Merge remote-tracking branch 'origin/topic/awelzel/4845-get-current-packet-header-fragmented-syn'

* origin/topic/awelzel/4845-get-current-packet-header-fragmented-syn:
  iosource/Packet: Allow ToRawPktHdrVal() for reassembled packets

.cirrus.yml

@@ -48,53 +48,108 @@ freebsd_environment: &FREEBSD_ENVIRONMENT
     ZEEK_CI_CPUS: 8
     ZEEK_CI_BTEST_JOBS: 8
 
-builds_only_if_template: &BUILDS_ONLY_IF_TEMPLATE
-  # Rules for skipping builds:
-  # - Do not run builds for anything that's cron triggered
-  # - Don't do darwin builds on zeek-security repo because they use up a ton of compute credits.
-  # - Always build PRs, but not if they come from dependabot
-  # - Always build master and release/* builds from the main repo
+only_if_pr_master_release: &ONLY_IF_PR_MASTER_RELEASE
   only_if: >
-    ( $CIRRUS_CRON == '' ) &&
-    ( ( $CIRRUS_PR != '' && $CIRRUS_BRANCH !=~ 'dependabot/.*' ) ||
     ( ( $CIRRUS_REPO_NAME == 'zeek' || $CIRRUS_REPO_NAME == 'zeek-security' ) &&
-      (
+      ( $CIRRUS_CRON != 'weekly' ) &&
+      ( $CIRRUS_PR != '' ||
         $CIRRUS_BRANCH == 'master' ||
         $CIRRUS_BRANCH =~ 'release/.*'
       )
-    ) )
+    )
 
-skip_task_on_pr: &SKIP_TASK_ON_PR
-  # Skip this task on PRs if it does not have the CI: Full label,
-  # it continues to run for direct pushes to master/release.
-  skip: >
-    ( ! ($CIRRUS_PR == '' || $CIRRUS_PR_LABELS =~ '.*CI: Full.*') || $CIRRUS_PR_LABELS =~ '.*CI: Skip All' )
-
-skip_cluster_test_on_pr: &SKIP_CLUSTER_TEST_ON_PR
-  # Skip this task on PRs if it does not have the CI: Full label,
-  # it continues to run for direct pushes to master/release.
-  skip: >
-    ( ! ($CIRRUS_PR == '' || $CIRRUS_PR_LABELS =~ '.*CI: (Full|Cluster Test).*') || $CIRRUS_PR_LABELS =~ '.*CI: Skip All' )
-
-zam_skip_task_on_pr: &ZAM_SKIP_TASK_ON_PR
-  # Skip this task on PRs unless it has the "CI: Full" or "CI: ZAM" label
-  # or files in src/script_opt/** were modified.
-  # It continues to run for direct pushes to master/release, as
-  # CIRRUS_PR will be empty.
-  skip: >
-    ( ! ($CIRRUS_PR == '' || $CIRRUS_PR_LABELS =~ '.*CI: (Full|ZAM).*' || changesInclude('src/script_opt/**')) || $CIRRUS_PR_LABELS =~ '.*CI: Skip All' )
-
-benchmark_only_if_template: &BENCHMARK_ONLY_IF_TEMPLATE
-  # only_if condition for cron-triggered benchmarking tests.
-  # These currently do not run for release/.*
+only_if_pr_master_release_nightly: &ONLY_IF_PR_MASTER_RELEASE_NIGHTLY
   only_if: >
-    ( $CIRRUS_REPO_NAME == 'zeek' || $CIRRUS_REPO_NAME == 'zeek-security' ) &&
-    ( $CIRRUS_CRON == 'benchmark-nightly' ||
-      $CIRRUS_PR_LABELS =~ '.*CI: (Full|Benchmark).*' )
+    ( ( $CIRRUS_REPO_NAME == 'zeek' || $CIRRUS_REPO_NAME == 'zeek-security' ) &&
+      ( $CIRRUS_CRON != 'weekly' ) &&
+      ( $CIRRUS_PR != '' ||
+        $CIRRUS_BRANCH == 'master' ||
+        $CIRRUS_BRANCH =~ 'release/.*' ||
+        ( $CIRRUS_CRON == 'nightly' && $CIRRUS_BRANCH == 'master' )
+      )
+    )
+
+only_if_pr_release_and_nightly: &ONLY_IF_PR_RELEASE_AND_NIGHTLY
+  only_if: >
+    ( ( $CIRRUS_REPO_NAME == 'zeek' || $CIRRUS_REPO_NAME == 'zeek-security' ) &&
+      ( $CIRRUS_CRON != 'weekly' ) &&
+      ( $CIRRUS_PR != '' ||
+        $CIRRUS_BRANCH =~ 'release/.*' ||
+        ( $CIRRUS_CRON == 'nightly' && $CIRRUS_BRANCH == 'master' )
+      )
+    )
+
+only_if_pr_nightly: &ONLY_IF_PR_NIGHTLY
+  only_if: >
+    ( ( $CIRRUS_REPO_NAME == 'zeek' || $CIRRUS_REPO_NAME == 'zeek-security' ) &&
+      ( $CIRRUS_CRON != 'weekly' ) &&
+      ( $CIRRUS_PR != '' ||
+        ( $CIRRUS_CRON == 'nightly' && $CIRRUS_BRANCH == 'master' )
+      )
+    )
+
+only_if_release_tag_nightly: &ONLY_IF_RELEASE_TAG_NIGHTLY
+  only_if: >
+    ( ( $CIRRUS_REPO_NAME == 'zeek' ) &&
+      ( $CIRRUS_CRON != 'weekly' ) &&
+      ( ( $CIRRUS_BRANCH =~ 'release/.*' && $CIRRUS_TAG =~ 'v[0-9]+\.[0-9]+\.[0-9]+(-rc[0-9]+)?$' ) ||
+        ( $CIRRUS_CRON == 'nightly' && $CIRRUS_BRANCH == 'master' )
+      )
+    )
+
+only_if_nightly: &ONLY_IF_NIGHTLY
+  only_if: >
+    ( ( $CIRRUS_REPO_NAME == 'zeek' ) &&
+      ( $CIRRUS_CRON == 'nightly' && $CIRRUS_BRANCH == 'master' )
+    )
+
+only_if_weekly: &ONLY_IF_WEEKLY
+  only_if: >
+    ( ( $CIRRUS_REPO_NAME == 'zeek' || $CIRRUS_REPO_NAME == 'zeek-security' ) &&
+      ( $CIRRUS_CRON == 'weekly' && $CIRRUS_BRANCH == 'master' )
+    )
+
+skip_if_pr_skip_all: &SKIP_IF_PR_SKIP_ALL
+  skip: >
+    ( $CIRRUS_PR != '' && $CIRRUS_PR_LABELS =~ ".*CI: Skip All.*" )
+
+skip_if_pr_not_full_ci: &SKIP_IF_PR_NOT_FULL_CI
+  skip: >
+    ( ( $CIRRUS_PR != '' && $CIRRUS_PR_LABELS !=~ ".*CI: Full.*") ||
+      ( $CIRRUS_PR_LABELS =~ ".*CI: Skip All.*" )
+    )
+
+skip_if_pr_not_full_or_benchmark: &SKIP_IF_PR_NOT_FULL_OR_BENCHMARK
+  skip: >
+    ( ( $CIRRUS_PR != '' && $CIRRUS_PR_LABELS !=~ ".*CI: (Full|Benchmark).*" ) ||
+      ( $CIRRUS_PR_LABELS =~ ".*CI: Skip All.*" )
+    )
+
+skip_if_pr_not_full_or_cluster_test: &SKIP_IF_PR_NOT_FULL_OR_CLUSTER_TEST
+  skip: >
+    ( ( $CIRRUS_PR != '' && $CIRRUS_PR_LABELS !=~ ".*CI: (Full|Cluster Test).*" ) ||
+      ( $CIRRUS_PR_LABELS =~ ".*CI: Skip All.*" )
+    )
+
+skip_if_pr_not_full_or_zam: &SKIP_IF_PR_NOT_FULL_OR_ZAM
+  skip: >
+    ( ( $CIRRUS_PR != '' && $CIRRUS_PR_LABELS !=~ ".*CI: (Full|ZAM).*" ) ||
+      ( $CIRRUS_PR_LABELS =~ ".*CI: Skip All.*" )
+    )
+
+skip_if_pr_not_full_or_zeekctl: &SKIP_IF_PR_NOT_FULL_OR_ZEEKCTL
+  skip: >
+    ( ( $CIRRUS_PR != '' && $CIRRUS_PR_LABELS !=~ ".*CI: (Full|Zeekctl).*" ) ||
+      ( $CIRRUS_PR_LABELS =~ ".*CI: Skip All.*" )
+    )
+
+skip_if_pr_not_full_or_windows: &SKIP_IF_PR_NOT_FULL_OR_WINDOWS
+  skip: >
+    ( ( $CIRRUS_PR != '' && $CIRRUS_PR_LABELS !=~ ".*CI: (Full|Windows).*" ) ||
+      ( $CIRRUS_PR_LABELS =~ ".*CI: Skip All.*" )
+    )
 
 ci_template: &CI_TEMPLATE
-  << : *BUILDS_ONLY_IF_TEMPLATE
-
   # Default timeout is 60 minutes, Cirrus hard limit is 120 minutes for free
   # tasks, so may as well ask for full time.
   timeout_in: 120m
@@ -136,12 +191,9 @@ ci_template: &CI_TEMPLATE
       # Evit some of the cached build artifacts not used in this build.
       CCACHE_MAXSIZE=${ZEEK_CCACHE_PRUNE_SIZE} ccache -c
 
-  # Always skip all tasks on a PR if the "CI: Skip All" label is applied.
-  skip: >
-    $CIRRUS_PR != '' && $CIRRUS_PR_LABELS =~ '.*CI: Skip All.*'
-
 env:
   CIRRUS_WORKING_DIR: /zeek
+  CIRRUS_LOG_TIMESTAMP: true
   ZEEK_CI_CPUS: *CPUS
   ZEEK_CI_BTEST_JOBS: *BTEST_JOBS
   ZEEK_CI_BTEST_RETRIES: *BTEST_RETRIES
@@ -192,6 +244,10 @@ fedora42_task:
     dockerfile: ci/fedora-42/Dockerfile
     << : *RESOURCES_TEMPLATE
   << : *CI_TEMPLATE
+  << : *ONLY_IF_PR_MASTER_RELEASE
+  << : *SKIP_IF_PR_SKIP_ALL
+  env:
+    ZEEK_CI_CONFIGURE_FLAGS: *BINARY_CONFIG
 
 fedora41_task:
   container:
@@ -199,14 +255,71 @@ fedora41_task:
     dockerfile: ci/fedora-41/Dockerfile
     << : *RESOURCES_TEMPLATE
   << : *CI_TEMPLATE
-  << : *SKIP_TASK_ON_PR
+  << : *ONLY_IF_PR_MASTER_RELEASE
+  << : *SKIP_IF_PR_NOT_FULL_CI
 
 centosstream9_task:
   container:
-    # Stream 9 EOL: Around Dec 2027
+    # Stream 9 EOL: 31 May 2027
     dockerfile: ci/centos-stream-9/Dockerfile
     << : *RESOURCES_TEMPLATE
   << : *CI_TEMPLATE
+  << : *ONLY_IF_PR_MASTER_RELEASE
+  << : *SKIP_IF_PR_NOT_FULL_CI
+
+centosstream10_task:
+  container:
+    # Stream 10 EOL: 01 January 2030
+    dockerfile: ci/centos-stream-10/Dockerfile
+    << : *RESOURCES_TEMPLATE
+  << : *CI_TEMPLATE
+  << : *ONLY_IF_PR_MASTER_RELEASE
+  << : *SKIP_IF_PR_NOT_FULL_CI
+
+debian13_task:
+  container:
+    # Debian 13 (trixie) EOL: TBD
+    dockerfile: ci/debian-13/Dockerfile
+    << : *RESOURCES_TEMPLATE
+  << : *CI_TEMPLATE
+  << : *ONLY_IF_PR_MASTER_RELEASE
+  << : *SKIP_IF_PR_NOT_FULL_CI
+
+arm_debian13_task:
+  arm_container:
+    # Debian 13 (trixie) EOL: TBD
+    dockerfile: ci/debian-13/Dockerfile
+    << : *RESOURCES_TEMPLATE
+  << : *CI_TEMPLATE
+  << : *ONLY_IF_PR_MASTER_RELEASE
+  << : *SKIP_IF_PR_SKIP_ALL
+
+debian13_static_task:
+  container:
+    # Just use a recent/common distro to run a static compile test.
+    # Debian 13 (trixie) EOL: TBD
+    dockerfile: ci/debian-13/Dockerfile
+    << : *RESOURCES_TEMPLATE
+  << : *CI_TEMPLATE
+  << : *ONLY_IF_PR_MASTER_RELEASE
+  << : *SKIP_IF_PR_NOT_FULL_CI
+  env:
+    ZEEK_CI_CONFIGURE_FLAGS: *STATIC_CONFIG
+
+debian13_binary_task:
+  container:
+    # Just use a recent/common distro to run binary mode compile test.
+    # As of 2024-03, the used configure flags are equivalent to the flags
+    # that we use to create binary packages.
+    # Just use a recent/common distro to run a static compile test.
+    # Debian 13 (trixie) EOL: TBD
+    dockerfile: ci/debian-13/Dockerfile
+    << : *RESOURCES_TEMPLATE
+  << : *CI_TEMPLATE
+  << : *ONLY_IF_PR_MASTER_RELEASE
+  << : *SKIP_IF_PR_NOT_FULL_CI
+  env:
+    ZEEK_CI_CONFIGURE_FLAGS: *BINARY_CONFIG
 
 debian12_task:
   container:
@@ -214,48 +327,8 @@ debian12_task:
     dockerfile: ci/debian-12/Dockerfile
     << : *RESOURCES_TEMPLATE
   << : *CI_TEMPLATE
-
-arm_debian12_task:
-  arm_container:
-    # Debian 12 (bookworm) EOL: TBD
-    dockerfile: ci/debian-12/Dockerfile
-    << : *RESOURCES_TEMPLATE
-  << : *CI_TEMPLATE
-  env:
-    ZEEK_CI_CONFIGURE_FLAGS: *NO_SPICY_CONFIG
-
-debian12_static_task:
-  container:
-    # Just use a recent/common distro to run a static compile test.
-    # Debian 12 (bookworm) EOL: TBD
-    dockerfile: ci/debian-12/Dockerfile
-    << : *RESOURCES_TEMPLATE
-  << : *CI_TEMPLATE
-  << : *SKIP_TASK_ON_PR
-  env:
-    ZEEK_CI_CONFIGURE_FLAGS: *STATIC_CONFIG
-
-debian12_binary_task:
-  container:
-    # Just use a recent/common distro to run binary mode compile test.
-    # As of 2024-03, the used configure flags are equivalent to the flags
-    # that we use to create binary packages.
-    # Just use a recent/common distro to run a static compile test.
-    # Debian 12 (bookworm) EOL: TBD
-    dockerfile: ci/debian-12/Dockerfile
-    << : *RESOURCES_TEMPLATE
-  << : *CI_TEMPLATE
-  << : *SKIP_TASK_ON_PR
-  env:
-    ZEEK_CI_CONFIGURE_FLAGS: *BINARY_CONFIG
-
-debian11_task:
-  container:
-    # Debian 11 EOL: June 2026
-    dockerfile: ci/debian-11/Dockerfile
-    << : *RESOURCES_TEMPLATE
-  << : *CI_TEMPLATE
-  << : *SKIP_TASK_ON_PR
+  << : *ONLY_IF_PR_MASTER_RELEASE
+  << : *SKIP_IF_PR_NOT_FULL_CI
 
 opensuse_leap_15_6_task:
   container:
@@ -263,6 +336,8 @@ opensuse_leap_15_6_task:
     dockerfile: ci/opensuse-leap-15.6/Dockerfile
     << : *RESOURCES_TEMPLATE
   << : *CI_TEMPLATE
+  << : *ONLY_IF_PR_MASTER_RELEASE
+  << : *SKIP_IF_PR_NOT_FULL_CI
 
 opensuse_tumbleweed_task:
   container:
@@ -271,7 +346,8 @@ opensuse_tumbleweed_task:
     << : *RESOURCES_TEMPLATE
   prepare_script: ./ci/opensuse-tumbleweed/prepare.sh
   << : *CI_TEMPLATE
-#  << : *SKIP_TASK_ON_PR
+  << : *ONLY_IF_PR_MASTER_RELEASE
+  << : *SKIP_IF_PR_NOT_FULL_CI
 
 weekly_current_gcc_task:
   container:
@@ -280,8 +356,7 @@ weekly_current_gcc_task:
     << : *RESOURCES_TEMPLATE
   prepare_script: ./ci/opensuse-tumbleweed/prepare-weekly.sh
   << : *CI_TEMPLATE
-  only_if: >
-    $CIRRUS_REPO_NAME == 'zeek' && $CIRRUS_BRANCH == 'master' && $CIRRUS_CRON == 'weekly'
+  << : *ONLY_IF_WEEKLY
   env:
     ZEEK_CI_COMPILER: gcc
 
@@ -292,33 +367,42 @@ weekly_current_clang_task:
     << : *RESOURCES_TEMPLATE
   prepare_script: ./ci/opensuse-tumbleweed/prepare-weekly.sh
   << : *CI_TEMPLATE
-  only_if: >
-    $CIRRUS_REPO_NAME == 'zeek' && $CIRRUS_BRANCH == 'master' && $CIRRUS_CRON == 'weekly'
+  << : *ONLY_IF_WEEKLY
   env:
     ZEEK_CI_COMPILER: clang
 
-ubuntu24_10_task:
+ubuntu25_04_task:
   container:
-    # Ubuntu 24.10 EOL: 2025-07-30
-    dockerfile: ci/ubuntu-24.10/Dockerfile
+    # Ubuntu 25.04 EOL: 2026-01-31
+    dockerfile: ci/ubuntu-25.04/Dockerfile
     << : *RESOURCES_TEMPLATE
   << : *CI_TEMPLATE
-  << : *SKIP_TASK_ON_PR
+  << : *ONLY_IF_PR_MASTER_RELEASE
+  << : *SKIP_IF_PR_NOT_FULL_CI
 
-ubuntu24_task:
+ubuntu24_04_task:
   container:
     # Ubuntu 24.04 EOL: Jun 2029
     dockerfile: ci/ubuntu-24.04/Dockerfile
     << : *RESOURCES_TEMPLATE
   << : *CI_TEMPLATE
+  << : *ONLY_IF_PR_MASTER_RELEASE
+  << : *SKIP_IF_PR_SKIP_ALL
+  env:
+    ZEEK_CI_CREATE_ARTIFACT: 1
+  upload_binary_artifacts:
+    path: build.tgz
+  benchmark_script: ./ci/benchmark.sh
 
 # Same as above, but running the ZAM tests instead of the regular tests.
-ubuntu24_zam_task:
+ubuntu24_04_zam_task:
   container:
     # Ubuntu 24.04 EOL: Jun 2029
     dockerfile: ci/ubuntu-24.04/Dockerfile
     << : *RESOURCES_TEMPLATE
   << : *CI_TEMPLATE
+  << : *ONLY_IF_PR_MASTER_RELEASE
+  << : *SKIP_IF_PR_NOT_FULL_OR_ZAM
   env:
     ZEEK_CI_SKIP_UNIT_TESTS: 1
     ZEEK_CI_SKIP_EXTERNAL_BTESTS: 1
@@ -327,62 +411,41 @@ ubuntu24_zam_task:
     ZEEK_CI_BTEST_JOBS: 3
 
 # Same as above, but using Clang and libc++
-ubuntu24_clang_libcpp_task:
+ubuntu24_04_clang_libcpp_task:
   container:
     # Ubuntu 24.04 EOL: Jun 2029
     dockerfile: ci/ubuntu-24.04/Dockerfile
     << : *RESOURCES_TEMPLATE
   << : *CI_TEMPLATE
-  << : *SKIP_TASK_ON_PR
+  << : *ONLY_IF_PR_MASTER_RELEASE
+  << : *SKIP_IF_PR_NOT_FULL_CI
   env:
     CC: clang-19
     CXX: clang++-19
     CXXFLAGS: -stdlib=libc++
 
-ubuntu24_clang_tidy_task:
+ubuntu24_04_clang_tidy_task:
   container:
     # Ubuntu 24.04 EOL: Jun 2029
     dockerfile: ci/ubuntu-24.04/Dockerfile
     << : *RESOURCES_TEMPLATE
   << : *CI_TEMPLATE
-  # Run on merges to master and release/.* and benchmark-nightly cron.
-  only_if: >
-    ( $CIRRUS_REPO_NAME == 'zeek' || $CIRRUS_REPO_NAME == 'zeek-security' ) &&
-      $CIRRUS_BRANCH == 'master' ||
-      $CIRRUS_BRANCH =~ 'release/.*' ||
-      $CIRRUS_CRON == 'benchmark-nightly'
+  << : *ONLY_IF_PR_MASTER_RELEASE
+  << : *SKIP_IF_PR_NOT_FULL_CI
   env:
     CC: clang-19
     CXX: clang++-19
     ZEEK_CI_CONFIGURE_FLAGS: *CLANG_TIDY_CONFIG
 
-
-ubuntu22_task:
-  container:
-    # Ubuntu 22.04 EOL: June 2027
-    dockerfile: ci/ubuntu-22.04/Dockerfile
-    << : *RESOURCES_TEMPLATE
-  << : *CI_TEMPLATE
-  env:
-    ZEEK_CI_CREATE_ARTIFACT: 1
-  upload_binary_artifacts:
-    path: build.tgz
-  benchmark_script: ./ci/benchmark.sh
-  # Run on PRs, merges to master and release/.* and benchmark-nightly cron.
-  only_if: >
-    ( $CIRRUS_PR != '' && $CIRRUS_BRANCH !=~ 'dependabot/.*' ) ||
-    ( ( $CIRRUS_REPO_NAME == 'zeek' || $CIRRUS_REPO_NAME == 'zeek-security' ) &&
-        $CIRRUS_BRANCH == 'master' ||
-        $CIRRUS_BRANCH =~ 'release/.*' ||
-        $CIRRUS_CRON == 'benchmark-nightly' )
-
 # Also enable Spicy SSL for this
-ubuntu22_spicy_task:
+ubuntu24_04_spicy_task:
   container:
-    # Ubuntu 22.04 EOL: April 2027
-    dockerfile: ci/ubuntu-22.04/Dockerfile
+    # Ubuntu 24.04 EOL: Jun 2029
+    dockerfile: ci/ubuntu-24.04/Dockerfile
     << : *RESOURCES_TEMPLATE
   << : *CI_TEMPLATE
+  << : *ONLY_IF_PR_MASTER_RELEASE
+  << : *SKIP_IF_PR_NOT_FULL_OR_BENCHMARK
   env:
     ZEEK_CI_CREATE_ARTIFACT: 1
     ZEEK_CI_CONFIGURE_FLAGS: *SPICY_SSL_CONFIG
@@ -390,14 +453,15 @@ ubuntu22_spicy_task:
   upload_binary_artifacts:
     path: build.tgz
   benchmark_script: ./ci/benchmark.sh
-  << : *BENCHMARK_ONLY_IF_TEMPLATE
 
-ubuntu22_spicy_head_task:
+ubuntu24_04_spicy_head_task:
   container:
-    # Ubuntu 22.04 EOL: April 2027
-    dockerfile: ci/ubuntu-22.04/Dockerfile
+    # Ubuntu 24.04 EOL: Jun 2029
+    dockerfile: ci/ubuntu-24.04/Dockerfile
     << : *RESOURCES_TEMPLATE
   << : *CI_TEMPLATE
+  << : *ONLY_IF_PR_MASTER_RELEASE_NIGHTLY
+  << : *SKIP_IF_PR_NOT_FULL_OR_BENCHMARK
   env:
     ZEEK_CI_CREATE_ARTIFACT: 1
     ZEEK_CI_CONFIGURE_FLAGS: *SPICY_SSL_CONFIG
@@ -407,10 +471,15 @@ ubuntu22_spicy_head_task:
   upload_binary_artifacts:
     path: build.tgz
   benchmark_script: ./ci/benchmark.sh
-  # Don't run this job on release branches. It tests against spicy HEAD, which
-  # will frequently require other fixes that won't be in a release branch.
-  skip: $CIRRUS_BRANCH =~ 'release/.*'
-  << : *BENCHMARK_ONLY_IF_TEMPLATE
+
+ubuntu22_04_task:
+  container:
+    # Ubuntu 22.04 EOL: June 2027
+    dockerfile: ci/ubuntu-22.04/Dockerfile
+    << : *RESOURCES_TEMPLATE
+  << : *CI_TEMPLATE
+  << : *ONLY_IF_PR_MASTER_RELEASE
+  << : *SKIP_IF_PR_NOT_FULL_CI
 
 alpine_task:
   container:
@@ -420,6 +489,8 @@ alpine_task:
     dockerfile: ci/alpine/Dockerfile
     << : *RESOURCES_TEMPLATE
   << : *CI_TEMPLATE
+  << : *ONLY_IF_PR_MASTER_RELEASE
+  << : *SKIP_IF_PR_NOT_FULL_CI
 
 # Cirrus only supports the following macos runner currently, selecting
 # anything else automatically upgrades to this one.
@@ -432,6 +503,8 @@ macos_sequoia_task:
     image: ghcr.io/cirruslabs/macos-runner:sequoia
   prepare_script: ./ci/macos/prepare.sh
   << : *CI_TEMPLATE
+  << : *ONLY_IF_PR_MASTER_RELEASE
+  << : *SKIP_IF_PR_SKIP_ALL
   << : *MACOS_ENVIRONMENT
 
 # FreeBSD EOL timelines: https://www.freebsd.org/security/#sup
@@ -443,6 +516,8 @@ freebsd14_task:
 
   prepare_script: ./ci/freebsd/prepare.sh
   << : *CI_TEMPLATE
+  << : *ONLY_IF_PR_MASTER_RELEASE
+  << : *SKIP_IF_PR_SKIP_ALL
   << : *FREEBSD_ENVIRONMENT
 
 freebsd13_task:
@@ -453,7 +528,8 @@ freebsd13_task:
 
   prepare_script: ./ci/freebsd/prepare.sh
   << : *CI_TEMPLATE
-  << : *SKIP_TASK_ON_PR
+  << : *ONLY_IF_PR_MASTER_RELEASE
+  << : *SKIP_IF_PR_NOT_FULL_CI
   << : *FREEBSD_ENVIRONMENT
 
 asan_sanitizer_task:
@@ -463,6 +539,8 @@ asan_sanitizer_task:
     << : *RESOURCES_TEMPLATE
 
   << : *CI_TEMPLATE
+  << : *ONLY_IF_PR_MASTER_RELEASE
+  << : *SKIP_IF_PR_SKIP_ALL
   test_fuzzers_script: ./ci/test-fuzzers.sh
   coverage_script: ./ci/upload-coverage.sh
   env:
@@ -479,6 +557,8 @@ asan_sanitizer_zam_task:
     << : *RESOURCES_TEMPLATE
 
   << : *CI_TEMPLATE
+  << : *ONLY_IF_PR_NIGHTLY
+  << : *SKIP_IF_PR_NOT_FULL_OR_ZAM
   env:
     ZEEK_CI_CONFIGURE_FLAGS: *ASAN_SANITIZER_CONFIG
     ASAN_OPTIONS: detect_leaks=1:detect_odr_violation=0
@@ -487,7 +567,6 @@ asan_sanitizer_zam_task:
     ZEEK_CI_BTEST_EXTRA_ARGS: -a zam
     # Use a lower number of jobs due to OOM issues with ZAM tasks
     ZEEK_CI_BTEST_JOBS: 3
-  << : *ZAM_SKIP_TASK_ON_PR
 
 ubsan_sanitizer_task:
   container:
@@ -496,7 +575,8 @@ ubsan_sanitizer_task:
     << : *RESOURCES_TEMPLATE
 
   << : *CI_TEMPLATE
-  << : *SKIP_TASK_ON_PR
+  << : *ONLY_IF_PR_NIGHTLY
+  << : *SKIP_IF_PR_NOT_FULL_CI
   test_fuzzers_script: ./ci/test-fuzzers.sh
   env:
     CC: clang-19
@@ -512,6 +592,8 @@ ubsan_sanitizer_zam_task:
     << : *RESOURCES_TEMPLATE
 
   << : *CI_TEMPLATE
+  << : *ONLY_IF_PR_NIGHTLY
+  << : *SKIP_IF_PR_NOT_FULL_OR_ZAM
   env:
     CC: clang-19
     CXX: clang++-19
@@ -523,7 +605,6 @@ ubsan_sanitizer_zam_task:
     ZEEK_CI_BTEST_EXTRA_ARGS: -a zam
     # Use a lower number of jobs due to OOM issues with ZAM tasks
     ZEEK_CI_BTEST_JOBS: 3
-  << : *ZAM_SKIP_TASK_ON_PR
 
 tsan_sanitizer_task:
   container:
@@ -532,7 +613,8 @@ tsan_sanitizer_task:
     << : *RESOURCES_TEMPLATE
 
   << : *CI_TEMPLATE
-  << : *SKIP_TASK_ON_PR
+  << : *ONLY_IF_PR_NIGHTLY
+  << : *SKIP_IF_PR_NOT_FULL_CI
   env:
     CC: clang-19
     CXX: clang++-19
@@ -556,13 +638,12 @@ windows_task:
   prepare_script: ci/windows/prepare.cmd
   build_script: ci/windows/build.cmd
   test_script: ci/windows/test.cmd
+  << : *ONLY_IF_PR_MASTER_RELEASE
+  << : *SKIP_IF_PR_NOT_FULL_OR_WINDOWS
   env:
     ZEEK_CI_CPUS: 8
     # Give verbose error output on a test failure.
     CTEST_OUTPUT_ON_FAILURE: 1
-  << : *BUILDS_ONLY_IF_TEMPLATE
-  skip: >
-    $CIRRUS_PR != '' && $CIRRUS_PR_LABELS =~ '.*CI: Skip All.*'
 
 
 # Container images
@@ -643,22 +724,18 @@ arm64_container_image_docker_builder:
   env:
     CIRRUS_ARCH: arm64
   << : *DOCKER_BUILD_TEMPLATE
-  << : *SKIP_TASK_ON_PR
+  << : *ONLY_IF_RELEASE_TAG_NIGHTLY
 
 amd64_container_image_docker_builder:
   env:
     CIRRUS_ARCH: amd64
   << : *DOCKER_BUILD_TEMPLATE
-  << : *SKIP_CLUSTER_TEST_ON_PR
+  << : *ONLY_IF_PR_MASTER_RELEASE_NIGHTLY
+  << : *SKIP_IF_PR_NOT_FULL_OR_CLUSTER_TEST
 
 container_image_manifest_docker_builder:
   cpu: 1
-  # Push master builds to zeek/zeek-dev, or tagged release branches to zeek/zeek
-  only_if: >
-    ( $CIRRUS_CRON == '' ) &&
-    ( $CIRRUS_REPO_FULL_NAME == 'zeek/zeek' &&
-      ( $CIRRUS_BRANCH == 'master' ||
-        $CIRRUS_TAG =~ 'v[0-9]+\.[0-9]+\.[0-9]+(-rc[0-9]+)?$' ) )
+  << : *ONLY_IF_RELEASE_TAG_NIGHTLY
   env:
     DOCKER_USERNAME: ENCRYPTED[!505b3dee552a395730a7e79e6aab280ffbe1b84ec62ae7616774dfefe104e34f896d2e20ce3ad701f338987c13c33533!]
     DOCKER_PASSWORD: ENCRYPTED[!6c4b2f6f0e5379ef1091719cc5d2d74c90cfd2665ac786942033d6d924597ffb95dbbc1df45a30cc9ddeec76c07ac620!]
@@ -737,8 +814,7 @@ container_image_manifest_docker_builder:
 # images from the public ECR repository to stay within free-tier bounds.
 public_ecr_cleanup_docker_builder:
   cpu: 1
-  only_if: >
-    $CIRRUS_CRON == '' && $CIRRUS_REPO_FULL_NAME == 'zeek/zeek' && $CIRRUS_BRANCH == 'master'
+  << : *ONLY_IF_NIGHTLY
   env:
     AWS_ACCESS_KEY_ID: ENCRYPTED[!eff52f6442e1bc78bce5b15a23546344df41bf519f6201924cb70c7af12db23f442c0e5f2b3687c2d856ceb11fcb8c49!]
     AWS_SECRET_ACCESS_KEY: ENCRYPTED[!748bc302dd196140a5fa8e89c9efd148882dc846d4e723787d2de152eb136fa98e8dea7e6d2d6779d94f72dd3c088228!]
@@ -778,7 +854,8 @@ cluster_testing_docker_builder:
       path: "testing/external/zeek-testing-cluster/.tmp/**"
   depends_on:
     - amd64_container_image
-  << : *SKIP_CLUSTER_TEST_ON_PR
+  << : *ONLY_IF_PR_RELEASE_AND_NIGHTLY
+  << : *SKIP_IF_PR_NOT_FULL_OR_CLUSTER_TEST
 
 
 # Test zeekctl upon master and release pushes and also when
@@ -789,16 +866,11 @@ cluster_testing_docker_builder:
 zeekctl_debian12_task:
   cpu: *CPUS
   memory: *MEMORY
-  only_if: >
-    ( $CIRRUS_CRON == 'zeekctl-nightly' ) ||
-    ( $CIRRUS_PR != '' && $CIRRUS_PR_LABELS =~ '.*CI: (Zeekctl|Full).*' ) ||
-    ( $CIRRUS_REPO_NAME == 'zeek' && (
-        $CIRRUS_BRANCH == 'master' ||
-        $CIRRUS_BRANCH =~ 'release/.*' )
-    )
+  << : *ONLY_IF_PR_MASTER_RELEASE
+  << : *SKIP_IF_PR_NOT_FULL_OR_ZEEKCTL
   container:
-    # Debian 12 (bookworm) EOL: TBD
-    dockerfile: ci/debian-12/Dockerfile
+    # Debian 13 (trixie) EOL: TBD
+    dockerfile: ci/debian-13/Dockerfile
     << : *RESOURCES_TEMPLATE
   sync_submodules_script: git submodule update --recursive --init
   always:
@@ -821,8 +893,8 @@ include_plugins_debian12_task:
   cpu: *CPUS
   memory: *MEMORY
   container:
-    # Debian 12 (bookworm) EOL: TBD
-    dockerfile: ci/debian-12/Dockerfile
+    # Debian 13 (trixie) EOL: TBD
+    dockerfile: ci/debian-13/Dockerfile
     << : *RESOURCES_TEMPLATE
   sync_submodules_script: git submodule update --recursive --init
   fetch_external_plugins_script:
@@ -853,5 +925,5 @@ include_plugins_debian12_task:
   on_failure:
     upload_include_plugins_testing_artifacts:
       path: "testing/builtin-plugins/.tmp/**"
-  << : *BUILDS_ONLY_IF_TEMPLATE
-  << : *SKIP_TASK_ON_PR
+  << : *ONLY_IF_PR_MASTER_RELEASE
+  << : *SKIP_IF_PR_NOT_FULL_CI

.clang-tidy

@@ -3,6 +3,7 @@ Checks: [-*,
          performance-*,
          modernize-*,
          readability-isolate-declaration,
+         readability-container-contains,
 
          # Enable a very limited number of the cppcoreguidelines checkers.
          # See the notes for some of the rest of them below.

.github/workflows/generate-docs.yml

@@ -70,12 +70,10 @@ jobs:
             sqlite3 \
             swig \
             zlib1g-dev
-          # Many distros adhere to PEP 394's recommendation for `python` =
-          # `python2` so this is a simple workaround until we drop Python 2
-          # support and explicitly use `python3` for all invocations.
-          sudo ln -sf /usr/bin/python3 /usr/local/bin/python
-          sudo pip3 install --break-system-packages -r doc/requirements.txt
-          sudo pip3 install --break-system-packages pre-commit
+          python3 -m venv ci-docs-venv
+          source ci-docs-venv/bin/activate
+          pip3 install -r doc/requirements.txt
+          pip3 install pre-commit
 
       - name: ccache
         uses: hendrikmuhs/ccache-action@v1.2
@@ -112,6 +110,7 @@ jobs:
 
       - name: Generate Docs
         run: |
+          source ci-docs-venv/bin/activate
           git config --global user.name zeek-bot
           git config --global user.email info@zeek.org
 

.gitmodules

@@ -1,9 +1,6 @@
 [submodule "auxil/zeek-aux"]
 	path = auxil/zeek-aux
 	url = https://github.com/zeek/zeek-aux
-[submodule "auxil/binpac"]
-	path = auxil/binpac
-	url = https://github.com/zeek/binpac
 [submodule "auxil/zeekctl"]
 	path = auxil/zeekctl
 	url = https://github.com/zeek/zeekctl
@@ -13,18 +10,12 @@
 [submodule "cmake"]
 	path = cmake
 	url = https://github.com/zeek/cmake
-[submodule "src/3rdparty"]
-	path = src/3rdparty
-	url = https://github.com/zeek/zeek-3rdparty
 [submodule "auxil/broker"]
 	path = auxil/broker
 	url = https://github.com/zeek/broker
 [submodule "auxil/netcontrol-connectors"]
 	path = auxil/netcontrol-connectors
 	url = https://github.com/zeek/zeek-netcontrol
-[submodule "auxil/bifcl"]
-	path = auxil/bifcl
-	url = https://github.com/zeek/bifcl
 [submodule "doc"]
 	path = doc
 	url = https://github.com/zeek/zeek-docs
@@ -46,9 +37,6 @@
 [submodule "auxil/zeek-client"]
 	path = auxil/zeek-client
 	url = https://github.com/zeek/zeek-client
-[submodule "auxil/gen-zam"]
-	path = auxil/gen-zam
-	url = https://github.com/zeek/gen-zam
 [submodule "auxil/c-ares"]
 	path = auxil/c-ares
 	url = https://github.com/c-ares/c-ares
@@ -58,9 +46,6 @@
 [submodule "auxil/spicy"]
 	path = auxil/spicy
 	url = https://github.com/zeek/spicy
-[submodule "auxil/zeek-af_packet-plugin"]
-	path = auxil/zeek-af_packet-plugin
-	url = https://github.com/zeek/zeek-af_packet-plugin.git
 [submodule "auxil/libunistd"]
 	path = auxil/libunistd
 	url = https://github.com/zeek/libunistd

.pre-commit-config.yaml

@@ -10,7 +10,7 @@ repos:
     language: python
     files: '\.(h|c|cpp|cc|spicy|evt)$'
     types: [file]
-    exclude: '^(testing/btest/(Baseline|plugins|spicy|scripts)/.*|testing/builtin-plugins/.*)$'
+    exclude: '^(testing/btest/(Baseline|plugins|spicy|scripts)/.*|testing/builtin-plugins/.*|src/3rdparty/.*)$'
 
   - id: btest-command-commented
     name: Check that all BTest command lines are commented out
@@ -19,25 +19,26 @@ repos:
     files: '^testing/btest/.*$'
 
 - repo: https://github.com/pre-commit/mirrors-clang-format
-  rev: v20.1.7
+  rev: v20.1.8
   hooks:
   - id: clang-format
     types_or:
       - "c"
       - "c++"
       - "json"
+    exclude: '^src/3rdparty/.*'
 
 - repo: https://github.com/maxwinterstein/shfmt-py
-  rev: v3.11.0.2
+  rev: v3.12.0.1
   hooks:
     - id: shfmt
       args: ["-w", "-i", "4", "-ci"]
 
 - repo: https://github.com/astral-sh/ruff-pre-commit
-  rev: v0.12.1
+  rev: v0.12.8
   hooks:
-    - id: ruff
-      args: [--fix]
+    - id: ruff-check
+      args: ["--fix"]
     - id: ruff-format
 
 - repo: https://github.com/cheshirekow/cmake-format-precommit
@@ -46,10 +47,10 @@ repos:
   - id: cmake-format
 
 - repo: https://github.com/crate-ci/typos
-  rev: v1.33.1
+  rev: v1.35.3
   hooks:
     - id: typos
-      exclude: '^(.typos.toml|src/SmithWaterman.cc|testing/.*|auxil/.*|scripts/base/frameworks/files/magic/.*|CHANGES|scripts/base/protocols/ssl/mozilla-ca-list.zeek)$'
+      exclude: '^(.typos.toml|src/SmithWaterman.cc|testing/.*|auxil/.*|scripts/base/frameworks/files/magic/.*|CHANGES|scripts/base/protocols/ssl/mozilla-ca-list.zeek|src/3rdparty/.*)$'
 
 - repo: https://github.com/bbannier/spicy-format
   rev: v0.26.0

.typos.toml

@@ -38,6 +38,7 @@ extend-ignore-re = [
     "\"BaR\"",
     "\"xFoObar\"",
     "\"FoO\"",
+    "Smoot",
 ]
 
 extend-ignore-identifiers-re = [

CHANGES

@@ -1,3 +1,729 @@
+8.1.0-dev.626 | 2025-10-02 14:10:44 +0200
+
+  * GH-4845: iosource/Packet: Allow ToRawPktHdrVal() for reassembled packets (Arne Welzel, Corelight)
+
+    Closes #4845
+
+8.1.0-dev.624 | 2025-10-02 09:15:17 +0200
+
+  * Conn: Improve packing, drop bitfields and boolenize (Arne Welzel, Corelight)
+
+    There's a few holes in Conn, particularly now that TransportProto has
+    become a uint8_t. Pack things a bit more neatly.
+
+  * Conn: Lazily initialize weird_state when needed (Arne Welzel, Corelight)
+
+    A std::unordered_map takes 56 bytes on my system. Switch to a unique_ptr
+    an initialize weird_state lazily. That saves ~48 bytes per connection.
+    Particularly for scan or non-weird traffic, this should allow some
+    memory savings.
+
+8.1.0-dev.621 | 2025-09-30 20:46:27 +0000
+
+  * GH-2686: fixes for re-declaring type identifiers in inconsistent ways - addresses GH-2686 (Vern Paxson, Corelight)
+
+8.1.0-dev.619 | 2025-09-30 20:45:19 +0000
+
+  * Fix for standalone initializations that require BiFs, and streamlining of standalone BiF-tracking (Vern Paxson, Corelight)
+
+8.1.0-dev.617 | 2025-09-30 20:12:14 +0000
+
+  * fixed bug in logic for including/excluding files for script optimization (Vern Paxson, Corelight)
+
+8.1.0-dev.615 | 2025-09-30 19:12:05 +0000
+
+  * Remove checks for OpenSSL 1.x versions (Tim Wojtulewicz, Corelight)
+
+  * Remove some additional LibreSSL checks (Tim Wojtulewicz, Corelight)
+
+8.1.0-dev.612 | 2025-09-29 18:04:24 +0200
+
+  * Supervisor: Make last_signal atomic to squelch data race (Arne Welzel, Corelight)
+
+    When the stem process terminates and SIGCHLD is sent to the supervisor,
+    the signal might be handled by the main thread or any other threads that
+    aren't blocking SIGCHLD explicitly. Convert last_signal to a std::atomic<int>
+    such that non-main threads can safely set last_signal without triggering
+    data race as reported by TSAN. This doesn't make it less racy to work
+    last_signal, but it appears we only use it for debug printing anyhow and
+    another option might have been to just remove last_signal altogether.
+
+    Follow-up for #4849
+
+8.1.0-dev.610 | 2025-09-29 08:21:01 -0700
+
+  * Update docs submodule [nomail] (Tim Wojtulewicz, Corelight)
+
+8.1.0-dev.609 | 2025-09-29 13:08:15 +0200
+
+  * cluster/zeromq: Fix Cluster::subscribe() block if not initialized (Arne Welzel, Corelight)
+
+    If Cluster::init() hasn't been invoked yet, Cluster::subscribe() with the
+    ZeroMQ backend would block because the main_inproc socket didn't
+    yet have a connection from the child thread. Prevent this by connecting
+    the main and child socket pair at construction time.
+
+    This will queue the subscriptions and start processing them once the
+    child thread has started.
+
+8.1.0-dev.607 | 2025-09-26 14:19:40 -0700
+
+  * Fixes for -O gen-standalone-C++ for tracking BiFs, lambdas, attribute types, and independent globals (Vern Paxson, Corelight)
+
+8.1.0-dev.605 | 2025-09-26 11:19:17 -0700
+
+  * OpaqueVal, OCSP, X509: drop outdated LibreSSL guards to fix OpenBSD (Klemens Nanni)
+    build
+
+    Whatever is used with recent OpenSSL is also available with latest LibreSSL
+    on OpenBSD 7.8-beta as of today.
+
+    Some of these hunks have been in the net/bro port for years, others I
+    recently added whilst gradually updating from 6.0.5 to 8.0.1.
+
+8.1.0-dev.603 | 2025-09-26 02:56:58 +0000
+
+  * Ignore src/3rdparty for pre-commit (Tim Wojtulewicz, Corelight)
+
+  * Merge src/3rdparty repo into the main Zeek repo (Tim Wojtulewicz, Corelight)
+
+  * cluster/zeromq: Improve EINTR handling (Arne Welzel, Corelight)
+
+    When using ZeroMQ also within the Supervisor process, zmq::poll() and
+    recv() were observed to return EINTR, handle these.
+
+8.1.0-dev.572 | 2025-09-25 13:52:46 +0200
+
+  * cluster/zeromq: Improve EINTR handling (Arne Welzel, Corelight)
+
+    When using ZeroMQ also within the Supervisor process, zmq::poll() and
+    recv() were observed to return EINTR, handle these.
+
+8.1.0-dev.570 | 2025-09-23 09:05:53 -0700
+
+  * BTests & baselines for testing selective skipping of script optimization (Vern Paxson, Corelight)
+
+  * added &no_ZAM_opt/&no_CPP_opt attributes and --no-opt-files/--no-opt-funcs for controlling skipping script optimization (Vern Paxson, Corelight)
+
+8.1.0-dev.567 | 2025-09-23 13:07:24 +0200
+
+  * GH-4842: utils/decompose_uri: Support URIs containing IPv6 addresses (Arne Welzel, Corelight)
+
+    An URI containing a bracketed or non-bracketed IPv6 address of the form
+    http://[::1]:42 was previously split on the first colon for port extraction,
+    causing a subsequent to_count() call to fail. Harden this to check for a
+    digits in the last :[0-9]+ component.
+
+    Fixes #4842
+
+8.1.0-dev.565 | 2025-09-22 07:45:57 -0700
+
+  * Restore the SetType constructor and destructor (Tim Wojtulewicz, Corelight)
+
+8.1.0-dev.563 | 2025-09-22 07:33:42 -0700
+
+  * Bump libkqueue to latest upstream master (Tim Wojtulewicz, Corelight)
+
+  * Fix clang-tidy finding in recent script_opt changes (Tim Wojtulewicz, Corelight)
+
+8.1.0-dev.559 | 2025-09-17 14:28:32 -0700
+
+  * tracking of event groups for compilation to standalone-C++ (Vern Paxson, Corelight)
+
+  * Deprecate SetType, as it can be replaced by TableType (Vern Paxson, Corelight)
+
+8.1.0-dev.555 | 2025-09-17 08:44:43 -0700
+
+  * Avoid starting up storage expiration thread with zero backends (Tim Wojtulewicz, Corelight)
+
+  * Remove a couple of obsolete TODOs in storage manager (Tim Wojtulewicz, Corelight)
+
+8.1.0-dev.552 | 2025-09-16 13:31:39 -0700
+
+  * Reword comment about when ZeroMQ is required for the build (Tim Wojtulewicz, Corelight)
+
+  * CI: Add label for enabling the Windows build in PRs (Tim Wojtulewicz)
+
+  * Enable building ZeroMQ support on Windows (Tim Wojtulewicz)
+
+8.1.0-dev.548 | 2025-09-16 12:41:52 -0700
+
+  * Force SQLite to be in thread-safe mode during CMake (Tim Wojtulewicz, Corelight)
+
+    This allows us to remove the need to check for thread-safe mode in the
+    various SQLite plugins. See https://www.sqlite.org/compile.html#threadsafe
+    for why `1` is a good choice here.
+
+8.1.0-dev.546 | 2025-09-16 11:31:04 -0700
+
+  * full tracking of the characteristics of globals when compiling scripts to C++ (Vern Paxson, Corelight)
+
+8.1.0-dev.544 | 2025-09-16 11:02:35 -0700
+
+  * fix for associating attributes with globals for -O gen-standalone-C++ (Vern Paxson, Corelight)
+
+8.1.0-dev.542 | 2025-09-16 10:53:04 -0700
+
+  * fix for tracking identifiers and aggregates when compiling to standalone-C++ (Vern Paxson, Corelight)
+
+8.1.0-dev.540 | 2025-09-16 10:52:44 -0700
+
+  * fix for '?' operator precedence when compiling scripts to C++ (Vern Paxson, Corelight)
+
+8.1.0-dev.538 | 2025-09-12 09:24:51 -0700
+
+  * for -O gen-standalone-C++, make the presence of uncompilable functions fatal unless -O allow-cond is used (Vern Paxson, Corelight)
+
+8.1.0-dev.536 | 2025-09-12 09:24:12 -0700
+
+  * Bump zeekctl for new trace-summary/pysubnettree versions (Tim Wojtulewicz, Corelight)
+
+8.1.0-dev.534 | 2025-09-11 10:57:17 -0700
+
+  * Clean up initialization of DNS_Interpreter (Tim Wojtulewicz, Corelight)
+
+  * Switch types used in DNS analyzer to be more consistent (Tim Wojtulewicz, Corelight)
+
+8.1.0-dev.531 | 2025-09-08 11:19:36 -0700
+
+  * btest/core/suspend_processing: Add WebSocket example (Arne Welzel, Corelight)
+
+    Add a test/example forwarding all new_connection() events produced during
+    `zeek -r wikipedia.trace` as my_new_connection() to a WebSocket client.
+
+    This is mostly to demonstrate and verify usage of suspend_processing(),
+    websocket_client_added(), resume_processing(), Pcap::file_done(),
+    websocket_client_lost() and terminate() together.
+
+8.1.0-dev.529 | 2025-09-08 11:02:39 -0700
+
+  * Bump Spicy for Coverity fixes (Evan Typanski, Corelight)
+
+8.1.0-dev.527 | 2025-09-08 11:02:05 -0700
+
+  * Update src/3rdparty submodule for doctest v2.4.12 and include fix (Tim Wojtulewicz, Corelight)
+
+8.1.0-dev.525 | 2025-09-06 04:26:32 +0000
+
+  * CI: Fix warning about ENV usage in CI dockerfiles (Tim Wojtulewicz, Corelight)
+
+    This also bumps the dates on the DOCKERFILE_VERSION values
+
+  * CI: Add centos stream 10 build (Tim Wojtulewicz, Corelight)
+
+8.1.0-dev.522 | 2025-09-05 19:52:31 -0700
+
+  * Bump zeek-client to pull in more resilient controller I/O (Christian Kreibich, Corelight)
+
+8.1.0-dev.520 | 2025-09-05 12:56:24 -0700
+
+  * Bump auxil/spicy to latest development snapshot (Benjamin Bannier, Corelight)
+
+8.1.0-dev.518 | 2025-09-04 16:21:52 -0700
+
+  * libkqueue: Switch to using HEAD from upstream master (Tim Wojtulewicz, Corelight)
+
+    The upstream recently merge Christian's fixes for the fd_map allocations
+    so that patch doesn't need to be in our fork anymore. I also tested removing
+    the patch about timers from our fork, and it seems to only matter for Linux
+    kernels earlier than 5.3. All of our supported platforms are newer than
+    that.
+
+8.1.0-dev.516 | 2025-09-04 09:04:27 -0700
+
+  * Bump auxil/spicy to latest development snapshot (Benjamin Bannier, Corelight)
+
+  * Fix clang-tidy warning from recent SVCB merge (Tim Wojtulewicz, Corelight)
+
+  * NEWS: Add note about DNS SVCB changes (Tim Wojtulewicz, Corelight)
+
+8.1.0-dev.511 | 2025-09-03 15:35:49 -0700
+
+  * Parse SVCB/HTTPS SvcParams list (Klemens Nanni)
+
+    Add full support for RFC 9460's SvcParams list.
+
+    Amend the existing `dns_svcb_rr` record by a vector of new
+    `dns_svcb_param` records containing aptly typed SvcParamKey and
+    SvcParamValue pairs.  Example output:
+
+    ```
+    @load base/protocols/dns
+    event dns_HTTPS( c: connection , msg: dns_msg , ans: dns_answer , https: dns_svcb_rr ) {
+    	for (_, param in https$svc_params)
+    		print to_json(param);  # filter uninitialised values
+    }
+    ```
+
+    ```
+    $ dig https cloudflare-ech.com +short | tr [:space:] \\n
+    1
+    .
+    alpn="h3,h2"
+    ipv4hint=104.18.10.118,104.18.11.118
+    ech=AEX+DQBBHgAgACBGL2e9TiFwjK/w1Zg9AmRm7mgXHz3PjffP0mTFNMxmDQAEAAEAAQASY2xvdWRmbGFyZS1lY2guY29tAAA=
+    ipv6hint=2606:4700::6812:a76,2606:4700::6812:b76
+    ```
+
+    ```
+    {"key":1,"alpn":["h3","h2"]}
+    {"key":4,"hint":["104.18.10.118","104.18.11.118"]}
+    {"key":5,"ech":"AEX+DQBBHgAgACBGL2e9TiFwjK/w1Zg9AmRm7mgXHz3PjffP0mTFNMxmDQAEAAEAAQASY2xvdWRmbGFyZS1lY2guY29tAAA="}
+    {"key":6,"hint":["2606:4700::6812:a76","2606:4700::6812:b76"]}
+    ```
+
+    Values with malformed data or belonging to invalid/reserved keys
+    are passed raw bytes in network order for script-level inspection.
+
+    Follow up to "Initial Support to DNS SVCB/HTTPS RR"
+    https://github.com/zeek/zeek/pull/1808
+
+  * Clang-tidy fixes for recent IDPtr changes (Tim Wojtulewicz, Corelight)
+
+8.1.0-dev.508 | 2025-09-03 15:02:41 -0700
+
+  * fixup! fixup! shift much of the internal use of ID* identifier pointers over to IDPtr objects (Vern Paxson, Corelight)
+
+  * fixup! shift much of the internal use of ID* identifier pointers over to IDPtr objects (Vern Paxson, Corelight)
+
+  * fixup! shift much of the internal use of ID* identifier pointers over to IDPtr objects (Vern Paxson, Corelight)
+
+  * fixup! shift much of the internal use of ID* identifier pointers over to IDPtr objects (Vern Paxson, Corelight)
+
+  * annotate a number of BTests as unsuited for -O gen-C++ testing due to multiple Zeek runs (Vern Paxson, Corelight)
+
+  * BTest baseline updates for -O gen-C++ - all minor tweaks (Vern Paxson, Corelight)
+
+  * BTest updates for script optimization tracking of BiFs (Vern Paxson, Corelight)
+
+  * regression test for former ASAN issue with script optimization of lambdas (Vern Paxson, Corelight)
+
+  * shift much of the internal use of ID* identifier pointers over to IDPtr objects (Vern Paxson, Corelight)
+
+  * maintenance update for script optimization's knowledge of BiFs (Vern Paxson, Corelight)
+
+  * logger fix for interoperability with -O gen-C++ code (Vern Paxson, Corelight)
+
+8.1.0-dev.496 | 2025-09-03 16:17:23 -0400
+
+  * tightened up parsing of $field=X record constructor expressions (Vern Paxson, Corelight)
+
+8.1.0-dev.494 | 2025-09-03 14:50:12 +0200
+
+  * Remove unnecessary peer signature from test `scripts.base.protocols.bittorrent.tracker` (Benjamin Bannier, Corelight)
+
+8.1.0-dev.492 | 2025-09-02 16:00:17 -0700
+
+  * Remove some unnecessary #includes from binpac source files (Tim Wojtulewicz, Corelight)
+
+8.1.0-dev.490 | 2025-09-02 11:47:44 -0700
+
+  * Fix a few more random clang-tidy findings (Tim Wojtulewicz, Corelight)
+
+  * Use std::numbers::pi instead of hard-coded value (Tim Wojtulewicz, Corelight)
+  
+  * Use std::scoped_lock instead of std::lock_guard (Tim Wojtulewicz, Corelight)
+
+  * Use .contains() instead of .find() or .count() (Tim Wojtulewicz, Corelight)
+
+8.1.0-dev.484 | 2025-08-29 21:53:19 -0700
+
+  * Bump zeek-testing-cluster to pull in WebSocket TLS updates (Christian Kreibich, Corelight)
+
+  * Bump zeek-client to pull in TLS config updates (Christian Kreibich, Corelight)
+
+  * Management framework: add TLS options for controller's websocket server (Arne Welzel, Corelight)
+
+8.1.0-dev.480 | 2025-08-29 15:08:29 -0700
+
+  * Move benchmarking to Ubnutu 24 task, add to normal PR builds (Tim Wojtulewicz, Corelight)
+
+    We already have a "regular Linux" build for PRs, but I'm adding this one
+    in so that we have benchmark coverage on PRs as well.
+
+  * Reorder ubuntu builds so 22.04 comes last (Tim Wojtulewicz, Corelight)
+
+  * Rename Ubuntu-based tasks to have the full version name (Tim Wojtulewicz, Corelight)
+
+  * Switch Zeek Spicy builds to Ubuntu 24 (Tim Wojtulewicz, Corelight)
+
+8.1.0-dev.475 | 2025-08-28 15:19:59 -0700
+
+  * gen-zam: Move source files up a directory (Tim Wojtulewicz, Corelight)
+
+  * bifcl: Fix clang-tidy warnings (Tim Wojtulewicz, Corelight)
+
+  * bifcl: Run clang-format on c++ code embedded in builtin-func.{l,y} (Tim Wojtulewicz, Corelight)
+
+  * binpac: Fix a large number of clang-tidy warnings (Tim Wojtulewicz, Corelight)
+
+  * binpac: Replace delete_list macro uses (Tim Wojtulewicz, Corelight)
+
+  * binpac: Run clang-format on c++ code embedded in pac_parse.{ll,yy} (Tim Wojtulewicz, Corelight)
+
+8.1.0-dev.468 | 2025-08-28 13:06:25 -0700
+
+  * Link Prometheus symbols via Broker instead of directly (Tim Wojtulewicz, Corelight)
+
+  * CI: Restore build steps for ubuntu22 task (Tim Wojtulewicz, Corelight)
+
+    This got accidentally deleted during the reorganization of the CI tasks
+
+  * Fix ruff check legacy alias for pre-commit (Tim Wojtulewicz, Corelight)
+
+8.1.0-dev.461 | 2025-08-26 14:39:50 -0700
+
+  * CI: Rework layout of when each task runs (Tim Wojtulewicz, Corelight)
+
+8.1.0-dev.459 | 2025-08-26 08:59:20 -0700
+
+  * Fix installation of symlink with `DESTDIR` (Benjamin Bannier, Corelight)
+
+    We install test data which we also make available under an alternative
+    path for backwards compatibility. The installation of this symlink did
+    not take `DESTDIR` installs like used by Zeek's packaging into account
+    which caused installations from packages to behave different from
+    installs from source.
+
+    This patch fixes the symlink to respect a possible `DESTDIR`.
+
+    Closes #3266.
+
+8.1.0-dev.456 | 2025-08-25 12:29:32 -0700
+
+  * Bump zeekctl (Arne Welzel, Corelight)
+
+8.1.0-dev.454 | 2025-08-22 10:12:35 +0200
+
+  * cluster/Backend: Fallback to current network time when current event has not timestamp (Arne Welzel, Corelight)
+
+    When a WebSocket client sends an event to Zeek without explicit network
+    timestamp metadata, Zeek would use -1.0 as a timestamp for any events
+    published while handling this event. Instead, it seems far more sensible
+    to use the current network time in that scenario.
+
+  * cluster/serializer/broker: Do not send empty metadata vectors around (Arne Welzel, Corelight)
+
+    Event when there's no metadata attached to an event, we'd still use the
+    constructor passing an empty metadata vector, resulting in an on-the-wire
+    representation with an empty trailing vector.
+
+    Particularly visible when just snooping events via websocat. There also
+    seems to be some bug with the timestamp -1 handling.
+
+8.1.0-dev.451 | 2025-08-21 17:10:05 -0700
+
+  * Minor fixes to a few NEWS entries. (Christian Kreibich, Corelight)
+
+8.1.0-dev.449 | 2025-08-21 17:02:51 +0200
+
+  * README.md: Add Mastodon and Bluesky links (Arne Welzel, Corelight)
+
+  * README: Drop "Follow us on Twitter" (Arne Welzel, Corelight)
+
+  * Merge a number of submodules into Zeek (Tim Wojtulewicz, Corelight)
+
+    The code from the gen-zam, bifcl, binpac and zeek-af_packet-plugin
+    submodules has been merged into the Zeek tree.
+
+8.1.0-dev.101 | 2025-08-20 11:38:37 -0400
+
+  * Fix Coverity issue with new `Attributes` ctor (Evan Typanski, Corelight)
+
+8.1.0-dev.99 | 2025-08-19 15:08:35 -0700
+
+  * CI: Update CentOS 9 to Python 3.13 (Tim Wojtulewicz, Corelight)
+
+8.1.0-dev.97 | 2025-08-19 20:27:13 +0200
+
+  * logging/Manager: Also pass non-null vector and set (Arne Welzel, Corelight)
+
+    Primarily to align with strings and also to keep the plugin
+    API the same.
+
+  * logging/Manager: Non-null strings for empty strings (Arne Welzel, Corelight)
+
+    After #4724, empty strings would result in nullptrs being stored in the
+    threading::Value's string_val.data field instead of a valid pointer to
+    an empty strings. This upsets UBSAN's  nonnull check for memcpy()
+
+        [01:29:45.807]   ../../src/SerializationFormat.cc:80:33: runtime error: null pointer passed as argument 2, which is declared to never be null
+        [01:29:45.807]   /usr/include/string.h:44:28: note: nonnull attribute specified here
+        [01:29:45.807]       #0 0x5b2e9c933a3f in zeek::detail::SerializationFormat::WriteData(void const*, unsigned long) /zeek/build/src/../../src/SerializationFormat.cc:80:5
+        [01:29:45.807]       #1 0x5b2e9c935184 in zeek::detail::BinarySerializationFormat::Write(char const*, int, char const*) /zeek/build/src/../../src/SerializationFormat.cc:371:40
+
+    Continue to allocate the empty string for now as a fix.
+
+  * CI: Ubuntu 24.10 is eol, add Ubuntu 25.04 (Johanna Amann, Corelight)
+
+8.1.0-dev.91 | 2025-08-18 14:59:41 -0700
+
+  * Add a missing header for the broker cluster serializer (Tim Wojtulewicz, Corelight)
+
+8.1.0-dev.89 | 2025-08-18 14:46:57 -0700
+
+  * Bump auxil/spicy to latest development snapshot (Benjamin Bannier, Corelight)
+
+8.1.0-dev.87 | 2025-08-18 14:39:10 -0400
+
+  * Fix parameter attributes pretending to be records (Evan Typanski, Corelight)
+
+    Parameters relied on is_record for a couple of validations, but they are
+    not records and should not be treated as such. This way we can validate
+    &optional better.
+
+  * Only allow `&optional` in records (Evan Typanski, Corelight)
+
+    There was some confusing behavior with &optional and locals, so this
+    should get rid of that by making it an error. However, there is a case
+    where function parameters are still allowed to have &optional - this is
+    because there are checks for &default in parameters as well.
+
+8.1.0-dev.83 | 2025-08-18 09:40:41 -0700
+
+  * Add `record_type_to_vector` deprecation to NEWS (Evan Typanski, Corelight)
+
+8.1.0-dev.81 | 2025-08-18 16:44:30 +0200
+
+  * cluster/serializer/broker: Drop unused include (Arne Welzel, Corelight)
+
+  * cluster/serializer/broker: fixup inconsistent param comment (Arne Welzel, Corelight)
+
+    References #4754 #4756
+
+8.1.0-dev.78 | 2025-08-18 13:13:21 +0200
+
+  * Changed behavior of var-extraction-uri.zeek from policy/protocol/http to extract only the URI parameter names. Do not include the path in the first parameter name. Only extract uri vars if parameters actually exist. (Benjamin Grap)
+
+8.1.0-dev.76 | 2025-08-18 09:40:05 +0200
+
+  * Report PostgreSQL login success only after ReadyForQuery (Fupeng Zhao)
+
+    Previously, Zeek treated the receipt of `AuthenticationOk` as a
+    successful login. However, according to the PostgreSQL
+    Frontend/Backend Protocol, the startup phase is not complete until
+    the server sends `ReadyForQuery`. It is still possible for the server
+    to emit an `ErrorResponse` (e.g. ERRCODE_INVALID_AUTHORIZATION_SPECIFICATION)
+    after `AuthenticationOk` but before `ReadyForQuery`.
+
+    This change updates the PostgreSQL analyzer to defer reporting login
+    success until `ReadyForQuery` is observed. This prevents false
+    positives in cases where authentication succeeds but session startup
+    fails.
+
+8.1.0-dev.74 | 2025-08-17 17:28:59 +0200
+
+  * maintenance updates for ZAM BiF-tracking (Vern Paxson, Corelight)
+
+  * fix line numbers associated with "if" and initialization statements (Vern Paxson, Corelight)
+
+8.1.0-dev.71 | 2025-08-17 16:56:57 +0200
+
+  * cluster/serializer/broker: Do not special case Broker::Data anymore (Arne Welzel, Corelight)
+
+    The previous approach ignored the fact that nested / inner values might
+    also be Broker::Data values. I'm not super sure about the validity of
+    the test, because it's essentially demonstrating any-nesting, but
+    it's not leading to extra Broker::Data encoding.
+
+  * broker/Data: Support unwrapping Broker::Data records (Arne Welzel, Corelight)
+
+    Calling val_to_data() on a Broker::Data ends up wrapping the
+    Broker::Data record instead of using the contained broker::value
+    directly.
+
+    Seems this should be the default behavior and wonder if the flag
+    even makes sense, but for a 8.0 backport that seems more reasonable.
+
+8.1.0-dev.68 | 2025-08-15 15:20:36 -0700
+
+  * Revert "Move BinPAC, bifcl, af_packet, and gen_zam submodules into main zeek repo" (Tim Wojtulewicz)
+
+  * Fix some clang-tidy findings in generated BIF code (Tim Wojtulewicz, Corelight)
+
+  * Fix clang-tidy and pre-commit warnings for gen-zam code files (Tim Wojtulewicz, Corelight)
+
+  * Move gen-zam code into the main Zeek repository (Tim Wojtulewicz, Corelight)
+
+    This is based on commit 56a6db00b887c79d26f303676677cb490d1c296d from
+    the gen-zam repository.
+
+  * Move zeek-af_packet-plugin code into the main Zeek repository (Tim Wojtulewicz, Corelight)
+
+    This is based on commit b89a6f64123f778090d1dd6ec48e6b8e8906ea11 from
+    the zeek-af_packet-plugin repository.
+
+  * Move the bifcl code into the main Zeek repository (Tim Wojtulewicz, Corelight)
+
+    This is based on commit 5947749f7850b075f11d6a2aaefe7dad4f63cb62f from
+    the bifcl repository.
+
+  * Fix clang-tidy findings in the binpac lib code (Tim Wojtulewicz, Corelight)
+
+  * Add copyright headers to all of the binpac source files (Tim Wojtulewicz, Corelight)
+
+  * Move binpac code into the main Zeek repository (Tim Wojtulewicz, Corelight)
+
+    This is based on commit 48f75b5f6415fe9d597e3e991cec635b1bc400dc from
+    the binpac repository.
+
+8.1.0-dev.56 | 2025-08-13 21:20:50 +0200
+
+  * ci: Run zeekctl and builtin tasks with Debian 13, too (Arne Welzel, Corelight)
+
+  * ci: Prepend timestamps to output (Arne Welzel, Corelight)
+
+  * ci: Enable Spicy for arm_debian13 (Arne Welzel, Corelight)
+
+  * ci: Add Debian 13.0 (trixie) (Arne Welzel, Corelight)
+
+  * docker: Bump to debian:trixie-slim (Arne Welzel, Corelight)
+
+8.1.0-dev.50 | 2025-08-12 17:42:46 -0700
+
+  * Add NEWS entry for field length limiting (Tim Wojtulewicz, Corelight)
+
+  * Tag truncated values with a flag, plus pack threading::Value better (Tim Wojtulewicz, Corelight)
+
+  * Remove length limiting on string fields for HTTP (Tim Wojtulewicz, Corelight)
+
+  * Make total_size counter a member in logging::Manager (Tim Wojtulewicz, Corelight)
+
+  * Remove using numeric_limits and just check for zero instead (Tim Wojtulewicz, Corelight)
+
+  * Expand the size of the log-size filters for x509 (Tim Wojtulewicz, Corelight)
+
+  * Add options to filter at the stream level as well as globally (Tim Wojtulewicz, Corelight)
+
+  * Add a weird that gets emitted when strings/containers are over the limits (Tim Wojtulewicz, Corelight)
+
+  * Add metrics to track string and container fields limited by length (Tim Wojtulewicz, Corelight)
+
+  * Replace unused stream argument from RecordToLogRecord with WriterInfo (Tim Wojtulewicz, Corelight)
+
+    This also adds a WriterInfo argument to ValToLogVal and passes the one from
+    RecordToLogRecord into it.
+
+  * Implement string- and container-length filtering at the log record level (Tim Wojtulewicz, Corelight)
+
+8.1.0-dev.38 | 2025-08-12 12:38:24 -0700
+
+  * Bump pre-commit hooks (Benjamin Bannier, Corelight)
+
+  * Bump auxil/spicy to latest development snapshot (Benjamin Bannier, Corelight)
+
+8.1.0-dev.35 | 2025-08-12 11:01:12 -0700
+
+  * Pass DNS complete_flag along as a uint8_t instead of a String (Tim Wojtulewicz, Corelight)
+
+  * Update docs submodule with 8.1 deprecation removals (Tim Wojtulewicz, Corelight)
+
+  * Update zeekjs submodule with 8.1 deprecation fixes (Tim Wojtulewicz, Corelight)
+
+  * Remove deprecations tagged for v8.1 (Tim Wojtulewicz, Corelight)
+
+8.1.0-dev.29 | 2025-08-11 11:37:22 -0700
+
+  * smb2/read: Parse only 1 byte for data_offset, ignore reserved1 (Arne Welzel, Corelight)
+
+    A user provided a SMB2 pcap with the reserved1 field of a ReadResponse
+    set to 1 instead of 0. This confused the padding computation due to
+    including this byte into the offset. Properly split data_offset and
+    reserved1 into individual byte fields.
+
+    Closes #4730
+
+8.1.0-dev.27 | 2025-08-10 21:28:33 -0700
+
+  * GH-4176: cluster: Add on_subscribe() and on_unsubscribe() hooks (Arne Welzel, Corelight)
+
+    Closes #4176
+
+8.1.0-dev.24 | 2025-08-08 14:23:51 +0200
+
+  * GH-4176: cluster: Add on_subscribe() and on_unsubscribe() hooks (Arne Welzel, Corelight)
+
+8.1.0-dev.22 | 2025-08-07 08:39:27 -0700
+
+  * Update zeek-aux to remove BRO_DIST from plugin skeleton (Tim Wojtulewicz, Corelight)
+
+  * cmake_minimum_required() should come before project() (Tim Wojtulewicz, Corelight)
+
+8.1.0-dev.19 | 2025-08-07 10:48:40 +0200
+
+  * btest/tap-analyzer: Update existing test and add new one for UpdateConnVal() (Arne Welzel, Corelight)
+
+    This also changes the output of connection UIDs from the tap analyzer to be
+    prefixed with C for easier correlation with other logs.
+
+  * SessionAdapter: Keep tap_analyzers until destruction (Arne Welzel, Corelight)
+
+    connection_state_remove() is invoked after Done(), so it's not a good
+    idea to remove the tap analyzers before in case they have up-to-date
+    information for the connection val.
+
+  * tcp,udp,icmp adapters: Move TapPacket() to earlier (Arne Welzel, Corelight)
+
+    Writing a test, the packet was tapped after protocol analysis at least
+    for TCP. Ensure tapping happens before. The adapter->Process() moving
+    after pkt->session made me a bit wondering if things are underspecified
+    here, but seems reasonable to set the session on pkt before adapter->Process().
+
+  * tcp,udp,icmp adapters: Fix UpdateConnVal() superclass call (Arne Welzel, Corelight)
+
+    Now that SessionAdapter implements UpdateConnVal(), the individual
+    adapters need to call that instead of Analyzer::UpdateConnVal()
+
+    Thanks clang-tidy.
+
+8.1.0-dev.14 | 2025-08-06 14:37:50 +0100
+
+  * Add proto to analyzer.log (Johanna Amann, Corelight)
+
+    The analyzer.log file was missing the protocol field to distinguish
+    tcp/udp connections.
+
+8.1.0-dev.12 | 2025-08-06 09:10:08 -0400
+
+  * GH-4722: Fix record coercion with compatible types (Evan Typanski, Corelight)
+
+    Fixes #4722
+
+8.1.0-dev.10 | 2025-08-06 14:27:32 +0200
+
+  * TapAnalyzer: Fix docstring (Arne Welzel, Corelight)
+
+    Relates to #4337 #4725 #4734
+
+  * btest/plugins/tap-analyzer: Update baseline (Arne Welzel, Corelight)
+
+    Relates to #4337 #4725 #4734
+
+8.1.0-dev.7 | 2025-08-05 20:00:19 +0200
+
+  * TapAnalyzer: More verdict to action rename (Arne Welzel, Corelight)
+
+    Relates to #4725 #4337
+
+8.1.0-dev.5 | 2025-08-05 19:48:50 +0200
+
+  * IPBasedAnalyzer: Call TapPacket() when skipping (Arne Welzel, Corelight)
+
+    When skip_further_processing() is called, a TapAnalyzer should still see
+    the packets as skipped with SkipReason "skipping".
+
+  * SessionAdapter: Introduce TapAnalyzer for session adapter (Arne Welzel, Corelight)
+
+    This commit introduces a mechanism to attach light weight analyzers to
+    the root analyzer of sessions in order to tap into the packets delivered
+    to child analyzer.
+
+8.1.0-dev.2 | 2025-08-04 09:35:41 -0700
+
+  * Compile contributors for Zeek 8.0 in the NEWS file (Christian Kreibich, Corelight)
+
+  * Start of 8.1.0 development (Tim Wojtulewicz, Corelight)
+
 8.0.0-dev.827 | 2025-08-01 17:10:13 +0200
 
   * ci/windows: No ZeroMQ cluster backend (Arne Welzel, Corelight)

CMakeLists.txt

@@ -396,14 +396,14 @@ endfunction ()
 
 add_zeek_dynamic_plugin_build_interface_include_directories(
     ${PROJECT_SOURCE_DIR}/src/include
-    ${PROJECT_SOURCE_DIR}/auxil/binpac/lib
+    ${PROJECT_SOURCE_DIR}/tools/binpac/lib
     ${PROJECT_SOURCE_DIR}/auxil/broker/libbroker
     ${PROJECT_SOURCE_DIR}/auxil/paraglob/include
     ${PROJECT_SOURCE_DIR}/auxil/prometheus-cpp/core/include
     ${PROJECT_SOURCE_DIR}/auxil/expected-lite/include
     ${CMAKE_BINARY_DIR}/src
     ${CMAKE_BINARY_DIR}/src/include
-    ${CMAKE_BINARY_DIR}/auxil/binpac/lib
+    ${CMAKE_BINARY_DIR}/tools/binpac/lib
     ${CMAKE_BINARY_DIR}/auxil/broker/libbroker
     ${CMAKE_BINARY_DIR}/auxil/prometheus-cpp/core/include)
 
@@ -892,28 +892,26 @@ if (BUILD_STATIC_BINPAC)
     set(ENABLE_STATIC_ONLY true)
 endif ()
 
-add_subdirectory(auxil/binpac)
+add_subdirectory(tools/binpac)
 set(ENABLE_STATIC_ONLY ${ENABLE_STATIC_ONLY_SAVED})
 
 # FIXME: avoid hard-coding a path for multi-config generator support. See the
 # TODO in ZeekPluginConfig.cmake.in.
-set(BINPAC_EXE_PATH "${CMAKE_BINARY_DIR}/auxil/binpac/src/binpac${CMAKE_EXECUTABLE_SUFFIX}")
+set(BINPAC_EXE_PATH "${CMAKE_BINARY_DIR}/tools/binpac/src/binpac${CMAKE_EXECUTABLE_SUFFIX}")
 set(_binpac_exe_path "included")
 
 # Need to call find_package so it sets up the include paths used by plugin builds.
 find_package(BinPAC REQUIRED)
 add_executable(Zeek::BinPAC ALIAS binpac)
 
-add_subdirectory(auxil/bifcl)
+add_subdirectory(tools/bifcl)
 add_executable(Zeek::BifCl ALIAS bifcl)
 # FIXME: avoid hard-coding a path for multi-config generator support. See the
 # TODO in ZeekPluginConfig.cmake.in.
-set(BIFCL_EXE_PATH "${CMAKE_BINARY_DIR}/auxil/bifcl/bifcl${CMAKE_EXECUTABLE_SUFFIX}")
+set(BIFCL_EXE_PATH "${CMAKE_BINARY_DIR}/tools/bifcl/bifcl${CMAKE_EXECUTABLE_SUFFIX}")
 set(_bifcl_exe_path "included")
 
-if (NOT GEN_ZAM_EXE_PATH)
-    add_subdirectory(auxil/gen-zam)
-endif ()
+add_subdirectory(tools/gen-zam)
 
 if (ENABLE_JEMALLOC)
     if (${CMAKE_SYSTEM_NAME} MATCHES "FreeBSD")
@@ -1189,18 +1187,6 @@ endif ()
 # Tell the plugin code that we're building as part of the main tree.
 set(ZEEK_PLUGIN_INTERNAL_BUILD true CACHE INTERNAL "" FORCE)
 
-set(ZEEK_HAVE_AF_PACKET no)
-if (${CMAKE_SYSTEM_NAME} MATCHES Linux)
-    if (NOT DISABLE_AF_PACKET)
-        if (NOT AF_PACKET_PLUGIN_PATH)
-            set(AF_PACKET_PLUGIN_PATH ${CMAKE_SOURCE_DIR}/auxil/zeek-af_packet-plugin)
-        endif ()
-
-        list(APPEND ZEEK_INCLUDE_PLUGINS ${AF_PACKET_PLUGIN_PATH})
-        set(ZEEK_HAVE_AF_PACKET yes)
-    endif ()
-endif ()
-
 set(ZEEK_HAVE_JAVASCRIPT no)
 if (NOT DISABLE_JAVASCRIPT)
     set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${PROJECT_SOURCE_DIR}/auxil/zeekjs/cmake)
@@ -1220,6 +1206,7 @@ if (NOT DISABLE_JAVASCRIPT)
     endif ()
 endif ()
 
+set(ZEEK_HAVE_AF_PACKET no CACHE INTERNAL "Zeek has AF_PACKET support")
 set(ZEEK_HAVE_JAVASCRIPT ${ZEEK_HAVE_JAVASCRIPT} CACHE INTERNAL "Zeek has JavaScript support")
 
 set(DEFAULT_ZEEKPATH_PATHS
@@ -1558,11 +1545,8 @@ message("")
 
 output_summary_bool("AF_PACKET" ${ZEEK_HAVE_AF_PACKET})
 output_summary_bool("Aux. Tools" ${INSTALL_AUX_TOOLS})
-output_summary_line("BifCL" ${_bifcl_exe_path})
-output_summary_line("BinPAC" ${_binpac_exe_path})
 output_summary_bool("BTest" ${INSTALL_BTEST})
 output_summary_line("BTest tooling" ${_install_btest_tools_msg})
-output_summary_line("Gen-ZAM" ${_gen_zam_exe_path})
 output_summary_bool("JavaScript" ${ZEEK_HAVE_JAVASCRIPT})
 output_summary_line("Spicy" ${_spicy})
 output_summary_bool("Spicy analyzers" ${USE_SPICY_ANALYZERS})

NEWS

@@ -6,17 +6,73 @@ release. For an exhaustive list of changes, see the ``CHANGES`` file
 Zeek 8.1.0
 ==========
 
-We would like to thank ... for their contributions to this release.
+We would like to thank @chrisjlly, Klemens Nanni (@klemensn), and Klemens Nanni
+(@klemens-ya) for their contributions to this release.
 
 Breaking Changes
 ----------------
 
+- Python 3.10 is now required for Zeek and all of its associated subprojects.
+
+- The ``&optional`` script attribute will now error when applied to anything that's
+  not a record field. Previously, this would have surprising behavior.
+
+- The BinPAC, Bifcl, and Gen-ZAM tools have all moved directly into the Zeek repo, which
+  should ease maintenance on them a bit. They were moved from the ``auxil`` directory to the
+  tools directory. Along with this, the ``--gen-zam`` argument for ``configure`` was
+  removed and the internal version will always be used.
+
+- The zeek-af_packet-plugin git submodule was moved directly into the Zeek repo. This used
+  to live in the ``auxil`` directory, after having moved there from an external plugin.
+  It is now built as part of main Zeek build whenever building on Linux.
+
 New Functionality
 -----------------
 
+- A new TapAnalyzer class was added allowing to tap into all packets delivered
+  to child analyzers attached to session adapters.
+
+- Two new hooks, ``Cluster::on_subscribe()`` and ``Cluster::on_unsubscribe()`` have
+  been added to allow observing ``Subscribe()`` and ``Unsubscribe()`` calls on
+  backends by Zeek scripts.
+
+- The ability to control the length of strings and containers in log output was added. The
+  maximum length of individual log fields can be set, as well as the total length of all
+  string or container fields in a single log record. This feature is controlled via four
+  new script-level variables:
+
+	Log::default_max_field_string_bytes
+	Log::default_max_total_string_bytes
+	Log::default_max_field_container_elements
+	Log::default_max_total_container_elements
+
+  When one of the ``field`` limits is reached, the individual field is truncated. When one
+  of the ``total`` limits is reached, all further strings will returned as empty and all
+  further container elements will not be output. See the documentation for those variables
+  for more detail.
+
+  The above variables control the truncation globally, but they can also be set for log
+  streams individually. This is controlled by variables with the same names that can be
+  set when the log stream is created.
+
+  Two new weirds were added to report the truncation: ``log_string_field_truncated`` and
+  ``log_container_field_truncated``. New metrics were added to track how many truncations
+  have occurred: ``zeek_log_writer_truncated_string_fields_total`` and
+  ``zeek_log_writer_truncated_containers_total``. The metrics are reported for each log
+  stream.
+
+- The DNS analyzer now returns the set of parameters for SVCB data. It previously handled
+  SVCB packets, but omitted the parameters while parsing.
+
 Changed Functionality
 ---------------------
 
+- The var-extraction-uri.zeek policy does not include the path in the ``uri_vars``
+  field anymore.
+
+- The ``get_current_packet_header()`` now populates the returned record also for
+  fragmented IP datagrams.
+
 Removed Functionality
 ---------------------
 
@@ -27,7 +83,13 @@ Deprecated Functionality
 Zeek 8.0.0
 ==========
 
-We would like to thank Bhaskar Bhar (@bhaskarbhar) for their contributions to this
+We would like to thank @aidans111, Anthony Verez (@netantho), Baa (@Baa14453),
+Bhaskar Bhar (@bhaskarbhar), @dwhitemv25, EdKo (@ephikos), @edoardomich, Fupeng
+Zhao (@AmazingPP), hendrik.schwartke@os-s.de (@hendrikschwartke), @i2z1, Jan
+Grashöfer (@J-Gras) Jean-Samuel Marier, Justin Azoff (@JustinAzoff), Mario D
+(@mari0d), Markus Elfring (@elfring), Peter Cullen (@pbcullen), Sean Donaghy,
+Simeon Miteff (@simeonmiteff), Steve Smoot (@stevesmoot), @timo-mue,
+@wojciech-graj, and Xiaochuan Ye (@XueSongTap) for their contributions to this
 release.
 
 Breaking Changes
@@ -303,7 +365,7 @@ New Functionality
 - Zeek now supports extracting the PPPoE session ID. The ``PacketAnalyzer::PPPoE::session_id``
   BiF can be used to get the session ID of the current packet.
 
-  The ``onn/pppoe-session-id-logging.zeek`` policy script adds pppoe session IDs to the
+  The ``conn/pppoe-session-id-logging.zeek`` policy script adds pppoe session IDs to the
   connection log.
 
   The ``get_conn_stats()`` function's return value now includes the number of packets
@@ -373,7 +435,7 @@ Changed Functionality
   times in X509 certificates as local times.
 
 - The PPPoE parser now respects the size value given in the PPPoE header. Data
-  beyon the size given in the header will be truncated.
+  beyond the size given in the header will be truncated.
 
 - Record fields with ``&default`` attributes initializing empty ``vector``, ``table``
   or ``set`` instances are now deferred until they are accessed, potentially
@@ -440,6 +502,9 @@ Deprecated Functionality
   ``std::string`` and ``std::string_view`` added ``begins_with`` and ``ends_with`` methods
   in C++ 20, and those should be used instead.
 
+- The ``record_type_to_vector`` BIF is deprecated in favor of using the newly ordered
+  ``record_fields`` BIF.
+
 Zeek 7.2.0
 ==========
 
@@ -762,7 +827,7 @@ New Functionality
     some updates to Zeek's internal DNS resolver due to changes in the c-ares
     API. At least version v1.28.0 is now required to build Zeek.
 
-  - Python 3.9 is now required for Zeek and all of it's associated subprojects.
+  - Python 3.9 is now required for Zeek and all of its associated subprojects.
 
 - IP-based connections that were previously not logged due to using an unknown
   IP protocol (e.g. not TCP, UDP, or ICMP) now appear in conn.log. All conn.log
@@ -853,7 +918,7 @@ New Functionality
   analyzer used for processing the packet when the event is raised. The
   ``unknown_protocol.log`` file was extended to include this information.
 
-- The MySQL analyzer now generates a ``mysql_user_change()`` event when the user
+- The MySQL analyzer now generates a ``mysql_change_user()`` event when the user
   changes mid-session via the ``COM_USER_CHANGE`` command.
 
 - The DNS analyzer was extended to support TKEY RRs (RFC 2390). A corresponding

README

@@ -3,7 +3,7 @@ The Zeek Network Security Monitor
 =================================
 
 Zeek is a powerful framework for network traffic analysis and security
-monitoring. Follow us on Twitter at @zeekurity.
+monitoring.
 
 Key Features
 ============

README.md

@@ -15,14 +15,15 @@ traffic analysis and security monitoring.
 [_Development_](#development) —
 [_License_](#license)
 
-Follow us on Twitter at [@zeekurity](https://twitter.com/zeekurity).
-
 [![Coverage Status](https://coveralls.io/repos/github/zeek/zeek/badge.svg?branch=master)](https://coveralls.io/github/zeek/zeek?branch=master)
 [![Build Status](https://img.shields.io/cirrus/github/zeek/zeek)](https://cirrus-ci.com/github/zeek/zeek)
 
 [![Slack](https://img.shields.io/badge/slack-@zeek-brightgreen.svg?logo=slack)](https://zeek.org/slack)
 [![Discourse](https://img.shields.io/discourse/status?server=https%3A%2F%2Fcommunity.zeek.org)](https://community.zeek.org)
 
+[![Mastodon](https://img.shields.io/badge/mastodon-@zeek@infosec.exchange-brightgreen.svg?logo=mastodon)](https://infosec.exchange/@zeek)
+[![Bluesky](https://img.shields.io/badge/bluesky-@zeek-brightgreen.svg?logo=bluesky)](https://bsky.app/profile/zeek.org)
+
 </h4>
 
 

VERSION

@@ -1 +1 @@
-8.1.0-dev.0
+8.1.0-dev.626

auxil/bifcl

@@ -1 +0,0 @@
-Subproject commit 5947749f7850b075f11d6a2aaefe7dad4f63cb62

auxil/binpac

@@ -1 +0,0 @@
-Subproject commit 48f75b5f6415fe9d597e3e991cec635b1bc400dc

auxil/gen-zam

@@ -1 +0,0 @@
-Subproject commit 56a6db00b887c79d26f303676677cb490d1c296d

auxil/libkqueue

@@ -1 +1 @@
-Subproject commit af5612bb0db00831b82e706d63176094243c8ad7
+Subproject commit ea30540c77679ced3ce7886199384e8743628921

auxil/spicy

@@ -1 +1 @@
-Subproject commit 140e88c9a8e04eca801bbd891e085cc180eee43f
+Subproject commit 7635e113080be6fc20cb308636c8c38565c95c8a

auxil/zeek-af_packet-plugin

@@ -1 +0,0 @@
-Subproject commit b89a6f64123f778090d1dd6ec48e6b8e8906ea11

auxil/zeek-aux

@@ -1 +1 @@
-Subproject commit 6c72725b184cc5fd7d12cea5084f0f51de3e82e3
+Subproject commit 9a51ce1940a808aaad253077905c2b34f15f1e08

auxil/zeek-client

@@ -1 +1 @@
-Subproject commit 4440c7a05ba4be229ac88d70e8f4eef2465afc50
+Subproject commit 16849ca3ec2f8637e3f8ef8ee27e2c279724387f

auxil/zeekctl

@@ -1 +1 @@
-Subproject commit 93459b37c3deab4bec9e886211672024fa3e4759
+Subproject commit 485abcad45daeea6d09680e5fc7d29e97d2e3fbe

auxil/zeekjs

@@ -1 +1 @@
-Subproject commit df112dc70ad7c2b854e4aeaad0742cdfa3927945
+Subproject commit e5985abfffc1ef5ead3a0bab196fa5d86bc5276f

ci/alpine/Dockerfile

@@ -2,7 +2,7 @@ FROM alpine:latest
 
 # A version field to invalidate Cirrus's build cache when needed, as suggested in
 # https://github.com/cirruslabs/cirrus-ci-docs/issues/544#issuecomment-566066822
-ENV DOCKERFILE_VERSION 20250306
+ENV DOCKERFILE_VERSION=20250905
 
 RUN apk add --no-cache \
   bash \

ci/centos-stream-10/Dockerfile

@@ -0,0 +1,49 @@
+FROM quay.io/centos/centos:stream10
+
+# A version field to invalidate Cirrus's build cache when needed, as suggested in
+# https://github.com/cirruslabs/cirrus-ci-docs/issues/544#issuecomment-566066822
+ENV DOCKERFILE_VERSION=20250905
+
+# dnf config-manager isn't available at first, and
+# we need it to install the CRB repo below.
+RUN dnf -y install 'dnf-command(config-manager)'
+
+# What used to be powertools is now called "CRB".
+# We need it for some of the packages installed below.
+# https://docs.fedoraproject.org/en-US/epel/
+RUN dnf config-manager --set-enabled crb
+RUN dnf -y install \
+    https://dl.fedoraproject.org/pub/epel/epel-release-latest-10.noarch.rpm
+
+# The --nobest flag is hopefully temporary. Without it we currently hit
+# package versioning conflicts around OpenSSL.
+RUN dnf -y --nobest install \
+    bison \
+    ccache \
+    cmake \
+    cppzmq-devel \
+    diffutils \
+    flex \
+    gcc \
+    gcc-c++ \
+    git \
+    jq \
+    libpcap-devel \
+    make \
+    openssl \
+    openssl-devel \
+    procps-ng \
+    python3 \
+    python3-devel \
+    python3-pip\
+    sqlite \
+    swig \
+    tar \
+    which \
+    zlib-devel \
+  && dnf clean all && rm -rf /var/cache/dnf
+
+# Set the crypto policy to allow SHA-1 certificates - which we have in our tests
+RUN dnf -y --nobest install crypto-policies-scripts && update-crypto-policies --set LEGACY
+
+RUN pip3 install websockets junit2html

ci/centos-stream-9/Dockerfile

@@ -2,7 +2,7 @@ FROM quay.io/centos/centos:stream9
 
 # A version field to invalidate Cirrus's build cache when needed, as suggested in
 # https://github.com/cirruslabs/cirrus-ci-docs/issues/544#issuecomment-566066822
-ENV DOCKERFILE_VERSION 20241024
+ENV DOCKERFILE_VERSION=20250905
 
 # dnf config-manager isn't available at first, and
 # we need it to install the CRB repo below.
@@ -34,9 +34,9 @@ RUN dnf -y --nobest install \
     openssl \
     openssl-devel \
     procps-ng \
-    python3 \
-    python3-devel \
-    python3-pip\
+    python3.13 \
+    python3.13-devel \
+    python3.13-pip\
     sqlite \
     swig \
     tar \
@@ -47,4 +47,8 @@ RUN dnf -y --nobest install \
 # Set the crypto policy to allow SHA-1 certificates - which we have in our tests
 RUN dnf -y --nobest install crypto-policies-scripts && update-crypto-policies --set LEGACY
 
+# Override the default python3.9 installation paths with 3.13
+RUN alternatives --install /usr/bin/python3 python3 /usr/bin/python3.13 10
+RUN alternatives --install /usr/bin/pip3 pip3 /usr/bin/pip3.13 10
+
 RUN pip3 install websockets junit2html

ci/debian-12/Dockerfile

@@ -4,7 +4,7 @@ ENV DEBIAN_FRONTEND="noninteractive" TZ="America/Los_Angeles"
 
 # A version field to invalidate Cirrus's build cache when needed, as suggested in
 # https://github.com/cirruslabs/cirrus-ci-docs/issues/544#issuecomment-566066822
-ENV DOCKERFILE_VERSION 20241024
+ENV DOCKERFILE_VERSION=20250905
 
 RUN apt-get update && apt-get -y install \
     bison \

ci/debian-13/Dockerfile

@@ -1,32 +1,36 @@
-FROM debian:11
+FROM debian:13
 
 ENV DEBIAN_FRONTEND="noninteractive" TZ="America/Los_Angeles"
 
 # A version field to invalidate Cirrus's build cache when needed, as suggested in
 # https://github.com/cirruslabs/cirrus-ci-docs/issues/544#issuecomment-566066822
-ENV DOCKERFILE_VERSION 20241024
+ENV DOCKERFILE_VERSION=20250905
 
 RUN apt-get update && apt-get -y install \
     bison \
     bsdmainutils \
     ccache \
     cmake \
+    cppzmq-dev \
     curl \
+    dnsmasq \
     flex \
     g++ \
     gcc \
     git \
     jq \
     libkrb5-dev \
+    libnats-dev \
     libnode-dev \
     libpcap-dev \
+    librdkafka-dev \
     libssl-dev \
     libuv1-dev \
-    libzmq3-dev \
     make \
     python3 \
     python3-dev \
     python3-pip\
+    python3-websockets \
     sqlite3 \
     swig \
     wget \
@@ -35,4 +39,6 @@ RUN apt-get update && apt-get -y install \
   && apt autoclean \
   && rm -rf /var/lib/apt/lists/*
 
-RUN pip3 install websockets junit2html
+# Debian trixie really doesn't like using pip to install system wide stuff, but
+# doesn't seem there's a python3-junit2html package, so not sure what we'd break.
+RUN pip3 install --break-system-packages junit2html

ci/fedora-41/Dockerfile

@@ -2,7 +2,7 @@ FROM fedora:41
 
 # A version field to invalidate Cirrus's build cache when needed, as suggested in
 # https://github.com/cirruslabs/cirrus-ci-docs/issues/544#issuecomment-566066822
-ENV DOCKERFILE_VERSION 20250203
+ENV DOCKERFILE_VERSION=20250905
 
 RUN dnf -y install \
     bison \

ci/fedora-42/Dockerfile

@@ -2,7 +2,7 @@ FROM fedora:42
 
 # A version field to invalidate Cirrus's build cache when needed, as suggested in
 # https://github.com/cirruslabs/cirrus-ci-docs/issues/544#issuecomment-566066822
-ENV DOCKERFILE_VERSION 20250508
+ENV DOCKERFILE_VERSION=20250905
 
 RUN dnf -y install \
     bison \

ci/opensuse-leap-15.6/Dockerfile

@@ -2,7 +2,7 @@ FROM opensuse/leap:15.6
 
 # A version field to invalidate Cirrus's build cache when needed, as suggested in
 # https://github.com/cirruslabs/cirrus-ci-docs/issues/544#issuecomment-566066822
-ENV DOCKERFILE_VERSION 20241024
+ENV DOCKERFILE_VERSION=20250905
 
 RUN zypper addrepo https://download.opensuse.org/repositories/openSUSE:Leap:15.6:Update/standard/openSUSE:Leap:15.6:Update.repo \
  && zypper refresh \

ci/opensuse-tumbleweed/Dockerfile

@@ -2,7 +2,7 @@ FROM opensuse/tumbleweed
 
 # A version field to invalidate Cirrus's build cache when needed, as suggested in
 # https://github.com/cirruslabs/cirrus-ci-docs/issues/544#issuecomment-566066822
-ENV DOCKERFILE_VERSION 20250502
+ENV DOCKERFILE_VERSION=20250905
 
 # Remove the repo-openh264 repository, it caused intermittent issues
 # and we should not be needing any packages from it.

ci/ubuntu-22.04/Dockerfile

@@ -4,7 +4,7 @@ ENV DEBIAN_FRONTEND="noninteractive" TZ="America/Los_Angeles"
 
 # A version field to invalidate Cirrus's build cache when needed, as suggested in
 # https://github.com/cirruslabs/cirrus-ci-docs/issues/544#issuecomment-566066822
-ENV DOCKERFILE_VERSION 20241024
+ENV DOCKERFILE_VERSION=20250905
 
 RUN apt-get update && apt-get -y install \
     bc \

ci/ubuntu-24.04/Dockerfile

@@ -4,7 +4,7 @@ ENV DEBIAN_FRONTEND="noninteractive" TZ="America/Los_Angeles"
 
 # A version field to invalidate Cirrus's build cache when needed, as suggested in
 # https://github.com/cirruslabs/cirrus-ci-docs/issues/544#issuecomment-566066822
-ENV DOCKERFILE_VERSION 20250522
+ENV DOCKERFILE_VERSION=20250905
 
 RUN apt-get update && apt-get -y install \
     bc \
@@ -32,7 +32,9 @@ RUN apt-get update && apt-get -y install \
     make \
     python3 \
     python3-dev \
+    python3-git \
     python3-pip \
+    python3-semantic-version \
     redis-server \
     ruby \
     sqlite3 \

ci/ubuntu-25.04/Dockerfile

@@ -1,10 +1,10 @@
-FROM ubuntu:24.10
+FROM ubuntu:25.04
 
 ENV DEBIAN_FRONTEND="noninteractive" TZ="America/Los_Angeles"
 
 # A version field to invalidate Cirrus's build cache when needed, as suggested in
 # https://github.com/cirruslabs/cirrus-ci-docs/issues/544#issuecomment-566066822
-ENV DOCKERFILE_VERSION 20241115
+ENV DOCKERFILE_VERSION=20250905
 
 RUN apt-get update && apt-get -y install \
     bc \

ci/windows/Dockerfile

@@ -5,7 +5,7 @@ SHELL [ "powershell" ]
 
 # A version field to invalidatea Cirrus's build cache when needed, as suggested in
 # https://github.com/cirruslabs/cirrus-ci-docs/issues/544#issuecomment-566066822
-ENV DOCKERFILE_VERSION 20250528
+ENV DOCKERFILE_VERSION=20250905
 
 RUN Set-ExecutionPolicy Unrestricted -Force
 

ci/windows/build.cmd

@@ -7,5 +7,5 @@ call "c:\Program Files (x86)\Microsoft Visual Studio\2022\BuildTools\VC\Auxiliar
 mkdir build
 cd build
 
-cmake.exe .. -DCMAKE_BUILD_TYPE=release -DVCPKG_TARGET_TRIPLET="x64-windows-static" -DENABLE_ZEEK_UNIT_TESTS=yes -DENABLE_CLUSTER_BACKEND_ZEROMQ=no -G Ninja
+cmake.exe .. -DCMAKE_BUILD_TYPE=release -DVCPKG_TARGET_TRIPLET="x64-windows-static" -DENABLE_ZEEK_UNIT_TESTS=yes -G Ninja
 cmake.exe --build .

cmake

@@ -1 +1 @@
-Subproject commit 913c8a1a9cdece3461a39a07c330b89baa09be99
+Subproject commit d51c6990446cf70cb9c01bca17dad171a1db05d3

cmake_templates/zeek-config-paths.h.in

@@ -3,8 +3,6 @@
 #pragma once
 
 constexpr char ZEEK_SCRIPT_INSTALL_PATH[] = "@ZEEK_SCRIPT_INSTALL_PATH@";
-[[deprecated("Remove in v8.1. Use ZEEK_PLUGIN_INSTALL_PATH")]] constexpr char BRO_PLUGIN_INSTALL_PATH[] =
-    "@ZEEK_PLUGIN_DIR@";
 constexpr char ZEEK_PLUGIN_INSTALL_PATH[] = "@ZEEK_PLUGIN_DIR@";
 constexpr char DEFAULT_ZEEKPATH[] = "@DEFAULT_ZEEKPATH@";
 constexpr char ZEEK_SPICY_MODULE_PATH[] = "@ZEEK_SPICY_MODULE_PATH@";

configure

@@ -90,15 +90,9 @@ Usage: $0 [OPTION]... [VAR=VALUE]...
     --disable-zkg          don't install zkg
 
   Required Packages in Non-Standard Locations:
-    --with-bifcl=PATH      path to Zeek BIF compiler executable
-                           (useful for cross-compiling)
-    --with-binpac=PATH     path to BinPAC executable
-                           (useful for cross-compiling)
     --with-bison=PATH      path to bison executable
     --with-broker=PATH     path to Broker install root
                            (Zeek uses an embedded version by default)
-    --with-gen-zam=PATH    path to Gen-ZAM code generator
-                           (Zeek uses an embedded version by default)
     --with-flex=PATH       path to flex executable
     --with-libkqueue=PATH  path to libkqueue install root
                            (Zeek uses an embedded version by default)
@@ -184,8 +178,6 @@ builddir=build
 CMakeCacheEntries=""
 display_cmake=0
 has_disable_archiver=0
-has_with_binpac=0
-has_with_bifcl=0
 
 # parse arguments
 while [ $# -ne 0 ]; do
@@ -369,15 +361,9 @@ while [ $# -ne 0 ]; do
         --disable-zkg)
             append_cache_entry INSTALL_ZKG BOOL false
             ;;
-        --with-bifcl=*)
-            has_with_bifcl=1
-            ;;
         --with-bind=*)
             append_cache_entry BIND_ROOT_DIR PATH $optarg
             ;;
-        --with-binpac=*)
-            has_with_binpac=1
-            ;;
         --with-bison=*)
             append_cache_entry BISON_EXECUTABLE PATH $optarg
             ;;
@@ -390,9 +376,6 @@ while [ $# -ne 0 ]; do
         --with-flex=*)
             append_cache_entry FLEX_EXECUTABLE PATH $optarg
             ;;
-        --with-gen-zam=*)
-            append_cache_entry GEN_ZAM_EXE_PATH PATH $optarg
-            ;;
         --with-geoip=*)
             append_cache_entry LibMMDB_ROOT_DIR PATH $optarg
             ;;
@@ -508,15 +491,3 @@ eval ${cmake} 2>&1
 echo "# This is the command used to configure this build" >config.status
 echo $command >>config.status
 chmod u+x config.status
-
-if [ $has_with_bifcl -eq 1 ]; then
-    echo
-    echo "The --with-bifcl option has been deprecated and will be removed in v8.1."
-    echo "The bundled version of bifcl will always be used."
-fi
-
-if [ $has_with_binpac -eq 1 ]; then
-    echo
-    echo "The --with-binpac option has been deprecated and will be removed in v8.1."
-    echo "The bundled version of BinPAC will always be used.\n"
-fi

doc

@@ -1 +1 @@
-Subproject commit 1ce37d96e268134100fbc6793c0c64d48e162337
+Subproject commit 8f38ae2fd563314393eb1ca58c827d26e9966520

docker/builder.Dockerfile

@@ -1,7 +1,7 @@
 # See the file "COPYING" in the main distribution directory for copyright.
 
 # Layer to build Zeek.
-FROM debian:bookworm-slim
+FROM debian:13-slim
 
 # Make the shell split commands in the log so we can determine reasons for
 # failures more easily.
@@ -16,6 +16,7 @@ RUN echo 'Acquire::https::timeout "180";' >> /etc/apt/apt.conf.d/99-timeouts
 
 # Configure system for build.
 RUN apt-get -q update \
+ && apt-get upgrade -q -y \
  && apt-get install -q -y --no-install-recommends \
      bind9 \
      bison \
@@ -36,7 +37,7 @@ RUN apt-get -q update \
      libz-dev \
      make \
      python3-minimal \
-     python3.11-dev \
+     python3-dev \
      swig \
      ninja-build \
      python3-pip \

docker/final.Dockerfile

@@ -1,7 +1,7 @@
 # See the file "COPYING" in the main distribution directory for copyright.
 
 # Final layer containing all artifacts.
-FROM debian:bookworm-slim
+FROM debian:13-slim
 
 # Make the shell split commands in the log so we can determine reasons for
 # failures more easily.
@@ -15,14 +15,15 @@ RUN echo 'Acquire::http::timeout "180";' > /etc/apt/apt.conf.d/99-timeouts
 RUN echo 'Acquire::https::timeout "180";' >> /etc/apt/apt.conf.d/99-timeouts
 
 RUN apt-get -q update \
+ && apt-get upgrade -q -y \
  && apt-get install -q -y --no-install-recommends \
      ca-certificates \
      git \
      jq \
      libmaxminddb0 \
-     libnode108 \
+     libnode115 \
      libpcap0.8 \
-     libpython3.11 \
+     libpython3.13 \
      libssl3 \
      libuv1 \
      libz1 \

scripts/base/files/x509/main.zeek

@@ -105,6 +105,29 @@ export {
 
 	## Event for accessing logged records.
 	global log_x509: event(rec: Info);
+
+	## The maximum number of bytes that a single string field can contain when
+	## logging. If a string reaches this limit, the log output for the field will be
+	## truncated. Setting this to zero disables the limiting.
+	##
+	## .. zeek:see:: Log::default_max_field_string_bytes
+	const default_max_field_string_bytes = Log::default_max_field_string_bytes &redef;
+
+	## The maximum number of elements a single container field can contain when
+	## logging. If a container reaches this limit, the log output for the field will
+	## be truncated. Setting this to zero disables the limiting.
+	##
+	## .. zeek:see:: Log::default_max_field_container_elements
+	const default_max_field_container_elements = 500 &redef;
+
+	## The maximum total number of container elements a record may log. This is the
+	## sum of all container elements logged for the record. If this limit is reached,
+	## all further containers will be logged as empty containers. If the limit is
+	## reached while processing a container, the container will be truncated in the
+	## output. Setting this to zero disables the limiting.
+	##
+	## .. zeek:see:: Log::default_max_total_container_elements
+	const default_max_total_container_elements = 1500 &redef;
 }
 
 global known_log_certs_with_broker: set[LogCertHash] &create_expire=relog_known_certificates_after &backend=Broker::MEMORY;
@@ -117,7 +140,12 @@ redef record Files::Info += {
 
 event zeek_init() &priority=5
 	{
-	Log::create_stream(X509::LOG, Log::Stream($columns=Info, $ev=log_x509, $path="x509", $policy=log_policy));
+	# x509 can have some very large certificates and very large sets of URIs. Expand the log size filters
+	# so that we're not truncating those.
+	Log::create_stream(X509::LOG, Log::Stream($columns=Info, $ev=log_x509, $path="x509", $policy=log_policy,
+	                                          $max_field_string_bytes=X509::default_max_field_string_bytes,
+	                                          $max_field_container_elements=X509::default_max_field_container_elements,
+	                                          $max_total_container_elements=X509::default_max_total_container_elements));
 
 	# We use MIME types internally to distinguish between user and CA certificates.
 	# The first certificate in a connection always gets tagged as user-cert, all
@@ -225,4 +253,3 @@ event file_state_remove(f: fa_file) &priority=5
 
 	Log::write(LOG, f$info$x509);
 	}
-

scripts/base/frameworks/analyzer/dpd.zeek

@@ -6,9 +6,6 @@
 module DPD;
 
 export {
-	## Deprecated, please see https://github.com/zeek/zeek/pull/4200 for details
-	option max_violations: table[Analyzer::Tag] of count = table() &deprecated="Remove in v8.1: This has become non-functional in Zeek 7.2, see PR #4200" &default = 5;
-
 	## Analyzers which you don't want to remove on violations.
 	option ignore_violations: set[Analyzer::Tag] = set();
 
@@ -110,4 +107,3 @@ event analyzer_violation_info(atype: AllAnalyzers::Tag, info: AnalyzerViolationI
 	if ( disabled )
 		event analyzer_failed(network_time(), atype, info);
 	}
-

scripts/base/frameworks/analyzer/logging.zeek

@@ -23,8 +23,10 @@ export {
 		uid:            string            &log &optional;
 		## File UID if available.
 		fuid:           string            &log &optional;
-		## Connection identifier if available
+		## Connection identifier if available.
 		id:             conn_id           &log &optional;
+		## Transport protocol for the violation, if available.
+		proto:          transport_proto   &log &optional;
 		## Failure or violation reason, if available.
 		failure_reason: string            &log;
 		## Data causing failure or violation if available. Truncated
@@ -62,6 +64,7 @@ function log_analyzer_failure(ts: time, atype: AllAnalyzers::Tag, info: Analyzer
 		{
 		rec$id = info$c$id;
 		rec$uid = info$c$uid;
+		rec$proto = get_port_transport_proto(info$c$id$orig_p);
 		}
 
 	if ( info?$f )

scripts/base/frameworks/broker/main.zeek

@@ -28,7 +28,7 @@ export {
 
 	## Default address on which to listen for WebSocket connections.
 	##
-	## .. zeek:see:: Broker::listen_websocket
+	## .. zeek:see:: Cluster::listen_websocket
 	const default_listen_address_websocket = getenv("ZEEK_DEFAULT_LISTEN_ADDRESS") &redef;
 
 	## Default interval to retry connecting to a peer if it cannot be made to
@@ -69,11 +69,6 @@ export {
 	## all peers.
 	const ssl_keyfile = "" &redef;
 
-	## The number of buffered messages at the Broker/CAF layer after which
-	## a subscriber considers themselves congested (i.e. tune the congestion
-	## control mechanisms).
-	const congestion_queue_size = 200 &redef &deprecated="Remove in v8.1. Non-functional since v5.0";
-
 	## The max number of log entries per log stream to batch together when
 	## sending log messages to a remote logger.
 	const log_batch_size = 400 &redef;
@@ -319,27 +314,6 @@ export {
 	                        p: port &default = default_port,
 	                        retry: interval &default = default_listen_retry): port;
 
-	## Listen for remote connections using WebSocket.
-	##
-	## a: an address string on which to accept connections, e.g.
-	##    "127.0.0.1".  An empty string refers to INADDR_ANY.
-	##
-	## p: the TCP port to listen on. The value 0 means that the OS should choose
-	##    the next available free port.
-	##
-	## retry: If non-zero, retries listening in regular intervals if the port cannot be
-	##        acquired immediately. 0 disables retries.  If the
-	##        ZEEK_DEFAULT_LISTEN_RETRY environment variable is set (as number
-	##        of seconds), it overrides any value given here.
-	##
-	## Returns: the bound port or 0/? on failure.
-	##
-	## .. zeek:see:: Broker::status
-	global listen_websocket: function(a: string &default = default_listen_address_websocket,
-	                                  p: port &default = default_port_websocket,
-	                                  retry: interval &default = default_listen_retry): port
-		&deprecated="Remove in v8.1. Switch to Cluster::listen_websocket() instead.";
-
 	## Initiate a remote connection.
 	##
 	## a: an address to connect to, e.g. "localhost" or "127.0.0.1".
@@ -450,29 +424,6 @@ export {
 	##
 	## Returns: true if a new event forwarding/subscription is now registered.
 	global forward: function(topic_prefix: string): bool;
-
-	## Automatically send an event to any interested peers whenever it is
-	## locally dispatched. (For example, using "event my_event(...);" in a
-	## script.)
-	##
-	## topic: a topic string associated with the event message.
-	##        Peers advertise interest by registering a subscription to some
-	##        prefix of this topic name.
-	##
-	## ev: a Zeek event value.
-	##
-	## Returns: true if automatic event sending is now enabled.
-	global auto_publish: function(topic: string, ev: any): bool &deprecated="Remove in v8.1. Switch to explicit Cluster::publish() calls. Auto-publish won't work with all cluster backends.";
-
-	## Stop automatically sending an event to peers upon local dispatch.
-	##
-	## topic: a topic originally given to :zeek:see:`Broker::auto_publish`.
-	##
-	## ev: an event originally given to :zeek:see:`Broker::auto_publish`.
-	##
-	## Returns: true if automatic events will not occur for the topic/event
-	##          pair.
-	global auto_unpublish: function(topic: string, ev: any): bool &deprecated="Remove in v8.1. See Broker::auto_publish()";
 }
 
 @load base/bif/comm.bif
@@ -514,31 +465,6 @@ function listen(a: string, p: port, retry: interval): port
 	return bound;
 	}
 
-event retry_listen_websocket(a: string, p: port, retry: interval)
-	{
-@pragma push ignore-deprecations
-	listen_websocket(a, p, retry);
-@pragma pop ignore-deprecations
-	}
-
-function listen_websocket(a: string, p: port, retry: interval): port
-	{
-	local bound = __listen(a, p, Broker::WEBSOCKET);
-
-	if ( bound == 0/tcp )
-		{
-		local e = getenv("ZEEK_DEFAULT_LISTEN_RETRY");
-
-		if ( e != "" )
-			retry = double_to_interval(to_double(e));
-
-		if ( retry != 0secs )
-			schedule retry { retry_listen_websocket(a, p, retry) };
-		}
-
-	return bound;
-	}
-
 function peer(a: string, p: port, retry: interval): bool
 	{
 	return __peer(a, p, retry);
@@ -593,13 +519,3 @@ function unsubscribe(topic_prefix: string): bool
 	{
 	return __unsubscribe(topic_prefix);
 	}
-
-function auto_publish(topic: string, ev: any): bool
-	{
-	return __auto_publish(topic, ev);
-	}
-
-function auto_unpublish(topic: string, ev: any): bool
-	{
-	return __auto_unpublish(topic, ev);
-	}

scripts/base/frameworks/cluster/main.zeek

@@ -362,8 +362,6 @@ export {
 
 	## WebSocket server options to pass to :zeek:see:`Cluster::listen_websocket`.
 	type WebSocketServerOptions: record {
-		## The host address to listen on.
-		listen_host: string &optional &deprecated="Remove in v8.1: Use $listen_addr instead.";
 		## The address to listen on, cannot be used together with ``listen_host``.
 		listen_addr: addr &optional;
 		## The port the WebSocket server is supposed to listen on.
@@ -401,6 +399,20 @@ export {
 		## The value of the X-Application-Name HTTP header, if any.
 		application_name: string &optional;
 	};
+
+	## A hook invoked for every :zeek:see:`Cluster::subscribe` call.
+	##
+	## Breaking from this hook has no effect.
+	##
+	## topic: The topic string as given to :zeek:see:`Cluster::subscribe`.
+	global on_subscribe: hook(topic: string);
+
+	## A hook invoked for every :zeek:see:`Cluster::subscribe` call.
+	##
+	## Breaking from this hook has no effect.
+	##
+	## topic: The topic string as given to :zeek:see:`Cluster::subscribe`.
+	global on_unsubscribe: hook(topic: string);
 }
 
 # Needs declaration of Cluster::Event type.

scripts/base/frameworks/logging/main.zeek

@@ -422,6 +422,26 @@ export {
 		## .. :zeek:see:`Log::default_max_delay_queue_size`
 		## .. :zeek:see:`Log::set_max_delay_queue_size`
 		max_delay_queue_size: count &default=default_max_delay_queue_size;
+
+		## Maximum string size for field in a log record from this stream.
+		##
+		## .. :zeek:see:`Log::default_max_field_string_bytes`
+		max_field_string_bytes: count &default=Log::default_max_field_string_bytes;
+
+		## Maximum total string size in a log record from this stream.
+		##
+		## .. :zeek:see:`Log::default_max_total_string_bytes`
+		max_total_string_bytes: count &default=Log::default_max_total_string_bytes;
+
+		## Maximum container elements for field in a log record from this stream.
+		##
+		## .. :zeek:see:`Log::default_max_field_container_elements`
+		max_field_container_elements: count &default=Log::default_max_field_container_elements;
+
+		## Maximum total container elements in a log record from this stream.
+		##
+		## .. :zeek:see:`Log::default_max_total_container_elements`
+		max_total_container_elements: count &default=Log::default_max_total_container_elements;
 	};
 
 	## Sentinel value for indicating that a filter was not found when looked up.

scripts/base/frameworks/telemetry/main.zeek

@@ -245,16 +245,6 @@ export {
 	                                          label_values: labels_vector,
 	                                          measurement: double): bool;
 
-	## Interval at which the :zeek:see:`Telemetry::sync` hook is invoked.
-	##
-	## By default, the hook is invoked on demand, setting this option to
-	## a positive interval allows to invoke it regularly, too. Regular
-	## invocations are relative to Zeek's network time.
-	##
-	## Note that on-demand hook invocation will happen even if this
-	## is set.
-	option sync_interval = 0sec &deprecated="Remove in 8.1. If you require regular sync invocation, do so explicitly in a scheduled event.";
-
 	## Collect all counter and gauge metrics matching the given *name* and *prefix*.
 	##
 	## For histogram metrics, use the :zeek:see:`Telemetry::collect_histogram_metrics`.
@@ -465,14 +455,6 @@ function collect_histogram_metrics(prefix: string, name: string): vector of Hist
 	return Telemetry::__collect_histogram_metrics(prefix, name);
 	}
 
-event run_sync_hook()
-	{
-	hook Telemetry::sync();
-@pragma push ignore-deprecations
-	schedule sync_interval { run_sync_hook() };
-@pragma pop ignore-deprecations
-	}
-
 # Expose the Zeek version as Prometheus style info metric
 global version_gauge_family = Telemetry::register_gauge_family(Telemetry::MetricOpts(
 	$prefix="zeek",
@@ -485,11 +467,6 @@ global version_gauge_family = Telemetry::register_gauge_family(Telemetry::Metric
 
 event zeek_init()
 	{
-@pragma push ignore-deprecations
-	if ( sync_interval > 0sec )
-		schedule sync_interval { run_sync_hook() };
-@pragma pop ignore-deprecations
-
 	local v = Version::info;
 	local labels = vector(cat(v$version_number),
 	                      cat(v$major), cat(v$minor), cat (v$patch),

scripts/base/init-bare.zeek

@@ -1726,18 +1726,6 @@ const tcp_excessive_data_without_further_acks = 10 * 1024 * 1024 &redef;
 ## buffering.
 const tcp_max_old_segments = 0 &redef;
 
-## For services without a handler, these sets define originator-side ports
-## that still trigger reassembly.
-##
-## .. zeek:see:: tcp_reassembler_ports_resp
-const tcp_reassembler_ports_orig: set[port] = {} &redef &deprecated="Remove in v8.1. Non-functional since v4.1";
-
-## For services without a handler, these sets define responder-side ports
-## that still trigger reassembly.
-##
-## .. zeek:see:: tcp_reassembler_ports_orig
-const tcp_reassembler_ports_resp: set[port] = {} &redef &deprecated="Remove in v8.1. Non-functional since v4.1";
-
 ## Defines destination TCP ports for which the contents of the originator stream
 ## should be delivered via :zeek:see:`tcp_contents`.
 ##
@@ -3063,9 +3051,8 @@ type dns_binds_rr: record {
 	algorithm: count;	##< Algorithm for Public Key.
 	key_id: count;		##< key tag.
 	removal_flag: count;	##< rm flag.
-	complte_flag: string &deprecated="Remove in v8.1: Use complete_flag instead.";	##< complete flag.
-	is_query: count;	##< The RR is a query/Response.
 	complete_flag: count;	##< complete flag.
+	is_query: count;	##< The RR is a query/Response.
 };
 
 ## A Private RR type LOC record.
@@ -3084,12 +3071,30 @@ type dns_loc_rr: record {
 	is_query: count;	##< The RR is a query/Response.
 };
 
-## DNS SVCB and HTTPS RRs
+## A SvcParamKey with an optional SvcParamValue.
+#
+## .. zeek:see:: dns_svcb_rr
+type dns_svcb_param: record {
+	key: count;				##< SvcParamKey
+	mandatory: vector of count &optional;	##< "mandatory" SvcParamKey values
+	alpn: vector of string &optional;	##< "alpn" IDs
+	p: count &optional;			##< "port" number, TCP or UDP
+	hint: vector of addr &optional;		##< "ipv4hint" or "ipv6hint" IP addresses
+	ech: string &optional;			##< "ech" base64 encoded ECHConfigList blob
+	raw: string &optional;			##< reserved key's or malformed value
+};
+
+type dns_svcb_param_vec: vector of dns_svcb_param;
+
+## A SVCB or HTTPS record.
+##
+## See also RFC 9460 - Service Binding and Parameter Specification via the DNS (SVCB and HTTPS Resource Records).
 ##
 ## .. zeek:see:: dns_SVCB dns_HTTPS
 type dns_svcb_rr: record {
-	svc_priority: count;	##< Service priority for the current record, 0 indicates that this record is in AliasMode and cannot carry svc_params; otherwise this is in ServiceMode, and may include svc_params
+	svc_priority: count;				##< Service priority. If zero, the record is in AliasMode and has no SvcParam.
 	target_name: string;				##< Target name, the hostname of the service endpoint.
+	svc_params: dns_svcb_param_vec &optional;	##< Service parameters, if any.
 };
 
 ## A NAPTR record.
@@ -3756,6 +3761,31 @@ export {
 	## higher than this limit, but it prevents runaway-sized log entries from causing
 	## problems.
 	const max_log_record_size = 1024*1024*64 &redef;
+
+	## The maximum number of bytes that a single string field can contain when
+	## logging. If a string reaches this limit, the log output for the field will be
+	## truncated. Setting this to zero disables the limiting.
+	const default_max_field_string_bytes = 4096 &redef;
+
+	## The maximum number of elements a single container field can contain when
+	## logging. If a container reaches this limit, the log output for the field will
+	## be truncated. Setting this to zero disables the limiting.
+	const default_max_field_container_elements = 100 &redef;
+
+	## The maximum total bytes a record may log for string fields. This is the sum of
+	## all bytes in string fields logged for the record. If this limit is reached, all
+	## further string fields will be logged as empty strings. Any containers holding
+	## string fields will be logged as empty containers. If the limit is reached while
+	## processing a container holding string fields, the container will be truncated
+	## in the log output. Setting this to zero disables the limiting.
+	const default_max_total_string_bytes = 256000 &redef;
+
+	## The maximum total number of container elements a record may log. This is the
+	## sum of all container elements logged for the record. If this limit is reached,
+	## all further containers will be logged as empty containers. If the limit is
+	## reached while processing a container, the container will be truncated in the
+	## output. Setting this to zero disables the limiting.
+	const default_max_total_container_elements = 500 &redef;
 }
 
 module POP3;
@@ -5679,6 +5709,31 @@ export {
 	};
 }
 
+module AF_Packet;
+
+export {
+	## Size of the ring-buffer.
+	const buffer_size = 128 * 1024 * 1024 &redef;
+	## Size of an individual block. Needs to be a multiple of page size.
+	const block_size = 4096 * 8 &redef;
+	## Retire timeout for a single block.
+	const block_timeout = 10msec &redef;
+	## Toggle whether to use hardware timestamps.
+	const enable_hw_timestamping = F &redef;
+	## Toggle whether to use PACKET_FANOUT.
+	const enable_fanout = T &redef;
+	## Toggle defragmentation of IP packets using PACKET_FANOUT_FLAG_DEFRAG.
+	const enable_defrag = F &redef;
+	## Fanout mode.
+	const fanout_mode = FANOUT_HASH &redef;
+	## Fanout ID.
+	const fanout_id = 23 &redef;
+	## Link type (default Ethernet).
+	const link_type = 1 &redef;
+	## Checksum validation mode.
+	const checksum_validation_mode: ChecksumMode = CHECKSUM_ON &redef;
+}
+
 module DCE_RPC;
 
 export {

scripts/base/protocols/dns/consts.zeek

@@ -182,8 +182,9 @@ export {
 		[4] = "SHA384",
 	} &default = function(n: count): string { return fmt("digest-%d", n); };
 
-	## SVCB/HTTPS SvcParam keys, as defined in
-	## https://www.ietf.org/archive/id/draft-ietf-dnsop-svcb-https-07.txt, sec 14.3.2
+	## SVCB/HTTPS SvcParam keys as defined in
+	## https://datatracker.ietf.org/doc/html/rfc9460#name-initial-contents
+	## Keep in sync with src/analyzer/protocol/dns/DNS.h SVCPARAM_Key.
 	const svcparam_keys = {
 		[0] = "mandatory",
 		[1] = "alpn",

scripts/base/protocols/http/main.zeek

@@ -139,6 +139,14 @@ export {
 	## out and request/response tracking reset to prevent unbounded
 	## state growth.
 	option max_pending_requests = 100;
+
+	## The maximum number of bytes that a single string field can contain when
+	## logging. If a string reaches this limit, the log output for the field will be
+	## truncated. Setting this to zero disables the limiting. HTTP has no maximum
+	## length for various fields such as the URI, so this is set to zero by default.
+	##
+	## .. zeek:see:: Log::default_max_field_string_bytes
+	const default_max_field_string_bytes = 0 &redef;
 }
 
 # Add the http state tracking fields to the connection record.
@@ -156,7 +164,8 @@ redef likely_server_ports += { ports };
 # Initialize the HTTP logging stream and ports.
 event zeek_init() &priority=5
 	{
-	Log::create_stream(HTTP::LOG, Log::Stream($columns=Info, $ev=log_http, $path="http", $policy=log_policy));
+	Log::create_stream(HTTP::LOG, Log::Stream($columns=Info, $ev=log_http, $path="http", $policy=log_policy,
+	                                          $max_field_string_bytes=HTTP::default_max_field_string_bytes));
 	Analyzer::register_for_ports(Analyzer::ANALYZER_HTTP, ports);
 	}
 
@@ -394,4 +403,3 @@ hook finalize_http(c: connection)
 			}
 		}
 	}
-

scripts/base/protocols/postgresql/main.zeek

@@ -53,7 +53,7 @@ export {
 		user: string &optional;
 		database: string &optional;
 		application_name: string &optional;
-		rows: count &default=0;
+		rows: count &optional;
 		errors: vector of string;
 	};
 
@@ -197,8 +197,6 @@ event PostgreSQL::authentication_ok(c: connection) {
 
 	c$postgresql$backend = "auth_ok";
 	c$postgresql$success = T;
-
-	emit_log(c);
 }
 
 event PostgreSQL::terminate(c: connection) {
@@ -224,6 +222,9 @@ event PostgreSQL::simple_query(c: connection, query: string) {
 event PostgreSQL::data_row(c: connection, column_values: count) {
 	hook set_session(c);
 
+	if ( ! c$postgresql_state?$rows )
+		c$postgresql_state$rows = 0;
+
 	++c$postgresql_state$rows;
 }
 
@@ -236,7 +237,11 @@ event PostgreSQL::ready_for_query(c: connection, transaction_status: string) {
 	if ( ! c$postgresql?$success )
 		c$postgresql$success = transaction_status == "I" || transaction_status == "T";
 
+	if ( c$postgresql_state?$rows ) {
 		c$postgresql$rows = c$postgresql_state$rows;
+		delete c$postgresql_state$rows;
+	}
+
 	emit_log(c);
 }
 

scripts/base/utils/urls.zeek

@@ -117,15 +117,14 @@ function decompose_uri(uri: string): URI
 			}
 		}
 
-	if ( /:/ in s )
+	if ( /:[0-9]*$/ in s )
 		{
-		# Parse location and port.
-		parts = split_string1(s, /:/);
-		u$netlocation = parts[0];
-		if ( parts[1] != "" )
-		    {
-		    u$portnum = to_count(parts[1]);
-		    }
+		# Input ends with a numeric port or just colon: Strip it
+		# for netlocation and convert any port digits into portnum.
+		u$netlocation = gsub(s, /:[0-9]*$/, "");
+		local portstr = s[|u$netlocation| + 1:];
+		if ( portstr != "" )
+			u$portnum = to_count(portstr);
 		}
 	else
 		{

scripts/policy/frameworks/analyzer/deprecated-dpd-log.zeek

@@ -1,96 +0,0 @@
-##! Creates the now deprecated dpd.logfile.
-# Remove in v8.1
-
-module DPD;
-
-export {
-	## Add the DPD logging stream identifier.
-	redef enum Log::ID += { LOG };
-
-	## A default logging policy hook for the stream.
-	global log_policy: Log::PolicyHook;
-
-	## The record type defining the columns to log in the DPD logging stream.
-	type Info: record {
-		## Timestamp for when protocol analysis failed.
-		ts:             time            &log;
-		## Connection unique ID.
-		uid:            string          &log;
-		## Connection ID containing the 4-tuple which identifies endpoints.
-		id:             conn_id         &log;
-		## Transport protocol for the violation.
-		proto:          transport_proto &log;
-		## The analyzer that generated the violation.
-		analyzer:       string          &log;
-		## The textual reason for the analysis failure.
-		failure_reason: string          &log;
-	};
-}
-
-redef record connection += {
-	dpd: Info &optional;
-	## The set of services (analyzers) for which Zeek has observed a
-	## violation after the same service had previously been confirmed.
-	service_violation: set[string] &default=set() &ordered &deprecated="Remove in v8.1. Consider using failed_analyzers instead";
-
-};
-
-event zeek_init() &priority=5
-	{
-	Log::create_stream(DPD::LOG, Log::Stream($columns=Info, $path="dpd", $policy=log_policy));
-	}
-
-# before the same event in dpd.zeek
-event analyzer_violation_info(atype: AllAnalyzers::Tag, info: AnalyzerViolationInfo) &priority=15
-	{
-	if ( ! is_protocol_analyzer(atype) && ! is_packet_analyzer(atype) )
-		return;
-
-	if ( ! info?$c )
-		return;
-
-	local c = info$c;
-	local analyzer = Analyzer::name(atype);
-	# If the service hasn't been confirmed yet, or already failed,
-	# don't generate a log message for the protocol violation.
-	if ( analyzer !in c$service || analyzer in c$service_violation )
-		return;
-
-	add c$service_violation[analyzer];
-
-	local dpd: Info;
-	dpd$ts = network_time();
-	dpd$uid = c$uid;
-	dpd$id = c$id;
-	dpd$proto = get_port_transport_proto(c$id$orig_p);
-	dpd$analyzer = analyzer;
-
-	# Encode data into the reason if there's any as done for the old
-	# analyzer_violation event, previously.
-	local reason = info$reason;
-	if ( info?$data )
-		{
-		local ellipsis = |info$data| > 40 ? "..." : "";
-		local data = info$data[0:40];
-		reason = fmt("%s [%s%s]", reason, data, ellipsis);
-		}
-
-	dpd$failure_reason = reason;
-	c$dpd = dpd;
-	}
-
-event analyzer_violation_info(atype: AllAnalyzers::Tag, info: AnalyzerViolationInfo ) &priority=-5
-	{
-	if ( ! is_protocol_analyzer(atype) && ! is_packet_analyzer(atype) )
-		return;
-
-	if ( ! info?$c )
-		return;
-
-	if ( info$c?$dpd )
-		{
-		Log::write(DPD::LOG, info$c$dpd);
-		delete info$c$dpd;
-		}
-	}
-

scripts/policy/frameworks/dpd/detect-protocols.zeek

@@ -1,3 +0,0 @@
-@deprecated("frameworks/dpd/detect-protocols.zeek moved to frameworks/analyzer/detect-protocols.zeek. Please switch to frameworks/analyzer/detect-protocols.zeek. Remove in 8.1")
-
-@load frameworks/analyzer/detect-protocols.zeek

scripts/policy/frameworks/dpd/packet-segment-logging.zeek

@@ -1,34 +0,0 @@
-@deprecated("Please switch to frameworks/analyzer/packet-segment-logging, which logs to analyzer.log. Remove in 8.1")
-
-##! This script enables logging of packet segment data when a protocol
-##! parsing violation is encountered.  The amount of data from the
-##! packet logged is set by the :zeek:see:`DPD::packet_segment_size` variable.
-##! A caveat to logging packet data is that in some cases, the packet may
-##! not be the packet that actually caused the protocol violation.
-
-@load frameworks/analyzer/deprecated-dpd-log
-
-module DPD;
-
-export {
-	redef record Info += {
-		## A chunk of the payload that most likely resulted in the
-		## analyzer violation.
-		packet_segment: string &optional &log;
-	};
-
-	## Size of the packet segment to display in the DPD log.
-	option packet_segment_size: int = 255;
-}
-
-
-event analyzer_violation_info(atype: AllAnalyzers::Tag, info: AnalyzerViolationInfo) &priority=4
-	{
-	if ( ! is_protocol_analyzer(atype) && ! is_packet_analyzer(atype) )
-		return;
-
-	if ( ! info?$c || ! info$c?$dpd )
-		return;
-
-	info$c$dpd$packet_segment = fmt("%s", sub_bytes(get_current_packet()$data, 0, packet_segment_size));
-	}

scripts/policy/frameworks/management/controller/config.zeek

@@ -61,6 +61,14 @@ export {
 	## for websocket clients.
 	const default_port_websocket = 2149/tcp &redef;
 
+	## TLS options for the controller's WebSocket server. The default is
+	## to operate unencrypted. To replicate Broker's default encryption
+	## without endpoint validation, set the
+	## :zeek:field:`Cluster::WebSocketTLSOptions$ca_file` field to
+	## "NONE" and :zeek:field:`Cluster::WebSocketTLSOptions$ciphers` to
+	## "AECDH-AES256-SHA@SECLEVEL=0:AECDH-AES256-SHA:P-384".
+	const tls_options_websocket = Cluster::WebSocketTLSOptions() &redef;
+
 	## Whether the controller should auto-assign Broker listening ports to
 	## cluster nodes that need them and don't have them explicitly specified
 	## in cluster configurations.

scripts/policy/frameworks/management/controller/main.zeek

@@ -4,6 +4,7 @@
 ##! by the supervisor.
 
 @load base/frameworks/broker
+@load base/frameworks/cluster
 
 @load policy/frameworks/management
 @load policy/frameworks/management/agent/config # For the agent topic prefix
@@ -1644,9 +1645,10 @@ event zeek_init()
 
 	if ( cni$bound_port != 0/unknown )
 		{
-@pragma push ignore-deprecations
-		Broker::listen_websocket(cat(cni$address), cni$bound_port);
-@pragma pop ignore-deprecations
+		local ws_opts = Cluster::WebSocketServerOptions($listen_addr=to_addr(cni$address),
+		                                                $listen_port=cni$bound_port,
+		                                                $tls_options=Management::Controller::tls_options_websocket);
+		Cluster::listen_websocket(ws_opts);
 		websocket_info = fmt("websocket port %s:%s", cni$address, cni$bound_port);
 		}
 

scripts/policy/protocols/http/detect-sqli.zeek

@@ -1,142 +0,0 @@
-##! SQL injection attack detection in HTTP.
-
-## This package is deprecated in favor of detect-sql-injection.zeek.
-##
-## The replacement script annotates the notices it generates with
-## an associated $uid connection identifier; always provides an attacker
-## IP address in the $src field; and always provides a victim IP address
-## in the $dst field. The notices generated by this script, on the other
-## hand, lack a $uid identifier, and do not provide $dst information.
-## In addition, for SQL_Injection_Victim notices, this script provides the
-## victim's IP address in the $src field, which some find counter-intuitive.
-##
-## In addition, the replacement script removes support for generating
-## Notice emails.
-
-@deprecated "Remove in v8.1: Switch to the improved detect-sql-injection script"
-
-@load base/frameworks/notice
-@load base/frameworks/sumstats
-@load base/protocols/http
-
-module HTTP;
-
-export {
-	redef enum Notice::Type += {
-		## Indicates that a host performing SQL injection attacks was
-		## detected.
-		SQL_Injection_Attacker,
-		## Indicates that a host was seen to have SQL injection attacks
-		## against it.  This is tracked by IP address as opposed to
-		## hostname.
-		SQL_Injection_Victim,
-	};
-
-	redef enum Tags += {
-		## Indicator of a URI based SQL injection attack.
-		URI_SQLI,
-		## Indicator of client body based SQL injection attack.  This is
-		## typically the body content of a POST request. Not implemented
-		## yet.
-		POST_SQLI,
-		## Indicator of a cookie based SQL injection attack. Not
-		## implemented yet.
-		COOKIE_SQLI,
-	};
-
-	## Defines the threshold that determines if an SQL injection attack
-	## is ongoing based on the number of requests that appear to be SQL
-	## injection attacks.
-	const sqli_requests_threshold: double = 50.0 &redef;
-
-	## Interval at which to watch for the
-	## :zeek:id:`HTTP::sqli_requests_threshold` variable to be crossed.
-	## At the end of each interval the counter is reset.
-	const sqli_requests_interval = 5min &redef;
-
-	## Collecting samples will add extra data to notice emails
-	## by collecting some sample SQL injection url paths.  Disable
-	## sample collection by setting this value to 0.
-	const collect_SQLi_samples = 5 &redef;
-
-	## Regular expression is used to match URI based SQL injections.
-	const match_sql_injection_uri =
-		  /[\?&][^[:blank:]\x00-\x1f\|\+]+?=[\-[:alnum:]%]+([[:blank:]\x00-\x1f\+]|\/\*.*?\*\/)*['"]?([[:blank:]\x00-\x1f\+]|\/\*.*?\*\/|\)?;)+.*?([hH][aA][vV][iI][nN][gG]|[uU][nN][iI][oO][nN]|[eE][xX][eE][cC]|[sS][eE][lL][eE][cC][tT]|[dD][eE][lL][eE][tT][eE]|[dD][rR][oO][pP]|[dD][eE][cC][lL][aA][rR][eE]|[cC][rR][eE][aA][tT][eE]|[iI][nN][sS][eE][rR][tT])([[:blank:]\x00-\x1f\+]|\/\*.*?\*\/)+/
-		| /[\?&][^[:blank:]\x00-\x1f\|\+]+?=[\-0-9%]+([[:blank:]\x00-\x1f\+]|\/\*.*?\*\/)*['"]?([[:blank:]\x00-\x1f\+]|\/\*.*?\*\/|\)?;)+([xX]?[oO][rR]|[nN]?[aA][nN][dD])([[:blank:]\x00-\x1f\+]|\/\*.*?\*\/)+['"]?(([^a-zA-Z&]+)?=|[eE][xX][iI][sS][tT][sS])/
-		| /[\?&][^[:blank:]\x00-\x1f\+]+?=[\-0-9%]*([[:blank:]\x00-\x1f\+]|\/\*.*?\*\/)*['"]([[:blank:]\x00-\x1f]|\/\*.*?\*\/)*(-|=|\+|\|\|)([[:blank:]\x00-\x1f\+]|\/\*.*?\*\/)*([0-9]|\(?[cC][oO][nN][vV][eE][rR][tT]|[cC][aA][sS][tT])/
-		| /[\?&][^[:blank:]\x00-\x1f\|\+]+?=([[:blank:]\x00-\x1f\+]|\/\*.*?\*\/)*['"]([[:blank:]\x00-\x1f\+]|\/\*.*?\*\/|;)*([xX]?[oO][rR]|[nN]?[aA][nN][dD]|[hH][aA][vV][iI][nN][gG]|[uU][nN][iI][oO][nN]|[eE][xX][eE][cC]|[sS][eE][lL][eE][cC][tT]|[dD][eE][lL][eE][tT][eE]|[dD][rR][oO][pP]|[dD][eE][cC][lL][aA][rR][eE]|[cC][rR][eE][aA][tT][eE]|[rR][eE][gG][eE][xX][pP]|[iI][nN][sS][eE][rR][tT])([[:blank:]\x00-\x1f\+]|\/\*.*?\*\/|[\[(])+[a-zA-Z&]{2,}/
-		| /[\?&][^[:blank:]\x00-\x1f\+]+?=[^\.]*?([cC][hH][aA][rR]|[aA][sS][cC][iI][iI]|[sS][uU][bB][sS][tT][rR][iI][nN][gG]|[tT][rR][uU][nN][cC][aA][tT][eE]|[vV][eE][rR][sS][iI][oO][nN]|[lL][eE][nN][gG][tT][hH])\(/
-		| /\/\*![[:digit:]]{5}.*?\*\// &redef;
-
-	## A hook that can be used to prevent specific requests from being counted
-	## as an injection attempt.  Use a 'break' statement to exit the hook
-	## early and ignore the request.
-	global HTTP::sqli_policy: hook(c: connection, method: string, unescaped_URI: string);
-}
-
-function format_sqli_samples(samples: vector of SumStats::Observation): string
-	{
-	local ret = "SQL Injection samples\n---------------------";
-	for ( i in samples )
-		ret += "\n" + samples[i]$str;
-	return ret;
-	}
-
-event zeek_init() &priority=3
-	{
-	# Add filters to the metrics so that the metrics framework knows how to
-	# determine when it looks like an actual attack and how to respond when
-	# thresholds are crossed.
-	local r1 = SumStats::Reducer($stream="http.sqli.attacker", $apply=set(SumStats::SUM, SumStats::SAMPLE), $num_samples=collect_SQLi_samples);
-	SumStats::create(SumStats::SumStat($name="detect-sqli-attackers",
-		$epoch=sqli_requests_interval,
-		$reducers=set(r1),
-		$threshold_val(key: SumStats::Key, result: SumStats::Result) =
-			{
-			return result["http.sqli.attacker"]$sum;
-			},
-		$threshold=sqli_requests_threshold,
-		$threshold_crossed(key: SumStats::Key, result: SumStats::Result) =
-			{
-			local r = result["http.sqli.attacker"];
-			NOTICE(Notice::Info($note=SQL_Injection_Attacker,
-			                    $msg="An SQL injection attacker was discovered!",
-			                    $email_body_sections=vector(format_sqli_samples(r$samples)),
-			                    $src=key$host,
-			                    $identifier=cat(key$host)));
-			}));
-
-	local r2 = SumStats::Reducer($stream="http.sqli.victim", $apply=set(SumStats::SUM, SumStats::SAMPLE), $num_samples=collect_SQLi_samples);
-	SumStats::create(SumStats::SumStat($name="detect-sqli-victims",
-		$epoch=sqli_requests_interval,
-		$reducers=set(r2),
-		$threshold_val(key: SumStats::Key, result: SumStats::Result) =
-			{
-			return result["http.sqli.victim"]$sum;
-			},
-		$threshold=sqli_requests_threshold,
-		$threshold_crossed(key: SumStats::Key, result: SumStats::Result) =
-			{
-			local r = result["http.sqli.victim"];
-			NOTICE(Notice::Info($note=SQL_Injection_Victim,
-			                    $msg="An SQL injection victim was discovered!",
-			                    $email_body_sections=vector(format_sqli_samples(r$samples)),
-			                    $src=key$host,
-			                    $identifier=cat(key$host)));
-			}));
-	}
-
-event http_request(c: connection, method: string, original_URI: string,
-                   unescaped_URI: string, version: string) &priority=3
-	{
-	if ( ! hook HTTP::sqli_policy(c, method, unescaped_URI) )
-		return;
-
-	if ( match_sql_injection_uri in unescaped_URI )
-		{
-		add c$http$tags[URI_SQLI];
-
-		SumStats::observe("http.sqli.attacker", SumStats::Key($host=c$id$orig_h), SumStats::Observation($str=original_URI));
-		SumStats::observe("http.sqli.victim",   SumStats::Key($host=c$id$resp_h), SumStats::Observation($str=original_URI));
-		}
-	}

scripts/policy/protocols/http/var-extraction-uri.zeek

@@ -13,5 +13,9 @@ redef record Info += {
 event http_request(c: connection, method: string, original_URI: string,
                    unescaped_URI: string, version: string) &priority=2
 	{
-	c$http$uri_vars = extract_keys(original_URI, /&/);
+	local param_parts = split_string1(original_URI, /\?/);
+	if ( |param_parts| > 1 )
+		{
+		c$http$uri_vars = extract_keys(param_parts[1], /&/);
+		}
 	}

scripts/policy/protocols/ssl/log-certs-base64.zeek

@@ -10,6 +10,9 @@ redef record X509::Info += {
 	cert: string &log &optional;
 };
 
+## Certificates can be large and we don't want to risk truncating the output.
+redef X509::default_max_field_string_bytes = 0;
+
 event x509_certificate(f: fa_file, cert_ref: opaque of x509, cert: X509::Certificate) &priority=1
 	{
 	if ( ! f$info?$x509 )

scripts/test-all-policy.zeek

@@ -11,7 +11,6 @@
 
 @load frameworks/analyzer/debug-logging.zeek
 @load frameworks/analyzer/detect-protocols.zeek
-# @load frameworks/analyzer/deprecated-dpd-log.zeek
 @load frameworks/analyzer/packet-segment-logging.zeek
 # @load frameworks/control/controllee.zeek
 # @load frameworks/control/controller.zeek
@@ -50,8 +49,6 @@
 @load frameworks/management/request.zeek
 @load frameworks/management/types.zeek
 @load frameworks/management/util.zeek
-# @load frameworks/dpd/detect-protocols.zeek
-# @load frameworks/dpd/packet-segment-logging.zeek
 @load frameworks/intel/do_notice.zeek
 @load frameworks/intel/do_expire.zeek
 @load frameworks/intel/whitelist.zeek
@@ -126,7 +123,6 @@
 @load protocols/ftp/detect-bruteforcing.zeek
 @load protocols/ftp/detect.zeek
 @load protocols/ftp/software.zeek
-# @load protocols/http/detect-sqli.zeek
 @load protocols/http/detect-sql-injection.zeek
 @load protocols/http/detect-webapps.zeek
 @load protocols/http/header-names.zeek

scripts/zeekygen/__load__.zeek

@@ -1,14 +1,8 @@
 @load test-all-policy.zeek
 
 # Scripts which are commented out in test-all-policy.zeek.
-@load frameworks/analyzer/deprecated-dpd-log.zeek
 @load frameworks/conn_key/vlan_fivetuple.zeek
 
-# Remove in v8.1: replaced by frameworks/analyzer/detect-protocols.zeek
-@pragma push ignore-deprecations
-@load frameworks/dpd/detect-protocols.zeek
-@pragma pop ignore-deprecations
-
 @load protocols/ssl/decryption.zeek
 @ifdef ( Cluster::CLUSTER_BACKEND_ZEROMQ )
 @load frameworks/cluster/backend/zeromq/connect.zeek
@@ -17,11 +11,6 @@
 @load frameworks/control/controllee.zeek
 @load frameworks/control/controller.zeek
 
-# Remove in v8.1: replaced by frameworks/analyzer/packet-segment-logging.zeek
-@pragma push ignore-deprecations
-@load frameworks/dpd/packet-segment-logging.zeek
-@pragma pop ignore-deprecations
-
 @load frameworks/management/agent/main.zeek
 @load frameworks/management/controller/main.zeek
 @load frameworks/management/node/__load__.zeek
@@ -31,9 +20,6 @@
 @load policy/misc/dump-events.zeek
 @load policy/protocols/conn/speculative-service.zeek
 
-# Remove in v8.1: This script is deprecated and conflicts with detect-sql-injection.zeek
-# @load policy/protocols/http/detect-sqli.zeek
-
 @if ( have_spicy() )
 # Loading this messes up documentation of some elements defined elsewhere.
 # @load frameworks/spicy/record-spicy-batch.zeek

src/3rdparty

@@ -1 +0,0 @@
-Subproject commit 22b2618417bfb587aa9183e1d7774d7fb5023ccf

src/3rdparty/ConvertUTF.c

@@ -0,0 +1,771 @@
+/*===--- ConvertUTF.c - Universal Character Names conversions ---------------===
+ *
+ *                     The LLVM Compiler Infrastructure
+ *
+ * This file is distributed under the University of Illinois Open Source
+ * License:
+ *
+ *    University of Illinois/NCSA
+ *    Open Source License
+ *
+ *    Copyright (c) 2003-2014 University of Illinois at Urbana-Champaign.
+ *    All rights reserved.
+ *
+ *    Developed by:
+ *
+ *        LLVM Team
+ *
+ *        University of Illinois at Urbana-Champaign
+ *
+ *        http://llvm.org
+ *
+ *    Permission is hereby granted, free of charge, to any person
+ *    obtaining a copy of this software and associated documentation
+ *    files (the "Software"), to deal with the Software without
+ *    restriction, including without limitation the rights to use,
+ *    copy, modify, merge, publish, distribute, sublicense, and/or
+ *    sell copies of the Software, and to permit persons to whom the
+ *    Software is furnished to do so, subject to the following
+ *    conditions:
+ *
+ *        * Redistributions of source code must retain the above
+ *          copyright notice, this list of conditions and the
+ *          following disclaimers.
+ *
+ *        * Redistributions in binary form must reproduce the
+ *          above copyright notice, this list of conditions and
+ *          the following disclaimers in the documentation and/or
+ *          other materials provided with the distribution.
+ *
+ *        * Neither the names of the LLVM Team, University of
+ *          Illinois at Urbana-Champaign, nor the names of its
+ *          contributors may be used to endorse or promote
+ *          products derived from this Software without specific
+ *          prior written permission.
+ *
+ *    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ *    EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ *    OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ *    NONINFRINGEMENT.  IN NO EVENT SHALL THE CONTRIBUTORS OR
+ *    COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ *    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ *    ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
+ *    USE OR OTHER DEALINGS WITH THE SOFTWARE.
+ *
+ *===------------------------------------------------------------------------=*/
+/*
+ * Copyright © 1991-2015 Unicode, Inc. All rights reserved.
+ * Distributed under the Terms of Use in
+ * http://www.unicode.org/copyright.html.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of the Unicode data files and any associated documentation
+ * (the "Data Files") or Unicode software and any associated documentation
+ * (the "Software") to deal in the Data Files or Software
+ * without restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, and/or sell copies of
+ * the Data Files or Software, and to permit persons to whom the Data Files
+ * or Software are furnished to do so, provided that
+ * (a) this copyright and permission notice appear with all copies
+ * of the Data Files or Software,
+ * (b) this copyright and permission notice appear in associated
+ * documentation, and
+ * (c) there is clear notice in each modified Data File or in the Software
+ * as well as in the documentation associated with the Data File(s) or
+ * Software that the data or software has been modified.
+ *
+ * THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF
+ * ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
+ * WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT OF THIRD PARTY RIGHTS.
+ * IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS INCLUDED IN THIS
+ * NOTICE BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL
+ * DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE,
+ * DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
+ * TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+ * PERFORMANCE OF THE DATA FILES OR SOFTWARE.
+ *
+ * Except as contained in this notice, the name of a copyright holder
+ * shall not be used in advertising or otherwise to promote the sale,
+ * use or other dealings in these Data Files or Software without prior
+ * written authorization of the copyright holder.
+ */
+
+/* ---------------------------------------------------------------------
+
+    Conversions between UTF32, UTF-16, and UTF-8. Source code file.
+    Author: Mark E. Davis, 1994.
+    Rev History: Rick McGowan, fixes & updates May 2001.
+    Sept 2001: fixed const & error conditions per
+        mods suggested by S. Parent & A. Lillich.
+    June 2002: Tim Dodd added detection and handling of incomplete
+        source sequences, enhanced error detection, added casts
+        to eliminate compiler warnings.
+    July 2003: slight mods to back out aggressive FFFE detection.
+    Jan 2004: updated switches in from-UTF8 conversions.
+    Oct 2004: updated to use UNI_MAX_LEGAL_UTF32 in UTF-32 conversions.
+
+    See the header file "ConvertUTF.h" for complete documentation.
+
+------------------------------------------------------------------------ */
+
+
+#include "ConvertUTF.h"
+#ifdef CVTUTF_DEBUG
+#include <stdio.h>
+#endif
+#include <assert.h>
+
+static const int halfShift  = 10; /* used for shifting by 10 bits */
+
+static const UTF32 halfBase = 0x0010000UL;
+static const UTF32 halfMask = 0x3FFUL;
+
+#define UNI_SUR_HIGH_START  (UTF32)0xD800
+#define UNI_SUR_HIGH_END    (UTF32)0xDBFF
+#define UNI_SUR_LOW_START   (UTF32)0xDC00
+#define UNI_SUR_LOW_END     (UTF32)0xDFFF
+#define false      0
+#define true        1
+
+/* --------------------------------------------------------------------- */
+
+/*
+ * Index into the table below with the first byte of a UTF-8 sequence to
+ * get the number of trailing bytes that are supposed to follow it.
+ * Note that *legal* UTF-8 values can't have 4 or 5-bytes. The table is
+ * left as-is for anyone who may want to do such conversion, which was
+ * allowed in earlier algorithms.
+ */
+static const char trailingBytesForUTF8[256] = {
+    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+    1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
+    2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, 3,3,3,3,3,3,3,3,4,4,4,4,5,5,5,5
+};
+
+/*
+ * Magic values subtracted from a buffer value during UTF8 conversion.
+ * This table contains as many values as there might be trailing bytes
+ * in a UTF-8 sequence.
+ */
+static const UTF32 offsetsFromUTF8[6] = { 0x00000000UL, 0x00003080UL, 0x000E2080UL,
+                     0x03C82080UL, 0xFA082080UL, 0x82082080UL };
+
+/*
+ * Once the bits are split out into bytes of UTF-8, this is a mask OR-ed
+ * into the first byte, depending on how many bytes follow.  There are
+ * as many entries in this table as there are UTF-8 sequence types.
+ * (I.e., one byte sequence, two byte... etc.). Remember that sequencs
+ * for *legal* UTF-8 will be 4 or fewer bytes total.
+ */
+static const UTF8 firstByteMark[7] = { 0x00, 0x00, 0xC0, 0xE0, 0xF0, 0xF8, 0xFC };
+
+/* --------------------------------------------------------------------- */
+
+/* The interface converts a whole buffer to avoid function-call overhead.
+ * Constants have been gathered. Loops & conditionals have been removed as
+ * much as possible for efficiency, in favor of drop-through switches.
+ * (See "Note A" at the bottom of the file for equivalent code.)
+ * If your compiler supports it, the "isLegalUTF8" call can be turned
+ * into an inline function.
+ */
+
+
+/* --------------------------------------------------------------------- */
+
+ConversionResult ConvertUTF32toUTF16 (
+        const UTF32** sourceStart, const UTF32* sourceEnd,
+        UTF16** targetStart, UTF16* targetEnd, ConversionFlags flags) {
+    ConversionResult result = conversionOK;
+    const UTF32* source = *sourceStart;
+    UTF16* target = *targetStart;
+    while (source < sourceEnd) {
+        UTF32 ch;
+        if (target >= targetEnd) {
+            result = targetExhausted; break;
+        }
+        ch = *source++;
+        if (ch <= UNI_MAX_BMP) { /* Target is a character <= 0xFFFF */
+            /* UTF-16 surrogate values are illegal in UTF-32; 0xffff or 0xfffe are both reserved values */
+            if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_LOW_END) {
+                if (flags == strictConversion) {
+                    --source; /* return to the illegal value itself */
+                    result = sourceIllegal;
+                    break;
+                } else {
+                    *target++ = UNI_REPLACEMENT_CHAR;
+                }
+            } else {
+                *target++ = (UTF16)ch; /* normal case */
+            }
+        } else if (ch > UNI_MAX_LEGAL_UTF32) {
+            if (flags == strictConversion) {
+                result = sourceIllegal;
+            } else {
+                *target++ = UNI_REPLACEMENT_CHAR;
+            }
+        } else {
+            /* target is a character in range 0xFFFF - 0x10FFFF. */
+            if (target + 1 >= targetEnd) {
+                --source; /* Back up source pointer! */
+                result = targetExhausted; break;
+            }
+            ch -= halfBase;
+            *target++ = (UTF16)((ch >> halfShift) + UNI_SUR_HIGH_START);
+            *target++ = (UTF16)((ch & halfMask) + UNI_SUR_LOW_START);
+        }
+    }
+    *sourceStart = source;
+    *targetStart = target;
+    return result;
+}
+
+/* --------------------------------------------------------------------- */
+
+ConversionResult ConvertUTF16toUTF32 (
+        const UTF16** sourceStart, const UTF16* sourceEnd,
+        UTF32** targetStart, UTF32* targetEnd, ConversionFlags flags) {
+    ConversionResult result = conversionOK;
+    const UTF16* source = *sourceStart;
+    UTF32* target = *targetStart;
+    UTF32 ch, ch2;
+    while (source < sourceEnd) {
+        const UTF16* oldSource = source; /*  In case we have to back up because of target overflow. */
+        ch = *source++;
+        /* If we have a surrogate pair, convert to UTF32 first. */
+        if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_HIGH_END) {
+            /* If the 16 bits following the high surrogate are in the source buffer... */
+            if (source < sourceEnd) {
+                ch2 = *source;
+                /* If it's a low surrogate, convert to UTF32. */
+                if (ch2 >= UNI_SUR_LOW_START && ch2 <= UNI_SUR_LOW_END) {
+                    ch = ((ch - UNI_SUR_HIGH_START) << halfShift)
+                        + (ch2 - UNI_SUR_LOW_START) + halfBase;
+                    ++source;
+                } else if (flags == strictConversion) { /* it's an unpaired high surrogate */
+                    --source; /* return to the illegal value itself */
+                    result = sourceIllegal;
+                    break;
+                }
+            } else { /* We don't have the 16 bits following the high surrogate. */
+                --source; /* return to the high surrogate */
+                result = sourceExhausted;
+                break;
+            }
+        } else if (flags == strictConversion) {
+            /* UTF-16 surrogate values are illegal in UTF-32 */
+            if (ch >= UNI_SUR_LOW_START && ch <= UNI_SUR_LOW_END) {
+                --source; /* return to the illegal value itself */
+                result = sourceIllegal;
+                break;
+            }
+        }
+        if (target >= targetEnd) {
+            source = oldSource; /* Back up source pointer! */
+            result = targetExhausted; break;
+        }
+        *target++ = ch;
+    }
+    *sourceStart = source;
+    *targetStart = target;
+#ifdef CVTUTF_DEBUG
+if (result == sourceIllegal) {
+    fprintf(stderr, "ConvertUTF16toUTF32 illegal seq 0x%04x,%04x\n", ch, ch2);
+    fflush(stderr);
+}
+#endif
+    return result;
+}
+ConversionResult ConvertUTF16toUTF8 (
+        const UTF16** sourceStart, const UTF16* sourceEnd,
+        UTF8** targetStart, UTF8* targetEnd, ConversionFlags flags) {
+    ConversionResult result = conversionOK;
+    const UTF16* source = *sourceStart;
+    UTF8* target = *targetStart;
+    while (source < sourceEnd) {
+        UTF32 ch;
+        unsigned short bytesToWrite = 0;
+        const UTF32 byteMask = 0xBF;
+        const UTF32 byteMark = 0x80;
+        const UTF16* oldSource = source; /* In case we have to back up because of target overflow. */
+        ch = *source++;
+        /* If we have a surrogate pair, convert to UTF32 first. */
+        if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_HIGH_END) {
+            /* If the 16 bits following the high surrogate are in the source buffer... */
+            if (source < sourceEnd) {
+                UTF32 ch2 = *source;
+                /* If it's a low surrogate, convert to UTF32. */
+                if (ch2 >= UNI_SUR_LOW_START && ch2 <= UNI_SUR_LOW_END) {
+                    ch = ((ch - UNI_SUR_HIGH_START) << halfShift)
+                        + (ch2 - UNI_SUR_LOW_START) + halfBase;
+                    ++source;
+                } else if (flags == strictConversion) { /* it's an unpaired high surrogate */
+                    --source; /* return to the illegal value itself */
+                    result = sourceIllegal;
+                    break;
+                }
+            } else { /* We don't have the 16 bits following the high surrogate. */
+                --source; /* return to the high surrogate */
+                result = sourceExhausted;
+                break;
+            }
+        } else if (flags == strictConversion) {
+            /* UTF-16 surrogate values are illegal in UTF-32 */
+            if (ch >= UNI_SUR_LOW_START && ch <= UNI_SUR_LOW_END) {
+                --source; /* return to the illegal value itself */
+                result = sourceIllegal;
+                break;
+            }
+        }
+        /* Figure out how many bytes the result will require */
+        if (ch < (UTF32)0x80) {      bytesToWrite = 1;
+        } else if (ch < (UTF32)0x800) {     bytesToWrite = 2;
+        } else if (ch < (UTF32)0x10000) {   bytesToWrite = 3;
+        } else if (ch < (UTF32)0x110000) {  bytesToWrite = 4;
+        } else {                            bytesToWrite = 3;
+                                            ch = UNI_REPLACEMENT_CHAR;
+        }
+
+        target += bytesToWrite;
+        if (target > targetEnd) {
+            source = oldSource; /* Back up source pointer! */
+            target -= bytesToWrite; result = targetExhausted; break;
+        }
+        switch (bytesToWrite) { /* note: everything falls through. */
+            case 4: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6;
+            case 3: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6;
+            case 2: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6;
+            case 1: *--target =  (UTF8)(ch | firstByteMark[bytesToWrite]);
+        }
+        target += bytesToWrite;
+    }
+    *sourceStart = source;
+    *targetStart = target;
+    return result;
+}
+
+/* --------------------------------------------------------------------- */
+
+ConversionResult ConvertUTF32toUTF8 (
+        const UTF32** sourceStart, const UTF32* sourceEnd,
+        UTF8** targetStart, UTF8* targetEnd, ConversionFlags flags) {
+    ConversionResult result = conversionOK;
+    const UTF32* source = *sourceStart;
+    UTF8* target = *targetStart;
+    while (source < sourceEnd) {
+        UTF32 ch;
+        unsigned short bytesToWrite = 0;
+        const UTF32 byteMask = 0xBF;
+        const UTF32 byteMark = 0x80;
+        ch = *source++;
+        if (flags == strictConversion ) {
+            /* UTF-16 surrogate values are illegal in UTF-32 */
+            if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_LOW_END) {
+                --source; /* return to the illegal value itself */
+                result = sourceIllegal;
+                break;
+            }
+        }
+        /*
+         * Figure out how many bytes the result will require. Turn any
+         * illegally large UTF32 things (> Plane 17) into replacement chars.
+         */
+        if (ch < (UTF32)0x80) {      bytesToWrite = 1;
+        } else if (ch < (UTF32)0x800) {     bytesToWrite = 2;
+        } else if (ch < (UTF32)0x10000) {   bytesToWrite = 3;
+        } else if (ch <= UNI_MAX_LEGAL_UTF32) {  bytesToWrite = 4;
+        } else {                            bytesToWrite = 3;
+                                            ch = UNI_REPLACEMENT_CHAR;
+                                            result = sourceIllegal;
+        }
+
+        target += bytesToWrite;
+        if (target > targetEnd) {
+            --source; /* Back up source pointer! */
+            target -= bytesToWrite; result = targetExhausted; break;
+        }
+        switch (bytesToWrite) { /* note: everything falls through. */
+            case 4: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6;
+            case 3: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6;
+            case 2: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6;
+            case 1: *--target = (UTF8) (ch | firstByteMark[bytesToWrite]);
+        }
+        target += bytesToWrite;
+    }
+    *sourceStart = source;
+    *targetStart = target;
+    return result;
+}
+
+/* --------------------------------------------------------------------- */
+
+/*
+ * Utility routine to tell whether a sequence of bytes is legal UTF-8.
+ * This must be called with the length pre-determined by the first byte.
+ * If not calling this from ConvertUTF8to*, then the length can be set by:
+ *  length = trailingBytesForUTF8[*source]+1;
+ * and the sequence is illegal right away if there aren't that many bytes
+ * available.
+ * If presented with a length > 4, this returns false.  The Unicode
+ * definition of UTF-8 goes up to 4-byte sequences.
+ */
+
+static Boolean isLegalUTF8(const UTF8 *source, int length) {
+    UTF8 a;
+    const UTF8 *srcptr = source+length;
+    switch (length) {
+    default: return false;
+        /* Everything else falls through when "true"... */
+    case 4: if ((a = (*--srcptr)) < 0x80 || a > 0xBF) return false;
+    case 3: if ((a = (*--srcptr)) < 0x80 || a > 0xBF) return false;
+    case 2: if ((a = (*--srcptr)) < 0x80 || a > 0xBF) return false;
+
+        switch (*source) {
+            /* no fall-through in this inner switch */
+            case 0xE0: if (a < 0xA0) return false; break;
+            case 0xED: if (a > 0x9F) return false; break;
+            case 0xF0: if (a < 0x90) return false; break;
+            case 0xF4: if (a > 0x8F) return false; break;
+            default:   if (a < 0x80) return false;
+        }
+
+    case 1: if (*source >= 0x80 && *source < 0xC2) return false;
+    }
+    if (*source > 0xF4) return false;
+    return true;
+}
+
+/* --------------------------------------------------------------------- */
+
+/*
+ * Exported function to return whether a UTF-8 sequence is legal or not.
+ * This is not used here; it's just exported.
+ */
+Boolean isLegalUTF8Sequence(const UTF8 *source, const UTF8 *sourceEnd) {
+    int length = trailingBytesForUTF8[*source]+1;
+    if (length > sourceEnd - source) {
+        return false;
+    }
+    return isLegalUTF8(source, length);
+}
+
+/* --------------------------------------------------------------------- */
+
+static unsigned
+findMaximalSubpartOfIllFormedUTF8Sequence(const UTF8 *source,
+                                          const UTF8 *sourceEnd) {
+  UTF8 b1, b2, b3;
+
+  assert(!isLegalUTF8Sequence(source, sourceEnd));
+
+  /*
+   * Unicode 6.3.0, D93b:
+   *
+   *   Maximal subpart of an ill-formed subsequence: The longest code unit
+   *   subsequence starting at an unconvertible offset that is either:
+   *   a. the initial subsequence of a well-formed code unit sequence, or
+   *   b. a subsequence of length one.
+   */
+
+  if (source == sourceEnd)
+    return 0;
+
+  /*
+   * Perform case analysis.  See Unicode 6.3.0, Table 3-7. Well-Formed UTF-8
+   * Byte Sequences.
+   */
+
+  b1 = *source;
+  ++source;
+  if (b1 >= 0xC2 && b1 <= 0xDF) {
+    /*
+     * First byte is valid, but we know that this code unit sequence is
+     * invalid, so the maximal subpart has to end after the first byte.
+     */
+    return 1;
+  }
+
+  if (source == sourceEnd)
+    return 1;
+
+  b2 = *source;
+  ++source;
+
+  if (b1 == 0xE0) {
+    return (b2 >= 0xA0 && b2 <= 0xBF) ? 2 : 1;
+  }
+  if (b1 >= 0xE1 && b1 <= 0xEC) {
+    return (b2 >= 0x80 && b2 <= 0xBF) ? 2 : 1;
+  }
+  if (b1 == 0xED) {
+    return (b2 >= 0x80 && b2 <= 0x9F) ? 2 : 1;
+  }
+  if (b1 >= 0xEE && b1 <= 0xEF) {
+    return (b2 >= 0x80 && b2 <= 0xBF) ? 2 : 1;
+  }
+  if (b1 == 0xF0) {
+    if (b2 >= 0x90 && b2 <= 0xBF) {
+      if (source == sourceEnd)
+        return 2;
+
+      b3 = *source;
+      return (b3 >= 0x80 && b3 <= 0xBF) ? 3 : 2;
+    }
+    return 1;
+  }
+  if (b1 >= 0xF1 && b1 <= 0xF3) {
+    if (b2 >= 0x80 && b2 <= 0xBF) {
+      if (source == sourceEnd)
+        return 2;
+
+      b3 = *source;
+      return (b3 >= 0x80 && b3 <= 0xBF) ? 3 : 2;
+    }
+    return 1;
+  }
+  if (b1 == 0xF4) {
+    if (b2 >= 0x80 && b2 <= 0x8F) {
+      if (source == sourceEnd)
+        return 2;
+
+      b3 = *source;
+      return (b3 >= 0x80 && b3 <= 0xBF) ? 3 : 2;
+    }
+    return 1;
+  }
+
+  assert((b1 >= 0x80 && b1 <= 0xC1) || b1 >= 0xF5);
+  /*
+   * There are no valid sequences that start with these bytes.  Maximal subpart
+   * is defined to have length 1 in these cases.
+   */
+  return 1;
+}
+
+/* --------------------------------------------------------------------- */
+
+/*
+ * Exported function to return the total number of bytes in a codepoint
+ * represented in UTF-8, given the value of the first byte.
+ */
+unsigned getNumBytesForUTF8(UTF8 first) {
+  return trailingBytesForUTF8[first] + 1;
+}
+
+/* --------------------------------------------------------------------- */
+
+/*
+ * Exported function to return whether a UTF-8 string is legal or not.
+ * This is not used here; it's just exported.
+ */
+Boolean isLegalUTF8String(const UTF8 **source, const UTF8 *sourceEnd) {
+    while (*source != sourceEnd) {
+        int length = trailingBytesForUTF8[**source] + 1;
+        if (length > sourceEnd - *source || !isLegalUTF8(*source, length))
+            return false;
+        *source += length;
+    }
+    return true;
+}
+
+/* --------------------------------------------------------------------- */
+
+ConversionResult ConvertUTF8toUTF16 (
+        const UTF8** sourceStart, const UTF8* sourceEnd,
+        UTF16** targetStart, UTF16* targetEnd, ConversionFlags flags) {
+    ConversionResult result = conversionOK;
+    const UTF8* source = *sourceStart;
+    UTF16* target = *targetStart;
+    while (source < sourceEnd) {
+        UTF32 ch = 0;
+        unsigned short extraBytesToRead = trailingBytesForUTF8[*source];
+        if (extraBytesToRead >= sourceEnd - source) {
+            result = sourceExhausted; break;
+        }
+        /* Do this check whether lenient or strict */
+        if (!isLegalUTF8(source, extraBytesToRead+1)) {
+            result = sourceIllegal;
+            break;
+        }
+        /*
+         * The cases all fall through. See "Note A" below.
+         */
+        switch (extraBytesToRead) {
+            case 5: ch += *source++; ch <<= 6; /* remember, illegal UTF-8 */
+            case 4: ch += *source++; ch <<= 6; /* remember, illegal UTF-8 */
+            case 3: ch += *source++; ch <<= 6;
+            case 2: ch += *source++; ch <<= 6;
+            case 1: ch += *source++; ch <<= 6;
+            case 0: ch += *source++;
+        }
+        ch -= offsetsFromUTF8[extraBytesToRead];
+
+        if (target >= targetEnd) {
+            source -= (extraBytesToRead+1); /* Back up source pointer! */
+            result = targetExhausted; break;
+        }
+        if (ch <= UNI_MAX_BMP) { /* Target is a character <= 0xFFFF */
+            /* UTF-16 surrogate values are illegal in UTF-32 */
+            if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_LOW_END) {
+                if (flags == strictConversion) {
+                    source -= (extraBytesToRead+1); /* return to the illegal value itself */
+                    result = sourceIllegal;
+                    break;
+                } else {
+                    *target++ = UNI_REPLACEMENT_CHAR;
+                }
+            } else {
+                *target++ = (UTF16)ch; /* normal case */
+            }
+        } else if (ch > UNI_MAX_UTF16) {
+            if (flags == strictConversion) {
+                result = sourceIllegal;
+                source -= (extraBytesToRead+1); /* return to the start */
+                break; /* Bail out; shouldn't continue */
+            } else {
+                *target++ = UNI_REPLACEMENT_CHAR;
+            }
+        } else {
+            /* target is a character in range 0xFFFF - 0x10FFFF. */
+            if (target + 1 >= targetEnd) {
+                source -= (extraBytesToRead+1); /* Back up source pointer! */
+                result = targetExhausted; break;
+            }
+            ch -= halfBase;
+            *target++ = (UTF16)((ch >> halfShift) + UNI_SUR_HIGH_START);
+            *target++ = (UTF16)((ch & halfMask) + UNI_SUR_LOW_START);
+        }
+    }
+    *sourceStart = source;
+    *targetStart = target;
+    return result;
+}
+
+/* --------------------------------------------------------------------- */
+
+static ConversionResult ConvertUTF8toUTF32Impl(
+        const UTF8** sourceStart, const UTF8* sourceEnd,
+        UTF32** targetStart, UTF32* targetEnd, ConversionFlags flags,
+        Boolean InputIsPartial) {
+    ConversionResult result = conversionOK;
+    const UTF8* source = *sourceStart;
+    UTF32* target = *targetStart;
+    while (source < sourceEnd) {
+        UTF32 ch = 0;
+        unsigned short extraBytesToRead = trailingBytesForUTF8[*source];
+        if (extraBytesToRead >= sourceEnd - source) {
+            if (flags == strictConversion || InputIsPartial) {
+                result = sourceExhausted;
+                break;
+            } else {
+                result = sourceIllegal;
+
+                /*
+                 * Replace the maximal subpart of ill-formed sequence with
+                 * replacement character.
+                 */
+                source += findMaximalSubpartOfIllFormedUTF8Sequence(source,
+                                                                    sourceEnd);
+                *target++ = UNI_REPLACEMENT_CHAR;
+                continue;
+            }
+        }
+        if (target >= targetEnd) {
+            result = targetExhausted; break;
+        }
+
+        /* Do this check whether lenient or strict */
+        if (!isLegalUTF8(source, extraBytesToRead+1)) {
+            result = sourceIllegal;
+            if (flags == strictConversion) {
+                /* Abort conversion. */
+                break;
+            } else {
+                /*
+                 * Replace the maximal subpart of ill-formed sequence with
+                 * replacement character.
+                 */
+                source += findMaximalSubpartOfIllFormedUTF8Sequence(source,
+                                                                    sourceEnd);
+                *target++ = UNI_REPLACEMENT_CHAR;
+                continue;
+            }
+        }
+        /*
+         * The cases all fall through. See "Note A" below.
+         */
+        switch (extraBytesToRead) {
+            case 5: ch += *source++; ch <<= 6;
+            case 4: ch += *source++; ch <<= 6;
+            case 3: ch += *source++; ch <<= 6;
+            case 2: ch += *source++; ch <<= 6;
+            case 1: ch += *source++; ch <<= 6;
+            case 0: ch += *source++;
+        }
+        ch -= offsetsFromUTF8[extraBytesToRead];
+
+        if (ch <= UNI_MAX_LEGAL_UTF32) {
+            /*
+             * UTF-16 surrogate values are illegal in UTF-32, and anything
+             * over Plane 17 (> 0x10FFFF) is illegal.
+             */
+            if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_LOW_END) {
+                if (flags == strictConversion) {
+                    source -= (extraBytesToRead+1); /* return to the illegal value itself */
+                    result = sourceIllegal;
+                    break;
+                } else {
+                    *target++ = UNI_REPLACEMENT_CHAR;
+                }
+            } else {
+                *target++ = ch;
+            }
+        } else { /* i.e., ch > UNI_MAX_LEGAL_UTF32 */
+            result = sourceIllegal;
+            *target++ = UNI_REPLACEMENT_CHAR;
+        }
+    }
+    *sourceStart = source;
+    *targetStart = target;
+    return result;
+}
+
+ConversionResult ConvertUTF8toUTF32Partial(const UTF8 **sourceStart,
+                                           const UTF8 *sourceEnd,
+                                           UTF32 **targetStart,
+                                           UTF32 *targetEnd,
+                                           ConversionFlags flags) {
+  return ConvertUTF8toUTF32Impl(sourceStart, sourceEnd, targetStart, targetEnd,
+                                flags, /*InputIsPartial=*/true);
+}
+
+ConversionResult ConvertUTF8toUTF32(const UTF8 **sourceStart,
+                                    const UTF8 *sourceEnd, UTF32 **targetStart,
+                                    UTF32 *targetEnd, ConversionFlags flags) {
+  return ConvertUTF8toUTF32Impl(sourceStart, sourceEnd, targetStart, targetEnd,
+                                flags, /*InputIsPartial=*/false);
+}
+
+/* ---------------------------------------------------------------------
+
+    Note A.
+    The fall-through switches in UTF-8 reading code save a
+    temp variable, some decrements & conditionals.  The switches
+    are equivalent to the following loop:
+        {
+            int tmpBytesToRead = extraBytesToRead+1;
+            do {
+                ch += *source++;
+                --tmpBytesToRead;
+                if (tmpBytesToRead) ch <<= 6;
+            } while (tmpBytesToRead > 0);
+        }
+    In UTF-8 writing code, the switches on "bytesToWrite" are
+    similarly unrolled loops.
+
+   --------------------------------------------------------------------- */

src/3rdparty/ConvertUTF.h

@@ -0,0 +1,249 @@
+/*===--- ConvertUTF.h - Universal Character Names conversions ---------------===
+ *
+ *                     The LLVM Compiler Infrastructure
+ *
+ * This file is distributed under the University of Illinois Open Source
+ * License:
+ *
+ *    University of Illinois/NCSA
+ *    Open Source License
+ *
+ *    Copyright (c) 2003-2014 University of Illinois at Urbana-Champaign.
+ *    All rights reserved.
+ *
+ *    Developed by:
+ *
+ *        LLVM Team
+ *
+ *        University of Illinois at Urbana-Champaign
+ *
+ *        http://llvm.org
+ *
+ *    Permission is hereby granted, free of charge, to any person
+ *    obtaining a copy of this software and associated documentation
+ *    files (the "Software"), to deal with the Software without
+ *    restriction, including without limitation the rights to use,
+ *    copy, modify, merge, publish, distribute, sublicense, and/or
+ *    sell copies of the Software, and to permit persons to whom the
+ *    Software is furnished to do so, subject to the following
+ *    conditions:
+ *
+ *        * Redistributions of source code must retain the above
+ *          copyright notice, this list of conditions and the
+ *          following disclaimers.
+ *
+ *        * Redistributions in binary form must reproduce the
+ *          above copyright notice, this list of conditions and
+ *          the following disclaimers in the documentation and/or
+ *          other materials provided with the distribution.
+ *
+ *        * Neither the names of the LLVM Team, University of
+ *          Illinois at Urbana-Champaign, nor the names of its
+ *          contributors may be used to endorse or promote
+ *          products derived from this Software without specific
+ *          prior written permission.
+ *
+ *    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ *    EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ *    OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ *    NONINFRINGEMENT.  IN NO EVENT SHALL THE CONTRIBUTORS OR
+ *    COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ *    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ *    ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
+ *    USE OR OTHER DEALINGS WITH THE SOFTWARE.
+ *
+ *==------------------------------------------------------------------------==*/
+/*
+ * Copyright © 1991-2015 Unicode, Inc. All rights reserved.
+ * Distributed under the Terms of Use in
+ * http://www.unicode.org/copyright.html.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of the Unicode data files and any associated documentation
+ * (the "Data Files") or Unicode software and any associated documentation
+ * (the "Software") to deal in the Data Files or Software
+ * without restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, and/or sell copies of
+ * the Data Files or Software, and to permit persons to whom the Data Files
+ * or Software are furnished to do so, provided that
+ * (a) this copyright and permission notice appear with all copies
+ * of the Data Files or Software,
+ * (b) this copyright and permission notice appear in associated
+ * documentation, and
+ * (c) there is clear notice in each modified Data File or in the Software
+ * as well as in the documentation associated with the Data File(s) or
+ * Software that the data or software has been modified.
+ *
+ * THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF
+ * ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
+ * WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT OF THIRD PARTY RIGHTS.
+ * IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS INCLUDED IN THIS
+ * NOTICE BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL
+ * DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE,
+ * DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
+ * TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+ * PERFORMANCE OF THE DATA FILES OR SOFTWARE.
+ *
+ * Except as contained in this notice, the name of a copyright holder
+ * shall not be used in advertising or otherwise to promote the sale,
+ * use or other dealings in these Data Files or Software without prior
+ * written authorization of the copyright holder.
+ */
+
+/* ---------------------------------------------------------------------
+
+    Conversions between UTF32, UTF-16, and UTF-8.  Header file.
+
+    Several funtions are included here, forming a complete set of
+    conversions between the three formats.  UTF-7 is not included
+    here, but is handled in a separate source file.
+
+    Each of these routines takes pointers to input buffers and output
+    buffers.  The input buffers are const.
+
+    Each routine converts the text between *sourceStart and sourceEnd,
+    putting the result into the buffer between *targetStart and
+    targetEnd. Note: the end pointers are *after* the last item: e.g.
+    *(sourceEnd - 1) is the last item.
+
+    !!! NOTE: The source and end pointers must be aligned properly !!!
+
+    The return result indicates whether the conversion was successful,
+    and if not, whether the problem was in the source or target buffers.
+    (Only the first encountered problem is indicated.)
+
+    After the conversion, *sourceStart and *targetStart are both
+    updated to point to the end of last text successfully converted in
+    the respective buffers.
+
+    Input parameters:
+        sourceStart - pointer to a pointer to the source buffer.
+                The contents of this are modified on return so that
+                it points at the next thing to be converted.
+        targetStart - similarly, pointer to pointer to the target buffer.
+        sourceEnd, targetEnd - respectively pointers to the ends of the
+                two buffers, for overflow checking only.
+
+    These conversion functions take a ConversionFlags argument. When this
+    flag is set to strict, both irregular sequences and isolated surrogates
+    will cause an error.  When the flag is set to lenient, both irregular
+    sequences and isolated surrogates are converted.
+
+    Whether the flag is strict or lenient, all illegal sequences will cause
+    an error return. This includes sequences such as: <F4 90 80 80>, <C0 80>,
+    or <A0> in UTF-8, and values above 0x10FFFF in UTF-32. Conformant code
+    must check for illegal sequences.
+
+    When the flag is set to lenient, characters over 0x10FFFF are converted
+    to the replacement character; otherwise (when the flag is set to strict)
+    they constitute an error.
+
+    Output parameters:
+        The value "sourceIllegal" is returned from some routines if the input
+        sequence is malformed.  When "sourceIllegal" is returned, the source
+        value will point to the illegal value that caused the problem. E.g.,
+        in UTF-8 when a sequence is malformed, it points to the start of the
+        malformed sequence.
+
+    Author: Mark E. Davis, 1994.
+    Rev History: Rick McGowan, fixes & updates May 2001.
+         Fixes & updates, Sept 2001.
+
+------------------------------------------------------------------------ */
+
+#pragma once
+
+/* ---------------------------------------------------------------------
+    The following 4 definitions are compiler-specific.
+    The C standard does not guarantee that wchar_t has at least
+    16 bits, so wchar_t is no less portable than unsigned short!
+    All should be unsigned values to avoid sign extension during
+    bit mask & shift operations.
+------------------------------------------------------------------------ */
+
+typedef unsigned int    UTF32;  /* at least 32 bits */
+typedef unsigned short  UTF16;  /* at least 16 bits */
+typedef unsigned char   UTF8;   /* typically 8 bits */
+typedef unsigned char   Boolean; /* 0 or 1 */
+
+/* Some fundamental constants */
+#define UNI_REPLACEMENT_CHAR (UTF32)0x0000FFFD
+#define UNI_MAX_BMP (UTF32)0x0000FFFF
+#define UNI_MAX_UTF16 (UTF32)0x0010FFFF
+#define UNI_MAX_UTF32 (UTF32)0x7FFFFFFF
+#define UNI_MAX_LEGAL_UTF32 (UTF32)0x0010FFFF
+
+#define UNI_MAX_UTF8_BYTES_PER_CODE_POINT 4
+
+#define UNI_UTF16_BYTE_ORDER_MARK_NATIVE  0xFEFF
+#define UNI_UTF16_BYTE_ORDER_MARK_SWAPPED 0xFFFE
+
+typedef enum {
+  conversionOK,           /* conversion successful */
+  sourceExhausted,        /* partial character in source, but hit end */
+  targetExhausted,        /* insuff. room in target for conversion */
+  sourceIllegal           /* source sequence is illegal/malformed */
+} ConversionResult;
+
+typedef enum {
+  strictConversion = 0,
+  lenientConversion
+} ConversionFlags;
+
+/* This is for C++ and does no harm in C */
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+ConversionResult ConvertUTF8toUTF16 (
+  const UTF8** sourceStart, const UTF8* sourceEnd,
+  UTF16** targetStart, UTF16* targetEnd, ConversionFlags flags);
+
+/**
+ * Convert a partial UTF8 sequence to UTF32.  If the sequence ends in an
+ * incomplete code unit sequence, returns \c sourceExhausted.
+ */
+ConversionResult ConvertUTF8toUTF32Partial(
+  const UTF8** sourceStart, const UTF8* sourceEnd,
+  UTF32** targetStart, UTF32* targetEnd, ConversionFlags flags);
+
+/**
+ * Convert a partial UTF8 sequence to UTF32.  If the sequence ends in an
+ * incomplete code unit sequence, returns \c sourceIllegal.
+ */
+ConversionResult ConvertUTF8toUTF32(
+  const UTF8** sourceStart, const UTF8* sourceEnd,
+  UTF32** targetStart, UTF32* targetEnd, ConversionFlags flags);
+
+/* NOTE: The source and end pointers must be aligned properly. */
+ConversionResult ConvertUTF16toUTF8 (
+  const UTF16** sourceStart, const UTF16* sourceEnd,
+  UTF8** targetStart, UTF8* targetEnd, ConversionFlags flags);
+
+/* NOTE: The source and end pointers must be aligned properly. */
+ConversionResult ConvertUTF32toUTF8 (
+  const UTF32** sourceStart, const UTF32* sourceEnd,
+  UTF8** targetStart, UTF8* targetEnd, ConversionFlags flags);
+
+/* NOTE: The source and end pointers must be aligned properly. */
+ConversionResult ConvertUTF16toUTF32 (
+  const UTF16** sourceStart, const UTF16* sourceEnd,
+  UTF32** targetStart, UTF32* targetEnd, ConversionFlags flags);
+
+/* NOTE: The source and end pointers must be aligned properly. */
+ConversionResult ConvertUTF32toUTF16 (
+  const UTF32** sourceStart, const UTF32* sourceEnd,
+  UTF16** targetStart, UTF16* targetEnd, ConversionFlags flags);
+
+Boolean isLegalUTF8Sequence(const UTF8 *source, const UTF8 *sourceEnd);
+
+Boolean isLegalUTF8String(const UTF8 **source, const UTF8 *sourceEnd);
+
+unsigned getNumBytesForUTF8(UTF8 firstByte);
+
+#ifdef __cplusplus
+}
+#endif
+
+/* --------------------------------------------------------------------- */

src/3rdparty/bsd-getopt-long.c

@@ -0,0 +1,524 @@
+/*    $OpenBSD: getopt_long.c,v 1.17 2004/06/03 18:46:52 millert Exp $    */
+/*    $NetBSD: getopt_long.c,v 1.15 2002/01/31 22:43:40 tv Exp $    */
+
+/*
+ * Copyright (c) 2002 Todd C. Miller <Todd.Miller@courtesan.com>
+ *
+ * Permission to use, copy, modify, and distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND TODD C. MILLER DISCLAIMS ALL
+ * WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL TODD C. MILLER BE LIABLE
+ * FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
+ * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+ * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+/*-
+ * Copyright (c) 2000 The NetBSD Foundation, Inc.
+ * All rights reserved.
+ *
+ * This code is derived from software contributed to The NetBSD Foundation
+ * by Dieter Baron and Thomas Klausner.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. All advertising materials mentioning features or use of this software
+ *    must display the following acknowledgement:
+ *        This product includes software developed by the NetBSD
+ *        Foundation, Inc. and its contributors.
+ * 4. Neither the name of The NetBSD Foundation nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#define IN_GETOPT_LONG_C 1
+
+#include <zeek/zeek-config.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <stdio.h>
+#include <string.h>
+
+#ifndef HAVE_GETOPT_LONG
+
+# include "bsd-getopt-long.h"
+
+# ifdef WITH_DMALLOC
+#  include <dmalloc.h>
+# endif
+
+int    pure_opterr = 1;        /* if error message should be printed */
+int    pure_optind = 1;        /* index into parent argv vector */
+int    pure_optopt = '?';        /* character checked for validity */
+int    pure_optreset;        /* reset getopt */
+const char    *pure_optarg;        /* argument associated with option */
+
+# define PRINT_ERROR    ((pure_opterr) && (*options != ':'))
+
+# define FLAG_PERMUTE    0x01    /* permute non-options to the end of argv */
+# define FLAG_ALLARGS    0x02    /* treat non-options as args to option "-1" */
+# define FLAG_LONGONLY   0x04    /* operate as pure_getopt_long_only */
+
+/* return values */
+# define    BADCH       (int)'?'
+# define    BADARG      ((*options == ':') ? (int)':' : (int)'?')
+# define    INORDER     (int)1
+
+# define    EMSG        ""
+
+static int pure_getopt_internal(int, char * const *, const char *,
+                                const struct pure_option *, int *, int);
+static int pure_parse_long_options(char * const *, const char *,
+                                   const struct pure_option *, int *, int);
+static int pure_gcd(int, int);
+static void pure_permute_args(int, int, int, char * const *);
+
+static const char *pure_place = EMSG; /* option letter processing */
+
+/* XXX: set pure_optreset to 1 rather than these two */
+static int nonopt_start = -1; /* first non option argument (for permute) */
+static int nonopt_end = -1;   /* first option after non options (for permute) */
+
+/* Error messages */
+static const char *recargchar = "option requires an argument -- %c\n";
+static const char *recargstring = "option requires an argument -- %s\n";
+static const char *ambig = "ambiguous option -- %.*s\n";
+static const char *noarg = "option doesn't take an argument -- %.*s\n";
+static const char *illoptchar = "unknown option -- %c\n";
+static const char *illoptstring = "unknown option -- %s\n";
+
+/*
+ * Compute the greatest common divisor of a and b.
+ */
+static int pure_gcd(int a, int b)
+{
+    int c;
+    
+    c = a % b;
+    while (c != 0) {
+        a = b;
+        b = c;
+        c = a % b;
+    }    
+    return b;
+}
+
+/*
+ * Exchange the block from nonopt_start to nonopt_end with the block
+ * from nonopt_end to opt_end (keeping the same order of arguments
+ * in each block).
+ */
+static void pure_permute_args(int panonopt_start, int panonopt_end, 
+                              int opt_end, char * const *nargv)
+{
+    int cstart, cyclelen, i, j, ncycle, nnonopts, nopts, pos;
+    char *swap;
+    
+    /*
+     * compute lengths of blocks and number and size of cycles
+     */
+    nnonopts = panonopt_end - panonopt_start;
+    nopts = opt_end - panonopt_end;
+    ncycle = pure_gcd(nnonopts, nopts);
+    cyclelen = (opt_end - panonopt_start) / ncycle;
+    
+    for (i = 0; i < ncycle; i++) {
+        cstart = panonopt_end+i;
+        pos = cstart;
+        for (j = 0; j < cyclelen; j++) {
+            if (pos >= panonopt_end)
+                pos -= nnonopts;
+            else
+                pos += nopts;
+            swap = nargv[pos];
+            /* LINTED const cast */
+            ((char **) nargv)[pos] = nargv[cstart];
+            /* LINTED const cast */
+            ((char **)nargv)[cstart] = swap;
+        }
+    }
+}
+
+/*
+ * pure_parse_long_options --
+ *    Parse long options in argc/argv argument vector.
+ * Returns -1 if short_too is set and the option does not match long_options.
+ */
+static int pure_parse_long_options(char * const *nargv, const char *options,
+                                   const struct pure_option *long_options,
+                                   int *idx, int short_too)
+{
+    const char *current_argv, *has_equal;
+    size_t current_argv_len;
+    int i, match;
+    
+    current_argv = pure_place;
+    match = -1;
+    
+    pure_optind++;
+    
+    if ((has_equal = strchr(current_argv, '=')) != NULL) {
+        /* argument found (--option=arg) */
+        current_argv_len = has_equal - current_argv;
+        has_equal++;
+    } else
+        current_argv_len = strlen(current_argv);
+    
+    for (i = 0; long_options[i].name; i++) {
+        /* find matching long option */
+        if (strncmp(current_argv, long_options[i].name,
+            current_argv_len))
+            continue;
+        
+        if (strlen(long_options[i].name) == current_argv_len) {
+            /* exact match */
+            match = i;
+            break;
+        }
+        /*
+         * If this is a known short option, don't allow
+         * a partial match of a single character.
+         */
+        if (short_too && current_argv_len == 1)
+            continue;
+        
+        if (match == -1)    /* partial match */
+            match = i;
+        else {
+            /* ambiguous abbreviation */
+            if (PRINT_ERROR)
+                fprintf(stderr, ambig, (int)current_argv_len,
+                        current_argv);
+            pure_optopt = 0;
+            return BADCH;
+        }
+    }
+    if (match != -1) {        /* option found */
+        if (long_options[match].has_arg == no_argument
+            && has_equal) {
+            if (PRINT_ERROR)
+                fprintf(stderr, noarg, (int)current_argv_len,
+                        current_argv);
+            /*
+             * XXX: GNU sets pure_optopt to val regardless of flag
+             */
+            if (long_options[match].flag == NULL)
+                pure_optopt = long_options[match].val;
+            else
+                pure_optopt = 0;
+            return BADARG;
+        }
+        if (long_options[match].has_arg == required_argument ||
+            long_options[match].has_arg == optional_argument) {
+            if (has_equal)
+                pure_optarg = has_equal;
+            else if (long_options[match].has_arg ==
+                     required_argument) {
+                /*
+                 * optional argument doesn't use next nargv
+                 */
+                pure_optarg = nargv[pure_optind++];
+            }
+        }
+        if ((long_options[match].has_arg == required_argument)
+            && (pure_optarg == NULL)) {
+            /*
+             * Missing argument; leading ':' indicates no error
+             * should be generated.
+             */
+            if (PRINT_ERROR)
+                fprintf(stderr, recargstring,
+                        current_argv);
+            /*
+             * XXX: GNU sets pure_optopt to val regardless of flag
+             */
+            if (long_options[match].flag == NULL)
+                pure_optopt = long_options[match].val;
+            else
+                pure_optopt = 0;
+            --pure_optind;
+            return BADARG;
+        }
+    } else {            /* unknown option */
+        if (short_too) {
+            --pure_optind;
+            return -1;
+        }
+        if (PRINT_ERROR)
+            fprintf(stderr, illoptstring, current_argv);
+        pure_optopt = 0;
+        return BADCH;
+    }
+    if (idx)
+        *idx = match;
+    if (long_options[match].flag) {
+        *long_options[match].flag = long_options[match].val;
+        return 0;
+    } else
+        return long_options[match].val;
+}
+
+/*
+ * getopt_internal --
+ *    Parse argc/argv argument vector.  Called by user level routines.
+ */
+static int pure_getopt_internal(int nargc, char * const *nargv,
+                                const char *options,
+                                const struct pure_option *long_options,
+                                int *idx, int flags)
+{
+    char *oli;                /* option letter list index */
+    int optchar, short_too;
+    static int posixly_correct = -1;
+    
+    if (options == NULL)
+        return -1;
+    
+    /*
+     * Disable GNU extensions if POSIXLY_CORRECT is set or options
+     * string begins with a '+'.
+     */
+    if (posixly_correct == -1)
+        posixly_correct = (getenv("POSIXLY_CORRECT") != NULL);
+    if (posixly_correct || *options == '+')
+        flags &= ~FLAG_PERMUTE;
+    else if (*options == '-')
+        flags |= FLAG_ALLARGS;
+    if (*options == '+' || *options == '-')
+        options++;
+    
+    /*
+     * XXX Some GNU programs (like cvs) set pure_optind to 0 instead of
+     * XXX using pure_optreset.  Work around this braindamage.
+     */
+    if (pure_optind == 0)
+        pure_optind = pure_optreset = 1;
+    
+    pure_optarg = NULL;
+    if (pure_optreset)
+        nonopt_start = nonopt_end = -1;
+    start:
+    if (pure_optreset || !*pure_place) {        /* update scanning pointer */
+        pure_optreset = 0;
+        if (pure_optind >= nargc) {          /* end of argument vector */
+            pure_place = EMSG;
+            if (nonopt_end != -1) {
+                /* do permutation, if we have to */
+                pure_permute_args(nonopt_start, nonopt_end,
+                                  pure_optind, nargv);
+                pure_optind -= nonopt_end - nonopt_start;
+            }
+            else if (nonopt_start != -1) {
+                /*
+                 * If we skipped non-options, set pure_optind
+                 * to the first of them.
+                 */
+                pure_optind = nonopt_start;
+            }
+            nonopt_start = nonopt_end = -1;
+            return -1;
+        }
+        if (*(pure_place = nargv[pure_optind]) != '-' ||
+            (pure_place[1] == '\0' && strchr(options, '-') == NULL)) {
+            pure_place = EMSG;        /* found non-option */
+            if (flags & FLAG_ALLARGS) {
+                /*
+                 * GNU extension:
+                 * return non-option as argument to option 1
+                 */
+                pure_optarg = nargv[pure_optind++];
+                return INORDER;
+            }
+            if (!(flags & FLAG_PERMUTE)) {
+                /*
+                 * If no permutation wanted, stop parsing
+                 * at first non-option.
+                 */
+                return -1;
+            }
+            /* do permutation */
+            if (nonopt_start == -1)
+                nonopt_start = pure_optind;
+            else if (nonopt_end != -1) {
+                pure_permute_args(nonopt_start, nonopt_end,
+                                  pure_optind, nargv);
+                nonopt_start = pure_optind -
+                    (nonopt_end - nonopt_start);
+                nonopt_end = -1;
+            }
+            pure_optind++;
+            /* process next argument */
+            goto start;
+        }
+        if (nonopt_start != -1 && nonopt_end == -1)
+            nonopt_end = pure_optind;
+
+        /*
+         * Check for "--" or "--foo" with no long options
+         * but if pure_place is simply "-" leave it unmolested.
+         */
+
+        if (pure_place[1] != '\0' && *++pure_place == '-' &&
+            (pure_place[1] == '\0' || long_options == NULL)) {
+            pure_optind++;
+            pure_place = EMSG;
+            /*
+             * We found an option (--), so if we skipped
+             * non-options, we have to permute.
+             */
+            if (nonopt_end != -1) {
+                pure_permute_args(nonopt_start, nonopt_end,
+                                  pure_optind, nargv);
+                pure_optind -= nonopt_end - nonopt_start;
+            }
+            nonopt_start = nonopt_end = -1;
+            return -1;
+        }
+    }
+    
+    /*
+     * Check long options if:
+     *  1) we were passed some
+     *  2) the arg is not just "-"
+     *  3) either the arg starts with -- we are pure_getopt_long_only()
+     */
+    if (long_options != NULL && pure_place != nargv[pure_optind] &&
+        (*pure_place == '-' || (flags & FLAG_LONGONLY))) {
+        short_too = 0;
+        if (*pure_place == '-')
+            pure_place++;        /* --foo long option */
+        else if (*pure_place != ':' && strchr(options, *pure_place) != NULL)
+            short_too = 1;        /* could be short option too */
+        
+        optchar = pure_parse_long_options(nargv, options, long_options,
+                                          idx, short_too);
+        if (optchar != -1) {
+            pure_place = EMSG;
+            return optchar;
+        }
+    }
+    
+    if ((optchar = (int) *pure_place++) == ':' ||
+        (optchar == '-' && *pure_place != '\0') ||        
+        (oli = strchr(options, optchar)) == NULL) { 
+         /*
+          * If the user specified "-" and '-' isn't listed in
+          * options, return -1 (non-option) as per POSIX.
+          * Otherwise, it is an unknown option character (or :').
+          */
+        if (optchar == '-' && *pure_place == '\0')
+            return -1;
+        if (!*pure_place)
+            ++pure_optind;
+        if (PRINT_ERROR)
+            fprintf(stderr, illoptchar, optchar);
+        pure_optopt = optchar;
+        return BADCH;
+    }
+    if (long_options != NULL && optchar == 'W' && oli[1] == ';') {
+        /* -W long-option */
+        if (*pure_place)            /* no space */
+            /* NOTHING */;
+        else if (++pure_optind >= nargc) {    /* no arg */
+            pure_place = EMSG;
+            if (PRINT_ERROR)
+                fprintf(stderr, recargchar, optchar);
+            pure_optopt = optchar;
+            return BADARG;
+        } else                /* white space */
+            pure_place = nargv[pure_optind];
+        optchar = pure_parse_long_options(nargv, options, long_options,
+                                          idx, 0);
+        pure_place = EMSG;
+        return optchar;
+    }
+    if (*++oli != ':') {            /* doesn't take argument */
+        if (!*pure_place)
+            ++pure_optind;
+    } else {                /* takes (optional) argument */
+        pure_optarg = NULL;
+        if (*pure_place)            /* no white space */
+            pure_optarg = pure_place;
+        /* XXX: disable test for :: if PC? (GNU doesn't) */
+        else if (oli[1] != ':') {    /* arg not optional */
+            if (++pure_optind >= nargc) {    /* no arg */
+                pure_place = EMSG;
+                if (PRINT_ERROR)
+                    fprintf(stderr, recargchar, optchar);
+                pure_optopt = optchar;
+                return BADARG;
+            } else {
+                pure_optarg = nargv[pure_optind];
+            }
+        }
+        pure_place = EMSG;
+        ++pure_optind;
+    }
+    /* dump back option letter */
+    return optchar;
+}
+
+/*
+ * getopt --
+ *    Parse argc/argv argument vector.
+ */
+int pure_getopt(int nargc, char * const *nargv, const char *options)
+{
+    
+    /*
+     * We dont' pass FLAG_PERMUTE to pure_getopt_internal() since
+     * the BSD getopt(3) (unlike GNU) has never done this.
+     *
+     * Furthermore, since many privileged programs call getopt()
+     * before dropping privileges it makes sense to keep things
+     * as simple (and bug-free) as possible.
+     */
+    return pure_getopt_internal(nargc, nargv, options, NULL, NULL, 0);
+}
+
+/*
+ * pure_getopt_long --
+ *    Parse argc/argv argument vector.
+ */
+int pure_getopt_long(int nargc, char * const *nargv, const char *options,
+                     const struct pure_option *long_options, int *idx)
+{
+    return pure_getopt_internal(nargc, nargv, options, long_options, idx,
+                                FLAG_PERMUTE);
+}
+
+/*
+ * pure_getopt_long_only --
+ *    Parse argc/argv argument vector.
+ */
+int pure_getopt_long_only(int nargc, char * const *nargv,
+                               const char *options,
+                               const struct pure_option *long_options,
+                               int *idx)
+{
+    return pure_getopt_internal(nargc, nargv, options, long_options, idx,
+                                FLAG_PERMUTE|FLAG_LONGONLY);
+}
+
+#endif

src/3rdparty/bsd-getopt-long.h

@@ -0,0 +1,133 @@
+/*    $OpenBSD: getopt_long.c,v 1.13 2003/06/03 01:52:40 millert Exp $    */
+/*    $NetBSD: getopt_long.c,v 1.15 2002/01/31 22:43:40 tv Exp $    */
+
+/*
+ * Copyright (c) 2002 Todd C. Miller <Todd.Miller@courtesan.com>
+ *
+ * Permission to use, copy, modify, and distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND TODD C. MILLER DISCLAIMS ALL
+ * WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL TODD C. MILLER BE LIABLE
+ * FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
+ * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+ * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+/*-
+ * Copyright (c) 2000 The NetBSD Foundation, Inc.
+ * All rights reserved.
+ *
+ * This code is derived from software contributed to The NetBSD Foundation
+ * by Dieter Baron and Thomas Klausner.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. All advertising materials mentioning features or use of this software
+ *    must display the following acknowledgement:
+ *        This product includes software developed by the NetBSD
+ *        Foundation, Inc. and its contributors.
+ * 4. Neither the name of The NetBSD Foundation nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#pragma once
+
+#ifndef HAVE_GETOPT_LONG
+
+/*
+ * GNU-like getopt_long() and 4.4BSD getsubopt()/optreset extensions
+ */
+# ifndef no_argument
+#  define no_argument        0
+# endif
+# ifndef required_argument
+#  define required_argument  1
+# endif
+# ifndef optional_argument
+#  define optional_argument  2
+# endif
+
+struct pure_option {
+    /* name of long option */
+    const char *name;
+    /*
+     * one of no_argument, required_argument, and optional_argument:
+     * whether option takes an argument
+     */
+    int has_arg;
+    /* if not NULL, set *flag to val when option found */
+    int *flag;
+    /* if flag not NULL, value to set *flag to; else return value */
+    int val;
+};
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+int pure_getopt_long(int nargc, char * const *nargv, const char *options,
+                     const struct pure_option *long_options, int *idx);
+
+int pure_getopt_long_only(int nargc, char * const *nargv,
+                          const char *options,
+                          const struct pure_option *long_options,
+                          int *idx);
+
+int pure_getopt(int nargc, char * const *nargv, const char *options);
+
+extern const char *pure_optarg;            /* getopt(3) external variables */
+extern int pure_opterr;
+extern int pure_optind;
+extern int pure_optopt;
+extern int pure_optreset;
+
+#ifdef __cplusplus
+}
+#endif
+
+/* prefix+macros just to avoid clashes with existing getopt() implementations */
+
+# ifndef IN_GETOPT_LONG_C
+#  undef option
+#  define option pure_option
+#  undef getopt_long
+#  define getopt_long(A, B, C, D, E) pure_getopt_long(A, B, C, D, E)
+#  undef getopt_long_only
+#  define getopt_long_only(A, B, C, D, E) pure_getopt_long_only(A, B, C, D, E)
+#  undef getopt
+#  define getopt(A, B, C) pure_getopt(A, B, C)
+#  undef optarg
+#  define optarg pure_optarg
+#  undef opterr
+#  define opterr pure_opterr
+#  undef optind
+#  define optind pure_optind
+#  undef optopt
+#  define optopt pure_optopt
+#  undef optreset
+#  define optreset pure_optreset
+# endif
+
+#endif

src/3rdparty/in_cksum.cc

@@ -0,0 +1,141 @@
+// Modified from tcpdump v4.9.3's in_cksum.c (which itself was a modified
+// version of FreeBSD's in_cksum.c).
+
+/* in_cksum.c
+ * 4.4-Lite-2 Internet checksum routine, modified to take a vector of
+ * pointers/lengths giving the pieces to be checksummed.  Also using
+ * Tahoe/CGI version of ADDCARRY(x) macro instead of from portable version.
+ */
+
+/*
+ * Copyright (c) 1988, 1992, 1993
+ *	The Regents of the University of California.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. Neither the name of the University nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ *	@(#)in_cksum.c	8.1 (Berkeley) 6/10/93
+ */
+
+#include "zeek/net_util.h"
+
+namespace zeek::detail {
+
+#define ADDCARRY(x)  {if ((x) > 65535) (x) -= 65535;}
+#define REDUCE {l_util.l = sum; sum = l_util.s[0] + l_util.s[1]; ADDCARRY(sum);}
+
+uint16_t in_cksum(const struct checksum_block *vec, int veclen)
+{
+	const uint16_t *w;
+	int sum = 0;
+	int mlen = 0;
+	int byte_swapped = 0;
+
+	union {
+		uint8_t		c[2];
+		uint16_t	s;
+	} s_util;
+	union {
+		uint16_t	s[2];
+		uint32_t	l;
+	} l_util;
+
+	for (; veclen != 0; vec++, veclen--) {
+		if (vec->len == 0)
+			continue;
+		w = reinterpret_cast<const uint16_t *>(vec->block);
+		if (mlen == -1) {
+			/*
+			 * The first byte of this chunk is the continuation
+			 * of a word spanning between this chunk and the
+			 * last chunk.
+			 *
+			 * s_util.c[0] is already saved when scanning previous
+			 * chunk.
+			 */
+			s_util.c[1] = *(const uint8_t *)w;
+			sum += s_util.s;
+			w = reinterpret_cast<const uint16_t *>((const uint8_t *)w + 1);
+			mlen = vec->len - 1;
+		} else
+			mlen = vec->len;
+		/*
+		 * Force to even boundary.
+		 */
+		if ((1 & (uintptr_t) w) && (mlen > 0)) {
+			REDUCE;
+			sum <<= 8;
+			s_util.c[0] = *(const uint8_t *)w;
+			w = reinterpret_cast<const uint16_t *>((const uint8_t *)w + 1);
+			mlen--;
+			byte_swapped = 1;
+		}
+		/*
+		 * Unroll the loop to make overhead from
+		 * branches &c small.
+		 */
+		while ((mlen -= 32) >= 0) {
+			sum += w[0]; sum += w[1]; sum += w[2]; sum += w[3];
+			sum += w[4]; sum += w[5]; sum += w[6]; sum += w[7];
+			sum += w[8]; sum += w[9]; sum += w[10]; sum += w[11];
+			sum += w[12]; sum += w[13]; sum += w[14]; sum += w[15];
+			w += 16;
+		}
+		mlen += 32;
+		while ((mlen -= 8) >= 0) {
+			sum += w[0]; sum += w[1]; sum += w[2]; sum += w[3];
+			w += 4;
+		}
+		mlen += 8;
+		if (mlen == 0 && byte_swapped == 0)
+			continue;
+		REDUCE;
+		while ((mlen -= 2) >= 0) {
+			sum += *w++;
+		}
+		if (byte_swapped) {
+			REDUCE;
+			sum <<= 8;
+			byte_swapped = 0;
+			if (mlen == -1) {
+				s_util.c[1] = *(const uint8_t *)w;
+				sum += s_util.s;
+				mlen = 0;
+			} else
+				mlen = -1;
+		} else if (mlen == -1)
+			s_util.c[0] = *(const uint8_t *)w;
+	}
+	if (mlen == -1) {
+		/* The last mbuf has odd # of bytes. Follow the
+		   standard (the odd byte may be shifted left by 8 bits
+		   or not as determined by endian-ness of the machine) */
+		s_util.c[1] = 0;
+		sum += s_util.s;
+	}
+	REDUCE;
+	return sum;
+}
+
+} // namespace zeek

src/3rdparty/jthread.hpp

@@ -0,0 +1,153 @@
+// Copied from https://github.com/josuttis/jthread,
+// used under CC-BY-4.0.
+
+// -----------------------------------------------------
+// cooperative interruptable and joining thread:
+// -----------------------------------------------------
+#ifndef JTHREAD_HPP
+#define JTHREAD_HPP
+
+#include <functional> // for invoke()
+#include <future>
+#include <iostream> // for debugging output
+#include <thread>
+#include <type_traits>
+
+#include "stop_token.hpp"
+
+namespace nonstd {
+
+//*****************************************
+//* class jthread
+//* - joining std::thread with signaling stop/end support
+//*****************************************
+class jthread {
+public:
+    //*****************************************
+    //* standardized API:
+    //*****************************************
+    // - cover full API of std::thread
+    //   to be able to switch from std::thread to std::jthread
+
+    // types are those from std::thread:
+    using id = ::std::thread::id;
+    using native_handle_type = ::std::thread::native_handle_type;
+
+    // construct/copy/destroy:
+    jthread() noexcept;
+    // template <typename F, typename... Args> explicit jthread(F&& f, Args&&... args);
+    //  THE constructor that starts the thread:
+    //  - NOTE: does SFINAE out copy constructor semantics
+    template<typename Callable, typename... Args,
+             typename = ::std::enable_if_t<! ::std::is_same_v<::std::decay_t<Callable>, jthread>>>
+    explicit jthread(Callable&& cb, Args&&... args);
+    ~jthread();
+
+    jthread(const jthread&) = delete;
+    jthread(jthread&&) noexcept = default;
+    jthread& operator=(const jthread&) = delete;
+    jthread& operator=(jthread&&) noexcept;
+
+    // members:
+    void swap(jthread&) noexcept;
+    bool joinable() const noexcept;
+    void join();
+    void detach();
+
+    id get_id() const noexcept;
+    native_handle_type native_handle();
+
+    // static members:
+    static unsigned hardware_concurrency() noexcept { return ::std::thread::hardware_concurrency(); };
+
+    //*****************************************
+    // - supplementary API:
+    //   - for the calling thread:
+    [[nodiscard]] stop_source get_stop_source() noexcept;
+    [[nodiscard]] stop_token get_stop_token() const noexcept;
+    bool request_stop() noexcept { return get_stop_source().request_stop(); }
+
+
+    //*****************************************
+    //* implementation:
+    //*****************************************
+
+private:
+    //*** API for the starting thread:
+    stop_source _stopSource; // stop_source for started thread
+    ::std::thread _thread{}; // started thread (if any)
+};
+
+
+//**********************************************************************
+
+//*****************************************
+//* implementation of class jthread
+//*****************************************
+
+// default constructor:
+inline jthread::jthread() noexcept : _stopSource{nostopstate} {}
+
+// THE constructor that starts the thread:
+// - NOTE: declaration does SFINAE out copy constructor semantics
+template<typename Callable, typename... Args, typename>
+inline jthread::jthread(Callable&& cb, Args&&... args)
+    : _stopSource{}, // initialize stop_source
+      _thread{
+          [](stop_token st, auto&& cb, auto&&... args) { // called lambda in the thread
+              // perform tasks of the thread:
+              if constexpr ( std::is_invocable_v<Callable, stop_token, Args...> ) {
+                  // pass the stop_token as first argument to the started thread:
+                  ::std::invoke(::std::forward<decltype(cb)>(cb), std::move(st),
+                                ::std::forward<decltype(args)>(args)...);
+              }
+              else {
+                  // started thread does not expect a stop token:
+                  ::std::invoke(::std::forward<decltype(cb)>(cb), ::std::forward<decltype(args)>(args)...);
+              }
+          },
+          _stopSource.get_token(),      // not captured due to possible races if immediately set
+          ::std::forward<Callable>(cb), // pass callable
+          ::std::forward<Args>(args)... // pass arguments for callable
+      } {}
+
+// move assignment operator:
+inline jthread& jthread::operator=(jthread&& t) noexcept {
+    if ( joinable() ) { // if not joined/detached, signal stop and wait for end:
+        request_stop();
+        join();
+    }
+
+    _thread = std::move(t._thread);
+    _stopSource = std::move(t._stopSource);
+    return *this;
+}
+
+// destructor:
+inline jthread::~jthread() {
+    if ( joinable() ) { // if not joined/detached, signal stop and wait for end:
+        request_stop();
+        join();
+    }
+}
+
+
+// others:
+inline bool jthread::joinable() const noexcept { return _thread.joinable(); }
+inline void jthread::join() { _thread.join(); }
+inline void jthread::detach() { _thread.detach(); }
+inline typename jthread::id jthread::get_id() const noexcept { return _thread.get_id(); }
+inline typename jthread::native_handle_type jthread::native_handle() { return _thread.native_handle(); }
+
+inline stop_source jthread::get_stop_source() noexcept { return _stopSource; }
+inline stop_token jthread::get_stop_token() const noexcept { return _stopSource.get_token(); }
+
+inline void jthread::swap(jthread& t) noexcept {
+    std::swap(_stopSource, t._stopSource);
+    std::swap(_thread, t._thread);
+}
+
+
+} // namespace nonstd
+
+#endif // JTHREAD_HPP

src/3rdparty/modp_numtoa.c

@@ -0,0 +1,527 @@
+/* -*- mode: c; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 4 -*- */
+/* vi: set expandtab shiftwidth=4 tabstop=4: */
+
+#include "modp_numtoa.h"
+
+#include <stdint.h>
+#include <stdio.h>
+#include <math.h>
+#include <limits.h>
+#include <float.h>
+
+// other interesting references on num to string convesion
+// http://www.jb.man.ac.uk/~slowe/cpp/itoa.html
+// and http://www.ddj.com/dept/cpp/184401596?pgno=6
+
+// Version 19-Nov-2007
+// Fixed round-to-even rules to match printf
+//   thanks to Johannes Otepka
+
+/**
+ * Powers of 10
+ * 10^0 to 10^9
+ */
+static const double _pow10[] = {1, 10, 100, 1000, 10000, 100000, 1000000,
+                               10000000, 100000000, 1000000000};
+static const double _pow10r[] = {1, .1, .01, .001, .0001, .00001, .000001,
+                                .0000001, .00000001, .000000001};
+
+static void strreverse(char* begin, char* end)
+{
+    char aux;
+    while (end > begin)
+        aux = *end, *end-- = *begin, *begin++ = aux;
+}
+
+// Expects 'str' to have been made using "%e" scientific notation format string
+// Returns the number of characters removed
+static size_t sn_strip_trailing_zeros(char* str)
+	{
+	char* frac = 0;
+
+	for ( ; ; )
+		{
+		if ( *str == '.' )
+			{
+			frac = str + 1;
+			break;
+			}
+
+		if ( *str == 0 )
+			break;
+
+		++str;
+		}
+
+	if ( ! frac )
+		return 0;
+
+	char* start_dec = frac;
+	char* exp = 0;
+	char* trailing_zeros = 0;
+
+	for ( ; ; )
+		{
+		if ( *frac == 0 )
+			break;
+
+		if ( *frac == 'e' )
+			{
+			exp = frac;
+			break;
+			}
+
+		if ( *frac == '0' )
+			{
+			if ( ! trailing_zeros )
+				trailing_zeros = frac;
+			}
+		else
+			trailing_zeros = 0;
+
+		++frac;
+		}
+
+	if ( trailing_zeros == start_dec )
+		--trailing_zeros;
+
+	if ( ! trailing_zeros || ! exp )
+		return 0;
+
+	char* start_exp = exp;
+
+	for ( ; ; )
+		{
+		*trailing_zeros = *exp;
+
+		if ( *exp == 0 )
+			break;
+
+		++trailing_zeros;
+		++exp;
+		}
+
+	return exp - start_exp;
+	}
+
+size_t modp_itoa10(int32_t value, char* str)
+{
+    char* wstr=str;
+    // Take care of sign
+    unsigned int uvalue = (value < 0) ? -value : value;
+    // Conversion. Number is reversed.
+    do *wstr++ = (char)(48 + (uvalue % 10)); while(uvalue /= 10);
+    if (value < 0) *wstr++ = '-';
+    *wstr='\0';
+
+    // Reverse string
+    strreverse(str,wstr-1);
+    return wstr - str;
+}
+
+size_t modp_uitoa10(uint32_t value, char* str)
+{
+    char* wstr=str;
+    // Conversion. Number is reversed.
+    do *wstr++ = (char)(48 + (value % 10)); while (value /= 10);
+    *wstr='\0';
+    // Reverse string
+    strreverse(str, wstr-1);
+    return wstr - str;
+}
+
+size_t modp_litoa10(int64_t value, char* str)
+{
+    char* wstr=str;
+    uint64_t uvalue = (value < 0) ? (value == INT64_MIN ? (uint64_t)(INT64_MAX) + 1 : -value) : value;
+
+    // Conversion. Number is reversed.
+    do *wstr++ = (char)(48 + (uvalue % 10)); while(uvalue /= 10);
+    if (value < 0) *wstr++ = '-';
+    *wstr='\0';
+
+    // Reverse string
+    strreverse(str,wstr-1);
+    return wstr - str;
+}
+
+size_t modp_ulitoa10(uint64_t value, char* str)
+{
+    char* wstr=str;
+    // Conversion. Number is reversed.
+    do *wstr++ = (char)(48 + (value % 10)); while (value /= 10);
+    *wstr='\0';
+    // Reverse string
+    strreverse(str, wstr-1);
+    return wstr - str;
+}
+
+size_t modp_dtoa(double value, char* str, int prec)
+{
+    /* Hacky test for NaN
+     * under -fast-math this won't work, but then you also won't
+     * have correct nan values anyways.  The alternative is
+     * to link with libmath (bad) or hack IEEE double bits (bad)
+     */
+    if (! (value == value)) {
+        str[0] = 'n'; str[1] = 'a'; str[2] = 'n'; str[3] = '\0';
+        return 3;
+    }
+
+    /* we'll work in positive values and deal with the
+       negative sign issue later */
+    int neg = 0;
+    if (value < 0) {
+        neg = 1;
+        value = -value;
+    }
+
+    /* if input is larger than thres_max, revert to exponential */
+    const double thres_max = (double)(INT_MAX);
+
+    /* for very large numbers switch back to native sprintf for exponentials.
+       anyone want to write code to replace this? */
+    /*
+      normal printf behavior is to print EVERY whole number digit
+      which can be 100s of characters overflowing your buffers == bad
+    */
+    if (value >= thres_max) {
+#pragma GCC diagnostic push
+        // Ignore the diagnostic warning about sprintf being deprecated here. We
+        // have to assume that the size of the string is long enough here
+        // because we'd have to change the function definition to support
+        // passing a length.
+#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
+        int n = sprintf(str, "%.*e", DBL_DECIMAL_DIG - 1, neg ? -value : value);
+#pragma GCC diagnostic pop
+        n -= sn_strip_trailing_zeros(str);
+        return n;
+    }
+
+    double diff = 0.0;
+    char* wstr = str;
+
+    if (prec < 0) {
+        prec = 0;
+    } else if (prec > 9) {
+        /* precision of >= 10 can lead to overflow errors */
+        prec = 9;
+    }
+
+    int whole = (int) value;
+    double tmp = (value - whole) * _pow10[prec];
+    uint32_t frac = (uint32_t)(tmp);
+    diff = tmp - frac;
+
+    if (diff > 0.5) {
+        ++frac;
+        /* handle rollover, e.g.  case 0.99 with prec 1 is 1.0  */
+        if (frac >= _pow10[prec]) {
+            frac = 0;
+            ++whole;
+        }
+    } else if (diff == 0.5 && ((frac == 0) || (frac & 1))) {
+        /* if halfway, round up if odd, OR
+           if last digit is 0.  That last part is strange */
+        ++frac;
+    }
+
+    if (prec == 0) {
+        diff = value - whole;
+        if (diff > 0.5) {
+            /* greater than 0.5, round up, e.g. 1.6 -> 2 */
+            ++whole;
+        } else if (diff == 0.5 && (whole & 1)) {
+            /* exactly 0.5 and ODD, then round up */
+            /* 1.5 -> 2, but 2.5 -> 2 */
+            ++whole;
+        }
+    } else {
+        int count = prec;
+        // now do fractional part, as an unsigned number
+        do {
+            --count;
+            *wstr++ = (char)(48 + (frac % 10));
+        } while (frac /= 10);
+        // add extra 0s
+        while (count-- > 0) *wstr++ = '0';
+        // add decimal
+        *wstr++ = '.';
+    }
+
+    // do whole part
+    // Take care of sign
+    // Conversion. Number is reversed.
+    do *wstr++ = (char)(48 + (whole % 10)); while (whole /= 10);
+    if (neg) {
+        *wstr++ = '-';
+    }
+    *wstr='\0';
+    strreverse(str, wstr-1);
+    return wstr - str;
+}
+
+
+// This is near identical to modp_dtoa above
+//   The differnce is noted below
+size_t modp_dtoa2(double value, char* str, int prec)
+{
+    /* Hacky test for NaN
+     * under -fast-math this won't work, but then you also won't
+     * have correct nan values anyways.  The alternative is
+     * to link with libmath (bad) or hack IEEE double bits (bad)
+     */
+    if (! (value == value)) {
+        str[0] = 'n'; str[1] = 'a'; str[2] = 'n'; str[3] = '\0';
+        return 3;
+    }
+
+    /* we'll work in positive values and deal with the
+       negative sign issue later */
+    int neg = 0;
+    if (value < 0) {
+        neg = 1;
+        value = -value;
+    }
+
+    /* if input is larger than thres_max, revert to exponential */
+    const double thres_max = (double)(INT_MAX);
+
+    /* for very large numbers switch back to native sprintf for exponentials.
+       anyone want to write code to replace this? */
+    /*
+      normal printf behavior is to print EVERY whole number digit
+      which can be 100s of characters overflowing your buffers == bad
+    */
+    if (value >= thres_max) {
+#pragma GCC diagnostic push
+        // Ignore the diagnostic warning about sprintf being deprecated here. We
+        // have to assume that the size of the string is long enough here
+        // because we'd have to change the function definition to support
+        // passing a length.
+#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
+        int n = sprintf(str, "%.*e", DBL_DECIMAL_DIG - 1, neg ? -value : value);
+#pragma GCC diagnostic pop
+        n -= sn_strip_trailing_zeros(str);
+        return n;
+    }
+
+    int count;
+    double diff = 0.0;
+    char* wstr = str;
+
+    if (prec < 0) {
+        prec = 0;
+    } else if (prec > 9) {
+        /* precision of >= 10 can lead to overflow errors */
+        prec = 9;
+    }
+
+    double smallest = _pow10r[prec];
+
+    if (value != 0.0 && value < smallest) {
+#pragma GCC diagnostic push
+        // Ignore the diagnostic warning about sprintf being deprecated here. We
+        // have to assume that the size of the string is long enough here
+        // because we'd have to change the function definition to support
+        // passing a length.
+#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
+        int n = sprintf(str, "%.*e", DBL_DECIMAL_DIG - 1, neg ? -value : value);
+#pragma GCC diagnostic pop
+        n -= sn_strip_trailing_zeros(str);
+        return n;
+    }
+
+    int whole = (int) value;
+    double tmp = (value - whole) * _pow10[prec];
+    uint32_t frac = (uint32_t)(tmp);
+    diff = tmp - frac;
+
+    if (diff > 0.5) {
+        ++frac;
+        /* handle rollover, e.g.  case 0.99 with prec 1 is 1.0  */
+        if (frac >= _pow10[prec]) {
+            frac = 0;
+            ++whole;
+        }
+    } else if (diff == 0.5 && ((frac == 0) || (frac & 1))) {
+        /* if halfway, round up if odd, OR
+           if last digit is 0.  That last part is strange */
+        ++frac;
+    }
+
+    if (prec == 0) {
+        diff = value - whole;
+        if (diff > 0.5) {
+            /* greater than 0.5, round up, e.g. 1.6 -> 2 */
+            ++whole;
+        } else if (diff == 0.5 && (whole & 1)) {
+            /* exactly 0.5 and ODD, then round up */
+            /* 1.5 -> 2, but 2.5 -> 2 */
+            ++whole;
+        }
+
+        //vvvvvvvvvvvvvvvvvvv  Diff from modp_dto2
+    } else if (frac) {
+        count = prec;
+        // now do fractional part, as an unsigned number
+        // we know it is not 0 but we can have leading zeros, these
+        // should be removed
+        while (!(frac % 10)) {
+            --count;
+            frac /= 10;
+        }
+        //^^^^^^^^^^^^^^^^^^^  Diff from modp_dto2
+
+        // now do fractional part, as an unsigned number
+        do {
+            --count;
+            *wstr++ = (char)(48 + (frac % 10));
+        } while (frac /= 10);
+        // add extra 0s
+        while (count-- > 0) *wstr++ = '0';
+        // add decimal
+        *wstr++ = '.';
+    }
+
+    // do whole part
+    // Take care of sign
+    // Conversion. Number is reversed.
+    do *wstr++ = (char)(48 + (whole % 10)); while (whole /= 10);
+    if (neg) {
+        *wstr++ = '-';
+    }
+    *wstr='\0';
+    strreverse(str, wstr-1);
+    return wstr - str;
+}
+
+// This is near identical to modp_dtoa2 above, excep that it never uses
+// exponential notation and requires a buffer length.
+size_t modp_dtoa3(double value, char* str, int n, int prec)
+{
+    /* Hacky test for NaN
+     * under -fast-math this won't work, but then you also won't
+     * have correct nan values anyways.  The alternative is
+     * to link with libmath (bad) or hack IEEE double bits (bad)
+     */
+    if (! (value == value)) {
+        str[0] = 'n'; str[1] = 'a'; str[2] = 'n'; str[3] = '\0';
+        return 3;
+    }
+
+    /* we'll work in positive values and deal with the
+       negative sign issue later */
+    int neg = 0;
+    if (value < 0) {
+        neg = 1;
+        value = -value;
+    }
+
+    if (prec < 0) {
+        prec = 0;
+    } else if (prec > 9) {
+        /* precision of >= 10 can lead to overflow errors */
+        prec = 9;
+    }
+
+    /* if input is larger than thres_max, revert to exponential */
+    const double thres_max = (double)(INT_MAX);
+
+    /* for very large numbers switch back to native sprintf for exponentials.
+       anyone want to write code to replace this? */
+    /*
+      normal printf behavior is to print EVERY whole number digit
+      which can be 100s of characters overflowing your buffers == bad
+    */
+    if (value >= thres_max) {
+        /* ---- Modified part, compared to modp_dtoa3. */
+        int i = snprintf(str, n, "%.*f", prec, neg ? -value : value);
+
+        if ( i < 0 || i >= n ) {
+        // Error or truncated output.
+            snprintf(str, n, "NAN");
+            return 3;
+            }
+
+        /* Remove trailing zeros. */
+
+        char* p;
+        for ( p = str + i - 1; p >= str && *p == '0'; --p );
+
+        if ( p >= str && *p == '.' )
+            --p;
+
+        *++p = '\0';
+        return p - str - 1;
+
+        /* ---- End of modified part.. */
+    }
+
+    int count;
+    double diff = 0.0;
+    char* wstr = str;
+
+    int whole = (int) value;
+    double tmp = (value - whole) * _pow10[prec];
+    uint32_t frac = (uint32_t)(tmp);
+    diff = tmp - frac;
+
+    if (diff > 0.5) {
+        ++frac;
+        /* handle rollover, e.g.  case 0.99 with prec 1 is 1.0  */
+        if (frac >= _pow10[prec]) {
+            frac = 0;
+            ++whole;
+        }
+    } else if (diff == 0.5 && ((frac == 0) || (frac & 1))) {
+        /* if halfway, round up if odd, OR
+           if last digit is 0.  That last part is strange */
+        ++frac;
+    }
+
+    if (prec == 0) {
+        diff = value - whole;
+        if (diff > 0.5) {
+            /* greater than 0.5, round up, e.g. 1.6 -> 2 */
+            ++whole;
+        } else if (diff == 0.5 && (whole & 1)) {
+            /* exactly 0.5 and ODD, then round up */
+            /* 1.5 -> 2, but 2.5 -> 2 */
+            ++whole;
+        }
+
+        //vvvvvvvvvvvvvvvvvvv  Diff from modp_dto2
+    } else if (frac) {
+        count = prec;
+        // now do fractional part, as an unsigned number
+        // we know it is not 0 but we can have leading zeros, these
+        // should be removed
+        while (!(frac % 10)) {
+            --count;
+            frac /= 10;
+        }
+        //^^^^^^^^^^^^^^^^^^^  Diff from modp_dto2
+
+        // now do fractional part, as an unsigned number
+        do {
+            --count;
+            *wstr++ = (char)(48 + (frac % 10));
+        } while (frac /= 10);
+        // add extra 0s
+        while (count-- > 0) *wstr++ = '0';
+        // add decimal
+        *wstr++ = '.';
+    }
+
+    // do whole part
+    // Take care of sign
+    // Conversion. Number is reversed.
+    do *wstr++ = (char)(48 + (whole % 10)); while (whole /= 10);
+    if (neg) {
+        *wstr++ = '-';
+    }
+    *wstr='\0';
+    strreverse(str, wstr-1);
+    return wstr - str;
+}

src/3rdparty/modp_numtoa.h

@@ -0,0 +1,116 @@
+/* -*- mode: c++; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 4 -*- */
+/* vi: set expandtab shiftwidth=4 tabstop=4: */
+
+/**
+ * \file
+ *
+ * <pre>
+ * Copyright &copy; 2007, Nick Galbreath -- nickg [at] modp [dot] com
+ * All rights reserved.
+ * http://code.google.com/p/stringencoders/
+ * Released under the bsd license.
+ * </pre>
+ *
+ * This defines signed/unsigned integer, and 'double' to char buffer
+ * converters.  The standard way of doing this is with "sprintf", however
+ * these functions are
+ *   * guarenteed maximum size output
+ *   * 5-20x faster!
+ *   * core-dump safe
+ *
+ *
+ */
+
+#pragma once
+
+#ifdef __cplusplus
+#define BEGIN_C extern "C" {
+#define END_C }
+#else
+#define BEGIN_C
+#define END_C
+#endif
+
+BEGIN_C
+
+#include <stdint.h>
+#include <stddef.h>
+
+/** \brief convert an signed integer to char buffer, return # characters added
+ *
+ * \param[in] value
+ * \param[out] buf the output buffer.  Should be 16 chars or more.
+ */
+size_t modp_itoa10(int32_t value, char* buf);
+
+/** \brief convert an unsigned integer to char buffer, return # characters added
+ *
+ * \param[in] value
+ * \param[out] buf The output buffer, should be 16 chars or more.
+ */
+size_t modp_uitoa10(uint32_t value, char* buf);
+
+/** \brief convert an signed long integer to char buffer, return # characters added
+ *
+ * \param[in] value
+ * \param[out] buf the output buffer.  Should be 24 chars or more.
+ */
+size_t modp_litoa10(int64_t value, char* buf);
+
+/** \brief convert an unsigned long integer to char buffer, return # characters added
+ *
+ * \param[in] value
+ * \param[out] buf The output buffer, should be 24 chars or more.
+ */
+size_t modp_ulitoa10(uint64_t value, char* buf);
+
+/** \brief convert a floating point number to char buffer with
+ *         fixed-precision format, return # characters added
+ *
+ * This is similar to "%.[0-9]f" in the printf style.  It will include
+ * trailing zeros
+ *
+ * If the input value is greater than 1<<31, then the output format
+ * will be switched exponential format and include as many precision digits
+ * as needed to preserve information.
+ *
+ * \param[in] value
+ * \param[out] buf  The allocated output buffer.  Should be 32 chars or more.
+ * \param[in] precision  Number of digits to the right of the decimal point.
+ *    Can only be 0-9.
+ */
+size_t modp_dtoa(double value, char* buf, int precision);
+
+/** \brief convert a floating point number to char buffer with a
+ *         variable-precision format, and no trailing zero, return
+ *         number of characters added
+ *
+ * This is similar to "%.[0-9]f" in the printf style, except it will
+ * NOT include trailing zeros after the decimal point.  This type
+ * of format oddly does not exists with printf.
+ *
+ * If the input value is greater than 1<<31, then the output format
+ * will be switched exponential format and include as many precision digits
+ * as needed to preserve information.
+ *
+ * If a non-zero input value is less than 10^(-precision), the output format
+ * will be switched exponential format and include as many precision digits
+ * as needed to preserve information.
+ *
+ * \param[in] value
+ * \param[out] buf  The allocated output buffer.  Should be 32 chars or more.
+ * \param[in] precision  Number of digits to the right of the decimal point.
+ *    Can only be 0-9.
+ */
+size_t modp_dtoa2(double value, char* buf, int precision);
+
+/** \brief convert a floating point number to char buffer with a
+ *         variable-precision format, no trailing zeros, and no
+ *   	scientific notation, return number of characters added
+ *
+ *  Other than avoiding scientific notation, this is the same as mop_dtoa2. It does however
+ *  require the max buffer length. The buffer will always be null-terminated.
+ */
+size_t modp_dtoa3(double value, char* buf, int n, int precision);
+
+END_C

src/3rdparty/patricia.h

@@ -0,0 +1,209 @@
+/*
+ * $Id: patricia.h,v 1.6 2005/12/07 20:53:01 dplonka Exp $
+ * Dave Plonka <plonka@doit.wisc.edu>
+ *
+ * This product includes software developed by the University of Michigan,
+ * Merit Network, Inc., and their contributors.
+ *
+ * This file had been called "radix.h" in the MRT sources.
+ *
+ * I renamed it to "patricia.h" since it's not an implementation of a general
+ * radix trie.  Also, pulled in various requirements from "mrt.h" and added
+ * some other things it could be used as a standalone API.
+ */
+
+/*
+ * This code originates from Dave Plonka's Net::Security perl module. An
+ * adaptation of it in C is kept at
+ * https://github.com/CAIDA/cc-common/tree/master/libpatricia. That repository
+ * is considered the upstream version for Zeek's fork. We make some custom
+ * changes to this upstream:
+ * - Replace void_fn_t with data_fn_t and prefix_data_fn_t
+ * - Add patricia_search_all method
+ *
+ * The current version is based on commit
+ * fd262ab5ac5bae8b0d4a8b5e2e723115b1846376 from that repo.
+ */
+
+/* From copyright.txt:
+ *
+ * Copyright (c) 1997, 1998, 1999
+ *
+ *
+ * The Regents of the University of Michigan ("The Regents") and Merit Network,
+ * Inc.  All rights reserved.
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 1.  Redistributions of source code must retain the above
+ *     copyright notice, this list of conditions and the
+ *     following disclaimer.
+ * 2.  Redistributions in binary form must reproduce the above
+ *     copyright notice, this list of conditions and the
+ *     following disclaimer in the documentation and/or other
+ *     materials provided with the distribution.
+ * 3.  All advertising materials mentioning features or use of
+ *     this software must display the following acknowledgement:
+ * This product includes software developed by the University of Michigan, Merit
+ * Network, Inc., and their contributors.
+ * 4.  Neither the name of the University, Merit Network, nor the
+ *     names of their contributors may be used to endorse or
+ *     promote products derived from this software without
+ *     specific prior written permission.
+ * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS" AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef _PATRICIA_H
+#define _PATRICIA_H
+
+#define HAVE_IPV6
+
+/* typedef unsigned int u_int; */
+typedef void (*void_fn_t)();
+/* { from defs.h */
+#define prefix_touchar(prefix) ((u_char *)&(prefix)->add.sin)
+#define MAXLINE 1024
+#define BIT_TEST(f, b) ((f) & (b))
+/* } */
+
+#define addroute make_and_lookup
+
+#include <sys/types.h> /* for u_* definitions (on FreeBSD 5) */
+
+#include <errno.h> /* for EAFNOSUPPORT */
+#ifndef EAFNOSUPPORT
+#defined EAFNOSUPPORT WSAEAFNOSUPPORT
+#include <winsock.h>
+#else
+#include <netinet/in.h> /* for struct in_addr */
+#endif
+
+#include <sys/socket.h> /* for AF_INET */
+
+/* { from mrt.h */
+
+typedef struct _prefix4_t {
+  u_short family; /* AF_INET | AF_INET6 */
+  u_short bitlen; /* same as mask? */
+  int ref_count;  /* reference count */
+  struct in_addr sin;
+} prefix4_t;
+
+typedef struct _prefix_t {
+  u_short family; /* AF_INET | AF_INET6 */
+  u_short bitlen; /* same as mask? */
+  int ref_count;  /* reference count */
+  union {
+    struct in_addr sin;
+#ifdef HAVE_IPV6
+    struct in6_addr sin6;
+#endif /* IPV6 */
+  } add;
+} prefix_t;
+
+typedef void (*data_fn_t)(void *);
+typedef void (*prefix_data_fn_t)(prefix_t *, void *);
+
+/* } */
+
+typedef struct _patricia_node_t {
+  u_int bit;                       /* flag if this node used */
+  prefix_t *prefix;                /* who we are in patricia tree */
+  struct _patricia_node_t *l, *r;  /* left and right children */
+  struct _patricia_node_t *parent; /* may be used */
+  void *data;                      /* pointer to data */
+  void *user1; /* pointer to usr data (ex. route flap info) */
+} patricia_node_t;
+
+typedef struct _patricia_tree_t {
+  patricia_node_t *head;
+  u_int maxbits;       /* for IP, 32 bit addresses */
+  int num_active_node; /* for debug purpose */
+} patricia_tree_t;
+
+patricia_node_t *patricia_search_exact(patricia_tree_t *patricia,
+                                       prefix_t *prefix);
+int patricia_search_all(patricia_tree_t *patricia, prefix_t *prefix,
+                        patricia_node_t ***list, int *n);
+patricia_node_t *patricia_search_best(patricia_tree_t *patricia,
+                                      prefix_t *prefix);
+patricia_node_t *patricia_search_best2(patricia_tree_t *patricia,
+                                       prefix_t *prefix, int inclusive);
+patricia_node_t *patricia_lookup(patricia_tree_t *patricia, prefix_t *prefix);
+void patricia_remove(patricia_tree_t *patricia, patricia_node_t *node);
+patricia_tree_t *New_Patricia(int maxbits);
+void Clear_Patricia(patricia_tree_t *patricia, data_fn_t func);
+void Destroy_Patricia(patricia_tree_t *patricia, data_fn_t func);
+
+void patricia_process(patricia_tree_t *patricia, prefix_data_fn_t func);
+
+void Deref_Prefix(prefix_t *prefix);
+char *prefix_toa(prefix_t *prefix);
+
+/* { from demo.c */
+
+prefix_t *ascii2prefix(int family, char *string);
+
+patricia_node_t *make_and_lookup(patricia_tree_t *tree, char *string);
+
+/* } */
+
+#define PATRICIA_MAXBITS (sizeof(struct in6_addr) * 8)
+#define PATRICIA_NBIT(x) (0x80 >> ((x)&0x7f))
+#define PATRICIA_NBYTE(x) ((x) >> 3)
+
+#define PATRICIA_DATA_GET(node, type) (type *)((node)->data)
+#define PATRICIA_DATA_SET(node, value) ((node)->data = (void *)(value))
+
+#define PATRICIA_WALK(Xhead, Xnode)                                            \
+  do {                                                                         \
+    patricia_node_t *Xstack[PATRICIA_MAXBITS + 1];                             \
+    patricia_node_t **Xsp = Xstack;                                            \
+    patricia_node_t *Xrn = (Xhead);                                            \
+    while ((Xnode = Xrn)) {                                                    \
+      if (Xnode->prefix)
+
+#define PATRICIA_WALK_ALL(Xhead, Xnode)                                        \
+  do {                                                                         \
+    patricia_node_t *Xstack[PATRICIA_MAXBITS + 1];                             \
+    patricia_node_t **Xsp = Xstack;                                            \
+    patricia_node_t *Xrn = (Xhead);                                            \
+    while ((Xnode = Xrn)) {                                                    \
+      if (1)
+
+#define PATRICIA_WALK_BREAK                                                    \
+  {                                                                            \
+    if (Xsp != Xstack) {                                                       \
+      Xrn = *(--Xsp);                                                          \
+    } else {                                                                   \
+      Xrn = (patricia_node_t *)0;                                              \
+    }                                                                          \
+    continue;                                                                  \
+  }
+
+#define PATRICIA_WALK_END                                                      \
+  if (Xrn->l) {                                                                \
+    if (Xrn->r) {                                                              \
+      *Xsp++ = Xrn->r;                                                         \
+    }                                                                          \
+    Xrn = Xrn->l;                                                              \
+  } else if (Xrn->r) {                                                         \
+    Xrn = Xrn->r;                                                              \
+  } else if (Xsp != Xstack) {                                                  \
+    Xrn = *(--Xsp);                                                            \
+  } else {                                                                     \
+    Xrn = (patricia_node_t *)0;                                                \
+  }                                                                            \
+  }                                                                            \
+  }                                                                            \
+  while (0)
+
+#endif /* _PATRICIA_H */

src/3rdparty/setsignal.c

@@ -0,0 +1,52 @@
+/*
+ * See the file "COPYING" in the main distribution directory for copyright.
+ */
+
+#include "zeek/zeek-config.h"			/* must appear before first ifdef */
+
+#include <sys/types.h>
+
+#ifdef HAVE_MEMORY_H
+#include <memory.h>
+#endif
+#include <signal.h>
+#ifdef HAVE_SIGACTION
+#include <string.h>
+#endif
+
+#include "setsignal.h"
+
+/*
+ * An os independent signal() with BSD semantics, e.g. the signal
+ * catcher is restored following service of the signal.
+ *
+ * When sigset() is available, signal() has SYSV semantics and sigset()
+ * has BSD semantics and call interface. Unfortunately, Linux does not
+ * have sigset() so we use the more complicated sigaction() interface
+ * there.
+ *
+ * Did I mention that signals suck?
+ */
+RETSIGTYPE
+(*setsignal (int sig, RETSIGTYPE (*func)(int)))(int)
+{
+#ifdef HAVE_SIGACTION
+	struct sigaction old, new;
+
+	memset(&new, 0, sizeof(new));
+	new.sa_handler = func;
+#ifdef SA_RESTART
+	new.sa_flags |= SA_RESTART;
+#endif
+	if (sigaction(sig, &new, &old) < 0)
+		return (SIG_ERR);
+	return (old.sa_handler);
+
+#else
+#ifdef HAVE_SIGSET
+	return (sigset(sig, func));
+#else
+	return (signal(sig, func));
+#endif
+#endif
+}

src/3rdparty/setsignal.h

@@ -0,0 +1,7 @@
+/*
+ * See the file "COPYING" in the main distribution directory for copyright.
+ *
+ */
+#pragma once
+
+RETSIGTYPE (*setsignal(int, RETSIGTYPE (*)(int)))(int);

src/3rdparty/stop_token.hpp

@@ -0,0 +1,510 @@
+// Copied from https://github.com/josuttis/jthread,
+// used under CC-BY-4.0.
+
+#pragma once
+// <stop_token> header
+
+#include <atomic>
+#include <thread>
+#include <type_traits>
+#include <utility>
+#ifdef SAFE
+#include <iostream>
+#endif
+
+#if defined(__x86_64__) || defined(_M_X64)
+#include <immintrin.h>
+#endif
+
+namespace nonstd {
+inline void __spin_yield() noexcept {
+    // TODO: Platform-specific code here
+#if defined(__x86_64__) || defined(_M_X64)
+    _mm_pause();
+#endif
+}
+
+
+//-----------------------------------------------
+// internal types for shared stop state
+//-----------------------------------------------
+
+struct __stop_callback_base {
+    void (*__callback_)(__stop_callback_base*) = nullptr;
+
+    __stop_callback_base* __next_ = nullptr;
+    __stop_callback_base** __prev_ = nullptr;
+    bool* __isRemoved_ = nullptr;
+    std::atomic<bool> __callbackFinishedExecuting_{false};
+
+    void __execute() noexcept { __callback_(this); }
+
+protected:
+    // it shall only by us who deletes this
+    // (workaround for virtual __execute() and destructor)
+    ~__stop_callback_base() = default;
+};
+
+struct __stop_state {
+public:
+    void __add_token_reference() noexcept { __state_.fetch_add(__token_ref_increment, std::memory_order_relaxed); }
+
+    void __remove_token_reference() noexcept {
+        auto __oldState = __state_.fetch_sub(__token_ref_increment, std::memory_order_acq_rel);
+        if ( __oldState < (__token_ref_increment + __source_ref_increment) ) {
+            delete this;
+        }
+    }
+
+    void __add_source_reference() noexcept { __state_.fetch_add(__source_ref_increment, std::memory_order_relaxed); }
+
+    void __remove_source_reference() noexcept {
+        auto __oldState = __state_.fetch_sub(__source_ref_increment, std::memory_order_acq_rel);
+        if ( __oldState < (__token_ref_increment + __source_ref_increment) ) {
+            delete this;
+        }
+    }
+
+    bool __request_stop() noexcept {
+        if ( ! __try_lock_and_signal_until_signalled() ) {
+            // Stop has already been requested.
+            return false;
+        }
+
+        // Set the 'stop_requested' signal and acquired the lock.
+
+        __signallingThread_ = std::this_thread::get_id();
+
+        while ( __head_ != nullptr ) {
+            // Dequeue the head of the queue
+            auto* __cb = __head_;
+            __head_ = __cb->__next_;
+            const bool anyMore = __head_ != nullptr;
+            if ( anyMore ) {
+                __head_->__prev_ = &__head_;
+            }
+            // Mark this item as removed from the list.
+            __cb->__prev_ = nullptr;
+
+            // Don't hold lock while executing callback
+            // so we don't block other threads from deregistering callbacks.
+            __unlock();
+
+            // TRICKY: Need to store a flag on the stack here that the callback
+            // can use to signal that the destructor was executed inline
+            // during the call. If the destructor was executed inline then
+            // it's not safe to dereference __cb after __execute() returns.
+            // If the destructor runs on some other thread then the other
+            // thread will block waiting for this thread to signal that the
+            // callback has finished executing.
+            bool __isRemoved = false;
+            __cb->__isRemoved_ = &__isRemoved;
+
+            __cb->__execute();
+
+            if ( ! __isRemoved ) {
+                __cb->__isRemoved_ = nullptr;
+                __cb->__callbackFinishedExecuting_.store(true, std::memory_order_release);
+            }
+
+            if ( ! anyMore ) {
+                // This was the last item in the queue when we dequeued it.
+                // No more items should be added to the queue after we have
+                // marked the state as interrupted, only removed from the queue.
+                // Avoid acquring/releasing the lock in this case.
+                return true;
+            }
+
+            __lock();
+        }
+
+        __unlock();
+
+        return true;
+    }
+
+    bool __is_stop_requested() noexcept { return __is_stop_requested(__state_.load(std::memory_order_acquire)); }
+
+    bool __is_stop_requestable() noexcept { return __is_stop_requestable(__state_.load(std::memory_order_acquire)); }
+
+    bool __try_add_callback(__stop_callback_base* __cb, bool __incrementRefCountIfSuccessful) noexcept {
+        std::uint64_t __oldState;
+        goto __load_state;
+        do {
+            goto __check_state;
+            do {
+                __spin_yield();
+            __load_state:
+                __oldState = __state_.load(std::memory_order_acquire);
+            __check_state:
+                if ( __is_stop_requested(__oldState) ) {
+                    __cb->__execute();
+                    return false;
+                }
+                else if ( ! __is_stop_requestable(__oldState) ) {
+                    return false;
+                }
+            } while ( __is_locked(__oldState) );
+        } while ( ! __state_.compare_exchange_weak(__oldState, __oldState | __locked_flag, std::memory_order_acquire) );
+
+        // Push callback onto callback list.
+        __cb->__next_ = __head_;
+        if ( __cb->__next_ != nullptr ) {
+            __cb->__next_->__prev_ = &__cb->__next_;
+        }
+        __cb->__prev_ = &__head_;
+        __head_ = __cb;
+
+        if ( __incrementRefCountIfSuccessful ) {
+            __unlock_and_increment_token_ref_count();
+        }
+        else {
+            __unlock();
+        }
+
+        // Successfully added the callback.
+        return true;
+    }
+
+    void __remove_callback(__stop_callback_base* __cb) noexcept {
+        __lock();
+
+        if ( __cb->__prev_ != nullptr ) {
+            // Still registered, not yet executed
+            // Just remove from the list.
+            *__cb->__prev_ = __cb->__next_;
+            if ( __cb->__next_ != nullptr ) {
+                __cb->__next_->__prev_ = __cb->__prev_;
+            }
+
+            __unlock_and_decrement_token_ref_count();
+
+            return;
+        }
+
+        __unlock();
+
+        // Callback has either already executed or is executing
+        // concurrently on another thread.
+
+        if ( __signallingThread_ == std::this_thread::get_id() ) {
+            // Callback executed on this thread or is still currently executing
+            // and is deregistering itself from within the callback.
+            if ( __cb->__isRemoved_ != nullptr ) {
+                // Currently inside the callback, let the __request_stop() method
+                // know the object is about to be destructed and that it should
+                // not try to access the object when the callback returns.
+                *__cb->__isRemoved_ = true;
+            }
+        }
+        else {
+            // Callback is currently executing on another thread,
+            // block until it finishes executing.
+            while ( ! __cb->__callbackFinishedExecuting_.load(std::memory_order_acquire) ) {
+                __spin_yield();
+            }
+        }
+
+        __remove_token_reference();
+    }
+
+private:
+    static bool __is_locked(std::uint64_t __state) noexcept { return (__state & __locked_flag) != 0; }
+
+    static bool __is_stop_requested(std::uint64_t __state) noexcept { return (__state & __stop_requested_flag) != 0; }
+
+    static bool __is_stop_requestable(std::uint64_t __state) noexcept {
+        // Interruptible if it has already been interrupted or if there are
+        // still interrupt_source instances in existence.
+        return __is_stop_requested(__state) || (__state >= __source_ref_increment);
+    }
+
+    bool __try_lock_and_signal_until_signalled() noexcept {
+        std::uint64_t __oldState = __state_.load(std::memory_order_acquire);
+        do {
+            if ( __is_stop_requested(__oldState) )
+                return false;
+            while ( __is_locked(__oldState) ) {
+                __spin_yield();
+                __oldState = __state_.load(std::memory_order_acquire);
+                if ( __is_stop_requested(__oldState) )
+                    return false;
+            }
+        } while ( ! __state_.compare_exchange_weak(__oldState, __oldState | __stop_requested_flag | __locked_flag,
+                                                   std::memory_order_acq_rel, std::memory_order_acquire) );
+        return true;
+    }
+
+    void __lock() noexcept {
+        auto __oldState = __state_.load(std::memory_order_relaxed);
+        do {
+            while ( __is_locked(__oldState) ) {
+                __spin_yield();
+                __oldState = __state_.load(std::memory_order_relaxed);
+            }
+        } while ( ! __state_.compare_exchange_weak(__oldState, __oldState | __locked_flag, std::memory_order_acquire,
+                                                   std::memory_order_relaxed) );
+    }
+
+    void __unlock() noexcept { __state_.fetch_sub(__locked_flag, std::memory_order_release); }
+
+    void __unlock_and_increment_token_ref_count() noexcept {
+        __state_.fetch_sub(__locked_flag - __token_ref_increment, std::memory_order_release);
+    }
+
+    void __unlock_and_decrement_token_ref_count() noexcept {
+        auto __oldState = __state_.fetch_sub(__locked_flag + __token_ref_increment, std::memory_order_acq_rel);
+        // Check if new state is less than __token_ref_increment which would
+        // indicate that this was the last reference.
+        if ( __oldState < (__locked_flag + __token_ref_increment + __token_ref_increment) ) {
+            delete this;
+        }
+    }
+
+    static constexpr std::uint64_t __stop_requested_flag = 1u;
+    static constexpr std::uint64_t __locked_flag = 2u;
+    static constexpr std::uint64_t __token_ref_increment = 4u;
+    static constexpr std::uint64_t __source_ref_increment = static_cast<std::uint64_t>(1u) << 33u;
+
+    // bit 0 - stop-requested
+    // bit 1 - locked
+    // bits 2-32 - token ref count (31 bits)
+    // bits 33-63 - source ref count (31 bits)
+    std::atomic<std::uint64_t> __state_{__source_ref_increment};
+    __stop_callback_base* __head_ = nullptr;
+    std::thread::id __signallingThread_{};
+};
+
+
+//-----------------------------------------------
+// forward declarations
+//-----------------------------------------------
+
+class stop_source;
+template<typename _Callback>
+class stop_callback;
+
+// std::nostopstate
+// - to initialize a stop_source without shared stop state
+struct nostopstate_t {
+    explicit nostopstate_t() = default;
+};
+inline constexpr nostopstate_t nostopstate{};
+
+
+//-----------------------------------------------
+// stop_token
+//-----------------------------------------------
+
+class stop_token {
+public:
+    // construct:
+    // - TODO: explicit?
+    stop_token() noexcept : __state_(nullptr) {}
+
+    // copy/move/assign/destroy:
+    stop_token(const stop_token& __it) noexcept : __state_(__it.__state_) {
+        if ( __state_ != nullptr ) {
+            __state_->__add_token_reference();
+        }
+    }
+
+    stop_token(stop_token&& __it) noexcept : __state_(std::exchange(__it.__state_, nullptr)) {}
+
+    ~stop_token() {
+        if ( __state_ != nullptr ) {
+            __state_->__remove_token_reference();
+        }
+    }
+
+    stop_token& operator=(const stop_token& __it) noexcept {
+        if ( __state_ != __it.__state_ ) {
+            stop_token __tmp{__it};
+            swap(__tmp);
+        }
+        return *this;
+    }
+
+    stop_token& operator=(stop_token&& __it) noexcept {
+        stop_token __tmp{std::move(__it)};
+        swap(__tmp);
+        return *this;
+    }
+
+    void swap(stop_token& __it) noexcept { std::swap(__state_, __it.__state_); }
+
+    // stop handling:
+    [[nodiscard]] bool stop_requested() const noexcept {
+        return __state_ != nullptr && __state_->__is_stop_requested();
+    }
+
+    [[nodiscard]] bool stop_possible() const noexcept {
+        return __state_ != nullptr && __state_->__is_stop_requestable();
+    }
+
+    [[nodiscard]] friend bool operator==(const stop_token& __a, const stop_token& __b) noexcept {
+        return __a.__state_ == __b.__state_;
+    }
+    [[nodiscard]] friend bool operator!=(const stop_token& __a, const stop_token& __b) noexcept {
+        return __a.__state_ != __b.__state_;
+    }
+
+private:
+    friend class stop_source;
+    template<typename _Callback>
+    friend class stop_callback;
+
+    explicit stop_token(__stop_state* __state) noexcept : __state_(__state) {
+        if ( __state_ != nullptr ) {
+            __state_->__add_token_reference();
+        }
+    }
+
+    __stop_state* __state_;
+};
+
+
+//-----------------------------------------------
+// stop_source
+//-----------------------------------------------
+
+class stop_source {
+public:
+    stop_source() : __state_(new __stop_state()) {}
+
+    explicit stop_source(nostopstate_t) noexcept : __state_(nullptr) {}
+
+    ~stop_source() {
+        if ( __state_ != nullptr ) {
+            __state_->__remove_source_reference();
+        }
+    }
+
+    stop_source(const stop_source& __other) noexcept : __state_(__other.__state_) {
+        if ( __state_ != nullptr ) {
+            __state_->__add_source_reference();
+        }
+    }
+
+    stop_source(stop_source&& __other) noexcept : __state_(std::exchange(__other.__state_, nullptr)) {}
+
+    stop_source& operator=(stop_source&& __other) noexcept {
+        stop_source __tmp{std::move(__other)};
+        swap(__tmp);
+        return *this;
+    }
+
+    stop_source& operator=(const stop_source& __other) noexcept {
+        if ( __state_ != __other.__state_ ) {
+            stop_source __tmp{__other};
+            swap(__tmp);
+        }
+        return *this;
+    }
+
+    [[nodiscard]] bool stop_requested() const noexcept {
+        return __state_ != nullptr && __state_->__is_stop_requested();
+    }
+
+    [[nodiscard]] bool stop_possible() const noexcept { return __state_ != nullptr; }
+
+    bool request_stop() noexcept {
+        if ( __state_ != nullptr ) {
+            return __state_->__request_stop();
+        }
+        return false;
+    }
+
+    [[nodiscard]] stop_token get_token() const noexcept { return stop_token{__state_}; }
+
+    void swap(stop_source& __other) noexcept { std::swap(__state_, __other.__state_); }
+
+    [[nodiscard]] friend bool operator==(const stop_source& __a, const stop_source& __b) noexcept {
+        return __a.__state_ == __b.__state_;
+    }
+    [[nodiscard]] friend bool operator!=(const stop_source& __a, const stop_source& __b) noexcept {
+        return __a.__state_ != __b.__state_;
+    }
+
+private:
+    __stop_state* __state_;
+};
+
+
+//-----------------------------------------------
+// stop_callback
+//-----------------------------------------------
+
+template<typename _Callback>
+// requires Destructible<_Callback> && Invocable<_Callback>
+class [[nodiscard]] stop_callback : private __stop_callback_base {
+public:
+    using callback_type = _Callback;
+
+    template<typename _CB, std::enable_if_t<std::is_constructible_v<_Callback, _CB>, int> = 0>
+    // requires Constructible<Callback, C>
+    explicit stop_callback(const stop_token& __token,
+                           _CB&& __cb) noexcept(std::is_nothrow_constructible_v<_Callback, _CB>)
+        : __stop_callback_base{[](__stop_callback_base* __that) noexcept {
+              static_cast<stop_callback*>(__that)->__execute();
+          }},
+          __state_(nullptr),
+          __cb_(static_cast<_CB&&>(__cb)) {
+        if ( __token.__state_ != nullptr && __token.__state_->__try_add_callback(this, true) ) {
+            __state_ = __token.__state_;
+        }
+    }
+
+    template<typename _CB, std::enable_if_t<std::is_constructible_v<_Callback, _CB>, int> = 0>
+    // requires Constructible<Callback, C>
+    explicit stop_callback(stop_token&& __token, _CB&& __cb) noexcept(std::is_nothrow_constructible_v<_Callback, _CB>)
+        : __stop_callback_base{[](__stop_callback_base* __that) noexcept {
+              static_cast<stop_callback*>(__that)->__execute();
+          }},
+          __state_(nullptr),
+          __cb_(static_cast<_CB&&>(__cb)) {
+        if ( __token.__state_ != nullptr && __token.__state_->__try_add_callback(this, false) ) {
+            __state_ = std::exchange(__token.__state_, nullptr);
+        }
+    }
+
+    ~stop_callback() {
+#ifdef SAFE
+        if ( __inExecute_.load() ) {
+            std::cerr << "*** OOPS: ~stop_callback() while callback executed\n";
+        }
+#endif
+        if ( __state_ != nullptr ) {
+            __state_->__remove_callback(this);
+        }
+    }
+
+    stop_callback& operator=(const stop_callback&) = delete;
+    stop_callback& operator=(stop_callback&&) = delete;
+    stop_callback(const stop_callback&) = delete;
+    stop_callback(stop_callback&&) = delete;
+
+private:
+    void __execute() noexcept {
+        // Executed in a noexcept context
+        // If it throws then we call std::terminate().
+#ifdef SAFE
+        __inExecute_.store(true);
+        __cb_();
+        __inExecute_.store(false);
+#else
+        __cb_();
+#endif
+    }
+
+    __stop_state* __state_;
+    _Callback __cb_;
+#ifdef SAFE
+    std::atomic<bool> __inExecute_{false};
+#endif
+};
+
+template<typename _Callback>
+stop_callback(stop_token, _Callback) -> stop_callback<_Callback>;
+
+} // namespace nonstd

src/3rdparty/strsep.c

@@ -0,0 +1,93 @@
+/*-
+ * Copyright (c) 1990, 1993
+ *	The Regents of the University of California.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. All advertising materials mentioning features or use of this software
+ *    must display the following acknowledgement:
+ *	This product includes software developed by the University of
+ *	California, Berkeley and its contributors.
+ * 4. Neither the name of the University nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include "zeek/zeek-config.h"
+
+#ifndef HAVE_STRSEP
+
+#include <sys/types.h>
+
+#include <string.h>
+#include <stdio.h>
+
+char *strsep(char **, const char *);
+
+#if defined(LIBC_SCCS) && !defined(lint)
+static char sccsid[] = "@(#)strsep.c	8.1 (Berkeley) 6/4/93";
+#endif /* LIBC_SCCS and not lint */
+#ifndef lint
+static const char rcsid[] =
+  "$FreeBSD: src/lib/libc/string/strsep.c,v 1.2.12.1 2001/07/09 23:30:07 obrien Exp $";
+#endif
+
+/*
+ * Get next token from string *stringp, where tokens are possibly-empty
+ * strings separated by characters from delim.
+ *
+ * Writes NULs into the string at *stringp to end tokens.
+ * delim need not remain constant from call to call.
+ * On return, *stringp points past the last NUL written (if there might
+ * be further tokens), or is NULL (if there are definitely no more tokens).
+ *
+ * If *stringp is NULL, strsep returns NULL.
+ */
+char *
+strsep(stringp, delim)
+	register char **stringp;
+	register const char *delim;
+{
+	register char *s;
+	register const char *spanp;
+	register int c, sc;
+	char *tok;
+
+	if ((s = *stringp) == NULL)
+		return (NULL);
+	for (tok = s;;) {
+		c = *s++;
+		spanp = delim;
+		do {
+			if ((sc = *spanp++) == c) {
+				if (c == 0)
+					s = NULL;
+				else
+					s[-1] = 0;
+				*stringp = s;
+				return (tok);
+			}
+		} while (sc != 0);
+	}
+	/* NOTREACHED */
+}
+
+#endif

src/3rdparty/zeek_inet_ntop.c

@@ -0,0 +1,217 @@
+/* Taken/adapted from FreeBSD 9.0.0 inet_ntop.c (CVS revision 1.3.16.1.2.1) */
+/*
+ * Copyright (c) 2004 by Internet Systems Consortium, Inc. ("ISC")
+ * Copyright (c) 1996-1999 by Internet Software Consortium.
+ *
+ * Permission to use, copy, modify, and distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS.  IN NO EVENT SHALL ISC BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT
+ * OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#include "zeek_inet_ntop.h"
+
+#include <sys/param.h>
+#include <sys/types.h>
+#include <sys/socket.h>
+
+#include <netinet/in.h>
+#include <arpa/inet.h>
+#include <arpa/nameser.h>
+
+#include <errno.h>
+#include <stdio.h>
+#include <string.h>
+
+/*%
+ * WARNING: Don't even consider trying to compile this on a system where
+ * sizeof(int) < 4.  sizeof(int) > 4 is fine; all the world's not a VAX.
+ */
+
+static const char *zeek_inet_ntop4(const u_char *src, char *dst, socklen_t size);
+static const char *zeek_inet_ntop6(const u_char *src, char *dst, socklen_t size);
+
+/* char *
+ * zeek_inet_ntop(af, src, dst, size)
+ *	convert a network format address to presentation format.
+ * return:
+ *	pointer to presentation format address (`dst'), or NULL (see errno).
+ * author:
+ *	Paul Vixie, 1996.
+ */
+const char *
+zeek_inet_ntop(int af, const void * __restrict src, char * __restrict dst,
+    socklen_t size)
+{
+	switch (af) {
+	case AF_INET:
+		return (zeek_inet_ntop4(src, dst, size));
+	case AF_INET6:
+		return (zeek_inet_ntop6(src, dst, size));
+	default:
+		errno = EAFNOSUPPORT;
+		return (NULL);
+	}
+	/* NOTREACHED */
+}
+
+/* const char *
+ * zeek_inet_ntop4(src, dst, size)
+ *	format an IPv4 address
+ * return:
+ *	`dst' (as a const)
+ * notes:
+ *	(1) uses no statics
+ *	(2) takes a u_char* not an in_addr as input
+ * author:
+ *	Paul Vixie, 1996.  Modified by Jon Siwek, 2012, to replace strlcpy
+ */
+static const char *
+zeek_inet_ntop4(const u_char *src, char *dst, socklen_t size)
+{
+	static const char fmt[] = "%u.%u.%u.%u";
+	char tmp[sizeof "255.255.255.255"];
+	int l;
+
+	l = snprintf(tmp, sizeof(tmp), fmt, src[0], src[1], src[2], src[3]);
+	if (l <= 0 || (socklen_t) l >= size) {
+		errno = ENOSPC;
+		return (NULL);
+	}
+	strncpy(dst, tmp, size - 1);
+	dst[size - 1] = 0;
+	return (dst);
+}
+
+/* const char *
+ * zeek_inet_ntop6(src, dst, size)
+ *	convert IPv6 binary address into presentation (printable) format
+ * author:
+ *	Paul Vixie, 1996.  Modified by Jon Siwek, 2012, for IPv4-translated format
+ */
+static const char *
+zeek_inet_ntop6(const u_char *src, char *dst, socklen_t size)
+{
+	/*
+	 * Note that int32_t and int16_t need only be "at least" large enough
+	 * to contain a value of the specified size.  On some systems, like
+	 * Crays, there is no such thing as an integer variable with 16 bits.
+	 * Keep this in mind if you think this function should have been coded
+	 * to use pointer overlays.  All the world's not a VAX.
+	 */
+	char tmp[sizeof("ffff:ffff:ffff:ffff:ffff:ffff:255.255.255.255")], *tp;
+	struct { int base, len; } best, cur;
+	u_int words[NS_IN6ADDRSZ / NS_INT16SZ];
+	int i;
+
+	/*
+	 * Preprocess:
+	 *	Copy the input (bytewise) array into a wordwise array.
+	 *	Find the longest run of 0x00's in src[] for :: shorthanding.
+	 */
+	memset(words, '\0', sizeof words);
+	for (i = 0; i < NS_IN6ADDRSZ; i++)
+		words[i / 2] |= (src[i] << ((1 - (i % 2)) << 3));
+	best.base = -1;
+	best.len = 0;
+	cur.base = -1;
+	cur.len = 0;
+	for (i = 0; i < (NS_IN6ADDRSZ / NS_INT16SZ); i++) {
+		if (words[i] == 0) {
+			if (cur.base == -1)
+				cur.base = i, cur.len = 1;
+			else
+				cur.len++;
+		} else {
+			if (cur.base != -1) {
+				if (best.base == -1 || cur.len > best.len)
+					best = cur;
+				cur.base = -1;
+			}
+		}
+	}
+	if (cur.base != -1) {
+		if (best.base == -1 || cur.len > best.len)
+			best = cur;
+	}
+	if (best.base != -1 && best.len < 2)
+		best.base = -1;
+
+	/*
+	 * Format the result.
+	 */
+	int remaining = sizeof(tmp);
+	tp = tmp;
+	for (i = 0; i < (NS_IN6ADDRSZ / NS_INT16SZ) && remaining > 0; i++) {
+		/* Are we inside the best run of 0x00's? */
+		if (best.base != -1 && i >= best.base &&
+		    i < (best.base + best.len)) {
+			if (i == best.base)
+				{
+				*tp++ = ':';
+				remaining--;
+				}
+			continue;
+		}
+
+		/* Are we following an initial run of 0x00s or any real hex? */
+		if (i != 0)
+			{
+			*tp++ = ':';
+			remaining--;
+			}
+
+		/* Is this address an encapsulated IPv4? */
+		if (i == 6 && best.base == 0 && (best.len == 6 ||
+		    (best.len == 7 && words[7] != 0x0001) ||
+		    (best.len == 5 && words[5] == 0xffff) ||
+		    (best.len == 4 && words[4] == 0xffff && words[5] == 0))) {
+			if (!zeek_inet_ntop4(src+12, tp, sizeof tmp - (tp - tmp)))
+				return (NULL);
+			tp += strlen(tp);
+			remaining -= strlen(tp);
+			break;
+		}
+
+		// snprintf() returns the number of characters that were written not
+		// including the null character. We can use that to increase the
+		// pointer as we're moving forward. Unfortunately, snprintf() can also
+		// return more than the value passed if it would have stepped off the
+		// end.
+		int ret = snprintf(tp, remaining, "%x" , words[i]);
+		if ( ret < remaining )
+			tp += ret;
+
+		// Even if we returned too much data, subtract from remaining so that
+		// the failure cases below get triggered.
+		remaining -= ret;
+	}
+
+	/* Was it a trailing run of 0x00's? */
+	if (remaining >= 2 &&
+	    best.base != -1 &&
+	    (best.base + best.len) == (NS_IN6ADDRSZ / NS_INT16SZ)) {
+		*tp++ = ':';
+		remaining--;
+	}
+
+	if ( remaining >= 1 ) {
+		*tp++ = '\0';
+		remaining--;
+	}
+
+	else if ( remaining <= 0 ) {
+		errno = ENOSPC;
+		return (NULL);
+	}
+
+	strcpy(dst, tmp);
+	return (dst);
+}

src/3rdparty/zeek_inet_ntop.h

@@ -0,0 +1,16 @@
+#pragma once
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <sys/types.h>
+#include <sys/socket.h>
+
+const char *
+zeek_inet_ntop(int af, const void * __restrict src, char * __restrict dst,
+    socklen_t size);
+
+#ifdef __cplusplus
+}
+#endif

src/Attr.cc

@@ -41,6 +41,8 @@ const char* attr_name(AttrTag t) {
 		"&is_assigned",
 		"&is_used",
 		"&ordered",
+		"&no_ZAM_opt",
+		"&no_CPP_opt",
 	};
     // clang-format on
 
@@ -166,13 +168,9 @@ detail::TraversalCode Attr::Traverse(detail::TraversalCallback* cb) const {
     HANDLE_TC_ATTR_POST(tc);
 }
 
-Attributes::Attributes(TypePtr t, bool arg_in_record, bool is_global)
-    : Attributes(std::vector<AttrPtr>{}, std::move(t), arg_in_record, is_global) {}
-
-Attributes::Attributes(std::vector<AttrPtr> a, TypePtr t, bool arg_in_record, bool is_global) : type(std::move(t)) {
+Attributes::Attributes(std::vector<AttrPtr> a, TypePtr t, bool in_record, bool is_global, bool is_param)
+    : type(std::move(t)), in_record(in_record), global_var(is_global), is_param(is_param) {
     attrs.reserve(a.size());
-    in_record = arg_in_record;
-    global_var = is_global;
 
     SetLocationInfo(&start_location, &end_location);
 
@@ -198,9 +196,19 @@ void Attributes::AddAttr(AttrPtr attr, bool is_redef) {
             return true;
         }
 
-        return new_tag == ATTR_LOG || new_tag == ATTR_OPTIONAL || new_tag == ATTR_REDEF ||
-               new_tag == ATTR_BROKER_STORE_ALLOW_COMPLEX || new_tag == ATTR_RAW_OUTPUT ||
-               new_tag == ATTR_ERROR_HANDLER || new_tag == ATTR_IS_USED;
+        static const std::set<AttrTag> acceptable = {
+            ATTR_BROKER_STORE_ALLOW_COMPLEX,
+            ATTR_ERROR_HANDLER,
+            ATTR_IS_USED,
+            ATTR_LOG,
+            ATTR_NO_CPP_OPT,
+            ATTR_NO_ZAM_OPT,
+            ATTR_OPTIONAL,
+            ATTR_RAW_OUTPUT,
+            ATTR_REDEF,
+        };
+
+        return acceptable.contains(new_tag);
     };
 
     // A `redef` is allowed to overwrite an existing attribute instead of
@@ -221,7 +229,7 @@ void Attributes::AddAttr(AttrPtr attr, bool is_redef) {
     // that's a signal to skip the checking. If the type is error,
     // there's no point checking attributes either.
     if ( type && ! IsErrorType(type->Tag()) ) {
-        if ( ! CheckAttr(attr.get()) ) {
+        if ( ! CheckAttr(attr.get(), type) ) {
             // Get rid of it, so we don't get error cascades down the line.
             RemoveAttr(attr->Tag());
             return;
@@ -289,7 +297,7 @@ void Attributes::DescribeReST(ODesc* d, bool shorten) const {
     }
 }
 
-bool Attributes::CheckAttr(Attr* a) {
+bool Attributes::CheckAttr(Attr* a, const TypePtr& attrs_t) {
     switch ( a->Tag() ) {
         case ATTR_DEPRECATED:
         case ATTR_REDEF:
@@ -297,13 +305,11 @@ bool Attributes::CheckAttr(Attr* a) {
         case ATTR_IS_USED: break;
 
         case ATTR_OPTIONAL:
-            if ( global_var )
-                return AttrError("&optional is not valid for global variables");
+            if ( ! in_record )
+                return AttrError("&optional is only valid for record fields");
 
-            // Remove in v8.1: Call AttrError()
-            if ( in_record && Find(ATTR_DEFAULT) )
-                zeek::reporter->Deprecation(
-                    "Remove in v8.1: Using &default and &optional together results in &default behavior");
+            if ( Find(ATTR_DEFAULT) )
+                return AttrError("Using &default and &optional together results in &default behavior");
 
             break;
 
@@ -333,7 +339,7 @@ bool Attributes::CheckAttr(Attr* a) {
                 return AttrError("&default and &default_insert cannot be used together");
 
             std::string err_msg;
-            if ( ! check_default_attr(a, type, global_var, in_record, err_msg) && ! err_msg.empty() )
+            if ( ! check_default_attr(a, type, global_var, in_record || is_param, err_msg) && ! err_msg.empty() )
                 return AttrError(err_msg.c_str());
             break;
         }
@@ -342,13 +348,11 @@ bool Attributes::CheckAttr(Attr* a) {
             if ( Find(ATTR_DEFAULT_INSERT) )
                 return AttrError("&default and &default_insert cannot be used together");
 
-            // Remove in v8.1: Call AttrError()
             if ( in_record && Find(ATTR_OPTIONAL) )
-                zeek::reporter->Deprecation(
-                    "Remove in v8.1: Using &default and &optional together results in &default behavior");
+                return AttrError("Using &default and &optional together results in &default behavior");
 
             std::string err_msg;
-            if ( ! check_default_attr(a, type, global_var, in_record, err_msg) && ! err_msg.empty() )
+            if ( ! check_default_attr(a, type, global_var, in_record || is_param, err_msg) && ! err_msg.empty() )
                 return AttrError(err_msg.c_str());
             break;
         }
@@ -549,6 +553,15 @@ bool Attributes::CheckAttr(Attr* a) {
                 return AttrError("&ordered only applicable to tables");
             break;
 
+        case ATTR_NO_ZAM_OPT:
+        case ATTR_NO_CPP_OPT: {
+            if ( attrs_t->Tag() != TYPE_FUNC ) {
+                bool is_no_zam = a->Tag() == ATTR_NO_ZAM_OPT;
+                Error(util::fmt("&no_%s_opt must apply to a function", is_no_zam ? "ZAM" : "CPP"), attrs_t.get());
+                return false;
+            }
+        } break;
+
         default: BadTag("Attributes::CheckAttr", attr_name(a->Tag()));
     }
 
@@ -590,7 +603,7 @@ bool Attributes::operator==(const Attributes& other) const {
     return true;
 }
 
-bool check_default_attr(Attr* a, const TypePtr& type, bool global_var, bool in_record, std::string& err_msg) {
+bool check_default_attr(Attr* a, const TypePtr& type, bool global_var, bool record_like, std::string& err_msg) {
     ASSERT(a->Tag() == ATTR_DEFAULT || a->Tag() == ATTR_DEFAULT_INSERT);
     std::string aname = attr_name(a->Tag());
     // &default is allowed for global tables, since it's used in
@@ -602,7 +615,7 @@ bool check_default_attr(Attr* a, const TypePtr& type, bool global_var, bool in_r
 
     const auto& atype = a->GetExpr()->GetType();
 
-    if ( type->Tag() != TYPE_TABLE || (type->IsSet() && ! in_record) ) {
+    if ( type->Tag() != TYPE_TABLE || (type->IsSet() && ! record_like) ) {
         if ( same_type(atype, type) )
             // Ok.
             return true;
@@ -632,7 +645,7 @@ bool check_default_attr(Attr* a, const TypePtr& type, bool global_var, bool in_r
     TableType* tt = type->AsTableType();
     const auto& ytype = tt->Yield();
 
-    if ( ! in_record ) { // &default applies to the type itself.
+    if ( ! record_like ) { // &default applies to the type itself.
         if ( same_type(atype, ytype) )
             return true;
 

src/Attr.h

@@ -50,6 +50,8 @@ enum AttrTag : uint8_t {
     ATTR_IS_ASSIGNED, // to suppress usage warnings
     ATTR_IS_USED,     // to suppress usage warnings
     ATTR_ORDERED,     // used to store tables in ordered mode
+    ATTR_NO_ZAM_OPT,  // avoid ZAM optimization
+    ATTR_NO_CPP_OPT,  // avoid -O gen-C++ optimization
     NUM_ATTRS         // this item should always be last
 };
 
@@ -108,8 +110,14 @@ protected:
 // Manages a collection of attributes.
 class Attributes final : public Obj {
 public:
-    Attributes(std::vector<AttrPtr> a, TypePtr t, bool in_record, bool is_global);
-    Attributes(TypePtr t, bool in_record, bool is_global);
+    Attributes(std::vector<AttrPtr> a, TypePtr t, bool in_record, bool is_global, bool is_param);
+
+    Attributes(TypePtr t, bool in_record, bool is_global)
+        : Attributes(std::vector<AttrPtr>{}, std::move(t), in_record, is_global, false) {}
+
+    Attributes(std::vector<AttrPtr> a, TypePtr t, bool in_record, bool is_global)
+        : Attributes(std::move(a), std::move(t), in_record, is_global, false) {}
+
 
     ~Attributes() override = default;
 
@@ -132,7 +140,7 @@ public:
 
 protected:
     // Returns true if the attribute is okay, false if not.
-    bool CheckAttr(Attr* attr);
+    bool CheckAttr(Attr* attr, const TypePtr& attrs_t);
 
     // Reports an attribute error and returns false (handy for CheckAttr()).
     bool AttrError(const char* msg);
@@ -142,6 +150,7 @@ protected:
 
     bool in_record;
     bool global_var;
+    bool is_param;
 };
 
 // Checks whether default attribute "a" is compatible with the given type.

src/CMakeLists.txt

@@ -467,10 +467,18 @@ set(THIRD_PARTY_SRCS
     3rdparty/strsep.c
     3rdparty/zeek_inet_ntop.c)
 
-if (USE_SQLITE AND WNOERROR_FLAG)
+if (USE_SQLITE)
+    if (WNOERROR_FLAG)
         set_source_files_properties(3rdparty/sqlite3.c PROPERTIES COMPILE_FLAGS ${WNOERROR_FLAG})
     endif ()
 
+    # Always force building SQLite in thread-safe mode. This lets us remove sqlite3_threadsafe()
+    # checks in various places, since the library will always be guaranteed to be in thread-safe
+    # mode. This value is the default, but it's set here to be explicit about it.
+    set_source_files_properties(3rdparty/sqlite3.c PROPERTIES COMPILE_DEFINITIONS
+                                                              SQLITE_THREADSAFE=1)
+endif ()
+
 set_source_files_properties(${THIRD_PARTY_SRCS} PROPERTIES SKIP_LINTING ON)
 
 # Highwayhash. Highwayhash is a bit special since it has architecture dependent

src/Conn.cc

@@ -4,6 +4,7 @@
 
 #include <binpac.h>
 #include <cctype>
+#include <memory>
 
 #include "zeek/Desc.h"
 #include "zeek/ID.h"
@@ -12,6 +13,7 @@
 #include "zeek/RunState.h"
 #include "zeek/Timer.h"
 #include "zeek/TunnelEncapsulation.h"
+#include "zeek/WeirdState.h"
 #include "zeek/analyzer/Analyzer.h"
 #include "zeek/analyzer/Manager.h"
 #include "zeek/analyzer/protocol/pia/PIA.h"
@@ -39,47 +41,10 @@ Connection::Connection(zeek::IPBasedConnKeyPtr k, double t, uint32_t flow, const
     resp_port = key->DstPort();
     proto = key->GetTransportProto();
 
-    Init(flow, pkt);
-}
-
-#pragma GCC diagnostic push
-#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
-Connection::Connection(const detail::ConnKey& k, double t, const ConnTuple* id, uint32_t flow, const Packet* pkt)
-    : Session(t, connection_timeout, connection_status_update, detail::connection_status_update_interval) {
-    orig_addr = id->src_addr;
-    resp_addr = id->dst_addr;
-    orig_port = id->src_port;
-    resp_port = id->dst_port;
-
-    key = std::make_unique<zeek::IPConnKey>();
-    key->InitTuple(id->src_addr, id->src_port, id->dst_addr, id->dst_port, id->proto, id->is_one_way);
-    key->Init(*pkt);
-
-    proto = key->GetTransportProto();
-
-    Init(flow, pkt);
-}
-#pragma GCC diagnostic pop
-
-Connection::~Connection() {
-    if ( ! finished )
-        reporter->InternalError("Done() not called before destruction of Connection");
-
-    CancelTimers();
-
-    if ( conn_val )
-        conn_val->SetOrigin(nullptr);
-
-    delete adapter;
-
-    --current_connections;
-}
-
-void Connection::Init(uint32_t flow, const Packet* pkt) {
     orig_flow_label = flow;
     resp_flow_label = 0;
-    saw_first_orig_packet = 1;
-    saw_first_resp_packet = 0;
+    saw_first_orig_packet = true;
+    saw_first_resp_packet = false;
 
     if ( pkt->l2_src )
         memcpy(orig_l2_addr, pkt->l2_src, sizeof(orig_l2_addr));
@@ -94,11 +59,11 @@ void Connection::Init(uint32_t flow, const Packet* pkt) {
     vlan = pkt->vlan;
     inner_vlan = pkt->inner_vlan;
 
-    weird = 0;
+    weird = false;
 
     suppress_event = 0;
 
-    finished = 0;
+    finished = false;
 
     adapter = nullptr;
     primary_PIA = nullptr;
@@ -109,6 +74,20 @@ void Connection::Init(uint32_t flow, const Packet* pkt) {
     encapsulation = pkt->encap;
 }
 
+Connection::~Connection() {
+    if ( ! finished )
+        reporter->InternalError("Done() not called before destruction of Connection");
+
+    CancelTimers();
+
+    if ( conn_val )
+        conn_val->SetOrigin(nullptr);
+
+    delete adapter;
+
+    --current_connections;
+}
+
 void Connection::CheckEncapsulation(const std::shared_ptr<EncapsulationStack>& arg_encap) {
     if ( encapsulation && arg_encap ) {
         if ( *encapsulation != *arg_encap ) {
@@ -140,7 +119,7 @@ void Connection::CheckEncapsulation(const std::shared_ptr<EncapsulationStack>& a
 }
 
 void Connection::Done() {
-    finished = 1;
+    finished = true;
 
     if ( adapter ) {
         if ( ConnTransport() == TRANSPORT_TCP ) {
@@ -294,7 +273,7 @@ void Connection::RemovalEvent() {
 }
 
 void Connection::Weird(const char* name, const char* addl, const char* source) {
-    weird = 1;
+    weird = true;
     reporter->Weird(this, name, addl ? addl : "", source ? source : "");
 }
 
@@ -416,13 +395,16 @@ void Connection::CheckFlowLabel(bool is_orig, uint32_t flow_label) {
     }
 
     if ( is_orig )
-        saw_first_orig_packet = 1;
+        saw_first_orig_packet = true;
     else
-        saw_first_resp_packet = 1;
+        saw_first_resp_packet = true;
 }
 
 bool Connection::PermitWeird(const char* name, uint64_t threshold, uint64_t rate, double duration) {
-    return detail::PermitWeird(weird_state, name, threshold, rate, duration);
+    if ( ! weird_state )
+        weird_state = std::make_unique<detail::WeirdStateMap>();
+
+    return detail::PermitWeird(*weird_state, name, threshold, rate, duration);
 }
 
 } // namespace zeek

src/Conn.h

@@ -53,21 +53,6 @@ enum ConnEventToFlag : uint8_t {
     NUM_EVENTS_TO_FLAG,
 };
 
-// Deprecated without replacement: remove in v8.1.
-// XXX using [[deprecated]] for the whole struct leads to hard errors on FreeBSD/MacOS.
-#pragma GCC diagnostic push
-#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
-struct ConnTuple {
-#pragma GCC diagnostic pop
-    [[deprecated("Remove in v8.1: Switch to new conn_key framework")]] IPAddr src_addr;
-    [[deprecated("Remove in v8.1: Switch to new conn_key framework")]] IPAddr dst_addr;
-    [[deprecated("Remove in v8.1: Switch to new conn_key framework")]] uint32_t src_port = 0;
-    [[deprecated("Remove in v8.1: Switch to new conn_key framework")]] uint32_t dst_port = 0;
-    [[deprecated("Remove in v8.1: Switch to new conn_key framework")]] uint16_t proto = UNKNOWN_IP_PROTO;
-    [[deprecated("Remove in v8.1: Switch to new conn_key framework")]] bool is_one_way =
-        false; // if true, don't canonicalize order
-};
-
 static inline int addr_port_canon_lt(const IPAddr& addr1, uint32_t p1, const IPAddr& addr2, uint32_t p2) {
     return addr1 < addr2 || (addr1 == addr2 && p1 < p2);
 }
@@ -76,9 +61,6 @@ class Connection final : public session::Session {
 public:
     Connection(zeek::IPBasedConnKeyPtr k, double t, uint32_t flow, const Packet* pkt);
 
-    [[deprecated("Remove in v8.1. Switch to ConnKey factories and the new zeek::ConnKey tree.")]]
-    Connection(const detail::ConnKey& k, double t, const ConnTuple* id, uint32_t flow, const Packet* pkt);
-
     ~Connection() override;
 
     /**
@@ -165,7 +147,7 @@ public:
     void RemovalEvent() override;
 
     void Weird(const char* name, const char* addl = "", const char* source = "");
-    bool DidWeird() const { return weird != 0; }
+    bool DidWeird() const { return weird; }
 
     inline bool FlagEvent(ConnEventToFlag e) {
         if ( e >= 0 && e < NUM_EVENTS_TO_FLAG ) {
@@ -213,16 +195,11 @@ public:
     static void InitPostScript();
 
 private:
-    // Common initialization for the constructors. This can move back into the
-    // (sole) constructor when we remove the deprecated one in 8.1.
-    void Init(uint32_t flow, const Packet* pkt);
-
     friend class session::detail::Timer;
 
     IPAddr orig_addr;
     IPAddr resp_addr;
     uint32_t orig_port, resp_port;             // in network order
-    TransportProto proto;
     uint32_t orig_flow_label, resp_flow_label; // most recent IPv6 flow labels
     uint32_t vlan, inner_vlan;                 // VLAN this connection traverses, if available
     u_char orig_l2_addr[Packet::L2_ADDR_LEN];  // Link-layer originator address, if available
@@ -230,19 +207,21 @@ private:
     int suppress_event;                        // suppress certain events to once per conn.
     RecordValPtr conn_val;
     std::shared_ptr<EncapsulationStack> encapsulation; // tunnels
-    uint8_t tunnel_changes = 0;
 
     IPBasedConnKeyPtr key;
 
-    unsigned int weird : 1;
-    unsigned int finished : 1;
-    unsigned int saw_first_orig_packet : 1, saw_first_resp_packet : 1;
+    TransportProto proto;
+    uint8_t tunnel_changes = 0;
+    bool weird;
+    bool finished;
+    bool saw_first_orig_packet;
+    bool saw_first_resp_packet;
 
     packet_analysis::IP::SessionAdapter* adapter;
     analyzer::pia::PIA* primary_PIA;
 
     UID uid; // Globally unique connection ID.
-    detail::WeirdStateMap weird_state;
+    std::unique_ptr<detail::WeirdStateMap> weird_state;
 
     // Count number of connections.
     static uint64_t total_connections;

src/DNS_Mgr.cc

@@ -473,20 +473,20 @@ void DNS_Mgr::Done() {
 }
 
 void DNS_Mgr::RegisterSocket(int fd, bool read, bool write) {
-    if ( read && socket_fds.count(fd) == 0 ) {
+    if ( read && ! socket_fds.contains(fd) ) {
         socket_fds.insert(fd);
         iosource_mgr->RegisterFd(fd, this, IOSource::READ);
     }
-    else if ( ! read && socket_fds.count(fd) != 0 ) {
+    else if ( ! read && socket_fds.contains(fd) ) {
         socket_fds.erase(fd);
         iosource_mgr->UnregisterFd(fd, this, IOSource::READ);
     }
 
-    if ( write && write_socket_fds.count(fd) == 0 ) {
+    if ( write && ! write_socket_fds.contains(fd) ) {
         write_socket_fds.insert(fd);
         iosource_mgr->RegisterFd(fd, this, IOSource::WRITE);
     }
-    else if ( ! write && write_socket_fds.count(fd) != 0 ) {
+    else if ( ! write && write_socket_fds.contains(fd) ) {
         write_socket_fds.erase(fd);
         iosource_mgr->UnregisterFd(fd, this, IOSource::WRITE);
     }
@@ -1323,7 +1323,7 @@ double DNS_Mgr::GetNextTimeout() {
 }
 
 void DNS_Mgr::ProcessFd(int fd, int flags) {
-    if ( socket_fds.count(fd) != 0 ) {
+    if ( socket_fds.contains(fd) ) {
         int read_fd = (flags & IOSource::ProcessFlags::READ) != 0 ? fd : ARES_SOCKET_BAD;
         int write_fd = (flags & IOSource::ProcessFlags::WRITE) != 0 ? fd : ARES_SOCKET_BAD;
         ares_process_fd(channel, read_fd, write_fd);
@@ -1410,7 +1410,7 @@ TableValPtr DNS_Mgr::empty_addr_set() {
     auto addr_t = base_type(TYPE_ADDR);
     auto set_index = make_intrusive<TypeList>(addr_t);
     set_index->Append(std::move(addr_t));
-    auto s = make_intrusive<SetType>(std::move(set_index), nullptr);
+    auto s = make_intrusive<TableType>(std::move(set_index), nullptr);
     return make_intrusive<TableVal>(std::move(s));
 }
 

src/DebugLogger.cc

@@ -148,7 +148,7 @@ bool DebugLogger::CheckStreams(const std::set<std::string>& plugin_names) {
         if ( ! stream.starts_with("plugin-") )
             continue;
 
-        if ( available_plugin_streams.count(stream) == 0 ) {
+        if ( ! available_plugin_streams.contains(stream) ) {
             reporter->Error("No plugin debug stream '%s' found", stream.c_str());
             ok = false;
         }
@@ -180,7 +180,7 @@ void DebugLogger::Log(DebugStream stream, const char* fmt, ...) {
 void DebugLogger::Log(const plugin::Plugin& plugin, const char* fmt, ...) {
     if ( ! all ) {
         std::string tok = PluginStreamName(plugin.Name());
-        if ( enabled_streams.find(tok) == enabled_streams.end() )
+        if ( ! enabled_streams.contains(tok) )
             return;
     }
 

src/Desc.h

@@ -144,10 +144,6 @@ public:
         return byte_vec(t);
     }
 
-    [[deprecated("Remove in v8.1. Use Size() that returns size_t instead.")]]
-    int Len() const {
-        return static_cast<int>(offset);
-    }
     size_t Size() const { return offset; }
 
     void Clear();

src/Event.cc

@@ -41,16 +41,6 @@ RecordValPtr detail::MetadataEntry::BuildVal() const {
     return rv;
 }
 
-Event::Event(const EventHandlerPtr& arg_handler, zeek::Args arg_args, util::detail::SourceID arg_src,
-             analyzer::ID arg_aid, Obj* arg_obj, double arg_ts)
-    : handler(arg_handler),
-      args(std::move(arg_args)),
-      meta(detail::MakeEventMetadataVector(arg_ts)),
-      src(arg_src),
-      aid(arg_aid),
-      obj(zeek::NewRef{}, arg_obj),
-      next_event(nullptr) {}
-
 Event::Event(detail::EventMetadataVectorPtr arg_meta, const EventHandlerPtr& arg_handler, zeek::Args arg_args,
              util::detail::SourceID arg_src, analyzer::ID arg_aid, Obj* arg_obj)
     : handler(arg_handler),
@@ -58,8 +48,7 @@ Event::Event(detail::EventMetadataVectorPtr arg_meta, const EventHandlerPtr& arg
       meta(std::move(arg_meta)),
       src(arg_src),
       aid(arg_aid),
-      obj(zeek::NewRef{}, arg_obj),
-      next_event(nullptr) {}
+      obj(zeek::NewRef{}, arg_obj) {}
 
 zeek::VectorValPtr Event::MetadataValues(const EnumValPtr& id) const {
     static const auto& any_vec_t = zeek::id::find_type<zeek::VectorType>("any_vec");
@@ -135,14 +124,8 @@ void Event::Dispatch(bool no_remote) {
         reporter->BeginErrorHandler();
 
     try {
-#pragma GCC diagnostic push
-#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
-        // Replace in v8.1 with handler->Call(&args).
-        handler->Call(&args, no_remote, Time());
-#pragma GCC diagnostic pop
-    }
-
-    catch ( InterpreterException& e ) {
+        handler->Call(&args);
+    } catch ( InterpreterException& e ) {
         // Already reported.
     }
 
@@ -161,15 +144,12 @@ EventMgr::~EventMgr() {
     }
 }
 
-void EventMgr::Enqueue(const EventHandlerPtr& h, Args vl, util::detail::SourceID src, analyzer::ID aid, Obj* obj,
-                       DeprecatedTimestamp deprecated_ts) {
+void EventMgr::Enqueue(const EventHandlerPtr& h, Args vl, util::detail::SourceID src, analyzer::ID aid, Obj* obj) {
     detail::EventMetadataVectorPtr meta;
 
-    double ts = double(deprecated_ts);
-
     // If this is a local event and EventMetadata::add_network_timestamp is
     // enabled, automatically set the network timestamp for this event to the
-    // current network time when it is < 0 (default of deprecated_ts is -1.0).
+    // current network time.
     //
     // See the other Enqueue() implementation for the local vs broker/remote
     // motivation of want_network_timestamp.
@@ -178,19 +158,8 @@ void EventMgr::Enqueue(const EventHandlerPtr& h, Args vl, util::detail::SourceID
         ((src == util::detail::SOURCE_LOCAL) ||
          (src == util::detail::SOURCE_BROKER && BifConst::EventMetadata::add_missing_remote_network_timestamp));
 
-    if ( want_network_timestamp ) {
-        if ( ts < 0.0 )
-            ts = run_state::network_time;
-
-        // In v8.1 when the deprecated_ts parameters is gone: Just use run_state::network_time directly here.
-        meta = detail::MakeEventMetadataVector(ts);
-    }
-    else if ( ts >= 0.0 ) {
-        // EventMetadata::add_network_timestamp is false, but EventMgr::Enqueue()
-        // with an explicit (non-negative) timestamp is used. That's a deprecated
-        // API, but we continue to support it until v8.1.
-        meta = detail::MakeEventMetadataVector(ts);
-    }
+    if ( want_network_timestamp )
+        meta = detail::MakeEventMetadataVector(run_state::network_time);
 
     QueueEvent(new Event(std::move(meta), h, std::move(vl), src, aid, obj));
 }
@@ -261,14 +230,6 @@ void EventMgr::QueueEvent(Event* event) {
     ++event_mgr.num_events_queued;
 }
 
-void EventMgr::Dispatch(Event* event, bool no_remote) {
-    Event* old_current = current;
-    current = event;
-    event->Dispatch(no_remote);
-    current = old_current;
-    Unref(event);
-}
-
 void EventMgr::Dispatch(const EventHandlerPtr& h, zeek::Args vl) {
     detail::EventMetadataVectorPtr meta;
 
@@ -284,11 +245,11 @@ void EventMgr::Dispatch(const EventHandlerPtr& h, zeek::Args vl) {
     if ( done )
         return;
 
-#pragma GCC diagnostic push
-#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
-    // TODO: Open-code the old Dispatch() implementation here in v8.1.
-    Dispatch(ev);
-#pragma GCC diagnostic pop
+    Event* old_current = current;
+    current = ev;
+    ev->Dispatch();
+    current = old_current;
+    Unref(ev);
 }
 
 void EventMgr::Drain() {

src/Event.h

@@ -55,10 +55,6 @@ constexpr double NO_TIMESTAMP = -1.0;
 
 class Event final : public Obj {
 public:
-    [[deprecated("Remove in v8.1: Do not instantiate raw events. Use EventMgr::Dispatch() or EventMgr::Enqueue().")]]
-    Event(const EventHandlerPtr& handler, zeek::Args args, util::detail::SourceID src = util::detail::SOURCE_LOCAL,
-          analyzer::ID aid = 0, Obj* obj = nullptr, double ts = run_state::network_time);
-
     void SetNext(Event* n) { next_event = n; }
     Event* NextEvent() const { return next_event; }
 
@@ -99,12 +95,10 @@ private:
     util::detail::SourceID src;
     analyzer::ID aid;
     zeek::IntrusivePtr<Obj> obj;
-    Event* next_event;
+    Event* next_event = nullptr;
 };
 
 class EventMgr final : public Obj, public iosource::IOSource {
-    class DeprecatedTimestamp;
-
 public:
     ~EventMgr() override;
 
@@ -123,7 +117,7 @@ public:
      * (defaults to current network time - deprecated).
      */
     void Enqueue(const EventHandlerPtr& h, zeek::Args vl, util::detail::SourceID src = util::detail::SOURCE_LOCAL,
-                 analyzer::ID aid = 0, Obj* obj = nullptr, DeprecatedTimestamp ts = {});
+                 analyzer::ID aid = 0, Obj* obj = nullptr);
 
     /**
      * A version of Enqueue() taking a variable number of arguments.
@@ -147,9 +141,6 @@ public:
     void Enqueue(detail::EventMetadataVectorPtr meta, const EventHandlerPtr& h, zeek::Args vl,
                  util::detail::SourceID src = util::detail::SOURCE_LOCAL, analyzer::ID aid = 0, Obj* obj = nullptr);
 
-    [[deprecated("Remove in v8.1: Use Dispatch(handler, args) instead.")]]
-    void Dispatch(Event* event, bool no_remote = false);
-
     // Dispatch an event with the given handler and arguments immediately.
     //
     // While the event is technically not queued, HookQueueEvent() is
@@ -196,24 +187,6 @@ public:
     uint64_t num_events_dispatched = 0;
 
 private:
-    /**
-     * Helper class to produce a compile time warning if Enqueue() is called with an explicit timestamp.
-     *
-     * Remove in v8.1.
-     */
-    class DeprecatedTimestamp {
-    public:
-        DeprecatedTimestamp() : d(-1.0) {}
-        [[deprecated("Use overload EventMgr::Enqueue(EventMetadataVectorPtr meta, ...) to pass timestamp metadata")]]
-        /*implicit*/ DeprecatedTimestamp(double d)
-            : d(d) {}
-
-        explicit operator double() const { return d; }
-
-    private:
-        double d;
-    };
-
     void QueueEvent(Event* event);
 
     Event* current = nullptr;

src/EventHandler.cc

@@ -22,9 +22,7 @@ EventHandler::EventHandler(std::string arg_name) {
     generate_always = false;
 }
 
-EventHandler::operator bool() const {
-    return enabled && ((local && local->HasEnabledBodies()) || generate_always || ! auto_publish.empty());
-}
+EventHandler::operator bool() const { return enabled && ((local && local->HasEnabledBodies()) || generate_always); }
 
 const FuncTypePtr& EventHandler::GetType(bool check_export) {
     if ( type )
@@ -44,7 +42,7 @@ const FuncTypePtr& EventHandler::GetType(bool check_export) {
 
 void EventHandler::SetFunc(FuncPtr f) { local = std::move(f); }
 
-void EventHandler::Call(Args* vl, bool no_remote, double ts) {
+void EventHandler::Call(Args* vl) {
     if ( ! call_count ) {
         static auto eh_invocations_family =
             telemetry_mgr->CounterFamily("zeek", "event-handler-invocations", {"name"},
@@ -58,40 +56,6 @@ void EventHandler::Call(Args* vl, bool no_remote, double ts) {
     if ( new_event )
         NewEvent(vl);
 
-    if ( ! no_remote ) {
-        if ( ! auto_publish.empty() ) {
-            // Send event in form [name, xs...] where xs represent the arguments.
-            BrokerListBuilder xs;
-            xs.Reserve(vl->size());
-            bool valid_args = true;
-
-            for ( const auto& v : *vl ) {
-                if ( ! xs.Add(v) ) {
-                    valid_args = false;
-                    auto_publish.clear();
-                    reporter->Error("failed auto-remote event '%s', disabled", Name());
-                    break;
-                }
-            }
-
-            if ( valid_args ) {
-                auto ev_args = std::move(xs).Build();
-
-                for ( auto it = auto_publish.begin();; ) {
-                    const auto& topic = *it;
-                    ++it;
-
-                    if ( it != auto_publish.end() )
-                        broker_mgr->PublishEvent(topic, Name(), ev_args, ts);
-                    else {
-                        broker_mgr->PublishEvent(topic, Name(), std::move(ev_args), ts);
-                        break;
-                    }
-                }
-            }
-        }
-    }
-
     if ( local )
         // No try/catch here; we pass exceptions upstream.
         local->Invoke(vl);

src/EventHandler.h

@@ -35,28 +35,8 @@ public:
 
     void SetFunc(FuncPtr f);
 
-    [[deprecated("Remove in v8.1, use explicit Publish().")]]
-    void AutoPublish(std::string topic) {
-        auto_publish.insert(std::move(topic));
-    }
-
-    [[deprecated("Remove in v8.1.")]]
-    void AutoUnpublish(const std::string& topic) {
-        auto_publish.erase(topic);
-    }
-
-    [[deprecated(
-        "Remove in v8.1. The no_remote and ts parameters are AutoPublish() specific and won't have an effect "
-        "in the future. Use Call(args)")]]
-    void Call(zeek::Args* vl, bool no_remote = false, double ts = run_state::network_time);
-
     // Call the function associated with this handler.
-    void Call(zeek::Args* vl) {
-#pragma GCC diagnostic push
-#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
-        Call(vl, false, run_state::network_time);
-#pragma GCC diagnostic pop
-    }
+    void Call(zeek::Args* vl);
 
     // Returns true if there is at least one local or remote handler.
     explicit operator bool() const;
@@ -89,8 +69,6 @@ private:
 
     // Initialize this lazy, so we don't expose metrics for 0 values.
     std::shared_ptr<zeek::telemetry::Counter> call_count;
-
-    std::unordered_set<std::string> auto_publish;
 };
 
 // Encapsulates a ptr to an event handler to overload the boolean operator.

src/EventRegistry.cc

@@ -55,7 +55,7 @@ EventHandler* EventRegistry::Lookup(std::string_view name) {
 }
 
 bool EventRegistry::NotOnlyRegisteredFromScript(std::string_view name) {
-    return not_only_from_script.count(std::string(name)) > 0;
+    return not_only_from_script.contains(std::string(name));
 }
 
 EventRegistry::string_list EventRegistry::Match(RE_Matcher* pattern) {
@@ -174,12 +174,12 @@ namespace {
 class EventMetadataTypeRejector : public detail::TraversalCallback {
 public:
     detail::TraversalCode PreType(const Type* t) override {
-        if ( visited.count(t) > 0 )
+        if ( visited.contains(t) )
             return detail::TC_ABORTSTMT;
 
         visited.insert(t);
 
-        if ( reject.count(t->Tag()) )
+        if ( reject.contains(t->Tag()) )
             rejected.push_back(t);
 
         return detail::TC_CONTINUE;

src/EventRegistry.h

@@ -216,6 +216,16 @@ public:
      */
     void AddFunc(detail::ScriptFuncPtr f);
 
+    /**
+     * @return The name associated with the group.
+     */
+    const auto& GetName() const { return name; }
+
+    /**
+     * @return The type of group.
+     */
+    const auto& GetEventGroupKind() const { return kind; }
+
 private:
     void UpdateFuncBodies();
 

src/EventTrace.cc

@@ -732,7 +732,7 @@ void ValTraceMgr::TraceEventValues(std::shared_ptr<EventTrace> et, const zeek::A
     // remember them so we can catch uses of them in future events.
     for ( auto i = num_vals; i < vals.size(); ++i ) {
         processed_vals.insert(vals[i].get());
-        ASSERT(val_names.count(vals[i].get()) > 0);
+        ASSERT(val_names.contains(vals[i].get()));
     }
 }
 
@@ -794,8 +794,8 @@ void ValTraceMgr::NewVal(ValPtr v) {
 }
 
 void ValTraceMgr::ValUsed(const ValPtr& v) {
-    ASSERT(val_names.count(v.get()) > 0);
-    if ( processed_vals.count(v.get()) > 0 )
+    ASSERT(val_names.contains(v.get()));
+    if ( processed_vals.contains(v.get()) )
         // We saw this value when processing a previous event.
         globals.insert(v.get());
 }
@@ -818,17 +818,17 @@ void ValTraceMgr::AssessChange(const ValTrace* vt, const ValTrace* prev_vt) {
         bool needs_lhs = d->NeedsLHS();
         bool is_first_def = false;
 
-        if ( needs_lhs && val_names.count(v) == 0 ) {
+        if ( needs_lhs && ! val_names.contains(v) ) {
             TrackVar(v);
             is_first_def = true;
         }
 
-        ASSERT(val_names.count(v) > 0);
+        ASSERT(val_names.contains(v));
 
         // The "/" in the following is just to have a delimiter
         // to make sure the string is unambiguous.
         auto full_delta = val_names[v] + "/" + rhs;
-        if ( previous_deltas.count(full_delta) > 0 )
+        if ( previous_deltas.contains(full_delta) )
             continue;
 
         previous_deltas.insert(std::move(full_delta));
@@ -849,7 +849,7 @@ void ValTraceMgr::TrackVar(const Val* v) {
 
 std::string ValTraceMgr::GenValName(const ValPtr& v) {
     if ( IsAggr(v->GetType()) && ! IsUnspecifiedAggregate(v) ) { // Aggregate shouldn't exist; create it
-        ASSERT(val_map.count(v.get()) == 0);
+        ASSERT(! val_map.contains(v.get()));
         NewVal(v);
         return val_names[v.get()];
     }
@@ -1012,7 +1012,7 @@ void EventTraceMgr::Generate() {
 }
 
 void EventTraceMgr::StartEvent(const ScriptFunc* ev, const zeek::Args* args) {
-    if ( script_events.count(ev->GetName()) > 0 )
+    if ( script_events.contains(ev->GetName()) )
         return;
 
     auto nt = run_state::network_time;
@@ -1029,7 +1029,7 @@ void EventTraceMgr::StartEvent(const ScriptFunc* ev, const zeek::Args* args) {
 }
 
 void EventTraceMgr::EndEvent(const ScriptFunc* ev, const zeek::Args* args) {
-    if ( script_events.count(ev->GetName()) > 0 )
+    if ( script_events.contains(ev->GetName()) )
         return;
 
     if ( run_state::network_time > 0.0 && ev->GetName() != "zeek_init" )

src/EventTrace.h

@@ -354,7 +354,7 @@ public:
 
     // Returns true if the script variable associated with the given value
     // needs to be global (because it's used across multiple events).
-    bool IsGlobal(const ValPtr& v) const { return globals.count(v.get()) > 0; }
+    bool IsGlobal(const ValPtr& v) const { return globals.contains(v.get()); }
 
     // Returns or sets the "base time" from which eligible times are
     // transformed into offsets rather than maintained as absolute