Merge remote-tracking branch 'origin/master' into topic/johanna/gh-859

.cirrus.yml

@@ -83,7 +83,7 @@ env:
   # This is the https endpoint host and port used for benchmarking. It's kept
   # encrypted as a security measure to avoid leaking the host's information.
   ZEEK_BENCHMARK_HOST: ENCRYPTED[62ecdc93e839800d754d09d9a9070e9cb9b209e7d7dd2472ba38648f786ff272d0e0ea71233d0910025f2c6f3771259c]
-  ZEEK_BENCHMARK_PORT: ENCRYPTED[fb34ae2d51bac798fc01da052f3772154e17bbe2c1c5615509e82935248e748053fda399a0caf909632b6272cebff9f4]
+  ZEEK_BENCHMARK_PORT: ENCRYPTED[b97fabf4d6bd5eef107c8469c5cb2c44e0107d89c220f43e7d1e7bdfb32dbdc2620855fee8e5a8d889458d5a6ac3e5c7]
 
   # The repo token used for uploading data to Coveralls.io
   ZEEK_COVERALLS_REPO_TOKEN: ENCRYPTED[7ffd1e041f848f02b62f5abc7fda8a5a8a1561fbb2b46d88cefb67c74408ddeef6ea6f3b279c7953ca14ae9b4d050e2d]
@@ -91,6 +91,13 @@ env:
 # Linux EOL timelines: https://linuxlifecycle.com/
 # Fedora (~13 months): https://fedoraproject.org/wiki/Fedora_Release_Life_Cycle
 
+fedora34_task:
+  container:
+    # Fedora 34 EOL: Around May 2022
+    dockerfile: ci/fedora-34/Dockerfile
+    << : *RESOURCES_TEMPLATE
+  << : *CI_TEMPLATE
+
 fedora33_task:
   container:
     # Fedora 33 EOL: Around November 2022
@@ -98,13 +105,6 @@ fedora33_task:
     << : *RESOURCES_TEMPLATE
   << : *CI_TEMPLATE
 
-fedora32_task:
-  container:
-    # Fedora 32 EOL: Around May 2021
-    dockerfile: ci/fedora-32/Dockerfile
-    << : *RESOURCES_TEMPLATE
-  << : *CI_TEMPLATE
-
 centosstream8_task:
   container:
     # Stream 8 support should be 5 years, so until 2024. but I cannot find a concrete timeline --cpk
@@ -169,6 +169,13 @@ opensuse_leap_15_2_task:
     << : *RESOURCES_TEMPLATE
   << : *CI_TEMPLATE
 
+opensuse_leap_15_3_task:
+  container:
+    # Opensuse Leap 15.3 EOL: TBD
+    dockerfile: ci/opensuse-leap-15.3/Dockerfile
+    << : *RESOURCES_TEMPLATE
+  << : *CI_TEMPLATE
+
 ubuntu20_task:
   container:
     # Ubuntu 20.04 EOL: April 2025
@@ -205,19 +212,29 @@ alpine_task:
 # We aim to support both the current and previous macOS release.
 macos_big_sur_task:
   macos_instance:
-    image: big-sur-base
+    image: big-sur-xcode-12.5
   prepare_script: ./ci/macos/prepare.sh
   << : *CI_TEMPLATE
   << : *MACOS_RESOURCES_TEMPLATE
 
 macos_catalina_task:
   macos_instance:
-    image: catalina-xcode-11.6
+    image: catalina-xcode
   prepare_script: ./ci/macos/prepare.sh
   << : *CI_TEMPLATE
   << : *MACOS_RESOURCES_TEMPLATE
 
 # FreeBSD EOL timelines: https://www.freebsd.org/security/security.html#sup
+freebsd13_task:
+  freebsd_instance:
+    # FreeBSD 13 EOL: January 31, 2026
+    image_family: freebsd-13-0
+    cpu: 8
+    # Not allowed to request less than 8GB for an 8 CPU FreeBSD VM.
+    memory: 8GB
+  prepare_script: ./ci/freebsd/prepare.sh
+  << : *CI_TEMPLATE
+
 freebsd12_task:
   freebsd_instance:
     # FreeBSD 12 EOL: June 30, 2024

.gitignore

@@ -1,4 +1,7 @@
-build
+# Ignore anything prefixed with build since people
+# tend to name all of their build directories prefixed that way.
+build*
+
 tmp
 *.gcov
 
@@ -11,3 +14,6 @@ cmake-build-*
 
 # ignore pyenv local settings
 .python-version
+
+# clangd
+.cache

CHANGES

@@ -1,3 +1,434 @@
+4.1.0-dev.818 | 2021-06-28 13:50:13 -0700
+
+  * GH-1216: Enable Mobile IPv6 support by default (Tim Wojtulewicz, Corelight)
+
+    This removes the ENABLE_MOBILE_IPV6 #define variable. It also marks the
+    --enable-mobile-ipv6 configure argument as deprecated.
+
+4.1.0-dev.816 | 2021-06-28 11:08:29 -0700
+
+  * GH-572: Mark MemoryAllocation() and related methods deprecated (Tim Wojtulewicz, Corelight)
+
+4.1.0-dev.814 | 2021-06-28 11:06:39 -0700
+
+  * Check for -1 return from FieldOffset() in Val::HasField()
+
+    Fixes Coverity 1457804 (Tim Wojtulewicz, Corelight)
+
+4.1.0-dev.812 | 2021-06-28 11:02:46 -0700
+
+  * whoops overlooked the need to canonicalize filenames (Vern Paxson, Corelight)
+
+  * another set of tweaks per review comments (Vern Paxson, Corelight)
+
+  * addressed a number of code review comments (Vern Paxson, Corelight)
+
+  * baseline updates for merge (Vern Paxson, Corelight)
+
+  * Merge remote-tracking branch 'origin/master' into topic/vern/ZAM-prep (Vern Paxson, Corelight)
+
+  * support "any" coercions for "-O gen-C++" (Vern Paxson, Corelight)
+
+  * better descriptions for named record constructors (Vern Paxson, Corelight)
+
+  * test suite baseline updates for "-a opt" optimize-AST alternative (Vern Paxson, Corelight)
+
+  * test suite baseline updates for "-a xform" alternative / AST transformation (Vern Paxson, Corelight)
+
+  * error propagation fix for AST reduction (Vern Paxson, Corelight)
+
+  * updates to "-a inline" test suite alternative baseline (Vern Paxson, Corelight)
+
+  * updates for the main test suite baseline (Vern Paxson, Corelight)
+
+  * updates to test suite tests for compatibility with upcoming ZAM functionality (Vern Paxson, Corelight)
+
+  * "-O compile-all" option to specify compilation of inlined functions (Vern Paxson, Corelight)
+
+  * compile inlined functions if they're also used indirectly (Vern Paxson, Corelight)
+
+  * provide ZAM-generated code with low-level access to record fields (Vern Paxson, Corelight)
+
+  * fix for cloning records with fields of type "any" (Vern Paxson, Corelight)
+
+  * direct access for ZAM to VectorVal internal vector (Vern Paxson, Corelight)
+
+  * ZVal constructors, accessors & methods in support of ZAM (Vern Paxson, Corelight)
+
+  * switch ZVal representation of types from Type objects to TypeVal's (Vern Paxson, Corelight)
+
+  * revised error-reporting interface for ZVal's, to accommodate ZAM inner loop (Vern Paxson, Corelight)
+
+  * faster construction of records by factoring static decisions into RecordType's (Vern Paxson, Corelight)
+
+  * make "switch" internals accessible to ZAM; tidying of same (Vern Paxson, Corelight)
+
+  * factor out "cast" functionality to make available to lower-level ZAM access (Vern Paxson, Corelight)
+
+  * tidying for check_and_promote_expr (Vern Paxson, Corelight)
+
+  * employ explicit conversions to/from "any" and "vector of any" types (Vern Paxson, Corelight)
+
+  * more robust treatment of arithmetic coercions (Vern Paxson, Corelight)
+
+  * support for constructing VectorVal's directly from underlying ZVal vectors (Vern Paxson, Corelight)
+
+  * support for ensuring that a vector can be treated as having a homogeneous type (Vern Paxson, Corelight)
+
+  * factoring out of low-level vector indexing to make available to ZAM (Vern Paxson, Corelight)
+
+  * minor changes for more robust behavior in the face of errors (Vern Paxson, Corelight)
+
+  * gracefully deal with "eval" exceptions that occur during AST reduction (Vern Paxson, Corelight)
+
+  * directly construct records of known types, rather than requiring coercion (Vern Paxson, Corelight)
+
+  * fixes for treating WhileStmt's "loop_cond_pred_stmt" as a first-class citizen (Vern Paxson, Corelight)
+
+  * support for profiling function bodies w/o needing accompanying ScriptFunc object (Vern Paxson, Corelight)
+
+  * support for Frame's having call locations even if no associated CallExpr (Vern Paxson, Corelight)
+
+  * fix for AST optimization altering top-level body statement (Vern Paxson, Corelight)
+
+  * fix for analyzing variable usage inside of table initializers (Vern Paxson, Corelight)
+
+  * fix for inlining type-based switch statements (Vern Paxson, Corelight)
+
+  * fix for computing |size| of files and subnets (Vern Paxson, Corelight)
+
+  * fix for tracking the effects of += operations (Vern Paxson, Corelight)
+
+  * Expr method to invert the sense of a relational (Vern Paxson, Corelight)
+
+  * Trigger constructor (and factoring) to support lower-level constructions (Vern Paxson, Corelight)
+
+  * fixed / removed out-of-date comments, tidied check_and_promote_args() interface (Vern Paxson, Corelight)
+
+  * convert scopes to be managed using IntrusivePtr's (Vern Paxson, Corelight)
+
+  * various accessors used by ZAM compiler (Vern Paxson, Corelight)
+
+  * Update submodule(s) [nomail] (Tim Wojtulewicz, Corelight)
+
+4.1.0-dev.764 | 2021-06-27 10:50:19 -0700
+
+  * Remove unnecessary -B arguments from Zeek invocations in testsuite
+
+    Now that Zeek no longer silently accepts -B when not compiled in debug
+    mode, these tests were failing. (Christian Kreibich, Corelight)
+
+  * Fix perftools-enabled build (Christian Kreibich, Corelight)
+
+  * Minor tweaks to single-character command line option handling
+
+    - Use of "-B" now triggers usage output and errors out when Zeek
+      wasn't built with debugging support.
+
+    - Always keep the perftools options (-m, -M) in the optparse string,
+      for consistency with other flags dependent on configuration. We
+      still fall through to usage and erroring out as before.
+
+    - Minor indentation fix. (Christian Kreibich, Corelight)
+
+4.1.0-dev.760 | 2021-06-27 10:46:01 -0700
+
+  * Add btest test case (Luke Cesarz)
+
+  * Fix typo (Luke Cesarz)
+
+  * Fix segfault with incomplete connection
+
+    Add required HasField check before GetFieldAs call (Luke Cesarz)
+
+4.1.0-dev.755 | 2021-06-23 13:53:54 -0700
+
+  * Call brew update-reset in ci/macos/prepare.sh
+
+    This fixes some issues with the Catalina builds when it prepares the image
+    and the base Cirrus image has old recipes for Homebrew. The VM then has
+    to build a bunch of packages it shouldn't need to. (Tim Wojtulewicz, Corelight)
+
+  * GH-1368: Use --osx-sysroot for macOS CI builds (Tim Wojtulewicz, Corelight)
+
+4.1.0-dev.750 | 2021-06-21 16:14:03 -0700
+
+  * Remove the Stepping Stone analyzer
+
+    This commit removes the stepping stone analyzer. It has been deactivated
+    by default since at least Zeek 2.0, is dysfunctional in cluster settings
+    and has a bunch of other issued.
+
+    Relates to GH-1573 (Johanna Amann, Corelight)
+
+4.1.0-dev.748 | 2021-06-21 15:41:29 -0700
+
+  * Drop Fedora 32 from CI, now past EOL (Christian Kreibich, Corelight)
+
+  * Remove a double-defined TLS cert from a Broker btest (Christian Kreibich, Corelight)
+
+4.1.0-dev.744 | 2021-06-21 09:17:36 +0200
+
+  * Add a new field `email_dest` to NOTICEs, which defines where to
+    send email to. The email-related NOTICE actions fill this now, and
+    then emails will be sent to all recorded addresses at the end of
+    NOTICE processing. This makes email generation more consistent and
+    extensible. (Vlad Grigorescu)
+
+  * Add page and email administrator to mails processed by hostnames
+    extension. (Vlad Grigorescu)
+
+4.1.0-dev.731 | 2021-06-17 10:40:58 +0100
+
+  * Change SSH version field to be `&optional`.
+
+    In version 3.3.0-dev.537 we added handling for SSH version 1.99 which
+    used a SSH version of 0 to indicate weird cases where no version could be
+    determined.
+
+    This patch is a fixup for that patch. We now use an `&optional` version value.
+    If no SSH version can be eixtracted the version will be unset; additionally a
+    `conn_weird` event will be raised. See GH-1590. (Benjamin Bannier, Corelight)
+
+4.1.0-dev.727 | 2021-06-14 16:19:34 -0700
+
+  * Bump Highwayhash submodule to pull in fix for FreeBSD (Christian Kreibich, Corelight)
+
+4.1.0-dev.725 | 2021-06-11 11:54:30 -0700
+
+  * Fixes for the builtin plugin functionality (Seth Hall, Corelight)
+
+4.1.0-dev.722 | 2021-06-10 10:42:57 -0700
+
+  * Added --include-plugins configure argument (Seth Hall, Corelight)
+
+4.1.0-dev.720 | 2021-06-10 11:29:19 +0100
+
+  * Introduce script-land option LogAscii::logdir that can be used to set
+    the logging directory. (Henrik Kramselund Jereminsen)
+
+4.1.0-dev.715 | 2021-06-09 09:12:26 -0700
+
+  * Fix macOS Big Sur builds on Cirrus
+
+    - Upgrade the Big Sur VM to use the Xcode 12.5 version. This has a newer
+      version of brew installed on it that fixes an issue with an EOL package host
+      that finally shut down for good recently.
+    - Use 'brew upgrade' for openssl and cmake, since those are both present on the
+      base VM. This prevents 'brew install' from printing an error if the package
+      exists but is out of date. (Tim Wojtulewicz, Corelight)
+
+4.1.0-dev.713 | 2021-06-08 13:54:28 -0700
+
+  * Add OpenSUSE Leap 15.3 to testing (Johanna Amann, Corelight)
+
+4.1.0-dev.709 | 2021-06-07 09:41:28 +0200
+
+  * Improve assignment operators for IntrusivePtr. (Dominik Charousset, Corelight)
+
+  * Fix docs for `ProcStats`: `mem` is in bytes, not KB. (Arne Welzel, Corelight)
+
+4.1.0-dev.704 | 2021-06-04 08:29:18 -0700
+
+  * Add deprecated headers for UDP and ICMP analyzers (Tim Wojtulewicz, Corelight)
+
+  * Fix handling of IP packets with bogus IP header lengths
+
+    Credit to OSS-Fuzz for discovery
+    https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=34711
+    (Link to details becomes public 30 days after patch release) (Tim Wojtulewicz, Corelight)
+
+4.1.0-dev.700 | 2021-06-03 09:27:57 -0700
+
+  * Make update-traces fail when the curl invocation fails (Christian Kreibich, Corelight)
+
+4.1.0-dev.697 | 2021-06-02 15:08:12 -0700
+
+  * Add FreeBSD 13 to CI (Christian Kreibich, Corelight)
+
+  * Add Fedora 34 to CI (Christian Kreibich, Corelight)
+
+4.1.0-dev.693 | 2021-06-02 13:22:09 -0700
+
+  * Label session adapters in the output of zeek -NN (Tim Wojtulewicz, Corelight)
+
+  * Split session adapter code into separate files from the analyzers (Tim Wojtulewicz, Corelight)
+
+  * Move adapter-specific code back into the adapter (Tim Wojtulewicz, Corelight)
+
+  * Move ICMP counterpart methods outside of ICMPAnalyzer class
+
+    These were previously global methods in the old analyzer, and moving them
+    to be private members of ICMPAnalyzer broke the usage of them by at least
+    one external plugin. (Tim Wojtulewicz, Corelight)
+
+  * Remove obsolete Skipping()/SetSkip() from Connection (Tim Wojtulewicz, Corelight)
+
+  * Remove some code from IPBasedAnalyzer and children that was waiting for TCP to be implemented (Tim Wojtulewicz, Corelight)
+
+  * Move TCPStateStats object out of session_mgr (Tim Wojtulewicz, Corelight)
+
+  * Move analyzer-to-port mapping out of analyzer::Manager into packet analyzers (Tim Wojtulewicz, Corelight)
+
+  * Move packet parsing code out of adapter into analyzer (Tim Wojtulewicz, Corelight)
+
+  * Move old TCP analyzer into analyzer adapter in packet analysis tree (Tim Wojtulewicz, Corelight)
+
+4.1.0-dev.681 | 2021-06-02 09:57:41 -0700
+
+  * Add some extra length checking when parsing mobile ipv6 packets
+
+    Credit to OSS-Fuzz for discovery
+    https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=34263
+    (Link to details becomes public 30 days after patch release) (Tim Wojtulewicz, Corelight)
+
+4.1.0-dev.679 | 2021-06-01 19:11:40 -0700
+
+  * Replace toupper() usages in netbios decoding BIFs (Jon Siwek, Corelight)
+
+    This avoids potential for locale-dependent results of toupper() by
+    instead using a function that simply maps ASCII characters a-z to A-Z.
+
+4.1.0-dev.676 | 2021-06-01 10:19:19 -0700
+
+  * Integrate review feedback (Dominik Charousset, Corelight)
+
+  * Sync new broker options, fix name inconsistencies (Dominik Charousset, Corelight)
+
+  * Integrate new Broker metric exporter parameters (Dominik Charousset, Corelight)
+
+4.1.0-dev.671 | 2021-06-01 09:51:38 -0700
+
+  * Update detect-MHR.zeek (Chris C)
+
+    Update Virustotal URL to current
+    option match_sub_url = "https://www.virustotal.com/gui/search/%s"
+
+4.1.0-dev.669 | 2021-06-01 09:39:30 -0700
+
+  * GH-839: Fix use of &optional sub-records within table/set indices (Jon Siwek, Corelight)
+
+4.1.0-dev.666 | 2021-05-26 10:51:51 -0700
+
+  * Ensure SessionAdapter members are initialized (Tim Wojtulewicz, Corelight)
+
+    Fixes Coverity #1453273
+
+4.1.0-dev.665 | 2021-05-26 08:07:26 +0200
+
+  * Extend the file analyzer API to set source manually. (Robin
+    Sommer, Corelight)
+
+4.1.0-dev.661 | 2021-05-24 15:03:54 -0700
+
+  * Update Broker submodule for bump of embedded CAF to 0.18.3 (Jon Siwek, Corelight)
+
+4.1.0-dev.660 | 2021-05-24 12:38:44 -0700
+
+  * Add type field to session::Key to help avoid collisions in map (Tim Wojtulewicz, Corelight)
+
+  * Move bad UDP checksum handling into adapter object (Tim Wojtulewicz, Corelight)
+
+  * Rename IPBasedTransportAnalyzer to SessionAdapter (Tim Wojtulewicz, Corelight)
+
+    This also also combines the old TransportLayerAnalyzer class into
+    SessionAdapter, and removes the old class. This requires naming changes
+    in a few places but no functionality changes.
+
+  * Move building session analyzer tree out of analyzer::Manager (Tim Wojtulewicz, Corelight)
+
+  * Rework the packet flow through the IP-based analyzers (Tim Wojtulewicz, Corelight)
+
+  * Add new UDP packet analyzer, remove old one (Tim Wojtulewicz, Corelight)
+
+  * Add new ICMP packet analyzer, remove old one (Tim Wojtulewicz, Corelight)
+
+  * Add base class for IP-based packet analyzers (Tim Wojtulewicz, Corelight)
+
+  * Move SessionManager::ParseIPPacket to IP analyzer's namespace (Tim Wojtulewicz, Corelight)
+
+  * Added skeletons for TCP/UDP/ICMP packet analysis plugins. (Tim Wojtulewicz, Corelight)
+
+    This includes integration into the IP plugin and calling of the sessions code from each plugin.
+
+4.1.0-dev.646 | 2021-05-18 11:47:25 -0700
+
+  * Omit unneeded decimal points in modp_dtoa2() scientific notation output (Jon Siwek, Corelight)
+
+    For example, "1e-13" is now used instead of "1.e-13".
+
+  * GH-1244: Change modp_dtoa2() to use scientific notation for small values (Jon Siwek, Corelight)
+
+    This fixes problems where printing floating point numbers less than
+    10^-6 output as "0.0".  Such numbers now use using scientific notation
+    and preserve the value's actual floating point representation.
+
+4.1.0-dev.643 | 2021-05-17 11:57:58 -0700
+
+  * GH-1546: Make DictIterator() public, add copy/move operators (Tim Wojtulewicz, Corelight)
+
+4.1.0-dev.641 | 2021-05-17 11:28:11 -0700
+
+  * GH-1558: Fix reading `vector of enum` types from config files (Jon Siwek, Corelight)
+
+  * GH-1555: Fix reading empty set[enum] values from config files (Jon Siwek, Corelight)
+
+4.1.0-dev.638 | 2021-05-17 13:08:28 +0100
+
+  * Manual page updates (Henrik Kramselund Jereminsen)
+
+4.1.0-dev.631 | 2021-05-11 09:26:37 -0700
+
+  * Add unit tests to ZeekString.cc (Tim Wojtulewicz)
+
+4.1.0-dev.628 | 2021-05-10 12:44:25 -0700
+
+  * Add experimental support for translating Zeek scripts to equivalent C++ (Vern Paxson, Corelight)
+
+    The generated C++ can then be compiled directly into the `zeek` binary,
+    replacing use of the interpreter and producing better runtime performance.
+    See `src/script_opt/CPP/README.md` for a guide on how to use this feature.
+
+  * Add new "-a cpp" btest alternative (Vern Paxson, Corelight)
+
+  * Add VectorVal methods to leverage ZVal representation (Vern Paxson, Corelight)
+
+  * Fix backtrace BiF to avoid iterator invalidation & support compiled code (Vern Paxson, Corelight)
+
+4.1.0-dev.593 | 2021-05-10 10:17:34 +0100
+
+  * Explain zeek-config options in help output (Christian Kreibich, Corelight)
+
+  * Sort variables at top of zeek-config alphabetically (Christian Kreibich, Corelight)
+
+  * Install Zeek's btest tooling with the distribution
+
+    This creates $PREFIX/share/btest in the install tree, with the
+    following folders:
+
+    - scripts/ for the canonifiers
+    - data/ for random.seed
+    - data/pcaps for the test pcaps
+
+    The pcaps can be skipped by configuring with --disable-btest-pcaps. (Christian Kreibich, Corelight)
+
+4.1.0-dev.587 | 2021-05-05 14:05:51 +0000
+
+  * Merge remote-tracking branch 'origin/topic/timw/session-coverity'
+
+    * origin/topic/timw/session-coverity:
+      Minor cleanup in IPAddr.h
+      Fix a few Coverity warnings from the session manager work (Tim Wojtulewicz)
+
+  * Minor cleanup in IPAddr.h (Tim Wojtulewicz, Corelight)
+
+  * Fix a few Coverity warnings from the session manager work
+
+    - Be explicit about setting the copied flag in session::Key. Coverity seems
+      confused about when that flag is set if it gets set by default
+      initialization. This should fix 1452757 and 1452759.
+    - Explicitly copy the fields in ConnKey instead of using memcpy. Fixes
+      1452758. (Tim Wojtulewicz, Corelight)
 
 4.1.0-dev.583 | 2021-05-03 18:21:33 -0700
 

CMakeLists.txt

@@ -490,7 +490,7 @@ endif()
 # Tell the plugin code that we're building as part of the main tree.
 set(ZEEK_PLUGIN_INTERNAL_BUILD true CACHE INTERNAL "" FORCE)
 
-set(DEFAULT_ZEEKPATH .:${ZEEK_SCRIPT_INSTALL_PATH}:${ZEEK_SCRIPT_INSTALL_PATH}/policy:${ZEEK_SCRIPT_INSTALL_PATH}/site)
+set(DEFAULT_ZEEKPATH .:${ZEEK_SCRIPT_INSTALL_PATH}:${ZEEK_SCRIPT_INSTALL_PATH}/policy:${ZEEK_SCRIPT_INSTALL_PATH}/site:${ZEEK_SCRIPT_INSTALL_PATH}/builtin-plugins)
 
 if ( NOT BINARY_PACKAGING_MODE )
     set(ZEEK_DIST ${CMAKE_SOURCE_DIR})
@@ -539,6 +539,9 @@ if ( GooglePerftools_INCLUDE_DIR )
     set(ZEEK_CONFIG_GooglePerftools_INCLUDE_DIR ${GooglePerftools_INCLUDE_DIR})
 endif ()
 
+set(ZEEK_CONFIG_BTEST_TOOLS_DIR ${ZEEK_ROOT_DIR}/share/btest)
+install(DIRECTORY DESTINATION ${ZEEK_CONFIG_BTEST_TOOLS_DIR})
+
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/zeek-config.in
                ${CMAKE_CURRENT_BINARY_DIR}/zeek-config @ONLY)
 install(PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/zeek-config DESTINATION bin)
@@ -570,12 +573,41 @@ if ( INSTALL_ZKG )
         DESTINATION ${ZEEK_ZKG_CONFIG_DIR} RENAME config)
 endif ()
 
+########################################################################
+## Look for external plugins to build in
+
+string(REPLACE "," " " _build_in_plugins "${ZEEK_INCLUDE_PLUGINS}")
+separate_arguments(_build_in_plugins)
+foreach(plugin_dir ${_build_in_plugins})
+    if ( NOT IS_ABSOLUTE "${plugin_dir}/CMakeLists.txt" )
+      message(FATAL_ERROR "Plugins to build in need to be defined with absolute path! ${plugin_dir}")
+    endif()
+
+    if ( NOT EXISTS "${plugin_dir}/CMakeLists.txt" )
+      message(FATAL_ERROR "No plugin found at ${plugin_dir}!")
+    endif()
+
+    get_filename_component(plugin_name ${plugin_dir} NAME)
+
+    # Create a list of plugin directories that will then be added in the src/CMakeLists.txt
+    list(APPEND BUILTIN_PLUGIN_LIST ${plugin_dir})
+
+    message(STATUS "  Building in plugin: ${plugin_name} (${plugin_dir})")
+
+    if ( "${ZEEK_BUILTIN_PLUGINS}" STREQUAL "" )
+        set(ZEEK_BUILTIN_PLUGINS ${plugin_name})
+    else ()
+        set(ZEEK_BUILTIN_PLUGINS "${ZEEK_BUILTIN_PLUGINS}, ${plugin_name}")
+    endif ()
+endforeach()
+
 ########################################################################
 ## Recurse on sub-directories
 
 add_subdirectory(src)
 add_subdirectory(scripts)
 add_subdirectory(man)
+add_subdirectory(testing)
 
 include(CheckOptionalBuildSources)
 
@@ -608,6 +640,12 @@ if (CMAKE_BUILD_TYPE)
     string(TOUPPER ${CMAKE_BUILD_TYPE} BuildType)
 endif ()
 
+if ( INSTALL_BTEST_PCAPS )
+    set(_install_btest_tools_msg "all")
+else ()
+    set(_install_btest_tools_msg "no pcaps")
+endif ()
+
 message(
     "\n====================|  Zeek Build Summary  |===================="
     "\n"
@@ -619,6 +657,7 @@ message(
     "\nZeek Script Path:  ${ZEEK_SCRIPT_INSTALL_PATH}"
     "\nDebug mode:        ${ENABLE_DEBUG}"
     "\nUnit tests:        ${ENABLE_ZEEK_UNIT_TESTS}"
+    "\nBuiltin Plugins:   ${ZEEK_BUILTIN_PLUGINS}"
     "\n"
     "\nCC:                ${CMAKE_C_COMPILER}"
     "\nCFLAGS:            ${CMAKE_C_FLAGS} ${CMAKE_C_FLAGS_${BuildType}}"
@@ -629,6 +668,7 @@ message(
     "\nZeekControl:       ${INSTALL_ZEEKCTL}"
     "\nAux. Tools:        ${INSTALL_AUX_TOOLS}"
     "\nBTest:             ${INSTALL_BTEST}"
+    "\nBTest tooling:     ${_install_btest_tools_msg}"
     "\nzkg:               ${INSTALL_ZKG}"
     "\n"
     "\nlibmaxminddb:      ${USE_GEOIP}"

NEWS

@@ -49,15 +49,53 @@ New Functionality
 - A Telemetry API was added to assist in gathering arbitrary runtime metrics
   and allows potential export to Prometheus.
 
+- Experimental support for translating Zeek scripts to equivalent C++.
+  The generated C++ can then be compiled directly into the `zeek` binary,
+  replacing use of the interpreter and producing better runtime performance.
+  See `src/script_opt/CPP/README.md` for a guide on how to use this feature.
+
+- Support for more generic session management. The NetSessions class has been
+  renamed to SessionMgr (with the old name marked deprecated). The new
+  class allows plugins to take advantage of session management similar to how
+  Connection objects were handled previously, but without the need to be based
+  on IP-based protocols.
+
+- The ASCII writer gained a new option LogAscii::logdir, which can be used to
+  change the logging output directory.
+
+- Added a ``--include-plugins`` argument to ``configure``. This argument
+  takes a semicolon separated list of paths containing plugins that will be
+  statically built into Zeek.
+
 Changed Functionality
 ---------------------
 
+- The default IP-based transport protocols (UDP, TCP, and ICMP) have been
+  moved to the packet analysis framework. This change allows us to move other
+  analyzers in the future that better align with the packet analysis framework
+  than they do with session analysis.
+
+- The version field in ssh.log is now optional and will not be set if we cannot
+  determine the version that was negotiated by the client and server.
+
+- Add a new field `email_dest` to NOTICEs, which defines where to
+  send email to. The email-related NOTICE actions fill this now, and
+  then emails will be sent to all recorded addresses at the end of
+  NOTICE processing. This makes email generation more consistent and
+  extensible.
+
+- Add page and email administrator to mails processed by hostnames extension.
+
 Removed Functionality
 ---------------------
 
 - Support for the RocksDB Broker data store was previously broken and unusable,
   so all code/options related to it are now removed.
 
+- Support for the ENABLE_MOBILE_IPV6 compiler variable has been removed. Mobile
+  IPv6 is now enabled by default. The --enable-mobile-ipv6 returns a warning
+  that it will be removed in v5.1 and no longer has any effect.
+
 Deprecated Functionality
 ------------------------
 
@@ -79,6 +117,11 @@ Deprecated Functionality
 
 - ``supervisor_rotation_format_func`` is renamed to ``archiver_rotation_format_func``
 
+- The ```MemoryAllocation()``` function implemented by a number of interfaces
+  is now deprecated. In testing we found that the values returned were mostly
+  incorrect and weren't useful. The ```val_size``` and ```global_sizes``` BIF
+  methods have also both been marked deprecated.
+
 Zeek 4.0.0
 ==========
 

VERSION

@@ -1 +1 @@
-4.1.0-dev.583
+4.1.0-dev.818

auxil/broker

@@ -1 +1 @@
-Subproject commit 6ba5a134b0a6c29997bda652ca3a8b5b4da7cd9a
+Subproject commit 3c00df399e05d70b2aefdeb8fef6b47f08d0b289

auxil/btest

@@ -1 +1 @@
-Subproject commit 72b8e3a6a5863715cbbce873cd8e1bf5358086ab
+Subproject commit 1be682d0744f201551ada8dc568820c5f91a049c

auxil/highwayhash

@@ -1 +1 @@
-Subproject commit bb8b31c7377e047049466246bb1d2c00db8bc0bc
+Subproject commit ea06651bd11387f5aac694c819314536fe7df060

auxil/package-manager

@@ -1 +1 @@
-Subproject commit dedfdf443cebc27e4eeb121072578a9d4216ea7f
+Subproject commit bb76430eafab98b93570348103ec0c8edfdba9c0

auxil/zeek-archiver

@@ -1 +1 @@
-Subproject commit 7fe80115aa159119febc623cec0660774bcae50c
+Subproject commit d3e55991cbe69f37966207479492edd38d548b1d

auxil/zeekctl

@@ -1 +1 @@
-Subproject commit c383d515ef89bd8c4d9785ed23ef6e1c2aafe6ce
+Subproject commit d31885671d74932d951778c029fa74d44cf3e542

ci/build.sh

@@ -3,6 +3,12 @@
 set -e
 set -x
 
+# If we're on macOS, use --osx-sysroot to ensure we can find the SDKs from Xcode. This avoids
+# some problems with Catalina specifically, but it doesn't break anything on Big Sur either.
+if [ "${CIRRUS_OS}" == "darwin" ]; then
+    export ZEEK_CI_CONFIGURE_FLAGS="${ZEEK_CI_CONFIGURE_FLAGS} --osx-sysroot=$(xcrun --show-sdk-path)"
+fi
+
 if [ "${ZEEK_CI_CREATE_ARTIFACT}" != "1" ]; then
     ./configure ${ZEEK_CI_CONFIGURE_FLAGS}
     cd build

ci/fedora-34/Dockerfile

@@ -1,4 +1,4 @@
-FROM fedora:32
+FROM fedora:34
 
 RUN dnf -y install \
     bison \

ci/macos/prepare.sh

@@ -5,4 +5,6 @@ sysctl hw.model hw.machine hw.ncpu hw.physicalcpu hw.logicalcpu
 set -e
 set -x
 
-brew install cmake swig openssl bison
+brew update-reset
+brew upgrade cmake openssl
+brew install swig bison

ci/opensuse-leap-15.3/Dockerfile

@@ -0,0 +1,25 @@
+FROM opensuse/leap:15.3
+
+RUN zypper in -y \
+  cmake \
+  make \
+  gcc \
+  gcc-c++ \
+  python3 \
+  python3-devel \
+  flex \
+  bison \
+  libpcap-devel \
+  libopenssl-devel \
+  zlib-devel \
+  swig \
+  git \
+  curl \
+  python3-pip \
+  which \
+  gzip \
+  tar \
+  && rm -rf /var/cache/zypp
+
+
+RUN pip3 install junit2html

cmake

@@ -1 +1 @@
-Subproject commit 74259745dea5ee4889d1ac1f4ebde4e2c59c329a
+Subproject commit cce53d15008a26dcb1b7eb534a78f52f9355c676

configure

@@ -13,21 +13,23 @@ usage="\
 Usage: $0 [OPTION]... [VAR=VALUE]...
 
   Build Options:
-    --cmake=PATH           custom path to a CMake binary
+    --cmake=PATH                custom path to a CMake binary
-    --builddir=DIR         place build files in directory [build]
+    --builddir=DIR              place build files in directory [build]
-    --build-dir=DIR        alias for --builddir
+    --build-dir=DIR             alias for --builddir
-    --build-type=TYPE      set CMake build type [RelWithDebInfo]:
+    --build-type=TYPE           set CMake build type [RelWithDebInfo]:
-                             - Debug: optimizations off, debug symbols + flags
+                                  - Debug: optimizations off, debug symbols + flags
-                             - MinSizeRel: size optimizations, debugging off
+                                  - MinSizeRel: size optimizations, debugging off
-                             - Release: optimizations on, debugging off
+                                  - Release: optimizations on, debugging off
-                             - RelWithDebInfo: optimizations on,
+                                  - RelWithDebInfo: optimizations on,
-                                   debug symbols on, debug flags off
+                                        debug symbols on, debug flags off
-    --generator=GENERATOR  CMake generator to use (see cmake --help)
+    --generator=GENERATOR       CMake generator to use (see cmake --help)
-    --ccache               use ccache to speed up recompilation (requires
+    --ccache                    use ccache to speed up recompilation (requires
-                           ccache installation and CMake 3.10+)
+                                ccache installation and CMake 3.10+)
-    --toolchain=PATH       path to a CMAKE_TOOLCHAIN_FILE
+    --toolchain=PATH            path to a CMAKE_TOOLCHAIN_FILE
-                           (useful for cross-compiling)
+                                (useful for cross-compiling)
-    --sanitizers=LIST      comma-separated list of sanitizer names to enable
+    --sanitizers=LIST           comma-separated list of sanitizer names to enable
+    --include-plugins=PATHS     paths containing plugins to build directly into Zeek
+                                (semicolon delimited and quoted when multiple)
 
   Installation Directories:
     --prefix=PREFIX        installation directory [/usr/local/zeek]
@@ -66,6 +68,7 @@ Usage: $0 [OPTION]... [VAR=VALUE]...
     --disable-auxtools     don't build or install auxiliary tools
     --disable-archiver     don't build or install zeek-archiver tool
     --disable-btest        don't install BTest
+    --disable-btest-pcaps  don't install Zeek's BTest input pcaps
     --disable-python       don't try to build python bindings for Broker
     --disable-broker-tests don't try to build Broker unit tests
     --disable-zkg          don't install zkg
@@ -162,12 +165,15 @@ append_cache_entry ENABLE_JEMALLOC      BOOL   false
 append_cache_entry BUILD_SHARED_LIBS    BOOL   true
 append_cache_entry INSTALL_AUX_TOOLS    BOOL   true
 append_cache_entry INSTALL_BTEST        BOOL   true
+append_cache_entry INSTALL_BTEST_PCAPS  BOOL   true
 append_cache_entry INSTALL_ZEEK_ARCHIVER BOOL  true
 append_cache_entry INSTALL_ZEEKCTL      BOOL   true
 append_cache_entry INSTALL_ZKG          BOOL   true
 append_cache_entry CPACK_SOURCE_IGNORE_FILES STRING
-append_cache_entry ENABLE_MOBILE_IPV6   BOOL   false
 append_cache_entry ZEEK_SANITIZERS      STRING ""
+append_cache_entry ZEEK_INCLUDE_PLUGINS STRING ""
+
+has_enable_mobile_ipv6=0
 
 # parse arguments
 while [ $# -ne 0 ]; do
@@ -206,6 +212,9 @@ while [ $# -ne 0 ]; do
         --toolchain=*)
             append_cache_entry CMAKE_TOOLCHAIN_FILE  PATH   $optarg
             ;;
+        --include-plugins=*)
+            append_cache_entry ZEEK_INCLUDE_PLUGINS  STRING   $optarg
+            ;;
         --prefix=*)
             prefix=$optarg
             append_cache_entry CMAKE_INSTALL_PREFIX PATH   $optarg
@@ -257,7 +266,7 @@ while [ $# -ne 0 ]; do
             append_cache_entry ENABLE_DEBUG         BOOL   true
             ;;
         --enable-mobile-ipv6)
-            append_cache_entry ENABLE_MOBILE_IPV6         BOOL   true
+            has_enable_mobile_ipv6=1
             ;;
         --enable-perftools)
             append_cache_entry ENABLE_PERFTOOLS     BOOL   true
@@ -293,6 +302,9 @@ while [ $# -ne 0 ]; do
         --disable-btest)
             append_cache_entry INSTALL_BTEST BOOL false
             ;;
+        --disable-btest-pcaps)
+            append_cache_entry INSTALL_BTEST_PCAPS BOOL false
+            ;;
         --disable-python)
             append_cache_entry DISABLE_PYTHON_BINDINGS     BOOL   true
             ;;
@@ -429,3 +441,8 @@ fi
 echo "# This is the command used to configure this build" > config.status
 echo $command >> config.status
 chmod u+x config.status
+
+if [ $has_enable_mobile_ipv6 -eq 1 ]; then
+    echo
+    echo "NOTE: The --enable-mobile-ipv6 argument no longer has any effect and will be removed in v5.1."
+fi

doc

@@ -1 +1 @@
-Subproject commit e9f80f75548a12cfbc711a4c69d3135e0ad105e3
+Subproject commit c6a93225ed2c5644d9caa167e4e36cab26c5d10b

man/zeek.8

@@ -16,6 +16,8 @@ tasks, including detecting malware by interfacing to external registries,
 reporting vulnerable versions of software seen on the network, identifying
 popular web applications, detecting SSH brute-forcing, validating SSL
 certificate chains, among others.
+
+You must have the necessary permissions to access to the files or interfaces specified.
 .SH OPTIONS
 .TP
 .B <file>
@@ -148,6 +150,31 @@ Output file for script execution statistics
 .TP
 .B ZEEK_DISABLE_ZEEKYGEN
 Disable Zeekygen (Broxygen) documentation support
+.SH OUTPUT FORMAT
+Output is written in multiple files depending on configuration. The default
+location is the current directory.
+
+The output written by Zeek can be formatted in multiple ways using the
+logging framework.
+.PP
+The default are files in human-readable (ASCII) format. The data is organized
+into columns (tab-delimited). The data can be processed using, e.g., the \fBzeek-cut\fR tool.
+
+
+.SH EXAMPLES
+Read a capture file and generate the default logs:
+.br
+    # zeek -r test-capture.pcap
+.PP
+When running on live traffic, Zeek is usually started by running \fBzeekctl\fR. To configure
+Zeek with an initial configuration, install, and restart:
+.br
+    # zeekctl deploy
+
+Note: the zeekctl configuration may need to be updated before first use. Especially the
+network interface used should be the correct one.
+.SH SEE ALSO
+zeekctl(8) zeek-cut(1)
 .SH AUTHOR
 .B zeek
 was written by The Zeek Project <info@zeek.org>.

scripts/base/frameworks/analyzer/main.zeek

@@ -124,7 +124,6 @@ export {
 	## A set of analyzers to disable by default at startup. The default set
 	## contains legacy analyzers that are no longer supported.
 	global disabled_analyzers: set[Analyzer::Tag] = {
-		ANALYZER_STEPPINGSTONE,
 		ANALYZER_TCPSTATS,
 	} &redef;
 }

scripts/base/frameworks/broker/main.zeek

@@ -31,7 +31,7 @@ export {
 	## authenticated.
 	const disable_ssl = F &redef;
 
-	## Path to a file containing concatenated trusted certificates 
+	## Path to a file containing concatenated trusted certificates
 	## in PEM format. If set, Zeek will require valid certificates for
 	## all peers.
 	const ssl_cafile = "" &redef;
@@ -122,6 +122,37 @@ export {
 	## done reading the pcap.
 	option peer_counts_as_iosource = T;
 
+	## Port for Broker's metric exporter. Setting this to a valid TCP port causes
+	## Broker to make metrics available to Prometheus scrapers via HTTP. Zeek
+	## overrides any value provided in zeek_init or earlier at startup if the
+	## environment variable BROKER_METRICS_PORT is defined.
+	const metrics_port = 0/unknown &redef;
+
+	## Frequency for publishing scraped metrics to the target topic. Zeek
+	## overrides any value provided in zeek_init or earlier at startup if the
+	## environment variable BROKER_METRICS_EXPORT_INTERVAL is defined.
+	option metrics_export_interval = 1 sec;
+
+	## Target topic for the metrics. Setting a non-empty string starts the
+	## periodic publishing of local metrics. Zeek overrides any value provided in
+	## zeek_init or earlier at startup if the environment variable
+	## BROKER_METRICS_EXPORT_TOPIC is defined.
+	option metrics_export_topic = "";
+
+	## ID for the metrics exporter. When setting a target topic for the
+	## exporter, Broker sets this option to the suffix of the new topic *unless*
+	## the ID is a non-empty string. Since setting a topic starts the periodic
+	## publishing of events, we recommend setting the ID always first or avoid
+	## setting it at all if the topic suffix serves as a good-enough ID. Zeek
+	## overrides any value provided in zeek_init or earlier at startup if the
+	## environment variable BROKER_METRICS_ENDPOINT_NAME is defined.
+	option metrics_export_endpoint_name = "";
+
+	## Selects prefixes from the local metrics. Only metrics with prefixes
+	## listed in this variable are included when publishing local metrics.
+	## Setting an empty vector selects *all* metrics.
+	option metrics_export_prefixes: vector of string = vector();
+
 	## The default topic prefix where logs will be published.  The log's stream
 	## id is appended when writing to a particular stream.
 	const default_log_topic_prefix = "zeek/logs/" &redef;
@@ -385,9 +416,53 @@ event Broker::log_flush() &priority=10
 	schedule Broker::log_batch_interval { Broker::log_flush() };
 	}
 
+function update_metrics_export_interval(id: string, val: interval): interval
+	{
+	Broker::__set_metrics_export_interval(val);
+	return val;
+	}
+
+function update_metrics_export_topic(id: string, val: string): string
+	{
+	Broker::__set_metrics_export_topic(val);
+	return val;
+	}
+
+function update_metrics_export_endpoint_name(id: string, val: string): string
+	{
+	Broker::__set_metrics_export_endpoint_name(val);
+	return val;
+	}
+
+function update_metrics_export_prefixes(id: string, filter: vector of string): vector of string
+	{
+	Broker::__set_metrics_export_prefixes(filter);
+	return filter;
+	}
+
 event zeek_init()
 	{
 	schedule Broker::log_batch_interval { Broker::log_flush() };
+	# interval
+	update_metrics_export_interval("Broker::metrics_export_interval",
+	                               Broker::metrics_export_interval);
+	Option::set_change_handler("Broker::metrics_export_interval",
+	                           update_metrics_export_interval);
+	# topic
+	update_metrics_export_topic("Broker::metrics_export_topic",
+	                            Broker::metrics_export_topic);
+	Option::set_change_handler("Broker::metrics_export_topic",
+	                           update_metrics_export_topic);
+	# endpoint name
+	update_metrics_export_endpoint_name("Broker::metrics_export_endpoint_name",
+	                                    Broker::metrics_export_endpoint_name);
+	Option::set_change_handler("Broker::metrics_export_endpoint_name",
+	                           update_metrics_export_endpoint_name);
+	# prefixes
+	update_metrics_export_prefixes("Broker::metrics_export_prefixes",
+	                               Broker::metrics_export_prefixes);
+	Option::set_change_handler("Broker::metrics_export_prefixes",
+	                           update_metrics_export_prefixes);
 	}
 
 event retry_listen(a: string, p: port, retry: interval)

scripts/base/frameworks/logging/writers/ascii.zeek

@@ -54,6 +54,11 @@ export {
 	## This option is also available as a per-filter ``$config`` option.
 	const gzip_file_extension = "gz" &redef;
 
+	## Define the default logging directory. If empty, logs are written
+	## to the current working directory.
+	##
+	const logdir = "" &redef;
+
 	## Format of timestamps when writing out JSON. By default, the JSON
 	## formatter will use double values for timestamps which represent the
 	## number of seconds from the UNIX epoch.

scripts/base/frameworks/notice/actions/email_admin.zeek

@@ -18,18 +18,15 @@ export {
 	};
 }
 
-hook notice(n: Notice::Info) &priority=-5
+hook notice(n: Notice::Info)
 	{
 	if ( |Site::local_admins| > 0 &&
 	     ACTION_EMAIL_ADMIN in n$actions )
 		{
 		local email = "";
 		if ( n?$src && |Site::get_emails(n$src)| > 0 )
-			email = fmt("%s, %s", email, Site::get_emails(n$src));
+			add n$email_dest[Site::get_emails(n$src)];
 		if ( n?$dst && |Site::get_emails(n$dst)| > 0 )
-			email = fmt("%s, %s", email, Site::get_emails(n$dst));
+			add n$email_dest[Site::get_emails(n$dst)];
-		
-		if ( email != "" )
-			email_notice_to(n, email, T);
 		}
 	}

scripts/base/frameworks/notice/actions/page.zeek

@@ -11,14 +11,14 @@ export {
 		## variable.
 		ACTION_PAGE
 	};
-	
+
 	## Email address to send notices with the :zeek:enum:`Notice::ACTION_PAGE`
 	## action.
 	option mail_page_dest = "";
 }
 
-hook notice(n: Notice::Info) &priority=-5
+hook notice(n: Notice::Info)
 	{
 	if ( ACTION_PAGE in n$actions )
-		email_notice_to(n, mail_page_dest, F);
+		add n$email_dest[mail_page_dest];
 	}

scripts/base/frameworks/notice/main.zeek

@@ -136,6 +136,9 @@ export {
 		## The actions which have been applied to this notice.
 		actions:        ActionSet      &log &default=ActionSet();
 
+		## The email address(es) where to send this notice
+		email_dest:     set[string]    &log &default=set();
+
 		## By adding chunks of text into this element, other scripts
 		## can expand on notices that are being emailed.  The normal
 		## way to add text is to extend the vector by handling the
@@ -510,10 +513,17 @@ hook Notice::policy(n: Notice::Info) &priority=10
 	add n$actions[ACTION_LOG];
 	}
 
-hook Notice::notice(n: Notice::Info) &priority=-5
+hook Notice::notice(n: Notice::Info)
 	{
 	if ( ACTION_EMAIL in n$actions )
-		email_notice_to(n, mail_dest, T);
+		add n$email_dest[mail_dest];
+	}
+
+hook Notice::notice(n: Notice::Info) &priority=-5
+	{
+	for ( dest in n$email_dest )
+		email_notice_to(n, dest, T);
+
 	if ( ACTION_LOG in n$actions )
 		Log::write(Notice::LOG, n);
 	if ( ACTION_ALARM in n$actions )

scripts/base/init-bare.zeek

@@ -635,7 +635,7 @@ type ProcStats: record {
 	real_time: interval;          ##< Elapsed real time since Zeek started running.
 	user_time: interval;          ##< User CPU seconds.
 	system_time: interval;        ##< System CPU seconds.
-	mem: count;                   ##< Maximum memory consumed, in KB.
+	mem: count;                   ##< Maximum memory consumed, in bytes.
 	minor_faults: count;          ##< Page faults not requiring actual I/O.
 	major_faults: count;          ##< Page faults requiring actual I/O.
 	num_swap: count;              ##< Times swapped out.
@@ -1933,6 +1933,7 @@ type gtp_delete_pdp_ctx_response_elements: record {
 @load base/frameworks/supervisor/api
 @load base/bif/supervisor.bif
 @load base/bif/packet_analysis.bif
+@load base/bif/CPP-load.bif
 
 ## Internal function.
 function add_interface(iold: string, inew: string): string
@@ -4117,15 +4118,6 @@ type PE::SectionHeader: record {
 }
 module GLOBAL;
 
-## Internal to the stepping stone detector.
-const stp_delta: interval &redef;
-
-## Internal to the stepping stone detector.
-const stp_idle_min: interval &redef;
-
-## Internal to the stepping stone detector.
-global stp_skip_src: set[addr] &redef;
-
 ## Description of a signature match.
 ##
 ## .. zeek:see:: signature_match

scripts/base/packet-protocols/__load__.zeek

@@ -15,3 +15,6 @@
 @load base/packet-protocols/gre
 @load base/packet-protocols/iptunnel
 @load base/packet-protocols/vntag
+@load base/packet-protocols/udp
+@load base/packet-protocols/tcp
+@load base/packet-protocols/icmp

scripts/base/packet-protocols/icmp/__load__.zeek

@@ -0,0 +1 @@
+@load ./main

scripts/base/packet-protocols/icmp/main.zeek

@@ -0,0 +1,5 @@
+module PacketAnalyzer::ICMP;
+
+#event zeek_init() &priority=20
+#	{
+#	}

scripts/base/packet-protocols/ip/main.zeek

@@ -1,8 +1,22 @@
 module PacketAnalyzer::IP;
 
+const IPPROTO_TCP : count = 6;
+const IPPROTO_UDP : count = 17;
+const IPPROTO_ICMP : count = 1;
+const IPPROTO_ICMP6 : count = 58;
+
+const IPPROTO_IPIP : count = 4;
+const IPPROTO_IPV6 : count = 41;
+const IPPROTO_GRE : count = 47;
+
 event zeek_init() &priority=20
 	{
-	PacketAnalyzer::register_packet_analyzer(PacketAnalyzer::ANALYZER_IP, 4, PacketAnalyzer::ANALYZER_IPTUNNEL);
+	PacketAnalyzer::register_packet_analyzer(PacketAnalyzer::ANALYZER_IP, IPPROTO_IPIP, PacketAnalyzer::ANALYZER_IPTUNNEL);
-	PacketAnalyzer::register_packet_analyzer(PacketAnalyzer::ANALYZER_IP, 41, PacketAnalyzer::ANALYZER_IPTUNNEL);
+	PacketAnalyzer::register_packet_analyzer(PacketAnalyzer::ANALYZER_IP, IPPROTO_IPV6, PacketAnalyzer::ANALYZER_IPTUNNEL);
-	PacketAnalyzer::register_packet_analyzer(PacketAnalyzer::ANALYZER_IP, 47, PacketAnalyzer::ANALYZER_GRE);
+	PacketAnalyzer::register_packet_analyzer(PacketAnalyzer::ANALYZER_IP, IPPROTO_GRE, PacketAnalyzer::ANALYZER_GRE);
+
+	PacketAnalyzer::register_packet_analyzer(PacketAnalyzer::ANALYZER_IP, IPPROTO_TCP, PacketAnalyzer::ANALYZER_TCP);
+	PacketAnalyzer::register_packet_analyzer(PacketAnalyzer::ANALYZER_IP, IPPROTO_UDP, PacketAnalyzer::ANALYZER_UDP);
+	PacketAnalyzer::register_packet_analyzer(PacketAnalyzer::ANALYZER_IP, IPPROTO_ICMP, PacketAnalyzer::ANALYZER_ICMP);
+	PacketAnalyzer::register_packet_analyzer(PacketAnalyzer::ANALYZER_IP, IPPROTO_ICMP6, PacketAnalyzer::ANALYZER_ICMP);
 	}

scripts/base/packet-protocols/tcp/__load__.zeek

@@ -0,0 +1 @@
+@load ./main

scripts/base/packet-protocols/tcp/main.zeek

@@ -0,0 +1,5 @@
+module PacketAnalyzer::TCP;
+
+#event zeek_init() &priority=20
+#	{
+#	}

scripts/base/packet-protocols/udp/__load__.zeek

@@ -0,0 +1 @@
+@load ./main

scripts/base/packet-protocols/udp/main.zeek

@@ -0,0 +1,5 @@
+module PacketAnalyzer::UDP;
+
+#event zeek_init() &priority=20
+#	{
+#	}

scripts/base/protocols/ssh/main.zeek

@@ -20,8 +20,12 @@ export {
 		uid:             string       &log;
 		## The connection's 4-tuple of endpoint addresses/ports.
 		id:              conn_id      &log;
-		## SSH major version (1 or 2)
+		## SSH major version (1, 2, or unset). The version can be unset if the
-		version:         count        &log;
+		## client and server version strings are unset, malformed or incompatible
+		## so no common version can be extracted. If no version can be extracted
+		## even though both client and server versions are set a weird
+		## will be generated.
+		version:         count        &log &optional;
 		## Authentication result (T=success, F=failure, unset=unknown)
 		auth_success:    bool         &log &optional;
 		## The number of authentication attemps we observed. There's always
@@ -155,65 +159,82 @@ function set_session(c: connection)
 		}
 	}
 
-function set_version(c: connection, version: string)
+function set_version(c: connection)
-    {
+	{
-    if ( c$ssh?$server && c$ssh?$client && |c$ssh$client| > 4 && |c$ssh$server| > 4 )
+	# We always either set the version field to a concrete value, or unset it.
-        {
+	delete c$ssh$version;
-        if ( c$ssh$client[4] == "1" && c$ssh$server[4] == "2" )
+
-            {
+	# If either the client or server string is unset we cannot compute a
-            # SSH199 vs SSH2 -> 2
+	# version and return early. We do not raise a weird in this case as we
-            if ( ( |c$ssh$client| > 7 ) && ( c$ssh$client[6] == "9" ) && ( c$ssh$client[7] == "9" ) )
+	# might arrive here while having only seen one side of the handshake.
-                c$ssh$version = 2;
+	const has_server = c$ssh?$server && |c$ssh$server| > 0;
-            # SSH1 vs SSH2 -> Undefined
+	const has_client = c$ssh?$client && |c$ssh$client| > 0;
-            else
+	if ( ! ( has_server && has_client ) )
-                c$ssh$version = 0;
+		return;
-            }
+
-        else if ( c$ssh$client[4] == "2" && c$ssh$server[4] == "1" )
+	if ( |c$ssh$client| > 4 && |c$ssh$server| > 4 )
-            {
+		{
-            # SSH2 vs SSH199 -> 2
+		if ( c$ssh$client[4] == "1" && c$ssh$server[4] == "2" )
-            if ( ( |c$ssh$server| > 7 ) && ( c$ssh$server[6] == "9" ) && ( c$ssh$server[7] == "9" ) )
+			{
-                c$ssh$version = 2;
+			# SSH199 vs SSH2 -> 2
-            else
+			if ( ( |c$ssh$client| > 7 ) && ( c$ssh$client[6] == "9" ) && ( c$ssh$client[7] == "9" ) )
-            # SSH2 vs SSH1 -> Undefined
+				c$ssh$version = 2;
-                c$ssh$version = 0;
+			# SSH1 vs SSH2 -> Undefined
-            }
+			else
-        else if ( c$ssh$client[4] == "1" && c$ssh$server[4] == "1" )
+				Reporter::conn_weird("SSH_version_mismatch", c, fmt("%s vs %s", c$ssh$server, c$ssh$client));
-            {
+				return;
-            # SSH1 vs SSH199 -> 1
+			}
-            if ( ( |c$ssh$server| > 7 ) && ( c$ssh$server[6] == "9" ) && ( c$ssh$server[7] == "9" ) )
+		else if ( c$ssh$client[4] == "2" && c$ssh$server[4] == "1" )
-                {
+			{
-                # SSH199 vs SSH199
+			# SSH2 vs SSH199 -> 2
-                if (( |c$ssh$client| > 7 ) && ( c$ssh$client[6] == "9" ) && ( c$ssh$client[7] == "9" ))
+			if ( ( |c$ssh$server| > 7 ) && ( c$ssh$server[6] == "9" ) && ( c$ssh$server[7] == "9" ) )
-                    c$ssh$version = 2;
+				c$ssh$version = 2;
-                else
+			else
-                    c$ssh$version = 1;
+				# SSH2 vs SSH1 -> Undefined
-                }
+				Reporter::conn_weird("SSH_version_mismatch", c, fmt("%s vs %s", c$ssh$server, c$ssh$client));
-            else
+				return;
-                {
+			}
-                # SSH1 vs SSH1 -> 1
+		else if ( c$ssh$client[4] == "1" && c$ssh$server[4] == "1" )
-                c$ssh$version = 1;
+			{
-                }
+			# SSH1 vs SSH199 -> 1
-            }
+			if ( ( |c$ssh$server| > 7 ) && ( c$ssh$server[6] == "9" ) && ( c$ssh$server[7] == "9" ) )
-        # SSH2 vs SSH2
+				{
-        else if (c$ssh$client[4] == "2" && c$ssh$server[4] == "2" )
+				# SSH199 vs SSH199
-            {
+				if (( |c$ssh$client| > 7 ) && ( c$ssh$client[6] == "9" ) && ( c$ssh$client[7] == "9" ))
-            c$ssh$version = 2;
+					c$ssh$version = 2;
-            }
+				else
-        }
+					c$ssh$version = 1;
-    }
+				}
+			else
+				{
+				# SSH1 vs SSH1 -> 1
+				c$ssh$version = 1;
+				}
+			}
+		# SSH2 vs SSH2
+		else if (c$ssh$client[4] == "2" && c$ssh$server[4] == "2" )
+			{
+			c$ssh$version = 2;
+			}
+
+		return;
+		}
+
+	Reporter::conn_weird("SSH_cannot_determine_version", c, fmt("%s vs %s", c$ssh$server, c$ssh$client));
+	}
 
 event ssh_server_version(c: connection, version: string)
 	{
 	set_session(c);
 	c$ssh$server = version;
-	set_version(c, version);
+	set_version(c);
 	}
 
 event ssh_client_version(c: connection, version: string)
 	{
 	set_session(c);
 	c$ssh$client = version;
-	set_version(c, version);
+	set_version(c);
 	}
 
 event ssh_auth_attempted(c: connection, authenticated: bool) &priority=5

scripts/base/protocols/ssl/main.zeek

@@ -141,8 +141,8 @@ export {
 	## record as it is sent on to the logging framework.
 	global log_ssl: event(rec: Info);
 
-	# Hook that can be used to perform actions right before the log record
+	## Hook that can be used to perform actions right before the log record
-	# is written.
+	## is written.
 	global ssl_finishing: hook(c: connection);
 
 	## SSL finalization hook.  Remaining SSL info may get logged when it's called.

scripts/policy/frameworks/files/detect-MHR.zeek

@@ -26,7 +26,7 @@ export {
 	## The Match notice has a sub message with a URL where you can get more
 	## information about the file. The %s will be replaced with the SHA-1
 	## hash of the file.
-	option match_sub_url = "https://www.virustotal.com/en/search/?query=%s";
+	option match_sub_url = "https://www.virustotal.com/gui/search/%s";
 
 	## The malware hash registry runs each malware sample through several
 	## A/V engines.  Team Cymru returns a percentage to indicate how

scripts/policy/frameworks/notice/extend-email/hostnames.zeek

@@ -13,13 +13,14 @@ module Notice;
 # reference to the original notice)
 global tmp_notice_storage: table[string] of Notice::Info &create_expire=max_email_delay+10secs;
 
-hook notice(n: Notice::Info) &priority=10
+# Run after e-mail address is set, but before e-mail is sent.
+hook notice(n: Notice::Info) &priority=-1
 	{
 	if ( ! n?$src && ! n?$dst )
 		return;
 
 	# This should only be done for notices that are being sent to email.
-	if ( ACTION_EMAIL !in n$actions )
+	if ( ! n?$email_dest )
 		return;
 
 	# I'm not recovering gracefully from the when statements because I want

src/Attr.cc

@@ -359,7 +359,7 @@ void Attributes::CheckAttr(Attr* a)
 				// Ok.
 				break;
 
-			auto e = check_and_promote_expr(a->GetExpr().get(), type.get());
+			auto e = check_and_promote_expr(a->GetExpr(), type);
 
 			if ( e )
 				{
@@ -399,7 +399,7 @@ void Attributes::CheckAttr(Attr* a)
 					// Ok.
 					break;
 
-				auto e = check_and_promote_expr(a->GetExpr().get(), ytype.get());
+				auto e = check_and_promote_expr(a->GetExpr(), ytype);
 
 				if ( e )
 					{
@@ -425,7 +425,7 @@ void Attributes::CheckAttr(Attr* a)
 
 			if ( (atype->Tag() == TYPE_TABLE && atype->AsTableType()->IsUnspecifiedTable()) )
 				{
-				auto e = check_and_promote_expr(a->GetExpr().get(), type.get());
+				auto e = check_and_promote_expr(a->GetExpr(), type);
 
 				if ( e )
 					{

src/CCL.h

@@ -26,6 +26,7 @@ public:
 	void ReplaceSyms(int_list* new_syms)
 				{ delete syms; syms = new_syms; }
 
+	[[deprecated("Remove in v5.1. MemoryAllocation() is deprecated and will be removed. See GHI-572.")]]
 	unsigned int MemoryAllocation() const;
 
 protected:

src/CMakeLists.txt

@@ -124,6 +124,9 @@ set(BIF_SRCS
     # it's needed before parsing the packet protocol scripts, which happen
     # very near to the start of parsing.
     packet_analysis/packet_analysis.bif
+    # The C++ loading BIF is treated like other top-level BIFs to give
+    # us flexibility regarding when it's called.
+    script_opt/CPP/CPP-load.bif
 )
 
 foreach (bift ${BIF_SRCS})
@@ -170,6 +173,32 @@ add_subdirectory(session)
 
 add_subdirectory(fuzzers)
 
+########################################################################
+## Build in the discovered external plugins and create the autogenerated scripts.
+
+set(PRELOAD_SCRIPT ${CMAKE_BINARY_DIR}/scripts/builtin-plugins/__preload__.zeek)
+file(WRITE ${PRELOAD_SCRIPT} "# Warning, this is an autogenerated file!\n")
+set(LOAD_SCRIPT ${CMAKE_BINARY_DIR}/scripts/builtin-plugins/__load__.zeek)
+file(WRITE ${LOAD_SCRIPT} "# Warning, this is an autogenerated file!\n")
+
+foreach (plugin_dir ${BUILTIN_PLUGIN_LIST})
+    get_filename_component(plugin_name ${plugin_dir} NAME)
+
+    if(IS_DIRECTORY "${plugin_dir}/cmake")
+        set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${plugin_dir}/cmake")
+    endif()
+
+    # Setup the include path for built source artifacts.
+    include_directories(AFTER
+                        ${plugin_dir}/src
+                        ${CMAKE_CURRENT_BINARY_DIR}/builtin-plugins/${plugin_name})
+
+    add_subdirectory(${plugin_dir} ${CMAKE_CURRENT_BINARY_DIR}/builtin-plugins/${plugin_name})
+endforeach()
+
+install(FILES ${PRELOAD_SCRIPT} DESTINATION ${ZEEK_SCRIPT_INSTALL_PATH}/builtin-plugins/)
+install(FILES ${LOAD_SCRIPT} DESTINATION ${ZEEK_SCRIPT_INSTALL_PATH}/builtin-plugins/)
+
 ########################################################################
 ## bro target
 
@@ -217,6 +246,10 @@ add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/DebugCmdConstants.h
                    WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
 )
 
+set(_gen_zeek_script_cpp ${CMAKE_CURRENT_BINARY_DIR}/../CPP-gen.cc)
+add_custom_command(OUTPUT ${_gen_zeek_script_cpp}
+		    COMMAND ${CMAKE_COMMAND} -E touch ${_gen_zeek_script_cpp})
+
 set_source_files_properties(nb_dns.c PROPERTIES COMPILE_FLAGS
                             -fno-strict-aliasing)
 
@@ -327,6 +360,27 @@ set(MAIN_SRCS
     plugin/Manager.cc
     plugin/Plugin.cc
 
+    script_opt/CPP/Attrs.cc
+    script_opt/CPP/Consts.cc
+    script_opt/CPP/DeclFunc.cc
+    script_opt/CPP/Driver.cc
+    script_opt/CPP/Emit.cc
+    script_opt/CPP/Exprs.cc
+    script_opt/CPP/Func.cc
+    script_opt/CPP/GenFunc.cc
+    script_opt/CPP/HashMgr.cc
+    script_opt/CPP/Inits.cc
+    script_opt/CPP/RuntimeInit.cc
+    script_opt/CPP/RuntimeOps.cc
+    script_opt/CPP/RuntimeVec.cc
+    script_opt/CPP/Stmts.cc
+    script_opt/CPP/Tracker.cc
+    script_opt/CPP/Types.cc
+    script_opt/CPP/Util.cc
+    script_opt/CPP/Vars.cc
+
+    ${_gen_zeek_script_cpp}
+
     script_opt/DefItem.cc
     script_opt/DefSetsMgr.cc
     script_opt/Expr.cc

src/CompHash.cc

@@ -509,6 +509,9 @@ int CompositeHash::SingleTypeKeySize(Type* bt, const Val* v,
 
 		case TYPE_RECORD:
 			{
+			if ( ! v )
+				return (optional && ! calc_static_size) ? sz : 0;
+
 			const RecordVal* rv = v ? v->AsRecordVal() : nullptr;
 			RecordType* rt = bt->AsRecordType();
 			int num_fields = rt->NumFields();

src/CompHash.h

@@ -30,6 +30,7 @@ public:
 	// Given a hash key, recover the values used to create it.
 	ListValPtr RecoverVals(const HashKey& k) const;
 
+	[[deprecated("Remove in v5.1. MemoryAllocation() is deprecated and will be removed. See GHI-572.")]]
 	unsigned int MemoryAllocation() const { return padded_sizeof(*this) + util::pad_size(size); }
 
 protected:

src/Conn.cc

@@ -19,6 +19,8 @@
 #include "zeek/analyzer/Analyzer.h"
 #include "zeek/analyzer/Manager.h"
 #include "zeek/iosource/IOSource.h"
+#include "zeek/packet_analysis/protocol/ip/SessionAdapter.h"
+#include "zeek/packet_analysis/protocol/tcp/TCP.h"
 
 namespace zeek {
 
@@ -54,7 +56,6 @@ Connection::Connection(const detail::ConnKey& k, double t,
 	vlan = pkt->vlan;
 	inner_vlan = pkt->inner_vlan;
 
-	skip = 0;
 	weird = 0;
 
 	suppress_event = 0;
@@ -64,7 +65,7 @@ Connection::Connection(const detail::ConnKey& k, double t,
 	hist_seen = 0;
 	history = "";
 
-	root_analyzer = nullptr;
+	adapter = nullptr;
 	primary_PIA = nullptr;
 
 	++current_connections;
@@ -83,7 +84,7 @@ Connection::~Connection()
 	if ( conn_val )
 		conn_val->SetOrigin(nullptr);
 
-	delete root_analyzer;
+	delete adapter;
 
 	--current_connections;
 	}
@@ -124,23 +125,23 @@ void Connection::CheckEncapsulation(const std::shared_ptr<EncapsulationStack>& a
 
 void Connection::Done()
 	{
-	// TODO: this still doesn't feel like the right place to do this, but it's better
-	// here than in SessionManager. This really should be down in the TCP analyzer
-	// somewhere, but it's session-related, so maybe not?
-	if ( ConnTransport() == TRANSPORT_TCP )
-		{
-		auto ta = static_cast<analyzer::tcp::TCP_Analyzer*>(GetRootAnalyzer());
-		assert(ta->IsAnalyzer("TCP"));
-		analyzer::tcp::TCP_Endpoint* to = ta->Orig();
-		analyzer::tcp::TCP_Endpoint* tr = ta->Resp();
-
-		session_mgr->tcp_stats.StateLeft(to->state, tr->state);
-		}
-
 	finished = 1;
 
-	if ( root_analyzer && ! root_analyzer->IsFinished() )
+	if ( adapter )
-		root_analyzer->Done();
+		{
+		if ( ConnTransport() == TRANSPORT_TCP )
+			{
+			auto* ta = static_cast<packet_analysis::TCP::TCPSessionAdapter*>(adapter);
+			assert(ta->IsAnalyzer("TCP"));
+			analyzer::tcp::TCP_Endpoint* to = ta->Orig();
+			analyzer::tcp::TCP_Endpoint* tr = ta->Resp();
+
+			packet_analysis::TCP::TCPAnalyzer::GetStats().StateLeft(to->state, tr->state);
+			}
+
+		if ( ! adapter->IsFinished() )
+			adapter->Done();
+		}
 	}
 
 void Connection::NextPacket(double t, bool is_orig,
@@ -153,14 +154,14 @@ void Connection::NextPacket(double t, bool is_orig,
 	run_state::current_timestamp = t;
 	run_state::current_pkt = pkt;
 
-	if ( Skipping() )
+	if ( adapter )
-		return;
-
-	if ( root_analyzer )
 		{
+		if ( adapter->Skipping() )
+			return;
+
 		record_current_packet = record_packet;
 		record_current_content = record_content;
-		root_analyzer->NextPacket(len, data, is_orig, -1, ip, caplen);
+		adapter->NextPacket(len, data, is_orig, -1, ip, caplen);
 		record_packet = record_current_packet;
 		record_content = record_current_content;
 		}
@@ -173,7 +174,7 @@ void Connection::NextPacket(double t, bool is_orig,
 
 bool Connection::IsReuse(double t, const u_char* pkt)
 	{
-	return root_analyzer && root_analyzer->IsReuse(t, pkt);
+	return adapter && adapter->IsReuse(t, pkt);
 	}
 
 bool Connection::ScaledHistoryEntry(char code, uint32_t& counter,
@@ -275,8 +276,8 @@ const RecordValPtr& Connection::GetVal()
 
 		}
 
-	if ( root_analyzer )
+	if ( adapter )
-		root_analyzer->UpdateConnVal(conn_val.get());
+		adapter->UpdateConnVal(conn_val.get());
 
 	conn_val->AssignTime(3, start_time);	// ###
 	conn_val->AssignInterval(4, last_time - start_time);
@@ -289,17 +290,17 @@ const RecordValPtr& Connection::GetVal()
 
 analyzer::Analyzer* Connection::FindAnalyzer(analyzer::ID id)
 	{
-	return root_analyzer ? root_analyzer->FindChild(id) : nullptr;
+	return adapter ? adapter->FindChild(id) : nullptr;
 	}
 
 analyzer::Analyzer* Connection::FindAnalyzer(const analyzer::Tag& tag)
 	{
-	return root_analyzer ? root_analyzer->FindChild(tag) : nullptr;
+	return adapter ? adapter->FindChild(tag) : nullptr;
 	}
 
 analyzer::Analyzer* Connection::FindAnalyzer(const char* name)
 	{
-	return root_analyzer->FindChild(name);
+	return adapter->FindChild(name);
 	}
 
 void Connection::AppendAddl(const char* str)
@@ -370,8 +371,8 @@ void Connection::FlipRoles()
 
 	conn_val = nullptr;
 
-	if ( root_analyzer )
+	if ( adapter )
-		root_analyzer->FlipRoles();
+		adapter->FlipRoles();
 
 	analyzer_mgr->ApplyScheduledAnalyzers(this);
 
@@ -380,17 +381,23 @@ void Connection::FlipRoles()
 
 unsigned int Connection::MemoryAllocation() const
 	{
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
 	return session::Session::MemoryAllocation() + padded_sizeof(*this)
 		+ (timers.MemoryAllocation() - padded_sizeof(timers))
 		+ (conn_val ? conn_val->MemoryAllocation() : 0)
-		+ (root_analyzer ? root_analyzer->MemoryAllocation(): 0)
+		+ (adapter ? adapter->MemoryAllocation(): 0)
 		// primary_PIA is already contained in the analyzer tree.
 		;
+#pragma GCC diagnostic pop
 	}
 
 unsigned int Connection::MemoryAllocationVal() const
 	{
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
 	return conn_val ? conn_val->MemoryAllocation() : 0;
+#pragma GCC diagnostic pop
 	}
 
 void Connection::Describe(ODesc* d) const
@@ -448,10 +455,10 @@ void Connection::IDString(ODesc* d) const
 	d->Add(ntohs(resp_port));
 	}
 
-void Connection::SetRootAnalyzer(analyzer::TransportLayerAnalyzer* analyzer,
+void Connection::SetSessionAdapter(packet_analysis::IP::SessionAdapter* aa,
-                                 analyzer::pia::PIA* pia)
+                                   analyzer::pia::PIA* pia)
 	{
-	root_analyzer = analyzer;
+	adapter = aa;
 	primary_PIA = pia;
 	}
 

src/Conn.h

@@ -41,12 +41,8 @@ class RuleHdrTest;
 
 } // namespace detail
 
-namespace analyzer {
+namespace analyzer { class Analyzer; }
-
+namespace packet_analysis::IP { class SessionAdapter; }
-class TransportLayerAnalyzer;
-class Analyzer;
-
-} // namespace analyzer
 
 enum ConnEventToFlag {
 	NUL_IN_LINE,
@@ -117,7 +113,10 @@ public:
 	// should be marked invalid.
 	const detail::ConnKey& Key() const	{ return key; }
 	session::detail::Key SessionKey(bool copy) const override
-		{ return session::detail::Key{&key, sizeof(key), copy}; }
+		{
+		return session::detail::Key{
+			&key, sizeof(key), session::detail::Key::CONNECTION_KEY_TYPE, copy};
+		}
 
 	const IPAddr& OrigAddr() const		{ return orig_addr; }
 	const IPAddr& RespAddr() const		{ return resp_addr; }
@@ -144,14 +143,6 @@ public:
 			return "unknown";
 		}
 
-	// FIXME: Now this is in Analyzer and should eventually be removed here.
-	//
-	// If true, skip processing of remainder of connection.  Note
-	// that this does not in itself imply that record_packets is false;
-	// we might want instead to process the connection off-line.
-	void SetSkip(bool do_skip)		{ skip = do_skip ? 1 : 0; }
-	bool Skipping() const			{ return skip; }
-
 	// Returns true if the packet reflects a reuse of this
 	// connection (i.e., not a continuation but the beginning of
 	// a new connection).
@@ -196,7 +187,9 @@ public:
 	// Statistics.
 
 	// Just a lower bound.
+	[[deprecated("Remove in v5.1. MemoryAllocation() is deprecated and will be removed. See GHI-572.")]]
 	unsigned int MemoryAllocation() const override;
+	[[deprecated("Remove in v5.1. MemoryAllocation() is deprecated and will be removed. See GHI-572.")]]
 	unsigned int MemoryAllocationVal() const override;
 
 	static uint64_t TotalConnections()
@@ -231,8 +224,8 @@ public:
 	void AddHistory(char code)	{ history += code; }
 
 	// Sets the root of the analyzer tree as well as the primary PIA.
-	void SetRootAnalyzer(analyzer::TransportLayerAnalyzer* analyzer, analyzer::pia::PIA* pia);
+	void SetSessionAdapter(packet_analysis::IP::SessionAdapter* aa, analyzer::pia::PIA* pia);
-	analyzer::TransportLayerAnalyzer* GetRootAnalyzer()	{ return root_analyzer; }
+	packet_analysis::IP::SessionAdapter* GetSessionAdapter()	{ return adapter; }
 	analyzer::pia::PIA* GetPrimaryPIA()	{ return primary_PIA; }
 
 	// Sets the transport protocol in use.
@@ -271,7 +264,6 @@ private:
 
 	detail::ConnKey key;
 
-	unsigned int skip:1;
 	unsigned int weird:1;
 	unsigned int finished:1;
 	unsigned int saw_first_orig_packet:1, saw_first_resp_packet:1;
@@ -279,7 +271,7 @@ private:
 	uint32_t hist_seen;
 	std::string history;
 
-	analyzer::TransportLayerAnalyzer* root_analyzer;
+	packet_analysis::IP::SessionAdapter* adapter;
 	analyzer::pia::PIA* primary_PIA;
 
 	UID uid;	// Globally unique connection ID.

src/DFA.cc

@@ -426,10 +426,13 @@ unsigned int DFA_Machine::MemoryAllocation() const
 	dfa_state_cache->GetStats(&s);
 
 	// FIXME: Count *ec?
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
 	return padded_sizeof(*this)
 		+ s.mem
 		+ padded_sizeof(*start_state)
 		+ nfa->MemoryAllocation();
+#pragma GCC diagnostic pop
 	}
 
 bool DFA_Machine::StateSetToDFA_State(NFA_state_list* state_set,

src/DFA.h

@@ -123,6 +123,7 @@ public:
 	void Describe(ODesc* d) const override;
 	void Dump(FILE* f);
 
+	[[deprecated("Remove in v5.1. MemoryAllocation() is deprecated and will be removed. See GHI-572.")]]
 	unsigned int MemoryAllocation() const;
 
 protected:

src/Debug.cc

@@ -972,10 +972,7 @@ ValPtr dbg_eval_expr(const char* expr)
 
 	const ScriptFunc* func = frame->GetFunction();
 	if ( func )
-		{
-		Ref(func->GetScope());
 		push_existing_scope(func->GetScope());
-		}
 
 	// ### Possibly push a debugger-local scope?
 

src/DebugCmds.cc

@@ -61,7 +61,7 @@ static void lookup_global_symbols_regex(const string& orig_regex, vector<ID*>& m
 		return;
 		}
 
-	Scope* global = global_scope();
+	auto global = global_scope();
 	const auto& syms = global->Vars();
 
 	ID* nextid;

src/Dict.cc

@@ -265,6 +265,11 @@ TEST_CASE("dict new iteration")
 		count++;
 		}
 
+	PDict<uint32_t>::iterator it;
+	it = dict.begin();
+	it = dict.end();
+	PDict<uint32_t>::iterator it2 = it;
+
 	CHECK(count == 2);
 
 	delete key;
@@ -767,7 +772,10 @@ void Dictionary::DumpKeys() const
 	if ( binary )
 		{
 		char key = char(random() % 26) + 'A';
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
 		sprintf(key_file, "%d.%d.%zu-%c.key", Length(), max_distance, MemoryAllocation()/Length(), key);
+#pragma GCC diagnostic pop
 		std::ofstream f(key_file, std::ios::binary|std::ios::out|std::ios::trunc);
 		for ( int idx = 0; idx < Capacity(); idx++ )
 			if ( ! table[idx].Empty() )
@@ -780,7 +788,10 @@ void Dictionary::DumpKeys() const
 	else
 		{
 		char key = char(random() % 26) + 'A';
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
 		sprintf(key_file, "%d.%d.%zu-%d.ckey",Length(), max_distance, MemoryAllocation()/Length(), key);
+#pragma GCC diagnostic pop
 		std::ofstream f(key_file, std::ios::out|std::ios::trunc);
 		for ( int idx = 0; idx < Capacity(); idx++ )
 			if ( ! table[idx].Empty() )
@@ -828,10 +839,13 @@ void Dictionary::Dump(int level) const
 	int distances[DICT_NUM_DISTANCES];
 	int max_distance = 0;
 	DistanceStats(max_distance, distances, DICT_NUM_DISTANCES);
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
 	printf("cap %'7d ent %'7d %'-7d load %.2f max_dist %2d mem %10zu mem/ent %3zu key/ent %3d lg %2d remaps %1d remap_end %4d ",
 		Capacity(), Length(), MaxLength(), (double)Length()/(table? Capacity() : 1),
 		max_distance, MemoryAllocation(), (MemoryAllocation())/(Length()?Length():1), key_size / (Length()?Length():1),
 		log2_buckets, remaps, remap_end);
+#pragma GCC diagnostic pop
 	if ( Length() > 0 )
 		{
 		for (int i = 0; i < DICT_NUM_DISTANCES-1; i++)
@@ -1558,8 +1572,11 @@ DictIterator::DictIterator(const Dictionary* d, detail::DictEntry* begin, detail
 
 DictIterator::~DictIterator()
 	{
-	assert(dict->num_iterators > 0);
+	if ( dict )
-	dict->num_iterators--;
+		{
+		assert(dict->num_iterators > 0);
+		dict->num_iterators--;
+		}
 	}
 
 DictIterator& DictIterator::operator++()
@@ -1574,6 +1591,80 @@ DictIterator& DictIterator::operator++()
 	return *this;
 	}
 
+DictIterator::DictIterator(const DictIterator& that)
+	{
+	if ( this == &that )
+		return;
+
+	if ( dict )
+		{
+		assert(dict->num_iterators > 0);
+		dict->num_iterators--;
+		}
+
+	dict = that.dict;
+	curr = that.curr;
+	end = that.end;
+	dict->num_iterators++;
+	}
+
+DictIterator& DictIterator::operator=(const DictIterator& that)
+	{
+	if ( this == &that )
+		return *this;
+
+	if ( dict )
+		{
+		assert(dict->num_iterators > 0);
+		dict->num_iterators--;
+		}
+
+	dict = that.dict;
+	curr = that.curr;
+	end = that.end;
+	dict->num_iterators++;
+
+	return *this;
+	}
+
+DictIterator::DictIterator(DictIterator&& that)
+	{
+	if ( this == &that )
+		return;
+
+	if ( dict )
+		{
+		assert(dict->num_iterators > 0);
+		dict->num_iterators--;
+		}
+
+	dict = that.dict;
+	curr = that.curr;
+	end = that.end;
+
+	that.dict = nullptr;
+	}
+
+DictIterator& DictIterator::operator=(DictIterator&& that)
+	{
+	if ( this == &that )
+		return *this;
+
+	if ( dict )
+		{
+		assert(dict->num_iterators > 0);
+		dict->num_iterators--;
+		}
+
+	dict = that.dict;
+	curr = that.curr;
+	end = that.end;
+
+	that.dict = nullptr;
+
+	return *this;
+	}
+
 
 
 

src/Dict.h

@@ -159,8 +159,14 @@ public:
 	using difference_type = std::ptrdiff_t;
 	using iterator_category = std::forward_iterator_tag;
 
+	DictIterator() = default;
 	~DictIterator();
 
+	DictIterator(const DictIterator& that);
+	DictIterator& operator=(const DictIterator& that);
+	DictIterator(DictIterator&& that);
+	DictIterator& operator=(DictIterator&& that);
+
 	reference operator*() { return *curr; }
 	pointer operator->() { return curr; }
 
@@ -171,9 +177,9 @@ public:
 	bool operator!=( const DictIterator& that ) const { return !(*this == that); }
 
 private:
+
 	friend class Dictionary;
 
-	DictIterator() = default;
 	DictIterator(const Dictionary* d, detail::DictEntry* begin, detail::DictEntry* end);
 
 	Dictionary* dict = nullptr;
@@ -331,6 +337,7 @@ public:
 	// Remove all entries.
 	void Clear();
 
+	[[deprecated("Remove in v5.1. MemoryAllocation() is deprecated and will be removed. See GHI-572.")]]
 	size_t MemoryAllocation() const;
 
 	/// The capacity of the table, Buckets + Overflow Size.

src/Expr.cc

@@ -55,7 +55,7 @@ const char* expr_name(BroExprTag t)
 		"inline()",
 		"[]=", "$=",
 		"vec+=",
-		"to_any_coerce", "from_any_coerce",
+		"to_any_coerce", "from_any_coerce", "from_any_vec_coerce",
 		"any[]",
 		"nop",
 
@@ -211,6 +211,11 @@ ExprPtr Expr::MakeLvalue()
 	return {NewRef{}, this};
 	}
 
+bool Expr::InvertSense()
+	{
+	return false;
+	}
+
 void Expr::EvalIntoAggregate(const zeek::Type* /* t */, Val* /* aggr */,
                              Frame* /* f */) const
 	{
@@ -568,10 +573,16 @@ void NameExpr::ExprDescribe(ODesc* d) const
 ConstExpr::ConstExpr(ValPtr arg_val)
 	: Expr(EXPR_CONST), val(std::move(arg_val))
 	{
-	if ( val->GetType()->Tag() == TYPE_LIST && val->AsListVal()->Length() == 1 )
+	if ( val )
-		val = val->AsListVal()->Idx(0);
+		{
+		if ( val->GetType()->Tag() == TYPE_LIST &&
+		     val->AsListVal()->Length() == 1 )
+			val = val->AsListVal()->Idx(0);
 
-	SetType(val->GetType());
+		SetType(val->GetType());
+		}
+	else
+		SetError();
 	}
 
 void ConstExpr::ExprDescribe(ODesc* d) const
@@ -1386,9 +1397,12 @@ SizeExpr::SizeExpr(ExprPtr arg_op)
 	if ( IsError() )
 		return;
 
-	if ( op->GetType()->Tag() == TYPE_ANY )
+	auto& t = op->GetType();
+
+	if ( t->Tag() == TYPE_ANY )
 		SetType(base_type(TYPE_ANY));
-	else if ( op->GetType()->InternalType() == TYPE_INTERNAL_DOUBLE )
+	else if ( t->Tag() == TYPE_FILE || t->Tag() == TYPE_SUBNET ||
+	          t->InternalType() == TYPE_INTERNAL_DOUBLE )
 		SetType(base_type(TYPE_DOUBLE));
 	else
 		SetType(base_type(TYPE_COUNT));
@@ -2043,6 +2057,12 @@ ValPtr EqExpr::Fold(Val* v1, Val* v2) const
 		return BinaryExpr::Fold(v1, v2);
 	}
 
+bool EqExpr::InvertSense()
+	{
+	tag = (tag == EXPR_EQ ? EXPR_NE : EXPR_EQ);
+	return true;
+	}
+
 RelExpr::RelExpr(BroExprTag arg_tag, ExprPtr arg_op1, ExprPtr arg_op2)
 	: BinaryExpr(arg_tag, std::move(arg_op1), std::move(arg_op2))
 	{
@@ -2100,6 +2120,19 @@ void RelExpr::Canonicize()
 		}
 	}
 
+bool RelExpr::InvertSense()
+	{
+	switch ( tag ) {
+	case EXPR_LT:   tag = EXPR_GE; return true;
+	case EXPR_LE:   tag = EXPR_GT; return true;
+	case EXPR_GE:   tag = EXPR_LT; return true;
+	case EXPR_GT:   tag = EXPR_LE; return true;
+
+	default:
+		return false;
+	}
+	}
+
 CondExpr::CondExpr(ExprPtr arg_op1, ExprPtr arg_op2, ExprPtr arg_op3)
 	: Expr(EXPR_COND),
 	  op1(std::move(arg_op1)), op2(std::move(arg_op2)), op3(std::move(arg_op3))
@@ -2900,7 +2933,7 @@ ValPtr IndexExpr::Eval(Frame* f) const
 		{
 		VectorVal* v_v1 = v1->AsVectorVal();
 		VectorVal* v_v2 = indv->AsVectorVal();
-		auto v_result = make_intrusive<VectorVal>(GetType<VectorType>());
+		auto vt = cast_intrusive<VectorType>(GetType());
 
 		// Booleans select each element (or not).
 		if ( IsBool(v_v2->GetType()->Yield()->Tag()) )
@@ -2911,23 +2944,11 @@ ValPtr IndexExpr::Eval(Frame* f) const
 				return nullptr;
 				}
 
-			for ( unsigned int i = 0; i < v_v2->Size(); ++i )
+			return vector_bool_select(vt, v_v1, v_v2);
-				{
-				if ( v_v2->BoolAt(i) )
-					v_result->Assign(v_result->Size() + 1, v_v1->ValAt(i));
-				}
 			}
 		else
-			{ // The elements are indices.
+			// Elements are indices.
-			// ### Should handle negative indices here like
+			return vector_int_select(vt, v_v1, v_v2);
-			// S does, i.e., by excluding those elements.
-			// Probably only do this if *all* are negative.
-			v_result->Resize(v_v2->Size());
-			for ( unsigned int i = 0; i < v_v2->Size(); ++i )
-				v_result->Assign(i, v_v1->ValAt(v_v2->ValAt(i)->CoerceToInt()));
-			}
-
-		return v_result;
 		}
 	else
 		return Fold(v1.get(), v2.get());
@@ -2954,7 +2975,7 @@ ValPtr IndexExpr::Fold(Val* v1, Val* v2) const
 		break;
 
 	case TYPE_TABLE:
-		v = v1->AsTableVal()->FindOrDefault({NewRef{}, v2}); // Then, we jump into the TableVal here.
+		v = v1->AsTableVal()->FindOrDefault({NewRef{}, v2});
 		break;
 
 	case TYPE_STRING:
@@ -3029,6 +3050,35 @@ VectorValPtr index_slice(VectorVal* vect, int _first, int _last)
 	return result;
 	}
 
+VectorValPtr vector_bool_select(VectorTypePtr vt, const VectorVal* v1,
+                                const VectorVal* v2)
+	{
+	auto v_result = make_intrusive<VectorVal>(std::move(vt));
+
+	for ( unsigned int i = 0; i < v2->Size(); ++i )
+		if ( v2->BoolAt(i) )
+			v_result->Assign(v_result->Size() + 1, v1->ValAt(i));
+
+	return v_result;
+	}
+
+VectorValPtr vector_int_select(VectorTypePtr vt, const VectorVal* v1,
+                               const VectorVal* v2)
+	{
+	auto v_result = make_intrusive<VectorVal>(std::move(vt));
+
+	// The elements are indices.
+	//
+	// ### Should handle negative indices here like S does, i.e.,
+	// by excluding those elements.  Probably only do this if *all*
+	// are negative.
+	v_result->Resize(v2->Size());
+	for ( unsigned int i = 0; i < v2->Size(); ++i )
+		v_result->Assign(i, v1->ValAt(v2->ValAt(i)->CoerceToInt()));
+
+	return v_result;
+	}
+
 void IndexExpr::Assign(Frame* f, ValPtr v)
 	{
 	if ( IsError() )
@@ -3191,8 +3241,10 @@ void HasFieldExpr::ExprDescribe(ODesc* d) const
 		d->Add(field);
 	}
 
+
 RecordConstructorExpr::RecordConstructorExpr(ListExprPtr constructor_list)
-	: Expr(EXPR_RECORD_CONSTRUCTOR), op(std::move(constructor_list))
+	: Expr(EXPR_RECORD_CONSTRUCTOR), op(std::move(constructor_list)),
+	  map(std::nullopt)
 	{
 	if ( IsError() )
 		return;
@@ -3221,8 +3273,45 @@ RecordConstructorExpr::RecordConstructorExpr(ListExprPtr constructor_list)
 	SetType(make_intrusive<RecordType>(record_types));
 	}
 
-RecordConstructorExpr::~RecordConstructorExpr()
+RecordConstructorExpr::RecordConstructorExpr(RecordTypePtr known_rt,
+                                             ListExprPtr constructor_list)
+: Expr(EXPR_RECORD_CONSTRUCTOR), op(std::move(constructor_list))
 	{
+	if ( IsError() )
+		return;
+
+	SetType(known_rt);
+
+	const auto& exprs = op->AsListExpr()->Exprs();
+	map = std::vector<int>(exprs.length());
+
+	int i = 0;
+	for ( const auto& e : exprs )
+		{
+		if ( e->Tag() != EXPR_FIELD_ASSIGN )
+			{
+			Error("bad type in record constructor", e);
+			SetError();
+			continue;
+			}
+
+		auto field = e->AsFieldAssignExpr();
+		int index = known_rt->FieldOffset(field->FieldName());
+
+		if ( index < 0 )
+			{
+			Error("no such field in record", e);
+			SetError();
+			continue;
+			}
+
+		auto known_ft = known_rt->GetFieldType(index);
+
+		if ( ! field->PromoteTo(known_ft) )
+			SetError();
+
+		(*map)[i++] = index;
+		}
 	}
 
 ValPtr RecordConstructorExpr::InitVal(const zeek::Type* t, ValPtr aggr) const
@@ -3262,13 +3351,16 @@ ValPtr RecordConstructorExpr::Eval(Frame* f) const
 	const auto& exprs = op->Exprs();
 	auto rt = cast_intrusive<RecordType>(type);
 
-	if ( exprs.length() != rt->NumFields() )
+	if ( ! map && exprs.length() != rt->NumFields() )
 		RuntimeErrorWithCallStack("inconsistency evaluating record constructor");
 
 	auto rv = make_intrusive<RecordVal>(std::move(rt));
 
 	for ( int i = 0; i < exprs.length(); ++i )
-		rv->Assign(i, exprs[i]->Eval(f));
+		{
+		int ind = map ? (*map)[i] : i;
+		rv->Assign(ind, exprs[i]->Eval(f));
+		}
 
 	return rv;
 	}
@@ -3280,9 +3372,21 @@ bool RecordConstructorExpr::IsPure() const
 
 void RecordConstructorExpr::ExprDescribe(ODesc* d) const
 	{
-	d->Add("[");
+	auto& tn = type->GetName();
-	op->Describe(d);
+
-	d->Add("]");
+	if ( tn.size() > 0 )
+		{
+		d->Add(tn);
+		d->Add("(");
+		op->Describe(d);
+		d->Add(")");
+		}
+	else
+		{
+		d->Add("[");
+		op->Describe(d);
+		d->Add("]");
+		}
 	}
 
 TraversalCode RecordConstructorExpr::Traverse(TraversalCallback* cb) const
@@ -3348,9 +3452,9 @@ TableConstructorExpr::TableConstructorExpr(ListExprPtr constructor_list,
 		if ( expr->Tag() != EXPR_ASSIGN )
 			continue;
 
-		auto idx_expr = expr->AsAssignExpr()->Op1();
+		auto idx_expr = expr->AsAssignExpr()->GetOp1();
-		auto val_expr = expr->AsAssignExpr()->Op2();
+		auto val_expr = expr->AsAssignExpr()->GetOp2();
-		auto yield_type = GetType()->AsTableType()->Yield().get();
+		auto yield_type = GetType()->AsTableType()->Yield();
 
 		// Promote LHS
 		assert(idx_expr->Tag() == EXPR_LIST);
@@ -3361,17 +3465,14 @@ TableConstructorExpr::TableConstructorExpr(ListExprPtr constructor_list,
 
 		loop_over_list(idx_exprs, j)
 			{
-			Expr* idx = idx_exprs[j];
+			ExprPtr idx = {NewRef{}, idx_exprs[j]};
 
-			auto promoted_idx = check_and_promote_expr(idx, indices[j].get());
+			auto promoted_idx = check_and_promote_expr(idx, indices[j]);
 
 			if ( promoted_idx )
 				{
-				if ( promoted_idx.get() != idx )
+				if ( promoted_idx != idx )
-					{
+					Unref(idx_exprs.replace(j, promoted_idx.release()));
-					Unref(idx);
-					idx_exprs.replace(j, promoted_idx.release());
-					}
 
 				continue;
 				}
@@ -3483,7 +3584,7 @@ SetConstructorExpr::SetConstructorExpr(ListExprPtr constructor_list,
 	if ( indices.size() == 1 )
 		{
 		if ( ! check_and_promote_exprs_to_type(op->AsListExpr(),
-		                                       indices[0].get()) )
+		                                       indices[0]) )
 			ExprError("inconsistent type in set constructor");
 		}
 
@@ -3602,7 +3703,7 @@ VectorConstructorExpr::VectorConstructorExpr(ListExprPtr constructor_list,
 		}
 
 	if ( ! check_and_promote_exprs_to_type(op->AsListExpr(),
-					       type->AsVectorType()->Yield().get()) )
+					       type->AsVectorType()->Yield()) )
 		ExprError("inconsistent types in vector constructor");
 	}
 
@@ -3670,6 +3771,12 @@ FieldAssignExpr::FieldAssignExpr(const char* arg_field_name, ExprPtr value)
 	SetType(op->GetType());
 	}
 
+bool FieldAssignExpr::PromoteTo(TypePtr t)
+	{
+	op = check_and_promote_expr(op, t);
+	return op != nullptr;
+	}
+
 void FieldAssignExpr::EvalIntoAggregate(const zeek::Type* t, Val* aggr, Frame* f)
 	const
 	{
@@ -3707,7 +3814,11 @@ void FieldAssignExpr::ExprDescribe(ODesc* d) const
 	d->Add("$");
 	d->Add(FieldName());
 	d->Add("=");
-	op->Describe(d);
+
+	if ( op )
+		op->Describe(d);
+	else
+		d->Add("<error>");
 	}
 
 ArithCoerceExpr::ArithCoerceExpr(ExprPtr arg_op, TypeTag t)
@@ -3739,49 +3850,36 @@ ArithCoerceExpr::ArithCoerceExpr(ExprPtr arg_op, TypeTag t)
 		ExprError("bad coercion value");
 	}
 
-ValPtr ArithCoerceExpr::FoldSingleVal(Val* v, InternalTypeTag t) const
+ValPtr ArithCoerceExpr::FoldSingleVal(ValPtr v, const TypePtr& t) const
 	{
-	switch ( t ) {
+	return check_and_promote(v, t.get(), false, location);
-	case TYPE_INTERNAL_DOUBLE:
-		return make_intrusive<DoubleVal>(v->CoerceToDouble());
-
-	case TYPE_INTERNAL_INT:
-		return val_mgr->Int(v->CoerceToInt());
-
-	case TYPE_INTERNAL_UNSIGNED:
-		return val_mgr->Count(v->CoerceToUnsigned());
-
-	default:
-		RuntimeErrorWithCallStack("bad type in CoerceExpr::Fold");
-		return nullptr;
-	}
 	}
 
 ValPtr ArithCoerceExpr::Fold(Val* v) const
 	{
-	InternalTypeTag t = type->InternalType();
+	auto t = GetType();
 
 	if ( ! is_vector(v) )
 		{
 		// Our result type might be vector, in which case this
 		// invocation is being done per-element rather than on
-		// the whole vector.  Correct the type tag if necessary.
+		// the whole vector.  Correct the type if so.
 		if ( type->Tag() == TYPE_VECTOR )
-			t = GetType()->AsVectorType()->Yield()->InternalType();
+			t = t->AsVectorType()->Yield();
 
-		return FoldSingleVal(v, t);
+		return FoldSingleVal({NewRef{}, v}, t);
 		}
 
-	t = GetType()->AsVectorType()->Yield()->InternalType();
-
 	VectorVal* vv = v->AsVectorVal();
-	auto result = make_intrusive<VectorVal>(GetType<VectorType>());
+	auto result = make_intrusive<VectorVal>(cast_intrusive<VectorType>(t));
+
+	auto yt = t->AsVectorType()->Yield();
 
 	for ( unsigned int i = 0; i < vv->Size(); ++i )
 		{
 		auto elt = vv->ValAt(i);
 		if ( elt )
-			result->Assign(i, FoldSingleVal(elt.get(), t));
+			result->Assign(i, FoldSingleVal(elt, yt));
 		else
 			result->Assign(i, nullptr);
 		}
@@ -4554,9 +4652,9 @@ void LambdaExpr::CheckCaptures()
 		}
 	}
 
-Scope* LambdaExpr::GetScope() const
+ScopePtr LambdaExpr::GetScope() const
 	{
-	return ingredients->scope.get();
+	return ingredients->scope;
 	}
 
 ValPtr LambdaExpr::Eval(Frame* f) const
@@ -4879,12 +4977,12 @@ ValPtr ListExpr::InitVal(const zeek::Type* t, ValPtr aggr) const
 
 		loop_over_list(exprs, i)
 			{
-			Expr* e = exprs[i];
+			ExprPtr e = {NewRef{}, exprs[i]};
 			const auto& vyt = vec->GetType()->AsVectorType()->Yield();
-			auto promoted_e = check_and_promote_expr(e, vyt.get());
+			auto promoted_e = check_and_promote_expr(e, vyt);
 
 			if ( promoted_e )
-				e = promoted_e.get();
+				e = promoted_e;
 
 			if ( ! vec->Assign(i, e->Eval(nullptr)) )
 				{
@@ -5106,34 +5204,38 @@ CastExpr::CastExpr(ExprPtr arg_op, TypePtr t)
 		ExprError("cast not supported");
 	}
 
-ValPtr CastExpr::Eval(Frame* f) const
+ValPtr CastExpr::Fold(Val* v) const
 	{
-	if ( IsError() )
+	std::string error;
-		return nullptr;
+	auto res = cast_value({NewRef{}, v}, GetType(), error);
 
-	auto v = op->Eval(f);
+	if ( ! res )
+		RuntimeError(error.c_str());
 
-	if ( ! v )
+	return res;
-		return nullptr;
+	}
 
-	auto nv = cast_value_to_type(v.get(), GetType().get());
+ValPtr cast_value(ValPtr v, const TypePtr& t, std::string& error)
+	{
+	auto nv = cast_value_to_type(v.get(), t.get());
 
 	if ( nv )
 		return nv;
 
 	ODesc d;
+
 	d.Add("invalid cast of value with type '");
 	v->GetType()->Describe(&d);
 	d.Add("' to type '");
-	GetType()->Describe(&d);
+	t->Describe(&d);
 	d.Add("'");
 
 	if ( same_type(v->GetType(), Broker::detail::DataVal::ScriptDataType()) &&
-		 ! v->AsRecordVal()->HasField(0) )
+	     ! v->AsRecordVal()->HasField(0) )
 		d.Add(" (nil $data field)");
 
-	RuntimeError(d.Description());
+	error = d.Description();
-	return nullptr;  // not reached.
+	return nullptr;
 	}
 
 void CastExpr::ExprDescribe(ODesc* d) const
@@ -5180,34 +5282,42 @@ ExprPtr get_assign_expr(ExprPtr op1, ExprPtr op2, bool is_init)
 			std::move(op1), std::move(op2), is_init);
 	}
 
-ExprPtr check_and_promote_expr(Expr* const e, zeek::Type* t)
+ExprPtr check_and_promote_expr(ExprPtr e, TypePtr t)
 	{
 	const auto& et = e->GetType();
 	TypeTag e_tag = et->Tag();
 	TypeTag t_tag = t->Tag();
 
-	if ( t->Tag() == TYPE_ANY )
+	if ( t_tag == TYPE_ANY )
-		return {NewRef{}, e};
+		{
+		if ( e_tag != TYPE_ANY )
+			return make_intrusive<CoerceToAnyExpr>(e);
+
+		return e;
+		}
+
+	if ( e_tag == TYPE_ANY )
+		return make_intrusive<CoerceFromAnyExpr>(e, t);
 
 	if ( EitherArithmetic(t_tag, e_tag) )
 		{
 		if ( e_tag == t_tag )
-			return {NewRef{}, e};
+			return e;
 
 		if ( ! BothArithmetic(t_tag, e_tag) )
 			{
-			t->Error("arithmetic mixed with non-arithmetic", e);
+			t->Error("arithmetic mixed with non-arithmetic", e.get());
 			return nullptr;
 			}
 
 		TypeTag mt = max_type(t_tag, e_tag);
 		if ( mt != t_tag )
 			{
-			t->Error("over-promotion of arithmetic value", e);
+			t->Error("over-promotion of arithmetic value", e.get());
 			return nullptr;
 			}
 
-		return make_intrusive<ArithCoerceExpr>(IntrusivePtr{NewRef{}, e}, t_tag);
+		return make_intrusive<ArithCoerceExpr>(e, t_tag);
 		}
 
 	if ( t->Tag() == TYPE_RECORD && et->Tag() == TYPE_RECORD )
@@ -5216,14 +5326,13 @@ ExprPtr check_and_promote_expr(Expr* const e, zeek::Type* t)
 		RecordType* et_r = et->AsRecordType();
 
 		if ( same_type(t, et) )
-			return {NewRef{}, e};
+			return e;
 
 		if ( record_promotion_compatible(t_r, et_r) )
-			return make_intrusive<RecordCoerceExpr>(
+			return make_intrusive<RecordCoerceExpr>(e,
-				IntrusivePtr{NewRef{}, e},
 				IntrusivePtr{NewRef{}, t_r});
 
-		t->Error("incompatible record types", e);
+		t->Error("incompatible record types", e.get());
 		return nullptr;
 		}
 
@@ -5232,23 +5341,21 @@ ExprPtr check_and_promote_expr(Expr* const e, zeek::Type* t)
 		{
 		if ( t->Tag() == TYPE_TABLE && et->Tag() == TYPE_TABLE &&
 			  et->AsTableType()->IsUnspecifiedTable() )
-			return make_intrusive<TableCoerceExpr>(
+			return make_intrusive<TableCoerceExpr>(e,
-				IntrusivePtr{NewRef{}, e},
 				IntrusivePtr{NewRef{}, t->AsTableType()});
 
 		if ( t->Tag() == TYPE_VECTOR && et->Tag() == TYPE_VECTOR &&
 		     et->AsVectorType()->IsUnspecifiedVector() )
-			return make_intrusive<VectorCoerceExpr>(
+			return make_intrusive<VectorCoerceExpr>(e,
-				IntrusivePtr{NewRef{}, e},
 				IntrusivePtr{NewRef{}, t->AsVectorType()});
 
 		if ( t->Tag() != TYPE_ERROR && et->Tag() != TYPE_ERROR )
-			t->Error("type clash", e);
+			t->Error("type clash", e.get());
 
 		return nullptr;
 		}
 
-	return {NewRef{}, e};
+	return e;
 	}
 
 bool check_and_promote_exprs(ListExpr* const elements, TypeList* types)
@@ -5267,8 +5374,8 @@ bool check_and_promote_exprs(ListExpr* const elements, TypeList* types)
 
 	loop_over_list(el, i)
 		{
-		Expr* e = el[i];
+		ExprPtr e = {NewRef{}, el[i]};
-		auto promoted_e = check_and_promote_expr(e, tl[i].get());
+		auto promoted_e = check_and_promote_expr(e, tl[i]);
 
 		if ( ! promoted_e )
 			{
@@ -5276,17 +5383,14 @@ bool check_and_promote_exprs(ListExpr* const elements, TypeList* types)
 			return false;
 			}
 
-		if ( promoted_e.get() != e )
+		if ( promoted_e != e )
-			{
+			Unref(el.replace(i, promoted_e.release()));
-			Unref(e);
-			el.replace(i, promoted_e.release());
-			}
 		}
 
 	return true;
 	}
 
-bool check_and_promote_args(ListExpr* const args, RecordType* types)
+bool check_and_promote_args(ListExpr* const args, const RecordType* types)
 	{
 	ExprPList& el = args->Exprs();
 	int ntypes = types->NumFields();
@@ -5303,7 +5407,7 @@ bool check_and_promote_args(ListExpr* const args, RecordType* types)
 		// arguments using &default expressions.
 		for ( int i = ntypes - 1; i >= el.length(); --i )
 			{
-			TypeDecl* td = types->FieldDecl(i);
+			auto td = types->FieldDecl(i);
 			const auto& def_attr = td->attrs ? td->attrs->Find(ATTR_DEFAULT).get() : nullptr;
 
 			if ( ! def_attr )
@@ -5340,29 +5444,26 @@ bool check_and_promote_args(ListExpr* const args, RecordType* types)
 	return rval;
 	}
 
-bool check_and_promote_exprs_to_type(ListExpr* const elements, zeek::Type* type)
+bool check_and_promote_exprs_to_type(ListExpr* const elements, TypePtr t)
 	{
 	ExprPList& el = elements->Exprs();
 
-	if ( type->Tag() == TYPE_ANY )
+	if ( t->Tag() == TYPE_ANY )
 		return true;
 
 	loop_over_list(el, i)
 		{
-		Expr* e = el[i];
+		ExprPtr e = {NewRef{}, el[i]};
-		auto promoted_e = check_and_promote_expr(e, type);
+		auto promoted_e = check_and_promote_expr(e, t);
 
 		if ( ! promoted_e )
 			{
-			e->Error("type mismatch", type);
+			e->Error("type mismatch", t.get());
 			return false;
 			}
 
-		if ( promoted_e.get() != e )
+		if ( promoted_e != e )
-			{
+			Unref(el.replace(i, promoted_e.release()));
-			Unref(e);
-			el.replace(i, promoted_e.release());
-			}
 		}
 
 	return true;

src/Expr.h

@@ -27,6 +27,7 @@ class Frame;
 class Scope;
 struct function_ingredients;
 using IDPtr = IntrusivePtr<ID>;
+using ScopePtr = IntrusivePtr<Scope>;
 
 enum BroExprTag : int {
 	EXPR_ANY = -1,
@@ -71,7 +72,7 @@ enum BroExprTag : int {
 	// ASTs produced by parsing .zeek script files.
 	EXPR_INDEX_ASSIGN, EXPR_FIELD_LHS_ASSIGN,
 	EXPR_APPEND_TO,
-	EXPR_TO_ANY_COERCE, EXPR_FROM_ANY_COERCE,
+	EXPR_TO_ANY_COERCE, EXPR_FROM_ANY_COERCE, EXPR_FROM_ANY_VEC_COERCE,
 	EXPR_ANY_INDEX,
 
 	EXPR_NOP,
@@ -94,12 +95,16 @@ class ForExpr;
 class HasFieldExpr;
 class IndexAssignExpr;
 class IndexExpr;
-class IsExpr;
 class InlineExpr;
+class IsExpr;
 class LambdaExpr;
 class ListExpr;
 class NameExpr;
+class RecordCoerceExpr;
+class RecordConstructorExpr;
 class RefExpr;
+class SetConstructorExpr;
+class TableConstructorExpr;
 
 class Expr;
 using CallExprPtr = IntrusivePtr<CallExpr>;
@@ -192,6 +197,11 @@ public:
 	// the current value of expr (this is the default method).
 	virtual ExprPtr MakeLvalue();
 
+	// Invert the sense of the operation.  Returns true if the expression
+	// was invertible (currently only true for relational/equality
+	// expressions), false otherwise.
+	virtual bool InvertSense();
+
 	// Marks the expression as one requiring (or at least appearing
 	// with) parentheses.  Used for pretty-printing.
 	void MarkParen()		{ paren = true; }
@@ -215,12 +225,16 @@ public:
 	ZEEK_EXPR_ACCESSOR_DECLS(HasFieldExpr)
 	ZEEK_EXPR_ACCESSOR_DECLS(IndexAssignExpr)
 	ZEEK_EXPR_ACCESSOR_DECLS(IndexExpr)
-	ZEEK_EXPR_ACCESSOR_DECLS(IsExpr)
 	ZEEK_EXPR_ACCESSOR_DECLS(InlineExpr)
+	ZEEK_EXPR_ACCESSOR_DECLS(IsExpr)
 	ZEEK_EXPR_ACCESSOR_DECLS(LambdaExpr)
 	ZEEK_EXPR_ACCESSOR_DECLS(ListExpr)
 	ZEEK_EXPR_ACCESSOR_DECLS(NameExpr)
+	ZEEK_EXPR_ACCESSOR_DECLS(RecordCoerceExpr)
+	ZEEK_EXPR_ACCESSOR_DECLS(RecordConstructorExpr)
 	ZEEK_EXPR_ACCESSOR_DECLS(RefExpr)
+	ZEEK_EXPR_ACCESSOR_DECLS(SetConstructorExpr)
+	ZEEK_EXPR_ACCESSOR_DECLS(TableConstructorExpr)
 
 	void Describe(ODesc* d) const override final;
 
@@ -782,6 +796,7 @@ public:
 	ExprPtr Duplicate() override;
 	bool WillTransform(Reducer* c) const override;
 	ExprPtr Reduce(Reducer* c, StmtPtr& red_stmt) override;
+	bool InvertSense() override;
 
 protected:
 	ValPtr Fold(Val* v1, Val* v2) const override;
@@ -796,6 +811,7 @@ public:
 	ExprPtr Duplicate() override;
 	bool WillTransform(Reducer* c) const override;
 	ExprPtr Reduce(Reducer* c, StmtPtr& red_stmt) override;
+	bool InvertSense() override;
 };
 
 class CondExpr final : public Expr {
@@ -957,6 +973,15 @@ extern VectorValPtr index_slice(VectorVal* vect, int first, int last);
 // (exactly) two values.
 extern StringValPtr index_string(const String* s, const ListVal* lv);
 
+// Returns a vector indexed by a boolean vector.
+extern VectorValPtr vector_bool_select(VectorTypePtr vt, const VectorVal* v1,
+                                       const VectorVal* v2);
+
+// Returns a vector indexed by a numeric vector (which specifies the
+// indices to select).
+extern VectorValPtr vector_int_select(VectorTypePtr vt, const VectorVal* v1,
+                                      const VectorVal* v2);
+
 class IndexExprWhen final : public IndexExpr {
 public:
 	static inline std::vector<ValPtr> results = {};
@@ -1046,9 +1071,13 @@ protected:
 class RecordConstructorExpr final : public Expr {
 public:
 	explicit RecordConstructorExpr(ListExprPtr constructor_list);
-	~RecordConstructorExpr() override;
 
-	ListExpr* Op() const	{ return op.get(); }
+        // This form is used to construct records of a known (ultimate) type.
+	explicit RecordConstructorExpr(RecordTypePtr known_rt,
+	                               ListExprPtr constructor_list);
+
+	ListExprPtr Op() const    { return op; }
+	const auto& Map() const	{ return map; }
 
 	ValPtr Eval(Frame* f) const override;
 
@@ -1069,6 +1098,7 @@ protected:
 	void ExprDescribe(ODesc* d) const override;
 
 	ListExprPtr op;
+	std::optional<std::vector<int>> map;
 };
 
 class TableConstructorExpr final : public UnaryExpr {
@@ -1149,6 +1179,14 @@ public:
 
 	const char* FieldName() const	{ return field_name.c_str(); }
 
+	// When these are first constructed, we don't know the type.
+	// The following method coerces/promotes the assignment expression
+	// as needed, once we do know the type.
+	//
+	// Returns true on success, false if the types were incompatible
+	// (in which case an error is reported).
+	bool PromoteTo(TypePtr t);
+
 	void EvalIntoAggregate(const zeek::Type* t, Val* aggr, Frame* f) const override;
 	bool IsRecordElement(TypeDecl* td) const override;
 
@@ -1175,7 +1213,7 @@ public:
 	ExprPtr Reduce(Reducer* c, StmtPtr& red_stmt) override;
 
 protected:
-	ValPtr FoldSingleVal(Val* v, InternalTypeTag t) const;
+	ValPtr FoldSingleVal(ValPtr v, const TypePtr& t) const;
 	ValPtr Fold(Val* v) const override;
 };
 
@@ -1291,6 +1329,7 @@ public:
 
 	Expr* Func() const	{ return func.get(); }
 	ListExpr* Args() const	{ return args.get(); }
+	ListExprPtr ArgsPtr() const	{ return args; }
 
 	bool IsPure() const override;
 
@@ -1332,7 +1371,7 @@ public:
 	ValPtr Eval(Frame* f) const override;
 	TraversalCode Traverse(TraversalCallback* cb) const override;
 
-	Scope* GetScope() const;
+	ScopePtr GetScope() const;
 
 	// Optimization-related:
 	ExprPtr Duplicate() override;
@@ -1442,10 +1481,14 @@ public:
 	ExprPtr Duplicate() override;
 
 protected:
-	ValPtr Eval(Frame* f) const override;
+	ValPtr Fold(Val* v) const override;
 	void ExprDescribe(ODesc* d) const override;
 };
 
+// Returns the value 'v' cast to type 't'.  On an error, returns nil
+// and populates "error" with an error message.
+extern ValPtr cast_value(ValPtr v, const TypePtr& t, std::string& error);
+
 class IsExpr final : public UnaryExpr {
 public:
 	IsExpr(ExprPtr op, TypePtr t);
@@ -1583,6 +1626,20 @@ protected:
 	ExprPtr Duplicate() override;
 };
 
+// ... and for conversion from a "vector of any" type.
+class CoerceFromAnyVecExpr : public UnaryExpr {
+public:
+	// to_type is yield type, not VectorType.
+	CoerceFromAnyVecExpr(ExprPtr op, TypePtr to_type);
+
+	// Can't use UnaryExpr's Eval() because it will do folding
+	// over the individual vector elements.
+	ValPtr Eval(Frame* f) const override;
+
+protected:
+	ExprPtr Duplicate() override;
+};
+
 // Expression used to explicitly capture [a, b, c, ...] = x assignments.
 class AnyIndexExpr : public UnaryExpr {
 public:
@@ -1635,26 +1692,22 @@ ExprPtr get_assign_expr(
 	ExprPtr op1,
 	ExprPtr op2, bool is_init);
 
-// Type-check the given expression(s) against the given type(s).  Complain
-// if the expression cannot match the given type, returning 0.  If it can
-// match, promote it as necessary (modifying the ref parameter accordingly)
-// and return 1.
-//
-// The second, third, and fourth forms are for promoting a list of
-// expressions (which is updated in place) to either match a list of
-// types or a single type.
-//
-// Note, the type is not "const" because it can be ref'd.
-
 /**
- * Returns nullptr if the expression cannot match or a promoted
+ * Type-check the given expression(s) against the given type(s).  Complain
- * expression.
+ * if the expression cannot match the given type, returning nullptr;
+ * otherwise, returns an expression reflecting the promotion.
+ *
+ * The second, third, and fourth forms are for promoting a list of
+ * expressions (which is updated in place) to either match a list of
+ * types or a single type.
+ *
+ * Note, the type is not "const" because it can be ref'd.
  */
-extern ExprPtr check_and_promote_expr(Expr* e, Type* t);
+extern ExprPtr check_and_promote_expr(ExprPtr e, TypePtr t);
 
 extern bool check_and_promote_exprs(ListExpr* elements, TypeList* types);
-extern bool check_and_promote_args(ListExpr* args, RecordType* types);
+extern bool check_and_promote_args(ListExpr* args, const RecordType* types);
-extern bool check_and_promote_exprs_to_type(ListExpr* elements, Type* type);
+extern bool check_and_promote_exprs_to_type(ListExpr* elements, TypePtr type);
 
 // Returns a ListExpr simplified down to a list a values, or nil
 // if they couldn't all be reduced.

src/Frag.h

@@ -83,6 +83,7 @@ public:
 
 	size_t Size() const	{ return fragments.size(); }
 	size_t MaxFragments() const 	{ return max_fragments; }
+	[[deprecated("Remove in v5.1. MemoryAllocation() is deprecated and will be removed. See GHI-572.")]]
 	uint32_t MemoryAllocation() const;
 
 private:

src/Frame.cc

@@ -430,7 +430,8 @@ std::pair<bool, FramePtr> Frame::Unserialize(const broker::vector& data,
 
 	if ( captures || *has_name == "CopyFrame" )
 		{
-		ASSERT(captures && *has_name == "CopyFrame");
+		if ( captures )
+			ASSERT(*has_name == "CopyFrame");
 
 		auto has_body = broker::get_if<broker::vector>(*where);
 		if ( ! has_body )
@@ -618,7 +619,7 @@ void Frame::CaptureClosure(Frame* c, IDPList arg_outer_ids)
 
 const detail::Location* Frame::GetCallLocation() const
 	{
-	return call ? call->GetLocationInfo() : nullptr;
+	return call ? call->GetLocationInfo() : call_loc;
 	}
 
 void Frame::SetTrigger(trigger::TriggerPtr arg_trigger)

src/Frame.h

@@ -262,6 +262,8 @@ public:
 	void SetCall(const CallExpr* arg_call)	{ call = arg_call; }
 	void ClearCall()			{ call = nullptr; }
 	const CallExpr* GetCall() const		{ return call; }
+
+	void SetCallLoc(const Location* loc)	{ call_loc = loc; }
 	const detail::Location* GetCallLocation() const;
 
 	void SetDelayed()	{ delayed = true; }
@@ -389,7 +391,8 @@ private:
 	Stmt* next_stmt;
 
 	trigger::TriggerPtr trigger;
-	const CallExpr* call;
+	const CallExpr* call = nullptr;
+	const Location* call_loc = nullptr;	// only needed if call is nil
 
 	std::unique_ptr<std::vector<ScriptFunc*>> functions_with_closure_frame_reference;
 };

src/Func.cc

@@ -44,7 +44,7 @@
 #include "zeek/File.h"
 #include "zeek/Frame.h"
 #include "zeek/Var.h"
-#include "zeek/analyzer/protocol/login/Login.h"
+#include "zeek/analyzer/protocol/tcp/TCP.h"
 #include "zeek/session/Manager.h"
 #include "zeek/RE.h"
 #include "zeek/Event.h"
@@ -62,6 +62,7 @@
 #include "option.bif.func_h"
 #include "supervisor.bif.func_h"
 #include "packet_analysis.bif.func_h"
+#include "CPP-load.bif.func_h"
 
 #include "zeek.bif.func_def"
 #include "stats.bif.func_def"
@@ -70,6 +71,7 @@
 #include "option.bif.func_def"
 #include "supervisor.bif.func_def"
 #include "packet_analysis.bif.func_def"
+#include "CPP-load.bif.func_def"
 
 extern	RETSIGTYPE sig_handler(int signo);
 
@@ -199,8 +201,8 @@ void Func::DescribeDebug(ODesc* d, const Args* args) const
 detail::TraversalCode Func::Traverse(detail::TraversalCallback* cb) const
 	{
 	// FIXME: Make a fake scope for builtins?
-	detail::Scope* old_scope = cb->current_scope;
+	auto old_scope = cb->current_scope;
-	cb->current_scope = scope.get();
+	cb->current_scope = scope;
 
 	detail::TraversalCode tc = cb->PreFunction(this);
 	HANDLE_TC_STMT_PRE(tc);
@@ -305,11 +307,35 @@ ScriptFunc::ScriptFunc(const IDPtr& arg_id, StmtPtr arg_body,
 		Body b;
 		b.stmts = AddInits(std::move(arg_body), aggr_inits);
 		current_body = b.stmts;
-		b.priority = priority;
+		current_priority = b.priority = priority;
 		bodies.push_back(b);
 		}
 	}
 
+ScriptFunc::ScriptFunc(std::string _name, FuncTypePtr ft,
+	               std::vector<StmtPtr> bs, std::vector<int> priorities)
+	{
+	name = std::move(_name);
+	frame_size = ft->ParamList()->GetTypes().size();
+	type = std::move(ft);
+
+	auto n = bs.size();
+	ASSERT(n == priorities.size());
+
+	for ( auto i = 0u; i < n; ++i )
+		{
+		Body b;
+		b.stmts = std::move(bs[i]);
+		b.priority = priorities[i];
+		bodies.push_back(b);
+		}
+
+	sort(bodies.begin(), bodies.end());
+
+	current_body = bodies[0].stmts;
+	current_priority = bodies[0].priority;
+	}
+
 ScriptFunc::~ScriptFunc()
 	{
 	if ( ! weak_closure_ref )
@@ -542,9 +568,8 @@ void ScriptFunc::AddBody(StmtPtr new_body,
 
 	Body b;
 	b.stmts = new_body;
-	b.priority = priority;
-
 	current_body = new_body;
+	current_priority = b.priority = priority;
 
 	bodies.push_back(b);
 	sort(bodies.begin(), bodies.end());
@@ -558,6 +583,7 @@ void ScriptFunc::ReplaceBody(const StmtPtr& old_body, StmtPtr new_body)
 		if ( body.stmts.get() == old_body.get() )
 			{
 			body.stmts = new_body;
+			current_priority = body.priority;
 			found_it = true;
 			}
 

src/Func.h

@@ -12,7 +12,7 @@
 #include "zeek/ZeekList.h"
 #include "zeek/Stmt.h"
 #include "zeek/Obj.h"
-#include "zeek/IntrusivePtr.h"
+#include "zeek/Scope.h"
 #include "zeek/Type.h" /* for function_flavor */
 #include "zeek/TraverseTypes.h"
 #include "zeek/ZeekArgs.h"
@@ -103,7 +103,7 @@ public:
 	                     size_t new_frame_size, int priority = 0);
 
 	virtual void SetScope(detail::ScopePtr newscope);
-	virtual detail::Scope* GetScope() const		{ return scope.get(); }
+	virtual detail::ScopePtr GetScope() const	{ return scope; }
 
 	const FuncTypePtr& GetType() const
 		{ return type; }
@@ -151,6 +151,10 @@ public:
 	        const std::vector<IDPtr>& inits,
 	        size_t frame_size, int priority);
 
+	// For compiled scripts.
+	ScriptFunc(std::string name, FuncTypePtr ft,
+	           std::vector<StmtPtr> bodies, std::vector<int> priorities);
+
 	~ScriptFunc() override;
 
 	bool IsPure() const override;
@@ -239,6 +243,7 @@ public:
 				detail::StmtPtr new_body);
 
 	StmtPtr CurrentBody() const		{ return current_body; }
+	int CurrentPriority() const		{ return current_priority; }
 
 	/**
 	 * Returns the function's frame size.
@@ -307,8 +312,11 @@ private:
 
 	OffsetMap* captures_offset_mapping = nullptr;
 
-	// The most recently added/updated body.
+	// The most recently added/updated body ...
 	StmtPtr current_body;
+
+	// ... and its priority.
+	int current_priority;
 };
 
 using built_in_func = BifReturnVal (*)(Frame* frame, const Args* args);

src/Hash.h

@@ -261,6 +261,7 @@ public:
 	int Size() const	{ return size; }
 	hash_t Hash() const	{ return hash; }
 
+	[[deprecated("Remove in v5.1. MemoryAllocation() is deprecated and will be removed. See GHI-572.")]]
 	unsigned int MemoryAllocation() const	{ return padded_sizeof(*this) + util::pad_size(size); }
 
 	static hash_t HashBytes(const void* bytes, int size);

src/ID.cc

@@ -285,6 +285,11 @@ const AttrPtr& ID::GetAttr(AttrTag t) const
 	return attrs ? attrs->Find(t) : Attr::nil;
 	}
 
+void ID::AddInitExpr(ExprPtr init_expr)
+	{
+	init_exprs.emplace_back(std::move(init_expr));
+	}
+
 bool ID::IsDeprecated() const
 	{
 	return GetAttr(ATTR_DEPRECATED) != nullptr;

src/ID.h

@@ -112,6 +112,10 @@ public:
 
 	const AttrPtr& GetAttr(AttrTag t) const;
 
+	void AddInitExpr(ExprPtr init_expr);
+	const std::vector<ExprPtr>& GetInitExprs() const
+		{ return init_exprs; }
+
 	bool IsDeprecated() const;
 
 	void MakeDeprecated(ExprPtr deprecation);
@@ -156,6 +160,13 @@ protected:
 	int offset;
 	ValPtr val;
 	AttributesPtr attrs;
+
+	// Expressions used to initialize the identifier, for use by
+	// the scripts-to-C++ compiler.  We need to track all of them
+	// because it's possible that a global value gets created using
+	// one of the earlier instances rather than the last one.
+	std::vector<ExprPtr> init_exprs;
+
 	// contains list of functions that are called when an option changes
 	std::multimap<int, FuncPtr> option_handlers;
 

src/IP.cc

@@ -157,7 +157,6 @@ RecordValPtr IPv6_Hdr::ToVal(VectorValPtr chain) const
 		}
 		break;
 
-#ifdef ENABLE_MOBILE_IPV6
 	case IPPROTO_MOBILITY:
 		{
 		static auto ip6_mob_type = id::find_type<RecordType>("ip6_mobility_hdr");
@@ -290,7 +289,6 @@ RecordValPtr IPv6_Hdr::ToVal(VectorValPtr chain) const
 		rv->Assign(5, std::move(msg));
 		}
 		break;
-#endif //ENABLE_MOBILE_IPV6
 
 	default:
 		break;
@@ -445,9 +443,7 @@ static inline bool isIPv6ExtHeader(uint8_t type)
 	case IPPROTO_FRAGMENT:
 	case IPPROTO_AH:
 	case IPPROTO_ESP:
-#ifdef ENABLE_MOBILE_IPV6
 	case IPPROTO_MOBILITY:
-#endif
 		return true;
 	default:
 		return false;
@@ -457,9 +453,7 @@ static inline bool isIPv6ExtHeader(uint8_t type)
 IPv6_Hdr_Chain::~IPv6_Hdr_Chain()
 	{
 	for ( size_t i = 0; i < chain.size(); ++i ) delete chain[i];
-#ifdef ENABLE_MOBILE_IPV6
 	delete homeAddr;
-#endif
 	delete finalDst;
 	}
 
@@ -509,11 +503,9 @@ void IPv6_Hdr_Chain::Init(const struct ip6_hdr* ip6, int total_len,
 		if ( current_type == IPPROTO_ROUTING )
 			ProcessRoutingHeader((const struct ip6_rthdr*) hdrs, cur_len);
 
-#ifdef ENABLE_MOBILE_IPV6
 		// Only Mobile IPv6 has a destination option we care about right now.
 		if ( current_type == IPPROTO_DSTOPTS )
 			ProcessDstOpts((const struct ip6_dest*) hdrs, cur_len);
-#endif
 
 		hdrs += cur_len;
 		length += cur_len;
@@ -521,9 +513,7 @@ void IPv6_Hdr_Chain::Init(const struct ip6_hdr* ip6, int total_len,
 
 		} while ( current_type != IPPROTO_FRAGMENT &&
 				  current_type != IPPROTO_ESP &&
-#ifdef ENABLE_MOBILE_IPV6
 				  current_type != IPPROTO_MOBILITY &&
-#endif
 				  isIPv6ExtHeader(next_type) );
 	}
 
@@ -540,10 +530,8 @@ bool IPv6_Hdr_Chain::IsFragment() const
 
 IPAddr IPv6_Hdr_Chain::SrcAddr() const
 	{
-#ifdef ENABLE_MOBILE_IPV6
 	if ( homeAddr )
 		return IPAddr(*homeAddr);
-#endif
 	if ( chain.empty() )
 		{
 		reporter->InternalWarning("empty IPv6 header chain");
@@ -595,7 +583,6 @@ void IPv6_Hdr_Chain::ProcessRoutingHeader(const struct ip6_rthdr* r, uint16_t le
 		}
 		break;
 
-#ifdef ENABLE_MOBILE_IPV6
 	case 2: // Defined by Mobile IPv6 RFC 6275.
 		{
 		if ( r->ip6r_segleft > 0 )
@@ -607,7 +594,6 @@ void IPv6_Hdr_Chain::ProcessRoutingHeader(const struct ip6_rthdr* r, uint16_t le
 			}
 		}
 		break;
-#endif
 
 	default:
 		reporter->Weird(SrcAddr(), DstAddr(), "unknown_routing_type",
@@ -616,9 +602,14 @@ void IPv6_Hdr_Chain::ProcessRoutingHeader(const struct ip6_rthdr* r, uint16_t le
 	}
 	}
 
-#ifdef ENABLE_MOBILE_IPV6
 void IPv6_Hdr_Chain::ProcessDstOpts(const struct ip6_dest* d, uint16_t len)
 	{
+	// Skip two bytes to get the beginning of the first option structure. These
+	// two bytes are the protocol for the next header and extension header length,
+	// already known to exist before calling this method.  See header format:
+	// https://datatracker.ietf.org/doc/html/rfc8200#section-4.6
+	assert(len >= 2);
+
 	const u_char* data = (const u_char*) d;
 	len -= 2 * sizeof(uint8_t);
 	data += 2* sizeof(uint8_t);
@@ -627,35 +618,45 @@ void IPv6_Hdr_Chain::ProcessDstOpts(const struct ip6_dest* d, uint16_t len)
 		{
 		const struct ip6_opt* opt = (const struct ip6_opt*) data;
 		switch ( opt->ip6o_type ) {
-		case 201: // Home Address Option, Mobile IPv6 RFC 6275 section 6.3
+		case 0:
-			{
+			// If option type is zero, it's a Pad0 and can be just a single
-			if ( opt->ip6o_len == 16 )
+			// byte in width. Skip over it.
-				if ( homeAddr )
-					reporter->Weird(SrcAddr(), DstAddr(), "multiple_home_addr_opts");
-				else
-					homeAddr = new IPAddr(*((const in6_addr*)(data + 2)));
-			else
-				reporter->Weird(SrcAddr(), DstAddr(), "bad_home_addr_len");
-			}
-			break;
-
-		default:
-			break;
-		}
-
-		if ( opt->ip6o_type == 0 )
-			{
 			data += sizeof(uint8_t);
 			len -= sizeof(uint8_t);
-			}
+			break;
-		else
+		default:
 			{
-			data += 2 * sizeof(uint8_t) + opt->ip6o_len;
+			// Double-check that the len can hold the whole option structure.
-			len -= 2 * sizeof(uint8_t) + opt->ip6o_len;
+			// Otherwise we get a buffer-overflow when we check the option_len.
+			// Also check that it holds everything for the option itself.
+			if ( len < sizeof(struct ip6_opt) ||
+			     len < sizeof(struct ip6_opt) + opt->ip6o_len )
+				{
+				reporter->Weird(SrcAddr(), DstAddr(), "bad_ipv6_dest_opt_len");
+				len = 0;
+				break;
+				}
+
+			if ( opt->ip6o_type == 201 ) // Home Address Option, Mobile IPv6 RFC 6275 section 6.3
+				{
+				if ( opt->ip6o_len == sizeof(struct in6_addr) )
+					{
+					if ( homeAddr )
+						reporter->Weird(SrcAddr(), DstAddr(), "multiple_home_addr_opts");
+					else
+						homeAddr = new IPAddr(*((const in6_addr*)(data + sizeof(struct ip6_opt))));
+					}
+				else
+					reporter->Weird(SrcAddr(), DstAddr(), "bad_home_addr_len");
+				}
+
+			data += sizeof(struct ip6_opt) + opt->ip6o_len;
+			len -= sizeof(struct ip6_opt) + opt->ip6o_len;
 			}
+			break;
 		}
 	}
-#endif
+	}
 
 VectorValPtr IPv6_Hdr_Chain::ToVal() const
 	{
@@ -695,11 +696,9 @@ VectorValPtr IPv6_Hdr_Chain::ToVal() const
 		case IPPROTO_ESP:
 			ext_hdr->Assign(6, std::move(v));
 			break;
-#ifdef ENABLE_MOBILE_IPV6
 		case IPPROTO_MOBILITY:
 			ext_hdr->Assign(7, std::move(v));
 			break;
-#endif
 		default:
 			reporter->InternalWarning("IPv6_Hdr_Chain bad header %d", type);
 			continue;
@@ -732,10 +731,8 @@ IPv6_Hdr_Chain* IPv6_Hdr_Chain::Copy(const ip6_hdr* new_hdr) const
 	IPv6_Hdr_Chain* rval = new IPv6_Hdr_Chain;
 	rval->length = length;
 
-#ifdef ENABLE_MOBILE_IPV6
 	if ( homeAddr )
 		rval->homeAddr = new IPAddr(*homeAddr);
-#endif
 
 	if ( finalDst )
 		rval->finalDst = new IPAddr(*finalDst);

src/IP.h

@@ -26,8 +26,6 @@ using VectorValPtr = IntrusivePtr<VectorVal>;
 
 namespace detail { class FragReassembler; }
 
-#ifdef ENABLE_MOBILE_IPV6
-
 #ifndef IPPROTO_MOBILITY
 #define IPPROTO_MOBILITY 135
 #endif
@@ -40,8 +38,6 @@ struct ip6_mobility {
 	uint16_t ip6mob_chksum;
 };
 
-#endif //ENABLE_MOBILE_IPV6
-
 /**
  * Base class for IPv6 header/extensions.
  */
@@ -66,9 +62,7 @@ public:
 		case IPPROTO_ROUTING:
 		case IPPROTO_FRAGMENT:
 		case IPPROTO_AH:
-#ifdef ENABLE_MOBILE_IPV6
 		case IPPROTO_MOBILITY:
-#endif
 			((ip6_ext*)data)->ip6e_nxt = next_type;
 			break;
 		case IPPROTO_ESP:
@@ -93,9 +87,7 @@ public:
 		case IPPROTO_ROUTING:
 		case IPPROTO_FRAGMENT:
 		case IPPROTO_AH:
-#ifdef ENABLE_MOBILE_IPV6
 		case IPPROTO_MOBILITY:
-#endif
 			return ((ip6_ext*)data)->ip6e_nxt;
 		case IPPROTO_ESP:
 		default:
@@ -114,9 +106,7 @@ public:
 		case IPPROTO_HOPOPTS:
 		case IPPROTO_DSTOPTS:
 		case IPPROTO_ROUTING:
-#ifdef ENABLE_MOBILE_IPV6
 		case IPPROTO_MOBILITY:
-#endif
 			return 8 + 8 * ((ip6_ext*)data)->ip6e_len;
 		case IPPROTO_FRAGMENT:
 			return 8;
@@ -262,13 +252,11 @@ protected:
 	 */
 	void ProcessRoutingHeader(const struct ip6_rthdr* r, uint16_t len);
 
-#ifdef ENABLE_MOBILE_IPV6
 	/**
 	 * Inspect a Destination Option header's options for things we need to
 	 * remember, such as the Home Address option from Mobile IPv6.
 	 */
 	void ProcessDstOpts(const struct ip6_dest* d, uint16_t len);
-#endif
 
 	std::vector<IPv6_Hdr*> chain;
 
@@ -277,12 +265,10 @@ protected:
 	 */
 	uint16_t length = 0;
 
-#ifdef ENABLE_MOBILE_IPV6
 	/**
 	 * Home Address of the packet's source as defined by Mobile IPv6 (RFC 6275).
 	 */
 	IPAddr* homeAddr = nullptr;
-#endif
 
 	/**
 	 * The final destination address in chain's first Routing header that has
@@ -394,7 +380,6 @@ public:
 		return ((const u_char*) ip6) + ip6_hdrs->TotalLength();
 		}
 
-#ifdef ENABLE_MOBILE_IPV6
 	/**
 	 * Returns a pointer to the mobility header of the IP packet, if present,
 	 * else a null pointer.
@@ -408,7 +393,6 @@ public:
 		else
 			return (const ip6_mobility*)(*ip6_hdrs)[ip6_hdrs->Size()-1]->Data();
 		}
-#endif
 
 	/**
 	 * Returns the length of the IP packet's payload (length of packet minus

src/IPAddr.cc

@@ -44,12 +44,26 @@ ConnKey::ConnKey(const IPAddr& src, const IPAddr& dst, uint16_t src_port,
 		}
 	}
 
-detail::ConnKey::ConnKey(const ConnTuple& id)
+ConnKey::ConnKey(const ConnTuple& id)
 	: ConnKey(id.src_addr, id.dst_addr, id.src_port, id.dst_port,
 	          id.proto, id.is_one_way)
 	{
 	}
 
+ConnKey& ConnKey::operator=(const ConnKey& rhs)
+	{
+	if ( this == &rhs )
+		return *this;
+
+	memcpy(&ip1, &rhs.ip1, sizeof(in6_addr));
+	memcpy(&ip2, &rhs.ip2, sizeof(in6_addr));
+	port1 = rhs.port1;
+	port2 = rhs.port2;
+	transport = rhs.transport;
+
+	return *this;
+	}
+
 } // namespace detail
 
 IPAddr::IPAddr(const String& s)

src/IPAddr.h

@@ -4,13 +4,13 @@
 
 #include <netinet/in.h>
 #include <arpa/inet.h>
-#include <string.h>
+#include <cstring>
 #include <string>
 #include <memory>
 
 #include "zeek/threading/SerialTypes.h"
 
-typedef in_addr in4_addr;
+using in4_addr = in_addr;
 
 namespace zeek {
 
@@ -29,12 +29,9 @@ struct ConnKey {
 	TransportProto transport;
 
 	ConnKey(const IPAddr& src, const IPAddr& dst, uint16_t src_port,
-	          uint16_t dst_port, TransportProto t, bool one_way);
+	        uint16_t dst_port, TransportProto t, bool one_way);
 	ConnKey(const ConnTuple& conn);
-	ConnKey(const ConnKey& rhs)
+	ConnKey(const ConnKey& rhs)	{ *this = rhs; }
-		{
-		*this = rhs;
-		}
 
 	bool operator<(const ConnKey& rhs) const { return memcmp(this, &rhs, sizeof(ConnKey)) < 0; }
 	bool operator<=(const ConnKey& rhs) const { return memcmp(this, &rhs, sizeof(ConnKey)) <= 0; }
@@ -43,13 +40,7 @@ struct ConnKey {
 	bool operator>=(const ConnKey& rhs) const { return memcmp(this, &rhs, sizeof(ConnKey)) >= 0; }
 	bool operator>(const ConnKey& rhs) const { return memcmp(this, &rhs, sizeof(ConnKey)) > 0; }
 
-	ConnKey& operator=(const ConnKey& rhs)
+	ConnKey& operator=(const ConnKey& rhs);
-		{
-		if ( this != &rhs )
-			memcpy(this, &rhs, sizeof(ConnKey));
-
-		return *this;
-		}
 };
 
 using ConnIDKey [[deprecated("Remove in v5.1. Use zeek::detail::ConnKey.")]] = ConnKey;
@@ -64,7 +55,7 @@ public:
 	/**
 	 * Address family.
 	 */
-	typedef IPFamily Family;
+	using Family = IPFamily;
 
 	/**
 	 * Byte order.
@@ -387,11 +378,13 @@ public:
 		return ! ( addr1 <= addr2 );
 		}
 
-	/** Converts the address into the type used internally by the
+	/**
-	  * inter-thread communication.
+	 * Converts the address into the type used internally by the
-	  */
+	 * inter-thread communication.
+	 */
 	void ConvertToThreadingValue(threading::Value::addr_t* v) const;
 
+	[[deprecated("Remove in v5.1. MemoryAllocation() is deprecated and will be removed. See GHI-572.")]]
 	unsigned int MemoryAllocation() const { return padded_sizeof(*this); }
 
 	/**
@@ -607,7 +600,8 @@ public:
 	 */
 	uint8_t LengthIPv6() const { return length; }
 
-	/** Returns true if the given address is part of the prefix.
+	/**
+	 * Returns true if the given address is part of the prefix.
 	 *
 	 * @param addr The address to test.
 	 */
@@ -643,15 +637,17 @@ public:
 	 */
 	std::unique_ptr<detail::HashKey> MakeHashKey() const;
 
-	/** Converts the prefix into the type used internally by the
+	/**
-	  * inter-thread communication.
+	 * Converts the prefix into the type used internally by the
-	  */
+	 * inter-thread communication.
+	 */
 	void ConvertToThreadingValue(threading::Value::subnet_t* v) const
 		{
 		v->length = length;
 		prefix.ConvertToThreadingValue(&v->prefix);
 		}
 
+	[[deprecated("Remove in v5.1. MemoryAllocation() is deprecated and will be removed. See GHI-572.")]]
 	unsigned int MemoryAllocation() const { return padded_sizeof(*this); }
 
 	/**

src/IntrusivePtr.h

@@ -137,12 +137,29 @@ public:
 		return std::exchange(ptr_, nullptr);
 		}
 
-	IntrusivePtr& operator=(IntrusivePtr other) noexcept
+	IntrusivePtr& operator=(const IntrusivePtr& other) noexcept
+		{
+		IntrusivePtr tmp{other};
+		swap(tmp);
+		return *this;
+		}
+
+	IntrusivePtr& operator=(IntrusivePtr&& other) noexcept
 		{
 		swap(other);
 		return *this;
 		}
 
+	IntrusivePtr& operator=(std::nullptr_t) noexcept
+		{
+		if ( ptr_ )
+			{
+			Unref(ptr_);
+			ptr_ = nullptr;
+			}
+		return *this;
+		}
+
 	pointer get() const noexcept
 		{
 		return ptr_;

src/List.h

@@ -158,6 +158,7 @@ public:
 		return max_entries;
 		}
 
+	[[deprecated("Remove in v5.1. MemoryAllocation() is deprecated and will be removed. See GHI-572.")]]
 	int MemoryAllocation() const
 		{ return padded_sizeof(*this) + util::pad_size(max_entries * sizeof(T)); }
 

src/NFA.cc

@@ -157,9 +157,12 @@ void NFA_State::Dump(FILE* f)
 
 unsigned int NFA_State::TotalMemoryAllocation() const
 	{
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
 	return padded_sizeof(*this)
 		+ xtions.MemoryAllocation() - padded_sizeof(xtions)
 		+ (epsclosure ? epsclosure->MemoryAllocation() : 0);
+#pragma GCC diagnostic pop
 	}
 
 NFA_Machine::NFA_Machine(NFA_State* first, NFA_State* final)

src/NFA.h

@@ -62,6 +62,7 @@ public:
 	void Dump(FILE* f);
 
 	// Recursivly count all the reachable states.
+	[[deprecated("Remove in v5.1. MemoryAllocation() is deprecated and will be removed. See GHI-572.")]]
 	unsigned int TotalMemoryAllocation() const;
 
 protected:
@@ -118,8 +119,14 @@ public:
 	void Describe(ODesc* d) const override;
 	void Dump(FILE* f);
 
+	[[deprecated("Remove in v5.1. MemoryAllocation() is deprecated and will be removed. See GHI-572.")]]
 	unsigned int MemoryAllocation() const
-		{ return padded_sizeof(*this) + first_state->TotalMemoryAllocation(); }
+		{
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
+		return padded_sizeof(*this) + first_state->TotalMemoryAllocation();
+#pragma GCC diagnostic pop
+		}
 
 protected:
 	NFA_State* first_state;

src/NetVar.cc

@@ -157,9 +157,6 @@ int dns_skip_all_auth;
 int dns_skip_all_addl;
 int dns_max_queries;
 
-double stp_delta;
-double stp_idle_min;
-
 double table_expire_interval;
 double table_expire_delay;
 int table_incremental_step;
@@ -312,11 +309,6 @@ void init_net_var()
 	dns_skip_all_addl = id::find_val("dns_skip_all_addl")->AsBool();
 	dns_max_queries = id::find_val("dns_max_queries")->AsCount();
 
-	stp_delta = 0.0;
-	if ( const auto& v = id::find_val("stp_delta") ) stp_delta = v->AsInterval();
-	stp_idle_min = 0.0;
-	if ( const auto& v = id::find_val("stp_idle_min") ) stp_delta = v->AsInterval();
-
 	orig_addr_anonymization = 0;
 	if ( const auto& id = id::find("orig_addr_anonymization") )
 		if ( const auto& v = id->GetVal() )

src/NetVar.h

@@ -59,8 +59,6 @@ extern int dns_skip_all_auth;
 extern int dns_skip_all_addl;
 extern int dns_max_queries;
 
-extern double stp_delta;
-extern double stp_idle_min;
 extern double table_expire_interval;
 extern double table_expire_delay;
 extern int table_incremental_step;

src/Options.cc

@@ -158,30 +158,52 @@ static void set_analysis_option(const char* opt, Options& opts)
 	if ( util::streq(opt, "help") )
 		{
 		fprintf(stderr, "--optimize options:\n");
+		fprintf(stderr, "    all	equivalent to \"inline\" and \"activate\"\n");
+		fprintf(stderr, "    add-C++	generate private C++ for any missing script bodies\n");
+		fprintf(stderr, "    compile-all	*if* compiling, compile all scripts, even inlined ones\n");
 		fprintf(stderr, "    dump-uds	dump use-defs to stdout; implies xform\n");
 		fprintf(stderr, "    dump-xform	dump transformed scripts to stdout; implies xform\n");
+		fprintf(stderr, "    gen-C++	generate C++ script bodies\n");
+		fprintf(stderr, "    gen-standalone-C++	generate \"standalone\" C++ script bodies\n");
 		fprintf(stderr, "    help	print this list\n");
 		fprintf(stderr, "    inline	inline function calls\n");
 		fprintf(stderr, "    optimize-AST	optimize the (transformed) AST; implies xform\n");
 		fprintf(stderr, "    recursive	report on recursive functions and exit\n");
+		fprintf(stderr, "    report-C++	report available C++ script bodies and exit\n");
+		fprintf(stderr, "    update-C++	generate reusable C++ for any missing script bodies\n");
+		fprintf(stderr, "    use-C++	use available C++ script bodies\n");
 		fprintf(stderr, "    xform	tranform scripts to \"reduced\" form\n");
 		exit(0);
 		}
 
 	auto& a_o = opts.analysis_options;
 
-	if ( util::streq(opt, "dump-uds") )
+	if ( util::streq(opt, "add-C++") )
+		a_o.add_CPP = true;
+	else if ( util::streq(opt, "compile-all") )
+		a_o.activate = a_o.compile_all = true;
+	else if ( util::streq(opt, "dump-uds") )
 		a_o.activate = a_o.dump_uds = true;
 	else if ( util::streq(opt, "dump-xform") )
 		a_o.activate = a_o.dump_xform = true;
+	else if ( util::streq(opt, "gen-C++") )
+		a_o.gen_CPP = true;
+	else if ( util::streq(opt, "gen-standalone-C++") )
+		a_o.gen_standalone_CPP = true;
 	else if ( util::streq(opt, "inline") )
 		a_o.inliner = true;
-	else if ( util::streq(opt, "recursive") )
-		a_o.inliner = a_o.report_recursive = true;
-	else if ( util::streq(opt, "xform") )
-		a_o.activate = true;
 	else if ( util::streq(opt, "optimize-AST") )
 		a_o.activate = a_o.optimize_AST = true;
+	else if ( util::streq(opt, "recursive") )
+		a_o.inliner = a_o.report_recursive = true;
+	else if ( util::streq(opt, "report-C++") )
+		a_o.report_CPP = true;
+	else if ( util::streq(opt, "update-C++") )
+		a_o.update_CPP = true;
+	else if ( util::streq(opt, "use-C++") )
+		a_o.use_CPP = true;
+	else if ( util::streq(opt, "xform") )
+		a_o.activate = true;
 
 	else
 		{
@@ -331,13 +353,9 @@ Options parse_cmdline(int argc, char** argv)
 	};
 
 	char opts[256];
-	util::safe_strncpy(opts, "B:e:f:G:H:I:i:j::n:O:o:p:r:s:T:t:U:w:X:CDFNPQSWabdhuv",
+	util::safe_strncpy(opts, "B:e:f:G:H:I:i:j::n:O:o:p:r:s:T:t:U:w:X:CDFMNPQSWabdhmuv",
 	                         sizeof(opts));
 
-#ifdef USE_PERFTOOLS_DEBUG
-	strncat(opts, "mM", 2);
-#endif
-
 	int op;
 	int long_optsind;
 	opterr = 0;
@@ -425,15 +443,19 @@ Options parse_cmdline(int argc, char** argv)
 		case 'w':
 			rval.pcap_output_file = optarg;
 			break;
+
+#ifdef DEBUG
 		case 'B':
 			rval.debug_log_streams = optarg;
 			break;
+#endif
+
 		case 'C':
 			rval.ignore_checksums = true;
 			break;
 		case 'D':
-		    rval.deterministic_mode = true;
+			rval.deterministic_mode = true;
-		    break;
+			break;
 		case 'E':
 			rval.pseudo_realtime = 1.0;
 			if ( optarg )

src/RE.cc

@@ -416,14 +416,20 @@ unsigned int Specific_RE_Matcher::MemoryAllocation() const
 	{
 	unsigned int size = 0;
 
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
 	for ( int i = 0; i < ccl_list.length(); ++i )
 		size += ccl_list[i]->MemoryAllocation();
+#pragma GCC diagnostic pop
 
 	size += util::pad_size(sizeof(CCL*) * ccl_dict.size());
 	for ( const auto& entry : ccl_dict )
 		{
 		size += padded_sizeof(std::string) + util::pad_size(sizeof(std::string::value_type) * entry.first.size());
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
 		size += entry.second->MemoryAllocation();
+#pragma GCC diagnostic pop
 		}
 
 	for ( const auto& entry : defs )
@@ -432,6 +438,8 @@ unsigned int Specific_RE_Matcher::MemoryAllocation() const
 		size += padded_sizeof(std::string) + util::pad_size(sizeof(std::string::value_type) * entry.second.size());
 		}
 
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
 	return size + padded_sizeof(*this)
 		+ (pattern_text ? util::pad_size(strlen(pattern_text) + 1) : 0)
 		+ ccl_list.MemoryAllocation() - padded_sizeof(ccl_list)
@@ -440,6 +448,7 @@ unsigned int Specific_RE_Matcher::MemoryAllocation() const
 		+ padded_sizeof(*any_ccl)
 		+ padded_sizeof(*accepted) // NOLINT(bugprone-sizeof-container)
 		+ accepted->size() * padded_sizeof(AcceptingSet::key_type);
+#pragma GCC diagnostic pop
 	}
 
 static RE_Matcher* matcher_merge(const RE_Matcher* re1, const RE_Matcher* re2,

src/RE.h

@@ -115,6 +115,7 @@ public:
 
 	void Dump(FILE* f);
 
+	[[deprecated("Remove in v5.1. MemoryAllocation() is deprecated and will be removed. See GHI-572.")]]
 	unsigned int MemoryAllocation() const;
 
 protected:
@@ -232,11 +233,15 @@ public:
 	// the main ("explicit") constructor was used.
 	const char* OrigText() const	{ return orig_text.c_str(); }
 
+	[[deprecated("Remove in v5.1. MemoryAllocation() is deprecated and will be removed. See GHI-572.")]]
 	unsigned int MemoryAllocation() const
 		{
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
 		return padded_sizeof(*this)
 			+ (re_anywhere ? re_anywhere->MemoryAllocation() : 0)
 			+ (re_exact ? re_exact->MemoryAllocation() : 0);
+#pragma GCC diagnostic pop
 		}
 
 protected:

src/Reassem.h

@@ -282,9 +282,11 @@ public:
 	void Describe(ODesc* d) const override;
 
 	// Sum over all data buffered in some reassembler.
+	[[deprecated("Remove in v5.1. MemoryAllocation() is deprecated and will be removed. See GHI-572.")]]
 	static uint64_t TotalMemoryAllocation()	{ return total_size; }
 
 	// Data buffered by type of reassembler.
+	[[deprecated("Remove in v5.1. MemoryAllocation() is deprecated and will be removed. See GHI-572.")]]
 	static uint64_t MemoryAllocation(ReassemblerType rtype);
 
 	void SetMaxOldBlocks(uint32_t count)	{ max_old_blocks = count; }

src/Reporter.cc

@@ -187,6 +187,21 @@ void Reporter::RuntimeError(const detail::Location* location, const char* fmt, .
 	throw InterpreterException();
 	}
 
+void Reporter::CPPRuntimeError(const char* fmt, ...)
+	{
+	++errors;
+	va_list ap;
+	va_start(ap, fmt);
+	FILE* out = EmitToStderr(errors_to_stderr) ? stderr : nullptr;
+	DoLog("runtime error in compiled code", reporter_error, out, nullptr, nullptr, true, true, "", fmt, ap);
+	va_end(ap);
+
+	if ( abort_on_scripting_errors )
+		abort();
+
+	throw InterpreterException();
+	}
+
 void Reporter::InternalError(const char* fmt, ...)
 	{
 	va_list ap;
@@ -365,6 +380,10 @@ bool Reporter::PermitFlowWeird(const char* name,
 
 bool Reporter::PermitExpiredConnWeird(const char* name, const RecordVal& conn_id)
 	{
+	if ( !conn_id.HasField("orig_h") || !conn_id.HasField("resp_h") ||
+	     !conn_id.HasField("orig_p") || !conn_id.HasField("resp_p") )
+		return false;
+
 	auto conn_tuple = std::make_tuple(conn_id.GetFieldAs<AddrVal>("orig_h"),
 	                                  conn_id.GetFieldAs<AddrVal>("resp_h"),
 	                                  conn_id.GetFieldAs<PortVal>("orig_p")->Port(),

src/Reporter.h

@@ -100,6 +100,10 @@ public:
 	// function will not return but raise an InterpreterException.
 	[[noreturn]] void RuntimeError(const detail::Location* location, const char* fmt, ...) __attribute__((format(printf, 3, 4)));
 
+	// Report a runtime error in executing a compiled script. This
+	// function will not return but raise an InterpreterException.
+	[[noreturn]] void CPPRuntimeError(const char* fmt, ...) __attribute__((format(printf, 2, 3)));
+
 	// Report a traffic weirdness, i.e., an unexpected protocol situation
 	// that may lead to incorrectly processing a connnection.
 	void Weird(const char* name, const char* addl = "", const char* source = "");	// Raises net_weird().

src/RuleCondition.cc

@@ -25,12 +25,12 @@ namespace zeek::detail {
 bool RuleConditionTCPState::DoMatch(Rule* rule, RuleEndpointState* state,
 					const u_char* data, int len)
 	{
-	analyzer::Analyzer* root = state->GetAnalyzer()->Conn()->GetRootAnalyzer();
+	auto* adapter = state->GetAnalyzer()->Conn()->GetSessionAdapter();
 
-	if ( ! root || ! root->IsAnalyzer("TCP") )
+	if ( ! adapter || ! adapter->IsAnalyzer("TCP") )
 		return false;
 
-	auto* ta = static_cast<analyzer::tcp::TCP_Analyzer*>(root);
+	auto* ta = static_cast<packet_analysis::TCP::TCPSessionAdapter*>(adapter);
 
 	if ( tcpstates & RULE_STATE_STATELESS )
 		return true;
@@ -57,9 +57,9 @@ void RuleConditionTCPState::PrintDebug()
 bool RuleConditionUDPState::DoMatch(Rule* rule, RuleEndpointState* state,
                                     const u_char* data, int len)
 	{
-	analyzer::Analyzer* root = state->GetAnalyzer()->Conn()->GetRootAnalyzer();
+	auto* adapter = state->GetAnalyzer()->Conn()->GetSessionAdapter();
 
-	if ( ! root || ! root->IsAnalyzer("UDP") )
+	if ( ! adapter || ! adapter->IsAnalyzer("UDP") )
 		return false;
 
 	if ( states & RULE_STATE_STATELESS )

src/RunState.cc

@@ -37,7 +37,6 @@ extern "C" {
 #include "zeek/plugin/Manager.h"
 #include "zeek/broker/Manager.h"
 #include "zeek/packet_analysis/Manager.h"
-#include "zeek/analyzer/protocol/stepping-stone/SteppingStone.h"
 
 extern "C" {
 extern int select(int, fd_set *, fd_set *, fd_set *, struct timeval *);
@@ -46,8 +45,6 @@ extern int select(int, fd_set *, fd_set *, fd_set *, struct timeval *);
 static double last_watchdog_proc_time = 0.0;	// value of above during last watchdog
 extern int signal_val;
 
-using namespace zeek::analyzer::stepping_stone;
-
 namespace zeek::run_state {
 namespace detail {
 
@@ -197,9 +194,6 @@ void init_run(const std::optional<std::string>& interface,
 
 	session_mgr = new session::Manager();
 
-	// Initialize the stepping stone manager. We intentionally throw away the result here.
-	SteppingStoneManager::Get();
-
 	if ( do_watchdog )
 		{
 		// Set up the watchdog to make sure we don't wedge.
@@ -414,7 +408,6 @@ void delete_run()
 	util::detail::set_processing_status("TERMINATING", "delete_run");
 
 	delete session_mgr;
-	delete SteppingStoneManager::Get();
 
 	for ( int i = 0; i < zeek::detail::NUM_ADDR_ANONYMIZATION_METHODS; ++i )
 		delete zeek::detail::ip_anonymizer[i];

src/Scope.cc

@@ -12,10 +12,8 @@
 
 namespace zeek::detail {
 
-using scope_list = PList<Scope>;
+static std::vector<ScopePtr> scopes;
-
+static ScopePtr top_scope;
-static scope_list scopes;
-static Scope* top_scope;
 
 Scope::Scope(IDPtr id,
              std::unique_ptr<std::vector<AttrPtr>> al)
@@ -117,9 +115,9 @@ const IDPtr& lookup_ID(const char* name, const char* curr_module,
 	bool need_export = check_export && (ID_module != GLOBAL_MODULE_NAME &&
 	                                    ID_module != curr_module);
 
-	for ( int i = scopes.length() - 1; i >= 0; --i )
+	for ( auto s_i = scopes.rbegin(); s_i != scopes.rend(); ++s_i )
 		{
-		const auto& id = scopes[i]->Find(fullname);
+		const auto& id = (*s_i)->Find(fullname);
 
 		if ( id )
 			{
@@ -172,16 +170,15 @@ IDPtr install_ID(const char* name, const char* module_name,
 	return id;
 	}
 
-void push_existing_scope(Scope* scope)
+void push_existing_scope(ScopePtr scope)
 	{
 	top_scope = scope;
 	scopes.push_back(scope);
 	}
 
-void push_scope(IDPtr id,
+void push_scope(IDPtr id, std::unique_ptr<std::vector<AttrPtr>> attrs)
-                std::unique_ptr<std::vector<AttrPtr>> attrs)
 	{
-	top_scope = new Scope(std::move(id), std::move(attrs));
+	top_scope = make_intrusive<Scope>(std::move(id), std::move(attrs));
 	scopes.push_back(top_scope);
 	}
 
@@ -191,19 +188,19 @@ ScopePtr pop_scope()
 		reporter->InternalError("scope underflow");
 	scopes.pop_back();
 
-	Scope* old_top = top_scope;
+	auto old_top = top_scope;
 
 	top_scope = scopes.empty() ? nullptr : scopes.back();
 
-	return {AdoptRef{}, old_top};
+	return old_top;
 	}
 
-Scope* current_scope()
+ScopePtr current_scope()
 	{
 	return top_scope;
 	}
 
-Scope* global_scope()
+ScopePtr global_scope()
 	{
 	return scopes.empty() ? 0 : scopes.front();
 	}

src/Scope.h

@@ -96,12 +96,12 @@ extern IDPtr install_ID(
 	bool is_global, bool is_export);
 
 extern void push_scope(IDPtr id, std::unique_ptr<std::vector<AttrPtr>> attrs);
-extern void push_existing_scope(Scope* scope);
+extern void push_existing_scope(ScopePtr scope);
 
 // Returns the one popped off.
 extern ScopePtr pop_scope();
-extern Scope* current_scope();
+extern ScopePtr current_scope();
-extern Scope* global_scope();
+extern ScopePtr global_scope();
 
 // Current module (identified by its name).
 extern std::string current_module;

src/Stats.cc

@@ -15,6 +15,7 @@
 #include "zeek/broker/Manager.h"
 #include "zeek/input.h"
 #include "zeek/Func.h"
+#include "zeek/packet_analysis/protocol/tcp/TCP.h"
 
 uint64_t zeek::detail::killed_by_inactivity = 0;
 uint64_t& killed_by_inactivity = zeek::detail::killed_by_inactivity;
@@ -126,7 +127,10 @@ void ProfileLogger::Log()
 		run_state::network_time, (utime + stime) - (first_utime + first_stime),
 		utime - first_utime, stime - first_stime, rtime - first_rtime));
 
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
 	int conn_mem_use = expensive ? session_mgr->SessionMemoryUsage() : 0;
+#pragma GCC diagnostic pop
 	double avg_conn_mem_use = 0;
 
 	if ( expensive && session_mgr->CurrentSessions() != 0 )
@@ -134,6 +138,8 @@ void ProfileLogger::Log()
 
 	// TODO: This previously output the number of connections, but now that we're storing sessions
 	// as well as connections, this might need to be renamed.
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
 	file->Write(util::fmt("%.06f Conns: total=%" PRIu64 " current=%" PRIu64 "/%" PRIi32 " mem=%" PRIi32 "K avg=%.1f table=%" PRIu32 "K connvals=%" PRIu32 "K\n",
 		run_state::network_time,
 		Connection::TotalConnections(),
@@ -144,6 +150,7 @@ void ProfileLogger::Log()
 		expensive ? session_mgr->MemoryAllocation() / 1024 : 0,
 		expensive ? session_mgr->SessionMemoryUsageVals() / 1024 : 0
 		));
+#pragma GCC diagnostic pop
 
 	session::Stats s;
 	session_mgr->GetStats(s);
@@ -155,7 +162,7 @@ void ProfileLogger::Log()
 		s.num_ICMP_conns, s.max_ICMP_conns
 		));
 
-	session_mgr->tcp_stats.PrintStats(file,
+	packet_analysis::TCP::TCPAnalyzer::GetStats().PrintStats(file,
 			util::fmt("%.06f TCP-States:", run_state::network_time));
 
 	// Alternatively, if you prefer more compact output...
@@ -177,8 +184,11 @@ void ProfileLogger::Log()
 	file->Write(util::fmt("%.06f Connections expired due to inactivity: %" PRIu64 "\n",
 	                      run_state::network_time, killed_by_inactivity));
 
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
 	file->Write(util::fmt("%.06f Total reassembler data: %" PRIu64 "K\n", run_state::network_time,
 	                      Reassembler::TotalMemoryAllocation() / 1024));
+#pragma GCC diagnostic pop
 
 	// Signature engine.
 	if ( expensive && rule_matcher )
@@ -273,7 +283,10 @@ void ProfileLogger::Log()
 				{
 				const auto& v = id->GetVal();
 
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
 				size = v->MemoryAllocation();
+#pragma GCC diagnostic pop
 				mem += size;
 
 				bool print = false;

src/Stmt.cc

@@ -34,6 +34,7 @@ const char* stmt_name(StmtTag t)
 		"<init>", "fallthrough", "while",
 		"catch-return",
 		"check-any-length",
+		"compiled-C++",
 		"null",
 	};
 
@@ -709,7 +710,7 @@ void SwitchStmt::Init()
 	t->Append(e->GetType());
 	comp_hash = new CompositeHash(std::move(t));
 
-	case_label_value_map.SetDeleteFunc(int_del_func);
+	case_label_hash_map.SetDeleteFunc(int_del_func);
 	}
 
 SwitchStmt::SwitchStmt(ExprPtr index, case_list* arg_cases)
@@ -854,12 +855,13 @@ bool SwitchStmt::AddCaseLabelValueMapping(const Val* v, int idx)
 		                        type_name(e->GetType()->Tag()));
 		}
 
-	int* label_idx = case_label_value_map.Lookup(hk.get());
+	int* label_idx = case_label_hash_map.Lookup(hk.get());
 
 	if ( label_idx )
 		return false;
 
-	case_label_value_map.Insert(hk.get(), new int(idx));
+	case_label_value_map[v] = idx;
+	case_label_hash_map.Insert(hk.get(), new int(idx));
 	return true;
 	}
 
@@ -883,7 +885,7 @@ std::pair<int, ID*> SwitchStmt::FindCaseLabelMatch(const Val* v) const
 	ID* label_id = nullptr;
 
 	// Find matching expression cases.
-	if ( case_label_value_map.Length() )
+	if ( case_label_hash_map.Length() )
 		{
 		auto hk = comp_hash->MakeHashKey(*v, true);
 
@@ -896,7 +898,7 @@ std::pair<int, ID*> SwitchStmt::FindCaseLabelMatch(const Val* v) const
 			return std::make_pair(-1, nullptr);
 			}
 
-		if ( auto i = case_label_value_map.Lookup(hk.get()) )
+		if ( auto i = case_label_hash_map.Lookup(hk.get()) )
 			label_idx = *i;
 		}
 
@@ -1108,7 +1110,10 @@ WhileStmt::~WhileStmt() = default;
 
 bool WhileStmt::IsPure() const
 	{
-	return loop_condition->IsPure() && body->IsPure();
+	if ( loop_condition->IsPure() && body->IsPure() )
+		return ! loop_cond_pred_stmt || loop_cond_pred_stmt->IsPure();
+	else
+		return false;
 	}
 
 void WhileStmt::StmtDescribe(ODesc* d) const
@@ -1118,6 +1123,13 @@ void WhileStmt::StmtDescribe(ODesc* d) const
 	if ( d->IsReadable() )
 		d->Add("(");
 
+	if ( loop_cond_pred_stmt )
+		{
+		d->Add(" {");
+		loop_cond_pred_stmt->Describe(d);
+		d->Add("} ");
+		}
+
 	loop_condition->Describe(d);
 
 	if ( d->IsReadable() )
@@ -1135,6 +1147,12 @@ TraversalCode WhileStmt::Traverse(TraversalCallback* cb) const
 	TraversalCode tc = cb->PreStmt(this);
 	HANDLE_TC_STMT_PRE(tc);
 
+	if ( loop_cond_pred_stmt )
+		{
+		tc = loop_cond_pred_stmt->Traverse(cb);
+		HANDLE_TC_STMT_PRE(tc);
+		}
+
 	tc = loop_condition->Traverse(cb);
 	HANDLE_TC_STMT_PRE(tc);
 
@@ -1513,7 +1531,7 @@ TraversalCode FallthroughStmt::Traverse(TraversalCallback* cb) const
 ReturnStmt::ReturnStmt(ExprPtr arg_e)
 	: ExprStmt(STMT_RETURN, std::move(arg_e))
 	{
-	Scope* s = current_scope();
+	auto s = current_scope();
 
 	if ( ! s || ! s->GetID() )
 		{
@@ -1547,7 +1565,7 @@ ReturnStmt::ReturnStmt(ExprPtr arg_e)
 
 	else
 		{
-		auto promoted_e = check_and_promote_expr(e.get(), yt.get());
+		auto promoted_e = check_and_promote_expr(e, yt);
 
 		if ( promoted_e )
 			e = std::move(promoted_e);

src/Stmt.h

@@ -16,9 +16,12 @@ class CompositeHash;
 class NameExpr;
 using NameExprPtr = IntrusivePtr<zeek::detail::NameExpr>;
 
+class ZAMCompiler;	// for "friend" declarations
+
 class ExprListStmt : public Stmt {
 public:
 	const ListExpr* ExprList() const	{ return l.get(); }
+	const ListExprPtr& ExprListPtr() const	{ return l; }
 
 	TraversalCode Traverse(TraversalCallback* cb) const override;
 
@@ -180,15 +183,24 @@ public:
 	bool NoFlowAfter(bool ignore_break) const override;
 
 protected:
+	friend class ZAMCompiler;
+
+	int DefaultCaseIndex() const	{ return default_case_idx; }
+	const auto& ValueMap() const	{ return case_label_value_map; }
+	const std::vector<std::pair<ID*, int>>* TypeMap() const
+		{ return &case_label_type_list; }
+	const CompositeHash* CompHash() const   { return comp_hash; }
+
 	ValPtr DoExec(Frame* f, Val* v, StmtFlowType& flow) override;
 	bool IsPure() const override;
 
 	// Initialize composite hash and case label map.
 	void Init();
 
-	// Adds an entry in case_label_value_map for the given value to associate it
+	// Adds entries in case_label_value_map and case_label_hash_map
-	// with the given index in the cases list.  If the entry already exists,
+	// for the given value to associate it with the given index in
-	// returns false, else returns true.
+	// the cases list.  If the entry already exists, returns false,
+	// else returns true.
 	bool AddCaseLabelValueMapping(const Val* v, int idx);
 
 	// Adds an entry in case_label_type_map for the given type (w/ ID) to
@@ -205,7 +217,8 @@ protected:
 	case_list* cases;
 	int default_case_idx;
 	CompositeHash* comp_hash;
-	PDict<int> case_label_value_map;
+	std::unordered_map<const Val*, int> case_label_value_map;
+	PDict<int> case_label_hash_map;
 	std::vector<std::pair<ID*, int>> case_label_type_list;
 };
 
@@ -523,6 +536,7 @@ public:
 	const Stmt* Body() const	{ return s1.get(); }
 	const Expr* TimeoutExpr() const	{ return timeout.get(); }
 	const Stmt* TimeoutBody() const	{ return s2.get(); }
+	bool IsReturn() const		{ return is_return; }
 
 	void StmtDescribe(ODesc* d) const override;
 
@@ -597,6 +611,8 @@ class CheckAnyLenStmt : public ExprStmt {
 public:
 	explicit CheckAnyLenStmt(ExprPtr e, int expected_len);
 
+	int ExpectedLen() const		{ return expected_len; }
+
 	ValPtr Exec(Frame* f, StmtFlowType& flow) override;
 
 	StmtPtr Duplicate() override;

src/StmtEnums.h

@@ -20,6 +20,7 @@ enum StmtTag {
 	STMT_WHILE,
 	STMT_CATCH_RETURN,	// for reduced InlineExpr's
 	STMT_CHECK_ANY_LEN,	// internal reduced statement
+	STMT_CPP,	// compiled C++
 	STMT_NULL
 #define NUM_STMTS (int(STMT_NULL) + 1)
 };

src/Traverse.h

@@ -2,6 +2,7 @@
 
 #pragma once
 
+#include "zeek/Scope.h"
 #include "zeek/TraverseTypes.h"
 
 namespace zeek {
@@ -10,7 +11,6 @@ class Func;
 
 namespace detail {
 
-class Scope;
 class Stmt;
 class Expr;
 class ID;
@@ -38,7 +38,7 @@ public:
 	virtual TraversalCode PreDecl(const ID*) { return TC_CONTINUE; }
 	virtual TraversalCode PostDecl(const ID*) { return TC_CONTINUE; }
 
-	Scope* current_scope;
+	ScopePtr current_scope;
 };
 
 TraversalCode traverse_all(TraversalCallback* cb);

src/Trigger.cc

@@ -98,15 +98,45 @@ protected:
 	double time;
 };
 
-Trigger::Trigger(Expr* arg_cond, Stmt* arg_body,
+Trigger::Trigger(const Expr* cond, Stmt* body, Stmt* timeout_stmts,
-                 Stmt* arg_timeout_stmts,
+                 Expr* timeout_expr, Frame* frame,
-                 Expr* arg_timeout, Frame* arg_frame,
+                 bool is_return, const Location* location)
-                 bool arg_is_return, const Location* arg_location)
+	{
+	timeout_value = -1;
+
+	if ( timeout_expr )
+		{
+		ValPtr timeout_val;
+
+		try
+			{
+			timeout_val = timeout_expr->Eval(frame);
+			}
+		catch ( InterpreterException& )
+			{ /* Already reported */ }
+
+		if ( timeout_val )
+			timeout_value = timeout_val->AsInterval();
+		}
+
+	Init(cond, body, timeout_stmts, frame, is_return, location);
+	}
+
+Trigger::Trigger(const Expr* cond, Stmt* body, Stmt* timeout_stmts,
+                 double timeout, Frame* frame,
+                 bool is_return, const Location* location)
+	{
+	timeout_value = timeout;
+	Init(cond, body, timeout_stmts, frame, is_return, location);
+	}
+
+void Trigger::Init(const Expr* arg_cond, Stmt* arg_body, Stmt* arg_timeout_stmts,
+                   Frame* arg_frame, bool arg_is_return,
+                   const Location* arg_location)
 	{
 	cond = arg_cond;
 	body = arg_body;
 	timeout_stmts = arg_timeout_stmts;
-	timeout = arg_timeout;
 	frame = arg_frame->Clone();
 	timer = nullptr;
 	delayed = false;
@@ -114,7 +144,6 @@ Trigger::Trigger(Expr* arg_cond, Stmt* arg_body,
 	attached = nullptr;
 	is_return = arg_is_return;
 	location = arg_location;
-	timeout_value = -1;
 
 	DBG_LOG(DBG_NOTIFIERS, "%s: instantiating", Name());
 
@@ -132,23 +161,6 @@ Trigger::Trigger(Expr* arg_cond, Stmt* arg_body,
 		arg_frame->SetDelayed();
 		}
 
-	ValPtr timeout_val;
-
-	if ( arg_timeout )
-		{
-		try
-			{
-			timeout_val = arg_timeout->Eval(arg_frame);
-			}
-		catch ( InterpreterException& )
-			{ /* Already reported */ }
-		}
-
-	if ( timeout_val )
-		{
-		timeout_value = timeout_val->AsInterval();
-		}
-
 	// Make sure we don't get deleted if somebody calls a method like
 	// Timeout() while evaluating the trigger.
 	Ref(this);
@@ -198,7 +210,7 @@ Trigger::~Trigger()
 	// point.
 	}
 
-void Trigger::Init(std::vector<ValPtr> index_expr_results)
+void Trigger::ReInit(std::vector<ValPtr> index_expr_results)
 	{
 	assert(! disabled);
 	UnregisterAll();
@@ -276,7 +288,7 @@ bool Trigger::Eval()
 		// Not true. Perhaps next time...
 		DBG_LOG(DBG_NOTIFIERS, "%s: trigger condition is false", Name());
 		Unref(f);
-		Init(std::move(index_expr_results));
+		ReInit(std::move(index_expr_results));
 		return false;
 		}
 

src/Trigger.h

@@ -38,7 +38,9 @@ public:
 	// instantiation.  Note that if the condition is already true, the
 	// statements are executed immediately and the object is deleted
 	// right away.
-	Trigger(Expr* cond, Stmt* body, Stmt* timeout_stmts, Expr* timeout,
+	Trigger(const Expr* cond, Stmt* body, Stmt* timeout_stmts, Expr* timeout,
+		Frame* f, bool is_return, const Location* loc);
+	Trigger(const Expr* cond, Stmt* body, Stmt* timeout_stmts, double timeout,
 		Frame* f, bool is_return, const Location* loc);
 	~Trigger() override;
 
@@ -95,12 +97,16 @@ private:
 	friend class TriggerTraversalCallback;
 	friend class TriggerTimer;
 
-	void Init(std::vector<IntrusivePtr<Val>> index_expr_results);
+	void Init(const Expr* cond, Stmt* body, Stmt* timeout_stmts, Frame* frame,
+	          bool is_return, const Location* location);
+
+	void ReInit(std::vector<IntrusivePtr<Val>> index_expr_results);
+
 	void Register(ID* id);
 	void Register(Val* val);
 	void UnregisterAll();
 
-	Expr* cond;
+	const Expr* cond;
 	Stmt* body;
 	Stmt* timeout_stmts;
 	Expr* timeout;

src/Type.cc

@@ -239,7 +239,7 @@ int Type::MatchesIndex(detail::ListExpr* const index) const
 		if ( index->Exprs().length() != 1 && index->Exprs().length() != 2 )
 			return DOES_NOT_MATCH_INDEX;
 
-		if ( check_and_promote_exprs_to_type(index, zeek::base_type(TYPE_INT).get()) )
+		if ( check_and_promote_exprs_to_type(index, zeek::base_type(TYPE_INT)) )
 			return MATCHES_INDEX_SCALAR;
 		}
 
@@ -335,6 +335,8 @@ unsigned int TypeList::MemoryAllocation() const
 	{
 	unsigned int size = 0;
 
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
 	for ( const auto& t : types )
 		size += t->MemoryAllocation();
 
@@ -343,6 +345,7 @@ unsigned int TypeList::MemoryAllocation() const
 	return Type::MemoryAllocation()
 		+ padded_sizeof(*this) - padded_sizeof(Type)
 		+ size;
+#pragma GCC diagnostic pop
 	}
 
 int IndexType::MatchesIndex(detail::ListExpr* const index) const
@@ -892,6 +895,39 @@ void TypeDecl::DescribeReST(ODesc* d, bool roles_only) const
 		}
 	}
 
+
+// The following tracks how to initialize a given field, for fast execution
+// of Create().
+
+class FieldInit {
+public:
+	// The type of initialization for the field.
+	enum {
+		R_INIT_NONE,	// skip this entry
+
+		R_INIT_DIRECT,	// look in direct_init for raw value
+		R_INIT_DIRECT_MANAGED,	// same, but managed type
+
+		R_INIT_DEF,	// look in def_expr for expression
+
+		R_INIT_RECORD,	// field requires a new record
+		R_INIT_TABLE,	// field requires a new table/set
+		R_INIT_VECTOR,	// field requires a new vector
+	} init_type;
+
+	// For R_INIT_DIRECT/R_INIT_DIRECT_MANAGED:
+	ZVal direct_init;
+
+	detail::ExprPtr def_expr;
+	TypePtr def_type;
+	bool def_coerce = false;	// whether coercion's required
+
+	RecordTypePtr r_type;	// for R_INIT_RECORD
+	TableTypePtr t_type;	// for R_INIT_TABLE
+	detail::AttributesPtr attrs;	// attributes for R_INIT_TABLE
+	VectorTypePtr v_type;	// for R_INIT_VECTOR
+};
+
 RecordType::RecordType(type_decl_list* arg_types) : Type(TYPE_RECORD)
 	{
 	types = arg_types;
@@ -928,13 +964,81 @@ RecordType::~RecordType()
 
 		delete types;
 		}
+
+	for ( auto fi : field_inits )
+		delete fi;
 	}
 
 void RecordType::AddField(unsigned int field, const TypeDecl* td)
 	{
+	ASSERT(field == field_inits.size());
 	ASSERT(field == managed_fields.size());
 
 	managed_fields.push_back(ZVal::IsManagedType(td->type));
+
+	auto init = new FieldInit();
+	init->init_type = FieldInit::R_INIT_NONE;
+
+	init->attrs = td->attrs;
+	auto a = init->attrs;
+
+	auto type = td->type;
+
+	auto def_attr = a ? a->Find(detail::ATTR_DEFAULT) : nullptr;
+	auto def_expr = def_attr ? def_attr->GetExpr() : nullptr;
+
+	if ( def_expr )
+		{
+		if ( type->Tag() == TYPE_RECORD &&
+		     def_expr->GetType()->Tag() == TYPE_RECORD &&
+		     ! same_type(def_expr->GetType(), type) )
+			init->def_coerce = true;
+
+		if ( def_expr->Tag() == detail::EXPR_CONST )
+			{
+			auto v = def_expr->Eval(nullptr);
+
+			if ( ZVal::IsManagedType(type) )
+				init->init_type =
+					FieldInit::R_INIT_DIRECT_MANAGED;
+			else
+				init->init_type = FieldInit::R_INIT_DIRECT;
+
+			init->direct_init = ZVal(v, type);
+			}
+
+		else
+			{
+			init->init_type = FieldInit::R_INIT_DEF;
+			init->def_expr = def_expr;
+			init->def_type = def_expr->GetType();
+			}
+		}
+
+	else if ( ! (a && a->Find(detail::ATTR_OPTIONAL)) )
+		{
+		TypeTag tag = type->Tag();
+
+		if ( tag == TYPE_RECORD )
+			{
+			init->init_type = FieldInit::R_INIT_RECORD;
+			init->r_type = cast_intrusive<RecordType>(type);
+			}
+
+		else if ( tag == TYPE_TABLE )
+			{
+			init->init_type = FieldInit::R_INIT_TABLE;
+			init->t_type = cast_intrusive<TableType>(type);
+			}
+
+		else if ( tag == TYPE_VECTOR )
+			{
+			init->init_type = FieldInit::R_INIT_VECTOR;
+			init->v_type = cast_intrusive<VectorType>(type);
+			}
+		}
+
+	field_inits.push_back(init);
 	}
 
 bool RecordType::HasField(const char* field) const
@@ -1129,6 +1233,67 @@ void RecordType::AddFieldsDirectly(const type_decl_list& others,
 	num_fields = types->length();
 	}
 
+void RecordType::Create(std::vector<std::optional<ZVal>>& r) const
+	{
+	int n = NumFields();
+
+	for ( int i = 0; i < n; ++i )
+		{
+		auto& init = field_inits[i];
+
+		ZVal r_i;
+
+		switch ( init->init_type ) {
+		case FieldInit::R_INIT_NONE:
+			r.push_back(std::nullopt);
+			continue;
+
+		case FieldInit::R_INIT_DIRECT:
+			r_i = init->direct_init;
+			break;
+
+		case FieldInit::R_INIT_DIRECT_MANAGED:
+			r_i = init->direct_init;
+			zeek::Ref(r_i.ManagedVal());
+			break;
+
+		case FieldInit::R_INIT_DEF:
+			{
+			auto v = init->def_expr->Eval(nullptr);
+			if ( v )
+				{
+				const auto& t = init->def_type;
+
+				if ( init->def_coerce )
+					{
+					auto rt = cast_intrusive<RecordType>(t);
+					v = v->AsRecordVal()->CoerceTo(rt);
+					}
+
+				r_i = ZVal(v, t);
+				}
+			else
+				reporter->Error("failed &default in record creation");
+			}
+			break;
+
+		case FieldInit::R_INIT_RECORD:
+			r_i = ZVal(new RecordVal(init->r_type));
+			break;
+
+		case FieldInit::R_INIT_TABLE:
+			r_i = ZVal(new TableVal(init->t_type, init->attrs));
+			break;
+
+		case FieldInit::R_INIT_VECTOR:
+			r_i = ZVal(new VectorVal(init->v_type));
+			break;
+		}
+
+		r.push_back(r_i);
+		}
+	}
+
 void RecordType::DescribeFields(ODesc* d) const
 	{
 	if ( d->IsReadable() )

src/Type.h

@@ -18,6 +18,7 @@
 namespace zeek {
 
 class Val;
+union ZVal;
 class EnumVal;
 class TableVal;
 using ValPtr = IntrusivePtr<Val>;
@@ -33,7 +34,7 @@ using ListExprPtr = IntrusivePtr<ListExpr>;
 
 } // namespace detail
 
-// BRO types.
+// Zeek types.
 enum TypeTag {
 	TYPE_VOID,      // 0
 	TYPE_BOOL,      // 1
@@ -171,7 +172,7 @@ public:
 
 	explicit Type(TypeTag tag, bool base_type = false);
 
-	// Performs a shallow clone operation of the Bro type.
+	// Performs a shallow clone operation of the Zeek type.
 	// This especially means that especially for tables the types
 	// are not recursively cloned; altering one type will in this case
 	// alter one of them.
@@ -249,6 +250,7 @@ public:
 	void Describe(ODesc* d) const override;
 	virtual void DescribeReST(ODesc* d, bool roles_only = false) const;
 
+	[[deprecated("Remove in v5.1. MemoryAllocation() is deprecated and will be removed. See GHI-572.")]]
 	virtual unsigned MemoryAllocation() const;
 
 	void SetName(const std::string& arg_name) { name = arg_name; }
@@ -346,6 +348,7 @@ public:
 
 	void Describe(ODesc* d) const override;
 
+	[[deprecated("Remove in v5.1. MemoryAllocation() is deprecated and will be removed. See GHI-572.")]]
 	unsigned int MemoryAllocation() const override;
 
 protected:
@@ -559,6 +562,11 @@ public:
 
 using type_decl_list = PList<TypeDecl>;
 
+// The following tracks how to initialize a given field.  We don't define
+// it here because it requires pulling in a bunch of low-level headers that
+// would be nice to avoid.
+class FieldInit;
+
 class RecordType final : public Type {
 public:
 	explicit RecordType(type_decl_list* types);
@@ -636,6 +644,13 @@ public:
 	void AddFieldsDirectly(const type_decl_list& types,
 				bool add_log_attr = false);
 
+	/**
+	 *
+	 * Populates a new instance of the record with its initial values.
+	 * @param r  The record's underlying value vector.
+	 */
+	void Create(std::vector<std::optional<ZVal>>& r) const;
+
 	void Describe(ODesc* d) const override;
 	void DescribeReST(ODesc* d, bool roles_only = false) const override;
 	void DescribeFields(ODesc* d) const;
@@ -660,6 +675,10 @@ protected:
 
 	void AddField(unsigned int field, const TypeDecl* td);
 
+	// Maps each field to how to initialize it.  Uses pointers due to
+	// keeping the FieldInit definition private to Type.cc (see above).
+	std::vector<FieldInit*> field_inits;
+
 	// If we were willing to bound the size of records, then we could
 	// use std::bitset here instead.
 	std::vector<bool> managed_fields;
@@ -747,11 +766,14 @@ public:
 
 	const EnumValPtr& GetEnumVal(bro_int_t i);
 
+	// Only for use by C++-generated code.  Non-protected because we
+	// don't know in advance the names of the functions that will
+	// access it.
+	void AddNameInternal(const std::string& full_name, bro_int_t val);
+
 protected:
 	void AddNameInternal(const std::string& module_name,
-			const char* name, bro_int_t val, bool is_export);
+	                     const char* name, bro_int_t val, bool is_export);
-
-	void AddNameInternal(const std::string& full_name, bro_int_t val);
 
 	void CheckAndAddName(const std::string& module_name,
 	                     const char* name, bro_int_t val, bool is_export,

src/Val.cc

@@ -84,6 +84,7 @@ CONVERTERS(TYPE_STRING, StringVal*, Val::AsStringVal)
 CONVERTERS(TYPE_VECTOR, VectorVal*, Val::AsVectorVal)
 CONVERTERS(TYPE_ENUM, EnumVal*, Val::AsEnumVal)
 CONVERTERS(TYPE_OPAQUE, OpaqueVal*, Val::AsOpaqueVal)
+CONVERTERS(TYPE_TYPE, TypeVal*, Val::AsTypeVal)
 
 ValPtr Val::CloneState::NewClone(Val* src, ValPtr dst)
 	{
@@ -230,8 +231,6 @@ ValPtr Val::SizeVal() const
 	{
 	switch ( type->InternalType() ) {
 	case TYPE_INTERNAL_INT:
-		// Return abs value. However abs() only works on ints and llabs
-		// doesn't work on Mac OS X 10.5. So we do it by hand
 		if ( AsInt() < 0 )
 			return val_mgr->Count(-AsInt());
 		else
@@ -817,7 +816,10 @@ AddrVal::~AddrVal()
 
 unsigned int AddrVal::MemoryAllocation() const
 	{
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
 	return padded_sizeof(*this) + addr_val->MemoryAllocation();
+#pragma GCC diagnostic pop
 	}
 
 ValPtr AddrVal::SizeVal() const
@@ -882,7 +884,10 @@ int SubNetVal::Width() const
 
 unsigned int SubNetVal::MemoryAllocation() const
 	{
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
 	return padded_sizeof(*this) + subnet_val->MemoryAllocation();
+#pragma GCC diagnostic pop
 	}
 
 ValPtr SubNetVal::SizeVal() const
@@ -1010,7 +1015,10 @@ void StringVal::ValDescribe(ODesc* d) const
 
 unsigned int StringVal::MemoryAllocation() const
 	{
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
 	return padded_sizeof(*this) + string_val->MemoryAllocation();
+#pragma GCC diagnostic pop
 	}
 
 StringValPtr StringVal::Replace(
@@ -1220,7 +1228,10 @@ void PatternVal::ValDescribe(ODesc* d) const
 
 unsigned int PatternVal::MemoryAllocation() const
 	{
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
 	return padded_sizeof(*this) + re_val->MemoryAllocation();
+#pragma GCC diagnostic pop
 	}
 
 ValPtr PatternVal::DoClone(CloneState* state)
@@ -1333,12 +1344,15 @@ ValPtr ListVal::DoClone(CloneState* state)
 
 unsigned int ListVal::MemoryAllocation() const
 	{
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
 	unsigned int size = 0;
 	for ( const auto& val : vals )
 		size += val->MemoryAllocation();
 
 	size += util::pad_size(vals.capacity() * sizeof(decltype(vals)::value_type));
 	return size + padded_sizeof(*this) + type->MemoryAllocation();
+#pragma GCC diagnostic pop
 	}
 
 TableEntryVal* TableEntryVal::Clone(Val::CloneState* state)
@@ -2771,6 +2785,8 @@ unsigned int TableVal::MemoryAllocation() const
 	{
 	unsigned int size = 0;
 
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
 	for ( const auto& ve : *table_val )
 		{
 		auto* tv = ve.GetValue<TableEntryVal*>();
@@ -2781,6 +2797,7 @@ unsigned int TableVal::MemoryAllocation() const
 
 	return size + padded_sizeof(*this) + table_val->MemoryAllocation()
 		+ table_hash->MemoryAllocation();
+#pragma GCC diagnostic pop
 	}
 
 std::unique_ptr<detail::HashKey> TableVal::MakeHashKey(const Val& index) const
@@ -2858,62 +2875,18 @@ RecordVal::RecordVal(RecordTypePtr t, bool init_fields)
 	if ( run_state::is_parsing )
 		parse_time_records[rt.get()].emplace_back(NewRef{}, this);
 
-	if ( ! init_fields )
+	if ( init_fields )
-		return;
-
-	// Initialize to default values from RecordType (which are nil
-	// by default).
-	for ( int i = 0; i < n; ++i )
 		{
-		detail::Attributes* a = rt->FieldDecl(i)->attrs.get();
+		try
-		detail::Attr* def_attr = a ? a->Find(detail::ATTR_DEFAULT).get() : nullptr;
-		ValPtr def;
-
-		if ( def_attr )
-			try
-				{
-				def = def_attr->GetExpr()->Eval(nullptr);
-				}
-			catch ( InterpreterException& )
-				{
-				if ( run_state::is_parsing )
-					parse_time_records[rt.get()].pop_back();
-
-				delete record_val;
-				throw;
-				}
-
-		const auto& type = rt->FieldDecl(i)->type;
-
-		if ( def && type->Tag() == TYPE_RECORD &&
-		     def->GetType()->Tag() == TYPE_RECORD &&
-		     ! same_type(def->GetType(), type) )
 			{
-			auto tmp = def->AsRecordVal()->CoerceTo(cast_intrusive<RecordType>(type));
+			rt->Create(*record_val);
-
-			if ( tmp )
-				def = std::move(tmp);
 			}
-
+		catch ( InterpreterException& e )
-		if ( ! def && ! (a && a->Find(detail::ATTR_OPTIONAL)) )
 			{
-			TypeTag tag = type->Tag();
+			if ( run_state::is_parsing )
-
+				parse_time_records[rt.get()].pop_back();
-			if ( tag == TYPE_RECORD )
+			throw;
-				def = make_intrusive<RecordVal>(cast_intrusive<RecordType>(type));
-
-			else if ( tag == TYPE_TABLE )
-				def = make_intrusive<TableVal>(IntrusivePtr{NewRef{}, type->AsTableType()},
-				                                     IntrusivePtr{NewRef{}, a});
-
-			else if ( tag == TYPE_VECTOR )
-				def = make_intrusive<VectorVal>(cast_intrusive<VectorType>(type));
 			}
-
-		if ( def )
-			record_val->emplace_back(ZVal(def, def->GetType()));
-		else
-			record_val->emplace_back(std::nullopt);
 		}
 	}
 
@@ -2987,7 +2960,8 @@ void RecordVal::ResizeParseTimeRecords(RecordType* revised_rt)
 		if ( required_length > current_length )
 			{
 			for ( auto i = current_length; i < required_length; ++i )
-				rv->AppendField(revised_rt->FieldDefault(i));
+				rv->AppendField(revised_rt->FieldDefault(i),
+				                revised_rt->GetFieldType(i));
 			}
 		}
 	}
@@ -3165,7 +3139,7 @@ ValPtr RecordVal::DoClone(CloneState* state)
 	// record. As we cannot guarantee that it will ber zeroed out at the
 	// approproate time (as it seems to be guaranteed for the original record)
 	// we don't touch it.
-	auto rv = make_intrusive<RecordVal>(GetType<RecordType>(), false);
+	auto rv = make_intrusive<RecordVal>(rt, false);
 	rv->origin = nullptr;
 	state->NewClone(this, rv);
 
@@ -3174,7 +3148,7 @@ ValPtr RecordVal::DoClone(CloneState* state)
 		{
 		auto f_i = GetField(i);
 		auto v = f_i ? f_i->Clone(state) : nullptr;
-  		rv->AppendField(std::move(v));
+		rv->AppendField(std::move(v), rt->GetFieldType(i));
 		}
 
 	return rv;
@@ -3189,7 +3163,10 @@ unsigned int RecordVal::MemoryAllocation() const
 		{
 		auto f_i = GetField(i);
 		if ( f_i )
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
 			size += f_i->MemoryAllocation();
+#pragma GCC diagnostic pop
 		}
 
 	size += util::pad_size(record_val->capacity() * sizeof(ZVal));
@@ -3230,9 +3207,15 @@ ValPtr TypeVal::DoClone(CloneState* state)
 	return {NewRef{}, this};
 	}
 
-VectorVal::VectorVal(VectorTypePtr t) : Val(t)
+VectorVal::VectorVal(VectorTypePtr t) :
+VectorVal(t, new vector<std::optional<ZVal>>())
 	{
-	vector_val = new vector<std::optional<ZVal>>();
+	}
+
+VectorVal::VectorVal(VectorTypePtr t, std::vector<std::optional<ZVal>>* vals)
+: Val(t)
+	{
+	vector_val = vals;
 	yield_type = t->Yield();
 
 	auto y_tag = yield_type->Tag();
@@ -3660,6 +3643,50 @@ VectorValPtr VectorVal::Order(Func* cmp_func)
 	return result_v;
 	}
 
+bool VectorVal::Concretize(const TypePtr& t)
+	{
+	if ( ! any_yield )
+		// Could do a same_type() call here, but really this case
+		// shouldn't happen in any case.
+		return yield_type->Tag() == t->Tag();
+
+	if ( ! vector_val )
+		// Trivially concretized.
+		return true;
+
+	auto n = vector_val->size();
+	for ( auto i = 0U; i < n; ++i )
+		{
+		auto& v = (*vector_val)[i];
+		if ( ! v )
+			// Vector hole does not require concretization.
+			continue;
+
+		auto& vt_i = yield_types ? (*yield_types)[i] : yield_type;
+		if ( vt_i->Tag() == TYPE_ANY )
+			{ // Do the concretization.
+			ValPtr any_v = {NewRef{}, v->AsAny()};
+			auto& vt = any_v->GetType();
+			if ( vt->Tag() != t->Tag() )
+				return false;
+
+			v = ZVal(any_v, t);
+			}
+
+		else if ( vt_i->Tag() != t->Tag() )
+			return false;
+		}
+
+	// Require that this vector be treated consistently in the future.
+	yield_type = t;
+	managed_yield = ZVal::IsManagedType(yield_type);
+	delete yield_types;
+	yield_types = nullptr;
+	any_yield = false;
+
+	return true;
+	}
+
 unsigned int VectorVal::Resize(unsigned int new_num_elements)
 	{
 	unsigned int oldsize = vector_val->size();

src/Val.h

@@ -45,6 +45,8 @@ class PrefixTable;
 class CompositeHash;
 class HashKey;
 
+class ZBody;
+
 } // namespace detail
 
 namespace run_state {
@@ -110,6 +112,7 @@ public:
 	virtual ValPtr SizeVal() const;
 
 	// Bytes in total value object.
+	[[deprecated("Remove in v5.1. MemoryAllocation() is deprecated and will be removed. See GHI-572.")]]
 	virtual unsigned int MemoryAllocation() const;
 
 	// Add this value to the given value (if appropriate).
@@ -186,6 +189,9 @@ UNDERLYING_ACCESSOR_DECL(TypeVal, zeek::Type*, AsType)
 	OpaqueVal* AsOpaqueVal();
 	const OpaqueVal* AsOpaqueVal() const;
 
+	TypeVal* AsTypeVal();
+	const TypeVal* AsTypeVal() const;
+
 	void Describe(ODesc* d) const override;
 	virtual void DescribeReST(ODesc* d) const;
 
@@ -450,10 +456,11 @@ public:
 	// Returns a masked port number
 	static uint32_t Mask(uint32_t port_num, TransportProto port_type);
 
-protected:
+	// Only meant for use by ValManager and compiled-to-C++ script
-	friend class ValManager;
+	// functions.
 	PortVal(uint32_t p);
 
+protected:
 	void ValDescribe(ODesc* d) const override;
 	ValPtr DoClone(CloneState* state) override;
 
@@ -480,6 +487,7 @@ public:
 
 	const IPAddr& Get() const	{ return *addr_val; }
 
+	[[deprecated("Remove in v5.1. MemoryAllocation() is deprecated and will be removed. See GHI-572.")]]
 	unsigned int MemoryAllocation() const override;
 
 protected:
@@ -509,6 +517,7 @@ public:
 
 	const IPPrefix& Get() const	{ return *subnet_val; }
 
+	[[deprecated("Remove in v5.1. MemoryAllocation() is deprecated and will be removed. See GHI-572.")]]
 	unsigned int MemoryAllocation() const override;
 
 protected:
@@ -544,6 +553,7 @@ public:
 
 	const String* Get() const	{ return string_val; }
 
+	[[deprecated("Remove in v5.1. MemoryAllocation() is deprecated and will be removed. See GHI-572.")]]
 	unsigned int MemoryAllocation() const override;
 
 	StringValPtr Replace(RE_Matcher* re, const String& repl,
@@ -605,6 +615,7 @@ public:
 
 	const RE_Matcher* Get() const	{ return re_val; }
 
+	[[deprecated("Remove in v5.1. MemoryAllocation() is deprecated and will be removed. See GHI-572.")]]
 	unsigned int MemoryAllocation() const override;
 
 protected:
@@ -654,6 +665,7 @@ public:
 
 	void Describe(ODesc* d) const override;
 
+	[[deprecated("Remove in v5.1. MemoryAllocation() is deprecated and will be removed. See GHI-572.")]]
 	unsigned int MemoryAllocation() const override;
 
 protected:
@@ -927,6 +939,7 @@ public:
 	// the function in the frame allowing it to capture its closure.
 	void InitDefaultFunc(detail::Frame* f);
 
+	[[deprecated("Remove in v5.1. MemoryAllocation() is deprecated and will be removed. See GHI-572.")]]
 	unsigned int MemoryAllocation() const override;
 
 	void ClearTimer(detail::Timer* t)
@@ -1173,13 +1186,15 @@ public:
 	/**
 	 * Appends a value to the record's fields.  The caller is responsible
 	 * for ensuring that fields are appended in the correct order and
-	 * with the correct type.
+	 * with the correct type.  The type needs to be passed in because
+	 * it's unsafe to take it from v when the field's type is "any" while
+	 * v is a concrete type.
 	 * @param v  The value to append.
 	 */
-	void AppendField(ValPtr v)
+	void AppendField(ValPtr v, const TypePtr& t)
 		{
 		if ( v )
-			record_val->emplace_back(ZVal(v, v->GetType()));
+			record_val->emplace_back(ZVal(v, t));
 		else
 			record_val->emplace_back(std::nullopt);
 		}
@@ -1202,6 +1217,18 @@ public:
 		return (*record_val)[field] ? true : false;
 		}
 
+	/**
+	 * Returns true if the given field is in the record, false if
+	 * it's missing.
+	 * @param field  The field name to retrieve.
+	 * @return  Whether there's a value for the given field name.
+	 */
+	bool HasField(const char *field) const
+		{
+		int idx = GetType()->AsRecordType()->FieldOffset(field);
+		return (idx != -1) && HasField(idx);
+		}
+
 	/**
 	 * Returns the value of a given field index.
 	 * @param field  The field index to retrieve.
@@ -1380,6 +1407,7 @@ public:
 	RecordValPtr CoerceTo(RecordTypePtr other,
 	                      bool allow_orphaning = false);
 
+	[[deprecated("Remove in v5.1. MemoryAllocation() is deprecated and will be removed. See GHI-572.")]]
 	unsigned int MemoryAllocation() const override;
 	void DescribeReST(ODesc* d) const override;
 
@@ -1393,6 +1421,23 @@ public:
 	static void DoneParsing();
 
 protected:
+	friend class zeek::detail::ZBody;
+
+	// For use by low-level ZAM instructions.  Caller assumes
+	// responsibility for memory management.  The first version
+	// allows manipulation of whether the field is present at all.
+	// The second version ensures that the optional value is present.
+	std::optional<ZVal>& RawOptField(int field)
+		{ return (*record_val)[field]; }
+
+	ZVal& RawField(int field)
+		{
+		auto& f = RawOptField(field);
+		if ( ! f )
+			f = ZVal();
+		return *f;
+		}
+
 	ValPtr DoClone(CloneState* state) override;
 
 	void AddedField(int field)
@@ -1436,6 +1481,8 @@ protected:
 	friend class Val;
 	friend class EnumType;
 
+	friend EnumValPtr make_enum__CPP(TypePtr t, int i);
+
 	template<class T, class... Ts>
 	friend IntrusivePtr<T> make_intrusive(Ts&&... args);
 
@@ -1468,6 +1515,8 @@ protected:
 class VectorVal final : public Val, public notifier::detail::Modifiable {
 public:
 	explicit VectorVal(VectorTypePtr t);
+	VectorVal(VectorTypePtr t, std::vector<std::optional<ZVal>>* vals);
+
 	~VectorVal() override;
 
 	ValPtr SizeVal() const override;
@@ -1548,8 +1597,29 @@ public:
 	 */
 	VectorValPtr Order(Func* cmp_func = nullptr);
 
+	/**
+	 * Ensures that the vector can be used as a "vector of t".  In
+	 * general, this is only relevant for objects that are typed as
+	 * "vector of any", making sure that each element is in fact
+	 * of type "t", and is internally represented as such so that
+	 * this object can be used directly without any special-casing.
+	 *
+	 * Returns true if the object is compatible with "vector of t"
+	 * (including if it's not a vector-of-any but instead already a
+	 * vector-of-t), false if not compatible.
+	 * @param t  The yield type to concretize to.
+	 * @return  True if the object is compatible with vector-of-t, false
+	 * if not.
+	 */
+	bool Concretize(const TypePtr& t);
+
 	ValPtr ValAt(unsigned int index) const	{ return At(index); }
 
+	bool Has(unsigned int index) const
+		// Version to use once std::optional implementation is merged.
+		// { return index < vector_val->size() && vector_val[index]; }
+		{ return At(index) != nullptr; }
+
 	/**
 	 * Returns the given element in a given underlying representation.
 	 * Enables efficient vector access.  Caller must ensure that the
@@ -1558,8 +1628,12 @@ public:
 	 * @param index  The position in the vector of the element to return.
 	 * @return  The element's underlying value.
 	 */
+	bro_int_t IntAt(unsigned int index) const
+		{ return (*vector_val)[index]->int_val; }
 	bro_uint_t CountAt(unsigned int index) const
 		{ return (*vector_val)[index]->uint_val; }
+	double DoubleAt(unsigned int index) const
+		{ return (*vector_val)[index]->double_val; }
 	const RecordVal* RecordValAt(unsigned int index) const
 		{ return (*vector_val)[index]->record_val; }
 	bool BoolAt(unsigned int index) const
@@ -1569,6 +1643,10 @@ public:
 	const String* StringAt(unsigned int index) const
 		{ return StringValAt(index)->AsString(); }
 
+	// Only intended for low-level access by compiled code.
+	const auto& RawVec() const	{ return vector_val; }
+	auto& RawVec()			{ return vector_val; }
+
 protected:
 	/**
 	 * Returns the element at a given index or nullptr if it does not exist.

src/Var.cc

@@ -672,7 +672,7 @@ void begin_func(IDPtr id, const char* module_name,
 
 class OuterIDBindingFinder : public TraversalCallback {
 public:
-	OuterIDBindingFinder(Scope* s)
+	OuterIDBindingFinder(ScopePtr s)
 		{
 		scopes.emplace_back(s);
 		}
@@ -680,7 +680,7 @@ public:
 	TraversalCode PreExpr(const Expr*) override;
 	TraversalCode PostExpr(const Expr*) override;
 
-	std::vector<Scope*> scopes;
+	std::vector<ScopePtr> scopes;
 	std::unordered_set<ID*> outer_id_references;
 };
 
@@ -766,7 +766,7 @@ void end_func(StmtPtr body)
 	ingredients.release();
 	}
 
-IDPList gather_outer_ids(Scope* scope, Stmt* body)
+IDPList gather_outer_ids(ScopePtr scope, StmtPtr body)
 	{
 	OuterIDBindingFinder cb(scope);
 	body->Traverse(&cb);

src/Var.h

@@ -20,6 +20,7 @@ class Expr;
 class Scope;
 class Stmt;
 using StmtPtr = IntrusivePtr<Stmt>;
+using ScopePtr = IntrusivePtr<Scope>;
 
 enum DeclType { VAR_REGULAR, VAR_CONST, VAR_REDEF, VAR_OPTION, };
 
@@ -41,7 +42,7 @@ extern void begin_func(IDPtr id, const char* module_name, FunctionFlavor flavor,
 extern void end_func(StmtPtr body);
 
 // Gather all IDs referenced inside a body that aren't part of a given scope.
-extern IDPList gather_outer_ids(Scope* scope, Stmt* body);
+extern IDPList gather_outer_ids(ScopePtr scope, StmtPtr body);
 
 } // namespace detail
 } // namespace zeek

src/ZVal.cc

@@ -9,7 +9,7 @@
 using namespace zeek;
 
 
-bool ZVal::zval_was_nil = false;
+bool* ZVal::zval_was_nil_addr = nullptr;
 
 
 ZVal::ZVal(ValPtr v, const TypePtr& t)
@@ -24,7 +24,7 @@ ZVal::ZVal(ValPtr v, const TypePtr& t)
 		return;
 		}
 
-	auto vt = v->GetType();
+	const auto& vt = v->GetType();
 
 	if ( vt->Tag() != t->Tag() && t->Tag() != TYPE_ANY )
 		{
@@ -102,7 +102,7 @@ ZVal::ZVal(ValPtr v, const TypePtr& t)
 		break;
 
 	case TYPE_TYPE:
-		type_val = t->Ref();
+		type_val = v.release()->AsTypeVal();
 		break;
 
 	case TYPE_ERROR:
@@ -222,15 +222,6 @@ ValPtr ZVal::ToVal(const TypePtr& t) const
 	case TYPE_ENUM:
 		return t->AsEnumType()->GetEnumVal(int_val);
 
-	case TYPE_ANY:
-		return {NewRef{}, any_val};
-
-	case TYPE_TYPE:
-		{
-		TypePtr tp = {NewRef{}, type_val};
-		return make_intrusive<TypeVal>(tp);
-		}
-
 	case TYPE_FUNC:
 		if ( func_val )
 			{
@@ -258,6 +249,8 @@ ValPtr ZVal::ToVal(const TypePtr& t) const
 	case TYPE_RECORD:	v = record_val; break;
 	case TYPE_VECTOR:	v = vector_val; break;
 	case TYPE_PATTERN:	v = re_val; break;
+	case TYPE_ANY:		v = any_val; break;
+	case TYPE_TYPE:		v = type_val; break;
 
 	case TYPE_ERROR:
 	case TYPE_TIMER:
@@ -271,7 +264,8 @@ ValPtr ZVal::ToVal(const TypePtr& t) const
 	if ( v )
 		return {NewRef{}, v};
 
-	zval_was_nil = true;
+	if ( zval_was_nil_addr )
+		*zval_was_nil_addr = true;
 
 	return nullptr;
 	}

src/ZVal.h

@@ -18,7 +18,7 @@ class RecordVal;
 class StringVal;
 class SubNetVal;
 class TableVal;
-class Type;
+class TypeVal;
 class Val;
 class VectorVal;
 
@@ -31,6 +31,7 @@ using RecordValPtr = IntrusivePtr<RecordVal>;
 using StringValPtr = IntrusivePtr<StringVal>;
 using SubNetValPtr = IntrusivePtr<SubNetVal>;
 using TableValPtr = IntrusivePtr<TableVal>;
+using TypeValPtr = IntrusivePtr<TypeVal>;
 using ValPtr = IntrusivePtr<Val>;
 using VectorValPtr = IntrusivePtr<VectorVal>;
 
@@ -69,9 +70,10 @@ union ZVal {
 	ZVal(OpaqueVal* v)	{ opaque_val = v; }
 	ZVal(PatternVal* v)	{ re_val = v; }
 	ZVal(TableVal* v)	{ table_val = v; }
+	ZVal(TypeVal* v)	{ type_val = v; }
 	ZVal(RecordVal* v)	{ record_val = v; }
 	ZVal(VectorVal* v)	{ vector_val = v; }
-	ZVal(Type* v)		{ type_val = v; }
+	ZVal(Val* v)		{ any_val = v; }
 
 	ZVal(StringValPtr v)	{ string_val = v.release(); }
 	ZVal(AddrValPtr v)	{ addr_val = v.release(); }
@@ -80,21 +82,14 @@ union ZVal {
 	ZVal(OpaqueValPtr v)	{ opaque_val = v.release(); }
 	ZVal(PatternValPtr v)	{ re_val = v.release(); }
 	ZVal(TableValPtr v)	{ table_val = v.release(); }
+	ZVal(TypeValPtr v)	{ type_val = v.release(); }
 	ZVal(RecordValPtr v)	{ record_val = v.release(); }
 	ZVal(VectorValPtr v)	{ vector_val = v.release(); }
-	ZVal(TypePtr v)		{ type_val = v.release(); }
 
 	// Convert to a higher-level script value.  The caller needs to
 	// ensure that they're providing the correct type.
 	ValPtr ToVal(const TypePtr& t) const;
 
-	// Whether a ZVal was accessed that was missing (a nil pointer).
-	// Used to generate run-time error messages.
-	static bool ZValNilStatus()		{ return zval_was_nil; }
-
-	// Resets the notion of low-level-error-occurred.
-	static void ClearZValNilStatus()	{ zval_was_nil = false; }
-
 	bro_int_t AsInt() const		{ return int_val; }
 	bro_uint_t AsCount() const	{ return uint_val; }
 	double AsDouble() const		{ return double_val; }
@@ -110,10 +105,31 @@ union ZVal {
 	TableVal* AsTable() const	{ return table_val; }
 	RecordVal* AsRecord() const	{ return record_val; }
 	VectorVal* AsVector() const	{ return vector_val; }
-	Type* AsType() const		{ return type_val; }
+	TypeVal* AsType() const		{ return type_val; }
 	Val* AsAny() const		{ return any_val; }
 
 	Obj* ManagedVal() const		{ return managed_val; }
+	void ClearManagedVal()		{ managed_val = nullptr; }
+
+	// The following return references that can be used to
+	// populate the ZVal.  Handy for compiled ZAM code.
+	bro_int_t& AsIntRef()		{ return int_val; }
+	bro_uint_t& AsCountRef()	{ return uint_val; }
+	double& AsDoubleRef()		{ return double_val; }
+	StringVal*& AsStringRef()	{ return string_val; }
+	AddrVal*& AsAddrRef()		{ return addr_val; }
+	SubNetVal*& AsSubNetRef()	{ return subnet_val; }
+	File*& AsFileRef()		{ return file_val; }
+	Func*& AsFuncRef()		{ return func_val; }
+	ListVal*& AsListRef()		{ return list_val; }
+	OpaqueVal*& AsOpaqueRef()	{ return opaque_val; }
+	PatternVal*& AsPatternRef()	{ return re_val; }
+	TableVal*& AsTableRef()		{ return table_val; }
+	RecordVal*& AsRecordRef()	{ return record_val; }
+	VectorVal*& AsVectorRef()	{ return vector_val; }
+	TypeVal*& AsTypeRef()		{ return type_val; }
+	Val*& AsAnyRef()		{ return any_val; }
+	Obj*& ManagedValRef()		{ return managed_val; }
 
 	// True if a given type is one for which we manage the associated
 	// memory internally.
@@ -133,6 +149,14 @@ union ZVal {
 			DeleteManagedType(v);
 		}
 
+	// Specifies the address of a flag to set if a ZVal is accessed
+	// that was missing (a nil pointer).  Used to generate run-time
+	// error messages.  We use an address-based interface so that
+	// this flag can be combined with a general-purpose error flag,
+	// allowing inner loops to only have to test a single flag.
+	static void SetZValNilStatusAddr(bool* _zval_was_nil_addr)
+		{ zval_was_nil_addr = _zval_was_nil_addr; }
+
 private:
 	friend class RecordVal;
 	friend class VectorVal;
@@ -161,7 +185,7 @@ private:
 	TableVal* table_val;
 	RecordVal* record_val;
 	VectorVal* vector_val;
-	Type* type_val;
+	TypeVal* type_val;
 
 	// Used for "any" values.
 	Val* any_val;
@@ -175,7 +199,7 @@ private:
 	// because often the caller won't have direct access to the
 	// particular ZVal that produces the issue, and just wants to
 	// know whether it occurred at some point.
-	static bool zval_was_nil;
+	static bool* zval_was_nil_addr;
 };
 
 } // zeek

src/ZeekString.cc

@@ -6,18 +6,23 @@
 #include <ctype.h>
 #include <algorithm>
 #include <iostream>
+#include <sstream>	// Needed for unit testing
 
 #include "zeek/Val.h"
 #include "zeek/ID.h"
 #include "zeek/Reporter.h"
 #include "zeek/util.h"
 
+#include "zeek/3rdparty/doctest.h"
+
 #ifdef DEBUG
 #define DEBUG_STR(msg) DBG_LOG(zeek::DBG_STRING, msg)
 #else
 #define DEBUG_STR(msg)
 #endif
 
+using namespace std::string_literals;
+
 namespace zeek {
 
 constexpr int String::EXPANDED_STRING;
@@ -101,6 +106,19 @@ bool String::operator<(const String &bs) const
 	return Bstr_cmp(this, &bs) < 0;
 	}
 
+bool String::operator==(std::string_view s) const
+	{
+	if ( static_cast<size_t>(n) != s.size() )
+		return false;
+
+	if ( b == nullptr )
+		{
+		return s.size() == 0;
+		}
+
+	return (memcmp(b, s.data(), n) == 0);
+	}
+
 void String::Adopt(byte_vec bytes, int len)
 	{
 	Reset();
@@ -169,6 +187,7 @@ const char* String::CheckString() const
 		{
 		// Either an embedded NUL, or no final NUL.
 		char* exp_s = Render();
+
 		if ( nulTerm )
 			reporter->Error("string with embedded NUL: \"%s\"", exp_s);
 		else
@@ -343,7 +362,7 @@ String::Vec* String::Split(const String::IdxVec& indices) const
 	return result;
 	}
 
-VectorVal* String:: VecToPolicy(Vec* vec)
+VectorVal* String::VecToPolicy(Vec* vec)
 	{
 	auto result = make_intrusive<VectorVal>(id::string_vec);
 
@@ -390,11 +409,10 @@ char* String::VecToString(const Vec* vec)
 	return strdup(result.c_str());
 	}
 
-bool StringLenCmp::operator()(String * const& bst1,
+bool StringLenCmp::operator()(String* const& bst1, String* const& bst2)
-				 String * const& bst2)
 	{
 	return _increasing ? (bst1->Len() < bst2->Len()) :
-				(bst1->Len() > bst2->Len());
+	                     (bst1->Len() > bst2->Len());
 	}
 
 std::ostream& operator<<(std::ostream& os, const String& bs)
@@ -496,3 +514,235 @@ void delete_strings(std::vector<const String*>& v)
 	}
 
 } // namespace zeek
+
+TEST_SUITE_BEGIN("ZeekString");
+
+TEST_CASE("construction")
+	{
+	zeek::String s1{};
+	CHECK_EQ(s1.Len(), 0);
+	CHECK_EQ(s1.Bytes(), nullptr);
+	CHECK_EQ(s1, "");
+
+	std::string text = "abcdef";
+	zeek::byte_vec text2 = new u_char[7];
+	memcpy(text2, text.c_str(), 7);
+
+	zeek::String s2{text2, 6, false};
+	CHECK_EQ(s2.Len(), 6);
+
+	zeek::String s3{text2, 6, true};
+	CHECK_EQ(s3.Len(), 6);
+
+	zeek::String s4{"abcdef"};
+	CHECK_EQ(s4.Len(), 6);
+
+	zeek::String s5{std::string("abcdef")};
+	CHECK_EQ(s5.Len(), 6);
+
+	zeek::String s6{s5};
+	CHECK_EQ(s6.Len(), 6);
+
+	zeek::String s7{true, text2, 6};
+	CHECK_EQ(s7.Len(), 6);
+	CHECK_EQ(s7.Bytes(), text2);
+
+	// Construct a temporary reporter object for the next two tests
+	zeek::reporter = new zeek::Reporter(false);
+
+	zeek::byte_vec text3 = new u_char[7];
+	memcpy(text3, text.c_str(), 7);
+	zeek::String s8{false, text3, 6};
+	CHECK_EQ(std::string(s8.CheckString()), "<string-with-NUL>");
+
+	zeek::byte_vec text4 = new u_char[7];
+	memcpy(text4, text.c_str(), 7);
+	text4[2] = '\0';
+	zeek::String s9{false, text4, 6};
+	CHECK_EQ(std::string(s9.CheckString()), "<string-with-NUL>");
+
+	delete zeek::reporter;
+
+	zeek::byte_vec text5 = (zeek::byte_vec)malloc(7);
+	memcpy(text5, text.c_str(), 7);
+	zeek::String s10{true, text5, 6};
+	s10.SetUseFreeToDelete(1);
+	CHECK_EQ(s10.Bytes(), text5);
+	}
+
+TEST_CASE("set/assignment/comparison")
+	{
+	zeek::String s{"abc"};
+	CHECK_EQ(s, "abc");
+
+	s.Set("def");
+	CHECK_EQ(s, "def");
+
+	s.Set(std::string("ghi"));
+	CHECK_EQ(s, "ghi");
+
+	zeek::String s2{"abc"};
+	s.Set(s2);
+	CHECK_EQ(s, "abc");
+
+	zeek::String s3{"def"};
+	s = s3;
+	CHECK_EQ(s, "def");
+	CHECK_EQ(s, s3);
+	CHECK(s2 < s3);
+
+	s.Set("ghi");
+	CHECK_FALSE(s < s2);
+
+	std::string text = "abcdef";
+	zeek::byte_vec text2 = new u_char[7];
+	memcpy(text2, text.c_str(), 7);
+	s.Adopt(text2, 7);
+
+	CHECK_EQ(s, "abcdef");
+	CHECK_FALSE(s == s2);
+
+	// This is a clearly invalid string and we probably shouldn't allow it to be
+	// constructed, but this test covers one if statement in Bstr_eq.
+	zeek::String s4(false, nullptr, 3);
+	CHECK_FALSE(s4 == s2);
+
+	zeek::String s5{};
+	CHECK_LT(s5, s);
+	CHECK_FALSE(s < s5);
+	}
+
+TEST_CASE("searching/modification")
+	{
+	zeek::String s{"this is a test"};
+	auto* ss = s.GetSubstring(5, 4);
+	CHECK_EQ(*ss, "is a");
+	delete ss;
+
+	auto* ss2 = s.GetSubstring(-1, 4);
+	CHECK_EQ(ss2, nullptr);
+	ss2 = s.GetSubstring(s.Len() + 5, 4);
+	CHECK_EQ(ss2, nullptr);
+
+	zeek::String s2{"test"};
+	CHECK_EQ(s.FindSubstring(&s2), 10);
+
+	s2.ToUpper();
+	CHECK_EQ(s2, "TEST");
+
+	zeek::String::IdxVec indexes;
+	zeek::String::Vec* splits = s.Split(indexes);
+	CHECK_EQ(splits, nullptr);
+
+	indexes.insert(indexes.end(), {4, 7, 9, -1, 30});
+	splits = s.Split(indexes);
+	CHECK_EQ(splits->size(), 4);
+	CHECK_EQ(*(splits->at(0)), "this");
+	CHECK_EQ(*(splits->at(1)), " is");
+	CHECK_EQ(*(splits->at(2)), " a");
+	CHECK_EQ(*(splits->at(3)), " test");
+
+	zeek::String* s3 = concatenate(*splits);
+	CHECK_EQ(s.Len(), s3->Len());
+	CHECK_EQ(s, *s3);
+	delete s3;
+
+	char* temp = zeek::String::VecToString(splits);
+	CHECK_EQ(std::string(temp), "[this, is, a, test,]");
+	free(temp);
+
+ 	for ( auto* entry : *splits )
+		delete entry;
+	delete splits;
+	}
+
+TEST_CASE("rendering")
+	{
+	zeek::String s1("\\abcd\'\"");
+	auto* r = s1.Render(zeek::String::ESC_ESC);
+	CHECK_EQ(std::string(r), "\\\\abcd\'\"");
+	delete [] r;
+
+	r = s1.Render(zeek::String::ESC_QUOT);
+	CHECK_EQ(std::string(r), "\\abcd\\\'\\\"");
+	delete [] r;
+
+	r = s1.Render(zeek::String::ESC_ESC | zeek::String::ESC_QUOT | zeek::String::ESC_SER);
+	CHECK_EQ(std::string(r), "10 \\\\abcd\\\'\\\"");
+	delete [] r;
+
+	zeek::byte_vec text = new u_char[6];
+	text[0] = 3;
+	text[1] = 4;
+	text[2] = 5;
+	text[3] = 6;
+	text[4] = '\\';
+	text[5] = '\'';
+	zeek::String s2(false, text, 6);
+
+	r = s2.Render(zeek::String::ESC_HEX);
+	CHECK_EQ(std::string(r), "\\x03\\x04\\x05\\x06\\\'");
+	delete [] r;
+
+	int test_length = 0;
+	r = s2.Render(zeek::String::ESC_DOT, &test_length);
+	CHECK_EQ(std::string(r), "....\\\'");
+	CHECK_EQ(test_length, 7);
+	delete [] r;
+
+	r = s2.Render(zeek::String::BRO_STRING_LITERAL);
+	CHECK_EQ(std::string(r), "\\x03\\x04\\x05\\x06\\\\\\\'");
+	delete [] r;
+
+	std::ostringstream os1;
+	// This uses ESC_HEX, so it should be the same as the test above
+	os1 << s2;
+	CHECK_EQ(os1.str(), "\\x03\\x04\\x05\\x06\\\'");
+
+	std::ostringstream os2;
+	s2.Render(os2, zeek::String::ESC_HEX);
+	CHECK_EQ(os2.str(), "\\x03\\x04\\x05\\x06\\\'");
+	}
+
+TEST_CASE("read")
+	{
+	std::string text1("5 abcde");
+	std::istringstream iss1(text1);
+	zeek::String s1{};
+	s1.Read(iss1);
+	CHECK_EQ(s1, "abcde");
+
+	std::string text2("abcde");
+	std::istringstream iss2(text2);
+	zeek::String s2{};
+	// Setting to something else disables reading the serialization format
+	s2.Read(iss2, zeek::String::ESC_HEX);
+	CHECK_EQ(s2, text2);
+	}
+
+TEST_CASE("misc")
+	{
+	std::vector<const zeek::String*> sv = {new zeek::String{}, new zeek::String{}};
+	CHECK_EQ(sv.size(), 2);
+	zeek::delete_strings(sv);
+	CHECK_EQ(sv.size(), 0);
+
+	std::vector<zeek::data_chunk_t> dv = {{5, "abcde"}, {6, "fghijk"}};
+	auto* s = zeek::concatenate(dv);
+	CHECK_EQ(*s, "abcdefghijk");
+	delete s;
+
+	std::vector<zeek::String*> sv2 = {new zeek::String{"abcde"}, new zeek::String{"fghi"}};
+	std::sort(sv2.begin(), sv2.end(), zeek::StringLenCmp(true));
+	CHECK_EQ(*(sv2.front()), "fghi");
+	CHECK_EQ(*(sv2.back()), "abcde");
+
+	std::sort(sv2.begin(), sv2.end(), zeek::StringLenCmp(false));
+	CHECK_EQ(*(sv2.front()), "abcde");
+	CHECK_EQ(*(sv2.back()), "fghi");
+
+	for ( auto* entry : sv2 )
+		delete entry;
+	}
+
+TEST_SUITE_END();

src/ZeekString.h

@@ -19,6 +19,12 @@ class VectorVal;
 
 typedef u_char* byte_vec;
 
+/**
+ * A container type for holding blocks of byte data. This can be used for
+ * character strings, but is not limited to that alone. This class provides
+ * methods for rendering byte data into character strings, including
+ * conversions of non-printable characters into other representations.
+ */
 class String {
 public:
 	typedef std::vector<String*> Vec;
@@ -49,6 +55,7 @@ public:
 	const String& operator=(const String& bs);
 	bool operator==(const String& bs) const;
 	bool operator<(const String& bs) const;
+	bool operator==(std::string_view s) const;
 
 	byte_vec Bytes() const	{ return b; }
 	int Len() const	{ return n; }
@@ -71,6 +78,13 @@ public:
 	void SetUseFreeToDelete(int use_it)
 		{ use_free_to_delete = use_it; }
 
+	/**
+	 * Returns a character-string representation of the stored bytes. This
+	 * method doesn't do any extra rendering or character conversions. If
+	 * null characters are found in the middle of the data or if the data
+	 * is missing a closing null character, an error string is returned and
+	 * a error is reported.
+	 */
 	const char* CheckString() const;
 
 	enum render_style {
@@ -116,6 +130,7 @@ public:
 	// XXX and to_upper; the latter doesn't use String::ToUpper().
 	void ToUpper();
 
+	[[deprecated("Remove in v5.1. MemoryAllocation() is deprecated and will be removed. See GHI-572.")]]
 	unsigned int MemoryAllocation() const;
 
 	// Returns new string containing the substring of this string,

src/analyzer/Analyzer.cc

@@ -764,6 +764,8 @@ void Analyzer::AppendNewChildren()
 
 unsigned int Analyzer::MemoryAllocation() const
 	{
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
 	unsigned int mem = padded_sizeof(*this)
 		+ (timers.MemoryAllocation() - padded_sizeof(timers));
 
@@ -777,6 +779,7 @@ unsigned int Analyzer::MemoryAllocation() const
 		mem += a->MemoryAllocation();
 
 	return mem;
+#pragma GCC diagnostic pop
 	}
 
 void Analyzer::UpdateConnVal(RecordVal *conn_val)
@@ -880,31 +883,4 @@ void SupportAnalyzer::ForwardUndelivered(uint64_t seq, int len, bool is_orig)
 	}
 
 
-void TransportLayerAnalyzer::Done()
-	{
-	Analyzer::Done();
-	}
-
-void TransportLayerAnalyzer::SetContentsFile(unsigned int /* direction */,
-                                             FilePtr /* f */)
-	{
-	reporter->Error("analyzer type does not support writing to a contents file");
-	}
-
-FilePtr TransportLayerAnalyzer::GetContentsFile(unsigned int /* direction */) const
-	{
-	reporter->Error("analyzer type does not support writing to a contents file");
-	return nullptr;
-	}
-
-void TransportLayerAnalyzer::PacketContents(const u_char* data, int len)
-	{
-	if ( packet_contents && len > 0 )
-		{
-		String* cbs = new String(data, len, true);
-		auto contents = make_intrusive<StringVal>(cbs);
-		EnqueueConnEvent(packet_contents, ConnVal(), std::move(contents));
-		}
-	}
-
 } // namespace zeek::analyzer