Merge branch 'master' into topic/vlad/page_hostnames

.cirrus.yml

@@ -13,6 +13,7 @@ memory: &MEMORY 4GB
 config: &CONFIG --build-type=release --enable-cpp-tests --disable-broker-tests --prefix=$CIRRUS_WORKING_DIR/install
 static_config: &STATIC_CONFIG --build-type=release --enable-cpp-tests --disable-broker-tests --enable-static-broker --enable-static-binpac --prefix=$CIRRUS_WORKING_DIR/install
 sanitizer_config: &SANITIZER_CONFIG --build-type=debug --enable-cpp-tests --disable-broker-tests --sanitizers=address,undefined --enable-fuzzers --enable-coverage
+mobile_ipv6_config: &MOBILE_IPV6_CONFIG --build-type=release --enable-cpp-tests --enable-mobile-ipv6 --disable-broker-tests --prefix=$CIRRUS_WORKING_DIR/install
 
 resources_template: &RESOURCES_TEMPLATE
   cpu: *CPUS
@@ -82,7 +83,7 @@ env:
   # This is the https endpoint host and port used for benchmarking. It's kept
   # encrypted as a security measure to avoid leaking the host's information.
   ZEEK_BENCHMARK_HOST: ENCRYPTED[62ecdc93e839800d754d09d9a9070e9cb9b209e7d7dd2472ba38648f786ff272d0e0ea71233d0910025f2c6f3771259c]
-  ZEEK_BENCHMARK_PORT: ENCRYPTED[fb34ae2d51bac798fc01da052f3772154e17bbe2c1c5615509e82935248e748053fda399a0caf909632b6272cebff9f4]
+  ZEEK_BENCHMARK_PORT: ENCRYPTED[b97fabf4d6bd5eef107c8469c5cb2c44e0107d89c220f43e7d1e7bdfb32dbdc2620855fee8e5a8d889458d5a6ac3e5c7]
 
   # The repo token used for uploading data to Coveralls.io
   ZEEK_COVERALLS_REPO_TOKEN: ENCRYPTED[7ffd1e041f848f02b62f5abc7fda8a5a8a1561fbb2b46d88cefb67c74408ddeef6ea6f3b279c7953ca14ae9b4d050e2d]
@@ -90,6 +91,13 @@ env:
 # Linux EOL timelines: https://linuxlifecycle.com/
 # Fedora (~13 months): https://fedoraproject.org/wiki/Fedora_Release_Life_Cycle
 
+fedora34_task:
+  container:
+    # Fedora 34 EOL: Around May 2022
+    dockerfile: ci/fedora-34/Dockerfile
+    << : *RESOURCES_TEMPLATE
+  << : *CI_TEMPLATE
+
 fedora33_task:
   container:
     # Fedora 33 EOL: Around November 2022
@@ -168,6 +176,13 @@ opensuse_leap_15_2_task:
     << : *RESOURCES_TEMPLATE
   << : *CI_TEMPLATE
 
+opensuse_leap_15_3_task:
+  container:
+    # Opensuse Leap 15.3 EOL: TBD
+    dockerfile: ci/opensuse-leap-15.3/Dockerfile
+    << : *RESOURCES_TEMPLATE
+  << : *CI_TEMPLATE
+
 ubuntu20_task:
   container:
     # Ubuntu 20.04 EOL: April 2025
@@ -181,6 +196,8 @@ ubuntu18_task:
     dockerfile: ci/ubuntu-18.04/Dockerfile
     << : *RESOURCES_TEMPLATE
   << : *CI_TEMPLATE
+  env:
+    ZEEK_CI_CONFIGURE_FLAGS: *MOBILE_IPV6_CONFIG
 
 ubuntu16_task:
   container:
@@ -202,7 +219,7 @@ alpine_task:
 # We aim to support both the current and previous macOS release.
 macos_big_sur_task:
   macos_instance:
-    image: big-sur-base
+    image: big-sur-xcode-12.5
   prepare_script: ./ci/macos/prepare.sh
   << : *CI_TEMPLATE
   << : *MACOS_RESOURCES_TEMPLATE
@@ -215,6 +232,16 @@ macos_catalina_task:
   << : *MACOS_RESOURCES_TEMPLATE
 
 # FreeBSD EOL timelines: https://www.freebsd.org/security/security.html#sup
+freebsd13_task:
+  freebsd_instance:
+    # FreeBSD 13 EOL: January 31, 2026
+    image_family: freebsd-13-0
+    cpu: 8
+    # Not allowed to request less than 8GB for an 8 CPU FreeBSD VM.
+    memory: 8GB
+  prepare_script: ./ci/freebsd/prepare.sh
+  << : *CI_TEMPLATE
+
 freebsd12_task:
   freebsd_instance:
     # FreeBSD 12 EOL: June 30, 2024

.gitignore

@@ -1,4 +1,7 @@
-build
+# Ignore anything prefixed with build since people
+# tend to name all of their build directories prefixed that way.
+build*
+
 tmp
 *.gcov
 
@@ -11,3 +14,6 @@ cmake-build-*
 
 # ignore pyenv local settings
 .python-version
+
+# clangd
+.cache

CHANGES

@@ -1,3 +1,410 @@
+4.1.0-dev.722 | 2021-06-10 10:42:57 -0700
+
+  * Added --include-plugins configure argument (Seth Hall, Corelight)
+
+4.1.0-dev.720 | 2021-06-10 11:29:19 +0100
+
+  * Introduce script-land option LogAscii::logdir that can be used to set
+    the logging directory. (Henrik Kramselund Jereminsen)
+
+4.1.0-dev.715 | 2021-06-09 09:12:26 -0700
+
+  * Fix macOS Big Sur builds on Cirrus
+
+    - Upgrade the Big Sur VM to use the Xcode 12.5 version. This has a newer
+      version of brew installed on it that fixes an issue with an EOL package host
+      that finally shut down for good recently.
+    - Use 'brew upgrade' for openssl and cmake, since those are both present on the
+      base VM. This prevents 'brew install' from printing an error if the package
+      exists but is out of date. (Tim Wojtulewicz, Corelight)
+
+4.1.0-dev.713 | 2021-06-08 13:54:28 -0700
+
+  * Add OpenSUSE Leap 15.3 to testing (Johanna Amann, Corelight)
+
+4.1.0-dev.709 | 2021-06-07 09:41:28 +0200
+
+  * Improve assignment operators for IntrusivePtr. (Dominik Charousset, Corelight)
+
+  * Fix docs for `ProcStats`: `mem` is in bytes, not KB. (Arne Welzel, Corelight)
+
+4.1.0-dev.704 | 2021-06-04 08:29:18 -0700
+
+  * Add deprecated headers for UDP and ICMP analyzers (Tim Wojtulewicz, Corelight)
+
+  * Fix handling of IP packets with bogus IP header lengths
+
+    Credit to OSS-Fuzz for discovery
+    https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=34711
+    (Link to details becomes public 30 days after patch release) (Tim Wojtulewicz, Corelight)
+
+4.1.0-dev.700 | 2021-06-03 09:27:57 -0700
+
+  * Make update-traces fail when the curl invocation fails (Christian Kreibich, Corelight)
+
+4.1.0-dev.697 | 2021-06-02 15:08:12 -0700
+
+  * Add FreeBSD 13 to CI (Christian Kreibich, Corelight)
+
+  * Add Fedora 34 to CI (Christian Kreibich, Corelight)
+
+4.1.0-dev.693 | 2021-06-02 13:22:09 -0700
+
+  * Label session adapters in the output of zeek -NN (Tim Wojtulewicz, Corelight)
+
+  * Split session adapter code into separate files from the analyzers (Tim Wojtulewicz, Corelight)
+
+  * Move adapter-specific code back into the adapter (Tim Wojtulewicz, Corelight)
+
+  * Move ICMP counterpart methods outside of ICMPAnalyzer class
+
+    These were previously global methods in the old analyzer, and moving them
+    to be private members of ICMPAnalyzer broke the usage of them by at least
+    one external plugin. (Tim Wojtulewicz, Corelight)
+
+  * Remove obsolete Skipping()/SetSkip() from Connection (Tim Wojtulewicz, Corelight)
+
+  * Remove some code from IPBasedAnalyzer and children that was waiting for TCP to be implemented (Tim Wojtulewicz, Corelight)
+
+  * Move TCPStateStats object out of session_mgr (Tim Wojtulewicz, Corelight)
+
+  * Move analyzer-to-port mapping out of analyzer::Manager into packet analyzers (Tim Wojtulewicz, Corelight)
+
+  * Move packet parsing code out of adapter into analyzer (Tim Wojtulewicz, Corelight)
+
+  * Move old TCP analyzer into analyzer adapter in packet analysis tree (Tim Wojtulewicz, Corelight)
+
+4.1.0-dev.681 | 2021-06-02 09:57:41 -0700
+
+  * Add some extra length checking when parsing mobile ipv6 packets
+
+    Credit to OSS-Fuzz for discovery
+    https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=34263
+    (Link to details becomes public 30 days after patch release) (Tim Wojtulewicz, Corelight)
+
+4.1.0-dev.679 | 2021-06-01 19:11:40 -0700
+
+  * Replace toupper() usages in netbios decoding BIFs (Jon Siwek, Corelight)
+
+    This avoids potential for locale-dependent results of toupper() by
+    instead using a function that simply maps ASCII characters a-z to A-Z.
+
+4.1.0-dev.676 | 2021-06-01 10:19:19 -0700
+
+  * Integrate review feedback (Dominik Charousset, Corelight)
+
+  * Sync new broker options, fix name inconsistencies (Dominik Charousset, Corelight)
+
+  * Integrate new Broker metric exporter parameters (Dominik Charousset, Corelight)
+
+4.1.0-dev.671 | 2021-06-01 09:51:38 -0700
+
+  * Update detect-MHR.zeek (Chris C)
+
+    Update Virustotal URL to current
+    option match_sub_url = "https://www.virustotal.com/gui/search/%s"
+
+4.1.0-dev.669 | 2021-06-01 09:39:30 -0700
+
+  * GH-839: Fix use of &optional sub-records within table/set indices (Jon Siwek, Corelight)
+
+4.1.0-dev.666 | 2021-05-26 10:51:51 -0700
+
+  * Ensure SessionAdapter members are initialized (Tim Wojtulewicz, Corelight)
+
+    Fixes Coverity #1453273
+
+4.1.0-dev.665 | 2021-05-26 08:07:26 +0200
+
+  * Extend the file analyzer API to set source manually. (Robin
+    Sommer, Corelight)
+
+4.1.0-dev.661 | 2021-05-24 15:03:54 -0700
+
+  * Update Broker submodule for bump of embedded CAF to 0.18.3 (Jon Siwek, Corelight)
+
+4.1.0-dev.660 | 2021-05-24 12:38:44 -0700
+
+  * Add type field to session::Key to help avoid collisions in map (Tim Wojtulewicz, Corelight)
+
+  * Move bad UDP checksum handling into adapter object (Tim Wojtulewicz, Corelight)
+
+  * Rename IPBasedTransportAnalyzer to SessionAdapter (Tim Wojtulewicz, Corelight)
+
+    This also also combines the old TransportLayerAnalyzer class into
+    SessionAdapter, and removes the old class. This requires naming changes
+    in a few places but no functionality changes.
+
+  * Move building session analyzer tree out of analyzer::Manager (Tim Wojtulewicz, Corelight)
+
+  * Rework the packet flow through the IP-based analyzers (Tim Wojtulewicz, Corelight)
+
+  * Add new UDP packet analyzer, remove old one (Tim Wojtulewicz, Corelight)
+
+  * Add new ICMP packet analyzer, remove old one (Tim Wojtulewicz, Corelight)
+
+  * Add base class for IP-based packet analyzers (Tim Wojtulewicz, Corelight)
+
+  * Move SessionManager::ParseIPPacket to IP analyzer's namespace (Tim Wojtulewicz, Corelight)
+
+  * Added skeletons for TCP/UDP/ICMP packet analysis plugins. (Tim Wojtulewicz, Corelight)
+
+    This includes integration into the IP plugin and calling of the sessions code from each plugin.
+
+4.1.0-dev.646 | 2021-05-18 11:47:25 -0700
+
+  * Omit unneeded decimal points in modp_dtoa2() scientific notation output (Jon Siwek, Corelight)
+
+    For example, "1e-13" is now used instead of "1.e-13".
+
+  * GH-1244: Change modp_dtoa2() to use scientific notation for small values (Jon Siwek, Corelight)
+
+    This fixes problems where printing floating point numbers less than
+    10^-6 output as "0.0".  Such numbers now use using scientific notation
+    and preserve the value's actual floating point representation.
+
+4.1.0-dev.643 | 2021-05-17 11:57:58 -0700
+
+  * GH-1546: Make DictIterator() public, add copy/move operators (Tim Wojtulewicz, Corelight)
+
+4.1.0-dev.641 | 2021-05-17 11:28:11 -0700
+
+  * GH-1558: Fix reading `vector of enum` types from config files (Jon Siwek, Corelight)
+
+  * GH-1555: Fix reading empty set[enum] values from config files (Jon Siwek, Corelight)
+
+4.1.0-dev.638 | 2021-05-17 13:08:28 +0100
+
+  * Manual page updates (Henrik Kramselund Jereminsen)
+
+4.1.0-dev.631 | 2021-05-11 09:26:37 -0700
+
+  * Add unit tests to ZeekString.cc (Tim Wojtulewicz)
+
+4.1.0-dev.628 | 2021-05-10 12:44:25 -0700
+
+  * Add experimental support for translating Zeek scripts to equivalent C++ (Vern Paxson, Corelight)
+
+    The generated C++ can then be compiled directly into the `zeek` binary,
+    replacing use of the interpreter and producing better runtime performance.
+    See `src/script_opt/CPP/README.md` for a guide on how to use this feature.
+
+  * Add new "-a cpp" btest alternative (Vern Paxson, Corelight)
+
+  * Add VectorVal methods to leverage ZVal representation (Vern Paxson, Corelight)
+
+  * Fix backtrace BiF to avoid iterator invalidation & support compiled code (Vern Paxson, Corelight)
+
+4.1.0-dev.593 | 2021-05-10 10:17:34 +0100
+
+  * Explain zeek-config options in help output (Christian Kreibich, Corelight)
+
+  * Sort variables at top of zeek-config alphabetically (Christian Kreibich, Corelight)
+
+  * Install Zeek's btest tooling with the distribution
+
+    This creates $PREFIX/share/btest in the install tree, with the
+    following folders:
+
+    - scripts/ for the canonifiers
+    - data/ for random.seed
+    - data/pcaps for the test pcaps
+
+    The pcaps can be skipped by configuring with --disable-btest-pcaps. (Christian Kreibich, Corelight)
+
+4.1.0-dev.587 | 2021-05-05 14:05:51 +0000
+
+  * Merge remote-tracking branch 'origin/topic/timw/session-coverity'
+
+    * origin/topic/timw/session-coverity:
+      Minor cleanup in IPAddr.h
+      Fix a few Coverity warnings from the session manager work (Tim Wojtulewicz)
+
+  * Minor cleanup in IPAddr.h (Tim Wojtulewicz, Corelight)
+
+  * Fix a few Coverity warnings from the session manager work
+
+    - Be explicit about setting the copied flag in session::Key. Coverity seems
+      confused about when that flag is set if it gets set by default
+      initialization. This should fix 1452757 and 1452759.
+    - Explicitly copy the fields in ConnKey instead of using memcpy. Fixes
+      1452758. (Tim Wojtulewicz, Corelight)
+
+4.1.0-dev.583 | 2021-05-03 18:21:33 -0700
+
+  * switched RecordVal's to use std::optional for tracking missing fields (Vern Paxson, Corelight)
+
+  * added constructors for directly building ZVal's (Vern Paxson, Corelight)
+
+4.1.0-dev.580 | 2021-04-30 18:29:22 -0700
+
+  * Add missing zeek/ prefix to a telemetry header's includes (Jon Siwek, Corelight)
+
+4.1.0-dev.578 | 2021-04-30 09:42:26 -0700
+
+  * GH-1534: Fix excessive coredump for duplicate enum definitions
+
+    An adequate error message was previously reported for duplicate enum
+    definitions, this just now prevents trying to access it as a constant in
+    subsequent parsing and further generating a coredump. (Jon Siwek, Corelight)
+
+4.1.0-dev.576 | 2021-04-30 09:40:18 -0700
+
+  * Fixes to `decode_netbios_name` and `decode_netbios_name_type` BIFs
+
+    Fixes to `decode_netbios_name`:
+
+    * Improve validation that input string is a NetBIOS encoding
+      (32 bytes, with characters ranging from 'A' to 'P').  This helps
+      prevent Undefined Behavior of left-shifting negative values.
+      Invalid encodings now cause a return-value of an empty string.
+
+    * More liberal in what decoded characters are allowed.  Namely,
+      spaces are now allowed (but any trailing null-bytes and spaces
+      are trimmed, similar to before).
+
+    Fixes to `decode_netbios_name_type`:
+
+    * Improve validation that input string is a NetBIOS encoding
+      (32 bytes, with characters ranging from 'A' to 'P').  This helps
+      prevent Undefined Behavior of left-shifting negative values and
+      a heap-buffer-overread when the input string is too small.
+      Invalid encodings now cause a return-value of 256. (Jon Siwek, Corelight)
+
+4.1.0-dev.573 | 2021-04-29 11:29:39 -0700
+
+  * Rename ConnID and ConnIDKey (Tim Wojtulewicz, Corelight)
+
+  * Remove Session prefix from some session-related classes and files (Tim Wojtulewicz, Corelight)
+
+  * Move session code into new directory and into zeek::session namespace (Tim Wojtulewicz, Corelight)
+
+  * Move SessionKey into a separate file, added comments (Tim Wojtulewicz, Corelight)
+
+  * Review cleanup
+
+    - Add constructors for ConnIDKey, remove BuildConnIDKey()
+    - Rename protocol stats classes and move to implementation file
+    - Rename "num" field of protocol stats to "active"
+    - Explicitly delete copy operations for SessionKey
+    - Change argument for ProtocolStats methods to const-reference
+    - Make key validity methods in Session not be virtual
+    - Rename Session::ClearKey and Session::IsKeyValid (Tim Wojtulewicz, Corelight)
+
+  * Rework stats to store handles to the counters (Tim Wojtulewicz, Corelight)
+
+  * Rename some connection-specific methods in Session and SessionManager (Tim Wojtulewicz, Corelight)
+
+  * Add deprecated version of Sessions.h (Tim Wojtulewicz, Corelight)
+
+  * Rename NetSessions to SessionManager
+
+    This also includes:
+    - Deprecating the NetSessions name.
+    - Renaming the zeek::sessions global to zeek::session_mgr and deprecating the old name.
+    - Renaming Sessions.{h,cc} to SessionManager.{h,cc}. (Tim Wojtulewicz, Corelight)
+
+  * Store a single map of Sessions instead of split maps of Connections.
+
+    This commit also includes:
+    - Storing the transport protocol in ConnID and ConnIDKey to allow tcp and
+      udp connections from the same IP/Port combinations. This happens in the
+      core.cisco-fabric-path test, for example.
+    - Lots of test updates. The reasons for these are two fold. First, with
+      the change to only store a single map means that TCP, UDP, and ICMP
+      connections are now mixed. When Zeek drains the map at shutdown, it drains
+      each of those protocols together instead of separately. The second is
+      because of how Sessions are stored in the map. We're now storing them
+      keyed by the hash of the key stored by the Session objects, which causes
+      them to again be in the map in a different order. (Tim Wojtulewicz, Corelight)
+
+  * Convert session stats to use the new telemetry API (Tim Wojtulewicz, Corelight)
+
+  * Add test for get_conn_stats BIF before reworking session stats (Tim Wojtulewicz, Corelight)
+
+  * Add new Session base class
+
+    This is mostly code copied from the existing Connection class, as that class now
+    inherits from Session. (Tim Wojtulewicz, Corelight)
+
+  * Move packet filter out of NetSessions (Tim Wojtulewicz, Corelight)
+
+  * Move some code out of NetSessions
+
+    - TCPStateStats update when a session is removed was moved to Connection
+    - Stepping Stone manager moved to a singleton object in SteppingStoneManager (Tim Wojtulewicz, Corelight)
+
+4.1.0-dev.557 | 2021-04-29 09:00:51 -0700
+
+  * Add basic testing for Geneve protocol analyzer
+
+    The added pcap file was downloaded from an attachment to
+    https://gitlab.com/wireshark/wireshark/-/issues/10193 without explicit
+    license. (Benjamin Bannier, Corelight)
+
+  * GH-1517: Add Geneve decap support
+
+    This patch adds the ability to decap Geneve packets to process the inner
+    payload. The structure of the analyzer borrows heavily from the VXLAN
+    analyzer. (Benjamin Bannier, Corelight)
+
+4.1.0-dev.554 | 2021-04-28 13:45:53 -0700
+
+  * Add a fatal error condition for invalid Dictionary insertion distances (Jon Siwek, Corelight)
+
+    When choosing poor/aggressive values for `table_expire_interval`,
+    `table_expire_delay`, and/or `table_incremental_step` that tend to
+    leave tables in state of constant table-expiry-iteration, the underlying
+    Dictionary is never allowed the chance to complete remapping operations
+    which re-position entries to more ideal locations (e.g. after
+    reallocating the table to be able to store more entries).
+
+    That situation not only leads to the Dictionary generally having a less
+    efficient structure, but eventually, the lack of re-positioning may
+    cause an insertion to calculate the new entry's
+    distance-from-ideal-position to be a value requiring a full 16-bits or
+    more (>=65535), but an entry only allows storing 16-bit distance values,
+    with 65535 being a sentinel value that is supposed to indicate an empty
+    entry.  Dictionary operations may start misbehaving if that's allowed to
+    happen.
+
+  * Fix using clear_table() within an &expire_func (Jon Siwek, Corelight)
+
+    This previously crashed since clear_table()/TableVal::RemoveAll() left
+    behind a stale iterator to the old table causing a heap-use-after-free
+    when resuming table expiry iteration in TableVal::DoExpire().
+
+  * Remove saving/restoring of value pointer after calling expire_func (Jon Siwek, Corelight)
+
+    It's no longer used for anything.  Previously, it was used to detect
+    whether the expiry batch finished iterating the entire table or not, but
+    that's now determined by directly checking if the iterator itself
+    signifies the end of the table.
+
+  * Avoid allocating a HashKey for no-op table expiry iterations (Jon Siwek, Corelight)
+
+4.1.0-dev.549 | 2021-04-28 13:09:30 -0700
+
+  * Fix -Wsign-compare warnings in Debug{Cmds}.cc (Jon Siwek, Corelight)
+
+4.1.0-dev.547 | 2021-04-28 09:27:15 -0700
+
+  * GH-1528: Remove broken Queue/PQueue class, replace with std::deque (Tim Wojtulewicz, Corelight)
+
+4.1.0-dev.545 | 2021-04-26 11:14:49 -0700
+
+  * GH-1483: Improve error for mismatched container initialization types (Jon Siwek, Corelight)
+
+    For example, trying to assign a vector, table, set, or record
+    constructor expression to a global variable of a different type now
+    provides a more explanatory error message than the previous
+    "Val::CONVERTER" fatal-error and coredump.
+
+4.1.0-dev.543 | 2021-04-26 09:53:53 -0700
+
+  * Update btest baselines for --enable-mobile-ipv6 builds (Tim Wojtulewicz, Corelight)
+
+  * Build ubuntu18 on CI with --enable-mobile-ipv6 (Tim Wojtulewicz, Corelight)
 
 4.1.0-dev.539 | 2021-04-20 15:21:03 -0700
 

CMakeLists.txt

@@ -539,6 +539,9 @@ if ( GooglePerftools_INCLUDE_DIR )
     set(ZEEK_CONFIG_GooglePerftools_INCLUDE_DIR ${GooglePerftools_INCLUDE_DIR})
 endif ()
 
+set(ZEEK_CONFIG_BTEST_TOOLS_DIR ${ZEEK_ROOT_DIR}/share/btest)
+install(DIRECTORY DESTINATION ${ZEEK_CONFIG_BTEST_TOOLS_DIR})
+
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/zeek-config.in
                ${CMAKE_CURRENT_BINARY_DIR}/zeek-config @ONLY)
 install(PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/zeek-config DESTINATION bin)
@@ -570,12 +573,41 @@ if ( INSTALL_ZKG )
         DESTINATION ${ZEEK_ZKG_CONFIG_DIR} RENAME config)
 endif ()
 
+########################################################################
+## Look for external plugins to build in
+
+string(REPLACE "," " " _build_in_plugins "${ZEEK_INCLUDE_PLUGINS}")
+separate_arguments(_build_in_plugins)
+foreach(plugin_dir ${_build_in_plugins})
+    if ( NOT IS_ABSOLUTE "${plugin_dir}/CMakeLists.txt" )
+      message(FATAL_ERROR "Plugins to build in need to be defined with absolute path! ${plugin_dir}")
+    endif()
+
+    if ( NOT EXISTS "${plugin_dir}/CMakeLists.txt" )
+      message(FATAL_ERROR "No plugin found at ${plugin_dir}!")
+    endif()
+
+    get_filename_component(plugin_name ${plugin_dir} NAME)
+
+    # Create a list of plugin directories that will then be added in the src/CMakeLists.txt
+    list(APPEND BUILTIN_PLUGIN_LIST ${plugin_dir})
+
+    message(STATUS "  Building in plugin: ${plugin_name} (${plugin_dir})")
+
+    if ( "${ZEEK_BUILTIN_PLUGINS}" STREQUAL "" )
+        set(ZEEK_BUILTIN_PLUGINS ${plugin_name})
+    else ()
+        set(ZEEK_BUILTIN_PLUGINS "${ZEEK_BUILTIN_PLUGINS}, ${plugin_name}")
+    endif ()
+endforeach()
+
 ########################################################################
 ## Recurse on sub-directories
 
 add_subdirectory(src)
 add_subdirectory(scripts)
 add_subdirectory(man)
+add_subdirectory(testing)
 
 include(CheckOptionalBuildSources)
 
@@ -608,6 +640,12 @@ if (CMAKE_BUILD_TYPE)
     string(TOUPPER ${CMAKE_BUILD_TYPE} BuildType)
 endif ()
 
+if ( INSTALL_BTEST_PCAPS )
+    set(_install_btest_tools_msg "all")
+else ()
+    set(_install_btest_tools_msg "no pcaps")
+endif ()
+
 message(
     "\n====================|  Zeek Build Summary  |===================="
     "\n"
@@ -619,6 +657,7 @@ message(
     "\nZeek Script Path:  ${ZEEK_SCRIPT_INSTALL_PATH}"
     "\nDebug mode:        ${ENABLE_DEBUG}"
     "\nUnit tests:        ${ENABLE_ZEEK_UNIT_TESTS}"
+    "\nBuiltin Plugins:   ${ZEEK_BUILTIN_PLUGINS}"
     "\n"
     "\nCC:                ${CMAKE_C_COMPILER}"
     "\nCFLAGS:            ${CMAKE_C_FLAGS} ${CMAKE_C_FLAGS_${BuildType}}"
@@ -629,6 +668,7 @@ message(
     "\nZeekControl:       ${INSTALL_ZEEKCTL}"
     "\nAux. Tools:        ${INSTALL_AUX_TOOLS}"
     "\nBTest:             ${INSTALL_BTEST}"
+    "\nBTest tooling:     ${_install_btest_tools_msg}"
     "\nzkg:               ${INSTALL_ZKG}"
     "\n"
     "\nlibmaxminddb:      ${USE_GEOIP}"

NEWS

@@ -49,9 +49,32 @@ New Functionality
 - A Telemetry API was added to assist in gathering arbitrary runtime metrics
   and allows potential export to Prometheus.
 
+- Experimental support for translating Zeek scripts to equivalent C++.
+  The generated C++ can then be compiled directly into the `zeek` binary,
+  replacing use of the interpreter and producing better runtime performance.
+  See `src/script_opt/CPP/README.md` for a guide on how to use this feature.
+
+- Support for more generic session management. The NetSessions class has been
+  renamed to SessionMgr (with the old name marked deprecated). The new
+  class allows plugins to take advantage of session management similar to how
+  Connection objects were handled previously, but without the need to be based
+  on IP-based protocols.
+
+- The ASCII writer gained a new option LogAscii::logdir, which can be used to
+  change the logging output directory.
+
+- Added a ``--include-plugins`` argument to ``configure``. This argument
+  takes a semicolon separated list of paths containing plugins that will be
+  statically built into Zeek.
+
 Changed Functionality
 ---------------------
 
+- The default IP-based transport protocols (UDP, TCP, and ICMP) have been
+  moved to the packet analysis framework. This change allows us to move other
+  analyzers in the future that better align with the packet analysis framework
+  than they do with session analysis.
+
 Removed Functionality
 ---------------------
 

VERSION

@@ -1 +1 @@
-4.1.0-dev.539
+4.1.0-dev.722

auxil/bifcl

@@ -1 +1 @@
-Subproject commit 919416a0a3457c95f23b67b39ce8e4da0b84c00b
+Subproject commit 0bb0ebe8e056871a03cf965351c2b56ed8507de1

auxil/broker

@@ -1 +1 @@
-Subproject commit 6ba5a134b0a6c29997bda652ca3a8b5b4da7cd9a
+Subproject commit ae97a637abcc763f09fb9dc984588c2f602ea954

auxil/btest

@@ -1 +1 @@
-Subproject commit 72b8e3a6a5863715cbbce873cd8e1bf5358086ab
+Subproject commit 1be682d0744f201551ada8dc568820c5f91a049c

auxil/package-manager

@@ -1 +1 @@
-Subproject commit ccdd3f305cb56040e212dab9ec9239a4bd0cc62b
+Subproject commit 8d9e5a10c45891d37e35f265fbe1522a5cad1236

auxil/zeek-archiver

@@ -1 +1 @@
-Subproject commit 7fe80115aa159119febc623cec0660774bcae50c
+Subproject commit 82f1938baa2350f18e832d9cd369c416ba843b21

ci/fedora-34/Dockerfile

@@ -0,0 +1,23 @@
+FROM fedora:34
+
+RUN dnf -y install \
+    bison \
+    cmake \
+    diffutils \
+    findutils \
+    flex \
+    git \
+    gcc \
+    gcc-c++ \
+    libpcap-devel \
+    make \
+    openssl-devel \
+    python3-devel \
+    python3-pip\
+    sqlite \
+    swig \
+    which \
+    zlib-devel \
+  && dnf clean all && rm -rf /var/cache/dnf
+
+RUN pip3 install junit2html

ci/macos/prepare.sh

@@ -5,4 +5,5 @@ sysctl hw.model hw.machine hw.ncpu hw.physicalcpu hw.logicalcpu
 set -e
 set -x
 
-brew install cmake swig openssl bison
+brew upgrade cmake openssl
+brew install swig bison

ci/opensuse-leap-15.3/Dockerfile

@@ -0,0 +1,25 @@
+FROM opensuse/leap:15.3
+
+RUN zypper in -y \
+  cmake \
+  make \
+  gcc \
+  gcc-c++ \
+  python3 \
+  python3-devel \
+  flex \
+  bison \
+  libpcap-devel \
+  libopenssl-devel \
+  zlib-devel \
+  swig \
+  git \
+  curl \
+  python3-pip \
+  which \
+  gzip \
+  tar \
+  && rm -rf /var/cache/zypp
+
+
+RUN pip3 install junit2html

configure

@@ -13,21 +13,23 @@ usage="\
 Usage: $0 [OPTION]... [VAR=VALUE]...
 
   Build Options:
-    --cmake=PATH           custom path to a CMake binary
+    --cmake=PATH                custom path to a CMake binary
-    --builddir=DIR         place build files in directory [build]
+    --builddir=DIR              place build files in directory [build]
-    --build-dir=DIR        alias for --builddir
+    --build-dir=DIR             alias for --builddir
-    --build-type=TYPE      set CMake build type [RelWithDebInfo]:
+    --build-type=TYPE           set CMake build type [RelWithDebInfo]:
-                             - Debug: optimizations off, debug symbols + flags
+                                  - Debug: optimizations off, debug symbols + flags
-                             - MinSizeRel: size optimizations, debugging off
+                                  - MinSizeRel: size optimizations, debugging off
-                             - Release: optimizations on, debugging off
+                                  - Release: optimizations on, debugging off
-                             - RelWithDebInfo: optimizations on,
+                                  - RelWithDebInfo: optimizations on,
-                                   debug symbols on, debug flags off
+                                        debug symbols on, debug flags off
-    --generator=GENERATOR  CMake generator to use (see cmake --help)
+    --generator=GENERATOR       CMake generator to use (see cmake --help)
-    --ccache               use ccache to speed up recompilation (requires
+    --ccache                    use ccache to speed up recompilation (requires
-                           ccache installation and CMake 3.10+)
+                                ccache installation and CMake 3.10+)
-    --toolchain=PATH       path to a CMAKE_TOOLCHAIN_FILE
+    --toolchain=PATH            path to a CMAKE_TOOLCHAIN_FILE
-                           (useful for cross-compiling)
+                                (useful for cross-compiling)
-    --sanitizers=LIST      comma-separated list of sanitizer names to enable
+    --sanitizers=LIST           comma-separated list of sanitizer names to enable
+    --include-plugins=PATHS     paths containing plugins to build directly into Zeek
+                                (semicolon delimited and quoted when multiple)
 
   Installation Directories:
     --prefix=PREFIX        installation directory [/usr/local/zeek]
@@ -66,6 +68,7 @@ Usage: $0 [OPTION]... [VAR=VALUE]...
     --disable-auxtools     don't build or install auxiliary tools
     --disable-archiver     don't build or install zeek-archiver tool
     --disable-btest        don't install BTest
+    --disable-btest-pcaps  don't install Zeek's BTest input pcaps
     --disable-python       don't try to build python bindings for Broker
     --disable-broker-tests don't try to build Broker unit tests
     --disable-zkg          don't install zkg
@@ -162,12 +165,14 @@ append_cache_entry ENABLE_JEMALLOC      BOOL   false
 append_cache_entry BUILD_SHARED_LIBS    BOOL   true
 append_cache_entry INSTALL_AUX_TOOLS    BOOL   true
 append_cache_entry INSTALL_BTEST        BOOL   true
+append_cache_entry INSTALL_BTEST_PCAPS  BOOL   true
 append_cache_entry INSTALL_ZEEK_ARCHIVER BOOL  true
 append_cache_entry INSTALL_ZEEKCTL      BOOL   true
 append_cache_entry INSTALL_ZKG          BOOL   true
 append_cache_entry CPACK_SOURCE_IGNORE_FILES STRING
 append_cache_entry ENABLE_MOBILE_IPV6   BOOL   false
 append_cache_entry ZEEK_SANITIZERS      STRING ""
+append_cache_entry ZEEK_INCLUDE_PLUGINS STRING ""
 
 # parse arguments
 while [ $# -ne 0 ]; do
@@ -206,6 +211,9 @@ while [ $# -ne 0 ]; do
         --toolchain=*)
             append_cache_entry CMAKE_TOOLCHAIN_FILE  PATH   $optarg
             ;;
+        --include-plugins=*)
+            append_cache_entry ZEEK_INCLUDE_PLUGINS  STRING   $optarg
+            ;;
         --prefix=*)
             prefix=$optarg
             append_cache_entry CMAKE_INSTALL_PREFIX PATH   $optarg
@@ -293,6 +301,9 @@ while [ $# -ne 0 ]; do
         --disable-btest)
             append_cache_entry INSTALL_BTEST BOOL false
             ;;
+        --disable-btest-pcaps)
+            append_cache_entry INSTALL_BTEST_PCAPS BOOL false
+            ;;
         --disable-python)
             append_cache_entry DISABLE_PYTHON_BINDINGS     BOOL   true
             ;;

doc

@@ -1 +1 @@
-Subproject commit d273f2be33866ce8240bd83632def252aca69d1b
+Subproject commit 0c08d66af3b782d532f760bd1c28941aaa081921

man/zeek.8

@@ -16,6 +16,8 @@ tasks, including detecting malware by interfacing to external registries,
 reporting vulnerable versions of software seen on the network, identifying
 popular web applications, detecting SSH brute-forcing, validating SSL
 certificate chains, among others.
+
+You must have the necessary permissions to access to the files or interfaces specified.
 .SH OPTIONS
 .TP
 .B <file>
@@ -148,6 +150,31 @@ Output file for script execution statistics
 .TP
 .B ZEEK_DISABLE_ZEEKYGEN
 Disable Zeekygen (Broxygen) documentation support
+.SH OUTPUT FORMAT
+Output is written in multiple files depending on configuration. The default
+location is the current directory.
+
+The output written by Zeek can be formatted in multiple ways using the
+logging framework.
+.PP
+The default are files in human-readable (ASCII) format. The data is organized
+into columns (tab-delimited). The data can be processed using, e.g., the \fBzeek-cut\fR tool.
+
+
+.SH EXAMPLES
+Read a capture file and generate the default logs:
+.br
+    # zeek -r test-capture.pcap
+.PP
+When running on live traffic, Zeek is usually started by running \fBzeekctl\fR. To configure
+Zeek with an initial configuration, install, and restart:
+.br
+    # zeekctl deploy
+
+Note: the zeekctl configuration may need to be updated before first use. Especially the
+network interface used should be the correct one.
+.SH SEE ALSO
+zeekctl(8) zeek-cut(1)
 .SH AUTHOR
 .B zeek
 was written by The Zeek Project <info@zeek.org>.

scripts/base/frameworks/broker/main.zeek

@@ -31,7 +31,7 @@ export {
 	## authenticated.
 	const disable_ssl = F &redef;
 
-	## Path to a file containing concatenated trusted certificates 
+	## Path to a file containing concatenated trusted certificates
 	## in PEM format. If set, Zeek will require valid certificates for
 	## all peers.
 	const ssl_cafile = "" &redef;
@@ -122,6 +122,37 @@ export {
 	## done reading the pcap.
 	option peer_counts_as_iosource = T;
 
+	## Port for Broker's metric exporter. Setting this to a valid TCP port causes
+	## Broker to make metrics available to Prometheus scrapers via HTTP. Zeek
+	## overrides any value provided in zeek_init or earlier at startup if the
+	## environment variable BROKER_METRICS_PORT is defined.
+	const metrics_port = 0/unknown &redef;
+
+	## Frequency for publishing scraped metrics to the target topic. Zeek
+	## overrides any value provided in zeek_init or earlier at startup if the
+	## environment variable BROKER_METRICS_EXPORT_INTERVAL is defined.
+	option metrics_export_interval = 1 sec;
+
+	## Target topic for the metrics. Setting a non-empty string starts the
+	## periodic publishing of local metrics. Zeek overrides any value provided in
+	## zeek_init or earlier at startup if the environment variable
+	## BROKER_METRICS_EXPORT_TOPIC is defined.
+	option metrics_export_topic = "";
+
+	## ID for the metrics exporter. When setting a target topic for the
+	## exporter, Broker sets this option to the suffix of the new topic *unless*
+	## the ID is a non-empty string. Since setting a topic starts the periodic
+	## publishing of events, we recommend setting the ID always first or avoid
+	## setting it at all if the topic suffix serves as a good-enough ID. Zeek
+	## overrides any value provided in zeek_init or earlier at startup if the
+	## environment variable BROKER_METRICS_ENDPOINT_NAME is defined.
+	option metrics_export_endpoint_name = "";
+
+	## Selects prefixes from the local metrics. Only metrics with prefixes
+	## listed in this variable are included when publishing local metrics.
+	## Setting an empty vector selects *all* metrics.
+	option metrics_export_prefixes: vector of string = vector();
+
 	## The default topic prefix where logs will be published.  The log's stream
 	## id is appended when writing to a particular stream.
 	const default_log_topic_prefix = "zeek/logs/" &redef;
@@ -385,9 +416,53 @@ event Broker::log_flush() &priority=10
 	schedule Broker::log_batch_interval { Broker::log_flush() };
 	}
 
+function update_metrics_export_interval(id: string, val: interval): interval
+	{
+	Broker::__set_metrics_export_interval(val);
+	return val;
+	}
+
+function update_metrics_export_topic(id: string, val: string): string
+	{
+	Broker::__set_metrics_export_topic(val);
+	return val;
+	}
+
+function update_metrics_export_endpoint_name(id: string, val: string): string
+	{
+	Broker::__set_metrics_export_endpoint_name(val);
+	return val;
+	}
+
+function update_metrics_export_prefixes(id: string, filter: vector of string): vector of string
+	{
+	Broker::__set_metrics_export_prefixes(filter);
+	return filter;
+	}
+
 event zeek_init()
 	{
 	schedule Broker::log_batch_interval { Broker::log_flush() };
+	# interval
+	update_metrics_export_interval("Broker::metrics_export_interval",
+	                               Broker::metrics_export_interval);
+	Option::set_change_handler("Broker::metrics_export_interval",
+	                           update_metrics_export_interval);
+	# topic
+	update_metrics_export_topic("Broker::metrics_export_topic",
+	                            Broker::metrics_export_topic);
+	Option::set_change_handler("Broker::metrics_export_topic",
+	                           update_metrics_export_topic);
+	# endpoint name
+	update_metrics_export_endpoint_name("Broker::metrics_export_endpoint_name",
+	                                    Broker::metrics_export_endpoint_name);
+	Option::set_change_handler("Broker::metrics_export_endpoint_name",
+	                           update_metrics_export_endpoint_name);
+	# prefixes
+	update_metrics_export_prefixes("Broker::metrics_export_prefixes",
+	                               Broker::metrics_export_prefixes);
+	Option::set_change_handler("Broker::metrics_export_prefixes",
+	                           update_metrics_export_prefixes);
 	}
 
 event retry_listen(a: string, p: port, retry: interval)

scripts/base/frameworks/logging/writers/ascii.zeek

@@ -54,6 +54,11 @@ export {
 	## This option is also available as a per-filter ``$config`` option.
 	const gzip_file_extension = "gz" &redef;
 
+	## Define the default logging directory. If empty, logs are written
+	## to the current working directory.
+	##
+	const logdir = "" &redef;
+
 	## Format of timestamps when writing out JSON. By default, the JSON
 	## formatter will use double values for timestamps which represent the
 	## number of seconds from the UNIX epoch.

scripts/base/frameworks/tunnels/main.zeek

@@ -93,7 +93,7 @@ export {
 const ayiya_ports = { 5072/udp };
 const teredo_ports = { 3544/udp };
 const gtpv1_ports = { 2152/udp, 2123/udp };
-redef likely_server_ports += { ayiya_ports, teredo_ports, gtpv1_ports, vxlan_ports };
+redef likely_server_ports += { ayiya_ports, teredo_ports, gtpv1_ports, vxlan_ports, geneve_ports };
 
 event zeek_init() &priority=5
 	{
@@ -103,6 +103,7 @@ event zeek_init() &priority=5
 	Analyzer::register_for_ports(Analyzer::ANALYZER_TEREDO, teredo_ports);
 	Analyzer::register_for_ports(Analyzer::ANALYZER_GTPV1, gtpv1_ports);
 	Analyzer::register_for_ports(Analyzer::ANALYZER_VXLAN, vxlan_ports);
+	Analyzer::register_for_ports(Analyzer::ANALYZER_GENEVE, geneve_ports);
 	}
 
 function register_all(ecv: EncapsulatingConnVector)

scripts/base/init-bare.zeek

@@ -635,7 +635,7 @@ type ProcStats: record {
 	real_time: interval;          ##< Elapsed real time since Zeek started running.
 	user_time: interval;          ##< User CPU seconds.
 	system_time: interval;        ##< System CPU seconds.
-	mem: count;                   ##< Maximum memory consumed, in KB.
+	mem: count;                   ##< Maximum memory consumed, in bytes.
 	minor_faults: count;          ##< Page faults not requiring actual I/O.
 	major_faults: count;          ##< Page faults requiring actual I/O.
 	num_swap: count;              ##< Times swapped out.
@@ -1933,6 +1933,7 @@ type gtp_delete_pdp_ctx_response_elements: record {
 @load base/frameworks/supervisor/api
 @load base/bif/supervisor.bif
 @load base/bif/packet_analysis.bif
+@load base/bif/CPP-load.bif
 
 ## Internal function.
 function add_interface(iold: string, inew: string): string
@@ -5029,6 +5030,12 @@ export {
 	## if you customize this, you may still want to manually ensure that
 	## :zeek:see:`likely_server_ports` also gets populated accordingly.
 	const vxlan_ports: set[port] = { 4789/udp } &redef;
+
+	## The set of UDP ports used for Geneve traffic.  Traffic using this
+	## UDP destination port will attempt to be decapsulated.  Note that if
+	## if you customize this, you may still want to manually ensure that
+	## :zeek:see:`likely_server_ports` also gets populated accordingly.
+	const geneve_ports: set[port] = { 6081/udp } &redef;
 } # end export
 
 module Reporter;

scripts/base/packet-protocols/__load__.zeek

@@ -15,3 +15,6 @@
 @load base/packet-protocols/gre
 @load base/packet-protocols/iptunnel
 @load base/packet-protocols/vntag
+@load base/packet-protocols/udp
+@load base/packet-protocols/tcp
+@load base/packet-protocols/icmp

scripts/base/packet-protocols/icmp/__load__.zeek

@@ -0,0 +1 @@
+@load ./main

scripts/base/packet-protocols/icmp/main.zeek

@@ -0,0 +1,5 @@
+module PacketAnalyzer::ICMP;
+
+#event zeek_init() &priority=20
+#	{
+#	}

scripts/base/packet-protocols/ip/main.zeek

@@ -1,8 +1,22 @@
 module PacketAnalyzer::IP;
 
+const IPPROTO_TCP : count = 6;
+const IPPROTO_UDP : count = 17;
+const IPPROTO_ICMP : count = 1;
+const IPPROTO_ICMP6 : count = 58;
+
+const IPPROTO_IPIP : count = 4;
+const IPPROTO_IPV6 : count = 41;
+const IPPROTO_GRE : count = 47;
+
 event zeek_init() &priority=20
 	{
-	PacketAnalyzer::register_packet_analyzer(PacketAnalyzer::ANALYZER_IP, 4, PacketAnalyzer::ANALYZER_IPTUNNEL);
+	PacketAnalyzer::register_packet_analyzer(PacketAnalyzer::ANALYZER_IP, IPPROTO_IPIP, PacketAnalyzer::ANALYZER_IPTUNNEL);
-	PacketAnalyzer::register_packet_analyzer(PacketAnalyzer::ANALYZER_IP, 41, PacketAnalyzer::ANALYZER_IPTUNNEL);
+	PacketAnalyzer::register_packet_analyzer(PacketAnalyzer::ANALYZER_IP, IPPROTO_IPV6, PacketAnalyzer::ANALYZER_IPTUNNEL);
-	PacketAnalyzer::register_packet_analyzer(PacketAnalyzer::ANALYZER_IP, 47, PacketAnalyzer::ANALYZER_GRE);
+	PacketAnalyzer::register_packet_analyzer(PacketAnalyzer::ANALYZER_IP, IPPROTO_GRE, PacketAnalyzer::ANALYZER_GRE);
+
+	PacketAnalyzer::register_packet_analyzer(PacketAnalyzer::ANALYZER_IP, IPPROTO_TCP, PacketAnalyzer::ANALYZER_TCP);
+	PacketAnalyzer::register_packet_analyzer(PacketAnalyzer::ANALYZER_IP, IPPROTO_UDP, PacketAnalyzer::ANALYZER_UDP);
+	PacketAnalyzer::register_packet_analyzer(PacketAnalyzer::ANALYZER_IP, IPPROTO_ICMP, PacketAnalyzer::ANALYZER_ICMP);
+	PacketAnalyzer::register_packet_analyzer(PacketAnalyzer::ANALYZER_IP, IPPROTO_ICMP6, PacketAnalyzer::ANALYZER_ICMP);
 	}

scripts/base/packet-protocols/tcp/__load__.zeek

@@ -0,0 +1 @@
+@load ./main

scripts/base/packet-protocols/tcp/main.zeek

@@ -0,0 +1,5 @@
+module PacketAnalyzer::TCP;
+
+#event zeek_init() &priority=20
+#	{
+#	}

scripts/base/packet-protocols/udp/__load__.zeek

@@ -0,0 +1 @@
+@load ./main

scripts/base/packet-protocols/udp/main.zeek

@@ -0,0 +1,5 @@
+module PacketAnalyzer::UDP;
+
+#event zeek_init() &priority=20
+#	{
+#	}

scripts/base/protocols/dns/main.zeek

@@ -434,7 +434,11 @@ event dns_request(c: connection, msg: dns_msg, query: string, qtype: count, qcla
 	#       worked into the query/response in some fashion.
 	if ( c$id$resp_p == 137/udp )
 		{
-		query = decode_netbios_name(query);
+		local decoded_query = decode_netbios_name(query);
+
+		if ( |decoded_query| != 0 )
+			query = decoded_query;
+
 		if ( c$dns$qtype_name == "SRV" )
 			{
 			# The SRV RFC used the ID used for NetBios Status RRs.

scripts/base/protocols/ssl/main.zeek

@@ -111,8 +111,8 @@ export {
 	## record as it is sent on to the logging framework.
 	global log_ssl: event(rec: Info);
 
-	# Hook that can be used to perform actions right before the log record
+	## Hook that can be used to perform actions right before the log record
-	# is written.
+	## is written.
 	global ssl_finishing: hook(c: connection);
 
 	## SSL finalization hook.  Remaining SSL info may get logged when it's called.

scripts/policy/frameworks/files/detect-MHR.zeek

@@ -26,7 +26,7 @@ export {
 	## The Match notice has a sub message with a URL where you can get more
 	## information about the file. The %s will be replaced with the SHA-1
 	## hash of the file.
-	option match_sub_url = "https://www.virustotal.com/en/search/?query=%s";
+	option match_sub_url = "https://www.virustotal.com/gui/search/%s";
 
 	## The malware hash registry runs each malware sample through several
 	## A/V engines.  Team Cymru returns a percentage to indicate how

src/CMakeLists.txt

@@ -124,6 +124,9 @@ set(BIF_SRCS
     # it's needed before parsing the packet protocol scripts, which happen
     # very near to the start of parsing.
     packet_analysis/packet_analysis.bif
+    # The C++ loading BIF is treated like other top-level BIFs to give
+    # us flexibility regarding when it's called.
+    script_opt/CPP/CPP-load.bif
 )
 
 foreach (bift ${BIF_SRCS})
@@ -166,9 +169,46 @@ add_subdirectory(input)
 add_subdirectory(iosource)
 add_subdirectory(logging)
 add_subdirectory(probabilistic)
+add_subdirectory(session)
 
 add_subdirectory(fuzzers)
 
+########################################################################
+## Build in the discovered external plugins and create the autogenerated scripts.
+
+set(PRELOAD_SCRIPT ${CMAKE_BINARY_DIR}/scripts/builtin-plugins/__preload__.zeek)
+file(WRITE ${PRELOAD_SCRIPT} "# Warning, this is an autogenerated file!\n")
+set(LOAD_SCRIPT ${CMAKE_BINARY_DIR}/scripts/builtin-plugins/__load__.zeek)
+file(WRITE ${LOAD_SCRIPT} "# Warning, this is an autogenerated file!\n")
+
+foreach (plugin_dir ${BUILTIN_PLUGIN_LIST})
+    get_filename_component(plugin_name ${plugin_dir} NAME)
+
+    if(IS_DIRECTORY "${plugin_dir}/cmake")
+        set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${plugin_dir}/cmake")
+    endif()
+
+    # Setup the include path for built source artifacts.
+    include_directories(AFTER
+                        ${plugin_dir}/src
+                        ${CMAKE_CURRENT_BINARY_DIR}/builtin-plugins/${plugin_name})
+
+    add_subdirectory(${plugin_dir} ${CMAKE_CURRENT_BINARY_DIR}/builtin-plugins/${plugin_name})
+
+    set(preload_script ${plugin_dir}/scripts/__preload__.zeek)
+    if (EXISTS ${plugin_dir}/scripts/__preload__.zeek)
+        file(APPEND ${CMAKE_BINARY_DIR}/scripts/builtin-plugins/__preload__.zeek "\n@load ${preload_script}")
+    endif()
+    set(load_script ${plugin_dir}/scripts/__load__.zeek)
+    if (EXISTS ${plugin_dir}/scripts/__load__.zeek)
+        file(APPEND ${CMAKE_BINARY_DIR}/scripts/builtin-plugins/__load__.zeek "\n@load ${load_script}")
+    endif()
+endforeach()
+
+install(FILES ${PRELOAD_SCRIPT} DESTINATION ${ZEEK_SCRIPT_INSTALL_PATH}/builtin-plugins/)
+install(FILES ${LOAD_SCRIPT} DESTINATION ${ZEEK_SCRIPT_INSTALL_PATH}/builtin-plugins/)
+
+
 ########################################################################
 ## bro target
 
@@ -216,6 +256,10 @@ add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/DebugCmdConstants.h
                    WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
 )
 
+set(_gen_zeek_script_cpp ${CMAKE_CURRENT_BINARY_DIR}/../CPP-gen.cc)
+add_custom_command(OUTPUT ${_gen_zeek_script_cpp}
+		    COMMAND ${CMAKE_COMMAND} -E touch ${_gen_zeek_script_cpp})
+
 set_source_files_properties(nb_dns.c PROPERTIES COMPILE_FLAGS
                             -fno-strict-aliasing)
 
@@ -286,7 +330,6 @@ set(MAIN_SRCS
     Scope.cc
     ScriptCoverageManager.cc
     SerializationFormat.cc
-    Sessions.cc
     SmithWaterman.cc
     Stats.cc
     Stmt.cc
@@ -327,6 +370,27 @@ set(MAIN_SRCS
     plugin/Manager.cc
     plugin/Plugin.cc
 
+    script_opt/CPP/Attrs.cc
+    script_opt/CPP/Consts.cc
+    script_opt/CPP/DeclFunc.cc
+    script_opt/CPP/Driver.cc
+    script_opt/CPP/Emit.cc
+    script_opt/CPP/Exprs.cc
+    script_opt/CPP/Func.cc
+    script_opt/CPP/GenFunc.cc
+    script_opt/CPP/HashMgr.cc
+    script_opt/CPP/Inits.cc
+    script_opt/CPP/RuntimeInit.cc
+    script_opt/CPP/RuntimeOps.cc
+    script_opt/CPP/RuntimeVec.cc
+    script_opt/CPP/Stmts.cc
+    script_opt/CPP/Tracker.cc
+    script_opt/CPP/Types.cc
+    script_opt/CPP/Util.cc
+    script_opt/CPP/Vars.cc
+
+    ${_gen_zeek_script_cpp}
+
     script_opt/DefItem.cc
     script_opt/DefSetsMgr.cc
     script_opt/Expr.cc

src/CompHash.cc

@@ -509,6 +509,9 @@ int CompositeHash::SingleTypeKeySize(Type* bt, const Val* v,
 
 		case TYPE_RECORD:
 			{
+			if ( ! v )
+				return (optional && ! calc_static_size) ? sz : 0;
+
 			const RecordVal* rv = v ? v->AsRecordVal() : nullptr;
 			RecordType* rt = bt->AsRecordType();
 			int num_fields = rt->NumFields();

src/Conn.cc

@@ -10,7 +10,7 @@
 #include "zeek/RunState.h"
 #include "zeek/NetVar.h"
 #include "zeek/Event.h"
-#include "zeek/Sessions.h"
+#include "zeek/session/Manager.h"
 #include "zeek/Reporter.h"
 #include "zeek/Timer.h"
 #include "zeek/iosource/IOSource.h"
@@ -19,56 +19,20 @@
 #include "zeek/analyzer/Analyzer.h"
 #include "zeek/analyzer/Manager.h"
 #include "zeek/iosource/IOSource.h"
+#include "zeek/packet_analysis/protocol/ip/SessionAdapter.h"
+#include "zeek/packet_analysis/protocol/tcp/TCP.h"
 
 namespace zeek {
-namespace detail {
-
-void ConnectionTimer::Init(Connection* arg_conn, timer_func arg_timer,
-                           bool arg_do_expire)
-	{
-	conn = arg_conn;
-	timer = arg_timer;
-	do_expire = arg_do_expire;
-	Ref(conn);
-	}
-
-ConnectionTimer::~ConnectionTimer()
-	{
-	if ( conn->RefCnt() < 1 )
-		reporter->InternalError("reference count inconsistency in ~ConnectionTimer");
-
-	conn->RemoveTimer(this);
-	Unref(conn);
-	}
-
-void ConnectionTimer::Dispatch(double t, bool is_expire)
-	{
-	if ( is_expire && ! do_expire )
-		return;
-
-	// Remove ourselves from the connection's set of timers so
-	// it doesn't try to cancel us.
-	conn->RemoveTimer(this);
-
-	(conn->*timer)(t);
-
-	if ( conn->RefCnt() < 1 )
-		reporter->InternalError("reference count inconsistency in ConnectionTimer::Dispatch");
-	}
-
-} // namespace detail
 
 uint64_t Connection::total_connections = 0;
 uint64_t Connection::current_connections = 0;
 
-Connection::Connection(NetSessions* s, const detail::ConnIDKey& k, double t,
+Connection::Connection(const detail::ConnKey& k, double t,
-                       const ConnID* id, uint32_t flow, const Packet* pkt)
+                       const ConnTuple* id, uint32_t flow, const Packet* pkt)
+	: Session(t, connection_timeout, connection_status_update,
+	          detail::connection_status_update_interval),
+	  key(k)
 	{
-	sessions = s;
-	key = k;
-	key_valid = true;
-	start_time = last_time = t;
-
 	orig_addr = id->src_addr;
 	resp_addr = id->dst_addr;
 	orig_port = id->src_port;
@@ -92,25 +56,16 @@ Connection::Connection(NetSessions* s, const detail::ConnIDKey& k, double t,
 	vlan = pkt->vlan;
 	inner_vlan = pkt->inner_vlan;
 
-	is_active = 1;
-	skip = 0;
 	weird = 0;
 
 	suppress_event = 0;
 
-	record_contents = record_packets = 1;
-	record_current_packet = record_current_content = 0;
-
-	timers_canceled = 0;
-	inactivity_timeout = 0;
-	installed_status_timer = 0;
-
 	finished = 0;
 
 	hist_seen = 0;
 	history = "";
 
-	root_analyzer = nullptr;
+	adapter = nullptr;
 	primary_PIA = nullptr;
 
 	++current_connections;
@@ -129,7 +84,7 @@ Connection::~Connection()
 	if ( conn_val )
 		conn_val->SetOrigin(nullptr);
 
-	delete root_analyzer;
+	delete adapter;
 
 	--current_connections;
 	}
@@ -141,7 +96,7 @@ void Connection::CheckEncapsulation(const std::shared_ptr<EncapsulationStack>& a
 		if ( *encapsulation != *arg_encap )
 			{
 			if ( tunnel_changed )
-				EnqueueEvent(tunnel_changed, nullptr, ConnVal(),
+				EnqueueEvent(tunnel_changed, nullptr, GetVal(),
 				             arg_encap->ToVal());
 
 			encapsulation = std::make_shared<EncapsulationStack>(*arg_encap);
@@ -153,7 +108,7 @@ void Connection::CheckEncapsulation(const std::shared_ptr<EncapsulationStack>& a
 		if ( tunnel_changed )
 			{
 			EncapsulationStack empty;
-			EnqueueEvent(tunnel_changed, nullptr, ConnVal(), empty.ToVal());
+			EnqueueEvent(tunnel_changed, nullptr, GetVal(), empty.ToVal());
 			}
 
 		encapsulation = nullptr;
@@ -162,7 +117,7 @@ void Connection::CheckEncapsulation(const std::shared_ptr<EncapsulationStack>& a
 	else if ( arg_encap )
 		{
 		if ( tunnel_changed )
-			EnqueueEvent(tunnel_changed, nullptr, ConnVal(), arg_encap->ToVal());
+			EnqueueEvent(tunnel_changed, nullptr, GetVal(), arg_encap->ToVal());
 
 		encapsulation = std::make_shared<EncapsulationStack>(*arg_encap);
 		}
@@ -172,8 +127,21 @@ void Connection::Done()
 	{
 	finished = 1;
 
-	if ( root_analyzer && ! root_analyzer->IsFinished() )
+	if ( adapter )
-		root_analyzer->Done();
+		{
+		if ( ConnTransport() == TRANSPORT_TCP )
+			{
+			auto* ta = static_cast<packet_analysis::TCP::TCPSessionAdapter*>(adapter);
+			assert(ta->IsAnalyzer("TCP"));
+			analyzer::tcp::TCP_Endpoint* to = ta->Orig();
+			analyzer::tcp::TCP_Endpoint* tr = ta->Resp();
+
+			packet_analysis::TCP::TCPAnalyzer::GetStats().StateLeft(to->state, tr->state);
+			}
+
+		if ( ! adapter->IsFinished() )
+			adapter->Done();
+		}
 	}
 
 void Connection::NextPacket(double t, bool is_orig,
@@ -186,14 +154,14 @@ void Connection::NextPacket(double t, bool is_orig,
 	run_state::current_timestamp = t;
 	run_state::current_pkt = pkt;
 
-	if ( Skipping() )
+	if ( adapter )
-		return;
-
-	if ( root_analyzer )
 		{
+		if ( adapter->Skipping() )
+			return;
+
 		record_current_packet = record_packet;
 		record_current_content = record_content;
-		root_analyzer->NextPacket(len, data, is_orig, -1, ip, caplen);
+		adapter->NextPacket(len, data, is_orig, -1, ip, caplen);
 		record_packet = record_current_packet;
 		record_content = record_current_content;
 		}
@@ -204,15 +172,9 @@ void Connection::NextPacket(double t, bool is_orig,
 	run_state::current_pkt = nullptr;
 	}
 
-void Connection::SetLifetime(double lifetime)
-	{
-	ADD_TIMER(&Connection::DeleteTimer, run_state::network_time + lifetime, 0,
-	          detail::TIMER_CONN_DELETE);
-	}
-
 bool Connection::IsReuse(double t, const u_char* pkt)
 	{
-	return root_analyzer && root_analyzer->IsReuse(t, pkt);
+	return adapter && adapter->IsReuse(t, pkt);
 	}
 
 bool Connection::ScaledHistoryEntry(char code, uint32_t& counter,
@@ -252,83 +214,13 @@ void Connection::HistoryThresholdEvent(EventHandlerPtr e, bool is_orig,
 		return;
 
 	EnqueueEvent(e, nullptr,
-		ConnVal(),
+		GetVal(),
 		val_mgr->Bool(is_orig),
 		val_mgr->Count(threshold)
 	);
 	}
 
-void Connection::DeleteTimer(double /* t */)
+const RecordValPtr& Connection::GetVal()
-	{
-	if ( is_active )
-		Event(connection_timeout, nullptr);
-
-	sessions->Remove(this);
-	}
-
-void Connection::InactivityTimer(double t)
-	{
-	if ( last_time + inactivity_timeout <= t )
-		{
-		Event(connection_timeout, nullptr);
-		sessions->Remove(this);
-		++detail::killed_by_inactivity;
-		}
-	else
-		ADD_TIMER(&Connection::InactivityTimer,
-		          last_time + inactivity_timeout, 0,
-		          detail::TIMER_CONN_INACTIVITY);
-	}
-
-void Connection::RemoveConnectionTimer(double t)
-	{
-	RemovalEvent();
-	sessions->Remove(this);
-	}
-
-void Connection::SetInactivityTimeout(double timeout)
-	{
-	if ( timeout == inactivity_timeout )
-		return;
-
-	// First cancel and remove any existing inactivity timer.
-	for ( const auto& timer : timers )
-		if ( timer->Type() == detail::TIMER_CONN_INACTIVITY )
-			{
-			detail::timer_mgr->Cancel(timer);
-			break;
-			}
-
-	if ( timeout )
-		ADD_TIMER(&Connection::InactivityTimer,
-		          last_time + timeout, 0, detail::TIMER_CONN_INACTIVITY);
-
-	inactivity_timeout = timeout;
-	}
-
-void Connection::EnableStatusUpdateTimer()
-	{
-	if ( installed_status_timer )
-		return;
-
-	if ( connection_status_update && zeek::detail::connection_status_update_interval )
-		{
-		ADD_TIMER(&Connection::StatusUpdateTimer,
-		          run_state::network_time + detail::connection_status_update_interval, 0,
-		          detail::TIMER_CONN_STATUS_UPDATE);
-		installed_status_timer = 1;
-		}
-	}
-
-void Connection::StatusUpdateTimer(double t)
-	{
-	EnqueueEvent(connection_status_update, nullptr, ConnVal());
-	ADD_TIMER(&Connection::StatusUpdateTimer,
-	          run_state::network_time + detail::connection_status_update_interval, 0,
-	          detail::TIMER_CONN_STATUS_UPDATE);
-	}
-
-const RecordValPtr& Connection::ConnVal()
 	{
 	if ( ! conn_val )
 		{
@@ -384,8 +276,8 @@ const RecordValPtr& Connection::ConnVal()
 
 		}
 
-	if ( root_analyzer )
+	if ( adapter )
-		root_analyzer->UpdateConnVal(conn_val.get());
+		adapter->UpdateConnVal(conn_val.get());
 
 	conn_val->AssignTime(3, start_time);	// ###
 	conn_val->AssignInterval(4, last_time - start_time);
@@ -398,22 +290,22 @@ const RecordValPtr& Connection::ConnVal()
 
 analyzer::Analyzer* Connection::FindAnalyzer(analyzer::ID id)
 	{
-	return root_analyzer ? root_analyzer->FindChild(id) : nullptr;
+	return adapter ? adapter->FindChild(id) : nullptr;
 	}
 
 analyzer::Analyzer* Connection::FindAnalyzer(const analyzer::Tag& tag)
 	{
-	return root_analyzer ? root_analyzer->FindChild(tag) : nullptr;
+	return adapter ? adapter->FindChild(tag) : nullptr;
 	}
 
 analyzer::Analyzer* Connection::FindAnalyzer(const char* name)
 	{
-	return root_analyzer->FindChild(name);
+	return adapter->FindChild(name);
 	}
 
 void Connection::AppendAddl(const char* str)
 	{
-	const auto& cv = ConnVal();
+	const auto& cv = GetVal();
 
 	const char* old = cv->GetFieldAs<StringVal>(6)->CheckString();
 	const char* format = *old ? "%s %s" : "%s%s";
@@ -444,25 +336,7 @@ void Connection::Match(detail::Rule::PatternType type, const u_char* data, int l
 void Connection::RemovalEvent()
 	{
 	if ( connection_state_remove )
-		EnqueueEvent(connection_state_remove, nullptr, ConnVal());
+		EnqueueEvent(connection_state_remove, nullptr, GetVal());
-	}
-
-void Connection::Event(EventHandlerPtr f, analyzer::Analyzer* analyzer, const char* name)
-	{
-	if ( ! f )
-		return;
-
-	if ( name )
-		EnqueueEvent(f, analyzer, make_intrusive<StringVal>(name), ConnVal());
-	else
-		EnqueueEvent(f, analyzer, ConnVal());
-	}
-
-void Connection::EnqueueEvent(EventHandlerPtr f, analyzer::Analyzer* a,
-                              Args args)
-	{
-	// "this" is passed as a cookie for the event
-	event_mgr.Enqueue(f, std::move(args), util::detail::SOURCE_LOCAL, a ? a->GetID() : 0, this);
 	}
 
 void Connection::Weird(const char* name, const char* addl, const char* source)
@@ -471,44 +345,6 @@ void Connection::Weird(const char* name, const char* addl, const char* source)
 	reporter->Weird(this, name, addl ? addl : "", source ? source : "");
 	}
 
-void Connection::AddTimer(timer_func timer, double t, bool do_expire,
-                          detail::TimerType type)
-	{
-	if ( timers_canceled )
-		return;
-
-	// If the key is cleared, the connection isn't stored in the connection
-	// table anymore and will soon be deleted. We're not installing new
-	// timers anymore then.
-	if ( ! key_valid )
-		return;
-
-	detail::Timer* conn_timer = new detail::ConnectionTimer(this, timer, t, do_expire, type);
-	detail::timer_mgr->Add(conn_timer);
-	timers.push_back(conn_timer);
-	}
-
-void Connection::RemoveTimer(detail::Timer* t)
-	{
-	timers.remove(t);
-	}
-
-void Connection::CancelTimers()
-	{
-	// We are going to cancel our timers which, in turn, may cause them to
-	// call RemoveTimer(), which would then modify the list we're just
-	// traversing. Thus, we first make a copy of the list which we then
-	// iterate through.
-	TimerPList tmp(timers.length());
-	std::copy(timers.begin(), timers.end(), std::back_inserter(tmp));
-
-	for ( const auto& timer : tmp )
-		detail::timer_mgr->Cancel(timer);
-
-	timers_canceled = 1;
-	timers.clear();
-	}
-
 void Connection::FlipRoles()
 	{
 	IPAddr tmp_addr = resp_addr;
@@ -535,8 +371,8 @@ void Connection::FlipRoles()
 
 	conn_val = nullptr;
 
-	if ( root_analyzer )
+	if ( adapter )
-		root_analyzer->FlipRoles();
+		adapter->FlipRoles();
 
 	analyzer_mgr->ApplyScheduledAnalyzers(this);
 
@@ -545,25 +381,22 @@ void Connection::FlipRoles()
 
 unsigned int Connection::MemoryAllocation() const
 	{
-	return padded_sizeof(*this)
+	return session::Session::MemoryAllocation() + padded_sizeof(*this)
 		+ (timers.MemoryAllocation() - padded_sizeof(timers))
 		+ (conn_val ? conn_val->MemoryAllocation() : 0)
-		+ (root_analyzer ? root_analyzer->MemoryAllocation(): 0)
+		+ (adapter ? adapter->MemoryAllocation(): 0)
 		// primary_PIA is already contained in the analyzer tree.
 		;
 	}
 
-unsigned int Connection::MemoryAllocationConnVal() const
+unsigned int Connection::MemoryAllocationVal() const
 	{
 	return conn_val ? conn_val->MemoryAllocation() : 0;
 	}
 
 void Connection::Describe(ODesc* d) const
 	{
-	d->Add(start_time);
+	session::Session::Describe(d);
-	d->Add("(");
-	d->Add(last_time);
-	d->AddSP(")");
 
 	switch ( proto ) {
 		case TRANSPORT_TCP:
@@ -616,10 +449,10 @@ void Connection::IDString(ODesc* d) const
 	d->Add(ntohs(resp_port));
 	}
 
-void Connection::SetRootAnalyzer(analyzer::TransportLayerAnalyzer* analyzer,
+void Connection::SetSessionAdapter(packet_analysis::IP::SessionAdapter* aa,
-                                 analyzer::pia::PIA* pia)
+                                   analyzer::pia::PIA* pia)
 	{
-	root_analyzer = analyzer;
+	adapter = aa;
 	primary_PIA = pia;
 	}
 
@@ -639,7 +472,7 @@ void Connection::CheckFlowLabel(bool is_orig, uint32_t flow_label)
 		     (is_orig ? saw_first_orig_packet : saw_first_resp_packet) )
 			{
 			EnqueueEvent(connection_flow_label_changed, nullptr,
-				ConnVal(),
+				GetVal(),
 				val_mgr->Bool(is_orig),
 				val_mgr->Count(my_flow_label),
 				val_mgr->Count(flow_label)

src/Conn.h

@@ -16,6 +16,7 @@
 #include "zeek/WeirdState.h"
 #include "zeek/ZeekArgs.h"
 #include "zeek/IntrusivePtr.h"
+#include "zeek/session/Session.h"
 #include "zeek/iosource/Packet.h"
 
 #include "zeek/analyzer/Tag.h"
@@ -24,7 +25,6 @@
 namespace zeek {
 
 class Connection;
-class NetSessions;
 class EncapsulationStack;
 class Val;
 class RecordVal;
@@ -32,21 +32,17 @@ class RecordVal;
 using ValPtr = IntrusivePtr<Val>;
 using RecordValPtr = IntrusivePtr<RecordVal>;
 
+namespace session { class Manager; }
 namespace detail {
 
-class ConnectionTimer;
 class Specific_RE_Matcher;
 class RuleEndpointState;
 class RuleHdrTest;
 
 } // namespace detail
 
-namespace analyzer {
+namespace analyzer { class Analyzer; }
-
+namespace packet_analysis::IP { class SessionAdapter; }
-class TransportLayerAnalyzer;
-class Analyzer;
-
-} // namespace analyzer
 
 enum ConnEventToFlag {
 	NUL_IN_LINE,
@@ -55,41 +51,46 @@ enum ConnEventToFlag {
 	NUM_EVENTS_TO_FLAG,
 };
 
-typedef void (Connection::*timer_func)(double t);
+struct ConnTuple {
-
-struct ConnID {
 	IPAddr src_addr;
 	IPAddr dst_addr;
 	uint32_t src_port;
 	uint32_t dst_port;
 	bool is_one_way;	// if true, don't canonicalize order
+	TransportProto proto;
 };
 
+using ConnID [[deprecated("Remove in v5.1. Use zeek::ConnTuple.")]] = ConnTuple;
+
 static inline int addr_port_canon_lt(const IPAddr& addr1, uint32_t p1,
                                      const IPAddr& addr2, uint32_t p2)
 	{
 	return addr1 < addr2 || (addr1 == addr2 && p1 < p2);
 	}
 
-class Connection final : public Obj {
+class Connection final : public session::Session {
 public:
 
-	Connection(NetSessions* s, const detail::ConnIDKey& k, double t, const ConnID* id,
+	Connection(const detail::ConnKey& k, double t, const ConnTuple* id,
 	           uint32_t flow, const Packet* pkt);
 	~Connection() override;
 
-	// Invoked when an encapsulation is discovered. It records the
+	/**
-	// encapsulation with the connection and raises a "tunnel_changed"
+	 * Invoked when an encapsulation is discovered. It records the encapsulation
-	// event if it's different from the previous encapsulation (or the
+	 * with the connection and raises a "tunnel_changed" event if it's different
-	// first encountered). encap can be null to indicate no
+	 * from the previous encapsulation or if it's the first one encountered.
-	// encapsulation.
+	 *
+	 * @param encap The new encapsulation. Can be set to null to indicated no
+	 * encapsulation or clear an old one.
+	 */
 	void CheckEncapsulation(const std::shared_ptr<EncapsulationStack>& encap);
 
-	// Invoked when connection is about to be removed.  Use Ref(this)
+	/**
-	// inside Done to keep the connection object around (though it'll
+	 * Invoked when the session is about to be removed. Use Ref(this)
-	// no longer be accessible from the dictionary of active
+	 * inside Done to keep the session object around, though it'll
-	// connections).
+	 * no longer be accessible from the SessionManager.
-	void Done();
+	 */
+	void Done() override;
 
 	// Process the connection's next packet.  "data" points just
 	// beyond the IP header.  It's updated to point just beyond
@@ -110,14 +111,12 @@ public:
 	// Keys are only considered valid for a connection when a
 	// connection is in the session map. If it is removed, the key
 	// should be marked invalid.
-	const detail::ConnIDKey& Key() const	{ return key; }
+	const detail::ConnKey& Key() const	{ return key; }
-	void ClearKey()					{ key_valid = false; }
+	session::detail::Key SessionKey(bool copy) const override
-	bool IsKeyValid() const			{ return key_valid; }
+		{
-
+		return session::detail::Key{
-	double StartTime() const		{ return start_time; }
+			&key, sizeof(key), session::detail::Key::CONNECTION_KEY_TYPE, copy};
-	void SetStartTime(double t)		{ start_time = t; }
+		}
-	double LastTime() const			{ return last_time; }
-	void SetLastTime(double t) 		{ last_time = t; }
 
 	const IPAddr& OrigAddr() const		{ return orig_addr; }
 	const IPAddr& RespAddr() const		{ return resp_addr; }
@@ -132,52 +131,31 @@ public:
 	analyzer::Analyzer* FindAnalyzer(const char* name);	// find first in tree.
 
 	TransportProto ConnTransport() const { return proto; }
-
+	std::string TransportIdentifier() const override
-	// True if we should record subsequent packets (either headers or
+		{
-	// in their entirety, depending on record_contents).  We still
+		if ( proto == TRANSPORT_TCP )
-	// record subsequent SYN/FIN/RST, regardless of how this is set.
+			return "tcp";
-	bool RecordPackets() const		{ return record_packets; }
+		else if ( proto == TRANSPORT_UDP )
-	void SetRecordPackets(bool do_record)	{ record_packets = do_record ? 1 : 0; }
+			return "udp";
-
+		else if ( proto == TRANSPORT_ICMP )
-	// True if we should record full packets for this connection,
+			return "icmp";
-	// false if we should just record headers.
+		else
-	bool RecordContents() const		{ return record_contents; }
+			return "unknown";
-	void SetRecordContents(bool do_record)	{ record_contents = do_record ? 1 : 0; }
+		}
-
-	// Set whether to record *current* packet header/full.
-	void SetRecordCurrentPacket(bool do_record)
-		{ record_current_packet = do_record ? 1 : 0; }
-	void SetRecordCurrentContent(bool do_record)
-		{ record_current_content = do_record ? 1 : 0; }
-
-	// FIXME: Now this is in Analyzer and should eventually be removed here.
-	//
-	// If true, skip processing of remainder of connection.  Note
-	// that this does not in itself imply that record_packets is false;
-	// we might want instead to process the connection off-line.
-	void SetSkip(bool do_skip)		{ skip = do_skip ? 1 : 0; }
-	bool Skipping() const			{ return skip; }
-
-	// Arrange for the connection to expire after the given amount of time.
-	void SetLifetime(double lifetime);
 
 	// Returns true if the packet reflects a reuse of this
 	// connection (i.e., not a continuation but the beginning of
 	// a new connection).
 	bool IsReuse(double t, const u_char* pkt);
 
-	// Get/set the inactivity timeout for this connection.
-	void SetInactivityTimeout(double timeout);
-	double InactivityTimeout() const	{ return inactivity_timeout; }
-
-	// Activate connection_status_update timer.
-	void EnableStatusUpdateTimer();
-
 	/**
 	 * Returns the associated "connection" record.
 	 */
-	const RecordValPtr& ConnVal();
+	const RecordValPtr& GetVal() override;
 
+	/**
+	 * Append additional entries to the history field in the connection record.
+	 */
 	void AppendAddl(const char* str);
 
 	void Match(detail::Rule::PatternType type, const u_char* data, int len,
@@ -186,36 +164,11 @@ public:
 	/**
 	 * Generates connection removal event(s).
 	 */
-	void RemovalEvent();
+	void RemovalEvent() override;
-
-	// If a handler exists for 'f', an event will be generated.  If 'name' is
-	// given that event's first argument will be it, and it's second will be
-	// the connection value.  If 'name' is null, then the event's first
-	// argument is the connection value.
-	void Event(EventHandlerPtr f, analyzer::Analyzer* analyzer, const char* name = nullptr);
-
-	/**
-	 * Enqueues an event associated with this connection and given analyzer.
-	 */
-	void EnqueueEvent(EventHandlerPtr f, analyzer::Analyzer* analyzer,
-	                  Args args);
-
-	/**
-	 * A version of EnqueueEvent() taking a variable number of arguments.
-	 */
-	template <class... Args>
-	std::enable_if_t<
-		std::is_convertible_v<
-			std::tuple_element_t<0, std::tuple<Args...>>, ValPtr>>
-	EnqueueEvent(EventHandlerPtr h, analyzer::Analyzer* analyzer, Args&&... args)
-		{ return EnqueueEvent(h, analyzer, zeek::Args{std::forward<Args>(args)...}); }
 
 	void Weird(const char* name, const char* addl = "", const char* source = "");
 	bool DidWeird() const	{ return weird != 0; }
 
-	// Cancel all associated timers.
-	void CancelTimers();
-
 	inline bool FlagEvent(ConnEventToFlag e)
 		{
 		if ( e >= 0 && e < NUM_EVENTS_TO_FLAG )
@@ -234,8 +187,8 @@ public:
 	// Statistics.
 
 	// Just a lower bound.
-	unsigned int MemoryAllocation() const;
+	unsigned int MemoryAllocation() const override;
-	unsigned int MemoryAllocationConnVal() const;
+	unsigned int MemoryAllocationVal() const override;
 
 	static uint64_t TotalConnections()
 		{ return total_connections; }
@@ -268,11 +221,9 @@ public:
 
 	void AddHistory(char code)	{ history += code; }
 
-	void DeleteTimer(double t);
-
 	// Sets the root of the analyzer tree as well as the primary PIA.
-	void SetRootAnalyzer(analyzer::TransportLayerAnalyzer* analyzer, analyzer::pia::PIA* pia);
+	void SetSessionAdapter(packet_analysis::IP::SessionAdapter* aa, analyzer::pia::PIA* pia);
-	analyzer::TransportLayerAnalyzer* GetRootAnalyzer()	{ return root_analyzer; }
+	packet_analysis::IP::SessionAdapter* GetSessionAdapter()	{ return adapter; }
 	analyzer::pia::PIA* GetPrimaryPIA()	{ return primary_PIA; }
 
 	// Sets the transport protocol in use.
@@ -293,28 +244,9 @@ public:
 	bool PermitWeird(const char* name, uint64_t threshold, uint64_t rate,
 	                 double duration);
 
-protected:
+private:
 
-	// Add the given timer to expire at time t.  If do_expire
+	friend class session::detail::Timer;
-	// is true, then the timer is also evaluated when Bro terminates,
-	// otherwise not.
-	void AddTimer(timer_func timer, double t, bool do_expire,
-	              detail::TimerType type);
-
-	void RemoveTimer(detail::Timer* t);
-
-	// Allow other classes to access pointers to these:
-	friend class detail::ConnectionTimer;
-
-	void InactivityTimer(double t);
-	void StatusUpdateTimer(double t);
-	void RemoveConnectionTimer(double t);
-
-	NetSessions* sessions;
-	detail::ConnIDKey key;
-	bool key_valid;
-
-	TimerPList timers;
 
 	IPAddr orig_addr;
 	IPAddr resp_addr;
@@ -324,59 +256,28 @@ protected:
 	uint32_t vlan, inner_vlan;	// VLAN this connection traverses, if available
 	u_char orig_l2_addr[Packet::L2_ADDR_LEN];	// Link-layer originator address, if available
 	u_char resp_l2_addr[Packet::L2_ADDR_LEN];	// Link-layer responder address, if available
-	double start_time, last_time;
+	int suppress_event;	// suppress certain events to once per conn.
-	double inactivity_timeout;
 	RecordValPtr conn_val;
 	std::shared_ptr<EncapsulationStack> encapsulation; // tunnels
-	int suppress_event;	// suppress certain events to once per conn.
 
-	unsigned int installed_status_timer:1;
+	detail::ConnKey key;
-	unsigned int timers_canceled:1;
+
-	unsigned int is_active:1;
-	unsigned int skip:1;
 	unsigned int weird:1;
 	unsigned int finished:1;
-	unsigned int record_packets:1, record_contents:1;
-	unsigned int record_current_packet:1, record_current_content:1;
 	unsigned int saw_first_orig_packet:1, saw_first_resp_packet:1;
 
-	// Count number of connections.
-	static uint64_t total_connections;
-	static uint64_t current_connections;
-
-	std::string history;
 	uint32_t hist_seen;
+	std::string history;
 
-	analyzer::TransportLayerAnalyzer* root_analyzer;
+	packet_analysis::IP::SessionAdapter* adapter;
 	analyzer::pia::PIA* primary_PIA;
 
 	UID uid;	// Globally unique connection ID.
 	detail::WeirdStateMap weird_state;
+
+	// Count number of connections.
+	static uint64_t total_connections;
+	static uint64_t current_connections;
 };
 
-namespace detail {
-
-class ConnectionTimer final : public Timer {
-public:
-	ConnectionTimer(Connection* arg_conn, timer_func arg_timer,
-	                double arg_t, bool arg_do_expire, TimerType arg_type)
-		: Timer(arg_t, arg_type)
-		{ Init(arg_conn, arg_timer, arg_do_expire); }
-	~ConnectionTimer() override;
-
-	void Dispatch(double t, bool is_expire) override;
-
-protected:
-
-	void Init(Connection* conn, timer_func timer, bool do_expire);
-
-	Connection* conn;
-	timer_func timer;
-	bool do_expire;
-};
-
-} // namespace detail
 } // namespace zeek
-
-#define ADD_TIMER(timer, t, do_expire, type) \
-	AddTimer(timer_func(timer), (t), (do_expire), (type))

src/DbgBreakpoint.cc

@@ -8,7 +8,6 @@
 
 #include "zeek/Desc.h"
 #include "zeek/ID.h"
-#include "zeek/Queue.h"
 #include "zeek/Debug.h"
 #include "zeek/Scope.h"
 #include "zeek/Frame.h"

src/Debug.h

@@ -5,9 +5,9 @@
 #include <vector>
 #include <map>
 #include <string>
+#include <deque>
 
 #include "zeek/Obj.h"
-#include "zeek/Queue.h"
 #include "zeek/StmtEnums.h"
 #include "zeek/util.h"
 
@@ -38,7 +38,7 @@ public:
 };
 
 class StmtLocMapping;
-using Filemap = PQueue<StmtLocMapping>; // mapping for a single file
+using Filemap = std::deque<StmtLocMapping*>; // mapping for a single file
 
 using BPIDMapType = std::map<int, DbgBreakpoint*>;
 using BPMapType = std::multimap<const Stmt*, DbgBreakpoint*>;

src/DebugCmds.cc

@@ -26,11 +26,10 @@
 
 using namespace std;
 
-zeek::PQueue<zeek::detail::DebugCmdInfo> zeek::detail::g_DebugCmdInfos;
-zeek::PQueue<zeek::detail::DebugCmdInfo>& g_DebugCmdInfos = zeek::detail::g_DebugCmdInfos;
-
 namespace zeek::detail {
 
+DebugCmdInfoQueue g_DebugCmdInfos;
+
 //
 // Helper routines
 //
@@ -154,7 +153,7 @@ DebugCmdInfo::DebugCmdInfo(DebugCmd arg_cmd, const char* const* arg_names,
 
 const DebugCmdInfo* get_debug_cmd_info(DebugCmd cmd)
 	{
-	if ( (int) cmd < g_DebugCmdInfos.length() )
+	if ( (int) cmd < g_DebugCmdInfos.size() )
 		return g_DebugCmdInfos[(int) cmd];
 	else
 		return nullptr;

src/DebugCmds.h

@@ -6,8 +6,7 @@
 #include <stdlib.h>
 #include <string>
 #include <vector>
-
+#include <deque>
-#include "zeek/Queue.h"
 
 // This file is generated during the build.
 #include "DebugCmdConstants.h"
@@ -45,11 +44,12 @@ protected:
 	bool repeatable;
 };
 
-extern PQueue<DebugCmdInfo> g_DebugCmdInfos;
+using DebugCmdInfoQueue = std::deque<DebugCmdInfo*>;
+extern DebugCmdInfoQueue g_DebugCmdInfos;
 
 void init_global_dbg_constants ();
 
-#define num_debug_cmds() (g_DebugCmdInfos.length())
+#define num_debug_cmds() (static_cast<int>(g_DebugCmdInfos.size()))
 
 // Looks up the info record and returns it; if cmd is not found returns 0.
 const DebugCmdInfo* get_debug_cmd_info(DebugCmd cmd);

src/Dict.cc

@@ -265,6 +265,11 @@ TEST_CASE("dict new iteration")
 		count++;
 		}
 
+	PDict<uint32_t>::iterator it;
+	it = dict.begin();
+	it = dict.end();
+	PDict<uint32_t>::iterator it2 = it;
+
 	CHECK(count == 2);
 
 	delete key;
@@ -1027,8 +1032,14 @@ int Dictionary::LookupIndex(const void* key, int key_size, detail::hash_t hash,
 		*insert_position = i;
 
 	if ( insert_distance )
+		{
 		*insert_distance = i - bucket;
 
+		if ( *insert_distance >= detail::TOO_FAR_TO_REACH )
+			reporter->FatalErrorWithCore("Dictionary (size %d) insertion distance too far: %d",
+			                             Length(), *insert_distance);
+		}
+
 	return -1;
 	}
 
@@ -1552,8 +1563,11 @@ DictIterator::DictIterator(const Dictionary* d, detail::DictEntry* begin, detail
 
 DictIterator::~DictIterator()
 	{
-	assert(dict->num_iterators > 0);
+	if ( dict )
-	dict->num_iterators--;
+		{
+		assert(dict->num_iterators > 0);
+		dict->num_iterators--;
+		}
 	}
 
 DictIterator& DictIterator::operator++()
@@ -1568,6 +1582,80 @@ DictIterator& DictIterator::operator++()
 	return *this;
 	}
 
+DictIterator::DictIterator(const DictIterator& that)
+	{
+	if ( this == &that )
+		return;
+
+	if ( dict )
+		{
+		assert(dict->num_iterators > 0);
+		dict->num_iterators--;
+		}
+
+	dict = that.dict;
+	curr = that.curr;
+	end = that.end;
+	dict->num_iterators++;
+	}
+
+DictIterator& DictIterator::operator=(const DictIterator& that)
+	{
+	if ( this == &that )
+		return *this;
+
+	if ( dict )
+		{
+		assert(dict->num_iterators > 0);
+		dict->num_iterators--;
+		}
+
+	dict = that.dict;
+	curr = that.curr;
+	end = that.end;
+	dict->num_iterators++;
+
+	return *this;
+	}
+
+DictIterator::DictIterator(DictIterator&& that)
+	{
+	if ( this == &that )
+		return;
+
+	if ( dict )
+		{
+		assert(dict->num_iterators > 0);
+		dict->num_iterators--;
+		}
+
+	dict = that.dict;
+	curr = that.curr;
+	end = that.end;
+
+	that.dict = nullptr;
+	}
+
+DictIterator& DictIterator::operator=(DictIterator&& that)
+	{
+	if ( this == &that )
+		return *this;
+
+	if ( dict )
+		{
+		assert(dict->num_iterators > 0);
+		dict->num_iterators--;
+		}
+
+	dict = that.dict;
+	curr = that.curr;
+	end = that.end;
+
+	that.dict = nullptr;
+
+	return *this;
+	}
+
 
 
 

src/Dict.h

@@ -159,8 +159,14 @@ public:
 	using difference_type = std::ptrdiff_t;
 	using iterator_category = std::forward_iterator_tag;
 
+	DictIterator() = default;
 	~DictIterator();
 
+	DictIterator(const DictIterator& that);
+	DictIterator& operator=(const DictIterator& that);
+	DictIterator(DictIterator&& that);
+	DictIterator& operator=(DictIterator&& that);
+
 	reference operator*() { return *curr; }
 	pointer operator->() { return curr; }
 
@@ -171,9 +177,9 @@ public:
 	bool operator!=( const DictIterator& that ) const { return !(*this == that); }
 
 private:
+
 	friend class Dictionary;
 
-	DictIterator() = default;
 	DictIterator(const Dictionary* d, detail::DictEntry* begin, detail::DictEntry* end);
 
 	Dictionary* dict = nullptr;

src/Expr.cc

@@ -24,6 +24,18 @@
 
 namespace zeek::detail {
 
+static bool init_tag_check(const Expr* expr, const char* name,
+                           TypeTag expect_tag, TypeTag init_tag)
+	{
+	if ( expect_tag == init_tag )
+		return true;
+
+	auto msg = util::fmt("unexpected use of %s in '%s' initialization",
+	                     name, type_name(init_tag));
+	expr->Error(msg);
+	return false;
+	}
+
 const char* expr_name(BroExprTag t)
 	{
 	static const char* expr_names[int(NUM_EXPRS)] = {
@@ -3215,6 +3227,15 @@ RecordConstructorExpr::~RecordConstructorExpr()
 
 ValPtr RecordConstructorExpr::InitVal(const zeek::Type* t, ValPtr aggr) const
 	{
+	if ( IsError() )
+		{
+		Error("bad record initializer");
+		return nullptr;
+		}
+
+	if ( ! init_tag_check(this, "record constructor", TYPE_RECORD, t->Tag()) )
+		return nullptr;
+
 	auto v = Eval(nullptr);
 
 	if ( v )
@@ -3394,6 +3415,9 @@ ValPtr TableConstructorExpr::InitVal(const zeek::Type* t, ValPtr aggr) const
 	if ( IsError() )
 		return nullptr;
 
+	if ( ! init_tag_check(this, "table constructor", TYPE_TABLE, t->Tag()) )
+		return nullptr;
+
 	auto tt = GetType<TableType>();
 
 	auto tval = aggr ?
@@ -3508,6 +3532,9 @@ ValPtr SetConstructorExpr::InitVal(const zeek::Type* t, ValPtr aggr) const
 	if ( IsError() )
 		return nullptr;
 
+	if ( ! init_tag_check(this, "set constructor", TYPE_TABLE, t->Tag()) )
+		return nullptr;
+
 	const auto& index_type = t->AsTableType()->GetIndices();
 	auto tt = GetType<TableType>();
 	auto tval = aggr ?
@@ -3606,6 +3633,9 @@ ValPtr VectorConstructorExpr::InitVal(const zeek::Type* t, ValPtr aggr) const
 	if ( IsError() )
 		return nullptr;
 
+	if ( ! init_tag_check(this, "vector constructor", TYPE_VECTOR, t->Tag()) )
+		return nullptr;
+
 	auto vt = GetType<VectorType>();
 	auto vec = aggr ?
 	        VectorValPtr{AdoptRef{}, aggr.release()->AsVectorVal()} :
@@ -3864,6 +3894,15 @@ RecordCoerceExpr::RecordCoerceExpr(ExprPtr arg_op, RecordTypePtr r)
 
 ValPtr RecordCoerceExpr::InitVal(const zeek::Type* t, ValPtr aggr) const
 	{
+	if ( IsError() )
+		{
+		Error("bad record initializer");
+		return nullptr;
+		}
+
+	if ( ! init_tag_check(this, "record", TYPE_RECORD, t->Tag()) )
+		return nullptr;
+
 	if ( auto v = Eval(nullptr) )
 		{
 		RecordVal* rv = v->AsRecordVal();

src/Frag.cc

@@ -6,7 +6,7 @@
 #include "zeek/Hash.h"
 #include "zeek/IP.h"
 #include "zeek/NetVar.h"
-#include "zeek/Sessions.h"
+#include "zeek/session/Manager.h"
 #include "zeek/Reporter.h"
 #include "zeek/RunState.h"
 
@@ -29,7 +29,7 @@ void FragTimer::Dispatch(double t, bool /* is_expire */)
 		reporter->InternalWarning("fragment timer dispatched w/o reassembler");
 	}
 
-FragReassembler::FragReassembler(NetSessions* arg_s,
+FragReassembler::FragReassembler(session::Manager* arg_s,
                                  const std::unique_ptr<IP_Hdr>& ip, const u_char* pkt,
                                  const FragReassemblerKey& k, double t)
 	: Reassembler(0, REASSEM_FRAG)
@@ -355,7 +355,7 @@ FragReassembler* FragmentManager::NextFragment(double t, const std::unique_ptr<I
 
 	if ( ! f )
 		{
-		f = new FragReassembler(sessions, ip, pkt, key, t);
+		f = new FragReassembler(session_mgr, ip, pkt, key, t);
 		fragments[key] = f;
 		if ( fragments.size() > max_fragments )
 			max_fragments = fragments.size();

src/Frag.h

@@ -12,9 +12,10 @@
 
 namespace zeek {
 
-class NetSessions;
 class IP_Hdr;
 
+namespace session { class Manager; }
+
 namespace detail {
 
 class FragReassembler;
@@ -24,8 +25,8 @@ using FragReassemblerKey = std::tuple<IPAddr, IPAddr, bro_uint_t>;
 
 class FragReassembler : public Reassembler {
 public:
-	FragReassembler(NetSessions* s, const std::unique_ptr<IP_Hdr>& ip, const u_char* pkt,
+	FragReassembler(session::Manager* s, const std::unique_ptr<IP_Hdr>& ip,
-	                const FragReassemblerKey& k, double t);
+	                const u_char* pkt, const FragReassemblerKey& k, double t);
 	~FragReassembler() override;
 
 	void AddFragment(double t, const std::unique_ptr<IP_Hdr>& ip, const u_char* pkt);
@@ -44,7 +45,7 @@ protected:
 
 	u_char* proto_hdr;
 	std::unique_ptr<IP_Hdr> reassembled_pkt;
-	NetSessions* s;
+	session::Manager* s;
 	uint64_t frag_size;	// size of fully reassembled fragment
 	FragReassemblerKey key;
 	uint16_t next_proto; // first IPv6 fragment header's next proto field

src/Frame.cc

@@ -430,7 +430,8 @@ std::pair<bool, FramePtr> Frame::Unserialize(const broker::vector& data,
 
 	if ( captures || *has_name == "CopyFrame" )
 		{
-		ASSERT(captures && *has_name == "CopyFrame");
+		if ( captures )
+			ASSERT(*has_name == "CopyFrame");
 
 		auto has_body = broker::get_if<broker::vector>(*where);
 		if ( ! has_body )

src/Func.cc

@@ -44,8 +44,8 @@
 #include "zeek/File.h"
 #include "zeek/Frame.h"
 #include "zeek/Var.h"
-#include "zeek/analyzer/protocol/login/Login.h"
+#include "zeek/analyzer/protocol/tcp/TCP.h"
-#include "zeek/Sessions.h"
+#include "zeek/session/Manager.h"
 #include "zeek/RE.h"
 #include "zeek/Event.h"
 #include "zeek/Traverse.h"
@@ -62,6 +62,7 @@
 #include "option.bif.func_h"
 #include "supervisor.bif.func_h"
 #include "packet_analysis.bif.func_h"
+#include "CPP-load.bif.func_h"
 
 #include "zeek.bif.func_def"
 #include "stats.bif.func_def"
@@ -70,6 +71,7 @@
 #include "option.bif.func_def"
 #include "supervisor.bif.func_def"
 #include "packet_analysis.bif.func_def"
+#include "CPP-load.bif.func_def"
 
 extern	RETSIGTYPE sig_handler(int signo);
 
@@ -305,11 +307,35 @@ ScriptFunc::ScriptFunc(const IDPtr& arg_id, StmtPtr arg_body,
 		Body b;
 		b.stmts = AddInits(std::move(arg_body), aggr_inits);
 		current_body = b.stmts;
-		b.priority = priority;
+		current_priority = b.priority = priority;
 		bodies.push_back(b);
 		}
 	}
 
+ScriptFunc::ScriptFunc(std::string _name, FuncTypePtr ft,
+	               std::vector<StmtPtr> bs, std::vector<int> priorities)
+	{
+	name = std::move(_name);
+	frame_size = ft->ParamList()->GetTypes().size();
+	type = std::move(ft);
+
+	auto n = bs.size();
+	ASSERT(n == priorities.size());
+
+	for ( auto i = 0u; i < n; ++i )
+		{
+		Body b;
+		b.stmts = std::move(bs[i]);
+		b.priority = priorities[i];
+		bodies.push_back(b);
+		}
+
+	sort(bodies.begin(), bodies.end());
+
+	current_body = bodies[0].stmts;
+	current_priority = bodies[0].priority;
+	}
+
 ScriptFunc::~ScriptFunc()
 	{
 	if ( ! weak_closure_ref )
@@ -542,9 +568,8 @@ void ScriptFunc::AddBody(StmtPtr new_body,
 
 	Body b;
 	b.stmts = new_body;
-	b.priority = priority;
-
 	current_body = new_body;
+	current_priority = b.priority = priority;
 
 	bodies.push_back(b);
 	sort(bodies.begin(), bodies.end());
@@ -558,6 +583,7 @@ void ScriptFunc::ReplaceBody(const StmtPtr& old_body, StmtPtr new_body)
 		if ( body.stmts.get() == old_body.get() )
 			{
 			body.stmts = new_body;
+			current_priority = body.priority;
 			found_it = true;
 			}
 

src/Func.h

@@ -151,6 +151,10 @@ public:
 	        const std::vector<IDPtr>& inits,
 	        size_t frame_size, int priority);
 
+	// For compiled scripts.
+	ScriptFunc(std::string name, FuncTypePtr ft,
+	           std::vector<StmtPtr> bodies, std::vector<int> priorities);
+
 	~ScriptFunc() override;
 
 	bool IsPure() const override;
@@ -239,6 +243,7 @@ public:
 				detail::StmtPtr new_body);
 
 	StmtPtr CurrentBody() const		{ return current_body; }
+	int CurrentPriority() const		{ return current_priority; }
 
 	/**
 	 * Returns the function's frame size.
@@ -307,8 +312,11 @@ private:
 
 	OffsetMap* captures_offset_mapping = nullptr;
 
-	// The most recently added/updated body.
+	// The most recently added/updated body ...
 	StmtPtr current_body;
+
+	// ... and its priority.
+	int current_priority;
 };
 
 using built_in_func = BifReturnVal (*)(Frame* frame, const Args* args);

src/ID.cc

@@ -285,6 +285,11 @@ const AttrPtr& ID::GetAttr(AttrTag t) const
 	return attrs ? attrs->Find(t) : Attr::nil;
 	}
 
+void ID::AddInitExpr(ExprPtr init_expr)
+	{
+	init_exprs.emplace_back(std::move(init_expr));
+	}
+
 bool ID::IsDeprecated() const
 	{
 	return GetAttr(ATTR_DEPRECATED) != nullptr;

src/ID.h

@@ -112,6 +112,10 @@ public:
 
 	const AttrPtr& GetAttr(AttrTag t) const;
 
+	void AddInitExpr(ExprPtr init_expr);
+	const std::vector<ExprPtr>& GetInitExprs() const
+		{ return init_exprs; }
+
 	bool IsDeprecated() const;
 
 	void MakeDeprecated(ExprPtr deprecation);
@@ -156,6 +160,13 @@ protected:
 	int offset;
 	ValPtr val;
 	AttributesPtr attrs;
+
+	// Expressions used to initialize the identifier, for use by
+	// the scripts-to-C++ compiler.  We need to track all of them
+	// because it's possible that a global value gets created using
+	// one of the earlier instances rather than the last one.
+	std::vector<ExprPtr> init_exprs;
+
 	// contains list of functions that are called when an option changes
 	std::multimap<int, FuncPtr> option_handlers;
 

src/IP.cc

@@ -619,6 +619,12 @@ void IPv6_Hdr_Chain::ProcessRoutingHeader(const struct ip6_rthdr* r, uint16_t le
 #ifdef ENABLE_MOBILE_IPV6
 void IPv6_Hdr_Chain::ProcessDstOpts(const struct ip6_dest* d, uint16_t len)
 	{
+	// Skip two bytes to get the beginning of the first option structure. These
+	// two bytes are the protocol for the next header and extension header length,
+	// already known to exist before calling this method.  See header format:
+	// https://datatracker.ietf.org/doc/html/rfc8200#section-4.6
+	assert(len >= 2);
+
 	const u_char* data = (const u_char*) d;
 	len -= 2 * sizeof(uint8_t);
 	data += 2* sizeof(uint8_t);
@@ -627,34 +633,45 @@ void IPv6_Hdr_Chain::ProcessDstOpts(const struct ip6_dest* d, uint16_t len)
 		{
 		const struct ip6_opt* opt = (const struct ip6_opt*) data;
 		switch ( opt->ip6o_type ) {
-		case 201: // Home Address Option, Mobile IPv6 RFC 6275 section 6.3
+		case 0:
-			{
+			// If option type is zero, it's a Pad0 and can be just a single
-			if ( opt->ip6o_len == 16 )
+			// byte in width. Skip over it.
-				if ( homeAddr )
-					reporter->Weird(SrcAddr(), DstAddr(), "multiple_home_addr_opts");
-				else
-					homeAddr = new IPAddr(*((const in6_addr*)(data + 2)));
-			else
-				reporter->Weird(SrcAddr(), DstAddr(), "bad_home_addr_len");
-			}
-			break;
-
-		default:
-			break;
-		}
-
-		if ( opt->ip6o_type == 0 )
-			{
 			data += sizeof(uint8_t);
 			len -= sizeof(uint8_t);
-			}
+			break;
-		else
+		default:
 			{
-			data += 2 * sizeof(uint8_t) + opt->ip6o_len;
+			// Double-check that the len can hold the whole option structure.
-			len -= 2 * sizeof(uint8_t) + opt->ip6o_len;
+			// Otherwise we get a buffer-overflow when we check the option_len.
+			// Also check that it holds everything for the option itself.
+			if ( len < sizeof(struct ip6_opt) ||
+			     len < sizeof(struct ip6_opt) + opt->ip6o_len )
+				{
+				reporter->Weird(SrcAddr(), DstAddr(), "bad_ipv6_dest_opt_len");
+				len = 0;
+				break;
+				}
+
+			if ( opt->ip6o_type == 201 ) // Home Address Option, Mobile IPv6 RFC 6275 section 6.3
+				{
+				if ( opt->ip6o_len == sizeof(struct in6_addr) )
+					{
+					if ( homeAddr )
+						reporter->Weird(SrcAddr(), DstAddr(), "multiple_home_addr_opts");
+					else
+						homeAddr = new IPAddr(*((const in6_addr*)(data + sizeof(struct ip6_opt))));
+					}
+				else
+					reporter->Weird(SrcAddr(), DstAddr(), "bad_home_addr_len");
+				}
+
+			data += sizeof(struct ip6_opt) + opt->ip6o_len;
+			len -= sizeof(struct ip6_opt) + opt->ip6o_len;
 			}
+			break;
 		}
 	}
+	}
 #endif
 
 VectorValPtr IPv6_Hdr_Chain::ToVal() const

src/IPAddr.cc

@@ -15,36 +15,57 @@
 namespace zeek {
 
 const IPAddr IPAddr::v4_unspecified = IPAddr(in4_addr{});
-
 const IPAddr IPAddr::v6_unspecified = IPAddr();
 
-detail::ConnIDKey detail::BuildConnIDKey(const ConnID& id)
+namespace detail {
-	{
-	ConnIDKey key;
 
-	// Lookup up connection based on canonical ordering, which is
+ConnKey::ConnKey(const IPAddr& src, const IPAddr& dst, uint16_t src_port,
+                 uint16_t dst_port, TransportProto t, bool one_way)
+	: transport(t)
+	{
+ 	// Lookup up connection based on canonical ordering, which is
 	// the smaller of <src addr, src port> and <dst addr, dst port>
 	// followed by the other.
-	if ( id.is_one_way ||
+	if ( one_way ||
-	     addr_port_canon_lt(id.src_addr, id.src_port, id.dst_addr, id.dst_port)
+	     addr_port_canon_lt(src, src_port, dst, dst_port)
 	   )
 		{
-		key.ip1 = id.src_addr.in6;
+		ip1 = src.in6;
-		key.ip2 = id.dst_addr.in6;
+		ip2 = dst.in6;
-		key.port1 = id.src_port;
+		port1 = src_port;
-		key.port2 = id.dst_port;
+		port2 = dst_port;
 		}
 	else
 		{
-		key.ip1 = id.dst_addr.in6;
+		ip1 = dst.in6;
-		key.ip2 = id.src_addr.in6;
+		ip2 = src.in6;
-		key.port1 = id.dst_port;
+		port1 = dst_port;
-		key.port2 = id.src_port;
+		port2 = src_port;
 		}
-
-	return key;
 	}
 
+ConnKey::ConnKey(const ConnTuple& id)
+	: ConnKey(id.src_addr, id.dst_addr, id.src_port, id.dst_port,
+	          id.proto, id.is_one_way)
+	{
+	}
+
+ConnKey& ConnKey::operator=(const ConnKey& rhs)
+	{
+	if ( this == &rhs )
+		return *this;
+
+	memcpy(&ip1, &rhs.ip1, sizeof(in6_addr));
+	memcpy(&ip2, &rhs.ip2, sizeof(in6_addr));
+	port1 = rhs.port1;
+	port2 = rhs.port2;
+	transport = rhs.transport;
+
+	return *this;
+	}
+
+} // namespace detail
+
 IPAddr::IPAddr(const String& s)
 	{
 	Init(s.CheckString());

src/IPAddr.h

@@ -4,60 +4,46 @@
 
 #include <netinet/in.h>
 #include <arpa/inet.h>
-#include <string.h>
+#include <cstring>
 #include <string>
 #include <memory>
 
 #include "zeek/threading/SerialTypes.h"
 
-typedef in_addr in4_addr;
+using in4_addr = in_addr;
 
 namespace zeek {
 
 class String;
-struct ConnID;
+struct ConnTuple;
 
 namespace detail {
 
 class HashKey;
 
-struct ConnIDKey {
+struct ConnKey {
 	in6_addr ip1;
 	in6_addr ip2;
 	uint16_t port1;
 	uint16_t port2;
+	TransportProto transport;
 
-	ConnIDKey() : port1(0), port2(0)
+	ConnKey(const IPAddr& src, const IPAddr& dst, uint16_t src_port,
-		{
+	        uint16_t dst_port, TransportProto t, bool one_way);
-		memset(&ip1, 0, sizeof(in6_addr));
+	ConnKey(const ConnTuple& conn);
-		memset(&ip2, 0, sizeof(in6_addr));
+	ConnKey(const ConnKey& rhs)	{ *this = rhs; }
-		}
 
-	ConnIDKey(const ConnIDKey& rhs)
+	bool operator<(const ConnKey& rhs) const { return memcmp(this, &rhs, sizeof(ConnKey)) < 0; }
-		{
+	bool operator<=(const ConnKey& rhs) const { return memcmp(this, &rhs, sizeof(ConnKey)) <= 0; }
-		*this = rhs;
+	bool operator==(const ConnKey& rhs) const { return memcmp(this, &rhs, sizeof(ConnKey)) == 0; }
-		}
+	bool operator!=(const ConnKey& rhs) const { return memcmp(this, &rhs, sizeof(ConnKey)) != 0; }
+	bool operator>=(const ConnKey& rhs) const { return memcmp(this, &rhs, sizeof(ConnKey)) >= 0; }
+	bool operator>(const ConnKey& rhs) const { return memcmp(this, &rhs, sizeof(ConnKey)) > 0; }
 
-	bool operator<(const ConnIDKey& rhs) const { return memcmp(this, &rhs, sizeof(ConnIDKey)) < 0; }
+	ConnKey& operator=(const ConnKey& rhs);
-	bool operator<=(const ConnIDKey& rhs) const { return memcmp(this, &rhs, sizeof(ConnIDKey)) <= 0; }
-	bool operator==(const ConnIDKey& rhs) const { return memcmp(this, &rhs, sizeof(ConnIDKey)) == 0; }
-	bool operator!=(const ConnIDKey& rhs) const { return memcmp(this, &rhs, sizeof(ConnIDKey)) != 0; }
-	bool operator>=(const ConnIDKey& rhs) const { return memcmp(this, &rhs, sizeof(ConnIDKey)) >= 0; }
-	bool operator>(const ConnIDKey& rhs) const { return memcmp(this, &rhs, sizeof(ConnIDKey)) > 0; }
-
-	ConnIDKey& operator=(const ConnIDKey& rhs)
-		{
-		if ( this != &rhs )
-			memcpy(this, &rhs, sizeof(ConnIDKey));
-
-		return *this;
-		}
 };
 
-/**
+using ConnIDKey [[deprecated("Remove in v5.1. Use zeek::detail::ConnKey.")]] = ConnKey;
- * Returns a map key for a given ConnID.
- */
-ConnIDKey BuildConnIDKey(const ConnID& id);
 
 } // namespace detail
 
@@ -69,7 +55,7 @@ public:
 	/**
 	 * Address family.
 	 */
-	typedef IPFamily Family;
+	using Family = IPFamily;
 
 	/**
 	 * Byte order.
@@ -392,13 +378,12 @@ public:
 		return ! ( addr1 <= addr2 );
 		}
 
-	/** Converts the address into the type used internally by the
+	/**
-	  * inter-thread communication.
+	 * Converts the address into the type used internally by the
-	  */
+	 * inter-thread communication.
+	 */
 	void ConvertToThreadingValue(threading::Value::addr_t* v) const;
 
-	friend detail::ConnIDKey detail::BuildConnIDKey(const ConnID& id);
-
 	unsigned int MemoryAllocation() const { return padded_sizeof(*this); }
 
 	/**
@@ -450,6 +435,7 @@ public:
 	static const IPAddr v6_unspecified;
 
 private:
+	friend struct detail::ConnKey;
 	friend class IPPrefix;
 
 	/**
@@ -613,7 +599,8 @@ public:
 	 */
 	uint8_t LengthIPv6() const { return length; }
 
-	/** Returns true if the given address is part of the prefix.
+	/**
+	 * Returns true if the given address is part of the prefix.
 	 *
 	 * @param addr The address to test.
 	 */
@@ -649,9 +636,10 @@ public:
 	 */
 	std::unique_ptr<detail::HashKey> MakeHashKey() const;
 
-	/** Converts the prefix into the type used internally by the
+	/**
-	  * inter-thread communication.
+	 * Converts the prefix into the type used internally by the
-	  */
+	 * inter-thread communication.
+	 */
 	void ConvertToThreadingValue(threading::Value::subnet_t* v) const
 		{
 		v->length = length;

src/IntrusivePtr.h

@@ -137,12 +137,29 @@ public:
 		return std::exchange(ptr_, nullptr);
 		}
 
-	IntrusivePtr& operator=(IntrusivePtr other) noexcept
+	IntrusivePtr& operator=(const IntrusivePtr& other) noexcept
+		{
+		IntrusivePtr tmp{other};
+		swap(tmp);
+		return *this;
+		}
+
+	IntrusivePtr& operator=(IntrusivePtr&& other) noexcept
 		{
 		swap(other);
 		return *this;
 		}
 
+	IntrusivePtr& operator=(std::nullptr_t) noexcept
+		{
+		if ( ptr_ )
+			{
+			Unref(ptr_);
+			ptr_ = nullptr;
+			}
+		return *this;
+		}
+
 	pointer get() const noexcept
 		{
 		return ptr_;

src/Options.cc

@@ -158,26 +158,45 @@ static void set_analysis_option(const char* opt, Options& opts)
 	if ( util::streq(opt, "help") )
 		{
 		fprintf(stderr, "--optimize options:\n");
+		fprintf(stderr, "    all	equivalent to \"inline\" and \"activate\"\n");
+		fprintf(stderr, "    add-C++	generate private C++ for any missing script bodies\n");
 		fprintf(stderr, "    dump-uds	dump use-defs to stdout; implies xform\n");
 		fprintf(stderr, "    dump-xform	dump transformed scripts to stdout; implies xform\n");
+		fprintf(stderr, "    gen-C++	generate C++ script bodies\n");
+		fprintf(stderr, "    gen-standalone-C++	generate \"standalone\" C++ script bodies\n");
 		fprintf(stderr, "    help	print this list\n");
 		fprintf(stderr, "    inline	inline function calls\n");
 		fprintf(stderr, "    optimize-AST	optimize the (transformed) AST; implies xform\n");
 		fprintf(stderr, "    recursive	report on recursive functions and exit\n");
+		fprintf(stderr, "    report-C++	report available C++ script bodies and exit\n");
+		fprintf(stderr, "    update-C++	generate reusable C++ for any missing script bodies\n");
+		fprintf(stderr, "    use-C++	use available C++ script bodies\n");
 		fprintf(stderr, "    xform	tranform scripts to \"reduced\" form\n");
 		exit(0);
 		}
 
 	auto& a_o = opts.analysis_options;
 
-	if ( util::streq(opt, "dump-uds") )
+	if ( util::streq(opt, "add-C++") )
+		a_o.add_CPP = true;
+	else if ( util::streq(opt, "dump-uds") )
 		a_o.activate = a_o.dump_uds = true;
 	else if ( util::streq(opt, "dump-xform") )
 		a_o.activate = a_o.dump_xform = true;
+	else if ( util::streq(opt, "gen-C++") )
+		a_o.gen_CPP = true;
+	else if ( util::streq(opt, "gen-standalone-C++") )
+		a_o.gen_standalone_CPP = true;
 	else if ( util::streq(opt, "inline") )
 		a_o.inliner = true;
 	else if ( util::streq(opt, "recursive") )
 		a_o.inliner = a_o.report_recursive = true;
+	else if ( util::streq(opt, "report-C++") )
+		a_o.report_CPP = true;
+	else if ( util::streq(opt, "update-C++") )
+		a_o.update_CPP = true;
+	else if ( util::streq(opt, "use-C++") )
+		a_o.use_CPP = true;
 	else if ( util::streq(opt, "xform") )
 		a_o.activate = true;
 	else if ( util::streq(opt, "optimize-AST") )

src/Queue.h

@@ -26,11 +26,11 @@
 namespace zeek {
 
 template<typename T>
-class Queue {
+class [[deprecated("Remove in v5.1. This class is deprecated (and is likely broken, see #1528). Use std::vector<T>.")]] Queue {
 public:
 	explicit Queue(int size = 0)
 		{
-		const int DEFAULT_CHUNK_SIZE = 10;
+		constexpr int DEFAULT_CHUNK_SIZE = 10;
 		chunk_size = DEFAULT_CHUNK_SIZE;
 
 		head = tail = num_entries = 0;
@@ -55,6 +55,7 @@ public:
 	~Queue()		{ delete[] entries; }
 
 	int length() const	{ return num_entries; }
+	int capacity() const	{ return max_entries; }
 	int resize(int new_size = 0)	// 0 => size to fit current number of entries
 		{
 		if ( new_size < num_entries )
@@ -197,6 +198,6 @@ protected:
 
 
 template<typename T>
-using PQueue = Queue<T*>;
+using PQueue [[deprecated("Remove in v5.1. This class is deprecated (and is likely broken, see #1528). Use std::vector<T*>.")]] = Queue<T*>;
 
 } // namespace zeek

src/Reporter.cc

@@ -187,6 +187,21 @@ void Reporter::RuntimeError(const detail::Location* location, const char* fmt, .
 	throw InterpreterException();
 	}
 
+void Reporter::CPPRuntimeError(const char* fmt, ...)
+	{
+	++errors;
+	va_list ap;
+	va_start(ap, fmt);
+	FILE* out = EmitToStderr(errors_to_stderr) ? stderr : nullptr;
+	DoLog("runtime error in compiled code", reporter_error, out, nullptr, nullptr, true, true, "", fmt, ap);
+	va_end(ap);
+
+	if ( abort_on_scripting_errors )
+		abort();
+
+	throw InterpreterException();
+	}
+
 void Reporter::InternalError(const char* fmt, ...)
 	{
 	va_list ap;
@@ -439,7 +454,7 @@ void Reporter::Weird(Connection* conn, const char* name, const char* addl, const
 			return;
 	}
 
-	WeirdHelper(conn_weird, {conn->ConnVal()->Ref(), new StringVal(addl), new StringVal(source)},
+	WeirdHelper(conn_weird, {conn->GetVal()->Ref(), new StringVal(addl), new StringVal(source)},
 	            "%s", name);
 	}
 
@@ -601,7 +616,7 @@ void Reporter::DoLog(const char* prefix, EventHandlerPtr event, FILE* out,
 			vl.emplace_back(make_intrusive<StringVal>(loc_str.c_str()));
 
 		if ( conn )
-			vl.emplace_back(conn->ConnVal());
+			vl.emplace_back(conn->GetVal());
 
 		if ( addl )
 			for ( auto v : *addl )

src/Reporter.h

@@ -100,6 +100,10 @@ public:
 	// function will not return but raise an InterpreterException.
 	[[noreturn]] void RuntimeError(const detail::Location* location, const char* fmt, ...) __attribute__((format(printf, 3, 4)));
 
+	// Report a runtime error in executing a compiled script. This
+	// function will not return but raise an InterpreterException.
+	[[noreturn]] void CPPRuntimeError(const char* fmt, ...) __attribute__((format(printf, 2, 3)));
+
 	// Report a traffic weirdness, i.e., an unexpected protocol situation
 	// that may lead to incorrectly processing a connnection.
 	void Weird(const char* name, const char* addl = "", const char* source = "");	// Raises net_weird().

src/RuleCondition.cc

@@ -25,12 +25,12 @@ namespace zeek::detail {
 bool RuleConditionTCPState::DoMatch(Rule* rule, RuleEndpointState* state,
 					const u_char* data, int len)
 	{
-	analyzer::Analyzer* root = state->GetAnalyzer()->Conn()->GetRootAnalyzer();
+	auto* adapter = state->GetAnalyzer()->Conn()->GetSessionAdapter();
 
-	if ( ! root || ! root->IsAnalyzer("TCP") )
+	if ( ! adapter || ! adapter->IsAnalyzer("TCP") )
 		return false;
 
-	auto* ta = static_cast<analyzer::tcp::TCP_Analyzer*>(root);
+	auto* ta = static_cast<packet_analysis::TCP::TCPSessionAdapter*>(adapter);
 
 	if ( tcpstates & RULE_STATE_STATELESS )
 		return true;
@@ -57,9 +57,9 @@ void RuleConditionTCPState::PrintDebug()
 bool RuleConditionUDPState::DoMatch(Rule* rule, RuleEndpointState* state,
                                     const u_char* data, int len)
 	{
-	analyzer::Analyzer* root = state->GetAnalyzer()->Conn()->GetRootAnalyzer();
+	auto* adapter = state->GetAnalyzer()->Conn()->GetSessionAdapter();
 
-	if ( ! root || ! root->IsAnalyzer("UDP") )
+	if ( ! adapter || ! adapter->IsAnalyzer("UDP") )
 		return false;
 
 	if ( states & RULE_STATE_STATELESS )

src/RunState.cc

@@ -24,7 +24,7 @@ extern "C" {
 };
 
 #include "zeek/NetVar.h"
-#include "zeek/Sessions.h"
+#include "zeek/session/Manager.h"
 #include "zeek/Event.h"
 #include "zeek/Timer.h"
 #include "zeek/ID.h"
@@ -37,6 +37,7 @@ extern "C" {
 #include "zeek/plugin/Manager.h"
 #include "zeek/broker/Manager.h"
 #include "zeek/packet_analysis/Manager.h"
+#include "zeek/analyzer/protocol/stepping-stone/SteppingStone.h"
 
 extern "C" {
 extern int select(int, fd_set *, fd_set *, fd_set *, struct timeval *);
@@ -45,6 +46,8 @@ extern int select(int, fd_set *, fd_set *, fd_set *, struct timeval *);
 static double last_watchdog_proc_time = 0.0;	// value of above during last watchdog
 extern int signal_val;
 
+using namespace zeek::analyzer::stepping_stone;
+
 namespace zeek::run_state {
 namespace detail {
 
@@ -192,7 +195,10 @@ void init_run(const std::optional<std::string>& interface,
 
 	zeek::detail::init_ip_addr_anonymizers();
 
-	sessions = new NetSessions();
+	session_mgr = new session::Manager();
+
+	// Initialize the stepping stone manager. We intentionally throw away the result here.
+	SteppingStoneManager::Get();
 
 	if ( do_watchdog )
 		{
@@ -384,13 +390,13 @@ void finish_run(int drain_events)
 
 	if ( drain_events )
 		{
-		if ( sessions )
+		if ( session_mgr )
-			sessions->Drain();
+			session_mgr->Drain();
 
 		event_mgr.Drain();
 
-		if ( sessions )
+		if ( session_mgr )
-			sessions->Done();
+			session_mgr->Done();
 		}
 
 #ifdef DEBUG
@@ -407,7 +413,8 @@ void delete_run()
 	{
 	util::detail::set_processing_status("TERMINATING", "delete_run");
 
-	delete sessions;
+	delete session_mgr;
+	delete SteppingStoneManager::Get();
 
 	for ( int i = 0; i < zeek::detail::NUM_ADDR_ANONYMIZATION_METHODS; ++i )
 		delete zeek::detail::ip_anonymizer[i];

src/Sessions.cc

@@ -1,786 +0,0 @@
-// See the file "COPYING" in the main distribution directory for copyright.
-
-#include "zeek/zeek-config.h"
-#include "zeek/Sessions.h"
-
-#include <netinet/in.h>
-#include <arpa/inet.h>
-
-#include <stdlib.h>
-#include <unistd.h>
-
-#include <pcap.h>
-
-#include "zeek/Desc.h"
-#include "zeek/RunState.h"
-#include "zeek/Event.h"
-#include "zeek/Timer.h"
-#include "zeek/NetVar.h"
-#include "zeek/Reporter.h"
-#include "zeek/RuleMatcher.h"
-#include "zeek/TunnelEncapsulation.h"
-
-#include "zeek/analyzer/protocol/icmp/ICMP.h"
-#include "zeek/analyzer/protocol/udp/UDP.h"
-#include "zeek/analyzer/protocol/stepping-stone/SteppingStone.h"
-#include "zeek/analyzer/Manager.h"
-
-#include "zeek/iosource/IOSource.h"
-#include "zeek/packet_analysis/Manager.h"
-
-#include "zeek/analyzer/protocol/stepping-stone/events.bif.h"
-
-// These represent NetBIOS services on ephemeral ports.  They're numbered
-// so that we can use a single int to hold either an actual TCP/UDP server
-// port or one of these.
-enum NetBIOS_Service {
-	NETBIOS_SERVICE_START = 0x10000L,	// larger than any port
-	NETBIOS_SERVICE_DCE_RPC,
-};
-
-zeek::NetSessions* zeek::sessions;
-zeek::NetSessions*& sessions = zeek::sessions;
-
-namespace zeek {
-
-NetSessions::NetSessions()
-	{
-	if ( stp_correlate_pair )
-		stp_manager = new analyzer::stepping_stone::SteppingStoneManager();
-	else
-		stp_manager = nullptr;
-
-	packet_filter = nullptr;
-
-	memset(&stats, 0, sizeof(SessionStats));
-	}
-
-NetSessions::~NetSessions()
-	{
-	delete packet_filter;
-	delete stp_manager;
-
-	for ( const auto& entry : tcp_conns )
-		Unref(entry.second);
-	for ( const auto& entry : udp_conns )
-		Unref(entry.second);
-	for ( const auto& entry : icmp_conns )
-		Unref(entry.second);
-
-	detail::fragment_mgr->Clear();
-	}
-
-void NetSessions::Done()
-	{
-	}
-
-void NetSessions::ProcessTransportLayer(double t, const Packet* pkt, size_t remaining)
-	{
-	const std::unique_ptr<IP_Hdr>& ip_hdr = pkt->ip_hdr;
-
-	uint32_t len = ip_hdr->TotalLen();
-	uint16_t ip_hdr_len = ip_hdr->HdrLen();
-
-	if ( len < ip_hdr_len )
-		{
-		sessions->Weird("bogus_IP_header_lengths", pkt);
-		return;
-		}
-
-	len -= ip_hdr_len;	// remove IP header
-
-	int proto = ip_hdr->NextProto();
-
-	if ( CheckHeaderTrunc(proto, len, remaining, pkt) )
-		return;
-
-	const u_char* data = ip_hdr->Payload();
-
-	ConnID id;
-	id.src_addr = ip_hdr->SrcAddr();
-	id.dst_addr = ip_hdr->DstAddr();
-	ConnectionMap* d = nullptr;
-	BifEnum::Tunnel::Type tunnel_type = BifEnum::Tunnel::IP;
-
-	switch ( proto ) {
-	case IPPROTO_TCP:
-		{
-		const struct tcphdr* tp = (const struct tcphdr *) data;
-		id.src_port = tp->th_sport;
-		id.dst_port = tp->th_dport;
-		id.is_one_way = false;
-		d = &tcp_conns;
-		break;
-		}
-
-	case IPPROTO_UDP:
-		{
-		const struct udphdr* up = (const struct udphdr *) data;
-		id.src_port = up->uh_sport;
-		id.dst_port = up->uh_dport;
-		id.is_one_way = false;
-		d = &udp_conns;
-		break;
-		}
-
-	case IPPROTO_ICMP:
-		{
-		const struct icmp* icmpp = (const struct icmp *) data;
-
-		id.src_port = icmpp->icmp_type;
-		id.dst_port = analyzer::icmp::ICMP4_counterpart(icmpp->icmp_type,
-		                                                icmpp->icmp_code,
-		                                                id.is_one_way);
-
-		id.src_port = htons(id.src_port);
-		id.dst_port = htons(id.dst_port);
-
-		d = &icmp_conns;
-		break;
-		}
-
-	case IPPROTO_ICMPV6:
-		{
-		const struct icmp* icmpp = (const struct icmp *) data;
-
-		id.src_port = icmpp->icmp_type;
-		id.dst_port = analyzer::icmp::ICMP6_counterpart(icmpp->icmp_type,
-		                                                icmpp->icmp_code,
-		                                                id.is_one_way);
-
-		id.src_port = htons(id.src_port);
-		id.dst_port = htons(id.dst_port);
-
-		d = &icmp_conns;
-		break;
-		}
-
-	default:
-		Weird("unknown_protocol", pkt, util::fmt("%d", proto));
-		return;
-	}
-
-	detail::ConnIDKey key = detail::BuildConnIDKey(id);
-	Connection* conn = nullptr;
-
-	// FIXME: The following is getting pretty complex. Need to split up
-	// into separate functions.
-	auto it = d->find(key);
-	if ( it != d->end() )
-		conn = it->second;
-
-	if ( ! conn )
-		{
-		conn = NewConn(key, t, &id, data, proto, ip_hdr->FlowLabel(), pkt);
-		if ( conn )
-			InsertConnection(d, key, conn);
-		}
-	else
-		{
-		// We already know that connection.
-		if ( conn->IsReuse(t, data) )
-			{
-			conn->Event(connection_reused, nullptr);
-
-			Remove(conn);
-			conn = NewConn(key, t, &id, data, proto, ip_hdr->FlowLabel(), pkt);
-			if ( conn )
-				InsertConnection(d, key, conn);
-			}
-		else
-			{
-			conn->CheckEncapsulation(pkt->encap);
-			}
-		}
-
-	if ( ! conn )
-		return;
-
-	int record_packet = 1;	// whether to record the packet at all
-	int record_content = 1;	// whether to record its data
-
-	bool is_orig = (id.src_addr == conn->OrigAddr()) &&
-			(id.src_port == conn->OrigPort());
-
-	conn->CheckFlowLabel(is_orig, ip_hdr->FlowLabel());
-
-	ValPtr pkt_hdr_val;
-
-	if ( ipv6_ext_headers && ip_hdr->NumHeaders() > 1 )
-		{
-		pkt_hdr_val = ip_hdr->ToPktHdrVal();
-		conn->EnqueueEvent(ipv6_ext_headers, nullptr, conn->ConnVal(),
-		                   pkt_hdr_val);
-		}
-
-	if ( new_packet )
-		conn->EnqueueEvent(new_packet, nullptr, conn->ConnVal(), pkt_hdr_val ?
-		                   std::move(pkt_hdr_val) : ip_hdr->ToPktHdrVal());
-
-	conn->NextPacket(t, is_orig, ip_hdr.get(), len, remaining, data,
-	                 record_packet, record_content, pkt);
-
-	// We skip this block for reassembled packets because the pointer
-	// math wouldn't work.
-	if ( ! ip_hdr->reassembled && record_packet )
-		{
-		if ( record_content )
-			pkt->dump_packet = true;	// save the whole thing
-
-		else
-			{
-			int hdr_len = data - pkt->data;
-			packet_mgr->DumpPacket(pkt, hdr_len);	// just save the header
-			}
-		}
-	}
-
-int NetSessions::ParseIPPacket(int caplen, const u_char* const pkt, int proto,
-                               IP_Hdr*& inner)
-	{
-	if ( proto == IPPROTO_IPV6 )
-		{
-		if ( caplen < (int)sizeof(struct ip6_hdr) )
-			return -1;
-
-		const struct ip6_hdr* ip6 = (const struct ip6_hdr*) pkt;
-		inner = new IP_Hdr(ip6, false, caplen);
-		if ( ( ip6->ip6_ctlun.ip6_un2_vfc & 0xF0 ) != 0x60 )
-			return -2;
-		}
-
-	else if ( proto == IPPROTO_IPV4 )
-		{
-		if ( caplen < (int)sizeof(struct ip) )
-			return -1;
-
-		const struct ip* ip4 = (const struct ip*) pkt;
-		inner = new IP_Hdr(ip4, false);
-		if ( ip4->ip_v != 4 )
-			return -2;
-		}
-
-	else
-		{
-		reporter->InternalWarning("Bad IP protocol version in ParseIPPacket");
-		return -1;
-		}
-
-	if ( (uint32_t)caplen != inner->TotalLen() )
-		return (uint32_t)caplen < inner->TotalLen() ? -1 : 1;
-
-	return 0;
-	}
-
-bool NetSessions::CheckHeaderTrunc(int proto, uint32_t len, uint32_t caplen,
-                                   const Packet* p)
-	{
-	uint32_t min_hdr_len = 0;
-	switch ( proto ) {
-	case IPPROTO_TCP:
-		min_hdr_len = sizeof(struct tcphdr);
-		break;
-	case IPPROTO_UDP:
-		min_hdr_len = sizeof(struct udphdr);
-		break;
-	case IPPROTO_ICMP:
-	case IPPROTO_ICMPV6:
-	default:
-		// Use for all other packets.
-		min_hdr_len = ICMP_MINLEN;
-		break;
-	}
-
-	if ( len < min_hdr_len )
-		{
-		Weird("truncated_header", p);
-		return true;
-		}
-
-	if ( caplen < min_hdr_len )
-		{
-		Weird("internally_truncated_header", p);
-		return true;
-		}
-
-	return false;
-	}
-
-Connection* NetSessions::FindConnection(Val* v)
-	{
-	const auto& vt = v->GetType();
-	if ( ! IsRecord(vt->Tag()) )
-		return nullptr;
-
-	RecordType* vr = vt->AsRecordType();
-	auto vl = v->As<RecordVal*>();
-
-	int orig_h, orig_p;	// indices into record's value list
-	int resp_h, resp_p;
-
-	if ( vr == id::conn_id )
-		{
-		orig_h = 0;
-		orig_p = 1;
-		resp_h = 2;
-		resp_p = 3;
-		}
-
-	else
-		{
-		// While it's not a conn_id, it may have equivalent fields.
-		orig_h = vr->FieldOffset("orig_h");
-		resp_h = vr->FieldOffset("resp_h");
-		orig_p = vr->FieldOffset("orig_p");
-		resp_p = vr->FieldOffset("resp_p");
-
-		if ( orig_h < 0 || resp_h < 0 || orig_p < 0 || resp_p < 0 )
-			return nullptr;
-
-		// ### we ought to check that the fields have the right
-		// types, too.
-		}
-
-	const IPAddr& orig_addr = vl->GetFieldAs<AddrVal>(orig_h);
-	const IPAddr& resp_addr = vl->GetFieldAs<AddrVal>(resp_h);
-
-	auto orig_portv = vl->GetFieldAs<PortVal>(orig_p);
-	auto resp_portv = vl->GetFieldAs<PortVal>(resp_p);
-
-	ConnID id;
-
-	id.src_addr = orig_addr;
-	id.dst_addr = resp_addr;
-
-	id.src_port = htons((unsigned short) orig_portv->Port());
-	id.dst_port = htons((unsigned short) resp_portv->Port());
-
-	id.is_one_way = false;	// ### incorrect for ICMP connections
-
-	detail::ConnIDKey key = detail::BuildConnIDKey(id);
-	ConnectionMap* d;
-
-	if ( orig_portv->IsTCP() )
-		d = &tcp_conns;
-	else if ( orig_portv->IsUDP() )
-		d = &udp_conns;
-	else if ( orig_portv->IsICMP() )
-		d = &icmp_conns;
-	else
-		{
-		// This can happen due to pseudo-connections we
-		// construct, for example for packet headers embedded
-		// in ICMPs.
-		return nullptr;
-		}
-
-	Connection* conn = nullptr;
-	auto it = d->find(key);
-	if ( it != d->end() )
-		conn = it->second;
-
-	return conn;
-	}
-
-void NetSessions::Remove(Connection* c)
-	{
-	if ( c->IsKeyValid() )
-		{
-		const detail::ConnIDKey& key = c->Key();
-		c->CancelTimers();
-
-		if ( c->ConnTransport() == TRANSPORT_TCP )
-			{
-			auto ta = static_cast<analyzer::tcp::TCP_Analyzer*>(c->GetRootAnalyzer());
-			assert(ta->IsAnalyzer("TCP"));
-			analyzer::tcp::TCP_Endpoint* to = ta->Orig();
-			analyzer::tcp::TCP_Endpoint* tr = ta->Resp();
-
-			tcp_stats.StateLeft(to->state, tr->state);
-			}
-
-		c->Done();
-		c->RemovalEvent();
-
-		// Zero out c's copy of the key, so that if c has been Ref()'d
-		// up, we know on a future call to Remove() that it's no
-		// longer in the dictionary.
-		c->ClearKey();
-
-		switch ( c->ConnTransport() ) {
-		case TRANSPORT_TCP:
-			if ( tcp_conns.erase(key) == 0 )
-				reporter->InternalWarning("connection missing");
-			break;
-
-		case TRANSPORT_UDP:
-			if ( udp_conns.erase(key) == 0 )
-				reporter->InternalWarning("connection missing");
-			break;
-
-		case TRANSPORT_ICMP:
-			if ( icmp_conns.erase(key) == 0 )
-				reporter->InternalWarning("connection missing");
-			break;
-
-		case TRANSPORT_UNKNOWN:
-			reporter->InternalWarning("unknown transport when removing connection");
-			break;
-		}
-
-		Unref(c);
-		}
-	}
-
-void NetSessions::Insert(Connection* c)
-	{
-	assert(c->IsKeyValid());
-
-	Connection* old = nullptr;
-
-	switch ( c->ConnTransport() ) {
-	// Remove first. Otherwise the map would still reference the old key for
-	// already existing connections.
-
-	case TRANSPORT_TCP:
-		old = LookupConn(tcp_conns, c->Key());
-		tcp_conns.erase(c->Key());
-		InsertConnection(&tcp_conns, c->Key(), c);
-		break;
-
-	case TRANSPORT_UDP:
-		old = LookupConn(udp_conns, c->Key());
-		udp_conns.erase(c->Key());
-		InsertConnection(&udp_conns, c->Key(), c);
-		break;
-
-	case TRANSPORT_ICMP:
-		old = LookupConn(icmp_conns, c->Key());
-		icmp_conns.erase(c->Key());
-		InsertConnection(&icmp_conns, c->Key(), c);
-		break;
-
-	default:
-		reporter->InternalWarning("unknown connection type");
-		Unref(c);
-		return;
-	}
-
-	if ( old && old != c )
-		{
-		// Some clean-ups similar to those in Remove() (but invisible
-		// to the script layer).
-		old->CancelTimers();
-		old->ClearKey();
-		Unref(old);
-		}
-	}
-
-void NetSessions::Drain()
-	{
-	for ( const auto& entry : tcp_conns )
-		{
-		Connection* tc = entry.second;
-		tc->Done();
-		tc->RemovalEvent();
-		}
-
-	for ( const auto& entry : udp_conns )
-		{
-		Connection* uc = entry.second;
-		uc->Done();
-		uc->RemovalEvent();
-		}
-
-	for ( const auto& entry : icmp_conns )
-		{
-		Connection* ic = entry.second;
-		ic->Done();
-		ic->RemovalEvent();
-		}
-	}
-
-void NetSessions::Clear()
-	{
-	for ( const auto& entry : tcp_conns )
-		Unref(entry.second);
-	for ( const auto& entry : udp_conns )
-		Unref(entry.second);
-	for ( const auto& entry : icmp_conns )
-		Unref(entry.second);
-
-	tcp_conns.clear();
-	udp_conns.clear();
-	icmp_conns.clear();
-
-	detail::fragment_mgr->Clear();
-	}
-
-void NetSessions::GetStats(SessionStats& s) const
-	{
-	s.num_TCP_conns = tcp_conns.size();
-	s.cumulative_TCP_conns = stats.cumulative_TCP_conns;
-	s.num_UDP_conns = udp_conns.size();
-	s.cumulative_UDP_conns = stats.cumulative_UDP_conns;
-	s.num_ICMP_conns = icmp_conns.size();
-	s.cumulative_ICMP_conns = stats.cumulative_ICMP_conns;
-	s.num_fragments = detail::fragment_mgr->Size();
-	s.num_packets = packet_mgr->PacketsProcessed();
-
-	s.max_TCP_conns = stats.max_TCP_conns;
-	s.max_UDP_conns = stats.max_UDP_conns;
-	s.max_ICMP_conns = stats.max_ICMP_conns;
-	s.max_fragments = detail::fragment_mgr->MaxFragments();
-	}
-
-Connection* NetSessions::NewConn(const detail::ConnIDKey& k, double t, const ConnID* id,
-                                 const u_char* data, int proto, uint32_t flow_label,
-                                 const Packet* pkt)
-	{
-	// FIXME: This should be cleaned up a bit, it's too protocol-specific.
-	// But I'm not yet sure what the right abstraction for these things is.
-	int src_h = ntohs(id->src_port);
-	int dst_h = ntohs(id->dst_port);
-	int flags = 0;
-
-	// Hmm... This is not great.
-	TransportProto tproto = TRANSPORT_UNKNOWN;
-	switch ( proto ) {
-		case IPPROTO_ICMP:
-			tproto = TRANSPORT_ICMP;
-			break;
-		case IPPROTO_TCP:
-			tproto = TRANSPORT_TCP;
-			break;
-		case IPPROTO_UDP:
-			tproto = TRANSPORT_UDP;
-			break;
-		case IPPROTO_ICMPV6:
-			tproto = TRANSPORT_ICMP;
-			break;
-		default:
-			reporter->InternalWarning("unknown transport protocol");
-			return nullptr;
-	};
-
-	if ( tproto == TRANSPORT_TCP )
-		{
-		const struct tcphdr* tp = (const struct tcphdr*) data;
-		flags = tp->th_flags;
-		}
-
-	bool flip = false;
-
-	if ( ! WantConnection(src_h, dst_h, tproto, flags, flip) )
-		return nullptr;
-
-	Connection* conn = new Connection(this, k, t, id, flow_label, pkt);
-	conn->SetTransport(tproto);
-
-	if ( flip )
-		conn->FlipRoles();
-
-	if ( ! analyzer_mgr->BuildInitialAnalyzerTree(conn) )
-		{
-		conn->Done();
-		Unref(conn);
-		return nullptr;
-		}
-
-	if ( new_connection )
-		conn->Event(new_connection, nullptr);
-
-	return conn;
-	}
-
-Connection* NetSessions::LookupConn(const ConnectionMap& conns, const detail::ConnIDKey& key)
-	{
-	auto it = conns.find(key);
-	if ( it != conns.end() )
-		return it->second;
-
-	return nullptr;
-	}
-
-bool NetSessions::IsLikelyServerPort(uint32_t port, TransportProto proto) const
-	{
-	// We keep a cached in-core version of the table to speed up the lookup.
-	static std::set<bro_uint_t> port_cache;
-	static bool have_cache = false;
-
-	if ( ! have_cache )
-		{
-		auto likely_server_ports = id::find_val<TableVal>("likely_server_ports");
-		auto lv = likely_server_ports->ToPureListVal();
-		for ( int i = 0; i < lv->Length(); i++ )
-			port_cache.insert(lv->Idx(i)->InternalUnsigned());
-		have_cache = true;
-		}
-
-	// We exploit our knowledge of PortVal's internal storage mechanism
-	// here.
-	if ( proto == TRANSPORT_TCP )
-		port |= TCP_PORT_MASK;
-	else if ( proto == TRANSPORT_UDP )
-		port |= UDP_PORT_MASK;
-	else if ( proto == TRANSPORT_ICMP )
-		port |= ICMP_PORT_MASK;
-
-	return port_cache.find(port) != port_cache.end();
-	}
-
-bool NetSessions::WantConnection(uint16_t src_port, uint16_t dst_port,
-                                 TransportProto transport_proto,
-                                 uint8_t tcp_flags, bool& flip_roles)
-	{
-	flip_roles = false;
-
-	if ( transport_proto == TRANSPORT_TCP )
-		{
-		if ( ! (tcp_flags & TH_SYN) || (tcp_flags & TH_ACK) )
-			{
-			// The new connection is starting either without a SYN,
-			// or with a SYN ack. This means it's a partial connection.
-			if ( ! zeek::detail::partial_connection_ok )
-				return false;
-
-			if ( tcp_flags & TH_SYN && ! zeek::detail::tcp_SYN_ack_ok )
-				return false;
-
-			// Try to guess true responder by the port numbers.
-			// (We might also think that for SYN acks we could
-			// safely flip the roles, but that doesn't work
-			// for stealth scans.)
-			if ( IsLikelyServerPort(src_port, TRANSPORT_TCP) )
-				{ // connection is a candidate for flipping
-				if ( IsLikelyServerPort(dst_port, TRANSPORT_TCP) )
-					// Hmmm, both source and destination
-					// are plausible.  Heuristic: flip only
-					// if (1) this isn't a SYN ACK (to avoid
-					// confusing stealth scans) and
-					// (2) dest port > src port (to favor
-					// more plausible servers).
-					flip_roles = ! (tcp_flags & TH_SYN) && src_port < dst_port;
-				else
-					// Source is plausible, destination isn't.
-					flip_roles = true;
-				}
-			}
-		}
-
-	else if ( transport_proto == TRANSPORT_UDP )
-		flip_roles =
-			IsLikelyServerPort(src_port, TRANSPORT_UDP) &&
-			! IsLikelyServerPort(dst_port, TRANSPORT_UDP);
-
-	return true;
-	}
-
-void NetSessions::Weird(const char* name, const Packet* pkt, const char* addl, const char* source)
-	{
-	const char* weird_name = name;
-
-	if ( pkt )
-		{
-		pkt->dump_packet = true;
-
-		if ( pkt->encap && pkt->encap->LastType() != BifEnum::Tunnel::NONE )
-			weird_name = util::fmt("%s_in_tunnel", name);
-
-		if ( pkt->ip_hdr )
-			{
-			reporter->Weird(pkt->ip_hdr->SrcAddr(), pkt->ip_hdr->DstAddr(), weird_name, addl, source);
-			return;
-			}
-		}
-
-	reporter->Weird(weird_name, addl, source);
-	}
-
-void NetSessions::Weird(const char* name, const IP_Hdr* ip, const char* addl)
-	{
-	reporter->Weird(ip->SrcAddr(), ip->DstAddr(), name, addl);
-	}
-
-unsigned int NetSessions::ConnectionMemoryUsage()
-	{
-	unsigned int mem = 0;
-
-	if ( run_state::terminating )
-		// Connections have been flushed already.
-		return 0;
-
-	for ( const auto& entry : tcp_conns )
-		mem += entry.second->MemoryAllocation();
-
-	for ( const auto& entry : udp_conns )
-		mem += entry.second->MemoryAllocation();
-
-	for ( const auto& entry : icmp_conns )
-		mem += entry.second->MemoryAllocation();
-
-	return mem;
-	}
-
-unsigned int NetSessions::ConnectionMemoryUsageConnVals()
-	{
-	unsigned int mem = 0;
-
-	if ( run_state::terminating )
-		// Connections have been flushed already.
-		return 0;
-
-	for ( const auto& entry : tcp_conns )
-		mem += entry.second->MemoryAllocationConnVal();
-
-	for ( const auto& entry : udp_conns )
-		mem += entry.second->MemoryAllocationConnVal();
-
-	for ( const auto& entry : icmp_conns )
-		mem += entry.second->MemoryAllocationConnVal();
-
-	return mem;
-	}
-
-unsigned int NetSessions::MemoryAllocation()
-	{
-	if ( run_state::terminating )
-		// Connections have been flushed already.
-		return 0;
-
-	return ConnectionMemoryUsage()
-		+ padded_sizeof(*this)
-		+ (tcp_conns.size() * (sizeof(ConnectionMap::key_type) + sizeof(ConnectionMap::value_type)))
-		+ (udp_conns.size() * (sizeof(ConnectionMap::key_type) + sizeof(ConnectionMap::value_type)))
-		+ (icmp_conns.size() * (sizeof(ConnectionMap::key_type) + sizeof(ConnectionMap::value_type)))
-		+ detail::fragment_mgr->MemoryAllocation();
-		// FIXME: MemoryAllocation() not implemented for rest.
-		;
-	}
-
-void NetSessions::InsertConnection(ConnectionMap* m, const detail::ConnIDKey& key, Connection* conn)
-	{
-	(*m)[key] = conn;
-
-	switch ( conn->ConnTransport() )
-		{
-		case TRANSPORT_TCP:
-			stats.cumulative_TCP_conns++;
-			if ( m->size() > stats.max_TCP_conns )
-				stats.max_TCP_conns = m->size();
-			break;
-		case TRANSPORT_UDP:
-			stats.cumulative_UDP_conns++;
-			if ( m->size() > stats.max_UDP_conns )
-				stats.max_UDP_conns = m->size();
-			break;
-		case TRANSPORT_ICMP:
-			stats.cumulative_ICMP_conns++;
-			if ( m->size() > stats.max_ICMP_conns )
-				stats.max_ICMP_conns = m->size();
-			break;
-		default: break;
-		}
-	}
-
-} // namespace zeek

src/Sessions.h

@@ -1,181 +1,2 @@
-// See the file "COPYING" in the main distribution directory for copyright.
+#warning "This file is deprecated and will be removed in v5.1. Use session/Manager.h instead."
-
+#include "zeek/session/Manager.h"
-#pragma once
-
-#include <sys/types.h> // for u_char
-#include <map>
-#include <utility>
-
-#include "zeek/Frag.h"
-#include "zeek/PacketFilter.h"
-#include "zeek/NetVar.h"
-#include "zeek/analyzer/protocol/tcp/Stats.h"
-
-class ConnCompressor;
-
-namespace zeek {
-
-class EncapsulationStack;
-class Packet;
-class Connection;
-struct ConnID;
-
-namespace analyzer::stepping_stone { class SteppingStoneManager; }
-
-struct SessionStats {
-	size_t num_TCP_conns;
-	size_t max_TCP_conns;
-	uint64_t cumulative_TCP_conns;
-
-	size_t num_UDP_conns;
-	size_t max_UDP_conns;
-	uint64_t cumulative_UDP_conns;
-
-	size_t num_ICMP_conns;
-	size_t max_ICMP_conns;
-	uint64_t cumulative_ICMP_conns;
-
-	size_t num_fragments;
-	size_t max_fragments;
-	uint64_t num_packets;
-};
-
-class NetSessions {
-public:
-	NetSessions();
-	~NetSessions();
-
-	void Done();	// call to drain events before destructing
-
-	// Looks up the connection referred to by the given Val,
-	// which should be a conn_id record.  Returns nil if there's
-	// no such connection or the Val is ill-formed.
-	Connection* FindConnection(Val* v);
-
-	void Remove(Connection* c);
-	void Insert(Connection* c);
-
-	// Generating connection_pending events for all connections
-	// that are still active.
-	void Drain();
-
-	// Clears the session maps.
-	void Clear();
-
-	void GetStats(SessionStats& s) const;
-
-	void Weird(const char* name, const Packet* pkt,
-	           const char* addl = "", const char* source = "");
-	void Weird(const char* name, const IP_Hdr* ip,
-	           const char* addl = "");
-
-	detail::PacketFilter* GetPacketFilter(bool init=true)
-		{
-		if ( ! packet_filter && init )
-			packet_filter = new detail::PacketFilter(detail::packet_filter_default);
-		return packet_filter;
-		}
-
-	analyzer::stepping_stone::SteppingStoneManager* GetSTPManager()	{ return stp_manager; }
-
-	unsigned int CurrentConnections()
-		{
-		return tcp_conns.size() + udp_conns.size() + icmp_conns.size();
-		}
-
-	/**
-	 * Main entry point for processing packets destined for session analyzers. This
-	 * method is called by the packet analysis manager when after it has processed
-	 * an IP-based packet, and shouldn't be called directly from other places.
-	 *
-	 * @param t The timestamp for this packet.
-	 * @param pkt The packet being processed.
-	 * @param len The number of bytes that haven't been processed yet by packet
-	 * analysis.
-	 */
-	void ProcessTransportLayer(double t, const Packet *pkt, size_t len);
-
-	/**
-	 * Returns a wrapper IP_Hdr object if \a pkt appears to be a valid IPv4
-	 * or IPv6 header based on whether it's long enough to contain such a header,
-	 * if version given in the header matches the proto argument, and also checks
-	 * that the payload length field of that header matches the actual
-	 * length of \a pkt given by \a caplen.
-	 *
-	 * @param caplen The length of \a pkt in bytes.
-	 * @param pkt The inner IP packet data.
-	 * @param proto Either IPPROTO_IPV6 or IPPROTO_IPV4 to indicate which IP
-	 *        protocol \a pkt corresponds to.
-	 * @param inner The inner IP packet wrapper pointer to be allocated/assigned
-	 *        if \a pkt looks like a valid IP packet or at least long enough
-	 *        to hold an IP header.
-	 * @return 0 If the inner IP packet appeared valid, else -1 if \a caplen
-	 *         is greater than the supposed IP packet's payload length field, -2
-	 *         if the version of the inner header does not match proto or
-	 *         1 if \a caplen is less than the supposed packet's payload length.
-	 *         In the -1 case, \a inner may still be non-null if \a caplen was
-	 *         long enough to be an IP header, and \a inner is always non-null
-	 *         for other return values.
-	 */
-	int ParseIPPacket(int caplen, const u_char* const pkt, int proto,
-	                  IP_Hdr*& inner);
-
-	unsigned int ConnectionMemoryUsage();
-	unsigned int ConnectionMemoryUsageConnVals();
-	unsigned int MemoryAllocation();
-	analyzer::tcp::TCPStateStats tcp_stats;	// keeps statistics on TCP states
-
-protected:
-	friend class ConnCompressor;
-
-	using ConnectionMap = std::map<detail::ConnIDKey, Connection*>;
-
-	Connection* NewConn(const detail::ConnIDKey& k, double t, const ConnID* id,
-	                    const u_char* data, int proto, uint32_t flow_label,
-	                    const Packet* pkt);
-
-	Connection* LookupConn(const ConnectionMap& conns, const detail::ConnIDKey& key);
-
-	// Returns true if the port corresonds to an application
-	// for which there's a Bro analyzer (even if it might not
-	// be used by the present policy script), or it's more
-	// generally a likely server port, false otherwise.
-	//
-	// Note, port is in host order.
-	bool IsLikelyServerPort(uint32_t port, TransportProto transport_proto) const;
-
-	// Upon seeing the first packet of a connection, checks whether
-	// we want to analyze it (e.g., we may not want to look at partial
-	// connections), and, if yes, whether we should flip the roles of
-	// originator and responder (based on known ports or such).
-	// Use tcp_flags=0 for non-TCP.
-	bool WantConnection(uint16_t src_port, uint16_t dest_port,
-	                    TransportProto transport_proto,
-	                    uint8_t tcp_flags, bool& flip_roles);
-
-	// For a given protocol, checks whether the header's length as derived
-	// from lower-level headers or the length actually captured is less
-	// than that protocol's minimum header size.
-	bool CheckHeaderTrunc(int proto, uint32_t len, uint32_t caplen, const Packet *pkt);
-
-	// Inserts a new connection into the sessions map. If a connection with
-	// the same key already exists in the map, it will be overwritten by
-	// the new one.  Connection count stats get updated either way (so most
-	// cases should likely check that the key is not already in the map to
-	// avoid unnecessary incrementing of connecting counts).
-	void InsertConnection(ConnectionMap* m, const detail::ConnIDKey& key, Connection* conn);
-
-	ConnectionMap tcp_conns;
-	ConnectionMap udp_conns;
-	ConnectionMap icmp_conns;
-
-	SessionStats stats;
-
-	analyzer::stepping_stone::SteppingStoneManager* stp_manager;
-	detail::PacketFilter* packet_filter;
-};
-
-// Manager for the currently active sessions.
-extern NetSessions* sessions;
-
-} // namespace zeek

src/Stats.cc

@@ -7,7 +7,7 @@
 #include "zeek/RunState.h"
 #include "zeek/NetVar.h"
 #include "zeek/ID.h"
-#include "zeek/Sessions.h"
+#include "zeek/session/Manager.h"
 #include "zeek/Scope.h"
 #include "zeek/DNS_Mgr.h"
 #include "zeek/Trigger.h"
@@ -15,6 +15,7 @@
 #include "zeek/broker/Manager.h"
 #include "zeek/input.h"
 #include "zeek/Func.h"
+#include "zeek/packet_analysis/protocol/tcp/TCP.h"
 
 uint64_t zeek::detail::killed_by_inactivity = 0;
 uint64_t& killed_by_inactivity = zeek::detail::killed_by_inactivity;
@@ -126,25 +127,27 @@ void ProfileLogger::Log()
 		run_state::network_time, (utime + stime) - (first_utime + first_stime),
 		utime - first_utime, stime - first_stime, rtime - first_rtime));
 
-	int conn_mem_use = expensive ? sessions->ConnectionMemoryUsage() : 0;
+	int conn_mem_use = expensive ? session_mgr->SessionMemoryUsage() : 0;
 	double avg_conn_mem_use = 0;
 
-	if ( expensive && sessions->CurrentConnections() != 0 )
+	if ( expensive && session_mgr->CurrentSessions() != 0 )
-		avg_conn_mem_use = conn_mem_use / static_cast<double>(sessions->CurrentConnections());
+		avg_conn_mem_use = conn_mem_use / static_cast<double>(session_mgr->CurrentSessions());
 
+	// TODO: This previously output the number of connections, but now that we're storing sessions
+	// as well as connections, this might need to be renamed.
 	file->Write(util::fmt("%.06f Conns: total=%" PRIu64 " current=%" PRIu64 "/%" PRIi32 " mem=%" PRIi32 "K avg=%.1f table=%" PRIu32 "K connvals=%" PRIu32 "K\n",
 		run_state::network_time,
 		Connection::TotalConnections(),
 		Connection::CurrentConnections(),
-		sessions->CurrentConnections(),
+		session_mgr->CurrentSessions(),
 		conn_mem_use,
 		avg_conn_mem_use,
-		expensive ? sessions->MemoryAllocation() / 1024 : 0,
+		expensive ? session_mgr->MemoryAllocation() / 1024 : 0,
-		expensive ? sessions->ConnectionMemoryUsageConnVals() / 1024 : 0
+		expensive ? session_mgr->SessionMemoryUsageVals() / 1024 : 0
 		));
 
-	SessionStats s;
+	session::Stats s;
-	sessions->GetStats(s);
+	session_mgr->GetStats(s);
 
 	file->Write(util::fmt("%.06f Conns: tcp=%zu/%zu udp=%zu/%zu icmp=%zu/%zu\n",
 		run_state::network_time,
@@ -153,22 +156,22 @@ void ProfileLogger::Log()
 		s.num_ICMP_conns, s.max_ICMP_conns
 		));
 
-	sessions->tcp_stats.PrintStats(file,
+	packet_analysis::TCP::TCPAnalyzer::GetStats().PrintStats(file,
 			util::fmt("%.06f TCP-States:", run_state::network_time));
 
 	// Alternatively, if you prefer more compact output...
 	/*
 	file->Write(util::fmt("%.8f TCP-States: I=%d S=%d SA=%d SR=%d E=%d EF=%d ER=%d F=%d P=%d\n",
 		       run_state::network_time,
-		       sessions->tcp_stats.StateInactive(),
+		       session_mgr->tcp_stats.StateInactive(),
-		       sessions->tcp_stats.StateRequest(),
+		       session_mgr->tcp_stats.StateRequest(),
-		       sessions->tcp_stats.StateSuccRequest(),
+		       session_mgr->tcp_stats.StateSuccRequest(),
-		       sessions->tcp_stats.StateRstRequest(),
+		       session_mgr->tcp_stats.StateRstRequest(),
-		       sessions->tcp_stats.StateEstablished(),
+		       session_mgr->tcp_stats.StateEstablished(),
-		       sessions->tcp_stats.StateHalfClose(),
+		       session_mgr->tcp_stats.StateHalfClose(),
-		       sessions->tcp_stats.StateHalfRst(),
+		       session_mgr->tcp_stats.StateHalfRst(),
-		       sessions->tcp_stats.StateClosed(),
+		       session_mgr->tcp_stats.StateClosed(),
-		       sessions->tcp_stats.StatePartial()
+		       session_mgr->tcp_stats.StatePartial()
 		       ));
 	*/
 

src/Stmt.cc

@@ -34,6 +34,7 @@ const char* stmt_name(StmtTag t)
 		"<init>", "fallthrough", "while",
 		"catch-return",
 		"check-any-length",
+		"compiled-C++",
 		"null",
 	};
 
@@ -150,7 +151,7 @@ bool Stmt::SetLocationInfo(const Location* start, const Location* end)
 	// Optimistically just put it at the end.
 	map.push_back(new_mapping);
 
-	int curr_idx = map.length() - 1;
+	size_t curr_idx = map.size() - 1;
 	if ( curr_idx == 0 )
 		return true;
 

src/StmtEnums.h

@@ -20,6 +20,7 @@ enum StmtTag {
 	STMT_WHILE,
 	STMT_CATCH_RETURN,	// for reduced InlineExpr's
 	STMT_CHECK_ANY_LEN,	// internal reduced statement
+	STMT_CPP,	// compiled C++
 	STMT_NULL
 #define NUM_STMTS (int(STMT_NULL) + 1)
 };

src/Type.h

@@ -747,11 +747,14 @@ public:
 
 	const EnumValPtr& GetEnumVal(bro_int_t i);
 
+	// Only for use by C++-generated code.  Non-protected because we
+	// don't know in advance the names of the functions that will
+	// access it.
+	void AddNameInternal(const std::string& full_name, bro_int_t val);
+
 protected:
 	void AddNameInternal(const std::string& module_name,
-			const char* name, bro_int_t val, bool is_export);
+	                     const char* name, bro_int_t val, bool is_export);
-
-	void AddNameInternal(const std::string& full_name, bro_int_t val);
 
 	void CheckAndAddName(const std::string& module_name,
 	                     const char* name, bro_int_t val, bool is_export,

src/Val.cc

@@ -1459,6 +1459,8 @@ TableVal::~TableVal()
 
 void TableVal::RemoveAll()
 	{
+	delete expire_iterator;
+	expire_iterator = nullptr;
 	// Here we take the brute force approach.
 	delete table_val;
 	table_val = new PDict<TableEntryVal>;
@@ -2542,16 +2544,12 @@ void TableVal::DoExpire(double t)
 		expire_iterator = new RobustDictIterator(std::move(it));
 		}
 
-	std::unique_ptr<detail::HashKey> k = nullptr;
-	TableEntryVal* v = nullptr;
-	TableEntryVal* v_saved = nullptr;
 	bool modified = false;
 
 	for ( int i = 0; i < zeek::detail::table_incremental_step &&
 		      *expire_iterator != table_val->end_robust(); ++i, ++(*expire_iterator) )
 		{
-		k = (*expire_iterator)->GetHashKey();
+		auto v = (*expire_iterator)->GetValue<TableEntryVal*>();
-		v = (*expire_iterator)->GetValue<TableEntryVal*>();
 
 		if ( v->ExpireAccessTime() == 0 )
 			{
@@ -2564,6 +2562,7 @@ void TableVal::DoExpire(double t)
 
 		else if ( v->ExpireAccessTime() + timeout < t )
 			{
+			auto k = (*expire_iterator)->GetHashKey();
 			ListValPtr idx = nullptr;
 
 			if ( expire_func )
@@ -2574,12 +2573,14 @@ void TableVal::DoExpire(double t)
 				// It's possible that the user-provided
 				// function modified or deleted the table
 				// value, so look it up again.
-				v_saved = v;
 				v = table_val->Lookup(k.get());
 
 				if ( ! v )
 					{ // user-provided function deleted it
-					v = v_saved;
+					if ( ! expire_iterator )
+						// Entire table got dropped (e.g. clear_table() / RemoveAll())
+						break;
+
 					continue;
 					}
 
@@ -2618,7 +2619,7 @@ void TableVal::DoExpire(double t)
 	if ( modified )
 		Modified();
 
-	if ( (*expire_iterator) == table_val->end_robust() )
+	if ( ! expire_iterator || (*expire_iterator) == table_val->end_robust() )
 		{
 		delete expire_iterator;
 		expire_iterator = nullptr;
@@ -2845,11 +2846,9 @@ RecordVal::RecordVal(RecordTypePtr t, bool init_fields)
 
 	int n = rt->NumFields();
 
-	record_val = new std::vector<ZVal>;
+	record_val = new std::vector<std::optional<ZVal>>;
 	record_val->reserve(n);
 
-	is_in_record = new std::vector<bool>(n, false);
-
 	if ( run_state::is_parsing )
 		parse_time_records[rt.get()].emplace_back(NewRef{}, this);
 
@@ -2875,7 +2874,6 @@ RecordVal::RecordVal(RecordTypePtr t, bool init_fields)
 					parse_time_records[rt.get()].pop_back();
 
 				delete record_val;
-				delete is_in_record;
 				throw;
 				}
 
@@ -2907,15 +2905,9 @@ RecordVal::RecordVal(RecordTypePtr t, bool init_fields)
 			}
 
 		if ( def )
-			{
 			record_val->emplace_back(ZVal(def, def->GetType()));
-			(*is_in_record)[i] = true;
-			}
 		else
-			{
+			record_val->emplace_back(std::nullopt);
-			record_val->emplace_back(ZVal());
-			(*is_in_record)[i] = false;
-			}
 		}
 	}
 
@@ -2925,10 +2917,9 @@ RecordVal::~RecordVal()
 
 	for ( unsigned int i = 0; i < n; ++i )
 		if ( HasField(i) && IsManaged(i) )
-			ZVal::DeleteManagedType((*record_val)[i]);
+			ZVal::DeleteManagedType(*(*record_val)[i]);
 
 	delete record_val;
-	delete is_in_record;
 	}
 
 ValPtr RecordVal::SizeVal() const
@@ -2944,7 +2935,6 @@ void RecordVal::Assign(int field, ValPtr new_val)
 
 		auto t = rt->GetFieldType(field);
 		(*record_val)[field] = ZVal(new_val, t);
-		(*is_in_record)[field] = true;
 		Modified();
 		}
 	else
@@ -2956,10 +2946,9 @@ void RecordVal::Remove(int field)
 	if ( HasField(field) )
 		{
 		if ( IsManaged(field) )
-			ZVal::DeleteManagedType((*record_val)[field]);
+			ZVal::DeleteManagedType(*(*record_val)[field]);
 
-		(*record_val)[field] = ZVal();
+		(*record_val)[field] = std::nullopt;
-		(*is_in_record)[field] = false;
 
 		Modified();
 		}
@@ -2991,8 +2980,6 @@ void RecordVal::ResizeParseTimeRecords(RecordType* revised_rt)
 
 		if ( required_length > current_length )
 			{
-			rv->Reserve(required_length);
-
 			for ( auto i = current_length; i < required_length; ++i )
 				rv->AppendField(revised_rt->FieldDefault(i));
 			}
@@ -3202,13 +3189,6 @@ unsigned int RecordVal::MemoryAllocation() const
 	size += util::pad_size(record_val->capacity() * sizeof(ZVal));
 	size += padded_sizeof(*record_val);
 
-	// It's tricky sizing is_in_record since it's a std::vector
-	// specialization.  We approximate this by not scaling capacity()
-	// by sizeof(bool) but just using its raw value.  That's still
-	// presumably going to be an overestimate.
-	size += util::pad_size(is_in_record->capacity());
-	size += padded_sizeof(*is_in_record);
-
 	return size + padded_sizeof(*this);
 	}
 

src/Val.h

@@ -450,10 +450,11 @@ public:
 	// Returns a masked port number
 	static uint32_t Mask(uint32_t port_num, TransportProto port_type);
 
-protected:
+	// Only meant for use by ValManager and compiled-to-C++ script
-	friend class ValManager;
+	// functions.
 	PortVal(uint32_t p);
 
+protected:
 	void ValDescribe(ODesc* d) const override;
 	ValPtr DoClone(CloneState* state) override;
 
@@ -1103,13 +1104,13 @@ public:
 	// The following provide efficient record field assignments.
 	void Assign(int field, bool new_val)
 		{
-		(*record_val)[field].int_val = int(new_val);
+		(*record_val)[field] = ZVal(bro_int_t(new_val));
 		AddedField(field);
 		}
 
 	void Assign(int field, int new_val)
 		{
-		(*record_val)[field].int_val = new_val;
+		(*record_val)[field] = ZVal(bro_int_t(new_val));
 		AddedField(field);
 		}
 
@@ -1118,18 +1119,18 @@ public:
 	// than the other.
 	void Assign(int field, uint32_t new_val)
 		{
-		(*record_val)[field].uint_val = new_val;
+		(*record_val)[field] = ZVal(bro_uint_t(new_val));
 		AddedField(field);
 		}
 	void Assign(int field, uint64_t new_val)
 		{
-		(*record_val)[field].uint_val = new_val;
+		(*record_val)[field] = ZVal(bro_uint_t(new_val));
 		AddedField(field);
 		}
 
 	void Assign(int field, double new_val)
 		{
-		(*record_val)[field].double_val = new_val;
+		(*record_val)[field] = ZVal(new_val);
 		AddedField(field);
 		}
 
@@ -1144,8 +1145,9 @@ public:
 
 	void Assign(int field, StringVal* new_val)
 		{
-		ZVal::DeleteManagedType((*record_val)[field]);
+		if ( HasField(field) )
-		(*record_val)[field].string_val = new_val;
+			ZVal::DeleteManagedType(*(*record_val)[field]);
+		(*record_val)[field] = ZVal(new_val);
 		AddedField(field);
 		}
 	void Assign(int field, const char* new_val)
@@ -1177,29 +1179,9 @@ public:
 	void AppendField(ValPtr v)
 		{
 		if ( v )
-			{
-			(*is_in_record)[record_val->size()] = true;
 			record_val->emplace_back(ZVal(v, v->GetType()));
-			}
 		else
-			{
+			record_val->emplace_back(std::nullopt);
-			(*is_in_record)[record_val->size()] = false;
-			record_val->emplace_back(ZVal());
-			}
-		}
-
-	/**
-	 * Ensures that the record has enough internal storage for the
-	 * given number of fields.
-	 * @param n  The number of fields.
-	 */
-	void Reserve(unsigned int n)
-		{
-		record_val->reserve(n);
-		is_in_record->reserve(n);
-
-		for ( auto i = is_in_record->size(); i < n; ++i )
-			is_in_record->emplace_back(false);
 		}
 
 	/**
@@ -1217,7 +1199,7 @@ public:
 	 */
 	bool HasField(int field) const
 		{
-		return (*is_in_record)[field];
+		return (*record_val)[field] ? true : false;
 		}
 
 	/**
@@ -1230,7 +1212,7 @@ public:
 		if ( ! HasField(field) )
 			return nullptr;
 
-		return (*record_val)[field].ToVal(rt->GetFieldType(field));
+		return (*record_val)[field]->ToVal(rt->GetFieldType(field));
 		}
 
 	/**
@@ -1304,33 +1286,33 @@ public:
 		if constexpr ( std::is_same_v<T, BoolVal> ||
 		               std::is_same_v<T, IntVal> ||
 		               std::is_same_v<T, EnumVal> )
-			return record_val->operator[](field).int_val;
+			return record_val->operator[](field)->int_val;
 		else if constexpr ( std::is_same_v<T, CountVal> )
-			return record_val->operator[](field).uint_val;
+			return record_val->operator[](field)->uint_val;
 		else if constexpr ( std::is_same_v<T, DoubleVal> ||
 		                    std::is_same_v<T, TimeVal> ||
 		                    std::is_same_v<T, IntervalVal> )
-			return record_val->operator[](field).double_val;
+			return record_val->operator[](field)->double_val;
 		else if constexpr ( std::is_same_v<T, PortVal> )
-			return val_mgr->Port(record_val->at(field).uint_val);
+			return val_mgr->Port(record_val->at(field)->uint_val);
 		else if constexpr ( std::is_same_v<T, StringVal> )
-			return record_val->operator[](field).string_val->Get();
+			return record_val->operator[](field)->string_val->Get();
 		else if constexpr ( std::is_same_v<T, AddrVal> )
-			return record_val->operator[](field).addr_val->Get();
+			return record_val->operator[](field)->addr_val->Get();
 		else if constexpr ( std::is_same_v<T, SubNetVal> )
-			return record_val->operator[](field).subnet_val->Get();
+			return record_val->operator[](field)->subnet_val->Get();
 		else if constexpr ( std::is_same_v<T, File> )
-			return *(record_val->operator[](field).file_val);
+			return *(record_val->operator[](field)->file_val);
 		else if constexpr ( std::is_same_v<T, Func> )
-			return *(record_val->operator[](field).func_val);
+			return *(record_val->operator[](field)->func_val);
 		else if constexpr ( std::is_same_v<T, PatternVal> )
-			return record_val->operator[](field).re_val->Get();
+			return record_val->operator[](field)->re_val->Get();
 		else if constexpr ( std::is_same_v<T, RecordVal> )
-			return record_val->operator[](field).record_val;
+			return record_val->operator[](field)->record_val;
 		else if constexpr ( std::is_same_v<T, VectorVal> )
-			return record_val->operator[](field).vector_val;
+			return record_val->operator[](field)->vector_val;
 		else if constexpr ( std::is_same_v<T, TableVal> )
-			return record_val->operator[](field).table_val->Get();
+			return record_val->operator[](field)->table_val->Get();
 		else
 			{
 			// It's an error to reach here, although because of
@@ -1345,12 +1327,12 @@ public:
 	T GetFieldAs(int field) const
 		{
 		if constexpr ( std::is_integral_v<T> && std::is_signed_v<T> )
-			return record_val->operator[](field).int_val;
+			return record_val->operator[](field)->int_val;
 		else if constexpr ( std::is_integral_v<T> &&
 					std::is_unsigned_v<T> )
-			return record_val->operator[](field).uint_val;
+			return record_val->operator[](field)->uint_val;
 		else if constexpr ( std::is_floating_point_v<T> )
-			return record_val->operator[](field).double_val;
+			return record_val->operator[](field)->double_val;
 
 		// Note: we could add other types here using type traits,
 		// such as is_same_v<T, std::string>, etc.
@@ -1415,7 +1397,6 @@ protected:
 
 	void AddedField(int field)
 		{
-		(*is_in_record)[field] = true;
 		Modified();
 		}
 
@@ -1428,7 +1409,7 @@ private:
 	void DeleteFieldIfManaged(unsigned int field)
 		{
 		if ( HasField(field) && IsManaged(field) )
-			ZVal::DeleteManagedType((*record_val)[field]);
+			ZVal::DeleteManagedType(*(*record_val)[field]);
 		}
 
 	bool IsManaged(unsigned int offset) const
@@ -1441,12 +1422,7 @@ private:
 	RecordTypePtr rt;
 
 	// Low-level values of each of the fields.
-	std::vector<ZVal>* record_val;
+	std::vector<std::optional<ZVal>>* record_val;
-
-	// Whether a given field exists - for optional fields, and because
-	// Zeek does not enforce that non-optional fields are actually
-	// present.
-	std::vector<bool>* is_in_record;
 
 	// Whether a given field requires explicit memory management.
 	const std::vector<bool>& is_managed;
@@ -1460,6 +1436,8 @@ protected:
 	friend class Val;
 	friend class EnumType;
 
+	friend EnumValPtr make_enum__CPP(TypePtr t, int i);
+
 	template<class T, class... Ts>
 	friend IntrusivePtr<T> make_intrusive(Ts&&... args);
 
@@ -1574,6 +1552,11 @@ public:
 
 	ValPtr ValAt(unsigned int index) const	{ return At(index); }
 
+	bool Has(unsigned int index) const
+		// Version to use once std::optional implementation is merged.
+		// { return index < vector_val->size() && vector_val[index]; }
+		{ return At(index) != nullptr; }
+
 	/**
 	 * Returns the given element in a given underlying representation.
 	 * Enables efficient vector access.  Caller must ensure that the
@@ -1582,8 +1565,12 @@ public:
 	 * @param index  The position in the vector of the element to return.
 	 * @return  The element's underlying value.
 	 */
+	bro_int_t IntAt(unsigned int index) const
+		{ return (*vector_val)[index]->int_val; }
 	bro_uint_t CountAt(unsigned int index) const
 		{ return (*vector_val)[index]->uint_val; }
+	double DoubleAt(unsigned int index) const
+		{ return (*vector_val)[index]->double_val; }
 	const RecordVal* RecordValAt(unsigned int index) const
 		{ return (*vector_val)[index]->record_val; }
 	bool BoolAt(unsigned int index) const

src/ZVal.h

@@ -8,19 +8,31 @@
 
 namespace zeek {
 
-class StringVal;
 class AddrVal;
-class SubNetVal;
 class File;
 class Func;
 class ListVal;
 class OpaqueVal;
 class PatternVal;
-class TableVal;
 class RecordVal;
-class VectorVal;
+class StringVal;
+class SubNetVal;
+class TableVal;
 class Type;
 class Val;
+class VectorVal;
+
+using AddrValPtr = IntrusivePtr<AddrVal>;
+using EnumValPtr = IntrusivePtr<EnumVal>;
+using ListValPtr = IntrusivePtr<ListVal>;
+using OpaqueValPtr = IntrusivePtr<OpaqueVal>;
+using PatternValPtr = IntrusivePtr<PatternVal>;
+using RecordValPtr = IntrusivePtr<RecordVal>;
+using StringValPtr = IntrusivePtr<StringVal>;
+using SubNetValPtr = IntrusivePtr<SubNetVal>;
+using TableValPtr = IntrusivePtr<TableVal>;
+using ValPtr = IntrusivePtr<Val>;
+using VectorValPtr = IntrusivePtr<VectorVal>;
 
 // Note that a ZVal by itself is ambiguous: it doesn't track its type.
 // This makes them consume less memory and cheaper to copy.  It does
@@ -43,6 +55,35 @@ union ZVal {
 	// Construct an empty value compatible with the given type.
 	ZVal(const TypePtr& t);
 
+	// Construct directly.
+	ZVal(bro_int_t v)	{ int_val = v; }
+	ZVal(bro_uint_t v)	{ uint_val = v; }
+	ZVal(double v)		{ double_val = v; }
+
+	ZVal(StringVal* v)	{ string_val = v; }
+	ZVal(AddrVal* v)	{ addr_val = v; }
+	ZVal(SubNetVal* v)	{ subnet_val = v; }
+	ZVal(File* v)		{ file_val = v; }
+	ZVal(Func* v)		{ func_val = v; }
+	ZVal(ListVal* v)	{ list_val = v; }
+	ZVal(OpaqueVal* v)	{ opaque_val = v; }
+	ZVal(PatternVal* v)	{ re_val = v; }
+	ZVal(TableVal* v)	{ table_val = v; }
+	ZVal(RecordVal* v)	{ record_val = v; }
+	ZVal(VectorVal* v)	{ vector_val = v; }
+	ZVal(Type* v)		{ type_val = v; }
+
+	ZVal(StringValPtr v)	{ string_val = v.release(); }
+	ZVal(AddrValPtr v)	{ addr_val = v.release(); }
+	ZVal(SubNetValPtr v)	{ subnet_val = v.release(); }
+	ZVal(ListValPtr v)	{ list_val = v.release(); }
+	ZVal(OpaqueValPtr v)	{ opaque_val = v.release(); }
+	ZVal(PatternValPtr v)	{ re_val = v.release(); }
+	ZVal(TableValPtr v)	{ table_val = v.release(); }
+	ZVal(RecordValPtr v)	{ record_val = v.release(); }
+	ZVal(VectorValPtr v)	{ vector_val = v.release(); }
+	ZVal(TypePtr v)		{ type_val = v.release(); }
+
 	// Convert to a higher-level script value.  The caller needs to
 	// ensure that they're providing the correct type.
 	ValPtr ToVal(const TypePtr& t) const;

src/ZeekString.cc

@@ -6,18 +6,23 @@
 #include <ctype.h>
 #include <algorithm>
 #include <iostream>
+#include <sstream>	// Needed for unit testing
 
 #include "zeek/Val.h"
 #include "zeek/ID.h"
 #include "zeek/Reporter.h"
 #include "zeek/util.h"
 
+#include "zeek/3rdparty/doctest.h"
+
 #ifdef DEBUG
 #define DEBUG_STR(msg) DBG_LOG(zeek::DBG_STRING, msg)
 #else
 #define DEBUG_STR(msg)
 #endif
 
+using namespace std::string_literals;
+
 namespace zeek {
 
 constexpr int String::EXPANDED_STRING;
@@ -101,6 +106,19 @@ bool String::operator<(const String &bs) const
 	return Bstr_cmp(this, &bs) < 0;
 	}
 
+bool String::operator==(std::string_view s) const
+	{
+	if ( static_cast<size_t>(n) != s.size() )
+		return false;
+
+	if ( b == nullptr )
+		{
+		return s.size() == 0;
+		}
+
+	return (memcmp(b, s.data(), n) == 0);
+	}
+
 void String::Adopt(byte_vec bytes, int len)
 	{
 	Reset();
@@ -169,6 +187,7 @@ const char* String::CheckString() const
 		{
 		// Either an embedded NUL, or no final NUL.
 		char* exp_s = Render();
+
 		if ( nulTerm )
 			reporter->Error("string with embedded NUL: \"%s\"", exp_s);
 		else
@@ -343,7 +362,7 @@ String::Vec* String::Split(const String::IdxVec& indices) const
 	return result;
 	}
 
-VectorVal* String:: VecToPolicy(Vec* vec)
+VectorVal* String::VecToPolicy(Vec* vec)
 	{
 	auto result = make_intrusive<VectorVal>(id::string_vec);
 
@@ -390,11 +409,10 @@ char* String::VecToString(const Vec* vec)
 	return strdup(result.c_str());
 	}
 
-bool StringLenCmp::operator()(String * const& bst1,
+bool StringLenCmp::operator()(String* const& bst1, String* const& bst2)
-				 String * const& bst2)
 	{
 	return _increasing ? (bst1->Len() < bst2->Len()) :
-				(bst1->Len() > bst2->Len());
+	                     (bst1->Len() > bst2->Len());
 	}
 
 std::ostream& operator<<(std::ostream& os, const String& bs)
@@ -496,3 +514,235 @@ void delete_strings(std::vector<const String*>& v)
 	}
 
 } // namespace zeek
+
+TEST_SUITE_BEGIN("ZeekString");
+
+TEST_CASE("construction")
+	{
+	zeek::String s1{};
+	CHECK_EQ(s1.Len(), 0);
+	CHECK_EQ(s1.Bytes(), nullptr);
+	CHECK_EQ(s1, "");
+
+	std::string text = "abcdef";
+	zeek::byte_vec text2 = new u_char[7];
+	memcpy(text2, text.c_str(), 7);
+
+	zeek::String s2{text2, 6, false};
+	CHECK_EQ(s2.Len(), 6);
+
+	zeek::String s3{text2, 6, true};
+	CHECK_EQ(s3.Len(), 6);
+
+	zeek::String s4{"abcdef"};
+	CHECK_EQ(s4.Len(), 6);
+
+	zeek::String s5{std::string("abcdef")};
+	CHECK_EQ(s5.Len(), 6);
+
+	zeek::String s6{s5};
+	CHECK_EQ(s6.Len(), 6);
+
+	zeek::String s7{true, text2, 6};
+	CHECK_EQ(s7.Len(), 6);
+	CHECK_EQ(s7.Bytes(), text2);
+
+	// Construct a temporary reporter object for the next two tests
+	zeek::reporter = new zeek::Reporter(false);
+
+	zeek::byte_vec text3 = new u_char[7];
+	memcpy(text3, text.c_str(), 7);
+	zeek::String s8{false, text3, 6};
+	CHECK_EQ(std::string(s8.CheckString()), "<string-with-NUL>");
+
+	zeek::byte_vec text4 = new u_char[7];
+	memcpy(text4, text.c_str(), 7);
+	text4[2] = '\0';
+	zeek::String s9{false, text4, 6};
+	CHECK_EQ(std::string(s9.CheckString()), "<string-with-NUL>");
+
+	delete zeek::reporter;
+
+	zeek::byte_vec text5 = (zeek::byte_vec)malloc(7);
+	memcpy(text5, text.c_str(), 7);
+	zeek::String s10{true, text5, 6};
+	s10.SetUseFreeToDelete(1);
+	CHECK_EQ(s10.Bytes(), text5);
+	}
+
+TEST_CASE("set/assignment/comparison")
+	{
+	zeek::String s{"abc"};
+	CHECK_EQ(s, "abc");
+
+	s.Set("def");
+	CHECK_EQ(s, "def");
+
+	s.Set(std::string("ghi"));
+	CHECK_EQ(s, "ghi");
+
+	zeek::String s2{"abc"};
+	s.Set(s2);
+	CHECK_EQ(s, "abc");
+
+	zeek::String s3{"def"};
+	s = s3;
+	CHECK_EQ(s, "def");
+	CHECK_EQ(s, s3);
+	CHECK(s2 < s3);
+
+	s.Set("ghi");
+	CHECK_FALSE(s < s2);
+
+	std::string text = "abcdef";
+	zeek::byte_vec text2 = new u_char[7];
+	memcpy(text2, text.c_str(), 7);
+	s.Adopt(text2, 7);
+
+	CHECK_EQ(s, "abcdef");
+	CHECK_FALSE(s == s2);
+
+	// This is a clearly invalid string and we probably shouldn't allow it to be
+	// constructed, but this test covers one if statement in Bstr_eq.
+	zeek::String s4(false, nullptr, 3);
+	CHECK_FALSE(s4 == s2);
+
+	zeek::String s5{};
+	CHECK_LT(s5, s);
+	CHECK_FALSE(s < s5);
+	}
+
+TEST_CASE("searching/modification")
+	{
+	zeek::String s{"this is a test"};
+	auto* ss = s.GetSubstring(5, 4);
+	CHECK_EQ(*ss, "is a");
+	delete ss;
+
+	auto* ss2 = s.GetSubstring(-1, 4);
+	CHECK_EQ(ss2, nullptr);
+	ss2 = s.GetSubstring(s.Len() + 5, 4);
+	CHECK_EQ(ss2, nullptr);
+
+	zeek::String s2{"test"};
+	CHECK_EQ(s.FindSubstring(&s2), 10);
+
+	s2.ToUpper();
+	CHECK_EQ(s2, "TEST");
+
+	zeek::String::IdxVec indexes;
+	zeek::String::Vec* splits = s.Split(indexes);
+	CHECK_EQ(splits, nullptr);
+
+	indexes.insert(indexes.end(), {4, 7, 9, -1, 30});
+	splits = s.Split(indexes);
+	CHECK_EQ(splits->size(), 4);
+	CHECK_EQ(*(splits->at(0)), "this");
+	CHECK_EQ(*(splits->at(1)), " is");
+	CHECK_EQ(*(splits->at(2)), " a");
+	CHECK_EQ(*(splits->at(3)), " test");
+
+	zeek::String* s3 = concatenate(*splits);
+	CHECK_EQ(s.Len(), s3->Len());
+	CHECK_EQ(s, *s3);
+	delete s3;
+
+	char* temp = zeek::String::VecToString(splits);
+	CHECK_EQ(std::string(temp), "[this, is, a, test,]");
+	free(temp);
+
+ 	for ( auto* entry : *splits )
+		delete entry;
+	delete splits;
+	}
+
+TEST_CASE("rendering")
+	{
+	zeek::String s1("\\abcd\'\"");
+	auto* r = s1.Render(zeek::String::ESC_ESC);
+	CHECK_EQ(std::string(r), "\\\\abcd\'\"");
+	delete [] r;
+
+	r = s1.Render(zeek::String::ESC_QUOT);
+	CHECK_EQ(std::string(r), "\\abcd\\\'\\\"");
+	delete [] r;
+
+	r = s1.Render(zeek::String::ESC_ESC | zeek::String::ESC_QUOT | zeek::String::ESC_SER);
+	CHECK_EQ(std::string(r), "10 \\\\abcd\\\'\\\"");
+	delete [] r;
+
+	zeek::byte_vec text = new u_char[6];
+	text[0] = 3;
+	text[1] = 4;
+	text[2] = 5;
+	text[3] = 6;
+	text[4] = '\\';
+	text[5] = '\'';
+	zeek::String s2(false, text, 6);
+
+	r = s2.Render(zeek::String::ESC_HEX);
+	CHECK_EQ(std::string(r), "\\x03\\x04\\x05\\x06\\\'");
+	delete [] r;
+
+	int test_length = 0;
+	r = s2.Render(zeek::String::ESC_DOT, &test_length);
+	CHECK_EQ(std::string(r), "....\\\'");
+	CHECK_EQ(test_length, 7);
+	delete [] r;
+
+	r = s2.Render(zeek::String::BRO_STRING_LITERAL);
+	CHECK_EQ(std::string(r), "\\x03\\x04\\x05\\x06\\\\\\\'");
+	delete [] r;
+
+	std::ostringstream os1;
+	// This uses ESC_HEX, so it should be the same as the test above
+	os1 << s2;
+	CHECK_EQ(os1.str(), "\\x03\\x04\\x05\\x06\\\'");
+
+	std::ostringstream os2;
+	s2.Render(os2, zeek::String::ESC_HEX);
+	CHECK_EQ(os2.str(), "\\x03\\x04\\x05\\x06\\\'");
+	}
+
+TEST_CASE("read")
+	{
+	std::string text1("5 abcde");
+	std::istringstream iss1(text1);
+	zeek::String s1{};
+	s1.Read(iss1);
+	CHECK_EQ(s1, "abcde");
+
+	std::string text2("abcde");
+	std::istringstream iss2(text2);
+	zeek::String s2{};
+	// Setting to something else disables reading the serialization format
+	s2.Read(iss2, zeek::String::ESC_HEX);
+	CHECK_EQ(s2, text2);
+	}
+
+TEST_CASE("misc")
+	{
+	std::vector<const zeek::String*> sv = {new zeek::String{}, new zeek::String{}};
+	CHECK_EQ(sv.size(), 2);
+	zeek::delete_strings(sv);
+	CHECK_EQ(sv.size(), 0);
+
+	std::vector<zeek::data_chunk_t> dv = {{5, "abcde"}, {6, "fghijk"}};
+	auto* s = zeek::concatenate(dv);
+	CHECK_EQ(*s, "abcdefghijk");
+	delete s;
+
+	std::vector<zeek::String*> sv2 = {new zeek::String{"abcde"}, new zeek::String{"fghi"}};
+	std::sort(sv2.begin(), sv2.end(), zeek::StringLenCmp(true));
+	CHECK_EQ(*(sv2.front()), "fghi");
+	CHECK_EQ(*(sv2.back()), "abcde");
+
+	std::sort(sv2.begin(), sv2.end(), zeek::StringLenCmp(false));
+	CHECK_EQ(*(sv2.front()), "abcde");
+	CHECK_EQ(*(sv2.back()), "fghi");
+
+	for ( auto* entry : sv2 )
+		delete entry;
+	}
+
+TEST_SUITE_END();

src/ZeekString.h

@@ -19,6 +19,12 @@ class VectorVal;
 
 typedef u_char* byte_vec;
 
+/**
+ * A container type for holding blocks of byte data. This can be used for
+ * character strings, but is not limited to that alone. This class provides
+ * methods for rendering byte data into character strings, including
+ * conversions of non-printable characters into other representations.
+ */
 class String {
 public:
 	typedef std::vector<String*> Vec;
@@ -49,6 +55,7 @@ public:
 	const String& operator=(const String& bs);
 	bool operator==(const String& bs) const;
 	bool operator<(const String& bs) const;
+	bool operator==(std::string_view s) const;
 
 	byte_vec Bytes() const	{ return b; }
 	int Len() const	{ return n; }
@@ -71,6 +78,13 @@ public:
 	void SetUseFreeToDelete(int use_it)
 		{ use_free_to_delete = use_it; }
 
+	/**
+	 * Returns a character-string representation of the stored bytes. This
+	 * method doesn't do any extra rendering or character conversions. If
+	 * null characters are found in the middle of the data or if the data
+	 * is missing a closing null character, an error string is returned and
+	 * a error is reported.
+	 */
 	const char* CheckString() const;
 
 	enum render_style {

src/analyzer/Analyzer.cc

@@ -787,7 +787,7 @@ void Analyzer::UpdateConnVal(RecordVal *conn_val)
 
 const RecordValPtr& Analyzer::ConnVal()
 	{
-	return conn->ConnVal();
+	return conn->GetVal();
 	}
 
 void Analyzer::Event(EventHandlerPtr f, const char* name)
@@ -880,31 +880,4 @@ void SupportAnalyzer::ForwardUndelivered(uint64_t seq, int len, bool is_orig)
 	}
 
 
-void TransportLayerAnalyzer::Done()
-	{
-	Analyzer::Done();
-	}
-
-void TransportLayerAnalyzer::SetContentsFile(unsigned int /* direction */,
-                                             FilePtr /* f */)
-	{
-	reporter->Error("analyzer type does not support writing to a contents file");
-	}
-
-FilePtr TransportLayerAnalyzer::GetContentsFile(unsigned int /* direction */) const
-	{
-	reporter->Error("analyzer type does not support writing to a contents file");
-	return nullptr;
-	}
-
-void TransportLayerAnalyzer::PacketContents(const u_char* data, int len)
-	{
-	if ( packet_contents && len > 0 )
-		{
-		String* cbs = new String(data, len, true);
-		auto contents = make_intrusive<StringVal>(cbs);
-		EnqueueConnEvent(packet_contents, ConnVal(), std::move(contents));
-		}
-	}
-
 } // namespace zeek::analyzer

src/analyzer/Analyzer.h

@@ -25,6 +25,7 @@ using FilePtr = zeek::IntrusivePtr<File>;
 using RecordValPtr = zeek::IntrusivePtr<RecordVal>;
 
 namespace detail { class Rule; }
+namespace packet_analysis::IP { class IPBasedAnalyzer; }
 
 } // namespace zeek
 
@@ -601,6 +602,7 @@ protected:
 	friend class Manager;
 	friend class zeek::Connection;
 	friend class zeek::analyzer::tcp::TCP_ApplicationAnalyzer;
+	friend class zeek::packet_analysis::IP::IPBasedAnalyzer;
 
 	/**
 	 * Return a string represantation of an analyzer, containing its name
@@ -844,83 +846,4 @@ private:
 #define CONTENTS_RESP 2
 #define CONTENTS_BOTH 3
 
-/**
- * Base class for analyzers parsing transport-layer protocols.
- */
-class TransportLayerAnalyzer : public Analyzer {
-public:
-	/**
-	 * Constructor.
-	 *
-	 * @param name A name for the protocol the analyzer is parsing. The
-	 * name must match the one the corresponding Component registers.
-	 *
-	 * @param conn The connection the analyzer is associated with.
-	 */
-	TransportLayerAnalyzer(const char* name, Connection* conn)
-		: Analyzer(name, conn)	{ pia = nullptr; }
-
-	/**
-	 * Overridden from parent class.
-	 */
-	void Done() override;
-
-	/**
-	 * Returns true if the analyzer determines that in fact a new
-	 * connection has started without the connection statement having
-	 * terminated the previous one, i.e., the new data is arriving at
-	 * what's the analyzer for the previous instance. This is used only
-	 * for TCP.
-	 */
-	virtual bool IsReuse(double t, const u_char* pkt) = 0;
-
-	/**
-	 * Associates a file with the analyzer in which to record all
-	 * analyzed input. This must only be called with derived classes that
-	 * overide the method; the default implementation will abort.
-	 *
-	 * @param direction One of the CONTENTS_* constants indicating which
-	 * direction of the input stream is to be recorded.
-	 *
-	 * @param f The file to record to.
-	 *
-	 */
-	virtual void SetContentsFile(unsigned int direction, FilePtr f);
-
-	/**
-	 * Returns an associated contents file, if any.  This must only be
-	 * called with derived classes that overide the method; the default
-	 * implementation will abort.
-	 *
-	 * @param direction One of the CONTENTS_* constants indicating which
-	 * direction the query is for.
-	 */
-	virtual FilePtr GetContentsFile(unsigned int direction) const;
-
-	/**
-	 * Associates a PIA with this analyzer. A PIA takes the
-	 * transport-layer input and determine which protocol analyzer(s) to
-	 * use for parsing it.
-	 */
-	void SetPIA(analyzer::pia::PIA* arg_PIA)	{ pia = arg_PIA; }
-
-	/**
-	 * Returns the associated PIA, or null of none. Does not take
-	 * ownership.
-	 */
-	analyzer::pia::PIA* GetPIA() const		{ return pia; }
-
-	/**
-	 * Helper to raise a \c packet_contents event.
-	 *
-	 * @param data The dass to pass to the event.
-	 *
-	 * @param len The length of \a data.
-	 */
-	void PacketContents(const u_char* data, int len);
-
-private:
-	analyzer::pia::PIA* pia;
-};
-
 } // namespace zeek::analyzer

src/analyzer/Component.cc

@@ -8,8 +8,9 @@
 
 namespace zeek::analyzer {
 
-Component::Component(const std::string& name, factory_callback arg_factory, Tag::subtype_t arg_subtype, bool arg_enabled, bool arg_partial)
+Component::Component(const std::string& name, factory_callback arg_factory, Tag::subtype_t arg_subtype,
-	: plugin::Component(plugin::component::ANALYZER, name),
+                     bool arg_enabled, bool arg_partial, bool arg_adapter)
+	: plugin::Component(arg_adapter ? plugin::component::SESSION_ADAPTER : plugin::component::ANALYZER, name),
 	  plugin::TaggedComponent<analyzer::Tag>(arg_subtype)
 	{
 	factory = arg_factory;

src/analyzer/Component.h

@@ -53,12 +53,16 @@ public:
 	 * manager, including from script-land.
 	 *
 	 * @param partial If true, the analyzer can deal with payload from
-	 * partial connections, i.e., when Bro enters the stream mid-way
+	 * partial connections, i.e., when Zeek enters the stream mid-way
 	 * after not seeing the beginning. Note that handling of partial
 	 * connections has generally not seen much testing yet as virtually
 	 * no existing analyzer supports it.
+	 *
+	 * @param adapter If true, this analyzer is a session adapter from
+	 * the packet analyzer framework.
 	 */
-	Component(const std::string& name, factory_callback factory, Tag::subtype_t subtype = 0, bool enabled = true, bool partial = false);
+	Component(const std::string& name, factory_callback factory, Tag::subtype_t subtype = 0,
+	          bool enabled = true, bool partial = false, bool adapter = false);
 
 	/**
 	 * Destructor.

src/analyzer/Manager.cc

@@ -8,11 +8,11 @@
 #include "zeek/RunState.h"
 
 #include "zeek/analyzer/protocol/conn-size/ConnSize.h"
-#include "zeek/analyzer/protocol/icmp/ICMP.h"
 #include "zeek/analyzer/protocol/pia/PIA.h"
 #include "zeek/analyzer/protocol/stepping-stone/SteppingStone.h"
 #include "zeek/analyzer/protocol/tcp/TCP.h"
-#include "zeek/analyzer/protocol/udp/UDP.h"
+#include "zeek/packet_analysis/protocol/ip/IPBasedAnalyzer.h"
+#include "zeek/packet_analysis/protocol/ip/SessionAdapter.h"
 
 #include "zeek/plugin/Manager.h"
 
@@ -66,12 +66,6 @@ Manager::Manager()
 
 Manager::~Manager()
 	{
-	for ( analyzer_map_by_port::const_iterator i = analyzers_by_port_tcp.begin(); i != analyzers_by_port_tcp.end(); i++ )
-		delete i->second;
-
-	for ( analyzer_map_by_port::const_iterator i = analyzers_by_port_udp.begin(); i != analyzers_by_port_udp.end(); i++ )
-		delete i->second;
-
 	// Clean up expected-connection table.
 	while ( conns_by_timeout.size() )
 		{
@@ -81,14 +75,6 @@ Manager::~Manager()
 		}
 	}
 
-void Manager::InitPreScript()
-	{
-	// Cache these tags.
-	analyzer_connsize = GetComponentTag("CONNSIZE");
-	analyzer_stepping = GetComponentTag("STEPPINGSTONE");
-	analyzer_tcpstats = GetComponentTag("TCPSTATS");
-	}
-
 void Manager::InitPostScript()
 	{
 	const auto& id = detail::global_scope()->Find("Tunnel::vxlan_ports");
@@ -115,24 +101,16 @@ void Manager::DumpDebug()
 	DBG_LOG(DBG_ANALYZER, " ");
 	DBG_LOG(DBG_ANALYZER, "Analyzers by port:");
 
-	for ( analyzer_map_by_port::const_iterator i = analyzers_by_port_tcp.begin(); i != analyzers_by_port_tcp.end(); i++ )
+	if ( packet_analysis::AnalyzerPtr tcp = packet_mgr->GetAnalyzer("TCP") )
 		{
-		std::string s;
+		auto* ipba = static_cast<packet_analysis::IP::IPBasedAnalyzer*>(tcp.get());
-
+		ipba->DumpPortDebug();
-		for ( tag_set::const_iterator j = i->second->begin(); j != i->second->end(); j++ )
-			s += std::string(GetComponentName(*j)) + " ";
-
-		DBG_LOG(DBG_ANALYZER, "    %d/tcp: %s", i->first, s.c_str());
 		}
 
-	for ( analyzer_map_by_port::const_iterator i = analyzers_by_port_udp.begin(); i != analyzers_by_port_udp.end(); i++ )
+	if ( packet_analysis::AnalyzerPtr udp = packet_mgr->GetAnalyzer("UDP") )
 		{
-		std::string s;
+		auto* ipba = static_cast<packet_analysis::IP::IPBasedAnalyzer*>(udp.get());
-
+		ipba->DumpPortDebug();
-		for ( tag_set::const_iterator j = i->second->begin(); j != i->second->end(); j++ )
-			s += std::string(GetComponentName(*j)) + " ";
-
-		DBG_LOG(DBG_ANALYZER, "    %d/udp: %s", i->first, s.c_str());
 		}
 
 #endif
@@ -254,34 +232,38 @@ bool Manager::UnregisterAnalyzerForPort(EnumVal* val, PortVal* port)
 
 bool Manager::RegisterAnalyzerForPort(const Tag& tag, TransportProto proto, uint32_t port)
 	{
-	tag_set* l = LookupPort(proto, port, true);
+	// TODO: this class is becoming more generic and removing a lot of the
+	// checks for protocols, but this part might need to stay like this.
+	packet_analysis::AnalyzerPtr analyzer;
+	if ( proto == TRANSPORT_TCP )
+		analyzer = packet_mgr->GetAnalyzer("TCP");
+	else if ( proto == TRANSPORT_UDP )
+		analyzer = packet_mgr->GetAnalyzer("UDP");
 
-	if ( ! l )
+	if ( ! analyzer )
 		return false;
 
-#ifdef DEBUG
+	auto* ipba = static_cast<packet_analysis::IP::IPBasedAnalyzer*>(analyzer.get());
-	const char* name = GetComponentName(tag).c_str();
-	DBG_LOG(DBG_ANALYZER, "Registering analyzer %s for port %" PRIu32 "/%d", name, port, proto);
-#endif
 
-	l->insert(tag);
+	return ipba->RegisterAnalyzerForPort(tag, port);
-	return true;
 	}
 
 bool Manager::UnregisterAnalyzerForPort(const Tag& tag, TransportProto proto, uint32_t port)
 	{
-	tag_set* l = LookupPort(proto, port, true);
+	// TODO: this class is becoming more generic and removing a lot of the
+	// checks for protocols, but this part might need to stay like this.
+	packet_analysis::AnalyzerPtr analyzer;
+	if ( proto == TRANSPORT_TCP )
+		analyzer = packet_mgr->GetAnalyzer("TCP");
+	else if ( proto == TRANSPORT_UDP )
+		analyzer = packet_mgr->GetAnalyzer("UDP");
 
-	if ( ! l )
+	if ( ! analyzer )
-		return true;  // still a "successful" unregistration
+		return false;
 
-#ifdef DEBUG
+	auto* ipba = static_cast<packet_analysis::IP::IPBasedAnalyzer*>(analyzer.get());
-	const char* name = GetComponentName(tag).c_str();
-	DBG_LOG(DBG_ANALYZER, "Unregistering analyzer %s for port %" PRIu32 "/%d", name, port, proto);
-#endif
 
-	l->erase(tag);
+	return ipba->UnregisterAnalyzerForPort(tag, port);
-	return true;
 	}
 
 Analyzer* Manager::InstantiateAnalyzer(const Tag& tag, Connection* conn)
@@ -323,181 +305,6 @@ Analyzer* Manager::InstantiateAnalyzer(const char* name, Connection* conn)
 	return tag ? InstantiateAnalyzer(tag, conn) : nullptr;
 	}
 
-Manager::tag_set* Manager::LookupPort(TransportProto proto, uint32_t port, bool add_if_not_found)
-	{
-	analyzer_map_by_port* m = nullptr;
-
-	switch ( proto ) {
-	case TRANSPORT_TCP:
-		m = &analyzers_by_port_tcp;
-		break;
-
-	case TRANSPORT_UDP:
-		m = &analyzers_by_port_udp;
-		break;
-
-	default:
-		reporter->InternalWarning("unsupported transport protocol in analyzer::Manager::LookupPort");
-		return nullptr;
-	}
-
-	analyzer_map_by_port::const_iterator i = m->find(port);
-
-	if ( i != m->end() )
-		return i->second;
-
-	if ( ! add_if_not_found )
-		return nullptr;
-
-	tag_set* l = new tag_set;
-	m->insert(std::make_pair(port, l));
-	return l;
-	}
-
-Manager::tag_set* Manager::LookupPort(PortVal* val, bool add_if_not_found)
-	{
-	return LookupPort(val->PortType(), val->Port(), add_if_not_found);
-	}
-
-bool Manager::BuildInitialAnalyzerTree(Connection* conn)
-	{
-	analyzer::tcp::TCP_Analyzer* tcp = nullptr;
-	TransportLayerAnalyzer* root = nullptr;
-	analyzer::pia::PIA* pia = nullptr;
-	bool check_port = false;
-
-	switch ( conn->ConnTransport() ) {
-
-	case TRANSPORT_TCP:
-		root = tcp = new analyzer::tcp::TCP_Analyzer(conn);
-		pia = new analyzer::pia::PIA_TCP(conn);
-		check_port = true;
-		DBG_ANALYZER(conn, "activated TCP analyzer");
-		break;
-
-	case TRANSPORT_UDP:
-		root = new analyzer::udp::UDP_Analyzer(conn);
-		pia = new analyzer::pia::PIA_UDP(conn);
-		check_port = true;
-		DBG_ANALYZER(conn, "activated UDP analyzer");
-		break;
-
-	case TRANSPORT_ICMP: {
-		root = new analyzer::icmp::ICMP_Analyzer(conn);
-		DBG_ANALYZER(conn, "activated ICMP analyzer");
-		break;
-		}
-
-	default:
-		reporter->InternalWarning("unknown protocol can't build analyzer tree");
-		return false;
-	}
-
-	bool scheduled = ApplyScheduledAnalyzers(conn, false, root);
-
-	// Hmm... Do we want *just* the expected analyzer, or all
-	// other potential analyzers as well?  For now we only take
-	// the scheduled ones.
-	if ( ! scheduled )
-		{ // Let's see if it's a port we know.
-		if ( check_port && ! zeek::detail::dpd_ignore_ports )
-			{
-			int resp_port = ntohs(conn->RespPort());
-			tag_set* ports = LookupPort(conn->ConnTransport(), resp_port, false);
-
-			if ( ports )
-				{
-				for ( tag_set::const_iterator j = ports->begin(); j != ports->end(); ++j )
-					{
-					Analyzer* analyzer = analyzer_mgr->InstantiateAnalyzer(*j, conn);
-
-					if ( ! analyzer )
-						continue;
-
-					root->AddChildAnalyzer(analyzer, false);
-					DBG_ANALYZER_ARGS(conn, "activated %s analyzer due to port %d",
-					                  analyzer_mgr->GetComponentName(*j).c_str(), resp_port);
-					}
-				}
-			}
-		}
-
-	if ( tcp )
-		{
-		// We have to decide whether to reassamble the stream.
-		// We turn it on right away if we already have an app-layer
-		// analyzer, reassemble_first_packets is true, or the user
-		// asks us to do so.  In all other cases, reassembly may
-		// be turned on later by the TCP PIA.
-
-		bool reass = root->GetChildren().size() ||
-				zeek::detail::dpd_reassemble_first_packets ||
-				zeek::detail::tcp_content_deliver_all_orig ||
-				zeek::detail::tcp_content_deliver_all_resp;
-
-		if ( tcp_contents && ! reass )
-			{
-			static auto tcp_content_delivery_ports_orig = id::find_val<TableVal>("tcp_content_delivery_ports_orig");
-			static auto tcp_content_delivery_ports_resp = id::find_val<TableVal>("tcp_content_delivery_ports_resp");
-			const auto& dport = val_mgr->Port(ntohs(conn->RespPort()), TRANSPORT_TCP);
-
-			if ( ! reass )
-				reass = (bool)tcp_content_delivery_ports_orig->FindOrDefault(dport);
-
-			if ( ! reass )
-				reass = (bool)tcp_content_delivery_ports_resp->FindOrDefault(dport);
-			}
-
-		if ( reass )
-			tcp->EnableReassembly();
-
-		if ( IsEnabled(analyzer_stepping) )
-			{
-			// Add a SteppingStone analyzer if requested.  The port
-			// should really not be hardcoded here, but as it can
-			// handle non-reassembled data, it doesn't really fit into
-			// our general framing ...  Better would be to turn it
-			// on *after* we discover we have interactive traffic.
-			uint16_t resp_port = ntohs(conn->RespPort());
-			if ( resp_port == 22 || resp_port == 23 || resp_port == 513 )
-				{
-				static auto stp_skip_src = id::find_val<TableVal>("stp_skip_src");
-				auto src = make_intrusive<AddrVal>(conn->OrigAddr());
-
-				if ( ! stp_skip_src->FindOrDefault(src) )
-					tcp->AddChildAnalyzer(new analyzer::stepping_stone::SteppingStone_Analyzer(conn), false);
-				}
-			}
-
-		if ( IsEnabled(analyzer_tcpstats) )
-			// Add TCPStats analyzer. This needs to see packets so
-			// we cannot add it as a normal child.
-			tcp->AddChildPacketAnalyzer(new analyzer::tcp::TCPStats_Analyzer(conn));
-
-		if ( IsEnabled(analyzer_connsize) )
-			// Add ConnSize analyzer. Needs to see packets, not stream.
-			tcp->AddChildPacketAnalyzer(new analyzer::conn_size::ConnSize_Analyzer(conn));
-		}
-
-	else
-		{
-		if ( IsEnabled(analyzer_connsize) )
-			// Add ConnSize analyzer. Needs to see packets, not stream.
-			root->AddChildAnalyzer(new analyzer::conn_size::ConnSize_Analyzer(conn));
-		}
-
-	if ( pia )
-		root->AddChildAnalyzer(pia->AsAnalyzer());
-
-	conn->SetRootAnalyzer(root, pia);
-	root->Init();
-	root->InitChildren();
-
-	PLUGIN_HOOK_VOID(HOOK_SETUP_ANALYZER_TREE, HookSetupAnalyzerTree(conn));
-
-	return true;
-	}
-
 void Manager::ExpireScheduledAnalyzers()
 	{
 	if ( ! run_state::network_time )
@@ -607,10 +414,11 @@ Manager::tag_set Manager::GetScheduled(const Connection* conn)
 	return result;
 	}
 
-bool Manager::ApplyScheduledAnalyzers(Connection* conn, bool init, TransportLayerAnalyzer* parent)
+bool Manager::ApplyScheduledAnalyzers(Connection* conn, bool init,
+                                      packet_analysis::IP::SessionAdapter* parent)
 	{
 	if ( ! parent )
-		parent = conn->GetRootAnalyzer();
+		parent = conn->GetSessionAdapter();
 
 	if ( ! parent )
 		return false;
@@ -628,7 +436,7 @@ bool Manager::ApplyScheduledAnalyzers(Connection* conn, bool init, TransportLaye
 
 		if ( scheduled_analyzer_applied )
 			conn->EnqueueEvent(scheduled_analyzer_applied, nullptr,
-			                   conn->ConnVal(), it->AsVal());
+			                   conn->GetVal(), it->AsVal());
 
 		DBG_ANALYZER_ARGS(conn, "activated %s analyzer as scheduled",
 		                  analyzer_mgr->GetComponentName(*it).c_str());

src/analyzer/Manager.h

@@ -35,6 +35,14 @@
 #include "zeek/analyzer/analyzer.bif.h"
 
 namespace zeek {
+
+namespace packet_analysis::IP {
+
+class IPBasedAnalyzer;
+class SessionAdapter;
+
+} // namespace packet_analysis::IP
+
 namespace analyzer {
 
 /**
@@ -59,12 +67,6 @@ public:
 	 */
 	~Manager();
 
-	/**
-	 * First-stage initializion of the manager. This is called early on
-	 * during Bro's initialization, before any scripts are processed.
-	 */
-	void InitPreScript();
-
 	/**
 	 * Second-stage initialization of the manager. This is called late
 	 * during Bro's initialization after any scripts are processed.
@@ -238,17 +240,6 @@ public:
 	 */
 	Analyzer* InstantiateAnalyzer(const char* name, Connection* c);
 
-	/**
-	 * Given the first packet of a connection, builds its initial
-	 * analyzer tree.
-	 *
-	 * @param conn The connection to add the initial set of analyzers to.
-	 *
-	 * @return False if the tree cannot be build; that's usually an
-	 * internally error.
-	 */
-	bool BuildInitialAnalyzerTree(Connection* conn);
-
 	/**
 	 * Schedules a particular analyzer for an upcoming connection. Once
 	 * the connection is seen, BuildInitAnalyzerTree() will add the
@@ -313,7 +304,7 @@ public:
 	 * @return True if at least one scheduled analyzer was found.
 	 */
 	bool ApplyScheduledAnalyzers(Connection* conn, bool init_and_event = true,
-	                             TransportLayerAnalyzer* parent = nullptr);
+	                             packet_analysis::IP::SessionAdapter* parent = nullptr);
 
 	/**
 	 * Schedules a particular analyzer for an upcoming connection. Once
@@ -345,22 +336,13 @@ public:
 
 private:
 
-	using tag_set = std::set<Tag>;
+	friend class packet_analysis::IP::IPBasedAnalyzer;
-	using analyzer_map_by_port = std::map<uint32_t, tag_set*>;
 
-	tag_set* LookupPort(PortVal* val, bool add_if_not_found);
+	using tag_set = std::set<Tag>;
-	tag_set* LookupPort(TransportProto proto, uint32_t port, bool add_if_not_found);
 
 	tag_set GetScheduled(const Connection* conn);
 	void ExpireScheduledAnalyzers();
 
-	analyzer_map_by_port analyzers_by_port_tcp;
-	analyzer_map_by_port analyzers_by_port_udp;
-
-	Tag analyzer_connsize;
-	Tag analyzer_stepping;
-	Tag analyzer_tcpstats;
-
 	//// Data structures to track analyzed scheduled for future connections.
 
 	// The index for a scheduled connection.

src/analyzer/protocol/CMakeLists.txt

@@ -9,11 +9,11 @@ add_subdirectory(dns)
 add_subdirectory(file)
 add_subdirectory(finger)
 add_subdirectory(ftp)
+add_subdirectory(geneve)
 add_subdirectory(gnutella)
 add_subdirectory(gssapi)
 add_subdirectory(gtpv1)
 add_subdirectory(http)
-add_subdirectory(icmp)
 add_subdirectory(ident)
 add_subdirectory(imap)
 add_subdirectory(irc)
@@ -44,7 +44,6 @@ add_subdirectory(stepping-stone)
 add_subdirectory(syslog)
 add_subdirectory(tcp)
 add_subdirectory(teredo)
-add_subdirectory(udp)
 add_subdirectory(vxlan)
 add_subdirectory(xmpp)
 add_subdirectory(zip)

src/analyzer/protocol/ayiya/AYIYA.cc

@@ -4,6 +4,7 @@
 
 #include "zeek/Func.h"
 #include "zeek/packet_analysis/protocol/iptunnel/IPTunnel.h"
+#include "zeek/packet_analysis/protocol/ip/IP.h"
 
 namespace zeek::analyzer::ayiya {
 
@@ -45,8 +46,8 @@ void AYIYA_Analyzer::DeliverPacket(int len, const u_char* data, bool orig, uint6
 	caplen -= inner_packet_offset;
 	inner_packet_offset = -1;
 
-	IP_Hdr* inner = nullptr;
+	std::unique_ptr<IP_Hdr> inner;
-	int result = sessions->ParseIPPacket(len, data, next_header, inner);
+	int result = packet_analysis::IP::ParsePacket(len, data, next_header, inner);
 
 	if ( result == 0 )
 		{
@@ -65,9 +66,6 @@ void AYIYA_Analyzer::DeliverPacket(int len, const u_char* data, bool orig, uint6
 	else
 		ProtocolViolation("AYIYA payload length",
 		                  reinterpret_cast<const char*>(data), len);
-
-	if ( result != 0 )
-		delete inner;
 	}
 
 } // namespace zeek::analyzer::ayiya

src/analyzer/protocol/ayiya/ayiya-analyzer.pac

@@ -1,5 +1,4 @@
 %extern{
-#include "zeek/Sessions.h"
 #include "zeek/Conn.h"
 #include "zeek/analyzer/protocol/ayiya/AYIYA.h"
 %}

src/analyzer/protocol/conn-size/functions.bif

@@ -1,11 +1,11 @@
 %%{
 #include "zeek/analyzer/protocol/conn-size/ConnSize.h"
 #include "zeek/Reporter.h"
-#include "zeek/Sessions.h"
+#include "zeek/session/Manager.h"
 
 static zeek::analyzer::Analyzer* GetConnsizeAnalyzer(zeek::Val* cid)
 	{
-	zeek::Connection* c = zeek::sessions->FindConnection(cid);
+	zeek::Connection* c = zeek::session_mgr->FindConnection(cid);
 	if ( ! c )
 		return nullptr;
 

src/analyzer/protocol/dhcp/DHCP.h

@@ -1,7 +1,5 @@
 #pragma once
 
-#include "zeek/analyzer/protocol/udp/UDP.h"
-
 #include "analyzer/protocol/dhcp/dhcp_pac.h"
 
 namespace zeek::analyzer::dhcp {

src/analyzer/protocol/dnp3/DNP3.h

@@ -1,7 +1,6 @@
 #pragma once
 
 #include "zeek/analyzer/protocol/tcp/TCP.h"
-#include "zeek/analyzer/protocol/udp/UDP.h"
 
 #include "analyzer/protocol/dnp3/dnp3_pac.h"
 

src/analyzer/protocol/dns/DNS.cc

@@ -11,7 +11,7 @@
 
 #include "zeek/ZeekString.h"
 #include "zeek/NetVar.h"
-#include "zeek/Sessions.h"
+#include "zeek/session/Manager.h"
 #include "zeek/Event.h"
 #include "zeek/RunState.h"
 
@@ -346,7 +346,7 @@ bool DNS_Interpreter::ParseAnswer(detail::DNS_MsgInfo* msg,
 		case detail::TYPE_DS:
 			status = ParseRR_DS(msg, data, len, rdlength, msg_start);
 			break;
-		
+
 		case detail::TYPE_BINDS:
 			status = ParseRR_BINDS(msg, data, len, rdlength, msg_start);
 			break;
@@ -354,7 +354,7 @@ bool DNS_Interpreter::ParseAnswer(detail::DNS_MsgInfo* msg,
 		case detail::TYPE_SSHFP:
 			status = ParseRR_SSHFP(msg, data, len, rdlength, msg_start);
 			break;
-		
+
 		case detail::TYPE_LOC:
 			status = ParseRR_LOC(msg, data, len, rdlength, msg_start);
 			break;
@@ -2261,7 +2261,7 @@ void DNS_Analyzer::ExpireTimer(double t)
 	if ( t - Conn()->LastTime() >= zeek::detail::dns_session_timeout - 1.0 || run_state::terminating )
 		{
 		Event(connection_timeout);
-		sessions->Remove(Conn());
+		session_mgr->Remove(Conn());
 		}
 	else
 		ADD_ANALYZER_TIMER(&DNS_Analyzer::ExpireTimer,

src/analyzer/protocol/geneve/CMakeLists.txt

@@ -0,0 +1,8 @@
+include(ZeekPlugin)
+
+include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR})
+
+zeek_plugin_begin(Zeek Geneve)
+zeek_plugin_cc(Geneve.cc Plugin.cc)
+zeek_plugin_bif(events.bif)
+zeek_plugin_end()

src/analyzer/protocol/geneve/Geneve.cc

@@ -0,0 +1,88 @@
+// See the file  in the main distribution directory for copyright.
+
+#include "zeek/analyzer/protocol/geneve/Geneve.h"
+
+#include "zeek/Conn.h"
+#include "zeek/IP.h"
+#include "zeek/RunState.h"
+#include "zeek/packet_analysis/protocol/iptunnel/IPTunnel.h"
+
+#include "zeek/analyzer/protocol/geneve/events.bif.h"
+
+namespace zeek::analyzer::geneve {
+
+void Geneve_Analyzer::Done()
+	{
+	Analyzer::Done();
+	Event(udp_session_done);
+	}
+
+void Geneve_Analyzer::DeliverPacket(int len, const u_char* data, bool orig,
+                                    uint64_t seq, const IP_Hdr* ip, int caplen)
+	{
+	Analyzer::DeliverPacket(len, data, orig, seq, ip, caplen);
+
+	// Outer Ethernet, IP, and UDP layers already skipped.
+	// Also, generic UDP analyzer already checked/guarantees caplen >= len.
+
+	constexpr auto tunnel_header_len = 8;
+
+	if ( len < tunnel_header_len )
+		{
+		ProtocolViolation("Geneve header truncation", reinterpret_cast<const char*>(data), len);
+		return;
+		}
+
+	auto outer = Conn()->GetEncapsulation();
+
+	if ( outer && outer->Depth() >= BifConst::Tunnel::max_depth )
+		{
+		Weird("tunnel_depth");
+		return;
+		}
+
+	if ( ! outer )
+		outer = std::make_shared<EncapsulationStack>();
+
+	EncapsulatingConn inner(Conn(), BifEnum::Tunnel::GENEVE);
+	outer->Add(inner);
+
+	auto tunnel_opt_len = data[0] << 1;
+	auto vni = (data[4] << 16) + (data[5] << 8) + (data[6] << 0);
+
+	if ( len < tunnel_header_len  + tunnel_opt_len )
+		{
+		ProtocolViolation("Geneve option header truncation", reinterpret_cast<const char*>(data), len);
+		return;
+		}
+
+	// Skip over the Geneve headers and create a new packet.
+	data += tunnel_header_len + tunnel_opt_len;
+	caplen -= tunnel_header_len + tunnel_opt_len;
+	len -= tunnel_header_len + tunnel_opt_len;
+
+	pkt_timeval ts;
+	ts.tv_sec = static_cast<time_t>(run_state::current_timestamp);
+	ts.tv_usec = static_cast<suseconds_t>((run_state::current_timestamp - static_cast<double>(ts.tv_sec)) * 1000000);
+	Packet pkt(DLT_EN10MB, &ts, caplen, len, data);
+	pkt.encap = outer;
+
+	if ( ! packet_mgr->ProcessInnerPacket(&pkt) )
+		{
+		ProtocolViolation("Geneve invalid inner packet");
+		return;
+		}
+
+	// This isn't really an error. It's just that the inner packet wasn't an IP packet (like ARP).
+	// Just return without reporting a violation.
+	if ( ! pkt.ip_hdr )
+		return;
+
+	ProtocolConfirmation();
+
+	if ( geneve_packet )
+		Conn()->EnqueueEvent(geneve_packet, nullptr, ConnVal(),
+		                     pkt.ip_hdr->ToPktHdrVal(), val_mgr->Count(vni));
+	}
+
+} // namespace zeek::analyzer::geneve

src/analyzer/protocol/geneve/Geneve.h

@@ -0,0 +1,24 @@
+// See the file  in the main distribution directory for copyright.
+
+#pragma once
+
+#include "zeek/analyzer/Analyzer.h"
+
+namespace zeek::analyzer::geneve {
+
+class Geneve_Analyzer final : public analyzer::Analyzer {
+public:
+	explicit Geneve_Analyzer(Connection* conn)
+	    : Analyzer("Geneve", conn)
+		{}
+
+	void Done() override;
+
+	void DeliverPacket(int len, const u_char* data, bool orig,
+	                   uint64_t seq, const IP_Hdr* ip, int caplen) override;
+
+	static analyzer::Analyzer* Instantiate(Connection* conn)
+		{ return new Geneve_Analyzer(conn); }
+};
+
+} // namespace zeek::analyzer::vxlan

src/analyzer/protocol/geneve/Plugin.cc

@@ -0,0 +1,22 @@
+// See the file  in the main distribution directory for copyright.
+
+#include "zeek/plugin/Plugin.h"
+#include "zeek/analyzer/Component.h"
+#include "zeek/analyzer/protocol/geneve/Geneve.h"
+
+namespace zeek::plugin::detail::Zeek_Geneve {
+
+class Plugin : public zeek::plugin::Plugin {
+public:
+	zeek::plugin::Configuration Configure() override
+		{
+		AddComponent(new zeek::analyzer::Component("Geneve", zeek::analyzer::geneve::Geneve_Analyzer::Instantiate));
+
+		zeek::plugin::Configuration config;
+		config.name = "Zeek::Geneve";
+		config.description = "Geneve analyzer";
+		return config;
+		}
+} plugin;
+
+} // namespace zeek::plugin::detail::Zeek_Geneve

src/analyzer/protocol/geneve/events.bif

@@ -0,0 +1,12 @@
+## Generated for any packet encapsulated in a Geneve tunnel.
+## See :rfc:`8926` for more information about the VXLAN protocol.
+##
+## outer: The Geneve tunnel connection.
+##
+## inner: The Geneve-encapsulated Ethernet packet header and transport header.
+##
+## vni: Geneve Network Identifier.
+##
+## .. note:: Since this event may be raised on a per-packet basis, handling
+##    it may become particularly expensive for real-time analysis.
+event geneve_packet%(outer: connection, inner: pkt_hdr, vni: count%);

src/analyzer/protocol/gnutella/Gnutella.cc

@@ -129,7 +129,7 @@ bool Gnutella_Analyzer::IsHTTP(std::string header)
 		if ( Parent()->IsAnalyzer("TCP") )
 			{
 			// Replay buffered data.
-			analyzer::pia::PIA* pia = static_cast<analyzer::TransportLayerAnalyzer *>(Parent())->GetPIA();
+			analyzer::pia::PIA* pia = static_cast<packet_analysis::IP::SessionAdapter*>(Parent())->GetPIA();
 			if ( pia )
 				static_cast<analyzer::pia::PIA_TCP *>(pia)->ReplayStreamBuffer(a);
 			}

src/analyzer/protocol/gtpv1/GTPv1.cc

@@ -2,6 +2,7 @@
 
 #include "zeek/analyzer/protocol/gtpv1/GTPv1.h"
 #include "zeek/packet_analysis/protocol/iptunnel/IPTunnel.h"
+#include "zeek/packet_analysis/protocol/ip/IP.h"
 
 #include "zeek/analyzer/protocol/gtpv1/events.bif.h"
 
@@ -46,8 +47,8 @@ void GTPv1_Analyzer::DeliverPacket(int len, const u_char* data, bool orig, uint6
 	caplen -= inner_packet_offset;
 	inner_packet_offset = -1;
 
-	IP_Hdr* inner = nullptr;
+	std::unique_ptr<IP_Hdr> inner = nullptr;
-	int result = sessions->ParseIPPacket(len, data, next_header, inner);
+	int result = packet_analysis::IP::ParsePacket(len, data, next_header, inner);
 
 	if ( result == 0 )
 		{
@@ -76,9 +77,6 @@ void GTPv1_Analyzer::DeliverPacket(int len, const u_char* data, bool orig, uint6
 	else
 		ProtocolViolation("GTPv1 payload length",
 		                  reinterpret_cast<const char*>(odata), olen);
-
-	if ( result != 0 )
-		delete inner;
 	}
 
 } // namespace zeek::analyzer::gtpv1

src/analyzer/protocol/gtpv1/gtpv1-analyzer.pac

@@ -1,5 +1,4 @@
 %extern{
-#include "zeek/Sessions.h"
 #include "zeek/ZeekString.h"
 #include "zeek/analyzer/protocol/gtpv1/GTPv1.h"
 %}

src/analyzer/protocol/icmp/ICMP.h

@@ -1,99 +1,15 @@
-// See the file "COPYING" in the main distribution directory for copyright.
+#warning "Remove in v5.1. This analyzer has been moved to packet analysis, use 'zeek/packet_analysis/protocol/icmp/ICMPSessionAdapter.h' and/or 'zeek/packet_analysis/protocol/icmp/ICMP.h'."
 
-#pragma once
+#include "zeek/packet_analysis/protocol/icmp/ICMPSessionAdapter.h"
+#include "zeek/packet_analysis/protocol/icmp/ICMP.h"
 
-#include "zeek/RuleMatcher.h"
+namespace zeek::analyzer::icmp {
-#include "zeek/analyzer/Analyzer.h"
-#include "zeek/net_util.h"
 
-namespace zeek {
+using ICMP_Analyzer [[deprecated("Remove in v5.1. Use zeek::packet_analysis::ICMP::ICMPSessionAdapter.")]] =
+	zeek::packet_analysis::ICMP::ICMPSessionAdapter;
+constexpr auto ICMP4_counterpart [[deprecated("Remove in v5.1. Use zeek::packet_analysis::ICMP::ICMP4_counterpart.")]] =
+	zeek::packet_analysis::ICMP::ICMP4_counterpart;
+constexpr auto ICMP6_counterpart [[deprecated("Remove in v5.1. Use zeek::packet_analysis::ICMP::ICMP6_counterpart.")]] =
+	zeek::packet_analysis::ICMP::ICMP6_counterpart;
 
-class VectorVal;
+} // namespace zeek::analyzer::icmp
-using VectorValPtr = IntrusivePtr<VectorVal>;
-
-namespace analyzer::icmp {
-
-enum ICMP_EndpointState {
-	ICMP_INACTIVE,	// no packet seen
-	ICMP_ACTIVE,	// packets seen
-};
-
-// We do not have an PIA for ICMP (yet) and therefore derive from
-// RuleMatcherState to perform our own matching.
-class ICMP_Analyzer final : public analyzer::TransportLayerAnalyzer {
-public:
-	explicit ICMP_Analyzer(Connection* conn);
-
-	void UpdateConnVal(RecordVal *conn_val) override;
-
-	static analyzer::Analyzer* Instantiate(Connection* conn)
-		{ return new ICMP_Analyzer(conn); }
-
-protected:
-	void Done() override;
-	void DeliverPacket(int len, const u_char* data, bool orig,
-	                   uint64_t seq, const IP_Hdr* ip, int caplen) override;
-	bool IsReuse(double t, const u_char* pkt) override;
-	unsigned int MemoryAllocation() const override;
-
-	void ICMP_Sent(const struct icmp* icmpp, int len, int caplen, int icmpv6,
-	               const u_char* data, const IP_Hdr* ip_hdr);
-
-	void Echo(double t, const struct icmp* icmpp, int len,
-			 int caplen, const u_char*& data, const IP_Hdr* ip_hdr);
-	void Redirect(double t, const struct icmp* icmpp, int len,
-			 int caplen, const u_char*& data, const IP_Hdr* ip_hdr);
-	void RouterAdvert(double t, const struct icmp* icmpp, int len,
-			 int caplen, const u_char*& data, const IP_Hdr* ip_hdr);
-	void NeighborAdvert(double t, const struct icmp* icmpp, int len,
-			 int caplen, const u_char*& data, const IP_Hdr* ip_hdr);
-	void NeighborSolicit(double t, const struct icmp* icmpp, int len,
-			 int caplen, const u_char*& data, const IP_Hdr* ip_hdr);
-	void RouterSolicit(double t, const struct icmp* icmpp, int len,
-			 int caplen, const u_char*& data, const IP_Hdr* ip_hdr);
-
-	void Describe(ODesc* d) const;
-
-	RecordValPtr BuildInfo(const struct icmp* icmpp, int len,
-	                       bool icmpv6, const IP_Hdr* ip_hdr);
-
-	void NextICMP4(double t, const struct icmp* icmpp, int len, int caplen,
-	               const u_char*& data, const IP_Hdr* ip_hdr );
-
-	RecordValPtr ExtractICMP4Context(int len, const u_char*& data);
-
-	void Context4(double t, const struct icmp* icmpp, int len, int caplen,
-	              const u_char*& data, const IP_Hdr* ip_hdr);
-
-	TransportProto GetContextProtocol(const IP_Hdr* ip_hdr, uint32_t* src_port,
-	                                  uint32_t* dst_port);
-
-	void NextICMP6(double t, const struct icmp* icmpp, int len, int caplen,
-	               const u_char*& data, const IP_Hdr* ip_hdr );
-
-	RecordValPtr ExtractICMP6Context(int len, const u_char*& data);
-
-	void Context6(double t, const struct icmp* icmpp, int len, int caplen,
-	              const u_char*& data, const IP_Hdr* ip_hdr);
-
-	// RFC 4861 Neighbor Discover message options
-	VectorValPtr BuildNDOptionsVal(int caplen, const u_char* data);
-
-	RecordValPtr icmp_conn_val;
-	int type;
-	int code;
-	int request_len, reply_len;
-
-	detail::RuleMatcherState matcher_state;
-
-private:
-	void UpdateEndpointVal(const ValPtr& endp, bool is_orig);
-};
-
-// Returns the counterpart type to the given type (e.g., the counterpart
-// to ICMP_ECHOREPLY is ICMP_ECHO).
-extern int ICMP4_counterpart(int icmp_type, int icmp_code, bool& is_one_way);
-extern int ICMP6_counterpart(int icmp_type, int icmp_code, bool& is_one_way);
-
-} // namespace analyzer::icmp
-} // namespace zeek

src/analyzer/protocol/icmp/Plugin.cc

@@ -1,22 +0,0 @@
-// See the file  in the main distribution directory for copyright.
-
-#include "zeek/plugin/Plugin.h"
-#include "zeek/analyzer/Component.h"
-#include "zeek/analyzer/protocol/icmp/ICMP.h"
-
-namespace zeek::plugin::detail::Zeek_ICMP {
-
-class Plugin : public zeek::plugin::Plugin {
-public:
-	zeek::plugin::Configuration Configure() override
-		{
-		AddComponent(new zeek::analyzer::Component("ICMP", zeek::analyzer::icmp::ICMP_Analyzer::Instantiate));
-
-		zeek::plugin::Configuration config;
-		config.name = "Zeek::ICMP";
-		config.description = "ICMP analyzer";
-		return config;
-		}
-} plugin;
-
-} // namespace zeek::plugin::detail::Zeek_ICMP

src/analyzer/protocol/login/RSH.cc

@@ -35,7 +35,7 @@ Contents_Rsh_Analyzer::~Contents_Rsh_Analyzer()
 
 void Contents_Rsh_Analyzer::DoDeliver(int len, const u_char* data)
 	{
-	analyzer::tcp::TCP_Analyzer* tcp = static_cast<analyzer::tcp::TCP_ApplicationAnalyzer*>(Parent())->TCP();
+	auto* tcp = static_cast<analyzer::tcp::TCP_ApplicationAnalyzer*>(Parent())->TCP();
 	assert(tcp);
 
 	int endp_state = IsOrig() ? tcp->OrigState() : tcp->RespState();

src/analyzer/protocol/login/functions.bif

@@ -2,7 +2,7 @@
 %%{
 #include "zeek/analyzer/protocol/login/Login.h"
 #include "zeek/Reporter.h"
-#include "zeek/Sessions.h"
+#include "zeek/session/Manager.h"
 %%}
 
 ## Returns the state of the given login (Telnet or Rlogin) connection.
@@ -26,7 +26,7 @@
 ## .. zeek:see:: set_login_state
 function get_login_state%(cid: conn_id%): count
 	%{
-	zeek::Connection* c = sessions->FindConnection(cid);
+	zeek::Connection* c = session_mgr->FindConnection(cid);
 	if ( ! c )
 		return zeek::val_mgr->False();
 
@@ -50,7 +50,7 @@ function get_login_state%(cid: conn_id%): count
 ## .. zeek:see:: get_login_state
 function set_login_state%(cid: conn_id, new_state: count%): bool
 	%{
-	zeek::Connection* c = sessions->FindConnection(cid);
+	zeek::Connection* c = session_mgr->FindConnection(cid);
 	if ( ! c )
 		return zeek::val_mgr->False();
 

src/analyzer/protocol/ncp/NCP.cc

@@ -7,8 +7,6 @@
 #include <string>
 #include <map>
 
-#include "zeek/Sessions.h"
-
 #include "zeek/analyzer/protocol/ncp/events.bif.h"
 #include "zeek/analyzer/protocol/ncp/consts.bif.h"
 

src/analyzer/protocol/netbios/NetbiosSSN.cc

@@ -7,7 +7,7 @@
 
 #include "zeek/ZeekString.h"
 #include "zeek/NetVar.h"
-#include "zeek/Sessions.h"
+#include "zeek/session/Manager.h"
 #include "zeek/Event.h"
 #include "zeek/RunState.h"
 
@@ -531,7 +531,7 @@ void NetbiosSSN_Analyzer::ExpireTimer(double t)
 		     netbios_ssn_session_timeout - 1.0 )
 		{
 		Event(connection_timeout);
-		sessions->Remove(Conn());
+		session_mgr->Remove(Conn());
 		}
 	else
 		ADD_ANALYZER_TIMER(&NetbiosSSN_Analyzer::ExpireTimer,

src/analyzer/protocol/netbios/NetbiosSSN.h

@@ -2,7 +2,6 @@
 
 #pragma once
 
-#include "zeek/analyzer/protocol/udp/UDP.h"
 #include "zeek/analyzer/protocol/tcp/TCP.h"
 
 namespace zeek::analyzer::netbios_ssn {

src/analyzer/protocol/netbios/functions.bif

@@ -1,53 +1,101 @@
 %%{
 #include "zeek/Reporter.h"
+
+// Like toupper(), but avoid potential for locale-dependence.
+static char netbios_toupper(char c)
+	{
+	if ( c >= 'a' && c <= 'z' )
+		return c - 32;
+	return c;
+	}
 %%}
 
 ## Decode a NetBIOS name.  See https://jeffpar.github.io/kbarchive/kb/194/Q194203/.
 ##
 ## name: The encoded NetBIOS name, e.g., ``"FEEIEFCAEOEFFEECEJEPFDCAEOEBENEF"``.
 ##
-## Returns: The decoded NetBIOS name, e.g., ``"THE NETBIOS NAME"``.
+## Returns: The decoded NetBIOS name, e.g., ``"THE NETBIOS NAM"``.  An empty
+##          string is returned if the argument is not a valid NetBIOS encoding
+##          (though an encoding that would decode to something that includes
+##          only null-bytes or space-characters also yields an empty string).
 ##
 ## .. zeek:see:: decode_netbios_name_type
 function decode_netbios_name%(name: string%): string
 	%{
+	if ( name->Len() != 32 )
+		return val_mgr->EmptyString();
+
 	char buf[16];
-	char result[16];
 	const u_char* s = name->Bytes();
 	int i, j;
+	int length = 0;
 
 	for ( i = 0, j = 0; i < 16; ++i )
 		{
-		char c0 = (j < name->Len()) ? toupper(s[j++]) : 'A';
+		char c0 = netbios_toupper(s[j++]);
-		char c1 = (j < name->Len()) ? toupper(s[j++]) : 'A';
+		char c1 = netbios_toupper(s[j++]);
-		buf[i] = ((c0 - 'A') << 4) + (c1 - 'A');
-		}
 
-	for ( i = 0; i < 15; ++i )
+		if ( c0 < 'A' || c0 > 'P' || c1 < 'A' || c1 > 'P' )
-		{
+			return val_mgr->EmptyString();
-		if ( isalnum(buf[i]) || ispunct(buf[i]) ||
+
+		buf[i] = ((c0 - 'A') << 4) + (c1 - 'A');
+
+		if ( isalnum(buf[i]) || ispunct(buf[i]) || buf[i] == ' ' ||
 		     // \x01\x02 is seen in at least one case as the first two bytes.
 		     // I think that any \x01 and \x02 should always be passed through.
 		     buf[i] < 3 )
-			result[i] = buf[i];
+			++length;
 		else
 			break;
 		}
 
-	return zeek::make_intrusive<zeek::StringVal>(i, result);
+
+	// The 16th byte indicates the suffix/type, so don't include it
+	if ( length == 16 )
+		length = 15;
+
+	// Walk back and remove any trailing spaces or nulls
+	for ( ; ; )
+		{
+		if ( length == 0 )
+			return val_mgr->EmptyString();
+
+		auto c = buf[length - 1];
+
+		if ( c != ' ' && c != 0 )
+			break;
+
+		--length;
+		}
+
+	return zeek::make_intrusive<zeek::StringVal>(length, buf);
 	%}
 
 ## Converts a NetBIOS name type to its corresponding numeric value.
 ## See https://en.wikipedia.org/wiki/NetBIOS#NetBIOS_Suffixes.
 ##
-## name: The NetBIOS name type.
+## name: An encoded NetBIOS name.
 ##
-## Returns: The numeric value of *name*.
+## Returns: The numeric value of *name* or 256 if it's not a valid encoding.
 ##
 ## .. zeek:see:: decode_netbios_name
 function decode_netbios_name_type%(name: string%): count
 	%{
+	if ( name->Len() != 32 )
+		return val_mgr->Count(256);
+
 	const u_char* s = name->Bytes();
-	char return_val = ((toupper(s[30]) - 'A') << 4) + (toupper(s[31]) - 'A');
+
+	for ( auto i = 0; i < 32; ++i )
+		{
+		char c = netbios_toupper(s[i]);
+
+		if ( c < 'A' || c > 'P' )
+			return val_mgr->Count(256);
+		}
+
+	char c0 = toupper(s[30]);
+	char c1 = toupper(s[31]);
+	char return_val = ((c0 - 'A') << 4) + (c1 - 'A');
 	return zeek::val_mgr->Count(return_val);
 	%}