Merge branch 'master' into topic/vern/script-inlining

.cirrus.yml

@@ -1,3 +1,10 @@
+##################################################################################
+# Please note:                                                                   #
+#                                                                                #
+# After updating this file, please also update CI column of the support matrix   #
+# at https://github.com/zeek/zeek/wiki/Zeek-Operating-System-Support-Matrix      #
+##################################################################################
+
 cpus: &CPUS 4
 btest_jobs: &BTEST_JOBS 4
 btest_retries: &BTEST_RETRIES 2
@@ -11,6 +18,15 @@ resources_template: &RESOURCES_TEMPLATE
   cpu: *CPUS
   memory: *MEMORY
 
+macos_resources_template: &MACOS_RESOURCES_TEMPLATE
+  # cpu/memory setting is implicitly 2 core / 4 thread and 8GB, and
+  # trying to set it explicitly results in an error.
+  env:
+    ZEEK_CI_CPUS: 4
+    ZEEK_CI_BTEST_JOBS: 4
+    # No permission to write to default location of /zeek
+    CIRRUS_WORKING_DIR: /tmp/zeek
+
 ci_template: &CI_TEMPLATE
   only_if: >
     $CIRRUS_PR != '' ||
@@ -159,21 +175,21 @@ ubuntu16_task:
     << : *RESOURCES_TEMPLATE
   << : *CI_TEMPLATE
 
-# Apple doesn't publish official long-term support timelines, so easiest
+# Apple doesn't publish official long-term support timelines.
-# option is to only support the latest macOS release or whatever latest
+# We aim to support both the current and previous macOS release.
-# image is available.
+macos_big_sur_task:
-macos_task:
   osx_instance:
-    image: catalina-base
+    image: big-sur-base
-    # cpu/memory setting is implicitly 2 core / 4 thread and 8GB, and
-    # trying to set it explicitly results in an error.
   prepare_script: ./ci/macos/prepare.sh
   << : *CI_TEMPLATE
-  env:
+  << : *MACOS_RESOURCES_TEMPLATE
-    ZEEK_CI_CPUS: 4
+
-    ZEEK_CI_BTEST_JOBS: 4
+macos_catalina_task:
-    # No permission to write to default location of /zeek
+  osx_instance:
-    CIRRUS_WORKING_DIR: /tmp/zeek
+    image: catalina-base
+  prepare_script: ./ci/macos/prepare.sh
+  << : *CI_TEMPLATE
+  << : *MACOS_RESOURCES_TEMPLATE
 
 # FreeBSD EOL timelines: https://www.freebsd.org/security/security.html#sup
 freebsd12_task:

.lgtm.yml

@@ -0,0 +1,45 @@
+# LGTM config file docs: https://lgtm.com/help/lgtm/lgtm.yml-configuration-file
+
+# Results from files under any classifier will be excluded from LGTM stats.
+path_classifiers:
+  library:
+    - "auxil/broker/caf/"
+    - "auxil/libkqueue/"
+    - "auxil/highwayhash/"
+    - "auxil/rapidjson/"
+    - "src/3rdparty/"
+
+# Filter out alerts that aren't concerning.
+queries:
+  - exclude: cpp/use-of-goto
+  - exclude: cpp/short-global-name
+  - exclude: cpp/fixme-comment
+  - exclude: cpp/function-in-block
+
+extraction:
+  cpp:
+    prepare:
+      # Ubuntu packages to install.
+      packages:
+        - cmake
+        - make
+        - ninja-build
+        - gcc
+        - g++
+        - flex
+        - bison
+        - libpcap-dev
+        - libssl-dev
+        - python3
+        - python3-dev
+        - swig
+        - zlib1g-dev
+        - libkrb5-dev
+
+    configure:
+      command:
+        - ./configure --build-type=debug --generator=Ninja
+
+    index:
+      build_command:
+        - ( cd build && ninja )

CHANGES

@@ -1,4 +1,139 @@
 
+4.1.0-dev.27 | 2021-01-06 20:42:35 -0800
+
+  * GH-1347: Update cmake module to fix ZeekPluginDynamic's find_package(CAF) (Jon Siwek, Corelight)
+
+4.1.0-dev.25 | 2020-12-28 15:37:20 -0700
+
+  * Add LGTM config file (Jon Siwek, Corelight)
+
+4.1.0-dev.23 | 2020-12-23 10:36:22 -0800
+
+  * Fix subtle race on data store initialization (Dominik Charousset, Corelight)
+
+4.1.0-dev.19 | 2020-12-22 10:58:13 -0800
+
+  * Add new TLS 1.3 constants. (Johanna Amann)
+
+  * Fix TLS 1.3 session resumption detection. (Johanna Amann)
+
+    Now we detect TLS 1.3 session resumption by looking if both sides have
+    the PSK extension set, which is much more exact than the previous
+    approach.
+
+  * Introduce ssl_probable_encrypted_handshake_message event (Johanna Amann)
+
+    This event is raised for messages that (probably) are TLS 1.3 handshake
+    messages, including finished. This allows scripts to examine handshake
+    messages without having to handle all encrypted messages.
+
+  * SSL Analyzer: ignore CCS for TLS 1.3 (Johanna Amann)
+
+    In TLS 1.3, the ChangeCipherSpec message is meaningless; it only is
+    included to convince middleboxes that the devices actually are speaking
+    TLS 1.2. Nowadays some TLS 1.3 implementations also just don't send the
+    packet.
+
+    In a push to unify our handling of TLS 1.3 connections - ignore CCS and
+    always go with application data packet counting to determine if
+    connections are or are not encrypted.
+
+  * GH-1323: TLS analyzer: change logic to track TLS 1.3 connection establishment (Johanna Amann)
+
+    This commit changes the logic that is used to tracks connection
+    establishment - and moves it from scriptland into the core.
+
+    TLS 1.3 connection establishment is much more finnicky for us than the
+    establishment of earlier versions - since we cannot rely on the CCS
+    message anymore (which is meaningless and not sent in a lot of cases).
+
+    With this commit, the ssl_encrypted_data message gets raised for
+    encrypted TLS 1.3 handshake messages - which is much more correct than
+    the behavior before that just interpreted them as plaintext messages.
+
+4.1.0-dev.12 | 2020-12-21 10:55:13 -0800
+
+  * Add MacOS Big Sur to CI (Christian Kreibich, Corelight)
+
+4.1.0-dev.10 | 2020-12-17 16:11:10 -0800
+
+  * Remove explicit use of config mode for finding CAF (Dominik Charousset, Corelight)
+
+4.1.0-dev.7 | 2020-12-16 15:10:04 -0800
+
+  * logging/ascii: Atomically create .shadow files using rename() (Arne Welzel, Corelight)
+
+    A logger process being terminated/killed while in the process of creating
+    a new .shadow file may leave an empty (invalid) one around. This in turn
+    causes the logger to error and exit during startup.
+
+4.1.0-dev.4 | 2020-12-15 08:00:44 -0800
+
+  * Add test case to cover weird EDNS ECS parsing situations (Jon Siwek, Corelight)
+
+  * Fix EDNS ECS option parsing bugs (Jon Siwek, Corelight)
+
+    * The parsing of IPv6 addresses tried to fill a stack-buffer with as
+      much data as supplied in the Option even if it was in excess of the
+      desired prefix or maximum IPv6 address size.  This could result in an
+      overflow of that stack-buffer.
+
+    * The parsing of IPv4 addresses would overwrite the storage used for
+      that address as many times as there were bytes in the Option in excess
+      of the desired prefix length or maximum IPv4 address size.  This could
+      cause the resulting IPv4 address to be derived from the incorrect
+      data.
+
+    * Upon encountering unexpected/excessive option-length or source-prefix
+      parameters, the data pointer used for parsing was also not always
+      advanced to the start of the next alleged option's data.  Assuming all
+      other parsing code correctly guards against invalid input, there's no
+      further harm from that other than the subsequent parsing being more
+      likely to encounter unexpected values and emitting more Weirds.
+
+    Credit to OSS-Fuzz for discovery
+    https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=28336
+    (Link to details becomes public 30 days after patch release)
+
+4.1.0-dev.1 | 2020-12-14 22:27:57 -0800
+
+  * Fix local.zeek compatibility test (Jon Siwek, Corelight)
+
+3.3.0-dev.664 | 2020-12-14 21:23:01 -0800
+
+  * Update submodules (Jon Siwek, Corelight)
+
+3.3.0-dev.663 | 2020-12-14 14:27:45 -0800
+
+  * Rename a 'do_net_run' variable to 'do_run_loop'
+
+    For clarity, since the net_run() function was renamed to run_loop(). (Jon Siwek, Corelight)
+
+  * GH-1329: call Zeek's cleanup function from standalone fuzzer driver (Jon Siwek, Corelight)
+
+    Otherwise, the global Broker manager object containing CAF/threading
+    logic is never destructed and can result in a heap-use-after-free if it
+    tries to access other global objects after they're cleaned up from
+    __cxa_finalize().
+
+3.3.0-dev.660 | 2020-12-14 10:55:15 -0800
+
+  * Fix typo in table iterator invalidation test comment (Tim Wojtulewicz, Corelight)
+
+  * GH-1328: Improve behavior of Dictionary iterator invalidation (Jon Siwek, Corelight)
+
+    Previously, an assertion was triggered in debug builds upon any attempt
+    to insert or remove a Dictionary entry while any iteration of that
+    Dictionary is underway and also even in cases where Dictionary membership
+    was not actually modified (and thus invalidates a loop).
+
+    Now, it emits run-time warnings regardless of build-type and only when
+    insert/remove operations truly change the Dictionary membership.  In the
+    context of a Zeek script causing an invalidation, the warning message
+    also now helps pinpoint the exact expression that causes it.
+
+  * Add a note about aligning .cirrus.yml to platform support matrix wiki page (Johanna Amann, Corelight)
+
 3.3.0-dev.650 | 2020-12-12 20:24:19 -0800
 
   * Install zkg as part of the Zeek distribution. (Christian Kreibich, Corelight)

CMakeLists.txt

@@ -314,11 +314,11 @@ endif ()
 if ( CAF_ROOT OR BROKER_ROOT_DIR )
   # TODO: drop < 3.12 compatibility check when raising the minimum CMake version
   if ( CAF_ROOT AND CMAKE_VERSION VERSION_LESS 3.12 )
-    find_package(CAF ${CAF_VERSION_MIN_REQUIRED} REQUIRED CONFIG
+    find_package(CAF ${CAF_VERSION_MIN_REQUIRED} REQUIRED
                  COMPONENTS openssl test io core
                  PATHS "${CAF_ROOT}")
   else ()
-    find_package(CAF ${CAF_VERSION_MIN_REQUIRED} REQUIRED CONFIG
+    find_package(CAF ${CAF_VERSION_MIN_REQUIRED} REQUIRED
                  COMPONENTS openssl test io core)
   endif ()
   message(STATUS "Using system CAF version ${CAF_VERSION}")

NEWS

@@ -3,6 +3,21 @@ This document summarizes the most important changes in the current Zeek
 release. For an exhaustive list of changes, see the ``CHANGES`` file
 (note that submodules, such as Broker, come with their own ``CHANGES``.)
 
+Zeek 4.1.0
+==========
+
+New Functionality
+-----------------
+
+Changed Functionality
+---------------------
+
+Removed Functionality
+---------------------
+
+Deprecated Functionality
+------------------------
+
 Zeek 4.0.0
 ==========
 
@@ -162,6 +177,8 @@ New Functionality
   The zkg source tree resides in ``auxil/package-manager`` as an
   additional Git submodule.
 
+- Added a new ``ssl_probable_encrypted_handshake_message`` event, which
+  is raised for encrypted TLS 1.3 handshake messages.
 
 Changed Functionality
 ---------------------
@@ -241,6 +258,31 @@ Changed Functionality
   can be triggered by anybody controlling one of the endpoints (instead
   of both).  For discussion, see https://github.com/zeek/zeek/issues/343.
 
+- TLS 1.3 support was improved in several ways:
+
+  * In the past, some TLS 1.3 sessions were misidentified as using session
+    resumption when, in fact, they were not resumed. This was caused by
+    the TLS session ID which no longer has any meaning in TLS 1.3. This was
+    fixed.
+
+  * Similarly, in the past, TLS 1.3 sessions that use TLS 1.3 PSKs for
+    session resumption were not marked as resumed. This also was fixed.
+
+  * The way in which session establishment for TLS 1.3 is performed was
+    rewritten. This causes the ``ssl_encrypted_data`` event to be correctly
+    raised; in the past this did not work for some sessions. A new
+    ``ssl_probable_encrypted_handshake_message`` event was added that is
+    raised for encrypted TLS 1.3 handshake packets.
+
+  * In the same vein, hello retry requests in TLS 1.3 should now always
+    be handled correctly; in the past this only happened in some cases.
+
+    Please note: When a connection uses Hello Retry requests you will see
+    two client hello and two server hello events in a single connection.
+
+    This happened in the past, but may become more common now; this might
+    trigger unexpected behavior in your scripts.
+
 Removed Functionality
 ---------------------
 
@@ -268,6 +310,10 @@ Removed Functionality
   the Zeek distribution and will get built unless overridden with the
   ``--with-caf=`` configuration option.
 
+- ``server_appdata`` and ``client_appdata`` were removed from ``SSL::Info``.
+  These variables were only used internally, and did not give a correct counts
+  in all circumstances.
+
 Deprecated Functionality
 ------------------------
 

VERSION

@@ -1 +1 @@
-3.3.0-dev.650
+4.1.0-dev.27

auxil/bifcl

@@ -1 +1 @@
-Subproject commit 5a45ae8d0f61e7ae7fa3ed0ea5841e8347e40926
+Subproject commit 624e5525f54d4a15c35687d42301045f1769d3e1

auxil/binpac

@@ -1 +1 @@
-Subproject commit 1078f4e9d6065ae47cf6fca9bd8e98183f913b98
+Subproject commit c33447a02166dc8e2f436b6c0411891431a04d13

auxil/broker

@@ -1 +1 @@
-Subproject commit 7878a6d816fdae370e86f13ae08c6762b6d11347
+Subproject commit c97549bf8491aaf9e8d6777588a46af8af2e99ec

auxil/btest

@@ -1 +1 @@
-Subproject commit 93227bc064ccf51857d0dd83cefaa8325c2f0ce0
+Subproject commit b1de8a637bc006587f5863ada6275aa76c77be7a

auxil/netcontrol-connectors

@@ -1 +1 @@
-Subproject commit 94e1c36512adb47b43c157b87c500176ffb668e2
+Subproject commit fa4df170279b13bcf36d273a2ba91e2b63b32152

auxil/package-manager

@@ -1 +1 @@
-Subproject commit 5b85a7620ed28ac2c4c96e43621d1e10b39e5ae3
+Subproject commit b820f9d73221882a5f9932849ff8c15a58ea5bde

auxil/paraglob

@@ -1 +1 @@
-Subproject commit 61af2af691a0d3b04c740fe8a9df6e5b410c464e
+Subproject commit a4efe7a6508333a2e4f50f66223d212770e0f098

auxil/zeek-archiver

@@ -1 +1 @@
-Subproject commit 37d9e97833aab3e6c24fdeb8c8f5385b878f8290
+Subproject commit 49c49193f8869b9bce8dfc992901d7ec60c1d37e

auxil/zeek-aux

@@ -1 +1 @@
-Subproject commit 633271fa09ec69006b43122deea02dc18a6c6907
+Subproject commit b2ce33bc3e4f57a72cd7bb7db2e3c7dc62e7a202

auxil/zeekctl

@@ -1 +1 @@
-Subproject commit 2736291513f384292c36220e9e57b439fcde3b4f
+Subproject commit e2cee1d50a66a11391fefe896b2f18e64849009e

cmake

@@ -1 +1 @@
-Subproject commit 40251ae850dee52eae8eb05e552c165e2deef354
+Subproject commit 74259745dea5ee4889d1ac1f4ebde4e2c59c329a

doc

@@ -1 +1 @@
-Subproject commit bc112906578b01df6392c804142173ed41cf8aea
+Subproject commit 92e5541f4691b35ffccfd0090d97eb028d190ede

scripts/base/frameworks/cluster/broker-stores.zeek

@@ -37,7 +37,11 @@ event zeek_init()
 	}
 
 # Send the auto masters we created to the newly connected node
-event Broker::peer_added(endpoint: Broker::EndpointInfo, msg: string) &priority=1
+# Note: this is specifically chosen to be higher priority than the
+# Broker::peer_added event in cluster/main.zeek which sends the
+# Cluster::hello event to prevent a race on whether that Cluster::hello
+# ends up generating Cluster::node_up before Broker::announce_masters.
+event Broker::peer_added(endpoint: Broker::EndpointInfo, msg: string) &priority=11
 	{
 	if ( ! Cluster::is_enabled() )
 		return;

scripts/base/protocols/ssl/consts.zeek

@@ -137,6 +137,53 @@ export {
 		[120] = "no_application_protocol",
 	} &default=function(i: count):string { return fmt("unknown-%d", i); };
 
+	# Map SSL Extension values to consts for easier readability of code.
+	# More information can be found here:
+	# http://www.iana.org/assignments/tls-extensiontype-values/tls-extensiontype-values.xml
+	const SSL_EXTENSION_SERVER_NAME = 0;
+	const SSL_EXTENSION_MAX_FRAGMENT_LENGTH = 1;
+	const SSL_EXTENSION_CLIENT_CERTIFICATE_URL = 2;
+	const SSL_EXTENSION_TRUSTED_CA_KEYS = 3;
+	const SSL_EXTENSION_TRUNCATED_HMAC = 4;
+	const SSL_EXTENSION_STATUS_REQUEST = 5;
+	const SSL_EXTENSION_USER_MAPPING = 6;
+	const SSL_EXTENSION_CLIENT_AUTHZ = 7;
+	const SSL_EXTENSION_SERVER_AUTHZ = 8;
+	const SSL_EXTENSION_CERT_TYPE = 9;
+	const SSL_EXTENSION_SUPPORTED_GROUPS = 10;
+	const SSL_EXTENSION_EC_POINT_FORMATS = 11;
+	const SSL_EXTENSION_SRP = 12;
+	const SSL_EXTENSION_SIGNATURE_ALGORITHMS = 13;
+	const SSL_EXTENSION_USE_SRTP = 14;
+	const SSL_EXTENSION_HEARTBEAT = 15;
+	const SSL_EXTENSION_APPLICATION_LAYER_PROTOCOL_NEGOTIATION = 16;
+	const SSL_EXTENSION_STATUS_REQUEST_V2 = 17;
+	const SSL_EXTENSION_SIGNED_CERTIFICATE_TIMESTAMP = 18;
+	const SSL_EXTENSION_CLIENT_CERTIFICATE_TYPE = 19;
+	const SSL_EXTENSION_SERVER_CERTIFICATE_TYPE = 20;
+	const SSL_EXTENSION_PADDING = 21;
+	const SSL_EXTENSION_ENCRYPT_THEN_MAC = 22;
+	const SSL_EXTENSION_EXTENDED_MASTER_SECRET = 23;
+	const SSL_EXTENSION_TOKEN_BINDING = 24;
+	const SSL_EXTENSION_CACHED_INFO = 25;
+	const SSL_EXTENSION_SESSIONTICKET_TLS = 35;
+	const SSL_EXTENSION_KEY_SHARE = 40;
+	const SSL_EXTENSION_PRE_SHARED_KEY = 41;
+	const SSL_EXTENSION_EARLY_DATA = 42;
+	const SSL_EXTENSION_SUPPORTED_VERSIONS = 43;
+	const SSL_EXTENSION_COOKIE = 44;
+	const SSL_EXTENSION_PSK_KEY_EXCHANGE_MODES = 45;
+	const SSL_EXTENSION_TICKETEARLYDATAINFO = 46;
+	const SSL_EXTENSION_CERTIFICATE_AUTHORITIES = 47;
+	const SSL_EXTENSION_OID_FILTERS = 48;
+	const SSL_EXTENSION_NEXT_PROTOCOL_NEGOTIATION = 13172;
+	const SSL_EXTENSION_ORIGIN_BOUND_CERTIFICATES = 13175;
+	const SSL_EXTENSION_ENCRYPTED_CLIENT_CERTIFICATES = 13180;
+	const SSL_EXTENSION_CHANNEL_ID = 30031;
+	const SSL_EXTENSION_CHANNEL_ID_NEW = 30032;
+	const SSL_EXTENSION_PADDING_TEMP = 35655;
+	const SSL_EXTENSION_RENEGOTIATION_INFO = 65281;
+
 	## Mapping between numeric codes and human readable strings for SSL/TLS
 	## extensions.
 	# More information can be found here:

scripts/base/protocols/ssl/main.zeek

@@ -46,12 +46,10 @@ export {
 		## by the client. This value is used to determine if a session
 		## is being resumed. It's not logged.
 		client_key_exchange_seen: bool     &default=F;
-		## Count to track if the server already sent an application data
+		## Track if the client sent a pre-shared-key extension.
-		## packet for TLS 1.3. Used to track when a session was established.
+		## Used to determine if a TLS 1.3 session is being resumed.
-		server_appdata:   count            &default=0;
+		## Not logged.
-		## Flag to track if the client already sent an application data
+		client_psk_seen: bool     &default=F;
-		## packet for TLS 1.3. Used to track when a session was established.
-		client_appdata:   bool             &default=F;
 
 		## Last alert that was seen during the connection.
 		last_alert:       string           &log &optional;
@@ -237,7 +235,7 @@ event ssl_server_hello(c: connection, version: count, record_version: count, pos
 		}
 	c$ssl$cipher = cipher_desc[cipher];
 
-	if ( c$ssl?$session_id && c$ssl$session_id == bytestring_to_hexstr(session_id) )
+	if ( c$ssl?$session_id && c$ssl$session_id == bytestring_to_hexstr(session_id) && c$ssl$version_num/0xFF != 0x7F && c$ssl$version_num != TLSv13 )
 		c$ssl$resumed = T;
 	}
 
@@ -305,10 +303,16 @@ event ssl_extension(c: connection, is_orig: bool, code: count, val: string) &pri
 	{
 	set_session(c);
 
-	if ( is_orig && SSL::extensions[code] == "SessionTicket TLS" && |val| > 0 )
+	if ( is_orig && code == SSL_EXTENSION_SESSIONTICKET_TLS && |val| > 0 )
 		# In this case, we might have an empty ID. Set back to F in client_hello event
 		# if it is not empty after all.
 		c$ssl$client_ticket_empty_session_seen = T;
+	else if ( is_orig && code == SSL_EXTENSION_PRE_SHARED_KEY )
+		# In this case, the client sent a PSK extension which can be used for resumption
+		c$ssl$client_psk_seen = T;
+	else if ( ! is_orig && code == SSL_EXTENSION_PRE_SHARED_KEY && c$ssl$client_psk_seen )
+		# In this case, the server accepted the PSK offered by the client.
+		c$ssl$resumed = T;
 	}
 
 event ssl_change_cipher_spec(c: connection, is_orig: bool) &priority=5
@@ -370,41 +374,8 @@ event ssl_plaintext_data(c: connection, is_orig: bool, record_version: count, co
 	if ( ! c$ssl?$version || c$ssl$established || content_type != APPLICATION_DATA )
 		return;
 
-	if ( c$ssl$version_num/0xFF != 0x7F && c$ssl$version_num != TLSv13 )
+	local wi = Weird::Info($ts=network_time(), $name="ssl_early_application_data", $uid=c$uid, $id=c$id);
-		{
+	Weird::weird(wi);
-		local wi = Weird::Info($ts=network_time(), $name="ssl_early_application_data", $uid=c$uid, $id=c$id);
-		Weird::weird(wi);
-		return;
-		}
-
-	if ( is_orig )
-		{
-		c$ssl$client_appdata = T;
-		return;
-		}
-
-	if ( c$ssl$client_appdata && c$ssl$server_appdata == 0 )
-		{
-		# something went wrong in the handshake here - we can't say if it was established. Just abort.
-		return;
-		}
-  else if ( ! c$ssl$client_appdata && c$ssl$server_appdata == 0 )
-		{
-		c$ssl$server_appdata = 1;
-		return;
-		}
-	else if ( c$ssl$client_appdata && c$ssl$server_appdata == 1 )
-		{
-		# wait for one more packet before we believe it was established. This one could be an encrypted alert.
-		c$ssl$server_appdata = 2;
-		return;
-		}
-	else if ( c$ssl$client_appdata && c$ssl$server_appdata == 2 )
-		{
-		set_ssl_established(c);
-		event ssl_established(c);
-		return;
-		}
 	}
 
 event protocol_violation(c: connection, atype: Analyzer::Tag, aid: count,

src/Dict.cc

@@ -217,6 +217,77 @@ TEST_CASE("dict iteration")
 	delete key2;
 	}
 
+TEST_CASE("dict iterator invalidation")
+	{
+	PDict<uint32_t> dict;
+
+	uint32_t val = 15;
+	uint32_t key_val = 5;
+	auto key = new detail::HashKey(key_val);
+
+	uint32_t val2 = 10;
+	uint32_t key_val2 = 25;
+	auto key2 = new detail::HashKey(key_val2);
+
+	uint32_t val3 = 42;
+	uint32_t key_val3 = 37;
+	auto key3 = new detail::HashKey(key_val3);
+
+	dict.Insert(key, &val);
+	dict.Insert(key2, &val2);
+
+	detail::HashKey* it_key;
+	bool iterators_invalidated = false;
+	IterCookie* it = dict.InitForIteration();
+	CHECK(it != nullptr);
+
+	while ( uint32_t* entry = dict.NextEntry(it_key, it) )
+		{
+		iterators_invalidated = false;
+		dict.Remove(key3, &iterators_invalidated);
+		// Key doesn't exist, nothing to remove, iteration not invalidated.
+		CHECK(!iterators_invalidated);
+
+		iterators_invalidated = false;
+		dict.Insert(key, &val2, &iterators_invalidated);
+		// Key exists, value gets overwritten, iteration not invalidated.
+		CHECK(!iterators_invalidated);
+
+		iterators_invalidated = false;
+		dict.Remove(key2, &iterators_invalidated);
+		// Key exists, gets removed, iteration is invalidated.
+		CHECK(iterators_invalidated);
+
+		delete it_key;
+		dict.StopIteration(it);
+		break;
+		}
+
+	it = dict.InitForIteration();
+	CHECK(it != nullptr);
+
+	while ( uint32_t* entry = dict.NextEntry(it_key, it) )
+		{
+		iterators_invalidated = false;
+		dict.Insert(key3, &val3, &iterators_invalidated);
+		// Key doesn't exist, gets inserted, iteration is invalidated.
+		CHECK(iterators_invalidated);
+
+		delete it_key;
+		dict.StopIteration(it);
+		break;
+		}
+
+	CHECK(dict.Length() == 2);
+	CHECK(*static_cast<uint32_t*>(dict.Lookup(key)) == val2);
+	CHECK(*static_cast<uint32_t*>(dict.Lookup(key3)) == val3);
+	CHECK(static_cast<uint32_t*>(dict.Lookup(key2)) == nullptr);
+
+	delete key;
+	delete key2;
+	delete key3;
+	}
+
 TEST_SUITE_END();
 
 /////////////////////////////////////////////////////////////////////////////////////////////////
@@ -718,13 +789,10 @@ int Dictionary::LookupIndex(const void* key, int key_size, detail::hash_t hash,
 // Insert
 //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
 
-void* Dictionary::Insert(void* key, int key_size, detail::hash_t hash, void* val, bool copy_key)
+void* Dictionary::Insert(void* key, int key_size, detail::hash_t hash, void* val, bool copy_key, bool* iterators_invalidated)
 	{
 	ASSERT_VALID(this);
 
-	// Allow insertions only if there's no active non-robust iterations.
-	ASSERT(num_iterators == 0 || (cookies && cookies->size() == num_iterators));
-
 	// Initialize the table if it hasn't been done yet. This saves memory storing a bunch
 	// of empty dicts.
 	if ( ! table )
@@ -762,6 +830,14 @@ void* Dictionary::Insert(void* key, int key_size, detail::hash_t hash, void* val
 		}
 	else
 		{
+		if ( ! HaveOnlyRobustIterators() )
+			{
+			if ( iterators_invalidated )
+				*iterators_invalidated = true;
+			else
+				reporter->InternalWarning("Dictionary::Insert() possibly caused iterator invalidation");
+			}
+
 		// Allocate memory for key if necesary. Key is updated to reflect internal key if necessary.
 		detail::DictEntry entry(key, key_size, hash, val, insert_distance, copy_key);
 		InsertRelocateAndAdjust(entry, insert_position);
@@ -879,16 +955,24 @@ void Dictionary::SizeUp()
 // Remove
 /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
 
-void* Dictionary::Remove(const void* key, int key_size, detail::hash_t hash, bool dont_delete)
+void* Dictionary::Remove(const void* key, int key_size, detail::hash_t hash, bool dont_delete, bool* iterators_invalidated)
 	{//cookie adjustment: maintain inserts here. maintain next in lower level version.
 	ASSERT_VALID(this);
-	ASSERT(num_iterators == 0 || (cookies && cookies->size() == num_iterators)); //only robust iterators exist.
+
 	ASSERT(! dont_delete); //this is a poorly designed flag. if on, the internal has nowhere to return and memory is lost.
 
 	int position = LookupIndex(key, key_size, hash);
 	if ( position < 0 )
 		return nullptr;
 
+	if ( ! HaveOnlyRobustIterators() )
+		{
+		if ( iterators_invalidated )
+			*iterators_invalidated = true;
+		else
+			reporter->InternalWarning("Dictionary::Remove() possibly caused iterator invalidation");
+		}
+
 	detail::DictEntry entry = RemoveRelocateAndAdjust(position);
 	num_entries--;
 	ASSERT(num_entries >= 0);

src/Dict.h

@@ -164,20 +164,26 @@ public:
 	void* Lookup(const void* key, int key_size, detail::hash_t h) const;
 
 	// Returns previous value, or 0 if none.
-	void* Insert(detail::HashKey* key, void* val)
+	// If iterators_invalidated is supplied, its value is set to true
-		{ return Insert(key->TakeKey(), key->Size(), key->Hash(), val, false); }
+	// if the removal may have invalidated any existing iterators.
+	void* Insert(detail::HashKey* key, void* val, bool* iterators_invalidated = nullptr)
+		{ return Insert(key->TakeKey(), key->Size(), key->Hash(), val, false, iterators_invalidated); }
 
 	// If copy_key is true, then the key is copied, otherwise it's assumed
 	// that it's a heap pointer that now belongs to the Dictionary to
 	// manage as needed.
-	void* Insert(void* key, int key_size, detail::hash_t hash, void* val, bool copy_key);
+	// If iterators_invalidated is supplied, its value is set to true
+	// if the removal may have invalidated any existing iterators.
+	void* Insert(void* key, int key_size, detail::hash_t hash, void* val, bool copy_key, bool* iterators_invalidated = nullptr);
 
 	// Removes the given element.  Returns a pointer to the element in
 	// case it needs to be deleted.  Returns 0 if no such element exists.
 	// If dontdelete is true, the key's bytes will not be deleted.
-	void* Remove(const detail::HashKey* key)
+	// If iterators_invalidated is supplied, its value is set to true
-		{ return Remove(key->Key(), key->Size(), key->Hash()); }
+	// if the removal may have invalidated any existing iterators.
-	void* Remove(const void* key, int key_size, detail::hash_t hash, bool dont_delete = false);
+	void* Remove(const detail::HashKey* key, bool* iterators_invalidated = nullptr)
+		{ return Remove(key->Key(), key->Size(), key->Hash(), false, iterators_invalidated); }
+	void* Remove(const void* key, int key_size, detail::hash_t hash, bool dont_delete = false, bool* iterators_invalidated = nullptr);
 
 	// Number of entries.
 	int Length() const
@@ -337,6 +343,9 @@ private:
 
 	void SizeUp();
 
+	bool HaveOnlyRobustIterators() const
+		{ return num_iterators == 0 || (cookies && cookies->size() == num_iterators); }
+
 	//alligned on 8-bytes with 4-leading bytes. 7*8=56 bytes a dictionary.
 
 	// when sizeup but the current mapping is in progress. the current mapping will be ignored
@@ -380,13 +389,13 @@ public:
 		}
 	T* Lookup(const detail::HashKey* key) const
 		{ return (T*) Dictionary::Lookup(key); }
-	T* Insert(const char* key, T* val)
+	T* Insert(const char* key, T* val, bool* iterators_invalidated = nullptr)
 		{
 		detail::HashKey h(key);
-		return (T*) Dictionary::Insert(&h, (void*) val);
+		return (T*) Dictionary::Insert(&h, (void*) val, iterators_invalidated);
 		}
-	T* Insert(detail::HashKey* key, T* val)
+	T* Insert(detail::HashKey* key, T* val, bool* iterators_invalidated = nullptr)
-		{ return (T*) Dictionary::Insert(key, (void*) val); }
+		{ return (T*) Dictionary::Insert(key, (void*) val, iterators_invalidated); }
 	T* NthEntry(int n) const
 		{ return (T*) Dictionary::NthEntry(n); }
 	T* NthEntry(int n, const char*& key) const
@@ -401,10 +410,10 @@ public:
 		}
 	T* NextEntry(detail::HashKey*& h, IterCookie*& cookie) const
 		{ return (T*) Dictionary::NextEntry(h, cookie, true); }
-	T* RemoveEntry(const detail::HashKey* key)
+	T* RemoveEntry(const detail::HashKey* key, bool* iterators_invalidated = nullptr)
-		{ return (T*) Remove(key->Key(), key->Size(), key->Hash()); }
+		{ return (T*) Remove(key->Key(), key->Size(), key->Hash(), false, iterators_invalidated); }
-	T* RemoveEntry(const detail::HashKey& key)
+	T* RemoveEntry(const detail::HashKey& key, bool* iterators_invalidated = nullptr)
-		{ return (T*) Remove(key.Key(), key.Size(), key.Hash()); }
+		{ return (T*) Remove(key.Key(), key.Size(), key.Hash(), false, iterators_invalidated); }
 };
 
 } // namespace zeek

src/Expr.cc

@@ -2654,7 +2654,17 @@ void IndexExpr::Add(Frame* f)
 	if ( ! v2 )
 		return;
 
-	v1->AsTableVal()->Assign(std::move(v2), nullptr);
+	bool iterators_invalidated = false;
+	v1->AsTableVal()->Assign(std::move(v2), nullptr, true, &iterators_invalidated);
+
+	if ( iterators_invalidated )
+		{
+		ODesc d;
+		Describe(&d);
+		reporter->PushLocation(GetLocationInfo());
+		reporter->Warning("possible loop/iterator invalidation caused by expression: %s", d.Description());
+		reporter->PopLocation();
+		}
 	}
 
 void IndexExpr::Delete(Frame* f)
@@ -2672,7 +2682,17 @@ void IndexExpr::Delete(Frame* f)
 	if ( ! v2 )
 		return;
 
-	v1->AsTableVal()->Remove(*v2);
+	bool iterators_invalidated = false;
+	v1->AsTableVal()->Remove(*v2, true, &iterators_invalidated);
+
+	if ( iterators_invalidated )
+		{
+		ODesc d;
+		Describe(&d);
+		reporter->PushLocation(GetLocationInfo());
+		reporter->Warning("possible loop/iterator invalidation caused by expression: %s", d.Description());
+		reporter->PopLocation();
+		}
 	}
 
 ExprPtr IndexExpr::MakeLvalue()
@@ -2893,7 +2913,10 @@ void IndexExpr::Assign(Frame* f, ValPtr v)
 		}
 
 	case TYPE_TABLE:
-		if ( ! v1->AsTableVal()->Assign(std::move(v2), std::move(v)) )
+		{
+		bool iterators_invalidated = false;
+
+		if ( ! v1->AsTableVal()->Assign(std::move(v2), std::move(v), true, &iterators_invalidated) )
 			{
 			v = std::move(v_extra);
 
@@ -2911,6 +2934,16 @@ void IndexExpr::Assign(Frame* f, ValPtr v)
 			else
 				RuntimeErrorWithCallStack("assignment failed with null value");
 			}
+
+		if ( iterators_invalidated )
+			{
+			ODesc d;
+			Describe(&d);
+			reporter->PushLocation(GetLocationInfo());
+			reporter->Warning("possible loop/iterator invalidation caused by expression: %s", d.Description());
+			reporter->PopLocation();
+			}
+		}
 		break;
 
 	case TYPE_STRING:

src/Val.cc

@@ -1548,7 +1548,8 @@ void TableVal::CheckExpireAttr(detail::AttrTag at)
 		}
 	}
 
-bool TableVal::Assign(ValPtr index, ValPtr new_val, bool broker_forward)
+bool TableVal::Assign(ValPtr index, ValPtr new_val, bool broker_forward,
+                      bool* iterators_invalidated)
 	{
 	auto k = MakeHashKey(*index);
 
@@ -1558,7 +1559,7 @@ bool TableVal::Assign(ValPtr index, ValPtr new_val, bool broker_forward)
 		return false;
 		}
 
-	return Assign(std::move(index), std::move(k), std::move(new_val), broker_forward);
+	return Assign(std::move(index), std::move(k), std::move(new_val), broker_forward, iterators_invalidated);
 	}
 
 bool TableVal::Assign(Val* index, Val* new_val)
@@ -1567,7 +1568,7 @@ bool TableVal::Assign(Val* index, Val* new_val)
 	}
 
 bool TableVal::Assign(ValPtr index, std::unique_ptr<detail::HashKey> k,
-                      ValPtr new_val, bool broker_forward)
+                      ValPtr new_val, bool broker_forward, bool* iterators_invalidated)
 	{
 	bool is_set = table_type->IsSet();
 
@@ -1576,7 +1577,7 @@ bool TableVal::Assign(ValPtr index, std::unique_ptr<detail::HashKey> k,
 
 	TableEntryVal* new_entry_val = new TableEntryVal(std::move(new_val));
 	detail::HashKey k_copy(k->Key(), k->Size(), k->Hash());
-	TableEntryVal* old_entry_val = AsNonConstTable()->Insert(k.get(), new_entry_val);
+	TableEntryVal* old_entry_val = AsNonConstTable()->Insert(k.get(), new_entry_val, iterators_invalidated);
 
 	// If the dictionary index already existed, the insert may free up the
 	// memory allocated to the key bytes, so have to assume k is invalid
@@ -2263,11 +2264,11 @@ void TableVal::SendToStore(const Val* index, const TableEntryVal* new_entry_val,
 		}
 	}
 
-ValPtr TableVal::Remove(const Val& index, bool broker_forward)
+ValPtr TableVal::Remove(const Val& index, bool broker_forward, bool* iterators_invalidated)
 	{
 	auto k = MakeHashKey(index);
 
-	TableEntryVal* v = k ? AsNonConstTable()->RemoveEntry(k.get()) : nullptr;
+	TableEntryVal* v = k ? AsNonConstTable()->RemoveEntry(k.get(), iterators_invalidated) : nullptr;
 	ValPtr va;
 
 	if ( v )
@@ -2293,9 +2294,9 @@ ValPtr TableVal::Remove(const Val& index, bool broker_forward)
 	return va;
 	}
 
-ValPtr TableVal::Remove(const detail::HashKey& k)
+ValPtr TableVal::Remove(const detail::HashKey& k, bool* iterators_invalidated)
 	{
-	TableEntryVal* v = AsNonConstTable()->RemoveEntry(k);
+	TableEntryVal* v = AsNonConstTable()->RemoveEntry(k, iterators_invalidated);
 	ValPtr va;
 
 	if ( v )

src/Val.h

@@ -792,9 +792,12 @@ public:
 	 * must be nullptr.
 	 * @param broker_forward Controls if the value will be forwarded to attached
 	 *        Broker stores.
+	 * @param iterators_invalidated  if supplied, gets set to true if the operation
+	 *        may have invalidated existing iterators.
 	 * @return  True if the assignment type-checked.
 	 */
-	bool Assign(ValPtr index, ValPtr new_val, bool broker_forward = true);
+	bool Assign(ValPtr index, ValPtr new_val, bool broker_forward = true,
+	            bool* iterators_invalidated = nullptr);
 
 	/**
 	 * Assigns a value at an associated index in the table (or in the
@@ -803,13 +806,16 @@ public:
 	 * (if needed, the index val can be recovered from the hash key).
 	 * @param k  A precomputed hash key to use.
 	 * @param new_val  The value to assign at the index.  For a set, this
+	 * @param iterators_invalidated  if supplied, gets set to true if the operation
+	 *        may have invalidated existing iterators.
 	 * must be nullptr.
 	 * @param broker_forward Controls if the value will be forwarded to attached
 	 *        Broker stores.
 	 * @return  True if the assignment type-checked.
 	 */
 	bool Assign(ValPtr index, std::unique_ptr<detail::HashKey> k,
-	            ValPtr new_val, bool broker_forward = true);
+	            ValPtr new_val, bool broker_forward = true,
+	            bool* iterators_invalidated = nullptr);
 
 	// Returns true if the assignment typechecked, false if not. The
 	// methods take ownership of new_val, but not of the index.  If we're
@@ -943,19 +949,23 @@ public:
 	 * @param index  The index to remove.
 	 * @param broker_forward Controls if the remove operation will be forwarded to attached
 	 *        Broker stores.
+	 * @param iterators_invalidated  if supplied, gets set to true if the operation
+	 *        may have invalidated existing iterators.
 	 * @return  The value associated with the index if it exists, else nullptr.
 	 * For a sets that don't really contain associated values, a placeholder
 	 * value is returned to differentiate it from non-existent index (nullptr),
 	 * but otherwise has no meaning in relation to the set's contents.
 	 */
-	ValPtr Remove(const Val& index, bool broker_forward = true);
+	ValPtr Remove(const Val& index, bool broker_forward = true, bool* iterators_invalidated = nullptr);
 
 	/**
 	 * Same as Remove(const Val&), but uses a precomputed hash key.
 	 * @param k  The hash key to lookup.
+	 * @param iterators_invalidated  if supplied, gets set to true if the operation
+	 *        may have invalidated existing iterators.
 	 * @return  Same as Remove(const Val&).
 	 */
-	ValPtr Remove(const detail::HashKey& k);
+	ValPtr Remove(const detail::HashKey& k, bool* iterators_invalidated = nullptr);
 
 	[[deprecated("Remove in v4.1.  Use Remove().")]]
 	Val* Delete(const Val* index)

src/analyzer/protocol/dns/DNS.cc

@@ -742,9 +742,12 @@ bool DNS_Interpreter::ParseRR_EDNS(detail::DNS_MsgInfo* msg,
 			case detail::TYPE_ECS:
 				{
 				// must be 4 bytes + variable number of octets for address
-				if ( option_len <= 4 ) {
+				if ( option_len <= 4 )
+					{
+					analyzer->Weird("EDNS_ECS_invalid_option_len");
+					data += option_len;
 					break;
-				}
+					}
 
 				detail::EDNS_ECS opt{};
 				uint16_t ecs_family = ExtractShort(data, option_len);
@@ -758,27 +761,73 @@ bool DNS_Interpreter::ParseRR_EDNS(detail::DNS_MsgInfo* msg,
 				// padding with 0 bits to pad to the end of the last octet needed.
 				if ( ecs_family == L3_IPV4 )
 					{
+					if ( opt.ecs_src_pfx_len > 32 )
+						{
+						analyzer->Weird("EDNS_ECS_invalid_addr_v4_prefix",
+						                util::fmt("%" PRIu16 " bits", opt.ecs_src_pfx_len));
+						data += option_len;
+						break;
+						}
+
+					if ( opt.ecs_src_pfx_len > option_len * 8 )
+						{
+						analyzer->Weird("EDNS_ECS_invalid_addr_v4",
+						                util::fmt("need %" PRIu16 " bits, have %d bits",
+						                          opt.ecs_src_pfx_len, option_len * 8));
+						data += option_len;
+						break;
+						}
+
 					opt.ecs_family = make_intrusive<StringVal>("v4");
 					uint32_t addr = 0;
-					for (uint16_t shift_factor = 3; option_len > 0; option_len--)
+					uint16_t shift_factor = 3;
+					int bits_left = opt.ecs_src_pfx_len;
+
+					while ( bits_left > 0 )
 						{
 						addr |= data[0] << (shift_factor * 8);
 						data++;
 						shift_factor--;
+						option_len--;
+						bits_left -= 8;
 						}
+
 					addr = htonl(addr);
 					opt.ecs_addr = make_intrusive<AddrVal>(addr);
 					}
 				else if ( ecs_family == L3_IPV6 )
 					{
+					if ( opt.ecs_src_pfx_len > 128 )
+						{
+						analyzer->Weird("EDNS_ECS_invalid_addr_v6_prefix",
+						                util::fmt("%" PRIu16 " bits", opt.ecs_src_pfx_len));
+						data += option_len;
+						break;
+						}
+
+					if ( opt.ecs_src_pfx_len > option_len * 8 )
+						{
+						analyzer->Weird("EDNS_ECS_invalid_addr_v6",
+						                util::fmt("need %" PRIu16 " bits, have %d bits",
+						                          opt.ecs_src_pfx_len, option_len * 8));
+						data += option_len;
+						break;
+						}
+
 					opt.ecs_family = make_intrusive<StringVal>("v6");
 					uint32_t addr[4] = { 0 };
-					for (uint16_t i = 0, shift_factor = 15; option_len > 0; option_len--)
+					uint16_t shift_factor = 15;
+					int bits_left = opt.ecs_src_pfx_len;
+					int i = 0;
+
+					while ( bits_left > 0 )
 						{
 						addr[i / 4] |= data[0] << ((shift_factor % 4) * 8);
 						data++;
 						i++;
 						shift_factor--;
+						option_len--;
+						bits_left -= 8;
 						}
 
 					for (uint8_t i = 0; i < 4; i++)
@@ -799,6 +848,7 @@ bool DNS_Interpreter::ParseRR_EDNS(detail::DNS_MsgInfo* msg,
 					msg->BuildHdrVal(),
 					msg->BuildEDNS_ECS_Val(&opt)
 				);
+				data += option_len;
 				break;
 				} // END EDNS ECS
 

src/analyzer/protocol/ssl/DTLS.cc

@@ -48,6 +48,10 @@ void DTLS_Analyzer::EndOfData(bool is_orig)
 	handshake_interp->FlowEOF(is_orig);
 	}
 
+uint16_t DTLS_Analyzer::GetNegotiatedVersion() const
+  {
+  return handshake_interp->chosen_version();
+  }
 
 void DTLS_Analyzer::SendHandshake(uint16_t raw_tls_version, uint8_t msg_type, uint32_t length, const u_char* begin, const u_char* end, bool orig)
 	{

src/analyzer/protocol/ssl/DTLS.h

@@ -21,7 +21,8 @@ public:
 	void EndOfData(bool is_orig) override;
 
 	void SendHandshake(uint16_t raw_tls_version, uint8_t msg_type, uint32_t length, const u_char* begin, const u_char* end, bool orig);
-
+	// Get the TLS version that the server chose. 0 if not yet known.
+	uint16_t GetNegotiatedVersion() const;
 
 	static analyzer::Analyzer* Instantiate(Connection* conn)
 		{ return new DTLS_Analyzer(conn); }

src/analyzer/protocol/ssl/SSL.cc

@@ -48,6 +48,11 @@ void SSL_Analyzer::StartEncryption()
 	interp->setEstablished();
 	}
 
+uint16_t SSL_Analyzer::GetNegotiatedVersion() const
+	{
+	return handshake_interp->chosen_version();
+	}
+
 void SSL_Analyzer::DeliverStream(int len, const u_char* data, bool orig)
 	{
 	analyzer::tcp::TCP_ApplicationAnalyzer::DeliverStream(len, data, orig);

src/analyzer/protocol/ssl/SSL.h

@@ -24,6 +24,8 @@ public:
 
 	// Tell the analyzer that encryption has started.
 	void StartEncryption();
+	// Get the TLS version that the server chose. 0 if not yet known.
+	uint16_t GetNegotiatedVersion() const;
 
 	// Overriden from analyzer::tcp::TCP_ApplicationAnalyzer.
 	void EndpointEOF(bool is_orig) override;

src/analyzer/protocol/ssl/events.bif

@@ -559,9 +559,42 @@ event ssl_plaintext_data%(c: connection, is_orig: bool, record_version: count, c
 ## length: length of the entire message.
 ##
 ## .. zeek:see::  ssl_client_hello ssl_established ssl_extension ssl_server_hello
-##    ssl_alert ssl_heartbeat
+##    ssl_alert ssl_heartbeat ssl_probable_encrypted_handshake_message
 event ssl_encrypted_data%(c: connection, is_orig: bool, record_version: count, content_type: count, length: count%);
 
+## This event is generated for application data records of TLS 1.3 connections of which
+## we suspect that they contain handshake messages.
+##
+## In TLS 1.3, large parts of the handshake are encrypted; the only cleartext packets
+## typically exchanged are the client hello and the server hello. The first few packets
+## after the client and server hello, however, are a continuation of the handshake and
+## still include handshake data.
+##
+## This event is raised for these packets of which we suspect that they are handshake records,
+## including the finished record.
+##
+## The heuristic for this is: all application data record after the server hello are
+## handshake records until at least one application data record has been received
+## from both the server and the client. Typically, the server will send more records
+## before the client sends the first application data record; and the first application
+## data record of the client will typically include the finished message.
+##
+## Given the encrypted nature of the protocol, in some cases this determination is
+## not correct; the client can send more handshake packets before the finished message, e.g.,
+## when client certificates are used.
+##
+## Note that :zeek:see::ssl_encrypted_data is also raised for these messages.
+##
+## c: The connection.
+##
+## is_orig: True if event is raised for originator side of the connection.
+##
+## length: length of the entire message.
+##
+## .. zeek:see::  ssl_client_hello ssl_established ssl_server_hello
+##    ssl_encrypted_data
+event ssl_probable_encrypted_handshake_message%(c: connection, is_orig: bool, length: count%);
+
 ## This event contains the OCSP response contained in a Certificate Status Request
 ## message, when the client requested OCSP stapling and the server supports it.
 ## See description in :rfc:`6066`.

src/analyzer/protocol/ssl/ssl-dtls-analyzer.pac

@@ -45,9 +45,16 @@ refine connection SSL_Conn += {
 
 	function proc_ciphertext_record(rec : SSLRecord) : bool
 		%{
-		 if ( client_state_ == STATE_ENCRYPTED &&
+		if ( established_ == false && determine_tls13() == 1 )
-		      server_state_ == STATE_ENCRYPTED &&
+			{
-		      established_ == false )
+			if ( ssl_probable_encrypted_handshake_message )
+				zeek::BifEvent::enqueue_ssl_probable_encrypted_handshake_message(zeek_analyzer(),
+					zeek_analyzer()->Conn(), ${rec.is_orig}, ${rec.length});
+			}
+
+		if ( client_state_ == STATE_ENCRYPTED &&
+		     server_state_ == STATE_ENCRYPTED &&
+		     established_ == false )
 			{
 			established_ = true;
 			if ( ssl_established )

src/analyzer/protocol/ssl/ssl-dtls-protocol.pac

@@ -44,8 +44,15 @@ enum AnalyzerState {
 type ChangeCipherSpec(rec: SSLRecord) = record {
 	type : uint8;
 } &length = 1, &let {
-	state_changed : bool =
+	# I know this looks a bit weird. Basically - in TLS 1.3, CCS is meaningless
-		$context.connection.startEncryption(rec.is_orig);
+	# fluff that just is used to pretend to TLS 1.2 devices listening in that
+	# yes, this is TLS. Since we want to know which packets come after this,
+	# and since we do have special handling for TLS 1.3 - let's ignore it in
+	# that case.
+	state_changed : bool = case $context.connection.determine_tls13() of {
+		1 -> false;
+		0 -> $context.connection.startEncryption(rec.is_orig);
+	};
 };
 
 
@@ -130,4 +137,14 @@ refine connection SSL_Conn += {
 			server_state_ = STATE_ENCRYPTED;
 		return true;
 		%}
+
+	function determine_tls13() : int
+		%{
+		// let's be conservative and only return yes if it has a valid TLS 1.3 version number here.
+		uint16_t negotiated_version = zeek_analyzer()->GetNegotiatedVersion();
+		if ( negotiated_version == TLSv13 || negotiated_version/0xFF == 0x7F )
+			return 1;
+
+		return 0;
+		%}
 };

src/analyzer/protocol/ssl/ssl-protocol.pac

@@ -30,10 +30,10 @@ type SSLRecord(is_orig: bool) = record {
 		UNKNOWN_VERSION -> 0;
 		SSLv20 -> (((head0 & 0x7f) << 8) | head1) - 3;
 		default -> (head3 << 8) | head4;
-	};
+	} &requires(version);
 };
 
-type RecordText(rec: SSLRecord) = case $context.connection.state(rec.is_orig) of {
+type RecordText(rec: SSLRecord) = case $context.connection.determine_state(rec.is_orig, rec.content_type) of {
 	STATE_ENCRYPTED
 		-> ciphertext : CiphertextRecord(rec);
 	default
@@ -137,6 +137,49 @@ type SSLPDU(is_orig: bool) = record {
 
 refine connection SSL_Conn += {
 
+	## So - this falls a bit under the envelope of dirty hack - but I don't
+	## really have a better idea. This function determines if a packet should
+	## be handled as an encrypted or as a plaintext packet.
+	##
+	## For TLS 1.2 and below - this is relatively straightforward. Everything
+	## that arrives before CCS (Change Cipher Spec) is a plaintext record. And
+	## everything that arrives after CCS will be encrypted.
+	##
+	## TLS 1.3, however, messes this up a bunch. Some clients still choose to
+	## send a CCS message. The message, however, is pretty much meaningless
+	## from a protocol perspective - and just ignored by the other side. Also -
+	## it is not necessary to send it and some implementations just don't.
+	##
+	## So - what we do here is that we enable the encrypted flag when we get
+	## the first application data in a connection that negotiated TLS 1.3.
+	##
+	## This is correct insofar as the packet will be encrypted. We sadly loose
+	## a bit of context here - we can't really say when we get the first packet
+	## that uses the final cryptographic key material - and will contain content
+	## data. We just don't have that information available in TLS 1.3 anymore.
+	function determine_state(is_orig: bool, content_type: int) : int
+		%{
+		int current_state = state(is_orig);
+		if ( current_state == STATE_ENCRYPTED || content_type != APPLICATION_DATA )
+			return current_state;
+
+		// state = STATE_CLEAR && content_type == APPLICATION_DATA
+		uint16_t negotiated_version = zeek_analyzer()->GetNegotiatedVersion();
+
+		// in theory, we should check for TLS13 or draft-TLS13 instead of doing the reverse.
+		// But - people use weird version numbers. And all of those weird version numbers are
+		// some sort of TLS1.3. So - let's do it this way round instead.
+		if ( negotiated_version != SSLv20 && negotiated_version != SSLv30 && negotiated_version != TLSv10 && negotiated_version != TLSv11 && negotiated_version != TLSv12 )
+			{
+			// well, it seems like this is a TLS 1.3 (or equivalent) applicatio data packet. Let's enable encryption
+			// and handle it as encrypted.
+			startEncryption(is_orig);
+			return STATE_ENCRYPTED;
+			}
+
+		return current_state; // has to be STATE_CLEAR
+		%}
+
 	function determine_ssl_record_layer(head0 : uint8, head1 : uint8,
 					head2 : uint8, head3: uint8, head4: uint8, is_orig: bool) : int
 		%{

src/analyzer/protocol/ssl/tls-handshake-protocol.pac

@@ -795,10 +795,14 @@ type SupportedVersions(rec: HandshakeRecord) = record {
 	versions: uint16[] &until($input.length() == 0);
 } &length=length+1;
 
+# If the server sends it, this is the authorative version. Set it.
 type OneSupportedVersion(rec: HandshakeRecord) = record {
 	version: uint16;
+} &let {
+	version_set : bool = $context.connection.set_version(version);
 };
 
+
 type PSKKeyExchangeModes(rec: HandshakeRecord) = record {
 	length: uint8;
 	modes: uint8[] &until($input.length() == 0);
@@ -944,6 +948,7 @@ refine connection Handshake_Conn += {
 	%init{
 		chosen_cipher_ = NO_CHOSEN_CIPHER;
 		chosen_version_ = UNKNOWN_VERSION;
+
 		record_version_ = 0;
 	%}
 
@@ -955,10 +960,18 @@ refine connection Handshake_Conn += {
 		return true;
 		%}
 
-	function chosen_version() : int %{ return chosen_version_; %}
+	function chosen_version() : uint16 %{ return chosen_version_; %}
 
+	# This function is called several times in certain circumstances.
+	# If it is called twice, it is first called due to the supported_versions
+	# field in the server hello - and then again due to the outer version in
+	# the server hello. So - once we have a version here, let's just stick
+	# with it.
 	function set_version(version: uint16) : bool
 		%{
+		if ( chosen_version_ != UNKNOWN_VERSION )
+			return false;
+
 		chosen_version_ = version;
 		return true;
 		%}

src/fuzzers/standalone-driver.cc

@@ -6,6 +6,8 @@
 #include <memory>
 #include <chrono>
 
+#include "zeek/zeek-setup.h"
+
 extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size);
 extern "C" int LLVMFuzzerInitialize(int* argc, char*** argv);
 
@@ -64,4 +66,5 @@ int main(int argc, char** argv)
 	auto fuzz_dt = duration<double>(agg_stop - fuzz_start).count();
 	printf("Processed %d inputs in %fs (%fs w/ initialization), avg = %fs\n",
 	       num_inputs, fuzz_dt, agg_dt, fuzz_dt / num_inputs);
+	return zeek::detail::cleanup(false);
 	}

src/logging/writers/ascii/Ascii.cc

@@ -455,7 +455,8 @@ bool Ascii::DoInit(const WriterInfo& info, int num_fields, const threading::Fiel
 		if ( use_shadow )
 			{
 			auto sfname = shadow_file_prefix + fname;
-			auto sfd = open(sfname.data(), O_WRONLY | O_CREAT | O_TRUNC, 0666);
+			auto tmp_sfname = ".tmp" + sfname;
+			auto sfd = open(tmp_sfname.data(), O_WRONLY | O_CREAT | O_TRUNC, 0666);
 
 			if ( sfd < 0 )
 				{
@@ -474,6 +475,17 @@ bool Ascii::DoInit(const WriterInfo& info, int num_fields, const threading::Fiel
 			util::safe_write(sfd, "\n", 1);
 
 			util::safe_close(sfd);
+
+			if ( rename(tmp_sfname.data(), sfname.data()) == -1 )
+				{
+				Error(Fmt("Unable to rename %s to %s: %s",
+					  tmp_sfname.data(), sfname.data(),
+					  Strerror(errno)));
+
+				unlink(tmp_sfname.data());
+
+				return false;
+				}
 			}
 		}
 

src/main.cc

@@ -16,11 +16,11 @@ int main(int argc, char** argv)
 		return setup_result.code;
 
 	auto& options = setup_result.options;
-	auto do_net_run = zeek::iosource_mgr->Size() > 0 ||
+	auto do_run_loop = zeek::iosource_mgr->Size() > 0 ||
-	                  zeek::run_state::detail::have_pending_timers ||
+	                   zeek::run_state::detail::have_pending_timers ||
-	                  zeek::BifConst::exit_only_after_terminate;
+	                   zeek::BifConst::exit_only_after_terminate;
 
-	if ( do_net_run )
+	if ( do_run_loop )
 		{
 		if ( zeek::detail::profiling_logger )
 			zeek::detail::profiling_logger->Log();
@@ -78,5 +78,5 @@ int main(int argc, char** argv)
 			}
 		}
 
-	return zeek::detail::cleanup(do_net_run);
+	return zeek::detail::cleanup(do_run_loop);
 	}

src/zeek-setup.cc

@@ -918,9 +918,9 @@ SetupResult setup(int argc, char** argv, Options* zopts)
 	return {0, std::move(options)};
 	}
 
-int cleanup(bool did_net_run)
+int cleanup(bool did_run_loop )
 	{
-	if ( did_net_run )
+	if ( did_run_loop )
 		done_with_network();
 
 	run_state::detail::delete_run();

src/zeek-setup.h

@@ -24,8 +24,8 @@ SetupResult setup(int argc, char** argv, Options* options = nullptr);
 
 /**
  * Cleans up Zeek's global state.
- * @param did_net_run  whether the net_run() was called.
+ * @param did_run_loop  whether the run_loop() function was called.
  */
-int cleanup(bool did_net_run);
+int cleanup(bool did_run_loop);
 
 } // namespace zeek::detail

testing/btest/Baseline/language.table-set-iterator-invalidation/out

@@ -0,0 +1,42 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+warning in <...>/table-set-iterator-invalidation.zeek, line 21: possible loop/iterator invalidation caused by expression: t[4]
+warning in <...>/table-set-iterator-invalidation.zeek, line 30: possible loop/iterator invalidation caused by expression: t[4]
+warning in <...>/table-set-iterator-invalidation.zeek, line 53: possible loop/iterator invalidation caused by expression: s[4]
+warning in <...>/table-set-iterator-invalidation.zeek, line 62: possible loop/iterator invalidation caused by expression: s[4]
+{
+[2] = 2,
+[1] = 1,
+[3] = 3
+}
+{
+[2] = 2,
+[4] = four,
+[3] = 3,
+[1] = 1
+}
+{
+[2] = 2,
+[1] = 1,
+[3] = 3
+}
+{
+[2] = 2,
+[1] = 1,
+[3] = 3
+}
+{
+2,
+4,
+3,
+1
+}
+{
+2,
+1,
+3
+}
+{
+2,
+1,
+3
+}

testing/btest/Baseline/scripts.base.protocols.dns.dns-edns-ecs/weird.log

@@ -0,0 +1,15 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	weird
+#open XXXX-XX-XX-XX-XX-XX
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	name	addl	notice	peer	source
+#types	time	string	addr	port	addr	port	string	string	bool	string	string
+XXXXXXXXXX.XXXXXX	CHhAvVGS1DHFjwGM9	74.125.47.13	57157	192.168.90.10	53	EDNS_ECS_invalid_addr_v4	need 32 bits, have 24 bits	F	zeek	DNS
+XXXXXXXXXX.XXXXXX	ClEkJM2Vm5giqnMf4h	74.125.73.76	55744	192.168.90.10	53	EDNS_ECS_invalid_addr_v4_prefix	255 bits	F	zeek	DNS
+XXXXXXXXXX.XXXXXX	C4J4Th3PJpwUYZZ6gc	2a00:1450:4013:c03::10a	46433	2001:470:765b::a25:53	53	EDNS_ECS_invalid_addr_v6_prefix	255 bits	F	zeek	DNS
+XXXXXXXXXX.XXXXXX	CtPZjS20MLrsMUOJi2	2a00:1450:4013:c06::105	63369	2001:470:765b::a25:53	53	EDNS_ECS_invalid_addr_v6	need 66 bits, have 56 bits	F	zeek	DNS
+XXXXXXXXXX.XXXXXX	CUM0KZ3MLUfNB0cl11	2a00:1450:400c:c00::106	54430	2001:470:765b::a25:53	53	EDNS_ECS_invalid_option_len	-	F	zeek	DNS
+#close XXXX-XX-XX-XX-XX-XX

testing/btest/Baseline/scripts.base.protocols.ssl.tls13-version/ssl.log

@@ -7,5 +7,5 @@
 #open XXXX-XX-XX-XX-XX-XX
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	version	cipher	curve	server_name	resumed	last_alert	next_protocol	established	cert_chain_fuids	client_cert_chain_fuids	subject	issuer	client_subject	client_issuer
 #types	time	string	addr	port	addr	port	string	string	string	string	bool	string	string	bool	vector[string]	vector[string]	string	string	string	string
-XXXXXXXXXX.XXXXXX	CHhAvVGS1DHFjwGM9	192.168.86.23	63449	52.32.149.186	443	TLSv13-draft23	TLS_AES_128_GCM_SHA256	x25519	tls13.crypto.mozilla.org	T	-	-	T	-	-	-	-	-	-
+XXXXXXXXXX.XXXXXX	CHhAvVGS1DHFjwGM9	192.168.86.23	63449	52.32.149.186	443	TLSv13-draft23	TLS_AES_128_GCM_SHA256	x25519	tls13.crypto.mozilla.org	F	-	-	T	-	-	-	-	-	-
 #close XXXX-XX-XX-XX-XX-XX

testing/btest/Baseline/scripts.base.protocols.ssl.tls13/.stdout

@@ -1,4 +1,5 @@
 ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+tls13draft16-chrome55.0.2879.0-canary-aborted.pcap
 key_share, [orig_h=192.168.6.203, orig_p=53226/tcp, resp_h=52.32.149.186, resp_p=443/tcp], T
 unknown-27242
 x25519
@@ -7,6 +8,7 @@ key_share, [orig_h=192.168.6.203, orig_p=53227/tcp, resp_h=52.32.149.186, resp_p
 unknown-19018
 x25519
 client, TLSv10, TLSv12
+tls13draft16-chrome55.0.2879.0-canary.pcap
 key_share, [orig_h=192.168.6.203, orig_p=53994/tcp, resp_h=138.68.41.77, resp_p=443/tcp], T
 unknown-43690
 x25519
@@ -14,6 +16,13 @@ client, TLSv10, TLSv12
 key_share, [orig_h=192.168.6.203, orig_p=53994/tcp, resp_h=138.68.41.77, resp_p=443/tcp], F
 x25519
 server, TLSv10, TLSv13-draft14
+encrypted, [orig_h=192.168.6.203, orig_p=53994/tcp, resp_h=138.68.41.77, resp_p=443/tcp], F, TLSv10, 23
+encrypted, [orig_h=192.168.6.203, orig_p=53994/tcp, resp_h=138.68.41.77, resp_p=443/tcp], F, TLSv10, 23
+encrypted, [orig_h=192.168.6.203, orig_p=53994/tcp, resp_h=138.68.41.77, resp_p=443/tcp], F, TLSv10, 23
+encrypted, [orig_h=192.168.6.203, orig_p=53994/tcp, resp_h=138.68.41.77, resp_p=443/tcp], F, TLSv10, 23
+established, [orig_h=192.168.6.203, orig_p=53994/tcp, resp_h=138.68.41.77, resp_p=443/tcp]
+encrypted, [orig_h=192.168.6.203, orig_p=53994/tcp, resp_h=138.68.41.77, resp_p=443/tcp], T, TLSv10, 23
+encrypted, [orig_h=192.168.6.203, orig_p=53994/tcp, resp_h=138.68.41.77, resp_p=443/tcp], F, TLSv10, 23
 key_share, [orig_h=192.168.6.203, orig_p=53996/tcp, resp_h=138.68.41.77, resp_p=443/tcp], T
 unknown-60138
 x25519
@@ -21,17 +30,30 @@ client, TLSv10, TLSv12
 key_share, [orig_h=192.168.6.203, orig_p=53996/tcp, resp_h=138.68.41.77, resp_p=443/tcp], F
 x25519
 server, TLSv10, TLSv13-draft14
+encrypted, [orig_h=192.168.6.203, orig_p=53996/tcp, resp_h=138.68.41.77, resp_p=443/tcp], F, TLSv10, 23
+encrypted, [orig_h=192.168.6.203, orig_p=53996/tcp, resp_h=138.68.41.77, resp_p=443/tcp], F, TLSv10, 23
+encrypted, [orig_h=192.168.6.203, orig_p=53996/tcp, resp_h=138.68.41.77, resp_p=443/tcp], F, TLSv10, 23
+encrypted, [orig_h=192.168.6.203, orig_p=53996/tcp, resp_h=138.68.41.77, resp_p=443/tcp], F, TLSv10, 23
 established, [orig_h=192.168.6.203, orig_p=53996/tcp, resp_h=138.68.41.77, resp_p=443/tcp]
 encrypted, [orig_h=192.168.6.203, orig_p=53996/tcp, resp_h=138.68.41.77, resp_p=443/tcp], T, TLSv10, 23
+encrypted, [orig_h=192.168.6.203, orig_p=53996/tcp, resp_h=138.68.41.77, resp_p=443/tcp], T, TLSv10, 23
+encrypted, [orig_h=192.168.6.203, orig_p=53996/tcp, resp_h=138.68.41.77, resp_p=443/tcp], T, TLSv10, 23
+encrypted, [orig_h=192.168.6.203, orig_p=53996/tcp, resp_h=138.68.41.77, resp_p=443/tcp], T, TLSv10, 23
+encrypted, [orig_h=192.168.6.203, orig_p=53996/tcp, resp_h=138.68.41.77, resp_p=443/tcp], T, TLSv10, 23
+encrypted, [orig_h=192.168.6.203, orig_p=53996/tcp, resp_h=138.68.41.77, resp_p=443/tcp], F, TLSv10, 23
 encrypted, [orig_h=192.168.6.203, orig_p=53996/tcp, resp_h=138.68.41.77, resp_p=443/tcp], F, TLSv10, 23
 encrypted, [orig_h=192.168.6.203, orig_p=53996/tcp, resp_h=138.68.41.77, resp_p=443/tcp], T, TLSv10, 23
 encrypted, [orig_h=192.168.6.203, orig_p=53996/tcp, resp_h=138.68.41.77, resp_p=443/tcp], F, TLSv10, 23
+encrypted, [orig_h=192.168.6.203, orig_p=53996/tcp, resp_h=138.68.41.77, resp_p=443/tcp], T, TLSv10, 23
+encrypted, [orig_h=192.168.6.203, orig_p=53996/tcp, resp_h=138.68.41.77, resp_p=443/tcp], F, TLSv10, 23
+tls13draft16-ff52.a01-aborted.pcap
 key_share, [orig_h=192.150.187.20, orig_p=54980/tcp, resp_h=52.32.149.186, resp_p=443/tcp], T
 x25519
 secp256r1
 secp384r1
 client, TLSv10, TLSv12
 client, TLSv10, TLSv12
+tls13draft16-ff52.a01.pcap
 key_share, [orig_h=192.150.187.20, orig_p=36778/tcp, resp_h=138.68.41.77, resp_p=443/tcp], T
 x25519
 secp256r1
@@ -40,6 +62,12 @@ client, TLSv10, TLSv12
 key_share, [orig_h=192.150.187.20, orig_p=36778/tcp, resp_h=138.68.41.77, resp_p=443/tcp], F
 secp384r1
 server, TLSv10, TLSv13-draft16
+encrypted, [orig_h=192.150.187.20, orig_p=36778/tcp, resp_h=138.68.41.77, resp_p=443/tcp], F, TLSv10, 23
+encrypted, [orig_h=192.150.187.20, orig_p=36778/tcp, resp_h=138.68.41.77, resp_p=443/tcp], F, TLSv10, 23
+encrypted, [orig_h=192.150.187.20, orig_p=36778/tcp, resp_h=138.68.41.77, resp_p=443/tcp], F, TLSv10, 23
+encrypted, [orig_h=192.150.187.20, orig_p=36778/tcp, resp_h=138.68.41.77, resp_p=443/tcp], F, TLSv10, 23
+encrypted, [orig_h=192.150.187.20, orig_p=36778/tcp, resp_h=138.68.41.77, resp_p=443/tcp], F, TLSv10, 23
+encrypted, [orig_h=192.150.187.20, orig_p=36778/tcp, resp_h=138.68.41.77, resp_p=443/tcp], F, TLSv10, 23
 key_share, [orig_h=192.150.187.20, orig_p=36782/tcp, resp_h=138.68.41.77, resp_p=443/tcp], T
 x25519
 secp256r1
@@ -48,13 +76,23 @@ client, TLSv10, TLSv12
 key_share, [orig_h=192.150.187.20, orig_p=36782/tcp, resp_h=138.68.41.77, resp_p=443/tcp], F
 secp384r1
 server, TLSv10, TLSv13-draft16
+encrypted, [orig_h=192.150.187.20, orig_p=36782/tcp, resp_h=138.68.41.77, resp_p=443/tcp], F, TLSv10, 23
+encrypted, [orig_h=192.150.187.20, orig_p=36782/tcp, resp_h=138.68.41.77, resp_p=443/tcp], F, TLSv10, 23
+encrypted, [orig_h=192.150.187.20, orig_p=36782/tcp, resp_h=138.68.41.77, resp_p=443/tcp], F, TLSv10, 23
+encrypted, [orig_h=192.150.187.20, orig_p=36782/tcp, resp_h=138.68.41.77, resp_p=443/tcp], F, TLSv10, 23
 established, [orig_h=192.150.187.20, orig_p=36782/tcp, resp_h=138.68.41.77, resp_p=443/tcp]
 encrypted, [orig_h=192.150.187.20, orig_p=36782/tcp, resp_h=138.68.41.77, resp_p=443/tcp], T, TLSv10, 23
+encrypted, [orig_h=192.150.187.20, orig_p=36782/tcp, resp_h=138.68.41.77, resp_p=443/tcp], T, TLSv10, 23
+encrypted, [orig_h=192.150.187.20, orig_p=36782/tcp, resp_h=138.68.41.77, resp_p=443/tcp], T, TLSv10, 23
+encrypted, [orig_h=192.150.187.20, orig_p=36782/tcp, resp_h=138.68.41.77, resp_p=443/tcp], F, TLSv10, 23
 encrypted, [orig_h=192.150.187.20, orig_p=36782/tcp, resp_h=138.68.41.77, resp_p=443/tcp], F, TLSv10, 23
 encrypted, [orig_h=192.150.187.20, orig_p=36782/tcp, resp_h=138.68.41.77, resp_p=443/tcp], T, TLSv10, 23
 encrypted, [orig_h=192.150.187.20, orig_p=36782/tcp, resp_h=138.68.41.77, resp_p=443/tcp], F, TLSv10, 23
 encrypted, [orig_h=192.150.187.20, orig_p=36782/tcp, resp_h=138.68.41.77, resp_p=443/tcp], T, TLSv10, 23
 encrypted, [orig_h=192.150.187.20, orig_p=36782/tcp, resp_h=138.68.41.77, resp_p=443/tcp], F, TLSv10, 23
+encrypted, [orig_h=192.150.187.20, orig_p=36782/tcp, resp_h=138.68.41.77, resp_p=443/tcp], T, TLSv10, 23
+encrypted, [orig_h=192.150.187.20, orig_p=36782/tcp, resp_h=138.68.41.77, resp_p=443/tcp], F, TLSv10, 23
+tls13_psk_succesfull.pcap
 key_share, [orig_h=192.168.178.80, orig_p=54220/tcp, resp_h=174.138.9.219, resp_p=443/tcp], T
 x25519
 client, TLSv10, TLSv12
@@ -69,16 +107,21 @@ encrypted, [orig_h=192.168.178.80, orig_p=54220/tcp, resp_h=174.138.9.219, resp_
 encrypted, [orig_h=192.168.178.80, orig_p=54220/tcp, resp_h=174.138.9.219, resp_p=443/tcp], T, TLSv12, 23
 encrypted, [orig_h=192.168.178.80, orig_p=54220/tcp, resp_h=174.138.9.219, resp_p=443/tcp], F, TLSv12, 23
 encrypted, [orig_h=192.168.178.80, orig_p=54220/tcp, resp_h=174.138.9.219, resp_p=443/tcp], T, TLSv12, 23
+hrr.pcap
 key_share, [orig_h=10.192.48.168, orig_p=63564/tcp, resp_h=64.233.185.139, resp_p=443/tcp], T
 secp224r1
 client, TLSv10, TLSv12
 key_share, [orig_h=10.192.48.168, orig_p=63564/tcp, resp_h=64.233.185.139, resp_p=443/tcp], F
 secp256r1
 server, TLSv12, TLSv12
+key_share, [orig_h=10.192.48.168, orig_p=63564/tcp, resp_h=64.233.185.139, resp_p=443/tcp], T
+secp256r1
+client, TLSv12, TLSv12
+key_share, [orig_h=10.192.48.168, orig_p=63564/tcp, resp_h=64.233.185.139, resp_p=443/tcp], F
+secp256r1
+server, TLSv12, TLSv12
+encrypted, [orig_h=10.192.48.168, orig_p=63564/tcp, resp_h=64.233.185.139, resp_p=443/tcp], F, TLSv12, 23
 established, [orig_h=10.192.48.168, orig_p=63564/tcp, resp_h=64.233.185.139, resp_p=443/tcp]
-encrypted, [orig_h=10.192.48.168, orig_p=63564/tcp, resp_h=64.233.185.139, resp_p=443/tcp], T, TLSv12, 22
-encrypted, [orig_h=10.192.48.168, orig_p=63564/tcp, resp_h=64.233.185.139, resp_p=443/tcp], F, TLSv12, 22
-encrypted, [orig_h=10.192.48.168, orig_p=63564/tcp, resp_h=64.233.185.139, resp_p=443/tcp], F, TLSv12, 23
 encrypted, [orig_h=10.192.48.168, orig_p=63564/tcp, resp_h=64.233.185.139, resp_p=443/tcp], T, TLSv12, 23
 encrypted, [orig_h=10.192.48.168, orig_p=63564/tcp, resp_h=64.233.185.139, resp_p=443/tcp], F, TLSv12, 23
 encrypted, [orig_h=10.192.48.168, orig_p=63564/tcp, resp_h=64.233.185.139, resp_p=443/tcp], T, TLSv12, 23

testing/btest/Baseline/scripts.base.protocols.ssl.tls13/ssl-out.log

@@ -18,7 +18,7 @@ XXXXXXXXXX.XXXXXX	ClEkJM2Vm5giqnMf4h	192.168.6.203	53227	52.32.149.186	443	-	-	-
 #open XXXX-XX-XX-XX-XX-XX
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	version	cipher	curve	server_name	resumed	last_alert	next_protocol	established	cert_chain_fuids	client_cert_chain_fuids	subject	issuer	client_subject	client_issuer
 #types	time	string	addr	port	addr	port	string	string	string	string	bool	string	string	bool	vector[string]	vector[string]	string	string	string	string
-XXXXXXXXXX.XXXXXX	CHhAvVGS1DHFjwGM9	192.168.6.203	53994	138.68.41.77	443	TLSv13-draft14	TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256	x25519	-	F	-	-	F	-	-	-	-	-	-
+XXXXXXXXXX.XXXXXX	CHhAvVGS1DHFjwGM9	192.168.6.203	53994	138.68.41.77	443	TLSv13-draft14	TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256	x25519	-	F	-	-	T	-	-	-	-	-	-
 XXXXXXXXXX.XXXXXX	ClEkJM2Vm5giqnMf4h	192.168.6.203	53996	138.68.41.77	443	TLSv13-draft14	TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256	x25519	-	F	-	-	T	-	-	-	-	-	-
 #close XXXX-XX-XX-XX-XX-XX
 #separator \x09
@@ -61,5 +61,5 @@ XXXXXXXXXX.XXXXXX	CHhAvVGS1DHFjwGM9	192.168.178.80	54220	174.138.9.219	443	TLSv1
 #open XXXX-XX-XX-XX-XX-XX
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	version	cipher	curve	server_name	resumed	last_alert	next_protocol	established	cert_chain_fuids	client_cert_chain_fuids	subject	issuer	client_subject	client_issuer
 #types	time	string	addr	port	addr	port	string	string	string	string	bool	string	string	bool	vector[string]	vector[string]	string	string	string	string
-XXXXXXXXXX.XXXXXX	CHhAvVGS1DHFjwGM9	10.192.48.168	63564	64.233.185.139	443	TLSv13	TLS_AES_256_GCM_SHA384	secp256r1	-	T	-	-	T	-	-	-	-	-	-
+XXXXXXXXXX.XXXXXX	CHhAvVGS1DHFjwGM9	10.192.48.168	63564	64.233.185.139	443	TLSv13	TLS_AES_256_GCM_SHA384	secp256r1	-	F	-	-	T	-	-	-	-	-	-
 #close XXXX-XX-XX-XX-XX-XX

testing/btest/Baseline/scripts.base.protocols.ssl.tls13_encrypted_handshake_events/.stdout

@@ -0,0 +1,13 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+Probable handshake, F, 23
+encrypted, [orig_h=192.168.186.133, orig_p=43056/tcp, resp_h=192.168.186.134, resp_p=9090/tcp], F, TLSv12, 23
+Probable handshake, F, 716
+encrypted, [orig_h=192.168.186.133, orig_p=43056/tcp, resp_h=192.168.186.134, resp_p=9090/tcp], F, TLSv12, 23
+Probable handshake, F, 281
+encrypted, [orig_h=192.168.186.133, orig_p=43056/tcp, resp_h=192.168.186.134, resp_p=9090/tcp], F, TLSv12, 23
+Probable handshake, F, 69
+encrypted, [orig_h=192.168.186.133, orig_p=43056/tcp, resp_h=192.168.186.134, resp_p=9090/tcp], F, TLSv12, 23
+Probable handshake, T, 69
+Established!
+encrypted, [orig_h=192.168.186.133, orig_p=43056/tcp, resp_h=192.168.186.134, resp_p=9090/tcp], T, TLSv12, 23
+encrypted, [orig_h=192.168.186.133, orig_p=43056/tcp, resp_h=192.168.186.134, resp_p=9090/tcp], T, TLSv12, 23

testing/btest/Baseline/scripts.base.protocols.ssl.tls13_encrypted_handshake_events/ssl.log

@@ -0,0 +1,11 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	ssl
+#open XXXX-XX-XX-XX-XX-XX
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	version	cipher	curve	server_name	resumed	last_alert	next_protocol	established	cert_chain_fuids	client_cert_chain_fuids	subject	issuer	client_subject	client_issuer
+#types	time	string	addr	port	addr	port	string	string	string	string	bool	string	string	bool	vector[string]	vector[string]	string	string	string	string
+XXXXXXXXXX.XXXXXX	CHhAvVGS1DHFjwGM9	192.168.186.133	43056	192.168.186.134	9090	TLSv13	TLS_AES_256_GCM_SHA384	secp256r1	-	F	-	-	T	-	-	-	-	-	-
+#close XXXX-XX-XX-XX-XX-XX

testing/btest/language/table-set-iterator-invalidation.zeek

@@ -0,0 +1,73 @@
+# @TEST-EXEC: zeek -b %INPUT >out 2>&1
+# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff out
+
+# Note that while modifying container membership during for-loop iteration is
+# supposed to be undefined-behavior, it should be practically ok to have this
+# test perform such operations if they always `break` out of the loop
+# immediately afterward.
+
+local t = table([1] = "one", [2] = "two", [3] = "three");
+
+for ( i in t )
+	# Modifying an existing element is not qualified as modifying membership,
+	# so this doesn't trigger a warning.
+	t[i] = cat(i);
+
+print t;
+
+for ( i in t )
+	{
+	# Adding an element in a loop should trigger a warning.
+	t[4] = "four";
+	break;
+	}
+
+print t;
+
+for ( i in t )
+	{
+	# Deleting an element in a loop should trigger a warning.
+	delete t[4];
+	break;
+	}
+
+print t;
+
+for ( i in t )
+	# Trying to delete a non-existent element within in a loop does not
+	# actually modify membership, so does not trigger a warning.
+	delete t[0];
+
+print t;
+
+local s = set(1, 2, 3);
+
+for ( n in s )
+	# Trying to add an existing element within in a loop does not
+	# actually modify membership, so does not trigger a warning.
+	add s[1];
+
+for ( n in s )
+	{
+	# Adding an element in a loop should trigger a warning.
+	add s[4];
+	break;
+	}
+
+print s;
+
+for ( n in s )
+	{
+	# Deleting an element in a loop should trigger a warning.
+	delete s[4];
+	break;
+	}
+
+print s;
+
+for ( n in s )
+	# Trying to delete a non-existent element within in a loop does not
+	# actually modify membership, so does not trigger a warning.
+	delete s[0];
+
+print s;

testing/btest/scripts/base/protocols/dns/dns-edns-ecs.zeek

@@ -1,6 +1,12 @@
+# Test-case for valid message format:
 # @TEST-EXEC: zeek -b -C -r $TRACES/dns-edns-ecs.pcap %INPUT > output
 # @TEST-EXEC: btest-diff output
 
+# Test-case for malformed messages:
+# @TEST-EXEC: zeek -b -C -r $TRACES/dns-edns-ecs-bad.pcap %INPUT
+# @TEST-EXEC: zeek -b -C -r $TRACES/dns-edns-ecs-weirds.pcap %INPUT base/frameworks/notice/weird
+# @TEST-EXEC: btest-diff weird.log
+
 @load policy/protocols/dns/auth-addl
 
 event dns_EDNS_ecs(c: connection, msg: dns_msg, opt: dns_edns_ecs) {

testing/btest/scripts/base/protocols/ssl/tls13.test

@@ -1,13 +1,19 @@
+# @TEST-EXEC: echo "tls13draft16-chrome55.0.2879.0-canary-aborted.pcap"
 # @TEST-EXEC: zeek -b -C -r $TRACES/tls/tls13draft16-chrome55.0.2879.0-canary-aborted.pcap %INPUT
 # @TEST-EXEC: cat ssl.log > ssl-out.log
+# @TEST-EXEC: echo "tls13draft16-chrome55.0.2879.0-canary.pcap"
 # @TEST-EXEC: zeek -b -C -r $TRACES/tls/tls13draft16-chrome55.0.2879.0-canary.pcap %INPUT
 # @TEST-EXEC: cat ssl.log >> ssl-out.log
+# @TEST-EXEC: echo "tls13draft16-ff52.a01-aborted.pcap"
 # @TEST-EXEC: zeek -b -C -r $TRACES/tls/tls13draft16-ff52.a01-aborted.pcap %INPUT
 # @TEST-EXEC: cat ssl.log >> ssl-out.log
+# @TEST-EXEC: echo "tls13draft16-ff52.a01.pcap"
 # @TEST-EXEC: zeek -b -C -r $TRACES/tls/tls13draft16-ff52.a01.pcap %INPUT
 # @TEST-EXEC: cat ssl.log >> ssl-out.log
+# @TEST-EXEC: echo "tls13_psk_succesfull.pcap"
 # @TEST-EXEC: zeek -b -C -r $TRACES/tls/tls13_psk_succesfull.pcap %INPUT
 # @TEST-EXEC: cat ssl.log >> ssl-out.log
+# @TEST-EXEC: echo "hrr.pcap"
 # @TEST-EXEC: zeek -b -C -r $TRACES/tls/hrr.pcap %INPUT
 # @TEST-EXEC: cat ssl.log >> ssl-out.log
 # @TEST-EXEC: btest-diff ssl-out.log

testing/btest/scripts/base/protocols/ssl/tls13_encrypted_handshake_events.test

@@ -0,0 +1,22 @@
+# @TEST-EXEC: zeek -b -C -r $TRACES/tls/tls13_wolfssl.pcap %INPUT
+# @TEST-EXEC: btest-diff ssl.log
+# @TEST-EXEC: btest-diff .stdout
+
+@load base/protocols/ssl
+
+redef SSL::disable_analyzer_after_detection=F;
+
+event ssl_encrypted_data(c: connection, is_orig: bool, record_version: count, content_type: count, length: count)
+	{
+	print "encrypted", c$id, is_orig, SSL::version_strings[record_version], content_type;
+	}
+
+event ssl_established(c: connection)
+	{
+	print "Established!";
+	}
+
+event ssl_probable_encrypted_handshake_message(c: connection, is_orig: bool, length: count)
+	{
+	print "Probable handshake", is_orig, length;
+	}

testing/btest/scripts/site/local-compat.test

@@ -15,7 +15,7 @@
 # # simply update this test's TEST-START-FILE with the latest contents
 # site/local.zeek.
 
-@TEST-START-FILE local-3.3.zeek
+@TEST-START-FILE local-4.1.zeek
 ##! Local site policy. Customize as appropriate.
 ##!
 ##! This file will not be overwritten when upgrading or reinstalling!
@@ -122,4 +122,7 @@ redef digest_salt = "Please change this value.";
 # Uncomment the following line to enable logging of link-layer addresses. Enabling
 # this adds the link-layer address for each connection endpoint to the conn.log file.
 # @load policy/protocols/conn/mac-logging
+
+# Uncomment this to source zkg's package state
+# @load packages
 @TEST-END-FILE