Merge remote-tracking branch 'origin/master' into topic/johanna/spicy-tls

* origin/master:
  Update doc submodule [nomail] [skip ci]
  build_inner_connection: Use the outer packet's timestamp
  build_inner_connection: Avoid one extra Init()
  packet_analysis: Do not run DetectProtocol() on disabled analyzers
  packet_analysis/Dispatcher: Do not index table twice
  packet_analysis: Avoid shared_ptr copying for analyzer lookups
  Update doc submodule [nomail] [skip ci]
  Update doc submodule [nomail] [skip ci]
  SSL: Add new extension types and ECH test
  Update `.git-blame-ignore-revs`
  Format JSON with clang-format
  Bump pre-commit hooks
  Reformat Zeek in Spicy style

.git-blame-ignore-revs

@@ -24,3 +24,12 @@ e97c14add5b04aedc7f3f9dba59f665cbad793af
 
 # Remove trailing whitespace from script files
 a6378531dbc5c357926d98fe785bb719cc70e1b4
+
+# clang-format: Reformat Zeek in Spicy style
+f5a76c1aedc7f8886bc6abef0dfaa8065684b1f6
+
+# clang-format: Bump pre-commit hooks.
+26d04fd9fca0868d9c81e02fbffb6f81a00b56e5
+
+# clang-format: Format JSON with clang-format
+e6256446ddef5c5d5240eefff974556f2e12ac46

CHANGES

@@ -1,3 +1,63 @@
+6.2.0-dev.93 | 2023-11-01 12:04:16 +0100
+
+  * build_inner_connection: Use the outer packet's timestamp (Arne Welzel, Corelight)
+
+    Don't construct the timeval based on run_state, just use the timestamp
+    of the outer packet to avoid the extra int/double conversions required.
+
+  * build_inner_connection: Avoid one extra Init() (Arne Welzel, Corelight)
+
+    Packet::Init() is not so cheap as one might think: It computes a
+    timestamp from { 0, 0 } using double division. Just avoid this
+    by not initializing an empty Packet.
+
+  * packet_analysis: Do not run DetectProtocol() on disabled analyzers (Arne Welzel, Corelight)
+
+    This came up when disabling the TEREDO analyzer but still seeing its
+    DetectProtocol() method prominently in flame graphs.
+
+  * packet_analysis/Dispatcher: Do not index table twice (Arne Welzel, Corelight)
+
+    It's okay to return the nullptr that's in the table, no need to check
+    for != nullptr before dereferencing again.
+
+  * GH-3379: packet_analysis: Avoid shared_ptr copying for analyzer lookups (Arne Welzel, Corelight)
+
+    For deeply encapsulated connections (think AWS traffic mirroring format
+    like IP,UDP,GENEVE,IP,UDP,VXLAN,ETH,IP,TCP), the Dispatcher::Lookup()
+    method is fairly visible in profiles when running in bare mode.
+
+    This changes the Analyzer::Lookup() and Dispatcher::Lookup() return value
+    breaking the API in favor of the performance improvement.
+
+    Relates to zeek/zeek#3379.
+
+6.2.0-dev.86 | 2023-10-31 16:17:33 +0000
+
+  * SSL: Add new extension types and ECH test (Johanna Amann, Corelight)
+
+    This commit adds a multitude of new extension types that were added in
+    the last few years; it also adds grease values to extensions, curves,
+    and ciphersuites.
+
+    Furthermore, it adds a test that contains a encrypted-client-hello
+    key-exchange (which uses several extension types that we do not have in
+    our baseline so far).
+
+6.2.0-dev.83 | 2023-10-30 12:34:26 -0700
+
+  * Format JSON with clang-format (Benjamin Bannier, Corelight)
+
+    We do not activate support for JavaScript at this time since most of our
+    JavaScript code is in BTest files to test zeekjs, but these files also
+    contain other languages which leads to largely misformated files.
+
+  * Reformat Zeek in Spicy style (Benjamin Bannier, Corelight)
+
+    This largely copies over Spicy's `.clang-format` configuration file. The
+    one place where we deviate is header include order since Zeek depends on
+    headers being included in a certain order.
+
 6.2.0-dev.78 | 2023-10-30 11:47:23 +0100
 
   * GH-3421: Normalize version strings in test (Benjamin Bannier, Corelight)

NEWS

@@ -9,6 +9,10 @@ Zeek 6.2.0
 Breaking Changes
 ----------------
 
+- The methods ``Dispatcher::Lookup()`` and ``Analyzer::Lookup()`` in the packet_analysis
+  namespace were changed to return a reference to a std::shared_ptr instead of a copy
+  for performance reasons.
+
 New Functionality
 -----------------
 

VERSION

@@ -1 +1 @@
-6.2.0-dev.78
+6.2.0-dev.93

auxil/spicy

@@ -1 +1 @@
-Subproject commit 7b8eff527f60ec58eff3242253bdc1f5f1fccbef
+Subproject commit d26c81c0a2982ef81339beebff455c23713fb526

cmake

@@ -1 +1 @@
-Subproject commit 98799bb51aabb282e7dd6372aea7dbcf909469ac
+Subproject commit f7b4fbe4892594034d3d9ca639c0ffa6a99fcbe5

doc

@@ -1 +1 @@
-Subproject commit 22fe25d980131abdfadb4bdb9390aee347e77023
+Subproject commit 172ca9edceb7dd2a7bcc5ed7e91d57c00066e3d4

scripts/base/protocols/ssl/consts.zeek

@@ -168,7 +168,20 @@ export {
 	const SSL_EXTENSION_EXTENDED_MASTER_SECRET = 23;
 	const SSL_EXTENSION_TOKEN_BINDING = 24;
 	const SSL_EXTENSION_CACHED_INFO = 25;
+	const SSL_EXTENSION_TLS_LTS = 26;
+	const SSL_EXTENSION_COMPRESS_CERTIFICATE = 27;
+	const SSL_EXTENSION_RECORD_SIZE_LIMIT = 28;
+	const SSL_EXTENSION_PWD_PROTECT = 29;
+	const SSL_EXTENSION_PWD_CLEAR = 30;
+	const SSL_EXTENSION_PASSWORD_SALT = 31;
+	const SSL_EXTENSION_TICKET_PINNING = 32;
+	const SSL_EXTENSION_TLS_CERT_WITH_EXTERN_PSK = 33;
+	const SSL_EXTENSION_DELEGATED_CREDENTIAL = 34;
 	const SSL_EXTENSION_SESSIONTICKET_TLS = 35;
+	const SSL_EXTENSION_TLMSP = 36;
+	const SSL_EXTENSION_TLMSP_PROXYING = 37;
+	const SSL_EXTENSION_TLMSP_DELEGATE = 38;
+	const SSL_EXTENSION_SUPPORTED_EKT_CIPHERS = 39;
 	const SSL_EXTENSION_KEY_SHARE_OLD = 40;
 	const SSL_EXTENSION_PRE_SHARED_KEY = 41;
 	const SSL_EXTENSION_EARLY_DATA = 42;
@@ -182,16 +195,22 @@ export {
 	const SSL_EXTENSION_SIGNATURE_ALGORITHMS_CERT = 50;
 	const SSL_EXTENSION_KEY_SHARE = 51;
 	const SSL_EXTENSION_TRANSPARENCY_INFO = 52;
-	const SSL_EXTENSION_CONNECTION_ID = 53;
+	const SSL_EXTENSION_CONNECTION_ID_DEPRECATED = 53;
+	const SSL_EXTENSION_CONNECTION_ID = 54;
 	const SSL_EXTENSION_EXTERNAL_ID_HASH = 55;
 	const SSL_EXTENSION_EXTERNAL_SESSION_ID = 56;
 	const SSL_EXTENSION_QUIC_TRANSPORT_PARAMETERS = 57;
+	const SSL_EXTENSION_TICKET_REQUEST = 58;
+	const SSL_EXTENSION_DNSSEC_CHAIN = 59;
+	const SSL_EXTENSION_SEQUENCE_NUMBER_ENCRYPTION_ALGORITHMS = 60;
 	const SSL_EXTENSION_NEXT_PROTOCOL_NEGOTIATION = 13172;
 	const SSL_EXTENSION_ORIGIN_BOUND_CERTIFICATES = 13175;
 	const SSL_EXTENSION_ENCRYPTED_CLIENT_CERTIFICATES = 13180;
+	const SSL_EXTENSION_APPLICATION_SETTING = 17513;
 	const SSL_EXTENSION_CHANNEL_ID = 30031;
 	const SSL_EXTENSION_CHANNEL_ID_NEW = 30032;
 	const SSL_EXTENSION_PADDING_TEMP = 35655;
+	const SSL_EXTENSION_ENCRYPTED_CLIENT_HELLO = 65037;
 	const SSL_EXTENSION_RENEGOTIATION_INFO = 65281;
 
 	## Mapping between numeric codes and human readable strings for SSL/TLS
@@ -225,7 +244,20 @@ export {
 		[23] = "extended_master_secret",
 		[24] = "token_binding", # temporary till 2017-03-06 - draft-ietf-tokbind-negotiation
 		[25] = "cached_info",
+		[26] = "tls_lts", # draft-gutmann-tls-lts
+		[27] = "compress_certificate", # RFC8879
+		[28] = "record_size_limit", # RFC8449
+		[29] = "pwd_protect", # RFC8492
+		[30] = "pwd_clear", # RFC8492
+		[31] = "password_salt", # RFC8492
+		[32] = "ticket_pinning", # RFC8672
+		[33] = "tls_cert_with_extern_psk", # RFC8773
+		[34] = "delegated_credential", # RFC9345
 		[35] = "SessionTicket TLS",
+		[36] = "TLMSP", # ETSI TS 103 523-2
+		[37] = "TLMSP_proxying", # ETSI TS 103 523-2
+		[38] = "TLMSP_delegate", # ETSI TS 103 523-2
+		[39] = "supported_ekt_ciphers", # RFC8870
 		[40] = "key_share_old", # new for TLS 1.3, used in some of the drafts. Did not make it into the RFC. Was used for extended_random before.
 		[41] = "pre_shared_key", # new for 1.3, see RFC 8446
 		[42] = "early_data", # new for 1.3, see RFC 8446
@@ -238,18 +270,41 @@ export {
 		[49] = "post_handshake_auth", # new for 1.3, see RFC 8446
 		[50] = "signature_algorithms_cert", # new for 1.3, see RFC 8446
 		[51] = "key_share", # new for 1.3, see RFC 8446
-		[52] = "transparency_info", # temporary - draft-ietf-trans-rfc6962-bis-34
+		[52] = "transparency_info", # RFC9162
-		[53] = "connection_id", # temporary -d draft-ietf-tls-dtls-connection-id
+		[53] = "connection_id_deprecated", # RFC9146
-		[55] = "external_id_hash", # temporary - RFC-ietf-mmusic-sdp-uks-07
+		[54] = "connection_id", # RFC9146
-		[56] = "external_session_id", # temporary - RFC-ietf-mmusic-sdp-uks-07
+		[55] = "external_id_hash", # RFC8844
-		[57] = "quic_transport_parameters", # temporary - draft-ietf-quic-tls-32
+		[56] = "external_session_id", # RFC8844
+		[57] = "quic_transport_parameters", # RFC9001
+		[58] = "ticket_request", # RFC9149]
+		[59] = "dnssec_chain", # RFC9102
+		[60] = "sequence_number_encryption_algorithms", # draft-pismenny-tls-dtls-plaintext-sequence-number-01
 		[13172] = "next_protocol_negotiation",
 		[13175] = "origin_bound_certificates",
 		[13180] = "encrypted_client_certificates",
+		[17513] = "application_setting", # draft-vvv-tls-alps-01.html
 		[30031] = "channel_id",
 		[30032] = "channel_id_new",
 		[35655] = "padding",
-		[65281] = "renegotiation_info"
+		[65037] = "encrypted_client_hello", # draft-ietf-tls-esni
+		[65281] = "renegotiation_info",
+		# GREASE values - rfc8701
+		[2570] = "grease_0x0A0A",
+		[6682] = "grease_0x1A1A",
+		[10794] = "grease_0x2A2A",
+		[14906] = "grease_0x3A3A",
+		[19018] = "grease_0x4A4A",
+		[23130] = "grease_0x5A5A",
+		[27242] = "grease_0x6A6A",
+		[31354] = "grease_0x7A7A",
+		[35466] = "grease_0x8A8A",
+		[39578] = "grease_0x9A9A",
+		[43690] = "grease_0xAAAA",
+		[47802] = "grease_0xBABA",
+		[51914] = "grease_0xCACA",
+		[56026] = "grease_0xDADA",
+		[60138] = "grease_0xEAEA",
+		[64250] = "grease_0xFAFA"
 	} &default=function(i: count):string { return fmt("unknown-%d", i); };
 
 	## Mapping between numeric codes and human readable string for SSL/TLS elliptic curves.
@@ -293,7 +348,24 @@ export {
 		[259] = "ffdhe6144",
 		[260] = "ffdhe8192",
 		[0xFF01] = "arbitrary_explicit_prime_curves",
-		[0xFF02] = "arbitrary_explicit_char2_curves"
+		[0xFF02] = "arbitrary_explicit_char2_curves",
+		# GREASE values - rfc8701
+		[2570] = "grease_0x0A0A",
+		[6682] = "grease_0x1A1A",
+		[10794] = "grease_0x2A2A",
+		[14906] = "grease_0x3A3A",
+		[19018] = "grease_0x4A4A",
+		[23130] = "grease_0x5A5A",
+		[27242] = "grease_0x6A6A",
+		[31354] = "grease_0x7A7A",
+		[35466] = "grease_0x8A8A",
+		[39578] = "grease_0x9A9A",
+		[43690] = "grease_0xAAAA",
+		[47802] = "grease_0xBABA",
+		[51914] = "grease_0xCACA",
+		[56026] = "grease_0xDADA",
+		[60138] = "grease_0xEAEA",
+		[64250] = "grease_0xFAFA"
 	} &default=function(i: count):string { return fmt("unknown-%d", i); };
 
 	## Mapping between numeric codes and human readable string for SSL/TLS EC point formats.
@@ -1081,6 +1153,23 @@ export {
 		[SSL_RSA_WITH_DES_CBC_MD5] = "SSL_RSA_WITH_DES_CBC_MD5",
 		[SSL_RSA_WITH_3DES_EDE_CBC_MD5] = "SSL_RSA_WITH_3DES_EDE_CBC_MD5",
 		[TLS_EMPTY_RENEGOTIATION_INFO_SCSV] = "TLS_EMPTY_RENEGOTIATION_INFO_SCSV",
+		# GREASE - rfc8701
+		[2570] = "grease_0x0A0A",
+		[6682] = "grease_0x1A1A",
+		[10794] = "grease_0x2A2A",
+		[14906] = "grease_0x3A3A",
+		[19018] = "grease_0x4A4A",
+		[23130] = "grease_0x5A5A",
+		[27242] = "grease_0x6A6A",
+		[31354] = "grease_0x7A7A",
+		[35466] = "grease_0x8A8A",
+		[39578] = "grease_0x9A9A",
+		[43690] = "grease_0xAAAA",
+		[47802] = "grease_0xBABA",
+		[51914] = "grease_0xCACA",
+		[56026] = "grease_0xDADA",
+		[60138] = "grease_0xEAEA",
+		[64250] = "grease_0xFAFA"
 	} &default=function(i: count):string { return fmt("unknown-%d", i); };
 
 }

src/analyzer/protocol/rdp/RDP.cc

@@ -11,6 +11,7 @@ RDP_Analyzer::RDP_Analyzer(Connection* c) : analyzer::tcp::TCP_ApplicationAnalyz
     interp = new binpac::RDP::RDP_Conn(this);
 
     had_gap = false;
+    // ssl = nullptr;
 }
 
 RDP_Analyzer::~RDP_Analyzer() { delete interp; }
@@ -43,18 +44,19 @@ void RDP_Analyzer::DeliverStream(int len, const u_char* data, bool orig) {
         // 0x01 is SSL/TLS
         // 0x03-0x04 is CredSSP which is effectively SSL/TLS
         if ( interp->encryption_method() > 0x00 ) {
-            /*			if ( ! ssl )
+            /*
-                            {
+    if ( ! ssl ) {
         ssl = new analyzer::ssl::SSL_Analyzer(Conn());
-                            if ( ! AddChildAnalyzer(ssl) )
+        if ( ! AddChildAnalyzer(ssl) ) {
-                                {
+            reporter->AnalyzerError(this,
-                                reporter->AnalyzerError(this, "failed to add TCP child analyzer "
+                                    "failed to add TCP child analyzer "
                                     "to RDP analyzer: already exists");
             return;
         }
     }
 
-                        ForwardStream(len, data, orig); */
+    ForwardStream(len, data, orig);
+             */
         }
         else {
             if ( rdp_native_encrypted_data )

src/analyzer/protocol/ssl/Plugin.cc

@@ -11,10 +11,8 @@ namespace zeek::plugin::detail::Zeek_SSL {
 class Plugin : public zeek::plugin::Plugin {
 public:
     zeek::plugin::Configuration Configure() override {
-        // AddComponent(
+        // AddComponent(new zeek::analyzer::Component("SSL", zeek::analyzer::ssl::SSL_Analyzer::Instantiate));
-        // 	new zeek::analyzer::Component("SSL", zeek::analyzer::ssl::SSL_Analyzer::Instantiate));
+        // AddComponent(new zeek::analyzer::Component("DTLS", zeek::analyzer::dtls::DTLS_Analyzer::Instantiate));
-        // AddComponent(new zeek::analyzer::Component(
-        // 	"DTLS", zeek::analyzer::dtls::DTLS_Analyzer::Instantiate));
 
         zeek::plugin::Configuration config;
         config.name = "Zeek::SSL";

src/packet_analysis/Analyzer.cc

@@ -53,23 +53,47 @@ bool Analyzer::IsAnalyzer(const char* name) {
     return packet_mgr->GetComponentName(tag) == name;
 }
 
-AnalyzerPtr Analyzer::Lookup(uint32_t identifier) const { return dispatcher.Lookup(identifier); }
+const AnalyzerPtr& Analyzer::Lookup(uint32_t identifier) const { return dispatcher.Lookup(identifier); }
 
-bool Analyzer::ForwardPacket(size_t len, const uint8_t* data, Packet* packet, uint32_t identifier) const {
+// Find the next inner analyzer using identifier or via DetectProtocol(),
-    auto inner_analyzer = Lookup(identifier);
+// otherwise return the default analyzer.
-    if ( ! inner_analyzer ) {
+const AnalyzerPtr& Analyzer::FindInnerAnalyzer(size_t len, const uint8_t* data, Packet* packet,
+                                               uint32_t identifier) const {
+    const auto& identifier_based_analyzer = Lookup(identifier);
+    if ( identifier_based_analyzer )
+        return identifier_based_analyzer;
+
+    const auto& detect_based_analyzer = DetectInnerAnalyzer(len, data, packet);
+    if ( detect_based_analyzer )
+        return detect_based_analyzer;
+
+    return default_analyzer;
+}
+
+// Find the next inner analyzer via DetectProtocol(), otherwise the default analyzer.
+const AnalyzerPtr& Analyzer::FindInnerAnalyzer(size_t len, const uint8_t* data, Packet* packet) const {
+    const auto& detect_based_analyzer = DetectInnerAnalyzer(len, data, packet);
+    if ( detect_based_analyzer )
+        return detect_based_analyzer;
+
+    return default_analyzer;
+}
+
+// Return an analyzer found via DetectProtocol() for the given data, else nil.
+const AnalyzerPtr& Analyzer::DetectInnerAnalyzer(size_t len, const uint8_t* data, Packet* packet) const {
     for ( const auto& child : analyzers_to_detect ) {
-            if ( child->DetectProtocol(len, data, packet) ) {
+        if ( child->IsEnabled() && child->DetectProtocol(len, data, packet) ) {
             DBG_LOG(DBG_PACKET_ANALYSIS, "Protocol detection in %s succeeded, next layer analyzer is %s",
                     GetAnalyzerName(), child->GetAnalyzerName());
-                inner_analyzer = child;
+            return child;
-                break;
-            }
         }
     }
 
-    if ( ! inner_analyzer )
+    return nil;
-        inner_analyzer = default_analyzer;
+}
+
+bool Analyzer::ForwardPacket(size_t len, const uint8_t* data, Packet* packet, uint32_t identifier) const {
+    const auto& inner_analyzer = FindInnerAnalyzer(len, data, packet, identifier);
 
     if ( ! inner_analyzer ) {
         DBG_LOG(DBG_PACKET_ANALYSIS, "Analysis in %s failed, could not find analyzer for identifier %#x.",
@@ -89,23 +113,12 @@ bool Analyzer::ForwardPacket(size_t len, const uint8_t* data, Packet* packet, ui
 
     DBG_LOG(DBG_PACKET_ANALYSIS, "Analysis in %s succeeded, next layer identifier is %#x.", GetAnalyzerName(),
             identifier);
+
     return inner_analyzer->AnalyzePacket(len, data, packet);
 }
 
 bool Analyzer::ForwardPacket(size_t len, const uint8_t* data, Packet* packet) const {
-    AnalyzerPtr inner_analyzer = nullptr;
+    const auto& inner_analyzer = FindInnerAnalyzer(len, data, packet);
-
-    for ( const auto& child : analyzers_to_detect ) {
-        if ( child->DetectProtocol(len, data, packet) ) {
-            DBG_LOG(DBG_PACKET_ANALYSIS, "Protocol detection in %s succeeded, next layer analyzer is %s",
-                    GetAnalyzerName(), child->GetAnalyzerName());
-            inner_analyzer = child;
-            break;
-        }
-    }
-
-    if ( ! inner_analyzer )
-        inner_analyzer = default_analyzer;
 
     if ( ! inner_analyzer ) {
         DBG_LOG(DBG_PACKET_ANALYSIS, "Analysis in %s stopped, no default analyzer available.", GetAnalyzerName());

src/packet_analysis/Analyzer.h

@@ -9,12 +9,16 @@
 #include "zeek/session/Session.h"
 
 namespace zeek::packet_analysis {
+class Analyzer;
+using AnalyzerPtr = std::shared_ptr<Analyzer>;
 
 /**
  * Main packet analyzer interface.
  */
 class Analyzer {
 public:
+    static inline AnalyzerPtr nil = nullptr;
+
     /**
      * Constructor.
      *
@@ -198,7 +202,7 @@ protected:
      * @return The analyzer registered for the given identifier. Returns a
      * nullptr if no analyzer is registered.
      */
-    AnalyzerPtr Lookup(uint32_t identifier) const;
+    const AnalyzerPtr& Lookup(uint32_t identifier) const;
 
     /**
      * Returns an analyzer based on a script-land definition.
@@ -256,6 +260,11 @@ private:
     void EnqueueAnalyzerViolationInfo(session::Session* session, const char* reason, const char* data, int len,
                                       const zeek::Tag& arg_tag);
 
+    // Internal helpers to find an appropriate next inner analyzer.
+    const AnalyzerPtr& FindInnerAnalyzer(size_t len, const uint8_t* data, Packet* packet, uint32_t identifier) const;
+    const AnalyzerPtr& FindInnerAnalyzer(size_t len, const uint8_t* data, Packet* packet) const;
+    const AnalyzerPtr& DetectInnerAnalyzer(size_t len, const uint8_t* data, Packet* packet) const;
+
     zeek::Tag tag;
     Dispatcher dispatcher;
     AnalyzerPtr default_analyzer = nullptr;
@@ -270,7 +279,4 @@ private:
 
     void Init(const zeek::Tag& tag);
 };
-
-using AnalyzerPtr = std::shared_ptr<Analyzer>;
-
 } // namespace zeek::packet_analysis

src/packet_analysis/Dispatcher.cc

@@ -48,12 +48,12 @@ void Dispatcher::Register(uint32_t identifier, AnalyzerPtr analyzer) {
     table[index] = std::move(analyzer);
 }
 
-AnalyzerPtr Dispatcher::Lookup(uint32_t identifier) const {
+const AnalyzerPtr& Dispatcher::Lookup(uint32_t identifier) const {
     int64_t index = identifier - lowest_identifier;
-    if ( index >= 0 && index < static_cast<int64_t>(table.size()) && table[index] != nullptr )
+    if ( index >= 0 && index < static_cast<int64_t>(table.size()) )
         return table[index];
 
-    return nullptr;
+    return Analyzer::nil;
 }
 
 size_t Dispatcher::Count() const {

src/packet_analysis/Dispatcher.h

@@ -35,7 +35,7 @@ public:
      * @return The analyzer registered for the given identifier. Returns a
      * nullptr if no analyzer is registered.
      */
-    AnalyzerPtr Lookup(uint32_t identifier) const;
+    const AnalyzerPtr& Lookup(uint32_t identifier) const;
 
     /**
      * Returns the number of registered analyzers.

src/packet_analysis/protocol/iptunnel/IPTunnel.cc

@@ -152,8 +152,6 @@ std::unique_ptr<Packet> build_inner_packet(Packet* outer_pkt, int* encap_index,
                                            std::shared_ptr<EncapsulationStack> encap_stack, uint32_t inner_cap_len,
                                            const u_char* data, int link_type, BifEnum::Tunnel::Type tunnel_type,
                                            const Tag& analyzer_tag) {
-    auto inner_pkt = std::make_unique<Packet>();
-
     assert(outer_pkt->cap_len >= inner_cap_len);
     assert(outer_pkt->len >= outer_pkt->cap_len - inner_cap_len);
 
@@ -166,10 +164,7 @@ std::unique_ptr<Packet> build_inner_packet(Packet* outer_pkt, int* encap_index,
     uint32_t consumed_len = outer_pkt->cap_len - inner_cap_len;
     uint32_t inner_wire_len = outer_pkt->len - consumed_len;
 
-    pkt_timeval ts;
+    auto inner_pkt = std::make_unique<Packet>(link_type, &outer_pkt->ts, inner_cap_len, inner_wire_len, data);
-    ts.tv_sec = static_cast<time_t>(run_state::current_timestamp);
-    ts.tv_usec = static_cast<suseconds_t>((run_state::current_timestamp - static_cast<double>(ts.tv_sec)) * 1000000);
-    inner_pkt->Init(link_type, &ts, inner_cap_len, inner_wire_len, data);
 
     *encap_index = 0;
     if ( outer_pkt->session ) {

src/packet_analysis/protocol/iptunnel/IPTunnel.h

@@ -77,6 +77,8 @@ protected:
  * The wire length (pkt->len) of the inner packet is computed based on the wire length
  * of the outer packet and the differences in capture lengths.
  *
+ * The inner packet's timestamp is set to the same value as the outer packet's timestamp.
+ *
  * @param outer_pkt The packet containing the encapsulation. This packet should contain
  * @param encap_index A return value for the current index into the encapsulation stack.
  * This is returned to allow analyzers to know what point in the stack they were operating

testing/btest/Baseline/scripts.base.protocols.ssl.tls13-encrypted-client-hello/.stdout

@@ -0,0 +1,53 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+T, grease_0xDADA
+T, renegotiation_info
+T, application_layer_protocol_negotiation
+T, signature_algorithms
+T, key_share
+T, psk_key_exchange_modes
+T, application_setting
+T, SessionTicket TLS
+T, supported_versions
+Curves, 192.168.20.65, 162.159.138.85
+grease_0x1A1A
+x25519
+secp256r1
+secp384r1
+T, supported_groups
+T, encrypted_client_hello
+T, extended_master_secret
+T, status_request
+T, signed_certificate_timestamp
+T, ec_point_formats
+T, server_name
+T, compress_certificate
+T, grease_0x9A9A
+T, padding
+F, supported_versions
+F, key_share
+T, grease_0xBABA
+Curves, 192.168.20.65, 162.159.138.85
+grease_0xDADA
+x25519
+secp256r1
+secp384r1
+T, supported_groups
+T, SessionTicket TLS
+T, application_setting
+T, ec_point_formats
+T, encrypted_client_hello
+T, renegotiation_info
+T, signed_certificate_timestamp
+T, status_request
+T, signature_algorithms
+T, compress_certificate
+T, psk_key_exchange_modes
+T, extended_master_secret
+T, server_name
+T, application_layer_protocol_negotiation
+T, supported_versions
+T, key_share
+T, grease_0xFAFA
+T, padding
+F, supported_versions
+F, key_share

testing/btest/Baseline/scripts.base.protocols.ssl.tls13-encrypted-client-hello/ssl.log

@@ -0,0 +1,12 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	ssl
+#open XXXX-XX-XX-XX-XX-XX
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	version	cipher	curve	server_name	resumed	last_alert	next_protocol	established	ssl_history	cert_chain_fps	client_cert_chain_fps	sni_matches_cert
+#types	time	string	addr	port	addr	port	string	string	string	string	bool	string	string	bool	string	vector[string]	vector[string]	bool
+XXXXXXXXXX.XXXXXX	CHhAvVGS1DHFjwGM9	192.168.20.65	51066	162.159.138.85	443	TLSv13	TLS_AES_128_GCM_SHA256	x25519	cloudflare-ech.com	F	-	-	T	CsiI	-	-	-
+XXXXXXXXXX.XXXXXX	ClEkJM2Vm5giqnMf4h	192.168.20.65	51071	162.159.138.85	443	TLSv13	TLS_AES_128_GCM_SHA256	x25519	cloudflare-ech.com	F	-	-	T	CsiI	-	-	-
+#close XXXX-XX-XX-XX-XX-XX

testing/btest/Baseline/scripts.base.protocols.ssl.tls13/.stdout

@@ -1,16 +1,16 @@
 ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
 tls13draft16-chrome55.0.2879.0-canary-aborted.pcap
 key_share, [orig_h=192.168.6.203, orig_p=53226/tcp, resp_h=52.32.149.186, resp_p=443/tcp], T
-unknown-27242
+grease_0x6A6A
 x25519
 client, TLSv10, TLSv12
 key_share, [orig_h=192.168.6.203, orig_p=53227/tcp, resp_h=52.32.149.186, resp_p=443/tcp], T
-unknown-19018
+grease_0x4A4A
 x25519
 client, TLSv10, TLSv12
 tls13draft16-chrome55.0.2879.0-canary.pcap
 key_share, [orig_h=192.168.6.203, orig_p=53994/tcp, resp_h=138.68.41.77, resp_p=443/tcp], T
-unknown-43690
+grease_0xAAAA
 x25519
 client, TLSv10, TLSv12
 key_share, [orig_h=192.168.6.203, orig_p=53994/tcp, resp_h=138.68.41.77, resp_p=443/tcp], F
@@ -24,7 +24,7 @@ established, [orig_h=192.168.6.203, orig_p=53994/tcp, resp_h=138.68.41.77, resp_
 encrypted, [orig_h=192.168.6.203, orig_p=53994/tcp, resp_h=138.68.41.77, resp_p=443/tcp], T, TLSv10, 23
 encrypted, [orig_h=192.168.6.203, orig_p=53994/tcp, resp_h=138.68.41.77, resp_p=443/tcp], F, TLSv10, 23
 key_share, [orig_h=192.168.6.203, orig_p=53996/tcp, resp_h=138.68.41.77, resp_p=443/tcp], T
-unknown-60138
+grease_0xEAEA
 x25519
 client, TLSv10, TLSv12
 key_share, [orig_h=192.168.6.203, orig_p=53996/tcp, resp_h=138.68.41.77, resp_p=443/tcp], F

testing/btest/scripts/base/protocols/ssl/tls13-encrypted-client-hello.test

@@ -0,0 +1,20 @@
+# @TEST-EXEC: zeek -b -C -r $TRACES/tls/tls13-ech.pcap %INPUT
+# @TEST-EXEC: btest-diff ssl.log
+# @TEST-EXEC: btest-diff .stdout
+
+# This is a trace that uses the new encrypted client hello extension to hide (among others)
+# the real value of the SNI.
+
+@load base/protocols/ssl
+
+event ssl_extension(c: connection, is_client: bool, code: count, val: string)
+	{
+	print is_client, SSL::extensions[code];
+	}
+
+event ssl_extension_elliptic_curves(c: connection, is_client: bool, curves: index_vec)
+	{
+	print "Curves", c$id$orig_h, c$id$resp_h;
+	for ( i in curves )
+		print SSL::ec_curves[curves[i]];
+	}