Merge remote-tracking branch 'origin/topic/awelzel/quic-draft-mvfst-versions'

* origin/topic/awelzel/quic-draft-mvfst-versions:
  quic: Handle and log unhandled_version
  quic: Support decryption of a few more versions

CHANGES

@@ -1,3 +1,11 @@
+6.2.0-dev.344 | 2024-01-10 14:07:18 +0100
+
+  * quic: Handle and log unhandled_version (Arne Welzel, Corelight)
+
+  * quic: Support decryption of a few more versions (Arne Welzel, Corelight)
+
+  * parse: Support @if conditionals in record definitions (Arne Welzel, Corelight)
+
 6.2.0-dev.339 | 2024-01-09 09:15:39 +0100
 
   * Dict: Invalidate iterators during Clear() (Arne Welzel, Corelight)

NEWS

@@ -133,7 +133,9 @@ New Functionality
   events.
 
 - The QUIC analyzer has been extended to support analyzing QUIC Version 2
-  INITIAL packets (RFC 9369).
+  INITIAL packets (RFC 9369). Additionally, prior draft and some of
+  Facebook's mvfst versions are supported. Unknown QUIC versions will now be
+  reported in ``quic.log`` as an entry with a ``U`` history field.
 
 - Conditional directives (``@if``, ``@ifdef``, ``@ifndef``, ``@else`` and
   ``@endif``) can now be placed within a record's definition to conditionally

VERSION

@@ -1 +1 @@
-6.2.0-dev.339
+6.2.0-dev.344

scripts/base/protocols/quic/consts.zeek

@@ -4,5 +4,24 @@ export {
 	const version_strings: table[count] of string = {
 		[0x00000001] = "1",
 		[0x6b3343cf] = "quicv2",
+		[0xff000016] = "draft-22",
+		[0xff000017] = "draft-23",
+		[0xff000018] = "draft-24",
+		[0xff000019] = "draft-25",
+		[0xff00001a] = "draft-26",
+		[0xff00001b] = "draft-27",
+		[0xff00001c] = "draft-28",
+		[0xff00001d] = "draft-29",
+		[0xff00001e] = "draft-30",
+		[0xff00001f] = "draft-30",
+		[0xff000020] = "draft-32",
+		[0xff000021] = "draft-33",
+		[0xff000022] = "draft-34",
+		[0xfaceb001] = "mvfst (faceb001)",
+		[0xfaceb002] = "mvfst (faceb002)",
+		[0xfaceb00e] = "mvfst (faceb00e)",
+		[0xfaceb011] = "mvfst (faceb011)",
+		[0xfaceb012] = "mvfst (faceb012)",
+		[0xfaceb013] = "mvfst (faceb013)",
 	} &default=function(version: count): string { return fmt("unknown-%x", version); };
 }

scripts/base/protocols/quic/main.zeek

@@ -157,6 +157,19 @@ event QUIC::retry_packet(c: connection, is_orig: bool, version: count, dcid: str
 	delete c$quic;
 	}
 
+# If we couldn't handle a version, log it as a single record.
+event QUIC::unhandled_version(c: connection, is_orig: bool, version: count, dcid: string, scid: string)
+	{
+	if ( ! c?$quic )
+		set_conn(c, is_orig, version, dcid, scid);
+
+	add_to_history(c, is_orig, "UNHANDLED_VERSION");
+
+	log_record(c$quic);
+
+	delete c$quic;
+	}
+
 # Upon a connection_close_frame(), if any c$quic state is pending to be logged, do so
 # now and prepare for a new entry.
 event QUIC::connection_close_frame(c: connection, is_orig: bool, version: count, dcid: string, scid: string, error_code: count, reason_phrase: string)

src/analyzer/protocol/quic/QUIC.spicy

@@ -27,9 +27,6 @@ function can_decrypt(long_header: LongHeaderPacket, context: ConnectionIDInfo, i
   if ( ! long_header.is_initial )
     return False;
 
-  if ( long_header.version != Version1 && long_header.version != Version2 )
-    return False;
-
   if ( is_client )
     return ! context.client_initial_processed;
 
@@ -80,6 +77,25 @@ type ConnectionIDInfo = struct {
 ##############
 # Definitions
 ##############
+const VersionDraft22: uint32 = 0xff000016;
+const VersionDraft23: uint32 = 0xff000017;
+const VersionDraft24: uint32 = 0xff000018;
+const VersionDraft25: uint32 = 0xff000019;
+const VersionDraft26: uint32 = 0xff00001a;
+const VersionDraft27: uint32 = 0xff00001b;
+const VersionDraft28: uint32 = 0xff00001c;
+const VersionDraft29: uint32 = 0xff00001d;
+const VersionDraft30: uint32 = 0xff00001e;
+const VersionDraft31: uint32 = 0xff00001f;
+const VersionDraft32: uint32 = 0xff000020;
+const VersionDraft33: uint32 = 0xff000021;
+const VersionDraft34: uint32 = 0xff000022;
+const VersionFace001: uint32 = 0xfaceb001;
+const VersionFace002: uint32 = 0xfaceb002;
+const VersionFace00e: uint32 = 0xfaceb00e;
+const VersionFace011: uint32 = 0xfaceb011;
+const VersionFace012: uint32 = 0xfaceb012;
+const VersionFace013: uint32 = 0xfaceb013;
 const Version1: uint32 = 0x00000001;
 const Version2: uint32 = 0x6b3343cf;
 
@@ -236,6 +252,25 @@ public type LongHeaderPacket = unit {
   src_conn_id: bytes &size=self.client_conn_id_length;
 
   switch ( self.version ) {
+    VersionDraft22,
+    VersionDraft23,
+    VersionDraft24,
+    VersionDraft25,
+    VersionDraft26,
+    VersionDraft27,
+    VersionDraft28,
+    VersionDraft29,
+    VersionDraft30,
+    VersionDraft31,
+    VersionDraft32,
+    VersionDraft33,
+    VersionDraft34,
+    VersionFace001,
+    VersionFace002,
+    VersionFace00e,
+    VersionFace011,
+    VersionFace012,
+    VersionFace013,
     Version1 -> v1: LongHeaderPacketV1(self);
     Version2 -> v2: LongHeaderPacketV2(self);
     * -> unknown: UnhandledVersion(self) {

src/analyzer/protocol/quic/decrypt_crypto.cc

@@ -24,7 +24,6 @@ refactors as C++ development is not our main profession.
 #include <array>
 #include <cstdint>
 #include <cstdlib>
-#include <cstring>
 #include <memory>
 #include <string>
 #include <vector>
@@ -208,7 +207,6 @@ hilti::rt::Bytes decrypt(const std::vector<uint8_t>& client_key, const hilti::rt
 // Pre-initialized SSL contexts for re-use. Not thread-safe. These are only used in expand-only mode
 // and have a fixed HKDF info set.
 struct HkdfCtx {
-    bool initialized = false;
     EVP_PKEY_CTX* client_in_ctx = nullptr;
     EVP_PKEY_CTX* server_in_ctx = nullptr;
     EVP_PKEY_CTX* key_info_ctx = nullptr;
@@ -216,7 +214,6 @@ struct HkdfCtx {
     EVP_PKEY_CTX* hp_info_ctx = nullptr;
 };
 
-
 struct HkdfCtxParam {
     EVP_PKEY_CTX** ctx;
     std::vector<uint8_t> info;
@@ -251,9 +248,9 @@ std::vector<uint8_t> hkdf_expand(EVP_PKEY_CTX* ctx, size_t out_len, const std::v
 
 class QuicPacketProtection {
 public:
-    std::vector<uint8_t> GetSecret(bool is_orig, const hilti::rt::Bytes& connection_id) {
+    std::vector<uint8_t> GetSecret(bool is_orig, uint32_t version, const hilti::rt::Bytes& connection_id) {
         const auto& ctxs = GetHkdfCtxs();
-        const auto initial_secret = hkdf_extract(GetInitialSalt(), connection_id);
+        const auto initial_secret = hkdf_extract(GetInitialSalt(version), connection_id);
         EVP_PKEY_CTX* ctx = is_orig ? ctxs.client_in_ctx : ctxs.server_in_ctx;
         return hkdf_expand(ctx, INITIAL_SECRET_LEN, initial_secret);
     }
@@ -273,7 +270,8 @@ public:
         return hkdf_expand(ctxs.hp_info_ctx, AEAD_HP_LEN, secret);
     }
 
-    virtual const std::vector<uint8_t>& GetInitialSalt() = 0;
+    virtual bool Supports(uint32_t version) const = 0;
+    virtual const std::vector<uint8_t>& GetInitialSalt(uint32_t version) const = 0;
     virtual HkdfCtx& GetHkdfCtxs() = 0;
 
     virtual ~QuicPacketProtection() = default;
@@ -293,34 +291,73 @@ public:
 // QUIC v1
 //
 // https://datatracker.ietf.org/doc/html/rfc9001
-std::vector<uint8_t> INITIAL_SALT_V1 = {0x38, 0x76, 0x2c, 0xf7, 0xf5, 0x59, 0x34, 0xb3, 0x4d, 0x17,
-                                        0x9a, 0xe6, 0xa4, 0xc8, 0x0c, 0xad, 0xcc, 0xbb, 0x7f, 0x0a};
-
-std::vector<uint8_t> CLIENT_INITIAL_INFO = {0x00, 0x20, 0x0f, 0x74, 0x6c, 0x73, 0x31, 0x33, 0x20, 0x63,
-                                            0x6c, 0x69, 0x65, 0x6e, 0x74, 0x20, 0x69, 0x6e, 0x00};
-
-std::vector<uint8_t> SERVER_INITIAL_INFO = {0x00, 0x20, 0x0f, 0x74, 0x6c, 0x73, 0x31, 0x33, 0x20, 0x73,
-                                            0x65, 0x72, 0x76, 0x65, 0x72, 0x20, 0x69, 0x6e, 0x00};
-
-std::vector<uint8_t> KEY_INFO = {0x00, 0x10, 0x0e, 0x74, 0x6c, 0x73, 0x31, 0x33, 0x20,
-                                 0x71, 0x75, 0x69, 0x63, 0x20, 0x6b, 0x65, 0x79, 0x00};
-
-std::vector<uint8_t> IV_INFO = {0x00, 0x0c, 0x0d, 0x74, 0x6c, 0x73, 0x31, 0x33, 0x20,
-                                0x71, 0x75, 0x69, 0x63, 0x20, 0x69, 0x76, 0x00};
-
-std::vector<uint8_t> HP_INFO = {0x00, 0x10, 0x0d, 0x74, 0x6c, 0x73, 0x31, 0x33, 0x20,
-                                0x71, 0x75, 0x69, 0x63, 0x20, 0x68, 0x70, 0x00};
-
-
 class QuicPacketProtectionV1 : public QuicPacketProtection {
 public:
-    virtual std::vector<uint8_t>& GetInitialSalt() override { return INITIAL_SALT_V1; }
+    virtual bool Supports(uint32_t version) const override {
+        // Quic V1
+        if ( version == 0x00000001 )
+            return true;
+
+        // Draft 22 through 34
+        if ( version >= 0xff000016 && version <= 0xff000022 )
+            return true;
+
+        // mvfst from facebook
+        if ( version == 0xfaceb001 || (version >= 0xfaceb002 && version <= 0xfaceb013) )
+            return true;
+
+        return false;
+    };
+
+    virtual const std::vector<uint8_t>& GetInitialSalt(uint32_t version) const override {
+        static std::vector<uint8_t> INITIAL_SALT_V1 = {0x38, 0x76, 0x2c, 0xf7, 0xf5, 0x59, 0x34, 0xb3, 0x4d, 0x17,
+                                                       0x9a, 0xe6, 0xa4, 0xc8, 0x0c, 0xad, 0xcc, 0xbb, 0x7f, 0x0a};
+
+        // https://insights.sei.cmu.edu/documents/4499/2023_017_001_890985.pdf
+        static std::vector<uint8_t> INITIAL_SALT_D22 = {0x7f, 0xbc, 0xdb, 0x0e, 0x7c, 0x66, 0xbb, 0xe9, 0x19, 0x3a,
+                                                        0x96, 0xcd, 0x21, 0x51, 0x9e, 0xbd, 0x7a, 0x02, 0x64, 0x4a};
+
+        static std::vector<uint8_t> INITIAL_SALT_D23_D28 = {0xc3, 0xee, 0xf7, 0x12, 0xc7, 0x2e, 0xbb, 0x5a, 0x11, 0xa7,
+                                                            0xd2, 0x43, 0x2b, 0xb4, 0x63, 0x65, 0xbe, 0xf9, 0xf5, 0x02};
+
+        static std::vector<uint8_t> INITIAL_SALT_D29_D32 = {0xaf, 0xbf, 0xec, 0x28, 0x99, 0x93, 0xd2, 0x4c, 0x9e, 0x97,
+                                                            0x86, 0xf1, 0x9c, 0x61, 0x11, 0xe0, 0x43, 0x90, 0xa8, 0x99};
+        if ( version == 0xff000016 )
+            return INITIAL_SALT_D22;
+
+        if ( version >= 0xff000017 && version <= 0xff00001c )
+            return INITIAL_SALT_D23_D28;
+
+        if ( version >= 0xff00001d && version <= 0xff000020 )
+            return INITIAL_SALT_D29_D32;
+
+        if ( version == 0xfaceb001 )
+            return INITIAL_SALT_D22;
+
+        if ( version >= 0xfaceb002 && version <= 0xfaceb013 )
+            return INITIAL_SALT_D23_D28;
+
+        return INITIAL_SALT_V1;
+    }
+
     virtual HkdfCtx& GetHkdfCtxs() override { return hkdf_ctxs; }
 
     // Pre-initialize SSL context for reuse with HKDF info set to version specific values.
     static void Initialize() {
-        if ( hkdf_ctxs.initialized )
+        std::vector<uint8_t> CLIENT_INITIAL_INFO = {0x00, 0x20, 0x0f, 0x74, 0x6c, 0x73, 0x31, 0x33, 0x20, 0x63,
-            return;
+                                                    0x6c, 0x69, 0x65, 0x6e, 0x74, 0x20, 0x69, 0x6e, 0x00};
+
+        std::vector<uint8_t> SERVER_INITIAL_INFO = {0x00, 0x20, 0x0f, 0x74, 0x6c, 0x73, 0x31, 0x33, 0x20, 0x73,
+                                                    0x65, 0x72, 0x76, 0x65, 0x72, 0x20, 0x69, 0x6e, 0x00};
+
+        std::vector<uint8_t> KEY_INFO = {0x00, 0x10, 0x0e, 0x74, 0x6c, 0x73, 0x31, 0x33, 0x20,
+                                         0x71, 0x75, 0x69, 0x63, 0x20, 0x6b, 0x65, 0x79, 0x00};
+
+        std::vector<uint8_t> IV_INFO = {0x00, 0x0c, 0x0d, 0x74, 0x6c, 0x73, 0x31, 0x33, 0x20,
+                                        0x71, 0x75, 0x69, 0x63, 0x20, 0x69, 0x76, 0x00};
+
+        std::vector<uint8_t> HP_INFO = {0x00, 0x10, 0x0d, 0x74, 0x6c, 0x73, 0x31, 0x33, 0x20,
+                                        0x71, 0x75, 0x69, 0x63, 0x20, 0x68, 0x70, 0x00};
 
         std::vector<HkdfCtxParam> hkdf_ctx_params = {
             {&hkdf_ctxs.client_in_ctx, CLIENT_INITIAL_INFO},
@@ -333,7 +370,6 @@ public:
         QuicPacketProtection::Initialize(hkdf_ctx_params);
 
         instance = std::make_unique<QuicPacketProtectionV1>();
-        hkdf_ctxs.initialized = true;
     }
 
     static HkdfCtx hkdf_ctxs;
@@ -347,32 +383,35 @@ std::unique_ptr<QuicPacketProtectionV1> QuicPacketProtectionV1::instance = nullp
 // QUIC v2
 //
 // https://datatracker.ietf.org/doc/rfc9369/
-std::vector<uint8_t> INITIAL_SALT_V2 = {0x0d, 0xed, 0xe3, 0xde, 0xf7, 0x00, 0xa6, 0xdb, 0x81, 0x93,
-                                        0x81, 0xbe, 0x6e, 0x26, 0x9d, 0xcb, 0xf9, 0xbd, 0x2e, 0xd9};
-
-std::vector<uint8_t> CLIENT_INITIAL_INFO_V2 = {0x00, 0x20, 0x0f, 0x74, 0x6c, 0x73, 0x31, 0x33, 0x20, 0x63,
-                                               0x6c, 0x69, 0x65, 0x6e, 0x74, 0x20, 0x69, 0x6e, 0x00};
-
-std::vector<uint8_t> SERVER_INITIAL_INFO_V2 = {0x00, 0x20, 0x0f, 0x74, 0x6c, 0x73, 0x31, 0x33, 0x20, 0x73,
-                                               0x65, 0x72, 0x76, 0x65, 0x72, 0x20, 0x69, 0x6e, 0x00};
-
-std::vector<uint8_t> KEY_INFO_V2 = {0x00, 0x10, 0x10, 0x74, 0x6c, 0x73, 0x31, 0x33, 0x20, 0x71,
-                                    0x75, 0x69, 0x63, 0x76, 0x32, 0x20, 0x6b, 0x65, 0x79, 0x00};
-
-std::vector<uint8_t> IV_INFO_V2 = {0x00, 0x0c, 0x0f, 0x74, 0x6c, 0x73, 0x31, 0x33, 0x20, 0x71,
-                                   0x75, 0x69, 0x63, 0x76, 0x32, 0x20, 0x69, 0x76, 0x00};
-
-std::vector<uint8_t> HP_INFO_V2 = {0x00, 0x10, 0x0f, 0x74, 0x6c, 0x73, 0x31, 0x33, 0x20, 0x71,
-                                   0x75, 0x69, 0x63, 0x76, 0x32, 0x20, 0x68, 0x70, 0x00};
-
 class QuicPacketProtectionV2 : public QuicPacketProtection {
 public:
-    virtual std::vector<uint8_t>& GetInitialSalt() override { return INITIAL_SALT_V2; }
+    virtual bool Supports(uint32_t version) const override { return version == 0x6b3343cf; }
+
+    virtual const std::vector<uint8_t>& GetInitialSalt(uint32_t version) const override {
+        static std::vector<uint8_t> INITIAL_SALT_V2 = {0x0d, 0xed, 0xe3, 0xde, 0xf7, 0x00, 0xa6, 0xdb, 0x81, 0x93,
+                                                       0x81, 0xbe, 0x6e, 0x26, 0x9d, 0xcb, 0xf9, 0xbd, 0x2e, 0xd9};
+
+        return INITIAL_SALT_V2;
+    }
+
     virtual HkdfCtx& GetHkdfCtxs() override { return hkdf_ctxs; }
 
     static void Initialize() {
-        if ( hkdf_ctxs.initialized )
+        std::vector<uint8_t> CLIENT_INITIAL_INFO_V2 = {0x00, 0x20, 0x0f, 0x74, 0x6c, 0x73, 0x31, 0x33, 0x20, 0x63,
-            return;
+                                                       0x6c, 0x69, 0x65, 0x6e, 0x74, 0x20, 0x69, 0x6e, 0x00};
+
+        std::vector<uint8_t> SERVER_INITIAL_INFO_V2 = {0x00, 0x20, 0x0f, 0x74, 0x6c, 0x73, 0x31, 0x33, 0x20, 0x73,
+                                                       0x65, 0x72, 0x76, 0x65, 0x72, 0x20, 0x69, 0x6e, 0x00};
+
+        std::vector<uint8_t> KEY_INFO_V2 = {0x00, 0x10, 0x10, 0x74, 0x6c, 0x73, 0x31, 0x33, 0x20, 0x71,
+                                            0x75, 0x69, 0x63, 0x76, 0x32, 0x20, 0x6b, 0x65, 0x79, 0x00};
+
+        std::vector<uint8_t> IV_INFO_V2 = {0x00, 0x0c, 0x0f, 0x74, 0x6c, 0x73, 0x31, 0x33, 0x20, 0x71,
+                                           0x75, 0x69, 0x63, 0x76, 0x32, 0x20, 0x69, 0x76, 0x00};
+
+        std::vector<uint8_t> HP_INFO_V2 = {0x00, 0x10, 0x0f, 0x74, 0x6c, 0x73, 0x31, 0x33, 0x20, 0x71,
+                                           0x75, 0x69, 0x63, 0x76, 0x32, 0x20, 0x68, 0x70, 0x00};
+
         std::vector<HkdfCtxParam> hkdf_ctx_params = {
             {&hkdf_ctxs.client_in_ctx, CLIENT_INITIAL_INFO_V2},
             {&hkdf_ctxs.server_in_ctx, SERVER_INITIAL_INFO_V2},
@@ -383,7 +422,6 @@ public:
 
         QuicPacketProtection::Initialize(hkdf_ctx_params);
         instance = std::make_unique<QuicPacketProtectionV2>();
-        hkdf_ctxs.initialized = true;
     }
 
     static HkdfCtx hkdf_ctxs;
@@ -404,6 +442,13 @@ hilti::rt::Bytes QUIC_decrypt_crypto_payload(const hilti::rt::integer::safe<uint
                                              const hilti::rt::integer::safe<uint64_t>& encrypted_offset,
                                              const hilti::rt::integer::safe<uint64_t>& payload_length,
                                              const hilti::rt::Bool& from_client) {
+    static bool initialized = false;
+    if ( ! initialized ) {
+        QuicPacketProtectionV1::Initialize();
+        QuicPacketProtectionV2::Initialize();
+        initialized = true;
+    }
+
     if ( payload_length < 20 )
         throw hilti::rt::RuntimeError(hilti::rt::fmt("payload too small %ld < 20", payload_length));
 
@@ -411,21 +456,20 @@ hilti::rt::Bytes QUIC_decrypt_crypto_payload(const hilti::rt::integer::safe<uint
         throw hilti::rt::RuntimeError(
             hilti::rt::fmt("packet too small %ld %ld", all_data.size(), encrypted_offset + payload_length));
 
+    uint32_t v = version;
     QuicPacketProtection* qpp = nullptr;
 
-    if ( version == 0x00000001 ) { // quicv1
+    if ( QuicPacketProtectionV1::instance->Supports(v) ) {
-        QuicPacketProtectionV1::Initialize();
         qpp = QuicPacketProtectionV1::instance.get();
     }
-    else if ( version == 0x6b3343cf ) { // quicv2
+    else if ( QuicPacketProtectionV2::instance->Supports(v) ) {
-        QuicPacketProtectionV2::Initialize();
         qpp = QuicPacketProtectionV2::instance.get();
     }
     else {
-        throw hilti::rt::RuntimeError(hilti::rt::fmt("unable to handle version %lx", version));
+        throw hilti::rt::RuntimeError(hilti::rt::fmt("unable to decrypt QUIC version 0x%lx", version));
     }
 
-    const auto& secret = qpp->GetSecret(from_client, connection_id);
+    const auto& secret = qpp->GetSecret(from_client, v, connection_id);
     std::vector<uint8_t> key = qpp->GetKey(secret);
     std::vector<uint8_t> iv = qpp->GetIv(secret);
     std::vector<uint8_t> hp = qpp->GetHp(secret);

testing/btest/Baseline/scripts.base.protocols.quic.vector-max-size-crash/quic.log

@@ -0,0 +1,11 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	quic
+#open XXXX-XX-XX-XX-XX-XX
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	version	client_initial_dcid	server_scid	server_name	client_protocol	history
+#types	time	string	addr	port	addr	port	string	string	string	string	string	string
+XXXXXXXXXX.XXXXXX	CHhAvVGS1DHFjwGM9	1.2.3.4	45492	7.7.7.7	443	unknown-10010000	00	-	-	-	U
+#close XXXX-XX-XX-XX-XX-XX

testing/btest/scripts/base/protocols/quic/vector-max-size-crash.zeek

@@ -6,6 +6,7 @@
 # @TEST-EXEC: zeek-cut -m ts uid cause analyzer_kind analyzer_name failure_reason < analyzer.log > analyzer.log.cut
 # @TEST-EXEC: btest-diff conn.log.cut
 # @TEST-EXEC: btest-diff out
+# @TEST-EXEC: btest-diff quic.log
 
 # @TEST-EXEC: TEST_DIFF_CANONIFIER='sed -r "s/\((.+)\.spicy:[0-9]+:[0-9]+\)/(\1.spicy:<line>:<column>)/g" | $SCRIPTS/diff-remove-abspath' btest-diff analyzer.log.cut