Merge remote-tracking branch 'origin/master' into topic/johanna/spicy-tls

* origin/master: (89 commits)
  Update doc submodule [nomail] [skip ci]
  Bump cmake submodule [nomail]
  testing/btest: Default to HILTI_JIT_PARALLELISM=1
  Revert "CI: Use ccache and a single CPU when building spicy analyzers for btests"
  Update doc submodule [nomail] [skip ci]
  CI: Use ccache and a single CPU when building spicy analyzers for btests
  Extend btest for logging of disabled analyzers
  Update zeek-aux submodule [nomail]
  Add logging of disabled analyzers to analyzer.log
  Bump auxil/spicy to latest development snapshot
  Management framework: bump cluster testsuite to pull in telemetry tests
  Management framework: bump zeek-client
  Management framework: augment deployed configs with instance IP addresses
  Management framework: add auto-enumeration of metrics ports
  Management framework: propagate metrics port from agent
  Management framework: add metrics port in management & Supervisor node records
  Harden the telemetry manager against unset Telemetry::metrics_address
  Comment-only tweaks for telemetry-related settings.
  Fix for --display-cmake in configure Moved build directory creation further down in the script so that --display-cmake has a chance to happen before build tree setup.
  Update submodules [nomail]
  ...

src/analyzer/protocol/http/HTTP.cc

@@ -1020,6 +1020,36 @@ void HTTP_Analyzer::Undelivered(uint64_t seq, int len, bool is_orig) {
     }
 }
 
+void HTTP_Analyzer::FlipRoles() {
+    analyzer::tcp::TCP_ApplicationAnalyzer::FlipRoles();
+
+    // If FlipRoles() is invoked after we've upgraded to something,
+    // don't do anything. This shouldn't happen as flipping of TCP
+    // connections currently happens before any data is transferred,
+    // but better safe than sorry.
+    if ( upgraded || pia ) {
+        Weird("HTTP_late_flip_roles");
+        return;
+    }
+
+    // If we haven't upgraded but saw request or replies, just bail
+    // for the rest of this connection. Again, this should never happen
+    // right now, but raise a weird in case it starts to happen.
+    if ( num_requests > 0 || num_replies > 0 ) {
+        Weird("HTTP_late_flip_roles");
+        SetSkip(true);
+        return;
+    }
+
+    // IsOrig() of the support analyzer has been updated, but we still need
+    // to change the analyzer's local state and the partial skipping setting.
+    bool skip_partial_orig = content_line_orig->SkipPartial();
+    bool skip_partial_resp = content_line_resp->SkipPartial();
+    std::swap(content_line_orig, content_line_resp);
+    content_line_orig->SetSkipPartial(skip_partial_orig);
+    content_line_resp->SetSkipPartial(skip_partial_resp);
+}
+
 void HTTP_Analyzer::EndpointEOF(bool is_orig) {
     analyzer::tcp::TCP_ApplicationAnalyzer::EndpointEOF(is_orig);
 

src/analyzer/protocol/http/HTTP.h

@@ -167,6 +167,7 @@ public:
     void Done() override;
     void DeliverStream(int len, const u_char* data, bool orig) override;
     void Undelivered(uint64_t seq, int len, bool orig) override;
+    void FlipRoles() override;
 
     // Overridden from analyzer::tcp::TCP_ApplicationAnalyzer
     void EndpointEOF(bool is_orig) override;

src/analyzer/protocol/ldap/ldap.evt

@@ -14,9 +14,9 @@ import LDAP;
 on LDAP::Message -> event LDAP::message($conn,
                                         self.messageID,
                                         self.opcode,
-                                        self.result.code,
-                                        self.result.matchedDN,
-                                        self.result.diagnosticMessage,
+                                        self.result_.code,
+                                        self.result_.matchedDN,
+                                        self.result_.diagnosticMessage,
                                         self.obj,
                                         self.arg);
 

src/analyzer/protocol/ldap/ldap.spicy

@@ -223,7 +223,7 @@ public type Message = unit {
   var opcode: ProtocolOpcode = ProtocolOpcode::Undef;
   var applicationBytes: bytes;
   var unsetResultDefault: Result;
-  var result: Result& = self.unsetResultDefault;
+  var result_: Result& = self.unsetResultDefault;
   var obj: string = "";
   var arg: string = "";
   var success: bool = False;
@@ -328,7 +328,7 @@ type BindRequest = unit(inout message: Message) {
 
 type BindResponse = unit(inout message: Message) {
   : Result {
-    message.result = $$;
+    message.result_ = $$;
   }
 
   # TODO: if we want to parse SASL credentials returned
@@ -777,7 +777,7 @@ type SearchResultEntry = unit(inout message: Message) {
 
 type SearchResultDone = unit(inout message: Message) {
   : Result {
-    message.result = $$;
+    message.result_ = $$;
   }
 };
 
@@ -800,7 +800,7 @@ type ModifyRequest = unit(inout message: Message) {
 
 type ModifyResponse = unit(inout message: Message) {
   : Result {
-    message.result = $$;
+    message.result_ = $$;
   }
 };
 
@@ -816,7 +816,7 @@ type ModifyResponse = unit(inout message: Message) {
 
 type AddResponse = unit(inout message: Message) {
   : Result {
-    message.result = $$;
+    message.result_ = $$;
   }
 };
 
@@ -832,7 +832,7 @@ type DelRequest = unit(inout message: Message) {
 
 type DelResponse = unit(inout message: Message) {
   : Result {
-    message.result = $$;
+    message.result_ = $$;
   }
 };
 
@@ -847,7 +847,7 @@ type DelResponse = unit(inout message: Message) {
 
 type ModDNResponse = unit(inout message: Message) {
   : Result {
-    message.result = $$;
+    message.result_ = $$;
   }
 };
 
@@ -862,7 +862,7 @@ type ModDNResponse = unit(inout message: Message) {
 
 type CompareResponse = unit(inout message: Message) {
   : Result {
-    message.result = $$;
+    message.result_ = $$;
   }
 };
 

src/analyzer/protocol/quic/QUIC.evt

@@ -19,6 +19,6 @@ on QUIC::HandshakePacket -> event QUIC::handshake_packet($conn, $is_orig, self.h
 on QUIC::ZeroRTTPacket -> event QUIC::zero_rtt_packet($conn, $is_orig, self.header.version, self.header.dest_conn_id, self.header.src_conn_id);
 
 on QUIC::ConnectionClosePayload -> event QUIC::connection_close_frame($conn, $is_orig, self.header.version, self.header.dest_conn_id, self.header.src_conn_id,
-                                                                      self.error_code.result, self.reason_phrase);
+                                                                      self.error_code.result_, self.reason_phrase);
 
 on QUIC::UnhandledVersion -> event QUIC::unhandled_version($conn, $is_orig, self.header.version, self.header.dest_conn_id, self.header.src_conn_id);

src/analyzer/protocol/quic/QUIC.spicy

@@ -157,7 +157,7 @@ type FrameType = enum {
 
 type VariableLengthInteger = unit {
   var bytes_to_parse: uint64;
-  var result: uint64;
+  var result_: uint64;
 
   # Value of the two most significant bits indicates number of bytes
   # to parse for the variable length integer.
@@ -166,11 +166,11 @@ type VariableLengthInteger = unit {
   # Section 16 and Appendix A
   : uint8 {
     self.bytes_to_parse = 2**((0xC0 & $$) >> 6);
-    self.result = $$ & 0x3F;
+    self.result_ = $$ & 0x3F;
   }
 
   : uint8[self.bytes_to_parse - 1] if (self.bytes_to_parse > 1) foreach {
-    self.result = (self.result << 8) | $$;
+    self.result_ = (self.result_ << 8) | $$;
   }
 };
 
@@ -185,8 +185,8 @@ public type LongHeaderPacketV1 = unit(inout outer: LongHeaderPacket) {
       outer.encrypted_offset = outer.offset() +
                                self.initial_hdr.length.bytes_to_parse +
                                self.initial_hdr.token_length.bytes_to_parse +
-                               self.initial_hdr.token_length.result;
-      outer.payload_length = self.initial_hdr.length.result;
+                               self.initial_hdr.token_length.result_;
+      outer.payload_length = self.initial_hdr.length.result_;
     }
 
     LongPacketTypeV1::ZERO_RTT -> zerortt_hdr : ZeroRTTPacket(outer);
@@ -204,8 +204,8 @@ public type LongHeaderPacketV2 = unit(inout outer: LongHeaderPacket) {
       outer.encrypted_offset = outer.offset() +
                                 self.initial_hdr.length.bytes_to_parse +
                                 self.initial_hdr.token_length.bytes_to_parse +
-                                self.initial_hdr.token_length.result;
-      outer.payload_length = self.initial_hdr.length.result;
+                                self.initial_hdr.token_length.result_;
+      outer.payload_length = self.initial_hdr.length.result_;
     }
 
     LongPacketTypeV2::ZERO_RTT -> zerortt_hdr : ZeroRTTPacket(outer);
@@ -281,7 +281,7 @@ public type Frame = unit(header: LongHeaderPacket, from_client: bool, crypto_sin
     FrameType::ACK2 -> b: ACKPayload;
     FrameType::CRYPTO -> c: CRYPTOPayload(from_client) {
       # Have the sink re-assemble potentially out-of-order cryptodata
-      crypto_sink.write(self.c.cryptodata, self.c.offset.result);
+      crypto_sink.write(self.c.cryptodata, self.c.offset.result_);
     }
     FrameType::CONNECTION_CLOSE1 -> : ConnectionClosePayload(header);
     FrameType::PADDING -> : skip /\x00*/;  # eat the padding
@@ -295,7 +295,7 @@ public type Frame = unit(header: LongHeaderPacket, from_client: bool, crypto_sin
 type CRYPTOPayload = unit(from_client: bool) {
   offset: VariableLengthInteger;
   length: VariableLengthInteger;
-  cryptodata: bytes &size=self.length.result;
+  cryptodata: bytes &size=self.length.result_;
 };
 
 type ACKPayload = unit {
@@ -313,7 +313,7 @@ type ConnectionClosePayload = unit(header: LongHeaderPacket) {
     -> frame_type: VariableLengthInteger;
   };
   reason_phrase_length: VariableLengthInteger;
-  reason_phrase: bytes &size=self.reason_phrase_length.result;
+  reason_phrase: bytes &size=self.reason_phrase_length.result_;
 };
 
 
@@ -326,7 +326,7 @@ type ConnectionClosePayload = unit(header: LongHeaderPacket) {
 type InitialPacket = unit(header: LongHeaderPacket) {
   var header: LongHeaderPacket = header;
   token_length: VariableLengthInteger;
-  token: bytes &size=self.token_length.result;
+  token: bytes &size=self.token_length.result_;
 
   # 5.4.2. Header Protection Sample
   #
@@ -336,25 +336,25 @@ type InitialPacket = unit(header: LongHeaderPacket) {
   #
   # Enforce 4 bytes Packet Number length + 16 bytes sample
   # ciphertext available.
-  length: VariableLengthInteger &requires=self.length.result >= 20;
+  length: VariableLengthInteger &requires=self.length.result_ >= 20;
 
   # Consume the remainder of payload. This
   # includes the packet number field, but we
   # do not know its length yet. We need the
   # payload for sampling, however.
-  payload: skip bytes &size=self.length.result;
+  payload: skip bytes &size=self.length.result_;
 };
 
 type ZeroRTTPacket = unit(header: LongHeaderPacket) {
   var header: LongHeaderPacket = header;
   length: VariableLengthInteger;
-  payload: skip bytes &size=self.length.result;
+  payload: skip bytes &size=self.length.result_;
 };
 
 type HandshakePacket = unit(header: LongHeaderPacket) {
   var header: LongHeaderPacket = header;
   length: VariableLengthInteger;
-  payload: skip bytes &size=self.length.result;
+  payload: skip bytes &size=self.length.result_;
 };
 
 

src/analyzer/protocol/ssh/ssh-protocol.pac

@@ -37,7 +37,6 @@ type SSH_Key_Exchange(is_orig: bool) = record {
 	key_ex: case $context.connection.get_version() of {
 		SSH1 -> ssh1_msg : SSH1_Key_Exchange(is_orig, packet_length);
 		SSH2 -> ssh2_msg : SSH2_Key_Exchange(is_orig, packet_length);
-		default -> terminate : bytestring &restofdata &transient;
 	};
 } &length = $context.flow.get_kex_length($context.connection.get_version(), packet_length);
 

src/analyzer/protocol/tcp/ContentLine.h

@@ -22,6 +22,9 @@ public:
     // If enabled, flag (first) line with embedded NUL. Default off.
     void SetIsNULSensitive(bool enable) { flag_NULs = enable; }
 
+    // Returns true if skipping data above a hole.
+    bool SkipPartial() const { return skip_partial; }
+
     // If enabled, skip data above a hole. Default off.
     void SetSkipPartial(bool enable) { skip_partial = enable; }