ldap: Implement extended request/response and StartTLS support

PCAP was produced with a local OpenLDAP server configured to support StartTLS.

This puts the Zeek calls into a separate ldap_zeek.spicy file/module
to separate it from LDAP.

NEWS

@@ -14,6 +14,10 @@ New Functionality
 
 * The LDAP analyzer now supports handling of non-sealed GSS-API WRAP tokens.
 
+* StartTLS support was added to the LDAP analyzer. The SSL analyzer is enabled
+  for connections where client and server negotiate to TLS through the extended
+  request/response mechanism.
+
 Changed Functionality
 ---------------------
 

scripts/base/protocols/ldap/consts.zeek

@@ -120,4 +120,11 @@ export {
 	    "searching", [ LDAP::SearchDerefAlias_DEREF_FINDING_BASE ] =
 	    "finding", [ LDAP::SearchDerefAlias_DEREF_ALWAYS ] = "always",  }
 	    &default="unknown";
+
+	const EXTENDED_REQUESTS = {
+	   # StartTLS, https://datatracker.ietf.org/doc/html/rfc4511#section-4.14.1
+	   [ "1.3.6.1.4.1.1466.20037" ] = "StartTLS",
+	   # whoami, https://datatracker.ietf.org/doc/html/rfc4532#section-2
+	   [ "1.3.6.1.4.1.4203.1.11.3" ] = "whoami",
+	} &default="unknown" &redef;
 }

scripts/base/protocols/ldap/main.zeek

@@ -258,6 +258,9 @@ event LDAP::message(c: connection,
       }
 
       m$object = object;
+
+      if ( opcode == LDAP::ProtocolOpcode_EXTENDED_REQUEST )
+        m$object += fmt(" (%s)", EXTENDED_REQUESTS[object]);
     }
 
     if ( argument != "" ) {

scripts/base/protocols/ldap/spicy-events.zeek

@@ -98,3 +98,44 @@ global LDAP::search_result_entry: event (
   message_id: int,
   object_name: string
 );
+
+## Event generated for each ExtendedRequest in LDAP messages.
+##
+## c: The connection.
+##
+## message_id: The messageID element.
+##
+## request_name: The name of the extended request.
+##
+## request_value: The value of the extended request (empty if missing).
+global LDAP::extended_request: event (
+  c: connection,
+  message_id: int,
+  request_name: string,
+  request_value: string
+);
+
+## Event generated for each ExtendedResponse in LDAP messages.
+##
+## c: The connection.
+##
+## message_id: The messageID element.
+##
+## result: The result code of the response.
+##
+## response_name: The name of the extended response (empty if missing).
+##
+## response_value: The value of the extended response (empty if missing).
+global LDAP::extended_response: event (
+  c: connection,
+  message_id: int,
+  result: LDAP::ResultCode,
+  response_name: string,
+  response_value: string
+);
+
+## Event generated when a plaintext LDAP connection switched to TLS.
+##
+## c: The connection.
+##
+global LDAP::starttls: event(c: connection);

src/analyzer/protocol/ldap/CMakeLists.txt

@@ -1,5 +1,5 @@
 spicy_add_analyzer(
     NAME LDAP
     PACKAGE_NAME spicy-ldap
-    SOURCES ldap.spicy ldap.evt asn1.spicy
+    SOURCES ldap.spicy ldap.evt asn1.spicy ldap_zeek.spicy
-    MODULES LDAP ASN1)
+    MODULES LDAP ASN1 LDAP_Zeek)

src/analyzer/protocol/ldap/ldap.evt

@@ -41,3 +41,18 @@ on LDAP::SearchRequest -> event LDAP::search_request($conn,
 on LDAP::SearchResultEntry -> event LDAP::search_result_entry($conn,
                                                               message.messageID,
                                                               self.objectName);
+
+on LDAP::ExtendedRequest -> event LDAP::extended_request($conn,
+                                                         message.messageID,
+                                                         self.requestName,
+                                                         self.requestValue);
+
+on LDAP::ExtendedResponse -> event LDAP::extended_response($conn,
+                                                           message.messageID,
+                                                           message.result_.code,
+                                                           self.responseName,
+                                                           self.responseValue);
+
+# Once switched into MessageMode::TLS, we won't parse messages anymore,
+# so this is raised just once.
+on LDAP::Message if (ctx.messageMode == LDAP::MessageMode::TLS) -> event LDAP::starttls($conn);

src/analyzer/protocol/ldap/ldap.spicy

@@ -130,29 +130,38 @@ public type Result = unit {
 const GSSAPI_MECH_MS_KRB5 = "1.2.840.48018.1.2.2";
 
 # Supported SASL stripping modes.
-type SaslStripping = enum {
+type MessageMode = enum {
-  MS_KRB5 = 1,  # Payload starts with a 4 byte length followed by a wrap token that may or may not be sealed.
+  MS_KRB5 = 1, # Payload starts with a 4 byte length followed by a wrap token that may or may not be sealed.
+  TLS = 2,     # Client/server used StartTLS, forward to SSL analyzer.
 };
 
 type Ctx = struct {
-  saslStripping: SaslStripping;  # Which mode of SASL stripping to use.
+  messageMode: MessageMode;   # Message dispatching mode
+  startTlsRequested: bool;    # Did the client use the StartTLS extended request?
 };
 
 #-----------------------------------------------------------------------------
 public type Messages = unit {
   %context = Ctx;
-  : SASLStrip(self.context())[];
+  : MessageDispatch(self.context())[];
 };
 
 #-----------------------------------------------------------------------------
-public type SASLStrip = unit(ctx: Ctx&) {
+public type MessageDispatch = unit(ctx: Ctx&) {
-  switch( ctx.saslStripping ) {
+  switch( ctx.messageMode ) {
-    SaslStripping::Undef -> : Message(ctx);
+    MessageMode::Undef -> : Message(ctx);
-    SaslStripping::MS_KRB5 -> : SaslMsKrb5Stripper(ctx);
+    MessageMode::MS_KRB5 -> : SaslMsKrb5Stripper(ctx);
+    MessageMode::TLS -> : TlsForward;
   };
 };
 
 
+#-----------------------------------------------------------------------------
+type TlsForward = unit {
+  # Just consume everything. This is hooked in ldap_zeek.spicy
+  chunk: bytes &chunked &eod;
+};
+
 type KrbWrapToken = unit {
   # https://datatracker.ietf.org/doc/html/rfc4121#section-4.2.6.2
 
@@ -223,6 +232,7 @@ public type Message = unit(ctx: Ctx&) {
   var arg: string = "";
   var seqHeaderLen: uint64;
   var msgLen: uint64;
+  var opLen: uint64;
 
   seqHeader: ASN1::ASN1Header &requires=($$.tag.class == ASN1::ASN1Class::Universal && $$.tag.type_ == ASN1::ASN1Type::Sequence) {
     self.msgLen = $$.len.len;
@@ -241,6 +251,7 @@ public type Message = unit(ctx: Ctx&) {
 
   protocolOp: ASN1::ASN1Header &requires=($$.tag.class == ASN1::ASN1Class::Application) {
     self.opcode = cast<ProtocolOpcode>(cast<uint8>($$.tag.type_));
+    self.opLen = $$.len.len;
   }
 
   switch ( self.opcode ) {
@@ -263,12 +274,12 @@ public type Message = unit(ctx: Ctx&) {
     # just commenting this out, it will stop processing LDAP Messages in this connection
     ProtocolOpcode::ADD_REQUEST             -> ADD_REQUEST:             NotImplemented(self);
     ProtocolOpcode::COMPARE_REQUEST         -> COMPARE_REQUEST:         NotImplemented(self);
-    ProtocolOpcode::EXTENDED_REQUEST        -> EXTENDED_REQUEST:        NotImplemented(self);
+    ProtocolOpcode::EXTENDED_REQUEST        -> EXTENDED_REQUEST:        ExtendedRequest(self, ctx);
-    ProtocolOpcode::EXTENDED_RESPONSE       -> EXTENDED_RESPONSE:       NotImplemented(self);
+    ProtocolOpcode::EXTENDED_RESPONSE       -> EXTENDED_RESPONSE:       ExtendedResponse(self, ctx);
     ProtocolOpcode::INTERMEDIATE_RESPONSE   -> INTERMEDIATE_RESPONSE:   NotImplemented(self);
     ProtocolOpcode::MOD_DN_REQUEST          -> MOD_DN_REQUEST:          NotImplemented(self);
     ProtocolOpcode::SEARCH_RESULT_REFERENCE -> SEARCH_RESULT_REFERENCE: NotImplemented(self);
-  } &size=self.protocolOp.len.len;
+  } &size=self.opLen;
 
   # Ensure some invariants hold after parsing the command.
   : void &requires=(self.offset() >= self.seqHeaderLen);
@@ -427,7 +438,7 @@ type BindResponse = unit(inout message: Message, ctx: Ctx&) {
         local token = self.serverSaslCreds[0].negTokenResp;
         if ( token.accepted && token?.supportedMechOid ) {
           if ( token.supportedMechOid == GSSAPI_MECH_MS_KRB5 ) {
-            ctx.saslStripping = SaslStripping::MS_KRB5;
+            ctx.messageMode = MessageMode::MS_KRB5;
           }
         }
       }
@@ -980,16 +991,61 @@ type AbandonRequest = unit(inout message: Message) {
 #-----------------------------------------------------------------------------
 # Extended Operation
 # https://tools.ietf.org/html/rfc4511#section-4.12
+type ExtendedRequest = unit(inout message: Message, ctx: Ctx&) {
+  var requestValue: bytes;
+  header: ASN1::ASN1Header &requires=($$.tag.class == ASN1::ASN1Class::ContextSpecific);
+  requestName: bytes &size=self.header.len.len &convert=$$.decode(spicy::Charset::ASCII) {
+    message.obj = $$;
+  }
 
-# TODO: implement ExtendedRequest
+  # If there's more byte to parse, it's the requestValue.
-# type ExtendedRequest = unit(inout message: Message) {
+  : ASN1::ASN1Message(False)
-#
+      &requires=($$.head.tag.class == ASN1::ASN1Class::ContextSpecific)
-# };
+      if ( message.opLen > self.offset() ) {
 
-# TODO: implement ExtendedResponse
+    self.requestValue = $$.application_data;
-# type ExtendedResponse = unit(inout message: Message) {
+  }
-#
+
-# };
+  on %done {
+    # Did the client request StartTLS?
+    #
+    # https://datatracker.ietf.org/doc/html/rfc4511#section-4.14.1
+    if ( self.requestName == "1.3.6.1.4.1.1466.20037" )
+      ctx.startTlsRequested = True;
+  }
+};
+
+#-----------------------------------------------------------------------------
+type ExtendedResponseEntry = unit(inout r: ExtendedResponse) {
+  : ASN1::ASN1Message(False) &requires=($$.head.tag.class == ASN1::ASN1Class::ContextSpecific) {
+    if ( $$.head.tag.type_ == ASN1::ASN1Type(10) )
+      r.responseName = $$.application_data;
+    else if ( $$.head.tag.type_ == ASN1::ASN1Type(11) )
+      r.responseValue = $$.application_data;
+    else
+      throw "Unhandled extended response tag %s" % $$.head.tag;
+  }
+};
+
+#-----------------------------------------------------------------------------
+type ExtendedResponse = unit(inout message: Message, ctx: Ctx&) {
+  var responseName: bytes;
+  var responseValue: bytes;
+  : Result {
+    message.result_ = $$;
+  }
+
+  # Try to parse two ASN1 entries if there are bytes left in the unit.
+  # Both are optional and identified by context specific tagging.
+  : ExtendedResponseEntry(self) if ( message.opLen > self.offset() );
+  : ExtendedResponseEntry(self) if ( message.opLen > self.offset() );
+
+  on %done {
+    # Client had requested StartTLS and it was successful? Switch to SSL.
+    if ( ctx.startTlsRequested && message.result_.code == ResultCode::SUCCESS )
+      ctx.messageMode = MessageMode::TLS;
+  }
+};
 
 #-----------------------------------------------------------------------------
 # IntermediateResponse Message

src/analyzer/protocol/ldap/ldap_zeek.spicy

@@ -0,0 +1,12 @@
+module LDAP_Zeek;
+
+import LDAP;
+import zeek;
+
+on LDAP::TlsForward::%init {
+  zeek::protocol_begin("SSL");
+}
+
+on LDAP::TlsForward::chunk {
+  zeek::protocol_data_in(zeek::is_orig(), self.chunk);
+}

testing/btest/Baseline/scripts.base.protocols.ldap.starttls/conn.log

@@ -0,0 +1,11 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	conn
+#open XXXX-XX-XX-XX-XX-XX
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	service	duration	orig_bytes	resp_bytes	conn_state	missed_bytes	history	orig_pkts	orig_ip_bytes	resp_pkts	resp_ip_bytes	tunnel_parents
+#types	time	string	addr	port	addr	port	enum	string	interval	count	count	string	count	string	count	count	count	count	set[string]
+XXXXXXXXXX.XXXXXX	CHhAvVGS1DHFjwGM9	127.0.0.1	45936	127.0.1.1	389	tcp	ldap_tcp,ssl	0.016922	683	3002	RSTO	0	ShADadFR	14	1407	14	3738	-
+#close XXXX-XX-XX-XX-XX-XX

testing/btest/Baseline/scripts.base.protocols.ldap.starttls/ldap.log

@@ -0,0 +1,11 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	ldap
+#open XXXX-XX-XX-XX-XX-XX
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	message_id	version	opcode	result	diagnostic_message	object	argument
+#types	time	string	addr	port	addr	port	int	int	string	string	string	string	string
+XXXXXXXXXX.XXXXXX	CHhAvVGS1DHFjwGM9	127.0.0.1	45936	127.0.1.1	389	1	-	extended	success	-	1.3.6.1.4.1.1466.20037 (StartTLS)	-
+#close XXXX-XX-XX-XX-XX-XX

testing/btest/Baseline/scripts.base.protocols.ldap.starttls/out

@@ -0,0 +1,4 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+CHhAvVGS1DHFjwGM9, extended_request, 1.3.6.1.4.1.1466.20037 (StartTLS), 
+CHhAvVGS1DHFjwGM9, extended_response, LDAP::ResultCode_SUCCESS, , 
+CHhAvVGS1DHFjwGM9, LDAP::starttls

testing/btest/Baseline/scripts.base.protocols.ldap.starttls/ssl.log

@@ -0,0 +1,11 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	ssl
+#open XXXX-XX-XX-XX-XX-XX
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	version	cipher	curve	server_name	resumed	last_alert	next_protocol	established	ssl_history	cert_chain_fps	client_cert_chain_fps	sni_matches_cert
+#types	time	string	addr	port	addr	port	string	string	string	string	bool	string	string	bool	string	vector[string]	vector[string]	bool
+XXXXXXXXXX.XXXXXX	CHhAvVGS1DHFjwGM9	127.0.0.1	45936	127.0.1.1	389	TLSv13	TLS_AES_256_GCM_SHA384	secp256r1	ubuntu-01.example.com	F	-	-	T	CsiI	-	-	-
+#close XXXX-XX-XX-XX-XX-XX

testing/btest/Baseline/scripts.base.protocols.ldap.who-am-i/conn.log

@@ -0,0 +1,11 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	conn
+#open XXXX-XX-XX-XX-XX-XX
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	service	duration	orig_bytes	resp_bytes	conn_state	missed_bytes	history	orig_pkts	orig_ip_bytes	resp_pkts	resp_ip_bytes	tunnel_parents
+#types	time	string	addr	port	addr	port	enum	string	interval	count	count	string	count	string	count	count	count	count	set[string]
+XXXXXXXXXX.XXXXXX	CHhAvVGS1DHFjwGM9	127.0.0.1	48122	127.0.1.1	389	tcp	ldap_tcp	0.001192	83	59	SF	0	ShADadFf	8	507	5	327	-
+#close XXXX-XX-XX-XX-XX-XX

testing/btest/Baseline/scripts.base.protocols.ldap.who-am-i/ldap.log

@@ -0,0 +1,13 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	ldap
+#open XXXX-XX-XX-XX-XX-XX
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	message_id	version	opcode	result	diagnostic_message	object	argument
+#types	time	string	addr	port	addr	port	int	int	string	string	string	string	string
+XXXXXXXXXX.XXXXXX	CHhAvVGS1DHFjwGM9	127.0.0.1	48122	127.0.1.1	389	1	3	bind simple	success	-	cn=admin,dc=example,dc=com	REDACTED
+XXXXXXXXXX.XXXXXX	CHhAvVGS1DHFjwGM9	127.0.0.1	48122	127.0.1.1	389	2	-	extended	success	-	1.3.6.1.4.1.4203.1.11.3 (whoami)	-
+XXXXXXXXXX.XXXXXX	CHhAvVGS1DHFjwGM9	127.0.0.1	48122	127.0.1.1	389	3	-	unbind	-	-	-	-
+#close XXXX-XX-XX-XX-XX-XX

testing/btest/Baseline/scripts.base.protocols.ldap.who-am-i/out

@@ -0,0 +1,3 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+CHhAvVGS1DHFjwGM9, extended_request, 1.3.6.1.4.1.4203.1.11.3 (whoami), 
+CHhAvVGS1DHFjwGM9, extended_response, LDAP::ResultCode_SUCCESS, , dn:cn=admin,dc=example,dc=com

testing/btest/scripts/base/protocols/ldap/starttls.zeek

@@ -0,0 +1,25 @@
+# Copyright (c) 2024 by the Zeek Project. See LICENSE for details.
+
+# @TEST-REQUIRES: have-spicy
+# @TEST-EXEC: zeek -C -r ${TRACES}/ldap/ldap-starttls.pcap %INPUT >out
+# @TEST-EXEC: cat conn.log | zeek-cut -Cn local_orig local_resp > conn.log2 && mv conn.log2 conn.log
+# @TEST-EXEC: btest-diff out
+# @TEST-EXEC: btest-diff conn.log
+# @TEST-EXEC: btest-diff ldap.log
+# @TEST-EXEC: btest-diff ssl.log
+# @TEST-EXEC: ! test -f dpd.log
+# @TEST-EXEC: ! test -f analyzer.log
+#
+# @TEST-DOC: LDAP supports StartTLS through extendedRequest 1.3.6.1.4.1.1466.20037
+
+event LDAP::extended_request(c: connection, message_id: int, request_name: string, request_value: string) {
+  print c$uid, "extended_request", fmt("%s (%s)", request_name, LDAP::EXTENDED_REQUESTS[request_name]), request_value;
+}
+
+event LDAP::extended_response(c: connection, message_id: int, result: LDAP::ResultCode, response_name: string, response_value: string) {
+  print c$uid, "extended_response", result, response_name, response_value;
+}
+
+event LDAP::starttls(c: connection) {
+  print c$uid, "LDAP::starttls";
+}

testing/btest/scripts/base/protocols/ldap/who-am-i.zeek

@@ -0,0 +1,20 @@
+# Copyright (c) 2024 by the Zeek Project. See LICENSE for details.
+
+# @TEST-REQUIRES: have-spicy
+# @TEST-EXEC: zeek -C -r ${TRACES}/ldap/ldap-who-am-i.pcap %INPUT >out
+# @TEST-EXEC: cat conn.log | zeek-cut -Cn local_orig local_resp > conn.log2 && mv conn.log2 conn.log
+# @TEST-EXEC: btest-diff out
+# @TEST-EXEC: btest-diff conn.log
+# @TEST-EXEC: btest-diff ldap.log
+# @TEST-EXEC: ! test -f dpd.log
+# @TEST-EXEC: ! test -f analyzer.log
+#
+# @TEST-DOC: Testing OpenLDAP's ldapwhoami utility with simple authentication.
+
+event LDAP::extended_request(c: connection, message_id: int, request_name: string, request_value: string) {
+  print c$uid, "extended_request", fmt("%s (%s)", request_name, LDAP::EXTENDED_REQUESTS[request_name]), request_value;
+}
+
+event LDAP::extended_response(c: connection, message_id: int, result: LDAP::ResultCode, response_name: string, response_value: string) {
+  print c$uid, "extended_response", result, response_name, response_value;
+}