Merge remote-tracking branch 'origin/topic/johanna/tls13'

BIT-1727 #merged

* origin/topic/johanna/tls13:
  Better way to deal with overloaded Assign constructors.
  A few tabbing fixes in TLS 1.3 support
  TLS 1.3 support.

src/analyzer/protocol/ssl/CMakeLists.txt

@@ -7,6 +7,7 @@ bro_plugin_begin(Bro SSL)
 bro_plugin_cc(SSL.cc DTLS.cc Plugin.cc)
 bro_plugin_bif(types.bif)
 bro_plugin_bif(events.bif)
+bro_plugin_bif(functions.bif)
 bro_plugin_pac(tls-handshake.pac tls-handshake-protocol.pac tls-handshake-analyzer.pac ssl-defs.pac
 	proc-client-hello.pac
 	proc-server-hello.pac

src/analyzer/protocol/ssl/SSL.cc

@@ -41,6 +41,13 @@ void SSL_Analyzer::EndpointEOF(bool is_orig)
 	handshake_interp->FlowEOF(is_orig);
 	}
 
+void SSL_Analyzer::StartEncryption()
+	{
+	interp->startEncryption(true);
+	interp->startEncryption(false);
+	interp->setEstablished();
+	}
+
 void SSL_Analyzer::DeliverStream(int len, const u_char* data, bool orig)
 	{
 	tcp::TCP_ApplicationAnalyzer::DeliverStream(len, data, orig);

src/analyzer/protocol/ssl/SSL.h

@@ -23,6 +23,9 @@ public:
 
 	void SendHandshake(const u_char* begin, const u_char* end, bool orig);
 
+	// Tell the analyzer that encryption has started.
+	void StartEncryption();
+
 	// Overriden from tcp::TCP_ApplicationAnalyzer.
 	virtual void EndpointEOF(bool is_orig);
 

src/analyzer/protocol/ssl/events.bif

@@ -44,9 +44,11 @@ event ssl_client_hello%(c: connection, version: count, possible_ts: time, client
 ##          :bro:id:`SSL::version_strings` table maps them to descriptive names.
 ##
 ## possible_ts: The current time as sent by the server. Note that SSL/TLS does
-##              not require clocks to be set correctly, so treat with care.
+##              not require clocks to be set correctly, so treat with care. This value
+##              is not sent in TLSv1.3.
 ##
-## session_id: The session ID as sent back by the server (if any).
+## session_id: The session ID as sent back by the server (if any). This value is not
+##             sent in TLSv1.3.
 ##
 ## server_random: The random value sent by the server. For version 2 connections,
 ##		  the connection-id is returned.
@@ -56,7 +58,8 @@ event ssl_client_hello%(c: connection, version: count, possible_ts: time, client
 ##         them to descriptive names.
 ##
 ## comp_method: The compression method chosen by the client. The values are
-##              standardized as part of the SSL/TLS protocol.
+##              standardized as part of the SSL/TLS protocol. This value is not
+##              sent in TLSv1.3.
 ##
 ## .. bro:see:: ssl_alert ssl_client_hello ssl_established ssl_extension
 ##    ssl_session_ticket_handshake x509_certificate ssl_server_curve
@@ -83,7 +86,7 @@ event ssl_server_hello%(c: connection, version: count, possible_ts: time, server
 ## .. bro:see:: ssl_alert ssl_client_hello ssl_established ssl_server_hello
 ##    ssl_session_ticket_handshake ssl_extension_ec_point_formats
 ##    ssl_extension_elliptic_curves ssl_extension_application_layer_protocol_negotiation
-##    ssl_extension_server_name ssl_extension_signature_algorithm
+##    ssl_extension_server_name ssl_extension_signature_algorithm ssl_extension_key_share
 event ssl_extension%(c: connection, is_orig: bool, code: count, val: string%);
 
 ## Generated for an SSL/TLS Elliptic Curves extension. This TLS extension is
@@ -100,6 +103,7 @@ event ssl_extension%(c: connection, is_orig: bool, code: count, val: string%);
 ##    ssl_session_ticket_handshake ssl_extension
 ##    ssl_extension_ec_point_formats ssl_extension_application_layer_protocol_negotiation
 ##    ssl_extension_server_name ssl_server_curve ssl_extension_signature_algorithm
+##    ssl_extension_key_share
 event ssl_extension_elliptic_curves%(c: connection, is_orig: bool, curves: index_vec%);
 
 ## Generated for an SSL/TLS Supported Point Formats extension. This TLS extension
@@ -117,6 +121,7 @@ event ssl_extension_elliptic_curves%(c: connection, is_orig: bool, curves: index
 ##    ssl_session_ticket_handshake ssl_extension
 ##    ssl_extension_elliptic_curves ssl_extension_application_layer_protocol_negotiation
 ##    ssl_extension_server_name ssl_server_curve ssl_extension_signature_algorithm
+##    ssl_extension_key_share
 event ssl_extension_ec_point_formats%(c: connection, is_orig: bool, point_formats: index_vec%);
 
 ## Generated for an Signature Algorithms extension. This TLS extension
@@ -133,9 +138,25 @@ event ssl_extension_ec_point_formats%(c: connection, is_orig: bool, point_format
 ## .. bro:see:: ssl_alert ssl_client_hello ssl_established ssl_server_hello
 ##    ssl_session_ticket_handshake ssl_extension
 ##    ssl_extension_elliptic_curves ssl_extension_application_layer_protocol_negotiation
-##    ssl_extension_server_name ssl_server_curve
+##    ssl_extension_server_name ssl_server_curve ssl_extension_key_share
 event ssl_extension_signature_algorithm%(c: connection, is_orig: bool, signature_algorithms: signature_and_hashalgorithm_vec%);
 
+## Generated for a Key Share extension. This TLS extension is defined in TLS1.3-draft16
+## and sent by the client and the server in the initial handshake. It gives the list of
+## named groups supported by the client and chosen by the server.
+##
+## c: The connection.
+##
+## is_orig: True if event is raised for originator side of the connection.
+##
+## curves: List of supported/chosen named groups.
+##
+## .. bro:see:: ssl_alert ssl_client_hello ssl_established ssl_server_hello
+##    ssl_session_ticket_handshake ssl_extension
+##    ssl_extension_elliptic_curves ssl_extension_application_layer_protocol_negotiation
+##    ssl_extension_server_name ssl_server_curve
+event ssl_extension_key_share%(c: connection, is_orig: bool, curves: index_vec%);
+
 ## Generated if a named curve is chosen by the server for an SSL/TLS connection.
 ## The curve is sent by the server in the ServerKeyExchange message as defined
 ## in :rfc:`4492`, in case an ECDH or ECDHE cipher suite is chosen.
@@ -147,7 +168,7 @@ event ssl_extension_signature_algorithm%(c: connection, is_orig: bool, signature
 ## .. bro:see:: ssl_alert ssl_client_hello ssl_established ssl_server_hello
 ##    ssl_session_ticket_handshake ssl_extension
 ##    ssl_extension_elliptic_curves ssl_extension_application_layer_protocol_negotiation
-##    ssl_extension_server_name
+##    ssl_extension_server_name ssl_extension_key_share
 event ssl_server_curve%(c: connection, curve: count%);
 
 ## Generated if a server uses a DH-anon or DHE cipher suite. This event contains
@@ -182,7 +203,7 @@ event ssl_dh_server_params%(c: connection, p: string, q: string, Ys: string%);
 ## .. bro:see:: ssl_alert ssl_client_hello ssl_established ssl_server_hello
 ##    ssl_session_ticket_handshake ssl_extension
 ##    ssl_extension_elliptic_curves ssl_extension_ec_point_formats
-##    ssl_extension_server_name
+##    ssl_extension_server_name ssl_extension_key_share
 event ssl_extension_application_layer_protocol_negotiation%(c: connection, is_orig: bool, protocols: string_vec%);
 
 ## Generated for an SSL/TLS Server Name extension. This SSL/TLS extension is
@@ -201,6 +222,7 @@ event ssl_extension_application_layer_protocol_negotiation%(c: connection, is_or
 ##    ssl_session_ticket_handshake ssl_extension
 ##    ssl_extension_elliptic_curves ssl_extension_ec_point_formats
 ##    ssl_extension_application_layer_protocol_negotiation
+##    ssl_extension_key_share
 event ssl_extension_server_name%(c: connection, is_orig: bool, names: string_vec%);
 
 ## Generated at the end of an SSL/TLS handshake. SSL/TLS sessions start with
@@ -284,6 +306,23 @@ event ssl_session_ticket_handshake%(c: connection, ticket_lifetime_hint: count,
 ##    ssl_alert ssl_encrypted_data
 event ssl_heartbeat%(c: connection, is_orig: bool, length: count, heartbeat_type: count, payload_length: count, payload: string%);
 
+## Generated for non-handshake SSL/TLS application_data messages that are sent before
+## full encryption starts. For TLS 1.2 and lower, this event should not be raised. For TLS 1.3,
+## it is used by Bro internally to determine if the connection has been completely setup.
+## This is necessary as TLS 1.3 does not have CCS anymore.
+##
+## c: The connection.
+##
+## is_orig: True if event is raised for originator side of the connection.
+##
+## content_type: message type as reported by TLS session layer.
+##
+## length: length of the entire heartbeat message.
+##
+## .. bro:see::  ssl_client_hello ssl_established ssl_extension ssl_server_hello
+##    ssl_alert ssl_heartbeat
+event ssl_application_data%(c: connection, is_orig: bool, length: count%);
+
 ## Generated for SSL/TLS messages that are sent after session encryption
 ## started.
 ##

src/analyzer/protocol/ssl/functions.bif

@@ -0,0 +1,16 @@
+
+%%{
+#include "analyzer/protocol/ssl/SSL.h"
+%%}
+
+## Sets if the SSL analyzer should consider the connection established (handshake
+## finished succesfully).
+##
+## c: The SSL connection.
+function set_ssl_established%(c: connection%): any
+	%{
+	analyzer::Analyzer* sa = c->FindAnalyzer("SSL");
+	if ( sa )
+		static_cast<analyzer::ssl::SSL_Analyzer*>(sa)->StartEncryption();
+	return 0;
+	%}

src/analyzer/protocol/ssl/ssl-defs.pac

@@ -65,12 +65,16 @@ function to_string_val(data : uint8[]) : StringVal
 
 function version_ok(vers : uint16) : bool
 	%{
+	if ( vers >> 8 == 0x7F ) // 1.3 draft
+		return true;
+
 	switch ( vers ) {
 	case SSLv20:
 	case SSLv30:
 	case TLSv10:
 	case TLSv11:
 	case TLSv12:
+	case TLSv13:
 	case DTLSv10:
 	case DTLSv12:
 		return true;
@@ -88,7 +92,7 @@ using std::string;
 #include "events.bif.h"
 %}
 
-# a maximum of 100k for one record seems safe 
+# a maximum of 100k for one record seems safe
 let MAX_DTLS_HANDSHAKE_RECORD: uint32 = 100000;
 
 enum ContentType {
@@ -112,6 +116,8 @@ enum SSLVersions {
 	TLSv10		= 0x0301,
 	TLSv11		= 0x0302,
 	TLSv12		= 0x0303,
+	TLSv13		= 0x0304,
+	TLSv13_draft	= 0x7F00, # the second byte actually defines the draft.
 
 	DTLSv10   = 0xFEFF,
 	# DTLSv11 does not exist.
@@ -139,7 +145,11 @@ enum SSLExtensions {
 	EXT_STATUS_REQUEST_V2 = 17,
 	EXT_SIGNED_CERTIFICATE_TIMESTAMP = 18,
 	EXT_SESSIONTICKET_TLS = 35,
-	EXT_EXTENDED_RANDOM = 40,
+	EXT_KEY_SHARE = 40,
+	EXT_PRE_SHARED_KEY = 41,
+	EXT_EARLY_DATA = 42,
+	EXT_SUPPORTED_VERSIONS = 43,
+	EXT_COOKIE = 44,
 	EXT_NEXT_PROTOCOL_NEGOTIATION = 13172,
 	EXT_ORIGIN_BOUND_CERTIFICATES = 13175,
 	EXT_ENCRYPTED_CLIENT_CERTIFICATES = 13180,

src/analyzer/protocol/ssl/ssl-dtls-analyzer.pac

@@ -23,6 +23,12 @@ refine connection SSL_Conn += {
 	%cleanup{
 	%}
 
+	function setEstablished() : bool
+		%{
+		established_ = true;
+		return true;
+		%}
+
 	function proc_alert(rec: SSLRecord, level : int, desc : int) : bool
 		%{
 		BifEvent::generate_ssl_alert(bro_analyzer(), bro_analyzer()->Conn(),
@@ -54,6 +60,14 @@ refine connection SSL_Conn += {
 		return true;
 		%}
 
+	function proc_application_record(rec : SSLRecord) : bool
+		%{
+		BifEvent::generate_ssl_application_data(bro_analyzer(),
+			bro_analyzer()->Conn(), ${rec.is_orig}, ${rec.length});
+
+		return true;
+		%}
+
 	function proc_heartbeat(rec : SSLRecord, type: uint8, payload_length: uint16, data: bytestring) : bool
 		%{
 		BifEvent::generate_ssl_heartbeat(bro_analyzer(),
@@ -101,6 +115,10 @@ refine typeattr CiphertextRecord += &let {
 	proc : bool = $context.connection.proc_ciphertext_record(rec);
 }
 
+refine typeattr ApplicationData += &let {
+	proc : bool = $context.connection.proc_application_record(rec);
+}
+
 refine typeattr ChangeCipherSpec += &let {
 	proc : bool = $context.connection.proc_ccs(rec);
 };

src/analyzer/protocol/ssl/ssl-dtls-protocol.pac

@@ -64,7 +64,7 @@ type Alert(rec: SSLRecord) = record {
 ######################################################################
 
 # Application data should always be encrypted, so we should not
-# reach this point.
+# reach this point, unless we are in TLS 1.3 ...
 type ApplicationData(rec: SSLRecord) = record {
 	data : bytestring &restofdata &transient;
 };
@@ -79,12 +79,11 @@ type Heartbeat(rec: SSLRecord) = record {
 	data : bytestring &restofdata;
 };
 
-
-
 ######################################################################
-# Fragmentation (6.2.1.)
+# Unknown Records (6.2.1.)
 ######################################################################
 
+# We should never reach this.
 type UnknownRecord(rec: SSLRecord) = record {
 	cont : bytestring &restofdata &transient;
 };

src/analyzer/protocol/ssl/tls-handshake-analyzer.pac

@@ -102,6 +102,29 @@ refine connection Handshake_Conn += {
 		return true;
 		%}
 
+	function proc_client_key_share(rec: HandshakeRecord, keyshare: KeyShareEntry[]) : bool
+		%{
+		VectorVal* nglist = new VectorVal(internal_type("index_vec")->AsVectorType());
+
+		if ( keyshare )
+			{
+			for ( unsigned int i = 0; i < keyshare->size(); ++i )
+				nglist->Assign(i, new Val((*keyshare)[i]->namedgroup(), TYPE_COUNT));
+			}
+
+		BifEvent::generate_ssl_extension_key_share(bro_analyzer(), bro_analyzer()->Conn(), ${rec.is_orig}, nglist);
+		return true;
+		%}
+
+	function proc_server_key_share(rec: HandshakeRecord, keyshare: KeyShareEntry) : bool
+		%{
+		VectorVal* nglist = new VectorVal(internal_type("index_vec")->AsVectorType());
+
+		nglist->Assign(0u, new Val(keyshare->namedgroup(), TYPE_COUNT));
+		BifEvent::generate_ssl_extension_key_share(bro_analyzer(), bro_analyzer()->Conn(), ${rec.is_orig}, nglist);
+		return true;
+		%}
+
 	function proc_signature_algorithm(rec: HandshakeRecord, supported_signature_algorithms: SignatureAndHashAlgorithm[]) : bool
 		%{
 		VectorVal* slist = new VectorVal(internal_type("signature_and_hashalgorithm_vec")->AsVectorType());
@@ -243,6 +266,13 @@ refine typeattr ServerHello += &let {
 			compression_method);
 };
 
+refine typeattr ServerHello13 += &let {
+	proc : bool = $context.connection.proc_server_hello(server_version,
+			0, random, 0, cipher_suite, 0,
+			0);
+};
+
+
 refine typeattr Certificate += &let {
 	proc : bool = $context.connection.proc_v3_certificate(rec.is_orig, certificates);
 };
@@ -267,6 +297,14 @@ refine typeattr EllipticCurves += &let {
 	proc : bool = $context.connection.proc_elliptic_curves(rec, elliptic_curve_list);
 };
 
+refine typeattr ServerHelloKeyShare += &let {
+	proc : bool = $context.connection.proc_server_key_share(rec, keyshare);
+};
+
+refine typeattr ClientHelloKeyShare += &let {
+	proc : bool = $context.connection.proc_client_key_share(rec, keyshares);
+};
+
 refine typeattr SignatureAlgorithm += &let {
 	proc : bool = $context.connection.proc_signature_algorithm(rec, supported_signature_algorithms);
 }

src/analyzer/protocol/ssl/tls-handshake-protocol.pac

@@ -33,7 +33,7 @@ type HandshakeRecord(is_orig: bool) = record {
 type Handshake(rec: HandshakeRecord) = case rec.msg_type of {
 	HELLO_REQUEST        -> hello_request        : HelloRequest(rec);
 	CLIENT_HELLO         -> client_hello         : ClientHello(rec);
-	SERVER_HELLO         -> server_hello         : ServerHello(rec);
+	SERVER_HELLO         -> server_hello         : ServerHelloChoice(rec);
 	HELLO_VERIFY_REQUEST -> hello_verify_request : HelloVerifyRequest(rec);
 	SESSION_TICKET       -> session_ticket       : SessionTicketHandshake(rec);
 	CERTIFICATE          -> certificate          : Certificate(rec);
@@ -97,8 +97,24 @@ type ClientHelloCookie(rec: HandshakeRecord) = record {
 # V3 Server Hello (7.4.1.3.)
 ######################################################################
 
-type ServerHello(rec: HandshakeRecord) = record {
-	server_version : uint16;
+# TLS 1.3 server hello is different from earlier versions. Trick around a
+# bit, route 1.3 requests to a different record than earlier.
+type ServerHelloChoice(rec: HandshakeRecord) = record {
+	server_version0 : uint8;
+	server_version1 : uint8;
+	hello: case parsed_version of {
+		TLSv13, TLSv13_draft -> hello13: ServerHello13(rec, server_version);
+		default -> helloclassic: ServerHello(rec, server_version);
+	} &requires(server_version) &requires(parsed_version);
+} &let {
+	server_version : uint16 = (server_version0 << 8) | server_version1;
+	parsed_version : uint16 = case server_version0 of {
+		0x7F -> 0x7F00; # map any draft version to 00
+		default -> server_version;
+	};
+};
+
+type ServerHello(rec: HandshakeRecord, server_version: uint16) = record {
 	gmt_unix_time : uint32;
 	random_bytes : bytestring &length = 28;
 	session_len : uint8;
@@ -114,6 +130,16 @@ type ServerHello(rec: HandshakeRecord) = record {
 		$context.connection.set_cipher(cipher_suite[0]);
 };
 
+type ServerHello13(rec: HandshakeRecord, server_version: uint16) = record {
+	random : bytestring &length = 32;
+	cipher_suite : uint16[1];
+	ext_len: uint16[] &until($element == 0 || $element != 0);
+	extensions : SSLExtension(rec)[] &until($input.length() == 0);
+} &let {
+	cipher_set : bool =
+		$context.connection.set_cipher(cipher_suite[0]);
+};
+
 ######################################################################
 # DTLS Hello Verify Request
 ######################################################################
@@ -459,6 +485,7 @@ type SSLExtension(rec: HandshakeRecord) = record {
 #		EXT_STATUS_REQUEST -> status_request: StatusRequest(rec)[] &until($element == 0 || $element != 0);
 		EXT_SERVER_NAME -> server_name: ServerNameExt(rec)[] &until($element == 0 || $element != 0);
 		EXT_SIGNATURE_ALGORITHMS -> signature_algorithm: SignatureAlgorithm(rec)[] &until($element == 0 || $element != 0);
+		EXT_KEY_SHARE -> key_share: KeyShare(rec)[] &until($element == 0 || $element != 0);
 		default -> data: bytestring &restofdata;
 	};
 } &length=data_len+4 &exportsourcedata;
@@ -502,6 +529,28 @@ type EcPointFormats(rec: HandshakeRecord) = record {
 	point_format_list: uint8[length];
 };
 
+type KeyShareEntry() = record {
+	namedgroup : uint16;
+	key_exchange_length : uint16;
+	key_exchange: bytestring &length=key_exchange_length &transient;
+};
+
+type ServerHelloKeyShare(rec: HandshakeRecord) = record {
+	keyshare : KeyShareEntry;
+};
+
+type ClientHelloKeyShare(rec: HandshakeRecord) = record {
+	length: uint16;
+	keyshares : KeyShareEntry[] &until($input.length() == 0);
+};
+
+type KeyShare(rec: HandshakeRecord) = case rec.msg_type of {
+	CLIENT_HELLO -> client_hello_keyshare : ClientHelloKeyShare(rec);
+	SERVER_HELLO -> server_hello_keyshare : ServerHelloKeyShare(rec);
+	# ... well, we don't parse hello retry requests yet, because I don't have an example of them on the wire.
+	default -> other : bytestring &restofdata &transient;
+};
+
 type SignatureAndHashAlgorithm() = record {
 	HashAlgorithm: uint8;
 	SignatureAlgorithm: uint8;