Merge branch 'topic/johanna/tls13-extensions' into topic/johanna/ocsp-sct-validate

CHANGES

@@ -1,4 +1,14 @@
 
+2.5-115 | 2017-03-23 07:25:41 -0700
+
+  * Fix a test that was failing on some platforms. (Daniel Thayer)
+
+  * Remove test for cluster catch and release. This test keeps failing
+    intermittently because of timing issues that are surprisingly hard
+    to fix. (Johanna Amann)
+
+  * Fix some Coverity warnings. (Daniel Thayer)
+
 2.5-106 | 2017-03-13 11:19:03 -0700
 
   * print version string to stdout on --version, instead

VERSION

@@ -1 +1 @@
-2.5-106
+2.5-115

aux/broctl

@@ -1 +1 @@
-Subproject commit 96583ab378b1de32ac9804246e1b0e2845fc8b3e
+Subproject commit cf7ea4e1ad18920058f32e95bbea3bdd765b6094

scripts/base/protocols/ssl/consts.bro

@@ -163,6 +163,10 @@ export {
 		[42] = "early_data", # new for 1.3, state of draft-16
 		[43] = "supported_versions", # new for 1.3, state of draft-16
 		[44] = "cookie", # new for 1.3, state of draft-16
+		[45] = "psk_key_exchange_modes", # new for 1.3, state of draft-18
+		[46] = "TicketEarlyDataInfo", # new for 1.3, state of draft-16
+		[47] = "certificate_authorities", # new for 1.3, state of draft-18
+		[48] = "oid_filters", # new for 1.3, state of draft-18
 		[13172] = "next_protocol_negotiation",
 		[13175] = "origin_bound_certificates",
 		[13180] = "encrypted_client_certificates",

scripts/base/protocols/ssl/dpd.sig

@@ -1,7 +1,7 @@
 signature dpd_ssl_server {
   ip-proto == tcp
   # Server hello.
-  payload /^((\x15\x03[\x00\x01\x02\x03]....)?\x16\x03[\x00\x01\x02\x03]..\x02...\x03[\x00\x01\x02\x03]|...?\x04..\x00\x02).*/
+  payload /^((\x15\x03[\x00\x01\x02\x03]....)?\x16\x03[\x00\x01\x02\x03]..\x02...((\x03[\x00\x01\x02\x03\x04])|(\x7F[\x00-\x50]))|...?\x04..\x00\x02).*/
   requires-reverse-signature dpd_ssl_client
   enable "ssl"
   tcp-state responder
@@ -10,7 +10,7 @@ signature dpd_ssl_server {
 signature dpd_ssl_client {
   ip-proto == tcp
   # Client hello.
-  payload /^(\x16\x03[\x00\x01\x02\x03]..\x01...\x03[\x00\x01\x02\x03]|...?\x01[\x00\x03][\x00\x01\x02\x03]).*/
+  payload /^(\x16\x03[\x00\x01\x02\x03]..\x01...\x03[\x00\x01\x02\x03]|...?\x01[\x00\x03][\x00\x01\x02\x03\x04]).*/
   tcp-state originator
 }
 

src/analyzer/protocol/ssl/events.bif

@@ -87,6 +87,7 @@ event ssl_server_hello%(c: connection, version: count, possible_ts: time, server
 ##    ssl_session_ticket_handshake ssl_extension_ec_point_formats
 ##    ssl_extension_elliptic_curves ssl_extension_application_layer_protocol_negotiation
 ##    ssl_extension_server_name ssl_extension_signature_algorithm ssl_extension_key_share
+##    ssl_extension_psk_key_exchange_modes ssl_extension_supported_versions
 event ssl_extension%(c: connection, is_orig: bool, code: count, val: string%);
 
 ## Generated for an SSL/TLS Elliptic Curves extension. This TLS extension is
@@ -104,6 +105,7 @@ event ssl_extension%(c: connection, is_orig: bool, code: count, val: string%);
 ##    ssl_extension_ec_point_formats ssl_extension_application_layer_protocol_negotiation
 ##    ssl_extension_server_name ssl_server_curve ssl_extension_signature_algorithm
 ##    ssl_extension_key_share
+##    ssl_extension_psk_key_exchange_modes ssl_extension_supported_versions
 event ssl_extension_elliptic_curves%(c: connection, is_orig: bool, curves: index_vec%);
 
 ## Generated for an SSL/TLS Supported Point Formats extension. This TLS extension
@@ -122,6 +124,7 @@ event ssl_extension_elliptic_curves%(c: connection, is_orig: bool, curves: index
 ##    ssl_extension_elliptic_curves ssl_extension_application_layer_protocol_negotiation
 ##    ssl_extension_server_name ssl_server_curve ssl_extension_signature_algorithm
 ##    ssl_extension_key_share
+##    ssl_extension_psk_key_exchange_modes ssl_extension_supported_versions
 event ssl_extension_ec_point_formats%(c: connection, is_orig: bool, point_formats: index_vec%);
 
 ## Generated for an Signature Algorithms extension. This TLS extension
@@ -139,6 +142,7 @@ event ssl_extension_ec_point_formats%(c: connection, is_orig: bool, point_format
 ##    ssl_session_ticket_handshake ssl_extension
 ##    ssl_extension_elliptic_curves ssl_extension_application_layer_protocol_negotiation
 ##    ssl_extension_server_name ssl_server_curve ssl_extension_key_share
+##    ssl_extension_psk_key_exchange_modes ssl_extension_supported_versions
 event ssl_extension_signature_algorithm%(c: connection, is_orig: bool, signature_algorithms: signature_and_hashalgorithm_vec%);
 
 ## Generated for a Key Share extension. This TLS extension is defined in TLS1.3-draft16
@@ -155,6 +159,7 @@ event ssl_extension_signature_algorithm%(c: connection, is_orig: bool, signature
 ##    ssl_session_ticket_handshake ssl_extension
 ##    ssl_extension_elliptic_curves ssl_extension_application_layer_protocol_negotiation
 ##    ssl_extension_server_name ssl_server_curve
+##    ssl_extension_psk_key_exchange_modes ssl_extension_supported_versions
 event ssl_extension_key_share%(c: connection, is_orig: bool, curves: index_vec%);
 
 ## Generated if a named curve is chosen by the server for an SSL/TLS connection.
@@ -169,6 +174,7 @@ event ssl_extension_key_share%(c: connection, is_orig: bool, curves: index_vec%)
 ##    ssl_session_ticket_handshake ssl_extension
 ##    ssl_extension_elliptic_curves ssl_extension_application_layer_protocol_negotiation
 ##    ssl_extension_server_name ssl_extension_key_share
+##    ssl_extension_psk_key_exchange_modes ssl_extension_supported_versions
 event ssl_server_curve%(c: connection, curve: count%);
 
 ## Generated if a server uses a DH-anon or DHE cipher suite. This event contains
@@ -204,6 +210,7 @@ event ssl_dh_server_params%(c: connection, p: string, q: string, Ys: string%);
 ##    ssl_session_ticket_handshake ssl_extension
 ##    ssl_extension_elliptic_curves ssl_extension_ec_point_formats
 ##    ssl_extension_server_name ssl_extension_key_share
+##    ssl_extension_psk_key_exchange_modes ssl_extension_supported_versions
 event ssl_extension_application_layer_protocol_negotiation%(c: connection, is_orig: bool, protocols: string_vec%);
 
 ## Generated for an SSL/TLS Server Name extension. This SSL/TLS extension is
@@ -223,6 +230,7 @@ event ssl_extension_application_layer_protocol_negotiation%(c: connection, is_or
 ##    ssl_extension_elliptic_curves ssl_extension_ec_point_formats
 ##    ssl_extension_application_layer_protocol_negotiation
 ##    ssl_extension_key_share
+##    ssl_extension_psk_key_exchange_modes ssl_extension_supported_versions
 event ssl_extension_server_name%(c: connection, is_orig: bool, names: string_vec%);
 
 ## Generated for the signed_certificate_timestamp TLS extension as defined in
@@ -247,6 +255,42 @@ event ssl_extension_server_name%(c: connection, is_orig: bool, names: string_vec
 ## signature: signature part of the digitally_signed struct
 event ssl_extension_signed_certificate_timestamp%(c: connection, is_orig: bool, version: count, logid: string, timestamp: count, signature_and_hashalgorithm: SSL::SignatureAndHashAlgorithm, signature: string%);
 
+## Generated for an TLS Supported Versions extension. This TLS extension
+## is defined in the TLS 1.3 rfc and sent by the client in the initial handshake.
+## It contains the TLS versions that it supports. This informaion can be used by
+## the server to choose the best TLS version o use.
+##
+## c: The connection.
+##
+## is_orig: True if event is raised for originator side of the connection.
+##
+## versions: List of supported TLS versions.
+##
+## .. bro:see:: ssl_alert ssl_client_hello ssl_established ssl_server_hello
+##    ssl_session_ticket_handshake ssl_extension
+##    ssl_extension_elliptic_curves ssl_extension_ec_point_formats
+##    ssl_extension_application_layer_protocol_negotiation
+##    ssl_extension_key_share ssl_extension_server_name
+##    ssl_extension_psk_key_exchange_modes
+event ssl_extension_supported_versions%(c: connection, is_orig: bool, versions: index_vec%);
+
+## Generated for an TLS Pre-Shared Key Exchange Modes extension. This TLS extension is defined
+## in the TLS 1.3 rfc and sent by the client in the initial handshake. It contains the
+## list of Pre-Shared Key Exchange Modes that it supports.
+## c: The connection.
+##
+## is_orig: True if event is raised for originator side of the connection.
+##
+## versions: List of supported Pre-Shared Key Exchange Modes.
+##
+## .. bro:see:: ssl_alert ssl_client_hello ssl_established ssl_server_hello
+##    ssl_session_ticket_handshake ssl_extension
+##    ssl_extension_elliptic_curves ssl_extension_ec_point_formats
+##    ssl_extension_application_layer_protocol_negotiation
+##    ssl_extension_key_share ssl_extension_server_name
+##    ssl_extension_supported_versions
+event ssl_extension_psk_key_exchange_modes%(c: connection, is_orig: bool, modes: index_vec%);
+
 ## Generated at the end of an SSL/TLS handshake. SSL/TLS sessions start with
 ## an unencrypted handshake, and Bro extracts as much information out of that
 ## as it can. This event signals the time when an SSL/TLS has finished the

src/analyzer/protocol/ssl/ssl-defs.pac

@@ -150,6 +150,10 @@ enum SSLExtensions {
 	EXT_EARLY_DATA = 42,
 	EXT_SUPPORTED_VERSIONS = 43,
 	EXT_COOKIE = 44,
+	EXT_PSK_KEY_EXCHANGE_MODES = 45,
+	EXT_TICKET_EARLY_DATA_INFO = 46,
+	EXT_CERTIFICATE_AUTHORITIES = 47,
+	EXT_OID_FILTERS = 48,
 	EXT_NEXT_PROTOCOL_NEGOTIATION = 13172,
 	EXT_ORIGIN_BOUND_CERTIFICATES = 13175,
 	EXT_ENCRYPTED_CLIENT_CERTIFICATES = 13180,

src/analyzer/protocol/ssl/tls-handshake-analyzer.pac

@@ -189,6 +189,38 @@ refine connection Handshake_Conn += {
 		return true;
 		%}
 
+	function proc_supported_versions(rec: HandshakeRecord, versions_list: uint16[]) : bool
+		%{
+		VectorVal* versions = new VectorVal(internal_type("index_vec")->AsVectorType());
+
+		if ( versions_list )
+			{
+			for ( int i = 0; i < versions_list->size(); ++i )
+				versions->Assign(i, new Val((*versions_list)[i], TYPE_COUNT));
+			}
+
+		BifEvent::generate_ssl_extension_supported_versions(bro_analyzer(), bro_analyzer()->Conn(),
+			${rec.is_orig}, versions);
+
+		return true;
+		%}
+
+	function proc_psk_key_exchange_modes(rec: HandshakeRecord, mode_list: uint8[]) : bool
+		%{
+		VectorVal* modes = new VectorVal(internal_type("index_vec")->AsVectorType());
+
+		if ( mode_list )
+			{
+			for ( int i = 0; i < mode_list->size(); ++i )
+				modes->Assign(i, new Val((*mode_list)[i], TYPE_COUNT));
+			}
+
+		BifEvent::generate_ssl_extension_psk_key_exchange_modes(bro_analyzer(), bro_analyzer()->Conn(),
+			${rec.is_orig}, modes);
+
+		return true;
+		%}
+
 	function proc_v3_certificate(is_orig: bool, cl : X509Certificate[]) : bool
 		%{
 		vector<X509Certificate*>* certs = cl;
@@ -364,6 +396,14 @@ refine typeattr DhServerKeyExchange += &let {
 	proc : bool = $context.connection.proc_dh_server_key_exchange(rec, dh_p, dh_g, dh_Ys);
 };
 
+refine typeattr SupportedVersions += &let {
+	proc : bool = $context.connection.proc_supported_versions(rec, versions);
+};
+
+refine typeattr PSKKeyExchangeModes += &let {
+	proc : bool = $context.connection.proc_psk_key_exchange_modes(rec, modes);
+};
+
 refine typeattr Handshake += &let {
 	proc : bool = $context.connection.proc_handshake(rec.is_orig, rec.msg_type, rec.msg_length);
 };

src/analyzer/protocol/ssl/tls-handshake-protocol.pac

@@ -499,12 +499,24 @@ type SSLExtension(rec: HandshakeRecord) = record {
 		EXT_SIGNATURE_ALGORITHMS -> signature_algorithm: SignatureAlgorithm(rec)[] &until($element == 0 || $element != 0);
 		EXT_SIGNED_CERTIFICATE_TIMESTAMP -> certificate_timestamp: SignedCertificateTimestampList(rec)[] &until($element == 0 || $element != 0);
 		EXT_KEY_SHARE -> key_share: KeyShare(rec)[] &until($element == 0 || $element != 0);
+		EXT_SUPPORTED_VERSIONS -> supported_versions: SupportedVersions(rec)[] &until($element == 0 || $element != 0);
+		EXT_PSK_KEY_EXCHANGE_MODES -> psk_key_exchange_modes: PSKKeyExchangeModes(rec)[] &until($element == 0 || $element != 0);
 		default -> data: bytestring &restofdata;
 	};
 } &length=data_len+4 &exportsourcedata;
 
 %include tls-handshake-signed_certificate_timestamp.pac
 
+type SupportedVersions(rec: HandshakeRecord) = record {
+	length: uint8;
+	versions: uint16[] &until($input.length() == 0);
+} &length=length+1;
+
+type PSKKeyExchangeModes(rec: HandshakeRecord) = record {
+	length: uint8;
+	modes: uint8[] &until($input.length() == 0);
+} &length=length+1;
+
 type ServerNameHostName() = record {
 	length: uint16;
 	host_name: bytestring &length=length;

testing/btest/Baseline/scripts.base.protocols.ssl.dpd/.stdout

@@ -8,3 +8,6 @@ Start test run
 Client hello, 10.0.0.80, 68.233.76.12, 771
 Start test run
 Client hello, 192.168.6.217, 67.207.128.99, 771
+Start test run
+Client hello, 192.168.6.240, 139.162.123.134, 771
+Client hello, 192.168.6.240, 139.162.123.134, 771

testing/btest/Baseline/scripts.base.protocols.ssl.tls-extension-events/.stdout

@@ -20,3 +20,69 @@ uncompressed
 ansiX962_compressed_prime
 ansiX962_compressed_char2
 ALPN, 192.168.4.149, 74.125.239.152, [spdy/3.1]
+Point formats, 192.168.6.240, 139.162.123.134, T
+uncompressed
+ansiX962_compressed_prime
+ansiX962_compressed_char2
+Curves, 192.168.6.240, 139.162.123.134
+x25519
+secp256r1
+secp521r1
+secp384r1
+signature_algorithm, 192.168.6.240, 139.162.123.134
+sha256, ecdsa
+sha384, ecdsa
+sha512, ecdsa
+unknown-8, unknown-4
+unknown-8, unknown-5
+unknown-8, unknown-6
+sha256, rsa
+sha384, rsa
+sha512, rsa
+sha1, ecdsa
+sha1, rsa
+sha1, dsa
+sha256, dsa
+sha384, dsa
+sha512, dsa
+supported_versions(, 192.168.6.240, 139.162.123.134
+TLSv13-draft19
+TLSv12
+TLSv11
+TLSv10
+psk_key_exchange_modes, 192.168.6.240, 139.162.123.134
+1
+0
+Point formats, 192.168.6.240, 139.162.123.134, T
+uncompressed
+ansiX962_compressed_prime
+ansiX962_compressed_char2
+Curves, 192.168.6.240, 139.162.123.134
+x25519
+secp256r1
+secp521r1
+secp384r1
+signature_algorithm, 192.168.6.240, 139.162.123.134
+sha256, ecdsa
+sha384, ecdsa
+sha512, ecdsa
+unknown-8, unknown-4
+unknown-8, unknown-5
+unknown-8, unknown-6
+sha256, rsa
+sha384, rsa
+sha512, rsa
+sha1, ecdsa
+sha1, rsa
+sha1, dsa
+sha256, dsa
+sha384, dsa
+sha512, dsa
+supported_versions(, 192.168.6.240, 139.162.123.134
+TLSv13-draft19
+TLSv12
+TLSv11
+TLSv10
+psk_key_exchange_modes, 192.168.6.240, 139.162.123.134
+1
+0

testing/btest/scripts/base/frameworks/input/missing-file-initially.bro

@@ -8,7 +8,7 @@
 # @TEST-EXEC: sleep 2; mv does-not-exist.dat does-not-exist-again.dat; echo "Streaming still works" >> does-not-exist-again.dat
 # @TEST-EXEC: btest-bg-wait -k 3
 # @TEST-EXEC: btest-diff bro/.stdout
-# @TEST-EXEC: btest-diff bro/.stderr
+# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-sort btest-diff bro/.stderr
 
 @TEST-START-FILE does-exist.dat
 #separator \x09

testing/btest/scripts/base/protocols/ssl/dpd.test

@@ -2,6 +2,7 @@
 # @TEST-EXEC: bro -b -r $TRACES/tls/ssl.v3.trace %INPUT
 # @TEST-EXEC: bro -b -r $TRACES/tls/tls1.2.trace %INPUT
 # @TEST-EXEC: bro -b -r $TRACES/tls/tls-early-alert.trace %INPUT
+# @TEST-EXEC: bro -b -r $TRACES/tls/tls-13draft19-early-data.pcap %INPUT
 # @TEST-EXEC: btest-diff .stdout
 
 @load base/frameworks/dpd

testing/btest/scripts/base/protocols/ssl/tls-extension-events.test

@@ -1,4 +1,5 @@
 # @TEST-EXEC: bro -C -r $TRACES/tls/chrome-34-google.trace %INPUT
+# @TEST-EXEC: bro -C -r $TRACES/tls/tls-13draft19-early-data.pcap %INPUT
 # @TEST-EXEC: btest-diff .stdout
 
 event ssl_extension_elliptic_curves(c: connection, is_orig: bool, curves: index_vec)
@@ -33,3 +34,17 @@ event ssl_extension_signature_algorithm(c: connection, is_orig: bool, signature_
 		print SSL::hash_algorithms[signature_algorithms[i]$HashAlgorithm], SSL::signature_algorithms[signature_algorithms[i]$SignatureAlgorithm];
 		}
 	}
+
+event ssl_extension_supported_versions(c: connection, is_orig: bool, versions: index_vec)
+	{
+	print "supported_versions(", c$id$orig_h, c$id$resp_h;
+	for ( i in versions )
+		print SSL::version_strings[versions[i]];
+	}
+
+event ssl_extension_psk_key_exchange_modes(c: connection, is_orig: bool, modes: index_vec)
+	{
+	print "psk_key_exchange_modes", c$id$orig_h, c$id$resp_h;
+	for ( i in modes )
+		print modes[i];
+	}