Merge branch 'topic/johanna/signed_certificate_timestamp' into topic/johanna/ocsp-new

src/analyzer/protocol/ssl/CMakeLists.txt

@@ -12,6 +12,7 @@ bro_plugin_pac(tls-handshake.pac tls-handshake-protocol.pac tls-handshake-analyz
 	proc-client-hello.pac
 	proc-server-hello.pac
 	proc-certificate.pac
+	tls-handshake-signed_certificate_timestamp.pac
 )
 bro_plugin_pac(ssl.pac ssl-dtls-analyzer.pac ssl-analyzer.pac ssl-dtls-protocol.pac ssl-protocol.pac ssl-defs.pac
 	proc-client-hello.pac

src/analyzer/protocol/ssl/events.bif

@@ -192,7 +192,7 @@ event ssl_dh_server_params%(c: connection, p: string, q: string, Ys: string%);
 ## the initial handshake. It contains the list of client supported application
 ## protocols by the client or the server, respectively.
 ##
-## At the moment it is mostly used to negotiate the use of SPDY / HTTP2-drafts.
+## At the moment it is mostly used to negotiate the use of SPDY / HTTP2.
 ##
 ## c: The connection.
 ##
@@ -225,6 +225,27 @@ event ssl_extension_application_layer_protocol_negotiation%(c: connection, is_or
 ##    ssl_extension_key_share
 event ssl_extension_server_name%(c: connection, is_orig: bool, names: string_vec%);
 
+## Generated for the signed_certificate_timestamp TLS extension as defined in
+## :rfc:`6962`. The extension is used to transmit signed proofs that are
+## used for Certificate Transparency.
+##
+## c: The connection.
+##
+## is_orig: True if event is raised for originator side of the connection.
+##
+## version: the version of the protocol to which the SCT conforms. Always
+##          should be 0 (representing version 1)
+##
+## logid: 32 bit key id
+##
+## timestamp: the current NTP Time
+##
+## signature_and_hashalgorithm: signature and hash algorithm used for the
+##                              digitally_signed struct
+##
+## signature: signature part of the digitally_signed struct
+event ssl_extension_signed_certificate_timestamp%(c: connection, is_orig: bool, version: count, logid: string, timestamp: time, signature_and_hashalgorithm: SSL::SignatureAndHashAlgorithm, signature: string%);
+
 ## Generated at the end of an SSL/TLS handshake. SSL/TLS sessions start with
 ## an unencrypted handshake, and Bro extracts as much information out of that
 ## as it can. This event signals the time when an SSL/TLS has finished the

src/analyzer/protocol/ssl/tls-handshake-analyzer.pac

@@ -249,6 +249,24 @@ refine connection Handshake_Conn += {
 		return true;
 		%}
 
+	function proc_signedcertificatetimestamp(rec: HandshakeRecord, version: uint8, logid: const_bytestring, timestamp: uint64, digitally_signed_algorithms: SignatureAndHashAlgorithm, digitally_signed_signature: const_bytestring) : bool
+		%{
+		RecordVal* ha = new RecordVal(BifType::Record::SSL::SignatureAndHashAlgorithm);
+		ha->Assign(0, new Val(digitally_signed_algorithms->HashAlgorithm(), TYPE_COUNT));
+		ha->Assign(1, new Val(digitally_signed_algorithms->SignatureAlgorithm(), TYPE_COUNT));
+
+		BifEvent::generate_ssl_extension_signed_certificate_timestamp(bro_analyzer(),
+			bro_analyzer()->Conn(), ${rec.is_orig},
+			version,
+			new StringVal(logid.length(), reinterpret_cast<const char*>(logid.begin())),
+			((double)timestamp)/1000,
+			ha,
+			new StringVal(digitally_signed_signature.length(), reinterpret_cast<const char*>(digitally_signed_signature.begin()))
+		);
+
+		return true;
+		%}
+
 	function proc_dh_server_key_exchange(rec: HandshakeRecord, p: bytestring, g: bytestring, Ys: bytestring) : bool
 		%{
 		BifEvent::generate_ssl_dh_server_params(bro_analyzer(),
@@ -269,7 +287,6 @@ refine connection Handshake_Conn += {
 		return true;
 		%}
 
-
 };
 
 refine typeattr ClientHello += &let {
@@ -351,3 +368,6 @@ refine typeattr Handshake += &let {
 	proc : bool = $context.connection.proc_handshake(rec.is_orig, rec.msg_type, rec.msg_length);
 };
 
+refine typeattr SignedCertificateTimestamp += &let {
+	proc : bool = $context.connection.proc_signedcertificatetimestamp(rec, version, logid, timestamp, digitally_signed_algorithms, digitally_signed_signature);
+};

src/analyzer/protocol/ssl/tls-handshake-protocol.pac

@@ -497,11 +497,14 @@ type SSLExtension(rec: HandshakeRecord) = record {
 #		EXT_STATUS_REQUEST -> status_request: StatusRequest(rec)[] &until($element == 0 || $element != 0);
 		EXT_SERVER_NAME -> server_name: ServerNameExt(rec)[] &until($element == 0 || $element != 0);
 		EXT_SIGNATURE_ALGORITHMS -> signature_algorithm: SignatureAlgorithm(rec)[] &until($element == 0 || $element != 0);
+		EXT_SIGNED_CERTIFICATE_TIMESTAMP -> certificate_timestamp: SignedCertificateTimestampList(rec)[] &until($element == 0 || $element != 0);
 		EXT_KEY_SHARE -> key_share: KeyShare(rec)[] &until($element == 0 || $element != 0);
 		default -> data: bytestring &restofdata;
 	};
 } &length=data_len+4 &exportsourcedata;
 
+%include tls-handshake-signed_certificate_timestamp.pac
+
 type ServerNameHostName() = record {
 	length: uint16;
 	host_name: bytestring &length=length;
@@ -563,11 +566,6 @@ type KeyShare(rec: HandshakeRecord) = case rec.msg_type of {
 	default -> other : bytestring &restofdata &transient;
 };
 
-type SignatureAndHashAlgorithm() = record {
-	HashAlgorithm: uint8;
-	SignatureAlgorithm: uint8;
-}
-
 type SignatureAlgorithm(rec: HandshakeRecord) = record {
 	length: uint16;
 	supported_signature_algorithms: SignatureAndHashAlgorithm[] &until($input.length() == 0);

src/analyzer/protocol/ssl/tls-handshake-signed_certificate_timestamp.pac

@@ -0,0 +1,28 @@
+# We keep this extension separate, because it also can be included in X.509 certificates.
+# If included there, it uses the exact same syntax and we just symlink it from the X.509
+# file analyzer tree.
+
+type SignatureAndHashAlgorithm() = record {
+	HashAlgorithm: uint8;
+	SignatureAlgorithm: uint8;
+}
+
+type SignedCertificateTimestampList(rec: HandshakeRecord) = record {
+	length: uint16;
+	SCTs: SignedCertificateTimestamp(rec)[] &until($input.length() == 0);
+} &length=length+2;
+
+type SignedCertificateTimestamp(rec: HandshakeRecord) = record {
+	# before - framing
+	length: uint16;
+	# from here: SignedCertificateTimestamp
+	version: uint8;
+	logid: bytestring &length=32;
+	timestamp: uint64;
+	extensions_length: uint16; # extensions are not actually defined yet, so we cannot parse them
+	extensions: bytestring &length=extensions_length;
+	digitally_signed_algorithms: SignatureAndHashAlgorithm;
+	digitally_signed_signature_length: uint16;
+	digitally_signed_signature: bytestring &length=digitally_signed_signature_length;
+} &length=length+2;
+

src/file_analysis/analyzer/x509/CMakeLists.txt

@@ -7,4 +7,5 @@ include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR}
 bro_plugin_begin(Bro X509)
 bro_plugin_cc(X509.cc OCSP.cc Plugin.cc)
 bro_plugin_bif(events.bif types.bif functions.bif ocsp_events.bif)
+bro_plugin_pac(x509-extension.pac x509-signed_certificate_timestamp.pac)
 bro_plugin_end()

src/file_analysis/analyzer/x509/X509.cc

@@ -4,6 +4,7 @@
 
 #include "X509.h"
 #include "Event.h"
+#include "x509-extension_pac.h"
 
 #include "events.bif.h"
 #include "types.bif.h"
@@ -301,6 +302,56 @@ void file_analysis::X509::ParseExtension(X509_EXTENSION* ex)
 
 	else if ( OBJ_obj2nid(ext_asn) == NID_subject_alt_name )
 		ParseSAN(ex);
+
+#ifdef NID_ct_cert_scts
+	else if ( OBJ_obj2nid(ext_asn) == NID_ct_cert_scts || OBJ_obj2nid(ext_asn) == NID_ct_precert_scts )
+#else
+	else if ( strcmp(oid, "1.3.6.1.4.1.11129.2.4.2") == 0 || strcmp(oid, "1.3.6.1.4.1.11129.2.4.4") == 0 )
+#endif
+		ParseSignedCertificateTimestamps(ex);
+	}
+
+void file_analysis::X509::ParseSignedCertificateTimestamps(X509_EXTENSION* ext)
+	{
+	// Ok, signed certificate timestamps are a bit of an odd case out; we don't
+	// want to use the (basically nonexistant) OpenSSL functionality to parse them.
+	// Instead we have our own, self-written binpac parser to parse just them,
+	// which we will initialize here and tear down immediately again.
+
+	ASN1_OCTET_STRING* ext_val = X509_EXTENSION_get_data(ext);
+	// the octet string of the extension contains the octet string which in turn
+	// contains the SCT. Obviously.
+
+	unsigned char* ext_val_copy = (unsigned char*) OPENSSL_malloc(ext_val->length);
+	unsigned char* ext_val_second_pointer = ext_val_copy;
+	memcpy(ext_val_copy, ext_val->data, ext_val->length);
+
+	ASN1_OCTET_STRING* inner = d2i_ASN1_OCTET_STRING(NULL, (const unsigned char**) &ext_val_copy, ext_val->length);
+	if ( !inner )
+		{
+		reporter->Error("X509::ParseSignedCertificateTimestamps could not parse inner octet string");
+		return;
+		}
+
+	binpac::X509Extension::MockConnection* conn = new binpac::X509Extension::MockConnection(this);
+	binpac::X509Extension::SignedCertTimestampExt* interp = new binpac::X509Extension::SignedCertTimestampExt(conn);
+
+	try
+		{
+		interp->NewData(inner->data, inner->data + inner->length);
+		}
+	catch( const binpac::Exception& e )
+		{
+		// throw a warning or sth
+		reporter->Error("X509::ParseSignedCertificateTimestamps could not parse SCT");
+		}
+
+	OPENSSL_free(ext_val_second_pointer);
+
+	interp->FlowEOF();
+
+	delete interp;
+	delete conn;
 	}
 
 void file_analysis::X509::ParseBasicConstraints(X509_EXTENSION* ex)

src/file_analysis/analyzer/x509/X509.h

@@ -58,6 +58,7 @@ private:
 	void ParseExtension(X509_EXTENSION* ex);
 	void ParseBasicConstraints(X509_EXTENSION* ex);
 	void ParseSAN(X509_EXTENSION* ex);
+	void ParseSignedCertificateTimestamps(X509_EXTENSION* ext);
 
 	std::string cert_data;
 

src/file_analysis/analyzer/x509/events.bif

@@ -55,3 +55,22 @@ event x509_ext_basic_constraints%(f: fa_file, ext: X509::BasicConstraints%);
 ##              x509_parse x509_verify
 ##              x509_get_certificate_string
 event x509_ext_subject_alternative_name%(f: fa_file, ext: X509::SubjectAlternativeName%);
+
+## Generated for the signed_certificate_timestamp X509 extension as defined in
+## :rfc:`6962`. The extension is used to transmit signed proofs that are
+## used for Certificate Transparency.
+##
+## f: The file.
+##
+## version: the version of the protocol to which the SCT conforms. Always
+##          should be 0 (representing version 1)
+##
+## logid: 32 bit key id
+##
+## timestamp: the timestamp of the sct
+##
+## signature_and_hashalgorithm: signature and hash algorithm used for the
+##                              digitally_signed struct
+##
+## signature: signature part of the digitally_signed struct
+event x509_ext_signed_certificate_timestamp%(f: fa_file, version: count, logid: string, timestamp: time, hash_algorithm: count, signature_algorithm: count, signature: string%);

src/file_analysis/analyzer/x509/x509-extension.pac

@@ -0,0 +1,54 @@
+# Binpac analyzer for X.509 extensions
+# we just use it for the SignedCertificateTimestamp at the moment
+
+%include binpac.pac
+%include bro.pac
+
+%extern{
+#include "types.bif.h"
+#include "file_analysis/File.h"
+#include "events.bif.h"
+%}
+
+analyzer X509Extension withcontext {
+	connection: MockConnection;
+	flow:       SignedCertTimestampExt;
+};
+
+connection MockConnection(bro_analyzer: BroFileAnalyzer) {
+	upflow = SignedCertTimestampExt;
+	downflow = SignedCertTimestampExt;
+};
+
+%include x509-signed_certificate_timestamp.pac
+
+# The base record
+type HandshakeRecord() = record {
+  signed_certificate_timestamp_list: SignedCertificateTimestampList(this)[] &transient;
+} &byteorder = bigendian;
+
+flow SignedCertTimestampExt {
+	flowunit = HandshakeRecord withcontext(connection, this);
+};
+
+refine connection MockConnection += {
+
+	function proc_signedcertificatetimestamp(rec: HandshakeRecord, version: uint8, logid: const_bytestring, timestamp: uint64, digitally_signed_algorithms: SignatureAndHashAlgorithm, digitally_signed_signature: const_bytestring) : bool
+		%{
+		BifEvent::generate_x509_ext_signed_certificate_timestamp((analyzer::Analyzer *) bro_analyzer(),
+			bro_analyzer()->GetFile()->GetVal()->Ref(),
+			version,
+			new StringVal(logid.length(), reinterpret_cast<const char*>(logid.begin())),
+			((double)timestamp)/1000,
+			digitally_signed_algorithms->HashAlgorithm(),
+			digitally_signed_algorithms->SignatureAlgorithm(),
+			new StringVal(digitally_signed_signature.length(), reinterpret_cast<const char*>(digitally_signed_signature.begin()))
+		);
+
+		return true;
+		%}
+};
+
+refine typeattr SignedCertificateTimestamp += &let {
+	proc : bool = $context.connection.proc_signedcertificatetimestamp(rec, version, logid, timestamp, digitally_signed_algorithms, digitally_signed_signature);
+};

src/file_analysis/analyzer/x509/x509-signed_certificate_timestamp.pac

@@ -0,0 +1 @@
+../../../analyzer/protocol/ssl/tls-handshake-signed_certificate_timestamp.pac

testing/btest/Baseline/scripts.base.files.x509.signed_certificate_timestamp/.stdout

@@ -0,0 +1,4 @@
+0, 1474927230.876, 4, 3
+0, 1474927232.863, 4, 3
+0, 1474927232.112, 4, 3
+0, 1474927232.304, 4, 3

testing/btest/Baseline/scripts.base.protocols.ssl.signed_certificate_timestamp/.stdout

@@ -0,0 +1,3 @@
+0, 1406997753.366, [HashAlgorithm=4, SignatureAlgorithm=3]
+0, 1407002457.456, [HashAlgorithm=4, SignatureAlgorithm=3]
+0, 1410299366.023, [HashAlgorithm=4, SignatureAlgorithm=3]

testing/btest/scripts/base/files/x509/signed_certificate_timestamp.test

@@ -0,0 +1,7 @@
+# @TEST-EXEC: bro -r $TRACES/tls/certificate-with-sct.pcap %INPUT
+# @TEST-EXEC: btest-diff .stdout
+
+event x509_ext_signed_certificate_timestamp(f: fa_file, version: count, logid: string, timestamp: time, hash_algorithm: count, signature_algorithm: count, signature: string)
+	{
+	print version, timestamp, hash_algorithm, signature_algorithm;
+	}

testing/btest/scripts/base/protocols/ssl/signed_certificate_timestamp.test

@@ -0,0 +1,7 @@
+# @TEST-EXEC: bro -r $TRACES/tls/signed_certificate_timestamp.pcap %INPUT
+# @TEST-EXEC: btest-diff .stdout
+
+event ssl_extension_signed_certificate_timestamp(c: connection, is_orig: bool, version: count, logid: string, timestamp: time, signature_and_hashalgorithm: SSL::SignatureAndHashAlgorithm, signature: string)
+	{
+	print version, timestamp, signature_and_hashalgorithm;
+	}