Merge remote-tracking branch 'origin/topic/johanna/x509-cn'

* origin/topic/johanna/x509-cn:
  Use our new features to send the CN and SAN fields of certificates to the intel framework.
  Do not log common name by default (it is most interesting for scripts) and add a test case.
  extract most specific common name from certificates

BIT-1323 #merged

CHANGES

@@ -1,4 +1,12 @@
 
+2.3-516 | 2015-03-04 12:30:06 -0800
+
+  * Extract most specific Common Name from SSL certificates (Johanna
+    Amann)
+
+  * Send CN and SAN fields of SSL certificates to the Intel framework.
+    (Johanna Amann)
+
 2.3-511 | 2015-03-02 18:07:17 -0800
 
   * Changes to plugin meta hooks for function calls. (Gilbert Clark)

VERSION

@@ -1 +1 @@
-2.3-511
+2.3-516

scripts/base/init-bare.bro

@@ -2774,19 +2774,20 @@ export {
 module X509;
 export {
 	type Certificate: record {
-		version: count;	##< Version number.
-		serial: string;	##< Serial number.
-		subject: string;	##< Subject.
-		issuer: string;	##< Issuer.
-		not_valid_before: time;	##< Timestamp before when certificate is not valid.
-		not_valid_after: time;	##< Timestamp after when certificate is not valid.
-		key_alg: string;	##< Name of the key algorithm
-		sig_alg: string;	##< Name of the signature algorithm
-		key_type: string &optional;	##< Key type, if key parseable by openssl (either rsa, dsa or ec)
-		key_length: count &optional;	##< Key length in bits
-		exponent: string &optional;	##< Exponent, if RSA-certificate
-		curve: string &optional;	##< Curve, if EC-certificate
-	} &log;
+		version: count &log;	##< Version number.
+		serial: string &log;	##< Serial number.
+		subject: string &log;	##< Subject.
+		issuer: string &log;	##< Issuer.
+		cn: string &optional; ##< Last (most specific) common name.
+		not_valid_before: time &log;	##< Timestamp before when certificate is not valid.
+		not_valid_after: time &log;	##< Timestamp after when certificate is not valid.
+		key_alg: string &log;	##< Name of the key algorithm
+		sig_alg: string &log;	##< Name of the signature algorithm
+		key_type: string &optional &log;	##< Key type, if key parseable by openssl (either rsa, dsa or ec)
+		key_length: count &optional &log;	##< Key length in bits
+		exponent: string &optional &log;	##< Exponent, if RSA-certificate
+		curve: string &optional &log;	##< Curve, if EC-certificate
+	};
 
 	type Extension: record {
 		name: string;	##< Long name of extension. oid if name not known

scripts/policy/frameworks/intel/seen/ssl.bro

@@ -10,3 +10,16 @@ event ssl_extension_server_name(c: connection, is_orig: bool, names: string_vec)
 		             $conn=c,
 		             $where=SSL::IN_SERVER_NAME]);
 	}
+
+event ssl_established(c: connection)
+	{
+	if ( ! c$ssl?$cert_chain || |c$ssl$cert_chain| == 0 ||
+	     ! c$ssl$cert_chain[0]?$x509 )
+		return;
+
+	if ( c$ssl$cert_chain[0]$x509?$certificate && c$ssl$cert_chain[0]$x509$certificate?$cn )
+		Intel::seen([$indicator=c$ssl$cert_chain[0]$x509$certificate$cn,
+			$indicator_type=Intel::DOMAIN,
+			$conn=c,
+			$where=X509::IN_CERT]);
+	}

scripts/policy/frameworks/intel/seen/x509.bro

@@ -2,6 +2,18 @@
 @load base/files/x509
 @load ./where-locations
 
+event x509_ext_subject_alternative_name(f: fa_file, ext: X509::SubjectAlternativeName)
+	{
+	if ( ext?$dns )
+		{
+		for ( i in ext$dns )
+			Intel::seen([$indicator=ext$dns[i],
+				$indicator_type=Intel::DOMAIN,
+				$f=f,
+				$where=X509::IN_CERT]);
+		}
+	}
+
 event x509_certificate(f: fa_file, cert_ref: opaque of x509, cert: X509::Certificate)
 	{
 	if ( /emailAddress=/ in cert$subject )

src/file_analysis/analyzer/x509/X509.cc

@@ -104,13 +104,35 @@ RecordVal* file_analysis::X509::ParseCertificate(X509Val* cert_val)
 	len = BIO_gets(bio, buf, sizeof(buf));
 	pX509Cert->Assign(2, new StringVal(len, buf));
 	BIO_reset(bio);
+
+	X509_NAME *subject_name = X509_get_subject_name(ssl_cert);
+	// extract the most specific (last) common name from the subject
+	int namepos = -1;
+	for ( ;; )
+		{
+		int j = X509_NAME_get_index_by_NID(subject_name, NID_commonName, namepos);
+		if ( j == -1 )
+			break;
+
+		namepos = j;
+		}
+
+	if ( namepos != -1 )
+		{
+		// we found a common name
+		ASN1_STRING_print(bio, X509_NAME_ENTRY_get_data(X509_NAME_get_entry(subject_name, namepos)));
+		len = BIO_gets(bio, buf, sizeof(buf));
+		pX509Cert->Assign(4, new StringVal(len, buf));
+		BIO_reset(bio);
+		}
+
 	X509_NAME_print_ex(bio, X509_get_issuer_name(ssl_cert), 0, XN_FLAG_RFC2253);
 	len = BIO_gets(bio, buf, sizeof(buf));
 	pX509Cert->Assign(3, new StringVal(len, buf));
 	BIO_free(bio);
 
-	pX509Cert->Assign(4, new Val(GetTimeFromAsn1(X509_get_notBefore(ssl_cert)), TYPE_TIME));
-	pX509Cert->Assign(5, new Val(GetTimeFromAsn1(X509_get_notAfter(ssl_cert)), TYPE_TIME));
+	pX509Cert->Assign(5, new Val(GetTimeFromAsn1(X509_get_notBefore(ssl_cert)), TYPE_TIME));
+	pX509Cert->Assign(6, new Val(GetTimeFromAsn1(X509_get_notAfter(ssl_cert)), TYPE_TIME));
 
 	// we only read 255 bytes because byte 256 is always 0.
 	// if the string is longer than 255, that will be our null-termination,
@@ -118,28 +140,28 @@ RecordVal* file_analysis::X509::ParseCertificate(X509Val* cert_val)
 	if ( ! i2t_ASN1_OBJECT(buf, 255, ssl_cert->cert_info->key->algor->algorithm) )
 		buf[0] = 0;
 
-	pX509Cert->Assign(6, new StringVal(buf));
+	pX509Cert->Assign(7, new StringVal(buf));
 
 	if ( ! i2t_ASN1_OBJECT(buf, 255, ssl_cert->sig_alg->algorithm) )
 		buf[0] = 0;
 
-	pX509Cert->Assign(7, new StringVal(buf));
+	pX509Cert->Assign(8, new StringVal(buf));
 
 	// Things we can do when we have the key...
 	EVP_PKEY *pkey = X509_extract_key(ssl_cert);
 	if ( pkey != NULL )
 		{
 		if ( pkey->type == EVP_PKEY_DSA )
-			pX509Cert->Assign(8, new StringVal("dsa"));
+			pX509Cert->Assign(9, new StringVal("dsa"));
 
 		else if ( pkey->type == EVP_PKEY_RSA )
 			{
-			pX509Cert->Assign(8, new StringVal("rsa"));
+			pX509Cert->Assign(9, new StringVal("rsa"));
 
 			char *exponent = BN_bn2dec(pkey->pkey.rsa->e);
 			if ( exponent != NULL )
 				{
-				pX509Cert->Assign(10, new StringVal(exponent));
+				pX509Cert->Assign(11, new StringVal(exponent));
 				OPENSSL_free(exponent);
 				exponent = NULL;
 				}
@@ -147,14 +169,14 @@ RecordVal* file_analysis::X509::ParseCertificate(X509Val* cert_val)
 #ifndef OPENSSL_NO_EC
 		else if ( pkey->type == EVP_PKEY_EC )
 			{
-			pX509Cert->Assign(8, new StringVal("ecdsa"));
-			pX509Cert->Assign(11, KeyCurve(pkey));
+			pX509Cert->Assign(9, new StringVal("ecdsa"));
+			pX509Cert->Assign(12, KeyCurve(pkey));
 			}
 #endif
 
 		unsigned int length = KeyLength(pkey);
 		if ( length > 0 )
-			pX509Cert->Assign(9, new Val(length, TYPE_COUNT));
+			pX509Cert->Assign(10, new Val(length, TYPE_COUNT));
 
 		EVP_PKEY_free(pkey);
 		}

testing/btest/Baseline/scripts.base.protocols.ssl.common_name/.stdout

@@ -0,0 +1,3 @@
+*.gstatic.com
+Google Internet Authority
+No CN

testing/btest/Baseline/scripts.policy.frameworks.intel.seen.certs/intel-all.log

@@ -0,0 +1,22 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	intel
+#open	2015-03-04-01-12-47
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	fuid	file_mime_type	file_desc	seen.indicator	seen.indicator_type	seen.where	seen.node	sources
+#types	time	string	addr	port	addr	port	string	string	string	string	enum	enum	string	set[string]
+1416942644.593119	CXWv6p3arKYeMETxOg	192.168.4.149	49422	23.92.19.75	443	F0txuw2pvrkZOn04a8	-	23.92.19.75:443/tcp	www.pantz.org	Intel::DOMAIN	X509::IN_CERT	bro	source1
+#close	2015-03-04-01-12-47
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	intel
+#open	2015-03-04-01-12-47
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	fuid	file_mime_type	file_desc	seen.indicator	seen.indicator_type	seen.where	seen.node	sources
+#types	time	string	addr	port	addr	port	string	string	string	string	enum	enum	string	set[string]
+1170717505.934612	CXWv6p3arKYeMETxOg	192.150.187.164	58868	194.127.84.106	443	-	-	-	www.dresdner-privat.de	Intel::DOMAIN	X509::IN_CERT	bro	source1
+1170717509.082241	CjhGID4nQcgTWjvg4c	192.150.187.164	58869	194.127.84.106	443	-	-	-	www.dresdner-privat.de	Intel::DOMAIN	X509::IN_CERT	bro	source1
+1170717512.108799	CCvvfg3TEfuqmmG4bh	192.150.187.164	58870	194.127.84.106	443	-	-	-	www.dresdner-privat.de	Intel::DOMAIN	X509::IN_CERT	bro	source1
+#close	2015-03-04-01-12-47

testing/btest/scripts/base/protocols/ssl/common_name.test

@@ -0,0 +1,13 @@
+# This tests a normal SSL connection and the log it outputs.
+
+# @TEST-EXEC: bro -r $TRACES/tls/tls-conn-with-extensions.trace %INPUT
+# @TEST-EXEC: bro -C -r $TRACES/tls/cert-no-cn.pcap %INPUT
+# @TEST-EXEC: btest-diff .stdout
+
+event x509_certificate(f: fa_file, cert_ref: opaque of x509, cert: X509::Certificate)
+	{
+	if ( cert?$cn )
+		print cert$cn;
+	else
+		print "No CN";
+	}

testing/btest/scripts/policy/frameworks/intel/seen/certs.bro

@@ -0,0 +1,18 @@
+# @TEST-EXEC: bro -r $TRACES/tls/ecdsa-cert.pcap %INPUT
+# @TEST-EXEC: cat intel.log > intel-all.log
+# @TEST-EXEC: bro -r $TRACES/tls/ssl.v3.trace %INPUT
+# @TEST-EXEC: cat intel.log >> intel-all.log
+# @TEST-EXEC: btest-diff intel-all.log
+
+@TEST-START-FILE intel.dat
+#fields	indicator	indicator_type	meta.source	meta.desc	meta.url
+www.pantz.org	Intel::DOMAIN	source1	test entry	http://some-data-distributor.com/100000
+www.dresdner-privat.de	Intel::DOMAIN	source1	test entry	http://some-data-distributor.com/100000
+@TEST-END-FILE
+
+@load base/frameworks/intel
+@load base/protocols/ssl
+@load frameworks/intel/seen
+
+redef Intel::read_files += { "intel.dat" };
+