Add verify functionality, including the ability to get the validated

chain. This means that it is now possible to get information about the
root-certificates that were used to secure a connection.

Intermediate commit before changing the script interface again.

addresses BIT-953, BIT-760

scripts/base/files/x509/main.bro

@@ -5,24 +5,27 @@ module X509;
 
 export {
 	redef enum Log::ID += { LOG };
+
+	redef record Files::Info += {
+	};
 }
 
-event x509_cert(f: fa_file, cert: X509::Certificate)
+event x509_cert(f: fa_file, cert_ref: opaque of x509, cert: X509::Certificate)
 	{
 	print cert;
 	}
 
-event x509_extension(f: fa_file, cert: X509::Certificate, ext: X509::Extension)
+event x509_extension(f: fa_file, cert_ref: opaque of x509, cert: X509::Certificate, ext: X509::Extension)
 {
 print ext;
 }
 
-event x509_ext_basic_constraints(f: fa_file, cert: X509::Certificate, ext: X509::BasicConstraints)
+event x509_ext_basic_constraints(f: fa_file, cert_ref: opaque of x509, cert: X509::Certificate, ext: X509::BasicConstraints)
 {
 print ext;
 }
 
-event x509_ext_subject_alternative_name(f: fa_file, cert: X509::Certificate, ext: X509::SubjectAlternativeName) 
+event x509_ext_subject_alternative_name(f: fa_file, cert_ref: opaque of x509, cert: X509::Certificate, ext: string_vec) 
 {
 print ext;
 }

scripts/base/init-bare.bro

@@ -46,6 +46,13 @@ type index_vec: vector of count;
 ##    directly and then remove this alias.
 type string_vec: vector of string;
 
+## A vector of x509 opaques.
+##
+## .. todo:: We need this type definition only for declaring builtin functions
+##    via ``bifcl``. We should extend ``bifcl`` to understand composite types
+##    directly and then remove this alias.
+type x509_opaque_vector: vector of opaque of x509;
+
 ## A vector of addresses.
 ##
 ## .. todo:: We need this type definition only for declaring builtin functions
@@ -2744,7 +2751,6 @@ export {
 module X509;
 export {
 	type X509::Certificate: record {
-		certificate: opaque of x509; ##< OpenSSL certificate reference
 		version: count;	##< Version number.
 		serial: string;	##< Serial number.
 		subject: string;	##< Subject.
@@ -2774,8 +2780,14 @@ export {
 		path_len: count &optional;
 	};
 	
-	type X509::SubjectAlternativeName: record {
+	## Result of an X509 certificate chain verification
-		names: vector of string;
+	type X509::Result: record {
+		## OpenSSL result code
+		result:	count;
+		## Result as string
+		result_string: string;
+		## References to the final certificate chain, if verification successful. End-host certificate is first.
+		chain_certs: vector of opaque of x509 &optional;
 	};
 }
 		

src/Type.h

@@ -73,6 +73,7 @@ class EnumType;
 class Serializer;
 class VectorType;
 class TypeType;
+class OpaqueType;
 
 const int DOES_NOT_MATCH_INDEX = 0;
 const int MATCHES_INDEX_SCALAR = 1;
@@ -204,6 +205,18 @@ public:
 		return (VectorType*) this;
 		}
 
+	OpaqueType* AsOpaqueType()
+		{
+		CHECK_TYPE_TAG(TYPE_OPAQUE, "BroType::AsOpaqueType");
+		return (OpaqueType*) this;
+		}
+
+	const OpaqueType* AsOpaqueType() const
+		{
+		CHECK_TYPE_TAG(TYPE_OPAQUE, "BroType::AsOpaqueType");
+		return (OpaqueType*) this;
+		}
+
 	VectorType* AsVectorType()
 	        {
 		CHECK_TYPE_TAG(TYPE_VECTOR, "BroType::AsVectorType");

src/file_analysis/analyzer/x509/X509.cc

@@ -49,7 +49,17 @@ bool file_analysis::X509::EndOfFile()
 		return false;
 		}
 
-	RecordVal* cert_record = ParseCertificate(ssl_cert); // cert_record takes ownership of ssl_cert
+	X509Val* cert_val = new X509Val(ssl_cert); // cert_val takes ownership of ssl_cert
+
+	RecordVal* cert_record = ParseCertificate(cert_val); // parse basic information into record
+
+	// and send	the record on to scriptland
+	val_list* vl = new val_list();
+	vl->append(GetFile()->GetVal()->Ref());
+	vl->append(cert_val->Ref());
+	vl->append(cert_record->Ref()); // we Ref it here, because we want to keep a copy around for now...
+
+	mgr.QueueEvent(x509_cert, vl);
 
 	// after parsing the certificate - parse the extensions...
 
@@ -60,7 +70,7 @@ bool file_analysis::X509::EndOfFile()
 		if ( !ex )
 			continue;
 
-		ParseExtension(ex, cert_record);
+		ParseExtension(ex, cert_record, cert_val);
 		}
 	
 	// X509_free(ssl_cert); We do _not_ free the certificate here. It is refcounted
@@ -69,34 +79,36 @@ bool file_analysis::X509::EndOfFile()
 	// The certificate will be freed when the last X509Val is Unref'd.
 	
 	Unref(cert_record); // Unref the RecordVal that we kept around from ParseCertificate
+	Unref(cert_val); // Same for cert_val
 
 	return false;
 	}
 
-RecordVal* file_analysis::X509::ParseCertificate(::X509* ssl_cert) 
+RecordVal* file_analysis::X509::ParseCertificate(X509Val* cert_val) 
 	{
+	::X509* ssl_cert = cert_val->GetCertificate();
+
 	char buf[256]; // we need a buffer for some of the openssl functions
 	memset(buf, 0, 256);
 	
 	RecordVal* pX509Cert = new RecordVal(BifType::Record::X509::Certificate);
 	BIO *bio = BIO_new(BIO_s_mem());
 
-	pX509Cert->Assign(0, new X509Val(ssl_cert)); // take ownership for cleanup
+	pX509Cert->Assign(0, new Val((uint64) X509_get_version(ssl_cert), TYPE_COUNT));
-	pX509Cert->Assign(1, new Val((uint64) X509_get_version(ssl_cert), TYPE_COUNT));
 	i2a_ASN1_INTEGER(bio, X509_get_serialNumber(ssl_cert));
 	int len = BIO_read(bio, &(*buf), sizeof buf);
-	pX509Cert->Assign(2, new StringVal(len, buf));
+	pX509Cert->Assign(1, new StringVal(len, buf));
 
 	X509_NAME_print_ex(bio, X509_get_subject_name(ssl_cert), 0, XN_FLAG_RFC2253);
 	len = BIO_gets(bio, &(*buf), sizeof buf);
-	pX509Cert->Assign(3, new StringVal(len, buf));
+	pX509Cert->Assign(2, new StringVal(len, buf));
 	X509_NAME_print_ex(bio, X509_get_issuer_name(ssl_cert), 0, XN_FLAG_RFC2253);
 	len = BIO_gets(bio, &(*buf), sizeof buf);
-	pX509Cert->Assign(4, new StringVal(len, buf));
+	pX509Cert->Assign(3, new StringVal(len, buf));
 	BIO_free(bio);
 
-	pX509Cert->Assign(5, new Val(get_time_from_asn1(X509_get_notBefore(ssl_cert)), TYPE_TIME));
+	pX509Cert->Assign(4, new Val(get_time_from_asn1(X509_get_notBefore(ssl_cert)), TYPE_TIME));
-	pX509Cert->Assign(6, new Val(get_time_from_asn1(X509_get_notAfter(ssl_cert)), TYPE_TIME));
+	pX509Cert->Assign(5, new Val(get_time_from_asn1(X509_get_notAfter(ssl_cert)), TYPE_TIME));
 
 	// we only read 255 bytes because byte 256 is always 0.
 	// if the string is longer than 255, that will be our null-termination,
@@ -141,16 +153,11 @@ RecordVal* file_analysis::X509::ParseCertificate(::X509* ssl_cert)
 			pX509Cert->Assign(9, new Val(length, TYPE_COUNT));
 		}
 
-	val_list* vl = new val_list();
-	vl->append(GetFile()->GetVal()->Ref());
-	vl->append(pX509Cert->Ref()); // we Ref it here, because we want to keep a copy around for now...
-
-	mgr.QueueEvent(x509_cert, vl);
 
 	return pX509Cert;
 	}
 
-void file_analysis::X509::ParseExtension(X509_EXTENSION* ex, RecordVal* r) 
+void file_analysis::X509::ParseExtension(X509_EXTENSION* ex, RecordVal* r, X509Val* cert_val) 
 	{
 	char name[256];
 	char oid[256];
@@ -196,6 +203,7 @@ void file_analysis::X509::ParseExtension(X509_EXTENSION* ex, RecordVal* r)
 	// but I am not sure if there is a better way to do it...
 	val_list* vl = new val_list();
 	vl->append(GetFile()->GetVal()->Ref());
+	vl->append(cert_val->Ref());
 	vl->append(r->Ref());
 	vl->append(pX509Ext);
 
@@ -203,12 +211,12 @@ void file_analysis::X509::ParseExtension(X509_EXTENSION* ex, RecordVal* r)
 
 	// look if we have a specialized handler for this event...
 	if ( OBJ_obj2nid(ext_asn) == NID_basic_constraints ) 
-		ParseBasicConstraints(ex, r);
+		ParseBasicConstraints(ex, r, cert_val);
 	else if ( OBJ_obj2nid(ext_asn) == NID_subject_alt_name )
-		ParseSAN(ex, r);
+		ParseSAN(ex, r, cert_val);
 	}
 
-void file_analysis::X509::ParseBasicConstraints(X509_EXTENSION* ex, RecordVal* r) 
+void file_analysis::X509::ParseBasicConstraints(X509_EXTENSION* ex, RecordVal* r, X509Val* cert_val) 
 	{
 	assert(OBJ_obj2nid(X509_EXTENSION_get_object(ex)) == NID_basic_constraints);
 	
@@ -226,6 +234,7 @@ void file_analysis::X509::ParseBasicConstraints(X509_EXTENSION* ex, RecordVal* r
 		}
 		val_list* vl = new val_list();
 		vl->append(GetFile()->GetVal()->Ref());
+		vl->append(cert_val->Ref());
 		vl->append(r->Ref());
 		vl->append(pBasicConstraint);
 
@@ -234,7 +243,7 @@ void file_analysis::X509::ParseBasicConstraints(X509_EXTENSION* ex, RecordVal* r
 		}
 	}
 
-void file_analysis::X509::ParseSAN(X509_EXTENSION* ext, RecordVal* r) 
+void file_analysis::X509::ParseSAN(X509_EXTENSION* ext, RecordVal* r, X509Val* cert_val) 
 	{
 	assert(OBJ_obj2nid(X509_EXTENSION_get_object(ext)) == NID_subject_alt_name);
 
@@ -273,13 +282,11 @@ void file_analysis::X509::ParseSAN(X509_EXTENSION* ext, RecordVal* r)
 			}
 		}
 
-		RecordVal* pSan = new RecordVal(BifType::Record::X509::SubjectAlternativeName);
-		pSan->Assign(0, names);
-
 		val_list* vl = new val_list();
 		vl->append(GetFile()->GetVal()->Ref());
+		vl->append(cert_val->Ref());
 		vl->append(r->Ref());
-		vl->append(pSan);
+		vl->append(names);
 
 		mgr.QueueEvent(x509_ext_basic_constraints, vl);
 	}

src/file_analysis/analyzer/x509/X509.h

@@ -12,6 +12,8 @@
 
 namespace file_analysis {
 
+class X509Val;
+
 class X509 : public file_analysis::Analyzer {
 public:
 	//~X509();
@@ -19,6 +21,8 @@ public:
 	static file_analysis::Analyzer* Instantiate(RecordVal* args, File* file)
 		{ return new X509(args, file); }
 	
+	static RecordVal* ParseCertificate(X509Val* cert_val);
+
 	virtual bool DeliverStream(const u_char* data, uint64 len);
 	virtual bool Undelivered(uint64 offset, uint64 len);	
 	virtual bool EndOfFile();
@@ -31,10 +35,9 @@ private:
 	static StringVal* key_curve(EVP_PKEY *key);
 	static unsigned int key_length(EVP_PKEY *key);
 
-	RecordVal* ParseCertificate(::X509* ssl_cert);
+	void ParseExtension(X509_EXTENSION* ex, RecordVal* r, X509Val* cert_val);
-	void ParseExtension(X509_EXTENSION* ex, RecordVal* r);
+	void ParseBasicConstraints(X509_EXTENSION* ex, RecordVal* r, X509Val* cert_val);
-	void ParseBasicConstraints(X509_EXTENSION* ex, RecordVal* r);
+	void ParseSAN(X509_EXTENSION* ex, RecordVal* r, X509Val* cert_val);
-	void ParseSAN(X509_EXTENSION* ex, RecordVal* r);
 
 	std::string cert_data;
 };

src/file_analysis/analyzer/x509/events.bif

@@ -1,4 +1,4 @@
-event x509_cert%(f: fa_file, cert: X509::Certificate%);
+event x509_cert%(f: fa_file, cert_ref: opaque of x509, cert: X509::Certificate%);
-event x509_extension%(f: fa_file, cert: X509::Certificate, ext: X509::Extension%);
+event x509_extension%(f: fa_file, cert_ref: opaque of x509, cert: X509::Certificate, ext: X509::Extension%);
-event x509_ext_basic_constraints%(f: fa_file, cert: X509::Certificate, ext: X509::BasicConstraints%);
+event x509_ext_basic_constraints%(f: fa_file, cert_ref: opaque of x509, cert: X509::Certificate, ext: X509::BasicConstraints%);
-event x509_ext_subject_alternative_name%(f: fa_file, cert: X509::Certificate, ext: X509::SubjectAlternativeName%);
+event x509_ext_subject_alternative_name%(f: fa_file, cert_ref: opaque of x509, cert: X509::Certificate, names: string_vec%);

src/file_analysis/analyzer/x509/functions.bif

@@ -0,0 +1,171 @@
+%%{
+#include "file_analysis/analyzer/x509/X509.h"
+#include "types.bif.h"
+
+#include <openssl/x509.h>
+#include <openssl/asn1.h>
+#include <openssl/x509_vfy.h>
+
+// This is the indexed map of X509 certificate stores.
+static map<Val*, X509_STORE*> x509_stores;
+
+// ### NOTE: while d2i_X509 does not take a const u_char** pointer,
+// here we assume d2i_X509 does not write to <data>, so it is safe to
+// convert data to a non-const pointer.  Could some X509 guru verify
+// this?
+
+X509* d2i_X509_(X509** px, const u_char** in, int len)
+	{
+#ifdef OPENSSL_D2I_X509_USES_CONST_CHAR
+	  return d2i_X509(px, in, len);
+#else
+	  return d2i_X509(px, (u_char**)in, len);
+#endif
+	}
+
+%%}
+
+## Parses a certificate into an X509::Certificate structure
+##
+## cert: The x509 certificicate opaque
+##
+## Returns: A X509::Certificate structure
+##
+## .. bro:see:: x509_verify
+function x509_parse%(cert: opaque of x509%): X509::Certificate
+	%{
+	assert(cert);
+	file_analysis::X509Val* h = (file_analysis::X509Val*) cert;
+	
+	return file_analysis::X509::ParseCertificate(h);
+	%}
+
+## Verifies a certificate.
+##
+## cert_val: The X.509 certificate in DER format.
+##
+## cert_stack: Specifies a certificate chain that is being used to validate
+##             the given certificate against the root store given in *root_certs*
+##
+## root_certs: A list of root certificates to validate the certificate chain
+##
+## Returns: A record of type X509::Result containing the result code of the verify
+##          operation. In case of success also returns the full certificate chain.
+##
+## .. bro:see:: x509_parse
+function x509_verify%(cert_val: opaque of x509, cert_stack: x509_opaque_vector, root_certs: table_string_of_string%): X509::Result
+	%{
+	X509_STORE* ctx = 0;
+	int i = 0;
+
+	// If this certificate store was built previously, just reuse the old one.
+	if ( x509_stores.count(root_certs) > 0 )
+		ctx = x509_stores[root_certs];
+
+	if ( ! ctx ) // lookup to see if we have this one built already!
+		{
+		ctx = X509_STORE_new();
+		TableVal* root_certs2 = root_certs->AsTableVal();
+		ListVal* idxs = root_certs2->ConvertToPureList();
+
+		// Build the validation store
+		for ( i = 0; i < idxs->Length(); ++i )
+			{
+			Val* key = idxs->Index(i);
+			StringVal *sv = root_certs2->Lookup(key)->AsStringVal();
+			const uint8* data = sv->Bytes();
+			X509* x = d2i_X509_(NULL, &data, sv->Len());
+			if ( ! x )
+				{
+				builtin_error(fmt("Root CA error: %s", ERR_error_string(ERR_peek_last_error(),NULL)));
+				return new Val((uint64) ERR_get_error(), TYPE_COUNT);
+				}
+			X509_STORE_add_cert(ctx, x);
+			}
+		delete idxs;
+
+		// Save the newly constructed certificate store into the cacheing map.
+		x509_stores[root_certs] = ctx;
+		}
+
+	assert(cert_val);
+	file_analysis::X509Val* cert_handle = (file_analysis::X509Val*) cert_val;
+
+	X509* cert = cert_handle->GetCertificate();
+	if ( ! cert )
+		{
+		builtin_error(fmt("No certificate in opaque"));
+		return new Val(-1, TYPE_COUNT);
+		}
+
+	STACK_OF(X509)* untrusted_certs = sk_X509_new_null();
+	if ( ! untrusted_certs )
+		{
+		builtin_error(fmt("Untrusted certificate stack initialization error: %s", ERR_error_string(ERR_peek_last_error(),NULL)));
+		return new Val((uint64) ERR_get_error(), TYPE_COUNT);
+		}
+
+	VectorVal *cert_stack_vec = cert_stack->AsVectorVal();
+	for ( i = 0; i < (int) cert_stack_vec->Size(); ++i )
+		{
+		Val *sv = cert_stack_vec->Lookup(i);
+		// Fixme: check type
+		X509* x = ((file_analysis::X509Val*) sv)->GetCertificate();
+		if ( ! x )
+			{
+			sk_X509_pop(untrusted_certs);
+			builtin_error(fmt("No certificate in opaque in stack"));
+			return new Val(-1, TYPE_COUNT);
+			}
+		sk_X509_push(untrusted_certs, x);
+		}
+
+	X509_STORE_CTX csc;
+	X509_STORE_CTX_init(&csc, ctx, cert, untrusted_certs);
+	X509_STORE_CTX_set_time(&csc, 0, (time_t) network_time);
+
+	int result = X509_verify_cert(&csc);
+
+	VectorVal* chainVector = 0;
+		if ( result == 1 ) // we have a valid chain. try to get it...
+		{
+		STACK_OF(X509)* chain = X509_STORE_CTX_get1_chain(&csc); // get1 = deep copy
+
+		if (!chain) 
+			{
+			reporter->Error("Encountered valid chain that could not be resolved");
+			goto x509_verify_chainerror;
+			}
+		
+		int num_certs = sk_X509_num(chain);
+		chainVector = new VectorVal(new VectorType(base_type(TYPE_OPAQUE)));
+
+		for ( int i = 0; i < num_certs; i++ ) 
+			{
+			X509* currcert = sk_X509_value(chain, i);
+			if ( !currcert )
+				{
+				reporter->InternalError("OpenSSL returned null certificate");
+				goto x509_verify_chainerror;
+				}
+
+			chainVector->Assign(i, new file_analysis::X509Val(currcert)); // X509Val takes ownership
+			}
+		}
+
+x509_verify_chainerror:
+
+	X509_STORE_CTX_cleanup(&csc);
+
+	if ( untrusted_certs )
+		sk_X509_pop(untrusted_certs);
+
+	RecordVal* rrecord = new RecordVal(BifType::Record::X509::Result);
+
+	rrecord->Assign(0, new Val((uint64) csc.error, TYPE_COUNT));
+	rrecord->Assign(1, new StringVal(X509_verify_cert_error_string(csc.error)));
+	if ( chainVector ) 
+		rrecord->Assign(2, chainVector);
+
+	return rrecord;
+	%}

src/file_analysis/analyzer/x509/types.bif

@@ -1,5 +1,5 @@
 type X509::Certificate: record;
 type X509::Extension: record;
 type X509::BasicConstraints: record;
-type X509::SubjectAlternativeName: record;
+type X509::Result: record;