X509 file analyzer nearly done. Verification and most other policy scripts

work fine now.

Todo:
 * update all baselines
 * fix the circular reference to the fa_file structure I introduced :)
   Sadly this does not seem to be entirely straightforward.

addresses BIT-953, BIT-760

scripts/base/files/x509/README

@@ -0,0 +1 @@
+Support for X509 certificates with the file analysis framework.

scripts/base/files/x509/main.bro

@@ -1,32 +1,77 @@
-
 @load base/frameworks/files
+@load base/files/hash
 
 module X509;
 
 export {
 	redef enum Log::ID += { LOG };
 
-	redef record Files::Info += {
+	type Info: record {
+		## current timestamp
+		ts: time &log &default=network_time();
+
+		## file id of this certificate
+		id: string &log;
+    
+		## Basic information about the certificate
+		certificate: X509::Certificate &log;
+
+		## The opaque wrapping the certificate. Mainly used
+		## for the verify operations
+		handle: opaque of x509;
+
+		## All extensions that were encountered in the certificate
+		extensions: vector of X509::Extension &default=vector();
+
+		## Subject alternative name extension of the certificate
+		san: string_vec &optional &log;
+
+		## Basic constraints extension of the certificate
+		basic_constraints: X509::BasicConstraints &optional &log;
 	};
+
+	## Event for accessing logged records.
+	global log_x509: event(rec: Info);
 }
 
-event x509_cert(f: fa_file, cert_ref: opaque of x509, cert: X509::Certificate)
+event bro_init() &priority=5
 	{
-	print cert;
+	Log::create_stream(X509::LOG, [$columns=Info, $ev=log_x509]);
 	}
 
-event x509_extension(f: fa_file, cert_ref: opaque of x509, cert: X509::Certificate, ext: X509::Extension)
+redef record fa_file += {
+	## Information about X509 certificates. This is used to keep
+	## certificate information until all events have been received.
+	x509: X509::Info &optional;
+};
+
+event x509_certificate(f: fa_file, cert_ref: opaque of x509, cert: X509::Certificate) &priority=5
 	{
-print ext;
+	f$x509 = [$id=f$id, $certificate=cert, $handle=cert_ref];
 	}
 
-event x509_ext_basic_constraints(f: fa_file, cert_ref: opaque of x509, cert: X509::Certificate, ext: X509::BasicConstraints)
+event x509_extension(f: fa_file, ext: X509::Extension) &priority=5
 	{
-print ext;
+	if ( f?$x509 )
+		f$x509$extensions[|f$x509$extensions|] = ext;
 	}
 
-event x509_ext_subject_alternative_name(f: fa_file, cert_ref: opaque of x509, cert: X509::Certificate, ext: string_vec) 
+event x509_ext_basic_constraints(f: fa_file, ext: X509::BasicConstraints) &priority=5
 	{
-print ext;
+	if ( f?$x509 )
+		f$x509$basic_constraints = ext;
 	}
 
+event x509_ext_subject_alternative_name(f: fa_file, names: string_vec) &priority=5
+	{
+	if ( f?$x509 )
+		f$x509$san = names;
+	}
+
+event file_state_remove(f: fa_file)
+	{
+	if ( f?$x509 )
+		{
+		Log::write(LOG, f$x509);
+		}
+	}

scripts/base/init-bare.bro

@@ -2765,7 +2765,7 @@ export {
 		curve: string &optional; ##< curve, if EC-certificate
 		#ca: bool &optional; ##< indicates the CA value in the X509v3 BasicConstraints extension
 		#path_len: count &optional; ##< indicates the path_length value in the X509v3 BasicConstraints extension
-	};
+	} &log;
 
 	type X509::Extension: record {
 		name: string; ##< long name of extension. oid if name not known
@@ -2778,7 +2778,7 @@ export {
 	type X509::BasicConstraints: record {
 		ca: bool; ##< CA flag set?
 		path_len: count &optional;
-	};
+	} &log;
 	
 	## Result of an X509 certificate chain verification
 	type X509::Result: record {

scripts/base/init-default.bro

@@ -38,6 +38,9 @@
 @load base/frameworks/sumstats
 @load base/frameworks/tunnels
 
+# needed for the SSL protocol
+@load base/files/x509
+
 @load base/protocols/conn
 @load base/protocols/dhcp
 @load base/protocols/dnp3
@@ -57,7 +60,6 @@
 @load base/files/hash
 @load base/files/extract
 @load base/files/unified2
-@load base/files/x509
 
 @load base/misc/find-checksum-offloading
 @load base/misc/find-filtered-trace

scripts/base/protocols/ssl/files.bro

@@ -6,9 +6,33 @@ module SSL;
 
 export {
 	redef record Info += {
-		## An ordered vector of file unique IDs which contains
+		## Chain of certificates offered by the server to validate its
-		## all the certificates sent over the connection
+		## complete signing chain.
-		fuids: vector of string &log &default=string_vec();
+		cert_chain: vector of fa_file &optional;
+
+		## An ordered vector of all certicate file unique IDs for the
+		## certificates offered by the server.
+		cert_chain_fuids: vector of string &optional &log;
+
+		## Chain of certificates offered by the client to validate its
+		## complete signing chain.
+		client_cert_chain: vector of fa_file &optional;
+		
+		## An ordered vector of all certicate file unique IDs for the
+		## certificates offered by the client.
+		client_cert_chain_fuids: vector of string &optional &log;
+		
+		## Subject of the X.509 certificate offered by the server.
+		subject:          string           &log &optional;
+		## Subject of the signer of the X.509 certificate offered by the
+		## server.
+		issuer:   string           &log &optional;
+		
+		## Subject of the X.509 certificate offered by the client.
+		client_subject:          string           &log &optional;
+		## Subject of the signer of the X.509 certificate offered by the
+		## client.
+		client_issuer:   string           &log &optional;
 	};
 
 	## Default file handle provider for SSL.
@@ -20,7 +44,7 @@ export {
 
 function get_file_handle(c: connection, is_orig: bool): string
 	{
-	return cat(Analyzer::ANALYZER_SMTP, c$start_time);
+	return cat(Analyzer::ANALYZER_SSL, c$start_time);
 	}
 
 function describe_file(f: fa_file): string
@@ -29,6 +53,8 @@ function describe_file(f: fa_file): string
 	if ( f$source != "SSL" )
 		return "";
 
+	# Fixme!
+
 	return "";
 	}
 
@@ -41,8 +67,46 @@ event bro_init() &priority=5
 
 event file_over_new_connection(f: fa_file, c: connection, is_orig: bool) &priority=5
 	{
-	if ( c?$ssl )
+	if ( ! c?$ssl )
-		c$ssl$fuids[|c$ssl$fuids|] = f$id;
+		return;
+
+	if ( ! c$ssl?$cert_chain )
+		{
+		c$ssl$cert_chain = vector();
+		c$ssl$client_cert_chain = vector();
+		c$ssl$cert_chain_fuids = string_vec();
+		c$ssl$client_cert_chain_fuids = string_vec();
+		}	
+
+	if ( is_orig )
+		{
+		c$ssl$client_cert_chain[|c$ssl$client_cert_chain|] = f;
+		c$ssl$client_cert_chain_fuids[|c$ssl$client_cert_chain_fuids|] = f$id;
+		}
+	else
+		{
+		c$ssl$cert_chain[|c$ssl$cert_chain|] = f;
+		c$ssl$cert_chain_fuids[|c$ssl$cert_chain_fuids|] = f$id;
+		}
 
 	Files::add_analyzer(f, Files::ANALYZER_X509);
+	# always calculate hashes for certificates
+	Files::add_analyzer(f, Files::ANALYZER_MD5);
+	Files::add_analyzer(f, Files::ANALYZER_SHA1);
+	}
+
+event ssl_established(c: connection) &priority=6
+	{
+	# update subject and issuer information
+	if ( c$ssl?$cert_chain && |c$ssl$cert_chain| > 0 )
+		{
+		c$ssl$subject = c$ssl$cert_chain[0]$x509$certificate$subject;
+		c$ssl$issuer = c$ssl$cert_chain[0]$x509$certificate$issuer;
+		}
+
+	if ( c$ssl?$client_cert_chain && |c$ssl$client_cert_chain| > 0 )
+		{
+		c$ssl$client_subject = c$ssl$client_cert_chain[0]$x509$certificate$subject;
+		c$ssl$client_issuer = c$ssl$client_cert_chain[0]$x509$certificate$issuer;
+		}
 	}

scripts/base/protocols/ssl/main.bro

@@ -24,36 +24,9 @@ export {
 		server_name:      string           &log &optional;
 		## Session ID offered by the client for session resumption.
 		session_id:       string           &log &optional;
-		## Subject of the X.509 certificate offered by the server.
-		subject:          string           &log &optional;
-		## Subject of the signer of the X.509 certificate offered by the
-		## server.
-		issuer_subject:   string           &log &optional;
-		## NotValidBefore field value from the server certificate.
-		not_valid_before: time             &log &optional;
-		## NotValidAfter field value from the server certificate.
-		not_valid_after:  time             &log &optional;
 		## Last alert that was seen during the connection.
 		last_alert:       string           &log &optional;
 
-		## Subject of the X.509 certificate offered by the client.
-		client_subject:          string           &log &optional;
-		## Subject of the signer of the X.509 certificate offered by the
-		## client.
-		client_issuer_subject:   string           &log &optional;
-
-		## Full binary server certificate stored in DER format.
-		cert:             string           &optional;
-		## Chain of certificates offered by the server to validate its
-		## complete signing chain.
-		cert_chain:       vector of string &optional;
-
-		## Full binary client certificate stored in DER format.
-		client_cert:             string           &optional;
-		## Chain of certificates offered by the client to validate its
-		## complete signing chain.
-		client_cert_chain:       vector of string &optional;
-
 		## The analyzer ID used for the analyzer instance attached
 		## to each connection.  It is not used for logging since it's a
 		## meaningless arbitrary number.
@@ -108,8 +81,7 @@ event bro_init() &priority=5
 function set_session(c: connection)
 	{
 	if ( ! c?$ssl )
-		c$ssl = [$ts=network_time(), $uid=c$uid, $id=c$id, $cert_chain=vector(),
+		c$ssl = [$ts=network_time(), $uid=c$uid, $id=c$id];
-		         $client_cert_chain=vector()];
 	}
 
 function delay_log(info: Info, token: string)
@@ -185,7 +157,7 @@ event ssl_alert(c: connection, is_orig: bool, level: count, desc: count) &priori
 	c$ssl$last_alert = alert_descriptions[desc];
 	}
 
-event ssl_established(c: connection) &priority=5
+event ssl_established(c: connection) &priority=7
 	{
 	set_session(c);
 	}

scripts/policy/frameworks/intel/seen/__load__.bro

@@ -7,3 +7,4 @@
 @load ./ssl
 @load ./smtp
 @load ./smtp-url-extraction
+@load ./x509

scripts/policy/frameworks/intel/seen/ssl.bro

@@ -2,27 +2,6 @@
 @load base/protocols/ssl
 @load ./where-locations
 
-event x509_certificate(c: connection, is_orig: bool, cert: X509, chain_idx: count, chain_len: count, der_cert: string)
-	{
-	if ( chain_idx == 0 )
-		{
-		if ( /emailAddress=/ in cert$subject )
-			{
-			local email = sub(cert$subject, /^.*emailAddress=/, "");
-			email = sub(email, /,.*$/, "");
-			Intel::seen([$indicator=email,
-			             $indicator_type=Intel::EMAIL,
-			             $conn=c,
-			             $where=(is_orig ? SSL::IN_CLIENT_CERT : SSL::IN_SERVER_CERT)]);
-			}
-
-		Intel::seen([$indicator=sha1_hash(der_cert),
-		             $indicator_type=Intel::CERT_HASH,
-		             $conn=c,
-		             $where=(is_orig ? SSL::IN_CLIENT_CERT : SSL::IN_SERVER_CERT)]);
-		}
-	}
-
 event ssl_extension(c: connection, is_orig: bool, code: count, val: string)
 	{
 	if ( is_orig && SSL::extensions[code] == "server_name" && 

scripts/policy/frameworks/intel/seen/where-locations.bro

@@ -21,9 +21,8 @@ export {
 		SMTP::IN_REPLY_TO,
 		SMTP::IN_X_ORIGINATING_IP_HEADER,
 		SMTP::IN_MESSAGE,
-		SSL::IN_SERVER_CERT,
-		SSL::IN_CLIENT_CERT,
 		SSL::IN_SERVER_NAME,
 		SMTP::IN_HEADER,
+		X509::IN_CERT,
 	};
 }

scripts/policy/frameworks/intel/seen/x509.bro

@@ -0,0 +1,16 @@
+@load base/frameworks/intel
+@load base/files/x509
+@load ./where-locations
+
+event x509_certificate(f: fa_file, cert_ref: opaque of x509, cert: X509::Certificate)
+	{
+	if ( /emailAddress=/ in cert$subject )
+		{
+		local email = sub(cert$subject, /^.*emailAddress=/, "");
+		email = sub(email, /,.*$/, "");
+		Intel::seen([$indicator=email,
+								 $indicator_type=Intel::EMAIL,
+								 $f=f,
+								 $where=X509::IN_CERT]);
+		}
+	}

scripts/policy/protocols/ssl/cert-hash.bro

@@ -1,22 +0,0 @@
-##! Calculate MD5 sums for server DER formatted certificates.
-
-@load base/protocols/ssl
-
-module SSL;
-
-export {
-	redef record Info += {
-		## MD5 sum of the raw server certificate.
-		cert_hash: string &log &optional;
-	};
-}
-
-event x509_certificate(c: connection, is_orig: bool, cert: X509, chain_idx: count, chain_len: count, der_cert: string) &priority=4
-	{
-	# We aren't tracking client certificates yet and we are also only tracking
-	# the primary cert.  Watch that this came from an SSL analyzed session too.
-	if ( is_orig || chain_idx != 0 || ! c?$ssl ) 
-		return;
-
-	c$ssl$cert_hash = md5_hash(der_cert);
-	}

scripts/policy/protocols/ssl/expiring-certs.bro

@@ -3,10 +3,7 @@
 ##! certificate.
 
 @load base/protocols/ssl
-@load base/frameworks/notice
+@load base/files/x509
-@load base/utils/directions-and-hosts
-
-@load protocols/ssl/cert-hash
 
 module SSL;
 
@@ -35,30 +32,31 @@ export {
 	const notify_when_cert_expiring_in = 30days &redef;
 }
 
-event x509_certificate(c: connection, is_orig: bool, cert: X509, chain_idx: count, chain_len: count, der_cert: string) &priority=3
+event ssl_established(c: connection) &priority=3
 	{
-	# If this isn't the host cert or we aren't interested in the server, just return.
+	# If there are no certificates or we are not interested in the server, just return.
-	if ( is_orig || 
+	if ( ! c$ssl?$cert_chain || |c$ssl$cert_chain| == 0 ||
-		 chain_idx != 0 ||
-		 ! c$ssl?$cert_hash || 
 		 ! addr_matches_host(c$id$resp_h, notify_certs_expiration) )
 		return;
 
+	local hash = c$ssl$cert_chain[0]$info$md5;
+	local cert = c$ssl$cert_chain[0]$x509$certificate;
+	
 	if ( cert$not_valid_before > network_time() )
 		NOTICE([$note=Certificate_Not_Valid_Yet,
 		        $conn=c, $suppress_for=1day,
 		        $msg=fmt("Certificate %s isn't valid until %T", cert$subject, cert$not_valid_before),
-		        $identifier=cat(c$id$resp_h, c$id$resp_p, c$ssl$cert_hash)]);
+		        $identifier=cat(c$id$resp_h, c$id$resp_p, hash)]);
 	
 	else if ( cert$not_valid_after < network_time() )
 		NOTICE([$note=Certificate_Expired,
 		        $conn=c, $suppress_for=1day,
 		        $msg=fmt("Certificate %s expired at %T", cert$subject, cert$not_valid_after),
-		        $identifier=cat(c$id$resp_h, c$id$resp_p, c$ssl$cert_hash)]);
+		        $identifier=cat(c$id$resp_h, c$id$resp_p, hash)]);
 	
 	else if ( cert$not_valid_after - notify_when_cert_expiring_in < network_time() )
 		NOTICE([$note=Certificate_Expires_Soon,
 		        $msg=fmt("Certificate %s is going to expire at %T", cert$subject, cert$not_valid_after),
 		        $conn=c, $suppress_for=1day,
-		        $identifier=cat(c$id$resp_h, c$id$resp_p, c$ssl$cert_hash)]);
+		        $identifier=cat(c$id$resp_h, c$id$resp_p, hash)]);
 	}

scripts/policy/protocols/ssl/extract-certs-pem.bro

@@ -10,8 +10,7 @@
 ##!
 
 @load base/protocols/ssl
-@load base/utils/directions-and-hosts
+@load base/files/x509
-@load protocols/ssl/cert-hash
 
 module SSL;
 
@@ -23,41 +22,31 @@ export {
 }
 
 # This is an internally maintained variable to prevent relogging of
-# certificates that have already been seen.  It is indexed on an md5 sum of
+# certificates that have already been seen.  It is indexed on an sha1 sum of
 # the certificate.
 global extracted_certs: set[string] = set() &read_expire=1hr &redef;
 
 event ssl_established(c: connection) &priority=5
 	{
-	if ( ! c$ssl?$cert )
+	if ( ! c$ssl?$cert_chain || |c$ssl$cert_chain| == 0 )
 		return;
 
 	if ( ! addr_matches_host(c$id$resp_h, extract_certs_pem) )
 		return;
 
-	if ( c$ssl$cert_hash in extracted_certs )
+	local hash = c$ssl$cert_chain[0]$info$sha1;
+	local cert = c$ssl$cert_chain[0]$x509$handle;
+
+	if ( hash in extracted_certs )
 		# If we already extracted this cert, don't do it again.
 		return;
 
-	add extracted_certs[c$ssl$cert_hash];
+	add extracted_certs[hash];
 	local filename = Site::is_local_addr(c$id$resp_h) ? "certs-local.pem" : "certs-remote.pem";
 	local outfile = open_for_append(filename);
+	enable_raw_output(outfile);
 
-	print outfile, "-----BEGIN CERTIFICATE-----";
+	print outfile, x509_get_certificate_string(cert, T);
 
-	# Encode to base64 and format to fit 50 lines. Otherwise openssl won't like it later.
-	local lines = split_all(encode_base64(c$ssl$cert), /.{50}/);
-	local i = 1;
-	for ( line in lines )
-		{
-		if ( |lines[i]| > 0 )
-			{
-			print outfile, lines[i];
-			}
-		i+=1;
-		}
-
-	print outfile, "-----END CERTIFICATE-----";
-	print outfile, "";
 	close(outfile);
 	}

scripts/policy/protocols/ssl/known-certs.bro

@@ -3,7 +3,7 @@
 
 @load base/utils/directions-and-hosts
 @load base/protocols/ssl
-@load protocols/ssl/cert-hash
+@load base/files/x509
 
 module Known;
 
@@ -33,7 +33,7 @@ export {
 	## The set of all known certificates to store for preventing duplicate 
 	## logging. It can also be used from other scripts to 
 	## inspect if a certificate has been seen in use. The string value 
-	## in the set is for storing the DER formatted certificate's MD5 hash.
+	## in the set is for storing the DER formatted certificate' SHA1 hash.
 	global certs: set[addr, string] &create_expire=1day &synchronized &redef;
 	
 	## Event that can be handled to access the loggable record as it is sent
@@ -46,16 +46,18 @@ event bro_init() &priority=5
 	Log::create_stream(Known::CERTS_LOG, [$columns=CertsInfo, $ev=log_known_certs]);
 	}
 
-event x509_certificate(c: connection, is_orig: bool, cert: X509, chain_idx: count, chain_len: count, der_cert: string) &priority=3
+event ssl_established(c: connection) &priority=3
 	{
-	# Make sure this is the server cert and we have a hash for it.
+	if ( ! c$ssl?$cert_chain || |c$ssl$cert_chain| < 1 )
-	if ( is_orig || chain_idx != 0 || ! c$ssl?$cert_hash ) 
 		return;
 
+	local hash = c$ssl$cert_chain[0]$info$sha1;
+	local cert = c$ssl$cert_chain[0]$x509$certificate;
+
 	local host = c$id$resp_h;
-	if ( [host, c$ssl$cert_hash] !in certs && addr_matches_host(host, cert_tracking) )
+	if ( [host, hash] !in certs && addr_matches_host(host, cert_tracking) )
 		{
-		add certs[host, c$ssl$cert_hash];
+		add certs[host, hash];
 		Log::write(Known::CERTS_LOG, [$ts=network_time(), $host=host,
 		                              $port_num=c$id$resp_p, $subject=cert$subject,
 		                              $issuer_subject=cert$issuer,

scripts/policy/protocols/ssl/notary.bro

@@ -16,7 +16,6 @@ export {
 }
 
 redef record SSL::Info += {
-	sha1:   string &log &optional;
 	notary: Response &log &optional;
 	};
 
@@ -38,14 +37,12 @@ function clear_waitlist(digest: string)
 		}
 	}
 
-event x509_certificate(c: connection, is_orig: bool, cert: X509,
+event ssl_established(c: connection) &priority=3
-                       chain_idx: count, chain_len: count, der_cert: string)
 	{
-	if ( is_orig || chain_idx != 0 || ! c?$ssl )
+	if ( ! c$ssl?$cert_chain || |c$ssl$cert_chain| == 0 )
 		return;
 
-	local digest = sha1_hash(der_cert);
+	local digest = c$ssl$cert_chain[0]$info$sha1;
-	c$ssl$sha1 = digest;
 
 	if ( digest in notary_cache )
 		{

scripts/policy/protocols/ssl/validate-certs.bro

@@ -2,7 +2,6 @@
 
 @load base/frameworks/notice
 @load base/protocols/ssl
-@load protocols/ssl/cert-hash
 
 module SSL;
 
@@ -19,9 +18,9 @@ export {
 		validation_status: string &log &optional;
 	};
 	
-	## MD5 hash values for recently validated certs along with the
+	## MD5 hash values for recently validated chains along with the
 	## validation status message are kept in this table to avoid constant
-	## validation every time the same certificate is seen.
+	## validation every time the same certificate chain is seen.
 	global recently_validated_certs: table[string] of string = table() 
 		&read_expire=5mins &synchronized &redef;
 }
@@ -29,18 +28,26 @@ export {
 event ssl_established(c: connection) &priority=3
 	{
 	# If there aren't any certs we can't very well do certificate validation.
-	if ( ! c$ssl?$cert || ! c$ssl?$cert_chain )
+	if ( ! c$ssl?$cert_chain || |c$ssl$cert_chain| == 0 )
 		return;
 
-	if ( c$ssl?$cert_hash && c$ssl$cert_hash in recently_validated_certs )
+	local chain_id = join_string_vec(c$ssl$cert_chain_fuids, ".");
+
+	local chain: vector of opaque of x509 = vector();
+	for ( i in c$ssl$cert_chain )
 		{
-		c$ssl$validation_status = recently_validated_certs[c$ssl$cert_hash];
+		chain[i] = c$ssl$cert_chain[i]$x509$handle;
+		}
+
+	if ( chain_id in recently_validated_certs )
+		{
+		c$ssl$validation_status = recently_validated_certs[chain_id];
 		}
 	else
 		{
-		local result = x509_verify(c$ssl$cert, c$ssl$cert_chain, root_certs);
+		local result = x509_verify(chain, root_certs);
-		c$ssl$validation_status = x509_err2str(result);
+		c$ssl$validation_status = result$result_string;
-		recently_validated_certs[c$ssl$cert_hash] = c$ssl$validation_status;
+		recently_validated_certs[chain_id] = result$result_string;
 		}
 		
 	if ( c$ssl$validation_status != "ok" )
@@ -48,7 +55,7 @@ event ssl_established(c: connection) &priority=3
 		local message = fmt("SSL certificate validation failed with (%s)", c$ssl$validation_status);
 		NOTICE([$note=Invalid_Server_Cert, $msg=message,
 		        $sub=c$ssl$subject, $conn=c,
-		        $identifier=cat(c$id$resp_h,c$id$resp_p,c$ssl$validation_status,c$ssl$cert_hash)]);
+		        $identifier=cat(c$id$resp_h,c$id$resp_p,c$ssl$validation_status)]);
 		}
 	}
 

src/file_analysis/analyzer/x509/X509.cc

@@ -59,7 +59,7 @@ bool file_analysis::X509::EndOfFile()
 	vl->append(cert_val->Ref());
 	vl->append(cert_record->Ref()); // we Ref it here, because we want to keep a copy around for now...
 
-	mgr.QueueEvent(x509_cert, vl);
+	mgr.QueueEvent(x509_certificate, vl);
 
 	// after parsing the certificate - parse the extensions...
 
@@ -70,7 +70,7 @@ bool file_analysis::X509::EndOfFile()
 		if ( !ex )
 			continue;
 
-		ParseExtension(ex, cert_record, cert_val);
+		ParseExtension(ex);
 		}
 	
 	// X509_free(ssl_cert); We do _not_ free the certificate here. It is refcounted
@@ -157,7 +157,7 @@ RecordVal* file_analysis::X509::ParseCertificate(X509Val* cert_val)
 	return pX509Cert;
 	}
 
-void file_analysis::X509::ParseExtension(X509_EXTENSION* ex, RecordVal* r, X509Val* cert_val) 
+void file_analysis::X509::ParseExtension(X509_EXTENSION* ex) 
 	{
 	char name[256];
 	char oid[256];
@@ -203,20 +203,18 @@ void file_analysis::X509::ParseExtension(X509_EXTENSION* ex, RecordVal* r, X509V
 	// but I am not sure if there is a better way to do it...
 	val_list* vl = new val_list();
 	vl->append(GetFile()->GetVal()->Ref());
-	vl->append(cert_val->Ref());
-	vl->append(r->Ref());
 	vl->append(pX509Ext);
 
 	mgr.QueueEvent(x509_extension, vl);
 
 	// look if we have a specialized handler for this event...
 	if ( OBJ_obj2nid(ext_asn) == NID_basic_constraints ) 
-		ParseBasicConstraints(ex, r, cert_val);
+		ParseBasicConstraints(ex);
 	else if ( OBJ_obj2nid(ext_asn) == NID_subject_alt_name )
-		ParseSAN(ex, r, cert_val);
+		ParseSAN(ex);
 	}
 
-void file_analysis::X509::ParseBasicConstraints(X509_EXTENSION* ex, RecordVal* r, X509Val* cert_val) 
+void file_analysis::X509::ParseBasicConstraints(X509_EXTENSION* ex) 
 	{
 	assert(OBJ_obj2nid(X509_EXTENSION_get_object(ex)) == NID_basic_constraints);
 	
@@ -234,8 +232,6 @@ void file_analysis::X509::ParseBasicConstraints(X509_EXTENSION* ex, RecordVal* r
 		}
 		val_list* vl = new val_list();
 		vl->append(GetFile()->GetVal()->Ref());
-		vl->append(cert_val->Ref());
-		vl->append(r->Ref());
 		vl->append(pBasicConstraint);
 
 		mgr.QueueEvent(x509_ext_basic_constraints, vl);
@@ -243,7 +239,7 @@ void file_analysis::X509::ParseBasicConstraints(X509_EXTENSION* ex, RecordVal* r
 		}
 	}
 
-void file_analysis::X509::ParseSAN(X509_EXTENSION* ext, RecordVal* r, X509Val* cert_val) 
+void file_analysis::X509::ParseSAN(X509_EXTENSION* ext) 
 	{
 	assert(OBJ_obj2nid(X509_EXTENSION_get_object(ext)) == NID_subject_alt_name);
 
@@ -284,11 +280,9 @@ void file_analysis::X509::ParseSAN(X509_EXTENSION* ext, RecordVal* r, X509Val* c
 
 		val_list* vl = new val_list();
 		vl->append(GetFile()->GetVal()->Ref());
-		vl->append(cert_val->Ref());
-		vl->append(r->Ref());
 		vl->append(names);
 
-		mgr.QueueEvent(x509_ext_basic_constraints, vl);
+		mgr.QueueEvent(x509_ext_subject_alternative_name, vl);
 	}
 
 StringVal* file_analysis::X509::key_curve(EVP_PKEY *key)

src/file_analysis/analyzer/x509/X509.h

@@ -35,9 +35,9 @@ private:
 	static StringVal* key_curve(EVP_PKEY *key);
 	static unsigned int key_length(EVP_PKEY *key);
 
-	void ParseExtension(X509_EXTENSION* ex, RecordVal* r, X509Val* cert_val);
+	void ParseExtension(X509_EXTENSION* ex);
-	void ParseBasicConstraints(X509_EXTENSION* ex, RecordVal* r, X509Val* cert_val);
+	void ParseBasicConstraints(X509_EXTENSION* ex);
-	void ParseSAN(X509_EXTENSION* ex, RecordVal* r, X509Val* cert_val);
+	void ParseSAN(X509_EXTENSION* ex);
 
 	std::string cert_data;
 };

src/file_analysis/analyzer/x509/events.bif

@@ -1,4 +1,4 @@
-event x509_cert%(f: fa_file, cert_ref: opaque of x509, cert: X509::Certificate%);
+event x509_certificate%(f: fa_file, cert_ref: opaque of x509, cert: X509::Certificate%);
-event x509_extension%(f: fa_file, cert_ref: opaque of x509, cert: X509::Certificate, ext: X509::Extension%);
+event x509_extension%(f: fa_file, ext: X509::Extension%);
-event x509_ext_basic_constraints%(f: fa_file, cert_ref: opaque of x509, cert: X509::Certificate, ext: X509::BasicConstraints%);
+event x509_ext_basic_constraints%(f: fa_file, ext: X509::BasicConstraints%);
-event x509_ext_subject_alternative_name%(f: fa_file, cert_ref: opaque of x509, cert: X509::Certificate, names: string_vec%);
+event x509_ext_subject_alternative_name%(f: fa_file, names: string_vec%);

src/file_analysis/analyzer/x509/functions.bif

@@ -23,11 +23,22 @@ X509* d2i_X509_(X509** px, const u_char** in, int len)
 #endif
 	}
 
+// construct an error record
+RecordVal* x509_error_record(uint64_t num, const char* reason)
+	{
+	RecordVal* rrecord = new RecordVal(BifType::Record::X509::Result);
+
+	rrecord->Assign(0, new Val(num, TYPE_COUNT));
+	rrecord->Assign(1, new StringVal(reason));
+
+	return rrecord;
+	}
+
 %%}
 
 ## Parses a certificate into an X509::Certificate structure
 ##
-## cert: The x509 certificicate opaque
+## cert: The X509 certificicate opaque handle
 ##
 ## Returns: A X509::Certificate structure
 ##
@@ -40,12 +51,50 @@ function x509_parse%(cert: opaque of x509%): X509::Certificate
 	return file_analysis::X509::ParseCertificate(h);
 	%}
 
+## Returns the string form of a certificate
+##
+## cert: The X509 certificate opaque handle
+##
+## pem: A boolean that specifies if the certificate is returned
+##      in pem-form (true), or as the raw ASN1 encoded binary
+##      (false).
+##
+## Returns: X509 certificate as a string
+
+function x509_get_certificate_string%(cert: opaque of x509, pem: bool &default=F%): string
+	%{
+	assert(cert);
+	file_analysis::X509Val* h = (file_analysis::X509Val*) cert;
+
+	BIO *bio = BIO_new(BIO_s_mem());
+	
+	if ( pem )
+		{
+		PEM_write_bio_X509(bio, h->GetCertificate());
+		}
+	else
+		{
+		i2d_X509_bio(bio, h->GetCertificate());
+		}
+	
+	BIO_flush(bio);
+	int length = BIO_pending(bio);
+	// use OPENSS_malloc here. Otherwhise, interesting problems will happen
+	char *buffer = (char*) OPENSSL_malloc(length);
+	BIO_read(bio, (void*) buffer, length);
+	StringVal* ext_val = new StringVal(length, buffer);
+	OPENSSL_free(buffer);
+	BIO_free_all(bio);
+			
+	return ext_val;
+	%}
+
+
 ## Verifies a certificate.
 ##
-## cert_val: The X.509 certificate in DER format.
+## certs: Specifies a certificate chain that is being used to validate
-##
+##        the given certificate against the root store given in *root_certs*.
-## cert_stack: Specifies a certificate chain that is being used to validate
+##        The host certificate has to be at index 0.
-##             the given certificate against the root store given in *root_certs*
 ##
 ## root_certs: A list of root certificates to validate the certificate chain
 ##
@@ -53,11 +102,28 @@ function x509_parse%(cert: opaque of x509%): X509::Certificate
 ##          operation. In case of success also returns the full certificate chain.
 ##
 ## .. bro:see:: x509_parse
-function x509_verify%(cert_val: opaque of x509, cert_stack: x509_opaque_vector, root_certs: table_string_of_string%): X509::Result
+function x509_verify%(certs: x509_opaque_vector, root_certs: table_string_of_string%): X509::Result
 	%{
 	X509_STORE* ctx = 0;
 	int i = 0;
 	
+	VectorVal *certs_vec = certs->AsVectorVal();
+	if ( certs_vec->Size() < 1 )
+		{
+		reporter->Error("No certificates given in vector");
+		return x509_error_record(-1, "no certificates");
+		}
+
+	// host certificate
+	unsigned int index = 0; // to prevent overloading to 0pointer
+	Val *sv = certs_vec->Lookup(index);
+	if ( !sv )
+		{
+		builtin_error("undefined value in certificate vector");
+		return x509_error_record(-1, "undefined value in certificate vector");
+		}
+	file_analysis::X509Val* cert_handle = (file_analysis::X509Val*) sv;	
+
 	// If this certificate store was built previously, just reuse the old one.
 	if ( x509_stores.count(root_certs) > 0 )
 		ctx = x509_stores[root_certs];
@@ -78,7 +144,7 @@ function x509_verify%(cert_val: opaque of x509, cert_stack: x509_opaque_vector,
 			if ( ! x )
 				{
 				builtin_error(fmt("Root CA error: %s", ERR_error_string(ERR_peek_last_error(),NULL)));
-				return new Val((uint64) ERR_get_error(), TYPE_COUNT);
+				return x509_error_record((uint64) ERR_get_error(), ERR_error_string(ERR_peek_last_error(),NULL));
 				}
 			X509_STORE_add_cert(ctx, x);
 			}
@@ -88,34 +154,30 @@ function x509_verify%(cert_val: opaque of x509, cert_stack: x509_opaque_vector,
 		x509_stores[root_certs] = ctx;
 		}
 
-	assert(cert_val);
-	file_analysis::X509Val* cert_handle = (file_analysis::X509Val*) cert_val;
-
 	X509* cert = cert_handle->GetCertificate();
 	if ( ! cert )
 		{
 		builtin_error(fmt("No certificate in opaque"));
-		return new Val(-1, TYPE_COUNT);
+		return x509_error_record(-1, "No certificate in opaque");
 		}
 
 	STACK_OF(X509)* untrusted_certs = sk_X509_new_null();
 	if ( ! untrusted_certs )
 		{
 		builtin_error(fmt("Untrusted certificate stack initialization error: %s", ERR_error_string(ERR_peek_last_error(),NULL)));
-		return new Val((uint64) ERR_get_error(), TYPE_COUNT);
+		return x509_error_record((uint64) ERR_get_error(), ERR_error_string(ERR_peek_last_error(),NULL));		
 		}
 
-	VectorVal *cert_stack_vec = cert_stack->AsVectorVal();
+	for ( i = 1; i < (int) certs_vec->Size(); ++i ) // start at 1 - 0 is host cert
-	for ( i = 0; i < (int) cert_stack_vec->Size(); ++i )
 		{
-		Val *sv = cert_stack_vec->Lookup(i);
+		Val *sv = certs_vec->Lookup(i);
 		// Fixme: check type
 		X509* x = ((file_analysis::X509Val*) sv)->GetCertificate();
 		if ( ! x )
 			{
 			sk_X509_pop(untrusted_certs);
 			builtin_error(fmt("No certificate in opaque in stack"));
-			return new Val(-1, TYPE_COUNT);
+			return x509_error_record(-1, "No certificate in opaque");
 			}
 		sk_X509_push(untrusted_certs, x);
 		}