Change x509 log - now certificates are only logged once per hour.

Add parsing of several more types to SAN extension.

Make error messages of x509 file analyzer more useful.

Fix file ID generation.

You apparently have to be very careful which EndOfFile function of
the file analysis framework you call... otherwhise it might try
to close another file id. This took me quite a while to find.

addresses BIT-953, BIT-760, BIT-1150

scripts/base/files/x509/main.bro

@@ -6,12 +6,15 @@ module X509;
 export {
 	redef enum Log::ID += { LOG };
 
+	## Set that keeps track of the certificates which were logged recently.
+	global cert_hashes: set[string] &create_expire=1hrs &synchronized &redef;
+
 	type Info: record {
 		## current timestamp
-		ts: time &log &default=network_time();
+		ts: time &log;
 
-		## file id of this certificate
-		id: string &log;
+		## SHA-1 hash of this certificate
+		sha1: string &log &optional;
     
 		## Basic information about the certificate
 		certificate: X509::Certificate &log;
@@ -24,7 +27,7 @@ export {
 		extensions: vector of X509::Extension &default=vector();
 
 		## Subject alternative name extension of the certificate
-		san: string_vec &optional &log;
+		san: X509::SubjectAlternativeName &optional &log;
 
 		## Basic constraints extension of the certificate
 		basic_constraints: X509::BasicConstraints &optional &log;
@@ -45,9 +48,20 @@ redef record Files::Info += {
 	x509: X509::Info &optional;
 };
 
+# Either, this event arrives first - then info$x509 does not exist 
+# yet and this is a no-op, and the sha1 value is set in x509_certificate.
+# Or the x509_certificate event arrives first - then the hash is set here.
+event file_hash(f: fa_file, kind: string, hash: string)
+	{
+	if ( f$info?$x509 && kind == "sha1" )
+		f$info$x509$sha1 = hash;
+	}
+
 event x509_certificate(f: fa_file, cert_ref: opaque of x509, cert: X509::Certificate) &priority=5
 	{
-	f$info$x509 = [$id=f$id, $certificate=cert, $handle=cert_ref];
+	f$info$x509 = [$ts=f$info$ts, $certificate=cert, $handle=cert_ref];
+	if ( f$info?$sha1 )
+		f$info$x509$sha1 = f$info$sha1;
 	}
 
 event x509_extension(f: fa_file, ext: X509::Extension) &priority=5
@@ -62,10 +76,10 @@ event x509_ext_basic_constraints(f: fa_file, ext: X509::BasicConstraints) &prior
 		f$info$x509$basic_constraints = ext;
 	}
 
-event x509_ext_subject_alternative_name(f: fa_file, names: string_vec) &priority=5
+event x509_ext_subject_alternative_name(f: fa_file, ext: X509::SubjectAlternativeName) &priority=5
 	{
 	if ( f$info?$x509 )
-		f$info$x509$san = names;
+		f$info$x509$san = ext;
 	}
 
 event file_state_remove(f: fa_file) &priority=5
@@ -73,5 +87,17 @@ event file_state_remove(f: fa_file) &priority=5
 	if ( ! f$info?$x509 )
 		return;
 
+	if ( ! f$info$x509?$sha1 )
+		{
+		Reporter::error(fmt("Certificate without a hash value. Logging skipped. File-id: %s", f$id));
+		return;
+		}
+
+	if ( f$info$x509$sha1 in cert_hashes )
+		# we already have seen & logged this certificate
+		return;
+
+	add cert_hashes[f$info$x509$sha1];
+
 	Log::write(LOG, f$info$x509);
 	}

scripts/base/init-bare.bro

@@ -2750,7 +2750,7 @@ export {
 
 module X509;
 export {
-	type X509::Certificate: record {
+	type Certificate: record {
 		version: count;	##< Version number.
 		serial: string;	##< Serial number.
 		subject: string;	##< Subject.
@@ -2767,7 +2767,7 @@ export {
 		#path_len: count &optional; ##< indicates the path_length value in the X509v3 BasicConstraints extension
 	} &log;
 
-	type X509::Extension: record {
+	type Extension: record {
 		name: string; ##< long name of extension. oid if name not known
 		short_name: string &optional; ##< short name of extension if known.
 		oid: string; ##< oid of extension
@@ -2775,13 +2775,21 @@ export {
 		value: string; ##< extension content parsed to string for known extensions. Raw data otherwise.
 	};
 
-	type X509::BasicConstraints: record {
+	type BasicConstraints: record {
 		ca: bool; ##< CA flag set?
-		path_len: count &optional;
+		path_len: count &optional; ##< maximum path length
 	} &log;
+
+	type SubjectAlternativeName: record {
+		dns: string_vec &optional &log; ##< list of DNS entries in SAN
+		uri: string_vec &optional &log; ##< list of URI entries in SAN
+		email: string_vec &optional &log; ##< list of email entries in SAN
+		ip: addr_vec &optional &log; ##< list of IP entries in SAN
+		other_fields: bool; ##< true if the certificate contained other, not recognized or parsed name fields
+	};
 	
 	## Result of an X509 certificate chain verification
-	type X509::Result: record {
+	type Result: record {
 		## OpenSSL result code
 		result:	count;
 		## Result as string

scripts/base/protocols/ssl/files.bro

@@ -11,29 +11,34 @@ export {
 		## complete signing chain.
 		cert_chain: vector of Files::Info &optional;
 
-		## An ordered vector of all certicate file unique IDs for the
+		## An ordered vector of all certicate sha1 hashes for the
 		## certificates offered by the server.
-		cert_chain_fuids: vector of string &optional &log;
+		cert_chain_sha1s: vector of string &optional &log;
 
 		## Chain of certificates offered by the client to validate its
 		## complete signing chain.
 		client_cert_chain: vector of Files::Info &optional;
 		
-		## An ordered vector of all certicate file unique IDs for the
+		## An ordered vector of all certicate sha1 hashes for the
 		## certificates offered by the client.
-		client_cert_chain_fuids: vector of string &optional &log;
+		client_cert_chain_sha1s: vector of string &optional &log;
 		
 		## Subject of the X.509 certificate offered by the server.
-		subject:          string           &log &optional;
+		subject: string &log &optional;
 		## Subject of the signer of the X.509 certificate offered by the
 		## server.
-		issuer:   string           &log &optional;
+		issuer: string &log &optional;
 		
 		## Subject of the X.509 certificate offered by the client.
-		client_subject:          string           &log &optional;
+		client_subject: string &log &optional;
 		## Subject of the signer of the X.509 certificate offered by the
 		## client.
-		client_issuer:   string           &log &optional;
+		client_issuer: string &log &optional;
+
+		## current number of certificates seen from either side. Used
+		## to create file handles
+		server_depth: count &default=0;
+		client_depth: count &default=0;
 	};
 
 	## Default file handle provider for SSL.
@@ -45,7 +50,22 @@ export {
 
 function get_file_handle(c: connection, is_orig: bool): string
 	{
-	return cat(Analyzer::ANALYZER_SSL, c$start_time);
+	set_session(c);
+
+	local depth: count;
+
+	if ( is_orig )
+		{
+		depth = c$ssl$client_depth;
+		++c$ssl$client_depth;
+		}
+	else
+		{
+		depth = c$ssl$server_depth;
+		++c$ssl$server_depth;
+		}
+
+	return cat(Analyzer::ANALYZER_SSL, c$start_time, is_orig, id_string(c$id), depth);
 	}
 
 function describe_file(f: fa_file): string
@@ -54,7 +74,19 @@ function describe_file(f: fa_file): string
 	if ( f$source != "SSL" )
 		return "";
 
-	# Fixme!
+	# It is difficult to reliably describe a certificate - especially since
+	# we do not know when this function is called (hence, if the data structures
+	# are already populated).
+	#
+	# Just return a bit of our connection information and hope that that is good enough.
+	for ( cid in f$conns )
+		{
+		if ( f$conns[cid]?$ssl )
+			{
+			local c = f$conns[cid];
+			return cat(c$id$resp_h, ":", c$id$resp_p);
+			}
+		}
 
 	return "";
 	}
@@ -75,30 +107,22 @@ event file_over_new_connection(f: fa_file, c: connection, is_orig: bool) &priori
 		{
 		c$ssl$cert_chain = vector();
 		c$ssl$client_cert_chain = vector();
-		c$ssl$cert_chain_fuids = string_vec();
-		c$ssl$client_cert_chain_fuids = string_vec();
 		}	
 
 	if ( is_orig )
-		{
 		c$ssl$client_cert_chain[|c$ssl$client_cert_chain|] = f$info;
-		c$ssl$client_cert_chain_fuids[|c$ssl$client_cert_chain_fuids|] = f$id;
-		}
 	else
-		{
 		c$ssl$cert_chain[|c$ssl$cert_chain|] = f$info;
-		c$ssl$cert_chain_fuids[|c$ssl$cert_chain_fuids|] = f$id;
-		}
 
 	Files::add_analyzer(f, Files::ANALYZER_X509);
-	# always calculate hashes for certificates
+	# always calculate hashes. SHA1 is always required for certificates.
 	Files::add_analyzer(f, Files::ANALYZER_MD5);
 	Files::add_analyzer(f, Files::ANALYZER_SHA1);
 	}
 
 event ssl_established(c: connection) &priority=6
 	{
-	# update subject and issuer information
+	# update subject and issuer information as well as sha1 hashes
 	if ( c$ssl?$cert_chain && |c$ssl$cert_chain| > 0 )
 		{
 		c$ssl$subject = c$ssl$cert_chain[0]$x509$certificate$subject;
@@ -110,4 +134,19 @@ event ssl_established(c: connection) &priority=6
 		c$ssl$client_subject = c$ssl$client_cert_chain[0]$x509$certificate$subject;
 		c$ssl$client_issuer = c$ssl$client_cert_chain[0]$x509$certificate$issuer;
 		}
+
+
+	if ( c$ssl?$cert_chain )
+		{
+		c$ssl$cert_chain_sha1s = string_vec();
+		for ( i in c$ssl$cert_chain )
+			c$ssl$cert_chain_sha1s[i] = c$ssl$cert_chain[i]$x509$sha1;
+		}
+
+	if ( c$ssl?$client_cert_chain )
+		{
+		c$ssl$client_cert_chain_sha1s = string_vec();
+		for ( i in c$ssl$client_cert_chain )
+			c$ssl$client_cert_chain_sha1s[i] = c$ssl$client_cert_chain[i]$x509$sha1;
+		}
 	}