Merge remote-tracking branch 'origin/topic/bernhard/file-analysis-x509'

* origin/topic/bernhard/file-analysis-x509:
  Forgot the preamble for the new leak test
  (hopefully) last change -> return real opaque vec instead of any_vec
  Fix dump-events - it cannot be used with ssl anymore, because openssl does not give the same string results in all versions.
  Finishing touches of the x509 file analyzer.
  Revert change to only log certificates once per hour.
  Change x509 log - now certificates are only logged once per hour.
  Fix circular reference problem and a few other small things.
  X509 file analyzer nearly done. Verification and most other policy scripts work fine now.
  Add verify functionality, including the ability to get the validated chain. This means that it is now possible to get information about the root-certificates that were used to secure a connection.
  Second try on the event interface.
  Backport crash fix that made it into master with the x509_extension backport from here.
  Make x509 certificates an opaque type
  rip out x509 code from ssl analyzer. Note that since at the moment the file analyzer does not yet re-populate the info record that means quite a lot of information is simply not available.
  parse out extension. One event for general extensions (just returns the openssl-parsed string-value), one event for basicconstraints (is a certificate a CA or not) and one event for subject-alternative-names (only DNS parts).
  Very basic file-analyzer for x509 certificates. Mostly ripped from the ssl-analyzer and the topic/bernhard/x509 branch.

scripts/base/protocols/ssl/__load__.bro

@@ -1,5 +1,6 @@
 @load ./consts
 @load ./main
 @load ./mozilla-ca-list
+@load ./files
 
 @load-sigs ./dpd.sig

scripts/base/protocols/ssl/files.bro

@@ -0,0 +1,149 @@
+@load ./main
+@load base/utils/conn-ids
+@load base/frameworks/files
+@load base/files/x509
+
+module SSL;
+
+export {
+	redef record Info += {
+		## Chain of certificates offered by the server to validate its
+		## complete signing chain.
+		cert_chain: vector of Files::Info &optional;
+
+		## An ordered vector of all certicate file unique IDs for the
+		## certificates offered by the server.
+		cert_chain_fuids: vector of string &optional &log;
+
+		## Chain of certificates offered by the client to validate its
+		## complete signing chain.
+		client_cert_chain: vector of Files::Info &optional;
+
+		## An ordered vector of all certicate file unique IDs for the
+		## certificates offered by the client.
+		client_cert_chain_fuids: vector of string &optional &log;
+
+		## Subject of the X.509 certificate offered by the server.
+		subject: string &log &optional;
+
+		## Subject of the signer of the X.509 certificate offered by the
+		## server.
+		issuer: string &log &optional;
+
+		## Subject of the X.509 certificate offered by the client.
+		client_subject: string &log &optional;
+
+		## Subject of the signer of the X.509 certificate offered by the
+		## client.
+		client_issuer: string &log &optional;
+
+		## Current number of certificates seen from either side. Used
+		## to create file handles.
+		server_depth: count &default=0;
+		client_depth: count &default=0;
+	};
+
+	## Default file handle provider for SSL.
+	global get_file_handle: function(c: connection, is_orig: bool): string;
+
+	## Default file describer for SSL.
+	global describe_file: function(f: fa_file): string;
+}
+
+function get_file_handle(c: connection, is_orig: bool): string
+	{
+	set_session(c);
+
+	local depth: count;
+
+	if ( is_orig )
+		{
+		depth = c$ssl$client_depth;
+		++c$ssl$client_depth;
+		}
+	else
+		{
+		depth = c$ssl$server_depth;
+		++c$ssl$server_depth;
+		}
+
+	return cat(Analyzer::ANALYZER_SSL, c$start_time, is_orig, id_string(c$id), depth);
+	}
+
+function describe_file(f: fa_file): string
+	{
+	if ( f$source != "SSL" || ! f?$info || ! f$info?$x509 || ! f$info$x509?$certificate )
+		return "";
+
+	# It is difficult to reliably describe a certificate - especially since
+	# we do not know when this function is called (hence, if the data structures
+	# are already populated).
+	#
+	# Just return a bit of our connection information and hope that that is good enough.
+	for ( cid in f$conns )
+		{
+		if ( f$conns[cid]?$ssl )
+			{
+			local c = f$conns[cid];
+			return cat(c$id$resp_h, ":", c$id$resp_p);
+			}
+		}
+
+	return cat("Serial: ", f$info$x509$certificate$serial, " Subject: ",
+		f$info$x509$certificate$subject, " Issuer: ",
+		f$info$x509$certificate$issuer);
+	}
+
+event bro_init() &priority=5
+	{
+	Files::register_protocol(Analyzer::ANALYZER_SSL,
+	                         [$get_file_handle = SSL::get_file_handle,
+	                          $describe        = SSL::describe_file]);
+	}
+
+event file_over_new_connection(f: fa_file, c: connection, is_orig: bool) &priority=5
+	{
+	if ( ! c?$ssl )
+		return;
+
+	if ( ! c$ssl?$cert_chain )
+		{
+		c$ssl$cert_chain = vector();
+		c$ssl$client_cert_chain = vector();
+		c$ssl$cert_chain_fuids = string_vec();
+		c$ssl$client_cert_chain_fuids = string_vec();
+		}
+
+	if ( is_orig )
+		{
+		c$ssl$client_cert_chain[|c$ssl$client_cert_chain|] = f$info;
+		c$ssl$client_cert_chain_fuids[|c$ssl$client_cert_chain_fuids|] = f$id;
+		}
+	else
+		{
+		c$ssl$cert_chain[|c$ssl$cert_chain|] = f$info;
+		c$ssl$cert_chain_fuids[|c$ssl$cert_chain_fuids|] = f$id;
+		}
+
+	Files::add_analyzer(f, Files::ANALYZER_X509);
+	# always calculate hashes. They are not necessary for base scripts
+	# but very useful for identification, and required for policy scripts
+	Files::add_analyzer(f, Files::ANALYZER_MD5);
+	Files::add_analyzer(f, Files::ANALYZER_SHA1);
+	}
+
+event ssl_established(c: connection) &priority=6
+	{
+	# update subject and issuer information
+	if ( c$ssl?$cert_chain && |c$ssl$cert_chain| > 0 )
+		{
+		c$ssl$subject = c$ssl$cert_chain[0]$x509$certificate$subject;
+		c$ssl$issuer = c$ssl$cert_chain[0]$x509$certificate$issuer;
+		}
+
+	if ( c$ssl?$client_cert_chain && |c$ssl$client_cert_chain| > 0 )
+		{
+		c$ssl$client_subject = c$ssl$client_cert_chain[0]$x509$certificate$subject;
+		c$ssl$client_issuer = c$ssl$client_cert_chain[0]$x509$certificate$issuer;
+		}
+	}

scripts/base/protocols/ssl/main.bro

@@ -24,36 +24,9 @@ export {
 		server_name:      string           &log &optional;
 		## Session ID offered by the client for session resumption.
 		session_id:       string           &log &optional;
-		## Subject of the X.509 certificate offered by the server.
-		subject:          string           &log &optional;
-		## Subject of the signer of the X.509 certificate offered by the
-		## server.
-		issuer_subject:   string           &log &optional;
-		## NotValidBefore field value from the server certificate.
-		not_valid_before: time             &log &optional;
-		## NotValidAfter field value from the server certificate.
-		not_valid_after:  time             &log &optional;
 		## Last alert that was seen during the connection.
 		last_alert:       string           &log &optional;
 
-		## Subject of the X.509 certificate offered by the client.
-		client_subject:          string           &log &optional;
-		## Subject of the signer of the X.509 certificate offered by the
-		## client.
-		client_issuer_subject:   string           &log &optional;
-
-		## Full binary server certificate stored in DER format.
-		cert:             string           &optional;
-		## Chain of certificates offered by the server to validate its
-		## complete signing chain.
-		cert_chain:       vector of string &optional;
-
-		## Full binary client certificate stored in DER format.
-		client_cert:             string           &optional;
-		## Chain of certificates offered by the client to validate its
-		## complete signing chain.
-		client_cert_chain:       vector of string &optional;
-
 		## The analyzer ID used for the analyzer instance attached
 		## to each connection.  It is not used for logging since it's a
 		## meaningless arbitrary number.
@@ -116,8 +89,7 @@ event bro_init() &priority=5
 function set_session(c: connection)
 	{
 	if ( ! c?$ssl )
-		c$ssl = [$ts=network_time(), $uid=c$uid, $id=c$id, $cert_chain=vector(),
-		         $client_cert_chain=vector()];
+		c$ssl = [$ts=network_time(), $uid=c$uid, $id=c$id];
 	}
 
 function delay_log(info: Info, token: string)
@@ -185,49 +157,6 @@ event ssl_server_hello(c: connection, version: count, possible_ts: time, server_
 	c$ssl$cipher = cipher_desc[cipher];
 	}
 
-event x509_certificate(c: connection, is_orig: bool, cert: X509, chain_idx: count, chain_len: count, der_cert: string) &priority=5
-	{
-	set_session(c);
-
-	# We aren't doing anything with client certificates yet.
-	if ( is_orig )
-		{
-		if ( chain_idx == 0 )
-			{
-			# Save the primary cert.
-			c$ssl$client_cert = der_cert;
-
-			# Also save other certificate information about the primary cert.
-			c$ssl$client_subject = cert$subject;
-			c$ssl$client_issuer_subject = cert$issuer;
-			}
-		else
-			{
-			# Otherwise, add it to the cert validation chain.
-			c$ssl$client_cert_chain[|c$ssl$client_cert_chain|] = der_cert;
-			}
-		}
-	else
-		{
-		if ( chain_idx == 0 )
-			{
-			# Save the primary cert.
-			c$ssl$cert = der_cert;
-
-			# Also save other certificate information about the primary cert.
-			c$ssl$subject = cert$subject;
-			c$ssl$issuer_subject = cert$issuer;
-			c$ssl$not_valid_before = cert$not_valid_before;
-			c$ssl$not_valid_after = cert$not_valid_after;
-			}
-		else
-			{
-			# Otherwise, add it to the cert validation chain.
-			c$ssl$cert_chain[|c$ssl$cert_chain|] = der_cert;
-			}
-		}
-	}
-
 event ssl_extension(c: connection, is_orig: bool, code: count, val: string) &priority=5
 	{
 	set_session(c);
@@ -243,7 +172,7 @@ event ssl_alert(c: connection, is_orig: bool, level: count, desc: count) &priori
 	c$ssl$last_alert = alert_descriptions[desc];
 	}
 
-event ssl_established(c: connection) &priority=5
+event ssl_established(c: connection) &priority=7
 	{
 	set_session(c);
 	c$ssl$established = T;