Merge remote-tracking branch 'origin/topic/bernhard/file-analysis-x509'

* origin/topic/bernhard/file-analysis-x509:
  x509 version always has to be incremented. Baselines already committed.
  Fix x509 certificate Version (the +1 was missing...).
  Add policy script to suppress non host-certificate logging in x509.log
  Forgot the preamble for the new leak test
  (hopefully) last change -> return real opaque vec instead of any_vec
  Fix dump-events - it cannot be used with ssl anymore, because openssl does not give the same string results in all versions.
  Finishing touches of the x509 file analyzer.
  Revert change to only log certificates once per hour.
  Change x509 log - now certificates are only logged once per hour.
  Fix circular reference problem and a few other small things.
  X509 file analyzer nearly done. Verification and most other policy scripts work fine now.
  Add verify functionality, including the ability to get the validated chain. This means that it is now possible to get information about the root-certificates that were used to secure a connection.
  Second try on the event interface.
  Backport crash fix that made it into master with the x509_extension backport from here.
  Make x509 certificates an opaque type
  rip out x509 code from ssl analyzer. Note that since at the moment the file analyzer does not yet re-populate the info record that means quite a lot of information is simply not available.
  parse out extension. One event for general extensions (just returns the openssl-parsed string-value), one event for basicconstraints (is a certificate a CA or not) and one event for subject-alternative-names (only DNS parts).
  Very basic file-analyzer for x509 certificates. Mostly ripped from the ssl-analyzer and the topic/bernhard/x509 branch.

BIT-1150 #merged

scripts/base/files/x509/README

@@ -0,0 +1 @@
+Support for X509 certificates with the file analysis framework.

scripts/base/files/x509/__load__.bro

@@ -0,0 +1 @@
+@load ./main

scripts/base/files/x509/main.bro

@@ -0,0 +1,77 @@
+@load base/frameworks/files
+@load base/files/hash
+
+module X509;
+
+export {
+	redef enum Log::ID += { LOG };
+
+	type Info: record {
+		## Current timestamp.
+		ts: time &log;
+
+		## File id of this certificate.
+		id: string &log;
+
+		## Basic information about the certificate.
+		certificate: X509::Certificate &log;
+
+		## The opaque wrapping the certificate. Mainly used
+		## for the verify operations.
+		handle: opaque of x509;
+
+		## All extensions that were encountered in the certificate.
+		extensions: vector of X509::Extension &default=vector();
+
+		## Subject alternative name extension of the certificate.
+		san: X509::SubjectAlternativeName &optional &log;
+
+		## Basic constraints extension of the certificate.
+		basic_constraints: X509::BasicConstraints &optional &log;
+	};
+
+	## Event for accessing logged records.
+	global log_x509: event(rec: Info);
+}
+
+event bro_init() &priority=5
+	{
+	Log::create_stream(X509::LOG, [$columns=Info, $ev=log_x509]);
+	}
+
+redef record Files::Info += {
+	## Information about X509 certificates. This is used to keep
+	## certificate information until all events have been received.
+	x509: X509::Info &optional;
+};
+
+event x509_certificate(f: fa_file, cert_ref: opaque of x509, cert: X509::Certificate) &priority=5
+	{
+	f$info$x509 = [$ts=f$info$ts, $id=f$id, $certificate=cert, $handle=cert_ref];
+	}
+
+event x509_extension(f: fa_file, ext: X509::Extension) &priority=5
+	{
+	if ( f$info?$x509 )
+		f$info$x509$extensions[|f$info$x509$extensions|] = ext;
+	}
+
+event x509_ext_basic_constraints(f: fa_file, ext: X509::BasicConstraints) &priority=5
+	{
+	if ( f$info?$x509 )
+		f$info$x509$basic_constraints = ext;
+	}
+
+event x509_ext_subject_alternative_name(f: fa_file, ext: X509::SubjectAlternativeName) &priority=5
+	{
+	if ( f$info?$x509 )
+		f$info$x509$san = ext;
+	}
+
+event file_state_remove(f: fa_file) &priority=5
+	{
+	if ( ! f$info?$x509 )
+		return;
+
+	Log::write(LOG, f$info$x509);
+	}

scripts/base/init-bare.bro

@@ -54,6 +54,13 @@ type any_vec: vector of any;
 ##    directly and then remove this alias.
 type string_vec: vector of string;
 
+## A vector of x509 opaques.
+##
+## .. todo:: We need this type definition only for declaring builtin functions
+##    via ``bifcl``. We should extend ``bifcl`` to understand composite types
+##    directly and then remove this alias.
+type x509_opaque_vector: vector of opaque of x509;
+
 ## A vector of addresses.
 ##
 ## .. todo:: We need this type definition only for declaring builtin functions
@@ -2421,29 +2428,6 @@ global dns_skip_all_addl = T &redef;
 ## traffic and do not process it.  Set to 0 to turn off this functionality.
 global dns_max_queries = 5;
 
-## An X509 certificate.
-##
-## .. bro:see:: x509_certificate
-type X509: record {
-	version: count;	##< Version number.
-	serial: string;	##< Serial number.
-	subject: string;	##< Subject.
-	issuer: string;	##< Issuer.
-	not_valid_before: time;	##< Timestamp before when certificate is not valid.
-	not_valid_after: time;	##< Timestamp after when certificate is not valid.
-};
-
-## An X509 extension.
-##
-## .. bro:see:: x509_extension
-type X509_extension_info: record {
-	name: string;	##< Long name of extension; oid if name not known.
-	short_name: string &optional;	##< Short name of extension if known.
-	oid: string;	##< Oid of extension.
-	critical: bool;	##< True if extension is critical.
-	value: string;	##< Extension content parsed to string for known extensions. Raw data otherwise.
-};
-
 ## HTTP session statistics.
 ##
 ## .. bro:see:: http_stats
@@ -2765,6 +2749,55 @@ export {
 	};
 }
 
+module X509;
+export {
+	type Certificate: record {
+		version: count;	##< Version number.
+		serial: string;	##< Serial number.
+		subject: string;	##< Subject.
+		issuer: string;	##< Issuer.
+		not_valid_before: time;	##< Timestamp before when certificate is not valid.
+		not_valid_after: time;	##< Timestamp after when certificate is not valid.
+		key_alg: string;	##< Name of the key algorithm
+		sig_alg: string;	##< Name of the signature algorithm
+		key_type: string &optional;	##< Key type, if key parseable by openssl (either rsa, dsa or ec)
+		key_length: count &optional;	##< Key length in bits
+		exponent: string &optional;	##< Exponent, if RSA-certificate
+		curve: string &optional;	##< Curve, if EC-certificate
+	} &log;
+
+	type Extension: record {
+		name: string;	##< Long name of extension. oid if name not known
+		short_name: string &optional;	##< Short name of extension if known
+		oid: string;	##< Oid of extension
+		critical: bool;	##< True if extension is critical
+		value: string;	##< Extension content parsed to string for known extensions. Raw data otherwise.
+	};
+
+	type BasicConstraints: record {
+		ca: bool;	##< CA flag set?
+		path_len: count &optional;	##< Maximum path length
+	} &log;
+
+	type SubjectAlternativeName: record {
+		dns: string_vec &optional &log;	##< List of DNS entries in SAN
+		uri: string_vec &optional &log;	##< List of URI entries in SAN
+		email: string_vec &optional &log;	##< List of email entries in SAN
+		ip: addr_vec &optional &log;	##< List of IP entries in SAN
+		other_fields: bool;	##< True if the certificate contained other, not recognized or parsed name fields
+	};
+
+	## Result of an X509 certificate chain verification
+	type Result: record {
+		## OpenSSL result code
+		result:	count;
+		## Result as string
+		result_string: string;
+		## References to the final certificate chain, if verification successful. End-host certificate is first.
+		chain_certs: vector of opaque of x509 &optional;
+	};
+}
+
 module SOCKS;
 export {
 	## This record is for a SOCKS client or server to provide either a

scripts/base/init-default.bro

@@ -57,7 +57,7 @@
 @load base/files/hash
 @load base/files/extract
 @load base/files/unified2
-
+@load base/files/x509
 
 @load base/misc/find-checksum-offloading
 @load base/misc/find-filtered-trace

scripts/base/protocols/ssl/__load__.bro

@@ -1,5 +1,6 @@
 @load ./consts
 @load ./main
 @load ./mozilla-ca-list
+@load ./files
 
 @load-sigs ./dpd.sig

scripts/base/protocols/ssl/files.bro

@@ -0,0 +1,149 @@
+@load ./main
+@load base/utils/conn-ids
+@load base/frameworks/files
+@load base/files/x509
+
+module SSL;
+
+export {
+	redef record Info += {
+		## Chain of certificates offered by the server to validate its
+		## complete signing chain.
+		cert_chain: vector of Files::Info &optional;
+
+		## An ordered vector of all certicate file unique IDs for the
+		## certificates offered by the server.
+		cert_chain_fuids: vector of string &optional &log;
+
+		## Chain of certificates offered by the client to validate its
+		## complete signing chain.
+		client_cert_chain: vector of Files::Info &optional;
+
+		## An ordered vector of all certicate file unique IDs for the
+		## certificates offered by the client.
+		client_cert_chain_fuids: vector of string &optional &log;
+
+		## Subject of the X.509 certificate offered by the server.
+		subject: string &log &optional;
+
+		## Subject of the signer of the X.509 certificate offered by the
+		## server.
+		issuer: string &log &optional;
+
+		## Subject of the X.509 certificate offered by the client.
+		client_subject: string &log &optional;
+
+		## Subject of the signer of the X.509 certificate offered by the
+		## client.
+		client_issuer: string &log &optional;
+
+		## Current number of certificates seen from either side. Used
+		## to create file handles.
+		server_depth: count &default=0;
+		client_depth: count &default=0;
+	};
+
+	## Default file handle provider for SSL.
+	global get_file_handle: function(c: connection, is_orig: bool): string;
+
+	## Default file describer for SSL.
+	global describe_file: function(f: fa_file): string;
+}
+
+function get_file_handle(c: connection, is_orig: bool): string
+	{
+	set_session(c);
+
+	local depth: count;
+
+	if ( is_orig )
+		{
+		depth = c$ssl$client_depth;
+		++c$ssl$client_depth;
+		}
+	else
+		{
+		depth = c$ssl$server_depth;
+		++c$ssl$server_depth;
+		}
+
+	return cat(Analyzer::ANALYZER_SSL, c$start_time, is_orig, id_string(c$id), depth);
+	}
+
+function describe_file(f: fa_file): string
+	{
+	if ( f$source != "SSL" || ! f?$info || ! f$info?$x509 || ! f$info$x509?$certificate )
+		return "";
+
+	# It is difficult to reliably describe a certificate - especially since
+	# we do not know when this function is called (hence, if the data structures
+	# are already populated).
+	#
+	# Just return a bit of our connection information and hope that that is good enough.
+	for ( cid in f$conns )
+		{
+		if ( f$conns[cid]?$ssl )
+			{
+			local c = f$conns[cid];
+			return cat(c$id$resp_h, ":", c$id$resp_p);
+			}
+		}
+
+	return cat("Serial: ", f$info$x509$certificate$serial, " Subject: ",
+		f$info$x509$certificate$subject, " Issuer: ",
+		f$info$x509$certificate$issuer);
+	}
+
+event bro_init() &priority=5
+	{
+	Files::register_protocol(Analyzer::ANALYZER_SSL,
+	                         [$get_file_handle = SSL::get_file_handle,
+	                          $describe        = SSL::describe_file]);
+	}
+
+event file_over_new_connection(f: fa_file, c: connection, is_orig: bool) &priority=5
+	{
+	if ( ! c?$ssl )
+		return;
+
+	if ( ! c$ssl?$cert_chain )
+		{
+		c$ssl$cert_chain = vector();
+		c$ssl$client_cert_chain = vector();
+		c$ssl$cert_chain_fuids = string_vec();
+		c$ssl$client_cert_chain_fuids = string_vec();
+		}
+
+	if ( is_orig )
+		{
+		c$ssl$client_cert_chain[|c$ssl$client_cert_chain|] = f$info;
+		c$ssl$client_cert_chain_fuids[|c$ssl$client_cert_chain_fuids|] = f$id;
+		}
+	else
+		{
+		c$ssl$cert_chain[|c$ssl$cert_chain|] = f$info;
+		c$ssl$cert_chain_fuids[|c$ssl$cert_chain_fuids|] = f$id;
+		}
+
+	Files::add_analyzer(f, Files::ANALYZER_X509);
+	# always calculate hashes. They are not necessary for base scripts
+	# but very useful for identification, and required for policy scripts
+	Files::add_analyzer(f, Files::ANALYZER_MD5);
+	Files::add_analyzer(f, Files::ANALYZER_SHA1);
+	}
+
+event ssl_established(c: connection) &priority=6
+	{
+	# update subject and issuer information
+	if ( c$ssl?$cert_chain && |c$ssl$cert_chain| > 0 )
+		{
+		c$ssl$subject = c$ssl$cert_chain[0]$x509$certificate$subject;
+		c$ssl$issuer = c$ssl$cert_chain[0]$x509$certificate$issuer;
+		}
+
+	if ( c$ssl?$client_cert_chain && |c$ssl$client_cert_chain| > 0 )
+		{
+		c$ssl$client_subject = c$ssl$client_cert_chain[0]$x509$certificate$subject;
+		c$ssl$client_issuer = c$ssl$client_cert_chain[0]$x509$certificate$issuer;
+		}
+	}

scripts/base/protocols/ssl/main.bro

@@ -24,36 +24,9 @@ export {
 		server_name:      string           &log &optional;
 		## Session ID offered by the client for session resumption.
 		session_id:       string           &log &optional;
-		## Subject of the X.509 certificate offered by the server.
-		subject:          string           &log &optional;
-		## Subject of the signer of the X.509 certificate offered by the
-		## server.
-		issuer_subject:   string           &log &optional;
-		## NotValidBefore field value from the server certificate.
-		not_valid_before: time             &log &optional;
-		## NotValidAfter field value from the server certificate.
-		not_valid_after:  time             &log &optional;
 		## Last alert that was seen during the connection.
 		last_alert:       string           &log &optional;
 
-		## Subject of the X.509 certificate offered by the client.
-		client_subject:          string           &log &optional;
-		## Subject of the signer of the X.509 certificate offered by the
-		## client.
-		client_issuer_subject:   string           &log &optional;
-
-		## Full binary server certificate stored in DER format.
-		cert:             string           &optional;
-		## Chain of certificates offered by the server to validate its
-		## complete signing chain.
-		cert_chain:       vector of string &optional;
-
-		## Full binary client certificate stored in DER format.
-		client_cert:             string           &optional;
-		## Chain of certificates offered by the client to validate its
-		## complete signing chain.
-		client_cert_chain:       vector of string &optional;
-
 		## The analyzer ID used for the analyzer instance attached
 		## to each connection.  It is not used for logging since it's a
 		## meaningless arbitrary number.
@@ -116,8 +89,7 @@ event bro_init() &priority=5
 function set_session(c: connection)
 	{
 	if ( ! c?$ssl )
-		c$ssl = [$ts=network_time(), $uid=c$uid, $id=c$id, $cert_chain=vector(),
-		         $client_cert_chain=vector()];
+		c$ssl = [$ts=network_time(), $uid=c$uid, $id=c$id];
 	}
 
 function delay_log(info: Info, token: string)
@@ -185,49 +157,6 @@ event ssl_server_hello(c: connection, version: count, possible_ts: time, server_
 	c$ssl$cipher = cipher_desc[cipher];
 	}
 
-event x509_certificate(c: connection, is_orig: bool, cert: X509, chain_idx: count, chain_len: count, der_cert: string) &priority=5
-	{
-	set_session(c);
-
-	# We aren't doing anything with client certificates yet.
-	if ( is_orig )
-		{
-		if ( chain_idx == 0 )
-			{
-			# Save the primary cert.
-			c$ssl$client_cert = der_cert;
-
-			# Also save other certificate information about the primary cert.
-			c$ssl$client_subject = cert$subject;
-			c$ssl$client_issuer_subject = cert$issuer;
-			}
-		else
-			{
-			# Otherwise, add it to the cert validation chain.
-			c$ssl$client_cert_chain[|c$ssl$client_cert_chain|] = der_cert;
-			}
-		}
-	else
-		{
-		if ( chain_idx == 0 )
-			{
-			# Save the primary cert.
-			c$ssl$cert = der_cert;
-
-			# Also save other certificate information about the primary cert.
-			c$ssl$subject = cert$subject;
-			c$ssl$issuer_subject = cert$issuer;
-			c$ssl$not_valid_before = cert$not_valid_before;
-			c$ssl$not_valid_after = cert$not_valid_after;
-			}
-		else
-			{
-			# Otherwise, add it to the cert validation chain.
-			c$ssl$cert_chain[|c$ssl$cert_chain|] = der_cert;
-			}
-		}
-	}
-
 event ssl_extension(c: connection, is_orig: bool, code: count, val: string) &priority=5
 	{
 	set_session(c);
@@ -243,7 +172,7 @@ event ssl_alert(c: connection, is_orig: bool, level: count, desc: count) &priori
 	c$ssl$last_alert = alert_descriptions[desc];
 	}
 
-event ssl_established(c: connection) &priority=5
+event ssl_established(c: connection) &priority=7
 	{
 	set_session(c);
 	c$ssl$established = T;