Merge remote-tracking branch 'origin/topic/bernhard/file-analysis-x509'

* origin/topic/bernhard/file-analysis-x509:
  Forgot the preamble for the new leak test
  (hopefully) last change -> return real opaque vec instead of any_vec
  Fix dump-events - it cannot be used with ssl anymore, because openssl does not give the same string results in all versions.
  Finishing touches of the x509 file analyzer.
  Revert change to only log certificates once per hour.
  Change x509 log - now certificates are only logged once per hour.
  Fix circular reference problem and a few other small things.
  X509 file analyzer nearly done. Verification and most other policy scripts work fine now.
  Add verify functionality, including the ability to get the validated chain. This means that it is now possible to get information about the root-certificates that were used to secure a connection.
  Second try on the event interface.
  Backport crash fix that made it into master with the x509_extension backport from here.
  Make x509 certificates an opaque type
  rip out x509 code from ssl analyzer. Note that since at the moment the file analyzer does not yet re-populate the info record that means quite a lot of information is simply not available.
  parse out extension. One event for general extensions (just returns the openssl-parsed string-value), one event for basicconstraints (is a certificate a CA or not) and one event for subject-alternative-names (only DNS parts).
  Very basic file-analyzer for x509 certificates. Mostly ripped from the ssl-analyzer and the topic/bernhard/x509 branch.

scripts/base/files/x509/README

@@ -0,0 +1 @@
+Support for X509 certificates with the file analysis framework.

scripts/base/files/x509/__load__.bro

@@ -0,0 +1 @@
+@load ./main

scripts/base/files/x509/main.bro

@@ -0,0 +1,77 @@
+@load base/frameworks/files
+@load base/files/hash
+
+module X509;
+
+export {
+	redef enum Log::ID += { LOG };
+
+	type Info: record {
+		## Current timestamp.
+		ts: time &log;
+
+		## File id of this certificate.
+		id: string &log;
+
+		## Basic information about the certificate.
+		certificate: X509::Certificate &log;
+
+		## The opaque wrapping the certificate. Mainly used
+		## for the verify operations.
+		handle: opaque of x509;
+
+		## All extensions that were encountered in the certificate.
+		extensions: vector of X509::Extension &default=vector();
+
+		## Subject alternative name extension of the certificate.
+		san: X509::SubjectAlternativeName &optional &log;
+
+		## Basic constraints extension of the certificate.
+		basic_constraints: X509::BasicConstraints &optional &log;
+	};
+
+	## Event for accessing logged records.
+	global log_x509: event(rec: Info);
+}
+
+event bro_init() &priority=5
+	{
+	Log::create_stream(X509::LOG, [$columns=Info, $ev=log_x509]);
+	}
+
+redef record Files::Info += {
+	## Information about X509 certificates. This is used to keep
+	## certificate information until all events have been received.
+	x509: X509::Info &optional;
+};
+
+event x509_certificate(f: fa_file, cert_ref: opaque of x509, cert: X509::Certificate) &priority=5
+	{
+	f$info$x509 = [$ts=f$info$ts, $id=f$id, $certificate=cert, $handle=cert_ref];
+	}
+
+event x509_extension(f: fa_file, ext: X509::Extension) &priority=5
+	{
+	if ( f$info?$x509 )
+		f$info$x509$extensions[|f$info$x509$extensions|] = ext;
+	}
+
+event x509_ext_basic_constraints(f: fa_file, ext: X509::BasicConstraints) &priority=5
+	{
+	if ( f$info?$x509 )
+		f$info$x509$basic_constraints = ext;
+	}
+
+event x509_ext_subject_alternative_name(f: fa_file, ext: X509::SubjectAlternativeName) &priority=5
+	{
+	if ( f$info?$x509 )
+		f$info$x509$san = ext;
+	}
+
+event file_state_remove(f: fa_file) &priority=5
+	{
+	if ( ! f$info?$x509 )
+		return;
+
+	Log::write(LOG, f$info$x509);
+	}

scripts/base/init-bare.bro

@@ -54,6 +54,13 @@ type any_vec: vector of any;
 ##    directly and then remove this alias.
 type string_vec: vector of string;
 
+## A vector of x509 opaques.
+##
+## .. todo:: We need this type definition only for declaring builtin functions
+##    via ``bifcl``. We should extend ``bifcl`` to understand composite types
+##    directly and then remove this alias.
+type x509_opaque_vector: vector of opaque of x509;
+
 ## A vector of addresses.
 ##
 ## .. todo:: We need this type definition only for declaring builtin functions
@@ -2421,29 +2428,6 @@ global dns_skip_all_addl = T &redef;
 ## traffic and do not process it.  Set to 0 to turn off this functionality.
 global dns_max_queries = 5;
 
-## An X509 certificate.
-##
-## .. bro:see:: x509_certificate
-type X509: record {
-	version: count;	##< Version number.
-	serial: string;	##< Serial number.
-	subject: string;	##< Subject.
-	issuer: string;	##< Issuer.
-	not_valid_before: time;	##< Timestamp before when certificate is not valid.
-	not_valid_after: time;	##< Timestamp after when certificate is not valid.
-};
-
-## An X509 extension.
-##
-## .. bro:see:: x509_extension
-type X509_extension_info: record {
-	name: string;	##< Long name of extension; oid if name not known.
-	short_name: string &optional;	##< Short name of extension if known.
-	oid: string;	##< Oid of extension.
-	critical: bool;	##< True if extension is critical.
-	value: string;	##< Extension content parsed to string for known extensions. Raw data otherwise.
-};
-
 ## HTTP session statistics.
 ##
 ## .. bro:see:: http_stats
@@ -2765,6 +2749,55 @@ export {
 	};
 }
 
+module X509;
+export {
+	type Certificate: record {
+		version: count;	##< Version number.
+		serial: string;	##< Serial number.
+		subject: string;	##< Subject.
+		issuer: string;	##< Issuer.
+		not_valid_before: time;	##< Timestamp before when certificate is not valid.
+		not_valid_after: time;	##< Timestamp after when certificate is not valid.
+		key_alg: string;	##< Name of the key algorithm
+		sig_alg: string;	##< Name of the signature algorithm
+		key_type: string &optional;	##< Key type, if key parseable by openssl (either rsa, dsa or ec)
+		key_length: count &optional;	##< Key length in bits
+		exponent: string &optional;	##< Exponent, if RSA-certificate
+		curve: string &optional;	##< Curve, if EC-certificate
+	} &log;
+
+	type Extension: record {
+		name: string;	##< Long name of extension. oid if name not known
+		short_name: string &optional;	##< Short name of extension if known
+		oid: string;	##< Oid of extension
+		critical: bool;	##< True if extension is critical
+		value: string;	##< Extension content parsed to string for known extensions. Raw data otherwise.
+	};
+
+	type BasicConstraints: record {
+		ca: bool;	##< CA flag set?
+		path_len: count &optional;	##< Maximum path length
+	} &log;
+
+	type SubjectAlternativeName: record {
+		dns: string_vec &optional &log;	##< List of DNS entries in SAN
+		uri: string_vec &optional &log;	##< List of URI entries in SAN
+		email: string_vec &optional &log;	##< List of email entries in SAN
+		ip: addr_vec &optional &log;	##< List of IP entries in SAN
+		other_fields: bool;	##< True if the certificate contained other, not recognized or parsed name fields
+	};
+
+	## Result of an X509 certificate chain verification
+	type Result: record {
+		## OpenSSL result code
+		result:	count;
+		## Result as string
+		result_string: string;
+		## References to the final certificate chain, if verification successful. End-host certificate is first.
+		chain_certs: vector of opaque of x509 &optional;
+	};
+}
+
 module SOCKS;
 export {
 	## This record is for a SOCKS client or server to provide either a

scripts/base/init-default.bro

@@ -57,7 +57,7 @@
 @load base/files/hash
 @load base/files/extract
 @load base/files/unified2
-
+@load base/files/x509
 
 @load base/misc/find-checksum-offloading
 @load base/misc/find-filtered-trace

scripts/base/protocols/ssl/__load__.bro

@@ -1,5 +1,6 @@
 @load ./consts
 @load ./main
 @load ./mozilla-ca-list
+@load ./files
 
 @load-sigs ./dpd.sig

scripts/base/protocols/ssl/files.bro

@@ -0,0 +1,149 @@
+@load ./main
+@load base/utils/conn-ids
+@load base/frameworks/files
+@load base/files/x509
+
+module SSL;
+
+export {
+	redef record Info += {
+		## Chain of certificates offered by the server to validate its
+		## complete signing chain.
+		cert_chain: vector of Files::Info &optional;
+
+		## An ordered vector of all certicate file unique IDs for the
+		## certificates offered by the server.
+		cert_chain_fuids: vector of string &optional &log;
+
+		## Chain of certificates offered by the client to validate its
+		## complete signing chain.
+		client_cert_chain: vector of Files::Info &optional;
+
+		## An ordered vector of all certicate file unique IDs for the
+		## certificates offered by the client.
+		client_cert_chain_fuids: vector of string &optional &log;
+
+		## Subject of the X.509 certificate offered by the server.
+		subject: string &log &optional;
+
+		## Subject of the signer of the X.509 certificate offered by the
+		## server.
+		issuer: string &log &optional;
+
+		## Subject of the X.509 certificate offered by the client.
+		client_subject: string &log &optional;
+
+		## Subject of the signer of the X.509 certificate offered by the
+		## client.
+		client_issuer: string &log &optional;
+
+		## Current number of certificates seen from either side. Used
+		## to create file handles.
+		server_depth: count &default=0;
+		client_depth: count &default=0;
+	};
+
+	## Default file handle provider for SSL.
+	global get_file_handle: function(c: connection, is_orig: bool): string;
+
+	## Default file describer for SSL.
+	global describe_file: function(f: fa_file): string;
+}
+
+function get_file_handle(c: connection, is_orig: bool): string
+	{
+	set_session(c);
+
+	local depth: count;
+
+	if ( is_orig )
+		{
+		depth = c$ssl$client_depth;
+		++c$ssl$client_depth;
+		}
+	else
+		{
+		depth = c$ssl$server_depth;
+		++c$ssl$server_depth;
+		}
+
+	return cat(Analyzer::ANALYZER_SSL, c$start_time, is_orig, id_string(c$id), depth);
+	}
+
+function describe_file(f: fa_file): string
+	{
+	if ( f$source != "SSL" || ! f?$info || ! f$info?$x509 || ! f$info$x509?$certificate )
+		return "";
+
+	# It is difficult to reliably describe a certificate - especially since
+	# we do not know when this function is called (hence, if the data structures
+	# are already populated).
+	#
+	# Just return a bit of our connection information and hope that that is good enough.
+	for ( cid in f$conns )
+		{
+		if ( f$conns[cid]?$ssl )
+			{
+			local c = f$conns[cid];
+			return cat(c$id$resp_h, ":", c$id$resp_p);
+			}
+		}
+
+	return cat("Serial: ", f$info$x509$certificate$serial, " Subject: ",
+		f$info$x509$certificate$subject, " Issuer: ",
+		f$info$x509$certificate$issuer);
+	}
+
+event bro_init() &priority=5
+	{
+	Files::register_protocol(Analyzer::ANALYZER_SSL,
+	                         [$get_file_handle = SSL::get_file_handle,
+	                          $describe        = SSL::describe_file]);
+	}
+
+event file_over_new_connection(f: fa_file, c: connection, is_orig: bool) &priority=5
+	{
+	if ( ! c?$ssl )
+		return;
+
+	if ( ! c$ssl?$cert_chain )
+		{
+		c$ssl$cert_chain = vector();
+		c$ssl$client_cert_chain = vector();
+		c$ssl$cert_chain_fuids = string_vec();
+		c$ssl$client_cert_chain_fuids = string_vec();
+		}
+
+	if ( is_orig )
+		{
+		c$ssl$client_cert_chain[|c$ssl$client_cert_chain|] = f$info;
+		c$ssl$client_cert_chain_fuids[|c$ssl$client_cert_chain_fuids|] = f$id;
+		}
+	else
+		{
+		c$ssl$cert_chain[|c$ssl$cert_chain|] = f$info;
+		c$ssl$cert_chain_fuids[|c$ssl$cert_chain_fuids|] = f$id;
+		}
+
+	Files::add_analyzer(f, Files::ANALYZER_X509);
+	# always calculate hashes. They are not necessary for base scripts
+	# but very useful for identification, and required for policy scripts
+	Files::add_analyzer(f, Files::ANALYZER_MD5);
+	Files::add_analyzer(f, Files::ANALYZER_SHA1);
+	}
+
+event ssl_established(c: connection) &priority=6
+	{
+	# update subject and issuer information
+	if ( c$ssl?$cert_chain && |c$ssl$cert_chain| > 0 )
+		{
+		c$ssl$subject = c$ssl$cert_chain[0]$x509$certificate$subject;
+		c$ssl$issuer = c$ssl$cert_chain[0]$x509$certificate$issuer;
+		}
+
+	if ( c$ssl?$client_cert_chain && |c$ssl$client_cert_chain| > 0 )
+		{
+		c$ssl$client_subject = c$ssl$client_cert_chain[0]$x509$certificate$subject;
+		c$ssl$client_issuer = c$ssl$client_cert_chain[0]$x509$certificate$issuer;
+		}
+	}

scripts/base/protocols/ssl/main.bro

@@ -24,36 +24,9 @@ export {
 		server_name:      string           &log &optional;
 		## Session ID offered by the client for session resumption.
 		session_id:       string           &log &optional;
-		## Subject of the X.509 certificate offered by the server.
-		subject:          string           &log &optional;
-		## Subject of the signer of the X.509 certificate offered by the
-		## server.
-		issuer_subject:   string           &log &optional;
-		## NotValidBefore field value from the server certificate.
-		not_valid_before: time             &log &optional;
-		## NotValidAfter field value from the server certificate.
-		not_valid_after:  time             &log &optional;
 		## Last alert that was seen during the connection.
 		last_alert:       string           &log &optional;
 
-		## Subject of the X.509 certificate offered by the client.
-		client_subject:          string           &log &optional;
-		## Subject of the signer of the X.509 certificate offered by the
-		## client.
-		client_issuer_subject:   string           &log &optional;
-
-		## Full binary server certificate stored in DER format.
-		cert:             string           &optional;
-		## Chain of certificates offered by the server to validate its
-		## complete signing chain.
-		cert_chain:       vector of string &optional;
-
-		## Full binary client certificate stored in DER format.
-		client_cert:             string           &optional;
-		## Chain of certificates offered by the client to validate its
-		## complete signing chain.
-		client_cert_chain:       vector of string &optional;
-
 		## The analyzer ID used for the analyzer instance attached
 		## to each connection.  It is not used for logging since it's a
 		## meaningless arbitrary number.
@@ -116,8 +89,7 @@ event bro_init() &priority=5
 function set_session(c: connection)
 	{
 	if ( ! c?$ssl )
-		c$ssl = [$ts=network_time(), $uid=c$uid, $id=c$id, $cert_chain=vector(),
+		c$ssl = [$ts=network_time(), $uid=c$uid, $id=c$id];
-		         $client_cert_chain=vector()];
 	}
 
 function delay_log(info: Info, token: string)
@@ -185,49 +157,6 @@ event ssl_server_hello(c: connection, version: count, possible_ts: time, server_
 	c$ssl$cipher = cipher_desc[cipher];
 	}
 
-event x509_certificate(c: connection, is_orig: bool, cert: X509, chain_idx: count, chain_len: count, der_cert: string) &priority=5
-	{
-	set_session(c);
-
-	# We aren't doing anything with client certificates yet.
-	if ( is_orig )
-		{
-		if ( chain_idx == 0 )
-			{
-			# Save the primary cert.
-			c$ssl$client_cert = der_cert;
-
-			# Also save other certificate information about the primary cert.
-			c$ssl$client_subject = cert$subject;
-			c$ssl$client_issuer_subject = cert$issuer;
-			}
-		else
-			{
-			# Otherwise, add it to the cert validation chain.
-			c$ssl$client_cert_chain[|c$ssl$client_cert_chain|] = der_cert;
-			}
-		}
-	else
-		{
-		if ( chain_idx == 0 )
-			{
-			# Save the primary cert.
-			c$ssl$cert = der_cert;
-
-			# Also save other certificate information about the primary cert.
-			c$ssl$subject = cert$subject;
-			c$ssl$issuer_subject = cert$issuer;
-			c$ssl$not_valid_before = cert$not_valid_before;
-			c$ssl$not_valid_after = cert$not_valid_after;
-			}
-		else
-			{
-			# Otherwise, add it to the cert validation chain.
-			c$ssl$cert_chain[|c$ssl$cert_chain|] = der_cert;
-			}
-		}
-	}
-
 event ssl_extension(c: connection, is_orig: bool, code: count, val: string) &priority=5
 	{
 	set_session(c);
@@ -243,7 +172,7 @@ event ssl_alert(c: connection, is_orig: bool, level: count, desc: count) &priori
 	c$ssl$last_alert = alert_descriptions[desc];
 	}
 
-event ssl_established(c: connection) &priority=5
+event ssl_established(c: connection) &priority=7
 	{
 	set_session(c);
 	c$ssl$established = T;

scripts/policy/frameworks/intel/seen/__load__.bro

@@ -7,3 +7,4 @@
 @load ./ssl
 @load ./smtp
 @load ./smtp-url-extraction
+@load ./x509

scripts/policy/frameworks/intel/seen/ssl.bro

@@ -2,27 +2,6 @@
 @load base/protocols/ssl
 @load ./where-locations
 
-event x509_certificate(c: connection, is_orig: bool, cert: X509, chain_idx: count, chain_len: count, der_cert: string)
-	{
-	if ( chain_idx == 0 )
-		{
-		if ( /emailAddress=/ in cert$subject )
-			{
-			local email = sub(cert$subject, /^.*emailAddress=/, "");
-			email = sub(email, /,.*$/, "");
-			Intel::seen([$indicator=email,
-			             $indicator_type=Intel::EMAIL,
-			             $conn=c,
-			             $where=(is_orig ? SSL::IN_CLIENT_CERT : SSL::IN_SERVER_CERT)]);
-			}
-
-		Intel::seen([$indicator=sha1_hash(der_cert),
-		             $indicator_type=Intel::CERT_HASH,
-		             $conn=c,
-		             $where=(is_orig ? SSL::IN_CLIENT_CERT : SSL::IN_SERVER_CERT)]);
-		}
-	}
-
 event ssl_extension(c: connection, is_orig: bool, code: count, val: string)
 	{
 	if ( is_orig && SSL::extensions[code] == "server_name" && 

scripts/policy/frameworks/intel/seen/where-locations.bro

@@ -21,9 +21,8 @@ export {
 		SMTP::IN_REPLY_TO,
 		SMTP::IN_X_ORIGINATING_IP_HEADER,
 		SMTP::IN_MESSAGE,
-		SSL::IN_SERVER_CERT,
-		SSL::IN_CLIENT_CERT,
 		SSL::IN_SERVER_NAME,
 		SMTP::IN_HEADER,
+		X509::IN_CERT,
 	};
 }

scripts/policy/frameworks/intel/seen/x509.bro

@@ -0,0 +1,16 @@
+@load base/frameworks/intel
+@load base/files/x509
+@load ./where-locations
+
+event x509_certificate(f: fa_file, cert_ref: opaque of x509, cert: X509::Certificate)
+	{
+	if ( /emailAddress=/ in cert$subject )
+		{
+		local email = sub(cert$subject, /^.*emailAddress=/, "");
+		email = sub(email, /,.*$/, "");
+		Intel::seen([$indicator=email,
+			     $indicator_type=Intel::EMAIL,
+			     $f=f,
+			     $where=X509::IN_CERT]);
+		}
+	}

scripts/policy/protocols/ssl/cert-hash.bro

@@ -1,22 +0,0 @@
-##! Calculate MD5 sums for server DER formatted certificates.
-
-@load base/protocols/ssl
-
-module SSL;
-
-export {
-	redef record Info += {
-		## MD5 sum of the raw server certificate.
-		cert_hash: string &log &optional;
-	};
-}
-
-event x509_certificate(c: connection, is_orig: bool, cert: X509, chain_idx: count, chain_len: count, der_cert: string) &priority=4
-	{
-	# We aren't tracking client certificates yet and we are also only tracking
-	# the primary cert.  Watch that this came from an SSL analyzed session too.
-	if ( is_orig || chain_idx != 0 || ! c?$ssl ) 
-		return;
-
-	c$ssl$cert_hash = md5_hash(der_cert);
-	}

scripts/policy/protocols/ssl/expiring-certs.bro

@@ -3,10 +3,7 @@
 ##! certificate.
 
 @load base/protocols/ssl
-@load base/frameworks/notice
+@load base/files/x509
-@load base/utils/directions-and-hosts
-
-@load protocols/ssl/cert-hash
 
 module SSL;
 
@@ -35,30 +32,31 @@ export {
 	const notify_when_cert_expiring_in = 30days &redef;
 }
 
-event x509_certificate(c: connection, is_orig: bool, cert: X509, chain_idx: count, chain_len: count, der_cert: string) &priority=3
+event ssl_established(c: connection) &priority=3
 	{
-	# If this isn't the host cert or we aren't interested in the server, just return.
+	# If there are no certificates or we are not interested in the server, just return.
-	if ( is_orig || 
+	if ( ! c$ssl?$cert_chain || |c$ssl$cert_chain| == 0 ||
-		 chain_idx != 0 ||
-		 ! c$ssl?$cert_hash || 
 		 ! addr_matches_host(c$id$resp_h, notify_certs_expiration) )
 		return;
 
+	local fuid = c$ssl$cert_chain_fuids[0];
+	local cert = c$ssl$cert_chain[0]$x509$certificate;
+	
 	if ( cert$not_valid_before > network_time() )
 		NOTICE([$note=Certificate_Not_Valid_Yet,
 		        $conn=c, $suppress_for=1day,
 		        $msg=fmt("Certificate %s isn't valid until %T", cert$subject, cert$not_valid_before),
-		        $identifier=cat(c$id$resp_h, c$id$resp_p, c$ssl$cert_hash)]);
+		        $fuid=fuid]);
 	
 	else if ( cert$not_valid_after < network_time() )
 		NOTICE([$note=Certificate_Expired,
 		        $conn=c, $suppress_for=1day,
 		        $msg=fmt("Certificate %s expired at %T", cert$subject, cert$not_valid_after),
-		        $identifier=cat(c$id$resp_h, c$id$resp_p, c$ssl$cert_hash)]);
+		        $fuid=fuid]);
 	
 	else if ( cert$not_valid_after - notify_when_cert_expiring_in < network_time() )
 		NOTICE([$note=Certificate_Expires_Soon,
 		        $msg=fmt("Certificate %s is going to expire at %T", cert$subject, cert$not_valid_after),
 		        $conn=c, $suppress_for=1day,
-		        $identifier=cat(c$id$resp_h, c$id$resp_p, c$ssl$cert_hash)]);
+		        $fuid=fuid]);
 	}

scripts/policy/protocols/ssl/extract-certs-pem.bro

@@ -10,8 +10,7 @@
 ##!
 
 @load base/protocols/ssl
-@load base/utils/directions-and-hosts
+@load base/files/x509
-@load protocols/ssl/cert-hash
 
 module SSL;
 
@@ -23,41 +22,31 @@ export {
 }
 
 # This is an internally maintained variable to prevent relogging of
-# certificates that have already been seen.  It is indexed on an md5 sum of
+# certificates that have already been seen.  It is indexed on an sha1 sum of
 # the certificate.
 global extracted_certs: set[string] = set() &read_expire=1hr &redef;
 
 event ssl_established(c: connection) &priority=5
 	{
-	if ( ! c$ssl?$cert )
+	if ( ! c$ssl?$cert_chain || |c$ssl$cert_chain| == 0 )
 		return;
 
 	if ( ! addr_matches_host(c$id$resp_h, extract_certs_pem) )
 		return;
 
-	if ( c$ssl$cert_hash in extracted_certs )
+	local hash = c$ssl$cert_chain[0]$sha1;
+	local cert = c$ssl$cert_chain[0]$x509$handle;
+
+	if ( hash in extracted_certs )
 		# If we already extracted this cert, don't do it again.
 		return;
 
-	add extracted_certs[c$ssl$cert_hash];
+	add extracted_certs[hash];
 	local filename = Site::is_local_addr(c$id$resp_h) ? "certs-local.pem" : "certs-remote.pem";
 	local outfile = open_for_append(filename);
+	enable_raw_output(outfile);
 
-	print outfile, "-----BEGIN CERTIFICATE-----";
+	print outfile, x509_get_certificate_string(cert, T);
 
-	# Encode to base64 and format to fit 50 lines. Otherwise openssl won't like it later.
-	local lines = split_all(encode_base64(c$ssl$cert), /.{50}/);
-	local i = 1;
-	for ( line in lines )
-		{
-		if ( |lines[i]| > 0 )
-			{
-			print outfile, lines[i];
-			}
-		i+=1;
-		}
-
-	print outfile, "-----END CERTIFICATE-----";
-	print outfile, "";
 	close(outfile);
 	}

scripts/policy/protocols/ssl/known-certs.bro

@@ -3,7 +3,7 @@
 
 @load base/utils/directions-and-hosts
 @load base/protocols/ssl
-@load protocols/ssl/cert-hash
+@load base/files/x509
 
 module Known;
 
@@ -33,7 +33,7 @@ export {
 	## The set of all known certificates to store for preventing duplicate 
 	## logging. It can also be used from other scripts to 
 	## inspect if a certificate has been seen in use. The string value 
-	## in the set is for storing the DER formatted certificate's MD5 hash.
+	## in the set is for storing the DER formatted certificate' SHA1 hash.
 	global certs: set[addr, string] &create_expire=1day &synchronized &redef;
 	
 	## Event that can be handled to access the loggable record as it is sent
@@ -46,16 +46,27 @@ event bro_init() &priority=5
 	Log::create_stream(Known::CERTS_LOG, [$columns=CertsInfo, $ev=log_known_certs]);
 	}
 
-event x509_certificate(c: connection, is_orig: bool, cert: X509, chain_idx: count, chain_len: count, der_cert: string) &priority=3
+event ssl_established(c: connection) &priority=3
 	{
-	# Make sure this is the server cert and we have a hash for it.
+	if ( ! c$ssl?$cert_chain || |c$ssl$cert_chain| < 1 )
-	if ( is_orig || chain_idx != 0 || ! c$ssl?$cert_hash ) 
 		return;
 
-	local host = c$id$resp_h;
+	local fuid = c$ssl$cert_chain_fuids[0];
-	if ( [host, c$ssl$cert_hash] !in certs && addr_matches_host(host, cert_tracking) )
+
+	if ( ! c$ssl$cert_chain[0]?$sha1 )
 		{
-		add certs[host, c$ssl$cert_hash];
+		Reporter::error(fmt("Certificate with fuid %s did not contain sha1 hash when checking for known certs. Aborting",
+			fuid));
+		return;
+		}
+
+	local hash = c$ssl$cert_chain[0]$sha1;
+	local cert = c$ssl$cert_chain[0]$x509$certificate;
+
+	local host = c$id$resp_h;
+	if ( [host, hash] !in certs && addr_matches_host(host, cert_tracking) )
+		{
+		add certs[host, hash];
 		Log::write(Known::CERTS_LOG, [$ts=network_time(), $host=host,
 		                              $port_num=c$id$resp_p, $subject=cert$subject,
 		                              $issuer_subject=cert$issuer,

scripts/policy/protocols/ssl/notary.bro

@@ -16,7 +16,6 @@ export {
 }
 
 redef record SSL::Info += {
-	sha1:   string &log &optional;
 	notary: Response &log &optional;
 	};
 
@@ -38,14 +37,12 @@ function clear_waitlist(digest: string)
 		}
 	}
 
-event x509_certificate(c: connection, is_orig: bool, cert: X509,
+event ssl_established(c: connection) &priority=3
-                       chain_idx: count, chain_len: count, der_cert: string)
 	{
-	if ( is_orig || chain_idx != 0 || ! c?$ssl )
+	if ( ! c$ssl?$cert_chain || |c$ssl$cert_chain| == 0 )
 		return;
 
-	local digest = sha1_hash(der_cert);
+	local digest = c$ssl$cert_chain[0]$sha1;
-	c$ssl$sha1 = digest;
 
 	if ( digest in notary_cache )
 		{

scripts/policy/protocols/ssl/validate-certs.bro

@@ -2,7 +2,6 @@
 
 @load base/frameworks/notice
 @load base/protocols/ssl
-@load protocols/ssl/cert-hash
 
 module SSL;
 
@@ -19,9 +18,9 @@ export {
 		validation_status: string &log &optional;
 	};
 	
-	## MD5 hash values for recently validated certs along with the
+	## MD5 hash values for recently validated chains along with the
 	## validation status message are kept in this table to avoid constant
-	## validation every time the same certificate is seen.
+	## validation every time the same certificate chain is seen.
 	global recently_validated_certs: table[string] of string = table() 
 		&read_expire=5mins &synchronized &redef;
 }
@@ -29,18 +28,26 @@ export {
 event ssl_established(c: connection) &priority=3
 	{
 	# If there aren't any certs we can't very well do certificate validation.
-	if ( ! c$ssl?$cert || ! c$ssl?$cert_chain )
+	if ( ! c$ssl?$cert_chain || |c$ssl$cert_chain| == 0 )
 		return;
 
-	if ( c$ssl?$cert_hash && c$ssl$cert_hash in recently_validated_certs )
+	local chain_id = join_string_vec(c$ssl$cert_chain_fuids, ".");
+
+	local chain: vector of opaque of x509 = vector();
+	for ( i in c$ssl$cert_chain )
 		{
-		c$ssl$validation_status = recently_validated_certs[c$ssl$cert_hash];
+		chain[i] = c$ssl$cert_chain[i]$x509$handle;
+		}
+
+	if ( chain_id in recently_validated_certs )
+		{
+		c$ssl$validation_status = recently_validated_certs[chain_id];
 		}
 	else
 		{
-		local result = x509_verify(c$ssl$cert, c$ssl$cert_chain, root_certs);
+		local result = x509_verify(chain, root_certs);
-		c$ssl$validation_status = x509_err2str(result);
+		c$ssl$validation_status = result$result_string;
-		recently_validated_certs[c$ssl$cert_hash] = c$ssl$validation_status;
+		recently_validated_certs[chain_id] = result$result_string;
 		}
 		
 	if ( c$ssl$validation_status != "ok" )
@@ -48,7 +55,7 @@ event ssl_established(c: connection) &priority=3
 		local message = fmt("SSL certificate validation failed with (%s)", c$ssl$validation_status);
 		NOTICE([$note=Invalid_Server_Cert, $msg=message,
 		        $sub=c$ssl$subject, $conn=c,
-		        $identifier=cat(c$id$resp_h,c$id$resp_p,c$ssl$validation_status,c$ssl$cert_hash)]);
+		        $identifier=cat(c$id$resp_h,c$id$resp_p,c$ssl$validation_status)]);
 		}
 	}
 

scripts/test-all-policy.bro

@@ -26,6 +26,7 @@
 @load frameworks/intel/seen/smtp.bro
 @load frameworks/intel/seen/ssl.bro
 @load frameworks/intel/seen/where-locations.bro
+@load frameworks/intel/seen/x509.bro
 @load frameworks/files/detect-MHR.bro
 @load frameworks/files/hash-all-files.bro
 @load frameworks/packet-filter/shunt.bro
@@ -82,7 +83,6 @@
 @load protocols/ssh/geo-data.bro
 @load protocols/ssh/interesting-hostnames.bro
 @load protocols/ssh/software.bro
-@load protocols/ssl/cert-hash.bro
 @load protocols/ssl/expiring-certs.bro
 @load protocols/ssl/extract-certs-pem.bro
 @load protocols/ssl/known-certs.bro

src/NetVar.cc

@@ -47,9 +47,6 @@ int tcp_max_initial_window;
 int tcp_max_above_hole_without_any_acks;
 int tcp_excessive_data_without_further_acks;
 
-RecordType* x509_type;
-RecordType* x509_extension_type;
-
 RecordType* socks_address;
 
 double non_analyzed_lifetime;
@@ -354,9 +351,6 @@ void init_net_var()
 	tcp_excessive_data_without_further_acks =
 		opt_internal_int("tcp_excessive_data_without_further_acks");
 
-	x509_type = internal_type("X509")->AsRecordType();
-	x509_extension_type = internal_type("X509_extension_info")->AsRecordType();
-
 	socks_address = internal_type("SOCKS::Address")->AsRecordType();
 
 	non_analyzed_lifetime = opt_internal_double("non_analyzed_lifetime");

src/NetVar.h

@@ -50,9 +50,6 @@ extern int tcp_max_initial_window;
 extern int tcp_max_above_hole_without_any_acks;
 extern int tcp_excessive_data_without_further_acks;
 
-extern RecordType* x509_type;
-extern RecordType* x509_extension_type;
-
 extern RecordType* socks_address;
 
 extern double non_analyzed_lifetime;

src/SerialTypes.h

@@ -111,6 +111,7 @@ SERIAL_VAL(ENTROPY_VAL, 19)
 SERIAL_VAL(TOPK_VAL, 20)
 SERIAL_VAL(BLOOMFILTER_VAL, 21)
 SERIAL_VAL(CARDINALITY_VAL, 22)
+SERIAL_VAL(X509_VAL, 23)
 
 #define SERIAL_EXPR(name, val) SERIAL_CONST(name, val, EXPR)
 SERIAL_EXPR(EXPR, 1)

src/Type.h

@@ -73,6 +73,7 @@ class EnumType;
 class Serializer;
 class VectorType;
 class TypeType;
+class OpaqueType;
 
 const int DOES_NOT_MATCH_INDEX = 0;
 const int MATCHES_INDEX_SCALAR = 1;
@@ -204,6 +205,18 @@ public:
 		return (VectorType*) this;
 		}
 
+	OpaqueType* AsOpaqueType()
+		{
+		CHECK_TYPE_TAG(TYPE_OPAQUE, "BroType::AsOpaqueType");
+		return (OpaqueType*) this;
+		}
+
+	const OpaqueType* AsOpaqueType() const
+		{
+		CHECK_TYPE_TAG(TYPE_OPAQUE, "BroType::AsOpaqueType");
+		return (OpaqueType*) this;
+		}
+
 	VectorType* AsVectorType()
 	        {
 		CHECK_TYPE_TAG(TYPE_VECTOR, "BroType::AsVectorType");
@@ -597,6 +610,7 @@ extern OpaqueType* entropy_type;
 extern OpaqueType* cardinality_type;
 extern OpaqueType* topk_type;
 extern OpaqueType* bloomfilter_type;
+extern OpaqueType* x509_opaque_type;
 
 // Returns the BRO basic (non-parameterized) type with the given type.
 extern BroType* base_type(TypeTag tag);

src/analyzer/protocol/ssl/CMakeLists.txt

@@ -6,6 +6,5 @@ include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DI
 bro_plugin_begin(Bro SSL)
 bro_plugin_cc(SSL.cc Plugin.cc)
 bro_plugin_bif(events.bif)
-bro_plugin_bif(functions.bif)
 bro_plugin_pac(ssl.pac ssl-analyzer.pac ssl-protocol.pac ssl-defs.pac)
 bro_plugin_end()

src/analyzer/protocol/ssl/Plugin.cc

@@ -7,5 +7,4 @@ BRO_PLUGIN_BEGIN(Bro, SSL)
 	BRO_PLUGIN_DESCRIPTION("SSL analyzer");
 	BRO_PLUGIN_ANALYZER("SSL", ssl::SSL_Analyzer);
 	BRO_PLUGIN_BIF_FILE(events);
-	BRO_PLUGIN_BIF_FILE(functions);
 BRO_PLUGIN_END

src/analyzer/protocol/ssl/SSL.h

@@ -27,8 +27,7 @@ public:
 	static bool Available()
 		{
 		return ( ssl_client_hello || ssl_server_hello ||
-			ssl_established || ssl_extension || ssl_alert ||
+			ssl_established || ssl_extension || ssl_alert );
-			x509_certificate || x509_extension || x509_error );
 		}
 
 protected:

src/analyzer/protocol/ssl/events.bif

@@ -25,7 +25,7 @@
 ##          :bro:id:`SSL::cipher_desc` table maps them to descriptive names.
 ##
 ## .. bro:see:: ssl_alert  ssl_established ssl_extension ssl_server_hello
-##    ssl_session_ticket_handshake x509_certificate x509_error x509_extension
+##    ssl_session_ticket_handshake x509_certificate
 event ssl_client_hello%(c: connection, version: count, possible_ts: time, client_random: string, session_id: string, ciphers: index_vec%);
 
 ## Generated for an SSL/TLS server's initial *hello* message. SSL/TLS sessions
@@ -58,7 +58,7 @@ event ssl_client_hello%(c: connection, version: count, possible_ts: time, client
 ##              standardized as part of the SSL/TLS protocol.
 ##
 ## .. bro:see:: ssl_alert ssl_client_hello ssl_established ssl_extension
-##    ssl_session_ticket_handshake x509_certificate x509_error x509_extension
+##    ssl_session_ticket_handshake x509_certificate
 event ssl_server_hello%(c: connection, version: count, possible_ts: time, server_random: string, session_id: string, cipher: count, comp_method: count%);
 
 ## Generated for SSL/TLS extensions seen in an initial handshake.  SSL/TLS
@@ -77,7 +77,7 @@ event ssl_server_hello%(c: connection, version: count, possible_ts: time, server
 ## val: The raw extension value that was sent in the message.
 ##
 ## .. bro:see:: ssl_alert ssl_client_hello ssl_established ssl_server_hello
-##    ssl_session_ticket_handshake x509_certificate x509_error x509_extension
+##    ssl_session_ticket_handshake
 event ssl_extension%(c: connection, is_orig: bool, code: count, val: string%);
 
 ## Generated at the end of an SSL/TLS handshake. SSL/TLS sessions start with
@@ -92,7 +92,7 @@ event ssl_extension%(c: connection, is_orig: bool, code: count, val: string%);
 ## c: The connection.
 ##
 ## .. bro:see:: ssl_alert ssl_client_hello  ssl_extension ssl_server_hello
-##    ssl_session_ticket_handshake x509_certificate x509_error x509_extension
+##    ssl_session_ticket_handshake x509_certificate
 event ssl_established%(c: connection%);
 
 ## Generated for SSL/TLS alert records. SSL/TLS sessions start with an
@@ -115,7 +115,7 @@ event ssl_established%(c: connection%);
 ##       defined as part of the SSL/TLS protocol.
 ##
 ## .. bro:see::  ssl_client_hello ssl_established ssl_extension ssl_server_hello
-##    ssl_session_ticket_handshake x509_certificate x509_error x509_extension
+##    ssl_session_ticket_handshake
 event ssl_alert%(c: connection, is_orig: bool, level: count, desc: count%);
 
 ## Generated for SSL/TLS handshake messages that are a part of the
@@ -136,68 +136,5 @@ event ssl_alert%(c: connection, is_orig: bool, level: count, desc: count%);
 ## ticket: The raw ticket data.
 ##
 ## .. bro:see::  ssl_client_hello ssl_established ssl_extension ssl_server_hello
-##    x509_certificate x509_error x509_extension ssl_alert
+##    ssl_alert
 event ssl_session_ticket_handshake%(c: connection, ticket_lifetime_hint: count, ticket: string%);
-
-## Generated for X509 certificates seen in SSL/TLS connections. During the
-## initial SSL/TLS handshake, certificates are exchanged in the clear. Bro
-## raises this event for each certificate seen (including both a site's primary
-## cert, and further certs sent as part of the validation chain).
-##
-## See `Wikipedia <http://en.wikipedia.org/wiki/X.509>`__ for more information
-## about the X.509 format.
-##
-## c: The connection.
-##
-## is_orig: True if event is raised for originator side of the connection.
-##
-## cert: The parsed certificate.
-##
-## chain_idx: The index in the validation chain that this cert has. Index zero
-##            indicates an endpoint's primary cert, while higher indices
-##            indicate the place in the validation chain (which has length
-##            *chain_len*).
-##
-## chain_len: The total length of the validation chain that this cert is part
-##            of.
-##
-## der_cert: The complete cert encoded in `DER
-##           <http://en.wikipedia.org/wiki/Distinguished_Encoding_Rules>`__
-##           format.
-##
-## .. bro:see:: ssl_alert ssl_client_hello ssl_established ssl_extension
-##    ssl_server_hello  x509_error x509_extension x509_verify
-event x509_certificate%(c: connection, is_orig: bool, cert: X509, chain_idx: count, chain_len: count, der_cert: string%);
-
-## Generated for X509 extensions seen in a certificate.
-##
-## See `Wikipedia <http://en.wikipedia.org/wiki/X.509>`__ for more information
-## about the X.509 format.
-##
-## c: The connection.
-##
-## is_orig: True if event is raised for originator side of the connection.
-##
-## cert: The parsed certificate.
-##
-## extension: The parsed extension.
-##
-## .. bro:see:: ssl_alert ssl_client_hello ssl_established ssl_extension
-##    ssl_server_hello x509_certificate x509_error  x509_verify
-event x509_extension%(c: connection, is_orig: bool, cert: X509, extension: X509_extension_info%);
-
-## Generated when errors occur during parsing an X509 certificate.
-##
-## See `Wikipedia <http://en.wikipedia.org/wiki/X.509>`__ for more information
-## about the X.509 format.
-##
-## c: The connection.
-##
-## is_orig: True if event is raised for originator side of the connection.
-##
-## err: An error code describing what went wrong. :bro:id:`SSL::x509_errors`
-##      maps error codes to a textual description.
-##
-## .. bro:see:: ssl_alert ssl_client_hello ssl_established ssl_extension
-##    ssl_server_hello x509_certificate  x509_extension x509_err2str x509_verify
-event x509_error%(c: connection, is_orig: bool, err: count%);

src/analyzer/protocol/ssl/functions.bif

@@ -1,132 +0,0 @@
-
-%%{
-#include <openssl/x509.h>
-#include <openssl/asn1.h>
-#include <openssl/x509_vfy.h>
-
-// This is the indexed map of X509 certificate stores.
-static map<Val*, X509_STORE*> x509_stores;
-
-// ### NOTE: while d2i_X509 does not take a const u_char** pointer,
-// here we assume d2i_X509 does not write to <data>, so it is safe to
-// convert data to a non-const pointer.  Could some X509 guru verify
-// this?
-
-X509* d2i_X509_(X509** px, const u_char** in, int len)
-	{
-#ifdef OPENSSL_D2I_X509_USES_CONST_CHAR
-	  return d2i_X509(px, in, len);
-#else
-	  return d2i_X509(px, (u_char**)in, len);
-#endif
-	}
-
-%%}
-
-
-## Verifies a certificate.
-##
-## der_cert: The X.509 certificate in DER format.
-##
-## cert_stack: Specifies a certificate chain to validate against, with index 0
-##             typically being the root CA. Bro uses the Mozilla root CA list
-##             by default.
-##
-## root_certs: A list of additional root certificates that extends
-##             *cert_stack*.
-##
-## Returns: A status code of the verification which can be converted into an
-##          ASCII string via :bro:id:`x509_err2str`.
-##
-## .. bro:see:: x509_err2str
-function x509_verify%(der_cert: string, cert_stack: string_vec, root_certs: table_string_of_string%): count
-	%{
-	X509_STORE* ctx = 0;
-	int i = 0;
-
-	// If this certificate store was built previously, just reuse the old one.
-	if ( x509_stores.count(root_certs) > 0 )
-		ctx = x509_stores[root_certs];
-
-	if ( ! ctx ) // lookup to see if we have this one built already!
-		{
-		ctx = X509_STORE_new();
-		TableVal* root_certs2 = root_certs->AsTableVal();
-		ListVal* idxs = root_certs2->ConvertToPureList();
-
-		// Build the validation store
-		for ( i = 0; i < idxs->Length(); ++i )
-			{
-			Val* key = idxs->Index(i);
-			StringVal *sv = root_certs2->Lookup(key)->AsStringVal();
-			const uint8* data = sv->Bytes();
-			X509* x = d2i_X509_(NULL, &data, sv->Len());
-			if ( ! x )
-				{
-				builtin_error(fmt("Root CA error: %s", ERR_error_string(ERR_peek_last_error(),NULL)));
-				return new Val((uint64) ERR_get_error(), TYPE_COUNT);
-				}
-			X509_STORE_add_cert(ctx, x);
-			}
-		delete idxs;
-
-		// Save the newly constructed certificate store into the cacheing map.
-		x509_stores[root_certs] = ctx;
-		}
-
-	const uint8 *cert_data = der_cert->Bytes();
-	X509* cert = d2i_X509_(NULL, &cert_data, der_cert->Len());
-	if ( ! cert )
-		{
-		builtin_error(fmt("Certificate error: %s", ERR_error_string(ERR_peek_last_error(),NULL)));
-		return new Val((uint64) ERR_get_error(), TYPE_COUNT);
-		}
-
-	STACK_OF(X509)* untrusted_certs = sk_X509_new_null();
-	if ( ! untrusted_certs )
-		{
-		builtin_error(fmt("Untrusted certificate stack initialization error: %s", ERR_error_string(ERR_peek_last_error(),NULL)));
-		return new Val((uint64) ERR_get_error(), TYPE_COUNT);
-		}
-
-	VectorVal *cert_stack_vec = cert_stack->AsVectorVal();
-	for ( i = 0; i < (int) cert_stack_vec->Size(); ++i )
-		{
-		StringVal *sv = cert_stack_vec->Lookup(i)->AsStringVal();
-		const uint8 *data = sv->Bytes();
-		X509* x = d2i_X509_(NULL, &data, sv->Len());
-		if ( ! x )
-			{
-			X509_free(cert);
-			sk_X509_pop_free(untrusted_certs, X509_free);
-			builtin_error(fmt("Untrusted certificate stack creation error: %s", ERR_error_string(ERR_peek_last_error(),NULL)));
-			return new Val((uint64) ERR_get_error(), TYPE_COUNT);
-			}
-		sk_X509_push(untrusted_certs, x);
-		}
-
-	X509_STORE_CTX csc;
-	X509_STORE_CTX_init(&csc, ctx, cert, untrusted_certs);
-	X509_STORE_CTX_set_time(&csc, 0, (time_t) network_time);
-
-	int result = X509_verify_cert(&csc);
-	X509_STORE_CTX_cleanup(&csc);
-
-	if ( untrusted_certs )
-		sk_X509_pop_free(untrusted_certs, X509_free);
-	X509_free(cert);
-
-	return new Val((uint64) csc.error, TYPE_COUNT);
-	%}
-
-## Converts a certificate verification error code into an ASCII string.
-##
-## err_num: The error code.
-##
-## Returns: A string representation of *err_num*.
-##
-## .. bro:see:: x509_verify
-function x509_err2str%(err_num: count%): string
-	%{
-	return new StringVal(X509_verify_cert_error_string(err_num));
-	%}

src/analyzer/protocol/ssl/ssl-analyzer.pac

@@ -8,9 +8,7 @@
 
 #include "util.h"
 
-#include <openssl/x509.h>
+#include "file_analysis/Manager.h"
-#include <openssl/x509v3.h>
-#include <openssl/asn1.h>
 %}
 
 
@@ -24,8 +22,6 @@
 	};
 
 	string orig_label(bool is_orig);
-	void free_X509(void *);
-	X509* d2i_X509_binpac(X509** px, const uint8** in, int len);
 	string handshake_type_label(int type);
 	%}
 
@@ -35,20 +31,6 @@ string orig_label(bool is_orig)
 		return string(is_orig ? "originator" :"responder");
 		}
 
-	void free_X509(void* cert)
-		{
-		X509_free((X509*) cert);
-		}
-
-	X509* d2i_X509_binpac(X509** px, const uint8** in, int len)
-		{
-#ifdef OPENSSL_D2I_X509_USES_CONST_CHAR
-		return d2i_X509(px, in, len);
-#else
-		return d2i_X509(px, (u_char**) in, len);
-#endif
-		}
-
 	string handshake_type_label(int type)
 		{
 		switch ( type ) {
@@ -249,113 +231,15 @@ refine connection SSL_Conn += {
 		if ( certificates->size() == 0 )
 			return true;
 
-		if ( x509_certificate )
-			{
-			STACK_OF(X509)* untrusted_certs = 0;
-
 		for ( unsigned int i = 0; i < certificates->size(); ++i )
 			{
 			const bytestring& cert = (*certificates)[i];
-				const uint8* data = cert.data();
-				X509* pTemp = d2i_X509_binpac(NULL, &data, cert.length());
-				if ( ! pTemp )
-					{
-					BifEvent::generate_x509_error(bro_analyzer(), bro_analyzer()->Conn(),
-					                              ${rec.is_orig}, ERR_get_error());
-					return false;
-					}
 
-				RecordVal* pX509Cert = new RecordVal(x509_type);
+			string fid = file_mgr->DataIn(reinterpret_cast<const u_char*>(cert.data()), cert.length(),
-				char tmp[256];
+						      bro_analyzer()->GetAnalyzerTag(), bro_analyzer()->Conn(),
-				BIO *bio = BIO_new(BIO_s_mem());
+						      ${rec.is_orig});
 
-				pX509Cert->Assign(0, new Val((uint64) X509_get_version(pTemp), TYPE_COUNT));
+			file_mgr->EndOfFile(fid);
-				i2a_ASN1_INTEGER(bio, X509_get_serialNumber(pTemp));
-				int len = BIO_read(bio, &(*tmp), sizeof tmp);
-				pX509Cert->Assign(1, new StringVal(len, tmp));
-
-				X509_NAME_print_ex(bio, X509_get_subject_name(pTemp), 0, XN_FLAG_RFC2253);
-				len = BIO_gets(bio, &(*tmp), sizeof tmp);
-				pX509Cert->Assign(2, new StringVal(len, tmp));
-				X509_NAME_print_ex(bio, X509_get_issuer_name(pTemp), 0, XN_FLAG_RFC2253);
-				len = BIO_gets(bio, &(*tmp), sizeof tmp);
-				pX509Cert->Assign(3, new StringVal(len, tmp));
-				BIO_free(bio);
-
-				pX509Cert->Assign(4, new Val(get_time_from_asn1(X509_get_notBefore(pTemp)), TYPE_TIME));
-				pX509Cert->Assign(5, new Val(get_time_from_asn1(X509_get_notAfter(pTemp)), TYPE_TIME));
-				StringVal* der_cert = new StringVal(cert.length(), (const char*) cert.data());
-
-				BifEvent::generate_x509_certificate(bro_analyzer(), bro_analyzer()->Conn(),
-							${rec.is_orig},
-							pX509Cert,
-							i, certificates->size(),
-							der_cert);
-
-				// Are there any X509 extensions?
-				//printf("Number of x509 extensions: %d\n", X509_get_ext_count(pTemp));
-				if ( x509_extension && X509_get_ext_count(pTemp) > 0 )
-					{
-					int num_ext = X509_get_ext_count(pTemp);
-					for ( int k = 0; k < num_ext; ++k )
-						{
-						char name[256];
-						char oid[256];
-
-						memset(name, 0, sizeof(name));
-						memset(oid, 0, sizeof(oid));
-
-						X509_EXTENSION* ex = X509_get_ext(pTemp, k);
-
-						if ( ! ex )
-							continue;
-
-						ASN1_OBJECT* ext_asn = X509_EXTENSION_get_object(ex);
-						const char* short_name = OBJ_nid2sn(OBJ_obj2nid(ext_asn));
-
-						OBJ_obj2txt(name, sizeof(name) - 1, ext_asn, 0);
-						OBJ_obj2txt(oid, sizeof(oid) - 1, ext_asn, 1);
-
-						int critical = 0;
-						if ( X509_EXTENSION_get_critical(ex) != 0 )
-							critical = 1;
-
-						BIO *bio = BIO_new(BIO_s_mem());
-						if( ! X509V3_EXT_print(bio, ex, 0, 0))
-							M_ASN1_OCTET_STRING_print(bio, ex->value);
-
-						BIO_flush(bio);
-						int length = BIO_pending(bio);
-
-						// Use OPENSSL_malloc here. Using new or anything else can lead
-						// to interesting, hard to debug segfaults.
-						char *buffer = (char*) OPENSSL_malloc(length);
-						BIO_read(bio, buffer, length);
-						StringVal* ext_val = new StringVal(length, buffer);
-						OPENSSL_free(buffer);
-
-						BIO_free_all(bio);
-
-						RecordVal* pX509Ext = new RecordVal(x509_extension_type);
-						pX509Ext->Assign(0, new StringVal(name));
-
-						if ( short_name && strlen(short_name) > 0 )
-							pX509Ext->Assign(1, new StringVal(short_name));
-
-						pX509Ext->Assign(2, new StringVal(oid));
-						pX509Ext->Assign(3, new Val(critical, TYPE_BOOL));
-						pX509Ext->Assign(4, ext_val);
-
-						BifEvent::generate_x509_extension(bro_analyzer(),
-									bro_analyzer()->Conn(),
-									${rec.is_orig},
-									pX509Cert->Ref(),
-									pX509Ext);
-						}
-					}
-
-				X509_free(pTemp);
-				}
 			}
 		return true;
 		%}

src/analyzer/protocol/ssl/ssl-protocol.pac

@@ -22,7 +22,6 @@ type uint24 = record {
 	};
 
 	string state_label(int state_nr);
-	double get_time_from_asn1(const ASN1_TIME * atime);
 %}
 
 extern type to_int;
@@ -146,105 +145,6 @@ enum AnalyzerState {
 			return string(fmt("UNKNOWN (%d)", state_nr));
 		}
 		}
-
-
-	double get_time_from_asn1(const ASN1_TIME * atime)
-		{
-		time_t lResult = 0;
-
-		char lBuffer[24];
-		char * pBuffer = lBuffer;
-
-		size_t lTimeLength = atime->length;
-		char * pString = (char *) atime->data;
-
-		if ( atime->type == V_ASN1_UTCTIME )
-			{
-			if ( lTimeLength < 11 || lTimeLength > 17 )
-				return 0;
-
-			memcpy(pBuffer, pString, 10);
-			pBuffer += 10;
-			pString += 10;
-			}
-		else
-			{
-			if ( lTimeLength < 13 )
-				return 0;
-
-			memcpy(pBuffer, pString, 12);
-			pBuffer += 12;
-			pString += 12;
-			}
-
-		if ((*pString == 'Z') || (*pString == '-') || (*pString == '+'))
-			{
-			*(pBuffer++) = '0';
-			*(pBuffer++) = '0';
-			}
-		else
-			{
-			*(pBuffer++) = *(pString++);
-			*(pBuffer++) = *(pString++);
-
-			// Skip any fractional seconds...
-			if (*pString == '.')
-				{
-				pString++;
-				while ((*pString >= '0') && (*pString <= '9'))
-					pString++;
-				}
-			}
-
-		*(pBuffer++) = 'Z';
-		*(pBuffer++) = '\0';
-
-		time_t lSecondsFromUTC;
-
-		if ( *pString == 'Z' )
-			lSecondsFromUTC = 0;
-
-		else
-			{
-			if ((*pString != '+') && (pString[5] != '-'))
-				return 0;
-
-			lSecondsFromUTC = ((pString[1]-'0') * 10 + (pString[2]-'0')) * 60;
-			lSecondsFromUTC += (pString[3]-'0') * 10 + (pString[4]-'0');
-
-			if (*pString == '-')
-				lSecondsFromUTC = -lSecondsFromUTC;
-			}
-
-		tm lTime;
-		lTime.tm_sec  = ((lBuffer[10] - '0') * 10) + (lBuffer[11] - '0');
-		lTime.tm_min  = ((lBuffer[8] - '0') * 10) + (lBuffer[9] - '0');
-		lTime.tm_hour = ((lBuffer[6] - '0') * 10) + (lBuffer[7] - '0');
-		lTime.tm_mday = ((lBuffer[4] - '0') * 10) + (lBuffer[5] - '0');
-		lTime.tm_mon  = (((lBuffer[2] - '0') * 10) + (lBuffer[3] - '0')) - 1;
-		lTime.tm_year = ((lBuffer[0] - '0') * 10) + (lBuffer[1] - '0');
-
-		if ( lTime.tm_year < 50 )
-			lTime.tm_year += 100; // RFC 2459
-
-		lTime.tm_wday = 0;
-		lTime.tm_yday = 0;
-		lTime.tm_isdst = 0;  // No DST adjustment requested
-
-		lResult = mktime(&lTime);
-
-		if ( lResult )
-			{
-			if ( 0 != lTime.tm_isdst )
-				lResult -= 3600;  // mktime may adjust for DST  (OS dependent)
-
-			lResult += lSecondsFromUTC;
-			}
-		else
-			lResult = 0;
-
-		return lResult;
-	}
 %}
 
 ######################################################################

src/file_analysis/Manager.h

@@ -108,7 +108,7 @@ public:
 	 *         cached and passed back in to a subsequent function call in order
 	 *         to avoid costly file handle lookups (which have to go through
 	 *         the \c get_file_handle script-layer event).  An empty string
-	 *         indicates the associate file is not going to be analyzed further.
+	 *         indicates the associated file is not going to be analyzed further.
 	 */
 	std::string DataIn(const u_char* data, uint64 len, analyzer::Tag tag,
 	                   Connection* conn, bool is_orig,

src/file_analysis/analyzer/CMakeLists.txt

@@ -2,3 +2,4 @@ add_subdirectory(data_event)
 add_subdirectory(extract)
 add_subdirectory(hash)
 add_subdirectory(unified2)
+add_subdirectory(x509)

src/file_analysis/analyzer/x509/CMakeLists.txt

@@ -0,0 +1,10 @@
+
+include(BroPlugin)
+
+include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR}
+                           ${CMAKE_CURRENT_BINARY_DIR})
+
+bro_plugin_begin(Bro X509)
+bro_plugin_cc(X509.cc Plugin.cc)
+bro_plugin_bif(events.bif types.bif functions.bif)
+bro_plugin_end()

src/file_analysis/analyzer/x509/Plugin.cc

@@ -0,0 +1,11 @@
+#include "plugin/Plugin.h"
+
+#include "X509.h"
+
+BRO_PLUGIN_BEGIN(Bro, X509)
+	BRO_PLUGIN_DESCRIPTION("X509 certificate parser");
+	BRO_PLUGIN_FILE_ANALYZER("X509", X509);
+	BRO_PLUGIN_BIF_FILE(events);
+	BRO_PLUGIN_BIF_FILE(types);
+	BRO_PLUGIN_BIF_FILE(functions);
+BRO_PLUGIN_END

src/file_analysis/analyzer/x509/X509.cc

@@ -0,0 +1,587 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#include <string>
+
+#include "X509.h"
+#include "Event.h"
+
+#include "events.bif.h"
+#include "types.bif.h"
+
+#include "file_analysis/Manager.h"
+
+#include <openssl/x509.h>
+#include <openssl/x509v3.h>
+#include <openssl/asn1.h>
+#include <openssl/opensslconf.h>
+
+using namespace file_analysis;
+
+IMPLEMENT_SERIAL(X509Val, SER_X509_VAL);
+
+file_analysis::X509::X509(RecordVal* args, file_analysis::File* file)
+	: file_analysis::Analyzer(file_mgr->GetComponentTag("X509"), args, file)
+	{
+	cert_data.clear();
+	}
+
+bool file_analysis::X509::DeliverStream(const u_char* data, uint64 len)
+	{
+	// just add it to the data we have so far, since we cannot do anything else anyways...
+	cert_data.append(reinterpret_cast<const char*>(data), len);
+	return true;
+	}
+
+bool file_analysis::X509::Undelivered(uint64 offset, uint64 len)
+	{
+	return false;
+	}
+
+bool file_analysis::X509::EndOfFile()
+	{
+	// ok, now we can try to parse the certificate with openssl. Should
+	// be rather straightforward...
+	const unsigned char* cert_char = reinterpret_cast<const unsigned char*>(cert_data.data());
+
+	::X509* ssl_cert = d2i_X509(NULL, &cert_char, cert_data.size());
+	if ( ! ssl_cert )
+		{
+		reporter->Error("Could not parse X509 certificate (fuid %s)", GetFile()->GetID().c_str());
+		return false;
+		}
+
+	X509Val* cert_val = new X509Val(ssl_cert); // cert_val takes ownership of ssl_cert
+
+	RecordVal* cert_record = ParseCertificate(cert_val); // parse basic information into record
+
+	// and send the record on to scriptland
+	val_list* vl = new val_list();
+	vl->append(GetFile()->GetVal()->Ref());
+	vl->append(cert_val->Ref());
+	vl->append(cert_record->Ref()); // we Ref it here, because we want to keep a copy around for now...
+	mgr.QueueEvent(x509_certificate, vl);
+
+	// after parsing the certificate - parse the extensions...
+
+	int num_ext = X509_get_ext_count(ssl_cert);
+	for ( int k = 0; k < num_ext; ++k )
+		{
+		X509_EXTENSION* ex = X509_get_ext(ssl_cert, k);
+		if ( ! ex )
+			continue;
+
+		ParseExtension(ex);
+		}
+
+	// X509_free(ssl_cert); We do _not_ free the certificate here. It is refcounted
+	// inside the X509Val that is sent on in the cert record to scriptland.
+	//
+	// The certificate will be freed when the last X509Val is Unref'd.
+
+	Unref(cert_record); // Unref the RecordVal that we kept around from ParseCertificate
+	Unref(cert_val); // Same for cert_val
+
+	return false;
+	}
+
+RecordVal* file_analysis::X509::ParseCertificate(X509Val* cert_val)
+	{
+	::X509* ssl_cert = cert_val->GetCertificate();
+
+	char buf[256]; // we need a buffer for some of the openssl functions
+	memset(buf, 0, sizeof(buf));
+
+	RecordVal* pX509Cert = new RecordVal(BifType::Record::X509::Certificate);
+	BIO *bio = BIO_new(BIO_s_mem());
+
+	pX509Cert->Assign(0, new Val((uint64) X509_get_version(ssl_cert), TYPE_COUNT));
+	i2a_ASN1_INTEGER(bio, X509_get_serialNumber(ssl_cert));
+	int len = BIO_read(bio, &(*buf), sizeof(buf));
+	pX509Cert->Assign(1, new StringVal(len, buf));
+
+	X509_NAME_print_ex(bio, X509_get_subject_name(ssl_cert), 0, XN_FLAG_RFC2253);
+	len = BIO_gets(bio, &(*buf), sizeof(buf));
+	pX509Cert->Assign(2, new StringVal(len, buf));
+	X509_NAME_print_ex(bio, X509_get_issuer_name(ssl_cert), 0, XN_FLAG_RFC2253);
+	len = BIO_gets(bio, &(*buf), sizeof(buf));
+	pX509Cert->Assign(3, new StringVal(len, buf));
+	BIO_free(bio);
+
+	pX509Cert->Assign(4, new Val(GetTimeFromAsn1(X509_get_notBefore(ssl_cert)), TYPE_TIME));
+	pX509Cert->Assign(5, new Val(GetTimeFromAsn1(X509_get_notAfter(ssl_cert)), TYPE_TIME));
+
+	// we only read 255 bytes because byte 256 is always 0.
+	// if the string is longer than 255, that will be our null-termination,
+	// otherwhise i2t does null-terminate.
+	if ( ! i2t_ASN1_OBJECT(buf, 255, ssl_cert->cert_info->key->algor->algorithm) )
+		buf[0] = 0;
+
+	pX509Cert->Assign(6, new StringVal(buf));
+
+	if ( ! i2t_ASN1_OBJECT(buf, 255, ssl_cert->sig_alg->algorithm) )
+		buf[0] = 0;
+
+	pX509Cert->Assign(7, new StringVal(buf));
+
+	// Things we can do when we have the key...
+	EVP_PKEY *pkey = X509_extract_key(ssl_cert);
+	if ( pkey != NULL )
+		{
+		if ( pkey->type == EVP_PKEY_DSA )
+			pX509Cert->Assign(8, new StringVal("dsa"));
+
+		else if ( pkey->type == EVP_PKEY_RSA )
+			{
+			pX509Cert->Assign(8, new StringVal("rsa"));
+
+			char *exponent = BN_bn2dec(pkey->pkey.rsa->e);
+			if ( exponent != NULL )
+				{
+				pX509Cert->Assign(10, new StringVal(exponent));
+				OPENSSL_free(exponent);
+				exponent = NULL;
+				}
+			}
+#ifndef OPENSSL_NO_EC
+		else if ( pkey->type == EVP_PKEY_EC )
+			{
+			pX509Cert->Assign(8, new StringVal("dsa"));
+			pX509Cert->Assign(11, KeyCurve(pkey));
+			}
+#endif
+
+		unsigned int length = KeyLength(pkey);
+		if ( length > 0 )
+			pX509Cert->Assign(9, new Val(length, TYPE_COUNT));
+		}
+
+
+	return pX509Cert;
+	}
+
+void file_analysis::X509::ParseExtension(X509_EXTENSION* ex)
+	{
+	char name[256];
+	char oid[256];
+
+	ASN1_OBJECT* ext_asn = X509_EXTENSION_get_object(ex);
+	const char* short_name = OBJ_nid2sn(OBJ_obj2nid(ext_asn));
+
+	OBJ_obj2txt(name, 255, ext_asn, 0);
+	OBJ_obj2txt(oid, 255, ext_asn, 1);
+
+	int critical = 0;
+	if ( X509_EXTENSION_get_critical(ex) != 0 )
+		critical = 1;
+
+	BIO *bio = BIO_new(BIO_s_mem());
+	if( ! X509V3_EXT_print(bio, ex, 0, 0))
+		M_ASN1_OCTET_STRING_print(bio,ex->value);
+
+	BIO_flush(bio);
+	int length = BIO_pending(bio);
+
+	// Use OPENSSL_malloc here. Using new or anything else can lead
+	// to interesting, hard to debug segfaults.
+	char *buffer = (char*) OPENSSL_malloc(length);
+	BIO_read(bio, (void*)buffer, length);
+	StringVal* ext_val = new StringVal(length, buffer);
+	OPENSSL_free(buffer);
+	BIO_free_all(bio);
+
+	RecordVal* pX509Ext = new RecordVal(BifType::Record::X509::Extension);
+	pX509Ext->Assign(0, new StringVal(name));
+
+	if ( short_name and strlen(short_name) > 0 )
+		pX509Ext->Assign(1, new StringVal(short_name));
+
+	pX509Ext->Assign(2, new StringVal(oid));
+	pX509Ext->Assign(3, new Val(critical, TYPE_BOOL));
+	pX509Ext->Assign(4, ext_val);
+
+	// send off generic extension event
+	//
+	// and then look if we have a specialized event for the extension we just
+	// parsed. And if we have it, we send the specialized event on top of the
+	// generic event that we just had. I know, that is... kind of not nice,
+	// but I am not sure if there is a better way to do it...
+	val_list* vl = new val_list();
+	vl->append(GetFile()->GetVal()->Ref());
+	vl->append(pX509Ext);
+
+	mgr.QueueEvent(x509_extension, vl);
+
+	// look if we have a specialized handler for this event...
+	if ( OBJ_obj2nid(ext_asn) == NID_basic_constraints )
+		ParseBasicConstraints(ex);
+
+	else if ( OBJ_obj2nid(ext_asn) == NID_subject_alt_name )
+		ParseSAN(ex);
+	}
+
+void file_analysis::X509::ParseBasicConstraints(X509_EXTENSION* ex)
+	{
+	assert(OBJ_obj2nid(X509_EXTENSION_get_object(ex)) == NID_basic_constraints);
+
+	RecordVal* pBasicConstraint = new RecordVal(BifType::Record::X509::BasicConstraints);
+	BASIC_CONSTRAINTS *constr = (BASIC_CONSTRAINTS *) X509V3_EXT_d2i(ex);
+
+	if ( constr )
+		{
+		pBasicConstraint->Assign(0, new Val(constr->ca ? 1 : 0, TYPE_BOOL));
+
+		if ( constr->pathlen )
+			pBasicConstraint->Assign(1, new Val((int32_t) ASN1_INTEGER_get(constr->pathlen), TYPE_COUNT));
+
+		val_list* vl = new val_list();
+		vl->append(GetFile()->GetVal()->Ref());
+		vl->append(pBasicConstraint);
+
+		mgr.QueueEvent(x509_ext_basic_constraints, vl);
+		}
+
+	else
+		reporter->Error("Certificate with invalid BasicConstraint. fuid %s", GetFile()->GetID().c_str());
+	}
+
+void file_analysis::X509::ParseSAN(X509_EXTENSION* ext)
+	{
+	assert(OBJ_obj2nid(X509_EXTENSION_get_object(ext)) == NID_subject_alt_name);
+
+	GENERAL_NAMES *altname = (GENERAL_NAMES*)X509V3_EXT_d2i(ext);
+	if ( ! altname )
+		{
+		reporter->Error("Could not parse subject alternative names. fuid %s", GetFile()->GetID().c_str());
+		return;
+		}
+
+	VectorVal* names = 0;
+	VectorVal* emails = 0;
+	VectorVal* uris = 0;
+	VectorVal* ips = 0;
+
+	unsigned int otherfields = 0;
+
+	for ( int i = 0; i < sk_GENERAL_NAME_num(altname); i++ )
+		{
+		GENERAL_NAME *gen = sk_GENERAL_NAME_value(altname, i);
+		assert(gen);
+
+		if ( gen->type == GEN_DNS || gen->type == GEN_URI || gen->type == GEN_EMAIL )
+			{
+			if ( ASN1_STRING_type(gen->d.ia5) != V_ASN1_IA5STRING )
+				{
+				reporter->Error("DNS-field does not contain an IA5String. fuid %s", GetFile()->GetID().c_str());
+				continue;
+				}
+
+			const char* name = (const char*) ASN1_STRING_data(gen->d.ia5);
+			StringVal* bs = new StringVal(name);
+
+			switch ( gen->type )
+				{
+				case GEN_DNS:
+					if ( names == 0 )
+						names = new VectorVal(internal_type("string_vec")->AsVectorType());
+
+					names->Assign(names->Size(), bs);
+					break;
+
+				case GEN_URI:
+					if ( uris == 0 )
+						uris = new VectorVal(internal_type("string_vec")->AsVectorType());
+
+					uris->Assign(uris->Size(), bs);
+					break;
+
+				case GEN_EMAIL:
+					if ( emails == 0 )
+						emails = new VectorVal(internal_type("string_vec")->AsVectorType());
+
+					emails->Assign(emails->Size(), bs);
+					break;
+				}
+			}
+
+		else if ( gen->type == GEN_IPADD )
+			{
+				if ( ips == 0 )
+					ips = new VectorVal(internal_type("addr_vec")->AsVectorType());
+
+				uint32* addr = (uint32*) gen->d.ip->data;
+
+				if( gen->d.ip->length == 4 )
+					ips->Assign(ips->Size(), new AddrVal(*addr));
+
+				else if ( gen->d.ip->length == 16 )
+					ips->Assign(ips->Size(), new AddrVal(addr));
+
+				else
+					{
+					reporter->Error("Weird IP address length %d in subject alternative name. fuid %s", gen->d.ip->length, GetFile()->GetID().c_str());
+					continue;
+					}
+			}
+
+		else
+			{
+			// reporter->Error("Subject alternative name contained unsupported fields. fuid %s", GetFile()->GetID().c_str());
+			// This happens quite often - just mark it
+			otherfields = 1;
+			continue;
+			}
+		}
+
+		RecordVal* sanExt = new RecordVal(BifType::Record::X509::SubjectAlternativeName);
+
+		if ( names != 0 )
+			sanExt->Assign(0, names);
+
+		if ( uris != 0 )
+			sanExt->Assign(1, uris);
+
+		if ( emails != 0 )
+			sanExt->Assign(2, emails);
+
+		if ( ips != 0 )
+			sanExt->Assign(3, ips);
+
+		sanExt->Assign(4, new Val(otherfields, TYPE_BOOL));
+
+		val_list* vl = new val_list();
+		vl->append(GetFile()->GetVal()->Ref());
+		vl->append(sanExt);
+		mgr.QueueEvent(x509_ext_subject_alternative_name, vl);
+	}
+
+StringVal* file_analysis::X509::KeyCurve(EVP_PKEY *key)
+	{
+	assert(key != NULL);
+
+#ifdef OPENSSL_NO_EC
+	// well, we do not have EC-Support...
+	return NULL;
+#else
+	if ( key->type != EVP_PKEY_EC )
+		{
+		// no EC-key - no curve name
+		return NULL;
+		}
+
+	const EC_GROUP *group;
+	int nid;
+	if ( (group = EC_KEY_get0_group(key->pkey.ec)) == NULL)
+		// I guess we could not parse this
+		return NULL;
+
+	nid = EC_GROUP_get_curve_name(group);
+	if ( nid == 0 )
+		// and an invalid nid...
+		return NULL;
+
+	const char * curve_name = OBJ_nid2sn(nid);
+	if ( curve_name == NULL )
+		return NULL;
+
+	return new StringVal(curve_name);
+#endif
+	}
+
+unsigned int file_analysis::X509::KeyLength(EVP_PKEY *key)
+	{
+	assert(key != NULL);
+
+	switch(key->type) {
+	case EVP_PKEY_RSA:
+		return BN_num_bits(key->pkey.rsa->n);
+
+	case EVP_PKEY_DSA:
+		return BN_num_bits(key->pkey.dsa->p);
+
+#ifndef OPENSSL_NO_EC
+	case EVP_PKEY_EC:
+		{
+		BIGNUM* ec_order = BN_new();
+		if ( ! ec_order )
+			// could not malloc bignum?
+			return 0;
+
+		const EC_GROUP *group = EC_KEY_get0_group(key->pkey.ec);
+		if ( ! group )
+			// unknown ex-group
+			return 0;
+
+		if ( ! EC_GROUP_get_order(group, ec_order, NULL) )
+			// could not get ec-group-order
+			return 0;
+
+		unsigned int length = BN_num_bits(ec_order);
+		BN_free(ec_order);
+		return length;
+		}
+#endif
+	default:
+		return 0; // unknown public key type
+	}
+
+	reporter->InternalError("cannot be reached");
+	}
+
+double file_analysis::X509::GetTimeFromAsn1(const ASN1_TIME* atime)
+	{
+	time_t lResult = 0;
+
+	char lBuffer[24];
+	char* pBuffer = lBuffer;
+
+	size_t lTimeLength = atime->length;
+	char * pString = (char *) atime->data;
+
+	if ( atime->type == V_ASN1_UTCTIME )
+		{
+		if ( lTimeLength < 11 || lTimeLength > 17 )
+			return 0;
+
+		memcpy(pBuffer, pString, 10);
+		pBuffer += 10;
+		pString += 10;
+		}
+
+	else
+		{
+		if ( lTimeLength < 13 )
+			return 0;
+
+		memcpy(pBuffer, pString, 12);
+		pBuffer += 12;
+		pString += 12;
+		}
+
+	if ((*pString == 'Z') || (*pString == '-') || (*pString == '+'))
+		{
+		*(pBuffer++) = '0';
+		*(pBuffer++) = '0';
+		}
+
+	else
+		{
+		*(pBuffer++) = *(pString++);
+		*(pBuffer++) = *(pString++);
+
+		// Skip any fractional seconds...
+		if (*pString == '.')
+			{
+			pString++;
+			while ((*pString >= '0') && (*pString <= '9'))
+				pString++;
+			}
+		}
+
+	*(pBuffer++) = 'Z';
+	*(pBuffer++) = '\0';
+
+	time_t lSecondsFromUTC;
+
+	if ( *pString == 'Z' )
+		lSecondsFromUTC = 0;
+
+	else
+		{
+		if ((*pString != '+') && (pString[5] != '-'))
+			return 0;
+
+		lSecondsFromUTC = ((pString[1]-'0') * 10 + (pString[2]-'0')) * 60;
+		lSecondsFromUTC += (pString[3]-'0') * 10 + (pString[4]-'0');
+
+		if (*pString == '-')
+			lSecondsFromUTC = -lSecondsFromUTC;
+		}
+
+	tm lTime;
+	lTime.tm_sec  = ((lBuffer[10] - '0') * 10) + (lBuffer[11] - '0');
+	lTime.tm_min  = ((lBuffer[8] - '0') * 10) + (lBuffer[9] - '0');
+	lTime.tm_hour = ((lBuffer[6] - '0') * 10) + (lBuffer[7] - '0');
+	lTime.tm_mday = ((lBuffer[4] - '0') * 10) + (lBuffer[5] - '0');
+	lTime.tm_mon  = (((lBuffer[2] - '0') * 10) + (lBuffer[3] - '0')) - 1;
+	lTime.tm_year = ((lBuffer[0] - '0') * 10) + (lBuffer[1] - '0');
+
+	if ( lTime.tm_year < 50 )
+		lTime.tm_year += 100; // RFC 2459
+
+	lTime.tm_wday = 0;
+	lTime.tm_yday = 0;
+	lTime.tm_isdst = 0;  // No DST adjustment requested
+
+	lResult = mktime(&lTime);
+
+	if ( lResult )
+		{
+		if ( 0 != lTime.tm_isdst )
+			lResult -= 3600;  // mktime may adjust for DST  (OS dependent)
+
+		lResult += lSecondsFromUTC;
+		}
+
+	else
+		lResult = 0;
+
+	return lResult;
+}
+
+X509Val::X509Val(::X509* arg_certificate) : OpaqueVal(x509_opaque_type)
+	{
+	certificate = arg_certificate;
+	}
+
+X509Val::X509Val() : OpaqueVal(x509_opaque_type)
+	{
+	certificate = 0;
+	}
+
+X509Val::~X509Val()
+	{
+	if ( certificate )
+		X509_free(certificate);
+	}
+
+::X509* X509Val::GetCertificate() const
+	{
+	return certificate;
+	}
+
+bool X509Val::DoSerialize(SerialInfo* info) const
+	{
+	DO_SERIALIZE(SER_X509_VAL, OpaqueVal);
+
+	unsigned char *buf = NULL;
+
+	int length = i2d_X509(certificate, &buf);
+
+	if ( length < 0 )
+		return false;
+
+	bool res = SERIALIZE_STR(reinterpret_cast<const char*>(buf), length);
+
+	OPENSSL_free(buf);
+	return res;
+	}
+
+bool X509Val::DoUnserialize(UnserialInfo* info)
+	{
+	DO_UNSERIALIZE(OpaqueVal)
+
+	int length;
+	unsigned char *certbuf, *opensslbuf;
+
+	if ( ! UNSERIALIZE_STR(reinterpret_cast<char **>(&certbuf), &length) )
+		return false;
+
+	opensslbuf = certbuf; // OpenSSL likes to shift pointers around. really.
+	certificate = d2i_X509(NULL, const_cast<const unsigned char**>(&opensslbuf), length);
+	delete[] certbuf;
+
+	if ( !certificate )
+		return false;
+
+	return true;
+	}

src/file_analysis/analyzer/x509/X509.h

@@ -0,0 +1,101 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#ifndef FILE_ANALYSIS_X509_H
+#define FILE_ANALYSIS_X509_H
+
+#include <string>
+
+#include "Val.h"
+#include "../File.h"
+#include "Analyzer.h"
+
+#include <openssl/x509.h>
+#include <openssl/asn1.h>
+
+namespace file_analysis {
+
+class X509Val;
+
+class X509 : public file_analysis::Analyzer {
+public:
+	virtual bool DeliverStream(const u_char* data, uint64 len);
+	virtual bool Undelivered(uint64 offset, uint64 len);
+	virtual bool EndOfFile();
+
+	/**
+	 * Converts an X509 certificate into a \c X509::Certificate record
+	 * value. This is a static function that can be called from external,
+	 * it doesn't depend on the state of any particular file analyzer.
+	 *
+	 * @param cert_val The certificate to converts.
+	 *
+	 * @param Returns the new record value and passes ownership to
+	 * caller.
+	 */
+	static RecordVal* ParseCertificate(X509Val* cert_val);
+
+	static file_analysis::Analyzer* Instantiate(RecordVal* args, File* file)
+		{ return new X509(args, file); }
+
+protected:
+	X509(RecordVal* args, File* file);
+
+private:
+	void ParseExtension(X509_EXTENSION* ex);
+	void ParseBasicConstraints(X509_EXTENSION* ex);
+	void ParseSAN(X509_EXTENSION* ex);
+
+	std::string cert_data;
+
+	// Helpers for ParseCertificate.
+	static double GetTimeFromAsn1(const ASN1_TIME * atime);
+	static StringVal* KeyCurve(EVP_PKEY *key);
+	static unsigned int KeyLength(EVP_PKEY *key);
+};
+
+/**
+ * This class wraps an OpenSSL X509 data structure.
+ *
+ * We need these to be able to pass OpenSSL pointers around in Bro
+ * script-land. Otherwise, we cannot verify certificates from Bro
+ * scriptland
+ */
+class X509Val : public OpaqueVal {
+public:
+	/**
+	 * Construct an X509Val.
+	 *
+	 * @param certificate specifies the wrapped OpenSSL certificate
+	 *
+	 * @return A newly initialized X509Val.
+	 */
+	explicit X509Val(::X509* certificate);
+
+	/**
+	 * Destructor.
+	 */
+	~X509Val();
+
+	/**
+	 * Get the wrapped X509 certificate. Please take care, that the
+	 * internal OpenSSL reference counting stays the same.
+	 *
+	 * @return The wrapped OpenSSL X509 certificate.
+	 */
+	::X509* GetCertificate() const;
+
+protected:
+	/**
+	 * Construct an empty X509Val. Only used for deserialization
+	 */
+	X509Val();
+
+private:
+	::X509* certificate; // the wrapped certificate
+
+	DECLARE_SERIAL(X509Val);
+};
+
+}
+
+#endif

src/file_analysis/analyzer/x509/events.bif

@@ -0,0 +1,57 @@
+## Generated for encountered X509 certificates, e.g., in the clear SSL/TLS 
+## connection handshake.
+##
+## See `Wikipedia <http://en.wikipedia.org/wiki/X.509>`__ for more information
+## about the X.509 format.
+##
+## f: The file.
+##
+## cert_ref: An opaque pointer to the underlying OpenSSL data structure of the
+##           certificate.
+##
+## cert: The parsed certificate information.
+##
+## .. bro:see:: x509_extension x509_ext_basic_constraints
+##              x509_ext_subject_alternative_name x509_parse x509_verify
+##              x509_get_certificate_string
+event x509_certificate%(f: fa_file, cert_ref: opaque of x509, cert: X509::Certificate%);
+
+## Generated for X509 extensions seen in a certificate.
+##
+## See `Wikipedia <http://en.wikipedia.org/wiki/X.509>`__ for more information
+## about the X.509 format.
+##
+## f: The file.
+##
+## ext: The parsed extension.
+##
+## .. bro:see:: x509_certificate x509_ext_basic_constraints
+##              x509_ext_subject_alternative_name x509_parse x509_verify
+##              x509_get_certificate_string
+event x509_extension%(f: fa_file, ext: X509::Extension%);
+
+## Generated for the X509 basic constraints extension seen in a certificate.
+## This extension can be used to identify the subject of a certificate as a CA.
+##
+## f: The file.
+##
+## ext: The parsed basic constraints extension.
+##
+## .. bro:see:: x509_certificate x509_extension
+##              x509_ext_subject_alternative_name x509_parse x509_verify
+##              x509_get_certificate_string
+event x509_ext_basic_constraints%(f: fa_file, ext: X509::BasicConstraints%);
+
+## Generated for the X509 subject alternative name extension seen in a certificate.
+## This extension can be used to allow additional entities to be bound to the subject
+## of the certificate. Usually it is used to specify one or multiple DNS names for 
+## which a certificate is valid.
+##
+## f: The file.
+##
+## ext: The parsed subject alternative name extension.
+##
+## .. bro:see:: x509_certificate x509_extension x509_ext_basic_constraints
+##              x509_parse x509_verify
+##              x509_get_certificate_string
+event x509_ext_subject_alternative_name%(f: fa_file, ext: X509::SubjectAlternativeName%);

src/file_analysis/analyzer/x509/functions.bif

@@ -0,0 +1,245 @@
+%%{
+#include "file_analysis/analyzer/x509/X509.h"
+#include "types.bif.h"
+
+#include <openssl/x509.h>
+#include <openssl/asn1.h>
+#include <openssl/x509_vfy.h>
+
+// This is the indexed map of X509 certificate stores.
+static map<Val*, X509_STORE*> x509_stores;
+
+// ### NOTE: while d2i_X509 does not take a const u_char** pointer,
+// here we assume d2i_X509 does not write to <data>, so it is safe to
+// convert data to a non-const pointer.  Could some X509 guru verify
+// this?
+
+X509* d2i_X509_(X509** px, const u_char** in, int len)
+	{
+#ifdef OPENSSL_D2I_X509_USES_CONST_CHAR
+	  return d2i_X509(px, in, len);
+#else
+	  return d2i_X509(px, (u_char**)in, len);
+#endif
+	}
+
+// construct an error record
+RecordVal* x509_error_record(uint64_t num, const char* reason)
+	{
+	RecordVal* rrecord = new RecordVal(BifType::Record::X509::Result);
+
+	rrecord->Assign(0, new Val(num, TYPE_COUNT));
+	rrecord->Assign(1, new StringVal(reason));
+
+	return rrecord;
+	}
+
+%%}
+
+## Parses a certificate into an X509::Certificate structure.
+##
+## cert: The X509 certificicate opaque handle
+##
+## Returns: A X509::Certificate structure
+##
+## .. bro:see:: x509_certificate x509_extension x509_ext_basic_constraints
+##              x509_ext_subject_alternative_name x509_verify
+##              x509_get_certificate_string
+function x509_parse%(cert: opaque of x509%): X509::Certificate
+	%{
+	assert(cert);
+	file_analysis::X509Val* h = (file_analysis::X509Val*) cert;
+
+	return file_analysis::X509::ParseCertificate(h);
+	%}
+
+## Returns the string form of a certificate.
+##
+## cert: The X509 certificate opaque handle
+##
+## pem: A boolean that specifies if the certificate is returned
+##      in pem-form (true), or as the raw ASN1 encoded binary
+##      (false).
+##
+## Returns: X509 certificate as a string
+##
+## .. bro:see:: x509_certificate x509_extension x509_ext_basic_constraints
+##              x509_ext_subject_alternative_name x509_parse x509_verify
+function x509_get_certificate_string%(cert: opaque of x509, pem: bool &default=F%): string
+	%{
+	assert(cert);
+	file_analysis::X509Val* h = (file_analysis::X509Val*) cert;
+
+	BIO *bio = BIO_new(BIO_s_mem());
+
+	if ( pem )
+		PEM_write_bio_X509(bio, h->GetCertificate());
+
+	else
+		i2d_X509_bio(bio, h->GetCertificate());
+
+	BIO_flush(bio);
+	int length = BIO_pending(bio);
+	// use OPENSS_malloc here. Otherwhise, interesting problems will happen.
+	char *buffer = (char*) OPENSSL_malloc(length);
+	BIO_read(bio, (void*) buffer, length);
+	StringVal* ext_val = new StringVal(length, buffer);
+	OPENSSL_free(buffer);
+	BIO_free_all(bio);
+
+	return ext_val;
+	%}
+
+
+## Verifies a certificate.
+##
+## certs: Specifies a certificate chain that is being used to validate
+##        the given certificate against the root store given in *root_certs*.
+##        The host certificate has to be at index 0.
+##
+## root_certs: A list of root certificates to validate the certificate chain
+##
+## verify_time: Time for the validity check of the certificates.
+##
+## Returns: A record of type X509::Result containing the result code of the verify
+##          operation. In case of success also returns the full certificate chain.
+##
+## .. bro:see:: x509_certificate x509_extension x509_ext_basic_constraints
+##              x509_ext_subject_alternative_name x509_parse
+##              x509_get_certificate_string
+function x509_verify%(certs: x509_opaque_vector, root_certs: table_string_of_string, verify_time: time &default=network_time()%): X509::Result
+	%{
+	X509_STORE* ctx = 0;
+	int i = 0;
+
+	VectorVal *certs_vec = certs->AsVectorVal();
+	if ( certs_vec->Size() < 1 )
+		{
+		reporter->Error("No certificates given in vector");
+		return x509_error_record(-1, "no certificates");
+		}
+
+	// host certificate
+	unsigned int index = 0; // to prevent overloading to 0pointer
+	Val *sv = certs_vec->Lookup(index);
+	if ( !sv )
+		{
+		builtin_error("undefined value in certificate vector");
+		return x509_error_record(-1, "undefined value in certificate vector");
+		}
+
+	file_analysis::X509Val* cert_handle = (file_analysis::X509Val*) sv;
+
+	// If this certificate store was built previously, just reuse the old one.
+	if ( x509_stores.count(root_certs) > 0 )
+		ctx = x509_stores[root_certs];
+
+	if ( ! ctx ) // lookup to see if we have this one built already!
+		{
+		ctx = X509_STORE_new();
+		TableVal* root_certs2 = root_certs->AsTableVal();
+		ListVal* idxs = root_certs2->ConvertToPureList();
+
+		// Build the validation store
+		for ( i = 0; i < idxs->Length(); ++i )
+			{
+			Val* key = idxs->Index(i);
+			StringVal *sv = root_certs2->Lookup(key)->AsStringVal();
+			const uint8* data = sv->Bytes();
+			X509* x = d2i_X509_(NULL, &data, sv->Len());
+			if ( ! x )
+				{
+				builtin_error(fmt("Root CA error: %s", ERR_error_string(ERR_peek_last_error(),NULL)));
+				return x509_error_record((uint64) ERR_get_error(), ERR_error_string(ERR_peek_last_error(),NULL));
+				}
+
+			X509_STORE_add_cert(ctx, x);
+			}
+
+		delete idxs;
+
+		// Save the newly constructed certificate store into the cacheing map.
+		x509_stores[root_certs] = ctx;
+		}
+
+	X509* cert = cert_handle->GetCertificate();
+	if ( ! cert )
+		{
+		builtin_error(fmt("No certificate in opaque"));
+		return x509_error_record(-1, "No certificate in opaque");
+		}
+
+	STACK_OF(X509)* untrusted_certs = sk_X509_new_null();
+	if ( ! untrusted_certs )
+		{
+		builtin_error(fmt("Untrusted certificate stack initialization error: %s", ERR_error_string(ERR_peek_last_error(),NULL)));
+		return x509_error_record((uint64) ERR_get_error(), ERR_error_string(ERR_peek_last_error(),NULL));
+		}
+
+	for ( i = 1; i < (int) certs_vec->Size(); ++i ) // start at 1 - 0 is host cert
+		{
+		Val *sv = certs_vec->Lookup(i);
+		// Fixme: check type
+		X509* x = ((file_analysis::X509Val*) sv)->GetCertificate();
+		if ( ! x )
+			{
+			sk_X509_pop(untrusted_certs);
+			builtin_error(fmt("No certificate in opaque in stack"));
+			return x509_error_record(-1, "No certificate in opaque");
+			}
+
+		sk_X509_push(untrusted_certs, x);
+		}
+
+	X509_STORE_CTX csc;
+	X509_STORE_CTX_init(&csc, ctx, cert, untrusted_certs);
+	X509_STORE_CTX_set_time(&csc, 0, (time_t) verify_time);
+	X509_STORE_CTX_set_flags(&csc, X509_V_FLAG_USE_CHECK_TIME);
+
+	int result = X509_verify_cert(&csc);
+
+	VectorVal* chainVector = 0;
+
+	if ( result == 1 ) // we have a valid chain. try to get it...
+		{
+		STACK_OF(X509)* chain = X509_STORE_CTX_get1_chain(&csc); // get1 = deep copy
+
+		if ( ! chain )
+			{
+			reporter->Error("Encountered valid chain that could not be resolved");
+			goto x509_verify_chainerror;
+			}
+
+		int num_certs = sk_X509_num(chain);
+		chainVector = new VectorVal(internal_type("x509_opaque_vector")->AsVectorType());
+
+		for ( int i = 0; i < num_certs; i++ )
+			{
+			X509* currcert = sk_X509_value(chain, i);
+			if ( !currcert )
+				{
+				reporter->InternalError("OpenSSL returned null certificate");
+				goto x509_verify_chainerror;
+				}
+
+			chainVector->Assign(i, new file_analysis::X509Val(currcert)); // X509Val takes ownership
+			}
+		}
+
+x509_verify_chainerror:
+
+	X509_STORE_CTX_cleanup(&csc);
+
+	if ( untrusted_certs )
+		sk_X509_pop(untrusted_certs);
+
+	RecordVal* rrecord = new RecordVal(BifType::Record::X509::Result);
+
+	rrecord->Assign(0, new Val((uint64) csc.error, TYPE_COUNT));
+	rrecord->Assign(1, new StringVal(X509_verify_cert_error_string(csc.error)));
+
+	if ( chainVector )
+		rrecord->Assign(2, chainVector);
+
+	return rrecord;
+	%}

src/file_analysis/analyzer/x509/types.bif

@@ -0,0 +1,5 @@
+type X509::Certificate: record;
+type X509::Extension: record;
+type X509::BasicConstraints: record;
+type X509::SubjectAlternativeName: record;
+type X509::Result: record;

src/main.cc

@@ -130,6 +130,7 @@ OpaqueType* entropy_type = 0;
 OpaqueType* cardinality_type = 0;
 OpaqueType* topk_type = 0;
 OpaqueType* bloomfilter_type = 0;
+OpaqueType* x509_opaque_type = 0;
 
 // Keep copy of command line
 int bro_argc;
@@ -865,6 +866,7 @@ int main(int argc, char** argv)
 	cardinality_type = new OpaqueType("cardinality");
 	topk_type = new OpaqueType("topk");
 	bloomfilter_type = new OpaqueType("bloomfilter");
+	x509_opaque_type = new OpaqueType("x509");
 
 	// The leak-checker tends to produce some false
 	// positives (memory which had already been

testing/btest/Baseline/bifs.x509_verify/.stdout

@@ -0,0 +1,7 @@
+Validation result: certificate has expired
+Validation result: ok
+Resulting chain:
+Fingerprint: 70829f77ff4b6e908324a3f4e1940fce6c489098, Subject: CN=www.tobu-estate.com,OU=Terms of use at www.verisign.com/rpa (c)05,O=TOBU RAILWAY Co.\,Ltd.,L=Sumida-ku,ST=Tokyo,C=JP
+Fingerprint: 5deb8f339e264c19f6686f5f8f32b54a4c46b476, Subject: CN=VeriSign Class 3 Secure Server CA - G3,OU=Terms of use at https://www.verisign.com/rpa (c)10,OU=VeriSign Trust Network,O=VeriSign\, Inc.,C=US
+Fingerprint: 32f30882622b87cf8856c63db873df0853b4dd27, Subject: CN=VeriSign Class 3 Public Primary Certification Authority - G5,OU=(c) 2006 VeriSign\, Inc. - For authorized use only,OU=VeriSign Trust Network,O=VeriSign\, Inc.,C=US
+Fingerprint: 742c3192e607e424eb4549542be1bbc53e6174e2, Subject: OU=Class 3 Public Primary Certification Authority,O=VeriSign\, Inc.,C=US

testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log

@@ -3,7 +3,7 @@
 #empty_field	(empty)
 #unset_field	-
 #path	loaded_scripts
-#open	2013-10-30-16-52-11
+#open	2014-03-04-06-37-10
 #fields	name
 #types	string
 scripts/base/init-bare.bro
@@ -56,7 +56,6 @@ scripts/base/init-bare.bro
     build/scripts/base/bif/plugins/Bro_SOCKS.events.bif.bro
     build/scripts/base/bif/plugins/Bro_SSH.events.bif.bro
     build/scripts/base/bif/plugins/Bro_SSL.events.bif.bro
-    build/scripts/base/bif/plugins/Bro_SSL.functions.bif.bro
     build/scripts/base/bif/plugins/Bro_SteppingStone.events.bif.bro
     build/scripts/base/bif/plugins/Bro_Syslog.events.bif.bro
     build/scripts/base/bif/plugins/Bro_TCP.events.bif.bro
@@ -65,6 +64,9 @@ scripts/base/init-bare.bro
     build/scripts/base/bif/plugins/Bro_UDP.events.bif.bro
     build/scripts/base/bif/plugins/Bro_Unified2.events.bif.bro
     build/scripts/base/bif/plugins/Bro_Unified2.types.bif.bro
+    build/scripts/base/bif/plugins/Bro_X509.events.bif.bro
+    build/scripts/base/bif/plugins/Bro_X509.functions.bif.bro
+    build/scripts/base/bif/plugins/Bro_X509.types.bif.bro
     build/scripts/base/bif/plugins/Bro_ZIP.events.bif.bro
   scripts/base/frameworks/logging/__load__.bro
     scripts/base/frameworks/logging/main.bro
@@ -101,4 +103,4 @@ scripts/base/init-bare.bro
     build/scripts/base/bif/top-k.bif.bro
 scripts/policy/misc/loaded-scripts.bro
   scripts/base/utils/paths.bro
-#close	2013-10-30-16-52-11
+#close	2014-03-04-06-37-10

testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log

@@ -3,7 +3,7 @@
 #empty_field	(empty)
 #unset_field	-
 #path	loaded_scripts
-#open	2014-01-31-22-54-38
+#open	2014-03-13-22-14-10
 #fields	name
 #types	string
 scripts/base/init-bare.bro
@@ -56,7 +56,6 @@ scripts/base/init-bare.bro
     build/scripts/base/bif/plugins/Bro_SOCKS.events.bif.bro
     build/scripts/base/bif/plugins/Bro_SSH.events.bif.bro
     build/scripts/base/bif/plugins/Bro_SSL.events.bif.bro
-    build/scripts/base/bif/plugins/Bro_SSL.functions.bif.bro
     build/scripts/base/bif/plugins/Bro_SteppingStone.events.bif.bro
     build/scripts/base/bif/plugins/Bro_Syslog.events.bif.bro
     build/scripts/base/bif/plugins/Bro_TCP.events.bif.bro
@@ -65,6 +64,9 @@ scripts/base/init-bare.bro
     build/scripts/base/bif/plugins/Bro_UDP.events.bif.bro
     build/scripts/base/bif/plugins/Bro_Unified2.events.bif.bro
     build/scripts/base/bif/plugins/Bro_Unified2.types.bif.bro
+    build/scripts/base/bif/plugins/Bro_X509.events.bif.bro
+    build/scripts/base/bif/plugins/Bro_X509.functions.bif.bro
+    build/scripts/base/bif/plugins/Bro_X509.types.bif.bro
     build/scripts/base/bif/plugins/Bro_ZIP.events.bif.bro
   scripts/base/frameworks/logging/__load__.bro
     scripts/base/frameworks/logging/main.bro
@@ -187,6 +189,11 @@ scripts/base/init-default.bro
         scripts/base/protocols/ssl/consts.bro
         scripts/base/protocols/ssl/main.bro
         scripts/base/protocols/ssl/mozilla-ca-list.bro
+        scripts/base/protocols/ssl/files.bro
+          scripts/base/files/x509/__load__.bro
+            scripts/base/files/x509/main.bro
+              scripts/base/files/hash/__load__.bro
+                scripts/base/files/hash/main.bro
   scripts/base/protocols/http/__load__.bro
     scripts/base/protocols/http/main.bro
     scripts/base/protocols/http/entities.bro
@@ -213,8 +220,6 @@ scripts/base/init-default.bro
     scripts/base/protocols/syslog/consts.bro
     scripts/base/protocols/syslog/main.bro
   scripts/base/protocols/tunnels/__load__.bro
-  scripts/base/files/hash/__load__.bro
-    scripts/base/files/hash/main.bro
   scripts/base/files/extract/__load__.bro
     scripts/base/files/extract/main.bro
   scripts/base/files/unified2/__load__.bro
@@ -222,4 +227,4 @@ scripts/base/init-default.bro
   scripts/base/misc/find-checksum-offloading.bro
   scripts/base/misc/find-filtered-trace.bro
 scripts/policy/misc/loaded-scripts.bro
-#close	2014-01-31-22-54-38
+#close	2014-03-13-22-14-10

testing/btest/Baseline/scripts.base.protocols.ftp.gridftp/ssl.log

@@ -3,9 +3,9 @@
 #empty_field	(empty)
 #unset_field	-
 #path	ssl
-#open	2014-03-04-22-24-11
+#open	2014-03-13-20-45-24
-#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	version	cipher	server_name	session_id	subject	issuer_subject	not_valid_before	not_valid_after	last_alert	client_subject	client_issuer_subject	established
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	version	cipher	server_name	session_id	last_alert	established	cert_chain_fuids	client_cert_chain_fuids	subject	issuer	client_subject	client_issuer
-#types	time	string	addr	port	addr	port	string	string	string	string	string	string	time	time	string	string	string	bool
+#types	time	string	addr	port	addr	port	string	string	string	string	string	bool	vector[string]	vector[string]	string	string	string	string
-1348168976.508038	CXWv6p3arKYeMETxOg	192.168.57.103	60108	192.168.57.101	2811	TLSv10	TLS_RSA_WITH_AES_256_CBC_SHA	-	-	CN=host/alpha,OU=simpleCA-alpha,OU=GlobusTest,O=Grid	CN=Globus Simple CA,OU=simpleCA-alpha,OU=GlobusTest,O=Grid	1348161979.000000	1379697979.000000	-	CN=917532944,CN=Jon Siwek,OU=local,OU=simpleCA-alpha,OU=GlobusTest,O=Grid	CN=Jon Siwek,OU=local,OU=simpleCA-alpha,OU=GlobusTest,O=Grid	T
+1348168976.508038	CXWv6p3arKYeMETxOg	192.168.57.103	60108	192.168.57.101	2811	TLSv10	TLS_RSA_WITH_AES_256_CBC_SHA	-	-	-	T	FBtbj87tgpyeDSj31,F8TfgZ31c1dFu8Kt2k	FVNYOh2BeQBb7MpCPe,FwjBou1e5DbpE0eOgk,FbYQmk4x4M4Bx3PZme	CN=host/alpha,OU=simpleCA-alpha,OU=GlobusTest,O=Grid	CN=Globus Simple CA,OU=simpleCA-alpha,OU=GlobusTest,O=Grid	CN=917532944,CN=Jon Siwek,OU=local,OU=simpleCA-alpha,OU=GlobusTest,O=Grid	CN=Jon Siwek,OU=local,OU=simpleCA-alpha,OU=GlobusTest,O=Grid
-1348168976.551422	CjhGID4nQcgTWjvg4c	192.168.57.103	35391	192.168.57.101	55968	TLSv10	TLS_RSA_WITH_NULL_SHA	-	-	CN=932373381,CN=917532944,CN=Jon Siwek,OU=local,OU=simpleCA-alpha,OU=GlobusTest,O=Grid	CN=917532944,CN=Jon Siwek,OU=local,OU=simpleCA-alpha,OU=GlobusTest,O=Grid	1348168676.000000	1348206441.000000	-	CN=917532944,CN=Jon Siwek,OU=local,OU=simpleCA-alpha,OU=GlobusTest,O=Grid	CN=Jon Siwek,OU=local,OU=simpleCA-alpha,OU=GlobusTest,O=Grid	T
+1348168976.551422	CjhGID4nQcgTWjvg4c	192.168.57.103	35391	192.168.57.101	55968	TLSv10	TLS_RSA_WITH_NULL_SHA	-	-	-	T	F4SSqN31HDIrrH5Q8h,FJHp5Pf6VLQsRQK3,FHACqa3dX9BXRV2av,FNnDVT1NURRWeoLLN3	FFWYVj4BcvQb35WIaf,Fj16G835fnJgnVlKU6,FGONoc1Nj0Ka5zlxDa	CN=932373381,CN=917532944,CN=Jon Siwek,OU=local,OU=simpleCA-alpha,OU=GlobusTest,O=Grid	CN=917532944,CN=Jon Siwek,OU=local,OU=simpleCA-alpha,OU=GlobusTest,O=Grid	CN=917532944,CN=Jon Siwek,OU=local,OU=simpleCA-alpha,OU=GlobusTest,O=Grid	CN=Jon Siwek,OU=local,OU=simpleCA-alpha,OU=GlobusTest,O=Grid
-#close	2014-03-04-22-24-11
+#close	2014-03-13-20-45-24

testing/btest/Baseline/scripts.base.protocols.ftp.gridftp/x509.log

@@ -0,0 +1,21 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	x509
+#open	2014-03-13-20-45-24
+#fields	ts	id	certificate.version	certificate.serial	certificate.subject	certificate.issuer	certificate.not_valid_before	certificate.not_valid_after	certificate.key_alg	certificate.sig_alg	certificate.key_type	certificate.key_length	certificate.exponent	certificate.curve	san.dns	san.uri	san.email	san.ip	basic_constraints.ca	basic_constraints.path_len
+#types	time	string	count	string	string	string	time	time	string	string	string	count	string	string	vector[string]	vector[string]	vector[string]	vector[addr]	bool	count
+1348168976.510615	FBtbj87tgpyeDSj31	2	01	CN=host/alpha,OU=simpleCA-alpha,OU=GlobusTest,O=Grid	CN=Globus Simple CA,OU=simpleCA-alpha,OU=GlobusTest,O=Grid	1348161979.000000	1379697979.000000	rsaEncryption	sha1WithRSAEncryption	rsa	1024	65537	-	-	-	-	-	-	-
+1348168976.510615	F8TfgZ31c1dFu8Kt2k	2	EA83D17188B68E4D	CN=Globus Simple CA,OU=simpleCA-alpha,OU=GlobusTest,O=Grid	CN=Globus Simple CA,OU=simpleCA-alpha,OU=GlobusTest,O=Grid	1348161502.000000	1505841502.000000	rsaEncryption	sha1WithRSAEncryption	rsa	1024	65537	-	-	-	-	-	T	-
+1348168976.514202	FVNYOh2BeQBb7MpCPe	2	36B07110	CN=917532944,CN=Jon Siwek,OU=local,OU=simpleCA-alpha,OU=GlobusTest,O=Grid	CN=Jon Siwek,OU=local,OU=simpleCA-alpha,OU=GlobusTest,O=Grid	1348162941.000000	1348206441.000000	rsaEncryption	sha1WithRSAEncryption	rsa	512	65537	-	-	-	-	-	-	-
+1348168976.514202	FwjBou1e5DbpE0eOgk	2	02	CN=Jon Siwek,OU=local,OU=simpleCA-alpha,OU=GlobusTest,O=Grid	CN=Globus Simple CA,OU=simpleCA-alpha,OU=GlobusTest,O=Grid	1348162263.000000	1379698263.000000	rsaEncryption	sha1WithRSAEncryption	rsa	1024	65537	-	-	-	-	-	-	-
+1348168976.514202	FbYQmk4x4M4Bx3PZme	2	EA83D17188B68E4D	CN=Globus Simple CA,OU=simpleCA-alpha,OU=GlobusTest,O=Grid	CN=Globus Simple CA,OU=simpleCA-alpha,OU=GlobusTest,O=Grid	1348161502.000000	1505841502.000000	rsaEncryption	sha1WithRSAEncryption	rsa	1024	65537	-	-	-	-	-	T	-
+1348168976.551554	F4SSqN31HDIrrH5Q8h	2	3792E385	CN=932373381,CN=917532944,CN=Jon Siwek,OU=local,OU=simpleCA-alpha,OU=GlobusTest,O=Grid	CN=917532944,CN=Jon Siwek,OU=local,OU=simpleCA-alpha,OU=GlobusTest,O=Grid	1348168676.000000	1348206441.000000	rsaEncryption	sha1WithRSAEncryption	rsa	512	65537	-	-	-	-	-	-	-
+1348168976.551554	FJHp5Pf6VLQsRQK3	2	36B07110	CN=917532944,CN=Jon Siwek,OU=local,OU=simpleCA-alpha,OU=GlobusTest,O=Grid	CN=Jon Siwek,OU=local,OU=simpleCA-alpha,OU=GlobusTest,O=Grid	1348162941.000000	1348206441.000000	rsaEncryption	sha1WithRSAEncryption	rsa	512	65537	-	-	-	-	-	-	-
+1348168976.551554	FHACqa3dX9BXRV2av	2	02	CN=Jon Siwek,OU=local,OU=simpleCA-alpha,OU=GlobusTest,O=Grid	CN=Globus Simple CA,OU=simpleCA-alpha,OU=GlobusTest,O=Grid	1348162263.000000	1379698263.000000	rsaEncryption	sha1WithRSAEncryption	rsa	1024	65537	-	-	-	-	-	-	-
+1348168976.551554	FNnDVT1NURRWeoLLN3	2	EA83D17188B68E4D	CN=Globus Simple CA,OU=simpleCA-alpha,OU=GlobusTest,O=Grid	CN=Globus Simple CA,OU=simpleCA-alpha,OU=GlobusTest,O=Grid	1348161502.000000	1505841502.000000	rsaEncryption	sha1WithRSAEncryption	rsa	1024	65537	-	-	-	-	-	T	-
+1348168976.554445	FFWYVj4BcvQb35WIaf	2	36B07110	CN=917532944,CN=Jon Siwek,OU=local,OU=simpleCA-alpha,OU=GlobusTest,O=Grid	CN=Jon Siwek,OU=local,OU=simpleCA-alpha,OU=GlobusTest,O=Grid	1348162941.000000	1348206441.000000	rsaEncryption	sha1WithRSAEncryption	rsa	512	65537	-	-	-	-	-	-	-
+1348168976.554445	Fj16G835fnJgnVlKU6	2	02	CN=Jon Siwek,OU=local,OU=simpleCA-alpha,OU=GlobusTest,O=Grid	CN=Globus Simple CA,OU=simpleCA-alpha,OU=GlobusTest,O=Grid	1348162263.000000	1379698263.000000	rsaEncryption	sha1WithRSAEncryption	rsa	1024	65537	-	-	-	-	-	-	-
+1348168976.554445	FGONoc1Nj0Ka5zlxDa	2	EA83D17188B68E4D	CN=Globus Simple CA,OU=simpleCA-alpha,OU=GlobusTest,O=Grid	CN=Globus Simple CA,OU=simpleCA-alpha,OU=GlobusTest,O=Grid	1348161502.000000	1505841502.000000	rsaEncryption	sha1WithRSAEncryption	rsa	1024	65537	-	-	-	-	-	T	-
+#close	2014-03-13-20-45-24

testing/btest/Baseline/scripts.base.protocols.ssl.basic/ssl.log

@@ -3,8 +3,8 @@
 #empty_field	(empty)
 #unset_field	-
 #path	ssl
-#open	2014-03-04-22-02-50
+#open	2014-03-13-20-45-46
-#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	version	cipher	server_name	session_id	subject	issuer_subject	not_valid_before	not_valid_after	last_alert	client_subject	client_issuer_subject	established
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	version	cipher	server_name	session_id	last_alert	established	cert_chain_fuids	client_cert_chain_fuids	subject	issuer	client_subject	client_issuer
-#types	time	string	addr	port	addr	port	string	string	string	string	string	string	time	time	string	string	string	bool
+#types	time	string	addr	port	addr	port	string	string	string	string	string	bool	vector[string]	vector[string]	string	string	string	string
-1335538392.319381	CXWv6p3arKYeMETxOg	192.168.1.105	62045	74.125.224.79	443	TLSv10	TLS_ECDHE_RSA_WITH_RC4_128_SHA	ssl.gstatic.com	-	CN=*.gstatic.com,O=Google Inc,L=Mountain View,ST=California,C=US	CN=Google Internet Authority,O=Google Inc,C=US	1334102677.000000	1365639277.000000	-	-	-	T
+1335538392.319381	CXWv6p3arKYeMETxOg	192.168.1.105	62045	74.125.224.79	443	TLSv10	TLS_ECDHE_RSA_WITH_RC4_128_SHA	ssl.gstatic.com	-	-	T	F6wfNWn8LR755SYo7,FJl60T1mOolaez9T0h	(empty)	CN=*.gstatic.com,O=Google Inc,L=Mountain View,ST=California,C=US	CN=Google Internet Authority,O=Google Inc,C=US	-	-
-#close	2014-03-04-22-02-50
+#close	2014-03-13-20-45-46

testing/btest/Baseline/scripts.base.protocols.ssl.basic/x509.log

@@ -0,0 +1,11 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	x509
+#open	2014-03-13-20-45-46
+#fields	ts	id	certificate.version	certificate.serial	certificate.subject	certificate.issuer	certificate.not_valid_before	certificate.not_valid_after	certificate.key_alg	certificate.sig_alg	certificate.key_type	certificate.key_length	certificate.exponent	certificate.curve	san.dns	san.uri	san.email	san.ip	basic_constraints.ca	basic_constraints.path_len
+#types	time	string	count	string	string	string	time	time	string	string	string	count	string	string	vector[string]	vector[string]	vector[string]	vector[addr]	bool	count
+1335538392.343624	F6wfNWn8LR755SYo7	2	36F5DA5300000000505E	CN=*.gstatic.com,O=Google Inc,L=Mountain View,ST=California,C=US	CN=Google Internet Authority,O=Google Inc,C=US	1334102677.000000	1365639277.000000	rsaEncryption	sha1WithRSAEncryption	rsa	1024	65537	-	*.gstatic.com,gstatic.com,*.metric.gstatic.com	-	-	-	-	-
+1335538392.343624	FJl60T1mOolaez9T0h	2	0B6771	CN=Google Internet Authority,O=Google Inc,C=US	OU=Equifax Secure Certificate Authority,O=Equifax,C=US	1244493807.000000	1370634207.000000	rsaEncryption	sha1WithRSAEncryption	rsa	1024	65537	-	-	-	-	-	T	0
+#close	2014-03-13-20-45-46

testing/btest/Baseline/scripts.base.protocols.ssl.tls-1.2-handshake-failure/ssl.log

@@ -3,8 +3,8 @@
 #empty_field	(empty)
 #unset_field	-
 #path	ssl
-#open	2014-03-04-21-57-58
+#open	2014-03-13-20-46-30
-#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	version	cipher	server_name	session_id	subject	issuer_subject	not_valid_before	not_valid_after	last_alert	client_subject	client_issuer_subject	established
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	version	cipher	server_name	session_id	last_alert	established	cert_chain_fuids	client_cert_chain_fuids	subject	issuer	client_subject	client_issuer
-#types	time	string	addr	port	addr	port	string	string	string	string	string	string	time	time	string	string	string	bool
+#types	time	string	addr	port	addr	port	string	string	string	string	string	bool	vector[string]	vector[string]	string	string	string	string
-1393957586.786031	CXWv6p3arKYeMETxOg	192.168.4.149	53525	74.125.239.37	443	-	-	-	-	-	-	-	-	handshake_failure	-	-	F
+1393957586.786031	CXWv6p3arKYeMETxOg	192.168.4.149	53525	74.125.239.37	443	-	-	-	-	handshake_failure	F	-	-	-	-	-	-
-#close	2014-03-04-21-57-58
+#close	2014-03-13-20-46-30

testing/btest/Baseline/scripts.base.protocols.ssl.tls-1.2/ssl.log

@@ -3,8 +3,8 @@
 #empty_field	(empty)
 #unset_field	-
 #path	ssl
-#open	2014-03-04-22-03-00
+#open	2014-03-13-20-46-09
-#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	version	cipher	server_name	session_id	subject	issuer_subject	not_valid_before	not_valid_after	last_alert	client_subject	client_issuer_subject	established
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	version	cipher	server_name	session_id	last_alert	established	cert_chain_fuids	client_cert_chain_fuids	subject	issuer	client_subject	client_issuer
-#types	time	string	addr	port	addr	port	string	string	string	string	string	string	time	time	string	string	string	bool
+#types	time	string	addr	port	addr	port	string	string	string	string	string	bool	vector[string]	vector[string]	string	string	string	string
-1357328848.549370	CXWv6p3arKYeMETxOg	10.0.0.80	56637	68.233.76.12	443	TLSv12	TLS_RSA_WITH_RC4_128_MD5	-	-	CN=*.taleo.net,OU=Comodo PremiumSSL Wildcard,OU=Web,O=Taleo Inc.,street=4140 Dublin Boulevard,street=Suite 400,L=Dublin,ST=CA,postalCode=94568,C=US	CN=COMODO High-Assurance Secure Server CA,O=COMODO CA Limited,L=Salford,ST=Greater Manchester,C=GB	1304467200.000000	1467676799.000000	-	-	-	T
+1357328848.549370	CXWv6p3arKYeMETxOg	10.0.0.80	56637	68.233.76.12	443	TLSv12	TLS_RSA_WITH_RC4_128_MD5	-	-	-	T	FlnQzb2dJK4p9jXwmd,FaDzX22O4j3kFF6Jqg,F9Tsjm3OdCmGGw43Yh	(empty)	CN=*.taleo.net,OU=Comodo PremiumSSL Wildcard,OU=Web,O=Taleo Inc.,street=4140 Dublin Boulevard,street=Suite 400,L=Dublin,ST=CA,postalCode=94568,C=US	CN=COMODO High-Assurance Secure Server CA,O=COMODO CA Limited,L=Salford,ST=Greater Manchester,C=GB	-	-
-#close	2014-03-04-22-03-00
+#close	2014-03-13-20-46-09

testing/btest/Baseline/scripts.base.protocols.ssl.tls-1.2/x509.log

@@ -0,0 +1,12 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	x509
+#open	2014-03-13-20-46-09
+#fields	ts	id	certificate.version	certificate.serial	certificate.subject	certificate.issuer	certificate.not_valid_before	certificate.not_valid_after	certificate.key_alg	certificate.sig_alg	certificate.key_type	certificate.key_length	certificate.exponent	certificate.curve	san.dns	san.uri	san.email	san.ip	basic_constraints.ca	basic_constraints.path_len
+#types	time	string	count	string	string	string	time	time	string	string	string	count	string	string	vector[string]	vector[string]	vector[string]	vector[addr]	bool	count
+1357328848.591964	FlnQzb2dJK4p9jXwmd	2	99FAA8037A4EB2FAEF84EB5E55D5B8C8	CN=*.taleo.net,OU=Comodo PremiumSSL Wildcard,OU=Web,O=Taleo Inc.,street=4140 Dublin Boulevard,street=Suite 400,L=Dublin,ST=CA,postalCode=94568,C=US	CN=COMODO High-Assurance Secure Server CA,O=COMODO CA Limited,L=Salford,ST=Greater Manchester,C=GB	1304467200.000000	1467676799.000000	rsaEncryption	sha1WithRSAEncryption	rsa	2048	65537	-	*.taleo.net,taleo.net	-	-	-	F	-
+1357328848.591964	FaDzX22O4j3kFF6Jqg	2	1690C329B6780607511F05B0344846CB	CN=COMODO High-Assurance Secure Server CA,O=COMODO CA Limited,L=Salford,ST=Greater Manchester,C=GB	CN=AddTrust External CA Root,OU=AddTrust External TTP Network,O=AddTrust AB,C=SE	1271376000.000000	1590835718.000000	rsaEncryption	sha1WithRSAEncryption	rsa	2048	65537	-	-	-	-	-	T	0
+1357328848.591964	F9Tsjm3OdCmGGw43Yh	2	01	CN=AddTrust External CA Root,OU=AddTrust External TTP Network,O=AddTrust AB,C=SE	CN=AddTrust External CA Root,OU=AddTrust External TTP Network,O=AddTrust AB,C=SE	959683718.000000	1590835718.000000	rsaEncryption	sha1WithRSAEncryption	rsa	2048	65537	-	-	-	-	-	T	-
+#close	2014-03-13-20-46-09

testing/btest/Baseline/scripts.policy.misc.dump-events/all-events-no-args.log

@@ -1,36 +1,118 @@
          0.000000 bro_init
          0.000000 filter_change_tracking
-1170717505.366729 ChecksumOffloading::check
+1254722767.492060 protocol_confirmation
-1170717505.366729 filter_change_tracking
+1254722767.492060 ChecksumOffloading::check
-1170717505.366729 new_connection
+1254722767.492060 filter_change_tracking
-1170717505.548308 connection_established
+1254722767.492060 new_connection
-1170717505.549109 protocol_confirmation
+1254722767.492060 dns_message
-1170717505.549109 ssl_client_hello
+1254722767.492060 dns_request
-1170717505.734145 ssl_server_hello
+1254722767.492060 dns_end
-1170717505.735416 x509_certificate
+1254722767.526085 dns_message
-1170717505.735416 x509_certificate
+1254722767.526085 dns_CNAME_reply
-1170717505.934612 ssl_established
+1254722767.526085 dns_A_reply
-1170717508.515696 new_connection
+1254722767.526085 dns_end
-1170717508.696747 connection_established
+1254722767.529046 new_connection
-1170717508.697180 protocol_confirmation
+1254722767.875996 connection_established
-1170717508.697180 ssl_client_hello
+1254722768.219663 smtp_reply
-1170717508.881857 ssl_server_hello
+1254722768.219663 smtp_reply
-1170717508.883051 x509_certificate
+1254722768.219663 smtp_reply
-1170717508.883051 x509_certificate
+1254722768.224809 protocol_confirmation
-1170717509.082241 ssl_established
+1254722768.224809 smtp_request
-1170717511.541455 new_connection
+1254722768.566183 smtp_reply
-1170717511.722589 connection_established
+1254722768.566183 smtp_reply
-1170717511.722913 protocol_confirmation
+1254722768.566183 smtp_reply
-1170717511.722913 ssl_client_hello
+1254722768.566183 smtp_reply
-1170717511.908619 ssl_server_hello
+1254722768.566183 smtp_reply
-1170717511.909717 x509_certificate
+1254722768.566183 smtp_reply
-1170717511.909717 x509_certificate
+1254722768.568729 smtp_request
-1170717512.108799 ssl_established
+1254722768.911081 smtp_reply
-1170717528.851698 ChecksumOffloading::check
+1254722768.911655 smtp_request
-1170717528.851698 connection_state_remove
+1254722769.253544 smtp_reply
-1170717531.882302 net_done
+1254722769.254118 smtp_request
-1170717531.882302 filter_change_tracking
+1254722769.613798 smtp_reply
-1170717531.882302 connection_state_remove
+1254722769.614414 smtp_request
-1170717531.882302 connection_state_remove
+1254722769.956765 smtp_reply
-1170717531.882302 bro_done
+1254722769.957250 smtp_request
-1170717531.882302 ChecksumOffloading::check
+1254722770.319708 smtp_reply
+1254722770.320203 smtp_request
+1254722770.320203 mime_begin_entity
+1254722770.661679 smtp_reply
+1254722770.692743 mime_one_header
+1254722770.692743 mime_one_header
+1254722770.692743 mime_one_header
+1254722770.692743 mime_one_header
+1254722770.692743 mime_one_header
+1254722770.692743 mime_one_header
+1254722770.692743 mime_one_header
+1254722770.692743 mime_one_header
+1254722770.692743 mime_one_header
+1254722770.692743 mime_one_header
+1254722770.692743 mime_one_header
+1254722770.692743 mime_one_header
+1254722770.692743 mime_begin_entity
+1254722770.692743 mime_one_header
+1254722770.692743 mime_begin_entity
+1254722770.692743 mime_one_header
+1254722770.692743 mime_one_header
+1254722770.692743 get_file_handle
+1254722770.692743 mime_end_entity
+1254722770.692743 get_file_handle
+1254722770.692743 file_new
+1254722770.692743 file_over_new_connection
+1254722770.692743 file_state_remove
+1254722770.692743 get_file_handle
+1254722770.692743 mime_begin_entity
+1254722770.692743 mime_one_header
+1254722770.692743 mime_one_header
+1254722770.692786 get_file_handle
+1254722770.692786 file_new
+1254722770.692786 file_over_new_connection
+1254722770.692804 get_file_handle
+1254722770.692804 mime_end_entity
+1254722770.692804 get_file_handle
+1254722770.692804 file_state_remove
+1254722770.692804 get_file_handle
+1254722770.692804 mime_end_entity
+1254722770.692804 get_file_handle
+1254722770.692804 get_file_handle
+1254722770.692804 mime_begin_entity
+1254722770.692804 mime_one_header
+1254722770.692804 mime_one_header
+1254722770.692804 mime_one_header
+1254722770.692823 get_file_handle
+1254722770.692823 file_new
+1254722770.692823 file_over_new_connection
+1254722770.692823 get_file_handle
+1254722770.695115 new_connection
+1254722771.469814 get_file_handle
+1254722771.494181 get_file_handle
+1254722771.494181 get_file_handle
+1254722771.494199 get_file_handle
+1254722771.834628 get_file_handle
+1254722771.834655 get_file_handle
+1254722771.834655 get_file_handle
+1254722771.858316 get_file_handle
+1254722771.858334 get_file_handle
+1254722771.858334 mime_end_entity
+1254722771.858334 get_file_handle
+1254722771.858334 file_state_remove
+1254722771.858334 get_file_handle
+1254722771.858334 mime_end_entity
+1254722771.858334 get_file_handle
+1254722771.858334 get_file_handle
+1254722771.858334 get_file_handle
+1254722771.858334 get_file_handle
+1254722771.858334 smtp_request
+1254722772.248789 smtp_reply
+1254722774.763825 smtp_request
+1254722775.105467 smtp_reply
+1254722776.690444 new_connection
+1254722776.690444 net_done
+1254722776.690444 ChecksumOffloading::check
+1254722776.690444 connection_state_remove
+1254722776.690444 filter_change_tracking
+1254722776.690444 connection_state_remove
+1254722776.690444 connection_state_remove
+1254722776.690444 connection_state_remove
+1254722776.690444 bro_done
+1254722776.690444 ChecksumOffloading::check

testing/btest/Baseline/scripts.policy.misc.dump-events/smtp-events.log

@@ -0,0 +1,190 @@
+1254722768.219663 smtp_reply
+                  [0] c: connection      = [id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], orig=[size=0, state=4, num_pkts=2, num_bytes_ip=88, flow_label=0], resp=[size=181, state=4, num_pkts=1, num_bytes_ip=48, flow_label=0], start_time=1254722767.529046, duration=0.690617, service={^J^J}, addl=, hot=0, history=ShAd, uid=CjhGID4nQcgTWjvg4c, tunnel=<uninitialized>, dpd=<uninitialized>, conn=<uninitialized>, extract_orig=F, extract_resp=F, dhcp=<uninitialized>, dnp3=<uninitialized>, dns=<uninitialized>, dns_state=<uninitialized>, ftp=<uninitialized>, ftp_data_reuse=F, ssl=<uninitialized>, http=<uninitialized>, http_state=<uninitialized>, irc=<uninitialized>, modbus=<uninitialized>, smtp=<uninitialized>, smtp_state=<uninitialized>, socks=<uninitialized>, ssh=<uninitialized>, syslog=<uninitialized>]
+                  [1] is_orig: bool      = F
+                  [2] code: count        = 220
+                  [3] cmd: string        = >
+                  [4] msg: string        = xc90.websitewelcome.com ESMTP Exim 4.69 #1 Mon, 05 Oct 2009 01:05:54 -0500 
+                  [5] cont_resp: bool    = T
+
+1254722768.219663 smtp_reply
+                  [0] c: connection      = [id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], orig=[size=0, state=4, num_pkts=2, num_bytes_ip=88, flow_label=0], resp=[size=181, state=4, num_pkts=1, num_bytes_ip=48, flow_label=0], start_time=1254722767.529046, duration=0.690617, service={^J^J}, addl=, hot=0, history=ShAd, uid=CjhGID4nQcgTWjvg4c, tunnel=<uninitialized>, dpd=<uninitialized>, conn=<uninitialized>, extract_orig=F, extract_resp=F, dhcp=<uninitialized>, dnp3=<uninitialized>, dns=<uninitialized>, dns_state=<uninitialized>, ftp=<uninitialized>, ftp_data_reuse=F, ssl=<uninitialized>, http=<uninitialized>, http_state=<uninitialized>, irc=<uninitialized>, modbus=<uninitialized>, smtp=[ts=1254722768.219663, uid=CjhGID4nQcgTWjvg4c, id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], trans_depth=1, helo=<uninitialized>, mailfrom=<uninitialized>, rcptto=<uninitialized>, date=<uninitialized>, from=<uninitialized>, to=<uninitialized>, reply_to=<uninitialized>, msg_id=<uninitialized>, in_reply_to=<uninitialized>, subject=<uninitialized>, x_originating_ip=<uninitialized>, first_received=<uninitialized>, second_received=<uninitialized>, last_reply=220 xc90.websitewelcome.com ESMTP Exim 4.69 #1 Mon, 05 Oct 2009 01:05:54 -0500 , path=[74.53.140.153, 10.10.1.4], user_agent=<uninitialized>, process_received_from=T, has_client_activity=F, entity=<uninitialized>, fuids=[]], smtp_state=[helo=<uninitialized>, messages_transferred=0, pending_messages=<uninitialized>, mime_depth=0], socks=<uninitialized>, ssh=<uninitialized>, syslog=<uninitialized>]
+                  [1] is_orig: bool      = F
+                  [2] code: count        = 220
+                  [3] cmd: string        = >
+                  [4] msg: string        = We do not authorize the use of this system to transport unsolicited, 
+                  [5] cont_resp: bool    = T
+
+1254722768.219663 smtp_reply
+                  [0] c: connection      = [id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], orig=[size=0, state=4, num_pkts=2, num_bytes_ip=88, flow_label=0], resp=[size=181, state=4, num_pkts=1, num_bytes_ip=48, flow_label=0], start_time=1254722767.529046, duration=0.690617, service={^J^J}, addl=, hot=0, history=ShAd, uid=CjhGID4nQcgTWjvg4c, tunnel=<uninitialized>, dpd=<uninitialized>, conn=<uninitialized>, extract_orig=F, extract_resp=F, dhcp=<uninitialized>, dnp3=<uninitialized>, dns=<uninitialized>, dns_state=<uninitialized>, ftp=<uninitialized>, ftp_data_reuse=F, ssl=<uninitialized>, http=<uninitialized>, http_state=<uninitialized>, irc=<uninitialized>, modbus=<uninitialized>, smtp=[ts=1254722768.219663, uid=CjhGID4nQcgTWjvg4c, id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], trans_depth=1, helo=<uninitialized>, mailfrom=<uninitialized>, rcptto=<uninitialized>, date=<uninitialized>, from=<uninitialized>, to=<uninitialized>, reply_to=<uninitialized>, msg_id=<uninitialized>, in_reply_to=<uninitialized>, subject=<uninitialized>, x_originating_ip=<uninitialized>, first_received=<uninitialized>, second_received=<uninitialized>, last_reply=220 We do not authorize the use of this system to transport unsolicited, , path=[74.53.140.153, 10.10.1.4], user_agent=<uninitialized>, process_received_from=T, has_client_activity=F, entity=<uninitialized>, fuids=[]], smtp_state=[helo=<uninitialized>, messages_transferred=0, pending_messages=<uninitialized>, mime_depth=0], socks=<uninitialized>, ssh=<uninitialized>, syslog=<uninitialized>]
+                  [1] is_orig: bool      = F
+                  [2] code: count        = 220
+                  [3] cmd: string        = >
+                  [4] msg: string        = and/or bulk e-mail.
+                  [5] cont_resp: bool    = F
+
+1254722768.224809 smtp_request
+                  [0] c: connection      = [id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], orig=[size=9, state=4, num_pkts=2, num_bytes_ip=88, flow_label=0], resp=[size=181, state=4, num_pkts=2, num_bytes_ip=269, flow_label=0], start_time=1254722767.529046, duration=0.695763, service={^J^ISMTP^J}, addl=, hot=0, history=ShAdD, uid=CjhGID4nQcgTWjvg4c, tunnel=<uninitialized>, dpd=<uninitialized>, conn=<uninitialized>, extract_orig=F, extract_resp=F, dhcp=<uninitialized>, dnp3=<uninitialized>, dns=<uninitialized>, dns_state=<uninitialized>, ftp=<uninitialized>, ftp_data_reuse=F, ssl=<uninitialized>, http=<uninitialized>, http_state=<uninitialized>, irc=<uninitialized>, modbus=<uninitialized>, smtp=[ts=1254722768.219663, uid=CjhGID4nQcgTWjvg4c, id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], trans_depth=1, helo=<uninitialized>, mailfrom=<uninitialized>, rcptto=<uninitialized>, date=<uninitialized>, from=<uninitialized>, to=<uninitialized>, reply_to=<uninitialized>, msg_id=<uninitialized>, in_reply_to=<uninitialized>, subject=<uninitialized>, x_originating_ip=<uninitialized>, first_received=<uninitialized>, second_received=<uninitialized>, last_reply=220 and/or bulk e-mail., path=[74.53.140.153, 10.10.1.4], user_agent=<uninitialized>, process_received_from=T, has_client_activity=F, entity=<uninitialized>, fuids=[]], smtp_state=[helo=<uninitialized>, messages_transferred=0, pending_messages=<uninitialized>, mime_depth=0], socks=<uninitialized>, ssh=<uninitialized>, syslog=<uninitialized>]
+                  [1] is_orig: bool      = T
+                  [2] command: string    = EHLO
+                  [3] arg: string        = GP
+
+1254722768.566183 smtp_reply
+                  [0] c: connection      = [id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], orig=[size=9, state=4, num_pkts=3, num_bytes_ip=137, flow_label=0], resp=[size=318, state=4, num_pkts=3, num_bytes_ip=309, flow_label=0], start_time=1254722767.529046, duration=1.037137, service={^J^ISMTP^J}, addl=, hot=0, history=ShAdDa, uid=CjhGID4nQcgTWjvg4c, tunnel=<uninitialized>, dpd=<uninitialized>, conn=<uninitialized>, extract_orig=F, extract_resp=F, dhcp=<uninitialized>, dnp3=<uninitialized>, dns=<uninitialized>, dns_state=<uninitialized>, ftp=<uninitialized>, ftp_data_reuse=F, ssl=<uninitialized>, http=<uninitialized>, http_state=<uninitialized>, irc=<uninitialized>, modbus=<uninitialized>, smtp=[ts=1254722768.219663, uid=CjhGID4nQcgTWjvg4c, id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], trans_depth=1, helo=GP, mailfrom=<uninitialized>, rcptto=<uninitialized>, date=<uninitialized>, from=<uninitialized>, to=<uninitialized>, reply_to=<uninitialized>, msg_id=<uninitialized>, in_reply_to=<uninitialized>, subject=<uninitialized>, x_originating_ip=<uninitialized>, first_received=<uninitialized>, second_received=<uninitialized>, last_reply=220 and/or bulk e-mail., path=[74.53.140.153, 10.10.1.4], user_agent=<uninitialized>, process_received_from=T, has_client_activity=T, entity=<uninitialized>, fuids=[]], smtp_state=[helo=GP, messages_transferred=0, pending_messages=<uninitialized>, mime_depth=0], socks=<uninitialized>, ssh=<uninitialized>, syslog=<uninitialized>]
+                  [1] is_orig: bool      = F
+                  [2] code: count        = 250
+                  [3] cmd: string        = EHLO
+                  [4] msg: string        = xc90.websitewelcome.com Hello GP [122.162.143.157]
+                  [5] cont_resp: bool    = T
+
+1254722768.566183 smtp_reply
+                  [0] c: connection      = [id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], orig=[size=9, state=4, num_pkts=3, num_bytes_ip=137, flow_label=0], resp=[size=318, state=4, num_pkts=3, num_bytes_ip=309, flow_label=0], start_time=1254722767.529046, duration=1.037137, service={^J^ISMTP^J}, addl=, hot=0, history=ShAdDa, uid=CjhGID4nQcgTWjvg4c, tunnel=<uninitialized>, dpd=<uninitialized>, conn=<uninitialized>, extract_orig=F, extract_resp=F, dhcp=<uninitialized>, dnp3=<uninitialized>, dns=<uninitialized>, dns_state=<uninitialized>, ftp=<uninitialized>, ftp_data_reuse=F, ssl=<uninitialized>, http=<uninitialized>, http_state=<uninitialized>, irc=<uninitialized>, modbus=<uninitialized>, smtp=[ts=1254722768.219663, uid=CjhGID4nQcgTWjvg4c, id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], trans_depth=1, helo=GP, mailfrom=<uninitialized>, rcptto=<uninitialized>, date=<uninitialized>, from=<uninitialized>, to=<uninitialized>, reply_to=<uninitialized>, msg_id=<uninitialized>, in_reply_to=<uninitialized>, subject=<uninitialized>, x_originating_ip=<uninitialized>, first_received=<uninitialized>, second_received=<uninitialized>, last_reply=250 xc90.websitewelcome.com Hello GP [122.162.143.157], path=[74.53.140.153, 10.10.1.4], user_agent=<uninitialized>, process_received_from=T, has_client_activity=T, entity=<uninitialized>, fuids=[]], smtp_state=[helo=GP, messages_transferred=0, pending_messages=<uninitialized>, mime_depth=0], socks=<uninitialized>, ssh=<uninitialized>, syslog=<uninitialized>]
+                  [1] is_orig: bool      = F
+                  [2] code: count        = 250
+                  [3] cmd: string        = EHLO
+                  [4] msg: string        = SIZE 52428800
+                  [5] cont_resp: bool    = T
+
+1254722768.566183 smtp_reply
+                  [0] c: connection      = [id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], orig=[size=9, state=4, num_pkts=3, num_bytes_ip=137, flow_label=0], resp=[size=318, state=4, num_pkts=3, num_bytes_ip=309, flow_label=0], start_time=1254722767.529046, duration=1.037137, service={^J^ISMTP^J}, addl=, hot=0, history=ShAdDa, uid=CjhGID4nQcgTWjvg4c, tunnel=<uninitialized>, dpd=<uninitialized>, conn=<uninitialized>, extract_orig=F, extract_resp=F, dhcp=<uninitialized>, dnp3=<uninitialized>, dns=<uninitialized>, dns_state=<uninitialized>, ftp=<uninitialized>, ftp_data_reuse=F, ssl=<uninitialized>, http=<uninitialized>, http_state=<uninitialized>, irc=<uninitialized>, modbus=<uninitialized>, smtp=[ts=1254722768.219663, uid=CjhGID4nQcgTWjvg4c, id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], trans_depth=1, helo=GP, mailfrom=<uninitialized>, rcptto=<uninitialized>, date=<uninitialized>, from=<uninitialized>, to=<uninitialized>, reply_to=<uninitialized>, msg_id=<uninitialized>, in_reply_to=<uninitialized>, subject=<uninitialized>, x_originating_ip=<uninitialized>, first_received=<uninitialized>, second_received=<uninitialized>, last_reply=250 SIZE 52428800, path=[74.53.140.153, 10.10.1.4], user_agent=<uninitialized>, process_received_from=T, has_client_activity=T, entity=<uninitialized>, fuids=[]], smtp_state=[helo=GP, messages_transferred=0, pending_messages=<uninitialized>, mime_depth=0], socks=<uninitialized>, ssh=<uninitialized>, syslog=<uninitialized>]
+                  [1] is_orig: bool      = F
+                  [2] code: count        = 250
+                  [3] cmd: string        = EHLO
+                  [4] msg: string        = PIPELINING
+                  [5] cont_resp: bool    = T
+
+1254722768.566183 smtp_reply
+                  [0] c: connection      = [id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], orig=[size=9, state=4, num_pkts=3, num_bytes_ip=137, flow_label=0], resp=[size=318, state=4, num_pkts=3, num_bytes_ip=309, flow_label=0], start_time=1254722767.529046, duration=1.037137, service={^J^ISMTP^J}, addl=, hot=0, history=ShAdDa, uid=CjhGID4nQcgTWjvg4c, tunnel=<uninitialized>, dpd=<uninitialized>, conn=<uninitialized>, extract_orig=F, extract_resp=F, dhcp=<uninitialized>, dnp3=<uninitialized>, dns=<uninitialized>, dns_state=<uninitialized>, ftp=<uninitialized>, ftp_data_reuse=F, ssl=<uninitialized>, http=<uninitialized>, http_state=<uninitialized>, irc=<uninitialized>, modbus=<uninitialized>, smtp=[ts=1254722768.219663, uid=CjhGID4nQcgTWjvg4c, id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], trans_depth=1, helo=GP, mailfrom=<uninitialized>, rcptto=<uninitialized>, date=<uninitialized>, from=<uninitialized>, to=<uninitialized>, reply_to=<uninitialized>, msg_id=<uninitialized>, in_reply_to=<uninitialized>, subject=<uninitialized>, x_originating_ip=<uninitialized>, first_received=<uninitialized>, second_received=<uninitialized>, last_reply=250 PIPELINING, path=[74.53.140.153, 10.10.1.4], user_agent=<uninitialized>, process_received_from=T, has_client_activity=T, entity=<uninitialized>, fuids=[]], smtp_state=[helo=GP, messages_transferred=0, pending_messages=<uninitialized>, mime_depth=0], socks=<uninitialized>, ssh=<uninitialized>, syslog=<uninitialized>]
+                  [1] is_orig: bool      = F
+                  [2] code: count        = 250
+                  [3] cmd: string        = EHLO
+                  [4] msg: string        = AUTH PLAIN LOGIN
+                  [5] cont_resp: bool    = T
+
+1254722768.566183 smtp_reply
+                  [0] c: connection      = [id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], orig=[size=9, state=4, num_pkts=3, num_bytes_ip=137, flow_label=0], resp=[size=318, state=4, num_pkts=3, num_bytes_ip=309, flow_label=0], start_time=1254722767.529046, duration=1.037137, service={^J^ISMTP^J}, addl=, hot=0, history=ShAdDa, uid=CjhGID4nQcgTWjvg4c, tunnel=<uninitialized>, dpd=<uninitialized>, conn=<uninitialized>, extract_orig=F, extract_resp=F, dhcp=<uninitialized>, dnp3=<uninitialized>, dns=<uninitialized>, dns_state=<uninitialized>, ftp=<uninitialized>, ftp_data_reuse=F, ssl=<uninitialized>, http=<uninitialized>, http_state=<uninitialized>, irc=<uninitialized>, modbus=<uninitialized>, smtp=[ts=1254722768.219663, uid=CjhGID4nQcgTWjvg4c, id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], trans_depth=1, helo=GP, mailfrom=<uninitialized>, rcptto=<uninitialized>, date=<uninitialized>, from=<uninitialized>, to=<uninitialized>, reply_to=<uninitialized>, msg_id=<uninitialized>, in_reply_to=<uninitialized>, subject=<uninitialized>, x_originating_ip=<uninitialized>, first_received=<uninitialized>, second_received=<uninitialized>, last_reply=250 AUTH PLAIN LOGIN, path=[74.53.140.153, 10.10.1.4], user_agent=<uninitialized>, process_received_from=T, has_client_activity=T, entity=<uninitialized>, fuids=[]], smtp_state=[helo=GP, messages_transferred=0, pending_messages=<uninitialized>, mime_depth=0], socks=<uninitialized>, ssh=<uninitialized>, syslog=<uninitialized>]
+                  [1] is_orig: bool      = F
+                  [2] code: count        = 250
+                  [3] cmd: string        = EHLO
+                  [4] msg: string        = STARTTLS
+                  [5] cont_resp: bool    = T
+
+1254722768.566183 smtp_reply
+                  [0] c: connection      = [id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], orig=[size=9, state=4, num_pkts=3, num_bytes_ip=137, flow_label=0], resp=[size=318, state=4, num_pkts=3, num_bytes_ip=309, flow_label=0], start_time=1254722767.529046, duration=1.037137, service={^J^ISMTP^J}, addl=, hot=0, history=ShAdDa, uid=CjhGID4nQcgTWjvg4c, tunnel=<uninitialized>, dpd=<uninitialized>, conn=<uninitialized>, extract_orig=F, extract_resp=F, dhcp=<uninitialized>, dnp3=<uninitialized>, dns=<uninitialized>, dns_state=<uninitialized>, ftp=<uninitialized>, ftp_data_reuse=F, ssl=<uninitialized>, http=<uninitialized>, http_state=<uninitialized>, irc=<uninitialized>, modbus=<uninitialized>, smtp=[ts=1254722768.219663, uid=CjhGID4nQcgTWjvg4c, id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], trans_depth=1, helo=GP, mailfrom=<uninitialized>, rcptto=<uninitialized>, date=<uninitialized>, from=<uninitialized>, to=<uninitialized>, reply_to=<uninitialized>, msg_id=<uninitialized>, in_reply_to=<uninitialized>, subject=<uninitialized>, x_originating_ip=<uninitialized>, first_received=<uninitialized>, second_received=<uninitialized>, last_reply=250 STARTTLS, path=[74.53.140.153, 10.10.1.4], user_agent=<uninitialized>, process_received_from=T, has_client_activity=T, entity=<uninitialized>, fuids=[]], smtp_state=[helo=GP, messages_transferred=0, pending_messages=<uninitialized>, mime_depth=0], socks=<uninitialized>, ssh=<uninitialized>, syslog=<uninitialized>]
+                  [1] is_orig: bool      = F
+                  [2] code: count        = 250
+                  [3] cmd: string        = EHLO
+                  [4] msg: string        = HELP
+                  [5] cont_resp: bool    = F
+
+1254722768.568729 smtp_request
+                  [0] c: connection      = [id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], orig=[size=21, state=4, num_pkts=3, num_bytes_ip=137, flow_label=0], resp=[size=318, state=4, num_pkts=4, num_bytes_ip=486, flow_label=0], start_time=1254722767.529046, duration=1.039683, service={^J^ISMTP^J}, addl=, hot=0, history=ShAdDa, uid=CjhGID4nQcgTWjvg4c, tunnel=<uninitialized>, dpd=<uninitialized>, conn=<uninitialized>, extract_orig=F, extract_resp=F, dhcp=<uninitialized>, dnp3=<uninitialized>, dns=<uninitialized>, dns_state=<uninitialized>, ftp=<uninitialized>, ftp_data_reuse=F, ssl=<uninitialized>, http=<uninitialized>, http_state=<uninitialized>, irc=<uninitialized>, modbus=<uninitialized>, smtp=[ts=1254722768.219663, uid=CjhGID4nQcgTWjvg4c, id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], trans_depth=1, helo=GP, mailfrom=<uninitialized>, rcptto=<uninitialized>, date=<uninitialized>, from=<uninitialized>, to=<uninitialized>, reply_to=<uninitialized>, msg_id=<uninitialized>, in_reply_to=<uninitialized>, subject=<uninitialized>, x_originating_ip=<uninitialized>, first_received=<uninitialized>, second_received=<uninitialized>, last_reply=250 HELP, path=[74.53.140.153, 10.10.1.4], user_agent=<uninitialized>, process_received_from=T, has_client_activity=T, entity=<uninitialized>, fuids=[]], smtp_state=[helo=GP, messages_transferred=0, pending_messages=<uninitialized>, mime_depth=0], socks=<uninitialized>, ssh=<uninitialized>, syslog=<uninitialized>]
+                  [1] is_orig: bool      = T
+                  [2] command: string    = AUTH
+                  [3] arg: string        = LOGIN
+
+1254722768.911081 smtp_reply
+                  [0] c: connection      = [id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], orig=[size=21, state=4, num_pkts=4, num_bytes_ip=189, flow_label=0], resp=[size=336, state=4, num_pkts=4, num_bytes_ip=486, flow_label=0], start_time=1254722767.529046, duration=1.382035, service={^J^ISMTP^J}, addl=, hot=0, history=ShAdDa, uid=CjhGID4nQcgTWjvg4c, tunnel=<uninitialized>, dpd=<uninitialized>, conn=<uninitialized>, extract_orig=F, extract_resp=F, dhcp=<uninitialized>, dnp3=<uninitialized>, dns=<uninitialized>, dns_state=<uninitialized>, ftp=<uninitialized>, ftp_data_reuse=F, ssl=<uninitialized>, http=<uninitialized>, http_state=<uninitialized>, irc=<uninitialized>, modbus=<uninitialized>, smtp=[ts=1254722768.219663, uid=CjhGID4nQcgTWjvg4c, id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], trans_depth=1, helo=GP, mailfrom=<uninitialized>, rcptto=<uninitialized>, date=<uninitialized>, from=<uninitialized>, to=<uninitialized>, reply_to=<uninitialized>, msg_id=<uninitialized>, in_reply_to=<uninitialized>, subject=<uninitialized>, x_originating_ip=<uninitialized>, first_received=<uninitialized>, second_received=<uninitialized>, last_reply=250 HELP, path=[74.53.140.153, 10.10.1.4], user_agent=<uninitialized>, process_received_from=T, has_client_activity=T, entity=<uninitialized>, fuids=[]], smtp_state=[helo=GP, messages_transferred=0, pending_messages=<uninitialized>, mime_depth=0], socks=<uninitialized>, ssh=<uninitialized>, syslog=<uninitialized>]
+                  [1] is_orig: bool      = F
+                  [2] code: count        = 334
+                  [3] cmd: string        = AUTH
+                  [4] msg: string        = VXNlcm5hbWU6
+                  [5] cont_resp: bool    = F
+
+1254722768.911655 smtp_request
+                  [0] c: connection      = [id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], orig=[size=51, state=4, num_pkts=4, num_bytes_ip=189, flow_label=0], resp=[size=336, state=4, num_pkts=5, num_bytes_ip=544, flow_label=0], start_time=1254722767.529046, duration=1.382609, service={^J^ISMTP^J}, addl=, hot=0, history=ShAdDa, uid=CjhGID4nQcgTWjvg4c, tunnel=<uninitialized>, dpd=<uninitialized>, conn=<uninitialized>, extract_orig=F, extract_resp=F, dhcp=<uninitialized>, dnp3=<uninitialized>, dns=<uninitialized>, dns_state=<uninitialized>, ftp=<uninitialized>, ftp_data_reuse=F, ssl=<uninitialized>, http=<uninitialized>, http_state=<uninitialized>, irc=<uninitialized>, modbus=<uninitialized>, smtp=[ts=1254722768.219663, uid=CjhGID4nQcgTWjvg4c, id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], trans_depth=1, helo=GP, mailfrom=<uninitialized>, rcptto=<uninitialized>, date=<uninitialized>, from=<uninitialized>, to=<uninitialized>, reply_to=<uninitialized>, msg_id=<uninitialized>, in_reply_to=<uninitialized>, subject=<uninitialized>, x_originating_ip=<uninitialized>, first_received=<uninitialized>, second_received=<uninitialized>, last_reply=334 VXNlcm5hbWU6, path=[74.53.140.153, 10.10.1.4], user_agent=<uninitialized>, process_received_from=T, has_client_activity=T, entity=<uninitialized>, fuids=[]], smtp_state=[helo=GP, messages_transferred=0, pending_messages=<uninitialized>, mime_depth=0], socks=<uninitialized>, ssh=<uninitialized>, syslog=<uninitialized>]
+                  [1] is_orig: bool      = T
+                  [2] command: string    = **
+                  [3] arg: string        = Z3VycGFydGFwQHBhdHJpb3RzLmlu
+
+1254722769.253544 smtp_reply
+                  [0] c: connection      = [id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], orig=[size=51, state=4, num_pkts=5, num_bytes_ip=259, flow_label=0], resp=[size=354, state=4, num_pkts=5, num_bytes_ip=544, flow_label=0], start_time=1254722767.529046, duration=1.724498, service={^J^ISMTP^J}, addl=, hot=0, history=ShAdDa, uid=CjhGID4nQcgTWjvg4c, tunnel=<uninitialized>, dpd=<uninitialized>, conn=<uninitialized>, extract_orig=F, extract_resp=F, dhcp=<uninitialized>, dnp3=<uninitialized>, dns=<uninitialized>, dns_state=<uninitialized>, ftp=<uninitialized>, ftp_data_reuse=F, ssl=<uninitialized>, http=<uninitialized>, http_state=<uninitialized>, irc=<uninitialized>, modbus=<uninitialized>, smtp=[ts=1254722768.219663, uid=CjhGID4nQcgTWjvg4c, id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], trans_depth=1, helo=GP, mailfrom=<uninitialized>, rcptto=<uninitialized>, date=<uninitialized>, from=<uninitialized>, to=<uninitialized>, reply_to=<uninitialized>, msg_id=<uninitialized>, in_reply_to=<uninitialized>, subject=<uninitialized>, x_originating_ip=<uninitialized>, first_received=<uninitialized>, second_received=<uninitialized>, last_reply=334 VXNlcm5hbWU6, path=[74.53.140.153, 10.10.1.4], user_agent=<uninitialized>, process_received_from=T, has_client_activity=T, entity=<uninitialized>, fuids=[]], smtp_state=[helo=GP, messages_transferred=0, pending_messages=<uninitialized>, mime_depth=0], socks=<uninitialized>, ssh=<uninitialized>, syslog=<uninitialized>]
+                  [1] is_orig: bool      = F
+                  [2] code: count        = 334
+                  [3] cmd: string        = AUTH_ANSWER
+                  [4] msg: string        = UGFzc3dvcmQ6
+                  [5] cont_resp: bool    = F
+
+1254722769.254118 smtp_request
+                  [0] c: connection      = [id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], orig=[size=69, state=4, num_pkts=5, num_bytes_ip=259, flow_label=0], resp=[size=354, state=4, num_pkts=6, num_bytes_ip=602, flow_label=0], start_time=1254722767.529046, duration=1.725072, service={^J^ISMTP^J}, addl=, hot=0, history=ShAdDa, uid=CjhGID4nQcgTWjvg4c, tunnel=<uninitialized>, dpd=<uninitialized>, conn=<uninitialized>, extract_orig=F, extract_resp=F, dhcp=<uninitialized>, dnp3=<uninitialized>, dns=<uninitialized>, dns_state=<uninitialized>, ftp=<uninitialized>, ftp_data_reuse=F, ssl=<uninitialized>, http=<uninitialized>, http_state=<uninitialized>, irc=<uninitialized>, modbus=<uninitialized>, smtp=[ts=1254722768.219663, uid=CjhGID4nQcgTWjvg4c, id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], trans_depth=1, helo=GP, mailfrom=<uninitialized>, rcptto=<uninitialized>, date=<uninitialized>, from=<uninitialized>, to=<uninitialized>, reply_to=<uninitialized>, msg_id=<uninitialized>, in_reply_to=<uninitialized>, subject=<uninitialized>, x_originating_ip=<uninitialized>, first_received=<uninitialized>, second_received=<uninitialized>, last_reply=334 UGFzc3dvcmQ6, path=[74.53.140.153, 10.10.1.4], user_agent=<uninitialized>, process_received_from=T, has_client_activity=T, entity=<uninitialized>, fuids=[]], smtp_state=[helo=GP, messages_transferred=0, pending_messages=<uninitialized>, mime_depth=0], socks=<uninitialized>, ssh=<uninitialized>, syslog=<uninitialized>]
+                  [1] is_orig: bool      = T
+                  [2] command: string    = **
+                  [3] arg: string        = cHVuamFiQDEyMw==
+
+1254722769.613798 smtp_reply
+                  [0] c: connection      = [id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], orig=[size=69, state=4, num_pkts=6, num_bytes_ip=317, flow_label=0], resp=[size=384, state=4, num_pkts=6, num_bytes_ip=602, flow_label=0], start_time=1254722767.529046, duration=2.084752, service={^J^ISMTP^J}, addl=, hot=0, history=ShAdDa, uid=CjhGID4nQcgTWjvg4c, tunnel=<uninitialized>, dpd=<uninitialized>, conn=<uninitialized>, extract_orig=F, extract_resp=F, dhcp=<uninitialized>, dnp3=<uninitialized>, dns=<uninitialized>, dns_state=<uninitialized>, ftp=<uninitialized>, ftp_data_reuse=F, ssl=<uninitialized>, http=<uninitialized>, http_state=<uninitialized>, irc=<uninitialized>, modbus=<uninitialized>, smtp=[ts=1254722768.219663, uid=CjhGID4nQcgTWjvg4c, id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], trans_depth=1, helo=GP, mailfrom=<uninitialized>, rcptto=<uninitialized>, date=<uninitialized>, from=<uninitialized>, to=<uninitialized>, reply_to=<uninitialized>, msg_id=<uninitialized>, in_reply_to=<uninitialized>, subject=<uninitialized>, x_originating_ip=<uninitialized>, first_received=<uninitialized>, second_received=<uninitialized>, last_reply=334 UGFzc3dvcmQ6, path=[74.53.140.153, 10.10.1.4], user_agent=<uninitialized>, process_received_from=T, has_client_activity=T, entity=<uninitialized>, fuids=[]], smtp_state=[helo=GP, messages_transferred=0, pending_messages=<uninitialized>, mime_depth=0], socks=<uninitialized>, ssh=<uninitialized>, syslog=<uninitialized>]
+                  [1] is_orig: bool      = F
+                  [2] code: count        = 235
+                  [3] cmd: string        = AUTH_ANSWER
+                  [4] msg: string        = Authentication succeeded
+                  [5] cont_resp: bool    = F
+
+1254722769.614414 smtp_request
+                  [0] c: connection      = [id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], orig=[size=105, state=4, num_pkts=6, num_bytes_ip=317, flow_label=0], resp=[size=384, state=4, num_pkts=7, num_bytes_ip=672, flow_label=0], start_time=1254722767.529046, duration=2.085368, service={^J^ISMTP^J}, addl=, hot=0, history=ShAdDa, uid=CjhGID4nQcgTWjvg4c, tunnel=<uninitialized>, dpd=<uninitialized>, conn=<uninitialized>, extract_orig=F, extract_resp=F, dhcp=<uninitialized>, dnp3=<uninitialized>, dns=<uninitialized>, dns_state=<uninitialized>, ftp=<uninitialized>, ftp_data_reuse=F, ssl=<uninitialized>, http=<uninitialized>, http_state=<uninitialized>, irc=<uninitialized>, modbus=<uninitialized>, smtp=[ts=1254722768.219663, uid=CjhGID4nQcgTWjvg4c, id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], trans_depth=1, helo=GP, mailfrom=<uninitialized>, rcptto=<uninitialized>, date=<uninitialized>, from=<uninitialized>, to=<uninitialized>, reply_to=<uninitialized>, msg_id=<uninitialized>, in_reply_to=<uninitialized>, subject=<uninitialized>, x_originating_ip=<uninitialized>, first_received=<uninitialized>, second_received=<uninitialized>, last_reply=235 Authentication succeeded, path=[74.53.140.153, 10.10.1.4], user_agent=<uninitialized>, process_received_from=T, has_client_activity=T, entity=<uninitialized>, fuids=[]], smtp_state=[helo=GP, messages_transferred=0, pending_messages=<uninitialized>, mime_depth=0], socks=<uninitialized>, ssh=<uninitialized>, syslog=<uninitialized>]
+                  [1] is_orig: bool      = T
+                  [2] command: string    = MAIL
+                  [3] arg: string        = FROM: <gurpartap@patriots.in>
+
+1254722769.956765 smtp_reply
+                  [0] c: connection      = [id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], orig=[size=105, state=4, num_pkts=7, num_bytes_ip=393, flow_label=0], resp=[size=392, state=4, num_pkts=7, num_bytes_ip=672, flow_label=0], start_time=1254722767.529046, duration=2.427719, service={^J^ISMTP^J}, addl=, hot=0, history=ShAdDa, uid=CjhGID4nQcgTWjvg4c, tunnel=<uninitialized>, dpd=<uninitialized>, conn=<uninitialized>, extract_orig=F, extract_resp=F, dhcp=<uninitialized>, dnp3=<uninitialized>, dns=<uninitialized>, dns_state=<uninitialized>, ftp=<uninitialized>, ftp_data_reuse=F, ssl=<uninitialized>, http=<uninitialized>, http_state=<uninitialized>, irc=<uninitialized>, modbus=<uninitialized>, smtp=[ts=1254722768.219663, uid=CjhGID4nQcgTWjvg4c, id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], trans_depth=1, helo=GP, mailfrom=<gurpartap@patriots.in>, rcptto=<uninitialized>, date=<uninitialized>, from=<uninitialized>, to=<uninitialized>, reply_to=<uninitialized>, msg_id=<uninitialized>, in_reply_to=<uninitialized>, subject=<uninitialized>, x_originating_ip=<uninitialized>, first_received=<uninitialized>, second_received=<uninitialized>, last_reply=235 Authentication succeeded, path=[74.53.140.153, 10.10.1.4], user_agent=<uninitialized>, process_received_from=T, has_client_activity=T, entity=<uninitialized>, fuids=[]], smtp_state=[helo=GP, messages_transferred=0, pending_messages=<uninitialized>, mime_depth=0], socks=<uninitialized>, ssh=<uninitialized>, syslog=<uninitialized>]
+                  [1] is_orig: bool      = F
+                  [2] code: count        = 250
+                  [3] cmd: string        = MAIL
+                  [4] msg: string        = OK
+                  [5] cont_resp: bool    = F
+
+1254722769.957250 smtp_request
+                  [0] c: connection      = [id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], orig=[size=144, state=4, num_pkts=7, num_bytes_ip=393, flow_label=0], resp=[size=392, state=4, num_pkts=8, num_bytes_ip=720, flow_label=0], start_time=1254722767.529046, duration=2.428204, service={^J^ISMTP^J}, addl=, hot=0, history=ShAdDa, uid=CjhGID4nQcgTWjvg4c, tunnel=<uninitialized>, dpd=<uninitialized>, conn=<uninitialized>, extract_orig=F, extract_resp=F, dhcp=<uninitialized>, dnp3=<uninitialized>, dns=<uninitialized>, dns_state=<uninitialized>, ftp=<uninitialized>, ftp_data_reuse=F, ssl=<uninitialized>, http=<uninitialized>, http_state=<uninitialized>, irc=<uninitialized>, modbus=<uninitialized>, smtp=[ts=1254722768.219663, uid=CjhGID4nQcgTWjvg4c, id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], trans_depth=1, helo=GP, mailfrom=<gurpartap@patriots.in>, rcptto=<uninitialized>, date=<uninitialized>, from=<uninitialized>, to=<uninitialized>, reply_to=<uninitialized>, msg_id=<uninitialized>, in_reply_to=<uninitialized>, subject=<uninitialized>, x_originating_ip=<uninitialized>, first_received=<uninitialized>, second_received=<uninitialized>, last_reply=250 OK, path=[74.53.140.153, 10.10.1.4], user_agent=<uninitialized>, process_received_from=T, has_client_activity=T, entity=<uninitialized>, fuids=[]], smtp_state=[helo=GP, messages_transferred=0, pending_messages=<uninitialized>, mime_depth=0], socks=<uninitialized>, ssh=<uninitialized>, syslog=<uninitialized>]
+                  [1] is_orig: bool      = T
+                  [2] command: string    = RCPT
+                  [3] arg: string        = TO: <raj_deol2002in@yahoo.co.in>
+
+1254722770.319708 smtp_reply
+                  [0] c: connection      = [id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], orig=[size=144, state=4, num_pkts=8, num_bytes_ip=472, flow_label=0], resp=[size=406, state=4, num_pkts=8, num_bytes_ip=720, flow_label=0], start_time=1254722767.529046, duration=2.790662, service={^J^ISMTP^J}, addl=, hot=0, history=ShAdDa, uid=CjhGID4nQcgTWjvg4c, tunnel=<uninitialized>, dpd=<uninitialized>, conn=<uninitialized>, extract_orig=F, extract_resp=F, dhcp=<uninitialized>, dnp3=<uninitialized>, dns=<uninitialized>, dns_state=<uninitialized>, ftp=<uninitialized>, ftp_data_reuse=F, ssl=<uninitialized>, http=<uninitialized>, http_state=<uninitialized>, irc=<uninitialized>, modbus=<uninitialized>, smtp=[ts=1254722768.219663, uid=CjhGID4nQcgTWjvg4c, id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], trans_depth=1, helo=GP, mailfrom=<gurpartap@patriots.in>, rcptto={^J^I<raj_deol2002in@yahoo.co.in>^J}, date=<uninitialized>, from=<uninitialized>, to=<uninitialized>, reply_to=<uninitialized>, msg_id=<uninitialized>, in_reply_to=<uninitialized>, subject=<uninitialized>, x_originating_ip=<uninitialized>, first_received=<uninitialized>, second_received=<uninitialized>, last_reply=250 OK, path=[74.53.140.153, 10.10.1.4], user_agent=<uninitialized>, process_received_from=T, has_client_activity=T, entity=<uninitialized>, fuids=[]], smtp_state=[helo=GP, messages_transferred=0, pending_messages=<uninitialized>, mime_depth=0], socks=<uninitialized>, ssh=<uninitialized>, syslog=<uninitialized>]
+                  [1] is_orig: bool      = F
+                  [2] code: count        = 250
+                  [3] cmd: string        = RCPT
+                  [4] msg: string        = Accepted
+                  [5] cont_resp: bool    = F
+
+1254722770.320203 smtp_request
+                  [0] c: connection      = [id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], orig=[size=150, state=4, num_pkts=8, num_bytes_ip=472, flow_label=0], resp=[size=406, state=4, num_pkts=9, num_bytes_ip=774, flow_label=0], start_time=1254722767.529046, duration=2.791157, service={^J^ISMTP^J}, addl=, hot=0, history=ShAdDa, uid=CjhGID4nQcgTWjvg4c, tunnel=<uninitialized>, dpd=<uninitialized>, conn=<uninitialized>, extract_orig=F, extract_resp=F, dhcp=<uninitialized>, dnp3=<uninitialized>, dns=<uninitialized>, dns_state=<uninitialized>, ftp=<uninitialized>, ftp_data_reuse=F, ssl=<uninitialized>, http=<uninitialized>, http_state=<uninitialized>, irc=<uninitialized>, modbus=<uninitialized>, smtp=[ts=1254722768.219663, uid=CjhGID4nQcgTWjvg4c, id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], trans_depth=1, helo=GP, mailfrom=<gurpartap@patriots.in>, rcptto={^J^I<raj_deol2002in@yahoo.co.in>^J}, date=<uninitialized>, from=<uninitialized>, to=<uninitialized>, reply_to=<uninitialized>, msg_id=<uninitialized>, in_reply_to=<uninitialized>, subject=<uninitialized>, x_originating_ip=<uninitialized>, first_received=<uninitialized>, second_received=<uninitialized>, last_reply=250 Accepted, path=[74.53.140.153, 10.10.1.4], user_agent=<uninitialized>, process_received_from=T, has_client_activity=T, entity=<uninitialized>, fuids=[]], smtp_state=[helo=GP, messages_transferred=0, pending_messages=<uninitialized>, mime_depth=0], socks=<uninitialized>, ssh=<uninitialized>, syslog=<uninitialized>]
+                  [1] is_orig: bool      = T
+                  [2] command: string    = DATA
+                  [3] arg: string        = 
+
+1254722770.661679 smtp_reply
+                  [0] c: connection      = [id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], orig=[size=150, state=4, num_pkts=9, num_bytes_ip=518, flow_label=0], resp=[size=462, state=4, num_pkts=9, num_bytes_ip=774, flow_label=0], start_time=1254722767.529046, duration=3.132633, service={^J^ISMTP^J}, addl=, hot=0, history=ShAdDa, uid=CjhGID4nQcgTWjvg4c, tunnel=<uninitialized>, dpd=<uninitialized>, conn=<uninitialized>, extract_orig=F, extract_resp=F, dhcp=<uninitialized>, dnp3=<uninitialized>, dns=<uninitialized>, dns_state=<uninitialized>, ftp=<uninitialized>, ftp_data_reuse=F, ssl=<uninitialized>, http=<uninitialized>, http_state=<uninitialized>, irc=<uninitialized>, modbus=<uninitialized>, smtp=[ts=1254722768.219663, uid=CjhGID4nQcgTWjvg4c, id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], trans_depth=1, helo=GP, mailfrom=<gurpartap@patriots.in>, rcptto={^J^I<raj_deol2002in@yahoo.co.in>^J}, date=<uninitialized>, from=<uninitialized>, to=<uninitialized>, reply_to=<uninitialized>, msg_id=<uninitialized>, in_reply_to=<uninitialized>, subject=<uninitialized>, x_originating_ip=<uninitialized>, first_received=<uninitialized>, second_received=<uninitialized>, last_reply=250 Accepted, path=[74.53.140.153, 10.10.1.4], user_agent=<uninitialized>, process_received_from=T, has_client_activity=T, entity=[filename=<uninitialized>], fuids=[]], smtp_state=[helo=GP, messages_transferred=0, pending_messages=<uninitialized>, mime_depth=1], socks=<uninitialized>, ssh=<uninitialized>, syslog=<uninitialized>]
+                  [1] is_orig: bool      = F
+                  [2] code: count        = 354
+                  [3] cmd: string        = DATA
+                  [4] msg: string        = Enter message, ending with "." on a line by itself
+                  [5] cont_resp: bool    = F
+
+1254722771.858334 smtp_request
+                  [0] c: connection      = [id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], orig=[size=14699, state=4, num_pkts=23, num_bytes_ip=21438, flow_label=0], resp=[size=462, state=4, num_pkts=15, num_bytes_ip=1070, flow_label=0], start_time=1254722767.529046, duration=4.329288, service={^J^ISMTP^J}, addl=, hot=0, history=ShAdDa, uid=CjhGID4nQcgTWjvg4c, tunnel=<uninitialized>, dpd=<uninitialized>, conn=<uninitialized>, extract_orig=F, extract_resp=F, dhcp=<uninitialized>, dnp3=<uninitialized>, dns=<uninitialized>, dns_state=<uninitialized>, ftp=<uninitialized>, ftp_data_reuse=F, ssl=<uninitialized>, http=<uninitialized>, http_state=<uninitialized>, irc=<uninitialized>, modbus=<uninitialized>, smtp=[ts=1254722768.219663, uid=CjhGID4nQcgTWjvg4c, id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], trans_depth=1, helo=GP, mailfrom=<gurpartap@patriots.in>, rcptto={^J^I<raj_deol2002in@yahoo.co.in>^J}, date=Mon, 5 Oct 2009 11:36:07 +0530, from="Gurpartap Singh" <gurpartap@patriots.in>, to={^J^I<raj_deol2002in@yahoo.co.in>^J}, reply_to=<uninitialized>, msg_id=<000301ca4581$ef9e57f0$cedb07d0$@in>, in_reply_to=<uninitialized>, subject=SMTP, x_originating_ip=<uninitialized>, first_received=<uninitialized>, second_received=<uninitialized>, last_reply=354 Enter message, ending with "." on a line by itself, path=[74.53.140.153, 10.10.1.4], user_agent=Microsoft Office Outlook 12.0, process_received_from=T, has_client_activity=T, entity=<uninitialized>, fuids=[Fel9gs4OtNEV6gUJZ5, Ft4M3f2yMvLlmwtbq9, FL9Y0d45OI4LpS6fmh]], smtp_state=[helo=GP, messages_transferred=0, pending_messages=<uninitialized>, mime_depth=5], socks=<uninitialized>, ssh=<uninitialized>, syslog=<uninitialized>]
+                  [1] is_orig: bool      = T
+                  [2] command: string    = .
+                  [3] arg: string        = .
+
+1254722772.248789 smtp_reply
+                  [0] c: connection      = [id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], orig=[size=14699, state=4, num_pkts=24, num_bytes_ip=21507, flow_label=0], resp=[size=490, state=4, num_pkts=21, num_bytes_ip=1310, flow_label=0], start_time=1254722767.529046, duration=4.719743, service={^J^ISMTP^J}, addl=, hot=0, history=ShAdDa, uid=CjhGID4nQcgTWjvg4c, tunnel=<uninitialized>, dpd=<uninitialized>, conn=<uninitialized>, extract_orig=F, extract_resp=F, dhcp=<uninitialized>, dnp3=<uninitialized>, dns=<uninitialized>, dns_state=<uninitialized>, ftp=<uninitialized>, ftp_data_reuse=F, ssl=<uninitialized>, http=<uninitialized>, http_state=<uninitialized>, irc=<uninitialized>, modbus=<uninitialized>, smtp=[ts=1254722768.219663, uid=CjhGID4nQcgTWjvg4c, id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], trans_depth=1, helo=GP, mailfrom=<gurpartap@patriots.in>, rcptto={^J^I<raj_deol2002in@yahoo.co.in>^J}, date=Mon, 5 Oct 2009 11:36:07 +0530, from="Gurpartap Singh" <gurpartap@patriots.in>, to={^J^I<raj_deol2002in@yahoo.co.in>^J}, reply_to=<uninitialized>, msg_id=<000301ca4581$ef9e57f0$cedb07d0$@in>, in_reply_to=<uninitialized>, subject=SMTP, x_originating_ip=<uninitialized>, first_received=<uninitialized>, second_received=<uninitialized>, last_reply=354 Enter message, ending with "." on a line by itself, path=[74.53.140.153, 10.10.1.4], user_agent=Microsoft Office Outlook 12.0, process_received_from=T, has_client_activity=T, entity=<uninitialized>, fuids=[Fel9gs4OtNEV6gUJZ5, Ft4M3f2yMvLlmwtbq9, FL9Y0d45OI4LpS6fmh]], smtp_state=[helo=GP, messages_transferred=0, pending_messages=<uninitialized>, mime_depth=5], socks=<uninitialized>, ssh=<uninitialized>, syslog=<uninitialized>]
+                  [1] is_orig: bool      = F
+                  [2] code: count        = 250
+                  [3] cmd: string        = .
+                  [4] msg: string        = OK id=1Mugho-0003Dg-Un
+                  [5] cont_resp: bool    = F
+
+1254722774.763825 smtp_request
+                  [0] c: connection      = [id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], orig=[size=14705, state=4, num_pkts=25, num_bytes_ip=21547, flow_label=0], resp=[size=490, state=4, num_pkts=22, num_bytes_ip=1378, flow_label=0], start_time=1254722767.529046, duration=7.234779, service={^J^ISMTP^J}, addl=, hot=0, history=ShAdDa, uid=CjhGID4nQcgTWjvg4c, tunnel=<uninitialized>, dpd=<uninitialized>, conn=<uninitialized>, extract_orig=F, extract_resp=F, dhcp=<uninitialized>, dnp3=<uninitialized>, dns=<uninitialized>, dns_state=<uninitialized>, ftp=<uninitialized>, ftp_data_reuse=F, ssl=<uninitialized>, http=<uninitialized>, http_state=<uninitialized>, irc=<uninitialized>, modbus=<uninitialized>, smtp=[ts=1254722772.248789, uid=CjhGID4nQcgTWjvg4c, id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], trans_depth=2, helo=GP, mailfrom=<uninitialized>, rcptto=<uninitialized>, date=<uninitialized>, from=<uninitialized>, to=<uninitialized>, reply_to=<uninitialized>, msg_id=<uninitialized>, in_reply_to=<uninitialized>, subject=<uninitialized>, x_originating_ip=<uninitialized>, first_received=<uninitialized>, second_received=<uninitialized>, last_reply=<uninitialized>, path=[74.53.140.153, 10.10.1.4], user_agent=<uninitialized>, process_received_from=T, has_client_activity=F, entity=<uninitialized>, fuids=[]], smtp_state=[helo=GP, messages_transferred=1, pending_messages=<uninitialized>, mime_depth=5], socks=<uninitialized>, ssh=<uninitialized>, syslog=<uninitialized>]
+                  [1] is_orig: bool      = T
+                  [2] command: string    = QUIT
+                  [3] arg: string        = 
+
+1254722775.105467 smtp_reply
+                  [0] c: connection      = [id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], orig=[size=14705, state=5, num_pkts=27, num_bytes_ip=21633, flow_label=0], resp=[size=538, state=4, num_pkts=22, num_bytes_ip=1378, flow_label=0], start_time=1254722767.529046, duration=7.576421, service={^J^ISMTP^J}, addl=, hot=0, history=ShAdDaF, uid=CjhGID4nQcgTWjvg4c, tunnel=<uninitialized>, dpd=<uninitialized>, conn=<uninitialized>, extract_orig=F, extract_resp=F, dhcp=<uninitialized>, dnp3=<uninitialized>, dns=<uninitialized>, dns_state=<uninitialized>, ftp=<uninitialized>, ftp_data_reuse=F, ssl=<uninitialized>, http=<uninitialized>, http_state=<uninitialized>, irc=<uninitialized>, modbus=<uninitialized>, smtp=[ts=1254722772.248789, uid=CjhGID4nQcgTWjvg4c, id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], trans_depth=2, helo=GP, mailfrom=<uninitialized>, rcptto=<uninitialized>, date=<uninitialized>, from=<uninitialized>, to=<uninitialized>, reply_to=<uninitialized>, msg_id=<uninitialized>, in_reply_to=<uninitialized>, subject=<uninitialized>, x_originating_ip=<uninitialized>, first_received=<uninitialized>, second_received=<uninitialized>, last_reply=<uninitialized>, path=[74.53.140.153, 10.10.1.4], user_agent=<uninitialized>, process_received_from=T, has_client_activity=F, entity=<uninitialized>, fuids=[]], smtp_state=[helo=GP, messages_transferred=1, pending_messages=<uninitialized>, mime_depth=5], socks=<uninitialized>, ssh=<uninitialized>, syslog=<uninitialized>]
+                  [1] is_orig: bool      = F
+                  [2] code: count        = 221
+                  [3] cmd: string        = QUIT
+                  [4] msg: string        = xc90.websitewelcome.com closing connection
+                  [5] cont_resp: bool    = F
+

testing/btest/Baseline/scripts.policy.protocols.ssl.expiring-certs/notice.log

@@ -0,0 +1,11 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	notice
+#open	2014-03-13-21-37-53
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	fuid	file_mime_type	file_desc	proto	note	msg	sub	src	dst	p	n	peer_descr	actions	suppress_for	dropped	remote_location.country_code	remote_location.region	remote_location.city	remote_location.latitude	remote_location.longitude
+#types	time	string	addr	port	addr	port	string	string	string	enum	enum	string	string	addr	addr	port	count	string	table[enum]	interval	bool	string	string	string	double	double
+1394745603.293028	CXWv6p3arKYeMETxOg	192.168.4.149	60539	87.98.220.10	443	F1fX1R2cDOzbvg17ye	-	-	tcp	SSL::Certificate_Expired	Certificate CN=www.spidh.org,OU=COMODO SSL,OU=Domain Control Validated expired at 2014-03-04-23:59:59.000000000	-	192.168.4.149	87.98.220.10	443	-	bro	Notice::ACTION_LOG	86400.000000	F	-	-	-	-	-
+1394745619.197766	CjhGID4nQcgTWjvg4c	192.168.4.149	60540	122.1.240.204	443	F6NAbK127LhNBaEe5c	-	-	tcp	SSL::Certificate_Expires_Soon	Certificate CN=www.tobu-estate.com,OU=Terms of use at www.verisign.com/rpa (c)05,O=TOBU RAILWAY Co.\,Ltd.,L=Sumida-ku,ST=Tokyo,C=JP is going to expire at 2014-03-14-23:59:59.000000000	-	192.168.4.149	122.1.240.204	443	-	bro	Notice::ACTION_LOG	86400.000000	F	-	-	-	-	-
+#close	2014-03-13-21-37-53

testing/btest/Baseline/scripts.policy.protocols.ssl.extract-certs-pem/certs-remote.pem

@@ -1,34 +1,26 @@
 -----BEGIN CERTIFICATE-----
-MIIEfDCCA+WgAwIBAgIQBKeBFvADKDvaK4RiBJ+eyzANBgkqhk
+MIIEfDCCA+WgAwIBAgIQBKeBFvADKDvaK4RiBJ+eyzANBgkqhkiG9w0BAQUFADCB
-iG9w0BAQUFADCBujEfMB0GA1UEChMWVmVyaVNpZ24gVHJ1c3Qg
+ujEfMB0GA1UEChMWVmVyaVNpZ24gVHJ1c3QgTmV0d29yazEXMBUGA1UECxMOVmVy
-TmV0d29yazEXMBUGA1UECxMOVmVyaVNpZ24sIEluYy4xMzAxBg
+aVNpZ24sIEluYy4xMzAxBgNVBAsTKlZlcmlTaWduIEludGVybmF0aW9uYWwgU2Vy
-NVBAsTKlZlcmlTaWduIEludGVybmF0aW9uYWwgU2VydmVyIENB
+dmVyIENBIC0gQ2xhc3MgMzFJMEcGA1UECxNAd3d3LnZlcmlzaWduLmNvbS9DUFMg
-IC0gQ2xhc3MgMzFJMEcGA1UECxNAd3d3LnZlcmlzaWduLmNvbS
+SW5jb3JwLmJ5IFJlZi4gTElBQklMSVRZIExURC4oYyk5NyBWZXJpU2lnbjAeFw0w
-9DUFMgSW5jb3JwLmJ5IFJlZi4gTElBQklMSVRZIExURC4oYyk5
+NjExMTQwMDAwMDBaFw0wNzExMTQyMzU5NTlaMIHAMQswCQYDVQQGEwJERTEPMA0G
-NyBWZXJpU2lnbjAeFw0wNjExMTQwMDAwMDBaFw0wNzExMTQyMz
+A1UECBMGQmF5ZXJuMREwDwYDVQQHFAhNdWVuY2hlbjE3MDUGA1UEChQuQUdJUyBB
-U5NTlaMIHAMQswCQYDVQQGEwJERTEPMA0GA1UECBMGQmF5ZXJu
+bGxpYW56IERyZXNkbmVyIEluZm9ybWF0aW9uc3N5c3RlbWUgR21iSDEzMDEGA1UE
-MREwDwYDVQQHFAhNdWVuY2hlbjE3MDUGA1UEChQuQUdJUyBBbG
+CxQqVGVybXMgb2YgdXNlIGF0IHd3dy52ZXJpc2lnbi5jb20vcnBhIChjKTAwMR8w
-xpYW56IERyZXNkbmVyIEluZm9ybWF0aW9uc3N5c3RlbWUgR21i
+HQYDVQQDFBZ3d3cuZHJlc2RuZXItcHJpdmF0LmRlMIGfMA0GCSqGSIb3DQEBAQUA
-SDEzMDEGA1UECxQqVGVybXMgb2YgdXNlIGF0IHd3dy52ZXJpc2
+A4GNADCBiQKBgQDrqHR+++O06r6LHD3t6oYEYlHgKlqehm+Yy7zF7cXIyladTJJY
-lnbi5jb20vcnBhIChjKTAwMR8wHQYDVQQDFBZ3d3cuZHJlc2Ru
+4WsTb7y35S6YQPeP1qPACqtGUhs4/AUg54Duxl3VuwP8xYO6mmcI/Sy6owiU8LMf
-ZXItcHJpdmF0LmRlMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQ
+Fij2BWZbv3+oWfq+mWs2YrhuxoNHU2MPWrRRwYioVbnUMW09KkqVCtF7hwIDAQAB
-KBgQDrqHR+++O06r6LHD3t6oYEYlHgKlqehm+Yy7zF7cXIylad
+o4IBeTCCAXUwCQYDVR0TBAIwADALBgNVHQ8EBAMCBaAwRgYDVR0fBD8wPTA7oDmg
-TJJY4WsTb7y35S6YQPeP1qPACqtGUhs4/AUg54Duxl3VuwP8xY
+N4Y1aHR0cDovL2NybC52ZXJpc2lnbi5jb20vQ2xhc3MzSW50ZXJuYXRpb25hbFNl
-O6mmcI/Sy6owiU8LMfFij2BWZbv3+oWfq+mWs2YrhuxoNHU2MP
+cnZlci5jcmwwRAYDVR0gBD0wOzA5BgtghkgBhvhFAQcXAzAqMCgGCCsGAQUFBwIB
-WrRRwYioVbnUMW09KkqVCtF7hwIDAQABo4IBeTCCAXUwCQYDVR
+FhxodHRwczovL3d3dy52ZXJpc2lnbi5jb20vcnBhMCgGA1UdJQQhMB8GCWCGSAGG
-0TBAIwADALBgNVHQ8EBAMCBaAwRgYDVR0fBD8wPTA7oDmgN4Y1
++EIEAQYIKwYBBQUHAwEGCCsGAQUFBwMCMDQGCCsGAQUFBwEBBCgwJjAkBggrBgEF
-aHR0cDovL2NybC52ZXJpc2lnbi5jb20vQ2xhc3MzSW50ZXJuYX
+BQcwAYYYaHR0cDovL29jc3AudmVyaXNpZ24uY29tMG0GCCsGAQUFBwEMBGEwX6Fd
-Rpb25hbFNlcnZlci5jcmwwRAYDVR0gBD0wOzA5BgtghkgBhvhF
+oFswWTBXMFUWCWltYWdlL2dpZjAhMB8wBwYFKw4DAhoEFI/l0xqGrI2Oa8PPgGrU
-AQcXAzAqMCgGCCsGAQUFBwIBFhxodHRwczovL3d3dy52ZXJpc2
+SBgsexkuMCUWI2h0dHA6Ly9sb2dvLnZlcmlzaWduLmNvbS92c2xvZ28uZ2lmMA0G
-lnbi5jb20vcnBhMCgGA1UdJQQhMB8GCWCGSAGG+EIEAQYIKwYB
+CSqGSIb3DQEBBQUAA4GBAC9z4m/BniN+WVCJlXhv6QS9mFRTYOwIUtIKKZKabarV
-BQUHAwEGCCsGAQUFBwMCMDQGCCsGAQUFBwEBBCgwJjAkBggrBg
+sWfBYt7JGE5XPWmcsgNmkgO76E3FmNQvQtm20uCXEFh2z+fWp8y72yXuQl3L8HSr
-EFBQcwAYYYaHR0cDovL29jc3AudmVyaXNpZ24uY29tMG0GCCsG
+0lTl6LpRD6TDPjT6UvKg5nr0j9x2Qr09/HjAt+teLR/FoF7foBGH+MNYEMh5KPjk
-AQUFBwEMBGEwX6FdoFswWTBXMFUWCWltYWdlL2dpZjAhMB8wBw
-YFKw4DAhoEFI/l0xqGrI2Oa8PPgGrUSBgsexkuMCUWI2h0dHA6
-Ly9sb2dvLnZlcmlzaWduLmNvbS92c2xvZ28uZ2lmMA0GCSqGSI
-b3DQEBBQUAA4GBAC9z4m/BniN+WVCJlXhv6QS9mFRTYOwIUtIK
-KZKabarVsWfBYt7JGE5XPWmcsgNmkgO76E3FmNQvQtm20uCXEF
-h2z+fWp8y72yXuQl3L8HSr0lTl6LpRD6TDPjT6UvKg5nr0j9x2
-Qr09/HjAt+teLR/FoF7foBGH+MNYEMh5KPjk
 -----END CERTIFICATE-----
-

testing/btest/Baseline/scripts.policy.protocols.ssl.known-certs/known_certs.log

@@ -0,0 +1,10 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	known_certs
+#open	2014-03-13-21-47-24
+#fields	ts	host	port_num	subject	issuer_subject	serial
+#types	time	addr	port	string	string	string
+1394747126.871404	74.125.239.129	443	CN=*.google.com,O=Google Inc,L=Mountain View,ST=California,C=US	CN=Google Internet Authority G2,O=Google Inc,C=US	4A2C8628C1010633
+#close	2014-03-13-21-47-24

testing/btest/Baseline/scripts.policy.protocols.ssl.known-certs/ssl.log

@@ -0,0 +1,11 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	ssl
+#open	2014-03-13-21-47-24
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	version	cipher	server_name	session_id	last_alert	established	cert_chain_fuids	client_cert_chain_fuids	subject	issuer	client_subject	client_issuer
+#types	time	string	addr	port	addr	port	string	string	string	string	string	bool	vector[string]	vector[string]	string	string	string	string
+1394747126.855035	CXWv6p3arKYeMETxOg	192.168.4.149	60623	74.125.239.129	443	TLSv12	TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256	-	-	-	T	FlaIzV19yTmBYwWwc6,F0BeiV3cMsGkNML0P2,F6PfYi2WUoPdIJrhpg	(empty)	CN=*.google.com,O=Google Inc,L=Mountain View,ST=California,C=US	CN=Google Internet Authority G2,O=Google Inc,C=US	-	-
+1394747129.505622	CjhGID4nQcgTWjvg4c	192.168.4.149	60624	74.125.239.129	443	TLSv12	TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256	-	-	-	T	FOye6a4kt8a7QChqw3,FytlLr3jOQenFAVtYi,FEmnxy4DGbxkmtQJS1	(empty)	CN=*.google.com,O=Google Inc,L=Mountain View,ST=California,C=US	CN=Google Internet Authority G2,O=Google Inc,C=US	-	-
+#close	2014-03-13-21-47-24

testing/btest/Baseline/scripts.policy.protocols.ssl.known-certs/x509.log

@@ -0,0 +1,15 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	x509
+#open	2014-03-13-21-47-24
+#fields	ts	id	certificate.version	certificate.serial	certificate.subject	certificate.issuer	certificate.not_valid_before	certificate.not_valid_after	certificate.key_alg	certificate.sig_alg	certificate.key_type	certificate.key_length	certificate.exponent	certificate.curve	san.dns	san.uri	san.email	san.ip	basic_constraints.ca	basic_constraints.path_len
+#types	time	string	count	string	string	string	time	time	string	string	string	count	string	string	vector[string]	vector[string]	vector[string]	vector[addr]	bool	count
+1394747126.862409	FlaIzV19yTmBYwWwc6	2	4A2C8628C1010633	CN=*.google.com,O=Google Inc,L=Mountain View,ST=California,C=US	CN=Google Internet Authority G2,O=Google Inc,C=US	1393341558.000000	1401062400.000000	rsaEncryption	sha1WithRSAEncryption	rsa	2048	65537	-	*.google.com,*.android.com,*.appengine.google.com,*.cloud.google.com,*.google-analytics.com,*.google.ca,*.google.cl,*.google.co.in,*.google.co.jp,*.google.co.uk,*.google.com.ar,*.google.com.au,*.google.com.br,*.google.com.co,*.google.com.mx,*.google.com.tr,*.google.com.vn,*.google.de,*.google.es,*.google.fr,*.google.hu,*.google.it,*.google.nl,*.google.pl,*.google.pt,*.googleapis.cn,*.googlecommerce.com,*.googlevideo.com,*.gstatic.com,*.gvt1.com,*.urchin.com,*.url.google.com,*.youtube-nocookie.com,*.youtube.com,*.youtubeeducation.com,*.ytimg.com,android.com,g.co,goo.gl,google-analytics.com,google.com,googlecommerce.com,urchin.com,youtu.be,youtube.com,youtubeeducation.com	-	-	-	F	-
+1394747126.862409	F0BeiV3cMsGkNML0P2	2	023A69	CN=Google Internet Authority G2,O=Google Inc,C=US	CN=GeoTrust Global CA,O=GeoTrust Inc.,C=US	1365174955.000000	1428160555.000000	rsaEncryption	sha1WithRSAEncryption	rsa	2048	65537	-	-	-	-	-	T	0
+1394747126.862409	F6PfYi2WUoPdIJrhpg	2	12BBE6	CN=GeoTrust Global CA,O=GeoTrust Inc.,C=US	OU=Equifax Secure Certificate Authority,O=Equifax,C=US	1021953600.000000	1534824000.000000	rsaEncryption	sha1WithRSAEncryption	rsa	2048	65537	-	-	-	-	-	T	-
+1394747129.512954	FOye6a4kt8a7QChqw3	2	4A2C8628C1010633	CN=*.google.com,O=Google Inc,L=Mountain View,ST=California,C=US	CN=Google Internet Authority G2,O=Google Inc,C=US	1393341558.000000	1401062400.000000	rsaEncryption	sha1WithRSAEncryption	rsa	2048	65537	-	*.google.com,*.android.com,*.appengine.google.com,*.cloud.google.com,*.google-analytics.com,*.google.ca,*.google.cl,*.google.co.in,*.google.co.jp,*.google.co.uk,*.google.com.ar,*.google.com.au,*.google.com.br,*.google.com.co,*.google.com.mx,*.google.com.tr,*.google.com.vn,*.google.de,*.google.es,*.google.fr,*.google.hu,*.google.it,*.google.nl,*.google.pl,*.google.pt,*.googleapis.cn,*.googlecommerce.com,*.googlevideo.com,*.gstatic.com,*.gvt1.com,*.urchin.com,*.url.google.com,*.youtube-nocookie.com,*.youtube.com,*.youtubeeducation.com,*.ytimg.com,android.com,g.co,goo.gl,google-analytics.com,google.com,googlecommerce.com,urchin.com,youtu.be,youtube.com,youtubeeducation.com	-	-	-	F	-
+1394747129.512954	FytlLr3jOQenFAVtYi	2	023A69	CN=Google Internet Authority G2,O=Google Inc,C=US	CN=GeoTrust Global CA,O=GeoTrust Inc.,C=US	1365174955.000000	1428160555.000000	rsaEncryption	sha1WithRSAEncryption	rsa	2048	65537	-	-	-	-	-	T	0
+1394747129.512954	FEmnxy4DGbxkmtQJS1	2	12BBE6	CN=GeoTrust Global CA,O=GeoTrust Inc.,C=US	OU=Equifax Secure Certificate Authority,O=Equifax,C=US	1021953600.000000	1534824000.000000	rsaEncryption	sha1WithRSAEncryption	rsa	2048	65537	-	-	-	-	-	T	-
+#close	2014-03-13-21-47-24

testing/btest/Baseline/scripts.policy.protocols.ssl.validate-certs/ssl.log

@@ -0,0 +1,11 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	ssl
+#open	2014-03-13-21-53-03
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	version	cipher	server_name	session_id	last_alert	established	cert_chain_fuids	client_cert_chain_fuids	subject	issuer	client_subject	client_issuer	validation_status
+#types	time	string	addr	port	addr	port	string	string	string	string	string	bool	vector[string]	vector[string]	string	string	string	string	string
+1394745602.951961	CXWv6p3arKYeMETxOg	192.168.4.149	60539	87.98.220.10	443	TLSv10	TLS_DHE_RSA_WITH_AES_256_CBC_SHA	-	-	-	T	F1fX1R2cDOzbvg17ye,FqPEQR2eytAQybroyl	(empty)	CN=www.spidh.org,OU=COMODO SSL,OU=Domain Control Validated	CN=COMODO SSL CA,O=COMODO CA Limited,L=Salford,ST=Greater Manchester,C=GB	-	-	certificate has expired
+1394745618.791420	CjhGID4nQcgTWjvg4c	192.168.4.149	60540	122.1.240.204	443	TLSv10	TLS_RSA_WITH_AES_256_CBC_SHA	-	-	-	T	F6NAbK127LhNBaEe5c,FDhmPt28vyXlGMTxP7,F0ROCKibhE1KntJ1h	(empty)	CN=www.tobu-estate.com,OU=Terms of use at www.verisign.com/rpa (c)05,O=TOBU RAILWAY Co.\,Ltd.,L=Sumida-ku,ST=Tokyo,C=JP	CN=VeriSign Class 3 Secure Server CA - G3,OU=Terms of use at https://www.verisign.com/rpa (c)10,OU=VeriSign Trust Network,O=VeriSign\, Inc.,C=US	-	-	ok
+#close	2014-03-13-21-53-03

testing/btest/bifs/x509_verify.bro

@@ -0,0 +1,25 @@
+# @TEST-EXEC: bro -r $TRACES/tls/tls-expired-cert.trace %INPUT 
+# @TEST-EXEC: btest-diff .stdout
+
+event ssl_established(c: connection) &priority=3
+	{
+	local chain: vector of opaque of x509 = vector();
+	for ( i in c$ssl$cert_chain )
+		{
+		chain[i] = c$ssl$cert_chain[i]$x509$handle;
+		}
+
+	local result = x509_verify(chain, SSL::root_certs);
+	print fmt("Validation result: %s", result$result_string);
+	if ( result$result != 0 ) # not ok
+		return;
+
+	print "Resulting chain:";
+	for ( i in result$chain_certs )
+		{
+		local cert = result$chain_certs[i];
+		local certinfo = x509_parse(cert);
+		local sha1 = sha1_hash(x509_get_certificate_string(cert));
+		print fmt("Fingerprint: %s, Subject: %s", sha1, certinfo$subject);
+		}
+	}

testing/btest/core/leaks/x509_verify.bro

@@ -0,0 +1,33 @@
+# Needs perftools support.
+#
+# @TEST-GROUP: leaks
+#
+# @TEST-REQUIRES: bro  --help 2>&1 | grep -q mem-leaks
+#
+# @TEST-EXEC: HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local btest-bg-run bro bro -b -m -r $TRACES/tls/tls-expired-cert.trace %INPUT
+# @TEST-EXEC: btest-bg-wait 15
+
+@load base/protocols/ssl
+
+event ssl_established(c: connection) &priority=3
+	{
+	local chain: vector of opaque of x509 = vector();
+	for ( i in c$ssl$cert_chain )
+		{
+		chain[i] = c$ssl$cert_chain[i]$x509$handle;
+		}
+
+	local result = x509_verify(chain, SSL::root_certs);
+	print fmt("Validation result: %s", result$result_string);
+	if ( result$result != 0 ) # not ok
+		return;
+
+	print "Resulting chain:";
+	for ( i in result$chain_certs )
+		{
+		local cert = result$chain_certs[i];
+		local certinfo = x509_parse(cert);
+		local sha1 = sha1_hash(x509_get_certificate_string(cert));
+		print fmt("Fingerprint: %s, Subject: %s", sha1, certinfo$subject);
+		}
+	}

testing/btest/scripts/base/protocols/ftp/gridftp.test

@@ -2,6 +2,7 @@
 # @TEST-EXEC: btest-diff notice.log
 # @TEST-EXEC: btest-diff conn.log
 # @TEST-EXEC: btest-diff ssl.log
+# @TEST-EXEC: btest-diff x509.log
 
 @load base/protocols/ftp/gridftp
 

testing/btest/scripts/base/protocols/ssl/basic.test

@@ -1,4 +1,5 @@
 # This tests a normal SSL connection and the log it outputs.
 
-# @TEST-EXEC: bro -r $TRACES/tls-conn-with-extensions.trace %INPUT
+# @TEST-EXEC: bro -r $TRACES/tls/tls-conn-with-extensions.trace %INPUT
 # @TEST-EXEC: btest-diff ssl.log
+# @TEST-EXEC: btest-diff x509.log

testing/btest/scripts/base/protocols/ssl/tls-1.2-ciphers.test

@@ -1,4 +1,4 @@
-# @TEST-EXEC: bro -r $TRACES/tls1.2.trace %INPUT
+# @TEST-EXEC: bro -r $TRACES/tls/tls1.2.trace %INPUT
 # @TEST-EXEC: btest-diff .stdout
 
 event ssl_client_hello(c: connection, version: count, possible_ts: time, client_random: string, session_id: string, ciphers: index_vec) 

testing/btest/scripts/base/protocols/ssl/tls-1.2-handshake-failure.test

@@ -1,2 +1,2 @@
-# @TEST-EXEC: bro -r $TRACES/tls-1.2-handshake-failure.trace %INPUT
+# @TEST-EXEC: bro -r $TRACES/tls/tls-1.2-handshake-failure.trace %INPUT
 # @TEST-EXEC: btest-diff ssl.log

testing/btest/scripts/base/protocols/ssl/tls-1.2-random.test

@@ -1,4 +1,4 @@
-# @TEST-EXEC: bro -r $TRACES/tls1.2.trace %INPUT
+# @TEST-EXEC: bro -r $TRACES/tls/tls1.2.trace %INPUT
 # @TEST-EXEC: btest-diff .stdout
 
 event ssl_client_hello(c: connection, version: count, possible_ts: time, client_random: string, session_id: string, ciphers: index_vec) 

testing/btest/scripts/base/protocols/ssl/tls-1.2.test

@@ -1,2 +1,3 @@
-# @TEST-EXEC: bro -r $TRACES/tls1.2.trace %INPUT
+# @TEST-EXEC: bro -r $TRACES/tls/tls1.2.trace %INPUT
 # @TEST-EXEC: btest-diff ssl.log
+# @TEST-EXEC: btest-diff x509.log

testing/btest/scripts/base/protocols/ssl/x509_extensions.test

@@ -1,7 +1,7 @@
-# @TEST-EXEC: bro -r $TRACES/tls1.2.trace %INPUT
+# @TEST-EXEC: bro -r $TRACES/tls/tls1.2.trace %INPUT
 # @TEST-EXEC: btest-diff .stdout
 
-event x509_extension(c: connection, is_orig: bool, cert:X509, extension: X509_extension_info) 
+event x509_extension(f: fa_file, extension: X509::Extension)
 {
 	# The formatting of CRL Distribution Points varies between OpenSSL versions. Skip it
 	# for the test.

testing/btest/scripts/policy/misc/dump-events.bro

@@ -1,7 +1,7 @@
-# @TEST-EXEC: bro -r $TRACES/ssl.v3.trace policy/misc/dump-events.bro >all-events.log
+# @TEST-EXEC: bro -r $TRACES/smtp.trace policy/misc/dump-events.bro >all-events.log
-# @TEST-EXEC: bro -r $TRACES/ssl.v3.trace policy/misc/dump-events.bro DumpEvents::include_args=F >all-events-no-args.log
+# @TEST-EXEC: bro -r $TRACES/smtp.trace policy/misc/dump-events.bro DumpEvents::include_args=F >all-events-no-args.log
-# @TEST-EXEC: bro -r $TRACES/ssl.v3.trace policy/misc/dump-events.bro DumpEvents::include=/ssl_/ >ssl-events.log
+# @TEST-EXEC: bro -r $TRACES/smtp.trace policy/misc/dump-events.bro DumpEvents::include=/smtp_/ >smtp-events.log
 # 
 # @TEST-EXEC: btest-diff all-events.log
 # @TEST-EXEC: btest-diff all-events-no-args.log
-# @TEST-EXEC: btest-diff ssl-events.log
+# @TEST-EXEC: btest-diff smtp-events.log

testing/btest/scripts/policy/protocols/ssl/expiring-certs.bro

@@ -0,0 +1,7 @@
+# @TEST-EXEC: bro -r $TRACES/tls/tls-expired-cert.trace %INPUT
+# @TEST-EXEC: btest-diff notice.log
+
+@load protocols/ssl/expiring-certs
+
+redef SSL::notify_certs_expiration = ALL_HOSTS;
+

testing/btest/scripts/policy/protocols/ssl/extract-certs-pem.bro

@@ -1,4 +1,4 @@
-# @TEST-EXEC: bro -r $TRACES/ssl.v3.trace %INPUT 
+# @TEST-EXEC: bro -r $TRACES/tls/ssl.v3.trace %INPUT
 # @TEST-EXEC: btest-diff certs-remote.pem
 
 @load protocols/ssl/extract-certs-pem

testing/btest/scripts/policy/protocols/ssl/known-certs.bro

@@ -0,0 +1,9 @@
+# @TEST-EXEC: bro -r $TRACES/tls/google-duplicate.trace %INPUT
+# @TEST-EXEC: btest-diff ssl.log
+# @TEST-EXEC: btest-diff x509.log
+# @TEST-EXEC: btest-diff known_certs.log
+
+@load protocols/ssl/known-certs
+
+redef Known::cert_tracking = ALL_HOSTS;
+

testing/btest/scripts/policy/protocols/ssl/validate-certs.bro

@@ -0,0 +1,4 @@
+# @TEST-EXEC: bro -r $TRACES/tls/tls-expired-cert.trace %INPUT
+# @TEST-EXEC: btest-diff ssl.log
+
+@load protocols/ssl/validate-certs