Merge remote-tracking branch 'origin/topic/matthias/notary'

* origin/topic/matthias/notary:
  Small cosmetic changes.
  Give log buffer the correct name.
  Simplify delayed logging of SSL records.
  Implement delay-token style SSL logging.
  More style tweaks: replace spaces with tabs.
  Factor notary code into separte file.
  Adhere to Bro coding style guidelines.
  Enhance ssl.log with information from notary.

Closes #928

CHANGES

@@ -1,4 +1,12 @@
 
+2.1-328 | 2013-02-05 01:34:29 -0500
+
+  * New script to query the ICSI Certificate Notary 
+    (http://notary.icsi.berkeley.edu/) over DNS and add information
+    to the SSL log at runtime. (Matthias Vallentin) 
+
+  * Add delayed logging to SSL base scripts. (Matthias Vallentin)
+
 2.1-319 | 2013-02-04 09:45:34 -0800
 
   * Update input tests to use exit_only_after_terminate. (Bernhard

VERSION

@@ -1 +1 @@
-2.1-319
+2.1-328

doc/scripts/DocSourcesList.cmake

@@ -162,6 +162,7 @@ rest_target(${psd} policy/protocols/ssl/cert-hash.bro)
 rest_target(${psd} policy/protocols/ssl/expiring-certs.bro)
 rest_target(${psd} policy/protocols/ssl/extract-certs-pem.bro)
 rest_target(${psd} policy/protocols/ssl/known-certs.bro)
+rest_target(${psd} policy/protocols/ssl/notary.bro)
 rest_target(${psd} policy/protocols/ssl/validate-certs.bro)
 rest_target(${psd} policy/tuning/defaults/packet-fragments.bro)
 rest_target(${psd} policy/tuning/defaults/warnings.bro)

scripts/base/protocols/ssl/main.bro

@@ -72,6 +72,17 @@ export {
 	## utility.
 	const openssl_util = "openssl" &redef;
 
+	## The maximum amount of time a script can delay records from being logged.
+	const max_log_delay = 15secs &redef;
+
+	## Delays an SSL record for a specific token: the record will not be logged
+	## as longs the token exists or until :bro:id:`SSL::max_log_delay` elapses.
+	global delay_log: function(info: Info, token: string);
+
+	## Undelays an SSL record for a previously inserted token, allowing the
+	## record to be logged.
+	global undelay_log: function(info: Info, token: string);
+
 	## Event that can be handled to access the SSL
 	## record as it is sent on to the logging framework.
 	global log_ssl: event(rec: Info);
@@ -81,6 +92,13 @@ redef record connection += {
 	ssl: Info &optional;
 };
 
+redef record Info += {
+		# Adding a string "token" to this set will cause the SSL script
+		# to delay logging the record until either the token has been removed or
+		# the record has been delayed for :bro:id:`SSL::max_log_delay`.
+		delay_tokens: set[string] &optional;
+};
+
 event bro_init() &priority=5
 	{
 	Log::create_stream(SSL::LOG, [$columns=Info, $ev=log_ssl]);
@@ -115,6 +133,13 @@ redef likely_server_ports += {
 	989/tcp, 990/tcp, 992/tcp, 993/tcp, 995/tcp, 5223/tcp
 };
 
+# A queue that buffers log records.
+global log_delay_queue: table[count] of Info;
+# The top queue index where records are added.
+global log_delay_queue_head = 0;
+# The bottom queue index that points to the next record to be flushed.
+global log_delay_queue_tail = 0;
+
 function set_session(c: connection)
 	{
 	if ( ! c?$ssl )
@@ -122,12 +147,65 @@ function set_session(c: connection)
 		         $client_cert_chain=vector()];
 	}
 
+function delay_log(info: Info, token: string)
+	{
+	info$delay_tokens = set();
+	add info$delay_tokens[token];
+
+	log_delay_queue[log_delay_queue_head] = info;
+	++log_delay_queue_head;
+	}
+
+function undelay_log(info: Info, token: string)
+	{
+	if ( token in info$delay_tokens )
+		delete info$delay_tokens[token];
+	}
+
+global log_record: function(info: Info);
+
+event delay_logging(info: Info)
+	{
+	log_record(info);
+	}
+
+function log_record(info: Info)
+	{
+	if ( ! info?$delay_tokens || |info$delay_tokens| == 0 )
+		{
+		Log::write(SSL::LOG, info);
+		}
+	else
+		{
+		for ( unused_index in log_delay_queue )
+			{
+			if ( log_delay_queue_head == log_delay_queue_tail )
+				return;
+			if ( |log_delay_queue[log_delay_queue_tail]$delay_tokens| > 0 )
+				{
+				if ( info$ts + max_log_delay > network_time() )
+					{
+					schedule 1sec { delay_logging(info) };
+					return;
+					}
+				else
+					{
+					Reporter::info(fmt("SSL delay tokens not released in time (%s)",
+					                   info$delay_tokens));
+					}
+				}
+			Log::write(SSL::LOG, log_delay_queue[log_delay_queue_tail]);
+			delete log_delay_queue[log_delay_queue_tail];
+			++log_delay_queue_tail;
+			}
+		}
+	}
+
 function finish(c: connection)
 	{
-	Log::write(SSL::LOG, c$ssl);
+	log_record(c$ssl);
 	if ( disable_analyzer_after_detection && c?$ssl && c$ssl?$analyzer_id )
 		disable_analyzer(c$id, c$ssl$analyzer_id);
-	delete c$ssl;
 	}
 
 event ssl_client_hello(c: connection, version: count, possible_ts: time, session_id: string, ciphers: count_set) &priority=5
@@ -228,3 +306,15 @@ event protocol_violation(c: connection, atype: count, aid: count,
 	if ( c?$ssl )
 		finish(c);
 	}
+
+event bro_done()
+	{
+	if ( |log_delay_queue| == 0 )
+		return;
+	for ( unused_index in log_delay_queue )
+		{
+		Log::write(SSL::LOG, log_delay_queue[log_delay_queue_tail]);
+		delete log_delay_queue[log_delay_queue_tail];
+		++log_delay_queue_tail;
+		}
+	}

scripts/policy/protocols/ssl/notary.bro

@@ -0,0 +1,105 @@
+@load base/protocols/ssl
+
+module CertNotary;
+
+export {
+	## A response from the ICSI certificate notary.
+	type Response: record {
+		first_seen: count &log &optional;
+		last_seen:  count &log &optional;
+		times_seen: count &log &optional;
+		valid:      bool  &log &optional;
+	};
+
+	## The notary domain to query.
+	const domain = "notary.icsi.berkeley.edu" &redef;
+}
+
+redef record SSL::Info += {
+	sha1:   string &log &optional;
+	notary: Response &log &optional;
+	};
+
+# The DNS cache of notary responses.
+global notary_cache: table[string] of Response &create_expire = 1 hr;
+
+# The records that wait for a notary response identified by the cert digest.
+# Each digest refers to a list of connection UIDs which are updated when a DNS
+# reply arrives asynchronously.
+global waitlist: table[string] of vector of SSL::Info;
+
+function clear_waitlist(digest: string)
+	{
+	if ( digest in waitlist )
+		{
+		for ( i in waitlist[digest] )
+			SSL::undelay_log(waitlist[digest][i], "notary");
+		delete waitlist[digest];
+		}
+	}
+
+event x509_certificate(c: connection, is_orig: bool, cert: X509,
+                       chain_idx: count, chain_len: count, der_cert: string)
+	{
+	if ( is_orig || chain_idx != 0 || ! c?$ssl )
+		return;
+
+	local digest = sha1_hash(der_cert);
+	c$ssl$sha1 = digest;
+
+	if ( digest in notary_cache )
+		{
+		c$ssl$notary = notary_cache[digest];
+		return;
+		}
+
+	SSL::delay_log(c$ssl, "notary");
+
+	local waits_already = digest in waitlist;
+	if ( ! waits_already )
+		waitlist[digest] = vector();
+	waitlist[digest][|waitlist[digest]|] = c$ssl;
+	if ( waits_already )
+		return;
+
+	when ( local str = lookup_hostname_txt(fmt("%s.%s", digest, domain)) )
+		{
+		notary_cache[digest] = [];
+
+		# Parse notary answer.
+		if ( str == "<???>" ) # NXDOMAIN
+			{
+			clear_waitlist(digest);
+			return;
+			}
+		local fields = split(str, / /);
+		if ( |fields| != 5 ) # version 1 has 5 fields.
+			{
+			clear_waitlist(digest);
+			return;
+			}
+		local version = split(fields[1], /=/)[2];
+		if ( version != "1" )
+			{
+			clear_waitlist(digest);
+			return;
+			}
+		local r = notary_cache[digest];
+		r$first_seen = to_count(split(fields[2], /=/)[2]);
+		r$last_seen = to_count(split(fields[3], /=/)[2]);
+		r$times_seen = to_count(split(fields[4], /=/)[2]);
+		r$valid = split(fields[5], /=/)[2] == "1";
+
+		# Assign notary answer to all records waiting for this digest.
+		if ( digest in waitlist )
+			{
+			for ( i in waitlist[digest] )
+				{
+				local info = waitlist[digest][i];
+				SSL::undelay_log(info, "notary");
+				info$notary = r;
+				}
+			delete waitlist[digest];
+			}
+		}
+	}

scripts/site/local.bro

@@ -56,6 +56,10 @@ redef Software::vulnerable_versions += {
 # This script enables SSL/TLS certificate validation.
 @load protocols/ssl/validate-certs
 
+# This script checks each SSL certificate hash against the ICSI certificate
+# notary service.
+@load protocols/ssl/notary
+
 # If you have libGeoIP support built in, do some geographic detections and 
 # logging for SSH traffic.
 @load protocols/ssh/geo-data

scripts/test-all-policy.bro

@@ -68,6 +68,7 @@
 @load protocols/ssl/expiring-certs.bro
 @load protocols/ssl/extract-certs-pem.bro
 @load protocols/ssl/known-certs.bro
+#@load protocols/ssl/notary.bro
 @load protocols/ssl/validate-certs.bro
 @load tuning/__load__.bro
 @load tuning/defaults/__load__.bro