From ff8184242aa8383775a42406b6fd1fb01266462d Mon Sep 17 00:00:00 2001
From: Matthias Vallentin <vallentin@icir.org>
Date: Fri, 21 Dec 2012 17:03:39 -0800
Subject: [PATCH 1/8] Enhance ssl.log with information from notary.

This commit brings enhances each log line with the data from the notary when
available. The added fields include:

  - notary.first_seen
  - notary.last_seen
  - notary.times_seen
  - notary.valid

The semantics of these fields map 1-to-1 to the corresponding fields in DNS TXT
lookups from the notary. The implementation of this feature required a bit
plumbing: when Bro finishes the analysis, the log record is copied into table
indexed by connection ID where it remains until either Bro terminates or the
answer of the notary arrives. The script accummulates requests for a given
digest into a "waitlist," to avoid multiple redundant lookups for high-profile
websites who receive a large chunk of traffic. When a DNS reply arrives
asynchronously, the when handler clears the waitlist and assigns the
information to all records in the buffered.

The script also adds Each log entry into a double-ended queue to make sure the
records arrive on disk in the same way Bro sees them. Each reply also triggers
a sweep through this deque which flushes the buffer up to the first outstanding
reply.

Here is an example from the public M57 trace from 2009:

  % bro-cut ts id.orig_h id.resp_h server_name notary.first_seen notary.last_seen notary.times_seen notary.valid < ssl.log
  1258562650.121682 192.168.1.104 208.97.132.223  mail.m57.biz  - - - -
  1258535660.267128 192.168.1.104 65.55.184.16  - - - - -
  1258561662.604948 192.168.1.105 66.235.128.158  - - - - -
  1258561885.571010 192.168.1.105 65.55.184.155 www.update.microsoft.com  - - - -
  1258563578.455331 192.168.1.103 208.97.132.223  - - - - -
  1258563716.527681 192.168.1.103 96.6.248.124  - - - - -
  1258563884.667153 192.168.1.103 66.235.139.152  - - - - -
  1258564818.755676 192.168.1.103 12.41.118.177 - - - - -
  1258564821.637874 192.168.1.103 12.41.118.177 - - - - -
  1258564821.637871 192.168.1.103 12.41.118.177 - - - - -
  1258564821.637876 192.168.1.103 12.41.118.177 - - - - -
  1258564821.638126 192.168.1.103 12.41.118.177 - - - - -
  1258562467.525034 192.168.1.104 208.97.132.223  mail.m57.biz  15392 15695 301 F
  1258563063.965975 192.168.1.104 63.245.209.105  aus2.mozilla.org  - - - -
  1258563064.091396 192.168.1.104 63.245.209.91 addons.mozilla.org  - - - -
  1258563329.202273 192.168.1.103 208.97.132.223  - 15392 15695 301 F
  1258563712.945933 192.168.1.103 65.55.16.121  - - - - -
  1258563714.044500 192.168.1.103 65.54.186.79  - - - - -
  1258563716.146680 192.168.1.103 96.6.248.124  - - - - -
  1258563737.432312 192.168.1.103 96.6.245.186  - - - - -
  1258563716.526933 192.168.1.103 96.6.245.186  - - - - -
  1258563716.527430 192.168.1.103 96.6.245.186  - - - - -
  1258563716.527179 192.168.1.103 96.6.245.186  - - - - -
  1258563716.527683 192.168.1.103 96.6.245.186  - - - - -
  1258563716.527432 192.168.1.103 96.6.245.186  - - - - -
  1258563751.178683 192.168.1.103 66.235.139.152  - - - - -
  1258563751.171938 192.168.1.103 65.54.234.75  - - - - -
  1258563751.182433 192.168.1.103 65.242.27.35  - - - - -
  1258563883.414188 192.168.1.103 65.55.16.121  - - - - -
  1258563884.702380 192.168.1.103 65.242.27.35  - - - - -
  1258563885.678766 192.168.1.103 65.54.186.79  - - - - -
  1258563886.124987 192.168.1.103 65.54.186.79  - - - - -
  1258564027.877525 192.168.1.103 65.54.234.75  - - - - -
  1258564688.206859 192.168.1.103 65.54.186.107 - - - - -
  1258567162.001225 192.168.1.105 208.97.132.223  mail.m57.biz  - - - -
  1258568040.512840 192.168.1.103 208.97.132.223  - - - - -
  1258564688.577376 192.168.1.103 207.46.120.170  - - - - -
  1258564723.029005 192.168.1.103 65.54.186.107 - - - - -
  1258564723.784032 192.168.1.103 65.55.194.249 - - - - -
  1258564748.521756 192.168.1.103 65.54.186.107 - - - - -
  1258564817.601152 192.168.1.103 12.41.118.177 - - - - -
  1258565684.353653 192.168.1.105 208.97.132.223  mail.m57.biz  15392 15695 301 F
  1258565710.188691 192.168.1.105 74.125.155.109  pop.gmail.com - - - -
  1258566061.103696 192.168.1.103 208.97.132.223  - 15392 15695 301 F
  1258566893.914987 192.168.1.102 208.97.132.223  - 15392 15695 301 F
---
 scripts/base/protocols/ssl/main.bro | 157 +++++++++++++++++++++++++++-
 1 file changed, 156 insertions(+), 1 deletion(-)

diff --git a/scripts/base/protocols/ssl/main.bro b/scripts/base/protocols/ssl/main.bro
index 6b434ae09d..60f17ea7ce 100644
--- a/scripts/base/protocols/ssl/main.bro
+++ b/scripts/base/protocols/ssl/main.bro
@@ -8,6 +8,14 @@ module SSL;
 export {
 	redef enum Log::ID += { LOG };
 
+	## A response from the ICSI certificate notary.
+	type NotaryResponse: record {
+		first_seen: count &log &optional;
+		last_seen: count &log &optional;
+		times_seen: count &log &optional;
+		valid: bool &log &optional;
+	};
+
 	type Info: record {
 		## Time when the SSL connection was first detected.
 		ts:               time             &log;
@@ -72,15 +80,48 @@ export {
 	## utility.
 	const openssl_util = "openssl" &redef;
 
+	## Flag that determines whether to use the ICSI certificate notary to enhance
+	## the SSL log records.
+	const use_notary = T &redef;
+
 	## Event that can be handled to access the SSL
 	## record as it is sent on to the logging framework.
 	global log_ssl: event(rec: Info);
 }
 
+# TODO: Maybe wrap these in @ifdef's? Otherwise we carry the extra baggage
+# around all the time.
+redef record Info += {
+	sha1_digest: string &optional;
+	notary: NotaryResponse &log &optional;
+	};
+
 redef record connection += {
 	ssl: Info &optional;
 };
 
+# The DNS cache of notary responses.
+global notary_cache: table[string] of NotaryResponse &create_expire = 1 hr;
+
+# The buffered SSL log records.
+global records: table[string] of Info;
+
+# The records that wait for a notary response identified by the cert digest.
+# Each digest refers to a list of connection UIDs which are updated when a DNS
+# reply arrives asynchronously.
+global waiting: table[string] of vector of string;
+
+# A double-ended queue that determines the log record order in which logs have
+# to written out to disk.
+global deque: table[count] of string;
+
+# The top-most deque index.
+global head = 0;
+
+# The bottom deque index that points to the next record to be flushed as soon
+# as the notary response arrives.
+global tail = 0;
+
 event bro_init() &priority=5
 	{
 	Log::create_stream(SSL::LOG, [$columns=Info, $ev=log_ssl]);
@@ -122,9 +163,103 @@ function set_session(c: connection)
 		         $client_cert_chain=vector()];
 	}
 
+function clear_waitlist(digest: string)
+	{
+	if ( digest in waiting )
+		{
+		for ( i in waiting[digest] )
+			{
+			local uid = waiting[digest][i];
+			records[uid]$notary = [];
+			}
+		delete waiting[digest];
+		}
+	}
+
+function lookup_cert_hash(uid: string, digest: string)
+  {
+  # Add the record ID to the list of waiting IDs for this digest.
+  local waits_already = digest in waiting;
+  if ( ! waits_already )
+		waiting[digest] = vector();
+	waiting[digest][|waiting[digest]|] = uid;
+	if ( waits_already )
+		return;
+
+	local domain = "%s.notary.icsi.berkeley.edu";
+	when ( local str = lookup_hostname_txt(fmt(domain, digest)) )
+		{
+		# Cache every response for a digest.
+		# TODO: should we ignore failing answers?
+		notary_cache[digest] = [];
+
+		# Parse notary answer.
+		if ( str == "<???>" )
+			{
+			# TODO: Should we handle NXDOMAIN separately?
+			clear_waitlist(digest);
+			return;
+			}
+		local fields = split(str, / /);
+		if ( |fields| != 5 )	# version 1 has 5 fields.
+			{
+			clear_waitlist(digest);
+			return;
+			}
+		local version = split(fields[1], /=/)[2];
+		if ( version != "1" )
+		{
+			clear_waitlist(digest);
+			return;
+		}
+		local r = notary_cache[digest];
+		r$first_seen = to_count(split(fields[2], /=/)[2]);
+		r$last_seen = to_count(split(fields[3], /=/)[2]);
+		r$times_seen = to_count(split(fields[4], /=/)[2]);
+		r$valid = split(fields[5], /=/)[2] == "1";
+
+		# Assign notary answer to all waiting records.
+		if ( digest in waiting )
+		{
+			for ( i in waiting[digest] )
+				records[waiting[digest][i]]$notary = r;
+			delete waiting[digest];
+		}
+
+		# Flush all records up to the record which still awaits an answer.
+		local current: string;
+		for ( unused_index in deque )
+			{
+				current = deque[tail];
+				local info = records[current];
+				if ( ! info?$notary )
+					break;
+				Log::write(SSL::LOG, info);
+				delete deque[tail];
+				delete records[current];
+				++tail;
+			}
+		}
+	}
+
 function finish(c: connection)
 	{
-	Log::write(SSL::LOG, c$ssl);
+	if ( use_notary && c$ssl?$sha1_digest)
+		{
+		local digest = c$ssl$sha1_digest;
+		if ( c$ssl$sha1_digest in notary_cache )
+			c$ssl$notary = notary_cache[digest];
+		else
+			lookup_cert_hash(c$uid, digest);
+		records[c$uid] = c$ssl; # Copy the record.
+		deque[head] = c$uid;
+		++head;
+		}
+	else
+		{
+		Log::write(SSL::LOG, c$ssl);
+		}
+
 	if ( disable_analyzer_after_detection && c?$ssl && c$ssl?$analyzer_id )
 		disable_analyzer(c$id, c$ssl$analyzer_id);
 	delete c$ssl;
@@ -181,6 +316,9 @@ event x509_certificate(c: connection, is_orig: bool, cert: X509, chain_idx: coun
 			c$ssl$issuer_subject = cert$issuer;
 			c$ssl$not_valid_before = cert$not_valid_before;
 			c$ssl$not_valid_after = cert$not_valid_after;
+
+			if ( use_notary )
+				c$ssl$sha1_digest = sha1_hash(der_cert);
 			}
 		else
 			{
@@ -228,3 +366,20 @@ event protocol_violation(c: connection, atype: count, aid: count,
 	if ( c?$ssl )
 		finish(c);
 	}
+
+event bro_done()
+	{
+	if ( ! use_notary || |deque| == 0 )
+		return;
+
+	local current: string;
+	for ( unused_index in deque )
+		{
+		current = deque[tail];
+		local info = records[current];
+		Log::write(SSL::LOG, info);
+		delete deque[tail];
+		delete records[current];
+		++tail;
+		}
+	}

From 7355a0089ae4c1d61f4e9f842763e8689514769a Mon Sep 17 00:00:00 2001
From: Matthias Vallentin <vallentin@icir.org>
Date: Fri, 21 Dec 2012 17:17:58 -0800
Subject: [PATCH 2/8] Adhere to Bro coding style guidelines.

---
 scripts/base/protocols/ssl/main.bro | 24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/scripts/base/protocols/ssl/main.bro b/scripts/base/protocols/ssl/main.bro
index 60f17ea7ce..e36fbd90e6 100644
--- a/scripts/base/protocols/ssl/main.bro
+++ b/scripts/base/protocols/ssl/main.bro
@@ -208,10 +208,10 @@ function lookup_cert_hash(uid: string, digest: string)
 			}
 		local version = split(fields[1], /=/)[2];
 		if ( version != "1" )
-		{
+			{
 			clear_waitlist(digest);
 			return;
-		}
+			}
 		local r = notary_cache[digest];
 		r$first_seen = to_count(split(fields[2], /=/)[2]);
 		r$last_seen = to_count(split(fields[3], /=/)[2]);
@@ -220,24 +220,24 @@ function lookup_cert_hash(uid: string, digest: string)
 
 		# Assign notary answer to all waiting records.
 		if ( digest in waiting )
-		{
+			{
 			for ( i in waiting[digest] )
 				records[waiting[digest][i]]$notary = r;
 			delete waiting[digest];
-		}
+			}
 
 		# Flush all records up to the record which still awaits an answer.
 		local current: string;
 		for ( unused_index in deque )
 			{
-				current = deque[tail];
-				local info = records[current];
-				if ( ! info?$notary )
-					break;
-				Log::write(SSL::LOG, info);
-				delete deque[tail];
-				delete records[current];
-				++tail;
+			current = deque[tail];
+			local info = records[current];
+			if ( ! info?$notary )
+				break;
+			Log::write(SSL::LOG, info);
+			delete deque[tail];
+			delete records[current];
+			++tail;
 			}
 		}
 	}

From 382262e286bb923ca38dd156217e14988656bc2f Mon Sep 17 00:00:00 2001
From: Matthias Vallentin <vallentin@icir.org>
Date: Fri, 21 Dec 2012 17:56:31 -0800
Subject: [PATCH 3/8] Factor notary code into separte file.

There exists one complication: the new file notary.bro requires the definition
of the SSL::Info record, but as does main.bro. Because I did not really know
where to put the common code (it's not a constant, so ssl/const.bro does not
really fit), I put it into __load.bro__ so that it sticks out for now. If
anybody has an idea how to solve this elegantly, please let me know.
---
 scripts/base/protocols/ssl/__load__.bro |  64 ++++++-
 scripts/base/protocols/ssl/main.bro     | 218 +-----------------------
 scripts/base/protocols/ssl/notary.bro   | 162 ++++++++++++++++++
 3 files changed, 232 insertions(+), 212 deletions(-)
 create mode 100644 scripts/base/protocols/ssl/notary.bro

diff --git a/scripts/base/protocols/ssl/__load__.bro b/scripts/base/protocols/ssl/__load__.bro
index eaaa13cd76..e97d1681fc 100644
--- a/scripts/base/protocols/ssl/__load__.bro
+++ b/scripts/base/protocols/ssl/__load__.bro
@@ -1,3 +1,65 @@
 @load ./consts
+
+module SSL;
+
+export {
+	redef enum Log::ID += { LOG };
+
+	type Info: record {
+		## Time when the SSL connection was first detected.
+		ts:               time             &log;
+		## Unique ID for the connection.
+		uid:         string          &log;
+		## The connection's 4-tuple of endpoint addresses/ports.
+		id:               conn_id          &log;
+		## SSL/TLS version that the server offered.
+		version:          string           &log &optional;
+		## SSL/TLS cipher suite that the server chose.
+		cipher:           string           &log &optional;
+		## Value of the Server Name Indicator SSL/TLS extension.  It
+		## indicates the server name that the client was requesting.
+		server_name:      string           &log &optional;
+		## Session ID offered by the client for session resumption.
+		session_id:       string           &log &optional;
+		## Subject of the X.509 certificate offered by the server.
+		subject:          string           &log &optional;
+		## Subject of the signer of the X.509 certificate offered by the server.
+		issuer_subject:   string           &log &optional;
+		## NotValidBefore field value from the server certificate.
+		not_valid_before: time             &log &optional;
+		## NotValidAfter field value from the server certificate.
+		not_valid_after:  time             &log &optional;
+		## Last alert that was seen during the connection.
+		last_alert:       string           &log &optional;
+
+		## Subject of the X.509 certificate offered by the client.
+		client_subject:          string           &log &optional;
+		## Subject of the signer of the X.509 certificate offered by the client.
+		client_issuer_subject:   string           &log &optional;
+
+		## Full binary server certificate stored in DER format.
+		cert:             string           &optional;
+		## Chain of certificates offered by the server to validate its
+		## complete signing chain.
+		cert_chain:       vector of string &optional;
+
+		## Full binary client certificate stored in DER format.
+		client_cert:             string           &optional;
+		## Chain of certificates offered by the client to validate its
+		## complete signing chain.
+		client_cert_chain:       vector of string &optional;
+
+		## The analyzer ID used for the analyzer instance attached
+		## to each connection.  It is not used for logging since it's a
+		## meaningless arbitrary number.
+		analyzer_id:      count            &optional;
+	};
+}
+
+redef record connection += {
+	ssl: Info &optional;
+};
+
+@load ./notary
 @load ./main
-@load ./mozilla-ca-list
\ No newline at end of file
+@load ./mozilla-ca-list
diff --git a/scripts/base/protocols/ssl/main.bro b/scripts/base/protocols/ssl/main.bro
index e36fbd90e6..f02602235c 100644
--- a/scripts/base/protocols/ssl/main.bro
+++ b/scripts/base/protocols/ssl/main.bro
@@ -6,66 +6,6 @@
 module SSL;
 
 export {
-	redef enum Log::ID += { LOG };
-
-	## A response from the ICSI certificate notary.
-	type NotaryResponse: record {
-		first_seen: count &log &optional;
-		last_seen: count &log &optional;
-		times_seen: count &log &optional;
-		valid: bool &log &optional;
-	};
-
-	type Info: record {
-		## Time when the SSL connection was first detected.
-		ts:               time             &log;
-		## Unique ID for the connection.
-		uid:         string          &log;
-		## The connection's 4-tuple of endpoint addresses/ports.
-		id:               conn_id          &log;
-		## SSL/TLS version that the server offered.
-		version:          string           &log &optional;
-		## SSL/TLS cipher suite that the server chose.
-		cipher:           string           &log &optional;
-		## Value of the Server Name Indicator SSL/TLS extension.  It
-		## indicates the server name that the client was requesting.
-		server_name:      string           &log &optional;
-		## Session ID offered by the client for session resumption.
-		session_id:       string           &log &optional;
-		## Subject of the X.509 certificate offered by the server.
-		subject:          string           &log &optional;
-		## Subject of the signer of the X.509 certificate offered by the server.
-		issuer_subject:   string           &log &optional;
-		## NotValidBefore field value from the server certificate.
-		not_valid_before: time             &log &optional;
-		## NotValidAfter field value from the server certificate.
-		not_valid_after:  time             &log &optional;
-		## Last alert that was seen during the connection.
-		last_alert:       string           &log &optional;
-
-		## Subject of the X.509 certificate offered by the client.
-		client_subject:          string           &log &optional;
-		## Subject of the signer of the X.509 certificate offered by the client.
-		client_issuer_subject:   string           &log &optional;
-
-		## Full binary server certificate stored in DER format.
-		cert:             string           &optional;
-		## Chain of certificates offered by the server to validate its
-		## complete signing chain.
-		cert_chain:       vector of string &optional;
-
-		## Full binary client certificate stored in DER format.
-		client_cert:             string           &optional;
-		## Chain of certificates offered by the client to validate its
-		## complete signing chain.
-		client_cert_chain:       vector of string &optional;
-
-		## The analyzer ID used for the analyzer instance attached
-		## to each connection.  It is not used for logging since it's a
-		## meaningless arbitrary number.
-		analyzer_id:      count            &optional;
-	};
-
 	## The default root CA bundle.  By loading the
 	## mozilla-ca-list.bro script it will be set to Mozilla's root CA list.
 	const root_certs: table[string] of string = {} &redef;
@@ -80,48 +20,11 @@ export {
 	## utility.
 	const openssl_util = "openssl" &redef;
 
-	## Flag that determines whether to use the ICSI certificate notary to enhance
-	## the SSL log records.
-	const use_notary = T &redef;
-
 	## Event that can be handled to access the SSL
 	## record as it is sent on to the logging framework.
 	global log_ssl: event(rec: Info);
 }
 
-# TODO: Maybe wrap these in @ifdef's? Otherwise we carry the extra baggage
-# around all the time.
-redef record Info += {
-	sha1_digest: string &optional;
-	notary: NotaryResponse &log &optional;
-	};
-
-redef record connection += {
-	ssl: Info &optional;
-};
-
-# The DNS cache of notary responses.
-global notary_cache: table[string] of NotaryResponse &create_expire = 1 hr;
-
-# The buffered SSL log records.
-global records: table[string] of Info;
-
-# The records that wait for a notary response identified by the cert digest.
-# Each digest refers to a list of connection UIDs which are updated when a DNS
-# reply arrives asynchronously.
-global waiting: table[string] of vector of string;
-
-# A double-ended queue that determines the log record order in which logs have
-# to written out to disk.
-global deque: table[count] of string;
-
-# The top-most deque index.
-global head = 0;
-
-# The bottom deque index that points to the next record to be flushed as soon
-# as the notary response arrives.
-global tail = 0;
-
 event bro_init() &priority=5
 	{
 	Log::create_stream(SSL::LOG, [$columns=Info, $ev=log_ssl]);
@@ -163,102 +66,15 @@ function set_session(c: connection)
 		         $client_cert_chain=vector()];
 	}
 
-function clear_waitlist(digest: string)
-	{
-	if ( digest in waiting )
-		{
-		for ( i in waiting[digest] )
-			{
-			local uid = waiting[digest][i];
-			records[uid]$notary = [];
-			}
-		delete waiting[digest];
-		}
-	}
-
-function lookup_cert_hash(uid: string, digest: string)
-  {
-  # Add the record ID to the list of waiting IDs for this digest.
-  local waits_already = digest in waiting;
-  if ( ! waits_already )
-		waiting[digest] = vector();
-	waiting[digest][|waiting[digest]|] = uid;
-	if ( waits_already )
-		return;
-
-	local domain = "%s.notary.icsi.berkeley.edu";
-	when ( local str = lookup_hostname_txt(fmt(domain, digest)) )
-		{
-		# Cache every response for a digest.
-		# TODO: should we ignore failing answers?
-		notary_cache[digest] = [];
-
-		# Parse notary answer.
-		if ( str == "<???>" )
-			{
-			# TODO: Should we handle NXDOMAIN separately?
-			clear_waitlist(digest);
-			return;
-			}
-		local fields = split(str, / /);
-		if ( |fields| != 5 )	# version 1 has 5 fields.
-			{
-			clear_waitlist(digest);
-			return;
-			}
-		local version = split(fields[1], /=/)[2];
-		if ( version != "1" )
-			{
-			clear_waitlist(digest);
-			return;
-			}
-		local r = notary_cache[digest];
-		r$first_seen = to_count(split(fields[2], /=/)[2]);
-		r$last_seen = to_count(split(fields[3], /=/)[2]);
-		r$times_seen = to_count(split(fields[4], /=/)[2]);
-		r$valid = split(fields[5], /=/)[2] == "1";
-
-		# Assign notary answer to all waiting records.
-		if ( digest in waiting )
-			{
-			for ( i in waiting[digest] )
-				records[waiting[digest][i]]$notary = r;
-			delete waiting[digest];
-			}
-
-		# Flush all records up to the record which still awaits an answer.
-		local current: string;
-		for ( unused_index in deque )
-			{
-			current = deque[tail];
-			local info = records[current];
-			if ( ! info?$notary )
-				break;
-			Log::write(SSL::LOG, info);
-			delete deque[tail];
-			delete records[current];
-			++tail;
-			}
-		}
-	}
-
 function finish(c: connection)
 	{
-	if ( use_notary && c$ssl?$sha1_digest)
-		{
-		local digest = c$ssl$sha1_digest;
-		if ( c$ssl$sha1_digest in notary_cache )
-			c$ssl$notary = notary_cache[digest];
-		else
-			lookup_cert_hash(c$uid, digest);
-		records[c$uid] = c$ssl; # Copy the record.
-		deque[head] = c$uid;
-		++head;
-		}
-	else
-		{
-		Log::write(SSL::LOG, c$ssl);
-		}
+# TODO: This dummy flag merely exists to check whether the notary script is
+# loaded. There's probably a better way to incorporate the notary.
+@ifdef( Notary::enabled )
+	Notary::push(c$ssl);
+@else
+	Log::write(SSL::LOG, c$ssl);
+@endif
 
 	if ( disable_analyzer_after_detection && c?$ssl && c$ssl?$analyzer_id )
 		disable_analyzer(c$id, c$ssl$analyzer_id);
@@ -316,9 +132,6 @@ event x509_certificate(c: connection, is_orig: bool, cert: X509, chain_idx: coun
 			c$ssl$issuer_subject = cert$issuer;
 			c$ssl$not_valid_before = cert$not_valid_before;
 			c$ssl$not_valid_after = cert$not_valid_after;
-
-			if ( use_notary )
-				c$ssl$sha1_digest = sha1_hash(der_cert);
 			}
 		else
 			{
@@ -366,20 +179,3 @@ event protocol_violation(c: connection, atype: count, aid: count,
 	if ( c?$ssl )
 		finish(c);
 	}
-
-event bro_done()
-	{
-	if ( ! use_notary || |deque| == 0 )
-		return;
-
-	local current: string;
-	for ( unused_index in deque )
-		{
-		current = deque[tail];
-		local info = records[current];
-		Log::write(SSL::LOG, info);
-		delete deque[tail];
-		delete records[current];
-		++tail;
-		}
-	}
diff --git a/scripts/base/protocols/ssl/notary.bro b/scripts/base/protocols/ssl/notary.bro
new file mode 100644
index 0000000000..6cf687ea90
--- /dev/null
+++ b/scripts/base/protocols/ssl/notary.bro
@@ -0,0 +1,162 @@
+module Notary;
+
+export {
+	# Flag to tell the SSL analysis script that it should buffer logs instead of
+	# flushing them directly.
+	const enabled = T;
+
+	## A response from the ICSI certificate notary.
+	type Response: record {
+		first_seen: count &log &optional;
+		last_seen: count &log &optional;
+		times_seen: count &log &optional;
+		valid: bool &log &optional;
+	};
+
+  ## Hands over an SSL record to the Notary module. This is an ownership
+  ## transfer, i.e., the caller does not need to call Log::write on this record
+  ## anymore.
+  global push: function(info: SSL::Info);
+
+  ## The notary domain to query.
+	const domain = "notary.icsi.berkeley.edu" &redef;
+}
+
+redef record SSL::Info += {
+	sha1_digest: string &optional;
+	notary: Response &log &optional;
+	};
+
+# The DNS cache of notary responses.
+global notary_cache: table[string] of Response &create_expire = 1 hr;
+
+# The buffered SSL log records.
+global records: table[string] of SSL::Info;
+
+# The records that wait for a notary response identified by the cert digest.
+# Each digest refers to a list of connection UIDs which are updated when a DNS
+# reply arrives asynchronously.
+global waiting: table[string] of vector of string;
+
+# A double-ended queue that determines the log record order in which logs have
+# to written out to disk.
+global deque: table[count] of string;
+
+# The top-most deque index.
+global head = 0;
+
+# The bottom deque index that points to the next record to be flushed as soon
+# as the notary response arrives.
+global tail = 0;
+
+function clear_waitlist(digest: string)
+	{
+	if ( digest in waiting )
+		{
+		for ( i in waiting[digest] )
+			{
+			local uid = waiting[digest][i];
+			records[uid]$notary = [];
+			}
+		delete waiting[digest];
+		}
+	}
+
+function flush(evict_all: bool)
+  {
+  local current: string;
+  for ( unused_index in deque )
+    {
+    current = deque[tail];
+    local info = records[current];
+    if ( ! evict_all && ! info?$notary )
+      break;
+    Log::write(SSL::LOG, info);
+    delete deque[tail];
+    delete records[current];
+    ++tail;
+    }
+  }
+
+function lookup_cert_hash(uid: string, digest: string)
+  {
+  # Add the record ID to the list of waiting IDs for this digest.
+  local waits_already = digest in waiting;
+  if ( ! waits_already )
+		waiting[digest] = vector();
+	waiting[digest][|waiting[digest]|] = uid;
+	if ( waits_already )
+		return;
+
+	when ( local str = lookup_hostname_txt(fmt("%s.%s", digest, domain)) )
+		{
+		# Cache every response for a digest.
+		notary_cache[digest] = [];
+
+		# Parse notary answer.
+		if ( str == "<???>" )
+			{
+			# TODO: Should we handle NXDOMAIN separately?
+			clear_waitlist(digest);
+			return;
+			}
+		local fields = split(str, / /);
+		if ( |fields| != 5 )	# version 1 has 5 fields.
+			{
+			clear_waitlist(digest);
+			return;
+			}
+		local version = split(fields[1], /=/)[2];
+		if ( version != "1" )
+			{
+			clear_waitlist(digest);
+			return;
+			}
+		local r = notary_cache[digest];
+		r$first_seen = to_count(split(fields[2], /=/)[2]);
+		r$last_seen = to_count(split(fields[3], /=/)[2]);
+		r$times_seen = to_count(split(fields[4], /=/)[2]);
+		r$valid = split(fields[5], /=/)[2] == "1";
+
+		# Assign notary answer to all waiting records.
+		if ( digest in waiting )
+			{
+			for ( i in waiting[digest] )
+				records[waiting[digest][i]]$notary = r;
+			delete waiting[digest];
+			}
+
+		flush(F);
+		}
+	}
+
+function push(info: SSL::Info)
+	{
+	if ( ! info?$sha1_digest )
+	  return;
+
+  local digest = info$sha1_digest;
+  if ( info$sha1_digest in notary_cache )
+    info$notary = notary_cache[digest];
+  else
+    lookup_cert_hash(info$uid, digest);
+  records[info$uid] = info;
+  deque[head] = info$uid;
+  ++head;
+	}
+
+event x509_certificate(c: connection, is_orig: bool, cert: X509,
+    chain_idx: count, chain_len: count, der_cert: string)
+	{
+	if ( is_orig || chain_idx != 0 || ! c?$ssl )
+	  return;
+
+  c$ssl$sha1_digest = sha1_hash(der_cert);
+  }
+
+event bro_done()
+	{
+	if ( |deque| == 0 )
+		return;
+  flush(T);
+	}

From 8a569facd623d5d15d5e6d42ce3f549bcf4d94c2 Mon Sep 17 00:00:00 2001
From: Matthias Vallentin <vallentin@icir.org>
Date: Fri, 21 Dec 2012 18:04:19 -0800
Subject: [PATCH 4/8] More style tweaks: replace spaces with tabs.

---
 scripts/base/protocols/ssl/notary.bro | 68 +++++++++++++--------------
 1 file changed, 34 insertions(+), 34 deletions(-)

diff --git a/scripts/base/protocols/ssl/notary.bro b/scripts/base/protocols/ssl/notary.bro
index 6cf687ea90..4ae03fd3f8 100644
--- a/scripts/base/protocols/ssl/notary.bro
+++ b/scripts/base/protocols/ssl/notary.bro
@@ -13,12 +13,12 @@ export {
 		valid: bool &log &optional;
 	};
 
-  ## Hands over an SSL record to the Notary module. This is an ownership
-  ## transfer, i.e., the caller does not need to call Log::write on this record
-  ## anymore.
-  global push: function(info: SSL::Info);
+	## Hands over an SSL record to the Notary module. This is an ownership
+	## transfer, i.e., the caller does not need to call Log::write on this record
+	## anymore.
+	global push: function(info: SSL::Info);
 
-  ## The notary domain to query.
+	## The notary domain to query.
 	const domain = "notary.icsi.berkeley.edu" &redef;
 }
 
@@ -63,26 +63,26 @@ function clear_waitlist(digest: string)
 	}
 
 function flush(evict_all: bool)
-  {
-  local current: string;
-  for ( unused_index in deque )
-    {
-    current = deque[tail];
-    local info = records[current];
-    if ( ! evict_all && ! info?$notary )
-      break;
-    Log::write(SSL::LOG, info);
-    delete deque[tail];
-    delete records[current];
-    ++tail;
-    }
-  }
+	{
+	local current: string;
+	for ( unused_index in deque )
+		{
+		current = deque[tail];
+		local info = records[current];
+		if ( ! evict_all && ! info?$notary )
+			break;
+		Log::write(SSL::LOG, info);
+		delete deque[tail];
+		delete records[current];
+		++tail;
+		}
+	}
 
 function lookup_cert_hash(uid: string, digest: string)
-  {
-  # Add the record ID to the list of waiting IDs for this digest.
-  local waits_already = digest in waiting;
-  if ( ! waits_already )
+	j{
+	j# Add the record ID to the list of waiting IDs for this digest.
+	jlocal waits_already = digest in waiting;
+	jif ( ! waits_already )
 		waiting[digest] = vector();
 	waiting[digest][|waiting[digest]|] = uid;
 	if ( waits_already )
@@ -135,14 +135,14 @@ function push(info: SSL::Info)
 	if ( ! info?$sha1_digest )
 	  return;
 
-  local digest = info$sha1_digest;
-  if ( info$sha1_digest in notary_cache )
-    info$notary = notary_cache[digest];
-  else
-    lookup_cert_hash(info$uid, digest);
-  records[info$uid] = info;
-  deque[head] = info$uid;
-  ++head;
+	local digest = info$sha1_digest;
+	if ( info$sha1_digest in notary_cache )
+		info$notary = notary_cache[digest];
+	else
+		lookup_cert_hash(info$uid, digest);
+	records[info$uid] = info;
+	deque[head] = info$uid;
+	++head;
 	}
 
 event x509_certificate(c: connection, is_orig: bool, cert: X509,
@@ -151,12 +151,12 @@ event x509_certificate(c: connection, is_orig: bool, cert: X509,
 	if ( is_orig || chain_idx != 0 || ! c?$ssl )
 	  return;
 
-  c$ssl$sha1_digest = sha1_hash(der_cert);
-  }
+	c$ssl$sha1_digest = sha1_hash(der_cert);
+	}
 
 event bro_done()
 	{
 	if ( |deque| == 0 )
 		return;
-  flush(T);
+	flush(T);
 	}

From 9e81342c92d1c56b4030b3a29e97c8c977155348 Mon Sep 17 00:00:00 2001
From: Matthias Vallentin <vallentin@icir.org>
Date: Sat, 22 Dec 2012 20:30:17 -0800
Subject: [PATCH 5/8] Implement delay-token style SSL logging.

This commit moves the notary script into the policy directory, along with some
architectural changes: the main SSL script now has functionality to add and
remove tokens for a given record. When adding a token, the script delays the
logging until the token has been removed or until the record exceeds a maximum
delay time.

As before, the base SSL script stores all records sequentially and buffers even
non-delayed records for the sake of having an ordered log file. If this turns
out to be not so important, we can easily revert to a simpler logic.

(This is still WiP, some debuggin statements still linger.)
---
 scripts/base/protocols/ssl/__load__.bro       |  62 -------
 scripts/base/protocols/ssl/main.bro           | 167 +++++++++++++++++-
 .../{base => policy}/protocols/ssl/notary.bro | 109 +++---------
 scripts/site/local.bro                        |   4 +
 4 files changed, 192 insertions(+), 150 deletions(-)
 rename scripts/{base => policy}/protocols/ssl/notary.bro (53%)

diff --git a/scripts/base/protocols/ssl/__load__.bro b/scripts/base/protocols/ssl/__load__.bro
index e97d1681fc..239438047c 100644
--- a/scripts/base/protocols/ssl/__load__.bro
+++ b/scripts/base/protocols/ssl/__load__.bro
@@ -1,65 +1,3 @@
 @load ./consts
-
-module SSL;
-
-export {
-	redef enum Log::ID += { LOG };
-
-	type Info: record {
-		## Time when the SSL connection was first detected.
-		ts:               time             &log;
-		## Unique ID for the connection.
-		uid:         string          &log;
-		## The connection's 4-tuple of endpoint addresses/ports.
-		id:               conn_id          &log;
-		## SSL/TLS version that the server offered.
-		version:          string           &log &optional;
-		## SSL/TLS cipher suite that the server chose.
-		cipher:           string           &log &optional;
-		## Value of the Server Name Indicator SSL/TLS extension.  It
-		## indicates the server name that the client was requesting.
-		server_name:      string           &log &optional;
-		## Session ID offered by the client for session resumption.
-		session_id:       string           &log &optional;
-		## Subject of the X.509 certificate offered by the server.
-		subject:          string           &log &optional;
-		## Subject of the signer of the X.509 certificate offered by the server.
-		issuer_subject:   string           &log &optional;
-		## NotValidBefore field value from the server certificate.
-		not_valid_before: time             &log &optional;
-		## NotValidAfter field value from the server certificate.
-		not_valid_after:  time             &log &optional;
-		## Last alert that was seen during the connection.
-		last_alert:       string           &log &optional;
-
-		## Subject of the X.509 certificate offered by the client.
-		client_subject:          string           &log &optional;
-		## Subject of the signer of the X.509 certificate offered by the client.
-		client_issuer_subject:   string           &log &optional;
-
-		## Full binary server certificate stored in DER format.
-		cert:             string           &optional;
-		## Chain of certificates offered by the server to validate its
-		## complete signing chain.
-		cert_chain:       vector of string &optional;
-
-		## Full binary client certificate stored in DER format.
-		client_cert:             string           &optional;
-		## Chain of certificates offered by the client to validate its
-		## complete signing chain.
-		client_cert_chain:       vector of string &optional;
-
-		## The analyzer ID used for the analyzer instance attached
-		## to each connection.  It is not used for logging since it's a
-		## meaningless arbitrary number.
-		analyzer_id:      count            &optional;
-	};
-}
-
-redef record connection += {
-	ssl: Info &optional;
-};
-
-@load ./notary
 @load ./main
 @load ./mozilla-ca-list
diff --git a/scripts/base/protocols/ssl/main.bro b/scripts/base/protocols/ssl/main.bro
index f02602235c..642d93eb96 100644
--- a/scripts/base/protocols/ssl/main.bro
+++ b/scripts/base/protocols/ssl/main.bro
@@ -6,6 +6,63 @@
 module SSL;
 
 export {
+	redef enum Log::ID += { LOG };
+
+	type Info: record {
+		## Time when the SSL connection was first detected.
+		ts:               time             &log;
+		## Unique ID for the connection.
+		uid:         string          &log;
+		## The connection's 4-tuple of endpoint addresses/ports.
+		id:               conn_id          &log;
+		## SSL/TLS version that the server offered.
+		version:          string           &log &optional;
+		## SSL/TLS cipher suite that the server chose.
+		cipher:           string           &log &optional;
+		## Value of the Server Name Indicator SSL/TLS extension.  It
+		## indicates the server name that the client was requesting.
+		server_name:      string           &log &optional;
+		## Session ID offered by the client for session resumption.
+		session_id:       string           &log &optional;
+		## Subject of the X.509 certificate offered by the server.
+		subject:          string           &log &optional;
+		## Subject of the signer of the X.509 certificate offered by the server.
+		issuer_subject:   string           &log &optional;
+		## NotValidBefore field value from the server certificate.
+		not_valid_before: time             &log &optional;
+		## NotValidAfter field value from the server certificate.
+		not_valid_after:  time             &log &optional;
+		## Last alert that was seen during the connection.
+		last_alert:       string           &log &optional;
+
+		## Subject of the X.509 certificate offered by the client.
+		client_subject:          string           &log &optional;
+		## Subject of the signer of the X.509 certificate offered by the client.
+		client_issuer_subject:   string           &log &optional;
+
+		## Full binary server certificate stored in DER format.
+		cert:             string           &optional;
+		## Chain of certificates offered by the server to validate its
+		## complete signing chain.
+		cert_chain:       vector of string &optional;
+
+		## Full binary client certificate stored in DER format.
+		client_cert:             string           &optional;
+		## Chain of certificates offered by the client to validate its
+		## complete signing chain.
+		client_cert_chain:       vector of string &optional;
+
+		## The analyzer ID used for the analyzer instance attached
+		## to each connection.  It is not used for logging since it's a
+		## meaningless arbitrary number.
+		analyzer_id:      count            &optional;
+
+		## Adding a string "token" to this set will cause the SSL script
+    ## to delay logging the record until either the token has been removed or
+    ## the record has been delayed for :bro:id:`SSL::max_log_delay`.
+	  delay_tokens: set[string] &optional;	
+	};
+
 	## The default root CA bundle.  By loading the
 	## mozilla-ca-list.bro script it will be set to Mozilla's root CA list.
 	const root_certs: table[string] of string = {} &redef;
@@ -20,11 +77,24 @@ export {
 	## utility.
 	const openssl_util = "openssl" &redef;
 
+	## The maximum amount of time a plugin can delay records from being logged.
+	const max_log_delay = 15secs &redef;
+
+	## TODO: document.
+	global add_delayed_record: function(info: Info, token: string);
+
+	## TODO: document.
+	global clear_delayed_record: function(uid: string, token: string) : Info;
+
 	## Event that can be handled to access the SSL
 	## record as it is sent on to the logging framework.
 	global log_ssl: event(rec: Info);
 }
 
+redef record connection += {
+	ssl: Info &optional;
+};
+
 event bro_init() &priority=5
 	{
 	Log::create_stream(SSL::LOG, [$columns=Info, $ev=log_ssl]);
@@ -59,6 +129,20 @@ redef likely_server_ports += {
 	989/tcp, 990/tcp, 992/tcp, 993/tcp, 995/tcp, 5223/tcp
 };
 
+# The buffered SSL log records.
+global records: table[string] of Info;
+
+# A double-ended queue that determines the log record order in which logs have
+# to written out to disk.
+global deque: table[count] of string;
+
+# The top-most deque index.
+global head = 0;
+
+# The bottom deque index that points to the next record to be flushed as soon
+# as the notary response arrives.
+global tail = 0;
+
 function set_session(c: connection)
 	{
 	if ( ! c?$ssl )
@@ -66,16 +150,69 @@ function set_session(c: connection)
 		         $client_cert_chain=vector()];
 	}
 
+function add_delayed_record(info: Info, token: string)
+	{
+	if ( info$uid in records )
+		{
+		print fmt("----- ignoring duplicate %s -----", info$uid);
+		return;
+		}
+
+	records[info$uid] = info;
+	deque[head] = info$uid;
+	++head;
+
+	info$delay_tokens = set();
+	add info$delay_tokens[token];
+	}
+
+function clear_delayed_record(uid: string, token: string) : Info
+	{
+	local info = records[uid];
+	delete info$delay_tokens[token];
+	return info;
+	}
+
+global log_record: function(info: Info);
+
+event delay_logging(info: Info)
+	{
+	log_record(info);
+	}
+
+function log_record(info: Info)
+	{
+	for ( unused_index in records )
+		{
+		if ( head == tail )
+			return;
+		local uid = deque[tail];
+		if ( |records[uid]$delay_tokens| > 0 )
+			{
+			if ( info$ts + max_log_delay > network_time() )
+				{
+				schedule 1sec { delay_logging(info) };
+				return;
+				}
+			else
+				{
+				event reporter_info(
+						network_time(),
+						fmt("SSL delay tokens not released in time (%s)",
+								info$delay_tokens),
+						"");
+				}
+			}
+		Log::write(SSL::LOG, records[uid]);
+		delete records[uid];
+		delete deque[tail];
+		++tail;
+		}
+	}
+
 function finish(c: connection)
 	{
-# TODO: This dummy flag merely exists to check whether the notary script is
-# loaded. There's probably a better way to incorporate the notary.
-@ifdef( Notary::enabled )
-	Notary::push(c$ssl);
-@else
-	Log::write(SSL::LOG, c$ssl);
-@endif
-
+	log_record(c$ssl);
 	if ( disable_analyzer_after_detection && c?$ssl && c$ssl?$analyzer_id )
 		disable_analyzer(c$id, c$ssl$analyzer_id);
 	delete c$ssl;
@@ -179,3 +316,17 @@ event protocol_violation(c: connection, atype: count, aid: count,
 	if ( c?$ssl )
 		finish(c);
 	}
+
+event bro_done()
+	{
+	if ( |records| == 0 )
+		return;
+	for ( unused_index in records )
+		{
+		local uid = deque[tail];
+		Log::write(SSL::LOG, records[uid]);
+		delete records[uid];
+		delete deque[tail];
+		++tail;
+		}
+	}
diff --git a/scripts/base/protocols/ssl/notary.bro b/scripts/policy/protocols/ssl/notary.bro
similarity index 53%
rename from scripts/base/protocols/ssl/notary.bro
rename to scripts/policy/protocols/ssl/notary.bro
index 4ae03fd3f8..f720359aa6 100644
--- a/scripts/base/protocols/ssl/notary.bro
+++ b/scripts/policy/protocols/ssl/notary.bro
@@ -1,10 +1,6 @@
 module Notary;
 
 export {
-	# Flag to tell the SSL analysis script that it should buffer logs instead of
-	# flushing them directly.
-	const enabled = T;
-
 	## A response from the ICSI certificate notary.
 	type Response: record {
 		first_seen: count &log &optional;
@@ -13,83 +9,66 @@ export {
 		valid: bool &log &optional;
 	};
 
-	## Hands over an SSL record to the Notary module. This is an ownership
-	## transfer, i.e., the caller does not need to call Log::write on this record
-	## anymore.
-	global push: function(info: SSL::Info);
-
 	## The notary domain to query.
 	const domain = "notary.icsi.berkeley.edu" &redef;
 }
 
 redef record SSL::Info += {
-	sha1_digest: string &optional;
+	sha1: string &log &optional;
 	notary: Response &log &optional;
 	};
 
 # The DNS cache of notary responses.
 global notary_cache: table[string] of Response &create_expire = 1 hr;
 
-# The buffered SSL log records.
-global records: table[string] of SSL::Info;
-
 # The records that wait for a notary response identified by the cert digest.
 # Each digest refers to a list of connection UIDs which are updated when a DNS
 # reply arrives asynchronously.
 global waiting: table[string] of vector of string;
 
-# A double-ended queue that determines the log record order in which logs have
-# to written out to disk.
-global deque: table[count] of string;
-
-# The top-most deque index.
-global head = 0;
-
-# The bottom deque index that points to the next record to be flushed as soon
-# as the notary response arrives.
-global tail = 0;
-
 function clear_waitlist(digest: string)
 	{
+	print "----- clearing waitlist -----";
 	if ( digest in waiting )
 		{
 		for ( i in waiting[digest] )
 			{
-			local uid = waiting[digest][i];
-			records[uid]$notary = [];
+      print fmt("----- retrieving %s -----", waiting[digest][i]);
+			local info = SSL::clear_delayed_record(waiting[digest][i], "notary");
+			info$notary = [];
 			}
 		delete waiting[digest];
 		}
 	}
 
-function flush(evict_all: bool)
+event x509_certificate(c: connection, is_orig: bool, cert: X509,
+    chain_idx: count, chain_len: count, der_cert: string)
 	{
-	local current: string;
-	for ( unused_index in deque )
-		{
-		current = deque[tail];
-		local info = records[current];
-		if ( ! evict_all && ! info?$notary )
-			break;
-		Log::write(SSL::LOG, info);
-		delete deque[tail];
-		delete records[current];
-		++tail;
-		}
-	}
+	if ( is_orig || chain_idx != 0 || ! c?$ssl )
+	  return;
 
-function lookup_cert_hash(uid: string, digest: string)
-	j{
-	j# Add the record ID to the list of waiting IDs for this digest.
-	jlocal waits_already = digest in waiting;
-	jif ( ! waits_already )
+  local digest = sha1_hash(der_cert);
+	c$ssl$sha1 = digest;
+
+	if ( digest in notary_cache )
+    {
+		c$ssl$notary = notary_cache[digest];
+		return;
+		}
+
+  print fmt("----- adding %s -----", c$ssl$uid);
+  SSL::add_delayed_record(c$ssl, "notary");
+
+	local waits_already = digest in waiting;
+	if ( ! waits_already )
 		waiting[digest] = vector();
-	waiting[digest][|waiting[digest]|] = uid;
+	waiting[digest][|waiting[digest]|] = c$uid;
 	if ( waits_already )
 		return;
 
 	when ( local str = lookup_hostname_txt(fmt("%s.%s", digest, domain)) )
 		{
+    print fmt("----- when for %s: %s -----", digest, str);
 		# Cache every response for a digest.
 		notary_cache[digest] = [];
 
@@ -122,41 +101,11 @@ function lookup_cert_hash(uid: string, digest: string)
 		if ( digest in waiting )
 			{
 			for ( i in waiting[digest] )
-				records[waiting[digest][i]]$notary = r;
+			{
+			  local info = SSL::clear_delayed_record(waiting[digest][i], "notary");
+				info$notary = r;
+      }
 			delete waiting[digest];
 			}
-
-		flush(F);
 		}
 	}
-
-function push(info: SSL::Info)
-	{
-	if ( ! info?$sha1_digest )
-	  return;
-
-	local digest = info$sha1_digest;
-	if ( info$sha1_digest in notary_cache )
-		info$notary = notary_cache[digest];
-	else
-		lookup_cert_hash(info$uid, digest);
-	records[info$uid] = info;
-	deque[head] = info$uid;
-	++head;
-	}
-
-event x509_certificate(c: connection, is_orig: bool, cert: X509,
-    chain_idx: count, chain_len: count, der_cert: string)
-	{
-	if ( is_orig || chain_idx != 0 || ! c?$ssl )
-	  return;
-
-	c$ssl$sha1_digest = sha1_hash(der_cert);
-	}
-
-event bro_done()
-	{
-	if ( |deque| == 0 )
-		return;
-	flush(T);
-	}
diff --git a/scripts/site/local.bro b/scripts/site/local.bro
index db1a786839..a080300185 100644
--- a/scripts/site/local.bro
+++ b/scripts/site/local.bro
@@ -56,6 +56,10 @@ redef Software::vulnerable_versions += {
 # This script enables SSL/TLS certificate validation.
 @load protocols/ssl/validate-certs
 
+# This script checks each SSL certificate hash against the ICSI certificate
+# notary service.
+@load protocols/ssl/notary
+
 # If you have libGeoIP support built in, do some geographic detections and 
 # logging for SSH traffic.
 @load protocols/ssh/geo-data

From 7ff15f4599e36c300515955b5f9664cbe5480980 Mon Sep 17 00:00:00 2001
From: Matthias Vallentin <vallentin@icir.org>
Date: Mon, 24 Dec 2012 22:57:15 -0800
Subject: [PATCH 6/8] Simplify delayed logging of SSL records.

---
 scripts/base/protocols/ssl/main.bro     | 54 +++++++++----------------
 scripts/policy/protocols/ssl/notary.bro | 46 +++++++++------------
 2 files changed, 39 insertions(+), 61 deletions(-)

diff --git a/scripts/base/protocols/ssl/main.bro b/scripts/base/protocols/ssl/main.bro
index 642d93eb96..249f8b9609 100644
--- a/scripts/base/protocols/ssl/main.bro
+++ b/scripts/base/protocols/ssl/main.bro
@@ -80,11 +80,13 @@ export {
 	## The maximum amount of time a plugin can delay records from being logged.
 	const max_log_delay = 15secs &redef;
 
-	## TODO: document.
-	global add_delayed_record: function(info: Info, token: string);
+	## Delays an SSL record for a specific token: the record will not be logged
+	## as longs the token exists or until :bro:id:`SSL::max_log_delay` elapses.
+	global delay_record: function(info: Info, token: string);
 
-	## TODO: document.
-	global clear_delayed_record: function(uid: string, token: string) : Info;
+	## Undelays an SSL record for a previously inserted token, allowing the
+	## record to be logged.
+	global undelay_record: function(info: Info, token: string);
 
 	## Event that can be handled to access the SSL
 	## record as it is sent on to the logging framework.
@@ -129,12 +131,9 @@ redef likely_server_ports += {
 	989/tcp, 990/tcp, 992/tcp, 993/tcp, 995/tcp, 5223/tcp
 };
 
-# The buffered SSL log records.
-global records: table[string] of Info;
-
 # A double-ended queue that determines the log record order in which logs have
 # to written out to disk.
-global deque: table[count] of string;
+global deque: table[count] of Info;
 
 # The top-most deque index.
 global head = 0;
@@ -150,27 +149,19 @@ function set_session(c: connection)
 		         $client_cert_chain=vector()];
 	}
 
-function add_delayed_record(info: Info, token: string)
+function delay_record(info: Info, token: string)
 	{
-	if ( info$uid in records )
-		{
-		print fmt("----- ignoring duplicate %s -----", info$uid);
-		return;
-		}
-
-	records[info$uid] = info;
-	deque[head] = info$uid;
-	++head;
-
 	info$delay_tokens = set();
 	add info$delay_tokens[token];
+
+	deque[head] = info;
+	++head;
 	}
 
-function clear_delayed_record(uid: string, token: string) : Info
+function undelay_record(info: Info, token: string)
 	{
-	local info = records[uid];
-	delete info$delay_tokens[token];
-	return info;
+	if ( token in info$delay_tokens )
+		delete info$delay_tokens[token];
 	}
 
 global log_record: function(info: Info);
@@ -182,12 +173,11 @@ event delay_logging(info: Info)
 
 function log_record(info: Info)
 	{
-	for ( unused_index in records )
+	for ( unused_index in deque )
 		{
 		if ( head == tail )
 			return;
-		local uid = deque[tail];
-		if ( |records[uid]$delay_tokens| > 0 )
+		if ( |deque[tail]$delay_tokens| > 0 )
 			{
 			if ( info$ts + max_log_delay > network_time() )
 				{
@@ -203,8 +193,7 @@ function log_record(info: Info)
 						"");
 				}
 			}
-		Log::write(SSL::LOG, records[uid]);
-		delete records[uid];
+		Log::write(SSL::LOG, deque[tail]);
 		delete deque[tail];
 		++tail;
 		}
@@ -215,7 +204,6 @@ function finish(c: connection)
 	log_record(c$ssl);
 	if ( disable_analyzer_after_detection && c?$ssl && c$ssl?$analyzer_id )
 		disable_analyzer(c$id, c$ssl$analyzer_id);
-	delete c$ssl;
 	}
 
 event ssl_client_hello(c: connection, version: count, possible_ts: time, session_id: string, ciphers: count_set) &priority=5
@@ -319,13 +307,11 @@ event protocol_violation(c: connection, atype: count, aid: count,
 
 event bro_done()
 	{
-	if ( |records| == 0 )
+	if ( |deque| == 0 )
 		return;
-	for ( unused_index in records )
+	for ( unused_index in deque )
 		{
-		local uid = deque[tail];
-		Log::write(SSL::LOG, records[uid]);
-		delete records[uid];
+		Log::write(SSL::LOG, deque[tail]);
 		delete deque[tail];
 		++tail;
 		}
diff --git a/scripts/policy/protocols/ssl/notary.bro b/scripts/policy/protocols/ssl/notary.bro
index f720359aa6..40afc977c2 100644
--- a/scripts/policy/protocols/ssl/notary.bro
+++ b/scripts/policy/protocols/ssl/notary.bro
@@ -24,20 +24,15 @@ global notary_cache: table[string] of Response &create_expire = 1 hr;
 # The records that wait for a notary response identified by the cert digest.
 # Each digest refers to a list of connection UIDs which are updated when a DNS
 # reply arrives asynchronously.
-global waiting: table[string] of vector of string;
+global wait_list: table[string] of vector of SSL::Info;
 
 function clear_waitlist(digest: string)
 	{
-	print "----- clearing waitlist -----";
-	if ( digest in waiting )
+	if ( digest in wait_list )
 		{
-		for ( i in waiting[digest] )
-			{
-      print fmt("----- retrieving %s -----", waiting[digest][i]);
-			local info = SSL::clear_delayed_record(waiting[digest][i], "notary");
-			info$notary = [];
-			}
-		delete waiting[digest];
+		for ( i in wait_list[digest] )
+			SSL::undelay_record(wait_list[digest][i], "notary");
+		delete wait_list[digest];
 		}
 	}
 
@@ -56,31 +51,27 @@ event x509_certificate(c: connection, is_orig: bool, cert: X509,
 		return;
 		}
 
-  print fmt("----- adding %s -----", c$ssl$uid);
-  SSL::add_delayed_record(c$ssl, "notary");
+  SSL::delay_record(c$ssl, "notary");
 
-	local waits_already = digest in waiting;
+	local waits_already = digest in wait_list;
 	if ( ! waits_already )
-		waiting[digest] = vector();
-	waiting[digest][|waiting[digest]|] = c$uid;
+		wait_list[digest] = vector();
+	wait_list[digest][|wait_list[digest]|] = c$ssl;
 	if ( waits_already )
 		return;
 
 	when ( local str = lookup_hostname_txt(fmt("%s.%s", digest, domain)) )
 		{
-    print fmt("----- when for %s: %s -----", digest, str);
-		# Cache every response for a digest.
 		notary_cache[digest] = [];
 
 		# Parse notary answer.
-		if ( str == "<???>" )
+		if ( str == "<???>" ) # NXDOMAIN
 			{
-			# TODO: Should we handle NXDOMAIN separately?
 			clear_waitlist(digest);
 			return;
 			}
 		local fields = split(str, / /);
-		if ( |fields| != 5 )	# version 1 has 5 fields.
+		if ( |fields| != 5 ) # version 1 has 5 fields.
 			{
 			clear_waitlist(digest);
 			return;
@@ -97,15 +88,16 @@ event x509_certificate(c: connection, is_orig: bool, cert: X509,
 		r$times_seen = to_count(split(fields[4], /=/)[2]);
 		r$valid = split(fields[5], /=/)[2] == "1";
 
-		# Assign notary answer to all waiting records.
-		if ( digest in waiting )
+		# Assign notary answer to all records waiting for this digest.
+		if ( digest in wait_list )
 			{
-			for ( i in waiting[digest] )
-			{
-			  local info = SSL::clear_delayed_record(waiting[digest][i], "notary");
+			for ( i in wait_list[digest] )
+        {
+			  local info = wait_list[digest][i];
+			  SSL::undelay_record(info, "notary");
 				info$notary = r;
-      }
-			delete waiting[digest];
+        }
+			delete wait_list[digest];
 			}
 		}
 	}

From 32a0ead698c467a9a89f5b58687db2f7e2812d3c Mon Sep 17 00:00:00 2001
From: Matthias Vallentin <vallentin@icir.org>
Date: Mon, 24 Dec 2012 23:06:56 -0800
Subject: [PATCH 7/8] Give log buffer the correct name.

---
 scripts/base/protocols/ssl/main.bro | 28 +++++++++++++---------------
 1 file changed, 13 insertions(+), 15 deletions(-)

diff --git a/scripts/base/protocols/ssl/main.bro b/scripts/base/protocols/ssl/main.bro
index 249f8b9609..e798695948 100644
--- a/scripts/base/protocols/ssl/main.bro
+++ b/scripts/base/protocols/ssl/main.bro
@@ -131,15 +131,13 @@ redef likely_server_ports += {
 	989/tcp, 990/tcp, 992/tcp, 993/tcp, 995/tcp, 5223/tcp
 };
 
-# A double-ended queue that determines the log record order in which logs have
-# to written out to disk.
-global deque: table[count] of Info;
+# A queue that buffers log records.
+global queue: table[count] of Info;
 
-# The top-most deque index.
+# The top queue index where records are added.
 global head = 0;
 
-# The bottom deque index that points to the next record to be flushed as soon
-# as the notary response arrives.
+# The bottom queue index that points to the next record to be flushed.
 global tail = 0;
 
 function set_session(c: connection)
@@ -154,7 +152,7 @@ function delay_record(info: Info, token: string)
 	info$delay_tokens = set();
 	add info$delay_tokens[token];
 
-	deque[head] = info;
+	queue[head] = info;
 	++head;
 	}
 
@@ -173,11 +171,11 @@ event delay_logging(info: Info)
 
 function log_record(info: Info)
 	{
-	for ( unused_index in deque )
+	for ( unused_index in queue )
 		{
 		if ( head == tail )
 			return;
-		if ( |deque[tail]$delay_tokens| > 0 )
+		if ( |queue[tail]$delay_tokens| > 0 )
 			{
 			if ( info$ts + max_log_delay > network_time() )
 				{
@@ -193,8 +191,8 @@ function log_record(info: Info)
 						"");
 				}
 			}
-		Log::write(SSL::LOG, deque[tail]);
-		delete deque[tail];
+		Log::write(SSL::LOG, queue[tail]);
+		delete queue[tail];
 		++tail;
 		}
 	}
@@ -307,12 +305,12 @@ event protocol_violation(c: connection, atype: count, aid: count,
 
 event bro_done()
 	{
-	if ( |deque| == 0 )
+	if ( |queue| == 0 )
 		return;
-	for ( unused_index in deque )
+	for ( unused_index in queue )
 		{
-		Log::write(SSL::LOG, deque[tail]);
-		delete deque[tail];
+		Log::write(SSL::LOG, queue[tail]);
+		delete queue[tail];
 		++tail;
 		}
 	}

From a635f96518ddeefe553ef4fd98213bc07ad63226 Mon Sep 17 00:00:00 2001
From: Matthias Vallentin <vallentin@icir.org>
Date: Tue, 15 Jan 2013 14:56:02 -0800
Subject: [PATCH 8/8] Small cosmetic changes.

---
 scripts/policy/protocols/ssl/notary.bro | 24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/scripts/policy/protocols/ssl/notary.bro b/scripts/policy/protocols/ssl/notary.bro
index 40afc977c2..0dc96e3570 100644
--- a/scripts/policy/protocols/ssl/notary.bro
+++ b/scripts/policy/protocols/ssl/notary.bro
@@ -24,15 +24,15 @@ global notary_cache: table[string] of Response &create_expire = 1 hr;
 # The records that wait for a notary response identified by the cert digest.
 # Each digest refers to a list of connection UIDs which are updated when a DNS
 # reply arrives asynchronously.
-global wait_list: table[string] of vector of SSL::Info;
+global waitlist: table[string] of vector of SSL::Info;
 
 function clear_waitlist(digest: string)
 	{
-	if ( digest in wait_list )
+	if ( digest in waitlist )
 		{
-		for ( i in wait_list[digest] )
-			SSL::undelay_record(wait_list[digest][i], "notary");
-		delete wait_list[digest];
+		for ( i in waitlist[digest] )
+			SSL::undelay_record(waitlist[digest][i], "notary");
+		delete waitlist[digest];
 		}
 	}
 
@@ -53,10 +53,10 @@ event x509_certificate(c: connection, is_orig: bool, cert: X509,
 
   SSL::delay_record(c$ssl, "notary");
 
-	local waits_already = digest in wait_list;
+	local waits_already = digest in waitlist;
 	if ( ! waits_already )
-		wait_list[digest] = vector();
-	wait_list[digest][|wait_list[digest]|] = c$ssl;
+		waitlist[digest] = vector();
+	waitlist[digest][|waitlist[digest]|] = c$ssl;
 	if ( waits_already )
 		return;
 
@@ -89,15 +89,15 @@ event x509_certificate(c: connection, is_orig: bool, cert: X509,
 		r$valid = split(fields[5], /=/)[2] == "1";
 
 		# Assign notary answer to all records waiting for this digest.
-		if ( digest in wait_list )
+		if ( digest in waitlist )
 			{
-			for ( i in wait_list[digest] )
+			for ( i in waitlist[digest] )
         {
-			  local info = wait_list[digest][i];
+			  local info = waitlist[digest][i];
 			  SSL::undelay_record(info, "notary");
 				info$notary = r;
         }
-			delete wait_list[digest];
+			delete waitlist[digest];
 			}
 		}
 	}