Merge remote branch 'origin/topic/seth/notice-email-delay'

* origin/topic/seth/notice-email-delay:
  The hostname notice email extension works now.
  Fixed more bugs with delayed emails.
  Working around a problem with setting default container types.
  Ugh, still major failure.  I'm just cutting the timeout handling for now.
  Fixed a small bug major problem with email delay timeout catching.
  Initial fixes for the problem of async actions with notice email extensions.

Closes #727.

CHANGES

@@ -1,4 +1,15 @@
 
+2.0-beta-139 | 2011-12-19 07:06:29 -0800
+
+  * The hostname notice email extension works now, plus a general
+    mechanism for adding delayed information to notices. (Seth Hall)
+
+  * Fix &default fields in records not being initialized in coerced
+    assignments. Addresses #722. (Jon Siwek)
+
+  * Make log headers include the type of data stored inside a set or
+    vector ("vector[string]"). (Bernhard Amann)
+
 2.0-beta-126 | 2011-12-18 15:18:05 -0800
 
   * DNS updates.  (Seth Hall)

VERSION

@@ -1 +1 @@
-2.0-beta-126
+2.0-beta-139

scripts/base/frameworks/notice/extend-email/hostnames.bro

@@ -2,7 +2,12 @@
 
 module Notice;
 
-# This probably doesn't actually work due to the async lookup_addr.
+# We have to store references to the notices here because the when statement
+# clones the frame which doesn't give us access to modify values outside
+# of it's execution scope. (we get a clone of the notice instead of a
+# reference to the original notice)
+global tmp_notice_storage: table[string] of Notice::Info &create_expire=max_email_delay+10secs;
+
 event Notice::notice(n: Notice::Info) &priority=10
 	{
 	if ( ! n?$src && ! n?$dst )
@@ -12,21 +17,31 @@ event Notice::notice(n: Notice::Info) &priority=10
 	if ( ACTION_EMAIL !in n$actions )
 		return;
 
+	# I'm not recovering gracefully from the when statements because I want
+	# the notice framework to detect that something has exceeded the maximum
+	# allowed email delay and tell the user.
+	local uid = unique_id("");
+	tmp_notice_storage[uid] = n;
+
 	local output = "";
 	if ( n?$src )
 		{
+		add n$email_delay_tokens["hostnames-src"];
 		when ( local src_name = lookup_addr(n$src) )
 			{
-			output = string_cat("orig_h/src hostname: ", src_name, "\n");
+			output = string_cat("orig/src hostname: ", src_name, "\n");
-			n$email_body_sections[|n$email_body_sections|] = output;
+			tmp_notice_storage[uid]$email_body_sections[|tmp_notice_storage[uid]$email_body_sections|] = output;
+			delete tmp_notice_storage[uid]$email_delay_tokens["hostnames-src"];
 			}
 		}
 	if ( n?$dst )
 		{
+		add n$email_delay_tokens["hostnames-dst"];
 		when ( local dst_name = lookup_addr(n$dst) )
 			{
-			output = string_cat("resp_h/dst hostname: ", dst_name, "\n");
+			output = string_cat("resp/dst hostname: ", dst_name, "\n");
-			n$email_body_sections[|n$email_body_sections|] = output;
+			tmp_notice_storage[uid]$email_body_sections[|tmp_notice_storage[uid]$email_body_sections|] = output;
+			delete tmp_notice_storage[uid]$email_delay_tokens["hostnames-dst"];
 			}
 		}
 	}

scripts/base/frameworks/notice/main.bro

@@ -96,7 +96,13 @@ export {
 		## expand on notices that are being emailed.  The normal way to add text
 		## is to extend the vector by handling the :bro:id:`Notice::notice`
 		## event and modifying the notice in place.
-		email_body_sections:  vector of string &default=vector();
+		email_body_sections:  vector of string &optional;
+
+		## Adding a string "token" to this set will cause the notice framework's
+		## built-in emailing functionality to delay sending the email until
+		## either the token has been removed or the email has been delayed
+		## for :bro:id:`max_email_delay`.
+		email_delay_tokens:   set[string] &optional;
 
 		## This field is to be provided when a notice is generated for the
 		## purpose of deduplicating notices.  The identifier string should
@@ -203,6 +209,8 @@ export {
 	const reply_to            = "" &redef;
 	## Text string prefixed to the subject of all emails sent out.
 	const mail_subject_prefix = "[Bro]" &redef;
+	## The maximum amount of time a plugin can delay email from being sent.
+	const max_email_delay     = 15secs &redef;
 
 	## A log postprocessing function that implements emailing the contents
 	## of a log upon rotation to any configured :bro:id:`Notice::mail_dest`.
@@ -347,11 +355,35 @@ function email_headers(subject_desc: string, dest: string): string
 	return header_text;
 	}
 
+event delay_sending_email(n: Notice::Info, dest: string, extend: bool)
+	{
+	email_notice_to(n, dest, extend);
+	}
+
 function email_notice_to(n: Notice::Info, dest: string, extend: bool)
 	{
 	if ( reading_traces() || dest == "" )
 		return;
 
+	if ( extend )
+		{
+		if ( |n$email_delay_tokens| > 0 )
+			{
+			# If we still are within the max_email_delay, keep delaying.
+			if ( n$ts + max_email_delay > network_time() )
+				{
+				schedule 1sec { delay_sending_email(n, dest, extend) };
+				return;
+				}
+			else
+				{
+				event reporter_info(network_time(),
+					fmt("Notice email delay tokens weren't released in time (%s).", n$email_delay_tokens),
+					"");
+				}
+			}
+		}
+
 	local email_text = email_headers(fmt("%s", n$note), dest);
 
 	# First off, finish the headers and include the human readable messages
@@ -377,9 +409,10 @@ function email_notice_to(n: Notice::Info, dest: string, extend: bool)
 	# Add the extended information if it's requested.
 	if ( extend )
 		{
+		email_text = string_cat(email_text, "\nEmail Extensions\n");
+		email_text = string_cat(email_text,   "----------------\n");
 		for ( i in n$email_body_sections )
 			{
-			email_text = string_cat(email_text, "******************\n");
 			email_text = string_cat(email_text, n$email_body_sections[i], "\n");
 			}
 		}
@@ -475,6 +508,11 @@ function apply_policy(n: Notice::Info)
 	if ( ! n?$actions )
 		n$actions = set();
 
+	if ( ! n?$email_body_sections )
+		n$email_body_sections = vector();
+	if ( ! n?$email_delay_tokens )
+		n$email_delay_tokens = set();
+
 	if ( ! n?$policy_items )
 		n$policy_items = set();