Merge remote-tracking branch 'origin/fastpath'

* origin/fastpath:
  Normalize Notice::Type identifiers per convention. (closes #484)
  Another fix to the default-loaded-scripts test.
  Add new piped_exec BiF.
  Revert "Fixes for email_notice_to() function."
  Fixes for email_notice_to() function.

policy/frameworks/notice/base/main.bro

@@ -227,35 +227,31 @@ function email_notice_to(n: Notice::Info, dest: string, extend: bool)
 	if ( reading_traces() || dest == "" )
 		return;
 		
-	local email_text = cat(
+	local email_text = string_cat(
 		"From: ", mail_from, "\n",
-		"Subject: ", mail_subject_prefix, " ", n$note, "\n",
+		"Subject: ", mail_subject_prefix, " ", fmt("%s", n$note), "\n",
 		"To: ", dest, "\n",
 		# TODO: BiF to get version (the resource_usage Bif seems like overkill).
 		"User-Agent: Bro-IDS/?.?.?\n"); 
 	
 	if ( reply_to != "" )
-		email_text = cat(email_text, "Reply-To: ", reply_to, "\n");
+		email_text = string_cat(email_text, "Reply-To: ", reply_to, "\n");
 	
 	# The notice emails always start off with the human readable message.
-	email_text = cat(email_text, "\n", n$msg, "\n");
+	email_text = string_cat(email_text, "\n", n$msg, "\n");
 	
 	# Add the extended information if it's requested.
 	if ( extend )
 		{
 		for ( i in n$email_body_sections )
 			{
-			email_text = cat(email_text, "******************\n");
-			email_text = cat(email_text, n$email_body_sections[i], "\n");
+			email_text = string_cat(email_text, "******************\n");
+			email_text = string_cat(email_text, n$email_body_sections[i], "\n");
 			}
 		}
 	
-	email_text = cat(email_text, "\n\n--\n[Automatically generated]\n\n");
-	
-	local mail_cmd =
-		fmt("echo \"%s\" | %s -t -oi  %s",
-			str_shell_escape(email_text), sendmail);
-	system(mail_cmd);
+	email_text = string_cat(email_text, "\n\n--\n[Automatically generated]\n\n");
+	piped_exec(fmt("%s -t -oi", sendmail), email_text);
 	}
 
 event notice(n: Notice::Info) &priority=-5

policy/frameworks/notice/base/weird.bro

@@ -8,13 +8,13 @@ export {
 	
 	redef enum Notice::Type += {
 		## Generic unusual but alarm-worthy activity.
-		WeirdActivity,
+		Weird_Activity,
 		## Possible evasion; usually just chud.
-		RetransmissionInconsistency,
+		Retransmission_Inconsistency,
 		## Could mean packet drop; could also be chud.
-		AckAboveHole,
+		Ack_Above_Hole,
 		## Data has sequence hole; perhaps due to filtering.
-		ContentGap,
+		Content_Gap,
 	};
 	
 	type Info: record {
@@ -295,7 +295,7 @@ function report_weird(t: time, name: string, id: string, have_conn: bool,
 	if ( action in notice_actions && ! no_log )
 		{
 		local n: Notice::Info;
-		n$note = WeirdActivity;
+		n$note = Weird_Activity;
 		n$msg = info$msg;
 		if ( have_conn )
 			n$conn = current_conn;
@@ -401,7 +401,7 @@ event rexmit_inconsistency(c: connection, t1: string, t2: string)
 	{
 	if ( c$id !in did_inconsistency_msg )
 		{
-		NOTICE([$note=RetransmissionInconsistency, 
+		NOTICE([$note=Retransmission_Inconsistency,
 		        $conn=c,
 		        $msg=fmt("%s rexmit inconsistency (%s) (%s)",
 		                 id_string(c$id), t1, t2)]);
@@ -411,13 +411,13 @@ event rexmit_inconsistency(c: connection, t1: string, t2: string)
 
 event ack_above_hole(c: connection)
 	{
-	NOTICE([$note=AckAboveHole, $conn=c,
+	NOTICE([$note=Ack_Above_Hole, $conn=c,
 	        $msg=fmt("%s ack above a hole", id_string(c$id))]);
 	}
 
 event content_gap(c: connection, is_orig: bool, seq: count, length: count)
 	{
-	NOTICE([$note=ContentGap, $conn=c,
+	NOTICE([$note=Content_Gap, $conn=c,
 	        $msg=fmt("%s content gap (%s %d/%d)%s",
 	                 id_string(c$id), is_orig ? ">" : "<", seq, length,
 	                 is_external_connection(c) ? " [external]" : "")]);

policy/frameworks/packet-filter/netstats.bro

@@ -7,7 +7,7 @@ module PacketFilter;
 export {
 	redef enum Notice::Type += {
 		## Bro reported packets dropped by the packet filter.
-		DroppedPackets,
+		Dropped_Packets,
 	};
 	
 	## This is the interval between individual statistics collection.
@@ -22,7 +22,7 @@ event net_stats_update(last_stat: NetStats)
 		{
 		local new_recvd = ns$pkts_recvd - last_stat$pkts_recvd;
 		local new_link = ns$pkts_link - last_stat$pkts_link;
-		NOTICE([$note=DroppedPackets,
+		NOTICE([$note=Dropped_Packets,
 		        $msg=fmt("%d packets dropped after filtering, %d received%s",
 		                 new_dropped, new_recvd + new_dropped,
 		                 new_link != 0 ? fmt(", %d on link", new_link) : "")]);

policy/hot.conn.bro

@@ -1,10 +1,10 @@
 @load hot
 
 redef enum Notice += {
-	SensitiveConnection,	# connection marked "hot"
+	Sensitive_Connection,	# connection marked "hot"
 };
 
-# Whether to translate the local address in SensitiveConnection notices
+# Whether to translate the local address in Sensitive_Connection notices
 # to a hostname.  Meant as a demonstration of the "when" construct.
 const xlate_hot_local_addr = F &redef;
 
@@ -40,16 +40,16 @@ function full_id_string(c: connection): string
 	}
 
 
-# Low-level routine that generates the actual SensitiveConnection
+# Low-level routine that generates the actual Sensitive_Connection
 # notice associated with a "hot" connection.
 function do_hot_notice(c: connection, dir: string, host: string)
 	{
-	NOTICE([$note=SensitiveConnection, $conn=c,
+	NOTICE([$note=Sensitive_Connection, $conn=c,
 		$msg=fmt("hot: %s %s local host: %s",
 			full_id_string(c), dir, host)]);
 	}
 
-# Generate a SensitiveConnection notice with the local hostname
+# Generate a Sensitive_Connection notice with the local hostname
 # translated.  Mostly intended as a demonstration of using "when".
 function gen_hot_notice_with_hostnames(c: connection)
 	{
@@ -78,7 +78,7 @@ function log_hot_conn(c: connection)
 		if ( xlate_hot_local_addr )
 			gen_hot_notice_with_hostnames(c);
 		else
-			NOTICE([$note=SensitiveConnection, $conn=c,
+			NOTICE([$note=Sensitive_Connection, $conn=c,
 				$msg=fmt("hot: %s", full_id_string(c))]);
 
 		add hot_conns_reported[c$id][msg];

policy/protocols/dns/base/detect.bro

@@ -2,7 +2,7 @@
 ##!
 ##! Notices raised:
 ##!
-##!   * :bro:enum:`DNS::ExternalName`
+##!   * :bro:enum:`DNS::External_Name`
 ##!
 ##!     A remote host resolves to a local host, but the name is not considered
 ##!     to be within a local zone.  :bro:id:`local_zones` variable **must**
@@ -17,7 +17,7 @@ export {
 		## Raised when a non-local name is found to be pointing at a local host.
 		## This only works appropriately when all of your authoritative DNS
 		## servers are located in your :bro:id:`Site::local_nets`.
-		ExternalName, 
+		External_Name,
 		};
 }
 
@@ -32,7 +32,7 @@ event dns_A_reply(c: connection, msg: dns_msg, ans: dns_answer, a: addr) &priori
 	     !Site::is_local_addr(c$id$resp_h) && # response from an external nameserver
 	     !Site::is_local_name(ans$query) )    # name isn't in a local zone.
 		{
-		NOTICE([$note=ExternalName,
+		NOTICE([$note=External_Name,
 		        $msg=fmt("%s is pointing to a local host - %s.", ans$query, a),
 		        $conn=c]);
 		}

policy/protocols/http/base/file-ident.bro

@@ -16,7 +16,7 @@ export {
 	redef enum Notice::Type += {
 		# This notice is thrown when the file extension doesn't 
 		# seem to match the file contents.
-		IncorrectFileType,
+		Incorrect_File_Type,
 	};
 
 	redef record Info += {
@@ -59,7 +59,7 @@ event signature_match(state: signature_state, msg: string, data: string) &priori
 		{
 		local url = build_url_http(c$http);
 		local message = fmt("%s %s %s", msg, c$http$method, url);
-		NOTICE([$note=IncorrectFileType,
+		NOTICE([$note=Incorrect_File_Type,
 		        $msg=message,
 		        $conn=c,
 		        $method=c$http$method,

policy/tuning/defaults/remove-high-volume-notices.bro

@@ -3,11 +3,9 @@
 
 # Remove these notices from logging since they can be too noisy.
 redef Notice::ignored_types += {
-	Weird::ContentGap,
-	Weird::AckAboveHole,
-	Weird::RetransmissionInconsistency,
+	Weird::Content_Gap,
+	Weird::Ack_Above_Hole,
+	Weird::Retransmission_Inconsistency,
 	## Only allow these to go in the weird log.
-	Weird::WeirdActivity,
-	#DynDisable::ProtocolViolation,
-	
-};
+	Weird::Weird_Activity,
+};

src/bro.bif

@@ -9,6 +9,7 @@
 #include <algorithm>
 #include <cmath>
 #include <sys/stat.h>
+#include <cstdio>
 
 #include "Reporter.h"
 
@@ -3613,3 +3614,24 @@ function NFS3::mode2string%(mode: count%): string
 
 	return new StringVal(str);
 	%}
+
+## Opens a program with popen() and writes a given string to the returned
+## stream to send it to the opened process's stdin.
+## program: a string naming the program to execute
+## to_write: a string to pipe to the opened program's process over stdin
+## Returns: F if popen'ing the program failed, else T
+function piped_exec%(program: string, to_write: string%): bool
+	%{
+	const char* prog = program->CheckString();
+	FILE* f = popen(prog, "w");
+	if ( ! f )
+		{
+		reporter->Error("Failed to popen %s", prog);
+		return new Val(false, TYPE_BOOL);
+		}
+
+	fprintf(f, "%s", to_write->CheckString());
+    pclose(f);
+
+    return new Val(true, TYPE_BOOL);
+	%}

testing/btest/Baseline/bifs.piped_exec/output

@@ -0,0 +1,2 @@
+hello world
+foobar

testing/btest/Baseline/language.rare-events/notice.log

@@ -1,2 +0,0 @@
-# ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	victim	note	msg	sub	src	dst	p	n	action	tag	do_alarm
-1308596064.17872	-	-	-	-	-	-	PacketFilter::DroppedPackets	2 packets dropped after filtering, 1109 received, 10000 on link	-	-	-	-	-	Notice::ACTION_FILE	@UWkUyAuUGXf	F

testing/btest/bifs/piped_exec.bro

@@ -0,0 +1,6 @@
+# @TEST-EXEC: bro %INPUT >output
+# @TEST-EXEC: btest-diff output
+
+global cmds = "print \"hello world\";";
+cmds = string_cat(cmds, "\nprint \"foobar\";");
+piped_exec("bro", cmds);

testing/btest/policy/misc/default-loaded-scripts.test

@@ -7,6 +7,6 @@
 
 # @TEST-EXEC: bro misc/loaded-scripts
 # @TEST-EXEC: test -e loaded_scripts.log
-# @TEST-EXEC: cat loaded_scripts.log | awk 'NR>1{print $2}' | sed ':a;$!N;s/^\(.*\).*\n\1.*/\1/;ta' >prefix
+# @TEST-EXEC: cat loaded_scripts.log | awk 'NR>1{print $2}' | sed -e ':a' -e '$!N' -e 's/^\(.*\).*\n\1.*/\1/' -e 'ta' >prefix
 # @TEST-EXEC: cat loaded_scripts.log | sed "s#`cat prefix`##g" >canonified_loaded_scripts.log
 # @TEST-EXEC: btest-diff canonified_loaded_scripts.log