Merge remote-tracking branch 'origin/master' into topic/seth/notice-suppression

aux/binpac

@@ -1 +1 @@
-Subproject commit 032b4e0f028a08257be0c703b27a7559e57bd40a
+Subproject commit 4fc13f7c6987b4163609e3df7a31f38501411cb7

aux/bro-aux

@@ -1 +1 @@
-Subproject commit 04d149a194e06ed5410ea3af924ff48b9129cd3b
+Subproject commit 86990f1640d986e39d5bb1287dbeb03b59a464f0

aux/broccoli

@@ -1 +1 @@
-Subproject commit 89620cc8e500855fb763281000cbe2a24290a829
+Subproject commit 6df97331bb74d02ef2252138b301e4ca14523962

aux/btest

@@ -1 +1 @@
-Subproject commit d1c620d98ce9d9c0b203314108b413784965d2ed
+Subproject commit ab78a66dd782f165ddf921faaf1f065b2f987481

scripts/base/frameworks/cluster/main.bro

@@ -86,15 +86,15 @@ function local_node_type(): NodeType
 	return is_enabled() ? nodes[node]$node_type : NONE;
 	}
 
-event remote_connection_handshake_done(p: event_peer)
+event remote_connection_handshake_done(p: event_peer) &priority=5
 	{
-	if ( nodes[p$descr]$node_type == WORKER )
+	if ( p$descr in nodes && nodes[p$descr]$node_type == WORKER )
 		++worker_count;
 	}
 
-event remote_connection_closed(p: event_peer)
+event remote_connection_closed(p: event_peer) &priority=5
 	{
-	if ( nodes[p$descr]$node_type == WORKER )
+	if ( p$descr in nodes && nodes[p$descr]$node_type == WORKER )
 		--worker_count;
 	}
 

scripts/base/frameworks/dpd/main.bro

@@ -25,8 +25,7 @@ export {
 		
 		## Disabled analyzer IDs.  This is only for internal tracking 
 		## so as to not attempt to disable analyzers multiple times.
-		# TODO: This is waiting on ticket #460 to remove the '0'.
-		disabled_aids:  set[count]      &default=set(0);
+		disabled_aids:  set[count]      &default=set();
 	};
 	
 	## Ignore violations which go this many bytes into the connection.

scripts/base/protocols/http/main.bro

@@ -30,10 +30,20 @@ export {
 		referrer:                string   &log &optional;
 		## The value of the User-Agent header from the client.
 		user_agent:              string   &log &optional;
-		## The value of the Content-Length header from the client.
-		request_content_length:  count    &log &optional;
-		## The value of the Content-Length header from the server.
-		response_content_length: count    &log &optional;
+		## The actual uncompressed content size of the data transferred from
+		## the client.
+		request_body_len:        count    &log &optional;
+		## This indicates whether or not there was an interruption while the
+		## request body was being sent.
+		request_body_interrupted: bool     &log &default=F;
+		## The actual uncompressed content size of the data transferred from
+		## the server.
+		response_body_len:       count    &log &optional;
+		## This indicates whether or not there was an interruption while the
+		## request body was being sent.  An interruption could cause hash
+		## calculation to fail and a number of other problems since the 
+		## analyzer may not be able to get back on track with the connection.
+		response_body_interrupted: bool     &log &default=F;
 		## The status code returned by the server.
 		status_code:             count    &log &optional;
 		## The status message returned by the server.
@@ -174,9 +184,6 @@ event http_header(c: connection, is_orig: bool, name: string, value: string) &pr
 			# The split is done to remove the occasional port value that shows up here.
 			c$http$host = split1(value, /:/)[1];
 		
-		else if ( name == "CONTENT-LENGTH" )
-			c$http$request_content_length = extract_count(value);
-			
 		else if ( name == "USER-AGENT" )
 			c$http$user_agent = value;
 			
@@ -201,7 +208,7 @@ event http_header(c: connection, is_orig: bool, name: string, value: string) &pr
 					}
 				else
 					{
-					c$http$username = "<problem-decoding>";
+					c$http$username = fmt("<problem-decoding> (%s)", value);
 					if ( c$http$capture_password )
 						c$http$password = userpass;
 					}
@@ -212,9 +219,7 @@ event http_header(c: connection, is_orig: bool, name: string, value: string) &pr
 		}
 	else # server headers
 		{
-		if ( name == "CONTENT-LENGTH" )
-			c$http$response_content_length = extract_count(value);
-		else if ( name == "CONTENT-DISPOSITION" &&
+		if ( name == "CONTENT-DISPOSITION" &&
 		     /[fF][iI][lL][eE][nN][aA][mM][eE]/ in value )
 			c$http$filename = extract_filename_from_content_disposition(value);
 		}
@@ -223,6 +228,17 @@ event http_header(c: connection, is_orig: bool, name: string, value: string) &pr
 event http_message_done(c: connection, is_orig: bool, stat: http_message_stat) &priority = 5
 	{
 	set_state(c, F, is_orig);
+	
+	if ( is_orig )
+		{
+		c$http$request_body_len = stat$body_length;
+		c$http$request_body_interrupted = stat$interrupted;
+		}
+	else
+		{
+		c$http$response_body_len = stat$body_length;
+		c$http$response_body_interrupted = stat$interrupted;
+		}
 	}
 	
 event http_message_done(c: connection, is_orig: bool, stat: http_message_stat) &priority = -5

scripts/base/protocols/ssl/main.bro

@@ -1,15 +1,10 @@
 @load ./consts
-@load base/frameworks/notice
 
 module SSL;
 
 export {
 	redef enum Log::ID += { LOG };
 
-	redef enum Notice::Type += {
-		Self_Signed_Cert
-	};
-
 	type Info: record {
 		ts:               time             &log;
 		uid:              string           &log;
@@ -76,6 +71,20 @@ function set_session(c: connection)
 		c$ssl = [$ts=network_time(), $uid=c$uid, $id=c$id, $cert_chain=vector()];
 	}
 	
+function finish(c: connection, violation: bool)
+	{
+	Log::write(SSL::LOG, c$ssl);
+	if ( delete_certs_after_logging )
+		{
+		if ( c$ssl?$cert )
+			delete c$ssl$cert;
+		if ( c$ssl?$cert_chain )
+			delete c$ssl$cert_chain;
+		}
+	if ( violation )
+		delete c$ssl;
+	}
+
 event ssl_client_hello(c: connection, version: count, possible_ts: time, session_id: string, ciphers: count_set) &priority=5
 	{
 	set_session(c);
@@ -125,14 +134,12 @@ event ssl_established(c: connection) &priority=5
 	
 event ssl_established(c: connection) &priority=-5
 	{
-	Log::write(SSL::LOG, c$ssl);
+	finish(c, F);
+	}
 	
-	if ( delete_certs_after_logging )
+event protocol_violation(c: connection, atype: count, aid: count,
+                         reason: string) &priority=5
 	{
-		if ( c$ssl?$cert )
-			delete c$ssl$cert;
-		if ( c$ssl?$cert_chain )
-			delete c$ssl$cert_chain;
+	if ( c?$ssl )
+		finish(c, T);
 	}
-	}
-	

scripts/policy/protocols/conn/known-services.bro

@@ -7,9 +7,9 @@
 
 module Known;
 
+export {
 	redef enum Log::ID += { SERVICES_LOG };
 	
-export {
 	type Info: record {
 		ts:             time            &log;
 		host:           addr            &log;
@@ -33,7 +33,7 @@ redef record connection += {
 	known_services_watch: bool &default=F;
 };
 
-event bro_init()
+event bro_init() &priority=5
 	{
 	Log::create_stream(Known::SERVICES_LOG, [$columns=Info,
 	                                         $ev=log_known_services]);

scripts/policy/protocols/http/detect-sqli.bro

@@ -1,19 +1,19 @@
 ##! SQL injection detection in HTTP.
 
-@load base/frameworks/notice/main
-@load base/frameworks/metrics/main
-@load base/protocols/http/main
+@load base/frameworks/notice
+@load base/frameworks/metrics
+@load base/protocols/http
 
 module HTTP;
 
 export {
 	redef enum Notice::Type += {
 		SQL_Injection_Attacker,
-		SQL_Injection_Attack,
+		SQL_Injection_Attack_Against,
 	};
 	
 	redef enum Metrics::ID += {
-		SQL_ATTACKS,
+		SQL_ATTACKER,
 		SQL_ATTACKS_AGAINST,
 	};
 
@@ -27,6 +27,16 @@ export {
 		COOKIE_SQLI,
 	};
 	
+	## This defines the threshold that determines if an SQL injection attack
+	## is ongoing based on the number of requests that appear to be SQL 
+	## injection attacks.
+	const sqli_requests_threshold = 50 &redef;
+	
+	## Interval at which to watch for the :bro:id:`sqli_requests_threshold`
+	## variable to be crossed.  At the end of each interval the counter is 
+	## reset.
+	const sqli_requests_interval = 5min &redef;
+
 	## This regular expression is used to match URI based SQL injections
 	const match_sql_injection_uri = 
 		  /[\?&][^[:blank:]\x00-\x37\|]+?=[\-[:alnum:]%]+([[:blank:]\x00-\x37]|\/\*.*?\*\/)*['"]?([[:blank:]\x00-\x37]|\/\*.*?\*\/|\)?;)+.*?([hH][aA][vV][iI][nN][gG]|[uU][nN][iI][oO][nN]|[eE][xX][eE][cC]|[sS][eE][lL][eE][cC][tT]|[dD][eE][lL][eE][tT][eE]|[dD][rR][oO][pP]|[dD][eE][cC][lL][aA][rR][eE]|[cC][rR][eE][aA][tT][eE]|[iI][nN][sS][eE][rR][tT])([[:blank:]\x00-\x37]|\/\*.*?\*\/)+/
@@ -37,15 +47,16 @@ export {
 		| /\/\*![[:digit:]]{5}.*?\*\// &redef;
 }
 
-event bro_init()
+event bro_init() &priority=3
 	{
-	Metrics::add_filter(SQL_ATTACKS, [$log=F,
-	                                  $break_interval=5mins, 
+	Metrics::add_filter(SQL_ATTACKER, [$log=F,
+	                                   $notice_threshold=sqli_requests_threshold,
+	                                   $break_interval=sqli_requests_interval, 
 	                                   $note=SQL_Injection_Attacker]);
 	Metrics::add_filter(SQL_ATTACKS_AGAINST, [$log=F, 
-	                                          $break_interval=5mins, 
-	                                          $note=SQL_Injection_Attack, 
-	                                          $notice_threshold=50]);
+	                                          $notice_threshold=sqli_requests_threshold,
+	                                          $break_interval=sqli_requests_interval, 
+	                                          $note=SQL_Injection_Attack_Against]);
 	}
 
 event http_request(c: connection, method: string, original_URI: string,
@@ -55,7 +66,7 @@ event http_request(c: connection, method: string, original_URI: string,
 		{
 		add c$http$tags[URI_SQLI];
 		
-		Metrics::add_data(SQL_ATTACKS, [$host=c$id$orig_h], 1);
+		Metrics::add_data(SQL_ATTACKER, [$host=c$id$orig_h], 1);
 		Metrics::add_data(SQL_ATTACKS_AGAINST, [$host=c$id$resp_h], 1);
 		}
 	}

scripts/policy/protocols/http/header-names.bro

@@ -0,0 +1,45 @@
+##! Extract and include the header names used for each request in the HTTP
+##! logging stream.  The headers in the logging stream will be stored in the
+##! same order which they were seen on the wire.
+
+@load base/protocols/http/main
+
+module HTTP;
+
+export {
+	redef record Info += {
+		## The vector of HTTP header names sent by the client.  No header 
+		## values are included here, just the header names.
+		client_header_names:  vector of string &log &optional;
+		
+		## The vector of HTTP header names sent by the server.  No header 
+		## values are included here, just the header names.
+		server_headers_names:  vector of string &log &optional;
+	};
+	
+	## A boolean value to determine if client header names are to be logged.
+	const log_client_header_names = T &redef;
+	
+	## A boolean value to determine if server header names are to be logged.
+	const log_server_header_names = F &redef;
+}
+
+event http_header(c: connection, is_orig: bool, name: string, value: string) &priority=3
+	{
+	if ( ! is_orig || ! c?$http )
+		return;
+	
+	if ( log_client_header_names )
+		{
+		if ( ! c$http?$client_header_names )
+			c$http$client_header_names = vector();
+		c$http$client_header_names[|c$http$client_header_names|] = name;
+		}
+		
+	if ( log_server_header_names )
+		{
+		if ( ! c$http?$server_header_names )
+			c$http$server_header_names = vector();
+		c$http$server_header_names[|c$http$server_header_names|] = name;
+		}
+	}

scripts/policy/protocols/http/headers.bro

@@ -1,25 +0,0 @@
-##! Extract and include the header keys used for each request in the log.
-
-@load base/protocols/http/main
-
-module HTTP;
-
-export {
-	redef record Info += {
-		## The vector of HTTP headers.  No header values are included here, just
-		## the header names.
-		## TODO: with an empty vector as &default, the vector isn't coerced to the
-		## correct type.
-		headers:  vector of string &log &optional;
-	};
-}
-
-event http_header(c: connection, is_orig: bool, name: string, value: string) &priority=4
-	{
-	if ( ! is_orig )
-		return;
-	
-	if ( ! c$http?$headers )
-		c$http$headers = vector();
-	c$http$headers[|c$http$headers|] = name;
-	}

scripts/policy/protocols/ssl/known-certs.bro

@@ -34,7 +34,7 @@ export {
 	global log_known_certs: event(rec: Info);
 }
 
-event bro_init()
+event bro_init() &priority=5
 	{
 	Log::create_stream(Known::CERTS_LOG, [$columns=Info, $ev=log_known_certs]);
 	}

src/Desc.cc

@@ -232,33 +232,37 @@ static const char* find_first_unprintable(ODesc* d, const char* bytes, unsigned
 
 void ODesc::AddBytes(const void* bytes, unsigned int n)
 	{
+	if ( ! escape )
+	    {
+	    AddBytesRaw(bytes, n);
+	    return;
+	    }
+
 	const char* s = (const char*) bytes;
 	const char* e = (const char*) bytes + n;
 
 	while ( s < e )
 		{
-		const char* t1 = escape ? (const char*) memchr(s, escape[0], e - s) : e;
-		const char* t2 = find_first_unprintable(this, s, t1 ? e - t1 : e - s);
+		const char* t1 = (const char*) memchr(s, escape[0], e - s);
 
-		if ( t2 && (t2 < t1 || ! t1) )
+		if ( ! t1 )
+		    t1 = e;
+
+		const char* t2 = find_first_unprintable(this, s, t1 - s);
+
+		if ( t2 && t2 < t1 )
 			{
 			AddBytesRaw(s, t2 - s);
 
 			char hex[6] = "\\x00";
 			hex[2] = hex_chars[((*t2) & 0xf0) >> 4];
 			hex[3] = hex_chars[(*t2) & 0x0f];
-			AddBytesRaw(hex, sizeof(hex));
+			AddBytesRaw(hex, 4);
 
 			s = t2 + 1;
 			continue;
 			}
 
-		if ( ! escape )
-			break;
-
-		if ( ! t1 )
-			break;
-
 		if ( memcmp(t1, escape, escape_len) != 0 )
 		    break;
 
@@ -269,7 +273,7 @@ void ODesc::AddBytes(const void* bytes, unsigned int n)
 			char hex[5] = "\\x00";
 			hex[2] = hex_chars[((*t1) & 0xf0) >> 4];
 			hex[3] = hex_chars[(*t1) & 0x0f];
-			AddBytesRaw(hex, sizeof(hex));
+			AddBytesRaw(hex, 4);
 			++t1;
 			}
 

src/bro.bif

@@ -3423,7 +3423,7 @@ function unique_id_from%(pool: int, prefix: string%) : string
 #include <openssl/x509_vfy.h>
 
 // This is the indexed map of X509 certificate stores.
-static map<BroString, X509_STORE*> x509_stores;
+static map<Val*, X509_STORE*> x509_stores;
 
 // ### NOTE: while d2i_X509 does not take a const u_char** pointer,
 // here we assume d2i_X509 does not write to <data>, so it is safe to
@@ -3448,9 +3448,8 @@ function x509_verify%(der_cert: string, cert_stack: string_vec, root_certs: tabl
 	int i = 0;
 
 	// If this certificate store was built previously, just reuse the old one.
-	BroString* s = convert_index_to_string(root_certs);
-	if ( x509_stores.count(*s) > 0 )
-		ctx = x509_stores[*s];
+	if ( x509_stores.count(root_certs) > 0 )
+		ctx = x509_stores[root_certs];
 
 	if ( ! ctx ) // lookup to see if we have this one built already!
 		{
@@ -3475,9 +3474,8 @@ function x509_verify%(der_cert: string, cert_stack: string_vec, root_certs: tabl
 		delete idxs;
 
 		// Save the newly constructed certificate store into the cacheing map.
-		x509_stores[*s] = ctx;
+		x509_stores[root_certs] = ctx;
 		}
-	delete s;
 
 	const uint8 *cert_data = der_cert->Bytes();
 	X509* cert = d2i_X509_(NULL, &cert_data, der_cert->Len());

testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape-odd-url/http.log

@@ -0,0 +1,5 @@
+#separator \x09
+#path	http
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	method	host	uri	referrer	user_agent	request_content_length	response_content_length	status_code	status_msg	filename	tags	username	password	proxied	mime_type	md5	extraction_file
+#types	time	string	addr	port	addr	port	string	string	string	string	string	count	count	count	string	string	table	string	string	table	string	string	file
+1315799856.264750	UWkUyAuUGXf	10.0.1.104	64216	193.40.5.162	80	GET	lepo.it.da.ut.ee	/~cect/teoreetilised seminarid_2010/arheoloogia_uurimisr\xfchma_seminar/Joyce et al - The Languages of Archaeology ~ Dialogue, Narrative and Writing.pdf	-	Wget/1.12 (darwin10.8.0)	-	346	404	Not Found	-	-	-	-	-	text/html	-	-

testing/btest/scripts/base/frameworks/logging/ascii-escape-odd-url.bro

@@ -0,0 +1,4 @@
+#
+# @TEST-EXEC: bro -C -r $TRACES/www-odd-url.trace
+# @TEST-EXEC: btest-diff http.log
+

testing/external/scripts/update-traces

@@ -19,7 +19,7 @@ cfg=traces.cfg
 for p in .proxy ../.proxy; do
     if [ -e $p ]; then
         proxy=`cat $p | head -1 | awk '{print $1}'`
-        echo Using proxy $p ...
+        echo Using proxy $proxy ...
         proxy="ALL_PROXY=$proxy"
         break
     fi
@@ -73,6 +73,8 @@ cat $cfg | while read line; do
         echo "`basename $file` already available."
     fi
 
+    rm -f $fp.tmp
+
 done