Merge remote-tracking branch 'origin/master' into topic/seth/notice-suppression

scripts/base/frameworks/cluster/main.bro

@@ -86,15 +86,15 @@ function local_node_type(): NodeType
 	return is_enabled() ? nodes[node]$node_type : NONE;
 	}
 
-event remote_connection_handshake_done(p: event_peer)
+event remote_connection_handshake_done(p: event_peer) &priority=5
 	{
-	if ( nodes[p$descr]$node_type == WORKER )
+	if ( p$descr in nodes && nodes[p$descr]$node_type == WORKER )
 		++worker_count;
 	}
 
-event remote_connection_closed(p: event_peer)
+event remote_connection_closed(p: event_peer) &priority=5
 	{
-	if ( nodes[p$descr]$node_type == WORKER )
+	if ( p$descr in nodes && nodes[p$descr]$node_type == WORKER )
 		--worker_count;
 	}
 

scripts/base/frameworks/dpd/main.bro

@@ -25,8 +25,7 @@ export {
 		
 		## Disabled analyzer IDs.  This is only for internal tracking 
 		## so as to not attempt to disable analyzers multiple times.
-		# TODO: This is waiting on ticket #460 to remove the '0'.
-		disabled_aids:  set[count]      &default=set(0);
+		disabled_aids:  set[count]      &default=set();
 	};
 	
 	## Ignore violations which go this many bytes into the connection.

scripts/base/protocols/http/main.bro

@@ -30,10 +30,20 @@ export {
 		referrer:                string   &log &optional;
 		## The value of the User-Agent header from the client.
 		user_agent:              string   &log &optional;
-		## The value of the Content-Length header from the client.
-		request_content_length:  count    &log &optional;
-		## The value of the Content-Length header from the server.
-		response_content_length: count    &log &optional;
+		## The actual uncompressed content size of the data transferred from
+		## the client.
+		request_body_len:        count    &log &optional;
+		## This indicates whether or not there was an interruption while the
+		## request body was being sent.
+		request_body_interrupted: bool     &log &default=F;
+		## The actual uncompressed content size of the data transferred from
+		## the server.
+		response_body_len:       count    &log &optional;
+		## This indicates whether or not there was an interruption while the
+		## request body was being sent.  An interruption could cause hash
+		## calculation to fail and a number of other problems since the 
+		## analyzer may not be able to get back on track with the connection.
+		response_body_interrupted: bool     &log &default=F;
 		## The status code returned by the server.
 		status_code:             count    &log &optional;
 		## The status message returned by the server.
@@ -174,9 +184,6 @@ event http_header(c: connection, is_orig: bool, name: string, value: string) &pr
 			# The split is done to remove the occasional port value that shows up here.
 			c$http$host = split1(value, /:/)[1];
 		
-		else if ( name == "CONTENT-LENGTH" )
-			c$http$request_content_length = extract_count(value);
-			
 		else if ( name == "USER-AGENT" )
 			c$http$user_agent = value;
 			
@@ -201,7 +208,7 @@ event http_header(c: connection, is_orig: bool, name: string, value: string) &pr
 					}
 				else
 					{
-					c$http$username = "<problem-decoding>";
+					c$http$username = fmt("<problem-decoding> (%s)", value);
 					if ( c$http$capture_password )
 						c$http$password = userpass;
 					}
@@ -212,10 +219,8 @@ event http_header(c: connection, is_orig: bool, name: string, value: string) &pr
 		}
 	else # server headers
 		{
-		if ( name == "CONTENT-LENGTH" )
-			c$http$response_content_length = extract_count(value);
-		else if ( name == "CONTENT-DISPOSITION" &&
-		          /[fF][iI][lL][eE][nN][aA][mM][eE]/ in value )
+		if ( name == "CONTENT-DISPOSITION" &&
+		     /[fF][iI][lL][eE][nN][aA][mM][eE]/ in value )
 			c$http$filename = extract_filename_from_content_disposition(value);
 		}
 	}
@@ -223,6 +228,17 @@ event http_header(c: connection, is_orig: bool, name: string, value: string) &pr
 event http_message_done(c: connection, is_orig: bool, stat: http_message_stat) &priority = 5
 	{
 	set_state(c, F, is_orig);
+	
+	if ( is_orig )
+		{
+		c$http$request_body_len = stat$body_length;
+		c$http$request_body_interrupted = stat$interrupted;
+		}
+	else
+		{
+		c$http$response_body_len = stat$body_length;
+		c$http$response_body_interrupted = stat$interrupted;
+		}
 	}
 	
 event http_message_done(c: connection, is_orig: bool, stat: http_message_stat) &priority = -5

scripts/base/protocols/ssl/main.bro

@@ -1,15 +1,10 @@
 @load ./consts
-@load base/frameworks/notice
 
 module SSL;
 
 export {
 	redef enum Log::ID += { LOG };
 
-	redef enum Notice::Type += {
-		Self_Signed_Cert
-	};
-
 	type Info: record {
 		ts:               time             &log;
 		uid:              string           &log;
@@ -75,6 +70,20 @@ function set_session(c: connection)
 	if ( ! c?$ssl )
 		c$ssl = [$ts=network_time(), $uid=c$uid, $id=c$id, $cert_chain=vector()];
 	}
+	
+function finish(c: connection, violation: bool)
+	{
+	Log::write(SSL::LOG, c$ssl);
+	if ( delete_certs_after_logging )
+		{
+		if ( c$ssl?$cert )
+			delete c$ssl$cert;
+		if ( c$ssl?$cert_chain )
+			delete c$ssl$cert_chain;
+		}
+	if ( violation )
+		delete c$ssl;
+	}
 
 event ssl_client_hello(c: connection, version: count, possible_ts: time, session_id: string, ciphers: count_set) &priority=5
 	{
@@ -125,14 +134,12 @@ event ssl_established(c: connection) &priority=5
 	
 event ssl_established(c: connection) &priority=-5
 	{
-	Log::write(SSL::LOG, c$ssl);
-	
-	if ( delete_certs_after_logging )
-		{
-		if ( c$ssl?$cert )
-			delete c$ssl$cert;
-		if ( c$ssl?$cert_chain )
-			delete c$ssl$cert_chain;
-		}
+	finish(c, F);
 	}
 	
+event protocol_violation(c: connection, atype: count, aid: count,
+                         reason: string) &priority=5
+	{
+	if ( c?$ssl )
+		finish(c, T);
+	}

scripts/policy/protocols/conn/known-services.bro

@@ -7,9 +7,9 @@
 
 module Known;
 
-redef enum Log::ID += { SERVICES_LOG };
-
 export {
+	redef enum Log::ID += { SERVICES_LOG };
+	
 	type Info: record {
 		ts:             time            &log;
 		host:           addr            &log;
@@ -33,10 +33,10 @@ redef record connection += {
 	known_services_watch: bool &default=F;
 };
 
-event bro_init()
+event bro_init() &priority=5
 	{
 	Log::create_stream(Known::SERVICES_LOG, [$columns=Info,
-	                                        $ev=log_known_services]);
+	                                         $ev=log_known_services]);
 	}
 	
 function known_services_done(c: connection)

scripts/policy/protocols/http/detect-sqli.bro

@@ -1,19 +1,19 @@
 ##! SQL injection detection in HTTP.
 
-@load base/frameworks/notice/main
-@load base/frameworks/metrics/main
-@load base/protocols/http/main
+@load base/frameworks/notice
+@load base/frameworks/metrics
+@load base/protocols/http
 
 module HTTP;
 
 export {
 	redef enum Notice::Type += {
 		SQL_Injection_Attacker,
-		SQL_Injection_Attack,
+		SQL_Injection_Attack_Against,
 	};
 	
 	redef enum Metrics::ID += {
-		SQL_ATTACKS,
+		SQL_ATTACKER,
 		SQL_ATTACKS_AGAINST,
 	};
 
@@ -26,6 +26,16 @@ export {
 		## Indicator of a cookie based SQL injection attack. Not implemented yet.
 		COOKIE_SQLI,
 	};
+	
+	## This defines the threshold that determines if an SQL injection attack
+	## is ongoing based on the number of requests that appear to be SQL 
+	## injection attacks.
+	const sqli_requests_threshold = 50 &redef;
+	
+	## Interval at which to watch for the :bro:id:`sqli_requests_threshold`
+	## variable to be crossed.  At the end of each interval the counter is 
+	## reset.
+	const sqli_requests_interval = 5min &redef;
 
 	## This regular expression is used to match URI based SQL injections
 	const match_sql_injection_uri = 
@@ -37,15 +47,16 @@ export {
 		| /\/\*![[:digit:]]{5}.*?\*\// &redef;
 }
 
-event bro_init()
+event bro_init() &priority=3
 	{
-	Metrics::add_filter(SQL_ATTACKS, [$log=F,
-	                                  $break_interval=5mins, 
-	                                  $note=SQL_Injection_Attacker]);
+	Metrics::add_filter(SQL_ATTACKER, [$log=F,
+	                                   $notice_threshold=sqli_requests_threshold,
+	                                   $break_interval=sqli_requests_interval, 
+	                                   $note=SQL_Injection_Attacker]);
 	Metrics::add_filter(SQL_ATTACKS_AGAINST, [$log=F, 
-	                                          $break_interval=5mins, 
-	                                          $note=SQL_Injection_Attack, 
-	                                          $notice_threshold=50]);
+	                                          $notice_threshold=sqli_requests_threshold,
+	                                          $break_interval=sqli_requests_interval, 
+	                                          $note=SQL_Injection_Attack_Against]);
 	}
 
 event http_request(c: connection, method: string, original_URI: string,
@@ -55,7 +66,7 @@ event http_request(c: connection, method: string, original_URI: string,
 		{
 		add c$http$tags[URI_SQLI];
 		
-		Metrics::add_data(SQL_ATTACKS, [$host=c$id$orig_h], 1);
+		Metrics::add_data(SQL_ATTACKER, [$host=c$id$orig_h], 1);
 		Metrics::add_data(SQL_ATTACKS_AGAINST, [$host=c$id$resp_h], 1);
 		}
 	}

scripts/policy/protocols/http/header-names.bro

@@ -0,0 +1,45 @@
+##! Extract and include the header names used for each request in the HTTP
+##! logging stream.  The headers in the logging stream will be stored in the
+##! same order which they were seen on the wire.
+
+@load base/protocols/http/main
+
+module HTTP;
+
+export {
+	redef record Info += {
+		## The vector of HTTP header names sent by the client.  No header 
+		## values are included here, just the header names.
+		client_header_names:  vector of string &log &optional;
+		
+		## The vector of HTTP header names sent by the server.  No header 
+		## values are included here, just the header names.
+		server_headers_names:  vector of string &log &optional;
+	};
+	
+	## A boolean value to determine if client header names are to be logged.
+	const log_client_header_names = T &redef;
+	
+	## A boolean value to determine if server header names are to be logged.
+	const log_server_header_names = F &redef;
+}
+
+event http_header(c: connection, is_orig: bool, name: string, value: string) &priority=3
+	{
+	if ( ! is_orig || ! c?$http )
+		return;
+	
+	if ( log_client_header_names )
+		{
+		if ( ! c$http?$client_header_names )
+			c$http$client_header_names = vector();
+		c$http$client_header_names[|c$http$client_header_names|] = name;
+		}
+		
+	if ( log_server_header_names )
+		{
+		if ( ! c$http?$server_header_names )
+			c$http$server_header_names = vector();
+		c$http$server_header_names[|c$http$server_header_names|] = name;
+		}
+	}

scripts/policy/protocols/http/headers.bro

@@ -1,25 +0,0 @@
-##! Extract and include the header keys used for each request in the log.
-
-@load base/protocols/http/main
-
-module HTTP;
-
-export {
-	redef record Info += {
-		## The vector of HTTP headers.  No header values are included here, just
-		## the header names.
-		## TODO: with an empty vector as &default, the vector isn't coerced to the
-		## correct type.
-		headers:  vector of string &log &optional;
-	};
-}
-
-event http_header(c: connection, is_orig: bool, name: string, value: string) &priority=4
-	{
-	if ( ! is_orig )
-		return;
-	
-	if ( ! c$http?$headers )
-		c$http$headers = vector();
-	c$http$headers[|c$http$headers|] = name;
-	}

scripts/policy/protocols/ssl/known-certs.bro

@@ -34,7 +34,7 @@ export {
 	global log_known_certs: event(rec: Info);
 }
 
-event bro_init()
+event bro_init() &priority=5
 	{
 	Log::create_stream(Known::CERTS_LOG, [$columns=Info, $ev=log_known_certs]);
 	}