Merge branch 'master' into topic/jsiwek/autodoc-fixes

Conflicts:
	scripts/CMakeLists.txt
	scripts/base/frameworks/cluster/setup-connections.bro
	scripts/base/frameworks/communication/__load__.bro
	scripts/base/frameworks/metrics/conn-example.bro
	scripts/base/frameworks/metrics/http-example.bro
	scripts/site/local.bro

scripts/policy/protocols/http/detect-MHR.bro

@@ -1,7 +1,7 @@
 ##! This script takes MD5 sums of files transferred over HTTP and checks them with
 ##! Team Cymru's Malware Hash Registry (http://www.team-cymru.org/Services/MHR/).
 ##! By default, not all file transfers will have MD5 sums calculated.  Read the
-##! documentation for the protocols/http/file-hash.bro script to see how to 
+##! documentation for the base/protocols/http/file-hash.bro script to see how to 
 ##! configure which transfers will have hashes calculated.
 
 export {

scripts/policy/protocols/http/detect-sqli.bro

@@ -9,17 +9,17 @@ export {
 	};
 	
 	redef enum Metrics::ID += {
-		SQL_ATTACKER,
-		SQL_ATTACK_AGAINST,
+		SQL_ATTACKS,
+		SQL_ATTACKS_AGAINST,
 	};
 
 	redef enum Tags += {
 		## Indicator of a URI based SQL injection attack.
 		URI_SQLI,
 		## Indicator of client body based SQL injection attack.  This is 
-		## typically the body content of a POST request. Not implemented yet!
+		## typically the body content of a POST request. Not implemented yet.
 		POST_SQLI,
-		## Indicator of a cookie based SQL injection attack. Not implemented yet!
+		## Indicator of a cookie based SQL injection attack. Not implemented yet.
 		COOKIE_SQLI,
 	};
 
@@ -30,13 +30,18 @@ export {
 		| /[\?&][^[:blank:]\x00-\x37]+?=[\-0-9%]*([[:blank:]\x00-\x37]|\/\*.*?\*\/)*['"]([[:blank:]\x00-\x37]|\/\*.*?\*\/)*(-|=|\+|\|\|)([[:blank:]\x00-\x37]|\/\*.*?\*\/)*([0-9]|\(?[cC][oO][nN][vV][eE][rR][tT]|[cC][aA][sS][tT])/
 		| /[\?&][^[:blank:]\x00-\x37\|]+?=([[:blank:]\x00-\x37]|\/\*.*?\*\/)*['"]([[:blank:]\x00-\x37]|\/\*.*?\*\/|;)*([xX]?[oO][rR]|[nN]?[aA][nN][dD]|[hH][aA][vV][iI][nN][gG]|[uU][nN][iI][oO][nN]|[eE][xX][eE][cC]|[sS][eE][lL][eE][cC][tT]|[dD][eE][lL][eE][tT][eE]|[dD][rR][oO][pP]|[dD][eE][cC][lL][aA][rR][eE]|[cC][rR][eE][aA][tT][eE]|[rR][eE][gG][eE][xX][pP]|[iI][nN][sS][eE][rR][tT])([[:blank:]\x00-\x37]|\/\*.*?\*\/|[\[(])+[a-zA-Z&]{2,}/
 		| /[\?&][^[:blank:]\x00-\x37]+?=[^\.]*?([cC][hH][aA][rR]|[aA][sS][cC][iI][iI]|[sS][uU][bB][sS][tT][rR][iI][nN][gG]|[tT][rR][uU][nN][cC][aA][tT][eE]|[vV][eE][rR][sS][iI][oO][nN]|[lL][eE][nN][gG][tT][hH])\(/
-		| /\/\*![[:digit:]]{5}.*?\*\//;
+		| /\/\*![[:digit:]]{5}.*?\*\// &redef;
 }
 
 event bro_init()
 	{
-	Metrics::add_filter(SQL_ATTACKER, [$break_interval=5mins, $note=SQL_Injection_Attack]);
-	Metrics::add_filter(SQL_ATTACK_AGAINST, [$break_interval=5mins, $note=SQL_Injection_Attack]);
+	Metrics::add_filter(SQL_ATTACKS, [$log=T, 
+	                                  $break_interval=1mins, 
+	                                  $note=SQL_Injection_Attacker]);
+	Metrics::add_filter(SQL_ATTACKS_AGAINST, [$log=T, 
+	                                          $break_interval=1mins, 
+	                                          $note=SQL_Injection_Attack, 
+	                                          $notice_thresholds=vector(10,100)]);
 	}
 
 event http_request(c: connection, method: string, original_URI: string,
@@ -46,7 +51,7 @@ event http_request(c: connection, method: string, original_URI: string,
 		{
 		add c$http$tags[URI_SQLI];
 		
-		Metrics::add_data(SQL_ATTACKER, [$host=c$id$orig_h], 1);
-		Metrics::add_data(SQL_ATTACK_AGAINST, [$host=c$id$resp_h], 1);
+		Metrics::add_data(SQL_ATTACKS, [$host=c$id$orig_h]);
+		Metrics::add_data(SQL_ATTACKS_AGAINST, [$host=c$id$resp_h]);
 		}
 	}

scripts/policy/protocols/smtp/blocklists.bro

@@ -0,0 +1,58 @@
+
+@load base/protocols/smtp
+
+module SMTP;
+
+export {
+	redef enum Notice::Type += { 
+		## Indicates that the server sent a reply mentioning an SMTP block list.
+		Blocklist_Error_Message,
+		## Indicates the client's address is seen in the block list error message.
+		Blocklist_Blocked_Host,
+	};
+
+	# This matches content in SMTP error messages that indicate some
+	# block list doesn't like the connection/mail.
+	const blocklist_error_messages = 
+		  /spamhaus\.org\//
+		| /sophos\.com\/security\//
+		| /spamcop\.net\/bl/
+		| /cbl\.abuseat\.org\// 
+		| /sorbs\.net\// 
+		| /bsn\.borderware\.com\//
+		| /mail-abuse\.com\//
+		| /b\.barracudacentral\.com\//
+		| /psbl\.surriel\.com\// 
+		| /antispam\.imp\.ch\// 
+		| /dyndns\.com\/.*spam/
+		| /rbl\.knology\.net\//
+		| /intercept\.datapacket\.net\//
+		| /uceprotect\.net\//
+		| /hostkarma\.junkemailfilter\.com\// &redef;
+	
+}
+
+event smtp_reply(c: connection, is_orig: bool, code: count, cmd: string,
+                 msg: string, cont_resp: bool) &priority=3
+	{
+	if ( code >= 400 && code != 421 )
+		{
+		# Raise a notice when an SMTP error about a block list is discovered.
+		if ( blocklist_error_messages in msg )
+			{
+			local note = Blocklist_Error_Message;
+			local message = fmt("%s received an error message mentioning an SMTP block list", c$id$orig_h);
+
+			# Determine if the originator's IP address is in the message.
+			local ips = find_ip_addresses(msg);
+			local text_ip = "";
+			if ( |ips| > 0 && to_addr(ips[0]) == c$id$orig_h )
+				{
+				note = Blocklist_Blocked_Host;
+				message = fmt("%s is on an SMTP block list", c$id$orig_h);
+				}
+			
+			NOTICE([$note=note, $conn=c, $msg=message, $sub=msg]);
+			}
+		}
+	}

scripts/policy/protocols/smtp/software.bro

@@ -43,10 +43,10 @@ export {
 	                   | /ZimbraWebClient/ &redef;
 }
 
-event smtp_data(c: connection, is_orig: bool, data: string) &priority=4
+event mime_one_header(c: connection, h: mime_header_rec) &priority=4
 	{
-	if ( c$smtp$current_header == "USER-AGENT" &&
-	     webmail_user_agents in c$smtp$user_agent )
+	if ( ! c?$smtp ) return;
+	if ( h$name == "USER-AGENT" && webmail_user_agents in c$smtp$user_agent )
 		c$smtp$is_webmail = T;
 	}
 

scripts/policy/protocols/ssh/detect-bruteforcing.bro

@@ -0,0 +1,79 @@
+
+module SSH;
+
+export {
+	redef enum Notice::Type += {
+		## Indicates that a host has been identified as crossing the 
+		## :bro:id:`password_guesses_limit` threshold with heuristically
+		## determined failed logins.
+		Password_Guessing,
+		## Indicates that a host previously identified as a "password guesser"
+		## has now had a heuristically successful login attempt.
+		Login_By_Password_Guesser,
+	};
+
+	## The number of failed SSH connections before a host is designated as
+	## guessing passwords.
+	const password_guesses_limit = 30 &redef;
+
+	## The amount of time to remember presumed non-successful logins to build
+	## model of a password guesser.
+	const guessing_timeout = 30 mins &redef;
+
+	## This value can be used to exclude hosts or entire networks from being 
+	## tracked as potential "guessers".  There are cases where the success
+	## heuristic fails and this acts as the whitelist.  The index represents 
+	## client subnets and the yield value represents server subnets.
+	const ignore_guessers: table[subnet] of subnet &redef;
+
+	## Keeps count of how many rejections a host has had.
+	global password_rejections: table[addr] of TrackCount 
+		&write_expire=guessing_timeout
+		&synchronized;
+
+	## Keeps track of hosts identified as guessing passwords.
+	global password_guessers: set[addr] &read_expire=guessing_timeout+1hr &synchronized;
+}
+
+event SSH::heuristic_successful_login(c: connection)
+	{
+	local id = c$id;
+	
+	# TODO: this should be migrated to the metrics framework.
+	if ( id$orig_h in password_rejections &&
+	     password_rejections[id$orig_h]$n > password_guesses_limit &&
+	     id$orig_h !in password_guessers )
+		{
+		add password_guessers[id$orig_h];
+		NOTICE([$note=Login_By_Password_Guesser,
+		        $conn=c,
+		        $n=password_rejections[id$orig_h]$n,
+		        $msg=fmt("Successful SSH login by password guesser %s", id$orig_h),
+		        $sub=fmt("%d failed logins", password_rejections[id$orig_h]$n)]);
+		}
+	}
+
+event SSH::heuristic_failed_login(c: connection)
+	{
+	local id = c$id;
+	
+	# presumed failure
+	if ( id$orig_h !in password_rejections )
+		password_rejections[id$orig_h] = new_track_count();
+	
+	# Track the number of rejections
+	# TODO: this should be migrated to the metrics framework.
+	if ( ! (id$orig_h in ignore_guessers &&
+	        id$resp_h in ignore_guessers[id$orig_h]) )
+		++password_rejections[id$orig_h]$n;
+	
+	if ( default_check_threshold(password_rejections[id$orig_h]) )
+		{
+		add password_guessers[id$orig_h];
+		NOTICE([$note=Password_Guessing,
+		        $conn=c,
+		        $msg=fmt("SSH password guessing by %s", id$orig_h),
+		        $sub=fmt("%d apparently failed logins", password_rejections[id$orig_h]$n),
+		        $n=password_rejections[id$orig_h]$n]);
+		}
+	}

scripts/policy/protocols/ssh/geo-data.bro

@@ -0,0 +1,39 @@
+##! This implements all of the additional information and geodata detections 
+##! for SSH analysis.
+
+module SSH;
+
+export {
+	redef enum Notice::Type += {
+		## If an SSH login is seen to or from a "watched" country based on the
+		## :bro:id:`SSH::watched_countries` variable then this notice will
+		## be generated.
+		Login_From_Watched_Country,
+	};
+	
+	## The set of countries for which you'd like to throw notices upon 
+	## successful login
+	const watched_countries: set[string] = {"RO"} &redef;
+
+	redef record Info += {
+		## Add geographic data related to the "remote" host of the connection.
+		remote_location: geo_location &log &optional;
+	};
+}
+
+event SSH::heuristic_successful_login(c: connection) &priority=5
+	{
+	local location: geo_location;
+	location = (c$ssh$direction == OUTBOUND) ? 
+		lookup_location(c$id$resp_h) : lookup_location(c$id$orig_h);
+	
+	# Add the location data to the SSH record.
+	c$ssh$remote_location = location;
+	
+	if ( location$country_code in watched_countries )
+		{
+		NOTICE([$note=Login_From_Watched_Country,
+		        $conn=c,
+		        $msg=fmt("SSH login from watched country: %s", location$country_code)]);
+		}
+	}

scripts/policy/protocols/ssh/interesting-hostnames.bro

@@ -0,0 +1,50 @@
+
+module SSH;
+
+export {
+	redef enum Notice::Type += {
+		## Generated if a login originates from a host matched by the 
+		## :bro:id:`interesting_hostnames` regular expression.
+		Login_From_Interesting_Hostname,
+		## Generated if a login goes to a host matched by the 
+		## :bro:id:`interesting_hostnames` regular expression.
+		Login_To_Interesting_Hostname,
+	};
+	
+	## Strange/bad host names to see successful SSH logins from or to.
+	const interesting_hostnames =
+			/^d?ns[0-9]*\./ |
+			/^smtp[0-9]*\./ |
+			/^mail[0-9]*\./ |
+			/^pop[0-9]*\./  |
+			/^imap[0-9]*\./ |
+			/^www[0-9]*\./  |
+			/^ftp[0-9]*\./  &redef;
+}
+
+event SSH::heuristic_successful_login(c: connection)
+	{
+	# Check to see if this login came from an interesting hostname.
+	when ( local orig_hostname = lookup_addr(c$id$orig_h) )
+		{
+		if ( interesting_hostnames in orig_hostname )
+			{
+			NOTICE([$note=Login_From_Interesting_Hostname,
+			        $conn=c,
+			        $msg=fmt("Interesting login from hostname: %s", orig_hostname),
+			        $sub=orig_hostname]);
+			}
+		}
+	# Check to see if this login went to an interesting hostname.
+	when ( local resp_hostname = lookup_addr(c$id$orig_h) )
+		{
+		if ( interesting_hostnames in resp_hostname )
+			{
+			NOTICE([$note=Login_To_Interesting_Hostname,
+			        $conn=c,
+			        $msg=fmt("Interesting login to hostname: %s", resp_hostname),
+			        $sub=resp_hostname]);
+			}
+		}
+	}
+

scripts/policy/protocols/ssh/software.bro

@@ -3,8 +3,8 @@ module SSH;
 
 export {
 	redef enum Software::Type += {
-		SSH_SERVER,
-		SSH_CLIENT,
+		SERVER,
+		CLIENT,
 	};
 }
 
@@ -12,7 +12,7 @@ event ssh_client_version(c: connection, version: string) &priority=4
 	{
 	# Get rid of the protocol information when passing to the software framework.
 	local cleaned_version = sub(version, /^SSH[0-9\.\-]+/, "");
-	local si = Software::parse(cleaned_version, c$id$orig_h, SSH_CLIENT);
+	local si = Software::parse(cleaned_version, c$id$orig_h, CLIENT);
 	Software::found(c$id, si);
 	}
 
@@ -20,6 +20,6 @@ event ssh_server_version(c: connection, version: string) &priority=4
 	{
 	# Get rid of the protocol information when passing to the software framework.
 	local cleaned_version = sub(version, /SSH[0-9\.\-]{2,}/, "");
-	local si = Software::parse(cleaned_version, c$id$resp_h, SSH_SERVER);
+	local si = Software::parse(cleaned_version, c$id$resp_h, SERVER);
 	Software::found(c$id, si);
 	}