Merge remote-tracking branch 'origin/topic/jsiwek/unit-tests'

* origin/topic/jsiwek/unit-tests:
  Fix utils/conn-ids test due to renamed conn-ids.bro
  Moving the test for site.bro to live w/ other utils/ tests.
  Fix test due to moving of site.bro
  More policy/utils unit tests and documentation.
  Updating documentation for some utils/ policy scripts
  Add unit tests for utils/paths.bro with some changes
  Adding unit tests for utils.
  Adding test for utils/addrs.bro.
  Add unit test for site.bro.

Conflicts:
	policy/utils/site.bro

Closes #525.

policy/protocols/ftp/base.bro

@@ -156,7 +156,7 @@ function ftp_message(s: Info)
 		
 		local arg = s$cmdarg$arg;
 		if ( s$cmdarg$cmd in file_cmds )
-			arg = fmt("ftp://%s%s", s$id$resp_h, absolute_path(s$cwd, arg));
+			arg = fmt("ftp://%s%s", s$id$resp_h, build_path_compressed(s$cwd, arg));
 		
 		s$ts=s$cmdarg$ts;
 		s$command=s$cmdarg$cmd;
@@ -279,13 +279,13 @@ event ftp_reply(c: connection, code: count, msg: string, cont_resp: bool) &prior
 	if ( [c$ftp$cmdarg$cmd, code] in directory_cmds )
 		{
 		if ( c$ftp$cmdarg$cmd == "CWD" )
-			c$ftp$cwd = build_full_path(c$ftp$cwd, c$ftp$cmdarg$arg);
+			c$ftp$cwd = build_path(c$ftp$cwd, c$ftp$cmdarg$arg);
 
 		else if ( c$ftp$cmdarg$cmd == "CDUP" )
 			c$ftp$cwd = cat(c$ftp$cwd, "/..");
 
 		else if ( c$ftp$cmdarg$cmd == "PWD" || c$ftp$cmdarg$cmd == "XPWD" )
-			c$ftp$cwd = extract_directory(msg);
+			c$ftp$cwd = extract_path(msg);
 		}
 	
 	# In case there are multiple commands queued, go ahead and remove the

policy/utils/addrs.bro

@@ -27,36 +27,65 @@ const ip_addr_regex =
     /(([0-9A-Fa-f]{1,4}:){6,6})([0-9]+)\.([0-9]+)\.([0-9]+)\.([0-9]+)/ | # 6Hex4Dec
     /(([0-9A-Fa-f]{1,4}(:[0-9A-Fa-f]{1,4})*)?)::(([0-9A-Fa-f]{1,4}:)*)([0-9]+)\.([0-9]+)\.([0-9]+)\.([0-9]+)/; # CompressedHex4Dec
 
-## Takes a string and returns T or F if the string appears to be a full and 
+## Checks if all elements of a string array are a valid octet value.
-## valid IP address.
+## octets: an array of strings to check for valid octet values.
+## Returns: T if every element is between 0 and 255, inclusive, else F.
+function has_valid_octets(octets: string_array): bool
+	{
+	local num = 0;
+	for ( i in octets )
+		{
+		num = to_count(octets[i]);
+		if ( num < 0 || 255 < num )
+			return F;
+		}
+	return T;
+	}
+
+## Checks if a string appears to be a valid IPv4 or IPv6 address.
+## ip_str: the string to check for valid IP formatting.
+## Returns: T if the string is a valid IPv4 or IPv6 address format.
 function is_valid_ip(ip_str: string): bool
 	{
+	local octets: string_array;
 	if ( ip_str == ipv4_addr_regex )
 		{
-		local octets = split(ip_str, /\./);
+		octets = split(ip_str, /\./);
 		if ( |octets| != 4 )
 			return F;
 		
-		local num=0;
+		return has_valid_octets(octets);
-		for ( i in octets )
-			{
-			num = to_count(octets[i]);
-			if ( num < 0 || 255 < num )
-				return F;
-			}
-		return T;
 		}
 	else if ( ip_str == ipv6_addr_regex )
 		{
-		# TODO: make this work correctly.
+		if ( ip_str == ipv6_hex4dec_regex ||
-		return T;
+		     ip_str == ipv6_compressed_hex4dec_regex )
+			{
+			# the regexes for hybrid IPv6-IPv4 address formats don't for valid
+			# octets within the IPv4 part, so do that now
+			octets = split(ip_str, /\./);
+			if ( |octets| != 4 )
+				return F;
+
+			# get rid of remaining IPv6 stuff in first octet
+			local tmp = split(octets[1], /:/);
+			octets[1] = tmp[|tmp|];
+
+			return has_valid_octets(octets);
+			}
+		else
+			{
+			# pure IPv6 address formats that only use hex digits don't need
+			# any additional checks -- the regexes should be complete
+			return T;
+			}
 		}
 	return F;
 	}
 
-## This outputs a string_array of ip addresses extracted from a string.
+## Extracts all IP (v4 or v6) address strings from a given string.
-## given: "this is 1.1.1.1 a test 2.2.2.2 string with ip addresses 3.3.3.3"
+## input: a string that may contain an IP address anywhere within it.
-## outputs: { [0] = 1.1.1.1, [1] = 2.2.2.2, [2] = 3.3.3.3 }
+## Returns: an array containing all valid IP address strings found in input.
 function find_ip_addresses(input: string): string_array
 	{
 	local parts = split_all(input, ip_addr_regex);

policy/utils/conn-ids.bro

@@ -4,15 +4,17 @@ module GLOBAL;
 
 export {
 	## Takes a conn_id record and returns a string representation with the 
-	## the general data flow appearing to be toward the right.
+	## the general data flow appearing to be from the connection originator
+	## on the left to the responder on the right.
 	global id_string: function(id: conn_id): string;
 	
 	## Takes a conn_id record and returns a string representation with the 
-	## the general data flow appearing to be toward the left.
+	## the general data flow appearing to be from the connection responder
+	## on the right to the originator on the left.
 	global reverse_id_string: function(id: conn_id): string;
 	
-	## Calls either the :bro:id:`id_string` or :bro:id:`reverse_id_string`
+	## Calls :bro:id:`id_string` or :bro:id:`reverse_id_string` if the second
-	## function depending on the second argument.
+	## argument is T or F, respectively.
 	global directed_id_string: function(id: conn_id, is_orig: bool): string;
 }
 

policy/utils/directions-and-hosts.bro

@@ -1,15 +1,53 @@
 
-type Direction: enum { INBOUND, OUTBOUND, BIDIRECTIONAL, NO_DIRECTION };
+type Direction: enum {
+	## The connection originator is not within the locally-monitored network,
+	## but the other endpoint is.
+	INBOUND,
+	## The connection originator is within the locally-monitored network,
+	## but the other endpoint is not.
+	OUTBOUND,
+	## Only one endpoint is within the locally-monitored network, meaning
+	## the connection is either outbound or inbound.
+	BIDIRECTIONAL,
+	## This value doesn't match any connection.
+	NO_DIRECTION
+};
+
+## Checks whether a given connection is of a given direction with respect
+## to the locally-monitored network.
+## id: a connection record containing the originator/responder hosts.
+## d: a direction with respect to the locally-monitored network
+## Returns: T if the two connection endpoints match the given direction, else F.
 function id_matches_direction(id: conn_id, d: Direction): bool
 	{
 	if ( d == NO_DIRECTION ) return F;
 
-	return ( d == BIDIRECTIONAL ||
+	local o_local = Site::is_local_addr(id$orig_h);
-	        (d == OUTBOUND && Site::is_local_addr(id$orig_h)) ||
+	local r_local = Site::is_local_addr(id$resp_h);
-	        (d == INBOUND && Site::is_local_addr(id$resp_h)) );
+
+	if ( d == BIDIRECTIONAL )
+		return (o_local && !r_local) || (!o_local && r_local);
+	else if ( d == OUTBOUND )
+		return o_local && !r_local;
+	else if ( d == INBOUND )
+		return !o_local && r_local;
 	}
-	
+
-type Host: enum { LOCAL_HOSTS, REMOTE_HOSTS, ALL_HOSTS, NO_HOSTS };
+type Host: enum {
+	## A host within the locally-monitored network.
+	LOCAL_HOSTS,
+	## A host not within the locally-monitored network.
+	REMOTE_HOSTS,
+	## Any host.
+	ALL_HOSTS,
+	## This value doesn't match any host.
+	NO_HOSTS
+};
+
+## Checks whether a given host (IP address) matches a given host type.
+## ip: address of a host
+## h: a host type
+## Returns: T if the given host matches the given type, else F.
 function addr_matches_host(ip: addr, h: Host): bool
 	{
 	if ( h == NO_HOSTS ) return F;

policy/utils/paths.bro

@@ -2,31 +2,41 @@
 
 const absolute_path_pat = /(\/|[A-Za-z]:[\\\/]).*/;
 
-## Given an arbitrary string, this should extract a single directory with
+## Given an arbitrary string, extracts a single, absolute path (directory
-## filename if it's included.
+## with filename).
 ## TODO: Make this work on Window's style directories.
-function extract_directory(input: string): string
+## input: a string that may contain an absolute path
+## Returns: the first absolute path found in input string, else an empty string
+function extract_path(input: string): string
 	{
-	const dir_pattern = /\"([^\"]|\"\")*(\/|\\)([^\"]|\"\")*\"/;
+	const dir_pattern = /(\/|[A-Za-z]:[\\\/])([^\"\ ]|(\\\ ))*/;
 	local parts = split_all(input, dir_pattern);
 
-	# This basically indicates no identifiable directory was found.
 	if ( |parts| < 3 )
 		return "";
 
-	local d = parts[2];
+	return parts[2];
-	return sub_bytes(d, 2, int_to_count(|d| - 2));
 	}
 
-## Process ..'s and eliminate duplicate '/'s
+## Compresses a given path by removing '..'s and the parent directory it
+## references and also removing '/'s.
+## dir: a path string, either relative or absolute
+## Returns: a compressed version of the input path
 function compress_path(dir: string): string
 	{
-	const cdup_sep = /((\/)+([^\/]|\\\/)+)?((\/)+\.\.(\/)+)/;
+	const cdup_sep = /((\/)*([^\/]|\\\/)+)?((\/)+\.\.(\/)*)/;
 
 	local parts = split_n(dir, cdup_sep, T, 1);
 	if ( length(parts) > 1 )
 		{
-		parts[2] = "/";
+		# reaching a point with two parent dir references back-to-back means
+		# we don't know about anything higher in the tree to pop off
+		if ( parts[2] == "../.." )
+			return cat_string_array(parts);
+		if ( sub_bytes(parts[2], 0, 1) == "/" )
+			parts[2] = "/";
+		else
+			parts[2] = "";
 		dir = cat_string_array(parts);
 		return compress_path(dir);
 		}
@@ -38,24 +48,27 @@ function compress_path(dir: string): string
 			parts[i] = "/";
 	dir = cat_string_array(parts);
 
+	# remove trailing slashes from path
+	if ( |dir| > 1 && sub_bytes(dir, |dir|, 1) == "/" )
+		dir = sub_bytes(dir, 0, |dir| - 1);
+
 	return dir;
 	}
 
-## Computes the absolute path with current working directory.
+## Constructs a path to a file given a directory and a file name.
-function absolute_path(cwd: string, file_name: string): string
+## dir: the directory in which the file lives
-	{
+## file_name: the name of the file
-	local abs_file_name: string;
+## Returns: the concatenation of the directory path and file name, or just
-	if ( file_name == absolute_path_pat ) # start with '/' or 'A:\'
+##          the file name if it's already an absolute path
-		abs_file_name = file_name;
+function build_path(dir: string, file_name: string): string
-	else
-		abs_file_name = string_cat(cwd, "/", file_name);
-	return compress_path(abs_file_name);
-	}
-
-## Takes a directory and a filename and combines them together into a full
-## filename with path.
-function build_full_path(cwd: string, file_name: string): string
 	{
 	return (file_name == absolute_path_pat) ?
-		file_name : cat(cwd, "/", file_name);
+		file_name : cat(dir, "/", file_name);
+	}
+
+## Returns a compressed path to a file given a directory and file name.
+## See :bro:id`build_path` and :bro:id:`compress_path`.
+function build_path_compressed(dir: string, file_name: string): string
+	{
+	return compress_path(build_path(dir, file_name));
 	}

policy/utils/pattern.bro

@@ -1,11 +1,13 @@
 ##! Functions for creating and working with patterns.
 
-## This function only works at or before init time.  Given a pattern as a string
+## Given a pattern as a string with two tildes (~~) contained in it, it will
-## with two tildes (~~) contained in it, it will return a pattern with the
+## return a pattern with string set's elements OR'd together where the
-## set[string] elements OR'd together where the double-tilde was given.
+## double-tilde was given (this function only works at or before init time).
-## If a literal backslash is include in 'pat', it needs to be given as a double
+## ss: a set of strings to OR together
-## backslash due to Bro's string parsing reducing it to a single backslash
+## pat: the pattern containing a "~~"  in it.  If a literal backslash is
-## upon rendering.
+##      included, it needs to be escaped with another backslash due to Bro's
+##      string parsing reducing it to a single backslash upon rendering.
+## Returns: the input pattern with "~~" replaced by OR'd elements of input set
 function set_to_regex(ss: set[string], pat: string): pattern
 	{
 	local i: count = 0;
@@ -31,15 +33,13 @@ type PatternMatchResult: record {
 
 ## Matches the given pattern against the given string, returning
 ## a :bro:type:`PatternMatchResult` record.
-## For example:
+## For example: ``match_pattern("foobar", /o*[a-k]/)`` returns
-##     match_pattern("foobar", /o*[a-k]/)
+## ``[matched=T, str=f, off=1]``,  because the *first* match is for
-## returns:
+## zero o's followed by an [a-k], but ``match_pattern("foobar", /o+[a-k]/)``
-##     [matched=T, str=f, off=1]
+## returns ``[matched=T, str=oob, off=2]``
-## because the *first* match is for zero o's followed by an [a-k],
+## s: a string to match against
-## while:
+## p: a pattern to match
-##     match_pattern("foobar", /o+[a-k]/)
+## Returns: a record indicating the match status
-## returns:
-##     [matched=T, str=oob, off=2]
 function match_pattern(s: string, p: pattern): PatternMatchResult
 	{
 	local a = split_n(s, p, T, 1);

policy/utils/site.bro

@@ -105,8 +105,11 @@ function find_all_emails(ip: addr): set[string]
 		tmp_subnet = mask_addr(ip, one_to_32[i]);
 		for ( email in local_admins[tmp_subnet] )
 			{
-			if ( email != "" )
+			for ( email in local_admins[tmp_ip] )
-				add output_values[email];
+				{
+				if ( email != "" )
+					add output_values[email];
+				}
 			}
 		}
 	return output_values;

policy/utils/strings.bro

@@ -9,8 +9,11 @@ function is_string_binary(s: string): bool
 	return byte_len(gsub(s, /[\x00-\x7f]/, "")) * 100 / |s| >= 25;
 	}
 
-## Takes a :bro:type:`set[string]` and joins each element together with the 
+## Joins a set of string together, with elements delimited by a constant string.
-## second argument.
+## ss: a set of strings to join
+## j: the string used to join set elements
+## Returns: a string composed of the all elements of the set, delimited by the
+##          joining string.
 function join_string_set(ss: set[string], j: string): string
 	{
 	local output="";
@@ -26,9 +29,11 @@ function join_string_set(ss: set[string], j: string): string
 	return output;
 	}
 
-## Given a string, returns an escaped version.  This means that
+## Given a string, returns an escaped version.
-## (1) any occurrences of any character in "chars" are escaped using '\', and
+## s: a string to escape
-## (2) any '\'s are likewise escaped.
+## chars: a string containing all the characters that need to be escaped
+## Returns: a string with all occurrences of any character in ``chars`` escaped
+##          using ``\``, and any literal ``\`` characters likewise escaped.
 function string_escape(s: string, chars: string): string
 	{
 	s = subst_string(s, "\\", "\\\\");
@@ -38,6 +43,9 @@ function string_escape(s: string, chars: string): string
 	}
 
 ## Cut a number of character from the end of the given string.
+## s: a string to trim
+## tail_len: the number of characters to remove from end of string
+## Returns: the string in ``s`` with ``tail_len`` characters removed from end
 function cut_tail(s: string, tail_len: count): string
 	{
 	if ( tail_len > |s| )

policy/utils/thresholds.bro

@@ -22,8 +22,12 @@ export {
 		30, 100, 1000, 10000, 100000, 1000000, 10000000,
 	} &redef;
 	
-	## This will check if a :bro:type:`TrackCount` variable has crossed the 
+	## This will check if a :bro:type:`TrackCount` variable has crossed any
-	## thresholds given in the first value.
+	## thresholds in a given set.
+	## v: a vector holding counts that represent thresholds
+	## tracker: the record being used to track event counter and currently
+	##          monitored threshold value
+	## Returns: T if a threshold has been crossed, else F
 	global check_threshold: function(v: vector of count, tracker: TrackCount): bool;
 	
 	## This will use the :bro:id:`default_notice_thresholds` variable to check 

testing/btest/Baseline/policy.utils.addrs/output

@@ -0,0 +1,43 @@
+============ test ipv4 regex
+T
+T
+T
+T
+T
+T
+T
+F
+F
+F
+T
+T
+============ test ipv6 regex
+T
+T
+T
+T
+T
+F
+F
+F
+F
+F
+T
+T
+============ test ipv6-ipv4 hybrid regexes
+T
+T
+F
+F
+F
+============ test find_ip_addresses()
+{
+[0] = 1.1.1.1,
+[2] = 3.3.3.3,
+[1] = 2.2.2.2
+}
+{
+[0] = 1.1.1.1,
+[2] = 3.3.3.3,
+[1] = 0:0:0:0:0:0:0:0
+}

testing/btest/Baseline/policy.utils.conn-ids/output

@@ -0,0 +1,6 @@
+10.0.0.100:10000 > 10.0.0.200:20000
+10.0.0.100:10000 < 10.0.0.200:20000
+10.0.0.100:10000 > 10.0.0.200:20000
+10.0.0.100:10000 < 10.0.0.200:20000
+T
+T

testing/btest/Baseline/policy.utils.directions-and-hosts/output

@@ -0,0 +1,24 @@
+LOCAL_HOSTS(10.0.0.100) == T: SUCCESS
+REMOTE_HOSTS(10.0.0.100) == F: SUCCESS
+ALL_HOSTS(10.0.0.100) == T: SUCCESS
+NO_HOSTS(10.0.0.100) == F: SUCCESS
+LOCAL_HOSTS(192.168.1.100) == F: SUCCESS
+REMOTE_HOSTS(192.168.1.100) == T: SUCCESS
+ALL_HOSTS(192.168.1.100) == T: SUCCESS
+NO_HOSTS(192.168.1.100) == F: SUCCESS
+INBOUND(o: 10.0.0.100, r: 10.0.0.200) == F: SUCCESS
+INBOUND(o: 10.0.0.100, r: 192.168.1.100) == F: SUCCESS
+INBOUND(o: 192.168.1.100, r: 10.0.0.100) == T: SUCCESS
+INBOUND(o: 192.168.1.100, r: 192.168.1.200) == F: SUCCESS
+OUTBOUND(o: 10.0.0.100, r: 10.0.0.200) == F: SUCCESS
+OUTBOUND(o: 10.0.0.100, r: 192.168.1.100) == T: SUCCESS
+OUTBOUND(o: 192.168.1.100, r: 10.0.0.100) == F: SUCCESS
+OUTBOUND(o: 192.168.1.100, r: 192.168.1.200) == F: SUCCESS
+BIDIRECTIONAL(o: 10.0.0.100, r: 10.0.0.200) == F: SUCCESS
+BIDIRECTIONAL(o: 10.0.0.100, r: 192.168.1.100) == T: SUCCESS
+BIDIRECTIONAL(o: 192.168.1.100, r: 10.0.0.100) == T: SUCCESS
+BIDIRECTIONAL(o: 192.168.1.100, r: 192.168.1.200) == F: SUCCESS
+NO_DIRECTION(o: 10.0.0.100, r: 10.0.0.200) == F: SUCCESS
+NO_DIRECTION(o: 10.0.0.100, r: 192.168.1.100) == F: SUCCESS
+NO_DIRECTION(o: 192.168.1.100, r: 10.0.0.100) == F: SUCCESS
+NO_DIRECTION(o: 192.168.1.100, r: 192.168.1.200) == F: SUCCESS

testing/btest/Baseline/policy.utils.files/output

@@ -0,0 +1,32 @@
+test-prefix_141.142.220.118:48649-208.80.152.118:80_test-suffix
+test-prefix_141.142.220.118:48649-208.80.152.118:80
+141.142.220.118:48649-208.80.152.118:80_test-suffix
+141.142.220.118:48649-208.80.152.118:80
+test-prefix_141.142.220.118:49997-208.80.152.3:80_test-suffix
+test-prefix_141.142.220.118:49997-208.80.152.3:80
+141.142.220.118:49997-208.80.152.3:80_test-suffix
+141.142.220.118:49997-208.80.152.3:80
+test-prefix_141.142.220.118:49996-208.80.152.3:80_test-suffix
+test-prefix_141.142.220.118:49996-208.80.152.3:80
+141.142.220.118:49996-208.80.152.3:80_test-suffix
+141.142.220.118:49996-208.80.152.3:80
+test-prefix_141.142.220.118:49998-208.80.152.3:80_test-suffix
+test-prefix_141.142.220.118:49998-208.80.152.3:80
+141.142.220.118:49998-208.80.152.3:80_test-suffix
+141.142.220.118:49998-208.80.152.3:80
+test-prefix_141.142.220.118:50000-208.80.152.3:80_test-suffix
+test-prefix_141.142.220.118:50000-208.80.152.3:80
+141.142.220.118:50000-208.80.152.3:80_test-suffix
+141.142.220.118:50000-208.80.152.3:80
+test-prefix_141.142.220.118:49999-208.80.152.3:80_test-suffix
+test-prefix_141.142.220.118:49999-208.80.152.3:80
+141.142.220.118:49999-208.80.152.3:80_test-suffix
+141.142.220.118:49999-208.80.152.3:80
+test-prefix_141.142.220.118:50001-208.80.152.3:80_test-suffix
+test-prefix_141.142.220.118:50001-208.80.152.3:80
+141.142.220.118:50001-208.80.152.3:80_test-suffix
+141.142.220.118:50001-208.80.152.3:80
+test-prefix_141.142.220.118:35642-208.80.152.2:80_test-suffix
+test-prefix_141.142.220.118:35642-208.80.152.2:80
+141.142.220.118:35642-208.80.152.2:80_test-suffix
+141.142.220.118:35642-208.80.152.2:80

testing/btest/Baseline/policy.utils.numbers/output

@@ -0,0 +1,7 @@
+0
+13
+13
+13
+13
+13
+1

testing/btest/Baseline/policy.utils.paths/output

@@ -0,0 +1,84 @@
+test compress_path()
+===============================
+Given : foo//bar
+Expect: foo/bar
+Result: foo/bar
+Result: SUCCESS
+===============================
+Given : foo//bar/..
+Expect: foo
+Result: foo
+Result: SUCCESS
+===============================
+Given : foo/bar/../..
+Expect: 
+Result: 
+Result: SUCCESS
+===============================
+Given : foo//bar/../..
+Expect: 
+Result: 
+Result: SUCCESS
+===============================
+Given : /foo/../bar
+Expect: /bar
+Result: /bar
+Result: SUCCESS
+===============================
+Given : /foo/../bar/..
+Expect: /
+Result: /
+Result: SUCCESS
+===============================
+Given : /foo/baz/../..
+Expect: /
+Result: /
+Result: SUCCESS
+===============================
+Given : ../..
+Expect: ../..
+Result: ../..
+Result: SUCCESS
+===============================
+Given : foo/../../..
+Expect: ../..
+Result: ../..
+Result: SUCCESS
+===============================
+test extract_path()
+===============================
+Given : "/this/is/a/dir" is current directory
+Expect: /this/is/a/dir
+Result: /this/is/a/dir
+Result: SUCCESS
+===============================
+Given : /this/is/a/dir is current directory
+Expect: /this/is/a/dir
+Result: /this/is/a/dir
+Result: SUCCESS
+===============================
+Given : /this/is/a/dir\ is\ current\ directory
+Expect: /this/is/a/dir\ is\ current\ directory
+Result: /this/is/a/dir\ is\ current\ directory
+Result: SUCCESS
+===============================
+Given : hey, /foo/bar/baz.bro is a cool script
+Expect: /foo/bar/baz.bro
+Result: /foo/bar/baz.bro
+Result: SUCCESS
+===============================
+Given : here's two dirs: /foo/bar and /foo/baz
+Expect: /foo/bar
+Result: /foo/bar
+Result: SUCCESS
+===============================
+test build_path_compressed()
+===============================
+/home/bro/policy/somefile.bro
+/usr/local/bro/share/bro/somefile.bro
+/usr/local/bro/somefile.bro
+===============================
+test build_full_path()
+===============================
+/home/bro//policy/somefile.bro
+/usr/local/bro/share/bro/somefile.bro

testing/btest/Baseline/policy.utils.pattern/output

@@ -0,0 +1,6 @@
+/^?((blarg|blah|bleh))$?/
+T
+/^?(foo(blarg|blah|bleh)bar)$?/
+T
+[matched=T, str=blah, off=4]
+[matched=F, str=, off=0]

testing/btest/Baseline/policy.utils.site/output

@@ -0,0 +1,2 @@
+other-site-admin@example.com, site-admin@example.com
+other-site-admin@example.com, net-admin@example.com, site-admin@example.com

testing/btest/Baseline/policy.utils.strings/output

@@ -0,0 +1,13 @@
+'hello' is NOT considered binary
+'\xff\xff\xff\0' IS considered binary
+'\0\0\xff\0' IS considered binary
+'\0\0\0\0' is NOT considered binary
+two, one, three
+one
+hell\o w\orl\d
+\\hello world\\
+hello world
+hello worl
+hello
+
+

testing/btest/Baseline/policy.utils.thresholds/output

@@ -0,0 +1,45 @@
+Iteration: 0, threshold check: F
+[n=0, index=0]
+Iteration: 1, threshold check: F
+[n=1, index=0]
+Iteration: 2, threshold check: T
+[n=2, index=1]
+Iteration: 3, threshold check: F
+[n=3, index=1]
+Iteration: 4, threshold check: T
+[n=4, index=2]
+Iteration: 5, threshold check: F
+[n=5, index=2]
+Iteration: 6, threshold check: T
+[n=6, index=3]
+Iteration: 7, threshold check: F
+[n=7, index=3]
+Iteration: 8, threshold check: T
+[n=8, index=4]
+Iteration: 9, threshold check: F
+[n=9, index=4]
+Iteration: 10, threshold check: T
+[n=10, index=5]
+====================================
+Iteration: 0, threshold check: F
+[n=0, index=0]
+Iteration: 1, threshold check: F
+[n=1, index=0]
+Iteration: 2, threshold check: T
+[n=2, index=1]
+Iteration: 3, threshold check: F
+[n=3, index=1]
+Iteration: 4, threshold check: T
+[n=4, index=2]
+Iteration: 5, threshold check: F
+[n=5, index=2]
+Iteration: 6, threshold check: T
+[n=6, index=3]
+Iteration: 7, threshold check: F
+[n=7, index=3]
+Iteration: 8, threshold check: T
+[n=8, index=4]
+Iteration: 9, threshold check: F
+[n=9, index=4]
+Iteration: 10, threshold check: T
+[n=10, index=5]

testing/btest/policy/utils/addrs.test

@@ -0,0 +1,104 @@
+# @TEST-EXEC: bro %INPUT > output
+# @TEST-EXEC: btest-diff output
+
+@load utils/addrs
+
+event bro_init()
+	{
+	local ip = "0.0.0.0";
+
+	print "============ test ipv4 regex";
+	print ip == ipv4_addr_regex;
+	print is_valid_ip(ip);
+	ip = "1.1.1.1";
+	print ip == ipv4_addr_regex;
+	print is_valid_ip(ip);
+	ip = "255.255.255.255";
+	print ip == ipv4_addr_regex;
+	print is_valid_ip(ip);
+	ip = "255.255.255.256";
+	print ip == ipv4_addr_regex; # the regex doesn't check for 0-255
+	print is_valid_ip(ip);       # but is_valid_ip() will
+	ip = "255.255.255.255.255";
+	print ip == ipv4_addr_regex;
+	print is_valid_ip(ip);
+	ip = "192.168.1.100";
+	print ip == ipv4_addr_regex;
+	print is_valid_ip(ip);
+
+	print "============ test ipv6 regex";
+
+	ip = "2001:0db8:85a3:0000:0000:8a2e:0370:7334";
+	print is_valid_ip(ip);
+
+	# test for case insensitivity
+	ip = "2001:0DB8:85A3:0000:0000:8A2E:0370:7334";
+	print is_valid_ip(ip);
+
+	# any case mixture is allowed
+	ip = "2001:0dB8:85a3:0000:0000:8A2E:0370:7334";
+	print is_valid_ip(ip);
+
+	# leading zeroes of a 16-bit group may be omitted
+	ip = "2001:db8:85a3:0:0:8a2e:370:7334";
+	print is_valid_ip(ip);
+
+	# a single occurrence of consecutive groups of zeroes may be replaced by ::
+	ip = "2001:db8:85a3::8a2e:370:7334";
+	print is_valid_ip(ip);
+
+	# this should fail because we don't have enough 16-bit groups
+	ip = "2001:db8:85a3:8a2e:370:7334";
+	print is_valid_ip(ip);
+
+	# this should fail because of an invalid hex digit
+	ip = "2001:gb8:85a3::8a2e:370:7334";
+	print is_valid_ip(ip);
+
+	# this should fail because we have too many 16-bit groups
+	ip = "2001:0db8:85a3:0000:0000:8a2e:0370:7334:1111";
+	print is_valid_ip(ip);
+
+	# this should fail because one group isn't 16-bits
+	ip = "2001:0db8:85a3:0000:0000:8a2e00:0370:7334";
+	print is_valid_ip(ip);
+
+	# this should fail because we can't have more than one ::
+	ip = "2001::85a3::7334";
+	print is_valid_ip(ip);
+
+	# all zeroes should work
+	ip = "0:0:0:0:0:0:0:0";
+	print is_valid_ip(ip);
+
+	# all zeroes condensed should work
+	ip = "::";
+	print is_valid_ip(ip);
+
+	print "============ test ipv6-ipv4 hybrid regexes";
+
+	# hybrid ipv6-ipv4 address should work
+	ip = "2001:db8:0:0:0:FFFF:192.168.0.5";
+	print is_valid_ip(ip);
+
+	# hybrid ipv6-ipv4 address with zero ommission should work
+	ip = "2001:db8::FFFF:192.168.0.5";
+	print is_valid_ip(ip);
+
+	# hybrid format with more than six 16-bit groups should fail
+	ip = "2001:db8:0:0:0:0:FFFF:192.168.0.5";
+	print is_valid_ip(ip);
+
+	# hybrid format without a 4 octet ipv4 part should fail
+	ip = "2001:db8:0:0:0:FFFF:192.168.0";
+	print is_valid_ip(ip);
+
+	# hybrid format's ipv4 part should test that all octet's are 0-255
+	ip = "2001:db8:0:0:0:FFFF:192.168.0.256";
+	print is_valid_ip(ip);
+
+	print "============ test find_ip_addresses()";
+	print find_ip_addresses("this is 1.1.1.1 a test 2.2.2.2 string with ip addresses 3.3.3.3");
+	print find_ip_addresses("this is 1.1.1.1 a test 0:0:0:0:0:0:0:0 string with ip addresses 3.3.3.3");
+
+	}

testing/btest/policy/utils/conn-ids.test

@@ -0,0 +1,14 @@
+# @TEST-EXEC: bro %INPUT >output
+# @TEST-EXEC: btest-diff output
+
+@load utils/conn-ids
+
+global c: conn_id = [ $orig_h = 10.0.0.100, $orig_p = 10000,
+                      $resp_h = 10.0.0.200, $resp_p = 20000 ];
+
+print id_string(c);
+print reverse_id_string(c);
+print directed_id_string(c, T);
+print directed_id_string(c, F);
+print id_string(c) == directed_id_string(c, T);
+print reverse_id_string(c) == directed_id_string(c, F);

testing/btest/policy/utils/directions-and-hosts.test

@@ -0,0 +1,72 @@
+# @TEST-EXEC: bro %INPUT >output
+# @TEST-EXEC: btest-diff output
+
+@load utils/site
+@load utils/directions-and-hosts
+
+redef Site::local_nets += { 10.0.0.0/8 };
+
+global local_ip = 10.0.0.100;
+global remote_ip = 192.168.1.100;
+
+global local2local: conn_id = [
+    $orig_h = 10.0.0.100, $orig_p = 10000,
+    $resp_h = 10.0.0.200, $resp_p = 20000 ];
+
+global local2remote: conn_id = [
+    $orig_h = 10.0.0.100,    $orig_p = 10000,
+    $resp_h = 192.168.1.100, $resp_p = 20000 ];
+
+global remote2local: conn_id = [
+    $orig_h = 192.168.1.100,    $orig_p = 10000,
+    $resp_h = 10.0.0.100,       $resp_p = 20000 ];
+
+global remote2remote: conn_id = [
+    $orig_h = 192.168.1.100, $orig_p = 10000,
+    $resp_h = 192.168.1.200, $resp_p = 20000 ];
+
+function test_host(ip: addr, h: Host, expect: bool)
+	{
+	local result = addr_matches_host(ip, h);
+	print fmt("%s(%s) == %s: %s", h, ip, expect,
+	          result == expect ? "SUCCESS" : "FAIL");
+	}
+
+function test_dir(id: conn_id, d: Direction, expect: bool)
+	{
+	local result = id_matches_direction(id, d);
+	print fmt("%s(o: %s, r: %s) == %s: %s", d, id$orig_h, id$resp_h, expect,
+	          result == expect ? "SUCCESS" : "FAIL");
+	}
+
+event bro_init()
+	{
+	test_host(local_ip,  LOCAL_HOSTS,  T);
+	test_host(local_ip,  REMOTE_HOSTS, F);
+	test_host(local_ip,  ALL_HOSTS,    T);
+	test_host(local_ip,  NO_HOSTS,     F);
+	test_host(remote_ip, LOCAL_HOSTS,  F);
+	test_host(remote_ip, REMOTE_HOSTS, T);
+	test_host(remote_ip, ALL_HOSTS,    T);
+	test_host(remote_ip, NO_HOSTS,     F);
+
+	test_dir(local2local,   INBOUND,       F);
+	test_dir(local2remote,  INBOUND,       F);
+	test_dir(remote2local,  INBOUND,       T);
+	test_dir(remote2remote, INBOUND,       F);
+
+	test_dir(local2local,   OUTBOUND,      F);
+	test_dir(local2remote,  OUTBOUND,      T);
+	test_dir(remote2local,  OUTBOUND,      F);
+	test_dir(remote2remote, OUTBOUND,      F);
+
+	test_dir(local2local,   BIDIRECTIONAL, F);
+	test_dir(local2remote,  BIDIRECTIONAL, T);
+	test_dir(remote2local,  BIDIRECTIONAL, T);
+	test_dir(remote2remote, BIDIRECTIONAL, F);
+
+	test_dir(local2local,   NO_DIRECTION,  F);
+	test_dir(local2remote,  NO_DIRECTION,  F);
+	test_dir(remote2local,  NO_DIRECTION,  F);
+	test_dir(remote2remote, NO_DIRECTION,  F);
+	}

testing/btest/policy/utils/files.test

@@ -0,0 +1,12 @@
+# @TEST-EXEC: bro -r $TRACES/wikipedia.trace %INPUT >output
+# @TEST-EXEC: btest-diff output
+
+@load utils/files
+
+event connection_established(c: connection)
+	{
+	print generate_extraction_filename("test-prefix", c, "test-suffix");
+	print generate_extraction_filename("test-prefix", c, "");
+	print generate_extraction_filename("", c, "test-suffix");
+	print generate_extraction_filename("", c, "");
+	}

testing/btest/policy/utils/numbers.test

@@ -0,0 +1,12 @@
+# @TEST-EXEC: bro %INPUT >output
+# @TEST-EXEC: btest-diff output
+
+@load utils/numbers
+
+print extract_count("These aren't the numbers you're looking for.");
+print extract_count("13These aren't the numbers you're looking for.");
+print extract_count("13 These aren't the numbers you're looking for.");
+print extract_count("These aren't the 13 numbers you're looking for.");
+print extract_count("These aren't the numbers you're looking for.13");
+print extract_count("These aren't the numbers you're looking for. 13");
+print extract_count("These aren't the 1abc3 numbers you're looking for.");

testing/btest/policy/utils/paths.test

@@ -0,0 +1,57 @@
+# @TEST-EXEC: bro %INPUT >output
+# @TEST-EXEC: btest-diff output
+
+@load utils/paths
+
+function test_extract(str: string, expect: string)
+	{
+	local result = extract_path(str);
+	print fmt("Given : %s", str);
+	print fmt("Expect: %s", expect);
+	print fmt("Result: %s", result);
+	print fmt("Result: %s", result == expect ? "SUCCESS" : "FAIL");
+	print "===============================";
+	}
+
+function test_compress(str: string, expect: string)
+	{
+	local result = compress_path(str);
+	print fmt("Given : %s", str);
+	print fmt("Expect: %s", expect);
+	print fmt("Result: %s", result);
+	print fmt("Result: %s", result == expect ? "SUCCESS" : "FAIL");
+	print "===============================";
+	}
+
+print "test compress_path()";
+print "===============================";
+test_compress("foo//bar", "foo/bar");
+test_compress("foo//bar/..", "foo");
+test_compress("foo/bar/../..", "");
+test_compress("foo//bar/../..", "");
+test_compress("/foo/../bar", "/bar");
+test_compress("/foo/../bar/..", "/");
+test_compress("/foo/baz/../..", "/");
+test_compress("../..", "../..");
+test_compress("foo/../../..", "../..");
+
+print "test extract_path()";
+print "===============================";
+test_extract("\"/this/is/a/dir\" is current directory", "/this/is/a/dir");
+test_extract("/this/is/a/dir is current directory", "/this/is/a/dir");
+test_extract("/this/is/a/dir\\ is\\ current\\ directory", "/this/is/a/dir\\ is\\ current\\ directory");
+test_extract("hey, /foo/bar/baz.bro is a cool script", "/foo/bar/baz.bro");
+test_extract("here's two dirs: /foo/bar and /foo/baz", "/foo/bar");
+
+print "test build_path_compressed()";
+print "===============================";
+print build_path_compressed("/home/bro/", "policy/somefile.bro");
+print build_path_compressed("/home/bro/", "/usr/local/bro/share/bro/somefile.bro");
+print build_path_compressed("/home/bro/", "/usr/local/bro/share/../../bro/somefile.bro");
+
+print "===============================";
+print "test build_full_path()";
+print "===============================";
+print build_path("/home/bro/", "policy/somefile.bro");
+print build_path("/home/bro/", "/usr/local/bro/share/bro/somefile.bro");
+

testing/btest/policy/utils/pattern.test

@@ -0,0 +1,16 @@
+# @TEST-EXEC: bro %INPUT >output
+# @TEST-EXEC: btest-diff output
+
+@load utils/pattern
+
+global r1 = set_to_regex(set("blah", "bleh", "blarg"), "(~~)");
+global r2 = set_to_regex(set("blah", "bleh", "blarg"), "foo(~~)bar");
+
+print r1;
+print "blah" == r1;
+
+print r2;
+print "fooblargbar" == r2;
+
+print match_pattern("123blah123", r1);
+print match_pattern("no match here", r1);

testing/btest/policy/utils/site.test

@@ -0,0 +1,18 @@
+# @TEST-EXEC: bro %INPUT > output
+# @TEST-EXEC: btest-diff output
+
+@load utils/site
+
+global a = { "site-admin@example.com", "other-site-admin@example.com" };
+global b = { "net-admin@example.com" };
+
+redef Site::local_admins += {
+    [141.142.0.0/16] = a,
+    [141.142.100.0/24] = b,
+};
+
+event bro_init()
+    {
+    print Site::get_emails(141.142.1.1);
+    print Site::get_emails(141.142.100.100);
+    }

testing/btest/policy/utils/strings.test

@@ -0,0 +1,29 @@
+# @TEST-EXEC: bro %INPUT >output
+# @TEST-EXEC: btest-diff output
+
+@load utils/strings
+
+function test_binary_string(s: string)
+	{
+	if ( is_string_binary(s) )
+		print fmt("'%s' IS considered binary", s);
+	else
+		print fmt("'%s' is NOT considered binary", s);
+	}
+
+test_binary_string("\x68\x65\x6C\x6C\x6F");
+test_binary_string("\xFF\xFF\xFF\x00");
+test_binary_string("\x00\x00\xFF\x00");
+test_binary_string("\x00\x00\x00\x00");
+
+print join_string_set(set("one", "two", "three"), ", ");
+print join_string_set(set("one"), ", ");
+
+print string_escape("hello world", "od");
+print string_escape("\\hello world\\", "");
+
+print cut_tail("hello world", 0);
+print cut_tail("hello world", 1);
+print cut_tail("hello world", 6);
+print cut_tail("hello world", 11);
+print cut_tail("hello world", 12);

testing/btest/policy/utils/thresholds.test

@@ -0,0 +1,28 @@
+# @TEST-EXEC: bro %INPUT >output
+# @TEST-EXEC: btest-diff output
+
+@load utils/thresholds
+
+redef default_notice_thresholds = { 2, 4, 6, 8, 10 };
+const my_thresholds: vector of count = { 2, 4, 6, 8, 10 };
+const loop_v: vector of count = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 };
+global track_count: TrackCount;
+
+for ( i in loop_v )
+	{
+	print fmt("Iteration: %s, threshold check: %s", i,
+	          check_threshold(my_thresholds, track_count));
+	print track_count;
+	++track_count$n;
+	}
+
+track_count$n = 0; track_count$index = 0;
+
+print "====================================";
+for ( i in loop_v )
+	{
+	print fmt("Iteration: %s, threshold check: %s", i,
+	          default_check_threshold(track_count));
+	print track_count;
+	++track_count$n;
+	}