Merge branch 'master', remote-tracking branch 'origin/fastpath'

* origin/fastpath:
  Fix PktSrc setting next_timestamp even when no packet available
  Fix lack of NUL-termination in to_upper/to_lower BIF's return val.
  Fixing unit tests and some minor bugs.
  Fix broctl cluster log rotation (addresses #619)

* master:

doc/scripts/DocSourcesList.cmake

@@ -102,7 +102,6 @@ rest_target(${psd} policy/integration/barnyard2/main.bro)
 rest_target(${psd} policy/integration/barnyard2/types.bro)
 rest_target(${psd} policy/misc/analysis-groups.bro)
 rest_target(${psd} policy/misc/loaded-scripts.bro)
-rest_target(${psd} policy/misc/pf-ring-load-balancing.bro)
 rest_target(${psd} policy/misc/profiling.bro)
 rest_target(${psd} policy/misc/trim-trace-file.bro)
 rest_target(${psd} policy/protocols/conn/known-hosts.bro)
@@ -115,7 +114,7 @@ rest_target(${psd} policy/protocols/http/detect-MHR.bro)
 rest_target(${psd} policy/protocols/http/detect-intel.bro)
 rest_target(${psd} policy/protocols/http/detect-sqli.bro)
 rest_target(${psd} policy/protocols/http/detect-webapps.bro)
-rest_target(${psd} policy/protocols/http/headers.bro)
+rest_target(${psd} policy/protocols/http/header-names.bro)
 rest_target(${psd} policy/protocols/http/software.bro)
 rest_target(${psd} policy/protocols/http/var-extraction-cookies.bro)
 rest_target(${psd} policy/protocols/http/var-extraction-uri.bro)

scripts/base/frameworks/logging/main.bro

@@ -192,10 +192,7 @@ function default_path_func(id: ID, path: string, rec: any) : string
 				output = cat(output, "_", module_parts[3]);
 			if ( 4 in module_parts && module_parts[4] != "" )
 				output = cat(output, sub_bytes(module_parts[4],1,1), "_", sub_bytes(module_parts[4], 2, |module_parts[4]|));
-			# TODO: There seems to be some problem with the split function 
-			#       not putting \0 at the end of the string.  fmt will make
-			#       a better internal string.
-			return fmt("%s", to_lower(output));
+			return to_lower(output);
 			}
 		
 		# Example: Notice::POLICY_LOG -> "notice_policy"
@@ -205,10 +202,7 @@ function default_path_func(id: ID, path: string, rec: any) : string
 		return cat(to_lower(parts[1]),"_",to_lower(parts[2]));
 		}
 	else
-		{
-		# In case there is a logging stream in the global namespace.
 		return to_lower(id_str);
-		}
 	}
 
 # Run post-processor on file. If there isn't any postprocessor defined,
@@ -241,8 +235,7 @@ function create_stream(id: ID, stream: Stream) : bool
 
 function disable_stream(id: ID) : bool
 	{
-	if ( ! __disable_stream(id) )
-		return F;
+	return __disable_stream(id);
 	}
 
 function add_filter(id: ID, filter: Filter) : bool

scripts/policy/protocols/http/header-names.bro

@@ -14,7 +14,7 @@ export {
 		
 		## The vector of HTTP header names sent by the server.  No header 
 		## values are included here, just the header names.
-		server_headers_names:  vector of string &log &optional;
+		server_header_names:  vector of string &log &optional;
 	};
 	
 	## A boolean value to determine if client header names are to be logged.

scripts/policy/protocols/ssh/detect-bruteforcing.bro

@@ -1,4 +1,5 @@
 
+@load base/protocols/ssh
 @load base/frameworks/metrics
 @load base/frameworks/notice
 @load base/frameworks/intel
@@ -72,4 +73,4 @@ event SSH::heuristic_failed_login(c: connection)
 	if ( ! (id$orig_h in ignore_guessers &&
 	        id$resp_h in ignore_guessers[id$orig_h]) )
 		Metrics::add_data(FAILED_LOGIN, [$host=id$orig_h], 1);
-	}
+	}

scripts/test-all-policy.bro

@@ -25,6 +25,7 @@
 @load integration/barnyard2/types.bro
 @load misc/analysis-groups.bro
 @load misc/loaded-scripts.bro
+@load misc/profiling.bro
 @load misc/trim-trace-file.bro
 @load protocols/conn/known-hosts.bro
 @load protocols/conn/known-services.bro
@@ -37,11 +38,16 @@
 @load protocols/http/detect-MHR.bro
 @load protocols/http/detect-sqli.bro
 @load protocols/http/detect-webapps.bro
+@load protocols/http/header-names.bro
 @load protocols/http/software.bro
 @load protocols/http/var-extraction-cookies.bro
 @load protocols/http/var-extraction-uri.bro
+@load protocols/smtp/blocklists.bro
 @load protocols/smtp/detect-suspicious-orig.bro
 @load protocols/smtp/software.bro
+@load protocols/ssh/detect-bruteforcing.bro
+@load protocols/ssh/geo-data.bro
+@load protocols/ssh/interesting-hostnames.bro
 @load protocols/ssh/software.bro
 @load protocols/ssl/known-certs.bro
 @load protocols/ssl/validate-certs.bro

src/Func.cc

@@ -334,7 +334,7 @@ Val* BroFunc::Call(val_list* args, Frame* parent) const
 	     (flow != FLOW_RETURN /* we fell off the end */ ||
 	      ! result /* explicit return with no result */) &&
 	     ! f->HasDelayed() )
-		reporter->Warning("non-void function returns without a value:", id->Name());
+		reporter->Warning("non-void function returns without a value: %s", id->Name());
 
 	if ( result && g_trace_state.DoTrace() )
 		{

src/LogMgr.cc

@@ -1266,20 +1266,31 @@ LogWriter* LogMgr::CreateWriter(EnumVal* id, EnumVal* writer, string path,
 	winfo->postprocessor = 0;
 
 	// Search for a corresponding filter for the writer/path pair and use its
-	// rotation settings.
+	// rotation settings.  If no matching filter is found, fall back on
+	// looking up the logging framework's default rotation interval.
+	bool found_filter_match = false;
 	list<Filter*>::const_iterator it;
+
 	for ( it = stream->filters.begin(); it != stream->filters.end(); ++it )
 		{
 		Filter* f = *it;
 		if ( f->writer->AsEnum() == writer->AsEnum() &&
 		     f->path == winfo->writer->Path() )
 			{
+			found_filter_match = true;
 			winfo->interval = f->interval;
 			winfo->postprocessor = f->postprocessor;
 			break;
 			}
 		}
 
+	if ( ! found_filter_match )
+		{
+		ID* id = global_scope()->Lookup("Log::default_rotation_interval");
+		assert(id);
+		winfo->interval = id->ID_Val()->AsInterval();
+		}
+
 	InstallRotationTimer(winfo);
 
 	stream->writers.insert(

src/PktSrc.cc

@@ -21,6 +21,7 @@ PktSrc::PktSrc()
 	{
 	interface = readfile = 0;
 	data = last_data = 0;
+	memset(&hdr, 0, sizeof(hdr));
 	hdr_size = 0;
 	datalink = 0;
 	netmask = 0xffffff00;
@@ -75,7 +76,9 @@ int PktSrc::ExtractNextPacket()
 		}
 
 	data = last_data = pcap_next(pd, &hdr);
-	next_timestamp = hdr.ts.tv_sec + double(hdr.ts.tv_usec) / 1e6;
+
+	if ( data )
+		next_timestamp = hdr.ts.tv_sec + double(hdr.ts.tv_usec) / 1e6;
 
 	if ( pseudo_realtime )
 		current_wallclock = current_time(true);

src/strings.bif

@@ -493,7 +493,7 @@ function to_lower%(str: string%): string
 			*ls++ = s[i];
 		}
 
-	return new StringVal(new BroString(1, byte_vec(lower_s), n));
+	return new StringVal(n, lower_s);
 	%}
 
 function to_upper%(str: string%): string
@@ -511,7 +511,7 @@ function to_upper%(str: string%): string
 			*us++ = s[i];
 		}
 
-	return new StringVal(new BroString(1, byte_vec(upper_s), n));
+	return new StringVal(n, upper_s);
 	%}
 
 function clean%(str: string%): string

testing/btest/Baseline/scripts.policy.misc.bare-loaded-scripts/canonified_loaded_scripts.log

@@ -1,4 +1,7 @@
-# depth	name
+#separator \x09
+#path	loaded_scripts
+#fields	depth	name
+#types	count	string
 0	scripts/base/init-bare.bro
 1	build/src/base/const.bif.bro
 1	build/src/base/types.bif.bro

testing/btest/Baseline/scripts.policy.misc.default-loaded-scripts/canonified_loaded_scripts.log

@@ -1,4 +1,7 @@
-# depth	name
+#separator \x09
+#path	loaded_scripts
+#fields	depth	name
+#types	count	string
 0	scripts/base/init-bare.bro
 1	build/src/base/const.bif.bro
 1	build/src/base/types.bif.bro

testing/btest/doc/blacklist-reminder.test

@@ -1,8 +0,0 @@
-# This test will fail if there are Bro scripts that have been temporarily
-# blacklisted from the documentation generation process for some reason
-# (e.g. they're a work-in-progress or otherwise fail to parse).  It's meant
-# to serve as a reminder that some future action may be needed to generate
-# documentation for the blacklisted scripts.
-#
-# @TEST-EXEC: $DIST/doc/scripts/genDocSourcesList.sh
-# @TEST-EXEC: btest-diff .stderr

testing/btest/scripts/base/init-default-coverage.bro

@@ -5,7 +5,7 @@
 #@TEST-EXEC: test -e $DIST/scripts/base/init-default.bro
 #@TEST-EXEC: ( cd $DIST/scripts/base && find . -name '*.bro' ) | sort >"all scripts found"
 #@TEST-EXEC: bro misc/loaded-scripts
-#@TEST-EXEC: cat loaded_scripts.log | egrep -v '/build/|/loaded-scripts.bro' | awk 'NR>1{print $2}' | sed 's#/./#/#g'  >loaded_scripts.log.tmp
+#@TEST-EXEC: cat loaded_scripts.log | egrep -v '/build/|/loaded-scripts.bro|#' | awk 'NR>1{print $2}' | sed 's#/./#/#g'  >loaded_scripts.log.tmp
 #@TEST-EXEC: cat loaded_scripts.log.tmp | sed -e ':a' -e '$!N' -e 's/^\(.*\).*\n\1.*/\1/' -e 'ta' >prefix
 #@TEST-EXEC: cat loaded_scripts.log.tmp | sed "s#`cat prefix`#./#g" | sort >init-default.bro
 #@TEST-EXEC: diff -u "all scripts found" init-default.bro 1>&2

testing/btest/scripts/policy/misc/bare-loaded-scripts.test

@@ -7,6 +7,6 @@
 
 # @TEST-EXEC: bro -b misc/loaded-scripts
 # @TEST-EXEC: test -e loaded_scripts.log
-# @TEST-EXEC: cat loaded_scripts.log | awk 'NR>1{print $2}' | sed -e ':a' -e '$!N' -e 's/^\(.*\).*\n\1.*/\1/' -e 'ta' >prefix
+# @TEST-EXEC: cat loaded_scripts.log | egrep -v '#' | awk 'NR>1{print $2}' | sed -e ':a' -e '$!N' -e 's/^\(.*\).*\n\1.*/\1/' -e 'ta' >prefix
 # @TEST-EXEC: cat loaded_scripts.log | sed "s#`cat prefix`##g" >canonified_loaded_scripts.log
 # @TEST-EXEC: btest-diff canonified_loaded_scripts.log

testing/btest/scripts/policy/misc/default-loaded-scripts.test

@@ -7,6 +7,6 @@
 
 # @TEST-EXEC: bro misc/loaded-scripts
 # @TEST-EXEC: test -e loaded_scripts.log
-# @TEST-EXEC: cat loaded_scripts.log | awk 'NR>1{print $2}' | sed -e ':a' -e '$!N' -e 's/^\(.*\).*\n\1.*/\1/' -e 'ta' >prefix
+# @TEST-EXEC: cat loaded_scripts.log | egrep -v '#' | awk 'NR>1{print $2}' | sed -e ':a' -e '$!N' -e 's/^\(.*\).*\n\1.*/\1/' -e 'ta' >prefix
 # @TEST-EXEC: cat loaded_scripts.log | sed "s#`cat prefix`##g" >canonified_loaded_scripts.log
 # @TEST-EXEC: btest-diff canonified_loaded_scripts.log