Merge remote-tracking branch 'origin/fastpath' into fastpath

CHANGES

@@ -1,4 +1,58 @@
 
+2.1-beta-28 | 2012-08-21 08:32:42 -0700
+
+  * Linking ES docs into logging document. (Robin Sommer)
+
+2.1-beta-27 | 2012-08-20 20:06:20 -0700
+
+  * Add the Stream record to Log:active_streams to make more dynamic
+    logging possible. (Seth Hall)
+
+  * Fix portability of printing to files returned by
+    open("/dev/stderr"). (Jon Siwek)
+
+  * Fix mime type diff canonifier to also skip mime_desc columns. (Jon
+    Siwek)
+
+  * Unit test tweaks/fixes. (Jon Siwek)
+
+    - Some baselines for tests in "leaks" group were outdated.
+
+    - Changed a few of the cluster/communication tests to terminate
+      more explicitly instead of relying on btest-bg-wait to kill
+      processes.  This makes the tests finish faster in the success case
+      and makes the reason for failing clearer in the that case.
+
+  * Fix memory leak of serialized IDs when compiled with
+    --enable-debug. (Jon Siwek)
+
+2.1-beta-21 | 2012-08-16 11:48:56 -0700
+
+  * Installing a handler for running out of memory in "new". Bro will
+    now print an error message in that case rather than abort with an
+    uncaught exception. (Robin Sommer)
+
+2.1-beta-20 | 2012-08-16 11:43:31 -0700
+
+  * Fixed potential problems with ElasticSearch output plugin. (Seth
+    Hall)
+
+2.1-beta-13 | 2012-08-10 12:28:04 -0700
+
+  * Reporter warnings and error now print to stderr by default. New
+	options Reporter::warnings_to_stderr and
+	Reporter::errors_to_stderr to disable. (Seth Hall)
+
+2.1-beta-9 | 2012-08-10 12:24:29 -0700
+
+  * Add more BIF tests. (Daniel Thayer)
+
+2.1-beta-6 | 2012-08-10 12:22:52 -0700
+
+  * Fix bug in input framework with an edge case. (Bernhard Amann)
+    
+  * Fix small bug in input framework test script. (Bernhard Amann)
+    
 2.1-beta-3 | 2012-08-03 10:46:49 -0700
 
   * Merge branch 'master' of ssh://git.bro-ids.org/bro (Robin Sommer)

VERSION

@@ -1 +1 @@
-2.0-914
+2.1-beta-28

aux/broctl

@@ -1 +1 @@
-Subproject commit 903108f6b43ad228309713da880026d50add41f4
+Subproject commit 6d0eb6083acdc77e0a912bec0fb23df79b98da63

doc/logging.rst

@@ -383,3 +383,4 @@ Bro supports the following output formats other than ASCII:
    :maxdepth: 1
 
    logging-dataseries
+   logging-elasticsearch

scripts/base/frameworks/logging/main.bro

@@ -329,9 +329,9 @@ export {
 	global run_rotation_postprocessor_cmd: function(info: RotationInfo, npath: string) : bool;
 
 	## The streams which are currently active and not disabled.
-	## This set is not meant to be modified by users!  Only use it for
+	## This table is not meant to be modified by users!  Only use it for
 	## examining which streams are active.
-	global active_streams: set[ID] = set();
+	global active_streams: table[ID] of Stream = table();
 }
 
 # We keep a script-level copy of all filters so that we can manipulate them.
@@ -417,7 +417,7 @@ function create_stream(id: ID, stream: Stream) : bool
 	if ( ! __create_stream(id, stream) )
 		return F;
 
-	add active_streams[id];
+	active_streams[id] = stream;
 
 	return add_default_filter(id);
 	}

scripts/base/frameworks/reporter/main.bro

@@ -36,24 +36,55 @@ export {
 		## Not all reporter messages will have locations in them though.
 		location: string &log &optional;
 	};
+
+	## Tunable for sending reporter warning messages to STDERR.  The option to
+	## turn it off is presented here in case Bro is being run by some
+	## external harness and shouldn't output anything to the console.
+	const warnings_to_stderr = T &redef;
+
+	## Tunable for sending reporter error messages to STDERR.  The option to
+	## turn it off is presented here in case Bro is being run by some
+	## external harness and shouldn't output anything to the console.
+	const errors_to_stderr = T &redef;
 }
 
+global stderr: file;
+
 event bro_init() &priority=5
 	{
 	Log::create_stream(Reporter::LOG, [$columns=Info]);
+
+	if ( errors_to_stderr || warnings_to_stderr )
+		stderr = open("/dev/stderr");
 	}
 
-event reporter_info(t: time, msg: string, location: string)
+event reporter_info(t: time, msg: string, location: string) &priority=-5
 	{
 	Log::write(Reporter::LOG, [$ts=t, $level=INFO, $message=msg, $location=location]);
 	}
 
-event reporter_warning(t: time, msg: string, location: string)
+event reporter_warning(t: time, msg: string, location: string) &priority=-5
 	{
+	if ( warnings_to_stderr )
+		{
+		if ( t > double_to_time(0.0) )
+			print stderr, fmt("WARNING: %.6f %s (%s)", t, msg, location);
+		else
+			print stderr, fmt("WARNING: %s (%s)", msg, location);
+		}
+
 	Log::write(Reporter::LOG, [$ts=t, $level=WARNING, $message=msg, $location=location]);
 	}
 
-event reporter_error(t: time, msg: string, location: string)
+event reporter_error(t: time, msg: string, location: string) &priority=-5
 	{
+	if ( errors_to_stderr )
+		{
+		if ( t > double_to_time(0.0) )
+			print stderr, fmt("ERROR: %.6f %s (%s)", t, msg, location);
+		else
+			print stderr, fmt("ERROR: %s (%s)", msg, location);
+		}
+
 	Log::write(Reporter::LOG, [$ts=t, $level=ERROR, $message=msg, $location=location]);
 	}

scripts/policy/tuning/logs-to-elasticsearch.bro

@@ -8,13 +8,13 @@ export {
 
 	## Optionally ignore any :bro:type:`Log::ID` from being sent to
 	## ElasticSearch with this script.
-	const excluded_log_ids: set[string] = set("Communication::LOG") &redef;
+	const excluded_log_ids: set[Log::ID] &redef;
 
 	## If you want to explicitly only send certain :bro:type:`Log::ID` 
 	## streams, add them to this set.  If the set remains empty, all will 
 	## be sent.  The :bro:id:`LogElasticSearch::excluded_log_ids` option will remain in 
 	## effect as well.
-	const send_logs: set[string] = set() &redef;
+	const send_logs: set[Log::ID] &redef;
 }
 
 event bro_init() &priority=-5
@@ -24,8 +24,8 @@ event bro_init() &priority=-5
 	
 	for ( stream_id in Log::active_streams )
 		{
-		if ( fmt("%s", stream_id) in excluded_log_ids ||
+		if ( stream_id in excluded_log_ids ||
-		     (|send_logs| > 0 && fmt("%s", stream_id) !in send_logs) )
+		     (|send_logs| > 0 && stream_id !in send_logs) )
 			next;
 
 		local filter: Log::Filter = [$name = "default-es",

src/File.cc

@@ -138,11 +138,22 @@ BroFile::BroFile(FILE* arg_f, const char* arg_name, const char* arg_access)
 BroFile::BroFile(const char* arg_name, const char* arg_access, BroType* arg_t)
 	{
 	Init();
-
+	f = 0;
 	name = copy_string(arg_name);
 	access = copy_string(arg_access);
 	t = arg_t ? arg_t : base_type(TYPE_STRING);
-	if ( ! Open() )
+
+	if ( streq(name, "/dev/stdin") )
+		f = stdin;
+	else if ( streq(name, "/dev/stdout") )
+		f = stdout;
+	else if ( streq(name, "/dev/stderr") )
+		f = stderr;
+
+	if ( f )
+		is_open = 1;
+
+	else if ( ! Open() )
 		{
 		reporter->Error("cannot open %s: %s", name, strerror(errno));
 		is_open = 0;
@@ -342,8 +353,8 @@ int BroFile::Close()
 
 	FinishEncrypt();
 
-	// Do not close stdout/stderr.
+	// Do not close stdin/stdout/stderr.
-	if ( f == stdout || f == stderr )
+	if ( f == stdin || f == stdout || f == stderr )
 		return 0;
 
 	if ( is_in_cache )
@@ -523,6 +534,10 @@ RecordVal* BroFile::Rotate()
 	if ( ! is_open )
 		return 0;
 
+	// Do not rotate stdin/stdout/stderr.
+	if ( f == stdin || f == stdout || f == stderr )
+		return 0;
+
 	if ( okay_to_manage && ! is_in_cache )
 		BringIntoCache();
 

src/RemoteSerializer.cc

@@ -2897,11 +2897,6 @@ void RemoteSerializer::GotID(ID* id, Val* val)
 					(desc && *desc) ? desc : "not set"),
 			current_peer);
 
-#ifdef USE_PERFTOOLS_DEBUG
-		// May still be cached, but we don't care.
-		heap_checker->IgnoreObject(id);
-#endif
-
 		Unref(id);
 		return;
 		}

src/Val.cc

@@ -64,7 +64,7 @@ Val::~Val()
 
 	Unref(type);
 #ifdef DEBUG
-	Unref(bound_id);
+	delete [] bound_id;
 #endif
 	}
 

src/Val.h

@@ -347,13 +347,15 @@ public:
 #ifdef DEBUG
 	// For debugging, we keep a reference to the global ID to which a
 	// value has been bound *last*.
-	ID* GetID() const	{ return bound_id; }
+	ID* GetID() const
+		{
+		return bound_id ? global_scope()->Lookup(bound_id) : 0;
+		}
+
 	void SetID(ID* id)
 		{
-		if ( bound_id )
+		delete [] bound_id;
-			::Unref(bound_id);
+		bound_id = id ? copy_string(id->Name()) : 0;
-		bound_id = id;
-		::Ref(bound_id);
 		}
 #endif
 
@@ -401,8 +403,8 @@ protected:
 	RecordVal* attribs;
 
 #ifdef DEBUG
-	// For debugging, we keep the ID to which a Val is bound.
+	// For debugging, we keep the name of the ID to which a Val is bound.
-	ID* bound_id;
+	const char* bound_id;
 #endif
 
 };

src/bro.bif

@@ -3787,7 +3787,7 @@ static GeoIP* open_geoip_db(GeoIPDBTypes type)
 		geoip = GeoIP_open_type(type, GEOIP_MEMORY_CACHE);
 
 	if ( ! geoip )
-		reporter->Warning("Failed to open GeoIP database: %s",
+		reporter->Info("Failed to open GeoIP database: %s",
 		                  GeoIPDBFileName[type]);
 	return geoip;
 	}
@@ -3827,7 +3827,7 @@ function lookup_location%(a: addr%) : geo_location
 			if ( ! geoip )
 				builtin_error("Can't initialize GeoIP City/Country database");
 			else
-				reporter->Warning("Fell back to GeoIP Country database");
+				reporter->Info("Fell back to GeoIP Country database");
 			}
 		else
 			have_city_db = true;

src/logging/writers/DataSeries.cc

@@ -243,8 +243,25 @@ bool DataSeries::OpenLog(string path)
 	log_file->writeExtentLibrary(log_types);
 
 	for( size_t i = 0; i < schema_list.size(); ++i )
-		extents.insert(std::make_pair(schema_list[i].field_name,
+		{
-					      GeneralField::create(log_series, schema_list[i].field_name)));
+		string fn = schema_list[i].field_name;
+		GeneralField* gf = 0;
+#ifdef USE_PERFTOOLS_DEBUG
+		{
+		// GeneralField isn't cleaning up some results of xml parsing, reported
+		// here: https://github.com/dataseries/DataSeries/issues/1
+		// Ignore for now to make leak tests pass.  There's confidence that
+		// we do clean up the GeneralField* since the ExtentSeries dtor for
+		// member log_series would trigger an assert if dynamically allocated
+		// fields aren't deleted beforehand.
+		HeapLeakChecker::Disabler disabler;
+#endif
+		gf = GeneralField::create(log_series, fn);
+#ifdef USE_PERFTOOLS_DEBUG
+		}
+#endif
+		extents.insert(std::make_pair(fn, gf));
+		}
 
 	if ( ds_extent_size < ROW_MIN )
 		{

src/logging/writers/ElasticSearch.cc

@@ -371,7 +371,11 @@ bool ElasticSearch::HTTPSend(CURL *handle)
 	// The best (only?) way to disable that is to just use HTTP 1.0
 	curl_easy_setopt(handle, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_0);
 
-	//curl_easy_setopt(handle, CURLOPT_TIMEOUT_MS, transfer_timeout);
+	// Some timeout options.  These will need more attention later.
+	curl_easy_setopt(handle, CURLOPT_NOSIGNAL, 1);
+	curl_easy_setopt(handle, CURLOPT_CONNECTTIMEOUT_MS, transfer_timeout);
+	curl_easy_setopt(handle, CURLOPT_TIMEOUT_MS, transfer_timeout*2);
+	curl_easy_setopt(handle, CURLOPT_DNS_CACHE_TIMEOUT, 60*60);
 
 	CURLcode return_code = curl_easy_perform(handle);
 

src/main.cc

@@ -337,6 +337,8 @@ void terminate_bro()
 	delete log_mgr;
 	delete thread_mgr;
 	delete reporter;
+
+	reporter = 0;
 	}
 
 void termination_signal()
@@ -380,6 +382,8 @@ static void bro_new_handler()
 
 int main(int argc, char** argv)
 	{
+	std::set_new_handler(bro_new_handler);
+
 	brofiler.ReadStats();
 
 	bro_argc = argc;

src/util.cc

@@ -1383,7 +1383,13 @@ void safe_close(int fd)
 
 void out_of_memory(const char* where)
 	{
-	reporter->FatalError("out of memory in %s.\n", where);
+	fprintf(stderr, "out of memory in %s.\n", where);
+
+	if ( reporter )
+		// Guess that might fail here if memory is really tight ...
+		reporter->FatalError("out of memory in %s.\n", where);
+
+	abort();
 	}
 
 void get_memory_usage(unsigned int* total, unsigned int* malloced)

testing/btest/Baseline/bifs.analyzer_name/out

@@ -0,0 +1 @@
+PIA_TCP

testing/btest/Baseline/bifs.capture_state_updates/out

@@ -0,0 +1 @@
+T

testing/btest/Baseline/bifs.entropy_test/out

@@ -0,0 +1,2 @@
+[entropy=4.715374, chi_square=591.981818, mean=75.472727, monte_carlo_pi=4.0, serial_correlation=-0.11027]
+[entropy=2.083189, chi_square=3906.018182, mean=69.054545, monte_carlo_pi=4.0, serial_correlation=0.849402]

testing/btest/Baseline/bifs.global_sizes/out

@@ -0,0 +1 @@
+found bro_init

testing/btest/Baseline/bifs.identify_data/out

@@ -0,0 +1,4 @@
+ASCII text, with no line terminators
+text/plain; charset=us-ascii
+PNG image data
+image/png; charset=binary

testing/btest/Baseline/bifs.is_local_interface/out

@@ -0,0 +1,4 @@
+T
+F
+F
+T

testing/btest/Baseline/bifs.reading_traces/out1

@@ -0,0 +1 @@
+F

testing/btest/Baseline/bifs.reading_traces/out2

@@ -0,0 +1 @@
+T

testing/btest/Baseline/bifs.strftime/out

@@ -0,0 +1,4 @@
+1970-01-01 00:00:00
+000000 19700101
+1973-11-29 21:33:09
+213309 19731129

testing/btest/Baseline/core.leaks.basic-cluster/manager-1.metrics.log

@@ -3,8 +3,10 @@
 #empty_field	(empty)
 #unset_field	-
 #path	metrics
+#open	2012-07-20-01-50-41
 #fields	ts	metric_id	filter_name	index.host	index.str	index.network	value
 #types	time	enum	string	addr	string	subnet	count
-1331256494.591966	TEST_METRIC	foo-bar	6.5.4.3	-	-	4
+1342749041.601712	TEST_METRIC	foo-bar	6.5.4.3	-	-	4
-1331256494.591966	TEST_METRIC	foo-bar	7.2.1.5	-	-	2
+1342749041.601712	TEST_METRIC	foo-bar	7.2.1.5	-	-	2
-1331256494.591966	TEST_METRIC	foo-bar	1.2.3.4	-	-	6
+1342749041.601712	TEST_METRIC	foo-bar	1.2.3.4	-	-	6
+#close	2012-07-20-01-50-49

testing/btest/Baseline/core.leaks.remote/sender.test.failure.log

@@ -3,8 +3,10 @@
 #empty_field	(empty)
 #unset_field	-
 #path	test.failure
+#open	2012-07-20-01-50-18
 #fields	t	id.orig_h	id.orig_p	id.resp_h	id.resp_p	status	country
 #types	time	addr	port	addr	port	string	string
-1331256472.375609	1.2.3.4	1234	2.3.4.5	80	failure	US
+1342749018.970682	1.2.3.4	1234	2.3.4.5	80	failure	US
-1331256472.375609	1.2.3.4	1234	2.3.4.5	80	failure	UK
+1342749018.970682	1.2.3.4	1234	2.3.4.5	80	failure	UK
-1331256472.375609	1.2.3.4	1234	2.3.4.5	80	failure	MX
+1342749018.970682	1.2.3.4	1234	2.3.4.5	80	failure	MX
+#close	2012-07-20-01-50-18

testing/btest/Baseline/core.leaks.remote/sender.test.log

@@ -3,10 +3,12 @@
 #empty_field	(empty)
 #unset_field	-
 #path	test
+#open	2012-07-20-01-50-18
 #fields	t	id.orig_h	id.orig_p	id.resp_h	id.resp_p	status	country
 #types	time	addr	port	addr	port	string	string
-1331256472.375609	1.2.3.4	1234	2.3.4.5	80	success	unknown
+1342749018.970682	1.2.3.4	1234	2.3.4.5	80	success	unknown
-1331256472.375609	1.2.3.4	1234	2.3.4.5	80	failure	US
+1342749018.970682	1.2.3.4	1234	2.3.4.5	80	failure	US
-1331256472.375609	1.2.3.4	1234	2.3.4.5	80	failure	UK
+1342749018.970682	1.2.3.4	1234	2.3.4.5	80	failure	UK
-1331256472.375609	1.2.3.4	1234	2.3.4.5	80	success	BR
+1342749018.970682	1.2.3.4	1234	2.3.4.5	80	success	BR
-1331256472.375609	1.2.3.4	1234	2.3.4.5	80	failure	MX
+1342749018.970682	1.2.3.4	1234	2.3.4.5	80	failure	MX
+#close	2012-07-20-01-50-18

testing/btest/Baseline/core.leaks.remote/sender.test.success.log

@@ -3,7 +3,9 @@
 #empty_field	(empty)
 #unset_field	-
 #path	test.success
+#open	2012-07-20-01-50-18
 #fields	t	id.orig_h	id.orig_p	id.resp_h	id.resp_p	status	country
 #types	time	addr	port	addr	port	string	string
-1331256472.375609	1.2.3.4	1234	2.3.4.5	80	success	unknown
+1342749018.970682	1.2.3.4	1234	2.3.4.5	80	success	unknown
-1331256472.375609	1.2.3.4	1234	2.3.4.5	80	success	BR
+1342749018.970682	1.2.3.4	1234	2.3.4.5	80	success	BR
+#close	2012-07-20-01-50-18

testing/btest/Baseline/core.reporter-error-in-handler/output

@@ -1,2 +1,3 @@
-error in /da/home/robin/bro/master/testing/btest/.tmp/core.reporter-error-in-handler/reporter-error-in-handler.bro, line 22: no such index (a[2])
+error in /home/jsiwek/bro/testing/btest/.tmp/core.reporter-error-in-handler/reporter-error-in-handler.bro, line 22: no such index (a[2])
+ERROR: no such index (a[1]) (/home/jsiwek/bro/testing/btest/.tmp/core.reporter-error-in-handler/reporter-error-in-handler.bro, line 28)
 1st error printed on script level

testing/btest/Baseline/core.reporter-runtime-error/output

@@ -1 +1,2 @@
-error in /da/home/robin/bro/master/testing/btest/.tmp/core.reporter-runtime-error/reporter-runtime-error.bro, line 12: no such index (a[1])
+error in /home/jsiwek/bro/testing/btest/.tmp/core.reporter-runtime-error/reporter-runtime-error.bro, line 12: no such index (a[1])
+ERROR: no such index (a[2]) (/home/jsiwek/bro/testing/btest/.tmp/core.reporter-runtime-error/reporter-runtime-error.bro, line 9)

testing/btest/Baseline/core.reporter/logger-test.log

@@ -1,6 +1,6 @@
-reporter_info|init test-info|/da/home/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 8|0.000000
+reporter_info|init test-info|/home/jsiwek/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 8|0.000000
-reporter_warning|init test-warning|/da/home/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 9|0.000000
+reporter_warning|init test-warning|/home/jsiwek/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 9|0.000000
-reporter_error|init test-error|/da/home/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 10|0.000000
+reporter_error|init test-error|/home/jsiwek/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 10|0.000000
-reporter_info|done test-info|/da/home/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 15|0.000000
+reporter_info|done test-info|/home/jsiwek/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 15|0.000000
-reporter_warning|done test-warning|/da/home/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 16|0.000000
+reporter_warning|done test-warning|/home/jsiwek/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 16|0.000000
-reporter_error|done test-error|/da/home/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 17|0.000000
+reporter_error|done test-error|/home/jsiwek/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 17|0.000000

testing/btest/Baseline/core.reporter/output

@@ -1,3 +1,7 @@
-/da/home/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 52: pre test-info
+/home/jsiwek/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 52: pre test-info
-warning in /da/home/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 53: pre test-warning
+warning in /home/jsiwek/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 53: pre test-warning
-error in /da/home/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 54: pre test-error
+error in /home/jsiwek/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 54: pre test-error
+WARNING: init test-warning (/home/jsiwek/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 9)
+ERROR: init test-error (/home/jsiwek/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 10)
+WARNING: done test-warning (/home/jsiwek/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 16)
+ERROR: done test-error (/home/jsiwek/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 17)

testing/btest/Baseline/coverage.bare-mode-errors/unique_errors_no_elasticsearch

@@ -0,0 +1 @@
+error: unknown writer type requested

testing/btest/Baseline/scripts.base.frameworks.reporter.disable-stderr/reporter.log

@@ -0,0 +1,10 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	reporter
+#open	2012-08-10-20-09-16
+#fields	ts	level	message	location
+#types	time	enum	string	string
+0.000000	Reporter::ERROR	no such index (test[3])	/da/home/robin/bro/master/testing/btest/.tmp/scripts.base.frameworks.reporter.disable-stderr/disable-stderr.bro, line 12
+#close	2012-08-10-20-09-16

testing/btest/Baseline/scripts.base.frameworks.reporter.stderr/.stderr

@@ -0,0 +1 @@
+ERROR: no such index (test[3]) (/blah/testing/btest/.tmp/scripts.base.frameworks.reporter.stderr/stderr.bro, line 9)

testing/btest/Baseline/scripts.base.frameworks.reporter.stderr/reporter.log

@@ -0,0 +1,10 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	reporter
+#open	2012-08-10-20-09-23
+#fields	ts	level	message	location
+#types	time	enum	string	string
+0.000000	Reporter::ERROR	no such index (test[3])	/da/home/robin/bro/master/testing/btest/.tmp/scripts.base.frameworks.reporter.stderr/stderr.bro, line 9
+#close	2012-08-10-20-09-23

testing/btest/bifs/analyzer_name.bro

@@ -0,0 +1,9 @@
+#
+# @TEST-EXEC: bro %INPUT >out
+# @TEST-EXEC: btest-diff out
+
+event bro_init()
+	{
+	local a = 1;
+	print analyzer_name(a);
+	}

testing/btest/bifs/bro_version.bro

@@ -0,0 +1,9 @@
+#
+# @TEST-EXEC: bro %INPUT
+
+event bro_init()
+	{
+	local a = bro_version();
+	if ( |a| == 0 )
+		exit(1);
+	}

testing/btest/bifs/capture_state_updates.bro

@@ -0,0 +1,9 @@
+#
+# @TEST-EXEC: bro %INPUT >out
+# @TEST-EXEC: btest-diff out
+# @TEST-EXEC: test -f testfile
+
+event bro_init()
+	{
+	print capture_state_updates("testfile");
+	}

testing/btest/bifs/checkpoint_state.bro

@@ -0,0 +1,10 @@
+#
+# @TEST-EXEC: bro %INPUT
+# @TEST-EXEC: test -f .state/state.bst
+
+event bro_init()
+	{
+	local a = checkpoint_state();
+	if ( a != T )
+		exit(1);
+	}

testing/btest/bifs/current_analyzer.bro

@@ -0,0 +1,11 @@
+#
+# @TEST-EXEC: bro %INPUT
+
+event bro_init()
+	{
+	local a = current_analyzer();
+	if ( a != 0 )
+		exit(1);
+
+	# TODO: add a test for non-zero return value
+	}

testing/btest/bifs/current_time.bro

@@ -0,0 +1,9 @@
+#
+# @TEST-EXEC: bro %INPUT
+
+event bro_init()
+	{
+	local a = current_time();
+	if ( a <= double_to_time(0) )
+		exit(1);
+	}

testing/btest/bifs/entropy_test.bro

@@ -0,0 +1,24 @@
+#
+# @TEST-EXEC: bro %INPUT >out
+# @TEST-EXEC: btest-diff out
+
+event bro_init()
+	{
+	local a = "dh3Hie02uh^s#Sdf9L3frd243h$d78r2G4cM6*Q05d(7rh46f!0|4-f";
+	if ( entropy_test_init(1) != T )
+		exit(1);
+
+	if ( entropy_test_add(1, a) != T )
+		exit(1);
+
+	print entropy_test_finish(1);
+
+	local b = "0011000aaabbbbcccc000011111000000000aaaabbbbcccc0000000";
+	if ( entropy_test_init(2) != T )
+		exit(1);
+
+	if ( entropy_test_add(2, b) != T )
+		exit(1);
+
+	print entropy_test_finish(2);
+	}

testing/btest/bifs/get_matcher_stats.bro

@@ -0,0 +1,9 @@
+#
+# @TEST-EXEC: bro %INPUT
+
+event bro_init()
+	{
+	local a = get_matcher_stats();
+	if ( a$matchers == 0 )
+		exit(1);
+	}

testing/btest/bifs/gethostname.bro

@@ -0,0 +1,9 @@
+#
+# @TEST-EXEC: bro %INPUT
+
+event bro_init()
+	{
+	local a = gethostname();
+	if ( |a| == 0 )
+		exit(1);
+	}

testing/btest/bifs/getpid.bro

@@ -0,0 +1,9 @@
+#
+# @TEST-EXEC: bro %INPUT
+
+event bro_init()
+	{
+	local a = getpid();
+	if ( a == 0 )
+		exit(1);
+	}

testing/btest/bifs/global_sizes.bro

@@ -0,0 +1,16 @@
+#
+# @TEST-EXEC: bro %INPUT >out
+# @TEST-EXEC: btest-diff out
+
+event bro_init()
+	{
+	local a = global_sizes();
+	for ( i in a )
+		{
+		# the table is quite large, so just look for one item we expect
+		if ( i == "bro_init" )
+			print "found bro_init";
+
+		}
+
+	}

testing/btest/bifs/identify_data.bro

@@ -0,0 +1,16 @@
+#
+# @TEST-EXEC: bro %INPUT >out
+# @TEST-EXEC: btest-diff out
+
+event bro_init()
+	{
+	# plain text
+	local a = "This is a test";
+	print identify_data(a, F);
+	print identify_data(a, T);
+
+	# PNG image
+	local b = "\x89\x50\x4e\x47\x0d\x0a\x1a\x0a";
+	print identify_data(b, F);
+	print identify_data(b, T);
+	}

testing/btest/bifs/is_local_interface.bro

@@ -0,0 +1,11 @@
+#
+# @TEST-EXEC: bro %INPUT >out
+# @TEST-EXEC: btest-diff out
+
+event bro_init()
+	{
+	print is_local_interface(127.0.0.1);
+	print is_local_interface(1.2.3.4);
+	print is_local_interface([2607::a:b:c:d]);
+	print is_local_interface([::1]);
+	}

testing/btest/bifs/reading_traces.bro

@@ -0,0 +1,10 @@
+
+# @TEST-EXEC: bro %INPUT >out1
+# @TEST-EXEC: btest-diff out1
+# @TEST-EXEC: bro -r $TRACES/web.trace %INPUT >out2
+# @TEST-EXEC: btest-diff out2
+
+event bro_init()
+	{
+	print reading_traces();
+	}

testing/btest/bifs/resource_usage.bro

@@ -0,0 +1,9 @@
+#
+# @TEST-EXEC: bro %INPUT
+
+event bro_init()
+	{
+	local a = resource_usage();
+	if ( a$version != bro_version() )
+		exit(1);
+	}

testing/btest/bifs/strftime.bro

@@ -0,0 +1,17 @@
+#
+# @TEST-EXEC: bro %INPUT >out
+# @TEST-EXEC: btest-diff out
+
+event bro_init()
+	{
+	local f1 = "%Y-%m-%d %H:%M:%S";
+	local f2 = "%H%M%S %Y%m%d";
+
+	local a = double_to_time(0);
+	print strftime(f1, a);
+	print strftime(f2, a);
+
+	a = double_to_time(123456789);
+	print strftime(f1, a);
+	print strftime(f2, a);
+	}

testing/btest/core/leaks/basic-cluster.bro

@@ -9,7 +9,7 @@
 # @TEST-EXEC: sleep 1
 # @TEST-EXEC: btest-bg-run worker-1  HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local BROPATH=$BROPATH:.. CLUSTER_NODE=worker-1  bro -m -r $TRACES/web.trace --pseudo-realtime %INPUT
 # @TEST-EXEC: btest-bg-run worker-2  HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local BROPATH=$BROPATH:.. CLUSTER_NODE=worker-2  bro -m -r $TRACES/web.trace --pseudo-realtime %INPUT
-# @TEST-EXEC: btest-bg-wait -k 30
+# @TEST-EXEC: btest-bg-wait 40
 # @TEST-EXEC: btest-diff manager-1/metrics.log
 
 @TEST-START-FILE cluster-layout.bro
@@ -40,3 +40,24 @@ event bro_init() &priority=5
 		Metrics::add_data(TEST_METRIC, [$host=7.2.1.5], 1);
 		}
 	}
+
+event remote_connection_closed(p: event_peer)
+	{
+	terminate();
+	}
+
+@if ( Cluster::local_node_type() == Cluster::MANAGER )
+
+global n = 0;
+
+event Metrics::log_metrics(rec: Metrics::Info)
+	{
+	n = n + 1;
+	if ( n == 3 )
+		{
+		terminate_communication();
+		terminate();
+		}
+	}
+
+@endif

testing/btest/core/leaks/remote.bro

@@ -4,17 +4,19 @@
 #
 # @TEST-REQUIRES: bro  --help 2>&1 | grep -q mem-leaks
 #
-# @TEST-EXEC: btest-bg-run sender HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local bro -m --pseudo-realtime %INPUT ../sender.bro
+# @TEST-EXEC: btest-bg-run sender HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local bro -b -m --pseudo-realtime %INPUT ../sender.bro
 # @TEST-EXEC: sleep 1
-# @TEST-EXEC: btest-bg-run receiver HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local bro -m --pseudo-realtime %INPUT ../receiver.bro
+# @TEST-EXEC: btest-bg-run receiver HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local bro -b -m --pseudo-realtime %INPUT ../receiver.bro
 # @TEST-EXEC: sleep 1
-# @TEST-EXEC: btest-bg-wait -k 10
+# @TEST-EXEC: btest-bg-wait 30
 # @TEST-EXEC: btest-diff sender/test.log
 # @TEST-EXEC: btest-diff sender/test.failure.log
 # @TEST-EXEC: btest-diff sender/test.success.log
-# @TEST-EXEC: cmp receiver/test.log sender/test.log
+# @TEST-EXEC: ( cd sender && for i in *.log; do cat $i | $SCRIPTS/diff-remove-timestamps >c.$i; done )
-# @TEST-EXEC: cmp receiver/test.failure.log sender/test.failure.log
+# @TEST-EXEC: ( cd receiver && for i in *.log; do cat $i | $SCRIPTS/diff-remove-timestamps >c.$i; done )
-# @TEST-EXEC: cmp receiver/test.success.log sender/test.success.log
+# @TEST-EXEC: cmp receiver/c.test.log sender/c.test.log
+# @TEST-EXEC: cmp receiver/c.test.failure.log sender/c.test.failure.log
+# @TEST-EXEC: cmp receiver/c.test.success.log sender/c.test.success.log
 
 # This is the common part loaded by both sender and receiver.
 module Test;
@@ -43,10 +45,10 @@ event bro_init()
 
 @TEST-START-FILE sender.bro
 
-module Test;
-
 @load frameworks/communication/listen
 
+module Test;
+
 function fail(rec: Log): bool
 	{
 	return rec$status != "success";
@@ -68,14 +70,27 @@ event remote_connection_handshake_done(p: event_peer)
 	Log::write(Test::LOG, [$t=network_time(), $id=cid, $status="failure", $country="MX"]);
 	disconnect(p);
 	}
+
+event remote_connection_closed(p: event_peer)
+	{
+	terminate();
+	}
+
 @TEST-END-FILE
 
 @TEST-START-FILE receiver.bro
 
 #####
 
+@load base/frameworks/communication
+
 redef Communication::nodes += {
     ["foo"] = [$host = 127.0.0.1, $connect=T, $request_logs=T]
 };
 
+event remote_connection_closed(p: event_peer)
+	{
+	terminate();
+	}
+
 @TEST-END-FILE

testing/btest/coverage/bare-mode-errors.test

@@ -10,4 +10,5 @@
 # @TEST-EXEC: test -d $DIST/scripts
 # @TEST-EXEC: for script in `find $DIST/scripts -name \*\.bro -not -path '*/site/*'`; do echo $script; if echo "$script" | egrep -q 'communication/listen|controllee'; then rm -rf load_attempt .bgprocs; btest-bg-run load_attempt bro -b $script; btest-bg-wait -k 2; cat load_attempt/.stderr >>allerrors; else bro -b $script 2>>allerrors; fi done || exit 0
 # @TEST-EXEC: cat allerrors | grep -v "received termination signal" | sort | uniq > unique_errors
-# @TEST-EXEC: btest-diff unique_errors
+# @TEST-EXEC: if [ $(grep -c CURL_INCLUDE_DIR-NOTFOUND $BUILD/CMakeCache.txt) -ne 0 ]; then cp unique_errors unique_errors_no_elasticsearch; fi
+# @TEST-EXEC: if [ $(grep -c CURL_INCLUDE_DIR-NOTFOUND $BUILD/CMakeCache.txt) -ne 0 ]; then btest-diff unique_errors_no_elasticsearch; else btest-diff unique_errors; fi

testing/btest/scripts/base/frameworks/logging/remote.bro

@@ -1,10 +1,10 @@
 # @TEST-SERIALIZE: comm
 #
-# @TEST-EXEC: btest-bg-run sender bro --pseudo-realtime %INPUT ../sender.bro
+# @TEST-EXEC: btest-bg-run sender bro -b --pseudo-realtime %INPUT ../sender.bro
 # @TEST-EXEC: sleep 1
-# @TEST-EXEC: btest-bg-run receiver bro --pseudo-realtime %INPUT ../receiver.bro
+# @TEST-EXEC: btest-bg-run receiver bro -b --pseudo-realtime %INPUT ../receiver.bro
 # @TEST-EXEC: sleep 1
-# @TEST-EXEC: btest-bg-wait -k 10
+# @TEST-EXEC: btest-bg-wait 15
 # @TEST-EXEC: btest-diff sender/test.log
 # @TEST-EXEC: btest-diff sender/test.failure.log
 # @TEST-EXEC: btest-diff sender/test.success.log
@@ -41,10 +41,10 @@ event bro_init()
 
 @TEST-START-FILE sender.bro
 
-module Test;
-
 @load frameworks/communication/listen
 
+module Test;
+
 function fail(rec: Log): bool
 	{
 	return rec$status != "success";
@@ -66,14 +66,27 @@ event remote_connection_handshake_done(p: event_peer)
 	Log::write(Test::LOG, [$t=network_time(), $id=cid, $status="failure", $country="MX"]);
 	disconnect(p);
 	}
+
+event remote_connection_closed(p: event_peer)
+	{
+	terminate();
+	}
+
 @TEST-END-FILE
 
 @TEST-START-FILE receiver.bro
 
 #####
 
+@load base/frameworks/communication
+
 redef Communication::nodes += {
     ["foo"] = [$host = 127.0.0.1, $connect=T, $request_logs=T]
 };
 
+event remote_connection_closed(p: event_peer)
+	{
+	terminate();
+	}
+
 @TEST-END-FILE

testing/btest/scripts/base/frameworks/logging/rotate-custom.bro

@@ -1,7 +1,7 @@
 #
 # @TEST-EXEC: bro -b -r ${TRACES}/rotation.trace %INPUT | egrep "test|test2" | sort >out.tmp
 # @TEST-EXEC: cat out.tmp pp.log | sort >out
-# @TEST-EXEC: for i in `ls test*.log | sort`; do printf '> %s\n' $i; cat $i; done | sort | uniq >>out
+# @TEST-EXEC: for i in `ls test*.log | sort`; do printf '> %s\n' $i; cat $i; done | sort | $SCRIPTS/diff-remove-timestamps | uniq >>out
 # @TEST-EXEC: btest-diff out
 # @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-sort btest-diff .stderr
 

testing/btest/scripts/base/frameworks/metrics/basic-cluster.bro

@@ -5,7 +5,7 @@
 # @TEST-EXEC: sleep 1
 # @TEST-EXEC: btest-bg-run worker-1  BROPATH=$BROPATH:.. CLUSTER_NODE=worker-1 bro %INPUT
 # @TEST-EXEC: btest-bg-run worker-2  BROPATH=$BROPATH:.. CLUSTER_NODE=worker-2 bro %INPUT
-# @TEST-EXEC: btest-bg-wait -k 10
+# @TEST-EXEC: btest-bg-wait 20
 # @TEST-EXEC: btest-diff manager-1/metrics.log
 
 @TEST-START-FILE cluster-layout.bro
@@ -36,3 +36,24 @@ event bro_init() &priority=5
 		Metrics::add_data(TEST_METRIC, [$host=7.2.1.5], 1);
 		}
 	}
+
+event remote_connection_closed(p: event_peer)
+	{
+	terminate();
+	}
+
+@if ( Cluster::local_node_type() == Cluster::MANAGER )
+
+global n = 0;
+
+event Metrics::log_metrics(rec: Metrics::Info)
+	{
+	n = n + 1;
+	if ( n == 3 )
+		{
+		terminate_communication();
+		terminate();
+		}
+	}
+
+@endif

testing/btest/scripts/base/frameworks/metrics/cluster-intermediate-update.bro

@@ -5,7 +5,7 @@
 # @TEST-EXEC: sleep 1
 # @TEST-EXEC: btest-bg-run worker-1  BROPATH=$BROPATH:.. CLUSTER_NODE=worker-1 bro %INPUT 
 # @TEST-EXEC: btest-bg-run worker-2  BROPATH=$BROPATH:.. CLUSTER_NODE=worker-2 bro %INPUT
-# @TEST-EXEC: btest-bg-wait -k 10
+# @TEST-EXEC: btest-bg-wait 20
 # @TEST-EXEC: btest-diff manager-1/notice.log
 
 @TEST-START-FILE cluster-layout.bro
@@ -37,6 +37,21 @@ event bro_init() &priority=5
 		 $log=T]);
 	}
 
+event remote_connection_closed(p: event_peer)
+	{
+	terminate();
+	}
+
+@if ( Cluster::local_node_type() == Cluster::MANAGER )
+
+event Notice::log_notice(rec: Notice::Info)
+	{
+	terminate_communication();
+	terminate();
+	}
+
+@endif
+
 @if ( Cluster::local_node_type() == Cluster::WORKER )
 
 event do_metrics(i: count)

testing/btest/scripts/base/frameworks/reporter/disable-stderr.bro

@@ -0,0 +1,13 @@
+# @TEST-EXEC: bro %INPUT
+# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff .stderr
+# @TEST-EXEC: TEST_DIFF_CANONIFIER="$SCRIPTS/diff-remove-abspath | $SCRIPTS/diff-remove-timestamps" btest-diff reporter.log
+
+redef Reporter::warnings_to_stderr = F;
+redef Reporter::errors_to_stderr = F;
+
+global test: table[count] of string = {};
+
+event bro_init()
+	{
+	print test[3];
+	}

testing/btest/scripts/base/frameworks/reporter/stderr.bro

@@ -0,0 +1,10 @@
+# @TEST-EXEC: bro %INPUT
+# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff .stderr
+# @TEST-EXEC: TEST_DIFF_CANONIFIER="$SCRIPTS/diff-remove-abspath | $SCRIPTS/diff-remove-timestamps" btest-diff reporter.log
+
+global test: table[count] of string = {};
+
+event bro_init()
+	{
+	print test[3];
+	}

testing/scripts/diff-remove-mime-types

@@ -3,20 +3,27 @@
 # A diff canonifier that removes all MIME types because libmagic output
 # can differ between installations.
 
-BEGIN { FS="\t"; OFS="\t"; column = -1; }
+BEGIN { FS="\t"; OFS="\t"; type_col = -1; desc_col = -1 }
 
 /^#fields/ {
     for ( i = 2; i < NF; ++i )
+        {
         if ( $i == "mime_type" )
-            column = i-1;
+            type_col = i-1;
+        if ( $i == "mime_desc" )
+            desc_col = i-1;
+        }
 }
 
-column >= 0 {
+function remove_mime (n) {
-    if ( $column != "-" )
+    if ( n >= 0 && $n != "-" )
         # Mark that it's set, but ignore content.
-        $column = "+";
+        $n = "+"
 }
 
+remove_mime(type_col)
+remove_mime(desc_col)
+
 {
     print;
 }