Merge remote-tracking branch 'origin/master' into topic/bernhard/hyperloglog

testing/btest/scripts/base/frameworks/file-analysis/bifs/remove_action.bro

@@ -11,8 +11,8 @@ redef test_get_file_name = function(f: fa_file): string
 event file_new(f: fa_file) &priority=-10
 	{
 	for ( tag in test_file_analyzers )
-		FileAnalysis::remove_analyzer(f, tag);
+		Files::remove_analyzer(f, tag);
 	local filename = test_get_file_name(f);
-	FileAnalysis::remove_analyzer(f, [$tag=FileAnalysis::ANALYZER_EXTRACT,
-	                                $extract_filename=filename]);
+	Files::remove_analyzer(f, Files::ANALYZER_EXTRACT,
+	                       [$extract_filename=filename]);
 	}

testing/btest/scripts/base/frameworks/file-analysis/bifs/set_timeout_interval.bro

@@ -20,7 +20,7 @@ redef default_file_timeout_interval = 2sec;
 event file_timeout(f: fa_file)
 	{
 	if ( timeout_cnt < 1 )
-		FileAnalysis::set_timeout_interval(f, f$timeout_interval);
+		Files::set_timeout_interval(f, f$timeout_interval);
 	else
 		terminate();
 	++timeout_cnt;

testing/btest/scripts/base/frameworks/file-analysis/bifs/stop.bro

@@ -4,5 +4,5 @@
 
 event file_new(f: fa_file)
 	{
-	FileAnalysis::stop(f);
+	Files::stop(f);
 	}

testing/btest/scripts/base/frameworks/file-analysis/irc.bro

@@ -4,7 +4,17 @@
 
 redef test_file_analysis_source = "IRC_DATA";
 
-redef test_get_file_name = function(f: fa_file): string
+global first: bool = T;
+
+function myfile(f: fa_file): string
 	{
-	return "thefile";
-	};
+	if ( first )
+		{
+		first = F;
+		return "thefile";
+		}
+	else
+		return "";
+	}
+
+redef test_get_file_name = myfile;

testing/btest/scripts/base/frameworks/file-analysis/logging.bro

@@ -1,5 +1,5 @@
 # @TEST-EXEC: bro -r $TRACES/http/get.trace $SCRIPTS/file-analysis-test.bro %INPUT
-# @TEST-EXEC: btest-diff file_analysis.log
+# @TEST-EXEC: btest-diff files.log
 
 redef test_file_analysis_source = "HTTP";
 

testing/btest/scripts/base/frameworks/intel/cluster-transparency.bro

@@ -28,7 +28,7 @@ event remote_connection_handshake_done(p: event_peer)
 	# Insert the data once both workers are connected.
 	if ( Cluster::local_node_type() == Cluster::MANAGER && Cluster::worker_count == 2 )
 		{
-		Intel::insert([$host=1.2.3.4,$meta=[$source="manager"]]);
+		Intel::insert([$indicator="1.2.3.4", $indicator_type=Intel::ADDR, $meta=[$source="manager"]]);
 		}
 	}
 
@@ -39,7 +39,7 @@ event Intel::cluster_new_item(item: Intel::Item)
 	if ( ! is_remote_event() )
 		return;
 
-	print fmt("cluster_new_item: %s inserted by %s (from peer: %s)", item$host, item$meta$source, get_event_peer()$descr);
+	print fmt("cluster_new_item: %s inserted by %s (from peer: %s)", item$indicator, item$meta$source, get_event_peer()$descr);
 
 	if ( ! sent_data )
 		{
@@ -47,9 +47,9 @@ event Intel::cluster_new_item(item: Intel::Item)
 		# full cluster is constructed.
 		sent_data = T;
 		if ( Cluster::node == "worker-1" )
-			Intel::insert([$host=123.123.123.123,$meta=[$source="worker-1"]]);
+			Intel::insert([$indicator="123.123.123.123", $indicator_type=Intel::ADDR, $meta=[$source="worker-1"]]);
 		if ( Cluster::node == "worker-2" )
-			Intel::insert([$host=4.3.2.1,$meta=[$source="worker-2"]]);
+			Intel::insert([$indicator="4.3.2.1", $indicator_type=Intel::ADDR, $meta=[$source="worker-2"]]);
 		}
 
 	# We're forcing worker-2 to do a lookup when it has three intelligence items

testing/btest/scripts/base/frameworks/intel/input-and-match.bro

@@ -5,10 +5,10 @@
 # @TEST-EXEC: btest-diff broproc/intel.log
 
 @TEST-START-FILE intel.dat
-#fields	host	net	str	str_type	meta.source	meta.desc	meta.url
-1.2.3.4	-	-	-	source1	this host is just plain baaad	http://some-data-distributor.com/1234
-1.2.3.4	-	-	-	source1	this host is just plain baaad	http://some-data-distributor.com/1234
--	-	e@mail.com	Intel::EMAIL	source1	Phishing email source	http://some-data-distributor.com/100000
+#fields	indicator	indicator_type	meta.source	meta.desc	meta.url
+1.2.3.4	Intel::ADDR	source1	this host is just plain baaad	http://some-data-distributor.com/1234
+1.2.3.4	Intel::ADDR	source1	this host is just plain baaad	http://some-data-distributor.com/1234
+e@mail.com	Intel::EMAIL	source1	Phishing email source	http://some-data-distributor.com/100000
 @TEST-END-FILE
 
 @load frameworks/communication/listen
@@ -18,8 +18,8 @@ redef enum Intel::Where += { SOMEWHERE };
 
 event do_it()
 	{
-	Intel::seen([$str="e@mail.com",
-	             $str_type=Intel::EMAIL,
+	Intel::seen([$indicator="e@mail.com",
+	             $indicator_type=Intel::EMAIL,
 	             $where=SOMEWHERE]);
 
 	Intel::seen([$host=1.2.3.4,

testing/btest/scripts/base/frameworks/intel/read-file-dist-cluster.bro

@@ -19,10 +19,10 @@ redef Cluster::nodes = {
 @TEST-END-FILE
 
 @TEST-START-FILE intel.dat
-#fields	host	net	str	str_type	meta.source	meta.desc	meta.url
-1.2.3.4	-	-	-	source1	this host is just plain baaad	http://some-data-distributor.com/1234
-1.2.3.4	-	-	-	source1	this host is just plain baaad	http://some-data-distributor.com/1234
--	-	e@mail.com	Intel::EMAIL	source1	Phishing email source	http://some-data-distributor.com/100000
+#fields	indicator	indicator_type	meta.source	meta.desc	meta.url
+1.2.3.4	Intel::ADDR	source1	this host is just plain baaad	http://some-data-distributor.com/1234
+1.2.3.4	Intel::ADDR	source1	this host is just plain baaad	http://some-data-distributor.com/1234
+e@mail.com	Intel::EMAIL	source1	Phishing email source	http://some-data-distributor.com/100000
 @TEST-END-FILE
 
 @load base/frameworks/control
@@ -41,7 +41,7 @@ redef enum Intel::Where += {
 event do_it()
 	{
 	Intel::seen([$host=1.2.3.4, $where=Intel::IN_A_TEST]);
-	Intel::seen([$str="e@mail.com", $str_type=Intel::EMAIL, $where=Intel::IN_A_TEST]);
+	Intel::seen([$indicator="e@mail.com", $indicator_type=Intel::EMAIL, $where=Intel::IN_A_TEST]);
 	}
 
 event bro_init()

testing/btest/scripts/base/protocols/dns/dns-key.bro

@@ -0,0 +1,4 @@
+# Making sure DNSKEY gets logged as such.
+#
+# @TEST-EXEC: bro -r $TRACES/dns-dnskey.trace
+# @TEST-EXEC: btest-diff dns.log

testing/btest/scripts/base/protocols/ftp/ftp-extract.bro

@@ -1,10 +0,0 @@
-# This tests FTP file extraction.
-#
-# @TEST-EXEC: bro -r $TRACES/ftp/ipv4.trace %INPUT
-# @TEST-EXEC: btest-diff conn.log
-# @TEST-EXEC: btest-diff ftp.log
-# @TEST-EXEC: cat ftp-item-*.dat | sort > extractions
-# @TEST-EXEC: btest-diff extractions
-
-redef FTP::logged_commands += {"LIST"};
-redef FTP::extract_file_types=/.*/;

testing/btest/scripts/base/protocols/http/http-extract-files.bro

@@ -1,6 +0,0 @@
-# @TEST-EXEC: bro -C -r $TRACES/web.trace %INPUT
-# @TEST-EXEC: btest-diff http.log
-# @TEST-EXEC: mv http-item-*.dat http-item.dat
-# @TEST-EXEC: btest-diff http-item.dat
-
-redef HTTP::extract_file_types += /text\/html/;

testing/btest/scripts/base/protocols/http/http-mime-and-md5.bro

@@ -1,6 +0,0 @@
-# This tests md5 calculation for a specified mime type.
-
-# @TEST-EXEC: bro -r $TRACES/http/pipelined-requests.trace %INPUT > output
-# @TEST-EXEC: btest-diff http.log
-
-redef HTTP::generate_md5 += /image\/png/;

testing/btest/scripts/base/protocols/http/multipart-extract.bro

@@ -1,5 +1,9 @@
 # @TEST-EXEC: bro -C -r $TRACES/http/multipart.trace %INPUT
 # @TEST-EXEC: btest-diff http.log
-# @TEST-EXEC: cat http-item-* | sort > extractions
+# @TEST-EXEC: cat extract_files/http-item-* | sort > extractions
 
-redef HTTP::extract_file_types += /.*/;
+event file_new(f: fa_file)
+	{
+	local fname = fmt("http-item-%s", f$id);
+	Files::add_analyzer(f, Files::ANALYZER_EXTRACT, [$extract_filename=fname]);
+	}

testing/btest/scripts/base/protocols/irc/dcc-extract.test

@@ -1,11 +0,0 @@
-# This tests that the contents of a DCC transfer negotiated with IRC can be
-# correctly extracted.
-
-# @TEST-EXEC: bro -r $TRACES/irc-dcc-send.trace %INPUT
-# @TEST-EXEC: btest-diff irc.log
-# @TEST-EXEC: mv irc-dcc-item-*.dat irc-dcc-item.dat
-# @TEST-EXEC: btest-diff irc-dcc-item.dat
-# @TEST-EXEC: bro -r $TRACES/irc-dcc-send.trace %INPUT IRC::extraction_prefix="test"
-# @TEST-EXEC: test -e test-*.dat
-
-redef IRC::extract_file_types=/.*/;

testing/btest/scripts/base/protocols/smtp/mime-extract.test

@@ -1,11 +0,0 @@
-# @TEST-EXEC: bro -r $TRACES/smtp.trace %INPUT
-# @TEST-EXEC: btest-diff smtp_entities.log
-# @TEST-EXEC: cat smtp-entity-*.dat | sort > extractions
-# @TEST-EXEC: btest-diff extractions
-# @TEST-EXEC: bro -r $TRACES/smtp.trace %INPUT SMTP::extraction_prefix="test"
-# @TEST-EXEC: cnt=0 && for f in test-*.dat; do cnt=$((cnt+1)); done && echo $cnt >filecount
-# @TEST-EXEC: btest-diff filecount
-
-@load base/protocols/smtp
-
-redef SMTP::extract_file_types=/text\/plain/;

testing/btest/scripts/base/protocols/smtp/mime.test

@@ -1,6 +0,0 @@
-# @TEST-EXEC: bro -r $TRACES/smtp.trace %INPUT
-# @TEST-EXEC: btest-diff smtp_entities.log
-
-@load base/protocols/smtp
-
-redef SMTP::generate_md5=/text\/plain/;

testing/btest/scripts/base/utils/active-http.test

@@ -0,0 +1,28 @@
+# @TEST-REQUIRES: which httpd
+# @TEST-REQUIRES: which python
+#
+# @TEST-EXEC: btest-bg-run httpd python $SCRIPTS/httpd.py --max 1
+# @TEST-EXEC: sleep 3
+# @TEST-EXEC: btest-bg-run bro bro -b %INPUT
+# @TEST-EXEC: btest-bg-wait 15
+# @TEST-EXEC: btest-diff bro/.stdout
+
+@load base/utils/active-http
+
+redef exit_only_after_terminate = T;
+
+event bro_init()
+	{
+	local req = ActiveHTTP::Request($url="localhost:32123");
+
+	when ( local resp = ActiveHTTP::request(req) )
+		{
+		print resp;
+		terminate();
+		}
+	timeout 1min
+		{
+		print "HTTP request timeout";
+		terminate();
+		}
+	}

testing/btest/scripts/base/utils/dir.test

@@ -0,0 +1,58 @@
+# @TEST-EXEC: btest-bg-run bro bro -b ../dirtest.bro
+# @TEST-EXEC: btest-bg-wait 10
+# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-sort btest-diff bro/.stdout
+
+@TEST-START-FILE dirtest.bro
+
+@load base/utils/dir
+
+redef exit_only_after_terminate = T;
+
+global c: count = 0;
+
+function check_terminate_condition()
+	{
+	c += 1;
+
+	if ( c == 10 )
+		terminate();
+	}
+
+function new_file1(fname: string)
+	{
+	print "new_file1", fname;
+	check_terminate_condition();
+	}
+
+function new_file2(fname: string)
+	{
+	print "new_file2", fname;
+	check_terminate_condition();
+	}
+
+event change_things()
+	{
+	system("touch ../testdir/newone");
+	system("rm ../testdir/bye && touch ../testdir/bye");
+	}
+
+event bro_init()
+	{
+	Dir::monitor("../testdir", new_file1, .5sec);
+	Dir::monitor("../testdir", new_file2, 1sec);
+	schedule 1sec { change_things() };
+	}
+
+@TEST-END-FILE
+
+@TEST-START-FILE testdir/hi
+123
+@TEST-END-FILE
+
+@TEST-START-FILE testdir/howsitgoing
+abc
+@TEST-END-FILE
+
+@TEST-START-FILE testdir/bye
+!@#
+@TEST-END-FILE

testing/btest/scripts/base/utils/exec.test

@@ -0,0 +1,74 @@
+# @TEST-EXEC: btest-bg-run bro bro -b ../exectest.bro
+# @TEST-EXEC: btest-bg-wait 10
+# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-sort btest-diff bro/.stdout
+
+@TEST-START-FILE exectest.bro
+
+@load base/utils/exec
+
+redef exit_only_after_terminate = T;
+
+global c: count = 0;
+
+function check_exit_condition()
+	{
+	c += 1;
+
+	if ( c == 4 )
+		terminate();
+	}
+
+function test_cmd(label: string, cmd: Exec::Command)
+	{
+	when ( local result = Exec::run(cmd) )
+		{
+		print label, result;
+		check_exit_condition();
+		}
+	}
+
+event bro_init()
+	{
+	test_cmd("test1", [$cmd="bash ../somescript.sh",
+	                   $read_files=set("out1", "out2")]);
+	test_cmd("test2", [$cmd="bash ../nofiles.sh"]);
+	test_cmd("test3", [$cmd="bash ../suicide.sh"]);
+	test_cmd("test4", [$cmd="bash ../stdin.sh", $stdin="hibye"]);
+	}
+
+@TEST-END-FILE
+
+@TEST-START-FILE somescript.sh
+#! /usr/bin/env bash
+echo "insert text here" > out1
+echo "and here" >> out1
+echo "insert more text here" > out2
+echo "and there" >> out2
+echo "done"
+echo "exit"
+echo "stop"
+@TEST-END-FILE
+
+@TEST-START-FILE nofiles.sh
+#! /usr/bin/env bash
+echo "here's something on stdout"
+echo "some more stdout"
+echo "last stdout"
+echo "and some stderr" 1>&2
+echo "more stderr" 1>&2
+echo "last stderr" 1>&2
+exit 1
+@TEST-END-FILE
+
+@TEST-START-FILE suicide.sh
+#! /usr/bin/env bash
+echo "FML"
+kill -9 $$
+echo "nope"
+@TEST-END-FILE
+
+@TEST-START-FILE stdin.sh
+#! /usr/bin/env bash
+read -r line
+echo "$line"
+@TEST-END-FILE

testing/btest/scripts/base/utils/files.test

@@ -11,3 +11,10 @@ event connection_established(c: connection)
 	print generate_extraction_filename("", c, "test-suffix");
 	print generate_extraction_filename("", c, "");
 	}
+
+event bro_init()
+	{
+	print extract_filename_from_content_disposition("attachment; filename=Economy");
+	print extract_filename_from_content_disposition("attachment; name=\"US-$ rates\"");
+	print extract_filename_from_content_disposition("attachment; filename*=iso-8859-1'en'%A3%20rates");
+	}