Merge remote-tracking branch 'origin/topic/jsiwek/reorg-followup'

2025-10-10 18:48:20 +00:00 · 2011-08-25 16:44:31 -04:00 · 2011-08-25 16:44:31 -04:00 · fc5f22cb5d
commit fc5f22cb5d
parent 562abfb0d1 e37430ef66
234 changed files with 295 additions and 104 deletions
--- a/testing/btest/scripts/base/frameworks/cluster/start-it-up.bro
+++ b/testing/btest/scripts/base/frameworks/cluster/start-it-up.bro
@ -0,0 +1,26 @@
+# @TEST-EXEC: btest-bg-run manager-1 BROPATH=$BROPATH:.. CLUSTER_NODE=manager-1 bro %INPUT
+# @TEST-EXEC: btest-bg-run proxy-1   BROPATH=$BROPATH:.. CLUSTER_NODE=proxy-1 bro %INPUT
+# @TEST-EXEC: btest-bg-run proxy-2   BROPATH=$BROPATH:.. CLUSTER_NODE=proxy-2 bro %INPUT
+# @TEST-EXEC: btest-bg-run worker-1  BROPATH=$BROPATH:.. CLUSTER_NODE=worker-1 bro %INPUT
+# @TEST-EXEC: btest-bg-run worker-2  BROPATH=$BROPATH:.. CLUSTER_NODE=worker-2 bro %INPUT
+# @TEST-EXEC: btest-bg-wait -k 2
+# @TEST-EXEC: btest-diff manager-1/.stdout
+# @TEST-EXEC: btest-diff proxy-1/.stdout
+# @TEST-EXEC: btest-diff proxy-2/.stdout
+# @TEST-EXEC: btest-diff worker-1/.stdout
+# @TEST-EXEC: btest-diff worker-2/.stdout
+
+@TEST-START-FILE cluster-layout.bro
+redef Cluster::nodes = {
+	["manager-1"] = [$node_type=Cluster::MANAGER, $ip=127.0.0.1, $p=37757/tcp, $workers=set("worker-1")],
+	["proxy-1"] = [$node_type=Cluster::PROXY,     $ip=127.0.0.1, $p=37758/tcp, $manager="manager-1", $workers=set("worker-1")],
+	["proxy-2"] = [$node_type=Cluster::PROXY,     $ip=127.0.0.1, $p=37759/tcp, $manager="manager-1", $workers=set("worker-2")],
+	["worker-1"] = [$node_type=Cluster::WORKER,   $ip=127.0.0.1, $p=37760/tcp, $manager="manager-1", $proxy="proxy-1", $interface="eth0"],
+	["worker-2"] = [$node_type=Cluster::WORKER,   $ip=127.0.0.1, $p=37761/tcp, $manager="manager-1", $proxy="proxy-2", $interface="eth1"],
+};
+@TEST-END-FILE
+
+event remote_connection_handshake_done(p: event_peer)
+	{
+	print "Connected to a peer";
+	}
--- a/testing/btest/scripts/base/frameworks/control/configuration_update.bro
+++ b/testing/btest/scripts/base/frameworks/control/configuration_update.bro
@ -0,0 +1,26 @@
+# @TEST-EXEC: btest-bg-run controllee  BROPATH=$BROPATH:.. bro %INPUT frameworks/control/controllee Communication::listen_port_clear=65531/tcp 
+# @TEST-EXEC: btest-bg-run controller  BROPATH=$BROPATH:.. bro %INPUT test-redef frameworks/control/controller Control::host=127.0.0.1 Control::host_port=65531/tcp Control::cmd=configuration_update
+# @TEST-EXEC: btest-bg-run controller2 BROPATH=$BROPATH:.. bro %INPUT frameworks/control/controller Control::host=127.0.0.1 Control::host_port=65531/tcp Control::cmd=shutdown
+# @TEST-EXEC: btest-bg-wait 1
+# @TEST-EXEC: btest-diff controllee/.stdout
+
+redef Communication::nodes = {
+	# We're waiting for connections from this host for control.
+	["control"] = [$host=127.0.0.1, $class="control", $events=Control::controller_events],
+};
+
+const test_var = "ORIGINAL VALUE (this should be printed out first)" &redef;
+
+@TEST-START-FILE test-redef.bro
+redef test_var = "NEW VALUE (this should be printed out second)";
+@TEST-END-FILE
+
+event bro_init()
+	{
+	print test_var;
+	}
+	
+event bro_done()
+	{
+	print test_var;
+	}
--- a/testing/btest/scripts/base/frameworks/control/id_value.bro
+++ b/testing/btest/scripts/base/frameworks/control/id_value.bro
@ -0,0 +1,23 @@
+# @TEST-EXEC: btest-bg-run controllee  BROPATH=$BROPATH:.. bro %INPUT only-for-controllee frameworks/control/controllee Communication::listen_port_clear=65532/tcp 
+# @TEST-EXEC: btest-bg-run controller  BROPATH=$BROPATH:.. bro %INPUT frameworks/control/controller Control::host=127.0.0.1 Control::host_port=65532/tcp Control::cmd=id_value Control::arg=test_var
+# @TEST-EXEC: btest-bg-wait -k 1
+# @TEST-EXEC: btest-diff controller/.stdout
+
+redef Communication::nodes = {
+	# We're waiting for connections from this host for control.
+	["control"] = [$host=127.0.0.1, $class="control", $events=Control::controller_events],
+};
+
+# This value shouldn't ever be printed to the controllers stdout.
+const test_var = "Original value" &redef;
+
+@TEST-START-FILE only-for-controllee.bro
+# This is only loaded on the controllee, but it's sent to the controller 
+# and should be printed there.
+redef test_var = "This is the value from the controllee";
+@TEST-END-FILE
+
+event Control::id_value_response(id: string, val: string)
+	{
+	print fmt("Got an id_value_response(%s, %s) event", id, val);
+	}
--- a/testing/btest/scripts/base/frameworks/control/shutdown.bro
+++ b/testing/btest/scripts/base/frameworks/control/shutdown.bro
@ -0,0 +1,8 @@
+# @TEST-EXEC: btest-bg-run controllee BROPATH=$BROPATH:.. bro %INPUT frameworks/control/controllee Communication::listen_port_clear=65530/tcp 
+# @TEST-EXEC: btest-bg-run controller BROPATH=$BROPATH:.. bro %INPUT frameworks/control/controller Control::host=127.0.0.1 Control::host_port=65530/tcp Control::cmd=shutdown
+# @TEST-EXEC: btest-bg-wait 1
+
+redef Communication::nodes = {
+	# We're waiting for connections from this host for control.
+	["control"] = [$host=127.0.0.1, $class="control", $events=Control::controller_events],
+};
--- a/testing/btest/scripts/base/frameworks/intel/insert-and-matcher.bro
+++ b/testing/btest/scripts/base/frameworks/intel/insert-and-matcher.bro
@ -0,0 +1,34 @@
+#
+# @TEST-EXEC: bro %INPUT >out
+# @TEST-EXEC: btest-diff out
+
+event bro_init()
+	{
+	Intel::insert([$ip=1.2.3.4, $tags=set("zeustracker.abuse.ch", "malicious")]);
+	Intel::insert([$str="http://www.google.com/", $subtype="url", $tags=set("infrastructure", "google")]);
+	Intel::insert([$str="Ab439G32F...", $subtype="x509_cert", $tags=set("bad")]);
+	Intel::insert([$str="Ab439G32F...", $tags=set("bad")]);
+	}
+
+event bro_done()
+	{
+	local orig_h = 1.2.3.4;
+	
+	if ( Intel::matcher([$ip=orig_h, $and_tags=set("malicious")]) )
+		print "VALID";
+	
+	if ( Intel::matcher([$ip=orig_h, $and_tags=set("don't match")]) )
+		print "INVALID";
+	
+	if ( Intel::matcher([$ip=orig_h, $pred=function(meta: Intel::MetaData): bool { return T; } ]) )
+		print "VALID";
+		
+	if ( Intel::matcher([$ip=orig_h, $pred=function(meta: Intel::MetaData): bool { return F; } ]) )
+		print "INVALID";
+		
+	if ( Intel::matcher([$str="http://www.google.com/", $subtype="url", $tags=set("google")]) )
+		print "VALID";
+		
+	if ( Intel::matcher([$str="http://www.example.com", $subtype="url"]) )
+		print "INVALID";
+	}
--- a/testing/btest/scripts/base/frameworks/logging/adapt-filter.bro
+++ b/testing/btest/scripts/base/frameworks/logging/adapt-filter.bro
@ -0,0 +1,33 @@
+
+# @TEST-EXEC: bro -b %INPUT 
+# @TEST-EXEC: btest-diff ssh-new-default.log
+# @TEST-EXEC: test '!' -e ssh.log
+
+module SSH;
+
+export {
+	# Create a new ID for our log stream
+	redef enum Log::ID += { SSH };
+
+	# Define a record with all the columns the log file can have.
+	# (I'm using a subset of fields from ssh-ext for demonstration.)
+	type Info: record {
+		t: time;
+		id: conn_id; # Will be rolled out into individual columns.
+		status: string &optional;
+		country: string &default="unknown";
+	} &log;
+}
+
+event bro_init()
+{
+	Log::create_stream(SSH, [$columns=Info]);
+
+	local filter = Log::get_filter(SSH, "default");
+	filter$path= "ssh-new-default";
+	Log::add_filter(SSH, filter);
+
+    local cid = [$orig_h=1.2.3.4, $orig_p=1234/tcp, $resp_h=2.3.4.5, $resp_p=80/tcp];
+	Log::write(SSH, [$t=network_time(), $id=cid, $status="success"]);
+	Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="US"]);
+}
--- a/testing/btest/scripts/base/frameworks/logging/ascii-binary.bro
+++ b/testing/btest/scripts/base/frameworks/logging/ascii-binary.bro
@ -0,0 +1,25 @@
+#
+# @TEST-EXEC: bro -b %INPUT
+# @TEST-EXEC: btest-diff ssh.log
+
+module SSH;
+
+export {
+	redef enum Log::ID += { SSH };
+
+	type Info: record {
+		data: string;
+		data2: string;
+	} &log;
+}
+
+redef LogAscii::separator = "|";
+
+event bro_init()
+{
+	Log::create_stream(SSH, [$columns=Info]);
+	Log::write(SSH, [$data="abc\n\xffdef", $data2="DATA2"]);
+	Log::write(SSH, [$data="abc|\xffdef", $data2="DATA2"]);
+	Log::write(SSH, [$data="abc\xff|def", $data2="DATA2"]);
+}
+
--- a/testing/btest/scripts/base/frameworks/logging/ascii-empty.bro
+++ b/testing/btest/scripts/base/frameworks/logging/ascii-empty.bro
@ -0,0 +1,38 @@
+#
+# @TEST-EXEC: bro -b %INPUT
+# @TEST-EXEC: btest-diff ssh.log
+
+redef LogAscii::output_to_stdout = F;
+redef LogAscii::separator = "|";
+redef LogAscii::empty_field = "EMPTY";
+redef LogAscii::unset_field = "NOT-SET";
+redef LogAscii::header_prefix = "PREFIX<>";
+
+module SSH;
+
+export {
+	redef enum Log::ID += { SSH };
+
+	type Log: record {
+		t: time;
+		id: conn_id; # Will be rolled out into individual columns.
+		status: string &optional;
+		country: string &default="unknown";
+		b: bool &optional;
+	} &log;
+}
+
+event bro_init()
+{
+	Log::create_stream(SSH, [$columns=Log]);
+
+    local cid = [$orig_h=1.2.3.4, $orig_p=1234/tcp, $resp_h=2.3.4.5, $resp_p=80/tcp];
+
+	Log::write(SSH, [$t=network_time(), $id=cid, $status="success"]);
+	Log::write(SSH, [$t=network_time(), $id=cid, $country="US"]);
+	Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="UK"]);
+	Log::write(SSH, [$t=network_time(), $id=cid, $country="BR"]);
+	Log::write(SSH, [$t=network_time(), $id=cid, $b=T, $status="failure", $country=""]);
+	
+}
+
--- a/testing/btest/scripts/base/frameworks/logging/ascii-escape.bro
+++ b/testing/btest/scripts/base/frameworks/logging/ascii-escape.bro
@ -0,0 +1,32 @@
+#
+# @TEST-EXEC: bro -b %INPUT 
+# @TEST-EXEC: btest-diff ssh.log
+
+redef LogAscii::separator = "||";
+
+module SSH;
+
+export {
+	redef enum Log::ID += { SSH };
+
+	type Log: record {
+		t: time;
+		id: conn_id; # Will be rolled out into individual columns.
+		status: string &optional;
+		country: string &default="unknown";
+	} &log;
+}
+
+event bro_init()
+{
+	Log::create_stream(SSH, [$columns=Log]);
+
+    local cid = [$orig_h=1.2.3.4, $orig_p=1234/tcp, $resp_h=2.3.4.5, $resp_p=80/tcp];
+
+	Log::write(SSH, [$t=network_time(), $id=cid, $status="success"]);
+	Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="US"]);
+	Log::write(SSH, [$t=network_time(), $id=cid, $status="fa||ure", $country="UK"]);
+	Log::write(SSH, [$t=network_time(), $id=cid, $status="su||ess", $country="BR"]);
+	Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="MX"]);	
+}
+
--- a/testing/btest/scripts/base/frameworks/logging/ascii-options.bro
+++ b/testing/btest/scripts/base/frameworks/logging/ascii-options.bro
@ -0,0 +1,35 @@
+#
+# @TEST-EXEC: bro -b %INPUT
+# @TEST-EXEC: btest-diff ssh.log
+
+redef LogAscii::output_to_stdout = F;
+redef LogAscii::separator = "|";
+redef LogAscii::include_header = F;
+
+module SSH;
+
+export {
+	redef enum Log::ID += { SSH };
+
+	type Log: record {
+		t: time;
+		id: conn_id; # Will be rolled out into individual columns.
+		status: string &optional;
+		country: string &default="unknown";
+	} &log;
+}
+
+event bro_init()
+{
+	Log::create_stream(SSH, [$columns=Log]);
+
+    local cid = [$orig_h=1.2.3.4, $orig_p=1234/tcp, $resp_h=2.3.4.5, $resp_p=80/tcp];
+
+	Log::write(SSH, [$t=network_time(), $id=cid, $status="success"]);
+	Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="US"]);
+	Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="UK"]);
+	Log::write(SSH, [$t=network_time(), $id=cid, $status="success", $country="BR"]);
+	Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="MX"]);
+	
+}
+
--- a/testing/btest/scripts/base/frameworks/logging/ascii-timestamps.bro
+++ b/testing/btest/scripts/base/frameworks/logging/ascii-timestamps.bro
@ -0,0 +1,27 @@
+#
+# @TEST-EXEC: bro -b %INPUT
+# @TEST-EXEC: btest-diff test.log
+
+module Test;
+
+export {
+	redef enum Log::ID += { TEST };
+
+	type Info: record {
+		data: time &log;
+	};
+}
+
+event bro_init()
+{
+	Log::create_stream(TEST, [$columns=Info]);
+	Log::write(TEST, [$data=double_to_time(1234567890)]);
+	Log::write(TEST, [$data=double_to_time(1234567890.0)]);
+	Log::write(TEST, [$data=double_to_time(1234567890.01)]);
+	Log::write(TEST, [$data=double_to_time(1234567890.001)]);
+	Log::write(TEST, [$data=double_to_time(1234567890.0001)]);
+	Log::write(TEST, [$data=double_to_time(1234567890.00001)]);
+	Log::write(TEST, [$data=double_to_time(1234567890.000001)]);
+	Log::write(TEST, [$data=double_to_time(1234567890.0000001)]);
+}
+
--- a/testing/btest/scripts/base/frameworks/logging/attr-extend.bro
+++ b/testing/btest/scripts/base/frameworks/logging/attr-extend.bro
@ -0,0 +1,37 @@
+#
+# @TEST-EXEC: bro -b %INPUT
+# @TEST-EXEC: btest-diff ssh.log
+
+module SSH;
+
+export {
+	redef enum Log::ID += { SSH };
+
+	type Log: record {
+		t: time;
+		id: conn_id;
+		status: string &optional &log;
+		country: string &default="unknown" &log;
+	};
+}
+
+redef record Log += {
+	a1: count &log &optional;
+	a2: count &optional;
+};
+
+redef record Log += {
+	b1: count &optional;
+	b2: count &optional;
+} &log;
+
+
+event bro_init()
+{
+	Log::create_stream(SSH, [$columns=Log]);
+
+    local cid = [$orig_h=1.2.3.4, $orig_p=1234/tcp, $resp_h=2.3.4.5, $resp_p=80/tcp];
+
+	Log::write(SSH, [$t=network_time(), $id=cid, $status="success", $a1=1, $a2=2, $b1=3, $b2=4]);
+}
+
--- a/testing/btest/scripts/base/frameworks/logging/attr.bro
+++ b/testing/btest/scripts/base/frameworks/logging/attr.bro
@ -0,0 +1,31 @@
+#
+# @TEST-EXEC: bro -b %INPUT
+# @TEST-EXEC: btest-diff ssh.log
+
+module SSH;
+
+export {
+	redef enum Log::ID += { SSH };
+
+	type Log: record {
+		t: time;
+		id: conn_id;
+		status: string &optional &log;
+		country: string &default="unknown" &log;
+	};
+}
+
+event bro_init()
+{
+	Log::create_stream(SSH, [$columns=Log]);
+
+    local cid = [$orig_h=1.2.3.4, $orig_p=1234/tcp, $resp_h=2.3.4.5, $resp_p=80/tcp];
+
+	Log::write(SSH, [$t=network_time(), $id=cid, $status="success"]);
+	Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="US"]);
+	Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="UK"]);
+	Log::write(SSH, [$t=network_time(), $id=cid, $status="success", $country="BR"]);
+	Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="MX"]);
+	
+}
+
--- a/testing/btest/scripts/base/frameworks/logging/disable-stream.bro
+++ b/testing/btest/scripts/base/frameworks/logging/disable-stream.bro
@ -0,0 +1,33 @@
+#
+# @TEST-EXEC: bro -b %INPUT
+# @TEST-EXEC: test '!' -e ssh.log
+
+module SSH;
+
+export {
+	redef enum Log::ID += { SSH };
+
+	type Log: record {
+		t: time;
+		id: conn_id; # Will be rolled out into individual columns.
+		status: string &optional;
+		country: string &default="unknown";
+	} &log;
+}
+
+event bro_init()
+{
+	Log::create_stream(SSH, [$columns=Log]);
+
+	Log::disable_stream(SSH);
+
+    local cid = [$orig_h=1.2.3.4, $orig_p=1234/tcp, $resp_h=2.3.4.5, $resp_p=80/tcp];
+
+	Log::write(SSH, [$t=network_time(), $id=cid, $status="success"]);
+	Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="US"]);
+	Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="UK"]);
+	Log::write(SSH, [$t=network_time(), $id=cid, $status="success", $country="BR"]);
+	Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="MX"]);
+	
+}
+
--- a/testing/btest/scripts/base/frameworks/logging/empty-event.bro
+++ b/testing/btest/scripts/base/frameworks/logging/empty-event.bro
@ -0,0 +1,33 @@
+#
+# @TEST-EXEC: bro -b %INPUT
+# @TEST-EXEC: btest-diff ssh.log
+
+module SSH;
+
+export {
+	redef enum Log::ID += { SSH };
+
+	type Log: record {
+		t: time;
+		id: conn_id; # Will be rolled out into individual columns.
+		status: string &optional;
+		country: string &default="unknown";
+	} &log;
+}
+
+global log_ssh: event(rec: Log);
+
+event bro_init()
+{
+	Log::create_stream(SSH, [$columns=Log, $ev=log_ssh]);
+
+    local cid = [$orig_h=1.2.3.4, $orig_p=1234/tcp, $resp_h=2.3.4.5, $resp_p=80/tcp];
+
+	Log::write(SSH, [$t=network_time(), $id=cid, $status="success"]);
+	Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="US"]);
+	Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="UK"]);
+	Log::write(SSH, [$t=network_time(), $id=cid, $status="success", $country="BR"]);
+	Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="MX"]);
+	
+}
+
--- a/testing/btest/scripts/base/frameworks/logging/events.bro
+++ b/testing/btest/scripts/base/frameworks/logging/events.bro
@ -0,0 +1,37 @@
+
+# @TEST-EXEC: bro -b %INPUT >output
+# @TEST-EXEC: btest-diff output
+
+module SSH;
+
+export {
+	# Create a new ID for our log stream
+	redef enum Log::ID += { SSH };
+
+	# Define a record with all the columns the log file can have.
+	# (I'm using a subset of fields from ssh-ext for demonstration.)
+	type Log: record {
+		t: time;
+		id: conn_id; # Will be rolled out into individual columns.
+		status: string &optional;
+		country: string &default="unknown";
+	} &log;
+}
+
+global ssh_log: event(rec: Log);
+
+event bro_init()
+{
+	Log::create_stream(SSH, [$columns=Log, $ev=ssh_log]);
+
+    local cid = [$orig_h=1.2.3.4, $orig_p=1234/tcp, $resp_h=2.3.4.5, $resp_p=80/tcp];
+	local r: Log = [$t=network_time(), $id=cid, $status="success"];
+	Log::write(SSH, r);
+	Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="US"]);
+	
+}
+
+event ssh_log(rec: Log)
+	{
+	print rec;
+	}
--- a/testing/btest/scripts/base/frameworks/logging/exclude.bro
+++ b/testing/btest/scripts/base/frameworks/logging/exclude.bro
@ -0,0 +1,34 @@
+#
+# @TEST-EXEC: bro -b %INPUT
+# @TEST-EXEC: btest-diff ssh.log
+
+module SSH;
+
+export {
+	redef enum Log::ID += { SSH };
+
+	type Log: record {
+		t: time;
+		id: conn_id; # Will be rolled out into individual columns.
+		status: string &optional;
+		country: string &default="unknown";
+	} &log;
+}
+
+event bro_init()
+{
+	Log::create_stream(SSH, [$columns=Log]);
+
+	Log::remove_default_filter(SSH);
+	Log::add_filter(SSH, [$name="f1", $exclude=set("t", "id.orig_h")]);
+
+	local cid = [$orig_h=1.2.3.4, $orig_p=1234/tcp, $resp_h=2.3.4.5, $resp_p=80/tcp];
+
+	Log::write(SSH, [$t=network_time(), $id=cid, $status="success"]);
+	Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="US"]);
+	Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="UK"]);
+	Log::write(SSH, [$t=network_time(), $id=cid, $status="success", $country="BR"]);
+	Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="MX"]);
+	
+}
+
--- a/testing/btest/scripts/base/frameworks/logging/file.bro
+++ b/testing/btest/scripts/base/frameworks/logging/file.bro
@ -0,0 +1,23 @@
+#
+# @TEST-EXEC: bro -b %INPUT
+# @TEST-EXEC: btest-diff ssh.log
+
+module SSH;
+
+export {
+	redef enum Log::ID += { SSH };
+
+	type Log: record {
+		t: time;
+		f: file;
+	} &log;
+}
+
+const foo_log = open_log_file("Foo") &redef;
+
+event bro_init()
+{
+	Log::create_stream(SSH, [$columns=Log]);
+	Log::write(SSH, [$t=network_time(), $f=foo_log]);
+}
+
--- a/testing/btest/scripts/base/frameworks/logging/include.bro
+++ b/testing/btest/scripts/base/frameworks/logging/include.bro
@ -0,0 +1,34 @@
+#
+# @TEST-EXEC: bro -b %INPUT
+# @TEST-EXEC: btest-diff ssh.log
+
+module SSH;
+
+export {
+	redef enum Log::ID += { SSH };
+
+	type Log: record {
+		t: time;
+		id: conn_id; # Will be rolled out into individual columns.
+		status: string &optional;
+		country: string &default="unknown";
+	} &log;
+}
+
+event bro_init()
+{
+	Log::create_stream(SSH, [$columns=Log]);
+
+	Log::remove_default_filter(SSH);
+	Log::add_filter(SSH, [$name="default", $include=set("t", "id.orig_h")]);
+
+	local cid = [$orig_h=1.2.3.4, $orig_p=1234/tcp, $resp_h=2.3.4.5, $resp_p=80/tcp];
+
+	Log::write(SSH, [$t=network_time(), $id=cid, $status="success"]);
+	Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="US"]);
+	Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="UK"]);
+	Log::write(SSH, [$t=network_time(), $id=cid, $status="success", $country="BR"]);
+	Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="MX"]);
+	
+}
+
--- a/testing/btest/scripts/base/frameworks/logging/no-local.bro
+++ b/testing/btest/scripts/base/frameworks/logging/no-local.bro
@ -0,0 +1,33 @@
+#
+# @TEST-EXEC: bro -b %INPUT
+# @TEST-EXEC: test '!' -e ssh.log
+
+module SSH;
+
+export {
+	redef enum Log::ID += { SSH };
+
+	type Log: record {
+		t: time;
+		id: conn_id; # Will be rolled out into individual columns.
+		status: string &optional;
+		country: string &default="unknown";
+	} &log;
+}
+
+redef Log::enable_local_logging = F;
+
+event bro_init()
+{
+	Log::create_stream(SSH, [$columns=Log]);
+
+    local cid = [$orig_h=1.2.3.4, $orig_p=1234/tcp, $resp_h=2.3.4.5, $resp_p=80/tcp];
+
+	Log::write(SSH, [$t=network_time(), $id=cid, $status="success"]);
+	Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="US"]);
+	Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="UK"]);
+	Log::write(SSH, [$t=network_time(), $id=cid, $status="success", $country="BR"]);
+	Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="MX"]);
+	
+}
+
--- a/testing/btest/scripts/base/frameworks/logging/path-func.bro
+++ b/testing/btest/scripts/base/frameworks/logging/path-func.bro
@ -0,0 +1,48 @@
+
+# @TEST-EXEC: bro -b %INPUT
+# @TEST-EXEC: ( ls static-*; cat static-* ) >output
+# @TEST-EXEC: btest-diff output
+
+module SSH;
+
+export {
+	# Create a new ID for our log stream
+	redef enum Log::ID += { SSH };
+
+	# Define a record with all the columns the log file can have.
+	# (I'm using a subset of fields from ssh-ext for demonstration.)
+	type Log: record {
+		t: time;
+		id: conn_id; # Will be rolled out into individual columns.
+		status: string &optional;
+		country: string &default="unknown";
+	} &log;
+}
+
+global c = -1;
+
+function path_func(id: Log::ID, path: string, rec: Log) : string
+	{
+	c = (c + 1) % 3;
+
+	return fmt("%s-%d-%s", path, c, rec$country);
+	}
+
+event bro_init()
+{
+	Log::create_stream(SSH, [$columns=Log]);
+	Log::remove_default_filter(SSH);
+
+	Log::add_filter(SSH, [$name="dyn", $path="static-prefix", $path_func=path_func]);
+
+	Log::set_buf(SSH, F);
+
+    local cid = [$orig_h=1.2.3.4, $orig_p=1234/tcp, $resp_h=2.3.4.5, $resp_p=80/tcp];
+	Log::write(SSH, [$t=network_time(), $id=cid, $status="success"]);
+	Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="US"]);
+	Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="UK"]);
+	Log::write(SSH, [$t=network_time(), $id=cid, $status="success", $country="BR"]);
+	Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="MX"]);
+	Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="MX2"]);
+	Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="MX3"]);
+}
--- a/testing/btest/scripts/base/frameworks/logging/pred.bro
+++ b/testing/btest/scripts/base/frameworks/logging/pred.bro
@ -0,0 +1,39 @@
+
+# @TEST-EXEC: bro -b %INPUT 
+# @TEST-EXEC: btest-diff ssh.success.log
+# @TEST-EXEC: btest-diff ssh.failure.log
+
+module SSH;
+
+export {
+	# Create a new ID for our log stream
+	redef enum Log::ID += { SSH };
+
+	# Define a record with all the columns the log file can have.
+	# (I'm using a subset of fields from ssh-ext for demonstration.)
+	type Log: record {
+		t: time;
+		id: conn_id; # Will be rolled out into individual columns.
+		status: string &optional;
+		country: string &default="unknown";
+	} &log;
+}
+
+function fail(rec: Log): bool
+	{
+	return rec$status != "success";
+	}
+
+event bro_init()
+{
+	Log::create_stream(SSH, [$columns=Log]);
+	Log::remove_default_filter(SSH);
+	Log::add_filter(SSH, [$name="f1", $path="ssh.success", $pred=function(rec: Log): bool { return rec$status == "success"; }]);
+	Log::add_filter(SSH, [$name="f2", $path="ssh.failure", $pred=fail]);
+
+    local cid = [$orig_h=1.2.3.4, $orig_p=1234/tcp, $resp_h=2.3.4.5, $resp_p=80/tcp];
+	local r: Log = [$t=network_time(), $id=cid, $status="success"];
+	Log::write(SSH, r);
+	Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="US"]);
+	
+}
--- a/testing/btest/scripts/base/frameworks/logging/remote-types.bro
+++ b/testing/btest/scripts/base/frameworks/logging/remote-types.bro
@ -0,0 +1,87 @@
+#
+# @TEST-EXEC: btest-bg-run sender bro --pseudo-realtime %INPUT ../sender.bro
+# @TEST-EXEC: btest-bg-run receiver bro --pseudo-realtime %INPUT ../receiver.bro
+# @TEST-EXEC: btest-bg-wait -k 1
+# @TEST-EXEC: btest-diff receiver/test.log
+# @TEST-EXEC: cmp receiver/test.log sender/test.log
+
+# Remote version testing all types.
+
+# This is the common part loaded by both sender and receiver.
+
+redef LogAscii::empty_field = "EMPTY";
+
+module Test;
+
+export {
+	# Create a new ID for our log stream
+	redef enum Log::ID += { TEST };
+
+	type Log: record {
+		b: bool;
+		i: int;
+		e: Log::ID;
+		c: count;
+		p: port;
+		sn: subnet;
+		a: addr;
+		d: double;
+		t: time;
+		iv: interval;
+		s: string;
+		sc: set[count];
+		ss: set[string];
+		se: set[string];
+		vc: vector of count;
+		ve: vector of string;
+	} &log;
+}
+
+event bro_init()
+{
+	Log::create_stream(TEST, [$columns=Log]);
+}
+
+#####
+
+@TEST-START-FILE sender.bro
+
+module Test;
+
+@load frameworks/communication/listen-clear
+
+event remote_connection_handshake_done(p: event_peer)
+	{
+	local empty_set: set[string];
+	local empty_vector: vector of string;
+
+	Log::write(TEST, [
+		$b=T,
+		$i=-42,
+		$e=TEST,
+		$c=21,
+		$p=123/tcp,
+		$sn=10.0.0.1/24,
+		$a=1.2.3.4,
+		$d=3.14,
+		$t=network_time(),
+		$iv=100secs,
+		$s="hurz",
+		$sc=set(1,2,3,4),
+		$ss=set("AA", "BB", "CC"),
+		$se=empty_set,
+		$vc=vector(10, 20, 30),
+		$ve=empty_vector
+		]);
+	}
+@TEST-END-FILE
+
+@TEST-START-FILE receiver.bro
+
+#####
+
+redef Communication::nodes += {
+    ["foo"] = [$host = 127.0.0.1, $connect=T, $request_logs=T]
+};
+
+@TEST-END-FILE
--- a/testing/btest/scripts/base/frameworks/logging/remote.bro
+++ b/testing/btest/scripts/base/frameworks/logging/remote.bro
@ -0,0 +1,75 @@
+#
+# @TEST-EXEC: btest-bg-run sender bro --pseudo-realtime %INPUT ../sender.bro
+# @TEST-EXEC: sleep 1
+# @TEST-EXEC: btest-bg-run receiver bro --pseudo-realtime %INPUT ../receiver.bro
+# @TEST-EXEC: sleep 1
+# @TEST-EXEC: btest-bg-wait -k 1
+# @TEST-EXEC: btest-diff sender/test.log
+# @TEST-EXEC: btest-diff sender/test.failure.log
+# @TEST-EXEC: btest-diff sender/test.success.log
+# @TEST-EXEC: cmp receiver/test.log sender/test.log
+# @TEST-EXEC: cmp receiver/test.failure.log sender/test.failure.log
+# @TEST-EXEC: cmp receiver/test.success.log sender/test.success.log
+
+# This is the common part loaded by both sender and receiver.
+module Test;
+
+export {
+	# Create a new ID for our log stream
+	redef enum Log::ID += { TEST };
+
+	# Define a record with all the columns the log file can have.
+	# (I'm using a subset of fields from ssh-ext for demonstration.)
+	type Log: record {
+		t: time;
+		id: conn_id; # Will be rolled out into individual columns.
+		status: string &optional;
+		country: string &default="unknown";
+	} &log;
+}
+
+event bro_init()
+{
+	Log::create_stream(TEST, [$columns=Log]);
+	Log::add_filter(TEST, [$name="f1", $path="test.success", $pred=function(rec: Log): bool { return rec$status == "success"; }]);
+}
+
+#####
+
+@TEST-START-FILE sender.bro
+
+module Test;
+
+@load frameworks/communication/listen-clear
+
+function fail(rec: Log): bool
+	{
+	return rec$status != "success";
+	}
+
+event remote_connection_handshake_done(p: event_peer)
+	{
+	Log::add_filter(TEST, [$name="f2", $path="test.failure", $pred=fail]);
+
+    local cid = [$orig_h=1.2.3.4, $orig_p=1234/tcp, $resp_h=2.3.4.5, $resp_p=80/tcp];
+
+	local r: Log = [$t=network_time(), $id=cid, $status="success"];
+
+	# Log something.
+	Log::write(TEST, r);
+	Log::write(TEST, [$t=network_time(), $id=cid, $status="failure", $country="US"]);
+	Log::write(TEST, [$t=network_time(), $id=cid, $status="failure", $country="UK"]);
+	Log::write(TEST, [$t=network_time(), $id=cid, $status="success", $country="BR"]);
+	Log::write(TEST, [$t=network_time(), $id=cid, $status="failure", $country="MX"]);
+	}
+@TEST-END-FILE
+
+@TEST-START-FILE receiver.bro
+
+#####
+
+redef Communication::nodes += {
+    ["foo"] = [$host = 127.0.0.1, $connect=T, $request_logs=T]
+};
+
+@TEST-END-FILE
--- a/testing/btest/scripts/base/frameworks/logging/remove.bro
+++ b/testing/btest/scripts/base/frameworks/logging/remove.bro
@ -0,0 +1,41 @@
+#
+# @TEST-EXEC: bro -b -B logging %INPUT
+# @TEST-EXEC: btest-diff ssh.log
+# @TEST-EXEC: btest-diff ssh.failure.log
+
+module SSH;
+
+export {
+	# Create a new ID for our log stream
+	redef enum Log::ID += { SSH };
+
+	# Define a record with all the columns the log file can have.
+	# (I'm using a subset of fields from ssh-ext for demonstration.)
+	type Log: record {
+		t: time;
+		id: conn_id; # Will be rolled out into individual columns.
+		status: string &optional;
+		country: string &default="unknown";
+	} &log;
+}
+
+event bro_init()
+{
+	Log::create_stream(SSH, [$columns=Log]);
+	Log::add_filter(SSH, [$name="f1", $path="ssh.failure", $pred=function(rec: Log): bool { return rec$status == "failure"; }]);
+
+    local cid = [$orig_h=1.2.3.4, $orig_p=1234/tcp, $resp_h=2.3.4.5, $resp_p=80/tcp];
+
+	# Log something.
+	Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="US"]);
+	Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="UK"]);
+
+	Log::remove_filter(SSH, "f1");
+	Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="BR"]);
+
+	Log::remove_filter(SSH, "default");
+	Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="MX"]);
+
+	Log::remove_filter(SSH, "doesn-not-exist");
+}
+
--- a/testing/btest/scripts/base/frameworks/logging/rotate-custom.bro
+++ b/testing/btest/scripts/base/frameworks/logging/rotate-custom.bro
@ -0,0 +1,43 @@
+#
+# @TEST-EXEC: bro -b -r %DIR/rotation.trace %INPUT 2>&1 | egrep "test|test2" | sort >out
+# @TEST-EXEC: for i in `ls test*.log | sort`; do printf '> %s\n' $i; cat $i; done | sort | uniq >>out
+# @TEST-EXEC: btest-diff out
+
+module Test;
+
+export {
+	# Create a new ID for our log stream
+	redef enum Log::ID += { Test };
+
+	# Define a record with all the columns the log file can have.
+	# (I'm using a subset of fields from ssh-ext for demonstration.)
+	type Log: record {
+		t: time;
+		id: conn_id; # Will be rolled out into individual columns.
+	} &log;
+}
+
+redef Log::default_rotation_interval = 1hr;
+redef Log::default_rotation_postprocessor_cmd = "echo 1st";
+
+function custom_rotate(info: Log::RotationInfo) : bool
+{
+    print "custom rotate", info;
+    return T;
+}
+
+redef Log::rotation_control += {
+	[Log::WRITER_ASCII, "test2"] = [$interv=30mins, $postprocessor=custom_rotate]
+};
+
+event bro_init()
+{
+	Log::create_stream(Test, [$columns=Log]);
+	Log::add_filter(Test, [$name="2nd", $path="test2"]);
+
+}
+
+event new_connection(c: connection)
+	{
+	Log::write(Test, [$t=network_time(), $id=c$id]);
+	}
--- a/testing/btest/scripts/base/frameworks/logging/rotate.bro
+++ b/testing/btest/scripts/base/frameworks/logging/rotate.bro
@ -0,0 +1,31 @@
+#
+# @TEST-EXEC: bro -r %DIR/rotation.trace %INPUT 2>&1 | grep "test" >out
+# @TEST-EXEC: for i in test.*.log; do printf '> %s\n' $i; cat $i; done >>out
+# @TEST-EXEC: btest-diff out
+
+module Test;
+
+export {
+	# Create a new ID for our log stream
+	redef enum Log::ID += { Test };
+
+	# Define a record with all the columns the log file can have.
+	# (I'm using a subset of fields from ssh-ext for demonstration.)
+	type Log: record {
+		t: time;
+		id: conn_id; # Will be rolled out into individual columns.
+	} &log;
+}
+
+redef Log::default_rotation_interval = 1hr;
+redef Log::default_rotation_postprocessor_cmd = "echo";
+
+event bro_init()
+{
+	Log::create_stream(Test, [$columns=Log]);
+}
+
+event new_connection(c: connection)
+	{
+	Log::write(Test, [$t=network_time(), $id=c$id]);
+	}
--- a/testing/btest/scripts/base/frameworks/logging/rotation.trace
+++ b/testing/btest/scripts/base/frameworks/logging/rotation.trace
--- a/testing/btest/scripts/base/frameworks/logging/stdout.bro
+++ b/testing/btest/scripts/base/frameworks/logging/stdout.bro
@ -0,0 +1,36 @@
+#
+# @TEST-EXEC: bro -b %INPUT >output
+# @TEST-EXEC: btest-diff output
+# @TEST-EXEC: test '!' -e ssh.log
+
+module SSH;
+
+export {
+	redef enum Log::ID += { SSH };
+
+	type Log: record {
+		t: time;
+		id: conn_id; # Will be rolled out into individual columns.
+		status: string &optional;
+		country: string &default="unknown";
+	} &log;
+}
+
+event bro_init()
+{
+	Log::create_stream(SSH, [$columns=Log]);
+
+	local filter = Log::get_filter(SSH, "default");
+	filter$path= "/dev/stdout";
+	Log::add_filter(SSH, filter);
+
+    local cid = [$orig_h=1.2.3.4, $orig_p=1234/tcp, $resp_h=2.3.4.5, $resp_p=80/tcp];
+
+	Log::write(SSH, [$t=network_time(), $id=cid, $status="success"]);
+	Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="US"]);
+	Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="UK"]);
+	Log::write(SSH, [$t=network_time(), $id=cid, $status="success", $country="BR"]);
+	Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="MX"]);
+	
+}
+
--- a/testing/btest/scripts/base/frameworks/logging/test-logging.bro
+++ b/testing/btest/scripts/base/frameworks/logging/test-logging.bro
@ -0,0 +1,31 @@
+#
+# @TEST-EXEC: bro -b %INPUT
+# @TEST-EXEC: btest-diff ssh.log
+
+module SSH;
+
+export {
+	redef enum Log::ID += { SSH };
+
+	type Log: record {
+		t: time;
+		id: conn_id; # Will be rolled out into individual columns.
+		status: string &optional;
+		country: string &default="unknown";
+	} &log;
+}
+
+event bro_init()
+{
+	Log::create_stream(SSH, [$columns=Log]);
+
+    local cid = [$orig_h=1.2.3.4, $orig_p=1234/tcp, $resp_h=2.3.4.5, $resp_p=80/tcp];
+
+	Log::write(SSH, [$t=network_time(), $id=cid, $status="success"]);
+	Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="US"]);
+	Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="UK"]);
+	Log::write(SSH, [$t=network_time(), $id=cid, $status="success", $country="BR"]);
+	Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="MX"]);
+	
+}
+
--- a/testing/btest/scripts/base/frameworks/logging/types.bro
+++ b/testing/btest/scripts/base/frameworks/logging/types.bro
@ -0,0 +1,70 @@
+#
+# @TEST-EXEC: bro -b %INPUT
+# @TEST-EXEC: btest-diff ssh.log
+#
+# Testing all possible types.
+
+redef LogAscii::empty_field = "EMPTY";
+
+module SSH;
+
+export {
+	redef enum Log::ID += { SSH };
+
+	type Log: record {
+		b: bool;
+		i: int;
+		e: Log::ID;
+		c: count;
+		p: port;
+		sn: subnet;
+		a: addr;
+		d: double;
+		t: time;
+		iv: interval;
+		s: string;
+		sc: set[count];
+		ss: set[string];
+		se: set[string];
+		vc: vector of count;
+		ve: vector of string;
+		f: function(i: count) : string;
+	} &log;
+}
+
+function foo(i : count) : string
+	{
+	if ( i > 0 )
+		return "Foo";
+	else
+		return "Bar";
+	}
+
+event bro_init()
+{
+	Log::create_stream(SSH, [$columns=Log]);
+
+	local empty_set: set[string];
+	local empty_vector: vector of string;
+
+	Log::write(SSH, [
+		$b=T,
+		$i=-42,
+		$e=SSH,
+		$c=21,
+		$p=123/tcp,
+		$sn=10.0.0.1/24,
+		$a=1.2.3.4,
+		$d=3.14,
+		$t=network_time(),
+		$iv=100secs,
+		$s="hurz",
+		$sc=set(1,2,3,4),
+		$ss=set("AA", "BB", "CC"),
+		$se=empty_set,
+		$vc=vector(10, 20, 30),
+		$ve=empty_vector,
+		$f=foo
+		]);
+}
+
--- a/testing/btest/scripts/base/frameworks/logging/unset-record.bro
+++ b/testing/btest/scripts/base/frameworks/logging/unset-record.bro
@ -0,0 +1,28 @@
+#
+# @TEST-EXEC: bro -b %INPUT
+# @TEST-EXEC: btest-diff testing.log
+
+redef enum Log::ID += { TESTING };
+
+type Foo: record {
+	val1: count;
+	val2: count;
+} &log;
+
+type Bar: record {
+	a: Foo   &log &optional;
+	b: count &log;
+};
+
+event bro_init()
+{
+    Log::create_stream(TESTING, [$columns=Bar]);
+
+    local x: Bar;
+
+    x = [$b=6];
+    Log::write(TESTING, x);
+
+    x = [$a=[$val1=1,$val2=2], $b=3];
+    Log::write(TESTING, x);
+}
--- a/testing/btest/scripts/base/frameworks/logging/vec.bro
+++ b/testing/btest/scripts/base/frameworks/logging/vec.bro
@ -0,0 +1,27 @@
+#
+# @TEST-EXEC: bro -b %INPUT
+# @TEST-EXEC: btest-diff ssh.log
+
+module SSH;
+
+export {
+	redef enum Log::ID += { SSH };
+
+	type Log: record {
+        vec: vector of string &log;
+	};
+}
+
+event bro_init()
+{
+	Log::create_stream(SSH, [$columns=Log]);
+
+    local v: vector of string;
+
+	v[1] = "2";
+	v[4] = "5";
+
+	Log::write(SSH, [$vec=v]);
+}
+
+
--- a/testing/btest/scripts/base/frameworks/metrics/basic-cluster.bro
+++ b/testing/btest/scripts/base/frameworks/metrics/basic-cluster.bro
@ -0,0 +1,34 @@
+# @TEST-EXEC: btest-bg-run manager-1 BROPATH=$BROPATH:.. CLUSTER_NODE=manager-1 bro %INPUT
+# @TEST-EXEC: btest-bg-run proxy-1   BROPATH=$BROPATH:.. CLUSTER_NODE=proxy-1 bro %INPUT
+# @TEST-EXEC: sleep 1
+# @TEST-EXEC: btest-bg-run worker-1  BROPATH=$BROPATH:.. CLUSTER_NODE=worker-1 bro %INPUT
+# @TEST-EXEC: btest-bg-run worker-2  BROPATH=$BROPATH:.. CLUSTER_NODE=worker-2 bro %INPUT
+# @TEST-EXEC: btest-bg-wait -k 6
+# @TEST-EXEC: btest-diff manager-1/metrics.log
+
+@TEST-START-FILE cluster-layout.bro
+redef Cluster::nodes = {
+	["manager-1"] = [$node_type=Cluster::MANAGER, $ip=127.0.0.1, $p=37757/tcp, $workers=set("worker-1")],
+	["proxy-1"] = [$node_type=Cluster::PROXY,     $ip=127.0.0.1, $p=37758/tcp, $manager="manager-1", $workers=set("worker-1")],
+	["worker-1"] = [$node_type=Cluster::WORKER,   $ip=127.0.0.1, $p=37760/tcp, $manager="manager-1", $proxy="proxy-1", $interface="eth0"],
+	["worker-2"] = [$node_type=Cluster::WORKER,   $ip=127.0.0.1, $p=37761/tcp, $manager="manager-1", $proxy="proxy-1", $interface="eth1"],
+};
+@TEST-END-FILE
+
+redef enum Metrics::ID += {
+	TEST_METRIC,
+};
+
+event bro_init() &priority=5
+	{
+	Metrics::add_filter(TEST_METRIC, 
+		[$name="foo-bar",
+		 $break_interval=3secs]);
+		
+	if ( Cluster::local_node_type() == Cluster::WORKER )
+		{
+		Metrics::add_data(TEST_METRIC, [$host=1.2.3.4], 3);
+		Metrics::add_data(TEST_METRIC, [$host=6.5.4.3], 2);
+		Metrics::add_data(TEST_METRIC, [$host=7.2.1.5], 1);
+		}
+	}
--- a/testing/btest/scripts/base/frameworks/metrics/basic.bro
+++ b/testing/btest/scripts/base/frameworks/metrics/basic.bro
@ -0,0 +1,16 @@
+# @TEST-EXEC: bro %INPUT
+# @TEST-EXEC: btest-diff metrics.log
+
+redef enum Metrics::ID += {
+	TEST_METRIC,
+};
+
+event bro_init() &priority=5
+	{
+	Metrics::add_filter(TEST_METRIC, 
+		[$name="foo-bar",
+		 $break_interval=3secs]);
+	Metrics::add_data(TEST_METRIC, [$host=1.2.3.4], 3);
+	Metrics::add_data(TEST_METRIC, [$host=6.5.4.3], 2);
+	Metrics::add_data(TEST_METRIC, [$host=7.2.1.5], 1);
+	}
--- a/testing/btest/scripts/base/frameworks/metrics/notice.bro
+++ b/testing/btest/scripts/base/frameworks/metrics/notice.bro
@ -0,0 +1,24 @@
+# @TEST-EXEC: bro %INPUT
+# @TEST-EXEC: btest-diff notice.log
+
+
+redef enum Notice::Type += {
+	Test_Notice,
+};
+
+redef enum Metrics::ID += {
+	TEST_METRIC,
+};
+
+event bro_init() &priority=5
+	{
+	Metrics::add_filter(TEST_METRIC, 
+		[$name="foo-bar",
+		 $break_interval=3secs,
+		 $note=Test_Notice,
+		 $notice_threshold=2,
+		 $log=F]);
+	Metrics::add_data(TEST_METRIC, [$host=1.2.3.4], 3);
+	Metrics::add_data(TEST_METRIC, [$host=6.5.4.3], 2);
+	Metrics::add_data(TEST_METRIC, [$host=7.2.1.5], 1);
+	}
--- a/testing/btest/scripts/base/frameworks/software/version-parsing.bro
+++ b/testing/btest/scripts/base/frameworks/software/version-parsing.bro
@ -0,0 +1,113 @@
+# @TEST-EXEC: bro %INPUT > output
+# @TEST-EXEC: btest-diff output
+
+global ts = network_time();
+global host = 0.0.0.0;
+
+global matched_software: table[string] of Software::Info = {
+	["OpenSSH_4.4"] = 
+		[$name="OpenSSH", $version=[$major=4,$minor=4], $host=host, $ts=ts],
+	["OpenSSH_5.2"] = 
+		[$name="OpenSSH", $version=[$major=5,$minor=2], $host=host, $ts=ts],
+	["Apache/2.0.63 (Unix) mod_auth_kerb/5.3 mod_ssl/2.0.63 OpenSSL/0.9.7a mod_fastcgi/2.4.2"] =
+		[$name="Apache", $version=[$major=2,$minor=0,$minor2=63,$addl="Unix"], $host=host, $ts=ts],
+	["Apache/1.3.19 (Unix)"] =
+		[$name="Apache", $version=[$major=1,$minor=3,$minor2=19,$addl="Unix"], $host=host, $ts=ts],
+	["ProFTPD 1.2.5rc1 Server (Debian)"] =
+		[$name="ProFTPD", $version=[$major=1,$minor=2,$minor2=5,$addl="rc1"], $host=host, $ts=ts],
+	["wu-2.4.2-academ[BETA-18-VR14](1)"] = 
+		[$name="wu", $version=[$major=2,$minor=4,$minor2=2,$addl="academ"], $host=host, $ts=ts],
+	["wu-2.6.2(1)"] =
+		[$name="wu", $version=[$major=2,$minor=6,$minor2=2,$addl="1"], $host=host, $ts=ts],
+	["Java1.2.2-JDeveloper"] =
+		[$name="Java", $version=[$major=1,$minor=2,$minor2=2,$addl="JDeveloper"], $host=host, $ts=ts],
+	["Java/1.6.0_13"] = 
+		[$name="Java", $version=[$major=1,$minor=6,$minor2=0,$addl="13"], $host=host, $ts=ts],
+	["Python-urllib/3.1"] = 
+		[$name="Python-urllib", $version=[$major=3,$minor=1], $host=host, $ts=ts],
+	["libwww-perl/5.820"] = 
+		[$name="libwww-perl", $version=[$major=5,$minor=820], $host=host, $ts=ts],
+	["Wget/1.9+cvs-stable (Red Hat modified)"] = 
+		[$name="Wget", $version=[$major=1,$minor=9,$addl="+cvs"], $host=host, $ts=ts],
+	["Wget/1.11.4 (Red Hat modified)"] = 
+		[$name="Wget", $version=[$major=1,$minor=11,$minor2=4,$addl="Red Hat modified"], $host=host, $ts=ts],
+	["curl/7.15.1 (i486-pc-linux-gnu) libcurl/7.15.1 OpenSSL/0.9.8a zlib/1.2.3 libidn/0.5.18"] =
+		[$name="curl", $version=[$major=7,$minor=15,$minor2=1,$addl="i486-pc-linux-gnu"], $host=host, $ts=ts],
+	["Apache"] = 
+		[$name="Apache", $host=host, $ts=ts],
+	["Zope/(Zope 2.7.8-final, python 2.3.5, darwin) ZServer/1.1 Plone/Unknown"] =
+		[$name="Zope/(Zope", $version=[$major=2,$minor=7,$minor2=8,$addl="final"], $host=host, $ts=ts],
+	["The Bat! (v2.00.9) Personal"] =
+		[$name="The Bat!", $version=[$major=2,$minor=0,$minor2=9,$addl="Personal"], $host=host, $ts=ts],
+	["Flash/10,2,153,1"] =
+		[$name="Flash", $version=[$major=10,$minor=2,$minor2=153,$addl="1"], $host=host, $ts=ts],
+	["mt2/1.2.3.967 Oct 13 2010-13:40:24 ord-pixel-x2 pid 0x35a3 13731"] = 
+		[$name="mt2", $version=[$major=1,$minor=2,$minor2=3,$addl="967"], $host=host, $ts=ts],
+	["CacheFlyServe v26b"] =
+		[$name="CacheFlyServe", $version=[$major=26,$addl="b"], $host=host, $ts=ts],
+	["Apache/2.0.46 (Win32) mod_ssl/2.0.46 OpenSSL/0.9.7b mod_jk2/2.0.4"] =
+		[$name="Apache", $version=[$major=2,$minor=0,$minor2=46,$addl="Win32"], $host=host, $ts=ts],
+	# I have no clue how I'd support this without a special case.
+	#["Apache mod_fcgid/2.3.6 mod_auth_passthrough/2.1 mod_bwlimited/1.4 FrontPage/5.0.2.2635"] =
+	#	[$name="Apache", $version=[], $host=host, $ts=ts],
+	["Apple iPhone v4.3.1 Weather v1.0.0.8G4"] =
+		[$name="Apple iPhone", $version=[$major=4,$minor=3,$minor2=1,$addl="Weather"], $host=host, $ts=ts],
+	["Mozilla/5.0 (iPhone; U; CPU iPhone OS 4_3_2 like Mac OS X; en-us) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8H7 Safari/6533.18.5"] =
+		[$name="Safari", $version=[$major=5,$minor=0,$minor2=2,$addl="Mobile"], $host=host, $ts=ts],
+	["Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_7; en-US) AppleWebKit/534.16 (KHTML, like Gecko) Chrome/10.0.648.205 Safari/534.16"] = 
+		[$name="Chrome", $version=[$major=10,$minor=0,$minor2=648,$addl="205"], $host=host, $ts=ts],
+	["Opera/9.80 (Windows NT 6.1; U; sv) Presto/2.7.62 Version/11.01"] =
+		[$name="Opera", $version=[$major=11,$minor=1], $host=host, $ts=ts],
+	["Mozilla/5.0 (Windows; U; Windows NT 5.1; de; rv:1.9.2.11) Gecko/20101013 Lightning/1.0b2 Thunderbird/3.1.5"] =
+		[$name="Thunderbird", $version=[$major=3,$minor=1,$minor2=5], $host=host, $ts=ts],
+	["iTunes/9.0 (Macintosh; Intel Mac OS X 10.5.8) AppleWebKit/531.9"] = 
+		[$name="iTunes", $version=[$major=9,$minor=0,$addl="Macintosh"], $host=host, $ts=ts],
+	["Java1.3.1_04"] =
+		[$name="Java", $version=[$major=1,$minor=3,$minor2=1,$addl="04"], $host=host, $ts=ts],
+	["Mozilla/5.0 (Linux; U; Android 2.3.3; zh-tw; HTC Pyramid Build/GRI40) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1"] = 
+		[$name="Safari", $version=[$major=4,$minor=0,$addl="Mobile"], $host=host, $ts=ts],
+	["Opera/9.80 (J2ME/MIDP; Opera Mini/9.80 (S60; SymbOS; Opera Mobi/23.348; U; en) Presto/2.5.25 Version/10.54"] = 
+		[$name="Opera Mini", $version=[$major=10,$minor=54], $host=host, $ts=ts],
+	["Opera/9.80 (J2ME/MIDP; Opera Mini/5.0.18741/18.794; U; en) Presto/2.4.15"] =
+		[$name="Opera Mini", $version=[$major=5,$minor=0,$minor2=18741], $host=host, $ts=ts],
+	["Opera/9.80 (Windows NT 5.1; Opera Mobi/49; U; en) Presto/2.4.18 Version/10.00"] =
+		[$name="Opera Mobi", $version=[$major=10,$minor=0], $host=host, $ts=ts],
+	["Mozilla/4.0 (compatible; MSIE 8.0; Android 2.2.2; Linux; Opera Mobi/ADR-1103311355; en) Opera 11.00"] =
+		[$name="Opera", $version=[$major=11,$minor=0], $host=host, $ts=ts],
+	["Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0; GTB5; SLCC1; .NET CLR 2.0.50727; Media Center PC 5.0; .NET CLR 3.0.04506; InfoPath.2)"] =
+		[$name="MSIE", $version=[$major=7,$minor=0], $host=host, $ts=ts],
+	["Mozilla/4.0 (compatible; MSIE 7.0b; Windows NT 5.1; Media Center PC 3.0; .NET CLR 1.0.3705; .NET CLR 1.1.4322; .NET CLR 2.0.50727; InfoPath.1)"] =
+		[$name="MSIE", $version=[$major=7,$minor=0,$addl="b"], $host=host, $ts=ts],
+	["Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.7.2) Gecko/20040804 Netscape/7.2 (ax)"] =
+		[$name="Netscape", $version=[$major=7,$minor=2], $host=host, $ts=ts],
+	["The Bat! (3.0.1 RC3) Professional"] =
+		[$name="The Bat!", $version=[$major=3,$minor=0,$minor2=1,$addl="RC3"], $host=host, $ts=ts],
+	# This is an FTP client (found with CLNT command)
+	["Total Commander"] =
+		[$name="Total Commander", $version=[], $host=host, $ts=ts],
+	["(vsFTPd 2.0.5)"] =
+		[$name="vsFTPd", $version=[$major=2,$minor=0,$minor2=5], $host=host, $ts=ts],
+	["Apple Mail (2.1084)"] = 
+		[$name="Apple Mail", $version=[$major=2,$minor=1084], $host=host, $ts=ts],
+};
+
+event bro_init()
+	{
+	for ( sw in matched_software )
+		{
+		local output = Software::parse(sw, host, Software::UNKNOWN);
+		local baseline: Software::Info;
+		baseline = matched_software[sw];
+		if ( baseline$name == output$name &&
+		     Software::cmp_versions(baseline$version,output$version) == 0 )
+			print fmt("success on: %s", sw);
+		else
+			{
+			print fmt("failure on: %s", sw);
+			print fmt("    test name:        %s", output$name);
+			print fmt("    test version:     %s", output$version);
+			print fmt("    baseline name:    %s", baseline$name);
+			print fmt("    baseline version: %s", baseline$version);
+			}
+        }
+	}
--- a/testing/btest/scripts/base/init-default-coverage.bro
+++ b/testing/btest/scripts/base/init-default-coverage.bro
@ -0,0 +1,11 @@
+# Makes sure that all base/* scripts are loaded by default via init-default.bro;
+# and that all scripts loaded there in there actually exist.
+
+@TEST-EXEC: test -d $DIST/scripts/base
+@TEST-EXEC: test -e $DIST/scripts/base/init-default.bro
+@TEST-EXEC: ( cd $DIST/scripts/base && find . -name '*.bro' ) | sort >"all scripts found"
+@TEST-EXEC: bro misc/loaded-scripts
+@TEST-EXEC: cat loaded_scripts.log | egrep -v '/build/|/loaded-scripts.bro' | awk 'NR>1{print $2}' | sed 's#/./#/#g'  >loaded_scripts.log.tmp
+@TEST-EXEC: cat loaded_scripts.log.tmp | sed -e ':a' -e '$!N' -e 's/^\(.*\).*\n\1.*/\1/' -e 'ta' >prefix
+@TEST-EXEC: cat loaded_scripts.log.tmp | sed "s#`cat prefix`#./#g" | sort >init-default.bro
+@TEST-EXEC: diff -u "all scripts found" init-default.bro 1>&2
--- a/testing/btest/scripts/base/protocols/http/http-header-crlf.bro
+++ b/testing/btest/scripts/base/protocols/http/http-header-crlf.bro
@ -0,0 +1,10 @@
+# This tests for what looks like a problem in the HTTP parser:
+# it gets confused whether it's in a header or not; it should
+# not report that weird.
+#
+# @TEST-EXEC: bro -r $TRACES/http-byteranges.trace %INPUT
+# @TEST-EXEC: grep -q http_no_crlf_in_header_list weird.log && exit 1 || exit 0
+
+# The base analysis scripts are loaded by default.
+#@load base/protocols/http
+
--- a/testing/btest/scripts/base/protocols/http/http-mime-and-md5.bro
+++ b/testing/btest/scripts/base/protocols/http/http-mime-and-md5.bro
@ -0,0 +1,22 @@
+# This tests md5 calculation for a specified mime type.  The http.log
+# will normalize mime types other than the target type to prevent sensitivity
+# to varying versions of libmagic.
+
+# @TEST-REQUIRES: grep -q '#define HAVE_LIBMAGIC' $BUILD/config.h
+# @TEST-EXEC: bro -r $TRACES/http-pipelined-requests.trace %INPUT > output
+# @TEST-EXEC: btest-diff http.log
+
+redef HTTP::generate_md5 += /image\/png/;
+
+event bro_init()
+	{
+	Log::remove_default_filter(HTTP::HTTP);
+	Log::add_filter(HTTP::HTTP, [$name="normalized-mime-types",
+	                             $pred=function(rec: HTTP::Info): bool
+		{
+		if ( rec?$mime_type && HTTP::generate_md5 != rec$mime_type )
+				rec$mime_type = "FAKE_MIME";
+		return T;
+		}
+	]);
+	}
--- a/testing/btest/scripts/base/protocols/http/http-pipelining.bro
+++ b/testing/btest/scripts/base/protocols/http/http-pipelining.bro
@ -0,0 +1,9 @@
+# @TEST-EXEC: bro -r $TRACES/http-pipelined-requests.trace %INPUT > output
+# @TEST-EXEC: btest-diff http.log
+
+# mime type is irrelevant to this test, so filter it out
+event bro_init()
+	{
+	Log::remove_default_filter(HTTP::HTTP);
+	Log::add_filter(HTTP::HTTP, [$name="less-mime-types", $exclude=set("mime_type")]);
+	}
--- a/testing/btest/scripts/base/protocols/irc/basic.test
+++ b/testing/btest/scripts/base/protocols/irc/basic.test
@ -0,0 +1,12 @@
+# This tests that basic IRC commands (NICK, USER, JOIN, DCC SEND)
+# are logged for a client.
+
+# @TEST-EXEC: bro -r $TRACES/irc-dcc-send.trace %INPUT
+# @TEST-EXEC: btest-diff irc.log
+
+# dcc mime types are irrelevant to this test, so filter it out
+event bro_init()
+	{
+	Log::remove_default_filter(IRC::IRC);
+	Log::add_filter(IRC::IRC, [$name="remove-mime", $exclude=set("dcc_mime_type")]);
+	}
--- a/testing/btest/scripts/base/protocols/irc/dcc-extract.test
+++ b/testing/btest/scripts/base/protocols/irc/dcc-extract.test
@ -0,0 +1,27 @@
+# This tests that the contents of a DCC transfer negotiated with IRC can be
+# correctly extracted.  The mime type of the file transferred is normalized
+# to prevent sensitivity to libmagic version being used.
+
+# @TEST-REQUIRES: grep -q '#define HAVE_LIBMAGIC' $BUILD/config.h
+# @TEST-EXEC: bro -r $TRACES/irc-dcc-send.trace %INPUT
+# @TEST-EXEC: btest-diff irc.log
+# @TEST-EXEC: btest-diff irc-dcc-item_192.168.1.77:57655-209.197.168.151:1024_1.dat
+# @TEST-EXEC: bro -r $TRACES/irc-dcc-send.trace %INPUT IRC::extraction_prefix="test"
+# @TEST-EXEC: test -e test_192.168.1.77:57655-209.197.168.151:1024_1.dat
+
+redef IRC::extract_file_types=/.*/;
+
+event bro_init()
+	{
+	Log::remove_default_filter(IRC::IRC);
+	Log::add_filter(IRC::IRC, [$name="normalized-mime-types",
+	                           $pred=function(rec: IRC::Info): bool
+		{
+		if ( rec?$dcc_mime_type )
+			{
+			rec$dcc_mime_type = "FAKE_MIME";
+			}
+		return T;
+		}
+	]);
+	}
--- a/testing/btest/scripts/base/protocols/smtp/basic.test
+++ b/testing/btest/scripts/base/protocols/smtp/basic.test
@ -0,0 +1,4 @@
+# @TEST-EXEC: bro -r $TRACES/smtp.trace %INPUT
+# @TEST-EXEC: btest-diff smtp.log
+
+@load base/protocols/smtp
--- a/testing/btest/scripts/base/protocols/smtp/mime-extract.test
+++ b/testing/btest/scripts/base/protocols/smtp/mime-extract.test
@ -0,0 +1,25 @@
+# @TEST-REQUIRES: grep -q '#define HAVE_LIBMAGIC' $BUILD/config.h
+# @TEST-EXEC: bro -r $TRACES/smtp.trace %INPUT
+# @TEST-EXEC: btest-diff smtp_entities.log
+# @TEST-EXEC: btest-diff smtp-entity_10.10.1.4:1470-74.53.140.153:25_1.dat
+# @TEST-EXEC: btest-diff smtp-entity_10.10.1.4:1470-74.53.140.153:25_2.dat
+# @TEST-EXEC: bro -r $TRACES/smtp.trace %INPUT SMTP::extraction_prefix="test"
+# @TEST-EXEC: test -e test_10.10.1.4:1470-74.53.140.153:25_1.dat
+# @TEST-EXEC: test -e test_10.10.1.4:1470-74.53.140.153:25_2.dat
+
+@load base/protocols/smtp
+
+redef SMTP::extract_file_types=/text\/plain/;
+
+event bro_init()
+	{
+	Log::remove_default_filter(SMTP::SMTP_ENTITIES);
+	Log::add_filter(SMTP::SMTP_ENTITIES, [$name="normalized-mime-types",
+									  	$pred=function(rec: SMTP::EntityInfo): bool
+		{
+		if ( rec?$mime_type )
+			rec$mime_type = "FAKE_MIME";
+		return T;
+		}
+	]);
+	}
--- a/testing/btest/scripts/base/protocols/smtp/mime.test
+++ b/testing/btest/scripts/base/protocols/smtp/mime.test
@ -0,0 +1,23 @@
+# Checks logging of mime types and md5 calculation.  Mime type in the log
+# is normalized to prevent sensitivity to libmagic version.
+
+# @TEST-REQUIRES: grep -q '#define HAVE_LIBMAGIC' $BUILD/config.h
+# @TEST-EXEC: bro -r $TRACES/smtp.trace %INPUT
+# @TEST-EXEC: btest-diff smtp_entities.log
+
+@load base/protocols/smtp
+
+redef SMTP::generate_md5=/text\/plain/;
+
+event bro_init()
+	{
+	Log::remove_default_filter(SMTP::SMTP_ENTITIES);
+	Log::add_filter(SMTP::SMTP_ENTITIES, [$name="normalized-mime-types",
+									  $pred=function(rec: SMTP::EntityInfo): bool
+		{
+		if ( rec?$mime_type )
+			rec$mime_type = "FAKE_MIME";
+		return T;
+		}
+	]);
+	}
--- a/testing/btest/scripts/base/utils/addrs.test
+++ b/testing/btest/scripts/base/utils/addrs.test
@ -0,0 +1,105 @@
+# @TEST-EXEC: bro %INPUT > output
+# @TEST-EXEC: btest-diff output
+
+# This is loaded by default
+#@load base/utils/addrs
+
+event bro_init()
+	{
+	local ip = "0.0.0.0";
+
+	print "============ test ipv4 regex";
+	print ip == ipv4_addr_regex;
+	print is_valid_ip(ip);
+	ip = "1.1.1.1";
+	print ip == ipv4_addr_regex;
+	print is_valid_ip(ip);
+	ip = "255.255.255.255";
+	print ip == ipv4_addr_regex;
+	print is_valid_ip(ip);
+	ip = "255.255.255.256";
+	print ip == ipv4_addr_regex; # the regex doesn't check for 0-255
+	print is_valid_ip(ip);       # but is_valid_ip() will
+	ip = "255.255.255.255.255";
+	print ip == ipv4_addr_regex;
+	print is_valid_ip(ip);
+	ip = "192.168.1.100";
+	print ip == ipv4_addr_regex;
+	print is_valid_ip(ip);
+
+	print "============ test ipv6 regex";
+
+	ip = "2001:0db8:85a3:0000:0000:8a2e:0370:7334";
+	print is_valid_ip(ip);
+
+	# test for case insensitivity
+	ip = "2001:0DB8:85A3:0000:0000:8A2E:0370:7334";
+	print is_valid_ip(ip);
+
+	# any case mixture is allowed
+	ip = "2001:0dB8:85a3:0000:0000:8A2E:0370:7334";
+	print is_valid_ip(ip);
+
+	# leading zeroes of a 16-bit group may be omitted
+	ip = "2001:db8:85a3:0:0:8a2e:370:7334";
+	print is_valid_ip(ip);
+
+	# a single occurrence of consecutive groups of zeroes may be replaced by ::
+	ip = "2001:db8:85a3::8a2e:370:7334";
+	print is_valid_ip(ip);
+
+	# this should fail because we don't have enough 16-bit groups
+	ip = "2001:db8:85a3:8a2e:370:7334";
+	print is_valid_ip(ip);
+
+	# this should fail because of an invalid hex digit
+	ip = "2001:gb8:85a3::8a2e:370:7334";
+	print is_valid_ip(ip);
+
+	# this should fail because we have too many 16-bit groups
+	ip = "2001:0db8:85a3:0000:0000:8a2e:0370:7334:1111";
+	print is_valid_ip(ip);
+
+	# this should fail because one group isn't 16-bits
+	ip = "2001:0db8:85a3:0000:0000:8a2e00:0370:7334";
+	print is_valid_ip(ip);
+
+	# this should fail because we can't have more than one ::
+	ip = "2001::85a3::7334";
+	print is_valid_ip(ip);
+
+	# all zeroes should work
+	ip = "0:0:0:0:0:0:0:0";
+	print is_valid_ip(ip);
+
+	# all zeroes condensed should work
+	ip = "::";
+	print is_valid_ip(ip);
+
+	print "============ test ipv6-ipv4 hybrid regexes";
+
+	# hybrid ipv6-ipv4 address should work
+	ip = "2001:db8:0:0:0:FFFF:192.168.0.5";
+	print is_valid_ip(ip);
+
+	# hybrid ipv6-ipv4 address with zero ommission should work
+	ip = "2001:db8::FFFF:192.168.0.5";
+	print is_valid_ip(ip);
+
+	# hybrid format with more than six 16-bit groups should fail
+	ip = "2001:db8:0:0:0:0:FFFF:192.168.0.5";
+	print is_valid_ip(ip);
+
+	# hybrid format without a 4 octet ipv4 part should fail
+	ip = "2001:db8:0:0:0:FFFF:192.168.0";
+	print is_valid_ip(ip);
+
+	# hybrid format's ipv4 part should test that all octet's are 0-255
+	ip = "2001:db8:0:0:0:FFFF:192.168.0.256";
+	print is_valid_ip(ip);
+
+	print "============ test find_ip_addresses()";
+	print find_ip_addresses("this is 1.1.1.1 a test 2.2.2.2 string with ip addresses 3.3.3.3");
+	print find_ip_addresses("this is 1.1.1.1 a test 0:0:0:0:0:0:0:0 string with ip addresses 3.3.3.3");
+
+	}
--- a/testing/btest/scripts/base/utils/conn-ids.test
+++ b/testing/btest/scripts/base/utils/conn-ids.test
@ -0,0 +1,15 @@
+# @TEST-EXEC: bro %INPUT >output
+# @TEST-EXEC: btest-diff output
+
+# This is loaded by default.
+#@load base/utils/conn-ids
+
+global c: conn_id = [ $orig_h = 10.0.0.100, $orig_p = 10000,
+                      $resp_h = 10.0.0.200, $resp_p = 20000 ];
+
+print id_string(c);
+print reverse_id_string(c);
+print directed_id_string(c, T);
+print directed_id_string(c, F);
+print id_string(c) == directed_id_string(c, T);
+print reverse_id_string(c) == directed_id_string(c, F);
--- a/testing/btest/scripts/base/utils/directions-and-hosts.test
+++ b/testing/btest/scripts/base/utils/directions-and-hosts.test
@ -0,0 +1,73 @@
+# @TEST-EXEC: bro %INPUT >output
+# @TEST-EXEC: btest-diff output
+
+# These are loaded by default.
+#@load base/utils/site
+#@load base/utils/directions-and-hosts
+
+redef Site::local_nets += { 10.0.0.0/8 };
+
+global local_ip = 10.0.0.100;
+global remote_ip = 192.168.1.100;
+
+global local2local: conn_id = [
+    $orig_h = 10.0.0.100, $orig_p = 10000,
+    $resp_h = 10.0.0.200, $resp_p = 20000 ];
+
+global local2remote: conn_id = [
+    $orig_h = 10.0.0.100,    $orig_p = 10000,
+    $resp_h = 192.168.1.100, $resp_p = 20000 ];
+
+global remote2local: conn_id = [
+    $orig_h = 192.168.1.100,    $orig_p = 10000,
+    $resp_h = 10.0.0.100,       $resp_p = 20000 ];
+
+global remote2remote: conn_id = [
+    $orig_h = 192.168.1.100, $orig_p = 10000,
+    $resp_h = 192.168.1.200, $resp_p = 20000 ];
+
+function test_host(ip: addr, h: Host, expect: bool)
+	{
+	local result = addr_matches_host(ip, h);
+	print fmt("%s(%s) == %s: %s", h, ip, expect,
+	          result == expect ? "SUCCESS" : "FAIL");
+	}
+
+function test_dir(id: conn_id, d: Direction, expect: bool)
+	{
+	local result = id_matches_direction(id, d);
+	print fmt("%s(o: %s, r: %s) == %s: %s", d, id$orig_h, id$resp_h, expect,
+	          result == expect ? "SUCCESS" : "FAIL");
+	}
+
+event bro_init()
+	{
+	test_host(local_ip,  LOCAL_HOSTS,  T);
+	test_host(local_ip,  REMOTE_HOSTS, F);
+	test_host(local_ip,  ALL_HOSTS,    T);
+	test_host(local_ip,  NO_HOSTS,     F);
+	test_host(remote_ip, LOCAL_HOSTS,  F);
+	test_host(remote_ip, REMOTE_HOSTS, T);
+	test_host(remote_ip, ALL_HOSTS,    T);
+	test_host(remote_ip, NO_HOSTS,     F);
+
+	test_dir(local2local,   INBOUND,       F);
+	test_dir(local2remote,  INBOUND,       F);
+	test_dir(remote2local,  INBOUND,       T);
+	test_dir(remote2remote, INBOUND,       F);
+
+	test_dir(local2local,   OUTBOUND,      F);
+	test_dir(local2remote,  OUTBOUND,      T);
+	test_dir(remote2local,  OUTBOUND,      F);
+	test_dir(remote2remote, OUTBOUND,      F);
+
+	test_dir(local2local,   BIDIRECTIONAL, F);
+	test_dir(local2remote,  BIDIRECTIONAL, T);
+	test_dir(remote2local,  BIDIRECTIONAL, T);
+	test_dir(remote2remote, BIDIRECTIONAL, F);
+
+	test_dir(local2local,   NO_DIRECTION,  F);
+	test_dir(local2remote,  NO_DIRECTION,  F);
+	test_dir(remote2local,  NO_DIRECTION,  F);
+	test_dir(remote2remote, NO_DIRECTION,  F);
+	}
--- a/testing/btest/scripts/base/utils/files.test
+++ b/testing/btest/scripts/base/utils/files.test
@ -0,0 +1,13 @@
+# @TEST-EXEC: bro -r $TRACES/wikipedia.trace %INPUT >output
+# @TEST-EXEC: btest-diff output
+
+# This is loaded by default.
+#@load base/utils/files
+
+event connection_established(c: connection)
+	{
+	print generate_extraction_filename("test-prefix", c, "test-suffix");
+	print generate_extraction_filename("test-prefix", c, "");
+	print generate_extraction_filename("", c, "test-suffix");
+	print generate_extraction_filename("", c, "");
+	}
--- a/testing/btest/scripts/base/utils/numbers.test
+++ b/testing/btest/scripts/base/utils/numbers.test
@ -0,0 +1,13 @@
+# @TEST-EXEC: bro %INPUT >output
+# @TEST-EXEC: btest-diff output
+
+# This is loaded by default.
+#@load base/utils/numbers
+
+print extract_count("These aren't the numbers you're looking for.");
+print extract_count("13These aren't the numbers you're looking for.");
+print extract_count("13 These aren't the numbers you're looking for.");
+print extract_count("These aren't the 13 numbers you're looking for.");
+print extract_count("These aren't the numbers you're looking for.13");
+print extract_count("These aren't the numbers you're looking for. 13");
+print extract_count("These aren't the 1abc3 numbers you're looking for.");
--- a/testing/btest/scripts/base/utils/paths.test
+++ b/testing/btest/scripts/base/utils/paths.test
@ -0,0 +1,58 @@
+# @TEST-EXEC: bro %INPUT >output
+# @TEST-EXEC: btest-diff output
+
+# This is loaded by default.
+#@load base/utils/paths
+
+function test_extract(str: string, expect: string)
+	{
+	local result = extract_path(str);
+	print fmt("Given : %s", str);
+	print fmt("Expect: %s", expect);
+	print fmt("Result: %s", result);
+	print fmt("Result: %s", result == expect ? "SUCCESS" : "FAIL");
+	print "===============================";
+	}
+
+function test_compress(str: string, expect: string)
+	{
+	local result = compress_path(str);
+	print fmt("Given : %s", str);
+	print fmt("Expect: %s", expect);
+	print fmt("Result: %s", result);
+	print fmt("Result: %s", result == expect ? "SUCCESS" : "FAIL");
+	print "===============================";
+	}
+
+print "test compress_path()";
+print "===============================";
+test_compress("foo//bar", "foo/bar");
+test_compress("foo//bar/..", "foo");
+test_compress("foo/bar/../..", "");
+test_compress("foo//bar/../..", "");
+test_compress("/foo/../bar", "/bar");
+test_compress("/foo/../bar/..", "/");
+test_compress("/foo/baz/../..", "/");
+test_compress("../..", "../..");
+test_compress("foo/../../..", "../..");
+
+print "test extract_path()";
+print "===============================";
+test_extract("\"/this/is/a/dir\" is current directory", "/this/is/a/dir");
+test_extract("/this/is/a/dir is current directory", "/this/is/a/dir");
+test_extract("/this/is/a/dir\\ is\\ current\\ directory", "/this/is/a/dir\\ is\\ current\\ directory");
+test_extract("hey, /foo/bar/baz.bro is a cool script", "/foo/bar/baz.bro");
+test_extract("here's two dirs: /foo/bar and /foo/baz", "/foo/bar");
+
+print "test build_path_compressed()";
+print "===============================";
+print build_path_compressed("/home/bro/", "policy/somefile.bro");
+print build_path_compressed("/home/bro/", "/usr/local/bro/share/bro/somefile.bro");
+print build_path_compressed("/home/bro/", "/usr/local/bro/share/../../bro/somefile.bro");
+
+print "===============================";
+print "test build_full_path()";
+print "===============================";
+print build_path("/home/bro/", "policy/somefile.bro");
+print build_path("/home/bro/", "/usr/local/bro/share/bro/somefile.bro");
+
--- a/testing/btest/scripts/base/utils/pattern.test
+++ b/testing/btest/scripts/base/utils/pattern.test
@ -0,0 +1,17 @@
+# @TEST-EXEC: bro %INPUT >output
+# @TEST-EXEC: btest-diff output
+
+# This is loaded by default.
+#@load base/utils/pattern
+
+global r1 = set_to_regex(set("blah", "bleh", "blarg"), "(~~)");
+global r2 = set_to_regex(set("blah", "bleh", "blarg"), "foo(~~)bar");
+
+print r1;
+print "blah" == r1;
+
+print r2;
+print "fooblargbar" == r2;
+
+print match_pattern("123blah123", r1);
+print match_pattern("no match here", r1);
--- a/testing/btest/scripts/base/utils/site.test
+++ b/testing/btest/scripts/base/utils/site.test
@ -0,0 +1,19 @@
+# @TEST-EXEC: bro %INPUT > output
+# @TEST-EXEC: btest-diff output
+
+# This is loaded by default.
+#@load base/utils/site
+
+global a = { "site-admin@example.com", "other-site-admin@example.com" };
+global b = { "net-admin@example.com" };
+
+redef Site::local_admins += {
+    [141.142.0.0/16] = a,
+    [141.142.100.0/24] = b,
+};
+
+event bro_init()
+    {
+    print Site::get_emails(141.142.1.1);
+    print Site::get_emails(141.142.100.100);
+    }
--- a/testing/btest/scripts/base/utils/strings.test
+++ b/testing/btest/scripts/base/utils/strings.test
@ -0,0 +1,30 @@
+# @TEST-EXEC: bro %INPUT >output
+# @TEST-EXEC: btest-diff output
+
+# This is loaded by default.
+#@load base/utils/strings
+
+function test_binary_string(s: string)
+	{
+	if ( is_string_binary(s) )
+		print fmt("'%s' IS considered binary", s);
+	else
+		print fmt("'%s' is NOT considered binary", s);
+	}
+
+test_binary_string("\x68\x65\x6C\x6C\x6F");
+test_binary_string("\xFF\xFF\xFF\x00");
+test_binary_string("\x00\x00\xFF\x00");
+test_binary_string("\x00\x00\x00\x00");
+
+print join_string_set(set("one", "two", "three"), ", ");
+print join_string_set(set("one"), ", ");
+
+print string_escape("hello world", "od");
+print string_escape("\\hello world\\", "");
+
+print cut_tail("hello world", 0);
+print cut_tail("hello world", 1);
+print cut_tail("hello world", 6);
+print cut_tail("hello world", 11);
+print cut_tail("hello world", 12);
--- a/testing/btest/scripts/base/utils/thresholds.test
+++ b/testing/btest/scripts/base/utils/thresholds.test
@ -0,0 +1,29 @@
+# @TEST-EXEC: bro %INPUT >output
+# @TEST-EXEC: btest-diff output
+
+# This is loaded by default.
+#@load base/utils/thresholds
+
+redef default_notice_thresholds = { 2, 4, 6, 8, 10 };
+const my_thresholds: vector of count = { 2, 4, 6, 8, 10 };
+const loop_v: vector of count = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 };
+global track_count: TrackCount;
+
+for ( i in loop_v )
+	{
+	print fmt("Iteration: %s, threshold check: %s", i,
+	          check_threshold(my_thresholds, track_count));
+	print track_count;
+	++track_count$n;
+	}
+
+track_count$n = 0; track_count$index = 0;
+
+print "====================================";
+for ( i in loop_v )
+	{
+	print fmt("Iteration: %s, threshold check: %s", i,
+	          default_check_threshold(track_count));
+	print track_count;
+	++track_count$n;
+	}