Merge remote branch 'origin/topic/robin/logging-internals' into topic/policy-scripts-new

testing/btest/Baseline/logging.attr-extend/ssh.log

@@ -0,0 +1,2 @@
+# status	country	a1	b1	b2
+success	unknown	1	3	4

testing/btest/Baseline/logging.attr/ssh.log

@@ -0,0 +1,6 @@
+# status	country
+success	unknown
+failure	US
+failure	UK
+success	BR
+failure	MX

testing/btest/Baseline/logging.remote-types/receiver.ssh.log

@@ -1,2 +1,2 @@
-# b	i	e	c	p	sn	n	a	d	t	iv	s	sc	ss	se
-T	-42	SSH::SSH	21	123	10.0.0.0/24	10.0.0.0	1.2.3.4	3.14	1299727902.65854	100.0	hurz	4,1,3,2	CC,BB,AA	EMPTY
+# b	i	e	c	p	sn	n	a	d	t	iv	s	sc	ss	se	vc	ve
+T	-42	SSH::SSH	21	123	10.0.0.0/24	10.0.0.0	1.2.3.4	3.14	1301360085.98852	100.0	hurz	4,1,3,2	CC,BB,AA	EMPTY	10,20,30	EMPTY

testing/btest/Baseline/logging.rotate-custom/out

@@ -68,8 +68,6 @@
 # t	id.orig_h	id.orig_p	id.resp_h	id.resp_p
 1299499205.0	10.0.0.1	20	10.0.0.2	1033
 1299502795.0	10.0.0.2	20	10.0.0.3	9
-> test.log
-# t	id.orig_h	id.orig_p	id.resp_h	id.resp_p
 > test2-11-03-06_19.00.05.log
 # t	id.orig_h	id.orig_p	id.resp_h	id.resp_p
 1299466805.0	10.0.0.1	20	10.0.0.2	1024
@@ -132,3 +130,5 @@
 1299502795.0	10.0.0.2	20	10.0.0.3	9
 > test2.log
 # t	id.orig_h	id.orig_p	id.resp_h	id.resp_p
+> test.log
+# t	id.orig_h	id.orig_p	id.resp_h	id.resp_p

testing/btest/Baseline/logging.types/ssh.log

@@ -1,2 +1,2 @@
-# b	i	e	c	p	sn	n	a	d	t	iv	s	sc	ss	se
-T	-42	SSH::SSH	21	123	10.0.0.0/24	10.0.0.0	1.2.3.4	3.14	1299727493.47095	100.0	hurz	4,1,3,2	CC,BB,AA	EMPTY
+# b	i	e	c	p	sn	n	a	d	t	iv	s	sc	ss	se	vc	ve
+T	-42	SSH::SSH	21	123	10.0.0.0/24	10.0.0.0	1.2.3.4	3.14	1301359781.8203	100.0	hurz	4,1,3,2	CC,BB,AA	EMPTY	10,20,30	EMPTY

testing/btest/Baseline/logging.vec/ssh.log

@@ -0,0 +1,2 @@
+# vec
+-,2,-,-,5

testing/btest/logging/adapt-filter.bro

@@ -18,7 +18,7 @@ export {
 		id: conn_id; # Will be rolled out into individual columns.
 		status: string &optional;
 		country: string &default="unknown";
-	};
+	} &log;
 }
 
 event bro_init()

testing/btest/logging/ascii-empty.bro

@@ -19,7 +19,7 @@ export {
 		status: string &optional;
 		country: string &default="unknown";
 		b: bool &optional;
-	};
+	} &log;
 }
 
 event bro_init()

testing/btest/logging/ascii-escape.bro

@@ -14,7 +14,7 @@ export {
 		id: conn_id; # Will be rolled out into individual columns.
 		status: string &optional;
 		country: string &default="unknown";
-	};
+	} &log;
 }
 
 event bro_init()

testing/btest/logging/ascii-options.bro

@@ -16,7 +16,7 @@ export {
 		id: conn_id; # Will be rolled out into individual columns.
 		status: string &optional;
 		country: string &default="unknown";
-	};
+	} &log;
 }
 
 event bro_init()

testing/btest/logging/attr-extend.bro

@@ -0,0 +1,37 @@
+#
+# @TEST-EXEC: bro %INPUT
+# @TEST-EXEC: btest-diff ssh.log
+
+module SSH;
+
+export {
+	redef enum Log::ID += { SSH };
+
+	type Log: record {
+		t: time;
+		id: conn_id;
+		status: string &optional &log;
+		country: string &default="unknown" &log;
+	};
+}
+
+redef record Log += {
+	a1: count &log &optional;
+	a2: count &optional;
+};
+
+redef record Log += {
+	b1: count &optional;
+	b2: count &optional;
+} &log;
+
+
+event bro_init()
+{
+	Log::create_stream(SSH, [$columns=Log]);
+
+    local cid = [$orig_h=1.2.3.4, $orig_p=1234/tcp, $resp_h=2.3.4.5, $resp_p=80/tcp];
+
+	Log::write(SSH, [$t=network_time(), $id=cid, $status="success", $a1=1, $a2=2, $b1=3, $b2=4]);
+}
+

testing/btest/logging/attr.bro

@@ -0,0 +1,31 @@
+#
+# @TEST-EXEC: bro %INPUT
+# @TEST-EXEC: btest-diff ssh.log
+
+module SSH;
+
+export {
+	redef enum Log::ID += { SSH };
+
+	type Log: record {
+		t: time;
+		id: conn_id;
+		status: string &optional &log;
+		country: string &default="unknown" &log;
+	};
+}
+
+event bro_init()
+{
+	Log::create_stream(SSH, [$columns=Log]);
+
+    local cid = [$orig_h=1.2.3.4, $orig_p=1234/tcp, $resp_h=2.3.4.5, $resp_p=80/tcp];
+
+	Log::write(SSH, [$t=network_time(), $id=cid, $status="success"]);
+	Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="US"]);
+	Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="UK"]);
+	Log::write(SSH, [$t=network_time(), $id=cid, $status="success", $country="BR"]);
+	Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="MX"]);
+	
+}
+

testing/btest/logging/disable-stream.bro

@@ -12,7 +12,7 @@ export {
 		id: conn_id; # Will be rolled out into individual columns.
 		status: string &optional;
 		country: string &default="unknown";
-	};
+	} &log;
 }
 
 event bro_init()

testing/btest/logging/empty-event.bro

@@ -12,7 +12,7 @@ export {
 		id: conn_id; # Will be rolled out into individual columns.
 		status: string &optional;
 		country: string &default="unknown";
-	};
+	} &log;
 }
 
 global log_ssh: event(rec: Log);

testing/btest/logging/events.bro

@@ -17,7 +17,7 @@ export {
 		id: conn_id; # Will be rolled out into individual columns.
 		status: string &optional;
 		country: string &default="unknown";
-	};
+	} &log;
 }
 
 global ssh_log: event(rec: Log);

testing/btest/logging/no-local.bro

@@ -12,7 +12,7 @@ export {
 		id: conn_id; # Will be rolled out into individual columns.
 		status: string &optional;
 		country: string &default="unknown";
-	};
+	} &log;
 }
 
 redef Log::enable_local_logging = F;

testing/btest/logging/path-func.bro

@@ -18,7 +18,7 @@ export {
 		id: conn_id; # Will be rolled out into individual columns.
 		status: string &optional;
 		country: string &default="unknown";
-	};
+	} &log;
 }
 
 global c = -1;

testing/btest/logging/pred.bro

@@ -18,7 +18,7 @@ export {
 		id: conn_id; # Will be rolled out into individual columns.
 		status: string &optional;
 		country: string &default="unknown";
-	};
+	} &log;
 }
 
 function fail(rec: Log): bool

testing/btest/logging/remote-types.bro

@@ -35,7 +35,9 @@ export {
 		sc: set[count];
 		ss: set[string];
 		se: set[string];
-	};
+		vc: vector of count;
+		ve: vector of string;
+	} &log;
 }
 
 event bro_init()
@@ -54,6 +56,7 @@ module SSH;
 event remote_connection_handshake_done(p: event_peer)
 	{
 	local empty_set: set[string];
+	local empty_vector: vector of string;
 
 	Log::write(SSH, [
 		$b=T,
@@ -70,7 +73,9 @@ event remote_connection_handshake_done(p: event_peer)
 		$s="hurz",
 		$sc=set(1,2,3,4),
 		$ss=set("AA", "BB", "CC"),
-		$se=empty_set
+		$se=empty_set,
+		$vc=vector(10, 20, 30),
+		$ve=empty_vector
 		]);
 	}
 @TEST-END-FILE

testing/btest/logging/remote.bro

@@ -25,7 +25,7 @@ export {
 		id: conn_id; # Will be rolled out into individual columns.
 		status: string &optional;
 		country: string &default="unknown";
-	};
+	} &log;
 }
 
 event bro_init()

testing/btest/logging/remove.bro

@@ -16,7 +16,7 @@ export {
 		id: conn_id; # Will be rolled out into individual columns.
 		status: string &optional;
 		country: string &default="unknown";
-	};
+	} &log;
 }
 
 event bro_init()

testing/btest/logging/rotate-custom.bro

@@ -1,6 +1,6 @@
 #
 # @TEST-EXEC: bro -r %DIR/rotation.trace %INPUT >out
-# @TEST-EXEC: for i in test*.log; do printf '> %s\n' $i; cat $i; done >>out
+# @TEST-EXEC: for i in `ls test*.log | sort`; do printf '> %s\n' $i; cat $i; done >>out
 # @TEST-EXEC: btest-diff out
 
 module Test;
@@ -14,7 +14,7 @@ export {
 	type Log: record {
 		t: time;
 		id: conn_id; # Will be rolled out into individual columns.
-	};
+	} &log;
 }
 
 redef Log::default_rotation_interval = 1hr;

testing/btest/logging/rotate.bro

@@ -14,7 +14,7 @@ export {
 	type Log: record {
 		t: time;
 		id: conn_id; # Will be rolled out into individual columns.
-	};
+	} &log;
 }
 
 redef Log::default_rotation_interval = 1hr;

testing/btest/logging/stdout.bro

@@ -13,7 +13,7 @@ export {
 		id: conn_id; # Will be rolled out into individual columns.
 		status: string &optional;
 		country: string &default="unknown";
-	};
+	} &log;
 }
 
 event bro_init()

testing/btest/logging/test-logging.bro

@@ -12,7 +12,7 @@ export {
 		id: conn_id; # Will be rolled out into individual columns.
 		status: string &optional;
 		country: string &default="unknown";
-	};
+	} &log;
 }
 
 event bro_init()

testing/btest/logging/types.bro

@@ -27,7 +27,9 @@ export {
 		sc: set[count];
 		ss: set[string];
 		se: set[string];
-	};
+		vc: vector of count;
+		ve: vector of string;
+	} &log;
 }
 
 event bro_init()
@@ -35,6 +37,7 @@ event bro_init()
 	Log::create_stream(SSH, [$columns=Log]);
 
 	local empty_set: set[string];
+	local empty_vector: vector of string;
 
 	Log::write(SSH, [
 		$b=T,
@@ -51,7 +54,9 @@ event bro_init()
 		$s="hurz",
 		$sc=set(1,2,3,4),
 		$ss=set("AA", "BB", "CC"),
-		$se=empty_set
+		$se=empty_set,
+		$vc=vector(10, 20, 30),
+		$ve=empty_vector
 		]);
 }
 

testing/btest/logging/vec.bro

@@ -0,0 +1,27 @@
+#
+# @TEST-EXEC: bro %INPUT
+# @TEST-EXEC: btest-diff ssh.log
+
+module SSH;
+
+export {
+	redef enum Log::ID += { SSH };
+
+	type Log: record {
+        vec: vector of string &log;
+	};
+}
+
+event bro_init()
+{
+	Log::create_stream(SSH, [$columns=Log]);
+
+    local v: vector of string;
+
+	v[2] = "2";
+	v[5] = "5";
+
+	Log::write(SSH, [$vec=v]);
+}
+
+

testing/rec.bro

@@ -0,0 +1,17 @@
+# @TEST-EXEC: bro %INPUT >output
+# @TEST-EXEC: btest-diff output
+
+type Foo: record {
+        a: count;
+        b: count &optional;
+};
+
+redef record Foo += {
+        c: count &default=42;
+        d: count &optional;
+};
+
+global f: Foo = [$a=21];
+
+print f;
+

testing/rec2.bro

@@ -0,0 +1,17 @@
+# @TEST-EXEC: bro %INPUT >output
+# @TEST-EXEC: btest-diff output
+
+type Foo: record {
+        a: count;
+        b: count &optional;
+};
+
+redef record Foo += {
+        c: count &default=42;
+        d: string &optional;
+};
+
+global f: Foo = [$a=21, $d="XXX"];
+
+print f;
+

testing/wrong-rec.bro

@@ -0,0 +1,13 @@
+# @TEST-EXEC-FAIL: bro %INPUT >output 2>&1
+# @TEST-EXEC: btest-diff output
+
+type Foo: record {
+        a: count;
+        b: count &optional;
+};
+
+redef record Foo += {
+        c: count;
+        d: string &optional;
+};
+