Merge remote-tracking branch 'origin/topic/johanna/bit-1578'

One tweak: I made ts optional and set it to network_time() if not given.

BIT-1578 #merged

* origin/topic/johanna/bit-1578:
  Weird: fix potential small issue when ignoring duplicates
  Rewrite weird logging.

CHANGES

@@ -1,4 +1,29 @@
 
+2.4-640 | 2016-06-18 09:50:18 -0700
+
+  * Fix potential mismatches when ignoring duplicate weirds. (Johanna Amann)
+
+  * Weird: Rewrite internals of weird logging. (Johanna Amann)
+
+        - "flow weirds" now actually log information about the flow
+        that they occur in.
+
+        - weirds can now be generated by calling Weird::weird() with
+        the info record directly, allowing more fine-granular passing
+        of information. This is e.g. used for DNS weirds.
+
+    Addresses BIT-1578 (Johanna Amann)
+
+  * Exec: fix reader cleanup when using read_files, preventing file
+    descriptors from leaking every time it was used. (Johanna Amann)
+
+  * Raw Writer: Make code more c++11-y, remove raw pointers. (Johanna
+    Amann)
+
+  * Add separate section with logging changes to NEWS. (Seth Hall)
+
+  * Removed app-stats scripts. Addresses BIT-1171. (Seth Hall)
+
 2.4-631 | 2016-06-16 16:45:10 -0400
 
   * Fixed matching mail address intel and added test (Jan Grashoefer)

NEWS

@@ -157,6 +157,13 @@ Changed Functionality
   filled out in the first place) has been split into to
   "orig_filenames" and "resp_filenames".
 
+Removed Functionality
+---------------------
+
+    - The app-stats scripts have been removed because they weren't
+      being maintained and they were becoming inaccurate. They
+      were also prone to needing more regular updates as the internet
+      changed and will likely be more relevant if maintained externally.
 
 Deprecated Functionality
 ------------------------
@@ -167,11 +174,6 @@ Deprecated Functionality
       decode_base64() and encode_base64(), which take an optional
       parameter to change the Base64 alphabet.
 
-    - The app-stats scripts have been removed because they weren't
-      being maintained and they were becoming inaccurate. They
-      were also prone to needing more regular updates as the internet
-      changed and will likely be more relevant if maintained externally.
-
 Bro 2.4
 =======
 

VERSION

@@ -1 +1 @@
-2.4-631
+2.4-640

scripts/base/frameworks/notice/weird.bro

@@ -22,25 +22,41 @@ export {
 		Activity,
 	};
 
-	## The record type which contains the column fields of the weird log.
+	## The record which is used for representing and logging weirds.
 	type Info: record {
 		## The time when the weird occurred.
 		ts:     time    &log;
+
 		## If a connection is associated with this weird, this will be
 		## the connection's unique ID.
 		uid:    string  &log &optional;
+
 		## conn_id for the optional connection.
 		id:     conn_id &log &optional;
+
+		## A shorthand way of giving the uid and id to a weird.
+		conn:   connection &optional;
+
 		## The name of the weird that occurred.
 		name:   string  &log;
+
 		## Additional information accompanying the weird if any.
 		addl:   string  &log &optional;
+
 		## Indicate if this weird was also turned into a notice.
-		notice: bool    &log &default=F;
+		notice:  bool   &log &default=F;
+
 		## The peer that originated this weird.  This is helpful in
 		## cluster deployments if a particular cluster node is having
 		## trouble to help identify which node is having trouble.
-		peer:   string  &log &optional;
+		peer:   string  &log &optional &default=peer_description;
+
+		## This field is to be provided when a weird is generated for
+		## the purpose of deduplicating weirds. The identifier string
+		## should be unique for a single instance of the weird. This field
+		## is used to define when a weird is conceptually a duplicate of
+		## a previous weird.
+		identifier: string &optional;
 	};
 
 	## Types of actions that may be taken when handling weird activity events.
@@ -267,6 +283,8 @@ export {
 	##
 	## rec: The weird columns about to be logged to the weird stream.
 	global log_weird: event(rec: Info);
+
+	global weird: function(w: Weird::Info);
 }
 
 # These actions result in the output being limited and further redundant
@@ -289,9 +307,6 @@ const notice_actions = {
 	ACTION_NOTICE_ONCE,
 };
 
-# Used to pass the optional connection into report().
-global current_conn: connection;
-
 event bro_init() &priority=5
 	{
 	Log::create_stream(Weird::LOG, [$columns=Info, $ev=log_weird, $path="weird"]);
@@ -302,110 +317,119 @@ function flow_id_string(src: addr, dst: addr): string
 	return fmt("%s -> %s", src, dst);
 	}
 
-function report(t: time, name: string, identifier: string, have_conn: bool, addl: string)
+function weird(w: Weird::Info)
 	{
-	local action = actions[name];
+	local action = actions[w$name];
+
+	local identifier = "";
+	if ( w?$identifier )
+		identifier = w$identifier;
+	else
+		{
+		if ( w?$id )
+			identifier = id_string(w$id);
+		}
 
 	# If this weird is to be ignored let's drop out of here very early.
-	if ( action == ACTION_IGNORE || [name, identifier] in weird_ignore )
+	if ( action == ACTION_IGNORE || [w$name, identifier] in weird_ignore )
 		return;
 
+	if ( w?$conn )
+		{
+		w$uid = w$conn$uid;
+		w$id = w$conn$id;
+		}
+
+	if ( w?$id )
+		{
+		if ( [w$id$orig_h, w$name] in ignore_hosts ||
+				 [w$id$resp_h, w$name] in ignore_hosts )
+				 return;
+		}
+
 	if ( action in limiting_actions )
 		{
+		local notice_identifier = identifier;
 		if ( action in notice_actions )
 			{
 			# Handle notices
-			if ( have_conn && action == ACTION_NOTICE_PER_ORIG )
+			if ( w?$id && action == ACTION_NOTICE_PER_ORIG )
-				identifier = fmt("%s", current_conn$id$orig_h);
+				notice_identifier = fmt("%s", w$id$orig_h);
 			else if ( action == ACTION_NOTICE_ONCE )
-				identifier = "";
+				notice_identifier = "";
 
 			# If this weird was already noticed then we're done.
-			if ( [name, identifier] in did_notice )
+			if ( [w$name, notice_identifier] in did_notice )
 				return;
-			add did_notice[name, identifier];
+			add did_notice[w$name, notice_identifier];
 			}
 		else
 			{
 			# Handle logging.
-			if ( have_conn && action == ACTION_LOG_PER_ORIG )
+			if ( w?$id && action == ACTION_LOG_PER_ORIG )
-				identifier = fmt("%s", current_conn$id$orig_h);
+				notice_identifier = fmt("%s", w$id$orig_h);
 			else if ( action == ACTION_LOG_ONCE )
-				identifier = "";
+				notice_identifier = "";
 
 			# If this weird was already logged then we're done.
-			if ( [name, identifier] in did_log )
+			if ( [w$name, notice_identifier] in did_log )
 				return;
-			add did_log[name, identifier];
-			}
-		}
 
-	# Create the Weird::Info record.
+			add did_log[w$name, notice_identifier];
-	local info: Info;
+			}
-	info$ts = t;
-	info$name = name;
-	info$peer = peer_description;
-	if ( addl != "" )
-		info$addl = addl;
-	if ( have_conn )
-		{
-		info$uid = current_conn$uid;
-		info$id = current_conn$id;
 		}
 
 	if ( action in notice_actions )
 		{
-		info$notice = T;
+		w$notice = T;
 
 		local n: Notice::Info;
 		n$note = Activity;
-		n$msg = info$name;
+		n$msg = w$name;
-		if ( have_conn )
+		if ( w?$conn )
-			n$conn = current_conn;
+			n$conn = w$conn;
-		if ( info?$addl )
+		else
-			n$sub = info$addl;
+			{
+			if ( w?$uid )
+				n$uid = w$uid;
+			if ( w?$id )
+				n$id = w$id;
+			}
+		if ( w?$addl )
+			n$sub = w$addl;
 		NOTICE(n);
 		}
 
 	# This is for the temporary ignoring to reduce volume for identical weirds.
-	if ( name !in weird_do_not_ignore_repeats )
+	if ( w$name !in weird_do_not_ignore_repeats )
-		add weird_ignore[name, identifier];
+		add weird_ignore[w$name, identifier];
 
-	Log::write(Weird::LOG, info);
+	Log::write(Weird::LOG, w);
 	}
 
-function report_conn(t: time, name: string, identifier: string, addl: string, c: connection)
-	{
-	local cid = c$id;
-	if ( [cid$orig_h, name] in ignore_hosts ||
-	     [cid$resp_h, name] in ignore_hosts )
-		return;
-
-	current_conn = c;
-	report(t, name, identifier, T, addl);
-	}
-
-function report_orig(t: time, name: string, identifier: string, orig: addr)
-	{
-	if ( [orig, name] in ignore_hosts )
-		return;
-	
-	report(t, name, identifier, F, "");
-	}
-
-
 # The following events come from core generated weirds typically.
 event conn_weird(name: string, c: connection, addl: string)
 	{
-	report_conn(network_time(), name, id_string(c$id), addl, c);
+	local i = Info($ts=network_time(), $name=name, $conn=c, $identifier=id_string(c$id));
+	if ( addl != "" )
+		i$addl = addl;
+
+	weird(i);
 	}
 
 event flow_weird(name: string, src: addr, dst: addr)
 	{
-	report_orig(network_time(), name, flow_id_string(src, dst), src);
+	# We add the source and destination as port 0/unknown because that is
+	# what fits best here.
+	local id = conn_id($orig_h=src, $orig_p=count_to_port(0, unknown_transport),
+	                   $resp_h=dst, $resp_p=count_to_port(0, unknown_transport));
+
+	local i = Info($ts=network_time(), $name=name, $id=id, $identifier=flow_id_string(src,dst));
+	weird(i);
 	}
 
 event net_weird(name: string)
 	{
-	report(network_time(), name, "", F, "");
+	local i = Info($ts=network_time(), $name=name);
+	weird(i);
 	}

scripts/base/protocols/dns/main.bro

@@ -2,6 +2,7 @@
 ##! their responses.
 
 @load base/utils/queue
+@load base/frameworks/notice/weird
 @load ./consts
 
 module DNS;
@@ -176,8 +177,9 @@ function log_unmatched_msgs_queue(q: Queue::Queue)
 
 	for ( i in infos )
 		{
-		event flow_weird("dns_unmatched_msg",
+		local wi = Weird::Info($ts=network_time(), $name="dns_unmatched_msg", $uid=infos[i]$uid,
-		                 infos[i]$id$orig_h, infos[i]$id$resp_h);
+		                       $id=infos[i]$id);
+		Weird::weird(wi);
 		Log::write(DNS::LOG, infos[i]);
 		}
 	}
@@ -192,12 +194,14 @@ function log_unmatched_msgs(msgs: PendingMessages)
 
 function enqueue_new_msg(msgs: PendingMessages, id: count, msg: Info)
 	{
+	local wi: Weird::Info;
 	if ( id !in msgs )
 		{
 		if ( |msgs| > max_pending_query_ids )
 			{
-			event flow_weird("dns_unmatched_query_id_quantity",
+			wi = Weird::Info($ts=network_time(), $name="dns_unmatched_msg", $uid=msg$uid,
-			                 msg$id$orig_h, msg$id$resp_h);
+			                       $id=msg$id);
+			Weird::weird(wi);
 			# Throw away all unmatched on assumption they'll never be matched.
 			log_unmatched_msgs(msgs);
 			}
@@ -208,8 +212,9 @@ function enqueue_new_msg(msgs: PendingMessages, id: count, msg: Info)
 		{
 		if ( Queue::len(msgs[id]) > max_pending_msgs )
 			{
-			event flow_weird("dns_unmatched_msg_quantity",
+			wi = Weird::Info($ts=network_time(), $name="dns_unmatched_msg_quantity", $uid=msg$uid,
-			                 msg$id$orig_h, msg$id$resp_h);
+			                       $id=msg$id);
+			Weird::weird(wi);
 			log_unmatched_msgs_queue(msgs[id]);
 			# Throw away all unmatched on assumption they'll never be matched.
 			msgs[id] = Queue::init();

testing/btest/Baseline/core.checksums/bad.out

@@ -3,101 +3,101 @@
 #empty_field	(empty)
 #unset_field	-
 #path	weird
-#open	2013-08-26-19-02-06
+#open	2016-06-15-20-38-04
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	name	addl	notice	peer
 #types	time	string	addr	port	addr	port	string	string	bool	string
 1332784981.078396	-	-	-	-	-	bad_IP_checksum	-	F	bro
-#close	2013-08-26-19-02-06
+#close	2016-06-15-20-38-04
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	weird
-#open	2013-08-26-19-02-06
+#open	2016-06-15-20-38-06
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	name	addl	notice	peer
 #types	time	string	addr	port	addr	port	string	string	bool	string
 1332784885.686428	CXWv6p3arKYeMETxOg	127.0.0.1	30000	127.0.0.1	80	bad_TCP_checksum	-	F	bro
-#close	2013-08-26-19-02-06
+#close	2016-06-15-20-38-06
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	weird
-#open	2013-08-26-19-02-07
+#open	2016-06-15-20-38-08
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	name	addl	notice	peer
 #types	time	string	addr	port	addr	port	string	string	bool	string
 1332784933.501023	CXWv6p3arKYeMETxOg	127.0.0.1	30000	127.0.0.1	13000	bad_UDP_checksum	-	F	bro
-#close	2013-08-26-19-02-07
+#close	2016-06-15-20-38-08
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	weird
-#open	2013-08-26-19-02-07
+#open	2016-06-15-20-38-10
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	name	addl	notice	peer
 #types	time	string	addr	port	addr	port	string	string	bool	string
 1334075363.536871	CXWv6p3arKYeMETxOg	192.168.1.100	8	192.168.1.101	0	bad_ICMP_checksum	-	F	bro
-#close	2013-08-26-19-02-07
+#close	2016-06-15-20-38-10
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	weird
-#open	2013-08-26-19-02-08
+#open	2016-06-15-20-38-11
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	name	addl	notice	peer
 #types	time	string	addr	port	addr	port	string	string	bool	string
-1332785210.013051	-	-	-	-	-	routing0_hdr	-	F	bro
+1332785210.013051	-	2001:4f8:4:7:2e0:81ff:fe52:ffff	0	2001:78:1:32::2	0	routing0_hdr	-	F	bro
 1332785210.013051	CXWv6p3arKYeMETxOg	2001:4f8:4:7:2e0:81ff:fe52:ffff	30000	2001:78:1:32::2	80	bad_TCP_checksum	-	F	bro
-#close	2013-08-26-19-02-08
+#close	2016-06-15-20-38-12
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	weird
-#open	2013-08-26-19-02-09
+#open	2016-06-15-20-38-13
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	name	addl	notice	peer
 #types	time	string	addr	port	addr	port	string	string	bool	string
-1332782580.798420	-	-	-	-	-	routing0_hdr	-	F	bro
+1332782580.798420	-	2001:4f8:4:7:2e0:81ff:fe52:ffff	0	2001:78:1:32::2	0	routing0_hdr	-	F	bro
 1332782580.798420	CXWv6p3arKYeMETxOg	2001:4f8:4:7:2e0:81ff:fe52:ffff	30000	2001:78:1:32::2	13000	bad_UDP_checksum	-	F	bro
-#close	2013-08-26-19-02-09
+#close	2016-06-15-20-38-13
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	weird
-#open	2013-08-26-19-02-09
+#open	2016-06-15-20-38-15
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	name	addl	notice	peer
 #types	time	string	addr	port	addr	port	string	string	bool	string
-1334075111.800086	-	-	-	-	-	routing0_hdr	-	F	bro
+1334075111.800086	-	2001:4f8:4:7:2e0:81ff:fe52:ffff	0	2001:78:1:32::1	0	routing0_hdr	-	F	bro
 1334075111.800086	CXWv6p3arKYeMETxOg	2001:4f8:4:7:2e0:81ff:fe52:ffff	128	2001:78:1:32::1	129	bad_ICMP_checksum	-	F	bro
-#close	2013-08-26-19-02-09
+#close	2016-06-15-20-38-15
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	weird
-#open	2013-08-26-19-02-10
+#open	2016-06-15-20-38-16
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	name	addl	notice	peer
 #types	time	string	addr	port	addr	port	string	string	bool	string
 1332785250.469132	CXWv6p3arKYeMETxOg	2001:4f8:4:7:2e0:81ff:fe52:ffff	30000	2001:4f8:4:7:2e0:81ff:fe52:9a6b	80	bad_TCP_checksum	-	F	bro
-#close	2013-08-26-19-02-10
+#close	2016-06-15-20-38-17
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	weird
-#open	2013-08-26-19-02-10
+#open	2016-06-15-20-38-18
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	name	addl	notice	peer
 #types	time	string	addr	port	addr	port	string	string	bool	string
 1332781342.923813	CXWv6p3arKYeMETxOg	2001:4f8:4:7:2e0:81ff:fe52:ffff	30000	2001:4f8:4:7:2e0:81ff:fe52:9a6b	13000	bad_UDP_checksum	-	F	bro
-#close	2013-08-26-19-02-10
+#close	2016-06-15-20-38-18
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	weird
-#open	2013-08-26-19-02-11
+#open	2016-06-15-20-38-20
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	name	addl	notice	peer
 #types	time	string	addr	port	addr	port	string	string	bool	string
 1334074939.467194	CXWv6p3arKYeMETxOg	2001:4f8:4:7:2e0:81ff:fe52:ffff	128	2001:4f8:4:7:2e0:81ff:fe52:9a6b	129	bad_ICMP_checksum	-	F	bro
-#close	2013-08-26-19-02-11
+#close	2016-06-15-20-38-20

testing/btest/Baseline/core.checksums/good.out

@@ -3,68 +3,68 @@
 #empty_field	(empty)
 #unset_field	-
 #path	weird
-#open	2013-08-26-19-34-56
+#open	2016-06-15-20-38-20
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	name	addl	notice	peer
 #types	time	string	addr	port	addr	port	string	string	bool	string
 1334074939.467194	CXWv6p3arKYeMETxOg	2001:4f8:4:7:2e0:81ff:fe52:ffff	128	2001:4f8:4:7:2e0:81ff:fe52:9a6b	129	bad_ICMP_checksum	-	F	bro
-#close	2013-08-26-19-34-56
+#close	2016-06-15-20-38-20
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	weird
-#open	2013-08-26-19-34-57
+#open	2016-06-15-20-38-27
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	name	addl	notice	peer
 #types	time	string	addr	port	addr	port	string	string	bool	string
-1332785125.596793	-	-	-	-	-	routing0_hdr	-	F	bro
+1332785125.596793	-	2001:4f8:4:7:2e0:81ff:fe52:ffff	0	2001:78:1:32::2	0	routing0_hdr	-	F	bro
-#close	2013-08-26-19-34-57
+#close	2016-06-15-20-38-27
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	weird
-#open	2013-08-26-19-34-57
+#open	2016-06-15-20-38-28
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	name	addl	notice	peer
 #types	time	string	addr	port	addr	port	string	string	bool	string
-1332782508.592037	-	-	-	-	-	routing0_hdr	-	F	bro
+1332782508.592037	-	2001:4f8:4:7:2e0:81ff:fe52:ffff	0	2001:78:1:32::2	0	routing0_hdr	-	F	bro
-#close	2013-08-26-19-34-57
+#close	2016-06-15-20-38-29
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	weird
-#open	2013-08-26-19-34-57
+#open	2016-06-15-20-38-30
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	name	addl	notice	peer
 #types	time	string	addr	port	addr	port	string	string	bool	string
-1334075027.053380	-	-	-	-	-	routing0_hdr	-	F	bro
+1334075027.053380	-	2001:4f8:4:7:2e0:81ff:fe52:ffff	0	2001:78:1:32::1	0	routing0_hdr	-	F	bro
-#close	2013-08-26-19-34-57
+#close	2016-06-15-20-38-30
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	weird
-#open	2013-08-26-19-34-57
+#open	2016-06-15-20-38-30
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	name	addl	notice	peer
 #types	time	string	addr	port	addr	port	string	string	bool	string
-1334075027.053380	-	-	-	-	-	routing0_hdr	-	F	bro
+1334075027.053380	-	2001:4f8:4:7:2e0:81ff:fe52:ffff	0	2001:78:1:32::1	0	routing0_hdr	-	F	bro
-#close	2013-08-26-19-34-57
+#close	2016-06-15-20-38-30
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	weird
-#open	2013-08-26-19-34-57
+#open	2016-06-15-20-38-30
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	name	addl	notice	peer
 #types	time	string	addr	port	addr	port	string	string	bool	string
-1334075027.053380	-	-	-	-	-	routing0_hdr	-	F	bro
+1334075027.053380	-	2001:4f8:4:7:2e0:81ff:fe52:ffff	0	2001:78:1:32::1	0	routing0_hdr	-	F	bro
-#close	2013-08-26-19-34-57
+#close	2016-06-15-20-38-30
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	weird
-#open	2013-08-26-19-34-57
+#open	2016-06-15-20-38-30
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	name	addl	notice	peer
 #types	time	string	addr	port	addr	port	string	string	bool	string
-1334075027.053380	-	-	-	-	-	routing0_hdr	-	F	bro
+1334075027.053380	-	2001:4f8:4:7:2e0:81ff:fe52:ffff	0	2001:78:1:32::1	0	routing0_hdr	-	F	bro
-#close	2013-08-26-19-34-57
+#close	2016-06-15-20-38-30

testing/btest/Baseline/core.reassembly/output

@@ -3,7 +3,6 @@ flow weird, excessively_small_fragment, 164.1.123.163, 164.1.123.61
 flow weird, fragment_size_inconsistency, 164.1.123.163, 164.1.123.61
 flow weird, fragment_inconsistency, 164.1.123.163, 164.1.123.61
 flow weird, fragment_inconsistency, 164.1.123.163, 164.1.123.61
-flow weird, dns_unmatched_msg, 164.1.123.163, 164.1.123.61
 ----------------------
 flow weird, excessively_small_fragment, 164.1.123.163, 164.1.123.61
 flow weird, excessively_small_fragment, 164.1.123.163, 164.1.123.61

testing/btest/Baseline/plugins.hooks/output

@@ -238,7 +238,7 @@
 0.000000   MetaHookPost  CallFunction(Log::__create_stream, <frame>, (Weird::LOG, [columns=<no value description>, ev=Weird::log_weird, path=weird])) -> <no result>
 0.000000   MetaHookPost  CallFunction(Log::__create_stream, <frame>, (X509::LOG, [columns=<no value description>, ev=X509::log_x509, path=x509])) -> <no result>
 0.000000   MetaHookPost  CallFunction(Log::__create_stream, <frame>, (mysql::LOG, [columns=<no value description>, ev=MySQL::log_mysql, path=mysql])) -> <no result>
-0.000000   MetaHookPost  CallFunction(Log::__write, <frame>, (PacketFilter::LOG, [ts=1466108964.602, node=bro, filter=ip or not ip, init=T, success=T])) -> <no result>
+0.000000   MetaHookPost  CallFunction(Log::__write, <frame>, (PacketFilter::LOG, [ts=1466268606.345873, node=bro, filter=ip or not ip, init=T, success=T])) -> <no result>
 0.000000   MetaHookPost  CallFunction(Log::add_default_filter, <frame>, (Cluster::LOG)) -> <no result>
 0.000000   MetaHookPost  CallFunction(Log::add_default_filter, <frame>, (Communication::LOG)) -> <no result>
 0.000000   MetaHookPost  CallFunction(Log::add_default_filter, <frame>, (Conn::LOG)) -> <no result>
@@ -359,7 +359,7 @@
 0.000000   MetaHookPost  CallFunction(Log::create_stream, <frame>, (Weird::LOG, [columns=<no value description>, ev=Weird::log_weird, path=weird])) -> <no result>
 0.000000   MetaHookPost  CallFunction(Log::create_stream, <frame>, (X509::LOG, [columns=<no value description>, ev=X509::log_x509, path=x509])) -> <no result>
 0.000000   MetaHookPost  CallFunction(Log::create_stream, <frame>, (mysql::LOG, [columns=<no value description>, ev=MySQL::log_mysql, path=mysql])) -> <no result>
-0.000000   MetaHookPost  CallFunction(Log::write, <frame>, (PacketFilter::LOG, [ts=1466108964.602, node=bro, filter=ip or not ip, init=T, success=T])) -> <no result>
+0.000000   MetaHookPost  CallFunction(Log::write, <frame>, (PacketFilter::LOG, [ts=1466268606.345873, node=bro, filter=ip or not ip, init=T, success=T])) -> <no result>
 0.000000   MetaHookPost  CallFunction(NetControl::check_plugins, <frame>, ()) -> <no result>
 0.000000   MetaHookPost  CallFunction(NetControl::init, <null>, ()) -> <no result>
 0.000000   MetaHookPost  CallFunction(Notice::want_pp, <frame>, ()) -> <no result>
@@ -652,6 +652,7 @@
 0.000000   MetaHookPost  LoadFile(base<...>/unified2) -> -1
 0.000000   MetaHookPost  LoadFile(base<...>/urls) -> -1
 0.000000   MetaHookPost  LoadFile(base<...>/utils) -> -1
+0.000000   MetaHookPost  LoadFile(base<...>/weird) -> -1
 0.000000   MetaHookPost  LoadFile(base<...>/x509) -> -1
 0.000000   MetaHookPost  LoadFile(base<...>/xmpp) -> -1
 0.000000   MetaHookPost  QueueEvent(NetControl::init()) -> false
@@ -897,7 +898,7 @@
 0.000000   MetaHookPre   CallFunction(Log::__create_stream, <frame>, (Weird::LOG, [columns=<no value description>, ev=Weird::log_weird, path=weird]))
 0.000000   MetaHookPre   CallFunction(Log::__create_stream, <frame>, (X509::LOG, [columns=<no value description>, ev=X509::log_x509, path=x509]))
 0.000000   MetaHookPre   CallFunction(Log::__create_stream, <frame>, (mysql::LOG, [columns=<no value description>, ev=MySQL::log_mysql, path=mysql]))
-0.000000   MetaHookPre   CallFunction(Log::__write, <frame>, (PacketFilter::LOG, [ts=1466108964.602, node=bro, filter=ip or not ip, init=T, success=T]))
+0.000000   MetaHookPre   CallFunction(Log::__write, <frame>, (PacketFilter::LOG, [ts=1466268606.345873, node=bro, filter=ip or not ip, init=T, success=T]))
 0.000000   MetaHookPre   CallFunction(Log::add_default_filter, <frame>, (Cluster::LOG))
 0.000000   MetaHookPre   CallFunction(Log::add_default_filter, <frame>, (Communication::LOG))
 0.000000   MetaHookPre   CallFunction(Log::add_default_filter, <frame>, (Conn::LOG))
@@ -1018,7 +1019,7 @@
 0.000000   MetaHookPre   CallFunction(Log::create_stream, <frame>, (Weird::LOG, [columns=<no value description>, ev=Weird::log_weird, path=weird]))
 0.000000   MetaHookPre   CallFunction(Log::create_stream, <frame>, (X509::LOG, [columns=<no value description>, ev=X509::log_x509, path=x509]))
 0.000000   MetaHookPre   CallFunction(Log::create_stream, <frame>, (mysql::LOG, [columns=<no value description>, ev=MySQL::log_mysql, path=mysql]))
-0.000000   MetaHookPre   CallFunction(Log::write, <frame>, (PacketFilter::LOG, [ts=1466108964.602, node=bro, filter=ip or not ip, init=T, success=T]))
+0.000000   MetaHookPre   CallFunction(Log::write, <frame>, (PacketFilter::LOG, [ts=1466268606.345873, node=bro, filter=ip or not ip, init=T, success=T]))
 0.000000   MetaHookPre   CallFunction(NetControl::check_plugins, <frame>, ())
 0.000000   MetaHookPre   CallFunction(NetControl::init, <null>, ())
 0.000000   MetaHookPre   CallFunction(Notice::want_pp, <frame>, ())
@@ -1311,6 +1312,7 @@
 0.000000   MetaHookPre   LoadFile(base<...>/unified2)
 0.000000   MetaHookPre   LoadFile(base<...>/urls)
 0.000000   MetaHookPre   LoadFile(base<...>/utils)
+0.000000   MetaHookPre   LoadFile(base<...>/weird)
 0.000000   MetaHookPre   LoadFile(base<...>/x509)
 0.000000   MetaHookPre   LoadFile(base<...>/xmpp)
 0.000000   MetaHookPre   QueueEvent(NetControl::init())
@@ -1555,7 +1557,7 @@
 0.000000 | HookCallFunction Log::__create_stream(Weird::LOG, [columns=<no value description>, ev=Weird::log_weird, path=weird])
 0.000000 | HookCallFunction Log::__create_stream(X509::LOG, [columns=<no value description>, ev=X509::log_x509, path=x509])
 0.000000 | HookCallFunction Log::__create_stream(mysql::LOG, [columns=<no value description>, ev=MySQL::log_mysql, path=mysql])
-0.000000 | HookCallFunction Log::__write(PacketFilter::LOG, [ts=1466108964.602, node=bro, filter=ip or not ip, init=T, success=T])
+0.000000 | HookCallFunction Log::__write(PacketFilter::LOG, [ts=1466268606.345873, node=bro, filter=ip or not ip, init=T, success=T])
 0.000000 | HookCallFunction Log::add_default_filter(Cluster::LOG)
 0.000000 | HookCallFunction Log::add_default_filter(Communication::LOG)
 0.000000 | HookCallFunction Log::add_default_filter(Conn::LOG)
@@ -1676,7 +1678,7 @@
 0.000000 | HookCallFunction Log::create_stream(Weird::LOG, [columns=<no value description>, ev=Weird::log_weird, path=weird])
 0.000000 | HookCallFunction Log::create_stream(X509::LOG, [columns=<no value description>, ev=X509::log_x509, path=x509])
 0.000000 | HookCallFunction Log::create_stream(mysql::LOG, [columns=<no value description>, ev=MySQL::log_mysql, path=mysql])
-0.000000 | HookCallFunction Log::write(PacketFilter::LOG, [ts=1466108964.602, node=bro, filter=ip or not ip, init=T, success=T])
+0.000000 | HookCallFunction Log::write(PacketFilter::LOG, [ts=1466268606.345873, node=bro, filter=ip or not ip, init=T, success=T])
 0.000000 | HookCallFunction NetControl::check_plugins()
 0.000000 | HookCallFunction NetControl::init()
 0.000000 | HookCallFunction Notice::want_pp()

testing/btest/Baseline/scripts.base.protocols.dns.duplicate-reponses/weird.log

@@ -3,10 +3,10 @@
 #empty_field	(empty)
 #unset_field	-
 #path	weird
-#open	2015-03-19-15-44-23
+#open	2016-06-15-20-29-41
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	name	addl	notice	peer
 #types	time	string	addr	port	addr	port	string	string	bool	string
 1363716396.798286	CXWv6p3arKYeMETxOg	55.247.223.174	27285	222.195.43.124	53	DNS_RR_unknown_type	46	F	bro
 1363716396.798374	CXWv6p3arKYeMETxOg	55.247.223.174	27285	222.195.43.124	53	dns_unmatched_reply	-	F	bro
-1363716396.798374	-	-	-	-	-	dns_unmatched_msg	-	F	bro
+1363716396.798374	CXWv6p3arKYeMETxOg	55.247.223.174	27285	222.195.43.124	53	dns_unmatched_msg	-	F	bro
-#close	2015-03-19-15-44-23
+#close	2016-06-15-20-29-41