Merge branch 'master' into topic/jsiwek/faf-updates

Conflicts:
	magic

CHANGES

@@ -1,4 +1,22 @@
 
+2.1-824 | 2013-07-22 14:25:14 -0400
+
+  * Fixed a scriptland state issue that manifested especially badly on proxies. (Seth Hall)
+
+  * Another test fix. (Robin Sommer)
+
+  * Canonyfying the output of core.print-bpf-filters. (Robin Sommer)
+
+2.1-820 | 2013-07-18 12:30:04 -0700
+
+  * Extending external canonifier to remove fractional values from
+    capture_loss.log. (Robin Sommer)
+
+  * Canonifying internal order for plugins and their components to
+    make it deterministic. (Robin Sommer)
+
+  * Small raw reader tweaks that got left our earlier. (Robin Sommer)
+    
 2.1-814 | 2013-07-15 18:18:20 -0700
 
   * Fixing raw reader crash when accessing nonexistant file, and

VERSION

@@ -1 +1 @@
-2.1-814
+2.1-824

aux/btest

@@ -1 +1 @@
-Subproject commit c2e73c9e1efed6bfdf2d977d716c97773c39492e
+Subproject commit ce366206e3407e534a786ad572c342e9f9fef26b

scripts/base/protocols/dns/main.bro

@@ -207,6 +207,11 @@ event DNS::do_reply(c: connection, msg: dns_msg, ans: dns_answer, reply: string)
 	{
 	if ( ans$answer_type == DNS_ANS )
 		{
+		if ( ! c?$dns )
+			{
+			event conn_weird("dns_unmatched_reply", c, "");
+			hook set_session(c, msg, F);
+			}
 		c$dns$AA    = msg$AA;
 		c$dns$RA    = msg$RA;
 

scripts/base/protocols/irc/dcc-send.bro

@@ -85,5 +85,6 @@ event expected_connection_seen(c: connection, a: Analyzer::Tag) &priority=10
 
 event connection_state_remove(c: connection) &priority=-5
 	{
+	if ( [c$id$resp_h, c$id$resp_p] in dcc_expected_transfers )
 		delete dcc_expected_transfers[c$id$resp_h, c$id$resp_p];
 	}

src/analyzer/Component.h

@@ -72,7 +72,7 @@ public:
 	 * from what's passed to the constructor but upper-cased and
 	 * canonified to allow being part of a script-level ID.
 	 */
-	const char* Name() const	{ return name; }
+	virtual const char* Name() const	{ return name; }
 
 	/**
 	 * Returns a canonocalized version of the analyzer's name.  The

src/file_analysis/Component.h

@@ -64,7 +64,7 @@ public:
 	 * from what's passed to the constructor but upper-cased and
 	 * canonified to allow being part of a script-level ID.
 	 */
-	const char* Name() const	{ return name; }
+	virtual const char* Name() const	{ return name; }
 
 	/**
 	 * Returns a canonocalized version of the analyzer's name.  The

src/input/readers/Raw.cc

@@ -169,8 +169,8 @@ bool Raw::OpenInput()
 			Error(Fmt("Init: cannot open %s", fname.c_str()));
 			return false;
 			}
-		}
 		fcntl(fileno(file),  F_SETFD, FD_CLOEXEC);
+		}
 
 	return true;
 	}

src/plugin/Component.h

@@ -45,6 +45,12 @@ public:
 	 */
 	component::Type Type() const;
 
+	/**
+	 * Returns a descriptive name for the analyzer. This name must be
+	 * unique across all components of the same type.
+	 */
+	virtual const char* Name() const = 0;
+
 	/**
 	 * Returns a textual representation of the component. The default
 	 * version just output the type. Derived version should call the

src/plugin/Manager.cc

@@ -30,9 +30,18 @@ bool Manager::LoadPluginsFrom(const std::string& dir)
 	return false;
 	}
 
+static bool plugin_cmp(const Plugin* a, const Plugin* b)
+	{
+	return a->Name() < b->Name();
+	}
+
 bool Manager::RegisterPlugin(Plugin *plugin)
 	{
 	Manager::PluginsInternal()->push_back(plugin);
+
+	// Sort plugins by name to make sure we have a deterministic order.
+	PluginsInternal()->sort(plugin_cmp);
+
 	return true;
 	}
 

src/plugin/Plugin.cc

@@ -156,9 +156,18 @@ Plugin::component_list Plugin::Components() const
 	return components;
 	}
 
+static bool component_cmp(const Component* a, const Component* b)
+	{
+	return a->Name() < b->Name();
+	}
+
 void Plugin::AddComponent(Component* c)
 	{
 	components.push_back(c);
+
+	// Sort components by name to make sure we have a deterministic
+	// order.
+	components.sort(component_cmp);
 	}
 
 void Plugin::AddBifInitFunction(bif_init_func c)

testing/btest/Baseline/core.print-bpf-filters/conn.log

@@ -3,8 +3,8 @@
 #empty_field	(empty)
 #unset_field	-
 #path	conn
-#open	2013-07-08-20-05-18
+#open	2013-07-18-00-18-33
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	service	duration	orig_bytes	resp_bytes	conn_state	local_orig	missed_bytes	history	orig_pkts	orig_ip_bytes	resp_pkts	resp_ip_bytes	tunnel_parents
 #types	time	string	addr	port	addr	port	enum	string	interval	count	count	string	bool	count	string	count	count	count	count	table[string]
 1278600802.069419	UWkUyAuUGXf	10.20.80.1	50343	10.0.0.15	80	tcp	-	0.004152	9	3429	SF	-	0	ShADadfF	7	381	7	3801	(empty)
-#close	2013-07-08-20-05-18
+#close	2013-07-18-00-18-33

testing/btest/Baseline/core.print-bpf-filters/output

@@ -3,38 +3,28 @@
 #empty_field	(empty)
 #unset_field	-
 #path	packet_filter
-#open	2013-07-08-20-05-17
+#open	2013-07-19-02-54-13
 #fields	ts	node	filter	init	success
 #types	time	string	string	bool	bool
-1373313917.926565	-	ip or not ip	T	T
+1374202453.158981	-	ip or not ip	T	T
-#close	2013-07-08-20-05-17
+#close	2013-07-19-02-54-13
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	packet_filter
-#open	2013-07-08-20-05-18
+#open	2013-07-19-02-54-13
 #fields	ts	node	filter	init	success
 #types	time	string	string	bool	bool
-1373313918.205206	-	port 42	T	T
+1374202453.437816	-	port 42	T	T
-#close	2013-07-08-20-05-18
+#close	2013-07-19-02-54-13
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	packet_filter
-#open	2013-07-08-20-05-18
+#open	2013-07-19-02-54-13
 #fields	ts	node	filter	init	success
 #types	time	string	string	bool	bool
-1373313918.491383	-	(vlan) and (ip or not ip)	T	T
+1374202453.715717	-	(vlan) and (ip or not ip)	T	T
-#close	2013-07-08-20-05-18
+#close	2013-07-19-02-54-13
-#separator \x09
-#set_separator	,
-#empty_field	(empty)
-#unset_field	-
-#path	packet_filter
-#open	2013-07-08-20-05-18
-#fields	ts	node	filter	init	success
-#types	time	string	string	bool	bool
-1373313918.795264	-	((((((((((((udp and port 3544) or (udp and port 514)) or ((tcp and port 2811) or (tcp and port 21))) or (tcp and port 502)) or ((((tcp and port 6669) or (tcp and port 6666)) or (tcp and port 6668)) or (tcp and port 6667))) or (tcp and port 1080)) or ((udp and port 2152) or (udp and port 2123))) or ((((((((tcp and port 631) or (tcp and port 8888)) or (tcp and port 3128)) or (tcp and port 80)) or (tcp and port 1080)) or (tcp and port 8000)) or (tcp and port 81)) or (tcp and port 8080))) or (udp and port 5072)) or ((tcp and port 25) or (tcp and port 587))) or (((((((((((tcp and port 5223) or (tcp and port 585)) or (tcp and port 614)) or (tcp and port 993)) or (tcp and port 636)) or (tcp and port 989)) or (tcp and port 995)) or (tcp and port 443)) or (tcp and port 563)) or (tcp and port 990)) or (tcp and port 992))) or (((((udp and port 5355) or (tcp and port 53)) or (udp and port 5353)) or (udp and port 137)) or (udp and port 53))) or (tcp and port 22)	T	T
-#close	2013-07-08-20-05-18

testing/btest/Baseline/core.print-bpf-filters/output2

@@ -0,0 +1,43 @@
+2 1080
+1 137
+1 21
+1 2123
+1 2152
+1 22
+1 25
+1 2811
+1 3128
+1 3544
+1 443
+1 502
+1 5072
+1 514
+1 5223
+2 53
+1 5353
+1 5355
+1 563
+1 585
+1 587
+1 614
+1 631
+1 636
+1 6666
+1 6667
+1 6668
+1 6669
+1 80
+1 8000
+1 8080
+1 81
+1 8888
+1 989
+1 990
+1 992
+1 993
+1 995
+40 and
+39 or
+40 port
+31 tcp
+9 udp

testing/btest/Baseline/scripts.base.protocols.dns.duplicate-reponses/dns.log

@@ -0,0 +1,11 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	dns
+#open	2013-07-18-13-21-52
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	trans_id	query	qclass	qclass_name	qtype	qtype_name	rcode	rcode_name	AA	TC	RD	RA	Z	answers	TTLs	rejected
+#types	time	string	addr	port	addr	port	enum	count	string	count	string	count	string	count	string	bool	bool	bool	bool	count	vector[string]	vector[interval]	bool
+1363716396.798072	UWkUyAuUGXf	55.247.223.174	27285	222.195.43.124	53	udp	21140	www.cmu.edu	1	C_INTERNET	1	A	0	NOERROR	T	F	F	F	1	www-cmu.andrew.cmu.edu,www-cmu-2.andrew.cmu.edu,128.2.10.163,www-cmu.andrew.cmu.edu	86400.000000,5.000000,21600.000000,86400.000000	F
+1363716396.798374	UWkUyAuUGXf	55.247.223.174	27285	222.195.43.124	53	udp	21140	-	-	-	-	-	0	NOERROR	T	F	F	F	0	www-cmu-2.andrew.cmu.edu,128.2.10.163	5.000000,21600.000000	F
+#close	2013-07-18-13-21-52

testing/btest/Baseline/scripts.base.protocols.dns.duplicate-reponses/weird.log

@@ -0,0 +1,11 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	weird
+#open	2013-07-18-13-21-52
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	name	addl	notice	peer
+#types	time	string	addr	port	addr	port	string	string	bool	string
+1363716396.798286	UWkUyAuUGXf	55.247.223.174	27285	222.195.43.124	53	DNS_RR_unknown_type	-	F	bro
+1363716396.798374	UWkUyAuUGXf	55.247.223.174	27285	222.195.43.124	53	dns_unmatched_reply	-	F	bro
+#close	2013-07-18-13-21-52

testing/btest/core/print-bpf-filters.bro

@@ -4,7 +4,12 @@
 # @TEST-EXEC: cat packet_filter.log >>output
 # @TEST-EXEC: bro -r $TRACES/mixed-vlan-mpls.trace PacketFilter::restricted_filter="vlan" >>output
 # @TEST-EXEC: cat packet_filter.log >>output
-# @TEST-EXEC: bro -r $TRACES/empty.trace PacketFilter::enable_auto_protocol_capture_filters=T >>output
-# @TEST-EXEC: cat packet_filter.log >>output
 # @TEST-EXEC: btest-diff output
 # @TEST-EXEC: btest-diff conn.log
+# 
+# The order in the output of enable_auto_protocol_capture_filters isn't
+# stable, for reasons not clear. We canonify it first.
+# @TEST-EXEC: bro -r $TRACES/empty.trace PacketFilter::enable_auto_protocol_capture_filters=T
+# @TEST-EXEC: cat packet_filter.log | bro-cut filter | sed 's#[()]##g' | tr ' ' '\n' | sort | uniq -c | awk '{print $1, $2}' >output2
+# @TEST-EXEC: btest-diff output2
+

testing/btest/scripts/base/protocols/dns/duplicate-reponses.bro

@@ -0,0 +1,5 @@
+# This tests the case where the DNS server responded with zero RRs.
+#
+# @TEST-EXEC: bro -r $TRACES/dns-two-responses.trace
+# @TEST-EXEC: btest-diff dns.log
+# @TEST-EXEC: btest-diff weird.log

testing/scripts/diff-canonifier-external

@@ -2,10 +2,17 @@
 #
 # Default canonifier used with the trace-based tests in testing/external/*.
 
+addl="cat"
+
+if [ "$1" == "capture_loss.log" ]; then
+    addl="`dirname $0`/diff-remove-fractions"
+fi
+
 `dirname $0`/diff-remove-timestamps \
     | `dirname $0`/diff-remove-uids \
     | `dirname $0`/diff-remove-file-ids \
     | `dirname $0`/diff-remove-x509-names \
     | `dirname $0`/diff-canon-notice-policy \
-    | `dirname $0`/diff-sort
+    | `dirname $0`/diff-sort \
+    | eval $addl
 

testing/scripts/diff-remove-fractions

@@ -0,0 +1,6 @@
+#! /usr/bin/env bash
+#
+# Replace fractions of double value (i.e., 3.14 -> 3.x).
+
+sed 's/\.[0-9]\{1,\}/.X/g'
+