Merge branch 'master' of ssh://git.bro-ids.org/bro

CHANGES

@@ -1,4 +1,62 @@
 
+1.6-dev-1316 | 2011-09-28 16:50:05 -0700
+
+  * Unit test cleanup. Updated README and collected coverage-related
+    tests in a common dir. (Jon Siwek)
+
+  * Fixes for known-services. (Seth Hall)
+
+  * Ported and 2.0ized the capture-loss script. (Seth Hall)
+
+  * Communication fix and extension.(Robin Sommer)
+
+    - Removing unnecessary log flushing. Closes #498.
+
+    - Adding new BiF disconnect() that shuts a connection to a peer down.
+
+    - terminate_connection() now first flushes any still buffered log
+      messages.
+
+  * Fix for high SSL memory usage by adding &transient attribute to
+    top-level SSL pac array type. Closes #574. (Robin Sommer)
+
+  * Fix a small bug in the metrics framework. (Seth Hall)
+
+  * Temporarily removing scripts that aren't ready to be included.
+    Will return before next release.  (Seth Hall)
+
+  * New SSL policy scripts.  (Seth Hall)
+
+    - protocols/ssl/expiring-certs uses time based information from
+      certificates to determine if they will expire soon, have already
+      expired, or haven't yet become valid.
+
+    - protocols/ssl/extract-certs-pem is a script for taking certs off
+      the line and converting them to PEM certificates with the openssl
+      command line tool then dumping them to a file.
+
+  * Notice::type_suppression_intervals: table[Notice::Type] of
+    interval can be used to modify the suppression intervals for
+    entire types of notices. (Seth Hall)
+
+  * EOF SSL protocol violations are only generated a single time now.
+    (Seth Hall)
+
+  * Script level fixes.  (Seth Hall)
+
+    - Fixed a type name conflict in the Known namespace.
+
+    - Fixed a DPD framework bug that was causing Reporter messages.
+
+    - Fixed the notice_policy log.
+
+    - Predicate functions are now logged.
+
+    - Predicate functions are now optional.  If not given, it's assumed that
+      the result should always apply. (Seth Hall)
+
+    - Fix a problem with accidental and mistaken HTTP log lines.
+
 1.6-dev-1293 | 2011-09-22 19:44:37 -0700
 
   * Smaller script tweaks. (Seth Hall)

VERSION

@@ -1 +1 @@
-1.6-dev-1293
+1.6-dev-1316

aux/broctl

@@ -1 +1 @@
-Subproject commit 01720883d2ba5584817964f6c30bef88b865726e
+Subproject commit f90d3eded266b4effbdd607f76768dd010c7f3b5

doc/scripts/DocSourcesList.cmake

@@ -42,6 +42,7 @@ rest_target(${psd} base/frameworks/notice/actions/add-geodata.bro)
 rest_target(${psd} base/frameworks/notice/actions/drop.bro)
 rest_target(${psd} base/frameworks/notice/actions/email_admin.bro)
 rest_target(${psd} base/frameworks/notice/actions/page.bro)
+rest_target(${psd} base/frameworks/notice/cluster.bro)
 rest_target(${psd} base/frameworks/notice/extend-email/hostnames.bro)
 rest_target(${psd} base/frameworks/notice/main.bro)
 rest_target(${psd} base/frameworks/notice/weird.bro)
@@ -125,6 +126,8 @@ rest_target(${psd} policy/protocols/ssh/detect-bruteforcing.bro)
 rest_target(${psd} policy/protocols/ssh/geo-data.bro)
 rest_target(${psd} policy/protocols/ssh/interesting-hostnames.bro)
 rest_target(${psd} policy/protocols/ssh/software.bro)
+rest_target(${psd} policy/protocols/ssl/expiring-certs.bro)
+rest_target(${psd} policy/protocols/ssl/extract-certs-pem.bro)
 rest_target(${psd} policy/protocols/ssl/known-certs.bro)
 rest_target(${psd} policy/protocols/ssl/validate-certs.bro)
 rest_target(${psd} policy/tuning/defaults/packet-fragments.bro)

scripts/base/frameworks/logging/__load__.bro

@@ -1,3 +1,3 @@
 @load ./main
-
+@load ./postprocessors
 @load ./writers/ascii

scripts/base/frameworks/logging/postprocessors/__load__.bro

@@ -0,0 +1 @@
+@load ./scp

scripts/base/frameworks/notice/__load__.bro

@@ -17,4 +17,4 @@
 
 @if ( Cluster::is_enabled() )
 @load ./cluster
-@endif
+@endif

scripts/base/protocols/http/main.bro

@@ -48,6 +48,10 @@ export {
 		status_code:             count    &log &optional;
 		## The status message returned by the server.
 		status_msg:              string   &log &optional;
+		## The last 1xx informational reply code returned by the server.
+		info_code:               count    &log &optional;
+		## The last 1xx informational reply message returned by the server.
+		info_msg:               string    &log &optional;
 		## The filename given in the Content-Disposition header
 		## sent by the server.
 		filename:                string   &log &optional;
@@ -111,6 +115,11 @@ redef capture_filters +=  {
 	["http"] = "tcp and port (80 or 81 or 631 or 1080 or 3138 or 8000 or 8080 or 8888)"
 };
 
+function code_in_range(c: count, min: count, max: count) : bool
+	{
+	return c >= min && c <= max;
+	}
+
 function new_http_session(c: connection): Info
 	{
 	local tmp: Info;
@@ -163,12 +172,21 @@ event http_reply(c: connection, version: string, code: count, reason: string) &p
 		local s: State;
 		c$http_state = s;
 		}
-	
+
-	++c$http_state$current_response;
+	# If the last response was an informational 1xx, we're still expecting
+	# the real response to the request, so don't create a new Info record yet.
+	if ( c$http_state$current_response !in c$http_state$pending ||
+	  ! code_in_range(c$http_state$pending[c$http_state$current_response]$status_code, 100, 199) )
+		++c$http_state$current_response;
 	set_state(c, F, F);
 	
 	c$http$status_code = code;
 	c$http$status_msg = reason;
+	if ( code_in_range(code, 100, 199) )
+		{
+		c$http$info_code = code;
+		c$http$info_msg = reason;
+		}
 	}
 	
 event http_header(c: connection, is_orig: bool, name: string, value: string) &priority=5
@@ -245,8 +263,13 @@ event http_message_done(c: connection, is_orig: bool, stat: http_message_stat) &
 	# The reply body is done so we're ready to log.
 	if ( ! is_orig )
 		{
-		Log::write(HTTP::LOG, c$http);
+		# If the response was an informational 1xx, we're still expecting
-		delete c$http_state$pending[c$http_state$current_response];
+		# the real response later, so we'll continue using the same record.
+		if ( ! code_in_range(c$http$status_code, 100, 199) )
+			{
+			Log::write(HTTP::LOG, c$http);
+			delete c$http_state$pending[c$http_state$current_response];
+			}
 		}
 	}
 

scripts/policy/protocols/ssl/expiring-certs.bro

@@ -5,6 +5,7 @@
 
 @load base/protocols/ssl
 @load base/frameworks/notice
+@load base/utils/directions-and-hosts
 
 module SSL;
 
@@ -58,5 +59,3 @@ event x509_certificate(c: connection, cert: X509, is_server: bool, chain_idx: co
 		        $conn=c, $suppress_for=1day,
 		        $identifier=fmt("%s:%d-%s", c$id$resp_h, c$id$resp_p, md5_hash(der_cert))]);
 	}
-	
-	

scripts/policy/protocols/ssl/extract-certs-pem.bro

@@ -14,6 +14,7 @@
 ##!
 
 @load base/protocols/ssl
+@load base/utils/directions-and-hosts
 
 module SSL;
 
@@ -45,4 +46,4 @@ event ssl_established(c: connection)
 	local side = Site::is_local_addr(c$id$resp_h) ? "local" : "remote";
 	local cmd = fmt("%s x509 -inform DER -outform PEM >> certs-%s.pem", openssl_util, side);
 	piped_exec(cmd, c$ssl$cert);
-	}
+	}

scripts/test-all-policy.bro

@@ -49,6 +49,8 @@
 @load protocols/ssh/geo-data.bro
 @load protocols/ssh/interesting-hostnames.bro
 @load protocols/ssh/software.bro
+@load protocols/ssl/expiring-certs.bro
+@load protocols/ssl/extract-certs-pem.bro
 @load protocols/ssl/known-certs.bro
 @load protocols/ssl/validate-certs.bro
 @load tuning/__load__.bro

src/HTTP.cc

@@ -5,6 +5,8 @@
 #include <ctype.h>
 #include <math.h>
 #include <stdlib.h>
+#include <string>
+#include <algorithm>
 
 #include "NetVar.h"
 #include "HTTP.h"
@@ -310,6 +312,67 @@ void HTTP_Entity::SubmitHeader(MIME_Header* h)
 			}
 		}
 
+	// Figure out content-length for HTTP 206 Partial Content response
+	// that uses multipart/byteranges content-type.
+	else if ( strcasecmp_n(h->get_name(), "content-range") == 0 && Parent() &&
+	          Parent()->MIMEContentType() == CONTENT_TYPE_MULTIPART &&
+		      http_message->MyHTTP_Analyzer()->HTTP_ReplyCode() == 206 )
+		{
+		data_chunk_t vt = h->get_value_token();
+		string byte_unit(vt.data, vt.length);
+		vt = h->get_value_after_token();
+		string byte_range(vt.data, vt.length);
+		byte_range.erase(remove(byte_range.begin(), byte_range.end(), ' '),
+		                 byte_range.end());
+
+		if ( byte_unit != "bytes" )
+			{
+			http_message->Weird("HTTP_content_range_unknown_byte_unit");
+			return;
+			}
+
+		size_t p = byte_range.find("/");
+		if ( p == string::npos )
+			{
+			http_message->Weird("HTTP_content_range_cannot_parse");
+			return;
+			}
+
+		string byte_range_resp_spec = byte_range.substr(0, p);
+		string instance_length = byte_range.substr(p + 1);
+
+		p = byte_range_resp_spec.find("-");
+		if ( p == string::npos )
+			{
+			http_message->Weird("HTTP_content_range_cannot_parse");
+			return;
+			}
+
+		string first_byte_pos = byte_range_resp_spec.substr(0, p);
+		string last_byte_pos = byte_range_resp_spec.substr(p + 1);
+
+		if ( DEBUG_http )
+			DEBUG_MSG("Parsed Content-Range: %s %s-%s/%s\n", byte_unit.c_str(),
+		              first_byte_pos.c_str(), last_byte_pos.c_str(),
+		              instance_length.c_str());
+
+		int64_t f, l;
+		atoi_n(first_byte_pos.size(), first_byte_pos.c_str(), 0, 10, f);
+		atoi_n(last_byte_pos.size(), last_byte_pos.c_str(), 0, 10, l);
+		int64_t len = l - f + 1;
+
+		if ( DEBUG_http )
+			DEBUG_MSG("Content-Range length = %"PRId64"\n", len);
+
+		if ( len > 0 )
+			content_length = len;
+		else
+			{
+			http_message->Weird("HTTP_non_positive_content_range");
+			return;
+			}
+		}
+
 	else if ( strcasecmp_n(h->get_name(), "transfer-encoding") == 0 )
 		{
 		data_chunk_t vt = h->get_value_token();
@@ -1305,7 +1368,9 @@ void HTTP_Analyzer::ReplyMade(const int interrupted, const char* msg)
 	if ( reply_message )
 		reply_message->Done(interrupted, msg);
 
-	if ( ! unanswered_requests.empty() )
+	// 1xx replies do not indicate the final response to a request,
+	// so don't pop an unanswered request in that case.
+	if ( (reply_code < 100 || reply_code >= 200) && ! unanswered_requests.empty() )
 		{
 		Unref(unanswered_requests.front());
 		unanswered_requests.pop();

src/HTTP.h

@@ -163,6 +163,9 @@ public:
 
 	void SkipEntityData(int is_orig);
 
+	int IsConnectionClose()		{ return connection_close; }
+	int HTTP_ReplyCode() const { return reply_code; };
+
 	// Overriden from Analyzer.
 	virtual void Done();
 	virtual void DeliverStream(int len, const u_char* data, bool orig);
@@ -183,8 +186,6 @@ public:
 			http_content_type || http_entity_data || http_message_done ||
 			http_event || http_stats) && !FLAGS_use_binpac; }
 
-	int IsConnectionClose()		{ return connection_close; }
-
 protected:
 	void GenStats();
 

src/MIME.h

@@ -95,6 +95,7 @@ public:
 	virtual void EndOfData();
 
 	MIME_Entity* Parent() const { return parent; }
+	int MIMEContentType() const { return content_type; }
 	StringVal* ContentType() const { return content_type_str; }
 	StringVal* ContentSubType() const { return content_subtype_str; }
 	int ContentTransferEncoding() const { return content_encoding; }

testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log

@@ -12,5 +12,7 @@
 1	scripts/base/frameworks/logging/__load__.bro
 2	scripts/base/frameworks/logging/./main.bro
 3	build/src/base/logging.bif.bro
+2	scripts/base/frameworks/logging/./postprocessors/__load__.bro
+3	scripts/base/frameworks/logging/./postprocessors/./scp.bro
 2	scripts/base/frameworks/logging/./writers/ascii.bro
 0	scripts/policy/misc/loaded-scripts.bro

testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log

@@ -12,6 +12,8 @@
 1	scripts/base/frameworks/logging/__load__.bro
 2	scripts/base/frameworks/logging/./main.bro
 3	build/src/base/logging.bif.bro
+2	scripts/base/frameworks/logging/./postprocessors/__load__.bro
+3	scripts/base/frameworks/logging/./postprocessors/./scp.bro
 2	scripts/base/frameworks/logging/./writers/ascii.bro
 0	scripts/base/init-default.bro
 1	scripts/base/utils/site.bro

testing/btest/Baseline/coverage.init-default/missing_loads

@@ -0,0 +1,6 @@
+-./frameworks/cluster/nodes/manager.bro
+-./frameworks/cluster/nodes/proxy.bro
+-./frameworks/cluster/nodes/worker.bro
+-./frameworks/cluster/setup-connections.bro
+-./frameworks/metrics/cluster.bro
+-./frameworks/notice/cluster.bro

testing/btest/Baseline/istate.events-ssl/receiver.http.log

@@ -1,5 +1,5 @@
 #separator \x09
 #path	http
-#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	method	host	uri	referrer	user_agent	request_body_len	request_body_interrupted	response_body_len	response_body_interrupted	status_code	status_msg	filename	tags	username	password	proxied	mime_type	md5	extraction_file
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	method	host	uri	referrer	user_agent	request_body_len	request_body_interrupted	response_body_len	response_body_interrupted	status_code	status_msg	info_code	info_msg	filename	tags	username	password	proxied	mime_type	md5	extraction_file
-#types	time	string	addr	port	addr	port	string	string	string	string	string	count	bool	count	bool	count	string	string	table	string	string	table	string	string	file
+#types	time	string	addr	port	addr	port	string	string	string	string	string	count	bool	count	bool	count	string	count	string	string	table	string	string	table	string	string	file
-1316124231.969273	arKYeMETxOg	141.42.64.125	56730	125.190.109.199	80	GET	www.icir.org	/	-	Wget/1.10	0	F	9130	F	200	OK	-	-	-	-	-	text/html	-	-
+1317149787.593092	arKYeMETxOg	141.42.64.125	56730	125.190.109.199	80	GET	www.icir.org	/	-	Wget/1.10	0	F	9130	F	200	OK	-	-	-	-	-	-	-	text/html	-	-

testing/btest/Baseline/istate.events-ssl/sender.http.log

@@ -1,5 +1,5 @@
 #separator \x09
 #path	http
-#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	method	host	uri	referrer	user_agent	request_body_len	request_body_interrupted	response_body_len	response_body_interrupted	status_code	status_msg	filename	tags	username	password	proxied	mime_type	md5	extraction_file
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	method	host	uri	referrer	user_agent	request_body_len	request_body_interrupted	response_body_len	response_body_interrupted	status_code	status_msg	info_code	info_msg	filename	tags	username	password	proxied	mime_type	md5	extraction_file
-#types	time	string	addr	port	addr	port	string	string	string	string	string	count	bool	count	bool	count	string	string	table	string	string	table	string	string	file
+#types	time	string	addr	port	addr	port	string	string	string	string	string	count	bool	count	bool	count	string	count	string	string	table	string	string	table	string	string	file
-1316124231.969273	arKYeMETxOg	141.42.64.125	56730	125.190.109.199	80	GET	www.icir.org	/	-	Wget/1.10	0	F	9130	F	200	OK	-	-	-	-	-	text/html	-	-
+1317149787.593092	arKYeMETxOg	141.42.64.125	56730	125.190.109.199	80	GET	www.icir.org	/	-	Wget/1.10	0	F	9130	F	200	OK	-	-	-	-	-	-	-	text/html	-	-

testing/btest/Baseline/istate.events/receiver.http.log

@@ -1,5 +1,5 @@
 #separator \x09
 #path	http
-#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	method	host	uri	referrer	user_agent	request_body_len	request_body_interrupted	response_body_len	response_body_interrupted	status_code	status_msg	filename	tags	username	password	proxied	mime_type	md5	extraction_file
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	method	host	uri	referrer	user_agent	request_body_len	request_body_interrupted	response_body_len	response_body_interrupted	status_code	status_msg	info_code	info_msg	filename	tags	username	password	proxied	mime_type	md5	extraction_file
-#types	time	string	addr	port	addr	port	string	string	string	string	string	count	bool	count	bool	count	string	string	table	string	string	table	string	string	file
+#types	time	string	addr	port	addr	port	string	string	string	string	string	count	bool	count	bool	count	string	count	string	string	table	string	string	table	string	string	file
-1316124240.720195	arKYeMETxOg	141.42.64.125	56730	125.190.109.199	80	GET	www.icir.org	/	-	Wget/1.10	0	F	9130	F	200	OK	-	-	-	-	-	text/html	-	-
+1317149750.648989	arKYeMETxOg	141.42.64.125	56730	125.190.109.199	80	GET	www.icir.org	/	-	Wget/1.10	0	F	9130	F	200	OK	-	-	-	-	-	-	-	text/html	-	-

testing/btest/Baseline/istate.events/sender.http.log

@@ -1,5 +1,5 @@
 #separator \x09
 #path	http
-#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	method	host	uri	referrer	user_agent	request_body_len	request_body_interrupted	response_body_len	response_body_interrupted	status_code	status_msg	filename	tags	username	password	proxied	mime_type	md5	extraction_file
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	method	host	uri	referrer	user_agent	request_body_len	request_body_interrupted	response_body_len	response_body_interrupted	status_code	status_msg	info_code	info_msg	filename	tags	username	password	proxied	mime_type	md5	extraction_file
-#types	time	string	addr	port	addr	port	string	string	string	string	string	count	bool	count	bool	count	string	string	table	string	string	table	string	string	file
+#types	time	string	addr	port	addr	port	string	string	string	string	string	count	bool	count	bool	count	string	count	string	string	table	string	string	table	string	string	file
-1316124240.720195	arKYeMETxOg	141.42.64.125	56730	125.190.109.199	80	GET	www.icir.org	/	-	Wget/1.10	0	F	9130	F	200	OK	-	-	-	-	-	text/html	-	-
+1317149750.648989	arKYeMETxOg	141.42.64.125	56730	125.190.109.199	80	GET	www.icir.org	/	-	Wget/1.10	0	F	9130	F	200	OK	-	-	-	-	-	-	-	text/html	-	-

testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape-odd-url/http.log

@@ -1,5 +1,5 @@
 #separator \x09
 #path	http
-#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	method	host	uri	referrer	user_agent	request_body_len	request_body_interrupted	response_body_len	response_body_interrupted	status_code	status_msg	filename	tags	username	password	proxied	mime_type	md5	extraction_file
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	method	host	uri	referrer	user_agent	request_body_len	request_body_interrupted	response_body_len	response_body_interrupted	status_code	status_msg	info_code	info_msg	filename	tags	username	password	proxied	mime_type	md5	extraction_file
-#types	time	string	addr	port	addr	port	string	string	string	string	string	count	bool	count	bool	count	string	string	table	string	string	table	string	string	file
+#types	time	string	addr	port	addr	port	string	string	string	string	string	count	bool	count	bool	count	string	count	string	string	table	string	string	table	string	string	file
-1315799856.264750	UWkUyAuUGXf	10.0.1.104	64216	193.40.5.162	80	GET	lepo.it.da.ut.ee	/~cect/teoreetilised seminarid_2010/arheoloogia_uurimisr\xfchma_seminar/Joyce et al - The Languages of Archaeology ~ Dialogue, Narrative and Writing.pdf	-	Wget/1.12 (darwin10.8.0)	0	F	346	F	404	Not Found	-	-	-	-	-	text/html	-	-
+1315799856.264750	UWkUyAuUGXf	10.0.1.104	64216	193.40.5.162	80	GET	lepo.it.da.ut.ee	/~cect/teoreetilised seminarid_2010/arheoloogia_uurimisr\xfchma_seminar/Joyce et al - The Languages of Archaeology ~ Dialogue, Narrative and Writing.pdf	-	Wget/1.12 (darwin10.8.0)	0	F	346	F	404	Not Found	-	-	-	-	-	-	-	text/html	-	-

testing/btest/Baseline/scripts.base.protocols.http.100-continue/http.log

@@ -0,0 +1,5 @@
+#separator \x09
+#path	http
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	method	host	uri	referrer	user_agent	request_body_len	request_body_interrupted	response_body_len	response_body_interrupted	status_code	status_msg	info_code	info_msg	filename	tags	username	password	proxied	mime_type	md5	extraction_file
+#types	time	string	addr	port	addr	port	string	string	string	string	string	count	bool	count	bool	count	string	count	string	string	table	string	string	table	string	string	file
+1237440095.634312	UWkUyAuUGXf	192.168.3.103	54102	128.146.216.51	80	POST	www.osu.edu	/	-	curl/7.17.1 (i386-apple-darwin8.11.1) libcurl/7.17.1 zlib/1.2.3	2001	F	60731	F	200	OK	100	Continue	-	-	-	-	-	text/html	-	-

testing/btest/Baseline/scripts.base.protocols.http.http-mime-and-md5/http.log

@@ -1,9 +1,9 @@
 #separator \x09
 #path	http
-#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	method	host	uri	referrer	user_agent	request_body_len	request_body_interrupted	response_body_len	response_body_interrupted	status_code	status_msg	filename	tags	username	password	proxied	mime_type	md5	extraction_file
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	method	host	uri	referrer	user_agent	request_body_len	request_body_interrupted	response_body_len	response_body_interrupted	status_code	status_msg	info_code	info_msg	filename	tags	username	password	proxied	mime_type	md5	extraction_file
-#types	time	string	addr	port	addr	port	string	string	string	string	string	count	bool	count	bool	count	string	string	table	string	string	table	string	string	file
+#types	time	string	addr	port	addr	port	string	string	string	string	string	count	bool	count	bool	count	string	count	string	string	table	string	string	table	string	string	file
-1258577884.844956	UWkUyAuUGXf	192.168.1.104	1673	63.245.209.11	80	GET	www.mozilla.org	/style/enhanced.css	http://www.mozilla.org/projects/calendar/	Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5	0	F	2675	F	200	OK	-	-	-	-	-	FAKE_MIME	-	-
+1258577884.844956	UWkUyAuUGXf	192.168.1.104	1673	63.245.209.11	80	GET	www.mozilla.org	/style/enhanced.css	http://www.mozilla.org/projects/calendar/	Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5	0	F	2675	F	200	OK	-	-	-	-	-	-	-	FAKE_MIME	-	-
-1258577884.960135	UWkUyAuUGXf	192.168.1.104	1673	63.245.209.11	80	GET	www.mozilla.org	/script/urchin.js	http://www.mozilla.org/projects/calendar/	Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5	0	F	21421	F	200	OK	-	-	-	-	-	FAKE_MIME	-	-
+1258577884.960135	UWkUyAuUGXf	192.168.1.104	1673	63.245.209.11	80	GET	www.mozilla.org	/script/urchin.js	http://www.mozilla.org/projects/calendar/	Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5	0	F	21421	F	200	OK	-	-	-	-	-	-	-	FAKE_MIME	-	-
-1258577885.317160	UWkUyAuUGXf	192.168.1.104	1673	63.245.209.11	80	GET	www.mozilla.org	/images/template/screen/bullet_utility.png	http://www.mozilla.org/style/screen.css	Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5	0	F	94	F	200	OK	-	-	-	-	-	FAKE_MIME	-	-
+1258577885.317160	UWkUyAuUGXf	192.168.1.104	1673	63.245.209.11	80	GET	www.mozilla.org	/images/template/screen/bullet_utility.png	http://www.mozilla.org/style/screen.css	Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5	0	F	94	F	200	OK	-	-	-	-	-	-	-	FAKE_MIME	-	-
-1258577885.349639	UWkUyAuUGXf	192.168.1.104	1673	63.245.209.11	80	GET	www.mozilla.org	/images/template/screen/key-point-top.png	http://www.mozilla.org/style/screen.css	Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5	0	F	2349	F	200	OK	-	-	-	-	-	image/png	e0029eea80812e9a8e57b8d05d52938a	-
+1258577885.349639	UWkUyAuUGXf	192.168.1.104	1673	63.245.209.11	80	GET	www.mozilla.org	/images/template/screen/key-point-top.png	http://www.mozilla.org/style/screen.css	Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5	0	F	2349	F	200	OK	-	-	-	-	-	-	-	image/png	e0029eea80812e9a8e57b8d05d52938a	-
-1258577885.394612	UWkUyAuUGXf	192.168.1.104	1673	63.245.209.11	80	GET	www.mozilla.org	/projects/calendar/images/header-sunbird.png	http://www.mozilla.org/projects/calendar/calendar.css	Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5	0	F	27579	F	200	OK	-	-	-	-	-	image/png	30aa926344f58019d047e85ba049ca1e	-
+1258577885.394612	UWkUyAuUGXf	192.168.1.104	1673	63.245.209.11	80	GET	www.mozilla.org	/projects/calendar/images/header-sunbird.png	http://www.mozilla.org/projects/calendar/calendar.css	Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5	0	F	27579	F	200	OK	-	-	-	-	-	-	-	image/png	30aa926344f58019d047e85ba049ca1e	-

testing/btest/Baseline/scripts.base.protocols.http.http-pipelining/http.log

@@ -1,9 +1,9 @@
 #separator \x09
 #path	http
-#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	method	host	uri	referrer	user_agent	request_body_len	request_body_interrupted	response_body_len	response_body_interrupted	status_code	status_msg	filename	tags	username	password	proxied	md5	extraction_file
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	method	host	uri	referrer	user_agent	request_body_len	request_body_interrupted	response_body_len	response_body_interrupted	status_code	status_msg	info_code	info_msg	filename	tags	username	password	proxied	md5	extraction_file
-#types	time	string	addr	port	addr	port	string	string	string	string	string	count	bool	count	bool	count	string	string	table	string	string	table	string	file
+#types	time	string	addr	port	addr	port	string	string	string	string	string	count	bool	count	bool	count	string	count	string	string	table	string	string	table	string	file
-1258577884.844956	UWkUyAuUGXf	192.168.1.104	1673	63.245.209.11	80	GET	www.mozilla.org	/style/enhanced.css	http://www.mozilla.org/projects/calendar/	Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5	0	F	2675	F	200	OK	-	-	-	-	-	-	-
+1258577884.844956	UWkUyAuUGXf	192.168.1.104	1673	63.245.209.11	80	GET	www.mozilla.org	/style/enhanced.css	http://www.mozilla.org/projects/calendar/	Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5	0	F	2675	F	200	OK	-	-	-	-	-	-	-	-	-
-1258577884.960135	UWkUyAuUGXf	192.168.1.104	1673	63.245.209.11	80	GET	www.mozilla.org	/script/urchin.js	http://www.mozilla.org/projects/calendar/	Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5	0	F	21421	F	200	OK	-	-	-	-	-	-	-
+1258577884.960135	UWkUyAuUGXf	192.168.1.104	1673	63.245.209.11	80	GET	www.mozilla.org	/script/urchin.js	http://www.mozilla.org/projects/calendar/	Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5	0	F	21421	F	200	OK	-	-	-	-	-	-	-	-	-
-1258577885.317160	UWkUyAuUGXf	192.168.1.104	1673	63.245.209.11	80	GET	www.mozilla.org	/images/template/screen/bullet_utility.png	http://www.mozilla.org/style/screen.css	Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5	0	F	94	F	200	OK	-	-	-	-	-	-	-
+1258577885.317160	UWkUyAuUGXf	192.168.1.104	1673	63.245.209.11	80	GET	www.mozilla.org	/images/template/screen/bullet_utility.png	http://www.mozilla.org/style/screen.css	Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5	0	F	94	F	200	OK	-	-	-	-	-	-	-	-	-
-1258577885.349639	UWkUyAuUGXf	192.168.1.104	1673	63.245.209.11	80	GET	www.mozilla.org	/images/template/screen/key-point-top.png	http://www.mozilla.org/style/screen.css	Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5	0	F	2349	F	200	OK	-	-	-	-	-	-	-
+1258577885.349639	UWkUyAuUGXf	192.168.1.104	1673	63.245.209.11	80	GET	www.mozilla.org	/images/template/screen/key-point-top.png	http://www.mozilla.org/style/screen.css	Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5	0	F	2349	F	200	OK	-	-	-	-	-	-	-	-	-
-1258577885.394612	UWkUyAuUGXf	192.168.1.104	1673	63.245.209.11	80	GET	www.mozilla.org	/projects/calendar/images/header-sunbird.png	http://www.mozilla.org/projects/calendar/calendar.css	Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5	0	F	27579	F	200	OK	-	-	-	-	-	-	-
+1258577885.394612	UWkUyAuUGXf	192.168.1.104	1673	63.245.209.11	80	GET	www.mozilla.org	/projects/calendar/images/header-sunbird.png	http://www.mozilla.org/projects/calendar/calendar.css	Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5	0	F	27579	F	200	OK	-	-	-	-	-	-	-	-	-

testing/btest/README

@@ -1,97 +1,85 @@
-BTest is simple framework for writing unit tests. Each test consists of a set
+This a test suite of small "unit tests" that verify individual pieces of Bro
-of command lines that will be executed, and success is determined based on
+functionality.  They all utilize BTest, a simple framework/driver for
-their exit codes. In addition, output can optionally be compared against a
+writing unit tests.  More information about BTest can be found at
-previously established baseline.
+http://www.bro-ids.org/development/btest.html
 
-More information about BTest can be found at http://www.icir.org/robin/btest/
+The test suite's BTest configuration is handled through the
+``btest.cfg`` file.  Of particular interest is the "TestDirs" settings,
+which specifies which directories BTest will recursively search for
+test files.
 
+Significant Subdirectories
+==========================
 
-This README contains the following sections:
+* Baseline/
-	* Contents of the testing/btest/ directory
+	Validated baselines for comparison against the output of each
-	* Running tests
+	test on future runs. If the new output differs from the Baseline
-	* Adding tests
+	output, then the test fails.
 
-
+* Traces/
-Contents of the testing/btest/ directory:
-
-Baseline/*/
-	The validated baselines for comparison against the output of each test on
-	future runs. If the new output differs from the Baseline output, then the
-	test fails.
-
-Scripts/
-	Shell scripts invoked by BTest to support testing.
-
-Traces/
 	Packet captures utilized by the various BTest tests.
 
-logging/
+* scripts/
-	Tests to validate the logging framework.
+    This hierarchy of tests emulates the hierarchy of the Bro scripts/
+    directory.
 
-policy/
+* coverage/
-	Tests of the functionality of Bro's bundled policy scripts.
+    This collection of tests relates to checking whether we're covering
+    everything we want to in terms of tests, documentation, and which
+    scripts get loaded in different Bro configurations.  These tests are
+    more prone to fail as new Bro scripts are developed and added to the
+    distribution -- checking the individual test's comments is the best
+    place to check for more details on what exactly the test is checking
+    and hints on how to fix it when it fails.
 
-software/
+Running Tests
-	Tests to validate Bro software not tested elsewhere.
+=============
 
-btest.cfg
+Either use the ``make all`` or ``make brief`` ``Makefile`` targets, or
-	Configuration file that specifies run-time settings for BTest. Of particular
+run ``btest`` directly with desired options/arguments.  Examples:
-	interest is the "TestDirs" settings, which specifies which directories BTest
+
-	will recursively search for test files.
+* btest <no arguments>
+	If you simply execute btest in this directory with no arguments,
+	then all directories listed as "TestDirs" in btest.cfg will be
+	searched recursively for test files.
 
 
-Running tests:
+* btest <btest options> test_directory
-
+	You can specify a directory on the command line to run just the
-btest  <no arguments>
+	tests contained in that directory. This is useful if you wish to
-	If you simply execute btest in this directory with no arguments, then all
+	run all of a given type of test, without running all the tests
-	directories listed as "TestDirs" in btest.cfg will be searched recursively
+	there are. For example, "btest scripts" will run all of the Bro
-	for	test files. This is how the NMI automated build & test environment
+	script unit tests.
-	invokes BTest to run all tests.
 
 
-btest  test_directory
+* btest <btest options> test_directory/test_file
-	You can specify a directory on the command line to run just the tests
+	You can specify a single test file to run just that test. This
-	contained in that directory. This is useful if you wish to run all of a
+	is useful when testing a single failing test or when developing
-	given type of test, without running all the tests there are. For example,
-	"btest policy" will run all of the tests for Bro's bundled policy scripts.
-
-
-btest  test_directory/test_file
-	You can specify a single test file to run just that test. This is useful
-	when testing a single aspect of Bro functionality, and also when developing
 	a new test.
 
+Adding Tests
+=============
 
+See either the `BTest documentation
+<http://www.bro-ids.org/development/btest.html>`_ or the existing unit
+tests for examples of what they actually look like.  The essential
+components of a new test include:
 
-Adding tests:
+* A test file in one of the subdirectories listed in the ``TestDirs``
+  of the ``btest.cfg`` file.
 
-See the documentation at http://www.icir.org/robin/btest/ for information on
+* If the unit test requires a known-good baseline output against which
-what BTests actually look like.
+  future tests will be compared (via ``btest-diff``), then that baseline
+  output will need to live in the ``Baseline`` directory.  Manually
+  adding that is possible, but it's easier to just use the ``-u`` or
+  ``-U`` options of ``btest`` to do it for you (using ``btest -d`` on a
+  test for which no baseline exists will show you the output so it can
+  be verified first before adding/updating the baseline output).
 
-The essential components of a new test include:
+If you create a new top-level testing directory for collecting related
-* A test file in a subdirectory of /testing/btest. This can be a sub-sub-
+tests, then you'll need to add it to the list of ``TestDirs`` in
-	directory, as the search for test files is recursive from the directories
+``btest.cfg``. Do this only if your test really doesn't fit logically in
-	listed as "TestDirs" in btest.cfg
+any of the extant directories.
-* A baseline for the output of your test. Although the baseline will be stored
-	in testing/btest/Baseline/ you should allow btest to copy the correct files
-	to that location, rather than copying them manually (see below).
 
-If you create a new subdirectory from testing/btest you'll need to add it to the
+Note that any new test you add this way will automatically be included
-list of "TestDirs" in btest.cfg. Do this only if your test really doesn't fit
+in the testing done in the NMI automated build & test environment.
-logically in any of the extant directories.
-
-While developing your test, you can specify the "-t" command-line option to make
-BTest preserve the testing/btest/.tmp directory. This directory holds the output
-from your test run; you can inspect it in place to ensure it is correct and as
-expected.
-
-Once you are satisfied with the results in testing/btest/.tmp you can make BTest
-store this output as the Baseline for the test by specifying the "-U" command-
-line option.
-
-When you are ready to commit your test to git, be sure the testing/btest/.tmp
-directory is deleted, and use "git status" to ensure you correctly identify all
-of the files that should be committed to the repository.
-
-Note that any new test you add this way will automatically be included in the
-testing done in the NMI automated build & test environment.

testing/btest/btest.cfg

@@ -1,5 +1,5 @@
 [btest]
-TestDirs    = doc bifs language core scripts istate
+TestDirs    = doc bifs language core scripts istate coverage
 TmpDir      = %(testbase)s/.tmp
 BaselineDir = %(testbase)s/Baseline
 IgnoreDirs  = .svn CVS .tmp

testing/btest/coverage/bare-load-baseline.test

@@ -1,5 +1,7 @@
 # This test is meant to cover whether the set of scripts that get loaded by
-# default in bare mode matches a baseline of known defaults.
+# default in bare mode matches a baseline of known defaults.  The baseline
+# should only need updating if something new is @load'd from init-bare.bro
+# (or from an @load'd descendent of it).
 #
 # As the output has absolute paths in it, we need to remove the common
 # prefix to make the test work everywhere. That's what the sed magic
@@ -7,6 +9,6 @@
 
 # @TEST-EXEC: bro -b misc/loaded-scripts
 # @TEST-EXEC: test -e loaded_scripts.log
-# @TEST-EXEC: cat loaded_scripts.log | egrep -v '#' | awk 'NR>1{print $2}' | sed -e ':a' -e '$!N' -e 's/^\(.*\).*\n\1.*/\1/' -e 'ta' >prefix
+# @TEST-EXEC: cat loaded_scripts.log | egrep -v '#' | awk 'NR>0{print $2}' | sed -e ':a' -e '$!N' -e 's/^\(.*\).*\n\1.*/\1/' -e 'ta' >prefix
 # @TEST-EXEC: cat loaded_scripts.log | sed "s#`cat prefix`##g" >canonified_loaded_scripts.log
 # @TEST-EXEC: btest-diff canonified_loaded_scripts.log

testing/btest/coverage/bare-mode-errors.test

@@ -1,6 +1,9 @@
-# Makes sure any given policy script in the scripts/ tree can be loaded in
+# Makes sure any given bro script in the scripts/ tree can be loaded in
-# bare mode.  btest-bg-run/btest-bg-wait are used to kill off scripts that
+# bare mode without error.  btest-bg-run/btest-bg-wait are used to kill off
-# block after loading, e.g. start listening on a socket.
+# scripts that block after loading, e.g. start listening on a socket.
+#
+# Commonly, this test may fail if one forgets to @load some base/ scripts
+# when writing a new bro scripts.
 #
 # @TEST-EXEC: test -d $DIST/scripts
 # @TEST-EXEC: for script in `find $DIST/scripts -name \*\.bro`; do echo $script;if [[ "$script" =~ listen-clear|listen-ssl|controllee ]]; then rm -rf load_attempt .bgprocs; btest-bg-run load_attempt bro -b $script; btest-bg-wait -k 2; cat load_attempt/.stderr >>allerrors; else bro -b $script 2>>allerrors; fi done || exit 0

testing/btest/coverage/default-load-baseline.test

@@ -1,5 +1,7 @@
 # This test is meant to cover whether the set of scripts that get loaded by
-# default matches a baseline of known defaults.
+# default matches a baseline of known defaults.  When new scripts are
+# added to the scripts/base/ directory, the baseline will usually just need
+# to be updated.
 #
 # As the output has absolute paths in it, we need to remove the common
 # prefix to make the test work everywhere. That's what the sed magic
@@ -7,6 +9,6 @@
 
 # @TEST-EXEC: bro misc/loaded-scripts
 # @TEST-EXEC: test -e loaded_scripts.log
-# @TEST-EXEC: cat loaded_scripts.log | egrep -v '#' | awk 'NR>1{print $2}' | sed -e ':a' -e '$!N' -e 's/^\(.*\).*\n\1.*/\1/' -e 'ta' >prefix
+# @TEST-EXEC: cat loaded_scripts.log | egrep -v '#' | awk 'NR>0{print $2}' | sed -e ':a' -e '$!N' -e 's/^\(.*\).*\n\1.*/\1/' -e 'ta' >prefix
 # @TEST-EXEC: cat loaded_scripts.log | sed "s#`cat prefix`##g" >canonified_loaded_scripts.log
 # @TEST-EXEC: btest-diff canonified_loaded_scripts.log

testing/btest/coverage/doc.test

@@ -1,5 +1,5 @@
-# This tests that we're generating policy script documentation for all the
+# This tests that we're generating bro script documentation for all the
-# available policy scripts.  If this fails, then the genDocSources.sh needs
+# available bro scripts.  If this fails, then the genDocSources.sh needs
 # to be run to produce a new DocSourcesList.cmake or genDocSources.sh needs
 # to be updated to blacklist undesired scripts.
 #

testing/btest/coverage/init-default.test

@@ -1,11 +1,18 @@
 # Makes sure that all base/* scripts are loaded by default via init-default.bro;
 # and that all scripts loaded there in there actually exist.
+#
+# This test will fail if a new bro script is added under the scripts/base/
+# directory and it is not also added as an @load in base/init-default.bro.
+# In some cases, a script in base is loaded based on the bro configuration
+# (e.g. cluster operation), and in such cases, the missing_loads baseline
+# can be adjusted to tolerate that.
 
 #@TEST-EXEC: test -d $DIST/scripts/base
 #@TEST-EXEC: test -e $DIST/scripts/base/init-default.bro
 #@TEST-EXEC: ( cd $DIST/scripts/base && find . -name '*.bro' ) | sort >"all scripts found"
 #@TEST-EXEC: bro misc/loaded-scripts
-#@TEST-EXEC: cat loaded_scripts.log | egrep -v '/build/|/loaded-scripts.bro|#' | awk 'NR>1{print $2}' | sed 's#/./#/#g'  >loaded_scripts.log.tmp
+#@TEST-EXEC: cat loaded_scripts.log | egrep -v '/build/|/loaded-scripts.bro|#' | awk 'NR>0{print $2}' | sed 's#/./#/#g'  >loaded_scripts.log.tmp
 #@TEST-EXEC: cat loaded_scripts.log.tmp | sed -e ':a' -e '$!N' -e 's/^\(.*\).*\n\1.*/\1/' -e 'ta' >prefix
 #@TEST-EXEC: cat loaded_scripts.log.tmp | sed "s#`cat prefix`#./#g" | sort >init-default.bro
-#@TEST-EXEC: diff -u "all scripts found" init-default.bro 1>&2
+#@TEST-EXEC: diff -u "all scripts found" init-default.bro | egrep "^-[^-]" > missing_loads
+#@TEST-EXEC: btest-diff missing_loads

testing/btest/coverage/test-all-policy.test

@@ -1,5 +1,9 @@
-# Makes sure that all policy/* scripts are loaded in test-all-policy.bro; and that
+# Makes sure that all policy/* scripts are loaded in
-# all scripts loaded there actually exist.
+# scripts/test-all-policy.bro and that all scripts loaded there actually exist.
+#
+# This test will fail if new bro scripts are added to the scripts/policy/
+# directory.  Correcting that just involves updating scripts/test-all-policy.bro
+# to @load the new bro scripts.
 
 @TEST-EXEC: test -e $DIST/scripts/test-all-policy.bro
 @TEST-EXEC: test -d $DIST/scripts

testing/btest/scripts/base/protocols/http/100-continue.bro

@@ -0,0 +1,12 @@
+# This tests that the HTTP analyzer does not generate an unmatched_HTTP_reply
+# weird as a result of seeing both a 1xx response and the real response to
+# a given request.  The http scripts should also be able log such replies
+# in a way that correlates the final response with the request.
+#
+# @TEST-EXEC: bro -r $TRACES/http-100-continue.trace %INPUT
+# @TEST-EXEC: grep -q unmatched_HTTP_reply weird.log && exit 1 || exit 0
+# @TEST-EXEC: btest-diff http.log
+
+# The base analysis scripts are loaded by default.
+#@load base/protocols/http
+