Merge remote-tracking branch 'origin/topic/seth/deflate-missing-headers-fix'

I've changed the dynamic allocation of the unzipbuf back to stack
allocation, hope I'm not not missing the reason for doing that ...

* origin/topic/seth/deflate-missing-headers-fix:
  Fixes an issue with missing zlib headers on deflated HTTP content.

BIT-1399 #merged

CHANGES

@@ -1,4 +1,9 @@
 
+2.4-12 | 2015-06-28 12:21:11 -0700
+
+  * Trying to decompress deflated HTTP content even when zlib headers
+    are missing. (Seth Hall)
+
 2.4-10 | 2015-06-25 07:11:17 -0700
 
   * Correct a name used in a header identifier (Justin Azoff)

VERSION

@@ -1 +1 @@
-2.4-10
+2.4-12

src/analyzer/protocol/zip/ZIP.cc

@@ -22,10 +22,9 @@ ZIP_Analyzer::ZIP_Analyzer(Connection* conn, bool orig, Method arg_method)
 	zip->next_in = 0;
 	zip->avail_in = 0;
 
-	// "15" here means maximum compression.  "32" is a gross overload
+	// "32" is a gross overload hack that means "check it
-	// hack that means "check it for whether it's a gzip file".  Sheesh.
+	// for whether it's a gzip file".  Sheesh.
-	zip_status = inflateInit2(zip, 15 + 32);
+	if ( inflateInit2(zip, MAX_WBITS + 32) != Z_OK )
-	if ( zip_status != Z_OK )
 		{
 		Weird("inflate_init_failed");
 		delete zip;
@@ -56,24 +55,25 @@ void ZIP_Analyzer::DeliverStream(int len, const u_char* data, bool orig)
 	static unsigned int unzip_size = 4096;
 	Bytef unzipbuf[unzip_size];
 
+	int allow_restart = 1;
+
 	zip->next_in = (Bytef*) data;
 	zip->avail_in = len;
 
-	do
+	Bytef *orig_next_in = zip->next_in;
+	size_t orig_avail_in = zip->avail_in;
+
+	while ( true )
 		{
 		zip->next_out = unzipbuf;
 		zip->avail_out = unzip_size;
 
 		zip_status = inflate(zip, Z_SYNC_FLUSH);
 
-		if ( zip_status != Z_STREAM_END &&
+		if ( zip_status == Z_STREAM_END ||
-		     zip_status != Z_OK &&
+		     zip_status == Z_OK )
-		     zip_status != Z_BUF_ERROR )
 			{
-			Weird("inflate_failed");
+			allow_restart = 0;
-			inflateEnd(zip);
-			break;
-			}
 
 			int have = unzip_size - zip->avail_out;
 			if ( have )
@@ -82,12 +82,36 @@ void ZIP_Analyzer::DeliverStream(int len, const u_char* data, bool orig)
 			if ( zip_status == Z_STREAM_END )
 				{
 				inflateEnd(zip);
-			delete zip;
+				return;
-			zip = 0;
-			break;
 				}
 
-		zip_status = Z_OK;
+			if ( zip->avail_in == 0 )
+				return;
+
+			}
+
+		else if ( allow_restart && zip_status == Z_DATA_ERROR )
+			{
+			// Some servers seem to not generate zlib headers,
+			// so this is an attempt to fix and continue anyway.
+			inflateEnd(zip);
+
+			if ( inflateInit2(zip, -MAX_WBITS) != Z_OK )
+				{
+				Weird("inflate_init_failed");
+				return;
+				}
+
+			zip->next_in = orig_next_in;
+			zip->avail_in = orig_avail_in;
+			allow_restart = 0;
+			continue;
+			}
+
+		else
+			{
+			Weird("inflate_failed");
+			return;
+			}
 		}
-	while ( zip->avail_out == 0 );
 	}

testing/btest/Baseline/scripts.base.protocols.http.missing-zlib-header/http.log

@@ -0,0 +1,11 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	http
+#open	2015-05-12-16-26-53
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	trans_depth	method	host	uri	referrer	user_agent	request_body_len	response_body_len	status_code	status_msg	info_code	info_msg	filename	tags	username	password	proxied	orig_fuids	orig_mime_types	resp_fuids	resp_mime_types
+#types	time	string	addr	port	addr	port	count	string	string	string	string	string	count	count	count	string	count	string	string	set[enum]	string	string	set[string]	vector[string]	vector[string]	vector[string]	vector[string]
+1232039472.314927	CXWv6p3arKYeMETxOg	237.244.174.255	1905	79.218.110.244	80	1	GET	ads1.msn.com	/library/dap.js	http://zone.msn.com/en/root/default.htm	Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; .NET CLR 1.0.3705; .NET CLR 1.1.4322; Media Center PC 4.0; .NET CLR 2.0.50727)	0	13249	200	OK	-	-	-	(empty)	-	-	-	-	-	FBcNS3RwceOxW15xg	text/plain
+1232039472.446194	CXWv6p3arKYeMETxOg	237.244.174.255	1905	79.218.110.244	80	2	GET	ads1.msn.com	/library/dap.js	http://zone.msn.com/en/root/default.htm	Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; .NET CLR 1.0.3705; .NET CLR 1.1.4322; Media Center PC 4.0; .NET CLR 2.0.50727)	0	13249	200	OK	-	-	-	(empty)	-	-	-	-	-	FDWU85N0DpedJPh93	text/plain
+#close	2015-05-12-16-26-53

testing/btest/scripts/base/protocols/http/missing-zlib-header.bro

@@ -0,0 +1,6 @@
+# This tests an issue where some web servers don't 
+# include an appropriate ZLIB header on deflated 
+# content.
+#
+# @TEST-EXEC: bro -r $TRACES/http/missing-zlib-header.pcap %INPUT
+# @TEST-EXEC: btest-diff http.log