Merge remote-tracking branch 'origin/topic/jsiwek/gh-1100-multipart-body-length'

* origin/topic/jsiwek/gh-1100-multipart-body-length:
  GH-1100: Fix reported body-length of HTTP messages w/ sub-entities

CHANGES

@@ -1,4 +1,13 @@
 
+3.3.0-dev.55 | 2020-08-10 09:57:36 -0700
+
+  * GH-1100: Fix reported body-length of HTTP messages w/ sub-entities (Jon Siwek, Corelight)
+
+    The body-lengths of sub-entities, like multipart messages, got counted
+    twice by mistake: once upon the end of the sub-entity and then again
+    upon the end of the top-level entity that contains all sub-entities.
+    The size of just the top-level entity is the correct one to use.
+
 3.3.0-dev.53 | 2020-08-05 12:35:50 -0700
 
   * Fix deprecation warnings in fuzz targets (Jon Siwek, Corelight)

VERSION

@@ -1 +1 @@
-3.3.0-dev.53
+3.3.0-dev.55

src/analyzer/protocol/http/HTTP.cc

@@ -692,8 +692,11 @@ void HTTP_Message::EndEntity(mime::MIME_Entity* entity)
 	if ( DEBUG_http )
 		DEBUG_MSG("%.6f: end entity (%d)\n", network_time, is_orig);
 
-	body_length += ((HTTP_Entity*) entity)->BodyLength();
+	if ( entity == top_level )
-	header_length += ((HTTP_Entity*) entity)->HeaderLength();
+		{
+		body_length += ((HTTP_Entity*) entity)->BodyLength();
+		header_length += ((HTTP_Entity*) entity)->HeaderLength();
+		}
 
 	if ( http_end_entity )
 		analyzer->EnqueueConnEvent(http_end_entity,

testing/btest/Baseline/scripts.base.protocols.http.multipart-body-length/http.log

@@ -0,0 +1,10 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	http
+#open	2020-08-04-21-12-03
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	trans_depth	method	host	uri	referrer	version	user_agent	origin	request_body_len	response_body_len	status_code	status_msg	info_code	info_msg	tags	username	password	proxied	orig_fuids	orig_filenames	orig_mime_types	resp_fuids	resp_filenames	resp_mime_types
+#types	time	string	addr	port	addr	port	count	string	string	string	string	string	string	string	count	count	count	string	count	string	set[enum]	string	string	set[string]	vector[string]	vector[string]	vector[string]	vector[string]	vector[string]	vector[string]
+1596519557.155703	CHhAvVGS1DHFjwGM9	127.0.0.1	45376	127.0.0.1	8000	1	POST	localhost	/	-	-	Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36	null	767	0	-	-	-	-	(empty)	-	-	-	FeaAeS2KrXTNRvTtBh	test.txt	text/plain	-	-	-
+#close	2020-08-04-21-12-03

testing/btest/Baseline/scripts.base.protocols.http.multipart-extract/http.log

@@ -6,5 +6,5 @@
 #open	2020-04-30-00-47-14
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	trans_depth	method	host	uri	referrer	version	user_agent	origin	request_body_len	response_body_len	status_code	status_msg	info_code	info_msg	tags	username	password	proxied	orig_fuids	orig_filenames	orig_mime_types	resp_fuids	resp_filenames	resp_mime_types
 #types	time	string	addr	port	addr	port	count	string	string	string	string	string	string	string	count	count	count	string	count	string	set[enum]	string	string	set[string]	vector[string]	vector[string]	vector[string]	vector[string]	vector[string]	vector[string]
-1369159408.455878	CHhAvVGS1DHFjwGM9	141.142.228.5	57262	54.243.88.146	80	1	POST	httpbin.org	/post	-	1.1	curl/7.30.0	-	370	465	200	OK	-	-	(empty)	-	-	-	F7GxTo3GBQtouewvQ3,FxoQEm2z4L7qn9fu89,FUSlBt3LAo6IXh1TPe	-	-	Fq3bOc1S5TIYOy2Yy	-	text/json
+1369159408.455878	CHhAvVGS1DHFjwGM9	141.142.228.5	57262	54.243.88.146	80	1	POST	httpbin.org	/post	-	1.1	curl/7.30.0	-	350	465	200	OK	-	-	(empty)	-	-	-	F7GxTo3GBQtouewvQ3,FxoQEm2z4L7qn9fu89,FUSlBt3LAo6IXh1TPe	-	-	Fq3bOc1S5TIYOy2Yy	-	text/json
 #close	2020-04-30-00-47-14

testing/btest/Baseline/scripts.base.protocols.http.multipart-file-limit/http-limit-ignored.log

@@ -6,5 +6,5 @@
 #open	2020-04-30-00-47-18
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	trans_depth	method	host	uri	referrer	version	user_agent	origin	request_body_len	response_body_len	status_code	status_msg	info_code	info_msg	tags	username	password	proxied	orig_fuids	orig_filenames	orig_mime_types	resp_fuids	resp_filenames	resp_mime_types
 #types	time	string	addr	port	addr	port	count	string	string	string	string	string	string	string	count	count	count	string	count	string	set[enum]	string	string	set[string]	vector[string]	vector[string]	vector[string]	vector[string]	vector[string]	vector[string]
-1369159408.455878	CHhAvVGS1DHFjwGM9	141.142.228.5	57262	54.243.88.146	80	1	POST	httpbin.org	/post	-	1.1	curl/7.30.0	-	370	465	200	OK	-	-	(empty)	-	-	-	F7GxTo3GBQtouewvQ3,FxoQEm2z4L7qn9fu89,FUSlBt3LAo6IXh1TPe	-	-	Fq3bOc1S5TIYOy2Yy	-	text/json
+1369159408.455878	CHhAvVGS1DHFjwGM9	141.142.228.5	57262	54.243.88.146	80	1	POST	httpbin.org	/post	-	1.1	curl/7.30.0	-	350	465	200	OK	-	-	(empty)	-	-	-	F7GxTo3GBQtouewvQ3,FxoQEm2z4L7qn9fu89,FUSlBt3LAo6IXh1TPe	-	-	Fq3bOc1S5TIYOy2Yy	-	text/json
 #close	2020-04-30-00-47-18

testing/btest/Baseline/scripts.base.protocols.http.multipart-file-limit/http-limited.log

@@ -6,5 +6,5 @@
 #open	2020-04-30-00-47-16
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	trans_depth	method	host	uri	referrer	version	user_agent	origin	request_body_len	response_body_len	status_code	status_msg	info_code	info_msg	tags	username	password	proxied	orig_fuids	orig_filenames	orig_mime_types	resp_fuids	resp_filenames	resp_mime_types
 #types	time	string	addr	port	addr	port	count	string	string	string	string	string	string	string	count	count	count	string	count	string	set[enum]	string	string	set[string]	vector[string]	vector[string]	vector[string]	vector[string]	vector[string]	vector[string]
-1369159408.455878	CHhAvVGS1DHFjwGM9	141.142.228.5	57262	54.243.88.146	80	1	POST	httpbin.org	/post	-	1.1	curl/7.30.0	-	370	465	200	OK	-	-	(empty)	-	-	-	F7GxTo3GBQtouewvQ3	-	-	Fq3bOc1S5TIYOy2Yy	-	text/json
+1369159408.455878	CHhAvVGS1DHFjwGM9	141.142.228.5	57262	54.243.88.146	80	1	POST	httpbin.org	/post	-	1.1	curl/7.30.0	-	350	465	200	OK	-	-	(empty)	-	-	-	F7GxTo3GBQtouewvQ3	-	-	Fq3bOc1S5TIYOy2Yy	-	text/json
 #close	2020-04-30-00-47-17

testing/btest/Baseline/scripts.base.protocols.http.multipart-file-limit/http.log

@@ -6,5 +6,5 @@
 #open	2020-04-30-00-47-15
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	trans_depth	method	host	uri	referrer	version	user_agent	origin	request_body_len	response_body_len	status_code	status_msg	info_code	info_msg	tags	username	password	proxied	orig_fuids	orig_filenames	orig_mime_types	resp_fuids	resp_filenames	resp_mime_types
 #types	time	string	addr	port	addr	port	count	string	string	string	string	string	string	string	count	count	count	string	count	string	set[enum]	string	string	set[string]	vector[string]	vector[string]	vector[string]	vector[string]	vector[string]	vector[string]
-1369159408.455878	CHhAvVGS1DHFjwGM9	141.142.228.5	57262	54.243.88.146	80	1	POST	httpbin.org	/post	-	1.1	curl/7.30.0	-	370	465	200	OK	-	-	(empty)	-	-	-	F7GxTo3GBQtouewvQ3,FxoQEm2z4L7qn9fu89,FUSlBt3LAo6IXh1TPe	-	-	Fq3bOc1S5TIYOy2Yy	-	text/json
+1369159408.455878	CHhAvVGS1DHFjwGM9	141.142.228.5	57262	54.243.88.146	80	1	POST	httpbin.org	/post	-	1.1	curl/7.30.0	-	350	465	200	OK	-	-	(empty)	-	-	-	F7GxTo3GBQtouewvQ3,FxoQEm2z4L7qn9fu89,FUSlBt3LAo6IXh1TPe	-	-	Fq3bOc1S5TIYOy2Yy	-	text/json
 #close	2020-04-30-00-47-16

testing/btest/scripts/base/protocols/http/multipart-body-length.zeek

@@ -0,0 +1,8 @@
+# @TEST-EXEC: zeek -C -r $TRACES/http/multipart-form-data.pcap %INPUT
+# @TEST-EXEC: btest-diff http.log
+
+# This test is mainly checking the request_body_len field for correctness.
+# Historical versions of Zeek would mistakenly count the body-lengths of the
+# multipart sub-entities twice: once upon the end of the sub-entity and then
+# again upon the end of the top-level enitity that contains all sub-entities.
+# The size of just the top-level enitity is the correct one to use.

testing/external/commit-hash.zeek-testing

@@ -1 +1 @@
-9d92ec99cadd04e95365dc2c3b507b7011db255a
+04e66b70e0cca94f0da5367e143cd8d6268ed153

testing/external/commit-hash.zeek-testing-private

@@ -1 +1 @@
-d6cd639023cfe26c4e2cf14a59e78599b22ed4d0
+8868d817d6da70b980fce8917082e15c217d55a8