Merge remote-tracking branch 'security/topic/awelzel/topic/awelzel/208-http-mime-nested-v2'

* security/topic/awelzel/topic/awelzel/208-http-mime-nested-v2:
  MIME: Cap nested MIME analysis depth to 100

CHANGES

@@ -1,3 +1,19 @@
+6.2.0-dev.426 | 2024-01-21 19:31:02 -0700
+
+  * MIME: Cap nested MIME analysis depth to 100 (Arne Welzel, Corelight)
+
+    OSS-Fuzz managed to produce a MIME multipart message construction with
+    thousands of nested entities (or that's what Zeek makes out of it anyhow).
+    Prevent such deep analysis by capping at a nesting depth of 100,
+    preventing unnecessary resource usage. A new weird named exceeded_mime_max_depth
+    is reported when this limit is reached.
+
+    This change reduces the runtime of the OSS-Fuzz reproducer from ~45 seconds
+    to ~2.5 seconds.
+
+    The test PCAP was produced from a Python script using the email package
+    and sending the rendered version via POST to a HTTP server.
+
 6.2.0-dev.423 | 2024-01-19 21:18:44 +0100
 
   * SMTP/BDAT: Use strtoull and bail on UULONG_MAX values (Arne Welzel, Corelight)

NEWS

@@ -184,6 +184,10 @@ Changed Functionality
   two encapsulation layers. Two layers are already easily reached in AWS GLB
   environments.
 
+- Nested MIME message analysis is now capped at a maximum depth of 100 to prevent
+  unbounded MIME message nesting. This limit is configurable with ``MIME::max_depth``.
+  A new weird named ``exceeded_mime_max_depth`` is reported when reached.
+
 Removed Functionality
 ---------------------
 

VERSION

@@ -1 +1 @@
-6.2.0-dev.423
+6.2.0-dev.426

scripts/base/init-bare.zeek

@@ -2780,6 +2780,16 @@ export {
 } # end export
 
 
+module MIME;
+export {
+	## Stop analysis of nested multipart MIME entities if this depth is
+	## reached. Setting this value to 0 removes the limit.
+	const max_depth = 100 &redef;
+
+} # end export
+
+
+
 module MOUNT3;
 export {
 

src/analyzer/protocol/mime/CMakeLists.txt

@@ -10,4 +10,5 @@ zeek_add_plugin(
     MIME.cc
     Plugin.cc
     BIFS
+    consts.bif
     events.bif)

src/analyzer/protocol/mime/MIME.cc

@@ -7,6 +7,7 @@
 #include "zeek/Base64.h"
 #include "zeek/NetVar.h"
 #include "zeek/Reporter.h"
+#include "zeek/analyzer/protocol/mime/consts.bif.h"
 #include "zeek/analyzer/protocol/mime/events.bif.h"
 #include "zeek/digest.h"
 #include "zeek/file_analysis/Manager.h"
@@ -450,8 +451,10 @@ MIME_Entity::MIME_Entity(MIME_Message* output_message, MIME_Entity* parent_entit
     init();
     parent = parent_entity;
     message = output_message;
-    if ( parent )
+    if ( parent ) {
         content_encoding = parent->ContentTransferEncoding();
+        depth = parent->Depth() + 1;
+    }
 
     want_all_headers = (bool)mime_all_headers;
 }
@@ -479,6 +482,7 @@ void MIME_Entity::init() {
 
     base64_decoder = nullptr;
 
+    depth = 0;
     data_buf_length = 0;
     data_buf_data = nullptr;
     data_buf_offset = -1;
@@ -1070,6 +1074,19 @@ void MIME_Entity::SubmitAllHeaders() { message->SubmitAllHeaders(headers); }
 
 void MIME_Entity::BeginChildEntity() {
     ASSERT(current_child_entity == nullptr);
+
+    // If the maximum depth for analysis is reached, don't create a new
+    // child entity. Instead, its header/body will be delivered to the
+    // current entity.
+    if ( zeek::BifConst::MIME::max_depth > 0 && Depth() >= zeek::BifConst::MIME::max_depth ) {
+        if ( message->GetAnalyzer() ) {
+            const char* addl = zeek::util::fmt("%" PRIu64, Depth());
+            message->GetAnalyzer()->Weird("exceeded_mime_max_depth", addl);
+        }
+
+        return;
+    }
+
     current_child_entity = NewChildEntity();
     message->BeginEntity(current_child_entity);
 }

src/analyzer/protocol/mime/MIME.h

@@ -101,6 +101,7 @@ public:
     virtual void EndOfData();
 
     MIME_Entity* Parent() const { return parent; }
+    zeek_uint_t Depth() const { return depth; };
     int MIMEContentType() const { return content_type; }
     const StringValPtr& GetContentType() const { return content_type_str; }
     const StringValPtr& GetContentSubType() const { return content_subtype_str; }
@@ -172,6 +173,7 @@ protected:
 
     zeek::detail::Base64Converter* base64_decoder;
 
+    zeek_uint_t depth;
     int data_buf_length;
     char* data_buf_data;
     int data_buf_offset;

src/analyzer/protocol/mime/consts.bif

@@ -0,0 +1 @@
+const MIME::max_depth: count;

testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log

@@ -176,6 +176,7 @@ scripts/base/init-frameworks-and-bifs.zeek
     build/scripts/base/bif/plugins/Zeek_KRB.events.bif.zeek
     build/scripts/base/bif/plugins/Zeek_Login.events.bif.zeek
     build/scripts/base/bif/plugins/Zeek_Login.functions.bif.zeek
+    build/scripts/base/bif/plugins/Zeek_MIME.consts.bif.zeek
     build/scripts/base/bif/plugins/Zeek_MIME.events.bif.zeek
     build/scripts/base/bif/plugins/Zeek_Modbus.events.bif.zeek
     build/scripts/base/bif/plugins/Zeek_MQTT.types.bif.zeek

testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log

@@ -176,6 +176,7 @@ scripts/base/init-frameworks-and-bifs.zeek
     build/scripts/base/bif/plugins/Zeek_KRB.events.bif.zeek
     build/scripts/base/bif/plugins/Zeek_Login.events.bif.zeek
     build/scripts/base/bif/plugins/Zeek_Login.functions.bif.zeek
+    build/scripts/base/bif/plugins/Zeek_MIME.consts.bif.zeek
     build/scripts/base/bif/plugins/Zeek_MIME.events.bif.zeek
     build/scripts/base/bif/plugins/Zeek_Modbus.events.bif.zeek
     build/scripts/base/bif/plugins/Zeek_MQTT.types.bif.zeek

testing/btest/Baseline/plugins.hooks/output

@@ -385,6 +385,7 @@
 0.000000   MetaHookPost  LoadFile(0, ./Zeek_KRB.types.bif.zeek, <...>/Zeek_KRB.types.bif.zeek) -> -1
 0.000000   MetaHookPost  LoadFile(0, ./Zeek_Login.events.bif.zeek, <...>/Zeek_Login.events.bif.zeek) -> -1
 0.000000   MetaHookPost  LoadFile(0, ./Zeek_Login.functions.bif.zeek, <...>/Zeek_Login.functions.bif.zeek) -> -1
+0.000000   MetaHookPost  LoadFile(0, ./Zeek_MIME.consts.bif.zeek, <...>/Zeek_MIME.consts.bif.zeek) -> -1
 0.000000   MetaHookPost  LoadFile(0, ./Zeek_MIME.events.bif.zeek, <...>/Zeek_MIME.events.bif.zeek) -> -1
 0.000000   MetaHookPost  LoadFile(0, ./Zeek_MQTT.events.bif.zeek, <...>/Zeek_MQTT.events.bif.zeek) -> -1
 0.000000   MetaHookPost  LoadFile(0, ./Zeek_MQTT.types.bif.zeek, <...>/Zeek_MQTT.types.bif.zeek) -> -1
@@ -673,6 +674,7 @@
 0.000000   MetaHookPost  LoadFileExtended(0, ./Zeek_KRB.types.bif.zeek, <...>/Zeek_KRB.types.bif.zeek) -> (-1, <no content>)
 0.000000   MetaHookPost  LoadFileExtended(0, ./Zeek_Login.events.bif.zeek, <...>/Zeek_Login.events.bif.zeek) -> (-1, <no content>)
 0.000000   MetaHookPost  LoadFileExtended(0, ./Zeek_Login.functions.bif.zeek, <...>/Zeek_Login.functions.bif.zeek) -> (-1, <no content>)
+0.000000   MetaHookPost  LoadFileExtended(0, ./Zeek_MIME.consts.bif.zeek, <...>/Zeek_MIME.consts.bif.zeek) -> (-1, <no content>)
 0.000000   MetaHookPost  LoadFileExtended(0, ./Zeek_MIME.events.bif.zeek, <...>/Zeek_MIME.events.bif.zeek) -> (-1, <no content>)
 0.000000   MetaHookPost  LoadFileExtended(0, ./Zeek_MQTT.events.bif.zeek, <...>/Zeek_MQTT.events.bif.zeek) -> (-1, <no content>)
 0.000000   MetaHookPost  LoadFileExtended(0, ./Zeek_MQTT.types.bif.zeek, <...>/Zeek_MQTT.types.bif.zeek) -> (-1, <no content>)
@@ -1309,6 +1311,7 @@
 0.000000   MetaHookPre   LoadFile(0, ./Zeek_KRB.types.bif.zeek, <...>/Zeek_KRB.types.bif.zeek)
 0.000000   MetaHookPre   LoadFile(0, ./Zeek_Login.events.bif.zeek, <...>/Zeek_Login.events.bif.zeek)
 0.000000   MetaHookPre   LoadFile(0, ./Zeek_Login.functions.bif.zeek, <...>/Zeek_Login.functions.bif.zeek)
+0.000000   MetaHookPre   LoadFile(0, ./Zeek_MIME.consts.bif.zeek, <...>/Zeek_MIME.consts.bif.zeek)
 0.000000   MetaHookPre   LoadFile(0, ./Zeek_MIME.events.bif.zeek, <...>/Zeek_MIME.events.bif.zeek)
 0.000000   MetaHookPre   LoadFile(0, ./Zeek_MQTT.events.bif.zeek, <...>/Zeek_MQTT.events.bif.zeek)
 0.000000   MetaHookPre   LoadFile(0, ./Zeek_MQTT.types.bif.zeek, <...>/Zeek_MQTT.types.bif.zeek)
@@ -1597,6 +1600,7 @@
 0.000000   MetaHookPre   LoadFileExtended(0, ./Zeek_KRB.types.bif.zeek, <...>/Zeek_KRB.types.bif.zeek)
 0.000000   MetaHookPre   LoadFileExtended(0, ./Zeek_Login.events.bif.zeek, <...>/Zeek_Login.events.bif.zeek)
 0.000000   MetaHookPre   LoadFileExtended(0, ./Zeek_Login.functions.bif.zeek, <...>/Zeek_Login.functions.bif.zeek)
+0.000000   MetaHookPre   LoadFileExtended(0, ./Zeek_MIME.consts.bif.zeek, <...>/Zeek_MIME.consts.bif.zeek)
 0.000000   MetaHookPre   LoadFileExtended(0, ./Zeek_MIME.events.bif.zeek, <...>/Zeek_MIME.events.bif.zeek)
 0.000000   MetaHookPre   LoadFileExtended(0, ./Zeek_MQTT.events.bif.zeek, <...>/Zeek_MQTT.events.bif.zeek)
 0.000000   MetaHookPre   LoadFileExtended(0, ./Zeek_MQTT.types.bif.zeek, <...>/Zeek_MQTT.types.bif.zeek)
@@ -2232,6 +2236,7 @@
 0.000000 | HookLoadFile  ./Zeek_KRB.types.bif.zeek <...>/Zeek_KRB.types.bif.zeek
 0.000000 | HookLoadFile  ./Zeek_Login.events.bif.zeek <...>/Zeek_Login.events.bif.zeek
 0.000000 | HookLoadFile  ./Zeek_Login.functions.bif.zeek <...>/Zeek_Login.functions.bif.zeek
+0.000000 | HookLoadFile  ./Zeek_MIME.consts.bif.zeek <...>/Zeek_MIME.consts.bif.zeek
 0.000000 | HookLoadFile  ./Zeek_MIME.events.bif.zeek <...>/Zeek_MIME.events.bif.zeek
 0.000000 | HookLoadFile  ./Zeek_MQTT.events.bif.zeek <...>/Zeek_MQTT.events.bif.zeek
 0.000000 | HookLoadFile  ./Zeek_MQTT.types.bif.zeek <...>/Zeek_MQTT.types.bif.zeek
@@ -2520,6 +2525,7 @@
 0.000000 | HookLoadFileExtended ./Zeek_KRB.types.bif.zeek <...>/Zeek_KRB.types.bif.zeek
 0.000000 | HookLoadFileExtended ./Zeek_Login.events.bif.zeek <...>/Zeek_Login.events.bif.zeek
 0.000000 | HookLoadFileExtended ./Zeek_Login.functions.bif.zeek <...>/Zeek_Login.functions.bif.zeek
+0.000000 | HookLoadFileExtended ./Zeek_MIME.consts.bif.zeek <...>/Zeek_MIME.consts.bif.zeek
 0.000000 | HookLoadFileExtended ./Zeek_MIME.events.bif.zeek <...>/Zeek_MIME.events.bif.zeek
 0.000000 | HookLoadFileExtended ./Zeek_MQTT.events.bif.zeek <...>/Zeek_MQTT.events.bif.zeek
 0.000000 | HookLoadFileExtended ./Zeek_MQTT.types.bif.zeek <...>/Zeek_MQTT.types.bif.zeek

testing/btest/Baseline/scripts.base.protocols.http.deeply-nested-mime/http.log

@@ -0,0 +1,12 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	http
+#open XXXX-XX-XX-XX-XX-XX
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	trans_depth	method	host	uri	referrer	version	user_agent	origin	request_body_len	response_body_len	status_code	status_msg	info_code	info_msg	tags	username	password	proxied	orig_fuids	orig_filenames	orig_mime_types	resp_fuids	resp_filenames	resp_mime_types
+#types	time	string	addr	port	addr	port	count	string	string	string	string	string	string	string	count	count	count	string	count	string	set[enum]	string	string	set[string]	vector[string]	vector[string]	vector[string]	vector[string]	vector[string]	vector[string]
+XXXXXXXXXX.XXXXXX	CHhAvVGS1DHFjwGM9	127.0.0.1	48724	127.0.0.1	8080	1	POST	localhost:8080	/	-	1.0	python-requests/2.25.1	-	0	497	501	Unsupported method ('POST')	-	-	(empty)	-	-	-	Fx3nQj3PL606RGh4A5,Fdjahd1XxlaUKC7vn8,Fe52j84XySnHI9mUz3,FxA6cG3PA85BxWwwue,FXB5842rxVtgRiI2Vb,Fw9fsX12UMGC0mGdXa,Fl4MFh2QO56Q341Pta,FYZdwQ25lK4xMv9Qob,FTr1zV3F6Fwdhnw5Ik,F6LDXlarWo5lMUQOi,F7m83Od9JOgipOzrc,FVzPLi3cocOyPUvib5,FH7s3jYnZ1R4gqO1g,FrlqlF3HqNAWohppk1,FobeqY2mIbUTMYzkva	-	text/plain,text/plain,text/plain,text/plain,text/plain,text/plain,text/plain,text/plain,text/plain,text/plain,text/plain,text/plain,text/plain,text/plain,text/plain	FWhx6m3AbtxIJtthb	-	text/html
+XXXXXXXXXX.XXXXXX	CHhAvVGS1DHFjwGM9	127.0.0.1	48724	127.0.0.1	8080	2	-	-	-	-	-	-	-	39686	0	-	-	-	-	(empty)	-	-	-	-	-	-	-	-	-
+#close XXXX-XX-XX-XX-XX-XX

testing/btest/Baseline/scripts.base.protocols.http.deeply-nested-mime/weird.log

@@ -0,0 +1,11 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	weird
+#open XXXX-XX-XX-XX-XX-XX
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	name	addl	notice	peer	source
+#types	time	string	addr	port	addr	port	string	string	bool	string	string
+XXXXXXXXXX.XXXXXX	CHhAvVGS1DHFjwGM9	127.0.0.1	48724	127.0.0.1	8080	exceeded_mime_max_depth	100	F	zeek	HTTP
+#close XXXX-XX-XX-XX-XX-XX

testing/btest/scripts/base/protocols/http/deeply-nested-mime.zeek

@@ -0,0 +1,7 @@
+# @TEST-DOC: HTTP POST request with 100 nestesd message/rfc822 entities, causing an analysis depth of 200 or so, Zeek stops at 100 and produces a weird.
+#
+# @TEST-EXEC: zeek -b -r $TRACES/http/deeply-nested-mime.pcap %INPUT
+# @TEST-EXEC: btest-diff http.log
+# @TEST-EXEC: btest-diff weird.log
+
+@load base/protocols/http