Merge remote-tracking branch 'origin/topic/jazoff/memory-reduction-mime'

* origin/topic/jazoff/memory-reduction-mime:
  Avoid buffering all http headers

Adjusted during merge:
  * whitespace
  * logic for HTTP needs to consider whether either mime_all_headers
    or http_all_headers events are being used

CHANGES

@@ -1,4 +1,11 @@
 
+3.1.0-dev.24 | 2019-08-12 19:30:26 -0700
+
+  * Avoid buffering all http/mime headers (Justin Azoff)
+
+    Only buffer all http/mime headers if the http_all_headers or
+    mime_all_headers events are in use.
+
 3.1.0-dev.22 | 2019-08-12 13:31:12 -0700
 
   * GH-535: fix typo of "C_HESIOD" in DNS::classes (Jon Siwek, Corelight)

VERSION

@@ -1 +1 @@
-3.1.0-dev.22
+3.1.0-dev.24

src/analyzer/protocol/http/HTTP.cc

@@ -53,6 +53,9 @@ HTTP_Entity::HTTP_Entity(HTTP_Message *arg_message, MIME_Entity* parent_entity,
 	offset = 0;
 	instance_length = -1; // unspecified
 	send_size = true;
+	// MIME_Entity already set want_all_headers depending on mime_all_headers
+	if ( ! want_all_headers )
+		want_all_headers = (bool)http_all_headers;
 	}
 
 void HTTP_Entity::EndOfData()
@@ -762,7 +765,10 @@ void HTTP_Message::SubmitAllHeaders(mime::MIME_HeaderList& hlist)
 
 void HTTP_Message::SubmitTrailingHeaders(mime::MIME_HeaderList& /* hlist */)
 	{
-	// Do nothing for now.
+	// Do nothing for now.  Note that if this ever changes do something
+	// which relies on the header list argument, that's currently not
+	// populated unless the http_all_headers or mime_all_headers events
+	// are being used (so you may need to change that, too).
 	}
 
 void HTTP_Message::SubmitData(int len, const char* buf)

src/analyzer/protocol/mime/MIME.cc

@@ -537,6 +537,8 @@ MIME_Entity::MIME_Entity(MIME_Message* output_message, MIME_Entity* parent_entit
 	message = output_message;
 	if ( parent )
 		content_encoding = parent->ContentTransferEncoding();
+
+	want_all_headers = (bool)mime_all_headers;
 	}
 
 void MIME_Entity::init()
@@ -569,6 +571,7 @@ void MIME_Entity::init()
 
 	message = 0;
 	delay_adding_implicit_CRLF = false;
+	want_all_headers = false;
 	}
 
 MIME_Entity::~MIME_Entity()
@@ -744,7 +747,11 @@ void MIME_Entity::FinishHeader()
 		{
 		ParseMIMEHeader(h);
 		SubmitHeader(h);
-		headers.push_back(h);
+
+		if ( want_all_headers )
+			headers.push_back(h);
+		else
+			delete h;
 		}
 	else
 		delete h;

src/analyzer/protocol/mime/MIME.h

@@ -172,6 +172,7 @@ protected:
 
 	MIME_Message* message;
 	bool delay_adding_implicit_CRLF;
+	bool want_all_headers;
 };
 
 // The reason I separate MIME_Message as an abstract class is to