SMTP: Add enable_mail_data_file_analysis

Enabling this option will instantiate a new fa_file instance for every
mail data (message content) transferred over an SMTP connection. A new
event smtp_mail_data_file() can be used to intercept and identify the
creation of such files.

scripts/base/init-bare.zeek

@@ -667,6 +667,12 @@ export {
 	## is introduced and a weird is raised. Conventionally, MIME messages
 	## have a maximum line length of 1000 octets when properly encoded.
 	const bdat_max_line_length = 4096 &redef;
+
+	## Whether to send data of individual top-level RFC822 messages
+	## in SMTP transactions to the file analysis framework.
+	##
+	## :zeek:see:`smtp_mail_data_file`.
+	const enable_mail_data_file_analysis = F &redef;
 }
 
 module TCP;

src/analyzer/protocol/smtp/SMTP.cc

@@ -10,6 +10,8 @@
 #include "zeek/analyzer/protocol/smtp/BDAT.h"
 #include "zeek/analyzer/protocol/smtp/consts.bif.h"
 #include "zeek/analyzer/protocol/smtp/events.bif.h"
+#include "zeek/file_analysis/File.h"
+#include "zeek/file_analysis/Manager.h"
 
 #undef SMTP_CMD_DEF
 #define SMTP_CMD_DEF(cmd) #cmd,
@@ -126,9 +128,13 @@ void SMTP_Analyzer::DeliverStream(int length, const u_char* line, bool orig) {
         if ( bdat->RemainingChunkSize() < static_cast<uint64_t>(bdat_len) )
             bdat_len = static_cast<int>(bdat->RemainingChunkSize());
 
-        if ( bdat_len > 0 )
+        if ( bdat_len > 0 ) {
             bdat->NextStream(bdat_len, line, orig);
 
+            if ( ! mail_fuid.empty() )
+                MailDataFileAnalysis(bdat_len, line);
+        }
+
         // All BDAT chunks seen?
         if ( bdat->IsLastChunk() && bdat->RemainingChunkSize() == 0 )
             UpdateState(detail::SMTP_CMD_END_OF_DATA, 0, orig);
@@ -844,7 +850,14 @@ void SMTP_Analyzer::UnexpectedReply(int cmd_code, int reply_code) {
     Unexpected(true, "unexpected reply", len, buf);
 }
 
-void SMTP_Analyzer::ProcessData(int length, const char* line) { mail->Deliver(length, line, true /* trailing_CRLF */); }
+void SMTP_Analyzer::ProcessData(int length, const char* line) {
+    mail->Deliver(length, line, true /* trailing_CRLF */);
+
+    if ( ! mail_fuid.empty() ) {
+        MailDataFileAnalysis(length, reinterpret_cast<const u_char*>(line));
+        MailDataFileAnalysis(2, reinterpret_cast<const u_char*>("\r\n"));
+    }
+}
 
 bool SMTP_Analyzer::ProcessBdatArg(int arg_len, const char* arg, bool orig) {
     // For the BDAT command, parse out the chunk-size from the line
@@ -880,6 +893,10 @@ bool SMTP_Analyzer::ProcessBdatArg(int arg_len, const char* arg, bool orig) {
     return true;
 }
 
+std::string SMTP_Analyzer::MailDataFileAnalysis(uint64_t len, const u_char* data) {
+    return file_mgr->DataIn(data, len, GetAnalyzerTag(), Conn(), true, mail_fuid, "message/rfc822");
+}
+
 void SMTP_Analyzer::BeginData(bool orig, detail::SMTP_State new_state) {
     state = new_state;
     skip_data = false; // reset the flag at the beginning of the mail
@@ -890,6 +907,14 @@ void SMTP_Analyzer::BeginData(bool orig, detail::SMTP_State new_state) {
     }
 
     mail = new analyzer::mime::MIME_Mail(this, orig);
+
+    if ( zeek::BifConst::SMTP::enable_mail_data_file_analysis ) {
+        mail_fuid = MailDataFileAnalysis(0, reinterpret_cast<const u_char*>(""));
+
+        auto* f = file_mgr->LookupFile(mail_fuid);
+        f->FileEvent(smtp_mail_data_file, zeek::Args{f->ToVal(), Conn()->GetVal()});
+        f->FileEventDrain();
+    }
 }
 
 void SMTP_Analyzer::EndData() {
@@ -911,6 +936,11 @@ void SMTP_Analyzer::EndData() {
         delete mail;
         mail = nullptr;
     }
+
+    if ( ! mail_fuid.empty() ) {
+        file_mgr->EndOfFile(mail_fuid);
+        mail_fuid.clear();
+    }
 }
 
 } // namespace zeek::analyzer::smtp

src/analyzer/protocol/smtp/SMTP.h

@@ -61,6 +61,7 @@ protected:
     void ProcessExtension(int ext_len, const char* ext);
     void ProcessData(int length, const char* line);
     bool ProcessBdatArg(int arg_len, const char* arg, bool orig);
+    std::string MailDataFileAnalysis(uint64_t len, const u_char* data);
 
     void UpdateState(int cmd_code, int reply_code, bool orig);
 
@@ -90,6 +91,7 @@ protected:
     std::unique_ptr<detail::SMTP_BDAT_Analyzer> bdat; // if set, BDAT chunk transfer active
 
     analyzer::mime::MIME_Mail* mail;
+    std::string mail_fuid; // fuid for mail data file analysis
 
 private:
     analyzer::tcp::ContentLine_Analyzer* cl_orig;

src/analyzer/protocol/smtp/consts.bif

@@ -1 +1,2 @@
 const SMTP::bdat_max_line_length: count;
+const SMTP::enable_mail_data_file_analysis: bool;

src/analyzer/protocol/smtp/events.bif

@@ -106,3 +106,15 @@ event smtp_unexpected%(c: connection, is_orig: bool, msg: string, detail: string
 ## c: The connection.
 ##
 event smtp_starttls%(c: connection%);
+
+## Generated when a new file is created for analyzing the full mail
+## of an RFC822 message in an SMTP transaction.
+##
+## This event can be leveraged to extract the full mail data to disk.
+##
+## c: The corresponding SMTP connection.
+##
+## f: The file corresponding to the mail data.
+##
+## .. zeek:see:: SMTP::enable_mail_data_file_analysis
+event smtp_mail_data_file%(f: fa_file, c: connection%);