diff --git a/CHANGES b/CHANGES index 56c3b12d67..4649be9832 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,12 @@ +6.2.0-dev.378 | 2024-01-12 10:48:33 +0100 + + * btest/smtp: Test with smtp-bdat-pipeline-8bitmime.pcap (Arne Welzel, Corelight) + + Not sure about the origin of this pcap, so adding it in a separate + commit, but it seems a nice real-world test case. + + * GH-3264: SMTP: Add BDAT support (Arne Welzel, Corelight) + 6.2.0-dev.375 | 2024-01-12 09:27:58 +0100 * Modernize various C++/Zeek-isms in the MMDB code. (Christian Kreibich, Corelight) diff --git a/NEWS b/NEWS index a0b92a94ec..330270c755 100644 --- a/NEWS +++ b/NEWS @@ -97,6 +97,10 @@ New Functionality Given this is the first iteration of this feature, feedback around usability and use-cases that aren't covered are more than welcome. +- The SMTP analyzer was extended to recognize and properly handle the BDAT command + from RFC 3030. This improves visibility into the SMTP protocol when mail agents + and servers support and use this extension. + - The event keyword in signatures was extended to support choosing a custom event to raise instead of ``signature_match()``. This can be more efficient in certain scenarios compared to funneling every match through a single event. diff --git a/VERSION b/VERSION index 968f8f8a09..b55b0a5a7b 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -6.2.0-dev.375 +6.2.0-dev.378 diff --git a/scripts/base/init-bare.zeek b/scripts/base/init-bare.zeek index 0313d63ae5..95cc63ec78 100644 --- a/scripts/base/init-bare.zeek +++ b/scripts/base/init-bare.zeek @@ -362,6 +362,13 @@ module FTP; ## raise a FTP_max_command_length_exceeded weird and are discarded. const max_command_length = 100 &redef; +module SMTP; + +## The maximum line length within a BDAT chunk before a forceful linebreak +## is introduced and a weird is raised. Conventionally, MIME messages +## have a maximum line length of 1000 octest when properly encoded. +const bdat_max_line_length = 4096 &redef; + module GLOBAL; ## Statistics about what a TCP endpoint sent. diff --git a/scripts/base/protocols/smtp/main.zeek b/scripts/base/protocols/smtp/main.zeek index cdb52f80b3..1db7ba87a0 100644 --- a/scripts/base/protocols/smtp/main.zeek +++ b/scripts/base/protocols/smtp/main.zeek @@ -247,13 +247,21 @@ event smtp_request(c: connection, is_orig: bool, command: string, arg: string) & c$smtp_state$trans_mail_from_seen = T; c$smtp_state$trans_rcpt_to_seen = F; # Reset state on MAIL FROM } - else if ( upper_command == "DATA" ) + else if ( upper_command == "DATA" || upper_command == "BDAT" ) { if ( mail_transaction_validation ) { if ( ! c$smtp_state$trans_rcpt_to_seen ) # mail from checked in rctp to mail_transaction_invalid(c, "data missing rcpt to"); } + + if ( upper_command == "BDAT" && ends_with(arg, " LAST") ) + { + # Reset state mail transaction state when we're seeing + # the last BDAT command. + c$smtp_state$trans_mail_from_seen = F; + c$smtp_state$trans_rcpt_to_seen = F; + } } else if ( upper_command == "." ) { diff --git a/src/analyzer/protocol/smtp/BDAT.cc b/src/analyzer/protocol/smtp/BDAT.cc new file mode 100644 index 0000000000..d51ab34ece --- /dev/null +++ b/src/analyzer/protocol/smtp/BDAT.cc @@ -0,0 +1,308 @@ +#include "zeek/analyzer/protocol/smtp/BDAT.h" + +#include "zeek/3rdparty/doctest.h" +#include "zeek/Conn.h" +#include "zeek/DebugLogger.h" +#include "zeek/analyzer/protocol/mime/MIME.h" +#include "zeek/util.h" + +namespace zeek::analyzer::smtp::detail { + + +struct BDATCmd parse_bdat_arg(int length, const char* arg) { + const char* arg_end = arg + length; + struct BDATCmd r = {0}; + + if ( *arg == '\0' || ! isdigit(*arg) ) { + r.error = "BDAT not followed by a valid chunk-size"; + return r; + } + + char* chunk_size_end = nullptr; + uint64_t chunk_size = strtoul(arg, &chunk_size_end, 10); + if ( *chunk_size_end != ' ' && chunk_size_end != arg_end ) { + r.error = "BDAT chunk-size not valid"; + return r; + } + + r.chunk_size = chunk_size; + r.is_last_chunk = strncasecmp(chunk_size_end, " LAST", 5) == 0; + return r; +} + + +SMTP_BDAT_Analyzer::SMTP_BDAT_Analyzer(Connection* conn, mime::MIME_Message* mail, size_t max_line_length) + : analyzer::Analyzer("SMTP_BDAT", conn), max_line_length(max_line_length), mail(mail) {} + +void SMTP_BDAT_Analyzer::NextChunk(ChunkType chunk_type, uint64_t chunk_size) { + DBG_LOG(DBG_ANALYZER, "BDAT: NextChunk size=%" PRIi64 " last=%d", chunk_size, chunk_type == ChunkType::Last); + assert(remaining_chunk_size == 0); + cur_chunk_type = chunk_type; + remaining_chunk_size = chunk_size; +} + +void SMTP_BDAT_Analyzer::DeliverStream(int len, const u_char* data, bool is_orig) { + analyzer::Analyzer::DeliverStream(len, data, is_orig); + assert(mail != nullptr); + assert(! IsFinished()); + + // Upstream analyzer delivers more data than we're + // expecting for the current chunk. Likely a logic + // error on their side. Truncate it. + if ( len > RemainingChunkSize() ) { + Weird("smtp_bdat_chunk_overflow"); + len = static_cast(RemainingChunkSize()); + } + + // If the buffer ends with a cr and the new data doesn't start with lf + // or it's empty, deliver everything in the buffer, including the cr. + if ( ! buf.empty() && buf[buf.size() - 1] == '\r' ) { + if ( len == 0 || (len > 0 && data[0] != '\n') ) { + Weird("smtp_bdat_line_cr_only"); + mail->Deliver(buf.size(), buf.data(), false /*trailing_crlf*/); + buf.resize(0); + } + } + + // Start searching for crlf at the end of the old buffer. + std::string::size_type i = buf.size() - 1; + + buf.append(reinterpret_cast(data), len); + + std::string::size_type line_start = 0; + for ( i = 0; i < buf.size(); i++ ) { + if ( i < buf.size() - 1 && buf[i] == '\r' && buf[i + 1] == '\n' ) { + // Found a match, buf[line_start, i) is the line we want to Deliver() + buf[i] = '\0'; + buf[i + 1] = '\0'; + mail->Deliver(i - line_start, &buf[line_start], true /*trailing_crlf*/); + line_start = i + 2; + i += 1; + } + else if ( buf[i] == '\n' ) { + // There's only a lf without a preceding cr, deliver the + // line including the lf, but trailing_CRLF set as false. + Weird("smtp_bdat_line_lf_only"); + mail->Deliver(i - line_start + 1, &buf[line_start], false /*trailing_crlf*/); + line_start = i + 1; + } + else if ( i - line_start >= max_line_length ) { + Weird("smtp_bdat_line_too_long", zeek::util::fmt("%zu", buf.size())); + mail->Deliver(i - line_start, &buf[line_start], false /*trailing_crlf*/); + line_start = i; + } + } + + // Trim everything that was delivered (might be nothing). + buf.erase(0, line_start); + remaining_chunk_size -= len; + + // If this is the last chunk and all data was received, Flush any + // remaining data out now. Done() is called by the owner of mail. + if ( IsLastChunk() && RemainingChunkSize() == 0 && buf.size() > 0 ) { + mail->Deliver(buf.size(), buf.data(), false /*trailing_crlf*/); // Maybe this should be true? + buf.erase(); + } +} + +void SMTP_BDAT_Analyzer::Done() { + analyzer::Analyzer::Done(); + + // Anything still buffered? Unexpected, but deliver it. + if ( ! buf.empty() ) { + Weird("smtp_bdat_undelivered_at_done"); + mail->Deliver(buf.size(), buf.data(), false /*trailing_crlf*/); + buf.erase(); + } +} + +} // namespace zeek::analyzer::smtp::detail + + +#include "zeek/analyzer/Analyzer.h" +#include "zeek/analyzer/Manager.h" + +namespace { + +using zeek::analyzer::smtp::detail::parse_bdat_arg; + +TEST_SUITE_BEGIN("bdat command parsing"); + +TEST_CASE("last chunk") { + std::string line = "86 LAST"; + const auto& [chunk_size, is_last_chunk, error] = parse_bdat_arg(line.size(), line.c_str()); + CHECK(error == nullptr); + CHECK(chunk_size == 86); + CHECK(is_last_chunk == true); +} + +TEST_CASE("last chunk lower") { + std::string line = "86 last"; + const auto& [chunk_size, is_last_chunk, error] = parse_bdat_arg(line.size(), line.c_str()); + CHECK(error == nullptr); + CHECK(chunk_size == 86); + CHECK(is_last_chunk == true); +} + +TEST_CASE("intermediate chunk") { + std::string line = "86"; + const auto& [chunk_size, is_last_chunk, error] = parse_bdat_arg(line.size(), line.c_str()); + CHECK(error == nullptr); + CHECK(chunk_size == 86); + CHECK(is_last_chunk == false); +} + +TEST_CASE("intermediate chunk rn") { + std::string line = "86\r\n"; + const auto& [chunk_size, is_last_chunk, error] = parse_bdat_arg(line.size() - 2, line.c_str()); + CHECK(error == nullptr); + CHECK(chunk_size == 86); + CHECK(is_last_chunk == false); +} + +TEST_CASE("space pre chunk size") { + std::string line = " 86 LAST"; + const auto& [chunk_size, is_last_chunk, error] = parse_bdat_arg(line.size(), line.c_str()); + REQUIRE(error != nullptr); + CHECK(error == std::string("BDAT not followed by a valid chunk-size")); +} + +TEST_CASE("non-numeric chunk size") { + std::string line = "scramble LAST"; + const auto& [chunk_size, is_last_chunk, error] = parse_bdat_arg(line.size(), line.c_str()); + REQUIRE(error != nullptr); + CHECK(error == std::string("BDAT not followed by a valid chunk-size")); +} + +TEST_CASE("missing space post chunk size") { + std::string line = "86LAST"; + const auto& [chunk_size, is_last_chunk, error] = parse_bdat_arg(line.size(), line.c_str()); + REQUIRE(error != nullptr); + CHECK(error == std::string("BDAT chunk-size not valid")); +} + +TEST_SUITE_END(); + +TEST_SUITE_BEGIN("bdat line analyzer"); + +using zeek::analyzer::smtp::detail::ChunkType; +using zeek::analyzer::smtp::detail::SMTP_BDAT_Analyzer; + +namespace mime = zeek::analyzer::mime; + +/** + * Helper class to test Deliver() calls. + */ +class Test_MIME_Message : public mime::MIME_Message { +public: + Test_MIME_Message(zeek::analyzer::Analyzer* a) : MIME_Message(a) {} + + void Deliver(int len, const char* data, bool trailing_CRLF) override { + assert(len >= 0); + // std::printf("Deliver: '%s' trailing_CRLF=%d\n", data, trailing_CRLF); + deliver_calls.emplace_back(std::string{data, static_cast(len)}, trailing_CRLF); + } + + + // Noops, should not be called + void BeginEntity(mime::MIME_Entity* entity) override {} + void EndEntity(mime::MIME_Entity* entity) override {} + void SubmitHeader(mime::MIME_Header* h) override {} + void SubmitAllHeaders(mime::MIME_HeaderList& hlist) override {} + void SubmitData(int len, const char* buf) override {} + bool RequestBuffer(int* plen, char** pbuf) override { return false; } + void SubmitAllData() {} + void SubmitEvent(int event_type, const char* detail) override {} + + const auto& DeliverCalls() { return deliver_calls; } + +private: + std::vector> deliver_calls; +}; + +TEST_CASE("line forward testing") { + zeek::Packet p; + zeek::ConnTuple t; + auto conn = std::make_unique(zeek::detail::ConnKey(t), 0, &t, 0, &p); + auto smtp_analyzer = + std::unique_ptr(zeek::analyzer_mgr->InstantiateAnalyzer("SMTP", conn.get())); + auto mail = std::make_unique(smtp_analyzer.get()); + auto bdat = std::make_unique(conn.get(), mail.get(), 128 /* max line length*/); + + auto deliver_all = [](const auto& ds, auto& bdat) { + for ( const auto& d : ds ) + bdat->NextStream(d.size(), reinterpret_cast(d.data()), true /*is_orig, irrelevant*/); + }; + + auto total_size = [](const auto& ds) { + uint64_t r = 0; + for ( const auto& d : ds ) + r += d.size(); + + return r; + }; + + // Helpers for type deduction. + std::vector deliveries; + std::vector> expected; + + SUBCASE("test two lines split in four") { + deliveries = {"MIME-", "Version: 1.0\r\n", "Subject: Zeek", " Logo\r\n"}; + bdat->NextChunk(ChunkType::Last, total_size(deliveries)); + deliver_all(deliveries, bdat); + + expected = {{"MIME-Version: 1.0", true}, {"Subject: Zeek Logo", true}}; + CHECK(mail->DeliverCalls() == expected); + } + + SUBCASE("split on cr") { + deliveries = {"MIME-", "Version: 1.0\r", "\nSubject: Zeek", " Logo\r", "\n"}; + bdat->NextChunk(ChunkType::Last, total_size(deliveries)); + deliver_all(deliveries, bdat); + + expected = {{"MIME-Version: 1.0", true}, {"Subject: Zeek Logo", true}}; + CHECK(mail->DeliverCalls() == expected); + } + + SUBCASE("cr without lf") { + // Currently, when there's just a \r, will deliver including the cr + deliveries = {"MIME-Version: 1.0\r", "Subject: Zeek", " Logo\r\n"}; + bdat->NextChunk(ChunkType::Last, total_size(deliveries)); + deliver_all(deliveries, bdat); + + expected = {{"MIME-Version: 1.0\r", false}, {"Subject: Zeek Logo", true}}; + CHECK(mail->DeliverCalls() == expected); + } + + SUBCASE("lf without cr") { + // When a line ends only with lf, will deliver it, but including the lf + deliveries = {"MIME-Version: 1.0\n", "Subject: Zeek", " Logo\n", "From: Zeek \r\n"}; + bdat->NextChunk(ChunkType::Last, total_size(deliveries)); + deliver_all(deliveries, bdat); + + expected = {{"MIME-Version: 1.0\n", false}, + {"Subject: Zeek Logo\n", false}, + {"From: Zeek ", true}}; + CHECK(mail->DeliverCalls() == expected); + } + + SUBCASE("max_line_length 10") { + bdat->Done(); // Assertion prevention. + bdat = std::make_unique(conn.get(), mail.get(), 10 /* max line length*/); + deliveries = {"1234567890123: 45\r\n", "X-Test: Y\r\n"}; + bdat->NextChunk(ChunkType::Last, total_size(deliveries)); + deliver_all(deliveries, bdat); + + expected = {{"1234567890", false}, {"123: 45", true}, {"X-Test: Y", true}}; + CHECK(mail->DeliverCalls() == expected); + } + + // Proper cleanup to avoid assertions + bdat->Done(); + mail->Done(); + smtp_analyzer->Done(); + conn->Done(); +} + +TEST_SUITE_END(); +} // namespace diff --git a/src/analyzer/protocol/smtp/BDAT.h b/src/analyzer/protocol/smtp/BDAT.h new file mode 100644 index 0000000000..bbee3f3ef6 --- /dev/null +++ b/src/analyzer/protocol/smtp/BDAT.h @@ -0,0 +1,110 @@ +// See the file "COPYING" in the main distribution directory for copyright. +#pragma once + +#include + +#include "zeek/Conn.h" +#include "zeek/analyzer/Analyzer.h" + +namespace zeek::analyzer { + +namespace mime { +class MIME_Message; +} + +namespace smtp { + +class SMTP_Analyzer; + +namespace detail { + +/** + * Parsed from a BDAT argument. + * + * If error is non-nil, parsing failed. + */ +struct BDATCmd { + uint64_t chunk_size = 0; + bool is_last_chunk = false; + const char* error = nullptr; +}; + +/** + * Helper to parse a BDAT argument. + * + * @param length Length of arg + * @param arg String following the "BDAT " part of the line. + */ +struct BDATCmd parse_bdat_arg(int length, const char* arg); + +/** + * The type of a BDAT chunk. + * + * Helper class to avoid true/false parameters. + */ +enum class ChunkType { + None, + Intermediate, + Last, +}; + +/** + * An analyzer to consume BDAT data. + * + * Yes, this is basically a small ContentLineAnalyzer, but instead + * of being hooked up as a SupportAnalyzer and assumes TCP, too, + * this directly forwards chunks into a MIME_Message instance. It's + * also BDAT chunk aware and knows when a chunk should have completed. + */ +class SMTP_BDAT_Analyzer : public zeek::analyzer::Analyzer { +public: + /** + * Constructor. + * + * @param conn The connection over which data is transferred. + * @param mail The MIME_Message to deliver lines to. + * @param max_line_length Maximum line length before forcefully delivering. + */ + SMTP_BDAT_Analyzer(zeek::Connection* conn, mime::MIME_Message* mail, size_t max_line_length); + + /** + * Setup state for the next BDAT chunk. + * + * @param chunk_size The size in octest of the next chunk. + * @param chunk_type Whether this is the last or an intermediate chunk. + */ + void NextChunk(smtp::detail::ChunkType chunk_type, uint64_t chunk_size); + + /** + * @see Analyzer::DeliverStream() + */ + void DeliverStream(int len, const u_char* data, bool is_orig) override; + + /** + * @see Analyzer::DeliverStream() + */ + void Done() override; + + /** + * @return The remaining size of the current chunk. + */ + int64_t RemainingChunkSize() const { return remaining_chunk_size; } + + /** + * @return true if the current chunk was started with LAST. + */ + bool IsLastChunk() const { return cur_chunk_type == ChunkType::Last; } + +private: + ChunkType cur_chunk_type = ChunkType::None; + uint64_t remaining_chunk_size = 0; + std::string buf; + + size_t max_line_length = 0; + + mime::MIME_Message* mail; // owned by SMTP analyzer. +}; + +} // namespace detail +} // namespace smtp +} // namespace zeek::analyzer diff --git a/src/analyzer/protocol/smtp/CMakeLists.txt b/src/analyzer/protocol/smtp/CMakeLists.txt index b302045985..88fb237e94 100644 --- a/src/analyzer/protocol/smtp/CMakeLists.txt +++ b/src/analyzer/protocol/smtp/CMakeLists.txt @@ -3,7 +3,9 @@ zeek_add_plugin( SMTP SOURCES SMTP.cc + BDAT.cc Plugin.cc BIFS + consts.bif events.bif functions.bif) diff --git a/src/analyzer/protocol/smtp/Plugin.cc b/src/analyzer/protocol/smtp/Plugin.cc index 306a6ce03c..b27747f2e3 100644 --- a/src/analyzer/protocol/smtp/Plugin.cc +++ b/src/analyzer/protocol/smtp/Plugin.cc @@ -3,6 +3,7 @@ #include "zeek/plugin/Plugin.h" #include "zeek/analyzer/Component.h" +#include "zeek/analyzer/protocol/smtp/BDAT.h" #include "zeek/analyzer/protocol/smtp/SMTP.h" namespace zeek::plugin::detail::Zeek_SMTP { @@ -11,6 +12,7 @@ class Plugin : public zeek::plugin::Plugin { public: zeek::plugin::Configuration Configure() override { AddComponent(new zeek::analyzer::Component("SMTP", zeek::analyzer::smtp::SMTP_Analyzer::Instantiate)); + AddComponent(new zeek::analyzer::Component("SMTP_BDAT", nullptr)); zeek::plugin::Configuration config; config.name = "Zeek::SMTP"; diff --git a/src/analyzer/protocol/smtp/SMTP.cc b/src/analyzer/protocol/smtp/SMTP.cc index 1b930fdb6d..6ba7838f31 100644 --- a/src/analyzer/protocol/smtp/SMTP.cc +++ b/src/analyzer/protocol/smtp/SMTP.cc @@ -10,6 +10,8 @@ #include "zeek/NetVar.h" #include "zeek/Reporter.h" #include "zeek/analyzer/Manager.h" +#include "zeek/analyzer/protocol/smtp/BDAT.h" +#include "zeek/analyzer/protocol/smtp/consts.bif.h" #include "zeek/analyzer/protocol/smtp/events.bif.h" #undef SMTP_CMD_DEF @@ -117,6 +119,29 @@ void SMTP_Analyzer::DeliverStream(int length, const u_char* line, bool orig) { // NOTE: do not use IsOrig() here, because of TURN command. bool is_sender = orig_is_sender ? orig : ! orig; + if ( is_sender && bdat ) { + // We're processing BDAT and have switched the ContentLine analyzer + // into plain mode to send us the full chunk. Ensure we only use up + // as much as we need in case we get more. + int64_t bdat_len = std::min(bdat->RemainingChunkSize(), static_cast(length)); + if ( bdat->RemainingChunkSize() > 0 ) + bdat->NextStream(bdat_len, line, orig); + + // All BDAT chunks seen? + if ( bdat->IsLastChunk() && bdat->RemainingChunkSize() == 0 ) + UpdateState(detail::SMTP_CMD_END_OF_DATA, 0, orig); + + line += bdat_len; + length -= bdat_len; + assert(length >= 0); + + // Anything left? Usually the remainder is zero as we're doing + // plain delivery. However, a "BDAT 0 LAST" empty chunk isn't + // delivered by the ContentLineAnalyzer. + if ( length == 0 ) + return; + } + #if 0 ### if ( line[length] != '\r' || line[length+1] != '\n' ) @@ -173,8 +198,8 @@ void SMTP_Analyzer::ProcessLine(int length, const char* line, bool orig) { expect_recver = true; } - else if ( state == detail::SMTP_IN_DATA ) { - // Check "." for end of data. + else if ( state == detail::SMTP_IN_DATA && ! bdat ) { + // Check "." for end of data for non-BDAT transfers. expect_recver = false; // ?? MAY server respond to mail data? if ( line[0] == '.' ) @@ -238,6 +263,40 @@ void SMTP_Analyzer::ProcessLine(int length, const char* line, bool orig) { RequestEvent(cmd_len, cmd, data_len, line); } + // For the BDAT command, parse out the chunk-size from the line + // and switch the ContentLineAnalyzer into plain delivery mode + // assuming things look valid. + if ( cmd_code == detail::SMTP_CMD_BDAT ) { + const auto [chunk_size, is_last_chunk, error] = detail::parse_bdat_arg(end_of_line - line, line); + if ( ! error ) { + assert(chunk_size >= 0); + auto* cl = orig ? cl_orig : cl_resp; + cl->SetPlainDelivery(chunk_size); + + if ( ! bdat ) { + assert(! mail); + // This is the first BDAT chunk. + BeginData(orig); + bdat = std::make_unique(Conn(), mail, + zeek::BifConst::SMTP::bdat_max_line_length); + } + + bdat->NextChunk(is_last_chunk ? detail::ChunkType::Last : detail::ChunkType::Intermediate, + chunk_size); + } + else { + AnalyzerViolation(error, line, length); + } + } + else if ( bdat ) { + // Non-BDAT command from client but still have BDAT state, + // close it out. This can happen when a client started to + // send BDAT chunks, but starts sending other commands without + // a last BDAT chunk. + Weird("smtp_missing_bdat_last_chunk"); + EndData(); + } + if ( cmd_code != detail::SMTP_CMD_END_OF_DATA ) UpdateState(cmd_code, 0, orig); } @@ -540,6 +599,37 @@ void SMTP_Analyzer::UpdateState(int cmd_code, int reply_code, bool orig) { } break; + case detail::SMTP_CMD_BDAT: + switch ( reply_code ) { + case 0: + if ( state != detail::SMTP_RCPT_OK ) + UnexpectedCommand(cmd_code, reply_code); + + assert(bdat); + state = detail::SMTP_IN_DATA; + break; + + case 250: break; // server accepted BDAT transfer. + + case 421: state = detail::SMTP_QUIT; break; + + case 500: + case 501: + case 503: + case 451: + case 554: + // Client may continue sending chunks if pipelined. We don't + // call EndData() here as it might be interesting what the + // client does send, even if the server isn't accepting it. + break; + + default: + UnexpectedReply(cmd_code, reply_code); + // Chunks might still be in-flight. See above. + break; + } + break; + case detail::SMTP_CMD_END_OF_DATA: switch ( reply_code ) { case 0: @@ -789,6 +879,11 @@ void SMTP_Analyzer::BeginData(bool orig) { } void SMTP_Analyzer::EndData() { + if ( bdat ) { + bdat->Done(); + bdat.reset(); + } + if ( ! mail ) Weird("smtp_unmatched_end_of_data"); else { diff --git a/src/analyzer/protocol/smtp/SMTP.h b/src/analyzer/protocol/smtp/SMTP.h index 91cad1e898..d93c110666 100644 --- a/src/analyzer/protocol/smtp/SMTP.h +++ b/src/analyzer/protocol/smtp/SMTP.h @@ -14,6 +14,8 @@ namespace zeek::analyzer::smtp { namespace detail { +class SMTP_BDAT_Analyzer; + enum SMTP_Cmd { #include "SMTP_cmd.def" }; @@ -83,6 +85,8 @@ protected: String* line_after_gap; // last line before the first reply // after a gap + std::unique_ptr bdat; // if set, BDAT chunk transfer active + analyzer::mime::MIME_Mail* mail; private: diff --git a/src/analyzer/protocol/smtp/consts.bif b/src/analyzer/protocol/smtp/consts.bif new file mode 100644 index 0000000000..379887d5d5 --- /dev/null +++ b/src/analyzer/protocol/smtp/consts.bif @@ -0,0 +1 @@ +const SMTP::bdat_max_line_length: count; diff --git a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log index 0bdbda72a4..37b0ea5102 100644 --- a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log @@ -229,6 +229,7 @@ scripts/base/init-frameworks-and-bifs.zeek build/scripts/base/bif/plugins/Zeek_SMB.events.bif.zeek build/scripts/base/bif/plugins/Zeek_SMB.consts.bif.zeek build/scripts/base/bif/plugins/Zeek_SMB.types.bif.zeek + build/scripts/base/bif/plugins/Zeek_SMTP.consts.bif.zeek build/scripts/base/bif/plugins/Zeek_SMTP.events.bif.zeek build/scripts/base/bif/plugins/Zeek_SMTP.functions.bif.zeek build/scripts/base/bif/plugins/Zeek_SNMP.events.bif.zeek diff --git a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log index 29feb83dfb..18aacdd010 100644 --- a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log @@ -229,6 +229,7 @@ scripts/base/init-frameworks-and-bifs.zeek build/scripts/base/bif/plugins/Zeek_SMB.events.bif.zeek build/scripts/base/bif/plugins/Zeek_SMB.consts.bif.zeek build/scripts/base/bif/plugins/Zeek_SMB.types.bif.zeek + build/scripts/base/bif/plugins/Zeek_SMTP.consts.bif.zeek build/scripts/base/bif/plugins/Zeek_SMTP.events.bif.zeek build/scripts/base/bif/plugins/Zeek_SMTP.functions.bif.zeek build/scripts/base/bif/plugins/Zeek_SNMP.events.bif.zeek diff --git a/testing/btest/Baseline/plugins.hooks/output b/testing/btest/Baseline/plugins.hooks/output index 33c34f0daf..f40858f96c 100644 --- a/testing/btest/Baseline/plugins.hooks/output +++ b/testing/btest/Baseline/plugins.hooks/output @@ -441,6 +441,7 @@ 0.000000 MetaHookPost LoadFile(0, ./Zeek_SMB.smb2_com_write.bif.zeek, <...>/Zeek_SMB.smb2_com_write.bif.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, ./Zeek_SMB.smb2_events.bif.zeek, <...>/Zeek_SMB.smb2_events.bif.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, ./Zeek_SMB.types.bif.zeek, <...>/Zeek_SMB.types.bif.zeek) -> -1 +0.000000 MetaHookPost LoadFile(0, ./Zeek_SMTP.consts.bif.zeek, <...>/Zeek_SMTP.consts.bif.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, ./Zeek_SMTP.events.bif.zeek, <...>/Zeek_SMTP.events.bif.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, ./Zeek_SMTP.functions.bif.zeek, <...>/Zeek_SMTP.functions.bif.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, ./Zeek_SNMP.events.bif.zeek, <...>/Zeek_SNMP.events.bif.zeek) -> -1 @@ -728,6 +729,7 @@ 0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_SMB.smb2_com_write.bif.zeek, <...>/Zeek_SMB.smb2_com_write.bif.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_SMB.smb2_events.bif.zeek, <...>/Zeek_SMB.smb2_events.bif.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_SMB.types.bif.zeek, <...>/Zeek_SMB.types.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_SMTP.consts.bif.zeek, <...>/Zeek_SMTP.consts.bif.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_SMTP.events.bif.zeek, <...>/Zeek_SMTP.events.bif.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_SMTP.functions.bif.zeek, <...>/Zeek_SMTP.functions.bif.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_SNMP.events.bif.zeek, <...>/Zeek_SNMP.events.bif.zeek) -> (-1, ) @@ -1363,6 +1365,7 @@ 0.000000 MetaHookPre LoadFile(0, ./Zeek_SMB.smb2_com_write.bif.zeek, <...>/Zeek_SMB.smb2_com_write.bif.zeek) 0.000000 MetaHookPre LoadFile(0, ./Zeek_SMB.smb2_events.bif.zeek, <...>/Zeek_SMB.smb2_events.bif.zeek) 0.000000 MetaHookPre LoadFile(0, ./Zeek_SMB.types.bif.zeek, <...>/Zeek_SMB.types.bif.zeek) +0.000000 MetaHookPre LoadFile(0, ./Zeek_SMTP.consts.bif.zeek, <...>/Zeek_SMTP.consts.bif.zeek) 0.000000 MetaHookPre LoadFile(0, ./Zeek_SMTP.events.bif.zeek, <...>/Zeek_SMTP.events.bif.zeek) 0.000000 MetaHookPre LoadFile(0, ./Zeek_SMTP.functions.bif.zeek, <...>/Zeek_SMTP.functions.bif.zeek) 0.000000 MetaHookPre LoadFile(0, ./Zeek_SNMP.events.bif.zeek, <...>/Zeek_SNMP.events.bif.zeek) @@ -1650,6 +1653,7 @@ 0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_SMB.smb2_com_write.bif.zeek, <...>/Zeek_SMB.smb2_com_write.bif.zeek) 0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_SMB.smb2_events.bif.zeek, <...>/Zeek_SMB.smb2_events.bif.zeek) 0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_SMB.types.bif.zeek, <...>/Zeek_SMB.types.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_SMTP.consts.bif.zeek, <...>/Zeek_SMTP.consts.bif.zeek) 0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_SMTP.events.bif.zeek, <...>/Zeek_SMTP.events.bif.zeek) 0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_SMTP.functions.bif.zeek, <...>/Zeek_SMTP.functions.bif.zeek) 0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_SNMP.events.bif.zeek, <...>/Zeek_SNMP.events.bif.zeek) @@ -2284,6 +2288,7 @@ 0.000000 | HookLoadFile ./Zeek_SMB.smb2_com_write.bif.zeek <...>/Zeek_SMB.smb2_com_write.bif.zeek 0.000000 | HookLoadFile ./Zeek_SMB.smb2_events.bif.zeek <...>/Zeek_SMB.smb2_events.bif.zeek 0.000000 | HookLoadFile ./Zeek_SMB.types.bif.zeek <...>/Zeek_SMB.types.bif.zeek +0.000000 | HookLoadFile ./Zeek_SMTP.consts.bif.zeek <...>/Zeek_SMTP.consts.bif.zeek 0.000000 | HookLoadFile ./Zeek_SMTP.events.bif.zeek <...>/Zeek_SMTP.events.bif.zeek 0.000000 | HookLoadFile ./Zeek_SMTP.functions.bif.zeek <...>/Zeek_SMTP.functions.bif.zeek 0.000000 | HookLoadFile ./Zeek_SNMP.events.bif.zeek <...>/Zeek_SNMP.events.bif.zeek @@ -2571,6 +2576,7 @@ 0.000000 | HookLoadFileExtended ./Zeek_SMB.smb2_com_write.bif.zeek <...>/Zeek_SMB.smb2_com_write.bif.zeek 0.000000 | HookLoadFileExtended ./Zeek_SMB.smb2_events.bif.zeek <...>/Zeek_SMB.smb2_events.bif.zeek 0.000000 | HookLoadFileExtended ./Zeek_SMB.types.bif.zeek <...>/Zeek_SMB.types.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_SMTP.consts.bif.zeek <...>/Zeek_SMTP.consts.bif.zeek 0.000000 | HookLoadFileExtended ./Zeek_SMTP.events.bif.zeek <...>/Zeek_SMTP.events.bif.zeek 0.000000 | HookLoadFileExtended ./Zeek_SMTP.functions.bif.zeek <...>/Zeek_SMTP.functions.bif.zeek 0.000000 | HookLoadFileExtended ./Zeek_SNMP.events.bif.zeek <...>/Zeek_SNMP.events.bif.zeek diff --git a/testing/btest/Baseline/scripts.base.protocols.smtp.bdat-0-last/conn.log.cut b/testing/btest/Baseline/scripts.base.protocols.smtp.bdat-0-last/conn.log.cut new file mode 100644 index 0000000000..6199830a5a --- /dev/null +++ b/testing/btest/Baseline/scripts.base.protocols.smtp.bdat-0-last/conn.log.cut @@ -0,0 +1,3 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +uid id.orig_h id.orig_p id.resp_h id.resp_p service duration +CHhAvVGS1DHFjwGM9 127.0.0.1 40864 127.0.0.1 25 smtp 0.414217 diff --git a/testing/btest/Baseline/scripts.base.protocols.smtp.bdat-0-last/out b/testing/btest/Baseline/scripts.base.protocols.smtp.bdat-0-last/out new file mode 100644 index 0000000000..c1d6eca389 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.protocols.smtp.bdat-0-last/out @@ -0,0 +1,6 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +CHhAvVGS1DHFjwGM9, T, EHLO, localhost +CHhAvVGS1DHFjwGM9, T, MAIL, FROM: +CHhAvVGS1DHFjwGM9, T, RCPT, TO: +CHhAvVGS1DHFjwGM9, T, BDAT, 0 LAST +CHhAvVGS1DHFjwGM9, T, QUIT, diff --git a/testing/btest/Baseline/scripts.base.protocols.smtp.bdat-0-last/smtp.log b/testing/btest/Baseline/scripts.base.protocols.smtp.bdat-0-last/smtp.log new file mode 100644 index 0000000000..5f6c652ff4 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.protocols.smtp.bdat-0-last/smtp.log @@ -0,0 +1,11 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path smtp +#open XXXX-XX-XX-XX-XX-XX +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth helo mailfrom rcptto date from to cc reply_to msg_id in_reply_to subject x_originating_ip first_received second_received last_reply path user_agent tls fuids +#types time string addr port addr port count string string set[string] string string set[string] set[string] string string string string addr string string string vector[addr] string bool vector[string] +XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 127.0.0.1 40864 127.0.0.1 25 1 localhost zeek@localhost root@localhost - - - - - - - - - - - 221 2.0.0 Bye 127.0.0.1,127.0.0.1 - F (empty) +#close XXXX-XX-XX-XX-XX-XX diff --git a/testing/btest/Baseline/scripts.base.protocols.smtp.bdat-multipart-chunked/conn.log.cut b/testing/btest/Baseline/scripts.base.protocols.smtp.bdat-multipart-chunked/conn.log.cut new file mode 100644 index 0000000000..1c39bad5c2 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.protocols.smtp.bdat-multipart-chunked/conn.log.cut @@ -0,0 +1,3 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +uid id.orig_h id.orig_p id.resp_h id.resp_p service duration +CHhAvVGS1DHFjwGM9 127.0.0.1 38718 127.0.0.1 25 smtp 2.832130 diff --git a/testing/btest/Baseline/scripts.base.protocols.smtp.bdat-multipart-chunked/files.log.cut b/testing/btest/Baseline/scripts.base.protocols.smtp.bdat-multipart-chunked/files.log.cut new file mode 100644 index 0000000000..376324ab23 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.protocols.smtp.bdat-multipart-chunked/files.log.cut @@ -0,0 +1,4 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +uid fuid source depth mime_type filename sha1 +CHhAvVGS1DHFjwGM9 F92xQs2qVTQbHLB0k3 SMTP 2 text/plain - 5cd7c323275d114627acbae92abc666d4335c8bb +CHhAvVGS1DHFjwGM9 FT8Wo02dSqdPNsIf3c SMTP 3 image/png zeek-logo.png e9a752f145e10688ab485277a723ecbcfb5c8a63 diff --git a/testing/btest/Baseline/scripts.base.protocols.smtp.bdat-multipart-chunked/out b/testing/btest/Baseline/scripts.base.protocols.smtp.bdat-multipart-chunked/out new file mode 100644 index 0000000000..dd777f665a --- /dev/null +++ b/testing/btest/Baseline/scripts.base.protocols.smtp.bdat-multipart-chunked/out @@ -0,0 +1,32 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +CHhAvVGS1DHFjwGM9, T, EHLO, localhost +CHhAvVGS1DHFjwGM9, T, MAIL, FROM: +CHhAvVGS1DHFjwGM9, T, RCPT, TO: +CHhAvVGS1DHFjwGM9, T, BDAT, 129 +CHhAvVGS1DHFjwGM9, T, BDAT, 129 +CHhAvVGS1DHFjwGM9, T, BDAT, 129 +CHhAvVGS1DHFjwGM9, T, BDAT, 129 +CHhAvVGS1DHFjwGM9, T, BDAT, 129 +CHhAvVGS1DHFjwGM9, T, BDAT, 129 +CHhAvVGS1DHFjwGM9, T, BDAT, 129 +CHhAvVGS1DHFjwGM9, T, BDAT, 129 +CHhAvVGS1DHFjwGM9, T, BDAT, 129 +CHhAvVGS1DHFjwGM9, T, BDAT, 129 +CHhAvVGS1DHFjwGM9, T, BDAT, 129 +CHhAvVGS1DHFjwGM9, T, BDAT, 129 +CHhAvVGS1DHFjwGM9, T, BDAT, 129 +CHhAvVGS1DHFjwGM9, T, BDAT, 129 +CHhAvVGS1DHFjwGM9, T, BDAT, 129 +CHhAvVGS1DHFjwGM9, T, BDAT, 129 +CHhAvVGS1DHFjwGM9, T, BDAT, 129 +CHhAvVGS1DHFjwGM9, T, BDAT, 129 +CHhAvVGS1DHFjwGM9, T, BDAT, 129 +CHhAvVGS1DHFjwGM9, T, BDAT, 129 +CHhAvVGS1DHFjwGM9, T, BDAT, 129 +CHhAvVGS1DHFjwGM9, T, BDAT, 129 +CHhAvVGS1DHFjwGM9, T, BDAT, 129 +CHhAvVGS1DHFjwGM9, T, BDAT, 129 +CHhAvVGS1DHFjwGM9, T, BDAT, 129 +CHhAvVGS1DHFjwGM9, T, BDAT, 129 +CHhAvVGS1DHFjwGM9, T, BDAT, 106 LAST +CHhAvVGS1DHFjwGM9, T, QUIT, diff --git a/testing/btest/Baseline/scripts.base.protocols.smtp.bdat-multipart-chunked/smtp.log b/testing/btest/Baseline/scripts.base.protocols.smtp.bdat-multipart-chunked/smtp.log new file mode 100644 index 0000000000..49f5f7e0b0 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.protocols.smtp.bdat-multipart-chunked/smtp.log @@ -0,0 +1,11 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path smtp +#open XXXX-XX-XX-XX-XX-XX +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth helo mailfrom rcptto date from to cc reply_to msg_id in_reply_to subject x_originating_ip first_received second_received last_reply path user_agent tls fuids +#types time string addr port addr port count string string set[string] string string set[string] set[string] string string string string addr string string string vector[addr] string bool vector[string] +XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 127.0.0.1 38718 127.0.0.1 25 1 localhost zeek@localhost root@localhost Tue, 12 Dec 2023 17:37:16 +0100 Arne Welzel Arne Welzel - - - Zeek Logo - - - 221 2.0.0 Bye 127.0.0.1,127.0.0.1 - F F92xQs2qVTQbHLB0k3,FT8Wo02dSqdPNsIf3c +#close XXXX-XX-XX-XX-XX-XX diff --git a/testing/btest/Baseline/scripts.base.protocols.smtp.bdat-multipart/conn.log.cut b/testing/btest/Baseline/scripts.base.protocols.smtp.bdat-multipart/conn.log.cut new file mode 100644 index 0000000000..727ae9c742 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.protocols.smtp.bdat-multipart/conn.log.cut @@ -0,0 +1,3 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +uid id.orig_h id.orig_p id.resp_h id.resp_p service duration +CHhAvVGS1DHFjwGM9 127.0.0.1 38848 127.0.0.1 25 smtp 0.393711 diff --git a/testing/btest/Baseline/scripts.base.protocols.smtp.bdat-multipart/files.log.cut b/testing/btest/Baseline/scripts.base.protocols.smtp.bdat-multipart/files.log.cut new file mode 100644 index 0000000000..89ea77854c --- /dev/null +++ b/testing/btest/Baseline/scripts.base.protocols.smtp.bdat-multipart/files.log.cut @@ -0,0 +1,4 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +uid source depth mime_type filename sha1 +CHhAvVGS1DHFjwGM9 SMTP 2 text/plain - 5cd7c323275d114627acbae92abc666d4335c8bb +CHhAvVGS1DHFjwGM9 SMTP 3 image/png zeek-logo.png e9a752f145e10688ab485277a723ecbcfb5c8a63 diff --git a/testing/btest/Baseline/scripts.base.protocols.smtp.bdat-multipart/out b/testing/btest/Baseline/scripts.base.protocols.smtp.bdat-multipart/out new file mode 100644 index 0000000000..c0b0a71df9 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.protocols.smtp.bdat-multipart/out @@ -0,0 +1,6 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +CHhAvVGS1DHFjwGM9, T, EHLO, localhost +CHhAvVGS1DHFjwGM9, T, MAIL, FROM: +CHhAvVGS1DHFjwGM9, T, RCPT, TO: +CHhAvVGS1DHFjwGM9, T, BDAT, 3460 LAST +CHhAvVGS1DHFjwGM9, T, QUIT, diff --git a/testing/btest/Baseline/scripts.base.protocols.smtp.bdat-multipart/smtp.log b/testing/btest/Baseline/scripts.base.protocols.smtp.bdat-multipart/smtp.log new file mode 100644 index 0000000000..218ba3db1e --- /dev/null +++ b/testing/btest/Baseline/scripts.base.protocols.smtp.bdat-multipart/smtp.log @@ -0,0 +1,11 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path smtp +#open XXXX-XX-XX-XX-XX-XX +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth helo mailfrom rcptto date from to cc reply_to msg_id in_reply_to subject x_originating_ip first_received second_received last_reply path user_agent tls fuids +#types time string addr port addr port count string string set[string] string string set[string] set[string] string string string string addr string string string vector[addr] string bool vector[string] +XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 127.0.0.1 38848 127.0.0.1 25 1 localhost zeek@localhost root@localhost Tue, 12 Dec 2023 17:37:16 +0100 Arne Welzel Arne Welzel - - - Zeek Logo - - - 221 2.0.0 Bye 127.0.0.1,127.0.0.1 - F FemNmn1XNlVbnEe4u3,FWMIGN1fQdGfq0zsg1 +#close XXXX-XX-XX-XX-XX-XX diff --git a/testing/btest/Baseline/scripts.base.protocols.smtp.bdat-pipeline-8bitmime/conn.log.cut b/testing/btest/Baseline/scripts.base.protocols.smtp.bdat-pipeline-8bitmime/conn.log.cut new file mode 100644 index 0000000000..48ff15048a --- /dev/null +++ b/testing/btest/Baseline/scripts.base.protocols.smtp.bdat-pipeline-8bitmime/conn.log.cut @@ -0,0 +1,3 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +uid id.orig_h id.orig_p id.resp_h id.resp_p service duration +CHhAvVGS1DHFjwGM9 23.128.96.18 56074 217.146.107.83 25 smtp 1.324926 diff --git a/testing/btest/Baseline/scripts.base.protocols.smtp.bdat-pipeline-8bitmime/files.log.cut b/testing/btest/Baseline/scripts.base.protocols.smtp.bdat-pipeline-8bitmime/files.log.cut new file mode 100644 index 0000000000..0b8e834546 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.protocols.smtp.bdat-pipeline-8bitmime/files.log.cut @@ -0,0 +1,4 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +uid source depth mime_type filename +CHhAvVGS1DHFjwGM9 SMTP 1 text/plain - +CHhAvVGS1DHFjwGM9 SMTP 2 text/x-diff - diff --git a/testing/btest/Baseline/scripts.base.protocols.smtp.bdat-pipeline-8bitmime/out b/testing/btest/Baseline/scripts.base.protocols.smtp.bdat-pipeline-8bitmime/out new file mode 100644 index 0000000000..c867091b99 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.protocols.smtp.bdat-pipeline-8bitmime/out @@ -0,0 +1,9 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +CHhAvVGS1DHFjwGM9, T, EHLO, vger.kernel.org +CHhAvVGS1DHFjwGM9, T, MAIL, From: BODY=8BITMIME SIZE=4333 +CHhAvVGS1DHFjwGM9, T, RCPT, To: +CHhAvVGS1DHFjwGM9, T, BDAT, 4404 LAST +CHhAvVGS1DHFjwGM9, T, MAIL, From: BODY=8BITMIME SIZE=8546 +CHhAvVGS1DHFjwGM9, T, RCPT, To: +CHhAvVGS1DHFjwGM9, T, BDAT, 8757 LAST +CHhAvVGS1DHFjwGM9, T, QUIT, diff --git a/testing/btest/Baseline/scripts.base.protocols.smtp.bdat-pipeline-8bitmime/smtp.log b/testing/btest/Baseline/scripts.base.protocols.smtp.bdat-pipeline-8bitmime/smtp.log new file mode 100644 index 0000000000..a460b30967 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.protocols.smtp.bdat-pipeline-8bitmime/smtp.log @@ -0,0 +1,12 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path smtp +#open XXXX-XX-XX-XX-XX-XX +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth helo mailfrom rcptto date from to cc reply_to msg_id in_reply_to subject x_originating_ip first_received second_received last_reply path user_agent tls fuids +#types time string addr port addr port count string string set[string] string string set[string] set[string] string string string string addr string string string vector[addr] string bool vector[string] +XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 23.128.96.18 56074 217.146.107.83 25 1 vger.kernel.org linux-kernel-owner@vger.kernel.org trafficwatcher@foundit.scootmail.com Thu, 4 Mar 2021 11:38:56 +0200 Andy Shevchenko Andrew Morton Lukasz Luba ,open list ,Daniel Lezcano ,"rafael@kernel.org" ,Andy Shevchenko - <20210303163125.dcc0a086a939a58ed30750e8@linux-foundation.org> Re: [PATCH 1/2] units: Add the HZ_PER_KHZ macro - by mail-pf1-x42f.google.com with SMTP id y67so641134pfb.2 for ; Thu, 04 Mar 2021 01:39:13 -0800 (PST) from mail-pf1-x42f.google.com (mail-pf1-x42f.google.com [IPv6:2607:f8b0:4864:20::42f]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id A02A7C061574 for ; Thu, 4 Mar 2021 01:39:13 -0800 (PST) 250 OK id=1lHkUh-008hPw-NY 217.146.107.83,23.128.96.18,23.128.96.19 - F FnJaFv4OCDjqLe4uN1 +XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 23.128.96.18 56074 217.146.107.83 25 1 vger.kernel.org linux-kernel-owner@vger.kernel.org trafficwatcher@foundit.scootmail.com Thu, 4 Mar 2021 10:38:07 +0100 Alejandro Colomar Amir Goldstein ,Luis Henriques ,linux-man@vger.kernel.org,Steve French ,Michael Kerrisk samba-technical ,Anna Schumaker ,Linux NFS Mailing List ,linux-fsdevel ,Alexander Viro ,Trond Myklebust ,Andreas Dilger ,Luis Lozano ,Ian Lance Taylor ,Olga Kornievskaia ,Miklos Szeredi ,linux-kernel ,Alejandro Colomar ,Walter Harms ,Christoph Hellwig ,Nicolas Boichat ,ceph-devel ,"Darrick J. Wong" ,Jeff Layton ,Greg KH ,Dave Chinner ,CIFS - <20210304093806.10589-1-alx.manpages@gmail.com> <20210224142307.7284-1-lhenriques@suse.de> [RFC v4] copy_file_range.2: Update cross-filesystem support for 5.12 - from localhost.localdomain ([170.253.51.130]) by smtp.googlemail.com with ESMTPSA id l2sm6127295wml.38.2021.03.04.01.40.17 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Thu, 04 Mar 2021 01:40:18 -0800 (PST) by mail-wr1-x431.google.com with SMTP id b18so20376314wrn.6; Thu, 04 Mar 2021 01:40:19 -0800 (PST) 221 scoot-81.wizint.net closing connection 217.146.107.83,23.128.96.18,23.128.96.19,170.253.51.130 git-send-email 2.30.1.721.g45526154a5 F FF6Eao4GW4grO0552g +#close XXXX-XX-XX-XX-XX-XX diff --git a/testing/btest/Baseline/scripts.base.protocols.smtp.bdat/conn.log.cut b/testing/btest/Baseline/scripts.base.protocols.smtp.bdat/conn.log.cut new file mode 100644 index 0000000000..0646c10bf5 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.protocols.smtp.bdat/conn.log.cut @@ -0,0 +1,3 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +uid id.orig_h id.orig_p id.resp_h id.resp_p service duration +CHhAvVGS1DHFjwGM9 127.0.0.1 52210 127.0.0.1 25 smtp 0.461731 diff --git a/testing/btest/Baseline/scripts.base.protocols.smtp.bdat/out b/testing/btest/Baseline/scripts.base.protocols.smtp.bdat/out new file mode 100644 index 0000000000..71716fae66 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.protocols.smtp.bdat/out @@ -0,0 +1,6 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +CHhAvVGS1DHFjwGM9, T, EHLO, localhost +CHhAvVGS1DHFjwGM9, T, MAIL, FROM: +CHhAvVGS1DHFjwGM9, T, RCPT, TO: +CHhAvVGS1DHFjwGM9, T, BDAT, 86 LAST +CHhAvVGS1DHFjwGM9, T, QUIT, diff --git a/testing/btest/Baseline/scripts.base.protocols.smtp.bdat/smtp.log b/testing/btest/Baseline/scripts.base.protocols.smtp.bdat/smtp.log new file mode 100644 index 0000000000..a3894d7025 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.protocols.smtp.bdat/smtp.log @@ -0,0 +1,11 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path smtp +#open XXXX-XX-XX-XX-XX-XX +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth helo mailfrom rcptto date from to cc reply_to msg_id in_reply_to subject x_originating_ip first_received second_received last_reply path user_agent tls fuids +#types time string addr port addr port count string string set[string] string string set[string] set[string] string string string string addr string string string vector[addr] string bool vector[string] +XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 127.0.0.1 52210 127.0.0.1 25 1 localhost zeek@localhost root@localhost - Sam@random.com Susan@random.com - - - - This is a bodyless test message - - - 221 2.0.0 Bye 127.0.0.1,127.0.0.1 - F (empty) +#close XXXX-XX-XX-XX-XX-XX diff --git a/testing/btest/Traces/smtp/rfc3030-bdat-0-last.pcap b/testing/btest/Traces/smtp/rfc3030-bdat-0-last.pcap new file mode 100644 index 0000000000..6ea54a7fe6 Binary files /dev/null and b/testing/btest/Traces/smtp/rfc3030-bdat-0-last.pcap differ diff --git a/testing/btest/Traces/smtp/rfc3030-bdat-example1.pcap b/testing/btest/Traces/smtp/rfc3030-bdat-example1.pcap new file mode 100644 index 0000000000..45689a9928 Binary files /dev/null and b/testing/btest/Traces/smtp/rfc3030-bdat-example1.pcap differ diff --git a/testing/btest/Traces/smtp/rfc3030-bdat-multipart-chunked.pcap b/testing/btest/Traces/smtp/rfc3030-bdat-multipart-chunked.pcap new file mode 100644 index 0000000000..c1bfed0ea5 Binary files /dev/null and b/testing/btest/Traces/smtp/rfc3030-bdat-multipart-chunked.pcap differ diff --git a/testing/btest/Traces/smtp/rfc3030-bdat-multipart.pcap b/testing/btest/Traces/smtp/rfc3030-bdat-multipart.pcap new file mode 100644 index 0000000000..7974ab798b Binary files /dev/null and b/testing/btest/Traces/smtp/rfc3030-bdat-multipart.pcap differ diff --git a/testing/btest/Traces/smtp/smtp-bdat-pipeline-8bitmime.pcap b/testing/btest/Traces/smtp/smtp-bdat-pipeline-8bitmime.pcap new file mode 100644 index 0000000000..73ad48a78f Binary files /dev/null and b/testing/btest/Traces/smtp/smtp-bdat-pipeline-8bitmime.pcap differ diff --git a/testing/btest/scripts/base/protocols/smtp/bdat-0-last.test b/testing/btest/scripts/base/protocols/smtp/bdat-0-last.test new file mode 100644 index 0000000000..dc82272774 --- /dev/null +++ b/testing/btest/scripts/base/protocols/smtp/bdat-0-last.test @@ -0,0 +1,16 @@ +# @TEST-DOC: PCAP just containing a BDAT 0 LAST command, postfix accepted it as a valid mail :-) +# +# @TEST-EXEC: zeek -b -r $TRACES/smtp/rfc3030-bdat-0-last.pcap %INPUT >out +# @TEST-EXEC: zeek-cut -m uid id.orig_h id.orig_p id.resp_h id.resp_p service duration < conn.log > conn.log.cut +# @TEST-EXEC: btest-diff out +# @TEST-EXEC: btest-diff conn.log.cut +# @TEST-EXEC: btest-diff smtp.log +# @TEST-EXEC: test ! -f weird.log + +@load base/protocols/conn +@load base/protocols/smtp + +event smtp_request(c: connection, is_orig: bool, command: string, arg: string) + { + print c$uid, is_orig, command, arg; + } diff --git a/testing/btest/scripts/base/protocols/smtp/bdat-multipart-chunked.test b/testing/btest/scripts/base/protocols/smtp/bdat-multipart-chunked.test new file mode 100644 index 0000000000..f72a94d1ac --- /dev/null +++ b/testing/btest/scripts/base/protocols/smtp/bdat-multipart-chunked.test @@ -0,0 +1,19 @@ +# @TEST-DOC: Multipart message transferred via BDAT and many chunks of size 129. +# +# @TEST-EXEC: zeek -b -r $TRACES/smtp/rfc3030-bdat-multipart-chunked.pcap %INPUT >out +# @TEST-EXEC: zeek-cut -m uid id.orig_h id.orig_p id.resp_h id.resp_p service duration < conn.log > conn.log.cut +# @TEST-EXEC: zeek-cut -m uid fuid source depth mime_type filename sha1 < files.log > files.log.cut +# @TEST-EXEC: btest-diff out +# @TEST-EXEC: btest-diff conn.log.cut +# @TEST-EXEC: btest-diff smtp.log +# @TEST-EXEC: btest-diff files.log.cut +# @TEST-EXEC: test ! -f weird.log + +@load base/protocols/conn +@load base/protocols/smtp +@load frameworks/files/hash-all-files + +event smtp_request(c: connection, is_orig: bool, command: string, arg: string) + { + print c$uid, is_orig, command, arg; + } diff --git a/testing/btest/scripts/base/protocols/smtp/bdat-multipart.test b/testing/btest/scripts/base/protocols/smtp/bdat-multipart.test new file mode 100644 index 0000000000..b46c2d0207 --- /dev/null +++ b/testing/btest/scripts/base/protocols/smtp/bdat-multipart.test @@ -0,0 +1,19 @@ +# @TEST-DOC: Multipart message transferred via BDAT as a single chunk. +# +# @TEST-EXEC: zeek -b -r $TRACES/smtp/rfc3030-bdat-multipart.pcap %INPUT >out +# @TEST-EXEC: zeek-cut -m uid id.orig_h id.orig_p id.resp_h id.resp_p service duration < conn.log > conn.log.cut +# @TEST-EXEC: zeek-cut -m uid source depth mime_type filename sha1 < files.log > files.log.cut +# @TEST-EXEC: btest-diff out +# @TEST-EXEC: btest-diff conn.log.cut +# @TEST-EXEC: btest-diff smtp.log +# @TEST-EXEC: btest-diff files.log.cut +# @TEST-EXEC: test ! -f weird.log + +@load base/protocols/conn +@load base/protocols/smtp +@load frameworks/files/hash-all-files + +event smtp_request(c: connection, is_orig: bool, command: string, arg: string) + { + print c$uid, is_orig, command, arg; + } diff --git a/testing/btest/scripts/base/protocols/smtp/bdat-pipeline-8bitmime.test b/testing/btest/scripts/base/protocols/smtp/bdat-pipeline-8bitmime.test new file mode 100644 index 0000000000..2b04bbdac9 --- /dev/null +++ b/testing/btest/scripts/base/protocols/smtp/bdat-pipeline-8bitmime.test @@ -0,0 +1,18 @@ +# @TEST-DOC: PCAP with kernel mailing list BDAT transfer. +# +# @TEST-EXEC: zeek -C -b -r $TRACES/smtp/smtp-bdat-pipeline-8bitmime.pcap %INPUT >out +# @TEST-EXEC: zeek-cut -m uid id.orig_h id.orig_p id.resp_h id.resp_p service duration < conn.log > conn.log.cut +# @TEST-EXEC: zeek-cut -m uid source depth mime_type filename sha1 < files.log > files.log.cut +# @TEST-EXEC: btest-diff out +# @TEST-EXEC: btest-diff conn.log.cut +# @TEST-EXEC: btest-diff smtp.log +# @TEST-EXEC: btest-diff files.log.cut +# @TEST-EXEC: test ! -f weird.log + +@load base/protocols/conn +@load base/protocols/smtp + +event smtp_request(c: connection, is_orig: bool, command: string, arg: string) + { + print c$uid, is_orig, command, arg; + } diff --git a/testing/btest/scripts/base/protocols/smtp/bdat.test b/testing/btest/scripts/base/protocols/smtp/bdat.test new file mode 100644 index 0000000000..d5fb94cb40 --- /dev/null +++ b/testing/btest/scripts/base/protocols/smtp/bdat.test @@ -0,0 +1,15 @@ +# @TEST-DOC: Reproduce the first BDAT example from RFC3030. +# +# @TEST-EXEC: zeek -b -r $TRACES/smtp/rfc3030-bdat-example1.pcap %INPUT >out +# @TEST-EXEC: zeek-cut -m uid id.orig_h id.orig_p id.resp_h id.resp_p service duration < conn.log > conn.log.cut +# @TEST-EXEC: btest-diff out +# @TEST-EXEC: btest-diff conn.log.cut +# @TEST-EXEC: btest-diff smtp.log +# @TEST-EXEC: test ! -f weird.log + +@load base/protocols/conn +@load base/protocols/smtp + +event smtp_request(c: connection, is_orig: bool, command: string, arg: string) { + print c$uid, is_orig, command, arg; +}