From 1b3b3892b587cef3bb8c7136286320158cf19105 Mon Sep 17 00:00:00 2001 From: cccs-jsjm <207595538+cccs-jsjm@users.noreply.github.com> Date: Mon, 14 Jul 2025 21:38:47 +0200 Subject: [PATCH] btest: Add tests for full email extraction --- .../extract_files.mail | 22 ++++++++++++++++++ .../files.cut | 4 ++++ .../files.cut | 6 +++++ .../files.cut | 8 +++++++ .../extract_files.mail | 22 ++++++++++++++++++ .../files.cut | 5 ++++ .../Traces/smtp/rfc3030-bdat-nonascii.pcap | Bin 0 -> 2388 bytes .../smtp/rfc822-msg-file-analysis-binary.test | 22 ++++++++++++++++++ .../rfc822-msg-file-analysis-multi-mail.test | 20 ++++++++++++++++ ...fc822-msg-file-analysis-multi-session.test | 17 ++++++++++++++ .../smtp/rfc822-msg-file-analysis.test | 21 +++++++++++++++++ 11 files changed, 147 insertions(+) create mode 100644 testing/btest/Baseline/scripts.base.protocols.smtp.rfc822-msg-file-analysis-binary/extract_files.mail create mode 100644 testing/btest/Baseline/scripts.base.protocols.smtp.rfc822-msg-file-analysis-binary/files.cut create mode 100644 testing/btest/Baseline/scripts.base.protocols.smtp.rfc822-msg-file-analysis-multi-mail/files.cut create mode 100644 testing/btest/Baseline/scripts.base.protocols.smtp.rfc822-msg-file-analysis-multi-session/files.cut create mode 100644 testing/btest/Baseline/scripts.base.protocols.smtp.rfc822-msg-file-analysis/extract_files.mail create mode 100644 testing/btest/Baseline/scripts.base.protocols.smtp.rfc822-msg-file-analysis/files.cut create mode 100644 testing/btest/Traces/smtp/rfc3030-bdat-nonascii.pcap create mode 100644 testing/btest/scripts/base/protocols/smtp/rfc822-msg-file-analysis-binary.test create mode 100644 testing/btest/scripts/base/protocols/smtp/rfc822-msg-file-analysis-multi-mail.test create mode 100644 testing/btest/scripts/base/protocols/smtp/rfc822-msg-file-analysis-multi-session.test create mode 100644 testing/btest/scripts/base/protocols/smtp/rfc822-msg-file-analysis.test diff --git a/testing/btest/Baseline/scripts.base.protocols.smtp.rfc822-msg-file-analysis-binary/extract_files.mail b/testing/btest/Baseline/scripts.base.protocols.smtp.rfc822-msg-file-analysis-binary/extract_files.mail new file mode 100644 index 0000000000..c60f3fc015 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.protocols.smtp.rfc822-msg-file-analysis-binary/extract_files.mail @@ -0,0 +1,22 @@ +Content-Type: multipart/mixed; boundary="===============6117237608014356945==" +MIME-Version: 1.0 +From: sender@example.com +To: recipient@example.com +Subject: subject + +--===============6117237608014356945== +Content-Type: text/plain; charset="us-ascii" +MIME-Version: 1.0 +Content-Transfer-Encoding: 7bit + +this is the body +--===============6117237608014356945== +MIME-Version: 1.0 +Content-Transfer-Encoding: 8bit +Content-Type: message/rfc822 +Content-Disposition: attachment; filename="test.msg" + +From: +Subject: écureuil + +--===============6117237608014356945==-- diff --git a/testing/btest/Baseline/scripts.base.protocols.smtp.rfc822-msg-file-analysis-binary/files.cut b/testing/btest/Baseline/scripts.base.protocols.smtp.rfc822-msg-file-analysis-binary/files.cut new file mode 100644 index 0000000000..2ca83c0b6f --- /dev/null +++ b/testing/btest/Baseline/scripts.base.protocols.smtp.rfc822-msg-file-analysis-binary/files.cut @@ -0,0 +1,4 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +fuid uid mime_type seen_bytes parent_fuid sha1 +FCg7geKxxmGkZIjg8 CHhAvVGS1DHFjwGM9 text/plain 16 FtRhTAoOcRvbFX8Gd - +FtRhTAoOcRvbFX8Gd CHhAvVGS1DHFjwGM9 message/rfc822 604 - 6ae7bc45d107228ca00b44d9d93ba7fdcf7255bf diff --git a/testing/btest/Baseline/scripts.base.protocols.smtp.rfc822-msg-file-analysis-multi-mail/files.cut b/testing/btest/Baseline/scripts.base.protocols.smtp.rfc822-msg-file-analysis-multi-mail/files.cut new file mode 100644 index 0000000000..c555a12503 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.protocols.smtp.rfc822-msg-file-analysis-multi-mail/files.cut @@ -0,0 +1,6 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +fuid uid mime_type seen_bytes parent_fuid sha1 +FmBdd33I01LDuH7oUb CHhAvVGS1DHFjwGM9 text/plain 754 FnJaFv4OCDjqLe4uN1 - +FnJaFv4OCDjqLe4uN1 CHhAvVGS1DHFjwGM9 message/rfc822 4404 - ee93d96ae2b5883e8dfe02aca71f799f2b47f1f6 +FKgT5hPFZTBbn1lp5 CHhAvVGS1DHFjwGM9 text/x-diff 4069 FkYtTJ1xa6wty5Ldhj - +FkYtTJ1xa6wty5Ldhj CHhAvVGS1DHFjwGM9 message/rfc822 8757 - 43a21580619bc8b129eb50cad9505a8469685149 diff --git a/testing/btest/Baseline/scripts.base.protocols.smtp.rfc822-msg-file-analysis-multi-session/files.cut b/testing/btest/Baseline/scripts.base.protocols.smtp.rfc822-msg-file-analysis-multi-session/files.cut new file mode 100644 index 0000000000..c298f43712 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.protocols.smtp.rfc822-msg-file-analysis-multi-session/files.cut @@ -0,0 +1,8 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +fuid uid mime_type seen_bytes parent_fuid sha1 +F3YoYf2TBYfB6459td CHhAvVGS1DHFjwGM9 text/plain 77 F8cH26hNF5zzM8RRh - +Fsvwh44sJq8mKeeSNf CHhAvVGS1DHFjwGM9 text/html 1868 F8cH26hNF5zzM8RRh - +FYOdN3NHoqhncZZKe CHhAvVGS1DHFjwGM9 text/plain 10809 F8cH26hNF5zzM8RRh - +F8cH26hNF5zzM8RRh CHhAvVGS1DHFjwGM9 message/rfc822 14545 - cfca0a07196a5d62e713d83b73e6290aea90435d +FlvZMD2G66eDZGt16c CUM0KZ3MLUfNB0cl11 text/plain 204 Fc5KpS3kUYqDLwWSMf - +Fc5KpS3kUYqDLwWSMf CUM0KZ3MLUfNB0cl11 message/rfc822 804 - e5409b0c77bae4d71cfddc4023a266d692d48663 diff --git a/testing/btest/Baseline/scripts.base.protocols.smtp.rfc822-msg-file-analysis/extract_files.mail b/testing/btest/Baseline/scripts.base.protocols.smtp.rfc822-msg-file-analysis/extract_files.mail new file mode 100644 index 0000000000..67702f7b6c --- /dev/null +++ b/testing/btest/Baseline/scripts.base.protocols.smtp.rfc822-msg-file-analysis/extract_files.mail @@ -0,0 +1,22 @@ +Content-Type: multipart/mixed; boundary="===============6117237608014356945==" +MIME-Version: 1.0 +From: sender@example.com +To: recipient@example.com +Subject: subject + +--===============6117237608014356945== +Content-Type: text/plain; charset="us-ascii" +MIME-Version: 1.0 +Content-Transfer-Encoding: 7bit + +this is the body +--===============6117237608014356945== +MIME-Version: 1.0 +Content-Transfer-Encoding: 8bit +Content-Type: message/rfc822 +Content-Disposition: attachment; filename="test.msg" + +From: + + +--===============6117237608014356945==-- diff --git a/testing/btest/Baseline/scripts.base.protocols.smtp.rfc822-msg-file-analysis/files.cut b/testing/btest/Baseline/scripts.base.protocols.smtp.rfc822-msg-file-analysis/files.cut new file mode 100644 index 0000000000..5b64a8b806 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.protocols.smtp.rfc822-msg-file-analysis/files.cut @@ -0,0 +1,5 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +fuid uid mime_type seen_bytes parent_fuid sha1 +FmtdjL2OsOZBPttkqk CHhAvVGS1DHFjwGM9 text/plain 16 Fmxhgo3R4TBQaBJPad - +F27fOV1yocRMx48oM4 CHhAvVGS1DHFjwGM9 - 2 Fmxhgo3R4TBQaBJPad - +Fmxhgo3R4TBQaBJPad CHhAvVGS1DHFjwGM9 message/rfc822 586 - f9bc3e476a513159f5c7c5869d47364514605b45 diff --git a/testing/btest/Traces/smtp/rfc3030-bdat-nonascii.pcap b/testing/btest/Traces/smtp/rfc3030-bdat-nonascii.pcap new file mode 100644 index 0000000000000000000000000000000000000000..4b6c831d9629d9eb9170b93845d1a6cf240df1d8 GIT binary patch literal 2388 zcmbuBO-~a+7{`ak7}KOG-iQY$<>G;bZJ{j`N{Xe(TcOg{c=51o2fC)aTX&~YjENUL zc=6!Hv+<(w0$w~A)T9{o;K`S>U%-Q(0H2x8cBjx7ivRGWTlSgX^MB?IzBgY#3!qcT z)4>5kecl)NjEj(dh0dTXLYG-u`t-f8_hA4F?FAa}vho3p)JxvCUMOe)I4$+%F97B2 zNhrb#YosrK9w_I0$_;pAA&YmkC^~xObtJsBDJvF2(Wr{6wysqxwo7rjZkpS;kSpe~ zOPo7~PC`MCI|mOeWbr!VqRyUIM{+yJ-0ojQC_BHr>KF2iDsA^ZbQW@0$l}nrfWAKjjY1}eKAzxEoS84JEG?{D^_gps!a^2j&I;zP z#}w`NLuQ&8efkTb+|0rki{#gqQXb7f|!alWGbve?N7_~ z>XPrb054g{;#I?d#ZxVV~hh;GyjV$p7fCt(B&S-gZIN__T8 z2*dM`xyVX16Wp-xHVtoB$l|D|Ks7y-aJvYZIfh!%5xO!vQ^av~%;!A`AjCoz;{_hH zcz16f;uSsKZy;xEi$c#zal1y6*sNEnQPUhcVj2xnp2X{R-70I&b}F3eR6G(%L`M^G zbzF^%jmF{=W3f~!917+Za#`g%aa_Z;k~lJ~hJtgBZ6@K6@y$RQnpvxoVcj-E!J?hS z{2uxCW`}pp<%+NpTjq`f#gw`q%*VQ)3 zh_j)ON27kfvxZx`CNx^#SvBWJ#z*&>Aa_itM8u9 files.cut +# @TEST-EXEC: btest-diff files.cut +# @TEST-EXEC: btest-diff --binary extract_files/mail +# @TEST-EXEC: grep -q "[^\x00-\x7f]" extract_files/mail + +@load base/files/hash +@load base/files/extract +@load base/protocols/smtp + +redef SMTP::enable_rfc822_msg_file_analysis = T; + +event file_over_new_connection(f: fa_file, c: connection, is_orig: bool) + { + if ( f$id == c$smtp$rfc822_msg_fuid ) + { + Files::add_analyzer(f, Files::ANALYZER_SHA1); + Files::add_analyzer(f, Files::ANALYZER_EXTRACT, [$extract_filename="mail"]); + } + } diff --git a/testing/btest/scripts/base/protocols/smtp/rfc822-msg-file-analysis-multi-mail.test b/testing/btest/scripts/base/protocols/smtp/rfc822-msg-file-analysis-multi-mail.test new file mode 100644 index 0000000000..e775624873 --- /dev/null +++ b/testing/btest/scripts/base/protocols/smtp/rfc822-msg-file-analysis-multi-mail.test @@ -0,0 +1,20 @@ +# @TEST-DOC: Test case for extracting multiple mails from the same SMTP session. +# +# @TEST-EXEC: zeek -C -b -r $TRACES/smtp/smtp-bdat-pipeline-8bitmime.pcap %INPUT +# @TEST-EXEC: zeek-cut -m fuid uid mime_type seen_bytes parent_fuid sha1 < files.log > files.cut +# @TEST-EXEC: btest-diff files.cut + +@load base/files/hash +@load base/files/extract +@load base/protocols/smtp + +redef SMTP::enable_rfc822_msg_file_analysis = T; + +event file_over_new_connection(f: fa_file, c: connection, is_orig: bool) + { + if ( f$id == c$smtp$rfc822_msg_fuid ) + { + Files::add_analyzer(f, Files::ANALYZER_EXTRACT); + Files::add_analyzer(f, Files::ANALYZER_SHA1); + } + } diff --git a/testing/btest/scripts/base/protocols/smtp/rfc822-msg-file-analysis-multi-session.test b/testing/btest/scripts/base/protocols/smtp/rfc822-msg-file-analysis-multi-session.test new file mode 100644 index 0000000000..442df93a48 --- /dev/null +++ b/testing/btest/scripts/base/protocols/smtp/rfc822-msg-file-analysis-multi-session.test @@ -0,0 +1,17 @@ +# @TEST-DOC: Test case for analyzing RFC822 messages from multiple SMTP sessions. +# +# @TEST-EXEC: zeek -C -b -r $TRACES/smtp.trace %INPUT +# @TEST-EXEC: zeek-cut -m fuid uid mime_type seen_bytes parent_fuid sha1 < files.log > files.cut +# @TEST-EXEC: btest-diff files.cut + +@load base/files/hash +@load base/files/extract +@load base/protocols/smtp + +redef SMTP::enable_rfc822_msg_file_analysis = T; + +event file_over_new_connection(f: fa_file, c: connection, is_orig: bool) + { + if ( f$id == c$smtp$rfc822_msg_fuid ) + Files::add_analyzer(f, Files::ANALYZER_SHA1); + } diff --git a/testing/btest/scripts/base/protocols/smtp/rfc822-msg-file-analysis.test b/testing/btest/scripts/base/protocols/smtp/rfc822-msg-file-analysis.test new file mode 100644 index 0000000000..f7c786d139 --- /dev/null +++ b/testing/btest/scripts/base/protocols/smtp/rfc822-msg-file-analysis.test @@ -0,0 +1,21 @@ +# @TEST-DOC: Test case for extracting a mail that contains attachments. +# +# @TEST-EXEC: zeek -C -b -r $TRACES/smtp-attachment-msg.pcap %INPUT +# @TEST-EXEC: zeek-cut -m fuid uid mime_type seen_bytes parent_fuid sha1 < files.log > files.cut +# @TEST-EXEC: btest-diff files.cut +# @TEST-EXEC: btest-diff --binary extract_files/mail + +@load base/files/hash +@load base/files/extract +@load base/protocols/smtp + +redef SMTP::enable_rfc822_msg_file_analysis = T; + +event file_over_new_connection(f: fa_file, c: connection, is_orig: bool) + { + if ( f$id == c$smtp$rfc822_msg_fuid ) + { + Files::add_analyzer(f, Files::ANALYZER_SHA1); + Files::add_analyzer(f, Files::ANALYZER_EXTRACT, [$extract_filename="mail"]); + } + }