diff --git a/CHANGES b/CHANGES index c62a770104..878ad01cb2 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,27 @@ +7.0.0-dev.8 | 2024-02-22 12:22:56 +0100 + + * testing: Update zeek-testing-private commit hash (Arne Welzel, Corelight) + + * ftp: Reset fuid after logging (Arne Welzel, Corelight) + + A user reported being confused about the fuid association of subsequent + FTP commands when a data transfer has completed. It seems reasonable to + unset fuid upon logging a FTP command which had a fuid. + + The current behavior results in the PORT or PASV commands after a RETR or STOR + to have the fuid of the prior file transfer. Similarly, any CWD or DEL commands + following a file transfer will unnecessarily be logged with the fuid of the + prior file transfer. + + This tickles the baselines for the private testing PCAP a lot, primarily + because there data connections in that pcap are never established properly. + E.g, the fuids FzDzid1Dxm9srVKHXf and FEfYX73q5C6GEQZXX9 have been re-used + for multiple commands. + + This may look like we're losing information, but the fuids vanishing + in the normal btests belong to a LIST command that isn't logged by + default into ftp.log. If it was, the fuid would be attached to it. + 7.0.0-dev.5 | 2024-02-22 12:01:33 +0100 * GH-3472: Allow delete statement for tables, sets and vectors (Arne Welzel, Corelight) diff --git a/NEWS b/NEWS index fcc86a5f6d..3c77ead2d2 100644 --- a/NEWS +++ b/NEWS @@ -21,6 +21,11 @@ New Functionality Changed Functionality --------------------- +- The ``ftp.log`` fuid field is now cleared after handling a command with a fuid + associated with it. Previously, fuid was sticky and any subsequent FTP command + would reproduce the same fuid, even if the command itself did not result in + a file transfer over a data connection (e.g., CWD, DEL, PASV, SIZE). + Removed Functionality --------------------- diff --git a/VERSION b/VERSION index f9f9d555f6..6e0e4a7832 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -7.0.0-dev.5 +7.0.0-dev.8 diff --git a/scripts/base/protocols/ftp/files.zeek b/scripts/base/protocols/ftp/files.zeek index 680e7f254d..e811e72364 100644 --- a/scripts/base/protocols/ftp/files.zeek +++ b/scripts/base/protocols/ftp/files.zeek @@ -7,11 +7,6 @@ module FTP; export { - redef record Info += { - ## File unique ID. - fuid: string &optional &log; - }; - ## Default file handle provider for FTP. global get_file_handle: function(c: connection, is_orig: bool): string; diff --git a/scripts/base/protocols/ftp/info.zeek b/scripts/base/protocols/ftp/info.zeek index d0122819fe..0e9cf2d21a 100644 --- a/scripts/base/protocols/ftp/info.zeek +++ b/scripts/base/protocols/ftp/info.zeek @@ -72,5 +72,8 @@ export { ## Determines if the password will be captured for this request. capture_password: bool &default=default_capture_password; + + ## File unique ID. + fuid: string &optional &log; }; } diff --git a/scripts/base/protocols/ftp/main.zeek b/scripts/base/protocols/ftp/main.zeek index 85b04038be..e33fc07891 100644 --- a/scripts/base/protocols/ftp/main.zeek +++ b/scripts/base/protocols/ftp/main.zeek @@ -207,13 +207,13 @@ function ftp_message(c: connection) if ( s?$cmdarg && s$command in logged_commands) Log::write(FTP::LOG, s); - # The MIME and file_size fields are specific to file transfer commands - # and may not be used in all commands so they need reset to "blank" - # values after logging. + # The MIME and file_size, data_channel and fuid fields are specific to + # file transfer commands and may not be used in all commands so they + # need reset to "blank" values after logging. delete s$mime_type; delete s$file_size; - # Same with data channel. delete s$data_channel; + delete s$fuid; } event sync_add_expected_data(s: Info, chan: ExpectedDataChannel) &is_used diff --git a/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-get-file-size/ftp.log b/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-get-file-size/ftp.log index e3ae3bb833..7103d05c6f 100644 --- a/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-get-file-size/ftp.log +++ b/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-get-file-size/ftp.log @@ -8,8 +8,8 @@ #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p user password command arg mime_type file_size reply_code reply_msg data_channel.passive data_channel.orig_h data_channel.resp_h data_channel.resp_p fuid #types time string addr port addr port string string string string string count count string bool addr addr port string XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 192.168.21.95 54089 164.107.123.6 21 - PASV - - - 227 Entering Passive Mode (164,107,123,6,183,187) T 192.168.21.95 164.107.123.6 47035 - -XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 192.168.21.95 54089 164.107.123.6 21 - PASV - - - 227 Entering Passive Mode (164,107,123,6,183,231) T 192.168.21.95 164.107.123.6 47079 FzwelK1cvu4OroNgn2 -XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 192.168.21.95 54089 164.107.123.6 21 - PASV - - - 227 Entering Passive Mode (164,107,123,6,183,211) T 192.168.21.95 164.107.123.6 47059 F9FJGR2omqil0TrC4l -XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 192.168.21.95 54089 164.107.123.6 21 - PASV - - - 227 Entering Passive Mode (164,107,123,6,183,197) T 192.168.21.95 164.107.123.6 47045 FbSjjXYPAIpF2a1F8 +XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 192.168.21.95 54089 164.107.123.6 21 - PASV - - - 227 Entering Passive Mode (164,107,123,6,183,231) T 192.168.21.95 164.107.123.6 47079 - +XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 192.168.21.95 54089 164.107.123.6 21 - PASV - - - 227 Entering Passive Mode (164,107,123,6,183,211) T 192.168.21.95 164.107.123.6 47059 - +XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 192.168.21.95 54089 164.107.123.6 21 - PASV - - - 227 Entering Passive Mode (164,107,123,6,183,197) T 192.168.21.95 164.107.123.6 47045 - XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 192.168.21.95 54089 164.107.123.6 21 - RETR ftp://164.107.123.6/mirror/internic/rfc/rfc1001.txt text/plain 154427 226 File send OK. - - - - FOICOh1qidx6BRr5b6 #close XXXX-XX-XX-XX-XX-XX diff --git a/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-ipv4/ftp.log b/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-ipv4/ftp.log index 61abca3308..e33958bcaf 100644 --- a/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-ipv4/ftp.log +++ b/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-ipv4/ftp.log @@ -8,9 +8,9 @@ #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p user password command arg mime_type file_size reply_code reply_msg data_channel.passive data_channel.orig_h data_channel.resp_h data_channel.resp_p fuid #types time string addr port addr port string string string string string count count string bool addr addr port string XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 141.142.220.235 50003 199.233.217.249 21 anonymous test PASV - - - 227 Entering Passive Mode (199,233,217,249,221,90) T 141.142.220.235 199.233.217.249 56666 - -XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 141.142.220.235 50003 199.233.217.249 21 anonymous test PASV - - - 227 Entering Passive Mode (199,233,217,249,221,91) T 141.142.220.235 199.233.217.249 56667 FEDbaA44a90dKFRzf3 +XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 141.142.220.235 50003 199.233.217.249 21 anonymous test PASV - - - 227 Entering Passive Mode (199,233,217,249,221,91) T 141.142.220.235 199.233.217.249 56667 - XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 141.142.220.235 50003 199.233.217.249 21 anonymous test RETR ftp://199.233.217.249/./robots.txt text/plain 77 226 Transfer complete. - - - - F6wZ8iqUoaPxGYCAj -XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 141.142.220.235 50003 199.233.217.249 21 anonymous test PORT 141,142,220,235,131,46 - - 200 PORT command successful. F 199.233.217.249 141.142.220.235 33582 F6wZ8iqUoaPxGYCAj -XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 141.142.220.235 50003 199.233.217.249 21 anonymous test PORT 141,142,220,235,147,203 - - 200 PORT command successful. F 199.233.217.249 141.142.220.235 37835 F7BoKm1QaCQXKASXLl +XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 141.142.220.235 50003 199.233.217.249 21 anonymous test PORT 141,142,220,235,131,46 - - 200 PORT command successful. F 199.233.217.249 141.142.220.235 33582 - +XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 141.142.220.235 50003 199.233.217.249 21 anonymous test PORT 141,142,220,235,147,203 - - 200 PORT command successful. F 199.233.217.249 141.142.220.235 37835 - XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 141.142.220.235 50003 199.233.217.249 21 anonymous test RETR ftp://199.233.217.249/./robots.txt text/plain 77 226 Transfer complete. - - - - FIXQK42rXgY0sM33Ue #close XXXX-XX-XX-XX-XX-XX diff --git a/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-ipv6/ftp.log b/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-ipv6/ftp.log index e87c705c90..1ce66a9e7d 100644 --- a/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-ipv6/ftp.log +++ b/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-ipv6/ftp.log @@ -8,10 +8,10 @@ #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p user password command arg mime_type file_size reply_code reply_msg data_channel.passive data_channel.orig_h data_channel.resp_h data_channel.resp_p fuid #types time string addr port addr port string string string string string count count string bool addr addr port string XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 2001:470:1f11:81f:c999:d94:aa7c:2e3e 49185 2001:470:4867:99::21 21 anonymous test EPSV - - - 229 Entering Extended Passive Mode (|||57086|) T 2001:470:1f11:81f:c999:d94:aa7c:2e3e 2001:470:4867:99::21 57086 - -XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 2001:470:1f11:81f:c999:d94:aa7c:2e3e 49185 2001:470:4867:99::21 21 anonymous test EPSV - - - 229 Entering Extended Passive Mode (|||57087|) T 2001:470:1f11:81f:c999:d94:aa7c:2e3e 2001:470:4867:99::21 57087 FRi0Gp33fe67ToMUyg -XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 2001:470:1f11:81f:c999:d94:aa7c:2e3e 49185 2001:470:4867:99::21 21 anonymous test EPSV - - - 229 Entering Extended Passive Mode (|||57088|) T 2001:470:1f11:81f:c999:d94:aa7c:2e3e 2001:470:4867:99::21 57088 FWRtZV2U54T8Pwebx1 +XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 2001:470:1f11:81f:c999:d94:aa7c:2e3e 49185 2001:470:4867:99::21 21 anonymous test EPSV - - - 229 Entering Extended Passive Mode (|||57087|) T 2001:470:1f11:81f:c999:d94:aa7c:2e3e 2001:470:4867:99::21 57087 - +XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 2001:470:1f11:81f:c999:d94:aa7c:2e3e 49185 2001:470:4867:99::21 21 anonymous test EPSV - - - 229 Entering Extended Passive Mode (|||57088|) T 2001:470:1f11:81f:c999:d94:aa7c:2e3e 2001:470:4867:99::21 57088 - XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 2001:470:1f11:81f:c999:d94:aa7c:2e3e 49185 2001:470:4867:99::21 21 anonymous test RETR ftp://[2001:470:4867:99::21]/robots.txt - 77 226 Transfer complete. - - - - FznRww2GL17njaey04 -XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 2001:470:1f11:81f:c999:d94:aa7c:2e3e 49185 2001:470:4867:99::21 21 anonymous test EPRT |2|2001:470:1f11:81f:c999:d94:aa7c:2e3e|49189| - - 200 EPRT command successful. F 2001:470:4867:99::21 2001:470:1f11:81f:c999:d94:aa7c:2e3e 49189 FznRww2GL17njaey04 +XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 2001:470:1f11:81f:c999:d94:aa7c:2e3e 49185 2001:470:4867:99::21 21 anonymous test EPRT |2|2001:470:1f11:81f:c999:d94:aa7c:2e3e|49189| - - 200 EPRT command successful. F 2001:470:4867:99::21 2001:470:1f11:81f:c999:d94:aa7c:2e3e 49189 - XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 2001:470:1f11:81f:c999:d94:aa7c:2e3e 49185 2001:470:4867:99::21 21 anonymous test RETR ftp://[2001:470:4867:99::21]/robots.txt - 77 226 Transfer complete. - - - - FqTT6724KXvAJtXx5j -XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 2001:470:1f11:81f:c999:d94:aa7c:2e3e 49185 2001:470:4867:99::21 21 anonymous test EPRT |2|2001:470:1f11:81f:c999:d94:aa7c:2e3e|49190| - - 200 EPRT command successful. F 2001:470:4867:99::21 2001:470:1f11:81f:c999:d94:aa7c:2e3e 49190 FqTT6724KXvAJtXx5j +XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 2001:470:1f11:81f:c999:d94:aa7c:2e3e 49185 2001:470:4867:99::21 21 anonymous test EPRT |2|2001:470:1f11:81f:c999:d94:aa7c:2e3e|49190| - - 200 EPRT command successful. F 2001:470:4867:99::21 2001:470:1f11:81f:c999:d94:aa7c:2e3e 49190 - #close XXXX-XX-XX-XX-XX-XX diff --git a/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-max-arg-length/ftp.log b/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-max-arg-length/ftp.log index fbeaa36ddf..655f5ab2cb 100644 --- a/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-max-arg-length/ftp.log +++ b/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-max-arg-length/ftp.log @@ -9,9 +9,9 @@ #types time string addr port addr port string string string string string count count string bool addr addr port string XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 141.142.220.235 50003 199.233.217.249 21 anonymous - USER anonymous - - 331 Guest login ok, t - - - - - XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 141.142.220.235 50003 199.233.217.249 21 anonymous test PASV - - - 227 Entering Passive T 141.142.220.235 199.233.217.249 56666 - -XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 141.142.220.235 50003 199.233.217.249 21 anonymous test PASV - - - 227 Entering Passive T 141.142.220.235 199.233.217.249 56667 FEDbaA44a90dKFRzf3 +XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 141.142.220.235 50003 199.233.217.249 21 anonymous test PASV - - - 227 Entering Passive T 141.142.220.235 199.233.217.249 56667 - XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 141.142.220.235 50003 199.233.217.249 21 anonymous test RETR ftp://199.233 text/plain 77 226 Transfer complete - - - - F6wZ8iqUoaPxGYCAj -XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 141.142.220.235 50003 199.233.217.249 21 anonymous test PORT 141,142,220,2 - - 200 PORT command succ F 199.233.217.249 141.142.220.235 33582 F6wZ8iqUoaPxGYCAj -XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 141.142.220.235 50003 199.233.217.249 21 anonymous test PORT 141,142,220,2 - - 200 PORT command succ F 199.233.217.249 141.142.220.235 37835 F7BoKm1QaCQXKASXLl +XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 141.142.220.235 50003 199.233.217.249 21 anonymous test PORT 141,142,220,2 - - 200 PORT command succ F 199.233.217.249 141.142.220.235 33582 - +XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 141.142.220.235 50003 199.233.217.249 21 anonymous test PORT 141,142,220,2 - - 200 PORT command succ F 199.233.217.249 141.142.220.235 37835 - XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 141.142.220.235 50003 199.233.217.249 21 anonymous test RETR ftp://199.233 text/plain 77 226 Transfer complete - - - - FIXQK42rXgY0sM33Ue #close XXXX-XX-XX-XX-XX-XX diff --git a/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-max-user-length/ftp.log b/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-max-user-length/ftp.log index 5d0a3165e5..737a360cc2 100644 --- a/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-max-user-length/ftp.log +++ b/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-max-user-length/ftp.log @@ -8,9 +8,9 @@ #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p user password command arg mime_type file_size reply_code reply_msg data_channel.passive data_channel.orig_h data_channel.resp_h data_channel.resp_p fuid #types time string addr port addr port string string string string string count count string bool addr addr port string XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 141.142.220.235 50003 199.233.217.249 21 anon te PASV - - - 227 Entering Passive Mode (199,233,217,249,221,90) T 141.142.220.235 199.233.217.249 56666 - -XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 141.142.220.235 50003 199.233.217.249 21 anon te PASV - - - 227 Entering Passive Mode (199,233,217,249,221,91) T 141.142.220.235 199.233.217.249 56667 FEDbaA44a90dKFRzf3 +XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 141.142.220.235 50003 199.233.217.249 21 anon te PASV - - - 227 Entering Passive Mode (199,233,217,249,221,91) T 141.142.220.235 199.233.217.249 56667 - XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 141.142.220.235 50003 199.233.217.249 21 anon te RETR ftp://199.233.217.249/./robots.txt text/plain 77 226 Transfer complete. - - - - F6wZ8iqUoaPxGYCAj -XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 141.142.220.235 50003 199.233.217.249 21 anon te PORT 141,142,220,235,131,46 - - 200 PORT command successful. F 199.233.217.249 141.142.220.235 33582 F6wZ8iqUoaPxGYCAj -XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 141.142.220.235 50003 199.233.217.249 21 anon te PORT 141,142,220,235,147,203 - - 200 PORT command successful. F 199.233.217.249 141.142.220.235 37835 F7BoKm1QaCQXKASXLl +XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 141.142.220.235 50003 199.233.217.249 21 anon te PORT 141,142,220,235,131,46 - - 200 PORT command successful. F 199.233.217.249 141.142.220.235 33582 - +XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 141.142.220.235 50003 199.233.217.249 21 anon te PORT 141,142,220,235,147,203 - - 200 PORT command successful. F 199.233.217.249 141.142.220.235 37835 - XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 141.142.220.235 50003 199.233.217.249 21 anon te RETR ftp://199.233.217.249/./robots.txt text/plain 77 226 Transfer complete. - - - - FIXQK42rXgY0sM33Ue #close XXXX-XX-XX-XX-XX-XX diff --git a/testing/external/commit-hash.zeek-testing-private b/testing/external/commit-hash.zeek-testing-private index 0ff8296fa2..54615157ba 100644 --- a/testing/external/commit-hash.zeek-testing-private +++ b/testing/external/commit-hash.zeek-testing-private @@ -1 +1 @@ -7693d38d870e7157bf78e7e14f0b2c0c3e871c62 +43987517cd3af2ad34ae50b217591d392d76e012