diff --git a/CHANGES b/CHANGES index 06adf41cb1..684df9042a 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,44 @@ +7.0.0-dev.26 | 2024-02-26 21:15:59 +0100 + + * signatures/iso-9660: Add \x01 suffix to CD001 (Arne Welzel, Corelight) + + As discussed with Tim, that should make it a bit more robust against + false positives. + + * test-all-policy: Do not load iso-9660.zeek (Arne Welzel, Corelight) + + Changing the default_file_bof_buffer_size has subtle impact on + MIME type detection and changed the zeek-testing baseline. Do + not load this new script via test-all-policy to avoid this. + + The new test was mainly an aid to understand what is actually going on. + In short, if default_file_bof_buffer_size is larger than the file MIME + detection only runs when the buffer is full, or when the file is removed. + When a file transfer happens over multiple HTTP connections, only + some or one of the http.log entries will have a proper response MIME type. + + PCAP extracted from 2009-M57-day11-18.trace.gz. + + * signatures: Move ISO 9660 signature to policy (Arne Welzel, Corelight) + + The previous "fix" caused significant performance degradation without + the signature ever having a chance to trigger. Moving it to policy + seems the best compromise, the alternative being outright removing it. + + * GH-3490: global_ids: Align script_id$type_name field with type_name() (Arne Welzel, Corelight) + + Populate script_id$type_name with what the type_name() bif + would produce for the same identifier. + + Closes #3490 + + * utils: Introduce packages.zeek with can_load() helper (Arne Welzel, Corelight) + + * GH-3594: zeek.bif: Add find_in_zeekpath() helper (Arne Welzel, Corelight) + + Relates to #3594. This helper can be used to determine the path that + will be used by @load, if at all. + 7.0.0-dev.16 | 2024-02-26 11:09:57 -0700 * Update Mozilla CA and Google CT list and related tests. (Johanna Amann, Corelight) diff --git a/NEWS b/NEWS index ca5a9b51fd..6d559b54e8 100644 --- a/NEWS +++ b/NEWS @@ -39,6 +39,13 @@ Changed Functionality returned by ``type_name()`` for each identifier. E.g, ``Site::local_nets`` has a type_name of ``set[subnet]`` rather than ``table``. +- The ISO 9660 file signature has been moved into the policy directory. The + signature has previously been non-functional due to implicit anchoring. Further, + this signature requires users to significantly increase their + ``default_file_bof_buffer_size``. Users can now enable this signature by loading + ``frameworks/signatures/iso-9660`` which also increases the BOF buffer sufficiently. + Note, doing so may increase memory and CPU usage significantly. + Removed Functionality --------------------- diff --git a/VERSION b/VERSION index 35a0ff0ad2..17ded3587f 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -7.0.0-dev.16 +7.0.0-dev.26 diff --git a/scripts/base/frameworks/files/magic/general.sig b/scripts/base/frameworks/files/magic/general.sig index c9a3df1f60..a676f328bc 100644 --- a/scripts/base/frameworks/files/magic/general.sig +++ b/scripts/base/frameworks/files/magic/general.sig @@ -296,18 +296,3 @@ signature file-windows-minidump { file-mime "application/x-windows-minidump", 50 file-magic /^MDMP/ } - -# ISO 9660 disk image: First 16 sectors (2k) are arbitrary data. -# The following sector is a volume descriptor with magic string "CD001" -# at offset 1: 16 * 2048 + 1 = 32769 -signature file-iso9660 { - file-mime "application/x-iso9660-image", 99 - file-magic /^.{32769}CD001/ -} - -# ISO 9660 disk image, magic string match in next volume descriptor. -# 17 * 2048 + 1 = 34817 -signature file-iso9660-2 { - file-mime "application/x-iso9660-image", 99 - file-magic /^.{34817}CD001/ -} diff --git a/scripts/policy/frameworks/signatures/iso-9660.sig b/scripts/policy/frameworks/signatures/iso-9660.sig new file mode 100644 index 0000000000..b4878a87eb --- /dev/null +++ b/scripts/policy/frameworks/signatures/iso-9660.sig @@ -0,0 +1,10 @@ +# ISO 9660 disk image: First 16 sectors (2k) are arbitrary data. +# The following sector is a volume descriptor with magic string "CD001" +# at offset 1: 16 * 2048 + 1 = 32769. +# +# However, we do not use exact offset matching /^.{32769}CD001/ as this +# results in major performance degradation. +signature file-iso9660 { + file-mime "application/x-iso9660-image", 99 + file-magic /.*CD001\x01/ +} diff --git a/scripts/policy/frameworks/signatures/iso-9660.zeek b/scripts/policy/frameworks/signatures/iso-9660.zeek new file mode 100644 index 0000000000..248b4c81e7 --- /dev/null +++ b/scripts/policy/frameworks/signatures/iso-9660.zeek @@ -0,0 +1,8 @@ +##! Load signature for ISO 9660 disk image and increase +##! default_file_bof_buffer_size to make it functional. +@load-sigs ./iso-9660 + +# CD001 string is in the 17th sector. +@if ( default_file_bof_buffer_size < (16 + 1) * 2048 ) +redef default_file_bof_buffer_size = (16 + 1) * 2048; +@endif diff --git a/scripts/test-all-policy.zeek b/scripts/test-all-policy.zeek index efa867c6bf..cf75ed5aa0 100644 --- a/scripts/test-all-policy.zeek +++ b/scripts/test-all-policy.zeek @@ -72,6 +72,7 @@ @load frameworks/notice/extend-email/hostnames.zeek @load files/x509/disable-certificate-events-known-certs.zeek @load frameworks/packet-filter/shunt.zeek +# @load frameworks/signatures/iso-9660.zeek @load frameworks/software/version-changes.zeek @load frameworks/software/vulnerable.zeek # @load frameworks/spicy/record-spicy-batch.zeek diff --git a/scripts/zeekygen/__load__.zeek b/scripts/zeekygen/__load__.zeek index 80a530bc3b..5cda9d6263 100644 --- a/scripts/zeekygen/__load__.zeek +++ b/scripts/zeekygen/__load__.zeek @@ -10,6 +10,7 @@ @load frameworks/management/node/__load__.zeek @load frameworks/management/node/main.zeek @load frameworks/files/extract-all-files.zeek +@load frameworks/signatures/iso-9660.zeek @load policy/misc/dump-events.zeek @load policy/protocols/conn/speculative-service.zeek diff --git a/testing/btest/Baseline/scripts.base.files.mime.vnd.ms-cab-compressed-multi-conn/files.log.cut b/testing/btest/Baseline/scripts.base.files.mime.vnd.ms-cab-compressed-multi-conn/files.log.cut new file mode 100644 index 0000000000..3299024504 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.files.mime.vnd.ms-cab-compressed-multi-conn/files.log.cut @@ -0,0 +1,7 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +fuid source mime_type filename +FJYJFE2AmtoQavCYEh HTTP application/vnd.ms-cab-compressed windows6.0-kb955430-x86-express_9c8a57958486102e94e591384dc69dd7e8d01169.cab +FJYJFE2AmtoQavCYEh HTTP application/vnd.ms-cab-compressed windows6.0-kb955430-x86-express_9c8a57958486102e94e591384dc69dd7e8d01169.cab +FJYJFE2AmtoQavCYEh HTTP application/vnd.ms-cab-compressed windows6.0-kb955430-x86-express_9c8a57958486102e94e591384dc69dd7e8d01169.cab +FJYJFE2AmtoQavCYEh HTTP application/vnd.ms-cab-compressed windows6.0-kb955430-x86-express_9c8a57958486102e94e591384dc69dd7e8d01169.cab +FJYJFE2AmtoQavCYEh HTTP application/vnd.ms-cab-compressed windows6.0-kb955430-x86-express_9c8a57958486102e94e591384dc69dd7e8d01169.cab diff --git a/testing/btest/Baseline/scripts.base.files.mime.vnd.ms-cab-compressed-multi-conn/http.log.cut b/testing/btest/Baseline/scripts.base.files.mime.vnd.ms-cab-compressed-multi-conn/http.log.cut new file mode 100644 index 0000000000..952f611b2b --- /dev/null +++ b/testing/btest/Baseline/scripts.base.files.mime.vnd.ms-cab-compressed-multi-conn/http.log.cut @@ -0,0 +1,7 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +uid method host status_code resp_fuids response_body_len resp_mime_types +CHhAvVGS1DHFjwGM9 GET au.download.windowsupdate.com 206 FJYJFE2AmtoQavCYEh 5473 - +ClEkJM2Vm5giqnMf4h GET au.download.windowsupdate.com 206 FJYJFE2AmtoQavCYEh 6622 - +C4J4Th3PJpwUYZZ6gc GET au.download.windowsupdate.com 206 FJYJFE2AmtoQavCYEh 7551 - +CtPZjS20MLrsMUOJi2 GET au.download.windowsupdate.com 206 FJYJFE2AmtoQavCYEh 11791 - +CUM0KZ3MLUfNB0cl11 GET au.download.windowsupdate.com 206 FJYJFE2AmtoQavCYEh 8181 application/vnd.ms-cab-compressed diff --git a/testing/btest/Traces/http/vnd.ms-cab-compressed-multi-conn.pcap b/testing/btest/Traces/http/vnd.ms-cab-compressed-multi-conn.pcap new file mode 100644 index 0000000000..db4aed21b4 Binary files /dev/null and b/testing/btest/Traces/http/vnd.ms-cab-compressed-multi-conn.pcap differ diff --git a/testing/btest/scripts/base/files/mime/iso-9660.zeek b/testing/btest/scripts/base/files/mime/iso-9660.zeek index 2d047f0364..6010ca0089 100644 --- a/testing/btest/scripts/base/files/mime/iso-9660.zeek +++ b/testing/btest/scripts/base/files/mime/iso-9660.zeek @@ -7,6 +7,8 @@ @load base/protocols/http @load base/frameworks/files +@load frameworks/signatures/iso-9660 + redef default_file_bof_buffer_size = 40000; event file_over_new_connection(f: fa_file, c: connection, is_orig: bool) diff --git a/testing/btest/scripts/base/files/mime/vnd.ms-cab-compressed-multi-conn.zeek b/testing/btest/scripts/base/files/mime/vnd.ms-cab-compressed-multi-conn.zeek new file mode 100644 index 0000000000..80b3895992 --- /dev/null +++ b/testing/btest/scripts/base/files/mime/vnd.ms-cab-compressed-multi-conn.zeek @@ -0,0 +1,22 @@ +# @TEST-DOC: Increasing default_file_bof_buffer_size has subtle impact on mime_type detection and association for partial file transfers over HTTP. Test mainly to aid understanding. +# +# @TEST-EXEC: zeek -b -r $TRACES/http/vnd.ms-cab-compressed-multi-conn.pcap %INPUT +# @TEST-EXEC: zeek-cut -m fuid source mime_type filename < files.log > files.log.cut +# @TEST-EXEC: btest-diff files.log.cut +# @TEST-EXEC: zeek-cut -m uid method host status_code resp_fuids response_body_len resp_mime_types < http.log > http.log.cut +# @TEST-EXEC: btest-diff http.log.cut + +@load base/protocols/http +@load base/frameworks/files + +# Increases default_file_bof_buffer_size, resulting in only one of the GET +# of http.log having the application/vnd.ms-cab-compressed associated. +@load policy/frameworks/signatures/iso-9660 + +redef LogAscii::use_json = F; + +event file_over_new_connection(f: fa_file, c: connection, is_orig: bool) + { + if ( f$source == "HTTP" ) + f$info$filename = split_string(c$http$uri, /\//)[-1]; + }