From e18edfa45236692984ebd1b85a0348206fcd3353 Mon Sep 17 00:00:00 2001 From: Johanna Amann Date: Wed, 6 Sep 2023 11:00:05 +0100 Subject: [PATCH] Add extract_limit_includes_missing option for file extraction Setting this option to false does not count missing bytes in files towards the extraction limits, and allows to extract data up to the desired limit, even when partial files are written. When missing bytes are encountered, files are now written as sparse files. Using this option requires the underlying storage and utilities to support sparse files. --- scripts/base/files/extract/main.zeek | 21 ++++++++-- src/file_analysis/analyzer/extract/Extract.cc | 35 ++++++++++++++-- src/file_analysis/analyzer/extract/Extract.h | 4 +- testing/btest/Baseline/plugins.hooks/output | 3 ++ .../1.out | 2 + .../2.out | 1 + .../3.out | 2 + .../extract_files.1 | 0 .../files-1.log | 11 +++++ .../files-2.log | 11 +++++ .../files-3.log | 11 +++++ .../spicy.file-analyzer-nested/output-max | 4 +- testing/btest/Traces/http/http-large-gap.pcap | Bin 0 -> 1339 bytes .../base/files/extract/limit-large-hole.zeek | 38 ++++++++++++++++++ 14 files changed, 132 insertions(+), 11 deletions(-) create mode 100644 testing/btest/Baseline/scripts.base.files.extract.limit-large-hole/1.out create mode 100644 testing/btest/Baseline/scripts.base.files.extract.limit-large-hole/2.out create mode 100644 testing/btest/Baseline/scripts.base.files.extract.limit-large-hole/3.out create mode 100644 testing/btest/Baseline/scripts.base.files.extract.limit-large-hole/extract_files.1 create mode 100644 testing/btest/Baseline/scripts.base.files.extract.limit-large-hole/files-1.log create mode 100644 testing/btest/Baseline/scripts.base.files.extract.limit-large-hole/files-2.log create mode 100644 testing/btest/Baseline/scripts.base.files.extract.limit-large-hole/files-3.log create mode 100644 testing/btest/Traces/http/http-large-gap.pcap create mode 100644 testing/btest/scripts/base/files/extract/limit-large-hole.zeek diff --git a/scripts/base/files/extract/main.zeek b/scripts/base/files/extract/main.zeek index 211b65536e..24cdb151c9 100644 --- a/scripts/base/files/extract/main.zeek +++ b/scripts/base/files/extract/main.zeek @@ -9,12 +9,17 @@ export { ## The default max size for extracted files (they won't exceed this ## number of bytes). A value of zero means unlimited. - ## - ## Note: Holes in files do not count towards these limits. Files with - ## holes are created as sparse files on disk. This means that their - ## apparent size can exceed this limit. option default_limit = 0; + ## This setting configures if the file extract limit is inclusive + ## of missing bytes. By default, missing bytes do count towards the + ## limit. + ## Setting this option to false changes this behavior so that missing + ## bytes no longer count towards these limits. Files with + ## missing bytes are created as sparse files on disk. Their apparent size + ## can exceed this file size limit. + option default_limit_includes_missing = T; + redef record Files::Info += { ## Local filename of extracted file. extracted: string &optional &log; @@ -41,6 +46,14 @@ export { ## :zeek:see:`FileExtract::set_limit` is called to increase the ## limit. A value of zero means "no limit". extract_limit: count &default=default_limit; + ## By default, missing bytes in files count towards the extract file size. + ## Missing bytes can, e.g., occur due to missed traffic, or offsets + ## used when downloading files. + ## Setting this option to false changes this behavior so that holes + ## in files do no longer count towards these limits. Files with + ## holes are created as sparse files on disk. Their apparent size + ## can exceed this file size limit. + extract_limit_includes_missing: bool &default=default_limit_includes_missing; }; ## Sets the maximum allowed extracted file size. diff --git a/src/file_analysis/analyzer/extract/Extract.cc b/src/file_analysis/analyzer/extract/Extract.cc index 284683f51e..9aad2863ac 100644 --- a/src/file_analysis/analyzer/extract/Extract.cc +++ b/src/file_analysis/analyzer/extract/Extract.cc @@ -13,9 +13,10 @@ namespace zeek::file_analysis::detail { Extract::Extract(RecordValPtr args, file_analysis::File* file, const std::string& arg_filename, - uint64_t arg_limit) + uint64_t arg_limit, bool arg_limit_includes_missing) : file_analysis::Analyzer(file_mgr->GetComponentTag("EXTRACT"), std::move(args), file), - filename(arg_filename), limit(arg_limit), written(0) + filename(arg_filename), limit(arg_limit), written(0), + limit_includes_missing(arg_limit_includes_missing) { char buf[128]; file_stream = fopen(filename.data(), "wb"); @@ -60,11 +61,14 @@ file_analysis::Analyzer* Extract::Instantiate(RecordValPtr args, file_analysis:: { const auto& fname = get_extract_field_val(args, "extract_filename"); const auto& limit = get_extract_field_val(args, "extract_limit"); + const auto& extract_limit_includes_missing = get_extract_field_val( + args, "extract_limit_includes_missing"); - if ( ! fname || ! limit ) + if ( ! fname || ! limit || ! extract_limit_includes_missing ) return nullptr; - return new Extract(std::move(args), file, fname->AsString()->CheckString(), limit->AsCount()); + return new Extract(std::move(args), file, fname->AsString()->CheckString(), limit->AsCount(), + extract_limit_includes_missing->AsBool()); } /** @@ -153,6 +157,29 @@ bool Extract::Undelivered(uint64_t offset, uint64_t len) if ( ! file_stream ) return false; + if ( limit_includes_missing ) + { + uint64_t towrite = 0; + bool limit_exceeded = check_limit_exceeded(limit, written, len, &towrite); + // if the limit is exceeded, we have to raise the event. This gives scripts the opportunity + // to raise the limit. + if ( limit_exceeded && file_extraction_limit ) + { + file_analysis::File* f = GetFile(); + f->FileEvent(file_extraction_limit, + {f->ToVal(), GetArgs(), val_mgr->Count(limit), val_mgr->Count(len)}); + // we have to check again if the limit is still exceedee + limit_exceeded = check_limit_exceeded(limit, written, len, &towrite); + } + + // if the limit is exceeded, abort and don't do anything - no reason to seek. + if ( limit_exceeded ) + return false; + + // if we don't skip holes, count this hole against the write limit + written += len; + } + if ( fseek(file_stream, len + offset, SEEK_SET) != 0 ) { char buf[128]; diff --git a/src/file_analysis/analyzer/extract/Extract.h b/src/file_analysis/analyzer/extract/Extract.h index 96a18c49f1..60249741ce 100644 --- a/src/file_analysis/analyzer/extract/Extract.h +++ b/src/file_analysis/analyzer/extract/Extract.h @@ -65,15 +65,17 @@ protected: * @param arg_filename a file system path which specifies the local file * to which the contents of the file will be extracted/written. * @param arg_limit the maximum allowed file size. + * @param arg_limit_includes_missing missing bytes count towards limit if true. */ Extract(RecordValPtr args, file_analysis::File* file, const std::string& arg_filename, - uint64_t arg_limit); + uint64_t arg_limit, bool arg_limit_includes_missing); private: std::string filename; FILE* file_stream; uint64_t limit; // the file extraction limit uint64_t written; // how many bytes we have written so far + bool limit_includes_missing; // do count missing bytes against limit if true }; } // namespace zeek::file_analysis::detail diff --git a/testing/btest/Baseline/plugins.hooks/output b/testing/btest/Baseline/plugins.hooks/output index d7242fc762..986b4f35c7 100644 --- a/testing/btest/Baseline/plugins.hooks/output +++ b/testing/btest/Baseline/plugins.hooks/output @@ -551,6 +551,7 @@ 0.000000 MetaHookPost CallFunction(Option::set_change_handler, , (FTP::max_reply_msg_length, Config::config_option_changed{ if ( == Config::location) return (Config::new_value)Config::log = Config::Info($ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value))if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, to_any_coerceConfig::log)return (Config::new_value)}, -100)) -> 0.000000 MetaHookPost CallFunction(Option::set_change_handler, , (FTP::max_user_length, Config::config_option_changed{ if ( == Config::location) return (Config::new_value)Config::log = Config::Info($ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value))if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, to_any_coerceConfig::log)return (Config::new_value)}, -100)) -> 0.000000 MetaHookPost CallFunction(Option::set_change_handler, , (FileExtract::default_limit, Config::config_option_changed{ if ( == Config::location) return (Config::new_value)Config::log = Config::Info($ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value))if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, to_any_coerceConfig::log)return (Config::new_value)}, -100)) -> +0.000000 MetaHookPost CallFunction(Option::set_change_handler, , (FileExtract::default_limit_includes_missing, Config::config_option_changed{ if ( == Config::location) return (Config::new_value)Config::log = Config::Info($ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value))if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, to_any_coerceConfig::log)return (Config::new_value)}, -100)) -> 0.000000 MetaHookPost CallFunction(Option::set_change_handler, , (Files::enable_reassembler, Config::config_option_changed{ if ( == Config::location) return (Config::new_value)Config::log = Config::Info($ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value))if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, to_any_coerceConfig::log)return (Config::new_value)}, -100)) -> 0.000000 MetaHookPost CallFunction(Option::set_change_handler, , (GridFTP::max_time, Config::config_option_changed{ if ( == Config::location) return (Config::new_value)Config::log = Config::Info($ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value))if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, to_any_coerceConfig::log)return (Config::new_value)}, -100)) -> 0.000000 MetaHookPost CallFunction(Option::set_change_handler, , (GridFTP::size_threshold, Config::config_option_changed{ if ( == Config::location) return (Config::new_value)Config::log = Config::Info($ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value))if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, to_any_coerceConfig::log)return (Config::new_value)}, -100)) -> @@ -2178,6 +2179,7 @@ 0.000000 MetaHookPre CallFunction(Option::set_change_handler, , (FTP::max_reply_msg_length, Config::config_option_changed{ if ( == Config::location) return (Config::new_value)Config::log = Config::Info($ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value))if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, to_any_coerceConfig::log)return (Config::new_value)}, -100)) 0.000000 MetaHookPre CallFunction(Option::set_change_handler, , (FTP::max_user_length, Config::config_option_changed{ if ( == Config::location) return (Config::new_value)Config::log = Config::Info($ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value))if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, to_any_coerceConfig::log)return (Config::new_value)}, -100)) 0.000000 MetaHookPre CallFunction(Option::set_change_handler, , (FileExtract::default_limit, Config::config_option_changed{ if ( == Config::location) return (Config::new_value)Config::log = Config::Info($ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value))if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, to_any_coerceConfig::log)return (Config::new_value)}, -100)) +0.000000 MetaHookPre CallFunction(Option::set_change_handler, , (FileExtract::default_limit_includes_missing, Config::config_option_changed{ if ( == Config::location) return (Config::new_value)Config::log = Config::Info($ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value))if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, to_any_coerceConfig::log)return (Config::new_value)}, -100)) 0.000000 MetaHookPre CallFunction(Option::set_change_handler, , (Files::enable_reassembler, Config::config_option_changed{ if ( == Config::location) return (Config::new_value)Config::log = Config::Info($ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value))if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, to_any_coerceConfig::log)return (Config::new_value)}, -100)) 0.000000 MetaHookPre CallFunction(Option::set_change_handler, , (GridFTP::max_time, Config::config_option_changed{ if ( == Config::location) return (Config::new_value)Config::log = Config::Info($ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value))if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, to_any_coerceConfig::log)return (Config::new_value)}, -100)) 0.000000 MetaHookPre CallFunction(Option::set_change_handler, , (GridFTP::size_threshold, Config::config_option_changed{ if ( == Config::location) return (Config::new_value)Config::log = Config::Info($ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value))if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, to_any_coerceConfig::log)return (Config::new_value)}, -100)) @@ -3804,6 +3806,7 @@ 0.000000 | HookCallFunction Option::set_change_handler(FTP::max_reply_msg_length, Config::config_option_changed{ if ( == Config::location) return (Config::new_value)Config::log = Config::Info($ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value))if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, to_any_coerceConfig::log)return (Config::new_value)}, -100) 0.000000 | HookCallFunction Option::set_change_handler(FTP::max_user_length, Config::config_option_changed{ if ( == Config::location) return (Config::new_value)Config::log = Config::Info($ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value))if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, to_any_coerceConfig::log)return (Config::new_value)}, -100) 0.000000 | HookCallFunction Option::set_change_handler(FileExtract::default_limit, Config::config_option_changed{ if ( == Config::location) return (Config::new_value)Config::log = Config::Info($ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value))if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, to_any_coerceConfig::log)return (Config::new_value)}, -100) +0.000000 | HookCallFunction Option::set_change_handler(FileExtract::default_limit_includes_missing, Config::config_option_changed{ if ( == Config::location) return (Config::new_value)Config::log = Config::Info($ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value))if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, to_any_coerceConfig::log)return (Config::new_value)}, -100) 0.000000 | HookCallFunction Option::set_change_handler(Files::enable_reassembler, Config::config_option_changed{ if ( == Config::location) return (Config::new_value)Config::log = Config::Info($ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value))if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, to_any_coerceConfig::log)return (Config::new_value)}, -100) 0.000000 | HookCallFunction Option::set_change_handler(GridFTP::max_time, Config::config_option_changed{ if ( == Config::location) return (Config::new_value)Config::log = Config::Info($ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value))if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, to_any_coerceConfig::log)return (Config::new_value)}, -100) 0.000000 | HookCallFunction Option::set_change_handler(GridFTP::size_threshold, Config::config_option_changed{ if ( == Config::location) return (Config::new_value)Config::log = Config::Info($ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value))if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, to_any_coerceConfig::log)return (Config::new_value)}, -100) diff --git a/testing/btest/Baseline/scripts.base.files.extract.limit-large-hole/1.out b/testing/btest/Baseline/scripts.base.files.extract.limit-large-hole/1.out new file mode 100644 index 0000000000..0ef9544205 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.files.extract.limit-large-hole/1.out @@ -0,0 +1,2 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +file_extraction_limit, 10, 2147483648 diff --git a/testing/btest/Baseline/scripts.base.files.extract.limit-large-hole/2.out b/testing/btest/Baseline/scripts.base.files.extract.limit-large-hole/2.out new file mode 100644 index 0000000000..49d861c74c --- /dev/null +++ b/testing/btest/Baseline/scripts.base.files.extract.limit-large-hole/2.out @@ -0,0 +1 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. diff --git a/testing/btest/Baseline/scripts.base.files.extract.limit-large-hole/3.out b/testing/btest/Baseline/scripts.base.files.extract.limit-large-hole/3.out new file mode 100644 index 0000000000..4f3357a79c --- /dev/null +++ b/testing/btest/Baseline/scripts.base.files.extract.limit-large-hole/3.out @@ -0,0 +1,2 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +file_extraction_limit, 1, 2 diff --git a/testing/btest/Baseline/scripts.base.files.extract.limit-large-hole/extract_files.1 b/testing/btest/Baseline/scripts.base.files.extract.limit-large-hole/extract_files.1 new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testing/btest/Baseline/scripts.base.files.extract.limit-large-hole/files-1.log b/testing/btest/Baseline/scripts.base.files.extract.limit-large-hole/files-1.log new file mode 100644 index 0000000000..127d2603f7 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.files.extract.limit-large-hole/files-1.log @@ -0,0 +1,11 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path files +#open XXXX-XX-XX-XX-XX-XX +#fields ts fuid uid id.orig_h id.orig_p id.resp_h id.resp_p source depth analyzers mime_type filename duration local_orig is_orig seen_bytes total_bytes missing_bytes overflow_bytes timedout parent_fuid extracted extracted_cutoff extracted_size +#types time string string addr port addr port string count set[string] string string interval bool bool count count count count bool string string bool count +XXXXXXXXXX.XXXXXX Fg5gNDmaUhHwqjbp8 CHhAvVGS1DHFjwGM9 192.168.65.2 53720 91.189.91.123 80 HTTP 0 EXTRACT - - 0.000000 F F 2 5037662208 2147483648 0 T - 1 T 10 +#close XXXX-XX-XX-XX-XX-XX diff --git a/testing/btest/Baseline/scripts.base.files.extract.limit-large-hole/files-2.log b/testing/btest/Baseline/scripts.base.files.extract.limit-large-hole/files-2.log new file mode 100644 index 0000000000..ffe8bed0ad --- /dev/null +++ b/testing/btest/Baseline/scripts.base.files.extract.limit-large-hole/files-2.log @@ -0,0 +1,11 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path files +#open XXXX-XX-XX-XX-XX-XX +#fields ts fuid uid id.orig_h id.orig_p id.resp_h id.resp_p source depth analyzers mime_type filename duration local_orig is_orig seen_bytes total_bytes missing_bytes overflow_bytes timedout parent_fuid extracted extracted_cutoff extracted_size +#types time string string addr port addr port string count set[string] string string interval bool bool count count count count bool string string bool count +XXXXXXXXXX.XXXXXX Fg5gNDmaUhHwqjbp8 CHhAvVGS1DHFjwGM9 192.168.65.2 53720 91.189.91.123 80 HTTP 0 EXTRACT - - 0.000000 F F 2 5037662208 2147483648 0 T - 2 F - +#close XXXX-XX-XX-XX-XX-XX diff --git a/testing/btest/Baseline/scripts.base.files.extract.limit-large-hole/files-3.log b/testing/btest/Baseline/scripts.base.files.extract.limit-large-hole/files-3.log new file mode 100644 index 0000000000..c2d233c3ec --- /dev/null +++ b/testing/btest/Baseline/scripts.base.files.extract.limit-large-hole/files-3.log @@ -0,0 +1,11 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path files +#open XXXX-XX-XX-XX-XX-XX +#fields ts fuid uid id.orig_h id.orig_p id.resp_h id.resp_p source depth analyzers mime_type filename duration local_orig is_orig seen_bytes total_bytes missing_bytes overflow_bytes timedout parent_fuid extracted extracted_cutoff extracted_size +#types time string string addr port addr port string count set[string] string string interval bool bool count count count count bool string string bool count +XXXXXXXXXX.XXXXXX Fg5gNDmaUhHwqjbp8 CHhAvVGS1DHFjwGM9 192.168.65.2 53720 91.189.91.123 80 HTTP 0 EXTRACT - - 0.000000 F F 2 5037662208 2147483648 0 T - 3 T 1 +#close XXXX-XX-XX-XX-XX-XX diff --git a/testing/btest/Baseline/spicy.file-analyzer-nested/output-max b/testing/btest/Baseline/spicy.file-analyzer-nested/output-max index c7a6a03585..f973bfca9d 100644 --- a/testing/btest/Baseline/spicy.file-analyzer-nested/output-max +++ b/testing/btest/Baseline/spicy.file-analyzer-nested/output-max @@ -1,7 +1,7 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. data3, FyjjRu4ARLzpsPLhNh, data3, Fz3QLf4Bn4qaQwyUdk, -depth warning, FyjjRu4ARLzpsPLhNh, [chunk_event=, stream_event=, extract_filename=, extract_limit=0], 2 -depth warning, Fz3QLf4Bn4qaQwyUdk, [chunk_event=, stream_event=, extract_filename=, extract_limit=0], 2 +depth warning, FyjjRu4ARLzpsPLhNh, [chunk_event=, stream_event=, extract_filename=, extract_limit=0, extract_limit_includes_missing=T], 2 +depth warning, Fz3QLf4Bn4qaQwyUdk, [chunk_event=, stream_event=, extract_filename=, extract_limit=0, extract_limit_includes_missing=T], 2 data2, F2Qpmk14ATv4vFSEsi, from 1:hello world data1, FcRmxz1fPbKQEgGGUi, hello world diff --git a/testing/btest/Traces/http/http-large-gap.pcap b/testing/btest/Traces/http/http-large-gap.pcap new file mode 100644 index 0000000000000000000000000000000000000000..649f5305fc5ee9c154594f0320b8d58444d367f1 GIT binary patch literal 1339 zcmaKrO=uHA7=~xlSnI-q@l-E$tX6ArH=D_JvsqLcwX{f!u}$%(2iYc*G;O+BvKzGs zQtCm47LVSfp7bDsB4RADf){J+q2i&vNb#UxFXAsqsLpJfpC+LTAIuCh5AXNA?|yjw zhHRS7{E5se0GuLgTKctEZYuKV4ye1RH-Jt6j-Y)VpriiuvsVuSj=}8U z)um++bge#Lprjec-)|Wg01oH2r%p!$+w!hsH$7+S>b6rdKgATud1juUC)1n0`nM~T zO6efDw47CymxR7Jvuph(K4z02QnF;Mw^Dk6{GE`GllQc$G)ri|tvOqvd6CfPCU!Ev zGdqYz#@1_4GSzx_ngOeXMyrbW9<|uSrkcb?LcG73*wq<=ZiEC;61?v8P+GUr{Kl4# zU~|+;8oV|ZktHE+CSi9d6m)w8565;VO-qF-oWPoiO(FkM2q(ulwolVzScOC57B-I} zkK~gSuPiBiap~vS0TZYAKn&{y5>BTQZbd*88wiK7LC)ITZ5+$7wBH|}9K{J%c_S^H zrQZCzRpHH?Yc(@MCYZZT22UqQrZ+)+F(`5Evjbcwp`ukuWYP=gC~>{3Cb5ypU4C37 zPEulFCJ>1-3~DJWt|j1!q;8SsIQFDwky(V&_%IaPVLvv=HROdJSrw7$RbW?dh-3S4 z>Jm<=FkoomVeCeNBp^R*9kA^<#IZe^Y4N?uNIV+HkpfMDfpjcS6FsWOuX^P?tuv&> zRM_l6VM)?NQRHP2wMUUJ8jbkFsJSp?-uB7dFp6V0B=bF3k6FViM5Uv0M`6*%a!1|m zqSq(O2#HGRS!moKy=#V%h=(;Rp48n}c#^FHsjGm;#zO^lh2W{eunnV}#tBg2bB&yD>f_UBz;*HI*Q2`0! q=!A>OP4^aKvqEAsPc?olBi87;)z+A)&^SuEW=Pj%8UuTX#+CnH@SCUr literal 0 HcmV?d00001 diff --git a/testing/btest/scripts/base/files/extract/limit-large-hole.zeek b/testing/btest/scripts/base/files/extract/limit-large-hole.zeek new file mode 100644 index 0000000000..221e91fd73 --- /dev/null +++ b/testing/btest/scripts/base/files/extract/limit-large-hole.zeek @@ -0,0 +1,38 @@ +# @TEST-EXEC: zeek -C -b -r $TRACES/http/http-large-gap.pcap %INPUT efname=1 FileExtract::default_limit_includes_missing=T +# @TEST-EXEC: btest-diff --binary extract_files/1 +# @TEST-EXEC: btest-diff 1.out +# @TEST-EXEC: mv files.log files-1.log +# @TEST-EXEC: btest-diff files-1.log +# @TEST-EXEC: zeek -C -b -r $TRACES/http/http-large-gap.pcap %INPUT efname=2 FileExtract::default_limit_includes_missing=F +# @TEST-EXEC: rm extract_files/2 +# @TEST-EXEC: btest-diff 2.out +# @TEST-EXEC: mv files.log files-2.log +# @TEST-EXEC: btest-diff files-2.log +# @TEST-EXEC: zeek -C -b -r $TRACES/http/http-large-gap.pcap %INPUT efname=3 FileExtract::default_limit_includes_missing=F max_extract=1 +# @TEST-EXEC: rm extract_files/3 +# @TEST-EXEC: btest-diff 3.out +# @TEST-EXEC: mv files.log files-3.log +# @TEST-EXEC: btest-diff files-3.log + +@load base/files/extract +@load base/protocols/http + +global outfile: file; +const max_extract: count = 10 &redef; +const efname: string = "0" &redef; + +event file_new(f: fa_file) + { + Files::add_analyzer(f, Files::ANALYZER_EXTRACT, + [$extract_filename=efname, $extract_limit=max_extract]); + } + +event file_extraction_limit(f: fa_file, args: any, limit: count, len: count) + { + print outfile, "file_extraction_limit", limit, len; + } + +event zeek_init() + { + outfile = open(fmt("%s.out", efname)); + }