From 339d46ae26e13ffb8dfc5c45036584913a8e52d7 Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Thu, 31 Jul 2025 13:16:16 -0700 Subject: [PATCH] Add a weird that gets emitted when strings/containers are over the limits --- src/logging/Manager.cc | 17 +++++++----- .../.stdout | 2 ++ .../weird.log | 11 ++++++++ .../.stdout | 2 ++ .../weird.log | 11 ++++++++ .../.stdout | 2 ++ .../weird.log | 11 ++++++++ .../.stdout | 2 ++ .../weird.log | 11 ++++++++ .../.stdout | 2 ++ .../weird.log | 11 ++++++++ .../.stdout | 2 ++ .../weird.log | 11 ++++++++ .../logging/field-length-limiting.zeek | 26 +++++++++++++------ 14 files changed, 107 insertions(+), 14 deletions(-) create mode 100644 testing/btest/Baseline/scripts.base.frameworks.logging.field-length-limiting-2/weird.log create mode 100644 testing/btest/Baseline/scripts.base.frameworks.logging.field-length-limiting-3/weird.log create mode 100644 testing/btest/Baseline/scripts.base.frameworks.logging.field-length-limiting-4/weird.log create mode 100644 testing/btest/Baseline/scripts.base.frameworks.logging.field-length-limiting-5/weird.log create mode 100644 testing/btest/Baseline/scripts.base.frameworks.logging.field-length-limiting-6/weird.log create mode 100644 testing/btest/Baseline/scripts.base.frameworks.logging.field-length-limiting/weird.log diff --git a/src/logging/Manager.cc b/src/logging/Manager.cc index b023506661..89c6e8266f 100644 --- a/src/logging/Manager.cc +++ b/src/logging/Manager.cc @@ -243,6 +243,7 @@ struct Manager::WriterInfo { bool from_remote = false; bool hook_initialized = false; string instantiating_filter; + string stream_name; std::shared_ptr total_writes; std::shared_ptr total_discarded_writes; @@ -1501,9 +1502,10 @@ threading::Value Manager::ValToLogVal(WriterInfo* info, std::optional& val size_t allowed_bytes = std::min( {static_cast(s->Len()), max_field_string_bytes, max_total_string_bytes - total_string_bytes}); - if ( allowed_bytes < static_cast(s->Len()) ) - // TODO: this could also log a reporter warning or a weird or something + if ( allowed_bytes < static_cast(s->Len()) ) { + reporter->Weird("log_string_field_truncated", util::fmt("%s", info->stream_name.c_str())); info->total_truncated_string_fields->Inc(); + } if ( allowed_bytes == 0 ) return lval; @@ -1556,9 +1558,10 @@ threading::Value Manager::ValToLogVal(WriterInfo* info, std::optional& val size_t allowed_elements = std::min({static_cast(set->Length()), max_field_container_elements, max_total_container_elements - total_container_elements}); - if ( allowed_elements < static_cast(set->Length()) ) - // TODO: this could also log a reporter warning or a weird or something + if ( allowed_elements < static_cast(set->Length()) ) { + reporter->Weird("log_container_field_truncated", util::fmt("%s", info->stream_name.c_str())); info->total_truncated_containers->Inc(); + } if ( allowed_elements == 0 ) return lval; @@ -1584,9 +1587,10 @@ threading::Value Manager::ValToLogVal(WriterInfo* info, std::optional& val size_t allowed_elements = std::min({static_cast(vec->Size()), max_field_container_elements, max_total_container_elements - total_container_elements}); - if ( allowed_elements < static_cast(vec->Size()) ) - // TODO: this could also log a reporter warning or a weird or something + if ( allowed_elements < static_cast(vec->Size()) ) { + reporter->Weird("log_container_field_truncated", util::fmt("%s", info->stream_name.c_str())); info->total_truncated_containers->Inc(); + } if ( allowed_elements == 0 ) return lval; @@ -1730,6 +1734,7 @@ WriterFrontend* Manager::CreateWriter(EnumVal* id, EnumVal* writer, WriterBacken winfo->from_remote = from_remote; winfo->hook_initialized = false; winfo->instantiating_filter = instantiating_filter; + winfo->stream_name = stream->name; // Search for a corresponding filter for the writer/path pair and use its // rotation settings. If no matching filter is found, fall back on diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.field-length-limiting-2/.stdout b/testing/btest/Baseline/scripts.base.frameworks.logging.field-length-limiting-2/.stdout index e40107ed88..8257d72161 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.field-length-limiting-2/.stdout +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.field-length-limiting-2/.stdout @@ -1,3 +1,5 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +Telemetry::COUNTER, zeek, zeek_log_writer_truncated_string_fields_total, [filter_name, module, path, stream, writer], [default, Weird, weird, Weird::LOG, Log::WRITER_ASCII], 0.0 Telemetry::COUNTER, zeek, zeek_log_writer_truncated_string_fields_total, [filter_name, module, path, stream, writer], [default, Test, test, Test::LOG, Log::WRITER_ASCII], 9.0 +Telemetry::COUNTER, zeek, zeek_log_writer_truncated_containers_total, [filter_name, module, path, stream, writer], [default, Weird, weird, Weird::LOG, Log::WRITER_ASCII], 0.0 Telemetry::COUNTER, zeek, zeek_log_writer_truncated_containers_total, [filter_name, module, path, stream, writer], [default, Test, test, Test::LOG, Log::WRITER_ASCII], 0.0 diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.field-length-limiting-2/weird.log b/testing/btest/Baseline/scripts.base.frameworks.logging.field-length-limiting-2/weird.log new file mode 100644 index 0000000000..fe5cd88cd4 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.field-length-limiting-2/weird.log @@ -0,0 +1,11 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path weird +#open XXXX-XX-XX-XX-XX-XX +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p name addl notice peer source +#types time string addr port addr port string string bool string string +XXXXXXXXXX.XXXXXX - - - - - log_string_field_truncated Test::LOG F zeek - +#close XXXX-XX-XX-XX-XX-XX diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.field-length-limiting-3/.stdout b/testing/btest/Baseline/scripts.base.frameworks.logging.field-length-limiting-3/.stdout index 51e81b6b71..57f1fd5f12 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.field-length-limiting-3/.stdout +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.field-length-limiting-3/.stdout @@ -1,3 +1,5 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +Telemetry::COUNTER, zeek, zeek_log_writer_truncated_string_fields_total, [filter_name, module, path, stream, writer], [default, Weird, weird, Weird::LOG, Log::WRITER_ASCII], 0.0 Telemetry::COUNTER, zeek, zeek_log_writer_truncated_string_fields_total, [filter_name, module, path, stream, writer], [default, Test, test, Test::LOG, Log::WRITER_ASCII], 12.0 +Telemetry::COUNTER, zeek, zeek_log_writer_truncated_containers_total, [filter_name, module, path, stream, writer], [default, Weird, weird, Weird::LOG, Log::WRITER_ASCII], 0.0 Telemetry::COUNTER, zeek, zeek_log_writer_truncated_containers_total, [filter_name, module, path, stream, writer], [default, Test, test, Test::LOG, Log::WRITER_ASCII], 0.0 diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.field-length-limiting-3/weird.log b/testing/btest/Baseline/scripts.base.frameworks.logging.field-length-limiting-3/weird.log new file mode 100644 index 0000000000..fe5cd88cd4 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.field-length-limiting-3/weird.log @@ -0,0 +1,11 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path weird +#open XXXX-XX-XX-XX-XX-XX +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p name addl notice peer source +#types time string addr port addr port string string bool string string +XXXXXXXXXX.XXXXXX - - - - - log_string_field_truncated Test::LOG F zeek - +#close XXXX-XX-XX-XX-XX-XX diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.field-length-limiting-4/.stdout b/testing/btest/Baseline/scripts.base.frameworks.logging.field-length-limiting-4/.stdout index 7094a3c95e..29978a303b 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.field-length-limiting-4/.stdout +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.field-length-limiting-4/.stdout @@ -1,3 +1,5 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +Telemetry::COUNTER, zeek, zeek_log_writer_truncated_string_fields_total, [filter_name, module, path, stream, writer], [default, Weird, weird, Weird::LOG, Log::WRITER_ASCII], 0.0 Telemetry::COUNTER, zeek, zeek_log_writer_truncated_string_fields_total, [filter_name, module, path, stream, writer], [default, Test, test, Test::LOG, Log::WRITER_ASCII], 0.0 +Telemetry::COUNTER, zeek, zeek_log_writer_truncated_containers_total, [filter_name, module, path, stream, writer], [default, Weird, weird, Weird::LOG, Log::WRITER_ASCII], 0.0 Telemetry::COUNTER, zeek, zeek_log_writer_truncated_containers_total, [filter_name, module, path, stream, writer], [default, Test, test, Test::LOG, Log::WRITER_ASCII], 2.0 diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.field-length-limiting-4/weird.log b/testing/btest/Baseline/scripts.base.frameworks.logging.field-length-limiting-4/weird.log new file mode 100644 index 0000000000..32aea67226 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.field-length-limiting-4/weird.log @@ -0,0 +1,11 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path weird +#open XXXX-XX-XX-XX-XX-XX +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p name addl notice peer source +#types time string addr port addr port string string bool string string +XXXXXXXXXX.XXXXXX - - - - - log_container_field_truncated Test::LOG F zeek - +#close XXXX-XX-XX-XX-XX-XX diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.field-length-limiting-5/.stdout b/testing/btest/Baseline/scripts.base.frameworks.logging.field-length-limiting-5/.stdout index 44df4ae86b..f2e2b796bd 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.field-length-limiting-5/.stdout +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.field-length-limiting-5/.stdout @@ -1,3 +1,5 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +Telemetry::COUNTER, zeek, zeek_log_writer_truncated_string_fields_total, [filter_name, module, path, stream, writer], [default, Weird, weird, Weird::LOG, Log::WRITER_ASCII], 0.0 Telemetry::COUNTER, zeek, zeek_log_writer_truncated_string_fields_total, [filter_name, module, path, stream, writer], [default, Test, test, Test::LOG, Log::WRITER_ASCII], 0.0 +Telemetry::COUNTER, zeek, zeek_log_writer_truncated_containers_total, [filter_name, module, path, stream, writer], [default, Weird, weird, Weird::LOG, Log::WRITER_ASCII], 0.0 Telemetry::COUNTER, zeek, zeek_log_writer_truncated_containers_total, [filter_name, module, path, stream, writer], [default, Test, test, Test::LOG, Log::WRITER_ASCII], 1.0 diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.field-length-limiting-5/weird.log b/testing/btest/Baseline/scripts.base.frameworks.logging.field-length-limiting-5/weird.log new file mode 100644 index 0000000000..32aea67226 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.field-length-limiting-5/weird.log @@ -0,0 +1,11 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path weird +#open XXXX-XX-XX-XX-XX-XX +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p name addl notice peer source +#types time string addr port addr port string string bool string string +XXXXXXXXXX.XXXXXX - - - - - log_container_field_truncated Test::LOG F zeek - +#close XXXX-XX-XX-XX-XX-XX diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.field-length-limiting-6/.stdout b/testing/btest/Baseline/scripts.base.frameworks.logging.field-length-limiting-6/.stdout index 7094a3c95e..29978a303b 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.field-length-limiting-6/.stdout +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.field-length-limiting-6/.stdout @@ -1,3 +1,5 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +Telemetry::COUNTER, zeek, zeek_log_writer_truncated_string_fields_total, [filter_name, module, path, stream, writer], [default, Weird, weird, Weird::LOG, Log::WRITER_ASCII], 0.0 Telemetry::COUNTER, zeek, zeek_log_writer_truncated_string_fields_total, [filter_name, module, path, stream, writer], [default, Test, test, Test::LOG, Log::WRITER_ASCII], 0.0 +Telemetry::COUNTER, zeek, zeek_log_writer_truncated_containers_total, [filter_name, module, path, stream, writer], [default, Weird, weird, Weird::LOG, Log::WRITER_ASCII], 0.0 Telemetry::COUNTER, zeek, zeek_log_writer_truncated_containers_total, [filter_name, module, path, stream, writer], [default, Test, test, Test::LOG, Log::WRITER_ASCII], 2.0 diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.field-length-limiting-6/weird.log b/testing/btest/Baseline/scripts.base.frameworks.logging.field-length-limiting-6/weird.log new file mode 100644 index 0000000000..32aea67226 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.field-length-limiting-6/weird.log @@ -0,0 +1,11 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path weird +#open XXXX-XX-XX-XX-XX-XX +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p name addl notice peer source +#types time string addr port addr port string string bool string string +XXXXXXXXXX.XXXXXX - - - - - log_container_field_truncated Test::LOG F zeek - +#close XXXX-XX-XX-XX-XX-XX diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.field-length-limiting/.stdout b/testing/btest/Baseline/scripts.base.frameworks.logging.field-length-limiting/.stdout index 354dee21f9..a9548ba949 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.field-length-limiting/.stdout +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.field-length-limiting/.stdout @@ -1,3 +1,5 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +Telemetry::COUNTER, zeek, zeek_log_writer_truncated_string_fields_total, [filter_name, module, path, stream, writer], [default, Weird, weird, Weird::LOG, Log::WRITER_ASCII], 2.0 Telemetry::COUNTER, zeek, zeek_log_writer_truncated_string_fields_total, [filter_name, module, path, stream, writer], [default, Test, test, Test::LOG, Log::WRITER_ASCII], 20.0 +Telemetry::COUNTER, zeek, zeek_log_writer_truncated_containers_total, [filter_name, module, path, stream, writer], [default, Weird, weird, Weird::LOG, Log::WRITER_ASCII], 0.0 Telemetry::COUNTER, zeek, zeek_log_writer_truncated_containers_total, [filter_name, module, path, stream, writer], [default, Test, test, Test::LOG, Log::WRITER_ASCII], 0.0 diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.field-length-limiting/weird.log b/testing/btest/Baseline/scripts.base.frameworks.logging.field-length-limiting/weird.log new file mode 100644 index 0000000000..ef2c56bc6b --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.field-length-limiting/weird.log @@ -0,0 +1,11 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path weird +#open XXXX-XX-XX-XX-XX-XX +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p name addl notice peer source +#types time string addr port addr port string string bool string string +XXXXXXXXXX.XXXXXX - - - - - log_s Test: F zeek - +#close XXXX-XX-XX-XX-XX-XX diff --git a/testing/btest/scripts/base/frameworks/logging/field-length-limiting.zeek b/testing/btest/scripts/base/frameworks/logging/field-length-limiting.zeek index cb05664065..4946c1b230 100644 --- a/testing/btest/scripts/base/frameworks/logging/field-length-limiting.zeek +++ b/testing/btest/scripts/base/frameworks/logging/field-length-limiting.zeek @@ -3,10 +3,12 @@ # @TEST-EXEC: zeek -b test.zeek %INPUT # @TEST-EXEC: btest-diff test.log # @TEST-EXEC: btest-diff .stdout +# @TEST-EXEC: btest-diff weird.log # @TEST-START-FILE test.zeek @load base/frameworks/telemetry +@load base/frameworks/notice/weird module Test; @@ -19,6 +21,16 @@ export { }; } +event log_telemetry() + { + local storage_metrics = Telemetry::collect_metrics("zeek", "log_writer_truncated*"); + for (i in storage_metrics) + { + local m = storage_metrics[i]; + print m$opts$metric_type, m$opts$prefix, m$opts$name, m$label_names, m$label_values, m$value; + } + } + event zeek_init() { Log::create_stream(LOG, [$columns=Info, $path="test"]); @@ -36,19 +48,17 @@ event zeek_init() Log::write(Test::LOG, rec); - local storage_metrics = Telemetry::collect_metrics("zeek", "log_writer_truncated*"); - for (i in storage_metrics) - { - local m = storage_metrics[i]; - print m$opts$metric_type, m$opts$prefix, m$opts$name, m$label_names, m$label_values, m$value; - } + # Do this as a separate event so the weirds get processed before we log the + # telemetry outout. See the comment below for the first test as to why. + event log_telemetry(); } - # @TEST-END-FILE test.zeek # Limit the individual fields to 5 bytes, but keep the total maximum large enough that it -# will write all of the fields. +# will write all of the fields. The weird test for this one will be off since it will +# limit the name of the weird. It will pass, but the fields in the log will get truncated +# like they're supposed to. redef Log::max_field_string_bytes = 5; # @TEST-START-NEXT