Mirror/zeek
1
0
Fork 0
mirror of https://github.com/zeek/zeek.git synced 2025-10-02 06:38:20 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Add a weird that gets emitted when strings/containers are over the limits

Browse source
This commit is contained in:
Tim Wojtulewicz 2025-07-31 13:16:16 -07:00
parent 837fde1a08
commit 339d46ae26
14 changed files with 107 additions and 14 deletions
Download patch file Download diff file Expand all files Collapse all files

17
src/logging/Manager.cc
View file

@ -243,6 +243,7 @@ struct Manager::WriterInfo {
bool from_remote = false;
bool hook_initialized = false;
string instantiating_filter;
string stream_name;
std::shared_ptr<telemetry::Counter> total_writes;
std::shared_ptr<telemetry::Counter> total_discarded_writes;
@ -1501,9 +1502,10 @@ threading::Value Manager::ValToLogVal(WriterInfo* info, std::optional<ZVal>& val
size_t allowed_bytes = std::min(
{static_cast<size_t>(s->Len()), max_field_string_bytes, max_total_string_bytes - total_string_bytes});
if ( allowed_bytes < static_cast<size_t>(s->Len()) )
// TODO: this could also log a reporter warning or a weird or something
if ( allowed_bytes < static_cast<size_t>(s->Len()) ) {
reporter->Weird("log_string_field_truncated", util::fmt("%s", info->stream_name.c_str()));
info->total_truncated_string_fields->Inc();
}
if ( allowed_bytes == 0 )
return lval;
@ -1556,9 +1558,10 @@ threading::Value Manager::ValToLogVal(WriterInfo* info, std::optional<ZVal>& val
size_t allowed_elements = std::min({static_cast<size_t>(set->Length()), max_field_container_elements,
max_total_container_elements - total_container_elements});
if ( allowed_elements < static_cast<size_t>(set->Length()) )
// TODO: this could also log a reporter warning or a weird or something
if ( allowed_elements < static_cast<size_t>(set->Length()) ) {
reporter->Weird("log_container_field_truncated", util::fmt("%s", info->stream_name.c_str()));
info->total_truncated_containers->Inc();
}
if ( allowed_elements == 0 )
return lval;
@ -1584,9 +1587,10 @@ threading::Value Manager::ValToLogVal(WriterInfo* info, std::optional<ZVal>& val
size_t allowed_elements = std::min({static_cast<size_t>(vec->Size()), max_field_container_elements,
max_total_container_elements - total_container_elements});
if ( allowed_elements < static_cast<size_t>(vec->Size()) )
// TODO: this could also log a reporter warning or a weird or something
if ( allowed_elements < static_cast<size_t>(vec->Size()) ) {
reporter->Weird("log_container_field_truncated", util::fmt("%s", info->stream_name.c_str()));
info->total_truncated_containers->Inc();
}
if ( allowed_elements == 0 )
return lval;
@ -1730,6 +1734,7 @@ WriterFrontend* Manager::CreateWriter(EnumVal* id, EnumVal* writer, WriterBacken
winfo->from_remote = from_remote;
winfo->hook_initialized = false;
winfo->instantiating_filter = instantiating_filter;
winfo->stream_name = stream->name;
// Search for a corresponding filter for the writer/path pair and use its
// rotation settings. If no matching filter is found, fall back on

2
testing/btest/Baseline/scripts.base.frameworks.logging.field-length-limiting-2/.stdout
View file

@ -1,3 +1,5 @@
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
Telemetry::COUNTER, zeek, zeek_log_writer_truncated_string_fields_total, [filter_name, module, path, stream, writer], [default, Weird, weird, Weird::LOG, Log::WRITER_ASCII], 0.0
Telemetry::COUNTER, zeek, zeek_log_writer_truncated_string_fields_total, [filter_name, module, path, stream, writer], [default, Test, test, Test::LOG, Log::WRITER_ASCII], 9.0
Telemetry::COUNTER, zeek, zeek_log_writer_truncated_containers_total, [filter_name, module, path, stream, writer], [default, Weird, weird, Weird::LOG, Log::WRITER_ASCII], 0.0
Telemetry::COUNTER, zeek, zeek_log_writer_truncated_containers_total, [filter_name, module, path, stream, writer], [default, Test, test, Test::LOG, Log::WRITER_ASCII], 0.0

11
testing/btest/Baseline/scripts.base.frameworks.logging.field-length-limiting-2/weird.log Normal file
View file

@ -0,0 +1,11 @@
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
#separator \x09
#set_separator ,
#empty_field (empty)
#unset_field -
#path weird
#open XXXX-XX-XX-XX-XX-XX
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p name addl notice peer source
#types time string addr port addr port string string bool string string
XXXXXXXXXX.XXXXXX - - - - - log_string_field_truncated Test::LOG F zeek -
#close XXXX-XX-XX-XX-XX-XX

2
testing/btest/Baseline/scripts.base.frameworks.logging.field-length-limiting-3/.stdout
View file

@ -1,3 +1,5 @@
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
Telemetry::COUNTER, zeek, zeek_log_writer_truncated_string_fields_total, [filter_name, module, path, stream, writer], [default, Weird, weird, Weird::LOG, Log::WRITER_ASCII], 0.0
Telemetry::COUNTER, zeek, zeek_log_writer_truncated_string_fields_total, [filter_name, module, path, stream, writer], [default, Test, test, Test::LOG, Log::WRITER_ASCII], 12.0
Telemetry::COUNTER, zeek, zeek_log_writer_truncated_containers_total, [filter_name, module, path, stream, writer], [default, Weird, weird, Weird::LOG, Log::WRITER_ASCII], 0.0
Telemetry::COUNTER, zeek, zeek_log_writer_truncated_containers_total, [filter_name, module, path, stream, writer], [default, Test, test, Test::LOG, Log::WRITER_ASCII], 0.0

11
testing/btest/Baseline/scripts.base.frameworks.logging.field-length-limiting-3/weird.log Normal file
View file

@ -0,0 +1,11 @@
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
#separator \x09
#set_separator ,
#empty_field (empty)
#unset_field -
#path weird
#open XXXX-XX-XX-XX-XX-XX
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p name addl notice peer source
#types time string addr port addr port string string bool string string
XXXXXXXXXX.XXXXXX - - - - - log_string_field_truncated Test::LOG F zeek -
#close XXXX-XX-XX-XX-XX-XX

2
testing/btest/Baseline/scripts.base.frameworks.logging.field-length-limiting-4/.stdout
View file

@ -1,3 +1,5 @@
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
Telemetry::COUNTER, zeek, zeek_log_writer_truncated_string_fields_total, [filter_name, module, path, stream, writer], [default, Weird, weird, Weird::LOG, Log::WRITER_ASCII], 0.0
Telemetry::COUNTER, zeek, zeek_log_writer_truncated_string_fields_total, [filter_name, module, path, stream, writer], [default, Test, test, Test::LOG, Log::WRITER_ASCII], 0.0
Telemetry::COUNTER, zeek, zeek_log_writer_truncated_containers_total, [filter_name, module, path, stream, writer], [default, Weird, weird, Weird::LOG, Log::WRITER_ASCII], 0.0
Telemetry::COUNTER, zeek, zeek_log_writer_truncated_containers_total, [filter_name, module, path, stream, writer], [default, Test, test, Test::LOG, Log::WRITER_ASCII], 2.0

11
testing/btest/Baseline/scripts.base.frameworks.logging.field-length-limiting-4/weird.log Normal file
View file

@ -0,0 +1,11 @@
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
#separator \x09
#set_separator ,
#empty_field (empty)
#unset_field -
#path weird
#open XXXX-XX-XX-XX-XX-XX
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p name addl notice peer source
#types time string addr port addr port string string bool string string
XXXXXXXXXX.XXXXXX - - - - - log_container_field_truncated Test::LOG F zeek -
#close XXXX-XX-XX-XX-XX-XX

2
testing/btest/Baseline/scripts.base.frameworks.logging.field-length-limiting-5/.stdout
View file

@ -1,3 +1,5 @@
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
Telemetry::COUNTER, zeek, zeek_log_writer_truncated_string_fields_total, [filter_name, module, path, stream, writer], [default, Weird, weird, Weird::LOG, Log::WRITER_ASCII], 0.0
Telemetry::COUNTER, zeek, zeek_log_writer_truncated_string_fields_total, [filter_name, module, path, stream, writer], [default, Test, test, Test::LOG, Log::WRITER_ASCII], 0.0
Telemetry::COUNTER, zeek, zeek_log_writer_truncated_containers_total, [filter_name, module, path, stream, writer], [default, Weird, weird, Weird::LOG, Log::WRITER_ASCII], 0.0
Telemetry::COUNTER, zeek, zeek_log_writer_truncated_containers_total, [filter_name, module, path, stream, writer], [default, Test, test, Test::LOG, Log::WRITER_ASCII], 1.0

11
testing/btest/Baseline/scripts.base.frameworks.logging.field-length-limiting-5/weird.log Normal file
View file

@ -0,0 +1,11 @@
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
#separator \x09
#set_separator ,
#empty_field (empty)
#unset_field -
#path weird
#open XXXX-XX-XX-XX-XX-XX
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p name addl notice peer source
#types time string addr port addr port string string bool string string
XXXXXXXXXX.XXXXXX - - - - - log_container_field_truncated Test::LOG F zeek -
#close XXXX-XX-XX-XX-XX-XX

2
testing/btest/Baseline/scripts.base.frameworks.logging.field-length-limiting-6/.stdout
View file

@ -1,3 +1,5 @@
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
Telemetry::COUNTER, zeek, zeek_log_writer_truncated_string_fields_total, [filter_name, module, path, stream, writer], [default, Weird, weird, Weird::LOG, Log::WRITER_ASCII], 0.0
Telemetry::COUNTER, zeek, zeek_log_writer_truncated_string_fields_total, [filter_name, module, path, stream, writer], [default, Test, test, Test::LOG, Log::WRITER_ASCII], 0.0
Telemetry::COUNTER, zeek, zeek_log_writer_truncated_containers_total, [filter_name, module, path, stream, writer], [default, Weird, weird, Weird::LOG, Log::WRITER_ASCII], 0.0
Telemetry::COUNTER, zeek, zeek_log_writer_truncated_containers_total, [filter_name, module, path, stream, writer], [default, Test, test, Test::LOG, Log::WRITER_ASCII], 2.0

11
testing/btest/Baseline/scripts.base.frameworks.logging.field-length-limiting-6/weird.log Normal file
View file

@ -0,0 +1,11 @@
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
#separator \x09
#set_separator ,
#empty_field (empty)
#unset_field -
#path weird
#open XXXX-XX-XX-XX-XX-XX
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p name addl notice peer source
#types time string addr port addr port string string bool string string
XXXXXXXXXX.XXXXXX - - - - - log_container_field_truncated Test::LOG F zeek -
#close XXXX-XX-XX-XX-XX-XX

2
testing/btest/Baseline/scripts.base.frameworks.logging.field-length-limiting/.stdout
View file

@ -1,3 +1,5 @@
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
Telemetry::COUNTER, zeek, zeek_log_writer_truncated_string_fields_total, [filter_name, module, path, stream, writer], [default, Weird, weird, Weird::LOG, Log::WRITER_ASCII], 2.0
Telemetry::COUNTER, zeek, zeek_log_writer_truncated_string_fields_total, [filter_name, module, path, stream, writer], [default, Test, test, Test::LOG, Log::WRITER_ASCII], 20.0
Telemetry::COUNTER, zeek, zeek_log_writer_truncated_containers_total, [filter_name, module, path, stream, writer], [default, Weird, weird, Weird::LOG, Log::WRITER_ASCII], 0.0
Telemetry::COUNTER, zeek, zeek_log_writer_truncated_containers_total, [filter_name, module, path, stream, writer], [default, Test, test, Test::LOG, Log::WRITER_ASCII], 0.0

11
testing/btest/Baseline/scripts.base.frameworks.logging.field-length-limiting/weird.log Normal file
View file

@ -0,0 +1,11 @@
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
#separator \x09
#set_separator ,
#empty_field (empty)
#unset_field -
#path weird
#open XXXX-XX-XX-XX-XX-XX
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p name addl notice peer source
#types time string addr port addr port string string bool string string
XXXXXXXXXX.XXXXXX - - - - - log_s Test: F zeek -
#close XXXX-XX-XX-XX-XX-XX

26
testing/btest/scripts/base/frameworks/logging/field-length-limiting.zeek
View file

@ -3,10 +3,12 @@
# @TEST-EXEC: zeek -b test.zeek %INPUT
# @TEST-EXEC: btest-diff test.log
# @TEST-EXEC: btest-diff .stdout
# @TEST-EXEC: btest-diff weird.log
# @TEST-START-FILE test.zeek
@load base/frameworks/telemetry
@load base/frameworks/notice/weird
module Test;
@ -19,6 +21,16 @@ export {
};
}
event log_telemetry()
{
local storage_metrics = Telemetry::collect_metrics("zeek", "log_writer_truncated*");
for (i in storage_metrics)
{
local m = storage_metrics[i];
print m$opts$metric_type, m$opts$prefix, m$opts$name, m$label_names, m$label_values, m$value;
}
}
event zeek_init()
{
Log::create_stream(LOG, [$columns=Info, $path="test"]);
@ -36,19 +48,17 @@ event zeek_init()
Log::write(Test::LOG, rec);
local storage_metrics = Telemetry::collect_metrics("zeek", "log_writer_truncated*");
for (i in storage_metrics)
{
local m = storage_metrics[i];
print m$opts$metric_type, m$opts$prefix, m$opts$name, m$label_names, m$label_values, m$value;
# Do this as a separate event so the weirds get processed before we log the
# telemetry outout. See the comment below for the first test as to why.
event log_telemetry();
}
}
# @TEST-END-FILE test.zeek
# Limit the individual fields to 5 bytes, but keep the total maximum large enough that it
# will write all of the fields.
# will write all of the fields. The weird test for this one will be off since it will
# limit the name of the weird. It will pass, but the fields in the log will get truncated
# like they're supposed to.
redef Log::max_field_string_bytes = 5;
# @TEST-START-NEXT