Add a weird that gets emitted when strings/containers are over the limits

src/logging/Manager.cc

@@ -243,6 +243,7 @@ struct Manager::WriterInfo {
     bool from_remote = false;
     bool hook_initialized = false;
     string instantiating_filter;
+    string stream_name;
 
     std::shared_ptr<telemetry::Counter> total_writes;
     std::shared_ptr<telemetry::Counter> total_discarded_writes;
@@ -1501,9 +1502,10 @@ threading::Value Manager::ValToLogVal(WriterInfo* info, std::optional<ZVal>& val
             size_t allowed_bytes = std::min(
                 {static_cast<size_t>(s->Len()), max_field_string_bytes, max_total_string_bytes - total_string_bytes});
 
-            if ( allowed_bytes < static_cast<size_t>(s->Len()) )
+            if ( allowed_bytes < static_cast<size_t>(s->Len()) ) {
-                // TODO: this could also log a reporter warning or a weird or something
+                reporter->Weird("log_string_field_truncated", util::fmt("%s", info->stream_name.c_str()));
                 info->total_truncated_string_fields->Inc();
+            }
 
             if ( allowed_bytes == 0 )
                 return lval;
@@ -1556,9 +1558,10 @@ threading::Value Manager::ValToLogVal(WriterInfo* info, std::optional<ZVal>& val
             size_t allowed_elements = std::min({static_cast<size_t>(set->Length()), max_field_container_elements,
                                                 max_total_container_elements - total_container_elements});
 
-            if ( allowed_elements < static_cast<size_t>(set->Length()) )
+            if ( allowed_elements < static_cast<size_t>(set->Length()) ) {
-                // TODO: this could also log a reporter warning or a weird or something
+                reporter->Weird("log_container_field_truncated", util::fmt("%s", info->stream_name.c_str()));
                 info->total_truncated_containers->Inc();
+            }
 
             if ( allowed_elements == 0 )
                 return lval;
@@ -1584,9 +1587,10 @@ threading::Value Manager::ValToLogVal(WriterInfo* info, std::optional<ZVal>& val
             size_t allowed_elements = std::min({static_cast<size_t>(vec->Size()), max_field_container_elements,
                                                 max_total_container_elements - total_container_elements});
 
-            if ( allowed_elements < static_cast<size_t>(vec->Size()) )
+            if ( allowed_elements < static_cast<size_t>(vec->Size()) ) {
-                // TODO: this could also log a reporter warning or a weird or something
+                reporter->Weird("log_container_field_truncated", util::fmt("%s", info->stream_name.c_str()));
                 info->total_truncated_containers->Inc();
+            }
 
             if ( allowed_elements == 0 )
                 return lval;
@@ -1730,6 +1734,7 @@ WriterFrontend* Manager::CreateWriter(EnumVal* id, EnumVal* writer, WriterBacken
     winfo->from_remote = from_remote;
     winfo->hook_initialized = false;
     winfo->instantiating_filter = instantiating_filter;
+    winfo->stream_name = stream->name;
 
     // Search for a corresponding filter for the writer/path pair and use its
     // rotation settings.  If no matching filter is found, fall back on

testing/btest/Baseline/scripts.base.frameworks.logging.field-length-limiting-2/.stdout

@@ -1,3 +1,5 @@
 ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+Telemetry::COUNTER, zeek, zeek_log_writer_truncated_string_fields_total, [filter_name, module, path, stream, writer], [default, Weird, weird, Weird::LOG, Log::WRITER_ASCII], 0.0
 Telemetry::COUNTER, zeek, zeek_log_writer_truncated_string_fields_total, [filter_name, module, path, stream, writer], [default, Test, test, Test::LOG, Log::WRITER_ASCII], 9.0
+Telemetry::COUNTER, zeek, zeek_log_writer_truncated_containers_total, [filter_name, module, path, stream, writer], [default, Weird, weird, Weird::LOG, Log::WRITER_ASCII], 0.0
 Telemetry::COUNTER, zeek, zeek_log_writer_truncated_containers_total, [filter_name, module, path, stream, writer], [default, Test, test, Test::LOG, Log::WRITER_ASCII], 0.0

testing/btest/Baseline/scripts.base.frameworks.logging.field-length-limiting-2/weird.log

@@ -0,0 +1,11 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	weird
+#open XXXX-XX-XX-XX-XX-XX
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	name	addl	notice	peer	source
+#types	time	string	addr	port	addr	port	string	string	bool	string	string
+XXXXXXXXXX.XXXXXX	-	-	-	-	-	log_string_field_truncated	Test::LOG	F	zeek	-
+#close XXXX-XX-XX-XX-XX-XX

testing/btest/Baseline/scripts.base.frameworks.logging.field-length-limiting-3/.stdout

@@ -1,3 +1,5 @@
 ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+Telemetry::COUNTER, zeek, zeek_log_writer_truncated_string_fields_total, [filter_name, module, path, stream, writer], [default, Weird, weird, Weird::LOG, Log::WRITER_ASCII], 0.0
 Telemetry::COUNTER, zeek, zeek_log_writer_truncated_string_fields_total, [filter_name, module, path, stream, writer], [default, Test, test, Test::LOG, Log::WRITER_ASCII], 12.0
+Telemetry::COUNTER, zeek, zeek_log_writer_truncated_containers_total, [filter_name, module, path, stream, writer], [default, Weird, weird, Weird::LOG, Log::WRITER_ASCII], 0.0
 Telemetry::COUNTER, zeek, zeek_log_writer_truncated_containers_total, [filter_name, module, path, stream, writer], [default, Test, test, Test::LOG, Log::WRITER_ASCII], 0.0

testing/btest/Baseline/scripts.base.frameworks.logging.field-length-limiting-3/weird.log

@@ -0,0 +1,11 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	weird
+#open XXXX-XX-XX-XX-XX-XX
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	name	addl	notice	peer	source
+#types	time	string	addr	port	addr	port	string	string	bool	string	string
+XXXXXXXXXX.XXXXXX	-	-	-	-	-	log_string_field_truncated	Test::LOG	F	zeek	-
+#close XXXX-XX-XX-XX-XX-XX

testing/btest/Baseline/scripts.base.frameworks.logging.field-length-limiting-4/.stdout

@@ -1,3 +1,5 @@
 ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+Telemetry::COUNTER, zeek, zeek_log_writer_truncated_string_fields_total, [filter_name, module, path, stream, writer], [default, Weird, weird, Weird::LOG, Log::WRITER_ASCII], 0.0
 Telemetry::COUNTER, zeek, zeek_log_writer_truncated_string_fields_total, [filter_name, module, path, stream, writer], [default, Test, test, Test::LOG, Log::WRITER_ASCII], 0.0
+Telemetry::COUNTER, zeek, zeek_log_writer_truncated_containers_total, [filter_name, module, path, stream, writer], [default, Weird, weird, Weird::LOG, Log::WRITER_ASCII], 0.0
 Telemetry::COUNTER, zeek, zeek_log_writer_truncated_containers_total, [filter_name, module, path, stream, writer], [default, Test, test, Test::LOG, Log::WRITER_ASCII], 2.0

testing/btest/Baseline/scripts.base.frameworks.logging.field-length-limiting-4/weird.log

@@ -0,0 +1,11 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	weird
+#open XXXX-XX-XX-XX-XX-XX
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	name	addl	notice	peer	source
+#types	time	string	addr	port	addr	port	string	string	bool	string	string
+XXXXXXXXXX.XXXXXX	-	-	-	-	-	log_container_field_truncated	Test::LOG	F	zeek	-
+#close XXXX-XX-XX-XX-XX-XX

testing/btest/Baseline/scripts.base.frameworks.logging.field-length-limiting-5/.stdout

@@ -1,3 +1,5 @@
 ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+Telemetry::COUNTER, zeek, zeek_log_writer_truncated_string_fields_total, [filter_name, module, path, stream, writer], [default, Weird, weird, Weird::LOG, Log::WRITER_ASCII], 0.0
 Telemetry::COUNTER, zeek, zeek_log_writer_truncated_string_fields_total, [filter_name, module, path, stream, writer], [default, Test, test, Test::LOG, Log::WRITER_ASCII], 0.0
+Telemetry::COUNTER, zeek, zeek_log_writer_truncated_containers_total, [filter_name, module, path, stream, writer], [default, Weird, weird, Weird::LOG, Log::WRITER_ASCII], 0.0
 Telemetry::COUNTER, zeek, zeek_log_writer_truncated_containers_total, [filter_name, module, path, stream, writer], [default, Test, test, Test::LOG, Log::WRITER_ASCII], 1.0

testing/btest/Baseline/scripts.base.frameworks.logging.field-length-limiting-5/weird.log

@@ -0,0 +1,11 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	weird
+#open XXXX-XX-XX-XX-XX-XX
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	name	addl	notice	peer	source
+#types	time	string	addr	port	addr	port	string	string	bool	string	string
+XXXXXXXXXX.XXXXXX	-	-	-	-	-	log_container_field_truncated	Test::LOG	F	zeek	-
+#close XXXX-XX-XX-XX-XX-XX

testing/btest/Baseline/scripts.base.frameworks.logging.field-length-limiting-6/.stdout

@@ -1,3 +1,5 @@
 ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+Telemetry::COUNTER, zeek, zeek_log_writer_truncated_string_fields_total, [filter_name, module, path, stream, writer], [default, Weird, weird, Weird::LOG, Log::WRITER_ASCII], 0.0
 Telemetry::COUNTER, zeek, zeek_log_writer_truncated_string_fields_total, [filter_name, module, path, stream, writer], [default, Test, test, Test::LOG, Log::WRITER_ASCII], 0.0
+Telemetry::COUNTER, zeek, zeek_log_writer_truncated_containers_total, [filter_name, module, path, stream, writer], [default, Weird, weird, Weird::LOG, Log::WRITER_ASCII], 0.0
 Telemetry::COUNTER, zeek, zeek_log_writer_truncated_containers_total, [filter_name, module, path, stream, writer], [default, Test, test, Test::LOG, Log::WRITER_ASCII], 2.0

testing/btest/Baseline/scripts.base.frameworks.logging.field-length-limiting-6/weird.log

@@ -0,0 +1,11 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	weird
+#open XXXX-XX-XX-XX-XX-XX
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	name	addl	notice	peer	source
+#types	time	string	addr	port	addr	port	string	string	bool	string	string
+XXXXXXXXXX.XXXXXX	-	-	-	-	-	log_container_field_truncated	Test::LOG	F	zeek	-
+#close XXXX-XX-XX-XX-XX-XX

testing/btest/Baseline/scripts.base.frameworks.logging.field-length-limiting/.stdout

@@ -1,3 +1,5 @@
 ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+Telemetry::COUNTER, zeek, zeek_log_writer_truncated_string_fields_total, [filter_name, module, path, stream, writer], [default, Weird, weird, Weird::LOG, Log::WRITER_ASCII], 2.0
 Telemetry::COUNTER, zeek, zeek_log_writer_truncated_string_fields_total, [filter_name, module, path, stream, writer], [default, Test, test, Test::LOG, Log::WRITER_ASCII], 20.0
+Telemetry::COUNTER, zeek, zeek_log_writer_truncated_containers_total, [filter_name, module, path, stream, writer], [default, Weird, weird, Weird::LOG, Log::WRITER_ASCII], 0.0
 Telemetry::COUNTER, zeek, zeek_log_writer_truncated_containers_total, [filter_name, module, path, stream, writer], [default, Test, test, Test::LOG, Log::WRITER_ASCII], 0.0

testing/btest/Baseline/scripts.base.frameworks.logging.field-length-limiting/weird.log

@@ -0,0 +1,11 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	weird
+#open XXXX-XX-XX-XX-XX-XX
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	name	addl	notice	peer	source
+#types	time	string	addr	port	addr	port	string	string	bool	string	string
+XXXXXXXXXX.XXXXXX	-	-	-	-	-	log_s	Test:	F	zeek	-
+#close XXXX-XX-XX-XX-XX-XX

testing/btest/scripts/base/frameworks/logging/field-length-limiting.zeek

@@ -3,10 +3,12 @@
 # @TEST-EXEC: zeek -b test.zeek %INPUT
 # @TEST-EXEC: btest-diff test.log
 # @TEST-EXEC: btest-diff .stdout
+# @TEST-EXEC: btest-diff weird.log
 
 # @TEST-START-FILE test.zeek
 
 @load base/frameworks/telemetry
+@load base/frameworks/notice/weird
 
 module Test;
 
@@ -19,6 +21,16 @@ export {
 	};
 }
 
+event log_telemetry()
+	{
+	local storage_metrics = Telemetry::collect_metrics("zeek", "log_writer_truncated*");
+	for (i in storage_metrics)
+		{
+		local m = storage_metrics[i];
+		print m$opts$metric_type, m$opts$prefix, m$opts$name, m$label_names, m$label_values, m$value;
+		}
+	}
+
 event zeek_init()
 	{
 	Log::create_stream(LOG, [$columns=Info, $path="test"]);
@@ -36,19 +48,17 @@ event zeek_init()
 
 	Log::write(Test::LOG, rec);
 
-	local storage_metrics = Telemetry::collect_metrics("zeek", "log_writer_truncated*");
+	# Do this as a separate event so the weirds get processed before we log the
-	for (i in storage_metrics)
+	# telemetry outout. See the comment below for the first test as to why.
-		{
+	event log_telemetry();
-		local m = storage_metrics[i];
-		print m$opts$metric_type, m$opts$prefix, m$opts$name, m$label_names, m$label_values, m$value;
-		}
 	}
 
-
 # @TEST-END-FILE test.zeek
 
 # Limit the individual fields to 5 bytes, but keep the total maximum large enough that it
-# will write all of the fields.
+# will write all of the fields. The weird test for this one will be off since it will
+# limit the name of the weird. It will pass, but the fields in the log will get truncated
+# like they're supposed to.
 redef Log::max_field_string_bytes = 5;
 
 # @TEST-START-NEXT