Add LogAscii::json_include_unset_fields flag to control unset field rendering

The flag controls whether JSON rendering includes unset &optional log fields
(F, the default), or includes them with a null value (T).

scripts/base/frameworks/logging/writers/ascii.zeek

@@ -66,6 +66,11 @@ export {
 	## This option is also available as a per-filter ``$config`` option.
 	const json_timestamps: JSON::TimestampFormat = JSON::TS_EPOCH &redef;
 
+	## Handling of optional fields when writing out JSON. By default the
+	## JSON formatter skips key and val when the field is absent. Setting
+	## the following field to T includes the key, with a null value.
+	const json_include_unset_fields = F &redef;
+
 	## If true, include lines with log meta information such as column names
 	## with types, the values of ASCII logging options that are in use, and
 	## the time when the file was opened and closed (the latter at the end).

src/logging/writers/ascii/Ascii.cc

@@ -197,6 +197,7 @@ Ascii::Ascii(WriterFrontend* frontend) : WriterBackend(frontend)
 	tsv = false;
 	use_json = false;
 	enable_utf_8 = false;
+	json_include_unset_fields = false;
 	formatter = nullptr;
 	gzip_level = 0;
 	gzfile = nullptr;
@@ -232,6 +233,8 @@ void Ascii::InitConfigOptions()
 	BifConst::LogAscii::json_timestamps->Describe(&tsfmt);
 	json_timestamps.assign((const char*)tsfmt.Bytes(), tsfmt.Len());
 
+	json_include_unset_fields = BifConst::LogAscii::json_include_unset_fields;
+
 	gzip_file_extension.assign((const char*)BifConst::LogAscii::gzip_file_extension->Bytes(),
 	                           BifConst::LogAscii::gzip_file_extension->Len());
 
@@ -329,6 +332,20 @@ bool Ascii::InitFilterOptions()
 		else if ( strcmp(i->first, "json_timestamps") == 0 )
 			json_timestamps.assign(i->second);
 
+		else if ( strcmp(i->first, "json_include_unset_fields") == 0 )
+			{
+			if ( strcmp(i->second, "T") == 0 )
+				json_include_unset_fields = true;
+			else if ( strcmp(i->second, "F") == 0 )
+				json_include_unset_fields = false;
+			else
+				{
+				Error("invalid value for 'json_include_unset_fields', must be "
+				      "a string and either \"T\" or \"F\"");
+				return false;
+				}
+			}
+
 		else if ( strcmp(i->first, "gzip_file_extension") == 0 )
 			gzip_file_extension.assign(i->second);
 
@@ -364,7 +381,7 @@ bool Ascii::InitFormatter()
 			return false;
 			}
 
-		formatter = new threading::formatter::JSON(this, tf);
+		formatter = new threading::formatter::JSON(this, tf, json_include_unset_fields);
 		// Using JSON implicitly turns off the header meta fields.
 		include_meta = false;
 		}

src/logging/writers/ascii/Ascii.h

@@ -78,6 +78,7 @@ private:
 	bool use_json;
 	bool enable_utf_8;
 	std::string json_timestamps;
+	bool json_include_unset_fields;
 	std::string logdir;
 
 	threading::Formatter* formatter;

src/logging/writers/ascii/ascii.bif

@@ -14,6 +14,7 @@ const use_json: bool;
 const enable_leftover_log_rotation: bool;
 const enable_utf_8: bool;
 const json_timestamps: JSON::TimestampFormat;
+const json_include_unset_fields: bool;
 const gzip_level: count;
 const gzip_file_extension: string;
 const logdir: string;

src/threading/formatters/JSON.cc

@@ -28,7 +28,8 @@ bool JSON::NullDoubleWriter::Double(double d)
 	return rapidjson::Writer<rapidjson::StringBuffer>::Double(d);
 	}
 
-JSON::JSON(MsgThread* t, TimeFormat tf) : Formatter(t), surrounding_braces(true)
+JSON::JSON(MsgThread* t, TimeFormat tf, bool arg_include_unset_fields)
+	: Formatter(t), surrounding_braces(true), include_unset_fields(arg_include_unset_fields)
 	{
 	timestamps = tf;
 	}
@@ -44,7 +45,7 @@ bool JSON::Describe(ODesc* desc, int num_fields, const Field* const* fields, Val
 
 	for ( int i = 0; i < num_fields; i++ )
 		{
-		if ( vals[i]->present )
+		if ( vals[i]->present || include_unset_fields )
 			BuildJSON(writer, vals[i], fields[i]->name);
 		}
 
@@ -62,7 +63,7 @@ bool JSON::Describe(ODesc* desc, Value* val, const std::string& name) const
 		return false;
 		}
 
-	if ( ! val->present || name.empty() )
+	if ( (! val->present && ! include_unset_fields) || name.empty() )
 		return true;
 
 	rapidjson::Document doc;
@@ -86,15 +87,15 @@ Value* JSON::ParseValue(const std::string& s, const std::string& name, TypeTag t
 
 void JSON::BuildJSON(NullDoubleWriter& writer, Value* val, const std::string& name) const
 	{
+	if ( ! name.empty() )
+		writer.Key(name);
+
 	if ( ! val->present )
 		{
 		writer.Null();
 		return;
 		}
 
-	if ( ! name.empty() )
-		writer.Key(name);
-
 	switch ( val->type )
 		{
 		case TYPE_BOOL:

src/threading/formatters/JSON.h

@@ -26,7 +26,7 @@ public:
 		          // elasticsearch).
 		};
 
-	JSON(MsgThread* t, TimeFormat tf);
+	JSON(MsgThread* t, TimeFormat tf, bool include_unset_fields = false);
 	~JSON() override;
 
 	bool Describe(ODesc* desc, Value* val, const std::string& name = "") const override;
@@ -50,6 +50,7 @@ private:
 
 	TimeFormat timestamps;
 	bool surrounding_braces;
+	bool include_unset_fields;
 	};
 
 	} // namespace zeek::threading::formatter

testing/btest/Baseline/scripts.base.frameworks.logging.ascii-json-optional/testing_nullfields.log

@@ -0,0 +1,2 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+{"ts":null,"msg":"Testing 1 2 3 "}

testing/btest/Baseline/scripts.base.frameworks.logging.ascii-json-optional/testing_nullfields_via_config.log

@@ -0,0 +1,2 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+{"ts":null,"msg":"Testing 1 2 3 "}

testing/btest/scripts/base/frameworks/logging/ascii-json-optional.zeek

@@ -1,10 +1,20 @@
+# This test verifies the behavior of the JSON writer regarding unset optional
+# values.  By default, such fields are skipped, while redef'ing
+# LogAscii::json_include_unset_fields=T or using a filter's config table to set a
+# field of the same name includes them with a null value.
 #
 # @TEST-EXEC: zeek -b %INPUT
 # @TEST-EXEC: btest-diff testing.log
+#
+# @TEST-EXEC: zeek -b %INPUT LogAscii::json_include_unset_fields=T Testing::logname=testing_nullfields
+# @TEST-EXEC: btest-diff testing_nullfields.log
+#
+# @TEST-EXEC: zeek -b %INPUT Testing::use_config_table=T Testing::logname=testing_nullfields_via_config
+# @TEST-EXEC: btest-diff testing_nullfields_via_config.log
 
 @load tuning/json-logs
 
-module testing;
+module Testing;
 
 export {
     redef enum Log::ID += { LOG };
@@ -15,13 +25,23 @@ export {
     };
 
     global log_test: event(rec: Info);
+
+    const logname = "testing" &redef;
+    const use_config_table = F &redef;
 }
 
 event zeek_init() &priority=5
 {
-    Log::create_stream(testing::LOG, [$columns=testing::Info, $ev=log_test]);
+    Log::create_stream(LOG, [$columns=Info, $ev=log_test, $path=logname]);
+
+    if ( use_config_table )
+        {
+        local f = Log::get_filter(LOG, "default");
+        f$config = table(["json_include_unset_fields"] = "T");
+        Log::add_filter(LOG, f);
+        }
+
     local info: Info;
     info$msg = "Testing 1 2 3 ";
-   Log::write(testing::LOG, info);
+    Log::write(LOG, info);
 }
-