diff --git a/scripts/base/frameworks/logging/writers/ascii.zeek b/scripts/base/frameworks/logging/writers/ascii.zeek index ce37a32537..bbeb94c172 100644 --- a/scripts/base/frameworks/logging/writers/ascii.zeek +++ b/scripts/base/frameworks/logging/writers/ascii.zeek @@ -66,6 +66,11 @@ export { ## This option is also available as a per-filter ``$config`` option. const json_timestamps: JSON::TimestampFormat = JSON::TS_EPOCH &redef; + ## Handling of optional fields when writing out JSON. By default the + ## JSON formatter skips key and val when the field is absent. Setting + ## the following field to T includes the key, with a null value. + const json_include_unset_fields = F &redef; + ## If true, include lines with log meta information such as column names ## with types, the values of ASCII logging options that are in use, and ## the time when the file was opened and closed (the latter at the end). diff --git a/src/logging/writers/ascii/Ascii.cc b/src/logging/writers/ascii/Ascii.cc index d70e23db99..0b189c75b5 100644 --- a/src/logging/writers/ascii/Ascii.cc +++ b/src/logging/writers/ascii/Ascii.cc @@ -197,6 +197,7 @@ Ascii::Ascii(WriterFrontend* frontend) : WriterBackend(frontend) tsv = false; use_json = false; enable_utf_8 = false; + json_include_unset_fields = false; formatter = nullptr; gzip_level = 0; gzfile = nullptr; @@ -232,6 +233,8 @@ void Ascii::InitConfigOptions() BifConst::LogAscii::json_timestamps->Describe(&tsfmt); json_timestamps.assign((const char*)tsfmt.Bytes(), tsfmt.Len()); + json_include_unset_fields = BifConst::LogAscii::json_include_unset_fields; + gzip_file_extension.assign((const char*)BifConst::LogAscii::gzip_file_extension->Bytes(), BifConst::LogAscii::gzip_file_extension->Len()); @@ -329,6 +332,20 @@ bool Ascii::InitFilterOptions() else if ( strcmp(i->first, "json_timestamps") == 0 ) json_timestamps.assign(i->second); + else if ( strcmp(i->first, "json_include_unset_fields") == 0 ) + { + if ( strcmp(i->second, "T") == 0 ) + json_include_unset_fields = true; + else if ( strcmp(i->second, "F") == 0 ) + json_include_unset_fields = false; + else + { + Error("invalid value for 'json_include_unset_fields', must be " + "a string and either \"T\" or \"F\""); + return false; + } + } + else if ( strcmp(i->first, "gzip_file_extension") == 0 ) gzip_file_extension.assign(i->second); @@ -364,7 +381,7 @@ bool Ascii::InitFormatter() return false; } - formatter = new threading::formatter::JSON(this, tf); + formatter = new threading::formatter::JSON(this, tf, json_include_unset_fields); // Using JSON implicitly turns off the header meta fields. include_meta = false; } diff --git a/src/logging/writers/ascii/Ascii.h b/src/logging/writers/ascii/Ascii.h index ae5afff999..b7eb1dc1d8 100644 --- a/src/logging/writers/ascii/Ascii.h +++ b/src/logging/writers/ascii/Ascii.h @@ -78,6 +78,7 @@ private: bool use_json; bool enable_utf_8; std::string json_timestamps; + bool json_include_unset_fields; std::string logdir; threading::Formatter* formatter; diff --git a/src/logging/writers/ascii/ascii.bif b/src/logging/writers/ascii/ascii.bif index c7d30ad531..38932ded36 100644 --- a/src/logging/writers/ascii/ascii.bif +++ b/src/logging/writers/ascii/ascii.bif @@ -14,6 +14,7 @@ const use_json: bool; const enable_leftover_log_rotation: bool; const enable_utf_8: bool; const json_timestamps: JSON::TimestampFormat; +const json_include_unset_fields: bool; const gzip_level: count; const gzip_file_extension: string; const logdir: string; diff --git a/src/threading/formatters/JSON.cc b/src/threading/formatters/JSON.cc index 5e3449cbdc..774b231f39 100644 --- a/src/threading/formatters/JSON.cc +++ b/src/threading/formatters/JSON.cc @@ -28,7 +28,8 @@ bool JSON::NullDoubleWriter::Double(double d) return rapidjson::Writer::Double(d); } -JSON::JSON(MsgThread* t, TimeFormat tf) : Formatter(t), surrounding_braces(true) +JSON::JSON(MsgThread* t, TimeFormat tf, bool arg_include_unset_fields) + : Formatter(t), surrounding_braces(true), include_unset_fields(arg_include_unset_fields) { timestamps = tf; } @@ -44,7 +45,7 @@ bool JSON::Describe(ODesc* desc, int num_fields, const Field* const* fields, Val for ( int i = 0; i < num_fields; i++ ) { - if ( vals[i]->present ) + if ( vals[i]->present || include_unset_fields ) BuildJSON(writer, vals[i], fields[i]->name); } @@ -62,7 +63,7 @@ bool JSON::Describe(ODesc* desc, Value* val, const std::string& name) const return false; } - if ( ! val->present || name.empty() ) + if ( (! val->present && ! include_unset_fields) || name.empty() ) return true; rapidjson::Document doc; @@ -86,15 +87,15 @@ Value* JSON::ParseValue(const std::string& s, const std::string& name, TypeTag t void JSON::BuildJSON(NullDoubleWriter& writer, Value* val, const std::string& name) const { + if ( ! name.empty() ) + writer.Key(name); + if ( ! val->present ) { writer.Null(); return; } - if ( ! name.empty() ) - writer.Key(name); - switch ( val->type ) { case TYPE_BOOL: diff --git a/src/threading/formatters/JSON.h b/src/threading/formatters/JSON.h index 0b6ae12b81..c946ab7bf2 100644 --- a/src/threading/formatters/JSON.h +++ b/src/threading/formatters/JSON.h @@ -26,7 +26,7 @@ public: // elasticsearch). }; - JSON(MsgThread* t, TimeFormat tf); + JSON(MsgThread* t, TimeFormat tf, bool include_unset_fields = false); ~JSON() override; bool Describe(ODesc* desc, Value* val, const std::string& name = "") const override; @@ -50,6 +50,7 @@ private: TimeFormat timestamps; bool surrounding_braces; + bool include_unset_fields; }; } // namespace zeek::threading::formatter diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-json-optional/testing_nullfields.log b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-json-optional/testing_nullfields.log new file mode 100644 index 0000000000..90269d5703 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-json-optional/testing_nullfields.log @@ -0,0 +1,2 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +{"ts":null,"msg":"Testing 1 2 3 "} diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-json-optional/testing_nullfields_via_config.log b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-json-optional/testing_nullfields_via_config.log new file mode 100644 index 0000000000..90269d5703 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-json-optional/testing_nullfields_via_config.log @@ -0,0 +1,2 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +{"ts":null,"msg":"Testing 1 2 3 "} diff --git a/testing/btest/scripts/base/frameworks/logging/ascii-json-optional.zeek b/testing/btest/scripts/base/frameworks/logging/ascii-json-optional.zeek index ec86557c4a..6ef40cae28 100644 --- a/testing/btest/scripts/base/frameworks/logging/ascii-json-optional.zeek +++ b/testing/btest/scripts/base/frameworks/logging/ascii-json-optional.zeek @@ -1,10 +1,20 @@ +# This test verifies the behavior of the JSON writer regarding unset optional +# values. By default, such fields are skipped, while redef'ing +# LogAscii::json_include_unset_fields=T or using a filter's config table to set a +# field of the same name includes them with a null value. # # @TEST-EXEC: zeek -b %INPUT # @TEST-EXEC: btest-diff testing.log +# +# @TEST-EXEC: zeek -b %INPUT LogAscii::json_include_unset_fields=T Testing::logname=testing_nullfields +# @TEST-EXEC: btest-diff testing_nullfields.log +# +# @TEST-EXEC: zeek -b %INPUT Testing::use_config_table=T Testing::logname=testing_nullfields_via_config +# @TEST-EXEC: btest-diff testing_nullfields_via_config.log @load tuning/json-logs -module testing; +module Testing; export { redef enum Log::ID += { LOG }; @@ -15,13 +25,23 @@ export { }; global log_test: event(rec: Info); + + const logname = "testing" &redef; + const use_config_table = F &redef; } event zeek_init() &priority=5 { - Log::create_stream(testing::LOG, [$columns=testing::Info, $ev=log_test]); + Log::create_stream(LOG, [$columns=Info, $ev=log_test, $path=logname]); + + if ( use_config_table ) + { + local f = Log::get_filter(LOG, "default"); + f$config = table(["json_include_unset_fields"] = "T"); + Log::add_filter(LOG, f); + } + local info: Info; info$msg = "Testing 1 2 3 "; - Log::write(testing::LOG, info); + Log::write(LOG, info); } -