diff --git a/scripts/base/frameworks/logging/writers/ascii.bro b/scripts/base/frameworks/logging/writers/ascii.bro index 68219cb964..2a887e64c0 100644 --- a/scripts/base/frameworks/logging/writers/ascii.bro +++ b/scripts/base/frameworks/logging/writers/ascii.bro @@ -25,10 +25,8 @@ export { const use_json = F &redef; ## By default, the JSON formatter will use double values for timestamps - ## which represent the number of seconds from the UNIX epoch. By setting - ## this to 'T', it will use the 8601 format. This is also available as - ## a per-filter $config option. - const json_iso_timestamps = F &redef; + ## which represent the number of seconds from the UNIX epoch. + const json_timestamps: JSON::TimestampFormat = JSON::TS_EPOCH &redef; ## If true, include lines with log meta information such as column names ## with types, the values of ASCII logging options that are in use, and diff --git a/scripts/base/init-bare.bro b/scripts/base/init-bare.bro index afb89ba1c7..0af4f0fb9d 100644 --- a/scripts/base/init-bare.bro +++ b/scripts/base/init-bare.bro @@ -3057,6 +3057,24 @@ const record_all_packets = F &redef; ## .. bro:see:: conn_stats const ignore_keep_alive_rexmit = F &redef; +module JSON; +export { + type TimestampFormat: enum { + ## Timestamps will be formatted as UNIX epoch doubles. This is + ## the format that Bro typically writes out timestamps. + TS_EPOCH, + ## Timestamps will be formatted as unsigned integers that + ## represent the number of milliseconds since the UNIX + ## epoch. + TS_MILLIS, + ## Timestamps will be formatted in the ISO8601 DateTime format. + ## Subseconds are also included which isn't actually part of the + ## standard but most things that parse ISO8601 seem to be able + ## to cope with that. + TS_ISO8601, + }; +} + module Tunnel; export { ## The maximum depth of a tunnel to decapsulate until giving up. diff --git a/src/logging.bif b/src/logging.bif index f12aa50c1c..062e4dbe31 100644 --- a/src/logging.bif +++ b/src/logging.bif @@ -78,7 +78,7 @@ const set_separator: string; const empty_field: string; const unset_field: string; const use_json: bool; -const json_iso_timestamps: bool; +const json_timestamps: JSON::TimestampFormat; # Options for the DataSeries writer. diff --git a/src/logging/writers/Ascii.cc b/src/logging/writers/Ascii.cc index 976c083eda..e0d1293b70 100644 --- a/src/logging/writers/Ascii.cc +++ b/src/logging/writers/Ascii.cc @@ -11,6 +11,7 @@ #include "Ascii.h" using namespace logging::writer; +using namespace threading; using threading::Value; using threading::Field; @@ -59,7 +60,6 @@ bool Ascii::DoInit(const WriterInfo& info, int num_fields, const Field* const * output_to_stdout = BifConst::LogAscii::output_to_stdout; include_meta = BifConst::LogAscii::include_meta; use_json = BifConst::LogAscii::use_json; - json_iso_timestamps = BifConst::LogAscii::json_iso_timestamps; separator.assign( (const char*) BifConst::LogAscii::separator->Bytes(), @@ -86,6 +86,13 @@ bool Ascii::DoInit(const WriterInfo& info, int num_fields, const Field* const * BifConst::LogAscii::meta_prefix->Len() ); + ODesc tsfmt; + BifConst::LogAscii::json_timestamps->Describe(&tsfmt); + json_timestamps.assign( + (const char*) tsfmt.Bytes(), + tsfmt.Len() + ); + // Set per-filter configuration options. for ( WriterInfo::config_map::const_iterator i = info.config.begin(); i != info.config.end(); i++ ) { @@ -142,13 +149,28 @@ bool Ascii::DoInit(const WriterInfo& info, int num_fields, const Field* const * else if ( strcmp(i->first, "meta_prefix") == 0 ) meta_prefix.assign(i->second); + + else if ( strcmp(i->first, "json_timestamps") == 0 ) + json_timestamps.assign(i->second); } - if ( use_json ) { + formatter::JSON::TimeFormat tf = formatter::JSON::TS_EPOCH; // Write out JSON formatted logs. - formatter = new threading::formatter::JSON(this, json_iso_timestamps); + if ( strcmp(json_timestamps.c_str(), "JSON::TS_EPOCH") == 0 ) + tf = formatter::JSON::TS_EPOCH; + else if ( strcmp(json_timestamps.c_str(), "JSON::TS_MILLIS") == 0 ) + tf = formatter::JSON::TS_MILLIS; + else if ( strcmp(json_timestamps.c_str(), "JSON::TS_ISO8601") == 0 ) + tf = formatter::JSON::TS_ISO8601; + else + { + Error(Fmt("Invalid JSON timestamp format: %s", json_timestamps.c_str())); + return false; + } + + formatter = new formatter::JSON(this, tf); // Using JSON implicitly turns off the header meta fields. include_meta = false; } @@ -157,7 +179,7 @@ bool Ascii::DoInit(const WriterInfo& info, int num_fields, const Field* const * // Use the default "Bro logs" format. desc.EnableEscaping(); desc.AddEscapeSequence(separator); - formatter = new threading::formatter::Ascii(this, threading::formatter::Ascii::SeparatorInfo(separator, set_separator, unset_field, empty_field)); + formatter = new formatter::Ascii(this, formatter::Ascii::SeparatorInfo(separator, set_separator, unset_field, empty_field)); } string path = info.path; diff --git a/src/logging/writers/Ascii.h b/src/logging/writers/Ascii.h index d131a289b4..a70f3c2c5e 100644 --- a/src/logging/writers/Ascii.h +++ b/src/logging/writers/Ascii.h @@ -47,14 +47,15 @@ private: bool output_to_stdout; bool include_meta; bool tsv; - bool use_json; - bool json_iso_timestamps; string separator; string set_separator; string empty_field; string unset_field; string meta_prefix; + + bool use_json; + string json_timestamps; threading::formatter::Formatter* formatter; }; diff --git a/src/logging/writers/ElasticSearch.cc b/src/logging/writers/ElasticSearch.cc index 0a18e624ea..45549cb565 100644 --- a/src/logging/writers/ElasticSearch.cc +++ b/src/logging/writers/ElasticSearch.cc @@ -52,7 +52,7 @@ ElasticSearch::ElasticSearch(WriterFrontend* frontend) : WriterBackend(frontend) curl_handle = HTTPSetup(); - json = new threading::formatter::JSON(this, false); + json = new threading::formatter::JSON(this, threading::formatter::JSON::TS_MILLIS); } ElasticSearch::~ElasticSearch() diff --git a/src/threading/formatters/JSON.cc b/src/threading/formatters/JSON.cc index 57e9d58527..0523d38f8b 100644 --- a/src/threading/formatters/JSON.cc +++ b/src/threading/formatters/JSON.cc @@ -10,9 +10,9 @@ using namespace threading::formatter; -JSON::JSON(MsgThread* t, bool json_iso_timestamps) : Formatter(t) +JSON::JSON(MsgThread* t, TimeFormat tf) : Formatter(t) { - iso_timestamps = json_iso_timestamps; + timestamps = tf; } JSON::~JSON() @@ -102,7 +102,7 @@ bool JSON::Describe(ODesc* desc, Value* val) const case TYPE_TIME: { - if ( iso_timestamps ) + if ( timestamps == TS_ISO8601 ) { char buffer[40]; time_t t = time_t(val->val.double_val); @@ -118,14 +118,18 @@ bool JSON::Describe(ODesc* desc, Value* val) const desc->AddRaw("\"", 1); } } - else + else if ( timestamps == TS_EPOCH ) + { + desc->Add(val->val.double_val); + } + else if ( timestamps == TS_MILLIS ) { // ElasticSearch uses milliseconds for timestamps and json only // supports signed ints (uints can be too large). uint64_t ts = (uint64_t) (val->val.double_val * 1000); if ( ts >= INT64_MAX ) { - thread->Error(thread->Fmt("time value too large for JSON: %" PRIu64, ts)); + thread->Error(thread->Fmt("time value too large for JSON milliseconds: %" PRIu64, ts)); desc->AddRaw("null", 4); } else diff --git a/src/threading/formatters/JSON.h b/src/threading/formatters/JSON.h index b15da075cd..d21a705ce1 100644 --- a/src/threading/formatters/JSON.h +++ b/src/threading/formatters/JSON.h @@ -13,7 +13,14 @@ namespace threading { namespace formatter { */ class JSON : public Formatter { public: - JSON(threading::MsgThread* t, bool json_iso_timestamps); + + enum TimeFormat { + TS_EPOCH, // Doubles that represents seconds from the UNIX epoch. + TS_ISO8601, // ISO 8601 defined human readable timestamp format. + TS_MILLIS // Milliseconds from the UNIX epoch. Some things need this (elasticsearch). + }; + + JSON(threading::MsgThread* t, TimeFormat tf); virtual ~JSON(); virtual bool Describe(ODesc* desc, threading::Value* val) const; @@ -23,7 +30,7 @@ public: virtual threading::Value* ParseValue(string s, string name, TypeTag type, TypeTag subtype = TYPE_ERROR) const; private: - bool iso_timestamps; + TimeFormat timestamps; }; }}