Expanded support for modifying the timestamp format in the JSON formatter.

scripts/base/frameworks/logging/writers/ascii.bro

@@ -25,10 +25,8 @@ export {
 	const use_json = F &redef;
 
 	## By default, the JSON formatter will use double values for timestamps
-	## which represent the number of seconds from the UNIX epoch.  By setting
+	## which represent the number of seconds from the UNIX epoch.
-	## this to 'T', it will use the 8601 format.  This is also available as 
+	const json_timestamps: JSON::TimestampFormat = JSON::TS_EPOCH &redef;
-	## a per-filter $config option.
-	const json_iso_timestamps = F &redef;
 
 	## If true, include lines with log meta information such as column names
 	## with types, the values of ASCII logging options that are in use, and

scripts/base/init-bare.bro

@@ -3057,6 +3057,24 @@ const record_all_packets = F &redef;
 ## .. bro:see:: conn_stats
 const ignore_keep_alive_rexmit = F &redef;
 
+module JSON;
+export {
+	type TimestampFormat: enum {
+		## Timestamps will be formatted as UNIX epoch doubles.  This is
+		## the format that Bro typically writes out timestamps.
+		TS_EPOCH,
+		## Timestamps will be formatted as unsigned integers that 
+		## represent the number of milliseconds since the UNIX
+		## epoch.
+		TS_MILLIS,
+		## Timestamps will be formatted in the ISO8601 DateTime format.
+		## Subseconds are also included which isn't actually part of the
+		## standard but most things that parse ISO8601 seem to be able
+		## to cope with that.
+		TS_ISO8601,
+	};
+}
+
 module Tunnel;
 export {
 	## The maximum depth of a tunnel to decapsulate until giving up.

src/logging.bif

@@ -78,7 +78,7 @@ const set_separator: string;
 const empty_field: string;
 const unset_field: string;
 const use_json: bool;
-const json_iso_timestamps: bool;
+const json_timestamps: JSON::TimestampFormat;
 
 # Options for the DataSeries writer.
 

src/logging/writers/Ascii.cc

@@ -11,6 +11,7 @@
 #include "Ascii.h"
 
 using namespace logging::writer;
+using namespace threading;
 using threading::Value;
 using threading::Field;
 
@@ -59,7 +60,6 @@ bool Ascii::DoInit(const WriterInfo& info, int num_fields, const Field* const *
 	output_to_stdout = BifConst::LogAscii::output_to_stdout;
 	include_meta = BifConst::LogAscii::include_meta;
 	use_json = BifConst::LogAscii::use_json;
-	json_iso_timestamps = BifConst::LogAscii::json_iso_timestamps;
 
 	separator.assign(
 			(const char*) BifConst::LogAscii::separator->Bytes(),
@@ -86,6 +86,13 @@ bool Ascii::DoInit(const WriterInfo& info, int num_fields, const Field* const *
 			BifConst::LogAscii::meta_prefix->Len()
 			);
 
+	ODesc tsfmt;
+	BifConst::LogAscii::json_timestamps->Describe(&tsfmt);
+	json_timestamps.assign(
+			(const char*) tsfmt.Bytes(),
+			tsfmt.Len()
+			);
+
 	// Set per-filter configuration options.
 	for ( WriterInfo::config_map::const_iterator i = info.config.begin(); i != info.config.end(); i++ )
 		{
@@ -142,13 +149,28 @@ bool Ascii::DoInit(const WriterInfo& info, int num_fields, const Field* const *
 
 		else if ( strcmp(i->first, "meta_prefix") == 0 )
 			meta_prefix.assign(i->second);
-		}
 		
+		else if ( strcmp(i->first, "json_timestamps") == 0 )
+			json_timestamps.assign(i->second);
+		}
 
 	if ( use_json )
 		{
+		formatter::JSON::TimeFormat tf = formatter::JSON::TS_EPOCH;
 		// Write out JSON formatted logs.
-		formatter = new threading::formatter::JSON(this, json_iso_timestamps);
+		if ( strcmp(json_timestamps.c_str(), "JSON::TS_EPOCH") == 0 )
+			tf = formatter::JSON::TS_EPOCH;
+		else if ( strcmp(json_timestamps.c_str(), "JSON::TS_MILLIS") == 0 )
+			tf = formatter::JSON::TS_MILLIS;
+		else if ( strcmp(json_timestamps.c_str(), "JSON::TS_ISO8601") == 0 )
+			tf = formatter::JSON::TS_ISO8601;
+		else
+			{
+			Error(Fmt("Invalid JSON timestamp format: %s", json_timestamps.c_str()));
+			return false;
+			}
+
+		formatter = new formatter::JSON(this, tf);
 		// Using JSON implicitly turns off the header meta fields.
 		include_meta = false;
 		}
@@ -157,7 +179,7 @@ bool Ascii::DoInit(const WriterInfo& info, int num_fields, const Field* const *
 		// Use the default "Bro logs" format.
 		desc.EnableEscaping();
 		desc.AddEscapeSequence(separator);
-		formatter = new threading::formatter::Ascii(this, threading::formatter::Ascii::SeparatorInfo(separator, set_separator, unset_field, empty_field));
+		formatter = new formatter::Ascii(this, formatter::Ascii::SeparatorInfo(separator, set_separator, unset_field, empty_field));
 		}
 
 	string path = info.path;

src/logging/writers/Ascii.h

@@ -47,8 +47,6 @@ private:
 	bool output_to_stdout;
 	bool include_meta;
 	bool tsv;
-	bool use_json;
-	bool json_iso_timestamps;
 
 	string separator;
 	string set_separator;
@@ -56,6 +54,9 @@ private:
 	string unset_field;
 	string meta_prefix;
 	
+	bool use_json;
+	string json_timestamps;
+
 	threading::formatter::Formatter* formatter;
 };
 

src/logging/writers/ElasticSearch.cc

@@ -52,7 +52,7 @@ ElasticSearch::ElasticSearch(WriterFrontend* frontend) : WriterBackend(frontend)
 
 	curl_handle = HTTPSetup();
 
-	json = new threading::formatter::JSON(this, false);
+	json = new threading::formatter::JSON(this, threading::formatter::JSON::TS_MILLIS);
 }
 
 ElasticSearch::~ElasticSearch()

src/threading/formatters/JSON.cc

@@ -10,9 +10,9 @@
 
 using namespace threading::formatter;
 
-JSON::JSON(MsgThread* t, bool json_iso_timestamps) : Formatter(t)
+JSON::JSON(MsgThread* t, TimeFormat tf) : Formatter(t)
 	{
-	iso_timestamps = json_iso_timestamps;
+	timestamps = tf;
 	}
 
 JSON::~JSON()
@@ -102,7 +102,7 @@ bool JSON::Describe(ODesc* desc, Value* val) const
 
 		case TYPE_TIME:
 			{
-			if ( iso_timestamps )
+			if ( timestamps == TS_ISO8601 )
 				{
 				char buffer[40];
 				time_t t = time_t(val->val.double_val);
@@ -118,14 +118,18 @@ bool JSON::Describe(ODesc* desc, Value* val) const
 					desc->AddRaw("\"", 1);
 					}
 				}
-			else
+			else if ( timestamps == TS_EPOCH )
+				{
+				desc->Add(val->val.double_val);
+				}
+			else if ( timestamps == TS_MILLIS )
 				{
 				// ElasticSearch uses milliseconds for timestamps and json only
 				// supports signed ints (uints can be too large).
 				uint64_t ts = (uint64_t) (val->val.double_val * 1000);
 				if ( ts >= INT64_MAX )
 					{
-					thread->Error(thread->Fmt("time value too large for JSON: %" PRIu64, ts));
+					thread->Error(thread->Fmt("time value too large for JSON milliseconds: %" PRIu64, ts));
 					desc->AddRaw("null", 4);
 					}
 				else

src/threading/formatters/JSON.h

@@ -13,7 +13,14 @@ namespace threading { namespace formatter {
   */
 class JSON : public Formatter {
 public:
-	JSON(threading::MsgThread* t, bool json_iso_timestamps);
+
+	enum TimeFormat {
+		TS_EPOCH,   // Doubles that represents seconds from the UNIX epoch.
+		TS_ISO8601, // ISO 8601 defined human readable timestamp format.
+		TS_MILLIS   // Milliseconds from the UNIX epoch.  Some things need this (elasticsearch).
+		};
+
+	JSON(threading::MsgThread* t, TimeFormat tf);
 	virtual ~JSON();
 
 	virtual bool Describe(ODesc* desc, threading::Value* val) const;
@@ -23,7 +30,7 @@ public:
 	virtual threading::Value* ParseValue(string s, string name, TypeTag type, TypeTag subtype = TYPE_ERROR) const;
 
 private:
-	bool iso_timestamps;
+	TimeFormat timestamps;
 };
 
 }}