From a56c3437151985a1d0e4c881c047796109fbd81d Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Mon, 10 Mar 2014 10:42:59 -0400 Subject: [PATCH 1/5] Refactored formatters and updated the the writers a bit. - Formatters have been abstracted similarly to readers and writers now. - The Ascii writer has a new option for writing out logs as JSON. - The Ascii writer now has all options availble as per-filter options as well as global. --- .../base/frameworks/logging/writers/ascii.bro | 21 ++- scripts/policy/tuning/json-logs.bro | 4 + scripts/test-all-policy.bro | 1 + src/CMakeLists.txt | 4 +- src/input/readers/Ascii.cc | 20 +- src/input/readers/Ascii.h | 4 +- src/input/readers/Benchmark.cc | 2 +- src/input/readers/Benchmark.h | 4 +- src/input/readers/SQLite.cc | 2 +- src/input/readers/SQLite.h | 4 +- src/logging.bif | 1 + src/logging/writers/Ascii.cc | 177 +++++++++++------- src/logging/writers/Ascii.h | 6 +- src/logging/writers/ElasticSearch.cc | 140 +------------- src/logging/writers/ElasticSearch.h | 3 +- src/logging/writers/SQLite.cc | 2 +- src/logging/writers/SQLite.h | 4 +- src/threading/Formatter.cc | 149 +++++++++++++++ .../{AsciiFormatter.h => Formatter.h} | 114 +++++------ .../Ascii.cc} | 176 ++++------------- src/threading/formatters/Ascii.h | 62 ++++++ src/threading/formatters/JSON.cc | 177 ++++++++++++++++++ src/threading/formatters/JSON.h | 30 +++ .../ssh.log | 1 + .../base/frameworks/logging/ascii-json.bro | 70 +++++++ 25 files changed, 750 insertions(+), 428 deletions(-) create mode 100644 scripts/policy/tuning/json-logs.bro create mode 100644 src/threading/Formatter.cc rename src/threading/{AsciiFormatter.h => Formatter.h} (55%) rename src/threading/{AsciiFormatter.cc => formatters/Ascii.cc} (67%) create mode 100644 src/threading/formatters/Ascii.h create mode 100644 src/threading/formatters/JSON.cc create mode 100644 src/threading/formatters/JSON.h create mode 100644 testing/btest/Baseline/scripts.base.frameworks.logging.ascii-json/ssh.log create mode 100644 testing/btest/scripts/base/frameworks/logging/ascii-json.bro diff --git a/scripts/base/frameworks/logging/writers/ascii.bro b/scripts/base/frameworks/logging/writers/ascii.bro index e510874951..ab86e4641d 100644 --- a/scripts/base/frameworks/logging/writers/ascii.bro +++ b/scripts/base/frameworks/logging/writers/ascii.bro @@ -17,27 +17,38 @@ module LogAscii; export { ## If true, output everything to stdout rather than ## into files. This is primarily for debugging purposes. + ## This is also available as a per-filter $config option. const output_to_stdout = F &redef; + ## If true, the default option will be to write logs in a JSON format. + ## This is also available as a per-filter $config option. + const use_json = F &redef; + ## If true, include lines with log meta information such as column names ## with types, the values of ASCII logging options that are in use, and ## the time when the file was opened and closed (the latter at the end). + ## If writing in JSON format, this is implicitly disabled. const include_meta = T &redef; - ## Prefix for lines with meta information. + ## Prefix for lines with meta information. This is also available as a + ## per-filter $config option. const meta_prefix = "#" &redef; - ## Separator between fields. + ## Separator between fields. This is also available as a per-filter + ## $config option. const separator = Log::separator &redef; - ## Separator between set elements. + ## Separator between set elements. This is also available as a + ## per-filter $config option. const set_separator = Log::set_separator &redef; ## String to use for empty fields. This should be different from - ## *unset_field* to make the output unambiguous. + ## *unset_field* to make the output unambiguous. This is also + ## available as a per-filter $config option. const empty_field = Log::empty_field &redef; - ## String to use for an unset &optional field. + ## String to use for an unset &optional field. This is also + ## available as a per-filter $config option. const unset_field = Log::unset_field &redef; } diff --git a/scripts/policy/tuning/json-logs.bro b/scripts/policy/tuning/json-logs.bro new file mode 100644 index 0000000000..e7daf35063 --- /dev/null +++ b/scripts/policy/tuning/json-logs.bro @@ -0,0 +1,4 @@ +##! Loading this script will cause all logs to be written +##! out as JSON by default. + +redef LogAscii::use_json=T; diff --git a/scripts/test-all-policy.bro b/scripts/test-all-policy.bro index 3a0bd17614..254ad69533 100644 --- a/scripts/test-all-policy.bro +++ b/scripts/test-all-policy.bro @@ -93,6 +93,7 @@ @load tuning/defaults/extracted_file_limits.bro @load tuning/defaults/packet-fragments.bro @load tuning/defaults/warnings.bro +@load tuning/json-logs.bro @load tuning/logs-to-elasticsearch.bro @load tuning/track-all-assets.bro diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index ecf8683ddd..0e990dd37b 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -335,11 +335,13 @@ set(bro_SRCS strsep.c modp_numtoa.c - threading/AsciiFormatter.cc threading/BasicThread.cc + threading/Formatter.cc threading/Manager.cc threading/MsgThread.cc threading/SerialTypes.cc + threading/formatters/Ascii.cc + threading/formatters/JSON.cc logging/Manager.cc logging/WriterBackend.cc diff --git a/src/input/readers/Ascii.cc b/src/input/readers/Ascii.cc index 3fc74480fc..4fd24de73d 100644 --- a/src/input/readers/Ascii.cc +++ b/src/input/readers/Ascii.cc @@ -69,13 +69,13 @@ Ascii::Ascii(ReaderFrontend *frontend) : ReaderBackend(frontend) unset_field.assign( (const char*) BifConst::InputAscii::unset_field->Bytes(), BifConst::InputAscii::unset_field->Len()); - ascii = new AsciiFormatter(this, AsciiFormatter::SeparatorInfo(set_separator, unset_field, empty_field)); + formatter = new threading::formatter::Ascii(this, threading::formatter::Ascii::SeparatorInfo(string(), set_separator, unset_field, empty_field)); } Ascii::~Ascii() { DoClose(); - delete ascii; + delete formatter; } void Ascii::DoClose() @@ -173,7 +173,6 @@ bool Ascii::ReadHeader(bool useCached) return false; } - FieldMapping f(field->name, field->type, field->subtype, ifields[field->name]); if ( field->secondary_name && strlen(field->secondary_name) != 0 ) @@ -200,7 +199,7 @@ bool Ascii::ReadHeader(bool useCached) bool Ascii::GetLine(string& str) { while ( getline(*file, str) ) - { + { if ( str[0] != '#' ) return true; @@ -282,7 +281,7 @@ bool Ascii::DoUpdate() file->sync(); - while ( GetLine(line ) ) + while ( GetLine(line) ) { // split on tabs bool error = false; @@ -332,7 +331,7 @@ bool Ascii::DoUpdate() return false; } - Value* val = ascii->ParseValue(stringfields[(*fit).position], (*fit).name, (*fit).type, (*fit).subtype); + Value* val = formatter->ParseValue(stringfields[(*fit).position], (*fit).name, (*fit).type, (*fit).subtype); if ( val == 0 ) { @@ -347,7 +346,7 @@ bool Ascii::DoUpdate() assert(val->type == TYPE_PORT ); // Error(Fmt("Got type %d != PORT with secondary position!", val->type)); - val->val.port_val.proto = ascii->ParseProto(stringfields[(*fit).secondary_position]); + val->val.port_val.proto = formatter->ParseProto(stringfields[(*fit).secondary_position]); } fields[fpos] = val; @@ -384,8 +383,9 @@ bool Ascii::DoUpdate() } bool Ascii::DoHeartbeat(double network_time, double current_time) -{ - switch ( Info().mode ) { + { + switch ( Info().mode ) + { case MODE_MANUAL: // yay, we do nothing :) break; @@ -398,7 +398,7 @@ bool Ascii::DoHeartbeat(double network_time, double current_time) default: assert(false); - } + } return true; } diff --git a/src/input/readers/Ascii.h b/src/input/readers/Ascii.h index 934ea9a258..5d6bc71d54 100644 --- a/src/input/readers/Ascii.h +++ b/src/input/readers/Ascii.h @@ -7,7 +7,7 @@ #include #include "../ReaderBackend.h" -#include "threading/AsciiFormatter.h" +#include "threading/formatters/Ascii.h" namespace input { namespace reader { @@ -64,7 +64,7 @@ private: string empty_field; string unset_field; - AsciiFormatter* ascii; + threading::formatter::Formatter* formatter; }; diff --git a/src/input/readers/Benchmark.cc b/src/input/readers/Benchmark.cc index ec6b382ebb..de7eae8cc8 100644 --- a/src/input/readers/Benchmark.cc +++ b/src/input/readers/Benchmark.cc @@ -29,7 +29,7 @@ Benchmark::Benchmark(ReaderFrontend *frontend) : ReaderBackend(frontend) heartbeatstarttime = 0; heartbeat_interval = double(BifConst::Threading::heartbeat_interval); - ascii = new AsciiFormatter(this, AsciiFormatter::SeparatorInfo()); + ascii = new threading::formatter::Ascii(this, threading::formatter::Ascii::SeparatorInfo()); } Benchmark::~Benchmark() diff --git a/src/input/readers/Benchmark.h b/src/input/readers/Benchmark.h index ad61b7d2a4..3296f3a85e 100644 --- a/src/input/readers/Benchmark.h +++ b/src/input/readers/Benchmark.h @@ -4,7 +4,7 @@ #define INPUT_READERS_BENCHMARK_H #include "../ReaderBackend.h" -#include "threading/AsciiFormatter.h" +#include "threading/formatters/Ascii.h" namespace input { namespace reader { @@ -40,7 +40,7 @@ private: double timedspread; double heartbeat_interval; - AsciiFormatter* ascii; + threading::formatter::Ascii* ascii; }; diff --git a/src/input/readers/SQLite.cc b/src/input/readers/SQLite.cc index 794b8059ba..d032f934a7 100644 --- a/src/input/readers/SQLite.cc +++ b/src/input/readers/SQLite.cc @@ -36,7 +36,7 @@ SQLite::SQLite(ReaderFrontend *frontend) BifConst::InputSQLite::empty_field->Len() ); - io = new AsciiFormatter(this, AsciiFormatter::SeparatorInfo(set_separator, unset_field, empty_field)); + io = new threading::formatter::Ascii(this, threading::formatter::Ascii::SeparatorInfo(string(), set_separator, unset_field, empty_field)); } SQLite::~SQLite() diff --git a/src/input/readers/SQLite.h b/src/input/readers/SQLite.h index cdea9808c0..a98b3e06b8 100644 --- a/src/input/readers/SQLite.h +++ b/src/input/readers/SQLite.h @@ -10,7 +10,7 @@ #include "../ReaderBackend.h" -#include "threading/AsciiFormatter.h" +#include "threading/formatters/Ascii.h" #include "3rdparty/sqlite3.h" namespace input { namespace reader { @@ -40,7 +40,7 @@ private: string query; sqlite3 *db; sqlite3_stmt *st; - AsciiFormatter* io; + threading::formatter::Ascii* io; string set_separator; string unset_field; diff --git a/src/logging.bif b/src/logging.bif index 1927e149c0..2e8a1eb3a4 100644 --- a/src/logging.bif +++ b/src/logging.bif @@ -77,6 +77,7 @@ const separator: string; const set_separator: string; const empty_field: string; const unset_field: string; +const use_json: bool; # Options for the DataSeries writer. diff --git a/src/logging/writers/Ascii.cc b/src/logging/writers/Ascii.cc index 1a9cc5c4cd..bd5d175d3a 100644 --- a/src/logging/writers/Ascii.cc +++ b/src/logging/writers/Ascii.cc @@ -10,8 +10,7 @@ #include "Ascii.h" -using namespace logging; -using namespace writer; +using namespace logging::writer; using threading::Value; using threading::Field; @@ -20,9 +19,46 @@ Ascii::Ascii(WriterFrontend* frontend) : WriterBackend(frontend) fd = 0; ascii_done = false; tsv = false; + } +Ascii::~Ascii() + { + if ( ! ascii_done ) + { + fprintf(stderr, "internal error: finish missing\n"); + abort(); + } + + delete formatter; + } + +bool Ascii::WriteHeaderField(const string& key, const string& val) + { + string str = meta_prefix + key + separator + val + "\n"; + + return safe_write(fd, str.c_str(), str.length()); + } + +void Ascii::CloseFile(double t) + { + if ( ! fd ) + return; + + if ( include_meta && ! tsv ) + WriteHeaderField("close", Timestamp(0)); + + safe_close(fd); + fd = 0; + } + +bool Ascii::DoInit(const WriterInfo& info, int num_fields, const Field* const * fields) + { + assert(! fd); + + // Set some default values. output_to_stdout = BifConst::LogAscii::output_to_stdout; include_meta = BifConst::LogAscii::include_meta; + use_json = BifConst::LogAscii::use_json; separator.assign( (const char*) BifConst::LogAscii::separator->Bytes(), @@ -49,48 +85,81 @@ Ascii::Ascii(WriterFrontend* frontend) : WriterBackend(frontend) BifConst::LogAscii::meta_prefix->Len() ); - desc.EnableEscaping(); - desc.AddEscapeSequence(separator); - - ascii = new AsciiFormatter(this, AsciiFormatter::SeparatorInfo(set_separator, unset_field, empty_field)); - } - -Ascii::~Ascii() - { - if ( ! ascii_done ) + // Set per-filter configuration options. + for ( WriterInfo::config_map::const_iterator i = info.config.begin(); i != info.config.end(); i++ ) { - fprintf(stderr, "internal error: finish missing\n"); - abort(); + if ( strcmp(i->first, "tsv") == 0 ) + { + if ( strcmp(i->second, "T") == 0 ) + tsv = true; + else if ( strcmp(i->second, "F") == 0 ) + tsv = false; + else + { + Error("invalid value for 'tsv', must be a string and either \"T\" or \"F\""); + return false; + } + } + + else if ( strcmp(i->first, "use_json") == 0 ) + { + if ( strcmp(i->second, "T") == 0 ) + use_json = true; + else if ( strcmp(i->second, "F") == 0 ) + use_json = false; + else + { + Error("invalid value for 'use_json', must be a string and either \"T\" or \"F\""); + return false; + } + } + + else if ( strcmp(i->first, "output_to_stdout") == 0 ) + { + if ( strcmp(i->second, "T") == 0 ) + output_to_stdout = true; + else if ( strcmp(i->second, "F") == 0 ) + output_to_stdout = false; + else + { + Error("invalid value for 'output_to_stdout', must be a string and either \"T\" or \"F\""); + return false; + } + } + + else if ( strcmp(i->first, "separator") == 0 ) + separator.assign(i->second); + + else if ( strcmp(i->first, "set_separator") == 0 ) + set_separator.assign(i->second); + + else if ( strcmp(i->first, "empty_field") == 0 ) + empty_field.assign(i->second); + + else if ( strcmp(i->first, "unset_field") == 0 ) + unset_field.assign(i->second); + + else if ( strcmp(i->first, "meta_prefix") == 0 ) + meta_prefix.assign(i->second); } - delete ascii; - } -bool Ascii::WriteHeaderField(const string& key, const string& val) - { - string str = meta_prefix + key + separator + val + "\n"; - - return safe_write(fd, str.c_str(), str.length()); - } - -void Ascii::CloseFile(double t) - { - if ( ! fd ) - return; - - if ( include_meta && ! tsv ) - WriteHeaderField("close", Timestamp(0)); - - safe_close(fd); - fd = 0; - } - -bool Ascii::DoInit(const WriterInfo& info, int num_fields, const Field* const * fields) - { - assert(! fd); + if ( use_json ) + { + // Write out JSON formatted logs. + formatter = new threading::formatter::JSON(this); + // Using JSON implicitly turns off the header meta fields. + include_meta = false; + } + else + { + // Use the default "Bro logs" format. + desc.EnableEscaping(); + desc.AddEscapeSequence(separator); + formatter = new threading::formatter::Ascii(this, threading::formatter::Ascii::SeparatorInfo(separator, set_separator, unset_field, empty_field)); + } string path = info.path; - if ( output_to_stdout ) path = "/dev/stdout"; @@ -106,24 +175,6 @@ bool Ascii::DoInit(const WriterInfo& info, int num_fields, const Field* const * return false; } - for ( WriterInfo::config_map::const_iterator i = info.config.begin(); i != info.config.end(); i++ ) - { - if ( strcmp(i->first, "tsv") == 0 ) - { - if ( strcmp(i->second, "T") == 0 ) - tsv = true; - - else if ( strcmp(i->second, "F") == 0 ) - tsv = false; - - else - { - Error("invalid value for 'tsv', must be a string and either \"T\" or \"F\""); - return false; - } - } - } - if ( include_meta ) { string names; @@ -208,17 +259,9 @@ bool Ascii::DoWrite(int num_fields, const Field* const * fields, DoInit(Info(), NumFields(), Fields()); desc.Clear(); - - for ( int i = 0; i < num_fields; i++ ) - { - if ( i > 0 ) - desc.AddRaw(separator); - - if ( ! ascii->Describe(&desc, vals[i], fields[i]->name) ) - return false; - } - - desc.AddRaw("\n", 1); + if ( ! formatter->Describe(&desc, num_fields, fields, vals) ) + return false; + desc.AddRaw("\n"); const char* bytes = (const char*)desc.Bytes(); int len = desc.Len(); diff --git a/src/logging/writers/Ascii.h b/src/logging/writers/Ascii.h index 0ddcada57b..08af465ff4 100644 --- a/src/logging/writers/Ascii.h +++ b/src/logging/writers/Ascii.h @@ -6,7 +6,8 @@ #define LOGGING_WRITER_ASCII_H #include "../WriterBackend.h" -#include "threading/AsciiFormatter.h" +#include "threading/formatters/Ascii.h" +#include "threading/formatters/JSON.h" namespace logging { namespace writer { @@ -46,6 +47,7 @@ private: bool output_to_stdout; bool include_meta; bool tsv; + bool use_json; string separator; string set_separator; @@ -53,7 +55,7 @@ private: string unset_field; string meta_prefix; - AsciiFormatter* ascii; + threading::formatter::Formatter* formatter; }; } diff --git a/src/logging/writers/ElasticSearch.cc b/src/logging/writers/ElasticSearch.cc index b22c3f4ab6..cec5bac80e 100644 --- a/src/logging/writers/ElasticSearch.cc +++ b/src/logging/writers/ElasticSearch.cc @@ -16,7 +16,6 @@ #include "BroString.h" #include "NetVar.h" #include "threading/SerialTypes.h" -#include "threading/AsciiFormatter.h" #include #include @@ -53,13 +52,13 @@ ElasticSearch::ElasticSearch(WriterFrontend* frontend) : WriterBackend(frontend) curl_handle = HTTPSetup(); - ascii = new AsciiFormatter(this, AsciiFormatter::SeparatorInfo()); + json = new threading::formatter::JSON(this); } ElasticSearch::~ElasticSearch() { delete [] cluster_name; - delete ascii; + delete json; } bool ElasticSearch::DoInit(const WriterInfo& info, int num_fields, const threading::Field* const* fields) @@ -98,133 +97,7 @@ bool ElasticSearch::BatchIndex() return true; } -bool ElasticSearch::AddValueToBuffer(ODesc* b, Value* val) - { - switch ( val->type ) - { - // ES treats 0 as false and any other value as true so bool types go here. - case TYPE_BOOL: - case TYPE_INT: - b->Add(val->val.int_val); - break; - case TYPE_COUNT: - case TYPE_COUNTER: - { - // ElasticSearch doesn't seem to support unsigned 64bit ints. - if ( val->val.uint_val >= INT64_MAX ) - { - Error(Fmt("count value too large: %" PRIu64, val->val.uint_val)); - b->AddRaw("null", 4); - } - else - b->Add(val->val.uint_val); - break; - } - - case TYPE_PORT: - b->Add(val->val.port_val.port); - break; - - case TYPE_SUBNET: - b->AddRaw("\"", 1); - b->Add(ascii->Render(val->val.subnet_val)); - b->AddRaw("\"", 1); - break; - - case TYPE_ADDR: - b->AddRaw("\"", 1); - b->Add(ascii->Render(val->val.addr_val)); - b->AddRaw("\"", 1); - break; - - case TYPE_DOUBLE: - case TYPE_INTERVAL: - b->Add(val->val.double_val); - break; - - case TYPE_TIME: - { - // ElasticSearch uses milliseconds for timestamps and json only - // supports signed ints (uints can be too large). - uint64_t ts = (uint64_t) (val->val.double_val * 1000); - if ( ts >= INT64_MAX ) - { - Error(Fmt("time value too large: %" PRIu64, ts)); - b->AddRaw("null", 4); - } - else - b->Add(ts); - break; - } - - case TYPE_ENUM: - case TYPE_STRING: - case TYPE_FILE: - case TYPE_FUNC: - { - b->AddRaw("\"", 1); - for ( int i = 0; i < val->val.string_val.length; ++i ) - { - char c = val->val.string_val.data[i]; - // 2byte Unicode escape special characters. - if ( c < 32 || c > 126 || c == '\n' || c == '"' || c == '\'' || c == '\\' || c == '&' ) - { - static const char hex_chars[] = "0123456789abcdef"; - b->AddRaw("\\u00", 4); - b->AddRaw(&hex_chars[(c & 0xf0) >> 4], 1); - b->AddRaw(&hex_chars[c & 0x0f], 1); - } - else - b->AddRaw(&c, 1); - } - b->AddRaw("\"", 1); - break; - } - - case TYPE_TABLE: - { - b->AddRaw("[", 1); - for ( int j = 0; j < val->val.set_val.size; j++ ) - { - if ( j > 0 ) - b->AddRaw(",", 1); - AddValueToBuffer(b, val->val.set_val.vals[j]); - } - b->AddRaw("]", 1); - break; - } - - case TYPE_VECTOR: - { - b->AddRaw("[", 1); - for ( int j = 0; j < val->val.vector_val.size; j++ ) - { - if ( j > 0 ) - b->AddRaw(",", 1); - AddValueToBuffer(b, val->val.vector_val.vals[j]); - } - b->AddRaw("]", 1); - break; - } - - default: - return false; - } - return true; - } - -bool ElasticSearch::AddFieldToBuffer(ODesc *b, Value* val, const Field* field) - { - if ( ! val->present ) - return false; - - b->AddRaw("\"", 1); - b->Add(field->name); - b->AddRaw("\":", 2); - AddValueToBuffer(b, val); - return true; - } bool ElasticSearch::DoWrite(int num_fields, const Field* const * fields, Value** vals) @@ -239,14 +112,7 @@ bool ElasticSearch::DoWrite(int num_fields, const Field* const * fields, buffer.Add(Info().path); buffer.AddRaw("\"}}\n", 4); - buffer.AddRaw("{", 1); - for ( int i = 0; i < num_fields; i++ ) - { - if ( i > 0 && buffer.Bytes()[buffer.Len()] != ',' && vals[i]->present ) - buffer.AddRaw(",", 1); - AddFieldToBuffer(&buffer, vals[i], fields[i]); - } - buffer.AddRaw("}\n", 2); + json->Describe(&buffer, num_fields, fields, vals); counter++; if ( counter >= BifConst::LogElasticSearch::max_batch_size || diff --git a/src/logging/writers/ElasticSearch.h b/src/logging/writers/ElasticSearch.h index b7aba09964..283fff2972 100644 --- a/src/logging/writers/ElasticSearch.h +++ b/src/logging/writers/ElasticSearch.h @@ -9,6 +9,7 @@ #define LOGGING_WRITER_ELASTICSEARCH_H #include +#include "threading/formatters/JSON.h" #include "../WriterBackend.h" namespace logging { namespace writer { @@ -73,7 +74,7 @@ private: uint64 batch_size; - AsciiFormatter* ascii; + threading::formatter::JSON* json; }; } diff --git a/src/logging/writers/SQLite.cc b/src/logging/writers/SQLite.cc index 25f5cb012e..968345696d 100644 --- a/src/logging/writers/SQLite.cc +++ b/src/logging/writers/SQLite.cc @@ -35,7 +35,7 @@ SQLite::SQLite(WriterFrontend* frontend) BifConst::LogSQLite::empty_field->Len() ); - io = new AsciiFormatter(this, AsciiFormatter::SeparatorInfo(set_separator, unset_field, empty_field)); + io = new threading::formatter::Ascii(this, threading::formatter::Ascii::SeparatorInfo(string(), set_separator, unset_field, empty_field)); } SQLite::~SQLite() diff --git a/src/logging/writers/SQLite.h b/src/logging/writers/SQLite.h index b70381ebf6..a962e903ff 100644 --- a/src/logging/writers/SQLite.h +++ b/src/logging/writers/SQLite.h @@ -9,7 +9,7 @@ #include "../WriterBackend.h" -#include "threading/AsciiFormatter.h" +#include "threading/formatters/Ascii.h" #include "3rdparty/sqlite3.h" namespace logging { namespace writer { @@ -51,7 +51,7 @@ private: string unset_field; string empty_field; - AsciiFormatter* io; + threading::formatter::Ascii* io; }; } diff --git a/src/threading/Formatter.cc b/src/threading/Formatter.cc new file mode 100644 index 0000000000..12037a741b --- /dev/null +++ b/src/threading/Formatter.cc @@ -0,0 +1,149 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#include "config.h" + +#include +#include + +#include "Formatter.h" +#include "bro_inet_ntop.h" + +using namespace threading; +using namespace formatter; +using threading::Value; +using threading::Field; + +Formatter::Formatter(threading::MsgThread* t) + { + thread = t; + } + +Formatter::~Formatter() + { + } + +bool Formatter::CheckNumberError(const string& s, const char* end) const + { + // Do this check first, before executing s.c_str() or similar. + // otherwise the value to which *end is pointing at the moment might + // be gone ... + bool endnotnull = (*end != '\0'); + + if ( s.length() == 0 ) + { + thread->Error("Got empty string for number field"); + return true; + } + + if ( end == s.c_str() ) { + thread->Error(thread->Fmt("String '%s' contained no parseable number", s.c_str())); + return true; + } + + if ( endnotnull ) + thread->Warning(thread->Fmt("Number '%s' contained non-numeric trailing characters. Ignored trailing characters '%s'", s.c_str(), end)); + + if ( errno == EINVAL ) + { + thread->Error(thread->Fmt("String '%s' could not be converted to a number", s.c_str())); + return true; + } + + else if ( errno == ERANGE ) + { + thread->Error(thread->Fmt("Number '%s' out of supported range.", s.c_str())); + return true; + } + + return false; + } + +string Formatter::Render(const threading::Value::addr_t& addr) const + { + if ( addr.family == IPv4 ) + { + char s[INET_ADDRSTRLEN]; + + if ( ! bro_inet_ntop(AF_INET, &addr.in.in4, s, INET_ADDRSTRLEN) ) + return ""; + else + return s; + } + else + { + char s[INET6_ADDRSTRLEN]; + + if ( ! bro_inet_ntop(AF_INET6, &addr.in.in6, s, INET6_ADDRSTRLEN) ) + return ""; + else + return s; + } + } + +TransportProto Formatter::ParseProto(const string &proto) const + { + if ( proto == "unknown" ) + return TRANSPORT_UNKNOWN; + else if ( proto == "tcp" ) + return TRANSPORT_TCP; + else if ( proto == "udp" ) + return TRANSPORT_UDP; + else if ( proto == "icmp" ) + return TRANSPORT_ICMP; + + thread->Error(thread->Fmt("Tried to parse invalid/unknown protocol: %s", proto.c_str())); + + return TRANSPORT_UNKNOWN; + } + + +// More or less verbose copy from IPAddr.cc -- which uses reporter. +threading::Value::addr_t Formatter::ParseAddr(const string &s) const + { + threading::Value::addr_t val; + + if ( s.find(':') == std::string::npos ) // IPv4. + { + val.family = IPv4; + + if ( inet_aton(s.c_str(), &(val.in.in4)) <= 0 ) + { + thread->Error(thread->Fmt("Bad address: %s", s.c_str())); + memset(&val.in.in4.s_addr, 0, sizeof(val.in.in4.s_addr)); + } + } + + else + { + val.family = IPv6; + if ( inet_pton(AF_INET6, s.c_str(), val.in.in6.s6_addr) <=0 ) + { + thread->Error(thread->Fmt("Bad address: %s", s.c_str())); + memset(val.in.in6.s6_addr, 0, sizeof(val.in.in6.s6_addr)); + } + } + + return val; + } + +string Formatter::Render(const threading::Value::subnet_t& subnet) const + { + char l[16]; + + if ( subnet.prefix.family == IPv4 ) + modp_uitoa10(subnet.length - 96, l); + else + modp_uitoa10(subnet.length, l); + + string s = Render(subnet.prefix) + "/" + l; + + return s; + } + +string Formatter::Render(double d) const + { + char buf[256]; + modp_dtoa(d, buf, 6); + return buf; + } + diff --git a/src/threading/AsciiFormatter.h b/src/threading/Formatter.h similarity index 55% rename from src/threading/AsciiFormatter.h rename to src/threading/Formatter.h index 1522e46191..c5f6cb4296 100644 --- a/src/threading/AsciiFormatter.h +++ b/src/threading/Formatter.h @@ -1,42 +1,20 @@ // See the file "COPYING" in the main distribution directory for copyright. -#ifndef THREADING_ASCII_FORMATTER_H -#define THREADING_ASCII_FORMATTER_H +#ifndef THREADING_FORMATTER_H +#define THREADING_FORMATTER_H #include "../Desc.h" #include "MsgThread.h" +namespace threading { namespace formatter { + /** - * A thread-safe class for converting values into a readable ASCII - * representation, and vice versa. This is a utility class that factors out - * common rendering/parsing code needed by a number of input/output threads. + * A thread-safe class for converting values into some textual format. + * This is a base class that implements the interface forcommon + * rendering/parsing code needed by a number of input/output threads. */ -class AsciiFormatter { +class Formatter { public: - /** - * A struct to pass the necessary configuration values to the - * AsciiFormatter module on initialization. - */ - struct SeparatorInfo - { - string set_separator; // Separator between set elements. - string unset_field; // String marking an unset field. - string empty_field; // String marking an empty (but set) field. - - /** - * Constructor that leaves separators etc unset to dummy - * values. Useful if you use only methods that don't need any - * of them, like StringToAddr, etc. - */ - SeparatorInfo(); - - /** - * Constructor that defines all the configuration options. - * Use if you need either ValToODesc or EntryToVal. - */ - SeparatorInfo(const string& set_separator, const string& unset_field, const string& empty_field); - }; - /** * Constructor. * @@ -44,31 +22,69 @@ public: * some of the thread's methods, e.g., for error reporting and * internal formatting. * - * @param info SeparatorInfo structure defining the necessary - * separators. */ - AsciiFormatter(threading::MsgThread* t, const SeparatorInfo info); + Formatter(threading::MsgThread* t); /** * Destructor. */ - ~AsciiFormatter(); + virtual ~Formatter(); /** - * Convert a threading value into a corresponding ASCII. - * representation. + * Convert a list of threading values into an implementation specific representation. + * + * @param desc The ODesc object to write to. + * + * @param num_fields The number of fields in the logging record. + * + * @param fields Information about the fields for each of the given log values. + * + * @param vals The field values. + * + * @return Returns true on success, false on error. Errors are also + * flagged via the reporter. + */ + virtual bool Describe(ODesc* desc, int num_fields, const threading::Field* const * fields, + threading::Value** vals) const = 0; + + /** + * Convert a threading value into an implementation specific representation. * * @param desc The ODesc object to write to. * * @param val the Value to render to the ODesc object. * - * @param The name of a field associated with the value. Used only - * for error reporting. + * @return Returns true on success, false on error. Errors are also + * flagged via the reporter. + */ + virtual bool Describe(ODesc* desc, threading::Value* val) const = 0; + + /** + * Convert a threading value into an implementation specific representation. + * + * @param desc The ODesc object to write to. + * + * @param val the Value to render to the ODesc object. + * + * @param The name of a field associated with the value. * * @return Returns true on success, false on error. Errors are also * flagged via the reporter. */ - bool Describe(ODesc* desc, threading::Value* val, const string& name) const; + virtual bool Describe(ODesc* desc, threading::Value* val, const string& name) const = 0; + + /** + * Convert a representation of a field into a value. + * + * @param s The string to parse. + * + * @param The name of a field associated with the value. Used only + * for error reporting. + * + * @return The new value, or null on error. Errors are also flagged + * via the reporter. + */ + virtual threading::Value* ParseValue(string s, string name, TypeTag type, TypeTag subtype = TYPE_ERROR) const = 0; /** * Convert an IP address into a string. @@ -98,19 +114,6 @@ public: */ string Render(double d) const; - /** - * Convert the ASCII representation of a field into a value. - * - * @param s The string to parse. - * - * @param The name of a field associated with the value. Used only - * for error reporting. - * - * @return The new value, or null on error. Errors are also flagged - * via the reporter. - */ - threading::Value* ParseValue(string s, string name, TypeTag type, TypeTag subtype = TYPE_ERROR) const; - /** * Convert a string into a TransportProto. The string must be one of * \c tcp, \c udp, \c icmp, or \c unknown. @@ -132,11 +135,12 @@ public: */ threading::Value::addr_t ParseAddr(const string &addr) const; -private: +protected: bool CheckNumberError(const string& s, const char * end) const; - SeparatorInfo separators; threading::MsgThread* thread; }; -#endif /* THREADING_ASCII_FORMATTER_H */ +}} + +#endif /* THREADING_FORMATTER_H */ diff --git a/src/threading/AsciiFormatter.cc b/src/threading/formatters/Ascii.cc similarity index 67% rename from src/threading/AsciiFormatter.cc rename to src/threading/formatters/Ascii.cc index 616abbe2b6..6457f3c782 100644 --- a/src/threading/AsciiFormatter.cc +++ b/src/threading/formatters/Ascii.cc @@ -5,35 +5,59 @@ #include #include -#include "AsciiFormatter.h" -#include "bro_inet_ntop.h" +#include "./Ascii.h" -AsciiFormatter::SeparatorInfo::SeparatorInfo() +using namespace threading::formatter; + +Ascii::SeparatorInfo::SeparatorInfo() { + this->separator = "SHOULD_NOT_BE_USED"; this->set_separator = "SHOULD_NOT_BE_USED"; this->unset_field = "SHOULD_NOT_BE_USED"; this->empty_field = "SHOULD_NOT_BE_USED"; } -AsciiFormatter::SeparatorInfo::SeparatorInfo(const string & set_separator, - const string & unset_field, const string & empty_field) +Ascii::SeparatorInfo::SeparatorInfo(const string & separator, + const string & set_separator, + const string & unset_field, + const string & empty_field) { + this->separator = separator; this->set_separator = set_separator; this->unset_field = unset_field; this->empty_field = empty_field; } -AsciiFormatter::AsciiFormatter(threading::MsgThread* t, const SeparatorInfo info) +Ascii::Ascii(threading::MsgThread* t, const SeparatorInfo info) : Formatter(t) { - thread = t; this->separators = info; } -AsciiFormatter::~AsciiFormatter() +Ascii::~Ascii() { } -bool AsciiFormatter::Describe(ODesc* desc, threading::Value* val, const string& name) const +bool Ascii::Describe(ODesc* desc, int num_fields, const threading::Field* const * fields, + threading::Value** vals) const + { + for ( int i = 0; i < num_fields; i++ ) + { + if ( i > 0 ) + desc->AddRaw(separators.separator); + + if ( ! Describe(desc, vals[i], fields[i]->name) ) + return false; + } + + return true; + } + +bool Ascii::Describe(ODesc* desc, threading::Value* val, const string& name) const + { + return Describe(desc, val); + } + +bool Ascii::Describe(ODesc* desc, threading::Value* val) const { if ( ! val->present ) { @@ -133,7 +157,7 @@ bool AsciiFormatter::Describe(ODesc* desc, threading::Value* val, const string& if ( j > 0 ) desc->AddRaw(separators.set_separator); - if ( ! Describe(desc, val->val.set_val.vals[j], name) ) + if ( ! Describe(desc, val->val.set_val.vals[j]) ) { desc->RemoveEscapeSequence(separators.set_separator); return false; @@ -158,7 +182,7 @@ bool AsciiFormatter::Describe(ODesc* desc, threading::Value* val, const string& if ( j > 0 ) desc->AddRaw(separators.set_separator); - if ( ! Describe(desc, val->val.vector_val.vals[j], name) ) + if ( ! Describe(desc, val->val.vector_val.vals[j]) ) { desc->RemoveEscapeSequence(separators.set_separator); return false; @@ -170,7 +194,7 @@ bool AsciiFormatter::Describe(ODesc* desc, threading::Value* val, const string& } default: - thread->Error(thread->Fmt("unsupported field format %d for %s", val->type, name.c_str())); + thread->Error(thread->Fmt("Ascii writer unsupported field format %d", val->type)); return false; } @@ -178,7 +202,7 @@ bool AsciiFormatter::Describe(ODesc* desc, threading::Value* val, const string& } -threading::Value* AsciiFormatter::ParseValue(string s, string name, TypeTag type, TypeTag subtype) const +threading::Value* Ascii::ParseValue(string s, string name, TypeTag type, TypeTag subtype) const { if ( s.compare(separators.unset_field) == 0 ) // field is not set... return new threading::Value(type, false); @@ -381,129 +405,3 @@ parse_error: delete val; return 0; } - -bool AsciiFormatter::CheckNumberError(const string& s, const char* end) const - { - // Do this check first, before executing s.c_str() or similar. - // otherwise the value to which *end is pointing at the moment might - // be gone ... - bool endnotnull = (*end != '\0'); - - if ( s.length() == 0 ) - { - thread->Error("Got empty string for number field"); - return true; - } - - if ( end == s.c_str() ) { - thread->Error(thread->Fmt("String '%s' contained no parseable number", s.c_str())); - return true; - } - - if ( endnotnull ) - thread->Warning(thread->Fmt("Number '%s' contained non-numeric trailing characters. Ignored trailing characters '%s'", s.c_str(), end)); - - if ( errno == EINVAL ) - { - thread->Error(thread->Fmt("String '%s' could not be converted to a number", s.c_str())); - return true; - } - - else if ( errno == ERANGE ) - { - thread->Error(thread->Fmt("Number '%s' out of supported range.", s.c_str())); - return true; - } - - return false; - } - -string AsciiFormatter::Render(const threading::Value::addr_t& addr) const - { - if ( addr.family == IPv4 ) - { - char s[INET_ADDRSTRLEN]; - - if ( ! bro_inet_ntop(AF_INET, &addr.in.in4, s, INET_ADDRSTRLEN) ) - return ""; - else - return s; - } - else - { - char s[INET6_ADDRSTRLEN]; - - if ( ! bro_inet_ntop(AF_INET6, &addr.in.in6, s, INET6_ADDRSTRLEN) ) - return ""; - else - return s; - } - } - -TransportProto AsciiFormatter::ParseProto(const string &proto) const - { - if ( proto == "unknown" ) - return TRANSPORT_UNKNOWN; - else if ( proto == "tcp" ) - return TRANSPORT_TCP; - else if ( proto == "udp" ) - return TRANSPORT_UDP; - else if ( proto == "icmp" ) - return TRANSPORT_ICMP; - - thread->Error(thread->Fmt("Tried to parse invalid/unknown protocol: %s", proto.c_str())); - - return TRANSPORT_UNKNOWN; - } - - -// More or less verbose copy from IPAddr.cc -- which uses reporter. -threading::Value::addr_t AsciiFormatter::ParseAddr(const string &s) const - { - threading::Value::addr_t val; - - if ( s.find(':') == std::string::npos ) // IPv4. - { - val.family = IPv4; - - if ( inet_aton(s.c_str(), &(val.in.in4)) <= 0 ) - { - thread->Error(thread->Fmt("Bad address: %s", s.c_str())); - memset(&val.in.in4.s_addr, 0, sizeof(val.in.in4.s_addr)); - } - } - - else - { - val.family = IPv6; - if ( inet_pton(AF_INET6, s.c_str(), val.in.in6.s6_addr) <=0 ) - { - thread->Error(thread->Fmt("Bad address: %s", s.c_str())); - memset(val.in.in6.s6_addr, 0, sizeof(val.in.in6.s6_addr)); - } - } - - return val; - } - -string AsciiFormatter::Render(const threading::Value::subnet_t& subnet) const - { - char l[16]; - - if ( subnet.prefix.family == IPv4 ) - modp_uitoa10(subnet.length - 96, l); - else - modp_uitoa10(subnet.length, l); - - string s = Render(subnet.prefix) + "/" + l; - - return s; - } - -string AsciiFormatter::Render(double d) const - { - char buf[256]; - modp_dtoa(d, buf, 6); - return buf; - } - diff --git a/src/threading/formatters/Ascii.h b/src/threading/formatters/Ascii.h new file mode 100644 index 0000000000..d4b8eec052 --- /dev/null +++ b/src/threading/formatters/Ascii.h @@ -0,0 +1,62 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#ifndef THREADING_FORMATTERS_ASCII_H +#define THREADING_FORMATTERS_ASCII_H + +#include "../Formatter.h" + +namespace threading { namespace formatter { + +class Ascii : public Formatter { +public: + /** + * A struct to pass the necessary configuration values to the + * Ascii module on initialization. + */ + struct SeparatorInfo + { + string separator; // Separator between columns + string set_separator; // Separator between set elements. + string unset_field; // String marking an unset field. + string empty_field; // String marking an empty (but set) field. + + /** + * Constructor that leaves separators etc unset to dummy + * values. Useful if you use only methods that don't need any + * of them, like StringToAddr, etc. + */ + SeparatorInfo(); + + /** + * Constructor that defines all the configuration options. + * Use if you need either ValToODesc or EntryToVal. + */ + SeparatorInfo(const string& separator, const string& set_separator, const string& unset_field, const string& empty_field); + }; + + /** + * Constructor. + * + * @param t The thread that uses this class instance. The class uses + * some of the thread's methods, e.g., for error reporting and + * internal formatting. + * + * @param info SeparatorInfo structure defining the necessary + * separators. + */ + Ascii(threading::MsgThread* t, const SeparatorInfo info); + virtual ~Ascii(); + + virtual bool Describe(ODesc* desc, threading::Value* val) const; + virtual bool Describe(ODesc* desc, threading::Value* val, const string& name) const; + virtual bool Describe(ODesc* desc, int num_fields, const threading::Field* const * fields, + threading::Value** vals) const; + virtual threading::Value* ParseValue(string s, string name, TypeTag type, TypeTag subtype = TYPE_ERROR) const; + +private: + SeparatorInfo separators; +}; + +}} + +#endif /* THREADING_FORMATTERS_ASCII_H */ diff --git a/src/threading/formatters/JSON.cc b/src/threading/formatters/JSON.cc new file mode 100644 index 0000000000..c1e68b27bb --- /dev/null +++ b/src/threading/formatters/JSON.cc @@ -0,0 +1,177 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#include "config.h" + +#include +#include + +#include "./JSON.h" + +using namespace threading::formatter; + +JSON::JSON(MsgThread* t) : Formatter(t) + { + } + +JSON::~JSON() + { + } + +bool JSON::Describe(ODesc* desc, int num_fields, const Field* const * fields, + Value** vals) const + { + desc->AddRaw("{"); + for ( int i = 0; i < num_fields; i++ ) + { + if ( i > 0 && + desc->Bytes()[desc->Len()] != ',' && vals[i]->present ) + { + desc->AddRaw(","); + } + + if ( ! Describe(desc, vals[i], fields[i]->name) ) + return false; + } + desc->AddRaw("}"); + return true; + } + +bool JSON::Describe(ODesc* desc, Value* val, const string& name) const + { + if ( ! val->present ) + return true; + + desc->AddRaw("\"", 1); + desc->Add(name); + desc->AddRaw("\":", 2); + + return Describe(desc, val); + } + +bool JSON::Describe(ODesc* desc, Value* val) const + { + if ( ! val->present ) + return true; + + switch ( val->type ) + { + case TYPE_BOOL: + desc->AddRaw(val->val.int_val == 0 ? "false" : "true"); + break; + + case TYPE_INT: + desc->Add(val->val.int_val); + break; + + case TYPE_COUNT: + case TYPE_COUNTER: + { + // JSON doesn't support unsigned 64bit ints. + if ( val->val.uint_val >= INT64_MAX ) + { + thread->Error(thread->Fmt("count value too large for JSON: %" PRIu64, val->val.uint_val)); + desc->AddRaw("null", 4); + } + else + desc->Add(val->val.uint_val); + break; + } + + case TYPE_PORT: + desc->Add(val->val.port_val.port); + break; + + case TYPE_SUBNET: + desc->AddRaw("\"", 1); + desc->Add(Render(val->val.subnet_val)); + desc->AddRaw("\"", 1); + break; + + case TYPE_ADDR: + desc->AddRaw("\"", 1); + desc->Add(Render(val->val.addr_val)); + desc->AddRaw("\"", 1); + break; + + case TYPE_DOUBLE: + case TYPE_INTERVAL: + desc->Add(val->val.double_val); + break; + + case TYPE_TIME: + { + // ElasticSearch uses milliseconds for timestamps and json only + // supports signed ints (uints can be too large). + uint64_t ts = (uint64_t) (val->val.double_val * 1000); + if ( ts >= INT64_MAX ) + { + thread->Error(thread->Fmt("time value too large for JSON: %" PRIu64, ts)); + desc->AddRaw("null", 4); + } + else + desc->Add(ts); + break; + } + + case TYPE_ENUM: + case TYPE_STRING: + case TYPE_FILE: + case TYPE_FUNC: + { + desc->AddRaw("\"", 1); + for ( int i = 0; i < val->val.string_val.length; ++i ) + { + char c = val->val.string_val.data[i]; + // 2byte Unicode escape special characters. + if ( c < 32 || c > 126 || c == '\n' || c == '"' || c == '\'' || c == '\\' || c == '&' ) + { + static const char hex_chars[] = "0123456789abcdef"; + desc->AddRaw("\\u00", 4); + desc->AddRaw(&hex_chars[(c & 0xf0) >> 4], 1); + desc->AddRaw(&hex_chars[c & 0x0f], 1); + } + else + desc->AddRaw(&c, 1); + } + desc->AddRaw("\"", 1); + break; + } + + case TYPE_TABLE: + { + desc->AddRaw("[", 1); + for ( int j = 0; j < val->val.set_val.size; j++ ) + { + if ( j > 0 ) + desc->AddRaw(",", 1); + Describe(desc, val->val.set_val.vals[j]); + } + desc->AddRaw("]", 1); + break; + } + + case TYPE_VECTOR: + { + desc->AddRaw("[", 1); + for ( int j = 0; j < val->val.vector_val.size; j++ ) + { + if ( j > 0 ) + desc->AddRaw(",", 1); + Describe(desc, val->val.vector_val.vals[j]); + } + desc->AddRaw("]", 1); + break; + } + + default: + return false; + } + + return true; + } + +threading::Value* JSON::ParseValue(string s, string name, TypeTag type, TypeTag subtype) const + { + thread->Error("JSON formatter does not support parsing yet."); + return NULL; + } diff --git a/src/threading/formatters/JSON.h b/src/threading/formatters/JSON.h new file mode 100644 index 0000000000..7b4c4ea328 --- /dev/null +++ b/src/threading/formatters/JSON.h @@ -0,0 +1,30 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#ifndef THREADING_FORMATTERS_JSON_H +#define THREADING_FORMATTERS_JSON_H + +#include "../Formatter.h" + +namespace threading { namespace formatter { + +/** + * A thread-safe class for converting values into a JSON representation + * and vice versa. + */ +class JSON : public Formatter { +public: + JSON(threading::MsgThread* t); + virtual ~JSON(); + + virtual bool Describe(ODesc* desc, threading::Value* val) const; + virtual bool Describe(ODesc* desc, threading::Value* val, const string& name) const; + virtual bool Describe(ODesc* desc, int num_fields, const threading::Field* const * fields, + threading::Value** vals) const; + virtual threading::Value* ParseValue(string s, string name, TypeTag type, TypeTag subtype = TYPE_ERROR) const; + +private: +}; + +}} + +#endif /* THREADING_FORMATTERS_JSON_H */ diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-json/ssh.log b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-json/ssh.log new file mode 100644 index 0000000000..7446047266 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-json/ssh.log @@ -0,0 +1 @@ +{"b":true,"i":-42,"e":"SSH::LOG","c":21,"p":123,"sn":"10.0.0.0/24","a":"1.2.3.4","d":3.14,"t":1394462315468,"iv":100.0,"s":"hurz","sc":[2,4,1,3],"ss":["CC","AA","BB"],"se":[],"vc":[10,20,30],"ve":[],"f":"SSH::foo\u000a{ \u000aif (0 < SSH::i) \u000a\u0009return (Foo);\u000aelse\u000a\u0009return (Bar);\u000a\u000a}"} diff --git a/testing/btest/scripts/base/frameworks/logging/ascii-json.bro b/testing/btest/scripts/base/frameworks/logging/ascii-json.bro new file mode 100644 index 0000000000..35cf7f1e80 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/logging/ascii-json.bro @@ -0,0 +1,70 @@ +# +# @TEST-EXEC: bro -b %INPUT +# @TEST-EXEC: btest-diff ssh.log +# +# Testing all possible types. + +redef LogAscii::use_json = T; + +module SSH; + +export { + redef enum Log::ID += { LOG }; + + type Log: record { + b: bool; + i: int; + e: Log::ID; + c: count; + p: port; + sn: subnet; + a: addr; + d: double; + t: time; + iv: interval; + s: string; + sc: set[count]; + ss: set[string]; + se: set[string]; + vc: vector of count; + ve: vector of string; + f: function(i: count) : string; + } &log; +} + +function foo(i : count) : string + { + if ( i > 0 ) + return "Foo"; + else + return "Bar"; + } + +event bro_init() +{ + Log::create_stream(SSH::LOG, [$columns=Log]); + + local empty_set: set[string]; + local empty_vector: vector of string; + + Log::write(SSH::LOG, [ + $b=T, + $i=-42, + $e=SSH::LOG, + $c=21, + $p=123/tcp, + $sn=10.0.0.1/24, + $a=1.2.3.4, + $d=3.14, + $t=network_time(), + $iv=100secs, + $s="hurz", + $sc=set(1,2,3,4), + $ss=set("AA", "BB", "CC"), + $se=empty_set, + $vc=vector(10, 20, 30), + $ve=empty_vector, + $f=foo + ]); +} + From c9aaf9e753f48008b258eff1ac6a60046a4d274c Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Mon, 10 Mar 2014 14:22:35 -0400 Subject: [PATCH 2/5] Added an option to the JSON formatter to use ISO 8601 for timestamps. - It's not *exactly* ISO 8601 which doesn't seem to support subseconds, but subseconds are very important to us and most things that support ISO8601 seem to also support subseconds in the way I'm implemented it. --- .../base/frameworks/logging/writers/ascii.bro | 6 +++ src/logging.bif | 1 + src/logging/writers/Ascii.cc | 3 +- src/logging/writers/Ascii.h | 1 + src/logging/writers/ElasticSearch.cc | 2 +- src/threading/formatters/JSON.cc | 37 +++++++++++++++---- src/threading/formatters/JSON.h | 3 +- .../ssh.log | 2 + .../ssh.log | 2 +- .../logging/ascii-json-iso-timestamps.bro | 31 ++++++++++++++++ .../base/frameworks/logging/ascii-json.bro | 2 +- 11 files changed, 77 insertions(+), 13 deletions(-) create mode 100644 testing/btest/Baseline/scripts.base.frameworks.logging.ascii-json-iso-timestamps/ssh.log create mode 100644 testing/btest/scripts/base/frameworks/logging/ascii-json-iso-timestamps.bro diff --git a/scripts/base/frameworks/logging/writers/ascii.bro b/scripts/base/frameworks/logging/writers/ascii.bro index ab86e4641d..68219cb964 100644 --- a/scripts/base/frameworks/logging/writers/ascii.bro +++ b/scripts/base/frameworks/logging/writers/ascii.bro @@ -24,6 +24,12 @@ export { ## This is also available as a per-filter $config option. const use_json = F &redef; + ## By default, the JSON formatter will use double values for timestamps + ## which represent the number of seconds from the UNIX epoch. By setting + ## this to 'T', it will use the 8601 format. This is also available as + ## a per-filter $config option. + const json_iso_timestamps = F &redef; + ## If true, include lines with log meta information such as column names ## with types, the values of ASCII logging options that are in use, and ## the time when the file was opened and closed (the latter at the end). diff --git a/src/logging.bif b/src/logging.bif index 2e8a1eb3a4..f12aa50c1c 100644 --- a/src/logging.bif +++ b/src/logging.bif @@ -78,6 +78,7 @@ const set_separator: string; const empty_field: string; const unset_field: string; const use_json: bool; +const json_iso_timestamps: bool; # Options for the DataSeries writer. diff --git a/src/logging/writers/Ascii.cc b/src/logging/writers/Ascii.cc index bd5d175d3a..976c083eda 100644 --- a/src/logging/writers/Ascii.cc +++ b/src/logging/writers/Ascii.cc @@ -59,6 +59,7 @@ bool Ascii::DoInit(const WriterInfo& info, int num_fields, const Field* const * output_to_stdout = BifConst::LogAscii::output_to_stdout; include_meta = BifConst::LogAscii::include_meta; use_json = BifConst::LogAscii::use_json; + json_iso_timestamps = BifConst::LogAscii::json_iso_timestamps; separator.assign( (const char*) BifConst::LogAscii::separator->Bytes(), @@ -147,7 +148,7 @@ bool Ascii::DoInit(const WriterInfo& info, int num_fields, const Field* const * if ( use_json ) { // Write out JSON formatted logs. - formatter = new threading::formatter::JSON(this); + formatter = new threading::formatter::JSON(this, json_iso_timestamps); // Using JSON implicitly turns off the header meta fields. include_meta = false; } diff --git a/src/logging/writers/Ascii.h b/src/logging/writers/Ascii.h index 08af465ff4..d131a289b4 100644 --- a/src/logging/writers/Ascii.h +++ b/src/logging/writers/Ascii.h @@ -48,6 +48,7 @@ private: bool include_meta; bool tsv; bool use_json; + bool json_iso_timestamps; string separator; string set_separator; diff --git a/src/logging/writers/ElasticSearch.cc b/src/logging/writers/ElasticSearch.cc index cec5bac80e..0a18e624ea 100644 --- a/src/logging/writers/ElasticSearch.cc +++ b/src/logging/writers/ElasticSearch.cc @@ -52,7 +52,7 @@ ElasticSearch::ElasticSearch(WriterFrontend* frontend) : WriterBackend(frontend) curl_handle = HTTPSetup(); - json = new threading::formatter::JSON(this); + json = new threading::formatter::JSON(this, false); } ElasticSearch::~ElasticSearch() diff --git a/src/threading/formatters/JSON.cc b/src/threading/formatters/JSON.cc index c1e68b27bb..57e9d58527 100644 --- a/src/threading/formatters/JSON.cc +++ b/src/threading/formatters/JSON.cc @@ -4,13 +4,15 @@ #include #include +#include #include "./JSON.h" using namespace threading::formatter; -JSON::JSON(MsgThread* t) : Formatter(t) +JSON::JSON(MsgThread* t, bool json_iso_timestamps) : Formatter(t) { + iso_timestamps = json_iso_timestamps; } JSON::~JSON() @@ -100,16 +102,35 @@ bool JSON::Describe(ODesc* desc, Value* val) const case TYPE_TIME: { - // ElasticSearch uses milliseconds for timestamps and json only - // supports signed ints (uints can be too large). - uint64_t ts = (uint64_t) (val->val.double_val * 1000); - if ( ts >= INT64_MAX ) + if ( iso_timestamps ) { - thread->Error(thread->Fmt("time value too large for JSON: %" PRIu64, ts)); - desc->AddRaw("null", 4); + char buffer[40]; + time_t t = time_t(val->val.double_val); + if ( strftime(buffer, sizeof(buffer), "%Y-%m-%dT%H:%M:%S", gmtime(&t)) == 0 ) + thread->Error(thread->Fmt("strftime error for JSON: %" PRIu64)); + else + { + double integ; + double frac = modf(val->val.double_val, &integ); + snprintf(buffer, sizeof(buffer), "%s.%06.0fZ", buffer, frac*1000000); + desc->AddRaw("\"", 1); + desc->Add(buffer); + desc->AddRaw("\"", 1); + } } else - desc->Add(ts); + { + // ElasticSearch uses milliseconds for timestamps and json only + // supports signed ints (uints can be too large). + uint64_t ts = (uint64_t) (val->val.double_val * 1000); + if ( ts >= INT64_MAX ) + { + thread->Error(thread->Fmt("time value too large for JSON: %" PRIu64, ts)); + desc->AddRaw("null", 4); + } + else + desc->Add(ts); + } break; } diff --git a/src/threading/formatters/JSON.h b/src/threading/formatters/JSON.h index 7b4c4ea328..b15da075cd 100644 --- a/src/threading/formatters/JSON.h +++ b/src/threading/formatters/JSON.h @@ -13,7 +13,7 @@ namespace threading { namespace formatter { */ class JSON : public Formatter { public: - JSON(threading::MsgThread* t); + JSON(threading::MsgThread* t, bool json_iso_timestamps); virtual ~JSON(); virtual bool Describe(ODesc* desc, threading::Value* val) const; @@ -23,6 +23,7 @@ public: virtual threading::Value* ParseValue(string s, string name, TypeTag type, TypeTag subtype = TYPE_ERROR) const; private: + bool iso_timestamps; }; }} diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-json-iso-timestamps/ssh.log b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-json-iso-timestamps/ssh.log new file mode 100644 index 0000000000..5673a0605a --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-json-iso-timestamps/ssh.log @@ -0,0 +1,2 @@ +{"t":"2008-07-09T16:13:30.005432Z"} +{"t":"1986-12-01T01:01:01.900000Z"} diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-json/ssh.log b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-json/ssh.log index 7446047266..9e4e74c018 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-json/ssh.log +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-json/ssh.log @@ -1 +1 @@ -{"b":true,"i":-42,"e":"SSH::LOG","c":21,"p":123,"sn":"10.0.0.0/24","a":"1.2.3.4","d":3.14,"t":1394462315468,"iv":100.0,"s":"hurz","sc":[2,4,1,3],"ss":["CC","AA","BB"],"se":[],"vc":[10,20,30],"ve":[],"f":"SSH::foo\u000a{ \u000aif (0 < SSH::i) \u000a\u0009return (Foo);\u000aelse\u000a\u0009return (Bar);\u000a\u000a}"} +{"b":true,"i":-42,"e":"SSH::LOG","c":21,"p":123,"sn":"10.0.0.0/24","a":"1.2.3.4","d":3.14,"t":1215620010543,"iv":100.0,"s":"hurz","sc":[2,4,1,3],"ss":["CC","AA","BB"],"se":[],"vc":[10,20,30],"ve":[],"f":"SSH::foo\u000a{ \u000aif (0 < SSH::i) \u000a\u0009return (Foo);\u000aelse\u000a\u0009return (Bar);\u000a\u000a}"} diff --git a/testing/btest/scripts/base/frameworks/logging/ascii-json-iso-timestamps.bro b/testing/btest/scripts/base/frameworks/logging/ascii-json-iso-timestamps.bro new file mode 100644 index 0000000000..f9541edda7 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/logging/ascii-json-iso-timestamps.bro @@ -0,0 +1,31 @@ +# +# @TEST-EXEC: bro -b %INPUT +# @TEST-EXEC: btest-diff ssh.log +# +# Testing all possible types. + +redef LogAscii::use_json = T; +redef LogAscii::json_iso_timestamps = T; + +module SSH; + +export { + redef enum Log::ID += { LOG }; + + type Log: record { + t: time; + } &log; +} + +event bro_init() +{ + Log::create_stream(SSH::LOG, [$columns=Log]); + Log::write(SSH::LOG, [ + $t=(strptime("%Y-%m-%dT%H:%M:%SZ", "2008-07-09T16:13:30Z") + 0.00543210 secs) + ]); + Log::write(SSH::LOG, [ + $t=(strptime("%Y-%m-%dT%H:%M:%SZ", "1986-12-01T01:01:01Z") + 0.90 secs) + ]); + +} + diff --git a/testing/btest/scripts/base/frameworks/logging/ascii-json.bro b/testing/btest/scripts/base/frameworks/logging/ascii-json.bro index 35cf7f1e80..2b6055930f 100644 --- a/testing/btest/scripts/base/frameworks/logging/ascii-json.bro +++ b/testing/btest/scripts/base/frameworks/logging/ascii-json.bro @@ -56,7 +56,7 @@ event bro_init() $sn=10.0.0.1/24, $a=1.2.3.4, $d=3.14, - $t=network_time(), + $t=(strptime("%Y-%m-%dT%H:%M:%SZ", "2008-07-09T16:13:30Z") + 0.543210 secs), $iv=100secs, $s="hurz", $sc=set(1,2,3,4), From 6cd9358a71aaf4d2e424a5d15964b83f03f1e1ac Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Wed, 12 Mar 2014 10:01:17 -0400 Subject: [PATCH 3/5] Ascii input reader now supports all config options per-input stream. --- src/input/readers/Ascii.cc | 59 ++++++++++++++++++++++++-------------- 1 file changed, 37 insertions(+), 22 deletions(-) diff --git a/src/input/readers/Ascii.cc b/src/input/readers/Ascii.cc index 4fd24de73d..393793c3cb 100644 --- a/src/input/readers/Ascii.cc +++ b/src/input/readers/Ascii.cc @@ -14,6 +14,7 @@ #include using namespace input::reader; +using namespace threading; using threading::Value; using threading::Field; @@ -50,27 +51,7 @@ Ascii::Ascii(ReaderFrontend *frontend) : ReaderBackend(frontend) { file = 0; mtime = 0; - - separator.assign( (const char*) BifConst::InputAscii::separator->Bytes(), - BifConst::InputAscii::separator->Len()); - - if ( separator.size() != 1 ) - Error("separator length has to be 1. Separator will be truncated."); - - set_separator.assign( (const char*) BifConst::InputAscii::set_separator->Bytes(), - BifConst::InputAscii::set_separator->Len()); - - if ( set_separator.size() != 1 ) - Error("set_separator length has to be 1. Separator will be truncated."); - - empty_field.assign( (const char*) BifConst::InputAscii::empty_field->Bytes(), - BifConst::InputAscii::empty_field->Len()); - - unset_field.assign( (const char*) BifConst::InputAscii::unset_field->Bytes(), - BifConst::InputAscii::unset_field->Len()); - - formatter = new threading::formatter::Ascii(this, threading::formatter::Ascii::SeparatorInfo(string(), set_separator, unset_field, empty_field)); -} + } Ascii::~Ascii() { @@ -90,7 +71,41 @@ void Ascii::DoClose() bool Ascii::DoInit(const ReaderInfo& info, int num_fields, const Field* const* fields) { - mtime = 0; + separator.assign( (const char*) BifConst::InputAscii::separator->Bytes(), + BifConst::InputAscii::separator->Len()); + + set_separator.assign( (const char*) BifConst::InputAscii::set_separator->Bytes(), + BifConst::InputAscii::set_separator->Len()); + + empty_field.assign( (const char*) BifConst::InputAscii::empty_field->Bytes(), + BifConst::InputAscii::empty_field->Len()); + + unset_field.assign( (const char*) BifConst::InputAscii::unset_field->Bytes(), + BifConst::InputAscii::unset_field->Len()); + + // Set per-filter configuration options. + for ( ReaderInfo::config_map::const_iterator i = info.config.begin(); i != info.config.end(); i++ ) + { + if ( strcmp(i->first, "separator") == 0 ) + separator.assign(i->second); + + else if ( strcmp(i->first, "set_separator") == 0 ) + set_separator.assign(i->second); + + else if ( strcmp(i->first, "empty_field") == 0 ) + empty_field.assign(i->second); + + else if ( strcmp(i->first, "unset_field") == 0 ) + unset_field.assign(i->second); + } + + if ( separator.size() != 1 ) + Error("separator length has to be 1. Separator will be truncated."); + + if ( set_separator.size() != 1 ) + Error("set_separator length has to be 1. Separator will be truncated."); + + formatter = new formatter::Ascii(this, formatter::Ascii::SeparatorInfo(separator, set_separator, unset_field, empty_field)); file = new ifstream(info.source); if ( ! file->is_open() ) From c591e4f57f11d46f31f332b4a2f2115fca370bf1 Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Wed, 12 Mar 2014 10:01:59 -0400 Subject: [PATCH 4/5] Expanded support for modifying the timestamp format in the JSON formatter. --- .../base/frameworks/logging/writers/ascii.bro | 6 ++-- scripts/base/init-bare.bro | 18 +++++++++++ src/logging.bif | 2 +- src/logging/writers/Ascii.cc | 30 ++++++++++++++++--- src/logging/writers/Ascii.h | 5 ++-- src/logging/writers/ElasticSearch.cc | 2 +- src/threading/formatters/JSON.cc | 14 +++++---- src/threading/formatters/JSON.h | 11 +++++-- 8 files changed, 69 insertions(+), 19 deletions(-) diff --git a/scripts/base/frameworks/logging/writers/ascii.bro b/scripts/base/frameworks/logging/writers/ascii.bro index 68219cb964..2a887e64c0 100644 --- a/scripts/base/frameworks/logging/writers/ascii.bro +++ b/scripts/base/frameworks/logging/writers/ascii.bro @@ -25,10 +25,8 @@ export { const use_json = F &redef; ## By default, the JSON formatter will use double values for timestamps - ## which represent the number of seconds from the UNIX epoch. By setting - ## this to 'T', it will use the 8601 format. This is also available as - ## a per-filter $config option. - const json_iso_timestamps = F &redef; + ## which represent the number of seconds from the UNIX epoch. + const json_timestamps: JSON::TimestampFormat = JSON::TS_EPOCH &redef; ## If true, include lines with log meta information such as column names ## with types, the values of ASCII logging options that are in use, and diff --git a/scripts/base/init-bare.bro b/scripts/base/init-bare.bro index afb89ba1c7..0af4f0fb9d 100644 --- a/scripts/base/init-bare.bro +++ b/scripts/base/init-bare.bro @@ -3057,6 +3057,24 @@ const record_all_packets = F &redef; ## .. bro:see:: conn_stats const ignore_keep_alive_rexmit = F &redef; +module JSON; +export { + type TimestampFormat: enum { + ## Timestamps will be formatted as UNIX epoch doubles. This is + ## the format that Bro typically writes out timestamps. + TS_EPOCH, + ## Timestamps will be formatted as unsigned integers that + ## represent the number of milliseconds since the UNIX + ## epoch. + TS_MILLIS, + ## Timestamps will be formatted in the ISO8601 DateTime format. + ## Subseconds are also included which isn't actually part of the + ## standard but most things that parse ISO8601 seem to be able + ## to cope with that. + TS_ISO8601, + }; +} + module Tunnel; export { ## The maximum depth of a tunnel to decapsulate until giving up. diff --git a/src/logging.bif b/src/logging.bif index f12aa50c1c..062e4dbe31 100644 --- a/src/logging.bif +++ b/src/logging.bif @@ -78,7 +78,7 @@ const set_separator: string; const empty_field: string; const unset_field: string; const use_json: bool; -const json_iso_timestamps: bool; +const json_timestamps: JSON::TimestampFormat; # Options for the DataSeries writer. diff --git a/src/logging/writers/Ascii.cc b/src/logging/writers/Ascii.cc index 976c083eda..e0d1293b70 100644 --- a/src/logging/writers/Ascii.cc +++ b/src/logging/writers/Ascii.cc @@ -11,6 +11,7 @@ #include "Ascii.h" using namespace logging::writer; +using namespace threading; using threading::Value; using threading::Field; @@ -59,7 +60,6 @@ bool Ascii::DoInit(const WriterInfo& info, int num_fields, const Field* const * output_to_stdout = BifConst::LogAscii::output_to_stdout; include_meta = BifConst::LogAscii::include_meta; use_json = BifConst::LogAscii::use_json; - json_iso_timestamps = BifConst::LogAscii::json_iso_timestamps; separator.assign( (const char*) BifConst::LogAscii::separator->Bytes(), @@ -86,6 +86,13 @@ bool Ascii::DoInit(const WriterInfo& info, int num_fields, const Field* const * BifConst::LogAscii::meta_prefix->Len() ); + ODesc tsfmt; + BifConst::LogAscii::json_timestamps->Describe(&tsfmt); + json_timestamps.assign( + (const char*) tsfmt.Bytes(), + tsfmt.Len() + ); + // Set per-filter configuration options. for ( WriterInfo::config_map::const_iterator i = info.config.begin(); i != info.config.end(); i++ ) { @@ -142,13 +149,28 @@ bool Ascii::DoInit(const WriterInfo& info, int num_fields, const Field* const * else if ( strcmp(i->first, "meta_prefix") == 0 ) meta_prefix.assign(i->second); + + else if ( strcmp(i->first, "json_timestamps") == 0 ) + json_timestamps.assign(i->second); } - if ( use_json ) { + formatter::JSON::TimeFormat tf = formatter::JSON::TS_EPOCH; // Write out JSON formatted logs. - formatter = new threading::formatter::JSON(this, json_iso_timestamps); + if ( strcmp(json_timestamps.c_str(), "JSON::TS_EPOCH") == 0 ) + tf = formatter::JSON::TS_EPOCH; + else if ( strcmp(json_timestamps.c_str(), "JSON::TS_MILLIS") == 0 ) + tf = formatter::JSON::TS_MILLIS; + else if ( strcmp(json_timestamps.c_str(), "JSON::TS_ISO8601") == 0 ) + tf = formatter::JSON::TS_ISO8601; + else + { + Error(Fmt("Invalid JSON timestamp format: %s", json_timestamps.c_str())); + return false; + } + + formatter = new formatter::JSON(this, tf); // Using JSON implicitly turns off the header meta fields. include_meta = false; } @@ -157,7 +179,7 @@ bool Ascii::DoInit(const WriterInfo& info, int num_fields, const Field* const * // Use the default "Bro logs" format. desc.EnableEscaping(); desc.AddEscapeSequence(separator); - formatter = new threading::formatter::Ascii(this, threading::formatter::Ascii::SeparatorInfo(separator, set_separator, unset_field, empty_field)); + formatter = new formatter::Ascii(this, formatter::Ascii::SeparatorInfo(separator, set_separator, unset_field, empty_field)); } string path = info.path; diff --git a/src/logging/writers/Ascii.h b/src/logging/writers/Ascii.h index d131a289b4..a70f3c2c5e 100644 --- a/src/logging/writers/Ascii.h +++ b/src/logging/writers/Ascii.h @@ -47,14 +47,15 @@ private: bool output_to_stdout; bool include_meta; bool tsv; - bool use_json; - bool json_iso_timestamps; string separator; string set_separator; string empty_field; string unset_field; string meta_prefix; + + bool use_json; + string json_timestamps; threading::formatter::Formatter* formatter; }; diff --git a/src/logging/writers/ElasticSearch.cc b/src/logging/writers/ElasticSearch.cc index 0a18e624ea..45549cb565 100644 --- a/src/logging/writers/ElasticSearch.cc +++ b/src/logging/writers/ElasticSearch.cc @@ -52,7 +52,7 @@ ElasticSearch::ElasticSearch(WriterFrontend* frontend) : WriterBackend(frontend) curl_handle = HTTPSetup(); - json = new threading::formatter::JSON(this, false); + json = new threading::formatter::JSON(this, threading::formatter::JSON::TS_MILLIS); } ElasticSearch::~ElasticSearch() diff --git a/src/threading/formatters/JSON.cc b/src/threading/formatters/JSON.cc index 57e9d58527..0523d38f8b 100644 --- a/src/threading/formatters/JSON.cc +++ b/src/threading/formatters/JSON.cc @@ -10,9 +10,9 @@ using namespace threading::formatter; -JSON::JSON(MsgThread* t, bool json_iso_timestamps) : Formatter(t) +JSON::JSON(MsgThread* t, TimeFormat tf) : Formatter(t) { - iso_timestamps = json_iso_timestamps; + timestamps = tf; } JSON::~JSON() @@ -102,7 +102,7 @@ bool JSON::Describe(ODesc* desc, Value* val) const case TYPE_TIME: { - if ( iso_timestamps ) + if ( timestamps == TS_ISO8601 ) { char buffer[40]; time_t t = time_t(val->val.double_val); @@ -118,14 +118,18 @@ bool JSON::Describe(ODesc* desc, Value* val) const desc->AddRaw("\"", 1); } } - else + else if ( timestamps == TS_EPOCH ) + { + desc->Add(val->val.double_val); + } + else if ( timestamps == TS_MILLIS ) { // ElasticSearch uses milliseconds for timestamps and json only // supports signed ints (uints can be too large). uint64_t ts = (uint64_t) (val->val.double_val * 1000); if ( ts >= INT64_MAX ) { - thread->Error(thread->Fmt("time value too large for JSON: %" PRIu64, ts)); + thread->Error(thread->Fmt("time value too large for JSON milliseconds: %" PRIu64, ts)); desc->AddRaw("null", 4); } else diff --git a/src/threading/formatters/JSON.h b/src/threading/formatters/JSON.h index b15da075cd..d21a705ce1 100644 --- a/src/threading/formatters/JSON.h +++ b/src/threading/formatters/JSON.h @@ -13,7 +13,14 @@ namespace threading { namespace formatter { */ class JSON : public Formatter { public: - JSON(threading::MsgThread* t, bool json_iso_timestamps); + + enum TimeFormat { + TS_EPOCH, // Doubles that represents seconds from the UNIX epoch. + TS_ISO8601, // ISO 8601 defined human readable timestamp format. + TS_MILLIS // Milliseconds from the UNIX epoch. Some things need this (elasticsearch). + }; + + JSON(threading::MsgThread* t, TimeFormat tf); virtual ~JSON(); virtual bool Describe(ODesc* desc, threading::Value* val) const; @@ -23,7 +30,7 @@ public: virtual threading::Value* ParseValue(string s, string name, TypeTag type, TypeTag subtype = TYPE_ERROR) const; private: - bool iso_timestamps; + TimeFormat timestamps; }; }} From ed7f658ee2f6bf63ac4b6a5bcc37e1182b6337d6 Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Wed, 12 Mar 2014 10:10:40 -0400 Subject: [PATCH 5/5] Updating a couple of tests. --- .../Baseline/scripts.base.frameworks.logging.ascii-json/ssh.log | 2 +- .../base/frameworks/logging/ascii-json-iso-timestamps.bro | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-json/ssh.log b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-json/ssh.log index 9e4e74c018..180d6a4374 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-json/ssh.log +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-json/ssh.log @@ -1 +1 @@ -{"b":true,"i":-42,"e":"SSH::LOG","c":21,"p":123,"sn":"10.0.0.0/24","a":"1.2.3.4","d":3.14,"t":1215620010543,"iv":100.0,"s":"hurz","sc":[2,4,1,3],"ss":["CC","AA","BB"],"se":[],"vc":[10,20,30],"ve":[],"f":"SSH::foo\u000a{ \u000aif (0 < SSH::i) \u000a\u0009return (Foo);\u000aelse\u000a\u0009return (Bar);\u000a\u000a}"} +{"b":true,"i":-42,"e":"SSH::LOG","c":21,"p":123,"sn":"10.0.0.0/24","a":"1.2.3.4","d":3.14,"t":1215620010.54321,"iv":100.0,"s":"hurz","sc":[2,4,1,3],"ss":["CC","AA","BB"],"se":[],"vc":[10,20,30],"ve":[],"f":"SSH::foo\u000a{ \u000aif (0 < SSH::i) \u000a\u0009return (Foo);\u000aelse\u000a\u0009return (Bar);\u000a\u000a}"} diff --git a/testing/btest/scripts/base/frameworks/logging/ascii-json-iso-timestamps.bro b/testing/btest/scripts/base/frameworks/logging/ascii-json-iso-timestamps.bro index f9541edda7..fa2a6f1efd 100644 --- a/testing/btest/scripts/base/frameworks/logging/ascii-json-iso-timestamps.bro +++ b/testing/btest/scripts/base/frameworks/logging/ascii-json-iso-timestamps.bro @@ -5,7 +5,7 @@ # Testing all possible types. redef LogAscii::use_json = T; -redef LogAscii::json_iso_timestamps = T; +redef LogAscii::json_timestamps = JSON::TS_ISO8601; module SSH;