diff --git a/scripts/base/frameworks/logging/writers/ascii.bro b/scripts/base/frameworks/logging/writers/ascii.bro index e510874951..ab86e4641d 100644 --- a/scripts/base/frameworks/logging/writers/ascii.bro +++ b/scripts/base/frameworks/logging/writers/ascii.bro @@ -17,27 +17,38 @@ module LogAscii; export { ## If true, output everything to stdout rather than ## into files. This is primarily for debugging purposes. + ## This is also available as a per-filter $config option. const output_to_stdout = F &redef; + ## If true, the default option will be to write logs in a JSON format. + ## This is also available as a per-filter $config option. + const use_json = F &redef; + ## If true, include lines with log meta information such as column names ## with types, the values of ASCII logging options that are in use, and ## the time when the file was opened and closed (the latter at the end). + ## If writing in JSON format, this is implicitly disabled. const include_meta = T &redef; - ## Prefix for lines with meta information. + ## Prefix for lines with meta information. This is also available as a + ## per-filter $config option. const meta_prefix = "#" &redef; - ## Separator between fields. + ## Separator between fields. This is also available as a per-filter + ## $config option. const separator = Log::separator &redef; - ## Separator between set elements. + ## Separator between set elements. This is also available as a + ## per-filter $config option. const set_separator = Log::set_separator &redef; ## String to use for empty fields. This should be different from - ## *unset_field* to make the output unambiguous. + ## *unset_field* to make the output unambiguous. This is also + ## available as a per-filter $config option. const empty_field = Log::empty_field &redef; - ## String to use for an unset &optional field. + ## String to use for an unset &optional field. This is also + ## available as a per-filter $config option. const unset_field = Log::unset_field &redef; } diff --git a/scripts/policy/tuning/json-logs.bro b/scripts/policy/tuning/json-logs.bro new file mode 100644 index 0000000000..e7daf35063 --- /dev/null +++ b/scripts/policy/tuning/json-logs.bro @@ -0,0 +1,4 @@ +##! Loading this script will cause all logs to be written +##! out as JSON by default. + +redef LogAscii::use_json=T; diff --git a/scripts/test-all-policy.bro b/scripts/test-all-policy.bro index 3a0bd17614..254ad69533 100644 --- a/scripts/test-all-policy.bro +++ b/scripts/test-all-policy.bro @@ -93,6 +93,7 @@ @load tuning/defaults/extracted_file_limits.bro @load tuning/defaults/packet-fragments.bro @load tuning/defaults/warnings.bro +@load tuning/json-logs.bro @load tuning/logs-to-elasticsearch.bro @load tuning/track-all-assets.bro diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index ecf8683ddd..0e990dd37b 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -335,11 +335,13 @@ set(bro_SRCS strsep.c modp_numtoa.c - threading/AsciiFormatter.cc threading/BasicThread.cc + threading/Formatter.cc threading/Manager.cc threading/MsgThread.cc threading/SerialTypes.cc + threading/formatters/Ascii.cc + threading/formatters/JSON.cc logging/Manager.cc logging/WriterBackend.cc diff --git a/src/input/readers/Ascii.cc b/src/input/readers/Ascii.cc index 3fc74480fc..4fd24de73d 100644 --- a/src/input/readers/Ascii.cc +++ b/src/input/readers/Ascii.cc @@ -69,13 +69,13 @@ Ascii::Ascii(ReaderFrontend *frontend) : ReaderBackend(frontend) unset_field.assign( (const char*) BifConst::InputAscii::unset_field->Bytes(), BifConst::InputAscii::unset_field->Len()); - ascii = new AsciiFormatter(this, AsciiFormatter::SeparatorInfo(set_separator, unset_field, empty_field)); + formatter = new threading::formatter::Ascii(this, threading::formatter::Ascii::SeparatorInfo(string(), set_separator, unset_field, empty_field)); } Ascii::~Ascii() { DoClose(); - delete ascii; + delete formatter; } void Ascii::DoClose() @@ -173,7 +173,6 @@ bool Ascii::ReadHeader(bool useCached) return false; } - FieldMapping f(field->name, field->type, field->subtype, ifields[field->name]); if ( field->secondary_name && strlen(field->secondary_name) != 0 ) @@ -200,7 +199,7 @@ bool Ascii::ReadHeader(bool useCached) bool Ascii::GetLine(string& str) { while ( getline(*file, str) ) - { + { if ( str[0] != '#' ) return true; @@ -282,7 +281,7 @@ bool Ascii::DoUpdate() file->sync(); - while ( GetLine(line ) ) + while ( GetLine(line) ) { // split on tabs bool error = false; @@ -332,7 +331,7 @@ bool Ascii::DoUpdate() return false; } - Value* val = ascii->ParseValue(stringfields[(*fit).position], (*fit).name, (*fit).type, (*fit).subtype); + Value* val = formatter->ParseValue(stringfields[(*fit).position], (*fit).name, (*fit).type, (*fit).subtype); if ( val == 0 ) { @@ -347,7 +346,7 @@ bool Ascii::DoUpdate() assert(val->type == TYPE_PORT ); // Error(Fmt("Got type %d != PORT with secondary position!", val->type)); - val->val.port_val.proto = ascii->ParseProto(stringfields[(*fit).secondary_position]); + val->val.port_val.proto = formatter->ParseProto(stringfields[(*fit).secondary_position]); } fields[fpos] = val; @@ -384,8 +383,9 @@ bool Ascii::DoUpdate() } bool Ascii::DoHeartbeat(double network_time, double current_time) -{ - switch ( Info().mode ) { + { + switch ( Info().mode ) + { case MODE_MANUAL: // yay, we do nothing :) break; @@ -398,7 +398,7 @@ bool Ascii::DoHeartbeat(double network_time, double current_time) default: assert(false); - } + } return true; } diff --git a/src/input/readers/Ascii.h b/src/input/readers/Ascii.h index 934ea9a258..5d6bc71d54 100644 --- a/src/input/readers/Ascii.h +++ b/src/input/readers/Ascii.h @@ -7,7 +7,7 @@ #include #include "../ReaderBackend.h" -#include "threading/AsciiFormatter.h" +#include "threading/formatters/Ascii.h" namespace input { namespace reader { @@ -64,7 +64,7 @@ private: string empty_field; string unset_field; - AsciiFormatter* ascii; + threading::formatter::Formatter* formatter; }; diff --git a/src/input/readers/Benchmark.cc b/src/input/readers/Benchmark.cc index ec6b382ebb..de7eae8cc8 100644 --- a/src/input/readers/Benchmark.cc +++ b/src/input/readers/Benchmark.cc @@ -29,7 +29,7 @@ Benchmark::Benchmark(ReaderFrontend *frontend) : ReaderBackend(frontend) heartbeatstarttime = 0; heartbeat_interval = double(BifConst::Threading::heartbeat_interval); - ascii = new AsciiFormatter(this, AsciiFormatter::SeparatorInfo()); + ascii = new threading::formatter::Ascii(this, threading::formatter::Ascii::SeparatorInfo()); } Benchmark::~Benchmark() diff --git a/src/input/readers/Benchmark.h b/src/input/readers/Benchmark.h index ad61b7d2a4..3296f3a85e 100644 --- a/src/input/readers/Benchmark.h +++ b/src/input/readers/Benchmark.h @@ -4,7 +4,7 @@ #define INPUT_READERS_BENCHMARK_H #include "../ReaderBackend.h" -#include "threading/AsciiFormatter.h" +#include "threading/formatters/Ascii.h" namespace input { namespace reader { @@ -40,7 +40,7 @@ private: double timedspread; double heartbeat_interval; - AsciiFormatter* ascii; + threading::formatter::Ascii* ascii; }; diff --git a/src/input/readers/SQLite.cc b/src/input/readers/SQLite.cc index 794b8059ba..d032f934a7 100644 --- a/src/input/readers/SQLite.cc +++ b/src/input/readers/SQLite.cc @@ -36,7 +36,7 @@ SQLite::SQLite(ReaderFrontend *frontend) BifConst::InputSQLite::empty_field->Len() ); - io = new AsciiFormatter(this, AsciiFormatter::SeparatorInfo(set_separator, unset_field, empty_field)); + io = new threading::formatter::Ascii(this, threading::formatter::Ascii::SeparatorInfo(string(), set_separator, unset_field, empty_field)); } SQLite::~SQLite() diff --git a/src/input/readers/SQLite.h b/src/input/readers/SQLite.h index cdea9808c0..a98b3e06b8 100644 --- a/src/input/readers/SQLite.h +++ b/src/input/readers/SQLite.h @@ -10,7 +10,7 @@ #include "../ReaderBackend.h" -#include "threading/AsciiFormatter.h" +#include "threading/formatters/Ascii.h" #include "3rdparty/sqlite3.h" namespace input { namespace reader { @@ -40,7 +40,7 @@ private: string query; sqlite3 *db; sqlite3_stmt *st; - AsciiFormatter* io; + threading::formatter::Ascii* io; string set_separator; string unset_field; diff --git a/src/logging.bif b/src/logging.bif index 1927e149c0..2e8a1eb3a4 100644 --- a/src/logging.bif +++ b/src/logging.bif @@ -77,6 +77,7 @@ const separator: string; const set_separator: string; const empty_field: string; const unset_field: string; +const use_json: bool; # Options for the DataSeries writer. diff --git a/src/logging/writers/Ascii.cc b/src/logging/writers/Ascii.cc index 1a9cc5c4cd..bd5d175d3a 100644 --- a/src/logging/writers/Ascii.cc +++ b/src/logging/writers/Ascii.cc @@ -10,8 +10,7 @@ #include "Ascii.h" -using namespace logging; -using namespace writer; +using namespace logging::writer; using threading::Value; using threading::Field; @@ -20,9 +19,46 @@ Ascii::Ascii(WriterFrontend* frontend) : WriterBackend(frontend) fd = 0; ascii_done = false; tsv = false; + } +Ascii::~Ascii() + { + if ( ! ascii_done ) + { + fprintf(stderr, "internal error: finish missing\n"); + abort(); + } + + delete formatter; + } + +bool Ascii::WriteHeaderField(const string& key, const string& val) + { + string str = meta_prefix + key + separator + val + "\n"; + + return safe_write(fd, str.c_str(), str.length()); + } + +void Ascii::CloseFile(double t) + { + if ( ! fd ) + return; + + if ( include_meta && ! tsv ) + WriteHeaderField("close", Timestamp(0)); + + safe_close(fd); + fd = 0; + } + +bool Ascii::DoInit(const WriterInfo& info, int num_fields, const Field* const * fields) + { + assert(! fd); + + // Set some default values. output_to_stdout = BifConst::LogAscii::output_to_stdout; include_meta = BifConst::LogAscii::include_meta; + use_json = BifConst::LogAscii::use_json; separator.assign( (const char*) BifConst::LogAscii::separator->Bytes(), @@ -49,48 +85,81 @@ Ascii::Ascii(WriterFrontend* frontend) : WriterBackend(frontend) BifConst::LogAscii::meta_prefix->Len() ); - desc.EnableEscaping(); - desc.AddEscapeSequence(separator); - - ascii = new AsciiFormatter(this, AsciiFormatter::SeparatorInfo(set_separator, unset_field, empty_field)); - } - -Ascii::~Ascii() - { - if ( ! ascii_done ) + // Set per-filter configuration options. + for ( WriterInfo::config_map::const_iterator i = info.config.begin(); i != info.config.end(); i++ ) { - fprintf(stderr, "internal error: finish missing\n"); - abort(); + if ( strcmp(i->first, "tsv") == 0 ) + { + if ( strcmp(i->second, "T") == 0 ) + tsv = true; + else if ( strcmp(i->second, "F") == 0 ) + tsv = false; + else + { + Error("invalid value for 'tsv', must be a string and either \"T\" or \"F\""); + return false; + } + } + + else if ( strcmp(i->first, "use_json") == 0 ) + { + if ( strcmp(i->second, "T") == 0 ) + use_json = true; + else if ( strcmp(i->second, "F") == 0 ) + use_json = false; + else + { + Error("invalid value for 'use_json', must be a string and either \"T\" or \"F\""); + return false; + } + } + + else if ( strcmp(i->first, "output_to_stdout") == 0 ) + { + if ( strcmp(i->second, "T") == 0 ) + output_to_stdout = true; + else if ( strcmp(i->second, "F") == 0 ) + output_to_stdout = false; + else + { + Error("invalid value for 'output_to_stdout', must be a string and either \"T\" or \"F\""); + return false; + } + } + + else if ( strcmp(i->first, "separator") == 0 ) + separator.assign(i->second); + + else if ( strcmp(i->first, "set_separator") == 0 ) + set_separator.assign(i->second); + + else if ( strcmp(i->first, "empty_field") == 0 ) + empty_field.assign(i->second); + + else if ( strcmp(i->first, "unset_field") == 0 ) + unset_field.assign(i->second); + + else if ( strcmp(i->first, "meta_prefix") == 0 ) + meta_prefix.assign(i->second); } - delete ascii; - } -bool Ascii::WriteHeaderField(const string& key, const string& val) - { - string str = meta_prefix + key + separator + val + "\n"; - - return safe_write(fd, str.c_str(), str.length()); - } - -void Ascii::CloseFile(double t) - { - if ( ! fd ) - return; - - if ( include_meta && ! tsv ) - WriteHeaderField("close", Timestamp(0)); - - safe_close(fd); - fd = 0; - } - -bool Ascii::DoInit(const WriterInfo& info, int num_fields, const Field* const * fields) - { - assert(! fd); + if ( use_json ) + { + // Write out JSON formatted logs. + formatter = new threading::formatter::JSON(this); + // Using JSON implicitly turns off the header meta fields. + include_meta = false; + } + else + { + // Use the default "Bro logs" format. + desc.EnableEscaping(); + desc.AddEscapeSequence(separator); + formatter = new threading::formatter::Ascii(this, threading::formatter::Ascii::SeparatorInfo(separator, set_separator, unset_field, empty_field)); + } string path = info.path; - if ( output_to_stdout ) path = "/dev/stdout"; @@ -106,24 +175,6 @@ bool Ascii::DoInit(const WriterInfo& info, int num_fields, const Field* const * return false; } - for ( WriterInfo::config_map::const_iterator i = info.config.begin(); i != info.config.end(); i++ ) - { - if ( strcmp(i->first, "tsv") == 0 ) - { - if ( strcmp(i->second, "T") == 0 ) - tsv = true; - - else if ( strcmp(i->second, "F") == 0 ) - tsv = false; - - else - { - Error("invalid value for 'tsv', must be a string and either \"T\" or \"F\""); - return false; - } - } - } - if ( include_meta ) { string names; @@ -208,17 +259,9 @@ bool Ascii::DoWrite(int num_fields, const Field* const * fields, DoInit(Info(), NumFields(), Fields()); desc.Clear(); - - for ( int i = 0; i < num_fields; i++ ) - { - if ( i > 0 ) - desc.AddRaw(separator); - - if ( ! ascii->Describe(&desc, vals[i], fields[i]->name) ) - return false; - } - - desc.AddRaw("\n", 1); + if ( ! formatter->Describe(&desc, num_fields, fields, vals) ) + return false; + desc.AddRaw("\n"); const char* bytes = (const char*)desc.Bytes(); int len = desc.Len(); diff --git a/src/logging/writers/Ascii.h b/src/logging/writers/Ascii.h index 0ddcada57b..08af465ff4 100644 --- a/src/logging/writers/Ascii.h +++ b/src/logging/writers/Ascii.h @@ -6,7 +6,8 @@ #define LOGGING_WRITER_ASCII_H #include "../WriterBackend.h" -#include "threading/AsciiFormatter.h" +#include "threading/formatters/Ascii.h" +#include "threading/formatters/JSON.h" namespace logging { namespace writer { @@ -46,6 +47,7 @@ private: bool output_to_stdout; bool include_meta; bool tsv; + bool use_json; string separator; string set_separator; @@ -53,7 +55,7 @@ private: string unset_field; string meta_prefix; - AsciiFormatter* ascii; + threading::formatter::Formatter* formatter; }; } diff --git a/src/logging/writers/ElasticSearch.cc b/src/logging/writers/ElasticSearch.cc index b22c3f4ab6..cec5bac80e 100644 --- a/src/logging/writers/ElasticSearch.cc +++ b/src/logging/writers/ElasticSearch.cc @@ -16,7 +16,6 @@ #include "BroString.h" #include "NetVar.h" #include "threading/SerialTypes.h" -#include "threading/AsciiFormatter.h" #include #include @@ -53,13 +52,13 @@ ElasticSearch::ElasticSearch(WriterFrontend* frontend) : WriterBackend(frontend) curl_handle = HTTPSetup(); - ascii = new AsciiFormatter(this, AsciiFormatter::SeparatorInfo()); + json = new threading::formatter::JSON(this); } ElasticSearch::~ElasticSearch() { delete [] cluster_name; - delete ascii; + delete json; } bool ElasticSearch::DoInit(const WriterInfo& info, int num_fields, const threading::Field* const* fields) @@ -98,133 +97,7 @@ bool ElasticSearch::BatchIndex() return true; } -bool ElasticSearch::AddValueToBuffer(ODesc* b, Value* val) - { - switch ( val->type ) - { - // ES treats 0 as false and any other value as true so bool types go here. - case TYPE_BOOL: - case TYPE_INT: - b->Add(val->val.int_val); - break; - case TYPE_COUNT: - case TYPE_COUNTER: - { - // ElasticSearch doesn't seem to support unsigned 64bit ints. - if ( val->val.uint_val >= INT64_MAX ) - { - Error(Fmt("count value too large: %" PRIu64, val->val.uint_val)); - b->AddRaw("null", 4); - } - else - b->Add(val->val.uint_val); - break; - } - - case TYPE_PORT: - b->Add(val->val.port_val.port); - break; - - case TYPE_SUBNET: - b->AddRaw("\"", 1); - b->Add(ascii->Render(val->val.subnet_val)); - b->AddRaw("\"", 1); - break; - - case TYPE_ADDR: - b->AddRaw("\"", 1); - b->Add(ascii->Render(val->val.addr_val)); - b->AddRaw("\"", 1); - break; - - case TYPE_DOUBLE: - case TYPE_INTERVAL: - b->Add(val->val.double_val); - break; - - case TYPE_TIME: - { - // ElasticSearch uses milliseconds for timestamps and json only - // supports signed ints (uints can be too large). - uint64_t ts = (uint64_t) (val->val.double_val * 1000); - if ( ts >= INT64_MAX ) - { - Error(Fmt("time value too large: %" PRIu64, ts)); - b->AddRaw("null", 4); - } - else - b->Add(ts); - break; - } - - case TYPE_ENUM: - case TYPE_STRING: - case TYPE_FILE: - case TYPE_FUNC: - { - b->AddRaw("\"", 1); - for ( int i = 0; i < val->val.string_val.length; ++i ) - { - char c = val->val.string_val.data[i]; - // 2byte Unicode escape special characters. - if ( c < 32 || c > 126 || c == '\n' || c == '"' || c == '\'' || c == '\\' || c == '&' ) - { - static const char hex_chars[] = "0123456789abcdef"; - b->AddRaw("\\u00", 4); - b->AddRaw(&hex_chars[(c & 0xf0) >> 4], 1); - b->AddRaw(&hex_chars[c & 0x0f], 1); - } - else - b->AddRaw(&c, 1); - } - b->AddRaw("\"", 1); - break; - } - - case TYPE_TABLE: - { - b->AddRaw("[", 1); - for ( int j = 0; j < val->val.set_val.size; j++ ) - { - if ( j > 0 ) - b->AddRaw(",", 1); - AddValueToBuffer(b, val->val.set_val.vals[j]); - } - b->AddRaw("]", 1); - break; - } - - case TYPE_VECTOR: - { - b->AddRaw("[", 1); - for ( int j = 0; j < val->val.vector_val.size; j++ ) - { - if ( j > 0 ) - b->AddRaw(",", 1); - AddValueToBuffer(b, val->val.vector_val.vals[j]); - } - b->AddRaw("]", 1); - break; - } - - default: - return false; - } - return true; - } - -bool ElasticSearch::AddFieldToBuffer(ODesc *b, Value* val, const Field* field) - { - if ( ! val->present ) - return false; - - b->AddRaw("\"", 1); - b->Add(field->name); - b->AddRaw("\":", 2); - AddValueToBuffer(b, val); - return true; - } bool ElasticSearch::DoWrite(int num_fields, const Field* const * fields, Value** vals) @@ -239,14 +112,7 @@ bool ElasticSearch::DoWrite(int num_fields, const Field* const * fields, buffer.Add(Info().path); buffer.AddRaw("\"}}\n", 4); - buffer.AddRaw("{", 1); - for ( int i = 0; i < num_fields; i++ ) - { - if ( i > 0 && buffer.Bytes()[buffer.Len()] != ',' && vals[i]->present ) - buffer.AddRaw(",", 1); - AddFieldToBuffer(&buffer, vals[i], fields[i]); - } - buffer.AddRaw("}\n", 2); + json->Describe(&buffer, num_fields, fields, vals); counter++; if ( counter >= BifConst::LogElasticSearch::max_batch_size || diff --git a/src/logging/writers/ElasticSearch.h b/src/logging/writers/ElasticSearch.h index b7aba09964..283fff2972 100644 --- a/src/logging/writers/ElasticSearch.h +++ b/src/logging/writers/ElasticSearch.h @@ -9,6 +9,7 @@ #define LOGGING_WRITER_ELASTICSEARCH_H #include +#include "threading/formatters/JSON.h" #include "../WriterBackend.h" namespace logging { namespace writer { @@ -73,7 +74,7 @@ private: uint64 batch_size; - AsciiFormatter* ascii; + threading::formatter::JSON* json; }; } diff --git a/src/logging/writers/SQLite.cc b/src/logging/writers/SQLite.cc index 25f5cb012e..968345696d 100644 --- a/src/logging/writers/SQLite.cc +++ b/src/logging/writers/SQLite.cc @@ -35,7 +35,7 @@ SQLite::SQLite(WriterFrontend* frontend) BifConst::LogSQLite::empty_field->Len() ); - io = new AsciiFormatter(this, AsciiFormatter::SeparatorInfo(set_separator, unset_field, empty_field)); + io = new threading::formatter::Ascii(this, threading::formatter::Ascii::SeparatorInfo(string(), set_separator, unset_field, empty_field)); } SQLite::~SQLite() diff --git a/src/logging/writers/SQLite.h b/src/logging/writers/SQLite.h index b70381ebf6..a962e903ff 100644 --- a/src/logging/writers/SQLite.h +++ b/src/logging/writers/SQLite.h @@ -9,7 +9,7 @@ #include "../WriterBackend.h" -#include "threading/AsciiFormatter.h" +#include "threading/formatters/Ascii.h" #include "3rdparty/sqlite3.h" namespace logging { namespace writer { @@ -51,7 +51,7 @@ private: string unset_field; string empty_field; - AsciiFormatter* io; + threading::formatter::Ascii* io; }; } diff --git a/src/threading/Formatter.cc b/src/threading/Formatter.cc new file mode 100644 index 0000000000..12037a741b --- /dev/null +++ b/src/threading/Formatter.cc @@ -0,0 +1,149 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#include "config.h" + +#include +#include + +#include "Formatter.h" +#include "bro_inet_ntop.h" + +using namespace threading; +using namespace formatter; +using threading::Value; +using threading::Field; + +Formatter::Formatter(threading::MsgThread* t) + { + thread = t; + } + +Formatter::~Formatter() + { + } + +bool Formatter::CheckNumberError(const string& s, const char* end) const + { + // Do this check first, before executing s.c_str() or similar. + // otherwise the value to which *end is pointing at the moment might + // be gone ... + bool endnotnull = (*end != '\0'); + + if ( s.length() == 0 ) + { + thread->Error("Got empty string for number field"); + return true; + } + + if ( end == s.c_str() ) { + thread->Error(thread->Fmt("String '%s' contained no parseable number", s.c_str())); + return true; + } + + if ( endnotnull ) + thread->Warning(thread->Fmt("Number '%s' contained non-numeric trailing characters. Ignored trailing characters '%s'", s.c_str(), end)); + + if ( errno == EINVAL ) + { + thread->Error(thread->Fmt("String '%s' could not be converted to a number", s.c_str())); + return true; + } + + else if ( errno == ERANGE ) + { + thread->Error(thread->Fmt("Number '%s' out of supported range.", s.c_str())); + return true; + } + + return false; + } + +string Formatter::Render(const threading::Value::addr_t& addr) const + { + if ( addr.family == IPv4 ) + { + char s[INET_ADDRSTRLEN]; + + if ( ! bro_inet_ntop(AF_INET, &addr.in.in4, s, INET_ADDRSTRLEN) ) + return ""; + else + return s; + } + else + { + char s[INET6_ADDRSTRLEN]; + + if ( ! bro_inet_ntop(AF_INET6, &addr.in.in6, s, INET6_ADDRSTRLEN) ) + return ""; + else + return s; + } + } + +TransportProto Formatter::ParseProto(const string &proto) const + { + if ( proto == "unknown" ) + return TRANSPORT_UNKNOWN; + else if ( proto == "tcp" ) + return TRANSPORT_TCP; + else if ( proto == "udp" ) + return TRANSPORT_UDP; + else if ( proto == "icmp" ) + return TRANSPORT_ICMP; + + thread->Error(thread->Fmt("Tried to parse invalid/unknown protocol: %s", proto.c_str())); + + return TRANSPORT_UNKNOWN; + } + + +// More or less verbose copy from IPAddr.cc -- which uses reporter. +threading::Value::addr_t Formatter::ParseAddr(const string &s) const + { + threading::Value::addr_t val; + + if ( s.find(':') == std::string::npos ) // IPv4. + { + val.family = IPv4; + + if ( inet_aton(s.c_str(), &(val.in.in4)) <= 0 ) + { + thread->Error(thread->Fmt("Bad address: %s", s.c_str())); + memset(&val.in.in4.s_addr, 0, sizeof(val.in.in4.s_addr)); + } + } + + else + { + val.family = IPv6; + if ( inet_pton(AF_INET6, s.c_str(), val.in.in6.s6_addr) <=0 ) + { + thread->Error(thread->Fmt("Bad address: %s", s.c_str())); + memset(val.in.in6.s6_addr, 0, sizeof(val.in.in6.s6_addr)); + } + } + + return val; + } + +string Formatter::Render(const threading::Value::subnet_t& subnet) const + { + char l[16]; + + if ( subnet.prefix.family == IPv4 ) + modp_uitoa10(subnet.length - 96, l); + else + modp_uitoa10(subnet.length, l); + + string s = Render(subnet.prefix) + "/" + l; + + return s; + } + +string Formatter::Render(double d) const + { + char buf[256]; + modp_dtoa(d, buf, 6); + return buf; + } + diff --git a/src/threading/AsciiFormatter.h b/src/threading/Formatter.h similarity index 55% rename from src/threading/AsciiFormatter.h rename to src/threading/Formatter.h index 1522e46191..c5f6cb4296 100644 --- a/src/threading/AsciiFormatter.h +++ b/src/threading/Formatter.h @@ -1,42 +1,20 @@ // See the file "COPYING" in the main distribution directory for copyright. -#ifndef THREADING_ASCII_FORMATTER_H -#define THREADING_ASCII_FORMATTER_H +#ifndef THREADING_FORMATTER_H +#define THREADING_FORMATTER_H #include "../Desc.h" #include "MsgThread.h" +namespace threading { namespace formatter { + /** - * A thread-safe class for converting values into a readable ASCII - * representation, and vice versa. This is a utility class that factors out - * common rendering/parsing code needed by a number of input/output threads. + * A thread-safe class for converting values into some textual format. + * This is a base class that implements the interface forcommon + * rendering/parsing code needed by a number of input/output threads. */ -class AsciiFormatter { +class Formatter { public: - /** - * A struct to pass the necessary configuration values to the - * AsciiFormatter module on initialization. - */ - struct SeparatorInfo - { - string set_separator; // Separator between set elements. - string unset_field; // String marking an unset field. - string empty_field; // String marking an empty (but set) field. - - /** - * Constructor that leaves separators etc unset to dummy - * values. Useful if you use only methods that don't need any - * of them, like StringToAddr, etc. - */ - SeparatorInfo(); - - /** - * Constructor that defines all the configuration options. - * Use if you need either ValToODesc or EntryToVal. - */ - SeparatorInfo(const string& set_separator, const string& unset_field, const string& empty_field); - }; - /** * Constructor. * @@ -44,31 +22,69 @@ public: * some of the thread's methods, e.g., for error reporting and * internal formatting. * - * @param info SeparatorInfo structure defining the necessary - * separators. */ - AsciiFormatter(threading::MsgThread* t, const SeparatorInfo info); + Formatter(threading::MsgThread* t); /** * Destructor. */ - ~AsciiFormatter(); + virtual ~Formatter(); /** - * Convert a threading value into a corresponding ASCII. - * representation. + * Convert a list of threading values into an implementation specific representation. + * + * @param desc The ODesc object to write to. + * + * @param num_fields The number of fields in the logging record. + * + * @param fields Information about the fields for each of the given log values. + * + * @param vals The field values. + * + * @return Returns true on success, false on error. Errors are also + * flagged via the reporter. + */ + virtual bool Describe(ODesc* desc, int num_fields, const threading::Field* const * fields, + threading::Value** vals) const = 0; + + /** + * Convert a threading value into an implementation specific representation. * * @param desc The ODesc object to write to. * * @param val the Value to render to the ODesc object. * - * @param The name of a field associated with the value. Used only - * for error reporting. + * @return Returns true on success, false on error. Errors are also + * flagged via the reporter. + */ + virtual bool Describe(ODesc* desc, threading::Value* val) const = 0; + + /** + * Convert a threading value into an implementation specific representation. + * + * @param desc The ODesc object to write to. + * + * @param val the Value to render to the ODesc object. + * + * @param The name of a field associated with the value. * * @return Returns true on success, false on error. Errors are also * flagged via the reporter. */ - bool Describe(ODesc* desc, threading::Value* val, const string& name) const; + virtual bool Describe(ODesc* desc, threading::Value* val, const string& name) const = 0; + + /** + * Convert a representation of a field into a value. + * + * @param s The string to parse. + * + * @param The name of a field associated with the value. Used only + * for error reporting. + * + * @return The new value, or null on error. Errors are also flagged + * via the reporter. + */ + virtual threading::Value* ParseValue(string s, string name, TypeTag type, TypeTag subtype = TYPE_ERROR) const = 0; /** * Convert an IP address into a string. @@ -98,19 +114,6 @@ public: */ string Render(double d) const; - /** - * Convert the ASCII representation of a field into a value. - * - * @param s The string to parse. - * - * @param The name of a field associated with the value. Used only - * for error reporting. - * - * @return The new value, or null on error. Errors are also flagged - * via the reporter. - */ - threading::Value* ParseValue(string s, string name, TypeTag type, TypeTag subtype = TYPE_ERROR) const; - /** * Convert a string into a TransportProto. The string must be one of * \c tcp, \c udp, \c icmp, or \c unknown. @@ -132,11 +135,12 @@ public: */ threading::Value::addr_t ParseAddr(const string &addr) const; -private: +protected: bool CheckNumberError(const string& s, const char * end) const; - SeparatorInfo separators; threading::MsgThread* thread; }; -#endif /* THREADING_ASCII_FORMATTER_H */ +}} + +#endif /* THREADING_FORMATTER_H */ diff --git a/src/threading/AsciiFormatter.cc b/src/threading/formatters/Ascii.cc similarity index 67% rename from src/threading/AsciiFormatter.cc rename to src/threading/formatters/Ascii.cc index 616abbe2b6..6457f3c782 100644 --- a/src/threading/AsciiFormatter.cc +++ b/src/threading/formatters/Ascii.cc @@ -5,35 +5,59 @@ #include #include -#include "AsciiFormatter.h" -#include "bro_inet_ntop.h" +#include "./Ascii.h" -AsciiFormatter::SeparatorInfo::SeparatorInfo() +using namespace threading::formatter; + +Ascii::SeparatorInfo::SeparatorInfo() { + this->separator = "SHOULD_NOT_BE_USED"; this->set_separator = "SHOULD_NOT_BE_USED"; this->unset_field = "SHOULD_NOT_BE_USED"; this->empty_field = "SHOULD_NOT_BE_USED"; } -AsciiFormatter::SeparatorInfo::SeparatorInfo(const string & set_separator, - const string & unset_field, const string & empty_field) +Ascii::SeparatorInfo::SeparatorInfo(const string & separator, + const string & set_separator, + const string & unset_field, + const string & empty_field) { + this->separator = separator; this->set_separator = set_separator; this->unset_field = unset_field; this->empty_field = empty_field; } -AsciiFormatter::AsciiFormatter(threading::MsgThread* t, const SeparatorInfo info) +Ascii::Ascii(threading::MsgThread* t, const SeparatorInfo info) : Formatter(t) { - thread = t; this->separators = info; } -AsciiFormatter::~AsciiFormatter() +Ascii::~Ascii() { } -bool AsciiFormatter::Describe(ODesc* desc, threading::Value* val, const string& name) const +bool Ascii::Describe(ODesc* desc, int num_fields, const threading::Field* const * fields, + threading::Value** vals) const + { + for ( int i = 0; i < num_fields; i++ ) + { + if ( i > 0 ) + desc->AddRaw(separators.separator); + + if ( ! Describe(desc, vals[i], fields[i]->name) ) + return false; + } + + return true; + } + +bool Ascii::Describe(ODesc* desc, threading::Value* val, const string& name) const + { + return Describe(desc, val); + } + +bool Ascii::Describe(ODesc* desc, threading::Value* val) const { if ( ! val->present ) { @@ -133,7 +157,7 @@ bool AsciiFormatter::Describe(ODesc* desc, threading::Value* val, const string& if ( j > 0 ) desc->AddRaw(separators.set_separator); - if ( ! Describe(desc, val->val.set_val.vals[j], name) ) + if ( ! Describe(desc, val->val.set_val.vals[j]) ) { desc->RemoveEscapeSequence(separators.set_separator); return false; @@ -158,7 +182,7 @@ bool AsciiFormatter::Describe(ODesc* desc, threading::Value* val, const string& if ( j > 0 ) desc->AddRaw(separators.set_separator); - if ( ! Describe(desc, val->val.vector_val.vals[j], name) ) + if ( ! Describe(desc, val->val.vector_val.vals[j]) ) { desc->RemoveEscapeSequence(separators.set_separator); return false; @@ -170,7 +194,7 @@ bool AsciiFormatter::Describe(ODesc* desc, threading::Value* val, const string& } default: - thread->Error(thread->Fmt("unsupported field format %d for %s", val->type, name.c_str())); + thread->Error(thread->Fmt("Ascii writer unsupported field format %d", val->type)); return false; } @@ -178,7 +202,7 @@ bool AsciiFormatter::Describe(ODesc* desc, threading::Value* val, const string& } -threading::Value* AsciiFormatter::ParseValue(string s, string name, TypeTag type, TypeTag subtype) const +threading::Value* Ascii::ParseValue(string s, string name, TypeTag type, TypeTag subtype) const { if ( s.compare(separators.unset_field) == 0 ) // field is not set... return new threading::Value(type, false); @@ -381,129 +405,3 @@ parse_error: delete val; return 0; } - -bool AsciiFormatter::CheckNumberError(const string& s, const char* end) const - { - // Do this check first, before executing s.c_str() or similar. - // otherwise the value to which *end is pointing at the moment might - // be gone ... - bool endnotnull = (*end != '\0'); - - if ( s.length() == 0 ) - { - thread->Error("Got empty string for number field"); - return true; - } - - if ( end == s.c_str() ) { - thread->Error(thread->Fmt("String '%s' contained no parseable number", s.c_str())); - return true; - } - - if ( endnotnull ) - thread->Warning(thread->Fmt("Number '%s' contained non-numeric trailing characters. Ignored trailing characters '%s'", s.c_str(), end)); - - if ( errno == EINVAL ) - { - thread->Error(thread->Fmt("String '%s' could not be converted to a number", s.c_str())); - return true; - } - - else if ( errno == ERANGE ) - { - thread->Error(thread->Fmt("Number '%s' out of supported range.", s.c_str())); - return true; - } - - return false; - } - -string AsciiFormatter::Render(const threading::Value::addr_t& addr) const - { - if ( addr.family == IPv4 ) - { - char s[INET_ADDRSTRLEN]; - - if ( ! bro_inet_ntop(AF_INET, &addr.in.in4, s, INET_ADDRSTRLEN) ) - return ""; - else - return s; - } - else - { - char s[INET6_ADDRSTRLEN]; - - if ( ! bro_inet_ntop(AF_INET6, &addr.in.in6, s, INET6_ADDRSTRLEN) ) - return ""; - else - return s; - } - } - -TransportProto AsciiFormatter::ParseProto(const string &proto) const - { - if ( proto == "unknown" ) - return TRANSPORT_UNKNOWN; - else if ( proto == "tcp" ) - return TRANSPORT_TCP; - else if ( proto == "udp" ) - return TRANSPORT_UDP; - else if ( proto == "icmp" ) - return TRANSPORT_ICMP; - - thread->Error(thread->Fmt("Tried to parse invalid/unknown protocol: %s", proto.c_str())); - - return TRANSPORT_UNKNOWN; - } - - -// More or less verbose copy from IPAddr.cc -- which uses reporter. -threading::Value::addr_t AsciiFormatter::ParseAddr(const string &s) const - { - threading::Value::addr_t val; - - if ( s.find(':') == std::string::npos ) // IPv4. - { - val.family = IPv4; - - if ( inet_aton(s.c_str(), &(val.in.in4)) <= 0 ) - { - thread->Error(thread->Fmt("Bad address: %s", s.c_str())); - memset(&val.in.in4.s_addr, 0, sizeof(val.in.in4.s_addr)); - } - } - - else - { - val.family = IPv6; - if ( inet_pton(AF_INET6, s.c_str(), val.in.in6.s6_addr) <=0 ) - { - thread->Error(thread->Fmt("Bad address: %s", s.c_str())); - memset(val.in.in6.s6_addr, 0, sizeof(val.in.in6.s6_addr)); - } - } - - return val; - } - -string AsciiFormatter::Render(const threading::Value::subnet_t& subnet) const - { - char l[16]; - - if ( subnet.prefix.family == IPv4 ) - modp_uitoa10(subnet.length - 96, l); - else - modp_uitoa10(subnet.length, l); - - string s = Render(subnet.prefix) + "/" + l; - - return s; - } - -string AsciiFormatter::Render(double d) const - { - char buf[256]; - modp_dtoa(d, buf, 6); - return buf; - } - diff --git a/src/threading/formatters/Ascii.h b/src/threading/formatters/Ascii.h new file mode 100644 index 0000000000..d4b8eec052 --- /dev/null +++ b/src/threading/formatters/Ascii.h @@ -0,0 +1,62 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#ifndef THREADING_FORMATTERS_ASCII_H +#define THREADING_FORMATTERS_ASCII_H + +#include "../Formatter.h" + +namespace threading { namespace formatter { + +class Ascii : public Formatter { +public: + /** + * A struct to pass the necessary configuration values to the + * Ascii module on initialization. + */ + struct SeparatorInfo + { + string separator; // Separator between columns + string set_separator; // Separator between set elements. + string unset_field; // String marking an unset field. + string empty_field; // String marking an empty (but set) field. + + /** + * Constructor that leaves separators etc unset to dummy + * values. Useful if you use only methods that don't need any + * of them, like StringToAddr, etc. + */ + SeparatorInfo(); + + /** + * Constructor that defines all the configuration options. + * Use if you need either ValToODesc or EntryToVal. + */ + SeparatorInfo(const string& separator, const string& set_separator, const string& unset_field, const string& empty_field); + }; + + /** + * Constructor. + * + * @param t The thread that uses this class instance. The class uses + * some of the thread's methods, e.g., for error reporting and + * internal formatting. + * + * @param info SeparatorInfo structure defining the necessary + * separators. + */ + Ascii(threading::MsgThread* t, const SeparatorInfo info); + virtual ~Ascii(); + + virtual bool Describe(ODesc* desc, threading::Value* val) const; + virtual bool Describe(ODesc* desc, threading::Value* val, const string& name) const; + virtual bool Describe(ODesc* desc, int num_fields, const threading::Field* const * fields, + threading::Value** vals) const; + virtual threading::Value* ParseValue(string s, string name, TypeTag type, TypeTag subtype = TYPE_ERROR) const; + +private: + SeparatorInfo separators; +}; + +}} + +#endif /* THREADING_FORMATTERS_ASCII_H */ diff --git a/src/threading/formatters/JSON.cc b/src/threading/formatters/JSON.cc new file mode 100644 index 0000000000..c1e68b27bb --- /dev/null +++ b/src/threading/formatters/JSON.cc @@ -0,0 +1,177 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#include "config.h" + +#include +#include + +#include "./JSON.h" + +using namespace threading::formatter; + +JSON::JSON(MsgThread* t) : Formatter(t) + { + } + +JSON::~JSON() + { + } + +bool JSON::Describe(ODesc* desc, int num_fields, const Field* const * fields, + Value** vals) const + { + desc->AddRaw("{"); + for ( int i = 0; i < num_fields; i++ ) + { + if ( i > 0 && + desc->Bytes()[desc->Len()] != ',' && vals[i]->present ) + { + desc->AddRaw(","); + } + + if ( ! Describe(desc, vals[i], fields[i]->name) ) + return false; + } + desc->AddRaw("}"); + return true; + } + +bool JSON::Describe(ODesc* desc, Value* val, const string& name) const + { + if ( ! val->present ) + return true; + + desc->AddRaw("\"", 1); + desc->Add(name); + desc->AddRaw("\":", 2); + + return Describe(desc, val); + } + +bool JSON::Describe(ODesc* desc, Value* val) const + { + if ( ! val->present ) + return true; + + switch ( val->type ) + { + case TYPE_BOOL: + desc->AddRaw(val->val.int_val == 0 ? "false" : "true"); + break; + + case TYPE_INT: + desc->Add(val->val.int_val); + break; + + case TYPE_COUNT: + case TYPE_COUNTER: + { + // JSON doesn't support unsigned 64bit ints. + if ( val->val.uint_val >= INT64_MAX ) + { + thread->Error(thread->Fmt("count value too large for JSON: %" PRIu64, val->val.uint_val)); + desc->AddRaw("null", 4); + } + else + desc->Add(val->val.uint_val); + break; + } + + case TYPE_PORT: + desc->Add(val->val.port_val.port); + break; + + case TYPE_SUBNET: + desc->AddRaw("\"", 1); + desc->Add(Render(val->val.subnet_val)); + desc->AddRaw("\"", 1); + break; + + case TYPE_ADDR: + desc->AddRaw("\"", 1); + desc->Add(Render(val->val.addr_val)); + desc->AddRaw("\"", 1); + break; + + case TYPE_DOUBLE: + case TYPE_INTERVAL: + desc->Add(val->val.double_val); + break; + + case TYPE_TIME: + { + // ElasticSearch uses milliseconds for timestamps and json only + // supports signed ints (uints can be too large). + uint64_t ts = (uint64_t) (val->val.double_val * 1000); + if ( ts >= INT64_MAX ) + { + thread->Error(thread->Fmt("time value too large for JSON: %" PRIu64, ts)); + desc->AddRaw("null", 4); + } + else + desc->Add(ts); + break; + } + + case TYPE_ENUM: + case TYPE_STRING: + case TYPE_FILE: + case TYPE_FUNC: + { + desc->AddRaw("\"", 1); + for ( int i = 0; i < val->val.string_val.length; ++i ) + { + char c = val->val.string_val.data[i]; + // 2byte Unicode escape special characters. + if ( c < 32 || c > 126 || c == '\n' || c == '"' || c == '\'' || c == '\\' || c == '&' ) + { + static const char hex_chars[] = "0123456789abcdef"; + desc->AddRaw("\\u00", 4); + desc->AddRaw(&hex_chars[(c & 0xf0) >> 4], 1); + desc->AddRaw(&hex_chars[c & 0x0f], 1); + } + else + desc->AddRaw(&c, 1); + } + desc->AddRaw("\"", 1); + break; + } + + case TYPE_TABLE: + { + desc->AddRaw("[", 1); + for ( int j = 0; j < val->val.set_val.size; j++ ) + { + if ( j > 0 ) + desc->AddRaw(",", 1); + Describe(desc, val->val.set_val.vals[j]); + } + desc->AddRaw("]", 1); + break; + } + + case TYPE_VECTOR: + { + desc->AddRaw("[", 1); + for ( int j = 0; j < val->val.vector_val.size; j++ ) + { + if ( j > 0 ) + desc->AddRaw(",", 1); + Describe(desc, val->val.vector_val.vals[j]); + } + desc->AddRaw("]", 1); + break; + } + + default: + return false; + } + + return true; + } + +threading::Value* JSON::ParseValue(string s, string name, TypeTag type, TypeTag subtype) const + { + thread->Error("JSON formatter does not support parsing yet."); + return NULL; + } diff --git a/src/threading/formatters/JSON.h b/src/threading/formatters/JSON.h new file mode 100644 index 0000000000..7b4c4ea328 --- /dev/null +++ b/src/threading/formatters/JSON.h @@ -0,0 +1,30 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#ifndef THREADING_FORMATTERS_JSON_H +#define THREADING_FORMATTERS_JSON_H + +#include "../Formatter.h" + +namespace threading { namespace formatter { + +/** + * A thread-safe class for converting values into a JSON representation + * and vice versa. + */ +class JSON : public Formatter { +public: + JSON(threading::MsgThread* t); + virtual ~JSON(); + + virtual bool Describe(ODesc* desc, threading::Value* val) const; + virtual bool Describe(ODesc* desc, threading::Value* val, const string& name) const; + virtual bool Describe(ODesc* desc, int num_fields, const threading::Field* const * fields, + threading::Value** vals) const; + virtual threading::Value* ParseValue(string s, string name, TypeTag type, TypeTag subtype = TYPE_ERROR) const; + +private: +}; + +}} + +#endif /* THREADING_FORMATTERS_JSON_H */ diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-json/ssh.log b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-json/ssh.log new file mode 100644 index 0000000000..7446047266 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-json/ssh.log @@ -0,0 +1 @@ +{"b":true,"i":-42,"e":"SSH::LOG","c":21,"p":123,"sn":"10.0.0.0/24","a":"1.2.3.4","d":3.14,"t":1394462315468,"iv":100.0,"s":"hurz","sc":[2,4,1,3],"ss":["CC","AA","BB"],"se":[],"vc":[10,20,30],"ve":[],"f":"SSH::foo\u000a{ \u000aif (0 < SSH::i) \u000a\u0009return (Foo);\u000aelse\u000a\u0009return (Bar);\u000a\u000a}"} diff --git a/testing/btest/scripts/base/frameworks/logging/ascii-json.bro b/testing/btest/scripts/base/frameworks/logging/ascii-json.bro new file mode 100644 index 0000000000..35cf7f1e80 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/logging/ascii-json.bro @@ -0,0 +1,70 @@ +# +# @TEST-EXEC: bro -b %INPUT +# @TEST-EXEC: btest-diff ssh.log +# +# Testing all possible types. + +redef LogAscii::use_json = T; + +module SSH; + +export { + redef enum Log::ID += { LOG }; + + type Log: record { + b: bool; + i: int; + e: Log::ID; + c: count; + p: port; + sn: subnet; + a: addr; + d: double; + t: time; + iv: interval; + s: string; + sc: set[count]; + ss: set[string]; + se: set[string]; + vc: vector of count; + ve: vector of string; + f: function(i: count) : string; + } &log; +} + +function foo(i : count) : string + { + if ( i > 0 ) + return "Foo"; + else + return "Bar"; + } + +event bro_init() +{ + Log::create_stream(SSH::LOG, [$columns=Log]); + + local empty_set: set[string]; + local empty_vector: vector of string; + + Log::write(SSH::LOG, [ + $b=T, + $i=-42, + $e=SSH::LOG, + $c=21, + $p=123/tcp, + $sn=10.0.0.1/24, + $a=1.2.3.4, + $d=3.14, + $t=network_time(), + $iv=100secs, + $s="hurz", + $sc=set(1,2,3,4), + $ss=set("AA", "BB", "CC"), + $se=empty_set, + $vc=vector(10, 20, 30), + $ve=empty_vector, + $f=foo + ]); +} +