diff --git a/scripts/base/frameworks/logging/main.bro b/scripts/base/frameworks/logging/main.bro index 8c2f507aff..054ad4a30b 100644 --- a/scripts/base/frameworks/logging/main.bro +++ b/scripts/base/frameworks/logging/main.bro @@ -17,6 +17,23 @@ export { ## anything else. const default_writer = WRITER_ASCII &redef; + ## Default separator between fields for logwriters. + ## Can be overwritten by individual writers. + const separator = "\t" &redef; + + ## Separator between set elements. + ## Can be overwritten by individual writers. + const set_separator = "," &redef; + + ## String to use for empty fields. This should be different from + ## *unset_field* to make the output non-ambigious. + ## Can be overwritten by individual writers. + const empty_field = "(empty)" &redef; + + ## String to use for an unset &optional field. + ## Can be overwritten by individual writers. + const unset_field = "-" &redef; + ## Type defining the content of a logging stream. type Stream: record { ## A record type defining the log's columns. diff --git a/scripts/base/frameworks/logging/writers/ascii.bro b/scripts/base/frameworks/logging/writers/ascii.bro index 800af51502..dc3910d767 100644 --- a/scripts/base/frameworks/logging/writers/ascii.bro +++ b/scripts/base/frameworks/logging/writers/ascii.bro @@ -25,17 +25,17 @@ export { const meta_prefix = "#" &redef; ## Separator between fields. - const separator = "\t" &redef; + const separator = Log::separator &redef; ## Separator between set elements. - const set_separator = "," &redef; + const set_separator = Log::set_separator &redef; ## String to use for empty fields. This should be different from ## *unset_field* to make the output non-ambigious. - const empty_field = "(empty)" &redef; + const empty_field = Log::empty_field &redef; ## String to use for an unset &optional field. - const unset_field = "-" &redef; + const unset_field = Log::unset_field &redef; } # Default function to postprocess a rotated ASCII log file. It moves the rotated diff --git a/src/AsciiInputOutput.cc b/src/AsciiInputOutput.cc index 8e0227b4ab..6ebc408dd4 100644 --- a/src/AsciiInputOutput.cc +++ b/src/AsciiInputOutput.cc @@ -2,12 +2,29 @@ #include "config.h" +#include "AsciiInputOutput.h" +#include "bro_inet_ntop.h" -bool AsciiInputOutput::ValToText(ODesc* desc, Value* val, const Field* field) +AsciiInputOutput::AsciiInputOutput(threading::MsgThread* t, const string & separator, const string & set_separator, + const string & empty_field, const string & unset_field) + { + thread = t; + this->separator = separator; + this->set_separator = set_separator; + this->empty_field = empty_field; + this->unset_field = unset_field; + } + + +AsciiInputOutput::~AsciiInputOutput() + { + } + +bool AsciiInputOutput::ValToODesc(ODesc* desc, threading::Value* val, const threading::Field* field) const { if ( ! val->present ) { - desc->AddN(unset_field, unset_field_len); + desc->Add(unset_field); return true; } @@ -63,11 +80,11 @@ bool AsciiInputOutput::ValToText(ODesc* desc, Value* val, const Field* field) if ( ! size ) { - desc->AddN(empty_field, empty_field_len); + desc->Add(empty_field); break; } - if ( size == unset_field_len && memcmp(data, unset_field, size) == 0 ) + if ( size == unset_field.size() && memcmp(data, unset_field.data(), size) == 0 ) { // The value we'd write out would match exactly the // place-holder we use for unset optional fields. We @@ -93,23 +110,23 @@ bool AsciiInputOutput::ValToText(ODesc* desc, Value* val, const Field* field) { if ( ! val->val.set_val.size ) { - desc->AddN(empty_field, empty_field_len); + desc->Add(empty_field); break; } - desc->AddEscapeSequence(set_separator, set_separator_len); + desc->AddEscapeSequence(set_separator); for ( int j = 0; j < val->val.set_val.size; j++ ) { if ( j > 0 ) - desc->AddRaw(set_separator, set_separator_len); + desc->AddRaw(set_separator); - if ( ! DoWriteOne(desc, val->val.set_val.vals[j], field) ) + if ( ! ValToODesc(desc, val->val.set_val.vals[j], field) ) { - desc->RemoveEscapeSequence(set_separator, set_separator_len); + desc->RemoveEscapeSequence(set_separator); return false; } } - desc->RemoveEscapeSequence(set_separator, set_separator_len); + desc->RemoveEscapeSequence(set_separator); break; } @@ -118,32 +135,76 @@ bool AsciiInputOutput::ValToText(ODesc* desc, Value* val, const Field* field) { if ( ! val->val.vector_val.size ) { - desc->AddN(empty_field, empty_field_len); + desc->Add(empty_field); break; } - desc->AddEscapeSequence(set_separator, set_separator_len); + desc->AddEscapeSequence(set_separator); for ( int j = 0; j < val->val.vector_val.size; j++ ) { if ( j > 0 ) - desc->AddRaw(set_separator, set_separator_len); + desc->AddRaw(set_separator); - if ( ! DoWriteOne(desc, val->val.vector_val.vals[j], field) ) + if ( ! ValToODesc(desc, val->val.vector_val.vals[j], field) ) { - desc->RemoveEscapeSequence(set_separator, set_separator_len); + desc->RemoveEscapeSequence(set_separator); return false; } } - desc->RemoveEscapeSequence(set_separator, set_separator_len); + desc->RemoveEscapeSequence(set_separator); break; } default: - Error(Fmt("unsupported field format %d for %s", val->type, field->name)); + thread->Error(thread->Fmt("unsupported field format %d for %s", val->type, field->name)); return false; } return true; } + +string AsciiInputOutput::Render(const threading::Value::addr_t& addr) + { + if ( addr.family == IPv4 ) + { + char s[INET_ADDRSTRLEN]; + + if ( ! bro_inet_ntop(AF_INET, &addr.in.in4, s, INET_ADDRSTRLEN) ) + return ""; + else + return s; + } + else + { + char s[INET6_ADDRSTRLEN]; + + if ( ! bro_inet_ntop(AF_INET6, &addr.in.in6, s, INET6_ADDRSTRLEN) ) + return ""; + else + return s; + } + } + +string AsciiInputOutput::Render(const threading::Value::subnet_t& subnet) + { + char l[16]; + + if ( subnet.prefix.family == IPv4 ) + modp_uitoa10(subnet.length - 96, l); + else + modp_uitoa10(subnet.length, l); + + string s = Render(subnet.prefix) + "/" + l; + + return s; + } + +string AsciiInputOutput::Render(double d) + { + char buf[256]; + modp_dtoa(d, buf, 6); + return buf; + } + diff --git a/src/AsciiInputOutput.h b/src/AsciiInputOutput.h index fd34b6d398..1f9d6810e0 100644 --- a/src/AsciiInputOutput.h +++ b/src/AsciiInputOutput.h @@ -3,12 +3,54 @@ #ifndef AsciiInputOutput_h #define AsciiInputOutput_h +#include "Desc.h" +#include "threading/MsgThread.h" + class AsciiInputOutput { public: + AsciiInputOutput(threading::MsgThread*, const string & separator, const string & set_separator, + const string & empty_field, const string & unset_field); + ~AsciiInputOutput(); + + // converts a threading value to the corresponding ascii representation // returns false & logs an error with reporter in case an error occurs - bool ValToText(ODesc* desc, Value* val, const Field* field); + bool ValToODesc(ODesc* desc, threading::Value* val, const threading::Field* field) const; + /** Helper method to render an IP address as a string. + * + * @param addr The address. + * + * @return An ASCII representation of the address. + */ + static string Render(const threading::Value::addr_t& addr); + + /** Helper method to render an subnet value as a string. + * + * @param addr The address. + * + * @return An ASCII representation of the address. + */ + static string Render(const threading::Value::subnet_t& subnet); + + /** Helper method to render a double in Bro's standard precision. + * + * @param d The double. + * + * @return An ASCII representation of the double. + */ + static string Render(double d); + + + private: + + string separator; + string set_separator; + string empty_field; + string unset_field; + string meta_prefix; + + threading::MsgThread* thread; }; #endif /* AsciiInputOuput_h */ diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 6867b9639c..fe6d5f27ff 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -284,6 +284,7 @@ set(bro_SRCS Analyzer.cc Anon.cc ARP.cc + AsciiInputOutput.cc Attr.cc AYIYA.cc BackDoor.cc diff --git a/src/Desc.h b/src/Desc.h index a131ce37dd..ddab14ef86 100644 --- a/src/Desc.h +++ b/src/Desc.h @@ -118,6 +118,7 @@ public: // Bypasses the escaping enabled via SetEscape(). void AddRaw(const char* s, int len) { AddBytesRaw(s, len); } + void AddRaw(const string &s) { AddBytesRaw(s.data(), s.size()); } // Returns the description as a string. const char* Description() const { return (const char*) base; } diff --git a/src/logging/WriterBackend.cc b/src/logging/WriterBackend.cc index 47fdec27ef..5e4230c8e3 100644 --- a/src/logging/WriterBackend.cc +++ b/src/logging/WriterBackend.cc @@ -1,7 +1,6 @@ // See the file "COPYING" in the main distribution directory for copyright. #include "util.h" -#include "bro_inet_ntop.h" #include "threading/SerialTypes.h" #include "Manager.h" @@ -328,46 +327,3 @@ bool WriterBackend::OnHeartbeat(double network_time, double current_time) SendOut(new FlushWriteBufferMessage(frontend)); return DoHeartbeat(network_time, current_time); } - -string WriterBackend::Render(const threading::Value::addr_t& addr) const - { - if ( addr.family == IPv4 ) - { - char s[INET_ADDRSTRLEN]; - - if ( ! bro_inet_ntop(AF_INET, &addr.in.in4, s, INET_ADDRSTRLEN) ) - return ""; - else - return s; - } - else - { - char s[INET6_ADDRSTRLEN]; - - if ( ! bro_inet_ntop(AF_INET6, &addr.in.in6, s, INET6_ADDRSTRLEN) ) - return ""; - else - return s; - } - } - -string WriterBackend::Render(const threading::Value::subnet_t& subnet) const - { - char l[16]; - - if ( subnet.prefix.family == IPv4 ) - modp_uitoa10(subnet.length - 96, l); - else - modp_uitoa10(subnet.length, l); - - string s = Render(subnet.prefix) + "/" + l; - - return s; - } - -string WriterBackend::Render(double d) const - { - char buf[256]; - modp_dtoa(d, buf, 6); - return buf; - } diff --git a/src/logging/WriterBackend.h b/src/logging/WriterBackend.h index 89185619c4..b326366b72 100644 --- a/src/logging/WriterBackend.h +++ b/src/logging/WriterBackend.h @@ -256,30 +256,6 @@ public: */ bool FinishedRotation(); - /** Helper method to render an IP address as a string. - * - * @param addr The address. - * - * @return An ASCII representation of the address. - */ - string Render(const threading::Value::addr_t& addr) const; - - /** Helper method to render an subnet value as a string. - * - * @param addr The address. - * - * @return An ASCII representation of the address. - */ - string Render(const threading::Value::subnet_t& subnet) const; - - /** Helper method to render a double in Bro's standard precision. - * - * @param d The double. - * - * @return An ASCII representation of the double. - */ - string Render(double d) const; - // Overridden from MsgThread. virtual bool OnHeartbeat(double network_time, double current_time); virtual bool OnFinish(double network_time); diff --git a/src/logging/writers/Ascii.cc b/src/logging/writers/Ascii.cc index c65b0701c3..9f045c1813 100644 --- a/src/logging/writers/Ascii.cc +++ b/src/logging/writers/Ascii.cc @@ -24,33 +24,35 @@ Ascii::Ascii(WriterFrontend* frontend) : WriterBackend(frontend) output_to_stdout = BifConst::LogAscii::output_to_stdout; include_meta = BifConst::LogAscii::include_meta; - separator_len = BifConst::LogAscii::separator->Len(); - separator = new char[separator_len]; - memcpy(separator, BifConst::LogAscii::separator->Bytes(), - separator_len); + separator.assign( + (const char*) BifConst::LogAscii::separator->Bytes(), + BifConst::LogAscii::separator->Len() + ); - set_separator_len = BifConst::LogAscii::set_separator->Len(); - set_separator = new char[set_separator_len]; - memcpy(set_separator, BifConst::LogAscii::set_separator->Bytes(), - set_separator_len); + set_separator.assign( + (const char*) BifConst::LogAscii::set_separator->Bytes(), + BifConst::LogAscii::set_separator->Len() + ); - empty_field_len = BifConst::LogAscii::empty_field->Len(); - empty_field = new char[empty_field_len]; - memcpy(empty_field, BifConst::LogAscii::empty_field->Bytes(), - empty_field_len); + empty_field.assign( + (const char*) BifConst::LogAscii::empty_field->Bytes(), + BifConst::LogAscii::empty_field->Len() + ); - unset_field_len = BifConst::LogAscii::unset_field->Len(); - unset_field = new char[unset_field_len]; - memcpy(unset_field, BifConst::LogAscii::unset_field->Bytes(), - unset_field_len); + unset_field.assign( + (const char*) BifConst::LogAscii::unset_field->Bytes(), + BifConst::LogAscii::unset_field->Len() + ); - meta_prefix_len = BifConst::LogAscii::meta_prefix->Len(); - meta_prefix = new char[meta_prefix_len]; - memcpy(meta_prefix, BifConst::LogAscii::meta_prefix->Bytes(), - meta_prefix_len); + meta_prefix.assign( + (const char*) BifConst::LogAscii::meta_prefix->Bytes(), + BifConst::LogAscii::meta_prefix->Len() + ); desc.EnableEscaping(); - desc.AddEscapeSequence(separator, separator_len); + desc.AddEscapeSequence(separator); + + io = new AsciiInputOutput(this, separator, set_separator, empty_field, unset_field); } Ascii::~Ascii() @@ -61,17 +63,12 @@ Ascii::~Ascii() abort(); } - delete [] separator; - delete [] set_separator; - delete [] empty_field; - delete [] unset_field; - delete [] meta_prefix; + delete io; } bool Ascii::WriteHeaderField(const string& key, const string& val) { - string str = string(meta_prefix, meta_prefix_len) + - key + string(separator, separator_len) + val + "\n"; + string str = meta_prefix + key + separator + val + "\n"; return safe_write(fd, str.c_str(), str.length()); } @@ -136,8 +133,8 @@ bool Ascii::DoInit(const WriterInfo& info, int num_fields, const Field* const * { if ( i > 0 ) { - names += string(separator, separator_len); - types += string(separator, separator_len); + names += separator; + types += separator; } names += string(fields[i]->name); @@ -154,20 +151,17 @@ bool Ascii::DoInit(const WriterInfo& info, int num_fields, const Field* const * return true; } - string str = string(meta_prefix, meta_prefix_len) + string str = meta_prefix + "separator " // Always use space as separator here. - + get_escaped_string(string(separator, separator_len), false) + + get_escaped_string(separator, false) + "\n"; if ( ! safe_write(fd, str.c_str(), str.length()) ) goto write_error; - if ( ! (WriteHeaderField("set_separator", get_escaped_string( - string(set_separator, set_separator_len), false)) && - WriteHeaderField("empty_field", get_escaped_string( - string(empty_field, empty_field_len), false)) && - WriteHeaderField("unset_field", get_escaped_string( - string(unset_field, unset_field_len), false)) && + if ( ! (WriteHeaderField("set_separator", get_escaped_string(set_separator, false)) && + WriteHeaderField("empty_field", get_escaped_string(empty_field, false)) && + WriteHeaderField("unset_field", get_escaped_string(unset_field, false)) && WriteHeaderField("path", get_escaped_string(path, false)) && WriteHeaderField("open", Timestamp(0))) ) goto write_error; @@ -205,151 +199,6 @@ bool Ascii::DoFinish(double network_time) return true; } - -bool Ascii::DoWriteOne(ODesc* desc, Value* val, const Field* field) - { - if ( ! val->present ) - { - desc->AddN(unset_field, unset_field_len); - return true; - } - - switch ( val->type ) { - - case TYPE_BOOL: - desc->Add(val->val.int_val ? "T" : "F"); - break; - - case TYPE_INT: - desc->Add(val->val.int_val); - break; - - case TYPE_COUNT: - case TYPE_COUNTER: - desc->Add(val->val.uint_val); - break; - - case TYPE_PORT: - desc->Add(val->val.port_val.port); - break; - - case TYPE_SUBNET: - desc->Add(Render(val->val.subnet_val)); - break; - - case TYPE_ADDR: - desc->Add(Render(val->val.addr_val)); - break; - - case TYPE_DOUBLE: - // Rendering via Add() truncates trailing 0s after the - // decimal point. The difference with TIME/INTERVAL is mainly - // to keep the log format consistent. - desc->Add(val->val.double_val); - break; - - case TYPE_INTERVAL: - case TYPE_TIME: - // Rendering via Render() keeps trailing 0s after the decimal - // point. The difference with DOUBLEis mainly to keep the log - // format consistent. - desc->Add(Render(val->val.double_val)); - break; - - case TYPE_ENUM: - case TYPE_STRING: - case TYPE_FILE: - case TYPE_FUNC: - { - int size = val->val.string_val.length; - const char* data = val->val.string_val.data; - - if ( ! size ) - { - desc->AddN(empty_field, empty_field_len); - break; - } - - if ( size == unset_field_len && memcmp(data, unset_field, size) == 0 ) - { - // The value we'd write out would match exactly the - // place-holder we use for unset optional fields. We - // escape the first character so that the output - // won't be ambigious. - static const char hex_chars[] = "0123456789abcdef"; - char hex[6] = "\\x00"; - hex[2] = hex_chars[((*data) & 0xf0) >> 4]; - hex[3] = hex_chars[(*data) & 0x0f]; - desc->AddRaw(hex, 4); - - ++data; - --size; - } - - if ( size ) - desc->AddN(data, size); - - break; - } - - case TYPE_TABLE: - { - if ( ! val->val.set_val.size ) - { - desc->AddN(empty_field, empty_field_len); - break; - } - - desc->AddEscapeSequence(set_separator, set_separator_len); - for ( int j = 0; j < val->val.set_val.size; j++ ) - { - if ( j > 0 ) - desc->AddRaw(set_separator, set_separator_len); - - if ( ! DoWriteOne(desc, val->val.set_val.vals[j], field) ) - { - desc->RemoveEscapeSequence(set_separator, set_separator_len); - return false; - } - } - desc->RemoveEscapeSequence(set_separator, set_separator_len); - - break; - } - - case TYPE_VECTOR: - { - if ( ! val->val.vector_val.size ) - { - desc->AddN(empty_field, empty_field_len); - break; - } - - desc->AddEscapeSequence(set_separator, set_separator_len); - for ( int j = 0; j < val->val.vector_val.size; j++ ) - { - if ( j > 0 ) - desc->AddRaw(set_separator, set_separator_len); - - if ( ! DoWriteOne(desc, val->val.vector_val.vals[j], field) ) - { - desc->RemoveEscapeSequence(set_separator, set_separator_len); - return false; - } - } - desc->RemoveEscapeSequence(set_separator, set_separator_len); - - break; - } - - default: - Error(Fmt("unsupported field format %d for %s", val->type, field->name)); - return false; - } - - return true; - } - bool Ascii::DoWrite(int num_fields, const Field* const * fields, Value** vals) { @@ -361,9 +210,9 @@ bool Ascii::DoWrite(int num_fields, const Field* const * fields, for ( int i = 0; i < num_fields; i++ ) { if ( i > 0 ) - desc.AddRaw(separator, separator_len); + desc.AddRaw(separator); - if ( ! DoWriteOne(&desc, vals[i], fields[i]) ) + if ( ! io->ValToODesc(&desc, vals[i], fields[i]) ) return false; } @@ -372,7 +221,7 @@ bool Ascii::DoWrite(int num_fields, const Field* const * fields, const char* bytes = (const char*)desc.Bytes(); int len = desc.Len(); - if ( strncmp(bytes, meta_prefix, meta_prefix_len) == 0 ) + if ( strncmp(bytes, meta_prefix.data(), meta_prefix.size()) == 0 ) { // It would so escape the first character. char buf[16]; diff --git a/src/logging/writers/Ascii.h b/src/logging/writers/Ascii.h index 37ec19aba6..3bd4c5dcf1 100644 --- a/src/logging/writers/Ascii.h +++ b/src/logging/writers/Ascii.h @@ -6,6 +6,7 @@ #define LOGGING_WRITER_ASCII_H #include "../WriterBackend.h" +#include "../../AsciiInputOutput.h" namespace logging { namespace writer { @@ -32,7 +33,6 @@ protected: private: bool IsSpecial(string path) { return path.find("/dev/") == 0; } - bool DoWriteOne(ODesc* desc, threading::Value* val, const threading::Field* field); bool WriteHeaderField(const string& key, const string& value); void CloseFile(double t); string Timestamp(double t); // Uses current time if t is zero. @@ -47,20 +47,13 @@ private: bool include_meta; bool only_single_header_row; - char* separator; - int separator_len; + string separator; + string set_separator; + string empty_field; + string unset_field; + string meta_prefix; - char* set_separator; - int set_separator_len; - - char* empty_field; - int empty_field_len; - - char* unset_field; - int unset_field_len; - - char* meta_prefix; - int meta_prefix_len; + AsciiInputOutput* io; }; } diff --git a/src/logging/writers/DataSeries.cc b/src/logging/writers/DataSeries.cc index bc5a82ec54..2adf05d522 100644 --- a/src/logging/writers/DataSeries.cc +++ b/src/logging/writers/DataSeries.cc @@ -46,10 +46,10 @@ std::string DataSeries::LogValueToString(threading::Value *val) } case TYPE_SUBNET: - return Render(val->val.subnet_val); + return AsciiInputOutput::Render(val->val.subnet_val); case TYPE_ADDR: - return Render(val->val.addr_val); + return AsciiInputOutput::Render(val->val.addr_val); // Note: These two cases are relatively special. We need to convert // these values into their integer equivalents to maximize precision. @@ -69,10 +69,10 @@ std::string DataSeries::LogValueToString(threading::Value *val) return ostr.str(); } else - return Render(val->val.double_val); + return AsciiInputOutput::Render(val->val.double_val); case TYPE_DOUBLE: - return Render(val->val.double_val); + return AsciiInputOutput::Render(val->val.double_val); case TYPE_ENUM: case TYPE_STRING: diff --git a/src/logging/writers/ElasticSearch.cc b/src/logging/writers/ElasticSearch.cc index ae825ac997..a5897e7272 100644 --- a/src/logging/writers/ElasticSearch.cc +++ b/src/logging/writers/ElasticSearch.cc @@ -16,6 +16,7 @@ #include "BroString.h" #include "NetVar.h" #include "threading/SerialTypes.h" +#include "../../AsciiInputOutput.h" #include #include @@ -124,13 +125,13 @@ bool ElasticSearch::AddValueToBuffer(ODesc* b, Value* val) case TYPE_SUBNET: b->AddRaw("\"", 1); - b->Add(Render(val->val.subnet_val)); + b->Add(AsciiInputOutput::Render(val->val.subnet_val)); b->AddRaw("\"", 1); break; case TYPE_ADDR: b->AddRaw("\"", 1); - b->Add(Render(val->val.addr_val)); + b->Add(AsciiInputOutput::Render(val->val.addr_val)); b->AddRaw("\"", 1); break; diff --git a/src/threading/MsgThread.h b/src/threading/MsgThread.h index e3e7c8500f..c5df6948a0 100644 --- a/src/threading/MsgThread.h +++ b/src/threading/MsgThread.h @@ -400,7 +400,7 @@ private: }; /** - * A paremeterized OututMessage that stores a pointer to an argument object. + * A paremeterized OutputMessage that stores a pointer to an argument object. * Normally, the objects will be used from the Process() callback. */ template