diff --git a/scripts/base/frameworks/input/main.bro b/scripts/base/frameworks/input/main.bro index 742dc65568..ac8f59a044 100644 --- a/scripts/base/frameworks/input/main.bro +++ b/scripts/base/frameworks/input/main.bro @@ -11,6 +11,24 @@ export { ## The default reader mode used. Defaults to `MANUAL`. const default_mode = MANUAL &redef; + ## Separator between fields. + ## Please note that the separator has to be exactly one character long + ## Can be overwritten by individual writers. + const separator = "\t" &redef; + + ## Separator between set elements. + ## Please note that the separator has to be exactly one character long + ## Can be overwritten by individual writers. + const set_separator = "," &redef; + + ## String to use for empty fields. + ## Can be overwritten by individual writers. + const empty_field = "(empty)" &redef; + + ## String to use for an unset &optional field. + ## Can be overwritten by individual writers. + const unset_field = "-" &redef; + ## Flag that controls if the input framework accepts records ## that contain types that are not supported (at the moment ## file and function). If true, the input framework will diff --git a/scripts/base/frameworks/input/readers/ascii.bro b/scripts/base/frameworks/input/readers/ascii.bro index 7fca1ad795..d46ab6f67a 100644 --- a/scripts/base/frameworks/input/readers/ascii.bro +++ b/scripts/base/frameworks/input/readers/ascii.bro @@ -7,15 +7,15 @@ module InputAscii; export { ## Separator between fields. ## Please note that the separator has to be exactly one character long - const separator = "\t" &redef; + const separator = Input::separator &redef; ## Separator between set elements. ## Please note that the separator has to be exactly one character long - const set_separator = "," &redef; + const set_separator = Input::set_separator &redef; ## String to use for empty fields. - const empty_field = "(empty)" &redef; + const empty_field = Input::empty_field &redef; ## String to use for an unset &optional field. - const unset_field = "-" &redef; + const unset_field = Input::unset_field &redef; } diff --git a/scripts/base/frameworks/logging/main.bro b/scripts/base/frameworks/logging/main.bro index 8c2f507aff..054ad4a30b 100644 --- a/scripts/base/frameworks/logging/main.bro +++ b/scripts/base/frameworks/logging/main.bro @@ -17,6 +17,23 @@ export { ## anything else. const default_writer = WRITER_ASCII &redef; + ## Default separator between fields for logwriters. + ## Can be overwritten by individual writers. + const separator = "\t" &redef; + + ## Separator between set elements. + ## Can be overwritten by individual writers. + const set_separator = "," &redef; + + ## String to use for empty fields. This should be different from + ## *unset_field* to make the output non-ambigious. + ## Can be overwritten by individual writers. + const empty_field = "(empty)" &redef; + + ## String to use for an unset &optional field. + ## Can be overwritten by individual writers. + const unset_field = "-" &redef; + ## Type defining the content of a logging stream. type Stream: record { ## A record type defining the log's columns. diff --git a/scripts/base/frameworks/logging/writers/ascii.bro b/scripts/base/frameworks/logging/writers/ascii.bro index 800af51502..dc3910d767 100644 --- a/scripts/base/frameworks/logging/writers/ascii.bro +++ b/scripts/base/frameworks/logging/writers/ascii.bro @@ -25,17 +25,17 @@ export { const meta_prefix = "#" &redef; ## Separator between fields. - const separator = "\t" &redef; + const separator = Log::separator &redef; ## Separator between set elements. - const set_separator = "," &redef; + const set_separator = Log::set_separator &redef; ## String to use for empty fields. This should be different from ## *unset_field* to make the output non-ambigious. - const empty_field = "(empty)" &redef; + const empty_field = Log::empty_field &redef; ## String to use for an unset &optional field. - const unset_field = "-" &redef; + const unset_field = Log::unset_field &redef; } # Default function to postprocess a rotated ASCII log file. It moves the rotated diff --git a/src/AsciiInputOutput.cc b/src/AsciiInputOutput.cc new file mode 100644 index 0000000000..dd7369e8eb --- /dev/null +++ b/src/AsciiInputOutput.cc @@ -0,0 +1,485 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#include "config.h" + +#include +#include +#include "AsciiInputOutput.h" +#include "bro_inet_ntop.h" + +AsciiInputOutput::AsciiInputOutput(threading::MsgThread* t) + { + thread = t; + } + +AsciiInputOutput::AsciiInputOutput(threading::MsgThread* t, const string & separator, const string & set_separator, + const string & empty_field, const string & unset_field) + { + thread = t; + this->separator = separator; + this->set_separator = set_separator; + this->empty_field = empty_field; + this->unset_field = unset_field; + } + + +AsciiInputOutput::~AsciiInputOutput() + { + } + +bool AsciiInputOutput::ValToODesc(ODesc* desc, threading::Value* val, const threading::Field* field) const + { + if ( ! val->present ) + { + desc->Add(unset_field); + return true; + } + + switch ( val->type ) { + + case TYPE_BOOL: + desc->Add(val->val.int_val ? "T" : "F"); + break; + + case TYPE_INT: + desc->Add(val->val.int_val); + break; + + case TYPE_COUNT: + case TYPE_COUNTER: + desc->Add(val->val.uint_val); + break; + + case TYPE_PORT: + desc->Add(val->val.port_val.port); + break; + + case TYPE_SUBNET: + desc->Add(Render(val->val.subnet_val)); + break; + + case TYPE_ADDR: + desc->Add(Render(val->val.addr_val)); + break; + + case TYPE_DOUBLE: + // Rendering via Add() truncates trailing 0s after the + // decimal point. The difference with TIME/INTERVAL is mainly + // to keep the log format consistent. + desc->Add(val->val.double_val); + break; + + case TYPE_INTERVAL: + case TYPE_TIME: + // Rendering via Render() keeps trailing 0s after the decimal + // point. The difference with DOUBLEis mainly to keep the log + // format consistent. + desc->Add(Render(val->val.double_val)); + break; + + case TYPE_ENUM: + case TYPE_STRING: + case TYPE_FILE: + case TYPE_FUNC: + { + int size = val->val.string_val.length; + const char* data = val->val.string_val.data; + + if ( ! size ) + { + desc->Add(empty_field); + break; + } + + if ( size == unset_field.size() && memcmp(data, unset_field.data(), size) == 0 ) + { + // The value we'd write out would match exactly the + // place-holder we use for unset optional fields. We + // escape the first character so that the output + // won't be ambigious. + static const char hex_chars[] = "0123456789abcdef"; + char hex[6] = "\\x00"; + hex[2] = hex_chars[((*data) & 0xf0) >> 4]; + hex[3] = hex_chars[(*data) & 0x0f]; + desc->AddRaw(hex, 4); + + ++data; + --size; + } + + if ( size ) + desc->AddN(data, size); + + break; + } + + case TYPE_TABLE: + { + if ( ! val->val.set_val.size ) + { + desc->Add(empty_field); + break; + } + + desc->AddEscapeSequence(set_separator); + for ( int j = 0; j < val->val.set_val.size; j++ ) + { + if ( j > 0 ) + desc->AddRaw(set_separator); + + if ( ! ValToODesc(desc, val->val.set_val.vals[j], field) ) + { + desc->RemoveEscapeSequence(set_separator); + return false; + } + } + desc->RemoveEscapeSequence(set_separator); + + break; + } + + case TYPE_VECTOR: + { + if ( ! val->val.vector_val.size ) + { + desc->Add(empty_field); + break; + } + + desc->AddEscapeSequence(set_separator); + for ( int j = 0; j < val->val.vector_val.size; j++ ) + { + if ( j > 0 ) + desc->AddRaw(set_separator); + + if ( ! ValToODesc(desc, val->val.vector_val.vals[j], field) ) + { + desc->RemoveEscapeSequence(set_separator); + return false; + } + } + desc->RemoveEscapeSequence(set_separator); + + break; + } + + default: + thread->Error(thread->Fmt("unsupported field format %d for %s", val->type, field->name)); + return false; + } + + return true; + } + + +threading::Value* AsciiInputOutput::EntryToVal(string s, string name, TypeTag type, TypeTag subtype) const + { + if ( s.compare(unset_field) == 0 ) // field is not set... + return new threading::Value(type, false); + + threading::Value* val = new threading::Value(type, true); + char* end = 0; + errno = 0; + + switch ( type ) { + case TYPE_ENUM: + case TYPE_STRING: + s = get_unescaped_string(s); + val->val.string_val.length = s.size(); + val->val.string_val.data = copy_string(s.c_str()); + break; + + case TYPE_BOOL: + if ( s == "T" ) + val->val.int_val = 1; + else if ( s == "F" ) + val->val.int_val = 0; + else + { + thread->Error(thread->Fmt("Field: %s Invalid value for boolean: %s", + name.c_str(), s.c_str())); + return 0; + } + break; + + case TYPE_INT: + val->val.int_val = strtoll(s.c_str(), &end, 10); + if ( CheckNumberError(s, end) ) + return 0; + break; + + case TYPE_DOUBLE: + case TYPE_TIME: + case TYPE_INTERVAL: + val->val.double_val = strtod(s.c_str(), &end); + if ( CheckNumberError(s, end) ) + return 0; + break; + + case TYPE_COUNT: + case TYPE_COUNTER: + val->val.uint_val = strtoull(s.c_str(), &end, 10); + if ( CheckNumberError(s, end) ) + return 0; + break; + + case TYPE_PORT: + val->val.port_val.port = strtoull(s.c_str(), &end, 10); + if ( CheckNumberError(s, end) ) + return 0; + + val->val.port_val.proto = TRANSPORT_UNKNOWN; + break; + + case TYPE_SUBNET: + { + s = get_unescaped_string(s); + size_t pos = s.find("/"); + if ( pos == s.npos ) + { + thread->Error(thread->Fmt("Invalid value for subnet: %s", s.c_str())); + return 0; + } + + uint8_t width = (uint8_t) strtol(s.substr(pos+1).c_str(), &end, 10); + + if ( CheckNumberError(s, end) ) + return 0; + + string addr = s.substr(0, pos); + + val->val.subnet_val.prefix = StringToAddr(addr); + val->val.subnet_val.length = width; + break; + } + + case TYPE_ADDR: + s = get_unescaped_string(s); + val->val.addr_val = StringToAddr(s); + break; + + case TYPE_TABLE: + case TYPE_VECTOR: + // First - common initialization + // Then - initialization for table. + // Then - initialization for vector. + // Then - common stuff + { + // how many entries do we have... + unsigned int length = 1; + for ( unsigned int i = 0; i < s.size(); i++ ) + { + if ( s[i] == set_separator[0] ) + length++; + } + + unsigned int pos = 0; + + if ( s.compare(empty_field) == 0 ) + length = 0; + + threading::Value** lvals = new threading::Value* [length]; + + if ( type == TYPE_TABLE ) + { + val->val.set_val.vals = lvals; + val->val.set_val.size = length; + } + + else if ( type == TYPE_VECTOR ) + { + val->val.vector_val.vals = lvals; + val->val.vector_val.size = length; + } + + else + assert(false); + + if ( length == 0 ) + break; //empty + + istringstream splitstream(s); + while ( splitstream ) + { + string element; + + if ( ! getline(splitstream, element, set_separator[0]) ) + break; + + if ( pos >= length ) + { + thread->Error(thread->Fmt("Internal error while parsing set. pos %d >= length %d." + " Element: %s", pos, length, element.c_str())); + break; + } + + threading::Value* newval = EntryToVal(element, name, subtype); + if ( newval == 0 ) + { + thread->Error("Error while reading set"); + return 0; + } + + lvals[pos] = newval; + + pos++; + } + + // Test if the string ends with a set_separator... or if the + // complete string is empty. In either of these cases we have + // to push an empty val on top of it. + if ( s.empty() || *s.rbegin() == set_separator[0] ) + { + lvals[pos] = EntryToVal("", name, subtype); + if ( lvals[pos] == 0 ) + { + thread->Error("Error while trying to add empty set element"); + return 0; + } + + pos++; + } + + if ( pos != length ) + { + thread->Error(thread->Fmt("Internal error while parsing set: did not find all elements: %s", s.c_str())); + return 0; + } + + break; + } + + default: + thread->Error(thread->Fmt("unsupported field format %d for %s", type, + name.c_str())); + return 0; + } + + return val; + } + +bool AsciiInputOutput::CheckNumberError(const string& s, const char * end) const + { + // Do this check first, before executing s.c_str() or similar. + // otherwise the value to which *end is pointing at the moment might + // be gone ... + bool endnotnull = (*end != '\0'); + + if ( s.length() == 0 ) + { + thread->Error("Got empty string for number field"); + return true; + } + + if ( end == s.c_str() ) { + thread->Error(thread->Fmt("String '%s' contained no parseable number", s.c_str())); + return true; + } + + if ( endnotnull ) + thread->Warning(thread->Fmt("Number '%s' contained non-numeric trailing characters. Ignored trailing characters '%s'", s.c_str(), end)); + + if ( errno == EINVAL ) + { + thread->Error(thread->Fmt("String '%s' could not be converted to a number", s.c_str())); + return true; + } + + else if ( errno == ERANGE ) + { + thread->Error(thread->Fmt("Number '%s' out of supported range.", s.c_str())); + return true; + } + + return false; + } + +string AsciiInputOutput::Render(const threading::Value::addr_t& addr) + { + if ( addr.family == IPv4 ) + { + char s[INET_ADDRSTRLEN]; + + if ( ! bro_inet_ntop(AF_INET, &addr.in.in4, s, INET_ADDRSTRLEN) ) + return ""; + else + return s; + } + else + { + char s[INET6_ADDRSTRLEN]; + + if ( ! bro_inet_ntop(AF_INET6, &addr.in.in6, s, INET6_ADDRSTRLEN) ) + return ""; + else + return s; + } + } + +TransportProto AsciiInputOutput::StringToProto(const string &proto) const + { + if ( proto == "unknown" ) + return TRANSPORT_UNKNOWN; + else if ( proto == "tcp" ) + return TRANSPORT_TCP; + else if ( proto == "udp" ) + return TRANSPORT_UDP; + else if ( proto == "icmp" ) + return TRANSPORT_ICMP; + + thread->Error(thread->Fmt("Tried to parse invalid/unknown protocol: %s", proto.c_str())); + + return TRANSPORT_UNKNOWN; + } + + +// More or less verbose copy from IPAddr.cc -- which uses reporter. +threading::Value::addr_t AsciiInputOutput::StringToAddr(const string &s) const + { + threading::Value::addr_t val; + + if ( s.find(':') == std::string::npos ) // IPv4. + { + val.family = IPv4; + + if ( inet_aton(s.c_str(), &(val.in.in4)) <= 0 ) + { + thread->Error(thread->Fmt("Bad address: %s", s.c_str())); + memset(&val.in.in4.s_addr, 0, sizeof(val.in.in4.s_addr)); + } + } + + else + { + val.family = IPv6; + if ( inet_pton(AF_INET6, s.c_str(), val.in.in6.s6_addr) <=0 ) + { + thread->Error(thread->Fmt("Bad address: %s", s.c_str())); + memset(val.in.in6.s6_addr, 0, sizeof(val.in.in6.s6_addr)); + } + } + + return val; + } + +string AsciiInputOutput::Render(const threading::Value::subnet_t& subnet) + { + char l[16]; + + if ( subnet.prefix.family == IPv4 ) + modp_uitoa10(subnet.length - 96, l); + else + modp_uitoa10(subnet.length, l); + + string s = Render(subnet.prefix) + "/" + l; + + return s; + } + +string AsciiInputOutput::Render(double d) + { + char buf[256]; + modp_dtoa(d, buf, 6); + return buf; + } + diff --git a/src/AsciiInputOutput.h b/src/AsciiInputOutput.h new file mode 100644 index 0000000000..1dd2a4dce0 --- /dev/null +++ b/src/AsciiInputOutput.h @@ -0,0 +1,81 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#ifndef AsciiInputOutput_h +#define AsciiInputOutput_h + +#include "Desc.h" +#include "threading/MsgThread.h" + +class AsciiInputOutput { + public: + // Constructor that leaves separators, etc empty. + // Use if you just need functionality like StringToAddr, etc. + AsciiInputOutput(threading::MsgThread*); + + // Constructor that defines all separators, etc. + // Use if you need either ValToODesc or EntryToVal. + AsciiInputOutput(threading::MsgThread*, const string & separator, const string & set_separator, + const string & empty_field, const string & unset_field); + ~AsciiInputOutput(); + + + // converts a threading value to the corresponding ascii representation + // returns false & logs an error with reporter in case an error occurs + bool ValToODesc(ODesc* desc, threading::Value* val, const threading::Field* field) const; + + // convert the ascii representation of a field into a Value + threading::Value* EntryToVal(string s, string name, TypeTag type, TypeTag subtype = TYPE_ERROR) const; + + /** Helper method to render an IP address as a string. + * + * @param addr The address. + * + * @return An ASCII representation of the address. + */ + static string Render(const threading::Value::addr_t& addr); + + /** Helper method to render an subnet value as a string. + * + * @param addr The address. + * + * @return An ASCII representation of the address. + */ + static string Render(const threading::Value::subnet_t& subnet); + + /** Helper method to render a double in Bro's standard precision. + * + * @param d The double. + * + * @return An ASCII representation of the double. + */ + static string Render(double d); + + /** + * Convert a string into a TransportProto. This is just a utility + * function for Readers. + * + * @param proto the transport protocol + */ + TransportProto StringToProto(const string &proto) const; + + /** + * Convert a string into a Value::addr_t. This is just a utility + * function for Readers. + * + * @param addr containing an ipv4 or ipv6 address + */ + threading::Value::addr_t StringToAddr(const string &addr) const; + + private: + bool CheckNumberError(const string& s, const char * end) const; + + string separator; + string set_separator; + string empty_field; + string unset_field; + string meta_prefix; + + threading::MsgThread* thread; +}; + +#endif /* AsciiInputOuput_h */ diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index cd27d9cadd..b759f146f7 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -284,6 +284,7 @@ set(bro_SRCS Analyzer.cc Anon.cc ARP.cc + AsciiInputOutput.cc Attr.cc AYIYA.cc BackDoor.cc diff --git a/src/Desc.h b/src/Desc.h index 9c60c68106..ddab14ef86 100644 --- a/src/Desc.h +++ b/src/Desc.h @@ -57,9 +57,13 @@ public: void AddEscapeSequence(const char* s) { escape_sequences.push_back(s); } void AddEscapeSequence(const char* s, size_t n) { escape_sequences.push_back(string(s, n)); } + void AddEscapeSequence(const string & s) + { escape_sequences.push_back(s); } void RemoveEscapeSequence(const char* s) { escape_sequences.remove(s); } void RemoveEscapeSequence(const char* s, size_t n) { escape_sequences.remove(string(s, n)); } + void RemoveEscapeSequence(const string & s) + { escape_sequences.remove(s); } void PushIndent(); void PopIndent(); @@ -114,6 +118,7 @@ public: // Bypasses the escaping enabled via SetEscape(). void AddRaw(const char* s, int len) { AddBytesRaw(s, len); } + void AddRaw(const string &s) { AddBytesRaw(s.data(), s.size()); } // Returns the description as a string. const char* Description() const { return (const char*) base; } diff --git a/src/input/ReaderBackend.cc b/src/input/ReaderBackend.cc index 74f5306271..0aaadc3cdc 100644 --- a/src/input/ReaderBackend.cc +++ b/src/input/ReaderBackend.cc @@ -281,50 +281,4 @@ bool ReaderBackend::OnHeartbeat(double network_time, double current_time) return DoHeartbeat(network_time, current_time); } -TransportProto ReaderBackend::StringToProto(const string &proto) - { - if ( proto == "unknown" ) - return TRANSPORT_UNKNOWN; - else if ( proto == "tcp" ) - return TRANSPORT_TCP; - else if ( proto == "udp" ) - return TRANSPORT_UDP; - else if ( proto == "icmp" ) - return TRANSPORT_ICMP; - - Error(Fmt("Tried to parse invalid/unknown protocol: %s", proto.c_str())); - - return TRANSPORT_UNKNOWN; - } - - -// More or less verbose copy from IPAddr.cc -- which uses reporter. -Value::addr_t ReaderBackend::StringToAddr(const string &s) - { - Value::addr_t val; - - if ( s.find(':') == std::string::npos ) // IPv4. - { - val.family = IPv4; - - if ( inet_aton(s.c_str(), &(val.in.in4)) <= 0 ) - { - Error(Fmt("Bad address: %s", s.c_str())); - memset(&val.in.in4.s_addr, 0, sizeof(val.in.in4.s_addr)); - } - } - - else - { - val.family = IPv6; - if ( inet_pton(AF_INET6, s.c_str(), val.in.in6.s6_addr) <=0 ) - { - Error(Fmt("Bad address: %s", s.c_str())); - memset(val.in.in6.s6_addr, 0, sizeof(val.in.in6.s6_addr)); - } - } - - return val; - } - } diff --git a/src/input/ReaderBackend.h b/src/input/ReaderBackend.h index 9fd6c06aa3..73e5475db6 100644 --- a/src/input/ReaderBackend.h +++ b/src/input/ReaderBackend.h @@ -315,21 +315,6 @@ protected: */ void EndCurrentSend(); - /** - * Convert a string into a TransportProto. This is just a utility - * function for Readers. - * - * @param proto the transport protocol - */ - TransportProto StringToProto(const string &proto); - - /** - * Convert a string into a Value::addr_t. This is just a utility - * function for Readers. - * - * @param addr containing an ipv4 or ipv6 address - */ - threading::Value::addr_t StringToAddr(const string &addr); private: // Frontend that instantiated us. This object must not be accessed diff --git a/src/input/readers/Ascii.cc b/src/input/readers/Ascii.cc index e9cba27205..0965b556f2 100644 --- a/src/input/readers/Ascii.cc +++ b/src/input/readers/Ascii.cc @@ -67,11 +67,14 @@ Ascii::Ascii(ReaderFrontend *frontend) : ReaderBackend(frontend) unset_field.assign( (const char*) BifConst::InputAscii::unset_field->Bytes(), BifConst::InputAscii::unset_field->Len()); + + io = new AsciiInputOutput(this, separator, set_separator, empty_field, unset_field); } Ascii::~Ascii() { DoClose(); + delete io; } void Ascii::DoClose() @@ -210,228 +213,7 @@ bool Ascii::GetLine(string& str) return false; } -bool Ascii::CheckNumberError(const string& s, const char * end) - { - // Do this check first, before executing s.c_str() or similar. - // otherwise the value to which *end is pointing at the moment might - // be gone ... - bool endnotnull = (*end != '\0'); - if ( s.length() == 0 ) - { - Error("Got empty string for number field"); - return true; - } - - if ( end == s.c_str() ) { - Error(Fmt("String '%s' contained no parseable number", s.c_str())); - return true; - } - - if ( endnotnull ) - Warning(Fmt("Number '%s' contained non-numeric trailing characters. Ignored trailing characters '%s'", s.c_str(), end)); - - if ( errno == EINVAL ) - { - Error(Fmt("String '%s' could not be converted to a number", s.c_str())); - return true; - } - - else if ( errno == ERANGE ) - { - Error(Fmt("Number '%s' out of supported range.", s.c_str())); - return true; - } - - return false; - } - - -Value* Ascii::EntryToVal(string s, FieldMapping field) - { - if ( s.compare(unset_field) == 0 ) // field is not set... - return new Value(field.type, false); - - Value* val = new Value(field.type, true); - char* end = 0; - errno = 0; - - switch ( field.type ) { - case TYPE_ENUM: - case TYPE_STRING: - s = get_unescaped_string(s); - val->val.string_val.length = s.size(); - val->val.string_val.data = copy_string(s.c_str()); - break; - - case TYPE_BOOL: - if ( s == "T" ) - val->val.int_val = 1; - else if ( s == "F" ) - val->val.int_val = 0; - else - { - Error(Fmt("Field: %s Invalid value for boolean: %s", - field.name.c_str(), s.c_str())); - return 0; - } - break; - - case TYPE_INT: - val->val.int_val = strtoll(s.c_str(), &end, 10); - if ( CheckNumberError(s, end) ) - return 0; - break; - - case TYPE_DOUBLE: - case TYPE_TIME: - case TYPE_INTERVAL: - val->val.double_val = strtod(s.c_str(), &end); - if ( CheckNumberError(s, end) ) - return 0; - break; - - case TYPE_COUNT: - case TYPE_COUNTER: - val->val.uint_val = strtoull(s.c_str(), &end, 10); - if ( CheckNumberError(s, end) ) - return 0; - break; - - case TYPE_PORT: - val->val.port_val.port = strtoull(s.c_str(), &end, 10); - if ( CheckNumberError(s, end) ) - return 0; - - val->val.port_val.proto = TRANSPORT_UNKNOWN; - break; - - case TYPE_SUBNET: - { - s = get_unescaped_string(s); - size_t pos = s.find("/"); - if ( pos == s.npos ) - { - Error(Fmt("Invalid value for subnet: %s", s.c_str())); - return 0; - } - - uint8_t width = (uint8_t) strtol(s.substr(pos+1).c_str(), &end, 10); - - if ( CheckNumberError(s, end) ) - return 0; - - string addr = s.substr(0, pos); - - val->val.subnet_val.prefix = StringToAddr(addr); - val->val.subnet_val.length = width; - break; - } - - case TYPE_ADDR: - s = get_unescaped_string(s); - val->val.addr_val = StringToAddr(s); - break; - - case TYPE_TABLE: - case TYPE_VECTOR: - // First - common initialization - // Then - initialization for table. - // Then - initialization for vector. - // Then - common stuff - { - // how many entries do we have... - unsigned int length = 1; - for ( unsigned int i = 0; i < s.size(); i++ ) - { - if ( s[i] == set_separator[0] ) - length++; - } - - unsigned int pos = 0; - - if ( s.compare(empty_field) == 0 ) - length = 0; - - Value** lvals = new Value* [length]; - - if ( field.type == TYPE_TABLE ) - { - val->val.set_val.vals = lvals; - val->val.set_val.size = length; - } - - else if ( field.type == TYPE_VECTOR ) - { - val->val.vector_val.vals = lvals; - val->val.vector_val.size = length; - } - - else - assert(false); - - if ( length == 0 ) - break; //empty - - istringstream splitstream(s); - while ( splitstream ) - { - string element; - - if ( ! getline(splitstream, element, set_separator[0]) ) - break; - - if ( pos >= length ) - { - Error(Fmt("Internal error while parsing set. pos %d >= length %d." - " Element: %s", pos, length, element.c_str())); - break; - } - - Value* newval = EntryToVal(element, field.subType()); - if ( newval == 0 ) - { - Error("Error while reading set"); - return 0; - } - - lvals[pos] = newval; - - pos++; - } - - // Test if the string ends with a set_separator... or if the - // complete string is empty. In either of these cases we have - // to push an empty val on top of it. - if ( s.empty() || *s.rbegin() == set_separator[0] ) - { - lvals[pos] = EntryToVal("", field.subType()); - if ( lvals[pos] == 0 ) - { - Error("Error while trying to add empty set element"); - return 0; - } - - pos++; - } - - if ( pos != length ) - { - Error(Fmt("Internal error while parsing set: did not find all elements: %s", s.c_str())); - return 0; - } - - break; - } - - default: - Error(Fmt("unsupported field format %d for %s", field.type, - field.name.c_str())); - return 0; - } - - return val; - } // read the entire file and send appropriate thingies back to InputMgr bool Ascii::DoUpdate() @@ -543,7 +325,7 @@ bool Ascii::DoUpdate() return false; } - Value* val = EntryToVal(stringfields[(*fit).position], *fit); + Value* val = io->EntryToVal(stringfields[(*fit).position], (*fit).name, (*fit).type, (*fit).subtype); if ( val == 0 ) { Error(Fmt("Could not convert line '%s' to Val. Ignoring line.", line.c_str())); @@ -557,7 +339,7 @@ bool Ascii::DoUpdate() assert(val->type == TYPE_PORT ); // Error(Fmt("Got type %d != PORT with secondary position!", val->type)); - val->val.port_val.proto = StringToProto(stringfields[(*fit).secondary_position]); + val->val.port_val.proto = io->StringToProto(stringfields[(*fit).secondary_position]); } fields[fpos] = val; diff --git a/src/input/readers/Ascii.h b/src/input/readers/Ascii.h index 6e693fc74b..ed641087c7 100644 --- a/src/input/readers/Ascii.h +++ b/src/input/readers/Ascii.h @@ -7,6 +7,7 @@ #include #include "../ReaderBackend.h" +#include "../../AsciiInputOutput.h" namespace input { namespace reader { @@ -47,8 +48,6 @@ private: bool ReadHeader(bool useCached); bool GetLine(string& str); - threading::Value* EntryToVal(string s, FieldMapping type); - bool CheckNumberError(const string& s, const char * end); ifstream* file; time_t mtime; @@ -64,6 +63,8 @@ private: string set_separator; string empty_field; string unset_field; + + AsciiInputOutput* io; }; diff --git a/src/input/readers/Benchmark.cc b/src/input/readers/Benchmark.cc index b8cec0f14d..4738c6e867 100644 --- a/src/input/readers/Benchmark.cc +++ b/src/input/readers/Benchmark.cc @@ -25,11 +25,15 @@ Benchmark::Benchmark(ReaderFrontend *frontend) : ReaderBackend(frontend) stopspreadat = int(BifConst::InputBenchmark::stopspreadat); timedspread = double(BifConst::InputBenchmark::timedspread); heartbeat_interval = double(BifConst::Threading::heartbeat_interval); + + io = new AsciiInputOutput(this); } Benchmark::~Benchmark() { DoClose(); + + delete io; } void Benchmark::DoClose() @@ -162,13 +166,13 @@ threading::Value* Benchmark::EntryToVal(TypeTag type, TypeTag subtype) case TYPE_SUBNET: { - val->val.subnet_val.prefix = StringToAddr("192.168.17.1"); + val->val.subnet_val.prefix = io->StringToAddr("192.168.17.1"); val->val.subnet_val.length = 16; } break; case TYPE_ADDR: - val->val.addr_val = StringToAddr("192.168.17.1"); + val->val.addr_val = io->StringToAddr("192.168.17.1"); break; case TYPE_TABLE: diff --git a/src/input/readers/Benchmark.h b/src/input/readers/Benchmark.h index bab564b12a..d259db463c 100644 --- a/src/input/readers/Benchmark.h +++ b/src/input/readers/Benchmark.h @@ -4,6 +4,7 @@ #define INPUT_READERS_BENCHMARK_H #include "../ReaderBackend.h" +#include "../../AsciiInputOutput.h" namespace input { namespace reader { @@ -38,6 +39,8 @@ private: double heartbeatstarttime; double timedspread; double heartbeat_interval; + + AsciiInputOutput* io; }; diff --git a/src/logging/WriterBackend.cc b/src/logging/WriterBackend.cc index 47fdec27ef..5e4230c8e3 100644 --- a/src/logging/WriterBackend.cc +++ b/src/logging/WriterBackend.cc @@ -1,7 +1,6 @@ // See the file "COPYING" in the main distribution directory for copyright. #include "util.h" -#include "bro_inet_ntop.h" #include "threading/SerialTypes.h" #include "Manager.h" @@ -328,46 +327,3 @@ bool WriterBackend::OnHeartbeat(double network_time, double current_time) SendOut(new FlushWriteBufferMessage(frontend)); return DoHeartbeat(network_time, current_time); } - -string WriterBackend::Render(const threading::Value::addr_t& addr) const - { - if ( addr.family == IPv4 ) - { - char s[INET_ADDRSTRLEN]; - - if ( ! bro_inet_ntop(AF_INET, &addr.in.in4, s, INET_ADDRSTRLEN) ) - return ""; - else - return s; - } - else - { - char s[INET6_ADDRSTRLEN]; - - if ( ! bro_inet_ntop(AF_INET6, &addr.in.in6, s, INET6_ADDRSTRLEN) ) - return ""; - else - return s; - } - } - -string WriterBackend::Render(const threading::Value::subnet_t& subnet) const - { - char l[16]; - - if ( subnet.prefix.family == IPv4 ) - modp_uitoa10(subnet.length - 96, l); - else - modp_uitoa10(subnet.length, l); - - string s = Render(subnet.prefix) + "/" + l; - - return s; - } - -string WriterBackend::Render(double d) const - { - char buf[256]; - modp_dtoa(d, buf, 6); - return buf; - } diff --git a/src/logging/WriterBackend.h b/src/logging/WriterBackend.h index 89185619c4..b326366b72 100644 --- a/src/logging/WriterBackend.h +++ b/src/logging/WriterBackend.h @@ -256,30 +256,6 @@ public: */ bool FinishedRotation(); - /** Helper method to render an IP address as a string. - * - * @param addr The address. - * - * @return An ASCII representation of the address. - */ - string Render(const threading::Value::addr_t& addr) const; - - /** Helper method to render an subnet value as a string. - * - * @param addr The address. - * - * @return An ASCII representation of the address. - */ - string Render(const threading::Value::subnet_t& subnet) const; - - /** Helper method to render a double in Bro's standard precision. - * - * @param d The double. - * - * @return An ASCII representation of the double. - */ - string Render(double d) const; - // Overridden from MsgThread. virtual bool OnHeartbeat(double network_time, double current_time); virtual bool OnFinish(double network_time); diff --git a/src/logging/writers/Ascii.cc b/src/logging/writers/Ascii.cc index c65b0701c3..9f045c1813 100644 --- a/src/logging/writers/Ascii.cc +++ b/src/logging/writers/Ascii.cc @@ -24,33 +24,35 @@ Ascii::Ascii(WriterFrontend* frontend) : WriterBackend(frontend) output_to_stdout = BifConst::LogAscii::output_to_stdout; include_meta = BifConst::LogAscii::include_meta; - separator_len = BifConst::LogAscii::separator->Len(); - separator = new char[separator_len]; - memcpy(separator, BifConst::LogAscii::separator->Bytes(), - separator_len); + separator.assign( + (const char*) BifConst::LogAscii::separator->Bytes(), + BifConst::LogAscii::separator->Len() + ); - set_separator_len = BifConst::LogAscii::set_separator->Len(); - set_separator = new char[set_separator_len]; - memcpy(set_separator, BifConst::LogAscii::set_separator->Bytes(), - set_separator_len); + set_separator.assign( + (const char*) BifConst::LogAscii::set_separator->Bytes(), + BifConst::LogAscii::set_separator->Len() + ); - empty_field_len = BifConst::LogAscii::empty_field->Len(); - empty_field = new char[empty_field_len]; - memcpy(empty_field, BifConst::LogAscii::empty_field->Bytes(), - empty_field_len); + empty_field.assign( + (const char*) BifConst::LogAscii::empty_field->Bytes(), + BifConst::LogAscii::empty_field->Len() + ); - unset_field_len = BifConst::LogAscii::unset_field->Len(); - unset_field = new char[unset_field_len]; - memcpy(unset_field, BifConst::LogAscii::unset_field->Bytes(), - unset_field_len); + unset_field.assign( + (const char*) BifConst::LogAscii::unset_field->Bytes(), + BifConst::LogAscii::unset_field->Len() + ); - meta_prefix_len = BifConst::LogAscii::meta_prefix->Len(); - meta_prefix = new char[meta_prefix_len]; - memcpy(meta_prefix, BifConst::LogAscii::meta_prefix->Bytes(), - meta_prefix_len); + meta_prefix.assign( + (const char*) BifConst::LogAscii::meta_prefix->Bytes(), + BifConst::LogAscii::meta_prefix->Len() + ); desc.EnableEscaping(); - desc.AddEscapeSequence(separator, separator_len); + desc.AddEscapeSequence(separator); + + io = new AsciiInputOutput(this, separator, set_separator, empty_field, unset_field); } Ascii::~Ascii() @@ -61,17 +63,12 @@ Ascii::~Ascii() abort(); } - delete [] separator; - delete [] set_separator; - delete [] empty_field; - delete [] unset_field; - delete [] meta_prefix; + delete io; } bool Ascii::WriteHeaderField(const string& key, const string& val) { - string str = string(meta_prefix, meta_prefix_len) + - key + string(separator, separator_len) + val + "\n"; + string str = meta_prefix + key + separator + val + "\n"; return safe_write(fd, str.c_str(), str.length()); } @@ -136,8 +133,8 @@ bool Ascii::DoInit(const WriterInfo& info, int num_fields, const Field* const * { if ( i > 0 ) { - names += string(separator, separator_len); - types += string(separator, separator_len); + names += separator; + types += separator; } names += string(fields[i]->name); @@ -154,20 +151,17 @@ bool Ascii::DoInit(const WriterInfo& info, int num_fields, const Field* const * return true; } - string str = string(meta_prefix, meta_prefix_len) + string str = meta_prefix + "separator " // Always use space as separator here. - + get_escaped_string(string(separator, separator_len), false) + + get_escaped_string(separator, false) + "\n"; if ( ! safe_write(fd, str.c_str(), str.length()) ) goto write_error; - if ( ! (WriteHeaderField("set_separator", get_escaped_string( - string(set_separator, set_separator_len), false)) && - WriteHeaderField("empty_field", get_escaped_string( - string(empty_field, empty_field_len), false)) && - WriteHeaderField("unset_field", get_escaped_string( - string(unset_field, unset_field_len), false)) && + if ( ! (WriteHeaderField("set_separator", get_escaped_string(set_separator, false)) && + WriteHeaderField("empty_field", get_escaped_string(empty_field, false)) && + WriteHeaderField("unset_field", get_escaped_string(unset_field, false)) && WriteHeaderField("path", get_escaped_string(path, false)) && WriteHeaderField("open", Timestamp(0))) ) goto write_error; @@ -205,151 +199,6 @@ bool Ascii::DoFinish(double network_time) return true; } - -bool Ascii::DoWriteOne(ODesc* desc, Value* val, const Field* field) - { - if ( ! val->present ) - { - desc->AddN(unset_field, unset_field_len); - return true; - } - - switch ( val->type ) { - - case TYPE_BOOL: - desc->Add(val->val.int_val ? "T" : "F"); - break; - - case TYPE_INT: - desc->Add(val->val.int_val); - break; - - case TYPE_COUNT: - case TYPE_COUNTER: - desc->Add(val->val.uint_val); - break; - - case TYPE_PORT: - desc->Add(val->val.port_val.port); - break; - - case TYPE_SUBNET: - desc->Add(Render(val->val.subnet_val)); - break; - - case TYPE_ADDR: - desc->Add(Render(val->val.addr_val)); - break; - - case TYPE_DOUBLE: - // Rendering via Add() truncates trailing 0s after the - // decimal point. The difference with TIME/INTERVAL is mainly - // to keep the log format consistent. - desc->Add(val->val.double_val); - break; - - case TYPE_INTERVAL: - case TYPE_TIME: - // Rendering via Render() keeps trailing 0s after the decimal - // point. The difference with DOUBLEis mainly to keep the log - // format consistent. - desc->Add(Render(val->val.double_val)); - break; - - case TYPE_ENUM: - case TYPE_STRING: - case TYPE_FILE: - case TYPE_FUNC: - { - int size = val->val.string_val.length; - const char* data = val->val.string_val.data; - - if ( ! size ) - { - desc->AddN(empty_field, empty_field_len); - break; - } - - if ( size == unset_field_len && memcmp(data, unset_field, size) == 0 ) - { - // The value we'd write out would match exactly the - // place-holder we use for unset optional fields. We - // escape the first character so that the output - // won't be ambigious. - static const char hex_chars[] = "0123456789abcdef"; - char hex[6] = "\\x00"; - hex[2] = hex_chars[((*data) & 0xf0) >> 4]; - hex[3] = hex_chars[(*data) & 0x0f]; - desc->AddRaw(hex, 4); - - ++data; - --size; - } - - if ( size ) - desc->AddN(data, size); - - break; - } - - case TYPE_TABLE: - { - if ( ! val->val.set_val.size ) - { - desc->AddN(empty_field, empty_field_len); - break; - } - - desc->AddEscapeSequence(set_separator, set_separator_len); - for ( int j = 0; j < val->val.set_val.size; j++ ) - { - if ( j > 0 ) - desc->AddRaw(set_separator, set_separator_len); - - if ( ! DoWriteOne(desc, val->val.set_val.vals[j], field) ) - { - desc->RemoveEscapeSequence(set_separator, set_separator_len); - return false; - } - } - desc->RemoveEscapeSequence(set_separator, set_separator_len); - - break; - } - - case TYPE_VECTOR: - { - if ( ! val->val.vector_val.size ) - { - desc->AddN(empty_field, empty_field_len); - break; - } - - desc->AddEscapeSequence(set_separator, set_separator_len); - for ( int j = 0; j < val->val.vector_val.size; j++ ) - { - if ( j > 0 ) - desc->AddRaw(set_separator, set_separator_len); - - if ( ! DoWriteOne(desc, val->val.vector_val.vals[j], field) ) - { - desc->RemoveEscapeSequence(set_separator, set_separator_len); - return false; - } - } - desc->RemoveEscapeSequence(set_separator, set_separator_len); - - break; - } - - default: - Error(Fmt("unsupported field format %d for %s", val->type, field->name)); - return false; - } - - return true; - } - bool Ascii::DoWrite(int num_fields, const Field* const * fields, Value** vals) { @@ -361,9 +210,9 @@ bool Ascii::DoWrite(int num_fields, const Field* const * fields, for ( int i = 0; i < num_fields; i++ ) { if ( i > 0 ) - desc.AddRaw(separator, separator_len); + desc.AddRaw(separator); - if ( ! DoWriteOne(&desc, vals[i], fields[i]) ) + if ( ! io->ValToODesc(&desc, vals[i], fields[i]) ) return false; } @@ -372,7 +221,7 @@ bool Ascii::DoWrite(int num_fields, const Field* const * fields, const char* bytes = (const char*)desc.Bytes(); int len = desc.Len(); - if ( strncmp(bytes, meta_prefix, meta_prefix_len) == 0 ) + if ( strncmp(bytes, meta_prefix.data(), meta_prefix.size()) == 0 ) { // It would so escape the first character. char buf[16]; diff --git a/src/logging/writers/Ascii.h b/src/logging/writers/Ascii.h index 37ec19aba6..3bd4c5dcf1 100644 --- a/src/logging/writers/Ascii.h +++ b/src/logging/writers/Ascii.h @@ -6,6 +6,7 @@ #define LOGGING_WRITER_ASCII_H #include "../WriterBackend.h" +#include "../../AsciiInputOutput.h" namespace logging { namespace writer { @@ -32,7 +33,6 @@ protected: private: bool IsSpecial(string path) { return path.find("/dev/") == 0; } - bool DoWriteOne(ODesc* desc, threading::Value* val, const threading::Field* field); bool WriteHeaderField(const string& key, const string& value); void CloseFile(double t); string Timestamp(double t); // Uses current time if t is zero. @@ -47,20 +47,13 @@ private: bool include_meta; bool only_single_header_row; - char* separator; - int separator_len; + string separator; + string set_separator; + string empty_field; + string unset_field; + string meta_prefix; - char* set_separator; - int set_separator_len; - - char* empty_field; - int empty_field_len; - - char* unset_field; - int unset_field_len; - - char* meta_prefix; - int meta_prefix_len; + AsciiInputOutput* io; }; } diff --git a/src/logging/writers/DataSeries.cc b/src/logging/writers/DataSeries.cc index bc5a82ec54..2adf05d522 100644 --- a/src/logging/writers/DataSeries.cc +++ b/src/logging/writers/DataSeries.cc @@ -46,10 +46,10 @@ std::string DataSeries::LogValueToString(threading::Value *val) } case TYPE_SUBNET: - return Render(val->val.subnet_val); + return AsciiInputOutput::Render(val->val.subnet_val); case TYPE_ADDR: - return Render(val->val.addr_val); + return AsciiInputOutput::Render(val->val.addr_val); // Note: These two cases are relatively special. We need to convert // these values into their integer equivalents to maximize precision. @@ -69,10 +69,10 @@ std::string DataSeries::LogValueToString(threading::Value *val) return ostr.str(); } else - return Render(val->val.double_val); + return AsciiInputOutput::Render(val->val.double_val); case TYPE_DOUBLE: - return Render(val->val.double_val); + return AsciiInputOutput::Render(val->val.double_val); case TYPE_ENUM: case TYPE_STRING: diff --git a/src/logging/writers/ElasticSearch.cc b/src/logging/writers/ElasticSearch.cc index ae825ac997..a5897e7272 100644 --- a/src/logging/writers/ElasticSearch.cc +++ b/src/logging/writers/ElasticSearch.cc @@ -16,6 +16,7 @@ #include "BroString.h" #include "NetVar.h" #include "threading/SerialTypes.h" +#include "../../AsciiInputOutput.h" #include #include @@ -124,13 +125,13 @@ bool ElasticSearch::AddValueToBuffer(ODesc* b, Value* val) case TYPE_SUBNET: b->AddRaw("\"", 1); - b->Add(Render(val->val.subnet_val)); + b->Add(AsciiInputOutput::Render(val->val.subnet_val)); b->AddRaw("\"", 1); break; case TYPE_ADDR: b->AddRaw("\"", 1); - b->Add(Render(val->val.addr_val)); + b->Add(AsciiInputOutput::Render(val->val.addr_val)); b->AddRaw("\"", 1); break; diff --git a/src/threading/MsgThread.h b/src/threading/MsgThread.h index e3e7c8500f..c5df6948a0 100644 --- a/src/threading/MsgThread.h +++ b/src/threading/MsgThread.h @@ -400,7 +400,7 @@ private: }; /** - * A paremeterized OututMessage that stores a pointer to an argument object. + * A paremeterized OutputMessage that stores a pointer to an argument object. * Normally, the objects will be used from the Process() callback. */ template