From bb76335e5a434a6737676f6afd520e60914fcc7c Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Mon, 3 Dec 2012 10:42:50 -0800 Subject: [PATCH 01/43] intermediate commit - it has been over a month since I touched this... --- src/AsciiInputOutput.cc | 149 ++++++++++++++++++++++++++++++++++++++++ src/AsciiInputOutput.h | 14 ++++ 2 files changed, 163 insertions(+) create mode 100644 src/AsciiInputOutput.cc create mode 100644 src/AsciiInputOutput.h diff --git a/src/AsciiInputOutput.cc b/src/AsciiInputOutput.cc new file mode 100644 index 0000000000..8e0227b4ab --- /dev/null +++ b/src/AsciiInputOutput.cc @@ -0,0 +1,149 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#include "config.h" + + +bool AsciiInputOutput::ValToText(ODesc* desc, Value* val, const Field* field) + { + if ( ! val->present ) + { + desc->AddN(unset_field, unset_field_len); + return true; + } + + switch ( val->type ) { + + case TYPE_BOOL: + desc->Add(val->val.int_val ? "T" : "F"); + break; + + case TYPE_INT: + desc->Add(val->val.int_val); + break; + + case TYPE_COUNT: + case TYPE_COUNTER: + desc->Add(val->val.uint_val); + break; + + case TYPE_PORT: + desc->Add(val->val.port_val.port); + break; + + case TYPE_SUBNET: + desc->Add(Render(val->val.subnet_val)); + break; + + case TYPE_ADDR: + desc->Add(Render(val->val.addr_val)); + break; + + case TYPE_DOUBLE: + // Rendering via Add() truncates trailing 0s after the + // decimal point. The difference with TIME/INTERVAL is mainly + // to keep the log format consistent. + desc->Add(val->val.double_val); + break; + + case TYPE_INTERVAL: + case TYPE_TIME: + // Rendering via Render() keeps trailing 0s after the decimal + // point. The difference with DOUBLEis mainly to keep the log + // format consistent. + desc->Add(Render(val->val.double_val)); + break; + + case TYPE_ENUM: + case TYPE_STRING: + case TYPE_FILE: + case TYPE_FUNC: + { + int size = val->val.string_val.length; + const char* data = val->val.string_val.data; + + if ( ! size ) + { + desc->AddN(empty_field, empty_field_len); + break; + } + + if ( size == unset_field_len && memcmp(data, unset_field, size) == 0 ) + { + // The value we'd write out would match exactly the + // place-holder we use for unset optional fields. We + // escape the first character so that the output + // won't be ambigious. + static const char hex_chars[] = "0123456789abcdef"; + char hex[6] = "\\x00"; + hex[2] = hex_chars[((*data) & 0xf0) >> 4]; + hex[3] = hex_chars[(*data) & 0x0f]; + desc->AddRaw(hex, 4); + + ++data; + --size; + } + + if ( size ) + desc->AddN(data, size); + + break; + } + + case TYPE_TABLE: + { + if ( ! val->val.set_val.size ) + { + desc->AddN(empty_field, empty_field_len); + break; + } + + desc->AddEscapeSequence(set_separator, set_separator_len); + for ( int j = 0; j < val->val.set_val.size; j++ ) + { + if ( j > 0 ) + desc->AddRaw(set_separator, set_separator_len); + + if ( ! DoWriteOne(desc, val->val.set_val.vals[j], field) ) + { + desc->RemoveEscapeSequence(set_separator, set_separator_len); + return false; + } + } + desc->RemoveEscapeSequence(set_separator, set_separator_len); + + break; + } + + case TYPE_VECTOR: + { + if ( ! val->val.vector_val.size ) + { + desc->AddN(empty_field, empty_field_len); + break; + } + + desc->AddEscapeSequence(set_separator, set_separator_len); + for ( int j = 0; j < val->val.vector_val.size; j++ ) + { + if ( j > 0 ) + desc->AddRaw(set_separator, set_separator_len); + + if ( ! DoWriteOne(desc, val->val.vector_val.vals[j], field) ) + { + desc->RemoveEscapeSequence(set_separator, set_separator_len); + return false; + } + } + desc->RemoveEscapeSequence(set_separator, set_separator_len); + + break; + } + + default: + Error(Fmt("unsupported field format %d for %s", val->type, field->name)); + return false; + } + + return true; + } + diff --git a/src/AsciiInputOutput.h b/src/AsciiInputOutput.h new file mode 100644 index 0000000000..fd34b6d398 --- /dev/null +++ b/src/AsciiInputOutput.h @@ -0,0 +1,14 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#ifndef AsciiInputOutput_h +#define AsciiInputOutput_h + +class AsciiInputOutput { + public: + // converts a threading value to the corresponding ascii representation + // returns false & logs an error with reporter in case an error occurs + bool ValToText(ODesc* desc, Value* val, const Field* field); + +}; + +#endif /* AsciiInputOuput_h */ From f62df0de82f866176f06954d215772667972283a Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Mon, 3 Dec 2012 11:01:28 -0800 Subject: [PATCH 02/43] std::string accessors to escape_sequence functionality --- src/Desc.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/Desc.h b/src/Desc.h index 9c60c68106..a131ce37dd 100644 --- a/src/Desc.h +++ b/src/Desc.h @@ -57,9 +57,13 @@ public: void AddEscapeSequence(const char* s) { escape_sequences.push_back(s); } void AddEscapeSequence(const char* s, size_t n) { escape_sequences.push_back(string(s, n)); } + void AddEscapeSequence(const string & s) + { escape_sequences.push_back(s); } void RemoveEscapeSequence(const char* s) { escape_sequences.remove(s); } void RemoveEscapeSequence(const char* s, size_t n) { escape_sequences.remove(string(s, n)); } + void RemoveEscapeSequence(const string & s) + { escape_sequences.remove(s); } void PushIndent(); void PopIndent(); From 501328d61ab5b7bf56202bc6385007d72c188092 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Mon, 3 Dec 2012 12:59:11 -0800 Subject: [PATCH 03/43] factor out ascii input/output. First step - factored out everything the logging classes use ( so only output ). Moved the script-level configuration to logging/main, and made the individual writers just refer to it - no idea if this is good design. It works. But I am happy about opinions :) Next step - add support for input... --- scripts/base/frameworks/logging/main.bro | 17 ++ .../base/frameworks/logging/writers/ascii.bro | 8 +- src/AsciiInputOutput.cc | 95 ++++++-- src/AsciiInputOutput.h | 44 +++- src/CMakeLists.txt | 1 + src/Desc.h | 1 + src/logging/WriterBackend.cc | 44 ---- src/logging/WriterBackend.h | 24 -- src/logging/writers/Ascii.cc | 221 +++--------------- src/logging/writers/Ascii.h | 21 +- src/logging/writers/DataSeries.cc | 8 +- src/logging/writers/ElasticSearch.cc | 5 +- src/threading/MsgThread.h | 2 +- 13 files changed, 194 insertions(+), 297 deletions(-) diff --git a/scripts/base/frameworks/logging/main.bro b/scripts/base/frameworks/logging/main.bro index 8c2f507aff..054ad4a30b 100644 --- a/scripts/base/frameworks/logging/main.bro +++ b/scripts/base/frameworks/logging/main.bro @@ -17,6 +17,23 @@ export { ## anything else. const default_writer = WRITER_ASCII &redef; + ## Default separator between fields for logwriters. + ## Can be overwritten by individual writers. + const separator = "\t" &redef; + + ## Separator between set elements. + ## Can be overwritten by individual writers. + const set_separator = "," &redef; + + ## String to use for empty fields. This should be different from + ## *unset_field* to make the output non-ambigious. + ## Can be overwritten by individual writers. + const empty_field = "(empty)" &redef; + + ## String to use for an unset &optional field. + ## Can be overwritten by individual writers. + const unset_field = "-" &redef; + ## Type defining the content of a logging stream. type Stream: record { ## A record type defining the log's columns. diff --git a/scripts/base/frameworks/logging/writers/ascii.bro b/scripts/base/frameworks/logging/writers/ascii.bro index 800af51502..dc3910d767 100644 --- a/scripts/base/frameworks/logging/writers/ascii.bro +++ b/scripts/base/frameworks/logging/writers/ascii.bro @@ -25,17 +25,17 @@ export { const meta_prefix = "#" &redef; ## Separator between fields. - const separator = "\t" &redef; + const separator = Log::separator &redef; ## Separator between set elements. - const set_separator = "," &redef; + const set_separator = Log::set_separator &redef; ## String to use for empty fields. This should be different from ## *unset_field* to make the output non-ambigious. - const empty_field = "(empty)" &redef; + const empty_field = Log::empty_field &redef; ## String to use for an unset &optional field. - const unset_field = "-" &redef; + const unset_field = Log::unset_field &redef; } # Default function to postprocess a rotated ASCII log file. It moves the rotated diff --git a/src/AsciiInputOutput.cc b/src/AsciiInputOutput.cc index 8e0227b4ab..6ebc408dd4 100644 --- a/src/AsciiInputOutput.cc +++ b/src/AsciiInputOutput.cc @@ -2,12 +2,29 @@ #include "config.h" +#include "AsciiInputOutput.h" +#include "bro_inet_ntop.h" -bool AsciiInputOutput::ValToText(ODesc* desc, Value* val, const Field* field) +AsciiInputOutput::AsciiInputOutput(threading::MsgThread* t, const string & separator, const string & set_separator, + const string & empty_field, const string & unset_field) + { + thread = t; + this->separator = separator; + this->set_separator = set_separator; + this->empty_field = empty_field; + this->unset_field = unset_field; + } + + +AsciiInputOutput::~AsciiInputOutput() + { + } + +bool AsciiInputOutput::ValToODesc(ODesc* desc, threading::Value* val, const threading::Field* field) const { if ( ! val->present ) { - desc->AddN(unset_field, unset_field_len); + desc->Add(unset_field); return true; } @@ -63,11 +80,11 @@ bool AsciiInputOutput::ValToText(ODesc* desc, Value* val, const Field* field) if ( ! size ) { - desc->AddN(empty_field, empty_field_len); + desc->Add(empty_field); break; } - if ( size == unset_field_len && memcmp(data, unset_field, size) == 0 ) + if ( size == unset_field.size() && memcmp(data, unset_field.data(), size) == 0 ) { // The value we'd write out would match exactly the // place-holder we use for unset optional fields. We @@ -93,23 +110,23 @@ bool AsciiInputOutput::ValToText(ODesc* desc, Value* val, const Field* field) { if ( ! val->val.set_val.size ) { - desc->AddN(empty_field, empty_field_len); + desc->Add(empty_field); break; } - desc->AddEscapeSequence(set_separator, set_separator_len); + desc->AddEscapeSequence(set_separator); for ( int j = 0; j < val->val.set_val.size; j++ ) { if ( j > 0 ) - desc->AddRaw(set_separator, set_separator_len); + desc->AddRaw(set_separator); - if ( ! DoWriteOne(desc, val->val.set_val.vals[j], field) ) + if ( ! ValToODesc(desc, val->val.set_val.vals[j], field) ) { - desc->RemoveEscapeSequence(set_separator, set_separator_len); + desc->RemoveEscapeSequence(set_separator); return false; } } - desc->RemoveEscapeSequence(set_separator, set_separator_len); + desc->RemoveEscapeSequence(set_separator); break; } @@ -118,32 +135,76 @@ bool AsciiInputOutput::ValToText(ODesc* desc, Value* val, const Field* field) { if ( ! val->val.vector_val.size ) { - desc->AddN(empty_field, empty_field_len); + desc->Add(empty_field); break; } - desc->AddEscapeSequence(set_separator, set_separator_len); + desc->AddEscapeSequence(set_separator); for ( int j = 0; j < val->val.vector_val.size; j++ ) { if ( j > 0 ) - desc->AddRaw(set_separator, set_separator_len); + desc->AddRaw(set_separator); - if ( ! DoWriteOne(desc, val->val.vector_val.vals[j], field) ) + if ( ! ValToODesc(desc, val->val.vector_val.vals[j], field) ) { - desc->RemoveEscapeSequence(set_separator, set_separator_len); + desc->RemoveEscapeSequence(set_separator); return false; } } - desc->RemoveEscapeSequence(set_separator, set_separator_len); + desc->RemoveEscapeSequence(set_separator); break; } default: - Error(Fmt("unsupported field format %d for %s", val->type, field->name)); + thread->Error(thread->Fmt("unsupported field format %d for %s", val->type, field->name)); return false; } return true; } + +string AsciiInputOutput::Render(const threading::Value::addr_t& addr) + { + if ( addr.family == IPv4 ) + { + char s[INET_ADDRSTRLEN]; + + if ( ! bro_inet_ntop(AF_INET, &addr.in.in4, s, INET_ADDRSTRLEN) ) + return ""; + else + return s; + } + else + { + char s[INET6_ADDRSTRLEN]; + + if ( ! bro_inet_ntop(AF_INET6, &addr.in.in6, s, INET6_ADDRSTRLEN) ) + return ""; + else + return s; + } + } + +string AsciiInputOutput::Render(const threading::Value::subnet_t& subnet) + { + char l[16]; + + if ( subnet.prefix.family == IPv4 ) + modp_uitoa10(subnet.length - 96, l); + else + modp_uitoa10(subnet.length, l); + + string s = Render(subnet.prefix) + "/" + l; + + return s; + } + +string AsciiInputOutput::Render(double d) + { + char buf[256]; + modp_dtoa(d, buf, 6); + return buf; + } + diff --git a/src/AsciiInputOutput.h b/src/AsciiInputOutput.h index fd34b6d398..1f9d6810e0 100644 --- a/src/AsciiInputOutput.h +++ b/src/AsciiInputOutput.h @@ -3,12 +3,54 @@ #ifndef AsciiInputOutput_h #define AsciiInputOutput_h +#include "Desc.h" +#include "threading/MsgThread.h" + class AsciiInputOutput { public: + AsciiInputOutput(threading::MsgThread*, const string & separator, const string & set_separator, + const string & empty_field, const string & unset_field); + ~AsciiInputOutput(); + + // converts a threading value to the corresponding ascii representation // returns false & logs an error with reporter in case an error occurs - bool ValToText(ODesc* desc, Value* val, const Field* field); + bool ValToODesc(ODesc* desc, threading::Value* val, const threading::Field* field) const; + /** Helper method to render an IP address as a string. + * + * @param addr The address. + * + * @return An ASCII representation of the address. + */ + static string Render(const threading::Value::addr_t& addr); + + /** Helper method to render an subnet value as a string. + * + * @param addr The address. + * + * @return An ASCII representation of the address. + */ + static string Render(const threading::Value::subnet_t& subnet); + + /** Helper method to render a double in Bro's standard precision. + * + * @param d The double. + * + * @return An ASCII representation of the double. + */ + static string Render(double d); + + + private: + + string separator; + string set_separator; + string empty_field; + string unset_field; + string meta_prefix; + + threading::MsgThread* thread; }; #endif /* AsciiInputOuput_h */ diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 6867b9639c..fe6d5f27ff 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -284,6 +284,7 @@ set(bro_SRCS Analyzer.cc Anon.cc ARP.cc + AsciiInputOutput.cc Attr.cc AYIYA.cc BackDoor.cc diff --git a/src/Desc.h b/src/Desc.h index a131ce37dd..ddab14ef86 100644 --- a/src/Desc.h +++ b/src/Desc.h @@ -118,6 +118,7 @@ public: // Bypasses the escaping enabled via SetEscape(). void AddRaw(const char* s, int len) { AddBytesRaw(s, len); } + void AddRaw(const string &s) { AddBytesRaw(s.data(), s.size()); } // Returns the description as a string. const char* Description() const { return (const char*) base; } diff --git a/src/logging/WriterBackend.cc b/src/logging/WriterBackend.cc index 47fdec27ef..5e4230c8e3 100644 --- a/src/logging/WriterBackend.cc +++ b/src/logging/WriterBackend.cc @@ -1,7 +1,6 @@ // See the file "COPYING" in the main distribution directory for copyright. #include "util.h" -#include "bro_inet_ntop.h" #include "threading/SerialTypes.h" #include "Manager.h" @@ -328,46 +327,3 @@ bool WriterBackend::OnHeartbeat(double network_time, double current_time) SendOut(new FlushWriteBufferMessage(frontend)); return DoHeartbeat(network_time, current_time); } - -string WriterBackend::Render(const threading::Value::addr_t& addr) const - { - if ( addr.family == IPv4 ) - { - char s[INET_ADDRSTRLEN]; - - if ( ! bro_inet_ntop(AF_INET, &addr.in.in4, s, INET_ADDRSTRLEN) ) - return ""; - else - return s; - } - else - { - char s[INET6_ADDRSTRLEN]; - - if ( ! bro_inet_ntop(AF_INET6, &addr.in.in6, s, INET6_ADDRSTRLEN) ) - return ""; - else - return s; - } - } - -string WriterBackend::Render(const threading::Value::subnet_t& subnet) const - { - char l[16]; - - if ( subnet.prefix.family == IPv4 ) - modp_uitoa10(subnet.length - 96, l); - else - modp_uitoa10(subnet.length, l); - - string s = Render(subnet.prefix) + "/" + l; - - return s; - } - -string WriterBackend::Render(double d) const - { - char buf[256]; - modp_dtoa(d, buf, 6); - return buf; - } diff --git a/src/logging/WriterBackend.h b/src/logging/WriterBackend.h index 89185619c4..b326366b72 100644 --- a/src/logging/WriterBackend.h +++ b/src/logging/WriterBackend.h @@ -256,30 +256,6 @@ public: */ bool FinishedRotation(); - /** Helper method to render an IP address as a string. - * - * @param addr The address. - * - * @return An ASCII representation of the address. - */ - string Render(const threading::Value::addr_t& addr) const; - - /** Helper method to render an subnet value as a string. - * - * @param addr The address. - * - * @return An ASCII representation of the address. - */ - string Render(const threading::Value::subnet_t& subnet) const; - - /** Helper method to render a double in Bro's standard precision. - * - * @param d The double. - * - * @return An ASCII representation of the double. - */ - string Render(double d) const; - // Overridden from MsgThread. virtual bool OnHeartbeat(double network_time, double current_time); virtual bool OnFinish(double network_time); diff --git a/src/logging/writers/Ascii.cc b/src/logging/writers/Ascii.cc index c65b0701c3..9f045c1813 100644 --- a/src/logging/writers/Ascii.cc +++ b/src/logging/writers/Ascii.cc @@ -24,33 +24,35 @@ Ascii::Ascii(WriterFrontend* frontend) : WriterBackend(frontend) output_to_stdout = BifConst::LogAscii::output_to_stdout; include_meta = BifConst::LogAscii::include_meta; - separator_len = BifConst::LogAscii::separator->Len(); - separator = new char[separator_len]; - memcpy(separator, BifConst::LogAscii::separator->Bytes(), - separator_len); + separator.assign( + (const char*) BifConst::LogAscii::separator->Bytes(), + BifConst::LogAscii::separator->Len() + ); - set_separator_len = BifConst::LogAscii::set_separator->Len(); - set_separator = new char[set_separator_len]; - memcpy(set_separator, BifConst::LogAscii::set_separator->Bytes(), - set_separator_len); + set_separator.assign( + (const char*) BifConst::LogAscii::set_separator->Bytes(), + BifConst::LogAscii::set_separator->Len() + ); - empty_field_len = BifConst::LogAscii::empty_field->Len(); - empty_field = new char[empty_field_len]; - memcpy(empty_field, BifConst::LogAscii::empty_field->Bytes(), - empty_field_len); + empty_field.assign( + (const char*) BifConst::LogAscii::empty_field->Bytes(), + BifConst::LogAscii::empty_field->Len() + ); - unset_field_len = BifConst::LogAscii::unset_field->Len(); - unset_field = new char[unset_field_len]; - memcpy(unset_field, BifConst::LogAscii::unset_field->Bytes(), - unset_field_len); + unset_field.assign( + (const char*) BifConst::LogAscii::unset_field->Bytes(), + BifConst::LogAscii::unset_field->Len() + ); - meta_prefix_len = BifConst::LogAscii::meta_prefix->Len(); - meta_prefix = new char[meta_prefix_len]; - memcpy(meta_prefix, BifConst::LogAscii::meta_prefix->Bytes(), - meta_prefix_len); + meta_prefix.assign( + (const char*) BifConst::LogAscii::meta_prefix->Bytes(), + BifConst::LogAscii::meta_prefix->Len() + ); desc.EnableEscaping(); - desc.AddEscapeSequence(separator, separator_len); + desc.AddEscapeSequence(separator); + + io = new AsciiInputOutput(this, separator, set_separator, empty_field, unset_field); } Ascii::~Ascii() @@ -61,17 +63,12 @@ Ascii::~Ascii() abort(); } - delete [] separator; - delete [] set_separator; - delete [] empty_field; - delete [] unset_field; - delete [] meta_prefix; + delete io; } bool Ascii::WriteHeaderField(const string& key, const string& val) { - string str = string(meta_prefix, meta_prefix_len) + - key + string(separator, separator_len) + val + "\n"; + string str = meta_prefix + key + separator + val + "\n"; return safe_write(fd, str.c_str(), str.length()); } @@ -136,8 +133,8 @@ bool Ascii::DoInit(const WriterInfo& info, int num_fields, const Field* const * { if ( i > 0 ) { - names += string(separator, separator_len); - types += string(separator, separator_len); + names += separator; + types += separator; } names += string(fields[i]->name); @@ -154,20 +151,17 @@ bool Ascii::DoInit(const WriterInfo& info, int num_fields, const Field* const * return true; } - string str = string(meta_prefix, meta_prefix_len) + string str = meta_prefix + "separator " // Always use space as separator here. - + get_escaped_string(string(separator, separator_len), false) + + get_escaped_string(separator, false) + "\n"; if ( ! safe_write(fd, str.c_str(), str.length()) ) goto write_error; - if ( ! (WriteHeaderField("set_separator", get_escaped_string( - string(set_separator, set_separator_len), false)) && - WriteHeaderField("empty_field", get_escaped_string( - string(empty_field, empty_field_len), false)) && - WriteHeaderField("unset_field", get_escaped_string( - string(unset_field, unset_field_len), false)) && + if ( ! (WriteHeaderField("set_separator", get_escaped_string(set_separator, false)) && + WriteHeaderField("empty_field", get_escaped_string(empty_field, false)) && + WriteHeaderField("unset_field", get_escaped_string(unset_field, false)) && WriteHeaderField("path", get_escaped_string(path, false)) && WriteHeaderField("open", Timestamp(0))) ) goto write_error; @@ -205,151 +199,6 @@ bool Ascii::DoFinish(double network_time) return true; } - -bool Ascii::DoWriteOne(ODesc* desc, Value* val, const Field* field) - { - if ( ! val->present ) - { - desc->AddN(unset_field, unset_field_len); - return true; - } - - switch ( val->type ) { - - case TYPE_BOOL: - desc->Add(val->val.int_val ? "T" : "F"); - break; - - case TYPE_INT: - desc->Add(val->val.int_val); - break; - - case TYPE_COUNT: - case TYPE_COUNTER: - desc->Add(val->val.uint_val); - break; - - case TYPE_PORT: - desc->Add(val->val.port_val.port); - break; - - case TYPE_SUBNET: - desc->Add(Render(val->val.subnet_val)); - break; - - case TYPE_ADDR: - desc->Add(Render(val->val.addr_val)); - break; - - case TYPE_DOUBLE: - // Rendering via Add() truncates trailing 0s after the - // decimal point. The difference with TIME/INTERVAL is mainly - // to keep the log format consistent. - desc->Add(val->val.double_val); - break; - - case TYPE_INTERVAL: - case TYPE_TIME: - // Rendering via Render() keeps trailing 0s after the decimal - // point. The difference with DOUBLEis mainly to keep the log - // format consistent. - desc->Add(Render(val->val.double_val)); - break; - - case TYPE_ENUM: - case TYPE_STRING: - case TYPE_FILE: - case TYPE_FUNC: - { - int size = val->val.string_val.length; - const char* data = val->val.string_val.data; - - if ( ! size ) - { - desc->AddN(empty_field, empty_field_len); - break; - } - - if ( size == unset_field_len && memcmp(data, unset_field, size) == 0 ) - { - // The value we'd write out would match exactly the - // place-holder we use for unset optional fields. We - // escape the first character so that the output - // won't be ambigious. - static const char hex_chars[] = "0123456789abcdef"; - char hex[6] = "\\x00"; - hex[2] = hex_chars[((*data) & 0xf0) >> 4]; - hex[3] = hex_chars[(*data) & 0x0f]; - desc->AddRaw(hex, 4); - - ++data; - --size; - } - - if ( size ) - desc->AddN(data, size); - - break; - } - - case TYPE_TABLE: - { - if ( ! val->val.set_val.size ) - { - desc->AddN(empty_field, empty_field_len); - break; - } - - desc->AddEscapeSequence(set_separator, set_separator_len); - for ( int j = 0; j < val->val.set_val.size; j++ ) - { - if ( j > 0 ) - desc->AddRaw(set_separator, set_separator_len); - - if ( ! DoWriteOne(desc, val->val.set_val.vals[j], field) ) - { - desc->RemoveEscapeSequence(set_separator, set_separator_len); - return false; - } - } - desc->RemoveEscapeSequence(set_separator, set_separator_len); - - break; - } - - case TYPE_VECTOR: - { - if ( ! val->val.vector_val.size ) - { - desc->AddN(empty_field, empty_field_len); - break; - } - - desc->AddEscapeSequence(set_separator, set_separator_len); - for ( int j = 0; j < val->val.vector_val.size; j++ ) - { - if ( j > 0 ) - desc->AddRaw(set_separator, set_separator_len); - - if ( ! DoWriteOne(desc, val->val.vector_val.vals[j], field) ) - { - desc->RemoveEscapeSequence(set_separator, set_separator_len); - return false; - } - } - desc->RemoveEscapeSequence(set_separator, set_separator_len); - - break; - } - - default: - Error(Fmt("unsupported field format %d for %s", val->type, field->name)); - return false; - } - - return true; - } - bool Ascii::DoWrite(int num_fields, const Field* const * fields, Value** vals) { @@ -361,9 +210,9 @@ bool Ascii::DoWrite(int num_fields, const Field* const * fields, for ( int i = 0; i < num_fields; i++ ) { if ( i > 0 ) - desc.AddRaw(separator, separator_len); + desc.AddRaw(separator); - if ( ! DoWriteOne(&desc, vals[i], fields[i]) ) + if ( ! io->ValToODesc(&desc, vals[i], fields[i]) ) return false; } @@ -372,7 +221,7 @@ bool Ascii::DoWrite(int num_fields, const Field* const * fields, const char* bytes = (const char*)desc.Bytes(); int len = desc.Len(); - if ( strncmp(bytes, meta_prefix, meta_prefix_len) == 0 ) + if ( strncmp(bytes, meta_prefix.data(), meta_prefix.size()) == 0 ) { // It would so escape the first character. char buf[16]; diff --git a/src/logging/writers/Ascii.h b/src/logging/writers/Ascii.h index 37ec19aba6..3bd4c5dcf1 100644 --- a/src/logging/writers/Ascii.h +++ b/src/logging/writers/Ascii.h @@ -6,6 +6,7 @@ #define LOGGING_WRITER_ASCII_H #include "../WriterBackend.h" +#include "../../AsciiInputOutput.h" namespace logging { namespace writer { @@ -32,7 +33,6 @@ protected: private: bool IsSpecial(string path) { return path.find("/dev/") == 0; } - bool DoWriteOne(ODesc* desc, threading::Value* val, const threading::Field* field); bool WriteHeaderField(const string& key, const string& value); void CloseFile(double t); string Timestamp(double t); // Uses current time if t is zero. @@ -47,20 +47,13 @@ private: bool include_meta; bool only_single_header_row; - char* separator; - int separator_len; + string separator; + string set_separator; + string empty_field; + string unset_field; + string meta_prefix; - char* set_separator; - int set_separator_len; - - char* empty_field; - int empty_field_len; - - char* unset_field; - int unset_field_len; - - char* meta_prefix; - int meta_prefix_len; + AsciiInputOutput* io; }; } diff --git a/src/logging/writers/DataSeries.cc b/src/logging/writers/DataSeries.cc index bc5a82ec54..2adf05d522 100644 --- a/src/logging/writers/DataSeries.cc +++ b/src/logging/writers/DataSeries.cc @@ -46,10 +46,10 @@ std::string DataSeries::LogValueToString(threading::Value *val) } case TYPE_SUBNET: - return Render(val->val.subnet_val); + return AsciiInputOutput::Render(val->val.subnet_val); case TYPE_ADDR: - return Render(val->val.addr_val); + return AsciiInputOutput::Render(val->val.addr_val); // Note: These two cases are relatively special. We need to convert // these values into their integer equivalents to maximize precision. @@ -69,10 +69,10 @@ std::string DataSeries::LogValueToString(threading::Value *val) return ostr.str(); } else - return Render(val->val.double_val); + return AsciiInputOutput::Render(val->val.double_val); case TYPE_DOUBLE: - return Render(val->val.double_val); + return AsciiInputOutput::Render(val->val.double_val); case TYPE_ENUM: case TYPE_STRING: diff --git a/src/logging/writers/ElasticSearch.cc b/src/logging/writers/ElasticSearch.cc index ae825ac997..a5897e7272 100644 --- a/src/logging/writers/ElasticSearch.cc +++ b/src/logging/writers/ElasticSearch.cc @@ -16,6 +16,7 @@ #include "BroString.h" #include "NetVar.h" #include "threading/SerialTypes.h" +#include "../../AsciiInputOutput.h" #include #include @@ -124,13 +125,13 @@ bool ElasticSearch::AddValueToBuffer(ODesc* b, Value* val) case TYPE_SUBNET: b->AddRaw("\"", 1); - b->Add(Render(val->val.subnet_val)); + b->Add(AsciiInputOutput::Render(val->val.subnet_val)); b->AddRaw("\"", 1); break; case TYPE_ADDR: b->AddRaw("\"", 1); - b->Add(Render(val->val.addr_val)); + b->Add(AsciiInputOutput::Render(val->val.addr_val)); b->AddRaw("\"", 1); break; diff --git a/src/threading/MsgThread.h b/src/threading/MsgThread.h index e3e7c8500f..c5df6948a0 100644 --- a/src/threading/MsgThread.h +++ b/src/threading/MsgThread.h @@ -400,7 +400,7 @@ private: }; /** - * A paremeterized OututMessage that stores a pointer to an argument object. + * A paremeterized OutputMessage that stores a pointer to an argument object. * Normally, the objects will be used from the Process() callback. */ template From 9b2265877d9be473d139d636bbdb7b944de63561 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Mon, 3 Dec 2012 13:41:19 -0800 Subject: [PATCH 04/43] and factor stuff out the input framework too. --- scripts/base/frameworks/input/main.bro | 18 ++ .../base/frameworks/input/readers/ascii.bro | 8 +- src/AsciiInputOutput.cc | 275 ++++++++++++++++++ src/AsciiInputOutput.h | 25 ++ src/input/ReaderBackend.cc | 46 --- src/input/ReaderBackend.h | 15 - src/input/readers/Ascii.cc | 228 +-------------- src/input/readers/Ascii.h | 5 +- src/input/readers/Benchmark.cc | 8 +- src/input/readers/Benchmark.h | 3 + 10 files changed, 339 insertions(+), 292 deletions(-) diff --git a/scripts/base/frameworks/input/main.bro b/scripts/base/frameworks/input/main.bro index 742dc65568..ac8f59a044 100644 --- a/scripts/base/frameworks/input/main.bro +++ b/scripts/base/frameworks/input/main.bro @@ -11,6 +11,24 @@ export { ## The default reader mode used. Defaults to `MANUAL`. const default_mode = MANUAL &redef; + ## Separator between fields. + ## Please note that the separator has to be exactly one character long + ## Can be overwritten by individual writers. + const separator = "\t" &redef; + + ## Separator between set elements. + ## Please note that the separator has to be exactly one character long + ## Can be overwritten by individual writers. + const set_separator = "," &redef; + + ## String to use for empty fields. + ## Can be overwritten by individual writers. + const empty_field = "(empty)" &redef; + + ## String to use for an unset &optional field. + ## Can be overwritten by individual writers. + const unset_field = "-" &redef; + ## Flag that controls if the input framework accepts records ## that contain types that are not supported (at the moment ## file and function). If true, the input framework will diff --git a/scripts/base/frameworks/input/readers/ascii.bro b/scripts/base/frameworks/input/readers/ascii.bro index 7fca1ad795..d46ab6f67a 100644 --- a/scripts/base/frameworks/input/readers/ascii.bro +++ b/scripts/base/frameworks/input/readers/ascii.bro @@ -7,15 +7,15 @@ module InputAscii; export { ## Separator between fields. ## Please note that the separator has to be exactly one character long - const separator = "\t" &redef; + const separator = Input::separator &redef; ## Separator between set elements. ## Please note that the separator has to be exactly one character long - const set_separator = "," &redef; + const set_separator = Input::set_separator &redef; ## String to use for empty fields. - const empty_field = "(empty)" &redef; + const empty_field = Input::empty_field &redef; ## String to use for an unset &optional field. - const unset_field = "-" &redef; + const unset_field = Input::unset_field &redef; } diff --git a/src/AsciiInputOutput.cc b/src/AsciiInputOutput.cc index 6ebc408dd4..dd7369e8eb 100644 --- a/src/AsciiInputOutput.cc +++ b/src/AsciiInputOutput.cc @@ -2,9 +2,16 @@ #include "config.h" +#include +#include #include "AsciiInputOutput.h" #include "bro_inet_ntop.h" +AsciiInputOutput::AsciiInputOutput(threading::MsgThread* t) + { + thread = t; + } + AsciiInputOutput::AsciiInputOutput(threading::MsgThread* t, const string & separator, const string & set_separator, const string & empty_field, const string & unset_field) { @@ -165,6 +172,228 @@ bool AsciiInputOutput::ValToODesc(ODesc* desc, threading::Value* val, const thre } +threading::Value* AsciiInputOutput::EntryToVal(string s, string name, TypeTag type, TypeTag subtype) const + { + if ( s.compare(unset_field) == 0 ) // field is not set... + return new threading::Value(type, false); + + threading::Value* val = new threading::Value(type, true); + char* end = 0; + errno = 0; + + switch ( type ) { + case TYPE_ENUM: + case TYPE_STRING: + s = get_unescaped_string(s); + val->val.string_val.length = s.size(); + val->val.string_val.data = copy_string(s.c_str()); + break; + + case TYPE_BOOL: + if ( s == "T" ) + val->val.int_val = 1; + else if ( s == "F" ) + val->val.int_val = 0; + else + { + thread->Error(thread->Fmt("Field: %s Invalid value for boolean: %s", + name.c_str(), s.c_str())); + return 0; + } + break; + + case TYPE_INT: + val->val.int_val = strtoll(s.c_str(), &end, 10); + if ( CheckNumberError(s, end) ) + return 0; + break; + + case TYPE_DOUBLE: + case TYPE_TIME: + case TYPE_INTERVAL: + val->val.double_val = strtod(s.c_str(), &end); + if ( CheckNumberError(s, end) ) + return 0; + break; + + case TYPE_COUNT: + case TYPE_COUNTER: + val->val.uint_val = strtoull(s.c_str(), &end, 10); + if ( CheckNumberError(s, end) ) + return 0; + break; + + case TYPE_PORT: + val->val.port_val.port = strtoull(s.c_str(), &end, 10); + if ( CheckNumberError(s, end) ) + return 0; + + val->val.port_val.proto = TRANSPORT_UNKNOWN; + break; + + case TYPE_SUBNET: + { + s = get_unescaped_string(s); + size_t pos = s.find("/"); + if ( pos == s.npos ) + { + thread->Error(thread->Fmt("Invalid value for subnet: %s", s.c_str())); + return 0; + } + + uint8_t width = (uint8_t) strtol(s.substr(pos+1).c_str(), &end, 10); + + if ( CheckNumberError(s, end) ) + return 0; + + string addr = s.substr(0, pos); + + val->val.subnet_val.prefix = StringToAddr(addr); + val->val.subnet_val.length = width; + break; + } + + case TYPE_ADDR: + s = get_unescaped_string(s); + val->val.addr_val = StringToAddr(s); + break; + + case TYPE_TABLE: + case TYPE_VECTOR: + // First - common initialization + // Then - initialization for table. + // Then - initialization for vector. + // Then - common stuff + { + // how many entries do we have... + unsigned int length = 1; + for ( unsigned int i = 0; i < s.size(); i++ ) + { + if ( s[i] == set_separator[0] ) + length++; + } + + unsigned int pos = 0; + + if ( s.compare(empty_field) == 0 ) + length = 0; + + threading::Value** lvals = new threading::Value* [length]; + + if ( type == TYPE_TABLE ) + { + val->val.set_val.vals = lvals; + val->val.set_val.size = length; + } + + else if ( type == TYPE_VECTOR ) + { + val->val.vector_val.vals = lvals; + val->val.vector_val.size = length; + } + + else + assert(false); + + if ( length == 0 ) + break; //empty + + istringstream splitstream(s); + while ( splitstream ) + { + string element; + + if ( ! getline(splitstream, element, set_separator[0]) ) + break; + + if ( pos >= length ) + { + thread->Error(thread->Fmt("Internal error while parsing set. pos %d >= length %d." + " Element: %s", pos, length, element.c_str())); + break; + } + + threading::Value* newval = EntryToVal(element, name, subtype); + if ( newval == 0 ) + { + thread->Error("Error while reading set"); + return 0; + } + + lvals[pos] = newval; + + pos++; + } + + // Test if the string ends with a set_separator... or if the + // complete string is empty. In either of these cases we have + // to push an empty val on top of it. + if ( s.empty() || *s.rbegin() == set_separator[0] ) + { + lvals[pos] = EntryToVal("", name, subtype); + if ( lvals[pos] == 0 ) + { + thread->Error("Error while trying to add empty set element"); + return 0; + } + + pos++; + } + + if ( pos != length ) + { + thread->Error(thread->Fmt("Internal error while parsing set: did not find all elements: %s", s.c_str())); + return 0; + } + + break; + } + + default: + thread->Error(thread->Fmt("unsupported field format %d for %s", type, + name.c_str())); + return 0; + } + + return val; + } + +bool AsciiInputOutput::CheckNumberError(const string& s, const char * end) const + { + // Do this check first, before executing s.c_str() or similar. + // otherwise the value to which *end is pointing at the moment might + // be gone ... + bool endnotnull = (*end != '\0'); + + if ( s.length() == 0 ) + { + thread->Error("Got empty string for number field"); + return true; + } + + if ( end == s.c_str() ) { + thread->Error(thread->Fmt("String '%s' contained no parseable number", s.c_str())); + return true; + } + + if ( endnotnull ) + thread->Warning(thread->Fmt("Number '%s' contained non-numeric trailing characters. Ignored trailing characters '%s'", s.c_str(), end)); + + if ( errno == EINVAL ) + { + thread->Error(thread->Fmt("String '%s' could not be converted to a number", s.c_str())); + return true; + } + + else if ( errno == ERANGE ) + { + thread->Error(thread->Fmt("Number '%s' out of supported range.", s.c_str())); + return true; + } + + return false; + } + string AsciiInputOutput::Render(const threading::Value::addr_t& addr) { if ( addr.family == IPv4 ) @@ -187,6 +416,52 @@ string AsciiInputOutput::Render(const threading::Value::addr_t& addr) } } +TransportProto AsciiInputOutput::StringToProto(const string &proto) const + { + if ( proto == "unknown" ) + return TRANSPORT_UNKNOWN; + else if ( proto == "tcp" ) + return TRANSPORT_TCP; + else if ( proto == "udp" ) + return TRANSPORT_UDP; + else if ( proto == "icmp" ) + return TRANSPORT_ICMP; + + thread->Error(thread->Fmt("Tried to parse invalid/unknown protocol: %s", proto.c_str())); + + return TRANSPORT_UNKNOWN; + } + + +// More or less verbose copy from IPAddr.cc -- which uses reporter. +threading::Value::addr_t AsciiInputOutput::StringToAddr(const string &s) const + { + threading::Value::addr_t val; + + if ( s.find(':') == std::string::npos ) // IPv4. + { + val.family = IPv4; + + if ( inet_aton(s.c_str(), &(val.in.in4)) <= 0 ) + { + thread->Error(thread->Fmt("Bad address: %s", s.c_str())); + memset(&val.in.in4.s_addr, 0, sizeof(val.in.in4.s_addr)); + } + } + + else + { + val.family = IPv6; + if ( inet_pton(AF_INET6, s.c_str(), val.in.in6.s6_addr) <=0 ) + { + thread->Error(thread->Fmt("Bad address: %s", s.c_str())); + memset(val.in.in6.s6_addr, 0, sizeof(val.in.in6.s6_addr)); + } + } + + return val; + } + string AsciiInputOutput::Render(const threading::Value::subnet_t& subnet) { char l[16]; diff --git a/src/AsciiInputOutput.h b/src/AsciiInputOutput.h index 1f9d6810e0..1dd2a4dce0 100644 --- a/src/AsciiInputOutput.h +++ b/src/AsciiInputOutput.h @@ -8,6 +8,12 @@ class AsciiInputOutput { public: + // Constructor that leaves separators, etc empty. + // Use if you just need functionality like StringToAddr, etc. + AsciiInputOutput(threading::MsgThread*); + + // Constructor that defines all separators, etc. + // Use if you need either ValToODesc or EntryToVal. AsciiInputOutput(threading::MsgThread*, const string & separator, const string & set_separator, const string & empty_field, const string & unset_field); ~AsciiInputOutput(); @@ -17,6 +23,9 @@ class AsciiInputOutput { // returns false & logs an error with reporter in case an error occurs bool ValToODesc(ODesc* desc, threading::Value* val, const threading::Field* field) const; + // convert the ascii representation of a field into a Value + threading::Value* EntryToVal(string s, string name, TypeTag type, TypeTag subtype = TYPE_ERROR) const; + /** Helper method to render an IP address as a string. * * @param addr The address. @@ -41,8 +50,24 @@ class AsciiInputOutput { */ static string Render(double d); + /** + * Convert a string into a TransportProto. This is just a utility + * function for Readers. + * + * @param proto the transport protocol + */ + TransportProto StringToProto(const string &proto) const; + + /** + * Convert a string into a Value::addr_t. This is just a utility + * function for Readers. + * + * @param addr containing an ipv4 or ipv6 address + */ + threading::Value::addr_t StringToAddr(const string &addr) const; private: + bool CheckNumberError(const string& s, const char * end) const; string separator; string set_separator; diff --git a/src/input/ReaderBackend.cc b/src/input/ReaderBackend.cc index 74f5306271..0aaadc3cdc 100644 --- a/src/input/ReaderBackend.cc +++ b/src/input/ReaderBackend.cc @@ -281,50 +281,4 @@ bool ReaderBackend::OnHeartbeat(double network_time, double current_time) return DoHeartbeat(network_time, current_time); } -TransportProto ReaderBackend::StringToProto(const string &proto) - { - if ( proto == "unknown" ) - return TRANSPORT_UNKNOWN; - else if ( proto == "tcp" ) - return TRANSPORT_TCP; - else if ( proto == "udp" ) - return TRANSPORT_UDP; - else if ( proto == "icmp" ) - return TRANSPORT_ICMP; - - Error(Fmt("Tried to parse invalid/unknown protocol: %s", proto.c_str())); - - return TRANSPORT_UNKNOWN; - } - - -// More or less verbose copy from IPAddr.cc -- which uses reporter. -Value::addr_t ReaderBackend::StringToAddr(const string &s) - { - Value::addr_t val; - - if ( s.find(':') == std::string::npos ) // IPv4. - { - val.family = IPv4; - - if ( inet_aton(s.c_str(), &(val.in.in4)) <= 0 ) - { - Error(Fmt("Bad address: %s", s.c_str())); - memset(&val.in.in4.s_addr, 0, sizeof(val.in.in4.s_addr)); - } - } - - else - { - val.family = IPv6; - if ( inet_pton(AF_INET6, s.c_str(), val.in.in6.s6_addr) <=0 ) - { - Error(Fmt("Bad address: %s", s.c_str())); - memset(val.in.in6.s6_addr, 0, sizeof(val.in.in6.s6_addr)); - } - } - - return val; - } - } diff --git a/src/input/ReaderBackend.h b/src/input/ReaderBackend.h index 9fd6c06aa3..73e5475db6 100644 --- a/src/input/ReaderBackend.h +++ b/src/input/ReaderBackend.h @@ -315,21 +315,6 @@ protected: */ void EndCurrentSend(); - /** - * Convert a string into a TransportProto. This is just a utility - * function for Readers. - * - * @param proto the transport protocol - */ - TransportProto StringToProto(const string &proto); - - /** - * Convert a string into a Value::addr_t. This is just a utility - * function for Readers. - * - * @param addr containing an ipv4 or ipv6 address - */ - threading::Value::addr_t StringToAddr(const string &addr); private: // Frontend that instantiated us. This object must not be accessed diff --git a/src/input/readers/Ascii.cc b/src/input/readers/Ascii.cc index e9cba27205..0965b556f2 100644 --- a/src/input/readers/Ascii.cc +++ b/src/input/readers/Ascii.cc @@ -67,11 +67,14 @@ Ascii::Ascii(ReaderFrontend *frontend) : ReaderBackend(frontend) unset_field.assign( (const char*) BifConst::InputAscii::unset_field->Bytes(), BifConst::InputAscii::unset_field->Len()); + + io = new AsciiInputOutput(this, separator, set_separator, empty_field, unset_field); } Ascii::~Ascii() { DoClose(); + delete io; } void Ascii::DoClose() @@ -210,228 +213,7 @@ bool Ascii::GetLine(string& str) return false; } -bool Ascii::CheckNumberError(const string& s, const char * end) - { - // Do this check first, before executing s.c_str() or similar. - // otherwise the value to which *end is pointing at the moment might - // be gone ... - bool endnotnull = (*end != '\0'); - if ( s.length() == 0 ) - { - Error("Got empty string for number field"); - return true; - } - - if ( end == s.c_str() ) { - Error(Fmt("String '%s' contained no parseable number", s.c_str())); - return true; - } - - if ( endnotnull ) - Warning(Fmt("Number '%s' contained non-numeric trailing characters. Ignored trailing characters '%s'", s.c_str(), end)); - - if ( errno == EINVAL ) - { - Error(Fmt("String '%s' could not be converted to a number", s.c_str())); - return true; - } - - else if ( errno == ERANGE ) - { - Error(Fmt("Number '%s' out of supported range.", s.c_str())); - return true; - } - - return false; - } - - -Value* Ascii::EntryToVal(string s, FieldMapping field) - { - if ( s.compare(unset_field) == 0 ) // field is not set... - return new Value(field.type, false); - - Value* val = new Value(field.type, true); - char* end = 0; - errno = 0; - - switch ( field.type ) { - case TYPE_ENUM: - case TYPE_STRING: - s = get_unescaped_string(s); - val->val.string_val.length = s.size(); - val->val.string_val.data = copy_string(s.c_str()); - break; - - case TYPE_BOOL: - if ( s == "T" ) - val->val.int_val = 1; - else if ( s == "F" ) - val->val.int_val = 0; - else - { - Error(Fmt("Field: %s Invalid value for boolean: %s", - field.name.c_str(), s.c_str())); - return 0; - } - break; - - case TYPE_INT: - val->val.int_val = strtoll(s.c_str(), &end, 10); - if ( CheckNumberError(s, end) ) - return 0; - break; - - case TYPE_DOUBLE: - case TYPE_TIME: - case TYPE_INTERVAL: - val->val.double_val = strtod(s.c_str(), &end); - if ( CheckNumberError(s, end) ) - return 0; - break; - - case TYPE_COUNT: - case TYPE_COUNTER: - val->val.uint_val = strtoull(s.c_str(), &end, 10); - if ( CheckNumberError(s, end) ) - return 0; - break; - - case TYPE_PORT: - val->val.port_val.port = strtoull(s.c_str(), &end, 10); - if ( CheckNumberError(s, end) ) - return 0; - - val->val.port_val.proto = TRANSPORT_UNKNOWN; - break; - - case TYPE_SUBNET: - { - s = get_unescaped_string(s); - size_t pos = s.find("/"); - if ( pos == s.npos ) - { - Error(Fmt("Invalid value for subnet: %s", s.c_str())); - return 0; - } - - uint8_t width = (uint8_t) strtol(s.substr(pos+1).c_str(), &end, 10); - - if ( CheckNumberError(s, end) ) - return 0; - - string addr = s.substr(0, pos); - - val->val.subnet_val.prefix = StringToAddr(addr); - val->val.subnet_val.length = width; - break; - } - - case TYPE_ADDR: - s = get_unescaped_string(s); - val->val.addr_val = StringToAddr(s); - break; - - case TYPE_TABLE: - case TYPE_VECTOR: - // First - common initialization - // Then - initialization for table. - // Then - initialization for vector. - // Then - common stuff - { - // how many entries do we have... - unsigned int length = 1; - for ( unsigned int i = 0; i < s.size(); i++ ) - { - if ( s[i] == set_separator[0] ) - length++; - } - - unsigned int pos = 0; - - if ( s.compare(empty_field) == 0 ) - length = 0; - - Value** lvals = new Value* [length]; - - if ( field.type == TYPE_TABLE ) - { - val->val.set_val.vals = lvals; - val->val.set_val.size = length; - } - - else if ( field.type == TYPE_VECTOR ) - { - val->val.vector_val.vals = lvals; - val->val.vector_val.size = length; - } - - else - assert(false); - - if ( length == 0 ) - break; //empty - - istringstream splitstream(s); - while ( splitstream ) - { - string element; - - if ( ! getline(splitstream, element, set_separator[0]) ) - break; - - if ( pos >= length ) - { - Error(Fmt("Internal error while parsing set. pos %d >= length %d." - " Element: %s", pos, length, element.c_str())); - break; - } - - Value* newval = EntryToVal(element, field.subType()); - if ( newval == 0 ) - { - Error("Error while reading set"); - return 0; - } - - lvals[pos] = newval; - - pos++; - } - - // Test if the string ends with a set_separator... or if the - // complete string is empty. In either of these cases we have - // to push an empty val on top of it. - if ( s.empty() || *s.rbegin() == set_separator[0] ) - { - lvals[pos] = EntryToVal("", field.subType()); - if ( lvals[pos] == 0 ) - { - Error("Error while trying to add empty set element"); - return 0; - } - - pos++; - } - - if ( pos != length ) - { - Error(Fmt("Internal error while parsing set: did not find all elements: %s", s.c_str())); - return 0; - } - - break; - } - - default: - Error(Fmt("unsupported field format %d for %s", field.type, - field.name.c_str())); - return 0; - } - - return val; - } // read the entire file and send appropriate thingies back to InputMgr bool Ascii::DoUpdate() @@ -543,7 +325,7 @@ bool Ascii::DoUpdate() return false; } - Value* val = EntryToVal(stringfields[(*fit).position], *fit); + Value* val = io->EntryToVal(stringfields[(*fit).position], (*fit).name, (*fit).type, (*fit).subtype); if ( val == 0 ) { Error(Fmt("Could not convert line '%s' to Val. Ignoring line.", line.c_str())); @@ -557,7 +339,7 @@ bool Ascii::DoUpdate() assert(val->type == TYPE_PORT ); // Error(Fmt("Got type %d != PORT with secondary position!", val->type)); - val->val.port_val.proto = StringToProto(stringfields[(*fit).secondary_position]); + val->val.port_val.proto = io->StringToProto(stringfields[(*fit).secondary_position]); } fields[fpos] = val; diff --git a/src/input/readers/Ascii.h b/src/input/readers/Ascii.h index 6e693fc74b..ed641087c7 100644 --- a/src/input/readers/Ascii.h +++ b/src/input/readers/Ascii.h @@ -7,6 +7,7 @@ #include #include "../ReaderBackend.h" +#include "../../AsciiInputOutput.h" namespace input { namespace reader { @@ -47,8 +48,6 @@ private: bool ReadHeader(bool useCached); bool GetLine(string& str); - threading::Value* EntryToVal(string s, FieldMapping type); - bool CheckNumberError(const string& s, const char * end); ifstream* file; time_t mtime; @@ -64,6 +63,8 @@ private: string set_separator; string empty_field; string unset_field; + + AsciiInputOutput* io; }; diff --git a/src/input/readers/Benchmark.cc b/src/input/readers/Benchmark.cc index b8cec0f14d..4738c6e867 100644 --- a/src/input/readers/Benchmark.cc +++ b/src/input/readers/Benchmark.cc @@ -25,11 +25,15 @@ Benchmark::Benchmark(ReaderFrontend *frontend) : ReaderBackend(frontend) stopspreadat = int(BifConst::InputBenchmark::stopspreadat); timedspread = double(BifConst::InputBenchmark::timedspread); heartbeat_interval = double(BifConst::Threading::heartbeat_interval); + + io = new AsciiInputOutput(this); } Benchmark::~Benchmark() { DoClose(); + + delete io; } void Benchmark::DoClose() @@ -162,13 +166,13 @@ threading::Value* Benchmark::EntryToVal(TypeTag type, TypeTag subtype) case TYPE_SUBNET: { - val->val.subnet_val.prefix = StringToAddr("192.168.17.1"); + val->val.subnet_val.prefix = io->StringToAddr("192.168.17.1"); val->val.subnet_val.length = 16; } break; case TYPE_ADDR: - val->val.addr_val = StringToAddr("192.168.17.1"); + val->val.addr_val = io->StringToAddr("192.168.17.1"); break; case TYPE_TABLE: diff --git a/src/input/readers/Benchmark.h b/src/input/readers/Benchmark.h index bab564b12a..d259db463c 100644 --- a/src/input/readers/Benchmark.h +++ b/src/input/readers/Benchmark.h @@ -4,6 +4,7 @@ #define INPUT_READERS_BENCHMARK_H #include "../ReaderBackend.h" +#include "../../AsciiInputOutput.h" namespace input { namespace reader { @@ -38,6 +39,8 @@ private: double heartbeatstarttime; double timedspread; double heartbeat_interval; + + AsciiInputOutput* io; }; From 22f2fc009c455ba290cf7af0a2506d944fff6c0a Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Mon, 3 Dec 2012 13:57:15 -0800 Subject: [PATCH 05/43] change constructors --- src/AsciiInputOutput.cc | 13 +++++++++++-- src/AsciiInputOutput.h | 8 +++++++- src/input/readers/Ascii.cc | 2 +- src/logging/writers/Ascii.cc | 2 +- 4 files changed, 20 insertions(+), 5 deletions(-) diff --git a/src/AsciiInputOutput.cc b/src/AsciiInputOutput.cc index dd7369e8eb..be15f44442 100644 --- a/src/AsciiInputOutput.cc +++ b/src/AsciiInputOutput.cc @@ -13,12 +13,21 @@ AsciiInputOutput::AsciiInputOutput(threading::MsgThread* t) } AsciiInputOutput::AsciiInputOutput(threading::MsgThread* t, const string & separator, const string & set_separator, - const string & empty_field, const string & unset_field) + const string & unset_field, const string & empty_field) { thread = t; this->separator = separator; this->set_separator = set_separator; + this->unset_field = unset_field; this->empty_field = empty_field; + } + +AsciiInputOutput::AsciiInputOutput(threading::MsgThread* t, const string & separator, const string & set_separator, + const string & unset_field) + { + thread = t; + this->separator = separator; + this->set_separator = set_separator; this->unset_field = unset_field; } @@ -275,7 +284,7 @@ threading::Value* AsciiInputOutput::EntryToVal(string s, string name, TypeTag ty unsigned int pos = 0; - if ( s.compare(empty_field) == 0 ) + if ( empty_field.size() > 0 && s.compare(empty_field) == 0 ) length = 0; threading::Value** lvals = new threading::Value* [length]; diff --git a/src/AsciiInputOutput.h b/src/AsciiInputOutput.h index 1dd2a4dce0..ba05fb85de 100644 --- a/src/AsciiInputOutput.h +++ b/src/AsciiInputOutput.h @@ -15,7 +15,13 @@ class AsciiInputOutput { // Constructor that defines all separators, etc. // Use if you need either ValToODesc or EntryToVal. AsciiInputOutput(threading::MsgThread*, const string & separator, const string & set_separator, - const string & empty_field, const string & unset_field); + const string & unset_field, const string & empty_field); + + // Constructor that defines all separators, etc, besides empty_field, which is not needed for many + // non-ascii-based io sources. + // Use if you need either ValToODesc or EntryToVal. + AsciiInputOutput(threading::MsgThread*, const string & separator, const string & set_separator, + const string & unset_field); ~AsciiInputOutput(); diff --git a/src/input/readers/Ascii.cc b/src/input/readers/Ascii.cc index 0965b556f2..0d45059a0f 100644 --- a/src/input/readers/Ascii.cc +++ b/src/input/readers/Ascii.cc @@ -68,7 +68,7 @@ Ascii::Ascii(ReaderFrontend *frontend) : ReaderBackend(frontend) unset_field.assign( (const char*) BifConst::InputAscii::unset_field->Bytes(), BifConst::InputAscii::unset_field->Len()); - io = new AsciiInputOutput(this, separator, set_separator, empty_field, unset_field); + io = new AsciiInputOutput(this, separator, set_separator, unset_field, empty_field); } Ascii::~Ascii() diff --git a/src/logging/writers/Ascii.cc b/src/logging/writers/Ascii.cc index 9f045c1813..c9ac42f774 100644 --- a/src/logging/writers/Ascii.cc +++ b/src/logging/writers/Ascii.cc @@ -52,7 +52,7 @@ Ascii::Ascii(WriterFrontend* frontend) : WriterBackend(frontend) desc.EnableEscaping(); desc.AddEscapeSequence(separator); - io = new AsciiInputOutput(this, separator, set_separator, empty_field, unset_field); + io = new AsciiInputOutput(this, separator, set_separator, unset_field, empty_field); } Ascii::~Ascii() From 86c724caa0a2ea70208a71adbb16eb8c12f7c145 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Mon, 3 Dec 2012 14:01:50 -0800 Subject: [PATCH 06/43] and thinking about it, ascii-io doesn't need the separator --- src/AsciiInputOutput.cc | 6 ++---- src/AsciiInputOutput.h | 4 ++-- src/input/readers/Ascii.cc | 2 +- src/logging/writers/Ascii.cc | 2 +- 4 files changed, 6 insertions(+), 8 deletions(-) diff --git a/src/AsciiInputOutput.cc b/src/AsciiInputOutput.cc index be15f44442..28736b9a77 100644 --- a/src/AsciiInputOutput.cc +++ b/src/AsciiInputOutput.cc @@ -12,21 +12,19 @@ AsciiInputOutput::AsciiInputOutput(threading::MsgThread* t) thread = t; } -AsciiInputOutput::AsciiInputOutput(threading::MsgThread* t, const string & separator, const string & set_separator, +AsciiInputOutput::AsciiInputOutput(threading::MsgThread* t, const string & set_separator, const string & unset_field, const string & empty_field) { thread = t; - this->separator = separator; this->set_separator = set_separator; this->unset_field = unset_field; this->empty_field = empty_field; } -AsciiInputOutput::AsciiInputOutput(threading::MsgThread* t, const string & separator, const string & set_separator, +AsciiInputOutput::AsciiInputOutput(threading::MsgThread* t, const string & set_separator, const string & unset_field) { thread = t; - this->separator = separator; this->set_separator = set_separator; this->unset_field = unset_field; } diff --git a/src/AsciiInputOutput.h b/src/AsciiInputOutput.h index ba05fb85de..16198eae54 100644 --- a/src/AsciiInputOutput.h +++ b/src/AsciiInputOutput.h @@ -14,13 +14,13 @@ class AsciiInputOutput { // Constructor that defines all separators, etc. // Use if you need either ValToODesc or EntryToVal. - AsciiInputOutput(threading::MsgThread*, const string & separator, const string & set_separator, + AsciiInputOutput(threading::MsgThread*, const string & set_separator, const string & unset_field, const string & empty_field); // Constructor that defines all separators, etc, besides empty_field, which is not needed for many // non-ascii-based io sources. // Use if you need either ValToODesc or EntryToVal. - AsciiInputOutput(threading::MsgThread*, const string & separator, const string & set_separator, + AsciiInputOutput(threading::MsgThread*, const string & set_separator, const string & unset_field); ~AsciiInputOutput(); diff --git a/src/input/readers/Ascii.cc b/src/input/readers/Ascii.cc index 0d45059a0f..e33844244a 100644 --- a/src/input/readers/Ascii.cc +++ b/src/input/readers/Ascii.cc @@ -68,7 +68,7 @@ Ascii::Ascii(ReaderFrontend *frontend) : ReaderBackend(frontend) unset_field.assign( (const char*) BifConst::InputAscii::unset_field->Bytes(), BifConst::InputAscii::unset_field->Len()); - io = new AsciiInputOutput(this, separator, set_separator, unset_field, empty_field); + io = new AsciiInputOutput(this, set_separator, unset_field, empty_field); } Ascii::~Ascii() diff --git a/src/logging/writers/Ascii.cc b/src/logging/writers/Ascii.cc index c9ac42f774..1403effe49 100644 --- a/src/logging/writers/Ascii.cc +++ b/src/logging/writers/Ascii.cc @@ -52,7 +52,7 @@ Ascii::Ascii(WriterFrontend* frontend) : WriterBackend(frontend) desc.EnableEscaping(); desc.AddEscapeSequence(separator); - io = new AsciiInputOutput(this, separator, set_separator, unset_field, empty_field); + io = new AsciiInputOutput(this, set_separator, unset_field, empty_field); } Ascii::~Ascii() From 35ac787cd8caa9e238a5832d15e0d15a946b7e14 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Thu, 13 Dec 2012 11:53:18 -0800 Subject: [PATCH 07/43] move AsciiInputOutput over to threading --- src/CMakeLists.txt | 2 +- src/input/readers/Ascii.h | 2 +- src/input/readers/Benchmark.h | 2 +- src/logging/writers/Ascii.h | 2 +- src/logging/writers/ElasticSearch.cc | 2 +- src/{ => threading}/AsciiInputOutput.cc | 2 +- src/{ => threading}/AsciiInputOutput.h | 4 ++-- 7 files changed, 8 insertions(+), 8 deletions(-) rename src/{ => threading}/AsciiInputOutput.cc (99%) rename src/{ => threading}/AsciiInputOutput.h (98%) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 3f414d0953..9c504d4ef5 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -286,7 +286,6 @@ set(bro_SRCS Analyzer.cc Anon.cc ARP.cc - AsciiInputOutput.cc Attr.cc AYIYA.cc BackDoor.cc @@ -426,6 +425,7 @@ set(bro_SRCS strsep.c modp_numtoa.c + threading/AsciiInputOutput.cc threading/BasicThread.cc threading/Manager.cc threading/MsgThread.cc diff --git a/src/input/readers/Ascii.h b/src/input/readers/Ascii.h index ed641087c7..52078bf3d6 100644 --- a/src/input/readers/Ascii.h +++ b/src/input/readers/Ascii.h @@ -7,7 +7,7 @@ #include #include "../ReaderBackend.h" -#include "../../AsciiInputOutput.h" +#include "../../threading/AsciiInputOutput.h" namespace input { namespace reader { diff --git a/src/input/readers/Benchmark.h b/src/input/readers/Benchmark.h index d259db463c..e83f446caa 100644 --- a/src/input/readers/Benchmark.h +++ b/src/input/readers/Benchmark.h @@ -4,7 +4,7 @@ #define INPUT_READERS_BENCHMARK_H #include "../ReaderBackend.h" -#include "../../AsciiInputOutput.h" +#include "../../threading/AsciiInputOutput.h" namespace input { namespace reader { diff --git a/src/logging/writers/Ascii.h b/src/logging/writers/Ascii.h index 5236932889..f85d240c95 100644 --- a/src/logging/writers/Ascii.h +++ b/src/logging/writers/Ascii.h @@ -6,7 +6,7 @@ #define LOGGING_WRITER_ASCII_H #include "../WriterBackend.h" -#include "../../AsciiInputOutput.h" +#include "../../threading/AsciiInputOutput.h" namespace logging { namespace writer { diff --git a/src/logging/writers/ElasticSearch.cc b/src/logging/writers/ElasticSearch.cc index a5897e7272..e9f5ab456b 100644 --- a/src/logging/writers/ElasticSearch.cc +++ b/src/logging/writers/ElasticSearch.cc @@ -16,7 +16,7 @@ #include "BroString.h" #include "NetVar.h" #include "threading/SerialTypes.h" -#include "../../AsciiInputOutput.h" +#include "../../threading/AsciiInputOutput.h" #include #include diff --git a/src/AsciiInputOutput.cc b/src/threading/AsciiInputOutput.cc similarity index 99% rename from src/AsciiInputOutput.cc rename to src/threading/AsciiInputOutput.cc index 28736b9a77..0f2198a229 100644 --- a/src/AsciiInputOutput.cc +++ b/src/threading/AsciiInputOutput.cc @@ -5,7 +5,7 @@ #include #include #include "AsciiInputOutput.h" -#include "bro_inet_ntop.h" +#include "../bro_inet_ntop.h" AsciiInputOutput::AsciiInputOutput(threading::MsgThread* t) { diff --git a/src/AsciiInputOutput.h b/src/threading/AsciiInputOutput.h similarity index 98% rename from src/AsciiInputOutput.h rename to src/threading/AsciiInputOutput.h index 16198eae54..b988bb9e63 100644 --- a/src/AsciiInputOutput.h +++ b/src/threading/AsciiInputOutput.h @@ -3,8 +3,8 @@ #ifndef AsciiInputOutput_h #define AsciiInputOutput_h -#include "Desc.h" -#include "threading/MsgThread.h" +#include "../Desc.h" +#include "MsgThread.h" class AsciiInputOutput { public: From f03a8cae0499f9614c7c2936eb1015503f362f0f Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Thu, 13 Dec 2012 11:59:59 -0800 Subject: [PATCH 08/43] harmonize function naming --- src/input/readers/Ascii.cc | 2 +- src/threading/AsciiInputOutput.cc | 6 +++--- src/threading/AsciiInputOutput.h | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/input/readers/Ascii.cc b/src/input/readers/Ascii.cc index 557de05675..950a3d5c9b 100644 --- a/src/input/readers/Ascii.cc +++ b/src/input/readers/Ascii.cc @@ -325,7 +325,7 @@ bool Ascii::DoUpdate() return false; } - Value* val = io->EntryToVal(stringfields[(*fit).position], (*fit).name, (*fit).type, (*fit).subtype); + Value* val = io->StringToVal(stringfields[(*fit).position], (*fit).name, (*fit).type, (*fit).subtype); if ( val == 0 ) { Error(Fmt("Could not convert line '%s' to Val. Ignoring line.", line.c_str())); diff --git a/src/threading/AsciiInputOutput.cc b/src/threading/AsciiInputOutput.cc index 0f2198a229..560a6dfc38 100644 --- a/src/threading/AsciiInputOutput.cc +++ b/src/threading/AsciiInputOutput.cc @@ -179,7 +179,7 @@ bool AsciiInputOutput::ValToODesc(ODesc* desc, threading::Value* val, const thre } -threading::Value* AsciiInputOutput::EntryToVal(string s, string name, TypeTag type, TypeTag subtype) const +threading::Value* AsciiInputOutput::StringToVal(string s, string name, TypeTag type, TypeTag subtype) const { if ( s.compare(unset_field) == 0 ) // field is not set... return new threading::Value(type, false); @@ -320,7 +320,7 @@ threading::Value* AsciiInputOutput::EntryToVal(string s, string name, TypeTag ty break; } - threading::Value* newval = EntryToVal(element, name, subtype); + threading::Value* newval = StringToVal(element, name, subtype); if ( newval == 0 ) { thread->Error("Error while reading set"); @@ -337,7 +337,7 @@ threading::Value* AsciiInputOutput::EntryToVal(string s, string name, TypeTag ty // to push an empty val on top of it. if ( s.empty() || *s.rbegin() == set_separator[0] ) { - lvals[pos] = EntryToVal("", name, subtype); + lvals[pos] = StringToVal("", name, subtype); if ( lvals[pos] == 0 ) { thread->Error("Error while trying to add empty set element"); diff --git a/src/threading/AsciiInputOutput.h b/src/threading/AsciiInputOutput.h index b988bb9e63..9f36175a5a 100644 --- a/src/threading/AsciiInputOutput.h +++ b/src/threading/AsciiInputOutput.h @@ -30,7 +30,7 @@ class AsciiInputOutput { bool ValToODesc(ODesc* desc, threading::Value* val, const threading::Field* field) const; // convert the ascii representation of a field into a Value - threading::Value* EntryToVal(string s, string name, TypeTag type, TypeTag subtype = TYPE_ERROR) const; + threading::Value* StringToVal(string s, string name, TypeTag type, TypeTag subtype = TYPE_ERROR) const; /** Helper method to render an IP address as a string. * From ff8184242aa8383775a42406b6fd1fb01266462d Mon Sep 17 00:00:00 2001 From: Matthias Vallentin Date: Fri, 21 Dec 2012 17:03:39 -0800 Subject: [PATCH 09/43] Enhance ssl.log with information from notary. This commit brings enhances each log line with the data from the notary when available. The added fields include: - notary.first_seen - notary.last_seen - notary.times_seen - notary.valid The semantics of these fields map 1-to-1 to the corresponding fields in DNS TXT lookups from the notary. The implementation of this feature required a bit plumbing: when Bro finishes the analysis, the log record is copied into table indexed by connection ID where it remains until either Bro terminates or the answer of the notary arrives. The script accummulates requests for a given digest into a "waitlist," to avoid multiple redundant lookups for high-profile websites who receive a large chunk of traffic. When a DNS reply arrives asynchronously, the when handler clears the waitlist and assigns the information to all records in the buffered. The script also adds Each log entry into a double-ended queue to make sure the records arrive on disk in the same way Bro sees them. Each reply also triggers a sweep through this deque which flushes the buffer up to the first outstanding reply. Here is an example from the public M57 trace from 2009: % bro-cut ts id.orig_h id.resp_h server_name notary.first_seen notary.last_seen notary.times_seen notary.valid < ssl.log 1258562650.121682 192.168.1.104 208.97.132.223 mail.m57.biz - - - - 1258535660.267128 192.168.1.104 65.55.184.16 - - - - - 1258561662.604948 192.168.1.105 66.235.128.158 - - - - - 1258561885.571010 192.168.1.105 65.55.184.155 www.update.microsoft.com - - - - 1258563578.455331 192.168.1.103 208.97.132.223 - - - - - 1258563716.527681 192.168.1.103 96.6.248.124 - - - - - 1258563884.667153 192.168.1.103 66.235.139.152 - - - - - 1258564818.755676 192.168.1.103 12.41.118.177 - - - - - 1258564821.637874 192.168.1.103 12.41.118.177 - - - - - 1258564821.637871 192.168.1.103 12.41.118.177 - - - - - 1258564821.637876 192.168.1.103 12.41.118.177 - - - - - 1258564821.638126 192.168.1.103 12.41.118.177 - - - - - 1258562467.525034 192.168.1.104 208.97.132.223 mail.m57.biz 15392 15695 301 F 1258563063.965975 192.168.1.104 63.245.209.105 aus2.mozilla.org - - - - 1258563064.091396 192.168.1.104 63.245.209.91 addons.mozilla.org - - - - 1258563329.202273 192.168.1.103 208.97.132.223 - 15392 15695 301 F 1258563712.945933 192.168.1.103 65.55.16.121 - - - - - 1258563714.044500 192.168.1.103 65.54.186.79 - - - - - 1258563716.146680 192.168.1.103 96.6.248.124 - - - - - 1258563737.432312 192.168.1.103 96.6.245.186 - - - - - 1258563716.526933 192.168.1.103 96.6.245.186 - - - - - 1258563716.527430 192.168.1.103 96.6.245.186 - - - - - 1258563716.527179 192.168.1.103 96.6.245.186 - - - - - 1258563716.527683 192.168.1.103 96.6.245.186 - - - - - 1258563716.527432 192.168.1.103 96.6.245.186 - - - - - 1258563751.178683 192.168.1.103 66.235.139.152 - - - - - 1258563751.171938 192.168.1.103 65.54.234.75 - - - - - 1258563751.182433 192.168.1.103 65.242.27.35 - - - - - 1258563883.414188 192.168.1.103 65.55.16.121 - - - - - 1258563884.702380 192.168.1.103 65.242.27.35 - - - - - 1258563885.678766 192.168.1.103 65.54.186.79 - - - - - 1258563886.124987 192.168.1.103 65.54.186.79 - - - - - 1258564027.877525 192.168.1.103 65.54.234.75 - - - - - 1258564688.206859 192.168.1.103 65.54.186.107 - - - - - 1258567162.001225 192.168.1.105 208.97.132.223 mail.m57.biz - - - - 1258568040.512840 192.168.1.103 208.97.132.223 - - - - - 1258564688.577376 192.168.1.103 207.46.120.170 - - - - - 1258564723.029005 192.168.1.103 65.54.186.107 - - - - - 1258564723.784032 192.168.1.103 65.55.194.249 - - - - - 1258564748.521756 192.168.1.103 65.54.186.107 - - - - - 1258564817.601152 192.168.1.103 12.41.118.177 - - - - - 1258565684.353653 192.168.1.105 208.97.132.223 mail.m57.biz 15392 15695 301 F 1258565710.188691 192.168.1.105 74.125.155.109 pop.gmail.com - - - - 1258566061.103696 192.168.1.103 208.97.132.223 - 15392 15695 301 F 1258566893.914987 192.168.1.102 208.97.132.223 - 15392 15695 301 F --- scripts/base/protocols/ssl/main.bro | 157 +++++++++++++++++++++++++++- 1 file changed, 156 insertions(+), 1 deletion(-) diff --git a/scripts/base/protocols/ssl/main.bro b/scripts/base/protocols/ssl/main.bro index 6b434ae09d..60f17ea7ce 100644 --- a/scripts/base/protocols/ssl/main.bro +++ b/scripts/base/protocols/ssl/main.bro @@ -8,6 +8,14 @@ module SSL; export { redef enum Log::ID += { LOG }; + ## A response from the ICSI certificate notary. + type NotaryResponse: record { + first_seen: count &log &optional; + last_seen: count &log &optional; + times_seen: count &log &optional; + valid: bool &log &optional; + }; + type Info: record { ## Time when the SSL connection was first detected. ts: time &log; @@ -72,15 +80,48 @@ export { ## utility. const openssl_util = "openssl" &redef; + ## Flag that determines whether to use the ICSI certificate notary to enhance + ## the SSL log records. + const use_notary = T &redef; + ## Event that can be handled to access the SSL ## record as it is sent on to the logging framework. global log_ssl: event(rec: Info); } +# TODO: Maybe wrap these in @ifdef's? Otherwise we carry the extra baggage +# around all the time. +redef record Info += { + sha1_digest: string &optional; + notary: NotaryResponse &log &optional; + }; + redef record connection += { ssl: Info &optional; }; +# The DNS cache of notary responses. +global notary_cache: table[string] of NotaryResponse &create_expire = 1 hr; + +# The buffered SSL log records. +global records: table[string] of Info; + +# The records that wait for a notary response identified by the cert digest. +# Each digest refers to a list of connection UIDs which are updated when a DNS +# reply arrives asynchronously. +global waiting: table[string] of vector of string; + +# A double-ended queue that determines the log record order in which logs have +# to written out to disk. +global deque: table[count] of string; + +# The top-most deque index. +global head = 0; + +# The bottom deque index that points to the next record to be flushed as soon +# as the notary response arrives. +global tail = 0; + event bro_init() &priority=5 { Log::create_stream(SSL::LOG, [$columns=Info, $ev=log_ssl]); @@ -122,9 +163,103 @@ function set_session(c: connection) $client_cert_chain=vector()]; } +function clear_waitlist(digest: string) + { + if ( digest in waiting ) + { + for ( i in waiting[digest] ) + { + local uid = waiting[digest][i]; + records[uid]$notary = []; + } + delete waiting[digest]; + } + } + +function lookup_cert_hash(uid: string, digest: string) + { + # Add the record ID to the list of waiting IDs for this digest. + local waits_already = digest in waiting; + if ( ! waits_already ) + waiting[digest] = vector(); + waiting[digest][|waiting[digest]|] = uid; + if ( waits_already ) + return; + + local domain = "%s.notary.icsi.berkeley.edu"; + when ( local str = lookup_hostname_txt(fmt(domain, digest)) ) + { + # Cache every response for a digest. + # TODO: should we ignore failing answers? + notary_cache[digest] = []; + + # Parse notary answer. + if ( str == "" ) + { + # TODO: Should we handle NXDOMAIN separately? + clear_waitlist(digest); + return; + } + local fields = split(str, / /); + if ( |fields| != 5 ) # version 1 has 5 fields. + { + clear_waitlist(digest); + return; + } + local version = split(fields[1], /=/)[2]; + if ( version != "1" ) + { + clear_waitlist(digest); + return; + } + local r = notary_cache[digest]; + r$first_seen = to_count(split(fields[2], /=/)[2]); + r$last_seen = to_count(split(fields[3], /=/)[2]); + r$times_seen = to_count(split(fields[4], /=/)[2]); + r$valid = split(fields[5], /=/)[2] == "1"; + + # Assign notary answer to all waiting records. + if ( digest in waiting ) + { + for ( i in waiting[digest] ) + records[waiting[digest][i]]$notary = r; + delete waiting[digest]; + } + + # Flush all records up to the record which still awaits an answer. + local current: string; + for ( unused_index in deque ) + { + current = deque[tail]; + local info = records[current]; + if ( ! info?$notary ) + break; + Log::write(SSL::LOG, info); + delete deque[tail]; + delete records[current]; + ++tail; + } + } + } + function finish(c: connection) { - Log::write(SSL::LOG, c$ssl); + if ( use_notary && c$ssl?$sha1_digest) + { + local digest = c$ssl$sha1_digest; + if ( c$ssl$sha1_digest in notary_cache ) + c$ssl$notary = notary_cache[digest]; + else + lookup_cert_hash(c$uid, digest); + records[c$uid] = c$ssl; # Copy the record. + deque[head] = c$uid; + ++head; + } + else + { + Log::write(SSL::LOG, c$ssl); + } + if ( disable_analyzer_after_detection && c?$ssl && c$ssl?$analyzer_id ) disable_analyzer(c$id, c$ssl$analyzer_id); delete c$ssl; @@ -181,6 +316,9 @@ event x509_certificate(c: connection, is_orig: bool, cert: X509, chain_idx: coun c$ssl$issuer_subject = cert$issuer; c$ssl$not_valid_before = cert$not_valid_before; c$ssl$not_valid_after = cert$not_valid_after; + + if ( use_notary ) + c$ssl$sha1_digest = sha1_hash(der_cert); } else { @@ -228,3 +366,20 @@ event protocol_violation(c: connection, atype: count, aid: count, if ( c?$ssl ) finish(c); } + +event bro_done() + { + if ( ! use_notary || |deque| == 0 ) + return; + + local current: string; + for ( unused_index in deque ) + { + current = deque[tail]; + local info = records[current]; + Log::write(SSL::LOG, info); + delete deque[tail]; + delete records[current]; + ++tail; + } + } From 7355a0089ae4c1d61f4e9f842763e8689514769a Mon Sep 17 00:00:00 2001 From: Matthias Vallentin Date: Fri, 21 Dec 2012 17:17:58 -0800 Subject: [PATCH 10/43] Adhere to Bro coding style guidelines. --- scripts/base/protocols/ssl/main.bro | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/scripts/base/protocols/ssl/main.bro b/scripts/base/protocols/ssl/main.bro index 60f17ea7ce..e36fbd90e6 100644 --- a/scripts/base/protocols/ssl/main.bro +++ b/scripts/base/protocols/ssl/main.bro @@ -208,10 +208,10 @@ function lookup_cert_hash(uid: string, digest: string) } local version = split(fields[1], /=/)[2]; if ( version != "1" ) - { + { clear_waitlist(digest); return; - } + } local r = notary_cache[digest]; r$first_seen = to_count(split(fields[2], /=/)[2]); r$last_seen = to_count(split(fields[3], /=/)[2]); @@ -220,24 +220,24 @@ function lookup_cert_hash(uid: string, digest: string) # Assign notary answer to all waiting records. if ( digest in waiting ) - { + { for ( i in waiting[digest] ) records[waiting[digest][i]]$notary = r; delete waiting[digest]; - } + } # Flush all records up to the record which still awaits an answer. local current: string; for ( unused_index in deque ) { - current = deque[tail]; - local info = records[current]; - if ( ! info?$notary ) - break; - Log::write(SSL::LOG, info); - delete deque[tail]; - delete records[current]; - ++tail; + current = deque[tail]; + local info = records[current]; + if ( ! info?$notary ) + break; + Log::write(SSL::LOG, info); + delete deque[tail]; + delete records[current]; + ++tail; } } } From 382262e286bb923ca38dd156217e14988656bc2f Mon Sep 17 00:00:00 2001 From: Matthias Vallentin Date: Fri, 21 Dec 2012 17:56:31 -0800 Subject: [PATCH 11/43] Factor notary code into separte file. There exists one complication: the new file notary.bro requires the definition of the SSL::Info record, but as does main.bro. Because I did not really know where to put the common code (it's not a constant, so ssl/const.bro does not really fit), I put it into __load.bro__ so that it sticks out for now. If anybody has an idea how to solve this elegantly, please let me know. --- scripts/base/protocols/ssl/__load__.bro | 64 ++++++- scripts/base/protocols/ssl/main.bro | 218 +----------------------- scripts/base/protocols/ssl/notary.bro | 162 ++++++++++++++++++ 3 files changed, 232 insertions(+), 212 deletions(-) create mode 100644 scripts/base/protocols/ssl/notary.bro diff --git a/scripts/base/protocols/ssl/__load__.bro b/scripts/base/protocols/ssl/__load__.bro index eaaa13cd76..e97d1681fc 100644 --- a/scripts/base/protocols/ssl/__load__.bro +++ b/scripts/base/protocols/ssl/__load__.bro @@ -1,3 +1,65 @@ @load ./consts + +module SSL; + +export { + redef enum Log::ID += { LOG }; + + type Info: record { + ## Time when the SSL connection was first detected. + ts: time &log; + ## Unique ID for the connection. + uid: string &log; + ## The connection's 4-tuple of endpoint addresses/ports. + id: conn_id &log; + ## SSL/TLS version that the server offered. + version: string &log &optional; + ## SSL/TLS cipher suite that the server chose. + cipher: string &log &optional; + ## Value of the Server Name Indicator SSL/TLS extension. It + ## indicates the server name that the client was requesting. + server_name: string &log &optional; + ## Session ID offered by the client for session resumption. + session_id: string &log &optional; + ## Subject of the X.509 certificate offered by the server. + subject: string &log &optional; + ## Subject of the signer of the X.509 certificate offered by the server. + issuer_subject: string &log &optional; + ## NotValidBefore field value from the server certificate. + not_valid_before: time &log &optional; + ## NotValidAfter field value from the server certificate. + not_valid_after: time &log &optional; + ## Last alert that was seen during the connection. + last_alert: string &log &optional; + + ## Subject of the X.509 certificate offered by the client. + client_subject: string &log &optional; + ## Subject of the signer of the X.509 certificate offered by the client. + client_issuer_subject: string &log &optional; + + ## Full binary server certificate stored in DER format. + cert: string &optional; + ## Chain of certificates offered by the server to validate its + ## complete signing chain. + cert_chain: vector of string &optional; + + ## Full binary client certificate stored in DER format. + client_cert: string &optional; + ## Chain of certificates offered by the client to validate its + ## complete signing chain. + client_cert_chain: vector of string &optional; + + ## The analyzer ID used for the analyzer instance attached + ## to each connection. It is not used for logging since it's a + ## meaningless arbitrary number. + analyzer_id: count &optional; + }; +} + +redef record connection += { + ssl: Info &optional; +}; + +@load ./notary @load ./main -@load ./mozilla-ca-list \ No newline at end of file +@load ./mozilla-ca-list diff --git a/scripts/base/protocols/ssl/main.bro b/scripts/base/protocols/ssl/main.bro index e36fbd90e6..f02602235c 100644 --- a/scripts/base/protocols/ssl/main.bro +++ b/scripts/base/protocols/ssl/main.bro @@ -6,66 +6,6 @@ module SSL; export { - redef enum Log::ID += { LOG }; - - ## A response from the ICSI certificate notary. - type NotaryResponse: record { - first_seen: count &log &optional; - last_seen: count &log &optional; - times_seen: count &log &optional; - valid: bool &log &optional; - }; - - type Info: record { - ## Time when the SSL connection was first detected. - ts: time &log; - ## Unique ID for the connection. - uid: string &log; - ## The connection's 4-tuple of endpoint addresses/ports. - id: conn_id &log; - ## SSL/TLS version that the server offered. - version: string &log &optional; - ## SSL/TLS cipher suite that the server chose. - cipher: string &log &optional; - ## Value of the Server Name Indicator SSL/TLS extension. It - ## indicates the server name that the client was requesting. - server_name: string &log &optional; - ## Session ID offered by the client for session resumption. - session_id: string &log &optional; - ## Subject of the X.509 certificate offered by the server. - subject: string &log &optional; - ## Subject of the signer of the X.509 certificate offered by the server. - issuer_subject: string &log &optional; - ## NotValidBefore field value from the server certificate. - not_valid_before: time &log &optional; - ## NotValidAfter field value from the server certificate. - not_valid_after: time &log &optional; - ## Last alert that was seen during the connection. - last_alert: string &log &optional; - - ## Subject of the X.509 certificate offered by the client. - client_subject: string &log &optional; - ## Subject of the signer of the X.509 certificate offered by the client. - client_issuer_subject: string &log &optional; - - ## Full binary server certificate stored in DER format. - cert: string &optional; - ## Chain of certificates offered by the server to validate its - ## complete signing chain. - cert_chain: vector of string &optional; - - ## Full binary client certificate stored in DER format. - client_cert: string &optional; - ## Chain of certificates offered by the client to validate its - ## complete signing chain. - client_cert_chain: vector of string &optional; - - ## The analyzer ID used for the analyzer instance attached - ## to each connection. It is not used for logging since it's a - ## meaningless arbitrary number. - analyzer_id: count &optional; - }; - ## The default root CA bundle. By loading the ## mozilla-ca-list.bro script it will be set to Mozilla's root CA list. const root_certs: table[string] of string = {} &redef; @@ -80,48 +20,11 @@ export { ## utility. const openssl_util = "openssl" &redef; - ## Flag that determines whether to use the ICSI certificate notary to enhance - ## the SSL log records. - const use_notary = T &redef; - ## Event that can be handled to access the SSL ## record as it is sent on to the logging framework. global log_ssl: event(rec: Info); } -# TODO: Maybe wrap these in @ifdef's? Otherwise we carry the extra baggage -# around all the time. -redef record Info += { - sha1_digest: string &optional; - notary: NotaryResponse &log &optional; - }; - -redef record connection += { - ssl: Info &optional; -}; - -# The DNS cache of notary responses. -global notary_cache: table[string] of NotaryResponse &create_expire = 1 hr; - -# The buffered SSL log records. -global records: table[string] of Info; - -# The records that wait for a notary response identified by the cert digest. -# Each digest refers to a list of connection UIDs which are updated when a DNS -# reply arrives asynchronously. -global waiting: table[string] of vector of string; - -# A double-ended queue that determines the log record order in which logs have -# to written out to disk. -global deque: table[count] of string; - -# The top-most deque index. -global head = 0; - -# The bottom deque index that points to the next record to be flushed as soon -# as the notary response arrives. -global tail = 0; - event bro_init() &priority=5 { Log::create_stream(SSL::LOG, [$columns=Info, $ev=log_ssl]); @@ -163,102 +66,15 @@ function set_session(c: connection) $client_cert_chain=vector()]; } -function clear_waitlist(digest: string) - { - if ( digest in waiting ) - { - for ( i in waiting[digest] ) - { - local uid = waiting[digest][i]; - records[uid]$notary = []; - } - delete waiting[digest]; - } - } - -function lookup_cert_hash(uid: string, digest: string) - { - # Add the record ID to the list of waiting IDs for this digest. - local waits_already = digest in waiting; - if ( ! waits_already ) - waiting[digest] = vector(); - waiting[digest][|waiting[digest]|] = uid; - if ( waits_already ) - return; - - local domain = "%s.notary.icsi.berkeley.edu"; - when ( local str = lookup_hostname_txt(fmt(domain, digest)) ) - { - # Cache every response for a digest. - # TODO: should we ignore failing answers? - notary_cache[digest] = []; - - # Parse notary answer. - if ( str == "" ) - { - # TODO: Should we handle NXDOMAIN separately? - clear_waitlist(digest); - return; - } - local fields = split(str, / /); - if ( |fields| != 5 ) # version 1 has 5 fields. - { - clear_waitlist(digest); - return; - } - local version = split(fields[1], /=/)[2]; - if ( version != "1" ) - { - clear_waitlist(digest); - return; - } - local r = notary_cache[digest]; - r$first_seen = to_count(split(fields[2], /=/)[2]); - r$last_seen = to_count(split(fields[3], /=/)[2]); - r$times_seen = to_count(split(fields[4], /=/)[2]); - r$valid = split(fields[5], /=/)[2] == "1"; - - # Assign notary answer to all waiting records. - if ( digest in waiting ) - { - for ( i in waiting[digest] ) - records[waiting[digest][i]]$notary = r; - delete waiting[digest]; - } - - # Flush all records up to the record which still awaits an answer. - local current: string; - for ( unused_index in deque ) - { - current = deque[tail]; - local info = records[current]; - if ( ! info?$notary ) - break; - Log::write(SSL::LOG, info); - delete deque[tail]; - delete records[current]; - ++tail; - } - } - } - function finish(c: connection) { - if ( use_notary && c$ssl?$sha1_digest) - { - local digest = c$ssl$sha1_digest; - if ( c$ssl$sha1_digest in notary_cache ) - c$ssl$notary = notary_cache[digest]; - else - lookup_cert_hash(c$uid, digest); - records[c$uid] = c$ssl; # Copy the record. - deque[head] = c$uid; - ++head; - } - else - { - Log::write(SSL::LOG, c$ssl); - } +# TODO: This dummy flag merely exists to check whether the notary script is +# loaded. There's probably a better way to incorporate the notary. +@ifdef( Notary::enabled ) + Notary::push(c$ssl); +@else + Log::write(SSL::LOG, c$ssl); +@endif if ( disable_analyzer_after_detection && c?$ssl && c$ssl?$analyzer_id ) disable_analyzer(c$id, c$ssl$analyzer_id); @@ -316,9 +132,6 @@ event x509_certificate(c: connection, is_orig: bool, cert: X509, chain_idx: coun c$ssl$issuer_subject = cert$issuer; c$ssl$not_valid_before = cert$not_valid_before; c$ssl$not_valid_after = cert$not_valid_after; - - if ( use_notary ) - c$ssl$sha1_digest = sha1_hash(der_cert); } else { @@ -366,20 +179,3 @@ event protocol_violation(c: connection, atype: count, aid: count, if ( c?$ssl ) finish(c); } - -event bro_done() - { - if ( ! use_notary || |deque| == 0 ) - return; - - local current: string; - for ( unused_index in deque ) - { - current = deque[tail]; - local info = records[current]; - Log::write(SSL::LOG, info); - delete deque[tail]; - delete records[current]; - ++tail; - } - } diff --git a/scripts/base/protocols/ssl/notary.bro b/scripts/base/protocols/ssl/notary.bro new file mode 100644 index 0000000000..6cf687ea90 --- /dev/null +++ b/scripts/base/protocols/ssl/notary.bro @@ -0,0 +1,162 @@ +module Notary; + +export { + # Flag to tell the SSL analysis script that it should buffer logs instead of + # flushing them directly. + const enabled = T; + + ## A response from the ICSI certificate notary. + type Response: record { + first_seen: count &log &optional; + last_seen: count &log &optional; + times_seen: count &log &optional; + valid: bool &log &optional; + }; + + ## Hands over an SSL record to the Notary module. This is an ownership + ## transfer, i.e., the caller does not need to call Log::write on this record + ## anymore. + global push: function(info: SSL::Info); + + ## The notary domain to query. + const domain = "notary.icsi.berkeley.edu" &redef; +} + +redef record SSL::Info += { + sha1_digest: string &optional; + notary: Response &log &optional; + }; + +# The DNS cache of notary responses. +global notary_cache: table[string] of Response &create_expire = 1 hr; + +# The buffered SSL log records. +global records: table[string] of SSL::Info; + +# The records that wait for a notary response identified by the cert digest. +# Each digest refers to a list of connection UIDs which are updated when a DNS +# reply arrives asynchronously. +global waiting: table[string] of vector of string; + +# A double-ended queue that determines the log record order in which logs have +# to written out to disk. +global deque: table[count] of string; + +# The top-most deque index. +global head = 0; + +# The bottom deque index that points to the next record to be flushed as soon +# as the notary response arrives. +global tail = 0; + +function clear_waitlist(digest: string) + { + if ( digest in waiting ) + { + for ( i in waiting[digest] ) + { + local uid = waiting[digest][i]; + records[uid]$notary = []; + } + delete waiting[digest]; + } + } + +function flush(evict_all: bool) + { + local current: string; + for ( unused_index in deque ) + { + current = deque[tail]; + local info = records[current]; + if ( ! evict_all && ! info?$notary ) + break; + Log::write(SSL::LOG, info); + delete deque[tail]; + delete records[current]; + ++tail; + } + } + +function lookup_cert_hash(uid: string, digest: string) + { + # Add the record ID to the list of waiting IDs for this digest. + local waits_already = digest in waiting; + if ( ! waits_already ) + waiting[digest] = vector(); + waiting[digest][|waiting[digest]|] = uid; + if ( waits_already ) + return; + + when ( local str = lookup_hostname_txt(fmt("%s.%s", digest, domain)) ) + { + # Cache every response for a digest. + notary_cache[digest] = []; + + # Parse notary answer. + if ( str == "" ) + { + # TODO: Should we handle NXDOMAIN separately? + clear_waitlist(digest); + return; + } + local fields = split(str, / /); + if ( |fields| != 5 ) # version 1 has 5 fields. + { + clear_waitlist(digest); + return; + } + local version = split(fields[1], /=/)[2]; + if ( version != "1" ) + { + clear_waitlist(digest); + return; + } + local r = notary_cache[digest]; + r$first_seen = to_count(split(fields[2], /=/)[2]); + r$last_seen = to_count(split(fields[3], /=/)[2]); + r$times_seen = to_count(split(fields[4], /=/)[2]); + r$valid = split(fields[5], /=/)[2] == "1"; + + # Assign notary answer to all waiting records. + if ( digest in waiting ) + { + for ( i in waiting[digest] ) + records[waiting[digest][i]]$notary = r; + delete waiting[digest]; + } + + flush(F); + } + } + +function push(info: SSL::Info) + { + if ( ! info?$sha1_digest ) + return; + + local digest = info$sha1_digest; + if ( info$sha1_digest in notary_cache ) + info$notary = notary_cache[digest]; + else + lookup_cert_hash(info$uid, digest); + records[info$uid] = info; + deque[head] = info$uid; + ++head; + } + +event x509_certificate(c: connection, is_orig: bool, cert: X509, + chain_idx: count, chain_len: count, der_cert: string) + { + if ( is_orig || chain_idx != 0 || ! c?$ssl ) + return; + + c$ssl$sha1_digest = sha1_hash(der_cert); + } + +event bro_done() + { + if ( |deque| == 0 ) + return; + flush(T); + } From 8a569facd623d5d15d5e6d42ce3f549bcf4d94c2 Mon Sep 17 00:00:00 2001 From: Matthias Vallentin Date: Fri, 21 Dec 2012 18:04:19 -0800 Subject: [PATCH 12/43] More style tweaks: replace spaces with tabs. --- scripts/base/protocols/ssl/notary.bro | 68 +++++++++++++-------------- 1 file changed, 34 insertions(+), 34 deletions(-) diff --git a/scripts/base/protocols/ssl/notary.bro b/scripts/base/protocols/ssl/notary.bro index 6cf687ea90..4ae03fd3f8 100644 --- a/scripts/base/protocols/ssl/notary.bro +++ b/scripts/base/protocols/ssl/notary.bro @@ -13,12 +13,12 @@ export { valid: bool &log &optional; }; - ## Hands over an SSL record to the Notary module. This is an ownership - ## transfer, i.e., the caller does not need to call Log::write on this record - ## anymore. - global push: function(info: SSL::Info); + ## Hands over an SSL record to the Notary module. This is an ownership + ## transfer, i.e., the caller does not need to call Log::write on this record + ## anymore. + global push: function(info: SSL::Info); - ## The notary domain to query. + ## The notary domain to query. const domain = "notary.icsi.berkeley.edu" &redef; } @@ -63,26 +63,26 @@ function clear_waitlist(digest: string) } function flush(evict_all: bool) - { - local current: string; - for ( unused_index in deque ) - { - current = deque[tail]; - local info = records[current]; - if ( ! evict_all && ! info?$notary ) - break; - Log::write(SSL::LOG, info); - delete deque[tail]; - delete records[current]; - ++tail; - } - } + { + local current: string; + for ( unused_index in deque ) + { + current = deque[tail]; + local info = records[current]; + if ( ! evict_all && ! info?$notary ) + break; + Log::write(SSL::LOG, info); + delete deque[tail]; + delete records[current]; + ++tail; + } + } function lookup_cert_hash(uid: string, digest: string) - { - # Add the record ID to the list of waiting IDs for this digest. - local waits_already = digest in waiting; - if ( ! waits_already ) + j{ + j# Add the record ID to the list of waiting IDs for this digest. + jlocal waits_already = digest in waiting; + jif ( ! waits_already ) waiting[digest] = vector(); waiting[digest][|waiting[digest]|] = uid; if ( waits_already ) @@ -135,14 +135,14 @@ function push(info: SSL::Info) if ( ! info?$sha1_digest ) return; - local digest = info$sha1_digest; - if ( info$sha1_digest in notary_cache ) - info$notary = notary_cache[digest]; - else - lookup_cert_hash(info$uid, digest); - records[info$uid] = info; - deque[head] = info$uid; - ++head; + local digest = info$sha1_digest; + if ( info$sha1_digest in notary_cache ) + info$notary = notary_cache[digest]; + else + lookup_cert_hash(info$uid, digest); + records[info$uid] = info; + deque[head] = info$uid; + ++head; } event x509_certificate(c: connection, is_orig: bool, cert: X509, @@ -151,12 +151,12 @@ event x509_certificate(c: connection, is_orig: bool, cert: X509, if ( is_orig || chain_idx != 0 || ! c?$ssl ) return; - c$ssl$sha1_digest = sha1_hash(der_cert); - } + c$ssl$sha1_digest = sha1_hash(der_cert); + } event bro_done() { if ( |deque| == 0 ) return; - flush(T); + flush(T); } From 9e81342c92d1c56b4030b3a29e97c8c977155348 Mon Sep 17 00:00:00 2001 From: Matthias Vallentin Date: Sat, 22 Dec 2012 20:30:17 -0800 Subject: [PATCH 13/43] Implement delay-token style SSL logging. This commit moves the notary script into the policy directory, along with some architectural changes: the main SSL script now has functionality to add and remove tokens for a given record. When adding a token, the script delays the logging until the token has been removed or until the record exceeds a maximum delay time. As before, the base SSL script stores all records sequentially and buffers even non-delayed records for the sake of having an ordered log file. If this turns out to be not so important, we can easily revert to a simpler logic. (This is still WiP, some debuggin statements still linger.) --- scripts/base/protocols/ssl/__load__.bro | 62 ------- scripts/base/protocols/ssl/main.bro | 167 +++++++++++++++++- .../{base => policy}/protocols/ssl/notary.bro | 109 +++--------- scripts/site/local.bro | 4 + 4 files changed, 192 insertions(+), 150 deletions(-) rename scripts/{base => policy}/protocols/ssl/notary.bro (53%) diff --git a/scripts/base/protocols/ssl/__load__.bro b/scripts/base/protocols/ssl/__load__.bro index e97d1681fc..239438047c 100644 --- a/scripts/base/protocols/ssl/__load__.bro +++ b/scripts/base/protocols/ssl/__load__.bro @@ -1,65 +1,3 @@ @load ./consts - -module SSL; - -export { - redef enum Log::ID += { LOG }; - - type Info: record { - ## Time when the SSL connection was first detected. - ts: time &log; - ## Unique ID for the connection. - uid: string &log; - ## The connection's 4-tuple of endpoint addresses/ports. - id: conn_id &log; - ## SSL/TLS version that the server offered. - version: string &log &optional; - ## SSL/TLS cipher suite that the server chose. - cipher: string &log &optional; - ## Value of the Server Name Indicator SSL/TLS extension. It - ## indicates the server name that the client was requesting. - server_name: string &log &optional; - ## Session ID offered by the client for session resumption. - session_id: string &log &optional; - ## Subject of the X.509 certificate offered by the server. - subject: string &log &optional; - ## Subject of the signer of the X.509 certificate offered by the server. - issuer_subject: string &log &optional; - ## NotValidBefore field value from the server certificate. - not_valid_before: time &log &optional; - ## NotValidAfter field value from the server certificate. - not_valid_after: time &log &optional; - ## Last alert that was seen during the connection. - last_alert: string &log &optional; - - ## Subject of the X.509 certificate offered by the client. - client_subject: string &log &optional; - ## Subject of the signer of the X.509 certificate offered by the client. - client_issuer_subject: string &log &optional; - - ## Full binary server certificate stored in DER format. - cert: string &optional; - ## Chain of certificates offered by the server to validate its - ## complete signing chain. - cert_chain: vector of string &optional; - - ## Full binary client certificate stored in DER format. - client_cert: string &optional; - ## Chain of certificates offered by the client to validate its - ## complete signing chain. - client_cert_chain: vector of string &optional; - - ## The analyzer ID used for the analyzer instance attached - ## to each connection. It is not used for logging since it's a - ## meaningless arbitrary number. - analyzer_id: count &optional; - }; -} - -redef record connection += { - ssl: Info &optional; -}; - -@load ./notary @load ./main @load ./mozilla-ca-list diff --git a/scripts/base/protocols/ssl/main.bro b/scripts/base/protocols/ssl/main.bro index f02602235c..642d93eb96 100644 --- a/scripts/base/protocols/ssl/main.bro +++ b/scripts/base/protocols/ssl/main.bro @@ -6,6 +6,63 @@ module SSL; export { + redef enum Log::ID += { LOG }; + + type Info: record { + ## Time when the SSL connection was first detected. + ts: time &log; + ## Unique ID for the connection. + uid: string &log; + ## The connection's 4-tuple of endpoint addresses/ports. + id: conn_id &log; + ## SSL/TLS version that the server offered. + version: string &log &optional; + ## SSL/TLS cipher suite that the server chose. + cipher: string &log &optional; + ## Value of the Server Name Indicator SSL/TLS extension. It + ## indicates the server name that the client was requesting. + server_name: string &log &optional; + ## Session ID offered by the client for session resumption. + session_id: string &log &optional; + ## Subject of the X.509 certificate offered by the server. + subject: string &log &optional; + ## Subject of the signer of the X.509 certificate offered by the server. + issuer_subject: string &log &optional; + ## NotValidBefore field value from the server certificate. + not_valid_before: time &log &optional; + ## NotValidAfter field value from the server certificate. + not_valid_after: time &log &optional; + ## Last alert that was seen during the connection. + last_alert: string &log &optional; + + ## Subject of the X.509 certificate offered by the client. + client_subject: string &log &optional; + ## Subject of the signer of the X.509 certificate offered by the client. + client_issuer_subject: string &log &optional; + + ## Full binary server certificate stored in DER format. + cert: string &optional; + ## Chain of certificates offered by the server to validate its + ## complete signing chain. + cert_chain: vector of string &optional; + + ## Full binary client certificate stored in DER format. + client_cert: string &optional; + ## Chain of certificates offered by the client to validate its + ## complete signing chain. + client_cert_chain: vector of string &optional; + + ## The analyzer ID used for the analyzer instance attached + ## to each connection. It is not used for logging since it's a + ## meaningless arbitrary number. + analyzer_id: count &optional; + + ## Adding a string "token" to this set will cause the SSL script + ## to delay logging the record until either the token has been removed or + ## the record has been delayed for :bro:id:`SSL::max_log_delay`. + delay_tokens: set[string] &optional; + }; + ## The default root CA bundle. By loading the ## mozilla-ca-list.bro script it will be set to Mozilla's root CA list. const root_certs: table[string] of string = {} &redef; @@ -20,11 +77,24 @@ export { ## utility. const openssl_util = "openssl" &redef; + ## The maximum amount of time a plugin can delay records from being logged. + const max_log_delay = 15secs &redef; + + ## TODO: document. + global add_delayed_record: function(info: Info, token: string); + + ## TODO: document. + global clear_delayed_record: function(uid: string, token: string) : Info; + ## Event that can be handled to access the SSL ## record as it is sent on to the logging framework. global log_ssl: event(rec: Info); } +redef record connection += { + ssl: Info &optional; +}; + event bro_init() &priority=5 { Log::create_stream(SSL::LOG, [$columns=Info, $ev=log_ssl]); @@ -59,6 +129,20 @@ redef likely_server_ports += { 989/tcp, 990/tcp, 992/tcp, 993/tcp, 995/tcp, 5223/tcp }; +# The buffered SSL log records. +global records: table[string] of Info; + +# A double-ended queue that determines the log record order in which logs have +# to written out to disk. +global deque: table[count] of string; + +# The top-most deque index. +global head = 0; + +# The bottom deque index that points to the next record to be flushed as soon +# as the notary response arrives. +global tail = 0; + function set_session(c: connection) { if ( ! c?$ssl ) @@ -66,16 +150,69 @@ function set_session(c: connection) $client_cert_chain=vector()]; } +function add_delayed_record(info: Info, token: string) + { + if ( info$uid in records ) + { + print fmt("----- ignoring duplicate %s -----", info$uid); + return; + } + + records[info$uid] = info; + deque[head] = info$uid; + ++head; + + info$delay_tokens = set(); + add info$delay_tokens[token]; + } + +function clear_delayed_record(uid: string, token: string) : Info + { + local info = records[uid]; + delete info$delay_tokens[token]; + return info; + } + +global log_record: function(info: Info); + +event delay_logging(info: Info) + { + log_record(info); + } + +function log_record(info: Info) + { + for ( unused_index in records ) + { + if ( head == tail ) + return; + local uid = deque[tail]; + if ( |records[uid]$delay_tokens| > 0 ) + { + if ( info$ts + max_log_delay > network_time() ) + { + schedule 1sec { delay_logging(info) }; + return; + } + else + { + event reporter_info( + network_time(), + fmt("SSL delay tokens not released in time (%s)", + info$delay_tokens), + ""); + } + } + Log::write(SSL::LOG, records[uid]); + delete records[uid]; + delete deque[tail]; + ++tail; + } + } + function finish(c: connection) { -# TODO: This dummy flag merely exists to check whether the notary script is -# loaded. There's probably a better way to incorporate the notary. -@ifdef( Notary::enabled ) - Notary::push(c$ssl); -@else - Log::write(SSL::LOG, c$ssl); -@endif - + log_record(c$ssl); if ( disable_analyzer_after_detection && c?$ssl && c$ssl?$analyzer_id ) disable_analyzer(c$id, c$ssl$analyzer_id); delete c$ssl; @@ -179,3 +316,17 @@ event protocol_violation(c: connection, atype: count, aid: count, if ( c?$ssl ) finish(c); } + +event bro_done() + { + if ( |records| == 0 ) + return; + for ( unused_index in records ) + { + local uid = deque[tail]; + Log::write(SSL::LOG, records[uid]); + delete records[uid]; + delete deque[tail]; + ++tail; + } + } diff --git a/scripts/base/protocols/ssl/notary.bro b/scripts/policy/protocols/ssl/notary.bro similarity index 53% rename from scripts/base/protocols/ssl/notary.bro rename to scripts/policy/protocols/ssl/notary.bro index 4ae03fd3f8..f720359aa6 100644 --- a/scripts/base/protocols/ssl/notary.bro +++ b/scripts/policy/protocols/ssl/notary.bro @@ -1,10 +1,6 @@ module Notary; export { - # Flag to tell the SSL analysis script that it should buffer logs instead of - # flushing them directly. - const enabled = T; - ## A response from the ICSI certificate notary. type Response: record { first_seen: count &log &optional; @@ -13,83 +9,66 @@ export { valid: bool &log &optional; }; - ## Hands over an SSL record to the Notary module. This is an ownership - ## transfer, i.e., the caller does not need to call Log::write on this record - ## anymore. - global push: function(info: SSL::Info); - ## The notary domain to query. const domain = "notary.icsi.berkeley.edu" &redef; } redef record SSL::Info += { - sha1_digest: string &optional; + sha1: string &log &optional; notary: Response &log &optional; }; # The DNS cache of notary responses. global notary_cache: table[string] of Response &create_expire = 1 hr; -# The buffered SSL log records. -global records: table[string] of SSL::Info; - # The records that wait for a notary response identified by the cert digest. # Each digest refers to a list of connection UIDs which are updated when a DNS # reply arrives asynchronously. global waiting: table[string] of vector of string; -# A double-ended queue that determines the log record order in which logs have -# to written out to disk. -global deque: table[count] of string; - -# The top-most deque index. -global head = 0; - -# The bottom deque index that points to the next record to be flushed as soon -# as the notary response arrives. -global tail = 0; - function clear_waitlist(digest: string) { + print "----- clearing waitlist -----"; if ( digest in waiting ) { for ( i in waiting[digest] ) { - local uid = waiting[digest][i]; - records[uid]$notary = []; + print fmt("----- retrieving %s -----", waiting[digest][i]); + local info = SSL::clear_delayed_record(waiting[digest][i], "notary"); + info$notary = []; } delete waiting[digest]; } } -function flush(evict_all: bool) +event x509_certificate(c: connection, is_orig: bool, cert: X509, + chain_idx: count, chain_len: count, der_cert: string) { - local current: string; - for ( unused_index in deque ) - { - current = deque[tail]; - local info = records[current]; - if ( ! evict_all && ! info?$notary ) - break; - Log::write(SSL::LOG, info); - delete deque[tail]; - delete records[current]; - ++tail; - } - } + if ( is_orig || chain_idx != 0 || ! c?$ssl ) + return; -function lookup_cert_hash(uid: string, digest: string) - j{ - j# Add the record ID to the list of waiting IDs for this digest. - jlocal waits_already = digest in waiting; - jif ( ! waits_already ) + local digest = sha1_hash(der_cert); + c$ssl$sha1 = digest; + + if ( digest in notary_cache ) + { + c$ssl$notary = notary_cache[digest]; + return; + } + + print fmt("----- adding %s -----", c$ssl$uid); + SSL::add_delayed_record(c$ssl, "notary"); + + local waits_already = digest in waiting; + if ( ! waits_already ) waiting[digest] = vector(); - waiting[digest][|waiting[digest]|] = uid; + waiting[digest][|waiting[digest]|] = c$uid; if ( waits_already ) return; when ( local str = lookup_hostname_txt(fmt("%s.%s", digest, domain)) ) { + print fmt("----- when for %s: %s -----", digest, str); # Cache every response for a digest. notary_cache[digest] = []; @@ -122,41 +101,11 @@ function lookup_cert_hash(uid: string, digest: string) if ( digest in waiting ) { for ( i in waiting[digest] ) - records[waiting[digest][i]]$notary = r; + { + local info = SSL::clear_delayed_record(waiting[digest][i], "notary"); + info$notary = r; + } delete waiting[digest]; } - - flush(F); } } - -function push(info: SSL::Info) - { - if ( ! info?$sha1_digest ) - return; - - local digest = info$sha1_digest; - if ( info$sha1_digest in notary_cache ) - info$notary = notary_cache[digest]; - else - lookup_cert_hash(info$uid, digest); - records[info$uid] = info; - deque[head] = info$uid; - ++head; - } - -event x509_certificate(c: connection, is_orig: bool, cert: X509, - chain_idx: count, chain_len: count, der_cert: string) - { - if ( is_orig || chain_idx != 0 || ! c?$ssl ) - return; - - c$ssl$sha1_digest = sha1_hash(der_cert); - } - -event bro_done() - { - if ( |deque| == 0 ) - return; - flush(T); - } diff --git a/scripts/site/local.bro b/scripts/site/local.bro index db1a786839..a080300185 100644 --- a/scripts/site/local.bro +++ b/scripts/site/local.bro @@ -56,6 +56,10 @@ redef Software::vulnerable_versions += { # This script enables SSL/TLS certificate validation. @load protocols/ssl/validate-certs +# This script checks each SSL certificate hash against the ICSI certificate +# notary service. +@load protocols/ssl/notary + # If you have libGeoIP support built in, do some geographic detections and # logging for SSH traffic. @load protocols/ssh/geo-data From 7ff15f4599e36c300515955b5f9664cbe5480980 Mon Sep 17 00:00:00 2001 From: Matthias Vallentin Date: Mon, 24 Dec 2012 22:57:15 -0800 Subject: [PATCH 14/43] Simplify delayed logging of SSL records. --- scripts/base/protocols/ssl/main.bro | 54 +++++++++---------------- scripts/policy/protocols/ssl/notary.bro | 46 +++++++++------------ 2 files changed, 39 insertions(+), 61 deletions(-) diff --git a/scripts/base/protocols/ssl/main.bro b/scripts/base/protocols/ssl/main.bro index 642d93eb96..249f8b9609 100644 --- a/scripts/base/protocols/ssl/main.bro +++ b/scripts/base/protocols/ssl/main.bro @@ -80,11 +80,13 @@ export { ## The maximum amount of time a plugin can delay records from being logged. const max_log_delay = 15secs &redef; - ## TODO: document. - global add_delayed_record: function(info: Info, token: string); + ## Delays an SSL record for a specific token: the record will not be logged + ## as longs the token exists or until :bro:id:`SSL::max_log_delay` elapses. + global delay_record: function(info: Info, token: string); - ## TODO: document. - global clear_delayed_record: function(uid: string, token: string) : Info; + ## Undelays an SSL record for a previously inserted token, allowing the + ## record to be logged. + global undelay_record: function(info: Info, token: string); ## Event that can be handled to access the SSL ## record as it is sent on to the logging framework. @@ -129,12 +131,9 @@ redef likely_server_ports += { 989/tcp, 990/tcp, 992/tcp, 993/tcp, 995/tcp, 5223/tcp }; -# The buffered SSL log records. -global records: table[string] of Info; - # A double-ended queue that determines the log record order in which logs have # to written out to disk. -global deque: table[count] of string; +global deque: table[count] of Info; # The top-most deque index. global head = 0; @@ -150,27 +149,19 @@ function set_session(c: connection) $client_cert_chain=vector()]; } -function add_delayed_record(info: Info, token: string) +function delay_record(info: Info, token: string) { - if ( info$uid in records ) - { - print fmt("----- ignoring duplicate %s -----", info$uid); - return; - } - - records[info$uid] = info; - deque[head] = info$uid; - ++head; - info$delay_tokens = set(); add info$delay_tokens[token]; + + deque[head] = info; + ++head; } -function clear_delayed_record(uid: string, token: string) : Info +function undelay_record(info: Info, token: string) { - local info = records[uid]; - delete info$delay_tokens[token]; - return info; + if ( token in info$delay_tokens ) + delete info$delay_tokens[token]; } global log_record: function(info: Info); @@ -182,12 +173,11 @@ event delay_logging(info: Info) function log_record(info: Info) { - for ( unused_index in records ) + for ( unused_index in deque ) { if ( head == tail ) return; - local uid = deque[tail]; - if ( |records[uid]$delay_tokens| > 0 ) + if ( |deque[tail]$delay_tokens| > 0 ) { if ( info$ts + max_log_delay > network_time() ) { @@ -203,8 +193,7 @@ function log_record(info: Info) ""); } } - Log::write(SSL::LOG, records[uid]); - delete records[uid]; + Log::write(SSL::LOG, deque[tail]); delete deque[tail]; ++tail; } @@ -215,7 +204,6 @@ function finish(c: connection) log_record(c$ssl); if ( disable_analyzer_after_detection && c?$ssl && c$ssl?$analyzer_id ) disable_analyzer(c$id, c$ssl$analyzer_id); - delete c$ssl; } event ssl_client_hello(c: connection, version: count, possible_ts: time, session_id: string, ciphers: count_set) &priority=5 @@ -319,13 +307,11 @@ event protocol_violation(c: connection, atype: count, aid: count, event bro_done() { - if ( |records| == 0 ) + if ( |deque| == 0 ) return; - for ( unused_index in records ) + for ( unused_index in deque ) { - local uid = deque[tail]; - Log::write(SSL::LOG, records[uid]); - delete records[uid]; + Log::write(SSL::LOG, deque[tail]); delete deque[tail]; ++tail; } diff --git a/scripts/policy/protocols/ssl/notary.bro b/scripts/policy/protocols/ssl/notary.bro index f720359aa6..40afc977c2 100644 --- a/scripts/policy/protocols/ssl/notary.bro +++ b/scripts/policy/protocols/ssl/notary.bro @@ -24,20 +24,15 @@ global notary_cache: table[string] of Response &create_expire = 1 hr; # The records that wait for a notary response identified by the cert digest. # Each digest refers to a list of connection UIDs which are updated when a DNS # reply arrives asynchronously. -global waiting: table[string] of vector of string; +global wait_list: table[string] of vector of SSL::Info; function clear_waitlist(digest: string) { - print "----- clearing waitlist -----"; - if ( digest in waiting ) + if ( digest in wait_list ) { - for ( i in waiting[digest] ) - { - print fmt("----- retrieving %s -----", waiting[digest][i]); - local info = SSL::clear_delayed_record(waiting[digest][i], "notary"); - info$notary = []; - } - delete waiting[digest]; + for ( i in wait_list[digest] ) + SSL::undelay_record(wait_list[digest][i], "notary"); + delete wait_list[digest]; } } @@ -56,31 +51,27 @@ event x509_certificate(c: connection, is_orig: bool, cert: X509, return; } - print fmt("----- adding %s -----", c$ssl$uid); - SSL::add_delayed_record(c$ssl, "notary"); + SSL::delay_record(c$ssl, "notary"); - local waits_already = digest in waiting; + local waits_already = digest in wait_list; if ( ! waits_already ) - waiting[digest] = vector(); - waiting[digest][|waiting[digest]|] = c$uid; + wait_list[digest] = vector(); + wait_list[digest][|wait_list[digest]|] = c$ssl; if ( waits_already ) return; when ( local str = lookup_hostname_txt(fmt("%s.%s", digest, domain)) ) { - print fmt("----- when for %s: %s -----", digest, str); - # Cache every response for a digest. notary_cache[digest] = []; # Parse notary answer. - if ( str == "" ) + if ( str == "" ) # NXDOMAIN { - # TODO: Should we handle NXDOMAIN separately? clear_waitlist(digest); return; } local fields = split(str, / /); - if ( |fields| != 5 ) # version 1 has 5 fields. + if ( |fields| != 5 ) # version 1 has 5 fields. { clear_waitlist(digest); return; @@ -97,15 +88,16 @@ event x509_certificate(c: connection, is_orig: bool, cert: X509, r$times_seen = to_count(split(fields[4], /=/)[2]); r$valid = split(fields[5], /=/)[2] == "1"; - # Assign notary answer to all waiting records. - if ( digest in waiting ) + # Assign notary answer to all records waiting for this digest. + if ( digest in wait_list ) { - for ( i in waiting[digest] ) - { - local info = SSL::clear_delayed_record(waiting[digest][i], "notary"); + for ( i in wait_list[digest] ) + { + local info = wait_list[digest][i]; + SSL::undelay_record(info, "notary"); info$notary = r; - } - delete waiting[digest]; + } + delete wait_list[digest]; } } } From 32a0ead698c467a9a89f5b58687db2f7e2812d3c Mon Sep 17 00:00:00 2001 From: Matthias Vallentin Date: Mon, 24 Dec 2012 23:06:56 -0800 Subject: [PATCH 15/43] Give log buffer the correct name. --- scripts/base/protocols/ssl/main.bro | 28 +++++++++++++--------------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/scripts/base/protocols/ssl/main.bro b/scripts/base/protocols/ssl/main.bro index 249f8b9609..e798695948 100644 --- a/scripts/base/protocols/ssl/main.bro +++ b/scripts/base/protocols/ssl/main.bro @@ -131,15 +131,13 @@ redef likely_server_ports += { 989/tcp, 990/tcp, 992/tcp, 993/tcp, 995/tcp, 5223/tcp }; -# A double-ended queue that determines the log record order in which logs have -# to written out to disk. -global deque: table[count] of Info; +# A queue that buffers log records. +global queue: table[count] of Info; -# The top-most deque index. +# The top queue index where records are added. global head = 0; -# The bottom deque index that points to the next record to be flushed as soon -# as the notary response arrives. +# The bottom queue index that points to the next record to be flushed. global tail = 0; function set_session(c: connection) @@ -154,7 +152,7 @@ function delay_record(info: Info, token: string) info$delay_tokens = set(); add info$delay_tokens[token]; - deque[head] = info; + queue[head] = info; ++head; } @@ -173,11 +171,11 @@ event delay_logging(info: Info) function log_record(info: Info) { - for ( unused_index in deque ) + for ( unused_index in queue ) { if ( head == tail ) return; - if ( |deque[tail]$delay_tokens| > 0 ) + if ( |queue[tail]$delay_tokens| > 0 ) { if ( info$ts + max_log_delay > network_time() ) { @@ -193,8 +191,8 @@ function log_record(info: Info) ""); } } - Log::write(SSL::LOG, deque[tail]); - delete deque[tail]; + Log::write(SSL::LOG, queue[tail]); + delete queue[tail]; ++tail; } } @@ -307,12 +305,12 @@ event protocol_violation(c: connection, atype: count, aid: count, event bro_done() { - if ( |deque| == 0 ) + if ( |queue| == 0 ) return; - for ( unused_index in deque ) + for ( unused_index in queue ) { - Log::write(SSL::LOG, deque[tail]); - delete deque[tail]; + Log::write(SSL::LOG, queue[tail]); + delete queue[tail]; ++tail; } } From 8000c40fee329e25d146b68e15fbb6ef82c599eb Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Mon, 7 Jan 2013 11:41:36 -0600 Subject: [PATCH 16/43] Fix memory leak in OpaqueType::DoUnserialize. --- src/Type.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Type.cc b/src/Type.cc index 412c25459d..1fb813efa1 100644 --- a/src/Type.cc +++ b/src/Type.cc @@ -1305,6 +1305,7 @@ bool OpaqueType::DoUnserialize(UnserialInfo* info) return false; name = n; + delete [] n; return true; } From e638f043010e14d4599fea53a75d6dbad66f3749 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Mon, 7 Jan 2013 09:47:09 -0800 Subject: [PATCH 17/43] Updating submodule(s). [nomail] --- aux/btest | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aux/btest b/aux/btest index d9c747f268..3585dc9a7a 160000 --- a/aux/btest +++ b/aux/btest @@ -1 +1 @@ -Subproject commit d9c747f268ca50275f3bc2e1581464b599d5a3ea +Subproject commit 3585dc9a7afe20d70cb77fc2472cc6bce3850b67 From 8b46bbb1c0b3aefea7fa683b53165e497a14056d Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Mon, 7 Jan 2013 13:29:05 -0600 Subject: [PATCH 18/43] Change substring index notation to use a colon (addresses #422). String slice notation is written as `s[1:2]` instead of `s[1, 2]` because the later is ambiguous with composite index types. --- src/Expr.cc | 16 +++++++++++++- src/Expr.h | 2 +- src/parse.y | 8 +++++++ testing/btest/core/leaks/string-indexing.bro | 22 ++++++++++---------- testing/btest/language/string-indexing.bro | 22 ++++++++++---------- 5 files changed, 46 insertions(+), 24 deletions(-) diff --git a/src/Expr.cc b/src/Expr.cc index eef0e7b69e..b3127d6a7f 100644 --- a/src/Expr.cc +++ b/src/Expr.cc @@ -2801,12 +2801,26 @@ bool AssignExpr::DoUnserialize(UnserialInfo* info) return UNSERIALIZE(&is_init); } -IndexExpr::IndexExpr(Expr* arg_op1, ListExpr* arg_op2) +IndexExpr::IndexExpr(Expr* arg_op1, ListExpr* arg_op2, bool is_string_slice) : BinaryExpr(EXPR_INDEX, arg_op1, arg_op2) { if ( IsError() ) return; + if ( is_string_slice ) + { + if ( ! IsString(op1->Type()->Tag()) ) + ExprError("slice notation indexing can apply only to strings"); + } + else if ( IsString(op1->Type()->Tag()) ) + { + if ( arg_op2->Exprs().length() != 1 ) + ExprError("invalid string index expression"); + } + + if ( IsError() ) + return; + int match_type = op1->Type()->MatchesIndex(arg_op2); if ( match_type == DOES_NOT_MATCH_INDEX ) SetError("not an index type"); diff --git a/src/Expr.h b/src/Expr.h index ea17c735b5..ac3ab0f4f2 100644 --- a/src/Expr.h +++ b/src/Expr.h @@ -646,7 +646,7 @@ protected: class IndexExpr : public BinaryExpr { public: - IndexExpr(Expr* op1, ListExpr* op2); + IndexExpr(Expr* op1, ListExpr* op2, bool is_string_slice = false); int CanAdd() const; int CanDel() const; diff --git a/src/parse.y b/src/parse.y index a16b742728..090786647e 100644 --- a/src/parse.y +++ b/src/parse.y @@ -418,6 +418,14 @@ expr: $$ = new IndexExpr($1, $3); } + | expr '[' expr ':' expr ']' + { + set_location(@1, @6); + ListExpr* le = new ListExpr($3); + le->Append($5); + $$ = new IndexExpr($1, le, true); + } + | expr '$' TOK_ID { set_location(@1, @3); diff --git a/testing/btest/core/leaks/string-indexing.bro b/testing/btest/core/leaks/string-indexing.bro index 40af2317e6..f9ea000ef9 100644 --- a/testing/btest/core/leaks/string-indexing.bro +++ b/testing/btest/core/leaks/string-indexing.bro @@ -11,16 +11,16 @@ event new_connection(c: connection) { local s = "0123456789"; print s[1]; - print s[1,2]; - print s[1,6]; - print s[0,20]; + print s[1:2]; + print s[1:6]; + print s[0:20]; print s[-2]; - print s[-3,-1]; - print s[-1,-10]; - print s[-1,0]; - print s[-1,5]; - print s[20, 23]; - print s[-20, 23]; - print s[0,5][2]; - print s[0,5][1,3][0]; + print s[-3:1]; + print s[-1:10]; + print s[-1:0]; + print s[-1:5]; + print s[20:23]; + print s[-20:23]; + print s[0:5][2]; + print s[0:5][1:3][0]; } diff --git a/testing/btest/language/string-indexing.bro b/testing/btest/language/string-indexing.bro index d05c1b6c5e..f991b3c5fa 100644 --- a/testing/btest/language/string-indexing.bro +++ b/testing/btest/language/string-indexing.bro @@ -3,15 +3,15 @@ local s = "0123456789"; print s[1]; -print s[1,2]; -print s[1,6]; -print s[0,20]; +print s[1:2]; +print s[1:6]; +print s[0:20]; print s[-2]; -print s[-3,-1]; -print s[-1,-10]; -print s[-1,0]; -print s[-1,5]; -print s[20, 23]; -print s[-20, 23]; -print s[0,5][2]; -print s[0,5][1,3][0]; +print s[-3:-1]; +print s[-1:-10]; +print s[-1:0]; +print s[-1:5]; +print s[20:23]; +print s[-20:23]; +print s[0:5][2]; +print s[0:5][1:3][0]; From 8695053e278158ce40cb2a3d6aa04ff60653ca0e Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Fri, 11 Jan 2013 11:43:15 -0600 Subject: [PATCH 19/43] Disable automatic case fallthrough in switch stmts. Addresses #754. Case bodies now don't require a "break" statement to prevent fallthrough to case bodies below. Empty case bodies generate an error message at parse-time to help indicate the absence of automatic fallthrough; to associate multiple values with a case, use "case 1, 2:" instead of "case 1: case 2:". --- src/Stmt.cc | 22 +++++++------------ src/Stmt.h | 6 +++++ testing/btest/core/leaks/switch-statement.bro | 17 ++++++-------- testing/btest/language/switch-statement.bro | 17 ++++++-------- 4 files changed, 28 insertions(+), 34 deletions(-) diff --git a/src/Stmt.cc b/src/Stmt.cc index 2cd7117ddb..3e37256338 100644 --- a/src/Stmt.cc +++ b/src/Stmt.cc @@ -701,6 +701,9 @@ SwitchStmt::SwitchStmt(Expr* index, case_list* arg_cases) : const Case* c = (*cases)[i]; const ListExpr* le = c->Cases(); + if ( ! c->Body() || c->Body()->AsStmtList()->Stmts().length() == 0 ) + c->Error("empty case label body does nothing"); + if ( le ) { if ( ! le->Type()->AsTypeList()->AllMatch(e->Type(), false) ) @@ -795,22 +798,13 @@ Val* SwitchStmt::DoExec(Frame* f, Val* v, stmt_flow_type& flow) const if ( matching_label_idx == -1 ) return 0; - for ( int i = matching_label_idx; i < cases->length(); ++i ) - { - const Case* c = (*cases)[i]; + const Case* c = (*cases)[matching_label_idx]; + flow = FLOW_NEXT; + rval = c->Body()->Exec(f, flow); + + if ( flow == FLOW_BREAK ) flow = FLOW_NEXT; - rval = c->Body()->Exec(f, flow); - - if ( flow == FLOW_BREAK ) - { - flow = FLOW_NEXT; - break; - } - - if ( flow == FLOW_RETURN ) - break; - } return rval; } diff --git a/src/Stmt.h b/src/Stmt.h index 497d7c97b1..32be7f33fc 100644 --- a/src/Stmt.h +++ b/src/Stmt.h @@ -46,6 +46,12 @@ public: return (StmtList*) this; } + const StmtList* AsStmtList() const + { + CHECK_TAG(tag, STMT_LIST, "Stmt::AsStmtList", stmt_name) + return (const StmtList*) this; + } + ForStmt* AsForStmt() { CHECK_TAG(tag, STMT_FOR, "Stmt::AsForStmt", stmt_name) diff --git a/testing/btest/core/leaks/switch-statement.bro b/testing/btest/core/leaks/switch-statement.bro index 24829006b5..6fbdb0d54a 100644 --- a/testing/btest/core/leaks/switch-statement.bro +++ b/testing/btest/core/leaks/switch-statement.bro @@ -157,6 +157,7 @@ function switch_break(v: count): string case 2: rval += "testing"; break; + rval += "ERROR"; case 3: rval += "tested"; } @@ -251,17 +252,17 @@ event new_connection(c: connection) test_switch( switch_subnet([fe80::1]/96) , "[fe80::0]" ); test_switch( switch_subnet(192.168.1.100/16) , "192.168.0.0/16" ); test_switch( switch_empty(2) , "n/a" ); - test_switch( switch_break(1) , "testtestingreturn" ); + test_switch( switch_break(1) , "testreturn" ); test_switch( switch_break(2) , "testingreturn" ); test_switch( switch_break(3) , "testedreturn" ); - test_switch( switch_default(1) , "123dr" ); - test_switch( switch_default(2) , "23dr" ); - test_switch( switch_default(3) , "3dr" ); + test_switch( switch_default(1) , "1r" ); + test_switch( switch_default(2) , "2r" ); + test_switch( switch_default(3) , "3r" ); test_switch( switch_default(4) , "dr" ); - test_switch( switch_default_placement(1) , "1d2r" ); + test_switch( switch_default_placement(1) , "1r" ); test_switch( switch_default_placement(2) , "2r" ); test_switch( switch_default_placement(3) , "3r" ); - test_switch( switch_default_placement(4) , "d2r" ); + test_switch( switch_default_placement(4) , "dr" ); local v = vector(0,1,2,3,4,5,6,7,9,10); local expect: string; @@ -271,16 +272,12 @@ event new_connection(c: connection) switch ( v[i] ) { case 1, 2: expect = "1,2"; - break; case 3, 4, 5: expect = "3,4,5"; - break; case 6, 7, 8, 9: expect = "6,7,8,9"; - break; default: expect = "n/a"; - break; } test_switch( switch_case_list(v[i]) , expect ); } diff --git a/testing/btest/language/switch-statement.bro b/testing/btest/language/switch-statement.bro index b8c34f77dc..dcf2a4c041 100644 --- a/testing/btest/language/switch-statement.bro +++ b/testing/btest/language/switch-statement.bro @@ -152,6 +152,7 @@ function switch_break(v: count): string case 2: rval += "testing"; break; + rval += "ERROR"; case 3: rval += "tested"; } @@ -246,17 +247,17 @@ event bro_init() test_switch( switch_subnet([fe80::1]/96) , "[fe80::0]" ); test_switch( switch_subnet(192.168.1.100/16) , "192.168.0.0/16" ); test_switch( switch_empty(2) , "n/a" ); - test_switch( switch_break(1) , "testtestingreturn" ); + test_switch( switch_break(1) , "testreturn" ); test_switch( switch_break(2) , "testingreturn" ); test_switch( switch_break(3) , "testedreturn" ); - test_switch( switch_default(1) , "123dr" ); - test_switch( switch_default(2) , "23dr" ); - test_switch( switch_default(3) , "3dr" ); + test_switch( switch_default(1) , "1r" ); + test_switch( switch_default(2) , "2r" ); + test_switch( switch_default(3) , "3r" ); test_switch( switch_default(4) , "dr" ); - test_switch( switch_default_placement(1) , "1d2r" ); + test_switch( switch_default_placement(1) , "1r" ); test_switch( switch_default_placement(2) , "2r" ); test_switch( switch_default_placement(3) , "3r" ); - test_switch( switch_default_placement(4) , "d2r" ); + test_switch( switch_default_placement(4) , "dr" ); local v = vector(0,1,2,3,4,5,6,7,9,10); local expect: string; @@ -266,16 +267,12 @@ event bro_init() switch ( v[i] ) { case 1, 2: expect = "1,2"; - break; case 3, 4, 5: expect = "3,4,5"; - break; case 6, 7, 8, 9: expect = "6,7,8,9"; - break; default: expect = "n/a"; - break; } test_switch( switch_case_list(v[i]) , expect ); } From 253b8201e810c822ea9a12ec4bf8ec909cd5f281 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Fri, 11 Jan 2013 10:01:47 -0800 Subject: [PATCH 20/43] port memory leak fix from master --- src/threading/AsciiInputOutput.cc | 43 +++++++++++++++++++++---------- 1 file changed, 30 insertions(+), 13 deletions(-) diff --git a/src/threading/AsciiInputOutput.cc b/src/threading/AsciiInputOutput.cc index 560a6dfc38..a9d62bd8ab 100644 --- a/src/threading/AsciiInputOutput.cc +++ b/src/threading/AsciiInputOutput.cc @@ -205,14 +205,14 @@ threading::Value* AsciiInputOutput::StringToVal(string s, string name, TypeTag t { thread->Error(thread->Fmt("Field: %s Invalid value for boolean: %s", name.c_str(), s.c_str())); - return 0; + goto parse_error; } break; case TYPE_INT: val->val.int_val = strtoll(s.c_str(), &end, 10); if ( CheckNumberError(s, end) ) - return 0; + goto parse_error; break; case TYPE_DOUBLE: @@ -220,20 +220,20 @@ threading::Value* AsciiInputOutput::StringToVal(string s, string name, TypeTag t case TYPE_INTERVAL: val->val.double_val = strtod(s.c_str(), &end); if ( CheckNumberError(s, end) ) - return 0; + goto parse_error; break; case TYPE_COUNT: case TYPE_COUNTER: val->val.uint_val = strtoull(s.c_str(), &end, 10); if ( CheckNumberError(s, end) ) - return 0; + goto parse_error; break; case TYPE_PORT: val->val.port_val.port = strtoull(s.c_str(), &end, 10); if ( CheckNumberError(s, end) ) - return 0; + goto parse_error; val->val.port_val.proto = TRANSPORT_UNKNOWN; break; @@ -245,13 +245,13 @@ threading::Value* AsciiInputOutput::StringToVal(string s, string name, TypeTag t if ( pos == s.npos ) { thread->Error(thread->Fmt("Invalid value for subnet: %s", s.c_str())); - return 0; + goto parse_error; } uint8_t width = (uint8_t) strtol(s.substr(pos+1).c_str(), &end, 10); if ( CheckNumberError(s, end) ) - return 0; + goto parse_error; string addr = s.substr(0, pos); @@ -281,6 +281,7 @@ threading::Value* AsciiInputOutput::StringToVal(string s, string name, TypeTag t } unsigned int pos = 0; + bool error = false; if ( empty_field.size() > 0 && s.compare(empty_field) == 0 ) length = 0; @@ -317,14 +318,16 @@ threading::Value* AsciiInputOutput::StringToVal(string s, string name, TypeTag t { thread->Error(thread->Fmt("Internal error while parsing set. pos %d >= length %d." " Element: %s", pos, length, element.c_str())); + error = true; break; } threading::Value* newval = StringToVal(element, name, subtype); if ( newval == 0 ) { - thread->Error("Error while reading set"); - return 0; + thread->Error("Error while reading set or vector"); + error = true; + break; } lvals[pos] = newval; @@ -335,22 +338,32 @@ threading::Value* AsciiInputOutput::StringToVal(string s, string name, TypeTag t // Test if the string ends with a set_separator... or if the // complete string is empty. In either of these cases we have // to push an empty val on top of it. - if ( s.empty() || *s.rbegin() == set_separator[0] ) + if ( ! error && (s.empty() || *s.rbegin() == set_separator[0]) ) { lvals[pos] = StringToVal("", name, subtype); if ( lvals[pos] == 0 ) { thread->Error("Error while trying to add empty set element"); - return 0; + goto parse_error; } pos++; } + if ( error ) { + // We had an error while reading a set or a vector. + // Hence we have to clean up the values that have + // been read so far + for ( unsigned int i = 0; i < pos; i++ ) + delete lvals[i]; + + goto parse_error; + } + if ( pos != length ) { thread->Error(thread->Fmt("Internal error while parsing set: did not find all elements: %s", s.c_str())); - return 0; + goto parse_error; } break; @@ -359,10 +372,14 @@ threading::Value* AsciiInputOutput::StringToVal(string s, string name, TypeTag t default: thread->Error(thread->Fmt("unsupported field format %d for %s", type, name.c_str())); - return 0; + goto parse_error; } return val; + +parse_error: + delete val; + return 0; } bool AsciiInputOutput::CheckNumberError(const string& s, const char * end) const From e2e22a707ba744ad7059942f3c79201cc9b2f757 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Sat, 12 Jan 2013 14:39:00 -0800 Subject: [PATCH 21/43] add the last of Robins suggestions (separate info-struct for constructors). This took a while. --- src/input/readers/Ascii.cc | 2 +- src/input/readers/Benchmark.cc | 2 +- src/logging/writers/Ascii.cc | 2 +- src/threading/AsciiInputOutput.cc | 45 ++++++++++++------------ src/threading/AsciiInputOutput.h | 58 +++++++++++++++++++++---------- 5 files changed, 65 insertions(+), 44 deletions(-) diff --git a/src/input/readers/Ascii.cc b/src/input/readers/Ascii.cc index 2af3a31a88..ab2c2721a0 100644 --- a/src/input/readers/Ascii.cc +++ b/src/input/readers/Ascii.cc @@ -68,7 +68,7 @@ Ascii::Ascii(ReaderFrontend *frontend) : ReaderBackend(frontend) unset_field.assign( (const char*) BifConst::InputAscii::unset_field->Bytes(), BifConst::InputAscii::unset_field->Len()); - io = new AsciiInputOutput(this, set_separator, unset_field, empty_field); + io = new AsciiInputOutput(this, AsciiInputOutput::SeparatorInfo(set_separator, unset_field, empty_field)); } Ascii::~Ascii() diff --git a/src/input/readers/Benchmark.cc b/src/input/readers/Benchmark.cc index 4738c6e867..9851f61b70 100644 --- a/src/input/readers/Benchmark.cc +++ b/src/input/readers/Benchmark.cc @@ -26,7 +26,7 @@ Benchmark::Benchmark(ReaderFrontend *frontend) : ReaderBackend(frontend) timedspread = double(BifConst::InputBenchmark::timedspread); heartbeat_interval = double(BifConst::Threading::heartbeat_interval); - io = new AsciiInputOutput(this); + io = new AsciiInputOutput(this, AsciiInputOutput::SeparatorInfo()); } Benchmark::~Benchmark() diff --git a/src/logging/writers/Ascii.cc b/src/logging/writers/Ascii.cc index 0ad7098852..8ed8797f09 100644 --- a/src/logging/writers/Ascii.cc +++ b/src/logging/writers/Ascii.cc @@ -52,7 +52,7 @@ Ascii::Ascii(WriterFrontend* frontend) : WriterBackend(frontend) desc.EnableEscaping(); desc.AddEscapeSequence(separator); - io = new AsciiInputOutput(this, set_separator, unset_field, empty_field); + io = new AsciiInputOutput(this, AsciiInputOutput::SeparatorInfo(set_separator, unset_field, empty_field)); } Ascii::~Ascii() diff --git a/src/threading/AsciiInputOutput.cc b/src/threading/AsciiInputOutput.cc index a9d62bd8ab..e64ca06b8f 100644 --- a/src/threading/AsciiInputOutput.cc +++ b/src/threading/AsciiInputOutput.cc @@ -7,24 +7,23 @@ #include "AsciiInputOutput.h" #include "../bro_inet_ntop.h" -AsciiInputOutput::AsciiInputOutput(threading::MsgThread* t) +AsciiInputOutput::AsciiInputOutput(threading::MsgThread* t, const SeparatorInfo info) { thread = t; + this->separators = info; } -AsciiInputOutput::AsciiInputOutput(threading::MsgThread* t, const string & set_separator, +AsciiInputOutput::SeparatorInfo::SeparatorInfo(const string & set_separator, const string & unset_field, const string & empty_field) { - thread = t; this->set_separator = set_separator; this->unset_field = unset_field; this->empty_field = empty_field; } -AsciiInputOutput::AsciiInputOutput(threading::MsgThread* t, const string & set_separator, +AsciiInputOutput::SeparatorInfo::SeparatorInfo(const string & set_separator, const string & unset_field) { - thread = t; this->set_separator = set_separator; this->unset_field = unset_field; } @@ -38,7 +37,7 @@ bool AsciiInputOutput::ValToODesc(ODesc* desc, threading::Value* val, const thre { if ( ! val->present ) { - desc->Add(unset_field); + desc->Add(separators.unset_field); return true; } @@ -94,11 +93,11 @@ bool AsciiInputOutput::ValToODesc(ODesc* desc, threading::Value* val, const thre if ( ! size ) { - desc->Add(empty_field); + desc->Add(separators.empty_field); break; } - if ( size == unset_field.size() && memcmp(data, unset_field.data(), size) == 0 ) + if ( size == separators.unset_field.size() && memcmp(data, separators.unset_field.data(), size) == 0 ) { // The value we'd write out would match exactly the // place-holder we use for unset optional fields. We @@ -124,23 +123,23 @@ bool AsciiInputOutput::ValToODesc(ODesc* desc, threading::Value* val, const thre { if ( ! val->val.set_val.size ) { - desc->Add(empty_field); + desc->Add(separators.empty_field); break; } - desc->AddEscapeSequence(set_separator); + desc->AddEscapeSequence(separators.set_separator); for ( int j = 0; j < val->val.set_val.size; j++ ) { if ( j > 0 ) - desc->AddRaw(set_separator); + desc->AddRaw(separators.set_separator); if ( ! ValToODesc(desc, val->val.set_val.vals[j], field) ) { - desc->RemoveEscapeSequence(set_separator); + desc->RemoveEscapeSequence(separators.set_separator); return false; } } - desc->RemoveEscapeSequence(set_separator); + desc->RemoveEscapeSequence(separators.set_separator); break; } @@ -149,23 +148,23 @@ bool AsciiInputOutput::ValToODesc(ODesc* desc, threading::Value* val, const thre { if ( ! val->val.vector_val.size ) { - desc->Add(empty_field); + desc->Add(separators.empty_field); break; } - desc->AddEscapeSequence(set_separator); + desc->AddEscapeSequence(separators.set_separator); for ( int j = 0; j < val->val.vector_val.size; j++ ) { if ( j > 0 ) - desc->AddRaw(set_separator); + desc->AddRaw(separators.set_separator); if ( ! ValToODesc(desc, val->val.vector_val.vals[j], field) ) { - desc->RemoveEscapeSequence(set_separator); + desc->RemoveEscapeSequence(separators.set_separator); return false; } } - desc->RemoveEscapeSequence(set_separator); + desc->RemoveEscapeSequence(separators.set_separator); break; } @@ -181,7 +180,7 @@ bool AsciiInputOutput::ValToODesc(ODesc* desc, threading::Value* val, const thre threading::Value* AsciiInputOutput::StringToVal(string s, string name, TypeTag type, TypeTag subtype) const { - if ( s.compare(unset_field) == 0 ) // field is not set... + if ( s.compare(separators.unset_field) == 0 ) // field is not set... return new threading::Value(type, false); threading::Value* val = new threading::Value(type, true); @@ -276,14 +275,14 @@ threading::Value* AsciiInputOutput::StringToVal(string s, string name, TypeTag t unsigned int length = 1; for ( unsigned int i = 0; i < s.size(); i++ ) { - if ( s[i] == set_separator[0] ) + if ( s[i] == separators.set_separator[0] ) length++; } unsigned int pos = 0; bool error = false; - if ( empty_field.size() > 0 && s.compare(empty_field) == 0 ) + if ( separators.empty_field.size() > 0 && s.compare(separators.empty_field) == 0 ) length = 0; threading::Value** lvals = new threading::Value* [length]; @@ -311,7 +310,7 @@ threading::Value* AsciiInputOutput::StringToVal(string s, string name, TypeTag t { string element; - if ( ! getline(splitstream, element, set_separator[0]) ) + if ( ! getline(splitstream, element, separators.set_separator[0]) ) break; if ( pos >= length ) @@ -338,7 +337,7 @@ threading::Value* AsciiInputOutput::StringToVal(string s, string name, TypeTag t // Test if the string ends with a set_separator... or if the // complete string is empty. In either of these cases we have // to push an empty val on top of it. - if ( ! error && (s.empty() || *s.rbegin() == set_separator[0]) ) + if ( ! error && (s.empty() || *s.rbegin() == separators.set_separator[0]) ) { lvals[pos] = StringToVal("", name, subtype); if ( lvals[pos] == 0 ) diff --git a/src/threading/AsciiInputOutput.h b/src/threading/AsciiInputOutput.h index 9f36175a5a..a8c4493914 100644 --- a/src/threading/AsciiInputOutput.h +++ b/src/threading/AsciiInputOutput.h @@ -8,22 +8,48 @@ class AsciiInputOutput { public: - // Constructor that leaves separators, etc empty. - // Use if you just need functionality like StringToAddr, etc. - AsciiInputOutput(threading::MsgThread*); - // Constructor that defines all separators, etc. - // Use if you need either ValToODesc or EntryToVal. - AsciiInputOutput(threading::MsgThread*, const string & set_separator, + /** + * A struct to pass the necessary initialization values to the AsciiInputOutput module + * on startup + */ + struct SeparatorInfo + { + //const string separator; + string set_separator; + string empty_field; + string unset_field; + string meta_prefix; + + // Constructor that leaves separators, etc empty. + // Use if you just need functionality like StringToAddr, etc. + SeparatorInfo() { }; + + // Constructor that defines all separators, etc. + // Use if you need either ValToODesc or EntryToVal. + SeparatorInfo(const string & set_separator, const string & unset_field, const string & empty_field); - - // Constructor that defines all separators, etc, besides empty_field, which is not needed for many - // non-ascii-based io sources. - // Use if you need either ValToODesc or EntryToVal. - AsciiInputOutput(threading::MsgThread*, const string & set_separator, - const string & unset_field); - ~AsciiInputOutput(); + // Constructor that defines all separators, etc, besides empty_field, which is not needed for many + // non-ascii-based io sources. + // Use if you need either ValToODesc or EntryToVal. + SeparatorInfo(const string & set_separator, + const string & unset_field); + }; + + /** + * Constructor + * + * @param t The thread that uses this class instance. Used to access thread + * message passing methods + * + * @param info + * SeparatorInfo structure defining the necessary separators + */ + AsciiInputOutput(threading::MsgThread* t, const SeparatorInfo info); + + // Destructor + ~AsciiInputOutput(); // converts a threading value to the corresponding ascii representation // returns false & logs an error with reporter in case an error occurs @@ -75,11 +101,7 @@ class AsciiInputOutput { private: bool CheckNumberError(const string& s, const char * end) const; - string separator; - string set_separator; - string empty_field; - string unset_field; - string meta_prefix; + SeparatorInfo separators; threading::MsgThread* thread; }; From a635f96518ddeefe553ef4fd98213bc07ad63226 Mon Sep 17 00:00:00 2001 From: Matthias Vallentin Date: Tue, 15 Jan 2013 14:56:02 -0800 Subject: [PATCH 22/43] Small cosmetic changes. --- scripts/policy/protocols/ssl/notary.bro | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/scripts/policy/protocols/ssl/notary.bro b/scripts/policy/protocols/ssl/notary.bro index 40afc977c2..0dc96e3570 100644 --- a/scripts/policy/protocols/ssl/notary.bro +++ b/scripts/policy/protocols/ssl/notary.bro @@ -24,15 +24,15 @@ global notary_cache: table[string] of Response &create_expire = 1 hr; # The records that wait for a notary response identified by the cert digest. # Each digest refers to a list of connection UIDs which are updated when a DNS # reply arrives asynchronously. -global wait_list: table[string] of vector of SSL::Info; +global waitlist: table[string] of vector of SSL::Info; function clear_waitlist(digest: string) { - if ( digest in wait_list ) + if ( digest in waitlist ) { - for ( i in wait_list[digest] ) - SSL::undelay_record(wait_list[digest][i], "notary"); - delete wait_list[digest]; + for ( i in waitlist[digest] ) + SSL::undelay_record(waitlist[digest][i], "notary"); + delete waitlist[digest]; } } @@ -53,10 +53,10 @@ event x509_certificate(c: connection, is_orig: bool, cert: X509, SSL::delay_record(c$ssl, "notary"); - local waits_already = digest in wait_list; + local waits_already = digest in waitlist; if ( ! waits_already ) - wait_list[digest] = vector(); - wait_list[digest][|wait_list[digest]|] = c$ssl; + waitlist[digest] = vector(); + waitlist[digest][|waitlist[digest]|] = c$ssl; if ( waits_already ) return; @@ -89,15 +89,15 @@ event x509_certificate(c: connection, is_orig: bool, cert: X509, r$valid = split(fields[5], /=/)[2] == "1"; # Assign notary answer to all records waiting for this digest. - if ( digest in wait_list ) + if ( digest in waitlist ) { - for ( i in wait_list[digest] ) + for ( i in waitlist[digest] ) { - local info = wait_list[digest][i]; + local info = waitlist[digest][i]; SSL::undelay_record(info, "notary"); info$notary = r; } - delete wait_list[digest]; + delete waitlist[digest]; } } } From f7679a3d50538a48b5f7cb46fad287eb1e420527 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Tue, 15 Jan 2013 15:03:20 -0800 Subject: [PATCH 23/43] add opaque type-ignoring for the accept_unsupported_types input framework option. Allows reading of records that contain &optional opaque-entries when accept_unsupported_types=T --- src/input/Manager.cc | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/input/Manager.cc b/src/input/Manager.cc index 33f612cddd..1082a003e9 100644 --- a/src/input/Manager.cc +++ b/src/input/Manager.cc @@ -794,15 +794,16 @@ bool Manager::UnrollRecordType(vector *fields, const RecordType *rec, if ( allow_file_func ) { if ( ( rec->FieldType(i)->Tag() == TYPE_FILE || - rec->FieldType(i)->Tag() == TYPE_FUNC ) && + rec->FieldType(i)->Tag() == TYPE_FUNC || + rec->FieldType(i)->Tag() == TYPE_OPAQUE ) && rec->FieldDecl(i)->FindAttr(ATTR_OPTIONAL) ) { - reporter->Info("Encountered incompatible type \"%s\" in table definition for ReaderFrontend. Ignoring field.", type_name(rec->FieldType(i)->Tag())); + reporter->Info("Encountered incompatible type \"%s\" in type definition for ReaderFrontend. Ignoring optional field.", type_name(rec->FieldType(i)->Tag())); continue; } } - reporter->Error("Incompatible type \"%s\" in table definition for ReaderFrontend", type_name(rec->FieldType(i)->Tag())); + reporter->Error("Incompatible type \"%s\" in type definition for ReaderFrontend", type_name(rec->FieldType(i)->Tag())); return false; } From be71a42f4c9cdde69b74f18203db062dbc18dea2 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Wed, 16 Jan 2013 16:17:17 -0600 Subject: [PATCH 24/43] Add "fallthrough" keyword, require a flow statement to end case blocks. Case blocks in switch statements now must end in a break, return, or fallthrough statement to give best mix of safety, readability, and flexibility. The new fallthrough keyword explicitly allows control to be passed to the next case block in a switch statement. Addresses #754. --- src/SerialTypes.h | 1 + src/Stmt.cc | 83 +++++++++++++++++-- src/Stmt.h | 18 +++- src/StmtEnums.h | 4 +- src/parse.y | 12 ++- src/scan.l | 1 + testing/btest/core/leaks/switch-statement.bro | 32 ++++--- testing/btest/language/switch-statement.bro | 32 ++++--- 8 files changed, 150 insertions(+), 33 deletions(-) diff --git a/src/SerialTypes.h b/src/SerialTypes.h index e103c1c40e..723badab1e 100644 --- a/src/SerialTypes.h +++ b/src/SerialTypes.h @@ -171,6 +171,7 @@ SERIAL_STMT(EVENT_BODY_LIST, 16) SERIAL_STMT(INIT_STMT, 17) SERIAL_STMT(NULL_STMT, 18) SERIAL_STMT(WHEN_STMT, 19) +SERIAL_STMT(FALLTHROUGH_STMT, 20) #define SERIAL_TYPE(name, val) SERIAL_CONST(name, val, BRO_TYPE) SERIAL_TYPE(BRO_TYPE, 1) diff --git a/src/Stmt.cc b/src/Stmt.cc index 3e37256338..cc506db985 100644 --- a/src/Stmt.cc +++ b/src/Stmt.cc @@ -23,7 +23,7 @@ const char* stmt_name(BroStmtTag t) "print", "event", "expr", "if", "when", "switch", "for", "next", "break", "return", "add", "delete", "list", "bodylist", - "", + "", "fallthrough", "null", }; @@ -584,6 +584,29 @@ bool IfStmt::DoUnserialize(UnserialInfo* info) return s2 != 0; } +static BroStmtTag get_last_stmt_tag(const Stmt* stmt) + { + if ( ! stmt ) return STMT_NULL; + + if ( stmt->Tag() != STMT_LIST ) return stmt->Tag(); + + const StmtList* stmts = stmt->AsStmtList(); + int len = stmts->Stmts().length(); + + if ( len == 0 ) return STMT_LIST; + + return get_last_stmt_tag(stmts->Stmts()[len - 1]); + } + +Case::Case(ListExpr* c, Stmt* arg_s) + : cases(simplify_expr_list(c, SIMPLIFY_GENERAL)), s(arg_s) + { + BroStmtTag t = get_last_stmt_tag(Body()); + + if ( t != STMT_BREAK && t != STMT_FALLTHROUGH && t != STMT_RETURN ) + Error("case block must end in break/fallthrough/return statement"); + } + Case::~Case() { Unref(cases); @@ -701,9 +724,6 @@ SwitchStmt::SwitchStmt(Expr* index, case_list* arg_cases) : const Case* c = (*cases)[i]; const ListExpr* le = c->Cases(); - if ( ! c->Body() || c->Body()->AsStmtList()->Stmts().length() == 0 ) - c->Error("empty case label body does nothing"); - if ( le ) { if ( ! le->Type()->AsTypeList()->AllMatch(e->Type(), false) ) @@ -798,12 +818,18 @@ Val* SwitchStmt::DoExec(Frame* f, Val* v, stmt_flow_type& flow) const if ( matching_label_idx == -1 ) return 0; - const Case* c = (*cases)[matching_label_idx]; + for ( int i = matching_label_idx; i < cases->length(); ++i ) + { + const Case* c = (*cases)[i]; - flow = FLOW_NEXT; - rval = c->Body()->Exec(f, flow); + flow = FLOW_NEXT; + rval = c->Body()->Exec(f, flow); - if ( flow == FLOW_BREAK ) + if ( flow == FLOW_BREAK || flow == FLOW_RETURN ) + break; + } + + if ( flow != FLOW_RETURN ) flow = FLOW_NEXT; return rval; @@ -1461,6 +1487,47 @@ bool BreakStmt::DoUnserialize(UnserialInfo* info) return true; } +Val* FallthroughStmt::Exec(Frame* /* f */, stmt_flow_type& flow) const + { + RegisterAccess(); + flow = FLOW_FALLTHROUGH; + return 0; + } + +int FallthroughStmt::IsPure() const + { + return 1; + } + +void FallthroughStmt::Describe(ODesc* d) const + { + Stmt::Describe(d); + Stmt::DescribeDone(d); + } + +TraversalCode FallthroughStmt::Traverse(TraversalCallback* cb) const + { + TraversalCode tc = cb->PreStmt(this); + HANDLE_TC_STMT_PRE(tc); + + tc = cb->PostStmt(this); + HANDLE_TC_STMT_POST(tc); + } + +IMPLEMENT_SERIAL(FallthroughStmt, SER_FALLTHROUGH_STMT); + +bool FallthroughStmt::DoSerialize(SerialInfo* info) const + { + DO_SERIALIZE(SER_FALLTHROUGH_STMT, Stmt); + return true; + } + +bool FallthroughStmt::DoUnserialize(UnserialInfo* info) + { + DO_UNSERIALIZE(Stmt); + return true; + } + ReturnStmt::ReturnStmt(Expr* arg_e) : ExprStmt(STMT_RETURN, arg_e) { Scope* s = current_scope(); diff --git a/src/Stmt.h b/src/Stmt.h index 32be7f33fc..32b90b4190 100644 --- a/src/Stmt.h +++ b/src/Stmt.h @@ -195,8 +195,7 @@ protected: class Case : public BroObj { public: - Case(ListExpr* c, Stmt* arg_s) : - cases(simplify_expr_list(c,SIMPLIFY_GENERAL)), s(arg_s) { } + Case(ListExpr* c, Stmt* arg_s); ~Case(); const ListExpr* Cases() const { return cases; } @@ -371,6 +370,21 @@ protected: DECLARE_SERIAL(BreakStmt); }; +class FallthroughStmt : public Stmt { +public: + FallthroughStmt() : Stmt(STMT_FALLTHROUGH) { } + + Val* Exec(Frame* f, stmt_flow_type& flow) const; + int IsPure() const; + + void Describe(ODesc* d) const; + + TraversalCode Traverse(TraversalCallback* cb) const; + +protected: + DECLARE_SERIAL(FallthroughStmt); +}; + class ReturnStmt : public ExprStmt { public: ReturnStmt(Expr* e); diff --git a/src/StmtEnums.h b/src/StmtEnums.h index f431e3fea1..1114816a93 100644 --- a/src/StmtEnums.h +++ b/src/StmtEnums.h @@ -16,6 +16,7 @@ typedef enum { STMT_ADD, STMT_DELETE, STMT_LIST, STMT_EVENT_BODY_LIST, STMT_INIT, + STMT_FALLTHROUGH, STMT_NULL #define NUM_STMTS (int(STMT_NULL) + 1) } BroStmtTag; @@ -24,7 +25,8 @@ typedef enum { FLOW_NEXT, // continue on to next statement FLOW_LOOP, // go to top of loop FLOW_BREAK, // break out of loop - FLOW_RETURN // return from function + FLOW_RETURN, // return from function + FLOW_FALLTHROUGH// fall through to next switch case } stmt_flow_type; extern const char* stmt_name(BroStmtTag t); diff --git a/src/parse.y b/src/parse.y index 090786647e..7ce1174595 100644 --- a/src/parse.y +++ b/src/parse.y @@ -8,8 +8,8 @@ %token TOK_ATENDIF TOK_ATELSE TOK_ATIF TOK_ATIFDEF TOK_ATIFNDEF %token TOK_BOOL TOK_BREAK TOK_CASE TOK_CONST %token TOK_CONSTANT TOK_COPY TOK_COUNT TOK_COUNTER TOK_DEFAULT TOK_DELETE -%token TOK_DOUBLE TOK_ELSE TOK_ENUM TOK_EVENT TOK_EXPORT TOK_FILE TOK_FOR -%token TOK_FUNCTION TOK_GLOBAL TOK_HOOK TOK_ID TOK_IF TOK_INT +%token TOK_DOUBLE TOK_ELSE TOK_ENUM TOK_EVENT TOK_EXPORT TOK_FALLTHROUGH +%token TOK_FILE TOK_FOR TOK_FUNCTION TOK_GLOBAL TOK_HOOK TOK_ID TOK_IF TOK_INT %token TOK_INTERVAL TOK_LIST TOK_LOCAL TOK_MODULE %token TOK_NEXT TOK_OF TOK_OPAQUE TOK_PATTERN TOK_PATTERN_TEXT %token TOK_PORT TOK_PRINT TOK_RECORD TOK_REDEF @@ -1436,6 +1436,14 @@ stmt: brofiler.AddStmt($$); } + | TOK_FALLTHROUGH ';' opt_no_test + { + set_location(@1, @2); + $$ = new FallthroughStmt; + if ( ! $3 ) + brofiler.AddStmt($$); + } + | TOK_RETURN ';' opt_no_test { set_location(@1, @2); diff --git a/src/scan.l b/src/scan.l index efcd273e36..ffbc125728 100644 --- a/src/scan.l +++ b/src/scan.l @@ -282,6 +282,7 @@ else return TOK_ELSE; enum return TOK_ENUM; event return TOK_EVENT; export return TOK_EXPORT; +fallthrough return TOK_FALLTHROUGH; file return TOK_FILE; for return TOK_FOR; function return TOK_FUNCTION; diff --git a/testing/btest/core/leaks/switch-statement.bro b/testing/btest/core/leaks/switch-statement.bro index 6fbdb0d54a..845915ae8a 100644 --- a/testing/btest/core/leaks/switch-statement.bro +++ b/testing/btest/core/leaks/switch-statement.bro @@ -148,18 +148,19 @@ function switch_empty(v: count): string return "n/a"; } -function switch_break(v: count): string +function switch_fallthrough(v: count): string { local rval = ""; switch ( v ) { case 1: rval += "test"; + fallthrough; case 2: rval += "testing"; - break; - rval += "ERROR"; + fallthrough; case 3: rval += "tested"; + break; } return rval + "return"; } @@ -170,12 +171,16 @@ function switch_default(v: count): string switch ( v ) { case 1: rval += "1"; + fallthrough; case 2: rval += "2"; + break; case 3: rval += "3"; + fallthrough; default: rval += "d"; + break; } return rval + "r"; } @@ -186,13 +191,16 @@ function switch_default_placement(v: count): string switch ( v ) { case 1: rval += "1"; + fallthrough; default: rval += "d"; + fallthrough; case 2: rval += "2"; break; case 3: rval += "3"; + break; } return rval + "r"; } @@ -252,17 +260,17 @@ event new_connection(c: connection) test_switch( switch_subnet([fe80::1]/96) , "[fe80::0]" ); test_switch( switch_subnet(192.168.1.100/16) , "192.168.0.0/16" ); test_switch( switch_empty(2) , "n/a" ); - test_switch( switch_break(1) , "testreturn" ); - test_switch( switch_break(2) , "testingreturn" ); - test_switch( switch_break(3) , "testedreturn" ); - test_switch( switch_default(1) , "1r" ); + test_switch( switch_fallthrough(1) , "testtestingtestedreturn" ); + test_switch( switch_fallthrough(2) , "testingtestedreturn" ); + test_switch( switch_fallthrough(3) , "testedreturn" ); + test_switch( switch_default(1) , "12r" ); test_switch( switch_default(2) , "2r" ); - test_switch( switch_default(3) , "3r" ); + test_switch( switch_default(3) , "3dr" ); test_switch( switch_default(4) , "dr" ); - test_switch( switch_default_placement(1) , "1r" ); + test_switch( switch_default_placement(1) , "1d2r" ); test_switch( switch_default_placement(2) , "2r" ); test_switch( switch_default_placement(3) , "3r" ); - test_switch( switch_default_placement(4) , "dr" ); + test_switch( switch_default_placement(4) , "d2r" ); local v = vector(0,1,2,3,4,5,6,7,9,10); local expect: string; @@ -272,12 +280,16 @@ event new_connection(c: connection) switch ( v[i] ) { case 1, 2: expect = "1,2"; + break; case 3, 4, 5: expect = "3,4,5"; + break; case 6, 7, 8, 9: expect = "6,7,8,9"; + break; default: expect = "n/a"; + break; } test_switch( switch_case_list(v[i]) , expect ); } diff --git a/testing/btest/language/switch-statement.bro b/testing/btest/language/switch-statement.bro index dcf2a4c041..152b14f87d 100644 --- a/testing/btest/language/switch-statement.bro +++ b/testing/btest/language/switch-statement.bro @@ -143,18 +143,19 @@ function switch_empty(v: count): string return "n/a"; } -function switch_break(v: count): string +function switch_fallthrough(v: count): string { local rval = ""; switch ( v ) { case 1: rval += "test"; + fallthrough; case 2: rval += "testing"; - break; - rval += "ERROR"; + fallthrough; case 3: rval += "tested"; + break; } return rval + "return"; } @@ -165,12 +166,16 @@ function switch_default(v: count): string switch ( v ) { case 1: rval += "1"; + fallthrough; case 2: rval += "2"; + break; case 3: rval += "3"; + fallthrough; default: rval += "d"; + break; } return rval + "r"; } @@ -181,13 +186,16 @@ function switch_default_placement(v: count): string switch ( v ) { case 1: rval += "1"; + fallthrough; default: rval += "d"; + fallthrough; case 2: rval += "2"; break; case 3: rval += "3"; + break; } return rval + "r"; } @@ -247,17 +255,17 @@ event bro_init() test_switch( switch_subnet([fe80::1]/96) , "[fe80::0]" ); test_switch( switch_subnet(192.168.1.100/16) , "192.168.0.0/16" ); test_switch( switch_empty(2) , "n/a" ); - test_switch( switch_break(1) , "testreturn" ); - test_switch( switch_break(2) , "testingreturn" ); - test_switch( switch_break(3) , "testedreturn" ); - test_switch( switch_default(1) , "1r" ); + test_switch( switch_fallthrough(1) , "testtestingtestedreturn" ); + test_switch( switch_fallthrough(2) , "testingtestedreturn" ); + test_switch( switch_fallthrough(3) , "testedreturn" ); + test_switch( switch_default(1) , "12r" ); test_switch( switch_default(2) , "2r" ); - test_switch( switch_default(3) , "3r" ); + test_switch( switch_default(3) , "3dr" ); test_switch( switch_default(4) , "dr" ); - test_switch( switch_default_placement(1) , "1r" ); + test_switch( switch_default_placement(1) , "1d2r" ); test_switch( switch_default_placement(2) , "2r" ); test_switch( switch_default_placement(3) , "3r" ); - test_switch( switch_default_placement(4) , "dr" ); + test_switch( switch_default_placement(4) , "d2r" ); local v = vector(0,1,2,3,4,5,6,7,9,10); local expect: string; @@ -267,12 +275,16 @@ event bro_init() switch ( v[i] ) { case 1, 2: expect = "1,2"; + break; case 3, 4, 5: expect = "3,4,5"; + break; case 6, 7, 8, 9: expect = "6,7,8,9"; + break; default: expect = "n/a"; + break; } test_switch( switch_case_list(v[i]) , expect ); } From 0a69b87f03b18f6c5b4e6952912b5390c9e698b1 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Thu, 17 Jan 2013 15:21:50 -0600 Subject: [PATCH 25/43] Fix uninitialized locals in event/hook handlers from having a value. Since values for local variables are referenced by offset within a Frame (not by identifier name), and event/hook handler bodies share a common Frame, the value offsets for local variables in different handlers may overlap. This meant locals in a handler without an initialization may actually end up referring to the value of a previous handler's local that has the same Frame offset. When executing the body, that can possibly result in a type-conflict error or give give unexpected results instead of a "use of uninitialized value" error. This patch makes it so uninitialized locals do always refer to a null value before executing the body of a event/hook handler, so that using them without assigning a value within the body will connsistently give a "use of uninitialized value" error. Addresses #932. --- src/Stmt.cc | 16 +++++++++---- src/Var.cc | 5 +--- .../Baseline/language.uninitialized-local/out | 2 ++ .../btest/language/uninitialized-local.bro | 23 +++++++++++++++++++ 4 files changed, 38 insertions(+), 8 deletions(-) create mode 100644 testing/btest/Baseline/language.uninitialized-local/out create mode 100644 testing/btest/language/uninitialized-local.bro diff --git a/src/Stmt.cc b/src/Stmt.cc index 2cd7117ddb..d7052d0b66 100644 --- a/src/Stmt.cc +++ b/src/Stmt.cc @@ -1789,13 +1789,21 @@ Val* InitStmt::Exec(Frame* f, stmt_flow_type& flow) const ID* aggr = (*inits)[i]; BroType* t = aggr->Type(); - Val* v; - if ( t->Tag() == TYPE_RECORD ) + Val* v = 0; + + switch ( t->Tag() ) { + case TYPE_RECORD: v = new RecordVal(t->AsRecordType()); - else if ( aggr->Type()->Tag() == TYPE_VECTOR ) + break; + case TYPE_VECTOR: v = new VectorVal(t->AsVectorType()); - else + break; + case TYPE_TABLE: v = new TableVal(t->AsTableType(), aggr->Attrs()); + break; + default: + break; + } f->SetElement(aggr->Offset(), v); } diff --git a/src/Var.cc b/src/Var.cc index b4d76097d3..0aadd93e92 100644 --- a/src/Var.cc +++ b/src/Var.cc @@ -243,10 +243,7 @@ Stmt* add_local(ID* id, BroType* t, init_class c, Expr* init, else { - if ( t->Tag() == TYPE_RECORD || t->Tag() == TYPE_TABLE || - t->Tag() == TYPE_VECTOR ) - current_scope()->AddInit(id); - + current_scope()->AddInit(id); return new NullStmt; } } diff --git a/testing/btest/Baseline/language.uninitialized-local/out b/testing/btest/Baseline/language.uninitialized-local/out new file mode 100644 index 0000000000..f803415fe6 --- /dev/null +++ b/testing/btest/Baseline/language.uninitialized-local/out @@ -0,0 +1,2 @@ +error in /Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/language.uninitialized-local/uninitialized-local.bro, line 16: value used but not set (my_string) +Continuing diff --git a/testing/btest/language/uninitialized-local.bro b/testing/btest/language/uninitialized-local.bro new file mode 100644 index 0000000000..e1cc178c0a --- /dev/null +++ b/testing/btest/language/uninitialized-local.bro @@ -0,0 +1,23 @@ +# @TEST-EXEC: bro -b %INPUT >out 2>&1 +# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff out + +event testit() &priority=10 + { + local my_count: count = 10; + } + +event testit() + { + # my_string's value occupies same Frame offset as my_count's from above + # handler, but execution of this handler body should still "initialize" + # it to a null value instead of referring to a left-over value of my_count. + local my_string: string; + local my_vector: vector of string; + my_vector[0] = my_string; + print "Continuing"; + } + +event bro_init() + { + event testit(); + } From 2823744ea5a5f315b0897731866ba60a37870ad3 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Thu, 17 Jan 2013 17:40:34 -0800 Subject: [PATCH 26/43] Removing unused class member. --- src/threading/BasicThread.h | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/threading/BasicThread.h b/src/threading/BasicThread.h index e17324e948..100efe8851 100644 --- a/src/threading/BasicThread.h +++ b/src/threading/BasicThread.h @@ -199,10 +199,6 @@ private: bool terminating; // Set to to true to signal termination. bool killed; // Set to true once forcefully killed. - // Used as a semaphore to tell the pthread thread when it may - // terminate. - pthread_mutex_t terminate; - // For implementing Fmt(). char* buf; unsigned int buf_len; From fdd11428c142868b2856978322f42204db6e1d40 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Fri, 18 Jan 2013 12:46:00 -0600 Subject: [PATCH 27/43] Change reporter messages to more reliably print to stderr. Moved this functionality to be internal instead of in the script-layer event handlers. The issue with the later is that bad things can happen between the time a reporter event handler is dispatched and the time it is executed, and if bro crashes in that time, the message may never be seen/logged. Addressed #930 (and revisits #836). --- scripts/base/frameworks/reporter/main.bro | 46 ++++--------------- scripts/base/init-bare.bro | 19 ++++++++ src/Reporter.cc | 37 ++++++++++++--- src/Reporter.h | 6 +++ src/main.cc | 2 + .../core.reporter-error-in-handler/output | 4 +- .../core.reporter-runtime-error/output | 4 +- .../Baseline/core.reporter/logger-test.log | 12 ++--- testing/btest/Baseline/core.reporter/output | 16 ++++--- .../.stderr | 2 +- .../reporter.log | 6 +-- 11 files changed, 90 insertions(+), 64 deletions(-) diff --git a/scripts/base/frameworks/reporter/main.bro b/scripts/base/frameworks/reporter/main.bro index edc5b1779a..249ecdac98 100644 --- a/scripts/base/frameworks/reporter/main.bro +++ b/scripts/base/frameworks/reporter/main.bro @@ -1,10 +1,15 @@ ##! This framework is intended to create an output and filtering path for ##! internal messages/warnings/errors. It should typically be loaded to -##! avoid Bro spewing internal messages to standard error and instead log -##! them to a file in a standard way. Note that this framework deals with -##! the handling of internally-generated reporter messages, for the -##! interface into actually creating reporter messages from the scripting -##! layer, use the built-in functions in :doc:`/scripts/base/reporter.bif`. +##! log such messages to a file in a standard way. For the options to +##! toggle whether messages are additionally written to STDERR, see +##! :bro:see:`Reporter::info_to_stderr`, +##! :bro:see:`Reporter::warnings_to_stderr`, and +##! :bro:see:`Reporter::errors_to_stderr`. +##! +##! Note that this framework deals with the handling of internally generated +##! reporter messages, for the interface in to actually creating interface +##! into actually creating reporter messages from the scripting layer, use +##! the built-in functions in :doc:`/scripts/base/reporter.bif`. module Reporter; @@ -36,26 +41,11 @@ export { ## Not all reporter messages will have locations in them though. location: string &log &optional; }; - - ## Tunable for sending reporter warning messages to STDERR. The option to - ## turn it off is presented here in case Bro is being run by some - ## external harness and shouldn't output anything to the console. - const warnings_to_stderr = T &redef; - - ## Tunable for sending reporter error messages to STDERR. The option to - ## turn it off is presented here in case Bro is being run by some - ## external harness and shouldn't output anything to the console. - const errors_to_stderr = T &redef; } -global stderr: file; - event bro_init() &priority=5 { Log::create_stream(Reporter::LOG, [$columns=Info]); - - if ( errors_to_stderr || warnings_to_stderr ) - stderr = open("/dev/stderr"); } event reporter_info(t: time, msg: string, location: string) &priority=-5 @@ -65,26 +55,10 @@ event reporter_info(t: time, msg: string, location: string) &priority=-5 event reporter_warning(t: time, msg: string, location: string) &priority=-5 { - if ( warnings_to_stderr ) - { - if ( t > double_to_time(0.0) ) - print stderr, fmt("WARNING: %.6f %s (%s)", t, msg, location); - else - print stderr, fmt("WARNING: %s (%s)", msg, location); - } - Log::write(Reporter::LOG, [$ts=t, $level=WARNING, $message=msg, $location=location]); } event reporter_error(t: time, msg: string, location: string) &priority=-5 { - if ( errors_to_stderr ) - { - if ( t > double_to_time(0.0) ) - print stderr, fmt("ERROR: %.6f %s (%s)", t, msg, location); - else - print stderr, fmt("ERROR: %s (%s)", msg, location); - } - Log::write(Reporter::LOG, [$ts=t, $level=ERROR, $message=msg, $location=location]); } diff --git a/scripts/base/init-bare.bro b/scripts/base/init-bare.bro index e5365a9428..1a1349dc94 100644 --- a/scripts/base/init-bare.bro +++ b/scripts/base/init-bare.bro @@ -2857,6 +2857,25 @@ export { } # end export module GLOBAL; +module Reporter; +export { + ## Tunable for sending reporter info messages to STDERR. The option to + ## turn it off is presented here in case Bro is being run by some + ## external harness and shouldn't output anything to the console. + const info_to_stderr = T &redef; + + ## Tunable for sending reporter warning messages to STDERR. The option to + ## turn it off is presented here in case Bro is being run by some + ## external harness and shouldn't output anything to the console. + const warnings_to_stderr = T &redef; + + ## Tunable for sending reporter error messages to STDERR. The option to + ## turn it off is presented here in case Bro is being run by some + ## external harness and shouldn't output anything to the console. + const errors_to_stderr = T &redef; +} +module GLOBAL; + ## Number of bytes per packet to capture from live interfaces. const snaplen = 8192 &redef; diff --git a/src/Reporter.cc b/src/Reporter.cc index 18f39ce4af..fc3dde3ca0 100644 --- a/src/Reporter.cc +++ b/src/Reporter.cc @@ -27,6 +27,13 @@ Reporter::Reporter() via_events = false; in_error_handler = 0; + // Always use stderr at startup/init before scripts have been fully parsed. + // Messages may otherwise be missed if an error occurs that prevents events + // from ever being dispatched. + info_to_stderr = true; + warnings_to_stderr = true; + errors_to_stderr = true; + openlog("bro", 0, LOG_LOCAL5); } @@ -35,11 +42,19 @@ Reporter::~Reporter() closelog(); } +void Reporter::InitOptions() + { + info_to_stderr = internal_const_val("Reporter::info_to_stderr")->AsBool(); + warnings_to_stderr = internal_const_val("Reporter::warnings_to_stderr")->AsBool(); + errors_to_stderr = internal_const_val("Reporter::errors_to_stderr")->AsBool(); + } + void Reporter::Info(const char* fmt, ...) { va_list ap; va_start(ap, fmt); - DoLog("", reporter_info, stderr, 0, 0, true, true, 0, fmt, ap); + FILE* out = info_to_stderr ? stderr : 0; + DoLog("", reporter_info, out, 0, 0, true, true, 0, fmt, ap); va_end(ap); } @@ -47,7 +62,8 @@ void Reporter::Warning(const char* fmt, ...) { va_list ap; va_start(ap, fmt); - DoLog("warning", reporter_warning, stderr, 0, 0, true, true, 0, fmt, ap); + FILE* out = warnings_to_stderr ? stderr : 0; + DoLog("warning", reporter_warning, out, 0, 0, true, true, 0, fmt, ap); va_end(ap); } @@ -56,7 +72,8 @@ void Reporter::Error(const char* fmt, ...) ++errors; va_list ap; va_start(ap, fmt); - DoLog("error", reporter_error, stderr, 0, 0, true, true, 0, fmt, ap); + FILE* out = errors_to_stderr ? stderr : 0; + DoLog("error", reporter_error, out, 0, 0, true, true, 0, fmt, ap); va_end(ap); } @@ -98,7 +115,9 @@ void Reporter::ExprRuntimeError(const Expr* expr, const char* fmt, ...) PushLocation(expr->GetLocationInfo()); va_list ap; va_start(ap, fmt); - DoLog("expression error", reporter_error, stderr, 0, 0, true, true, d.Description(), fmt, ap); + FILE* out = errors_to_stderr ? stderr : 0; + DoLog("expression error", reporter_error, out, 0, 0, true, true, + d.Description(), fmt, ap); va_end(ap); PopLocation(); throw InterpreterException(); @@ -122,7 +141,9 @@ void Reporter::InternalWarning(const char* fmt, ...) { va_list ap; va_start(ap, fmt); - DoLog("internal warning", reporter_warning, stderr, 0, 0, true, true, 0, fmt, ap); + FILE* out = warnings_to_stderr ? stderr : 0; + DoLog("internal warning", reporter_warning, out, 0, 0, true, true, 0, fmt, + ap); va_end(ap); } @@ -189,7 +210,9 @@ void Reporter::Weird(const IPAddr& orig, const IPAddr& resp, const char* name) WeirdFlowHelper(orig, resp, "%s", name); } -void Reporter::DoLog(const char* prefix, EventHandlerPtr event, FILE* out, Connection* conn, val_list* addl, bool location, bool time, const char* postfix, const char* fmt, va_list ap) +void Reporter::DoLog(const char* prefix, EventHandlerPtr event, FILE* out, + Connection* conn, val_list* addl, bool location, bool time, + const char* postfix, const char* fmt, va_list ap) { static char tmp[512]; @@ -298,7 +321,7 @@ void Reporter::DoLog(const char* prefix, EventHandlerPtr event, FILE* out, Conne mgr.QueueEvent(event, vl); } - else + if ( out ) { string s = ""; diff --git a/src/Reporter.h b/src/Reporter.h index e610e1519e..6a205cfd6d 100644 --- a/src/Reporter.h +++ b/src/Reporter.h @@ -43,6 +43,9 @@ public: Reporter(); ~Reporter(); + // Initialize reporter-sepcific options that are defined in script-layer. + void InitOptions(); + // Report an informational message, nothing that needs specific // attention. void Info(const char* fmt, ...) FMT_ATTR; @@ -126,6 +129,9 @@ private: int errors; bool via_events; int in_error_handler; + bool info_to_stderr; + bool warnings_to_stderr; + bool errors_to_stderr; std::list > locations; }; diff --git a/src/main.cc b/src/main.cc index 5999186240..59cdb23241 100644 --- a/src/main.cc +++ b/src/main.cc @@ -824,6 +824,8 @@ int main(int argc, char** argv) exit(1); } + reporter->InitOptions(); + init_general_global_var(); if ( user_pcap_filter ) diff --git a/testing/btest/Baseline/core.reporter-error-in-handler/output b/testing/btest/Baseline/core.reporter-error-in-handler/output index b20b1b2292..2f92afe6b0 100644 --- a/testing/btest/Baseline/core.reporter-error-in-handler/output +++ b/testing/btest/Baseline/core.reporter-error-in-handler/output @@ -1,3 +1,3 @@ -error in /home/jsiwek/bro/testing/btest/.tmp/core.reporter-error-in-handler/reporter-error-in-handler.bro, line 22: no such index (a[2]) -ERROR: no such index (a[1]) (/home/jsiwek/bro/testing/btest/.tmp/core.reporter-error-in-handler/reporter-error-in-handler.bro, line 28) +error in /Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/core.reporter-error-in-handler/reporter-error-in-handler.bro, line 28: no such index (a[1]) +error in /Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/core.reporter-error-in-handler/reporter-error-in-handler.bro, line 22: no such index (a[2]) 1st error printed on script level diff --git a/testing/btest/Baseline/core.reporter-runtime-error/output b/testing/btest/Baseline/core.reporter-runtime-error/output index 5a03f5feb2..c2ace6ceb6 100644 --- a/testing/btest/Baseline/core.reporter-runtime-error/output +++ b/testing/btest/Baseline/core.reporter-runtime-error/output @@ -1,2 +1,2 @@ -error in /home/jsiwek/bro/testing/btest/.tmp/core.reporter-runtime-error/reporter-runtime-error.bro, line 12: no such index (a[1]) -ERROR: no such index (a[2]) (/home/jsiwek/bro/testing/btest/.tmp/core.reporter-runtime-error/reporter-runtime-error.bro, line 9) +error in /Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/core.reporter-runtime-error/reporter-runtime-error.bro, line 12: no such index (a[1]) +error in /Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/core.reporter-runtime-error/reporter-runtime-error.bro, line 9: no such index (a[2]) diff --git a/testing/btest/Baseline/core.reporter/logger-test.log b/testing/btest/Baseline/core.reporter/logger-test.log index 5afd904b63..4ee0d03341 100644 --- a/testing/btest/Baseline/core.reporter/logger-test.log +++ b/testing/btest/Baseline/core.reporter/logger-test.log @@ -1,6 +1,6 @@ -reporter_info|init test-info|/home/jsiwek/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 8|0.000000 -reporter_warning|init test-warning|/home/jsiwek/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 9|0.000000 -reporter_error|init test-error|/home/jsiwek/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 10|0.000000 -reporter_info|done test-info|/home/jsiwek/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 15|0.000000 -reporter_warning|done test-warning|/home/jsiwek/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 16|0.000000 -reporter_error|done test-error|/home/jsiwek/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 17|0.000000 +reporter_info|init test-info|/Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 8|0.000000 +reporter_warning|init test-warning|/Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 9|0.000000 +reporter_error|init test-error|/Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 10|0.000000 +reporter_info|done test-info|/Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 15|0.000000 +reporter_warning|done test-warning|/Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 16|0.000000 +reporter_error|done test-error|/Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 17|0.000000 diff --git a/testing/btest/Baseline/core.reporter/output b/testing/btest/Baseline/core.reporter/output index f2c59259c2..24a12f9679 100644 --- a/testing/btest/Baseline/core.reporter/output +++ b/testing/btest/Baseline/core.reporter/output @@ -1,7 +1,9 @@ -/home/jsiwek/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 52: pre test-info -warning in /home/jsiwek/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 53: pre test-warning -error in /home/jsiwek/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 54: pre test-error -WARNING: init test-warning (/home/jsiwek/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 9) -ERROR: init test-error (/home/jsiwek/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 10) -WARNING: done test-warning (/home/jsiwek/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 16) -ERROR: done test-error (/home/jsiwek/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 17) +/Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 52: pre test-info +warning in /Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 53: pre test-warning +error in /Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 54: pre test-error +/Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 8: init test-info +warning in /Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 9: init test-warning +error in /Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 10: init test-error +/Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 15: done test-info +warning in /Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 16: done test-warning +error in /Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 17: done test-error diff --git a/testing/btest/Baseline/scripts.base.frameworks.reporter.stderr/.stderr b/testing/btest/Baseline/scripts.base.frameworks.reporter.stderr/.stderr index 78af1e7a73..dc5065f5c8 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.reporter.stderr/.stderr +++ b/testing/btest/Baseline/scripts.base.frameworks.reporter.stderr/.stderr @@ -1 +1 @@ -ERROR: no such index (test[3]) (/blah/testing/btest/.tmp/scripts.base.frameworks.reporter.stderr/stderr.bro, line 9) +error in /Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/scripts.base.frameworks.reporter.stderr/stderr.bro, line 9: no such index (test[3]) diff --git a/testing/btest/Baseline/scripts.base.frameworks.reporter.stderr/reporter.log b/testing/btest/Baseline/scripts.base.frameworks.reporter.stderr/reporter.log index b314bc45c3..391cf77a00 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.reporter.stderr/reporter.log +++ b/testing/btest/Baseline/scripts.base.frameworks.reporter.stderr/reporter.log @@ -3,8 +3,8 @@ #empty_field (empty) #unset_field - #path reporter -#open 2012-08-10-20-09-23 +#open 2013-01-18-18-29-30 #fields ts level message location #types time enum string string -0.000000 Reporter::ERROR no such index (test[3]) /da/home/robin/bro/master/testing/btest/.tmp/scripts.base.frameworks.reporter.stderr/stderr.bro, line 9 -#close 2012-08-10-20-09-23 +0.000000 Reporter::ERROR no such index (test[3]) /Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/scripts.base.frameworks.reporter.stderr/stderr.bro, line 9 +#close 2013-01-18-18-29-30 From 624980b98d7741f9af0cfb417336d149bb21e704 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Fri, 18 Jan 2013 12:56:23 -0600 Subject: [PATCH 28/43] Add a null value check in CompositeHash::ComputeHash. Because I guess aborting is nicer than segfaulting. Addresses #930. --- src/CompHash.cc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/CompHash.cc b/src/CompHash.cc index 86677f9719..306b3ea83e 100644 --- a/src/CompHash.cc +++ b/src/CompHash.cc @@ -273,6 +273,9 @@ char* CompositeHash::SingleValHash(int type_check, char* kp0, HashKey* CompositeHash::ComputeHash(const Val* v, int type_check) const { + if ( ! v ) + reporter->InternalError("null value given to CompositeHash::ComputeHash"); + if ( is_singleton ) return ComputeSingletonHash(v, type_check); From acafcfafd2cf5a5bd8b88b6256d21a5af6a92591 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Fri, 18 Jan 2013 13:15:34 -0600 Subject: [PATCH 29/43] Revert "Trick for parallelizing input framework unit tests." This reverts commit 43ed437daabb4575549e82f53881980353905242. The old way of doing the tests seems more reliable for now. --- .../.stderrwithoutfirstline | 14 +++++++------- .../.stderrwithoutfirstline | 6 +++--- .../bro..stderr | 10 +++++----- .../btest/scripts/base/frameworks/input/basic.bro | 6 +++++- .../scripts/base/frameworks/input/bignumber.bro | 7 ++++++- .../btest/scripts/base/frameworks/input/binary.bro | 7 ++++++- .../base/frameworks/input/empty-values-hashing.bro | 8 +++++++- .../scripts/base/frameworks/input/emptyvals.bro | 7 ++++++- .../btest/scripts/base/frameworks/input/event.bro | 7 ++++++- .../scripts/base/frameworks/input/executeraw.bro | 7 ++++++- .../base/frameworks/input/invalidnumbers.bro | 9 +++++++-- .../scripts/base/frameworks/input/invalidtext.bro | 9 +++++++-- .../scripts/base/frameworks/input/missing-file.bro | 7 ++++++- .../base/frameworks/input/onecolumn-norecord.bro | 7 ++++++- .../base/frameworks/input/onecolumn-record.bro | 7 ++++++- .../scripts/base/frameworks/input/optional.bro | 7 ++++++- .../btest/scripts/base/frameworks/input/port.bro | 7 ++++++- .../base/frameworks/input/predicate-stream.bro | 7 ++++++- .../scripts/base/frameworks/input/predicate.bro | 7 ++++++- .../base/frameworks/input/predicatemodify.bro | 7 ++++++- .../frameworks/input/predicatemodifyandreread.bro | 7 ++++++- .../input/predicaterefusesecondsamerecord.bro | 7 ++++++- .../btest/scripts/base/frameworks/input/raw.bro | 7 ++++++- .../btest/scripts/base/frameworks/input/repeat.bro | 7 ++++++- .../btest/scripts/base/frameworks/input/reread.bro | 6 +++++- .../scripts/base/frameworks/input/rereadraw.bro | 7 ++++++- .../btest/scripts/base/frameworks/input/set.bro | 7 ++++++- .../scripts/base/frameworks/input/setseparator.bro | 7 ++++++- .../base/frameworks/input/setspecialcases.bro | 8 +++++++- .../btest/scripts/base/frameworks/input/stream.bro | 6 +++++- .../scripts/base/frameworks/input/streamraw.bro | 7 ++++++- .../base/frameworks/input/subrecord-event.bro | 6 +++++- .../scripts/base/frameworks/input/subrecord.bro | 6 +++++- .../scripts/base/frameworks/input/tableevent.bro | 7 ++++++- .../scripts/base/frameworks/input/twotables.bro | 6 +++++- .../base/frameworks/input/unsupported_types.bro | 6 +++++- 36 files changed, 210 insertions(+), 50 deletions(-) diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.invalidnumbers/.stderrwithoutfirstline b/testing/btest/Baseline/scripts.base.frameworks.input.invalidnumbers/.stderrwithoutfirstline index 54005fb4b8..3ef51e40f2 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.input.invalidnumbers/.stderrwithoutfirstline +++ b/testing/btest/Baseline/scripts.base.frameworks.input.invalidnumbers/.stderrwithoutfirstline @@ -1,8 +1,8 @@ -1355266097.683599 error: ../input.log/Input::READER_ASCII: Number '12129223372036854775800' out of supported range. -1355266097.683599 error: ../input.log/Input::READER_ASCII: Could not convert line '12129223372036854775800 121218446744073709551612' to Val. Ignoring line. -1355266097.683599 warning: ../input.log/Input::READER_ASCII: Number '9223372036854775801TEXTHERE' contained non-numeric trailing characters. Ignored trailing characters 'TEXTHERE' -1355266097.683599 warning: ../input.log/Input::READER_ASCII: Number '1Justtext' contained non-numeric trailing characters. Ignored trailing characters 'Justtext' -1355266097.683599 error: ../input.log/Input::READER_ASCII: String 'Justtext' contained no parseable number -1355266097.683599 error: ../input.log/Input::READER_ASCII: Could not convert line 'Justtext 1' to Val. Ignoring line. -1355266097.683599 received termination signal +error: ../input.log/Input::READER_ASCII: Number '12129223372036854775800' out of supported range. +error: ../input.log/Input::READER_ASCII: Could not convert line '12129223372036854775800 121218446744073709551612' to Val. Ignoring line. +warning: ../input.log/Input::READER_ASCII: Number '9223372036854775801TEXTHERE' contained non-numeric trailing characters. Ignored trailing characters 'TEXTHERE' +warning: ../input.log/Input::READER_ASCII: Number '1Justtext' contained non-numeric trailing characters. Ignored trailing characters 'Justtext' +error: ../input.log/Input::READER_ASCII: String 'Justtext' contained no parseable number +error: ../input.log/Input::READER_ASCII: Could not convert line 'Justtext 1' to Val. Ignoring line. +received termination signal >>> diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.invalidtext/.stderrwithoutfirstline b/testing/btest/Baseline/scripts.base.frameworks.input.invalidtext/.stderrwithoutfirstline index f0545daeae..3d8ba5e267 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.input.invalidtext/.stderrwithoutfirstline +++ b/testing/btest/Baseline/scripts.base.frameworks.input.invalidtext/.stderrwithoutfirstline @@ -1,4 +1,4 @@ -1355265853.593476 error: ../input.log/Input::READER_ASCII: String 'l' contained no parseable number -1355265853.593476 error: ../input.log/Input::READER_ASCII: Could not convert line ' l' to Val. Ignoring line. -1355265853.593476 received termination signal +error: ../input.log/Input::READER_ASCII: String 'l' contained no parseable number +error: ../input.log/Input::READER_ASCII: Could not convert line ' l' to Val. Ignoring line. +received termination signal >>> diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.missing-file/bro..stderr b/testing/btest/Baseline/scripts.base.frameworks.input.missing-file/bro..stderr index e61280cdfc..4380007b93 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.input.missing-file/bro..stderr +++ b/testing/btest/Baseline/scripts.base.frameworks.input.missing-file/bro..stderr @@ -1,5 +1,5 @@ -1355265996.626106 error: does-not-exist.dat/Input::READER_ASCII: Init: cannot open does-not-exist.dat -1355265996.626106 error: does-not-exist.dat/Input::READER_ASCII: Init failed -1355265996.626106 warning: Stream input is already queued for removal. Ignoring remove. -1355265996.626106 error: does-not-exist.dat/Input::READER_ASCII: terminating thread -1355265996.626106 received termination signal +error: does-not-exist.dat/Input::READER_ASCII: Init: cannot open does-not-exist.dat +error: does-not-exist.dat/Input::READER_ASCII: Init failed +warning: Stream input is already queued for removal. Ignoring remove. +error: does-not-exist.dat/Input::READER_ASCII: terminating thread +received termination signal diff --git a/testing/btest/scripts/base/frameworks/input/basic.bro b/testing/btest/scripts/base/frameworks/input/basic.bro index fbb320e03f..dfac84d062 100644 --- a/testing/btest/scripts/base/frameworks/input/basic.bro +++ b/testing/btest/scripts/base/frameworks/input/basic.bro @@ -1,4 +1,7 @@ -# @TEST-EXEC: btest-bg-run bro bro -b --pseudo-realtime -r $TRACES/socks.trace %INPUT +# (uses listen.bro just to ensure input sources are more reliably fully-read). +# @TEST-SERIALIZE: comm +# +# @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: btest-bg-wait -k 5 # @TEST-EXEC: btest-diff out @@ -11,6 +14,7 @@ T -42 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz @TEST-END-FILE @load base/protocols/ssh +@load frameworks/communication/listen global outfile: file; diff --git a/testing/btest/scripts/base/frameworks/input/bignumber.bro b/testing/btest/scripts/base/frameworks/input/bignumber.bro index 098481a518..5b93472551 100644 --- a/testing/btest/scripts/base/frameworks/input/bignumber.bro +++ b/testing/btest/scripts/base/frameworks/input/bignumber.bro @@ -1,4 +1,7 @@ -# @TEST-EXEC: btest-bg-run bro bro -b --pseudo-realtime -r $TRACES/socks.trace %INPUT +# (uses listen.bro just to ensure input sources are more reliably fully-read). +# @TEST-SERIALIZE: comm +# +# @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: btest-bg-wait -k 5 # @TEST-EXEC: btest-diff out @@ -10,6 +13,8 @@ -9223372036854775800 18446744073709551612 @TEST-END-FILE +@load frameworks/communication/listen + global outfile: file; module A; diff --git a/testing/btest/scripts/base/frameworks/input/binary.bro b/testing/btest/scripts/base/frameworks/input/binary.bro index 7caa734d34..8d75abc5a9 100644 --- a/testing/btest/scripts/base/frameworks/input/binary.bro +++ b/testing/btest/scripts/base/frameworks/input/binary.bro @@ -1,4 +1,7 @@ -# @TEST-EXEC: btest-bg-run bro bro -b --pseudo-realtime -r $TRACES/socks.trace %INPUT +# (uses listen.bro just to ensure input sources are more reliably fully-read). +# @TEST-SERIALIZE: comm +# +# @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: btest-bg-wait -k 5 # @TEST-EXEC: btest-diff out @@ -22,6 +25,8 @@ abc\xff\x7cdef|DATA2 #end|2012-07-20-01-49-19 @TEST-END-FILE +@load frameworks/communication/listen + global outfile: file; global try: count; diff --git a/testing/btest/scripts/base/frameworks/input/empty-values-hashing.bro b/testing/btest/scripts/base/frameworks/input/empty-values-hashing.bro index 78f3d3a72e..c8760b467e 100644 --- a/testing/btest/scripts/base/frameworks/input/empty-values-hashing.bro +++ b/testing/btest/scripts/base/frameworks/input/empty-values-hashing.bro @@ -1,5 +1,8 @@ +# (uses listen.bro just to ensure input sources are more reliably fully-read). +# @TEST-SERIALIZE: comm +# # @TEST-EXEC: cp input1.log input.log -# @TEST-EXEC: btest-bg-run bro bro -b --pseudo-realtime -r $TRACES/socks.trace %INPUT +# @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: sleep 2 # @TEST-EXEC: cp input2.log input.log # @TEST-EXEC: btest-bg-wait -k 5 @@ -20,6 +23,9 @@ 2 TEST TEST @TEST-END-FILE +@load frameworks/communication/listen + + module A; type Idx: record { diff --git a/testing/btest/scripts/base/frameworks/input/emptyvals.bro b/testing/btest/scripts/base/frameworks/input/emptyvals.bro index e5e9bc22e3..94b0f1b620 100644 --- a/testing/btest/scripts/base/frameworks/input/emptyvals.bro +++ b/testing/btest/scripts/base/frameworks/input/emptyvals.bro @@ -1,4 +1,7 @@ -# @TEST-EXEC: btest-bg-run bro bro -b --pseudo-realtime -r $TRACES/socks.trace %INPUT +# (uses listen.bro just to ensure input sources are more reliably fully-read). +# @TEST-SERIALIZE: comm +# +# @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: btest-bg-wait -k 5 # @TEST-EXEC: btest-diff out @@ -11,6 +14,8 @@ T 1 - 2 @TEST-END-FILE +@load frameworks/communication/listen + global outfile: file; redef InputAscii::empty_field = "EMPTY"; diff --git a/testing/btest/scripts/base/frameworks/input/event.bro b/testing/btest/scripts/base/frameworks/input/event.bro index 7d553a53e3..ba47d5e3f2 100644 --- a/testing/btest/scripts/base/frameworks/input/event.bro +++ b/testing/btest/scripts/base/frameworks/input/event.bro @@ -1,4 +1,7 @@ -# @TEST-EXEC: btest-bg-run bro bro -b --pseudo-realtime -r $TRACES/socks.trace %INPUT +# (uses listen.bro just to ensure input sources are more reliably fully-read). +# @TEST-SERIALIZE: comm +# +# @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: btest-bg-wait -k 5 # @TEST-EXEC: btest-diff out @@ -16,6 +19,8 @@ 7 T @TEST-END-FILE +@load frameworks/communication/listen + global outfile: file; module A; diff --git a/testing/btest/scripts/base/frameworks/input/executeraw.bro b/testing/btest/scripts/base/frameworks/input/executeraw.bro index 09cd920bee..626b9cdfd2 100644 --- a/testing/btest/scripts/base/frameworks/input/executeraw.bro +++ b/testing/btest/scripts/base/frameworks/input/executeraw.bro @@ -1,4 +1,7 @@ -# @TEST-EXEC: btest-bg-run bro bro -b --pseudo-realtime -r $TRACES/socks.trace %INPUT +# (uses listen.bro just to ensure input sources are more reliably fully-read). +# @TEST-SERIALIZE: comm +# +# @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: btest-bg-wait -k 5 # @TEST-EXEC: cat out.tmp | sed 's/^ *//g' >out # @TEST-EXEC: btest-diff out @@ -14,6 +17,8 @@ sdf 3rw43wRRERLlL#RWERERERE. @TEST-END-FILE +@load frameworks/communication/listen + global outfile: file; type Val: record { diff --git a/testing/btest/scripts/base/frameworks/input/invalidnumbers.bro b/testing/btest/scripts/base/frameworks/input/invalidnumbers.bro index 3c467598ad..1deec605ae 100644 --- a/testing/btest/scripts/base/frameworks/input/invalidnumbers.bro +++ b/testing/btest/scripts/base/frameworks/input/invalidnumbers.bro @@ -1,8 +1,11 @@ -# @TEST-EXEC: btest-bg-run bro bro -b --pseudo-realtime -r $TRACES/socks.trace %INPUT +# (uses listen.bro just to ensure input sources are more reliably fully-read). +# @TEST-SERIALIZE: comm +# +# @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: btest-bg-wait -k 5 # @TEST-EXEC: btest-diff out # @TEST-EXEC: sed 1d .stderr > .stderrwithoutfirstline -# @TEST-EXEC: TEST_DIFF_CANONIFIER="$SCRIPTS/diff-remove-abspath | $SCRIPTS/diff-remove-timestamps" btest-diff .stderrwithoutfirstline +# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff .stderrwithoutfirstline @TEST-START-FILE input.log #separator \x09 @@ -14,6 +17,8 @@ Justtext 1 9223372036854775800 -18446744073709551612 @TEST-END-FILE +@load frameworks/communication/listen + global outfile: file; module A; diff --git a/testing/btest/scripts/base/frameworks/input/invalidtext.bro b/testing/btest/scripts/base/frameworks/input/invalidtext.bro index 668716d045..75efb1247d 100644 --- a/testing/btest/scripts/base/frameworks/input/invalidtext.bro +++ b/testing/btest/scripts/base/frameworks/input/invalidtext.bro @@ -1,8 +1,11 @@ -# @TEST-EXEC: btest-bg-run bro bro -b --pseudo-realtime -r $TRACES/socks.trace %INPUT +# (uses listen.bro just to ensure input sources are more reliably fully-read). +# @TEST-SERIALIZE: comm +# +# @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: btest-bg-wait -k 5 # @TEST-EXEC: btest-diff out # @TEST-EXEC: sed 1d .stderr > .stderrwithoutfirstline -# @TEST-EXEC: TEST_DIFF_CANONIFIER="$SCRIPTS/diff-remove-abspath | $SCRIPTS/diff-remove-timestamps" btest-diff .stderrwithoutfirstline +# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff .stderrwithoutfirstline @TEST-START-FILE input.log #separator \x09 @@ -12,6 +15,8 @@ 5 @TEST-END-FILE +@load frameworks/communication/listen + global outfile: file; module A; diff --git a/testing/btest/scripts/base/frameworks/input/missing-file.bro b/testing/btest/scripts/base/frameworks/input/missing-file.bro index 7048698221..aa5acf619e 100644 --- a/testing/btest/scripts/base/frameworks/input/missing-file.bro +++ b/testing/btest/scripts/base/frameworks/input/missing-file.bro @@ -1,7 +1,12 @@ -# @TEST-EXEC: btest-bg-run bro bro -b --pseudo-realtime -r $TRACES/socks.trace %INPUT +# (uses listen.bro just to ensure input sources are more reliably fully-read). +# @TEST-SERIALIZE: comm +# +# @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: btest-bg-wait -k 5 # @TEST-EXEC: btest-diff bro/.stderr +@load frameworks/communication/listen + global outfile: file; global try: count; diff --git a/testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro b/testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro index 505aa2245d..c08b1420fb 100644 --- a/testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro +++ b/testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro @@ -1,4 +1,7 @@ -# @TEST-EXEC: btest-bg-run bro bro -b --pseudo-realtime -r $TRACES/socks.trace %INPUT +# (uses listen.bro just to ensure input sources are more reliably fully-read). +# @TEST-SERIALIZE: comm +# +# @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: btest-bg-wait -k 5 # @TEST-EXEC: btest-diff out @@ -10,6 +13,8 @@ T -42 @TEST-END-FILE +@load frameworks/communication/listen + global outfile: file; redef InputAscii::empty_field = "EMPTY"; diff --git a/testing/btest/scripts/base/frameworks/input/onecolumn-record.bro b/testing/btest/scripts/base/frameworks/input/onecolumn-record.bro index e85267b4c3..9e420e75fe 100644 --- a/testing/btest/scripts/base/frameworks/input/onecolumn-record.bro +++ b/testing/btest/scripts/base/frameworks/input/onecolumn-record.bro @@ -1,4 +1,7 @@ -# @TEST-EXEC: btest-bg-run bro bro -b --pseudo-realtime -r $TRACES/socks.trace %INPUT +# (uses listen.bro just to ensure input sources are more reliably fully-read). +# @TEST-SERIALIZE: comm +# +# @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: btest-bg-wait -k 5 # @TEST-EXEC: btest-diff out @@ -10,6 +13,8 @@ T -42 @TEST-END-FILE +@load frameworks/communication/listen + global outfile: file; redef InputAscii::empty_field = "EMPTY"; diff --git a/testing/btest/scripts/base/frameworks/input/optional.bro b/testing/btest/scripts/base/frameworks/input/optional.bro index 670f0f48ba..2fe0e5c86f 100644 --- a/testing/btest/scripts/base/frameworks/input/optional.bro +++ b/testing/btest/scripts/base/frameworks/input/optional.bro @@ -1,4 +1,7 @@ -# @TEST-EXEC: btest-bg-run bro bro -b --pseudo-realtime -r $TRACES/socks.trace %INPUT +# (uses listen.bro just to ensure input sources are more reliably fully-read). +# @TEST-SERIALIZE: comm +# +# @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: btest-bg-wait -k 5 # @TEST-EXEC: btest-diff out @@ -16,6 +19,8 @@ 7 T @TEST-END-FILE +@load frameworks/communication/listen + global outfile: file; redef InputAscii::empty_field = "EMPTY"; diff --git a/testing/btest/scripts/base/frameworks/input/port.bro b/testing/btest/scripts/base/frameworks/input/port.bro index 2225132395..081c59559b 100644 --- a/testing/btest/scripts/base/frameworks/input/port.bro +++ b/testing/btest/scripts/base/frameworks/input/port.bro @@ -1,4 +1,7 @@ -# @TEST-EXEC: btest-bg-run bro bro -b --pseudo-realtime -r $TRACES/socks.trace %INPUT +# (uses listen.bro just to ensure input sources are more reliably fully-read). +# @TEST-SERIALIZE: comm +# +# @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: btest-bg-wait -k 5 # @TEST-EXEC: btest-diff out @@ -9,6 +12,8 @@ 1.2.3.6 30 unknown @TEST-END-FILE +@load frameworks/communication/listen + global outfile: file; redef InputAscii::empty_field = "EMPTY"; diff --git a/testing/btest/scripts/base/frameworks/input/predicate-stream.bro b/testing/btest/scripts/base/frameworks/input/predicate-stream.bro index b931bbc41e..8cf927e346 100644 --- a/testing/btest/scripts/base/frameworks/input/predicate-stream.bro +++ b/testing/btest/scripts/base/frameworks/input/predicate-stream.bro @@ -1,4 +1,7 @@ -# @TEST-EXEC: btest-bg-run bro bro -b --pseudo-realtime -r $TRACES/socks.trace %INPUT +# (uses listen.bro just to ensure input sources are more reliably fully-read). +# @TEST-SERIALIZE: comm +# +# @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: btest-bg-wait -k 5 # @TEST-EXEC: btest-diff out # @@ -20,6 +23,8 @@ 7 T @TEST-END-FILE +@load frameworks/communication/listen + global outfile: file; redef InputAscii::empty_field = "EMPTY"; diff --git a/testing/btest/scripts/base/frameworks/input/predicate.bro b/testing/btest/scripts/base/frameworks/input/predicate.bro index 9f2c4362de..8fb33242e8 100644 --- a/testing/btest/scripts/base/frameworks/input/predicate.bro +++ b/testing/btest/scripts/base/frameworks/input/predicate.bro @@ -1,4 +1,7 @@ -# @TEST-EXEC: btest-bg-run bro bro -b --pseudo-realtime -r $TRACES/socks.trace %INPUT +# (uses listen.bro just to ensure input sources are more reliably fully-read). +# @TEST-SERIALIZE: comm +# +# @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: btest-bg-wait -k 5 # @TEST-EXEC: btest-diff out @@ -16,6 +19,8 @@ 7 T @TEST-END-FILE +@load frameworks/communication/listen + global outfile: file; redef InputAscii::empty_field = "EMPTY"; diff --git a/testing/btest/scripts/base/frameworks/input/predicatemodify.bro b/testing/btest/scripts/base/frameworks/input/predicatemodify.bro index 4129a47873..17467bbc27 100644 --- a/testing/btest/scripts/base/frameworks/input/predicatemodify.bro +++ b/testing/btest/scripts/base/frameworks/input/predicatemodify.bro @@ -1,4 +1,7 @@ -# @TEST-EXEC: btest-bg-run bro bro -b --pseudo-realtime -r $TRACES/socks.trace %INPUT +# (uses listen.bro just to ensure input sources are more reliably fully-read). +# @TEST-SERIALIZE: comm +# +# @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: btest-bg-wait -k 5 # @TEST-EXEC: btest-diff out @@ -11,6 +14,8 @@ 2 T test2 idx2 @TEST-END-FILE +@load frameworks/communication/listen + global outfile: file; redef InputAscii::empty_field = "EMPTY"; diff --git a/testing/btest/scripts/base/frameworks/input/predicatemodifyandreread.bro b/testing/btest/scripts/base/frameworks/input/predicatemodifyandreread.bro index 17e7fa4bf2..5a9e993651 100644 --- a/testing/btest/scripts/base/frameworks/input/predicatemodifyandreread.bro +++ b/testing/btest/scripts/base/frameworks/input/predicatemodifyandreread.bro @@ -1,5 +1,8 @@ +# (uses listen.bro just to ensure input sources are more reliably fully-read). +# @TEST-SERIALIZE: comm +# # @TEST-EXEC: cp input1.log input.log -# @TEST-EXEC: btest-bg-run bro bro -b --pseudo-realtime -r $TRACES/socks.trace %INPUT +# @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: sleep 2 # @TEST-EXEC: cp input2.log input.log # @TEST-EXEC: sleep 2 @@ -55,6 +58,8 @@ 1 T test1 idx1 @TEST-END-FILE +@load frameworks/communication/listen + redef InputAscii::empty_field = "EMPTY"; module A; diff --git a/testing/btest/scripts/base/frameworks/input/predicaterefusesecondsamerecord.bro b/testing/btest/scripts/base/frameworks/input/predicaterefusesecondsamerecord.bro index 0df8b14dd6..ba0b468cdc 100644 --- a/testing/btest/scripts/base/frameworks/input/predicaterefusesecondsamerecord.bro +++ b/testing/btest/scripts/base/frameworks/input/predicaterefusesecondsamerecord.bro @@ -1,4 +1,7 @@ -# @TEST-EXEC: btest-bg-run bro bro -b --pseudo-realtime -r $TRACES/socks.trace %INPUT +# (uses listen.bro just to ensure input sources are more reliably fully-read). +# @TEST-SERIALIZE: comm +# +# @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: btest-bg-wait -k 5 # @TEST-EXEC: btest-diff out @@ -14,6 +17,8 @@ need-to-know 8c864306-d21a-37b1-8705-746a786719bf medium 95 1342569600 1.228.83. need-to-know 8c864306-d21a-37b1-8705-746a786719bf medium 65 1342656000 1.228.83.33 - - 9318 HANARO-AS Hanaro Telecom Inc. 1.224.0.0/13 apnic KR spam infrastructure spamming;malware domain public http://reputation.alienvault.com/reputation.generic @TEST-END-FILE +@load frameworks/communication/listen + global outfile: file; redef InputAscii::empty_field = "EMPTY"; diff --git a/testing/btest/scripts/base/frameworks/input/raw.bro b/testing/btest/scripts/base/frameworks/input/raw.bro index 7201e9ad3a..d15aec22bb 100644 --- a/testing/btest/scripts/base/frameworks/input/raw.bro +++ b/testing/btest/scripts/base/frameworks/input/raw.bro @@ -1,4 +1,7 @@ -# @TEST-EXEC: btest-bg-run bro bro -b --pseudo-realtime -r $TRACES/socks.trace %INPUT +# (uses listen.bro just to ensure input sources are more reliably fully-read). +# @TEST-SERIALIZE: comm +# +# @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: btest-bg-wait -k 5 # @TEST-EXEC: btest-diff out @@ -13,6 +16,8 @@ sdf 3rw43wRRERLlL#RWERERERE. @TEST-END-FILE +@load frameworks/communication/listen + global outfile: file; global try: count; diff --git a/testing/btest/scripts/base/frameworks/input/repeat.bro b/testing/btest/scripts/base/frameworks/input/repeat.bro index f29061fa8b..a966ac064e 100644 --- a/testing/btest/scripts/base/frameworks/input/repeat.bro +++ b/testing/btest/scripts/base/frameworks/input/repeat.bro @@ -1,4 +1,7 @@ -# @TEST-EXEC: btest-bg-run bro bro -b --pseudo-realtime -r $TRACES/socks.trace %INPUT +# (uses listen.bro just to ensure input sources are more reliably fully-read). +# @TEST-SERIALIZE: comm +# +# @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: btest-bg-wait -k 5 # @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-sort btest-diff out @@ -10,6 +13,8 @@ 1 T @TEST-END-FILE +@load frameworks/communication/listen + global outfile: file; global try: count; diff --git a/testing/btest/scripts/base/frameworks/input/reread.bro b/testing/btest/scripts/base/frameworks/input/reread.bro index bfb7b6fd84..11aa873f9d 100644 --- a/testing/btest/scripts/base/frameworks/input/reread.bro +++ b/testing/btest/scripts/base/frameworks/input/reread.bro @@ -1,5 +1,8 @@ +# (uses listen.bro just to ensure input sources are more reliably fully-read). +# @TEST-SERIALIZE: comm +# # @TEST-EXEC: cp input1.log input.log -# @TEST-EXEC: btest-bg-run bro bro -b --pseudo-realtime -r $TRACES/socks.trace %INPUT +# @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: sleep 2 # @TEST-EXEC: cp input2.log input.log # @TEST-EXEC: sleep 2 @@ -56,6 +59,7 @@ F -48 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz @TEST-END-FILE @load base/protocols/ssh +@load frameworks/communication/listen redef InputAscii::empty_field = "EMPTY"; diff --git a/testing/btest/scripts/base/frameworks/input/rereadraw.bro b/testing/btest/scripts/base/frameworks/input/rereadraw.bro index d1db2bb049..2fdcdc8f9e 100644 --- a/testing/btest/scripts/base/frameworks/input/rereadraw.bro +++ b/testing/btest/scripts/base/frameworks/input/rereadraw.bro @@ -1,4 +1,7 @@ -# @TEST-EXEC: btest-bg-run bro bro -b --pseudo-realtime -r $TRACES/socks.trace %INPUT +# (uses listen.bro just to ensure input sources are more reliably fully-read). +# @TEST-SERIALIZE: comm +# +# @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: btest-bg-wait -k 5 # @TEST-EXEC: btest-diff out @@ -13,6 +16,8 @@ sdf 3rw43wRRERLlL#RWERERERE. @TEST-END-FILE +@load frameworks/communication/listen + global outfile: file; global try: count; diff --git a/testing/btest/scripts/base/frameworks/input/set.bro b/testing/btest/scripts/base/frameworks/input/set.bro index a460f6788a..b2b5cea323 100644 --- a/testing/btest/scripts/base/frameworks/input/set.bro +++ b/testing/btest/scripts/base/frameworks/input/set.bro @@ -1,4 +1,7 @@ -# @TEST-EXEC: btest-bg-run bro bro -b --pseudo-realtime -r $TRACES/socks.trace %INPUT +# (uses listen.bro just to ensure input sources are more reliably fully-read). +# @TEST-SERIALIZE: comm +# +# @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: btest-bg-wait -k 5 # @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-sort btest-diff out @@ -13,6 +16,8 @@ 192.168.17.42 @TEST-END-FILE +@load frameworks/communication/listen + global outfile: file; redef InputAscii::empty_field = "EMPTY"; diff --git a/testing/btest/scripts/base/frameworks/input/setseparator.bro b/testing/btest/scripts/base/frameworks/input/setseparator.bro index 156f566d28..b7148d80bd 100644 --- a/testing/btest/scripts/base/frameworks/input/setseparator.bro +++ b/testing/btest/scripts/base/frameworks/input/setseparator.bro @@ -1,4 +1,7 @@ -# @TEST-EXEC: btest-bg-run bro bro -b --pseudo-realtime -r $TRACES/socks.trace %INPUT +# (uses listen.bro just to ensure input sources are more reliably fully-read). +# @TEST-SERIALIZE: comm +# +# @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: btest-bg-wait -k 5 # @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-sort btest-diff out @@ -10,6 +13,8 @@ redef InputAscii::set_separator = "|"; +@load frameworks/communication/listen + global outfile: file; module A; diff --git a/testing/btest/scripts/base/frameworks/input/setspecialcases.bro b/testing/btest/scripts/base/frameworks/input/setspecialcases.bro index 86e53feb0a..022eac9731 100644 --- a/testing/btest/scripts/base/frameworks/input/setspecialcases.bro +++ b/testing/btest/scripts/base/frameworks/input/setspecialcases.bro @@ -1,4 +1,7 @@ -# @TEST-EXEC: btest-bg-run bro bro -b --pseudo-realtime -r $TRACES/socks.trace %INPUT +# (uses listen.bro just to ensure input sources are more reliably fully-read). +# @TEST-SERIALIZE: comm +# +# @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: btest-bg-wait -k 5 # @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-sort btest-diff out @@ -13,6 +16,9 @@ 6 @TEST-END-FILE + +@load frameworks/communication/listen + global outfile: file; module A; diff --git a/testing/btest/scripts/base/frameworks/input/stream.bro b/testing/btest/scripts/base/frameworks/input/stream.bro index 684f4ea395..1ecd8a2eb0 100644 --- a/testing/btest/scripts/base/frameworks/input/stream.bro +++ b/testing/btest/scripts/base/frameworks/input/stream.bro @@ -1,5 +1,8 @@ +# (uses listen.bro just to ensure input sources are more reliably fully-read). +# @TEST-SERIALIZE: comm +# # @TEST-EXEC: cp input1.log input.log -# @TEST-EXEC: btest-bg-run bro bro -b --pseudo-realtime -r $TRACES/socks.trace %INPUT +# @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: sleep 3 # @TEST-EXEC: cat input2.log >> input.log # @TEST-EXEC: sleep 3 @@ -22,6 +25,7 @@ F -43 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz @TEST-END-FILE @load base/protocols/ssh +@load frameworks/communication/listen redef InputAscii::empty_field = "EMPTY"; diff --git a/testing/btest/scripts/base/frameworks/input/streamraw.bro b/testing/btest/scripts/base/frameworks/input/streamraw.bro index d8e43d6386..3bc06f7dea 100644 --- a/testing/btest/scripts/base/frameworks/input/streamraw.bro +++ b/testing/btest/scripts/base/frameworks/input/streamraw.bro @@ -1,5 +1,8 @@ +# (uses listen.bro just to ensure input sources are more reliably fully-read). +# @TEST-SERIALIZE: comm +# # @TEST-EXEC: cp input1.log input.log -# @TEST-EXEC: btest-bg-run bro bro -b --pseudo-realtime -r $TRACES/socks.trace %INPUT +# @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: sleep 3 # @TEST-EXEC: cat input2.log >> input.log # @TEST-EXEC: sleep 3 @@ -24,6 +27,8 @@ sdf 3rw43wRRERLlL#RWERERERE. @TEST-END-FILE +@load frameworks/communication/listen + module A; type Val: record { diff --git a/testing/btest/scripts/base/frameworks/input/subrecord-event.bro b/testing/btest/scripts/base/frameworks/input/subrecord-event.bro index 92ee6dd500..4e7dc1690a 100644 --- a/testing/btest/scripts/base/frameworks/input/subrecord-event.bro +++ b/testing/btest/scripts/base/frameworks/input/subrecord-event.bro @@ -1,4 +1,7 @@ -# @TEST-EXEC: btest-bg-run bro bro -b --pseudo-realtime -r $TRACES/socks.trace %INPUT +# (uses listen.bro just to ensure input sources are more reliably fully-read). +# @TEST-SERIALIZE: comm +# +# @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: btest-bg-wait -k 5 # @TEST-EXEC: btest-diff out @@ -11,6 +14,7 @@ T -42 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz @TEST-END-FILE @load base/protocols/ssh +@load frameworks/communication/listen global outfile: file; global try: count; diff --git a/testing/btest/scripts/base/frameworks/input/subrecord.bro b/testing/btest/scripts/base/frameworks/input/subrecord.bro index 11f247f764..512b8ec58f 100644 --- a/testing/btest/scripts/base/frameworks/input/subrecord.bro +++ b/testing/btest/scripts/base/frameworks/input/subrecord.bro @@ -1,4 +1,7 @@ -# @TEST-EXEC: btest-bg-run bro bro -b --pseudo-realtime -r $TRACES/socks.trace %INPUT +# (uses listen.bro just to ensure input sources are more reliably fully-read). +# @TEST-SERIALIZE: comm +# +# @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: btest-bg-wait -k 5 # @TEST-EXEC: btest-diff out @@ -11,6 +14,7 @@ T -42 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz @TEST-END-FILE @load base/protocols/ssh +@load frameworks/communication/listen global outfile: file; diff --git a/testing/btest/scripts/base/frameworks/input/tableevent.bro b/testing/btest/scripts/base/frameworks/input/tableevent.bro index db39fdf72b..723e519237 100644 --- a/testing/btest/scripts/base/frameworks/input/tableevent.bro +++ b/testing/btest/scripts/base/frameworks/input/tableevent.bro @@ -1,4 +1,7 @@ -# @TEST-EXEC: btest-bg-run bro bro -b --pseudo-realtime -r $TRACES/socks.trace %INPUT +# (uses listen.bro just to ensure input sources are more reliably fully-read). +# @TEST-SERIALIZE: comm +# +# @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: btest-bg-wait -k 5 # @TEST-EXEC: btest-diff out @@ -16,6 +19,8 @@ 7 T @TEST-END-FILE +@load frameworks/communication/listen + global outfile: file; global try: count; diff --git a/testing/btest/scripts/base/frameworks/input/twotables.bro b/testing/btest/scripts/base/frameworks/input/twotables.bro index 5540019971..83ae86cd46 100644 --- a/testing/btest/scripts/base/frameworks/input/twotables.bro +++ b/testing/btest/scripts/base/frameworks/input/twotables.bro @@ -1,5 +1,8 @@ +# (uses listen.bro just to ensure input sources are more reliably fully-read). +# @TEST-SERIALIZE: comm +# # @TEST-EXEC: cp input1.log input.log -# @TEST-EXEC: btest-bg-run bro bro -b --pseudo-realtime -r $TRACES/socks.trace %INPUT +# @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: sleep 5 # @TEST-EXEC: cp input3.log input.log # @TEST-EXEC: btest-bg-wait -k 10 @@ -31,6 +34,7 @@ F -44 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz @TEST-END-FILE @load base/protocols/ssh +@load frameworks/communication/listen redef InputAscii::empty_field = "EMPTY"; diff --git a/testing/btest/scripts/base/frameworks/input/unsupported_types.bro b/testing/btest/scripts/base/frameworks/input/unsupported_types.bro index 0651e0693e..e1350f61a9 100644 --- a/testing/btest/scripts/base/frameworks/input/unsupported_types.bro +++ b/testing/btest/scripts/base/frameworks/input/unsupported_types.bro @@ -1,4 +1,7 @@ -# @TEST-EXEC: btest-bg-run bro bro -b --pseudo-realtime -r $TRACES/socks.trace %INPUT +# (uses listen.bro just to ensure input sources are more reliably fully-read). +# @TEST-SERIALIZE: comm +# +# @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: btest-bg-wait -k 5 # @TEST-EXEC: btest-diff out @@ -11,6 +14,7 @@ whatever T -42 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.00 @TEST-END-FILE @load base/protocols/ssh +@load frameworks/communication/listen global outfile: file; From cce73a3aadc94215eb78b17d502cc5d66bb9a853 Mon Sep 17 00:00:00 2001 From: Gilbert Clark Date: Fri, 18 Jan 2013 14:23:34 -0500 Subject: [PATCH 30/43] Fixing variable size issues (was uint, should be long) with http response code. --- src/logging/writers/ElasticSearch.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/logging/writers/ElasticSearch.cc b/src/logging/writers/ElasticSearch.cc index ae825ac997..fccf33534e 100644 --- a/src/logging/writers/ElasticSearch.cc +++ b/src/logging/writers/ElasticSearch.cc @@ -402,7 +402,7 @@ bool ElasticSearch::HTTPSend(CURL *handle) case CURLE_OK: { - uint http_code = 0; + long http_code = 0; curl_easy_getinfo(curl_handle, CURLINFO_RESPONSE_CODE, &http_code); if ( http_code == 200 ) // Hopefully everything goes through here. From 4f39470c1bc37cce69c597ba5d5a28e1bb0548e1 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Fri, 18 Jan 2013 17:34:33 -0800 Subject: [PATCH 31/43] New option exit_only_after_terminate to prevent Bro from exiting. If set, the main loop won't terminate before somebody calls terminate(). This should make input framework testing more reliable I'd hope. --- scripts/base/init-bare.bro | 9 +++++++++ src/Net.cc | 3 ++- src/const.bif | 1 + 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/scripts/base/init-bare.bro b/scripts/base/init-bare.bro index e5365a9428..a01b52817c 100644 --- a/scripts/base/init-bare.bro +++ b/scripts/base/init-bare.bro @@ -2608,6 +2608,15 @@ const gap_report_freq = 1.0 sec &redef; ## .. bro:see:: content_gap gap_report partial_connection const report_gaps_for_partial = F &redef; +## Flag to prevent Bro from exiting automatically when input is exhausted. +## Normally Bro terminates when all packets sources have gone dry +## and communication isn't enabled. If this flag is set, Bro's main loop will +## instead keep idleing until :bro:see::`terminate` is explicitly called. +## +## This is mainly for testing purposes when termination behaviour needs to be +## controlled for reproducing results. +const exit_only_after_terminate = F &redef; + ## The CA certificate file to authorize remote Bros/Broccolis. ## ## .. bro:see:: ssl_private_key ssl_passphrase diff --git a/src/Net.cc b/src/Net.cc index 328998b011..73c618b8af 100644 --- a/src/Net.cc +++ b/src/Net.cc @@ -421,7 +421,8 @@ void net_run() set_processing_status("RUNNING", "net_run"); while ( io_sources.Size() || - (packet_sorter && ! packet_sorter->Empty()) ) + (packet_sorter && ! packet_sorter->Empty()) || + (BifConst::exit_only_after_terminate && ! terminating) ) { double ts; IOSource* src = io_sources.FindSoonest(&ts); diff --git a/src/const.bif b/src/const.bif index ea7dc03817..ea84b3363d 100644 --- a/src/const.bif +++ b/src/const.bif @@ -6,6 +6,7 @@ const ignore_keep_alive_rexmit: bool; const skip_http_data: bool; const use_conn_size_analyzer: bool; const report_gaps_for_partial: bool; +const exit_only_after_terminate: bool; const NFS3::return_data: bool; const NFS3::return_data_max: count; From c9412c427162d5c2003eba8114fd4911c7783072 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Wed, 23 Jan 2013 14:43:12 -0800 Subject: [PATCH 32/43] Updating submodule(s). [nomail] --- aux/btest | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aux/btest b/aux/btest index 3585dc9a7a..edf3ad17f2 160000 --- a/aux/btest +++ b/aux/btest @@ -1 +1 @@ -Subproject commit 3585dc9a7afe20d70cb77fc2472cc6bce3850b67 +Subproject commit edf3ad17f2d0e3c8892ffdfa306fe62354128fd8 From 0541c49a50d5b1682429bc3dd873fc375fb47378 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Wed, 23 Jan 2013 14:43:38 -0800 Subject: [PATCH 33/43] Changing btest call to use "-j" instead of "-j 5". --- testing/btest/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/testing/btest/Makefile b/testing/btest/Makefile index 93ccc8d5ec..ff63bdb601 100644 --- a/testing/btest/Makefile +++ b/testing/btest/Makefile @@ -6,13 +6,13 @@ all: cleanup btest-verbose coverage # Showing all tests. btest-verbose: - @$(BTEST) -j 5 -f $(DIAG) + @$(BTEST) -j -f $(DIAG) brief: cleanup btest-brief coverage # Brief output showing only failed tests. btest-brief: - @$(BTEST) -j 5 -b -f $(DIAG) + @$(BTEST) -j -b -f $(DIAG) coverage: @../scripts/coverage-calc ".tmp/script-coverage*" coverage.log `pwd`/../../scripts From b6c71f539089587f579f9e4a10b2763d152d789c Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Wed, 23 Jan 2013 18:11:49 -0800 Subject: [PATCH 34/43] Updating submodule(s). [nomail] --- aux/btest | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aux/btest b/aux/btest index edf3ad17f2..6c0f099c52 160000 --- a/aux/btest +++ b/aux/btest @@ -1 +1 @@ -Subproject commit edf3ad17f2d0e3c8892ffdfa306fe62354128fd8 +Subproject commit 6c0f099c52f9e7b5d6f3efafb63581a6cc0861a5 From ed165f22a248cd0cfc4e2af94b8c48ff4c4303aa Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Wed, 23 Jan 2013 19:43:27 -0800 Subject: [PATCH 35/43] Making a test portable. --- CHANGES | 4 ++++ VERSION | 2 +- testing/btest/language/switch-incomplete.bro | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/CHANGES b/CHANGES index ae5ae96a39..1029b7f639 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,8 @@ +2.1-304 | 2013-01-23 19:43:27 -0800 + + * Making a test portable. (Robin Sommer) + 2.1-302 | 2013-01-23 16:17:29 -0800 * Refactoring ASCII formatting/parsing from loggers/readers into a diff --git a/VERSION b/VERSION index 7ad39c568b..adf66ac1d3 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.1-302 +2.1-304 diff --git a/testing/btest/language/switch-incomplete.bro b/testing/btest/language/switch-incomplete.bro index c39f3f3688..7ee800b274 100644 --- a/testing/btest/language/switch-incomplete.bro +++ b/testing/btest/language/switch-incomplete.bro @@ -1,5 +1,5 @@ # @TEST-EXEC-FAIL: bro -b %INPUT >out 2>&1 -# @TEST-EXEC: btest-diff out +# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff out event bro_init() { From 69afc4a88269e497af4a9890f1a403e55694f934 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Thu, 24 Jan 2013 09:48:23 -0600 Subject: [PATCH 36/43] Add an error for record coercions that would orphan a field. These cases should be avoidable by fixing scripts where they occur and they can also help catch typos that would lead to unintentional runtime behavior. Adding this already revealed several scripts where a field in an inlined record was never removed after a code refactor. --- scripts/base/frameworks/signatures/main.bro | 8 +++---- scripts/base/protocols/http/file-hash.bro | 2 +- scripts/base/protocols/http/file-ident.bro | 4 +--- scripts/base/protocols/socks/main.bro | 2 +- scripts/policy/protocols/http/detect-MHR.bro | 2 +- src/Expr.cc | 7 ++++-- .../language.record-ceorce-orphan/out | 2 ++ .../btest/language/record-ceorce-orphan.bro | 22 +++++++++++++++++++ 8 files changed, 36 insertions(+), 13 deletions(-) create mode 100644 testing/btest/Baseline/language.record-ceorce-orphan/out create mode 100644 testing/btest/language/record-ceorce-orphan.bro diff --git a/scripts/base/frameworks/signatures/main.bro b/scripts/base/frameworks/signatures/main.bro index 26f78a68d1..4102214075 100644 --- a/scripts/base/frameworks/signatures/main.bro +++ b/scripts/base/frameworks/signatures/main.bro @@ -148,7 +148,7 @@ function has_signature_matched(id: string, orig: addr, resp: addr): bool event sig_summary(orig: addr, id: string, msg: string) { NOTICE([$note=Signature_Summary, $src=orig, - $filename=id, $msg=fmt("%s: %s", orig, msg), + $msg=fmt("%s: %s", orig, msg), $n=count_per_orig[orig,id] ]); } @@ -209,7 +209,6 @@ event signature_match(state: signature_state, msg: string, data: string) { NOTICE([$note=Count_Signature, $conn=state$conn, $msg=msg, - $filename=sig_id, $n=count_per_resp[dst,sig_id], $sub=fmt("%d matches of signature %s on host %s", count_per_resp[dst,sig_id], @@ -240,7 +239,7 @@ event signature_match(state: signature_state, msg: string, data: string) if ( notice ) NOTICE([$note=Sensitive_Signature, $conn=state$conn, $src=src_addr, - $dst=dst_addr, $filename=sig_id, $msg=fmt("%s: %s", src_addr, msg), + $dst=dst_addr, $msg=fmt("%s: %s", src_addr, msg), $sub=data]); if ( action == SIG_FILE_BUT_NO_SCAN || action == SIG_SUMMARY ) @@ -274,7 +273,7 @@ event signature_match(state: signature_state, msg: string, data: string) $src_addr=orig, $sig_id=sig_id, $event_msg=msg, $host_count=hcount, $sub_msg=horz_scan_msg]); - NOTICE([$note=Multiple_Sig_Responders, $src=orig, $filename=sig_id, + NOTICE([$note=Multiple_Sig_Responders, $src=orig, $msg=msg, $n=hcount, $sub=horz_scan_msg]); last_hthresh[orig] = hcount; @@ -295,7 +294,6 @@ event signature_match(state: signature_state, msg: string, data: string) $sub_msg=vert_scan_msg]); NOTICE([$note=Multiple_Signatures, $src=orig, $dst=resp, - $filename=sig_id, $msg=fmt("%s different signatures triggered", vcount), $n=vcount, $sub=vert_scan_msg]); diff --git a/scripts/base/protocols/http/file-hash.bro b/scripts/base/protocols/http/file-hash.bro index bc7547e51a..2545cbf817 100644 --- a/scripts/base/protocols/http/file-hash.bro +++ b/scripts/base/protocols/http/file-hash.bro @@ -73,7 +73,7 @@ event http_message_done(c: connection, is_orig: bool, stat: http_message_stat) & delete c$http$md5_handle; NOTICE([$note=MD5, $msg=fmt("%s %s %s", c$id$orig_h, c$http$md5, url), - $sub=c$http$md5, $conn=c, $URL=url]); + $sub=c$http$md5, $conn=c]); } } diff --git a/scripts/base/protocols/http/file-ident.bro b/scripts/base/protocols/http/file-ident.bro index b493f02bf0..706ea58558 100644 --- a/scripts/base/protocols/http/file-ident.bro +++ b/scripts/base/protocols/http/file-ident.bro @@ -68,9 +68,7 @@ event signature_match(state: signature_state, msg: string, data: string) &priori local message = fmt("%s %s %s", msg, c$http$method, url); NOTICE([$note=Incorrect_File_Type, $msg=message, - $conn=c, - $method=c$http$method, - $URL=url]); + $conn=c]); } } diff --git a/scripts/base/protocols/socks/main.bro b/scripts/base/protocols/socks/main.bro index 79ae4baa19..df5ee69f16 100644 --- a/scripts/base/protocols/socks/main.bro +++ b/scripts/base/protocols/socks/main.bro @@ -67,7 +67,7 @@ event socks_request(c: connection, version: count, request_type: count, # proxied connection. We treat this as a singular "tunnel". local cid = copy(c$id); cid$orig_p = 0/tcp; - Tunnel::register([$cid=cid, $tunnel_type=Tunnel::SOCKS, $payload_proxy=T]); + Tunnel::register([$cid=cid, $tunnel_type=Tunnel::SOCKS]); } event socks_reply(c: connection, version: count, reply: count, sa: SOCKS::Address, p: port) &priority=5 diff --git a/scripts/policy/protocols/http/detect-MHR.bro b/scripts/policy/protocols/http/detect-MHR.bro index 5abb98d39c..1898022978 100644 --- a/scripts/policy/protocols/http/detect-MHR.bro +++ b/scripts/policy/protocols/http/detect-MHR.bro @@ -37,7 +37,7 @@ event log_http(rec: HTTP::Info) local url = HTTP::build_url_http(rec); local message = fmt("%s %s %s", rec$id$orig_h, rec$md5, url); NOTICE([$note=Malware_Hash_Registry_Match, - $msg=message, $id=rec$id, $URL=url]); + $msg=message, $id=rec$id]); } } } diff --git a/src/Expr.cc b/src/Expr.cc index 9e71f27897..7b28d6f4df 100644 --- a/src/Expr.cc +++ b/src/Expr.cc @@ -3921,8 +3921,11 @@ RecordCoerceExpr::RecordCoerceExpr(Expr* op, RecordType* r) { int t_i = t_r->FieldOffset(sub_r->FieldName(i)); if ( t_i < 0 ) - // Orphane field in rhs, that's ok. - continue; + { + ExprError(fmt("orphaned field \"%s\" in record coercion", + sub_r->FieldName(i))); + break; + } BroType* sub_t_i = sub_r->FieldType(i); BroType* sup_t_i = t_r->FieldType(t_i); diff --git a/testing/btest/Baseline/language.record-ceorce-orphan/out b/testing/btest/Baseline/language.record-ceorce-orphan/out new file mode 100644 index 0000000000..aa42d13892 --- /dev/null +++ b/testing/btest/Baseline/language.record-ceorce-orphan/out @@ -0,0 +1,2 @@ +error in /Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/language.record-ceorce-orphan/record-ceorce-orphan.bro, line 19: orphaned field "wtf" in record coercion ((coerce [$a=test, $b=42, $wtf=1.0 sec] to record { a:string; b:count; c:interval; })) +error in /Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/language.record-ceorce-orphan/record-ceorce-orphan.bro, line 21: orphaned field "wtf" in record coercion ((coerce [$a=test, $b=42, $wtf=1.0 sec] to record { a:string; b:count; c:interval; })) diff --git a/testing/btest/language/record-ceorce-orphan.bro b/testing/btest/language/record-ceorce-orphan.bro new file mode 100644 index 0000000000..126b99d5ff --- /dev/null +++ b/testing/btest/language/record-ceorce-orphan.bro @@ -0,0 +1,22 @@ +# @TEST-EXEC-FAIL: bro -b %INPUT >out 2>&1 +# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff out + +type myrec: record { + a: string; + b: count; + c: interval &optional; +}; + +function myfunc(rec: myrec) + { + print rec; + } + +event bro_init() + { + # Orhpaned fields in a record coercion reflect a programming error, like a typo, so should + # be reported at parse-time to prevent unexpected run-time behavior. + local rec: myrec = [$a="test", $b=42, $wtf=1sec]; + print rec; + myfunc([$a="test", $b=42, $wtf=1sec]); + } From b72fbaf99fd8d40fe0bf38c81bf5f5921c762141 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Thu, 24 Jan 2013 10:26:54 -0600 Subject: [PATCH 37/43] Fix memory leak in some reporter messaging cases. Related to the changes in fdd11428. --- src/Reporter.cc | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/Reporter.cc b/src/Reporter.cc index fc3dde3ca0..6bc2577c72 100644 --- a/src/Reporter.cc +++ b/src/Reporter.cc @@ -320,6 +320,14 @@ void Reporter::DoLog(const char* prefix, EventHandlerPtr event, FILE* out, else mgr.QueueEvent(event, vl); } + else + { + if ( addl ) + { + loop_over_list(*addl, i) + Unref((*addl)[i]); + } + } if ( out ) { @@ -351,12 +359,6 @@ void Reporter::DoLog(const char* prefix, EventHandlerPtr event, FILE* out, if ( out ) fprintf(out, "%s", s.c_str()); - - if ( addl ) - { - loop_over_list(*addl, i) - Unref((*addl)[i]); - } } if ( alloced ) From 7d3b20d4fbe9f1bbe24aa249c2e4591feae4a381 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Fri, 25 Jan 2013 11:57:21 -0600 Subject: [PATCH 38/43] Fix runaway reference counting bug in record coercion. The RecordVal ctor refs the type arg via the MutableVal -> Val ctors, so this line was double incrementing the type's ref count, but could only decrement it once upon the Val's destruction. --- src/Expr.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Expr.cc b/src/Expr.cc index 9e71f27897..96ff79bb59 100644 --- a/src/Expr.cc +++ b/src/Expr.cc @@ -3967,7 +3967,7 @@ RecordCoerceExpr::~RecordCoerceExpr() Val* RecordCoerceExpr::Fold(Val* v) const { - RecordVal* val = new RecordVal(Type()->Ref()->AsRecordType()); + RecordVal* val = new RecordVal(Type()->AsRecordType()); RecordVal* rv = v->AsRecordVal(); for ( int i = 0; i < map_size; ++i ) From 8c807d19c37c6e719cb96394e11215e4894fc35c Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Wed, 30 Jan 2013 20:08:36 -0800 Subject: [PATCH 39/43] Fixing exit-after-terminate when used with bare mode. --- src/main.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main.cc b/src/main.cc index 5999186240..fffedee3dd 100644 --- a/src/main.cc +++ b/src/main.cc @@ -1053,7 +1053,7 @@ int main(int argc, char** argv) io_sources.Register(thread_mgr, true); - if ( io_sources.Size() > 0 || have_pending_timers ) + if ( io_sources.Size() > 0 || have_pending_timers || BifConst::exit_only_after_terminate ) { if ( profiling_logger ) profiling_logger->Log(); From b08bdbce46bdccbb8df5d4904ca87be4d1f36c9e Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Wed, 30 Jan 2013 20:09:19 -0800 Subject: [PATCH 40/43] Updating submodule(s). [nomail] --- aux/btest | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aux/btest b/aux/btest index 3585dc9a7a..ba0700fe44 160000 --- a/aux/btest +++ b/aux/btest @@ -1 +1 @@ -Subproject commit 3585dc9a7afe20d70cb77fc2472cc6bce3850b67 +Subproject commit ba0700fe448895b654b90d50f389f6f1341234cb From 01c2bf4e0e28f430940e10756fa47a1ef9ced9c0 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Wed, 30 Jan 2013 20:09:27 -0800 Subject: [PATCH 41/43] Updating submodule(s). [nomail] --- CHANGES | 8 ++++++++ VERSION | 2 +- aux/btest | 2 +- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/CHANGES b/CHANGES index 64b1eb0aa1..cc026a97eb 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,12 @@ +2.1-310 | 2013-01-30 20:09:27 -0800 + + * Add an error for record coercions that would orphan a field. (Jon + Siwek) + + * Fixing several scripts where a field in an inlined record was + never removed after a code refactor. (Jon Siwek) + 2.1-307 | 2013-01-25 13:50:57 -0800 * Fix runaway reference counting bug in record coercion. (Jon Siwek) diff --git a/VERSION b/VERSION index 3148d7e818..a04d961dd5 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.1-307 +2.1-310 diff --git a/aux/btest b/aux/btest index 6c0f099c52..ba0700fe44 160000 --- a/aux/btest +++ b/aux/btest @@ -1 +1 @@ -Subproject commit 6c0f099c52f9e7b5d6f3efafb63581a6cc0861a5 +Subproject commit ba0700fe448895b654b90d50f389f6f1341234cb From d3814594ff4db81b9dc5a45ec4f6419fe90fe972 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Fri, 1 Feb 2013 08:03:01 -0800 Subject: [PATCH 42/43] Updating submodule(s). [nomail] --- CHANGES | 4 ++++ VERSION | 2 +- aux/broccoli | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/CHANGES b/CHANGES index cc026a97eb..8b80bec68b 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,8 @@ +2.1-311 | 2013-02-01 08:03:01 -0800 + + * Updating submodule(s). + 2.1-310 | 2013-01-30 20:09:27 -0800 * Add an error for record coercions that would orphan a field. (Jon diff --git a/VERSION b/VERSION index a04d961dd5..44ee79325e 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.1-310 +2.1-311 diff --git a/aux/broccoli b/aux/broccoli index 073404dd29..c1ba9b44c4 160000 --- a/aux/broccoli +++ b/aux/broccoli @@ -1 +1 @@ -Subproject commit 073404dd29dc6e90ff0e4eb8bc836f8adbf3931e +Subproject commit c1ba9b44c4815c61c54c968f462ec5b0865e5990 From dd24d7af83abb68ccbf61488f246daf06e011972 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Fri, 1 Feb 2013 15:45:21 -0800 Subject: [PATCH 43/43] update input tests to use exit_only_after_terminate --- testing/btest/scripts/base/frameworks/input/basic.bro | 6 ++---- testing/btest/scripts/base/frameworks/input/bignumber.bro | 7 ++----- testing/btest/scripts/base/frameworks/input/binary.bro | 7 ++----- .../scripts/base/frameworks/input/empty-values-hashing.bro | 6 ++---- testing/btest/scripts/base/frameworks/input/emptyvals.bro | 5 +---- testing/btest/scripts/base/frameworks/input/event.bro | 5 +---- testing/btest/scripts/base/frameworks/input/executeraw.bro | 5 +---- .../btest/scripts/base/frameworks/input/invalidnumbers.bro | 5 +---- .../btest/scripts/base/frameworks/input/invalidtext.bro | 5 +---- .../btest/scripts/base/frameworks/input/missing-file.bro | 5 +---- .../scripts/base/frameworks/input/onecolumn-norecord.bro | 5 +---- .../scripts/base/frameworks/input/onecolumn-record.bro | 5 +---- testing/btest/scripts/base/frameworks/input/optional.bro | 5 +---- testing/btest/scripts/base/frameworks/input/port.bro | 5 +---- .../scripts/base/frameworks/input/predicate-stream.bro | 5 +---- testing/btest/scripts/base/frameworks/input/predicate.bro | 5 +---- .../scripts/base/frameworks/input/predicatemodify.bro | 5 +---- .../base/frameworks/input/predicatemodifyandreread.bro | 7 +++---- .../frameworks/input/predicaterefusesecondsamerecord.bro | 5 +---- testing/btest/scripts/base/frameworks/input/raw.bro | 5 +---- testing/btest/scripts/base/frameworks/input/repeat.bro | 5 +---- testing/btest/scripts/base/frameworks/input/reread.bro | 6 ++---- testing/btest/scripts/base/frameworks/input/rereadraw.bro | 5 +---- testing/btest/scripts/base/frameworks/input/set.bro | 5 +---- .../btest/scripts/base/frameworks/input/setseparator.bro | 5 +---- .../scripts/base/frameworks/input/setspecialcases.bro | 5 +---- testing/btest/scripts/base/frameworks/input/stream.bro | 5 +---- testing/btest/scripts/base/frameworks/input/streamraw.bro | 7 +++---- .../scripts/base/frameworks/input/subrecord-event.bro | 5 +---- testing/btest/scripts/base/frameworks/input/subrecord.bro | 5 +---- testing/btest/scripts/base/frameworks/input/tableevent.bro | 5 +---- testing/btest/scripts/base/frameworks/input/twotables.bro | 5 +---- .../scripts/base/frameworks/input/unsupported_types.bro | 5 +---- 33 files changed, 42 insertions(+), 134 deletions(-) diff --git a/testing/btest/scripts/base/frameworks/input/basic.bro b/testing/btest/scripts/base/frameworks/input/basic.bro index dfac84d062..ea6746c4db 100644 --- a/testing/btest/scripts/base/frameworks/input/basic.bro +++ b/testing/btest/scripts/base/frameworks/input/basic.bro @@ -1,10 +1,9 @@ -# (uses listen.bro just to ensure input sources are more reliably fully-read). -# @TEST-SERIALIZE: comm -# # @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: btest-bg-wait -k 5 # @TEST-EXEC: btest-diff out +redef exit_only_after_terminate = T; + @TEST-START-FILE input.log #separator \x09 #path ssh @@ -14,7 +13,6 @@ T -42 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz @TEST-END-FILE @load base/protocols/ssh -@load frameworks/communication/listen global outfile: file; diff --git a/testing/btest/scripts/base/frameworks/input/bignumber.bro b/testing/btest/scripts/base/frameworks/input/bignumber.bro index 5b93472551..19546d138c 100644 --- a/testing/btest/scripts/base/frameworks/input/bignumber.bro +++ b/testing/btest/scripts/base/frameworks/input/bignumber.bro @@ -1,10 +1,9 @@ -# (uses listen.bro just to ensure input sources are more reliably fully-read). -# @TEST-SERIALIZE: comm -# # @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: btest-bg-wait -k 5 # @TEST-EXEC: btest-diff out +redef exit_only_after_terminate = T; + @TEST-START-FILE input.log #separator \x09 #fields i c @@ -13,8 +12,6 @@ -9223372036854775800 18446744073709551612 @TEST-END-FILE -@load frameworks/communication/listen - global outfile: file; module A; diff --git a/testing/btest/scripts/base/frameworks/input/binary.bro b/testing/btest/scripts/base/frameworks/input/binary.bro index 8d75abc5a9..d8345dbfd3 100644 --- a/testing/btest/scripts/base/frameworks/input/binary.bro +++ b/testing/btest/scripts/base/frameworks/input/binary.bro @@ -1,10 +1,9 @@ -# (uses listen.bro just to ensure input sources are more reliably fully-read). -# @TEST-SERIALIZE: comm -# # @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: btest-bg-wait -k 5 # @TEST-EXEC: btest-diff out +redef exit_only_after_terminate = T; + redef InputAscii::separator = "|"; redef InputAscii::set_separator = ","; redef InputAscii::empty_field = "(empty)"; @@ -25,8 +24,6 @@ abc\xff\x7cdef|DATA2 #end|2012-07-20-01-49-19 @TEST-END-FILE -@load frameworks/communication/listen - global outfile: file; global try: count; diff --git a/testing/btest/scripts/base/frameworks/input/empty-values-hashing.bro b/testing/btest/scripts/base/frameworks/input/empty-values-hashing.bro index c8760b467e..5226586ad1 100644 --- a/testing/btest/scripts/base/frameworks/input/empty-values-hashing.bro +++ b/testing/btest/scripts/base/frameworks/input/empty-values-hashing.bro @@ -1,6 +1,3 @@ -# (uses listen.bro just to ensure input sources are more reliably fully-read). -# @TEST-SERIALIZE: comm -# # @TEST-EXEC: cp input1.log input.log # @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: sleep 2 @@ -23,8 +20,9 @@ 2 TEST TEST @TEST-END-FILE -@load frameworks/communication/listen +@load base/frameworks/communication # let network-time run +redef exit_only_after_terminate = T; module A; diff --git a/testing/btest/scripts/base/frameworks/input/emptyvals.bro b/testing/btest/scripts/base/frameworks/input/emptyvals.bro index 94b0f1b620..cef0606646 100644 --- a/testing/btest/scripts/base/frameworks/input/emptyvals.bro +++ b/testing/btest/scripts/base/frameworks/input/emptyvals.bro @@ -1,6 +1,3 @@ -# (uses listen.bro just to ensure input sources are more reliably fully-read). -# @TEST-SERIALIZE: comm -# # @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: btest-bg-wait -k 5 # @TEST-EXEC: btest-diff out @@ -14,7 +11,7 @@ T 1 - 2 @TEST-END-FILE -@load frameworks/communication/listen +redef exit_only_after_terminate = T; global outfile: file; diff --git a/testing/btest/scripts/base/frameworks/input/event.bro b/testing/btest/scripts/base/frameworks/input/event.bro index ba47d5e3f2..ac956a4a19 100644 --- a/testing/btest/scripts/base/frameworks/input/event.bro +++ b/testing/btest/scripts/base/frameworks/input/event.bro @@ -1,6 +1,3 @@ -# (uses listen.bro just to ensure input sources are more reliably fully-read). -# @TEST-SERIALIZE: comm -# # @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: btest-bg-wait -k 5 # @TEST-EXEC: btest-diff out @@ -19,7 +16,7 @@ 7 T @TEST-END-FILE -@load frameworks/communication/listen +redef exit_only_after_terminate = T; global outfile: file; diff --git a/testing/btest/scripts/base/frameworks/input/executeraw.bro b/testing/btest/scripts/base/frameworks/input/executeraw.bro index 626b9cdfd2..bcec76444f 100644 --- a/testing/btest/scripts/base/frameworks/input/executeraw.bro +++ b/testing/btest/scripts/base/frameworks/input/executeraw.bro @@ -1,6 +1,3 @@ -# (uses listen.bro just to ensure input sources are more reliably fully-read). -# @TEST-SERIALIZE: comm -# # @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: btest-bg-wait -k 5 # @TEST-EXEC: cat out.tmp | sed 's/^ *//g' >out @@ -17,7 +14,7 @@ sdf 3rw43wRRERLlL#RWERERERE. @TEST-END-FILE -@load frameworks/communication/listen +redef exit_only_after_terminate = T; global outfile: file; diff --git a/testing/btest/scripts/base/frameworks/input/invalidnumbers.bro b/testing/btest/scripts/base/frameworks/input/invalidnumbers.bro index 1deec605ae..3321b41253 100644 --- a/testing/btest/scripts/base/frameworks/input/invalidnumbers.bro +++ b/testing/btest/scripts/base/frameworks/input/invalidnumbers.bro @@ -1,6 +1,3 @@ -# (uses listen.bro just to ensure input sources are more reliably fully-read). -# @TEST-SERIALIZE: comm -# # @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: btest-bg-wait -k 5 # @TEST-EXEC: btest-diff out @@ -17,7 +14,7 @@ Justtext 1 9223372036854775800 -18446744073709551612 @TEST-END-FILE -@load frameworks/communication/listen +redef exit_only_after_terminate = T; global outfile: file; diff --git a/testing/btest/scripts/base/frameworks/input/invalidtext.bro b/testing/btest/scripts/base/frameworks/input/invalidtext.bro index 75efb1247d..ffca41dbee 100644 --- a/testing/btest/scripts/base/frameworks/input/invalidtext.bro +++ b/testing/btest/scripts/base/frameworks/input/invalidtext.bro @@ -1,6 +1,3 @@ -# (uses listen.bro just to ensure input sources are more reliably fully-read). -# @TEST-SERIALIZE: comm -# # @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: btest-bg-wait -k 5 # @TEST-EXEC: btest-diff out @@ -15,7 +12,7 @@ 5 @TEST-END-FILE -@load frameworks/communication/listen +redef exit_only_after_terminate = T; global outfile: file; diff --git a/testing/btest/scripts/base/frameworks/input/missing-file.bro b/testing/btest/scripts/base/frameworks/input/missing-file.bro index aa5acf619e..8fb12ba412 100644 --- a/testing/btest/scripts/base/frameworks/input/missing-file.bro +++ b/testing/btest/scripts/base/frameworks/input/missing-file.bro @@ -1,11 +1,8 @@ -# (uses listen.bro just to ensure input sources are more reliably fully-read). -# @TEST-SERIALIZE: comm -# # @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: btest-bg-wait -k 5 # @TEST-EXEC: btest-diff bro/.stderr -@load frameworks/communication/listen +redef exit_only_after_terminate = T; global outfile: file; global try: count; diff --git a/testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro b/testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro index c08b1420fb..989d6352a3 100644 --- a/testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro +++ b/testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro @@ -1,6 +1,3 @@ -# (uses listen.bro just to ensure input sources are more reliably fully-read). -# @TEST-SERIALIZE: comm -# # @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: btest-bg-wait -k 5 # @TEST-EXEC: btest-diff out @@ -13,7 +10,7 @@ T -42 @TEST-END-FILE -@load frameworks/communication/listen +redef exit_only_after_terminate = T; global outfile: file; diff --git a/testing/btest/scripts/base/frameworks/input/onecolumn-record.bro b/testing/btest/scripts/base/frameworks/input/onecolumn-record.bro index 9e420e75fe..54c8e9007e 100644 --- a/testing/btest/scripts/base/frameworks/input/onecolumn-record.bro +++ b/testing/btest/scripts/base/frameworks/input/onecolumn-record.bro @@ -1,6 +1,3 @@ -# (uses listen.bro just to ensure input sources are more reliably fully-read). -# @TEST-SERIALIZE: comm -# # @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: btest-bg-wait -k 5 # @TEST-EXEC: btest-diff out @@ -13,7 +10,7 @@ T -42 @TEST-END-FILE -@load frameworks/communication/listen +redef exit_only_after_terminate = T; global outfile: file; diff --git a/testing/btest/scripts/base/frameworks/input/optional.bro b/testing/btest/scripts/base/frameworks/input/optional.bro index 2fe0e5c86f..1fb9bce68b 100644 --- a/testing/btest/scripts/base/frameworks/input/optional.bro +++ b/testing/btest/scripts/base/frameworks/input/optional.bro @@ -1,6 +1,3 @@ -# (uses listen.bro just to ensure input sources are more reliably fully-read). -# @TEST-SERIALIZE: comm -# # @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: btest-bg-wait -k 5 # @TEST-EXEC: btest-diff out @@ -19,7 +16,7 @@ 7 T @TEST-END-FILE -@load frameworks/communication/listen +redef exit_only_after_terminate = T; global outfile: file; diff --git a/testing/btest/scripts/base/frameworks/input/port.bro b/testing/btest/scripts/base/frameworks/input/port.bro index 081c59559b..31776c5939 100644 --- a/testing/btest/scripts/base/frameworks/input/port.bro +++ b/testing/btest/scripts/base/frameworks/input/port.bro @@ -1,6 +1,3 @@ -# (uses listen.bro just to ensure input sources are more reliably fully-read). -# @TEST-SERIALIZE: comm -# # @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: btest-bg-wait -k 5 # @TEST-EXEC: btest-diff out @@ -12,7 +9,7 @@ 1.2.3.6 30 unknown @TEST-END-FILE -@load frameworks/communication/listen +redef exit_only_after_terminate = T; global outfile: file; diff --git a/testing/btest/scripts/base/frameworks/input/predicate-stream.bro b/testing/btest/scripts/base/frameworks/input/predicate-stream.bro index 8cf927e346..8caea9ad7a 100644 --- a/testing/btest/scripts/base/frameworks/input/predicate-stream.bro +++ b/testing/btest/scripts/base/frameworks/input/predicate-stream.bro @@ -1,6 +1,3 @@ -# (uses listen.bro just to ensure input sources are more reliably fully-read). -# @TEST-SERIALIZE: comm -# # @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: btest-bg-wait -k 5 # @TEST-EXEC: btest-diff out @@ -23,7 +20,7 @@ 7 T @TEST-END-FILE -@load frameworks/communication/listen +redef exit_only_after_terminate = T; global outfile: file; diff --git a/testing/btest/scripts/base/frameworks/input/predicate.bro b/testing/btest/scripts/base/frameworks/input/predicate.bro index 8fb33242e8..f9ff5c09ee 100644 --- a/testing/btest/scripts/base/frameworks/input/predicate.bro +++ b/testing/btest/scripts/base/frameworks/input/predicate.bro @@ -1,6 +1,3 @@ -# (uses listen.bro just to ensure input sources are more reliably fully-read). -# @TEST-SERIALIZE: comm -# # @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: btest-bg-wait -k 5 # @TEST-EXEC: btest-diff out @@ -19,7 +16,7 @@ 7 T @TEST-END-FILE -@load frameworks/communication/listen +redef exit_only_after_terminate = T; global outfile: file; diff --git a/testing/btest/scripts/base/frameworks/input/predicatemodify.bro b/testing/btest/scripts/base/frameworks/input/predicatemodify.bro index 17467bbc27..9d5eb3bd07 100644 --- a/testing/btest/scripts/base/frameworks/input/predicatemodify.bro +++ b/testing/btest/scripts/base/frameworks/input/predicatemodify.bro @@ -1,6 +1,3 @@ -# (uses listen.bro just to ensure input sources are more reliably fully-read). -# @TEST-SERIALIZE: comm -# # @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: btest-bg-wait -k 5 # @TEST-EXEC: btest-diff out @@ -14,7 +11,7 @@ 2 T test2 idx2 @TEST-END-FILE -@load frameworks/communication/listen +redef exit_only_after_terminate = T; global outfile: file; diff --git a/testing/btest/scripts/base/frameworks/input/predicatemodifyandreread.bro b/testing/btest/scripts/base/frameworks/input/predicatemodifyandreread.bro index 5a9e993651..174d145054 100644 --- a/testing/btest/scripts/base/frameworks/input/predicatemodifyandreread.bro +++ b/testing/btest/scripts/base/frameworks/input/predicatemodifyandreread.bro @@ -1,6 +1,3 @@ -# (uses listen.bro just to ensure input sources are more reliably fully-read). -# @TEST-SERIALIZE: comm -# # @TEST-EXEC: cp input1.log input.log # @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: sleep 2 @@ -58,7 +55,9 @@ 1 T test1 idx1 @TEST-END-FILE -@load frameworks/communication/listen +redef exit_only_after_terminate = T; + +@load base/frameworks/communication # let network-time run redef InputAscii::empty_field = "EMPTY"; diff --git a/testing/btest/scripts/base/frameworks/input/predicaterefusesecondsamerecord.bro b/testing/btest/scripts/base/frameworks/input/predicaterefusesecondsamerecord.bro index ba0b468cdc..247b301bfa 100644 --- a/testing/btest/scripts/base/frameworks/input/predicaterefusesecondsamerecord.bro +++ b/testing/btest/scripts/base/frameworks/input/predicaterefusesecondsamerecord.bro @@ -1,6 +1,3 @@ -# (uses listen.bro just to ensure input sources are more reliably fully-read). -# @TEST-SERIALIZE: comm -# # @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: btest-bg-wait -k 5 # @TEST-EXEC: btest-diff out @@ -17,7 +14,7 @@ need-to-know 8c864306-d21a-37b1-8705-746a786719bf medium 95 1342569600 1.228.83. need-to-know 8c864306-d21a-37b1-8705-746a786719bf medium 65 1342656000 1.228.83.33 - - 9318 HANARO-AS Hanaro Telecom Inc. 1.224.0.0/13 apnic KR spam infrastructure spamming;malware domain public http://reputation.alienvault.com/reputation.generic @TEST-END-FILE -@load frameworks/communication/listen +redef exit_only_after_terminate = T; global outfile: file; diff --git a/testing/btest/scripts/base/frameworks/input/raw.bro b/testing/btest/scripts/base/frameworks/input/raw.bro index d15aec22bb..0c4a438549 100644 --- a/testing/btest/scripts/base/frameworks/input/raw.bro +++ b/testing/btest/scripts/base/frameworks/input/raw.bro @@ -1,6 +1,3 @@ -# (uses listen.bro just to ensure input sources are more reliably fully-read). -# @TEST-SERIALIZE: comm -# # @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: btest-bg-wait -k 5 # @TEST-EXEC: btest-diff out @@ -16,7 +13,7 @@ sdf 3rw43wRRERLlL#RWERERERE. @TEST-END-FILE -@load frameworks/communication/listen +redef exit_only_after_terminate = T; global outfile: file; global try: count; diff --git a/testing/btest/scripts/base/frameworks/input/repeat.bro b/testing/btest/scripts/base/frameworks/input/repeat.bro index a966ac064e..08c7ab1592 100644 --- a/testing/btest/scripts/base/frameworks/input/repeat.bro +++ b/testing/btest/scripts/base/frameworks/input/repeat.bro @@ -1,6 +1,3 @@ -# (uses listen.bro just to ensure input sources are more reliably fully-read). -# @TEST-SERIALIZE: comm -# # @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: btest-bg-wait -k 5 # @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-sort btest-diff out @@ -13,7 +10,7 @@ 1 T @TEST-END-FILE -@load frameworks/communication/listen +redef exit_only_after_terminate = T; global outfile: file; global try: count; diff --git a/testing/btest/scripts/base/frameworks/input/reread.bro b/testing/btest/scripts/base/frameworks/input/reread.bro index 11aa873f9d..e7639d3e48 100644 --- a/testing/btest/scripts/base/frameworks/input/reread.bro +++ b/testing/btest/scripts/base/frameworks/input/reread.bro @@ -1,6 +1,3 @@ -# (uses listen.bro just to ensure input sources are more reliably fully-read). -# @TEST-SERIALIZE: comm -# # @TEST-EXEC: cp input1.log input.log # @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: sleep 2 @@ -59,8 +56,9 @@ F -48 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz @TEST-END-FILE @load base/protocols/ssh -@load frameworks/communication/listen +@load base/frameworks/communication # let network-time run +redef exit_only_after_terminate = T; redef InputAscii::empty_field = "EMPTY"; module A; diff --git a/testing/btest/scripts/base/frameworks/input/rereadraw.bro b/testing/btest/scripts/base/frameworks/input/rereadraw.bro index 2fdcdc8f9e..16f13c21e1 100644 --- a/testing/btest/scripts/base/frameworks/input/rereadraw.bro +++ b/testing/btest/scripts/base/frameworks/input/rereadraw.bro @@ -1,6 +1,3 @@ -# (uses listen.bro just to ensure input sources are more reliably fully-read). -# @TEST-SERIALIZE: comm -# # @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: btest-bg-wait -k 5 # @TEST-EXEC: btest-diff out @@ -16,7 +13,7 @@ sdf 3rw43wRRERLlL#RWERERERE. @TEST-END-FILE -@load frameworks/communication/listen +redef exit_only_after_terminate = T; global outfile: file; global try: count; diff --git a/testing/btest/scripts/base/frameworks/input/set.bro b/testing/btest/scripts/base/frameworks/input/set.bro index b2b5cea323..95a7a86a28 100644 --- a/testing/btest/scripts/base/frameworks/input/set.bro +++ b/testing/btest/scripts/base/frameworks/input/set.bro @@ -1,6 +1,3 @@ -# (uses listen.bro just to ensure input sources are more reliably fully-read). -# @TEST-SERIALIZE: comm -# # @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: btest-bg-wait -k 5 # @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-sort btest-diff out @@ -16,7 +13,7 @@ 192.168.17.42 @TEST-END-FILE -@load frameworks/communication/listen +redef exit_only_after_terminate = T; global outfile: file; diff --git a/testing/btest/scripts/base/frameworks/input/setseparator.bro b/testing/btest/scripts/base/frameworks/input/setseparator.bro index b7148d80bd..efc4b259f6 100644 --- a/testing/btest/scripts/base/frameworks/input/setseparator.bro +++ b/testing/btest/scripts/base/frameworks/input/setseparator.bro @@ -1,6 +1,3 @@ -# (uses listen.bro just to ensure input sources are more reliably fully-read). -# @TEST-SERIALIZE: comm -# # @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: btest-bg-wait -k 5 # @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-sort btest-diff out @@ -13,7 +10,7 @@ redef InputAscii::set_separator = "|"; -@load frameworks/communication/listen +redef exit_only_after_terminate = T; global outfile: file; diff --git a/testing/btest/scripts/base/frameworks/input/setspecialcases.bro b/testing/btest/scripts/base/frameworks/input/setspecialcases.bro index 022eac9731..27211a590e 100644 --- a/testing/btest/scripts/base/frameworks/input/setspecialcases.bro +++ b/testing/btest/scripts/base/frameworks/input/setspecialcases.bro @@ -1,6 +1,3 @@ -# (uses listen.bro just to ensure input sources are more reliably fully-read). -# @TEST-SERIALIZE: comm -# # @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: btest-bg-wait -k 5 # @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-sort btest-diff out @@ -17,7 +14,7 @@ @TEST-END-FILE -@load frameworks/communication/listen +redef exit_only_after_terminate = T; global outfile: file; diff --git a/testing/btest/scripts/base/frameworks/input/stream.bro b/testing/btest/scripts/base/frameworks/input/stream.bro index 1ecd8a2eb0..1d797f30d3 100644 --- a/testing/btest/scripts/base/frameworks/input/stream.bro +++ b/testing/btest/scripts/base/frameworks/input/stream.bro @@ -1,6 +1,3 @@ -# (uses listen.bro just to ensure input sources are more reliably fully-read). -# @TEST-SERIALIZE: comm -# # @TEST-EXEC: cp input1.log input.log # @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: sleep 3 @@ -25,7 +22,7 @@ F -43 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz @TEST-END-FILE @load base/protocols/ssh -@load frameworks/communication/listen +redef exit_only_after_terminate = T; redef InputAscii::empty_field = "EMPTY"; diff --git a/testing/btest/scripts/base/frameworks/input/streamraw.bro b/testing/btest/scripts/base/frameworks/input/streamraw.bro index 3bc06f7dea..a7cb718975 100644 --- a/testing/btest/scripts/base/frameworks/input/streamraw.bro +++ b/testing/btest/scripts/base/frameworks/input/streamraw.bro @@ -1,6 +1,3 @@ -# (uses listen.bro just to ensure input sources are more reliably fully-read). -# @TEST-SERIALIZE: comm -# # @TEST-EXEC: cp input1.log input.log # @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: sleep 3 @@ -10,6 +7,8 @@ # @TEST-EXEC: btest-bg-wait -k 5 # @TEST-EXEC: btest-diff out +redef exit_only_after_terminate = T; + @TEST-START-FILE input1.log sdfkh:KH;fdkncv;ISEUp34:Fkdj;YVpIODhfDF @TEST-END-FILE @@ -27,7 +26,7 @@ sdf 3rw43wRRERLlL#RWERERERE. @TEST-END-FILE -@load frameworks/communication/listen +@load base/frameworks/communication # let network-time run module A; diff --git a/testing/btest/scripts/base/frameworks/input/subrecord-event.bro b/testing/btest/scripts/base/frameworks/input/subrecord-event.bro index 4e7dc1690a..96774f9c29 100644 --- a/testing/btest/scripts/base/frameworks/input/subrecord-event.bro +++ b/testing/btest/scripts/base/frameworks/input/subrecord-event.bro @@ -1,6 +1,3 @@ -# (uses listen.bro just to ensure input sources are more reliably fully-read). -# @TEST-SERIALIZE: comm -# # @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: btest-bg-wait -k 5 # @TEST-EXEC: btest-diff out @@ -14,7 +11,7 @@ T -42 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz @TEST-END-FILE @load base/protocols/ssh -@load frameworks/communication/listen +redef exit_only_after_terminate = T; global outfile: file; global try: count; diff --git a/testing/btest/scripts/base/frameworks/input/subrecord.bro b/testing/btest/scripts/base/frameworks/input/subrecord.bro index 512b8ec58f..6f6c286828 100644 --- a/testing/btest/scripts/base/frameworks/input/subrecord.bro +++ b/testing/btest/scripts/base/frameworks/input/subrecord.bro @@ -1,6 +1,3 @@ -# (uses listen.bro just to ensure input sources are more reliably fully-read). -# @TEST-SERIALIZE: comm -# # @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: btest-bg-wait -k 5 # @TEST-EXEC: btest-diff out @@ -14,7 +11,7 @@ T -42 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz @TEST-END-FILE @load base/protocols/ssh -@load frameworks/communication/listen +redef exit_only_after_terminate = T; global outfile: file; diff --git a/testing/btest/scripts/base/frameworks/input/tableevent.bro b/testing/btest/scripts/base/frameworks/input/tableevent.bro index 723e519237..a409803440 100644 --- a/testing/btest/scripts/base/frameworks/input/tableevent.bro +++ b/testing/btest/scripts/base/frameworks/input/tableevent.bro @@ -1,6 +1,3 @@ -# (uses listen.bro just to ensure input sources are more reliably fully-read). -# @TEST-SERIALIZE: comm -# # @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: btest-bg-wait -k 5 # @TEST-EXEC: btest-diff out @@ -19,7 +16,7 @@ 7 T @TEST-END-FILE -@load frameworks/communication/listen +redef exit_only_after_terminate = T; global outfile: file; global try: count; diff --git a/testing/btest/scripts/base/frameworks/input/twotables.bro b/testing/btest/scripts/base/frameworks/input/twotables.bro index 83ae86cd46..79b33f7182 100644 --- a/testing/btest/scripts/base/frameworks/input/twotables.bro +++ b/testing/btest/scripts/base/frameworks/input/twotables.bro @@ -1,6 +1,3 @@ -# (uses listen.bro just to ensure input sources are more reliably fully-read). -# @TEST-SERIALIZE: comm -# # @TEST-EXEC: cp input1.log input.log # @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: sleep 5 @@ -34,7 +31,7 @@ F -44 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz @TEST-END-FILE @load base/protocols/ssh -@load frameworks/communication/listen +redef exit_only_after_terminate = T; redef InputAscii::empty_field = "EMPTY"; diff --git a/testing/btest/scripts/base/frameworks/input/unsupported_types.bro b/testing/btest/scripts/base/frameworks/input/unsupported_types.bro index e1350f61a9..937c963f27 100644 --- a/testing/btest/scripts/base/frameworks/input/unsupported_types.bro +++ b/testing/btest/scripts/base/frameworks/input/unsupported_types.bro @@ -1,6 +1,3 @@ -# (uses listen.bro just to ensure input sources are more reliably fully-read). -# @TEST-SERIALIZE: comm -# # @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: btest-bg-wait -k 5 # @TEST-EXEC: btest-diff out @@ -14,7 +11,7 @@ whatever T -42 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.00 @TEST-END-FILE @load base/protocols/ssh -@load frameworks/communication/listen +redef exit_only_after_terminate = T; global outfile: file;