From e8418ad5b0fae0b9ef5a797f1484be85c97f02b7 Mon Sep 17 00:00:00 2001 From: Johanna Amann Date: Wed, 1 Jun 2016 11:20:14 -0700 Subject: [PATCH] Ascii Input: Accept dos/windows newlines. The ascii reader now accepts \r\n newlines without complaining. Furthermore, the reader was slightly rewritten in a more c++11-y way, removing all raw pointers from the class. Addresses BIT-1198 --- src/input/readers/ascii/Ascii.cc | 42 ++++-------- src/input/readers/ascii/Ascii.h | 22 ++++--- .../scripts.base.frameworks.input.windows/out | 15 +++++ .../scripts/base/frameworks/input/windows.bro | 64 +++++++++++++++++++ 4 files changed, 107 insertions(+), 36 deletions(-) create mode 100644 testing/btest/Baseline/scripts.base.frameworks.input.windows/out create mode 100644 testing/btest/scripts/base/frameworks/input/windows.bro diff --git a/src/input/readers/ascii/Ascii.cc b/src/input/readers/ascii/Ascii.cc index 1bbcaea1d9..7b6d6d70bd 100644 --- a/src/input/readers/ascii/Ascii.cc +++ b/src/input/readers/ascii/Ascii.cc @@ -1,6 +1,5 @@ // See the file "COPYING" in the main distribution directory for copyright. -#include #include #include @@ -49,25 +48,15 @@ FieldMapping FieldMapping::subType() Ascii::Ascii(ReaderFrontend *frontend) : ReaderBackend(frontend) { - file = 0; mtime = 0; - formatter = 0; } Ascii::~Ascii() { - DoClose(); - delete formatter; } void Ascii::DoClose() { - if ( file != 0 ) - { - file->close(); - delete(file); - file = 0; - } } bool Ascii::DoInit(const ReaderInfo& info, int num_fields, const Field* const* fields) @@ -107,23 +96,19 @@ bool Ascii::DoInit(const ReaderInfo& info, int num_fields, const Field* const* f Error("set_separator length has to be 1. Separator will be truncated."); formatter::Ascii::SeparatorInfo sep_info(separator, set_separator, unset_field, empty_field); - formatter = new formatter::Ascii(this, sep_info); + formatter = unique_ptr(new formatter::Ascii(this, sep_info)); - file = new ifstream(info.source); - if ( ! file->is_open() ) + file.open(info.source); + if ( ! file.is_open() ) { Error(Fmt("Init: cannot open %s", info.source)); - delete(file); - file = 0; return false; } if ( ReadHeader(false) == false ) { Error(Fmt("Init: cannot open %s; headers are incorrect", info.source)); - file->close(); - delete(file); - file = 0; + file.close(); return false; } @@ -215,8 +200,11 @@ bool Ascii::ReadHeader(bool useCached) bool Ascii::GetLine(string& str) { - while ( getline(*file, str) ) + while ( getline(file, str) ) { + if ( str.back() == '\r' ) // deal with \r\n by removing \r + str.pop_back(); + if ( str[0] != '#' ) return true; @@ -258,24 +246,22 @@ bool Ascii::DoUpdate() { // dirty, fix me. (well, apparently after trying seeking, etc // - this is not that bad) - if ( file && file->is_open() ) + if ( file.is_open() ) { if ( Info().mode == MODE_STREAM ) { - file->clear(); // remove end of file evil bits + file.clear(); // remove end of file evil bits if ( !ReadHeader(true) ) return false; // header reading failed break; } - file->close(); - delete file; - file = 0; + file.close(); } - file = new ifstream(Info().source); - if ( ! file->is_open() ) + file.open(Info().source); + if ( ! file.is_open() ) { Error(Fmt("cannot open %s", Info().source)); return false; @@ -296,7 +282,7 @@ bool Ascii::DoUpdate() string line; - file->sync(); + file.sync(); while ( GetLine(line) ) { diff --git a/src/input/readers/ascii/Ascii.h b/src/input/readers/ascii/Ascii.h index fe9bb95845..d68267a392 100644 --- a/src/input/readers/ascii/Ascii.h +++ b/src/input/readers/ascii/Ascii.h @@ -5,6 +5,7 @@ #include #include +#include #include "input/ReaderBackend.h" #include "threading/formatters/Ascii.h" @@ -33,23 +34,28 @@ struct FieldMapping { */ class Ascii : public ReaderBackend { public: - Ascii(ReaderFrontend* frontend); + explicit Ascii(ReaderFrontend* frontend); ~Ascii(); + // prohibit copying and moving + Ascii(const Ascii&) = delete; + Ascii(Ascii&&) = delete; + Ascii& operator=(const Ascii&) = delete; + Ascii& operator=(Ascii&&) = delete; + static ReaderBackend* Instantiate(ReaderFrontend* frontend) { return new Ascii(frontend); } protected: - virtual bool DoInit(const ReaderInfo& info, int arg_num_fields, const threading::Field* const* fields); - virtual void DoClose(); - virtual bool DoUpdate(); - virtual bool DoHeartbeat(double network_time, double current_time); + bool DoInit(const ReaderInfo& info, int arg_num_fields, const threading::Field* const* fields) override; + void DoClose() override; + bool DoUpdate() override; + bool DoHeartbeat(double network_time, double current_time) override; private: - bool ReadHeader(bool useCached); bool GetLine(string& str); - ifstream* file; + ifstream file; time_t mtime; // map columns in the file to columns to send back to the manager @@ -64,7 +70,7 @@ private: string empty_field; string unset_field; - threading::formatter::Formatter* formatter; + std::unique_ptr formatter; }; diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.windows/out b/testing/btest/Baseline/scripts.base.frameworks.input.windows/out new file mode 100644 index 0000000000..c456298062 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.windows/out @@ -0,0 +1,15 @@ +{ +[-42] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, ns=4242, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +} +4242 diff --git a/testing/btest/scripts/base/frameworks/input/windows.bro b/testing/btest/scripts/base/frameworks/input/windows.bro new file mode 100644 index 0000000000..275f5e0713 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/input/windows.bro @@ -0,0 +1,64 @@ +# Test windows linebreaks + +# @TEST-EXEC: btest-bg-run bro bro -b %INPUT +# @TEST-EXEC: btest-bg-wait 10 +# @TEST-EXEC: btest-diff out + +redef exit_only_after_terminate = T; + +@TEST-START-FILE input.log +#separator \x09 +#path ssh +#fields b i e c p sn a d t iv s sc ss se vc ve ns +#types bool int enum count port subnet addr double time interval string table table table vector vector string +T -42 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY 4242 +@TEST-END-FILE + +@load base/protocols/ssh + +global outfile: file; + +redef InputAscii::empty_field = "EMPTY"; + +module A; + +type Idx: record { + i: int; +}; + +type Val: record { + b: bool; + e: Log::ID; + c: count; + p: port; + sn: subnet; + a: addr; + d: double; + t: time; + iv: interval; + s: string; + ns: string; + sc: set[count]; + ss: set[string]; + se: set[string]; + vc: vector of int; + ve: vector of int; +}; + +global servers: table[int] of Val = table(); + +event bro_init() + { + outfile = open("../out"); + # first read in the old stuff into the table... + Input::add_table([$source="../input.log", $name="ssh", $idx=Idx, $val=Val, $destination=servers]); + } + +event Input::end_of_data(name: string, source:string) + { + print outfile, servers; + print outfile, to_count(servers[-42]$ns); # try to actually use a string. If null-termination is wrong this will fail. + Input::remove("ssh"); + close(outfile); + terminate(); + }