From 124c985d7af91a98eb8a7aff8f66b0300849e854 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Sun, 26 Aug 2012 14:49:37 -0700 Subject: [PATCH 1/6] Bug found bei Keith & Seth: input framework was not handling counts and ints out of 32-bit-range correctly. Note - another bugfix will be coming later (problem reading sets containing zero-length-strings & un-escaping-bug in sets) --- src/input/readers/Ascii.cc | 6 +-- .../out | 3 ++ .../base/frameworks/input/bignumber.bro | 44 +++++++++++++++++++ 3 files changed, 50 insertions(+), 3 deletions(-) create mode 100644 testing/btest/Baseline/scripts.base.frameworks.input.bignumber/out create mode 100644 testing/btest/scripts/base/frameworks/input/bignumber.bro diff --git a/src/input/readers/Ascii.cc b/src/input/readers/Ascii.cc index fd936b07b6..28b1ed29c9 100644 --- a/src/input/readers/Ascii.cc +++ b/src/input/readers/Ascii.cc @@ -238,7 +238,7 @@ Value* Ascii::EntryToVal(string s, FieldMapping field) break; case TYPE_INT: - val->val.int_val = atoi(s.c_str()); + val->val.int_val = strtoll(s.c_str(), (char**) NULL, 10); break; case TYPE_DOUBLE: @@ -249,7 +249,7 @@ Value* Ascii::EntryToVal(string s, FieldMapping field) case TYPE_COUNT: case TYPE_COUNTER: - val->val.uint_val = atoi(s.c_str()); + val->val.uint_val = strtoull(s.c_str(),(char**) NULL, 10); break; case TYPE_PORT: @@ -344,7 +344,7 @@ Value* Ascii::EntryToVal(string s, FieldMapping field) if ( pos != length ) { - Error("Internal error while parsing set: did not find all elements"); + Error(Fmt("Internal error while parsing set: did not find all elements: %s", s.c_str())); return 0; } diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.bignumber/out b/testing/btest/Baseline/scripts.base.frameworks.input.bignumber/out new file mode 100644 index 0000000000..ab095ca36c --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.bignumber/out @@ -0,0 +1,3 @@ +{ +[9223372036854775800] = [c=18446744073709551612] +} diff --git a/testing/btest/scripts/base/frameworks/input/bignumber.bro b/testing/btest/scripts/base/frameworks/input/bignumber.bro new file mode 100644 index 0000000000..519992be05 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/input/bignumber.bro @@ -0,0 +1,44 @@ +# (uses listen.bro just to ensure input sources are more reliably fully-read). +# @TEST-SERIALIZE: comm +# +# @TEST-EXEC: btest-bg-run bro bro -b %INPUT +# @TEST-EXEC: btest-bg-wait -k 5 +# @TEST-EXEC: btest-diff out + +@TEST-START-FILE input.log +#separator \x09 +#fields i c +#types int count +9223372036854775800 18446744073709551612 +@TEST-END-FILE + +@load frameworks/communication/listen + +global outfile: file; + +module A; + +type Idx: record { + i: int; +}; + +type Val: record { + c: count; +}; + +global servers: table[int] of Val = table(); + +event bro_init() + { + outfile = open("../out"); + # first read in the old stuff into the table... + Input::add_table([$source="../input.log", $name="ssh", $idx=Idx, $val=Val, $destination=servers]); + Input::remove("ssh"); + } + +event Input::update_finished(name: string, source:string) + { + print outfile, servers; + close(outfile); + terminate(); + } From 977c1d7c5adbf1b3bb2be55a99c4bd018e78a524 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Sun, 26 Aug 2012 17:52:07 -0700 Subject: [PATCH 2/6] make set_separators different from , work for input framework. 1-line-patch + test. --- .../out | 10 ++++ .../base/frameworks/input/setseparator.bro | 46 +++++++++++++++++++ 2 files changed, 56 insertions(+) create mode 100644 testing/btest/Baseline/scripts.base.frameworks.input.setseparator/out create mode 100644 testing/btest/scripts/base/frameworks/input/setseparator.bro diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.setseparator/out b/testing/btest/Baseline/scripts.base.frameworks.input.setseparator/out new file mode 100644 index 0000000000..d0e0f53310 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.setseparator/out @@ -0,0 +1,10 @@ +{ +[1] = [s={ +b, +e, +d, +c, +f, +a +}, ss=[1, 2, 3, 4, 5, 6]] +} diff --git a/testing/btest/scripts/base/frameworks/input/setseparator.bro b/testing/btest/scripts/base/frameworks/input/setseparator.bro new file mode 100644 index 0000000000..44b9d08d54 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/input/setseparator.bro @@ -0,0 +1,46 @@ +# (uses listen.bro just to ensure input sources are more reliably fully-read). +# @TEST-SERIALIZE: comm +# +# @TEST-EXEC: btest-bg-run bro bro -b %INPUT +# @TEST-EXEC: btest-bg-wait -k 5 +# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-sort btest-diff out + +@TEST-START-FILE input.log +#separator \x09 +#fields i s ss +1 a|b|c|d|e|f 1|2|3|4|5|6 +@TEST-END-FILE + +redef InputAscii::set_separator = "|"; + +@load frameworks/communication/listen + +global outfile: file; + +module A; + +type Idx: record { + i: int; +}; + +type Val: record { + s: set[string]; + ss:vector of count; +}; + +global servers: table[int] of Val = table(); + +event bro_init() + { + outfile = open("../out"); + # first read in the old stuff into the table... + Input::add_table([$source="../input.log", $name="ssh", $idx=Idx, $val=Val, $destination=servers]); + Input::remove("ssh"); + } + +event Input::update_finished(name: string, source:string) + { + print outfile, servers; + close(outfile); + terminate(); + } From 6bf733ce513a39804ba73b1e281adba5322f2de6 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Sun, 26 Aug 2012 17:53:34 -0700 Subject: [PATCH 3/6] sorry. the patch for the set_separator. --- src/input/readers/Ascii.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/input/readers/Ascii.cc b/src/input/readers/Ascii.cc index 28b1ed29c9..e0be235700 100644 --- a/src/input/readers/Ascii.cc +++ b/src/input/readers/Ascii.cc @@ -288,7 +288,7 @@ Value* Ascii::EntryToVal(string s, FieldMapping field) // how many entries do we have... unsigned int length = 1; for ( unsigned int i = 0; i < s.size(); i++ ) - if ( s[i] == ',' ) length++; + if ( s[i] == set_separator[0] ) length++; unsigned int pos = 0; From a9e6d9ae8154eecb415f86ca9f786f21886fff94 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Sun, 26 Aug 2012 19:17:21 -0700 Subject: [PATCH 4/6] Fix two little bugs: Escaped ,'s in sets and vectors were unescaped before tokenization Handling of zero-length-strings as last element in a set was broken (sets ending with a ,). Hashing of lines just containing zero-length-strings was broken (now a \0 is appended to each string before it is hashed - giving us a hash of something for a line just consisting of \0s. This also allows to differentiate between vectors with varying numbers of zero-length-strings). --- src/input/Manager.cc | 6 ++- src/input/readers/Ascii.cc | 18 ++++++- .../out | 20 ++++++++ .../base/frameworks/input/setspecialcases.bro | 49 +++++++++++++++++++ 4 files changed, 90 insertions(+), 3 deletions(-) create mode 100644 testing/btest/Baseline/scripts.base.frameworks.input.setspecialcases/out create mode 100644 testing/btest/scripts/base/frameworks/input/setspecialcases.bro diff --git a/src/input/Manager.cc b/src/input/Manager.cc index 3c29f14928..07ce5b20fc 100644 --- a/src/input/Manager.cc +++ b/src/input/Manager.cc @@ -1718,7 +1718,7 @@ int Manager::GetValueLength(const Value* val) { case TYPE_STRING: case TYPE_ENUM: { - length += val->val.string_val.length; + length += val->val.string_val.length+1; break; } @@ -1818,7 +1818,9 @@ int Manager::CopyValue(char *data, const int startpos, const Value* val) case TYPE_ENUM: { memcpy(data+startpos, val->val.string_val.data, val->val.string_val.length); - return val->val.string_val.length; + // and add a \0 to the end. To be able to hash zero-length strings and differentiate from !present + memset(data+startpos+val->val.string_val.length, 0, 1); + return val->val.string_val.length+1; } case TYPE_ADDR: diff --git a/src/input/readers/Ascii.cc b/src/input/readers/Ascii.cc index e0be235700..4bf82c6a13 100644 --- a/src/input/readers/Ascii.cc +++ b/src/input/readers/Ascii.cc @@ -220,6 +220,7 @@ Value* Ascii::EntryToVal(string s, FieldMapping field) switch ( field.type ) { case TYPE_ENUM: case TYPE_STRING: + s = get_unescaped_string(s); val->val.string_val.length = s.size(); val->val.string_val.data = copy_string(s.c_str()); break; @@ -259,6 +260,7 @@ Value* Ascii::EntryToVal(string s, FieldMapping field) case TYPE_SUBNET: { + s = get_unescaped_string(s); size_t pos = s.find("/"); if ( pos == s.npos ) { @@ -275,6 +277,7 @@ Value* Ascii::EntryToVal(string s, FieldMapping field) } case TYPE_ADDR: + s = get_unescaped_string(s); val->val.addr_val = StringToAddr(s); break; @@ -342,6 +345,20 @@ Value* Ascii::EntryToVal(string s, FieldMapping field) pos++; } + // test if the string ends with a set_separator... if it does we have to push an zero-lenght + // val on top of it. + if ( *s.rbegin() == set_separator[0] ) + { + lvals[pos] = EntryToVal("", field.subType()); + if ( lvals[pos] == 0 ) + { + Error("Error while trying to add empty set element"); + return 0; + } + + pos++; + } + if ( pos != length ) { Error(Fmt("Internal error while parsing set: did not find all elements: %s", s.c_str())); @@ -438,7 +455,6 @@ bool Ascii::DoUpdate() if ( ! getline(splitstream, s, separator[0]) ) break; - s = get_unescaped_string(s); stringfields[pos] = s; pos++; diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.setspecialcases/out b/testing/btest/Baseline/scripts.base.frameworks.input.setspecialcases/out new file mode 100644 index 0000000000..28d1e025bf --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.setspecialcases/out @@ -0,0 +1,20 @@ +{ +[2] = [s={ +, +testing +}, s=[testing, , testing]], +[4] = [s={ +, +testing +}, s=[testing, ]], +[1] = [s={ +testing,testing,testing, +}, s=[testing,testing,testing,]], +[5] = [s={ + +}, s=[, , , ]], +[3] = [s={ +, +testing +}, s=[, testing]] +} diff --git a/testing/btest/scripts/base/frameworks/input/setspecialcases.bro b/testing/btest/scripts/base/frameworks/input/setspecialcases.bro new file mode 100644 index 0000000000..29819a795f --- /dev/null +++ b/testing/btest/scripts/base/frameworks/input/setspecialcases.bro @@ -0,0 +1,49 @@ +# (uses listen.bro just to ensure input sources are more reliably fully-read). +# @TEST-SERIALIZE: comm +# +# @TEST-EXEC: btest-bg-run bro bro -b %INPUT +# @TEST-EXEC: btest-bg-wait -k 5 +# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-sort btest-diff out + +@TEST-START-FILE input.log +#separator \x09 +#fields i s ss +1 testing\x2ctesting\x2ctesting\x2c testing\x2ctesting\x2ctesting\x2c +2 testing,,testing testing,,testing +3 ,testing ,testing +4 testing, testing, +5 ,,, ,,, +@TEST-END-FILE + + +@load frameworks/communication/listen + +global outfile: file; + +module A; + +type Idx: record { + i: int; +}; + +type Val: record { + s: set[string]; + s: vector of string; +}; + +global servers: table[int] of Val = table(); + +event bro_init() + { + outfile = open("../out"); + # first read in the old stuff into the table... + Input::add_table([$source="../input.log", $name="ssh", $idx=Idx, $val=Val, $destination=servers]); + Input::remove("ssh"); + } + +event Input::update_finished(name: string, source:string) + { + print outfile, servers; + close(outfile); + terminate(); + } From fbe464ffa348c59b980584ad321e206d9a794ac2 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Sun, 26 Aug 2012 20:26:08 -0700 Subject: [PATCH 5/6] another small bug found while searching for something else... ...one of the change events got the wrong parameters. This actually is a bit embarassing... --- src/input/Manager.cc | 2 +- .../scripts.base.frameworks.input.reread/out | 240 ++++++++++++++++-- 2 files changed, 223 insertions(+), 19 deletions(-) diff --git a/src/input/Manager.cc b/src/input/Manager.cc index 07ce5b20fc..44d7140485 100644 --- a/src/input/Manager.cc +++ b/src/input/Manager.cc @@ -1210,7 +1210,7 @@ void Manager::EndCurrentSend(ReaderFrontend* reader) Ref(predidx); Ref(val); Ref(ev); - SendEvent(stream->event, 3, ev, predidx, val); + SendEvent(stream->event, 4, stream->description->Ref(), ev, predidx, val); } if ( predidx ) // if we have a stream or an event... diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.reread/out b/testing/btest/Baseline/scripts.base.frameworks.input.reread/out index 8b55ced2ac..acc9bfe846 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.input.reread/out +++ b/testing/btest/Baseline/scripts.base.frameworks.input.reread/out @@ -1174,10 +1174,45 @@ BB }, vc=[10, 20, 30], ve=[]] ============EVENT============ Description -Input::EVENT_REMOVED +[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh, destination={ +[-48] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +}, idx=, val=, want_record=T, ev=line +{ +print A::outfile, ============EVENT============; +print A::outfile, Description; +print A::outfile, A::description; +print A::outfile, Type; +print A::outfile, A::tpe; +print A::outfile, Left; +print A::outfile, A::left; +print A::outfile, Right; +print A::outfile, A::right; +}, pred=anonymous-function +{ +print A::outfile, ============PREDICATE============; +print A::outfile, A::typ; +print A::outfile, A::left; +print A::outfile, A::right; +return (T); +}, config={ + +}] Type +Input::EVENT_REMOVED +Left [i=-43] -Left +Right [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ 2, 4, @@ -1190,13 +1225,47 @@ BB }, se={ }, vc=[10, 20, 30], ve=[]] -Right ============EVENT============ Description -Input::EVENT_REMOVED +[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh, destination={ +[-48] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +}, idx=, val=, want_record=T, ev=line +{ +print A::outfile, ============EVENT============; +print A::outfile, Description; +print A::outfile, A::description; +print A::outfile, Type; +print A::outfile, A::tpe; +print A::outfile, Left; +print A::outfile, A::left; +print A::outfile, Right; +print A::outfile, A::right; +}, pred=anonymous-function +{ +print A::outfile, ============PREDICATE============; +print A::outfile, A::typ; +print A::outfile, A::left; +print A::outfile, A::right; +return (T); +}, config={ + +}] Type +Input::EVENT_REMOVED +Left [i=-46] -Left +Right [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ 2, 4, @@ -1209,13 +1278,47 @@ BB }, se={ }, vc=[10, 20, 30], ve=[]] -Right ============EVENT============ Description -Input::EVENT_REMOVED +[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh, destination={ +[-48] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +}, idx=, val=, want_record=T, ev=line +{ +print A::outfile, ============EVENT============; +print A::outfile, Description; +print A::outfile, A::description; +print A::outfile, Type; +print A::outfile, A::tpe; +print A::outfile, Left; +print A::outfile, A::left; +print A::outfile, Right; +print A::outfile, A::right; +}, pred=anonymous-function +{ +print A::outfile, ============PREDICATE============; +print A::outfile, A::typ; +print A::outfile, A::left; +print A::outfile, A::right; +return (T); +}, config={ + +}] Type +Input::EVENT_REMOVED +Left [i=-44] -Left +Right [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ 2, 4, @@ -1228,13 +1331,47 @@ BB }, se={ }, vc=[10, 20, 30], ve=[]] -Right ============EVENT============ Description -Input::EVENT_REMOVED +[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh, destination={ +[-48] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +}, idx=, val=, want_record=T, ev=line +{ +print A::outfile, ============EVENT============; +print A::outfile, Description; +print A::outfile, A::description; +print A::outfile, Type; +print A::outfile, A::tpe; +print A::outfile, Left; +print A::outfile, A::left; +print A::outfile, Right; +print A::outfile, A::right; +}, pred=anonymous-function +{ +print A::outfile, ============PREDICATE============; +print A::outfile, A::typ; +print A::outfile, A::left; +print A::outfile, A::right; +return (T); +}, config={ + +}] Type +Input::EVENT_REMOVED +Left [i=-47] -Left +Right [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ 2, 4, @@ -1247,13 +1384,47 @@ BB }, se={ }, vc=[10, 20, 30], ve=[]] -Right ============EVENT============ Description -Input::EVENT_REMOVED +[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh, destination={ +[-48] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +}, idx=, val=, want_record=T, ev=line +{ +print A::outfile, ============EVENT============; +print A::outfile, Description; +print A::outfile, A::description; +print A::outfile, Type; +print A::outfile, A::tpe; +print A::outfile, Left; +print A::outfile, A::left; +print A::outfile, Right; +print A::outfile, A::right; +}, pred=anonymous-function +{ +print A::outfile, ============PREDICATE============; +print A::outfile, A::typ; +print A::outfile, A::left; +print A::outfile, A::right; +return (T); +}, config={ + +}] Type +Input::EVENT_REMOVED +Left [i=-45] -Left +Right [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ 2, 4, @@ -1266,13 +1437,47 @@ BB }, se={ }, vc=[10, 20, 30], ve=[]] -Right ============EVENT============ Description -Input::EVENT_REMOVED +[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh, destination={ +[-48] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +}, idx=, val=, want_record=T, ev=line +{ +print A::outfile, ============EVENT============; +print A::outfile, Description; +print A::outfile, A::description; +print A::outfile, Type; +print A::outfile, A::tpe; +print A::outfile, Left; +print A::outfile, A::left; +print A::outfile, Right; +print A::outfile, A::right; +}, pred=anonymous-function +{ +print A::outfile, ============PREDICATE============; +print A::outfile, A::typ; +print A::outfile, A::left; +print A::outfile, A::right; +return (T); +}, config={ + +}] Type -[i=-42] +Input::EVENT_REMOVED Left +[i=-42] +Right [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ 2, 4, @@ -1285,7 +1490,6 @@ BB }, se={ }, vc=[10, 20, 30], ve=[]] -Right ==========SERVERS============ { [-48] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ From 7e46936728f08b1214a6610e194793eb145a1f37 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Sun, 26 Aug 2012 20:49:21 -0700 Subject: [PATCH 6/6] Ok, this one is not really necessary for 2.1 and more of a nice-to-have Before this patch, empty values were not hashed at all. Which had the unfortunate side-effect that e.g. the lines TEST - and - TEST have the same hash values. On re-reads that means that the change will be ignored. This is probably pretty academic, but this patch changes it and adds a testcase. Output of the reread test changes due to re-ordering of the output (probably due to the fact that the internal hash values are changed and thus transferred in a different order) --- src/input/Manager.cc | 17 +- .../out | 155 +++++++++++ .../scripts.base.frameworks.input.reread/out | 248 +++++++++--------- .../frameworks/input/empty-values-hashing.bro | 89 +++++++ 4 files changed, 382 insertions(+), 127 deletions(-) create mode 100644 testing/btest/Baseline/scripts.base.frameworks.input.empty-values-hashing/out create mode 100644 testing/btest/scripts/base/frameworks/input/empty-values-hashing.bro diff --git a/src/input/Manager.cc b/src/input/Manager.cc index 44d7140485..e230c0e489 100644 --- a/src/input/Manager.cc +++ b/src/input/Manager.cc @@ -1911,11 +1911,16 @@ HashKey* Manager::HashValues(const int num_elements, const Value* const *vals) const Value* val = vals[i]; if ( val->present ) length += GetValueLength(val); - } - if ( length == 0 ) + // and in any case add 1 for the end-of-field-identifier + length++; + } + + + assert ( length >= num_elements ); + + if ( length == num_elements ) { - reporter->Error("Input reader sent line where all elements are null values. Ignoring line"); return NULL; } @@ -1929,6 +1934,12 @@ HashKey* Manager::HashValues(const int num_elements, const Value* const *vals) const Value* val = vals[i]; if ( val->present ) position += CopyValue(data, position, val); + + memset(data+position, 1, 1); // add end-of-field-marker. does not really matter which value it is, + // it just has to be... something + + position++; + } HashKey *key = new HashKey(data, length); diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.empty-values-hashing/out b/testing/btest/Baseline/scripts.base.frameworks.input.empty-values-hashing/out new file mode 100644 index 0000000000..474ef45cc2 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.empty-values-hashing/out @@ -0,0 +1,155 @@ +============PREDICATE============ +Input::EVENT_NEW +[i=1] +[s=, ss=TEST] +============PREDICATE============ +Input::EVENT_NEW +[i=2] +[s=, ss=] +============EVENT============ +Description +[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh, destination={ +[2] = [s=, ss=], +[1] = [s=, ss=TEST] +}, idx=, val=, want_record=T, ev=line +{ +print A::outfile, ============EVENT============; +print A::outfile, Description; +print A::outfile, A::description; +print A::outfile, Type; +print A::outfile, A::tpe; +print A::outfile, Left; +print A::outfile, A::left; +print A::outfile, Right; +print A::outfile, A::right; +}, pred=anonymous-function +{ +print A::outfile, ============PREDICATE============; +print A::outfile, A::typ; +print A::outfile, A::left; +print A::outfile, A::right; +return (T); +}, config={ + +}] +Type +Input::EVENT_NEW +Left +[i=1] +Right +[s=, ss=TEST] +============EVENT============ +Description +[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh, destination={ +[2] = [s=, ss=], +[1] = [s=, ss=TEST] +}, idx=, val=, want_record=T, ev=line +{ +print A::outfile, ============EVENT============; +print A::outfile, Description; +print A::outfile, A::description; +print A::outfile, Type; +print A::outfile, A::tpe; +print A::outfile, Left; +print A::outfile, A::left; +print A::outfile, Right; +print A::outfile, A::right; +}, pred=anonymous-function +{ +print A::outfile, ============PREDICATE============; +print A::outfile, A::typ; +print A::outfile, A::left; +print A::outfile, A::right; +return (T); +}, config={ + +}] +Type +Input::EVENT_NEW +Left +[i=2] +Right +[s=, ss=] +==========SERVERS============ +{ +[2] = [s=, ss=], +[1] = [s=, ss=TEST] +} +============PREDICATE============ +Input::EVENT_CHANGED +[i=1] +[s=TEST, ss=] +============PREDICATE============ +Input::EVENT_CHANGED +[i=2] +[s=TEST, ss=TEST] +============EVENT============ +Description +[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh, destination={ +[2] = [s=TEST, ss=TEST], +[1] = [s=TEST, ss=] +}, idx=, val=, want_record=T, ev=line +{ +print A::outfile, ============EVENT============; +print A::outfile, Description; +print A::outfile, A::description; +print A::outfile, Type; +print A::outfile, A::tpe; +print A::outfile, Left; +print A::outfile, A::left; +print A::outfile, Right; +print A::outfile, A::right; +}, pred=anonymous-function +{ +print A::outfile, ============PREDICATE============; +print A::outfile, A::typ; +print A::outfile, A::left; +print A::outfile, A::right; +return (T); +}, config={ + +}] +Type +Input::EVENT_CHANGED +Left +[i=1] +Right +[s=, ss=TEST] +============EVENT============ +Description +[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh, destination={ +[2] = [s=TEST, ss=TEST], +[1] = [s=TEST, ss=] +}, idx=, val=, want_record=T, ev=line +{ +print A::outfile, ============EVENT============; +print A::outfile, Description; +print A::outfile, A::description; +print A::outfile, Type; +print A::outfile, A::tpe; +print A::outfile, Left; +print A::outfile, A::left; +print A::outfile, Right; +print A::outfile, A::right; +}, pred=anonymous-function +{ +print A::outfile, ============PREDICATE============; +print A::outfile, A::typ; +print A::outfile, A::left; +print A::outfile, A::right; +return (T); +}, config={ + +}] +Type +Input::EVENT_CHANGED +Left +[i=2] +Right +[s=, ss=] +==========SERVERS============ +{ +[2] = [s=TEST, ss=TEST], +[1] = [s=TEST, ss=] +} +done diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.reread/out b/testing/btest/Baseline/scripts.base.frameworks.input.reread/out index acc9bfe846..538a6dec18 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.input.reread/out +++ b/testing/btest/Baseline/scripts.base.frameworks.input.reread/out @@ -1084,7 +1084,7 @@ BB } ============PREDICATE============ Input::EVENT_REMOVED -[i=-43] +[i=-44] [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ 2, 4, @@ -1096,6 +1096,21 @@ AA, BB }, se={ +}, vc=[10, 20, 30], ve=[]] +============PREDICATE============ +Input::EVENT_REMOVED +[i=-42] +[b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + }, vc=[10, 20, 30], ve=[]] ============PREDICATE============ Input::EVENT_REMOVED @@ -1111,21 +1126,6 @@ AA, BB }, se={ -}, vc=[10, 20, 30], ve=[]] -============PREDICATE============ -Input::EVENT_REMOVED -[i=-44] -[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ -2, -4, -1, -3 -}, ss={ -CC, -AA, -BB -}, se={ - }, vc=[10, 20, 30], ve=[]] ============PREDICATE============ Input::EVENT_REMOVED @@ -1159,7 +1159,113 @@ BB }, vc=[10, 20, 30], ve=[]] ============PREDICATE============ Input::EVENT_REMOVED +[i=-43] +[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +============EVENT============ +Description +[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh, destination={ +[-48] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +}, idx=, val=, want_record=T, ev=line +{ +print A::outfile, ============EVENT============; +print A::outfile, Description; +print A::outfile, A::description; +print A::outfile, Type; +print A::outfile, A::tpe; +print A::outfile, Left; +print A::outfile, A::left; +print A::outfile, Right; +print A::outfile, A::right; +}, pred=anonymous-function +{ +print A::outfile, ============PREDICATE============; +print A::outfile, A::typ; +print A::outfile, A::left; +print A::outfile, A::right; +return (T); +}, config={ + +}] +Type +Input::EVENT_REMOVED +Left +[i=-44] +Right +[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +============EVENT============ +Description +[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh, destination={ +[-48] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +}, idx=, val=, want_record=T, ev=line +{ +print A::outfile, ============EVENT============; +print A::outfile, Description; +print A::outfile, A::description; +print A::outfile, Type; +print A::outfile, A::tpe; +print A::outfile, Left; +print A::outfile, A::left; +print A::outfile, Right; +print A::outfile, A::right; +}, pred=anonymous-function +{ +print A::outfile, ============PREDICATE============; +print A::outfile, A::typ; +print A::outfile, A::left; +print A::outfile, A::right; +return (T); +}, config={ + +}] +Type +Input::EVENT_REMOVED +Left [i=-42] +Right [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ 2, 4, @@ -1207,59 +1313,6 @@ print A::outfile, A::right; return (T); }, config={ -}] -Type -Input::EVENT_REMOVED -Left -[i=-43] -Right -[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ -2, -4, -1, -3 -}, ss={ -CC, -AA, -BB -}, se={ - -}, vc=[10, 20, 30], ve=[]] -============EVENT============ -Description -[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh, destination={ -[-48] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ -2, -4, -1, -3 -}, ss={ -CC, -AA, -BB -}, se={ - -}, vc=[10, 20, 30], ve=[]] -}, idx=, val=, want_record=T, ev=line -{ -print A::outfile, ============EVENT============; -print A::outfile, Description; -print A::outfile, A::description; -print A::outfile, Type; -print A::outfile, A::tpe; -print A::outfile, Left; -print A::outfile, A::left; -print A::outfile, Right; -print A::outfile, A::right; -}, pred=anonymous-function -{ -print A::outfile, ============PREDICATE============; -print A::outfile, A::typ; -print A::outfile, A::left; -print A::outfile, A::right; -return (T); -}, config={ - }] Type Input::EVENT_REMOVED @@ -1313,59 +1366,6 @@ print A::outfile, A::right; return (T); }, config={ -}] -Type -Input::EVENT_REMOVED -Left -[i=-44] -Right -[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ -2, -4, -1, -3 -}, ss={ -CC, -AA, -BB -}, se={ - -}, vc=[10, 20, 30], ve=[]] -============EVENT============ -Description -[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh, destination={ -[-48] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ -2, -4, -1, -3 -}, ss={ -CC, -AA, -BB -}, se={ - -}, vc=[10, 20, 30], ve=[]] -}, idx=, val=, want_record=T, ev=line -{ -print A::outfile, ============EVENT============; -print A::outfile, Description; -print A::outfile, A::description; -print A::outfile, Type; -print A::outfile, A::tpe; -print A::outfile, Left; -print A::outfile, A::left; -print A::outfile, Right; -print A::outfile, A::right; -}, pred=anonymous-function -{ -print A::outfile, ============PREDICATE============; -print A::outfile, A::typ; -print A::outfile, A::left; -print A::outfile, A::right; -return (T); -}, config={ - }] Type Input::EVENT_REMOVED @@ -1476,9 +1476,9 @@ return (T); Type Input::EVENT_REMOVED Left -[i=-42] +[i=-43] Right -[b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ 2, 4, 1, diff --git a/testing/btest/scripts/base/frameworks/input/empty-values-hashing.bro b/testing/btest/scripts/base/frameworks/input/empty-values-hashing.bro new file mode 100644 index 0000000000..b66febba82 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/input/empty-values-hashing.bro @@ -0,0 +1,89 @@ +# (uses listen.bro just to ensure input sources are more reliably fully-read). +# @TEST-SERIALIZE: comm +# +# @TEST-EXEC: cp input1.log input.log +# @TEST-EXEC: btest-bg-run bro bro -b %INPUT +# @TEST-EXEC: sleep 2 +# @TEST-EXEC: cp input2.log input.log +# @TEST-EXEC: btest-bg-wait -k 5 +# @TEST-EXEC: btest-diff out + +@TEST-START-FILE input1.log +#separator \x09 +#fields i s ss +#types int sting string +1 - TEST +2 - - +@TEST-END-FILE +@TEST-START-FILE input2.log +#separator \x09 +#fields i s ss +#types int sting string +1 TEST - +2 TEST TEST +@TEST-END-FILE + +@load frameworks/communication/listen + + +module A; + +type Idx: record { + i: int; +}; + +type Val: record { + s: string; + ss: string; +}; + +global servers: table[int] of Val = table(); + +global outfile: file; + +global try: count; + +event line(description: Input::TableDescription, tpe: Input::Event, left: Idx, right: Val) + { + print outfile, "============EVENT============"; + print outfile, "Description"; + print outfile, description; + print outfile, "Type"; + print outfile, tpe; + print outfile, "Left"; + print outfile, left; + print outfile, "Right"; + print outfile, right; + } + +event bro_init() + { + outfile = open("../out"); + try = 0; + # first read in the old stuff into the table... + Input::add_table([$source="../input.log", $mode=Input::REREAD, $name="ssh", $idx=Idx, $val=Val, $destination=servers, $ev=line, + $pred(typ: Input::Event, left: Idx, right: Val) = { + print outfile, "============PREDICATE============"; + print outfile, typ; + print outfile, left; + print outfile, right; + return T; + } + ]); + } + + +event Input::update_finished(name: string, source: string) + { + print outfile, "==========SERVERS============"; + print outfile, servers; + + try = try + 1; + if ( try == 2 ) + { + print outfile, "done"; + close(outfile); + Input::remove("input"); + terminate(); + } + }