From 70f1403f1420b738d559c0675bd94703cd5af9aa Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Fri, 24 Aug 2012 13:18:51 -0700 Subject: [PATCH 01/20] Updating submodule(s). [nomail] --- CHANGES | 2 +- VERSION | 2 +- aux/broctl | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGES b/CHANGES index 066ee784a8..87da7378b0 100644 --- a/CHANGES +++ b/CHANGES @@ -1,5 +1,5 @@ -2.1-beta-54 | 2012-08-23 11:58:50 -0700 +2.1 | 2012-08-24 13:18:51 -0700 * Update documentation for builtin types. (Daniel Thayer) diff --git a/VERSION b/VERSION index fd6e9996db..879b416e60 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.1-beta-54 +2.1 diff --git a/aux/broctl b/aux/broctl index 5b3f9e5906..6b24757768 160000 --- a/aux/broctl +++ b/aux/broctl @@ -1 +1 @@ -Subproject commit 5b3f9e5906c90b76c5aa1626e112d4c991cb3fd8 +Subproject commit 6b24757768cd9aa742fd678d6864235519740ee8 From b5c694518904a5f122bc643c02f0518e11c3dade Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Fri, 24 Aug 2012 15:11:49 -0700 Subject: [PATCH 02/20] Updating submodule(s). [nomail] --- CHANGES | 2 +- aux/bro-aux | 2 +- aux/broctl | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGES b/CHANGES index 87da7378b0..1c6e9dfafe 100644 --- a/CHANGES +++ b/CHANGES @@ -1,5 +1,5 @@ -2.1 | 2012-08-24 13:18:51 -0700 +2.1 | 2012-08-24 15:11:49 -0700 * Update documentation for builtin types. (Daniel Thayer) diff --git a/aux/bro-aux b/aux/bro-aux index 4bc1a6f6a8..6748ec3a96 160000 --- a/aux/bro-aux +++ b/aux/bro-aux @@ -1 +1 @@ -Subproject commit 4bc1a6f6a8816dfacd8288fcf182ba35520e589b +Subproject commit 6748ec3a96d582a977cd9114ef19c76fe75c57ff diff --git a/aux/broctl b/aux/broctl index 6b24757768..2fb9ff62bf 160000 --- a/aux/broctl +++ b/aux/broctl @@ -1 +1 @@ -Subproject commit 6b24757768cd9aa742fd678d6864235519740ee8 +Subproject commit 2fb9ff62bf08f78071753016863640022fbfe338 From 124c985d7af91a98eb8a7aff8f66b0300849e854 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Sun, 26 Aug 2012 14:49:37 -0700 Subject: [PATCH 03/20] Bug found bei Keith & Seth: input framework was not handling counts and ints out of 32-bit-range correctly. Note - another bugfix will be coming later (problem reading sets containing zero-length-strings & un-escaping-bug in sets) --- src/input/readers/Ascii.cc | 6 +-- .../out | 3 ++ .../base/frameworks/input/bignumber.bro | 44 +++++++++++++++++++ 3 files changed, 50 insertions(+), 3 deletions(-) create mode 100644 testing/btest/Baseline/scripts.base.frameworks.input.bignumber/out create mode 100644 testing/btest/scripts/base/frameworks/input/bignumber.bro diff --git a/src/input/readers/Ascii.cc b/src/input/readers/Ascii.cc index fd936b07b6..28b1ed29c9 100644 --- a/src/input/readers/Ascii.cc +++ b/src/input/readers/Ascii.cc @@ -238,7 +238,7 @@ Value* Ascii::EntryToVal(string s, FieldMapping field) break; case TYPE_INT: - val->val.int_val = atoi(s.c_str()); + val->val.int_val = strtoll(s.c_str(), (char**) NULL, 10); break; case TYPE_DOUBLE: @@ -249,7 +249,7 @@ Value* Ascii::EntryToVal(string s, FieldMapping field) case TYPE_COUNT: case TYPE_COUNTER: - val->val.uint_val = atoi(s.c_str()); + val->val.uint_val = strtoull(s.c_str(),(char**) NULL, 10); break; case TYPE_PORT: @@ -344,7 +344,7 @@ Value* Ascii::EntryToVal(string s, FieldMapping field) if ( pos != length ) { - Error("Internal error while parsing set: did not find all elements"); + Error(Fmt("Internal error while parsing set: did not find all elements: %s", s.c_str())); return 0; } diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.bignumber/out b/testing/btest/Baseline/scripts.base.frameworks.input.bignumber/out new file mode 100644 index 0000000000..ab095ca36c --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.bignumber/out @@ -0,0 +1,3 @@ +{ +[9223372036854775800] = [c=18446744073709551612] +} diff --git a/testing/btest/scripts/base/frameworks/input/bignumber.bro b/testing/btest/scripts/base/frameworks/input/bignumber.bro new file mode 100644 index 0000000000..519992be05 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/input/bignumber.bro @@ -0,0 +1,44 @@ +# (uses listen.bro just to ensure input sources are more reliably fully-read). +# @TEST-SERIALIZE: comm +# +# @TEST-EXEC: btest-bg-run bro bro -b %INPUT +# @TEST-EXEC: btest-bg-wait -k 5 +# @TEST-EXEC: btest-diff out + +@TEST-START-FILE input.log +#separator \x09 +#fields i c +#types int count +9223372036854775800 18446744073709551612 +@TEST-END-FILE + +@load frameworks/communication/listen + +global outfile: file; + +module A; + +type Idx: record { + i: int; +}; + +type Val: record { + c: count; +}; + +global servers: table[int] of Val = table(); + +event bro_init() + { + outfile = open("../out"); + # first read in the old stuff into the table... + Input::add_table([$source="../input.log", $name="ssh", $idx=Idx, $val=Val, $destination=servers]); + Input::remove("ssh"); + } + +event Input::update_finished(name: string, source:string) + { + print outfile, servers; + close(outfile); + terminate(); + } From 977c1d7c5adbf1b3bb2be55a99c4bd018e78a524 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Sun, 26 Aug 2012 17:52:07 -0700 Subject: [PATCH 04/20] make set_separators different from , work for input framework. 1-line-patch + test. --- .../out | 10 ++++ .../base/frameworks/input/setseparator.bro | 46 +++++++++++++++++++ 2 files changed, 56 insertions(+) create mode 100644 testing/btest/Baseline/scripts.base.frameworks.input.setseparator/out create mode 100644 testing/btest/scripts/base/frameworks/input/setseparator.bro diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.setseparator/out b/testing/btest/Baseline/scripts.base.frameworks.input.setseparator/out new file mode 100644 index 0000000000..d0e0f53310 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.setseparator/out @@ -0,0 +1,10 @@ +{ +[1] = [s={ +b, +e, +d, +c, +f, +a +}, ss=[1, 2, 3, 4, 5, 6]] +} diff --git a/testing/btest/scripts/base/frameworks/input/setseparator.bro b/testing/btest/scripts/base/frameworks/input/setseparator.bro new file mode 100644 index 0000000000..44b9d08d54 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/input/setseparator.bro @@ -0,0 +1,46 @@ +# (uses listen.bro just to ensure input sources are more reliably fully-read). +# @TEST-SERIALIZE: comm +# +# @TEST-EXEC: btest-bg-run bro bro -b %INPUT +# @TEST-EXEC: btest-bg-wait -k 5 +# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-sort btest-diff out + +@TEST-START-FILE input.log +#separator \x09 +#fields i s ss +1 a|b|c|d|e|f 1|2|3|4|5|6 +@TEST-END-FILE + +redef InputAscii::set_separator = "|"; + +@load frameworks/communication/listen + +global outfile: file; + +module A; + +type Idx: record { + i: int; +}; + +type Val: record { + s: set[string]; + ss:vector of count; +}; + +global servers: table[int] of Val = table(); + +event bro_init() + { + outfile = open("../out"); + # first read in the old stuff into the table... + Input::add_table([$source="../input.log", $name="ssh", $idx=Idx, $val=Val, $destination=servers]); + Input::remove("ssh"); + } + +event Input::update_finished(name: string, source:string) + { + print outfile, servers; + close(outfile); + terminate(); + } From 6bf733ce513a39804ba73b1e281adba5322f2de6 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Sun, 26 Aug 2012 17:53:34 -0700 Subject: [PATCH 05/20] sorry. the patch for the set_separator. --- src/input/readers/Ascii.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/input/readers/Ascii.cc b/src/input/readers/Ascii.cc index 28b1ed29c9..e0be235700 100644 --- a/src/input/readers/Ascii.cc +++ b/src/input/readers/Ascii.cc @@ -288,7 +288,7 @@ Value* Ascii::EntryToVal(string s, FieldMapping field) // how many entries do we have... unsigned int length = 1; for ( unsigned int i = 0; i < s.size(); i++ ) - if ( s[i] == ',' ) length++; + if ( s[i] == set_separator[0] ) length++; unsigned int pos = 0; From a9e6d9ae8154eecb415f86ca9f786f21886fff94 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Sun, 26 Aug 2012 19:17:21 -0700 Subject: [PATCH 06/20] Fix two little bugs: Escaped ,'s in sets and vectors were unescaped before tokenization Handling of zero-length-strings as last element in a set was broken (sets ending with a ,). Hashing of lines just containing zero-length-strings was broken (now a \0 is appended to each string before it is hashed - giving us a hash of something for a line just consisting of \0s. This also allows to differentiate between vectors with varying numbers of zero-length-strings). --- src/input/Manager.cc | 6 ++- src/input/readers/Ascii.cc | 18 ++++++- .../out | 20 ++++++++ .../base/frameworks/input/setspecialcases.bro | 49 +++++++++++++++++++ 4 files changed, 90 insertions(+), 3 deletions(-) create mode 100644 testing/btest/Baseline/scripts.base.frameworks.input.setspecialcases/out create mode 100644 testing/btest/scripts/base/frameworks/input/setspecialcases.bro diff --git a/src/input/Manager.cc b/src/input/Manager.cc index 3c29f14928..07ce5b20fc 100644 --- a/src/input/Manager.cc +++ b/src/input/Manager.cc @@ -1718,7 +1718,7 @@ int Manager::GetValueLength(const Value* val) { case TYPE_STRING: case TYPE_ENUM: { - length += val->val.string_val.length; + length += val->val.string_val.length+1; break; } @@ -1818,7 +1818,9 @@ int Manager::CopyValue(char *data, const int startpos, const Value* val) case TYPE_ENUM: { memcpy(data+startpos, val->val.string_val.data, val->val.string_val.length); - return val->val.string_val.length; + // and add a \0 to the end. To be able to hash zero-length strings and differentiate from !present + memset(data+startpos+val->val.string_val.length, 0, 1); + return val->val.string_val.length+1; } case TYPE_ADDR: diff --git a/src/input/readers/Ascii.cc b/src/input/readers/Ascii.cc index e0be235700..4bf82c6a13 100644 --- a/src/input/readers/Ascii.cc +++ b/src/input/readers/Ascii.cc @@ -220,6 +220,7 @@ Value* Ascii::EntryToVal(string s, FieldMapping field) switch ( field.type ) { case TYPE_ENUM: case TYPE_STRING: + s = get_unescaped_string(s); val->val.string_val.length = s.size(); val->val.string_val.data = copy_string(s.c_str()); break; @@ -259,6 +260,7 @@ Value* Ascii::EntryToVal(string s, FieldMapping field) case TYPE_SUBNET: { + s = get_unescaped_string(s); size_t pos = s.find("/"); if ( pos == s.npos ) { @@ -275,6 +277,7 @@ Value* Ascii::EntryToVal(string s, FieldMapping field) } case TYPE_ADDR: + s = get_unescaped_string(s); val->val.addr_val = StringToAddr(s); break; @@ -342,6 +345,20 @@ Value* Ascii::EntryToVal(string s, FieldMapping field) pos++; } + // test if the string ends with a set_separator... if it does we have to push an zero-lenght + // val on top of it. + if ( *s.rbegin() == set_separator[0] ) + { + lvals[pos] = EntryToVal("", field.subType()); + if ( lvals[pos] == 0 ) + { + Error("Error while trying to add empty set element"); + return 0; + } + + pos++; + } + if ( pos != length ) { Error(Fmt("Internal error while parsing set: did not find all elements: %s", s.c_str())); @@ -438,7 +455,6 @@ bool Ascii::DoUpdate() if ( ! getline(splitstream, s, separator[0]) ) break; - s = get_unescaped_string(s); stringfields[pos] = s; pos++; diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.setspecialcases/out b/testing/btest/Baseline/scripts.base.frameworks.input.setspecialcases/out new file mode 100644 index 0000000000..28d1e025bf --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.setspecialcases/out @@ -0,0 +1,20 @@ +{ +[2] = [s={ +, +testing +}, s=[testing, , testing]], +[4] = [s={ +, +testing +}, s=[testing, ]], +[1] = [s={ +testing,testing,testing, +}, s=[testing,testing,testing,]], +[5] = [s={ + +}, s=[, , , ]], +[3] = [s={ +, +testing +}, s=[, testing]] +} diff --git a/testing/btest/scripts/base/frameworks/input/setspecialcases.bro b/testing/btest/scripts/base/frameworks/input/setspecialcases.bro new file mode 100644 index 0000000000..29819a795f --- /dev/null +++ b/testing/btest/scripts/base/frameworks/input/setspecialcases.bro @@ -0,0 +1,49 @@ +# (uses listen.bro just to ensure input sources are more reliably fully-read). +# @TEST-SERIALIZE: comm +# +# @TEST-EXEC: btest-bg-run bro bro -b %INPUT +# @TEST-EXEC: btest-bg-wait -k 5 +# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-sort btest-diff out + +@TEST-START-FILE input.log +#separator \x09 +#fields i s ss +1 testing\x2ctesting\x2ctesting\x2c testing\x2ctesting\x2ctesting\x2c +2 testing,,testing testing,,testing +3 ,testing ,testing +4 testing, testing, +5 ,,, ,,, +@TEST-END-FILE + + +@load frameworks/communication/listen + +global outfile: file; + +module A; + +type Idx: record { + i: int; +}; + +type Val: record { + s: set[string]; + s: vector of string; +}; + +global servers: table[int] of Val = table(); + +event bro_init() + { + outfile = open("../out"); + # first read in the old stuff into the table... + Input::add_table([$source="../input.log", $name="ssh", $idx=Idx, $val=Val, $destination=servers]); + Input::remove("ssh"); + } + +event Input::update_finished(name: string, source:string) + { + print outfile, servers; + close(outfile); + terminate(); + } From fbe464ffa348c59b980584ad321e206d9a794ac2 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Sun, 26 Aug 2012 20:26:08 -0700 Subject: [PATCH 07/20] another small bug found while searching for something else... ...one of the change events got the wrong parameters. This actually is a bit embarassing... --- src/input/Manager.cc | 2 +- .../scripts.base.frameworks.input.reread/out | 240 ++++++++++++++++-- 2 files changed, 223 insertions(+), 19 deletions(-) diff --git a/src/input/Manager.cc b/src/input/Manager.cc index 07ce5b20fc..44d7140485 100644 --- a/src/input/Manager.cc +++ b/src/input/Manager.cc @@ -1210,7 +1210,7 @@ void Manager::EndCurrentSend(ReaderFrontend* reader) Ref(predidx); Ref(val); Ref(ev); - SendEvent(stream->event, 3, ev, predidx, val); + SendEvent(stream->event, 4, stream->description->Ref(), ev, predidx, val); } if ( predidx ) // if we have a stream or an event... diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.reread/out b/testing/btest/Baseline/scripts.base.frameworks.input.reread/out index 8b55ced2ac..acc9bfe846 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.input.reread/out +++ b/testing/btest/Baseline/scripts.base.frameworks.input.reread/out @@ -1174,10 +1174,45 @@ BB }, vc=[10, 20, 30], ve=[]] ============EVENT============ Description -Input::EVENT_REMOVED +[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh, destination={ +[-48] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +}, idx=, val=, want_record=T, ev=line +{ +print A::outfile, ============EVENT============; +print A::outfile, Description; +print A::outfile, A::description; +print A::outfile, Type; +print A::outfile, A::tpe; +print A::outfile, Left; +print A::outfile, A::left; +print A::outfile, Right; +print A::outfile, A::right; +}, pred=anonymous-function +{ +print A::outfile, ============PREDICATE============; +print A::outfile, A::typ; +print A::outfile, A::left; +print A::outfile, A::right; +return (T); +}, config={ + +}] Type +Input::EVENT_REMOVED +Left [i=-43] -Left +Right [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ 2, 4, @@ -1190,13 +1225,47 @@ BB }, se={ }, vc=[10, 20, 30], ve=[]] -Right ============EVENT============ Description -Input::EVENT_REMOVED +[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh, destination={ +[-48] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +}, idx=, val=, want_record=T, ev=line +{ +print A::outfile, ============EVENT============; +print A::outfile, Description; +print A::outfile, A::description; +print A::outfile, Type; +print A::outfile, A::tpe; +print A::outfile, Left; +print A::outfile, A::left; +print A::outfile, Right; +print A::outfile, A::right; +}, pred=anonymous-function +{ +print A::outfile, ============PREDICATE============; +print A::outfile, A::typ; +print A::outfile, A::left; +print A::outfile, A::right; +return (T); +}, config={ + +}] Type +Input::EVENT_REMOVED +Left [i=-46] -Left +Right [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ 2, 4, @@ -1209,13 +1278,47 @@ BB }, se={ }, vc=[10, 20, 30], ve=[]] -Right ============EVENT============ Description -Input::EVENT_REMOVED +[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh, destination={ +[-48] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +}, idx=, val=, want_record=T, ev=line +{ +print A::outfile, ============EVENT============; +print A::outfile, Description; +print A::outfile, A::description; +print A::outfile, Type; +print A::outfile, A::tpe; +print A::outfile, Left; +print A::outfile, A::left; +print A::outfile, Right; +print A::outfile, A::right; +}, pred=anonymous-function +{ +print A::outfile, ============PREDICATE============; +print A::outfile, A::typ; +print A::outfile, A::left; +print A::outfile, A::right; +return (T); +}, config={ + +}] Type +Input::EVENT_REMOVED +Left [i=-44] -Left +Right [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ 2, 4, @@ -1228,13 +1331,47 @@ BB }, se={ }, vc=[10, 20, 30], ve=[]] -Right ============EVENT============ Description -Input::EVENT_REMOVED +[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh, destination={ +[-48] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +}, idx=, val=, want_record=T, ev=line +{ +print A::outfile, ============EVENT============; +print A::outfile, Description; +print A::outfile, A::description; +print A::outfile, Type; +print A::outfile, A::tpe; +print A::outfile, Left; +print A::outfile, A::left; +print A::outfile, Right; +print A::outfile, A::right; +}, pred=anonymous-function +{ +print A::outfile, ============PREDICATE============; +print A::outfile, A::typ; +print A::outfile, A::left; +print A::outfile, A::right; +return (T); +}, config={ + +}] Type +Input::EVENT_REMOVED +Left [i=-47] -Left +Right [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ 2, 4, @@ -1247,13 +1384,47 @@ BB }, se={ }, vc=[10, 20, 30], ve=[]] -Right ============EVENT============ Description -Input::EVENT_REMOVED +[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh, destination={ +[-48] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +}, idx=, val=, want_record=T, ev=line +{ +print A::outfile, ============EVENT============; +print A::outfile, Description; +print A::outfile, A::description; +print A::outfile, Type; +print A::outfile, A::tpe; +print A::outfile, Left; +print A::outfile, A::left; +print A::outfile, Right; +print A::outfile, A::right; +}, pred=anonymous-function +{ +print A::outfile, ============PREDICATE============; +print A::outfile, A::typ; +print A::outfile, A::left; +print A::outfile, A::right; +return (T); +}, config={ + +}] Type +Input::EVENT_REMOVED +Left [i=-45] -Left +Right [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ 2, 4, @@ -1266,13 +1437,47 @@ BB }, se={ }, vc=[10, 20, 30], ve=[]] -Right ============EVENT============ Description -Input::EVENT_REMOVED +[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh, destination={ +[-48] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +}, idx=, val=, want_record=T, ev=line +{ +print A::outfile, ============EVENT============; +print A::outfile, Description; +print A::outfile, A::description; +print A::outfile, Type; +print A::outfile, A::tpe; +print A::outfile, Left; +print A::outfile, A::left; +print A::outfile, Right; +print A::outfile, A::right; +}, pred=anonymous-function +{ +print A::outfile, ============PREDICATE============; +print A::outfile, A::typ; +print A::outfile, A::left; +print A::outfile, A::right; +return (T); +}, config={ + +}] Type -[i=-42] +Input::EVENT_REMOVED Left +[i=-42] +Right [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ 2, 4, @@ -1285,7 +1490,6 @@ BB }, se={ }, vc=[10, 20, 30], ve=[]] -Right ==========SERVERS============ { [-48] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ From 7e46936728f08b1214a6610e194793eb145a1f37 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Sun, 26 Aug 2012 20:49:21 -0700 Subject: [PATCH 08/20] Ok, this one is not really necessary for 2.1 and more of a nice-to-have Before this patch, empty values were not hashed at all. Which had the unfortunate side-effect that e.g. the lines TEST - and - TEST have the same hash values. On re-reads that means that the change will be ignored. This is probably pretty academic, but this patch changes it and adds a testcase. Output of the reread test changes due to re-ordering of the output (probably due to the fact that the internal hash values are changed and thus transferred in a different order) --- src/input/Manager.cc | 17 +- .../out | 155 +++++++++++ .../scripts.base.frameworks.input.reread/out | 248 +++++++++--------- .../frameworks/input/empty-values-hashing.bro | 89 +++++++ 4 files changed, 382 insertions(+), 127 deletions(-) create mode 100644 testing/btest/Baseline/scripts.base.frameworks.input.empty-values-hashing/out create mode 100644 testing/btest/scripts/base/frameworks/input/empty-values-hashing.bro diff --git a/src/input/Manager.cc b/src/input/Manager.cc index 44d7140485..e230c0e489 100644 --- a/src/input/Manager.cc +++ b/src/input/Manager.cc @@ -1911,11 +1911,16 @@ HashKey* Manager::HashValues(const int num_elements, const Value* const *vals) const Value* val = vals[i]; if ( val->present ) length += GetValueLength(val); - } - if ( length == 0 ) + // and in any case add 1 for the end-of-field-identifier + length++; + } + + + assert ( length >= num_elements ); + + if ( length == num_elements ) { - reporter->Error("Input reader sent line where all elements are null values. Ignoring line"); return NULL; } @@ -1929,6 +1934,12 @@ HashKey* Manager::HashValues(const int num_elements, const Value* const *vals) const Value* val = vals[i]; if ( val->present ) position += CopyValue(data, position, val); + + memset(data+position, 1, 1); // add end-of-field-marker. does not really matter which value it is, + // it just has to be... something + + position++; + } HashKey *key = new HashKey(data, length); diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.empty-values-hashing/out b/testing/btest/Baseline/scripts.base.frameworks.input.empty-values-hashing/out new file mode 100644 index 0000000000..474ef45cc2 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.empty-values-hashing/out @@ -0,0 +1,155 @@ +============PREDICATE============ +Input::EVENT_NEW +[i=1] +[s=, ss=TEST] +============PREDICATE============ +Input::EVENT_NEW +[i=2] +[s=, ss=] +============EVENT============ +Description +[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh, destination={ +[2] = [s=, ss=], +[1] = [s=, ss=TEST] +}, idx=, val=, want_record=T, ev=line +{ +print A::outfile, ============EVENT============; +print A::outfile, Description; +print A::outfile, A::description; +print A::outfile, Type; +print A::outfile, A::tpe; +print A::outfile, Left; +print A::outfile, A::left; +print A::outfile, Right; +print A::outfile, A::right; +}, pred=anonymous-function +{ +print A::outfile, ============PREDICATE============; +print A::outfile, A::typ; +print A::outfile, A::left; +print A::outfile, A::right; +return (T); +}, config={ + +}] +Type +Input::EVENT_NEW +Left +[i=1] +Right +[s=, ss=TEST] +============EVENT============ +Description +[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh, destination={ +[2] = [s=, ss=], +[1] = [s=, ss=TEST] +}, idx=, val=, want_record=T, ev=line +{ +print A::outfile, ============EVENT============; +print A::outfile, Description; +print A::outfile, A::description; +print A::outfile, Type; +print A::outfile, A::tpe; +print A::outfile, Left; +print A::outfile, A::left; +print A::outfile, Right; +print A::outfile, A::right; +}, pred=anonymous-function +{ +print A::outfile, ============PREDICATE============; +print A::outfile, A::typ; +print A::outfile, A::left; +print A::outfile, A::right; +return (T); +}, config={ + +}] +Type +Input::EVENT_NEW +Left +[i=2] +Right +[s=, ss=] +==========SERVERS============ +{ +[2] = [s=, ss=], +[1] = [s=, ss=TEST] +} +============PREDICATE============ +Input::EVENT_CHANGED +[i=1] +[s=TEST, ss=] +============PREDICATE============ +Input::EVENT_CHANGED +[i=2] +[s=TEST, ss=TEST] +============EVENT============ +Description +[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh, destination={ +[2] = [s=TEST, ss=TEST], +[1] = [s=TEST, ss=] +}, idx=, val=, want_record=T, ev=line +{ +print A::outfile, ============EVENT============; +print A::outfile, Description; +print A::outfile, A::description; +print A::outfile, Type; +print A::outfile, A::tpe; +print A::outfile, Left; +print A::outfile, A::left; +print A::outfile, Right; +print A::outfile, A::right; +}, pred=anonymous-function +{ +print A::outfile, ============PREDICATE============; +print A::outfile, A::typ; +print A::outfile, A::left; +print A::outfile, A::right; +return (T); +}, config={ + +}] +Type +Input::EVENT_CHANGED +Left +[i=1] +Right +[s=, ss=TEST] +============EVENT============ +Description +[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh, destination={ +[2] = [s=TEST, ss=TEST], +[1] = [s=TEST, ss=] +}, idx=, val=, want_record=T, ev=line +{ +print A::outfile, ============EVENT============; +print A::outfile, Description; +print A::outfile, A::description; +print A::outfile, Type; +print A::outfile, A::tpe; +print A::outfile, Left; +print A::outfile, A::left; +print A::outfile, Right; +print A::outfile, A::right; +}, pred=anonymous-function +{ +print A::outfile, ============PREDICATE============; +print A::outfile, A::typ; +print A::outfile, A::left; +print A::outfile, A::right; +return (T); +}, config={ + +}] +Type +Input::EVENT_CHANGED +Left +[i=2] +Right +[s=, ss=] +==========SERVERS============ +{ +[2] = [s=TEST, ss=TEST], +[1] = [s=TEST, ss=] +} +done diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.reread/out b/testing/btest/Baseline/scripts.base.frameworks.input.reread/out index acc9bfe846..538a6dec18 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.input.reread/out +++ b/testing/btest/Baseline/scripts.base.frameworks.input.reread/out @@ -1084,7 +1084,7 @@ BB } ============PREDICATE============ Input::EVENT_REMOVED -[i=-43] +[i=-44] [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ 2, 4, @@ -1096,6 +1096,21 @@ AA, BB }, se={ +}, vc=[10, 20, 30], ve=[]] +============PREDICATE============ +Input::EVENT_REMOVED +[i=-42] +[b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + }, vc=[10, 20, 30], ve=[]] ============PREDICATE============ Input::EVENT_REMOVED @@ -1111,21 +1126,6 @@ AA, BB }, se={ -}, vc=[10, 20, 30], ve=[]] -============PREDICATE============ -Input::EVENT_REMOVED -[i=-44] -[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ -2, -4, -1, -3 -}, ss={ -CC, -AA, -BB -}, se={ - }, vc=[10, 20, 30], ve=[]] ============PREDICATE============ Input::EVENT_REMOVED @@ -1159,7 +1159,113 @@ BB }, vc=[10, 20, 30], ve=[]] ============PREDICATE============ Input::EVENT_REMOVED +[i=-43] +[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +============EVENT============ +Description +[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh, destination={ +[-48] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +}, idx=, val=, want_record=T, ev=line +{ +print A::outfile, ============EVENT============; +print A::outfile, Description; +print A::outfile, A::description; +print A::outfile, Type; +print A::outfile, A::tpe; +print A::outfile, Left; +print A::outfile, A::left; +print A::outfile, Right; +print A::outfile, A::right; +}, pred=anonymous-function +{ +print A::outfile, ============PREDICATE============; +print A::outfile, A::typ; +print A::outfile, A::left; +print A::outfile, A::right; +return (T); +}, config={ + +}] +Type +Input::EVENT_REMOVED +Left +[i=-44] +Right +[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +============EVENT============ +Description +[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh, destination={ +[-48] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +}, idx=, val=, want_record=T, ev=line +{ +print A::outfile, ============EVENT============; +print A::outfile, Description; +print A::outfile, A::description; +print A::outfile, Type; +print A::outfile, A::tpe; +print A::outfile, Left; +print A::outfile, A::left; +print A::outfile, Right; +print A::outfile, A::right; +}, pred=anonymous-function +{ +print A::outfile, ============PREDICATE============; +print A::outfile, A::typ; +print A::outfile, A::left; +print A::outfile, A::right; +return (T); +}, config={ + +}] +Type +Input::EVENT_REMOVED +Left [i=-42] +Right [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ 2, 4, @@ -1207,59 +1313,6 @@ print A::outfile, A::right; return (T); }, config={ -}] -Type -Input::EVENT_REMOVED -Left -[i=-43] -Right -[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ -2, -4, -1, -3 -}, ss={ -CC, -AA, -BB -}, se={ - -}, vc=[10, 20, 30], ve=[]] -============EVENT============ -Description -[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh, destination={ -[-48] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ -2, -4, -1, -3 -}, ss={ -CC, -AA, -BB -}, se={ - -}, vc=[10, 20, 30], ve=[]] -}, idx=, val=, want_record=T, ev=line -{ -print A::outfile, ============EVENT============; -print A::outfile, Description; -print A::outfile, A::description; -print A::outfile, Type; -print A::outfile, A::tpe; -print A::outfile, Left; -print A::outfile, A::left; -print A::outfile, Right; -print A::outfile, A::right; -}, pred=anonymous-function -{ -print A::outfile, ============PREDICATE============; -print A::outfile, A::typ; -print A::outfile, A::left; -print A::outfile, A::right; -return (T); -}, config={ - }] Type Input::EVENT_REMOVED @@ -1313,59 +1366,6 @@ print A::outfile, A::right; return (T); }, config={ -}] -Type -Input::EVENT_REMOVED -Left -[i=-44] -Right -[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ -2, -4, -1, -3 -}, ss={ -CC, -AA, -BB -}, se={ - -}, vc=[10, 20, 30], ve=[]] -============EVENT============ -Description -[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh, destination={ -[-48] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ -2, -4, -1, -3 -}, ss={ -CC, -AA, -BB -}, se={ - -}, vc=[10, 20, 30], ve=[]] -}, idx=, val=, want_record=T, ev=line -{ -print A::outfile, ============EVENT============; -print A::outfile, Description; -print A::outfile, A::description; -print A::outfile, Type; -print A::outfile, A::tpe; -print A::outfile, Left; -print A::outfile, A::left; -print A::outfile, Right; -print A::outfile, A::right; -}, pred=anonymous-function -{ -print A::outfile, ============PREDICATE============; -print A::outfile, A::typ; -print A::outfile, A::left; -print A::outfile, A::right; -return (T); -}, config={ - }] Type Input::EVENT_REMOVED @@ -1476,9 +1476,9 @@ return (T); Type Input::EVENT_REMOVED Left -[i=-42] +[i=-43] Right -[b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ 2, 4, 1, diff --git a/testing/btest/scripts/base/frameworks/input/empty-values-hashing.bro b/testing/btest/scripts/base/frameworks/input/empty-values-hashing.bro new file mode 100644 index 0000000000..b66febba82 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/input/empty-values-hashing.bro @@ -0,0 +1,89 @@ +# (uses listen.bro just to ensure input sources are more reliably fully-read). +# @TEST-SERIALIZE: comm +# +# @TEST-EXEC: cp input1.log input.log +# @TEST-EXEC: btest-bg-run bro bro -b %INPUT +# @TEST-EXEC: sleep 2 +# @TEST-EXEC: cp input2.log input.log +# @TEST-EXEC: btest-bg-wait -k 5 +# @TEST-EXEC: btest-diff out + +@TEST-START-FILE input1.log +#separator \x09 +#fields i s ss +#types int sting string +1 - TEST +2 - - +@TEST-END-FILE +@TEST-START-FILE input2.log +#separator \x09 +#fields i s ss +#types int sting string +1 TEST - +2 TEST TEST +@TEST-END-FILE + +@load frameworks/communication/listen + + +module A; + +type Idx: record { + i: int; +}; + +type Val: record { + s: string; + ss: string; +}; + +global servers: table[int] of Val = table(); + +global outfile: file; + +global try: count; + +event line(description: Input::TableDescription, tpe: Input::Event, left: Idx, right: Val) + { + print outfile, "============EVENT============"; + print outfile, "Description"; + print outfile, description; + print outfile, "Type"; + print outfile, tpe; + print outfile, "Left"; + print outfile, left; + print outfile, "Right"; + print outfile, right; + } + +event bro_init() + { + outfile = open("../out"); + try = 0; + # first read in the old stuff into the table... + Input::add_table([$source="../input.log", $mode=Input::REREAD, $name="ssh", $idx=Idx, $val=Val, $destination=servers, $ev=line, + $pred(typ: Input::Event, left: Idx, right: Val) = { + print outfile, "============PREDICATE============"; + print outfile, typ; + print outfile, left; + print outfile, right; + return T; + } + ]); + } + + +event Input::update_finished(name: string, source: string) + { + print outfile, "==========SERVERS============"; + print outfile, servers; + + try = try + 1; + if ( try == 2 ) + { + print outfile, "done"; + close(outfile); + Input::remove("input"); + terminate(); + } + } From f133e8808a0f8b199f47141f497cb33ed6a6955f Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Sun, 26 Aug 2012 22:00:37 -0700 Subject: [PATCH 09/20] ok, this one might really be a bit too big for 2.1 Give all kinds of errors when encountering invalid numbers (like out-of-range-warnings, etc). --- src/input/readers/Ascii.cc | 57 ++++++++++++++++--- src/input/readers/Ascii.h | 1 + .../out | 3 +- .../.stderrwithoutfirstline | 8 +++ .../out | 5 ++ .../base/frameworks/input/bignumber.bro | 1 + .../base/frameworks/input/invalidnumbers.bro | 55 ++++++++++++++++++ 7 files changed, 122 insertions(+), 8 deletions(-) create mode 100644 testing/btest/Baseline/scripts.base.frameworks.input.invalidnumbers/.stderrwithoutfirstline create mode 100644 testing/btest/Baseline/scripts.base.frameworks.input.invalidnumbers/out create mode 100644 testing/btest/scripts/base/frameworks/input/invalidnumbers.bro diff --git a/src/input/readers/Ascii.cc b/src/input/readers/Ascii.cc index 4bf82c6a13..1923532103 100644 --- a/src/input/readers/Ascii.cc +++ b/src/input/readers/Ascii.cc @@ -11,6 +11,7 @@ #include #include #include +#include using namespace input::reader; using threading::Value; @@ -209,6 +210,34 @@ bool Ascii::GetLine(string& str) return false; } +bool Ascii::CheckNumberError(const string & s, const char * end) + { + + if ( s.length() == 0 ) + { + Error("Got empty string for number field"); + return true; + } + + if ( end == s.c_str() ) { + Error(Fmt("String '%s' contained no parseable number", s.c_str())); + return true; + } + + if ( *end != '\0' ) + Error(Fmt("Number '%s' contained non-numeric trailing characters. Ignored trailing characters '%s'", s.c_str(), end)); + + if ( errno == EINVAL ) + { + Error(Fmt("String '%s' could not be converted to a number", s.c_str())); + return true; + } + else if ( errno == ERANGE ) + Error(Fmt("Number '%s' out of supported range. Number was truncated", s.c_str())); + + return false; + } + Value* Ascii::EntryToVal(string s, FieldMapping field) { @@ -216,6 +245,8 @@ Value* Ascii::EntryToVal(string s, FieldMapping field) return new Value(field.type, false); Value* val = new Value(field.type, true); + char* end; + errno = 0; switch ( field.type ) { case TYPE_ENUM: @@ -239,22 +270,31 @@ Value* Ascii::EntryToVal(string s, FieldMapping field) break; case TYPE_INT: - val->val.int_val = strtoll(s.c_str(), (char**) NULL, 10); + val->val.int_val = strtoll(s.c_str(), &end, 10); + if ( CheckNumberError(s, end) ) + return 0; break; case TYPE_DOUBLE: case TYPE_TIME: case TYPE_INTERVAL: - val->val.double_val = atof(s.c_str()); + val->val.double_val = strtod(s.c_str(), &end); + if ( CheckNumberError(s, end) ) + return 0; break; case TYPE_COUNT: case TYPE_COUNTER: - val->val.uint_val = strtoull(s.c_str(),(char**) NULL, 10); + val->val.uint_val = strtoull(s.c_str(), &end, 10); + if ( CheckNumberError(s, end) ) + return 0; break; - + case TYPE_PORT: - val->val.port_val.port = atoi(s.c_str()); + val->val.port_val.port = strtoull(s.c_str(), &end, 10); + if ( CheckNumberError(s, end) ) + return 0; + val->val.port_val.proto = TRANSPORT_UNKNOWN; break; @@ -268,8 +308,11 @@ Value* Ascii::EntryToVal(string s, FieldMapping field) return 0; } - int width = atoi(s.substr(pos+1).c_str()); + uint8_t width = (uint8_t) strtol(s.substr(pos+1).c_str(), &end, 10); string addr = s.substr(0, pos); + + if ( CheckNumberError(s, end) ) + return 0; val->val.subnet_val.prefix = StringToAddr(addr); val->val.subnet_val.length = width; @@ -490,7 +533,7 @@ bool Ascii::DoUpdate() Value* val = EntryToVal(stringfields[(*fit).position], *fit); if ( val == 0 ) { - Error("Could not convert String value to Val"); + Error(Fmt("Could not convert line '%s' to Val. Aborting file read.", line.c_str())); return false; } diff --git a/src/input/readers/Ascii.h b/src/input/readers/Ascii.h index e1506cbe82..2228e491b0 100644 --- a/src/input/readers/Ascii.h +++ b/src/input/readers/Ascii.h @@ -48,6 +48,7 @@ private: bool ReadHeader(bool useCached); bool GetLine(string& str); threading::Value* EntryToVal(string s, FieldMapping type); + bool CheckNumberError(const string & s, const char * end); ifstream* file; time_t mtime; diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.bignumber/out b/testing/btest/Baseline/scripts.base.frameworks.input.bignumber/out index ab095ca36c..8b95ed8b19 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.input.bignumber/out +++ b/testing/btest/Baseline/scripts.base.frameworks.input.bignumber/out @@ -1,3 +1,4 @@ { -[9223372036854775800] = [c=18446744073709551612] +[9223372036854775800] = [c=18446744073709551612], +[-9223372036854775800] = [c=18446744073709551612] } diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.invalidnumbers/.stderrwithoutfirstline b/testing/btest/Baseline/scripts.base.frameworks.input.invalidnumbers/.stderrwithoutfirstline new file mode 100644 index 0000000000..bd32495a6f --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.invalidnumbers/.stderrwithoutfirstline @@ -0,0 +1,8 @@ +error: ../input.log/Input::READER_ASCII: Number '12129223372036854775800' out of supported range. Number was truncated +error: ../input.log/Input::READER_ASCII: Number '121218446744073709551612' out of supported range. Number was truncated +error: ../input.log/Input::READER_ASCII: Number '9223372036854775801TEXTHERE' contained non-numeric trailing characters. Ignored trailing characters 'TEXTHERE' +error: ../input.log/Input::READER_ASCII: Number '1Justtext' contained non-numeric trailing characters. Ignored trailing characters 'Justtext' +error: ../input2.log/Input::READER_ASCII: String 'Justtext' contained no parseable number +error: ../input2.log/Input::READER_ASCII: Could not convert line 'Justtext 1' to Val. Aborting file read. +received termination signal +>>> diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.invalidnumbers/out b/testing/btest/Baseline/scripts.base.frameworks.input.invalidnumbers/out new file mode 100644 index 0000000000..9be82c13a9 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.invalidnumbers/out @@ -0,0 +1,5 @@ +{ +[9223372036854775807] = [c=18446744073709551615], +[9223372036854775800] = [c=4], +[9223372036854775801] = [c=1] +} diff --git a/testing/btest/scripts/base/frameworks/input/bignumber.bro b/testing/btest/scripts/base/frameworks/input/bignumber.bro index 519992be05..250f84bbb2 100644 --- a/testing/btest/scripts/base/frameworks/input/bignumber.bro +++ b/testing/btest/scripts/base/frameworks/input/bignumber.bro @@ -10,6 +10,7 @@ #fields i c #types int count 9223372036854775800 18446744073709551612 +-9223372036854775800 18446744073709551612 @TEST-END-FILE @load frameworks/communication/listen diff --git a/testing/btest/scripts/base/frameworks/input/invalidnumbers.bro b/testing/btest/scripts/base/frameworks/input/invalidnumbers.bro new file mode 100644 index 0000000000..7914b53d94 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/input/invalidnumbers.bro @@ -0,0 +1,55 @@ +# (uses listen.bro just to ensure input sources are more reliably fully-read). +# @TEST-SERIALIZE: comm +# +# @TEST-EXEC: btest-bg-run bro bro -b %INPUT +# @TEST-EXEC: btest-bg-wait -k 5 +# @TEST-EXEC: btest-diff out +# @TEST-EXEC: sed 1d .stderr > .stderrwithoutfirstline +# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff .stderrwithoutfirstline + +@TEST-START-FILE input.log +#separator \x09 +#fields i c +#types int count +12129223372036854775800 121218446744073709551612 +9223372036854775801TEXTHERE 1Justtext +9223372036854775800 -18446744073709551612 +@TEST-END-FILE + +@TEST-START-FILE input2.log +#separator \x09 +#fields i c +#types int count +Justtext 1 +@TEST-END-FILE + + +@load frameworks/communication/listen + +global outfile: file; + +module A; + +type Idx: record { + i: int; +}; + +type Val: record { + c: count; +}; + +global servers: table[int] of Val = table(); + +event bro_init() + { + outfile = open("../out"); + # first read in the old stuff into the table... + Input::add_table([$source="../input.log", $name="ssh", $idx=Idx, $val=Val, $destination=servers]); + Input::remove("ssh"); + } + +event Input::update_finished(name: string, source:string) + { + print outfile, servers; + Input::add_table([$source="../input2.log", $name="ssh2", $idx=Idx, $val=Val, $destination=servers]); + } From a4ca5b0d829fa61a706913848620d85f2b125dd6 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Mon, 27 Aug 2012 09:49:57 -0700 Subject: [PATCH 10/20] fix handline of sets only containing a zero-length string. Thank you Robin... --- src/input/readers/Ascii.cc | 7 ++++--- .../scripts.base.frameworks.input.setspecialcases/out | 3 +++ .../scripts/base/frameworks/input/setspecialcases.bro | 1 + 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/input/readers/Ascii.cc b/src/input/readers/Ascii.cc index 4bf82c6a13..f1664a555a 100644 --- a/src/input/readers/Ascii.cc +++ b/src/input/readers/Ascii.cc @@ -345,9 +345,10 @@ Value* Ascii::EntryToVal(string s, FieldMapping field) pos++; } - // test if the string ends with a set_separator... if it does we have to push an zero-lenght - // val on top of it. - if ( *s.rbegin() == set_separator[0] ) + // test if the string ends with a set_separator... or if the complete string is + // empty. + // In either of these cases we have to push an empty val on top of it. + if ( s.empty() || *s.rbegin() == set_separator[0] ) { lvals[pos] = EntryToVal("", field.subType()); if ( lvals[pos] == 0 ) diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.setspecialcases/out b/testing/btest/Baseline/scripts.base.frameworks.input.setspecialcases/out index 28d1e025bf..62229f7f37 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.input.setspecialcases/out +++ b/testing/btest/Baseline/scripts.base.frameworks.input.setspecialcases/out @@ -7,6 +7,9 @@ testing , testing }, s=[testing, ]], +[6] = [s={ + +}, s=[]], [1] = [s={ testing,testing,testing, }, s=[testing,testing,testing,]], diff --git a/testing/btest/scripts/base/frameworks/input/setspecialcases.bro b/testing/btest/scripts/base/frameworks/input/setspecialcases.bro index 29819a795f..239bdfe7e7 100644 --- a/testing/btest/scripts/base/frameworks/input/setspecialcases.bro +++ b/testing/btest/scripts/base/frameworks/input/setspecialcases.bro @@ -13,6 +13,7 @@ 3 ,testing ,testing 4 testing, testing, 5 ,,, ,,, +6 @TEST-END-FILE From 5c486dae7e82ce308a6553a5dc53afb2fcae9ed8 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Mon, 27 Aug 2012 10:54:33 -0700 Subject: [PATCH 11/20] Ok, this one was a little bit sneaky. If I understand things correctly, calling other string functions on an stl string may alter the contents of the buffer to which earlier .c_str()-calls pointed. Kind of makes sense when thinking about it. Basically moving around a few lines should fix this. (And thank you again Robin) --- src/input/readers/Ascii.cc | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/input/readers/Ascii.cc b/src/input/readers/Ascii.cc index 1923532103..276391ef84 100644 --- a/src/input/readers/Ascii.cc +++ b/src/input/readers/Ascii.cc @@ -213,6 +213,9 @@ bool Ascii::GetLine(string& str) bool Ascii::CheckNumberError(const string & s, const char * end) { + bool endnotnull = (*end != '\0'); // do this check first, before executing s.c_str() or similar. + // otherwise the value to which *end is pointing at the moment might be gone... + if ( s.length() == 0 ) { Error("Got empty string for number field"); @@ -224,7 +227,7 @@ bool Ascii::CheckNumberError(const string & s, const char * end) return true; } - if ( *end != '\0' ) + if ( endnotnull ) Error(Fmt("Number '%s' contained non-numeric trailing characters. Ignored trailing characters '%s'", s.c_str(), end)); if ( errno == EINVAL ) @@ -309,11 +312,12 @@ Value* Ascii::EntryToVal(string s, FieldMapping field) } uint8_t width = (uint8_t) strtol(s.substr(pos+1).c_str(), &end, 10); - string addr = s.substr(0, pos); - + if ( CheckNumberError(s, end) ) return 0; + string addr = s.substr(0, pos); + val->val.subnet_val.prefix = StringToAddr(addr); val->val.subnet_val.length = width; break; From 56fa56ffa946581d7b4806b494821fe79f9974dc Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Mon, 27 Aug 2012 11:38:20 -0700 Subject: [PATCH 12/20] ...and another small change to error handling -> now errors in single lines do not kill processing, but simply ignore the line, log it, and continue. --- src/input/readers/Ascii.cc | 28 ++++++++++++++++--- .../.stderrwithoutfirstline | 12 ++++---- .../out | 1 - .../base/frameworks/input/invalidnumbers.bro | 11 ++------ 4 files changed, 32 insertions(+), 20 deletions(-) diff --git a/src/input/readers/Ascii.cc b/src/input/readers/Ascii.cc index 276391ef84..9c25953864 100644 --- a/src/input/readers/Ascii.cc +++ b/src/input/readers/Ascii.cc @@ -228,7 +228,7 @@ bool Ascii::CheckNumberError(const string & s, const char * end) } if ( endnotnull ) - Error(Fmt("Number '%s' contained non-numeric trailing characters. Ignored trailing characters '%s'", s.c_str(), end)); + Warning(Fmt("Number '%s' contained non-numeric trailing characters. Ignored trailing characters '%s'", s.c_str(), end)); if ( errno == EINVAL ) { @@ -236,7 +236,10 @@ bool Ascii::CheckNumberError(const string & s, const char * end) return true; } else if ( errno == ERANGE ) - Error(Fmt("Number '%s' out of supported range. Number was truncated", s.c_str())); + { + Error(Fmt("Number '%s' out of supported range.", s.c_str())); + return true; + } return false; } @@ -492,6 +495,7 @@ bool Ascii::DoUpdate() while ( GetLine(line ) ) { // split on tabs + bool error = false; istringstream splitstream(line); map stringfields; @@ -537,8 +541,9 @@ bool Ascii::DoUpdate() Value* val = EntryToVal(stringfields[(*fit).position], *fit); if ( val == 0 ) { - Error(Fmt("Could not convert line '%s' to Val. Aborting file read.", line.c_str())); - return false; + Error(Fmt("Could not convert line '%s' to Val. Ignoring line.", line.c_str())); + error = true; + break; } if ( (*fit).secondary_position != -1 ) @@ -555,6 +560,21 @@ bool Ascii::DoUpdate() fpos++; } + + if ( error ) + { + // encountered non-fatal error. ignoring line. + // first - delete all successfully read fields and the array structure. + + for ( int i = 0; i < fpos; i++ ) + delete fields[fpos]; + + delete[] fields; + continue; + } + + + //printf("fpos: %d, second.num_fields: %d\n", fpos, (*it).second.num_fields); assert ( fpos == NumFields() ); diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.invalidnumbers/.stderrwithoutfirstline b/testing/btest/Baseline/scripts.base.frameworks.input.invalidnumbers/.stderrwithoutfirstline index bd32495a6f..3ef51e40f2 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.input.invalidnumbers/.stderrwithoutfirstline +++ b/testing/btest/Baseline/scripts.base.frameworks.input.invalidnumbers/.stderrwithoutfirstline @@ -1,8 +1,8 @@ -error: ../input.log/Input::READER_ASCII: Number '12129223372036854775800' out of supported range. Number was truncated -error: ../input.log/Input::READER_ASCII: Number '121218446744073709551612' out of supported range. Number was truncated -error: ../input.log/Input::READER_ASCII: Number '9223372036854775801TEXTHERE' contained non-numeric trailing characters. Ignored trailing characters 'TEXTHERE' -error: ../input.log/Input::READER_ASCII: Number '1Justtext' contained non-numeric trailing characters. Ignored trailing characters 'Justtext' -error: ../input2.log/Input::READER_ASCII: String 'Justtext' contained no parseable number -error: ../input2.log/Input::READER_ASCII: Could not convert line 'Justtext 1' to Val. Aborting file read. +error: ../input.log/Input::READER_ASCII: Number '12129223372036854775800' out of supported range. +error: ../input.log/Input::READER_ASCII: Could not convert line '12129223372036854775800 121218446744073709551612' to Val. Ignoring line. +warning: ../input.log/Input::READER_ASCII: Number '9223372036854775801TEXTHERE' contained non-numeric trailing characters. Ignored trailing characters 'TEXTHERE' +warning: ../input.log/Input::READER_ASCII: Number '1Justtext' contained non-numeric trailing characters. Ignored trailing characters 'Justtext' +error: ../input.log/Input::READER_ASCII: String 'Justtext' contained no parseable number +error: ../input.log/Input::READER_ASCII: Could not convert line 'Justtext 1' to Val. Ignoring line. received termination signal >>> diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.invalidnumbers/out b/testing/btest/Baseline/scripts.base.frameworks.input.invalidnumbers/out index 9be82c13a9..56b2736006 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.input.invalidnumbers/out +++ b/testing/btest/Baseline/scripts.base.frameworks.input.invalidnumbers/out @@ -1,5 +1,4 @@ { -[9223372036854775807] = [c=18446744073709551615], [9223372036854775800] = [c=4], [9223372036854775801] = [c=1] } diff --git a/testing/btest/scripts/base/frameworks/input/invalidnumbers.bro b/testing/btest/scripts/base/frameworks/input/invalidnumbers.bro index 7914b53d94..3c755f1d08 100644 --- a/testing/btest/scripts/base/frameworks/input/invalidnumbers.bro +++ b/testing/btest/scripts/base/frameworks/input/invalidnumbers.bro @@ -13,17 +13,10 @@ #types int count 12129223372036854775800 121218446744073709551612 9223372036854775801TEXTHERE 1Justtext +Justtext 1 9223372036854775800 -18446744073709551612 @TEST-END-FILE -@TEST-START-FILE input2.log -#separator \x09 -#fields i c -#types int count -Justtext 1 -@TEST-END-FILE - - @load frameworks/communication/listen global outfile: file; @@ -51,5 +44,5 @@ event bro_init() event Input::update_finished(name: string, source:string) { print outfile, servers; - Input::add_table([$source="../input2.log", $name="ssh2", $idx=Idx, $val=Val, $destination=servers]); + terminate(); } From 26f5aee7f6376d65031517efa78a1a6e7cbf1b46 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Tue, 28 Aug 2012 00:44:39 -0700 Subject: [PATCH 13/20] on 32-bit machines only unsigned long longs are 64-bits long. Not just unsigned longs... Note that this means that up to now all outputs (including logs) of counts > 32 bits were broken on 32-bit systems. --- src/modp_numtoa.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/modp_numtoa.c b/src/modp_numtoa.c index 6deb8a70ed..6fa49b460f 100644 --- a/src/modp_numtoa.c +++ b/src/modp_numtoa.c @@ -56,7 +56,7 @@ void modp_uitoa10(uint32_t value, char* str) void modp_litoa10(int64_t value, char* str) { char* wstr=str; - unsigned long uvalue = (value < 0) ? -value : value; + unsigned long long uvalue = (value < 0) ? -value : value; // Conversion. Number is reversed. do *wstr++ = (char)(48 + (uvalue % 10)); while(uvalue /= 10); From 03f5795095642f89e11265ed36fda17f97a97ea9 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Tue, 28 Aug 2012 07:33:05 -0700 Subject: [PATCH 14/20] parse 64-bit consts correctly. --- src/scan.l | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/scan.l b/src/scan.l index 645ce659cd..3148ba58ad 100644 --- a/src/scan.l +++ b/src/scan.l @@ -439,7 +439,7 @@ F RET_CONST(new Val(false, TYPE_BOOL)) {D} { // TODO: check if we can use strtoull instead of atol, // and similarly for {HEX}. - RET_CONST(new Val(static_cast(atol(yytext)), + RET_CONST(new Val(static_cast(strtoll(yytext, (char**) NULL, 10)), TYPE_COUNT)) } {FLOAT} RET_CONST(new Val(atof(yytext), TYPE_DOUBLE)) @@ -483,7 +483,7 @@ F RET_CONST(new Val(false, TYPE_BOOL)) ({D}"."){3}{D} RET_CONST(new AddrVal(yytext)) -"0x"{HEX}+ RET_CONST(new Val(static_cast(strtol(yytext, 0, 16)), TYPE_COUNT)) +"0x"{HEX}+ RET_CONST(new Val(static_cast(strtoull(yytext, 0, 16)), TYPE_COUNT)) {H}("."{H})+ RET_CONST(dns_mgr->LookupHost(yytext)) From b815b7ca5c133960102409d32bb492080112dde0 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Tue, 28 Aug 2012 10:57:21 -0500 Subject: [PATCH 15/20] Fix uninitialized value for 'is_partial' in TCP analyzer. This led to non-deterministic behavior in cases where the first packet analyzed wasn't from the originator side (see the conditionals in TCP_Analyzer::CheckFlagCombos()). The 'short' test in private test suite showed this behavior most often. --- src/TCP.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/TCP.cc b/src/TCP.cc index 57e4449bf8..555adf1b57 100644 --- a/src/TCP.cc +++ b/src/TCP.cc @@ -46,6 +46,7 @@ TCP_Analyzer::TCP_Analyzer(Connection* conn) finished = 0; reassembling = 0; first_packet_seen = 0; + is_partial = 0; orig = new TCP_Endpoint(this, 1); resp = new TCP_Endpoint(this, 0); From 393ded1efe378a3f2109ccf49623e5050c12e048 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Tue, 28 Aug 2012 09:19:33 -0700 Subject: [PATCH 16/20] Set VERSION to 2.1-rc3 so that we don't get confused. --- CHANGES | 2 +- VERSION | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGES b/CHANGES index 02d7d74046..7df00f352c 100644 --- a/CHANGES +++ b/CHANGES @@ -1,5 +1,5 @@ -2.1 | 2012-08-24 15:11:49 -0700 +2.1-rc3 | 2012-08-24 15:11:49 -0700 * Input framework fixes, including: (Bernhard Amann) diff --git a/VERSION b/VERSION index 879b416e60..1537f322a9 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.1 +2.1-rc3 From cc49193f93ba8c60b65b61047a0874982ad93db3 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Tue, 28 Aug 2012 13:11:12 -0500 Subject: [PATCH 17/20] Remove automatic use of gperftools on non-Linux systems. --enable-perftools must now explicity be supplied to ./configure on non-Linux systems to link against the tcmalloc library that a gperftools installation provides. Linux systems still automatically link it if it's found. The rationale is that gperftools was developed and most throroughly tested on Linux so it's safer there. There especially seems to be potential problems with gperftools on OS X (e.g. see http://code.google.com/p/gperftools/issues/detail?id=413), and Bro currently doesn't work with gpertools there using clang or gcc. --- CMakeLists.txt | 29 ++++++++++++++++++----------- configure | 7 +++++++ 2 files changed, 25 insertions(+), 11 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index f667c0cfe0..2c8a726a1a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -88,24 +88,30 @@ if (LIBGEOIP_FOUND) list(APPEND OPTLIBS ${LibGeoIP_LIBRARY}) endif () -set(USE_PERFTOOLS false) +set(HAVE_PERFTOOLS false) set(USE_PERFTOOLS_DEBUG false) +set(USE_PERFTOOLS_TCMALLOC false) if (NOT DISABLE_PERFTOOLS) find_package(GooglePerftools) endif () if (GOOGLEPERFTOOLS_FOUND) - include_directories(BEFORE ${GooglePerftools_INCLUDE_DIR}) - set(USE_PERFTOOLS true) + set(HAVE_PERFTOOLS true) + # Non-Linux systems may not be well-supported by gperftools, so + # require explicit request from user to enable it in that case. + if (${CMAKE_SYSTEM_NAME} MATCHES "Linux" OR ENABLE_PERFTOOLS) + set(USE_PERFTOOLS_TCMALLOC true) - if (ENABLE_PERFTOOLS_DEBUG) - # Enable heap debugging with perftools. - set(USE_PERFTOOLS_DEBUG true) - list(APPEND OPTLIBS ${GooglePerftools_LIBRARIES_DEBUG}) - else () - # Link in tcmalloc for better performance. - list(APPEND OPTLIBS ${GooglePerftools_LIBRARIES}) + if (ENABLE_PERFTOOLS_DEBUG) + # Enable heap debugging with perftools. + set(USE_PERFTOOLS_DEBUG true) + include_directories(BEFORE ${GooglePerftools_INCLUDE_DIR}) + list(APPEND OPTLIBS ${GooglePerftools_LIBRARIES_DEBUG}) + else () + # Link in tcmalloc for better performance. + list(APPEND OPTLIBS ${GooglePerftools_LIBRARIES}) + endif () endif () endif () @@ -224,7 +230,8 @@ message( "\nAux. Tools: ${INSTALL_AUX_TOOLS}" "\n" "\nGeoIP: ${USE_GEOIP}" - "\nGoogle perftools: ${USE_PERFTOOLS}" + "\ngperftools found: ${HAVE_PERFTOOLS}" + "\n tcmalloc: ${USE_PERFTOOLS_TCMALLOC}" "\n debugging: ${USE_PERFTOOLS_DEBUG}" "\ncURL: ${USE_CURL}" "\n" diff --git a/configure b/configure index b4ca606103..8e4aaa8425 100755 --- a/configure +++ b/configure @@ -29,6 +29,8 @@ Usage: $0 [OPTION]... [VAR=VALUE]... Optional Features: --enable-debug compile in debugging mode --enable-mobile-ipv6 analyze mobile IPv6 features defined by RFC 6275 + --enable-perftools force use of Google perftools on non-Linux systems + (automatically on when perftools is present on Linux) --enable-perftools-debug use Google's perftools for debugging --disable-broccoli don't build or install the Broccoli library --disable-broctl don't install Broctl @@ -98,6 +100,7 @@ append_cache_entry PY_MOD_INSTALL_DIR PATH $prefix/lib/broctl append_cache_entry BRO_SCRIPT_INSTALL_PATH STRING $prefix/share/bro append_cache_entry BRO_ETC_INSTALL_DIR PATH $prefix/etc append_cache_entry ENABLE_DEBUG BOOL false +append_cache_entry ENABLE_PERFTOOLS BOOL false append_cache_entry ENABLE_PERFTOOLS_DEBUG BOOL false append_cache_entry BinPAC_SKIP_INSTALL BOOL true append_cache_entry BUILD_SHARED_LIBS BOOL true @@ -146,7 +149,11 @@ while [ $# -ne 0 ]; do --enable-mobile-ipv6) append_cache_entry ENABLE_MOBILE_IPV6 BOOL true ;; + --enable-perftools) + append_cache_entry ENABLE_PERFTOOLS BOOL true + ;; --enable-perftools-debug) + append_cache_entry ENABLE_PERFTOOLS BOOL true append_cache_entry ENABLE_PERFTOOLS_DEBUG BOOL true ;; --disable-broccoli) From e4b7ffa8ac0718ace6d37371c8283efc50502c4f Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Tue, 28 Aug 2012 16:44:30 -0700 Subject: [PATCH 18/20] Updating CHANGES and VERSION. --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 2097bb1d94..9459d4ba2a 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.1-rc3-5 +1.1 From b915db86d5c7b30c7d50d8b5ddfbbbdadd32107d Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Tue, 28 Aug 2012 16:46:37 -0700 Subject: [PATCH 19/20] Updating CHANGES and VERSION. --- CHANGES | 2 +- VERSION | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGES b/CHANGES index 232e2faa19..516c36974e 100644 --- a/CHANGES +++ b/CHANGES @@ -1,5 +1,5 @@ -1.1 | 2012-08-28 16:29:30 -0700 +2.1 | 2012-08-28 16:46:42 -0700 * Remove automatic use of gperftools on non-Linux systems. --enable-perftools must now explicity be supplied to ./configure diff --git a/VERSION b/VERSION index 9459d4ba2a..879b416e60 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.1 +2.1 From 22cf75dae553dc2aa2a103bf7721cd466b764d64 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Wed, 29 Aug 2012 08:09:44 -0700 Subject: [PATCH 20/20] Two fixes. - Typo in recent scanner fix. - Make bif.identify_magic robust against FreeBSD's libmagic config. --- CHANGES | 3 +++ src/scan.l | 4 +--- testing/btest/Baseline/bifs.identify_data/out | 2 +- testing/btest/bifs/identify_data.bro | 2 +- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/CHANGES b/CHANGES index 516c36974e..f8e4444f1d 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,9 @@ 2.1 | 2012-08-28 16:46:42 -0700 + * Make bif.identify_magic robust against FreeBSD's libmagic config. + (Robin Sommer) + * Remove automatic use of gperftools on non-Linux systems. --enable-perftools must now explicity be supplied to ./configure on non-Linux systems to link against the tcmalloc library. diff --git a/src/scan.l b/src/scan.l index 3148ba58ad..1b3d09f879 100644 --- a/src/scan.l +++ b/src/scan.l @@ -437,9 +437,7 @@ F RET_CONST(new Val(false, TYPE_BOOL)) } {D} { - // TODO: check if we can use strtoull instead of atol, - // and similarly for {HEX}. - RET_CONST(new Val(static_cast(strtoll(yytext, (char**) NULL, 10)), + RET_CONST(new Val(static_cast(strtoul(yytext, (char**) NULL, 10)), TYPE_COUNT)) } {FLOAT} RET_CONST(new Val(atof(yytext), TYPE_DOUBLE)) diff --git a/testing/btest/Baseline/bifs.identify_data/out b/testing/btest/Baseline/bifs.identify_data/out index a2872877f9..1cadefbf6e 100644 --- a/testing/btest/Baseline/bifs.identify_data/out +++ b/testing/btest/Baseline/bifs.identify_data/out @@ -1,4 +1,4 @@ ASCII text, with no line terminators text/plain; charset=us-ascii -PNG image data +PNG image image/png; charset=binary diff --git a/testing/btest/bifs/identify_data.bro b/testing/btest/bifs/identify_data.bro index 11824b5e85..39f289d40b 100644 --- a/testing/btest/bifs/identify_data.bro +++ b/testing/btest/bifs/identify_data.bro @@ -1,5 +1,5 @@ # -# @TEST-EXEC: bro %INPUT >out +# @TEST-EXEC: bro %INPUT | sed 's/PNG image data/PNG image/g' >out # @TEST-EXEC: btest-diff out event bro_init()