From 116079a9ad5e4e9d096be36e97879b69f036b4a2 Mon Sep 17 00:00:00 2001 From: Johanna Amann Date: Thu, 18 Jan 2018 11:09:12 -0800 Subject: [PATCH 1/4] Make parsing of booleans a little bit more lenient. This makes the input framework (and everything else that uses the Ascii parser) accept 0 and 1 as valid values for booleans. --- src/threading/formatters/Ascii.cc | 4 ++-- .../btest/Baseline/scripts.base.frameworks.input.basic/out | 2 +- testing/btest/scripts/base/frameworks/input/basic.bro | 5 +++-- testing/btest/scripts/base/frameworks/input/reread.bro | 2 +- 4 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/threading/formatters/Ascii.cc b/src/threading/formatters/Ascii.cc index e96290e793..6491063585 100644 --- a/src/threading/formatters/Ascii.cc +++ b/src/threading/formatters/Ascii.cc @@ -227,9 +227,9 @@ threading::Value* Ascii::ParseValue(const string& s, const string& name, TypeTag } case TYPE_BOOL: - if ( s == "T" ) + if ( s == "T" || s == "1" ) val->val.int_val = 1; - else if ( s == "F" ) + else if ( s == "F" || s == "0" ) val->val.int_val = 0; else { diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.basic/out b/testing/btest/Baseline/scripts.base.frameworks.input.basic/out index 3f288d5c54..5cc19d85a2 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.input.basic/out +++ b/testing/btest/Baseline/scripts.base.frameworks.input.basic/out @@ -1,5 +1,5 @@ { -[-42] = [b=T, e=SSH::LOG, c=21, p=123/unknown, pp=5/icmp, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, ns=4242, sc={ +[-42] = [b=T, bt=T, e=SSH::LOG, c=21, p=123/unknown, pp=5/icmp, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, ns=4242, sc={ 2, 4, 1, diff --git a/testing/btest/scripts/base/frameworks/input/basic.bro b/testing/btest/scripts/base/frameworks/input/basic.bro index e77a418f0d..356b87d70b 100644 --- a/testing/btest/scripts/base/frameworks/input/basic.bro +++ b/testing/btest/scripts/base/frameworks/input/basic.bro @@ -7,9 +7,9 @@ redef exit_only_after_terminate = T; @TEST-START-FILE input.log #separator \x09 #path ssh -#fields b i e c p pp sn a d t iv s sc ss se vc ve ns +#fields b bt i e c p pp sn a d t iv s sc ss se vc ve ns #types bool int enum count port port subnet addr double time interval string table table table vector vector string -T -42 SSH::LOG 21 123 5/icmp 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY 4242 +T 1 -42 SSH::LOG 21 123 5/icmp 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY 4242 @TEST-END-FILE @load base/protocols/ssh @@ -26,6 +26,7 @@ type Idx: record { type Val: record { b: bool; + bt: bool; e: Log::ID; c: count; p: port; diff --git a/testing/btest/scripts/base/frameworks/input/reread.bro b/testing/btest/scripts/base/frameworks/input/reread.bro index 4199093543..e4bb09df39 100644 --- a/testing/btest/scripts/base/frameworks/input/reread.bro +++ b/testing/btest/scripts/base/frameworks/input/reread.bro @@ -43,7 +43,7 @@ T -42 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz F -43 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a} F -44 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a} F -45 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a} -F -46 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a} +0 -46 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a} F -47 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a} F -48 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a} @TEST-END-FILE From 26ea1999ec7b3a7b4ecf2cecdcb8c09dc827b537 Mon Sep 17 00:00:00 2001 From: Johanna Amann Date: Fri, 10 Aug 2018 11:23:33 -0700 Subject: [PATCH 2/4] Ascii formatter: do not complain about port text. The ascii formatter already was happy to read ports in the form "42/tcp"; however it emitted a warning message for each line. This patch fixes this and adds a bit more testing for the existing behavior. --- src/threading/formatters/Ascii.cc | 14 ++++++ .../bro..stderr | 1 + .../bro.config.log | 34 +++++++------- .../bro..stderr | 2 + .../bro..stdout | 4 ++ .../scripts/base/frameworks/config/basic.bro | 5 +++ .../base/frameworks/input/port-embedded.bro | 44 +++++++++++++++++++ 7 files changed, 88 insertions(+), 16 deletions(-) create mode 100644 testing/btest/Baseline/scripts.base.frameworks.config.basic/bro..stderr create mode 100644 testing/btest/Baseline/scripts.base.frameworks.input.port-embedded/bro..stderr create mode 100644 testing/btest/Baseline/scripts.base.frameworks.input.port-embedded/bro..stdout create mode 100644 testing/btest/scripts/base/frameworks/input/port-embedded.bro diff --git a/src/threading/formatters/Ascii.cc b/src/threading/formatters/Ascii.cc index 6491063585..76b1ba4b04 100644 --- a/src/threading/formatters/Ascii.cc +++ b/src/threading/formatters/Ascii.cc @@ -261,8 +261,10 @@ threading::Value* Ascii::ParseValue(const string& s, const string& name, TypeTag break; case TYPE_PORT: + { val->val.port_val.proto = TRANSPORT_UNKNOWN; pos = s.find('/'); + string numberpart; if ( pos != std::string::npos && s.length() > pos + 1 ) { auto proto = s.substr(pos+1); @@ -272,10 +274,22 @@ threading::Value* Ascii::ParseValue(const string& s, const string& name, TypeTag val->val.port_val.proto = TRANSPORT_UDP; else if ( strtolower(proto) == "icmp" ) val->val.port_val.proto = TRANSPORT_ICMP; + else if ( strtolower(proto) == "unknown" ) + val->val.port_val.proto = TRANSPORT_UNKNOWN; + else + GetThread()->Warning(GetThread()->Fmt("Port '%s' contained unknown protocol '%s'", s.c_str(), proto.c_str())); + } + + // make the string end at the position of "/"; + if ( pos != std::string::npos && pos > 0 ) + { + numberpart = s.substr(0, pos); + start = numberpart.c_str(); } val->val.port_val.port = strtoull(start, &end, 10); if ( CheckNumberError(start, end) ) goto parse_error; + } break; case TYPE_SUBNET: diff --git a/testing/btest/Baseline/scripts.base.frameworks.config.basic/bro..stderr b/testing/btest/Baseline/scripts.base.frameworks.config.basic/bro..stderr new file mode 100644 index 0000000000..977e8fc37a --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.config.basic/bro..stderr @@ -0,0 +1 @@ +received termination signal diff --git a/testing/btest/Baseline/scripts.base.frameworks.config.basic/bro.config.log b/testing/btest/Baseline/scripts.base.frameworks.config.basic/bro.config.log index b1e03411e5..0d96d0f111 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.config.basic/bro.config.log +++ b/testing/btest/Baseline/scripts.base.frameworks.config.basic/bro.config.log @@ -3,21 +3,23 @@ #empty_field (empty) #unset_field - #path config -#open 2017-10-11-20-23-11 +#open 2018-08-10-18-16-52 #fields ts id old_value new_value location #types time string string string string -1507753391.587107 testbool T F ../configfile -1507753391.587107 testcount 0 1 ../configfile -1507753391.587107 testcount 1 2 ../configfile -1507753391.587107 testint 0 -1 ../configfile -1507753391.587107 testenum SSH::LOG Conn::LOG ../configfile -1507753391.587107 testport 42/tcp 45/unknown ../configfile -1507753391.587107 testaddr 127.0.0.1 127.0.0.1 ../configfile -1507753391.587107 testaddr 127.0.0.1 2607:f8b0:4005:801::200e ../configfile -1507753391.587107 testinterval 1.0 sec 60.0 ../configfile -1507753391.587107 testtime 0.0 1507321987.0 ../configfile -1507753391.587107 test_set (empty) b,c,a,d,erdbeerschnitzel ../configfile -1507753391.587107 test_vector (empty) 1,2,3,4,5,6 ../configfile -1507753391.587107 test_set b,c,a,d,erdbeerschnitzel (empty) ../configfile -1507753391.587107 test_set (empty) \x2d ../configfile -#close 2017-10-11-20-23-11 +1533925012.140634 testbool T F ../configfile +1533925012.140634 testcount 0 1 ../configfile +1533925012.140634 testcount 1 2 ../configfile +1533925012.140634 testint 0 -1 ../configfile +1533925012.140634 testenum SSH::LOG Conn::LOG ../configfile +1533925012.140634 testport 42/tcp 45/unknown ../configfile +1533925012.140634 testporttcp 40/udp 42/tcp ../configfile +1533925012.140634 testportudp 40/tcp 42/udp ../configfile +1533925012.140634 testaddr 127.0.0.1 127.0.0.1 ../configfile +1533925012.140634 testaddr 127.0.0.1 2607:f8b0:4005:801::200e ../configfile +1533925012.140634 testinterval 1.0 sec 60.0 ../configfile +1533925012.140634 testtime 0.0 1507321987.0 ../configfile +1533925012.140634 test_set (empty) b,c,a,d,erdbeerschnitzel ../configfile +1533925012.140634 test_vector (empty) 1,2,3,4,5,6 ../configfile +1533925012.140634 test_set b,c,a,d,erdbeerschnitzel (empty) ../configfile +1533925012.140634 test_set (empty) \x2d ../configfile +#close 2018-08-10-18-16-52 diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.port-embedded/bro..stderr b/testing/btest/Baseline/scripts.base.frameworks.input.port-embedded/bro..stderr new file mode 100644 index 0000000000..fee70a8699 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.port-embedded/bro..stderr @@ -0,0 +1,2 @@ +warning: ../input.log/Input::READER_ASCII: Port '50/trash' contained unknown protocol 'trash' +received termination signal diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.port-embedded/bro..stdout b/testing/btest/Baseline/scripts.base.frameworks.input.port-embedded/bro..stdout new file mode 100644 index 0000000000..d1d886b370 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.port-embedded/bro..stdout @@ -0,0 +1,4 @@ +[i=1.2.3.4], [p=80/tcp] +[i=1.2.3.5], [p=52/udp] +[i=1.2.3.6], [p=30/unknown] +[i=1.2.3.7], [p=50/unknown] diff --git a/testing/btest/scripts/base/frameworks/config/basic.bro b/testing/btest/scripts/base/frameworks/config/basic.bro index 3b72f6572d..f5a02983fd 100644 --- a/testing/btest/scripts/base/frameworks/config/basic.bro +++ b/testing/btest/scripts/base/frameworks/config/basic.bro @@ -1,6 +1,7 @@ # @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: btest-bg-wait 10 # @TEST-EXEC: btest-diff bro/config.log +# @TEST-EXEC: btest-diff bro/.stderr @load base/frameworks/config @load base/protocols/conn @@ -16,6 +17,8 @@ testcount 2 testint -1 testenum Conn::LOG testport 45 +testporttcp 42/tcp +testportudp 42/udp testaddr 127.0.0.1 testaddr 2607:f8b0:4005:801::200e testinterval 60 @@ -35,6 +38,8 @@ export { option testint: int = 0; option testenum = SSH::LOG; option testport = 42/tcp; + option testporttcp = 40/udp; + option testportudp = 40/tcp; option testaddr = 127.0.0.1; option testtime = network_time(); option testinterval = 1sec; diff --git a/testing/btest/scripts/base/frameworks/input/port-embedded.bro b/testing/btest/scripts/base/frameworks/input/port-embedded.bro new file mode 100644 index 0000000000..8aab733069 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/input/port-embedded.bro @@ -0,0 +1,44 @@ +# @TEST-EXEC: btest-bg-run bro bro -b %INPUT +# @TEST-EXEC: btest-bg-wait 10 +# @TEST-EXEC: btest-diff bro/.stdout +# @TEST-EXEC: btest-diff bro/.stderr + +@TEST-START-FILE input.log +#fields i p +1.2.3.4 80/tcp +1.2.3.5 52/udp +1.2.3.6 30/unknown +1.2.3.7 50/trash +@TEST-END-FILE + +redef exit_only_after_terminate = T; + +redef InputAscii::empty_field = "EMPTY"; + +module A; + +type Idx: record { + i: addr; +}; + +type Val: record { + p: port; +}; + +global servers: table[addr] of Val = table(); + +event line(description: Input::TableDescription, tpe: Input::Event, left: Idx, right: Val) + { + print left, right; + } + +event bro_init() + { + Input::add_table([$source="../input.log", $name="input", $idx=Idx, $val=Val, $ev=line, $destination=servers]); + } + +event Input::end_of_data(name: string, source: string) + { + Input::remove("input"); + terminate(); + } From c34fbee0d1a766dd7e864b28dbbf6380360ec990 Mon Sep 17 00:00:00 2001 From: Johanna Amann Date: Fri, 10 Aug 2018 11:45:37 -0700 Subject: [PATCH 3/4] Make options redef-able by default. --- src/ID.cc | 16 ++++++++++++++++ src/ID.h | 2 +- testing/btest/Baseline/core.option-redef/.stdout | 1 + testing/btest/core/option-redef.bro | 4 ++++ 4 files changed, 22 insertions(+), 1 deletion(-) diff --git a/src/ID.cc b/src/ID.cc index 4216422225..a68abb6264 100644 --- a/src/ID.cc +++ b/src/ID.cc @@ -294,6 +294,22 @@ void ID::RemoveAttr(attr_tag a) } } +void ID::SetOption() + { + if ( is_option ) + return; + + is_option = true; + + // option implied redefinable + if ( ! IsRedefinable() ) + { + attr_list* attr = new attr_list; + attr->append(new Attr(ATTR_REDEF)); + AddAttrs(new Attributes(attr, Type(), false)); + } + } + void ID::EvalFunc(Expr* ef, Expr* ev) { Expr* arg1 = new ConstExpr(val->Ref()); diff --git a/src/ID.h b/src/ID.h index 442a13dfcc..18754584df 100644 --- a/src/ID.h +++ b/src/ID.h @@ -60,7 +60,7 @@ public: void SetConst() { is_const = true; } bool IsConst() const { return is_const; } - void SetOption() { is_option = true; } + void SetOption(); bool IsOption() const { return is_option; } void SetEnumConst() { is_enum_const = true; } diff --git a/testing/btest/Baseline/core.option-redef/.stdout b/testing/btest/Baseline/core.option-redef/.stdout index 1e8b314962..baf1966653 100644 --- a/testing/btest/Baseline/core.option-redef/.stdout +++ b/testing/btest/Baseline/core.option-redef/.stdout @@ -1 +1,2 @@ 6 +7 diff --git a/testing/btest/core/option-redef.bro b/testing/btest/core/option-redef.bro index 05706ab48b..3d67a9a755 100644 --- a/testing/btest/core/option-redef.bro +++ b/testing/btest/core/option-redef.bro @@ -2,11 +2,15 @@ # @TEST-EXEC: btest-diff .stdout # options are allowed to be redef-able. +# And they are even redef-able by default. option testopt = 5 &redef; redef testopt = 6; +option anotheropt = 6; +redef anotheropt = 7; event bro_init() { print testopt; + print anotheropt; } From 7b44a6499433aa1817a5c1b21b46f4e864621dae Mon Sep 17 00:00:00 2001 From: Johanna Amann Date: Fri, 10 Aug 2018 14:28:17 -0700 Subject: [PATCH 4/4] Fix test that fails now that options are automatically redefable. --- testing/btest/Baseline/core.option-errors-4/.stderr | 1 - testing/btest/core/option-errors.bro | 5 ----- 2 files changed, 6 deletions(-) delete mode 100644 testing/btest/Baseline/core.option-errors-4/.stderr diff --git a/testing/btest/Baseline/core.option-errors-4/.stderr b/testing/btest/Baseline/core.option-errors-4/.stderr deleted file mode 100644 index b443da2eb9..0000000000 --- a/testing/btest/Baseline/core.option-errors-4/.stderr +++ /dev/null @@ -1 +0,0 @@ -error in /Users/johanna/corelight/bro/testing/btest/.tmp/core.option-errors-4/option-errors.bro, line 2 and /Users/johanna/corelight/bro/testing/btest/.tmp/core.option-errors-4/option-errors.bro, line 3: already defined (testopt) diff --git a/testing/btest/core/option-errors.bro b/testing/btest/core/option-errors.bro index 6a53598650..6a9a8f1db6 100644 --- a/testing/btest/core/option-errors.bro +++ b/testing/btest/core/option-errors.bro @@ -11,8 +11,3 @@ option testbool : bool; option testopt = 5; testopt = 6; - -@TEST-START-NEXT - -option testopt = 5; -redef testopt = 6;