From 80b3b82b540e390ce4fd7a3f2abd68088c6bad80 Mon Sep 17 00:00:00 2001 From: Vern Paxson Date: Tue, 26 Jun 2018 15:59:41 -0700 Subject: [PATCH 01/24] implemented /re/i for case-insensitive patterns --- NEWS | 6 +++++ src/RE.cc | 19 ++++++++++++++ src/RE.h | 5 ++++ src/input.h | 1 - src/parse.y | 9 ++++--- src/re-parse.y | 69 +++++++++++++++++++++++++++++++++++--------------- src/re-scan.l | 21 ++++++++++++++- src/scan.l | 14 +++++++++- 8 files changed, 117 insertions(+), 27 deletions(-) diff --git a/NEWS b/NEWS index f594207f58..d075d4637b 100644 --- a/NEWS +++ b/NEWS @@ -255,6 +255,12 @@ New Functionality semi-present in previous versions of Bro, but required constants as as its operands; now you can use any pattern-valued expressions. +- You can now specify that a pattern should be match in a case-insensitive + fashion by adding 'i' to the end of its specification. So for example + /fOO/i == "Foo" yields T, as does /fOO/i in "xFoObar". Characters + enclosed in quotes however keep their casing, so /"fOO"/i in "xFoObar" + yields F, though it yields T for "xfOObar". + Changed Functionality --------------------- diff --git a/src/RE.cc b/src/RE.cc index 4d26ce2423..4e29fa8e92 100644 --- a/src/RE.cc +++ b/src/RE.cc @@ -102,6 +102,19 @@ void Specific_RE_Matcher::AddPat(const char* new_pat, pattern_text = s; } +void Specific_RE_Matcher::MakeCaseInsensitive() + { + const char fmt[] = "(+i %s)"; + int n = strlen(pattern_text) + strlen(fmt); + + char* s = new char[n + 5 /* slop */]; + + safe_snprintf(s, n + 5, fmt, pattern_text); + + delete [] pattern_text; + pattern_text = s; + } + int Specific_RE_Matcher::Compile(int lazy) { if ( ! pattern_text ) @@ -444,6 +457,12 @@ void RE_Matcher::AddPat(const char* new_pat) re_exact->AddPat(new_pat); } +void RE_Matcher::MakeCaseInsensitive() + { + re_anywhere->MakeCaseInsensitive(); + re_exact->MakeCaseInsensitive(); + } + int RE_Matcher::Compile(int lazy) { return re_anywhere->Compile(lazy) && re_exact->Compile(lazy); diff --git a/src/RE.h b/src/RE.h index 056c0d2183..06b0699864 100644 --- a/src/RE.h +++ b/src/RE.h @@ -54,6 +54,8 @@ public: void AddPat(const char* pat); + void MakeCaseInsensitive(); + void SetPat(const char* pat) { pattern_text = copy_string(pat); } int Compile(int lazy = 0); @@ -178,6 +180,9 @@ public: void AddPat(const char* pat); + // Makes the matcher as specified to date case-insensitive. + void MakeCaseInsensitive(); + int Compile(int lazy = 0); // Returns true if s exactly matches the pattern, false otherwise. diff --git a/src/input.h b/src/input.h index 3d0caa459a..230a10073a 100644 --- a/src/input.h +++ b/src/input.h @@ -23,7 +23,6 @@ extern void add_input_file_at_front(const char* file); extern void add_to_name_list(char* s, char delim, name_list& nl); extern void begin_RE(); -extern void end_RE(); extern void do_atif(Expr* expr); extern void do_atifdef(const char* id); diff --git a/src/parse.y b/src/parse.y index 34d6f31373..25b6c17873 100644 --- a/src/parse.y +++ b/src/parse.y @@ -14,7 +14,7 @@ %token TOK_DOUBLE TOK_ELSE TOK_ENUM TOK_EVENT TOK_EXPORT TOK_FALLTHROUGH %token TOK_FILE TOK_FOR TOK_FUNCTION TOK_GLOBAL TOK_HOOK TOK_ID TOK_IF TOK_INT %token TOK_INTERVAL TOK_LIST TOK_LOCAL TOK_MODULE -%token TOK_NEXT TOK_OF TOK_OPAQUE TOK_PATTERN TOK_PATTERN_TEXT +%token TOK_NEXT TOK_OF TOK_OPAQUE TOK_PATTERN TOK_PATTERN_END TOK_PATTERN_TEXT %token TOK_PORT TOK_PRINT TOK_RECORD TOK_REDEF %token TOK_REMOVE_FROM TOK_RETURN TOK_SCHEDULE TOK_SET %token TOK_STRING TOK_SUBNET TOK_SWITCH TOK_TABLE @@ -52,7 +52,7 @@ %left '$' '[' ']' '(' ')' TOK_HAS_FIELD TOK_HAS_ATTR %nonassoc TOK_AS TOK_IS -%type opt_no_test opt_no_test_block opt_deprecated +%type opt_no_test opt_no_test_block opt_deprecated TOK_PATTERN_END %type TOK_ID TOK_PATTERN_TEXT %type local_id global_id def_global_id event_id global_or_event_id resolve_id begin_func case_type %type local_id_list case_type_list @@ -723,13 +723,16 @@ expr: $$ = new ConstExpr($1); } - | '/' { begin_RE(); } TOK_PATTERN_TEXT { end_RE(); } '/' + | '/' { begin_RE(); } TOK_PATTERN_TEXT TOK_PATTERN_END { set_location(@3); RE_Matcher* re = new RE_Matcher($3); delete [] $3; + if ( $4 ) + re->MakeCaseInsensitive(); + re->Compile(); $$ = new ConstExpr(new PatternVal(re)); } diff --git a/src/re-parse.y b/src/re-parse.y index 3847c06f29..6834836f28 100644 --- a/src/re-parse.y +++ b/src/re-parse.y @@ -11,11 +11,13 @@ int csize = 256; int syntax_error = 0; +int is_letter(int sym); +int cupper(int sym); int clower(int sym); void yyerror(const char msg[]); %} -%token TOK_CHAR TOK_NUMBER TOK_CCL TOK_CCE +%token TOK_CHAR TOK_NUMBER TOK_CCL TOK_CCE TOK_CASE_INSENSITIVE %union { int int_val; @@ -126,12 +128,11 @@ singleton : singleton '*' | '(' re ')' { $$ = $2; } + | TOK_CASE_INSENSITIVE re ')' + { $$ = $2; case_insensitive = 0; } + | TOK_CHAR - { - if ( case_insensitive && $1 >= 'A' && $1 <= 'Z' ) - $1 = clower($1); - $$ = new NFA_Machine(new NFA_State($1, rem->EC())); - } + { $$ = new NFA_Machine(new NFA_State($1, rem->EC())); } | '^' { @@ -158,17 +159,29 @@ full_ccl : '[' ccl ']' ccl : ccl TOK_CHAR '-' TOK_CHAR { - if ( case_insensitive ) - { - if ( $2 >= 'A' && $2 <= 'Z' ) - $2 = clower($2); - if ( $4 >= 'A' && $4 <= 'Z' ) - $4 = clower($4); - } - if ( $2 > $4 ) synerr("negative range in character class"); + else if ( case_insensitive && + (is_letter($2) || is_letter($4)) ) + { + if ( is_letter($2) && is_letter($4) && + isupper($2) == isupper($4) ) + { // Compatible range, do both versions + int l2 = tolower($2); + int l4 = tolower($4); + + for ( int i = l2; i<= l4; ++i ) + { + $1->Add(i); + $1->Add(toupper(i)); + } + } + + else + synerr("ambiguous case-insensitive character class"); + } + else { for ( int i = $2; i <= $4; ++i ) @@ -178,10 +191,13 @@ ccl : ccl TOK_CHAR '-' TOK_CHAR | ccl TOK_CHAR { - if ( case_insensitive && $2 >= 'A' && $2 <= 'Z' ) - $2 = clower($2); - - $1->Add($2); + if ( case_insensitive && is_letter($2) ) + { + $1->Add(clower($2)); + $1->Add(cupper($2)); + } + else + $1->Add($2); } | ccl ccl_expr @@ -200,9 +216,10 @@ ccl_expr: TOK_CCE string : string TOK_CHAR { - if ( case_insensitive && $2 >= 'A' && $2 <= 'Z' ) - $2 = clower($2); - + // Even if case-insensitivity is set, + // leave this alone; that provides a way + // of "escaping" out of insensitivity + // if needed. $1->AppendState(new NFA_State($2, rem->EC())); } @@ -211,6 +228,16 @@ string : string TOK_CHAR ; %% +int is_letter(int sym) + { + return isascii(sym) && (islower(sym) || isupper(sym)); + } + +int cupper(int sym) + { + return (isascii(sym) && islower(sym)) ? toupper(sym) : sym; + } + int clower(int sym) { return (isascii(sym) && isupper(sym)) ? tolower(sym) : sym; diff --git a/src/re-scan.l b/src/re-scan.l index 8bd00c8bba..70bafd5649 100644 --- a/src/re-scan.l +++ b/src/re-scan.l @@ -114,6 +114,25 @@ CCL_EXPR ("[:"[[:alpha:]]+":]") } } + "(+i"[ \t]* case_insensitive = 1; return TOK_CASE_INSENSITIVE; + + [a-zA-Z] { + if ( case_insensitive ) + { + char c = yytext[0]; // unput trashes yytext! + // Push back the character inside a CCL, + // so the parser can then expand it. + unput(']'); + unput(c); + unput('['); + } + else + { + yylval.int_val = yytext[0]; + return TOK_CHAR; + } + } + [|*+?.(){}] return yytext[0]; . yylval.int_val = yytext[0]; return TOK_CHAR; \n return 0; // treat as end of pattern @@ -157,7 +176,7 @@ CCL_EXPR ("[:"[[:alpha:]]+":]") "[:upper:]" { BEGIN(SC_CCL); yylval.cce_val = - case_insensitive ? my_isupper : my_islower; + case_insensitive ? my_islower : my_isupper; return TOK_CCE; } diff --git a/src/scan.l b/src/scan.l index 3bbf6ec999..24e0547bfc 100644 --- a/src/scan.l +++ b/src/scan.l @@ -554,7 +554,19 @@ F RET_CONST(new Val(false, TYPE_BOOL)) return TOK_PATTERN_TEXT; } -[/\\\n] return yytext[0]; +"/" { + BEGIN(INITIAL); + yylval.b = false; + return TOK_PATTERN_END; + } + +"/i" { + BEGIN(INITIAL); + yylval.b = true; + return TOK_PATTERN_END; + } + +[\\\n] return yytext[0]; // should cause a parse error <*>. reporter->Error("unrecognized character - %s", yytext); From 4bd8f3a5d5bfe7c922c94e2bc66c93a230f17496 Mon Sep 17 00:00:00 2001 From: Vern Paxson Date: Tue, 26 Jun 2018 20:43:48 -0700 Subject: [PATCH 02/24] fix for handling [:(lower|upper):] in case-insensitive patterns --- src/re-scan.l | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/re-scan.l b/src/re-scan.l index 70bafd5649..d952ca2fd1 100644 --- a/src/re-scan.l +++ b/src/re-scan.l @@ -36,6 +36,8 @@ static int my_isprint(int c) { return isprint(c); } static int my_ispunct(int c) { return ispunct(c); } static int my_isspace(int c) { return isspace(c); } static int my_isxdigit(int c) { return isxdigit(c); } + +static int my_is_letter(int c) { return my_islower(c) || my_isupper(c); } %} %option caseless nodefault nostdinit noyywrap @@ -168,15 +170,22 @@ CCL_EXPR ("[:"[[:alpha:]]+":]") "[:cntrl:]" RET_CCE(my_iscntrl) "[:digit:]" RET_CCE(my_isdigit) "[:graph:]" RET_CCE(my_isgraph) - "[:lower:]" RET_CCE(my_islower) "[:print:]" RET_CCE(my_isprint) "[:punct:]" RET_CCE(my_ispunct) "[:space:]" RET_CCE(my_isspace) "[:xdigit:]" RET_CCE(my_isxdigit) + + "[:lower:]" { + BEGIN(SC_CCL); + yylval.cce_val = + case_insensitive ? my_is_letter : my_islower; + return TOK_CCE; + } + "[:upper:]" { BEGIN(SC_CCL); yylval.cce_val = - case_insensitive ? my_islower : my_isupper; + case_insensitive ? my_is_letter : my_isupper; return TOK_CCE; } From 9bdb24a719121be1fd8008ba56f36ab65b91d26b Mon Sep 17 00:00:00 2001 From: Vern Paxson Date: Tue, 26 Jun 2018 20:47:12 -0700 Subject: [PATCH 03/24] d'oh there's isalpha. I looked earlier for isletter :-P --- src/re-parse.y | 12 +++--------- src/re-scan.l | 6 ++---- 2 files changed, 5 insertions(+), 13 deletions(-) diff --git a/src/re-parse.y b/src/re-parse.y index 6834836f28..95573f6a22 100644 --- a/src/re-parse.y +++ b/src/re-parse.y @@ -11,7 +11,6 @@ int csize = 256; int syntax_error = 0; -int is_letter(int sym); int cupper(int sym); int clower(int sym); void yyerror(const char msg[]); @@ -163,9 +162,9 @@ ccl : ccl TOK_CHAR '-' TOK_CHAR synerr("negative range in character class"); else if ( case_insensitive && - (is_letter($2) || is_letter($4)) ) + (isalpha($2) || isalpha($4)) ) { - if ( is_letter($2) && is_letter($4) && + if ( isalpha($2) && isalpha($4) && isupper($2) == isupper($4) ) { // Compatible range, do both versions int l2 = tolower($2); @@ -191,7 +190,7 @@ ccl : ccl TOK_CHAR '-' TOK_CHAR | ccl TOK_CHAR { - if ( case_insensitive && is_letter($2) ) + if ( case_insensitive && isalpha($2) ) { $1->Add(clower($2)); $1->Add(cupper($2)); @@ -228,11 +227,6 @@ string : string TOK_CHAR ; %% -int is_letter(int sym) - { - return isascii(sym) && (islower(sym) || isupper(sym)); - } - int cupper(int sym) { return (isascii(sym) && islower(sym)) ? toupper(sym) : sym; diff --git a/src/re-scan.l b/src/re-scan.l index d952ca2fd1..0c6819bdd7 100644 --- a/src/re-scan.l +++ b/src/re-scan.l @@ -36,8 +36,6 @@ static int my_isprint(int c) { return isprint(c); } static int my_ispunct(int c) { return ispunct(c); } static int my_isspace(int c) { return isspace(c); } static int my_isxdigit(int c) { return isxdigit(c); } - -static int my_is_letter(int c) { return my_islower(c) || my_isupper(c); } %} %option caseless nodefault nostdinit noyywrap @@ -178,14 +176,14 @@ CCL_EXPR ("[:"[[:alpha:]]+":]") "[:lower:]" { BEGIN(SC_CCL); yylval.cce_val = - case_insensitive ? my_is_letter : my_islower; + case_insensitive ? my_isalpha : my_islower; return TOK_CCE; } "[:upper:]" { BEGIN(SC_CCL); yylval.cce_val = - case_insensitive ? my_is_letter : my_isupper; + case_insensitive ? my_isalpha : my_isupper; return TOK_CCE; } From cfe45e0af0d7c5123278579dd82b5b83bce5b387 Mon Sep 17 00:00:00 2001 From: Vern Paxson Date: Fri, 29 Jun 2018 11:30:00 -0700 Subject: [PATCH 04/24] documentation updates for case-insensitive patterns --- NEWS | 5 +++++ doc/script-reference/types.rst | 15 ++++++++++++--- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/NEWS b/NEWS index d075d4637b..5dbdbcaf5d 100644 --- a/NEWS +++ b/NEWS @@ -261,6 +261,11 @@ New Functionality enclosed in quotes however keep their casing, so /"fOO"/i in "xFoObar" yields F, though it yields T for "xfOObar". + You can achieve the same functionality for a subpattern enclosed in + parentheses by adding "+i" to the open parenthesis, optionally followed + by whitespace. So for example "/foo|(+i bar)/" will match "BaR", but + not "FoO". + Changed Functionality --------------------- diff --git a/doc/script-reference/types.rst b/doc/script-reference/types.rst index 438706c425..d24ce09b8c 100644 --- a/doc/script-reference/types.rst +++ b/doc/script-reference/types.rst @@ -198,9 +198,9 @@ Here is a more detailed description of each type: .. bro:type:: pattern - A type representing regular-expression patterns which can be used + A type representing regular-expression patterns that can be used for fast text-searching operations. Pattern constants are created - by enclosing text within forward slashes (/) and is the same syntax + by enclosing text within forward slashes (``/``) and use the same syntax as the patterns supported by the `flex lexical analyzer `_. The speed of regular expression matching does not depend on the complexity or @@ -244,13 +244,22 @@ Here is a more detailed description of each type: yields true, like in the similar example above. You can also create the conjunction (concatenation) of patterns using the ``&`` - operator. For example: + operator. For example:: /foo/ & /bar/ in "foobar" will yield true because the pattern /(foo)(bar)/ appears in the string "foobar". + When specifying a pattern, you can add a final ``i`` specifier to + mark it as case-insensitive. For example, ``/foo|bar/i`` will match + a "foo", "Foo", "BaR", etc. + + You can also introduce a case-insensitive sub-pattern by enclosing it + in ``(+i````)``. For clarity, you can optionally include + trailing whitespace after the ``+i`` designator. So, for example, + ``/foo|(+i bar)/`` will match "foo" and "BaR", but *not* "Foo". + .. bro:type:: port A type representing transport-level port numbers (besides TCP and From 5ce3d1b899dc27adb853a19f07da35b7deeea9f0 Mon Sep 17 00:00:00 2001 From: Vern Paxson Date: Fri, 29 Jun 2018 11:30:21 -0700 Subject: [PATCH 05/24] bug fix for recent memory leak patch --- src/RE.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/RE.cc b/src/RE.cc index 4e29fa8e92..cd37da18e9 100644 --- a/src/RE.cc +++ b/src/RE.cc @@ -168,7 +168,8 @@ int Specific_RE_Matcher::CompileSet(const string_list& set, const int_list& idx) if ( set_nfa != nfa ) Unref(set_nfa); - Unref(nfa); + else + Unref(nfa); nfa = 0; return 0; From a02d9e7f4a6fa4d6430d9ffbdcef3423810b140f Mon Sep 17 00:00:00 2001 From: Vern Paxson Date: Fri, 29 Jun 2018 11:35:22 -0700 Subject: [PATCH 06/24] document use of double quotes to escape case-insensitivity --- NEWS | 5 +++++ doc/script-reference/types.rst | 4 ++++ 2 files changed, 9 insertions(+) diff --git a/NEWS b/NEWS index 5dbdbcaf5d..69665cde26 100644 --- a/NEWS +++ b/NEWS @@ -266,6 +266,11 @@ New Functionality by whitespace. So for example "/foo|(+i bar)/" will match "BaR", but not "FoO". + For both ways of specifying case-insensitivity, characters enclosed in + double quotes maintain their case-sensitivity. So for example /"foo"/i + will not match "Foo", but it will match "foo". + + Changed Functionality --------------------- diff --git a/doc/script-reference/types.rst b/doc/script-reference/types.rst index d24ce09b8c..36ed0f5bfa 100644 --- a/doc/script-reference/types.rst +++ b/doc/script-reference/types.rst @@ -260,6 +260,10 @@ Here is a more detailed description of each type: trailing whitespace after the ``+i`` designator. So, for example, ``/foo|(+i bar)/`` will match "foo" and "BaR", but *not* "Foo". + For both ways of specifying case-insensitivity, characters enclosed + in double quotes maintain their case-sensitivity. So for example + /"foo"/i will not match "Foo", but it will match "foo". + .. bro:type:: port A type representing transport-level port numbers (besides TCP and From f5e89b96aec80be073e6ec90a5ff2b6fcdba4490 Mon Sep 17 00:00:00 2001 From: Vern Paxson Date: Fri, 29 Jun 2018 11:55:50 -0700 Subject: [PATCH 07/24] test suite update for case-insensitive patterns --- testing/btest/Baseline/language.pattern/out | 22 +++++++++++++ testing/btest/language/pattern.bro | 34 ++++++++++++++++++++- 2 files changed, 55 insertions(+), 1 deletion(-) diff --git a/testing/btest/Baseline/language.pattern/out b/testing/btest/Baseline/language.pattern/out index 9c801eb60b..5c31320da9 100644 --- a/testing/btest/Baseline/language.pattern/out +++ b/testing/btest/Baseline/language.pattern/out @@ -10,3 +10,25 @@ in operator (PASS) & operator (FAIL) | operator (PASS) | operator (FAIL) +/i pattern modifier (PASS) +/i pattern modifier (PASS) +/i double-quote escape (FAIL) +/i double-quote escape (PASS) +case-sensitive pattern (FAIL) +case-sensitive pattern (FAIL) +case-sensitive pattern (PASS) +/i pattern disjunction (PASS) +/i pattern disjunction (FAIL) +/i pattern disjunction (PASS) +/i pattern disjunction (PASS) +/i pattern concatenation (PASS) +/i pattern concatenation (FAIL) +/i pattern concatenation (FAIL) +/i pattern concatenation (PASS) +/i pattern concatenation (PASS) +/i pattern concatenation (FAIL) +/i pattern character class (FAIL) +/i pattern character class (PASS) +(+i ...) pattern construct (PASS) +(+i ...) pattern construct (FAIL) +(+i ...) pattern construct (PASS) diff --git a/testing/btest/language/pattern.bro b/testing/btest/language/pattern.bro index 1c137969eb..70eca233ea 100644 --- a/testing/btest/language/pattern.bro +++ b/testing/btest/language/pattern.bro @@ -22,15 +22,47 @@ event bro_init() test_case( "equality operator", "foo" == p1 ); test_case( "equality operator (order of operands)", p1 == "foo" ); + test_case( "inequality operator", "foobar" != p1 ); test_case( "inequality operator (order of operands)", p1 != "foobar" ); + test_case( "in operator", p1 in "foobar" ); test_case( "in operator", p2 in "foobar" ); test_case( "!in operator", p3 !in "foobar" ); + test_case( "& operator", p1 & p2 in "baroob" ); test_case( "& operator", p2 & p1 in "baroob" ); + test_case( "| operator", p1 | p2 in "lazybarlazy" ); test_case( "| operator", p3 | p4 in "xoob" ); -} + test_case( "/i pattern modifier", /fOO/i in "xFoObar" ); + test_case( "/i pattern modifier", /fOO/i == "Foo" ); + test_case( "/i double-quote escape", /"fOO"/i in "xFoObar" ); + test_case( "/i double-quote escape", /"fOO"/i in "xfOObar" ); + + test_case( "case-sensitive pattern", /fOO/ in "xFoObar" ); + test_case( "case-sensitive pattern", /fOO/ == "Foo" ); + test_case( "case-sensitive pattern", /fOO/ == "fOO" ); + + test_case( "/i pattern disjunction", /bar/i | /bez/ == "bez" ); + test_case( "/i pattern disjunction", /bar/i | /bez/ == "bEz" ); + test_case( "/i pattern disjunction", /bar/i | /bez/ == "bar" ); + test_case( "/i pattern disjunction", /bar/i | /bez/ == "bAr" ); + + test_case( "/i pattern concatenation", /bar/i & /bez/ == "barbez" ); + test_case( "/i pattern concatenation", /bar/i & /bez/ == "barbEz" ); + test_case( "/i pattern concatenation", /BAR/i & /bez/ == "barbEz" ); + test_case( "/i pattern concatenation", /bar/i & /bez/ == "bArbez" ); + test_case( "/i pattern concatenation", /BAR/i & /bez/ == "bArbez" ); + test_case( "/i pattern concatenation", /bar/i & /bez/ == "bArbEz" ); + + test_case( "/i pattern character class", /ba[0a-c99S-Z0]/i & /bEz/ == "bArbEz" ); + test_case( "/i pattern character class", /ba[0a-c99M-S0]/i & /bEz/ == "bArbEz" ); + + test_case( "(+i ...) pattern construct", /foo|(+i bar)/ in "xBAry" ); + test_case( "(+i ...) pattern construct", /foo|(+i bar)/ in "xFOoy" ); + test_case( "(+i ...) pattern construct", /foo|(+i bar)/ | /foo/i in "xFOoy" ); + +} From 726424f371c2b3d5f044a96a9143a8169bb728c4 Mon Sep 17 00:00:00 2001 From: Vern Paxson Date: Fri, 29 Jun 2018 12:00:37 -0700 Subject: [PATCH 08/24] nitlet in NEWS entry --- NEWS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NEWS b/NEWS index 69665cde26..a31491862f 100644 --- a/NEWS +++ b/NEWS @@ -255,7 +255,7 @@ New Functionality semi-present in previous versions of Bro, but required constants as as its operands; now you can use any pattern-valued expressions. -- You can now specify that a pattern should be match in a case-insensitive +- You can now specify that a pattern matches in a case-insensitive fashion by adding 'i' to the end of its specification. So for example /fOO/i == "Foo" yields T, as does /fOO/i in "xFoObar". Characters enclosed in quotes however keep their casing, so /"fOO"/i in "xFoObar" From 85c4b0d2859140f1be679601d87e75a60b3559bb Mon Sep 17 00:00:00 2001 From: Vern Paxson Date: Fri, 29 Jun 2018 13:01:05 -0700 Subject: [PATCH 09/24] use PCRE syntax instead of the beautiful new (?i ...) syntax --- NEWS | 5 ++--- doc/script-reference/types.rst | 5 ++--- src/RE.cc | 2 +- src/re-scan.l | 2 +- testing/btest/Baseline/language.pattern/out | 6 +++--- testing/btest/language/pattern.bro | 6 +++--- 6 files changed, 12 insertions(+), 14 deletions(-) diff --git a/NEWS b/NEWS index a31491862f..02e632865c 100644 --- a/NEWS +++ b/NEWS @@ -262,9 +262,8 @@ New Functionality yields F, though it yields T for "xfOObar". You can achieve the same functionality for a subpattern enclosed in - parentheses by adding "+i" to the open parenthesis, optionally followed - by whitespace. So for example "/foo|(+i bar)/" will match "BaR", but - not "FoO". + parentheses by adding "?i:" to the open parenthesis. So for example + "/foo|(?i:bar)/" will match "BaR", but not "FoO". For both ways of specifying case-insensitivity, characters enclosed in double quotes maintain their case-sensitivity. So for example /"foo"/i diff --git a/doc/script-reference/types.rst b/doc/script-reference/types.rst index 36ed0f5bfa..99dac0be48 100644 --- a/doc/script-reference/types.rst +++ b/doc/script-reference/types.rst @@ -256,9 +256,8 @@ Here is a more detailed description of each type: a "foo", "Foo", "BaR", etc. You can also introduce a case-insensitive sub-pattern by enclosing it - in ``(+i````)``. For clarity, you can optionally include - trailing whitespace after the ``+i`` designator. So, for example, - ``/foo|(+i bar)/`` will match "foo" and "BaR", but *not* "Foo". + in ``(?i:````)``. So, for example, ``/foo|(?i:bar)/`` will + match "foo" and "BaR", but *not* "Foo". For both ways of specifying case-insensitivity, characters enclosed in double quotes maintain their case-sensitivity. So for example diff --git a/src/RE.cc b/src/RE.cc index cd37da18e9..9c17f2f992 100644 --- a/src/RE.cc +++ b/src/RE.cc @@ -104,7 +104,7 @@ void Specific_RE_Matcher::AddPat(const char* new_pat, void Specific_RE_Matcher::MakeCaseInsensitive() { - const char fmt[] = "(+i %s)"; + const char fmt[] = "(?i:%s)"; int n = strlen(pattern_text) + strlen(fmt); char* s = new char[n + 5 /* slop */]; diff --git a/src/re-scan.l b/src/re-scan.l index 0c6819bdd7..292f7a2e02 100644 --- a/src/re-scan.l +++ b/src/re-scan.l @@ -114,7 +114,7 @@ CCL_EXPR ("[:"[[:alpha:]]+":]") } } - "(+i"[ \t]* case_insensitive = 1; return TOK_CASE_INSENSITIVE; + "(?i:" case_insensitive = 1; return TOK_CASE_INSENSITIVE; [a-zA-Z] { if ( case_insensitive ) diff --git a/testing/btest/Baseline/language.pattern/out b/testing/btest/Baseline/language.pattern/out index 5c31320da9..dac62ab0fa 100644 --- a/testing/btest/Baseline/language.pattern/out +++ b/testing/btest/Baseline/language.pattern/out @@ -29,6 +29,6 @@ case-sensitive pattern (PASS) /i pattern concatenation (FAIL) /i pattern character class (FAIL) /i pattern character class (PASS) -(+i ...) pattern construct (PASS) -(+i ...) pattern construct (FAIL) -(+i ...) pattern construct (PASS) +(?i:...) pattern construct (PASS) +(?i:...) pattern construct (FAIL) +(?i:...) pattern construct (PASS) diff --git a/testing/btest/language/pattern.bro b/testing/btest/language/pattern.bro index 70eca233ea..e427b70e80 100644 --- a/testing/btest/language/pattern.bro +++ b/testing/btest/language/pattern.bro @@ -61,8 +61,8 @@ event bro_init() test_case( "/i pattern character class", /ba[0a-c99S-Z0]/i & /bEz/ == "bArbEz" ); test_case( "/i pattern character class", /ba[0a-c99M-S0]/i & /bEz/ == "bArbEz" ); - test_case( "(+i ...) pattern construct", /foo|(+i bar)/ in "xBAry" ); - test_case( "(+i ...) pattern construct", /foo|(+i bar)/ in "xFOoy" ); - test_case( "(+i ...) pattern construct", /foo|(+i bar)/ | /foo/i in "xFOoy" ); + test_case( "(?i:...) pattern construct", /foo|(?i:bar)/ in "xBAry" ); + test_case( "(?i:...) pattern construct", /foo|(?i:bar)/ in "xFOoy" ); + test_case( "(?i:...) pattern construct", /foo|(?i:bar)/ | /foo/i in "xFOoy" ); } From c9ebe725f6b5c2acbe74d1595ebcc40ee9b7979b Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Mon, 2 Jul 2018 16:29:21 -0500 Subject: [PATCH 10/24] BIT-1941: improve reliability of broker.disconnect unit test --- CHANGES | 4 + VERSION | 2 +- .../recv.broker.filtered.log | 1 - .../Baseline/broker.disconnect/recv.recv.out | 2 +- .../recv2.broker.filtered.log | 1 - .../broker.disconnect/recv2.recv2.out | 2 +- .../send.broker.filtered.log | 4 - .../Baseline/broker.disconnect/send.send.out | 2 - testing/btest/broker/disconnect.bro | 109 ++++++++---------- testing/scripts/wait-for-file | 24 ++++ testing/scripts/wait-for-pid | 24 ++++ 11 files changed, 105 insertions(+), 70 deletions(-) delete mode 100644 testing/btest/Baseline/broker.disconnect/recv.broker.filtered.log delete mode 100644 testing/btest/Baseline/broker.disconnect/recv2.broker.filtered.log delete mode 100644 testing/btest/Baseline/broker.disconnect/send.broker.filtered.log create mode 100755 testing/scripts/wait-for-file create mode 100755 testing/scripts/wait-for-pid diff --git a/CHANGES b/CHANGES index 9bc5ff07ec..7bde326b16 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,8 @@ +2.5-721 | 2018-07-02 16:29:21 -0500 + + * BIT-1941: improve reliability of broker.disconnect unit test (Corelight) + 2.5-719 | 2018-06-27 20:02:52 -0500 * Fix some typos and formatting in NEWS and other documentation diff --git a/VERSION b/VERSION index 613011266e..a9a61666da 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.5-719 +2.5-721 diff --git a/testing/btest/Baseline/broker.disconnect/recv.broker.filtered.log b/testing/btest/Baseline/broker.disconnect/recv.broker.filtered.log deleted file mode 100644 index 18907995da..0000000000 --- a/testing/btest/Baseline/broker.disconnect/recv.broker.filtered.log +++ /dev/null @@ -1 +0,0 @@ -1502645128.235998 Broker::STATUS peer-added 127.0.0.1 XXX handshake successful diff --git a/testing/btest/Baseline/broker.disconnect/recv.recv.out b/testing/btest/Baseline/broker.disconnect/recv.recv.out index f2782a61b2..c7c5a6ea33 100644 --- a/testing/btest/Baseline/broker.disconnect/recv.recv.out +++ b/testing/btest/Baseline/broker.disconnect/recv.recv.out @@ -1,2 +1,2 @@ peer added, handshake successful -Something receiver, 1 +receiver got event, 1 diff --git a/testing/btest/Baseline/broker.disconnect/recv2.broker.filtered.log b/testing/btest/Baseline/broker.disconnect/recv2.broker.filtered.log deleted file mode 100644 index 6a698102f2..0000000000 --- a/testing/btest/Baseline/broker.disconnect/recv2.broker.filtered.log +++ /dev/null @@ -1 +0,0 @@ -1502645286.464675 Broker::STATUS peer-added 127.0.0.1 XXX handshake successful diff --git a/testing/btest/Baseline/broker.disconnect/recv2.recv2.out b/testing/btest/Baseline/broker.disconnect/recv2.recv2.out index ba803026ad..09a0133c8d 100644 --- a/testing/btest/Baseline/broker.disconnect/recv2.recv2.out +++ b/testing/btest/Baseline/broker.disconnect/recv2.recv2.out @@ -1,2 +1,2 @@ peer added, handshake successful -Something receiver, 2 +receiver got event, 2 diff --git a/testing/btest/Baseline/broker.disconnect/send.broker.filtered.log b/testing/btest/Baseline/broker.disconnect/send.broker.filtered.log deleted file mode 100644 index 2a82fdec5f..0000000000 --- a/testing/btest/Baseline/broker.disconnect/send.broker.filtered.log +++ /dev/null @@ -1,4 +0,0 @@ -1524513026.795171 Broker::STATUS peer-added 127.0.0.1 XXX received handshake from remote core -1524513033.340316 Broker::STATUS connection-terminated 127.0.0.1 XXX lost remote peer -1524513035.437373 Broker::STATUS peer-added 127.0.0.1 XXX received handshake from remote core -1524513041.743002 Broker::STATUS connection-terminated 127.0.0.1 XXX lost remote peer diff --git a/testing/btest/Baseline/broker.disconnect/send.send.out b/testing/btest/Baseline/broker.disconnect/send.send.out index 2ed842beb0..f75d91dc23 100644 --- a/testing/btest/Baseline/broker.disconnect/send.send.out +++ b/testing/btest/Baseline/broker.disconnect/send.send.out @@ -1,6 +1,4 @@ peer added, received handshake from remote core -Something sender, 1 peer lost, lost remote peer peer added, received handshake from remote core -Something sender, 2 peer lost, lost remote peer diff --git a/testing/btest/broker/disconnect.bro b/testing/btest/broker/disconnect.bro index cc1822f891..6a5201627d 100644 --- a/testing/btest/broker/disconnect.bro +++ b/testing/btest/broker/disconnect.bro @@ -1,22 +1,19 @@ # @TEST-SERIALIZE: comm -# + # @TEST-EXEC: btest-bg-run recv "bro -B broker -b ../recv.bro >recv.out" # @TEST-EXEC: btest-bg-run send "bro -B broker -b ../send.bro >send.out" -# -# @TEST-EXEC: sleep 6 && kill $(cat recv/.pid) && sleep 1 && echo 0 >recv/.exitcode + +# @TEST-EXEC: $SCRIPTS/wait-for-file recv/got-event 30 || (btest-bg-wait -k 1 && false) +# @TEST-EXEC: kill $(cat recv/.pid) +# @TEST-EXEC: $SCRIPTS/wait-for-pid $(cat recv/.pid) 10 || (btest-bg-wait -k 1 && false) +# @TEST-EXEC: echo 0 >recv/.exitcode + # @TEST-EXEC: btest-bg-run recv2 "bro -B broker -b ../recv.bro >recv2.out" -# -# @TEST-EXEC: btest-bg-wait 25 +# @TEST-EXEC: btest-bg-wait 30 + # @TEST-EXEC: btest-diff send/send.out # @TEST-EXEC: btest-diff recv/recv.out # @TEST-EXEC: btest-diff recv2/recv2.out -# -# @TEST-EXEC: cat send/broker.log | awk '/Broker::STATUS/ { $5="XXX"; print; }' >send/broker.filtered.log -# @TEST-EXEC: cat recv/broker.log | awk '/Broker::STATUS/ { $5="XXX"; print; }' >recv/broker.filtered.log -# @TEST-EXEC: cat recv2/broker.log | grep -v "lost remote peer" | awk '/Broker::STATUS/ { $5="XXX"; print; }' >recv2/broker.filtered.log -# @TEST-EXEC: btest-diff send/broker.filtered.log -# @TEST-EXEC: btest-diff recv/broker.filtered.log -# @TEST-EXEC: btest-diff recv2/broker.filtered.log @TEST-START-FILE send.bro @@ -24,42 +21,34 @@ redef Broker::default_connect_retry=1secs; redef Broker::default_listen_retry=1secs; redef exit_only_after_terminate = T; -event self_terminate() +global peers = 0; +const test_topic = "bro/test/my_topic"; + +event my_event(i: count) { - terminate(); + print "sender got event", i; } -event do_terminate() - { - schedule 2sec { self_terminate() }; - } - -event print_something(i: int) - { - print "Something sender", i; - } - event bro_init() - { - Broker::subscribe("bro/event/my_topic"); - Broker::auto_publish("bro/event/my_topic", print_something); - Broker::auto_publish("bro/event/my_topic", do_terminate); - Broker::peer("127.0.0.1"); - - schedule 3secs { print_something(1) }; - schedule 12secs { print_something(2) }; - schedule 13secs { do_terminate() }; - } + { + Broker::subscribe(test_topic); + Broker::peer("127.0.0.1"); + } event Broker::peer_lost(endpoint: Broker::EndpointInfo, msg: string) - { - print "peer lost", msg; - } + { + print "peer lost", msg; + + if ( peers == 2 ) + terminate(); + } event Broker::peer_added(endpoint: Broker::EndpointInfo, msg: string) - { - print "peer added", msg; - } + { + ++peers; + print "peer added", msg; + Broker::publish(test_topic, my_event, peers); + } @TEST-END-FILE @@ -70,31 +59,33 @@ redef Broker::default_connect_retry=1secs; redef Broker::default_listen_retry=1secs; redef exit_only_after_terminate = T; -event do_terminate() - { - terminate(); - } +const test_topic = "bro/test/my_topic"; -event print_something(i: int) - { - print "Something receiver", i; - } +event my_event(i: count) + { + print "receiver got event", i; + + if ( i == 1 ) + # In the first case, terminate via `kill` from btest command. + system("touch got-event"); + else + terminate(); + } event bro_init() - { - Broker::subscribe("bro/event/my_topic"); - Broker::listen("127.0.0.1"); - } + { + Broker::subscribe(test_topic); + Broker::listen("127.0.0.1"); + } event Broker::peer_lost(endpoint: Broker::EndpointInfo, msg: string) - { - # In the 2nd run, this may be lost at termination, so don't output. - #print "peer lost", msg; - } + { + terminate(); + } event Broker::peer_added(endpoint: Broker::EndpointInfo, msg: string) - { - print "peer added", msg; - } + { + print "peer added", msg; + } @TEST-END-FILE diff --git a/testing/scripts/wait-for-file b/testing/scripts/wait-for-file new file mode 100755 index 0000000000..7a0a6f6874 --- /dev/null +++ b/testing/scripts/wait-for-file @@ -0,0 +1,24 @@ +#! /usr/bin/env bash + +# Sleeps until a file comes into existence. + +if [[ $# -ne 2 ]]; then + >&2 echo "usage: $0 " + exit 1 +fi + +wait_file=$1 +max_wait=$2 +wait_count=0 + +while [[ ! -e $wait_file ]]; do + let "wait_count += 1" + + if [[ $wait_count -ge $max_wait ]]; then + >&2 echo "error: file '$wait_file' does not exist after $max_wait seconds" + exit 1 + fi + + sleep 1 +done + diff --git a/testing/scripts/wait-for-pid b/testing/scripts/wait-for-pid new file mode 100755 index 0000000000..7aa0a927fe --- /dev/null +++ b/testing/scripts/wait-for-pid @@ -0,0 +1,24 @@ +#! /usr/bin/env bash + +# Sleeps until a process id no longer exists. + +if [[ $# -ne 2 ]]; then + >&2 echo "usage: $0 " + exit 1 +fi + +wait_pid=$1 +max_wait=$2 +wait_count=0 + +while kill -0 $wait_pid &> /dev/null; do + let "wait_count += 1" + + if [[ $wait_count -ge $max_wait ]]; then + >&2 echo "error: process $wait_pid still exists after $max_wait seconds" + exit 1 + fi + + sleep 1 +done + From acf1c591eac67f4731a277aa837223a4b26c8719 Mon Sep 17 00:00:00 2001 From: Liviu Valsan Date: Tue, 3 Jul 2018 15:08:21 +0200 Subject: [PATCH 11/24] Added support for making optional the extraction of DNS entries from X509 SAN as Intel::seen records. --- scripts/policy/frameworks/intel/seen/x509.bro | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/scripts/policy/frameworks/intel/seen/x509.bro b/scripts/policy/frameworks/intel/seen/x509.bro index c4f487947e..ba90a9610a 100644 --- a/scripts/policy/frameworks/intel/seen/x509.bro +++ b/scripts/policy/frameworks/intel/seen/x509.bro @@ -2,9 +2,16 @@ @load base/files/x509 @load ./where-locations +module Intel; + +export { + ## Enables the extraction of subject alternate names from the X509 SAN DNS field + const enable_x509_ext_subject_alternative_name = T &redef; +} + event x509_ext_subject_alternative_name(f: fa_file, ext: X509::SubjectAlternativeName) { - if ( ext?$dns ) + if ( enable_x509_ext_subject_alternative_name && ext?$dns ) { for ( i in ext$dns ) Intel::seen([$indicator=ext$dns[i], From 85e46f37cab925c772047c3bdc4ff3559f9ae552 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Tue, 3 Jul 2018 09:16:37 -0500 Subject: [PATCH 12/24] BIT-1941: teach diff-remove-timestamps about time 0 --- CHANGES | 4 ++++ VERSION | 2 +- testing/scripts/diff-remove-timestamps | 3 +-- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/CHANGES b/CHANGES index 7bde326b16..bd46f4e65c 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,8 @@ +2.5-722 | 2018-07-03 09:16:37 -0500 + + * BIT-1941: teach diff-remove-timestamps about time 0 (Corelight) + 2.5-721 | 2018-07-02 16:29:21 -0500 * BIT-1941: improve reliability of broker.disconnect unit test (Corelight) diff --git a/VERSION b/VERSION index a9a61666da..e1dbbe72b1 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.5-721 +2.5-722 diff --git a/testing/scripts/diff-remove-timestamps b/testing/scripts/diff-remove-timestamps index 44422f6f55..770a181c59 100755 --- a/testing/scripts/diff-remove-timestamps +++ b/testing/scripts/diff-remove-timestamps @@ -9,6 +9,5 @@ else sed="sed -E" fi -# The first sed uses a "basic" regexp, the 2nd a "modern:. -sed 's/[0-9]\{10\}\.[0-9]\{2,8\}/XXXXXXXXXX.XXXXXX/g' | \ +$sed 's/(0\.000000)|([0-9]{10}\.[0-9]{2,8})/XXXXXXXXXX.XXXXXX/g' | \ $sed 's/^ *#(open|close).(19|20)..-..-..-..-..-..$/#\1 XXXX-XX-XX-XX-XX-XX/g' From a6ddc882c3ba4308757bf3e7fe581ea367288d4e Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Tue, 3 Jul 2018 09:34:10 -0500 Subject: [PATCH 13/24] Fix unstable config framework test --- CHANGES | 4 ++++ VERSION | 2 +- .../btest/scripts/base/frameworks/config/several-files.bro | 3 +-- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/CHANGES b/CHANGES index bd46f4e65c..2a606b2956 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,8 @@ +2.5-723 | 2018-07-03 09:34:10 -0500 + + * Fix unstable config framework test (Corelight) + 2.5-722 | 2018-07-03 09:16:37 -0500 * BIT-1941: teach diff-remove-timestamps about time 0 (Corelight) diff --git a/VERSION b/VERSION index e1dbbe72b1..64feb3eb5a 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.5-722 +2.5-723 diff --git a/testing/btest/scripts/base/frameworks/config/several-files.bro b/testing/btest/scripts/base/frameworks/config/several-files.bro index 57d15c0075..c5ad563b4e 100644 --- a/testing/btest/scripts/base/frameworks/config/several-files.bro +++ b/testing/btest/scripts/base/frameworks/config/several-files.bro @@ -1,7 +1,6 @@ # @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: btest-bg-wait 10 -# @TEST-EXEC: sort bro/config.log >bro/config.log.tmp && mv bro/config.log.tmp bro/config.log -# @TEST-EXEC: btest-diff bro/config.log +# @TEST-EXEC: TEST_DIFF_CANONIFIER="$SCRIPTS/diff-canonifier | grep -v ^# | $SCRIPTS/diff-sort" btest-diff bro/config.log @load base/frameworks/config @load base/protocols/conn From df3ce608e300cea7342b7ca1f7ffc4ed3f4785b1 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Tue, 3 Jul 2018 10:25:14 -0500 Subject: [PATCH 14/24] Fix unstable cluster/logging test --- .../base/frameworks/logging/field-extension-cluster-error.bro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testing/btest/scripts/base/frameworks/logging/field-extension-cluster-error.bro b/testing/btest/scripts/base/frameworks/logging/field-extension-cluster-error.bro index 6ddcd0ddb7..9def14cc2a 100644 --- a/testing/btest/scripts/base/frameworks/logging/field-extension-cluster-error.bro +++ b/testing/btest/scripts/base/frameworks/logging/field-extension-cluster-error.bro @@ -5,7 +5,7 @@ # @TEST-EXEC: btest-bg-run worker-1 "cp ../cluster-layout.bro . && CLUSTER_NODE=worker-1 bro --pseudo-realtime -C -r $TRACES/wikipedia.trace %INPUT" # @TEST-EXEC: btest-bg-wait 20 # @TEST-EXEC: cat manager-1/reporter.log | grep -v "reporter/" > manager-reporter.log -# @TEST-EXEC: btest-diff manager-reporter.log +# @TEST-EXEC: TEST_DIFF_CANONIFIER="$SCRIPTS/diff-canonifier | grep -v ^# | $SCRIPTS/diff-sort" btest-diff manager-reporter.log @TEST-START-FILE cluster-layout.bro From 15d74ac081e7a5ca6c569f2021f371c17d71a5c4 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Tue, 3 Jul 2018 14:56:10 -0500 Subject: [PATCH 15/24] BIT-1941: improve unit test stability Mostly trying to standardize the way tests sleep for arbitrary amounts of time to make it easier to tell at which particular point the unit test actually may need the timeout interval increased (or else debugged further). --- CHANGES | 4 + VERSION | 2 +- .../out | 30 ++-- .../out | 144 +----------------- .../out | 144 +----------------- .../worker-2..stdout | 1 + testing/btest/core/leaks/basic-cluster.bro | 5 +- testing/btest/core/leaks/hll_cluster.bro | 5 +- testing/btest/core/leaks/input-raw.bro | 11 +- testing/btest/core/leaks/input-reread.bro | 19 ++- .../cluster/custom_pool_exclusivity.bro | 5 +- .../frameworks/cluster/custom_pool_limits.bro | 5 +- .../frameworks/cluster/log_distribution.bro | 6 +- .../frameworks/cluster/start-it-up-logger.bro | 7 +- .../base/frameworks/cluster/start-it-up.bro | 6 +- .../frameworks/cluster/topic_distribution.bro | 5 +- .../cluster/topic_distribution_bifs.bro | 5 +- .../base/frameworks/config/updates.bro | 14 +- .../control/configuration_update.bro | 12 +- .../frameworks/input/empty-values-hashing.bro | 6 +- .../input/missing-file-initially.bro | 15 +- .../input/predicatemodifyandreread.bro | 18 ++- .../frameworks/input/raw/executestdin.bro | 35 +++-- .../frameworks/input/raw/executestream.bro | 12 +- .../base/frameworks/input/raw/offset.bro | 6 +- .../base/frameworks/input/raw/streamraw.bro | 13 +- .../scripts/base/frameworks/input/reread.bro | 19 ++- .../scripts/base/frameworks/input/stream.bro | 12 +- .../base/frameworks/input/twotables.bro | 6 +- .../base/frameworks/intel/input-and-match.bro | 1 - .../intel/read-file-dist-cluster.bro | 5 +- .../base/frameworks/intel/updated-match.bro | 31 ++-- .../logging/field-extension-cluster-error.bro | 5 +- .../logging/field-extension-cluster.bro | 9 +- .../frameworks/netcontrol/basic-cluster.bro | 16 +- .../base/frameworks/notice/cluster.bro | 5 +- .../frameworks/notice/suppression-cluster.bro | 5 +- .../base/frameworks/openflow/log-cluster.bro | 16 +- .../frameworks/sumstats/basic-cluster.bro | 5 +- .../sumstats/cluster-intermediate-update.bro | 5 +- .../frameworks/sumstats/on-demand-cluster.bro | 5 +- .../frameworks/sumstats/sample-cluster.bro | 4 +- .../base/frameworks/sumstats/topk-cluster.bro | 4 +- 43 files changed, 302 insertions(+), 386 deletions(-) diff --git a/CHANGES b/CHANGES index 2a606b2956..8331fb1d70 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,8 @@ +2.5-725 | 2018-07-03 14:56:10 -0500 + + * BIT-1941: improve unit test stability (Corelight) + 2.5-723 | 2018-07-03 09:34:10 -0500 * Fix unstable config framework test (Corelight) diff --git a/VERSION b/VERSION index 64feb3eb5a..9f94bc17a4 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.5-723 +2.5-725 diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.raw.executestdin/out b/testing/btest/Baseline/scripts.base.frameworks.input.raw.executestdin/out index 23851022b5..e847bdab82 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.input.raw.executestdin/out +++ b/testing/btest/Baseline/scripts.base.frameworks.input.raw.executestdin/out @@ -1,20 +1,10 @@ -Input::EVENT_NEW, cat |, input0 -hello -Input::EVENT_NEW, cat |, input0 -there\x01\x02\x03\x04\x05\x01\x02\x03yay0 -Input::EVENT_NEW, cat |, input1 -hello -Input::EVENT_NEW, cat |, input1 -there\x01\x02\x03\x04\x05\x01\x02\x03yay01 -Input::EVENT_NEW, cat |, input2 -hello -Input::EVENT_NEW, cat |, input2 -there\x01\x02\x03\x04\x05\x01\x02\x03yay012 -Input::EVENT_NEW, cat |, input3 -hello -Input::EVENT_NEW, cat |, input3 -there\x01\x02\x03\x04\x05\x01\x02\x03yay0123 -Input::EVENT_NEW, cat |, input4 -hello -Input::EVENT_NEW, cat |, input4 -there\x01\x02\x03\x04\x05\x01\x02\x03yay01234 +Input::EVENT_NEW, cat |, input0, hello +Input::EVENT_NEW, cat |, input0, there\x01\x02\x03\x04\x05\x01\x02\x03yay0 +Input::EVENT_NEW, cat |, input1, hello +Input::EVENT_NEW, cat |, input1, there\x01\x02\x03\x04\x05\x01\x02\x03yay01 +Input::EVENT_NEW, cat |, input4, hello +Input::EVENT_NEW, cat |, input4, there\x01\x02\x03\x04\x05\x01\x02\x03yay01234 +Input::EVENT_NEW, cat |, input2, hello +Input::EVENT_NEW, cat |, input2, there\x01\x02\x03\x04\x05\x01\x02\x03yay012 +Input::EVENT_NEW, cat |, input3, hello +Input::EVENT_NEW, cat |, input3, there\x01\x02\x03\x04\x05\x01\x02\x03yay0123 diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.raw.executestream/out b/testing/btest/Baseline/scripts.base.frameworks.input.raw.executestream/out index 1705220b28..df7331e5e2 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.input.raw.executestream/out +++ b/testing/btest/Baseline/scripts.base.frameworks.input.raw.executestream/out @@ -1,153 +1,25 @@ -[source=tail -f ../input.log |, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=, want_record=F, ev=line -{ -print A::outfile, A::description; -print A::outfile, A::tpe; -print A::outfile, A::s; -A::try = A::try + 1; -if (8 == A::try) -{ -print A::outfile, done; -close(A::outfile); -Input::remove(input); -terminate(); -} - -}, error_ev=, config={ - -}] +tail -f ../input.log |, Input::READER_RAW, Input::STREAM, input Input::EVENT_NEW sdfkh:KH;fdkncv;ISEUp34:Fkdj;YVpIODhfDF -[source=tail -f ../input.log |, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=, want_record=F, ev=line -{ -print A::outfile, A::description; -print A::outfile, A::tpe; -print A::outfile, A::s; -A::try = A::try + 1; -if (8 == A::try) -{ -print A::outfile, done; -close(A::outfile); -Input::remove(input); -terminate(); -} - -}, error_ev=, config={ - -}] +tail -f ../input.log |, Input::READER_RAW, Input::STREAM, input Input::EVENT_NEW DSF"DFKJ"SDFKLh304yrsdkfj@#(*U$34jfDJup3UF -[source=tail -f ../input.log |, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=, want_record=F, ev=line -{ -print A::outfile, A::description; -print A::outfile, A::tpe; -print A::outfile, A::s; -A::try = A::try + 1; -if (8 == A::try) -{ -print A::outfile, done; -close(A::outfile); -Input::remove(input); -terminate(); -} - -}, error_ev=, config={ - -}] +tail -f ../input.log |, Input::READER_RAW, Input::STREAM, input Input::EVENT_NEW q3r3057fdf -[source=tail -f ../input.log |, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=, want_record=F, ev=line -{ -print A::outfile, A::description; -print A::outfile, A::tpe; -print A::outfile, A::s; -A::try = A::try + 1; -if (8 == A::try) -{ -print A::outfile, done; -close(A::outfile); -Input::remove(input); -terminate(); -} - -}, error_ev=, config={ - -}] +tail -f ../input.log |, Input::READER_RAW, Input::STREAM, input Input::EVENT_NEW sdfs\d -[source=tail -f ../input.log |, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=, want_record=F, ev=line -{ -print A::outfile, A::description; -print A::outfile, A::tpe; -print A::outfile, A::s; -A::try = A::try + 1; -if (8 == A::try) -{ -print A::outfile, done; -close(A::outfile); -Input::remove(input); -terminate(); -} - -}, error_ev=, config={ - -}] +tail -f ../input.log |, Input::READER_RAW, Input::STREAM, input Input::EVENT_NEW -[source=tail -f ../input.log |, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=, want_record=F, ev=line -{ -print A::outfile, A::description; -print A::outfile, A::tpe; -print A::outfile, A::s; -A::try = A::try + 1; -if (8 == A::try) -{ -print A::outfile, done; -close(A::outfile); -Input::remove(input); -terminate(); -} - -}, error_ev=, config={ - -}] +tail -f ../input.log |, Input::READER_RAW, Input::STREAM, input Input::EVENT_NEW dfsdf -[source=tail -f ../input.log |, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=, want_record=F, ev=line -{ -print A::outfile, A::description; -print A::outfile, A::tpe; -print A::outfile, A::s; -A::try = A::try + 1; -if (8 == A::try) -{ -print A::outfile, done; -close(A::outfile); -Input::remove(input); -terminate(); -} - -}, error_ev=, config={ - -}] +tail -f ../input.log |, Input::READER_RAW, Input::STREAM, input Input::EVENT_NEW sdf -[source=tail -f ../input.log |, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=, want_record=F, ev=line -{ -print A::outfile, A::description; -print A::outfile, A::tpe; -print A::outfile, A::s; -A::try = A::try + 1; -if (8 == A::try) -{ -print A::outfile, done; -close(A::outfile); -Input::remove(input); -terminate(); -} - -}, error_ev=, config={ - -}] +tail -f ../input.log |, Input::READER_RAW, Input::STREAM, input Input::EVENT_NEW 3rw43wRRERLlL#RWERERERE. done diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.raw.streamraw/out b/testing/btest/Baseline/scripts.base.frameworks.input.raw.streamraw/out index a2082f154b..16822c34a4 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.input.raw.streamraw/out +++ b/testing/btest/Baseline/scripts.base.frameworks.input.raw.streamraw/out @@ -1,153 +1,25 @@ -[source=../input.log, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=, want_record=F, ev=line -{ -print A::outfile, A::description; -print A::outfile, A::tpe; -print A::outfile, A::s; -A::try = A::try + 1; -if (8 == A::try) -{ -print A::outfile, done; -close(A::outfile); -Input::remove(input); -terminate(); -} - -}, error_ev=, config={ - -}] +../input.log, Input::READER_RAW, Input::STREAM, input Input::EVENT_NEW sdfkh:KH;fdkncv;ISEUp34:Fkdj;YVpIODhfDF -[source=../input.log, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=, want_record=F, ev=line -{ -print A::outfile, A::description; -print A::outfile, A::tpe; -print A::outfile, A::s; -A::try = A::try + 1; -if (8 == A::try) -{ -print A::outfile, done; -close(A::outfile); -Input::remove(input); -terminate(); -} - -}, error_ev=, config={ - -}] +../input.log, Input::READER_RAW, Input::STREAM, input Input::EVENT_NEW DSF"DFKJ"SDFKLh304yrsdkfj@#(*U$34jfDJup3UF -[source=../input.log, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=, want_record=F, ev=line -{ -print A::outfile, A::description; -print A::outfile, A::tpe; -print A::outfile, A::s; -A::try = A::try + 1; -if (8 == A::try) -{ -print A::outfile, done; -close(A::outfile); -Input::remove(input); -terminate(); -} - -}, error_ev=, config={ - -}] +../input.log, Input::READER_RAW, Input::STREAM, input Input::EVENT_NEW q3r3057fdf -[source=../input.log, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=, want_record=F, ev=line -{ -print A::outfile, A::description; -print A::outfile, A::tpe; -print A::outfile, A::s; -A::try = A::try + 1; -if (8 == A::try) -{ -print A::outfile, done; -close(A::outfile); -Input::remove(input); -terminate(); -} - -}, error_ev=, config={ - -}] +../input.log, Input::READER_RAW, Input::STREAM, input Input::EVENT_NEW sdfs\d -[source=../input.log, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=, want_record=F, ev=line -{ -print A::outfile, A::description; -print A::outfile, A::tpe; -print A::outfile, A::s; -A::try = A::try + 1; -if (8 == A::try) -{ -print A::outfile, done; -close(A::outfile); -Input::remove(input); -terminate(); -} - -}, error_ev=, config={ - -}] +../input.log, Input::READER_RAW, Input::STREAM, input Input::EVENT_NEW -[source=../input.log, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=, want_record=F, ev=line -{ -print A::outfile, A::description; -print A::outfile, A::tpe; -print A::outfile, A::s; -A::try = A::try + 1; -if (8 == A::try) -{ -print A::outfile, done; -close(A::outfile); -Input::remove(input); -terminate(); -} - -}, error_ev=, config={ - -}] +../input.log, Input::READER_RAW, Input::STREAM, input Input::EVENT_NEW dfsdf -[source=../input.log, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=, want_record=F, ev=line -{ -print A::outfile, A::description; -print A::outfile, A::tpe; -print A::outfile, A::s; -A::try = A::try + 1; -if (8 == A::try) -{ -print A::outfile, done; -close(A::outfile); -Input::remove(input); -terminate(); -} - -}, error_ev=, config={ - -}] +../input.log, Input::READER_RAW, Input::STREAM, input Input::EVENT_NEW sdf -[source=../input.log, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=, want_record=F, ev=line -{ -print A::outfile, A::description; -print A::outfile, A::tpe; -print A::outfile, A::s; -A::try = A::try + 1; -if (8 == A::try) -{ -print A::outfile, done; -close(A::outfile); -Input::remove(input); -terminate(); -} - -}, error_ev=, config={ - -}] +../input.log, Input::READER_RAW, Input::STREAM, input Input::EVENT_NEW 3rw43wRRERLlL#RWERERERE. done diff --git a/testing/btest/Baseline/scripts.base.frameworks.netcontrol.basic-cluster/worker-2..stdout b/testing/btest/Baseline/scripts.base.frameworks.netcontrol.basic-cluster/worker-2..stdout index c638f34077..587a51d2b8 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.netcontrol.basic-cluster/worker-2..stdout +++ b/testing/btest/Baseline/scripts.base.frameworks.netcontrol.basic-cluster/worker-2..stdout @@ -1,3 +1,4 @@ Rule added, worker-2:2, 4 Rule added, worker-2:3, 5 1 +Rule destroyed, worker-2:3, 5, 0 diff --git a/testing/btest/core/leaks/basic-cluster.bro b/testing/btest/core/leaks/basic-cluster.bro index 103146495b..8ea5d9c6dc 100644 --- a/testing/btest/core/leaks/basic-cluster.bro +++ b/testing/btest/core/leaks/basic-cluster.bro @@ -6,7 +6,6 @@ # @TEST-REQUIRES: bro --help 2>&1 | grep -q mem-leaks # # @TEST-EXEC: btest-bg-run manager-1 HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local BROPATH=$BROPATH:.. CLUSTER_NODE=manager-1 bro -m %INPUT -# @TEST-EXEC: sleep 1 # @TEST-EXEC: btest-bg-run worker-1 HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local BROPATH=$BROPATH:.. CLUSTER_NODE=worker-1 bro -m %INPUT # @TEST-EXEC: btest-bg-run worker-2 HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local BROPATH=$BROPATH:.. CLUSTER_NODE=worker-2 bro -m %INPUT # @TEST-EXEC: btest-bg-wait 60 @@ -19,6 +18,10 @@ redef Cluster::nodes = { }; @TEST-END-FILE +redef Cluster::retry_interval = 1sec; +redef Broker::default_listen_retry = 1sec; +redef Broker::default_connect_retry = 1sec; + redef Log::default_rotation_interval = 0secs; global n = 0; diff --git a/testing/btest/core/leaks/hll_cluster.bro b/testing/btest/core/leaks/hll_cluster.bro index f856f5d633..6a92f1a2e7 100644 --- a/testing/btest/core/leaks/hll_cluster.bro +++ b/testing/btest/core/leaks/hll_cluster.bro @@ -7,7 +7,6 @@ # # @TEST-EXEC: bro -m %INPUT>out # @TEST-EXEC: btest-bg-run manager-1 HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local BROPATH=$BROPATH:.. CLUSTER_NODE=manager-1 bro -m %INPUT -# @TEST-EXEC: sleep 2 # @TEST-EXEC: btest-bg-run worker-1 HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local BROPATH=$BROPATH:.. CLUSTER_NODE=worker-1 bro -m runnumber=1 %INPUT # @TEST-EXEC: btest-bg-run worker-2 HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local BROPATH=$BROPATH:.. CLUSTER_NODE=worker-2 bro -m runnumber=2 %INPUT # @TEST-EXEC: btest-bg-wait 60 @@ -24,6 +23,10 @@ redef Cluster::nodes = { }; @TEST-END-FILE +redef Cluster::retry_interval = 1sec; +redef Broker::default_listen_retry = 1sec; +redef Broker::default_connect_retry = 1sec; + redef Log::default_rotation_interval = 0secs; global hll_data: event(data: opaque of cardinality); diff --git a/testing/btest/core/leaks/input-raw.bro b/testing/btest/core/leaks/input-raw.bro index 602232da77..1a7315bc2a 100644 --- a/testing/btest/core/leaks/input-raw.bro +++ b/testing/btest/core/leaks/input-raw.bro @@ -6,9 +6,9 @@ # # @TEST-EXEC: cp input1.log input.log # @TEST-EXEC: HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local btest-bg-run bro bro -m -b %INPUT -# @TEST-EXEC: sleep 5 +# @TEST-EXEC: $SCRIPTS/wait-for-file bro/got2 8 || (btest-bg-wait -k 1 && false) # @TEST-EXEC: cat input2.log >> input.log -# @TEST-EXEC: sleep 5 +# @TEST-EXEC: $SCRIPTS/wait-for-file bro/got6 8 || (btest-bg-wait -k 1 && false) # @TEST-EXEC: cat input3.log >> input.log # @TEST-EXEC: btest-bg-wait 60 @@ -48,7 +48,12 @@ event line(description: Input::EventDescription, tpe: Input::Event, s: string) print outfile, s; try = try + 1; - if ( try == 16 ) + + if ( try == 2 ) + system("touch got2"); + else if ( try == 6 ) + system("touch got6"); + else if ( try == 16 ) { print outfile, "done"; close(outfile); diff --git a/testing/btest/core/leaks/input-reread.bro b/testing/btest/core/leaks/input-reread.bro index b3d1498bff..8b6295c15d 100644 --- a/testing/btest/core/leaks/input-reread.bro +++ b/testing/btest/core/leaks/input-reread.bro @@ -6,13 +6,13 @@ # # @TEST-EXEC: cp input1.log input.log # @TEST-EXEC: HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local btest-bg-run bro bro -m -b %INPUT -# @TEST-EXEC: sleep 60 +# @TEST-EXEC: $SCRIPTS/wait-for-file bro/got2 60 || (btest-bg-wait -k 1 && false) # @TEST-EXEC: cp input2.log input.log -# @TEST-EXEC: sleep 10 +# @TEST-EXEC: $SCRIPTS/wait-for-file bro/got4 10 || (btest-bg-wait -k 1 && false) # @TEST-EXEC: cp input3.log input.log -# @TEST-EXEC: sleep 10 +# @TEST-EXEC: $SCRIPTS/wait-for-file bro/got6 10 || (btest-bg-wait -k 1 && false) # @TEST-EXEC: cp input4.log input.log -# @TEST-EXEC: sleep 10 +# @TEST-EXEC: $SCRIPTS/wait-for-file bro/got8 10 || (btest-bg-wait -k 1 && false) # @TEST-EXEC: cp input5.log input.log # @TEST-EXEC: btest-bg-wait 120 @@ -145,7 +145,16 @@ event Input::end_of_data(name: string, source: string) } try = try + 1; - if ( try == 10 ) + + if ( try == 2 ) + system("touch got2"); + else if ( try == 4 ) + system("touch got4"); + else if ( try == 6 ) + system("touch got6"); + else if ( try == 8 ) + system("touch got8"); + else if ( try == 10 ) { print outfile, "done"; close(outfile); diff --git a/testing/btest/scripts/base/frameworks/cluster/custom_pool_exclusivity.bro b/testing/btest/scripts/base/frameworks/cluster/custom_pool_exclusivity.bro index 5f337093b0..c94b594daf 100644 --- a/testing/btest/scripts/base/frameworks/cluster/custom_pool_exclusivity.bro +++ b/testing/btest/scripts/base/frameworks/cluster/custom_pool_exclusivity.bro @@ -1,7 +1,6 @@ # @TEST-SERIALIZE: comm # # @TEST-EXEC: btest-bg-run manager-1 BROPATH=$BROPATH:.. CLUSTER_NODE=manager-1 bro %INPUT -# @TEST-EXEC: sleep 1 # @TEST-EXEC: btest-bg-run proxy-1 BROPATH=$BROPATH:.. CLUSTER_NODE=proxy-1 bro %INPUT # @TEST-EXEC: btest-bg-run proxy-2 BROPATH=$BROPATH:.. CLUSTER_NODE=proxy-2 bro %INPUT # @TEST-EXEC: btest-bg-wait 30 @@ -17,6 +16,10 @@ redef Cluster::nodes = { }; @TEST-END-FILE +redef Cluster::retry_interval = 1sec; +redef Broker::default_listen_retry = 1sec; +redef Broker::default_connect_retry = 1sec; + global my_pool_spec: Cluster::PoolSpec = Cluster::PoolSpec( $topic = "bro/cluster/pool/my_pool", diff --git a/testing/btest/scripts/base/frameworks/cluster/custom_pool_limits.bro b/testing/btest/scripts/base/frameworks/cluster/custom_pool_limits.bro index 10a190f016..cb099bc715 100644 --- a/testing/btest/scripts/base/frameworks/cluster/custom_pool_limits.bro +++ b/testing/btest/scripts/base/frameworks/cluster/custom_pool_limits.bro @@ -1,7 +1,6 @@ # @TEST-SERIALIZE: comm # # @TEST-EXEC: btest-bg-run manager-1 BROPATH=$BROPATH:.. CLUSTER_NODE=manager-1 bro %INPUT -# @TEST-EXEC: sleep 1 # @TEST-EXEC: btest-bg-run proxy-1 BROPATH=$BROPATH:.. CLUSTER_NODE=proxy-1 bro %INPUT # @TEST-EXEC: btest-bg-run proxy-2 BROPATH=$BROPATH:.. CLUSTER_NODE=proxy-2 bro %INPUT # @TEST-EXEC: btest-bg-wait 30 @@ -17,6 +16,10 @@ redef Cluster::nodes = { }; @TEST-END-FILE +redef Cluster::retry_interval = 1sec; +redef Broker::default_listen_retry = 1sec; +redef Broker::default_connect_retry = 1sec; + global my_pool_spec: Cluster::PoolSpec = Cluster::PoolSpec( $topic = "bro/cluster/pool/my_pool", diff --git a/testing/btest/scripts/base/frameworks/cluster/log_distribution.bro b/testing/btest/scripts/base/frameworks/cluster/log_distribution.bro index 02c1b7b6e7..5e710016ba 100644 --- a/testing/btest/scripts/base/frameworks/cluster/log_distribution.bro +++ b/testing/btest/scripts/base/frameworks/cluster/log_distribution.bro @@ -2,9 +2,7 @@ # # @TEST-EXEC: btest-bg-run logger-1 BROPATH=$BROPATH:.. CLUSTER_NODE=logger-1 bro %INPUT # @TEST-EXEC: btest-bg-run logger-2 BROPATH=$BROPATH:.. CLUSTER_NODE=logger-2 bro %INPUT -# @TEST-EXEC: sleep 1 # @TEST-EXEC: btest-bg-run manager BROPATH=$BROPATH:.. CLUSTER_NODE=manager bro %INPUT -# @TEST-EXEC: sleep 1 # @TEST-EXEC: btest-bg-run worker-1 BROPATH=$BROPATH:.. CLUSTER_NODE=worker-1 bro %INPUT # @TEST-EXEC: btest-bg-wait 30 # @TEST-EXEC: btest-diff logger-1/test.log @@ -22,6 +20,10 @@ redef Cluster::nodes = { @TEST-END-FILE +redef Cluster::retry_interval = 1sec; +redef Broker::default_listen_retry = 1sec; +redef Broker::default_connect_retry = 1sec; + redef Log::default_rotation_interval = 0sec; module Test; diff --git a/testing/btest/scripts/base/frameworks/cluster/start-it-up-logger.bro b/testing/btest/scripts/base/frameworks/cluster/start-it-up-logger.bro index 6fb834cc74..6bb9dcbc03 100644 --- a/testing/btest/scripts/base/frameworks/cluster/start-it-up-logger.bro +++ b/testing/btest/scripts/base/frameworks/cluster/start-it-up-logger.bro @@ -1,12 +1,9 @@ # @TEST-SERIALIZE: comm # # @TEST-EXEC: btest-bg-run logger-1 CLUSTER_NODE=logger-1 BROPATH=$BROPATH:.. bro %INPUT -# @TEST-EXEC: sleep 1 # @TEST-EXEC: btest-bg-run manager-1 CLUSTER_NODE=manager-1 BROPATH=$BROPATH:.. bro %INPUT -# @TEST-EXEC: sleep 1 # @TEST-EXEC: btest-bg-run proxy-1 CLUSTER_NODE=proxy-1 BROPATH=$BROPATH:.. bro %INPUT # @TEST-EXEC: btest-bg-run proxy-2 CLUSTER_NODE=proxy-2 BROPATH=$BROPATH:.. bro %INPUT -# @TEST-EXEC: sleep 1 # @TEST-EXEC: btest-bg-run worker-1 CLUSTER_NODE=worker-1 BROPATH=$BROPATH:.. bro %INPUT # @TEST-EXEC: btest-bg-run worker-2 CLUSTER_NODE=worker-2 BROPATH=$BROPATH:.. bro %INPUT # @TEST-EXEC: btest-bg-wait 30 @@ -29,6 +26,10 @@ redef Cluster::nodes = { }; @TEST-END-FILE +redef Cluster::retry_interval = 1sec; +redef Broker::default_listen_retry = 1sec; +redef Broker::default_connect_retry = 1sec; + global fully_connected: event(); global peer_count = 0; diff --git a/testing/btest/scripts/base/frameworks/cluster/start-it-up.bro b/testing/btest/scripts/base/frameworks/cluster/start-it-up.bro index 3b21cee3dc..be974c074f 100644 --- a/testing/btest/scripts/base/frameworks/cluster/start-it-up.bro +++ b/testing/btest/scripts/base/frameworks/cluster/start-it-up.bro @@ -1,10 +1,8 @@ # @TEST-SERIALIZE: comm # # @TEST-EXEC: btest-bg-run manager-1 BROPATH=$BROPATH:.. CLUSTER_NODE=manager-1 bro %INPUT -# @TEST-EXEC: sleep 1 # @TEST-EXEC: btest-bg-run proxy-1 BROPATH=$BROPATH:.. CLUSTER_NODE=proxy-1 bro %INPUT # @TEST-EXEC: btest-bg-run proxy-2 BROPATH=$BROPATH:.. CLUSTER_NODE=proxy-2 bro %INPUT -# @TEST-EXEC: sleep 1 # @TEST-EXEC: btest-bg-run worker-1 BROPATH=$BROPATH:.. CLUSTER_NODE=worker-1 bro %INPUT # @TEST-EXEC: btest-bg-run worker-2 BROPATH=$BROPATH:.. CLUSTER_NODE=worker-2 bro %INPUT # @TEST-EXEC: btest-bg-wait 30 @@ -24,6 +22,10 @@ redef Cluster::nodes = { }; @TEST-END-FILE +redef Cluster::retry_interval = 1sec; +redef Broker::default_listen_retry = 1sec; +redef Broker::default_connect_retry = 1sec; + global fully_connected: event(); global peer_count = 0; diff --git a/testing/btest/scripts/base/frameworks/cluster/topic_distribution.bro b/testing/btest/scripts/base/frameworks/cluster/topic_distribution.bro index 591a3329b7..e360ac55ef 100644 --- a/testing/btest/scripts/base/frameworks/cluster/topic_distribution.bro +++ b/testing/btest/scripts/base/frameworks/cluster/topic_distribution.bro @@ -1,7 +1,6 @@ # @TEST-SERIALIZE: comm # # @TEST-EXEC: btest-bg-run manager-1 BROPATH=$BROPATH:.. CLUSTER_NODE=manager-1 bro %INPUT -# @TEST-EXEC: sleep 1 # @TEST-EXEC: btest-bg-run proxy-1 BROPATH=$BROPATH:.. CLUSTER_NODE=proxy-1 bro %INPUT # @TEST-EXEC: btest-bg-run proxy-2 BROPATH=$BROPATH:.. CLUSTER_NODE=proxy-2 bro %INPUT # @TEST-EXEC: btest-bg-wait 30 @@ -17,6 +16,10 @@ redef Cluster::nodes = { }; @TEST-END-FILE +redef Cluster::retry_interval = 1sec; +redef Broker::default_listen_retry = 1sec; +redef Broker::default_connect_retry = 1sec; + global proxy_count = 0; event go_away() diff --git a/testing/btest/scripts/base/frameworks/cluster/topic_distribution_bifs.bro b/testing/btest/scripts/base/frameworks/cluster/topic_distribution_bifs.bro index 2f2462e752..9e79081906 100644 --- a/testing/btest/scripts/base/frameworks/cluster/topic_distribution_bifs.bro +++ b/testing/btest/scripts/base/frameworks/cluster/topic_distribution_bifs.bro @@ -1,7 +1,6 @@ # @TEST-SERIALIZE: comm # # @TEST-EXEC: btest-bg-run manager-1 BROPATH=$BROPATH:.. CLUSTER_NODE=manager-1 bro %INPUT -# @TEST-EXEC: sleep 1 # @TEST-EXEC: btest-bg-run proxy-1 BROPATH=$BROPATH:.. CLUSTER_NODE=proxy-1 bro %INPUT # @TEST-EXEC: btest-bg-run proxy-2 BROPATH=$BROPATH:.. CLUSTER_NODE=proxy-2 bro %INPUT # @TEST-EXEC: btest-bg-wait 30 @@ -19,6 +18,10 @@ redef Cluster::nodes = { }; @TEST-END-FILE +redef Cluster::retry_interval = 1sec; +redef Broker::default_listen_retry = 1sec; +redef Broker::default_connect_retry = 1sec; + global proxy_count = 0; global q = 0; diff --git a/testing/btest/scripts/base/frameworks/config/updates.bro b/testing/btest/scripts/base/frameworks/config/updates.bro index b9ecd013cd..1e523c752f 100644 --- a/testing/btest/scripts/base/frameworks/config/updates.bro +++ b/testing/btest/scripts/base/frameworks/config/updates.bro @@ -1,11 +1,11 @@ # @TEST-EXEC: btest-bg-run bro bro -b %INPUT -# @TEST-EXEC: sleep 2 +# @TEST-EXEC: $SCRIPTS/wait-for-file bro/got1 5 || (btest-bg-wait -k 1 && false) # @TEST-EXEC: mv configfile2 configfile # @TEST-EXEC: touch configfile -# @TEST-EXEC: sleep 2 +# @TEST-EXEC: $SCRIPTS/wait-for-file bro/got2 5 || (btest-bg-wait -k 1 && false) # @TEST-EXEC: mv configfile3 configfile # @TEST-EXEC: touch configfile -# @TEST-EXEC: sleep 2 +# @TEST-EXEC: $SCRIPTS/wait-for-file bro/got3 5 || (btest-bg-wait -k 1 && false) # @TEST-EXEC: mv configfile4 configfile # @TEST-EXEC: touch configfile # @TEST-EXEC: btest-bg-wait 10 @@ -103,6 +103,12 @@ event Input::end_of_data(name: string, source:string) eolcount += 1; - if ( eolcount == 4 ) + if ( eolcount == 1 ) + system("touch got1"); + else if ( eolcount == 2 ) + system("touch got2"); + else if ( eolcount == 3 ) + system("touch got3"); + else if ( eolcount == 4 ) terminate(); } diff --git a/testing/btest/scripts/base/frameworks/control/configuration_update.bro b/testing/btest/scripts/base/frameworks/control/configuration_update.bro index 5e459cc9f0..d3fef8e1b5 100644 --- a/testing/btest/scripts/base/frameworks/control/configuration_update.bro +++ b/testing/btest/scripts/base/frameworks/control/configuration_update.bro @@ -1,13 +1,14 @@ # @TEST-SERIALIZE: comm # # @TEST-EXEC: btest-bg-run controllee BROPATH=$BROPATH:.. bro %INPUT frameworks/control/controllee Broker::default_port=65531/tcp -# @TEST-EXEC: sleep 5 # @TEST-EXEC: btest-bg-run controller BROPATH=$BROPATH:.. bro %INPUT test-redef frameworks/control/controller Control::host=127.0.0.1 Control::host_port=65531/tcp Control::cmd=configuration_update -# @TEST-EXEC: sleep 5 -# @TEST-EXEC: btest-bg-run controller2 BROPATH=$BROPATH:.. bro %INPUT frameworks/control/controller Control::host=127.0.0.1 Control::host_port=65531/tcp Control::cmd=shutdown # @TEST-EXEC: btest-bg-wait 10 # @TEST-EXEC: btest-diff controllee/.stdout +redef Cluster::retry_interval = 1sec; +redef Broker::default_listen_retry = 1sec; +redef Broker::default_connect_retry = 1sec; + const test_var = "ORIGINAL VALUE (this should be printed out first)" &redef; @TEST-START-FILE test-redef.bro @@ -23,3 +24,8 @@ event bro_done() { print test_var; } + +event Broker::peer_lost(endpoint: Broker::EndpointInfo, msg: string) + { + terminate(); + } diff --git a/testing/btest/scripts/base/frameworks/input/empty-values-hashing.bro b/testing/btest/scripts/base/frameworks/input/empty-values-hashing.bro index d12ab864f2..f25c9bc3f6 100644 --- a/testing/btest/scripts/base/frameworks/input/empty-values-hashing.bro +++ b/testing/btest/scripts/base/frameworks/input/empty-values-hashing.bro @@ -1,6 +1,6 @@ # @TEST-EXEC: cp input1.log input.log # @TEST-EXEC: btest-bg-run bro bro -b %INPUT -# @TEST-EXEC: sleep 2 +# @TEST-EXEC: $SCRIPTS/wait-for-file bro/got1 5 || (btest-bg-wait -k 1 && false) # @TEST-EXEC: cp input2.log input.log # @TEST-EXEC: btest-bg-wait 10 # @TEST-EXEC: btest-diff out @@ -75,7 +75,9 @@ event Input::end_of_data(name: string, source: string) print outfile, servers; try = try + 1; - if ( try == 2 ) + if ( try == 1 ) + system("touch got1"); + else if ( try == 2 ) { print outfile, "done"; close(outfile); diff --git a/testing/btest/scripts/base/frameworks/input/missing-file-initially.bro b/testing/btest/scripts/base/frameworks/input/missing-file-initially.bro index 1de9fbd539..7c9f51994c 100644 --- a/testing/btest/scripts/base/frameworks/input/missing-file-initially.bro +++ b/testing/btest/scripts/base/frameworks/input/missing-file-initially.bro @@ -4,12 +4,12 @@ # failing behavior. # @TEST-EXEC: btest-bg-run bro bro %INPUT -# @TEST-EXEC: sleep 10 +# @TEST-EXEC: $SCRIPTS/wait-for-file bro/init 5 || (btest-bg-wait -k 1 && false) # @TEST-EXEC: mv does-exist.dat does-not-exist.dat -# @TEST-EXEC: sleep 2 +# @TEST-EXEC: $SCRIPTS/wait-for-file bro/next 5 || (btest-bg-wait -k 1 && false) # @TEST-EXEC: mv does-not-exist.dat does-not-exist-again.dat # @TEST-EXEC: echo "3 streaming still works" >> does-not-exist-again.dat -# @TEST-EXEC: btest-bg-wait -k 3 +# @TEST-EXEC: btest-bg-wait 5 # @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-sort btest-diff bro/.stdout # @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-sort btest-diff bro/.stderr @@ -31,9 +31,17 @@ type Val: record { line: string; }; +global line_count = 0; + event line(description: Input::EventDescription, tpe: Input::Event, v: Val) { print fmt("%s: %s", description$name, v$line); + ++line_count; + + if ( line_count == 4 ) + system("touch next"); + if ( line_count == 5 ) + terminate(); } event line2(description: Input::EventDescription, tpe: Input::Event, v: Val) @@ -49,4 +57,5 @@ event bro_init() Input::add_event([$source="../does-not-exist.dat", $name="inputmanual", $reader=Input::READER_ASCII, $mode=Input::MANUAL, $fields=Val, $ev=line, $want_record=T]); Input::add_event([$source="../does-not-exist.dat", $name="input2", $reader=Input::READER_ASCII, $mode=Input::REREAD, $fields=Val, $ev=line2, $want_record=T, $config=table(["fail_on_file_problem"] = "T")]); + system("touch init"); } diff --git a/testing/btest/scripts/base/frameworks/input/predicatemodifyandreread.bro b/testing/btest/scripts/base/frameworks/input/predicatemodifyandreread.bro index 526d1e113f..0ac5f104d0 100644 --- a/testing/btest/scripts/base/frameworks/input/predicatemodifyandreread.bro +++ b/testing/btest/scripts/base/frameworks/input/predicatemodifyandreread.bro @@ -1,12 +1,12 @@ # @TEST-EXEC: cp input1.log input.log # @TEST-EXEC: btest-bg-run bro bro -b %INPUT -# @TEST-EXEC: sleep 2 +# @TEST-EXEC: $SCRIPTS/wait-for-file bro/got1 5 || (btest-bg-wait -k 1 && false) # @TEST-EXEC: cp input2.log input.log -# @TEST-EXEC: sleep 2 +# @TEST-EXEC: $SCRIPTS/wait-for-file bro/got2 5 || (btest-bg-wait -k 1 && false) # @TEST-EXEC: cp input3.log input.log -# @TEST-EXEC: sleep 2 +# @TEST-EXEC: $SCRIPTS/wait-for-file bro/got3 5 || (btest-bg-wait -k 1 && false) # @TEST-EXEC: cp input4.log input.log -# @TEST-EXEC: sleep 2 +# @TEST-EXEC: $SCRIPTS/wait-for-file bro/got4 5 || (btest-bg-wait -k 1 && false) # @TEST-EXEC: cp input5.log input.log # @TEST-EXEC: btest-bg-wait 10 # @TEST-EXEC: btest-diff out @@ -96,7 +96,15 @@ event Input::end_of_data(name: string, source: string) try = try + 1; print outfile, fmt("Update_finished for %s, try %d", name, try); print outfile, servers; - + + if ( try == 1 ) + system("touch got1"); + else if ( try == 2 ) + system("touch got2"); + else if ( try == 3 ) + system("touch got3"); + else if ( try == 4 ) + system("touch got4"); if ( try == 5 ) { close(outfile); diff --git a/testing/btest/scripts/base/frameworks/input/raw/executestdin.bro b/testing/btest/scripts/base/frameworks/input/raw/executestdin.bro index 0edc53a0e4..b78dd4e0e3 100644 --- a/testing/btest/scripts/base/frameworks/input/raw/executestdin.bro +++ b/testing/btest/scripts/base/frameworks/input/raw/executestdin.bro @@ -7,6 +7,7 @@ redef exit_only_after_terminate = T; global outfile: file; global processes_finished: count = 0; +global lines_received: count = 0; global n: count = 0; global total_processes: count = 0; @@ -20,10 +21,23 @@ type Val: record { s: string; }; +global more_input: function(name_prefix: string); + +function check_terminate_condition() + { + if ( processes_finished != total_processes ) + return; + + if ( lines_received != (total_processes - 1) * 2 ) + return; + + terminate(); + } + event line(description: Input::EventDescription, tpe: Input::Event, s: string) { - print outfile, tpe, description$source, description$name; - print outfile, s; + ++lines_received; + print outfile, tpe, description$source, description$name, s; } event InputRaw::process_finished(name: string, source:string, exit_code:count, signal_exit:bool) @@ -31,10 +45,18 @@ event InputRaw::process_finished(name: string, source:string, exit_code:count, s print "process_finished", name, source; Input::remove(name); ++processes_finished; - if ( processes_finished == total_processes ) + if ( processes_finished == 1 ) + { + more_input("input"); + more_input("input"); + more_input("input"); + more_input("input"); + more_input("input"); + } + else if ( processes_finished == total_processes ) { close(outfile); - terminate(); + check_terminate_condition(); } } @@ -59,9 +81,4 @@ event bro_init() $reader=Input::READER_RAW, $mode=Input::STREAM, $name="input", $fields=Val, $ev=line, $want_record=F, $config=config_strings]); - more_input("input"); - more_input("input"); - more_input("input"); - more_input("input"); - more_input("input"); } diff --git a/testing/btest/scripts/base/frameworks/input/raw/executestream.bro b/testing/btest/scripts/base/frameworks/input/raw/executestream.bro index 77cb425fa4..240761ee03 100644 --- a/testing/btest/scripts/base/frameworks/input/raw/executestream.bro +++ b/testing/btest/scripts/base/frameworks/input/raw/executestream.bro @@ -1,8 +1,8 @@ # @TEST-EXEC: cp input1.log input.log # @TEST-EXEC: btest-bg-run bro bro -b %INPUT -# @TEST-EXEC: sleep 3 +# @TEST-EXEC: $SCRIPTS/wait-for-file bro/got1 5 || (btest-bg-wait -k 1 && false) # @TEST-EXEC: cat input2.log >> input.log -# @TEST-EXEC: sleep 3 +# @TEST-EXEC: $SCRIPTS/wait-for-file bro/got3 5 || (btest-bg-wait -k 1 && false) # @TEST-EXEC: cat input3.log >> input.log # @TEST-EXEC: btest-bg-wait 10 # @TEST-EXEC: btest-diff out @@ -38,12 +38,16 @@ global outfile: file; event line(description: Input::EventDescription, tpe: Input::Event, s: string) { - print outfile, description; + print outfile, description$source, description$reader, description$mode, description$name; print outfile, tpe; print outfile, s; try = try + 1; - if ( try == 8 ) + if ( try == 1 ) + system("touch got1"); + else if ( try == 3 ) + system("touch got3"); + else if ( try == 8 ) { print outfile, "done"; close(outfile); diff --git a/testing/btest/scripts/base/frameworks/input/raw/offset.bro b/testing/btest/scripts/base/frameworks/input/raw/offset.bro index 7ce040e1c9..f37fb9c28a 100644 --- a/testing/btest/scripts/base/frameworks/input/raw/offset.bro +++ b/testing/btest/scripts/base/frameworks/input/raw/offset.bro @@ -1,6 +1,6 @@ # @TEST-EXEC: cp input.log input2.log # @TEST-EXEC: btest-bg-run bro bro -b %INPUT -# @TEST-EXEC: sleep 2 +# @TEST-EXEC: $SCRIPTS/wait-for-file bro/got2 5 || (btest-bg-wait -k 1 && false) # @TEST-EXEC: echo "hi" >> input2.log # @TEST-EXEC: btest-bg-wait 10 # @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-sort btest-diff out @@ -24,7 +24,9 @@ event line(description: Input::EventDescription, tpe: Input::Event, s: string) { print outfile, s; try = try + 1; - if ( try == 3 ) + if ( try == 2 ) + system("touch got2"); + else if ( try == 3 ) { close(outfile); terminate(); diff --git a/testing/btest/scripts/base/frameworks/input/raw/streamraw.bro b/testing/btest/scripts/base/frameworks/input/raw/streamraw.bro index 6e45ba32b7..331db7eeb8 100644 --- a/testing/btest/scripts/base/frameworks/input/raw/streamraw.bro +++ b/testing/btest/scripts/base/frameworks/input/raw/streamraw.bro @@ -1,8 +1,8 @@ # @TEST-EXEC: cp input1.log input.log # @TEST-EXEC: btest-bg-run bro bro -b %INPUT -# @TEST-EXEC: sleep 3 +# @TEST-EXEC: $SCRIPTS/wait-for-file bro/got1 5 || (btest-bg-wait -k 1 && false) # @TEST-EXEC: cat input2.log >> input.log -# @TEST-EXEC: sleep 3 +# @TEST-EXEC: $SCRIPTS/wait-for-file bro/got3 5 || (btest-bg-wait -k 1 && false) # @TEST-EXEC: cat input3.log >> input.log # @TEST-EXEC: btest-bg-wait 10 # @TEST-EXEC: btest-diff out @@ -37,12 +37,17 @@ global outfile: file; event line(description: Input::EventDescription, tpe: Input::Event, s: string) { - print outfile, description; + print outfile, description$source, description$reader, description$mode, description$name; print outfile, tpe; print outfile, s; try = try + 1; - if ( try == 8 ) + + if ( try == 1 ) + system("touch got1"); + else if ( try == 3 ) + system("touch got3"); + else if ( try == 8 ) { print outfile, "done"; close(outfile); diff --git a/testing/btest/scripts/base/frameworks/input/reread.bro b/testing/btest/scripts/base/frameworks/input/reread.bro index d8cb868d22..4199093543 100644 --- a/testing/btest/scripts/base/frameworks/input/reread.bro +++ b/testing/btest/scripts/base/frameworks/input/reread.bro @@ -1,12 +1,12 @@ # @TEST-EXEC: cp input1.log input.log # @TEST-EXEC: btest-bg-run bro bro -b %INPUT -# @TEST-EXEC: sleep 2 +# @TEST-EXEC: $SCRIPTS/wait-for-file bro/got1 5 || (btest-bg-wait -k 1 && false) # @TEST-EXEC: cp input2.log input.log -# @TEST-EXEC: sleep 2 +# @TEST-EXEC: $SCRIPTS/wait-for-file bro/got2 5 || (btest-bg-wait -k 1 && false) # @TEST-EXEC: cp input3.log input.log -# @TEST-EXEC: sleep 2 +# @TEST-EXEC: $SCRIPTS/wait-for-file bro/got3 5 || (btest-bg-wait -k 1 && false) # @TEST-EXEC: cp input4.log input.log -# @TEST-EXEC: sleep 2 +# @TEST-EXEC: $SCRIPTS/wait-for-file bro/got4 5 || (btest-bg-wait -k 1 && false) # @TEST-EXEC: cp input5.log input.log # @TEST-EXEC: btest-bg-wait 10 # @TEST-EXEC: btest-diff out @@ -126,7 +126,16 @@ event Input::end_of_data(name: string, source: string) print outfile, servers; try = try + 1; - if ( try == 5 ) + + if ( try == 1 ) + system("touch got1"); + else if ( try == 2 ) + system("touch got2"); + else if ( try == 3 ) + system("touch got3"); + else if ( try == 4 ) + system("touch got4"); + else if ( try == 5 ) { print outfile, "done"; close(outfile); diff --git a/testing/btest/scripts/base/frameworks/input/stream.bro b/testing/btest/scripts/base/frameworks/input/stream.bro index ed497859aa..8ed498f074 100644 --- a/testing/btest/scripts/base/frameworks/input/stream.bro +++ b/testing/btest/scripts/base/frameworks/input/stream.bro @@ -1,8 +1,8 @@ # @TEST-EXEC: cp input1.log input.log # @TEST-EXEC: btest-bg-run bro bro -b %INPUT -# @TEST-EXEC: sleep 3 +# @TEST-EXEC: $SCRIPTS/wait-for-file bro/got1 5 || (btest-bg-wait -k 1 && false) # @TEST-EXEC: cat input2.log >> input.log -# @TEST-EXEC: sleep 3 +# @TEST-EXEC: $SCRIPTS/wait-for-file bro/got2 5 || (btest-bg-wait -k 1 && false) # @TEST-EXEC: cat input3.log >> input.log # @TEST-EXEC: btest-bg-wait 10 # @TEST-EXEC: btest-diff out @@ -66,8 +66,12 @@ event line(description: Input::TableDescription, tpe: Input::Event, left: Idx, r print outfile, servers; try = try + 1; - - if ( try == 3 ) + + if ( try == 1 ) + system("touch got1"); + else if ( try == 2 ) + system("touch got2"); + else if ( try == 3 ) { print outfile, "done"; close(outfile); diff --git a/testing/btest/scripts/base/frameworks/input/twotables.bro b/testing/btest/scripts/base/frameworks/input/twotables.bro index f0bedb2673..6f127ac4c2 100644 --- a/testing/btest/scripts/base/frameworks/input/twotables.bro +++ b/testing/btest/scripts/base/frameworks/input/twotables.bro @@ -1,6 +1,6 @@ # @TEST-EXEC: cp input1.log input.log # @TEST-EXEC: btest-bg-run bro bro -b %INPUT -# @TEST-EXEC: sleep 5 +# @TEST-EXEC: $SCRIPTS/wait-for-file bro/got2 5 || (btest-bg-wait -k 1 && false) # @TEST-EXEC: cp input3.log input.log # @TEST-EXEC: btest-bg-wait 10 # @TEST-EXEC: btest-diff event.out @@ -116,7 +116,9 @@ event Input::end_of_data(name: string, source: string) #print fin_out, servers; try = try + 1; - if ( try == 3 ) + if ( try == 2 ) + system("touch got2"); + else if ( try == 3 ) { print fin_out, "done"; print fin_out, servers; diff --git a/testing/btest/scripts/base/frameworks/intel/input-and-match.bro b/testing/btest/scripts/base/frameworks/intel/input-and-match.bro index 774f17fc57..8f74117201 100644 --- a/testing/btest/scripts/base/frameworks/intel/input-and-match.bro +++ b/testing/btest/scripts/base/frameworks/intel/input-and-match.bro @@ -1,4 +1,3 @@ -# @TEST-SERIALIZE: comm # @TEST-EXEC: btest-bg-run broproc bro %INPUT # @TEST-EXEC: btest-bg-wait -k 5 diff --git a/testing/btest/scripts/base/frameworks/intel/read-file-dist-cluster.bro b/testing/btest/scripts/base/frameworks/intel/read-file-dist-cluster.bro index 5488c4938e..b34e273d54 100644 --- a/testing/btest/scripts/base/frameworks/intel/read-file-dist-cluster.bro +++ b/testing/btest/scripts/base/frameworks/intel/read-file-dist-cluster.bro @@ -1,7 +1,6 @@ # @TEST-SERIALIZE: comm # # @TEST-EXEC: btest-bg-run manager-1 BROPATH=$BROPATH:.. CLUSTER_NODE=manager-1 bro %INPUT -# @TEST-EXEC: sleep 2 # @TEST-EXEC: btest-bg-run worker-1 BROPATH=$BROPATH:.. CLUSTER_NODE=worker-1 bro %INPUT # @TEST-EXEC: btest-bg-run worker-2 BROPATH=$BROPATH:.. CLUSTER_NODE=worker-2 bro %INPUT # @TEST-EXEC: btest-bg-wait -k 10 @@ -26,6 +25,10 @@ e@mail.com Intel::EMAIL source1 Phishing email source http://some-data-distribut @TEST-END-FILE @load base/frameworks/control +redef Cluster::retry_interval = 1sec; +redef Broker::default_listen_retry = 1sec; +redef Broker::default_connect_retry = 1sec; + redef Log::default_rotation_interval=0sec; module Intel; diff --git a/testing/btest/scripts/base/frameworks/intel/updated-match.bro b/testing/btest/scripts/base/frameworks/intel/updated-match.bro index fd7c738210..5cace1741e 100644 --- a/testing/btest/scripts/base/frameworks/intel/updated-match.bro +++ b/testing/btest/scripts/base/frameworks/intel/updated-match.bro @@ -1,12 +1,10 @@ -# @TEST-SERIALIZE: comm - # @TEST-EXEC: cp intel1.dat intel.dat # @TEST-EXEC: btest-bg-run broproc bro %INPUT -# @TEST-EXEC: sleep 2 +# @TEST-EXEC: $SCRIPTS/wait-for-file broproc/got1 5 || (btest-bg-wait -k 1 && false) # @TEST-EXEC: cp intel2.dat intel.dat -# @TEST-EXEC: sleep 2 +# @TEST-EXEC: $SCRIPTS/wait-for-file broproc/got2 5 || (btest-bg-wait -k 1 && false) # @TEST-EXEC: cp intel3.dat intel.dat -# @TEST-EXEC: btest-bg-wait 6 +# @TEST-EXEC: btest-bg-wait 10 # @TEST-EXEC: cat broproc/intel.log > output # @TEST-EXEC: cat broproc/notice.log >> output # @TEST-EXEC: btest-diff output @@ -35,6 +33,8 @@ redef Intel::read_files += { "../intel.dat" }; redef enum Intel::Where += { SOMEWHERE }; global runs = 0; +global entries_read = 0; + event do_it() { Intel::seen([$host=1.2.3.4, @@ -43,8 +43,11 @@ event do_it() $where=SOMEWHERE]); ++runs; - if ( runs < 3 ) - schedule 3sec { do_it() }; + + if ( runs == 1 ) + system("touch got1"); + if ( runs == 2 ) + system("touch got2"); } global log_lines = 0; @@ -55,7 +58,17 @@ event Intel::log_intel(rec: Intel::Info) terminate(); } -event bro_init() &priority=-10 +module Intel; + +event Intel::read_entry(desc: Input::EventDescription, tpe: Input::Event, item: Intel::Item) { - schedule 1sec { do_it() }; + ++entries_read; + print entries_read; + + if ( entries_read == 1 ) + event do_it(); + else if ( entries_read == 3 ) + event do_it(); + else if ( entries_read == 5 ) + event do_it(); } diff --git a/testing/btest/scripts/base/frameworks/logging/field-extension-cluster-error.bro b/testing/btest/scripts/base/frameworks/logging/field-extension-cluster-error.bro index 9def14cc2a..03108505d5 100644 --- a/testing/btest/scripts/base/frameworks/logging/field-extension-cluster-error.bro +++ b/testing/btest/scripts/base/frameworks/logging/field-extension-cluster-error.bro @@ -1,7 +1,6 @@ # @TEST-SERIALIZE: comm # # @TEST-EXEC: btest-bg-run manager-1 "cp ../cluster-layout.bro . && CLUSTER_NODE=manager-1 bro %INPUT" -# @TEST-EXEC: sleep 1 # @TEST-EXEC: btest-bg-run worker-1 "cp ../cluster-layout.bro . && CLUSTER_NODE=worker-1 bro --pseudo-realtime -C -r $TRACES/wikipedia.trace %INPUT" # @TEST-EXEC: btest-bg-wait 20 # @TEST-EXEC: cat manager-1/reporter.log | grep -v "reporter/" > manager-reporter.log @@ -21,6 +20,10 @@ redef Cluster::nodes = { redef exit_only_after_terminate = T; @endif +redef Cluster::retry_interval = 1sec; +redef Broker::default_listen_retry = 1sec; +redef Broker::default_connect_retry = 1sec; + redef Log::default_rotation_interval = 0secs; redef Log::default_scope_sep="_"; diff --git a/testing/btest/scripts/base/frameworks/logging/field-extension-cluster.bro b/testing/btest/scripts/base/frameworks/logging/field-extension-cluster.bro index 711a1286aa..3c464311f5 100644 --- a/testing/btest/scripts/base/frameworks/logging/field-extension-cluster.bro +++ b/testing/btest/scripts/base/frameworks/logging/field-extension-cluster.bro @@ -1,7 +1,6 @@ # @TEST-SERIALIZE: comm # # @TEST-EXEC: btest-bg-run manager-1 "cp ../cluster-layout.bro . && CLUSTER_NODE=manager-1 bro %INPUT" -# @TEST-EXEC: sleep 1 # @TEST-EXEC: btest-bg-run worker-1 "cp ../cluster-layout.bro . && CLUSTER_NODE=worker-1 bro --pseudo-realtime -C -r $TRACES/wikipedia.trace %INPUT" # @TEST-EXEC: btest-bg-wait 20 # @TEST-EXEC: btest-diff manager-1/http.log @@ -20,6 +19,9 @@ redef Cluster::nodes = { redef exit_only_after_terminate = T; @endif +redef Cluster::retry_interval = 1sec; +redef Broker::default_listen_retry = 1sec; +redef Broker::default_connect_retry = 1sec; redef Log::default_rotation_interval = 0secs; redef Log::default_scope_sep="_"; @@ -59,9 +61,12 @@ event bro_init() { if ( Cluster::node == "worker-1" ) Broker::subscribe("death"); + } +event Broker::peer_added(endpoint: Broker::EndpointInfo, msg: string) + { if ( Cluster::node == "manager-1" ) - schedule 13sec { kill_worker() }; + schedule 2sec { kill_worker() }; } event Broker::peer_lost(endpoint: Broker::EndpointInfo, msg: string) diff --git a/testing/btest/scripts/base/frameworks/netcontrol/basic-cluster.bro b/testing/btest/scripts/base/frameworks/netcontrol/basic-cluster.bro index bdf7f3f75d..fc9a308297 100644 --- a/testing/btest/scripts/base/frameworks/netcontrol/basic-cluster.bro +++ b/testing/btest/scripts/base/frameworks/netcontrol/basic-cluster.bro @@ -1,9 +1,10 @@ # @TEST-SERIALIZE: comm # # @TEST-EXEC: btest-bg-run manager-1 "cp ../cluster-layout.bro . && CLUSTER_NODE=manager-1 bro %INPUT" -# @TEST-EXEC: sleep 1 # @TEST-EXEC: btest-bg-run worker-1 "cp ../cluster-layout.bro . && CLUSTER_NODE=worker-1 bro --pseudo-realtime -C -r $TRACES/tls/ecdhe.pcap %INPUT" -# @TEST-EXEC: sleep 1 + +# @TEST-EXEC: $SCRIPTS/wait-for-pid $(cat worker-1/.pid) 10 || (btest-bg-wait -k 1 && false) + # @TEST-EXEC: btest-bg-run worker-2 "cp ../cluster-layout.bro . && CLUSTER_NODE=worker-2 bro --pseudo-realtime -C -r $TRACES/tls/ecdhe.pcap %INPUT" # @TEST-EXEC: btest-bg-wait 20 # @TEST-EXEC: btest-diff worker-1/.stdout @@ -17,6 +18,10 @@ redef Cluster::nodes = { }; @TEST-END-FILE +redef Cluster::retry_interval = 1sec; +redef Broker::default_listen_retry = 1sec; +redef Broker::default_connect_retry = 1sec; + redef Log::default_rotation_interval = 0secs; #redef exit_only_after_terminate = T; @@ -51,9 +56,14 @@ event terminate_me() { terminate(); } +global peers_lost = 0; + event Broker::peer_lost(endpoint: Broker::EndpointInfo, msg: string) { - schedule 1sec { terminate_me() }; + ++peers_lost; + + if ( peers_lost == 2 ) + schedule 2sec { terminate_me() }; } event NetControl::rule_added(r: NetControl::Rule, p: NetControl::PluginState, msg: string &default="") diff --git a/testing/btest/scripts/base/frameworks/notice/cluster.bro b/testing/btest/scripts/base/frameworks/notice/cluster.bro index 6784daf068..9bb80422b1 100644 --- a/testing/btest/scripts/base/frameworks/notice/cluster.bro +++ b/testing/btest/scripts/base/frameworks/notice/cluster.bro @@ -2,7 +2,6 @@ # # @TEST-EXEC: btest-bg-run manager-1 BROPATH=$BROPATH:.. CLUSTER_NODE=manager-1 bro %INPUT # @TEST-EXEC: btest-bg-run proxy-1 BROPATH=$BROPATH:.. CLUSTER_NODE=proxy-1 bro %INPUT -# @TEST-EXEC: sleep 2 # @TEST-EXEC: btest-bg-run worker-1 BROPATH=$BROPATH:.. CLUSTER_NODE=worker-1 bro %INPUT # @TEST-EXEC: btest-bg-wait 20 # @TEST-EXEC: btest-diff manager-1/notice.log @@ -15,6 +14,10 @@ redef Cluster::nodes = { }; @TEST-END-FILE +redef Cluster::retry_interval = 1sec; +redef Broker::default_listen_retry = 1sec; +redef Broker::default_connect_retry = 1sec; + redef Log::default_rotation_interval = 0secs; redef enum Notice::Type += { diff --git a/testing/btest/scripts/base/frameworks/notice/suppression-cluster.bro b/testing/btest/scripts/base/frameworks/notice/suppression-cluster.bro index c67512853f..6c9e429bc9 100644 --- a/testing/btest/scripts/base/frameworks/notice/suppression-cluster.bro +++ b/testing/btest/scripts/base/frameworks/notice/suppression-cluster.bro @@ -2,7 +2,6 @@ # # @TEST-EXEC: btest-bg-run manager-1 BROPATH=$BROPATH:.. CLUSTER_NODE=manager-1 bro %INPUT # @TEST-EXEC: btest-bg-run proxy-1 BROPATH=$BROPATH:.. CLUSTER_NODE=proxy-1 bro %INPUT -# @TEST-EXEC: sleep 2 # @TEST-EXEC: btest-bg-run worker-1 BROPATH=$BROPATH:.. CLUSTER_NODE=worker-1 bro %INPUT # @TEST-EXEC: btest-bg-run worker-2 BROPATH=$BROPATH:.. CLUSTER_NODE=worker-2 bro %INPUT # @TEST-EXEC: btest-bg-wait 20 @@ -17,6 +16,10 @@ redef Cluster::nodes = { }; @TEST-END-FILE +redef Cluster::retry_interval = 1sec; +redef Broker::default_listen_retry = 1sec; +redef Broker::default_connect_retry = 1sec; + redef Log::default_rotation_interval = 0secs; redef enum Notice::Type += { diff --git a/testing/btest/scripts/base/frameworks/openflow/log-cluster.bro b/testing/btest/scripts/base/frameworks/openflow/log-cluster.bro index c618be7e65..de957e720e 100644 --- a/testing/btest/scripts/base/frameworks/openflow/log-cluster.bro +++ b/testing/btest/scripts/base/frameworks/openflow/log-cluster.bro @@ -1,7 +1,6 @@ # @TEST-SERIALIZE: comm # # @TEST-EXEC: btest-bg-run manager-1 "cp ../cluster-layout.bro . && CLUSTER_NODE=manager-1 bro %INPUT" -# @TEST-EXEC: sleep 1 # @TEST-EXEC: btest-bg-run worker-1 "cp ../cluster-layout.bro . && CLUSTER_NODE=worker-1 bro --pseudo-realtime -C -r $TRACES/smtp.trace %INPUT" # @TEST-EXEC: btest-bg-wait 20 # @TEST-EXEC: btest-diff manager-1/openflow.log @@ -13,6 +12,9 @@ redef Cluster::nodes = { }; @TEST-END-FILE +redef Cluster::retry_interval = 1sec; +redef Broker::default_listen_retry = 1sec; +redef Broker::default_connect_retry = 1sec; redef Log::default_rotation_interval = 0secs; #redef exit_only_after_terminate = T; @@ -21,6 +23,18 @@ redef Log::default_rotation_interval = 0secs; global of_controller: OpenFlow::Controller; +@if ( Cluster::local_node_type() == Cluster::WORKER ) +event bro_init() + { + suspend_processing(); + } + +event Broker::peer_added(endpoint: Broker::EndpointInfo, msg: string) + { + continue_processing(); + } +@endif + event bro_init() { of_controller = OpenFlow::log_new(42); diff --git a/testing/btest/scripts/base/frameworks/sumstats/basic-cluster.bro b/testing/btest/scripts/base/frameworks/sumstats/basic-cluster.bro index 2c744228a0..e02b3143c5 100644 --- a/testing/btest/scripts/base/frameworks/sumstats/basic-cluster.bro +++ b/testing/btest/scripts/base/frameworks/sumstats/basic-cluster.bro @@ -1,7 +1,6 @@ # @TEST-SERIALIZE: comm # # @TEST-EXEC: btest-bg-run manager-1 BROPATH=$BROPATH:.. CLUSTER_NODE=manager-1 bro %INPUT -# @TEST-EXEC: sleep 1 # @TEST-EXEC: btest-bg-run worker-1 BROPATH=$BROPATH:.. CLUSTER_NODE=worker-1 bro %INPUT # @TEST-EXEC: btest-bg-run worker-2 BROPATH=$BROPATH:.. CLUSTER_NODE=worker-2 bro %INPUT # @TEST-EXEC: btest-bg-wait 15 @@ -16,6 +15,10 @@ redef Cluster::nodes = { }; @TEST-END-FILE +redef Cluster::retry_interval = 1sec; +redef Broker::default_listen_retry = 1sec; +redef Broker::default_connect_retry = 1sec; + redef Log::default_rotation_interval = 0secs; global n = 0; diff --git a/testing/btest/scripts/base/frameworks/sumstats/cluster-intermediate-update.bro b/testing/btest/scripts/base/frameworks/sumstats/cluster-intermediate-update.bro index ae0f093c27..18df7fe768 100644 --- a/testing/btest/scripts/base/frameworks/sumstats/cluster-intermediate-update.bro +++ b/testing/btest/scripts/base/frameworks/sumstats/cluster-intermediate-update.bro @@ -1,7 +1,6 @@ # @TEST-SERIALIZE: comm # # @TEST-EXEC: btest-bg-run manager-1 BROPATH=$BROPATH:.. CLUSTER_NODE=manager-1 bro %INPUT -# @TEST-EXEC: sleep 3 # @TEST-EXEC: btest-bg-run worker-1 BROPATH=$BROPATH:.. CLUSTER_NODE=worker-1 bro %INPUT # @TEST-EXEC: btest-bg-run worker-2 BROPATH=$BROPATH:.. CLUSTER_NODE=worker-2 bro %INPUT # @TEST-EXEC: btest-bg-wait 20 @@ -15,6 +14,10 @@ redef Cluster::nodes = { }; @TEST-END-FILE +redef Cluster::retry_interval = 1sec; +redef Broker::default_listen_retry = 1sec; +redef Broker::default_connect_retry = 1sec; + redef Log::default_rotation_interval = 0secs; event bro_init() &priority=5 diff --git a/testing/btest/scripts/base/frameworks/sumstats/on-demand-cluster.bro b/testing/btest/scripts/base/frameworks/sumstats/on-demand-cluster.bro index ac2aacc03c..adf61ffb82 100644 --- a/testing/btest/scripts/base/frameworks/sumstats/on-demand-cluster.bro +++ b/testing/btest/scripts/base/frameworks/sumstats/on-demand-cluster.bro @@ -1,7 +1,6 @@ # @TEST-SERIALIZE: comm # # @TEST-EXEC: btest-bg-run manager-1 BROPATH=$BROPATH:.. CLUSTER_NODE=manager-1 bro %INPUT -# @TEST-EXEC: sleep 1 # @TEST-EXEC: btest-bg-run worker-1 BROPATH=$BROPATH:.. CLUSTER_NODE=worker-1 bro %INPUT # @TEST-EXEC: btest-bg-run worker-2 BROPATH=$BROPATH:.. CLUSTER_NODE=worker-2 bro %INPUT # @TEST-EXEC: btest-bg-wait 15 @@ -17,6 +16,10 @@ redef Cluster::nodes = { }; @TEST-END-FILE +redef Cluster::retry_interval = 1sec; +redef Broker::default_listen_retry = 1sec; +redef Broker::default_connect_retry = 1sec; + redef Log::default_rotation_interval = 0secs; global n = 0; diff --git a/testing/btest/scripts/base/frameworks/sumstats/sample-cluster.bro b/testing/btest/scripts/base/frameworks/sumstats/sample-cluster.bro index 935a57bb5d..6426b42680 100644 --- a/testing/btest/scripts/base/frameworks/sumstats/sample-cluster.bro +++ b/testing/btest/scripts/base/frameworks/sumstats/sample-cluster.bro @@ -1,7 +1,6 @@ # @TEST-SERIALIZE: comm # # @TEST-EXEC: btest-bg-run manager-1 BROPATH=$BROPATH:.. CLUSTER_NODE=manager-1 bro %INPUT -# @TEST-EXEC: sleep 1 # @TEST-EXEC: btest-bg-run worker-1 BROPATH=$BROPATH:.. CLUSTER_NODE=worker-1 bro %INPUT # @TEST-EXEC: btest-bg-run worker-2 BROPATH=$BROPATH:.. CLUSTER_NODE=worker-2 bro %INPUT # @TEST-EXEC: btest-bg-wait 15 @@ -15,6 +14,9 @@ redef Cluster::nodes = { }; @TEST-END-FILE +redef Cluster::retry_interval = 1sec; +redef Broker::default_listen_retry = 1sec; +redef Broker::default_connect_retry = 1sec; redef Log::default_rotation_interval = 0secs; event bro_init() &priority=5 diff --git a/testing/btest/scripts/base/frameworks/sumstats/topk-cluster.bro b/testing/btest/scripts/base/frameworks/sumstats/topk-cluster.bro index 57a08aa040..3ab90e91b8 100644 --- a/testing/btest/scripts/base/frameworks/sumstats/topk-cluster.bro +++ b/testing/btest/scripts/base/frameworks/sumstats/topk-cluster.bro @@ -1,7 +1,6 @@ # @TEST-SERIALIZE: comm # # @TEST-EXEC: btest-bg-run manager-1 BROPATH=$BROPATH:.. CLUSTER_NODE=manager-1 bro %INPUT -# @TEST-EXEC: sleep 1 # @TEST-EXEC: btest-bg-run worker-1 BROPATH=$BROPATH:.. CLUSTER_NODE=worker-1 bro %INPUT # @TEST-EXEC: btest-bg-run worker-2 BROPATH=$BROPATH:.. CLUSTER_NODE=worker-2 bro %INPUT # @TEST-EXEC: btest-bg-wait 15 @@ -16,6 +15,9 @@ redef Cluster::nodes = { }; @TEST-END-FILE +redef Cluster::retry_interval = 1sec; +redef Broker::default_listen_retry = 1sec; +redef Broker::default_connect_retry = 1sec; redef Log::default_rotation_interval = 0secs; From b9a5d9ccbe0f86a67b9bc7b644d9cd849f49d050 Mon Sep 17 00:00:00 2001 From: Vern Paxson Date: Thu, 5 Jul 2018 10:13:20 -0700 Subject: [PATCH 16/24] de-restrict pattern-oriented BiFs to no longer require only running at init --- NEWS | 3 +++ src/bro.bif | 12 ------------ 2 files changed, 3 insertions(+), 12 deletions(-) diff --git a/NEWS b/NEWS index 8cd4f3c201..c898bcb3a1 100644 --- a/NEWS +++ b/NEWS @@ -249,6 +249,9 @@ New Functionality '^' are binary "and", "or" and "xor" operators, and '~' is a unary ones-complement operator. +- The string_to_pattern() built-in (and the now-deprecated merge_pattern() + built-in) is no longer restricted to only be called at initialization time. + Changed Functionality --------------------- diff --git a/src/bro.bif b/src/bro.bif index f0641104c9..835bcc3f7a 100644 --- a/src/bro.bif +++ b/src/bro.bif @@ -2960,12 +2960,6 @@ function uuid_to_string%(uuid: string%): string ## :bro:id:`bro_init`. function merge_pattern%(p1: pattern, p2: pattern%): pattern %{ - if ( bro_start_network_time != 0.0 ) - { - builtin_error("merge_pattern can only be called at init time"); - return 0; - } - reporter->Warning("merge_pattern() builtin-function has been deprecated"); RE_Matcher* re = new RE_Matcher(); @@ -3035,12 +3029,6 @@ function convert_for_pattern%(s: string%): string ## :bro:id:`bro_init`. function string_to_pattern%(s: string, convert: bool%): pattern %{ - if ( bro_start_network_time != 0.0 ) - { - builtin_error("string_to_pattern can only be called at init time"); - return 0; - } - const char* ss = (const char*) (s->Bytes()); int sn = s->Len(); char* pat; From 36400e2d6782c1bd997507b7ecf4b134fba6f74e Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Mon, 16 Jul 2018 10:12:36 -0500 Subject: [PATCH 17/24] Updating submodule(s). [nomail] --- aux/bro-aux | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aux/bro-aux b/aux/bro-aux index eeb677ff69..ed9764186e 160000 --- a/aux/bro-aux +++ b/aux/bro-aux @@ -1 +1 @@ -Subproject commit eeb677ff696f8ea3eaa43a765fe40da07ed5281d +Subproject commit ed9764186effc50172c2f6f77070ad35ff0e9002 From c09fe427a8783830b2a208d33b909d875054930d Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Mon, 16 Jul 2018 16:06:02 -0500 Subject: [PATCH 18/24] Improve Specific_RE_Matcher::CompileSet() error condition cleanup --- CHANGES | 5 +++++ VERSION | 2 +- src/RE.cc | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/CHANGES b/CHANGES index f8ed4a5982..e6463829e5 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,9 @@ +2.5-741 | 2018-07-16 16:06:02 -0500 + + * Improve Specific_RE_Matcher::CompileSet() error condition cleanup + (Jon Siwek, Corelight) + 2.5-740 | 2018-07-16 16:01:31 -0500 * Add support for case-insensitive patterns (Vern Paxson, Corelight) diff --git a/VERSION b/VERSION index d2ae53737a..86f56afeeb 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.5-740 +2.5-741 diff --git a/src/RE.cc b/src/RE.cc index 907670acd3..517fab4c91 100644 --- a/src/RE.cc +++ b/src/RE.cc @@ -166,7 +166,7 @@ int Specific_RE_Matcher::CompileSet(const string_list& set, const int_list& idx) { reporter->Error("error compiling pattern /%s/", set[i]); - if ( set_nfa != nfa ) + if ( set_nfa && set_nfa != nfa ) Unref(set_nfa); else Unref(nfa); From 4c072409f04122bf96916a7aca2851f8fd5fbf6e Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Mon, 16 Jul 2018 16:14:18 -0500 Subject: [PATCH 19/24] Updating submodule(s). [nomail] --- aux/bro-aux | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aux/bro-aux b/aux/bro-aux index ed9764186e..d2476564e6 160000 --- a/aux/bro-aux +++ b/aux/bro-aux @@ -1 +1 @@ -Subproject commit ed9764186effc50172c2f6f77070ad35ff0e9002 +Subproject commit d2476564e6934de8fcffc47bff1ae9733c3dde0c From 9caad8a04277b0993651415f1aa6d17b6caabba1 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Tue, 17 Jul 2018 14:20:19 -0500 Subject: [PATCH 20/24] Port broker::data variant usages to use CAF API directly Old code still all worked, but made use of Broker functions which now just redirect to CAF ones. --- CHANGES | 5 ++ VERSION | 2 +- aux/broker | 2 +- src/broker/Data.cc | 56 ++++++------- src/broker/Data.h | 4 +- src/broker/Manager.cc | 152 ++++++++++++++++++----------------- src/broker/data.bif | 4 +- src/logging/WriterBackend.cc | 18 ++--- 8 files changed, 125 insertions(+), 118 deletions(-) diff --git a/CHANGES b/CHANGES index e6463829e5..a4972bb8ea 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,9 @@ +2.5-743 | 2018-07-17 14:20:19 -0500 + + * Port broker::data variant usages to use CAF API directly + (Jon Siwek, Corelight) + 2.5-741 | 2018-07-16 16:06:02 -0500 * Improve Specific_RE_Matcher::CompileSet() error condition cleanup diff --git a/VERSION b/VERSION index 86f56afeeb..fd335625ba 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.5-741 +2.5-743 diff --git a/aux/broker b/aux/broker index 1d90b931cc..467024ab09 160000 --- a/aux/broker +++ b/aux/broker @@ -1 +1 @@ -Subproject commit 1d90b931cc888e31b0d4774dbae54f5758841ab3 +Subproject commit 467024ab09e3443798126e69054fb8862c69562f diff --git a/src/broker/Data.cc b/src/broker/Data.cc index 99c6e3ebef..b836b66002 100644 --- a/src/broker/Data.cc +++ b/src/broker/Data.cc @@ -215,7 +215,7 @@ struct val_converter { { auto expected_index_types = tt->Indices()->Types(); broker::vector composite_key; - auto indices = broker::get_if(item); + auto indices = caf::get_if(&item); if ( indices ) { @@ -283,7 +283,7 @@ struct val_converter { { auto expected_index_types = tt->Indices()->Types(); broker::vector composite_key; - auto indices = broker::get_if(item.first); + auto indices = caf::get_if(&item.first); if ( indices ) { @@ -384,7 +384,7 @@ struct val_converter { return nullptr; } - if ( broker::get_if(a[idx]) != nullptr ) + if ( caf::get_if(&a[idx]) != nullptr ) { rval->Assign(i, nullptr); ++idx; @@ -411,8 +411,8 @@ struct val_converter { if ( a.size() != 2 ) return nullptr; - auto exact_text = broker::get_if(a[0]); - auto anywhere_text = broker::get_if(a[1]); + auto exact_text = caf::get_if(&a[0]); + auto anywhere_text = caf::get_if(&a[1]); if ( ! exact_text || ! anywhere_text ) return nullptr; @@ -582,7 +582,7 @@ struct type_checker { for ( const auto& item : a ) { auto expected_index_types = tt->Indices()->Types(); - auto indices = broker::get_if(item); + auto indices = caf::get_if(&item); vector indices_to_check; if ( indices ) @@ -624,7 +624,7 @@ struct type_checker { auto expect = (*expected_index_types)[i]; auto& index_to_check = *(indices_to_check)[i]; - if ( ! broker::visit(type_checker{expect}, index_to_check) ) + if ( ! caf::visit(type_checker{expect}, index_to_check) ) return false; } } @@ -642,7 +642,7 @@ struct type_checker { for ( auto& item : a ) { auto expected_index_types = tt->Indices()->Types(); - auto indices = broker::get_if(item.first); + auto indices = caf::get_if(&item.first); vector indices_to_check; if ( indices ) @@ -689,11 +689,11 @@ struct type_checker { auto expect = (*expected_index_types)[i]; auto& index_to_check = *(indices_to_check)[i]; - if ( ! broker::visit(type_checker{expect}, index_to_check) ) + if ( ! caf::visit(type_checker{expect}, index_to_check) ) return false; } - if ( ! broker::visit(type_checker{tt->YieldType()}, + if ( ! caf::visit(type_checker{tt->YieldType()}, item.second) ) return false; } @@ -709,7 +709,7 @@ struct type_checker { for ( auto& item : a ) { - if ( ! broker::visit(type_checker{vt->YieldType()}, item) ) + if ( ! caf::visit(type_checker{vt->YieldType()}, item) ) return false; } @@ -725,13 +725,13 @@ struct type_checker { if ( idx >= a.size() ) return false; - if ( broker::get_if(a[idx]) != nullptr ) + if ( caf::get_if(&a[idx]) != nullptr ) { ++idx; continue; } - if ( ! broker::visit(type_checker{rt->FieldType(i)}, + if ( ! caf::visit(type_checker{rt->FieldType(i)}, a[idx]) ) return false; @@ -745,8 +745,8 @@ struct type_checker { if ( a.size() != 2 ) return false; - auto exact_text = broker::get_if(a[0]); - auto anywhere_text = broker::get_if(a[1]); + auto exact_text = caf::get_if(&a[0]); + auto anywhere_text = caf::get_if(&a[1]); if ( ! exact_text || ! anywhere_text ) return false; @@ -775,7 +775,7 @@ Val* bro_broker::data_to_val(broker::data d, BroType* type) if ( type->Tag() == TYPE_ANY ) return bro_broker::make_data_val(move(d)); - return broker::visit(val_converter{type}, std::move(d)); + return caf::visit(val_converter{type}, std::move(d)); } broker::expected bro_broker::val_to_data(Val* v) @@ -900,7 +900,7 @@ broker::expected bro_broker::val_to_data(Val* v) key = move(composite_key); if ( is_set ) - broker::get(rval).emplace(move(key)); + caf::get(rval).emplace(move(key)); else { auto val = val_to_data(entry->Value()); @@ -908,7 +908,7 @@ broker::expected bro_broker::val_to_data(Val* v) if ( ! val ) return broker::ec::invalid_data; - broker::get(rval).emplace(move(key), move(*val)); + caf::get(rval).emplace(move(key), move(*val)); } } @@ -1115,7 +1115,7 @@ struct data_type_getter { EnumVal* bro_broker::get_data_type(RecordVal* v, Frame* frame) { - return broker::visit(data_type_getter{}, opaque_field_to_data(v, frame)); + return caf::visit(data_type_getter{}, opaque_field_to_data(v, frame)); } broker::data& bro_broker::opaque_field_to_data(RecordVal* v, Frame* f) @@ -1131,7 +1131,7 @@ broker::data& bro_broker::opaque_field_to_data(RecordVal* v, Frame* f) bool bro_broker::DataVal::canCastTo(BroType* t) const { - return broker::visit(type_checker{t}, data); + return caf::visit(type_checker{t}, data); } Val* bro_broker::DataVal::castTo(BroType* t) @@ -1192,24 +1192,24 @@ broker::data bro_broker::threading_field_to_data(const threading::Field* f) threading::Field* bro_broker::data_to_threading_field(broker::data d) { - if ( ! broker::is(d) ) + if ( ! caf::holds_alternative(d) ) return nullptr; - auto& v = broker::get(d); - auto name = broker::get_if(v[0]); + auto& v = caf::get(d); + auto name = caf::get_if(&v[0]); auto secondary = v[1]; - auto type = broker::get_if(v[2]); - auto subtype = broker::get_if(v[3]); - auto optional = broker::get_if(v[4]); + auto type = caf::get_if(&v[2]); + auto subtype = caf::get_if(&v[3]); + auto optional = caf::get_if(&v[4]); if ( ! (name && type && subtype && optional) ) return nullptr; - if ( secondary != broker::nil && ! broker::is(secondary) ) + if ( secondary != broker::nil && ! caf::holds_alternative(secondary) ) return nullptr; return new threading::Field(name->c_str(), - secondary != broker::nil ? broker::get(secondary).c_str() : nullptr, + secondary != broker::nil ? caf::get(secondary).c_str() : nullptr, static_cast(*type), static_cast(*subtype), *optional); diff --git a/src/broker/Data.h b/src/broker/Data.h index 525faba5f6..e2a5968a82 100644 --- a/src/broker/Data.h +++ b/src/broker/Data.h @@ -210,11 +210,11 @@ broker::data& opaque_field_to_data(RecordVal* v, Frame* f); template T& require_data_type(broker::data& d, TypeTag tag, Frame* f) { - auto ptr = broker::get_if(d); + auto ptr = caf::get_if(&d); if ( ! ptr ) reporter->RuntimeError(f->GetCall()->GetLocationInfo(), "data is of type '%s' not of type '%s'", - broker::visit(type_name_getter{tag}, d), + caf::visit(type_name_getter{tag}, d), type_name(tag)); return *ptr; diff --git a/src/broker/Manager.cc b/src/broker/Manager.cc index 5def875ce7..a78ba2bea8 100644 --- a/src/broker/Manager.cc +++ b/src/broker/Manager.cc @@ -942,13 +942,13 @@ void Manager::Process() { had_input = true; - if ( auto stat = broker::get_if(status_msg) ) + if ( auto stat = caf::get_if(&status_msg) ) { ProcessStatus(std::move(*stat)); continue; } - if ( auto err = broker::get_if(status_msg) ) + if ( auto err = caf::get_if(&status_msg) ) { ProcessError(std::move(*err)); continue; @@ -1048,7 +1048,7 @@ void Manager::ProcessRelayEvent(broker::bro::RelayEvent ev) ++statistics.num_events_incoming; for ( auto& t : ev.topics() ) - PublishEvent(std::move(broker::get(t)), + PublishEvent(std::move(caf::get(t)), std::move(ev.name()), std::move(ev.args())); } @@ -1060,7 +1060,7 @@ void Manager::ProcessHandleAndRelayEvent(broker::bro::HandleAndRelayEvent ev) ProcessEvent(ev.name(), ev.args()); for ( auto& t : ev.topics() ) - PublishEvent(std::move(broker::get(t)), + PublishEvent(std::move(caf::get(t)), std::move(ev.name()), std::move(ev.args())); } @@ -1095,40 +1095,38 @@ bool bro_broker::Manager::ProcessLogCreate(broker::bro::LogCreate lc) } // Get log fields. + auto fields_data = caf::get_if(&lc.fields_data()); - try - { - auto fields_data = std::move(broker::get(lc.fields_data())); - auto num_fields = fields_data.size(); - auto fields = new threading::Field* [num_fields]; - - for ( auto i = 0u; i < num_fields; ++i ) - { - if ( auto field = data_to_threading_field(std::move(fields_data[i])) ) - fields[i] = field; - else - { - reporter->Warning("failed to convert remote log field # %d", i); - return false; - } - } - - if ( ! log_mgr->CreateWriterForRemoteLog(stream_id->AsEnumVal(), writer_id->AsEnumVal(), writer_info.get(), num_fields, fields) ) - { - ODesc d; - stream_id->Describe(&d); - reporter->Warning("failed to create remote log stream for %s locally", d.Description()); - } - - writer_info.release(); // log_mgr took ownership. - return true; - } - - catch (const broker::bad_variant_access& e) + if ( ! fields_data ) { reporter->Warning("failed to unpack remote log fields"); return false; } + + auto num_fields = fields_data->size(); + auto fields = new threading::Field* [num_fields]; + + for ( auto i = 0u; i < num_fields; ++i ) + { + if ( auto field = data_to_threading_field(std::move((*fields_data)[i])) ) + fields[i] = field; + else + { + reporter->Warning("failed to convert remote log field # %d", i); + delete [] fields; + return false; + } + } + + if ( ! log_mgr->CreateWriterForRemoteLog(stream_id->AsEnumVal(), writer_id->AsEnumVal(), writer_info.get(), num_fields, fields) ) + { + ODesc d; + stream_id->Describe(&d); + reporter->Warning("failed to create remote log stream for %s locally", d.Description()); + } + + writer_info.release(); // log_mgr took ownership. + return true; } bool bro_broker::Manager::ProcessLogWrite(broker::bro::LogWrite lw) @@ -1159,52 +1157,56 @@ bool bro_broker::Manager::ProcessLogWrite(broker::bro::LogWrite lw) } unref_guard writer_id_unreffer{writer_id}; + auto path = caf::get_if(&lw.path()); - try + if ( ! path ) { - auto& path = broker::get(lw.path()); - auto& serial_data = broker::get(lw.serial_data()); - - BinarySerializationFormat fmt; - fmt.StartRead(serial_data.data(), serial_data.size()); - - int num_fields; - bool success = fmt.Read(&num_fields, "num_fields"); - - if ( ! success ) - { - reporter->Warning("failed to unserialize remote log num fields for stream: %s", stream_id_name.data()); - return false; - } - - auto vals = new threading::Value* [num_fields]; - - for ( int i = 0; i < num_fields; ++i ) - { - vals[i] = new threading::Value; - - if ( ! vals[i]->Read(&fmt) ) - { - for ( int j = 0; j <=i; ++j ) - delete vals[j]; - - delete [] vals; - reporter->Warning("failed to unserialize remote log field %d for stream: %s", i, stream_id_name.data()); - - return false; - } - } - - log_mgr->WriteFromRemote(stream_id->AsEnumVal(), writer_id->AsEnumVal(), std::move(path), num_fields, vals); - fmt.EndRead(); - return true; - } - - catch ( const broker::bad_variant_access& e) - { - reporter->Warning("failed to unpack remote log values (bad variant) for stream: %s", stream_id_name.data()); + reporter->Warning("failed to unpack remote log values (bad path variant) for stream: %s", stream_id_name.data()); return false; } + + auto serial_data = caf::get_if(&lw.serial_data()); + + if ( ! serial_data ) + { + reporter->Warning("failed to unpack remote log values (bad serial_data variant) for stream: %s", stream_id_name.data()); + return false; + } + + BinarySerializationFormat fmt; + fmt.StartRead(serial_data->data(), serial_data->size()); + + int num_fields; + bool success = fmt.Read(&num_fields, "num_fields"); + + if ( ! success ) + { + reporter->Warning("failed to unserialize remote log num fields for stream: %s", stream_id_name.data()); + return false; + } + + auto vals = new threading::Value* [num_fields]; + + for ( int i = 0; i < num_fields; ++i ) + { + vals[i] = new threading::Value; + + if ( ! vals[i]->Read(&fmt) ) + { + for ( int j = 0; j <=i; ++j ) + delete vals[j]; + + delete [] vals; + reporter->Warning("failed to unserialize remote log field %d for stream: %s", i, stream_id_name.data()); + + return false; + } + } + + log_mgr->WriteFromRemote(stream_id->AsEnumVal(), writer_id->AsEnumVal(), + std::move(*path), num_fields, vals); + fmt.EndRead(); + return true; } bool Manager::ProcessIdentifierUpdate(broker::bro::IdentifierUpdate iu) diff --git a/src/broker/data.bif b/src/broker/data.bif index 658145089c..e874076434 100644 --- a/src/broker/data.bif +++ b/src/broker/data.bif @@ -457,7 +457,7 @@ function Broker::__record_lookup%(r: Broker::Data, idx: count%): Broker::Data auto& v = bro_broker::require_data_type(r->AsRecordVal(), TYPE_RECORD, frame); - if ( idx >= v.size() || broker::get_if(v[idx]) ) + if ( idx >= v.size() || caf::get_if(&v[idx]) ) return new RecordVal(BifType::Record::Broker::Data); return bro_broker::make_data_val(v[idx]); @@ -496,7 +496,7 @@ function Broker::__record_iterator_value%(it: opaque of Broker::RecordIterator%) return rval; } - if ( broker::get_if(*ri->it) ) + if ( caf::get_if(&(*ri->it)) ) return rval; // field isn't set rval->Assign(0, new bro_broker::DataVal(*ri->it)); diff --git a/src/logging/WriterBackend.cc b/src/logging/WriterBackend.cc index 69327e815d..4416e41d17 100644 --- a/src/logging/WriterBackend.cc +++ b/src/logging/WriterBackend.cc @@ -137,15 +137,15 @@ broker::data WriterBackend::WriterInfo::ToBroker() const bool WriterBackend::WriterInfo::FromBroker(broker::data d) { - if ( ! broker::is(d) ) + if ( ! caf::holds_alternative(d) ) return false; - auto v = broker::get(d); - auto bpath = broker::get_if(v[0]); - auto brotation_base = broker::get_if(v[1]); - auto brotation_interval = broker::get_if(v[2]); - auto bnetwork_time = broker::get_if(v[3]); - auto bconfig = broker::get_if(v[4]); + auto v = caf::get(d); + auto bpath = caf::get_if(&v[0]); + auto brotation_base = caf::get_if(&v[1]); + auto brotation_interval = caf::get_if(&v[2]); + auto bnetwork_time = caf::get_if(&v[3]); + auto bconfig = caf::get_if(&v[4]); if ( ! (bpath && brotation_base && brotation_interval && bnetwork_time && bconfig) ) return false; @@ -157,8 +157,8 @@ bool WriterBackend::WriterInfo::FromBroker(broker::data d) for ( auto i : *bconfig ) { - auto k = broker::get_if(i.first); - auto v = broker::get_if(i.second); + auto k = caf::get_if(&i.first); + auto v = caf::get_if(&i.second); if ( ! (k && v) ) return false; From 35b778eb4e18d0588d17a14bb2f77f7f4eda0c63 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Tue, 17 Jul 2018 14:42:52 -0500 Subject: [PATCH 21/24] Updating submodule(s). [nomail] --- aux/broker | 2 +- src/3rdparty | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/aux/broker b/aux/broker index 467024ab09..2285177616 160000 --- a/aux/broker +++ b/aux/broker @@ -1 +1 @@ -Subproject commit 467024ab09e3443798126e69054fb8862c69562f +Subproject commit 2285177616bf0e0bed6758bccf63f3dfee4d2b4f diff --git a/src/3rdparty b/src/3rdparty index 648ff9aee1..b7c6be774b 160000 --- a/src/3rdparty +++ b/src/3rdparty @@ -1 +1 @@ -Subproject commit 648ff9aee1bb568a1a8252bd6e8146a7c60a911e +Subproject commit b7c6be774b922be1e15f53571201c3be2bc28b75 From 1d1a63c16ccde3c7ce7b9f8932350a4d352da066 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Tue, 17 Jul 2018 16:46:16 -0500 Subject: [PATCH 22/24] Add explicit key in Travis known_hosts --- CHANGES | 4 ++++ VERSION | 2 +- testing/scripts/travis-job | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/CHANGES b/CHANGES index a4972bb8ea..5eb9d2dec0 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,8 @@ +2.5-745 | 2018-07-17 16:46:16 -0500 + + * Add explicit key in Travis known_hosts (Jon Siwek, Corelight) + 2.5-743 | 2018-07-17 14:20:19 -0500 * Port broker::data variant usages to use CAF API directly diff --git a/VERSION b/VERSION index fd335625ba..46f8a42564 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.5-743 +2.5-745 diff --git a/testing/scripts/travis-job b/testing/scripts/travis-job index f006b898ce..c6221d76a2 100644 --- a/testing/scripts/travis-job +++ b/testing/scripts/travis-job @@ -143,7 +143,7 @@ run() { chmod 600 travis_key mkdir -p ~/.ssh mv travis_key ~/.ssh/id_rsa - ssh-keyscan -H -p 22 -t rsa git.bro.org >> ~/.ssh/known_hosts + echo "git.bro.org ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEmlu+EaJfPKTVqoEUzb5JBEdvNiFxO2wm7Vl61dGBl57avakFl8YnRujbA2yxlpC2xnEKD5y++hXxtxRLefyCM=" >> ~/.ssh/known_hosts git clone ssh://git@git.bro.org/bro-testing-private rm ~/.ssh/id_rsa elif [ -n "${TRAVIS_PULL_REQUEST}" ] && [ "${TRAVIS_PULL_REQUEST}" != "false" ]; then From bf67076cdce063bcd7788dac449eb380943d34ab Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Tue, 17 Jul 2018 17:51:13 -0500 Subject: [PATCH 23/24] Improve an input framework unit test --- CHANGES | 4 ++ VERSION | 2 +- .../out | 36 ++++--------- .../base/frameworks/input/raw/stderr.bro | 52 +++++++++---------- 4 files changed, 38 insertions(+), 56 deletions(-) diff --git a/CHANGES b/CHANGES index 5eb9d2dec0..0f05f4fb59 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,8 @@ +2.5-746 | 2018-07-17 17:51:13 -0500 + + * Improve an input framework unit test (Jon Siwek, Corelight) + 2.5-745 | 2018-07-17 16:46:16 -0500 * Add explicit key in Travis known_hosts (Jon Siwek, Corelight) diff --git a/VERSION b/VERSION index 46f8a42564..fcd1315f19 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.5-745 +2.5-746 diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.raw.stderr/out b/testing/btest/Baseline/scripts.base.frameworks.input.raw.stderr/out index b7f857339d..65d0c26ab4 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.input.raw.stderr/out +++ b/testing/btest/Baseline/scripts.base.frameworks.input.raw.stderr/out @@ -1,27 +1,9 @@ -Input::EVENT_NEW -..: -F -Input::EVENT_NEW -bro -F -Input::EVENT_NEW -out -F -Input::EVENT_NEW -stderr.bro -F -Input::EVENT_NEW -stderr output contained nonexistant -T -Input::EVENT_NEW -stderr output contained nonexistant -T -Input::EVENT_NEW -stderr output contained nonexistant -T -done -End of Data event -input -Process finished event -input -Exit code != 0 +Input::EVENT_NEW line output (stderr=F): ../mydir: +Input::EVENT_NEW line output (stderr=F): a +Input::EVENT_NEW line output (stderr=F): b +Input::EVENT_NEW line output (stderr=F): c +Input::EVENT_NEW line output (stderr=T): +Input::EVENT_NEW line output (stderr=T): +Input::EVENT_NEW line output (stderr=T): +End of Data event, input +Process finished event, input, T diff --git a/testing/btest/scripts/base/frameworks/input/raw/stderr.bro b/testing/btest/scripts/base/frameworks/input/raw/stderr.bro index 8e3fcefe41..8ff4cc7f1b 100644 --- a/testing/btest/scripts/base/frameworks/input/raw/stderr.bro +++ b/testing/btest/scripts/base/frameworks/input/raw/stderr.bro @@ -1,3 +1,4 @@ +# @TEST-EXEC: mkdir mydir && touch mydir/a && touch mydir/b && touch mydir/c # @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: btest-bg-wait 10 # @TEST-EXEC: btest-diff out @@ -9,64 +10,59 @@ type Val: record { is_stderr: bool; }; -global try: count; +global try = 0; +global n = 0; global outfile: file; event line(description: Input::EventDescription, tpe: Input::Event, s: string, is_stderr: bool) { - print outfile, tpe; + local line_output = fmt("%s line output (stderr=%s): ", tpe, is_stderr); + if ( is_stderr ) { # work around localized error messages. and if some localization does not include the filename... well... that would be bad :) if ( strstr(s, "nonexistant") > 0 ) - { - print outfile, "stderr output contained nonexistant"; - } + line_output += ""; + else + line_output += ""; } else - { - print outfile, s; - } - print outfile, is_stderr; + line_output += s; - try = try + 1; - if ( try == 7 ) - { - print outfile, "done"; - Input::remove("input"); - } + print outfile, line_output; + ++try; + + if ( n == 2 && try == 7 ) + terminate(); } -global n = 0; - event Input::end_of_data(name: string, source:string) { - print outfile, "End of Data event"; - print outfile, name; + print outfile, "End of Data event", name; ++n; - if ( n == 2 ) + + if ( n == 2 && try == 7 ) terminate(); } event InputRaw::process_finished(name: string, source:string, exit_code:count, signal_exit:bool) { - print outfile, "Process finished event"; - print outfile, name; - if ( exit_code != 0 ) - print outfile, "Exit code != 0"; + print outfile, "Process finished event", name, exit_code != 0; ++n; - if ( n == 2 ) + + if ( n == 2 && try == 7 ) terminate(); } event bro_init() { - local config_strings: table[string] of string = { ["read_stderr"] = "1" }; outfile = open("../out"); - try = 0; - Input::add_event([$source="ls .. ../nonexistant ../nonexistant2 ../nonexistant3 |", $reader=Input::READER_RAW, $name="input", $fields=Val, $ev=line, $want_record=F, $config=config_strings, $mode=Input::STREAM]); + Input::add_event([$source="ls ../mydir ../nonexistant ../nonexistant2 ../nonexistant3 |", + $reader=Input::READER_RAW, $name="input", + $fields=Val, $ev=line, $want_record=F, + $config=config_strings, $mode=Input::STREAM]); } From d245513e0a3bc4924a7ec5f1bfb268d1af421b75 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Wed, 18 Jul 2018 09:51:13 -0500 Subject: [PATCH 24/24] Improve some netcontrol unit tests --- CHANGES | 4 ++++ VERSION | 2 +- .../base/frameworks/netcontrol/acld-hook.bro | 17 +++++++++++++++-- .../scripts/base/frameworks/netcontrol/acld.bro | 17 +++++++++++++++-- .../base/frameworks/netcontrol/broker.bro | 17 +++++++++++++++-- 5 files changed, 50 insertions(+), 7 deletions(-) diff --git a/CHANGES b/CHANGES index 0f05f4fb59..dd723f7fa7 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,8 @@ +2.5-747 | 2018-07-18 09:51:13 -0500 + + * Improve some netcontrol unit tests (Jon Siwek, Corelight) + 2.5-746 | 2018-07-17 17:51:13 -0500 * Improve an input framework unit test (Jon Siwek, Corelight) diff --git a/VERSION b/VERSION index fcd1315f19..c0d70f840f 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.5-746 +2.5-747 diff --git a/testing/btest/scripts/base/frameworks/netcontrol/acld-hook.bro b/testing/btest/scripts/base/frameworks/netcontrol/acld-hook.bro index 2928e3d9a0..a0ce9c44d6 100644 --- a/testing/btest/scripts/base/frameworks/netcontrol/acld-hook.bro +++ b/testing/btest/scripts/base/frameworks/netcontrol/acld-hook.bro @@ -11,22 +11,35 @@ @load base/frameworks/netcontrol redef exit_only_after_terminate = T; +global have_peer = F; +global did_init = F; + +event bro_init() + { + suspend_processing(); + } event NetControl::init() { - suspend_processing(); local netcontrol_acld = NetControl::create_acld(NetControl::AcldConfig($acld_host=127.0.0.1, $acld_port=Broker::default_port, $acld_topic="bro/event/netcontroltest")); NetControl::activate(netcontrol_acld, 0); } event NetControl::init_done() { - continue_processing(); + did_init = T; + + if ( did_init && have_peer ) + continue_processing(); } event Broker::peer_added(endpoint: Broker::EndpointInfo, msg: string) { print "Broker peer added", endpoint$network; + have_peer = T; + + if ( did_init && have_peer ) + continue_processing(); } event Broker::peer_lost(endpoint: Broker::EndpointInfo, msg: string) diff --git a/testing/btest/scripts/base/frameworks/netcontrol/acld.bro b/testing/btest/scripts/base/frameworks/netcontrol/acld.bro index b802536eec..7593790013 100644 --- a/testing/btest/scripts/base/frameworks/netcontrol/acld.bro +++ b/testing/btest/scripts/base/frameworks/netcontrol/acld.bro @@ -12,10 +12,16 @@ @load base/frameworks/netcontrol redef exit_only_after_terminate = T; +global have_peer = F; +global did_init = F; + +event bro_init() + { + suspend_processing(); + } event NetControl::init() { - suspend_processing(); local netcontrol_acld = NetControl::create_acld(NetControl::AcldConfig($acld_host=127.0.0.1, $acld_port=Broker::default_port, $acld_topic="bro/event/netcontroltest")); NetControl::activate(netcontrol_acld, 0); } @@ -23,11 +29,18 @@ event NetControl::init() event Broker::peer_added(endpoint: Broker::EndpointInfo, msg: string) { print "Broker peer added", endpoint$network; + have_peer = T; + + if ( did_init && have_peer ) + continue_processing(); } event NetControl::init_done() { - continue_processing(); + did_init = T; + + if ( did_init && have_peer ) + continue_processing(); } event Broker::peer_lost(endpoint: Broker::EndpointInfo, msg: string) diff --git a/testing/btest/scripts/base/frameworks/netcontrol/broker.bro b/testing/btest/scripts/base/frameworks/netcontrol/broker.bro index 75ad035f69..9e8bb65476 100644 --- a/testing/btest/scripts/base/frameworks/netcontrol/broker.bro +++ b/testing/btest/scripts/base/frameworks/netcontrol/broker.bro @@ -12,22 +12,35 @@ @load base/frameworks/netcontrol redef exit_only_after_terminate = T; +global have_peer = F; +global did_init = F; + +event bro_init() + { + suspend_processing(); + } event NetControl::init() { - suspend_processing(); local netcontrol_broker = NetControl::create_broker(NetControl::BrokerConfig($host=127.0.0.1, $bport=Broker::default_port, $topic="bro/event/netcontroltest"), T); NetControl::activate(netcontrol_broker, 0); } event NetControl::init_done() { - continue_processing(); + did_init = T; + + if ( did_init && have_peer ) + continue_processing(); } event Broker::peer_added(endpoint: Broker::EndpointInfo, msg: string) { print "Broker peer added", endpoint$network; + have_peer = T; + + if ( did_init && have_peer ) + continue_processing(); } event Broker::peer_lost(endpoint: Broker::EndpointInfo, msg: string)