From e50a8848aece186185c64b33185ecf6d957f4f7e Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Fri, 7 Feb 2020 19:58:35 -0800 Subject: [PATCH] Improve an input framework test A race condition could cause unstable output: if the thread reading the file is fast, often you see both "pred" functions execute and then both "line" events execute with both entries already in the table, but if the thread reading the file is slow, you see pred, event, pred, event, with only one entry available in the first event. --- CHANGES | 4 + VERSION | 2 +- .../events.out | 52 ++++++ .../out | 155 ------------------ .../preds.out | 16 ++ .../servers.out | 11 ++ .../input/empty-values-hashing.zeek | 65 +++++--- 7 files changed, 123 insertions(+), 182 deletions(-) create mode 100644 testing/btest/Baseline/scripts.base.frameworks.input.empty-values-hashing/events.out delete mode 100644 testing/btest/Baseline/scripts.base.frameworks.input.empty-values-hashing/out create mode 100644 testing/btest/Baseline/scripts.base.frameworks.input.empty-values-hashing/preds.out create mode 100644 testing/btest/Baseline/scripts.base.frameworks.input.empty-values-hashing/servers.out diff --git a/CHANGES b/CHANGES index 110fd3a10d..71bbe2b28a 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,8 @@ +3.1.0-dev.599 | 2020-02-07 19:58:35 -0800 + + * Improve an input framework test (Jon Siwek, Corelight) + 3.1.0-dev.598 | 2020-02-07 15:06:56 -0800 * Improve stability of a &expire_func btest (Jon Siwek, Corelight) diff --git a/VERSION b/VERSION index dd6629aa6b..a26e3f5f32 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.1.0-dev.598 +3.1.0-dev.599 diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.empty-values-hashing/events.out b/testing/btest/Baseline/scripts.base.frameworks.input.empty-values-hashing/events.out new file mode 100644 index 0000000000..015b7140a0 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.empty-values-hashing/events.out @@ -0,0 +1,52 @@ +============EVENT============ +Description + source, ../input.log + reader, Input::READER_ASCII + mode, Input::REREAD + name, ssh + destination[left = 1], [s=, ss=TEST] + idx, A::Idx + val, A::Val + want_record, T +Type, Input::EVENT_NEW +Left, [i=1] +Right, [s=, ss=TEST] +============EVENT============ +Description + source, ../input.log + reader, Input::READER_ASCII + mode, Input::REREAD + name, ssh + destination[left = 2], [s=, ss=] + idx, A::Idx + val, A::Val + want_record, T +Type, Input::EVENT_NEW +Left, [i=2] +Right, [s=, ss=] +============EVENT============ +Description + source, ../input.log + reader, Input::READER_ASCII + mode, Input::REREAD + name, ssh + destination[left = 1], [s=TEST, ss=] + idx, A::Idx + val, A::Val + want_record, T +Type, Input::EVENT_CHANGED +Left, [i=1] +Right, [s=, ss=TEST] +============EVENT============ +Description + source, ../input.log + reader, Input::READER_ASCII + mode, Input::REREAD + name, ssh + destination[left = 2], [s=TEST, ss=TEST] + idx, A::Idx + val, A::Val + want_record, T +Type, Input::EVENT_CHANGED +Left, [i=2] +Right, [s=, ss=] diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.empty-values-hashing/out b/testing/btest/Baseline/scripts.base.frameworks.input.empty-values-hashing/out deleted file mode 100644 index 6348fc6a6a..0000000000 --- a/testing/btest/Baseline/scripts.base.frameworks.input.empty-values-hashing/out +++ /dev/null @@ -1,155 +0,0 @@ -============PREDICATE============ -Input::EVENT_NEW -[i=1] -[s=, ss=TEST] -============PREDICATE============ -Input::EVENT_NEW -[i=2] -[s=, ss=] -============EVENT============ -Description -[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh, destination={ -[2] = [s=, ss=], -[1] = [s=, ss=TEST] -}, idx=A::Idx, val=A::Val, want_record=T, ev=line -{ -print A::outfile, ============EVENT============; -print A::outfile, Description; -print A::outfile, A::description; -print A::outfile, Type; -print A::outfile, A::tpe; -print A::outfile, Left; -print A::outfile, A::left; -print A::outfile, Right; -print A::outfile, A::right; -}, pred=anonymous-function -{ -print A::outfile, ============PREDICATE============; -print A::outfile, A::typ; -print A::outfile, A::left; -print A::outfile, A::right; -return (T); -}, error_ev=, config={ - -}] -Type -Input::EVENT_NEW -Left -[i=1] -Right -[s=, ss=TEST] -============EVENT============ -Description -[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh, destination={ -[2] = [s=, ss=], -[1] = [s=, ss=TEST] -}, idx=A::Idx, val=A::Val, want_record=T, ev=line -{ -print A::outfile, ============EVENT============; -print A::outfile, Description; -print A::outfile, A::description; -print A::outfile, Type; -print A::outfile, A::tpe; -print A::outfile, Left; -print A::outfile, A::left; -print A::outfile, Right; -print A::outfile, A::right; -}, pred=anonymous-function -{ -print A::outfile, ============PREDICATE============; -print A::outfile, A::typ; -print A::outfile, A::left; -print A::outfile, A::right; -return (T); -}, error_ev=, config={ - -}] -Type -Input::EVENT_NEW -Left -[i=2] -Right -[s=, ss=] -==========SERVERS============ -{ -[2] = [s=, ss=], -[1] = [s=, ss=TEST] -} -============PREDICATE============ -Input::EVENT_CHANGED -[i=1] -[s=TEST, ss=] -============PREDICATE============ -Input::EVENT_CHANGED -[i=2] -[s=TEST, ss=TEST] -============EVENT============ -Description -[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh, destination={ -[2] = [s=TEST, ss=TEST], -[1] = [s=TEST, ss=] -}, idx=A::Idx, val=A::Val, want_record=T, ev=line -{ -print A::outfile, ============EVENT============; -print A::outfile, Description; -print A::outfile, A::description; -print A::outfile, Type; -print A::outfile, A::tpe; -print A::outfile, Left; -print A::outfile, A::left; -print A::outfile, Right; -print A::outfile, A::right; -}, pred=anonymous-function -{ -print A::outfile, ============PREDICATE============; -print A::outfile, A::typ; -print A::outfile, A::left; -print A::outfile, A::right; -return (T); -}, error_ev=, config={ - -}] -Type -Input::EVENT_CHANGED -Left -[i=1] -Right -[s=, ss=TEST] -============EVENT============ -Description -[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh, destination={ -[2] = [s=TEST, ss=TEST], -[1] = [s=TEST, ss=] -}, idx=A::Idx, val=A::Val, want_record=T, ev=line -{ -print A::outfile, ============EVENT============; -print A::outfile, Description; -print A::outfile, A::description; -print A::outfile, Type; -print A::outfile, A::tpe; -print A::outfile, Left; -print A::outfile, A::left; -print A::outfile, Right; -print A::outfile, A::right; -}, pred=anonymous-function -{ -print A::outfile, ============PREDICATE============; -print A::outfile, A::typ; -print A::outfile, A::left; -print A::outfile, A::right; -return (T); -}, error_ev=, config={ - -}] -Type -Input::EVENT_CHANGED -Left -[i=2] -Right -[s=, ss=] -==========SERVERS============ -{ -[2] = [s=TEST, ss=TEST], -[1] = [s=TEST, ss=] -} -done diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.empty-values-hashing/preds.out b/testing/btest/Baseline/scripts.base.frameworks.input.empty-values-hashing/preds.out new file mode 100644 index 0000000000..88014bc67a --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.empty-values-hashing/preds.out @@ -0,0 +1,16 @@ +============PREDICATE============ +Input::EVENT_NEW +[i=1] +[s=, ss=TEST] +============PREDICATE============ +Input::EVENT_NEW +[i=2] +[s=, ss=] +============PREDICATE============ +Input::EVENT_CHANGED +[i=1] +[s=TEST, ss=] +============PREDICATE============ +Input::EVENT_CHANGED +[i=2] +[s=TEST, ss=TEST] diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.empty-values-hashing/servers.out b/testing/btest/Baseline/scripts.base.frameworks.input.empty-values-hashing/servers.out new file mode 100644 index 0000000000..3e96c24fae --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.empty-values-hashing/servers.out @@ -0,0 +1,11 @@ +==========SERVERS============ +{ +[2] = [s=, ss=], +[1] = [s=, ss=TEST] +} +==========SERVERS============ +{ +[2] = [s=TEST, ss=TEST], +[1] = [s=TEST, ss=] +} +done diff --git a/testing/btest/scripts/base/frameworks/input/empty-values-hashing.zeek b/testing/btest/scripts/base/frameworks/input/empty-values-hashing.zeek index 06dcc57fef..c6e945dbc3 100644 --- a/testing/btest/scripts/base/frameworks/input/empty-values-hashing.zeek +++ b/testing/btest/scripts/base/frameworks/input/empty-values-hashing.zeek @@ -3,7 +3,9 @@ # @TEST-EXEC: $SCRIPTS/wait-for-file zeek/got1 15|| (btest-bg-wait -k 1 && false) # @TEST-EXEC: mv input2.log input.log # @TEST-EXEC: btest-bg-wait 30 -# @TEST-EXEC: btest-diff out +# @TEST-EXEC: btest-diff servers.out +# @TEST-EXEC: btest-diff events.out +# @TEST-EXEC: btest-diff preds.out @TEST-START-FILE input1.log #separator \x09 @@ -33,54 +35,65 @@ type Val: record { ss: string; }; -global servers: table[int] of Val = table(); +type servers_type: table[int] of Val; +global servers: servers_type = table(); -global outfile: file; +global servers_file = open("../servers.out"); +global events_file = open("../events.out"); +global predicates_file = open("../preds.out"); global try: count; event line(description: Input::TableDescription, tpe: Input::Event, left: Idx, right: Val) { - print outfile, "============EVENT============"; - print outfile, "Description"; - print outfile, description; - print outfile, "Type"; - print outfile, tpe; - print outfile, "Left"; - print outfile, left; - print outfile, "Right"; - print outfile, right; + print events_file, "============EVENT============"; + print events_file, "Description"; + print events_file, " source", description$source; + print events_file, " reader", description$reader; + print events_file, " mode", description$mode; + print events_file, " name", description$name; + print events_file, fmt(" destination[left = %s]", left$i), + (description$destination as servers_type)[left$i]; + print events_file, " idx", description$idx; + print events_file, " val", description$val; + print events_file, " want_record", description$want_record; + print events_file, "Type", tpe; + print events_file, "Left", left; + print events_file, "Right", right; } event zeek_init() { - outfile = open("../out"); try = 0; # first read in the old stuff into the table... - Input::add_table([$source="../input.log", $mode=Input::REREAD, $name="ssh", $idx=Idx, $val=Val, $destination=servers, $ev=line, - $pred(typ: Input::Event, left: Idx, right: Val) = { - print outfile, "============PREDICATE============"; - print outfile, typ; - print outfile, left; - print outfile, right; - return T; - } + Input::add_table([$source="../input.log", $mode=Input::REREAD, $name="ssh", + $idx=Idx, $val=Val, $destination=servers, $ev=line, + $pred(typ: Input::Event, left: Idx, right: Val) = { + print predicates_file, "============PREDICATE============"; + print predicates_file, typ; + print predicates_file, left; + print predicates_file, right; + return T; + } ]); } event Input::end_of_data(name: string, source: string) { - print outfile, "==========SERVERS============"; - print outfile, servers; - + print servers_file, "==========SERVERS============"; + print servers_file, servers; + try = try + 1; + if ( try == 1 ) system("touch got1"); else if ( try == 2 ) { - print outfile, "done"; - close(outfile); + print servers_file, "done"; + close(servers_file); + close(events_file); + close(predicates_file); Input::remove("input"); terminate(); }