From 44ba9138c21fd1ee3e6de82e79984f85457b6bfb Mon Sep 17 00:00:00 2001 From: Craig Leres Date: Tue, 12 Jul 2022 17:57:50 -0700 Subject: [PATCH] Fix tail -F semantics when want_record=F and add tests for the new features While writing a test for the new "tail -F semantics" I found that the $want_record=F case was broken (errno 25). So instead of opening /dev/null when the input file is missing change READER_RAW to avoid I/O until it can be opened. Add two tests, one for when the event handler is called with a record and one for when it's called with a string. --- src/input/readers/raw/Raw.cc | 20 ++--- .../out | 10 +++ .../out | 10 +++ .../base/frameworks/input/raw/streamtail.zeek | 74 +++++++++++++++++++ .../frameworks/input/raw/streamtail2.zeek | 74 +++++++++++++++++++ 5 files changed, 179 insertions(+), 9 deletions(-) create mode 100644 testing/btest/Baseline/scripts.base.frameworks.input.raw.streamtail/out create mode 100644 testing/btest/Baseline/scripts.base.frameworks.input.raw.streamtail2/out create mode 100644 testing/btest/scripts/base/frameworks/input/raw/streamtail.zeek create mode 100644 testing/btest/scripts/base/frameworks/input/raw/streamtail2.zeek diff --git a/src/input/readers/raw/Raw.cc b/src/input/readers/raw/Raw.cc index 41bb956cd8..d5932ae1fa 100644 --- a/src/input/readers/raw/Raw.cc +++ b/src/input/readers/raw/Raw.cc @@ -281,14 +281,12 @@ bool Raw::OpenInput() else { file = std::unique_ptr(fopen(fname.c_str(), "r"), fclose); - if ( ! file && Info().mode == MODE_STREAM ) - { - // Watch /dev/null until the file appears - file = std::unique_ptr(fopen("/dev/null", "r"), fclose); - } - if ( ! file ) { + if ( Info().mode == MODE_STREAM ) + // Wait for file to appear + return true; + Error(Fmt("Init: cannot open %s", fname.c_str())); return false; } @@ -624,7 +622,7 @@ bool Raw::DoUpdate() break; // Is it the same file? - if ( sb.st_ino == ino && sb.st_dev == dev ) + if ( file && sb.st_ino == ino && sb.st_dev == dev ) break; // File was replaced @@ -638,10 +636,10 @@ bool Raw::DoUpdate() { // This is unlikely to fail Error(Fmt("Could not fstat %s", fname.c_str())); - fclose(tfile); return false; } - file.reset(nullptr); + if ( file ) + file.reset(nullptr); file = std::unique_ptr(tfile, fclose); ino = sb.st_ino; dev = sb.st_dev; @@ -661,6 +659,10 @@ bool Raw::DoUpdate() if ( stdin_towrite > 0 ) WriteToStdin(); + if ( ! file && Info().mode == MODE_STREAM ) + // Wait for file to appear + break; + int64_t length = GetLine(file.get()); // printf("Read %lld bytes\n", length); diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.raw.streamtail/out b/testing/btest/Baseline/scripts.base.frameworks.input.raw.streamtail/out new file mode 100644 index 0000000000..7c22e51a01 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.raw.streamtail/out @@ -0,0 +1,10 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +../input.log, Input::READER_RAW, Input::STREAM, input +Input::EVENT_NEW +sdfkh:KH;fdkncv;ISEUp34:Fkdj;YVpIODhfDF +../input.log, Input::READER_RAW, Input::STREAM, input +Input::EVENT_NEW +DSF"DFKJ"SDFKLh304yrsdkfj@#(*U$34jfDJup3UF +../input.log, Input::READER_RAW, Input::STREAM, input +Input::EVENT_NEW +3rw43wRRERLlL#RWERERERE. diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.raw.streamtail2/out b/testing/btest/Baseline/scripts.base.frameworks.input.raw.streamtail2/out new file mode 100644 index 0000000000..7c22e51a01 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.raw.streamtail2/out @@ -0,0 +1,10 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +../input.log, Input::READER_RAW, Input::STREAM, input +Input::EVENT_NEW +sdfkh:KH;fdkncv;ISEUp34:Fkdj;YVpIODhfDF +../input.log, Input::READER_RAW, Input::STREAM, input +Input::EVENT_NEW +DSF"DFKJ"SDFKLh304yrsdkfj@#(*U$34jfDJup3UF +../input.log, Input::READER_RAW, Input::STREAM, input +Input::EVENT_NEW +3rw43wRRERLlL#RWERERERE. diff --git a/testing/btest/scripts/base/frameworks/input/raw/streamtail.zeek b/testing/btest/scripts/base/frameworks/input/raw/streamtail.zeek new file mode 100644 index 0000000000..e1ec04ec80 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/input/raw/streamtail.zeek @@ -0,0 +1,74 @@ +# Test "tail -F" functionality (record version) + +# Start without the file +# @TEST-EXEC: rm -f input.log +# @TEST-EXEC: btest-bg-run zeek zeek -b %INPUT +# @TEST-EXEC: sleep 1 + +# Create the file +# @TEST-EXEC: cp input1.log input.log +# @TEST-EXEC: $SCRIPTS/wait-for-file zeek/got1 5 || (btest-bg-wait -k 1 && false) + +# Append to the file +# @TEST-EXEC: cat input2.log >> input.log +# @TEST-EXEC: $SCRIPTS/wait-for-file zeek/got2 5 || (btest-bg-wait -k 1 && false) + +# Move onto the file +# @TEST-EXEC: cp input3.log _input.log +# @TEST-EXEC: mv _input.log input.log + +# Done! +# @TEST-EXEC: btest-bg-wait 60 +# @TEST-EXEC: btest-diff out + +redef exit_only_after_terminate = T; + +@TEST-START-FILE input1.log +sdfkh:KH;fdkncv;ISEUp34:Fkdj;YVpIODhfDF +@TEST-END-FILE + +@TEST-START-FILE input2.log +DSF"DFKJ"SDFKLh304yrsdkfj@#(*U$34jfDJup3UF +@TEST-END-FILE + +@TEST-START-FILE input3.log +3rw43wRRERLlL#RWERERERE. +@TEST-END-FILE + +module A; + +type lineVal: record { + s: string; +}; + +global try: count; +global outfile: file; + +event line(description: Input::EventDescription, tpe: Input::Event, s: lineVal) + { + print outfile, description$source, description$reader, description$mode, description$name; + print outfile, tpe; + print outfile, s$s; + + try = try + 1; + + if ( try == 1 ) + system("touch got1"); + else if ( try == 2 ) + system("touch got2"); + else if ( try == 3 ) + { + close(outfile); + Input::remove("input"); + terminate(); + } + } + +event zeek_init() + { + outfile = open("../out"); + try = 0; + Input::add_event([$source="../input.log", + $reader=Input::READER_RAW, $mode=Input::STREAM, $name="input", + $fields=lineVal, $ev=line]); + } diff --git a/testing/btest/scripts/base/frameworks/input/raw/streamtail2.zeek b/testing/btest/scripts/base/frameworks/input/raw/streamtail2.zeek new file mode 100644 index 0000000000..fcc7cc826c --- /dev/null +++ b/testing/btest/scripts/base/frameworks/input/raw/streamtail2.zeek @@ -0,0 +1,74 @@ +# Test "tail -F" functionality (string version) + +# Start without the file +# @TEST-EXEC: rm -f input.log +# @TEST-EXEC: btest-bg-run zeek zeek -b %INPUT +# @TEST-EXEC: sleep 1 + +# Create the file +# @TEST-EXEC: cp input1.log input.log +# @TEST-EXEC: $SCRIPTS/wait-for-file zeek/got1 5 || (btest-bg-wait -k 1 && false) + +# Append to the file +# @TEST-EXEC: cat input2.log >> input.log +# @TEST-EXEC: $SCRIPTS/wait-for-file zeek/got2 5 || (btest-bg-wait -k 1 && false) + +# Move onto the file +# @TEST-EXEC: cp input3.log _input.log +# @TEST-EXEC: mv _input.log input.log + +# Done! +# @TEST-EXEC: btest-bg-wait 60 +# @TEST-EXEC: btest-diff out + +redef exit_only_after_terminate = T; + +@TEST-START-FILE input1.log +sdfkh:KH;fdkncv;ISEUp34:Fkdj;YVpIODhfDF +@TEST-END-FILE + +@TEST-START-FILE input2.log +DSF"DFKJ"SDFKLh304yrsdkfj@#(*U$34jfDJup3UF +@TEST-END-FILE + +@TEST-START-FILE input3.log +3rw43wRRERLlL#RWERERERE. +@TEST-END-FILE + +module A; + +type lineVal: record { + s: string; +}; + +global try: count; +global outfile: file; + +event line(description: Input::EventDescription, tpe: Input::Event, s: string) + { + print outfile, description$source, description$reader, description$mode, description$name; + print outfile, tpe; + print outfile, s; + + try = try + 1; + + if ( try == 1 ) + system("touch got1"); + else if ( try == 2 ) + system("touch got2"); + else if ( try == 3 ) + { + close(outfile); + Input::remove("input"); + terminate(); + } + } + +event zeek_init() + { + outfile = open("../out"); + try = 0; + Input::add_event([$source="../input.log", + $reader=Input::READER_RAW, $mode=Input::STREAM, $name="input", + $fields=lineVal, $want_record=F, $ev=line]); + }