Mirror/zeek
1
0
Fork 0
mirror of https://github.com/zeek/zeek.git synced 2025-10-02 14:48:21 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

Improve input framework re-read logic

Browse source 
Changed from checking for "has newer modification time" to "has
different modification time or inode number".
This commit is contained in:
Jon Siwek 2018-08-24 12:46:31 -05:00
parent 5c9813eadb
commit f41f392743
12 changed files with 38 additions and 19 deletions
Download patch file Download diff file Expand all files Collapse all files

5
src/input/readers/ascii/Ascii.cc
View file

@ -49,6 +49,7 @@ FieldMapping FieldMapping::subType()
Ascii::Ascii(ReaderFrontend *frontend) : ReaderBackend(frontend) Ascii::Ascii(ReaderFrontend *frontend) : ReaderBackend(frontend)
{ {
mtime = 0; mtime = 0;
ino = 0;
suppress_warnings = false; suppress_warnings = false;
fail_on_file_problem = false; fail_on_file_problem = false;
fail_on_invalid_lines = false; fail_on_invalid_lines = false;
@ -281,10 +282,12 @@ bool Ascii::DoUpdate()
return ! fail_on_file_problem; return ! fail_on_file_problem;
} }
if ( sb.st_mtime <= mtime ) // no change if ( sb.st_ino == ino && sb.st_mtime == mtime )
// no change
return true; return true;
mtime = sb.st_mtime; mtime = sb.st_mtime;
ino = sb.st_ino;
// file changed. reread. // file changed. reread.
// fallthrough // fallthrough

2
src/input/readers/ascii/Ascii.h
View file

@ -7,6 +7,7 @@
#include <vector> #include <vector>
#include <fstream> #include <fstream>
#include <memory> #include <memory>
#include <sys/types.h>
#include "input/ReaderBackend.h" #include "input/ReaderBackend.h"
#include "threading/formatters/Ascii.h" #include "threading/formatters/Ascii.h"
@ -63,6 +64,7 @@ private:
ifstream file; ifstream file;
time_t mtime; time_t mtime;
ino_t ino;
// map columns in the file to columns to send back to the manager // map columns in the file to columns to send back to the manager
vector<FieldMapping> columnMap; vector<FieldMapping> columnMap;

6
src/input/readers/binary/Binary.cc
View file

@ -14,7 +14,7 @@ using threading::Field;
streamsize Binary::chunk_size = 0; streamsize Binary::chunk_size = 0;
Binary::Binary(ReaderFrontend *frontend) Binary::Binary(ReaderFrontend *frontend)
: ReaderBackend(frontend), in(0), mtime(0), firstrun(true) : ReaderBackend(frontend), in(0), mtime(0), ino(0), firstrun(true)
{ {
if ( ! chunk_size ) if ( ! chunk_size )
{ {
@ -78,6 +78,7 @@ bool Binary::DoInit(const ReaderInfo& info, int num_fields,
{ {
in = 0; in = 0;
mtime = 0; mtime = 0;
ino = 0;
firstrun = true; firstrun = true;
if ( ! info.source || strlen(info.source) == 0 ) if ( ! info.source || strlen(info.source) == 0 )
@ -160,11 +161,12 @@ int Binary::UpdateModificationTime()
return -1; return -1;
} }
if ( sb.st_mtime <= mtime ) if ( sb.st_ino == ino && sb.st_mtime == mtime )
// no change // no change
return 0; return 0;
mtime = sb.st_mtime; mtime = sb.st_mtime;
ino = sb.st_ino;
return 1; return 1;
} }

2
src/input/readers/binary/Binary.h
View file

@ -4,6 +4,7 @@
#define INPUT_READERS_BINARY_H #define INPUT_READERS_BINARY_H
#include <fstream> #include <fstream>
#include <sys/types.h>
#include "input/ReaderBackend.h" #include "input/ReaderBackend.h"
@ -36,6 +37,7 @@ private:
string fname; string fname;
ifstream* in; ifstream* in;
time_t mtime; time_t mtime;
ino_t ino;
bool firstrun; bool firstrun;
// options set from the script-level. // options set from the script-level.

5
src/input/readers/config/Config.cc
View file

@ -23,6 +23,7 @@ using threading::Field;
Config::Config(ReaderFrontend *frontend) : ReaderBackend(frontend) Config::Config(ReaderFrontend *frontend) : ReaderBackend(frontend)
{ {
mtime = 0; mtime = 0;
ino = 0;
suppress_warnings = false; suppress_warnings = false;
fail_on_file_problem = false; fail_on_file_problem = false;
@ -146,10 +147,12 @@ bool Config::DoUpdate()
return ! fail_on_file_problem; return ! fail_on_file_problem;
} }
if ( sb.st_mtime <= mtime ) // no change if ( sb.st_ino == ino && sb.st_mtime == mtime )
// no change
return true; return true;
mtime = sb.st_mtime; mtime = sb.st_mtime;
ino = sb.st_ino;
// file changed. reread. // file changed. reread.
// fallthrough // fallthrough

2
src/input/readers/config/Config.h
View file

@ -8,6 +8,7 @@
#include <fstream> #include <fstream>
#include <memory> #include <memory>
#include <unordered_map> #include <unordered_map>
#include <sys/types.h>
#include "input/ReaderBackend.h" #include "input/ReaderBackend.h"
#include "threading/formatters/Ascii.h" #include "threading/formatters/Ascii.h"
@ -46,6 +47,7 @@ private:
ifstream file; ifstream file;
time_t mtime; time_t mtime;
ino_t ino;
bool fail_on_file_problem; bool fail_on_file_problem;
// this is an internal indicator in case the read is currently in a failed state // this is an internal indicator in case the read is currently in a failed state

5
src/input/readers/raw/Raw.cc
View file

@ -31,6 +31,7 @@ Raw::Raw(ReaderFrontend *frontend) : ReaderBackend(frontend), file(nullptr, fclo
execute = false; execute = false;
firstrun = true; firstrun = true;
mtime = 0; mtime = 0;
ino = 0;
forcekill = false; forcekill = false;
offset = 0; offset = 0;
separator.assign( (const char*) BifConst::InputRaw::record_separator->Bytes(), separator.assign( (const char*) BifConst::InputRaw::record_separator->Bytes(),
@ -341,6 +342,7 @@ bool Raw::DoInit(const ReaderInfo& info, int num_fields, const Field* const* fie
fname = info.source; fname = info.source;
mtime = 0; mtime = 0;
ino = 0;
execute = false; execute = false;
firstrun = true; firstrun = true;
int want_fields = 1; int want_fields = 1;
@ -553,11 +555,12 @@ bool Raw::DoUpdate()
return false; return false;
} }
if ( sb.st_mtime <= mtime ) if ( sb.st_ino == ino && sb.st_mtime == mtime )
// no change // no change
return true; return true;
mtime = sb.st_mtime; mtime = sb.st_mtime;
ino = sb.st_ino;
// file changed. reread. // file changed. reread.
// //
// fallthrough // fallthrough

2
src/input/readers/raw/Raw.h
View file

@ -6,6 +6,7 @@
#include <vector> #include <vector>
#include <memory> #include <memory>
#include <mutex> #include <mutex>
#include <sys/types.h>
#include "input/ReaderBackend.h" #include "input/ReaderBackend.h"
@ -51,6 +52,7 @@ private:
bool execute; bool execute;
bool firstrun; bool firstrun;
time_t mtime; time_t mtime;
ino_t ino;
// options set from the script-level. // options set from the script-level.
string separator; string separator;

4
testing/btest/scripts/base/frameworks/input/empty-values-hashing.bro
View file

@ -1,7 +1,7 @@
# @TEST-EXEC: cp input1.log input.log # @TEST-EXEC: mv input1.log input.log
# @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: btest-bg-run bro bro -b %INPUT
# @TEST-EXEC: $SCRIPTS/wait-for-file bro/got1 5 || (btest-bg-wait -k 1 && false) # @TEST-EXEC: $SCRIPTS/wait-for-file bro/got1 5 || (btest-bg-wait -k 1 && false)
# @TEST-EXEC: cp input2.log input.log # @TEST-EXEC: mv input2.log input.log
# @TEST-EXEC: btest-bg-wait 10 # @TEST-EXEC: btest-bg-wait 10
# @TEST-EXEC: btest-diff out # @TEST-EXEC: btest-diff out

10
testing/btest/scripts/base/frameworks/input/predicatemodifyandreread.bro
View file

@ -1,13 +1,13 @@
# @TEST-EXEC: cp input1.log input.log # @TEST-EXEC: mv input1.log input.log
# @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: btest-bg-run bro bro -b %INPUT
# @TEST-EXEC: $SCRIPTS/wait-for-file bro/got1 5 || (btest-bg-wait -k 1 && false) # @TEST-EXEC: $SCRIPTS/wait-for-file bro/got1 5 || (btest-bg-wait -k 1 && false)
# @TEST-EXEC: cp input2.log input.log # @TEST-EXEC: mv input2.log input.log
# @TEST-EXEC: $SCRIPTS/wait-for-file bro/got2 5 || (btest-bg-wait -k 1 && false) # @TEST-EXEC: $SCRIPTS/wait-for-file bro/got2 5 || (btest-bg-wait -k 1 && false)
# @TEST-EXEC: cp input3.log input.log # @TEST-EXEC: mv input3.log input.log
# @TEST-EXEC: $SCRIPTS/wait-for-file bro/got3 5 || (btest-bg-wait -k 1 && false) # @TEST-EXEC: $SCRIPTS/wait-for-file bro/got3 5 || (btest-bg-wait -k 1 && false)
# @TEST-EXEC: cp input4.log input.log # @TEST-EXEC: mv input4.log input.log
# @TEST-EXEC: $SCRIPTS/wait-for-file bro/got4 5 || (btest-bg-wait -k 1 && false) # @TEST-EXEC: $SCRIPTS/wait-for-file bro/got4 5 || (btest-bg-wait -k 1 && false)
# @TEST-EXEC: cp input5.log input.log # @TEST-EXEC: mv input5.log input.log
# @TEST-EXEC: btest-bg-wait 10 # @TEST-EXEC: btest-bg-wait 10
# @TEST-EXEC: btest-diff out # @TEST-EXEC: btest-diff out
# #

10
testing/btest/scripts/base/frameworks/input/reread.bro
View file

@ -1,13 +1,13 @@
# @TEST-EXEC: cp input1.log input.log # @TEST-EXEC: mv input1.log input.log
# @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: btest-bg-run bro bro -b %INPUT
# @TEST-EXEC: $SCRIPTS/wait-for-file bro/got1 5 || (btest-bg-wait -k 1 && false) # @TEST-EXEC: $SCRIPTS/wait-for-file bro/got1 5 || (btest-bg-wait -k 1 && false)
# @TEST-EXEC: cp input2.log input.log # @TEST-EXEC: mv input2.log input.log
# @TEST-EXEC: $SCRIPTS/wait-for-file bro/got2 5 || (btest-bg-wait -k 1 && false) # @TEST-EXEC: $SCRIPTS/wait-for-file bro/got2 5 || (btest-bg-wait -k 1 && false)
# @TEST-EXEC: cp input3.log input.log # @TEST-EXEC: mv input3.log input.log
# @TEST-EXEC: $SCRIPTS/wait-for-file bro/got3 5 || (btest-bg-wait -k 1 && false) # @TEST-EXEC: $SCRIPTS/wait-for-file bro/got3 5 || (btest-bg-wait -k 1 && false)
# @TEST-EXEC: cp input4.log input.log # @TEST-EXEC: mv input4.log input.log
# @TEST-EXEC: $SCRIPTS/wait-for-file bro/got4 5 || (btest-bg-wait -k 1 && false) # @TEST-EXEC: $SCRIPTS/wait-for-file bro/got4 5 || (btest-bg-wait -k 1 && false)
# @TEST-EXEC: cp input5.log input.log # @TEST-EXEC: mv input5.log input.log
# @TEST-EXEC: btest-bg-wait 10 # @TEST-EXEC: btest-bg-wait 10
# @TEST-EXEC: btest-diff out # @TEST-EXEC: btest-diff out

4
testing/btest/scripts/base/frameworks/input/twotables.bro
View file

@ -1,7 +1,7 @@
# @TEST-EXEC: cp input1.log input.log # @TEST-EXEC: mv input1.log input.log
# @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: btest-bg-run bro bro -b %INPUT
# @TEST-EXEC: $SCRIPTS/wait-for-file bro/got2 5 || (btest-bg-wait -k 1 && false) # @TEST-EXEC: $SCRIPTS/wait-for-file bro/got2 5 || (btest-bg-wait -k 1 && false)
# @TEST-EXEC: cp input3.log input.log # @TEST-EXEC: mv input3.log input.log
# @TEST-EXEC: btest-bg-wait 10 # @TEST-EXEC: btest-bg-wait 10
# @TEST-EXEC: btest-diff event.out # @TEST-EXEC: btest-diff event.out
# @TEST-EXEC: btest-diff pred1.out # @TEST-EXEC: btest-diff pred1.out