diff --git a/CHANGES b/CHANGES index 7b4686b22f..2d6596ef99 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,13 @@ +5.1.0-dev.172 | 2022-07-04 10:22:45 +0100 + + * The STREAM mode of the ASCII reader now behaves like `tail -F`: when file is + removed/replaced, it will start tracking the new file. See + https://github.com/zeek/zeek/pull/2097 for more detail. (Craig Leres) + + * Remove loops from Dict iterator invalidation unit test (Tim Wojtulewicz, Corelight) + + This fixes Coverity finding 1490366 + 5.1.0-dev.154 | 2022-07-01 14:10:58 -0700 * Remove unused util::detail::rand64bit method (Tim Wojtulewicz, Corelight) diff --git a/NEWS b/NEWS index 2a4a5a219c..4c4118999a 100644 --- a/NEWS +++ b/NEWS @@ -14,6 +14,10 @@ Breaking Changes variable was added to cover the transport layer. See this GitHub issue for more detail: https://github.com/zeek/zeek/issues/2183. +- The STREAM mode of the ASCII reader now behaves like `tail -F`: when file is + removed/replaced, it will start tracking the new file. See + https://github.com/zeek/zeek/pull/2097 for more detail + New Functionality ----------------- diff --git a/VERSION b/VERSION index 9d122c7c38..a45254e7fd 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -5.1.0-dev.154 +5.1.0-dev.172 diff --git a/doc b/doc index eafa706d64..bdb22dd2ce 160000 --- a/doc +++ b/doc @@ -1 +1 @@ -Subproject commit eafa706d64faa67379872354083f32338d5029a2 +Subproject commit bdb22dd2ce4618be8ddfdbf650a63cd385e6509c diff --git a/src/input/readers/raw/Raw.cc b/src/input/readers/raw/Raw.cc index 8c7fdd0f31..41bb956cd8 100644 --- a/src/input/readers/raw/Raw.cc +++ b/src/input/readers/raw/Raw.cc @@ -36,6 +36,7 @@ Raw::Raw(ReaderFrontend* frontend) firstrun = true; mtime = 0; ino = 0; + dev = 0; forcekill = false; offset = 0; separator.assign((const char*)BifConst::InputRaw::record_separator->Bytes(), @@ -280,12 +281,31 @@ bool Raw::OpenInput() else { file = std::unique_ptr(fopen(fname.c_str(), "r"), fclose); + if ( ! file && Info().mode == MODE_STREAM ) + { + // Watch /dev/null until the file appears + file = std::unique_ptr(fopen("/dev/null", "r"), fclose); + } + if ( ! file ) { Error(Fmt("Init: cannot open %s", fname.c_str())); return false; } + if ( Info().mode == MODE_STREAM ) + { + struct stat sb; + if ( fstat(fileno(file.get()), &sb) == -1 ) + { + // This is unlikely to fail + Error(Fmt("Could not get fstat for %s", fname.c_str())); + return false; + } + ino = sb.st_ino; + dev = sb.st_dev; + } + if ( ! SetFDFlags(fileno(file.get()), F_SETFD, FD_CLOEXEC) ) Warning(Fmt("Init: cannot set close-on-exec for %s", fname.c_str())); } @@ -346,6 +366,7 @@ bool Raw::DoInit(const ReaderInfo& info, int num_fields, const Field* const* fie fname = info.source; mtime = 0; ino = 0; + dev = 0; execute = false; firstrun = true; int want_fields = 1; @@ -574,25 +595,60 @@ bool Raw::DoUpdate() mtime = sb.st_mtime; ino = sb.st_ino; + dev = sb.st_dev; // file changed. reread. // // fallthrough } case MODE_MANUAL: - case MODE_STREAM: - if ( Info().mode == MODE_STREAM && file ) - { - clearerr(file.get()); // remove end of file evil bits - break; - } - CloseInput(); if ( ! OpenInput() ) return false; break; + case MODE_STREAM: + // Clear possible EOF condition + if ( file ) + clearerr(file.get()); + + // Done if reading from a pipe + if ( execute ) + break; + + // Check if the file has changed + struct stat sb; + if ( stat(fname.c_str(), &sb) == -1 ) + // File was removed + break; + + // Is it the same file? + if ( sb.st_ino == ino && sb.st_dev == dev ) + break; + + // File was replaced + FILE* tfile; + tfile = fopen(fname.c_str(), "r"); + if ( ! tfile ) + break; + + // Stat newly opened file + if ( fstat(fileno(tfile), &sb) == -1 ) + { + // This is unlikely to fail + Error(Fmt("Could not fstat %s", fname.c_str())); + fclose(tfile); + return false; + } + file.reset(nullptr); + file = std::unique_ptr(tfile, fclose); + ino = sb.st_ino; + dev = sb.st_dev; + offset = 0; + bufpos = 0; + break; + default: assert(false); } diff --git a/src/input/readers/raw/Raw.h b/src/input/readers/raw/Raw.h index 5e36848e26..7ebb334301 100644 --- a/src/input/readers/raw/Raw.h +++ b/src/input/readers/raw/Raw.h @@ -55,6 +55,7 @@ private: bool firstrun; time_t mtime; ino_t ino; + dev_t dev; // options set from the script-level. std::string separator;