Merge branch 'update' of https://github.com/leres/zeek

* 'update' of https://github.com/leres/zeek:
  Return false on error from the other place we call fstat()
  Address concerns raised by @0xxon; avoid the new code path when reading from a pipe and return false if fstat() fails after sucessfully opening the file (unlikely).
  Tweak some new comments
  Remove child program check, it's probably wrong given the test failures it causes
  Conform to style police
  Only set mtime and ino in Raw::OpenInput() do this for MODE_STREAM and avoid breaking MODE_REREAD
  Implement tail -F semantics for input framework MODE_STREAM Open /dev/null if the file is missing during init and wait for file to be created Collect initial ino, dev, and mtime when first opening the file Detect if the file has been replaced and open the new version
  Only set mtime and ino in Raw::OpenInput() do this for MODE_STREAM and avoid breaking MODE_REREAD
  Implement tail -F semantics for input framework MODE_STREAM Open /dev/null if the file is missing during init and wait for file to be created Collect initial ino, dev, and mtime when first opening the file Detect if the file has been replaced and open the new version

CHANGES

@@ -1,3 +1,13 @@
+5.1.0-dev.172 | 2022-07-04 10:22:45 +0100
+
+  * The STREAM mode of the ASCII reader now behaves like `tail -F`: when file is
+    removed/replaced, it will start tracking the new file. See
+    https://github.com/zeek/zeek/pull/2097 for more detail. (Craig Leres)
+
+  * Remove loops from Dict iterator invalidation unit test (Tim Wojtulewicz, Corelight)
+
+    This fixes Coverity finding 1490366
+
 5.1.0-dev.154 | 2022-07-01 14:10:58 -0700
 
   * Remove unused util::detail::rand64bit method (Tim Wojtulewicz, Corelight)

NEWS

@@ -14,6 +14,10 @@ Breaking Changes
   variable was added to cover the transport layer. See this GitHub issue for
   more detail: https://github.com/zeek/zeek/issues/2183.
 
+- The STREAM mode of the ASCII reader now behaves like `tail -F`: when file is 
+  removed/replaced, it will start tracking the new file. See
+  https://github.com/zeek/zeek/pull/2097 for more detail
+
 New Functionality
 -----------------
 

VERSION

@@ -1 +1 @@
-5.1.0-dev.154
+5.1.0-dev.172

doc

@@ -1 +1 @@
-Subproject commit eafa706d64faa67379872354083f32338d5029a2
+Subproject commit bdb22dd2ce4618be8ddfdbf650a63cd385e6509c

src/input/readers/raw/Raw.cc

@@ -36,6 +36,7 @@ Raw::Raw(ReaderFrontend* frontend)
 	firstrun = true;
 	mtime = 0;
 	ino = 0;
+	dev = 0;
 	forcekill = false;
 	offset = 0;
 	separator.assign((const char*)BifConst::InputRaw::record_separator->Bytes(),
@@ -280,12 +281,31 @@ bool Raw::OpenInput()
 	else
 		{
 		file = std::unique_ptr<FILE, int (*)(FILE*)>(fopen(fname.c_str(), "r"), fclose);
+		if ( ! file && Info().mode == MODE_STREAM )
+			{
+			// Watch /dev/null until the file appears
+			file = std::unique_ptr<FILE, int (*)(FILE*)>(fopen("/dev/null", "r"), fclose);
+			}
+
 		if ( ! file )
 			{
 			Error(Fmt("Init: cannot open %s", fname.c_str()));
 			return false;
 			}
 
+		if ( Info().mode == MODE_STREAM )
+			{
+			struct stat sb;
+			if ( fstat(fileno(file.get()), &sb) == -1 )
+				{
+				// This is unlikely to fail
+				Error(Fmt("Could not get fstat for %s", fname.c_str()));
+				return false;
+				}
+			ino = sb.st_ino;
+			dev = sb.st_dev;
+			}
+
 		if ( ! SetFDFlags(fileno(file.get()), F_SETFD, FD_CLOEXEC) )
 			Warning(Fmt("Init: cannot set close-on-exec for %s", fname.c_str()));
 		}
@@ -346,6 +366,7 @@ bool Raw::DoInit(const ReaderInfo& info, int num_fields, const Field* const* fie
 	fname = info.source;
 	mtime = 0;
 	ino = 0;
+	dev = 0;
 	execute = false;
 	firstrun = true;
 	int want_fields = 1;
@@ -574,25 +595,60 @@ bool Raw::DoUpdate()
 
 				mtime = sb.st_mtime;
 				ino = sb.st_ino;
+				dev = sb.st_dev;
 				// file changed. reread.
 				//
 				// fallthrough
 				}
 
 			case MODE_MANUAL:
-			case MODE_STREAM:
-				if ( Info().mode == MODE_STREAM && file )
-					{
-					clearerr(file.get()); // remove end of file evil bits
-					break;
-					}
-
 				CloseInput();
 				if ( ! OpenInput() )
 					return false;
 
 				break;
 
+			case MODE_STREAM:
+				// Clear possible EOF condition
+				if ( file )
+					clearerr(file.get());
+
+				// Done if reading from a pipe
+				if ( execute )
+					break;
+
+				// Check if the file has changed
+				struct stat sb;
+				if ( stat(fname.c_str(), &sb) == -1 )
+					// File was removed
+					break;
+
+				// Is it the same file?
+				if ( sb.st_ino == ino && sb.st_dev == dev )
+					break;
+
+				// File was replaced
+				FILE* tfile;
+				tfile = fopen(fname.c_str(), "r");
+				if ( ! tfile )
+					break;
+
+				// Stat newly opened file
+				if ( fstat(fileno(tfile), &sb) == -1 )
+					{
+					// This is unlikely to fail
+					Error(Fmt("Could not fstat %s", fname.c_str()));
+					fclose(tfile);
+					return false;
+					}
+				file.reset(nullptr);
+				file = std::unique_ptr<FILE, int (*)(FILE*)>(tfile, fclose);
+				ino = sb.st_ino;
+				dev = sb.st_dev;
+				offset = 0;
+				bufpos = 0;
+				break;
+
 			default:
 				assert(false);
 			}

src/input/readers/raw/Raw.h

@@ -55,6 +55,7 @@ private:
 	bool firstrun;
 	time_t mtime;
 	ino_t ino;
+	dev_t dev;
 
 	// options set from the script-level.
 	std::string separator;