Merge remote-tracking branch 'origin/topic/awelzel/pcap-reading-configurable-buffer'

* origin/topic/awelzel/pcap-reading-configurable-buffer:
  iosource/pcap: Support configurable buffer size
  util/setvbuf: Respect buf argument

NEWS

@@ -121,6 +121,9 @@ Changed Functionality
 - Parameter lists for functions, events and hooks now use commas instead of
   semicolons in error messages or when printing such functions.
 
+- The IO buffer size used for PCAP file reading is now always 128kb. This
+  new default can be changed via ``Pcap::bufsize_offline_bytes``.
+
 Removed Functionality
 ---------------------
 

scripts/base/init-bare.zeek

@@ -5286,6 +5286,11 @@ export {
 	## interfaces.
 	const bufsize = 128 &redef;
 
+	## Number of bytes to use for buffering file read operations when reading
+	## from a PCAP file. Setting this to 0 uses operating system defaults
+	## as chosen by fopen().
+	const bufsize_offline_bytes = 128 * 1024 &redef;
+
 	## Default timeout for packet sources without file descriptors.
 	##
 	## For libpcap based packet sources that do not provide a usable

src/iosource/pcap/Source.cc

@@ -10,6 +10,8 @@
 #include <pcap-int.h>
 #endif
 
+#include <stdio.h>
+
 #include "zeek/Event.h"
 #include "zeek/iosource/BPF_Program.h"
 #include "zeek/iosource/Packet.h"
@@ -176,10 +178,42 @@ void PcapSource::OpenOffline()
 	{
 	char errbuf[PCAP_ERRBUF_SIZE];
 
-	pd = pcap_open_offline(props.path.c_str(), errbuf);
+	FILE* f = nullptr;
+	if ( props.path == "-" )
+		{
+		f = stdin;
+		}
+	else
+		{
+		if ( f = fopen(props.path.c_str(), "rb"); ! f )
+			{
+			Error(util::fmt("unable to open %s: %s", props.path.c_str(), strerror(errno)));
+			return;
+			}
+
+		// Setup file IO buffering with a bufsize_offline_bytes sized
+		// buffer if set, otherwise use what fopen() took as the default.
+		if ( BifConst::Pcap::bufsize_offline_bytes != 0 )
+			{
+			iobuf.resize(BifConst::Pcap::bufsize_offline_bytes);
+			if ( util::detail::setvbuf(f, iobuf.data(), _IOFBF, iobuf.size()) != 0 )
+				{
+				Error(util::fmt("unable to setvbuf %s: %s", props.path.c_str(), strerror(errno)));
+				fclose(f);
+				return;
+				}
+			}
+		}
+
+	// pcap_fopen_offline() takes ownership of f on success and
+	// pcap_close() elsewhere should close it, too.
+	pd = pcap_fopen_offline(f, errbuf);
 
 	if ( ! pd )
 		{
+		if ( f != stdin )
+			fclose(f);
+
 		Error(errbuf);
 		return;
 		}

src/iosource/pcap/Source.h

@@ -4,6 +4,7 @@
 
 #include <sys/types.h> // for u_char
 #include <unistd.h>
+#include <vector>
 
 extern "C"
 	{
@@ -44,6 +45,9 @@ private:
 
 	pcap_t* pd;
 	struct pcap_stat prev_pstat = {0};
+
+	// Buffer provided to setvbuf() when reading from a PCAP file.
+	std::vector<char> iobuf;
 	};
 
 	} // namespace zeek::iosource::pcap

src/iosource/pcap/pcap.bif

@@ -3,6 +3,7 @@ module Pcap;
 
 const snaplen: count;
 const bufsize: count;
+const bufsize_offline_bytes: count;
 const non_fd_timeout: interval;
 
 %%{

src/util.cc

@@ -1025,7 +1025,7 @@ void set_thread_name(const char* name, pthread_t tid)
 int setvbuf(FILE* stream, char* buf, int type, size_t size)
 	{
 #ifndef _MSC_VER
-	return ::setvbuf(stream, NULL, type, size);
+	return ::setvbuf(stream, buf, type, size);
 #else
 	// TODO: this turns off buffering altogether because Windows wants us to pass a valid
 	// buffer and length if we're going to pass one of the other modes. We need to

testing/btest/Baseline/core.pcap.input-error/output2

@@ -1,3 +1,3 @@
 ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
 fatal error: problem with interface NO_SUCH_INTERFACE 
-fatal error: problem with trace file NO_SUCH_TRACE (NO_SUCH_TRACE: No such file or directory)
+fatal error: problem with trace file NO_SUCH_TRACE (unable to open NO_SUCH_TRACE: No such file or directory)

testing/btest/Baseline/core.pcap.wrong-format/output

@@ -0,0 +1,2 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+fatal error: problem with trace file not-a.pcap (unknown file format)

testing/btest/Baseline/core.pcap.wrong-format/output2

@@ -0,0 +1,2 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+fatal error: problem with trace file - (unknown file format)

testing/btest/core/pcap/wrong-format.zeek

@@ -0,0 +1,10 @@
+# @TEST-REQUIRES: test "${ZEEK_USE_CPP}" != "1"
+# @TEST-EXEC-FAIL: zeek -b -r not-a.pcap >output 2>&1
+# @TEST-EXEC: btest-diff output
+# @TEST-EXEC-FAIL: cat not-a.pcap | zeek -b -r - >output2 2>&1
+# @TEST-EXEC: btest-diff output2
+
+@TEST-START-FILE ./not-a.pcap
+%PDF-1.5
+This isn't an actual pdf file, and neither a PCAP.
+@TEST-END-FILE