Merge branch 'master' into topic/jsiwek/bit-1235

src/CMakeLists.txt

@@ -157,6 +157,7 @@ add_subdirectory(analyzer)
 add_subdirectory(broxygen)
 add_subdirectory(file_analysis)
 add_subdirectory(input)
+add_subdirectory(iosource)
 add_subdirectory(logging)
 add_subdirectory(probabilistic)
 
@@ -254,7 +255,6 @@ set(bro_SRCS
     Anon.cc
     Attr.cc
     Base64.cc
-    BPF_Program.cc
     Brofiler.cc
     BroString.cc
     CCL.cc
@@ -279,14 +279,13 @@ set(bro_SRCS
     EventRegistry.cc
     Expr.cc
     File.cc
-    FlowSrc.cc
+    Flare.cc
     Frag.cc
     Frame.cc
     Func.cc
     Hash.cc
     ID.cc
     IntSet.cc
-    IOSource.cc
     IP.cc
     IPAddr.cc
     List.cc
@@ -299,7 +298,7 @@ set(bro_SRCS
     OSFinger.cc
     PacketFilter.cc
     PersistenceSerializer.cc
-    PktSrc.cc
+    Pipe.cc
     PolicyFile.cc
     PrefixTable.cc
     PriorityQueue.cc

src/ChunkedIO.cc

@@ -210,6 +210,7 @@ bool ChunkedIOFd::WriteChunk(Chunk* chunk, bool partial)
 	else
 		pending_head = pending_tail = q;
 
+	write_flare.Fire();
 	return Flush();
 	}
 
@@ -232,6 +233,7 @@ bool ChunkedIOFd::PutIntoWriteBuffer(Chunk* chunk)
 	write_len += len;
 
 	delete chunk;
+	write_flare.Fire();
 
 	if ( network_time - last_flush > 0.005 )
 		FlushWriteBuffer();
@@ -269,6 +271,10 @@ bool ChunkedIOFd::FlushWriteBuffer()
 		if ( unsigned(written) == len )
 			{
 			write_pos = write_len = 0;
+
+			if ( ! pending_head )
+				write_flare.Extinguish();
+
 			return true;
 			}
 
@@ -318,7 +324,12 @@ bool ChunkedIOFd::Flush()
 			}
 		}
 
-	return FlushWriteBuffer();
+	bool rval = FlushWriteBuffer();
+
+	if ( ! pending_head && write_len == 0 )
+		write_flare.Extinguish();
+
+	return rval;
 	}
 
 uint32 ChunkedIOFd::ChunkAvailable()
@@ -394,6 +405,9 @@ bool ChunkedIOFd::Read(Chunk** chunk, bool may_block)
 #ifdef DEBUG_COMMUNICATION
 		AddToBuffer("<false:read-chunk>", true);
 #endif
+		if ( ! ChunkAvailable() )
+			read_flare.Extinguish();
+
 		return false;
 		}
 
@@ -402,9 +416,15 @@ bool ChunkedIOFd::Read(Chunk** chunk, bool may_block)
 #ifdef DEBUG_COMMUNICATION
 		AddToBuffer("<null:no-data>", true);
 #endif
+		read_flare.Extinguish();
 		return true;
 		}
 
+	if ( ChunkAvailable() )
+		read_flare.Fire();
+	else
+		read_flare.Extinguish();
+
 #ifdef DEBUG
 	if ( *chunk )
 		DBG_LOG(DBG_CHUNKEDIO, "read of size %d %s[%s]",
@@ -481,6 +501,9 @@ bool ChunkedIOFd::ReadChunk(Chunk** chunk, bool may_block)
 	read_pos = 0;
 	read_len = bytes_left;
 
+	if ( ! ChunkAvailable() )
+		read_flare.Extinguish();
+
 	// If allowed, wait a bit for something to read.
 	if ( may_block )
 		{
@@ -607,6 +630,14 @@ bool ChunkedIOFd::IsFillingUp()
 	return stats.pending > MAX_BUFFERED_CHUNKS_SOFT;
 	}
 
+iosource::FD_Set ChunkedIOFd::ExtraReadFDs() const
+	{
+	iosource::FD_Set rval;
+	rval.Insert(write_flare.FD());
+	rval.Insert(read_flare.FD());
+	return rval;
+	}
+
 void ChunkedIOFd::Clear()
 	{
 	while ( pending_head )
@@ -618,6 +649,9 @@ void ChunkedIOFd::Clear()
 		}
 
 	pending_head = pending_tail = 0;
+
+	if ( write_len == 0 )
+		write_flare.Extinguish();
 	}
 
 const char* ChunkedIOFd::Error()
@@ -830,6 +864,7 @@ bool ChunkedIOSSL::Write(Chunk* chunk)
 	else
 		write_head = write_tail = q;
 
+	write_flare.Fire();
 	Flush();
 	return true;
 	}
@@ -935,6 +970,7 @@ bool ChunkedIOSSL::Flush()
 		write_state = LEN;
 		}
 
+	write_flare.Extinguish();
 	return true;
 	}
 
@@ -1104,6 +1140,13 @@ bool ChunkedIOSSL::IsFillingUp()
 	return false;
 	}
 
+iosource::FD_Set ChunkedIOSSL::ExtraReadFDs() const
+	{
+	iosource::FD_Set rval;
+	rval.Insert(write_flare.FD());
+	return rval;
+	}
+
 void ChunkedIOSSL::Clear()
 	{
 	while ( write_head )
@@ -1114,6 +1157,7 @@ void ChunkedIOSSL::Clear()
 		write_head = next;
 		}
 	write_head = write_tail = 0;
+	write_flare.Extinguish();
 	}
 
 const char* ChunkedIOSSL::Error()

src/ChunkedIO.h

@@ -6,7 +6,8 @@
 #include "config.h"
 #include "List.h"
 #include "util.h"
-
+#include "Flare.h"
+#include "iosource/FD_Set.h"
 #include <list>
 
 #ifdef NEED_KRB5_H
@@ -95,6 +96,11 @@ public:
 	// Returns underlying fd if available, -1 otherwise.
 	virtual int Fd()	{ return -1; }
 
+	// Returns supplementary file descriptors that become read-ready in order
+	// to signal that there is some work that can be performed.
+	virtual iosource::FD_Set ExtraReadFDs() const
+		{ return iosource::FD_Set(); }
+
 	// Makes sure that no additional protocol data is written into
 	// the output stream.  If this is activated, the output cannot
 	// be read again by any of these classes!
@@ -177,6 +183,7 @@ public:
 	virtual void Clear();
 	virtual bool Eof()	{ return eof; }
 	virtual int Fd()	{ return fd; }
+	virtual iosource::FD_Set ExtraReadFDs() const;
 	virtual void Stats(char* buffer, int length);
 
 private:
@@ -240,6 +247,8 @@ private:
 	ChunkQueue* pending_tail;
 
 	pid_t pid;
+	bro::Flare write_flare;
+	bro::Flare read_flare;
 };
 
 // Chunked I/O using an SSL connection.
@@ -262,6 +271,7 @@ public:
 	virtual void Clear();
 	virtual bool Eof()	{ return eof; }
 	virtual int Fd()	{ return socket; }
+	virtual iosource::FD_Set ExtraReadFDs() const;
 	virtual void Stats(char* buffer, int length);
 
 private:
@@ -303,6 +313,8 @@ private:
 
 	// One SSL for all connections.
 	static SSL_CTX* ctx;
+
+	bro::Flare write_flare;
 };
 
 #include <zlib.h>
@@ -328,6 +340,8 @@ public:
 
 	virtual bool Eof()	{ return io->Eof(); }
 	virtual int Fd()	{ return io->Fd(); }
+	virtual iosource::FD_Set ExtraReadFDs() const
+		{ return io->ExtraReadFDs(); }
 	virtual void Stats(char* buffer, int length);
 
 	void EnableCompression(int level)

src/DNS_Mgr.cc

@@ -35,6 +35,7 @@
 #include "Net.h"
 #include "Var.h"
 #include "Reporter.h"
+#include "iosource/Manager.h"
 
 extern "C" {
 extern int select(int, fd_set *, fd_set *, fd_set *, struct timeval *);
@@ -404,17 +405,17 @@ DNS_Mgr::~DNS_Mgr()
 	delete [] dir;
 	}
 
-bool DNS_Mgr::Init()
+void DNS_Mgr::InitPostScript()
 	{
 	if ( did_init )
-		return true;
+		return;
 
 	const char* cache_dir = dir ? dir : ".";
 
 	if ( mode == DNS_PRIME && ! ensure_dir(cache_dir) )
 		{
 		did_init = 0;
-		return false;
+		return;
 		}
 
 	cache_name = new char[strlen(cache_dir) + 64];
@@ -433,14 +434,12 @@ bool DNS_Mgr::Init()
 
 	did_init = 1;
 
-	io_sources.Register(this, true);
+	iosource_mgr->Register(this, true);
 
 	// We never set idle to false, having the main loop only calling us from
 	// time to time. If we're issuing more DNS requests than we can handle
 	// in this way, we are having problems anyway ...
-	idle = true;
-
-	return true;
+	SetIdle(true);
 	}
 
 static TableVal* fake_name_lookup_result(const char* name)
@@ -1217,9 +1216,10 @@ void DNS_Mgr::IssueAsyncRequests()
 		}
 	}
 
-void  DNS_Mgr::GetFds(int* read, int* write, int* except)
+void DNS_Mgr::GetFds(iosource::FD_Set* read, iosource::FD_Set* write,
+                     iosource::FD_Set* except)
 	{
-	*read = nb_dns_fd(nb_dns);
+	read->Insert(nb_dns_fd(nb_dns));
 	}
 
 double DNS_Mgr::NextTimestamp(double* network_time)

src/DNS_Mgr.h

@@ -12,7 +12,7 @@
 #include "BroList.h"
 #include "Dict.h"
 #include "EventHandler.h"
-#include "IOSource.h"
+#include "iosource/IOSource.h"
 #include "IPAddr.h"
 
 class Val;
@@ -40,12 +40,12 @@ enum DNS_MgrMode {
 // Number of seconds we'll wait for a reply.
 #define DNS_TIMEOUT 5
 
-class DNS_Mgr : public IOSource {
+class DNS_Mgr : public iosource::IOSource {
 public:
 	DNS_Mgr(DNS_MgrMode mode);
 	virtual ~DNS_Mgr();
 
-	bool Init();
+	void InitPostScript();
 	void Flush();
 
 	// Looks up the address or addresses of the given host, and returns
@@ -132,7 +132,8 @@ protected:
 	void DoProcess(bool flush);
 
 	// IOSource interface.
-	virtual void GetFds(int* read, int* write, int* except);
+	virtual void GetFds(iosource::FD_Set* read, iosource::FD_Set* write,
+	                    iosource::FD_Set* except);
 	virtual double NextTimestamp(double* network_time);
 	virtual void Process();
 	virtual const char* Tag()	{ return "DNS_Mgr"; }

src/DebugLogger.cc

@@ -16,9 +16,10 @@ DebugLogger::Stream DebugLogger::streams[NUM_DBGS] = {
 	{ "compressor", 0, false }, {"string", 0, false },
 	{ "notifiers", 0, false },  { "main-loop", 0, false },
 	{ "dpd", 0, false }, { "tm", 0, false },
-	{ "logging", 0, false }, {"input", 0, false }, 
+	{ "logging", 0, false }, {"input", 0, false },
 	{ "threading", 0, false }, { "file_analysis", 0, false },
-	{ "plugins", 0, false }, { "broxygen", 0, false }
+	{ "plugins", 0, false }, { "broxygen", 0, false },
+	{ "pktio", 0, false}
 };
 
 DebugLogger::DebugLogger(const char* filename)

src/DebugLogger.h

@@ -31,6 +31,7 @@ enum DebugStream {
 	DBG_FILE_ANALYSIS,	// File analysis
 	DBG_PLUGINS,	// Plugin system
 	DBG_BROXYGEN,	// Broxygen
+	DBG_PKTIO,	// Packet sources and dumpers.
 
 	NUM_DBGS // Has to be last
 };

src/Flare.cc

@@ -0,0 +1,74 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#include "Flare.h"
+#include "Reporter.h"
+#include <unistd.h>
+#include <fcntl.h>
+#include <errno.h>
+
+using namespace bro;
+
+Flare::Flare()
+	: pipe(FD_CLOEXEC, FD_CLOEXEC, O_NONBLOCK, O_NONBLOCK)
+	{
+	}
+
+static void bad_pipe_op(const char* which)
+	{
+	char buf[256];
+	strerror_r(errno, buf, sizeof(buf));
+	reporter->FatalErrorWithCore("unexpected pipe %s failure: %s", which, buf);
+	}
+
+void Flare::Fire()
+	{
+	char tmp;
+
+	for ( ; ; )
+		{
+		int n = write(pipe.WriteFD(), &tmp, 1);
+
+		if ( n > 0 )
+			// Success -- wrote a byte to pipe.
+			break;
+
+		if ( n < 0 )
+			{
+			if ( errno == EAGAIN )
+				// Success: pipe is full and just need at least one byte in it.
+				break;
+
+			if ( errno == EINTR )
+				// Interrupted: try again.
+				continue;
+
+			bad_pipe_op("write");
+			}
+
+		// No error, but didn't write a byte: try again.
+		}
+	}
+
+void Flare::Extinguish()
+	{
+	char tmp[256];
+
+	for ( ; ; )
+		{
+		int n = read(pipe.ReadFD(), &tmp, sizeof(tmp));
+
+		if ( n >= 0 )
+			// Pipe may not be empty yet: try again.
+			continue;
+
+		if ( errno == EAGAIN )
+			// Success: pipe is now empty.
+			break;
+
+		if ( errno == EINTR )
+			// Interrupted: try again.
+			continue;
+
+		bad_pipe_op("read");
+		}
+	}

src/Flare.h

@@ -0,0 +1,44 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#ifndef BRO_FLARE_H
+#define BRO_FLARE_H
+
+#include "Pipe.h"
+
+namespace bro {
+
+class Flare {
+public:
+
+	/**
+	 * Create a flare object that can be used to signal a "ready" status via
+	 * a file descriptor that may be integrated with select(), poll(), etc.
+	 * Not thread-safe, but that should only require Fire()/Extinguish() calls
+	 * to be made mutually exclusive (across all copies of a Flare).
+	 */
+	Flare();
+
+	/**
+	 * @return a file descriptor that will become ready if the flare has been
+	 *         Fire()'d and not yet Extinguished()'d.
+	 */
+	int FD() const
+		{ return pipe.ReadFD(); }
+
+	/**
+	 * Put the object in the "ready" state.
+	 */
+	void Fire();
+
+	/**
+	 * Take the object out of the "ready" state.
+	 */
+	void Extinguish();
+
+private:
+	Pipe pipe;
+};
+
+} // namespace bro
+
+#endif // BRO_FLARE_H

src/FlowSrc.cc

@@ -1,228 +0,0 @@
-// See the file "COPYING" in the main distribution directory for copyright.
-//
-// Written by Bernhard Ager, TU Berlin (2006/2007).
-
-#include <stdio.h>
-#include <unistd.h>
-#include <fcntl.h>
-#include <netdb.h>
-
-#include "FlowSrc.h"
-#include "Net.h"
-#include "analyzer/protocol/netflow/netflow_pac.h"
-#include <errno.h>
-
-FlowSrc::FlowSrc()
-	{ // TODO: v9.
-	selectable_fd = -1;
-	idle = false;
-	data = 0;
-	pdu_len = -1;
-	exporter_ip = 0;
-	current_timestamp = next_timestamp = 0.0;
-	netflow_analyzer = new binpac::NetFlow::NetFlow_Analyzer();
-	}
-
-FlowSrc::~FlowSrc()
-	{
-	delete netflow_analyzer;
-	}
-
-void FlowSrc::GetFds(int* read, int* write, int* except)
-	{
-	if ( selectable_fd >= 0 )
-		*read = selectable_fd;
-	}
-
-double FlowSrc::NextTimestamp(double* network_time)
-	{
-	if ( ! data && ! ExtractNextPDU() )
-		return -1.0;
-	else
-		return next_timestamp;
-	}
-
-void FlowSrc::Process()
-	{
-	if ( ! data && ! ExtractNextPDU() )
-		return;
-
-	// This is normally done by calling net_packet_dispatch(),
-	// but as we don't have a packet to dispatch ...
-	net_update_time(next_timestamp);
-	expire_timers();
-
-	netflow_analyzer->downflow()->set_exporter_ip(exporter_ip);
-
-	// We handle exceptions in NewData (might have changed w/ new binpac).
-	netflow_analyzer->NewData(0, data, data + pdu_len);
-	data = 0;
-	}
-
-void FlowSrc::Close()
-	{
-	safe_close(selectable_fd);
-	}
-
-
-FlowSocketSrc::~FlowSocketSrc()
-	{
-	}
-
-int FlowSocketSrc::ExtractNextPDU()
-	{
-	sockaddr_in from;
-	socklen_t fromlen = sizeof(from);
-	pdu_len = recvfrom(selectable_fd, buffer, NF_MAX_PKT_SIZE, 0,
-				(struct sockaddr*) &from, &fromlen);
-	if ( pdu_len < 0 )
-		{
-		reporter->Error("problem reading NetFlow data from socket");
-		data = 0;
-		next_timestamp = -1.0;
-		closed = 1;
-		return 0;
-		}
-
-	if ( fromlen != sizeof(from) )
-		{
-		reporter->Error("malformed NetFlow PDU");
-		return 0;
-		}
-
-	data = buffer;
-	exporter_ip = from.sin_addr.s_addr;
-	next_timestamp = current_time();
-
-	if ( next_timestamp < current_timestamp )
-		next_timestamp = current_timestamp;
-	else
-		current_timestamp = next_timestamp;
-
-	return 1;
-	}
-
-FlowSocketSrc::FlowSocketSrc(const char* listen_parms)
-	{
-	int n = strlen(listen_parms) + 1;
-
-	char laddr[n], port[n], ident[n];
-	laddr[0] = port[0] = ident[0] = '\0';
-
-	int ret = sscanf(listen_parms, "%[^:]:%[^=]=%s", laddr, port, ident);
-	if ( ret < 2 )
-		{
-		snprintf(errbuf, BRO_FLOW_ERRBUF_SIZE,
-			"parsing your listen-spec went nuts: laddr='%s', port='%s'\n",
-			laddr[0] ? laddr : "", port[0] ? port : "");
-		closed = 1;
-		return;
-		}
-
-	const char* id = (ret == 3) ? ident : listen_parms;
-	netflow_analyzer->downflow()->set_identifier(id);
-
-	struct addrinfo aiprefs = {
-		0, PF_INET, SOCK_DGRAM, IPPROTO_UDP, 0, NULL, NULL, NULL
-	};
-	struct addrinfo* ainfo = 0;
-	if ( (ret = getaddrinfo(laddr, port, &aiprefs, &ainfo)) != 0 )
-		{
-		snprintf(errbuf, BRO_FLOW_ERRBUF_SIZE,
-				"getaddrinfo(%s, %s, ...): %s",
-				laddr, port, gai_strerror(ret));
-		closed = 1;
-		return;
-		}
-
-	if ( (selectable_fd = socket (PF_INET, SOCK_DGRAM, 0)) < 0 )
-		{
-		snprintf(errbuf, BRO_FLOW_ERRBUF_SIZE,
-				"socket: %s", strerror(errno));
-		closed = 1;
-		goto cleanup;
-		}
-
-	if ( bind (selectable_fd, ainfo->ai_addr, ainfo->ai_addrlen) < 0 )
-		{
-		snprintf(errbuf, BRO_FLOW_ERRBUF_SIZE,
-				"bind: %s", strerror(errno));
-		closed = 1;
-		goto cleanup;
-		}
-
-cleanup:
-	freeaddrinfo(ainfo);
-	}
-
-
-FlowFileSrc::~FlowFileSrc()
-	{
-	delete [] readfile;
-	}
-
-int FlowFileSrc::ExtractNextPDU()
-	{
-	FlowFileSrcPDUHeader pdu_header;
-
-	if ( read(selectable_fd, &pdu_header, sizeof(pdu_header)) <
-	     int(sizeof(pdu_header)) )
-		return Error(errno, "read header");
-
-	if ( pdu_header.pdu_length > NF_MAX_PKT_SIZE )
-		{
-		reporter->Error("NetFlow packet too long");
-
-		// Safely skip over the too-long PDU.
-		if ( lseek(selectable_fd, pdu_header.pdu_length, SEEK_CUR) < 0 )
-			return Error(errno, "lseek");
-		return 0;
-		}
-
-	if ( read(selectable_fd, buffer, pdu_header.pdu_length) <
-	     pdu_header.pdu_length )
-		return Error(errno, "read data");
-
-	if ( next_timestamp < pdu_header.network_time )
-		{
-		next_timestamp = pdu_header.network_time;
-		current_timestamp = pdu_header.network_time;
-		}
-	else
-		current_timestamp = next_timestamp;
-
-	data = buffer;
-	pdu_len = pdu_header.pdu_length;
-	exporter_ip = pdu_header.ipaddr;
-
-	return 1;
-	}
-
-FlowFileSrc::FlowFileSrc(const char* readfile)
-	{
-	int n = strlen(readfile) + 1;
-	char ident[n];
-	this->readfile = new char[n];
-
-	int ret = sscanf(readfile, "%[^=]=%s", this->readfile, ident);
-	const char* id = (ret == 2) ? ident : this->readfile;
-	netflow_analyzer->downflow()->set_identifier(id);
-
-	selectable_fd = open(this->readfile, O_RDONLY);
-	if ( selectable_fd < 0 )
-		{
-		closed = 1;
-		snprintf(errbuf, BRO_FLOW_ERRBUF_SIZE,
-				"open: %s", strerror(errno));
-		}
-	}
-
-int FlowFileSrc::Error(int errlvl, const char* errmsg)
-	{
-	snprintf(errbuf, BRO_FLOW_ERRBUF_SIZE,
-			"%s: %s", errmsg, strerror(errlvl));
-	data = 0;
-	next_timestamp = -1.0;
-	closed = 1;
-	return 0;
-	}

src/FlowSrc.h

@@ -1,84 +0,0 @@
-// See the file "COPYING" in the main distribution directory for copyright.
-//
-// Written by Bernhard Ager, TU Berlin (2006/2007).
-
-#ifndef flowsrc_h
-#define flowsrc_h
-
-#include "IOSource.h"
-#include "NetVar.h"
-#include "binpac.h"
-
-#define BRO_FLOW_ERRBUF_SIZE 512
-
-// TODO: 1500 is enough for v5 - how about the others?
-// 65536 would be enough for any UDP packet.
-#define NF_MAX_PKT_SIZE 8192
-
-struct FlowFileSrcPDUHeader {
-	double network_time;
-	int pdu_length;
-	uint32 ipaddr;
-};
-
-// Avoid including netflow_pac.h by explicitly declaring the NetFlow_Analyzer.
-namespace binpac {
-	namespace NetFlow {
-		class NetFlow_Analyzer;
-	}
-}
-
-class FlowSrc : public IOSource {
-public:
-	virtual ~FlowSrc();
-
-	// IOSource interface:
-	bool IsReady();
-	void GetFds(int* read, int* write, int* except);
-	double NextTimestamp(double* network_time);
-	void Process();
-
-	const char* Tag()		{ return "FlowSrc"; }
-	const char* ErrorMsg() const	{ return errbuf; }
-
-protected:
-	FlowSrc();
-
-	virtual int ExtractNextPDU() = 0;
-	virtual void Close();
-
-	int selectable_fd;
-
-	double current_timestamp;
-	double next_timestamp;
-	binpac::NetFlow::NetFlow_Analyzer* netflow_analyzer;
-
-	u_char buffer[NF_MAX_PKT_SIZE];
-	u_char* data;
-	int pdu_len;
-	uint32 exporter_ip;	// in network byte order
-
-	char errbuf[BRO_FLOW_ERRBUF_SIZE];
-};
-
-class FlowSocketSrc : public FlowSrc {
-public:
-	FlowSocketSrc(const char* listen_parms);
-	virtual ~FlowSocketSrc();
-
-	int ExtractNextPDU();
-};
-
-class FlowFileSrc : public FlowSrc {
-public:
-	FlowFileSrc(const char* readfile);
-	~FlowFileSrc();
-
-	int ExtractNextPDU();
-
-protected:
-	int Error(int errlvl, const char* errmsg);
-	char* readfile;
-};
-
-#endif

src/IOSource.cc

@@ -1,176 +0,0 @@
-#include <sys/types.h>
-#include <sys/time.h>
-#include <unistd.h>
-#include <assert.h>
-
-#include <algorithm>
-
-#include "util.h"
-#include "IOSource.h"
-
-IOSourceRegistry io_sources;
-
-IOSourceRegistry::~IOSourceRegistry()
-	{
-	for ( SourceList::iterator i = sources.begin(); i != sources.end(); ++i )
-		delete *i;
-
-	sources.clear();
-	}
-
-void IOSourceRegistry::RemoveAll()
-	{
-	// We're cheating a bit here ...
-	dont_counts = sources.size();
-	}
-
-IOSource* IOSourceRegistry::FindSoonest(double* ts)
-	{
-	// Remove sources which have gone dry. For simplicity, we only
-	// remove at most one each time.
-	for ( SourceList::iterator i = sources.begin();
-	      i != sources.end(); ++i )
-		if ( ! (*i)->src->IsOpen() )
-			{
-			delete *i;
-			sources.erase(i);
-			break;
-			}
-
-	// Ideally, we would always call select on the fds to see which
-	// are ready, and return the soonest. Unfortunately, that'd mean
-	// one select-call per packet, which we can't afford in high-volume
-	// environments.  Thus, we call select only every SELECT_FREQUENCY
-	// call (or if all sources report that they are dry).
-
-	++call_count;
-
-	IOSource* soonest_src = 0;
-	double soonest_ts = 1e20;
-	double soonest_local_network_time = 1e20;
-	bool all_idle = true;
-
-	// Find soonest source of those which tell us they have something to
-	// process.
-	for ( SourceList::iterator i = sources.begin(); i != sources.end(); ++i )
-		{
-		if ( ! (*i)->src->IsIdle() )
-			{
-			all_idle = false;
-			double local_network_time = 0;
-			double ts = (*i)->src->NextTimestamp(&local_network_time);
-			if ( ts > 0 && ts < soonest_ts )
-				{
-				soonest_ts = ts;
-				soonest_src = (*i)->src;
-				soonest_local_network_time =
-					local_network_time ?
-						local_network_time : ts;
-				}
-			}
-		}
-
-	// If we found one and aren't going to select this time,
-	// return it.
-	int maxx = 0;
-
-	if ( soonest_src && (call_count % SELECT_FREQUENCY) != 0 )
-		goto finished;
-
-	// Select on the join of all file descriptors.
-	fd_set fd_read, fd_write, fd_except;
-
-	FD_ZERO(&fd_read);
-	FD_ZERO(&fd_write);
-	FD_ZERO(&fd_except);
-
-	for ( SourceList::iterator i = sources.begin();
-	      i != sources.end(); ++i )
-		{
-		Source* src = (*i);
-
-		if ( ! src->src->IsIdle() )
-			// No need to select on sources which we know to
-			// be ready.
-			continue;
-
-		src->fd_read = src->fd_write = src->fd_except = 0;
-		src->src->GetFds(&src->fd_read, &src->fd_write, &src->fd_except);
-
-		FD_SET(src->fd_read, &fd_read);
-		FD_SET(src->fd_write, &fd_write);
-		FD_SET(src->fd_except, &fd_except);
-
-		maxx = max(src->fd_read, maxx);
-		maxx = max(src->fd_write, maxx);
-		maxx = max(src->fd_except, maxx);
-		}
-
-	// We can't block indefinitely even when all sources are dry:
-	// we're doing some IOSource-independent stuff in the main loop,
-	// so we need to return from time to time. (Instead of no time-out
-	// at all, we use a very small one. This lets FreeBSD trigger a
-	// BPF buffer switch on the next read when the hold buffer is empty
-	// while the store buffer isn't filled yet.
-
-	struct timeval timeout;
-
-	if ( all_idle )
-		{
-		// Interesting: when all sources are dry, simply sleeping a
-		// bit *without* watching for any fd becoming ready may
-		// decrease CPU load. I guess that's because it allows
-		// the kernel's packet buffers to fill. - Robin
-		timeout.tv_sec = 0;
-		timeout.tv_usec = 20; // SELECT_TIMEOUT;
-		select(0, 0, 0, 0, &timeout);
-		}
-
-	if ( ! maxx )
-		// No selectable fd at all.
-		goto finished;
-
-	timeout.tv_sec = 0;
-	timeout.tv_usec = 0;
-
-	if ( select(maxx + 1, &fd_read, &fd_write, &fd_except, &timeout) > 0 )
-		{ // Find soonest.
-		for ( SourceList::iterator i = sources.begin();
-		      i != sources.end(); ++i )
-			{
-			Source* src = (*i);
-
-			if ( ! src->src->IsIdle() )
-				continue;
-
-			if ( FD_ISSET(src->fd_read, &fd_read) ||
-			     FD_ISSET(src->fd_write, &fd_write) ||
-			     FD_ISSET(src->fd_except, &fd_except) )
-				{
-				double local_network_time = 0;
-				double ts = src->src->NextTimestamp(&local_network_time);
-				if ( ts > 0.0 && ts < soonest_ts )
-					{
-					soonest_ts = ts;
-					soonest_src = src->src;
-					soonest_local_network_time =
-						local_network_time ?
-							local_network_time : ts;
-					}
-				}
-			}
-		}
-
-finished:
-	*ts = soonest_local_network_time;
-	return soonest_src;
-	}
-
-void IOSourceRegistry::Register(IOSource* src, bool dont_count)
-	{
-	Source* s = new Source;
-	s->src = src;
-	if ( dont_count )
-		++dont_counts;
-	return sources.push_back(s);
-	}

src/IOSource.h

@@ -1,103 +0,0 @@
-// Interface for classes providing/consuming data during Bro's main loop.
-
-#ifndef iosource_h
-#define iosource_h
-
-#include <list>
-#include "Timer.h"
-
-using namespace std;
-
-class IOSource {
-public:
-	IOSource()	{ idle = closed = false; }
-	virtual ~IOSource()	{}
-
-	// Returns true if source has nothing ready to process.
-	bool IsIdle() const	{ return idle; }
-
-	// Returns true if more data is to be expected in the future.
-	// Otherwise, source may be removed.
-	bool IsOpen() const	{ return ! closed; }
-
-	// Returns select'able fds (leaves args untouched if we don't have
-	// selectable fds).
-	virtual void GetFds(int* read, int* write, int* except) = 0;
-
-	// The following two methods are only called when either IsIdle()
-	// returns false or select() on one of the fds indicates that there's
-	// data to process.
-
-	// Returns timestamp (in global network time) associated with next
-	// data item.  If the source wants the data item to be processed
-	// with a local network time, it sets the argument accordingly.
-	virtual double NextTimestamp(double* network_time) = 0;
-
-	// Processes and consumes next data item.
-	virtual void Process() = 0;
-
-	// Returns tag of timer manager associated with last processed
-	// data item, nil for global timer manager.
-	virtual TimerMgr::Tag* GetCurrentTag()	{ return 0; }
-
-	// Returns a descriptual tag for debugging.
-	virtual const char* Tag() = 0;
-
-protected:
-	// Derived classed are to set this to true if they have gone dry
-	// temporarily.
-	bool idle;
-
-	// Derived classed are to set this to true if they have gone dry
-	// permanently.
-	bool closed;
-};
-
-class IOSourceRegistry {
-public:
-	IOSourceRegistry()	{ call_count = 0; dont_counts = 0; }
-	~IOSourceRegistry();
-
-	// If dont_count is true, this source does not contribute to the
-	// number of IOSources returned by Size().  The effect is that
-	// if all sources but the non-counting ones have gone dry,
-	// processing will shut down.
-	void Register(IOSource* src, bool dont_count = false);
-
-	// This may block for some time.
-	IOSource* FindSoonest(double* ts);
-
-	int Size() const	{ return sources.size() - dont_counts; }
-
-	// Terminate IOSource processing immediately by removing all
-	// sources (and therefore returning a Size() of zero).
-	void Terminate()	{ RemoveAll(); }
-
-protected:
-	// When looking for a source with something to process,
-	// every SELECT_FREQUENCY calls we will go ahead and
-	// block on a select().
-	static const int SELECT_FREQUENCY = 25;
-
-	// Microseconds to wait in an empty select if no source is ready.
-	static const int SELECT_TIMEOUT = 50;
-
-	void RemoveAll();
-
-	unsigned int call_count;
-	int dont_counts;
-
-	struct Source {
-		IOSource* src;
-		int fd_read;
-		int fd_write;
-		int fd_except;
-	};
-
-	typedef list<Source*> SourceList;
-	SourceList sources;
-};
-
-extern IOSourceRegistry io_sources;
-
-#endif

src/Net.cc

@@ -29,6 +29,9 @@
 #include "Anon.h"
 #include "Serializer.h"
 #include "PacketDumper.h"
+#include "iosource/Manager.h"
+#include "iosource/PktSrc.h"
+#include "iosource/PktDumper.h"
 #include "plugin/Manager.h"
 
 extern "C" {
@@ -39,10 +42,7 @@ extern "C" {
 extern int select(int, fd_set *, fd_set *, fd_set *, struct timeval *);
 }
 
-PList(PktSrc) pkt_srcs;
-
-// FIXME: We should really merge PktDumper and PacketDumper.
-PktDumper* pkt_dumper = 0;
+iosource::PktDumper* pkt_dumper = 0;
 
 int reading_live = 0;
 int reading_traces = 0;
@@ -63,8 +63,8 @@ const u_char* current_pkt = 0;
 int current_dispatched = 0;
 int current_hdr_size = 0;
 double current_timestamp = 0.0;
-PktSrc* current_pktsrc = 0;
-IOSource* current_iosrc;
+iosource::PktSrc* current_pktsrc = 0;
+iosource::IOSource* current_iosrc = 0;
 
 std::list<ScannedFile> files_scanned;
 std::vector<string> sig_files;
@@ -113,17 +113,21 @@ RETSIGTYPE watchdog(int /* signo */)
 					// saving the packet which caused the
 					// watchdog to trigger may be helpful,
 					// so we'll save that one nevertheless.
-					pkt_dumper = new PktDumper("watchdog-pkt.pcap");
-					if ( pkt_dumper->IsError() )
+					pkt_dumper = iosource_mgr->OpenPktDumper("watchdog-pkt.pcap", false);
+					if ( ! pkt_dumper || pkt_dumper->IsError() )
 						{
-						reporter->Error("watchdog: can't open watchdog-pkt.pcap for writing\n");
-						delete pkt_dumper;
+						reporter->Error("watchdog: can't open watchdog-pkt.pcap for writing");
 						pkt_dumper = 0;
 						}
 					}
 
 				if ( pkt_dumper )
-					pkt_dumper->Dump(current_hdr, current_pkt);
+					{
+					iosource::PktDumper::Packet p;
+					p.hdr = current_hdr;
+					p.data = current_pkt;
+					pkt_dumper->Dump(&p);
+					}
 				}
 
 			net_get_final_stats();
@@ -149,118 +153,40 @@ void net_update_time(double new_network_time)
 	}
 
 void net_init(name_list& interfaces, name_list& readfiles,
-	      name_list& netflows, name_list& flowfiles,
-	        const char* writefile, const char* filter,
-			const char* secondary_filter, int do_watchdog)
+	      const char* writefile, int do_watchdog)
 	{
-	if ( readfiles.length() > 0 || flowfiles.length() > 0 )
+	if ( readfiles.length() > 0 )
 		{
 		reading_live = pseudo_realtime > 0.0;
 		reading_traces = 1;
 
 		for ( int i = 0; i < readfiles.length(); ++i )
 			{
-			PktFileSrc* ps = new PktFileSrc(readfiles[i], filter);
+			iosource::PktSrc* ps = iosource_mgr->OpenPktSrc(readfiles[i], false);
+			assert(ps);
 
 			if ( ! ps->IsOpen() )
-				reporter->FatalError("%s: problem with trace file %s - %s\n",
-					prog, readfiles[i], ps->ErrorMsg());
-			else
-				{
-				pkt_srcs.append(ps);
-				io_sources.Register(ps);
-				}
-
-			if ( secondary_filter )
-				{
-				// We use a second PktFileSrc for the
-				// secondary path.
-				PktFileSrc* ps = new PktFileSrc(readfiles[i],
-							secondary_filter,
-							TYPE_FILTER_SECONDARY);
-
-				if ( ! ps->IsOpen() )
-					reporter->FatalError("%s: problem with trace file %s - %s\n",
-						prog, readfiles[i],
-						ps->ErrorMsg());
-				else
-					{
-					pkt_srcs.append(ps);
-					io_sources.Register(ps);
-					}
-
-				ps->AddSecondaryTablePrograms();
-				}
-			}
-
-		for ( int i = 0; i < flowfiles.length(); ++i )
-			{
-			FlowFileSrc* fs = new FlowFileSrc(flowfiles[i]);
-
-			if ( ! fs->IsOpen() )
-				reporter->FatalError("%s: problem with netflow file %s - %s\n",
-					prog, flowfiles[i], fs->ErrorMsg());
-			else
-				{
-				io_sources.Register(fs);
-				}
+				reporter->FatalError("problem with trace file %s (%s)",
+						     readfiles[i],
+						     ps->ErrorMsg());
 			}
 		}
 
-	else if ((interfaces.length() > 0 || netflows.length() > 0))
+	else if ( interfaces.length() > 0 )
 		{
 		reading_live = 1;
 		reading_traces = 0;
 
 		for ( int i = 0; i < interfaces.length(); ++i )
 			{
-			PktSrc* ps;
-			ps = new PktInterfaceSrc(interfaces[i], filter);
+			iosource::PktSrc* ps = iosource_mgr->OpenPktSrc(interfaces[i], true);
+			assert(ps);
 
 			if ( ! ps->IsOpen() )
-				reporter->FatalError("%s: problem with interface %s - %s\n",
-					prog, interfaces[i], ps->ErrorMsg());
-			else
-				{
-				pkt_srcs.append(ps);
-				io_sources.Register(ps);
-				}
-
-			if ( secondary_filter )
-				{
-				PktSrc* ps;
-				ps = new PktInterfaceSrc(interfaces[i],
-					filter, TYPE_FILTER_SECONDARY);
-
-				if ( ! ps->IsOpen() )
-					reporter->Error("%s: problem with interface %s - %s\n",
-						prog, interfaces[i],
-						ps->ErrorMsg());
-				else
-					{
-					pkt_srcs.append(ps);
-					io_sources.Register(ps);
-					}
-
-				ps->AddSecondaryTablePrograms();
-				}
+				reporter->FatalError("problem with interface %s (%s)",
+						     interfaces[i],
+						     ps->ErrorMsg());
 			}
-
-		for ( int i = 0; i < netflows.length(); ++i )
-			{
-			FlowSocketSrc* fs = new FlowSocketSrc(netflows[i]);
-
-			if ( ! fs->IsOpen() )
-				{
-				reporter->Error("%s: problem with netflow socket %s - %s\n",
-					prog, netflows[i], fs->ErrorMsg());
-				delete fs;
-				}
-
-			else
-				io_sources.Register(fs);
-			}
-
 		}
 
 	else
@@ -272,12 +198,12 @@ void net_init(name_list& interfaces, name_list& readfiles,
 
 	if ( writefile )
 		{
-		// ### This will fail horribly if there are multiple
-		// interfaces with different-lengthed media.
-		pkt_dumper = new PktDumper(writefile);
-		if ( pkt_dumper->IsError() )
-			reporter->FatalError("%s: can't open write file \"%s\" - %s\n",
-				prog, writefile, pkt_dumper->ErrorMsg());
+		pkt_dumper = iosource_mgr->OpenPktDumper(writefile, false);
+		assert(pkt_dumper);
+
+		if ( ! pkt_dumper->IsOpen() )
+			reporter->FatalError("problem opening dump file %s (%s)",
+					     writefile, pkt_dumper->ErrorMsg());
 
 		ID* id = global_scope()->Lookup("trace_output_file");
 		if ( ! id )
@@ -298,7 +224,7 @@ void net_init(name_list& interfaces, name_list& readfiles,
 		}
 	}
 
-void expire_timers(PktSrc* src_ps)
+void expire_timers(iosource::PktSrc* src_ps)
 	{
 	SegmentProfiler(segment_logger, "expiring-timers");
 	TimerMgr* tmgr =
@@ -311,8 +237,8 @@ void expire_timers(PktSrc* src_ps)
 	}
 
 void net_packet_dispatch(double t, const struct pcap_pkthdr* hdr,
-			const u_char* pkt, int hdr_size,
-			PktSrc* src_ps)
+			 const u_char* pkt, int hdr_size,
+			 iosource::PktSrc* src_ps)
 	{
 	if ( ! bro_start_network_time )
 		bro_start_network_time = t;
@@ -368,11 +294,11 @@ void net_run()
 	{
 	set_processing_status("RUNNING", "net_run");
 
-	while ( io_sources.Size() ||
+	while ( iosource_mgr->Size() ||
 		(BifConst::exit_only_after_terminate && ! terminating) )
 		{
 		double ts;
-		IOSource* src = io_sources.FindSoonest(&ts);
+		iosource::IOSource* src = iosource_mgr->FindSoonest(&ts);
 
 #ifdef DEBUG
 		static int loop_counter = 0;
@@ -470,16 +396,19 @@ void net_run()
 
 void net_get_final_stats()
 	{
-	loop_over_list(pkt_srcs, i)
+	const iosource::Manager::PktSrcList& pkt_srcs(iosource_mgr->GetPktSrcs());
+
+	for ( iosource::Manager::PktSrcList::const_iterator i = pkt_srcs.begin();
+	      i != pkt_srcs.end(); i++ )
 		{
-		PktSrc* ps = pkt_srcs[i];
+		iosource::PktSrc* ps = *i;
 
 		if ( ps->IsLive() )
 			{
-			struct PktSrc::Stats s;
+			iosource::PktSrc::Stats s;
 			ps->Statistics(&s);
-			reporter->Info("%d packets received on interface %s, %d dropped\n",
-					s.received, ps->Interface(), s.dropped);
+			reporter->Info("%d packets received on interface %s, %d dropped",
+					s.received, ps->Path().c_str(), s.dropped);
 			}
 		}
 	}
@@ -499,8 +428,6 @@ void net_finish(int drain_events)
 			sessions->Done();
 		}
 
-	delete pkt_dumper;
-
 #ifdef DEBUG
 	extern int reassem_seen_bytes, reassem_copied_bytes;
 	// DEBUG_MSG("Reassembly (TCP and IP/Frag): %d bytes seen, %d bytes copied\n",
@@ -521,29 +448,6 @@ void net_delete()
 		delete ip_anonymizer[i];
 	}
 
-// net_packet_match
-//
-// Description:
-//  - Checks if a packet matches a filter. It just wraps up a call to
-//    [pcap.h's] bpf_filter().
-//
-// Inputs:
-//  - fp: a BPF-compiled filter
-//  - pkt: a pointer to the packet
-//  - len: the original packet length
-//  - caplen: the captured packet length. This is pkt length
-//
-// Output:
-//  - return: 1 if the packet matches the filter, 0 otherwise
-
-int net_packet_match(BPF_Program* fp, const u_char* pkt,
-		     u_int len, u_int caplen)
-	{
-	// NOTE: I don't like too much un-const'ing the pkt variable.
-	return bpf_filter(fp->GetProgram()->bf_insns, (u_char*) pkt, len, caplen);
-	}
-
-
 int _processing_suspended = 0;
 
 static double suspend_start = 0;
@@ -561,8 +465,12 @@ void net_continue_processing()
 	if ( _processing_suspended == 1 )
 		{
 		reporter->Info("processing continued");
-		loop_over_list(pkt_srcs, i)
-			pkt_srcs[i]->ContinueAfterSuspend();
+
+		const iosource::Manager::PktSrcList& pkt_srcs(iosource_mgr->GetPktSrcs());
+
+		for ( iosource::Manager::PktSrcList::const_iterator i = pkt_srcs.begin();
+		      i != pkt_srcs.end(); i++ )
+			(*i)->ContinueAfterSuspend();
 		}
 
 	--_processing_suspended;

src/Net.h

@@ -5,17 +5,15 @@
 
 #include "net_util.h"
 #include "util.h"
-#include "BPF_Program.h"
 #include "List.h"
-#include "PktSrc.h"
-#include "FlowSrc.h"
 #include "Func.h"
 #include "RemoteSerializer.h"
+#include "iosource/IOSource.h"
+#include "iosource/PktSrc.h"
+#include "iosource/PktDumper.h"
 
 extern void net_init(name_list& interfaces, name_list& readfiles,
-		name_list& netflows, name_list& flowfiles,
-		const char* writefile, const char* filter,
-		const char* secondary_filter, int do_watchdog);
+		const char* writefile, int do_watchdog);
 extern void net_run();
 extern void net_get_final_stats();
 extern void net_finish(int drain_events);
@@ -23,10 +21,8 @@ extern void net_delete();	// Reclaim all memory, etc.
 extern void net_update_time(double new_network_time);
 extern void net_packet_dispatch(double t, const struct pcap_pkthdr* hdr,
 			const u_char* pkt, int hdr_size,
-			PktSrc* src_ps);
-extern int net_packet_match(BPF_Program* fp, const u_char* pkt,
-			    u_int len, u_int caplen);
-extern void expire_timers(PktSrc* src_ps = 0);
+			iosource::PktSrc* src_ps);
+extern void expire_timers(iosource::PktSrc* src_ps = 0);
 extern void termination_signal();
 
 // Functions to temporarily suspend processing of live input (network packets
@@ -83,13 +79,10 @@ extern const u_char* current_pkt;
 extern int current_dispatched;
 extern int current_hdr_size;
 extern double current_timestamp;
-extern PktSrc* current_pktsrc;
-extern IOSource* current_iosrc;
+extern iosource::PktSrc* current_pktsrc;
+extern iosource::IOSource* current_iosrc;
 
-declare(PList,PktSrc);
-extern PList(PktSrc) pkt_srcs;
-
-extern PktDumper* pkt_dumper;	// where to save packets
+extern iosource::PktDumper* pkt_dumper;	// where to save packets
 
 extern char* writefile;
 

src/Pipe.cc

@@ -0,0 +1,83 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#include "Pipe.h"
+#include "Reporter.h"
+#include <unistd.h>
+#include <fcntl.h>
+#include <errno.h>
+#include <cstdio>
+
+using namespace bro;
+
+static void pipe_fail(int eno)
+	{
+	char tmp[256];
+	strerror_r(eno, tmp, sizeof(tmp));
+	reporter->FatalError("Pipe failure: %s", tmp);
+	}
+
+static void set_flags(int fd, int flags)
+	{
+	if ( flags )
+		fcntl(fd, F_SETFD, fcntl(fd, F_GETFD) | flags);
+	}
+
+static void set_status_flags(int fd, int flags)
+	{
+	if ( flags )
+		fcntl(fd, F_SETFL, fcntl(fd, F_GETFL) | flags);
+	}
+
+static int dup_or_fail(int fd, int flags)
+	{
+	int rval = dup(fd);
+
+	if ( rval < 0 )
+		pipe_fail(errno);
+
+	set_flags(fd, flags);
+	return rval;
+	}
+
+Pipe::Pipe(int flags0, int flags1, int status_flags0, int status_flags1)
+	{
+	// pipe2 can set flags atomically, but not yet available everywhere.
+	if ( ::pipe(fds) )
+		pipe_fail(errno);
+
+	flags[0] = flags0;
+	flags[1] = flags1;
+
+	set_flags(fds[0], flags[0]);
+	set_flags(fds[1], flags[1]);
+	set_status_flags(fds[0], status_flags0);
+	set_status_flags(fds[1], status_flags1);
+	}
+
+Pipe::~Pipe()
+	{
+	close(fds[0]);
+	close(fds[1]);
+	}
+
+Pipe::Pipe(const Pipe& other)
+	{
+	fds[0] = dup_or_fail(other.fds[0], other.flags[0]);
+	fds[1] = dup_or_fail(other.fds[1], other.flags[1]);
+	flags[0] = other.flags[0];
+	flags[1] = other.flags[1];
+	}
+
+Pipe& Pipe::operator=(const Pipe& other)
+	{
+	if ( this == &other )
+		return *this;
+
+	close(fds[0]);
+	close(fds[1]);
+	fds[0] = dup_or_fail(other.fds[0], other.flags[0]);
+	fds[1] = dup_or_fail(other.fds[1], other.flags[1]);
+	flags[0] = other.flags[0];
+	flags[1] = other.flags[1];
+	return *this;
+	}

src/Pipe.h

@@ -0,0 +1,56 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#ifndef BRO_PIPE_H
+#define BRO_PIPE_H
+
+namespace bro {
+
+class Pipe {
+public:
+
+	/**
+	 * Create a pair of file descriptors via pipe(), or aborts if it cannot.
+	 * @param flags0 file descriptor flags to set on read end of pipe.
+	 * @param flags1 file descriptor flags to set on write end of pipe.
+	 * @param status_flags0 descriptor status flags to set on read end of pipe.
+	 * @param status_flags1 descriptor status flags to set on write end of pipe.
+	 */
+	Pipe(int flags0 = 0, int flags1 = 0, int status_flags0 = 0,
+	     int status_flags1 = 0);
+
+	/**
+	  * Close the pair of file descriptors owned by the object.
+	  */
+	~Pipe();
+
+	/**
+	 * Make a copy of another Pipe object (file descriptors are dup'd).
+	 */
+	Pipe(const Pipe& other);
+
+	/**
+	 * Assign a Pipe object by closing file descriptors and duping those of
+	 * the other.
+	 */
+	Pipe& operator=(const Pipe& other);
+
+	/**
+	 * @return the file descriptor associated with the read-end of the pipe.
+	 */
+	int ReadFD() const
+		{ return fds[0]; }
+
+	/**
+	 * @return the file descriptor associated with the write-end of the pipe.
+	 */
+	int WriteFD() const
+		{ return fds[1]; }
+
+private:
+	int fds[2];
+	int flags[2];
+};
+
+} // namespace bro
+
+#endif // BRO_PIPE_H

src/PktSrc.cc

@@ -1,804 +0,0 @@
-// See the file "COPYING" in the main distribution directory for copyright.
-
-#include <errno.h>
-#include <sys/stat.h>
-
-#include "config.h"
-
-#include "util.h"
-#include "PktSrc.h"
-#include "Hash.h"
-#include "Net.h"
-#include "Sessions.h"
-
-
-// ### This needs auto-confing.
-#ifdef HAVE_PCAP_INT_H
-#include <pcap-int.h>
-#endif
-
-PktSrc::PktSrc()
-	{
-	interface = readfile = 0;
-	data = last_data = 0;
-	memset(&hdr, 0, sizeof(hdr));
-	hdr_size = 0;
-	datalink = 0;
-	netmask = 0xffffff00;
-	pd = 0;
-	idle = false;
-
-	next_sync_point = 0;
-	first_timestamp = current_timestamp = next_timestamp = 0.0;
-	first_wallclock = current_wallclock = 0;
-
-	stats.received = stats.dropped = stats.link = 0;
-	}
-
-PktSrc::~PktSrc()
-	{
-	Close();
-
-	loop_over_list(program_list, i)
-		delete program_list[i];
-
-	BPF_Program* code;
-	IterCookie* cookie = filters.InitForIteration();
-	while ( (code = filters.NextEntry(cookie)) )
-		delete code;
-
-	delete [] interface;
-	delete [] readfile;
-	}
-
-void PktSrc::GetFds(int* read, int* write, int* except)
-	{
-	if ( pseudo_realtime )
-		{
-		// Select would give erroneous results. But we simulate it
-		// by setting idle accordingly.
-		idle = CheckPseudoTime() == 0;
-		return;
-		}
-
-	if ( selectable_fd >= 0 )
-		*read = selectable_fd;
-	}
-
-int PktSrc::ExtractNextPacket()
-	{
-	// Don't return any packets if processing is suspended (except for the
-	// very first packet which we need to set up times).
-	if ( net_is_processing_suspended() && first_timestamp )
-		{
-		idle = true;
-		return 0;
-		}
-
-	data = last_data = pcap_next(pd, &hdr);
-
-	if ( data && (hdr.len == 0 || hdr.caplen == 0) )
-		{
-		sessions->Weird("empty_pcap_header", &hdr, data);
-		return 0;
-		}
-
-	if ( data )
-		next_timestamp = hdr.ts.tv_sec + double(hdr.ts.tv_usec) / 1e6;
-
-	if ( pseudo_realtime )
-		current_wallclock = current_time(true);
-
-	if ( ! first_timestamp )
-		first_timestamp = next_timestamp;
-
-	idle = (data == 0);
-
-	if ( data )
-		++stats.received;
-
-	// Source has gone dry.  If it's a network interface, this just means
-	// it's timed out. If it's a file, though, then the file has been
-	// exhausted.
-	if ( ! data && ! IsLive() )
-		{
-		closed = true;
-
-		if ( pseudo_realtime && using_communication )
-			{
-			if ( remote_trace_sync_interval )
-				remote_serializer->SendFinalSyncPoint();
-			else
-				remote_serializer->Terminate();
-			}
-		}
-
-	return data != 0;
-	}
-
-double PktSrc::NextTimestamp(double* local_network_time)
-	{
-	if ( ! data && ! ExtractNextPacket() )
-		return -1.0;
-
-	if ( pseudo_realtime )
-		{
-		// Delay packet if necessary.
-		double packet_time = CheckPseudoTime();
-		if ( packet_time )
-			return packet_time;
-
-		idle = true;
-		return -1.0;
-		}
-
-	return next_timestamp;
-	}
-
-void PktSrc::ContinueAfterSuspend()
-	{
-	current_wallclock = current_time(true);
-	}
-
-double PktSrc::CurrentPacketWallClock()
-	{
-	// We stop time when we are suspended.
-	if ( net_is_processing_suspended() )
-		current_wallclock = current_time(true);
-
-	return current_wallclock;
-	}
-
-double PktSrc::CheckPseudoTime()
-	{
-	if ( ! data && ! ExtractNextPacket() )
-		return 0;
-
-	if ( ! current_timestamp )
-		return bro_start_time;
-
-	if ( remote_trace_sync_interval )
-		{
-		if ( next_sync_point == 0 || next_timestamp >= next_sync_point )
-			{
-			int n = remote_serializer->SendSyncPoint();
-			next_sync_point = first_timestamp +
-						n * remote_trace_sync_interval;
-			remote_serializer->Log(RemoteSerializer::LogInfo,
-				fmt("stopping at packet %.6f, next sync-point at %.6f",
-					current_timestamp, next_sync_point));
-
-			return 0;
-			}
-		}
-
-	double pseudo_time = next_timestamp - first_timestamp;
-	double ct = (current_time(true) - first_wallclock) * pseudo_realtime;
-
-	return pseudo_time <= ct ? bro_start_time + pseudo_time : 0;
-	}
-
-void PktSrc::Process()
-	{
-	if ( ! data && ! ExtractNextPacket() )
-		return;
-
-	current_timestamp = next_timestamp;
-
-	int pkt_hdr_size = hdr_size;
-
-	// Unfortunately some packets on the link might have MPLS labels
-	// while others don't. That means we need to ask the link-layer if
-	// labels are in place.
-	bool have_mpls = false;
-
-	int protocol = 0;
-
-	switch ( datalink ) {
-	case DLT_NULL:
-		{
-		protocol = (data[3] << 24) + (data[2] << 16) + (data[1] << 8) + data[0];
-
-		// From the Wireshark Wiki: "AF_INET6, unfortunately, has
-		// different values in {NetBSD,OpenBSD,BSD/OS},
-		// {FreeBSD,DragonFlyBSD}, and {Darwin/Mac OS X}, so an IPv6
-		// packet might have a link-layer header with 24, 28, or 30
-		// as the AF_ value." As we may be reading traces captured on
-		// platforms other than what we're running on, we accept them
-		// all here.
-		if ( protocol != AF_INET
-		     && protocol != AF_INET6
-		     && protocol != 24
-		     && protocol != 28
-		     && protocol != 30 )
-			{
-			sessions->Weird("non_ip_packet_in_null_transport", &hdr, data);
-			data = 0;
-			return;
-			}
-
-		break;
-		}
-
-	case DLT_EN10MB:
-		{
-		// Get protocol being carried from the ethernet frame.
-		protocol = (data[12] << 8) + data[13];
-
-		switch ( protocol )
-			{
-			// MPLS carried over the ethernet frame.
-			case 0x8847:
-				// Remove the data link layer and denote a
-				// header size of zero before the IP header.
-				have_mpls = true;
-				data += get_link_header_size(datalink);
-				pkt_hdr_size = 0;
-				break;
-
-			// VLAN carried over the ethernet frame.
-			case 0x8100:
-				data += get_link_header_size(datalink);
-
-				// Check for MPLS in VLAN.
-				if ( ((data[2] << 8) + data[3]) == 0x8847 )
-					have_mpls = true;
-
-				data += 4; // Skip the vlan header
-				pkt_hdr_size = 0;
-
-				// Check for 802.1ah (Q-in-Q) containing IP.
-				// Only do a second layer of vlan tag
-				// stripping because there is no
-				// specification that allows for deeper
-				// nesting.
-				if ( ((data[2] << 8) + data[3]) == 0x0800 )
-					data += 4;
-
-				break;
-
-			// PPPoE carried over the ethernet frame.
-			case 0x8864:
-				data += get_link_header_size(datalink);
-				protocol = (data[6] << 8) + data[7];
-				data += 8; // Skip the PPPoE session and PPP header
-				pkt_hdr_size = 0;
-
-				if ( protocol != 0x0021 && protocol != 0x0057 )
-					{
-					// Neither IPv4 nor IPv6.
-					sessions->Weird("non_ip_packet_in_pppoe_encapsulation", &hdr, data);
-					data = 0;
-					return;
-					}
-				break;
-			}
-
-		break;
-		}
-
-	case DLT_PPP_SERIAL:
-		{
-		// Get PPP protocol.
-		protocol = (data[2] << 8) + data[3];
-
-		if ( protocol == 0x0281 )
-			{
-			// MPLS Unicast. Remove the data link layer and
-			// denote a header size of zero before the IP header.
-			have_mpls = true;
-			data += get_link_header_size(datalink);
-			pkt_hdr_size = 0;
-			}
-
-		else if ( protocol != 0x0021 && protocol != 0x0057 )
-			{
-			// Neither IPv4 nor IPv6.
-			sessions->Weird("non_ip_packet_in_ppp_encapsulation", &hdr, data);
-			data = 0;
-			return;
-			}
-		break;
-		}
-	}
-
-	if ( have_mpls )
-		{
-		// Skip the MPLS label stack.
-		bool end_of_stack = false;
-
-		while ( ! end_of_stack )
-			{
-			end_of_stack = *(data + 2) & 0x01;
-			data += 4;
-			}
-		}
-
-	if ( pseudo_realtime )
-		{
-		current_pseudo = CheckPseudoTime();
-		net_packet_dispatch(current_pseudo, &hdr, data, pkt_hdr_size, this);
-		if ( ! first_wallclock )
-			first_wallclock = current_time(true);
-		}
-
-	else
-		net_packet_dispatch(current_timestamp, &hdr, data, pkt_hdr_size, this);
-
-	data = 0;
-	}
-
-bool PktSrc::GetCurrentPacket(const struct pcap_pkthdr** arg_hdr,
-				const u_char** arg_pkt)
-	{
-	if ( ! last_data )
-		return false;
-
-	*arg_hdr = &hdr;
-	*arg_pkt = last_data;
-	return true;
-	}
-
-int PktSrc::PrecompileFilter(int index, const char* filter)
-	{
-	// Compile filter.
-	BPF_Program* code = new BPF_Program();
-
-	if ( ! code->Compile(pd, filter, netmask, errbuf, sizeof(errbuf)) )
-		{
-		delete code;
-		return 0;
-		}
-
-	// Store it in hash.
-	HashKey* hash = new HashKey(HashKey(bro_int_t(index)));
-	BPF_Program* oldcode = filters.Lookup(hash);
-	if ( oldcode )
-		delete oldcode;
-
-	filters.Insert(hash, code);
-	delete hash;
-
-	return 1;
-	}
-
-int PktSrc::SetFilter(int index)
-	{
-	// We don't want load-level filters for the secondary path.
-	if ( filter_type == TYPE_FILTER_SECONDARY && index > 0 )
-		return 1;
-
-	HashKey* hash = new HashKey(HashKey(bro_int_t(index)));
-	BPF_Program* code = filters.Lookup(hash);
-	delete hash;
-
-	if ( ! code )
-		{
-		safe_snprintf(errbuf, sizeof(errbuf),
-			      "No precompiled pcap filter for index %d",
-			      index);
-		return 0;
-		}
-
-	if ( pcap_setfilter(pd, code->GetProgram()) < 0 )
-		{
-		safe_snprintf(errbuf, sizeof(errbuf),
-			      "pcap_setfilter(%d): %s",
-			      index, pcap_geterr(pd));
-		return 0;
-		}
-
-#ifndef HAVE_LINUX
-	// Linux doesn't clear counters when resetting filter.
-	stats.received = stats.dropped = stats.link = 0;
-#endif
-
-	return 1;
-	}
-
-void PktSrc::SetHdrSize()
-	{
-	int dl = pcap_datalink(pd);
-	hdr_size = get_link_header_size(dl);
-
-	if ( hdr_size < 0 )
-		{
-		safe_snprintf(errbuf, sizeof(errbuf),
-			 "unknown data link type 0x%x", dl);
-		Close();
-		}
-
-	datalink = dl;
-	}
-
-void PktSrc::Close()
-	{
-	if ( pd )
-		{
-		pcap_close(pd);
-		pd = 0;
-		closed = true;
-		}
-	}
-
-void PktSrc::AddSecondaryTablePrograms()
-	{
-	BPF_Program* program;
-
-	loop_over_list(secondary_path->EventTable(), i)
-		{
-		SecondaryEvent* se = secondary_path->EventTable()[i];
-		program = new BPF_Program();
-
-		if ( ! program->Compile(snaplen, datalink, se->Filter(),
-					netmask, errbuf, sizeof(errbuf)) )
-			{
-			delete program;
-			Close();
-			return;
-			}
-
-		SecondaryProgram* sp = new SecondaryProgram(program, se);
-		program_list.append(sp);
-		}
-	}
-
-void PktSrc::Statistics(Stats* s)
-	{
-	if ( reading_traces )
-		s->received = s->dropped = s->link = 0;
-
-	else
-		{
-		struct pcap_stat pstat;
-		if ( pcap_stats(pd, &pstat) < 0 )
-			{
-			reporter->Error("problem getting packet filter statistics: %s",
-					ErrorMsg());
-			s->received = s->dropped = s->link = 0;
-			}
-
-		else
-			{
-			s->dropped = pstat.ps_drop;
-			s->link = pstat.ps_recv;
-			}
-		}
-
-	s->received = stats.received;
-
-	if ( pseudo_realtime )
-		s->dropped = 0;
-
-	stats.dropped = s->dropped;
-	}
-
-PktInterfaceSrc::PktInterfaceSrc(const char* arg_interface, const char* filter,
-					PktSrc_Filter_Type ft)
-: PktSrc()
-	{
-	char tmp_errbuf[PCAP_ERRBUF_SIZE];
-	filter_type = ft;
-
-	// Determine interface if not specified.
-	if ( ! arg_interface && ! (arg_interface = pcap_lookupdev(tmp_errbuf)) )
-		{
-		safe_snprintf(errbuf, sizeof(errbuf),
-			 "pcap_lookupdev: %s", tmp_errbuf);
-		return;
-		}
-
-	interface = copy_string(arg_interface);
-
-	// Determine network and netmask.
-	uint32 net;
-	if ( pcap_lookupnet(interface, &net, &netmask, tmp_errbuf) < 0 )
-		{
-		// ### The lookup can fail if no address is assigned to
-		// the interface; and libpcap doesn't have any useful notion
-		// of error codes, just error strings - how bogus - so we
-		// just kludge around the error :-(.
-		// sprintf(errbuf, "pcap_lookupnet %s", tmp_errbuf);
-		// return;
-		net = 0;
-		netmask = 0xffffff00;
-		}
-
-	// We use the smallest time-out possible to return almost immediately if
-	// no packets are available. (We can't use set_nonblocking() as it's
-	// broken on FreeBSD: even when select() indicates that we can read
-	// something, we may get nothing if the store buffer hasn't filled up
-	// yet.)
-	pd = pcap_open_live(interface, snaplen, 1, 1, tmp_errbuf);
-
-	if ( ! pd )
-		{
-		safe_snprintf(errbuf, sizeof(errbuf),
-			 "pcap_open_live: %s", tmp_errbuf);
-		closed = true;
-		return;
-		}
-
-	// ### This needs autoconf'ing.
-#ifdef HAVE_PCAP_INT_H
-	reporter->Info("pcap bufsize = %d\n", ((struct pcap *) pd)->bufsize);
-#endif
-
-#ifdef HAVE_LINUX
-	if ( pcap_setnonblock(pd, 1, tmp_errbuf) < 0 )
-		{
-		safe_snprintf(errbuf, sizeof(errbuf),
-			 "pcap_setnonblock: %s", tmp_errbuf);
-		pcap_close(pd);
-		closed = true;
-		return;
-		}
-#endif
-	selectable_fd = pcap_fileno(pd);
-
-	if ( PrecompileFilter(0, filter) && SetFilter(0) )
-		{
-		SetHdrSize();
-
-		if ( closed )
-			// Couldn't get header size.
-			return;
-
-		reporter->Info("listening on %s, capture length %d bytes\n", interface, snaplen);
-		}
-	else
-		closed = true;
-	}
-
-
-PktFileSrc::PktFileSrc(const char* arg_readfile, const char* filter,
-			PktSrc_Filter_Type ft)
-: PktSrc()
-	{
-	readfile = copy_string(arg_readfile);
-
-	filter_type = ft;
-
-	pd = pcap_open_offline((char*) readfile, errbuf);
-
-	if ( pd && PrecompileFilter(0, filter) && SetFilter(0) )
-		{
-		SetHdrSize();
-
-		if ( closed )
-			// Unknown link layer type.
-			return;
-
-		// We don't put file sources into non-blocking mode as
-		// otherwise we would not be able to identify the EOF.
-
-		selectable_fd = fileno(pcap_file(pd));
-
-		if ( selectable_fd < 0 )
-			reporter->InternalError("OS does not support selectable pcap fd");
-		}
-	else
-		closed = true;
-	}
-
-
-SecondaryPath::SecondaryPath()
-	{
-	filter = 0;
-
-	// Glue together the secondary filter, if exists.
-	Val* secondary_fv = internal_val("secondary_filters");
-	if ( secondary_fv->AsTableVal()->Size() == 0 )
-		return;
-
-	int did_first = 0;
-	const TableEntryValPDict* v = secondary_fv->AsTable();
-	IterCookie* c = v->InitForIteration();
-	TableEntryVal* tv;
-	HashKey* h;
-
-	while ( (tv = v->NextEntry(h, c)) )
-		{
-		// Get the index values.
-		ListVal* index =
-			secondary_fv->AsTableVal()->RecoverIndex(h);
-
-		const char* str =
-			index->Index(0)->Ref()->AsString()->CheckString();
-
-		if ( ++did_first == 1 )
-			{
-			filter = copy_string(str);
-			}
-		else
-			{
-			if ( strlen(filter) > 0 )
-				{
-				char* tmp_f = new char[strlen(str) + strlen(filter) + 32];
-				if ( strlen(str) == 0 )
-					sprintf(tmp_f, "%s", filter);
-				else
-					sprintf(tmp_f, "(%s) or (%s)", filter, str);
-				delete [] filter;
-				filter = tmp_f;
-				}
-			}
-
-		// Build secondary_path event table item and link it.
-		SecondaryEvent* se =
-			new SecondaryEvent(index->Index(0)->Ref()->AsString()->CheckString(),
-				tv->Value()->AsFunc() );
-
-		event_list.append(se);
-
-		delete h;
-		Unref(index);
-		}
-	}
-
-SecondaryPath::~SecondaryPath()
-	{
-	loop_over_list(event_list, i)
-		delete event_list[i];
-
-	delete [] filter;
-	}
-
-
-SecondaryProgram::~SecondaryProgram()
-	{
-	delete program;
-	}
-
-PktDumper::PktDumper(const char* arg_filename, bool arg_append)
-	{
-	filename[0] = '\0';
-	is_error = false;
-	append = arg_append;
-	dumper = 0;
-	open_time = 0.0;
-
-	// We need a pcap_t with a reasonable link-layer type. We try to get it
-	// from the packet sources. If not available, we fall back to Ethernet.
-	// FIXME: Perhaps we should make this configurable?
-	int linktype = -1;
-
-	if ( pkt_srcs.length() )
-		linktype = pkt_srcs[0]->LinkType();
-
-	if ( linktype < 0 )
-		linktype = DLT_EN10MB;
-
-	pd = pcap_open_dead(linktype, snaplen);
-	if ( ! pd )
-		{
-		Error("error for pcap_open_dead");
-		return;
-		}
-
-	if ( arg_filename )
-		Open(arg_filename);
-	}
-
-bool PktDumper::Open(const char* arg_filename)
-	{
-	if ( ! arg_filename && ! *filename )
-		{
-		Error("no filename given");
-		return false;
-		}
-
-	if ( arg_filename )
-		{
-		if ( dumper && streq(arg_filename, filename) )
-			// Already open.
-			return true;
-
-		safe_strncpy(filename, arg_filename, FNBUF_LEN);
-		}
-
-	if ( dumper )
-		Close();
-
-	struct stat s;
-	int exists = 0;
-
-	if ( append )
-		{
-		// See if output file already exists (and is non-empty).
-		exists = stat(filename, &s); ;
-
-		if ( exists < 0 && errno != ENOENT )
-			{
-			Error(fmt("can't stat file %s: %s", filename, strerror(errno)));
-			return false;
-			}
-		}
-
-	if ( ! append || exists < 0 || s.st_size == 0 )
-		{
-		// Open new file.
-		dumper =  pcap_dump_open(pd, filename);
-		if ( ! dumper )
-			{
-			Error(pcap_geterr(pd));
-			return false;
-			}
-		}
-
-	else
-		{
-		// Old file and we need to append, which, unfortunately,
-		// is not supported by libpcap. So, we have to hack a
-		// little bit, knowing that pcap_dumpter_t is, in fact,
-		// a FILE ... :-(
-		dumper = (pcap_dumper_t*) fopen(filename, "a");
-		if ( ! dumper )
-			{
-			Error(fmt("can't open dump %s: %s", filename, strerror(errno)));
-			return false;
-			}
-		}
-
-	open_time = network_time;
-	is_error = false;
-	return true;
-	}
-
-bool PktDumper::Close()
-	{
-	if ( dumper )
-		{
-		pcap_dump_close(dumper);
-		dumper = 0;
-		is_error = false;
-		}
-
-	return true;
-	}
-
-bool PktDumper::Dump(const struct pcap_pkthdr* hdr, const u_char* pkt)
-	{
-	if ( ! dumper )
-		return false;
-
-	if ( ! open_time )
-		open_time = network_time;
-
-	pcap_dump((u_char*) dumper, hdr, pkt);
-
-	return true;
-	}
-
-void PktDumper::Error(const char* errstr)
-	{
-	safe_strncpy(errbuf, errstr, sizeof(errbuf));
-	is_error = true;
-	}
-
-int get_link_header_size(int dl)
-	{
-	switch ( dl ) {
-	case DLT_NULL:
-		return 4;
-
-	case DLT_EN10MB:
-		return 14;
-
-	case DLT_FDDI:
-		return 13 + 8;	// fddi_header + LLC
-
-#ifdef DLT_LINUX_SLL
-	case DLT_LINUX_SLL:
-		return 16;
-#endif
-
-	case DLT_PPP_SERIAL:	// PPP_SERIAL
-		return 4;
-
-	case DLT_RAW:
-		return 0;
-	}
-
-	return -1;
-	}

src/PktSrc.h

@@ -1,258 +0,0 @@
-// See the file "COPYING" in the main distribution directory for copyright.
-
-#ifndef pktsrc_h
-#define pktsrc_h
-
-#include "Dict.h"
-#include "Expr.h"
-#include "BPF_Program.h"
-#include "IOSource.h"
-#include "RemoteSerializer.h"
-
-#define BRO_PCAP_ERRBUF_SIZE   PCAP_ERRBUF_SIZE + 256
-
-extern "C" {
-#include <pcap.h>
-}
-
-declare(PDict,BPF_Program);
-
-// Whether a PktSrc object is used by the normal filter structure or the
-// secondary-path structure.
-typedef enum {
-	TYPE_FILTER_NORMAL,  // the normal filter
-	TYPE_FILTER_SECONDARY,  // the secondary-path filter
-} PktSrc_Filter_Type;
-
-
-// {filter,event} tuples conforming the secondary path.
-class SecondaryEvent {
-public:
-	SecondaryEvent(const char* arg_filter, Func* arg_event)
-		{
-		filter = arg_filter;
-		event = arg_event;
-		}
-
-	const char* Filter()	{ return filter; }
-	Func* Event()		{ return event; }
-
-private:
-	const char* filter;
-	Func* event;
-};
-
-declare(PList,SecondaryEvent);
-typedef PList(SecondaryEvent) secondary_event_list;
-
-
-
-class SecondaryPath {
-public:
-	SecondaryPath();
-	~SecondaryPath();
-
-	secondary_event_list& EventTable()	{ return event_list; }
-	const char* Filter()			{ return filter; }
-
-private:
-	secondary_event_list event_list;
-	// OR'ed union of all SecondaryEvent filters
-	char* filter;
-};
-
-// Main secondary-path object.
-extern SecondaryPath* secondary_path;
-
-
-// {program, {filter,event}} tuple table.
-class SecondaryProgram {
-public:
-	SecondaryProgram(BPF_Program* arg_program, SecondaryEvent* arg_event)
-		{
-		program = arg_program;
-		event = arg_event;
-		}
-
-	~SecondaryProgram();
-
-	BPF_Program* Program()  { return program; }
-	SecondaryEvent* Event()	{ return event; }
-
-private:
-	// Associated program.
-	BPF_Program *program;
-
-	// Event that is run in case the program is matched.
-	SecondaryEvent* event;
-};
-
-declare(PList,SecondaryProgram);
-typedef PList(SecondaryProgram) secondary_program_list;
-
-
-
-class PktSrc : public IOSource {
-public:
-	~PktSrc();
-
-	// IOSource interface
-	bool IsReady();
-	void GetFds(int* read, int* write, int* except);
-	double NextTimestamp(double* local_network_time);
-	void Process();
-	const char* Tag()	{ return "PktSrc"; }
-
-	const char* ErrorMsg() const	{ return errbuf; }
-	void ClearErrorMsg()		{ *errbuf ='\0'; }
-
-	// Returns the packet last processed; false if there is no
-	// current packet available.
-	bool GetCurrentPacket(const pcap_pkthdr** hdr, const u_char** pkt);
-
-	int HdrSize() const		{ return hdr_size; }
-	int DataLink() const		{ return datalink; }
-
-	void ConsumePacket()	{ data = 0; }
-
-	int IsLive() const		{ return interface != 0; }
-
-	pcap_t* PcapHandle() const	{ return pd; }
-	int LinkType() const		{ return pcap_datalink(pd); }
-
-	const char* ReadFile() const	{ return readfile; }
-	const char* Interface() const	{ return interface; }
-	PktSrc_Filter_Type FilterType() const	{ return filter_type; }
-	void AddSecondaryTablePrograms();
-	const secondary_program_list& ProgramTable() const
-		{ return program_list; }
-
-	// Signal packet source that processing was suspended and is now going
-	// to be continued.
-	void ContinueAfterSuspend();
-
-	// Only valid in pseudo-realtime mode.
-	double CurrentPacketTimestamp()	{ return current_pseudo; }
-	double CurrentPacketWallClock();
-
-	struct Stats {
-		unsigned int received;	// pkts received (w/o drops)
-		unsigned int dropped;	// pkts dropped
-		unsigned int link;	// total packets on link
-					// (not always not available)
-	};
-
-	virtual void Statistics(Stats* stats);
-
-	// Precompiles a filter and associates the given index with it.
-	// Returns true on success, 0 if a problem occurred.
-	virtual int PrecompileFilter(int index, const char* filter);
-
-	// Activates the filter with the given index.
-	// Returns true on success, 0 if a problem occurred.
-	virtual int SetFilter(int index);
-
-protected:
-	PktSrc();
-
-	static const int PCAP_TIMEOUT = 20;
-
-	void SetHdrSize();
-
-	virtual void Close();
-
-	// Returns 1 on success, 0 on time-out/gone dry.
-	virtual int ExtractNextPacket();
-
-	// Checks if the current packet has a pseudo-time <= current_time.
-	// If yes, returns pseudo-time, otherwise 0.
-	double CheckPseudoTime();
-
-	double current_timestamp;
-	double next_timestamp;
-
-	// Only set in pseudo-realtime mode.
-	double first_timestamp;
-	double first_wallclock;
-	double current_wallclock;
-	double current_pseudo;
-
-	struct pcap_pkthdr hdr;
-	const u_char* data;	// contents of current packet
-	const u_char* last_data;	// same, but unaffected by consuming
-	int hdr_size;
-	int datalink;
-	double next_sync_point; // For trace synchronziation in pseudo-realtime
-
-	char* interface;	// nil if not reading from an interface
-	char* readfile;		// nil if not reading from a file
-
-	pcap_t* pd;
-	int selectable_fd;
-	uint32 netmask;
-	char errbuf[BRO_PCAP_ERRBUF_SIZE];
-
-	Stats stats;
-
-	PDict(BPF_Program) filters; // precompiled filters
-
-	PktSrc_Filter_Type filter_type; // normal path or secondary path
-	secondary_program_list program_list;
-};
-
-class PktInterfaceSrc : public PktSrc {
-public:
-	PktInterfaceSrc(const char* interface, const char* filter,
-			PktSrc_Filter_Type ft=TYPE_FILTER_NORMAL);
-};
-
-class PktFileSrc : public PktSrc {
-public:
-	PktFileSrc(const char* readfile, const char* filter,
-			PktSrc_Filter_Type ft=TYPE_FILTER_NORMAL);
-};
-
-
-extern int get_link_header_size(int dl);
-
-class PktDumper {
-public:
-	PktDumper(const char* file = 0, bool append = false);
-	~PktDumper()	{ Close(); }
-
-	bool Open(const char* file = 0);
-	bool Close();
-	bool Dump(const struct pcap_pkthdr* hdr, const u_char* pkt);
-
-	pcap_dumper_t* PcapDumper() 	{ return dumper; }
-
-	const char* FileName() const	{ return filename; }
-	bool IsError() const		{ return is_error; }
-	const char* ErrorMsg() const	{ return errbuf; }
-
-	// This heuristic will horribly fail if we're using packets
-	// with different link layers.  (If we can't derive a reasonable value
-	// from the packet sources, our fall-back is Ethernet.)
-	int HdrSize() const
-		{ return get_link_header_size(pcap_datalink(pd)); }
-
-	// Network time when dump file was opened.
-	double OpenTime() const		{ return open_time; }
-
-private:
-	void InitPd();
-	void Error(const char* str);
-
-	static const int FNBUF_LEN = 1024;
-	char filename[FNBUF_LEN];
-
-	bool append;
-	pcap_dumper_t* dumper;
-	pcap_t* pd;
-	double open_time;
-
-	bool is_error;
-	char errbuf[BRO_PCAP_ERRBUF_SIZE];
-};
-
-#endif

src/RemoteSerializer.cc

@@ -188,10 +188,10 @@
 #include "File.h"
 #include "Conn.h"
 #include "Reporter.h"
-#include "threading/SerialTypes.h"
-#include "logging/Manager.h"
 #include "IPAddr.h"
 #include "bro_inet_ntop.h"
+#include "iosource/Manager.h"
+#include "logging/Manager.h"
 #include "logging/logging.bif.h"
 
 extern "C" {
@@ -285,10 +285,10 @@ struct ping_args {
 	\
 	if ( ! c ) \
 		{ \
-		idle = io->IsIdle();\
+		SetIdle(io->IsIdle());\
 		return true; \
 		} \
-	idle = false; \
+	SetIdle(false); \
 	}
 
 static const char* msgToStr(int msg)
@@ -534,7 +534,6 @@ RemoteSerializer::RemoteSerializer()
 	current_sync_point = 0;
 	syncing_times = false;
 	io = 0;
-	closed = false;
 	terminating = false;
 	in_sync = 0;
 	last_flush = 0;
@@ -559,7 +558,7 @@ RemoteSerializer::~RemoteSerializer()
 	delete io;
 	}
 
-void RemoteSerializer::Init()
+void RemoteSerializer::Enable()
 	{
 	if ( initialized )
 		return;
@@ -572,7 +571,7 @@ void RemoteSerializer::Init()
 
 	Fork();
 
-	io_sources.Register(this);
+	iosource_mgr->Register(this);
 
 	Log(LogInfo, fmt("communication started, parent pid is %d, child pid is %d", getpid(), child_pid));
 	initialized = 1;
@@ -1276,7 +1275,7 @@ bool RemoteSerializer::Listen(const IPAddr& ip, uint16 port, bool expect_ssl,
 		return false;
 
 	listening = true;
-	closed = false;
+	SetClosed(false);
 	return true;
 	}
 
@@ -1345,7 +1344,7 @@ bool RemoteSerializer::StopListening()
 		return false;
 
 	listening = false;
-	closed = ! IsActive();
+	SetClosed(! IsActive());
 	return true;
 	}
 
@@ -1368,12 +1367,14 @@ void RemoteSerializer::Unregister(ID* id)
 			}
 	}
 
-void RemoteSerializer::GetFds(int* read, int* write, int* except)
+void RemoteSerializer::GetFds(iosource::FD_Set* read, iosource::FD_Set* write,
+                              iosource::FD_Set* except)
 	{
-	*read = io->Fd();
+	read->Insert(io->Fd());
+	read->Insert(io->ExtraReadFDs());
 
 	if ( io->CanWrite() )
-		*write = io->Fd();
+		write->Insert(io->Fd());
 	}
 
 double RemoteSerializer::NextTimestamp(double* local_network_time)
@@ -1383,7 +1384,7 @@ double RemoteSerializer::NextTimestamp(double* local_network_time)
 	if ( received_logs > 0 )
 		{
 		// If we processed logs last time, assume there's more.
-		idle = false;
+		SetIdle(false);
 		received_logs = 0;
 		return timer_mgr->Time();
 		}
@@ -1398,7 +1399,7 @@ double RemoteSerializer::NextTimestamp(double* local_network_time)
 		pt = timer_mgr->Time();
 
 	if ( packets.length() )
-		idle = false;
+		SetIdle(false);
 
 	if ( et >= 0 && (et < pt || pt < 0) )
 		return et;
@@ -1477,7 +1478,7 @@ void RemoteSerializer::Process()
 		}
 
 	if ( packets.length() )
-		idle = false;
+		SetIdle(false);
 	}
 
 void RemoteSerializer::Finish()
@@ -1509,7 +1510,7 @@ bool RemoteSerializer::Poll(bool may_block)
 		}
 
 	io->Flush();
-	idle = false;
+	SetIdle(false);
 
 	switch ( msgstate ) {
 	case TYPE:
@@ -1691,7 +1692,7 @@ bool RemoteSerializer::DoMessage()
 
 	case MSG_TERMINATE:
 		assert(terminating);
-		io_sources.Terminate();
+		iosource_mgr->Terminate();
 		return true;
 
 	case MSG_REMOTE_PRINT:
@@ -1879,7 +1880,7 @@ void RemoteSerializer::RemovePeer(Peer* peer)
 	delete peer->cache_out;
 	delete peer;
 
-	closed = ! IsActive();
+	SetClosed(! IsActive());
 
 	if ( in_sync == peer )
 		in_sync = 0;
@@ -2841,7 +2842,7 @@ void RemoteSerializer::GotEvent(const char* name, double time,
 	BufferedEvent* e = new BufferedEvent;
 
 	// Our time, not the time when the event was generated.
-	e->time = pkt_srcs.length() ?
+	e->time = iosource_mgr->GetPktSrcs().size() ?
 			time_t(network_time) : time_t(timer_mgr->Time());
 
 	e->src = current_peer->id;
@@ -3086,7 +3087,7 @@ RecordVal* RemoteSerializer::GetPeerVal(PeerID id)
 void RemoteSerializer::ChildDied()
 	{
 	Log(LogError, "child died");
-	closed = true;
+	SetClosed(true);
 	child_pid = 0;
 
 	// Shut down the main process as well.
@@ -3185,7 +3186,7 @@ void RemoteSerializer::FatalError(const char* msg)
 	Log(LogError, msg);
 	reporter->Error("%s", msg);
 
-	closed = true;
+	SetClosed(true);
 
 	if ( kill(child_pid, SIGQUIT) < 0 )
 		reporter->Warning("warning: cannot kill child pid %d, %s", child_pid, strerror(errno));
@@ -3356,6 +3357,15 @@ SocketComm::~SocketComm()
 
 static unsigned int first_rtime = 0;
 
+static void fd_vector_set(const std::vector<int>& fds, fd_set* set, int* max)
+	{
+	for ( size_t i = 0; i < fds.size(); ++i )
+		{
+		FD_SET(fds[i], set);
+		*max = ::max(fds[i], *max);
+		}
+	}
+
 void SocketComm::Run()
 	{
 	first_rtime = (unsigned int) current_time(true);
@@ -3377,10 +3387,9 @@ void SocketComm::Run()
 		FD_ZERO(&fd_write);
 		FD_ZERO(&fd_except);
 
-		int max_fd = 0;
-
+		int max_fd = io->Fd();
 		FD_SET(io->Fd(), &fd_read);
-		max_fd = io->Fd();
+		max_fd = std::max(max_fd, io->ExtraReadFDs().Set(&fd_read));
 
 		loop_over_list(peers, i)
 			{
@@ -3389,6 +3398,8 @@ void SocketComm::Run()
 				FD_SET(peers[i]->io->Fd(), &fd_read);
 				if ( peers[i]->io->Fd() > max_fd )
 					max_fd = peers[i]->io->Fd();
+				max_fd = std::max(max_fd,
+				                  peers[i]->io->ExtraReadFDs().Set(&fd_read));
 				}
 			else
 				{
@@ -3439,38 +3450,17 @@ void SocketComm::Run()
 		if ( ! io->IsFillingUp() && shutting_conns_down )
 			shutting_conns_down = false;
 
-		// We cannot rely solely on select() as the there may
-		// be some data left in our input/output queues. So, we use
-		// a small timeout for select and check for data
-		// manually afterwards.
-
 		static long selects = 0;
 		static long canwrites = 0;
-		static long timeouts = 0;
 
 		++selects;
 		if ( io->CanWrite() )
 			++canwrites;
 
-		// FIXME: Fine-tune this (timeouts, flush, etc.)
-		struct timeval small_timeout;
-		small_timeout.tv_sec = 0;
-		small_timeout.tv_usec =
-			io->CanWrite() || io->CanRead() ? 1 : 10;
-
-#if 0
-		if ( ! io->CanWrite() )
-			usleep(10);
-#endif
-
-		int a = select(max_fd + 1, &fd_read, &fd_write, &fd_except,
-				&small_timeout);
-
-		if ( a == 0 )
-			++timeouts;
+		int a = select(max_fd + 1, &fd_read, &fd_write, &fd_except, 0);
 
 		if ( selects % 100000 == 0 )
-			Log(fmt("selects=%ld canwrites=%ld timeouts=%ld", selects, canwrites, timeouts));
+			Log(fmt("selects=%ld canwrites=%ld", selects, canwrites));
 
 		if ( a < 0 )
 			// Ignore errors for now.
@@ -4212,6 +4202,7 @@ bool SocketComm::Listen()
 				safe_close(fd);
 				CloseListenFDs();
 				listen_next_try = time(0) + bind_retry_interval;
+				freeaddrinfo(res0);
 				return false;
 				}
 

src/RemoteSerializer.h

@@ -6,7 +6,7 @@
 #include "Dict.h"
 #include "List.h"
 #include "Serializer.h"
-#include "IOSource.h"
+#include "iosource/IOSource.h"
 #include "Stats.h"
 #include "File.h"
 #include "logging/WriterBackend.h"
@@ -22,13 +22,13 @@ namespace threading {
 }
 
 // This class handles the communication done in Bro's main loop.
-class RemoteSerializer : public Serializer, public IOSource {
+class RemoteSerializer : public Serializer, public iosource::IOSource {
 public:
 	RemoteSerializer();
 	virtual ~RemoteSerializer();
 
 	// Initialize the remote serializer (calling this will fork).
-	void Init();
+	void Enable();
 
 	// FIXME: Use SourceID directly (or rename everything to Peer*).
 	typedef SourceID PeerID;
@@ -140,7 +140,8 @@ public:
 	void Finish();
 
 	// Overidden from IOSource:
-	virtual void GetFds(int* read, int* write, int* except);
+	virtual void GetFds(iosource::FD_Set* read, iosource::FD_Set* write,
+	                    iosource::FD_Set* except);
 	virtual double NextTimestamp(double* local_network_time);
 	virtual void Process();
 	virtual TimerMgr::Tag* GetCurrentTag();

src/Serializer.cc

@@ -19,6 +19,7 @@
 #include "Conn.h"
 #include "Timer.h"
 #include "RemoteSerializer.h"
+#include "iosource/Manager.h"
 
 Serializer::Serializer(SerializationFormat* arg_format)
 	{
@@ -1045,7 +1046,7 @@ EventPlayer::EventPlayer(const char* file)
 		Error(fmt("event replayer: cannot open %s", file));
 
 	if ( ReadHeader() )
-		io_sources.Register(this);
+		iosource_mgr->Register(this);
 	}
 
 EventPlayer::~EventPlayer()
@@ -1067,9 +1068,10 @@ void EventPlayer::GotFunctionCall(const char* name, double time,
 	// We don't replay function calls.
 	}
 
-void EventPlayer::GetFds(int* read, int* write, int* except)
+void EventPlayer::GetFds(iosource::FD_Set* read, iosource::FD_Set* write,
+                         iosource::FD_Set* except)
 	{
-	*read = fd;
+	read->Insert(fd);
 	}
 
 double EventPlayer::NextTimestamp(double* local_network_time)
@@ -1085,7 +1087,7 @@ double EventPlayer::NextTimestamp(double* local_network_time)
 		{
 		UnserialInfo info(this);
 		Unserialize(&info);
-		closed = io->Eof();
+		SetClosed(io->Eof());
 		}
 
 	if ( ! ne_time )
@@ -1142,7 +1144,7 @@ bool Packet::Serialize(SerialInfo* info) const
 static BroFile* profiling_output = 0;
 
 #ifdef DEBUG
-static PktDumper* dump = 0;
+static iosource::PktDumper* dump = 0;
 #endif
 
 Packet* Packet::Unserialize(UnserialInfo* info)
@@ -1188,7 +1190,7 @@ Packet* Packet::Unserialize(UnserialInfo* info)
 	p->hdr = hdr;
 	p->pkt = (u_char*) pkt;
 	p->tag = tag;
-	p->hdr_size = get_link_header_size(p->link_type);
+	p->hdr_size = iosource::PktSrc::GetLinkHeaderSize(p->link_type);
 
 	delete [] tag;
 
@@ -1213,9 +1215,15 @@ Packet* Packet::Unserialize(UnserialInfo* info)
 	if ( debug_logger.IsEnabled(DBG_TM) )
 		{
 		if ( ! dump )
-			dump = new PktDumper("tm.pcap");
+			dump = iosource_mgr->OpenPktDumper("tm.pcap", true);
 
-		dump->Dump(p->hdr, p->pkt);
+		if ( dump )
+			{
+			iosource::PktDumper::Packet dp;
+			dp.hdr = p->hdr;
+			dp.data = p->pkt;
+			dump->Dump(&dp);
+			}
 		}
 #endif
 

src/Serializer.h

@@ -15,7 +15,7 @@
 #include "SerialInfo.h"
 #include "IP.h"
 #include "Timer.h"
-#include "IOSource.h"
+#include "iosource/IOSource.h"
 #include "Reporter.h"
 
 class SerializationCache;
@@ -350,12 +350,13 @@ public:
 };
 
 // Plays a file of events back.
-class EventPlayer : public FileSerializer, public IOSource {
+class EventPlayer : public FileSerializer, public iosource::IOSource {
 public:
 	EventPlayer(const char* file);
 	virtual ~EventPlayer();
 
-	virtual void GetFds(int* read, int* write, int* except);
+	virtual void GetFds(iosource::FD_Set* read, iosource::FD_Set* write,
+	                    iosource::FD_Set* except);
 	virtual double NextTimestamp(double* local_network_time);
 	virtual void Process();
 	virtual const char* Tag()	{ return "EventPlayer"; }

src/Sessions.cc

@@ -167,7 +167,7 @@ void NetSessions::Done()
 
 void NetSessions::DispatchPacket(double t, const struct pcap_pkthdr* hdr,
 			const u_char* pkt, int hdr_size,
-			PktSrc* src_ps)
+			iosource::PktSrc* src_ps)
 	{
 	const struct ip* ip_hdr = 0;
 	const u_char* ip_data = 0;
@@ -184,10 +184,7 @@ void NetSessions::DispatchPacket(double t, const struct pcap_pkthdr* hdr,
 		// Blanket encapsulation
 		hdr_size += encap_hdr_size;
 
-	if ( src_ps->FilterType() == TYPE_FILTER_NORMAL )
-		NextPacket(t, hdr, pkt, hdr_size);
-	else
-		NextPacketSecondary(t, hdr, pkt, hdr_size, src_ps);
+	NextPacket(t, hdr, pkt, hdr_size);
 	}
 
 void NetSessions::NextPacket(double t, const struct pcap_pkthdr* hdr,
@@ -262,53 +259,6 @@ void NetSessions::NextPacket(double t, const struct pcap_pkthdr* hdr,
 		DumpPacket(hdr, pkt);
 	}
 
-void NetSessions::NextPacketSecondary(double /* t */, const struct pcap_pkthdr* hdr,
-				const u_char* const pkt, int hdr_size,
-				const PktSrc* src_ps)
-	{
-	SegmentProfiler(segment_logger, "processing-secondary-packet");
-
-	++num_packets_processed;
-
-	uint32 caplen = hdr->caplen - hdr_size;
-	if ( caplen < sizeof(struct ip) )
-		{
-		Weird("truncated_IP", hdr, pkt);
-		return;
-		}
-
-	const struct ip* ip = (const struct ip*) (pkt + hdr_size);
-	if ( ip->ip_v == 4 )
-		{
-		const secondary_program_list& spt = src_ps->ProgramTable();
-
-		loop_over_list(spt, i)
-			{
-			SecondaryProgram* sp = spt[i];
-			if ( ! net_packet_match(sp->Program(), pkt,
-						hdr->len, hdr->caplen) )
-				continue;
-
-			val_list* args = new val_list;
-			StringVal* cmd_val =
-				new StringVal(sp->Event()->Filter());
-			args->append(cmd_val);
-			IP_Hdr ip_hdr(ip, false);
-			args->append(ip_hdr.BuildPktHdrVal());
-			// ### Need to queue event here.
-			try
-				{
-				sp->Event()->Event()->Call(args);
-				}
-
-			catch ( InterpreterException& e )
-				{ /* Already reported. */ }
-
-			delete args;
-			}
-		}
-	}
-
 int NetSessions::CheckConnectionTag(Connection* conn)
 	{
 	if ( current_iosrc->GetCurrentTag() )
@@ -1440,14 +1390,24 @@ void NetSessions::DumpPacket(const struct pcap_pkthdr* hdr,
 		return;
 
 	if ( len == 0 )
-		pkt_dumper->Dump(hdr, pkt);
+		{
+		iosource::PktDumper::Packet p;
+		p.hdr = hdr;
+		p.data = pkt;
+		pkt_dumper->Dump(&p);
+		}
+
 	else
 		{
 		struct pcap_pkthdr h = *hdr;
 		h.caplen = len;
 		if ( h.caplen > hdr->caplen )
 			reporter->InternalError("bad modified caplen");
-		pkt_dumper->Dump(&h, pkt);
+
+		iosource::PktDumper::Packet p;
+		p.hdr = &h;
+		p.data = pkt;
+		pkt_dumper->Dump(&p);
 		}
 	}
 

src/Sessions.h

@@ -69,11 +69,11 @@ public:
 	~NetSessions();
 
 	// Main entry point for packet processing. Dispatches the packet
-	// either through NextPacket() or NextPacketSecondary(), optionally
-	// employing the packet sorter first.
+	// either through NextPacket(), optionally employing the packet
+	// sorter first.
 	void DispatchPacket(double t, const struct pcap_pkthdr* hdr,
 			const u_char* const pkt, int hdr_size,
-			PktSrc* src_ps);
+			iosource::PktSrc* src_ps);
 
 	void Done();	// call to drain events before destructing
 
@@ -221,10 +221,6 @@ protected:
 	void NextPacket(double t, const struct pcap_pkthdr* hdr,
 			const u_char* const pkt, int hdr_size);
 
-	void NextPacketSecondary(double t, const struct pcap_pkthdr* hdr,
-			const u_char* const pkt, int hdr_size,
-			const PktSrc* src_ps);
-
 	// Record the given packet (if a dumper is active).  If len=0
 	// then the whole packet is recorded, otherwise just the first
 	// len bytes.

src/Type.cc

@@ -1381,6 +1381,11 @@ void OpaqueType::Describe(ODesc* d) const
 	d->Add(name.c_str());
 	}
 
+void OpaqueType::DescribeReST(ODesc* d, bool roles_only) const
+	{
+	d->Add(fmt(":bro:type:`%s` of %s", type_name(Tag()), name.c_str()));
+	}
+
 IMPLEMENT_SERIAL(OpaqueType, SER_OPAQUE_TYPE);
 
 bool OpaqueType::DoSerialize(SerialInfo* info) const

src/Type.h

@@ -534,6 +534,7 @@ public:
 	const string& Name() const { return name; }
 
 	void Describe(ODesc* d) const;
+	void DescribeReST(ODesc* d, bool roles_only = false) const;
 
 protected:
 	OpaqueType() { }

src/Val.cc

@@ -465,10 +465,7 @@ void Val::Describe(ODesc* d) const
 		d->SP();
 		}
 
-	if ( d->IsReadable() )
-		ValDescribe(d);
-	else
-		Val::ValDescribe(d);
+	ValDescribe(d);
 	}
 
 void Val::DescribeReST(ODesc* d) const

src/analyzer/Analyzer.cc

@@ -4,6 +4,7 @@
 
 #include "Analyzer.h"
 #include "Manager.h"
+#include "binpac.h"
 
 #include "analyzer/protocol/pia/PIA.h"
 #include "../Event.h"

src/analyzer/protocol/dhcp/dhcp-analyzer.pac

@@ -188,6 +188,9 @@ flow DHCP_Flow(is_orig: bool) {
 		switch ( type )
 			{
 			case DHCPOFFER:
+				if ( ! router_list )
+					router_list = new TableVal(dhcp_router_list);
+
 				BifEvent::generate_dhcp_offer(connection()->bro_analyzer(),
 							      connection()->bro_analyzer()->Conn(),
 							      dhcp_msg_val_->Ref(), new AddrVal(subnet_mask),
@@ -195,6 +198,9 @@ flow DHCP_Flow(is_orig: bool) {
 				break;
 
 			case DHCPACK:
+				if ( ! router_list )
+					router_list = new TableVal(dhcp_router_list);
+
 				BifEvent::generate_dhcp_ack(connection()->bro_analyzer(),
 							    connection()->bro_analyzer()->Conn(),
 							    dhcp_msg_val_->Ref(), new AddrVal(subnet_mask),
@@ -202,12 +208,14 @@ flow DHCP_Flow(is_orig: bool) {
 				break;
 
 			case DHCPNAK:
+				Unref(router_list);
 				BifEvent::generate_dhcp_nak(connection()->bro_analyzer(),
 							    connection()->bro_analyzer()->Conn(),
 							    dhcp_msg_val_->Ref(), host_name);
 				break;
 
 			default:
+				Unref(router_list);
 				Unref(host_name);
 				break;
 			}

src/analyzer/protocol/dhcp/events.bif

@@ -18,8 +18,8 @@
 ##
 event dhcp_discover%(c: connection, msg: dhcp_msg, req_addr: addr, host_name: string%);
 
-## Generated for DHCP messages of type *DHCPOFFER* (server to client in response to
-## DHCPDISCOVER with offer of configuration parameters).
+## Generated for DHCP messages of type *DHCPOFFER* (server to client in response
+## to DHCPDISCOVER with offer of configuration parameters).
 ##
 ## c: The connection record describing the underlying UDP flow.
 ##
@@ -34,7 +34,7 @@ event dhcp_discover%(c: connection, msg: dhcp_msg, req_addr: addr, host_name: st
 ## serv_addr: The server address specified by the message.
 ##
 ## host_name: Optional host name value. May differ from the host name requested
-## from the client. 
+##            from the client.
 ##
 ## .. bro:see:: dhcp_discover dhcp_request dhcp_decline dhcp_ack dhcp_nak
 ##              dhcp_release dhcp_inform
@@ -103,7 +103,7 @@ event dhcp_decline%(c: connection, msg: dhcp_msg, host_name: string%);
 ## serv_addr: The server address specified by the message.
 ##
 ## host_name: Optional host name value. May differ from the host name requested
-## from the client. 
+##            from the client.
 ##
 ## .. bro:see:: dhcp_discover dhcp_offer dhcp_request dhcp_decline dhcp_nak
 ##              dhcp_release dhcp_inform

src/analyzer/protocol/dnp3/dnp3-analyzer.pac

@@ -4,6 +4,24 @@ connection DNP3_Conn(bro_analyzer: BroAnalyzer) {
 	downflow = DNP3_Flow(false);
 };
 
+%header{
+    uint64 bytestring_to_time(const_bytestring time48, size_t length);
+    %}
+
+%code{
+    uint64 bytestring_to_time(const_bytestring time48, size_t length)
+        {
+        /* In DNP3, a timestamp is represented by 6 bytes since epoch
+           in milliseconds. The 6 bytes are stored in big endian format. */
+        uint64 epochTime = 0;
+
+        for ( unsigned int i = 0; i < length; i++ )
+            epochTime = time48[length - i - 1] + epochTime * 256;
+
+        return epochTime;
+        }
+    %}
+
 flow DNP3_Flow(is_orig: bool) {
 	flowunit = DNP3_PDU(is_orig) withcontext (connection, this);
 
@@ -222,7 +240,7 @@ flow DNP3_Flow(is_orig: bool) {
 			BifEvent::generate_dnp3_frozen_counter_32wFlagTime(
 				connection()->bro_analyzer(),
 				connection()->bro_analyzer()->Conn(),
-				is_orig(), flag, count_value, bytestring_to_val(time48));
+				is_orig(), flag, count_value, bytestring_to_time(time48, sizeof(time48)));
 			}
 
 		return true;
@@ -236,7 +254,7 @@ flow DNP3_Flow(is_orig: bool) {
 			BifEvent::generate_dnp3_frozen_counter_16wFlagTime(
 				connection()->bro_analyzer(),
 				connection()->bro_analyzer()->Conn(),
-				is_orig(), flag, count_value, bytestring_to_val(time48));
+				is_orig(), flag, count_value, bytestring_to_time(time48, sizeof(time48)));
 			}
 
 		return true;
@@ -390,7 +408,7 @@ flow DNP3_Flow(is_orig: bool) {
 			BifEvent::generate_dnp3_frozen_analog_input_32wTime(
 				connection()->bro_analyzer(),
 				connection()->bro_analyzer()->Conn(),
-				is_orig(), flag, frozen_value, bytestring_to_val(time48));
+				is_orig(), flag, frozen_value, bytestring_to_time(time48, sizeof(time48)));
 			}
 
 		return true;
@@ -404,7 +422,7 @@ flow DNP3_Flow(is_orig: bool) {
 			BifEvent::generate_dnp3_frozen_analog_input_16wTime(
 				connection()->bro_analyzer(),
 				connection()->bro_analyzer()->Conn(),
-				is_orig(), flag, frozen_value, bytestring_to_val(time48));
+				is_orig(), flag, frozen_value, bytestring_to_time(time48, sizeof(time48)));
 			}
 
 		return true;
@@ -502,7 +520,7 @@ flow DNP3_Flow(is_orig: bool) {
 			BifEvent::generate_dnp3_analog_input_event_32wTime(
 				connection()->bro_analyzer(),
 				connection()->bro_analyzer()->Conn(),
-				is_orig(), flag, value, bytestring_to_val(time48));
+				is_orig(), flag, value, bytestring_to_time(time48, sizeof(time48)));
 			}
 
 		return true;
@@ -516,7 +534,7 @@ flow DNP3_Flow(is_orig: bool) {
 			BifEvent::generate_dnp3_analog_input_event_16wTime(
 				connection()->bro_analyzer(),
 				connection()->bro_analyzer()->Conn(),
-				is_orig(), flag, value, bytestring_to_val(time48));
+				is_orig(), flag, value, bytestring_to_time(time48, sizeof(time48)));
 			}
 
 		return true;
@@ -558,7 +576,7 @@ flow DNP3_Flow(is_orig: bool) {
 			BifEvent::generate_dnp3_analog_input_event_SPwTime(
 				connection()->bro_analyzer(),
 				connection()->bro_analyzer()->Conn(),
-				is_orig(), flag, value, bytestring_to_val(time48));
+				is_orig(), flag, value, bytestring_to_time(time48, sizeof(time48)));
 			}
 
 		return true;
@@ -572,7 +590,7 @@ flow DNP3_Flow(is_orig: bool) {
 			BifEvent::generate_dnp3_analog_input_event_DPwTime(
 				connection()->bro_analyzer(),
 				connection()->bro_analyzer()->Conn(),
-				is_orig(), flag, value_low, value_high, bytestring_to_val(time48));
+				is_orig(), flag, value_low, value_high, bytestring_to_time(time48, sizeof(time48)));
 			}
 
 		return true;
@@ -614,7 +632,7 @@ flow DNP3_Flow(is_orig: bool) {
 			BifEvent::generate_dnp3_frozen_analog_input_event_32wTime(
 				connection()->bro_analyzer(),
 				connection()->bro_analyzer()->Conn(),
-				is_orig(), flag, frozen_value, bytestring_to_val(time48));
+				is_orig(), flag, frozen_value, bytestring_to_time(time48, sizeof(time48)));
 			}
 
 		return true;
@@ -628,7 +646,7 @@ flow DNP3_Flow(is_orig: bool) {
 			BifEvent::generate_dnp3_frozen_analog_input_event_16wTime(
 				connection()->bro_analyzer(),
 				connection()->bro_analyzer()->Conn(),
-				is_orig(), flag, frozen_value, bytestring_to_val(time48));
+				is_orig(), flag, frozen_value, bytestring_to_time(time48, sizeof(time48)));
 			}
 
 		return true;
@@ -670,7 +688,7 @@ flow DNP3_Flow(is_orig: bool) {
 			BifEvent::generate_dnp3_frozen_analog_input_event_SPwTime(
 				connection()->bro_analyzer(),
 				connection()->bro_analyzer()->Conn(),
-				is_orig(), flag, frozen_value, bytestring_to_val(time48));
+				is_orig(), flag, frozen_value, bytestring_to_time(time48, sizeof(time48)));
 			}
 
 		return true;
@@ -684,7 +702,7 @@ flow DNP3_Flow(is_orig: bool) {
 			BifEvent::generate_dnp3_frozen_analog_input_event_DPwTime(
 				connection()->bro_analyzer(),
 				connection()->bro_analyzer()->Conn(),
-				is_orig(), flag, frozen_value_low, frozen_value_high, bytestring_to_val(time48));
+				is_orig(), flag, frozen_value_low, frozen_value_high, bytestring_to_time(time48, sizeof(time48)));
 			}
 
 		return true;

src/analyzer/protocol/dnp3/events.bif

@@ -130,11 +130,11 @@ event dnp3_frozen_counter_16wFlag%(c: connection, is_orig: bool, flag:count, cou
 
 ## Generated for DNP3 objects with the group number 21 and variation number 5
 ## frozen counter 32 bit with flag and time
-event dnp3_frozen_counter_32wFlagTime%(c: connection, is_orig: bool, flag:count, count_value: count, time48: string%);
+event dnp3_frozen_counter_32wFlagTime%(c: connection, is_orig: bool, flag:count, count_value: count, time48: count%);
 
 ## Generated for DNP3 objects with the group number 21 and variation number 6
 ## frozen counter 16 bit with flag and time
-event dnp3_frozen_counter_16wFlagTime%(c: connection, is_orig: bool, flag:count, count_value: count, time48: string%);
+event dnp3_frozen_counter_16wFlagTime%(c: connection, is_orig: bool, flag:count, count_value: count, time48: count%);
 
 ## Generated for DNP3 objects with the group number 21 and variation number 9
 ## frozen counter 32 bit without flag
@@ -178,11 +178,11 @@ event dnp3_frozen_analog_input_16wFlag%(c: connection, is_orig: bool, flag: coun
 
 ## Generated for DNP3 objects with the group number 31 and variation number 3
 ## frozen analog input 32 bit with time-of-freeze
-event dnp3_frozen_analog_input_32wTime%(c: connection, is_orig: bool, flag: count, frozen_value: count, time48: string%);
+event dnp3_frozen_analog_input_32wTime%(c: connection, is_orig: bool, flag: count, frozen_value: count, time48: count%);
 
 ## Generated for DNP3 objects with the group number 31 and variation number 4
 ## frozen analog input 16 bit with time-of-freeze
-event dnp3_frozen_analog_input_16wTime%(c: connection, is_orig: bool, flag: count, frozen_value: count, time48: string%);
+event dnp3_frozen_analog_input_16wTime%(c: connection, is_orig: bool, flag: count, frozen_value: count, time48: count%);
 
 ## Generated for DNP3 objects with the group number 31 and variation number 5
 ## frozen analog input 32 bit without flag
@@ -210,11 +210,11 @@ event dnp3_analog_input_event_16woTime%(c: connection, is_orig: bool, flag: coun
 
 ## Generated for DNP3 objects with the group number 32 and variation number 3
 ## analog input event 32 bit with time
-event dnp3_analog_input_event_32wTime%(c: connection, is_orig: bool, flag: count, value: count, time48: string%);
+event dnp3_analog_input_event_32wTime%(c: connection, is_orig: bool, flag: count, value: count, time48: count%);
 
 ## Generated for DNP3 objects with the group number 32 and variation number 4
 ## analog input event 16 bit with time
-event dnp3_analog_input_event_16wTime%(c: connection, is_orig: bool, flag: count, value: count, time48: string%);
+event dnp3_analog_input_event_16wTime%(c: connection, is_orig: bool, flag: count, value: count, time48: count%);
 
 ## Generated for DNP3 objects with the group number 32 and variation number 5
 ## analog input event single-precision float point without time
@@ -226,11 +226,11 @@ event dnp3_analog_input_event_DPwoTime%(c: connection, is_orig: bool, flag: coun
 
 ## Generated for DNP3 objects with the group number 32 and variation number 7
 ## analog input event single-precision float point with time
-event dnp3_analog_input_event_SPwTime%(c: connection, is_orig: bool, flag: count, value: count, time48: string%);
+event dnp3_analog_input_event_SPwTime%(c: connection, is_orig: bool, flag: count, value: count, time48: count%);
 
 ## Generated for DNP3 objects with the group number 32 and variation number 8
 ## analog input event double-precisiion float point with time
-event dnp3_analog_input_event_DPwTime%(c: connection, is_orig: bool, flag: count, value_low: count, value_high: count, time48: string%);
+event dnp3_analog_input_event_DPwTime%(c: connection, is_orig: bool, flag: count, value_low: count, value_high: count, time48: count%);
 
 ## Generated for DNP3 objects with the group number 33 and variation number 1
 ## frozen analog input event 32 bit without time
@@ -242,11 +242,11 @@ event dnp3_frozen_analog_input_event_16woTime%(c: connection, is_orig: bool, fla
 
 ## Generated for DNP3 objects with the group number 33 and variation number 3
 ## frozen analog input event 32 bit with time
-event dnp3_frozen_analog_input_event_32wTime%(c: connection, is_orig: bool, flag: count, frozen_value: count, time48: string%);
+event dnp3_frozen_analog_input_event_32wTime%(c: connection, is_orig: bool, flag: count, frozen_value: count, time48: count%);
 
 ## Generated for DNP3 objects with the group number 33 and variation number 4
 ## frozen analog input event 16 bit with time
-event dnp3_frozen_analog_input_event_16wTime%(c: connection, is_orig: bool, flag: count, frozen_value: count, time48: string%);
+event dnp3_frozen_analog_input_event_16wTime%(c: connection, is_orig: bool, flag: count, frozen_value: count, time48: count%);
 
 ## Generated for DNP3 objects with the group number 33 and variation number 5
 ## frozen analog input event single-precision float point without time
@@ -258,11 +258,11 @@ event dnp3_frozen_analog_input_event_DPwoTime%(c: connection, is_orig: bool, fla
 
 ## Generated for DNP3 objects with the group number 33 and variation number 7
 ## frozen analog input event single-precision float point with time
-event dnp3_frozen_analog_input_event_SPwTime%(c: connection, is_orig: bool, flag: count, frozen_value: count, time48: string%);
+event dnp3_frozen_analog_input_event_SPwTime%(c: connection, is_orig: bool, flag: count, frozen_value: count, time48: count%);
 
 ## Generated for DNP3 objects with the group number 34 and variation number 8
 ## frozen analog input event double-precision float point with time
-event dnp3_frozen_analog_input_event_DPwTime%(c: connection, is_orig: bool, flag: count, frozen_value_low: count, frozen_value_high: count, time48: string%);
+event dnp3_frozen_analog_input_event_DPwTime%(c: connection, is_orig: bool, flag: count, frozen_value_low: count, frozen_value_high: count, time48: count%);
 
 ## g70
 event dnp3_file_transport%(c: connection, is_orig: bool, file_handle: count, block_num: count, file_data: string%);

src/analyzer/protocol/dns/DNS.cc

@@ -692,15 +692,23 @@ int DNS_Interpreter::ParseRR_EDNS(DNS_MsgInfo* msg,
 		data += rdlength;
 		len -= rdlength;
 		}
-	else
-		{ // no data, move on
-		data += rdlength;
-		len -= rdlength;
-		}
 
 	return 1;
 	}
 
+void DNS_Interpreter::ExtractOctets(const u_char*& data, int& len,
+                                    BroString** p)
+	{
+	uint16 dlen = ExtractShort(data, len);
+	dlen = min(len, static_cast<int>(dlen));
+
+	if ( p )
+		*p = new BroString(data, dlen, 0);
+
+	data += dlen;
+	len -= dlen;
+	}
+
 int DNS_Interpreter::ParseRR_TSIG(DNS_MsgInfo* msg,
 				const u_char*& data, int& len, int rdlength,
 				const u_char* msg_start)
@@ -718,24 +726,17 @@ int DNS_Interpreter::ParseRR_TSIG(DNS_MsgInfo* msg,
 	uint32 sign_time_sec = ExtractLong(data, len);
 	unsigned int sign_time_msec = ExtractShort(data, len);
 	unsigned int fudge = ExtractShort(data, len);
-
-	u_char request_MAC[16];
-	memcpy(request_MAC, data, sizeof(request_MAC));
-
-	// Here we adjust the size of the requested MAC + u_int16_t
-	// for length.  See RFC 2845, sec 2.3.
-	int n = sizeof(request_MAC) + sizeof(u_int16_t);
-	data += n;
-	len -= n;
-
+	BroString* request_MAC;
+	ExtractOctets(data, len, &request_MAC);
 	unsigned int orig_id = ExtractShort(data, len);
 	unsigned int rr_error = ExtractShort(data, len);
+	ExtractOctets(data, len, 0);  // Other Data
 
 	msg->tsig = new TSIG_DATA;
 
 	msg->tsig->alg_name =
 		new BroString(alg_name, alg_name_end - alg_name, 1);
-	msg->tsig->sig = new BroString(request_MAC, sizeof(request_MAC), 1);
+	msg->tsig->sig = request_MAC;
 	msg->tsig->time_s = sign_time_sec;
 	msg->tsig->time_ms = sign_time_msec;
 	msg->tsig->fudge = fudge;

src/analyzer/protocol/dns/DNS.h

@@ -180,6 +180,7 @@ protected:
 
 	uint16 ExtractShort(const u_char*& data, int& len);
 	uint32 ExtractLong(const u_char*& data, int& len);
+	void ExtractOctets(const u_char*& data, int& len, BroString** p);
 
 	int ParseRR_Name(DNS_MsgInfo* msg,
 				const u_char*& data, int& len, int rdlength,

src/analyzer/protocol/finger/Plugin.cc

@@ -1,6 +1,5 @@
 // See the file  in the main distribution directory for copyright.
 
-
 #include "plugin/Plugin.h"
 
 #include "Finger.h"

src/analyzer/protocol/http/HTTP.cc

@@ -243,10 +243,13 @@ int HTTP_Entity::Undelivered(int64_t len)
 		return 0;
 
 	if ( is_partial_content )
+		{
 		precomputed_file_id = file_mgr->Gap(body_length, len,
 		              http_message->MyHTTP_Analyzer()->GetAnalyzerTag(),
 		              http_message->MyHTTP_Analyzer()->Conn(),
 		              http_message->IsOrig(), precomputed_file_id);
+		offset += len;
+		}
 	else
 		precomputed_file_id = file_mgr->Gap(body_length, len,
 		                  http_message->MyHTTP_Analyzer()->GetAnalyzerTag(),
@@ -556,12 +559,9 @@ HTTP_Message::HTTP_Message(HTTP_Analyzer* arg_analyzer,
 
 	current_entity = 0;
 	top_level = new HTTP_Entity(this, 0, expect_body);
+	entity_data_buffer = 0;
 	BeginEntity(top_level);
 
-	buffer_offset = buffer_size = 0;
-	data_buffer = 0;
-	total_buffer_size = 0;
-
 	start_time = network_time;
 	body_length = 0;
 	content_gap_length = 0;
@@ -571,6 +571,7 @@ HTTP_Message::HTTP_Message(HTTP_Analyzer* arg_analyzer,
 HTTP_Message::~HTTP_Message()
 	{
 	delete top_level;
+	delete [] entity_data_buffer;
 	}
 
 Val* HTTP_Message::BuildMessageStat(const int interrupted, const char* msg)
@@ -618,22 +619,14 @@ void HTTP_Message::Done(const int interrupted, const char* detail)
 		}
 
 	MyHTTP_Analyzer()->HTTP_MessageDone(is_orig, this);
-
-	delete_strings(buffers);
-
-	if ( data_buffer )
-		{
-		delete data_buffer;
-		data_buffer = 0;
-		}
 	}
 
 int HTTP_Message::Undelivered(int64_t len)
 	{
-	if ( ! top_level )
-		return 0;
+	HTTP_Entity* e = current_entity ? current_entity
+	                                : static_cast<HTTP_Entity*>(top_level);
 
-	if ( ((HTTP_Entity*) top_level)->Undelivered(len) )
+	if ( e && e->Undelivered(len) )
 		{
 		content_gap_length += len;
 		return 1;
@@ -666,8 +659,6 @@ void HTTP_Message::EndEntity(mime::MIME_Entity* entity)
 	body_length += ((HTTP_Entity*) entity)->BodyLength();
 	header_length += ((HTTP_Entity*) entity)->HeaderLength();
 
-	DeliverEntityData();
-
 	if ( http_end_entity )
 		{
 		val_list* vl = new val_list();
@@ -741,31 +732,18 @@ void HTTP_Message::SubmitTrailingHeaders(mime::MIME_HeaderList& /* hlist */)
 
 void HTTP_Message::SubmitData(int len, const char* buf)
 	{
-	if ( buf != (const char*) data_buffer->Bytes() + buffer_offset ||
-	     buffer_offset + len > buffer_size )
-		{
-		reporter->AnalyzerError(MyHTTP_Analyzer(),
-		                                "HTTP message buffer misalignment");
-		return;
-		}
-
-	buffer_offset += len;
-	if ( buffer_offset >= buffer_size )
-		{
-		buffers.push_back(data_buffer);
-		data_buffer = 0;
-		}
+	if ( http_entity_data )
+		MyHTTP_Analyzer()->HTTP_EntityData(is_orig,
+		        new BroString(reinterpret_cast<const u_char*>(buf), len, 0));
 	}
 
 int HTTP_Message::RequestBuffer(int* plen, char** pbuf)
 	{
-	if ( ! data_buffer )
-		if ( ! InitBuffer(mime_segment_length) )
-			return 0;
-
-	*plen = data_buffer->Len() - buffer_offset;
-	*pbuf = (char*) data_buffer->Bytes() + buffer_offset;
+	if ( ! entity_data_buffer )
+		entity_data_buffer = new char[http_entity_data_delivery_size];
 
+	*plen = http_entity_data_delivery_size;
+	*pbuf = entity_data_buffer;
 	return 1;
 	}
 
@@ -806,9 +784,6 @@ void HTTP_Message::SetPlainDelivery(int64_t length)
 
 	if ( length > 0 && BifConst::skip_http_data )
 		content_line->SkipBytesAfterThisLine(length);
-
-	if ( ! data_buffer )
-		InitBuffer(length);
 	}
 
 void HTTP_Message::SkipEntityData()
@@ -817,87 +792,6 @@ void HTTP_Message::SkipEntityData()
 		current_entity->SkipBody();
 	}
 
-void HTTP_Message::DeliverEntityData()
-	{
-	if ( http_entity_data )
-		{
-		const BroString* entity_data = 0;
-
-		if ( data_buffer && buffer_offset > 0 )
-			{
-			if ( buffer_offset < buffer_size )
-				{
-				entity_data = new BroString(data_buffer->Bytes(), buffer_offset, 0);
-				delete data_buffer;
-				}
-			else
-				entity_data = data_buffer;
-
-			data_buffer = 0;
-
-			if ( buffers.empty() )
-				MyHTTP_Analyzer()->HTTP_EntityData(is_orig,
-								entity_data);
-			else
-				buffers.push_back(entity_data);
-
-			entity_data = 0;
-			}
-
-		if ( ! buffers.empty() )
-			{
-			if ( buffers.size() == 1 )
-				{
-				entity_data = buffers[0];
-				buffers.clear();
-				}
-			else
-				{
-				entity_data = concatenate(buffers);
-				delete_strings(buffers);
-				}
-
-			MyHTTP_Analyzer()->HTTP_EntityData(is_orig, entity_data);
-			}
-		}
-	else
-		{
-		delete_strings(buffers);
-
-		if ( data_buffer )
-			delete data_buffer;
-
-		data_buffer = 0;
-		}
-
-	total_buffer_size = 0;
-	}
-
-int HTTP_Message::InitBuffer(int64_t length)
-	{
-	if ( length <= 0 )
-		return 0;
-
-	if ( total_buffer_size >= http_entity_data_delivery_size )
-		DeliverEntityData();
-
-	if ( total_buffer_size + length > http_entity_data_delivery_size )
-		{
-		length = http_entity_data_delivery_size - total_buffer_size;
-		if ( length <= 0 )
-			return 0;
-		}
-
-	u_char* b = new u_char[length];
-	data_buffer = new BroString(0, b, length);
-
-	buffer_size = length;
-	total_buffer_size += length;
-	buffer_offset = 0;
-
-	return 1;
-	}
-
 void HTTP_Message::Weird(const char* msg)
 	{
 	analyzer->Weird(msg);
@@ -1844,7 +1738,7 @@ void HTTP_Analyzer::ParseVersion(data_chunk_t ver, const IPAddr& host,
 		}
 	}
 
-void HTTP_Analyzer::HTTP_EntityData(int is_orig, const BroString* entity_data)
+void HTTP_Analyzer::HTTP_EntityData(int is_orig, BroString* entity_data)
 	{
 	if ( http_entity_data )
 		{
@@ -1852,8 +1746,7 @@ void HTTP_Analyzer::HTTP_EntityData(int is_orig, const BroString* entity_data)
 		vl->append(BuildConnVal());
 		vl->append(new Val(is_orig, TYPE_BOOL));
 		vl->append(new Val(entity_data->Len(), TYPE_COUNT));
-		// FIXME: Make sure that removing the const here is indeed ok...
-		vl->append(new StringVal(const_cast<BroString*>(entity_data)));
+		vl->append(new StringVal(entity_data));
 		ConnectionEvent(http_entity_data, vl);
 		}
 	else

src/analyzer/protocol/http/HTTP.h

@@ -134,13 +134,7 @@ protected:
 	tcp::ContentLine_Analyzer* content_line;
 	bool is_orig;
 
-	vector<const BroString*> buffers;
-
-	// Controls the total buffer size within http_entity_data_delivery_size.
-	int total_buffer_size;
-
-	int buffer_offset, buffer_size;
-	BroString* data_buffer;
+	char* entity_data_buffer;
 
 	double start_time;
 
@@ -153,9 +147,6 @@ protected:
 
 	HTTP_Entity* current_entity;
 
-	int InitBuffer(int64_t length);
-	void DeliverEntityData();
-
 	Val* BuildMessageStat(const int interrupted, const char* msg);
 };
 
@@ -167,7 +158,7 @@ public:
 	void Undelivered(tcp::TCP_Endpoint* sender, uint64 seq, int len);
 
 	void HTTP_Header(int is_orig, mime::MIME_Header* h);
-	void HTTP_EntityData(int is_orig, const BroString* entity_data);
+	void HTTP_EntityData(int is_orig, BroString* entity_data);
 	void HTTP_MessageDone(int is_orig, HTTP_Message* message);
 	void HTTP_Event(const char* category, const char* detail);
 	void HTTP_Event(const char* category, StringVal *detail);

src/analyzer/protocol/mime/MIME.cc

@@ -142,8 +142,9 @@ int fputs(data_chunk_t b, FILE* fp)
 void MIME_Mail::Undelivered(int len)
 	{
 	// is_orig param not available, doesn't matter as long as it's consistent
-	file_mgr->Gap(cur_entity_len, len, analyzer->GetAnalyzerTag(), analyzer->Conn(),
-	              false);
+	cur_entity_id = file_mgr->Gap(cur_entity_len, len,
+	                              analyzer->GetAnalyzerTag(), analyzer->Conn(),
+	                              false, cur_entity_id);
 	}
 
 int strcasecmp_n(data_chunk_t s, const char* t)
@@ -644,11 +645,7 @@ void MIME_Entity::EndOfData()
 		if ( content_encoding == CONTENT_ENCODING_BASE64 )
 			FinishDecodeBase64();
 
-		if ( data_buf_offset > 0 )
-			{
-			SubmitData(data_buf_offset, data_buf_data);
-			data_buf_offset = -1;
-			}
+		FlushData();
 		}
 
 	message->EndEntity (this);
@@ -1002,6 +999,7 @@ void MIME_Entity::DecodeDataLine(int len, const char* data, int trailing_CRLF)
 			DecodeBinary(len, data, trailing_CRLF);
 			break;
 	}
+	FlushData();
 	}
 
 void MIME_Entity::DecodeBinary(int len, const char* data, int trailing_CRLF)
@@ -1201,6 +1199,15 @@ void MIME_Entity::DataOctets(int len, const char* data)
 		}
 	}
 
+void MIME_Entity::FlushData()
+	{
+	if ( data_buf_offset > 0 )
+		{
+		SubmitData(data_buf_offset, data_buf_data);
+		data_buf_offset = -1;
+		}
+	}
+
 void MIME_Entity::SubmitHeader(MIME_Header* h)
 	{
 	message->SubmitHeader(h);
@@ -1347,6 +1354,7 @@ MIME_Mail::~MIME_Mail()
 void MIME_Mail::BeginEntity(MIME_Entity* /* entity */)
 	{
 	cur_entity_len = 0;
+	cur_entity_id.clear();
 
 	if ( mime_begin_entity )
 		{
@@ -1386,6 +1394,7 @@ void MIME_Mail::EndEntity(MIME_Entity* /* entity */)
 		}
 
 	file_mgr->EndOfFile(analyzer->GetAnalyzerTag(), analyzer->Conn());
+	cur_entity_id.clear();
 	}
 
 void MIME_Mail::SubmitHeader(MIME_Header* h)
@@ -1448,8 +1457,9 @@ void MIME_Mail::SubmitData(int len, const char* buf)
 		}
 
 	// is_orig param not available, doesn't matter as long as it's consistent
-	file_mgr->DataIn(reinterpret_cast<const u_char*>(buf), len,
-	                 analyzer->GetAnalyzerTag(), analyzer->Conn(), false);
+	cur_entity_id = file_mgr->DataIn(reinterpret_cast<const u_char*>(buf), len,
+	                 analyzer->GetAnalyzerTag(), analyzer->Conn(), false,
+	                 cur_entity_id);
 
 	cur_entity_len += len;
 	buffer_start = (buf + len) - (char*)data_buffer->Bytes();

src/analyzer/protocol/mime/MIME.h

@@ -133,6 +133,7 @@ protected:
 	int GetDataBuffer();
 	void DataOctet(char ch);
 	void DataOctets(int len, const char* data);
+	void FlushData();
 	virtual void SubmitData(int len, const char* buf);
 
 	virtual void SubmitHeader(MIME_Header* h);
@@ -258,6 +259,7 @@ protected:
 	BroString* data_buffer;
 
 	uint64 cur_entity_len;
+	string cur_entity_id;
 };
 
 

src/analyzer/protocol/tcp/TCP_Reassembler.cc

@@ -117,6 +117,45 @@ void TCP_Reassembler::SetContentsFile(BroFile* f)
 	record_contents_file = f;
 	}
 
+static inline bool established(const TCP_Endpoint* a, const TCP_Endpoint* b)
+	{
+	return a->state == TCP_ENDPOINT_ESTABLISHED &&
+	       b->state == TCP_ENDPOINT_ESTABLISHED;
+	}
+
+static inline bool report_gap(const TCP_Endpoint* a, const TCP_Endpoint* b)
+	{
+	return content_gap &&
+	       ( BifConst::report_gaps_for_partial || established(a, b) );
+	}
+
+void TCP_Reassembler::Gap(uint64 seq, uint64 len)
+	{
+	// Only report on content gaps for connections that
+	// are in a cleanly established state.  In other
+	// states, these can arise falsely due to things
+	// like sequence number mismatches in RSTs, or
+	// unseen previous packets in partial connections.
+	// The one opportunity we lose here is on clean FIN
+	// handshakes, but Oh Well.
+
+	if ( report_gap(endp, endp->peer) )
+		{
+		val_list* vl = new val_list;
+		vl->append(dst_analyzer->BuildConnVal());
+		vl->append(new Val(IsOrig(), TYPE_BOOL));
+		vl->append(new Val(seq, TYPE_COUNT));
+		vl->append(new Val(len, TYPE_COUNT));
+		dst_analyzer->ConnectionEvent(content_gap, vl);
+		}
+
+	if ( type == Direct )
+		dst_analyzer->NextUndelivered(seq, len, IsOrig());
+	else
+		dst_analyzer->ForwardUndelivered(seq, len, IsOrig());
+
+	had_gap = true;
+	}
 
 void TCP_Reassembler::Undelivered(uint64 up_to_seq)
 	{
@@ -189,48 +228,35 @@ void TCP_Reassembler::Undelivered(uint64 up_to_seq)
 
 		if ( ! skip_deliveries )
 			{
-			// This can happen because we're processing a trace
-			// that's been filtered.  For example, if it's just
-			// SYN/FIN data, then there can be data in the FIN
-			// packet, but it's undelievered because it's out of
-			// sequence.
-
-			uint64 seq = last_reassem_seq;
-			uint64 len = up_to_seq - last_reassem_seq;
-
-			// Only report on content gaps for connections that
-			// are in a cleanly established state.  In other
-			// states, these can arise falsely due to things
-			// like sequence number mismatches in RSTs, or
-			// unseen previous packets in partial connections.
-			// The one opportunity we lose here is on clean FIN
-			// handshakes, but Oh Well.
-
-			if ( content_gap &&
-				(BifConst::report_gaps_for_partial ||
-					(endpoint->state == TCP_ENDPOINT_ESTABLISHED &&
-					peer->state == TCP_ENDPOINT_ESTABLISHED ) ) )
+			// If we have blocks that begin below up_to_seq, deliver them.
+			DataBlock* b = blocks;
+			while ( b )
 				{
-				val_list* vl = new val_list;
-				vl->append(dst_analyzer->BuildConnVal());
-				vl->append(new Val(IsOrig(), TYPE_BOOL));
-				vl->append(new Val(seq, TYPE_COUNT));
-				vl->append(new Val(len, TYPE_COUNT));
+				if ( b->seq < last_reassem_seq )
+					{
+					// Already delivered this block.
+					b = b->next;
+					continue;
+					}
 
-				dst_analyzer->ConnectionEvent(content_gap, vl);
+				if ( b->seq >= up_to_seq )
+					// Block is beyond what we need to process at this point.
+					break;
+
+				uint64 gap_at_seq = last_reassem_seq;
+				uint64 gap_len = b->seq - last_reassem_seq;
+
+				Gap(gap_at_seq, gap_len);
+				last_reassem_seq += gap_len;
+				BlockInserted(b);
+				// Inserting a block may cause trimming of what's buffered,
+				// so have to assume 'b' is invalid, hence re-assign to start.
+				b = blocks;
 				}
 
-			if ( type == Direct )
-				dst_analyzer->NextUndelivered(last_reassem_seq,
-								len, IsOrig());
-			else
-				{
-				dst_analyzer->ForwardUndelivered(last_reassem_seq,
-								len, IsOrig());
-				}
+			if ( up_to_seq > last_reassem_seq )
+				Gap(last_reassem_seq, up_to_seq - last_reassem_seq);
 			}
-
-		had_gap = true;
 		}
 
 	// We should record and match undelivered even if we are skipping
@@ -243,7 +269,8 @@ void TCP_Reassembler::Undelivered(uint64 up_to_seq)
 		MatchUndelivered(up_to_seq, false);
 
 	// But we need to re-adjust last_reassem_seq in either case.
-	last_reassem_seq = up_to_seq;	// we've done our best ...
+	if ( up_to_seq > last_reassem_seq )
+		last_reassem_seq = up_to_seq;	// we've done our best ...
 	}
 
 void TCP_Reassembler::MatchUndelivered(uint64 up_to_seq, bool use_last_upper)

src/analyzer/protocol/tcp/TCP_Reassembler.h

@@ -94,6 +94,7 @@ private:
 	DECLARE_SERIAL(TCP_Reassembler);
 
 	void Undelivered(uint64 up_to_seq);
+	void Gap(uint64 seq, uint64 len);
 
 	void RecordToSeq(uint64 start_seq, uint64 stop_seq, BroFile* f);
 	void RecordBlock(DataBlock* b, BroFile* f);

src/bro.bif

@@ -21,6 +21,7 @@
 #include "IPAddr.h"
 #include "util.h"
 #include "file_analysis/Manager.h"
+#include "iosource/Manager.h"
 
 using namespace std;
 
@@ -33,7 +34,7 @@ TableType* var_sizes;
 // and hence it's declared in NetVar.{h,cc}.
 extern RecordType* gap_info;
 
-static PktDumper* addl_pkt_dumper = 0;
+static iosource::PktDumper* addl_pkt_dumper = 0;
 
 bro_int_t parse_int(const char*& fmt)
 	{
@@ -1675,11 +1676,14 @@ function net_stats%(%): NetStats
 	unsigned int drop = 0;
 	unsigned int link = 0;
 
-	loop_over_list(pkt_srcs, i)
-		{
-		PktSrc* ps = pkt_srcs[i];
+	const iosource::Manager::PktSrcList& pkt_srcs(iosource_mgr->GetPktSrcs());
 
-		struct PktSrc::Stats stat;
+	for ( iosource::Manager::PktSrcList::const_iterator i = pkt_srcs.begin();
+	      i != pkt_srcs.end(); i++ )
+		{
+		iosource::PktSrc* ps = *i;
+
+		struct iosource::PktSrc::Stats stat;
 		ps->Statistics(&stat);
 		recv += stat.received;
 		drop += stat.dropped;
@@ -3224,10 +3228,15 @@ function dump_current_packet%(file_name: string%) : bool
 		return new Val(0, TYPE_BOOL);
 
 	if ( ! addl_pkt_dumper )
-		addl_pkt_dumper = new PktDumper(0, true);
+		addl_pkt_dumper = iosource_mgr->OpenPktDumper(file_name->CheckString(), true);
 
-	addl_pkt_dumper->Open(file_name->CheckString());
-	addl_pkt_dumper->Dump(hdr, pkt);
+	if ( addl_pkt_dumper )
+		{
+		iosource::PktDumper::Packet p;
+		p.hdr = hdr;
+		p.data = pkt;
+		addl_pkt_dumper->Dump(&p);
+		}
 
 	return new Val(! addl_pkt_dumper->IsError(), TYPE_BOOL);
 	%}
@@ -3284,10 +3293,15 @@ function dump_packet%(pkt: pcap_packet, file_name: string%) : bool
 	hdr.len = (*pkt_vl)[3]->AsCount();
 
 	if ( ! addl_pkt_dumper )
-		addl_pkt_dumper = new PktDumper(0, true);
+		addl_pkt_dumper = iosource_mgr->OpenPktDumper(file_name->CheckString(), true);
 
-	addl_pkt_dumper->Open(file_name->CheckString());
-	addl_pkt_dumper->Dump(&hdr, (*pkt_vl)[4]->AsString()->Bytes());
+	if ( addl_pkt_dumper )
+		{
+		iosource::PktDumper::Packet p;
+		p.hdr = &hdr;
+		p.data = (*pkt_vl)[4]->AsString()->Bytes();
+		addl_pkt_dumper->Dump(&p);
+		}
 
 	return new Val(addl_pkt_dumper->IsError(), TYPE_BOOL);
 	%}
@@ -4110,14 +4124,14 @@ function rotate_file_by_name%(f: string%): rotate_info
 	bool is_addl_pkt_dumper = false;
 
 	// Special case: one of current dump files.
-	if ( pkt_dumper && streq(pkt_dumper->FileName(), f->CheckString()) )
+	if ( pkt_dumper && streq(pkt_dumper->Path().c_str(), f->CheckString()) )
 		{
 		is_pkt_dumper = true;
 		pkt_dumper->Close();
 		}
 
 	if ( addl_pkt_dumper &&
-	     streq(addl_pkt_dumper->FileName(), f->CheckString()) )
+	     streq(addl_pkt_dumper->Path().c_str(), f->CheckString()) )
 		{
 		is_addl_pkt_dumper = true;
 		addl_pkt_dumper->Close();
@@ -4214,103 +4228,6 @@ function enable_raw_output%(f: file%): any
 #
 # ===========================================================================
 
-## Precompiles a PCAP filter and binds it to a given identifier.
-##
-## id: The PCAP identifier to reference the filter *s* later on.
-##
-## s: The PCAP filter. See ``man tcpdump`` for valid expressions.
-##
-## Returns: True if *s* is valid and precompiles successfully.
-##
-## .. bro:see:: install_pcap_filter
-##          install_src_addr_filter
-##          install_src_net_filter
-##          uninstall_src_addr_filter
-##          uninstall_src_net_filter
-##          install_dst_addr_filter
-##          install_dst_net_filter
-##          uninstall_dst_addr_filter
-##          uninstall_dst_net_filter
-##          pcap_error
-function precompile_pcap_filter%(id: PcapFilterID, s: string%): bool
-	%{
-	bool success = true;
-
-	loop_over_list(pkt_srcs, i)
-		{
-		pkt_srcs[i]->ClearErrorMsg();
-
-		if ( ! pkt_srcs[i]->PrecompileFilter(id->ForceAsInt(),
-							s->CheckString()) )
-			{
-			reporter->Error("precompile_pcap_filter: %s",
-				pkt_srcs[i]->ErrorMsg());
-			success = false;
-			}
-		}
-
-	return new Val(success, TYPE_BOOL);
-	%}
-
-## Installs a PCAP filter that has been precompiled with
-## :bro:id:`precompile_pcap_filter`.
-##
-## id: The PCAP filter id of a precompiled filter.
-##
-## Returns: True if the filter associated with *id* has been installed
-##          successfully.
-##
-## .. bro:see:: precompile_pcap_filter
-##              install_src_addr_filter
-##              install_src_net_filter
-##              uninstall_src_addr_filter
-##              uninstall_src_net_filter
-##              install_dst_addr_filter
-##              install_dst_net_filter
-##              uninstall_dst_addr_filter
-##              uninstall_dst_net_filter
-##              pcap_error
-function install_pcap_filter%(id: PcapFilterID%): bool
-	%{
-	bool success = true;
-
-	loop_over_list(pkt_srcs, i)
-		{
-		pkt_srcs[i]->ClearErrorMsg();
-
-		if ( ! pkt_srcs[i]->SetFilter(id->ForceAsInt()) )
-			success = false;
-		}
-
-	return new Val(success, TYPE_BOOL);
-	%}
-
-## Returns a string representation of the last PCAP error.
-##
-## Returns: A descriptive error message of the PCAP function that failed.
-##
-## .. bro:see:: precompile_pcap_filter
-##              install_pcap_filter
-##              install_src_addr_filter
-##              install_src_net_filter
-##              uninstall_src_addr_filter
-##              uninstall_src_net_filter
-##              install_dst_addr_filter
-##              install_dst_net_filter
-##              uninstall_dst_addr_filter
-##              uninstall_dst_net_filter
-function pcap_error%(%): string
-	%{
-	loop_over_list(pkt_srcs, i)
-		{
-		const char* err = pkt_srcs[i]->ErrorMsg();
-		if ( *err )
-			return new StringVal(err);
-		}
-
-	return new StringVal("no error");
-	%}
-
 ## Installs a filter to drop packets from a given IP source address with
 ## a certain probability if none of a given set of TCP flags are set.
 ## Note that for IPv6 packets with a Destination options header that has
@@ -4542,7 +4459,7 @@ function enable_communication%(%): any
 		return 0;
 
 	using_communication = 1;
-	remote_serializer->Init();
+	remote_serializer->Enable();
 	return 0;
 	%}
 

src/file_analysis/File.cc

@@ -437,7 +437,7 @@ void File::EndOfFile()
 void File::Gap(uint64 offset, uint64 len)
 	{
 	DBG_LOG(DBG_FILE_ANALYSIS, "[%s] Gap of size %" PRIu64 " at offset %" PRIu64,
-		id.c_str(), offset, len);
+		id.c_str(), len, offset);
 
 	analyzers.DrainModifications();
 

src/file_analysis/analyzer/x509/functions.bif

@@ -104,6 +104,39 @@ STACK_OF(X509)* x509_get_untrusted_stack(VectorVal* certs_vec)
 	return untrusted_certs;
 	}
 
+// We need this function to be able to identify the signer certificate of an
+// OCSP request out of a list of possible certificates.
+X509* x509_get_ocsp_signer(STACK_OF(X509) *certs, OCSP_RESPID *rid)
+	{
+	// We support two lookup types - either by response id or by key.
+	if ( rid->type == V_OCSP_RESPID_NAME )
+		return X509_find_by_subject(certs, rid->value.byName);
+
+	// There only should be name and type - but let's be sure...
+	if ( rid->type != V_OCSP_RESPID_KEY )
+		return 0;
+
+	// Just like OpenSSL, we just support SHA-1 lookups and bail out otherwhise.
+	if ( rid->value.byKey->length != SHA_DIGEST_LENGTH )
+		return 0;
+
+	unsigned char* key_hash = rid->value.byKey->data;
+	for ( int i = 0; i < sk_X509_num(certs); ++i )
+		{
+		unsigned char digest[SHA_DIGEST_LENGTH];
+		X509* cert = sk_X509_value(certs, i);
+		if ( ! X509_pubkey_digest(cert, EVP_sha1(), digest, NULL) )
+			// digest failed for this certificate, try with next
+			continue;
+
+		if ( memcmp(digest, key_hash, SHA_DIGEST_LENGTH) == 0 )
+			// keys match, return certificate
+			return cert;
+		}
+
+	return 0;
+	}
+
 %%}
 
 ## Parses a certificate into an X509::Certificate structure.
@@ -221,6 +254,7 @@ function x509_ocsp_verify%(certs: x509_opaque_vector, ocsp_reply: string, root_c
 	int out = -1;
 	int result = -1;
 	X509* issuer_certificate = 0;
+	X509* signer = 0;
 	OCSP_RESPONSE *resp = d2i_OCSP_RESPONSE(NULL, &start, ocsp_reply->Len());
 	if ( ! resp )
 		{
@@ -250,19 +284,47 @@ function x509_ocsp_verify%(certs: x509_opaque_vector, ocsp_reply: string, root_c
 	// inject the certificates in the certificate list of the OCSP reply, they actually are used during
 	// the lookup.
 	// Yay.
+
+	if ( ! basic->certs )
+		{
+		basic->certs = sk_X509_new_null();
+		if ( ! basic->certs )
+			{
+			rval = x509_result_record(-1, "Could not allocate basic x509 stack");
+			goto x509_ocsp_cleanup;
+			}
+		}
+
 	issuer_certificate = 0;
 	for ( int i = 0; i < sk_X509_num(untrusted_certs); i++)
 		{
 		sk_X509_push(basic->certs, X509_dup(sk_X509_value(untrusted_certs, i)));
 
-		if ( X509_NAME_cmp(X509_get_issuer_name(cert), X509_get_subject_name(sk_X509_value(untrusted_certs, i))) )
+		if ( X509_NAME_cmp(X509_get_issuer_name(cert), X509_get_subject_name(sk_X509_value(untrusted_certs, i))) == 0 )
 			issuer_certificate = sk_X509_value(untrusted_certs, i);
 		}
 
 	// Because we actually want to be able to give nice error messages that show why we were
 	// not able to verify the OCSP response - do our own verification logic first.
+	signer = x509_get_ocsp_signer(basic->certs, basic->tbsResponseData->responderId);
+
+	/*
+	Do this perhaps - OpenSSL also cannot do it, so I do not really feel bad about it.
+	Needs a different lookup because the root store is no stack of X509 certs
+
+	if ( !s igner )
+		// if we did not find it in the certificates that were sent, search in the root store
+		signer = x509_get_ocsp_signer(basic->certs, basic->tbsResponseData->responderId);
+	*/
+
+	if ( ! signer )
+		{
+		rval = x509_result_record(-1, "Could not find OCSP responder certificate");
+		goto x509_ocsp_cleanup;
+		}
+
 	csc = X509_STORE_CTX_new();
-	X509_STORE_CTX_init(csc, ctx, sk_X509_value(basic->certs, 0), basic->certs);
+	X509_STORE_CTX_init(csc, ctx, signer, basic->certs);
 	X509_STORE_CTX_set_time(csc, 0, (time_t) verify_time);
 	X509_STORE_CTX_set_purpose(csc, X509_PURPOSE_OCSP_HELPER);
 
@@ -281,7 +343,6 @@ function x509_ocsp_verify%(certs: x509_opaque_vector, ocsp_reply: string, root_c
 		goto x509_ocsp_cleanup;
 		}
 
-
 	// ok, now we verified the OCSP response. This means that we have a valid chain tying it
 	// to a root that we trust and that the signature also hopefully is valid. This does not yet
 	// mean that the ocsp response actually matches the certificate the server send us or that
@@ -322,7 +383,7 @@ function x509_ocsp_verify%(certs: x509_opaque_vector, ocsp_reply: string, root_c
 		goto x509_ocsp_cleanup;
 		}
 
-	if ( ! OCSP_id_cmp(certid, single->certId) )
+	if ( OCSP_id_cmp(certid, single->certId) != 0 )
 		return x509_result_record(-1, "OCSP reply is not for host certificate");
 
 	// next - check freshness of proof...

src/iosource/BPF_Program.cc

@@ -58,7 +58,14 @@ int pcap_compile_nopcap(int snaplen_arg, int linktype_arg,
 }
 #endif
 
-BPF_Program::BPF_Program() : m_compiled(), m_program()
+// Simple heuristic to identify filters that always match, so that we can
+// skip the filtering in that case. "ip or not ip" is Bro's default filter.
+static bool filter_matches_anything(const char *filter)
+	{
+	return (! filter) || strlen(filter) == 0 || strcmp(filter, "ip or not ip") == 0;
+	}
+
+BPF_Program::BPF_Program() : m_compiled(), m_matches_anything(false), m_program()
 	{
 	}
 
@@ -86,12 +93,14 @@ bool BPF_Program::Compile(pcap_t* pcap, const char* filter, uint32 netmask,
 		}
 
 	m_compiled = true;
+	m_matches_anything = filter_matches_anything(filter);
 
 	return true;
 	}
 
 bool BPF_Program::Compile(int snaplen, int linktype, const char* filter,
-				uint32 netmask, char* errbuf, bool optimize)
+			  uint32 netmask, char* errbuf, unsigned int errbuf_len,
+			  bool optimize)
 	{
 	FreeCode();
 
@@ -99,15 +108,23 @@ bool BPF_Program::Compile(int snaplen, int linktype, const char* filter,
 	char my_error[PCAP_ERRBUF_SIZE];
 
 	int err = pcap_compile_nopcap(snaplen, linktype, &m_program,
-				     (char *) filter, optimize, netmask, error);
+				     (char *) filter, optimize, netmask, my_error);
 	if ( err < 0 && errbuf )
-		safe_strncpy(errbuf, my_errbuf, PCAP_ERRBUF_SIZE);
+		safe_strncpy(errbuf, my_error, errbuf_len);
+		*errbuf = '\0';
 #else
 	int err = pcap_compile_nopcap(snaplen, linktype, &m_program,
 				     (char*) filter, optimize, netmask);
+
+	if ( err < 0 && errbuf && errbuf_len )
+		*errbuf = '\0';
 #endif
+
 	if ( err == 0 )
+		{
 		m_compiled = true;
+		m_matches_anything = filter_matches_anything(filter);
+		}
 
 	return err == 0;
 	}

src/iosource/BPF_Program.h

@@ -30,12 +30,17 @@ public:
 	// similarly to pcap_compile_nopcap(). Parameters are
 	// similar. Returns true on success.
 	bool Compile(int snaplen, int linktype, const char* filter,
-		uint32 netmask, char* errbuf = 0, bool optimize = true);
+		uint32 netmask, char* errbuf = 0, unsigned int errbuf_len = 0,
+		bool optimize = true);
 
 	// Returns true if this program currently contains compiled
 	// code, false otherwise.
 	bool IsCompiled()	{ return m_compiled; }
 
+	// Returns true if this program matches any packets. This is not
+	// comprehensive, but can identify a few cases where it does.
+	bool MatchesAnything() 	{ return m_matches_anything; }
+
 	// Accessor to the compiled program. Returns nil when
 	// no program is currently compiled.
 	bpf_program* GetProgram();
@@ -46,6 +51,7 @@ protected:
 	// (I like to prefix member variables with m_, makes it clear
 	// in the implementation whether it's a global or not. --ck)
 	bool m_compiled;
+	bool m_matches_anything;
 	struct bpf_program m_program;
 };
 

src/iosource/CMakeLists.txt

@@ -0,0 +1,23 @@
+
+include(BroSubdir)
+
+include_directories(BEFORE
+                    ${CMAKE_CURRENT_SOURCE_DIR}
+                    ${CMAKE_CURRENT_BINARY_DIR}
+)
+
+add_subdirectory(pcap)
+
+set(iosource_SRCS
+    BPF_Program.cc
+    Component.cc
+    Manager.cc
+    PktDumper.cc
+    PktSrc.cc
+)
+
+bif_target(pcap.bif)
+
+bro_add_subdir_library(iosource ${iosource_SRCS})
+add_dependencies(bro_iosource generate_outputs)
+

src/iosource/Component.cc

@@ -0,0 +1,165 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#include "Component.h"
+
+#include "Desc.h"
+#include "Reporter.h"
+
+using namespace iosource;
+
+Component::Component(const std::string& name)
+	: plugin::Component(plugin::component::IOSOURCE, name)
+	{
+	}
+
+Component::Component(plugin::component::Type type, const std::string& name)
+	: plugin::Component(type, name)
+	{
+	}
+
+Component::~Component()
+	{
+	}
+
+PktSrcComponent::PktSrcComponent(const std::string& arg_name, const std::string& arg_prefix, InputType arg_type, factory_callback arg_factory)
+	: iosource::Component(plugin::component::PKTSRC, arg_name)
+	{
+	tokenize_string(arg_prefix, ":", &prefixes);
+	type = arg_type;
+	factory = arg_factory;
+	}
+
+PktSrcComponent::~PktSrcComponent()
+	{
+	}
+
+const std::vector<std::string>& PktSrcComponent::Prefixes() const
+	{
+	return prefixes;
+	}
+
+bool PktSrcComponent::HandlesPrefix(const string& prefix) const
+	{
+	for ( std::vector<std::string>::const_iterator i = prefixes.begin();
+	      i != prefixes.end(); i++ )
+		{
+		if ( *i == prefix )
+			return true;
+		}
+
+	return false;
+	}
+
+bool PktSrcComponent::DoesLive() const
+	{
+	return type == LIVE || type == BOTH;
+	}
+
+bool PktSrcComponent::DoesTrace() const
+	{
+	return type == TRACE || type == BOTH;
+	}
+
+PktSrcComponent::factory_callback PktSrcComponent::Factory() const
+	{
+	return factory;
+	}
+
+void PktSrcComponent::DoDescribe(ODesc* d) const
+	{
+	iosource::Component::DoDescribe(d);
+
+	string prefs;
+
+	for ( std::vector<std::string>::const_iterator i = prefixes.begin();
+	      i != prefixes.end(); i++ )
+		{
+		if ( prefs.size() )
+			prefs += ", ";
+
+		prefs += '"' + *i + '"';
+		}
+
+	d->Add("interface prefix");
+	if ( prefixes.size() > 1 )
+		d->Add("es");
+
+	d->Add(" ");
+	d->Add(prefs);
+	d->Add("; supports ");
+
+	switch ( type ) {
+	case LIVE:
+		d->Add("live input");
+		break;
+
+	case TRACE:
+		d->Add("trace input");
+		break;
+
+	case BOTH:
+		d->Add("live and trace input");
+		break;
+
+	default:
+		reporter->InternalError("unknown PkrSrc type");
+	}
+
+	}
+
+PktDumperComponent::PktDumperComponent(const std::string& name, const std::string& arg_prefix, factory_callback arg_factory)
+	: plugin::Component(plugin::component::PKTDUMPER, name)
+	{
+	tokenize_string(arg_prefix, ":", &prefixes);
+	factory = arg_factory;
+	}
+
+PktDumperComponent::~PktDumperComponent()
+	{
+	}
+
+PktDumperComponent::factory_callback PktDumperComponent::Factory() const
+	{
+	return factory;
+	}
+
+const std::vector<std::string>& PktDumperComponent::Prefixes() const
+	{
+	return prefixes;
+	}
+
+bool PktDumperComponent::HandlesPrefix(const string& prefix) const
+	{
+	for ( std::vector<std::string>::const_iterator i = prefixes.begin();
+	      i != prefixes.end(); i++ )
+		{
+		if ( *i == prefix )
+			return true;
+		}
+
+	return false;
+	}
+
+void PktDumperComponent::DoDescribe(ODesc* d) const
+	{
+	plugin::Component::DoDescribe(d);
+
+	string prefs;
+
+	for ( std::vector<std::string>::const_iterator i = prefixes.begin();
+	      i != prefixes.end(); i++ )
+		{
+		if ( prefs.size() )
+			prefs += ", ";
+
+		prefs += '"' + *i + '"';
+		}
+
+	d->Add("dumper prefix");
+
+	if ( prefixes.size() > 1 )
+		d->Add("es");
+
+	d->Add(": ");
+	d->Add(prefs);
+	}

src/iosource/Component.h

@@ -0,0 +1,177 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#ifndef IOSOURCE_PLUGIN_COMPONENT_H
+#define IOSOURCE_PLUGIN_COMPONENT_H
+
+#include <string>
+#include <vector>
+
+#include "plugin/Component.h"
+
+namespace iosource {
+
+class IOSource;
+class PktSrc;
+class PktDumper;
+
+/**
+ * Component description for plugins providing IOSources.
+ */
+class Component : public plugin::Component {
+public:
+	typedef IOSource* (*factory_callback)();
+
+	/**
+	 * Constructor.
+	 *
+	 * @param name A descriptive name for the component.  This name must
+	 * be unique across all components of this type.
+	 */
+	Component(const std::string& name);
+
+	/**
+	 * Copy constructor.
+	 */
+	Component(const Component& other);
+
+	/**
+	 * Destructor.
+	 */
+	~Component();
+
+protected:
+	/**
+	 * Constructor to use by derived classes.
+	 *
+	 * @param type The type of the componnent.
+	 *
+	 * @param name A descriptive name for the component.  This name must
+	 * be unique across all components of this type.
+	 */
+	Component(plugin::component::Type type, const std::string& name);
+};
+
+/**
+ * Component description for plugins providing a PktSrc for packet input.
+ */
+class PktSrcComponent : public iosource::Component {
+public:
+	/**
+	 * Type of input a packet source supports.
+	 */
+	enum InputType {
+		LIVE,	///< Live input.
+		TRACE,		///< Offline input from trace file.
+		BOTH	///< Live input as well as offline.
+	};
+
+	typedef PktSrc* (*factory_callback)(const std::string& path, bool is_live);
+
+	/**
+	 * Constructor.
+	 *
+	 * @param name A descriptive name for the component.  This name must
+	 * be unique across all components of this type.
+	 *
+	 * @param prefixes The list of interface/file prefixes associated
+	 * with this component.
+	 *
+	 * @param type Type of input the component supports.
+	 *
+	 * @param factor Factory function to instantiate component.
+	 */
+	PktSrcComponent(const std::string& name, const std::string& prefixes, InputType type, factory_callback factory);
+
+	/**
+	 * Destructor.
+	 */
+	virtual ~PktSrcComponent();
+
+	/**
+	 * Returns the prefix(es) passed to the constructor.
+	 */
+	const std::vector<std::string>& Prefixes() const;
+
+	/**
+	 * Returns true if the given prefix is among the one specified for the component.
+	 */
+	bool HandlesPrefix(const std::string& prefix) const;
+
+	/**
+	 * Returns true if packet source instantiated by the component handle
+	 * live traffic.
+	 */
+	bool DoesLive() const;
+
+	/**
+	 * Returns true if packet source instantiated by the component handle
+	 * offline traces.
+	 */
+	bool DoesTrace() const;
+
+	/**
+	 * Returns the source's factory function.
+	 */
+	factory_callback Factory() const;
+
+	/**
+	 * Generates a human-readable description of the component. This goes
+	 * into the output of \c "bro -NN".
+	 */
+	virtual void DoDescribe(ODesc* d) const;
+
+private:
+	std::vector<std::string> prefixes;
+	InputType type;
+	factory_callback factory;
+};
+
+/**
+ * Component description for plugins providing a PktDumper for packet output.
+ *
+ * PktDumpers aren't IOSurces but we locate them here to keep them along with
+ * the PktSrc.
+ */
+class PktDumperComponent : public plugin::Component  {
+public:
+	typedef PktDumper* (*factory_callback)(const std::string& path, bool append);
+
+	/**
+	 * XXX
+	 */
+	PktDumperComponent(const std::string& name, const std::string& prefixes, factory_callback factory);
+
+	/**
+	 * Destructor.
+	 */
+	~PktDumperComponent();
+
+	/**
+	 * Returns the prefix(es) passed to the constructor.
+	 */
+	const std::vector<std::string>& Prefixes() const;
+
+	/**
+	 * Returns true if the given prefix is among the one specified for the component.
+	 */
+	bool HandlesPrefix(const std::string& prefix) const;
+
+	/**
+	 * Returns the source's factory function.
+	 */
+	factory_callback Factory() const;
+
+	/**
+	 * Generates a human-readable description of the component. This goes
+	 * into the output of \c "bro -NN".
+	 */
+	virtual void DoDescribe(ODesc* d) const;
+
+private:
+	std::vector<std::string> prefixes;
+	factory_callback factory;
+};
+
+}
+
+#endif

src/iosource/FD_Set.h

@@ -0,0 +1,104 @@
+#ifndef BRO_FD_SET_H
+#define BRO_FD_SET_H
+
+#include <set>
+#include <sys/select.h>
+
+namespace iosource {
+
+/**
+ * A container holding a set of file descriptors.
+ */
+class FD_Set {
+public:
+
+	/**
+	 * Constructor.  The set is initially empty.
+	 */
+	FD_Set() : max(-1), fds()
+		{ }
+
+	/**
+	 * Insert a file descriptor in to the set.
+	 * @param fd the fd to insert in the set.
+	 * @return false if fd was already in the set, else true.
+	 */
+	bool Insert(int fd)
+		{
+		if ( max < fd )
+			max = fd;
+
+		return fds.insert(fd).second;
+		}
+
+	/**
+	 * Inserts all the file descriptors from another set in to this one.
+	 * @param other a file descriptor set to merge in to this one.
+	 */
+	void Insert(const FD_Set& other)
+		{
+		for ( std::set<int>::const_iterator it = other.fds.begin();
+		      it != other.fds.end(); ++it )
+			Insert(*it);
+		}
+
+	/**
+	 * Empties the set.
+	 */
+	void Clear()
+		{ max = -1; fds.clear(); }
+
+	/**
+	 * Insert file descriptors in to a fd_set for use with select().
+	 * @return the greatest file descriptor inserted.
+	 */
+	int Set(fd_set* set) const
+		{
+		for ( std::set<int>::const_iterator it = fds.begin(); it != fds.end();
+		      ++it )
+			FD_SET(*it, set);
+
+		return max;
+		}
+
+	/**
+	 * @return Whether a file descriptor belonging to this set is within the
+	 *         fd_set arugment.
+	 */
+	bool Ready(fd_set* set) const
+		{
+		for ( std::set<int>::const_iterator it = fds.begin(); it != fds.end();
+		      ++it )
+			{
+			if ( FD_ISSET(*it, set) )
+				return true;
+			}
+
+		return false;
+		}
+
+	/**
+	 * @return whether any file descriptors have been added to the set.
+	 */
+	bool Empty() const
+		{
+		return fds.empty();
+		}
+
+	/**
+	 * @return the greatest file descriptor of all that have been added to the
+	 * set, or -1 if the set is empty.
+	 */
+	int Max() const
+		{
+		return max;
+		}
+
+private:
+	int max;
+	std::set<int> fds;
+};
+
+} // namespace bro
+
+#endif // BRO_FD_SET_H

src/iosource/IOSource.h

@@ -0,0 +1,140 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#ifndef IOSOURCE_IOSOURCE_H
+#define IOSOURCE_IOSOURCE_H
+
+extern "C" {
+#include <pcap.h>
+}
+
+#include <string>
+#include "FD_Set.h"
+#include "Timer.h"
+
+namespace iosource {
+
+/**
+ * Interface class for components providing/consuming data inside Bro's main
+ * loop.
+ */
+class IOSource {
+public:
+	/**
+	 * Constructor.
+	 */
+	IOSource()	{ idle = false; closed = false; }
+
+	/**
+	 * Destructor.
+	 */
+	virtual ~IOSource()	{}
+
+	/**
+	 * Returns true if source has nothing ready to process.
+	 */
+	bool IsIdle() const	{ return idle; }
+
+	/**
+	 * Returns true if more data is to be expected in the future.
+	 * Otherwise, source may be removed.
+	 */
+	bool IsOpen() const	{ return ! closed; }
+
+	/**
+	 * Initializes the source. Can be overwritten by derived classes.
+	 */
+	virtual void Init()	{ }
+
+	/**
+	 * Finalizes the source when it's being closed. Can be overwritten by
+	 * derived classes.
+	 */
+	virtual void Done()	{ }
+
+	/**
+	 * Returns select'able file descriptors for this source. Leaves the
+	 * passed values untouched if not available.
+	 *
+	 * @param read Pointer to container where to insert a read descriptor.
+	 *
+	 * @param write Pointer to container where to insert a write descriptor.
+	 *
+	 * @param except Pointer to container where to insert a except descriptor.
+	 */
+	virtual void GetFds(FD_Set* read, FD_Set* write, FD_Set* except) = 0;
+
+	/**
+	 * Returns the timestamp (in \a global network time) associated with
+	 * next data item from this source.  If the source wants the data
+	 * item to be processed with a local network time, it sets the
+	 * argument accordingly.
+	 *
+	 * This method will be called only when either IsIdle() returns
+	 * false, or select() on one of the fds returned by GetFDs()
+	 * indicates that there's data to process.
+	 *
+	 * Must be overridden by derived classes.
+	 *
+	 * @param network_time A pointer to store the \a local network time
+	 * associated with the next item (as opposed to global network time).
+	 *
+	 * @return The global network time of the next entry, or a value
+	 * smaller than zero if none is available currently.
+	 */
+	virtual double NextTimestamp(double* network_time) = 0;
+
+	/**
+	 * Processes and consumes next data item.
+	 *
+	 * This method will be called only when either IsIdle() returns
+	 * false, or select() on one of the fds returned by GetFDs()
+	 * indicates that there's data to process.
+	 *
+	 * Must be overridden by derived classes.
+	 */
+	virtual void Process() = 0;
+
+	/**
+	 * Returns the tag of the timer manafger associated with the last
+	 * procesees data item.
+	 *
+	 * Can be overridden by derived classes.
+	 *
+	 * @return The tag, or null for the global timer manager.
+	 * 
+	 */
+	virtual TimerMgr::Tag* GetCurrentTag()	{ return 0; }
+
+	/**
+	 * Returns a descriptual tag representing the source for debugging.
+	 *
+	 * Can be overridden by derived classes.
+	 *
+	 * @return The debugging name.
+	 */
+	virtual const char* Tag() = 0;
+
+protected:
+	/*
+	 * Callback for derived classes to call when they have gone dry
+	 * temporarily.
+	 *
+	 * @param is_idle True if the source is idle currently.
+	 */
+	void SetIdle(bool is_idle)	{ idle = is_idle; }
+
+	/*
+	 * Callback for derived class to call when they have shutdown.
+	 *
+	 * @param is_closed True if the source is now closed.
+	 */
+	void SetClosed(bool is_closed)	{ closed = is_closed; }
+
+private:
+	bool idle;
+	bool closed;
+};
+
+}
+
+#endif

src/iosource/Manager.cc

@@ -0,0 +1,314 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#include <sys/types.h>
+#include <sys/time.h>
+#include <unistd.h>
+#include <assert.h>
+
+#include <algorithm>
+
+#include "Manager.h"
+#include "IOSource.h"
+#include "PktSrc.h"
+#include "PktDumper.h"
+#include "plugin/Manager.h"
+
+#include "util.h"
+
+#define DEFAULT_PREFIX "pcap"
+
+using namespace iosource;
+
+Manager::~Manager()
+	{
+	for ( SourceList::iterator i = sources.begin(); i != sources.end(); ++i )
+		{
+		(*i)->src->Done();
+		delete *i;
+		}
+
+	sources.clear();
+
+	for ( PktDumperList::iterator i = pkt_dumpers.begin(); i != pkt_dumpers.end(); ++i )
+		{
+		(*i)->Done();
+		delete *i;
+		}
+
+	pkt_dumpers.clear();
+	}
+
+void Manager::RemoveAll()
+	{
+	// We're cheating a bit here ...
+	dont_counts = sources.size();
+	}
+
+IOSource* Manager::FindSoonest(double* ts)
+	{
+	// Remove sources which have gone dry. For simplicity, we only
+	// remove at most one each time.
+	for ( SourceList::iterator i = sources.begin();
+	      i != sources.end(); ++i )
+		if ( ! (*i)->src->IsOpen() )
+			{
+			(*i)->src->Done();
+			delete *i;
+			sources.erase(i);
+			break;
+			}
+
+	// Ideally, we would always call select on the fds to see which
+	// are ready, and return the soonest. Unfortunately, that'd mean
+	// one select-call per packet, which we can't afford in high-volume
+	// environments.  Thus, we call select only every SELECT_FREQUENCY
+	// call (or if all sources report that they are dry).
+
+	++call_count;
+
+	IOSource* soonest_src = 0;
+	double soonest_ts = 1e20;
+	double soonest_local_network_time = 1e20;
+	bool all_idle = true;
+
+	// Find soonest source of those which tell us they have something to
+	// process.
+	for ( SourceList::iterator i = sources.begin(); i != sources.end(); ++i )
+		{
+		if ( ! (*i)->src->IsIdle() )
+			{
+			all_idle = false;
+			double local_network_time = 0;
+			double ts = (*i)->src->NextTimestamp(&local_network_time);
+			if ( ts > 0 && ts < soonest_ts )
+				{
+				soonest_ts = ts;
+				soonest_src = (*i)->src;
+				soonest_local_network_time =
+					local_network_time ?
+						local_network_time : ts;
+				}
+			}
+		}
+
+	// If we found one and aren't going to select this time,
+	// return it.
+	int maxx = 0;
+
+	if ( soonest_src && (call_count % SELECT_FREQUENCY) != 0 )
+		goto finished;
+
+	// Select on the join of all file descriptors.
+	fd_set fd_read, fd_write, fd_except;
+
+	FD_ZERO(&fd_read);
+	FD_ZERO(&fd_write);
+	FD_ZERO(&fd_except);
+
+	for ( SourceList::iterator i = sources.begin();
+	      i != sources.end(); ++i )
+		{
+		Source* src = (*i);
+
+		if ( ! src->src->IsIdle() )
+			// No need to select on sources which we know to
+			// be ready.
+			continue;
+
+		src->Clear();
+		src->src->GetFds(&src->fd_read, &src->fd_write, &src->fd_except);
+		if ( src->fd_read.Empty() ) src->fd_read.Insert(0);
+		if ( src->fd_write.Empty() ) src->fd_write.Insert(0);
+		if ( src->fd_except.Empty() ) src->fd_except.Insert(0);
+		src->SetFds(&fd_read, &fd_write, &fd_except, &maxx);
+		}
+
+	// We can't block indefinitely even when all sources are dry:
+	// we're doing some IOSource-independent stuff in the main loop,
+	// so we need to return from time to time. (Instead of no time-out
+	// at all, we use a very small one. This lets FreeBSD trigger a
+	// BPF buffer switch on the next read when the hold buffer is empty
+	// while the store buffer isn't filled yet.
+
+	struct timeval timeout;
+
+	if ( all_idle )
+		{
+		// Interesting: when all sources are dry, simply sleeping a
+		// bit *without* watching for any fd becoming ready may
+		// decrease CPU load. I guess that's because it allows
+		// the kernel's packet buffers to fill. - Robin
+		timeout.tv_sec = 0;
+		timeout.tv_usec = 20; // SELECT_TIMEOUT;
+		select(0, 0, 0, 0, &timeout);
+		}
+
+	if ( ! maxx )
+		// No selectable fd at all.
+		goto finished;
+
+	timeout.tv_sec = 0;
+	timeout.tv_usec = 0;
+
+	if ( select(maxx + 1, &fd_read, &fd_write, &fd_except, &timeout) > 0 )
+		{ // Find soonest.
+		for ( SourceList::iterator i = sources.begin();
+		      i != sources.end(); ++i )
+			{
+			Source* src = (*i);
+
+			if ( ! src->src->IsIdle() )
+				continue;
+
+			if ( src->Ready(&fd_read, &fd_write, &fd_except) )
+				{
+				double local_network_time = 0;
+				double ts = src->src->NextTimestamp(&local_network_time);
+				if ( ts > 0.0 && ts < soonest_ts )
+					{
+					soonest_ts = ts;
+					soonest_src = src->src;
+					soonest_local_network_time =
+						local_network_time ?
+							local_network_time : ts;
+					}
+				}
+			}
+		}
+
+finished:
+	*ts = soonest_local_network_time;
+	return soonest_src;
+	}
+
+void Manager::Register(IOSource* src, bool dont_count)
+	{
+	src->Init();
+	Source* s = new Source;
+	s->src = src;
+	if ( dont_count )
+		++dont_counts;
+
+	sources.push_back(s);
+	}
+
+void Manager::Register(PktSrc* src)
+	{
+	pkt_srcs.push_back(src);
+	Register(src, false);
+	}
+
+static std::pair<std::string, std::string> split_prefix(std::string path)
+	{
+	// See if the path comes with a prefix telling us which type of
+	// PktSrc to use. If not, choose default.
+	std::string prefix;
+
+	std::string::size_type i = path.find("::");
+	if ( i != std::string::npos )
+		{
+		prefix = path.substr(0, i);
+		path = path.substr(i + 2, std::string::npos);
+		}
+
+	else
+		prefix= DEFAULT_PREFIX;
+
+	return std::make_pair(prefix, path);
+	}
+
+PktSrc* Manager::OpenPktSrc(const std::string& path, bool is_live)
+	{
+	std::pair<std::string, std::string> t = split_prefix(path);
+	std::string prefix = t.first;
+	std::string npath = t.second;
+
+	// Find the component providing packet sources of the requested prefix.
+
+	PktSrcComponent* component = 0;
+
+	std::list<PktSrcComponent*> all_components = plugin_mgr->Components<PktSrcComponent>();
+
+	for ( std::list<PktSrcComponent*>::const_iterator i = all_components.begin();
+	      i != all_components.end(); i++ )
+		{
+		PktSrcComponent* c = *i;
+
+		if ( c->HandlesPrefix(prefix) &&
+		     ((  is_live && c->DoesLive() ) ||
+		      (! is_live && c->DoesTrace())) )
+			{
+			component = c;
+			break;
+			}
+		}
+
+
+	if ( ! component )
+		reporter->FatalError("type of packet source '%s' not recognized, or mode not supported", prefix.c_str());
+
+	// Instantiate packet source.
+
+	PktSrc* ps = (*component->Factory())(npath, is_live);
+	assert(ps);
+
+	if ( ! ps->IsOpen() && ps->IsError() )
+		// Set an error message if it didn't open successfully.
+		ps->Error("could not open");
+
+	DBG_LOG(DBG_PKTIO, "Created packet source of type %s for %s", component->Name().c_str(), npath.c_str());
+
+	Register(ps);
+	return ps;
+	}
+
+
+PktDumper* Manager::OpenPktDumper(const string& path, bool append)
+	{
+	std::pair<std::string, std::string> t = split_prefix(path);
+	std::string prefix = t.first;
+	std::string npath = t.second;
+
+	// Find the component providing packet dumpers of the requested prefix.
+
+	PktDumperComponent* component = 0;
+
+	std::list<PktDumperComponent*> all_components = plugin_mgr->Components<PktDumperComponent>();
+
+	for ( std::list<PktDumperComponent*>::const_iterator i = all_components.begin();
+	      i != all_components.end(); i++ )
+		{
+		if ( (*i)->HandlesPrefix(prefix) )
+			{
+			component = (*i);
+			break;
+			}
+		}
+
+	if ( ! component )
+		reporter->FatalError("type of packet dumper '%s' not recognized", prefix.c_str());
+
+	// Instantiate packet dumper.
+
+	PktDumper* pd = (*component->Factory())(npath, append);
+	assert(pd);
+
+	if ( ! pd->IsOpen() && pd->IsError() )
+		// Set an error message if it didn't open successfully.
+		pd->Error("could not open");
+
+	DBG_LOG(DBG_PKTIO, "Created packer dumper of type %s for %s", component->Name().c_str(), npath.c_str());
+
+	pd->Init();
+	pkt_dumpers.push_back(pd);
+
+	return pd;
+	}
+
+void Manager::Source::SetFds(fd_set* read, fd_set* write, fd_set* except,
+                             int* maxx) const
+	{
+	*maxx = std::max(*maxx, fd_read.Set(read));
+	*maxx = std::max(*maxx, fd_write.Set(write));
+	*maxx = std::max(*maxx, fd_except.Set(except));
+	}

src/iosource/Manager.h

@@ -0,0 +1,146 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#ifndef IOSOURCE_MANAGER_H
+#define IOSOURCE_MANAGER_H
+
+#include <string>
+#include <list>
+#include "iosource/FD_Set.h"
+
+namespace iosource {
+
+class IOSource;
+class PktSrc;
+class PktDumper;
+
+/**
+ * Singleton class managing all IOSources.
+ */
+class Manager {
+public:
+	/**
+	 * Constructor.
+	 */
+	Manager()	{ call_count = 0; dont_counts = 0; }
+
+	/**
+	 * Destructor.
+	 */
+	~Manager();
+
+	/**
+	 * Registers an IOSource with the manager.
+	 *
+	 * @param src The source. The manager takes ownership.
+	 *
+	 * @param dont_count If true, this source does not contribute to the
+	 * number of IOSources returned by Size().  The effect is that if all
+	 * sources except for the non-counting ones have gone dry, processing
+	 * will shut down.
+	 */
+	void Register(IOSource* src, bool dont_count = false);
+
+	/**
+	 * Returns the packet source with the soonest available input. This
+	 * may block for a little while if all are dry.
+	 *
+	 * @param ts A pointer where to store the timestamp of the input that
+	 * the soonest source has available next.
+	 *
+	 * @return The source, or null if no source has input.
+	 */
+	IOSource* FindSoonest(double* ts);
+
+	/**
+	 * Returns the number of registered and still active sources,
+	 * excluding those that are registered as \a dont_cont.
+	 */
+	int Size() const	{ return sources.size() - dont_counts; }
+
+	typedef std::list<PktSrc *> PktSrcList;
+
+	/**
+	 * Returns a list of all registered PktSrc instances. This is a
+	 * subset of all registered IOSource instances.
+	 */
+	const PktSrcList& GetPktSrcs() const	{ return pkt_srcs; }
+
+	/**
+	 * Terminate all processing immediately by removing all sources (and
+	 * therefore now returning a Size() of zero).
+	 */
+	void Terminate()	{ RemoveAll(); }
+
+	/**
+	 * Opens a new packet source.
+	 *
+	 * @param path The interface or file name, as one would give to Bro \c -i.
+	 *
+	 * @param is_live True if \a path represents a live interface, false
+	 * for a file.
+	 *
+	 * @return The new packet source, or null if an error occured.
+	 */
+	PktSrc* OpenPktSrc(const std::string& path, bool is_live);
+
+	/**
+	 * Opens a new packet dumper.
+	 *
+	 * @param path The file name to dump into.
+	 *
+	 * @param append True to append if \a path already exists.
+ 	 *
+	 * @return The new packet dumper, or null if an error occured.
+	 */
+	PktDumper* OpenPktDumper(const std::string& path, bool append);
+
+private:
+	/**
+	 * When looking for a source with something to process, every
+	 * SELECT_FREQUENCY calls we will go ahead and block on a select().
+	 */
+	static const int SELECT_FREQUENCY = 25;
+
+	/**
+	 * Microseconds to wait in an empty select if no source is ready.
+	 */
+	static const int SELECT_TIMEOUT = 50;
+
+	void Register(PktSrc* src);
+	void RemoveAll();
+
+	unsigned int call_count;
+	int dont_counts;
+
+	struct Source {
+		IOSource* src;
+		FD_Set fd_read;
+		FD_Set fd_write;
+		FD_Set fd_except;
+
+		bool Ready(fd_set* read, fd_set* write, fd_set* except) const
+			{ return fd_read.Ready(read) || fd_write.Ready(write) ||
+			         fd_except.Ready(except); }
+
+		void SetFds(fd_set* read, fd_set* write, fd_set* except,
+		            int* maxx) const;
+
+		void Clear()
+			{ fd_read.Clear(); fd_write.Clear(); fd_except.Clear(); }
+	};
+
+	typedef std::list<Source*> SourceList;
+	SourceList sources;
+
+	typedef std::list<PktDumper *> PktDumperList;
+
+	PktSrcList pkt_srcs;
+	PktDumperList pkt_dumpers;
+};
+
+}
+
+extern iosource::Manager* iosource_mgr;
+
+#endif
+

src/iosource/PktDumper.cc

@@ -0,0 +1,84 @@
+
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#include <errno.h>
+#include <sys/stat.h>
+
+#include "config.h"
+
+#include "PktDumper.h"
+
+using namespace iosource;
+
+PktDumper::PktDumper()
+	{
+	is_open = false;
+	errmsg = "";
+	}
+
+PktDumper::~PktDumper()
+	{
+	}
+
+void PktDumper::Init()
+	{
+	Open();
+	}
+
+void PktDumper::Done()
+	{
+	Close();
+	}
+
+const std::string& PktDumper::Path() const
+	{
+	return props.path;
+	}
+
+bool PktDumper::IsOpen() const
+	{
+	return is_open;
+	}
+
+double PktDumper::OpenTime() const
+	{
+	return is_open ? props.open_time : 0;
+	}
+
+bool PktDumper::IsError() const
+	{
+	return errmsg.size();
+	}
+
+const char* PktDumper::ErrorMsg() const
+	{
+	return errmsg.size() ? errmsg.c_str() : 0;
+	}
+
+int PktDumper::HdrSize() const
+	{
+	return is_open ? props.hdr_size : -1;
+	}
+
+void PktDumper::Opened(const Properties& arg_props)
+	{
+	is_open = true;
+	props = arg_props;
+	DBG_LOG(DBG_PKTIO, "Opened dumper %s", props.path.c_str());
+	}
+
+void PktDumper::Closed()
+	{
+	is_open = false;
+	DBG_LOG(DBG_PKTIO, "Closed dumper %s", props.path.c_str());
+	props.path = "";
+	}
+
+void PktDumper::Error(const std::string& msg)
+	{
+	errmsg = msg;
+
+	DBG_LOG(DBG_PKTIO, "Error with dumper %s: %s",
+		IsOpen() ? props.path.c_str() : "<not open>",
+		msg.c_str());
+	}

src/iosource/PktDumper.h

@@ -0,0 +1,163 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#ifndef IOSOURCE_PKTSRC_PKTDUMPER_H
+#define IOSOURCE_PKTSRC_PKTDUMPER_H
+
+#include "IOSource.h"
+
+namespace iosource {
+
+/**
+ * Base class for packet dumpers.
+ */
+class PktDumper {
+public:
+	/**
+	 * Structure describing a packet.
+	 */
+	struct Packet {
+		/**
+		 * The pcap header associated with the packet.
+		 */
+		const struct pcap_pkthdr* hdr;
+
+		/**
+		 * The full content of the packet.
+		 */
+		const unsigned char* data;
+	};
+
+	/**
+	 * Constructor.
+	 */
+	PktDumper();
+
+	/**
+	 * Destructor.
+	 */
+	virtual ~PktDumper();
+
+	/**
+	 * Returns the path associated with the dumper.
+	 */
+	const std::string& Path() const;
+
+	/**
+	 * Returns true if the dumper is open for writing.
+	 */
+	bool IsOpen() const;
+
+	/**
+	 * Returns the time when the dumper was opened for writing.
+	 */
+	double OpenTime() const;
+
+	/**
+	 * Returns returns true if the dumper has encountered an error.
+	 */
+	bool IsError() const;
+
+	/**
+	 * Returns if the dumper has encountered an error, returns a
+	 * corresponding error message. Returns an emoty string otherwise.
+	 */
+	const char* ErrorMsg() const;
+
+	/**
+	 * Returns the size of the link-layer headers with this dumper.
+	 */
+	int HdrSize() const;
+
+	/**
+	 * Writes a packet to the dumper.
+	 *
+	 * @param pkt The packet to record.
+	 */
+	bool Record(const Packet* pkt);
+
+	// PktDumper interface for derived classes to implement.
+
+	/**
+	 * Called by the manager system to open the source.
+	 *
+	 * Derived classes must implement this method. If successful, the
+	 * implementation must call \a Opened(); if not, it must call Error()
+	 * with a corresponding message.
+	 */
+	virtual void Open() = 0;
+
+	/**
+	 * Called by the manager system to close the dumper.
+	 *
+	 * Derived classes must implement this method. If successful, the
+	 * implementation must call \a Closed(); if not, it must call Error()
+	 * with a corresponding message.
+	 */
+	virtual void Close() = 0;
+
+	/**
+	 * Called to write a packet to the dumper.
+	 *
+	 * Derived classes must implement this method.
+	 *
+	 * @param pkt The packet to record.
+	 *
+	 * @return True if succesful, false otherwise (in which case \a
+	 * Error() must have been called.)
+	 */
+	virtual bool Dump(const Packet* pkt) = 0;
+
+protected:
+	friend class Manager;
+
+	/**
+	 * Structure to pass back information about the packet dumper to the
+	 * base class. Derived class pass an instance of this to \a Opened().
+	 */
+	struct Properties {
+		std::string path;
+		int hdr_size;
+		double open_time;
+	};
+
+	/**
+	 * Called from the implementations of \a Open() to signal that the
+	 * source has been successully opened.
+	 *
+	 * @param props A properties instance describing the now open source.
+	 */
+	void Opened(const Properties& props);
+
+	/**
+	 * Called from the implementations of \a Close() to signal that the
+	 * source has been closed.
+	 */
+	void Closed();
+
+	/**
+	 * Called from derived classes to signal an error.
+	 *
+	 * @param msg A corresponding error message.
+	 */
+	void Error(const std::string& msg);
+
+	/**
+	 * Called by the manager to initialize the dumper.
+	 */
+	void Init();
+
+	/**
+	 * Called by the manager to shutdown the dumper.
+	 */
+	void Done();
+
+private:
+	bool is_open;
+	Properties props;
+
+	std::string errmsg;
+};
+
+}
+
+#endif

src/iosource/PktSrc.cc

@@ -0,0 +1,525 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#include <errno.h>
+#include <sys/stat.h>
+
+#include "config.h"
+
+#include "util.h"
+#include "PktSrc.h"
+#include "Hash.h"
+#include "Net.h"
+#include "Sessions.h"
+
+using namespace iosource;
+
+PktSrc::PktSrc()
+	{
+	have_packet = false;
+	errbuf = "";
+	SetClosed(true);
+
+	next_sync_point = 0;
+	first_timestamp = 0.0;
+	first_wallclock = current_wallclock = 0;
+	}
+
+PktSrc::~PktSrc()
+	{
+	BPF_Program* code;
+	IterCookie* cookie = filters.InitForIteration();
+	while ( (code = filters.NextEntry(cookie)) )
+		delete code;
+	}
+
+const std::string& PktSrc::Path() const
+	{
+	static std::string not_open("not open");
+	return IsOpen() ? props.path : not_open;
+	}
+
+const char* PktSrc::ErrorMsg() const
+	{
+	return errbuf.size() ? errbuf.c_str() : 0;
+	}
+
+int PktSrc::LinkType() const
+	{
+	return IsOpen() ? props.link_type : -1;
+	}
+
+uint32 PktSrc::Netmask() const
+	{
+	return IsOpen() ? props.netmask : PCAP_NETMASK_UNKNOWN;
+	}
+
+bool PktSrc::IsError() const
+	{
+	return ErrorMsg();
+	}
+
+int PktSrc::HdrSize() const
+	{
+	return IsOpen() ? props.hdr_size : -1;
+	}
+
+int PktSrc::SnapLen() const
+	{
+	return snaplen; // That's a global. Change?
+	}
+
+bool PktSrc::IsLive() const
+	{
+	return props.is_live;
+	}
+
+double PktSrc::CurrentPacketTimestamp()
+	{
+	return current_pseudo;
+	}
+
+double PktSrc::CurrentPacketWallClock()
+	{
+	// We stop time when we are suspended.
+	if ( net_is_processing_suspended() )
+		current_wallclock = current_time(true);
+
+	return current_wallclock;
+	}
+
+void PktSrc::Opened(const Properties& arg_props)
+	{
+	if ( arg_props.hdr_size < 0 )
+		{
+		char buf[512];
+		safe_snprintf(buf, sizeof(buf),
+			 "unknown data link type 0x%x", props.link_type);
+		Error(buf);
+		Close();
+		return;
+		}
+
+	props = arg_props;
+	SetClosed(false);
+
+	if ( ! PrecompileFilter(0, "") || ! SetFilter(0) )
+		{
+		Close();
+		return;
+		}
+
+	if ( props.is_live )
+		Info(fmt("listening on %s, capture length %d bytes\n", props.path.c_str(), SnapLen()));
+
+	DBG_LOG(DBG_PKTIO, "Opened source %s", props.path.c_str());
+	}
+
+void PktSrc::Closed()
+	{
+	SetClosed(true);
+
+	DBG_LOG(DBG_PKTIO, "Closed source %s", props.path.c_str());
+	}
+
+void PktSrc::Error(const std::string& msg)
+	{
+	// We don't report this immediately, Bro will ask us for the error
+	// once it notices we aren't open.
+	errbuf = msg;
+	DBG_LOG(DBG_PKTIO, "Error with source %s: %s",
+		IsOpen() ? props.path.c_str() : "<not open>",
+		msg.c_str());
+	}
+
+void PktSrc::Info(const std::string& msg)
+	{
+	reporter->Info("%s", msg.c_str());
+	}
+
+void PktSrc::Weird(const std::string& msg, const Packet* p)
+	{
+	sessions->Weird(msg.c_str(), p->hdr, p->data, 0);
+	}
+
+void PktSrc::InternalError(const std::string& msg)
+	{
+	reporter->InternalError("%s", msg.c_str());
+	}
+
+void PktSrc::ContinueAfterSuspend()
+	{
+	current_wallclock = current_time(true);
+	}
+
+int PktSrc::GetLinkHeaderSize(int link_type)
+	{
+	switch ( link_type ) {
+	case DLT_NULL:
+		return 4;
+
+	case DLT_EN10MB:
+		return 14;
+
+	case DLT_FDDI:
+		return 13 + 8;	// fddi_header + LLC
+
+#ifdef DLT_LINUX_SLL
+	case DLT_LINUX_SLL:
+		return 16;
+#endif
+
+	case DLT_PPP_SERIAL:	// PPP_SERIAL
+		return 4;
+
+	case DLT_RAW:
+		return 0;
+	}
+
+	return -1;
+	}
+
+double PktSrc::CheckPseudoTime()
+	{
+	if ( ! IsOpen() )
+		return 0;
+
+	if ( ! ExtractNextPacketInternal() )
+		return 0;
+
+	if ( remote_trace_sync_interval )
+		{
+		if ( next_sync_point == 0 || current_packet.ts >= next_sync_point )
+			{
+			int n = remote_serializer->SendSyncPoint();
+			next_sync_point = first_timestamp +
+						n * remote_trace_sync_interval;
+			remote_serializer->Log(RemoteSerializer::LogInfo,
+				fmt("stopping at packet %.6f, next sync-point at %.6f",
+					current_packet.ts, next_sync_point));
+
+			return 0;
+			}
+		}
+
+	double pseudo_time = current_packet.ts - first_timestamp;
+	double ct = (current_time(true) - first_wallclock) * pseudo_realtime;
+
+	return pseudo_time <= ct ? bro_start_time + pseudo_time : 0;
+	}
+
+void PktSrc::Init()
+	{
+	Open();
+	}
+
+void PktSrc::Done()
+	{
+	if ( IsOpen() )
+		Close();
+	}
+
+void PktSrc::GetFds(iosource::FD_Set* read, iosource::FD_Set* write,
+                    iosource::FD_Set* except)
+	{
+	if ( pseudo_realtime )
+		{
+		// Select would give erroneous results. But we simulate it
+		// by setting idle accordingly.
+		SetIdle(CheckPseudoTime() == 0);
+		return;
+		}
+
+	if ( IsOpen() && props.selectable_fd >= 0 )
+		read->Insert(props.selectable_fd);
+	}
+
+double PktSrc::NextTimestamp(double* local_network_time)
+	{
+	if ( ! IsOpen() )
+		return -1.0;
+
+	if ( ! ExtractNextPacketInternal() )
+		return -1.0;
+
+	if ( pseudo_realtime )
+		{
+		// Delay packet if necessary.
+		double packet_time = CheckPseudoTime();
+		if ( packet_time )
+			return packet_time;
+
+		SetIdle(true);
+		return -1.0;
+		}
+
+	return current_packet.ts;
+	}
+
+void PktSrc::Process()
+	{
+	if ( ! IsOpen() )
+		return;
+
+	if ( ! ExtractNextPacketInternal() )
+		return;
+
+	int pkt_hdr_size = props.hdr_size;
+
+	// Unfortunately some packets on the link might have MPLS labels
+	// while others don't. That means we need to ask the link-layer if
+	// labels are in place.
+	bool have_mpls = false;
+
+	int protocol = 0;
+	const u_char* data = current_packet.data;
+
+	switch ( props.link_type ) {
+	case DLT_NULL:
+		{
+		protocol = (data[3] << 24) + (data[2] << 16) + (data[1] << 8) + data[0];
+
+		// From the Wireshark Wiki: "AF_INET6, unfortunately, has
+		// different values in {NetBSD,OpenBSD,BSD/OS},
+		// {FreeBSD,DragonFlyBSD}, and {Darwin/Mac OS X}, so an IPv6
+		// packet might have a link-layer header with 24, 28, or 30
+		// as the AF_ value." As we may be reading traces captured on
+		// platforms other than what we're running on, we accept them
+		// all here.
+		if ( protocol != AF_INET
+		     && protocol != AF_INET6
+		     && protocol != 24
+		     && protocol != 28
+		     && protocol != 30 )
+			{
+			Weird("non_ip_packet_in_null_transport", &current_packet);
+			goto done;
+			}
+
+		break;
+		}
+
+	case DLT_EN10MB:
+		{
+		// Get protocol being carried from the ethernet frame.
+		protocol = (data[12] << 8) + data[13];
+
+		switch ( protocol )
+			{
+			// MPLS carried over the ethernet frame.
+			case 0x8847:
+				// Remove the data link layer and denote a
+				// header size of zero before the IP header.
+				have_mpls = true;
+				data += GetLinkHeaderSize(props.link_type);
+				pkt_hdr_size = 0;
+				break;
+
+			// VLAN carried over the ethernet frame.
+			case 0x8100:
+				data += GetLinkHeaderSize(props.link_type);
+
+				// Check for MPLS in VLAN.
+				if ( ((data[2] << 8) + data[3]) == 0x8847 )
+					have_mpls = true;
+
+				data += 4; // Skip the vlan header
+				pkt_hdr_size = 0;
+
+				// Check for 802.1ah (Q-in-Q) containing IP.
+				// Only do a second layer of vlan tag
+				// stripping because there is no
+				// specification that allows for deeper
+				// nesting.
+				if ( ((data[2] << 8) + data[3]) == 0x0800 )
+					data += 4;
+
+				break;
+
+			// PPPoE carried over the ethernet frame.
+			case 0x8864:
+				data += GetLinkHeaderSize(props.link_type);
+				protocol = (data[6] << 8) + data[7];
+				data += 8; // Skip the PPPoE session and PPP header
+				pkt_hdr_size = 0;
+
+				if ( protocol != 0x0021 && protocol != 0x0057 )
+					{
+					// Neither IPv4 nor IPv6.
+					Weird("non_ip_packet_in_pppoe_encapsulation", &current_packet);
+					goto done;
+					}
+				break;
+			}
+
+		break;
+		}
+
+	case DLT_PPP_SERIAL:
+		{
+		// Get PPP protocol.
+		protocol = (data[2] << 8) + data[3];
+
+		if ( protocol == 0x0281 )
+			{
+			// MPLS Unicast. Remove the data link layer and
+			// denote a header size of zero before the IP header.
+			have_mpls = true;
+				data += GetLinkHeaderSize(props.link_type);
+			pkt_hdr_size = 0;
+			}
+
+		else if ( protocol != 0x0021 && protocol != 0x0057 )
+			{
+			// Neither IPv4 nor IPv6.
+			Weird("non_ip_packet_in_ppp_encapsulation", &current_packet);
+			goto done;
+			}
+		break;
+		}
+	}
+
+	if ( have_mpls )
+		{
+		// Skip the MPLS label stack.
+		bool end_of_stack = false;
+
+		while ( ! end_of_stack )
+			{
+			end_of_stack = *(data + 2) & 0x01;
+			data += 4;
+			}
+		}
+
+	if ( pseudo_realtime )
+		{
+		current_pseudo = CheckPseudoTime();
+		net_packet_dispatch(current_pseudo, current_packet.hdr, data, pkt_hdr_size, this);
+		if ( ! first_wallclock )
+			first_wallclock = current_time(true);
+		}
+
+	else
+		net_packet_dispatch(current_packet.ts, current_packet.hdr, data, pkt_hdr_size, this);
+
+done:
+	have_packet = 0;
+	DoneWithPacket();
+	}
+
+const char* PktSrc::Tag()
+	{
+	return "PktSrc";
+	}
+
+bool PktSrc::ExtractNextPacketInternal()
+	{
+	if ( have_packet )
+		return true;
+
+	have_packet = false;
+
+	// Don't return any packets if processing is suspended (except for the
+	// very first packet which we need to set up times).
+	if ( net_is_processing_suspended() && first_timestamp )
+		{
+		SetIdle(true);
+		return 0;
+		}
+
+	if ( pseudo_realtime )
+		current_wallclock = current_time(true);
+
+	if ( ExtractNextPacket(&current_packet) )
+		{
+		if ( ! first_timestamp )
+			first_timestamp = current_packet.ts;
+
+		SetIdle(false);
+		have_packet = true;
+		return 1;
+		}
+
+	if ( pseudo_realtime && using_communication && ! IsOpen() )
+		{
+		// Source has gone dry, we're done.
+		if ( remote_trace_sync_interval )
+			remote_serializer->SendFinalSyncPoint();
+		else
+			remote_serializer->Terminate();
+		}
+
+	SetIdle(true);
+	return 0;
+	}
+
+bool PktSrc::PrecompileBPFFilter(int index, const std::string& filter)
+	{
+	if ( index < 0 )
+		return false;
+
+	char errbuf[PCAP_ERRBUF_SIZE];
+
+	// Compile filter.
+	BPF_Program* code = new BPF_Program();
+
+	if ( ! code->Compile(SnapLen(), LinkType(), filter.c_str(), Netmask(), errbuf, sizeof(errbuf)) )
+		{
+		string msg = fmt("cannot compile BPF filter \"%s\"", filter.c_str());
+
+		if ( *errbuf )
+			msg += ": " + string(errbuf);
+
+		Error(msg);
+
+		delete code;
+		return 0;
+		}
+
+	// Store it in hash.
+	HashKey* hash = new HashKey(HashKey(bro_int_t(index)));
+	BPF_Program* oldcode = filters.Lookup(hash);
+	if ( oldcode )
+		delete oldcode;
+
+	filters.Insert(hash, code);
+	delete hash;
+
+	return 1;
+	}
+
+BPF_Program* PktSrc::GetBPFFilter(int index)
+	{
+	if ( index < 0 )
+		return 0;
+
+	HashKey* hash = new HashKey(HashKey(bro_int_t(index)));
+	BPF_Program* code = filters.Lookup(hash);
+	delete hash;
+	return code;
+	}
+
+bool PktSrc::ApplyBPFFilter(int index, const struct pcap_pkthdr *hdr, const u_char *pkt)
+	{
+	BPF_Program* code = GetBPFFilter(index);
+
+	if ( ! code )
+		{
+		Error(fmt("BPF filter %d not compiled", index));
+		Close();
+		}
+
+	if ( code->MatchesAnything() )
+		return true;
+
+	return pcap_offline_filter(code->GetProgram(), hdr, pkt);
+	}
+
+bool PktSrc::GetCurrentPacket(const pcap_pkthdr** hdr, const u_char** pkt)
+	{
+	if ( ! have_packet )
+		return false;
+
+	*hdr = current_packet.hdr;
+	*pkt = current_packet.data;
+	return true;
+	}

src/iosource/PktSrc.h

@@ -0,0 +1,417 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#ifndef IOSOURCE_PKTSRC_PKTSRC_H
+#define IOSOURCE_PKTSRC_PKTSRC_H
+
+#include "IOSource.h"
+#include "BPF_Program.h"
+#include "Dict.h"
+
+declare(PDict,BPF_Program);
+
+namespace iosource {
+
+/**
+ * Base class for packet sources.
+ */
+class PktSrc : public IOSource {
+public:
+	/**
+	 * Struct for returning statistics on a packet source.
+	 */
+	struct Stats {
+		/**
+		 * Packets received by source after filtering (w/o drops).
+		 */
+		unsigned int received;
+
+		/**
+		 * Packets dropped by source.
+		 */
+		unsigned int dropped;	// pkts dropped
+
+		/**
+		 * Total number of packets on link before filtering.
+		 * Optional, can be left unset if not available.
+		 */
+		unsigned int link;
+
+		Stats()	{ received = dropped = link = 0; }
+	};
+
+	/**
+	 * Constructor.
+	 */
+	PktSrc();
+
+	/**
+	 * Destructor.
+	 */
+	virtual ~PktSrc();
+
+	/**
+	 * Returns the path associated with the source. This is the interface
+	 * name for live source, and a filename for offline sources.
+	 */
+	const std::string& Path() const;
+
+	/**
+	 * Returns true if this is a live source.
+	 */
+	bool IsLive() const;
+
+	/**
+	 * Returns the link type of the source.
+	 */
+	int LinkType() const;
+
+	/**
+	 * Returns the netmask associated with the source, or \c
+	 * PCAP_NETMASK_UNKNOWN if unknown.
+	 */
+	uint32 Netmask() const;
+
+	/**
+	 * Returns true if the source has flagged an error.
+	 */
+	bool IsError() const;
+
+	/**
+	 * If the source encountered an error, returns a corresponding error
+	 * message. Returns an empty string otherwise.
+	 */
+	const char* ErrorMsg() const;
+
+	/**
+	 * Returns the size of the link-layer header for this source.
+	 */
+	int HdrSize() const;
+
+	/**
+	 * Returns the snap length for this source.
+	 */
+	int SnapLen() const;
+
+	/**
+	 * In pseudo-realtime mode, returns the logical timestamp of the
+	 * current packet. Undefined if not running pseudo-realtime mode.
+	 */
+	double CurrentPacketTimestamp();
+
+	/**
+	 * In pseudo-realtime mode, returns the wall clock time associated
+	 * with current packet. Undefined if not running pseudo-realtime
+	 * mode.
+	 */
+	double CurrentPacketWallClock();
+
+	/**
+	 * Signals packet source that processing is going to be continued
+	 * after previous suspension.
+	 */
+	void ContinueAfterSuspend();
+
+	/**
+	 * Precompiles a BPF filter and associates the given index with it.
+	 * The compiled filter will be then available via \a GetBPFFilter().
+	 *
+	 * This is primarily a helper for packet source implementation that
+	 * want to apply BPF filtering to their packets.
+	 *
+	 * @param index The index to associate with the filter.
+	 *
+	 * @param BPF filter The filter string to precompile.
+	 *
+	 * @return True on success, false if a problem occurred.
+	 */
+	bool PrecompileBPFFilter(int index, const std::string& filter);
+
+	/**
+	 * Returns the precompiled BPF filter associated with a given index,
+	 * if any, as compiled by \a PrecompileBPFFilter().
+	 *
+	 * This is primarily a helper for packet source implementation that
+	 * want to apply BPF filtering to their packets.
+	 *
+	 * @return The BPF filter associated, or null if none has been
+	 * (successfully) compiled.
+	 */
+	BPF_Program* GetBPFFilter(int index);
+
+	/**
+	 * Applies a precompiled BPF filter to a packet. This will close the
+	 * source with an error message if no filter with that index has been
+	 * compiled.
+	 *
+	 * This is primarily a helper for packet source implementation that
+	 * want to apply BPF filtering to their packets.
+	 *
+	 * @param index The index of the filter to apply.
+	 *
+	 * @param hdr The header of the packet to filter.
+	 *
+	 * @param pkt The content of the packet to filter.
+	 *
+	 * @return True if it maches. 	 */
+	bool ApplyBPFFilter(int index, const struct pcap_pkthdr *hdr, const u_char *pkt);
+
+	/**
+	 * Returns the packet currently being processed, if available.
+	 *
+	 * @param hdr A pointer to pass the header of the current packet back.
+	 *
+	 * @param pkt A pointer to pass the content of the current packet
+	 * back.
+	 *
+	 * @return True if the current packet is available, or false if not.
+	 */
+	bool GetCurrentPacket(const pcap_pkthdr** hdr, const u_char** pkt);
+
+	// PacketSource interace for derived classes to override.
+
+	/**
+	 * Precompiles a filter and associates a given index with it. The
+	 * filter syntax is defined by the packet source's implenentation.
+	 *
+	 * Derived classes must implement this to implement their filtering.
+	 * If they want to use BPF but don't support it natively, they can
+	 * call the corresponding helper method provided by \a PktSrc.
+	 *
+	 * @param index The index to associate with the filter
+	 *
+	 * @param filter The filter string to precompile.
+	 *
+	 * @return True on success, false if a problem occurred or filtering
+	 * is not supported.
+	 */
+	virtual bool PrecompileFilter(int index, const std::string& filter) = 0;
+
+	/**
+	 * Activates a precompiled filter with the given index.
+	 *
+	 * Derived classes must implement this to implement their filtering.
+	 * If they want to use BPF but don't support it natively, they can
+	 * call the corresponding helper method provided by \a PktSrc.
+	 *
+	 * @param index The index of the filter to activate.
+	 *
+	 * @return True on success, false if a problem occurred or the
+	 * filtering is not supported.
+	 */
+	virtual bool SetFilter(int index) = 0;
+
+	/**
+	 * Returns current statistics about the source.
+	 *
+	 * Derived classes must implement this method.
+	 *
+	 * @param stats A statistics structure that the method fill out.
+	 */
+	virtual void Statistics(Stats* stats) = 0;
+
+	/**
+	 * Helper method to return the header size for a given link tyoe.
+	 *
+	 * @param link_type The link tyoe.
+	 *
+	 * @return The header size in bytes.
+	 */
+	static int GetLinkHeaderSize(int link_type);
+
+protected:
+	friend class Manager;
+
+	// Methods to use by derived classes.
+
+	/**
+	 * Structure to pass back information about the packet source to the
+	 * base class. Derived class pass an instance of this to \a Opened().
+	 */
+	struct Properties {
+		/**
+		 * The path associated with the source. This is the interface
+		 * name for live source, and a filename for offline sources.
+		 */
+		std::string path;
+
+		/**
+		 * A file descriptor suitable to use with \a select() for
+		 * determining if there's input available from this source.
+		 */
+		int selectable_fd;
+
+		/**
+		 * The link type for packets from this source.
+		 */
+		int link_type;
+
+		/**
+		 * The size of the link-layer header for packets from this
+		 * source. \a GetLinkHeaderSize() may be used to derive this
+		 * value.
+		 */
+		int hdr_size;
+
+		/**
+		 * The netmask associated with the source, or \c
+		 * PCAP_NETMASK_UNKNOWN if unknown.
+		 */
+		uint32 netmask;
+
+		/**
+		 * True if the source is reading live inout, false for
+		 * working offline.
+		 */
+		bool is_live;
+
+		Properties()
+			{
+			netmask = PCAP_NETMASK_UNKNOWN;
+			}
+	};
+
+	/**
+	 * Structure describing a packet.
+	 */
+	struct Packet {
+		/**
+		 *  Time associated with the packet.
+		 */
+		double ts;
+
+		/**
+		 *  The pcap header associated with the packet.
+		 */
+		const struct ::pcap_pkthdr* hdr;
+
+		/**
+		 * The full content of the packet.
+		 */
+		const u_char* data;
+	};
+
+	/**
+	 * Called from the implementations of \a Open() to signal that the
+	 * source has been successully opened.
+	 *
+	 * @param props A properties instance describing the now open source.
+	 */
+	void Opened(const Properties& props);
+
+	/**
+	 * Called from the implementations of \a Close() to signal that the
+	 * source has been closed.
+	 */
+	void Closed();
+
+	/**
+	 * Can be called from derived classes to send an informational
+	 * message to the user.
+	 *
+	 * @param msg The message to pass on.
+	 */
+	void Info(const std::string& msg);
+
+	/**
+	 * Can be called from derived classes to flag send an error.
+	 *
+	 * @param msg The message going with the error.
+	 */
+	void Error(const std::string& msg);
+
+	/**
+	 * Can be called from derived classes to flah a "weird" situation.
+	 *
+	 * @param msg The message to pass on.
+	 *
+	 * @param pkt The packet associated with the weird, or null if none.
+	 */
+	void Weird(const std::string& msg, const Packet* pkt);
+
+	/**
+	 * Can be called from derived classes to flag an internal error,
+	 * which will abort execution.
+	 *
+	 * @param msg The message to pass on.
+	 */
+	void InternalError(const std::string& msg);
+
+	// PktSrc interface for derived classes to implement.
+
+	/**
+	 * Called by the manager system to open the source.
+	 *
+	 * Derived classes must implement this method. If successful, the
+	 * implementation must call \a Opened(); if not, it must call Error()
+	 * with a corresponding message.
+	 */
+	virtual void Open() = 0;
+
+	/**
+	 * Called by the manager system to close the source.
+	 *
+	 * Derived classes must implement this method. If successful, the
+	 * implementation must call \a Closed(); if not, it must call Error()
+	 * with a corresponding message.
+	 */
+	virtual void Close() = 0;
+
+	/**
+	 * Provides the next packet from the source.
+	 *
+	 * @param pkt The packet structure to fill in with the packet's
+	 * information. The callee keep ownership of the data but must
+	 * guaranetee that it stays available at least until \a
+	 * DoneWithPacket() is called.  It is guaranteed that no two calls to
+	 * this method will hapen with \a DoneWithPacket() in between.
+	 *
+	 * @return True if a packet is available and *pkt* filled in. False
+	 * if not packet is available or an error occured (which must be
+	 * flageed via Error()).
+	 */
+	virtual bool ExtractNextPacket(Packet* pkt) = 0;
+
+	/**
+	 * Signals that the data of previously extracted packet will no
+	 * longer be needed.
+	 */
+	virtual void DoneWithPacket() = 0;
+
+private:
+	// Checks if the current packet has a pseudo-time <= current_time. If
+	// yes, returns pseudo-time, otherwise 0.
+	double CheckPseudoTime();
+
+	// Internal helper for ExtractNextPacket().
+	bool ExtractNextPacketInternal();
+
+	// IOSource interface implementation.
+	virtual void Init();
+	virtual void Done();
+	virtual void GetFds(iosource::FD_Set* read, iosource::FD_Set* write,
+	                    iosource::FD_Set* except);
+	virtual double NextTimestamp(double* local_network_time);
+	virtual void Process();
+	virtual const char* Tag();
+
+	Properties props;
+
+	bool have_packet;
+	Packet current_packet;
+
+	// For BPF filtering support.
+	PDict(BPF_Program) filters;
+
+	// Only set in pseudo-realtime mode.
+	double first_timestamp;
+	double first_wallclock;
+	double current_wallclock;
+	double current_pseudo;
+	double next_sync_point; // For trace synchronziation in pseudo-realtime
+
+	std::string errbuf;
+};
+
+}
+
+#endif

src/iosource/pcap.bif

@@ -0,0 +1,104 @@
+
+## Precompiles a PCAP filter and binds it to a given identifier.
+##
+## id: The PCAP identifier to reference the filter *s* later on.
+##
+## s: The PCAP filter. See ``man tcpdump`` for valid expressions.
+##
+## Returns: True if *s* is valid and precompiles successfully.
+##
+## .. bro:see:: install_pcap_filter
+##          install_src_addr_filter
+##          install_src_net_filter
+##          uninstall_src_addr_filter
+##          uninstall_src_net_filter
+##          install_dst_addr_filter
+##          install_dst_net_filter
+##          uninstall_dst_addr_filter
+##          uninstall_dst_net_filter
+##          pcap_error
+function precompile_pcap_filter%(id: PcapFilterID, s: string%): bool
+	%{
+	bool success = true;
+
+	const iosource::Manager::PktSrcList& pkt_srcs(iosource_mgr->GetPktSrcs());
+
+	for ( iosource::Manager::PktSrcList::const_iterator i = pkt_srcs.begin();
+	      i != pkt_srcs.end(); i++ )
+		{
+		iosource::PktSrc* ps = *i;
+
+		if ( ! ps->PrecompileFilter(id->ForceAsInt(),
+							s->CheckString()) )
+			success = false;
+		}
+
+	return new Val(success, TYPE_BOOL);
+	%}
+
+## Installs a PCAP filter that has been precompiled with
+## :bro:id:`precompile_pcap_filter`.
+##
+## id: The PCAP filter id of a precompiled filter.
+##
+## Returns: True if the filter associated with *id* has been installed
+##          successfully.
+##
+## .. bro:see:: precompile_pcap_filter
+##              install_src_addr_filter
+##              install_src_net_filter
+##              uninstall_src_addr_filter
+##              uninstall_src_net_filter
+##              install_dst_addr_filter
+##              install_dst_net_filter
+##              uninstall_dst_addr_filter
+##              uninstall_dst_net_filter
+##              pcap_error
+function install_pcap_filter%(id: PcapFilterID%): bool
+	%{
+	bool success = true;
+
+	const iosource::Manager::PktSrcList& pkt_srcs(iosource_mgr->GetPktSrcs());
+
+	for ( iosource::Manager::PktSrcList::const_iterator i = pkt_srcs.begin();
+	      i != pkt_srcs.end(); i++ )
+		{
+		iosource::PktSrc* ps = *i;
+
+		if ( ! ps->SetFilter(id->ForceAsInt()) )
+			success = false;
+		}
+
+	return new Val(success, TYPE_BOOL);
+	%}
+
+## Returns a string representation of the last PCAP error.
+##
+## Returns: A descriptive error message of the PCAP function that failed.
+##
+## .. bro:see:: precompile_pcap_filter
+##              install_pcap_filter
+##              install_src_addr_filter
+##              install_src_net_filter
+##              uninstall_src_addr_filter
+##              uninstall_src_net_filter
+##              install_dst_addr_filter
+##              install_dst_net_filter
+##              uninstall_dst_addr_filter
+##              uninstall_dst_net_filter
+function pcap_error%(%): string
+	%{
+	const iosource::Manager::PktSrcList& pkt_srcs(iosource_mgr->GetPktSrcs());
+
+	for ( iosource::Manager::PktSrcList::const_iterator i = pkt_srcs.begin();
+	      i != pkt_srcs.end(); i++ )
+		{
+		iosource::PktSrc* ps = *i;
+
+		const char* err = ps->ErrorMsg();
+		if ( *err )
+			return new StringVal(err);
+		}
+
+	return new StringVal("no error");
+	%}

src/iosource/pcap/CMakeLists.txt

@@ -0,0 +1,8 @@
+
+include(BroPlugin)
+
+include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR})
+
+bro_plugin_begin(Bro Pcap)
+bro_plugin_cc(Source.cc Dumper.cc Plugin.cc)
+bro_plugin_end()

src/iosource/pcap/Dumper.cc

@@ -0,0 +1,112 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#include <sys/stat.h>
+#include <errno.h>
+
+#include "Dumper.h"
+#include "../PktSrc.h"
+#include "../../Net.h"
+
+using namespace iosource::pcap;
+
+PcapDumper::PcapDumper(const std::string& path, bool arg_append)
+	{
+	append = arg_append;
+	props.path = path;
+	dumper = 0;
+	pd = 0;
+	}
+
+PcapDumper::~PcapDumper()
+	{
+	}
+
+void PcapDumper::Open()
+	{
+	int linktype = -1;
+
+	pd = pcap_open_dead(DLT_EN10MB, snaplen);
+	if ( ! pd )
+		{
+		Error("error for pcap_open_dead");
+		return;
+		}
+
+	if ( props.path.empty() )
+		{
+		Error("no filename given");
+		return;
+		}
+
+	struct stat s;
+	int exists = 0;
+
+	if ( append )
+		{
+		// See if output file already exists (and is non-empty).
+		exists = stat(props.path.c_str(), &s); ;
+
+		if ( exists < 0 && errno != ENOENT )
+			{
+			Error(fmt("can't stat file %s: %s", props.path.c_str(), strerror(errno)));
+			return;
+			}
+		}
+
+	if ( ! append || exists < 0 || s.st_size == 0 )
+		{
+		// Open new file.
+		dumper = pcap_dump_open(pd, props.path.c_str());
+		if ( ! dumper )
+			{
+			Error(pcap_geterr(pd));
+			return;
+			}
+		}
+
+	else
+		{
+		// Old file and we need to append, which, unfortunately,
+		// is not supported by libpcap. So, we have to hack a
+		// little bit, knowing that pcap_dumpter_t is, in fact,
+		// a FILE ... :-(
+		dumper = (pcap_dumper_t*) fopen(props.path.c_str(), "a");
+		if ( ! dumper )
+			{
+			Error(fmt("can't open dump %s: %s", props.path.c_str(), strerror(errno)));
+			return;
+			}
+		}
+
+	props.open_time = network_time;
+	props.hdr_size = PktSrc::GetLinkHeaderSize(pcap_datalink(pd));
+	Opened(props);
+	}
+
+void PcapDumper::Close()
+	{
+	if ( ! dumper )
+		return;
+
+	pcap_dump_close(dumper);
+	pcap_close(pd);
+	dumper = 0;
+	pd = 0;
+
+	Closed();
+	}
+
+bool PcapDumper::Dump(const Packet* pkt)
+	{
+	if ( ! dumper )
+		return false;
+
+	pcap_dump((u_char*) dumper, pkt->hdr, pkt->data);
+
+	return true;
+	}
+
+iosource::PktDumper* PcapDumper::Instantiate(const std::string& path, bool append)
+	{
+	return new PcapDumper(path, append);
+	}

src/iosource/pcap/Dumper.h

@@ -0,0 +1,41 @@
+// See the file  in the main distribution directory for copyright.
+
+#ifndef IOSOURCE_PKTSRC_PCAP_DUMPER_H
+#define IOSOURCE_PKTSRC_PCAP_DUMPER_H
+
+extern "C" {
+#include <pcap.h>
+}
+
+#include "../PktDumper.h"
+
+namespace iosource {
+namespace pcap {
+
+class PcapDumper : public PktDumper {
+public:
+	PcapDumper(const std::string& path, bool append);
+	virtual ~PcapDumper();
+
+	static PktDumper* Instantiate(const std::string& path, bool appen);
+
+protected:
+	// PktDumper interface.
+	virtual void Open();
+	virtual void Close();
+	virtual bool Dump(const Packet* pkt);
+
+private:
+	Properties props;
+
+	bool append;
+	pcap_dumper_t* dumper;
+	pcap_t* pd;
+};
+
+}
+}
+
+#endif
+
+

src/iosource/pcap/Plugin.cc

@@ -0,0 +1,27 @@
+// See the file  in the main distribution directory for copyright.
+
+#include "plugin/Plugin.h"
+
+#include "Source.h"
+#include "Dumper.h"
+
+namespace plugin {
+namespace Bro_Pcap {
+
+class Plugin : public plugin::Plugin {
+public:
+	plugin::Configuration Configure()
+		{
+		AddComponent(new ::iosource::PktSrcComponent("PcapReader", "pcap", ::iosource::PktSrcComponent::BOTH, ::iosource::pcap::PcapSource::Instantiate));
+		AddComponent(new ::iosource::PktDumperComponent("PcapWriter", "pcap", ::iosource::pcap::PcapDumper::Instantiate));
+
+		plugin::Configuration config;
+		config.name = "Bro::Pcap";
+		config.description = "Packet aquisition via libpcap";
+		return config;
+		}
+} plugin;
+
+}
+}
+

src/iosource/pcap/Source.cc

@@ -0,0 +1,272 @@
+// See the file  in the main distribution directory for copyright.
+
+#include <assert.h>
+
+#include "config.h"
+
+#include "Source.h"
+
+#ifdef HAVE_PCAP_INT_H
+#include <pcap-int.h>
+#endif
+
+using namespace iosource::pcap;
+
+PcapSource::~PcapSource()
+	{
+	Close();
+	}
+
+PcapSource::PcapSource(const std::string& path, bool is_live)
+	{
+	props.path = path;
+	props.is_live = is_live;
+	last_data = 0;
+	}
+
+void PcapSource::Open()
+	{
+	if ( props.is_live )
+		OpenLive();
+	else
+		OpenOffline();
+	}
+
+void PcapSource::Close()
+	{
+	if ( ! pd )
+		return;
+
+	pcap_close(pd);
+	pd = 0;
+	last_data = 0;
+
+	Closed();
+	}
+
+void PcapSource::OpenLive()
+	{
+	char errbuf[PCAP_ERRBUF_SIZE];
+	char tmp_errbuf[PCAP_ERRBUF_SIZE];
+
+	// Determine interface if not specified.
+	if ( props.path.empty() )
+		props.path = pcap_lookupdev(tmp_errbuf);
+
+	if ( props.path.empty() )
+		{
+		safe_snprintf(errbuf, sizeof(errbuf),
+			 "pcap_lookupdev: %s", tmp_errbuf);
+		Error(errbuf);
+		return;
+		}
+
+	// Determine network and netmask.
+	uint32 net;
+	if ( pcap_lookupnet(props.path.c_str(), &net, &props.netmask, tmp_errbuf) < 0 )
+		{
+		// ### The lookup can fail if no address is assigned to
+		// the interface; and libpcap doesn't have any useful notion
+		// of error codes, just error std::strings - how bogus - so we
+		// just kludge around the error :-(.
+		// sprintf(errbuf, "pcap_lookupnet %s", tmp_errbuf);
+		// return;
+		props.netmask = 0xffffff00;
+		}
+
+	// We use the smallest time-out possible to return almost immediately if
+	// no packets are available. (We can't use set_nonblocking() as it's
+	// broken on FreeBSD: even when select() indicates that we can read
+	// something, we may get nothing if the store buffer hasn't filled up
+	// yet.)
+	pd = pcap_open_live(props.path.c_str(), SnapLen(), 1, 1, tmp_errbuf);
+
+	if ( ! pd )
+		{
+		Error(tmp_errbuf);
+		return;
+		}
+
+	// ### This needs autoconf'ing.
+#ifdef HAVE_PCAP_INT_H
+	Info(fmt("pcap bufsize = %d\n", ((struct pcap *) pd)->bufsize));
+#endif
+
+#ifdef HAVE_LINUX
+	if ( pcap_setnonblock(pd, 1, tmp_errbuf) < 0 )
+		{
+		PcapError();
+		return;
+		}
+#endif
+
+	props.selectable_fd = pcap_fileno(pd);
+
+	SetHdrSize();
+
+	if ( ! pd )
+		// Was closed, couldn't get header size.
+		return;
+
+	props.is_live = true;
+
+	Opened(props);
+	}
+
+void PcapSource::OpenOffline()
+	{
+	char errbuf[PCAP_ERRBUF_SIZE];
+
+	pd = pcap_open_offline(props.path.c_str(), errbuf);
+
+	if ( ! pd )
+		{
+		Error(errbuf);
+		return;
+		}
+
+	SetHdrSize();
+
+	if ( ! pd )
+		// Was closed, unknown link layer type.
+		return;
+
+	props.selectable_fd = fileno(pcap_file(pd));
+
+	if ( props.selectable_fd < 0 )
+		InternalError("OS does not support selectable pcap fd");
+
+	props.is_live = false;
+	Opened(props);
+	}
+
+bool PcapSource::ExtractNextPacket(Packet* pkt)
+	{
+	if ( ! pd )
+		return false;
+
+	const u_char* data = pcap_next(pd, &current_hdr);
+
+	if ( ! data )
+		{
+		// Source has gone dry.  If it's a network interface, this just means
+		// it's timed out. If it's a file, though, then the file has been
+		// exhausted.
+		if ( ! props.is_live )
+			Close();
+
+		return false;
+		}
+
+	pkt->ts = current_hdr.ts.tv_sec + double(current_hdr.ts.tv_usec) / 1e6;
+	pkt->hdr = &current_hdr;
+	pkt->data = last_data = data;
+
+	if ( current_hdr.len == 0 || current_hdr.caplen == 0 )
+		{
+		Weird("empty_pcap_header", pkt);
+		return false;
+		}
+
+	last_hdr = current_hdr;
+	last_data = data;
+	++stats.received;
+	return true;
+	}
+
+void PcapSource::DoneWithPacket()
+	{
+	// Nothing to do.
+	}
+
+bool PcapSource::PrecompileFilter(int index, const std::string& filter)
+	{
+	return PktSrc::PrecompileBPFFilter(index, filter);
+	}
+
+bool PcapSource::SetFilter(int index)
+	{
+	if ( ! pd )
+		return true; // Prevent error message
+
+	char errbuf[PCAP_ERRBUF_SIZE];
+
+	BPF_Program* code = GetBPFFilter(index);
+
+	if ( ! code )
+		{
+		safe_snprintf(errbuf, sizeof(errbuf),
+			      "No precompiled pcap filter for index %d",
+			      index);
+		Error(errbuf);
+		return false;
+		}
+
+	if ( pcap_setfilter(pd, code->GetProgram()) < 0 )
+		{
+		PcapError();
+		return false;
+		}
+
+#ifndef HAVE_LINUX
+	// Linux doesn't clear counters when resetting filter.
+	stats.received = stats.dropped = stats.link = 0;
+#endif
+
+	return true;
+	}
+
+void PcapSource::Statistics(Stats* s)
+	{
+	char errbuf[PCAP_ERRBUF_SIZE];
+
+	if ( ! (props.is_live && pd) )
+		s->received = s->dropped = s->link = 0;
+
+	else
+		{
+		struct pcap_stat pstat;
+		if ( pcap_stats(pd, &pstat) < 0 )
+			{
+			PcapError();
+			s->received = s->dropped = s->link = 0;
+			}
+
+		else
+			{
+			s->dropped = pstat.ps_drop;
+			s->link = pstat.ps_recv;
+			}
+		}
+
+	s->received = stats.received;
+
+	if ( ! props.is_live )
+		s->dropped = 0;
+	}
+
+void PcapSource::PcapError()
+	{
+	if ( pd )
+		Error(fmt("pcap_error: %s", pcap_geterr(pd)));
+	else
+		Error("pcap_error: not open");
+
+	Close();
+	}
+
+void PcapSource::SetHdrSize()
+	{
+	if ( ! pd )
+		return;
+
+	char errbuf[PCAP_ERRBUF_SIZE];
+
+	props.link_type = pcap_datalink(pd);
+	props.hdr_size = GetLinkHeaderSize(props.link_type);
+	}
+
+iosource::PktSrc* PcapSource::Instantiate(const std::string& path, bool is_live)
+	{
+	return new PcapSource(path, is_live);
+	}

src/iosource/pcap/Source.h

@@ -0,0 +1,47 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#ifndef IOSOURCE_PKTSRC_PCAP_SOURCE_H
+#define IOSOURCE_PKTSRC_PCAP_SOURCE_H
+
+#include "../PktSrc.h"
+
+namespace iosource {
+namespace pcap {
+
+class PcapSource : public iosource::PktSrc {
+public:
+	PcapSource(const std::string& path, bool is_live);
+	virtual ~PcapSource();
+
+	static PktSrc* Instantiate(const std::string& path, bool is_live);
+
+protected:
+	// PktSrc interface.
+	virtual void Open();
+	virtual void Close();
+	virtual bool ExtractNextPacket(Packet* pkt);
+	virtual void DoneWithPacket();
+	virtual bool PrecompileFilter(int index, const std::string& filter);
+	virtual bool SetFilter(int index);
+	virtual void Statistics(Stats* stats);
+
+private:
+	void OpenLive();
+	void OpenOffline();
+	void PcapError();
+	void SetHdrSize();
+
+	Properties props;
+	Stats stats;
+
+	pcap_t *pd;
+
+	struct pcap_pkthdr current_hdr;
+	struct pcap_pkthdr last_hdr;
+	const u_char* last_data;
+};
+
+}
+}
+
+#endif

src/main.cc

@@ -57,6 +57,7 @@ extern "C" void OPENSSL_add_all_algorithms_conf(void);
 #include "plugin/Manager.h"
 #include "file_analysis/Manager.h"
 #include "broxygen/Manager.h"
+#include "iosource/Manager.h"
 
 #include "binpac_bro.h"
 
@@ -92,6 +93,7 @@ plugin::Manager* plugin_mgr = 0;
 analyzer::Manager* analyzer_mgr = 0;
 file_analysis::Manager* file_mgr = 0;
 broxygen::Manager* broxygen_mgr = 0;
+iosource::Manager* iosource_mgr = 0;
 Stmt* stmts;
 EventHandlerPtr net_done = 0;
 RuleMatcher* rule_matcher = 0;
@@ -108,7 +110,6 @@ int signal_val = 0;
 int optimize = 0;
 int do_notice_analysis = 0;
 int rule_bench = 0;
-SecondaryPath* secondary_path = 0;
 extern char version[];
 char* command_line_policy = 0;
 vector<string> params;
@@ -340,6 +341,10 @@ void terminate_bro()
 
 	terminating = true;
 
+	// File analysis termination may produce events, so do it early on in
+	// the termination process.
+	file_mgr->Terminate();
+
 	brofiler.WriteStats();
 
 	EventHandlerPtr bro_done = internal_handler("bro_done");
@@ -364,7 +369,6 @@ void terminate_bro()
 
 	mgr.Drain();
 
-	file_mgr->Terminate();
 	log_mgr->Terminate();
 	input_mgr->Terminate();
 	thread_mgr->Terminate();
@@ -375,20 +379,16 @@ void terminate_bro()
 
 	delete broxygen_mgr;
 	delete timer_mgr;
-	delete dns_mgr;
 	delete persistence_serializer;
-	delete event_player;
 	delete event_serializer;
 	delete state_serializer;
 	delete event_registry;
-	delete secondary_path;
-	delete remote_serializer;
 	delete analyzer_mgr;
 	delete file_mgr;
 	delete log_mgr;
 	delete plugin_mgr;
-	delete thread_mgr;
 	delete reporter;
+	delete iosource_mgr;
 
 	reporter = 0;
 	}
@@ -448,8 +448,6 @@ int main(int argc, char** argv)
 
 	name_list interfaces;
 	name_list read_files;
-	name_list netflows;
-	name_list flow_files;
 	name_list rule_files;
 	char* bst_file = 0;
 	char* id_name = 0;
@@ -551,7 +549,7 @@ int main(int argc, char** argv)
 	opterr = 0;
 
 	char opts[256];
-	safe_strncpy(opts, "B:D:e:f:I:i:K:l:n:p:R:r:s:T:t:U:w:x:X:y:Y:z:CFGLNOPSWabdghvZQ",
+	safe_strncpy(opts, "B:D:e:f:I:i:K:l:n:p:R:r:s:T:t:U:w:x:X:z:CFGLNOPSWabdghvZQ",
 		     sizeof(opts));
 
 #ifdef USE_PERFTOOLS_DEBUG
@@ -611,10 +609,6 @@ int main(int argc, char** argv)
 			writefile = optarg;
 			break;
 
-		case 'y':
-			flow_files.append(optarg);
-			break;
-
 		case 'z':
 			if ( streq(optarg, "notice") )
 				do_notice_analysis = 1;
@@ -708,10 +702,6 @@ int main(int argc, char** argv)
 			do_watchdog = 1;
 			break;
 
-		case 'Y':
-			netflows.append(optarg);
-			break;
-
 		case 'h':
 			usage();
 			break;
@@ -799,8 +789,7 @@ int main(int argc, char** argv)
 	// seed the PRNG. We should do this here (but at least Linux, FreeBSD
 	// and Solaris provide /dev/urandom).
 
-	if ( (interfaces.length() > 0 || netflows.length() > 0) &&
-	     (read_files.length() > 0 || flow_files.length() > 0 ))
+	if ( interfaces.length() > 0 && read_files.length() > 0 )
 		usage();
 
 #ifdef USE_IDMEF
@@ -823,7 +812,7 @@ int main(int argc, char** argv)
 	plugin_mgr->SearchDynamicPlugins(bro_plugin_path());
 
 	if ( optind == argc &&
-	     read_files.length() == 0 && flow_files.length() == 0 &&
+	     read_files.length() == 0 &&
 	     interfaces.length() == 0 &&
 	     ! (id_name || bst_file) && ! command_line_policy && ! print_plugins )
 		add_input_file("-");
@@ -850,6 +839,7 @@ int main(int argc, char** argv)
 	// policy, but we can't parse policy without DNS resolution.
 	dns_mgr->SetDir(".state");
 
+	iosource_mgr = new iosource::Manager();
 	persistence_serializer = new PersistenceSerializer();
 	remote_serializer = new RemoteSerializer();
 	event_registry = new EventRegistry();
@@ -863,9 +853,17 @@ int main(int argc, char** argv)
 	file_mgr->InitPreScript();
 	broxygen_mgr->InitPreScript();
 
+	bool missing_plugin = false;
+
 	for ( set<string>::const_iterator i = requested_plugins.begin();
 	      i != requested_plugins.end(); i++ )
-		plugin_mgr->ActivateDynamicPlugin(*i);
+		{
+		if ( ! plugin_mgr->ActivateDynamicPlugin(*i) )
+			missing_plugin = true;
+		}
+
+	if ( missing_plugin )
+		reporter->FatalError("Failed to activate requested dynamic plugin(s).");
 
 	plugin_mgr->ActivateDynamicPlugins(! bare_mode);
 
@@ -916,6 +914,7 @@ int main(int argc, char** argv)
 
 	analyzer_mgr->InitPostScript();
 	file_mgr->InitPostScript();
+	dns_mgr->InitPostScript();
 
 	if ( parse_only )
 		{
@@ -981,8 +980,7 @@ int main(int argc, char** argv)
 		// ### Add support for debug command file.
 		dbg_init_debugger(0);
 
-	if ( (flow_files.length() == 0 || read_files.length() == 0) &&
-	     (netflows.length() == 0 || interfaces.length() == 0) )
+	if ( read_files.length() == 0 && interfaces.length() == 0 )
 		{
 		Val* interfaces_val = internal_val("interfaces");
 		if ( interfaces_val )
@@ -999,13 +997,8 @@ int main(int argc, char** argv)
 
 	snaplen = internal_val("snaplen")->AsCount();
 
-	// Initialize the secondary path, if it's needed.
-	secondary_path = new SecondaryPath();
-
 	if ( dns_type != DNS_PRIME )
-		net_init(interfaces, read_files, netflows, flow_files,
-			writefile, "",
-			secondary_path->Filter(), do_watchdog);
+		net_init(interfaces, read_files, writefile, do_watchdog);
 
 	BroFile::SetDefaultRotation(log_rotate_interval, log_max_size);
 
@@ -1164,9 +1157,9 @@ int main(int argc, char** argv)
 
 	have_pending_timers = ! reading_traces && timer_mgr->Size() > 0;
 
-	io_sources.Register(thread_mgr, true);
+	iosource_mgr->Register(thread_mgr, true);
 
-	if ( io_sources.Size() > 0 ||
+	if ( iosource_mgr->Size() > 0 ||
 	     have_pending_timers ||
 	     BifConst::exit_only_after_terminate )
 		{

src/plugin/Component.cc

@@ -50,6 +50,18 @@ void Component::Describe(ODesc* d) const
 		d->Add("File Analyzer");
 		break;
 
+	case component::IOSOURCE:
+		d->Add("I/O Source");
+		break;
+
+	case component::PKTSRC:
+		d->Add("Packet Source");
+		break;
+
+	case component::PKTDUMPER:
+		d->Add("Packet Dumper");
+		break;
+
 	default:
 		reporter->InternalWarning("unknown component type in plugin::Component::Describe");
 		d->Add("<unknown component type>");

src/plugin/Component.h

@@ -12,20 +12,23 @@ namespace plugin {
 namespace component {
 
 /**
- * Component types. 
+ * Component types.
  */
 enum Type {
 	READER,	/// An input reader (not currently used).
 	WRITER,	/// A logging writer (not currenly used).
 	ANALYZER,	/// A protocol analyzer.
-	FILE_ANALYZER	/// A file analyzer.
+	FILE_ANALYZER,	/// A file analyzer.
+	IOSOURCE,	/// An I/O source, excluding packet sources.
+	PKTSRC,	/// A packet source.
+	PKTDUMPER	/// A packet dumper.
 	};
 }
 
 /**
  * Base class for plugin components. A component is a specific piece of
  * functionality that a plugin provides, such as a protocol analyzer or a log
- * writer. 
+ * writer.
  */
 class Component
 {

src/plugin/Plugin.h

@@ -9,6 +9,7 @@
 #include "config.h"
 #include "analyzer/Component.h"
 #include "file_analysis/Component.h"
+#include "iosource/Component.h"
 
 // We allow to override this externally for testing purposes.
 #ifndef BRO_PLUGIN_API_VERSION

src/threading/Manager.cc

@@ -11,7 +11,7 @@ Manager::Manager()
 	did_process = true;
 	next_beat = 0;
 	terminating = false;
-	idle = true;
+	SetIdle(true);
 	}
 
 Manager::~Manager()
@@ -47,8 +47,8 @@ void Manager::Terminate()
 	all_threads.clear();
 	msg_threads.clear();
 
-	idle = true;
-	closed = true;
+	SetIdle(true);
+	SetClosed(true);
 	terminating = false;
 	}
 
@@ -56,7 +56,7 @@ void Manager::AddThread(BasicThread* thread)
 	{
 	DBG_LOG(DBG_THREADING, "Adding thread %s ...", thread->Name());
 	all_threads.push_back(thread);
-	idle = false;
+	SetIdle(false);
 	}
 
 void Manager::AddMsgThread(MsgThread* thread)
@@ -65,7 +65,8 @@ void Manager::AddMsgThread(MsgThread* thread)
 	msg_threads.push_back(thread);
 	}
 
-void Manager::GetFds(int* read, int* write, int* except)
+void Manager::GetFds(iosource::FD_Set* read, iosource::FD_Set* write,
+                     iosource::FD_Set* except)
 	{
 	}
 

src/threading/Manager.h

@@ -4,7 +4,7 @@
 
 #include <list>
 
-#include "IOSource.h"
+#include "iosource/IOSource.h"
 
 #include "BasicThread.h"
 #include "MsgThread.h"
@@ -21,7 +21,7 @@ namespace threading {
  * their outgoing message queue on a regular basis and feeds data sent into
  * the rest of Bro. It also triggers the regular heartbeats.
  */
-class Manager : public IOSource
+class Manager : public iosource::IOSource
 {
 public:
 	/**
@@ -103,7 +103,8 @@ protected:
 	/**
 	 * Part of the IOSource interface.
 	 */
-	virtual void GetFds(int* read, int* write, int* except);
+	virtual void GetFds(iosource::FD_Set* read, iosource::FD_Set* write,
+	                    iosource::FD_Set* except);
 
 	/**
 	 * Part of the IOSource interface.

src/util.cc

@@ -43,6 +43,7 @@
 #include "NetVar.h"
 #include "Net.h"
 #include "Reporter.h"
+#include "iosource/Manager.h"
 
 /**
  * Return IP address without enclosing brackets and any leading 0x.
@@ -1424,11 +1425,13 @@ double current_time(bool real)
 
 	double t = double(tv.tv_sec) + double(tv.tv_usec) / 1e6;
 
-	if ( ! pseudo_realtime || real || pkt_srcs.length() == 0 )
+	const iosource::Manager::PktSrcList& pkt_srcs(iosource_mgr->GetPktSrcs());
+
+	if ( ! pseudo_realtime || real || pkt_srcs.empty() )
 		return t;
 
 	// This obviously only works for a single source ...
-	PktSrc* src = pkt_srcs[0];
+	iosource::PktSrc* src = pkt_srcs.front();
 
 	if ( net_is_processing_suspended() )
 		return src->CurrentPacketTimestamp();