Merge remote-tracking branch 'origin/topic/robin/pktsrc'

* origin/topic/robin/pktsrc:
  Fixing link-layer handling.
  Bugfixes and test updates.
  Fixing remaining tests.
  Adding test for dynamic packet dumper plugin.
  Test updates.
  Fixing Bro-level BPF filtering.
  A set of smaller API tweaks, and polishing.
  Adding test creating a dynamic pktsrc plugin.
  Interface tweaks for PktSrc, plus docs for PktSrc and PktDumper.
  A set of various fixes and smaller API tweaks, plus tests.
  Moving some of the BPF filtering code into base class.
  Small packet dumper API change.
  Moving Pkt{Src,Dumper} a directory level up.
  Removing FlowSrc.
  Removing netmap, remaining pieces of the 2ndary path, and left-over files of packet sorter.
  Prototype of a netmap packet source.
  A series of updates and tweaks to the new PktSrc interface.
  Fixing rebase relicts.
  IOSource reorg.

BIT-1243 #close

CHANGES

@@ -1,4 +1,28 @@
 
+2.3-161 | 2014-09-09 12:35:38 -0500
+
+  * Bugfixes and test updates/additions. (Robin Sommer)
+
+  * Interface tweaks and docs for PktSrc/PktDumper. (Robin Sommer)
+
+  * Moving PCAP-related bifs to iosource/pcap.bif. (Robin Sommer)
+
+  * Moving some of the BPF filtering code into base class.
+    This will allow packet sources that don't support BPF natively to
+    emulate the filtering via libpcap. (Robin Sommer)
+
+  * Moving Pkt{Src,Dumper} a directory level up and renaming
+    PktSourceComponent to PktSrcComponent. (Robin Sommer)
+
+  * Removing FlowSrc. (Robin Sommer)
+
+  * Removing remaining pieces of the 2ndary path, and left-over
+    files of packet sorter. (Robin Sommer)
+
+  * A bunch of infrastructure work to move IOSource, IOSourceRegistry
+    (now iosource::Manager) and PktSrc/PktDumper code into iosource/,
+    and over to a plugin structure. (Robin Sommer)
+
 2.3-137 | 2014-09-08 19:01:13 -0500
 
   * Fix Broxygen's rendering of opaque types. (Jon Siwek)

VERSION

@@ -1 +1 @@
-2.3-137
+2.3-161

aux/binpac

@@ -1 +1 @@
-Subproject commit 4e5969f5a40f5cc192a751375cb61131d32c0fc1
+Subproject commit b1031e97e1cb57df0e0405a6b5c6d0eb49c32c62

aux/bro-aux

@@ -1 +1 @@
-Subproject commit 181f084432e277f899140647d9b788059b3cccb1
+Subproject commit 92dea34b43f4109b97dc2cccda49dbb58c72f77a

aux/broccoli

@@ -1 +1 @@
-Subproject commit 6be54279bb7ecb5e03d8bcdc7660d323dc4de1bc
+Subproject commit 64134bc778b46307180192cff48f0d1f08a874e8

aux/broctl

@@ -1 +1 @@
-Subproject commit f0e0efda05e4b20924efc1b826ad5d85c8b65f83
+Subproject commit db3f7e375b785ee3ef9795bc4917d396871785ff

cmake

@@ -1 +1 @@
-Subproject commit aa15263ae39667e5e9bd73690b05aa4af9147ca3
+Subproject commit 0b22aeb9f30b1edad54c225ef3e431c68750480b

src/CMakeLists.txt

@@ -157,6 +157,7 @@ add_subdirectory(analyzer)
 add_subdirectory(broxygen)
 add_subdirectory(file_analysis)
 add_subdirectory(input)
+add_subdirectory(iosource)
 add_subdirectory(logging)
 add_subdirectory(probabilistic)
 
@@ -254,7 +255,6 @@ set(bro_SRCS
     Anon.cc
     Attr.cc
     Base64.cc
-    BPF_Program.cc
     Brofiler.cc
     BroString.cc
     CCL.cc
@@ -279,14 +279,12 @@ set(bro_SRCS
     EventRegistry.cc
     Expr.cc
     File.cc
-    FlowSrc.cc
     Frag.cc
     Frame.cc
     Func.cc
     Hash.cc
     ID.cc
     IntSet.cc
-    IOSource.cc
     IP.cc
     IPAddr.cc
     List.cc
@@ -299,7 +297,6 @@ set(bro_SRCS
     OSFinger.cc
     PacketFilter.cc
     PersistenceSerializer.cc
-    PktSrc.cc
     PolicyFile.cc
     PrefixTable.cc
     PriorityQueue.cc

src/DNS_Mgr.cc

@@ -35,6 +35,7 @@
 #include "Net.h"
 #include "Var.h"
 #include "Reporter.h"
+#include "iosource/Manager.h"
 
 extern "C" {
 extern int select(int, fd_set *, fd_set *, fd_set *, struct timeval *);
@@ -404,17 +405,17 @@ DNS_Mgr::~DNS_Mgr()
 	delete [] dir;
 	}
 
-bool DNS_Mgr::Init()
+void DNS_Mgr::InitPostScript()
 	{
 	if ( did_init )
-		return true;
+		return;
 
 	const char* cache_dir = dir ? dir : ".";
 
 	if ( mode == DNS_PRIME && ! ensure_dir(cache_dir) )
 		{
 		did_init = 0;
-		return false;
+		return;
 		}
 
 	cache_name = new char[strlen(cache_dir) + 64];
@@ -433,14 +434,12 @@ bool DNS_Mgr::Init()
 
 	did_init = 1;
 
-	io_sources.Register(this, true);
+	iosource_mgr->Register(this, true);
 
 	// We never set idle to false, having the main loop only calling us from
 	// time to time. If we're issuing more DNS requests than we can handle
 	// in this way, we are having problems anyway ...
-	idle = true;
+	SetIdle(true);
-
-	return true;
 	}
 
 static TableVal* fake_name_lookup_result(const char* name)

src/DNS_Mgr.h

@@ -12,7 +12,7 @@
 #include "BroList.h"
 #include "Dict.h"
 #include "EventHandler.h"
-#include "IOSource.h"
+#include "iosource/IOSource.h"
 #include "IPAddr.h"
 
 class Val;
@@ -40,12 +40,12 @@ enum DNS_MgrMode {
 // Number of seconds we'll wait for a reply.
 #define DNS_TIMEOUT 5
 
-class DNS_Mgr : public IOSource {
+class DNS_Mgr : public iosource::IOSource {
 public:
 	DNS_Mgr(DNS_MgrMode mode);
 	virtual ~DNS_Mgr();
 
-	bool Init();
+	void InitPostScript();
 	void Flush();
 
 	// Looks up the address or addresses of the given host, and returns

src/DebugLogger.cc

@@ -18,7 +18,8 @@ DebugLogger::Stream DebugLogger::streams[NUM_DBGS] = {
 	{ "dpd", 0, false }, { "tm", 0, false },
 	{ "logging", 0, false }, {"input", 0, false },
 	{ "threading", 0, false }, { "file_analysis", 0, false },
-	{ "plugins", 0, false }, { "broxygen", 0, false }
+	{ "plugins", 0, false }, { "broxygen", 0, false },
+	{ "pktio", 0, false}
 };
 
 DebugLogger::DebugLogger(const char* filename)

src/DebugLogger.h

@@ -31,6 +31,7 @@ enum DebugStream {
 	DBG_FILE_ANALYSIS,	// File analysis
 	DBG_PLUGINS,	// Plugin system
 	DBG_BROXYGEN,	// Broxygen
+	DBG_PKTIO,	// Packet sources and dumpers.
 
 	NUM_DBGS // Has to be last
 };

src/FlowSrc.cc

@@ -1,228 +0,0 @@
-// See the file "COPYING" in the main distribution directory for copyright.
-//
-// Written by Bernhard Ager, TU Berlin (2006/2007).
-
-#include <stdio.h>
-#include <unistd.h>
-#include <fcntl.h>
-#include <netdb.h>
-
-#include "FlowSrc.h"
-#include "Net.h"
-#include "analyzer/protocol/netflow/netflow_pac.h"
-#include <errno.h>
-
-FlowSrc::FlowSrc()
-	{ // TODO: v9.
-	selectable_fd = -1;
-	idle = false;
-	data = 0;
-	pdu_len = -1;
-	exporter_ip = 0;
-	current_timestamp = next_timestamp = 0.0;
-	netflow_analyzer = new binpac::NetFlow::NetFlow_Analyzer();
-	}
-
-FlowSrc::~FlowSrc()
-	{
-	delete netflow_analyzer;
-	}
-
-void FlowSrc::GetFds(int* read, int* write, int* except)
-	{
-	if ( selectable_fd >= 0 )
-		*read = selectable_fd;
-	}
-
-double FlowSrc::NextTimestamp(double* network_time)
-	{
-	if ( ! data && ! ExtractNextPDU() )
-		return -1.0;
-	else
-		return next_timestamp;
-	}
-
-void FlowSrc::Process()
-	{
-	if ( ! data && ! ExtractNextPDU() )
-		return;
-
-	// This is normally done by calling net_packet_dispatch(),
-	// but as we don't have a packet to dispatch ...
-	net_update_time(next_timestamp);
-	expire_timers();
-
-	netflow_analyzer->downflow()->set_exporter_ip(exporter_ip);
-
-	// We handle exceptions in NewData (might have changed w/ new binpac).
-	netflow_analyzer->NewData(0, data, data + pdu_len);
-	data = 0;
-	}
-
-void FlowSrc::Close()
-	{
-	safe_close(selectable_fd);
-	}
-
-
-FlowSocketSrc::~FlowSocketSrc()
-	{
-	}
-
-int FlowSocketSrc::ExtractNextPDU()
-	{
-	sockaddr_in from;
-	socklen_t fromlen = sizeof(from);
-	pdu_len = recvfrom(selectable_fd, buffer, NF_MAX_PKT_SIZE, 0,
-				(struct sockaddr*) &from, &fromlen);
-	if ( pdu_len < 0 )
-		{
-		reporter->Error("problem reading NetFlow data from socket");
-		data = 0;
-		next_timestamp = -1.0;
-		closed = 1;
-		return 0;
-		}
-
-	if ( fromlen != sizeof(from) )
-		{
-		reporter->Error("malformed NetFlow PDU");
-		return 0;
-		}
-
-	data = buffer;
-	exporter_ip = from.sin_addr.s_addr;
-	next_timestamp = current_time();
-
-	if ( next_timestamp < current_timestamp )
-		next_timestamp = current_timestamp;
-	else
-		current_timestamp = next_timestamp;
-
-	return 1;
-	}
-
-FlowSocketSrc::FlowSocketSrc(const char* listen_parms)
-	{
-	int n = strlen(listen_parms) + 1;
-
-	char laddr[n], port[n], ident[n];
-	laddr[0] = port[0] = ident[0] = '\0';
-
-	int ret = sscanf(listen_parms, "%[^:]:%[^=]=%s", laddr, port, ident);
-	if ( ret < 2 )
-		{
-		snprintf(errbuf, BRO_FLOW_ERRBUF_SIZE,
-			"parsing your listen-spec went nuts: laddr='%s', port='%s'\n",
-			laddr[0] ? laddr : "", port[0] ? port : "");
-		closed = 1;
-		return;
-		}
-
-	const char* id = (ret == 3) ? ident : listen_parms;
-	netflow_analyzer->downflow()->set_identifier(id);
-
-	struct addrinfo aiprefs = {
-		0, PF_INET, SOCK_DGRAM, IPPROTO_UDP, 0, NULL, NULL, NULL
-	};
-	struct addrinfo* ainfo = 0;
-	if ( (ret = getaddrinfo(laddr, port, &aiprefs, &ainfo)) != 0 )
-		{
-		snprintf(errbuf, BRO_FLOW_ERRBUF_SIZE,
-				"getaddrinfo(%s, %s, ...): %s",
-				laddr, port, gai_strerror(ret));
-		closed = 1;
-		return;
-		}
-
-	if ( (selectable_fd = socket (PF_INET, SOCK_DGRAM, 0)) < 0 )
-		{
-		snprintf(errbuf, BRO_FLOW_ERRBUF_SIZE,
-				"socket: %s", strerror(errno));
-		closed = 1;
-		goto cleanup;
-		}
-
-	if ( bind (selectable_fd, ainfo->ai_addr, ainfo->ai_addrlen) < 0 )
-		{
-		snprintf(errbuf, BRO_FLOW_ERRBUF_SIZE,
-				"bind: %s", strerror(errno));
-		closed = 1;
-		goto cleanup;
-		}
-
-cleanup:
-	freeaddrinfo(ainfo);
-	}
-
-
-FlowFileSrc::~FlowFileSrc()
-	{
-	delete [] readfile;
-	}
-
-int FlowFileSrc::ExtractNextPDU()
-	{
-	FlowFileSrcPDUHeader pdu_header;
-
-	if ( read(selectable_fd, &pdu_header, sizeof(pdu_header)) <
-	     int(sizeof(pdu_header)) )
-		return Error(errno, "read header");
-
-	if ( pdu_header.pdu_length > NF_MAX_PKT_SIZE )
-		{
-		reporter->Error("NetFlow packet too long");
-
-		// Safely skip over the too-long PDU.
-		if ( lseek(selectable_fd, pdu_header.pdu_length, SEEK_CUR) < 0 )
-			return Error(errno, "lseek");
-		return 0;
-		}
-
-	if ( read(selectable_fd, buffer, pdu_header.pdu_length) <
-	     pdu_header.pdu_length )
-		return Error(errno, "read data");
-
-	if ( next_timestamp < pdu_header.network_time )
-		{
-		next_timestamp = pdu_header.network_time;
-		current_timestamp = pdu_header.network_time;
-		}
-	else
-		current_timestamp = next_timestamp;
-
-	data = buffer;
-	pdu_len = pdu_header.pdu_length;
-	exporter_ip = pdu_header.ipaddr;
-
-	return 1;
-	}
-
-FlowFileSrc::FlowFileSrc(const char* readfile)
-	{
-	int n = strlen(readfile) + 1;
-	char ident[n];
-	this->readfile = new char[n];
-
-	int ret = sscanf(readfile, "%[^=]=%s", this->readfile, ident);
-	const char* id = (ret == 2) ? ident : this->readfile;
-	netflow_analyzer->downflow()->set_identifier(id);
-
-	selectable_fd = open(this->readfile, O_RDONLY);
-	if ( selectable_fd < 0 )
-		{
-		closed = 1;
-		snprintf(errbuf, BRO_FLOW_ERRBUF_SIZE,
-				"open: %s", strerror(errno));
-		}
-	}
-
-int FlowFileSrc::Error(int errlvl, const char* errmsg)
-	{
-	snprintf(errbuf, BRO_FLOW_ERRBUF_SIZE,
-			"%s: %s", errmsg, strerror(errlvl));
-	data = 0;
-	next_timestamp = -1.0;
-	closed = 1;
-	return 0;
-	}

src/FlowSrc.h

@@ -1,84 +0,0 @@
-// See the file "COPYING" in the main distribution directory for copyright.
-//
-// Written by Bernhard Ager, TU Berlin (2006/2007).
-
-#ifndef flowsrc_h
-#define flowsrc_h
-
-#include "IOSource.h"
-#include "NetVar.h"
-#include "binpac.h"
-
-#define BRO_FLOW_ERRBUF_SIZE 512
-
-// TODO: 1500 is enough for v5 - how about the others?
-// 65536 would be enough for any UDP packet.
-#define NF_MAX_PKT_SIZE 8192
-
-struct FlowFileSrcPDUHeader {
-	double network_time;
-	int pdu_length;
-	uint32 ipaddr;
-};
-
-// Avoid including netflow_pac.h by explicitly declaring the NetFlow_Analyzer.
-namespace binpac {
-	namespace NetFlow {
-		class NetFlow_Analyzer;
-	}
-}
-
-class FlowSrc : public IOSource {
-public:
-	virtual ~FlowSrc();
-
-	// IOSource interface:
-	bool IsReady();
-	void GetFds(int* read, int* write, int* except);
-	double NextTimestamp(double* network_time);
-	void Process();
-
-	const char* Tag()		{ return "FlowSrc"; }
-	const char* ErrorMsg() const	{ return errbuf; }
-
-protected:
-	FlowSrc();
-
-	virtual int ExtractNextPDU() = 0;
-	virtual void Close();
-
-	int selectable_fd;
-
-	double current_timestamp;
-	double next_timestamp;
-	binpac::NetFlow::NetFlow_Analyzer* netflow_analyzer;
-
-	u_char buffer[NF_MAX_PKT_SIZE];
-	u_char* data;
-	int pdu_len;
-	uint32 exporter_ip;	// in network byte order
-
-	char errbuf[BRO_FLOW_ERRBUF_SIZE];
-};
-
-class FlowSocketSrc : public FlowSrc {
-public:
-	FlowSocketSrc(const char* listen_parms);
-	virtual ~FlowSocketSrc();
-
-	int ExtractNextPDU();
-};
-
-class FlowFileSrc : public FlowSrc {
-public:
-	FlowFileSrc(const char* readfile);
-	~FlowFileSrc();
-
-	int ExtractNextPDU();
-
-protected:
-	int Error(int errlvl, const char* errmsg);
-	char* readfile;
-};
-
-#endif

src/IOSource.h

@@ -1,103 +0,0 @@
-// Interface for classes providing/consuming data during Bro's main loop.
-
-#ifndef iosource_h
-#define iosource_h
-
-#include <list>
-#include "Timer.h"
-
-using namespace std;
-
-class IOSource {
-public:
-	IOSource()	{ idle = closed = false; }
-	virtual ~IOSource()	{}
-
-	// Returns true if source has nothing ready to process.
-	bool IsIdle() const	{ return idle; }
-
-	// Returns true if more data is to be expected in the future.
-	// Otherwise, source may be removed.
-	bool IsOpen() const	{ return ! closed; }
-
-	// Returns select'able fds (leaves args untouched if we don't have
-	// selectable fds).
-	virtual void GetFds(int* read, int* write, int* except) = 0;
-
-	// The following two methods are only called when either IsIdle()
-	// returns false or select() on one of the fds indicates that there's
-	// data to process.
-
-	// Returns timestamp (in global network time) associated with next
-	// data item.  If the source wants the data item to be processed
-	// with a local network time, it sets the argument accordingly.
-	virtual double NextTimestamp(double* network_time) = 0;
-
-	// Processes and consumes next data item.
-	virtual void Process() = 0;
-
-	// Returns tag of timer manager associated with last processed
-	// data item, nil for global timer manager.
-	virtual TimerMgr::Tag* GetCurrentTag()	{ return 0; }
-
-	// Returns a descriptual tag for debugging.
-	virtual const char* Tag() = 0;
-
-protected:
-	// Derived classed are to set this to true if they have gone dry
-	// temporarily.
-	bool idle;
-
-	// Derived classed are to set this to true if they have gone dry
-	// permanently.
-	bool closed;
-};
-
-class IOSourceRegistry {
-public:
-	IOSourceRegistry()	{ call_count = 0; dont_counts = 0; }
-	~IOSourceRegistry();
-
-	// If dont_count is true, this source does not contribute to the
-	// number of IOSources returned by Size().  The effect is that
-	// if all sources but the non-counting ones have gone dry,
-	// processing will shut down.
-	void Register(IOSource* src, bool dont_count = false);
-
-	// This may block for some time.
-	IOSource* FindSoonest(double* ts);
-
-	int Size() const	{ return sources.size() - dont_counts; }
-
-	// Terminate IOSource processing immediately by removing all
-	// sources (and therefore returning a Size() of zero).
-	void Terminate()	{ RemoveAll(); }
-
-protected:
-	// When looking for a source with something to process,
-	// every SELECT_FREQUENCY calls we will go ahead and
-	// block on a select().
-	static const int SELECT_FREQUENCY = 25;
-
-	// Microseconds to wait in an empty select if no source is ready.
-	static const int SELECT_TIMEOUT = 50;
-
-	void RemoveAll();
-
-	unsigned int call_count;
-	int dont_counts;
-
-	struct Source {
-		IOSource* src;
-		int fd_read;
-		int fd_write;
-		int fd_except;
-	};
-
-	typedef list<Source*> SourceList;
-	SourceList sources;
-};
-
-extern IOSourceRegistry io_sources;
-
-#endif

src/Net.cc

@@ -29,6 +29,9 @@
 #include "Anon.h"
 #include "Serializer.h"
 #include "PacketDumper.h"
+#include "iosource/Manager.h"
+#include "iosource/PktSrc.h"
+#include "iosource/PktDumper.h"
 #include "plugin/Manager.h"
 
 extern "C" {
@@ -39,10 +42,7 @@ extern "C" {
 extern int select(int, fd_set *, fd_set *, fd_set *, struct timeval *);
 }
 
-PList(PktSrc) pkt_srcs;
+iosource::PktDumper* pkt_dumper = 0;
-
-// FIXME: We should really merge PktDumper and PacketDumper.
-PktDumper* pkt_dumper = 0;
 
 int reading_live = 0;
 int reading_traces = 0;
@@ -63,8 +63,8 @@ const u_char* current_pkt = 0;
 int current_dispatched = 0;
 int current_hdr_size = 0;
 double current_timestamp = 0.0;
-PktSrc* current_pktsrc = 0;
+iosource::PktSrc* current_pktsrc = 0;
-IOSource* current_iosrc;
+iosource::IOSource* current_iosrc = 0;
 
 std::list<ScannedFile> files_scanned;
 std::vector<string> sig_files;
@@ -113,17 +113,21 @@ RETSIGTYPE watchdog(int /* signo */)
 					// saving the packet which caused the
 					// watchdog to trigger may be helpful,
 					// so we'll save that one nevertheless.
-					pkt_dumper = new PktDumper("watchdog-pkt.pcap");
+					pkt_dumper = iosource_mgr->OpenPktDumper("watchdog-pkt.pcap", false);
-					if ( pkt_dumper->IsError() )
+					if ( ! pkt_dumper || pkt_dumper->IsError() )
 						{
-						reporter->Error("watchdog: can't open watchdog-pkt.pcap for writing\n");
+						reporter->Error("watchdog: can't open watchdog-pkt.pcap for writing");
-						delete pkt_dumper;
 						pkt_dumper = 0;
 						}
 					}
 
 				if ( pkt_dumper )
-					pkt_dumper->Dump(current_hdr, current_pkt);
+					{
+					iosource::PktDumper::Packet p;
+					p.hdr = current_hdr;
+					p.data = current_pkt;
+					pkt_dumper->Dump(&p);
+					}
 				}
 
 			net_get_final_stats();
@@ -149,118 +153,40 @@ void net_update_time(double new_network_time)
 	}
 
 void net_init(name_list& interfaces, name_list& readfiles,
-	      name_list& netflows, name_list& flowfiles,
+	      const char* writefile, int do_watchdog)
-	        const char* writefile, const char* filter,
-			const char* secondary_filter, int do_watchdog)
 	{
-	if ( readfiles.length() > 0 || flowfiles.length() > 0 )
+	if ( readfiles.length() > 0 )
 		{
 		reading_live = pseudo_realtime > 0.0;
 		reading_traces = 1;
 
 		for ( int i = 0; i < readfiles.length(); ++i )
 			{
-			PktFileSrc* ps = new PktFileSrc(readfiles[i], filter);
+			iosource::PktSrc* ps = iosource_mgr->OpenPktSrc(readfiles[i], false);
+			assert(ps);
 
 			if ( ! ps->IsOpen() )
-				reporter->FatalError("%s: problem with trace file %s - %s\n",
+				reporter->FatalError("problem with trace file %s (%s)",
-					prog, readfiles[i], ps->ErrorMsg());
+						     readfiles[i],
-			else
+						     ps->ErrorMsg());
-				{
-				pkt_srcs.append(ps);
-				io_sources.Register(ps);
-				}
-
-			if ( secondary_filter )
-				{
-				// We use a second PktFileSrc for the
-				// secondary path.
-				PktFileSrc* ps = new PktFileSrc(readfiles[i],
-							secondary_filter,
-							TYPE_FILTER_SECONDARY);
-
-				if ( ! ps->IsOpen() )
-					reporter->FatalError("%s: problem with trace file %s - %s\n",
-						prog, readfiles[i],
-						ps->ErrorMsg());
-				else
-					{
-					pkt_srcs.append(ps);
-					io_sources.Register(ps);
-					}
-
-				ps->AddSecondaryTablePrograms();
-				}
-			}
-
-		for ( int i = 0; i < flowfiles.length(); ++i )
-			{
-			FlowFileSrc* fs = new FlowFileSrc(flowfiles[i]);
-
-			if ( ! fs->IsOpen() )
-				reporter->FatalError("%s: problem with netflow file %s - %s\n",
-					prog, flowfiles[i], fs->ErrorMsg());
-			else
-				{
-				io_sources.Register(fs);
-				}
 			}
 		}
 
-	else if ((interfaces.length() > 0 || netflows.length() > 0))
+	else if ( interfaces.length() > 0 )
 		{
 		reading_live = 1;
 		reading_traces = 0;
 
 		for ( int i = 0; i < interfaces.length(); ++i )
 			{
-			PktSrc* ps;
+			iosource::PktSrc* ps = iosource_mgr->OpenPktSrc(interfaces[i], true);
-			ps = new PktInterfaceSrc(interfaces[i], filter);
+			assert(ps);
 
 			if ( ! ps->IsOpen() )
-				reporter->FatalError("%s: problem with interface %s - %s\n",
+				reporter->FatalError("problem with interface %s (%s)",
-					prog, interfaces[i], ps->ErrorMsg());
+						     interfaces[i],
-			else
+						     ps->ErrorMsg());
-				{
-				pkt_srcs.append(ps);
-				io_sources.Register(ps);
-				}
-
-			if ( secondary_filter )
-				{
-				PktSrc* ps;
-				ps = new PktInterfaceSrc(interfaces[i],
-					filter, TYPE_FILTER_SECONDARY);
-
-				if ( ! ps->IsOpen() )
-					reporter->Error("%s: problem with interface %s - %s\n",
-						prog, interfaces[i],
-						ps->ErrorMsg());
-				else
-					{
-					pkt_srcs.append(ps);
-					io_sources.Register(ps);
-					}
-
-				ps->AddSecondaryTablePrograms();
-				}
 			}
-
-		for ( int i = 0; i < netflows.length(); ++i )
-			{
-			FlowSocketSrc* fs = new FlowSocketSrc(netflows[i]);
-
-			if ( ! fs->IsOpen() )
-				{
-				reporter->Error("%s: problem with netflow socket %s - %s\n",
-					prog, netflows[i], fs->ErrorMsg());
-				delete fs;
-				}
-
-			else
-				io_sources.Register(fs);
-			}
-
 		}
 
 	else
@@ -272,12 +198,12 @@ void net_init(name_list& interfaces, name_list& readfiles,
 
 	if ( writefile )
 		{
-		// ### This will fail horribly if there are multiple
+		pkt_dumper = iosource_mgr->OpenPktDumper(writefile, false);
-		// interfaces with different-lengthed media.
+		assert(pkt_dumper);
-		pkt_dumper = new PktDumper(writefile);
+
-		if ( pkt_dumper->IsError() )
+		if ( ! pkt_dumper->IsOpen() )
-			reporter->FatalError("%s: can't open write file \"%s\" - %s\n",
+			reporter->FatalError("problem opening dump file %s (%s)",
-				prog, writefile, pkt_dumper->ErrorMsg());
+					     writefile, pkt_dumper->ErrorMsg());
 
 		ID* id = global_scope()->Lookup("trace_output_file");
 		if ( ! id )
@@ -298,7 +224,7 @@ void net_init(name_list& interfaces, name_list& readfiles,
 		}
 	}
 
-void expire_timers(PktSrc* src_ps)
+void expire_timers(iosource::PktSrc* src_ps)
 	{
 	SegmentProfiler(segment_logger, "expiring-timers");
 	TimerMgr* tmgr =
@@ -311,8 +237,8 @@ void expire_timers(PktSrc* src_ps)
 	}
 
 void net_packet_dispatch(double t, const struct pcap_pkthdr* hdr,
-			const u_char* pkt, int hdr_size,
+			 const u_char* pkt, int hdr_size,
-			PktSrc* src_ps)
+			 iosource::PktSrc* src_ps)
 	{
 	if ( ! bro_start_network_time )
 		bro_start_network_time = t;
@@ -368,11 +294,11 @@ void net_run()
 	{
 	set_processing_status("RUNNING", "net_run");
 
-	while ( io_sources.Size() ||
+	while ( iosource_mgr->Size() ||
 		(BifConst::exit_only_after_terminate && ! terminating) )
 		{
 		double ts;
-		IOSource* src = io_sources.FindSoonest(&ts);
+		iosource::IOSource* src = iosource_mgr->FindSoonest(&ts);
 
 #ifdef DEBUG
 		static int loop_counter = 0;
@@ -470,16 +396,19 @@ void net_run()
 
 void net_get_final_stats()
 	{
-	loop_over_list(pkt_srcs, i)
+	const iosource::Manager::PktSrcList& pkt_srcs(iosource_mgr->GetPktSrcs());
+
+	for ( iosource::Manager::PktSrcList::const_iterator i = pkt_srcs.begin();
+	      i != pkt_srcs.end(); i++ )
 		{
-		PktSrc* ps = pkt_srcs[i];
+		iosource::PktSrc* ps = *i;
 
 		if ( ps->IsLive() )
 			{
-			struct PktSrc::Stats s;
+			iosource::PktSrc::Stats s;
 			ps->Statistics(&s);
-			reporter->Info("%d packets received on interface %s, %d dropped\n",
+			reporter->Info("%d packets received on interface %s, %d dropped",
-					s.received, ps->Interface(), s.dropped);
+					s.received, ps->Path().c_str(), s.dropped);
 			}
 		}
 	}
@@ -499,8 +428,6 @@ void net_finish(int drain_events)
 			sessions->Done();
 		}
 
-	delete pkt_dumper;
-
 #ifdef DEBUG
 	extern int reassem_seen_bytes, reassem_copied_bytes;
 	// DEBUG_MSG("Reassembly (TCP and IP/Frag): %d bytes seen, %d bytes copied\n",
@@ -521,29 +448,6 @@ void net_delete()
 		delete ip_anonymizer[i];
 	}
 
-// net_packet_match
-//
-// Description:
-//  - Checks if a packet matches a filter. It just wraps up a call to
-//    [pcap.h's] bpf_filter().
-//
-// Inputs:
-//  - fp: a BPF-compiled filter
-//  - pkt: a pointer to the packet
-//  - len: the original packet length
-//  - caplen: the captured packet length. This is pkt length
-//
-// Output:
-//  - return: 1 if the packet matches the filter, 0 otherwise
-
-int net_packet_match(BPF_Program* fp, const u_char* pkt,
-		     u_int len, u_int caplen)
-	{
-	// NOTE: I don't like too much un-const'ing the pkt variable.
-	return bpf_filter(fp->GetProgram()->bf_insns, (u_char*) pkt, len, caplen);
-	}
-
-
 int _processing_suspended = 0;
 
 static double suspend_start = 0;
@@ -561,8 +465,12 @@ void net_continue_processing()
 	if ( _processing_suspended == 1 )
 		{
 		reporter->Info("processing continued");
-		loop_over_list(pkt_srcs, i)
+
-			pkt_srcs[i]->ContinueAfterSuspend();
+		const iosource::Manager::PktSrcList& pkt_srcs(iosource_mgr->GetPktSrcs());
+
+		for ( iosource::Manager::PktSrcList::const_iterator i = pkt_srcs.begin();
+		      i != pkt_srcs.end(); i++ )
+			(*i)->ContinueAfterSuspend();
 		}
 
 	--_processing_suspended;

src/Net.h

@@ -5,17 +5,15 @@
 
 #include "net_util.h"
 #include "util.h"
-#include "BPF_Program.h"
 #include "List.h"
-#include "PktSrc.h"
-#include "FlowSrc.h"
 #include "Func.h"
 #include "RemoteSerializer.h"
+#include "iosource/IOSource.h"
+#include "iosource/PktSrc.h"
+#include "iosource/PktDumper.h"
 
 extern void net_init(name_list& interfaces, name_list& readfiles,
-		name_list& netflows, name_list& flowfiles,
+		const char* writefile, int do_watchdog);
-		const char* writefile, const char* filter,
-		const char* secondary_filter, int do_watchdog);
 extern void net_run();
 extern void net_get_final_stats();
 extern void net_finish(int drain_events);
@@ -23,10 +21,8 @@ extern void net_delete();	// Reclaim all memory, etc.
 extern void net_update_time(double new_network_time);
 extern void net_packet_dispatch(double t, const struct pcap_pkthdr* hdr,
 			const u_char* pkt, int hdr_size,
-			PktSrc* src_ps);
+			iosource::PktSrc* src_ps);
-extern int net_packet_match(BPF_Program* fp, const u_char* pkt,
+extern void expire_timers(iosource::PktSrc* src_ps = 0);
-			    u_int len, u_int caplen);
-extern void expire_timers(PktSrc* src_ps = 0);
 extern void termination_signal();
 
 // Functions to temporarily suspend processing of live input (network packets
@@ -83,13 +79,10 @@ extern const u_char* current_pkt;
 extern int current_dispatched;
 extern int current_hdr_size;
 extern double current_timestamp;
-extern PktSrc* current_pktsrc;
+extern iosource::PktSrc* current_pktsrc;
-extern IOSource* current_iosrc;
+extern iosource::IOSource* current_iosrc;
 
-declare(PList,PktSrc);
+extern iosource::PktDumper* pkt_dumper;	// where to save packets
-extern PList(PktSrc) pkt_srcs;
-
-extern PktDumper* pkt_dumper;	// where to save packets
 
 extern char* writefile;
 

src/PktSrc.cc

@@ -1,804 +0,0 @@
-// See the file "COPYING" in the main distribution directory for copyright.
-
-#include <errno.h>
-#include <sys/stat.h>
-
-#include "config.h"
-
-#include "util.h"
-#include "PktSrc.h"
-#include "Hash.h"
-#include "Net.h"
-#include "Sessions.h"
-
-
-// ### This needs auto-confing.
-#ifdef HAVE_PCAP_INT_H
-#include <pcap-int.h>
-#endif
-
-PktSrc::PktSrc()
-	{
-	interface = readfile = 0;
-	data = last_data = 0;
-	memset(&hdr, 0, sizeof(hdr));
-	hdr_size = 0;
-	datalink = 0;
-	netmask = 0xffffff00;
-	pd = 0;
-	idle = false;
-
-	next_sync_point = 0;
-	first_timestamp = current_timestamp = next_timestamp = 0.0;
-	first_wallclock = current_wallclock = 0;
-
-	stats.received = stats.dropped = stats.link = 0;
-	}
-
-PktSrc::~PktSrc()
-	{
-	Close();
-
-	loop_over_list(program_list, i)
-		delete program_list[i];
-
-	BPF_Program* code;
-	IterCookie* cookie = filters.InitForIteration();
-	while ( (code = filters.NextEntry(cookie)) )
-		delete code;
-
-	delete [] interface;
-	delete [] readfile;
-	}
-
-void PktSrc::GetFds(int* read, int* write, int* except)
-	{
-	if ( pseudo_realtime )
-		{
-		// Select would give erroneous results. But we simulate it
-		// by setting idle accordingly.
-		idle = CheckPseudoTime() == 0;
-		return;
-		}
-
-	if ( selectable_fd >= 0 )
-		*read = selectable_fd;
-	}
-
-int PktSrc::ExtractNextPacket()
-	{
-	// Don't return any packets if processing is suspended (except for the
-	// very first packet which we need to set up times).
-	if ( net_is_processing_suspended() && first_timestamp )
-		{
-		idle = true;
-		return 0;
-		}
-
-	data = last_data = pcap_next(pd, &hdr);
-
-	if ( data && (hdr.len == 0 || hdr.caplen == 0) )
-		{
-		sessions->Weird("empty_pcap_header", &hdr, data);
-		return 0;
-		}
-
-	if ( data )
-		next_timestamp = hdr.ts.tv_sec + double(hdr.ts.tv_usec) / 1e6;
-
-	if ( pseudo_realtime )
-		current_wallclock = current_time(true);
-
-	if ( ! first_timestamp )
-		first_timestamp = next_timestamp;
-
-	idle = (data == 0);
-
-	if ( data )
-		++stats.received;
-
-	// Source has gone dry.  If it's a network interface, this just means
-	// it's timed out. If it's a file, though, then the file has been
-	// exhausted.
-	if ( ! data && ! IsLive() )
-		{
-		closed = true;
-
-		if ( pseudo_realtime && using_communication )
-			{
-			if ( remote_trace_sync_interval )
-				remote_serializer->SendFinalSyncPoint();
-			else
-				remote_serializer->Terminate();
-			}
-		}
-
-	return data != 0;
-	}
-
-double PktSrc::NextTimestamp(double* local_network_time)
-	{
-	if ( ! data && ! ExtractNextPacket() )
-		return -1.0;
-
-	if ( pseudo_realtime )
-		{
-		// Delay packet if necessary.
-		double packet_time = CheckPseudoTime();
-		if ( packet_time )
-			return packet_time;
-
-		idle = true;
-		return -1.0;
-		}
-
-	return next_timestamp;
-	}
-
-void PktSrc::ContinueAfterSuspend()
-	{
-	current_wallclock = current_time(true);
-	}
-
-double PktSrc::CurrentPacketWallClock()
-	{
-	// We stop time when we are suspended.
-	if ( net_is_processing_suspended() )
-		current_wallclock = current_time(true);
-
-	return current_wallclock;
-	}
-
-double PktSrc::CheckPseudoTime()
-	{
-	if ( ! data && ! ExtractNextPacket() )
-		return 0;
-
-	if ( ! current_timestamp )
-		return bro_start_time;
-
-	if ( remote_trace_sync_interval )
-		{
-		if ( next_sync_point == 0 || next_timestamp >= next_sync_point )
-			{
-			int n = remote_serializer->SendSyncPoint();
-			next_sync_point = first_timestamp +
-						n * remote_trace_sync_interval;
-			remote_serializer->Log(RemoteSerializer::LogInfo,
-				fmt("stopping at packet %.6f, next sync-point at %.6f",
-					current_timestamp, next_sync_point));
-
-			return 0;
-			}
-		}
-
-	double pseudo_time = next_timestamp - first_timestamp;
-	double ct = (current_time(true) - first_wallclock) * pseudo_realtime;
-
-	return pseudo_time <= ct ? bro_start_time + pseudo_time : 0;
-	}
-
-void PktSrc::Process()
-	{
-	if ( ! data && ! ExtractNextPacket() )
-		return;
-
-	current_timestamp = next_timestamp;
-
-	int pkt_hdr_size = hdr_size;
-
-	// Unfortunately some packets on the link might have MPLS labels
-	// while others don't. That means we need to ask the link-layer if
-	// labels are in place.
-	bool have_mpls = false;
-
-	int protocol = 0;
-
-	switch ( datalink ) {
-	case DLT_NULL:
-		{
-		protocol = (data[3] << 24) + (data[2] << 16) + (data[1] << 8) + data[0];
-
-		// From the Wireshark Wiki: "AF_INET6, unfortunately, has
-		// different values in {NetBSD,OpenBSD,BSD/OS},
-		// {FreeBSD,DragonFlyBSD}, and {Darwin/Mac OS X}, so an IPv6
-		// packet might have a link-layer header with 24, 28, or 30
-		// as the AF_ value." As we may be reading traces captured on
-		// platforms other than what we're running on, we accept them
-		// all here.
-		if ( protocol != AF_INET
-		     && protocol != AF_INET6
-		     && protocol != 24
-		     && protocol != 28
-		     && protocol != 30 )
-			{
-			sessions->Weird("non_ip_packet_in_null_transport", &hdr, data);
-			data = 0;
-			return;
-			}
-
-		break;
-		}
-
-	case DLT_EN10MB:
-		{
-		// Get protocol being carried from the ethernet frame.
-		protocol = (data[12] << 8) + data[13];
-
-		switch ( protocol )
-			{
-			// MPLS carried over the ethernet frame.
-			case 0x8847:
-				// Remove the data link layer and denote a
-				// header size of zero before the IP header.
-				have_mpls = true;
-				data += get_link_header_size(datalink);
-				pkt_hdr_size = 0;
-				break;
-
-			// VLAN carried over the ethernet frame.
-			case 0x8100:
-				data += get_link_header_size(datalink);
-
-				// Check for MPLS in VLAN.
-				if ( ((data[2] << 8) + data[3]) == 0x8847 )
-					have_mpls = true;
-
-				data += 4; // Skip the vlan header
-				pkt_hdr_size = 0;
-
-				// Check for 802.1ah (Q-in-Q) containing IP.
-				// Only do a second layer of vlan tag
-				// stripping because there is no
-				// specification that allows for deeper
-				// nesting.
-				if ( ((data[2] << 8) + data[3]) == 0x0800 )
-					data += 4;
-
-				break;
-
-			// PPPoE carried over the ethernet frame.
-			case 0x8864:
-				data += get_link_header_size(datalink);
-				protocol = (data[6] << 8) + data[7];
-				data += 8; // Skip the PPPoE session and PPP header
-				pkt_hdr_size = 0;
-
-				if ( protocol != 0x0021 && protocol != 0x0057 )
-					{
-					// Neither IPv4 nor IPv6.
-					sessions->Weird("non_ip_packet_in_pppoe_encapsulation", &hdr, data);
-					data = 0;
-					return;
-					}
-				break;
-			}
-
-		break;
-		}
-
-	case DLT_PPP_SERIAL:
-		{
-		// Get PPP protocol.
-		protocol = (data[2] << 8) + data[3];
-
-		if ( protocol == 0x0281 )
-			{
-			// MPLS Unicast. Remove the data link layer and
-			// denote a header size of zero before the IP header.
-			have_mpls = true;
-			data += get_link_header_size(datalink);
-			pkt_hdr_size = 0;
-			}
-
-		else if ( protocol != 0x0021 && protocol != 0x0057 )
-			{
-			// Neither IPv4 nor IPv6.
-			sessions->Weird("non_ip_packet_in_ppp_encapsulation", &hdr, data);
-			data = 0;
-			return;
-			}
-		break;
-		}
-	}
-
-	if ( have_mpls )
-		{
-		// Skip the MPLS label stack.
-		bool end_of_stack = false;
-
-		while ( ! end_of_stack )
-			{
-			end_of_stack = *(data + 2) & 0x01;
-			data += 4;
-			}
-		}
-
-	if ( pseudo_realtime )
-		{
-		current_pseudo = CheckPseudoTime();
-		net_packet_dispatch(current_pseudo, &hdr, data, pkt_hdr_size, this);
-		if ( ! first_wallclock )
-			first_wallclock = current_time(true);
-		}
-
-	else
-		net_packet_dispatch(current_timestamp, &hdr, data, pkt_hdr_size, this);
-
-	data = 0;
-	}
-
-bool PktSrc::GetCurrentPacket(const struct pcap_pkthdr** arg_hdr,
-				const u_char** arg_pkt)
-	{
-	if ( ! last_data )
-		return false;
-
-	*arg_hdr = &hdr;
-	*arg_pkt = last_data;
-	return true;
-	}
-
-int PktSrc::PrecompileFilter(int index, const char* filter)
-	{
-	// Compile filter.
-	BPF_Program* code = new BPF_Program();
-
-	if ( ! code->Compile(pd, filter, netmask, errbuf, sizeof(errbuf)) )
-		{
-		delete code;
-		return 0;
-		}
-
-	// Store it in hash.
-	HashKey* hash = new HashKey(HashKey(bro_int_t(index)));
-	BPF_Program* oldcode = filters.Lookup(hash);
-	if ( oldcode )
-		delete oldcode;
-
-	filters.Insert(hash, code);
-	delete hash;
-
-	return 1;
-	}
-
-int PktSrc::SetFilter(int index)
-	{
-	// We don't want load-level filters for the secondary path.
-	if ( filter_type == TYPE_FILTER_SECONDARY && index > 0 )
-		return 1;
-
-	HashKey* hash = new HashKey(HashKey(bro_int_t(index)));
-	BPF_Program* code = filters.Lookup(hash);
-	delete hash;
-
-	if ( ! code )
-		{
-		safe_snprintf(errbuf, sizeof(errbuf),
-			      "No precompiled pcap filter for index %d",
-			      index);
-		return 0;
-		}
-
-	if ( pcap_setfilter(pd, code->GetProgram()) < 0 )
-		{
-		safe_snprintf(errbuf, sizeof(errbuf),
-			      "pcap_setfilter(%d): %s",
-			      index, pcap_geterr(pd));
-		return 0;
-		}
-
-#ifndef HAVE_LINUX
-	// Linux doesn't clear counters when resetting filter.
-	stats.received = stats.dropped = stats.link = 0;
-#endif
-
-	return 1;
-	}
-
-void PktSrc::SetHdrSize()
-	{
-	int dl = pcap_datalink(pd);
-	hdr_size = get_link_header_size(dl);
-
-	if ( hdr_size < 0 )
-		{
-		safe_snprintf(errbuf, sizeof(errbuf),
-			 "unknown data link type 0x%x", dl);
-		Close();
-		}
-
-	datalink = dl;
-	}
-
-void PktSrc::Close()
-	{
-	if ( pd )
-		{
-		pcap_close(pd);
-		pd = 0;
-		closed = true;
-		}
-	}
-
-void PktSrc::AddSecondaryTablePrograms()
-	{
-	BPF_Program* program;
-
-	loop_over_list(secondary_path->EventTable(), i)
-		{
-		SecondaryEvent* se = secondary_path->EventTable()[i];
-		program = new BPF_Program();
-
-		if ( ! program->Compile(snaplen, datalink, se->Filter(),
-					netmask, errbuf, sizeof(errbuf)) )
-			{
-			delete program;
-			Close();
-			return;
-			}
-
-		SecondaryProgram* sp = new SecondaryProgram(program, se);
-		program_list.append(sp);
-		}
-	}
-
-void PktSrc::Statistics(Stats* s)
-	{
-	if ( reading_traces )
-		s->received = s->dropped = s->link = 0;
-
-	else
-		{
-		struct pcap_stat pstat;
-		if ( pcap_stats(pd, &pstat) < 0 )
-			{
-			reporter->Error("problem getting packet filter statistics: %s",
-					ErrorMsg());
-			s->received = s->dropped = s->link = 0;
-			}
-
-		else
-			{
-			s->dropped = pstat.ps_drop;
-			s->link = pstat.ps_recv;
-			}
-		}
-
-	s->received = stats.received;
-
-	if ( pseudo_realtime )
-		s->dropped = 0;
-
-	stats.dropped = s->dropped;
-	}
-
-PktInterfaceSrc::PktInterfaceSrc(const char* arg_interface, const char* filter,
-					PktSrc_Filter_Type ft)
-: PktSrc()
-	{
-	char tmp_errbuf[PCAP_ERRBUF_SIZE];
-	filter_type = ft;
-
-	// Determine interface if not specified.
-	if ( ! arg_interface && ! (arg_interface = pcap_lookupdev(tmp_errbuf)) )
-		{
-		safe_snprintf(errbuf, sizeof(errbuf),
-			 "pcap_lookupdev: %s", tmp_errbuf);
-		return;
-		}
-
-	interface = copy_string(arg_interface);
-
-	// Determine network and netmask.
-	uint32 net;
-	if ( pcap_lookupnet(interface, &net, &netmask, tmp_errbuf) < 0 )
-		{
-		// ### The lookup can fail if no address is assigned to
-		// the interface; and libpcap doesn't have any useful notion
-		// of error codes, just error strings - how bogus - so we
-		// just kludge around the error :-(.
-		// sprintf(errbuf, "pcap_lookupnet %s", tmp_errbuf);
-		// return;
-		net = 0;
-		netmask = 0xffffff00;
-		}
-
-	// We use the smallest time-out possible to return almost immediately if
-	// no packets are available. (We can't use set_nonblocking() as it's
-	// broken on FreeBSD: even when select() indicates that we can read
-	// something, we may get nothing if the store buffer hasn't filled up
-	// yet.)
-	pd = pcap_open_live(interface, snaplen, 1, 1, tmp_errbuf);
-
-	if ( ! pd )
-		{
-		safe_snprintf(errbuf, sizeof(errbuf),
-			 "pcap_open_live: %s", tmp_errbuf);
-		closed = true;
-		return;
-		}
-
-	// ### This needs autoconf'ing.
-#ifdef HAVE_PCAP_INT_H
-	reporter->Info("pcap bufsize = %d\n", ((struct pcap *) pd)->bufsize);
-#endif
-
-#ifdef HAVE_LINUX
-	if ( pcap_setnonblock(pd, 1, tmp_errbuf) < 0 )
-		{
-		safe_snprintf(errbuf, sizeof(errbuf),
-			 "pcap_setnonblock: %s", tmp_errbuf);
-		pcap_close(pd);
-		closed = true;
-		return;
-		}
-#endif
-	selectable_fd = pcap_fileno(pd);
-
-	if ( PrecompileFilter(0, filter) && SetFilter(0) )
-		{
-		SetHdrSize();
-
-		if ( closed )
-			// Couldn't get header size.
-			return;
-
-		reporter->Info("listening on %s, capture length %d bytes\n", interface, snaplen);
-		}
-	else
-		closed = true;
-	}
-
-
-PktFileSrc::PktFileSrc(const char* arg_readfile, const char* filter,
-			PktSrc_Filter_Type ft)
-: PktSrc()
-	{
-	readfile = copy_string(arg_readfile);
-
-	filter_type = ft;
-
-	pd = pcap_open_offline((char*) readfile, errbuf);
-
-	if ( pd && PrecompileFilter(0, filter) && SetFilter(0) )
-		{
-		SetHdrSize();
-
-		if ( closed )
-			// Unknown link layer type.
-			return;
-
-		// We don't put file sources into non-blocking mode as
-		// otherwise we would not be able to identify the EOF.
-
-		selectable_fd = fileno(pcap_file(pd));
-
-		if ( selectable_fd < 0 )
-			reporter->InternalError("OS does not support selectable pcap fd");
-		}
-	else
-		closed = true;
-	}
-
-
-SecondaryPath::SecondaryPath()
-	{
-	filter = 0;
-
-	// Glue together the secondary filter, if exists.
-	Val* secondary_fv = internal_val("secondary_filters");
-	if ( secondary_fv->AsTableVal()->Size() == 0 )
-		return;
-
-	int did_first = 0;
-	const TableEntryValPDict* v = secondary_fv->AsTable();
-	IterCookie* c = v->InitForIteration();
-	TableEntryVal* tv;
-	HashKey* h;
-
-	while ( (tv = v->NextEntry(h, c)) )
-		{
-		// Get the index values.
-		ListVal* index =
-			secondary_fv->AsTableVal()->RecoverIndex(h);
-
-		const char* str =
-			index->Index(0)->Ref()->AsString()->CheckString();
-
-		if ( ++did_first == 1 )
-			{
-			filter = copy_string(str);
-			}
-		else
-			{
-			if ( strlen(filter) > 0 )
-				{
-				char* tmp_f = new char[strlen(str) + strlen(filter) + 32];
-				if ( strlen(str) == 0 )
-					sprintf(tmp_f, "%s", filter);
-				else
-					sprintf(tmp_f, "(%s) or (%s)", filter, str);
-				delete [] filter;
-				filter = tmp_f;
-				}
-			}
-
-		// Build secondary_path event table item and link it.
-		SecondaryEvent* se =
-			new SecondaryEvent(index->Index(0)->Ref()->AsString()->CheckString(),
-				tv->Value()->AsFunc() );
-
-		event_list.append(se);
-
-		delete h;
-		Unref(index);
-		}
-	}
-
-SecondaryPath::~SecondaryPath()
-	{
-	loop_over_list(event_list, i)
-		delete event_list[i];
-
-	delete [] filter;
-	}
-
-
-SecondaryProgram::~SecondaryProgram()
-	{
-	delete program;
-	}
-
-PktDumper::PktDumper(const char* arg_filename, bool arg_append)
-	{
-	filename[0] = '\0';
-	is_error = false;
-	append = arg_append;
-	dumper = 0;
-	open_time = 0.0;
-
-	// We need a pcap_t with a reasonable link-layer type. We try to get it
-	// from the packet sources. If not available, we fall back to Ethernet.
-	// FIXME: Perhaps we should make this configurable?
-	int linktype = -1;
-
-	if ( pkt_srcs.length() )
-		linktype = pkt_srcs[0]->LinkType();
-
-	if ( linktype < 0 )
-		linktype = DLT_EN10MB;
-
-	pd = pcap_open_dead(linktype, snaplen);
-	if ( ! pd )
-		{
-		Error("error for pcap_open_dead");
-		return;
-		}
-
-	if ( arg_filename )
-		Open(arg_filename);
-	}
-
-bool PktDumper::Open(const char* arg_filename)
-	{
-	if ( ! arg_filename && ! *filename )
-		{
-		Error("no filename given");
-		return false;
-		}
-
-	if ( arg_filename )
-		{
-		if ( dumper && streq(arg_filename, filename) )
-			// Already open.
-			return true;
-
-		safe_strncpy(filename, arg_filename, FNBUF_LEN);
-		}
-
-	if ( dumper )
-		Close();
-
-	struct stat s;
-	int exists = 0;
-
-	if ( append )
-		{
-		// See if output file already exists (and is non-empty).
-		exists = stat(filename, &s); ;
-
-		if ( exists < 0 && errno != ENOENT )
-			{
-			Error(fmt("can't stat file %s: %s", filename, strerror(errno)));
-			return false;
-			}
-		}
-
-	if ( ! append || exists < 0 || s.st_size == 0 )
-		{
-		// Open new file.
-		dumper =  pcap_dump_open(pd, filename);
-		if ( ! dumper )
-			{
-			Error(pcap_geterr(pd));
-			return false;
-			}
-		}
-
-	else
-		{
-		// Old file and we need to append, which, unfortunately,
-		// is not supported by libpcap. So, we have to hack a
-		// little bit, knowing that pcap_dumpter_t is, in fact,
-		// a FILE ... :-(
-		dumper = (pcap_dumper_t*) fopen(filename, "a");
-		if ( ! dumper )
-			{
-			Error(fmt("can't open dump %s: %s", filename, strerror(errno)));
-			return false;
-			}
-		}
-
-	open_time = network_time;
-	is_error = false;
-	return true;
-	}
-
-bool PktDumper::Close()
-	{
-	if ( dumper )
-		{
-		pcap_dump_close(dumper);
-		dumper = 0;
-		is_error = false;
-		}
-
-	return true;
-	}
-
-bool PktDumper::Dump(const struct pcap_pkthdr* hdr, const u_char* pkt)
-	{
-	if ( ! dumper )
-		return false;
-
-	if ( ! open_time )
-		open_time = network_time;
-
-	pcap_dump((u_char*) dumper, hdr, pkt);
-
-	return true;
-	}
-
-void PktDumper::Error(const char* errstr)
-	{
-	safe_strncpy(errbuf, errstr, sizeof(errbuf));
-	is_error = true;
-	}
-
-int get_link_header_size(int dl)
-	{
-	switch ( dl ) {
-	case DLT_NULL:
-		return 4;
-
-	case DLT_EN10MB:
-		return 14;
-
-	case DLT_FDDI:
-		return 13 + 8;	// fddi_header + LLC
-
-#ifdef DLT_LINUX_SLL
-	case DLT_LINUX_SLL:
-		return 16;
-#endif
-
-	case DLT_PPP_SERIAL:	// PPP_SERIAL
-		return 4;
-
-	case DLT_RAW:
-		return 0;
-	}
-
-	return -1;
-	}

src/PktSrc.h

@@ -1,258 +0,0 @@
-// See the file "COPYING" in the main distribution directory for copyright.
-
-#ifndef pktsrc_h
-#define pktsrc_h
-
-#include "Dict.h"
-#include "Expr.h"
-#include "BPF_Program.h"
-#include "IOSource.h"
-#include "RemoteSerializer.h"
-
-#define BRO_PCAP_ERRBUF_SIZE   PCAP_ERRBUF_SIZE + 256
-
-extern "C" {
-#include <pcap.h>
-}
-
-declare(PDict,BPF_Program);
-
-// Whether a PktSrc object is used by the normal filter structure or the
-// secondary-path structure.
-typedef enum {
-	TYPE_FILTER_NORMAL,  // the normal filter
-	TYPE_FILTER_SECONDARY,  // the secondary-path filter
-} PktSrc_Filter_Type;
-
-
-// {filter,event} tuples conforming the secondary path.
-class SecondaryEvent {
-public:
-	SecondaryEvent(const char* arg_filter, Func* arg_event)
-		{
-		filter = arg_filter;
-		event = arg_event;
-		}
-
-	const char* Filter()	{ return filter; }
-	Func* Event()		{ return event; }
-
-private:
-	const char* filter;
-	Func* event;
-};
-
-declare(PList,SecondaryEvent);
-typedef PList(SecondaryEvent) secondary_event_list;
-
-
-
-class SecondaryPath {
-public:
-	SecondaryPath();
-	~SecondaryPath();
-
-	secondary_event_list& EventTable()	{ return event_list; }
-	const char* Filter()			{ return filter; }
-
-private:
-	secondary_event_list event_list;
-	// OR'ed union of all SecondaryEvent filters
-	char* filter;
-};
-
-// Main secondary-path object.
-extern SecondaryPath* secondary_path;
-
-
-// {program, {filter,event}} tuple table.
-class SecondaryProgram {
-public:
-	SecondaryProgram(BPF_Program* arg_program, SecondaryEvent* arg_event)
-		{
-		program = arg_program;
-		event = arg_event;
-		}
-
-	~SecondaryProgram();
-
-	BPF_Program* Program()  { return program; }
-	SecondaryEvent* Event()	{ return event; }
-
-private:
-	// Associated program.
-	BPF_Program *program;
-
-	// Event that is run in case the program is matched.
-	SecondaryEvent* event;
-};
-
-declare(PList,SecondaryProgram);
-typedef PList(SecondaryProgram) secondary_program_list;
-
-
-
-class PktSrc : public IOSource {
-public:
-	~PktSrc();
-
-	// IOSource interface
-	bool IsReady();
-	void GetFds(int* read, int* write, int* except);
-	double NextTimestamp(double* local_network_time);
-	void Process();
-	const char* Tag()	{ return "PktSrc"; }
-
-	const char* ErrorMsg() const	{ return errbuf; }
-	void ClearErrorMsg()		{ *errbuf ='\0'; }
-
-	// Returns the packet last processed; false if there is no
-	// current packet available.
-	bool GetCurrentPacket(const pcap_pkthdr** hdr, const u_char** pkt);
-
-	int HdrSize() const		{ return hdr_size; }
-	int DataLink() const		{ return datalink; }
-
-	void ConsumePacket()	{ data = 0; }
-
-	int IsLive() const		{ return interface != 0; }
-
-	pcap_t* PcapHandle() const	{ return pd; }
-	int LinkType() const		{ return pcap_datalink(pd); }
-
-	const char* ReadFile() const	{ return readfile; }
-	const char* Interface() const	{ return interface; }
-	PktSrc_Filter_Type FilterType() const	{ return filter_type; }
-	void AddSecondaryTablePrograms();
-	const secondary_program_list& ProgramTable() const
-		{ return program_list; }
-
-	// Signal packet source that processing was suspended and is now going
-	// to be continued.
-	void ContinueAfterSuspend();
-
-	// Only valid in pseudo-realtime mode.
-	double CurrentPacketTimestamp()	{ return current_pseudo; }
-	double CurrentPacketWallClock();
-
-	struct Stats {
-		unsigned int received;	// pkts received (w/o drops)
-		unsigned int dropped;	// pkts dropped
-		unsigned int link;	// total packets on link
-					// (not always not available)
-	};
-
-	virtual void Statistics(Stats* stats);
-
-	// Precompiles a filter and associates the given index with it.
-	// Returns true on success, 0 if a problem occurred.
-	virtual int PrecompileFilter(int index, const char* filter);
-
-	// Activates the filter with the given index.
-	// Returns true on success, 0 if a problem occurred.
-	virtual int SetFilter(int index);
-
-protected:
-	PktSrc();
-
-	static const int PCAP_TIMEOUT = 20;
-
-	void SetHdrSize();
-
-	virtual void Close();
-
-	// Returns 1 on success, 0 on time-out/gone dry.
-	virtual int ExtractNextPacket();
-
-	// Checks if the current packet has a pseudo-time <= current_time.
-	// If yes, returns pseudo-time, otherwise 0.
-	double CheckPseudoTime();
-
-	double current_timestamp;
-	double next_timestamp;
-
-	// Only set in pseudo-realtime mode.
-	double first_timestamp;
-	double first_wallclock;
-	double current_wallclock;
-	double current_pseudo;
-
-	struct pcap_pkthdr hdr;
-	const u_char* data;	// contents of current packet
-	const u_char* last_data;	// same, but unaffected by consuming
-	int hdr_size;
-	int datalink;
-	double next_sync_point; // For trace synchronziation in pseudo-realtime
-
-	char* interface;	// nil if not reading from an interface
-	char* readfile;		// nil if not reading from a file
-
-	pcap_t* pd;
-	int selectable_fd;
-	uint32 netmask;
-	char errbuf[BRO_PCAP_ERRBUF_SIZE];
-
-	Stats stats;
-
-	PDict(BPF_Program) filters; // precompiled filters
-
-	PktSrc_Filter_Type filter_type; // normal path or secondary path
-	secondary_program_list program_list;
-};
-
-class PktInterfaceSrc : public PktSrc {
-public:
-	PktInterfaceSrc(const char* interface, const char* filter,
-			PktSrc_Filter_Type ft=TYPE_FILTER_NORMAL);
-};
-
-class PktFileSrc : public PktSrc {
-public:
-	PktFileSrc(const char* readfile, const char* filter,
-			PktSrc_Filter_Type ft=TYPE_FILTER_NORMAL);
-};
-
-
-extern int get_link_header_size(int dl);
-
-class PktDumper {
-public:
-	PktDumper(const char* file = 0, bool append = false);
-	~PktDumper()	{ Close(); }
-
-	bool Open(const char* file = 0);
-	bool Close();
-	bool Dump(const struct pcap_pkthdr* hdr, const u_char* pkt);
-
-	pcap_dumper_t* PcapDumper() 	{ return dumper; }
-
-	const char* FileName() const	{ return filename; }
-	bool IsError() const		{ return is_error; }
-	const char* ErrorMsg() const	{ return errbuf; }
-
-	// This heuristic will horribly fail if we're using packets
-	// with different link layers.  (If we can't derive a reasonable value
-	// from the packet sources, our fall-back is Ethernet.)
-	int HdrSize() const
-		{ return get_link_header_size(pcap_datalink(pd)); }
-
-	// Network time when dump file was opened.
-	double OpenTime() const		{ return open_time; }
-
-private:
-	void InitPd();
-	void Error(const char* str);
-
-	static const int FNBUF_LEN = 1024;
-	char filename[FNBUF_LEN];
-
-	bool append;
-	pcap_dumper_t* dumper;
-	pcap_t* pd;
-	double open_time;
-
-	bool is_error;
-	char errbuf[BRO_PCAP_ERRBUF_SIZE];
-};
-
-#endif

src/RemoteSerializer.cc

@@ -188,10 +188,10 @@
 #include "File.h"
 #include "Conn.h"
 #include "Reporter.h"
-#include "threading/SerialTypes.h"
-#include "logging/Manager.h"
 #include "IPAddr.h"
 #include "bro_inet_ntop.h"
+#include "iosource/Manager.h"
+#include "logging/Manager.h"
 #include "logging/logging.bif.h"
 
 extern "C" {
@@ -285,10 +285,10 @@ struct ping_args {
 	\
 	if ( ! c ) \
 		{ \
-		idle = io->IsIdle();\
+		SetIdle(io->IsIdle());\
 		return true; \
 		} \
-	idle = false; \
+	SetIdle(false); \
 	}
 
 static const char* msgToStr(int msg)
@@ -534,7 +534,6 @@ RemoteSerializer::RemoteSerializer()
 	current_sync_point = 0;
 	syncing_times = false;
 	io = 0;
-	closed = false;
 	terminating = false;
 	in_sync = 0;
 	last_flush = 0;
@@ -559,7 +558,7 @@ RemoteSerializer::~RemoteSerializer()
 	delete io;
 	}
 
-void RemoteSerializer::Init()
+void RemoteSerializer::Enable()
 	{
 	if ( initialized )
 		return;
@@ -572,7 +571,7 @@ void RemoteSerializer::Init()
 
 	Fork();
 
-	io_sources.Register(this);
+	iosource_mgr->Register(this);
 
 	Log(LogInfo, fmt("communication started, parent pid is %d, child pid is %d", getpid(), child_pid));
 	initialized = 1;
@@ -1276,7 +1275,7 @@ bool RemoteSerializer::Listen(const IPAddr& ip, uint16 port, bool expect_ssl,
 		return false;
 
 	listening = true;
-	closed = false;
+	SetClosed(false);
 	return true;
 	}
 
@@ -1345,7 +1344,7 @@ bool RemoteSerializer::StopListening()
 		return false;
 
 	listening = false;
-	closed = ! IsActive();
+	SetClosed(! IsActive());
 	return true;
 	}
 
@@ -1383,7 +1382,7 @@ double RemoteSerializer::NextTimestamp(double* local_network_time)
 	if ( received_logs > 0 )
 		{
 		// If we processed logs last time, assume there's more.
-		idle = false;
+		SetIdle(false);
 		received_logs = 0;
 		return timer_mgr->Time();
 		}
@@ -1398,7 +1397,7 @@ double RemoteSerializer::NextTimestamp(double* local_network_time)
 		pt = timer_mgr->Time();
 
 	if ( packets.length() )
-		idle = false;
+		SetIdle(false);
 
 	if ( et >= 0 && (et < pt || pt < 0) )
 		return et;
@@ -1477,7 +1476,7 @@ void RemoteSerializer::Process()
 		}
 
 	if ( packets.length() )
-		idle = false;
+		SetIdle(false);
 	}
 
 void RemoteSerializer::Finish()
@@ -1509,7 +1508,7 @@ bool RemoteSerializer::Poll(bool may_block)
 		}
 
 	io->Flush();
-	idle = false;
+	SetIdle(false);
 
 	switch ( msgstate ) {
 	case TYPE:
@@ -1691,7 +1690,7 @@ bool RemoteSerializer::DoMessage()
 
 	case MSG_TERMINATE:
 		assert(terminating);
-		io_sources.Terminate();
+		iosource_mgr->Terminate();
 		return true;
 
 	case MSG_REMOTE_PRINT:
@@ -1879,7 +1878,7 @@ void RemoteSerializer::RemovePeer(Peer* peer)
 	delete peer->cache_out;
 	delete peer;
 
-	closed = ! IsActive();
+	SetClosed(! IsActive());
 
 	if ( in_sync == peer )
 		in_sync = 0;
@@ -2841,7 +2840,7 @@ void RemoteSerializer::GotEvent(const char* name, double time,
 	BufferedEvent* e = new BufferedEvent;
 
 	// Our time, not the time when the event was generated.
-	e->time = pkt_srcs.length() ?
+	e->time = iosource_mgr->GetPktSrcs().size() ?
 			time_t(network_time) : time_t(timer_mgr->Time());
 
 	e->src = current_peer->id;
@@ -3086,7 +3085,7 @@ RecordVal* RemoteSerializer::GetPeerVal(PeerID id)
 void RemoteSerializer::ChildDied()
 	{
 	Log(LogError, "child died");
-	closed = true;
+	SetClosed(true);
 	child_pid = 0;
 
 	// Shut down the main process as well.
@@ -3185,7 +3184,7 @@ void RemoteSerializer::FatalError(const char* msg)
 	Log(LogError, msg);
 	reporter->Error("%s", msg);
 
-	closed = true;
+	SetClosed(true);
 
 	if ( kill(child_pid, SIGQUIT) < 0 )
 		reporter->Warning("warning: cannot kill child pid %d, %s", child_pid, strerror(errno));

src/RemoteSerializer.h

@@ -6,7 +6,7 @@
 #include "Dict.h"
 #include "List.h"
 #include "Serializer.h"
-#include "IOSource.h"
+#include "iosource/IOSource.h"
 #include "Stats.h"
 #include "File.h"
 #include "logging/WriterBackend.h"
@@ -22,13 +22,13 @@ namespace threading {
 }
 
 // This class handles the communication done in Bro's main loop.
-class RemoteSerializer : public Serializer, public IOSource {
+class RemoteSerializer : public Serializer, public iosource::IOSource {
 public:
 	RemoteSerializer();
 	virtual ~RemoteSerializer();
 
 	// Initialize the remote serializer (calling this will fork).
-	void Init();
+	void Enable();
 
 	// FIXME: Use SourceID directly (or rename everything to Peer*).
 	typedef SourceID PeerID;

src/Serializer.cc

@@ -19,6 +19,7 @@
 #include "Conn.h"
 #include "Timer.h"
 #include "RemoteSerializer.h"
+#include "iosource/Manager.h"
 
 Serializer::Serializer(SerializationFormat* arg_format)
 	{
@@ -1045,7 +1046,7 @@ EventPlayer::EventPlayer(const char* file)
 		Error(fmt("event replayer: cannot open %s", file));
 
 	if ( ReadHeader() )
-		io_sources.Register(this);
+		iosource_mgr->Register(this);
 	}
 
 EventPlayer::~EventPlayer()
@@ -1085,7 +1086,7 @@ double EventPlayer::NextTimestamp(double* local_network_time)
 		{
 		UnserialInfo info(this);
 		Unserialize(&info);
-		closed = io->Eof();
+		SetClosed(io->Eof());
 		}
 
 	if ( ! ne_time )
@@ -1142,7 +1143,7 @@ bool Packet::Serialize(SerialInfo* info) const
 static BroFile* profiling_output = 0;
 
 #ifdef DEBUG
-static PktDumper* dump = 0;
+static iosource::PktDumper* dump = 0;
 #endif
 
 Packet* Packet::Unserialize(UnserialInfo* info)
@@ -1188,7 +1189,7 @@ Packet* Packet::Unserialize(UnserialInfo* info)
 	p->hdr = hdr;
 	p->pkt = (u_char*) pkt;
 	p->tag = tag;
-	p->hdr_size = get_link_header_size(p->link_type);
+	p->hdr_size = iosource::PktSrc::GetLinkHeaderSize(p->link_type);
 
 	delete [] tag;
 
@@ -1213,9 +1214,15 @@ Packet* Packet::Unserialize(UnserialInfo* info)
 	if ( debug_logger.IsEnabled(DBG_TM) )
 		{
 		if ( ! dump )
-			dump = new PktDumper("tm.pcap");
+			dump = iosource_mgr->OpenPktDumper("tm.pcap", true);
 
-		dump->Dump(p->hdr, p->pkt);
+		if ( dump )
+			{
+			iosource::PktDumper::Packet dp;
+			dp.hdr = p->hdr;
+			dp.data = p->pkt;
+			dump->Dump(&dp);
+			}
 		}
 #endif
 

src/Serializer.h

@@ -15,7 +15,7 @@
 #include "SerialInfo.h"
 #include "IP.h"
 #include "Timer.h"
-#include "IOSource.h"
+#include "iosource/IOSource.h"
 #include "Reporter.h"
 
 class SerializationCache;
@@ -350,7 +350,7 @@ public:
 };
 
 // Plays a file of events back.
-class EventPlayer : public FileSerializer, public IOSource {
+class EventPlayer : public FileSerializer, public iosource::IOSource {
 public:
 	EventPlayer(const char* file);
 	virtual ~EventPlayer();

src/Sessions.cc

@@ -167,7 +167,7 @@ void NetSessions::Done()
 
 void NetSessions::DispatchPacket(double t, const struct pcap_pkthdr* hdr,
 			const u_char* pkt, int hdr_size,
-			PktSrc* src_ps)
+			iosource::PktSrc* src_ps)
 	{
 	const struct ip* ip_hdr = 0;
 	const u_char* ip_data = 0;
@@ -184,10 +184,7 @@ void NetSessions::DispatchPacket(double t, const struct pcap_pkthdr* hdr,
 		// Blanket encapsulation
 		hdr_size += encap_hdr_size;
 
-	if ( src_ps->FilterType() == TYPE_FILTER_NORMAL )
+	NextPacket(t, hdr, pkt, hdr_size);
-		NextPacket(t, hdr, pkt, hdr_size);
-	else
-		NextPacketSecondary(t, hdr, pkt, hdr_size, src_ps);
 	}
 
 void NetSessions::NextPacket(double t, const struct pcap_pkthdr* hdr,
@@ -262,53 +259,6 @@ void NetSessions::NextPacket(double t, const struct pcap_pkthdr* hdr,
 		DumpPacket(hdr, pkt);
 	}
 
-void NetSessions::NextPacketSecondary(double /* t */, const struct pcap_pkthdr* hdr,
-				const u_char* const pkt, int hdr_size,
-				const PktSrc* src_ps)
-	{
-	SegmentProfiler(segment_logger, "processing-secondary-packet");
-
-	++num_packets_processed;
-
-	uint32 caplen = hdr->caplen - hdr_size;
-	if ( caplen < sizeof(struct ip) )
-		{
-		Weird("truncated_IP", hdr, pkt);
-		return;
-		}
-
-	const struct ip* ip = (const struct ip*) (pkt + hdr_size);
-	if ( ip->ip_v == 4 )
-		{
-		const secondary_program_list& spt = src_ps->ProgramTable();
-
-		loop_over_list(spt, i)
-			{
-			SecondaryProgram* sp = spt[i];
-			if ( ! net_packet_match(sp->Program(), pkt,
-						hdr->len, hdr->caplen) )
-				continue;
-
-			val_list* args = new val_list;
-			StringVal* cmd_val =
-				new StringVal(sp->Event()->Filter());
-			args->append(cmd_val);
-			IP_Hdr ip_hdr(ip, false);
-			args->append(ip_hdr.BuildPktHdrVal());
-			// ### Need to queue event here.
-			try
-				{
-				sp->Event()->Event()->Call(args);
-				}
-
-			catch ( InterpreterException& e )
-				{ /* Already reported. */ }
-
-			delete args;
-			}
-		}
-	}
-
 int NetSessions::CheckConnectionTag(Connection* conn)
 	{
 	if ( current_iosrc->GetCurrentTag() )
@@ -1440,14 +1390,24 @@ void NetSessions::DumpPacket(const struct pcap_pkthdr* hdr,
 		return;
 
 	if ( len == 0 )
-		pkt_dumper->Dump(hdr, pkt);
+		{
+		iosource::PktDumper::Packet p;
+		p.hdr = hdr;
+		p.data = pkt;
+		pkt_dumper->Dump(&p);
+		}
+
 	else
 		{
 		struct pcap_pkthdr h = *hdr;
 		h.caplen = len;
 		if ( h.caplen > hdr->caplen )
 			reporter->InternalError("bad modified caplen");
-		pkt_dumper->Dump(&h, pkt);
+
+		iosource::PktDumper::Packet p;
+		p.hdr = &h;
+		p.data = pkt;
+		pkt_dumper->Dump(&p);
 		}
 	}
 

src/Sessions.h

@@ -69,11 +69,11 @@ public:
 	~NetSessions();
 
 	// Main entry point for packet processing. Dispatches the packet
-	// either through NextPacket() or NextPacketSecondary(), optionally
+	// either through NextPacket(), optionally employing the packet
-	// employing the packet sorter first.
+	// sorter first.
 	void DispatchPacket(double t, const struct pcap_pkthdr* hdr,
 			const u_char* const pkt, int hdr_size,
-			PktSrc* src_ps);
+			iosource::PktSrc* src_ps);
 
 	void Done();	// call to drain events before destructing
 
@@ -221,10 +221,6 @@ protected:
 	void NextPacket(double t, const struct pcap_pkthdr* hdr,
 			const u_char* const pkt, int hdr_size);
 
-	void NextPacketSecondary(double t, const struct pcap_pkthdr* hdr,
-			const u_char* const pkt, int hdr_size,
-			const PktSrc* src_ps);
-
 	// Record the given packet (if a dumper is active).  If len=0
 	// then the whole packet is recorded, otherwise just the first
 	// len bytes.

src/analyzer/Analyzer.cc

@@ -4,6 +4,7 @@
 
 #include "Analyzer.h"
 #include "Manager.h"
+#include "binpac.h"
 
 #include "analyzer/protocol/pia/PIA.h"
 #include "../Event.h"

src/analyzer/protocol/finger/Plugin.cc

@@ -1,6 +1,5 @@
 // See the file  in the main distribution directory for copyright.
 
-
 #include "plugin/Plugin.h"
 
 #include "Finger.h"

src/bro.bif

@@ -21,6 +21,7 @@
 #include "IPAddr.h"
 #include "util.h"
 #include "file_analysis/Manager.h"
+#include "iosource/Manager.h"
 
 using namespace std;
 
@@ -33,7 +34,7 @@ TableType* var_sizes;
 // and hence it's declared in NetVar.{h,cc}.
 extern RecordType* gap_info;
 
-static PktDumper* addl_pkt_dumper = 0;
+static iosource::PktDumper* addl_pkt_dumper = 0;
 
 bro_int_t parse_int(const char*& fmt)
 	{
@@ -1675,11 +1676,14 @@ function net_stats%(%): NetStats
 	unsigned int drop = 0;
 	unsigned int link = 0;
 
-	loop_over_list(pkt_srcs, i)
+	const iosource::Manager::PktSrcList& pkt_srcs(iosource_mgr->GetPktSrcs());
-		{
-		PktSrc* ps = pkt_srcs[i];
 
-		struct PktSrc::Stats stat;
+	for ( iosource::Manager::PktSrcList::const_iterator i = pkt_srcs.begin();
+	      i != pkt_srcs.end(); i++ )
+		{
+		iosource::PktSrc* ps = *i;
+
+		struct iosource::PktSrc::Stats stat;
 		ps->Statistics(&stat);
 		recv += stat.received;
 		drop += stat.dropped;
@@ -3224,10 +3228,15 @@ function dump_current_packet%(file_name: string%) : bool
 		return new Val(0, TYPE_BOOL);
 
 	if ( ! addl_pkt_dumper )
-		addl_pkt_dumper = new PktDumper(0, true);
+		addl_pkt_dumper = iosource_mgr->OpenPktDumper(file_name->CheckString(), true);
 
-	addl_pkt_dumper->Open(file_name->CheckString());
+	if ( addl_pkt_dumper )
-	addl_pkt_dumper->Dump(hdr, pkt);
+		{
+		iosource::PktDumper::Packet p;
+		p.hdr = hdr;
+		p.data = pkt;
+		addl_pkt_dumper->Dump(&p);
+		}
 
 	return new Val(! addl_pkt_dumper->IsError(), TYPE_BOOL);
 	%}
@@ -3284,10 +3293,15 @@ function dump_packet%(pkt: pcap_packet, file_name: string%) : bool
 	hdr.len = (*pkt_vl)[3]->AsCount();
 
 	if ( ! addl_pkt_dumper )
-		addl_pkt_dumper = new PktDumper(0, true);
+		addl_pkt_dumper = iosource_mgr->OpenPktDumper(file_name->CheckString(), true);
 
-	addl_pkt_dumper->Open(file_name->CheckString());
+	if ( addl_pkt_dumper )
-	addl_pkt_dumper->Dump(&hdr, (*pkt_vl)[4]->AsString()->Bytes());
+		{
+		iosource::PktDumper::Packet p;
+		p.hdr = &hdr;
+		p.data = (*pkt_vl)[4]->AsString()->Bytes();
+		addl_pkt_dumper->Dump(&p);
+		}
 
 	return new Val(addl_pkt_dumper->IsError(), TYPE_BOOL);
 	%}
@@ -4110,14 +4124,14 @@ function rotate_file_by_name%(f: string%): rotate_info
 	bool is_addl_pkt_dumper = false;
 
 	// Special case: one of current dump files.
-	if ( pkt_dumper && streq(pkt_dumper->FileName(), f->CheckString()) )
+	if ( pkt_dumper && streq(pkt_dumper->Path().c_str(), f->CheckString()) )
 		{
 		is_pkt_dumper = true;
 		pkt_dumper->Close();
 		}
 
 	if ( addl_pkt_dumper &&
-	     streq(addl_pkt_dumper->FileName(), f->CheckString()) )
+	     streq(addl_pkt_dumper->Path().c_str(), f->CheckString()) )
 		{
 		is_addl_pkt_dumper = true;
 		addl_pkt_dumper->Close();
@@ -4214,103 +4228,6 @@ function enable_raw_output%(f: file%): any
 #
 # ===========================================================================
 
-## Precompiles a PCAP filter and binds it to a given identifier.
-##
-## id: The PCAP identifier to reference the filter *s* later on.
-##
-## s: The PCAP filter. See ``man tcpdump`` for valid expressions.
-##
-## Returns: True if *s* is valid and precompiles successfully.
-##
-## .. bro:see:: install_pcap_filter
-##          install_src_addr_filter
-##          install_src_net_filter
-##          uninstall_src_addr_filter
-##          uninstall_src_net_filter
-##          install_dst_addr_filter
-##          install_dst_net_filter
-##          uninstall_dst_addr_filter
-##          uninstall_dst_net_filter
-##          pcap_error
-function precompile_pcap_filter%(id: PcapFilterID, s: string%): bool
-	%{
-	bool success = true;
-
-	loop_over_list(pkt_srcs, i)
-		{
-		pkt_srcs[i]->ClearErrorMsg();
-
-		if ( ! pkt_srcs[i]->PrecompileFilter(id->ForceAsInt(),
-							s->CheckString()) )
-			{
-			reporter->Error("precompile_pcap_filter: %s",
-				pkt_srcs[i]->ErrorMsg());
-			success = false;
-			}
-		}
-
-	return new Val(success, TYPE_BOOL);
-	%}
-
-## Installs a PCAP filter that has been precompiled with
-## :bro:id:`precompile_pcap_filter`.
-##
-## id: The PCAP filter id of a precompiled filter.
-##
-## Returns: True if the filter associated with *id* has been installed
-##          successfully.
-##
-## .. bro:see:: precompile_pcap_filter
-##              install_src_addr_filter
-##              install_src_net_filter
-##              uninstall_src_addr_filter
-##              uninstall_src_net_filter
-##              install_dst_addr_filter
-##              install_dst_net_filter
-##              uninstall_dst_addr_filter
-##              uninstall_dst_net_filter
-##              pcap_error
-function install_pcap_filter%(id: PcapFilterID%): bool
-	%{
-	bool success = true;
-
-	loop_over_list(pkt_srcs, i)
-		{
-		pkt_srcs[i]->ClearErrorMsg();
-
-		if ( ! pkt_srcs[i]->SetFilter(id->ForceAsInt()) )
-			success = false;
-		}
-
-	return new Val(success, TYPE_BOOL);
-	%}
-
-## Returns a string representation of the last PCAP error.
-##
-## Returns: A descriptive error message of the PCAP function that failed.
-##
-## .. bro:see:: precompile_pcap_filter
-##              install_pcap_filter
-##              install_src_addr_filter
-##              install_src_net_filter
-##              uninstall_src_addr_filter
-##              uninstall_src_net_filter
-##              install_dst_addr_filter
-##              install_dst_net_filter
-##              uninstall_dst_addr_filter
-##              uninstall_dst_net_filter
-function pcap_error%(%): string
-	%{
-	loop_over_list(pkt_srcs, i)
-		{
-		const char* err = pkt_srcs[i]->ErrorMsg();
-		if ( *err )
-			return new StringVal(err);
-		}
-
-	return new StringVal("no error");
-	%}
-
 ## Installs a filter to drop packets from a given IP source address with
 ## a certain probability if none of a given set of TCP flags are set.
 ## Note that for IPv6 packets with a Destination options header that has
@@ -4542,7 +4459,7 @@ function enable_communication%(%): any
 		return 0;
 
 	using_communication = 1;
-	remote_serializer->Init();
+	remote_serializer->Enable();
 	return 0;
 	%}
 

src/iosource/BPF_Program.cc

@@ -58,7 +58,14 @@ int pcap_compile_nopcap(int snaplen_arg, int linktype_arg,
 }
 #endif
 
-BPF_Program::BPF_Program() : m_compiled(), m_program()
+// Simple heuristic to identify filters that always match, so that we can
+// skip the filtering in that case. "ip or not ip" is Bro's default filter.
+static bool filter_matches_anything(const char *filter)
+	{
+	return (! filter) || strlen(filter) == 0 || strcmp(filter, "ip or not ip") == 0;
+	}
+
+BPF_Program::BPF_Program() : m_compiled(), m_matches_anything(false), m_program()
 	{
 	}
 
@@ -86,12 +93,14 @@ bool BPF_Program::Compile(pcap_t* pcap, const char* filter, uint32 netmask,
 		}
 
 	m_compiled = true;
+	m_matches_anything = filter_matches_anything(filter);
 
 	return true;
 	}
 
 bool BPF_Program::Compile(int snaplen, int linktype, const char* filter,
-				uint32 netmask, char* errbuf, bool optimize)
+			  uint32 netmask, char* errbuf, unsigned int errbuf_len,
+			  bool optimize)
 	{
 	FreeCode();
 
@@ -99,15 +108,23 @@ bool BPF_Program::Compile(int snaplen, int linktype, const char* filter,
 	char my_error[PCAP_ERRBUF_SIZE];
 
 	int err = pcap_compile_nopcap(snaplen, linktype, &m_program,
-				     (char *) filter, optimize, netmask, error);
+				     (char *) filter, optimize, netmask, my_error);
 	if ( err < 0 && errbuf )
-		safe_strncpy(errbuf, my_errbuf, PCAP_ERRBUF_SIZE);
+		safe_strncpy(errbuf, my_error, errbuf_len);
+		*errbuf = '\0';
 #else
 	int err = pcap_compile_nopcap(snaplen, linktype, &m_program,
 				     (char*) filter, optimize, netmask);
+
+	if ( err < 0 && errbuf && errbuf_len )
+		*errbuf = '\0';
 #endif
+
 	if ( err == 0 )
+		{
 		m_compiled = true;
+		m_matches_anything = filter_matches_anything(filter);
+		}
 
 	return err == 0;
 	}

src/iosource/BPF_Program.h

@@ -30,12 +30,17 @@ public:
 	// similarly to pcap_compile_nopcap(). Parameters are
 	// similar. Returns true on success.
 	bool Compile(int snaplen, int linktype, const char* filter,
-		uint32 netmask, char* errbuf = 0, bool optimize = true);
+		uint32 netmask, char* errbuf = 0, unsigned int errbuf_len = 0,
+		bool optimize = true);
 
 	// Returns true if this program currently contains compiled
 	// code, false otherwise.
 	bool IsCompiled()	{ return m_compiled; }
 
+	// Returns true if this program matches any packets. This is not
+	// comprehensive, but can identify a few cases where it does.
+	bool MatchesAnything() 	{ return m_matches_anything; }
+
 	// Accessor to the compiled program. Returns nil when
 	// no program is currently compiled.
 	bpf_program* GetProgram();
@@ -46,6 +51,7 @@ protected:
 	// (I like to prefix member variables with m_, makes it clear
 	// in the implementation whether it's a global or not. --ck)
 	bool m_compiled;
+	bool m_matches_anything;
 	struct bpf_program m_program;
 };
 

src/iosource/CMakeLists.txt

@@ -0,0 +1,23 @@
+
+include(BroSubdir)
+
+include_directories(BEFORE
+                    ${CMAKE_CURRENT_SOURCE_DIR}
+                    ${CMAKE_CURRENT_BINARY_DIR}
+)
+
+add_subdirectory(pcap)
+
+set(iosource_SRCS
+    BPF_Program.cc
+    Component.cc
+    Manager.cc
+    PktDumper.cc
+    PktSrc.cc
+)
+
+bif_target(pcap.bif)
+
+bro_add_subdir_library(iosource ${iosource_SRCS})
+add_dependencies(bro_iosource generate_outputs)
+

src/iosource/Component.cc

@@ -0,0 +1,165 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#include "Component.h"
+
+#include "Desc.h"
+#include "Reporter.h"
+
+using namespace iosource;
+
+Component::Component(const std::string& name)
+	: plugin::Component(plugin::component::IOSOURCE, name)
+	{
+	}
+
+Component::Component(plugin::component::Type type, const std::string& name)
+	: plugin::Component(type, name)
+	{
+	}
+
+Component::~Component()
+	{
+	}
+
+PktSrcComponent::PktSrcComponent(const std::string& arg_name, const std::string& arg_prefix, InputType arg_type, factory_callback arg_factory)
+	: iosource::Component(plugin::component::PKTSRC, arg_name)
+	{
+	tokenize_string(arg_prefix, ":", &prefixes);
+	type = arg_type;
+	factory = arg_factory;
+	}
+
+PktSrcComponent::~PktSrcComponent()
+	{
+	}
+
+const std::vector<std::string>& PktSrcComponent::Prefixes() const
+	{
+	return prefixes;
+	}
+
+bool PktSrcComponent::HandlesPrefix(const string& prefix) const
+	{
+	for ( std::vector<std::string>::const_iterator i = prefixes.begin();
+	      i != prefixes.end(); i++ )
+		{
+		if ( *i == prefix )
+			return true;
+		}
+
+	return false;
+	}
+
+bool PktSrcComponent::DoesLive() const
+	{
+	return type == LIVE || type == BOTH;
+	}
+
+bool PktSrcComponent::DoesTrace() const
+	{
+	return type == TRACE || type == BOTH;
+	}
+
+PktSrcComponent::factory_callback PktSrcComponent::Factory() const
+	{
+	return factory;
+	}
+
+void PktSrcComponent::DoDescribe(ODesc* d) const
+	{
+	iosource::Component::DoDescribe(d);
+
+	string prefs;
+
+	for ( std::vector<std::string>::const_iterator i = prefixes.begin();
+	      i != prefixes.end(); i++ )
+		{
+		if ( prefs.size() )
+			prefs += ", ";
+
+		prefs += '"' + *i + '"';
+		}
+
+	d->Add("interface prefix");
+	if ( prefixes.size() > 1 )
+		d->Add("es");
+
+	d->Add(" ");
+	d->Add(prefs);
+	d->Add("; supports ");
+
+	switch ( type ) {
+	case LIVE:
+		d->Add("live input");
+		break;
+
+	case TRACE:
+		d->Add("trace input");
+		break;
+
+	case BOTH:
+		d->Add("live and trace input");
+		break;
+
+	default:
+		reporter->InternalError("unknown PkrSrc type");
+	}
+
+	}
+
+PktDumperComponent::PktDumperComponent(const std::string& name, const std::string& arg_prefix, factory_callback arg_factory)
+	: plugin::Component(plugin::component::PKTDUMPER, name)
+	{
+	tokenize_string(arg_prefix, ":", &prefixes);
+	factory = arg_factory;
+	}
+
+PktDumperComponent::~PktDumperComponent()
+	{
+	}
+
+PktDumperComponent::factory_callback PktDumperComponent::Factory() const
+	{
+	return factory;
+	}
+
+const std::vector<std::string>& PktDumperComponent::Prefixes() const
+	{
+	return prefixes;
+	}
+
+bool PktDumperComponent::HandlesPrefix(const string& prefix) const
+	{
+	for ( std::vector<std::string>::const_iterator i = prefixes.begin();
+	      i != prefixes.end(); i++ )
+		{
+		if ( *i == prefix )
+			return true;
+		}
+
+	return false;
+	}
+
+void PktDumperComponent::DoDescribe(ODesc* d) const
+	{
+	plugin::Component::DoDescribe(d);
+
+	string prefs;
+
+	for ( std::vector<std::string>::const_iterator i = prefixes.begin();
+	      i != prefixes.end(); i++ )
+		{
+		if ( prefs.size() )
+			prefs += ", ";
+
+		prefs += '"' + *i + '"';
+		}
+
+	d->Add("dumper prefix");
+
+	if ( prefixes.size() > 1 )
+		d->Add("es");
+
+	d->Add(": ");
+	d->Add(prefs);
+	}

src/iosource/Component.h

@@ -0,0 +1,177 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#ifndef IOSOURCE_PLUGIN_COMPONENT_H
+#define IOSOURCE_PLUGIN_COMPONENT_H
+
+#include <string>
+#include <vector>
+
+#include "plugin/Component.h"
+
+namespace iosource {
+
+class IOSource;
+class PktSrc;
+class PktDumper;
+
+/**
+ * Component description for plugins providing IOSources.
+ */
+class Component : public plugin::Component {
+public:
+	typedef IOSource* (*factory_callback)();
+
+	/**
+	 * Constructor.
+	 *
+	 * @param name A descriptive name for the component.  This name must
+	 * be unique across all components of this type.
+	 */
+	Component(const std::string& name);
+
+	/**
+	 * Copy constructor.
+	 */
+	Component(const Component& other);
+
+	/**
+	 * Destructor.
+	 */
+	~Component();
+
+protected:
+	/**
+	 * Constructor to use by derived classes.
+	 *
+	 * @param type The type of the componnent.
+	 *
+	 * @param name A descriptive name for the component.  This name must
+	 * be unique across all components of this type.
+	 */
+	Component(plugin::component::Type type, const std::string& name);
+};
+
+/**
+ * Component description for plugins providing a PktSrc for packet input.
+ */
+class PktSrcComponent : public iosource::Component {
+public:
+	/**
+	 * Type of input a packet source supports.
+	 */
+	enum InputType {
+		LIVE,	///< Live input.
+		TRACE,		///< Offline input from trace file.
+		BOTH	///< Live input as well as offline.
+	};
+
+	typedef PktSrc* (*factory_callback)(const std::string& path, bool is_live);
+
+	/**
+	 * Constructor.
+	 *
+	 * @param name A descriptive name for the component.  This name must
+	 * be unique across all components of this type.
+	 *
+	 * @param prefixes The list of interface/file prefixes associated
+	 * with this component.
+	 *
+	 * @param type Type of input the component supports.
+	 *
+	 * @param factor Factory function to instantiate component.
+	 */
+	PktSrcComponent(const std::string& name, const std::string& prefixes, InputType type, factory_callback factory);
+
+	/**
+	 * Destructor.
+	 */
+	virtual ~PktSrcComponent();
+
+	/**
+	 * Returns the prefix(es) passed to the constructor.
+	 */
+	const std::vector<std::string>& Prefixes() const;
+
+	/**
+	 * Returns true if the given prefix is among the one specified for the component.
+	 */
+	bool HandlesPrefix(const std::string& prefix) const;
+
+	/**
+	 * Returns true if packet source instantiated by the component handle
+	 * live traffic.
+	 */
+	bool DoesLive() const;
+
+	/**
+	 * Returns true if packet source instantiated by the component handle
+	 * offline traces.
+	 */
+	bool DoesTrace() const;
+
+	/**
+	 * Returns the source's factory function.
+	 */
+	factory_callback Factory() const;
+
+	/**
+	 * Generates a human-readable description of the component. This goes
+	 * into the output of \c "bro -NN".
+	 */
+	virtual void DoDescribe(ODesc* d) const;
+
+private:
+	std::vector<std::string> prefixes;
+	InputType type;
+	factory_callback factory;
+};
+
+/**
+ * Component description for plugins providing a PktDumper for packet output.
+ *
+ * PktDumpers aren't IOSurces but we locate them here to keep them along with
+ * the PktSrc.
+ */
+class PktDumperComponent : public plugin::Component  {
+public:
+	typedef PktDumper* (*factory_callback)(const std::string& path, bool append);
+
+	/**
+	 * XXX
+	 */
+	PktDumperComponent(const std::string& name, const std::string& prefixes, factory_callback factory);
+
+	/**
+	 * Destructor.
+	 */
+	~PktDumperComponent();
+
+	/**
+	 * Returns the prefix(es) passed to the constructor.
+	 */
+	const std::vector<std::string>& Prefixes() const;
+
+	/**
+	 * Returns true if the given prefix is among the one specified for the component.
+	 */
+	bool HandlesPrefix(const std::string& prefix) const;
+
+	/**
+	 * Returns the source's factory function.
+	 */
+	factory_callback Factory() const;
+
+	/**
+	 * Generates a human-readable description of the component. This goes
+	 * into the output of \c "bro -NN".
+	 */
+	virtual void DoDescribe(ODesc* d) const;
+
+private:
+	std::vector<std::string> prefixes;
+	factory_callback factory;
+};
+
+}
+
+#endif

src/iosource/IOSource.h

@@ -0,0 +1,140 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#ifndef IOSOURCE_IOSOURCE_H
+#define IOSOURCE_IOSOURCE_H
+
+extern "C" {
+#include <pcap.h>
+}
+
+#include <string>
+
+#include "Timer.h"
+
+namespace iosource {
+
+/**
+ * Interface class for components providing/consuming data inside Bro's main
+ * loop.
+ */
+class IOSource {
+public:
+	/**
+	 * Constructor.
+	 */
+	IOSource()	{ idle = false; closed = false; }
+
+	/**
+	 * Destructor.
+	 */
+	virtual ~IOSource()	{}
+
+	/**
+	 * Returns true if source has nothing ready to process.
+	 */
+	bool IsIdle() const	{ return idle; }
+
+	/**
+	 * Returns true if more data is to be expected in the future.
+	 * Otherwise, source may be removed.
+	 */
+	bool IsOpen() const	{ return ! closed; }
+
+	/**
+	 * Initializes the source. Can be overwritten by derived classes.
+	 */
+	virtual void Init()	{ }
+
+	/**
+	 * Finalizes the source when it's being closed. Can be overwritten by
+	 * derived classes.
+	 */
+	virtual void Done()	{ }
+
+	/**
+	 * Returns select'able file descriptors for this source. Leaves the
+	 * passed values untouched if not available.
+	 *
+	 * @param read Pointer to where to store a read descriptor.
+	 *
+	 * @param write Pointer to where to store a write descriptor.
+	 *
+	 * @param except Pointer to where to store a except descriptor.
+	 */
+	virtual void GetFds(int* read, int* write, int* except) = 0;
+
+	/**
+	 * Returns the timestamp (in \a global network time) associated with
+	 * next data item from this source.  If the source wants the data
+	 * item to be processed with a local network time, it sets the
+	 * argument accordingly.
+	 *
+	 * This method will be called only when either IsIdle() returns
+	 * false, or select() on one of the fds returned by GetFDs()
+	 * indicates that there's data to process.
+	 *
+	 * Must be overridden by derived classes.
+	 *
+	 * @param network_time A pointer to store the \a local network time
+	 * associated with the next item (as opposed to global network time).
+	 *
+	 * @return The global network time of the next entry, or a value
+	 * smaller than zero if none is available currently.
+	 */
+	virtual double NextTimestamp(double* network_time) = 0;
+
+	/**
+	 * Processes and consumes next data item.
+	 *
+	 * This method will be called only when either IsIdle() returns
+	 * false, or select() on one of the fds returned by GetFDs()
+	 * indicates that there's data to process.
+	 *
+	 * Must be overridden by derived classes.
+	 */
+	virtual void Process() = 0;
+
+	/**
+	 * Returns the tag of the timer manafger associated with the last
+	 * procesees data item.
+	 *
+	 * Can be overridden by derived classes.
+	 *
+	 * @return The tag, or null for the global timer manager.
+	 * 
+	 */
+	virtual TimerMgr::Tag* GetCurrentTag()	{ return 0; }
+
+	/**
+	 * Returns a descriptual tag representing the source for debugging.
+	 *
+	 * Can be overridden by derived classes.
+	 *
+	 * @return The debugging name.
+	 */
+	virtual const char* Tag() = 0;
+
+protected:
+	/*
+	 * Callback for derived classes to call when they have gone dry
+	 * temporarily.
+	 *
+	 * @param is_idle True if the source is idle currently.
+	 */
+	void SetIdle(bool is_idle)	{ idle = is_idle; }
+
+	/*
+	 * Callback for derived class to call when they have shutdown.
+	 *
+	 * @param is_closed True if the source is now closed.
+	 */
+	void SetClosed(bool is_closed)	{ closed = is_closed; }
+
+private:
+	bool idle;
+	bool closed;
+};
+
+}
+
+#endif

src/iosource/Manager.cc

@@ -1,3 +1,5 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
 #include <sys/types.h>
 #include <sys/time.h>
 #include <unistd.h>
@@ -5,26 +7,44 @@
 
 #include <algorithm>
 
-#include "util.h"
+#include "Manager.h"
 #include "IOSource.h"
+#include "PktSrc.h"
+#include "PktDumper.h"
+#include "plugin/Manager.h"
 
-IOSourceRegistry io_sources;
+#include "util.h"
 
-IOSourceRegistry::~IOSourceRegistry()
+#define DEFAULT_PREFIX "pcap"
+
+using namespace iosource;
+
+Manager::~Manager()
 	{
 	for ( SourceList::iterator i = sources.begin(); i != sources.end(); ++i )
+		{
+		(*i)->src->Done();
 		delete *i;
+		}
 
 	sources.clear();
+
+	for ( PktDumperList::iterator i = pkt_dumpers.begin(); i != pkt_dumpers.end(); ++i )
+		{
+		(*i)->Done();
+		delete *i;
+		}
+
+	pkt_dumpers.clear();
 	}
 
-void IOSourceRegistry::RemoveAll()
+void Manager::RemoveAll()
 	{
 	// We're cheating a bit here ...
 	dont_counts = sources.size();
 	}
 
-IOSource* IOSourceRegistry::FindSoonest(double* ts)
+IOSource* Manager::FindSoonest(double* ts)
 	{
 	// Remove sources which have gone dry. For simplicity, we only
 	// remove at most one each time.
@@ -32,6 +52,7 @@ IOSource* IOSourceRegistry::FindSoonest(double* ts)
 	      i != sources.end(); ++i )
 		if ( ! (*i)->src->IsOpen() )
 			{
+			(*i)->src->Done();
 			delete *i;
 			sources.erase(i);
 			break;
@@ -101,9 +122,9 @@ IOSource* IOSourceRegistry::FindSoonest(double* ts)
 		FD_SET(src->fd_write, &fd_write);
 		FD_SET(src->fd_except, &fd_except);
 
-		maxx = max(src->fd_read, maxx);
+		maxx = std::max(src->fd_read, maxx);
-		maxx = max(src->fd_write, maxx);
+		maxx = std::max(src->fd_write, maxx);
-		maxx = max(src->fd_except, maxx);
+		maxx = std::max(src->fd_except, maxx);
 		}
 
 	// We can't block indefinitely even when all sources are dry:
@@ -166,11 +187,126 @@ finished:
 	return soonest_src;
 	}
 
-void IOSourceRegistry::Register(IOSource* src, bool dont_count)
+void Manager::Register(IOSource* src, bool dont_count)
 	{
+	src->Init();
 	Source* s = new Source;
 	s->src = src;
 	if ( dont_count )
 		++dont_counts;
-	return sources.push_back(s);
+
+	sources.push_back(s);
+	}
+
+void Manager::Register(PktSrc* src)
+	{
+	pkt_srcs.push_back(src);
+	Register(src, false);
+	}
+
+static std::pair<std::string, std::string> split_prefix(std::string path)
+	{
+	// See if the path comes with a prefix telling us which type of
+	// PktSrc to use. If not, choose default.
+	std::string prefix;
+
+	std::string::size_type i = path.find(":");
+	if ( i != std::string::npos )
+		{
+		prefix = path.substr(0, i);
+		path = path.substr(++i, std::string::npos);
+		}
+
+	else
+		prefix= DEFAULT_PREFIX;
+
+	return std::make_pair(prefix, path);
+	}
+
+PktSrc* Manager::OpenPktSrc(const std::string& path, bool is_live)
+	{
+	std::pair<std::string, std::string> t = split_prefix(path);
+	std::string prefix = t.first;
+	std::string npath = t.second;
+
+	// Find the component providing packet sources of the requested prefix.
+
+	PktSrcComponent* component = 0;
+
+	std::list<PktSrcComponent*> all_components = plugin_mgr->Components<PktSrcComponent>();
+
+	for ( std::list<PktSrcComponent*>::const_iterator i = all_components.begin();
+	      i != all_components.end(); i++ )
+		{
+		PktSrcComponent* c = *i;
+
+		if ( c->HandlesPrefix(prefix) &&
+		     ((  is_live && c->DoesLive() ) ||
+		      (! is_live && c->DoesTrace())) )
+			{
+			component = c;
+			break;
+			}
+		}
+
+
+	if ( ! component )
+		reporter->FatalError("type of packet source '%s' not recognized, or mode not supported", prefix.c_str());
+
+	// Instantiate packet source.
+
+	PktSrc* ps = (*component->Factory())(npath, is_live);
+	assert(ps);
+
+	if ( ! ps->IsOpen() && ps->IsError() )
+		// Set an error message if it didn't open successfully.
+		ps->Error("could not open");
+
+	DBG_LOG(DBG_PKTIO, "Created packet source of type %s for %s", component->Name().c_str(), npath.c_str());
+
+	Register(ps);
+	return ps;
+	}
+
+
+PktDumper* Manager::OpenPktDumper(const string& path, bool append)
+	{
+	std::pair<std::string, std::string> t = split_prefix(path);
+	std::string prefix = t.first;
+	std::string npath = t.second;
+
+	// Find the component providing packet dumpers of the requested prefix.
+
+	PktDumperComponent* component = 0;
+
+	std::list<PktDumperComponent*> all_components = plugin_mgr->Components<PktDumperComponent>();
+
+	for ( std::list<PktDumperComponent*>::const_iterator i = all_components.begin();
+	      i != all_components.end(); i++ )
+		{
+		if ( (*i)->HandlesPrefix(prefix) )
+			{
+			component = (*i);
+			break;
+			}
+		}
+
+	if ( ! component )
+		reporter->FatalError("type of packet dumper '%s' not recognized", prefix.c_str());
+
+	// Instantiate packet dumper.
+
+	PktDumper* pd = (*component->Factory())(npath, append);
+	assert(pd);
+
+	if ( ! pd->IsOpen() && pd->IsError() )
+		// Set an error message if it didn't open successfully.
+		pd->Error("could not open");
+
+	DBG_LOG(DBG_PKTIO, "Created packer dumper of type %s for %s", component->Name().c_str(), npath.c_str());
+
+	pd->Init();
+	pkt_dumpers.push_back(pd);
+
+	return pd;
 	}

src/iosource/Manager.h

@@ -0,0 +1,135 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#ifndef IOSOURCE_MANAGER_H
+#define IOSOURCE_MANAGER_H
+
+#include <string>
+#include <list>
+
+namespace iosource {
+
+class IOSource;
+class PktSrc;
+class PktDumper;
+
+/**
+ * Singleton class managing all IOSources.
+ */
+class Manager {
+public:
+	/**
+	 * Constructor.
+	 */
+	Manager()	{ call_count = 0; dont_counts = 0; }
+
+	/**
+	 * Destructor.
+	 */
+	~Manager();
+
+	/**
+	 * Registers an IOSource with the manager.
+	 *
+	 * @param src The source. The manager takes ownership.
+	 *
+	 * @param dont_count If true, this source does not contribute to the
+	 * number of IOSources returned by Size().  The effect is that if all
+	 * sources except for the non-counting ones have gone dry, processing
+	 * will shut down.
+	 */
+	void Register(IOSource* src, bool dont_count = false);
+
+	/**
+	 * Returns the packet source with the soonest available input. This
+	 * may block for a little while if all are dry.
+	 *
+	 * @param ts A pointer where to store the timestamp of the input that
+	 * the soonest source has available next. 
+	 *
+	 * @return The source, or null if no source has input.
+	 */
+	IOSource* FindSoonest(double* ts);
+
+	/**
+	 * Returns the number of registered and still active sources,
+	 * excluding those that are registered as \a dont_cont.
+	 */
+	int Size() const	{ return sources.size() - dont_counts; }
+
+	typedef std::list<PktSrc *> PktSrcList;
+
+	/**
+	 * Returns a list of all registered PktSrc instances. This is a
+	 * subset of all registered IOSource instances.
+	 */
+	const PktSrcList& GetPktSrcs() const	{ return pkt_srcs; }
+
+	/**
+	 * Terminate all processing immediately by removing all sources (and
+	 * therefore now returning a Size() of zero).
+	 */
+	void Terminate()	{ RemoveAll(); }
+
+	/**
+	 * Opens a new packet source.
+	 *
+	 * @param path The interface or file name, as one would give to Bro \c -i.
+	 *
+	 * @param is_live True if \a path represents a live interface, false
+	 * for a file.
+	 *
+	 * @return The new packet source, or null if an error occured.
+	 */
+	PktSrc* OpenPktSrc(const std::string& path, bool is_live);
+
+	/**
+	 * Opens a new packet dumper.
+	 *
+	 * @param path The file name to dump into.
+	 *
+	 * @param append True to append if \a path already exists.
+ 	 *
+	 * @return The new packet dumper, or null if an error occured.
+	 */
+	PktDumper* OpenPktDumper(const std::string& path, bool append);
+
+private:
+	/**
+	 * When looking for a source with something to process, every
+	 * SELECT_FREQUENCY calls we will go ahead and block on a select().
+	 */
+	static const int SELECT_FREQUENCY = 25;
+
+	/**
+	 * Microseconds to wait in an empty select if no source is ready.
+	 */
+	static const int SELECT_TIMEOUT = 50;
+
+	void Register(PktSrc* src);
+	void RemoveAll();
+
+	unsigned int call_count;
+	int dont_counts;
+
+	struct Source {
+		IOSource* src;
+		int fd_read;
+		int fd_write;
+		int fd_except;
+	};
+
+	typedef std::list<Source*> SourceList;
+	SourceList sources;
+
+	typedef std::list<PktDumper *> PktDumperList;
+
+	PktSrcList pkt_srcs;
+	PktDumperList pkt_dumpers;
+};
+
+}
+
+extern iosource::Manager* iosource_mgr;
+
+#endif
+

src/iosource/PktDumper.cc

@@ -0,0 +1,84 @@
+
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#include <errno.h>
+#include <sys/stat.h>
+
+#include "config.h"
+
+#include "PktDumper.h"
+
+using namespace iosource;
+
+PktDumper::PktDumper()
+	{
+	is_open = false;
+	errmsg = "";
+	}
+
+PktDumper::~PktDumper()
+	{
+	}
+
+void PktDumper::Init()
+	{
+	Open();
+	}
+
+void PktDumper::Done()
+	{
+	Close();
+	}
+
+const std::string& PktDumper::Path() const
+	{
+	return props.path;
+	}
+
+bool PktDumper::IsOpen() const
+	{
+	return is_open;
+	}
+
+double PktDumper::OpenTime() const
+	{
+	return is_open ? props.open_time : 0;
+	}
+
+bool PktDumper::IsError() const
+	{
+	return errmsg.size();
+	}
+
+const char* PktDumper::ErrorMsg() const
+	{
+	return errmsg.size() ? errmsg.c_str() : 0;
+	}
+
+int PktDumper::HdrSize() const
+	{
+	return is_open ? props.hdr_size : -1;
+	}
+
+void PktDumper::Opened(const Properties& arg_props)
+	{
+	is_open = true;
+	props = arg_props;
+	DBG_LOG(DBG_PKTIO, "Opened dumper %s", props.path.c_str());
+	}
+
+void PktDumper::Closed()
+	{
+	is_open = false;
+	DBG_LOG(DBG_PKTIO, "Closed dumper %s", props.path.c_str());
+	props.path = "";
+	}
+
+void PktDumper::Error(const std::string& msg)
+	{
+	errmsg = msg;
+
+	DBG_LOG(DBG_PKTIO, "Error with dumper %s: %s",
+		IsOpen() ? props.path.c_str() : "<not open>",
+		msg.c_str());
+	}

src/iosource/PktDumper.h

@@ -0,0 +1,163 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#ifndef IOSOURCE_PKTSRC_PKTDUMPER_H
+#define IOSOURCE_PKTSRC_PKTDUMPER_H
+
+#include "IOSource.h"
+
+namespace iosource {
+
+/**
+ * Base class for packet dumpers.
+ */
+class PktDumper {
+public:
+	/**
+	 * Structure describing a packet.
+	 */
+	struct Packet {
+		/**
+		 * The pcap header associated with the packet.
+		 */
+		const struct pcap_pkthdr* hdr;
+
+		/**
+		 * The full content of the packet.
+		 */
+		const unsigned char* data;
+	};
+
+	/**
+	 * Constructor.
+	 */
+	PktDumper();
+
+	/**
+	 * Destructor.
+	 */
+	virtual ~PktDumper();
+
+	/**
+	 * Returns the path associated with the dumper.
+	 */
+	const std::string& Path() const;
+
+	/**
+	 * Returns true if the dumper is open for writing.
+	 */
+	bool IsOpen() const;
+
+	/**
+	 * Returns the time when the dumper was opened for writing.
+	 */
+	double OpenTime() const;
+
+	/**
+	 * Returns returns true if the dumper has encountered an error.
+	 */
+	bool IsError() const;
+
+	/**
+	 * Returns if the dumper has encountered an error, returns a
+	 * corresponding error message. Returns an emoty string otherwise.
+	 */
+	const char* ErrorMsg() const;
+
+	/**
+	 * Returns the size of the link-layer headers with this dumper.
+	 */
+	int HdrSize() const;
+
+	/**
+	 * Writes a packet to the dumper.
+	 *
+	 * @param pkt The packet to record.
+	 */
+	bool Record(const Packet* pkt);
+
+	// PktDumper interface for derived classes to implement.
+
+	/**
+	 * Called by the manager system to open the source.
+	 *
+	 * Derived classes must implement this method. If successful, the
+	 * implementation must call \a Opened(); if not, it must call Error()
+	 * with a corresponding message.
+	 */
+	virtual void Open() = 0;
+
+	/**
+	 * Called by the manager system to close the dumper.
+	 *
+	 * Derived classes must implement this method. If successful, the
+	 * implementation must call \a Closed(); if not, it must call Error()
+	 * with a corresponding message.
+	 */
+	virtual void Close() = 0;
+
+	/**
+	 * Called to write a packet to the dumper.
+	 *
+	 * Derived classes must implement this method.
+	 *
+	 * @param pkt The packet to record.
+	 *
+	 * @return True if succesful, false otherwise (in which case \a
+	 * Error() must have been called.)
+	 */
+	virtual bool Dump(const Packet* pkt) = 0;
+
+protected:
+	friend class Manager;
+
+	/**
+	 * Structure to pass back information about the packet dumper to the
+	 * base class. Derived class pass an instance of this to \a Opened().
+	 */
+	struct Properties {
+		std::string path;
+		int hdr_size;
+		double open_time;
+	};
+
+	/**
+	 * Called from the implementations of \a Open() to signal that the
+	 * source has been successully opened.
+	 *
+	 * @param props A properties instance describing the now open source.
+	 */
+	void Opened(const Properties& props);
+
+	/**
+	 * Called from the implementations of \a Close() to signal that the
+	 * source has been closed.
+	 */
+	void Closed();
+
+	/**
+	 * Called from derived classes to signal an error.
+	 *
+	 * @param msg A corresponding error message.
+	 */
+	void Error(const std::string& msg);
+
+	/**
+	 * Called by the manager to initialize the dumper.
+	 */
+	void Init();
+
+	/**
+	 * Called by the manager to shutdown the dumper.
+	 */
+	void Done();
+
+private:
+	bool is_open;
+	Properties props;
+
+	std::string errmsg;
+};
+
+}
+
+#endif

src/iosource/PktSrc.cc

@@ -0,0 +1,523 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#include <errno.h>
+#include <sys/stat.h>
+
+#include "config.h"
+
+#include "util.h"
+#include "PktSrc.h"
+#include "Hash.h"
+#include "Net.h"
+#include "Sessions.h"
+
+using namespace iosource;
+
+PktSrc::PktSrc()
+	{
+	have_packet = false;
+	errbuf = "";
+	SetClosed(true);
+
+	next_sync_point = 0;
+	first_timestamp = 0.0;
+	first_wallclock = current_wallclock = 0;
+	}
+
+PktSrc::~PktSrc()
+	{
+	BPF_Program* code;
+	IterCookie* cookie = filters.InitForIteration();
+	while ( (code = filters.NextEntry(cookie)) )
+		delete code;
+	}
+
+const std::string& PktSrc::Path() const
+	{
+	static std::string not_open("not open");
+	return IsOpen() ? props.path : not_open;
+	}
+
+const char* PktSrc::ErrorMsg() const
+	{
+	return errbuf.size() ? errbuf.c_str() : 0;
+	}
+
+int PktSrc::LinkType() const
+	{
+	return IsOpen() ? props.link_type : -1;
+	}
+
+uint32 PktSrc::Netmask() const
+	{
+	return IsOpen() ? props.netmask : PCAP_NETMASK_UNKNOWN;
+	}
+
+bool PktSrc::IsError() const
+	{
+	return ErrorMsg();
+	}
+
+int PktSrc::HdrSize() const
+	{
+	return IsOpen() ? props.hdr_size : -1;
+	}
+
+int PktSrc::SnapLen() const
+	{
+	return snaplen; // That's a global. Change?
+	}
+
+bool PktSrc::IsLive() const
+	{
+	return props.is_live;
+	}
+
+double PktSrc::CurrentPacketTimestamp()
+	{
+	return current_pseudo;
+	}
+
+double PktSrc::CurrentPacketWallClock()
+	{
+	// We stop time when we are suspended.
+	if ( net_is_processing_suspended() )
+		current_wallclock = current_time(true);
+
+	return current_wallclock;
+	}
+
+void PktSrc::Opened(const Properties& arg_props)
+	{
+	if ( arg_props.hdr_size < 0 )
+		{
+		char buf[512];
+		safe_snprintf(buf, sizeof(buf),
+			 "unknown data link type 0x%x", props.link_type);
+		Error(buf);
+		Close();
+		return;
+		}
+
+	props = arg_props;
+	SetClosed(false);
+
+	if ( ! PrecompileFilter(0, "") || ! SetFilter(0) )
+		{
+		Close();
+		return;
+		}
+
+	if ( props.is_live )
+		Info(fmt("listening on %s, capture length %d bytes\n", props.path.c_str(), SnapLen()));
+
+	DBG_LOG(DBG_PKTIO, "Opened source %s", props.path.c_str());
+	}
+
+void PktSrc::Closed()
+	{
+	SetClosed(true);
+
+	DBG_LOG(DBG_PKTIO, "Closed source %s", props.path.c_str());
+	}
+
+void PktSrc::Error(const std::string& msg)
+	{
+	// We don't report this immediately, Bro will ask us for the error
+	// once it notices we aren't open.
+	errbuf = msg;
+	DBG_LOG(DBG_PKTIO, "Error with source %s: %s",
+		IsOpen() ? props.path.c_str() : "<not open>",
+		msg.c_str());
+	}
+
+void PktSrc::Info(const std::string& msg)
+	{
+	reporter->Info("%s", msg.c_str());
+	}
+
+void PktSrc::Weird(const std::string& msg, const Packet* p)
+	{
+	sessions->Weird(msg.c_str(), p->hdr, p->data, 0);
+	}
+
+void PktSrc::InternalError(const std::string& msg)
+	{
+	reporter->InternalError("%s", msg.c_str());
+	}
+
+void PktSrc::ContinueAfterSuspend()
+	{
+	current_wallclock = current_time(true);
+	}
+
+int PktSrc::GetLinkHeaderSize(int link_type)
+	{
+	switch ( link_type ) {
+	case DLT_NULL:
+		return 4;
+
+	case DLT_EN10MB:
+		return 14;
+
+	case DLT_FDDI:
+		return 13 + 8;	// fddi_header + LLC
+
+#ifdef DLT_LINUX_SLL
+	case DLT_LINUX_SLL:
+		return 16;
+#endif
+
+	case DLT_PPP_SERIAL:	// PPP_SERIAL
+		return 4;
+
+	case DLT_RAW:
+		return 0;
+	}
+
+	return -1;
+	}
+
+double PktSrc::CheckPseudoTime()
+	{
+	if ( ! IsOpen() )
+		return 0;
+
+	if ( ! ExtractNextPacketInternal() )
+		return 0;
+
+	if ( remote_trace_sync_interval )
+		{
+		if ( next_sync_point == 0 || current_packet.ts >= next_sync_point )
+			{
+			int n = remote_serializer->SendSyncPoint();
+			next_sync_point = first_timestamp +
+						n * remote_trace_sync_interval;
+			remote_serializer->Log(RemoteSerializer::LogInfo,
+				fmt("stopping at packet %.6f, next sync-point at %.6f",
+					current_packet.ts, next_sync_point));
+
+			return 0;
+			}
+		}
+
+	double pseudo_time = current_packet.ts - first_timestamp;
+	double ct = (current_time(true) - first_wallclock) * pseudo_realtime;
+
+	return pseudo_time <= ct ? bro_start_time + pseudo_time : 0;
+	}
+
+void PktSrc::Init()
+	{
+	Open();
+	}
+
+void PktSrc::Done()
+	{
+	if ( IsOpen() )
+		Close();
+	}
+
+void PktSrc::GetFds(int* read, int* write, int* except)
+	{
+	if ( pseudo_realtime )
+		{
+		// Select would give erroneous results. But we simulate it
+		// by setting idle accordingly.
+		SetIdle(CheckPseudoTime() == 0);
+		return;
+		}
+
+	if ( IsOpen() && props.selectable_fd >= 0 )
+		*read = props.selectable_fd;
+	}
+
+double PktSrc::NextTimestamp(double* local_network_time)
+	{
+	if ( ! IsOpen() )
+		return -1.0;
+
+	if ( ! ExtractNextPacketInternal() )
+		return -1.0;
+
+	if ( pseudo_realtime )
+		{
+		// Delay packet if necessary.
+		double packet_time = CheckPseudoTime();
+		if ( packet_time )
+			return packet_time;
+
+		SetIdle(true);
+		return -1.0;
+		}
+
+	return current_packet.ts;
+	}
+
+void PktSrc::Process()
+	{
+	if ( ! IsOpen() )
+		return;
+
+	if ( ! ExtractNextPacketInternal() )
+		return;
+
+	int pkt_hdr_size = props.hdr_size;
+
+	// Unfortunately some packets on the link might have MPLS labels
+	// while others don't. That means we need to ask the link-layer if
+	// labels are in place.
+	bool have_mpls = false;
+
+	int protocol = 0;
+	const u_char* data = current_packet.data;
+
+	switch ( props.link_type ) {
+	case DLT_NULL:
+		{
+		protocol = (data[3] << 24) + (data[2] << 16) + (data[1] << 8) + data[0];
+
+		// From the Wireshark Wiki: "AF_INET6, unfortunately, has
+		// different values in {NetBSD,OpenBSD,BSD/OS},
+		// {FreeBSD,DragonFlyBSD}, and {Darwin/Mac OS X}, so an IPv6
+		// packet might have a link-layer header with 24, 28, or 30
+		// as the AF_ value." As we may be reading traces captured on
+		// platforms other than what we're running on, we accept them
+		// all here.
+		if ( protocol != AF_INET
+		     && protocol != AF_INET6
+		     && protocol != 24
+		     && protocol != 28
+		     && protocol != 30 )
+			{
+			Weird("non_ip_packet_in_null_transport", &current_packet);
+			goto done;
+			}
+
+		break;
+		}
+
+	case DLT_EN10MB:
+		{
+		// Get protocol being carried from the ethernet frame.
+		protocol = (data[12] << 8) + data[13];
+
+		switch ( protocol )
+			{
+			// MPLS carried over the ethernet frame.
+			case 0x8847:
+				// Remove the data link layer and denote a
+				// header size of zero before the IP header.
+				have_mpls = true;
+				data += GetLinkHeaderSize(props.link_type);
+				pkt_hdr_size = 0;
+				break;
+
+			// VLAN carried over the ethernet frame.
+			case 0x8100:
+				data += GetLinkHeaderSize(props.link_type);
+
+				// Check for MPLS in VLAN.
+				if ( ((data[2] << 8) + data[3]) == 0x8847 )
+					have_mpls = true;
+
+				data += 4; // Skip the vlan header
+				pkt_hdr_size = 0;
+
+				// Check for 802.1ah (Q-in-Q) containing IP.
+				// Only do a second layer of vlan tag
+				// stripping because there is no
+				// specification that allows for deeper
+				// nesting.
+				if ( ((data[2] << 8) + data[3]) == 0x0800 )
+					data += 4;
+
+				break;
+
+			// PPPoE carried over the ethernet frame.
+			case 0x8864:
+				data += GetLinkHeaderSize(props.link_type);
+				protocol = (data[6] << 8) + data[7];
+				data += 8; // Skip the PPPoE session and PPP header
+				pkt_hdr_size = 0;
+
+				if ( protocol != 0x0021 && protocol != 0x0057 )
+					{
+					// Neither IPv4 nor IPv6.
+					Weird("non_ip_packet_in_pppoe_encapsulation", &current_packet);
+					goto done;
+					}
+				break;
+			}
+
+		break;
+		}
+
+	case DLT_PPP_SERIAL:
+		{
+		// Get PPP protocol.
+		protocol = (data[2] << 8) + data[3];
+
+		if ( protocol == 0x0281 )
+			{
+			// MPLS Unicast. Remove the data link layer and
+			// denote a header size of zero before the IP header.
+			have_mpls = true;
+				data += GetLinkHeaderSize(props.link_type);
+			pkt_hdr_size = 0;
+			}
+
+		else if ( protocol != 0x0021 && protocol != 0x0057 )
+			{
+			// Neither IPv4 nor IPv6.
+			Weird("non_ip_packet_in_ppp_encapsulation", &current_packet);
+			goto done;
+			}
+		break;
+		}
+	}
+
+	if ( have_mpls )
+		{
+		// Skip the MPLS label stack.
+		bool end_of_stack = false;
+
+		while ( ! end_of_stack )
+			{
+			end_of_stack = *(data + 2) & 0x01;
+			data += 4;
+			}
+		}
+
+	if ( pseudo_realtime )
+		{
+		current_pseudo = CheckPseudoTime();
+		net_packet_dispatch(current_pseudo, current_packet.hdr, data, pkt_hdr_size, this);
+		if ( ! first_wallclock )
+			first_wallclock = current_time(true);
+		}
+
+	else
+		net_packet_dispatch(current_packet.ts, current_packet.hdr, data, pkt_hdr_size, this);
+
+done:
+	have_packet = 0;
+	DoneWithPacket();
+	}
+
+const char* PktSrc::Tag()
+	{
+	return "PktSrc";
+	}
+
+bool PktSrc::ExtractNextPacketInternal()
+	{
+	if ( have_packet )
+		return true;
+
+	have_packet = false;
+
+	// Don't return any packets if processing is suspended (except for the
+	// very first packet which we need to set up times).
+	if ( net_is_processing_suspended() && first_timestamp )
+		{
+		SetIdle(true);
+		return 0;
+		}
+
+	if ( pseudo_realtime )
+		current_wallclock = current_time(true);
+
+	if ( ExtractNextPacket(&current_packet) )
+		{
+		if ( ! first_timestamp )
+			first_timestamp = current_packet.ts;
+
+		have_packet = true;
+		return 1;
+		}
+
+	if ( pseudo_realtime && using_communication && ! IsOpen() )
+		{
+		// Source has gone dry, we're done.
+		if ( remote_trace_sync_interval )
+			remote_serializer->SendFinalSyncPoint();
+		else
+			remote_serializer->Terminate();
+		}
+
+	SetIdle(true);
+	return 0;
+	}
+
+bool PktSrc::PrecompileBPFFilter(int index, const std::string& filter)
+	{
+	if ( index < 0 )
+		return false;
+
+	char errbuf[PCAP_ERRBUF_SIZE];
+
+	// Compile filter.
+	BPF_Program* code = new BPF_Program();
+
+	if ( ! code->Compile(SnapLen(), LinkType(), filter.c_str(), Netmask(), errbuf, sizeof(errbuf)) )
+		{
+		string msg = fmt("cannot compile BPF filter \"%s\"", filter.c_str());
+
+		if ( *errbuf )
+			msg += ": " + string(errbuf);
+
+		Error(msg);
+
+		delete code;
+		return 0;
+		}
+
+	// Store it in hash.
+	HashKey* hash = new HashKey(HashKey(bro_int_t(index)));
+	BPF_Program* oldcode = filters.Lookup(hash);
+	if ( oldcode )
+		delete oldcode;
+
+	filters.Insert(hash, code);
+	delete hash;
+
+	return 1;
+	}
+
+BPF_Program* PktSrc::GetBPFFilter(int index)
+	{
+	if ( index < 0 )
+		return 0;
+
+	HashKey* hash = new HashKey(HashKey(bro_int_t(index)));
+	BPF_Program* code = filters.Lookup(hash);
+	delete hash;
+	return code;
+	}
+
+bool PktSrc::ApplyBPFFilter(int index, const struct pcap_pkthdr *hdr, const u_char *pkt)
+	{
+	BPF_Program* code = GetBPFFilter(index);
+
+	if ( ! code )
+		{
+		Error(fmt("BPF filter %d not compiled", index));
+		Close();
+		}
+
+	if ( code->MatchesAnything() )
+		return true;
+
+	return pcap_offline_filter(code->GetProgram(), hdr, pkt);
+	}
+
+bool PktSrc::GetCurrentPacket(const pcap_pkthdr** hdr, const u_char** pkt)
+	{
+	if ( ! have_packet )
+		return false;
+
+	*hdr = current_packet.hdr;
+	*pkt = current_packet.data;
+	return true;
+	}

src/iosource/PktSrc.h

@@ -0,0 +1,416 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#ifndef IOSOURCE_PKTSRC_PKTSRC_H
+#define IOSOURCE_PKTSRC_PKTSRC_H
+
+#include "IOSource.h"
+#include "BPF_Program.h"
+#include "Dict.h"
+
+declare(PDict,BPF_Program);
+
+namespace iosource {
+
+/**
+ * Base class for packet sources.
+ */
+class PktSrc : public IOSource {
+public:
+	/**
+	 * Struct for returning statistics on a packet source.
+	 */
+	struct Stats {
+		/**
+		 * Packets received by source after filtering (w/o drops).
+		 */
+		unsigned int received;
+
+		/**
+		 * Packets dropped by source.
+		 */
+		unsigned int dropped;	// pkts dropped
+
+		/**
+		 * Total number of packets on link before filtering.
+		 * Optional, can be left unset if not available.
+		 */
+		unsigned int link;
+
+		Stats()	{ received = dropped = link = 0; }
+	};
+
+	/**
+	 * Constructor.
+	 */
+	PktSrc();
+
+	/**
+	 * Destructor.
+	 */
+	virtual ~PktSrc();
+
+	/**
+	 * Returns the path associated with the source. This is the interface
+	 * name for live source, and a filename for offline sources.
+	 */
+	const std::string& Path() const;
+
+	/**
+	 * Returns true if this is a live source.
+	 */
+	bool IsLive() const;
+
+	/**
+	 * Returns the link type of the source.
+	 */
+	int LinkType() const;
+
+	/**
+	 * Returns the netmask associated with the source, or \c
+	 * PCAP_NETMASK_UNKNOWN if unknown.
+	 */
+	uint32 Netmask() const;
+
+	/**
+	 * Returns true if the source has flagged an error.
+	 */
+	bool IsError() const;
+
+	/**
+	 * If the source encountered an error, returns a corresponding error
+	 * message. Returns an empty string otherwise.
+	 */
+	const char* ErrorMsg() const;
+
+	/**
+	 * Returns the size of the link-layer header for this source.
+	 */
+	int HdrSize() const;
+
+	/**
+	 * Returns the snap length for this source.
+	 */
+	int SnapLen() const;
+
+	/**
+	 * In pseudo-realtime mode, returns the logical timestamp of the
+	 * current packet. Undefined if not running pseudo-realtime mode.
+	 */
+	double CurrentPacketTimestamp();
+
+	/**
+	 * In pseudo-realtime mode, returns the wall clock time associated
+	 * with current packet. Undefined if not running pseudo-realtime
+	 * mode.
+	 */
+	double CurrentPacketWallClock();
+
+	/**
+	 * Signals packet source that processing is going to be continued
+	 * after previous suspension.
+	 */
+	void ContinueAfterSuspend();
+
+	/**
+	 * Precompiles a BPF filter and associates the given index with it.
+	 * The compiled filter will be then available via \a GetBPFFilter().
+	 *
+	 * This is primarily a helper for packet source implementation that
+	 * want to apply BPF filtering to their packets.
+	 *
+	 * @param index The index to associate with the filter.
+	 *
+	 * @param BPF filter The filter string to precompile.
+	 *
+	 * @return True on success, false if a problem occurred.
+	 */
+	bool PrecompileBPFFilter(int index, const std::string& filter);
+
+	/**
+	 * Returns the precompiled BPF filter associated with a given index,
+	 * if any, as compiled by \a PrecompileBPFFilter().
+	 *
+	 * This is primarily a helper for packet source implementation that
+	 * want to apply BPF filtering to their packets.
+	 *
+	 * @return The BPF filter associated, or null if none has been
+	 * (successfully) compiled.
+	 */
+	BPF_Program* GetBPFFilter(int index);
+
+	/**
+	 * Applies a precompiled BPF filter to a packet. This will close the
+	 * source with an error message if no filter with that index has been
+	 * compiled.
+	 *
+	 * This is primarily a helper for packet source implementation that
+	 * want to apply BPF filtering to their packets.
+	 *
+	 * @param index The index of the filter to apply.
+	 *
+	 * @param hdr The header of the packet to filter.
+	 *
+	 * @param pkt The content of the packet to filter.
+	 *
+	 * @return True if it maches. 	 */
+	bool ApplyBPFFilter(int index, const struct pcap_pkthdr *hdr, const u_char *pkt);
+
+	/**
+	 * Returns the packet currently being processed, if available.
+	 *
+	 * @param hdr A pointer to pass the header of the current packet back.
+	 *
+	 * @param pkt A pointer to pass the content of the current packet
+	 * back.
+	 *
+	 * @return True if the current packet is available, or false if not.
+	 */
+	bool GetCurrentPacket(const pcap_pkthdr** hdr, const u_char** pkt);
+
+	// PacketSource interace for derived classes to override.
+
+	/**
+	 * Precompiles a filter and associates a given index with it. The
+	 * filter syntax is defined by the packet source's implenentation.
+	 *
+	 * Derived classes must implement this to implement their filtering.
+	 * If they want to use BPF but don't support it natively, they can
+	 * call the corresponding helper method provided by \a PktSrc.
+	 *
+	 * @param index The index to associate with the filter
+	 *
+	 * @param filter The filter string to precompile.
+	 *
+	 * @return True on success, false if a problem occurred or filtering
+	 * is not supported.
+	 */
+	virtual bool PrecompileFilter(int index, const std::string& filter) = 0;
+
+	/**
+	 * Activates a precompiled filter with the given index.
+	 *
+	 * Derived classes must implement this to implement their filtering.
+	 * If they want to use BPF but don't support it natively, they can
+	 * call the corresponding helper method provided by \a PktSrc.
+	 *
+	 * @param index The index of the filter to activate.
+	 *
+	 * @return True on success, false if a problem occurred or the
+	 * filtering is not supported.
+	 */
+	virtual bool SetFilter(int index) = 0;
+
+	/**
+	 * Returns current statistics about the source.
+	 *
+	 * Derived classes must implement this method.
+	 *
+	 * @param stats A statistics structure that the method fill out.
+	 */
+	virtual void Statistics(Stats* stats) = 0;
+
+	/**
+	 * Helper method to return the header size for a given link tyoe.
+	 *
+	 * @param link_type The link tyoe.
+	 *
+	 * @return The header size in bytes.
+	 */
+	static int GetLinkHeaderSize(int link_type);
+
+protected:
+	friend class Manager;
+
+	// Methods to use by derived classes.
+
+	/**
+	 * Structure to pass back information about the packet source to the
+	 * base class. Derived class pass an instance of this to \a Opened().
+	 */
+	struct Properties {
+		/**
+		 * The path associated with the source. This is the interface
+		 * name for live source, and a filename for offline sources.
+		 */
+		std::string path;
+
+		/**
+		 * A file descriptor suitable to use with \a select() for
+		 * determining if there's input available from this source.
+		 */
+		int selectable_fd;
+
+		/**
+		 * The link type for packets from this source.
+		 */
+		int link_type;
+
+		/**
+		 * The size of the link-layer header for packets from this
+		 * source. \a GetLinkHeaderSize() may be used to derive this
+		 * value.
+		 */
+		int hdr_size;
+
+		/**
+		 * The netmask associated with the source, or \c
+		 * PCAP_NETMASK_UNKNOWN if unknown.
+		 */
+		uint32 netmask;
+
+		/**
+		 * True if the source is reading live inout, false for
+		 * working offline.
+		 */
+		bool is_live;
+
+		Properties()
+			{
+			netmask = PCAP_NETMASK_UNKNOWN;
+			}
+	};
+
+	/**
+	 * Structure describing a packet.
+	 */
+	struct Packet {
+		/**
+		 *  Time associated with the packet.
+		 */
+		double ts;
+
+		/**
+		 *  The pcap header associated with the packet.
+		 */
+		const struct ::pcap_pkthdr* hdr;
+
+		/**
+		 * The full content of the packet.
+		 */
+		const u_char* data;
+	};
+
+	/**
+	 * Called from the implementations of \a Open() to signal that the
+	 * source has been successully opened.
+	 *
+	 * @param props A properties instance describing the now open source.
+	 */
+	void Opened(const Properties& props);
+
+	/**
+	 * Called from the implementations of \a Close() to signal that the
+	 * source has been closed.
+	 */
+	void Closed();
+
+	/**
+	 * Can be called from derived classes to send an informational
+	 * message to the user.
+	 *
+	 * @param msg The message to pass on.
+	 */
+	void Info(const std::string& msg);
+
+	/**
+	 * Can be called from derived classes to flag send an error.
+	 *
+	 * @param msg The message going with the error.
+	 */
+	void Error(const std::string& msg);
+
+	/**
+	 * Can be called from derived classes to flah a "weird" situation.
+	 *
+	 * @param msg The message to pass on.
+	 *
+	 * @param pkt The packet associated with the weird, or null if none.
+	 */
+	void Weird(const std::string& msg, const Packet* pkt);
+
+	/**
+	 * Can be called from derived classes to flag an internal error,
+	 * which will abort execution.
+	 *
+	 * @param msg The message to pass on.
+	 */
+	void InternalError(const std::string& msg);
+
+	// PktSrc interface for derived classes to implement.
+
+	/**
+	 * Called by the manager system to open the source.
+	 *
+	 * Derived classes must implement this method. If successful, the
+	 * implementation must call \a Opened(); if not, it must call Error()
+	 * with a corresponding message.
+	 */
+	virtual void Open() = 0;
+
+	/**
+	 * Called by the manager system to close the source.
+	 *
+	 * Derived classes must implement this method. If successful, the
+	 * implementation must call \a Closed(); if not, it must call Error()
+	 * with a corresponding message.
+	 */
+	virtual void Close() = 0;
+
+	/**
+	 * Provides the next packet from the source.
+	 *
+	 * @param pkt The packet structure to fill in with the packet's
+	 * information. The callee keep ownership of the data but must
+	 * guaranetee that it stays available at least until \a
+	 * DoneWithPacket() is called.  It is guaranteed that no two calls to
+	 * this method will hapen with \a DoneWithPacket() in between.
+	 *
+	 * @return True if a packet is available and *pkt* filled in. False
+	 * if not packet is available or an error occured (which must be
+	 * flageed via Error()).
+	 */
+	virtual bool ExtractNextPacket(Packet* pkt) = 0;
+
+	/**
+	 * Signals that the data of previously extracted packet will no
+	 * longer be needed.
+	 */
+	virtual void DoneWithPacket() = 0;
+
+private:
+	// Checks if the current packet has a pseudo-time <= current_time. If
+	// yes, returns pseudo-time, otherwise 0.
+	double CheckPseudoTime();
+
+	// Internal helper for ExtractNextPacket().
+	bool ExtractNextPacketInternal();
+
+	// IOSource interface implementation.
+	virtual void Init();
+	virtual void Done();
+	virtual void GetFds(int* read, int* write, int* except);
+	virtual double NextTimestamp(double* local_network_time);
+	virtual void Process();
+	virtual const char* Tag();
+
+	Properties props;
+
+	bool have_packet;
+	Packet current_packet;
+
+	// For BPF filtering support.
+	PDict(BPF_Program) filters;
+
+	// Only set in pseudo-realtime mode.
+	double first_timestamp;
+	double first_wallclock;
+	double current_wallclock;
+	double current_pseudo;
+	double next_sync_point; // For trace synchronziation in pseudo-realtime
+
+	std::string errbuf;
+};
+
+}
+
+#endif

src/iosource/pcap.bif

@@ -0,0 +1,104 @@
+
+## Precompiles a PCAP filter and binds it to a given identifier.
+##
+## id: The PCAP identifier to reference the filter *s* later on.
+##
+## s: The PCAP filter. See ``man tcpdump`` for valid expressions.
+##
+## Returns: True if *s* is valid and precompiles successfully.
+##
+## .. bro:see:: install_pcap_filter
+##          install_src_addr_filter
+##          install_src_net_filter
+##          uninstall_src_addr_filter
+##          uninstall_src_net_filter
+##          install_dst_addr_filter
+##          install_dst_net_filter
+##          uninstall_dst_addr_filter
+##          uninstall_dst_net_filter
+##          pcap_error
+function precompile_pcap_filter%(id: PcapFilterID, s: string%): bool
+	%{
+	bool success = true;
+
+	const iosource::Manager::PktSrcList& pkt_srcs(iosource_mgr->GetPktSrcs());
+
+	for ( iosource::Manager::PktSrcList::const_iterator i = pkt_srcs.begin();
+	      i != pkt_srcs.end(); i++ )
+		{
+		iosource::PktSrc* ps = *i;
+
+		if ( ! ps->PrecompileFilter(id->ForceAsInt(),
+							s->CheckString()) )
+			success = false;
+		}
+
+	return new Val(success, TYPE_BOOL);
+	%}
+
+## Installs a PCAP filter that has been precompiled with
+## :bro:id:`precompile_pcap_filter`.
+##
+## id: The PCAP filter id of a precompiled filter.
+##
+## Returns: True if the filter associated with *id* has been installed
+##          successfully.
+##
+## .. bro:see:: precompile_pcap_filter
+##              install_src_addr_filter
+##              install_src_net_filter
+##              uninstall_src_addr_filter
+##              uninstall_src_net_filter
+##              install_dst_addr_filter
+##              install_dst_net_filter
+##              uninstall_dst_addr_filter
+##              uninstall_dst_net_filter
+##              pcap_error
+function install_pcap_filter%(id: PcapFilterID%): bool
+	%{
+	bool success = true;
+
+	const iosource::Manager::PktSrcList& pkt_srcs(iosource_mgr->GetPktSrcs());
+
+	for ( iosource::Manager::PktSrcList::const_iterator i = pkt_srcs.begin();
+	      i != pkt_srcs.end(); i++ )
+		{
+		iosource::PktSrc* ps = *i;
+
+		if ( ! ps->SetFilter(id->ForceAsInt()) )
+			success = false;
+		}
+
+	return new Val(success, TYPE_BOOL);
+	%}
+
+## Returns a string representation of the last PCAP error.
+##
+## Returns: A descriptive error message of the PCAP function that failed.
+##
+## .. bro:see:: precompile_pcap_filter
+##              install_pcap_filter
+##              install_src_addr_filter
+##              install_src_net_filter
+##              uninstall_src_addr_filter
+##              uninstall_src_net_filter
+##              install_dst_addr_filter
+##              install_dst_net_filter
+##              uninstall_dst_addr_filter
+##              uninstall_dst_net_filter
+function pcap_error%(%): string
+	%{
+	const iosource::Manager::PktSrcList& pkt_srcs(iosource_mgr->GetPktSrcs());
+
+	for ( iosource::Manager::PktSrcList::const_iterator i = pkt_srcs.begin();
+	      i != pkt_srcs.end(); i++ )
+		{
+		iosource::PktSrc* ps = *i;
+
+		const char* err = ps->ErrorMsg();
+		if ( *err )
+			return new StringVal(err);
+		}
+
+	return new StringVal("no error");
+	%}

src/iosource/pcap/CMakeLists.txt

@@ -0,0 +1,8 @@
+
+include(BroPlugin)
+
+include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR})
+
+bro_plugin_begin(Bro Pcap)
+bro_plugin_cc(Source.cc Dumper.cc Plugin.cc)
+bro_plugin_end()

src/iosource/pcap/Dumper.cc

@@ -0,0 +1,112 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#include <sys/stat.h>
+#include <errno.h>
+
+#include "Dumper.h"
+#include "../PktSrc.h"
+#include "../../Net.h"
+
+using namespace iosource::pcap;
+
+PcapDumper::PcapDumper(const std::string& path, bool arg_append)
+	{
+	append = arg_append;
+	props.path = path;
+	dumper = 0;
+	pd = 0;
+	}
+
+PcapDumper::~PcapDumper()
+	{
+	}
+
+void PcapDumper::Open()
+	{
+	int linktype = -1;
+
+	pd = pcap_open_dead(DLT_EN10MB, snaplen);
+	if ( ! pd )
+		{
+		Error("error for pcap_open_dead");
+		return;
+		}
+
+	if ( props.path.empty() )
+		{
+		Error("no filename given");
+		return;
+		}
+
+	struct stat s;
+	int exists = 0;
+
+	if ( append )
+		{
+		// See if output file already exists (and is non-empty).
+		exists = stat(props.path.c_str(), &s); ;
+
+		if ( exists < 0 && errno != ENOENT )
+			{
+			Error(fmt("can't stat file %s: %s", props.path.c_str(), strerror(errno)));
+			return;
+			}
+		}
+
+	if ( ! append || exists < 0 || s.st_size == 0 )
+		{
+		// Open new file.
+		dumper = pcap_dump_open(pd, props.path.c_str());
+		if ( ! dumper )
+			{
+			Error(pcap_geterr(pd));
+			return;
+			}
+		}
+
+	else
+		{
+		// Old file and we need to append, which, unfortunately,
+		// is not supported by libpcap. So, we have to hack a
+		// little bit, knowing that pcap_dumpter_t is, in fact,
+		// a FILE ... :-(
+		dumper = (pcap_dumper_t*) fopen(props.path.c_str(), "a");
+		if ( ! dumper )
+			{
+			Error(fmt("can't open dump %s: %s", props.path.c_str(), strerror(errno)));
+			return;
+			}
+		}
+
+	props.open_time = network_time;
+	props.hdr_size = PktSrc::GetLinkHeaderSize(pcap_datalink(pd));
+	Opened(props);
+	}
+
+void PcapDumper::Close()
+	{
+	if ( ! dumper )
+		return;
+
+	pcap_dump_close(dumper);
+	pcap_close(pd);
+	dumper = 0;
+	pd = 0;
+
+	Closed();
+	}
+
+bool PcapDumper::Dump(const Packet* pkt)
+	{
+	if ( ! dumper )
+		return false;
+
+	pcap_dump((u_char*) dumper, pkt->hdr, pkt->data);
+
+	return true;
+	}
+
+iosource::PktDumper* PcapDumper::Instantiate(const std::string& path, bool append)
+	{
+	return new PcapDumper(path, append);
+	}

src/iosource/pcap/Dumper.h

@@ -0,0 +1,41 @@
+// See the file  in the main distribution directory for copyright.
+
+#ifndef IOSOURCE_PKTSRC_PCAP_DUMPER_H
+#define IOSOURCE_PKTSRC_PCAP_DUMPER_H
+
+extern "C" {
+#include <pcap.h>
+}
+
+#include "../PktDumper.h"
+
+namespace iosource {
+namespace pcap {
+
+class PcapDumper : public PktDumper {
+public:
+	PcapDumper(const std::string& path, bool append);
+	virtual ~PcapDumper();
+
+	static PktDumper* Instantiate(const std::string& path, bool appen);
+
+protected:
+	// PktDumper interface.
+	virtual void Open();
+	virtual void Close();
+	virtual bool Dump(const Packet* pkt);
+
+private:
+	Properties props;
+
+	bool append;
+	pcap_dumper_t* dumper;
+	pcap_t* pd;
+};
+
+}
+}
+
+#endif
+
+

src/iosource/pcap/Plugin.cc

@@ -0,0 +1,27 @@
+// See the file  in the main distribution directory for copyright.
+
+#include "plugin/Plugin.h"
+
+#include "Source.h"
+#include "Dumper.h"
+
+namespace plugin {
+namespace Bro_Pcap {
+
+class Plugin : public plugin::Plugin {
+public:
+	plugin::Configuration Configure()
+		{
+		AddComponent(new ::iosource::PktSrcComponent("PcapReader", "pcap", ::iosource::PktSrcComponent::BOTH, ::iosource::pcap::PcapSource::Instantiate));
+		AddComponent(new ::iosource::PktDumperComponent("PcapWriter", "pcap", ::iosource::pcap::PcapDumper::Instantiate));
+
+		plugin::Configuration config;
+		config.name = "Bro::Pcap";
+		config.description = "Packet aquisition via libpcap";
+		return config;
+		}
+} plugin;
+
+}
+}
+

src/iosource/pcap/Source.cc

@@ -0,0 +1,272 @@
+// See the file  in the main distribution directory for copyright.
+
+#include <assert.h>
+
+#include "config.h"
+
+#include "Source.h"
+
+#ifdef HAVE_PCAP_INT_H
+#include <pcap-int.h>
+#endif
+
+using namespace iosource::pcap;
+
+PcapSource::~PcapSource()
+	{
+	Close();
+	}
+
+PcapSource::PcapSource(const std::string& path, bool is_live)
+	{
+	props.path = path;
+	props.is_live = is_live;
+	last_data = 0;
+	}
+
+void PcapSource::Open()
+	{
+	if ( props.is_live )
+		OpenLive();
+	else
+		OpenOffline();
+	}
+
+void PcapSource::Close()
+	{
+	if ( ! pd )
+		return;
+
+	pcap_close(pd);
+	pd = 0;
+	last_data = 0;
+
+	Closed();
+	}
+
+void PcapSource::OpenLive()
+	{
+	char errbuf[PCAP_ERRBUF_SIZE];
+	char tmp_errbuf[PCAP_ERRBUF_SIZE];
+
+	// Determine interface if not specified.
+	if ( props.path.empty() )
+		props.path = pcap_lookupdev(tmp_errbuf);
+
+	if ( props.path.empty() )
+		{
+		safe_snprintf(errbuf, sizeof(errbuf),
+			 "pcap_lookupdev: %s", tmp_errbuf);
+		Error(errbuf);
+		return;
+		}
+
+	// Determine network and netmask.
+	uint32 net;
+	if ( pcap_lookupnet(props.path.c_str(), &net, &props.netmask, tmp_errbuf) < 0 )
+		{
+		// ### The lookup can fail if no address is assigned to
+		// the interface; and libpcap doesn't have any useful notion
+		// of error codes, just error std::strings - how bogus - so we
+		// just kludge around the error :-(.
+		// sprintf(errbuf, "pcap_lookupnet %s", tmp_errbuf);
+		// return;
+		props.netmask = 0xffffff00;
+		}
+
+	// We use the smallest time-out possible to return almost immediately if
+	// no packets are available. (We can't use set_nonblocking() as it's
+	// broken on FreeBSD: even when select() indicates that we can read
+	// something, we may get nothing if the store buffer hasn't filled up
+	// yet.)
+	pd = pcap_open_live(props.path.c_str(), SnapLen(), 1, 1, tmp_errbuf);
+
+	if ( ! pd )
+		{
+		Error(tmp_errbuf);
+		return;
+		}
+
+	// ### This needs autoconf'ing.
+#ifdef HAVE_PCAP_INT_H
+	Info("pcap bufsize = %d\n", ((struct pcap *) pd)->bufsize);
+#endif
+
+#ifdef HAVE_LINUX
+	if ( pcap_setnonblock(pd, 1, tmp_errbuf) < 0 )
+		{
+		PcapError();
+		return;
+		}
+#endif
+
+	props.selectable_fd = pcap_fileno(pd);
+
+	SetHdrSize();
+
+	if ( ! pd )
+		// Was closed, couldn't get header size.
+		return;
+
+	props.is_live = true;
+
+	Opened(props);
+	}
+
+void PcapSource::OpenOffline()
+	{
+	char errbuf[PCAP_ERRBUF_SIZE];
+
+	pd = pcap_open_offline(props.path.c_str(), errbuf);
+
+	if ( ! pd )
+		{
+		Error(errbuf);
+		return;
+		}
+
+	SetHdrSize();
+
+	if ( ! pd )
+		// Was closed, unknown link layer type.
+		return;
+
+	props.selectable_fd = fileno(pcap_file(pd));
+
+	if ( props.selectable_fd < 0 )
+		InternalError("OS does not support selectable pcap fd");
+
+	props.is_live = false;
+	Opened(props);
+	}
+
+bool PcapSource::ExtractNextPacket(Packet* pkt)
+	{
+	if ( ! pd )
+		return false;
+
+	const u_char* data = pcap_next(pd, &current_hdr);
+
+	if ( ! data )
+		{
+		// Source has gone dry.  If it's a network interface, this just means
+		// it's timed out. If it's a file, though, then the file has been
+		// exhausted.
+		if ( ! props.is_live )
+			Close();
+
+		return false;
+		}
+
+	pkt->ts = current_hdr.ts.tv_sec + double(current_hdr.ts.tv_usec) / 1e6;
+	pkt->hdr = &current_hdr;
+	pkt->data = last_data = data;
+
+	if ( current_hdr.len == 0 || current_hdr.caplen == 0 )
+		{
+		Weird("empty_pcap_header", pkt);
+		return false;
+		}
+
+	last_hdr = current_hdr;
+	last_data = data;
+	++stats.received;
+	return true;
+	}
+
+void PcapSource::DoneWithPacket()
+	{
+	// Nothing to do.
+	}
+
+bool PcapSource::PrecompileFilter(int index, const std::string& filter)
+	{
+	return PktSrc::PrecompileBPFFilter(index, filter);
+	}
+
+bool PcapSource::SetFilter(int index)
+	{
+	if ( ! pd )
+		return true; // Prevent error message
+
+	char errbuf[PCAP_ERRBUF_SIZE];
+
+	BPF_Program* code = GetBPFFilter(index);
+
+	if ( ! code )
+		{
+		safe_snprintf(errbuf, sizeof(errbuf),
+			      "No precompiled pcap filter for index %d",
+			      index);
+		Error(errbuf);
+		return false;
+		}
+
+	if ( pcap_setfilter(pd, code->GetProgram()) < 0 )
+		{
+		PcapError();
+		return false;
+		}
+
+#ifndef HAVE_LINUX
+	// Linux doesn't clear counters when resetting filter.
+	stats.received = stats.dropped = stats.link = 0;
+#endif
+
+	return true;
+	}
+
+void PcapSource::Statistics(Stats* s)
+	{
+	char errbuf[PCAP_ERRBUF_SIZE];
+
+	if ( ! (props.is_live && pd) )
+		s->received = s->dropped = s->link = 0;
+
+	else
+		{
+		struct pcap_stat pstat;
+		if ( pcap_stats(pd, &pstat) < 0 )
+			{
+			PcapError();
+			s->received = s->dropped = s->link = 0;
+			}
+
+		else
+			{
+			s->dropped = pstat.ps_drop;
+			s->link = pstat.ps_recv;
+			}
+		}
+
+	s->received = stats.received;
+
+	if ( ! props.is_live )
+		s->dropped = 0;
+	}
+
+void PcapSource::PcapError()
+	{
+	if ( pd )
+		Error(fmt("pcap_error: %s", pcap_geterr(pd)));
+	else
+		Error("pcap_error: not open");
+
+	Close();
+	}
+
+void PcapSource::SetHdrSize()
+	{
+	if ( ! pd )
+		return;
+
+	char errbuf[PCAP_ERRBUF_SIZE];
+
+	props.link_type = pcap_datalink(pd);
+	props.hdr_size = GetLinkHeaderSize(props.link_type);
+	}
+
+iosource::PktSrc* PcapSource::Instantiate(const std::string& path, bool is_live)
+	{
+	return new PcapSource(path, is_live);
+	}

src/iosource/pcap/Source.h

@@ -0,0 +1,47 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#ifndef IOSOURCE_PKTSRC_PCAP_SOURCE_H
+#define IOSOURCE_PKTSRC_PCAP_SOURCE_H
+
+#include "../PktSrc.h"
+
+namespace iosource {
+namespace pcap {
+
+class PcapSource : public iosource::PktSrc {
+public:
+	PcapSource(const std::string& path, bool is_live);
+	virtual ~PcapSource();
+
+	static PktSrc* Instantiate(const std::string& path, bool is_live);
+
+protected:
+	// PktSrc interface.
+	virtual void Open();
+	virtual void Close();
+	virtual bool ExtractNextPacket(Packet* pkt);
+	virtual void DoneWithPacket();
+	virtual bool PrecompileFilter(int index, const std::string& filter);
+	virtual bool SetFilter(int index);
+	virtual void Statistics(Stats* stats);
+
+private:
+	void OpenLive();
+	void OpenOffline();
+	void PcapError();
+	void SetHdrSize();
+
+	Properties props;
+	Stats stats;
+
+	pcap_t *pd;
+
+	struct pcap_pkthdr current_hdr;
+	struct pcap_pkthdr last_hdr;
+	const u_char* last_data;
+};
+
+}
+}
+
+#endif

src/main.cc

@@ -57,6 +57,7 @@ extern "C" void OPENSSL_add_all_algorithms_conf(void);
 #include "plugin/Manager.h"
 #include "file_analysis/Manager.h"
 #include "broxygen/Manager.h"
+#include "iosource/Manager.h"
 
 #include "binpac_bro.h"
 
@@ -92,6 +93,7 @@ plugin::Manager* plugin_mgr = 0;
 analyzer::Manager* analyzer_mgr = 0;
 file_analysis::Manager* file_mgr = 0;
 broxygen::Manager* broxygen_mgr = 0;
+iosource::Manager* iosource_mgr = 0;
 Stmt* stmts;
 EventHandlerPtr net_done = 0;
 RuleMatcher* rule_matcher = 0;
@@ -108,7 +110,6 @@ int signal_val = 0;
 int optimize = 0;
 int do_notice_analysis = 0;
 int rule_bench = 0;
-SecondaryPath* secondary_path = 0;
 extern char version[];
 char* command_line_policy = 0;
 vector<string> params;
@@ -375,20 +376,16 @@ void terminate_bro()
 
 	delete broxygen_mgr;
 	delete timer_mgr;
-	delete dns_mgr;
 	delete persistence_serializer;
-	delete event_player;
 	delete event_serializer;
 	delete state_serializer;
 	delete event_registry;
-	delete secondary_path;
-	delete remote_serializer;
 	delete analyzer_mgr;
 	delete file_mgr;
 	delete log_mgr;
 	delete plugin_mgr;
-	delete thread_mgr;
 	delete reporter;
+	delete iosource_mgr;
 
 	reporter = 0;
 	}
@@ -448,8 +445,6 @@ int main(int argc, char** argv)
 
 	name_list interfaces;
 	name_list read_files;
-	name_list netflows;
-	name_list flow_files;
 	name_list rule_files;
 	char* bst_file = 0;
 	char* id_name = 0;
@@ -551,7 +546,7 @@ int main(int argc, char** argv)
 	opterr = 0;
 
 	char opts[256];
-	safe_strncpy(opts, "B:D:e:f:I:i:K:l:n:p:R:r:s:T:t:U:w:x:X:y:Y:z:CFGLNOPSWabdghvZQ",
+	safe_strncpy(opts, "B:D:e:f:I:i:K:l:n:p:R:r:s:T:t:U:w:x:X:z:CFGLNOPSWabdghvZQ",
 		     sizeof(opts));
 
 #ifdef USE_PERFTOOLS_DEBUG
@@ -611,10 +606,6 @@ int main(int argc, char** argv)
 			writefile = optarg;
 			break;
 
-		case 'y':
-			flow_files.append(optarg);
-			break;
-
 		case 'z':
 			if ( streq(optarg, "notice") )
 				do_notice_analysis = 1;
@@ -708,10 +699,6 @@ int main(int argc, char** argv)
 			do_watchdog = 1;
 			break;
 
-		case 'Y':
-			netflows.append(optarg);
-			break;
-
 		case 'h':
 			usage();
 			break;
@@ -799,8 +786,7 @@ int main(int argc, char** argv)
 	// seed the PRNG. We should do this here (but at least Linux, FreeBSD
 	// and Solaris provide /dev/urandom).
 
-	if ( (interfaces.length() > 0 || netflows.length() > 0) &&
+	if ( interfaces.length() > 0 && read_files.length() > 0 )
-	     (read_files.length() > 0 || flow_files.length() > 0 ))
 		usage();
 
 #ifdef USE_IDMEF
@@ -823,7 +809,7 @@ int main(int argc, char** argv)
 	plugin_mgr->SearchDynamicPlugins(bro_plugin_path());
 
 	if ( optind == argc &&
-	     read_files.length() == 0 && flow_files.length() == 0 &&
+	     read_files.length() == 0 &&
 	     interfaces.length() == 0 &&
 	     ! (id_name || bst_file) && ! command_line_policy && ! print_plugins )
 		add_input_file("-");
@@ -850,6 +836,7 @@ int main(int argc, char** argv)
 	// policy, but we can't parse policy without DNS resolution.
 	dns_mgr->SetDir(".state");
 
+	iosource_mgr = new iosource::Manager();
 	persistence_serializer = new PersistenceSerializer();
 	remote_serializer = new RemoteSerializer();
 	event_registry = new EventRegistry();
@@ -916,6 +903,7 @@ int main(int argc, char** argv)
 
 	analyzer_mgr->InitPostScript();
 	file_mgr->InitPostScript();
+	dns_mgr->InitPostScript();
 
 	if ( parse_only )
 		{
@@ -981,8 +969,7 @@ int main(int argc, char** argv)
 		// ### Add support for debug command file.
 		dbg_init_debugger(0);
 
-	if ( (flow_files.length() == 0 || read_files.length() == 0) &&
+	if ( read_files.length() == 0 && interfaces.length() == 0 )
-	     (netflows.length() == 0 || interfaces.length() == 0) )
 		{
 		Val* interfaces_val = internal_val("interfaces");
 		if ( interfaces_val )
@@ -999,13 +986,8 @@ int main(int argc, char** argv)
 
 	snaplen = internal_val("snaplen")->AsCount();
 
-	// Initialize the secondary path, if it's needed.
-	secondary_path = new SecondaryPath();
-
 	if ( dns_type != DNS_PRIME )
-		net_init(interfaces, read_files, netflows, flow_files,
+		net_init(interfaces, read_files, writefile, do_watchdog);
-			writefile, "",
-			secondary_path->Filter(), do_watchdog);
 
 	BroFile::SetDefaultRotation(log_rotate_interval, log_max_size);
 
@@ -1164,9 +1146,9 @@ int main(int argc, char** argv)
 
 	have_pending_timers = ! reading_traces && timer_mgr->Size() > 0;
 
-	io_sources.Register(thread_mgr, true);
+	iosource_mgr->Register(thread_mgr, true);
 
-	if ( io_sources.Size() > 0 ||
+	if ( iosource_mgr->Size() > 0 ||
 	     have_pending_timers ||
 	     BifConst::exit_only_after_terminate )
 		{

src/plugin/Component.cc

@@ -50,6 +50,18 @@ void Component::Describe(ODesc* d) const
 		d->Add("File Analyzer");
 		break;
 
+	case component::IOSOURCE:
+		d->Add("I/O Source");
+		break;
+
+	case component::PKTSRC:
+		d->Add("Packet Source");
+		break;
+
+	case component::PKTDUMPER:
+		d->Add("Packet Dumper");
+		break;
+
 	default:
 		reporter->InternalWarning("unknown component type in plugin::Component::Describe");
 		d->Add("<unknown component type>");

src/plugin/Component.h

@@ -18,7 +18,10 @@ enum Type {
 	READER,	/// An input reader (not currently used).
 	WRITER,	/// A logging writer (not currenly used).
 	ANALYZER,	/// A protocol analyzer.
-	FILE_ANALYZER	/// A file analyzer.
+	FILE_ANALYZER,	/// A file analyzer.
+	IOSOURCE,	/// An I/O source, excluding packet sources.
+	PKTSRC,	/// A packet source.
+	PKTDUMPER	/// A packet dumper.
 	};
 }
 

src/plugin/Plugin.h

@@ -9,6 +9,7 @@
 #include "config.h"
 #include "analyzer/Component.h"
 #include "file_analysis/Component.h"
+#include "iosource/Component.h"
 
 // We allow to override this externally for testing purposes.
 #ifndef BRO_PLUGIN_API_VERSION

src/threading/Manager.cc

@@ -11,7 +11,7 @@ Manager::Manager()
 	did_process = true;
 	next_beat = 0;
 	terminating = false;
-	idle = true;
+	SetIdle(true);
 	}
 
 Manager::~Manager()
@@ -47,8 +47,8 @@ void Manager::Terminate()
 	all_threads.clear();
 	msg_threads.clear();
 
-	idle = true;
+	SetIdle(true);
-	closed = true;
+	SetClosed(true);
 	terminating = false;
 	}
 
@@ -56,7 +56,7 @@ void Manager::AddThread(BasicThread* thread)
 	{
 	DBG_LOG(DBG_THREADING, "Adding thread %s ...", thread->Name());
 	all_threads.push_back(thread);
-	idle = false;
+	SetIdle(false);
 	}
 
 void Manager::AddMsgThread(MsgThread* thread)

src/threading/Manager.h

@@ -4,7 +4,7 @@
 
 #include <list>
 
-#include "IOSource.h"
+#include "iosource/IOSource.h"
 
 #include "BasicThread.h"
 #include "MsgThread.h"
@@ -21,7 +21,7 @@ namespace threading {
  * their outgoing message queue on a regular basis and feeds data sent into
  * the rest of Bro. It also triggers the regular heartbeats.
  */
-class Manager : public IOSource
+class Manager : public iosource::IOSource
 {
 public:
 	/**

src/util.cc

@@ -43,6 +43,7 @@
 #include "NetVar.h"
 #include "Net.h"
 #include "Reporter.h"
+#include "iosource/Manager.h"
 
 /**
  * Return IP address without enclosing brackets and any leading 0x.
@@ -1424,11 +1425,13 @@ double current_time(bool real)
 
 	double t = double(tv.tv_sec) + double(tv.tv_usec) / 1e6;
 
-	if ( ! pseudo_realtime || real || pkt_srcs.length() == 0 )
+	const iosource::Manager::PktSrcList& pkt_srcs(iosource_mgr->GetPktSrcs());
+
+	if ( ! pseudo_realtime || real || pkt_srcs.empty() )
 		return t;
 
 	// This obviously only works for a single source ...
-	PktSrc* src = pkt_srcs[0];
+	iosource::PktSrc* src = pkt_srcs.front();
 
 	if ( net_is_processing_suspended() )
 		return src->CurrentPacketTimestamp();

testing/btest/Baseline/core.pcap.dumper/output

@@ -0,0 +1 @@
+00000010  ff ff 00 00 01 00 00 00  1d a2 b2 4e 73 00 07 00  | |	00000010  00 20 00 00 01 00 00 00  1d a2 b2 4e 73 00 07 00  |

testing/btest/Baseline/core.pcap.dynamic-filter/conn.log

@@ -0,0 +1,25 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	conn
+#open	2014-08-24-15-51-55
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	service	duration	orig_bytes	resp_bytes	conn_state	local_orig	missed_bytes	history	orig_pkts	orig_ip_bytes	resp_pkts	resp_ip_bytes	tunnel_parents
+#types	time	string	addr	port	addr	port	enum	string	interval	count	count	string	bool	count	string	count	count	count	count	set[string]
+1300475167.096535	CXWv6p3arKYeMETxOg	141.142.220.202	5353	224.0.0.251	5353	udp	dns	-	-	-	S0	-	0	D	1	73	0	0	(empty)
+1300475168.853899	CCvvfg3TEfuqmmG4bh	141.142.220.118	43927	141.142.2.2	53	udp	dns	0.000435	38	89	SF	-	0	Dd	1	66	1	117	(empty)
+1300475168.854378	CsRx2w45OKnoww6xl4	141.142.220.118	37676	141.142.2.2	53	udp	dns	0.000420	52	99	SF	-	0	Dd	1	80	1	127	(empty)
+1300475168.854837	CRJuHdVW0XPVINV8a	141.142.220.118	40526	141.142.2.2	53	udp	dns	0.000392	38	183	SF	-	0	Dd	1	66	1	211	(empty)
+1300475168.857956	CPbrpk1qSsw6ESzHV4	141.142.220.118	32902	141.142.2.2	53	udp	dns	0.000317	38	89	SF	-	0	Dd	1	66	1	117	(empty)
+1300475168.858306	C6pKV8GSxOnSLghOa	141.142.220.118	59816	141.142.2.2	53	udp	dns	0.000343	52	99	SF	-	0	Dd	1	80	1	127	(empty)
+1300475168.858713	CIPOse170MGiRM1Qf4	141.142.220.118	59714	141.142.2.2	53	udp	dns	0.000375	38	183	SF	-	0	Dd	1	66	1	211	(empty)
+1300475168.891644	C7XEbhP654jzLoe3a	141.142.220.118	58206	141.142.2.2	53	udp	dns	0.000339	38	89	SF	-	0	Dd	1	66	1	117	(empty)
+1300475168.892037	CJ3xTn1c4Zw9TmAE05	141.142.220.118	38911	141.142.2.2	53	udp	dns	0.000335	52	99	SF	-	0	Dd	1	80	1	127	(empty)
+1300475168.892414	CMXxB5GvmoxJFXdTa	141.142.220.118	59746	141.142.2.2	53	udp	dns	0.000421	38	183	SF	-	0	Dd	1	66	1	211	(empty)
+1300475168.893988	Caby8b1slFea8xwSmb	141.142.220.118	45000	141.142.2.2	53	udp	dns	0.000384	38	89	SF	-	0	Dd	1	66	1	117	(empty)
+1300475168.894422	Che1bq3i2rO3KD1Syg	141.142.220.118	48479	141.142.2.2	53	udp	dns	0.000317	52	99	SF	-	0	Dd	1	80	1	127	(empty)
+1300475168.894787	C3SfNE4BWaU4aSuwkc	141.142.220.118	48128	141.142.2.2	53	udp	dns	0.000423	38	183	SF	-	0	Dd	1	66	1	211	(empty)
+1300475168.901749	CEle3f3zno26fFZkrh	141.142.220.118	56056	141.142.2.2	53	udp	dns	0.000402	36	131	SF	-	0	Dd	1	64	1	159	(empty)
+1300475168.902195	CwSkQu4eWZCH7OONC1	141.142.220.118	55092	141.142.2.2	53	udp	dns	0.000374	36	198	SF	-	0	Dd	1	64	1	226	(empty)
+1300475168.652003	CjhGID4nQcgTWjvg4c	141.142.220.118	35634	208.80.152.2	80	tcp	-	-	-	-	OTH	-	0	D	1	515	0	0	(empty)
+#close	2014-08-24-15-51-55

testing/btest/Baseline/core.pcap.dynamic-filter/output

@@ -0,0 +1,30 @@
+1, [orig_h=141.142.220.202, orig_p=5353/udp, resp_h=224.0.0.251, resp_p=5353/udp]
+2, [orig_h=141.142.220.118, orig_p=35634/tcp, resp_h=208.80.152.2, resp_p=80/tcp]
+3, [orig_h=141.142.220.118, orig_p=43927/udp, resp_h=141.142.2.2, resp_p=53/udp]
+4, [orig_h=141.142.220.118, orig_p=43927/udp, resp_h=141.142.2.2, resp_p=53/udp]
+5, [orig_h=141.142.220.118, orig_p=37676/udp, resp_h=141.142.2.2, resp_p=53/udp]
+6, [orig_h=141.142.220.118, orig_p=37676/udp, resp_h=141.142.2.2, resp_p=53/udp]
+7, [orig_h=141.142.220.118, orig_p=40526/udp, resp_h=141.142.2.2, resp_p=53/udp]
+8, [orig_h=141.142.220.118, orig_p=40526/udp, resp_h=141.142.2.2, resp_p=53/udp]
+9, [orig_h=141.142.220.118, orig_p=32902/udp, resp_h=141.142.2.2, resp_p=53/udp]
+10, [orig_h=141.142.220.118, orig_p=32902/udp, resp_h=141.142.2.2, resp_p=53/udp]
+11, [orig_h=141.142.220.118, orig_p=59816/udp, resp_h=141.142.2.2, resp_p=53/udp]
+12, [orig_h=141.142.220.118, orig_p=59816/udp, resp_h=141.142.2.2, resp_p=53/udp]
+13, [orig_h=141.142.220.118, orig_p=59714/udp, resp_h=141.142.2.2, resp_p=53/udp]
+14, [orig_h=141.142.220.118, orig_p=59714/udp, resp_h=141.142.2.2, resp_p=53/udp]
+15, [orig_h=141.142.220.118, orig_p=58206/udp, resp_h=141.142.2.2, resp_p=53/udp]
+16, [orig_h=141.142.220.118, orig_p=58206/udp, resp_h=141.142.2.2, resp_p=53/udp]
+17, [orig_h=141.142.220.118, orig_p=38911/udp, resp_h=141.142.2.2, resp_p=53/udp]
+18, [orig_h=141.142.220.118, orig_p=38911/udp, resp_h=141.142.2.2, resp_p=53/udp]
+19, [orig_h=141.142.220.118, orig_p=59746/udp, resp_h=141.142.2.2, resp_p=53/udp]
+20, [orig_h=141.142.220.118, orig_p=59746/udp, resp_h=141.142.2.2, resp_p=53/udp]
+21, [orig_h=141.142.220.118, orig_p=45000/udp, resp_h=141.142.2.2, resp_p=53/udp]
+22, [orig_h=141.142.220.118, orig_p=45000/udp, resp_h=141.142.2.2, resp_p=53/udp]
+23, [orig_h=141.142.220.118, orig_p=48479/udp, resp_h=141.142.2.2, resp_p=53/udp]
+24, [orig_h=141.142.220.118, orig_p=48479/udp, resp_h=141.142.2.2, resp_p=53/udp]
+25, [orig_h=141.142.220.118, orig_p=48128/udp, resp_h=141.142.2.2, resp_p=53/udp]
+26, [orig_h=141.142.220.118, orig_p=48128/udp, resp_h=141.142.2.2, resp_p=53/udp]
+27, [orig_h=141.142.220.118, orig_p=56056/udp, resp_h=141.142.2.2, resp_p=53/udp]
+28, [orig_h=141.142.220.118, orig_p=56056/udp, resp_h=141.142.2.2, resp_p=53/udp]
+29, [orig_h=141.142.220.118, orig_p=55092/udp, resp_h=141.142.2.2, resp_p=53/udp]
+30, [orig_h=141.142.220.118, orig_p=55092/udp, resp_h=141.142.2.2, resp_p=53/udp]

testing/btest/Baseline/core.pcap.filter-error/output

@@ -0,0 +1,3 @@
+fatal error in /home/robin/bro/master/scripts/base/frameworks/packet-filter/./main.bro, line 282: Bad pcap filter 'kaputt'
+----
+error, cannot compile BPF filter "kaputt, too"

testing/btest/Baseline/core.pcap.input-error/output2

@@ -0,0 +1,2 @@
+fatal error: problem with interface NO_SUCH_INTERFACE 
+fatal error: problem with trace file NO_SUCH_TRACE (NO_SUCH_TRACE: No such file or directory)

testing/btest/Baseline/core.pcap.pseudo-realtime/output

@@ -0,0 +1 @@
+real time matches trace time

testing/btest/Baseline/core.pcap.read-trace-with-filter/conn.log

@@ -0,0 +1,10 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	conn
+#open	2014-08-23-18-29-48
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	service	duration	orig_bytes	resp_bytes	conn_state	local_orig	missed_bytes	history	orig_pkts	orig_ip_bytes	resp_pkts	resp_ip_bytes	tunnel_parents
+#types	time	string	addr	port	addr	port	enum	string	interval	count	count	string	bool	count	string	count	count	count	count	set[string]
+1300475168.892936	CXWv6p3arKYeMETxOg	141.142.220.118	50000	208.80.152.3	80	tcp	http	0.229603	1148	734	S1	-	0	ShADad	6	1468	4	950	(empty)
+#close	2014-08-23-18-29-48

testing/btest/Baseline/core.pcap.read-trace-with-filter/packet_filter.log

@@ -0,0 +1,10 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	packet_filter
+#open	2014-08-23-18-29-48
+#fields	ts	node	filter	init	success
+#types	time	string	string	bool	bool
+1408818588.510297	bro	port 50000	T	T
+#close	2014-08-23-18-29-48

testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log

@@ -3,7 +3,7 @@
 #empty_field	(empty)
 #unset_field	-
 #path	loaded_scripts
-#open	2014-08-14-04-31-10
+#open	2014-09-06-01-19-42
 #fields	name
 #types	string
 scripts/base/init-bare.bro
@@ -43,6 +43,7 @@ scripts/base/init-bare.bro
     scripts/base/frameworks/files/magic/__load__.bro
   build/scripts/base/bif/__load__.bro
     build/scripts/base/bif/broxygen.bif.bro
+    build/scripts/base/bif/pcap.bif.bro
     build/scripts/base/bif/bloom-filter.bif.bro
     build/scripts/base/bif/cardinality-counter.bif.bro
     build/scripts/base/bif/top-k.bif.bro
@@ -113,4 +114,4 @@ scripts/base/init-bare.bro
     build/scripts/base/bif/plugins/Bro_SQLiteWriter.sqlite.bif.bro
 scripts/policy/misc/loaded-scripts.bro
   scripts/base/utils/paths.bro
-#close	2014-08-14-04-31-10
+#close	2014-09-06-01-19-42

testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log

@@ -3,7 +3,7 @@
 #empty_field	(empty)
 #unset_field	-
 #path	loaded_scripts
-#open	2014-08-14-04-31-11
+#open	2014-09-06-01-20-32
 #fields	name
 #types	string
 scripts/base/init-bare.bro
@@ -43,6 +43,7 @@ scripts/base/init-bare.bro
     scripts/base/frameworks/files/magic/__load__.bro
   build/scripts/base/bif/__load__.bro
     build/scripts/base/bif/broxygen.bif.bro
+    build/scripts/base/bif/pcap.bif.bro
     build/scripts/base/bif/bloom-filter.bif.bro
     build/scripts/base/bif/cardinality-counter.bif.bro
     build/scripts/base/bif/top-k.bif.bro
@@ -242,4 +243,4 @@ scripts/base/init-default.bro
   scripts/base/misc/find-checksum-offloading.bro
   scripts/base/misc/find-filtered-trace.bro
 scripts/policy/misc/loaded-scripts.bro
-#close	2014-08-14-04-31-11
+#close	2014-09-06-01-20-32

testing/btest/Baseline/plugins.bifs-and-scripts-install/output

@@ -1,6 +1,6 @@
 Demo::Foo - <Insert description> (dynamic, version 1.0)
-    [Event] plugin_event
     [Function] hello_plugin_world
+    [Event] plugin_event
 
 plugin: automatically loaded at startup
 calling bif, Hello from the plugin!

testing/btest/Baseline/plugins.bifs-and-scripts/output

@@ -1,6 +1,6 @@
 Demo::Foo - <Insert description> (dynamic, version 1.0)
-    [Event] plugin_event
     [Function] hello_plugin_world
+    [Event] plugin_event
 
 ===
 plugin: automatically loaded at startup

testing/btest/Baseline/plugins.hooks/output

@@ -182,7 +182,7 @@
 0.000000   MetaHookPost  CallFunction(Log::__create_stream, (Unified2::LOG, [columns=<no value description>, ev=Unified2::log_unified2])) -> <null>
 0.000000   MetaHookPost  CallFunction(Log::__create_stream, (Weird::LOG, [columns=<no value description>, ev=Weird::log_weird])) -> <null>
 0.000000   MetaHookPost  CallFunction(Log::__create_stream, (X509::LOG, [columns=<no value description>, ev=X509::log_x509])) -> <null>
-0.000000   MetaHookPost  CallFunction(Log::__write, (PacketFilter::LOG, [ts=1405981560.501473, node=bro, filter=ip or not ip, init=T, success=T])) -> <null>
+0.000000   MetaHookPost  CallFunction(Log::__write, (PacketFilter::LOG, [ts=1409853900.737227, node=bro, filter=ip or not ip, init=T, success=T])) -> <null>
 0.000000   MetaHookPost  CallFunction(Log::add_default_filter, (Cluster::LOG)) -> <null>
 0.000000   MetaHookPost  CallFunction(Log::add_default_filter, (Communication::LOG)) -> <null>
 0.000000   MetaHookPost  CallFunction(Log::add_default_filter, (Conn::LOG)) -> <null>
@@ -273,8 +273,8 @@
 0.000000   MetaHookPost  CallFunction(Log::create_stream, (Unified2::LOG, [columns=<no value description>, ev=Unified2::log_unified2])) -> <null>
 0.000000   MetaHookPost  CallFunction(Log::create_stream, (Weird::LOG, [columns=<no value description>, ev=Weird::log_weird])) -> <null>
 0.000000   MetaHookPost  CallFunction(Log::create_stream, (X509::LOG, [columns=<no value description>, ev=X509::log_x509])) -> <null>
-0.000000   MetaHookPost  CallFunction(Log::default_path_func, (PacketFilter::LOG, , [ts=1405981560.501473, node=bro, filter=ip or not ip, init=T, success=T])) -> <null>
+0.000000   MetaHookPost  CallFunction(Log::default_path_func, (PacketFilter::LOG, , [ts=1409853900.737227, node=bro, filter=ip or not ip, init=T, success=T])) -> <null>
-0.000000   MetaHookPost  CallFunction(Log::write, (PacketFilter::LOG, [ts=1405981560.501473, node=bro, filter=ip or not ip, init=T, success=T])) -> <null>
+0.000000   MetaHookPost  CallFunction(Log::write, (PacketFilter::LOG, [ts=1409853900.737227, node=bro, filter=ip or not ip, init=T, success=T])) -> <null>
 0.000000   MetaHookPost  CallFunction(Notice::want_pp, ()) -> <null>
 0.000000   MetaHookPost  CallFunction(PacketFilter::build, ()) -> <null>
 0.000000   MetaHookPost  CallFunction(PacketFilter::combine_filters, (ip or not ip, and, )) -> <null>
@@ -316,7 +316,11 @@
 0.000000   MetaHookPost  LoadFile(../main) -> -1
 0.000000   MetaHookPost  LoadFile(./Bro_ARP.events.bif.bro) -> -1
 0.000000   MetaHookPost  LoadFile(./Bro_AYIYA.events.bif.bro) -> -1
+0.000000   MetaHookPost  LoadFile(./Bro_AsciiReader.ascii.bif.bro) -> -1
+0.000000   MetaHookPost  LoadFile(./Bro_AsciiWriter.ascii.bif.bro) -> -1
 0.000000   MetaHookPost  LoadFile(./Bro_BackDoor.events.bif.bro) -> -1
+0.000000   MetaHookPost  LoadFile(./Bro_BenchmarkReader.benchmark.bif.bro) -> -1
+0.000000   MetaHookPost  LoadFile(./Bro_BinaryReader.binary.bif.bro) -> -1
 0.000000   MetaHookPost  LoadFile(./Bro_BitTorrent.events.bif.bro) -> -1
 0.000000   MetaHookPost  LoadFile(./Bro_ConnSize.events.bif.bro) -> -1
 0.000000   MetaHookPost  LoadFile(./Bro_DCE_RPC.events.bif.bro) -> -1
@@ -347,16 +351,20 @@
 0.000000   MetaHookPost  LoadFile(./Bro_NetBIOS.events.bif.bro) -> -1
 0.000000   MetaHookPost  LoadFile(./Bro_NetBIOS.functions.bif.bro) -> -1
 0.000000   MetaHookPost  LoadFile(./Bro_NetFlow.events.bif.bro) -> -1
+0.000000   MetaHookPost  LoadFile(./Bro_NoneWriter.none.bif.bro) -> -1
 0.000000   MetaHookPost  LoadFile(./Bro_PIA.events.bif.bro) -> -1
 0.000000   MetaHookPost  LoadFile(./Bro_POP3.events.bif.bro) -> -1
 0.000000   MetaHookPost  LoadFile(./Bro_RADIUS.events.bif.bro) -> -1
 0.000000   MetaHookPost  LoadFile(./Bro_RPC.events.bif.bro) -> -1
+0.000000   MetaHookPost  LoadFile(./Bro_RawReader.raw.bif.bro) -> -1
 0.000000   MetaHookPost  LoadFile(./Bro_SMB.events.bif.bro) -> -1
 0.000000   MetaHookPost  LoadFile(./Bro_SMTP.events.bif.bro) -> -1
 0.000000   MetaHookPost  LoadFile(./Bro_SMTP.functions.bif.bro) -> -1
 0.000000   MetaHookPost  LoadFile(./Bro_SNMP.events.bif.bro) -> -1
 0.000000   MetaHookPost  LoadFile(./Bro_SNMP.types.bif.bro) -> -1
 0.000000   MetaHookPost  LoadFile(./Bro_SOCKS.events.bif.bro) -> -1
+0.000000   MetaHookPost  LoadFile(./Bro_SQLiteReader.sqlite.bif.bro) -> -1
+0.000000   MetaHookPost  LoadFile(./Bro_SQLiteWriter.sqlite.bif.bro) -> -1
 0.000000   MetaHookPost  LoadFile(./Bro_SSH.events.bif.bro) -> -1
 0.000000   MetaHookPost  LoadFile(./Bro_SSL.events.bif.bro) -> -1
 0.000000   MetaHookPost  LoadFile(./Bro_SteppingStone.events.bif.bro) -> -1
@@ -380,21 +388,20 @@
 0.000000   MetaHookPost  LoadFile(./cardinality-counter.bif.bro) -> -1
 0.000000   MetaHookPost  LoadFile(./const.bif.bro) -> -1
 0.000000   MetaHookPost  LoadFile(./consts) -> -1
-0.000000   MetaHookPost  LoadFile(./consts.bif.bro) -> -1
 0.000000   MetaHookPost  LoadFile(./consts.bro) -> -1
 0.000000   MetaHookPost  LoadFile(./contents) -> -1
 0.000000   MetaHookPost  LoadFile(./dcc-send) -> -1
 0.000000   MetaHookPost  LoadFile(./entities) -> -1
 0.000000   MetaHookPost  LoadFile(./event.bif.bro) -> -1
-0.000000   MetaHookPost  LoadFile(./events.bif.bro) -> -1
 0.000000   MetaHookPost  LoadFile(./exec) -> -1
 0.000000   MetaHookPost  LoadFile(./file_analysis.bif.bro) -> -1
 0.000000   MetaHookPost  LoadFile(./files) -> -1
-0.000000   MetaHookPost  LoadFile(./functions.bif.bro) -> -1
 0.000000   MetaHookPost  LoadFile(./gridftp) -> -1
 0.000000   MetaHookPost  LoadFile(./hll_unique) -> -1
+0.000000   MetaHookPost  LoadFile(./hooks.bif.bro) -> -1
 0.000000   MetaHookPost  LoadFile(./inactivity) -> -1
 0.000000   MetaHookPost  LoadFile(./info) -> -1
+0.000000   MetaHookPost  LoadFile(./init.bro) -> -1
 0.000000   MetaHookPost  LoadFile(./input) -> -1
 0.000000   MetaHookPost  LoadFile(./input.bif.bro) -> -1
 0.000000   MetaHookPost  LoadFile(./last) -> -1
@@ -408,6 +415,7 @@
 0.000000   MetaHookPost  LoadFile(./netstats) -> -1
 0.000000   MetaHookPost  LoadFile(./non-cluster) -> -1
 0.000000   MetaHookPost  LoadFile(./patterns) -> -1
+0.000000   MetaHookPost  LoadFile(./pcap.bif.bro) -> -1
 0.000000   MetaHookPost  LoadFile(./plugins) -> -1
 0.000000   MetaHookPost  LoadFile(./polling) -> -1
 0.000000   MetaHookPost  LoadFile(./postprocessors) -> -1
@@ -432,9 +440,7 @@
 0.000000   MetaHookPost  LoadFile(.<...>/ascii) -> -1
 0.000000   MetaHookPost  LoadFile(.<...>/benchmark) -> -1
 0.000000   MetaHookPost  LoadFile(.<...>/binary) -> -1
-0.000000   MetaHookPost  LoadFile(.<...>/dataseries) -> -1
 0.000000   MetaHookPost  LoadFile(.<...>/drop) -> -1
-0.000000   MetaHookPost  LoadFile(.<...>/elasticsearch) -> -1
 0.000000   MetaHookPost  LoadFile(.<...>/email_admin) -> -1
 0.000000   MetaHookPost  LoadFile(.<...>/hostnames) -> -1
 0.000000   MetaHookPost  LoadFile(.<...>/none) -> -1
@@ -699,7 +705,7 @@
 0.000000   MetaHookPre   CallFunction(Log::__create_stream, (Unified2::LOG, [columns=<no value description>, ev=Unified2::log_unified2]))
 0.000000   MetaHookPre   CallFunction(Log::__create_stream, (Weird::LOG, [columns=<no value description>, ev=Weird::log_weird]))
 0.000000   MetaHookPre   CallFunction(Log::__create_stream, (X509::LOG, [columns=<no value description>, ev=X509::log_x509]))
-0.000000   MetaHookPre   CallFunction(Log::__write, (PacketFilter::LOG, [ts=1405981560.501473, node=bro, filter=ip or not ip, init=T, success=T]))
+0.000000   MetaHookPre   CallFunction(Log::__write, (PacketFilter::LOG, [ts=1409853900.737227, node=bro, filter=ip or not ip, init=T, success=T]))
 0.000000   MetaHookPre   CallFunction(Log::add_default_filter, (Cluster::LOG))
 0.000000   MetaHookPre   CallFunction(Log::add_default_filter, (Communication::LOG))
 0.000000   MetaHookPre   CallFunction(Log::add_default_filter, (Conn::LOG))
@@ -790,8 +796,8 @@
 0.000000   MetaHookPre   CallFunction(Log::create_stream, (Unified2::LOG, [columns=<no value description>, ev=Unified2::log_unified2]))
 0.000000   MetaHookPre   CallFunction(Log::create_stream, (Weird::LOG, [columns=<no value description>, ev=Weird::log_weird]))
 0.000000   MetaHookPre   CallFunction(Log::create_stream, (X509::LOG, [columns=<no value description>, ev=X509::log_x509]))
-0.000000   MetaHookPre   CallFunction(Log::default_path_func, (PacketFilter::LOG, , [ts=1405981560.501473, node=bro, filter=ip or not ip, init=T, success=T]))
+0.000000   MetaHookPre   CallFunction(Log::default_path_func, (PacketFilter::LOG, , [ts=1409853900.737227, node=bro, filter=ip or not ip, init=T, success=T]))
-0.000000   MetaHookPre   CallFunction(Log::write, (PacketFilter::LOG, [ts=1405981560.501473, node=bro, filter=ip or not ip, init=T, success=T]))
+0.000000   MetaHookPre   CallFunction(Log::write, (PacketFilter::LOG, [ts=1409853900.737227, node=bro, filter=ip or not ip, init=T, success=T]))
 0.000000   MetaHookPre   CallFunction(Notice::want_pp, ())
 0.000000   MetaHookPre   CallFunction(PacketFilter::build, ())
 0.000000   MetaHookPre   CallFunction(PacketFilter::combine_filters, (ip or not ip, and, ))
@@ -833,7 +839,11 @@
 0.000000   MetaHookPre   LoadFile(../main)
 0.000000   MetaHookPre   LoadFile(./Bro_ARP.events.bif.bro)
 0.000000   MetaHookPre   LoadFile(./Bro_AYIYA.events.bif.bro)
+0.000000   MetaHookPre   LoadFile(./Bro_AsciiReader.ascii.bif.bro)
+0.000000   MetaHookPre   LoadFile(./Bro_AsciiWriter.ascii.bif.bro)
 0.000000   MetaHookPre   LoadFile(./Bro_BackDoor.events.bif.bro)
+0.000000   MetaHookPre   LoadFile(./Bro_BenchmarkReader.benchmark.bif.bro)
+0.000000   MetaHookPre   LoadFile(./Bro_BinaryReader.binary.bif.bro)
 0.000000   MetaHookPre   LoadFile(./Bro_BitTorrent.events.bif.bro)
 0.000000   MetaHookPre   LoadFile(./Bro_ConnSize.events.bif.bro)
 0.000000   MetaHookPre   LoadFile(./Bro_DCE_RPC.events.bif.bro)
@@ -864,16 +874,20 @@
 0.000000   MetaHookPre   LoadFile(./Bro_NetBIOS.events.bif.bro)
 0.000000   MetaHookPre   LoadFile(./Bro_NetBIOS.functions.bif.bro)
 0.000000   MetaHookPre   LoadFile(./Bro_NetFlow.events.bif.bro)
+0.000000   MetaHookPre   LoadFile(./Bro_NoneWriter.none.bif.bro)
 0.000000   MetaHookPre   LoadFile(./Bro_PIA.events.bif.bro)
 0.000000   MetaHookPre   LoadFile(./Bro_POP3.events.bif.bro)
 0.000000   MetaHookPre   LoadFile(./Bro_RADIUS.events.bif.bro)
 0.000000   MetaHookPre   LoadFile(./Bro_RPC.events.bif.bro)
+0.000000   MetaHookPre   LoadFile(./Bro_RawReader.raw.bif.bro)
 0.000000   MetaHookPre   LoadFile(./Bro_SMB.events.bif.bro)
 0.000000   MetaHookPre   LoadFile(./Bro_SMTP.events.bif.bro)
 0.000000   MetaHookPre   LoadFile(./Bro_SMTP.functions.bif.bro)
 0.000000   MetaHookPre   LoadFile(./Bro_SNMP.events.bif.bro)
 0.000000   MetaHookPre   LoadFile(./Bro_SNMP.types.bif.bro)
 0.000000   MetaHookPre   LoadFile(./Bro_SOCKS.events.bif.bro)
+0.000000   MetaHookPre   LoadFile(./Bro_SQLiteReader.sqlite.bif.bro)
+0.000000   MetaHookPre   LoadFile(./Bro_SQLiteWriter.sqlite.bif.bro)
 0.000000   MetaHookPre   LoadFile(./Bro_SSH.events.bif.bro)
 0.000000   MetaHookPre   LoadFile(./Bro_SSL.events.bif.bro)
 0.000000   MetaHookPre   LoadFile(./Bro_SteppingStone.events.bif.bro)
@@ -897,21 +911,20 @@
 0.000000   MetaHookPre   LoadFile(./cardinality-counter.bif.bro)
 0.000000   MetaHookPre   LoadFile(./const.bif.bro)
 0.000000   MetaHookPre   LoadFile(./consts)
-0.000000   MetaHookPre   LoadFile(./consts.bif.bro)
 0.000000   MetaHookPre   LoadFile(./consts.bro)
 0.000000   MetaHookPre   LoadFile(./contents)
 0.000000   MetaHookPre   LoadFile(./dcc-send)
 0.000000   MetaHookPre   LoadFile(./entities)
 0.000000   MetaHookPre   LoadFile(./event.bif.bro)
-0.000000   MetaHookPre   LoadFile(./events.bif.bro)
 0.000000   MetaHookPre   LoadFile(./exec)
 0.000000   MetaHookPre   LoadFile(./file_analysis.bif.bro)
 0.000000   MetaHookPre   LoadFile(./files)
-0.000000   MetaHookPre   LoadFile(./functions.bif.bro)
 0.000000   MetaHookPre   LoadFile(./gridftp)
 0.000000   MetaHookPre   LoadFile(./hll_unique)
+0.000000   MetaHookPre   LoadFile(./hooks.bif.bro)
 0.000000   MetaHookPre   LoadFile(./inactivity)
 0.000000   MetaHookPre   LoadFile(./info)
+0.000000   MetaHookPre   LoadFile(./init.bro)
 0.000000   MetaHookPre   LoadFile(./input)
 0.000000   MetaHookPre   LoadFile(./input.bif.bro)
 0.000000   MetaHookPre   LoadFile(./last)
@@ -925,6 +938,7 @@
 0.000000   MetaHookPre   LoadFile(./netstats)
 0.000000   MetaHookPre   LoadFile(./non-cluster)
 0.000000   MetaHookPre   LoadFile(./patterns)
+0.000000   MetaHookPre   LoadFile(./pcap.bif.bro)
 0.000000   MetaHookPre   LoadFile(./plugins)
 0.000000   MetaHookPre   LoadFile(./polling)
 0.000000   MetaHookPre   LoadFile(./postprocessors)
@@ -949,9 +963,7 @@
 0.000000   MetaHookPre   LoadFile(.<...>/ascii)
 0.000000   MetaHookPre   LoadFile(.<...>/benchmark)
 0.000000   MetaHookPre   LoadFile(.<...>/binary)
-0.000000   MetaHookPre   LoadFile(.<...>/dataseries)
 0.000000   MetaHookPre   LoadFile(.<...>/drop)
-0.000000   MetaHookPre   LoadFile(.<...>/elasticsearch)
 0.000000   MetaHookPre   LoadFile(.<...>/email_admin)
 0.000000   MetaHookPre   LoadFile(.<...>/hostnames)
 0.000000   MetaHookPre   LoadFile(.<...>/none)
@@ -1216,7 +1228,7 @@
 0.000000 | HookCallFunction Log::__create_stream(Unified2::LOG, [columns=<no value description>, ev=Unified2::log_unified2])
 0.000000 | HookCallFunction Log::__create_stream(Weird::LOG, [columns=<no value description>, ev=Weird::log_weird])
 0.000000 | HookCallFunction Log::__create_stream(X509::LOG, [columns=<no value description>, ev=X509::log_x509])
-0.000000 | HookCallFunction Log::__write(PacketFilter::LOG, [ts=1405981560.501473, node=bro, filter=ip or not ip, init=T, success=T])
+0.000000 | HookCallFunction Log::__write(PacketFilter::LOG, [ts=1409853900.737227, node=bro, filter=ip or not ip, init=T, success=T])
 0.000000 | HookCallFunction Log::add_default_filter(Cluster::LOG)
 0.000000 | HookCallFunction Log::add_default_filter(Communication::LOG)
 0.000000 | HookCallFunction Log::add_default_filter(Conn::LOG)
@@ -1307,8 +1319,8 @@
 0.000000 | HookCallFunction Log::create_stream(Unified2::LOG, [columns=<no value description>, ev=Unified2::log_unified2])
 0.000000 | HookCallFunction Log::create_stream(Weird::LOG, [columns=<no value description>, ev=Weird::log_weird])
 0.000000 | HookCallFunction Log::create_stream(X509::LOG, [columns=<no value description>, ev=X509::log_x509])
-0.000000 | HookCallFunction Log::default_path_func(PacketFilter::LOG, , [ts=1405981560.501473, node=bro, filter=ip or not ip, init=T, success=T])
+0.000000 | HookCallFunction Log::default_path_func(PacketFilter::LOG, , [ts=1409853900.737227, node=bro, filter=ip or not ip, init=T, success=T])
-0.000000 | HookCallFunction Log::write(PacketFilter::LOG, [ts=1405981560.501473, node=bro, filter=ip or not ip, init=T, success=T])
+0.000000 | HookCallFunction Log::write(PacketFilter::LOG, [ts=1409853900.737227, node=bro, filter=ip or not ip, init=T, success=T])
 0.000000 | HookCallFunction Notice::want_pp()
 0.000000 | HookCallFunction PacketFilter::build()
 0.000000 | HookCallFunction PacketFilter::combine_filters(ip or not ip, and, )

testing/btest/Baseline/plugins.pktdumper/output

@@ -0,0 +1,12 @@
+Demo::Foo - A Foo packet dumper (dynamic, version 1.0)
+    [Packet Dumper] FooPktDumper (dumper prefix: foo)
+
+===
+Dumping to XXX: 1373858797.646968 len 94
+Dumping to XXX: 1373858797.646998 len 94
+Dumping to XXX: 1373858797.647041 len 86
+Dumping to XXX: 1373858797.647147 len 98
+Dumping to XXX: 1373858797.647186 len 86
+Dumping to XXX: 1373858797.647250 len 86
+Dumping to XXX: 1373858797.647317 len 86
+Dumping to XXX: 1373858797.647350 len 86

testing/btest/Baseline/plugins.pktsrc/conn.log

@@ -0,0 +1,10 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	conn
+#open	2014-09-04-18-06-05
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	service	duration	orig_bytes	resp_bytes	conn_state	local_orig	missed_bytes	history	orig_pkts	orig_ip_bytes	resp_pkts	resp_ip_bytes	tunnel_parents
+#types	time	string	addr	port	addr	port	enum	string	interval	count	count	string	bool	count	string	count	count	count	count	set[string]
+1409193037.000000	CXWv6p3arKYeMETxOg	1.2.0.2	2527	1.2.0.3	6649	tcp	-	-	-	-	S0	-	0	S	1	64	0	0	(empty)
+#close	2014-09-04-18-06-05

testing/btest/Baseline/plugins.writer/output

@@ -2,21 +2,21 @@ Demo::Foo - A Foo test logging writer (dynamic, version 1.0)
     [Writer] Foo (Log::WRITER_FOO)
 
 ===
-[packet_filter] 1406831942.605829|bro|ip or not ip|T|T
 [conn] 1340213005.165293|CXWv6p3arKYeMETxOg|10.0.0.55|53994|60.190.189.214|8124|tcp|-|4.314406|0|0|S0|-|0|S|5|320|0|0|
-[tunnel] 1340213015.276495|-|10.0.0.55|0|60.190.189.214|8124|Tunnel::SOCKS|Tunnel::DISCOVER
-[socks] 1340213015.276495|CjhGID4nQcgTWjvg4c|10.0.0.55|53994|60.190.189.214|8124|5|-|succeeded|-|www.osnews.com|80|192.168.0.31|-|2688
-[http] 1340213019.013158|CjhGID4nQcgTWjvg4c|10.0.0.55|53994|60.190.189.214|8124|1|GET|www.osnews.com|/images/printer2.gif|http://www.osnews.com/|Mozilla/5.0 (Macintosh; Intel Mac OS X 10.7; rv:10.0.2) Gecko/20100101 Firefox/10.0.2|0|0|304|Not Modified|-|-|-||-|-|-|-|-|-|-
-[http] 1340213019.013426|CjhGID4nQcgTWjvg4c|10.0.0.55|53994|60.190.189.214|8124|2|GET|www.osnews.com|/img2/shorturl.jpg|http://www.osnews.com/|Mozilla/5.0 (Macintosh; Intel Mac OS X 10.7; rv:10.0.2) Gecko/20100101 Firefox/10.0.2|0|0|304|Not Modified|-|-|-||-|-|-|-|-|-|-
-[http] 1340213019.580162|CjhGID4nQcgTWjvg4c|10.0.0.55|53994|60.190.189.214|8124|3|GET|www.osnews.com|/images/icons/9.gif|http://www.osnews.com/|Mozilla/5.0 (Macintosh; Intel Mac OS X 10.7; rv:10.0.2) Gecko/20100101 Firefox/10.0.2|0|0|304|Not Modified|-|-|-||-|-|-|-|-|-|-
-[http] 1340213020.155861|CjhGID4nQcgTWjvg4c|10.0.0.55|53994|60.190.189.214|8124|4|GET|www.osnews.com|/images/icons/26.gif|http://www.osnews.com/|Mozilla/5.0 (Macintosh; Intel Mac OS X 10.7; rv:10.0.2) Gecko/20100101 Firefox/10.0.2|0|1368|200|OK|-|-|-||-|-|-|-|-|FBtZ7y1ppK8iIeY622|image/gif
-[files] 1340213020.732581|FBtZ7y1ppK8iIeY622|60.190.189.214|10.0.0.55|CjhGID4nQcgTWjvg4c|HTTP|0||image/gif|-|0.000000|-|F|1368|1368|0|0|F|-|-|-|-|-
-[http] 1340213020.732963|CjhGID4nQcgTWjvg4c|10.0.0.55|53994|60.190.189.214|8124|5|GET|www.osnews.com|/images/icons/17.gif|http://www.osnews.com/|Mozilla/5.0 (Macintosh; Intel Mac OS X 10.7; rv:10.0.2) Gecko/20100101 Firefox/10.0.2|0|0|304|Not Modified|-|-|-||-|-|-|-|-|-|-
-[http] 1340213021.300269|CjhGID4nQcgTWjvg4c|10.0.0.55|53994|60.190.189.214|8124|6|GET|www.osnews.com|/images/left.gif|http://www.osnews.com/|Mozilla/5.0 (Macintosh; Intel Mac OS X 10.7; rv:10.0.2) Gecko/20100101 Firefox/10.0.2|0|0|304|Not Modified|-|-|-||-|-|-|-|-|-|-
 [conn] 1340213010.582723|CjhGID4nQcgTWjvg4c|10.0.0.55|53994|60.190.189.214|8124|tcp|http,socks|13.839419|3860|2934|SF|-|0|ShADadfF|23|5080|20|3986|
-[http] 1340213021.861584|CjhGID4nQcgTWjvg4c|10.0.0.55|53994|60.190.189.214|8124|7|GET|www.osnews.com|/images/icons/32.gif|http://www.osnews.com/|Mozilla/5.0 (Macintosh; Intel Mac OS X 10.7; rv:10.0.2) Gecko/20100101 Firefox/10.0.2|0|0|304|Not Modified|-|-|-||-|-|-|-|-|-|-
 [conn] 1340213048.780152|CCvvfg3TEfuqmmG4bh|10.0.0.55|53994|60.190.189.214|8124|tcp|-|-|-|-|SH|-|0|F|1|52|0|0|
 [conn] 1340213097.272764|CsRx2w45OKnoww6xl4|10.0.0.55|53994|60.190.189.214|8124|tcp|-|-|-|-|SH|-|0|F|1|52|0|0|
 [conn] 1340213162.160367|CRJuHdVW0XPVINV8a|10.0.0.55|53994|60.190.189.214|8124|tcp|-|-|-|-|SH|-|0|F|1|52|0|0|
 [conn] 1340213226.561757|CPbrpk1qSsw6ESzHV4|10.0.0.55|53994|60.190.189.214|8124|tcp|-|-|-|-|SH|-|0|F|1|52|0|0|
 [conn] 1340213290.981995|C6pKV8GSxOnSLghOa|10.0.0.55|53994|60.190.189.214|8124|tcp|-|-|-|-|SH|-|0|F|1|52|0|0|
+[files] 1340213020.732581|FBtZ7y1ppK8iIeY622|60.190.189.214|10.0.0.55|CjhGID4nQcgTWjvg4c|HTTP|0||image/gif|-|0.000000|-|F|1368|1368|0|0|F|-|-|-|-|-
+[http] 1340213019.013158|CjhGID4nQcgTWjvg4c|10.0.0.55|53994|60.190.189.214|8124|1|GET|www.osnews.com|/images/printer2.gif|http://www.osnews.com/|Mozilla/5.0 (Macintosh; Intel Mac OS X 10.7; rv:10.0.2) Gecko/20100101 Firefox/10.0.2|0|0|304|Not Modified|-|-|-||-|-|-|-|-|-|-
+[http] 1340213019.013426|CjhGID4nQcgTWjvg4c|10.0.0.55|53994|60.190.189.214|8124|2|GET|www.osnews.com|/img2/shorturl.jpg|http://www.osnews.com/|Mozilla/5.0 (Macintosh; Intel Mac OS X 10.7; rv:10.0.2) Gecko/20100101 Firefox/10.0.2|0|0|304|Not Modified|-|-|-||-|-|-|-|-|-|-
+[http] 1340213019.580162|CjhGID4nQcgTWjvg4c|10.0.0.55|53994|60.190.189.214|8124|3|GET|www.osnews.com|/images/icons/9.gif|http://www.osnews.com/|Mozilla/5.0 (Macintosh; Intel Mac OS X 10.7; rv:10.0.2) Gecko/20100101 Firefox/10.0.2|0|0|304|Not Modified|-|-|-||-|-|-|-|-|-|-
+[http] 1340213020.155861|CjhGID4nQcgTWjvg4c|10.0.0.55|53994|60.190.189.214|8124|4|GET|www.osnews.com|/images/icons/26.gif|http://www.osnews.com/|Mozilla/5.0 (Macintosh; Intel Mac OS X 10.7; rv:10.0.2) Gecko/20100101 Firefox/10.0.2|0|1368|200|OK|-|-|-||-|-|-|-|-|FBtZ7y1ppK8iIeY622|image/gif
+[http] 1340213020.732963|CjhGID4nQcgTWjvg4c|10.0.0.55|53994|60.190.189.214|8124|5|GET|www.osnews.com|/images/icons/17.gif|http://www.osnews.com/|Mozilla/5.0 (Macintosh; Intel Mac OS X 10.7; rv:10.0.2) Gecko/20100101 Firefox/10.0.2|0|0|304|Not Modified|-|-|-||-|-|-|-|-|-|-
+[http] 1340213021.300269|CjhGID4nQcgTWjvg4c|10.0.0.55|53994|60.190.189.214|8124|6|GET|www.osnews.com|/images/left.gif|http://www.osnews.com/|Mozilla/5.0 (Macintosh; Intel Mac OS X 10.7; rv:10.0.2) Gecko/20100101 Firefox/10.0.2|0|0|304|Not Modified|-|-|-||-|-|-|-|-|-|-
+[http] 1340213021.861584|CjhGID4nQcgTWjvg4c|10.0.0.55|53994|60.190.189.214|8124|7|GET|www.osnews.com|/images/icons/32.gif|http://www.osnews.com/|Mozilla/5.0 (Macintosh; Intel Mac OS X 10.7; rv:10.0.2) Gecko/20100101 Firefox/10.0.2|0|0|304|Not Modified|-|-|-||-|-|-|-|-|-|-
+[packet_filter] 1409859343.786281|bro|ip or not ip|T|T
+[socks] 1340213015.276495|CjhGID4nQcgTWjvg4c|10.0.0.55|53994|60.190.189.214|8124|5|-|succeeded|-|www.osnews.com|80|192.168.0.31|-|2688
+[tunnel] 1340213015.276495|-|10.0.0.55|0|60.190.189.214|8124|Tunnel::SOCKS|Tunnel::DISCOVER

testing/btest/core/pcap/dumper.bro

@@ -0,0 +1,5 @@
+# @TEST-EXEC: bro -r $TRACES/workshop_2011_browse.trace -w dump
+# @TEST-EXEC: hexdump -C $TRACES/workshop_2011_browse.trace >1
+# @TEST-EXEC: hexdump -C dump >2
+# @TEST-EXEC: sdiff -s 1 2 >output || true
+# @TEST-EXEC: btest-diff output

testing/btest/core/pcap/dynamic-filter.bro

@@ -0,0 +1,32 @@
+# @TEST-EXEC: bro -C -r $TRACES/wikipedia.trace %INPUT >output
+# @TEST-EXEC: btest-diff output
+# @TEST-EXEC: btest-diff conn.log
+
+redef enum PcapFilterID += { A, B };
+
+global cnt = 0;
+
+event new_packet(c: connection, p: pkt_hdr)
+	{
+	++cnt;
+
+	print cnt, c$id;
+	
+	if ( cnt == 1 )
+		if ( ! install_pcap_filter(A) )
+			print "error 3";
+
+	if ( cnt == 2 )
+		if ( ! install_pcap_filter(B) )
+			print "error 4";
+	}
+
+event bro_init()
+	{
+	if ( ! precompile_pcap_filter(A, "port 80") )
+		print "error 1";
+	
+	if ( ! precompile_pcap_filter(B, "port 53") )
+		print "error 2";
+	}
+

testing/btest/core/pcap/filter-error.bro

@@ -0,0 +1,16 @@
+# @TEST-EXEC-FAIL: bro -r $TRACES/workshop_2011_browse.trace -f "kaputt" >>output 2>&1
+# @TEST-EXEC-FAIL: test -e conn.log
+# @TEST-EXEC: echo ---- >>output
+# @TEST-EXEC: bro -r $TRACES/workshop_2011_browse.trace  %INPUT >>output 2>&1
+# @TEST-EXEC: test -e conn.log
+# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff output
+
+redef enum PcapFilterID += { A };
+
+event bro_init()
+	{
+	if ( ! precompile_pcap_filter(A, "kaputt, too") )
+		print "error", pcap_error();
+	}
+
+

testing/btest/core/pcap/input-error.bro

@@ -0,0 +1,14 @@
+# @TEST-EXEC-FAIL: bro -i NO_SUCH_INTERFACE 2>&1  >>output 2>&1
+# @TEST-EXEC: cat output | sed 's/(.*)//g' >output2
+# @TEST-EXEC-FAIL: bro -r NO_SUCH_TRACE 2>&1 >>output2 2>&1
+# @TEST-EXEC: btest-diff output2
+
+redef enum PcapFilterID += { A };
+
+event bro_init()
+	{
+	if ( ! precompile_pcap_filter(A, "kaputt, too") )
+		print "error", pcap_error();
+	}
+
+

testing/btest/core/pcap/pseudo-realtime.bro

@@ -0,0 +1,42 @@
+# @TEST-EXEC: bro -C -r $TRACES/wikipedia.trace %INPUT --pseudo-realtime >output
+# @TEST-EXEC: btest-diff output
+
+global init = F;
+global last_network = network_time();
+global last_current = current_time();
+global cnt = 0;
+global an = 0secs;
+global ac = 0secs;
+
+event new_packet(c: connection, p: pkt_hdr)
+	{
+	local tn = network_time();
+	local tc = current_time();
+	local dn = tn - last_network;
+	local dc = tc - last_current;
+
+	last_network = tn;
+	last_current = tc;
+	++cnt;
+
+	if ( ! init ) 
+		{
+		init = T;
+		return;
+		}
+
+	an += dn;
+	ac += dc;
+	
+	# print fmt("num=%d agg_delta_network=%.1f agg_delta_real=%.1f", cnt, an, ac);
+	}
+
+event bro_done()
+	{
+	local d = (an - ac);
+	if ( d < 0 secs)
+		d = -d;
+	
+	print fmt("real time %s trace time", d < 1.0secs ? "matches" : "does NOT match");
+	}
+

testing/btest/core/pcap/read-trace-with-filter.bro

@@ -0,0 +1,3 @@
+# @TEST-EXEC: bro -C -r $TRACES/wikipedia.trace -f "port 50000"
+# @TEST-EXEC: btest-diff conn.log
+# @TEST-EXEC: btest-diff packet_filter.log

testing/btest/plugins/api-version-mismatch.sh

@@ -1,6 +1,6 @@
 # @TEST-EXEC: ${DIST}/aux/bro-aux/plugin-support/init-plugin Demo Foo
 # @TEST-EXEC: bash %INPUT
-# @TEST-EXEC: make BRO=${DIST}
+# @TEST-EXEC: ./configure --bro-dist=${DIST} && make
 # @TEST-EXEC-FAIL: BRO_PLUGIN_PATH=`pwd` bro -NN Demo::Foo >>output 2>&1
 # @TEST-EXEC: btest-diff output
 

testing/btest/plugins/bifs-and-scripts-install.sh

@@ -1,11 +1,14 @@
 # @TEST-EXEC: ${DIST}/aux/bro-aux/plugin-support/init-plugin Demo Foo
 # @TEST-EXEC: bash %INPUT
-# @TEST-EXEC: BRO_PLUGIN_INSTALL=`pwd`/test-install make BRO=${DIST}
+# @TEST-EXEC: ./configure --bro-dist=${DIST} --install-root=`pwd`/test-install
+# @TEST-EXEC: make
 # @TEST-EXEC: make install
 # @TEST-EXEC: BRO_PLUGIN_PATH=`pwd`/test-install bro -NN Demo::Foo >>output
 # @TEST-EXEC: BRO_PLUGIN_PATH=`pwd` bro demo/foo -r $TRACES/empty.trace >>output
 # @TEST-EXEC: TEST_DIFF_CANONIFIER= btest-diff output
 
+mkdir -p scripts/demo/foo/base/
+
 cat >scripts/__load__.bro <<EOF
 @load ./demo/foo/base/at-startup.bro
 EOF
@@ -21,8 +24,6 @@ event bro_init() &priority=-10
         }
 EOF
 
-mkdir -p scripts/demo/foo/base/
-
 cat >scripts/demo/foo/base/at-startup.bro <<EOF
 event bro_init() &priority=10
         {
@@ -31,17 +32,16 @@ event bro_init() &priority=10
         }
 EOF
 
-cat >src/functions.bif <<EOF
+cat >src/foo.bif <<EOF
 function hello_plugin_world%(%): string
         %{
         return new StringVal("Hello from the plugin!");
         %}
+
+event plugin_event%(foo: count%);
 EOF
 
 cat >activate.bro <<EOF
 @load-plugin Demo::Foo
 EOF
 
-cat >src/events.bif <<EOF
-event plugin_event%(foo: count%);
-EOF

testing/btest/plugins/bifs-and-scripts.sh

@@ -1,6 +1,6 @@
 # @TEST-EXEC: ${DIST}/aux/bro-aux/plugin-support/init-plugin Demo Foo
 # @TEST-EXEC: bash %INPUT
-# @TEST-EXEC: make BRO=${DIST}
+# @TEST-EXEC: ./configure --bro-dist=${DIST} && make
 # @TEST-EXEC: BRO_PLUGIN_PATH=`pwd` bro -NN Demo::Foo >>output
 
 # @TEST-EXEC: echo === >>output
@@ -23,6 +23,8 @@
 
 # @TEST-EXEC: TEST_DIFF_CANONIFIER= btest-diff output
 
+mkdir -p scripts/demo/foo/base/
+
 cat >scripts/__load__.bro <<EOF
 @load ./demo/foo/base/at-startup.bro
 EOF
@@ -38,8 +40,6 @@ event bro_init() &priority=-10
         }
 EOF
 
-mkdir -p scripts/demo/foo/base/
-
 cat >scripts/demo/foo/base/at-startup.bro <<EOF
 event bro_init() &priority=10
         {
@@ -48,17 +48,16 @@ event bro_init() &priority=10
         }
 EOF
 
-cat >src/functions.bif <<EOF
+cat >src/foo.bif <<EOF
 function hello_plugin_world%(%): string
         %{
         return new StringVal("Hello from the plugin!");
         %}
+
+event plugin_event%(foo: count%);
 EOF
 
 cat >activate.bro <<EOF
 @load-plugin Demo::Foo
 EOF
 
-cat >src/events.bif <<EOF
-event plugin_event%(foo: count%);
-EOF

testing/btest/plugins/file.bro

@@ -1,6 +1,6 @@
 # @TEST-EXEC: ${DIST}/aux/bro-aux/plugin-support/init-plugin Demo Foo
 # @TEST-EXEC: cp -r %DIR/file-plugin/* .
-# @TEST-EXEC: make BRO=${DIST}
+# @TEST-EXEC: ./configure --bro-dist=${DIST} && make
 # @TEST-EXEC: BRO_PLUGIN_PATH=`pwd` bro -NN Demo::Foo >>output
 # @TEST-EXEC: echo === >>output
 # @TEST-EXEC: BRO_PLUGIN_PATH=`pwd` bro -r $TRACES/ftp/retr.trace %INPUT >>output

testing/btest/plugins/hooks.bro

@@ -1,6 +1,6 @@
 # @TEST-EXEC: ${DIST}/aux/bro-aux/plugin-support/init-plugin Demo Hooks
 # @TEST-EXEC: cp -r %DIR/hooks-plugin/* .
-# @TEST-EXEC: make BRO=${DIST}
+# @TEST-EXEC: ./configure --bro-dist=${DIST} && make
 # @TEST-EXEC: BRO_PLUGIN_PATH=`pwd` bro -r $TRACES/http/get.trace %INPUT 2>&1 | $SCRIPTS/diff-remove-abspath | sort | uniq  >output
 # @TEST-EXEC: btest-diff output
 

testing/btest/plugins/init-plugin.bro

@@ -1,5 +1,5 @@
 # @TEST-EXEC: ${DIST}/aux/bro-aux/plugin-support/init-plugin Demo Foo
-# @TEST-EXEC: make BRO=${DIST}
+# @TEST-EXEC: ./configure --bro-dist=${DIST} && make
 # @TEST-EXEC: BRO_PLUGIN_PATH=`pwd` bro -NN Demo::Foo >>output
 # @TEST-EXEC: echo === >>output
 # @TEST-EXEC: BRO_PLUGIN_PATH=`pwd` bro -r $TRACES/port4242.trace >>output

testing/btest/plugins/pktdumper-plugin/CMakeLists.txt

@@ -0,0 +1,17 @@
+
+project(Bro-Plugin-Demo-Foo)
+
+cmake_minimum_required(VERSION 2.6.3)
+
+if ( NOT BRO_DIST )
+    message(FATAL_ERROR "BRO_DIST not set")
+endif ()
+
+set(CMAKE_MODULE_PATH ${BRO_DIST}/cmake)
+
+include(BroPlugin)
+
+bro_plugin_begin(Demo Foo)
+bro_plugin_cc(src/Plugin.cc)
+bro_plugin_cc(src/Foo.cc)
+bro_plugin_end()

testing/btest/plugins/pktdumper-plugin/src/Foo.cc

@@ -0,0 +1,40 @@
+
+#include <fcntl.h>
+#include <stdio.h>
+
+#include "Foo.h"
+
+using namespace plugin::Demo_Foo;
+
+Foo::Foo(const std::string& path, bool is_live)
+	{
+	props.path = path;
+	}
+
+Foo::~Foo()
+	{
+	}
+
+void Foo::Open()
+	{
+	props.open_time = network_time;
+	props.hdr_size = 0;
+	Opened(props);
+	}
+
+void Foo::Close()
+	{
+	Closed();
+	}
+
+bool Foo::Dump(const Packet* pkt)
+	{
+	double t = double(pkt->hdr->ts.tv_sec) + double(pkt->hdr->ts.tv_usec) / 1e6;
+	fprintf(stdout, "Dumping to %s: %.6f len %u\n", props.path.c_str(), t, (unsigned int)pkt->hdr->len);
+	return true;
+	}
+
+iosource::PktDumper* Foo::Instantiate(const std::string& path, bool append)
+	{
+	return new Foo(path, append);
+	}

testing/btest/plugins/pktdumper-plugin/src/Foo.h

@@ -0,0 +1,30 @@
+
+#ifndef BRO_PLUGIN_DEMO_FOO_H
+#define BRO_PLUGIN_DEMO_FOO_H
+
+#include <Val.h>
+#include <iosource/PktDumper.h>
+
+namespace plugin {
+namespace Demo_Foo {
+
+class Foo : public iosource::PktDumper {
+public:
+	Foo(const std::string& path, bool is_live);
+	virtual ~Foo();
+
+	static PktDumper* Instantiate(const std::string& path, bool append);
+
+protected:
+	virtual void Open();
+	virtual void Close();
+	virtual bool Dump(const Packet* pkt);
+
+private:
+	Properties props;
+};
+
+}
+}
+
+#endif

testing/btest/plugins/pktdumper-plugin/src/Plugin.cc

@@ -0,0 +1,20 @@
+
+#include "Plugin.h"
+
+#include "Foo.h"
+
+namespace plugin { namespace Demo_Foo { Plugin plugin; } }
+
+using namespace plugin::Demo_Foo;
+
+plugin::Configuration Plugin::Configure()
+	{
+	AddComponent(new ::iosource::PktDumperComponent("FooPktDumper", "foo", ::plugin::Demo_Foo::Foo::Instantiate));
+
+	plugin::Configuration config;
+	config.name = "Demo::Foo";
+	config.description = "A Foo packet dumper";
+	config.version.major = 1;
+	config.version.minor = 0;
+	return config;
+	}

testing/btest/plugins/pktdumper.bro

@@ -0,0 +1,8 @@
+# @TEST-EXEC: ${DIST}/aux/bro-aux/plugin-support/init-plugin Demo Foo
+# @TEST-EXEC: cp -r %DIR/pktdumper-plugin/* .
+# @TEST-EXEC: ./configure --bro-dist=${DIST} && make
+# @TEST-EXEC: BRO_PLUGIN_PATH=`pwd` bro -NN Demo::Foo  >>output
+# @TEST-EXEC: echo === >>output
+# @TEST-EXEC: BRO_PLUGIN_PATH=`pwd` bro -r $TRACES/port4242.trace -w foo:XXX %INPUT FilteredTraceDetection::enable=F >>output
+# @TEST-EXEC: btest-diff output
+

testing/btest/plugins/pktsrc-plugin/CMakeLists.txt

@@ -0,0 +1,17 @@
+
+project(Bro-Plugin-Demo-Foo)
+
+cmake_minimum_required(VERSION 2.6.3)
+
+if ( NOT BRO_DIST )
+    message(FATAL_ERROR "BRO_DIST not set")
+endif ()
+
+set(CMAKE_MODULE_PATH ${BRO_DIST}/cmake)
+
+include(BroPlugin)
+
+bro_plugin_begin(Demo Foo)
+bro_plugin_cc(src/Plugin.cc)
+bro_plugin_cc(src/Foo.cc)
+bro_plugin_end()

testing/btest/plugins/pktsrc-plugin/src/Foo.cc

@@ -0,0 +1,77 @@
+
+#include <fcntl.h>
+#include <stdio.h>
+
+#include "Foo.h"
+
+using namespace plugin::Demo_Foo;
+
+Foo::Foo(const std::string& path, bool is_live)
+	{
+	packet =
+	string("\x45\x00\x00\x40\x15\x55\x40\x00\x3e\x06\x25\x5b\x01\x02\x00\x02"
+	"\x01\x02\x00\x03\x09\xdf\x19\xf9\x5d\x8a\x36\x7c\x00\x00\x00\x00"
+	"\xb0\x02\x40\x00\x3c\x72\x00\x00\x02\x04\x05\x5c\x01\x03\x03\x00"
+	"\x01\x01\x08\x0a\x00\x00\x00\x00\x00\x00\x00\x00\x01\x01\x04\x02", 64);
+
+	props.path = path;
+	props.selectable_fd = open("/bin/sh", O_RDONLY); // any fd is fine.
+	props.link_type = DLT_RAW;
+	props.hdr_size = 0;
+	props.netmask = 0;
+	props.is_live = 0;
+	}
+
+iosource::PktSrc* Foo::Instantiate(const std::string& path, bool is_live)
+	{
+	return new Foo(path, is_live);
+	}
+
+void Foo::Open()
+	{
+	Opened(props);
+	}
+
+void Foo::Close()
+	{
+	Closed();
+	}
+
+bool Foo::ExtractNextPacket(Packet* pkt)
+	{
+	if ( packet.empty() )
+		{
+		Close();
+		return false;
+		}
+
+	hdr.ts.tv_sec = 1409193037;
+	hdr.ts.tv_usec = 0;
+	hdr.caplen = hdr.len = packet.size();
+	pkt->ts = hdr.ts.tv_sec;
+	pkt->hdr = &hdr;
+	pkt->data = (const u_char *)packet.c_str();
+	return true;
+	}
+
+void Foo::DoneWithPacket()
+	{
+	packet.clear();
+	}
+
+bool Foo::PrecompileFilter(int index, const std::string& filter)
+	{
+	// skip for the testing.
+	return true;
+	}
+
+bool Foo::SetFilter(int index)
+	{
+	// skip for the testing.
+	return true;
+	}
+
+void Foo::Statistics(Stats* stats)
+	{
+	// skip for the testing.
+	}

testing/btest/plugins/pktsrc-plugin/src/Foo.h

@@ -0,0 +1,35 @@
+
+#ifndef BRO_PLUGIN_DEMO_FOO_H
+#define BRO_PLUGIN_DEMO_FOO_H
+
+#include <Val.h>
+#include <iosource/PktSrc.h>
+
+namespace plugin {
+namespace Demo_Foo {
+
+class Foo : public iosource::PktSrc {
+public:
+	Foo(const std::string& path, bool is_live);
+
+	static PktSrc* Instantiate(const std::string& path, bool is_live);
+
+protected:
+	virtual void Open();
+	virtual void Close();
+	virtual bool ExtractNextPacket(Packet* pkt);
+	virtual void DoneWithPacket();
+	virtual bool PrecompileFilter(int index, const std::string& filter);
+	virtual bool SetFilter(int index);
+	virtual void Statistics(Stats* stats);
+
+private:
+	Properties props;
+	string packet;
+	struct pcap_pkthdr hdr;
+};
+
+}
+}
+
+#endif

testing/btest/plugins/pktsrc-plugin/src/Plugin.cc

@@ -0,0 +1,20 @@
+
+#include "Plugin.h"
+
+#include "Foo.h"
+
+namespace plugin { namespace Demo_Foo { Plugin plugin; } }
+
+using namespace plugin::Demo_Foo;
+
+plugin::Configuration Plugin::Configure()
+	{
+	AddComponent(new ::iosource::PktSrcComponent("FooPktSrc", "foo", ::iosource::PktSrcComponent::BOTH, ::plugin::Demo_Foo::Foo::Instantiate));
+
+	plugin::Configuration config;
+	config.name = "Demo::Foo";
+	config.description = "A Foo packet source";
+	config.version.major = 1;
+	config.version.minor = 0;
+	return config;
+	}

testing/btest/plugins/pktsrc.bro

@@ -0,0 +1,8 @@
+# @TEST-EXEC: ${DIST}/aux/bro-aux/plugin-support/init-plugin Demo Foo
+# @TEST-EXEC: cp -r %DIR/pktsrc-plugin/* .
+# @TEST-EXEC: ./configure --bro-dist=${DIST} && make
+# @TEST-EXEC: BRO_PLUGIN_PATH=`pwd` bro -NN Demo::Foo  >>output
+# @TEST-EXEC: echo === >>output
+# @TEST-EXEC: BRO_PLUGIN_PATH=`pwd` bro -r foo:XXX %INPUT FilteredTraceDetection::enable=F >>output
+# @TEST-EXEC: btest-diff conn.log
+

testing/btest/plugins/protocol-plugin/CMakeLists.txt

@@ -15,6 +15,5 @@ bro_plugin_begin(Demo Foo)
 bro_plugin_cc(src/Plugin.cc)
 bro_plugin_cc(src/Foo.cc)
 bro_plugin_bif(src/events.bif)
-bro_plugin_bif(src/functions.bif)
 bro_plugin_pac(src/foo.pac src/foo-protocol.pac src/foo-analyzer.pac)
 bro_plugin_end()

testing/btest/plugins/protocol.bro

@@ -1,6 +1,6 @@
 # @TEST-EXEC: ${DIST}/aux/bro-aux/plugin-support/init-plugin Demo Foo
 # @TEST-EXEC: cp -r %DIR/protocol-plugin/* .
-# @TEST-EXEC: make BRO=${DIST}
+# @TEST-EXEC: ./configure --bro-dist=${DIST} && make
 # @TEST-EXEC: BRO_PLUGIN_PATH=`pwd` bro -NN Demo::Foo >>output
 # @TEST-EXEC: echo === >>output
 # @TEST-EXEC: BRO_PLUGIN_PATH=`pwd` bro -r $TRACES/port4242.trace %INPUT >>output

testing/btest/plugins/reader.bro

@@ -1,6 +1,6 @@
 # @TEST-EXEC: ${DIST}/aux/bro-aux/plugin-support/init-plugin Demo Foo
 # @TEST-EXEC: cp -r %DIR/reader-plugin/* .
-# @TEST-EXEC: make BRO=${DIST}
+# @TEST-EXEC: ./configure --bro-dist=${DIST} && make
 # @TEST-EXEC: BRO_PLUGIN_PATH=`pwd` bro -NN Demo::Foo >>output
 # @TEST-EXEC: echo === >>output
 # @TEST-EXEC: BRO_PLUGIN_PATH=`pwd` btest-bg-run bro bro %INPUT

testing/btest/plugins/writer.bro

@@ -1,8 +1,8 @@
 # @TEST-EXEC: ${DIST}/aux/bro-aux/plugin-support/init-plugin Demo Foo
 # @TEST-EXEC: cp -r %DIR/writer-plugin/* .
-# @TEST-EXEC: make BRO=${DIST}
+# @TEST-EXEC: ./configure --bro-dist=${DIST} && make
 # @TEST-EXEC: BRO_PLUGIN_PATH=`pwd` bro -NN Demo::Foo >>output
 # @TEST-EXEC: echo === >>output
-# @TEST-EXEC: BRO_PLUGIN_PATH=`pwd` bro -r $TRACES/socks.trace Log::default_writer=Log::WRITER_FOO %INPUT >>output
+# @TEST-EXEC: BRO_PLUGIN_PATH=`pwd` bro -r $TRACES/socks.trace Log::default_writer=Log::WRITER_FOO %INPUT | sort >>output
-# @TEST-EXEC: TEST_DIFF_CANONIFIER= btest-diff output
+# @TEST-EXEC: TEST_DIFF_CANONIFIER=diff-remove-timestamps btest-diff output