Merge branch 'topic/rework-packets' of https://github.com/jsbarber/bro

* 'topic/rework-packets' of https://github.com/jsbarber/bro:
  One more tinker to Packet -- ensure no uninitialized values
  Packet::IP()-created IP_Hdr should not free
  Make enums work for non-C++11 config
  Refactor to make bro use a common Packet object. Do a better job of parsing layer 2 and keeping track of layer 3 proto. Add support for raw packet event, including Layer2 headers.

Conflicts:
	aux/plugins

CHANGES

@@ -1,4 +1,13 @@
 
+2.4-32 | 2015-07-16 17:21:29 -0700
+
+  * Refactor code to use a common Packet type throught. (Jeff Barber)
+
+  * Extend parsing layer 2 and keeping track of layer 3 protoco. (Jeff Barber)
+
+  * Add a raw_packet() event that generated for all packets and
+    include layer 2 information. (Jeff Barber)
+
 2.4-27 | 2015-07-15 13:31:49 -0700
 
   * Fix race condition in intel test. (Johanna Amann)

VERSION

@@ -1 +1 @@
-2.4-27
+2.4-32

aux/plugins

@@ -1 +1 @@
-Subproject commit 99d7519991b41a970809a99433ea9c7df42e9d93
+Subproject commit f2f4e0a52c65d4ba4eb2673555e9e8f9e424cd41

scripts/base/init-bare.bro

@@ -740,6 +740,7 @@ type pcap_packet: record {
 	caplen: count;	##< The number of bytes captured (<= *len*).
 	len: count;	##< The length of the packet in bytes, including link-level header.
 	data: string;	##< The payload of the packet, including link-level header.
+	link_type: link_encap;	##< Layer 2 link encapsulation type.
 };
 
 ## GeoIP location information.
@@ -1500,6 +1501,33 @@ type pkt_hdr: record {
 	icmp: icmp_hdr &optional;	##< The ICMP header if an ICMP packet.
 };
 
+## Values extracted from the layer 2 header.
+##
+## .. bro:see:: pkt_hdr
+type l2_hdr: record {
+	encap: link_encap;      ##< L2 link encapsulation.
+	len: count;		##< Total frame length on wire.
+	cap_len: count;		##< Captured length.
+	src: string &optional;	##< L2 source (if Ethernet).
+	dst: string &optional;	##< L2 destination (if Ethernet).
+	vlan: count &optional;	##< Outermost VLAN tag if any (and Ethernet).
+	eth_type: count &optional;	##< Innermost Ethertype (if Ethernet).
+	proto: layer3_proto;	##< L3 protocol.
+};
+
+## A raw packet header, consisting of L2 header and everything in
+## :bro:id:`pkt_hdr`. .
+##
+## .. bro:see:: raw_packet pkt_hdr
+type raw_pkt_hdr: record {
+	l2: l2_hdr;			##< The layer 2 header.
+	ip: ip4_hdr &optional;		##< The IPv4 header if an IPv4 packet.
+	ip6: ip6_hdr &optional;		##< The IPv6 header if an IPv6 packet.
+	tcp: tcp_hdr &optional;		##< The TCP header if a TCP packet.
+	udp: udp_hdr &optional;		##< The UDP header if a UDP packet.
+	icmp: icmp_hdr &optional;	##< The ICMP header if an ICMP packet.
+};
+
 ## A Teredo origin indication header.  See :rfc:`4380` for more information
 ## about the Teredo protocol.
 ##

src/CMakeLists.txt

@@ -297,6 +297,7 @@ set(bro_SRCS
     IntSet.cc
     IP.cc
     IPAddr.cc
+    Layer2.cc
     List.cc
     Reporter.cc
     NFA.cc

src/Conn.cc

@@ -241,12 +241,8 @@ void Connection::NextPacket(double t, int is_orig,
 			const u_char*& data,
 			int& record_packet, int& record_content,
 			// arguments for reproducing packets
-			const struct pcap_pkthdr* hdr,
+			const Packet *pkt)
-			const u_char* const pkt,
-			int hdr_size)
 	{
-	current_hdr = hdr;
-	current_hdr_size = hdr_size;
 	current_timestamp = t;
 	current_pkt = pkt;
 
@@ -264,8 +260,6 @@ void Connection::NextPacket(double t, int is_orig,
 	else
 		last_time = t;
 
-	current_hdr = 0;
-	current_hdr_size = 0;
 	current_timestamp = 0;
 	current_pkt = 0;
 	}

src/Conn.h

@@ -86,9 +86,7 @@ public:
 			const u_char*& data,
 			int& record_packet, int& record_content,
 			// arguments for reproducing packets
-			const struct pcap_pkthdr* hdr,
+			const Packet *pkt);
-			const u_char* const pkt,
-			int hdr_size);
 
 	HashKey* Key() const			{ return key; }
 	void ClearKey()				{ key = 0; }

src/IP.cc

@@ -327,24 +327,31 @@ RecordVal* IP_Hdr::BuildIPHdrVal() const
 RecordVal* IP_Hdr::BuildPktHdrVal() const
 	{
 	static RecordType* pkt_hdr_type = 0;
+
+	if ( ! pkt_hdr_type )
+		pkt_hdr_type = internal_type("pkt_hdr")->AsRecordType();
+
+	RecordVal* pkt_hdr = new RecordVal(pkt_hdr_type);
+	return BuildPktHdrVal(pkt_hdr, 0);
+	}
+
+RecordVal* IP_Hdr::BuildPktHdrVal(RecordVal* pkt_hdr, int sindex) const
+	{
 	static RecordType* tcp_hdr_type = 0;
 	static RecordType* udp_hdr_type = 0;
 	static RecordType* icmp_hdr_type = 0;
 
-	if ( ! pkt_hdr_type )
+	if ( ! tcp_hdr_type )
 		{
-		pkt_hdr_type = internal_type("pkt_hdr")->AsRecordType();
 		tcp_hdr_type = internal_type("tcp_hdr")->AsRecordType();
 		udp_hdr_type = internal_type("udp_hdr")->AsRecordType();
 		icmp_hdr_type = internal_type("icmp_hdr")->AsRecordType();
 		}
 
-	RecordVal* pkt_hdr = new RecordVal(pkt_hdr_type);
-
 	if ( ip4 )
-		pkt_hdr->Assign(0, BuildIPHdrVal());
+		pkt_hdr->Assign(sindex + 0, BuildIPHdrVal());
 	else
-		pkt_hdr->Assign(1, BuildIPHdrVal());
+		pkt_hdr->Assign(sindex + 1, BuildIPHdrVal());
 
 	// L4 header.
 	const u_char* data = Payload();
@@ -368,7 +375,7 @@ RecordVal* IP_Hdr::BuildPktHdrVal() const
 		tcp_hdr->Assign(6, new Val(tp->th_flags, TYPE_COUNT));
 		tcp_hdr->Assign(7, new Val(ntohs(tp->th_win), TYPE_COUNT));
 
-		pkt_hdr->Assign(2, tcp_hdr);
+		pkt_hdr->Assign(sindex + 2, tcp_hdr);
 		break;
 		}
 
@@ -381,7 +388,7 @@ RecordVal* IP_Hdr::BuildPktHdrVal() const
 		udp_hdr->Assign(1, new PortVal(ntohs(up->uh_dport), TRANSPORT_UDP));
 		udp_hdr->Assign(2, new Val(ntohs(up->uh_ulen), TYPE_COUNT));
 
-		pkt_hdr->Assign(3, udp_hdr);
+		pkt_hdr->Assign(sindex + 3, udp_hdr);
 		break;
 		}
 
@@ -392,7 +399,7 @@ RecordVal* IP_Hdr::BuildPktHdrVal() const
 
 		icmp_hdr->Assign(0, new Val(icmpp->icmp_type, TYPE_COUNT));
 
-		pkt_hdr->Assign(4, icmp_hdr);
+		pkt_hdr->Assign(sindex + 4, icmp_hdr);
 		break;
 		}
 

src/IP.h

@@ -574,8 +574,13 @@ public:
 	 */
 	RecordVal* BuildPktHdrVal() const;
 
-private:
+	/**
+	 * Same as above, but simply add our values into the record at the
+	 * specified starting index.
+	 */
+	RecordVal* BuildPktHdrVal(RecordVal* pkt_hdr, int sindex) const;
 
+private:
 	const struct ip* ip4;
 	const struct ip6_hdr* ip6;
 	bool del;

src/Net.cc

@@ -62,10 +62,8 @@ double bro_start_network_time;	// timestamp of first packet
 double last_watchdog_proc_time = 0.0;	// value of above during last watchdog
 bool terminating = false;	// whether we're done reading and finishing up
 
-const struct pcap_pkthdr* current_hdr = 0;
+const Packet *current_pkt = 0;
-const u_char* current_pkt = 0;
 int current_dispatched = 0;
-int current_hdr_size = 0;
 double current_timestamp = 0.0;
 iosource::PktSrc* current_pktsrc = 0;
 iosource::IOSource* current_iosrc = 0;
@@ -109,7 +107,7 @@ RETSIGTYPE watchdog(int /* signo */)
 			int frac_pst =
 				int((processing_start_time - int_pst) * 1e6);
 
-			if ( current_hdr )
+			if ( current_pkt )
 				{
 				if ( ! pkt_dumper )
 					{
@@ -126,12 +124,8 @@ RETSIGTYPE watchdog(int /* signo */)
 					}
 
 				if ( pkt_dumper )
-					{
+					pkt_dumper->Dump(current_pkt);
-					iosource::PktDumper::Packet p;
+
-					p.hdr = current_hdr;
-					p.data = current_pkt;
-					pkt_dumper->Dump(&p);
-					}
 				}
 
 			net_get_final_stats();
@@ -240,9 +234,7 @@ void expire_timers(iosource::PktSrc* src_ps)
 				max_timer_expires - current_dispatched);
 	}
 
-void net_packet_dispatch(double t, const struct pcap_pkthdr* hdr,
+void net_packet_dispatch(double t, const Packet* pkt, iosource::PktSrc* src_ps)
-			 const u_char* pkt, int hdr_size,
-			 iosource::PktSrc* src_ps)
 	{
 	if ( ! bro_start_network_time )
 		bro_start_network_time = t;
@@ -278,7 +270,7 @@ void net_packet_dispatch(double t, const struct pcap_pkthdr* hdr,
 			}
 		}
 
-	sessions->DispatchPacket(t, hdr, pkt, hdr_size, src_ps);
+	sessions->DispatchPacket(t, pkt, src_ps);
 	mgr.Drain();
 
 	if ( sp )

src/Net.h

@@ -19,8 +19,7 @@ extern void net_get_final_stats();
 extern void net_finish(int drain_events);
 extern void net_delete();	// Reclaim all memory, etc.
 extern void net_update_time(double new_network_time);
-extern void net_packet_dispatch(double t, const struct pcap_pkthdr* hdr,
+extern void net_packet_dispatch(double t, const Packet* pkt,
-			const u_char* pkt, int hdr_size,
 			iosource::PktSrc* src_ps);
 extern void expire_timers(iosource::PktSrc* src_ps = 0);
 extern void termination_signal();
@@ -74,10 +73,8 @@ extern bool using_communication;
 // Snaplen passed to libpcap.
 extern int snaplen;
 
-extern const struct pcap_pkthdr* current_hdr;
+extern const Packet* current_pkt;
-extern const u_char* current_pkt;
 extern int current_dispatched;
-extern int current_hdr_size;
 extern double current_timestamp;
 extern iosource::PktSrc* current_pktsrc;
 extern iosource::IOSource* current_iosrc;

src/Packet.h

@@ -0,0 +1,98 @@
+#ifndef packet_h
+#define packet_h
+
+#include "Desc.h"
+#include "IP.h"
+
+// A link-layer packet.
+//
+// Note that for serialization we don't use much of the support provided by
+// the serialization framework. Serialize/Unserialize do all the work by
+// themselves. In particular, Packets aren't derived from SerialObj. They are
+// completely seperate and self-contained entities, and we don't need any of
+// the sophisticated features like object caching.
+
+class Packet {
+public:
+	Packet()
+		{
+		struct timeval ts = {0, 0};
+		Init(0, &ts, 0, 0, 0);
+		}
+	// Construct and initialize from packet data.
+	//
+	// arg_free: If true makes an internal copy of the *data*. If false,
+	// stores just a pointer to *data*, which must remain valid.
+	Packet(int arg_link_type, struct timeval *arg_ts, uint32 arg_caplen,
+		uint32 arg_len, const u_char *arg_data, int arg_free = false,
+		std::string arg_tag = std::string(""), uint32 arg_hdrsize = 0,
+		int arg_l3_proto = -1)
+		{
+		Init(arg_link_type, arg_ts, arg_caplen, arg_len, arg_data, arg_free, arg_tag,
+			arg_hdrsize, arg_l3_proto);
+		}
+
+	~Packet()
+		{
+		if ( free )
+			delete [] data;
+		}
+
+	// Initialize with data from pointer.
+	//
+	// arg_free: If true makes an internal copy of the *data*. If false,
+	// stores just a pointer to *data*, which must remain valid.
+	void Init(int arg_link_type, struct timeval *arg_ts, uint32 arg_caplen,
+		uint32 arg_len, const u_char *arg_data, int arg_free = false,
+		std::string arg_tag = std::string(""), uint32 arg_hdrsize = 0,
+		int arg_l3_proto = -1)
+		{
+		link_type = arg_link_type;
+		ts = *arg_ts;
+		cap_len = arg_caplen;
+		len = arg_len;
+		free = arg_free;
+
+		if ( free )
+			{
+			data = new u_char[cap_len];
+			memcpy(const_cast<u_char *>(data), arg_data, cap_len);
+			}
+		else
+			data = arg_data;
+
+		hdr_size = arg_hdrsize;
+		l3_proto = arg_l3_proto;
+		tag = arg_tag;
+		time = ts.tv_sec + double(ts.tv_usec) / 1e6;
+		eth_type = 0;
+		vlan = 0;
+		}
+
+	const IP_Hdr IP() const
+		{ return IP_Hdr((struct ip *) (data + hdr_size), false); }
+
+	void Describe(ODesc* d) const;
+
+	bool Serialize(SerialInfo* info) const;
+	static Packet* Unserialize(UnserialInfo* info);
+
+	std::string tag;		/// Used in serialization
+	double time;			/// Timestamp reconstituted as float
+
+	struct timeval ts;		/// Capture timestamp
+	const u_char* data;		/// Packet data.
+	uint32 link_type;		/// pcap link_type (DLT_EN10MB, DLT_RAW, etc)
+	uint32 cap_len;			/// Captured packet length
+	uint32 len;			/// Actual length on wire
+	uint32 hdr_size;		/// Layer 2 header size
+	uint32 l3_proto;		/// Layer 3 protocol identified (if any)
+	uint32 eth_type;		/// If L2==ethernet, innermost ethertype field
+	uint32 vlan;			/// (Outermost) VLan tag if any, else 0
+
+private:
+	// should we delete associated packet memory upon destruction.
+	bool free;
+};
+
+#endif // packet_h

src/RemoteSerializer.cc

@@ -200,7 +200,7 @@ extern "C" {
 
 // Gets incremented each time there's an incompatible change
 // to the communication internals.
-static const unsigned short PROTOCOL_VERSION = 0x07;
+static const unsigned short PROTOCOL_VERSION = 0x08;
 
 static const char MSG_NONE = 0x00;
 static const char MSG_VERSION = 0x01;
@@ -1453,7 +1453,7 @@ void RemoteSerializer::Process()
 	if ( packets.length() )
 		{
 		BufferedPacket* bp = packets[0];
-		Packet* p = bp->p;
+		const Packet* p = bp->p;
 
 		// FIXME: The following chunk of code is copied from
 		// net_packet_dispatch().  We should change that function
@@ -1465,14 +1465,12 @@ void RemoteSerializer::Process()
 		current_dispatched =
 			tmgr->Advance(network_time, max_timer_expires);
 
-		current_hdr = p->hdr;
+		current_pkt = p;
-		current_pkt = p->pkt;
 		current_pktsrc = 0;
 		current_iosrc = this;
-		sessions->NextPacket(p->time, p->hdr, p->pkt, p->hdr_size);
+		sessions->NextPacket(p->time, p);
 		mgr.Drain();
 
-		current_hdr = 0;	// done with these
 		current_pkt = 0;
 		current_iosrc = 0;
 

src/Serializer.cc

@@ -1133,12 +1133,14 @@ void Packet::Describe(ODesc* d) const
 
 bool Packet::Serialize(SerialInfo* info) const
 	{
-	return SERIALIZE(uint32(hdr->ts.tv_sec)) &&
+	return SERIALIZE(uint32(ts.tv_sec)) &&
-		SERIALIZE(uint32(hdr->ts.tv_usec)) &&
+		SERIALIZE(uint32(ts.tv_usec)) &&
-		SERIALIZE(uint32(hdr->len)) &&
+		SERIALIZE(uint32(len)) &&
-		SERIALIZE(link_type) &&
+		SERIALIZE(uint32(link_type)) &&
-		info->s->Write(tag.c_str(), 0, "tag") &&
+		SERIALIZE(uint32(hdr_size)) &&
-		info->s->Write((const char*) pkt, hdr->caplen, "data");
+		SERIALIZE(uint32(l3_proto)) &&
+		info->s->Write(tag.c_str(), tag.length(), "tag") &&
+		info->s->Write((const char*)data, cap_len, "data");
 	}
 
 static BroFile* profiling_output = 0;
@@ -1149,49 +1151,31 @@ static iosource::PktDumper* dump = 0;
 
 Packet* Packet::Unserialize(UnserialInfo* info)
 	{
-	Packet* p = new Packet("", true);
+	struct timeval ts;
-	pcap_pkthdr* hdr = new pcap_pkthdr;
+	uint32 len, link_type, hdr_size, l3_proto;
 
-	uint32 tv_sec, tv_usec, len;
+	if ( ! (UNSERIALIZE((uint32 *)&ts.tv_sec) &&
-
+		UNSERIALIZE((uint32 *)&ts.tv_usec) &&
-	if ( ! (UNSERIALIZE(&tv_sec) &&
-		UNSERIALIZE(&tv_usec) &&
 		UNSERIALIZE(&len) &&
-		UNSERIALIZE(&p->link_type)) )
+		UNSERIALIZE(&link_type) &&
-		{
+		UNSERIALIZE(&hdr_size) &&
-		delete p;
+		UNSERIALIZE(&l3_proto)) )
-		delete hdr;
 		return 0;
-		}
-
-	hdr->ts.tv_sec = tv_sec;
-	hdr->ts.tv_usec = tv_usec;
-	hdr->len = len;
 
 	char* tag;
 	if ( ! info->s->Read((char**) &tag, 0, "tag") )
-		{
-		delete p;
-		delete hdr;
 		return 0;
-		}
 
-	char* pkt;
+	const u_char* pkt;
 	int caplen;
 	if ( ! info->s->Read((char**) &pkt, &caplen, "data") )
 		{
-		delete p;
-		delete hdr;
 		delete [] tag;
 		return 0;
 		}
 
-	hdr->caplen = uint32(caplen);
+	Packet *p = new Packet(link_type, &ts, caplen, len, pkt, true,
-	p->hdr = hdr;
+			std::string(tag), hdr_size, l3_proto);
-	p->pkt = (u_char*) pkt;
-	p->tag = tag;
-	p->hdr_size = iosource::PktSrc::GetLinkHeaderSize(p->link_type);
-
 	delete [] tag;
 
 	// For the global timer manager, we take the global network_time as the
@@ -1199,7 +1183,7 @@ Packet* Packet::Unserialize(UnserialInfo* info)
 	if ( p->tag == "" )
 		p->time = timer_mgr->Time();
 	else
-		p->time = p->hdr->ts.tv_sec + double(p->hdr->ts.tv_usec) / 1e6;
+		p->time = p->ts.tv_sec + double(p->ts.tv_usec) / 1e6;
 
 	if ( time_machine_profiling )
 		{
@@ -1208,7 +1192,7 @@ Packet* Packet::Unserialize(UnserialInfo* info)
 				new BroFile("tm-prof.packets.log", "w");
 
 		profiling_output->Write(fmt("%.6f %s %d\n", current_time(),
-			(p->tag != "" ? p->tag.c_str() : "-"), hdr->len));
+			(p->tag != "" ? p->tag.c_str() : "-"), p->len));
 		}
 
 #ifdef DEBUG
@@ -1219,10 +1203,7 @@ Packet* Packet::Unserialize(UnserialInfo* info)
 
 		if ( dump )
 			{
-			iosource::PktDumper::Packet dp;
+			dump->Dump(p);
-			dp.hdr = p->hdr;
-			dp.data = p->pkt;
-			dump->Dump(&dp);
 			}
 		}
 #endif

src/Serializer.h

@@ -378,64 +378,6 @@ protected:
 
 };
 
-
-// A link-layer packet.
-//
-// Eventually we should use something like this consistently throughout Bro,
-// replacing the current packet arguments in functions like *::NextPacket().
-// Before doing this, though, we should consider provisioning for packet
-// formats other than just libpcap by designing a more abstract interface.
-//
-// Note that for serialization we don't use much of the support provided by
-// the serialization framework. Serialize/Unserialize do all the work by
-// themselves. In particular, Packets aren't derived from SerialObj. They are
-// completely seperate and self-contained entities, and we don't need any of
-// the sophisticated features like object caching.
-
-class Packet {
-public:
-	// Argument is whether we should delete associatd memory upon
-	// destruction.
-	Packet(TimerMgr::Tag arg_tag, bool arg_free = false)
-		{
-		time = 0.0;
-		hdr = 0;
-		pkt = 0;
-		hdr_size = 0;
-		free = arg_free;
-		tag = arg_tag;
-		link_type = 0;
-		}
-
-	~Packet()
-		{
-		if ( free )
-			{
-			delete hdr;
-			delete [] pkt;
-			}
-		}
-
-	const IP_Hdr IP() const
-		{ return IP_Hdr((struct ip *) (pkt + hdr_size), true); }
-
-	void Describe(ODesc* d) const;
-
-	bool Serialize(SerialInfo* info) const;
-	static Packet* Unserialize(UnserialInfo* info);
-
-	const struct pcap_pkthdr* hdr;
-	const u_char* pkt;
-	TimerMgr::Tag tag;
-	uint32 link_type;
-
-	double time;
-	int hdr_size;
-
-private:
-	bool free;
-};
-
 extern FileSerializer* event_serializer;
 extern FileSerializer* state_serializer;
 

src/Sessions.cc

@@ -15,6 +15,7 @@
 #include "Sessions.h"
 #include "Reporter.h"
 #include "OSFinger.h"
+#include "Layer2.h"
 
 #include "analyzer/protocol/icmp/ICMP.h"
 #include "analyzer/protocol/udp/UDP.h"
@@ -165,98 +166,86 @@ void NetSessions::Done()
 	{
 	}
 
-void NetSessions::DispatchPacket(double t, const struct pcap_pkthdr* hdr,
+void NetSessions::DispatchPacket(double t, const Packet* pkt,
-			const u_char* pkt, int hdr_size,
 			iosource::PktSrc* src_ps)
 	{
-	const struct ip* ip_hdr = 0;
+	NextPacket(t, pkt);
-	const u_char* ip_data = 0;
+	}
-	int proto = 0;
 
-	if ( hdr->caplen >= hdr_size + sizeof(struct ip) )
+void NetSessions::NextPacket(double t, const Packet* pkt)
 	{
-		ip_hdr = reinterpret_cast<const struct ip*>(pkt + hdr_size);
+	if ( raw_packet )
-		if ( hdr->caplen >= unsigned(hdr_size + (ip_hdr->ip_hl << 2)) )
+		{
-			ip_data = pkt + hdr_size + (ip_hdr->ip_hl << 2);
+		val_list* vl = new val_list();
+		L2_Hdr l2_hdr(pkt);
+		vl->append(l2_hdr.BuildPktHdrVal());
+		mgr.QueueEvent(raw_packet, vl);
 		}
 
-	if ( encap_hdr_size > 0 && ip_data )
+	ProcNextPacket(t, pkt);
-		// Blanket encapsulation
-		hdr_size += encap_hdr_size;
-
-	NextPacket(t, hdr, pkt, hdr_size);
 	}
 
-void NetSessions::NextPacket(double t, const struct pcap_pkthdr* hdr,
+void NetSessions::ProcNextPacket(double t, const Packet *pkt)
-			     const u_char* const pkt, int hdr_size)
 	{
 	SegmentProfiler(segment_logger, "processing-packet");
 	if ( pkt_profiler )
-		pkt_profiler->ProfilePkt(t, hdr->caplen);
+		pkt_profiler->ProfilePkt(t, pkt->cap_len);
 
 	++num_packets_processed;
 
 	dump_this_packet = 0;
 
 	if ( record_all_packets )
-		DumpPacket(hdr, pkt);
+		DumpPacket(pkt);
 
-	// ### The following isn't really correct.  What we *should*
+	if ( pkt->hdr_size > pkt->cap_len )
-	// do is understanding the different link layers in order to
-	// find the network-layer protocol ID.  That's a big
-	// portability pain, though, unless we just assume everything's
-	// Ethernet .... not great, given the potential need to deal
-	// with PPP or FDDI (for some older traces).  So instead
-	// we look to see if what we have is consistent with an
-	// IPv4 packet.  If not, it's either ARP or IPv6 or weird.
-
-	if ( hdr_size > static_cast<int>(hdr->caplen) )
 		{
-		Weird("truncated_link_frame", hdr, pkt);
+		Weird("truncated_link_frame", pkt);
 		return;
 		}
 
-	uint32 caplen = hdr->caplen - hdr_size;
+	uint32 caplen = pkt->cap_len - pkt->hdr_size;
+
+	if ( pkt->l3_proto == AF_INET )
+		{
 		if ( caplen < sizeof(struct ip) )
 			{
-		Weird("truncated_IP", hdr, pkt);
+			Weird("truncated_IP", pkt);
 			return;
 			}
 
-	const struct ip* ip = (const struct ip*) (pkt + hdr_size);
+		const struct ip* ip = (const struct ip*) (pkt->data + pkt->hdr_size);
-
-	if ( ip->ip_v == 4 )
-		{
 		IP_Hdr ip_hdr(ip, false);
-		DoNextPacket(t, hdr, &ip_hdr, pkt, hdr_size, 0);
+		DoNextPacket(t, pkt, &ip_hdr, 0);
 		}
 
-	else if ( ip->ip_v == 6 )
+	else if ( pkt->l3_proto == AF_INET6 )
 		{
 		if ( caplen < sizeof(struct ip6_hdr) )
 			{
-			Weird("truncated_IP", hdr, pkt);
+			Weird("truncated_IP", pkt);
 			return;
 			}
 
-		IP_Hdr ip_hdr((const struct ip6_hdr*) (pkt + hdr_size), false, caplen);
+		IP_Hdr ip_hdr((const struct ip6_hdr*) (pkt->data + pkt->hdr_size), false, caplen);
-		DoNextPacket(t, hdr, &ip_hdr, pkt, hdr_size, 0);
+		DoNextPacket(t, pkt, &ip_hdr, 0);
 		}
 
-	else if ( analyzer::arp::ARP_Analyzer::IsARP(pkt, hdr_size) )
+	else if ( analyzer::arp::ARP_Analyzer::IsARP(pkt) )
 		{
 		if ( arp_analyzer )
-			arp_analyzer->NextPacket(t, hdr, pkt, hdr_size);
+			arp_analyzer->NextPacket(t, pkt);
 		}
 
 	else
 		{
-		Weird("unknown_packet_type", hdr, pkt);
+		Weird("unknown_packet_type", pkt);
 		return;
 		}
 
+
 	if ( dump_this_packet && ! record_all_packets )
-		DumpPacket(hdr, pkt);
+		DumpPacket(pkt);
 	}
 
 int NetSessions::CheckConnectionTag(Connection* conn)
@@ -337,26 +326,25 @@ static unsigned int gre_header_len(uint16 flags)
 	return len;
 	}
 
-void NetSessions::DoNextPacket(double t, const struct pcap_pkthdr* hdr,
+void NetSessions::DoNextPacket(double t, const Packet* pkt, const IP_Hdr* ip_hdr,
-				const IP_Hdr* ip_hdr, const u_char* const pkt,
+			       const EncapsulationStack* encapsulation)
-				int hdr_size, const EncapsulationStack* encapsulation)
 	{
-	uint32 caplen = hdr->caplen - hdr_size;
+	uint32 caplen = pkt->cap_len - pkt->hdr_size;
 	const struct ip* ip4 = ip_hdr->IP4_Hdr();
 
 	uint32 len = ip_hdr->TotalLen();
 	if ( len == 0 )
 		{
 		// TCP segmentation offloading can zero out the ip_len field.
-		Weird("ip_hdr_len_zero", hdr, pkt, encapsulation);
+		Weird("ip_hdr_len_zero", pkt, encapsulation);
 
 		// Cope with the zero'd out ip_len field by using the caplen.
-		len = hdr->caplen - hdr_size;
+		len = pkt->cap_len - pkt->hdr_size;
 		}
 
-	if ( hdr->len < len + hdr_size )
+	if ( pkt->len < len + pkt->hdr_size )
 		{
-		Weird("truncated_IP", hdr, pkt, encapsulation);
+		Weird("truncated_IP", pkt, encapsulation);
 		return;
 		}
 
@@ -368,7 +356,7 @@ void NetSessions::DoNextPacket(double t, const struct pcap_pkthdr* hdr,
 	if ( ! ignore_checksums && ip4 &&
 	     ones_complement_checksum((void*) ip4, ip_hdr_len, 0) != 0xffff )
 		{
-		Weird("bad_IP_checksum", hdr, pkt, encapsulation);
+		Weird("bad_IP_checksum", pkt, encapsulation);
 		return;
 		}
 
@@ -393,7 +381,7 @@ void NetSessions::DoNextPacket(double t, const struct pcap_pkthdr* hdr,
 			}
 		else
 			{
-			f = NextFragment(t, ip_hdr, pkt + hdr_size);
+			f = NextFragment(t, ip_hdr, pkt->data + pkt->hdr_size);
 			const IP_Hdr* ih = f->ReassembledPkt();
 			if ( ! ih )
 				// It didn't reassemble into anything yet.
@@ -437,7 +425,7 @@ void NetSessions::DoNextPacket(double t, const struct pcap_pkthdr* hdr,
 
 		if ( ! ignore_checksums && mobility_header_checksum(ip_hdr) != 0xffff )
 			{
-			Weird("bad_MH_checksum", hdr, pkt, encapsulation);
+			Weird("bad_MH_checksum", pkt, encapsulation);
 			return;
 			}
 
@@ -449,7 +437,7 @@ void NetSessions::DoNextPacket(double t, const struct pcap_pkthdr* hdr,
 			}
 
 		if ( ip_hdr->NextProto() != IPPROTO_NONE )
-			Weird("mobility_piggyback", hdr, pkt, encapsulation);
+			Weird("mobility_piggyback", pkt, encapsulation);
 
 		return;
 		}
@@ -457,7 +445,7 @@ void NetSessions::DoNextPacket(double t, const struct pcap_pkthdr* hdr,
 
 	int proto = ip_hdr->NextProto();
 
-	if ( CheckHeaderTrunc(proto, len, caplen, hdr, pkt, encapsulation) )
+	if ( CheckHeaderTrunc(proto, len, caplen, pkt, encapsulation) )
 		return;
 
 	const u_char* data = ip_hdr->Payload();
@@ -664,7 +652,7 @@ void NetSessions::DoNextPacket(double t, const struct pcap_pkthdr* hdr,
 		else
 			it->second.second = network_time;
 
-		DoNextInnerPacket(t, hdr, inner, encapsulation,
+		DoNextInnerPacket(t, pkt, inner, encapsulation,
 		                  ip_tunnels[tunnel_idx].first);
 
 		return;
@@ -677,13 +665,13 @@ void NetSessions::DoNextPacket(double t, const struct pcap_pkthdr* hdr,
 		// not sure the reason for the No Next header in the packet.
 		if ( ! ( encapsulation &&
 		     encapsulation->LastType() == BifEnum::Tunnel::TEREDO ) )
-			Weird("ipv6_no_next", hdr, pkt);
+			Weird("ipv6_no_next", pkt);
 
 		return;
 		}
 
 	default:
-		Weird(fmt("unknown_protocol_%d", proto), hdr, pkt, encapsulation);
+		Weird(fmt("unknown_protocol_%d", proto), pkt, encapsulation);
 		return;
 	}
 
@@ -756,8 +744,7 @@ void NetSessions::DoNextPacket(double t, const struct pcap_pkthdr* hdr,
 		        pkt_hdr_val ? pkt_hdr_val->Ref() : ip_hdr->BuildPktHdrVal());
 
 	conn->NextPacket(t, is_orig, ip_hdr, len, caplen, data,
-				record_packet, record_content,
+				record_packet, record_content, pkt);
-			        hdr, pkt, hdr_size);
 
 	if ( f )
 		{
@@ -772,40 +759,53 @@ void NetSessions::DoNextPacket(double t, const struct pcap_pkthdr* hdr,
 
 		else
 			{
-			int hdr_len = data - pkt;
+			int hdr_len = data - pkt->data;
-			DumpPacket(hdr, pkt, hdr_len);	// just save the header
+			DumpPacket(pkt, hdr_len);	// just save the header
 			}
 		}
 	}
 
-void NetSessions::DoNextInnerPacket(double t, const struct pcap_pkthdr* hdr,
+void NetSessions::DoNextInnerPacket(double t, const Packet* pkt,
 		const IP_Hdr* inner, const EncapsulationStack* prev,
 		const EncapsulatingConn& ec)
 	{
-	struct pcap_pkthdr fake_hdr;
+	uint32 caplen, len;
-	fake_hdr.caplen = fake_hdr.len = inner->TotalLen();
+	caplen = len = inner->TotalLen();
 
-	if ( hdr )
+	struct timeval ts;
-		fake_hdr.ts = hdr->ts;
+	int link_type;
+	int l3_proto;
+
+	if ( pkt )
+		ts = pkt->ts;
 	else
 		{
-		fake_hdr.ts.tv_sec = (time_t) network_time;
+		ts.tv_sec = (time_t) network_time;
-		fake_hdr.ts.tv_usec = (suseconds_t)
+		ts.tv_usec = (suseconds_t)
-		    ((network_time - (double)fake_hdr.ts.tv_sec) * 1000000);
+		    ((network_time - (double)ts.tv_sec) * 1000000);
 		}
 
-	const u_char* pkt = 0;
+	const u_char* data = 0;
 
 	if ( inner->IP4_Hdr() )
-		pkt = (const u_char*) inner->IP4_Hdr();
+		{
+		data = (const u_char*) inner->IP4_Hdr();
+		l3_proto = AF_INET;
+		}
 	else
-		pkt = (const u_char*) inner->IP6_Hdr();
+		{
+		data = (const u_char*) inner->IP6_Hdr();
+		l3_proto = AF_INET6;
+		}
 
 	EncapsulationStack* outer = prev ?
 			new EncapsulationStack(*prev) : new EncapsulationStack();
 	outer->Add(ec);
 
-	DoNextPacket(t, &fake_hdr, inner, pkt, 0, outer);
+	// Construct fake packet for DoNextPacket
+	Packet p;
+	p.Init(DLT_RAW, &ts, caplen, len, data, false, "", 0, l3_proto);
+	DoNextPacket(t, &p, inner, outer);
 
 	delete inner;
 	delete outer;
@@ -843,8 +843,7 @@ int NetSessions::ParseIPPacket(int caplen, const u_char* const pkt, int proto,
 	}
 
 bool NetSessions::CheckHeaderTrunc(int proto, uint32 len, uint32 caplen,
-                                   const struct pcap_pkthdr* h,
+                                   const Packet* p, const EncapsulationStack* encap)
-                                   const u_char* p, const EncapsulationStack* encap)
 	{
 	uint32 min_hdr_len = 0;
 	switch ( proto ) {
@@ -876,13 +875,13 @@ bool NetSessions::CheckHeaderTrunc(int proto, uint32 len, uint32 caplen,
 
 	if ( len < min_hdr_len )
 		{
-		Weird("truncated_header", h, p, encap);
+		Weird("truncated_header", p, encap);
 		return true;
 		}
 
 	if ( caplen < min_hdr_len )
 		{
-		Weird("internally_truncated_header", h, p, encap);
+		Weird("internally_truncated_header", p, encap);
 		return true;
 		}
 
@@ -1387,45 +1386,26 @@ void NetSessions::ExpireTimerMgrs()
 		}
 	}
 
-void NetSessions::DumpPacket(const struct pcap_pkthdr* hdr,
+void NetSessions::DumpPacket(const Packet *pkt, int len)
-				const u_char* pkt, int len)
 	{
 	if ( ! pkt_dumper )
 		return;
 
-	if ( len == 0 )
+	if ( len != 0 )
 		{
-		iosource::PktDumper::Packet p;
+		if ( (uint32)len > pkt->cap_len )
-		p.hdr = hdr;
+			reporter->Warning("bad modified caplen");
-		p.data = pkt;
-		pkt_dumper->Dump(&p);
-		}
-
 		else
-		{
+			const_cast<Packet *>(pkt)->cap_len = len;
-		struct pcap_pkthdr h = *hdr;
-		h.caplen = len;
-		if ( h.caplen > hdr->caplen )
-			reporter->InternalError("bad modified caplen");
-
-		iosource::PktDumper::Packet p;
-		p.hdr = &h;
-		p.data = pkt;
-		pkt_dumper->Dump(&p);
-		}
 		}
 
-void NetSessions::Internal(const char* msg, const struct pcap_pkthdr* hdr,
+	pkt_dumper->Dump(pkt);
-				const u_char* pkt)
-	{
-	DumpPacket(hdr, pkt);
-	reporter->InternalError("%s", msg);
 	}
 
-void NetSessions::Weird(const char* name, const struct pcap_pkthdr* hdr,
+void NetSessions::Weird(const char* name, const Packet* pkt,
-                        const u_char* pkt, const EncapsulationStack* encap)
+                        const EncapsulationStack* encap)
 	{
-	if ( hdr )
+	if ( pkt )
 		dump_this_packet = 1;
 
 	if ( encap && encap->LastType() != BifEnum::Tunnel::NONE )

src/Sessions.h

@@ -15,8 +15,6 @@
 
 #include <utility>
 
-struct pcap_pkthdr;
-
 class EncapsulationStack;
 class Connection;
 class OSFingerprint;
@@ -71,8 +69,7 @@ public:
 	// Main entry point for packet processing. Dispatches the packet
 	// either through NextPacket(), optionally employing the packet
 	// sorter first.
-	void DispatchPacket(double t, const struct pcap_pkthdr* hdr,
+	void DispatchPacket(double t, const Packet* pkt,
-			const u_char* const pkt, int hdr_size,
 			iosource::PktSrc* src_ps);
 
 	void Done();	// call to drain events before destructing
@@ -106,8 +103,8 @@ public:
 
 	void GetStats(SessionStats& s) const;
 
-	void Weird(const char* name, const struct pcap_pkthdr* hdr,
+	void Weird(const char* name, const Packet* pkt,
-	    const u_char* pkt, const EncapsulationStack* encap = 0);
+	    const EncapsulationStack* encap = 0);
 	void Weird(const char* name, const IP_Hdr* ip,
 	    const EncapsulationStack* encap = 0);
 
@@ -133,9 +130,8 @@ public:
 			icmp_conns.Length();
 		}
 
-	void DoNextPacket(double t, const struct pcap_pkthdr* hdr,
+	void DoNextPacket(double t, const Packet *pkt, const IP_Hdr* ip_hdr,
-			const IP_Hdr* ip_hdr, const u_char* const pkt,
+			const EncapsulationStack* encapsulation);
-			int hdr_size, const EncapsulationStack* encapsulation);
 
 	/**
 	 * Wrapper that recurses on DoNextPacket for encapsulated IP packets.
@@ -151,7 +147,7 @@ public:
 	 *        the most-recently found depth of encapsulation.
 	 * @param ec The most-recently found depth of encapsulation.
 	 */
-	void DoNextInnerPacket(double t, const struct pcap_pkthdr* hdr,
+	void DoNextInnerPacket(double t, const Packet *pkt,
 	                      const IP_Hdr* inner, const EncapsulationStack* prev,
 	                      const EncapsulatingConn& ec);
 
@@ -218,24 +214,20 @@ protected:
 				TransportProto transport_proto,
 				uint8 tcp_flags, bool& flip_roles);
 
-	void NextPacket(double t, const struct pcap_pkthdr* hdr,
+	void NextPacket(double t, const Packet* pkt);
-			const u_char* const pkt, int hdr_size);
+
+	void ProcNextPacket(double t, const Packet *pkt);
 
 	// Record the given packet (if a dumper is active).  If len=0
 	// then the whole packet is recorded, otherwise just the first
 	// len bytes.
-	void DumpPacket(const struct pcap_pkthdr* hdr, const u_char* pkt,
+	void DumpPacket(const Packet *pkt, int len=0);
-			int len=0);
-
-	void Internal(const char* msg, const struct pcap_pkthdr* hdr,
-			const u_char* pkt);
 
 	// For a given protocol, checks whether the header's length as derived
 	// from lower-level headers or the length actually captured is less
 	// than that protocol's minimum header size.
 	bool CheckHeaderTrunc(int proto, uint32 len, uint32 caplen,
-			      const struct pcap_pkthdr* hdr, const u_char* pkt,
+			      const Packet *pkt, const EncapsulationStack* encap);
-			      const EncapsulationStack* encap);
 
 	CompositeHash* ch;
 	PDict(Connection) tcp_conns;

src/analyzer/protocol/arp/ARP.cc

@@ -19,10 +19,14 @@ ARP_Analyzer::~ARP_Analyzer()
 	{
 	}
 
-bool ARP_Analyzer::IsARP(const u_char* pkt, int hdr_size)
+// Assumes pkt->hdr_size indicates size of the Layer 2 header.
+bool ARP_Analyzer::IsARP(const Packet* pkt)
 	{
+	if ( pkt->hdr_size < 2 )
+		return false;
+
 	unsigned short network_protocol =
-		*(unsigned short*) (pkt + hdr_size - 2);
+		*(unsigned short*) (pkt->data + pkt->hdr_size - 2);
 
 	switch ( ntohs(network_protocol) ) {
 	case ETHERTYPE_ARP:
@@ -93,16 +97,16 @@ bool ARP_Analyzer::IsARP(const u_char* pkt, int hdr_size)
 #endif
 
 
-void ARP_Analyzer::NextPacket(double t, const struct pcap_pkthdr* hdr,
+void ARP_Analyzer::NextPacket(double t, const Packet* pkt)
-				const u_char* const pkt, int hdr_size)
 	{
+	const u_char *data = pkt->data;
 	// Check whether the packet is OK ("inspired" in tcpdump's print-arp.c).
 	const struct arp_pkthdr* ah =
-		(const struct arp_pkthdr*) (pkt + hdr_size);
+		(const struct arp_pkthdr*) (data + pkt->hdr_size);
 
 	// Check the size.
-	int min_length = (ar_tpa(ah) - (char*) (pkt + hdr_size)) + ah->ar_pln;
+	int min_length = (ar_tpa(ah) - (char*) (data + pkt->hdr_size)) + ah->ar_pln;
-	int real_length = hdr->caplen - hdr_size;
+	int real_length = pkt->cap_len - pkt->hdr_size;
 	if ( min_length > real_length )
 		{
 		Corrupted("truncated_ARP");
@@ -158,7 +162,7 @@ void ARP_Analyzer::NextPacket(double t, const struct pcap_pkthdr* hdr,
 
 
 	// Check MAC src address = ARP sender MAC address.
-	if ( memcmp((const char*) (pkt+6), ar_sha(ah), ah->ar_hln) )
+	if ( memcmp((const char*) (data+6), ar_sha(ah), ah->ar_hln) )
 		{
 		BadARP(ah, "weird-arp-sha");
 		return;
@@ -167,12 +171,12 @@ void ARP_Analyzer::NextPacket(double t, const struct pcap_pkthdr* hdr,
 	// Check the code is supported.
 	switch ( ntohs(ah->ar_op) ) {
 	case ARPOP_REQUEST:
-		RREvent(arp_request, pkt+6, pkt,
+		RREvent(arp_request, data+6, data,
 				ar_spa(ah), ar_sha(ah), ar_tpa(ah), ar_tha(ah));
 		break;
 
 	case ARPOP_REPLY:
-		RREvent(arp_reply, pkt+6, pkt,
+		RREvent(arp_reply, data+6, data,
 				ar_spa(ah), ar_sha(ah), ar_tpa(ah), ar_tha(ah));
 		break;
 

src/analyzer/protocol/arp/ARP.h

@@ -24,6 +24,7 @@
 #endif
 
 #include "NetVar.h"
+#include "Packet.h"
 
 extern "C" {
 #include <pcap.h>
@@ -36,8 +37,7 @@ public:
 	ARP_Analyzer();
 	virtual ~ARP_Analyzer();
 
-	void NextPacket(double t, const struct pcap_pkthdr* hdr,
+	void NextPacket(double t, const Packet* pkt);
-			const u_char* const pkt, int hdr_size);
 
 	void Describe(ODesc* d) const;
 	void RREvent(EventHandlerPtr e, const u_char* src, const u_char* dst,
@@ -45,7 +45,7 @@ public:
 			const char* tpa, const char* tha);
 
 	// Whether a packet is of interest for ARP analysis.
-	static bool IsARP(const u_char* pkt, int hdr_size);
+	static bool IsARP(const Packet *pkt);
 
 protected:
 	AddrVal* ConstructAddrVal(const void* addr);

src/bro.bif

@@ -20,6 +20,7 @@
 #include "Reporter.h"
 #include "IPAddr.h"
 #include "util.h"
+#include "Packet.h"
 #include "file_analysis/Manager.h"
 #include "iosource/Manager.h"
 
@@ -3238,11 +3239,10 @@ const char* conn_id_string(Val* c)
 ## .. bro:see:: dump_packet get_current_packet send_current_packet
 function dump_current_packet%(file_name: string%) : bool
 	%{
-	const struct pcap_pkthdr* hdr;
+	const Packet* pkt;
-	const u_char* pkt;
 
 	if ( ! current_pktsrc ||
-	     ! current_pktsrc->GetCurrentPacket(&hdr, &pkt) )
+	     ! current_pktsrc->GetCurrentPacket(&pkt) )
 		return new Val(0, TYPE_BOOL);
 
 	if ( ! addl_pkt_dumper )
@@ -3250,13 +3250,10 @@ function dump_current_packet%(file_name: string%) : bool
 
 	if ( addl_pkt_dumper )
 		{
-		iosource::PktDumper::Packet p;
+		addl_pkt_dumper->Dump(pkt);
-		p.hdr = hdr;
-		p.data = pkt;
-		addl_pkt_dumper->Dump(&p);
 		}
 
-	return new Val(! addl_pkt_dumper->IsError(), TYPE_BOOL);
+	return new Val( addl_pkt_dumper && ! addl_pkt_dumper->IsError(), TYPE_BOOL);
 	%}
 
 ## Returns the currently processed PCAP packet.
@@ -3267,26 +3264,27 @@ function dump_current_packet%(file_name: string%) : bool
 ## .. bro:see:: dump_current_packet dump_packet send_current_packet
 function get_current_packet%(%) : pcap_packet
 	%{
-	const struct pcap_pkthdr* hdr;
+	const Packet* p;
-	const u_char* data;
 	RecordVal* pkt = new RecordVal(pcap_packet);
 
 	if ( ! current_pktsrc ||
-	     ! current_pktsrc->GetCurrentPacket(&hdr, &data) )
+	     ! current_pktsrc->GetCurrentPacket(&p) )
 		{
 		pkt->Assign(0, new Val(0, TYPE_COUNT));
 		pkt->Assign(1, new Val(0, TYPE_COUNT));
 		pkt->Assign(2, new Val(0, TYPE_COUNT));
 		pkt->Assign(3, new Val(0, TYPE_COUNT));
 		pkt->Assign(4, new StringVal(""));
+		pkt->Assign(5, new EnumVal(BifEnum::LINK_UNKNOWN, BifType::Enum::link_encap));
 		return pkt;
 		}
 
-	pkt->Assign(0, new Val(uint32(hdr->ts.tv_sec), TYPE_COUNT));
+	pkt->Assign(0, new Val(uint32(p->ts.tv_sec), TYPE_COUNT));
-	pkt->Assign(1, new Val(uint32(hdr->ts.tv_usec), TYPE_COUNT));
+	pkt->Assign(1, new Val(uint32(p->ts.tv_usec), TYPE_COUNT));
-	pkt->Assign(2, new Val(hdr->caplen, TYPE_COUNT));
+	pkt->Assign(2, new Val(p->cap_len, TYPE_COUNT));
-	pkt->Assign(3, new Val(hdr->len, TYPE_COUNT));
+	pkt->Assign(3, new Val(p->len, TYPE_COUNT));
-	pkt->Assign(4, new StringVal(hdr->caplen, (const char*) data));
+	pkt->Assign(4, new StringVal(p->cap_len, (const char*)p->data));
+	pkt->Assign(5, new EnumVal(p->link_type, BifType::Enum::link_encap));
 
 	return pkt;
 	%}
@@ -3302,26 +3300,29 @@ function get_current_packet%(%) : pcap_packet
 ## .. bro:see:: get_current_packet dump_current_packet send_current_packet
 function dump_packet%(pkt: pcap_packet, file_name: string%) : bool
 	%{
-	struct pcap_pkthdr hdr;
-	const val_list* pkt_vl = pkt->AsRecord();
-
-	hdr.ts.tv_sec = (*pkt_vl)[0]->AsCount();
-	hdr.ts.tv_usec = (*pkt_vl)[1]->AsCount();
-	hdr.caplen = (*pkt_vl)[2]->AsCount();
-	hdr.len = (*pkt_vl)[3]->AsCount();
-
 	if ( ! addl_pkt_dumper )
 		addl_pkt_dumper = iosource_mgr->OpenPktDumper(file_name->CheckString(), true);
 
 	if ( addl_pkt_dumper )
 		{
-		iosource::PktDumper::Packet p;
+		struct timeval ts;
-		p.hdr = &hdr;
+		uint32 caplen, len, link_type;
-		p.data = (*pkt_vl)[4]->AsString()->Bytes();
+		u_char *data;
+
+		const val_list* pkt_vl = pkt->AsRecord();
+
+		ts.tv_sec = (*pkt_vl)[0]->AsCount();
+		ts.tv_usec = (*pkt_vl)[1]->AsCount();
+		caplen = (*pkt_vl)[2]->AsCount();
+		len = (*pkt_vl)[3]->AsCount();
+		data = (*pkt_vl)[4]->AsString()->Bytes();
+		link_type = (*pkt_vl)[5]->AsEnum();
+		Packet p(link_type, &ts, caplen, len, data, true);
+
 		addl_pkt_dumper->Dump(&p);
 		}
 
-	return new Val(addl_pkt_dumper->IsError(), TYPE_BOOL);
+	return new Val(addl_pkt_dumper && ! addl_pkt_dumper->IsError(), TYPE_BOOL);
 	%}
 
 %%{
@@ -4800,20 +4801,16 @@ function send_ping%(p: event_peer, seq: count%) : bool
 ##              dump_packet dump_current_packet get_current_packet
 function send_current_packet%(p: event_peer%) : bool
 	%{
-	Packet pkt("");
+	const Packet* pkt;
 
 	if ( ! current_pktsrc ||
-	     ! current_pktsrc->GetCurrentPacket(&pkt.hdr, &pkt.pkt) )
+	     ! current_pktsrc->GetCurrentPacket(&pkt) )
 		return new Val(0, TYPE_BOOL);
 
 	RemoteSerializer::PeerID id = p->AsRecordVal()->Lookup(0)->AsCount();
 
-	pkt.time = pkt.hdr->ts.tv_sec + double(pkt.hdr->ts.tv_usec) / 1e6;
-	pkt.hdr_size = current_pktsrc->HdrSize();
-	pkt.link_type = current_pktsrc->LinkType();
-
 	SerialInfo info(remote_serializer);
-	return new Val(remote_serializer->SendPacket(&info, id, pkt), TYPE_BOOL);
+	return new Val(remote_serializer->SendPacket(&info, id, *pkt), TYPE_BOOL);
 	%}
 
 ## Returns the peer who generated the last event.

src/event.bif

@@ -225,17 +225,31 @@ event udp_session_done%(u: connection%);
 ##    ``ANALYZER_*`` constants right now.
 event scheduled_analyzer_applied%(c: connection, a: Analyzer::Tag%);
 
-## Generated for every packet Bro sees. This is a very low-level and expensive
+## Generated for every packet Bro sees that has a valid link-layer header. This
-## event that should be avoided when at all possible. It's usually infeasible to
+## is a very very low-level and expensive event that should be avoided when at all
-## handle when processing even medium volumes of traffic in real-time. That
+## possible. It's usually infeasible to handle when processing even medium volumes
-## said, if you work from a trace and want to do some packet-level analysis,
+## of traffic in real-time. That said, if you work from a trace and want to do some
-## it may come in handy.
+## packet-level analysis, it may come in handy. 
+##
+## p: Information from the header of the packet that triggered the event.
+##
+## .. bro:see:: new_packet packet_contents
+event raw_packet%(p: raw_pkt_hdr%);
+
+## Generated for all packets that make it into Bro's connection processing . In
+## contrast to :bro:id:`raw_packet` this filters out some packets that, e.g., don't
+## pass certain sanity checks. 
+## 
+## This is a very low-level and expensive event that should be avoided when at all
+## possible. It's usually infeasible to handle when processing even medium volumes
+## of traffic in real-time. That said, if you work from a trace and want to do some
+## packet-level analysis, it may come in handy.
 ##
 ## c: The connection the packet is part of.
 ##
 ## p: Information from the header of the packet that triggered the event.
 ##
-## .. bro:see:: tcp_packet packet_contents
+## .. bro:see:: tcp_packet packet_contents raw_packet
 event new_packet%(c: connection, p: pkt_hdr%);
 
 ## Generated for every IPv6 packet that contains extension headers.

src/iosource/PktDumper.h

@@ -3,6 +3,7 @@
 #ifndef IOSOURCE_PKTSRC_PKTDUMPER_H
 #define IOSOURCE_PKTSRC_PKTDUMPER_H
 
+#include "Packet.h"
 #include "IOSource.h"
 
 namespace iosource {
@@ -12,21 +13,6 @@ namespace iosource {
  */
 class PktDumper {
 public:
-	/**
-	 * Structure describing a packet.
-	 */
-	struct Packet {
-		/**
-		 * The pcap header associated with the packet.
-		 */
-		const struct pcap_pkthdr* hdr;
-
-		/**
-		 * The full content of the packet.
-		 */
-		const unsigned char* data;
-	};
-
 	/**
 	 * Constructor.
 	 */

src/iosource/PktSrc.cc

@@ -147,7 +147,7 @@ void PktSrc::Info(const std::string& msg)
 
 void PktSrc::Weird(const std::string& msg, const Packet* p)
 	{
-	sessions->Weird(msg.c_str(), p->hdr, p->data, 0);
+	sessions->Weird(msg.c_str(), p, 0);
 	}
 
 void PktSrc::InternalError(const std::string& msg)
@@ -197,20 +197,20 @@ double PktSrc::CheckPseudoTime()
 
 	if ( remote_trace_sync_interval )
 		{
-		if ( next_sync_point == 0 || current_packet.ts >= next_sync_point )
+		if ( next_sync_point == 0 || current_packet.time >= next_sync_point )
 			{
 			int n = remote_serializer->SendSyncPoint();
 			next_sync_point = first_timestamp +
 						n * remote_trace_sync_interval;
 			remote_serializer->Log(RemoteSerializer::LogInfo,
 				fmt("stopping at packet %.6f, next sync-point at %.6f",
-					current_packet.ts, next_sync_point));
+					current_packet.time, next_sync_point));
 
 			return 0;
 			}
 		}
 
-	double pseudo_time = current_packet.ts - first_timestamp;
+	double pseudo_time = current_packet.time - first_timestamp;
 	double ct = (current_time(true) - first_wallclock) * pseudo_realtime;
 
 	return pseudo_time <= ct ? bro_start_time + pseudo_time : 0;
@@ -273,7 +273,7 @@ double PktSrc::NextTimestamp(double* local_network_time)
 		return -1.0;
 		}
 
-	return current_packet.ts;
+	return current_packet.time;
 	}
 
 void PktSrc::Process()
@@ -284,20 +284,22 @@ void PktSrc::Process()
 	if ( ! ExtractNextPacketInternal() )
 		return;
 
-	int pkt_hdr_size = props.hdr_size;
-
 	// Unfortunately some packets on the link might have MPLS labels
 	// while others don't. That means we need to ask the link-layer if
 	// labels are in place.
 	bool have_mpls = false;
 
+	int l3_proto = 0;
 	int protocol = 0;
 	const u_char* data = current_packet.data;
 
+	current_packet.link_type = props.link_type;
+
 	switch ( props.link_type ) {
 	case DLT_NULL:
 		{
 		protocol = (data[3] << 24) + (data[2] << 16) + (data[1] << 8) + data[0];
+		data += GetLinkHeaderSize(props.link_type);
 
 		// From the Wireshark Wiki: "AF_INET6, unfortunately, has
 		// different values in {NetBSD,OpenBSD,BSD/OS},
@@ -306,16 +308,16 @@ void PktSrc::Process()
 		// as the AF_ value." As we may be reading traces captured on
 		// platforms other than what we're running on, we accept them
 		// all here.
-		if ( protocol != AF_INET
+		if ( protocol == 24 || protocol == 28 || protocol == 30 )
-		     && protocol != AF_INET6
+			protocol = AF_INET6;
-		     && protocol != 24
+
-		     && protocol != 28
+		if ( protocol != AF_INET && protocol != AF_INET6 )
-		     && protocol != 30 )
 			{
 			Weird("non_ip_packet_in_null_transport", &current_packet);
 			goto done;
 			}
 
+		l3_proto = protocol;
 		break;
 		}
 
@@ -323,47 +325,51 @@ void PktSrc::Process()
 		{
 		// Get protocol being carried from the ethernet frame.
 		protocol = (data[12] << 8) + data[13];
+		data += GetLinkHeaderSize(props.link_type);
+		current_packet.eth_type = protocol;
 
 		switch ( protocol )
 			{
 			// MPLS carried over the ethernet frame.
 			case 0x8847:
-				// Remove the data link layer and denote a
-				// header size of zero before the IP header.
 				have_mpls = true;
-				data += GetLinkHeaderSize(props.link_type);
-				pkt_hdr_size = 0;
 				break;
 
 			// VLAN carried over the ethernet frame.
+			// 802.1q / 802.1ad
 			case 0x8100:
-				data += GetLinkHeaderSize(props.link_type);
+			case 0x9100:
+				current_packet.vlan = ((data[0] << 8) + data[1]) & 0xfff;
+				protocol = ((data[2] << 8) + data[3]);
+				data += 4; // Skip the vlan header
 
 				// Check for MPLS in VLAN.
-				if ( ((data[2] << 8) + data[3]) == 0x8847 )
+				if ( protocol == 0x8847 )
+					{
 					have_mpls = true;
+					break;
+					}
 
+				// Check for double-tagged (802.1ad)
+				if ( protocol == 0x8100 || protocol == 0x9100 )
+					{
+					protocol = ((data[2] << 8) + data[3]);
 					data += 4; // Skip the vlan header
-				pkt_hdr_size = 0;
+					}
-
-				// Check for 802.1ah (Q-in-Q) containing IP.
-				// Only do a second layer of vlan tag
-				// stripping because there is no
-				// specification that allows for deeper
-				// nesting.
-				if ( ((data[2] << 8) + data[3]) == 0x0800 )
-					data += 4;
 
+				current_packet.eth_type = protocol;
 				break;
 
 			// PPPoE carried over the ethernet frame.
 			case 0x8864:
-				data += GetLinkHeaderSize(props.link_type);
 				protocol = (data[6] << 8) + data[7];
 				data += 8; // Skip the PPPoE session and PPP header
-				pkt_hdr_size = 0;
 
-				if ( protocol != 0x0021 && protocol != 0x0057 )
+				if ( protocol == 0x0021 )
+					l3_proto = AF_INET;
+				else if ( protocol == 0x0057 )
+					l3_proto = AF_INET6;
+				else
 					{
 					// Neither IPv4 nor IPv6.
 					Weird("non_ip_packet_in_pppoe_encapsulation", &current_packet);
@@ -372,6 +378,15 @@ void PktSrc::Process()
 				break;
 			}
 
+		// Normal path to determine Layer 3 protocol.
+		if ( ! have_mpls && ! l3_proto )
+			{
+			if ( protocol == 0x800 )
+				l3_proto = AF_INET;
+			else if ( protocol == 0x86dd )
+				l3_proto = AF_INET6;
+			}
+
 		break;
 		}
 
@@ -379,17 +394,19 @@ void PktSrc::Process()
 		{
 		// Get PPP protocol.
 		protocol = (data[2] << 8) + data[3];
+		data += GetLinkHeaderSize(props.link_type);
 
 		if ( protocol == 0x0281 )
 			{
 			// MPLS Unicast. Remove the data link layer and
 			// denote a header size of zero before the IP header.
 			have_mpls = true;
-				data += GetLinkHeaderSize(props.link_type);
-			pkt_hdr_size = 0;
 			}
-
+		else if ( protocol == 0x0021 )
-		else if ( protocol != 0x0021 && protocol != 0x0057 )
+			l3_proto = AF_INET;
+		else if ( protocol == 0x0057 )
+			l3_proto = AF_INET6;
+		else
 			{
 			// Neither IPv4 nor IPv6.
 			Weird("non_ip_packet_in_ppp_encapsulation", &current_packet);
@@ -397,6 +414,15 @@ void PktSrc::Process()
 			}
 		break;
 		}
+
+	default:
+		{
+		// Assume we're pointing at IP. Just figure out which version.
+		data += GetLinkHeaderSize(props.link_type);
+		const struct ip* ip = (const struct ip *)data;
+		l3_proto = ( ip->ip_v == 4 ) ? AF_INET : AF_INET6;
+		break;
+		}
 	}
 
 	if ( have_mpls )
@@ -408,19 +434,56 @@ void PktSrc::Process()
 			{
 			end_of_stack = *(data + 2) & 0x01;
 			data += 4;
+
+			if ( data >= current_packet.data + current_packet.cap_len )
+				{
+				Weird("no_mpls_payload", &current_packet);
+				goto done;
 				}
 			}
 
+		// We assume that what remains is IP
+		if ( data + sizeof(struct ip) >= current_packet.data + current_packet.cap_len )
+			{
+			Weird("no_ip_in_mpls_payload", &current_packet);
+			goto done;
+			}
+
+		const struct ip* ip = (const struct ip *)data;
+		l3_proto = ( ip->ip_v == 4 ) ? AF_INET : AF_INET6;
+		}
+
+	else if ( encap_hdr_size )
+		{
+		// Blanket encapsulation. We assume that what remains is IP.
+		data += encap_hdr_size;
+		if ( data + sizeof(struct ip) >= current_packet.data + current_packet.cap_len )
+			{
+			Weird("no_ip_left_after_encap", &current_packet);
+			goto done;
+			}
+
+		const struct ip* ip = (const struct ip *)data;
+		l3_proto = ( ip->ip_v == 4 ) ? AF_INET : AF_INET6;
+		}
+
+	// We've now determined (a) AF_INET (IPv4) vs (b) AF_INET6 (IPv6) vs
+	// (c) AF_UNSPEC (0 == anything else)
+	current_packet.l3_proto = l3_proto;
+
+	// Calculate how much header we've used up.
+	current_packet.hdr_size = (data - current_packet.data);
+
 	if ( pseudo_realtime )
 		{
 		current_pseudo = CheckPseudoTime();
-		net_packet_dispatch(current_pseudo, current_packet.hdr, data, pkt_hdr_size, this);
+		net_packet_dispatch(current_pseudo, &current_packet, this);
 		if ( ! first_wallclock )
 			first_wallclock = current_time(true);
 		}
 
 	else
-		net_packet_dispatch(current_packet.ts, current_packet.hdr, data, pkt_hdr_size, this);
+		net_packet_dispatch(current_packet.time, &current_packet, this);
 
 done:
 	have_packet = 0;
@@ -452,8 +515,10 @@ bool PktSrc::ExtractNextPacketInternal()
 
 	if ( ExtractNextPacket(&current_packet) )
 		{
+		current_packet.l3_proto = AF_UNSPEC;
+
 		if ( ! first_timestamp )
-			first_timestamp = current_packet.ts;
+			first_timestamp = current_packet.time;
 
 		SetIdle(false);
 		have_packet = true;
@@ -536,12 +601,11 @@ bool PktSrc::ApplyBPFFilter(int index, const struct pcap_pkthdr *hdr, const u_ch
 	return pcap_offline_filter(code->GetProgram(), hdr, pkt);
 	}
 
-bool PktSrc::GetCurrentPacket(const pcap_pkthdr** hdr, const u_char** pkt)
+bool PktSrc::GetCurrentPacket(const Packet** pkt)
 	{
 	if ( ! have_packet )
 		return false;
 
-	*hdr = current_packet.hdr;
+	*pkt = &current_packet;
-	*pkt = current_packet.data;
 	return true;
 	}

src/iosource/PktSrc.h

@@ -6,6 +6,7 @@
 #include "IOSource.h"
 #include "BPF_Program.h"
 #include "Dict.h"
+#include "Packet.h"
 
 declare(PDict,BPF_Program);
 
@@ -165,14 +166,12 @@ public:
 	/**
 	 * Returns the packet currently being processed, if available.
 	 *
-	 * @param hdr A pointer to pass the header of the current packet back.
-	 *
 	 * @param pkt A pointer to pass the content of the current packet
 	 * back.
 	 *
 	 * @return True if the current packet is available, or false if not.
 	 */
-	bool GetCurrentPacket(const pcap_pkthdr** hdr, const u_char** pkt);
+	bool GetCurrentPacket(const Packet** hdr);
 
 	// PacketSource interace for derived classes to override.
 
@@ -225,6 +224,13 @@ public:
 	 */
 	static int GetLinkHeaderSize(int link_type);
 
+	/**
+	 * Return the pcap link encapsulation type we started with.
+	 *
+	 * @return DLT_EN10MB (etc.)
+	 */
+	int GetLinkEncap(void);
+
 protected:
 	friend class Manager;
 
@@ -274,26 +280,6 @@ protected:
 		Properties();
 	};
 
-	/**
-	 * Structure describing a packet.
-	 */
-	struct Packet {
-		/**
-		 *  Time associated with the packet.
-		 */
-		double ts;
-
-		/**
-		 *  The pcap header associated with the packet.
-		 */
-		const struct ::pcap_pkthdr* hdr;
-
-		/**
-		 * The full content of the packet.
-		 */
-		const u_char* data;
-	};
-
 	/**
 	 * Called from the implementations of \a Open() to signal that the
 	 * source has been successully opened.

src/iosource/pcap/Dumper.cc

@@ -101,8 +101,12 @@ bool PcapDumper::Dump(const Packet* pkt)
 	if ( ! dumper )
 		return false;
 
-	pcap_dump((u_char*) dumper, pkt->hdr, pkt->data);
+	// Reconstitute the pcap_pkthdr.
+	const struct pcap_pkthdr phdr = {
+		.ts = pkt->ts, .caplen = pkt->cap_len, .len = pkt->len
+	};
 
+	pcap_dump((u_char*) dumper, &phdr, pkt->data);
 	return true;
 	}
 

src/iosource/pcap/Source.cc

@@ -167,9 +167,8 @@ bool PcapSource::ExtractNextPacket(Packet* pkt)
 		return false;
 		}
 
-	pkt->ts = current_hdr.ts.tv_sec + double(current_hdr.ts.tv_usec) / 1e6;
+	last_data = data;
-	pkt->hdr = &current_hdr;
+	pkt->Init(props.link_type, &current_hdr.ts, current_hdr.caplen, current_hdr.len, data);
-	pkt->data = last_data = data;
 
 	if ( current_hdr.len == 0 || current_hdr.caplen == 0 )
 		{

src/types.bif

@@ -184,6 +184,18 @@ type EncapsulatingConn: record;
 
 module GLOBAL;
 
+enum link_encap %{
+	LINK_ETHERNET,
+	LINK_UNKNOWN,
+%}
+
+enum layer3_proto %{
+	L3_IPV4,
+	L3_IPV6,
+	L3_ARP,
+	L3_UNKNOWN,
+%}
+
 type gtpv1_hdr: record;
 type gtp_create_pdp_ctx_request_elements: record;
 type gtp_create_pdp_ctx_response_elements: record;

testing/btest/Baseline/core.raw_packet/output

@@ -0,0 +1,48 @@
+[l2=[encap=LINK_ETHERNET, len=215, cap_len=215, src=e8:de:27:ff:c0:78, dst=ff:ff:ff:ff:ff:ff, vlan=<uninitialized>, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=201, id=0, ttl=64, p=17, src=192.168.1.1, dst=255.255.255.255], ip6=<uninitialized>, tcp=<uninitialized>, udp=[sport=40190/udp, dport=7437/udp, ulen=181], icmp=<uninitialized>]
+[l2=[encap=LINK_ETHERNET, len=68, cap_len=68, src=60:f8:1d:c9:8c:fa, dst=e8:de:27:ff:c0:78, vlan=<uninitialized>, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=54, id=52261, ttl=64, p=6, src=192.168.1.103, dst=64.4.23.176], ip6=<uninitialized>, tcp=[sport=65493/tcp, dport=40031/tcp, seq=2642773190, ack=2891276360, hl=32, dl=2, flags=24, win=4096], udp=<uninitialized>, icmp=<uninitialized>]
+[l2=[encap=LINK_ETHERNET, len=60, cap_len=60, src=e8:de:27:ff:c0:77, dst=01:80:c2:00:00:00, vlan=<uninitialized>, eth_type=38, proto=L3_UNKNOWN], ip=<uninitialized>, ip6=<uninitialized>, tcp=<uninitialized>, udp=<uninitialized>, icmp=<uninitialized>]
+[l2=[encap=LINK_ETHERNET, len=78, cap_len=78, src=60:f8:1d:c9:8c:fa, dst=e8:de:27:ff:c0:78, vlan=<uninitialized>, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=64, id=32575, ttl=64, p=17, src=192.168.1.103, dst=192.168.1.1], ip6=<uninitialized>, tcp=<uninitialized>, udp=[sport=65170/udp, dport=53/udp, ulen=44], icmp=<uninitialized>]
+[l2=[encap=LINK_ETHERNET, len=78, cap_len=78, src=60:f8:1d:c9:8c:fa, dst=e8:de:27:ff:c0:78, vlan=<uninitialized>, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=64, id=55466, ttl=64, p=17, src=192.168.1.103, dst=192.168.1.1], ip6=<uninitialized>, tcp=<uninitialized>, udp=[sport=53129/udp, dport=53/udp, ulen=44], icmp=<uninitialized>]
+[l2=[encap=LINK_ETHERNET, len=92, cap_len=92, src=60:f8:1d:c9:8c:fa, dst=e8:de:27:ff:c0:78, vlan=<uninitialized>, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=78, id=32240, ttl=64, p=17, src=192.168.1.103, dst=192.168.1.1], ip6=<uninitialized>, tcp=<uninitialized>, udp=[sport=53129/udp, dport=53/udp, ulen=58], icmp=<uninitialized>]
+[l2=[encap=LINK_ETHERNET, len=85, cap_len=85, src=60:f8:1d:c9:8c:fa, dst=e8:de:27:ff:c0:78, vlan=<uninitialized>, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=71, id=53895, ttl=64, p=17, src=192.168.1.103, dst=192.168.1.1], ip6=<uninitialized>, tcp=<uninitialized>, udp=[sport=57932/udp, dport=53/udp, ulen=51], icmp=<uninitialized>]
+[l2=[encap=LINK_ETHERNET, len=60, cap_len=60, src=e8:de:27:ff:c0:77, dst=01:80:c2:00:00:00, vlan=<uninitialized>, eth_type=38, proto=L3_UNKNOWN], ip=<uninitialized>, ip6=<uninitialized>, tcp=<uninitialized>, udp=<uninitialized>, icmp=<uninitialized>]
+[l2=[encap=LINK_ETHERNET, len=42, cap_len=42, src=00:50:56:3e:93:6b, dst=ff:ff:ff:ff:ff:ff, vlan=<uninitialized>, eth_type=2054, proto=L3_ARP], ip=<uninitialized>, ip6=<uninitialized>, tcp=<uninitialized>, udp=<uninitialized>, icmp=<uninitialized>]
+[l2=[encap=LINK_ETHERNET, len=42, cap_len=42, src=00:50:56:3e:93:6b, dst=ff:ff:ff:ff:ff:ff, vlan=<uninitialized>, eth_type=2054, proto=L3_ARP], ip=<uninitialized>, ip6=<uninitialized>, tcp=<uninitialized>, udp=<uninitialized>, icmp=<uninitialized>]
+[l2=[encap=LINK_ETHERNET, len=307, cap_len=307, src=e8:de:27:ff:c0:78, dst=01:00:5e:7f:ff:fa, vlan=<uninitialized>, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=293, id=0, ttl=4, p=17, src=192.168.1.1, dst=239.255.255.250], ip6=<uninitialized>, tcp=<uninitialized>, udp=[sport=45335/udp, dport=1900/udp, ulen=273], icmp=<uninitialized>]
+[l2=[encap=LINK_ETHERNET, len=316, cap_len=316, src=e8:de:27:ff:c0:78, dst=01:00:5e:7f:ff:fa, vlan=<uninitialized>, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=302, id=0, ttl=4, p=17, src=192.168.1.1, dst=239.255.255.250], ip6=<uninitialized>, tcp=<uninitialized>, udp=[sport=45335/udp, dport=1900/udp, ulen=282], icmp=<uninitialized>]
+[l2=[encap=LINK_ETHERNET, len=379, cap_len=379, src=e8:de:27:ff:c0:78, dst=01:00:5e:7f:ff:fa, vlan=<uninitialized>, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=365, id=0, ttl=4, p=17, src=192.168.1.1, dst=239.255.255.250], ip6=<uninitialized>, tcp=<uninitialized>, udp=[sport=45335/udp, dport=1900/udp, ulen=345], icmp=<uninitialized>]
+[l2=[encap=LINK_ETHERNET, len=371, cap_len=371, src=e8:de:27:ff:c0:78, dst=01:00:5e:7f:ff:fa, vlan=<uninitialized>, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=357, id=0, ttl=4, p=17, src=192.168.1.1, dst=239.255.255.250], ip6=<uninitialized>, tcp=<uninitialized>, udp=[sport=45335/udp, dport=1900/udp, ulen=337], icmp=<uninitialized>]
+[l2=[encap=LINK_ETHERNET, len=60, cap_len=60, src=e8:de:27:ff:c0:77, dst=01:80:c2:00:00:00, vlan=<uninitialized>, eth_type=38, proto=L3_UNKNOWN], ip=<uninitialized>, ip6=<uninitialized>, tcp=<uninitialized>, udp=<uninitialized>, icmp=<uninitialized>]
+[l2=[encap=LINK_ETHERNET, len=355, cap_len=355, src=e8:de:27:ff:c0:78, dst=01:00:5e:7f:ff:fa, vlan=<uninitialized>, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=341, id=0, ttl=4, p=17, src=192.168.1.1, dst=239.255.255.250], ip6=<uninitialized>, tcp=<uninitialized>, udp=[sport=45335/udp, dport=1900/udp, ulen=321], icmp=<uninitialized>]
+[l2=[encap=LINK_ETHERNET, len=42, cap_len=42, src=00:50:56:3e:93:6b, dst=ff:ff:ff:ff:ff:ff, vlan=<uninitialized>, eth_type=2054, proto=L3_ARP], ip=<uninitialized>, ip6=<uninitialized>, tcp=<uninitialized>, udp=<uninitialized>, icmp=<uninitialized>]
+[l2=[encap=LINK_ETHERNET, len=387, cap_len=387, src=e8:de:27:ff:c0:78, dst=01:00:5e:7f:ff:fa, vlan=<uninitialized>, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=373, id=0, ttl=4, p=17, src=192.168.1.1, dst=239.255.255.250], ip6=<uninitialized>, tcp=<uninitialized>, udp=[sport=45335/udp, dport=1900/udp, ulen=353], icmp=<uninitialized>]
+[l2=[encap=LINK_ETHERNET, len=316, cap_len=316, src=e8:de:27:ff:c0:78, dst=01:00:5e:7f:ff:fa, vlan=<uninitialized>, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=302, id=0, ttl=4, p=17, src=192.168.1.1, dst=239.255.255.250], ip6=<uninitialized>, tcp=<uninitialized>, udp=[sport=45335/udp, dport=1900/udp, ulen=282], icmp=<uninitialized>]
+[l2=[encap=LINK_ETHERNET, len=375, cap_len=375, src=e8:de:27:ff:c0:78, dst=01:00:5e:7f:ff:fa, vlan=<uninitialized>, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=361, id=0, ttl=4, p=17, src=192.168.1.1, dst=239.255.255.250], ip6=<uninitialized>, tcp=<uninitialized>, udp=[sport=45335/udp, dport=1900/udp, ulen=341], icmp=<uninitialized>]
+[l2=[encap=LINK_ETHERNET, len=369, cap_len=369, src=e8:de:27:ff:c0:78, dst=01:00:5e:7f:ff:fa, vlan=<uninitialized>, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=355, id=0, ttl=4, p=17, src=192.168.1.1, dst=239.255.255.250], ip6=<uninitialized>, tcp=<uninitialized>, udp=[sport=45335/udp, dport=1900/udp, ulen=335], icmp=<uninitialized>]
+[l2=[encap=LINK_ETHERNET, len=316, cap_len=316, src=e8:de:27:ff:c0:78, dst=01:00:5e:7f:ff:fa, vlan=<uninitialized>, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=302, id=0, ttl=4, p=17, src=192.168.1.1, dst=239.255.255.250], ip6=<uninitialized>, tcp=<uninitialized>, udp=[sport=45335/udp, dport=1900/udp, ulen=282], icmp=<uninitialized>]
+[l2=[encap=LINK_ETHERNET, len=371, cap_len=371, src=e8:de:27:ff:c0:78, dst=01:00:5e:7f:ff:fa, vlan=<uninitialized>, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=357, id=0, ttl=4, p=17, src=192.168.1.1, dst=239.255.255.250], ip6=<uninitialized>, tcp=<uninitialized>, udp=[sport=45335/udp, dport=1900/udp, ulen=337], icmp=<uninitialized>]
+[l2=[encap=LINK_ETHERNET, len=381, cap_len=381, src=e8:de:27:ff:c0:78, dst=01:00:5e:7f:ff:fa, vlan=<uninitialized>, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=367, id=0, ttl=4, p=17, src=192.168.1.1, dst=239.255.255.250], ip6=<uninitialized>, tcp=<uninitialized>, udp=[sport=45335/udp, dport=1900/udp, ulen=347], icmp=<uninitialized>]
+[l2=[encap=LINK_ETHERNET, len=215, cap_len=215, src=e8:de:27:ff:c0:78, dst=ff:ff:ff:ff:ff:ff, vlan=<uninitialized>, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=201, id=0, ttl=64, p=17, src=192.168.1.1, dst=255.255.255.255], ip6=<uninitialized>, tcp=<uninitialized>, udp=[sport=40190/udp, dport=7437/udp, ulen=181], icmp=<uninitialized>]
+[l2=[encap=LINK_ETHERNET, len=98, cap_len=98, src=00:50:56:3e:93:6b, dst=e8:de:27:ff:c0:78, vlan=<uninitialized>, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=84, id=29257, ttl=64, p=1, src=192.168.1.104, dst=192.168.1.1], ip6=<uninitialized>, tcp=<uninitialized>, udp=<uninitialized>, icmp=[icmp_type=8]]
+[l2=[encap=LINK_ETHERNET, len=98, cap_len=98, src=e8:de:27:ff:c0:78, dst=00:50:56:3e:93:6b, vlan=<uninitialized>, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=84, id=3684, ttl=64, p=1, src=192.168.1.1, dst=192.168.1.104], ip6=<uninitialized>, tcp=<uninitialized>, udp=<uninitialized>, icmp=[icmp_type=0]]
+[l2=[encap=LINK_ETHERNET, len=112, cap_len=112, src=60:f8:1d:c9:8c:fa, dst=e8:de:27:ff:c0:78, vlan=<uninitialized>, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=98, id=56893, ttl=64, p=6, src=192.168.1.103, dst=74.125.21.138], ip6=<uninitialized>, tcp=[sport=49171/tcp, dport=443/tcp, seq=3725176031, ack=445274592, hl=32, dl=46, flags=24, win=4096], udp=<uninitialized>, icmp=<uninitialized>]
+[l2=[encap=LINK_ETHERNET, len=66, cap_len=66, src=60:f8:1d:c9:8c:fa, dst=e8:de:27:ff:c0:78, vlan=<uninitialized>, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=52, id=22643, ttl=64, p=6, src=192.168.1.103, dst=74.125.21.138], ip6=<uninitialized>, tcp=[sport=49171/tcp, dport=443/tcp, seq=3725176077, ack=445274652, hl=32, dl=0, flags=16, win=4094], udp=<uninitialized>, icmp=<uninitialized>]
+[l2=[encap=LINK_ETHERNET, len=112, cap_len=112, src=60:f8:1d:c9:8c:fa, dst=e8:de:27:ff:c0:78, vlan=<uninitialized>, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=98, id=85, ttl=64, p=6, src=192.168.1.103, dst=74.125.21.138], ip6=<uninitialized>, tcp=[sport=49171/tcp, dport=443/tcp, seq=3725176077, ack=445274652, hl=32, dl=46, flags=24, win=4096], udp=<uninitialized>, icmp=<uninitialized>]
+[l2=[encap=LINK_ETHERNET, len=97, cap_len=97, src=60:f8:1d:c9:8c:fa, dst=e8:de:27:ff:c0:78, vlan=<uninitialized>, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=83, id=28558, ttl=64, p=6, src=192.168.1.103, dst=74.125.21.138], ip6=<uninitialized>, tcp=[sport=49171/tcp, dport=443/tcp, seq=3725176123, ack=445274652, hl=32, dl=31, flags=24, win=4096], udp=<uninitialized>, icmp=<uninitialized>]
+[l2=[encap=LINK_ETHERNET, len=66, cap_len=66, src=60:f8:1d:c9:8c:fa, dst=e8:de:27:ff:c0:78, vlan=<uninitialized>, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=52, id=36529, ttl=64, p=6, src=192.168.1.103, dst=74.125.21.138], ip6=<uninitialized>, tcp=[sport=49171/tcp, dport=443/tcp, seq=3725176154, ack=445274652, hl=32, dl=0, flags=17, win=4096], udp=<uninitialized>, icmp=<uninitialized>]
+[l2=[encap=LINK_ETHERNET, len=60, cap_len=60, src=e8:de:27:ff:c0:77, dst=01:80:c2:00:00:00, vlan=<uninitialized>, eth_type=38, proto=L3_UNKNOWN], ip=<uninitialized>, ip6=<uninitialized>, tcp=<uninitialized>, udp=<uninitialized>, icmp=<uninitialized>]
+[l2=[encap=LINK_ETHERNET, len=64, cap_len=64, src=00:19:06:ea:b8:c1, dst=ff:ff:ff:ff:ff:ff, vlan=123, eth_type=2054, proto=L3_ARP], ip=<uninitialized>, ip6=<uninitialized>, tcp=<uninitialized>, udp=<uninitialized>, icmp=<uninitialized>]
+[l2=[encap=LINK_ETHERNET, len=64, cap_len=64, src=00:18:73:de:57:c1, dst=ff:ff:ff:ff:ff:ff, vlan=123, eth_type=2054, proto=L3_ARP], ip=<uninitialized>, ip6=<uninitialized>, tcp=<uninitialized>, udp=<uninitialized>, icmp=<uninitialized>]
+[l2=[encap=LINK_ETHERNET, len=64, cap_len=64, src=00:18:73:de:57:c1, dst=ff:ff:ff:ff:ff:ff, vlan=123, eth_type=2054, proto=L3_ARP], ip=<uninitialized>, ip6=<uninitialized>, tcp=<uninitialized>, udp=<uninitialized>, icmp=<uninitialized>]
+[l2=[encap=LINK_ETHERNET, len=64, cap_len=64, src=00:19:06:ea:b8:c1, dst=00:18:73:de:57:c1, vlan=123, eth_type=2054, proto=L3_ARP], ip=<uninitialized>, ip6=<uninitialized>, tcp=<uninitialized>, udp=<uninitialized>, icmp=<uninitialized>]
+[l2=[encap=LINK_ETHERNET, len=118, cap_len=118, src=00:18:73:de:57:c1, dst=00:19:06:ea:b8:c1, vlan=123, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=100, id=5, ttl=255, p=1, src=192.168.123.2, dst=192.168.123.1], ip6=<uninitialized>, tcp=<uninitialized>, udp=<uninitialized>, icmp=[icmp_type=8]]
+[l2=[encap=LINK_ETHERNET, len=64, cap_len=64, src=00:19:06:ea:b8:c1, dst=ff:ff:ff:ff:ff:ff, vlan=123, eth_type=2054, proto=L3_ARP], ip=<uninitialized>, ip6=<uninitialized>, tcp=<uninitialized>, udp=<uninitialized>, icmp=<uninitialized>]
+[l2=[encap=LINK_ETHERNET, len=64, cap_len=64, src=00:18:73:de:57:c1, dst=00:19:06:ea:b8:c1, vlan=123, eth_type=2054, proto=L3_ARP], ip=<uninitialized>, ip6=<uninitialized>, tcp=<uninitialized>, udp=<uninitialized>, icmp=<uninitialized>]
+[l2=[encap=LINK_ETHERNET, len=118, cap_len=118, src=00:18:73:de:57:c1, dst=00:19:06:ea:b8:c1, vlan=123, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=100, id=6, ttl=255, p=1, src=192.168.123.2, dst=192.168.123.1], ip6=<uninitialized>, tcp=<uninitialized>, udp=<uninitialized>, icmp=[icmp_type=8]]
+[l2=[encap=LINK_ETHERNET, len=118, cap_len=118, src=00:19:06:ea:b8:c1, dst=00:18:73:de:57:c1, vlan=123, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=100, id=6, ttl=255, p=1, src=192.168.123.1, dst=192.168.123.2], ip6=<uninitialized>, tcp=<uninitialized>, udp=<uninitialized>, icmp=[icmp_type=0]]
+[l2=[encap=LINK_ETHERNET, len=118, cap_len=118, src=00:18:73:de:57:c1, dst=00:19:06:ea:b8:c1, vlan=123, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=100, id=7, ttl=255, p=1, src=192.168.123.2, dst=192.168.123.1], ip6=<uninitialized>, tcp=<uninitialized>, udp=<uninitialized>, icmp=[icmp_type=8]]
+[l2=[encap=LINK_ETHERNET, len=118, cap_len=118, src=00:19:06:ea:b8:c1, dst=00:18:73:de:57:c1, vlan=123, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=100, id=7, ttl=255, p=1, src=192.168.123.1, dst=192.168.123.2], ip6=<uninitialized>, tcp=<uninitialized>, udp=<uninitialized>, icmp=[icmp_type=0]]
+[l2=[encap=LINK_ETHERNET, len=118, cap_len=118, src=00:18:73:de:57:c1, dst=00:19:06:ea:b8:c1, vlan=123, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=100, id=8, ttl=255, p=1, src=192.168.123.2, dst=192.168.123.1], ip6=<uninitialized>, tcp=<uninitialized>, udp=<uninitialized>, icmp=[icmp_type=8]]
+[l2=[encap=LINK_ETHERNET, len=118, cap_len=118, src=00:19:06:ea:b8:c1, dst=00:18:73:de:57:c1, vlan=123, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=100, id=8, ttl=255, p=1, src=192.168.123.1, dst=192.168.123.2], ip6=<uninitialized>, tcp=<uninitialized>, udp=<uninitialized>, icmp=[icmp_type=0]]
+[l2=[encap=LINK_ETHERNET, len=118, cap_len=118, src=00:18:73:de:57:c1, dst=00:19:06:ea:b8:c1, vlan=123, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=100, id=9, ttl=255, p=1, src=192.168.123.2, dst=192.168.123.1], ip6=<uninitialized>, tcp=<uninitialized>, udp=<uninitialized>, icmp=[icmp_type=8]]
+[l2=[encap=LINK_ETHERNET, len=118, cap_len=118, src=00:19:06:ea:b8:c1, dst=00:18:73:de:57:c1, vlan=123, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=100, id=9, ttl=255, p=1, src=192.168.123.1, dst=192.168.123.2], ip6=<uninitialized>, tcp=<uninitialized>, udp=<uninitialized>, icmp=[icmp_type=0]]

testing/btest/core/raw_packet.bro

@@ -0,0 +1,9 @@
+# @TEST-EXEC: bro -b -r $TRACES/raw_packets.trace %INPUT >output
+# @TEST-EXEC: bro -b -r $TRACES/icmp_dot1q.trace %INPUT >>output
+# @TEST-EXEC: btest-diff output
+
+event raw_packet(p: raw_pkt_hdr)
+	{
+	print p;
+	}
+

testing/btest/plugins/pktdumper-plugin/src/Foo.cc

@@ -29,8 +29,8 @@ void Foo::Close()
 
 bool Foo::Dump(const Packet* pkt)
 	{
-	double t = double(pkt->hdr->ts.tv_sec) + double(pkt->hdr->ts.tv_usec) / 1e6;
+	double t = double(pkt->ts.tv_sec) + double(pkt->ts.tv_usec) / 1e6;
-	fprintf(stdout, "Dumping to %s: %.6f len %u\n", props.path.c_str(), t, (unsigned int)pkt->hdr->len);
+	fprintf(stdout, "Dumping to %s: %.6f len %u\n", props.path.c_str(), t, (unsigned int)pkt->len);
 	return true;
 	}
 

testing/btest/plugins/pktsrc-plugin/src/Foo.cc

@@ -45,12 +45,9 @@ bool Foo::ExtractNextPacket(Packet* pkt)
 		return false;
 		}
 
-	hdr.ts.tv_sec = 1409193037;
+	struct timeval ts = { 1409193037, 0 };
-	hdr.ts.tv_usec = 0;
+	pkt->Init(props.link_type, &ts, packet.size(), packet.size(), 
-	hdr.caplen = hdr.len = packet.size();
+		(const u_char *)packet.c_str());
-	pkt->ts = hdr.ts.tv_sec;
-	pkt->hdr = &hdr;
-	pkt->data = (const u_char *)packet.c_str();
 	return true;
 	}
 

testing/btest/plugins/pktsrc-plugin/src/Foo.h

@@ -26,7 +26,6 @@ protected:
 private:
 	Properties props;
 	string packet;
-	struct pcap_pkthdr hdr;
 };
 
 }