Merge remote-tracking branch 'origin/master' into topic/liangzhu/analyzer-ocsp

CHANGES

@@ -1,4 +1,61 @@
 
+2.4-41 | 2015-07-21 08:35:17 -0700
+
+  * Fixing compiler warning. (Robin Sommer)
+
+  * Updates to IANA TLS registry. (Johanna Amann)
+
+2.4-38 | 2015-07-20 15:30:35 -0700
+
+  * Refactor code to use a common Packet type throught. (Jeff
+    Barber/Robin Sommer)
+
+  * Extend parsing layer 2 and keeping track of layer 3 protoco. (Jeff Barber)
+
+  * Add a raw_packet() event that generated for all packets and
+    include layer 2 information. (Jeff Barber)
+
+2.4-27 | 2015-07-15 13:31:49 -0700
+
+  * Fix race condition in intel test. (Johanna Amann)
+
+2.4-24 | 2015-07-14 08:04:11 -0700
+
+  * Correct Perl package name on FreeBSD in documentation.(Justin Azoff)
+
+  * Adding an environment variable to BTest configuration for external
+    scripts. (Robin Sommer)
+
+2.4-20 | 2015-07-03 10:40:21 -0700
+
+  * Adding a weird for when truncated packets lead TCP reassembly to
+    ignore content. (Robin Sommer)
+
+2.4-19 | 2015-07-03 09:04:54 -0700
+
+  * A set of tests exercising IP defragmentation and TCP reassembly.
+    (Robin Sommer)
+
+2.4-17 | 2015-06-28 13:02:41 -0700
+
+  * BIT-1314: Add detection for Quantum Insert attacks. The TCP
+    reassembler can now keep a history of old TCP segments using the
+    tcp_max_old_segments option. An overlapping segment with different
+    data will then generate an rexmit_inconsistency event. The default
+    for tcp_max_old_segments is zero, which disabled any additional
+    buffering. (Yun Zheng Hu/Robin Sommer)
+
+2.4-14 | 2015-06-28 12:30:12 -0700
+
+  * BIT-1400: Allow '<' and '>' in MIME multipart boundaries. The spec
+    doesn't actually seem to permit these, but they seem to occur in
+    the wild. (Jon Siwek)
+
+2.4-12 | 2015-06-28 12:21:11 -0700
+
+  * BIT-1399: Trying to decompress deflated HTTP content even when
+    zlib headers are missing. (Seth Hall)
+
 2.4-10 | 2015-06-25 07:11:17 -0700
 
   * Correct a name used in a header identifier (Justin Azoff)

VERSION

@@ -1 +1 @@
-2.4-10
+2.4-41

aux/bro-aux

@@ -1 +1 @@
-Subproject commit 6d6679506d8762ddbba16f0b34f7ad253e3aac45
+Subproject commit 07af9748f40dc47d3a2b3290db494a90dcbddbdc

aux/broker

@@ -1 +1 @@
-Subproject commit f303cdbc60ad6eef35ebcd1473ee85b3123f5ef1
+Subproject commit d25efc7d5f495c30294b11180c1857477078f2d6

aux/btest

@@ -1 +1 @@
-Subproject commit 0e2da116a5e29baacaecc6daac7bc4bc9ff387c5
+Subproject commit a89cd0fda0f17f69b96c935959cae89145b92927

aux/plugins

@@ -1 +1 @@
-Subproject commit 99d7519991b41a970809a99433ea9c7df42e9d93
+Subproject commit 98ad8a5b97f601a3ec9a773d87582438212b8290

doc/install/install.rst

@@ -67,7 +67,7 @@ To install the required dependencies, you can use:
 
   .. console::
 
-      sudo pkg install bash cmake swig bison python perl py27-sqlite3
+      sudo pkg install bash cmake swig bison python perl5 py27-sqlite3
 
   Note that in older versions of FreeBSD, you might have to use the
   "pkg_add -r" command instead of "pkg install".

scripts/base/init-bare.bro

@@ -740,6 +740,7 @@ type pcap_packet: record {
 	caplen: count;	##< The number of bytes captured (<= *len*).
 	len: count;	##< The length of the packet in bytes, including link-level header.
 	data: string;	##< The payload of the packet, including link-level header.
+	link_type: link_encap;	##< Layer 2 link encapsulation type.
 };
 
 ## GeoIP location information.
@@ -954,6 +955,11 @@ const tcp_max_above_hole_without_any_acks = 16384 &redef;
 ## .. bro:see:: tcp_max_initial_window tcp_max_above_hole_without_any_acks
 const tcp_excessive_data_without_further_acks = 10 * 1024 * 1024 &redef;
 
+## Number of TCP segments to buffer beyond what's been acknowledged already
+## to detect retransmission inconsistencies. Zero disables any additonal
+## buffering.
+const tcp_max_old_segments = 0 &redef;
+
 ## For services without a handler, these sets define originator-side ports
 ## that still trigger reassembly.
 ##
@@ -1495,6 +1501,33 @@ type pkt_hdr: record {
 	icmp: icmp_hdr &optional;	##< The ICMP header if an ICMP packet.
 };
 
+## Values extracted from the layer 2 header.
+##
+## .. bro:see:: pkt_hdr
+type l2_hdr: record {
+	encap: link_encap;      ##< L2 link encapsulation.
+	len: count;		##< Total frame length on wire.
+	cap_len: count;		##< Captured length.
+	src: string &optional;	##< L2 source (if Ethernet).
+	dst: string &optional;	##< L2 destination (if Ethernet).
+	vlan: count &optional;	##< Outermost VLAN tag if any (and Ethernet).
+	eth_type: count &optional;	##< Innermost Ethertype (if Ethernet).
+	proto: layer3_proto;	##< L3 protocol.
+};
+
+## A raw packet header, consisting of L2 header and everything in
+## :bro:id:`pkt_hdr`. .
+##
+## .. bro:see:: raw_packet pkt_hdr
+type raw_pkt_hdr: record {
+	l2: l2_hdr;			##< The layer 2 header.
+	ip: ip4_hdr &optional;		##< The IPv4 header if an IPv4 packet.
+	ip6: ip6_hdr &optional;		##< The IPv6 header if an IPv6 packet.
+	tcp: tcp_hdr &optional;		##< The TCP header if a TCP packet.
+	udp: udp_hdr &optional;		##< The UDP header if a UDP packet.
+	icmp: icmp_hdr &optional;	##< The ICMP header if an ICMP packet.
+};
+
 ## A Teredo origin indication header.  See :rfc:`4380` for more information
 ## about the Teredo protocol.
 ##

scripts/base/protocols/ssl/consts.bro

@@ -120,9 +120,9 @@ export {
 		[18] = "signed_certificate_timestamp",
 		[19] = "client_certificate_type",
 		[20] = "server_certificate_type",
-		[21] = "padding", # temporary till 2015-03-12
+		[21] = "padding", # temporary till 2016-03-12
 		[22] = "encrypt_then_mac",
-		[23] = "extended_master_secret", # temporary till 2015-09-26
+		[23] = "extended_master_secret",
 		[35] = "SessionTicket TLS",
 		[40] = "extended_random",
 		[13172] = "next_protocol_negotiation",
@@ -169,7 +169,8 @@ export {
 		[256] = "ffdhe2048",
 		[257] = "ffdhe3072",
 		[258] = "ffdhe4096",
-		[259] = "ffdhe8192",
+		[259] = "ffdhe6144",
+		[260] = "ffdhe8192",
 		[0xFF01] = "arbitrary_explicit_prime_curves",
 		[0xFF02] = "arbitrary_explicit_char2_curves"
 	} &default=function(i: count):string { return fmt("unknown-%d", i); };

src/Conn.cc

@@ -241,12 +241,8 @@ void Connection::NextPacket(double t, int is_orig,
 			const u_char*& data,
 			int& record_packet, int& record_content,
 			// arguments for reproducing packets
-			const struct pcap_pkthdr* hdr,
+			const Packet *pkt)
-			const u_char* const pkt,
-			int hdr_size)
 	{
-	current_hdr = hdr;
-	current_hdr_size = hdr_size;
 	current_timestamp = t;
 	current_pkt = pkt;
 
@@ -264,8 +260,6 @@ void Connection::NextPacket(double t, int is_orig,
 	else
 		last_time = t;
 
-	current_hdr = 0;
-	current_hdr_size = 0;
 	current_timestamp = 0;
 	current_pkt = 0;
 	}

src/Conn.h

@@ -86,9 +86,7 @@ public:
 			const u_char*& data,
 			int& record_packet, int& record_content,
 			// arguments for reproducing packets
-			const struct pcap_pkthdr* hdr,
+			const Packet *pkt);
-			const u_char* const pkt,
-			int hdr_size);
 
 	HashKey* Key() const			{ return key; }
 	void ClearKey()				{ key = 0; }

src/IP.cc

@@ -327,24 +327,31 @@ RecordVal* IP_Hdr::BuildIPHdrVal() const
 RecordVal* IP_Hdr::BuildPktHdrVal() const
 	{
 	static RecordType* pkt_hdr_type = 0;
+
+	if ( ! pkt_hdr_type )
+		pkt_hdr_type = internal_type("pkt_hdr")->AsRecordType();
+
+	RecordVal* pkt_hdr = new RecordVal(pkt_hdr_type);
+	return BuildPktHdrVal(pkt_hdr, 0);
+	}
+
+RecordVal* IP_Hdr::BuildPktHdrVal(RecordVal* pkt_hdr, int sindex) const
+	{
 	static RecordType* tcp_hdr_type = 0;
 	static RecordType* udp_hdr_type = 0;
 	static RecordType* icmp_hdr_type = 0;
 
-	if ( ! pkt_hdr_type )
+	if ( ! tcp_hdr_type )
 		{
-		pkt_hdr_type = internal_type("pkt_hdr")->AsRecordType();
 		tcp_hdr_type = internal_type("tcp_hdr")->AsRecordType();
 		udp_hdr_type = internal_type("udp_hdr")->AsRecordType();
 		icmp_hdr_type = internal_type("icmp_hdr")->AsRecordType();
 		}
 
-	RecordVal* pkt_hdr = new RecordVal(pkt_hdr_type);
-
 	if ( ip4 )
-		pkt_hdr->Assign(0, BuildIPHdrVal());
+		pkt_hdr->Assign(sindex + 0, BuildIPHdrVal());
 	else
-		pkt_hdr->Assign(1, BuildIPHdrVal());
+		pkt_hdr->Assign(sindex + 1, BuildIPHdrVal());
 
 	// L4 header.
 	const u_char* data = Payload();
@@ -368,7 +375,7 @@ RecordVal* IP_Hdr::BuildPktHdrVal() const
 		tcp_hdr->Assign(6, new Val(tp->th_flags, TYPE_COUNT));
 		tcp_hdr->Assign(7, new Val(ntohs(tp->th_win), TYPE_COUNT));
 
-		pkt_hdr->Assign(2, tcp_hdr);
+		pkt_hdr->Assign(sindex + 2, tcp_hdr);
 		break;
 		}
 
@@ -381,7 +388,7 @@ RecordVal* IP_Hdr::BuildPktHdrVal() const
 		udp_hdr->Assign(1, new PortVal(ntohs(up->uh_dport), TRANSPORT_UDP));
 		udp_hdr->Assign(2, new Val(ntohs(up->uh_ulen), TYPE_COUNT));
 
-		pkt_hdr->Assign(3, udp_hdr);
+		pkt_hdr->Assign(sindex + 3, udp_hdr);
 		break;
 		}
 
@@ -392,7 +399,7 @@ RecordVal* IP_Hdr::BuildPktHdrVal() const
 
 		icmp_hdr->Assign(0, new Val(icmpp->icmp_type, TYPE_COUNT));
 
-		pkt_hdr->Assign(4, icmp_hdr);
+		pkt_hdr->Assign(sindex + 4, icmp_hdr);
 		break;
 		}
 

src/IP.h

@@ -574,8 +574,13 @@ public:
 	 */
 	RecordVal* BuildPktHdrVal() const;
 
-private:
+	/**
+	 * Same as above, but simply add our values into the record at the
+	 * specified starting index.
+	 */
+	RecordVal* BuildPktHdrVal(RecordVal* pkt_hdr, int sindex) const;
 
+private:
 	const struct ip* ip4;
 	const struct ip6_hdr* ip6;
 	bool del;

src/Net.cc

@@ -62,10 +62,8 @@ double bro_start_network_time;	// timestamp of first packet
 double last_watchdog_proc_time = 0.0;	// value of above during last watchdog
 bool terminating = false;	// whether we're done reading and finishing up
 
-const struct pcap_pkthdr* current_hdr = 0;
+const Packet *current_pkt = 0;
-const u_char* current_pkt = 0;
 int current_dispatched = 0;
-int current_hdr_size = 0;
 double current_timestamp = 0.0;
 iosource::PktSrc* current_pktsrc = 0;
 iosource::IOSource* current_iosrc = 0;
@@ -109,7 +107,7 @@ RETSIGTYPE watchdog(int /* signo */)
 			int frac_pst =
 				int((processing_start_time - int_pst) * 1e6);
 
-			if ( current_hdr )
+			if ( current_pkt )
 				{
 				if ( ! pkt_dumper )
 					{
@@ -126,12 +124,8 @@ RETSIGTYPE watchdog(int /* signo */)
 					}
 
 				if ( pkt_dumper )
-					{
+					pkt_dumper->Dump(current_pkt);
-					iosource::PktDumper::Packet p;
+
-					p.hdr = current_hdr;
-					p.data = current_pkt;
-					pkt_dumper->Dump(&p);
-					}
 				}
 
 			net_get_final_stats();
@@ -240,9 +234,7 @@ void expire_timers(iosource::PktSrc* src_ps)
 				max_timer_expires - current_dispatched);
 	}
 
-void net_packet_dispatch(double t, const struct pcap_pkthdr* hdr,
+void net_packet_dispatch(double t, const Packet* pkt, iosource::PktSrc* src_ps)
-			 const u_char* pkt, int hdr_size,
-			 iosource::PktSrc* src_ps)
 	{
 	if ( ! bro_start_network_time )
 		bro_start_network_time = t;
@@ -278,7 +270,7 @@ void net_packet_dispatch(double t, const struct pcap_pkthdr* hdr,
 			}
 		}
 
-	sessions->DispatchPacket(t, hdr, pkt, hdr_size, src_ps);
+	sessions->NextPacket(t, pkt);
 	mgr.Drain();
 
 	if ( sp )

src/Net.h

@@ -19,8 +19,7 @@ extern void net_get_final_stats();
 extern void net_finish(int drain_events);
 extern void net_delete();	// Reclaim all memory, etc.
 extern void net_update_time(double new_network_time);
-extern void net_packet_dispatch(double t, const struct pcap_pkthdr* hdr,
+extern void net_packet_dispatch(double t, const Packet* pkt,
-			const u_char* pkt, int hdr_size,
 			iosource::PktSrc* src_ps);
 extern void expire_timers(iosource::PktSrc* src_ps = 0);
 extern void termination_signal();
@@ -74,10 +73,8 @@ extern bool using_communication;
 // Snaplen passed to libpcap.
 extern int snaplen;
 
-extern const struct pcap_pkthdr* current_hdr;
+extern const Packet* current_pkt;
-extern const u_char* current_pkt;
 extern int current_dispatched;
-extern int current_hdr_size;
 extern double current_timestamp;
 extern iosource::PktSrc* current_pktsrc;
 extern iosource::IOSource* current_iosrc;

src/NetVar.cc

@@ -49,6 +49,7 @@ double tcp_partial_close_delay;
 int tcp_max_initial_window;
 int tcp_max_above_hole_without_any_acks;
 int tcp_excessive_data_without_further_acks;
+int tcp_max_old_segments;
 
 RecordType* socks_address;
 
@@ -225,8 +226,6 @@ int suppress_local_output;
 
 double timer_mgr_inactivity_timeout;
 
-int time_machine_profiling;
-
 StringVal* trace_output_file;
 
 int record_all_packets;
@@ -354,6 +353,7 @@ void init_net_var()
 		opt_internal_int("tcp_max_above_hole_without_any_acks");
 	tcp_excessive_data_without_further_acks =
 		opt_internal_int("tcp_excessive_data_without_further_acks");
+	tcp_max_old_segments = opt_internal_int("tcp_max_old_segments");
 
 	socks_address = internal_type("SOCKS::Address")->AsRecordType();
 
@@ -520,7 +520,6 @@ void init_net_var()
 
 	timer_mgr_inactivity_timeout =
 		opt_internal_double("timer_mgr_inactivity_timeout");
-	time_machine_profiling = opt_internal_int("time_machine_profiling");
 
 	script_id = internal_type("script_id")->AsRecordType();
 	id_table = internal_type("id_table")->AsTableType();

src/NetVar.h

@@ -52,6 +52,7 @@ extern double tcp_reset_delay;
 extern int tcp_max_initial_window;
 extern int tcp_max_above_hole_without_any_acks;
 extern int tcp_excessive_data_without_further_acks;
+extern int tcp_max_old_segments;
 
 extern RecordType* socks_address;
 
@@ -229,8 +230,6 @@ extern int suppress_local_output;
 
 extern double timer_mgr_inactivity_timeout;
 
-extern int time_machine_profiling;
-
 extern StringVal* trace_output_file;
 
 extern int record_all_packets;

src/Reassem.cc

@@ -34,12 +34,52 @@ uint64 Reassembler::total_size = 0;
 Reassembler::Reassembler(uint64 init_seq)
 	{
 	blocks = last_block = 0;
+	old_blocks = last_old_block = 0;
+	total_old_blocks = max_old_blocks = 0;
 	trim_seq = last_reassem_seq = init_seq;
 	}
 
 Reassembler::~Reassembler()
 	{
 	ClearBlocks();
+	ClearOldBlocks();
+	}
+
+void Reassembler::CheckOverlap(DataBlock *head, DataBlock *tail,
+					uint64 seq, uint64 len, const u_char* data)
+	{
+	if ( ! head || ! tail )
+		return;
+
+	uint64 upper = (seq + len);
+
+	for ( DataBlock* b = head; b; b = b->next )
+		{
+		uint64 nseq = seq;
+		uint64 nupper = upper;
+		const u_char* ndata = data;
+
+		if ( nupper <= b->seq )
+			continue;
+
+		if ( nseq >= b->upper )
+			continue;
+
+		if ( nseq < b->seq )
+			{
+			ndata += (b->seq - seq);
+			nseq = b->seq;
+			}
+
+		if ( nupper > b->upper )
+			nupper = b->upper;
+
+		uint64 overlap_offset = (nseq - b->seq);
+		uint64 overlap_len = (nupper - nseq);
+
+		if ( overlap_len )
+			Overlap(&b->block[overlap_offset], ndata, overlap_len);
+		}
 	}
 
 void Reassembler::NewBlock(double t, uint64 seq, uint64 len, const u_char* data)
@@ -49,10 +89,14 @@ void Reassembler::NewBlock(double t, uint64 seq, uint64 len, const u_char* data)
 
 	uint64 upper_seq = seq + len;
 
+	CheckOverlap(old_blocks, last_old_block, seq, len, data);
+
 	if ( upper_seq <= trim_seq )
 		// Old data, don't do any work for it.
 		return;
 
+	CheckOverlap(blocks, last_block, seq, len, data);
+
 	if ( seq < trim_seq )
 		{ // Partially old data, just keep the good stuff.
 		uint64 amount_old = trim_seq - seq;
@@ -119,7 +163,36 @@ uint64 Reassembler::TrimToSeq(uint64 seq)
 				num_missing += seq - blocks->upper;
 			}
 
-		delete blocks;
+		if ( max_old_blocks )
+			{
+			// Move block over to old_blocks queue.
+			blocks->next = 0;
+
+			if ( last_old_block )
+				{
+				blocks->prev = last_old_block;
+				last_old_block->next = blocks;
+				}
+			else
+				{
+				blocks->prev = 0;
+				old_blocks = blocks;
+				}
+
+			last_old_block = blocks;
+			total_old_blocks++;
+
+			while ( old_blocks && total_old_blocks > max_old_blocks )
+				{
+				DataBlock* next = old_blocks->next;
+				delete old_blocks;
+				old_blocks = next;
+				total_old_blocks--;
+				}
+			}
+
+		else
+			delete blocks;
 
 		blocks = b;
 		}
@@ -156,6 +229,18 @@ void Reassembler::ClearBlocks()
 	last_block = 0;
 	}
 
+void Reassembler::ClearOldBlocks()
+	{
+	while ( old_blocks )
+		{
+		DataBlock* b = old_blocks->next;
+		delete old_blocks;
+		old_blocks = b;
+		}
+
+	last_old_block = 0;
+	}
+
 uint64 Reassembler::TotalSize() const
 	{
 	uint64 size = 0;
@@ -218,7 +303,7 @@ DataBlock* Reassembler::AddAndCheck(DataBlock* b, uint64 seq, uint64 upper,
 		return new_b;
 		}
 
-	// The blocks overlap, complain.
+	// The blocks overlap.
 	if ( seq < b->seq )
 		{
 		// The new block has a prefix that comes before b.
@@ -239,8 +324,6 @@ DataBlock* Reassembler::AddAndCheck(DataBlock* b, uint64 seq, uint64 upper,
 	uint64 b_len = b->upper - overlap_start;
 	uint64 overlap_len = min(new_b_len, b_len);
 
-	Overlap(&b->block[overlap_offset], data, overlap_len);
-
 	if ( overlap_len < new_b_len )
 		{
 		// Recurse to resolve remainder of the new data.

src/Reassem.h

@@ -36,6 +36,7 @@ public:
 
 	// Delete all held blocks.
 	void ClearBlocks();
+	void ClearOldBlocks();
 
 	int HasBlocks() const		{ return blocks != 0; }
 	uint64 LastReassemSeq() const	{ return last_reassem_seq; }
@@ -50,6 +51,8 @@ public:
 	// Sum over all data buffered in some reassembler.
 	static uint64 TotalMemoryAllocation()	{ return total_size; }
 
+	void SetMaxOldBlocks(uint32 count)	{ max_old_blocks = count; }
+
 protected:
 	Reassembler()	{ }
 
@@ -65,10 +68,19 @@ protected:
 	DataBlock* AddAndCheck(DataBlock* b, uint64 seq,
 				uint64 upper, const u_char* data);
 
+	void CheckOverlap(DataBlock *head, DataBlock *tail,
+				uint64 seq, uint64 len, const u_char* data);
+
 	DataBlock* blocks;
 	DataBlock* last_block;
+
+	DataBlock* old_blocks;
+	DataBlock* last_old_block;
+
 	uint64 last_reassem_seq;
 	uint64 trim_seq;	// how far we've trimmed
+	uint32 max_old_blocks;
+	uint32 total_old_blocks;
 
 	static uint64 total_size;
 };

src/RemoteSerializer.cc

@@ -1453,7 +1453,7 @@ void RemoteSerializer::Process()
 	if ( packets.length() )
 		{
 		BufferedPacket* bp = packets[0];
-		Packet* p = bp->p;
+		const Packet* p = bp->p;
 
 		// FIXME: The following chunk of code is copied from
 		// net_packet_dispatch().  We should change that function
@@ -1465,14 +1465,12 @@ void RemoteSerializer::Process()
 		current_dispatched =
 			tmgr->Advance(network_time, max_timer_expires);
 
-		current_hdr = p->hdr;
+		current_pkt = p;
-		current_pkt = p->pkt;
 		current_pktsrc = 0;
 		current_iosrc = this;
-		sessions->NextPacket(p->time, p->hdr, p->pkt, p->hdr_size);
+		sessions->NextPacket(p->time, p);
 		mgr.Drain();
 
-		current_hdr = 0;	// done with these
 		current_pkt = 0;
 		current_iosrc = 0;
 

src/Serializer.cc

@@ -1122,110 +1122,3 @@ void EventPlayer::Process()
 
 	ne_time = 0;
 	}
-
-void Packet::Describe(ODesc* d) const
-	{
-	const IP_Hdr ip = IP();
-	d->Add(ip.SrcAddr());
-	d->Add("->");
-	d->Add(ip.DstAddr());
-	}
-
-bool Packet::Serialize(SerialInfo* info) const
-	{
-	return SERIALIZE(uint32(hdr->ts.tv_sec)) &&
-		SERIALIZE(uint32(hdr->ts.tv_usec)) &&
-		SERIALIZE(uint32(hdr->len)) &&
-		SERIALIZE(link_type) &&
-		info->s->Write(tag.c_str(), 0, "tag") &&
-		info->s->Write((const char*) pkt, hdr->caplen, "data");
-	}
-
-static BroFile* profiling_output = 0;
-
-#ifdef DEBUG
-static iosource::PktDumper* dump = 0;
-#endif
-
-Packet* Packet::Unserialize(UnserialInfo* info)
-	{
-	Packet* p = new Packet("", true);
-	pcap_pkthdr* hdr = new pcap_pkthdr;
-
-	uint32 tv_sec, tv_usec, len;
-
-	if ( ! (UNSERIALIZE(&tv_sec) &&
-		UNSERIALIZE(&tv_usec) &&
-		UNSERIALIZE(&len) &&
-		UNSERIALIZE(&p->link_type)) )
-		{
-		delete p;
-		delete hdr;
-		return 0;
-		}
-
-	hdr->ts.tv_sec = tv_sec;
-	hdr->ts.tv_usec = tv_usec;
-	hdr->len = len;
-
-	char* tag;
-	if ( ! info->s->Read((char**) &tag, 0, "tag") )
-		{
-		delete p;
-		delete hdr;
-		return 0;
-		}
-
-	char* pkt;
-	int caplen;
-	if ( ! info->s->Read((char**) &pkt, &caplen, "data") )
-		{
-		delete p;
-		delete hdr;
-		delete [] tag;
-		return 0;
-		}
-
-	hdr->caplen = uint32(caplen);
-	p->hdr = hdr;
-	p->pkt = (u_char*) pkt;
-	p->tag = tag;
-	p->hdr_size = iosource::PktSrc::GetLinkHeaderSize(p->link_type);
-
-	delete [] tag;
-
-	// For the global timer manager, we take the global network_time as the
-	// packet's timestamp for feeding it into our packet loop.
-	if ( p->tag == "" )
-		p->time = timer_mgr->Time();
-	else
-		p->time = p->hdr->ts.tv_sec + double(p->hdr->ts.tv_usec) / 1e6;
-
-	if ( time_machine_profiling )
-		{
-		if ( ! profiling_output )
-			profiling_output =
-				new BroFile("tm-prof.packets.log", "w");
-
-		profiling_output->Write(fmt("%.6f %s %d\n", current_time(),
-			(p->tag != "" ? p->tag.c_str() : "-"), hdr->len));
-		}
-
-#ifdef DEBUG
-	if ( debug_logger.IsEnabled(DBG_TM) )
-		{
-		if ( ! dump )
-			dump = iosource_mgr->OpenPktDumper("tm.pcap", true);
-
-		if ( dump )
-			{
-			iosource::PktDumper::Packet dp;
-			dp.hdr = p->hdr;
-			dp.data = p->pkt;
-			dump->Dump(&dp);
-			}
-		}
-#endif
-
-	return p;
-	}

src/Serializer.h

@@ -378,64 +378,6 @@ protected:
 
 };
 
-
-// A link-layer packet.
-//
-// Eventually we should use something like this consistently throughout Bro,
-// replacing the current packet arguments in functions like *::NextPacket().
-// Before doing this, though, we should consider provisioning for packet
-// formats other than just libpcap by designing a more abstract interface.
-//
-// Note that for serialization we don't use much of the support provided by
-// the serialization framework. Serialize/Unserialize do all the work by
-// themselves. In particular, Packets aren't derived from SerialObj. They are
-// completely seperate and self-contained entities, and we don't need any of
-// the sophisticated features like object caching.
-
-class Packet {
-public:
-	// Argument is whether we should delete associatd memory upon
-	// destruction.
-	Packet(TimerMgr::Tag arg_tag, bool arg_free = false)
-		{
-		time = 0.0;
-		hdr = 0;
-		pkt = 0;
-		hdr_size = 0;
-		free = arg_free;
-		tag = arg_tag;
-		link_type = 0;
-		}
-
-	~Packet()
-		{
-		if ( free )
-			{
-			delete hdr;
-			delete [] pkt;
-			}
-		}
-
-	const IP_Hdr IP() const
-		{ return IP_Hdr((struct ip *) (pkt + hdr_size), true); }
-
-	void Describe(ODesc* d) const;
-
-	bool Serialize(SerialInfo* info) const;
-	static Packet* Unserialize(UnserialInfo* info);
-
-	const struct pcap_pkthdr* hdr;
-	const u_char* pkt;
-	TimerMgr::Tag tag;
-	uint32 link_type;
-
-	double time;
-	int hdr_size;
-
-private:
-	bool free;
-};
-
 extern FileSerializer* event_serializer;
 extern FileSerializer* state_serializer;
 

src/Sessions.cc

@@ -165,98 +165,75 @@ void NetSessions::Done()
 	{
 	}
 
-void NetSessions::DispatchPacket(double t, const struct pcap_pkthdr* hdr,
+void NetSessions::NextPacket(double t, const Packet* pkt)
-			const u_char* pkt, int hdr_size,
-			iosource::PktSrc* src_ps)
 	{
-	const struct ip* ip_hdr = 0;
+	SegmentProfiler(segment_logger, "dispatching-packet");
-	const u_char* ip_data = 0;
-	int proto = 0;
 
-	if ( hdr->caplen >= hdr_size + sizeof(struct ip) )
+	if ( raw_packet )
 		{
-		ip_hdr = reinterpret_cast<const struct ip*>(pkt + hdr_size);
+		val_list* vl = new val_list();
-		if ( hdr->caplen >= unsigned(hdr_size + (ip_hdr->ip_hl << 2)) )
+		vl->append(pkt->BuildPktHdrVal());
-			ip_data = pkt + hdr_size + (ip_hdr->ip_hl << 2);
+		mgr.QueueEvent(raw_packet, vl);
 		}
 
-	if ( encap_hdr_size > 0 && ip_data )
-		// Blanket encapsulation
-		hdr_size += encap_hdr_size;
-
-	NextPacket(t, hdr, pkt, hdr_size);
-	}
-
-void NetSessions::NextPacket(double t, const struct pcap_pkthdr* hdr,
-			     const u_char* const pkt, int hdr_size)
-	{
-	SegmentProfiler(segment_logger, "processing-packet");
 	if ( pkt_profiler )
-		pkt_profiler->ProfilePkt(t, hdr->caplen);
+		pkt_profiler->ProfilePkt(t, pkt->cap_len);
 
 	++num_packets_processed;
 
 	dump_this_packet = 0;
 
 	if ( record_all_packets )
-		DumpPacket(hdr, pkt);
+		DumpPacket(pkt);
 
-	// ### The following isn't really correct.  What we *should*
+	if ( pkt->hdr_size > pkt->cap_len )
-	// do is understanding the different link layers in order to
-	// find the network-layer protocol ID.  That's a big
-	// portability pain, though, unless we just assume everything's
-	// Ethernet .... not great, given the potential need to deal
-	// with PPP or FDDI (for some older traces).  So instead
-	// we look to see if what we have is consistent with an
-	// IPv4 packet.  If not, it's either ARP or IPv6 or weird.
-
-	if ( hdr_size > static_cast<int>(hdr->caplen) )
 		{
-		Weird("truncated_link_frame", hdr, pkt);
+		Weird("truncated_link_frame", pkt);
 		return;
 		}
 
-	uint32 caplen = hdr->caplen - hdr_size;
+	uint32 caplen = pkt->cap_len - pkt->hdr_size;
-	if ( caplen < sizeof(struct ip) )
-		{
-		Weird("truncated_IP", hdr, pkt);
-		return;
-		}
 
-	const struct ip* ip = (const struct ip*) (pkt + hdr_size);
+	if ( pkt->l3_proto == L3_IPV4 )
-
-	if ( ip->ip_v == 4 )
 		{
-		IP_Hdr ip_hdr(ip, false);
+		if ( caplen < sizeof(struct ip) )
-		DoNextPacket(t, hdr, &ip_hdr, pkt, hdr_size, 0);
-		}
-
-	else if ( ip->ip_v == 6 )
-		{
-		if ( caplen < sizeof(struct ip6_hdr) )
 			{
-			Weird("truncated_IP", hdr, pkt);
+			Weird("truncated_IP", pkt);
 			return;
 			}
 
-		IP_Hdr ip_hdr((const struct ip6_hdr*) (pkt + hdr_size), false, caplen);
+		const struct ip* ip = (const struct ip*) (pkt->data + pkt->hdr_size);
-		DoNextPacket(t, hdr, &ip_hdr, pkt, hdr_size, 0);
+		IP_Hdr ip_hdr(ip, false);
+		DoNextPacket(t, pkt, &ip_hdr, 0);
 		}
 
-	else if ( analyzer::arp::ARP_Analyzer::IsARP(pkt, hdr_size) )
+	else if ( pkt->l3_proto == L3_IPV6 )
+		{
+		if ( caplen < sizeof(struct ip6_hdr) )
+			{
+			Weird("truncated_IP", pkt);
+			return;
+			}
+
+		IP_Hdr ip_hdr((const struct ip6_hdr*) (pkt->data + pkt->hdr_size), false, caplen);
+		DoNextPacket(t, pkt, &ip_hdr, 0);
+		}
+
+	else if ( pkt->l3_proto == L3_ARP )
 		{
 		if ( arp_analyzer )
-			arp_analyzer->NextPacket(t, hdr, pkt, hdr_size);
+			arp_analyzer->NextPacket(t, pkt);
 		}
 
 	else
 		{
-		Weird("unknown_packet_type", hdr, pkt);
+		Weird("unknown_packet_type", pkt);
 		return;
 		}
 
+
 	if ( dump_this_packet && ! record_all_packets )
-		DumpPacket(hdr, pkt);
+		DumpPacket(pkt);
 	}
 
 int NetSessions::CheckConnectionTag(Connection* conn)
@@ -337,26 +314,25 @@ static unsigned int gre_header_len(uint16 flags)
 	return len;
 	}
 
-void NetSessions::DoNextPacket(double t, const struct pcap_pkthdr* hdr,
+void NetSessions::DoNextPacket(double t, const Packet* pkt, const IP_Hdr* ip_hdr,
-				const IP_Hdr* ip_hdr, const u_char* const pkt,
+			       const EncapsulationStack* encapsulation)
-				int hdr_size, const EncapsulationStack* encapsulation)
 	{
-	uint32 caplen = hdr->caplen - hdr_size;
+	uint32 caplen = pkt->cap_len - pkt->hdr_size;
 	const struct ip* ip4 = ip_hdr->IP4_Hdr();
 
 	uint32 len = ip_hdr->TotalLen();
 	if ( len == 0 )
 		{
 		// TCP segmentation offloading can zero out the ip_len field.
-		Weird("ip_hdr_len_zero", hdr, pkt, encapsulation);
+		Weird("ip_hdr_len_zero", pkt, encapsulation);
 
 		// Cope with the zero'd out ip_len field by using the caplen.
-		len = hdr->caplen - hdr_size;
+		len = pkt->cap_len - pkt->hdr_size;
 		}
 
-	if ( hdr->len < len + hdr_size )
+	if ( pkt->len < len + pkt->hdr_size )
 		{
-		Weird("truncated_IP", hdr, pkt, encapsulation);
+		Weird("truncated_IP", pkt, encapsulation);
 		return;
 		}
 
@@ -368,7 +344,7 @@ void NetSessions::DoNextPacket(double t, const struct pcap_pkthdr* hdr,
 	if ( ! ignore_checksums && ip4 &&
 	     ones_complement_checksum((void*) ip4, ip_hdr_len, 0) != 0xffff )
 		{
-		Weird("bad_IP_checksum", hdr, pkt, encapsulation);
+		Weird("bad_IP_checksum", pkt, encapsulation);
 		return;
 		}
 
@@ -393,7 +369,7 @@ void NetSessions::DoNextPacket(double t, const struct pcap_pkthdr* hdr,
 			}
 		else
 			{
-			f = NextFragment(t, ip_hdr, pkt + hdr_size);
+			f = NextFragment(t, ip_hdr, pkt->data + pkt->hdr_size);
 			const IP_Hdr* ih = f->ReassembledPkt();
 			if ( ! ih )
 				// It didn't reassemble into anything yet.
@@ -437,7 +413,7 @@ void NetSessions::DoNextPacket(double t, const struct pcap_pkthdr* hdr,
 
 		if ( ! ignore_checksums && mobility_header_checksum(ip_hdr) != 0xffff )
 			{
-			Weird("bad_MH_checksum", hdr, pkt, encapsulation);
+			Weird("bad_MH_checksum", pkt, encapsulation);
 			return;
 			}
 
@@ -449,7 +425,7 @@ void NetSessions::DoNextPacket(double t, const struct pcap_pkthdr* hdr,
 			}
 
 		if ( ip_hdr->NextProto() != IPPROTO_NONE )
-			Weird("mobility_piggyback", hdr, pkt, encapsulation);
+			Weird("mobility_piggyback", pkt, encapsulation);
 
 		return;
 		}
@@ -457,7 +433,7 @@ void NetSessions::DoNextPacket(double t, const struct pcap_pkthdr* hdr,
 
 	int proto = ip_hdr->NextProto();
 
-	if ( CheckHeaderTrunc(proto, len, caplen, hdr, pkt, encapsulation) )
+	if ( CheckHeaderTrunc(proto, len, caplen, pkt, encapsulation) )
 		return;
 
 	const u_char* data = ip_hdr->Payload();
@@ -594,7 +570,7 @@ void NetSessions::DoNextPacket(double t, const struct pcap_pkthdr* hdr,
 
 			if ( ppp_proto != 0x0021 && ppp_proto != 0x0057 )
 				{
-				Weird("non_ip_packet_in_egre", ip_hdr, encapsulation);
+				Weird("non_ip_packet_in_encap", ip_hdr, encapsulation);
 				return;
 				}
 
@@ -664,7 +640,7 @@ void NetSessions::DoNextPacket(double t, const struct pcap_pkthdr* hdr,
 		else
 			it->second.second = network_time;
 
-		DoNextInnerPacket(t, hdr, inner, encapsulation,
+		DoNextInnerPacket(t, pkt, inner, encapsulation,
 		                  ip_tunnels[tunnel_idx].first);
 
 		return;
@@ -677,13 +653,13 @@ void NetSessions::DoNextPacket(double t, const struct pcap_pkthdr* hdr,
 		// not sure the reason for the No Next header in the packet.
 		if ( ! ( encapsulation &&
 		     encapsulation->LastType() == BifEnum::Tunnel::TEREDO ) )
-			Weird("ipv6_no_next", hdr, pkt);
+			Weird("ipv6_no_next", pkt);
 
 		return;
 		}
 
 	default:
-		Weird(fmt("unknown_protocol_%d", proto), hdr, pkt, encapsulation);
+		Weird(fmt("unknown_protocol_%d", proto), pkt, encapsulation);
 		return;
 	}
 
@@ -756,8 +732,7 @@ void NetSessions::DoNextPacket(double t, const struct pcap_pkthdr* hdr,
 		        pkt_hdr_val ? pkt_hdr_val->Ref() : ip_hdr->BuildPktHdrVal());
 
 	conn->NextPacket(t, is_orig, ip_hdr, len, caplen, data,
-				record_packet, record_content,
+				record_packet, record_content, pkt);
-			        hdr, pkt, hdr_size);
 
 	if ( f )
 		{
@@ -772,40 +747,53 @@ void NetSessions::DoNextPacket(double t, const struct pcap_pkthdr* hdr,
 
 		else
 			{
-			int hdr_len = data - pkt;
+			int hdr_len = data - pkt->data;
-			DumpPacket(hdr, pkt, hdr_len);	// just save the header
+			DumpPacket(pkt, hdr_len);	// just save the header
 			}
 		}
 	}
 
-void NetSessions::DoNextInnerPacket(double t, const struct pcap_pkthdr* hdr,
+void NetSessions::DoNextInnerPacket(double t, const Packet* pkt,
 		const IP_Hdr* inner, const EncapsulationStack* prev,
 		const EncapsulatingConn& ec)
 	{
-	struct pcap_pkthdr fake_hdr;
+	uint32 caplen, len;
-	fake_hdr.caplen = fake_hdr.len = inner->TotalLen();
+	caplen = len = inner->TotalLen();
 
-	if ( hdr )
+	struct timeval ts;
-		fake_hdr.ts = hdr->ts;
+	int link_type;
+	Layer3Proto l3_proto;
+
+	if ( pkt )
+		ts = pkt->ts;
 	else
 		{
-		fake_hdr.ts.tv_sec = (time_t) network_time;
+		ts.tv_sec = (time_t) network_time;
-		fake_hdr.ts.tv_usec = (suseconds_t)
+		ts.tv_usec = (suseconds_t)
-		    ((network_time - (double)fake_hdr.ts.tv_sec) * 1000000);
+		    ((network_time - (double)ts.tv_sec) * 1000000);
 		}
 
-	const u_char* pkt = 0;
+	const u_char* data = 0;
 
 	if ( inner->IP4_Hdr() )
-		pkt = (const u_char*) inner->IP4_Hdr();
+		{
+		data = (const u_char*) inner->IP4_Hdr();
+		l3_proto = L3_IPV4;
+		}
 	else
-		pkt = (const u_char*) inner->IP6_Hdr();
+		{
+		data = (const u_char*) inner->IP6_Hdr();
+		l3_proto = L3_IPV6;
+		}
 
 	EncapsulationStack* outer = prev ?
 			new EncapsulationStack(*prev) : new EncapsulationStack();
 	outer->Add(ec);
 
-	DoNextPacket(t, &fake_hdr, inner, pkt, 0, outer);
+	// Construct fake packet for DoNextPacket
+	Packet p;
+	p.Init(DLT_RAW, &ts, caplen, len, data, false, "");
+	DoNextPacket(t, &p, inner, outer);
 
 	delete inner;
 	delete outer;
@@ -843,8 +831,7 @@ int NetSessions::ParseIPPacket(int caplen, const u_char* const pkt, int proto,
 	}
 
 bool NetSessions::CheckHeaderTrunc(int proto, uint32 len, uint32 caplen,
-                                   const struct pcap_pkthdr* h,
+                                   const Packet* p, const EncapsulationStack* encap)
-                                   const u_char* p, const EncapsulationStack* encap)
 	{
 	uint32 min_hdr_len = 0;
 	switch ( proto ) {
@@ -876,13 +863,13 @@ bool NetSessions::CheckHeaderTrunc(int proto, uint32 len, uint32 caplen,
 
 	if ( len < min_hdr_len )
 		{
-		Weird("truncated_header", h, p, encap);
+		Weird("truncated_header", p, encap);
 		return true;
 		}
 
 	if ( caplen < min_hdr_len )
 		{
-		Weird("internally_truncated_header", h, p, encap);
+		Weird("internally_truncated_header", p, encap);
 		return true;
 		}
 
@@ -1387,45 +1374,26 @@ void NetSessions::ExpireTimerMgrs()
 		}
 	}
 
-void NetSessions::DumpPacket(const struct pcap_pkthdr* hdr,
+void NetSessions::DumpPacket(const Packet *pkt, int len)
-				const u_char* pkt, int len)
 	{
 	if ( ! pkt_dumper )
 		return;
 
-	if ( len == 0 )
+	if ( len != 0 )
 		{
-		iosource::PktDumper::Packet p;
+		if ( (uint32)len > pkt->cap_len )
-		p.hdr = hdr;
+			reporter->Warning("bad modified caplen");
-		p.data = pkt;
+		else
-		pkt_dumper->Dump(&p);
+			const_cast<Packet *>(pkt)->cap_len = len;
 		}
 
-	else
+	pkt_dumper->Dump(pkt);
-		{
-		struct pcap_pkthdr h = *hdr;
-		h.caplen = len;
-		if ( h.caplen > hdr->caplen )
-			reporter->InternalError("bad modified caplen");
-
-		iosource::PktDumper::Packet p;
-		p.hdr = &h;
-		p.data = pkt;
-		pkt_dumper->Dump(&p);
-		}
 	}
 
-void NetSessions::Internal(const char* msg, const struct pcap_pkthdr* hdr,
+void NetSessions::Weird(const char* name, const Packet* pkt,
-				const u_char* pkt)
+                        const EncapsulationStack* encap)
 	{
-	DumpPacket(hdr, pkt);
+	if ( pkt )
-	reporter->InternalError("%s", msg);
-	}
-
-void NetSessions::Weird(const char* name, const struct pcap_pkthdr* hdr,
-                        const u_char* pkt, const EncapsulationStack* encap)
-	{
-	if ( hdr )
 		dump_this_packet = 1;
 
 	if ( encap && encap->LastType() != BifEnum::Tunnel::NONE )

src/Sessions.h

@@ -15,8 +15,6 @@
 
 #include <utility>
 
-struct pcap_pkthdr;
-
 class EncapsulationStack;
 class Connection;
 class OSFingerprint;
@@ -68,12 +66,8 @@ public:
 	NetSessions();
 	~NetSessions();
 
-	// Main entry point for packet processing. Dispatches the packet
+	// Main entry point for packet processing.
-	// either through NextPacket(), optionally employing the packet
+	void NextPacket(double t, const Packet* pkt);
-	// sorter first.
-	void DispatchPacket(double t, const struct pcap_pkthdr* hdr,
-			const u_char* const pkt, int hdr_size,
-			iosource::PktSrc* src_ps);
 
 	void Done();	// call to drain events before destructing
 
@@ -106,8 +100,8 @@ public:
 
 	void GetStats(SessionStats& s) const;
 
-	void Weird(const char* name, const struct pcap_pkthdr* hdr,
+	void Weird(const char* name, const Packet* pkt,
-	    const u_char* pkt, const EncapsulationStack* encap = 0);
+	    const EncapsulationStack* encap = 0);
 	void Weird(const char* name, const IP_Hdr* ip,
 	    const EncapsulationStack* encap = 0);
 
@@ -133,9 +127,8 @@ public:
 			icmp_conns.Length();
 		}
 
-	void DoNextPacket(double t, const struct pcap_pkthdr* hdr,
+	void DoNextPacket(double t, const Packet *pkt, const IP_Hdr* ip_hdr,
-			const IP_Hdr* ip_hdr, const u_char* const pkt,
+			const EncapsulationStack* encapsulation);
-			int hdr_size, const EncapsulationStack* encapsulation);
 
 	/**
 	 * Wrapper that recurses on DoNextPacket for encapsulated IP packets.
@@ -151,7 +144,7 @@ public:
 	 *        the most-recently found depth of encapsulation.
 	 * @param ec The most-recently found depth of encapsulation.
 	 */
-	void DoNextInnerPacket(double t, const struct pcap_pkthdr* hdr,
+	void DoNextInnerPacket(double t, const Packet *pkt,
 	                      const IP_Hdr* inner, const EncapsulationStack* prev,
 	                      const EncapsulatingConn& ec);
 
@@ -218,24 +211,16 @@ protected:
 				TransportProto transport_proto,
 				uint8 tcp_flags, bool& flip_roles);
 
-	void NextPacket(double t, const struct pcap_pkthdr* hdr,
-			const u_char* const pkt, int hdr_size);
-
 	// Record the given packet (if a dumper is active).  If len=0
 	// then the whole packet is recorded, otherwise just the first
 	// len bytes.
-	void DumpPacket(const struct pcap_pkthdr* hdr, const u_char* pkt,
+	void DumpPacket(const Packet *pkt, int len=0);
-			int len=0);
-
-	void Internal(const char* msg, const struct pcap_pkthdr* hdr,
-			const u_char* pkt);
 
 	// For a given protocol, checks whether the header's length as derived
 	// from lower-level headers or the length actually captured is less
 	// than that protocol's minimum header size.
 	bool CheckHeaderTrunc(int proto, uint32 len, uint32 caplen,
-			      const struct pcap_pkthdr* hdr, const u_char* pkt,
+			      const Packet *pkt, const EncapsulationStack* encap);
-			      const EncapsulationStack* encap);
 
 	CompositeHash* ch;
 	PDict(Connection) tcp_conns;

src/analyzer/protocol/arp/ARP.cc

@@ -19,21 +19,6 @@ ARP_Analyzer::~ARP_Analyzer()
 	{
 	}
 
-bool ARP_Analyzer::IsARP(const u_char* pkt, int hdr_size)
-	{
-	unsigned short network_protocol =
-		*(unsigned short*) (pkt + hdr_size - 2);
-
-	switch ( ntohs(network_protocol) ) {
-	case ETHERTYPE_ARP:
-	case ETHERTYPE_REVARP:
-		return true;
-	default:
-		return false;
-	}
-	}
-
-
 // Argh! FreeBSD and Linux have almost completely different net/if_arp.h .
 // ... and on Solaris we are missing half of the ARPOP codes, so define
 // them here as necessary:
@@ -93,16 +78,16 @@ bool ARP_Analyzer::IsARP(const u_char* pkt, int hdr_size)
 #endif
 
 
-void ARP_Analyzer::NextPacket(double t, const struct pcap_pkthdr* hdr,
+void ARP_Analyzer::NextPacket(double t, const Packet* pkt)
-				const u_char* const pkt, int hdr_size)
 	{
+	const u_char *data = pkt->data;
 	// Check whether the packet is OK ("inspired" in tcpdump's print-arp.c).
 	const struct arp_pkthdr* ah =
-		(const struct arp_pkthdr*) (pkt + hdr_size);
+		(const struct arp_pkthdr*) (data + pkt->hdr_size);
 
 	// Check the size.
-	int min_length = (ar_tpa(ah) - (char*) (pkt + hdr_size)) + ah->ar_pln;
+	int min_length = (ar_tpa(ah) - (char*) (data + pkt->hdr_size)) + ah->ar_pln;
-	int real_length = hdr->caplen - hdr_size;
+	int real_length = pkt->cap_len - pkt->hdr_size;
 	if ( min_length > real_length )
 		{
 		Corrupted("truncated_ARP");
@@ -158,7 +143,7 @@ void ARP_Analyzer::NextPacket(double t, const struct pcap_pkthdr* hdr,
 
 
 	// Check MAC src address = ARP sender MAC address.
-	if ( memcmp((const char*) (pkt+6), ar_sha(ah), ah->ar_hln) )
+	if ( memcmp((const char*) (data+6), ar_sha(ah), ah->ar_hln) )
 		{
 		BadARP(ah, "weird-arp-sha");
 		return;
@@ -167,12 +152,12 @@ void ARP_Analyzer::NextPacket(double t, const struct pcap_pkthdr* hdr,
 	// Check the code is supported.
 	switch ( ntohs(ah->ar_op) ) {
 	case ARPOP_REQUEST:
-		RREvent(arp_request, pkt+6, pkt,
+		RREvent(arp_request, data+6, data,
 				ar_spa(ah), ar_sha(ah), ar_tpa(ah), ar_tha(ah));
 		break;
 
 	case ARPOP_REPLY:
-		RREvent(arp_reply, pkt+6, pkt,
+		RREvent(arp_reply, data+6, data,
 				ar_spa(ah), ar_sha(ah), ar_tpa(ah), ar_tha(ah));
 		break;
 

src/analyzer/protocol/arp/ARP.h

@@ -25,6 +25,8 @@
 
 #include "NetVar.h"
 
+class Packet;
+
 extern "C" {
 #include <pcap.h>
 }
@@ -36,17 +38,13 @@ public:
 	ARP_Analyzer();
 	virtual ~ARP_Analyzer();
 
-	void NextPacket(double t, const struct pcap_pkthdr* hdr,
+	void NextPacket(double t, const Packet* pkt);
-			const u_char* const pkt, int hdr_size);
 
 	void Describe(ODesc* d) const;
 	void RREvent(EventHandlerPtr e, const u_char* src, const u_char* dst,
 			const char* spa, const char* sha,
 			const char* tpa, const char* tha);
 
-	// Whether a packet is of interest for ARP analysis.
-	static bool IsARP(const u_char* pkt, int hdr_size);
-
 protected:
 	AddrVal* ConstructAddrVal(const void* addr);
 	StringVal* EthAddrToStr(const u_char* addr);

src/analyzer/protocol/http/HTTP.cc

@@ -1025,8 +1025,11 @@ void HTTP_Analyzer::DeliverStream(int len, const u_char* data, bool is_orig)
 				}
 			else
 				{
-				ProtocolViolation("not a http reply line");
+				if ( line != end_of_line )
-				reply_state = EXPECT_REPLY_NOTHING;
+					{
+					ProtocolViolation("not a http reply line");
+					reply_state = EXPECT_REPLY_NOTHING;
+					}
 				}
 
 			break;

src/analyzer/protocol/mime/MIME.cc

@@ -245,11 +245,16 @@ int MIME_get_field_name(int len, const char* data, data_chunk_t* name)
 	}
 
 // See RFC 2045, page 12.
-int MIME_is_tspecial (char ch)
+int MIME_is_tspecial (char ch, bool is_boundary = false)
 	{
-	return ch == '(' || ch == ')' || ch == '<' || ch == '>' || ch == '@' ||
+	if ( is_boundary )
-	       ch == ',' || ch == ';' || ch == ':' || ch == '\\' || ch == '"' ||
+		return ch == '(' || ch == ')' || ch == '@' ||
-	       ch == '/' || ch == '[' || ch == ']' || ch == '?' || ch == '=';
+		       ch == ',' || ch == ';' || ch == ':' || ch == '\\' || ch == '"' ||
+		       ch == '/' || ch == '[' || ch == ']' || ch == '?' || ch == '=';
+	else
+		return ch == '(' || ch == ')' || ch == '<' || ch == '>' || ch == '@' ||
+		       ch == ',' || ch == ';' || ch == ':' || ch == '\\' || ch == '"' ||
+		       ch == '/' || ch == '[' || ch == ']' || ch == '?' || ch == '=';
 	}
 
 int MIME_is_field_name_char (char ch)
@@ -257,26 +262,27 @@ int MIME_is_field_name_char (char ch)
 	return ch >= 33 && ch <= 126 && ch != ':';
 	}
 
-int MIME_is_token_char (char ch)
+int MIME_is_token_char (char ch, bool is_boundary = false)
 	{
-	return ch >= 33 && ch <= 126 && ! MIME_is_tspecial(ch);
+	return ch >= 33 && ch <= 126 && ! MIME_is_tspecial(ch, is_boundary);
 	}
 
 // See RFC 2045, page 12.
 // A token is composed of characters that are not SPACE, CTLs or tspecials
-int MIME_get_token(int len, const char* data, data_chunk_t* token)
+int MIME_get_token(int len, const char* data, data_chunk_t* token,
+                   bool is_boundary)
 	{
 	int i = MIME_skip_lws_comments(len, data);
 	while ( i < len )
 		{
 		int j;
 
-		if ( MIME_is_token_char(data[i]) )
+		if ( MIME_is_token_char(data[i], is_boundary) )
 			{
 			token->data = (data + i);
 			for ( j = i; j < len; ++j )
 				{
-				if ( ! MIME_is_token_char(data[j]) )
+				if ( ! MIME_is_token_char(data[j], is_boundary) )
 					break;
 				}
 
@@ -358,7 +364,7 @@ int MIME_get_quoted_string(int len, const char* data, data_chunk_t* str)
 	return -1;
 	}
 
-int MIME_get_value(int len, const char* data, BroString*& buf)
+int MIME_get_value(int len, const char* data, BroString*& buf, bool is_boundary)
 	{
 	int offset = MIME_skip_lws_comments(len, data);
 
@@ -379,7 +385,7 @@ int MIME_get_value(int len, const char* data, BroString*& buf)
 	else
 		{
 		data_chunk_t str;
-		int end = MIME_get_token(len, data, &str);
+		int end = MIME_get_token(len, data, &str, is_boundary);
 		if ( end < 0 )
 			return -1;
 
@@ -862,8 +868,22 @@ int MIME_Entity::ParseFieldParameters(int len, const char* data)
 		len -= offset;
 
 		BroString* val = 0;
-		// token or quoted-string
+
-		offset = MIME_get_value(len, data, val);
+		if ( current_field_type == MIME_CONTENT_TYPE &&
+		     content_type == CONTENT_TYPE_MULTIPART &&
+		     strcasecmp_n(attr, "boundary") == 0 )
+			{
+			// token or quoted-string (and some lenience for characters
+			// not explicitly allowed by the RFC, but encountered in the wild)
+			offset = MIME_get_value(len, data, val, true);
+			data_chunk_t vd = get_data_chunk(val);
+			multipart_boundary = new BroString((const u_char*)vd.data,
+			                                   vd.length, 1);
+			}
+		else
+			// token or quoted-string
+			offset = MIME_get_value(len, data, val);
+
 		if ( offset < 0 )
 			{
 			IllegalFormat("value not found in parameter specification");
@@ -873,8 +893,6 @@ int MIME_Entity::ParseFieldParameters(int len, const char* data)
 
 		data += offset;
 		len -= offset;
-
-		ParseParameter(attr, get_data_chunk(val));
 		delete val;
 		}
 
@@ -919,24 +937,6 @@ void MIME_Entity::ParseContentEncoding(data_chunk_t encoding_mechanism)
 	content_encoding = i;
 	}
 
-void MIME_Entity::ParseParameter(data_chunk_t attr, data_chunk_t val)
-	{
-	switch ( current_field_type ) {
-		case MIME_CONTENT_TYPE:
-			if ( content_type == CONTENT_TYPE_MULTIPART &&
-			     strcasecmp_n(attr, "boundary") == 0 )
-				multipart_boundary = new BroString((const u_char*)val.data, val.length, 1);
-			break;
-
-		case MIME_CONTENT_TRANSFER_ENCODING:
-			break;
-
-		default:
-			break;
-	}
-	}
-
-
 int MIME_Entity::CheckBoundaryDelimiter(int len, const char* data)
 	{
 	if ( ! multipart_boundary )

src/analyzer/protocol/mime/MIME.h

@@ -117,7 +117,6 @@ protected:
 
 	void ParseContentType(data_chunk_t type, data_chunk_t sub_type);
 	void ParseContentEncoding(data_chunk_t encoding_mechanism);
-	void ParseParameter(data_chunk_t attr, data_chunk_t val);
 
 	void BeginBody();
 	void NewDataLine(int len, const char* data, int trailing_CRLF);
@@ -276,9 +275,11 @@ extern int MIME_count_leading_lws(int len, const char* data);
 extern int MIME_count_trailing_lws(int len, const char* data);
 extern int MIME_skip_comments(int len, const char* data);
 extern int MIME_skip_lws_comments(int len, const char* data);
-extern int MIME_get_token(int len, const char* data, data_chunk_t* token);
+extern int MIME_get_token(int len, const char* data, data_chunk_t* token,
+                          bool is_boundary = false);
 extern int MIME_get_slash_token_pair(int len, const char* data, data_chunk_t* first, data_chunk_t* second);
-extern int MIME_get_value(int len, const char* data, BroString*& buf);
+extern int MIME_get_value(int len, const char* data, BroString*& buf,
+                          bool is_boundary = false);
 extern int MIME_get_field_name(int len, const char* data, data_chunk_t* name);
 extern BroString* MIME_decode_quoted_pairs(data_chunk_t buf);
 

src/analyzer/protocol/ssl/dtls-analyzer.pac

@@ -55,7 +55,7 @@ refine connection SSL_Conn += {
 
 		if ( length > MAX_DTLS_HANDSHAKE_RECORD )
 			{
-			bro_analyzer()->ProtocolViolation(fmt("DTLS record length %lld larger than allowed maximum.", length));
+			bro_analyzer()->ProtocolViolation(fmt("DTLS record length %" PRId64 " larger than allowed maximum.", length));
 			return true;
 			}
 

src/analyzer/protocol/tcp/TCP_Endpoint.cc

@@ -201,13 +201,18 @@ int TCP_Endpoint::DataSent(double t, uint64 seq, int len, int caplen,
 	{
 	int status = 0;
 
-	if ( contents_processor && caplen >= len )
+	if ( contents_processor )
-		status = contents_processor->DataSent(t, seq, len, data);
+		{
+		if ( caplen >= len )
+			status = contents_processor->DataSent(t, seq, len, data);
+		else
+			TCP()->Weird("truncated_tcp_payload");
+		}
 
 	if ( caplen <= 0 )
 		return status;
 
-	if ( contents_file && ! contents_processor && 
+	if ( contents_file && ! contents_processor &&
 	     seq + len > contents_start_seq )
 		{
 		int64 under_seq = contents_start_seq - seq;

src/analyzer/protocol/tcp/TCP_Reassembler.cc

@@ -42,6 +42,9 @@ TCP_Reassembler::TCP_Reassembler(analyzer::Analyzer* arg_dst_analyzer,
 	seq_to_skip = 0;
 	in_delivery = false;
 
+	if ( tcp_max_old_segments )
+		SetMaxOldBlocks(tcp_max_old_segments);
+
 	if ( tcp_contents )
 		{
 		// Val dst_port_val(ntohs(Conn()->RespPort()), TYPE_PORT);

src/analyzer/protocol/zip/ZIP.cc

@@ -22,10 +22,9 @@ ZIP_Analyzer::ZIP_Analyzer(Connection* conn, bool orig, Method arg_method)
 	zip->next_in = 0;
 	zip->avail_in = 0;
 
-	// "15" here means maximum compression.  "32" is a gross overload
+	// "32" is a gross overload hack that means "check it
-	// hack that means "check it for whether it's a gzip file".  Sheesh.
+	// for whether it's a gzip file".  Sheesh.
-	zip_status = inflateInit2(zip, 15 + 32);
+	if ( inflateInit2(zip, MAX_WBITS + 32) != Z_OK )
-	if ( zip_status != Z_OK )
 		{
 		Weird("inflate_init_failed");
 		delete zip;
@@ -56,38 +55,63 @@ void ZIP_Analyzer::DeliverStream(int len, const u_char* data, bool orig)
 	static unsigned int unzip_size = 4096;
 	Bytef unzipbuf[unzip_size];
 
+	int allow_restart = 1;
+
 	zip->next_in = (Bytef*) data;
 	zip->avail_in = len;
 
-	do
+	Bytef *orig_next_in = zip->next_in;
+	size_t orig_avail_in = zip->avail_in;
+
+	while ( true )
 		{
 		zip->next_out = unzipbuf;
 		zip->avail_out = unzip_size;
 
 		zip_status = inflate(zip, Z_SYNC_FLUSH);
 
-		if ( zip_status != Z_STREAM_END &&
+		if ( zip_status == Z_STREAM_END ||
-		     zip_status != Z_OK &&
+		     zip_status == Z_OK )
-		     zip_status != Z_BUF_ERROR )
+			{
+			allow_restart = 0;
+
+			int have = unzip_size - zip->avail_out;
+			if ( have )
+				ForwardStream(have, unzipbuf, IsOrig());
+
+			if ( zip_status == Z_STREAM_END )
+				{
+				inflateEnd(zip);
+				return;
+				}
+
+			if ( zip->avail_in == 0 )
+				return;
+
+			}
+
+		else if ( allow_restart && zip_status == Z_DATA_ERROR )
+			{
+			// Some servers seem to not generate zlib headers,
+			// so this is an attempt to fix and continue anyway.
+			inflateEnd(zip);
+
+			if ( inflateInit2(zip, -MAX_WBITS) != Z_OK )
+				{
+				Weird("inflate_init_failed");
+				return;
+				}
+
+			zip->next_in = orig_next_in;
+			zip->avail_in = orig_avail_in;
+			allow_restart = 0;
+			continue;
+			}
+
+		else
 			{
 			Weird("inflate_failed");
-			inflateEnd(zip);
+			return;
-			break;
 			}
-
-		int have = unzip_size - zip->avail_out;
-		if ( have )
-			ForwardStream(have, unzipbuf, IsOrig());
-
-		if ( zip_status == Z_STREAM_END )
-			{
-			inflateEnd(zip);
-			delete zip;
-			zip = 0;
-			break;
-			}
-
-		zip_status = Z_OK;
 		}
-	while ( zip->avail_out == 0 );
 	}

src/bro.bif

@@ -22,6 +22,7 @@
 #include "util.h"
 #include "file_analysis/Manager.h"
 #include "iosource/Manager.h"
+#include "iosource/Packet.h"
 
 using namespace std;
 
@@ -3238,11 +3239,10 @@ const char* conn_id_string(Val* c)
 ## .. bro:see:: dump_packet get_current_packet send_current_packet
 function dump_current_packet%(file_name: string%) : bool
 	%{
-	const struct pcap_pkthdr* hdr;
+	const Packet* pkt;
-	const u_char* pkt;
 
 	if ( ! current_pktsrc ||
-	     ! current_pktsrc->GetCurrentPacket(&hdr, &pkt) )
+	     ! current_pktsrc->GetCurrentPacket(&pkt) )
 		return new Val(0, TYPE_BOOL);
 
 	if ( ! addl_pkt_dumper )
@@ -3250,13 +3250,10 @@ function dump_current_packet%(file_name: string%) : bool
 
 	if ( addl_pkt_dumper )
 		{
-		iosource::PktDumper::Packet p;
+		addl_pkt_dumper->Dump(pkt);
-		p.hdr = hdr;
-		p.data = pkt;
-		addl_pkt_dumper->Dump(&p);
 		}
 
-	return new Val(! addl_pkt_dumper->IsError(), TYPE_BOOL);
+	return new Val( addl_pkt_dumper && ! addl_pkt_dumper->IsError(), TYPE_BOOL);
 	%}
 
 ## Returns the currently processed PCAP packet.
@@ -3267,26 +3264,27 @@ function dump_current_packet%(file_name: string%) : bool
 ## .. bro:see:: dump_current_packet dump_packet send_current_packet
 function get_current_packet%(%) : pcap_packet
 	%{
-	const struct pcap_pkthdr* hdr;
+	const Packet* p;
-	const u_char* data;
 	RecordVal* pkt = new RecordVal(pcap_packet);
 
 	if ( ! current_pktsrc ||
-	     ! current_pktsrc->GetCurrentPacket(&hdr, &data) )
+	     ! current_pktsrc->GetCurrentPacket(&p) )
 		{
 		pkt->Assign(0, new Val(0, TYPE_COUNT));
 		pkt->Assign(1, new Val(0, TYPE_COUNT));
 		pkt->Assign(2, new Val(0, TYPE_COUNT));
 		pkt->Assign(3, new Val(0, TYPE_COUNT));
 		pkt->Assign(4, new StringVal(""));
+		pkt->Assign(5, new EnumVal(BifEnum::LINK_UNKNOWN, BifType::Enum::link_encap));
 		return pkt;
 		}
 
-	pkt->Assign(0, new Val(uint32(hdr->ts.tv_sec), TYPE_COUNT));
+	pkt->Assign(0, new Val(uint32(p->ts.tv_sec), TYPE_COUNT));
-	pkt->Assign(1, new Val(uint32(hdr->ts.tv_usec), TYPE_COUNT));
+	pkt->Assign(1, new Val(uint32(p->ts.tv_usec), TYPE_COUNT));
-	pkt->Assign(2, new Val(hdr->caplen, TYPE_COUNT));
+	pkt->Assign(2, new Val(p->cap_len, TYPE_COUNT));
-	pkt->Assign(3, new Val(hdr->len, TYPE_COUNT));
+	pkt->Assign(3, new Val(p->len, TYPE_COUNT));
-	pkt->Assign(4, new StringVal(hdr->caplen, (const char*) data));
+	pkt->Assign(4, new StringVal(p->cap_len, (const char*)p->data));
+	pkt->Assign(5, new EnumVal(p->link_type, BifType::Enum::link_encap));
 
 	return pkt;
 	%}
@@ -3302,26 +3300,29 @@ function get_current_packet%(%) : pcap_packet
 ## .. bro:see:: get_current_packet dump_current_packet send_current_packet
 function dump_packet%(pkt: pcap_packet, file_name: string%) : bool
 	%{
-	struct pcap_pkthdr hdr;
-	const val_list* pkt_vl = pkt->AsRecord();
-
-	hdr.ts.tv_sec = (*pkt_vl)[0]->AsCount();
-	hdr.ts.tv_usec = (*pkt_vl)[1]->AsCount();
-	hdr.caplen = (*pkt_vl)[2]->AsCount();
-	hdr.len = (*pkt_vl)[3]->AsCount();
-
 	if ( ! addl_pkt_dumper )
 		addl_pkt_dumper = iosource_mgr->OpenPktDumper(file_name->CheckString(), true);
 
 	if ( addl_pkt_dumper )
 		{
-		iosource::PktDumper::Packet p;
+		struct timeval ts;
-		p.hdr = &hdr;
+		uint32 caplen, len, link_type;
-		p.data = (*pkt_vl)[4]->AsString()->Bytes();
+		u_char *data;
+
+		const val_list* pkt_vl = pkt->AsRecord();
+
+		ts.tv_sec = (*pkt_vl)[0]->AsCount();
+		ts.tv_usec = (*pkt_vl)[1]->AsCount();
+		caplen = (*pkt_vl)[2]->AsCount();
+		len = (*pkt_vl)[3]->AsCount();
+		data = (*pkt_vl)[4]->AsString()->Bytes();
+		link_type = (*pkt_vl)[5]->AsEnum();
+		Packet p(link_type, &ts, caplen, len, data, true);
+
 		addl_pkt_dumper->Dump(&p);
 		}
 
-	return new Val(addl_pkt_dumper->IsError(), TYPE_BOOL);
+	return new Val(addl_pkt_dumper && ! addl_pkt_dumper->IsError(), TYPE_BOOL);
 	%}
 
 %%{
@@ -4800,20 +4801,16 @@ function send_ping%(p: event_peer, seq: count%) : bool
 ##              dump_packet dump_current_packet get_current_packet
 function send_current_packet%(p: event_peer%) : bool
 	%{
-	Packet pkt("");
+	const Packet* pkt;
 
 	if ( ! current_pktsrc ||
-	     ! current_pktsrc->GetCurrentPacket(&pkt.hdr, &pkt.pkt) )
+	     ! current_pktsrc->GetCurrentPacket(&pkt) )
 		return new Val(0, TYPE_BOOL);
 
 	RemoteSerializer::PeerID id = p->AsRecordVal()->Lookup(0)->AsCount();
 
-	pkt.time = pkt.hdr->ts.tv_sec + double(pkt.hdr->ts.tv_usec) / 1e6;
-	pkt.hdr_size = current_pktsrc->HdrSize();
-	pkt.link_type = current_pktsrc->LinkType();
-
 	SerialInfo info(remote_serializer);
-	return new Val(remote_serializer->SendPacket(&info, id, pkt), TYPE_BOOL);
+	return new Val(remote_serializer->SendPacket(&info, id, *pkt), TYPE_BOOL);
 	%}
 
 ## Returns the peer who generated the last event.

src/event.bif

@@ -225,17 +225,31 @@ event udp_session_done%(u: connection%);
 ##    ``ANALYZER_*`` constants right now.
 event scheduled_analyzer_applied%(c: connection, a: Analyzer::Tag%);
 
-## Generated for every packet Bro sees. This is a very low-level and expensive
+## Generated for every packet Bro sees that have a valid link-layer header. This
-## event that should be avoided when at all possible. It's usually infeasible to
+## is a very very low-level and expensive event that should be avoided when at all
-## handle when processing even medium volumes of traffic in real-time. That
+## possible. It's usually infeasible to handle when processing even medium volumes
-## said, if you work from a trace and want to do some packet-level analysis,
+## of traffic in real-time. That said, if you work from a trace and want to do some
-## it may come in handy.
+## packet-level analysis, it may come in handy.
+##
+## p: Information from the header of the packet that triggered the event.
+##
+## .. bro:see:: new_packet packet_contents
+event raw_packet%(p: raw_pkt_hdr%);
+
+## Generated for all packets that make it into Bro's connection processing. In
+## contrast to :bro:id:`raw_packet` this filters out some more packets that don't
+## pass certain sanity checks.
+##
+## This is a very low-level and expensive event that should be avoided when at all
+## possible. It's usually infeasible to handle when processing even medium volumes
+## of traffic in real-time. That said, if you work from a trace and want to do some
+## packet-level analysis, it may come in handy.
 ##
 ## c: The connection the packet is part of.
 ##
 ## p: Information from the header of the packet that triggered the event.
 ##
-## .. bro:see:: tcp_packet packet_contents
+## .. bro:see:: tcp_packet packet_contents raw_packet
 event new_packet%(c: connection, p: pkt_hdr%);
 
 ## Generated for every IPv6 packet that contains extension headers.
@@ -282,7 +296,8 @@ event packet_contents%(c: connection, contents: string%);
 ## reassembling a TCP stream, Bro buffers all payload until it sees the
 ## responder acking it. If during that time, the sender resends a chunk of
 ## payload but with different content than originally, this event will be
-## raised.
+## raised. In addition, if :bro:id:`tcp_max_old_segments` is larger than zero,
+## mismatches with that older still-buffered data will likewise trigger the event.
 ##
 ## c: The connection showing the inconsistency.
 ##

src/iosource/CMakeLists.txt

@@ -12,6 +12,7 @@ set(iosource_SRCS
     BPF_Program.cc
     Component.cc
     Manager.cc
+    Packet.cc
     PktDumper.cc
     PktSrc.cc
 )

src/iosource/Packet.cc

@@ -0,0 +1,477 @@
+
+#include "Packet.h"
+#include "Sessions.h"
+#include "iosource/Manager.h"
+
+extern "C" {
+#ifdef HAVE_NET_ETHERNET_H
+#include <net/ethernet.h>
+#elif defined(HAVE_SYS_ETHERNET_H)
+#include <sys/ethernet.h>
+#elif defined(HAVE_NETINET_IF_ETHER_H)
+#include <netinet/if_ether.h>
+#elif defined(HAVE_NET_ETHERTYPES_H)
+#include <net/ethertypes.h>
+#endif
+}
+
+void Packet::Init(int arg_link_type, struct timeval *arg_ts, uint32 arg_caplen,
+		  uint32 arg_len, const u_char *arg_data, int arg_copy,
+		  std::string arg_tag)
+	{
+	if ( data && copy )
+		delete [] data;
+
+	link_type = arg_link_type;
+	ts = *arg_ts;
+	cap_len = arg_caplen;
+	len = arg_len;
+	tag = arg_tag;
+
+	copy = arg_copy;
+
+	if ( arg_data && arg_copy )
+		{
+		data = new u_char[arg_caplen];
+		memcpy(const_cast<u_char *>(data), arg_data, arg_caplen);
+		}
+	else
+		data = arg_data;
+
+	time = ts.tv_sec + double(ts.tv_usec) / 1e6;
+	hdr_size = GetLinkHeaderSize(arg_link_type);
+	l3_proto = L3_UNKNOWN;
+	eth_type = 0;
+	vlan = 0;
+
+	l2_valid = false;
+
+	if ( data )
+		ProcessLayer2();
+	}
+
+void Packet::Weird(const char* name)
+	{
+	sessions->Weird(name, this);
+	l2_valid = false;
+	}
+
+int Packet::GetLinkHeaderSize(int link_type)
+	{
+	switch ( link_type ) {
+	case DLT_NULL:
+		return 4;
+
+	case DLT_EN10MB:
+		return 14;
+
+	case DLT_FDDI:
+		return 13 + 8;	// fddi_header + LLC
+
+#ifdef DLT_LINUX_SLL
+	case DLT_LINUX_SLL:
+		return 16;
+#endif
+
+	case DLT_PPP_SERIAL:	// PPP_SERIAL
+		return 4;
+
+	case DLT_RAW:
+		return 0;
+	}
+
+	return -1;
+	}
+
+void Packet::ProcessLayer2()
+	{
+	l2_valid = true;
+
+	// Unfortunately some packets on the link might have MPLS labels
+	// while others don't. That means we need to ask the link-layer if
+	// labels are in place.
+	bool have_mpls = false;
+
+	const u_char* pdata = data;
+
+	switch ( link_type ) {
+	case DLT_NULL:
+		{
+		int protocol = (pdata[3] << 24) + (pdata[2] << 16) + (pdata[1] << 8) + pdata[0];
+		pdata += GetLinkHeaderSize(link_type);
+
+		// From the Wireshark Wiki: "AF_INET6, unfortunately, has
+		// different values in {NetBSD,OpenBSD,BSD/OS},
+		// {FreeBSD,DragonFlyBSD}, and {Darwin/Mac OS X}, so an IPv6
+		// packet might have a link-layer header with 24, 28, or 30
+		// as the AF_ value." As we may be reading traces captured on
+		// platforms other than what we're running on, we accept them
+		// all here.
+
+		if ( protocol == AF_INET )
+			l3_proto = L3_IPV4;
+		else if ( protocol == 24 || protocol == 28 || protocol == 30 )
+			l3_proto = L3_IPV6;
+		else
+			{
+			Weird("non_ip_packet_in_null_transport");
+			return;
+			}
+
+		break;
+		}
+
+	case DLT_EN10MB:
+		{
+		// Get protocol being carried from the ethernet frame.
+		int protocol = (pdata[12] << 8) + pdata[13];
+		pdata += GetLinkHeaderSize(link_type);
+		eth_type = protocol;
+
+		switch ( protocol )
+			{
+			// MPLS carried over the ethernet frame.
+			case 0x8847:
+				have_mpls = true;
+				break;
+
+			// VLAN carried over the ethernet frame.
+			// 802.1q / 802.1ad
+			case 0x8100:
+			case 0x9100:
+				vlan = ((pdata[0] << 8) + pdata[1]) & 0xfff;
+				protocol = ((pdata[2] << 8) + pdata[3]);
+				pdata += 4; // Skip the vlan header
+
+				// Check for MPLS in VLAN.
+				if ( protocol == 0x8847 )
+					{
+					have_mpls = true;
+					break;
+					}
+
+				// Check for double-tagged (802.1ad)
+				if ( protocol == 0x8100 || protocol == 0x9100 )
+					{
+					protocol = ((pdata[2] << 8) + pdata[3]);
+					pdata += 4; // Skip the vlan header
+					}
+
+				eth_type = protocol;
+				break;
+
+			// PPPoE carried over the ethernet frame.
+			case 0x8864:
+				protocol = (pdata[6] << 8) + pdata[7];
+				pdata += 8; // Skip the PPPoE session and PPP header
+
+				if ( protocol == 0x0021 )
+					l3_proto = L3_IPV4;
+				else if ( protocol == 0x0057 )
+					l3_proto = L3_IPV6;
+				else
+					{
+					// Neither IPv4 nor IPv6.
+					Weird("non_ip_packet_in_pppoe_encapsulation");
+					return;
+					}
+
+				break;
+			}
+
+		// Normal path to determine Layer 3 protocol.
+		if ( ! have_mpls && l3_proto == L3_UNKNOWN )
+			{
+			if ( protocol == 0x800 )
+				l3_proto = L3_IPV4;
+			else if ( protocol == 0x86dd )
+				l3_proto = L3_IPV6;
+			else if ( protocol == 0x0806 || protocol == 0x8035 )
+				l3_proto = L3_ARP;
+			else
+				{
+				// Neither IPv4 nor IPv6.
+				Weird("non_ip_packet_in_ethernet");
+				return;
+				}
+			}
+
+		break;
+		}
+
+	case DLT_PPP_SERIAL:
+		{
+		// Get PPP protocol.
+		int protocol = (pdata[2] << 8) + pdata[3];
+		pdata += GetLinkHeaderSize(link_type);
+
+		if ( protocol == 0x0281 )
+			{
+			// MPLS Unicast. Remove the pdata link layer and
+			// denote a header size of zero before the IP header.
+			have_mpls = true;
+			}
+		else if ( protocol == 0x0021 )
+			l3_proto = L3_IPV4;
+		else if ( protocol == 0x0057 )
+			l3_proto = L3_IPV6;
+		else
+			{
+			// Neither IPv4 nor IPv6.
+			Weird("non_ip_packet_in_ppp_encapsulation");
+			return;
+			}
+		break;
+		}
+
+	default:
+		{
+		// Assume we're pointing at IP. Just figure out which version.
+		pdata += GetLinkHeaderSize(link_type);
+		const struct ip* ip = (const struct ip *)pdata;
+
+		if ( ip->ip_v == 4 )
+			l3_proto = L3_IPV4;
+		else if ( ip->ip_v == 6 )
+			l3_proto = L3_IPV6;
+		else
+			{
+			// Neither IPv4 nor IPv6.
+			Weird("non_ip_packet");
+			return;
+			}
+
+		break;
+		}
+	}
+
+	if ( have_mpls )
+		{
+		// Skip the MPLS label stack.
+		bool end_of_stack = false;
+
+		while ( ! end_of_stack )
+			{
+			end_of_stack = *(pdata + 2) & 0x01;
+			pdata += 4;
+
+			if ( pdata >= pdata + cap_len )
+				{
+				Weird("no_mpls_payload");
+				return;
+				}
+			}
+
+		// We assume that what remains is IP
+		if ( pdata + sizeof(struct ip) >= data + cap_len )
+			{
+			Weird("no_ip_in_mpls_payload");
+			return;
+			}
+
+		const struct ip* ip = (const struct ip *)pdata;
+
+		if ( ip->ip_v == 4 )
+			l3_proto = L3_IPV4;
+		else if ( ip->ip_v == 6 )
+			l3_proto = L3_IPV6;
+		else
+			{
+			// Neither IPv4 nor IPv6.
+			Weird("no_ip_in_mpls_payload");
+			return;
+			}
+		}
+
+	else if ( encap_hdr_size )
+		{
+		// Blanket encapsulation. We assume that what remains is IP.
+		pdata += encap_hdr_size;
+		if ( pdata + sizeof(struct ip) >= data + cap_len )
+			{
+			Weird("no_ip_left_after_encap");
+			return;
+			}
+
+		const struct ip* ip = (const struct ip *)pdata;
+
+		if ( ip->ip_v == 4 )
+			l3_proto = L3_IPV4;
+		else if ( ip->ip_v == 6 )
+			l3_proto = L3_IPV6;
+		else
+			{
+			// Neither IPv4 nor IPv6.
+			Weird("no_ip_in_encap");
+			return;
+			}
+
+		}
+
+	// We've now determined (a) L3_IPV4 vs (b) L3_IPV6 vs
+	// (c) L3_ARP vs (d) L3_UNKNOWN.
+	l3_proto = l3_proto;
+
+	// Calculate how much header we've used up.
+	hdr_size = (pdata - data);
+}
+
+RecordVal* Packet::BuildPktHdrVal() const
+	{
+	static RecordType* l2_hdr_type = 0;
+	static RecordType* raw_pkt_hdr_type = 0;
+
+	if ( ! raw_pkt_hdr_type )
+		{
+		raw_pkt_hdr_type = internal_type("raw_pkt_hdr")->AsRecordType();
+		l2_hdr_type = internal_type("l2_hdr")->AsRecordType();
+		}
+
+	RecordVal* pkt_hdr = new RecordVal(raw_pkt_hdr_type);
+	RecordVal* l2_hdr = new RecordVal(l2_hdr_type);
+
+	int is_ethernet = (link_type == DLT_EN10MB) ? 1 : 0;
+
+	int l3 = BifEnum::L3_UNKNOWN;
+
+	if ( l3_proto == L3_IPV4 )
+		l3 = BifEnum::L3_IPV4;
+
+	else if ( l3_proto == L3_IPV6 )
+		l3 = BifEnum::L3_IPV6;
+
+	else if ( l3_proto == L3_ARP )
+		l3 = BifEnum::L3_ARP;
+
+	// l2_hdr layout:
+	//      encap: link_encap;      ##< L2 link encapsulation
+	//      len: count;		##< Total frame length on wire
+	//      cap_len: count;		##< Captured length
+	//      src: string &optional;  ##< L2 source (if ethernet)
+	//      dst: string &optional;  ##< L2 destination (if ethernet)
+	//      vlan: count &optional;  ##< VLAN tag if any (and ethernet)
+	//      ethertype: count &optional; ##< If ethernet
+	//      proto: layer3_proto;    ##< L3 proto
+
+	if ( is_ethernet )
+		{
+		// Ethernet header layout is:
+		//    dst[6bytes] src[6bytes] ethertype[2bytes]...
+		l2_hdr->Assign(0, new EnumVal(BifEnum::LINK_ETHERNET, BifType::Enum::link_encap));
+		l2_hdr->Assign(3, FmtEUI48(data + 6));	// src
+		l2_hdr->Assign(4, FmtEUI48(data));  	// dst
+
+		if ( vlan )
+			l2_hdr->Assign(5, new Val(vlan, TYPE_COUNT));
+
+		l2_hdr->Assign(6, new Val(eth_type, TYPE_COUNT));
+
+		if ( eth_type == ETHERTYPE_ARP || eth_type == ETHERTYPE_REVARP )
+			// We also identify ARP for L3 over ethernet
+			l3 = BifEnum::L3_ARP;
+		}
+	else
+		l2_hdr->Assign(0, new EnumVal(BifEnum::LINK_UNKNOWN, BifType::Enum::link_encap));
+
+	l2_hdr->Assign(1, new Val(len, TYPE_COUNT));
+	l2_hdr->Assign(2, new Val(cap_len, TYPE_COUNT));
+
+	l2_hdr->Assign(7, new EnumVal(l3, BifType::Enum::layer3_proto));
+
+	pkt_hdr->Assign(0, l2_hdr);
+
+	if ( l3_proto == L3_IPV4 )
+		{
+		IP_Hdr ip_hdr((const struct ip*)(data + hdr_size), false);
+		return ip_hdr.BuildPktHdrVal(pkt_hdr, 1);
+		}
+
+	else if ( l3_proto == L3_IPV6 )
+		{
+		IP_Hdr ip6_hdr((const struct ip6_hdr*)(data + hdr_size), false, cap_len);
+		return ip6_hdr.BuildPktHdrVal(pkt_hdr, 1);
+		}
+
+	else
+		return pkt_hdr;
+	}
+
+Val *Packet::FmtEUI48(const u_char *mac) const
+	{
+	char buf[20];
+	snprintf(buf, sizeof buf, "%02x:%02x:%02x:%02x:%02x:%02x",
+		 mac[0], mac[1], mac[2], mac[3], mac[4], mac[5]);
+	return new StringVal(buf);
+	}
+
+void Packet::Describe(ODesc* d) const
+	{
+	const IP_Hdr ip = IP();
+	d->Add(ip.SrcAddr());
+	d->Add("->");
+	d->Add(ip.DstAddr());
+	}
+
+bool Packet::Serialize(SerialInfo* info) const
+	{
+	return SERIALIZE(uint32(ts.tv_sec)) &&
+		SERIALIZE(uint32(ts.tv_usec)) &&
+		SERIALIZE(uint32(len)) &&
+		SERIALIZE(link_type) &&
+		info->s->Write(tag.c_str(), tag.length(), "tag") &&
+		info->s->Write((const char*)data, cap_len, "data");
+	}
+
+#ifdef DEBUG
+static iosource::PktDumper* dump = 0;
+#endif
+
+Packet* Packet::Unserialize(UnserialInfo* info)
+	{
+	struct timeval ts;
+	uint32 len, link_type;
+
+	if ( ! (UNSERIALIZE((uint32 *)&ts.tv_sec) &&
+		UNSERIALIZE((uint32 *)&ts.tv_usec) &&
+		UNSERIALIZE(&len) &&
+		UNSERIALIZE(&link_type)) )
+		return 0;
+
+	char* tag;
+	if ( ! info->s->Read((char**) &tag, 0, "tag") )
+		return 0;
+
+	const u_char* pkt;
+	int caplen;
+	if ( ! info->s->Read((char**) &pkt, &caplen, "data") )
+		{
+		delete [] tag;
+		return 0;
+		}
+
+	Packet *p = new Packet(link_type, &ts, caplen, len, pkt, true,
+			       std::string(tag));
+	delete [] tag;
+
+	// For the global timer manager, we take the global network_time as the
+	// packet's timestamp for feeding it into our packet loop.
+	if ( p->tag == "" )
+		p->time = timer_mgr->Time();
+	else
+		p->time = p->ts.tv_sec + double(p->ts.tv_usec) / 1e6;
+
+#ifdef DEBUG
+	if ( debug_logger.IsEnabled(DBG_TM) )
+		{
+		if ( ! dump )
+			dump = iosource_mgr->OpenPktDumper("tm.pcap", true);
+
+		if ( dump )
+			{
+			dump->Dump(p);
+			}
+		}
+#endif
+
+	return p;
+	}

src/iosource/Packet.h

@@ -0,0 +1,202 @@
+#ifndef packet_h
+#define packet_h
+
+#include "Desc.h"
+#include "IP.h"
+#include "NetVar.h"
+
+/**
+ * The Layer 3 type of a packet, as determined by the parsing code in Packet.
+ */
+enum Layer3Proto {
+	L3_UNKNOWN = -1,	/// Layer 3 type could not be determined.
+	L3_IPV4 = 1,	/// Layer 3 is IPv4.
+	L3_IPV6 = 2,	/// Layer 3 is IPv6.
+	L3_ARP = 3,	/// Layer 3 is ARP.
+};
+
+/**
+ * A link-layer packet.
+ *
+ * Note that for serialization we don't use much of the support provided by
+ * the serialization framework. Serialize/Unserialize do all the work by
+ * themselves. In particular, Packets aren't derived from SerialObj. They are
+ * completely seperate and self-contained entities, and we don't need any of
+ * the sophisticated features like object caching.
+ */
+class Packet {
+public:
+	/**
+	 * Construct and initialize from packet data.
+	 *
+	 * @param link_type The link type in the form of a \c DLT_* constant.
+	 *
+	 * @param ts The timestamp associated with the packet.
+	 *
+	 * @param caplen The number of bytes valid in *data*.
+	 *
+	 * @param len The wire length of the packet, which must be more or
+	 * equal *caplen* (but can't be less).
+	 *
+	 * @param data A pointer to the raw packet data, starting with the
+	 * layer 2 header. The pointer must remain valid for the lifetime of
+	 * the Packet instance, unless *copy* is true.
+	 *
+	 * @param copy If true, the constructor will make an internal copy of
+	 * *data*, so that the caller can release its version.
+	 *
+	 * @param tag A textual tag to associate with the packet for
+	 * differentiating the input streams.
+	 */
+	Packet(int link_type, struct timeval *ts, uint32 caplen,
+	       uint32 len, const u_char *data, int copy = false,
+	       std::string tag = std::string("")) : data(0)
+	       {
+	       Init(link_type, ts, caplen, len, data, copy, tag);
+	       }
+
+	/**
+	 * Default constructor. For internal use only.
+	 */
+	Packet() : data(0)
+		{
+		struct timeval ts = {0, 0};
+		Init(0, &ts, 0, 0, 0);
+		}
+
+	/**
+	 * Destructor.
+	 */
+	~Packet()
+		{
+		if ( copy )
+			delete [] data;
+		}
+
+	/**
+	 * (Re-)initialize from packet data.
+	 *
+	 * @param link_type The link type in the form of a \c DLT_* constant.
+	 *
+	 * @param ts The timestamp associated with the packet.
+	 *
+	 * @param caplen The number of bytes valid in *data*.
+	 *
+	 * @param len The wire length of the packet, which must be more or
+	 * equal *caplen* (but can't be less).
+	 *
+	 * @param data A pointer to the raw packet data, starting with the
+	 * layer 2 header. The pointer must remain valid for the lifetime of
+	 * the Packet instance, unless *copy* is true.
+	 *
+	 * @param copy If true, the constructor will make an internal copy of
+	 * *data*, so that the caller can release its version.
+	 *
+	 * @param tag A textual tag to associate with the packet for
+	 * differentiating the input streams.
+	 */
+	void Init(int link_type, struct timeval *ts, uint32 caplen,
+		uint32 len, const u_char *data, int copy = false,
+		std::string tag = std::string(""));
+
+	/**
+	 * Returns true if parsing the layer 2 fields failed, including when
+	 * no data was passed into the constructor in the first place.
+	 */
+	bool Layer2Valid()
+		{
+		return l2_valid;
+		}
+
+	/**
+	 * Interprets the Layer 3 of the packet as IP and returns a
+	 * correspondign object.
+	 */
+	const IP_Hdr IP() const
+		{ return IP_Hdr((struct ip *) (data + hdr_size), false); }
+
+	/**
+	 * Returns a \c raw_pkt_hdr RecordVal, which includes layer 2 and
+	 * also everything in IP_Hdr (i.e., IP4/6 + TCP/UDP/ICMP).
+	 */
+	RecordVal* BuildPktHdrVal() const;
+
+	/**
+	 * Static method returning the link-layer header size for a given
+	 * link type.
+	 *
+	 * @param link_type The link tyoe.
+	 *
+	 * @return The header size in bytes, or -1 if not known.
+	 */
+	static int GetLinkHeaderSize(int link_type);
+
+	/**
+	 * Describes the packet, with standard signature.
+	 */
+	void Describe(ODesc* d) const;
+
+	/**
+	 * Serializes the packet, with standard signature.
+	 */
+	bool Serialize(SerialInfo* info) const;
+
+	/**
+	 * Unserializes the packet, with standard signature.
+	 */
+	static Packet* Unserialize(UnserialInfo* info);
+
+	// These are passed in through the constructor.
+	std::string tag;		/// Used in serialization
+	double time;			/// Timestamp reconstituted as float
+	struct timeval ts;		/// Capture timestamp
+	const u_char* data;		/// Packet data.
+	uint32 len;			/// Actual length on wire
+	uint32 cap_len;			/// Captured packet length
+	uint32 link_type;		/// pcap link_type (DLT_EN10MB, DLT_RAW, etc)
+
+	// These are computed from Layer 2 data. These fields are only valid if
+	// Layer2Valid() returns true.
+
+	/**
+	 * Layer 2 header size. Valid iff Layer2Valid() returns true.
+	 */
+	uint32 hdr_size;
+
+	/**
+	 * Layer 3 protocol identified (if any). Valid iff Layer2Valid()
+	 * returns true.
+	 */
+	Layer3Proto l3_proto;		/// 
+
+	/**
+	 * If layer 2 is Ethernet, innermost ethertype field. Valid iff
+	 * Layer2Valid() returns true.
+	 */
+	uint32 eth_type;		/// 
+
+	/**
+	 * (Outermost) VLAN tag if any, else 0. Valid iff Layer2Valid()
+	 * returns true.
+	 */
+	uint32 vlan;			///
+
+private:
+	// Calculate layer 2 attributes. Sets
+	void ProcessLayer2();
+
+	// Wrapper to generate a packet-level weird.
+	void Weird(const char* name);
+
+	// Renders an MAC address into its ASCII representation.
+	Val *FmtEUI48(const u_char *mac) const;
+
+	// True if we need to delete associated packet memory upon
+	// destruction.
+	bool copy;
+
+	// True if L2 processing succeeded.
+	bool l2_valid;
+};
+
+#endif // packet_h

src/iosource/PktDumper.h

@@ -3,6 +3,7 @@
 #ifndef IOSOURCE_PKTSRC_PKTDUMPER_H
 #define IOSOURCE_PKTSRC_PKTDUMPER_H
 
+#include "Packet.h"
 #include "IOSource.h"
 
 namespace iosource {
@@ -12,21 +13,6 @@ namespace iosource {
  */
 class PktDumper {
 public:
-	/**
-	 * Structure describing a packet.
-	 */
-	struct Packet {
-		/**
-		 * The pcap header associated with the packet.
-		 */
-		const struct pcap_pkthdr* hdr;
-
-		/**
-		 * The full content of the packet.
-		 */
-		const unsigned char* data;
-	};
-
 	/**
 	 * Constructor.
 	 */

src/iosource/PktSrc.cc

@@ -17,7 +17,6 @@ PktSrc::Properties::Properties()
 	{
 	selectable_fd = -1;
 	link_type = -1;
-	hdr_size = -1;
 	netmask = NETMASK_UNKNOWN;
 	is_live = false;
 	}
@@ -67,11 +66,6 @@ bool PktSrc::IsError() const
 	return ErrorMsg();
 	}
 
-int PktSrc::HdrSize() const
-	{
-	return IsOpen() ? props.hdr_size : -1;
-	}
-
 int PktSrc::SnapLen() const
 	{
 	return snaplen; // That's a global. Change?
@@ -98,7 +92,7 @@ double PktSrc::CurrentPacketWallClock()
 
 void PktSrc::Opened(const Properties& arg_props)
 	{
-	if ( arg_props.hdr_size < 0 )
+	if ( Packet::GetLinkHeaderSize(arg_props.link_type) < 0 )
 		{
 		char buf[512];
 		safe_snprintf(buf, sizeof(buf),
@@ -147,7 +141,7 @@ void PktSrc::Info(const std::string& msg)
 
 void PktSrc::Weird(const std::string& msg, const Packet* p)
 	{
-	sessions->Weird(msg.c_str(), p->hdr, p->data, 0);
+	sessions->Weird(msg.c_str(), p, 0);
 	}
 
 void PktSrc::InternalError(const std::string& msg)
@@ -160,33 +154,6 @@ void PktSrc::ContinueAfterSuspend()
 	current_wallclock = current_time(true);
 	}
 
-int PktSrc::GetLinkHeaderSize(int link_type)
-	{
-	switch ( link_type ) {
-	case DLT_NULL:
-		return 4;
-
-	case DLT_EN10MB:
-		return 14;
-
-	case DLT_FDDI:
-		return 13 + 8;	// fddi_header + LLC
-
-#ifdef DLT_LINUX_SLL
-	case DLT_LINUX_SLL:
-		return 16;
-#endif
-
-	case DLT_PPP_SERIAL:	// PPP_SERIAL
-		return 4;
-
-	case DLT_RAW:
-		return 0;
-	}
-
-	return -1;
-	}
-
 double PktSrc::CheckPseudoTime()
 	{
 	if ( ! IsOpen() )
@@ -197,20 +164,20 @@ double PktSrc::CheckPseudoTime()
 
 	if ( remote_trace_sync_interval )
 		{
-		if ( next_sync_point == 0 || current_packet.ts >= next_sync_point )
+		if ( next_sync_point == 0 || current_packet.time >= next_sync_point )
 			{
 			int n = remote_serializer->SendSyncPoint();
 			next_sync_point = first_timestamp +
 						n * remote_trace_sync_interval;
 			remote_serializer->Log(RemoteSerializer::LogInfo,
 				fmt("stopping at packet %.6f, next sync-point at %.6f",
-					current_packet.ts, next_sync_point));
+					current_packet.time, next_sync_point));
 
 			return 0;
 			}
 		}
 
-	double pseudo_time = current_packet.ts - first_timestamp;
+	double pseudo_time = current_packet.time - first_timestamp;
 	double ct = (current_time(true) - first_wallclock) * pseudo_realtime;
 
 	return pseudo_time <= ct ? bro_start_time + pseudo_time : 0;
@@ -273,7 +240,7 @@ double PktSrc::NextTimestamp(double* local_network_time)
 		return -1.0;
 		}
 
-	return current_packet.ts;
+	return current_packet.time;
 	}
 
 void PktSrc::Process()
@@ -284,145 +251,20 @@ void PktSrc::Process()
 	if ( ! ExtractNextPacketInternal() )
 		return;
 
-	int pkt_hdr_size = props.hdr_size;
+	if ( current_packet.Layer2Valid() )
-
-	// Unfortunately some packets on the link might have MPLS labels
-	// while others don't. That means we need to ask the link-layer if
-	// labels are in place.
-	bool have_mpls = false;
-
-	int protocol = 0;
-	const u_char* data = current_packet.data;
-
-	switch ( props.link_type ) {
-	case DLT_NULL:
 		{
-		protocol = (data[3] << 24) + (data[2] << 16) + (data[1] << 8) + data[0];
+		if ( pseudo_realtime )
-
-		// From the Wireshark Wiki: "AF_INET6, unfortunately, has
-		// different values in {NetBSD,OpenBSD,BSD/OS},
-		// {FreeBSD,DragonFlyBSD}, and {Darwin/Mac OS X}, so an IPv6
-		// packet might have a link-layer header with 24, 28, or 30
-		// as the AF_ value." As we may be reading traces captured on
-		// platforms other than what we're running on, we accept them
-		// all here.
-		if ( protocol != AF_INET
-		     && protocol != AF_INET6
-		     && protocol != 24
-		     && protocol != 28
-		     && protocol != 30 )
 			{
-			Weird("non_ip_packet_in_null_transport", &current_packet);
+			current_pseudo = CheckPseudoTime();
-			goto done;
+			net_packet_dispatch(current_pseudo, &current_packet, this);
+			if ( ! first_wallclock )
+				first_wallclock = current_time(true);
 			}
 
-		break;
+		else
+			net_packet_dispatch(current_packet.time, &current_packet, this);
 		}
 
-	case DLT_EN10MB:
-		{
-		// Get protocol being carried from the ethernet frame.
-		protocol = (data[12] << 8) + data[13];
-
-		switch ( protocol )
-			{
-			// MPLS carried over the ethernet frame.
-			case 0x8847:
-				// Remove the data link layer and denote a
-				// header size of zero before the IP header.
-				have_mpls = true;
-				data += GetLinkHeaderSize(props.link_type);
-				pkt_hdr_size = 0;
-				break;
-
-			// VLAN carried over the ethernet frame.
-			case 0x8100:
-				data += GetLinkHeaderSize(props.link_type);
-
-				// Check for MPLS in VLAN.
-				if ( ((data[2] << 8) + data[3]) == 0x8847 )
-					have_mpls = true;
-
-				data += 4; // Skip the vlan header
-				pkt_hdr_size = 0;
-
-				// Check for 802.1ah (Q-in-Q) containing IP.
-				// Only do a second layer of vlan tag
-				// stripping because there is no
-				// specification that allows for deeper
-				// nesting.
-				if ( ((data[2] << 8) + data[3]) == 0x0800 )
-					data += 4;
-
-				break;
-
-			// PPPoE carried over the ethernet frame.
-			case 0x8864:
-				data += GetLinkHeaderSize(props.link_type);
-				protocol = (data[6] << 8) + data[7];
-				data += 8; // Skip the PPPoE session and PPP header
-				pkt_hdr_size = 0;
-
-				if ( protocol != 0x0021 && protocol != 0x0057 )
-					{
-					// Neither IPv4 nor IPv6.
-					Weird("non_ip_packet_in_pppoe_encapsulation", &current_packet);
-					goto done;
-					}
-				break;
-			}
-
-		break;
-		}
-
-	case DLT_PPP_SERIAL:
-		{
-		// Get PPP protocol.
-		protocol = (data[2] << 8) + data[3];
-
-		if ( protocol == 0x0281 )
-			{
-			// MPLS Unicast. Remove the data link layer and
-			// denote a header size of zero before the IP header.
-			have_mpls = true;
-				data += GetLinkHeaderSize(props.link_type);
-			pkt_hdr_size = 0;
-			}
-
-		else if ( protocol != 0x0021 && protocol != 0x0057 )
-			{
-			// Neither IPv4 nor IPv6.
-			Weird("non_ip_packet_in_ppp_encapsulation", &current_packet);
-			goto done;
-			}
-		break;
-		}
-	}
-
-	if ( have_mpls )
-		{
-		// Skip the MPLS label stack.
-		bool end_of_stack = false;
-
-		while ( ! end_of_stack )
-			{
-			end_of_stack = *(data + 2) & 0x01;
-			data += 4;
-			}
-		}
-
-	if ( pseudo_realtime )
-		{
-		current_pseudo = CheckPseudoTime();
-		net_packet_dispatch(current_pseudo, current_packet.hdr, data, pkt_hdr_size, this);
-		if ( ! first_wallclock )
-			first_wallclock = current_time(true);
-		}
-
-	else
-		net_packet_dispatch(current_packet.ts, current_packet.hdr, data, pkt_hdr_size, this);
-
-done:
 	have_packet = 0;
 	DoneWithPacket();
 	}
@@ -453,7 +295,7 @@ bool PktSrc::ExtractNextPacketInternal()
 	if ( ExtractNextPacket(&current_packet) )
 		{
 		if ( ! first_timestamp )
-			first_timestamp = current_packet.ts;
+			first_timestamp = current_packet.time;
 
 		SetIdle(false);
 		have_packet = true;
@@ -536,12 +378,11 @@ bool PktSrc::ApplyBPFFilter(int index, const struct pcap_pkthdr *hdr, const u_ch
 	return pcap_offline_filter(code->GetProgram(), hdr, pkt);
 	}
 
-bool PktSrc::GetCurrentPacket(const pcap_pkthdr** hdr, const u_char** pkt)
+bool PktSrc::GetCurrentPacket(const Packet** pkt)
 	{
 	if ( ! have_packet )
 		return false;
 
-	*hdr = current_packet.hdr;
+	*pkt = &current_packet;
-	*pkt = current_packet.data;
 	return true;
 	}

src/iosource/PktSrc.h

@@ -6,6 +6,7 @@
 #include "IOSource.h"
 #include "BPF_Program.h"
 #include "Dict.h"
+#include "Packet.h"
 
 declare(PDict,BPF_Program);
 
@@ -165,14 +166,12 @@ public:
 	/**
 	 * Returns the packet currently being processed, if available.
 	 *
-	 * @param hdr A pointer to pass the header of the current packet back.
-	 *
 	 * @param pkt A pointer to pass the content of the current packet
 	 * back.
 	 *
 	 * @return True if the current packet is available, or false if not.
 	 */
-	bool GetCurrentPacket(const pcap_pkthdr** hdr, const u_char** pkt);
+	bool GetCurrentPacket(const Packet** hdr);
 
 	// PacketSource interace for derived classes to override.
 
@@ -216,15 +215,6 @@ public:
 	 */
 	virtual void Statistics(Stats* stats) = 0;
 
-	/**
-	 * Helper method to return the header size for a given link tyoe.
-	 *
-	 * @param link_type The link tyoe.
-	 *
-	 * @return The header size in bytes.
-	 */
-	static int GetLinkHeaderSize(int link_type);
-
 protected:
 	friend class Manager;
 
@@ -252,13 +242,6 @@ protected:
 		 */
 		int link_type;
 
-		/**
-		 * The size of the link-layer header for packets from this
-		 * source. \a GetLinkHeaderSize() may be used to derive this
-		 * value.
-		 */
-		int hdr_size;
-
 		/**
 		 * Returns the netmask associated with the source, or \c
 		 * NETMASK_UNKNOWN if unknown.
@@ -274,26 +257,6 @@ protected:
 		Properties();
 	};
 
-	/**
-	 * Structure describing a packet.
-	 */
-	struct Packet {
-		/**
-		 *  Time associated with the packet.
-		 */
-		double ts;
-
-		/**
-		 *  The pcap header associated with the packet.
-		 */
-		const struct ::pcap_pkthdr* hdr;
-
-		/**
-		 * The full content of the packet.
-		 */
-		const u_char* data;
-	};
-
 	/**
 	 * Called from the implementations of \a Open() to signal that the
 	 * source has been successully opened.

src/iosource/pcap/Dumper.cc

@@ -79,7 +79,7 @@ void PcapDumper::Open()
 		}
 
 	props.open_time = network_time;
-	props.hdr_size = PktSrc::GetLinkHeaderSize(pcap_datalink(pd));
+	props.hdr_size = Packet::GetLinkHeaderSize(pcap_datalink(pd));
 	Opened(props);
 	}
 
@@ -101,8 +101,12 @@ bool PcapDumper::Dump(const Packet* pkt)
 	if ( ! dumper )
 		return false;
 
-	pcap_dump((u_char*) dumper, pkt->hdr, pkt->data);
+	// Reconstitute the pcap_pkthdr.
+	const struct pcap_pkthdr phdr = {
+		.ts = pkt->ts, .caplen = pkt->cap_len, .len = pkt->len
+	};
 
+	pcap_dump((u_char*) dumper, &phdr, pkt->data);
 	return true;
 	}
 

src/iosource/pcap/Source.cc

@@ -5,6 +5,7 @@
 #include "config.h"
 
 #include "Source.h"
+#include "iosource/Packet.h"
 
 #ifdef HAVE_PCAP_INT_H
 #include <pcap-int.h>
@@ -167,9 +168,8 @@ bool PcapSource::ExtractNextPacket(Packet* pkt)
 		return false;
 		}
 
-	pkt->ts = current_hdr.ts.tv_sec + double(current_hdr.ts.tv_usec) / 1e6;
+	last_data = data;
-	pkt->hdr = &current_hdr;
+	pkt->Init(props.link_type, &current_hdr.ts, current_hdr.caplen, current_hdr.len, data);
-	pkt->data = last_data = data;
 
 	if ( current_hdr.len == 0 || current_hdr.caplen == 0 )
 		{
@@ -275,7 +275,6 @@ void PcapSource::SetHdrSize()
 	char errbuf[PCAP_ERRBUF_SIZE];
 
 	props.link_type = pcap_datalink(pd);
-	props.hdr_size = GetLinkHeaderSize(props.link_type);
 	}
 
 iosource::PktSrc* PcapSource::Instantiate(const std::string& path, bool is_live)

src/types.bif

@@ -184,6 +184,18 @@ type EncapsulatingConn: record;
 
 module GLOBAL;
 
+enum link_encap %{
+	LINK_ETHERNET,
+	LINK_UNKNOWN,
+%}
+
+enum layer3_proto %{
+	L3_IPV4,
+	L3_IPV6,
+	L3_ARP,
+	L3_UNKNOWN,
+%}
+
 type gtpv1_hdr: record;
 type gtp_create_pdp_ctx_request_elements: record;
 type gtp_create_pdp_ctx_response_elements: record;

testing/btest/Baseline/core.raw_packet/output

@@ -0,0 +1,44 @@
+[l2=[encap=LINK_ETHERNET, len=215, cap_len=215, src=e8:de:27:ff:c0:78, dst=ff:ff:ff:ff:ff:ff, vlan=<uninitialized>, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=201, id=0, ttl=64, p=17, src=192.168.1.1, dst=255.255.255.255], ip6=<uninitialized>, tcp=<uninitialized>, udp=[sport=40190/udp, dport=7437/udp, ulen=181], icmp=<uninitialized>]
+[l2=[encap=LINK_ETHERNET, len=68, cap_len=68, src=60:f8:1d:c9:8c:fa, dst=e8:de:27:ff:c0:78, vlan=<uninitialized>, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=54, id=52261, ttl=64, p=6, src=192.168.1.103, dst=64.4.23.176], ip6=<uninitialized>, tcp=[sport=65493/tcp, dport=40031/tcp, seq=2642773190, ack=2891276360, hl=32, dl=2, flags=24, win=4096], udp=<uninitialized>, icmp=<uninitialized>]
+[l2=[encap=LINK_ETHERNET, len=78, cap_len=78, src=60:f8:1d:c9:8c:fa, dst=e8:de:27:ff:c0:78, vlan=<uninitialized>, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=64, id=32575, ttl=64, p=17, src=192.168.1.103, dst=192.168.1.1], ip6=<uninitialized>, tcp=<uninitialized>, udp=[sport=65170/udp, dport=53/udp, ulen=44], icmp=<uninitialized>]
+[l2=[encap=LINK_ETHERNET, len=78, cap_len=78, src=60:f8:1d:c9:8c:fa, dst=e8:de:27:ff:c0:78, vlan=<uninitialized>, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=64, id=55466, ttl=64, p=17, src=192.168.1.103, dst=192.168.1.1], ip6=<uninitialized>, tcp=<uninitialized>, udp=[sport=53129/udp, dport=53/udp, ulen=44], icmp=<uninitialized>]
+[l2=[encap=LINK_ETHERNET, len=92, cap_len=92, src=60:f8:1d:c9:8c:fa, dst=e8:de:27:ff:c0:78, vlan=<uninitialized>, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=78, id=32240, ttl=64, p=17, src=192.168.1.103, dst=192.168.1.1], ip6=<uninitialized>, tcp=<uninitialized>, udp=[sport=53129/udp, dport=53/udp, ulen=58], icmp=<uninitialized>]
+[l2=[encap=LINK_ETHERNET, len=85, cap_len=85, src=60:f8:1d:c9:8c:fa, dst=e8:de:27:ff:c0:78, vlan=<uninitialized>, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=71, id=53895, ttl=64, p=17, src=192.168.1.103, dst=192.168.1.1], ip6=<uninitialized>, tcp=<uninitialized>, udp=[sport=57932/udp, dport=53/udp, ulen=51], icmp=<uninitialized>]
+[l2=[encap=LINK_ETHERNET, len=42, cap_len=42, src=00:50:56:3e:93:6b, dst=ff:ff:ff:ff:ff:ff, vlan=<uninitialized>, eth_type=2054, proto=L3_ARP], ip=<uninitialized>, ip6=<uninitialized>, tcp=<uninitialized>, udp=<uninitialized>, icmp=<uninitialized>]
+[l2=[encap=LINK_ETHERNET, len=42, cap_len=42, src=00:50:56:3e:93:6b, dst=ff:ff:ff:ff:ff:ff, vlan=<uninitialized>, eth_type=2054, proto=L3_ARP], ip=<uninitialized>, ip6=<uninitialized>, tcp=<uninitialized>, udp=<uninitialized>, icmp=<uninitialized>]
+[l2=[encap=LINK_ETHERNET, len=307, cap_len=307, src=e8:de:27:ff:c0:78, dst=01:00:5e:7f:ff:fa, vlan=<uninitialized>, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=293, id=0, ttl=4, p=17, src=192.168.1.1, dst=239.255.255.250], ip6=<uninitialized>, tcp=<uninitialized>, udp=[sport=45335/udp, dport=1900/udp, ulen=273], icmp=<uninitialized>]
+[l2=[encap=LINK_ETHERNET, len=316, cap_len=316, src=e8:de:27:ff:c0:78, dst=01:00:5e:7f:ff:fa, vlan=<uninitialized>, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=302, id=0, ttl=4, p=17, src=192.168.1.1, dst=239.255.255.250], ip6=<uninitialized>, tcp=<uninitialized>, udp=[sport=45335/udp, dport=1900/udp, ulen=282], icmp=<uninitialized>]
+[l2=[encap=LINK_ETHERNET, len=379, cap_len=379, src=e8:de:27:ff:c0:78, dst=01:00:5e:7f:ff:fa, vlan=<uninitialized>, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=365, id=0, ttl=4, p=17, src=192.168.1.1, dst=239.255.255.250], ip6=<uninitialized>, tcp=<uninitialized>, udp=[sport=45335/udp, dport=1900/udp, ulen=345], icmp=<uninitialized>]
+[l2=[encap=LINK_ETHERNET, len=371, cap_len=371, src=e8:de:27:ff:c0:78, dst=01:00:5e:7f:ff:fa, vlan=<uninitialized>, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=357, id=0, ttl=4, p=17, src=192.168.1.1, dst=239.255.255.250], ip6=<uninitialized>, tcp=<uninitialized>, udp=[sport=45335/udp, dport=1900/udp, ulen=337], icmp=<uninitialized>]
+[l2=[encap=LINK_ETHERNET, len=355, cap_len=355, src=e8:de:27:ff:c0:78, dst=01:00:5e:7f:ff:fa, vlan=<uninitialized>, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=341, id=0, ttl=4, p=17, src=192.168.1.1, dst=239.255.255.250], ip6=<uninitialized>, tcp=<uninitialized>, udp=[sport=45335/udp, dport=1900/udp, ulen=321], icmp=<uninitialized>]
+[l2=[encap=LINK_ETHERNET, len=42, cap_len=42, src=00:50:56:3e:93:6b, dst=ff:ff:ff:ff:ff:ff, vlan=<uninitialized>, eth_type=2054, proto=L3_ARP], ip=<uninitialized>, ip6=<uninitialized>, tcp=<uninitialized>, udp=<uninitialized>, icmp=<uninitialized>]
+[l2=[encap=LINK_ETHERNET, len=387, cap_len=387, src=e8:de:27:ff:c0:78, dst=01:00:5e:7f:ff:fa, vlan=<uninitialized>, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=373, id=0, ttl=4, p=17, src=192.168.1.1, dst=239.255.255.250], ip6=<uninitialized>, tcp=<uninitialized>, udp=[sport=45335/udp, dport=1900/udp, ulen=353], icmp=<uninitialized>]
+[l2=[encap=LINK_ETHERNET, len=316, cap_len=316, src=e8:de:27:ff:c0:78, dst=01:00:5e:7f:ff:fa, vlan=<uninitialized>, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=302, id=0, ttl=4, p=17, src=192.168.1.1, dst=239.255.255.250], ip6=<uninitialized>, tcp=<uninitialized>, udp=[sport=45335/udp, dport=1900/udp, ulen=282], icmp=<uninitialized>]
+[l2=[encap=LINK_ETHERNET, len=375, cap_len=375, src=e8:de:27:ff:c0:78, dst=01:00:5e:7f:ff:fa, vlan=<uninitialized>, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=361, id=0, ttl=4, p=17, src=192.168.1.1, dst=239.255.255.250], ip6=<uninitialized>, tcp=<uninitialized>, udp=[sport=45335/udp, dport=1900/udp, ulen=341], icmp=<uninitialized>]
+[l2=[encap=LINK_ETHERNET, len=369, cap_len=369, src=e8:de:27:ff:c0:78, dst=01:00:5e:7f:ff:fa, vlan=<uninitialized>, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=355, id=0, ttl=4, p=17, src=192.168.1.1, dst=239.255.255.250], ip6=<uninitialized>, tcp=<uninitialized>, udp=[sport=45335/udp, dport=1900/udp, ulen=335], icmp=<uninitialized>]
+[l2=[encap=LINK_ETHERNET, len=316, cap_len=316, src=e8:de:27:ff:c0:78, dst=01:00:5e:7f:ff:fa, vlan=<uninitialized>, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=302, id=0, ttl=4, p=17, src=192.168.1.1, dst=239.255.255.250], ip6=<uninitialized>, tcp=<uninitialized>, udp=[sport=45335/udp, dport=1900/udp, ulen=282], icmp=<uninitialized>]
+[l2=[encap=LINK_ETHERNET, len=371, cap_len=371, src=e8:de:27:ff:c0:78, dst=01:00:5e:7f:ff:fa, vlan=<uninitialized>, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=357, id=0, ttl=4, p=17, src=192.168.1.1, dst=239.255.255.250], ip6=<uninitialized>, tcp=<uninitialized>, udp=[sport=45335/udp, dport=1900/udp, ulen=337], icmp=<uninitialized>]
+[l2=[encap=LINK_ETHERNET, len=381, cap_len=381, src=e8:de:27:ff:c0:78, dst=01:00:5e:7f:ff:fa, vlan=<uninitialized>, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=367, id=0, ttl=4, p=17, src=192.168.1.1, dst=239.255.255.250], ip6=<uninitialized>, tcp=<uninitialized>, udp=[sport=45335/udp, dport=1900/udp, ulen=347], icmp=<uninitialized>]
+[l2=[encap=LINK_ETHERNET, len=215, cap_len=215, src=e8:de:27:ff:c0:78, dst=ff:ff:ff:ff:ff:ff, vlan=<uninitialized>, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=201, id=0, ttl=64, p=17, src=192.168.1.1, dst=255.255.255.255], ip6=<uninitialized>, tcp=<uninitialized>, udp=[sport=40190/udp, dport=7437/udp, ulen=181], icmp=<uninitialized>]
+[l2=[encap=LINK_ETHERNET, len=98, cap_len=98, src=00:50:56:3e:93:6b, dst=e8:de:27:ff:c0:78, vlan=<uninitialized>, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=84, id=29257, ttl=64, p=1, src=192.168.1.104, dst=192.168.1.1], ip6=<uninitialized>, tcp=<uninitialized>, udp=<uninitialized>, icmp=[icmp_type=8]]
+[l2=[encap=LINK_ETHERNET, len=98, cap_len=98, src=e8:de:27:ff:c0:78, dst=00:50:56:3e:93:6b, vlan=<uninitialized>, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=84, id=3684, ttl=64, p=1, src=192.168.1.1, dst=192.168.1.104], ip6=<uninitialized>, tcp=<uninitialized>, udp=<uninitialized>, icmp=[icmp_type=0]]
+[l2=[encap=LINK_ETHERNET, len=112, cap_len=112, src=60:f8:1d:c9:8c:fa, dst=e8:de:27:ff:c0:78, vlan=<uninitialized>, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=98, id=56893, ttl=64, p=6, src=192.168.1.103, dst=74.125.21.138], ip6=<uninitialized>, tcp=[sport=49171/tcp, dport=443/tcp, seq=3725176031, ack=445274592, hl=32, dl=46, flags=24, win=4096], udp=<uninitialized>, icmp=<uninitialized>]
+[l2=[encap=LINK_ETHERNET, len=66, cap_len=66, src=60:f8:1d:c9:8c:fa, dst=e8:de:27:ff:c0:78, vlan=<uninitialized>, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=52, id=22643, ttl=64, p=6, src=192.168.1.103, dst=74.125.21.138], ip6=<uninitialized>, tcp=[sport=49171/tcp, dport=443/tcp, seq=3725176077, ack=445274652, hl=32, dl=0, flags=16, win=4094], udp=<uninitialized>, icmp=<uninitialized>]
+[l2=[encap=LINK_ETHERNET, len=112, cap_len=112, src=60:f8:1d:c9:8c:fa, dst=e8:de:27:ff:c0:78, vlan=<uninitialized>, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=98, id=85, ttl=64, p=6, src=192.168.1.103, dst=74.125.21.138], ip6=<uninitialized>, tcp=[sport=49171/tcp, dport=443/tcp, seq=3725176077, ack=445274652, hl=32, dl=46, flags=24, win=4096], udp=<uninitialized>, icmp=<uninitialized>]
+[l2=[encap=LINK_ETHERNET, len=97, cap_len=97, src=60:f8:1d:c9:8c:fa, dst=e8:de:27:ff:c0:78, vlan=<uninitialized>, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=83, id=28558, ttl=64, p=6, src=192.168.1.103, dst=74.125.21.138], ip6=<uninitialized>, tcp=[sport=49171/tcp, dport=443/tcp, seq=3725176123, ack=445274652, hl=32, dl=31, flags=24, win=4096], udp=<uninitialized>, icmp=<uninitialized>]
+[l2=[encap=LINK_ETHERNET, len=66, cap_len=66, src=60:f8:1d:c9:8c:fa, dst=e8:de:27:ff:c0:78, vlan=<uninitialized>, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=52, id=36529, ttl=64, p=6, src=192.168.1.103, dst=74.125.21.138], ip6=<uninitialized>, tcp=[sport=49171/tcp, dport=443/tcp, seq=3725176154, ack=445274652, hl=32, dl=0, flags=17, win=4096], udp=<uninitialized>, icmp=<uninitialized>]
+[l2=[encap=LINK_ETHERNET, len=64, cap_len=64, src=00:19:06:ea:b8:c1, dst=ff:ff:ff:ff:ff:ff, vlan=123, eth_type=2054, proto=L3_ARP], ip=<uninitialized>, ip6=<uninitialized>, tcp=<uninitialized>, udp=<uninitialized>, icmp=<uninitialized>]
+[l2=[encap=LINK_ETHERNET, len=64, cap_len=64, src=00:18:73:de:57:c1, dst=ff:ff:ff:ff:ff:ff, vlan=123, eth_type=2054, proto=L3_ARP], ip=<uninitialized>, ip6=<uninitialized>, tcp=<uninitialized>, udp=<uninitialized>, icmp=<uninitialized>]
+[l2=[encap=LINK_ETHERNET, len=64, cap_len=64, src=00:18:73:de:57:c1, dst=ff:ff:ff:ff:ff:ff, vlan=123, eth_type=2054, proto=L3_ARP], ip=<uninitialized>, ip6=<uninitialized>, tcp=<uninitialized>, udp=<uninitialized>, icmp=<uninitialized>]
+[l2=[encap=LINK_ETHERNET, len=64, cap_len=64, src=00:19:06:ea:b8:c1, dst=00:18:73:de:57:c1, vlan=123, eth_type=2054, proto=L3_ARP], ip=<uninitialized>, ip6=<uninitialized>, tcp=<uninitialized>, udp=<uninitialized>, icmp=<uninitialized>]
+[l2=[encap=LINK_ETHERNET, len=118, cap_len=118, src=00:18:73:de:57:c1, dst=00:19:06:ea:b8:c1, vlan=123, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=100, id=5, ttl=255, p=1, src=192.168.123.2, dst=192.168.123.1], ip6=<uninitialized>, tcp=<uninitialized>, udp=<uninitialized>, icmp=[icmp_type=8]]
+[l2=[encap=LINK_ETHERNET, len=64, cap_len=64, src=00:19:06:ea:b8:c1, dst=ff:ff:ff:ff:ff:ff, vlan=123, eth_type=2054, proto=L3_ARP], ip=<uninitialized>, ip6=<uninitialized>, tcp=<uninitialized>, udp=<uninitialized>, icmp=<uninitialized>]
+[l2=[encap=LINK_ETHERNET, len=64, cap_len=64, src=00:18:73:de:57:c1, dst=00:19:06:ea:b8:c1, vlan=123, eth_type=2054, proto=L3_ARP], ip=<uninitialized>, ip6=<uninitialized>, tcp=<uninitialized>, udp=<uninitialized>, icmp=<uninitialized>]
+[l2=[encap=LINK_ETHERNET, len=118, cap_len=118, src=00:18:73:de:57:c1, dst=00:19:06:ea:b8:c1, vlan=123, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=100, id=6, ttl=255, p=1, src=192.168.123.2, dst=192.168.123.1], ip6=<uninitialized>, tcp=<uninitialized>, udp=<uninitialized>, icmp=[icmp_type=8]]
+[l2=[encap=LINK_ETHERNET, len=118, cap_len=118, src=00:19:06:ea:b8:c1, dst=00:18:73:de:57:c1, vlan=123, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=100, id=6, ttl=255, p=1, src=192.168.123.1, dst=192.168.123.2], ip6=<uninitialized>, tcp=<uninitialized>, udp=<uninitialized>, icmp=[icmp_type=0]]
+[l2=[encap=LINK_ETHERNET, len=118, cap_len=118, src=00:18:73:de:57:c1, dst=00:19:06:ea:b8:c1, vlan=123, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=100, id=7, ttl=255, p=1, src=192.168.123.2, dst=192.168.123.1], ip6=<uninitialized>, tcp=<uninitialized>, udp=<uninitialized>, icmp=[icmp_type=8]]
+[l2=[encap=LINK_ETHERNET, len=118, cap_len=118, src=00:19:06:ea:b8:c1, dst=00:18:73:de:57:c1, vlan=123, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=100, id=7, ttl=255, p=1, src=192.168.123.1, dst=192.168.123.2], ip6=<uninitialized>, tcp=<uninitialized>, udp=<uninitialized>, icmp=[icmp_type=0]]
+[l2=[encap=LINK_ETHERNET, len=118, cap_len=118, src=00:18:73:de:57:c1, dst=00:19:06:ea:b8:c1, vlan=123, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=100, id=8, ttl=255, p=1, src=192.168.123.2, dst=192.168.123.1], ip6=<uninitialized>, tcp=<uninitialized>, udp=<uninitialized>, icmp=[icmp_type=8]]
+[l2=[encap=LINK_ETHERNET, len=118, cap_len=118, src=00:19:06:ea:b8:c1, dst=00:18:73:de:57:c1, vlan=123, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=100, id=8, ttl=255, p=1, src=192.168.123.1, dst=192.168.123.2], ip6=<uninitialized>, tcp=<uninitialized>, udp=<uninitialized>, icmp=[icmp_type=0]]
+[l2=[encap=LINK_ETHERNET, len=118, cap_len=118, src=00:18:73:de:57:c1, dst=00:19:06:ea:b8:c1, vlan=123, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=100, id=9, ttl=255, p=1, src=192.168.123.2, dst=192.168.123.1], ip6=<uninitialized>, tcp=<uninitialized>, udp=<uninitialized>, icmp=[icmp_type=8]]
+[l2=[encap=LINK_ETHERNET, len=118, cap_len=118, src=00:19:06:ea:b8:c1, dst=00:18:73:de:57:c1, vlan=123, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=100, id=9, ttl=255, p=1, src=192.168.123.1, dst=192.168.123.2], ip6=<uninitialized>, tcp=<uninitialized>, udp=<uninitialized>, icmp=[icmp_type=0]]

testing/btest/Baseline/core.reassembly/output

@@ -0,0 +1,32 @@
+----------------------
+flow weird, excessively_small_fragment, 164.1.123.163, 164.1.123.61
+flow weird, fragment_size_inconsistency, 164.1.123.163, 164.1.123.61
+flow weird, fragment_inconsistency, 164.1.123.163, 164.1.123.61
+flow weird, fragment_inconsistency, 164.1.123.163, 164.1.123.61
+flow weird, dns_unmatched_msg, 164.1.123.163, 164.1.123.61
+----------------------
+flow weird, excessively_small_fragment, 164.1.123.163, 164.1.123.61
+flow weird, excessively_small_fragment, 164.1.123.163, 164.1.123.61
+flow weird, fragment_overlap, 164.1.123.163, 164.1.123.61
+----------------------
+flow weird, fragment_with_DF, 210.54.213.247, 131.243.1.10
+flow weird, fragment_with_DF, 210.54.213.247, 131.243.1.10
+flow weird, fragment_with_DF, 210.54.213.247, 131.243.1.10
+flow weird, fragment_with_DF, 210.54.213.247, 131.243.1.10
+flow weird, fragment_with_DF, 210.54.213.247, 131.243.1.10
+----------------------
+flow weird, excessively_small_fragment, 128.32.46.142, 10.0.0.1
+flow weird, excessively_small_fragment, 128.32.46.142, 10.0.0.1
+flow weird, fragment_inconsistency, 128.32.46.142, 10.0.0.1
+----------------------
+net_weird, truncated_IP
+net_weird, truncated_IP
+net_weird, truncated_IP
+net_weird, truncated_IP
+rexmit_inconsistency, [orig_h=63.193.213.194, orig_p=2564/tcp, resp_h=128.3.97.175, resp_p=80/tcp], nlkmlpjfjjnoomfnqmdqgrdsgpefslhjrdjghsshrmosrkosidknnieiggpmnggelfhlkflfqojpjrsmeqghklmjlkdskjollmensjiqosemknoehellhlsspjfjpddfgqkemghskqosrksmkpsdomfoghllfokilshsisgpjhjoosidirlnmespjhdogdidoemejrnjjrookfrmiqllllqhlqfgolfqssfjrhrjhgfkpdnigiilrmnespjspeqjfedjhrkisjdhoofqdfeqnmihrelmildkngirkqorjslhmglripdojfedjjngjnpikoliqhdipgpshenekqiphmrsqmemghklodqnqoeggfkdqngrfollhjmddjreeghdqflohgrhqhelqsmdghgihpifpnikrddpmdfejhrhgfdfdlepmmhlhrnrslepqgmkopmdfogpoljeepqoemisfeksdeddiplnkfjddjioqhojlnmlirehidipdhqlddssssgpgikieeldsmfrkidpldsngdkidkoshkrofnonrrehghlmgmqshkedgpkpgjjkoneigsfjdlgjsngepfkndqoefqmsssrgegspromqepdpdeglmmegjljlmljeeorhhfmrohjeregpfshqjsqkekrihjdpfdjflgspepqjrqfemsjffmjfkhejdkrokmgdrhojgmgjpldjeiphroeheipolfmshoglkfnllfnhlflhlpddjflekhiqilefjpfqepdrrdokkjiekmelkhdpjlqjdlnfjemqdrksirdnjlrhrdijgqjhdqlidpfdisgrmnlfnsdlishlpfkshhglpdiqhpgmhpjdrpednjljfsqknsiqpfeqhlphgqdphflglpmqfkkhdjeodkelinkfpmfedidhphldmqjqggrljlhriehqqemeimkjhoqnsrdgengmgjokpeiijgrseppeoiflngggomdfjkndpqedhgnkiqlodkpjfkqoifidjmrdhhmglledkomllhpehdfjfdspmklkjdnhkdgpgqephfdfdrfplmepoegsekmrnikknelnprdpslmfkhghhooknieksjjhdeelidikndedijqqhfmphdondndpehmfoqelqigdpgioeljhedhfoeqlinriemqjigerkphgepqmiiidqlhriqioimpglonlsgomeloipndiihqqfiekkeriokrsjlmsjqiehqsrqkhdjlddjrrllirqkidqiggdrjpjirssgqepnqmhigfsqlekiqdddllnsjmroiofkieqnghddpjnhdjkfloilheljofddrkherkrieeoijrlfghiikmhpfdhekdjloejlmpperkgrhomedpfOOOOOOOOOOOOOOOOOOOOOOOOOOOO, nlkmlpjfjjnoomfnqmdqgrdsgpefslhjrdjghsshrmosrkosidknnieiggpmnggelfhlkflfqojpjrsmeqghklmjlkdskjollmensjiqosemknoehellhlsspjfjpddfgqkemghskqosrksmkpsdomfoghllfokilshsisgpjhjoosidirlnmespjhdogdidoemejrnjjrookfrmiqllllqhlqfgolfqssfjrhrjhgfkpdnigiilrmnespjspeqjfedjhrkisjdhoofqdfeqnmihrelmildkngirkqorjslhmglripdojfedjjngjnpikoliqhdipgpshenekqiphmrsqmemghklodqnqoeggfkdqngrfollhjmddjreeghdqflohgrhqhelqsmdghgihpifpnikrddpmdfejhrhgfdfdlepmmhlhrnrslepqgmkopmdfogpoljeepqoemisfeksdeddiplnkfjddjioqhojlnmlirehidipdhqlddssssgpgikieeldsmfrkidpldsngdkidkoshkrofnonrrehghlmgmqshkedgpkpgjjkoneigsfjdlgjsngepfkndqoefqmsssrgegspromqepdpdeglmmegjljlmljeeorhhfmrohjeregpfshqjsqkekrihjdpfdjflgspepqjrqfemsjffmjfkhejdkrokmgdrhojgmgjpldjeiphroeheipolfmshoglkfnllfnhlflhlpddjflekhiqilefjpfqepdrrdokkjiekmelkhdpjlqjdlnfjemqdrksirdnjlrhrdijgqjhdqlidpfdisgrmnlfnsdlishlpfkshhglpdiqhpgmhpjdrpednjljfsqknsiqpfeqhlphgqdphflglpmqfkkhdjeodkelinkfpmfedidhphldmqjqggrljlhriehqqemeimkjhoqnsrdgengmgjokpeiijgrseppeoiflngggomdfjkndpqedhgnkiqlodkpjfkqoifidjmrdhhmglledkomllhpehdfjfdspmklkjdnhkdgpgqephfdfdrfplmepoegsekmrnikknelnprdpslmfkhghhooknieksjjhdeelidikndedijqqhfmphdondndpehmfoqelqigdpgioeljhedhfoeqlinriemqjigerkphgepqmiiidqlhriqioimpglonlsgomeloipndiihqqfiekkeriokrsjlmsjqiehqsrqkhdjlddjrrllirqkidqiggdrjpjirssgqepnqmhigfsqlekiqdddllnsjmroiofkieqnghddpjnhdjkfloilheljofddrkherkrieeoijrlfghiikmhpfdhekdjloejlmpperkgrhomedpfqkrodjdmrqfpiodgphidfliidlhd
+rexmit_inconsistency, [orig_h=63.193.213.194, orig_p=2564/tcp, resp_h=128.3.97.175, resp_p=80/tcp], dgphrodofqhq, orgmmpelofil
+rexmit_inconsistency, [orig_h=63.193.213.194, orig_p=2564/tcp, resp_h=128.3.97.175, resp_p=80/tcp], lenhfdqhqfgs, dfpqssidkpdg
+rexmit_inconsistency, [orig_h=63.193.213.194, orig_p=2564/tcp, resp_h=128.3.97.175, resp_p=80/tcp], nlkmlpjfjjnoomfnqmdqgrdsgpefslhjrdjghsshrmosrkosidknnieiggpmnggelfhlkflfqojpjrsmeqghklmjlkdskjollmensjiqosemknoehellhlsspjfjpddfgqkemghskqosrksmkpsdomfoghllfokilshsisgpjhjoosidirlnmespjhdogdidoemejrnjjrookfrmiqllllqhlqfgolfqssfjrhrjhgfkpdnigiilrmnespjspeqjfedjhrkisjdhoofqdfeqnmihrelmildkngirkqorjslhmglripdojfedjjngjnpikoliqhdipgpshenekqiphmrsqmemghklodqnqoeggfkdqngrfollhjmddjreeghdqflohgrhqhelqsmdghgihpifpnikrddpmdfejhrhgfdfdlepmmhlhrnrslepqgmkopmdfogpoljeepqoemisfeksdeddiplnkfjddjioqhojlnmlirehidipdhqlddssssgpgikieeldsmfrkidpldsngdkidkoshkrofnonrrehghlmgmqshkedgpkpgjjkoneigsfjdlgjsngepfkndqoefqmsssrgegspromqepdpdeglmmegjljlmljeeorhhfmrohjeregpfshqjsqkekrihjdpfdjflgspepqjrqfemsjffmjfkhejdkrokmgdrhojgmgjpldjeiphroeheipolfmshoglkfnllfnhlflhlpddjflekhiqilefjpfqepdrrdokkjiekmelkhdpjlqjdlnfjemqdrksirdnjlrhrdijgqjhdqlidpfdisgrmnlfnsdlishlpfkshhglpdiqhpgmhpjdrpednjljfsqknsiqpfeqhlphgqdphflglpmqfkkhdjeodkelinkfpmfedidhphldmqjqggrljlhriehqqemeimkjhoqnsrdgengmgjokpeiijgrseppeoiflngggomdfjkndpqedhgnkiqlodkpjfkqoifidjmrdhhmglledkomllhpehdfjfdspmklkjdnhkdgpgqephfdfdrfplmepoegsekmrnikknelnprdpslmfkhghhooknieksjjhdeelidikndedijqqhfmphdondndpehmfoqelqigdpgioeljhedhfoeqlinriemqjigerkphgepqmiiidqlhriqioimpglonlsgomeloipndiihqqfiekkeriokrsjlmsjqiehqsrqkhdjlddjrrllirqkidqiggdrjpjirssgqepnqmhigfsqlekiqdddllnsjmroiofkieqnghddpjnhdjkfloilheljofddrkherkrieeoijrlfghiikmhpfdhekdjloejlmpperkgrhomedpfOOOOOOOOOOOOOOOOOOOOOOOOOOOO, nlkmlpjfjjnoomfnqmdqgrdsgpefslhjrdjghsshrmosrkosidknnieiggpmnggelfhlkflfqojpjrsmeqghklmjlkdskjollmensjiqosemknoehellhlsspjfjpddfgqkemghskqosrksmkpsdomfoghllfokilshsisgpjhjoosidirlnmespjhdogdidoemejrnjjrookfrmiqllllqhlqfgolfqssfjrhrjhgfkpdnigiilrmnespjspeqjfedjhrkisjdhoofqdfeqnmihrelmildkngirkqorjslhmglripdojfedjjngjnpikoliqhdipgpshenekqiphmrsqmemghklodqnqoeggfkdqngrfollhjmddjreeghdqflohgrhqhelqsmdghgihpifpnikrddpmdfejhrhgfdfdlepmmhlhrnrslepqgmkopmdfogpoljeepqoemisfeksdeddiplnkfjddjioqhojlnmlirehidipdhqlddssssgpgikieeldsmfrkidpldsngdkidkoshkrofnonrrehghlmgmqshkedgpkpgjjkoneigsfjdlgjsngepfkndqoefqmsssrgegspromqepdpdeglmmegjljlmljeeorhhfmrohjeregpfshqjsqkekrihjdpfdjflgspepqjrqfemsjffmjfkhejdkrokmgdrhojgmgjpldjeiphroeheipolfmshoglkfnllfnhlflhlpddjflekhiqilefjpfqepdrrdokkjiekmelkhdpjlqjdlnfjemqdrksirdnjlrhrdijgqjhdqlidpfdisgrmnlfnsdlishlpfkshhglpdiqhpgmhpjdrpednjljfsqknsiqpfeqhlphgqdphflglpmqfkkhdjeodkelinkfpmfedidhphldmqjqggrljlhriehqqemeimkjhoqnsrdgengmgjokpeiijgrseppeoiflngggomdfjkndpqedhgnkiqlodkpjfkqoifidjmrdhhmglledkomllhpehdfjfdspmklkjdnhkdgpgqephfdfdrfplmepoegsekmrnikknelnprdpslmfkhghhooknieksjjhdeelidikndedijqqhfmphdondndpehmfoqelqigdpgioeljhedhfoeqlinriemqjigerkphgepqmiiidqlhriqioimpglonlsgomeloipndiihqqfiekkeriokrsjlmsjqiehqsrqkhdjlddjrrllirqkidqiggdrjpjirssgqepnqmhigfsqlekiqdddllnsjmroiofkieqnghddpjnhdjkfloilheljofddrkherkrieeoijrlfghiikmhpfdhekdjloejlmpperkgrhomedpfqkrodjdmrqfpiodgphidfliislrr
+rexmit_inconsistency, [orig_h=63.193.213.194, orig_p=2564/tcp, resp_h=128.3.97.175, resp_p=80/tcp], iokgedlsdkjkiefgmeqkfjoh, ggdeolssksemrhedoledddml
+net_weird, truncated_IP
+rexmit_inconsistency, [orig_h=63.193.213.194, orig_p=2564/tcp, resp_h=128.3.97.175, resp_p=80/tcp], OOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOO HTTP/1.1\x0d\x0aHost: 127.0.0.1\x0d\x0aContent-Type: text/xml\x0d\x0aContent-length: 1\x0d\x0a\x0d\x0aO<?xml version="1.0"?>\x0d\x0a<g:searchrequest xmlns:g=, OOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOO HTTP/1.1\x0d\x0aHost: 127.0.0.1\x0d\x0aContent-Type: text/xml\x0d\x0aContent-length: 1\x0d\x0a\x0d\x0aO<?xml version="1.0"?igplqgeqsonkllfshdjplhjspmde

testing/btest/Baseline/core.tcp.quantum-insert/.stdout

@@ -0,0 +1,4 @@
+----- rexmit_inconsistency -----
+1429652006.683290 c: [orig_h=178.200.100.200, orig_p=39976/tcp, resp_h=96.126.98.124, resp_p=80/tcp]
+1429652006.683290 t1: HTTP/1.1 200 OK\x0d\x0aContent-Length: 5\x0d\x0a\x0d\x0aBANG!
+1429652006.683290 t2: HTTP/1.1 200 OK\x0d\x0aServer: nginx/1.4.4\x0d\x0aDate:

testing/btest/Baseline/scripts.base.protocols.http.missing-zlib-header/http.log

@@ -0,0 +1,11 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	http
+#open	2015-05-12-16-26-53
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	trans_depth	method	host	uri	referrer	user_agent	request_body_len	response_body_len	status_code	status_msg	info_code	info_msg	filename	tags	username	password	proxied	orig_fuids	orig_mime_types	resp_fuids	resp_mime_types
+#types	time	string	addr	port	addr	port	count	string	string	string	string	string	count	count	count	string	count	string	string	set[enum]	string	string	set[string]	vector[string]	vector[string]	vector[string]	vector[string]
+1232039472.314927	CXWv6p3arKYeMETxOg	237.244.174.255	1905	79.218.110.244	80	1	GET	ads1.msn.com	/library/dap.js	http://zone.msn.com/en/root/default.htm	Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; .NET CLR 1.0.3705; .NET CLR 1.1.4322; Media Center PC 4.0; .NET CLR 2.0.50727)	0	13249	200	OK	-	-	-	(empty)	-	-	-	-	-	FBcNS3RwceOxW15xg	text/plain
+1232039472.446194	CXWv6p3arKYeMETxOg	237.244.174.255	1905	79.218.110.244	80	2	GET	ads1.msn.com	/library/dap.js	http://zone.msn.com/en/root/default.htm	Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; .NET CLR 1.0.3705; .NET CLR 1.1.4322; Media Center PC 4.0; .NET CLR 2.0.50727)	0	13249	200	OK	-	-	-	(empty)	-	-	-	-	-	FDWU85N0DpedJPh93	text/plain
+#close	2015-05-12-16-26-53

testing/btest/core/raw_packet.bro

@@ -0,0 +1,9 @@
+# @TEST-EXEC: bro -b -r $TRACES/raw_packets.trace %INPUT >output
+# @TEST-EXEC: bro -b -r $TRACES/icmp_dot1q.trace %INPUT >>output
+# @TEST-EXEC: btest-diff output
+
+event raw_packet(p: raw_pkt_hdr)
+	{
+	print p;
+	}
+

testing/btest/core/reassembly.bro

@@ -0,0 +1,26 @@
+# @TEST-EXEC: bro -C -r $TRACES/ipv4/fragmented-1.pcap %INPUT >>output
+# @TEST-EXEC: bro -C -r $TRACES/ipv4/fragmented-2.pcap %INPUT >>output
+# @TEST-EXEC: bro -C -r $TRACES/ipv4/fragmented-3.pcap %INPUT >>output
+# @TEST-EXEC: bro -C -r $TRACES/ipv4/fragmented-4.pcap %INPUT >>output
+# @TEST-EXEC: bro -C -r $TRACES/tcp/reassembly.pcap %INPUT >>output
+# @TEST-EXEC: btest-diff output
+
+event bro_init()
+	{
+	print "----------------------";
+	}
+
+event flow_weird(name: string, src: addr, dst: addr)
+	{
+	print "flow weird", name, src, dst;
+	}
+
+event net_weird(name: string)
+	{
+	print "net_weird", name;
+	}
+
+event rexmit_inconsistency(c: connection, t1: string, t2: string)
+	{
+	print "rexmit_inconsistency", c$id, t1, t2 ;
+	}

testing/btest/core/tcp/quantum-insert.bro

@@ -0,0 +1,12 @@
+# @TEST-EXEC: bro -b -r $TRACES/tcp/qi_internet_SYNACK_curl_jsonip.pcap %INPUT
+# @TEST-EXEC: btest-diff .stdout
+
+# Quantum Insert like attack, overlapping TCP packet with different content
+redef tcp_max_old_segments = 10;
+event rexmit_inconsistency(c: connection, t1: string, t2: string)
+    {
+    print "----- rexmit_inconsistency -----";
+    print fmt("%.6f c: %s", network_time(), c$id);
+    print fmt("%.6f t1: %s", network_time(), t1);
+    print fmt("%.6f t2: %s", network_time(), t2);
+    }

testing/btest/plugins/pktdumper-plugin/src/Foo.cc

@@ -29,8 +29,8 @@ void Foo::Close()
 
 bool Foo::Dump(const Packet* pkt)
 	{
-	double t = double(pkt->hdr->ts.tv_sec) + double(pkt->hdr->ts.tv_usec) / 1e6;
+	double t = double(pkt->ts.tv_sec) + double(pkt->ts.tv_usec) / 1e6;
-	fprintf(stdout, "Dumping to %s: %.6f len %u\n", props.path.c_str(), t, (unsigned int)pkt->hdr->len);
+	fprintf(stdout, "Dumping to %s: %.6f len %u\n", props.path.c_str(), t, (unsigned int)pkt->len);
 	return true;
 	}
 

testing/btest/plugins/pktsrc-plugin/src/Foo.cc

@@ -17,7 +17,6 @@ Foo::Foo(const std::string& path, bool is_live)
 	props.path = path;
 	props.selectable_fd = open("/bin/sh", O_RDONLY); // any fd is fine.
 	props.link_type = DLT_RAW;
-	props.hdr_size = 0;
 	props.netmask = 0;
 	props.is_live = 0;
 	}
@@ -45,12 +44,9 @@ bool Foo::ExtractNextPacket(Packet* pkt)
 		return false;
 		}
 
-	hdr.ts.tv_sec = 1409193037;
+	struct timeval ts = { 1409193037, 0 };
-	hdr.ts.tv_usec = 0;
+	pkt->Init(props.link_type, &ts, packet.size(), packet.size(), 
-	hdr.caplen = hdr.len = packet.size();
+		(const u_char *)packet.c_str());
-	pkt->ts = hdr.ts.tv_sec;
-	pkt->hdr = &hdr;
-	pkt->data = (const u_char *)packet.c_str();
 	return true;
 	}
 

testing/btest/plugins/pktsrc-plugin/src/Foo.h

@@ -26,7 +26,6 @@ protected:
 private:
 	Properties props;
 	string packet;
-	struct pcap_pkthdr hdr;
 };
 
 }

testing/btest/scripts/base/protocols/http/missing-zlib-header.bro

@@ -0,0 +1,6 @@
+# This tests an issue where some web servers don't 
+# include an appropriate ZLIB header on deflated 
+# content.
+#
+# @TEST-EXEC: bro -r $TRACES/http/missing-zlib-header.pcap %INPUT
+# @TEST-EXEC: btest-diff http.log

testing/btest/scripts/policy/frameworks/intel/seen/certs.bro

@@ -16,3 +16,13 @@ www.dresdner-privat.de	Intel::DOMAIN	source1	test entry	http://some-data-distrib
 
 redef Intel::read_files += { "intel.dat" };
 
+event bro_init()
+	{
+	suspend_processing();
+	}
+
+event Input::end_of_data(name: string, source: string)
+	{
+	continue_processing();
+	}
+

testing/external/subdir-btest.cfg

@@ -15,6 +15,7 @@ TEST_DIFF_CANONIFIER=%(testbase)s/../../scripts/diff-canonifier-external
 TEST_DIFF_BRIEF=1
 TRACES=%(testbase)s/Traces
 SCRIPTS=%(testbase)s/../scripts
+SCRIPTS_LOCAL=%(testbase)s/scripts
 DIST=%(testbase)s/../../..
 BUILD=%(testbase)s/../../../build
 BRO_PROFILER_FILE=%(testbase)s/.tmp/script-coverage.XXXXXX