Merge branch 'topic/yunzheng/bit-1314'

I've worked on this a bit more:

    - Added tcp_max_old_segments to init-bare.bro.
    - Removed the existing call to Overlap() as that now led to
      duplicate events.
    - Fixed the code checking for overlaps, as it didn't catch all the
      cases.

BIT-1314 #merged
GitHub #31 merged

* topic/yunzheng/bit-1314:
  BIT-1314: Added QI test for rexmit_inconsistency
  BIT-1314: Add detection for Quantum Insert attacks

CHANGES

@@ -1,4 +1,13 @@
 
+2.4-17 | 2015-06-28 13:02:41 -0700
+
+  * BIT-1314: Add detection for Quantum Insert attacks. The TCP
+    reassembler can now keep a history of old TCP segments using the
+    tcp_max_old_segments option. An overlapping segment with different
+    data will then generate an rexmit_inconsistency event. The default
+    for tcp_max_old_segments is zero, which disabled any additional
+    buffering. (Yun Zheng Hu)
+
 2.4-14 | 2015-06-28 12:30:12 -0700
 
   * BIT-1400: Allow '<' and '>' in MIME multipart boundaries. The spec

VERSION

@@ -1 +1 @@
-2.4-14
+2.4-17

scripts/base/init-bare.bro

@@ -954,6 +954,11 @@ const tcp_max_above_hole_without_any_acks = 16384 &redef;
 ## .. bro:see:: tcp_max_initial_window tcp_max_above_hole_without_any_acks
 const tcp_excessive_data_without_further_acks = 10 * 1024 * 1024 &redef;
 
+## Number of TCP segments to buffer beyond what's been acknowledged already
+## to detect retransmission inconsistencies. Zero disables any additonal
+## buffering.
+const tcp_max_old_segments = 0 &redef;
+
 ## For services without a handler, these sets define originator-side ports
 ## that still trigger reassembly.
 ##

src/NetVar.cc

@@ -49,6 +49,7 @@ double tcp_partial_close_delay;
 int tcp_max_initial_window;
 int tcp_max_above_hole_without_any_acks;
 int tcp_excessive_data_without_further_acks;
+int tcp_max_old_segments;
 
 RecordType* socks_address;
 
@@ -354,6 +355,7 @@ void init_net_var()
 		opt_internal_int("tcp_max_above_hole_without_any_acks");
 	tcp_excessive_data_without_further_acks =
 		opt_internal_int("tcp_excessive_data_without_further_acks");
+	tcp_max_old_segments = opt_internal_int("tcp_max_old_segments");
 
 	socks_address = internal_type("SOCKS::Address")->AsRecordType();
 

src/NetVar.h

@@ -52,6 +52,7 @@ extern double tcp_reset_delay;
 extern int tcp_max_initial_window;
 extern int tcp_max_above_hole_without_any_acks;
 extern int tcp_excessive_data_without_further_acks;
+extern int tcp_max_old_segments;
 
 extern RecordType* socks_address;
 

src/Reassem.cc

@@ -34,12 +34,53 @@ uint64 Reassembler::total_size = 0;
 Reassembler::Reassembler(uint64 init_seq)
 	{
 	blocks = last_block = 0;
+	old_blocks = last_old_block = 0;
+	total_old_blocks = max_old_blocks = 0;
 	trim_seq = last_reassem_seq = init_seq;
 	}
 
 Reassembler::~Reassembler()
 	{
 	ClearBlocks();
+	ClearOldBlocks();
+	}
+
+void Reassembler::CheckOverlap(DataBlock *head, DataBlock *tail,
+					uint64 seq, uint64 len, const u_char* data)
+	{
+	if ( ! head || ! tail )
+		return;
+
+	uint64 orig_seq = seq;
+	uint64 orig_upper = seq + len;
+	const u_char* orig_data = data;
+
+	for ( DataBlock* b = head; b; b = b->next )
+		{
+		uint64 seq = orig_seq;
+		uint64 upper = orig_upper;
+		const u_char* data = orig_data;
+
+		if ( upper <= b->seq )
+			continue;
+
+		if ( seq >= b->upper )
+			continue;
+
+		if ( seq < b->seq )
+			{
+			data += (b->seq - seq);
+			seq = b->seq;
+			}
+
+		if ( upper > b->upper )
+			upper = b->upper;
+
+		uint64 overlap_offset = seq - b->seq;
+		uint64 overlap_len = upper - seq;
+		if ( overlap_len )
+			Overlap(&b->block[overlap_offset], data, overlap_len);
+		}
 	}
 
 void Reassembler::NewBlock(double t, uint64 seq, uint64 len, const u_char* data)
@@ -49,10 +90,14 @@ void Reassembler::NewBlock(double t, uint64 seq, uint64 len, const u_char* data)
 
 	uint64 upper_seq = seq + len;
 
+	CheckOverlap(old_blocks, last_old_block, seq, len, data);
+
 	if ( upper_seq <= trim_seq )
 		// Old data, don't do any work for it.
 		return;
 
+	CheckOverlap(blocks, last_block, seq, len, data);
+
 	if ( seq < trim_seq )
 		{ // Partially old data, just keep the good stuff.
 		uint64 amount_old = trim_seq - seq;
@@ -119,6 +164,35 @@ uint64 Reassembler::TrimToSeq(uint64 seq)
 				num_missing += seq - blocks->upper;
 			}
 
+		if ( max_old_blocks )
+			{
+			// Move block over to old_blocks queue.
+			blocks->next = 0;
+
+			if ( last_old_block )
+				{
+				blocks->prev = last_old_block;
+				last_old_block->next = blocks;
+				}
+			else
+				{
+				blocks->prev = 0;
+				old_blocks = blocks;
+				}
+
+			last_old_block = blocks;
+			total_old_blocks++;
+
+			while ( old_blocks && total_old_blocks > max_old_blocks )
+				{
+				DataBlock* next = old_blocks->next;
+				delete old_blocks;
+				old_blocks = next;
+				total_old_blocks--;
+				}
+			}
+
+		else
 			delete blocks;
 
 		blocks = b;
@@ -156,6 +230,18 @@ void Reassembler::ClearBlocks()
 	last_block = 0;
 	}
 
+void Reassembler::ClearOldBlocks()
+	{
+	while ( old_blocks )
+		{
+		DataBlock* b = old_blocks->next;
+		delete old_blocks;
+		old_blocks = b;
+		}
+
+	last_old_block = 0;
+	}
+
 uint64 Reassembler::TotalSize() const
 	{
 	uint64 size = 0;
@@ -239,7 +325,7 @@ DataBlock* Reassembler::AddAndCheck(DataBlock* b, uint64 seq, uint64 upper,
 	uint64 b_len = b->upper - overlap_start;
 	uint64 overlap_len = min(new_b_len, b_len);
 
-	Overlap(&b->block[overlap_offset], data, overlap_len);
+//	Overlap(&b->block[overlap_offset], data, overlap_len);
 
 	if ( overlap_len < new_b_len )
 		{

src/Reassem.h

@@ -36,6 +36,7 @@ public:
 
 	// Delete all held blocks.
 	void ClearBlocks();
+	void ClearOldBlocks();
 
 	int HasBlocks() const		{ return blocks != 0; }
 	uint64 LastReassemSeq() const	{ return last_reassem_seq; }
@@ -50,6 +51,8 @@ public:
 	// Sum over all data buffered in some reassembler.
 	static uint64 TotalMemoryAllocation()	{ return total_size; }
 
+	void SetMaxOldBlocks(uint32 count)	{ max_old_blocks = count; }
+
 protected:
 	Reassembler()	{ }
 
@@ -65,10 +68,19 @@ protected:
 	DataBlock* AddAndCheck(DataBlock* b, uint64 seq,
 				uint64 upper, const u_char* data);
 
+	void CheckOverlap(DataBlock *head, DataBlock *tail,
+				uint64 seq, uint64 len, const u_char* data);
+
 	DataBlock* blocks;
 	DataBlock* last_block;
+
+	DataBlock* old_blocks;
+	DataBlock* last_old_block;
+
 	uint64 last_reassem_seq;
 	uint64 trim_seq;	// how far we've trimmed
+	uint32 max_old_blocks;
+	uint32 total_old_blocks;
 
 	static uint64 total_size;
 };

src/analyzer/protocol/tcp/TCP_Reassembler.cc

@@ -42,6 +42,9 @@ TCP_Reassembler::TCP_Reassembler(analyzer::Analyzer* arg_dst_analyzer,
 	seq_to_skip = 0;
 	in_delivery = false;
 
+	if ( tcp_max_old_segments )
+		SetMaxOldBlocks(tcp_max_old_segments);
+
 	if ( tcp_contents )
 		{
 		// Val dst_port_val(ntohs(Conn()->RespPort()), TYPE_PORT);

src/event.bif

@@ -282,7 +282,8 @@ event packet_contents%(c: connection, contents: string%);
 ## reassembling a TCP stream, Bro buffers all payload until it sees the
 ## responder acking it. If during that time, the sender resends a chunk of
 ## payload but with different content than originally, this event will be
-## raised.
+## raised. In addition, if :bro:id:`tcp_max_old_segments` is larger than zero,
+## mismatches with that older still-buffered data will likewise trigger the event.
 ##
 ## c: The connection showing the inconsistency.
 ##

testing/btest/Baseline/core.tcp.quantum-insert/.stdout

@@ -0,0 +1,4 @@
+----- rexmit_inconsistency -----
+1429652006.683290 c: [orig_h=178.200.100.200, orig_p=39976/tcp, resp_h=96.126.98.124, resp_p=80/tcp]
+1429652006.683290 t1: HTTP/1.1 200 OK\x0d\x0aContent-Length: 5\x0d\x0a\x0d\x0aBANG!
+1429652006.683290 t2: HTTP/1.1 200 OK\x0d\x0aServer: nginx/1.4.4\x0d\x0aDate:

testing/btest/core/tcp/quantum-insert.bro

@@ -0,0 +1,12 @@
+# @TEST-EXEC: bro -b -r $TRACES/tcp/qi_internet_SYNACK_curl_jsonip.pcap %INPUT
+# @TEST-EXEC: btest-diff .stdout
+
+# Quantum Insert like attack, overlapping TCP packet with different content
+redef tcp_max_old_segments = 10;
+event rexmit_inconsistency(c: connection, t1: string, t2: string)
+    {
+    print "----- rexmit_inconsistency -----";
+    print fmt("%.6f c: %s", network_time(), c$id);
+    print fmt("%.6f t1: %s", network_time(), t1);
+    print fmt("%.6f t2: %s", network_time(), t2);
+    }