Switch one's complement checksum implementation

Borrows the `in_cksum` code from tcpdump, which borrowed from FreeBSD.
It handles unaligned data better and also unrolls the inner loop to
process 16 two-byte values at a time versus 2 one-byte values at a time
in the previous version.  Generally measured as ~1.5x faster in a
release build.  The new API should generally be more amenable to any
future optimization explorations since all relevant data blocks are
available within a single call rather than spread across multiple.

COPYING.3rdparty

@@ -250,6 +250,39 @@ PROJECT (https://github.com/zeek) UNDER BSD LICENCE.
 
 ==============================================================================
 
+%%% in_cksum.cc
+
+==============================================================================
+
+Copyright (c) 1988, 1992, 1993
+The Regents of the University of California.  All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+1. Redistributions of source code must retain the above copyright
+   notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+   notice, this list of conditions and the following disclaimer in the
+   documentation and/or other materials provided with the distribution.
+3. Neither the name of the University nor the names of its contributors
+   may be used to endorse or promote products derived from this software
+   without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+SUCH DAMAGE.
+
+==============================================================================
+
 %%% Patricia.c
 
 ==============================================================================

src/CMakeLists.txt

@@ -296,6 +296,7 @@ set(MAIN_SRCS
     legacy-netvar-init.cc
     bsd-getopt-long.c
     bro_inet_ntop.c
+    in_cksum.cc
     patricia.c
     setsignal.c
     strsep.c

src/Sessions.cc

@@ -269,7 +269,7 @@ void NetSessions::DoNextPacket(double t, const Packet* pkt, const IP_Hdr* ip_hdr
 		 return;
 
 	if ( ! pkt->l2_checksummed && ! zeek::detail::ignore_checksums && ip4 &&
-	     ones_complement_checksum((void*) ip4, ip_hdr_len, 0) != 0xffff )
+	     detail::in_cksum(reinterpret_cast<const uint8_t*>(ip4), ip_hdr_len) != 0xffff )
 		{
 		Weird("bad_IP_checksum", pkt, encapsulation);
 		return;

src/analyzer/protocol/icmp/ICMP.cc

@@ -346,7 +346,8 @@ RecordValPtr ICMP_Analyzer::ExtractICMP4Context(int len, const u_char*& data)
 		bad_hdr_len = 0;
 		ip_len = ip_hdr->TotalLen();
 		bad_checksum = ! run_state::current_pkt->l3_checksummed &&
-			(ones_complement_checksum((void*) ip_hdr->IP4_Hdr(), ip_hdr_len, 0) != 0xffff);
+		  (detail::in_cksum(reinterpret_cast<const uint8_t*>(ip_hdr->IP4_Hdr()),
+		                    ip_hdr_len) != 0xffff);
 
 		src_addr = ip_hdr->SrcAddr();
 		dst_addr = ip_hdr->DstAddr();

src/analyzer/protocol/tcp/TCP.cc

@@ -273,10 +273,10 @@ const struct tcphdr* TCP_Analyzer::ExtractTCP_Header(const u_char*& data,
 	}
 
 bool TCP_Analyzer::ValidateChecksum(const struct tcphdr* tp,
-				TCP_Endpoint* endpoint, int len, int caplen)
+				TCP_Endpoint* endpoint, int len, int caplen, bool ipv4)
 	{
 	if ( ! run_state::current_pkt->l3_checksummed && ! detail::ignore_checksums && caplen >= len &&
-	     ! endpoint->ValidChecksum(tp, len) )
+	     ! endpoint->ValidChecksum(tp, len, ipv4) )
 		{
 		Weird("bad_TCP_checksum");
 		endpoint->ChecksumError();
@@ -1060,7 +1060,7 @@ void TCP_Analyzer::DeliverPacket(int len, const u_char* data, bool is_orig,
 	TCP_Endpoint* endpoint = is_orig ? orig : resp;
 	TCP_Endpoint* peer = endpoint->peer;
 
-	if ( ! ValidateChecksum(tp, endpoint, len, caplen) )
+	if ( ! ValidateChecksum(tp, endpoint, len, caplen, ip->IP4_Hdr()) )
 		return;
 
 	uint32_t tcp_hdr_len = data - (const u_char*) tp;

src/analyzer/protocol/tcp/TCP.h

@@ -95,7 +95,7 @@ protected:
 	// Returns true if the checksum is valid, false if not (and in which
 	// case also updates the status history of the endpoint).
 	bool ValidateChecksum(const struct tcphdr* tp, TCP_Endpoint* endpoint,
-				int len, int caplen);
+				int len, int caplen, bool ipv4);
 
 	void SetPartialStatus(TCP_Flags flags, bool is_orig);
 

src/analyzer/protocol/tcp/TCP_Endpoint.cc

@@ -41,14 +41,6 @@ TCP_Endpoint::TCP_Endpoint(TCP_Analyzer* arg_analyzer, bool arg_is_orig)
 
 	src_addr = is_orig ? Conn()->RespAddr() : Conn()->OrigAddr();
 	dst_addr = is_orig ? Conn()->OrigAddr() : Conn()->RespAddr();
-
-	checksum_base = ones_complement_checksum(src_addr, 0);
-	checksum_base = ones_complement_checksum(dst_addr, checksum_base);
-	// Note, for IPv6, strictly speaking this field is 32 bits
-	// rather than 16 bits.  But because the upper bits are all zero,
-	// we get the same checksum either way.  The same applies to
-	// later when we add in the data length in ValidChecksum().
-	checksum_base += htons(IPPROTO_TCP);
 	}
 
 TCP_Endpoint::~TCP_Endpoint()
@@ -121,17 +113,12 @@ void TCP_Endpoint::SizeBufferedData(uint64_t& waiting_on_hole,
 		waiting_on_hole = waiting_on_ack = 0;
 	}
 
-bool TCP_Endpoint::ValidChecksum(const struct tcphdr* tp, int len) const
+bool TCP_Endpoint::ValidChecksum(const struct tcphdr* tp, int len, bool ipv4) const
 	{
-	uint32_t sum = checksum_base;
 	int tcp_len = tp->th_off * 4 + len;
 
-	if ( len % 2 == 1 )
+	auto sum = detail::ip_in_cksum(ipv4, src_addr, dst_addr, IPPROTO_TCP,
-		// Add in pad byte.
+	                               reinterpret_cast<const uint8_t*>(tp), tcp_len);
-		sum += htons(((const u_char*) tp)[tcp_len - 1] << 8);
-
-	sum += htons((unsigned short) tcp_len);	// fill out pseudo header
-	sum = ones_complement_checksum((void*) tp, tcp_len, sum);
 
 	return sum == 0xffff;
 	}

src/analyzer/protocol/tcp/TCP_Endpoint.h

@@ -165,7 +165,7 @@ public:
 	// WARNING: this is an O(n) operation and potentially very slow.
 	void SizeBufferedData(uint64_t& waiting_on_hole, uint64_t& waiting_on_ack);
 
-	bool ValidChecksum(const struct tcphdr* tp, int len) const;
+	bool ValidChecksum(const struct tcphdr* tp, int len, bool ipv4) const;
 
 	// Called to inform endpoint that it has generated a checksum error.
 	void ChecksumError();
@@ -211,7 +211,6 @@ public:
 	TCP_Reassembler* contents_processor;
 	TCP_Analyzer* tcp_analyzer;
 	FilePtr contents_file;
-	uint32_t checksum_base;
 
 	double start_time, last_time;
 	IPAddr src_addr; // the other endpoint

src/analyzer/protocol/udp/UDP.cc

@@ -260,22 +260,9 @@ void UDP_Analyzer::ChecksumEvent(bool is_orig, uint32_t threshold)
 
 bool UDP_Analyzer::ValidateChecksum(const IP_Hdr* ip, const udphdr* up, int len)
 	{
-	uint32_t sum;
+	auto sum = detail::ip_in_cksum(ip->IP4_Hdr(), ip->SrcAddr(), ip->DstAddr(),
-
+	                               IPPROTO_UDP,
-	if ( len % 2 == 1 )
+	                               reinterpret_cast<const uint8_t*>(up), len);
-		// Add in pad byte.
-		sum = htons(((const u_char*) up)[len - 1] << 8);
-	else
-		sum = 0;
-
-	sum = ones_complement_checksum(ip->SrcAddr(), sum);
-	sum = ones_complement_checksum(ip->DstAddr(), sum);
-	// Note, for IPv6, strictly speaking the protocol and length fields are
-	// 32 bits rather than 16 bits.  But because the upper bits are all zero,
-	// we get the same checksum either way.
-	sum += htons(IPPROTO_UDP);
-	sum += htons((unsigned short) len);
-	sum = ones_complement_checksum((void*) up, len, sum);
 
 	return sum == 0xffff;
 	}

src/in_cksum.cc

@@ -0,0 +1,141 @@
+// Modified from tcpdump v4.9.3's in_cksum.c (which itself was a modified
+// version of FreeBSD's in_cksum.c).
+
+/* in_cksum.c
+ * 4.4-Lite-2 Internet checksum routine, modified to take a vector of
+ * pointers/lengths giving the pieces to be checksummed.  Also using
+ * Tahoe/CGI version of ADDCARRY(x) macro instead of from portable version.
+ */
+
+/*
+ * Copyright (c) 1988, 1992, 1993
+ *	The Regents of the University of California.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. Neither the name of the University nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ *	@(#)in_cksum.c	8.1 (Berkeley) 6/10/93
+ */
+
+#include "net_util.h"
+
+namespace zeek::detail {
+
+#define ADDCARRY(x)  {if ((x) > 65535) (x) -= 65535;}
+#define REDUCE {l_util.l = sum; sum = l_util.s[0] + l_util.s[1]; ADDCARRY(sum);}
+
+uint16_t in_cksum(const struct checksum_block *vec, int veclen)
+{
+	const uint16_t *w;
+	int sum = 0;
+	int mlen = 0;
+	int byte_swapped = 0;
+
+	union {
+		uint8_t		c[2];
+		uint16_t	s;
+	} s_util;
+	union {
+		uint16_t	s[2];
+		uint32_t	l;
+	} l_util;
+
+	for (; veclen != 0; vec++, veclen--) {
+		if (vec->len == 0)
+			continue;
+		w = (const uint16_t *)(const void *)vec->block;
+		if (mlen == -1) {
+			/*
+			 * The first byte of this chunk is the continuation
+			 * of a word spanning between this chunk and the
+			 * last chunk.
+			 *
+			 * s_util.c[0] is already saved when scanning previous
+			 * chunk.
+			 */
+			s_util.c[1] = *(const uint8_t *)w;
+			sum += s_util.s;
+			w = (const uint16_t *)(const void *)((const uint8_t *)w + 1);
+			mlen = vec->len - 1;
+		} else
+			mlen = vec->len;
+		/*
+		 * Force to even boundary.
+		 */
+		if ((1 & (uintptr_t) w) && (mlen > 0)) {
+			REDUCE;
+			sum <<= 8;
+			s_util.c[0] = *(const uint8_t *)w;
+			w = (const uint16_t *)(const void *)((const uint8_t *)w + 1);
+			mlen--;
+			byte_swapped = 1;
+		}
+		/*
+		 * Unroll the loop to make overhead from
+		 * branches &c small.
+		 */
+		while ((mlen -= 32) >= 0) {
+			sum += w[0]; sum += w[1]; sum += w[2]; sum += w[3];
+			sum += w[4]; sum += w[5]; sum += w[6]; sum += w[7];
+			sum += w[8]; sum += w[9]; sum += w[10]; sum += w[11];
+			sum += w[12]; sum += w[13]; sum += w[14]; sum += w[15];
+			w += 16;
+		}
+		mlen += 32;
+		while ((mlen -= 8) >= 0) {
+			sum += w[0]; sum += w[1]; sum += w[2]; sum += w[3];
+			w += 4;
+		}
+		mlen += 8;
+		if (mlen == 0 && byte_swapped == 0)
+			continue;
+		REDUCE;
+		while ((mlen -= 2) >= 0) {
+			sum += *w++;
+		}
+		if (byte_swapped) {
+			REDUCE;
+			sum <<= 8;
+			byte_swapped = 0;
+			if (mlen == -1) {
+				s_util.c[1] = *(const uint8_t *)w;
+				sum += s_util.s;
+				mlen = 0;
+			} else
+				mlen = -1;
+		} else if (mlen == -1)
+			s_util.c[0] = *(const uint8_t *)w;
+	}
+	if (mlen == -1) {
+		/* The last mbuf has odd # of bytes. Follow the
+		   standard (the odd byte may be shifted left by 8 bits
+		   or not as determined by endian-ness of the machine) */
+		s_util.c[1] = 0;
+		sum += s_util.s;
+	}
+	REDUCE;
+	return sum;
+}
+
+} // namespace zeek

src/net_util.cc

@@ -16,8 +16,49 @@
 
 namespace zeek {
 
-// - adapted from tcpdump
+uint16_t detail::ip4_in_cksum(const IPAddr& src, const IPAddr& dst,
-// Returns the ones-complement checksum of a chunk of b short-aligned bytes.
+                              uint8_t next_proto, const uint8_t* data, int len)
+	{
+	constexpr auto nblocks = 2;
+	detail::checksum_block blocks[nblocks];
+
+	ipv4_pseudo_hdr ph;
+	memset(&ph, 0, sizeof(ph));
+
+	src.CopyIPv4(&ph.src);
+	dst.CopyIPv4(&ph.dst);
+	ph.len = htons(static_cast<uint16_t>(len));
+	ph.next_proto = next_proto;
+	blocks[0].block = reinterpret_cast<const uint8_t*>(&ph);
+	blocks[0].len = sizeof(ph);
+	blocks[1].block = data;
+	blocks[1].len = len;
+
+	return in_cksum(blocks, nblocks);
+	}
+
+uint16_t detail::ip6_in_cksum(const IPAddr& src, const IPAddr& dst,
+                              uint8_t next_proto, const uint8_t* data, int len)
+	{
+	constexpr auto nblocks = 2;
+	detail::checksum_block blocks[nblocks];
+
+	ipv6_pseudo_hdr ph;
+	memset(&ph, 0, sizeof(ph));
+
+	src.CopyIPv6(&ph.src);
+	dst.CopyIPv6(&ph.dst);
+	ph.len = htonl(static_cast<uint32_t>(len));
+	ph.next_proto = next_proto;
+	blocks[0].block = reinterpret_cast<const uint8_t*>(&ph);
+	blocks[0].len = sizeof(ph);
+	blocks[1].block = data;
+	blocks[1].len = len;
+
+	return in_cksum(blocks, nblocks);
+	}
+
+// Returns the ones-complement checksum of a chunk of 'b' bytes.
 int ones_complement_checksum(const void* p, int b, uint32_t sum)
 	{
 	const unsigned char* sp = (unsigned char*) p;
@@ -46,17 +87,7 @@ int ones_complement_checksum(const IPAddr& a, uint32_t sum)
 
 int icmp_checksum(const struct icmp* icmpp, int len)
 	{
-	uint32_t sum;
+	return detail::in_cksum(reinterpret_cast<const uint8_t*>(icmpp), len);
-
-	if ( len % 2 == 1 )
-		// Add in pad byte.
-		sum = htons(((const u_char*) icmpp)[len - 1] << 8);
-	else
-		sum = 0;
-
-	sum = ones_complement_checksum((void*) icmpp, len, sum);
-
-	return sum;
 	}
 
 #ifdef ENABLE_MOBILE_IPV6
@@ -89,26 +120,8 @@ int icmp6_checksum(const struct icmp* icmpp, const IP_Hdr* ip, int len)
 	{
 	// ICMP6 uses the same checksum function as ICMP4 but a different
 	// pseudo-header over which it is computed.
-	uint32_t sum;
+	return detail::ip6_in_cksum(ip->SrcAddr(), ip->DstAddr(), IPPROTO_ICMPV6,
-
+	                            reinterpret_cast<const uint8_t*>(icmpp), len);
-	if ( len % 2 == 1 )
-		// Add in pad byte.
-		sum = htons(((const u_char*) icmpp)[len - 1] << 8);
-	else
-		sum = 0;
-
-	// Pseudo-header as for UDP over IPv6 above.
-	sum = ones_complement_checksum(ip->SrcAddr(), sum);
-	sum = ones_complement_checksum(ip->DstAddr(), sum);
-	uint32_t l = htonl(len);
-	sum = ones_complement_checksum((void*) &l, 4, sum);
-
-	uint32_t addl_pseudo = htons(IPPROTO_ICMPV6);
-	sum = ones_complement_checksum((void*) &addl_pseudo, 4, sum);
-
-	sum = ones_complement_checksum((void*) icmpp, len, sum);
-
-	return sum;
 	}
 
 

src/net_util.h

@@ -121,7 +121,54 @@ ZEEK_FORWARD_DECLARE_NAMESPACED(IP_Hdr, zeek);
 
 namespace zeek {
 
-// Returns the ones-complement checksum of a chunk of b short-aligned bytes.
+namespace detail {
+
+struct checksum_block {
+	const uint8_t* block;
+	int len;
+};
+
+struct ipv4_pseudo_hdr {
+	in_addr  src;
+	in_addr  dst;
+	uint8_t  zero;
+	uint8_t  next_proto;
+	uint16_t len;
+};
+
+struct ipv6_pseudo_hdr {
+	in6_addr src;
+	in6_addr dst;
+	uint32_t len;
+	uint8_t  zero[3];
+	uint8_t  next_proto;
+};
+
+extern uint16_t in_cksum(const checksum_block* blocks, int num_blocks);
+
+inline uint16_t in_cksum(const uint8_t* data, int len)
+	{
+	checksum_block cb{data, len};
+	return in_cksum(&cb, 1);
+	}
+
+extern uint16_t ip4_in_cksum(const IPAddr& src, const IPAddr& dst,
+                             uint8_t next_proto, const uint8_t* data, int len);
+
+extern uint16_t ip6_in_cksum(const IPAddr& src, const IPAddr& dst,
+                             uint8_t next_proto, const uint8_t* data, int len);
+
+inline uint16_t ip_in_cksum(bool is_ipv4, const IPAddr& src, const IPAddr& dst,
+                            uint8_t next_proto, const uint8_t* data, int len)
+	{
+	if ( is_ipv4 )
+		return ip4_in_cksum(src, dst, next_proto, data, len);
+	return ip6_in_cksum(src, dst, next_proto, data, len);
+	}
+
+} // namespace zeek::detail
+
+// Returns the ones-complement checksum of a chunk of 'b' bytes.
 extern int ones_complement_checksum(const void* p, int b, uint32_t sum);
 
 extern int ones_complement_checksum(const IPAddr& a, uint32_t sum);