Merge branch 'master' of https://github.com/zeek/zeek

CHANGES

@@ -1,4 +1,89 @@
 
+3.2.0-dev.271 | 2020-03-17 22:52:40 -0700
+
+  * Scope: store IntrusivePtr in `local` (Max Kellermann)
+
+  * Scope: pass IntrusivePtr to AddInit() (Max Kellermann)
+
+  * DNS_Mgr: use class IntrusivePtr (Max Kellermann)
+
+  * Scope: use class IntrusivePtr (Max Kellermann)
+
+  * Attr: use class IntrusivePtr (Max Kellermann)
+
+  * Expr: check_and_promote_expr() returns IntrusivePtr (Max Kellermann)
+
+    Instead of returning a pseudo-boolean integer, it now returns a
+    referenced object or nullptr on error.  The old API was very error
+    prone because of its obscure reference counting semantics.
+
+  * Frame: use class IntrusivePtr (Max Kellermann)
+
+  * Val: RecordVal::LookupWithDefault() returns IntrusivePtr (Max Kellermann)
+
+  * Type: RecordType::FieldDefault() returns IntrusivePtr (Max Kellermann)
+
+    Fixes memory leak in EventHandler::NewEvent().
+
+  * Val: TableVal::Delete() returns IntrusivePtr (Max Kellermann)
+
+  * Type: base_type() returns IntrusivePtr (Max Kellermann)
+
+  * Type: init_type() returns IntrusivePtr (Max Kellermann)
+
+  * Type: merge_types() returns IntrusivePtr (Max Kellermann)
+
+  * Type: use class IntrusivePtr in VectorType (Max Kellermann)
+
+  * Type: use class IntrusivePtr in EnumType (Max Kellermann)
+
+  * Type: use class IntrusivePtr in FileType (Max Kellermann)
+
+  * Type: use class IntrusivePtr in TypeDecl (Max Kellermann)
+
+  * Type: make TypeDecl `final` and the dtor non-`virtual` (Max Kellermann)
+
+  * Type: use class IntrusivePtr in TypeType (Max Kellermann)
+
+  * Type: use class IntrusivePtr in FuncType (Max Kellermann)
+
+  * Type: use class IntrusivePtr in TypeList (Max Kellermann)
+
+  * Type: use class IntrusivePtr in IndexType (Max Kellermann)
+
+  * Val: use class IntrusivePtr in class TableVal (Max Kellermann)
+
+  * Val: use class IntrusivePtr in class TableEntryVal (Max Kellermann)
+
+  * CompHash: return IntrusivePtr (Max Kellermann)
+
+  * Type: return IntrusivePtr (Max Kellermann)
+
+  * Val: add TableVal::Assign() overload with IntrusivePtr (Max Kellermann)
+
+  * zeekygen/IdentifierInfo: use class IntrusivePtr more (Max Kellermann)
+
+  * Func: use class IntrusivePtr (Max Kellermann)
+
+  * GH-845: reference `id_list` before passing to new `BroFunc` (Max Kellermann)
+
+    Passing the `id_list` pointer to `BroFunc` transfers ownership of the
+    contained `ID` instances, because `~BroFunc()` unreferences them.
+    Therefore, we need to increase the reference counters for each
+    `BroFunc` instance to fix the use-after-free bug.
+
+  * input/Manager: fix three use-after-free bugs (Max Kellermann)
+
+3.2.0-dev.237 | 2020-03-17 16:54:35 -0700
+
+  * Cleanup in iosource/Packet (Tim Wojtulewicz, Corelight)
+
+  * Add ability for packet sources to flag a packet's l2 or l3 checksum as valid.
+
+    This lets packet source plugins implement handling of hardware checksum
+    offloading, if available. Setting the flags will skip the internal checksumming
+    for either layer 2 and/or layer 3. (Tim Wojtulewicz, Corelight)
+
 3.2.0-dev.234 | 2020-03-16 12:37:17 -0700
 
   * Remove an unused smb2_negotiate_response event handler. (M.Shirk)

VERSION

@@ -1 +1 @@
-3.2.0-dev.234
+3.2.0-dev.271

src/Sessions.cc

@@ -251,7 +251,7 @@ void NetSessions::DoNextPacket(double t, const Packet* pkt, const IP_Hdr* ip_hdr
 	if ( packet_filter && packet_filter->Match(ip_hdr, len, caplen) )
 		 return;
 
-	if ( ! ignore_checksums && ip4 &&
+	if ( ! pkt->l2_checksummed && ! ignore_checksums && ip4 &&
 	     ones_complement_checksum((void*) ip4, ip_hdr_len, 0) != 0xffff )
 		{
 		Weird("bad_IP_checksum", pkt, encapsulation);

src/analyzer/protocol/icmp/ICMP.cc

@@ -333,7 +333,7 @@ RecordVal* ICMP_Analyzer::ExtractICMP4Context(int len, const u_char*& data)
 		{
 		bad_hdr_len = 0;
 		ip_len = ip_hdr->TotalLen();
-		bad_checksum = (ones_complement_checksum((void*) ip_hdr->IP4_Hdr(), ip_hdr_len, 0) != 0xffff);
+		bad_checksum = ! current_pkt->l3_checksummed && (ones_complement_checksum((void*) ip_hdr->IP4_Hdr(), ip_hdr_len, 0) != 0xffff);
 
 		src_addr = ip_hdr->SrcAddr();
 		dst_addr = ip_hdr->DstAddr();

src/analyzer/protocol/tcp/TCP.cc

@@ -275,7 +275,7 @@ const struct tcphdr* TCP_Analyzer::ExtractTCP_Header(const u_char*& data,
 bool TCP_Analyzer::ValidateChecksum(const struct tcphdr* tp,
 				TCP_Endpoint* endpoint, int len, int caplen)
 	{
-	if ( ! ignore_checksums && caplen >= len &&
+	if ( ! current_pkt->l3_checksummed && ! ignore_checksums && caplen >= len &&
 	     ! endpoint->ValidChecksum(tp, len) )
 		{
 		Weird("bad_TCP_checksum");

src/analyzer/protocol/udp/UDP.cc

@@ -62,7 +62,7 @@ void UDP_Analyzer::DeliverPacket(int len, const u_char* data, bool is_orig,
 
 	int chksum = up->uh_sum;
 
-	auto validate_checksum = ! ignore_checksums && caplen >=len;
+	auto validate_checksum = ! current_pkt->l3_checksummed && ! ignore_checksums && caplen >=len;
 	constexpr auto vxlan_len = 8;
 	constexpr auto eth_len = 14;
 

src/iosource/Packet.cc

@@ -43,14 +43,17 @@ void Packet::Init(int arg_link_type, pkt_timeval *arg_ts, uint32_t arg_caplen,
 
 	time = ts.tv_sec + double(ts.tv_usec) / 1e6;
 	hdr_size = GetLinkHeaderSize(arg_link_type);
-	l3_proto = L3_UNKNOWN;
 	eth_type = 0;
 	vlan = 0;
 	inner_vlan = 0;
-	l2_src = 0;
-	l2_dst = 0;
 
+	l2_src = nullptr;
+	l2_dst = nullptr;
 	l2_valid = false;
+	l2_checksummed = false;
+
+	l3_proto = L3_UNKNOWN;
+	l3_checksummed = false;
 
 	if ( data && cap_len < hdr_size )
 		{
@@ -677,4 +680,3 @@ void Packet::Describe(ODesc* d) const
 	d->Add("->");
 	d->Add(ip.DstAddr());
 	}
-

src/iosource/Packet.h

@@ -19,8 +19,10 @@ class RecordVal;
 
 /**
  * The Layer 3 type of a packet, as determined by the parsing code in Packet.
+ * This enum is sized as an int32_t to make the Packet structure align
+ * correctly.
  */
-enum Layer3Proto {
+enum Layer3Proto : int32_t {
 	L3_UNKNOWN = -1,	/// Layer 3 type could not be determined.
 	L3_IPV4 = 1,		/// Layer 3 is IPv4.
 	L3_IPV6 = 2,		/// Layer 3 is IPv6.
@@ -110,7 +112,7 @@ public:
 	 * Returns true if parsing the layer 2 fields failed, including when
 	 * no data was passed into the constructor in the first place.
 	 */
-	bool Layer2Valid()
+	bool Layer2Valid() const
 		{
 		return l2_valid;
 		}
@@ -199,8 +201,20 @@ public:
 	 */
 	uint32_t inner_vlan;
 
+	/**
+	 * Indicates whether the layer 2 checksum was validated by the
+	 * hardware/kernel before being received by zeek.
+	 */
+	bool l2_checksummed;
+
+	/**
+	 * Indicates whether the layer 3 checksum was validated by the
+	 * hardware/kernel before being received by zeek.
+	 */
+	bool l3_checksummed;
+
 private:
-	// Calculate layer 2 attributes. Sets
+	// Calculate layer 2 attributes.
 	void ProcessLayer2();
 
 	// Wrapper to generate a packet-level weird.