Merge branch 'master' of https://github.com/rdenniston/zeek

* 'master' of https://github.com/rdenniston/zeek:
  Add linux netfilter NFLOG capture functionality initial commit

I made modifications:

  - Formatting / code style

  - More error handling and validity checks

  - The Type and Length value of TLVs is technically host order

  - Changed / fixed the Length value padding check: it's generally
    32-bit alignment, not just aligning any TLV less than 8 bytes.

CHANGES

@@ -1,4 +1,8 @@
 
+2.6-169 | 2019-03-19 19:12:47 -0700
+
+  * Add support for NFLOG link-layer type (Ryan Denniston)
+
 2.6-167 | 2019-03-18 13:58:28 -0700
 
   * GH-307: Build binpac as a shared lib, not static by default (Jon Siwek, Corelight)

NEWS

@@ -49,6 +49,8 @@ New Functionality
   - InputBinary::path_prefix
   - Intel::path_prefix
 
+- Support for NFLOG link-layer type.
+
 Changed Functionality
 ---------------------
 

VERSION

@@ -1 +1 @@
-2.6-167
+2.6-169

src/iosource/BPF_Program.cc

@@ -104,6 +104,17 @@ bool BPF_Program::Compile(int snaplen, int linktype, const char* filter,
 	{
 	FreeCode();
 
+	if ( linktype == DLT_NFLOG )
+		{
+		// No-op, NFLOG does not support BPF filters.
+		// Raising a warning might be good, but it would also be noisy
+		// since the default scripts will always attempt to compile
+		// and install a default filter
+		m_compiled = true;
+		m_matches_anything = true;
+		return true;
+		}
+
 #ifdef LIBPCAP_PCAP_COMPILE_NOPCAP_HAS_ERROR_PARAMETER
 	char my_error[PCAP_ERRBUF_SIZE];
 

src/iosource/Packet.cc

@@ -92,6 +92,14 @@ int Packet::GetLinkHeaderSize(int link_type)
 	case DLT_IEEE802_11_RADIO:	// 802.11 plus RadioTap
 		return 59;
 
+	case DLT_NFLOG:
+		// Linux netlink NETLINK NFLOG socket log messages
+		// The actual header size is variable, but we return the minimum
+		// expected size here, which is 4 bytes for the main header plus at
+		// least 2 bytes each for the type and length values assoicated with
+		// the final TLV carrying the packet payload.
+		return 8;
+
 	case DLT_RAW:
 		return 0;
 	}
@@ -395,6 +403,85 @@ void Packet::ProcessLayer2()
 		break;
 		}
 
+	case DLT_NFLOG:
+		{
+		// See https://www.tcpdump.org/linktypes/LINKTYPE_NFLOG.html
+
+		uint8 protocol = pdata[0];
+
+		if ( protocol == AF_INET )
+			l3_proto = L3_IPV4;
+		else if ( protocol == AF_INET6 )
+			l3_proto = L3_IPV6;
+		else
+			{
+			Weird("non_ip_in_nflog");
+			return;
+			}
+
+		uint8 version = pdata[1];
+
+		if ( version != 0 )
+			{
+			Weird("unknown_nflog_version");
+			return;
+			}
+
+		// Skip to TLVs.
+		pdata += 4;
+
+		uint16 tlv_len;
+		uint16 tlv_type;
+
+		while ( true )
+			{
+			if ( pdata + 4 >= end_of_data )
+				{
+				Weird("nflog_no_pcap_payload");
+				return;
+				}
+
+			// TLV Type and Length values are specified in host byte order
+			// (libpcap should have done any needed byteswapping already).
+
+			tlv_len = *(reinterpret_cast<const uint16*>(pdata));
+			tlv_type = *(reinterpret_cast<const uint16*>(pdata + 2));
+
+			auto constexpr nflog_type_payload = 9;
+
+			if ( tlv_type == nflog_type_payload )
+				{
+				// The raw packet payload follows this TLV.
+				pdata += 4;
+				break;
+				}
+			else
+				{
+				// The Length value includes the 4 octets for the Type and
+				// Length values, but TLVs are also implicitly padded to
+				// 32-bit alignments (that padding may not be included in
+				// the Length value).
+
+				if ( tlv_len < 4 )
+					{
+					Weird("nflog_bad_tlv_len");
+					return;
+					}
+				else
+					{
+					auto rem = tlv_len % 4;
+
+					if ( rem != 0 )
+						tlv_len += 4 - rem;
+					}
+
+				pdata += tlv_len;
+				}
+			}
+
+		break;
+		}
+
 	default:
 		{
 		// Assume we're pointing at IP. Just figure out which version.

src/iosource/pcap/Source.cc

@@ -247,11 +247,21 @@ bool PcapSource::SetFilter(int index)
 		return false;
 		}
 
+	if ( LinkType() == DLT_NFLOG )
+		{
+		// No-op, NFLOG does not support BPF filters.
+		// Raising a warning might be good, but it would also be noisy
+		// since the default scripts will always attempt to compile
+		// and install a default filter
+		}
+	else
+		{
 		if ( pcap_setfilter(pd, code->GetProgram()) < 0 )
 			{
 			PcapError();
 			return false;
 			}
+		}
 
 #ifndef HAVE_LINUX
 	// Linux doesn't clear counters when resetting filter.

testing/btest/Baseline/core.nflog/http.log

@@ -0,0 +1,10 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	http
+#open	2019-03-20-02-08-50
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	trans_depth	method	host	uri	referrer	version	user_agent	origin	request_body_len	response_body_len	status_code	status_msg	info_code	info_msg	tags	username	password	proxied	orig_fuids	orig_filenames	orig_mime_types	resp_fuids	resp_filenames	resp_mime_types
+#types	time	string	addr	port	addr	port	count	string	string	string	string	string	string	string	count	count	count	string	count	string	set[enum]	string	string	set[string]	vector[string]	vector[string]	vector[string]	vector[string]	vector[string]	vector[string]
+1470574779.148133	CHhAvVGS1DHFjwGM9	10.0.2.15	49762	93.184.216.34	80	1	GET	www.example.net	/	-	1.1	curl/7.50.1	-	0	1270	200	OK	-	-	(empty)	-	-	-	-	-	-	F57zxQB092IOFcB15	-	text/html
+#close	2019-03-20-02-08-50

testing/btest/core/nflog.bro

@@ -0,0 +1,4 @@
+# @TEST-EXEC: bro -C -r $TRACES/nflog-http.pcap %INPUT
+# @TEST-EXEC: btest-diff http.log
+
+@load base/protocols/http