Merge branch 'dns-original-query-case' of https://github.com/rvictory/zeek

Changes during merge
- Changed the policy script to use an event handler that behaves
  for like the base script: &priority=5, msg$opcode != early-out,
  no record field existence checks
- Also extended dns_query_reply event with original_query param
- Removed ExtractName overload, and just use default param

* 'dns-original-query-case' of https://github.com/rvictory/zeek:
  Fixed some places where tabs became spaces
  Stricter checking if we have a dns field on the connection being processed
  Modified the DNS protocol analyzer to add a new parameter to the dns_request event which includes the DNS query in its original case. Added a policy script that will add the original_case to the dns.log file as well. Created new btests to test both.

CHANGES

@@ -1,4 +1,16 @@
 
+3.2.0-dev.804 | 2020-06-25 23:58:59 -0700
+
+  * Extend dns_request, dns_reject, and dns_query_reply events with original_query param (Ryan Victory)
+
+    Where the "query" parameter is downcased, the "original_query" is
+    passed unmodified to these events.
+
+    Added a policy/protocols/dns/log-original-query-case.zeek script that adds
+    the original_query to the dns.log file as well.
+
+  * Convert to using permissions to check for access to cirrus variables in benchmark script (Tim Wojtulewicz, Corelight)
+
 3.2.0-dev.798 | 2020-06-19 13:25:02 -0700
 
   * Remove last_access_time from TableEntryVal. (Johanna Amann, Corelight)

VERSION

@@ -1 +1 @@
-3.2.0-dev.798
+3.2.0-dev.804

doc

@@ -1 +1 @@
-Subproject commit aea3b3a8f49699e7c140b612e87f36e8c37d6fa2
+Subproject commit 04d72500bfed53059aa06d26a826a501b87ae29c

scripts/policy/protocols/dns/log-original-query-case.zeek

@@ -0,0 +1,22 @@
+##! This script adds the query with its original letter casing
+##! to the DNS log.
+
+@load base/protocols/dns/main
+
+module DNS;
+
+export {
+	redef record Info += {
+		## Query with original letter casing
+		original_query: string &log &optional;
+	};
+}
+
+event dns_request(c: connection, msg: dns_msg, query: string, qtype: count, qclass: count, original_query: string) &priority=5
+	{
+	if ( msg$opcode != 0 )
+		# Currently only standard queries are tracked.
+		return;
+
+	c$dns$original_query = original_query;
+	}

scripts/test-all-policy.zeek

@@ -73,6 +73,7 @@
 @load protocols/dhcp/sub-opts.zeek
 @load protocols/dns/auth-addl.zeek
 @load protocols/dns/detect-external-names.zeek
+@load protocols/dns/log-original-query-case.zeek
 @load protocols/ftp/detect-bruteforcing.zeek
 @load protocols/ftp/detect.zeek
 @load protocols/ftp/software.zeek

src/analyzer/protocol/dns/DNS.cc

@@ -171,7 +171,7 @@ bool DNS_Interpreter::ParseQuestion(DNS_MsgInfo* msg,
 	u_char name[513];
 	int name_len = sizeof(name) - 1;
 
-	u_char* name_end = ExtractName(data, len, name, name_len, msg_start);
+	u_char* name_end = ExtractName(data, len, name, name_len, msg_start, false);
 	if ( ! name_end )
 		return false;
 
@@ -196,9 +196,16 @@ bool DNS_Interpreter::ParseQuestion(DNS_MsgInfo* msg,
 
 	if ( dns_event && ! msg->skip_event )
 		{
-		BroString* question_name =
+		BroString* original_name = new BroString(name, name_end - name, true);
-			new BroString(name, name_end - name, true);
+
-		SendReplyOrRejectEvent(msg, dns_event, data, len, question_name);
+		// Downcase the Name to normalize it
+		for ( u_char* np = name; np < name_end; ++np )
+			if ( isupper(*np) )
+				*np = tolower(*np);
+
+		BroString* question_name = new BroString(name, name_end - name, true);
+
+		SendReplyOrRejectEvent(msg, dns_event, data, len, question_name, original_name);
 		}
 	else
 		{
@@ -355,7 +362,7 @@ bool DNS_Interpreter::ParseAnswer(DNS_MsgInfo* msg,
 
 u_char* DNS_Interpreter::ExtractName(const u_char*& data, int& len,
 					u_char* name, int name_len,
-					const u_char* msg_start)
+					const u_char* msg_start, bool downcase)
 	{
 	u_char* name_start = name;
 
@@ -375,6 +382,7 @@ u_char* DNS_Interpreter::ExtractName(const u_char*& data, int& len,
 		}
 
 	// Convert labels to lower case for consistency.
+	if ( downcase )
 		for ( u_char* np = name_start; np < name; ++np )
 			if ( isupper(*np) )
 				*np = tolower(*np);
@@ -1388,7 +1396,8 @@ bool DNS_Interpreter::ParseRR_CAA(DNS_MsgInfo* msg,
 void DNS_Interpreter::SendReplyOrRejectEvent(DNS_MsgInfo* msg,
 						EventHandlerPtr event,
 						const u_char*& data, int& len,
-						BroString* question_name)
+						BroString* question_name,
+						BroString* original_name)
 	{
 	RR_Type qtype = RR_Type(ExtractShort(data, len));
 	int qclass = ExtractShort(data, len);
@@ -1400,7 +1409,8 @@ void DNS_Interpreter::SendReplyOrRejectEvent(DNS_MsgInfo* msg,
 		msg->BuildHdrVal(),
 		make_intrusive<StringVal>(question_name),
 		val_mgr->Count(qtype),
-		val_mgr->Count(qclass)
+		val_mgr->Count(qclass),
+		make_intrusive<StringVal>(original_name)
 	);
 	}
 

src/analyzer/protocol/dns/DNS.h

@@ -242,7 +242,7 @@ protected:
 
 	u_char* ExtractName(const u_char*& data, int& len,
 				u_char* label, int label_len,
-				const u_char* msg_start);
+				const u_char* msg_start, bool downcase = true);
 	bool ExtractLabel(const u_char*& data, int& len,
 			 u_char*& label, int& label_len,
 			 const u_char* msg_start);
@@ -308,7 +308,7 @@ protected:
 				const u_char* msg_start);
 	void SendReplyOrRejectEvent(DNS_MsgInfo* msg, EventHandlerPtr event,
 					const u_char*& data, int& len,
-					BroString* question_name);
+					BroString* question_name, BroString* original_name);
 
 	analyzer::Analyzer* analyzer;
 	bool first_message;

src/analyzer/protocol/dns/events.bif

@@ -34,12 +34,14 @@ event dns_message%(c: connection, is_orig: bool, msg: dns_msg, len: count%);
 ##
 ## msg: The parsed DNS message header.
 ##
-## query: The queried name.
+## query: The queried name (normalized to all lowercase).
 ##
 ## qtype: The queried resource record type.
 ##
 ## qclass: The queried resource record class.
 ##
+## original_query: The queried name, with the original case kept intact
+##
 ## .. zeek:see:: dns_AAAA_reply dns_A_reply dns_CNAME_reply dns_EDNS_addl
 ##    dns_HINFO_reply dns_MX_reply dns_NS_reply dns_PTR_reply dns_SOA_reply
 ##    dns_SRV_reply dns_TSIG_addl dns_TXT_reply dns_SPF_reply dns_WKS_reply dns_end
@@ -47,6 +49,7 @@ event dns_message%(c: connection, is_orig: bool, msg: dns_msg, len: count%);
 ##    dns_mapping_unverified dns_mapping_valid dns_message dns_query_reply
 ##    dns_rejected dns_max_queries dns_session_timeout dns_skip_addl
 ##    dns_skip_all_addl dns_skip_all_auth dns_skip_auth
+event dns_request%(c: connection, msg: dns_msg, query: string, qtype: count, qclass: count, original_query: string%);
 event dns_request%(c: connection, msg: dns_msg, query: string, qtype: count, qclass: count%);
 
 ## Generated for DNS replies that reject a query. This event is raised if a DNS
@@ -63,12 +66,14 @@ event dns_request%(c: connection, msg: dns_msg, query: string, qtype: count, qcl
 ##
 ## msg: The parsed DNS message header.
 ##
-## query: The queried name.
+## query: The queried name (normalized to all lowercase).
 ##
 ## qtype: The queried resource record type.
 ##
 ## qclass: The queried resource record class.
 ##
+## original_query: The queried name, with the original case kept intact
+##
 ## .. zeek:see:: dns_AAAA_reply dns_A_reply dns_CNAME_reply dns_EDNS_addl
 ##    dns_HINFO_reply dns_MX_reply dns_NS_reply dns_PTR_reply dns_SOA_reply
 ##    dns_SRV_reply dns_TSIG_addl dns_TXT_reply dns_SPF_reply dns_WKS_reply dns_end
@@ -76,6 +81,7 @@ event dns_request%(c: connection, msg: dns_msg, query: string, qtype: count, qcl
 ##    dns_mapping_unverified dns_mapping_valid dns_message dns_query_reply
 ##    dns_request dns_max_queries dns_session_timeout dns_skip_addl
 ##    dns_skip_all_addl dns_skip_all_auth dns_skip_auth
+event dns_rejected%(c: connection, msg: dns_msg, query: string, qtype: count, qclass: count, original_query: string%);
 event dns_rejected%(c: connection, msg: dns_msg, query: string, qtype: count, qclass: count%);
 
 ## Generated for each entry in the Question section of a DNS reply.
@@ -95,6 +101,8 @@ event dns_rejected%(c: connection, msg: dns_msg, query: string, qtype: count, qc
 ##
 ## qclass: The queried resource record class.
 ##
+## original_query: The queried name, with the original case kept intact
+##
 ## .. zeek:see:: dns_AAAA_reply dns_A_reply dns_CNAME_reply dns_EDNS_addl
 ##    dns_HINFO_reply dns_MX_reply dns_NS_reply dns_PTR_reply dns_SOA_reply
 ##    dns_SRV_reply dns_TSIG_addl dns_TXT_reply dns_SPF_reply dns_WKS_reply dns_end
@@ -102,6 +110,8 @@ event dns_rejected%(c: connection, msg: dns_msg, query: string, qtype: count, qc
 ##    dns_mapping_unverified dns_mapping_valid dns_message dns_rejected
 ##    dns_request dns_max_queries dns_session_timeout dns_skip_addl
 ##    dns_skip_all_addl dns_skip_all_auth dns_skip_auth
+event dns_query_reply%(c: connection, msg: dns_msg, query: string,
+			qtype: count, qclass: count, original_query: string%);
 event dns_query_reply%(c: connection, msg: dns_msg, query: string,
 			qtype: count, qclass: count%);
 

testing/btest/Baseline/scripts.policy.misc.dump-events/all-events.log

@@ -19,6 +19,7 @@
                   [2] query: string      = mail.patriots.in
                   [3] qtype: count       = 1
                   [4] qclass: count      = 1
+                  [5] original_query: string = mail.patriots.in
 
 1254722767.492060 protocol_confirmation
                   [0] c: connection      = [id=[orig_h=10.10.1.4, orig_p=56166/udp, resp_h=10.10.1.1, resp_p=53/udp], orig=[size=34, state=1, num_pkts=0, num_bytes_ip=0, flow_label=0, l2_addr=00:e0:1c:3c:17:c2], resp=[size=0, state=0, num_pkts=0, num_bytes_ip=0, flow_label=0, l2_addr=00:1f:33:d9:81:60], start_time=1254722767.49206, duration=0 secs, service={\x0a\x0a}, history=D, uid=CHhAvVGS1DHFjwGM9, tunnel=<uninitialized>, vlan=<uninitialized>, inner_vlan=<uninitialized>, successful=F, dpd=<uninitialized>, dpd_state=<uninitialized>, conn=<uninitialized>, extract_orig=F, extract_resp=F, thresholds=<uninitialized>, dce_rpc=<uninitialized>, dce_rpc_state=<uninitialized>, dce_rpc_backing=<uninitialized>, dhcp=<uninitialized>, dnp3=<uninitialized>, dns=[ts=1254722767.49206, uid=CHhAvVGS1DHFjwGM9, id=[orig_h=10.10.1.4, orig_p=56166/udp, resp_h=10.10.1.1, resp_p=53/udp], proto=udp, trans_id=31062, rtt=<uninitialized>, query=mail.patriots.in, qclass=1, qclass_name=C_INTERNET, qtype=1, qtype_name=A, rcode=<uninitialized>, rcode_name=<uninitialized>, AA=F, TC=F, RD=T, RA=F, Z=0, answers=<uninitialized>, TTLs=<uninitialized>, rejected=F, total_answers=<uninitialized>, total_replies=<uninitialized>, saw_query=F, saw_reply=F], dns_state=[pending_query=[ts=1254722767.49206, uid=CHhAvVGS1DHFjwGM9, id=[orig_h=10.10.1.4, orig_p=56166/udp, resp_h=10.10.1.1, resp_p=53/udp], proto=udp, trans_id=31062, rtt=<uninitialized>, query=mail.patriots.in, qclass=1, qclass_name=C_INTERNET, qtype=1, qtype_name=A, rcode=<uninitialized>, rcode_name=<uninitialized>, AA=F, TC=F, RD=T, RA=F, Z=0, answers=<uninitialized>, TTLs=<uninitialized>, rejected=F, total_answers=<uninitialized>, total_replies=<uninitialized>, saw_query=F, saw_reply=F], pending_queries=<uninitialized>, pending_replies=<uninitialized>], ftp=<uninitialized>, ftp_data_reuse=F, ssl=<uninitialized>, http=<uninitialized>, http_state=<uninitialized>, irc=<uninitialized>, krb=<uninitialized>, modbus=<uninitialized>, mysql=<uninitialized>, ntlm=<uninitialized>, ntp=<uninitialized>, radius=<uninitialized>, rdp=<uninitialized>, rfb=<uninitialized>, sip=<uninitialized>, sip_state=<uninitialized>, snmp=<uninitialized>, smb_state=<uninitialized>, smtp=<uninitialized>, smtp_state=<uninitialized>, socks=<uninitialized>, ssh=<uninitialized>, syslog=<uninitialized>]

testing/btest/Baseline/scripts.policy.protocols.dns.original_case/dns.log

@@ -0,0 +1,10 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	dns
+#open	2020-06-17-14-47-26
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	trans_id	rtt	query	qclass	qclass_name	qtype	qtype_name	rcode	rcode_name	AA	TC	RD	RA	Z	answers	TTLs	rejected	original_query
+#types	time	string	addr	port	addr	port	enum	count	interval	string	count	string	count	string	count	string	bool	bool	bool	bool	count	vector[string]	vector[interval]	bool	string
+1592402712.248901	CHhAvVGS1DHFjwGM9	192.168.3.138	63374	192.168.3.1	53	udp	20877	-	us.v27.distributed.net	1	C_INTERNET	1	A	-	-	F	F	T	F	2	-	-	F	Us.V27.DiStRiBuTeD.NET
+#close	2020-06-17-14-47-26

testing/btest/scripts/policy/protocols/dns/original_case.zeek

@@ -0,0 +1,3 @@
+# @TEST-EXEC: zeek -r $TRACES/dns_original_case.pcap %INPUT
+# @TEST-EXEC: btest-diff dns.log
+@load protocols/dns/log-original-query-case

testing/external/commit-hash.zeek-testing

@@ -1 +1 @@
-807dce8fd94d59e571994c033e333691f7ef27ba
+9d92ec99cadd04e95365dc2c3b507b7011db255a

testing/external/commit-hash.zeek-testing-private

@@ -1 +1 @@
-fa5c4dc4ea3481c7b273f0cfcc77497c0d32aa7c
+d6cd639023cfe26c4e2cf14a59e78599b22ed4d0