Merge branch 'topic/jgras/dpd-late-match' of https://github.com/J-Gras/zeek

* 'topic/jgras/dpd-late-match' of https://github.com/J-Gras/zeek:
  Improve dpd_late_match event generation.
  Improve logging of speculative service.
  Update test-all-policy script.
  Add speculative service script.
  Allow to handle late DPD matches.

CHANGES

@@ -1,4 +1,17 @@
 
+3.1.0-dev.124 | 2019-09-17 11:16:47 -0700
+
+  * Add speculative service script. (Jan Grashoefer)
+
+    The speculative service script handles dpd_late_match events to extend
+    conn.log with infos about potential protocol identifications.
+
+  * Allow to handle late DPD matches. (Jan Grashoefer)
+
+    If "dpd_match_only_beginning" is disabled, matches of protocol signatures
+    can be handeld using protocol_late_match. To prevent further matching in
+    this case, the new option "dpd_late_match_stop" may be activated.
+
 3.1.0-dev.118 | 2019-09-17 17:21:58 +0000
 
   * GH-566: Fix cases where ssh_encrypted_packet event wasn't raised.

NEWS

@@ -9,6 +9,16 @@ Zeek 3.1.0
 New Functionality
 -----------------
 
+- Add a new option, ``dpd_late_match_stop``, which can be used in conjuction
+  with the option ``dpd_match_only_beginning`` and the new event
+  ``protocol_late_match`` to help annotate the conn.log with a field
+  to speculate on the protocol/service in cases where the DPD buffer
+  was already exhausted and can't analyze the full connection anymore,
+  but where there was still a late signature match.  A new script,
+  ``policy/protocols/conn/speculative-service.zeek``, was added as an
+  example of how to perform this tuning and add a "speculative_service"
+  field to conn.log, but it's not loaded by default.
+
 Changed Functionality
 ---------------------
 

VERSION

@@ -1 +1 @@
-3.1.0-dev.118
+3.1.0-dev.124

doc

@@ -1 +1 @@
-Subproject commit a3dd09cd5acfe6aa848a84eae5fdb18d64d590e1
+Subproject commit 218907b6500d627904f6ba0767f7ba362bf825f7

scripts/base/init-bare.zeek

@@ -4684,6 +4684,18 @@ const dpd_buffer_size = 1024 &redef;
 ##    only signatures used for dynamic protocol detection.
 const dpd_match_only_beginning = T &redef;
 
+## If true, stops signature matching after a late match. A late match may occur
+## in case the DPD buffer is exhausted but a protocol signature matched. To
+## allow late matching, :zeek:see:`dpd_match_only_beginning` must be disabled.
+##
+## .. zeek:see:: dpd_reassemble_first_packets dpd_buffer_size
+##    dpd_match_only_beginning
+##
+## .. note:: Despite the name, this option stops *all* signature matching, not
+##    only signatures used for dynamic protocol detection but is triggered by
+##    DPD signatures only.
+const dpd_late_match_stop = F &redef;
+
 ## If true, don't consider any ports for deciding which protocol analyzer to
 ## use.
 ##

scripts/policy/protocols/conn/speculative-service.zeek

@@ -0,0 +1,36 @@
+##! This script adds information about matched DPD signatures to the connection
+##! log.
+
+@load base/protocols/conn
+
+module Conn;
+
+redef record Info += {
+	## Protocol that was determined by a matching signature after the beginning
+	## of a connection. In this situation no analyzer can be attached and hence
+	## the data cannot be analyzed nor the protocol can be confirmed.
+	speculative_service: string &log &optional;
+};
+
+redef record connection += {
+	speculative_service: set[string] &default=string_set();
+};
+
+redef dpd_match_only_beginning = F;
+redef dpd_late_match_stop = T;
+
+event protocol_late_match(c: connection, atype: Analyzer::Tag)
+	{
+	local analyzer = Analyzer::name(atype);
+	add c$speculative_service[analyzer];
+	}
+
+event connection_state_remove(c: connection)
+	{
+	local sp_service = "";
+	for ( s in c$speculative_service )
+		sp_service = sp_service == "" ? s : cat(sp_service, ",", s);
+
+	if ( sp_service != "" )
+		c$conn$speculative_service = to_lower(sp_service);
+	}

scripts/test-all-policy.zeek

@@ -67,6 +67,7 @@
 @load protocols/conn/mac-logging.zeek
 @load protocols/conn/vlan-logging.zeek
 @load protocols/conn/weirds.zeek
+#@load protocols/conn/speculative-service.zeek
 @load protocols/dhcp/msg-orig.zeek
 @load protocols/dhcp/software.zeek
 @load protocols/dhcp/sub-opts.zeek

scripts/zeekygen/__load__.zeek

@@ -6,6 +6,7 @@
 @load frameworks/control/controller.zeek
 @load frameworks/files/extract-all-files.zeek
 @load policy/misc/dump-events.zeek
+@load policy/protocols/conn/speculative-service.zeek
 
 @load ./example.zeek
 

src/NetVar.cc

@@ -164,6 +164,7 @@ RecordType* irc_join_info;
 int dpd_reassemble_first_packets;
 int dpd_buffer_size;
 int dpd_match_only_beginning;
+int dpd_late_match_stop;
 int dpd_ignore_ports;
 
 TableVal* likely_server_ports;
@@ -406,6 +407,7 @@ void init_net_var()
 		opt_internal_int("dpd_reassemble_first_packets");
 	dpd_buffer_size = opt_internal_int("dpd_buffer_size");
 	dpd_match_only_beginning = opt_internal_int("dpd_match_only_beginning");
+	dpd_late_match_stop = opt_internal_int("dpd_late_match_stop");
 	dpd_ignore_ports = opt_internal_int("dpd_ignore_ports");
 
 	likely_server_ports = internal_val("likely_server_ports")->AsTableVal();

src/NetVar.h

@@ -165,6 +165,7 @@ extern RecordType* irc_join_info;
 extern int dpd_reassemble_first_packets;
 extern int dpd_buffer_size;
 extern int dpd_match_only_beginning;
+extern int dpd_late_match_stop;
 extern int dpd_ignore_ports;
 
 extern TableVal* likely_server_ports;

src/analyzer/protocol/pia/PIA.cc

@@ -1,5 +1,6 @@
 #include "PIA.h"
 #include "RuleMatcher.h"
+#include "Event.h"
 #include "analyzer/protocol/tcp/TCP_Flags.h"
 #include "analyzer/protocol/tcp/TCP_Reassembler.h"
 
@@ -147,6 +148,20 @@ void PIA_UDP::ActivateAnalyzer(analyzer::Tag tag, const Rule* rule)
 		DBG_LOG(DBG_ANALYZER, "analyzer found but buffer already exceeded");
 		// FIXME: This is where to check whether an analyzer
 		// supports partial connections once we get such.
+
+		if ( protocol_late_match )
+			{
+			// Queue late match event
+			EnumVal *tval = tag ? tag.AsEnumVal() : GetAnalyzerTag().AsEnumVal();
+			Ref(tval);
+
+			mgr.QueueEventFast(protocol_late_match, {
+			    BuildConnVal(),
+			    tval,
+			});
+			}
+
+		pkt_buffer.state = dpd_late_match_stop ? SKIPPING : MATCHING_ONLY;
 		return;
 		}
 
@@ -282,6 +297,20 @@ void PIA_TCP::ActivateAnalyzer(analyzer::Tag tag, const Rule* rule)
 		DBG_LOG(DBG_ANALYZER, "analyzer found but buffer already exceeded");
 		// FIXME: This is where to check whether an analyzer supports
 		// partial connections once we get such.
+
+		if ( protocol_late_match )
+			{
+			// Queue late match event
+			EnumVal *tval = tag ? tag.AsEnumVal() : GetAnalyzerTag().AsEnumVal();
+			Ref(tval);
+
+			mgr.QueueEventFast(protocol_late_match, {
+			    BuildConnVal(),
+			    tval
+			});
+			}
+
+		stream_buffer.state = dpd_late_match_stop ? SKIPPING : MATCHING_ONLY;
 		return;
 		}
 

src/event.bif

@@ -369,6 +369,20 @@ event content_gap%(c: connection, is_orig: bool, seq: count, length: count%);
 ##    there (and thus in ``conn.log``).
 event protocol_confirmation%(c: connection, atype: Analyzer::Tag, aid: count%);
 
+## Generated if a DPD signature matched but the DPD buffer is already exhausted
+## and thus the analyzer could not be attached. While this does not confirm
+## that a protocol is actually used, it allows to retain that information.
+##
+## c: The connection.
+##
+## atype: The type of the analyzer confirming that its protocol is in
+##        use. The value is one of the ``Analyzer::ANALYZER_*`` constants. For example,
+##        ``Analyzer::ANALYZER_HTTP`` means the HTTP analyzer determined that it's indeed
+##        parsing an HTTP connection.
+##
+## .. bro:see:: dpd_buffer_size
+event protocol_late_match%(c: connection, atype: Analyzer::Tag%);
+
 ## Generated when a protocol analyzer determines that a connection it is parsing
 ## is not conforming to the protocol it expects. Zeek's dynamic protocol
 ## detection heuristically activates analyzers as soon as it believes a

testing/btest/Baseline/scripts.policy.protocols.conn.speculative-service/conn-post-large.log

@@ -0,0 +1,11 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	conn
+#open	2019-08-30-13-12-19
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	service	duration	orig_bytes	resp_bytes	conn_state	local_orig	local_resp	missed_bytes	history	orig_pkts	orig_ip_bytes	resp_pkts	resp_ip_bytes	tunnel_parents	speculative_service
+#types	time	string	addr	port	addr	port	enum	string	interval	count	count	string	bool	bool	count	string	count	count	count	count	set[string]	string
+1567010592.624680	CHhAvVGS1DHFjwGM9	127.0.0.1	37526	127.0.0.1	80	tcp	http	0.008395	61907	60478	SF	-	-	0	ShADadfF	10	62435	9	60954	-	http
+1567010639.143657	ClEkJM2Vm5giqnMf4h	127.0.0.1	60644	127.0.0.1	5000	tcp	-	0.015853	61917	60478	SF	-	-	0	ShADadfF	10	62445	9	60954	-	http
+#close	2019-08-30-13-12-19

testing/btest/Baseline/scripts.policy.protocols.conn.speculative-service/conn-wiki.log

@@ -0,0 +1,43 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	conn
+#open	2019-09-17-17-56-40
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	service	duration	orig_bytes	resp_bytes	conn_state	local_orig	local_resp	missed_bytes	history	orig_pkts	orig_ip_bytes	resp_pkts	resp_ip_bytes	tunnel_parents	speculative_service
+#types	time	string	addr	port	addr	port	enum	string	interval	count	count	string	bool	bool	count	string	count	count	count	count	set[string]	string
+1300475167.096535	CHhAvVGS1DHFjwGM9	141.142.220.202	5353	224.0.0.251	5353	udp	dns	-	-	-	S0	-	-	0	D	1	73	0	0	-	-
+1300475167.097012	ClEkJM2Vm5giqnMf4h	fe80::217:f2ff:fed7:cf65	5353	ff02::fb	5353	udp	dns	-	-	-	S0	-	-	0	D	1	199	0	0	-	-
+1300475167.099816	C4J4Th3PJpwUYZZ6gc	141.142.220.50	5353	224.0.0.251	5353	udp	dns	-	-	-	S0	-	-	0	D	1	179	0	0	-	-
+1300475168.853899	CmES5u32sYpV7JYN	141.142.220.118	43927	141.142.2.2	53	udp	dns	0.000435	38	89	SF	-	-	0	Dd	1	66	1	117	-	-
+1300475168.854378	CP5puj4I8PtEU4qzYg	141.142.220.118	37676	141.142.2.2	53	udp	dns	0.000420	52	99	SF	-	-	0	Dd	1	80	1	127	-	-
+1300475168.854837	C37jN32gN3y3AZzyf6	141.142.220.118	40526	141.142.2.2	53	udp	dns	0.000392	38	183	SF	-	-	0	Dd	1	66	1	211	-	-
+1300475168.857956	C0LAHyvtKSQHyJxIl	141.142.220.118	32902	141.142.2.2	53	udp	dns	0.000317	38	89	SF	-	-	0	Dd	1	66	1	117	-	-
+1300475168.858306	CFLRIC3zaTU1loLGxh	141.142.220.118	59816	141.142.2.2	53	udp	dns	0.000343	52	99	SF	-	-	0	Dd	1	80	1	127	-	-
+1300475168.858713	C9rXSW3KSpTYvPrlI1	141.142.220.118	59714	141.142.2.2	53	udp	dns	0.000375	38	183	SF	-	-	0	Dd	1	66	1	211	-	-
+1300475168.891644	C9mvWx3ezztgzcexV7	141.142.220.118	58206	141.142.2.2	53	udp	dns	0.000339	38	89	SF	-	-	0	Dd	1	66	1	117	-	-
+1300475168.892037	CNnMIj2QSd84NKf7U3	141.142.220.118	38911	141.142.2.2	53	udp	dns	0.000335	52	99	SF	-	-	0	Dd	1	80	1	127	-	-
+1300475168.892414	C7fIlMZDuRiqjpYbb	141.142.220.118	59746	141.142.2.2	53	udp	dns	0.000421	38	183	SF	-	-	0	Dd	1	66	1	211	-	-
+1300475168.893988	CpmdRlaUoJLN3uIRa	141.142.220.118	45000	141.142.2.2	53	udp	dns	0.000384	38	89	SF	-	-	0	Dd	1	66	1	117	-	-
+1300475168.894422	C1Xkzz2MaGtLrc1Tla	141.142.220.118	48479	141.142.2.2	53	udp	dns	0.000317	52	99	SF	-	-	0	Dd	1	80	1	127	-	-
+1300475168.894787	CqlVyW1YwZ15RhTBc4	141.142.220.118	48128	141.142.2.2	53	udp	dns	0.000423	38	183	SF	-	-	0	Dd	1	66	1	211	-	-
+1300475168.901749	CBA8792iHmnhPLksKa	141.142.220.118	56056	141.142.2.2	53	udp	dns	0.000402	36	131	SF	-	-	0	Dd	1	64	1	159	-	-
+1300475168.902195	CGLPPc35OzDQij1XX8	141.142.220.118	55092	141.142.2.2	53	udp	dns	0.000374	36	198	SF	-	-	0	Dd	1	64	1	226	-	-
+1300475169.899438	Cipfzj1BEnhejw8cGf	141.142.220.44	5353	224.0.0.251	5353	udp	dns	-	-	-	S0	-	-	0	D	1	85	0	0	-	-
+1300475170.862384	CV5WJ42jPYbNW9JNWf	141.142.220.226	137	141.142.220.255	137	udp	dns	2.613017	350	0	S0	-	-	0	D	7	546	0	0	-	-
+1300475171.675372	CPhDKt12KQPUVbQz06	fe80::3074:17d5:2052:c324	65373	ff02::1:3	5355	udp	dns	0.100096	66	0	S0	-	-	0	D	2	162	0	0	-	-
+1300475171.677081	CAnFrb2Cvxr5T7quOc	141.142.220.226	55131	224.0.0.252	5355	udp	dns	0.100021	66	0	S0	-	-	0	D	2	122	0	0	-	-
+1300475173.116749	C8rquZ3DjgNW06JGLl	fe80::3074:17d5:2052:c324	54213	ff02::1:3	5355	udp	dns	0.099801	66	0	S0	-	-	0	D	2	162	0	0	-	-
+1300475173.117362	CzrZOtXqhwwndQva3	141.142.220.226	55671	224.0.0.252	5355	udp	dns	0.099849	66	0	S0	-	-	0	D	2	122	0	0	-	-
+1300475173.153679	CaGCc13FffXe6RkQl9	141.142.220.238	56641	141.142.220.255	137	udp	dns	-	-	-	S0	-	-	0	D	1	78	0	0	-	-
+1300475168.652003	CtPZjS20MLrsMUOJi2	141.142.220.118	35634	208.80.152.2	80	tcp	-	0.061329	463	350	OTH	-	-	0	DdA	2	567	1	402	-	-
+1300475168.902635	CiyBAq1bBLNaTiTAc	141.142.220.118	35642	208.80.152.2	80	tcp	http	0.120041	534	412	S1	-	-	0	ShADad	4	750	3	576	-	-
+1300475168.855305	C3eiCBGOLw3VtHfOj	141.142.220.118	49996	208.80.152.3	80	tcp	http	0.218501	1171	733	S1	-	-	0	ShADad	6	1491	4	949	-	-
+1300475168.855330	CwjjYJ2WqgTbAqiHl6	141.142.220.118	49997	208.80.152.3	80	tcp	http	0.219720	1125	734	S1	-	-	0	ShADad	6	1445	4	950	-	-
+1300475168.859163	Ck51lg1bScffFj34Ri	141.142.220.118	49998	208.80.152.3	80	tcp	http	0.215893	1130	734	S1	-	-	0	ShADad	6	1450	4	950	-	-
+1300475168.892913	CykQaM33ztNt0csB9a	141.142.220.118	49999	208.80.152.3	80	tcp	http	0.220961	1137	733	S1	-	-	0	ShADad	6	1457	4	949	-	-
+1300475168.892936	CtxTCR2Yer0FR1tIBg	141.142.220.118	50000	208.80.152.3	80	tcp	http	0.229603	1148	734	S1	-	-	0	ShADad	6	1468	4	950	-	-
+1300475168.895267	CLNN1k2QMum1aexUK7	141.142.220.118	50001	208.80.152.3	80	tcp	http	0.227284	1178	734	S1	-	-	0	ShADad	6	1498	4	950	-	-
+1300475168.724007	CUM0KZ3MLUfNB0cl11	141.142.220.118	48649	208.80.152.118	80	tcp	http	0.119905	525	232	S1	-	-	0	ShADad	4	741	3	396	-	-
+1300475169.780331	CFSwNi4CNGxcuffo49	141.142.220.235	6705	173.192.163.128	80	tcp	-	-	-	-	OTH	-	-	0	^h	0	0	1	48	-	-
+#close	2019-09-17-17-56-40

testing/btest/scripts/policy/protocols/conn/speculative-service.zeek

@@ -0,0 +1,11 @@
+# A basic test of the speculative service detection
+
+# @TEST-EXEC: zeek -C -r $TRACES/http/http-post-large.pcap %INPUT
+# @TEST-EXEC: mv conn.log conn-post-large.log
+# @TEST-EXEC: btest-diff conn-post-large.log
+
+# @TEST-EXEC: zeek -C -r $TRACES/wikipedia.trace %INPUT
+# @TEST-EXEC: mv conn.log conn-wiki.log
+# @TEST-EXEC: btest-diff conn-wiki.log
+
+@load protocols/conn/speculative-service