Merge branch 'topic/jgras/dpd-late-match' of https://github.com/J-Gras/zeek

* 'topic/jgras/dpd-late-match' of https://github.com/J-Gras/zeek:
  Improve dpd_late_match event generation.
  Improve logging of speculative service.
  Update test-all-policy script.
  Add speculative service script.
  Allow to handle late DPD matches.

src/NetVar.cc

@@ -164,6 +164,7 @@ RecordType* irc_join_info;
 int dpd_reassemble_first_packets;
 int dpd_buffer_size;
 int dpd_match_only_beginning;
+int dpd_late_match_stop;
 int dpd_ignore_ports;
 
 TableVal* likely_server_ports;
@@ -406,6 +407,7 @@ void init_net_var()
 		opt_internal_int("dpd_reassemble_first_packets");
 	dpd_buffer_size = opt_internal_int("dpd_buffer_size");
 	dpd_match_only_beginning = opt_internal_int("dpd_match_only_beginning");
+	dpd_late_match_stop = opt_internal_int("dpd_late_match_stop");
 	dpd_ignore_ports = opt_internal_int("dpd_ignore_ports");
 
 	likely_server_ports = internal_val("likely_server_ports")->AsTableVal();

src/NetVar.h

@@ -165,6 +165,7 @@ extern RecordType* irc_join_info;
 extern int dpd_reassemble_first_packets;
 extern int dpd_buffer_size;
 extern int dpd_match_only_beginning;
+extern int dpd_late_match_stop;
 extern int dpd_ignore_ports;
 
 extern TableVal* likely_server_ports;

src/analyzer/protocol/pia/PIA.cc

@@ -1,5 +1,6 @@
 #include "PIA.h"
 #include "RuleMatcher.h"
+#include "Event.h"
 #include "analyzer/protocol/tcp/TCP_Flags.h"
 #include "analyzer/protocol/tcp/TCP_Reassembler.h"
 
@@ -147,6 +148,20 @@ void PIA_UDP::ActivateAnalyzer(analyzer::Tag tag, const Rule* rule)
 		DBG_LOG(DBG_ANALYZER, "analyzer found but buffer already exceeded");
 		// FIXME: This is where to check whether an analyzer
 		// supports partial connections once we get such.
+
+		if ( protocol_late_match )
+			{
+			// Queue late match event
+			EnumVal *tval = tag ? tag.AsEnumVal() : GetAnalyzerTag().AsEnumVal();
+			Ref(tval);
+
+			mgr.QueueEventFast(protocol_late_match, {
+			    BuildConnVal(),
+			    tval,
+			});
+			}
+
+		pkt_buffer.state = dpd_late_match_stop ? SKIPPING : MATCHING_ONLY;
 		return;
 		}
 
@@ -282,6 +297,20 @@ void PIA_TCP::ActivateAnalyzer(analyzer::Tag tag, const Rule* rule)
 		DBG_LOG(DBG_ANALYZER, "analyzer found but buffer already exceeded");
 		// FIXME: This is where to check whether an analyzer supports
 		// partial connections once we get such.
+
+		if ( protocol_late_match )
+			{
+			// Queue late match event
+			EnumVal *tval = tag ? tag.AsEnumVal() : GetAnalyzerTag().AsEnumVal();
+			Ref(tval);
+
+			mgr.QueueEventFast(protocol_late_match, {
+			    BuildConnVal(),
+			    tval
+			});
+			}
+
+		stream_buffer.state = dpd_late_match_stop ? SKIPPING : MATCHING_ONLY;
 		return;
 		}
 

src/event.bif

@@ -369,6 +369,20 @@ event content_gap%(c: connection, is_orig: bool, seq: count, length: count%);
 ##    there (and thus in ``conn.log``).
 event protocol_confirmation%(c: connection, atype: Analyzer::Tag, aid: count%);
 
+## Generated if a DPD signature matched but the DPD buffer is already exhausted
+## and thus the analyzer could not be attached. While this does not confirm
+## that a protocol is actually used, it allows to retain that information.
+##
+## c: The connection.
+##
+## atype: The type of the analyzer confirming that its protocol is in
+##        use. The value is one of the ``Analyzer::ANALYZER_*`` constants. For example,
+##        ``Analyzer::ANALYZER_HTTP`` means the HTTP analyzer determined that it's indeed
+##        parsing an HTTP connection.
+##
+## .. bro:see:: dpd_buffer_size
+event protocol_late_match%(c: connection, atype: Analyzer::Tag%);
+
 ## Generated when a protocol analyzer determines that a connection it is parsing
 ## is not conforming to the protocol it expects. Zeek's dynamic protocol
 ## detection heuristically activates analyzers as soon as it believes a