Analyzer failure logging: tweaks and test fixes

The main part of this commit are changes in tests. A lot of the tests
that previously relied on analyzer.log or dpd.log now use the new
analyzer-failed.log.

I verified all the changes and, as far as I can tell, everything
behaves as it should. This includes the external test baselines.

This change also enables logging of file and packet analyzer to
analyzer_failed.log and fixes some small behavior issues.

The analyzer_failed event is no longer raised when the removal of an
analyzer is vetoed.

If an analyzer is no longer active when an analyzer violation is raised,
currently the analyzer_failed event is raised. This can, e.g., happen
when an analyzer error happens at the very end of the connection. This
makes the behavior more similar to what happened in the past, and also
intuitively seems to make sense.

A bug introduced in the failed service logging was fixed.

scripts/policy/frameworks/analyzer/analyzer-debug-log.zeek

@@ -69,7 +69,7 @@ export {
 
 event zeek_init() &priority=5
 	{
-	Log::create_stream(LOG, [$columns=Info, $path="analyzer-debug", $policy=log_policy,
+	Log::create_stream(LOG, [$columns=Info, $path="analyzer_debug", $policy=log_policy,
 	                         $event_groups=set("Analyzer::DebugLogging")]);
 
 	local enable_handler = function(id: string, new_value: bool): bool {

scripts/policy/frameworks/analyzer/packet-segment-logging.zeek

@@ -0,0 +1,50 @@
+##! This script enables logging of packet segment data when a protocol
+##! parsing violation is encountered.  The amount of data from the
+##! packet logged is set by the :zeek:see:`Analyzer::Logging::packet_segment_size` variable.
+##! A caveat to logging packet data is that in some cases, the packet may
+##! not be the packet that actually caused the protocol violation.
+
+module Analyzer::Logging;
+
+export {
+	redef record connection += {
+		## A chunk of the payload that most likely resulted in a
+		## analyzer violation.
+		packet_segment: string &optional &log;
+	};
+
+	redef record Analyzer::Logging::Info += {
+		## A chunk of the payload that most likely resulted in the
+		## analyzer violation.
+		packet_segment: string &optional &log;
+	};
+
+	## Size of the packet segment to display in the DPD log.
+	option packet_segment_size: int = 255;
+}
+
+# stash the packet segment in the event causing the violation, so that it can be retrieved later.
+event analyzer_violation_info(atype: AllAnalyzers::Tag, info: AnalyzerViolationInfo) &priority=4
+	{
+	if ( ! is_protocol_analyzer(atype) && ! is_packet_analyzer(atype) )
+		return;
+
+	if ( ! info?$c || ! info?$aid )
+		return;
+
+	info$c$packet_segment = fmt("%s", get_current_packet()$data[:packet_segment_size]);
+	}
+
+hook Analyzer::Logging::log_policy(rec: Analyzer::Logging::Info, id: Log::ID, filter: Log::Filter)
+	{
+	if ( id != Analyzer::Logging::LOG )
+		return;
+
+	if ( ! rec?$id || ! connection_exists(rec$id) )
+		return;
+
+	local c = lookup_connection(rec$id);
+
+	if ( c?$packet_segment )
+		rec$packet_segment = c$packet_segment;
+	}

scripts/policy/frameworks/dpd/packet-segment-logging.zeek

@@ -1,9 +1,13 @@
+@deprecated("Please switch to frameworks/analyzer/packet-segment-logging, which logs to analyzer.log. Remove in 8.1")
+
 ##! This script enables logging of packet segment data when a protocol
 ##! parsing violation is encountered.  The amount of data from the
 ##! packet logged is set by the :zeek:see:`DPD::packet_segment_size` variable.
 ##! A caveat to logging packet data is that in some cases, the packet may
 ##! not be the packet that actually caused the protocol violation.
 
+@load frameworks/analyzer/dpd-log
+
 module DPD;
 
 export {

scripts/policy/protocols/conn/failed-service-logging.zeek

@@ -15,21 +15,19 @@ redef record Conn::Info += {
 	failed_service: set[string] &log &optional &ordered;
 };
 
-hook Analyzer::disabling_analyzer(c: connection, atype: AllAnalyzers::Tag, aid: count) &priority=-1000
+event analyzer_failed(ts: time, atype: AllAnalyzers::Tag, info: AnalyzerViolationInfo)
 	{
 	if ( ! is_protocol_analyzer(atype) && ! is_packet_analyzer(atype) )
 		return;
 
-	# Only add if previously confirmed
-	if ( Analyzer::name(atype) !in c$service && Analyzer::name(atype) !in c$service_violation )
-		return;
+	if ( ! info?$c )
+			return;
 
-	# Only log if dpd.zeek will disable
-	if ( atype in DPD::ignore_violations )
-		return;
+	local c = info$c;
 
-	local size = c$orig$size + c$resp$size;
-	if ( DPD::ignore_violations_after > 0 && size > DPD::ignore_violations_after )
+	# Only add if previously confirmed and not failed
+	local analyzer_name = Analyzer::name(atype);
+	if ( analyzer_name !in c$service || analyzer_name in c$service_violation )
 		return;
 
 	set_conn(c, F);