Merge remote-tracking branch 'jgras/topic/jgras/packet-analyzer-history-rework'

* jgras/topic/jgras/packet-analyzer-history-rework:
  Deprecate old unknown_protocol event signature
  Use raw pointer for packet analyzer history
  Update external baselines for analyzer history
  Update btest baselines for analyzer history
  Add packet analyzer history

CHANGES

@@ -1,3 +1,15 @@
+7.1.0-dev.147 | 2024-08-13 12:37:17 +0200
+
+  * Deprecate old unknown_protocol event signature (Jan Grashoefer, Corelight)
+
+  * Use raw pointer for packet analyzer history (Jan Grashoefer, Corelight)
+
+  * Update external baselines for analyzer history (Jan Grashoefer, Corelight)
+
+  * Update btest baselines for analyzer history (Jan Grashoefer, Corelight)
+
+  * Add packet analyzer history (Jan Grashoefer, Corelight)
+
 7.1.0-dev.141 | 2024-08-12 11:07:32 +0200
 
   * spicyz: Add back message about removed support for port / ports in evt (Arne Welzel, Corelight)

NEWS

@@ -25,6 +25,10 @@ New Functionality
   for connections where client and server negotiate to TLS through the extended
   request/response mechanism.
 
+* The ``unknown_protocols()`` event now includes the name of all packet
+  analyzer used for processing the packet when the event is raised. The
+  ``unknown_protocol.log`` file was extended to include this information.
+
 Changed Functionality
 ---------------------
 

VERSION

@@ -1 +1 @@
-7.1.0-dev.141
+7.1.0-dev.147

scripts/policy/misc/unknown-protocols.zeek

@@ -25,16 +25,23 @@ export {
 		## A certain number of bytes at the start of the unknown protocol's
 		## header.
 		first_bytes:  string   &log;
+
+		## The chain of packet analyzers that processed the packet up to this
+		## point. This includes the history of encapsulating packets in case
+		## of tunneling.
+		analyzer_history: vector of string &log;
 	};
 }
 
-event unknown_protocol(analyzer_name: string, protocol: count, first_bytes: string)
+event unknown_protocol(analyzer_name: string, protocol: count, first_bytes: string,
+	analyzer_history: string_vec)
 	{
 	local info : Info;
 	info$ts = network_time();
 	info$analyzer = analyzer_name;
 	info$protocol_id = fmt("0x%x", protocol);
 	info$first_bytes = bytestring_to_hexstr(first_bytes);
+	info$analyzer_history = analyzer_history;
 
 	Log::write(LOG, info);
 	}

src/event.bif

@@ -885,8 +885,13 @@ event Pcap::file_done%(path: string%);
 ##
 ## first_bytes: A certain number of bytes at the start of the unknown protocol's header.
 ##
+## analyzer_history: The chain of packet analyzers that processed the packet up to this
+##                   point. This includes the history of encapsulating packets in case
+##                   of tunneling.
+##
 ## .. zeek:see:: UnknownProtocol::first_bytes_count
-event unknown_protocol%(analyzer_name: string, protocol: count, first_bytes: string%);
+event unknown_protocol%(analyzer_name: string, protocol: count, first_bytes: string, analyzer_history: string_vec%);
+event unknown_protocol%(analyzer_name: string, protocol: count, first_bytes: string%) &deprecated="Remove in v8.1. Use the version that includes analyzer_history.";
 
 ## An event for handling packets that reached the end of processing without
 ## being marked as processed. Note that this event may lead to unpredictable

src/packet_analysis/Analyzer.cc

@@ -114,6 +114,7 @@ bool Analyzer::ForwardPacket(size_t len, const uint8_t* data, Packet* packet, ui
     DBG_LOG(DBG_PACKET_ANALYSIS, "Analysis in %s succeeded, next layer identifier is %#x.", GetAnalyzerName(),
             identifier);
 
+    packet_mgr->TrackAnalyzer(inner_analyzer.get());
     return inner_analyzer->AnalyzePacket(len, data, packet);
 }
 
@@ -129,6 +130,7 @@ bool Analyzer::ForwardPacket(size_t len, const uint8_t* data, Packet* packet) co
         return false;
     }
 
+    packet_mgr->TrackAnalyzer(inner_analyzer.get());
     return inner_analyzer->AnalyzePacket(len, data, packet);
 }
 

src/packet_analysis/Manager.cc

@@ -113,6 +113,7 @@ void Manager::ProcessPacket(Packet* packet) {
     }
 
     // Start packet analysis
+    analyzer_stack.clear();
     root_analyzer->ForwardPacket(packet->cap_len, packet->data, packet, packet->link_type);
 
     if ( ! packet->processed ) {
@@ -227,13 +228,24 @@ bool Manager::PermitUnknownProtocol(const std::string& analyzer, uint32_t protoc
     return false;
 }
 
+zeek::VectorValPtr Manager::BuildAnalyzerHistory() const {
+    auto history = zeek::make_intrusive<zeek::VectorVal>(zeek::id::string_vec);
+
+    for ( unsigned int i = 0; i < analyzer_stack.size(); i++ ) {
+        auto analyzer_name = analyzer_stack[i]->GetAnalyzerName();
+        history->Assign(i, make_intrusive<StringVal>(analyzer_name));
+    }
+
+    return history;
+}
+
 void Manager::ReportUnknownProtocol(const std::string& analyzer, uint32_t protocol, const uint8_t* data, size_t len) {
     if ( unknown_protocol ) {
         if ( PermitUnknownProtocol(analyzer, protocol) ) {
             int bytes_len = std::min(unknown_first_bytes_count, static_cast<uint64_t>(len));
 
             event_mgr.Enqueue(unknown_protocol, make_intrusive<StringVal>(analyzer), val_mgr->Count(protocol),
-                              make_intrusive<StringVal>(bytes_len, (const char*)data));
+                              make_intrusive<StringVal>(bytes_len, (const char*)data), BuildAnalyzerHistory());
         }
     }
 }

src/packet_analysis/Manager.h

@@ -176,6 +176,15 @@ public:
      */
     uint64_t GetUnprocessedCount() const { return total_not_processed; }
 
+    /**
+     * Tracks the given analyzer for the current packet's analyzer history.
+     * The packet analyzer history is implemented in form of a stack, which is reset on a
+     * call to ProcessPacket() but maintained throughout calls to ProcessInnerPacket().
+     *
+     * @param analyzer The analyzer to track.
+     */
+    void TrackAnalyzer(Analyzer* analyzer) { analyzer_stack.push_back(analyzer); }
+
 private:
     /**
      * Instantiates a new analyzer instance.
@@ -197,6 +206,14 @@ private:
      */
     AnalyzerPtr InstantiateAnalyzer(const std::string& name);
 
+    /**
+     * Generates a string vector that represents the analyzer history of the
+     * current packet based on the analyzers' tags.
+     *
+     * @return A vector of strings representing the packet analyzer history.
+     */
+    VectorValPtr BuildAnalyzerHistory() const;
+
     bool PermitUnknownProtocol(const std::string& analyzer, uint32_t protocol);
 
     std::map<std::string, AnalyzerPtr> analyzers;
@@ -216,6 +233,8 @@ private:
 
     uint64_t total_not_processed = 0;
     iosource::PktDumper* unprocessed_dumper = nullptr;
+
+    std::vector<Analyzer*> analyzer_stack;
 };
 
 } // namespace packet_analysis

testing/btest/Baseline/core.unknown-protocol-event/out

@@ -1,2 +1,2 @@
 ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
-ETHERNET, 35020, 02070400222d81db1004
+ETHERNET, 35020, 02070400222d81db1004, [ETHERNET]

testing/btest/Baseline/core.vntag/unknown_protocols.log

@@ -5,9 +5,9 @@
 #unset_field	-
 #path	unknown_protocols
 #open XXXX-XX-XX-XX-XX-XX
-#fields	ts	analyzer	protocol_id	first_bytes
-#types	time	string	string	string
-XXXXXXXXXX.XXXXXX	IP	0xfd	1b794b175fac06aba658
-XXXXXXXXXX.XXXXXX	IP	0xfd	9d6c1f9e20274bb66385
-XXXXXXXXXX.XXXXXX	IP	0xfd	06ffb64ded001f65f818
+#fields	ts	analyzer	protocol_id	first_bytes	analyzer_history
+#types	time	string	string	string	vector[string]
+XXXXXXXXXX.XXXXXX	IP	0xfd	1b794b175fac06aba658	ETHERNET,VNTAG,VLAN,IP
+XXXXXXXXXX.XXXXXX	IP	0xfd	9d6c1f9e20274bb66385	ETHERNET,VNTAG,VLAN,IP
+XXXXXXXXXX.XXXXXX	IP	0xfd	06ffb64ded001f65f818	ETHERNET,VNTAG,VLAN,IP
 #close XXXX-XX-XX-XX-XX-XX

testing/btest/Baseline/plugins.packet-protocol/output_orig

@@ -5,7 +5,7 @@
 #unset_field	-
 #path	unknown_protocols
 #open XXXX-XX-XX-XX-XX-XX
-#fields	ts	analyzer	protocol_id	first_bytes
-#types	time	string	string	string
-XXXXXXXXXX.XXXXXX	ETHERNET	0x88b5	4920616d20656e636170
+#fields	ts	analyzer	protocol_id	first_bytes	analyzer_history
+#types	time	string	string	string	vector[string]
+XXXXXXXXXX.XXXXXX	ETHERNET	0x88b5	4920616d20656e636170	ETHERNET
 #close XXXX-XX-XX-XX-XX-XX

testing/btest/core/unknown-protocol-event.zeek

@@ -1,6 +1,9 @@
 # @TEST-EXEC: zeek -b -r $TRACES/lldp.pcap %INPUT >out
 # @TEST-EXEC: btest-diff out
 
-event unknown_protocol(analyzer_name: string, protocol: count, first_bytes: string)
-	{ print analyzer_name, protocol, bytestring_to_hexstr(first_bytes); }
-
+event unknown_protocol(analyzer_name: string, protocol: count, first_bytes: string,
+	analyzer_history: string_vec)
+	{
+	print analyzer_name, protocol, bytestring_to_hexstr(first_bytes),
+		analyzer_history;
+	}

testing/external/commit-hash.zeek-testing

@@ -1 +1 @@
-3577ad0657c0cd61222d329a840e881dacf93f07
+4f808b8e1a4e99e738af85f15a5534c0ee99cdba