Merge remote-tracking branch 'origin/topic/awelzel/get-geneve-options'

* origin/topic/awelzel/get-geneve-options:
  PacketAnalyzer::Geneve: Add get_options()
  packet_analysis: Track data spans of packet analyzers

CHANGES

@@ -1,3 +1,19 @@
+7.2.0-dev.217 | 2025-02-22 12:33:25 -0800
+
+  * PacketAnalyzer::Geneve: Add get_options() (Arne Welzel, Corelight)
+
+    Allow to extract Geneve options on-demand, for example during a
+    new_connection() event.
+
+  * packet_analysis: Track data spans of packet analyzers (Arne Welzel, Corelight)
+
+    Do not just track the analyzer instance in the stack, but also the
+    data span it is given. This allows to extract more information on-demand
+    during event processing.
+
+    TrackAnalyzer() is technically a public API, but no one should use it
+    outside of the Analyzer's Forward methods itself.
+
 7.2.0-dev.214 | 2025-02-21 12:24:21 +0100
 
   * Fix bifs.to_count and bifs.to_int btests under ZAM (Tim Wojtulewicz, Corelight)

NEWS

@@ -24,6 +24,9 @@ New Functionality
   encrypted session information from a Kerberos response, including the cipher
   and encrypted data.
 
+- Geneve tunnel options of the current packet can be extracted from scripts
+  using the new PacketAnalyzer::Geneve::get_options() builtin function.
+
 Changed Functionality
 ---------------------
 

VERSION

@@ -1 +1 @@
-7.2.0-dev.214
+7.2.0-dev.217

scripts/base/packet-protocols/geneve/main.zeek

@@ -6,6 +6,18 @@ export {
 	## if you customize this, you may still want to manually ensure that
 	## :zeek:see:`likely_server_ports` also gets populated accordingly.
 	const geneve_ports: set[port] = { 6081/udp } &redef;
+
+	## A Geneve option.
+	type Option: record {
+		## The class of the option.
+		class: count;
+		## The critical bit of the type.
+		critical: bool;
+		## The type field of the option with the critical bit masked.
+		typ: count;
+		## The data field of the option.
+		data: string;
+	};
 }
 
 redef likely_server_ports += { geneve_ports };
@@ -25,3 +37,8 @@ event zeek_init() &priority=20
 	PacketAnalyzer::register_packet_analyzer(PacketAnalyzer::ANALYZER_GENEVE, 0x08DD, PacketAnalyzer::ANALYZER_IP);
 	PacketAnalyzer::register_packet_analyzer(PacketAnalyzer::ANALYZER_GENEVE, 0x0806, PacketAnalyzer::ANALYZER_ARP);
 	}
+
+module GLOBAL;
+
+type geneve_options_vec: vector of PacketAnalyzer::Geneve::Option;
+type geneve_options_vec_vec: vector of geneve_options_vec;

src/packet_analysis/Analyzer.cc

@@ -114,7 +114,7 @@ bool Analyzer::ForwardPacket(size_t len, const uint8_t* data, Packet* packet, ui
     DBG_LOG(DBG_PACKET_ANALYSIS, "Analysis in %s succeeded, next layer identifier is %#x.", GetAnalyzerName(),
             identifier);
 
-    packet_mgr->TrackAnalyzer(inner_analyzer.get());
+    packet_mgr->TrackAnalyzer(inner_analyzer.get(), len, data);
     return inner_analyzer->AnalyzePacket(len, data, packet);
 }
 
@@ -130,7 +130,7 @@ bool Analyzer::ForwardPacket(size_t len, const uint8_t* data, Packet* packet) co
         return false;
     }
 
-    packet_mgr->TrackAnalyzer(inner_analyzer.get());
+    packet_mgr->TrackAnalyzer(inner_analyzer.get(), len, data);
     return inner_analyzer->AnalyzePacket(len, data, packet);
 }
 

src/packet_analysis/Manager.cc

@@ -232,7 +232,7 @@ zeek::VectorValPtr Manager::BuildAnalyzerHistory() const {
     auto history = zeek::make_intrusive<zeek::VectorVal>(zeek::id::string_vec);
 
     for ( unsigned int i = 0; i < analyzer_stack.size(); i++ ) {
-        auto analyzer_name = analyzer_stack[i]->GetAnalyzerName();
+        auto analyzer_name = analyzer_stack[i].analyzer->GetAnalyzerName();
         history->Assign(i, make_intrusive<StringVal>(analyzer_name));
     }
 
@@ -249,3 +249,13 @@ void Manager::ReportUnknownProtocol(const std::string& analyzer, uint32_t protoc
         }
     }
 }
+
+std::vector<zeek::Span<const uint8_t>> Manager::GetAnalyzerData(const AnalyzerPtr& analyzer) {
+    std::vector<zeek::Span<const uint8_t>> result;
+    for ( const auto [sa, span] : analyzer_stack ) {
+        if ( sa == analyzer.get() )
+            result.push_back(span);
+    }
+
+    return result;
+}

src/packet_analysis/Manager.h

@@ -182,8 +182,21 @@ public:
      * call to ProcessPacket() but maintained throughout calls to ProcessInnerPacket().
      *
      * @param analyzer The analyzer to track.
+     * @param len The remaining length of the data in the packet being processed.
+     * @param data A pointer to the data
      */
-    void TrackAnalyzer(Analyzer* analyzer) { analyzer_stack.push_back(analyzer); }
+    void TrackAnalyzer(const Analyzer* analyzer, size_t len, const uint8_t* data) {
+        analyzer_stack.push_back({analyzer, {data, len}});
+    }
+
+    /**
+     * Get all tracked data spans for a given analyzer instance.
+     *
+     * @analyzer The analyzer instance.
+     *
+     * @returns An array of data spans.
+     */
+    std::vector<zeek::Span<const uint8_t>> GetAnalyzerData(const AnalyzerPtr& analyzer);
 
 private:
     /**
@@ -234,7 +247,12 @@ private:
     uint64_t total_not_processed = 0;
     iosource::PktDumper* unprocessed_dumper = nullptr;
 
-    std::vector<Analyzer*> analyzer_stack;
+    struct StackEntry {
+        const Analyzer* analyzer;
+        zeek::Span<const uint8_t> data; // Start of this layer, limited by span's size.
+    };
+
+    std::vector<StackEntry> analyzer_stack;
 };
 
 } // namespace packet_analysis

src/packet_analysis/protocol/geneve/CMakeLists.txt

@@ -5,4 +5,5 @@ zeek_add_plugin(
     Geneve.cc
     Plugin.cc
     BIFS
-    events.bif)
+    events.bif
+    functions.bif)

src/packet_analysis/protocol/geneve/Geneve.cc

@@ -2,11 +2,50 @@
 
 #include "zeek/packet_analysis/protocol/geneve/Geneve.h"
 
+#include "zeek/Span.h"
 #include "zeek/packet_analysis/protocol/geneve/events.bif.h"
 #include "zeek/packet_analysis/protocol/iptunnel/IPTunnel.h"
 
 using namespace zeek::packet_analysis::Geneve;
 
+void zeek::packet_analysis::Geneve::detail::parse_options(zeek::Span<const uint8_t> data, detail::Callback cb) {
+    size_t remaining = data.size();
+
+    if ( remaining < 8 )
+        return;
+
+    remaining -= 8;
+
+    uint8_t all_opt_len = (data[0] & 0x3F) * 4;
+
+    if ( remaining < all_opt_len )
+        return;
+
+    const uint8_t* p = &data[8];
+    const uint8_t* const end = &data[8] + all_opt_len;
+
+    while ( p < end ) {
+        auto remaining = end - p;
+        if ( remaining < 4 )
+            break;
+
+        uint16_t opt_class = ntohs(reinterpret_cast<const uint16_t*>(p)[0]);
+        bool opt_critical = (p[2] & 0x80) == 0x80;
+        uint8_t opt_type = p[2] & 0x7F;
+        uint8_t opt_len = (p[3] & 0x1F) * 4;
+
+        remaining -= 4;
+        p += 4;
+
+        if ( remaining < opt_len )
+            break;
+
+        cb(opt_class, opt_critical, opt_type, zeek::Span{p, opt_len});
+
+        p += opt_len;
+    }
+}
+
 GeneveAnalyzer::GeneveAnalyzer() : zeek::packet_analysis::Analyzer("Geneve") {}
 
 bool GeneveAnalyzer::AnalyzePacket(size_t len, const uint8_t* data, Packet* packet) {

src/packet_analysis/protocol/geneve/Geneve.h

@@ -2,11 +2,34 @@
 
 #pragma once
 
+#include <functional>
+
+#include "zeek/Span.h"
 #include "zeek/packet_analysis/Analyzer.h"
 #include "zeek/packet_analysis/Component.h"
 
 namespace zeek::packet_analysis::Geneve {
 
+namespace detail {
+
+/**
+ * Callback for parse_options(), passing the individual option pieces.
+ */
+using Callback =
+    std::function<void(uint16_t opt_class, bool opt_critical, uint8_t opt_type, zeek::Span<const uint8_t> opt_data)>;
+
+/**
+ * Parse Geneve options from the header data.
+ *
+ * For each option, the given callback is invoked.
+ *
+ * @param data The data span to treat as a Geneve header.
+ * @param cb The callback to invoke with each parsed option.
+ */
+void parse_options(zeek::Span<const uint8_t> data, Callback cb);
+
+} // namespace detail
+
 class GeneveAnalyzer : public zeek::packet_analysis::Analyzer {
 public:
     GeneveAnalyzer();

src/packet_analysis/protocol/geneve/functions.bif

@@ -0,0 +1,44 @@
+module PacketAnalyzer::Geneve;
+
+%%{
+#include "zeek/packet_analysis/Manager.h"
+#include "zeek/packet_analysis/protocol/geneve/Geneve.h"
+%%}
+
+## Returns all Geneve options from all layers of the current packet.
+##
+## The last entry in the outer vector are the options of the most
+## inner Geneve header.
+##
+## Returns a vector of vector of zeek::see:`PacketAnalyzer::Geneve::Option` records.
+function get_options%(%): geneve_options_vec_vec
+    %{
+    static const auto& analyzer = zeek::packet_mgr->GetAnalyzer("Geneve");
+    static const auto& rvtype = zeek::id::find_type<zeek::VectorType>("geneve_options_vec_vec");
+    static const auto& vtype = zeek::id::find_type<zeek::VectorType>("geneve_options_vec");
+    static const auto& rtype = zeek::id::find_type<zeek::RecordType>("PacketAnalyzer::Geneve::Option");
+
+    auto result = zeek::make_intrusive<zeek::VectorVal>(rvtype);
+    auto spans = zeek::packet_mgr->GetAnalyzerData(analyzer);
+    result->Reserve(spans.size());
+
+    for ( const auto& span : spans ) {
+        auto v = zeek::make_intrusive<zeek::VectorVal>(vtype);
+        auto cb = [&v](uint16_t opt_class, bool opt_critical, uint8_t opt_type, zeek::Span<const uint8_t> opt_data) -> void {
+            auto rv = zeek::make_intrusive<zeek::RecordVal>(rtype);
+            rv->Assign(0, zeek::val_mgr->Count(opt_class));
+            rv->Assign(1, zeek::val_mgr->Bool(opt_critical));
+            rv->Assign(2, zeek::val_mgr->Count(opt_type));
+            auto sv = zeek::make_intrusive<zeek::StringVal>(opt_data.size(), reinterpret_cast<const char*>(opt_data.data()));
+            rv->Assign(3, std::move(sv));
+
+            v->Append(std::move(rv));
+        };
+
+        zeek::packet_analysis::Geneve::detail::parse_options(span, cb);
+
+        result->Append(std::move(v));
+    }
+
+    return result;
+    %}

src/script_opt/FuncInfo.cc

@@ -112,6 +112,7 @@ static std::unordered_map<std::string, unsigned int> func_attrs = {
     {"Option::any_set_to_any_vec", ATTR_FOLDABLE},
     {"Option::set_change_handler", ATTR_NO_SCRIPT_SIDE_EFFECTS},
     {"PacketAnalyzer::GTPV1::remove_gtpv1_connection", ATTR_NO_SCRIPT_SIDE_EFFECTS},
+    {"PacketAnalyzer::Geneve::get_options", ATTR_NO_SCRIPT_SIDE_EFFECTS},
     {"PacketAnalyzer::TEREDO::remove_teredo_connection", ATTR_NO_SCRIPT_SIDE_EFFECTS},
     {"PacketAnalyzer::__disable_analyzer", ATTR_NO_SCRIPT_SIDE_EFFECTS},
     {"PacketAnalyzer::__enable_analyzer", ATTR_NO_SCRIPT_SIDE_EFFECTS},

testing/btest/Baseline/core.tunnels.geneve-get-options/out

@@ -0,0 +1,7 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+new_connection, ClEkJM2Vm5giqnMf4h, geneve-many-options.pcap
+opt, [class=256, critical=F, typ=1, data=1234567890abcde\x00]
+opt, [class=256, critical=F, typ=2, data=0123456789abcdef0123456789abcdef0\x00\x00\x00]
+opt, [class=256, critical=F, typ=3, data=0123456789\x00\x00]
+new_connection, ClEkJM2Vm5giqnMf4h, geneve-tagged-udp-packet.pcap
+opt, [class=65394, critical=F, typ=127, data=/home/esk/src/zeek//testing/btest/Traces/udp-packet.pcap]

testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log

@@ -264,6 +264,7 @@ scripts/base/init-frameworks-and-bifs.zeek
     build/scripts/base/bif/plugins/Zeek_UDP.events.bif.zeek
     build/scripts/base/bif/plugins/Zeek_ICMP.events.bif.zeek
     build/scripts/base/bif/plugins/Zeek_Geneve.events.bif.zeek
+    build/scripts/base/bif/plugins/Zeek_Geneve.functions.bif.zeek
     build/scripts/base/bif/plugins/Zeek_VXLAN.events.bif.zeek
     build/scripts/base/bif/plugins/Zeek_FileEntropy.events.bif.zeek
     build/scripts/base/bif/plugins/Zeek_FileExtract.events.bif.zeek

testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log

@@ -264,6 +264,7 @@ scripts/base/init-frameworks-and-bifs.zeek
     build/scripts/base/bif/plugins/Zeek_UDP.events.bif.zeek
     build/scripts/base/bif/plugins/Zeek_ICMP.events.bif.zeek
     build/scripts/base/bif/plugins/Zeek_Geneve.events.bif.zeek
+    build/scripts/base/bif/plugins/Zeek_Geneve.functions.bif.zeek
     build/scripts/base/bif/plugins/Zeek_VXLAN.events.bif.zeek
     build/scripts/base/bif/plugins/Zeek_FileEntropy.events.bif.zeek
     build/scripts/base/bif/plugins/Zeek_FileExtract.events.bif.zeek

testing/btest/Baseline/opt.ZAM-bif-tracking/output

@@ -1,2 +1,2 @@
 ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
-543 seen BiFs, 0 unseen BiFs (), 0 new BiFs ()
+544 seen BiFs, 0 unseen BiFs (), 0 new BiFs ()

testing/btest/Baseline/plugins.hooks/output

@@ -360,6 +360,7 @@
 0.000000   MetaHookPost  LoadFile(0, ./Zeek_GTPv1.events.bif.zeek, <...>/Zeek_GTPv1.events.bif.zeek) -> -1
 0.000000   MetaHookPost  LoadFile(0, ./Zeek_GTPv1.functions.bif.zeek, <...>/Zeek_GTPv1.functions.bif.zeek) -> -1
 0.000000   MetaHookPost  LoadFile(0, ./Zeek_Geneve.events.bif.zeek, <...>/Zeek_Geneve.events.bif.zeek) -> -1
+0.000000   MetaHookPost  LoadFile(0, ./Zeek_Geneve.functions.bif.zeek, <...>/Zeek_Geneve.functions.bif.zeek) -> -1
 0.000000   MetaHookPost  LoadFile(0, ./Zeek_Gnutella.events.bif.zeek, <...>/Zeek_Gnutella.events.bif.zeek) -> -1
 0.000000   MetaHookPost  LoadFile(0, ./Zeek_HTTP.events.bif.zeek, <...>/Zeek_HTTP.events.bif.zeek) -> -1
 0.000000   MetaHookPost  LoadFile(0, ./Zeek_HTTP.functions.bif.zeek, <...>/Zeek_HTTP.functions.bif.zeek) -> -1
@@ -667,6 +668,7 @@
 0.000000   MetaHookPost  LoadFileExtended(0, ./Zeek_GTPv1.events.bif.zeek, <...>/Zeek_GTPv1.events.bif.zeek) -> (-1, <no content>)
 0.000000   MetaHookPost  LoadFileExtended(0, ./Zeek_GTPv1.functions.bif.zeek, <...>/Zeek_GTPv1.functions.bif.zeek) -> (-1, <no content>)
 0.000000   MetaHookPost  LoadFileExtended(0, ./Zeek_Geneve.events.bif.zeek, <...>/Zeek_Geneve.events.bif.zeek) -> (-1, <no content>)
+0.000000   MetaHookPost  LoadFileExtended(0, ./Zeek_Geneve.functions.bif.zeek, <...>/Zeek_Geneve.functions.bif.zeek) -> (-1, <no content>)
 0.000000   MetaHookPost  LoadFileExtended(0, ./Zeek_Gnutella.events.bif.zeek, <...>/Zeek_Gnutella.events.bif.zeek) -> (-1, <no content>)
 0.000000   MetaHookPost  LoadFileExtended(0, ./Zeek_HTTP.events.bif.zeek, <...>/Zeek_HTTP.events.bif.zeek) -> (-1, <no content>)
 0.000000   MetaHookPost  LoadFileExtended(0, ./Zeek_HTTP.functions.bif.zeek, <...>/Zeek_HTTP.functions.bif.zeek) -> (-1, <no content>)
@@ -1307,6 +1309,7 @@
 0.000000   MetaHookPre   LoadFile(0, ./Zeek_GTPv1.events.bif.zeek, <...>/Zeek_GTPv1.events.bif.zeek)
 0.000000   MetaHookPre   LoadFile(0, ./Zeek_GTPv1.functions.bif.zeek, <...>/Zeek_GTPv1.functions.bif.zeek)
 0.000000   MetaHookPre   LoadFile(0, ./Zeek_Geneve.events.bif.zeek, <...>/Zeek_Geneve.events.bif.zeek)
+0.000000   MetaHookPre   LoadFile(0, ./Zeek_Geneve.functions.bif.zeek, <...>/Zeek_Geneve.functions.bif.zeek)
 0.000000   MetaHookPre   LoadFile(0, ./Zeek_Gnutella.events.bif.zeek, <...>/Zeek_Gnutella.events.bif.zeek)
 0.000000   MetaHookPre   LoadFile(0, ./Zeek_HTTP.events.bif.zeek, <...>/Zeek_HTTP.events.bif.zeek)
 0.000000   MetaHookPre   LoadFile(0, ./Zeek_HTTP.functions.bif.zeek, <...>/Zeek_HTTP.functions.bif.zeek)
@@ -1614,6 +1617,7 @@
 0.000000   MetaHookPre   LoadFileExtended(0, ./Zeek_GTPv1.events.bif.zeek, <...>/Zeek_GTPv1.events.bif.zeek)
 0.000000   MetaHookPre   LoadFileExtended(0, ./Zeek_GTPv1.functions.bif.zeek, <...>/Zeek_GTPv1.functions.bif.zeek)
 0.000000   MetaHookPre   LoadFileExtended(0, ./Zeek_Geneve.events.bif.zeek, <...>/Zeek_Geneve.events.bif.zeek)
+0.000000   MetaHookPre   LoadFileExtended(0, ./Zeek_Geneve.functions.bif.zeek, <...>/Zeek_Geneve.functions.bif.zeek)
 0.000000   MetaHookPre   LoadFileExtended(0, ./Zeek_Gnutella.events.bif.zeek, <...>/Zeek_Gnutella.events.bif.zeek)
 0.000000   MetaHookPre   LoadFileExtended(0, ./Zeek_HTTP.events.bif.zeek, <...>/Zeek_HTTP.events.bif.zeek)
 0.000000   MetaHookPre   LoadFileExtended(0, ./Zeek_HTTP.functions.bif.zeek, <...>/Zeek_HTTP.functions.bif.zeek)
@@ -2253,6 +2257,7 @@
 0.000000 | HookLoadFile  ./Zeek_GTPv1.events.bif.zeek <...>/Zeek_GTPv1.events.bif.zeek
 0.000000 | HookLoadFile  ./Zeek_GTPv1.functions.bif.zeek <...>/Zeek_GTPv1.functions.bif.zeek
 0.000000 | HookLoadFile  ./Zeek_Geneve.events.bif.zeek <...>/Zeek_Geneve.events.bif.zeek
+0.000000 | HookLoadFile  ./Zeek_Geneve.functions.bif.zeek <...>/Zeek_Geneve.functions.bif.zeek
 0.000000 | HookLoadFile  ./Zeek_Gnutella.events.bif.zeek <...>/Zeek_Gnutella.events.bif.zeek
 0.000000 | HookLoadFile  ./Zeek_HTTP.events.bif.zeek <...>/Zeek_HTTP.events.bif.zeek
 0.000000 | HookLoadFile  ./Zeek_HTTP.functions.bif.zeek <...>/Zeek_HTTP.functions.bif.zeek
@@ -2560,6 +2565,7 @@
 0.000000 | HookLoadFileExtended ./Zeek_GTPv1.events.bif.zeek <...>/Zeek_GTPv1.events.bif.zeek
 0.000000 | HookLoadFileExtended ./Zeek_GTPv1.functions.bif.zeek <...>/Zeek_GTPv1.functions.bif.zeek
 0.000000 | HookLoadFileExtended ./Zeek_Geneve.events.bif.zeek <...>/Zeek_Geneve.events.bif.zeek
+0.000000 | HookLoadFileExtended ./Zeek_Geneve.functions.bif.zeek <...>/Zeek_Geneve.functions.bif.zeek
 0.000000 | HookLoadFileExtended ./Zeek_Gnutella.events.bif.zeek <...>/Zeek_Gnutella.events.bif.zeek
 0.000000 | HookLoadFileExtended ./Zeek_HTTP.events.bif.zeek <...>/Zeek_HTTP.events.bif.zeek
 0.000000 | HookLoadFileExtended ./Zeek_HTTP.functions.bif.zeek <...>/Zeek_HTTP.functions.bif.zeek

testing/btest/Traces/README

@@ -46,3 +46,5 @@ Trace Index/Sources:
   VirusTotal reports that this file contains malware. The PE analyzer was originally added
   to decode info for malware, so this is expected. See
   https://zeekorg.slack.com/archives/CSZBXF6TH/p1738261449655049
+- tunnels/geneve-tagged-udp-packet.pcap
+  Provided by Eldon Koyle Corelight for testing.

testing/btest/core/tunnels/geneve-get-options.zeek

@@ -0,0 +1,20 @@
+# @TEST-EXEC: zeek -C -b -r $TRACES/tunnels/geneve-many-options.pcap %INPUT >>out
+# @TEST-EXEC: zeek -C -b -r $TRACES/tunnels/geneve-tagged-udp-packet.pcap %INPUT >>out
+# @TEST-EXEC: btest-diff out
+
+@load base/frameworks/tunnels
+@load base/protocols/conn
+
+
+event new_connection(c: connection)
+	{
+	if ( ! c?$tunnel )
+		return;
+
+	print "new_connection", c$uid, split_string(packet_source()$path, /\//)[-1];
+
+	for ( _, layer in PacketAnalyzer::Geneve::get_options() )
+		for ( _, opt in layer )
+			print "opt", opt;
+
+	}

testing/btest/opt/ZAM-bif-tracking.zeek

@@ -141,6 +141,7 @@ global known_BiFs = set(
 	"Option::set",
 	"Option::set_change_handler",
 	"PacketAnalyzer::GTPV1::remove_gtpv1_connection",
+	"PacketAnalyzer::Geneve::get_options",
 	"PacketAnalyzer::TEREDO::remove_teredo_connection",
 	"PacketAnalyzer::__disable_analyzer",
 	"PacketAnalyzer::__enable_analyzer",