SessionAdapter: Introduce TapAnalyzer for session adapter

This commit introduces a mechanism to attach light weight analyzers to
the root analyzer of sessions in order to tap into the packets delivered
to child analyzer.

NEWS

@@ -14,6 +14,10 @@ Breaking Changes
 New Functionality
 -----------------
 
+- A new TapAnalyzer class was added allowing to tap into all packets delivered
+  to child analyzers attached to session adapters.
+
+
 Changed Functionality
 ---------------------
 

src/packet_analysis/protocol/icmp/ICMP.cc

@@ -13,6 +13,7 @@
 #include "zeek/analyzer/Manager.h"
 #include "zeek/packet_analysis/protocol/icmp/ICMPSessionAdapter.h"
 #include "zeek/packet_analysis/protocol/icmp/events.bif.h"
+#include "zeek/packet_analysis/protocol/ip/SessionAdapter.h"
 #include "zeek/session/Manager.h"
 
 using namespace zeek::packet_analysis::ICMP;
@@ -80,6 +81,7 @@ void ICMPAnalyzer::DeliverPacket(Connection* c, double t, bool is_orig, int rema
 
         if ( chksum != 0xffff ) {
             adapter->Weird("bad_ICMP_checksum");
+            adapter->TapPacket(pkt, PacketAction::Skip, SkipReason::BadChecksum);
             return;
         }
     }
@@ -112,6 +114,9 @@ void ICMPAnalyzer::DeliverPacket(Connection* c, double t, bool is_orig, int rema
 
     ForwardPacket(std::min(len, remaining), data, pkt);
 
+    // Tap the packet before sending it to protocol analysis.
+    adapter->TapPacket(pkt);
+
     if ( remaining >= len )
         adapter->ForwardPacket(len, data, is_orig, -1, ip.get(), remaining);
 

src/packet_analysis/protocol/ip/SessionAdapter.cc

@@ -8,7 +8,14 @@
 
 using namespace zeek::packet_analysis::IP;
 
-void SessionAdapter::Done() { Analyzer::Done(); }
+void SessionAdapter::Done() {
+    Analyzer::Done();
+    for ( const auto& ta : tap_analyzers )
+        ta->Done();
+
+    // Ensure no more TapPacket() calls after Done() on TapAnalyzer instances.
+    tap_analyzers.clear();
+}
 
 bool SessionAdapter::IsReuse(double t, const u_char* pkt) { return parent->IsReuse(t, pkt); }
 
@@ -28,3 +35,37 @@ void SessionAdapter::PacketContents(const u_char* data, int len) {
         EnqueueConnEvent(packet_contents, ConnVal(), std::move(contents));
     }
 }
+
+void SessionAdapter::AddTapAnalyzer(TapAnalyzerPtr ta) {
+    assert(! IsFinished());
+    tap_analyzers.push_back(std::move(ta));
+    tap_analyzers.back()->Init();
+}
+
+bool SessionAdapter::RemoveTapAnalyzer(const TapAnalyzer* ta) {
+    // Find the raw pointer, call Done(), remove it, thereby destructing it.
+    for ( auto it = tap_analyzers.begin(); it != tap_analyzers.end(); ++it ) {
+        if ( it->get() == ta ) {
+            // Ensure Done() is called only after removal from tap_analyzers.
+            auto ptr{std::move(*it)};
+            tap_analyzers.erase(it);
+            ptr->Done();
+            ptr.reset();
+            return true;
+        }
+    }
+
+    return false;
+}
+
+void SessionAdapter::TapPacket(const Packet* pkt, PacketAction action, SkipReason skip_reason) const {
+    for ( const auto& ta : tap_analyzers )
+        ta->TapPacket(*pkt, action, skip_reason);
+}
+
+void SessionAdapter::UpdateConnVal(RecordVal* conn_val) {
+    Analyzer::UpdateConnVal(conn_val);
+
+    for ( const auto& ta : tap_analyzers )
+        ta->UpdateConnVal(conn_val);
+}

src/packet_analysis/protocol/ip/SessionAdapter.h

@@ -2,13 +2,90 @@
 
 #pragma once
 
+#include <memory>
+
 #include "zeek/analyzer/Analyzer.h"
+#include "zeek/iosource/Packet.h"
 
 namespace zeek::analyzer::pia {
 class PIA;
 }
 
-namespace zeek::packet_analysis::IP {
+namespace zeek::packet_analysis {
+
+/**
+ * Indicator for a TapAnalyzer to determine what will happen to a packet.
+ */
+enum class PacketAction : uint8_t {
+    Deliver, ///< Packet will be delivered to child protocol analyzers.
+    Skip,    ///< Processing of this packet will be skipped.
+};
+
+/**
+ * Reason why delivery of a packet would be skipped.
+ */
+enum class SkipReason : uint8_t {
+    None,           ///< None is used when the verdict is Deliver.
+    Unknown,        ///< Placeholder if no other value fits.
+    BadChecksum,    ///< The packet's checksum is invalid and ignore_checksums is false.
+    BadProtoHeader, ///< Something was off with the lengths or offsets in the protocol header.
+};
+
+/**
+ * A lightweight analyzer that receives all packets passing through session adapters.
+ *
+ * A use case of tap analyzers is to attach them during HookSetupAnalyzerTree() to
+ * observe all raw packets of a session, including those that are invalid or corrupt
+ * and aren't delivered to child protocol analyzers.
+ *
+ * The Packet class has an *is_orig* field if directionality is required. Additionally,
+ * the Connection instance available during HookSetupAnalyzerTree() can be stored into
+ * a custom TapAnalyzer, allowing to associate packets with a given Connection. However,
+ * the TapAnalyzer interface itself does not provide provisions for this use case.
+ */
+class TapAnalyzer {
+public:
+    virtual ~TapAnalyzer() = default;
+
+    /**
+     * Hook of a tap analyzer for receiving packet data.
+     *
+     * @param pkt The packet being processed.
+     * @param action Either Deliver or Skip as determined by session analyzers.
+     * @param skip_reason If verdict is Skip, an indication why this packet is skipped, otherwise None.
+     */
+    virtual void TapPacket(const Packet& pkt, PacketAction action, SkipReason skip_reason) = 0;
+
+    /**
+     * Hook for when the script-level connection record is updated.
+     *
+     * This is invoked when the session's UpdateConnVal() method has invoked UpdateConnVal()
+     * on all protocol analyzers attached to the session.
+     *
+     * @param conn_val The script-level connection record associated with the
+     *                 Connection this TapAnalyzer is attached to.
+     */
+    virtual void UpdateConnVal(RecordVal* conn_val) {}
+
+    /**
+     * Hook for initialization before tapping begins.
+     *
+     * This method is invoked after a tap analyzer has been added to a SessionAdapter.
+     */
+    virtual void Init() {};
+
+    /**
+     * Hook for when this analyzer is about to be removed and destructed.
+     *
+     * This is invoked when the session's Done() method is invoked, just before
+     * the TapAnalyzer instance is destroyed.
+     */
+    virtual void Done() {};
+};
+
+using TapAnalyzerPtr = std::unique_ptr<TapAnalyzer>;
+
+namespace IP {
 
 class IPBasedAnalyzer;
 
@@ -94,9 +171,45 @@ public:
      */
     void PacketContents(const u_char* data, int len);
 
+    /**
+     * Adds a TapAnalyzer instance to this session adapter.
+     *
+     * @param ta The TapAnalyzer instance to attach.
+     */
+    void AddTapAnalyzer(TapAnalyzerPtr ta);
+
+    /**
+     * Remove a TapAnalyzer instance by raw pointer.
+     *
+     * Note that the TapAnalyzer instance \a ta is pointing at will be destroyed
+     * during the call to RemoveTapAanalyzer() and should be discarded by the caller
+     * immediately. If you call RemoveTapAnalyzer() from within a TapAnalyzer's member
+     * function, ensure not accessing \a this afterwards.
+     *
+     * @param ta The raw pointer to the TapAnalyzer instance to remove.
+     */
+    bool RemoveTapAnalyzer(const TapAnalyzer* ta);
+
+    /**
+     * Helper to forward a packet to all attached TapAnalyzer instances.
+     *
+     * @param pkt The packet.
+     * @param verdict Whether the packet will be delivered or skipped.
+     * @param skip_reason If verdict is Skip, should be an indication why this packet is skipped.
+     */
+    void TapPacket(const Packet* pkt, PacketAction verdict = PacketAction::Deliver,
+                   SkipReason skip_reason = SkipReason::None) const;
+
+    /**
+     * Overridden from parent class, calling UpdateConnVal() on TapAnalyzer instances, too.
+     */
+    void UpdateConnVal(RecordVal* conn_val) override;
+
 protected:
     IPBasedAnalyzer* parent = nullptr;
     analyzer::pia::PIA* pia = nullptr;
+    std::vector<TapAnalyzerPtr> tap_analyzers;
 };
 
-} // namespace zeek::packet_analysis::IP
+} // namespace IP
+} // namespace zeek::packet_analysis

src/packet_analysis/protocol/tcp/TCP.cc

@@ -4,6 +4,7 @@
 
 #include "zeek/RunState.h"
 #include "zeek/analyzer/protocol/pia/PIA.h"
+#include "zeek/packet_analysis/protocol/ip/SessionAdapter.h"
 #include "zeek/packet_analysis/protocol/tcp/TCPSessionAdapter.h"
 
 using namespace zeek;
@@ -83,8 +84,10 @@ void TCPAnalyzer::DeliverPacket(Connection* c, double t, bool is_orig, int remai
     auto* adapter = static_cast<TCPSessionAdapter*>(c->GetSessionAdapter());
 
     const struct tcphdr* tp = ExtractTCP_Header(data, len, remaining, adapter);
-    if ( ! tp )
+    if ( ! tp ) {
+        adapter->TapPacket(pkt, PacketAction::Skip, SkipReason::BadProtoHeader);
         return;
+    }
 
     // We need the min() here because Ethernet frame padding can lead to
     // remaining > len.
@@ -95,8 +98,10 @@ void TCPAnalyzer::DeliverPacket(Connection* c, double t, bool is_orig, int remai
     analyzer::tcp::TCP_Endpoint* peer = endpoint->peer;
     const std::shared_ptr<IP_Hdr>& ip = pkt->ip_hdr;
 
-    if ( ! ValidateChecksum(ip.get(), tp, endpoint, len, remaining, adapter) )
+    if ( ! ValidateChecksum(ip.get(), tp, endpoint, len, remaining, adapter) ) {
+        adapter->TapPacket(pkt, PacketAction::Skip, SkipReason::BadChecksum);
         return;
+    }
 
     adapter->Process(is_orig, tp, len, ip, data, remaining);
 
@@ -107,6 +112,9 @@ void TCPAnalyzer::DeliverPacket(Connection* c, double t, bool is_orig, int remai
     // Send the packet back into the packet analysis framework.
     ForwardPacket(std::min(len, remaining), data, pkt);
 
+    // Tap the packet before sending it to session analysis.
+    adapter->TapPacket(pkt);
+
     // Call DeliverPacket on the adapter directly here. Normally we'd call ForwardPacket
     // but this adapter does some other things in its DeliverPacket with the packet children
     // analyzers.

src/packet_analysis/protocol/udp/UDP.cc

@@ -8,6 +8,7 @@
 #include "zeek/RunState.h"
 #include "zeek/analyzer/Manager.h"
 #include "zeek/analyzer/protocol/pia/PIA.h"
+#include "zeek/packet_analysis/protocol/ip/SessionAdapter.h"
 #include "zeek/packet_analysis/protocol/udp/UDPSessionAdapter.h"
 #include "zeek/packet_analysis/protocol/udp/events.bif.h"
 #include "zeek/session/Manager.h"
@@ -126,6 +127,7 @@ void UDPAnalyzer::DeliverPacket(Connection* c, double t, bool is_orig, int remai
 
         if ( bad ) {
             adapter->HandleBadChecksum(is_orig);
+            adapter->TapPacket(pkt, PacketAction::Skip, SkipReason::BadChecksum);
             return;
         }
     }
@@ -194,6 +196,9 @@ void UDPAnalyzer::DeliverPacket(Connection* c, double t, bool is_orig, int remai
     // detection has to be used.
     ForwardPacket(std::min(len, remaining), data, pkt, ntohs(c->RespPort()));
 
+    // Tap the packet before sending it to session analysis.
+    adapter->TapPacket(pkt);
+
     // Forward any data through session-analysis, too.
     adapter->ForwardPacket(std::min(len, remaining), data, is_orig, -1, ip.get(), pkt->cap_len);
 }

testing/btest/Baseline/plugins.tap-analyzer/output

@@ -0,0 +1,68 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+<...>/ip4-tcp-bad-chksum.pcap
+Init() uid=HhAvVGS1DHFjwGM9
+Analyzer added to HhAvVGS1DHFjwGM9
+Packet(len=54 orig=1, verdict=1 skip_reason=1) uid=HhAvVGS1DHFjwGM9
+Done() uid=HhAvVGS1DHFjwGM9
+===
+<...>/ip4-tcp-good-chksum.pcap
+Init() uid=HhAvVGS1DHFjwGM9
+Analyzer added to HhAvVGS1DHFjwGM9
+Packet(len=54 orig=1, verdict=0 skip_reason=0) uid=HhAvVGS1DHFjwGM9
+Done() uid=HhAvVGS1DHFjwGM9
+===
+<...>/ip4-udp-bad-chksum.pcap
+Init() uid=HhAvVGS1DHFjwGM9
+Analyzer added to HhAvVGS1DHFjwGM9
+Packet(len=46 orig=1, verdict=1 skip_reason=1) uid=HhAvVGS1DHFjwGM9
+Done() uid=HhAvVGS1DHFjwGM9
+===
+<...>/ip4-udp-good-chksum.pcap
+Init() uid=HhAvVGS1DHFjwGM9
+Analyzer added to HhAvVGS1DHFjwGM9
+Packet(len=46 orig=1, verdict=0 skip_reason=0) uid=HhAvVGS1DHFjwGM9
+Done() uid=HhAvVGS1DHFjwGM9
+===
+<...>/ip4-icmp-bad-chksum.pcap
+Init() uid=HhAvVGS1DHFjwGM9
+Analyzer added to HhAvVGS1DHFjwGM9
+Packet(len=42 orig=1, verdict=1 skip_reason=1) uid=HhAvVGS1DHFjwGM9
+Done() uid=HhAvVGS1DHFjwGM9
+===
+<...>/ip4-icmp-good-chksum.pcap
+Init() uid=HhAvVGS1DHFjwGM9
+Analyzer added to HhAvVGS1DHFjwGM9
+Packet(len=42 orig=1, verdict=0 skip_reason=0) uid=HhAvVGS1DHFjwGM9
+Done() uid=HhAvVGS1DHFjwGM9
+===
+<...>/ip6-icmp6-bad-chksum.pcap
+Init() uid=HhAvVGS1DHFjwGM9
+Analyzer added to HhAvVGS1DHFjwGM9
+Packet(len=69 orig=1, verdict=1 skip_reason=1) uid=HhAvVGS1DHFjwGM9
+Done() uid=HhAvVGS1DHFjwGM9
+===
+<...>/ip6-icmp6-good-chksum.pcap
+Init() uid=HhAvVGS1DHFjwGM9
+Analyzer added to HhAvVGS1DHFjwGM9
+Packet(len=69 orig=1, verdict=0 skip_reason=0) uid=HhAvVGS1DHFjwGM9
+Done() uid=HhAvVGS1DHFjwGM9
+===
+<...>/get.trace
+Init() uid=HhAvVGS1DHFjwGM9
+Analyzer added to HhAvVGS1DHFjwGM9
+Packet(len=78 orig=1, verdict=0 skip_reason=0) uid=HhAvVGS1DHFjwGM9
+Packet(len=74 orig=0, verdict=0 skip_reason=0) uid=HhAvVGS1DHFjwGM9
+Packet(len=66 orig=1, verdict=0 skip_reason=0) uid=HhAvVGS1DHFjwGM9
+Packet(len=202 orig=1, verdict=0 skip_reason=0) uid=HhAvVGS1DHFjwGM9
+Packet(len=66 orig=0, verdict=0 skip_reason=0) uid=HhAvVGS1DHFjwGM9
+Packet(len=1514 orig=0, verdict=0 skip_reason=0) uid=HhAvVGS1DHFjwGM9
+Packet(len=1514 orig=0, verdict=0 skip_reason=0) uid=HhAvVGS1DHFjwGM9
+Packet(len=1514 orig=0, verdict=0 skip_reason=0) uid=HhAvVGS1DHFjwGM9
+Packet(len=729 orig=0, verdict=0 skip_reason=0) uid=HhAvVGS1DHFjwGM9
+Packet(len=66 orig=1, verdict=0 skip_reason=0) uid=HhAvVGS1DHFjwGM9
+Packet(len=66 orig=1, verdict=0 skip_reason=0) uid=HhAvVGS1DHFjwGM9
+Packet(len=66 orig=1, verdict=0 skip_reason=0) uid=HhAvVGS1DHFjwGM9
+Packet(len=66 orig=0, verdict=0 skip_reason=0) uid=HhAvVGS1DHFjwGM9
+Packet(len=66 orig=1, verdict=0 skip_reason=0) uid=HhAvVGS1DHFjwGM9
+Done() uid=HhAvVGS1DHFjwGM9
+===

testing/btest/plugins/tap-analyzer-plugin/src/Plugin.cc

@@ -0,0 +1,60 @@
+#include "Plugin.h"
+
+#include <cstdio>
+#include <cstring>
+
+#include "zeek/Reporter.h"
+#include "zeek/analyzer/Analyzer.h"
+#include "zeek/analyzer/Manager.h"
+#include "zeek/analyzer/protocol/tcp/TCP.h"
+#include "zeek/packet_analysis/protocol/ip/SessionAdapter.h"
+
+namespace {
+class MyTapAnalyzer : public zeek::packet_analysis::TapAnalyzer {
+public:
+    MyTapAnalyzer(zeek::Connection* conn) : conn(conn) {}
+
+    void TapPacket(const zeek::Packet& pkt, zeek::packet_analysis::PacketAction verdict,
+                   const zeek::packet_analysis::SkipReason skip_reason) override {
+        std::printf("Packet(len=%d orig=%d, verdict=%d skip_reason=%d) uid=%s\n", pkt.len, pkt.is_orig,
+                    static_cast<int>(verdict), static_cast<int>(skip_reason), conn->GetUID().Base62().c_str());
+    }
+
+    void Init() override { std::printf("Init() uid=%s\n", conn->GetUID().Base62().c_str()); }
+
+    void Done() override { std::printf("Done() uid=%s\n", conn->GetUID().Base62().c_str()); }
+
+private:
+    zeek::Connection* conn = nullptr;
+};
+} // namespace
+
+
+namespace btest::plugin::Demo_TapAnalyzer {
+
+Plugin plugin;
+
+zeek::plugin::Configuration Plugin::Configure() {
+    EnableHook(zeek::plugin::HOOK_SETUP_ANALYZER_TREE);
+
+    zeek::plugin::Configuration config;
+    config.name = "Demo::TapAnalyzer";
+    config.description = "Testing the TapAnalyzer";
+    config.version = {1, 0, 0};
+    return config;
+}
+
+void Plugin::HookSetupAnalyzerTree(zeek::Connection* conn) {
+    // Init the uid for GetUID()
+    conn->GetVal();
+
+    auto analyzer = std::make_unique<MyTapAnalyzer>(conn);
+
+    auto* adapter = conn->GetSessionAdapter();
+    adapter->AddTapAnalyzer(std::move(analyzer));
+
+
+    std::printf("Analyzer added to %s\n", conn->GetUID().Base62().c_str());
+}
+
+} // namespace btest::plugin::Demo_TapAnalyzer

testing/btest/plugins/tap-analyzer-plugin/src/Plugin.h

@@ -0,0 +1,18 @@
+
+#pragma once
+
+#include "zeek/plugin/Plugin.h"
+
+namespace btest::plugin::Demo_TapAnalyzer {
+
+class Plugin : public zeek::plugin::Plugin {
+protected:
+    void HookSetupAnalyzerTree(zeek::Connection* conn) override;
+
+    // Overridden from zeek::plugin::Plugin.
+    zeek::plugin::Configuration Configure() override;
+};
+
+extern Plugin plugin;
+
+} // namespace btest::plugin::Demo_TapAnalyzer

testing/btest/plugins/tap-analyzer.zeek

@@ -0,0 +1,28 @@
+# @TEST-DOC: A plugin hooking HookSetupAnalyzerTree() to attach a TapAnalyzer to every connection.
+#
+# @TEST-EXEC: ${DIST}/auxil/zeek-aux/plugin-support/init-plugin -u . Demo TapAnalyzer
+# @TEST-EXEC: cp -r %DIR/tap-analyzer-plugin/* .
+# @TEST-EXEC: ./configure --zeek-dist=${DIST} && make
+#
+# @TEST-EXEC: ZEEK_PLUGIN_ACTIVATE="Demo::TapAnalyzer" ZEEK_PLUGIN_PATH=`pwd` zeek -b -r $TRACES/chksums/ip4-tcp-bad-chksum.pcap %INPUT >>output
+# @TEST-EXEC: ZEEK_PLUGIN_ACTIVATE="Demo::TapAnalyzer" ZEEK_PLUGIN_PATH=`pwd` zeek -b -r $TRACES/chksums/ip4-tcp-good-chksum.pcap %INPUT >>output
+# @TEST-EXEC: ZEEK_PLUGIN_ACTIVATE="Demo::TapAnalyzer" ZEEK_PLUGIN_PATH=`pwd` zeek -b -r $TRACES/chksums/ip4-udp-bad-chksum.pcap %INPUT >>output
+# @TEST-EXEC: ZEEK_PLUGIN_ACTIVATE="Demo::TapAnalyzer" ZEEK_PLUGIN_PATH=`pwd` zeek -b -r $TRACES/chksums/ip4-udp-good-chksum.pcap %INPUT >>output
+# @TEST-EXEC: ZEEK_PLUGIN_ACTIVATE="Demo::TapAnalyzer" ZEEK_PLUGIN_PATH=`pwd` zeek -b -r $TRACES/chksums/ip4-icmp-bad-chksum.pcap %INPUT >>output
+# @TEST-EXEC: ZEEK_PLUGIN_ACTIVATE="Demo::TapAnalyzer" ZEEK_PLUGIN_PATH=`pwd` zeek -b -r $TRACES/chksums/ip4-icmp-good-chksum.pcap %INPUT >>output
+# @TEST-EXEC: ZEEK_PLUGIN_ACTIVATE="Demo::TapAnalyzer" ZEEK_PLUGIN_PATH=`pwd` zeek -b -r $TRACES/chksums/ip6-icmp6-bad-chksum.pcap %INPUT >>output
+# @TEST-EXEC: ZEEK_PLUGIN_ACTIVATE="Demo::TapAnalyzer" ZEEK_PLUGIN_PATH=`pwd` zeek -b -r $TRACES/chksums/ip6-icmp6-good-chksum.pcap %INPUT >>output
+#
+# @TEST-EXEC: ZEEK_PLUGIN_ACTIVATE="Demo::TapAnalyzer" ZEEK_PLUGIN_PATH=`pwd` zeek -b -r $TRACES/http/get.trace %INPUT >>output
+#
+# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff output
+
+event zeek_init()
+	{
+	print packet_source()$path;
+	}
+
+event zeek_done()
+	{
+	print "===";
+	}