From d19fdfd17c4571b28182cccef9ef01512202146e Mon Sep 17 00:00:00 2001 From: Christian Kreibich Date: Tue, 10 Jun 2025 16:02:09 -0700 Subject: [PATCH 01/10] Add new ConnKey abstraction. --- src/CMakeLists.txt | 1 + src/ConnKey.h | 89 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 90 insertions(+) create mode 100644 src/ConnKey.h diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 31ae4fd073..3340139975 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -328,6 +328,7 @@ set(MAIN_SRCS CCL.cc CompHash.cc Conn.cc + ConnKey.h DFA.cc DbgBreakpoint.cc DbgHelp.cc diff --git a/src/ConnKey.h b/src/ConnKey.h new file mode 100644 index 0000000000..ec10104528 --- /dev/null +++ b/src/ConnKey.h @@ -0,0 +1,89 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#pragma once + +#include + +#include "zeek/IntrusivePtr.h" +#include "zeek/session/Key.h" + +namespace zeek { + +class Packet; + +class RecordVal; +using RecordValPtr = zeek::IntrusivePtr; + +/** + * Abstract ConnKey, for any type of connection. + */ +class ConnKey { +public: + virtual ~ConnKey() = default; + + /** + * Initialization of this key with the current packet. + * + * @param pkt The packet that's currently being processed. + */ + void Init(const Packet& pkt) { DoInit(pkt); } + + /** + * When Zeek renders a connection into a script-layer record, it calls this + * method to populate custom conn_id fields unique to this ConnKey, such as + * VLAN fields. This only needs to populate in fields in addition to Zeek's + * five-tuple (i.e., complete the record, not populate all of it). + * + * The default implementation does nothing. + * + * @param conn_id The conn_id record to populate. + */ + void PopulateConnIdVal(RecordVal& conn_id) { DoPopulateConnIdVal(conn_id); }; + + /** + * Return a non-owning session::detail::Key instance for connection lookups. + * + * Callers that need more than just a view of the key should copy the data. + * Callers are not supposed to hold on to the returned Key for longer than + * the ConnKey instance exists. + * + * @return A zeek::session::detail::Key + */ + zeek::session::detail::Key SessionKey() const { return DoSessionKey(); } + +protected: + /** + * Hook method for ConnKey::Init. + * + * Note that a given ConnKey instance may be re-used for different + * packets if it wasn't consumed to create a new connection. Therefore, + * implementers of this method are required to always set all fields + * that will affect the SessionKey result within DoInit anew. + * + * This a bit of an optimization done in the packet path that's shining + * through here. Rather than introducing a dedicated Reset method, + * implementers are asked to reset the key at initialization time + * which they most likely would do anyhow. + * + * @param pkt The packet that's currently being processed. + */ + virtual void DoInit(const Packet& pkt) {}; + + /** + * Hook method for ConnKey::PopulateConnIdVal. + * + * The default implementation does nothing. + */ + virtual void DoPopulateConnIdVal(RecordVal& conn_id) {} + + /** + * Hook method for implementing ConnKey::SessionKey. + * + * @return A zeek::session::detail::Key + */ + virtual session::detail::Key DoSessionKey() const = 0; +}; + +using ConnKeyPtr = std::unique_ptr; + +} // namespace zeek From 0c64f6a7b9f1020dee471bec029cda92eae627c0 Mon Sep 17 00:00:00 2001 From: Christian Kreibich Date: Wed, 9 Apr 2025 18:49:25 -0700 Subject: [PATCH 02/10] Establish plugin infrastructure for ConnKey factories. ConnKey factories are intermediaries that encapsulate the details of how to instantiate ConnKeys, which codify the hash input for connection lookups. --- scripts/base/init-bare.zeek | 13 ++++++++ src/CMakeLists.txt | 1 + src/conn_key/CMakeLists.txt | 1 + src/conn_key/Component.cc | 27 ++++++++++++++++ src/conn_key/Component.h | 45 +++++++++++++++++++++++++++ src/conn_key/Factory.h | 62 +++++++++++++++++++++++++++++++++++++ src/conn_key/Manager.cc | 39 +++++++++++++++++++++++ src/conn_key/Manager.h | 57 ++++++++++++++++++++++++++++++++++ src/plugin/Component.cc | 2 ++ src/plugin/Component.h | 1 + src/zeek-setup.cc | 5 +++ src/zeekygen/ScriptInfo.cc | 4 +++ 12 files changed, 257 insertions(+) create mode 100644 src/conn_key/CMakeLists.txt create mode 100644 src/conn_key/Component.cc create mode 100644 src/conn_key/Component.h create mode 100644 src/conn_key/Factory.h create mode 100644 src/conn_key/Manager.cc create mode 100644 src/conn_key/Manager.h diff --git a/scripts/base/init-bare.zeek b/scripts/base/init-bare.zeek index 712b3d6eb6..cbe4b6fb8b 100644 --- a/scripts/base/init-bare.zeek +++ b/scripts/base/init-bare.zeek @@ -629,6 +629,19 @@ export { const add_missing_remote_network_timestamp: bool = F &redef; } +module ConnKey; + +export { + ## The connection key factory to use for Zeek's internal connection + ## tracking. This is a ``ConnKey::Tag`` plugin component enum value, + ## and the default is Zeek's traditional 5-tuple-tracking based on + ## IP/port endpoint pairs, plus transport protocol. Plugins can provide + ## their own implementation. You'll usually not adjust this value in + ## isolation, but with a corresponding redef of the :zeek:type:`conn_id` + ## record to represent additional connection tuple members. + const factory = ConnKey::CONNKEY_FIVETUPLE &redef; +} + module FTP; export { diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 3340139975..bfceb7a671 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -197,6 +197,7 @@ gen_zam_target(${GEN_ZAM_SRC_DIR}) option(USE_SQLITE "Should Zeek use SQLite?" ON) add_subdirectory(analyzer) +add_subdirectory(conn_key) add_subdirectory(cluster) add_subdirectory(packet_analysis) add_subdirectory(broker) diff --git a/src/conn_key/CMakeLists.txt b/src/conn_key/CMakeLists.txt new file mode 100644 index 0000000000..33d10993ef --- /dev/null +++ b/src/conn_key/CMakeLists.txt @@ -0,0 +1 @@ +zeek_add_subdir_library(connkey SOURCES Factory.h Component.cc Manager.cc) diff --git a/src/conn_key/Component.cc b/src/conn_key/Component.cc new file mode 100644 index 0000000000..ae49e95f28 --- /dev/null +++ b/src/conn_key/Component.cc @@ -0,0 +1,27 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#include "zeek/conn_key/Component.h" + +#include "zeek/Desc.h" +#include "zeek/conn_key/Manager.h" + +using namespace zeek::conn_key; + +Component::Component(const std::string& name, factory_callback arg_factory, Tag::subtype_t arg_subtype) + : plugin::Component(plugin::component::CONNKEY, name, arg_subtype, conn_key_mgr->GetTagType()), + factory(std::move(arg_factory)) {} + +void Component::Initialize() { + InitializeTag(); + conn_key_mgr->RegisterComponent(this, "CONNKEY_"); +} + +void Component::DoDescribe(ODesc* d) const { + if ( factory ) { + d->Add("CONNKEY_"); + d->Add(CanonicalName()); + d->Add(", "); + } + + d->Add(Enabled() ? "enabled" : "disabled"); +} diff --git a/src/conn_key/Component.h b/src/conn_key/Component.h new file mode 100644 index 0000000000..c13f76e53e --- /dev/null +++ b/src/conn_key/Component.h @@ -0,0 +1,45 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#pragma once + +#include +#include + +#include "zeek/Tag.h" +#include "zeek/plugin/Component.h" + +namespace zeek::conn_key { + +class Factory; +using FactoryPtr = std::unique_ptr; + +class Component : public plugin::Component { +public: + using factory_callback = std::function; + + Component(const std::string& name, factory_callback factory, zeek::Tag::subtype_t subtype = 0); + ~Component() override = default; + + /** + * Initialization function. This function has to be called before any + * plugin component functionality is used; it is used to add the + * plugin component to the list of components and to initialize tags + */ + void Initialize() override; + + /** + * Returns the analyzer's factory function. + */ + factory_callback Factory() const { return factory; } + +protected: + /** + * Overridden from plugin::Component. + */ + void DoDescribe(ODesc* d) const override; + +private: + factory_callback factory; // The tuple factory's factory callback. +}; + +} // namespace zeek::conn_key diff --git a/src/conn_key/Factory.h b/src/conn_key/Factory.h new file mode 100644 index 0000000000..9eec55462c --- /dev/null +++ b/src/conn_key/Factory.h @@ -0,0 +1,62 @@ +// See the file "COPYING" in the main distribution directory for copyright. +#pragma once + +#include "zeek/ConnKey.h" +#include "zeek/util-types.h" + +namespace zeek { + +class Packet; +class RecordVal; +using RecordValPtr = IntrusivePtr; + +namespace conn_key { + +class Factory; +using FactoryPtr = std::unique_ptr; + +/** + * ConnKey factories instantiate derivatives of ConnKeys, to provide pluggable flow hashing. + */ +class Factory { +public: + virtual ~Factory() = default; + + /** + * Instantiates a clean ConnKey derivative and returns it. + * + * @return A unique pointer to the ConnKey instance. + */ + zeek::ConnKeyPtr NewConnKey() const { return DoNewConnKey(); } + + /** + * Instantiates a filled-in ConnKey derivative from a script-layer + * record, usually a conn_id instance. Implementations are free to + * implement this liberally, i.e. the input does not _have_ to be a + * conn_id. + * + * @param v The script-layer value providing key input. + * @return A unique pointer to the ConnKey instance, or an error message. + */ + zeek::expected ConnKeyFromVal(const zeek::Val& v) const { + return DoConnKeyFromVal(v); + } + +protected: + /** + * Hook for Factory::NewConnKey. + * + * @return A unique pointer to the ConnKey instance. + */ + virtual zeek::ConnKeyPtr DoNewConnKey() const = 0; + + /** + * Hook for Factory::ConnKeyFromVal. + * + * @return A unique pointer to the ConnKey instance, or an error message. + */ + virtual zeek::expected DoConnKeyFromVal(const zeek::Val& v) const = 0; +}; + +} // namespace conn_key +} // namespace zeek diff --git a/src/conn_key/Manager.cc b/src/conn_key/Manager.cc new file mode 100644 index 0000000000..8daeff57c8 --- /dev/null +++ b/src/conn_key/Manager.cc @@ -0,0 +1,39 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#include "zeek/conn_key/Manager.h" + +#include "zeek/conn_key/Component.h" + +using namespace zeek::conn_key; + +Manager::Manager() : plugin::ComponentManager("ConnKey", "Tag") {} + +void Manager::InitPostScript() { + const auto& factory_val = id::find_val("ConnKey::factory"); + factory = InstantiateFactory(factory_val); +} + +FactoryPtr Manager::InstantiateFactory(const zeek::EnumValPtr& tag) { + Component* c = Lookup(tag); + + if ( ! c ) { + reporter->FatalError( + "request to instantiate unknown connection tuple factory %s, please review ConnTuple::factory value", + tag->GetType()->AsEnumType()->Lookup(tag->Get())); + } + + if ( ! c->Factory() ) { + reporter->FatalError("factory %s cannot be instantiated dynamically", GetComponentName(tag).c_str()); + } + + FactoryPtr factory = c->Factory()(); + + if ( ! factory ) { + reporter->FatalError("factory instantiation failed"); + } + + // Could add validation of actual tag vs obtained one here, as we do e.g. in + // the packet_analysis Manager. + + return factory; +} diff --git a/src/conn_key/Manager.h b/src/conn_key/Manager.h new file mode 100644 index 0000000000..aeb49add5e --- /dev/null +++ b/src/conn_key/Manager.h @@ -0,0 +1,57 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#pragma once + +#include "zeek/Tag.h" +#include "zeek/conn_key/Component.h" +#include "zeek/conn_key/Factory.h" +#include "zeek/plugin/Component.h" +#include "zeek/plugin/ComponentManager.h" + +namespace zeek { + +namespace conn_key { + +/** + * This component manager is for registration of pluggable ConnKey factories + * that provide a zeek::plugin::component::CONNKEY component. + */ +class Manager : public plugin::ComponentManager { +public: + /** + * Constructor. + */ + Manager(); + + /** + * Destructor. + */ + ~Manager() = default; + + /** + * Hook called during Zeek's startup sequence at InitPostScript() time. + */ + void InitPostScript(); + + /** + * Return the instantiated Factory selected by the @c ConnKey::factory script-level variable. + * + * @return A reference to the selected see Factory. + */ + Factory& GetFactory() { return *factory; } + +private: + /** + * @return A pointer to a Factory given @arg tag. + */ + FactoryPtr InstantiateFactory(const EnumValPtr& tag); + + FactoryPtr factory; +}; + +} // namespace conn_key + +extern zeek::conn_key::Manager* conn_key_mgr; + + +} // namespace zeek diff --git a/src/plugin/Component.cc b/src/plugin/Component.cc index 6160484dcf..7d755fc803 100644 --- a/src/plugin/Component.cc +++ b/src/plugin/Component.cc @@ -52,6 +52,8 @@ void Component::Describe(ODesc* d) const { case component::STORAGE_SERIALIZER: d->Add("Storage Serializer"); break; + case component::CONNKEY: d->Add("ConnKey Factory"); break; + default: reporter->InternalWarning("unknown component type in plugin::Component::Describe"); d->Add(""); diff --git a/src/plugin/Component.h b/src/plugin/Component.h index 3dd07fd03c..de2e167181 100644 --- a/src/plugin/Component.h +++ b/src/plugin/Component.h @@ -38,6 +38,7 @@ enum Type : uint8_t { LOG_SERIALIZER, /// A serializer for log batches, used by cluster backends. STORAGE_BACKEND, /// A backend for the storage framework. STORAGE_SERIALIZER, /// A serializer for the storage framework. + CONNKEY, /// A factory for connection keys. }; } // namespace component diff --git a/src/zeek-setup.cc b/src/zeek-setup.cc index 5d42b27f38..d3c6000f84 100644 --- a/src/zeek-setup.cc +++ b/src/zeek-setup.cc @@ -54,6 +54,7 @@ #include "zeek/broker/Manager.h" #include "zeek/cluster/Backend.h" #include "zeek/cluster/Manager.h" +#include "zeek/conn_key/Manager.h" #include "zeek/file_analysis/Manager.h" #include "zeek/input.h" #include "zeek/input/Manager.h" @@ -161,6 +162,7 @@ void do_ssl_deinit() { zeek::ValManager* zeek::val_mgr = nullptr; zeek::packet_analysis::Manager* zeek::packet_mgr = nullptr; +zeek::conn_key::Manager* zeek::conn_key_mgr = nullptr; zeek::analyzer::Manager* zeek::analyzer_mgr = nullptr; zeek::plugin::Manager* zeek::plugin_mgr = nullptr; @@ -408,6 +410,7 @@ static void terminate_zeek() { delete zeekygen_mgr; delete packet_mgr; + delete conn_key_mgr; delete analyzer_mgr; delete file_mgr; delete cluster::manager; @@ -690,6 +693,7 @@ SetupResult setup(int argc, char** argv, Options* zopts) { iosource_mgr = new iosource::Manager(); event_registry = new EventRegistry(); packet_mgr = new packet_analysis::Manager(); + conn_key_mgr = new conn_key::Manager(); analyzer_mgr = new analyzer::Manager(); log_mgr = new logging::Manager(); input_mgr = new input::Manager(); @@ -838,6 +842,7 @@ SetupResult setup(int argc, char** argv, Options* zopts) { RecordType::InitPostScript(); + conn_key_mgr->InitPostScript(); telemetry_mgr->InitPostScript(); thread_mgr->InitPostScript(); iosource_mgr->InitPostScript(); diff --git a/src/zeekygen/ScriptInfo.cc b/src/zeekygen/ScriptInfo.cc index 103c7e273a..3b0dfbf4d2 100644 --- a/src/zeekygen/ScriptInfo.cc +++ b/src/zeekygen/ScriptInfo.cc @@ -309,6 +309,10 @@ void ScriptInfo::DoInitPostScript() { const auto& id = zeek::detail::global_scope()->Find("Input::Reader"); types.push_back(new IdentifierInfo(id, this)); } + else if ( name == "base/init-bare.zeek" ) { + const auto& id = zeek::detail::global_scope()->Find("ConnKey::Tag"); + types.push_back(new IdentifierInfo(id, this)); + } else if ( name == "base/frameworks/logging/main.zeek" ) { const auto& id = zeek::detail::global_scope()->Find("Log::Writer"); types.push_back(new IdentifierInfo(id, this)); From 5af8fc242aa99963fe62fca831ebab2425d8f9cf Mon Sep 17 00:00:00 2001 From: Christian Kreibich Date: Wed, 11 Jun 2025 13:20:42 -0700 Subject: [PATCH 03/10] Add IP-specific ConnKey implementation. The InitTuple() implementation here is a placeholder for a fuller one following later, when we do away with the need for ConnTuple. --- .../protocol/ip/CMakeLists.txt | 2 + .../protocol/ip/conn_key/CMakeLists.txt | 1 + .../protocol/ip/conn_key/IPBasedConnKey.cc | 3 + .../protocol/ip/conn_key/IPBasedConnKey.h | 123 ++++++++++++++++++ 4 files changed, 129 insertions(+) create mode 100644 src/packet_analysis/protocol/ip/conn_key/CMakeLists.txt create mode 100644 src/packet_analysis/protocol/ip/conn_key/IPBasedConnKey.cc create mode 100644 src/packet_analysis/protocol/ip/conn_key/IPBasedConnKey.h diff --git a/src/packet_analysis/protocol/ip/CMakeLists.txt b/src/packet_analysis/protocol/ip/CMakeLists.txt index 7e6d6ff8b4..fdefb433d0 100644 --- a/src/packet_analysis/protocol/ip/CMakeLists.txt +++ b/src/packet_analysis/protocol/ip/CMakeLists.txt @@ -1,3 +1,5 @@ zeek_add_plugin( PacketAnalyzer IP SOURCES IP.cc IPBasedAnalyzer.cc SessionAdapter.cc Plugin.cc) + +add_subdirectory(conn_key) diff --git a/src/packet_analysis/protocol/ip/conn_key/CMakeLists.txt b/src/packet_analysis/protocol/ip/conn_key/CMakeLists.txt new file mode 100644 index 0000000000..e3736f23ee --- /dev/null +++ b/src/packet_analysis/protocol/ip/conn_key/CMakeLists.txt @@ -0,0 +1 @@ +zeek_add_subdir_library(connkey-ip SOURCES IPBasedConnKey.cc) diff --git a/src/packet_analysis/protocol/ip/conn_key/IPBasedConnKey.cc b/src/packet_analysis/protocol/ip/conn_key/IPBasedConnKey.cc new file mode 100644 index 0000000000..ca4ac01e5e --- /dev/null +++ b/src/packet_analysis/protocol/ip/conn_key/IPBasedConnKey.cc @@ -0,0 +1,3 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#include "zeek/packet_analysis/protocol/ip/conn_key/IPBasedConnKey.h" diff --git a/src/packet_analysis/protocol/ip/conn_key/IPBasedConnKey.h b/src/packet_analysis/protocol/ip/conn_key/IPBasedConnKey.h new file mode 100644 index 0000000000..4202b3420d --- /dev/null +++ b/src/packet_analysis/protocol/ip/conn_key/IPBasedConnKey.h @@ -0,0 +1,123 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#pragma once + +#include + +#include "zeek/Conn.h" +#include "zeek/ConnKey.h" +#include "zeek/IPAddr.h" + +namespace zeek { + +namespace detail { + +/** + * Struct for embedding into an IPBasedConnKey. + */ +struct PackedConnTuple { + in6_addr ip1; + in6_addr ip2; + uint16_t port1 = 0; + uint16_t port2 = 0; + uint16_t proto = 0; +} __attribute__((packed, aligned)); + +} // namespace detail + +/** + * Abstract key class for IP-based connections. + * + * ConnKey instances for IP always hold a ConnTuple instance which is provided + * by the IPBasedAnalyzer. The InitConnTuple() method stores a normalized version + * in the tuple, losing the information about orig and responder. + */ +class IPBasedConnKey : public zeek::ConnKey { +public: + void InitTuple(const ConnTuple& ct) { InitPackedTuple(ct); } + + /** + * Return a modifiable reference to the embedded PackedConnTuple. + * + * This is virtual to give subclasses control over where + * to place the tuple within the key. + * + * @return A modifiable reference to the embedded PackedConnTuple. + */ + virtual detail::PackedConnTuple& PackedTuple() = 0; + + /** + * Return a non-modifiable reference to the embedded PackedConnTuple. + * + * This is virtual to give subclasses control over where + * to place the tuple within the key. + * + * @return A non-modifiable reference to the embedded PackedConnTuple. + */ + virtual const detail::PackedConnTuple& PackedTuple() const = 0; + +private: + /** + * Initialize a packed tuple from a ConnTuple instance. + */ + void InitPackedTuple(const ConnTuple& ct) { + auto& tuple = PackedTuple(); + + if ( ct.is_one_way || addr_port_canon_lt(ct.src_addr, ct.src_port, ct.dst_addr, ct.dst_port) ) { + ct.src_addr.CopyIPv6(&tuple.ip1); + ct.dst_addr.CopyIPv6(&tuple.ip2); + tuple.port1 = ct.src_port; + tuple.port2 = ct.dst_port; + } + else { + ct.dst_addr.CopyIPv6(&tuple.ip1); + ct.src_addr.CopyIPv6(&tuple.ip2); + tuple.port1 = ct.dst_port; + tuple.port2 = ct.src_port; + } + + tuple.proto = ct.proto; + } +}; + +using IPBasedConnKeyPtr = std::unique_ptr; + +/** + * The usual 5-tuple ConnKey, fully instantiable. + */ +class IPConnKey : public IPBasedConnKey { +public: + /** + * Constructor. + * + * Fill any holes in the key struct as we use the full tuple as a key. + */ + IPConnKey() { memset(static_cast(&key), 0, sizeof(key)); } + + /** + * @copydoc + */ + detail::PackedConnTuple& PackedTuple() override { return key.tuple; } + + /** + * @copydoc + */ + const detail::PackedConnTuple& PackedTuple() const override { return key.tuple; } + +protected: + /** + * @copydoc + */ + zeek::session::detail::Key DoSessionKey() const override { + return {reinterpret_cast(&key), sizeof(key), + // XXX: Not sure we need CONNECTION_KEY_TYPE? + session::detail::Key::CONNECTION_KEY_TYPE}; + } + +private: + struct { + struct detail::PackedConnTuple tuple; + } key; +}; + +} // namespace zeek From b8f82ff659e884ad093be68b0f314814df057ffb Mon Sep 17 00:00:00 2001 From: Christian Kreibich Date: Fri, 13 Jun 2025 12:50:16 -0700 Subject: [PATCH 04/10] Provide a connkey factory for Zeek's default five-tuples. Since the base factory is pure virtual this is now the first full implementation, but still a bit of a special case because it implements Zeek's default behavior and doesn't add "custom" content to the tuple. --- .../protocol/ip/conn_key/CMakeLists.txt | 2 + .../ip/conn_key/fivetuple/CMakeLists.txt | 3 + .../protocol/ip/conn_key/fivetuple/Factory.cc | 75 +++++++++++++++++++ .../protocol/ip/conn_key/fivetuple/Factory.h | 33 ++++++++ .../protocol/ip/conn_key/fivetuple/Plugin.cc | 24 ++++++ 5 files changed, 137 insertions(+) create mode 100644 src/packet_analysis/protocol/ip/conn_key/fivetuple/CMakeLists.txt create mode 100644 src/packet_analysis/protocol/ip/conn_key/fivetuple/Factory.cc create mode 100644 src/packet_analysis/protocol/ip/conn_key/fivetuple/Factory.h create mode 100644 src/packet_analysis/protocol/ip/conn_key/fivetuple/Plugin.cc diff --git a/src/packet_analysis/protocol/ip/conn_key/CMakeLists.txt b/src/packet_analysis/protocol/ip/conn_key/CMakeLists.txt index e3736f23ee..2a780c21df 100644 --- a/src/packet_analysis/protocol/ip/conn_key/CMakeLists.txt +++ b/src/packet_analysis/protocol/ip/conn_key/CMakeLists.txt @@ -1 +1,3 @@ zeek_add_subdir_library(connkey-ip SOURCES IPBasedConnKey.cc) + +add_subdirectory(fivetuple) diff --git a/src/packet_analysis/protocol/ip/conn_key/fivetuple/CMakeLists.txt b/src/packet_analysis/protocol/ip/conn_key/fivetuple/CMakeLists.txt new file mode 100644 index 0000000000..c243f41b01 --- /dev/null +++ b/src/packet_analysis/protocol/ip/conn_key/fivetuple/CMakeLists.txt @@ -0,0 +1,3 @@ +zeek_add_plugin( + Zeek ConnKey_Fivetuple + SOURCES Factory.cc Plugin.cc) diff --git a/src/packet_analysis/protocol/ip/conn_key/fivetuple/Factory.cc b/src/packet_analysis/protocol/ip/conn_key/fivetuple/Factory.cc new file mode 100644 index 0000000000..fbda9443d0 --- /dev/null +++ b/src/packet_analysis/protocol/ip/conn_key/fivetuple/Factory.cc @@ -0,0 +1,75 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#include "zeek/packet_analysis/protocol/ip/conn_key/fivetuple/Factory.h" + +#include "zeek/IP.h" +#include "zeek/Val.h" +#include "zeek/packet_analysis/protocol/ip/conn_key/IPBasedConnKey.h" +#include "zeek/util-types.h" + +namespace zeek::conn_key::fivetuple { + +zeek::ConnKeyPtr Factory::DoNewConnKey() const { return std::make_unique(); } + +zeek::expected Factory::DoConnKeyFromVal(const zeek::Val& v) const { + static auto unexpected_conn_id = zeek::unexpected{"invalid connection ID record encountered"}; + auto ck = NewConnKey(); + auto* ick = static_cast(ck.get()); + auto& pt = ick->PackedTuple(); + const auto& vt = v.GetType(); + + if ( ! IsRecord(vt->Tag()) ) + return unexpected_conn_id; + + auto* vr = vt->AsRecordType(); + auto vl = v.AsRecordVal(); + + // Indices into conn_id's record field value list: + int orig_h = 0, orig_p = 1, resp_h = 2, resp_p = 3, proto = 4; + + if ( vr != id::conn_id ) { + // While it's not a conn_id, it may have equivalent fields. + orig_h = vr->FieldOffset("orig_h"); + resp_h = vr->FieldOffset("resp_h"); + orig_p = vr->FieldOffset("orig_p"); + resp_p = vr->FieldOffset("resp_p"); + proto = vr->FieldOffset("proto"); + + // clang-format off + if ( orig_h < 0 || vr->GetFieldType(orig_h)->Tag() != TYPE_ADDR || + resp_h < 0 || vr->GetFieldType(resp_h)->Tag() != TYPE_ADDR || + orig_p < 0 || vr->GetFieldType(orig_p)->Tag() != TYPE_PORT || + resp_p < 0 || vr->GetFieldType(resp_p)->Tag() != TYPE_PORT || + proto < 0 || vr->GetFieldType(proto)->Tag() != TYPE_COUNT ) { + return unexpected_conn_id; + } + // clang-format on + } + + if ( ! vl->HasField(orig_h) || ! vl->HasField(resp_h) || ! vl->HasField(orig_p) || ! vl->HasField(resp_p) || + ! vl->HasField(proto) ) { + return unexpected_conn_id; + } + + const IPAddr& orig_addr = vl->GetFieldAs(orig_h); + const IPAddr& resp_addr = vl->GetFieldAs(resp_h); + + const auto& orig_portv = vl->GetFieldAs(orig_p); + const auto& resp_portv = vl->GetFieldAs(resp_p); + + const auto& protov = vl->GetField(proto); + auto proto16_t = static_cast(protov->AsCount()); + + if ( proto16_t == UNKNOWN_IP_PROTO ) + return zeek::unexpected( + "invalid connection ID record encountered: the proto field has the \"unknown\" 65535 value. " + "Did you forget to set it?"); + + ick->InitTuple(ConnTuple{orig_addr, resp_addr, htons(orig_portv->Port()), htons(resp_portv->Port()), proto16_t}); + + // Asserting here on the absence of errors can fail btests. + + return ck; +} + +} // namespace zeek::conn_key::fivetuple diff --git a/src/packet_analysis/protocol/ip/conn_key/fivetuple/Factory.h b/src/packet_analysis/protocol/ip/conn_key/fivetuple/Factory.h new file mode 100644 index 0000000000..f0ef6609a4 --- /dev/null +++ b/src/packet_analysis/protocol/ip/conn_key/fivetuple/Factory.h @@ -0,0 +1,33 @@ +// See the file "COPYING" in the main distribution directory for copyright. +#pragma once + +#include "zeek/ConnKey.h" +#include "zeek/conn_key/Factory.h" + +namespace zeek::conn_key::fivetuple { + +class Factory : public zeek::conn_key::Factory { +public: + static zeek::conn_key::FactoryPtr Instantiate() { return std::make_unique(); } + +protected: + /** + * Instantiates a clean ConnKey derivative and returns it. + * + * @return A unique pointer to the ConnKey instance. + */ + zeek::ConnKeyPtr DoNewConnKey() const override; + + /** + * Instantiates a filled-in ConnKey derivative from a script-layer + * value, usually a conn_id instance. Implementations are free to + * implement this liberally, i.e. the input does not _have_ to be a + * conn_id instance. + * + * @param v The script-layer value providing key input. + * @return A unique pointer to the ConnKey instance, or an error message. + */ + zeek::expected DoConnKeyFromVal(const zeek::Val& v) const override; +}; + +} // namespace zeek::conn_key::fivetuple diff --git a/src/packet_analysis/protocol/ip/conn_key/fivetuple/Plugin.cc b/src/packet_analysis/protocol/ip/conn_key/fivetuple/Plugin.cc new file mode 100644 index 0000000000..7e6d05a468 --- /dev/null +++ b/src/packet_analysis/protocol/ip/conn_key/fivetuple/Plugin.cc @@ -0,0 +1,24 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#include "zeek/plugin/Plugin.h" + +#include "zeek/conn_key/Component.h" +#include "zeek/packet_analysis/protocol/ip/conn_key/fivetuple/Factory.h" + +namespace zeek::plugin::Zeek_Conntuple_Fivetuple { + +class Plugin : public zeek::plugin::Plugin { +public: + zeek::plugin::Configuration Configure() override { + AddComponent(new conn_key::Component("Fivetuple", zeek::conn_key::fivetuple::Factory::Instantiate)); + + zeek::plugin::Configuration config; + config.name = "Zeek::ConnKey_Fivetuple"; + config.description = "ConnKey factory for Zeek's default IP/port/proto five-tuples"; + return config; + } +}; + +Plugin plugin; + +} // namespace zeek::plugin::Zeek_Conntuple_Fivetuple From 52d6228b06025ea1c4088707aeaeee5a254a1126 Mon Sep 17 00:00:00 2001 From: Christian Kreibich Date: Thu, 12 Jun 2025 12:59:45 -0700 Subject: [PATCH 05/10] Switch to virtualized use of new zeek::ConnKey class tree This touches quite a few places, but each just swaps out existing APIs and/or zeek::detail::ConnKey instances. --- src/Conn.cc | 68 +++++++++++++++---- src/Conn.h | 22 ++++-- src/analyzer/Analyzer.cc | 11 +-- src/analyzer/protocol/smtp/BDAT.cc | 6 +- src/packet_analysis/protocol/gtpv1/GTPv1.cc | 10 ++- src/packet_analysis/protocol/gtpv1/GTPv1.h | 6 +- .../protocol/gtpv1/functions.bif | 9 ++- .../protocol/ip/IPBasedAnalyzer.cc | 40 ++++++++--- .../protocol/ip/IPBasedAnalyzer.h | 3 +- src/packet_analysis/protocol/teredo/Teredo.cc | 10 ++- src/packet_analysis/protocol/teredo/Teredo.h | 5 +- .../protocol/teredo/functions.bif | 9 ++- src/session/Manager.cc | 21 +++--- src/session/Manager.h | 3 +- 14 files changed, 158 insertions(+), 65 deletions(-) diff --git a/src/Conn.cc b/src/Conn.cc index c00e86b337..97a2564d1a 100644 --- a/src/Conn.cc +++ b/src/Conn.cc @@ -23,8 +23,27 @@ namespace zeek { uint64_t Connection::total_connections = 0; uint64_t Connection::current_connections = 0; +Connection::Connection(zeek::IPBasedConnKeyPtr k, const zeek::ConnTuple& ct, double t, uint32_t flow, const Packet* pkt) + : Session(t, connection_timeout, connection_status_update, detail::connection_status_update_interval), + key(std::move(k)) { + orig_addr = ct.src_addr; + resp_addr = ct.dst_addr; + orig_port = ct.src_port; + resp_port = ct.dst_port; + + switch ( ct.proto ) { + case IPPROTO_TCP: proto = TRANSPORT_TCP; break; + case IPPROTO_UDP: proto = TRANSPORT_UDP; break; + case IPPROTO_ICMP: + case IPPROTO_ICMPV6: proto = TRANSPORT_ICMP; break; + default: proto = TRANSPORT_UNKNOWN; break; + } + + Init(flow, pkt); +} + Connection::Connection(const detail::ConnKey& k, double t, const ConnTuple* id, uint32_t flow, const Packet* pkt) - : Session(t, connection_timeout, connection_status_update, detail::connection_status_update_interval), key(k) { + : Session(t, connection_timeout, connection_status_update, detail::connection_status_update_interval) { orig_addr = id->src_addr; resp_addr = id->dst_addr; orig_port = id->src_port; @@ -38,6 +57,28 @@ Connection::Connection(const detail::ConnKey& k, double t, const ConnTuple* id, default: proto = TRANSPORT_UNKNOWN; break; } + key = std::make_unique(); + key->InitTuple(*id); + key->Init(*pkt); + + Init(flow, pkt); +} + +Connection::~Connection() { + if ( ! finished ) + reporter->InternalError("Done() not called before destruction of Connection"); + + CancelTimers(); + + if ( conn_val ) + conn_val->SetOrigin(nullptr); + + delete adapter; + + --current_connections; +} + +void Connection::Init(uint32_t flow, const Packet* pkt) { orig_flow_label = flow; resp_flow_label = 0; saw_first_orig_packet = 1; @@ -71,20 +112,6 @@ Connection::Connection(const detail::ConnKey& k, double t, const ConnTuple* id, encapsulation = pkt->encap; } -Connection::~Connection() { - if ( ! finished ) - reporter->InternalError("Done() not called before destruction of Connection"); - - CancelTimers(); - - if ( conn_val ) - conn_val->SetOrigin(nullptr); - - delete adapter; - - --current_connections; -} - void Connection::CheckEncapsulation(const std::shared_ptr& arg_encap) { if ( encapsulation && arg_encap ) { if ( *encapsulation != *arg_encap ) { @@ -157,6 +184,13 @@ void Connection::NextPacket(double t, bool is_orig, const IP_Hdr* ip, int len, i run_state::current_pkt = nullptr; } + +const ConnKey& Connection::Key() const { return *key; } + +session::detail::Key Connection::SessionKey(bool copy) const { return key->SessionKey(); } + +uint8_t Connection::KeyProto() const { return key->PackedTuple().proto; } + bool Connection::IsReuse(double t, const u_char* pkt) { return adapter && adapter->IsReuse(t, pkt); } namespace { @@ -186,6 +220,7 @@ const RecordValPtr& Connection::GetVal() { TransportProto prot_type = ConnTransport(); + // XXX this could technically move into IPBasedConnKey. auto id_val = make_intrusive(id::conn_id); id_val->Assign(0, make_intrusive(orig_addr)); id_val->Assign(1, val_mgr->Port(ntohs(orig_port), prot_type)); @@ -193,6 +228,9 @@ const RecordValPtr& Connection::GetVal() { id_val->Assign(3, val_mgr->Port(ntohs(resp_port), prot_type)); id_val->Assign(4, KeyProto()); + // Allow customized ConnKeys to augment the conn_id: + key->PopulateConnIdVal(*id_val); + auto orig_endp = make_intrusive(id::endpoint); orig_endp->Assign(0, 0); orig_endp->Assign(1, 0); diff --git a/src/Conn.h b/src/Conn.h index 95883b9b88..ca634a821f 100644 --- a/src/Conn.h +++ b/src/Conn.h @@ -5,6 +5,7 @@ #include #include +#include "zeek/ConnKey.h" #include "zeek/IPAddr.h" #include "zeek/IntrusivePtr.h" #include "zeek/Rule.h" @@ -27,6 +28,9 @@ class RecordVal; using ValPtr = IntrusivePtr; using RecordValPtr = IntrusivePtr; +class IPBasedConnKey; +using IPBasedConnKeyPtr = std::unique_ptr; + namespace detail { class Specific_RE_Matcher; @@ -64,6 +68,7 @@ static inline int addr_port_canon_lt(const IPAddr& addr1, uint32_t p1, const IPA class Connection final : public session::Session { public: + Connection(zeek::IPBasedConnKeyPtr k, const zeek::ConnTuple& ct, double t, uint32_t flow, const Packet* pkt); Connection(const detail::ConnKey& k, double t, const ConnTuple* id, uint32_t flow, const Packet* pkt); ~Connection() override; @@ -101,10 +106,12 @@ public: // Keys are only considered valid for a connection when a // connection is in the session map. If it is removed, the key // should be marked invalid. - const detail::ConnKey& Key() const { return key; } - session::detail::Key SessionKey(bool copy) const override { - return session::detail::Key{&key, sizeof(key), session::detail::Key::CONNECTION_KEY_TYPE, copy}; - } + // + // These touch the key, which we forward-declared above. Therefore this + // hides the implementation, which has the full class definition. + const ConnKey& Key() const; + session::detail::Key SessionKey(bool copy) const override; + uint8_t KeyProto() const; const IPAddr& OrigAddr() const { return orig_addr; } const IPAddr& RespAddr() const { return resp_addr; } @@ -130,8 +137,6 @@ public: return "unknown"; } - uint8_t KeyProto() const { return key.transport; } - // Returns true if the packet reflects a reuse of this // connection (i.e., not a continuation but the beginning of // a new connection). @@ -196,6 +201,9 @@ public: bool IsFinished() { return finished; } private: + // Common initialization for the constructors. + void Init(uint32_t flow, const Packet* pkt); + friend class session::detail::Timer; IPAddr orig_addr; @@ -211,7 +219,7 @@ private: std::shared_ptr encapsulation; // tunnels uint8_t tunnel_changes = 0; - detail::ConnKey key; + IPBasedConnKeyPtr key; unsigned int weird : 1; unsigned int finished : 1; diff --git a/src/analyzer/Analyzer.cc b/src/analyzer/Analyzer.cc index f1ef9ed459..fc73a468f7 100644 --- a/src/analyzer/Analyzer.cc +++ b/src/analyzer/Analyzer.cc @@ -8,6 +8,7 @@ #include "zeek/Conn.h" #include "zeek/Event.h" #include "zeek/analyzer/Manager.h" +#include "zeek/packet_analysis/protocol/ip/conn_key/IPBasedConnKey.h" #include "zeek/packet_analysis/protocol/tcp/TCPSessionAdapter.h" #include "zeek/3rdparty/doctest.h" @@ -806,8 +807,9 @@ TEST_SUITE("Analyzer management") { REQUIRE(zeek::analyzer_mgr); zeek::Packet p; - zeek::ConnTuple t; - auto conn = std::make_unique(zeek::detail::ConnKey(t), 0, &t, 0, &p); + zeek::ConnTuple ct; + zeek::IPBasedConnKeyPtr kp = std::make_unique(); + auto conn = std::make_unique(std::move(kp), ct, 0, 0, &p); auto* tcp = new zeek::packet_analysis::TCP::TCPSessionAdapter(conn.get()); conn->SetSessionAdapter(tcp, nullptr); @@ -838,8 +840,9 @@ TEST_SUITE("Analyzer management") { REQUIRE(zeek::analyzer_mgr); zeek::Packet p; - zeek::ConnTuple t; - auto conn = std::make_unique(zeek::detail::ConnKey(t), 0, &t, 0, &p); + zeek::ConnTuple ct; + zeek::IPBasedConnKeyPtr kp = std::make_unique(); + auto conn = std::make_unique(std::move(kp), ct, 0, 0, &p); auto ssh = zeek::analyzer_mgr->InstantiateAnalyzer("SSH", conn.get()); REQUIRE(ssh); diff --git a/src/analyzer/protocol/smtp/BDAT.cc b/src/analyzer/protocol/smtp/BDAT.cc index f83d011592..56fe42f4bb 100644 --- a/src/analyzer/protocol/smtp/BDAT.cc +++ b/src/analyzer/protocol/smtp/BDAT.cc @@ -5,6 +5,7 @@ #include "zeek/Conn.h" #include "zeek/DebugLogger.h" #include "zeek/analyzer/protocol/mime/MIME.h" +#include "zeek/packet_analysis/protocol/ip/conn_key/IPBasedConnKey.h" #include "zeek/util.h" #include "zeek/3rdparty/doctest.h" @@ -327,8 +328,9 @@ private: TEST_CASE("line forward testing") { zeek::Packet p; - zeek::ConnTuple t; - auto conn = std::make_unique(zeek::detail::ConnKey(t), 0, &t, 0, &p); + zeek::ConnTuple ct; + zeek::IPBasedConnKeyPtr kp = std::make_unique(); + auto conn = std::make_unique(std::move(kp), ct, 0, 0, &p); auto smtp_analyzer = std::unique_ptr(zeek::analyzer_mgr->InstantiateAnalyzer("SMTP", conn.get())); auto mail = std::make_unique(smtp_analyzer.get()); diff --git a/src/packet_analysis/protocol/gtpv1/GTPv1.cc b/src/packet_analysis/protocol/gtpv1/GTPv1.cc index 9dbfccdb80..547e145ba3 100644 --- a/src/packet_analysis/protocol/gtpv1/GTPv1.cc +++ b/src/packet_analysis/protocol/gtpv1/GTPv1.cc @@ -19,11 +19,15 @@ bool GTPv1_Analyzer::AnalyzePacket(size_t len, const uint8_t* data, Packet* pack } auto conn = static_cast(packet->session); - zeek::detail::ConnKey conn_key = conn->Key(); + const auto& key = conn->Key(); + auto sk = key.SessionKey(); - auto cm_it = conn_map.find(conn_key); + auto cm_it = conn_map.find(sk); if ( cm_it == conn_map.end() ) { - cm_it = conn_map.insert(cm_it, {conn_key, std::make_unique(this)}); + sk.CopyData(); // Copy key data to store in map. + auto [it, inserted] = conn_map.emplace(std::move(sk), std::make_unique(this)); + assert(inserted); + cm_it = it; // Let script land know about the state we created, so it will // register a conn removal hook for cleanup. diff --git a/src/packet_analysis/protocol/gtpv1/GTPv1.h b/src/packet_analysis/protocol/gtpv1/GTPv1.h index f48707dea1..846f550942 100644 --- a/src/packet_analysis/protocol/gtpv1/GTPv1.h +++ b/src/packet_analysis/protocol/gtpv1/GTPv1.h @@ -3,6 +3,7 @@ #pragma once #include "zeek/packet_analysis/Analyzer.h" +#include "zeek/session/Key.h" #include "packet_analysis/protocol/gtpv1/gtpv1_pac.h" @@ -27,11 +28,10 @@ public: gtp_hdr_val = std::move(val); } - void RemoveConnection(const zeek::detail::ConnKey& conn_key) { conn_map.erase(conn_key); } + void RemoveConnection(const zeek::session::detail::Key& conn_key) { conn_map.erase(conn_key); } protected: - using ConnMap = std::map>; - ConnMap conn_map; + std::map> conn_map; int inner_packet_offset = -1; uint8_t next_header = 0; diff --git a/src/packet_analysis/protocol/gtpv1/functions.bif b/src/packet_analysis/protocol/gtpv1/functions.bif index 05376a920e..d48cf8acce 100644 --- a/src/packet_analysis/protocol/gtpv1/functions.bif +++ b/src/packet_analysis/protocol/gtpv1/functions.bif @@ -2,6 +2,7 @@ module PacketAnalyzer::GTPV1; %%{ #include "zeek/Conn.h" +#include "zeek/conn_key/Manager.h" #include "zeek/session/Manager.h" #include "zeek/packet_analysis/Manager.h" #include "zeek/packet_analysis/protocol/gtpv1/GTPv1.h" @@ -12,8 +13,12 @@ function remove_gtpv1_connection%(cid: conn_id%) : bool zeek::packet_analysis::AnalyzerPtr gtpv1 = zeek::packet_mgr->GetAnalyzer("GTPv1"); if ( gtpv1 ) { - zeek::detail::ConnKey conn_key(cid); - static_cast(gtpv1.get())->RemoveConnection(conn_key); + auto r = zeek::conn_key_mgr->GetFactory().ConnKeyFromVal(*cid); + if ( ! r.has_value() ) + return zeek::val_mgr->False(); + + auto sk = r.value()->SessionKey(); + static_cast(gtpv1.get())->RemoveConnection(sk); } return zeek::val_mgr->True(); diff --git a/src/packet_analysis/protocol/ip/IPBasedAnalyzer.cc b/src/packet_analysis/protocol/ip/IPBasedAnalyzer.cc index 1b816a11b7..bfee8c3a04 100644 --- a/src/packet_analysis/protocol/ip/IPBasedAnalyzer.cc +++ b/src/packet_analysis/protocol/ip/IPBasedAnalyzer.cc @@ -7,6 +7,8 @@ #include "zeek/Val.h" #include "zeek/analyzer/Manager.h" #include "zeek/analyzer/protocol/pia/PIA.h" +#include "zeek/conn_key/Manager.h" +#include "zeek/packet_analysis/protocol/ip/conn_key/IPBasedConnKey.h" #include "zeek/plugin/Manager.h" #include "zeek/session/Manager.h" @@ -26,13 +28,30 @@ bool IPBasedAnalyzer::AnalyzePacket(size_t len, const uint8_t* data, Packet* pkt if ( ! BuildConnTuple(len, data, pkt, tuple) ) return false; - const std::shared_ptr& ip_hdr = pkt->ip_hdr; - zeek::detail::ConnKey key(tuple); + static IPBasedConnKeyPtr key; // Note, this is static for reuse: + if ( ! key ) { + ConnKeyPtr ck = conn_key_mgr->GetFactory().NewConnKey(); - Connection* conn = session_mgr->FindConnection(key); + // The IPBasedAnalyzer requires a factory that produces IPBasedConnKey instances. + // We could check with dynamic_cast, but that's probably slow, so assume plugin + // providers know what they're doing here and anyhow, we don't really have analyzers + // that instantiate non-IP connections today and definitely not here! + key = IPBasedConnKeyPtr(static_cast(ck.release())); + } + + // Initialize the key with the IP conn tuple and the packet as additional context. + // + // Custom IPConnKey implementations can fiddle with the Key through + // the DoInit(const Packet& pkt) hook called at this point. + key->InitTuple(tuple); + key->Init(*pkt); + + const std::shared_ptr& ip_hdr = pkt->ip_hdr; + + Connection* conn = session_mgr->FindConnection(*key); if ( ! conn ) { - conn = NewConn(&tuple, key, pkt); + conn = NewConn(tuple, std::move(key), pkt); if ( conn ) session_mgr->Insert(conn, false); } @@ -41,7 +60,7 @@ bool IPBasedAnalyzer::AnalyzePacket(size_t len, const uint8_t* data, Packet* pkt conn->Event(connection_reused, nullptr); session_mgr->Remove(conn); - conn = NewConn(&tuple, key, pkt); + conn = NewConn(tuple, std::move(key), pkt); if ( conn ) session_mgr->Insert(conn, false); } @@ -140,18 +159,19 @@ bool IPBasedAnalyzer::IsLikelyServerPort(uint32_t port) const { return port_cache.find(port) != port_cache.end(); } -zeek::Connection* IPBasedAnalyzer::NewConn(const ConnTuple* id, const zeek::detail::ConnKey& key, const Packet* pkt) { - int src_h = ntohs(id->src_port); - int dst_h = ntohs(id->dst_port); +zeek::Connection* IPBasedAnalyzer::NewConn(const ConnTuple& id, IPBasedConnKeyPtr key, const Packet* pkt) { + int src_h = ntohs(id.src_port); + int dst_h = ntohs(id.dst_port); bool flip = false; if ( ! WantConnection(src_h, dst_h, pkt->ip_hdr->Payload(), flip) ) return nullptr; - Connection* conn = new Connection(key, run_state::processing_start_time, id, pkt->ip_hdr->FlowLabel(), pkt); + Connection* conn = + new Connection(std::move(key), id, run_state::processing_start_time, pkt->ip_hdr->FlowLabel(), pkt); conn->SetTransport(transport); - if ( flip && ! id->dst_addr.IsBroadcast() ) + if ( flip && ! id.dst_addr.IsBroadcast() ) conn->FlipRoles(); BuildSessionAnalyzerTree(conn); diff --git a/src/packet_analysis/protocol/ip/IPBasedAnalyzer.h b/src/packet_analysis/protocol/ip/IPBasedAnalyzer.h index bc6d0d08e8..ecaaa0970a 100644 --- a/src/packet_analysis/protocol/ip/IPBasedAnalyzer.h +++ b/src/packet_analysis/protocol/ip/IPBasedAnalyzer.h @@ -7,6 +7,7 @@ #include "zeek/Tag.h" #include "zeek/packet_analysis/Analyzer.h" +#include "zeek/packet_analysis/protocol/ip/conn_key/IPBasedConnKey.h" namespace zeek::analyzer::pia { class PIA; @@ -184,7 +185,7 @@ private: * @param key A connection ID key generated from the ID. * @param pkt The packet associated with the new connection. */ - zeek::Connection* NewConn(const ConnTuple* id, const zeek::detail::ConnKey& key, const Packet* pkt); + zeek::Connection* NewConn(const ConnTuple& id, IPBasedConnKeyPtr key, const Packet* pkt); void BuildSessionAnalyzerTree(Connection* conn); diff --git a/src/packet_analysis/protocol/teredo/Teredo.cc b/src/packet_analysis/protocol/teredo/Teredo.cc index 68acf554bf..a2a0781b71 100644 --- a/src/packet_analysis/protocol/teredo/Teredo.cc +++ b/src/packet_analysis/protocol/teredo/Teredo.cc @@ -185,15 +185,19 @@ bool TeredoAnalyzer::AnalyzePacket(size_t len, const uint8_t* data, Packet* pack return false; } - zeek::detail::ConnKey conn_key = conn->Key(); - OrigRespMap::iterator or_it = orig_resp_map.find(conn_key); + const auto& k = conn->Key(); + auto sk = k.SessionKey(); + OrigRespMap::iterator or_it = orig_resp_map.find(sk); // The first time a teredo packet is parsed successfully, insert // state into orig_resp_map so we can confirm when both sides // see valid Teredo packets. Further, raise an event so that script // layer can install a connection removal hooks to cleanup later. if ( or_it == orig_resp_map.end() ) { - or_it = orig_resp_map.insert(or_it, {conn_key, {}}); + sk.CopyData(); // Copy key data to store in map. + auto [it, inserted] = orig_resp_map.emplace(std::move(sk), OrigResp{}); + assert(inserted); + or_it = it; packet->session->EnqueueEvent(new_teredo_state, nullptr, packet->session->GetVal()); } diff --git a/src/packet_analysis/protocol/teredo/Teredo.h b/src/packet_analysis/protocol/teredo/Teredo.h index 7e2ca41948..e29879f668 100644 --- a/src/packet_analysis/protocol/teredo/Teredo.h +++ b/src/packet_analysis/protocol/teredo/Teredo.h @@ -8,6 +8,7 @@ #include "zeek/RE.h" #include "zeek/Reporter.h" #include "zeek/packet_analysis/Analyzer.h" +#include "zeek/session/Key.h" namespace zeek::packet_analysis::teredo { @@ -44,7 +45,7 @@ public: bool DetectProtocol(size_t len, const uint8_t* data, Packet* packet) override; - void RemoveConnection(const zeek::detail::ConnKey& conn_key) { orig_resp_map.erase(conn_key); } + void RemoveConnection(const zeek::session::detail::Key& conn_key) { orig_resp_map.erase(conn_key); } protected: struct OrigResp { @@ -52,7 +53,7 @@ protected: bool valid_resp = false; bool confirmed = false; }; - using OrigRespMap = std::map; + using OrigRespMap = std::map; OrigRespMap orig_resp_map; std::unique_ptr pattern_re; diff --git a/src/packet_analysis/protocol/teredo/functions.bif b/src/packet_analysis/protocol/teredo/functions.bif index 8607712ca5..8b1a5eb48c 100644 --- a/src/packet_analysis/protocol/teredo/functions.bif +++ b/src/packet_analysis/protocol/teredo/functions.bif @@ -2,6 +2,7 @@ module PacketAnalyzer::TEREDO; %%{ #include "zeek/Conn.h" +#include "zeek/conn_key/Manager.h" #include "zeek/session/Manager.h" #include "zeek/packet_analysis/Manager.h" #include "zeek/packet_analysis/protocol/teredo/Teredo.h" @@ -12,8 +13,12 @@ function remove_teredo_connection%(cid: conn_id%) : bool zeek::packet_analysis::AnalyzerPtr teredo = zeek::packet_mgr->GetAnalyzer("Teredo"); if ( teredo ) { - zeek::detail::ConnKey conn_key(cid); - static_cast(teredo.get())->RemoveConnection(conn_key); + auto r = zeek::conn_key_mgr->GetFactory().ConnKeyFromVal(*cid); + if ( ! r.has_value() ) + return zeek::val_mgr->False(); + + auto sk = r.value()->SessionKey(); + static_cast(teredo.get())->RemoveConnection(sk); } return zeek::val_mgr->True(); diff --git a/src/session/Manager.cc b/src/session/Manager.cc index 32dfa4f226..ad1fc49b0d 100644 --- a/src/session/Manager.cc +++ b/src/session/Manager.cc @@ -17,6 +17,7 @@ #include "zeek/Stats.h" #include "zeek/Timer.h" #include "zeek/TunnelEncapsulation.h" +#include "zeek/conn_key/Manager.h" #include "zeek/packet_analysis/Manager.h" #include "zeek/session/Session.h" #include "zeek/telemetry/Manager.h" @@ -88,23 +89,23 @@ Manager::~Manager() { } Connection* Manager::FindConnection(Val* v) { - zeek::detail::ConnKey conn_key(v); + // XXX: This could in the future dispatch to different factories for + // different kinds of Vals. ``v`` will usually be a conn_id instance, which + // is IP-specific. If ``v`` is something else, maybe we'd like to use a + // different builder. + auto r = conn_key_mgr->GetFactory().ConnKeyFromVal(*v); - if ( ! conn_key.Valid() ) { + if ( ! r.has_value() ) { // Produce a loud error for invalid script-layer conn_id records. - const char* extra = ""; - if ( conn_key.transport == UNKNOWN_IP_PROTO ) - extra = ": the proto field has the \"unknown\" 65535 value. Did you forget to set it?"; - - zeek::emit_builtin_error(zeek::util::fmt("invalid connection ID record encountered%s", extra)); + zeek::emit_builtin_error(r.error().c_str()); return nullptr; } - return FindConnection(conn_key); + return FindConnection(*r.value()); } -Connection* Manager::FindConnection(const zeek::detail::ConnKey& conn_key) { - detail::Key key(&conn_key, sizeof(conn_key), detail::Key::CONNECTION_KEY_TYPE, false); +Connection* Manager::FindConnection(const zeek::ConnKey& conn_key) { + auto key = conn_key.SessionKey(); auto it = session_map.find(key); if ( it != session_map.end() ) diff --git a/src/session/Manager.h b/src/session/Manager.h index 5803673cdf..c4e1ef4f73 100644 --- a/src/session/Manager.h +++ b/src/session/Manager.h @@ -5,6 +5,7 @@ #include // for u_char #include +#include "zeek/ConnKey.h" #include "zeek/Frag.h" #include "zeek/session/Session.h" @@ -70,7 +71,7 @@ public: * @param conn_key The key for the connection to search for. * @return The connection, or nullptr if one doesn't exist. */ - Connection* FindConnection(const zeek::detail::ConnKey& conn_key); + Connection* FindConnection(const zeek::ConnKey& conn_key); void Remove(Session* s); void Insert(Session* c, bool remove_existing = true); From 7548dc9e961e48ef8e317a607d674f3461d01f81 Mon Sep 17 00:00:00 2001 From: Christian Kreibich Date: Thu, 12 Jun 2025 13:48:12 -0700 Subject: [PATCH 06/10] Deprecate the old Connection constructor and detail::ConnKey class. The new key-based Connection constructor replaces the former, and the new ConnKey class tree replaces the latter. --- src/Conn.cc | 3 +++ src/Conn.h | 6 +++++- src/IPAddr.cc | 3 +++ src/IPAddr.h | 11 +++++++---- 4 files changed, 18 insertions(+), 5 deletions(-) diff --git a/src/Conn.cc b/src/Conn.cc index 97a2564d1a..1092ac1d74 100644 --- a/src/Conn.cc +++ b/src/Conn.cc @@ -42,6 +42,8 @@ Connection::Connection(zeek::IPBasedConnKeyPtr k, const zeek::ConnTuple& ct, dou Init(flow, pkt); } +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wdeprecated-declarations" Connection::Connection(const detail::ConnKey& k, double t, const ConnTuple* id, uint32_t flow, const Packet* pkt) : Session(t, connection_timeout, connection_status_update, detail::connection_status_update_interval) { orig_addr = id->src_addr; @@ -63,6 +65,7 @@ Connection::Connection(const detail::ConnKey& k, double t, const ConnTuple* id, Init(flow, pkt); } +#pragma GCC diagnostic pop Connection::~Connection() { if ( ! finished ) diff --git a/src/Conn.h b/src/Conn.h index ca634a821f..2bff20b240 100644 --- a/src/Conn.h +++ b/src/Conn.h @@ -69,7 +69,10 @@ static inline int addr_port_canon_lt(const IPAddr& addr1, uint32_t p1, const IPA class Connection final : public session::Session { public: Connection(zeek::IPBasedConnKeyPtr k, const zeek::ConnTuple& ct, double t, uint32_t flow, const Packet* pkt); + + [[deprecated("Remove in v8.1. Switch to ConnKey factories and the new zeek::ConnKey tree.")]] Connection(const detail::ConnKey& k, double t, const ConnTuple* id, uint32_t flow, const Packet* pkt); + ~Connection() override; /** @@ -201,7 +204,8 @@ public: bool IsFinished() { return finished; } private: - // Common initialization for the constructors. + // Common initialization for the constructors. This can move back into the + // (sole) constructor when we remove the deprecated one in 8.1. void Init(uint32_t flow, const Packet* pkt); friend class session::detail::Timer; diff --git a/src/IPAddr.cc b/src/IPAddr.cc index c22498b22b..d8a777f59d 100644 --- a/src/IPAddr.cc +++ b/src/IPAddr.cc @@ -27,6 +27,8 @@ ConnKey::ConnKey(const ConnTuple& id) { Init(id.src_addr, id.dst_addr, id.src_port, id.dst_port, id.proto, id.is_one_way); } +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wdeprecated-declarations" ConnKey& ConnKey::operator=(const ConnKey& rhs) { if ( this == &rhs ) return *this; @@ -45,6 +47,7 @@ ConnKey& ConnKey::operator=(const ConnKey& rhs) { return *this; } +#pragma GCC diagnostic pop ConnKey::ConnKey(Val* v) { const auto& vt = v->GetType(); diff --git a/src/IPAddr.h b/src/IPAddr.h index 8e71010e98..341b1cc583 100644 --- a/src/IPAddr.h +++ b/src/IPAddr.h @@ -25,6 +25,7 @@ constexpr uint16_t INVALID_CONN_KEY_IP_PROTO = 65534; class HashKey; +// Deprecated: Remove the whole class in v8.1. Switch usage to the conntuple factories and the new zeek::ConnKey tree. class ConnKey { public: in6_addr ip1; @@ -33,10 +34,12 @@ public: uint16_t port2 = 0; uint16_t transport = INVALID_CONN_KEY_IP_PROTO; - ConnKey(const IPAddr& src, const IPAddr& dst, uint16_t src_port, uint16_t dst_port, uint16_t proto, bool one_way); - ConnKey(const ConnTuple& conn); - ConnKey(const ConnKey& rhs) { *this = rhs; } - ConnKey(Val* v); + [[deprecated("Remove in v8.1: Switch to new conn_key framework")]] ConnKey(const IPAddr& src, const IPAddr& dst, + uint16_t src_port, uint16_t dst_port, + uint16_t proto, bool one_way); + [[deprecated("Remove in v8.1: Switch to new conn_key framework")]] ConnKey(const ConnTuple& conn); + [[deprecated("Remove in v8.1: Switch to new conn_key framework")]] ConnKey(const ConnKey& rhs) { *this = rhs; } + [[deprecated("Remove in v8.1: Switch to new conn_key framework")]] ConnKey(Val* v); // FIXME: This is getting reworked as part of the connection tuple changes. Suppress // the clang-tidy warning for the time being. From a5122b503252fa1c8991ae9782210e623d7b22a1 Mon Sep 17 00:00:00 2001 From: Christian Kreibich Date: Thu, 12 Jun 2025 17:21:26 -0700 Subject: [PATCH 07/10] Deprecate ConnTuple and related APIs. Given IP-aware ConnKeys, ConnTuples aren't really required any more. ConnTuple had two benefits: - It preserved the original src/dst orientation from the packet headers it was based on, which IPBasedConnKey now tracks and provides accessor methods for. - In IPBasedAnalyzer::AnalyzePacket() its instance survived past the std:move() of the key into NewConn(), which we sidestep by keeping the original src address and port around until we need after the connection is obtained. --- src/Conn.cc | 14 ++--- src/Conn.h | 20 ++++--- src/IPAddr.cc | 4 +- src/analyzer/Analyzer.cc | 6 +-- src/analyzer/protocol/smtp/BDAT.cc | 3 +- src/packet_analysis/protocol/icmp/ICMP.cc | 17 +++--- src/packet_analysis/protocol/icmp/ICMP.h | 5 +- .../protocol/ip/IPBasedAnalyzer.cc | 42 ++++++++------- .../protocol/ip/IPBasedAnalyzer.h | 29 +++++++--- .../protocol/ip/conn_key/IPBasedConnKey.cc | 27 ++++++++++ .../protocol/ip/conn_key/IPBasedConnKey.h | 53 +++++++++++-------- .../protocol/ip/conn_key/fivetuple/Factory.cc | 2 +- src/packet_analysis/protocol/tcp/TCP.cc | 14 ++--- src/packet_analysis/protocol/tcp/TCP.h | 5 +- src/packet_analysis/protocol/udp/UDP.cc | 10 +--- src/packet_analysis/protocol/udp/UDP.h | 5 +- .../UnknownIPTransport.cc | 6 +-- .../unknown_ip_transport/UnknownIPTransport.h | 5 +- 18 files changed, 149 insertions(+), 118 deletions(-) diff --git a/src/Conn.cc b/src/Conn.cc index 1092ac1d74..86c2c690ad 100644 --- a/src/Conn.cc +++ b/src/Conn.cc @@ -23,15 +23,15 @@ namespace zeek { uint64_t Connection::total_connections = 0; uint64_t Connection::current_connections = 0; -Connection::Connection(zeek::IPBasedConnKeyPtr k, const zeek::ConnTuple& ct, double t, uint32_t flow, const Packet* pkt) +Connection::Connection(zeek::IPBasedConnKeyPtr k, double t, uint32_t flow, const Packet* pkt) : Session(t, connection_timeout, connection_status_update, detail::connection_status_update_interval), key(std::move(k)) { - orig_addr = ct.src_addr; - resp_addr = ct.dst_addr; - orig_port = ct.src_port; - resp_port = ct.dst_port; + orig_addr = key->SrcAddr(); + resp_addr = key->DstAddr(); + orig_port = key->SrcPort(); + resp_port = key->DstPort(); - switch ( ct.proto ) { + switch ( key->Proto() ) { case IPPROTO_TCP: proto = TRANSPORT_TCP; break; case IPPROTO_UDP: proto = TRANSPORT_UDP; break; case IPPROTO_ICMP: @@ -60,7 +60,7 @@ Connection::Connection(const detail::ConnKey& k, double t, const ConnTuple* id, } key = std::make_unique(); - key->InitTuple(*id); + key->InitTuple(id->src_addr, id->src_port, id->dst_addr, id->dst_port, id->proto, id->is_one_way); key->Init(*pkt); Init(flow, pkt); diff --git a/src/Conn.h b/src/Conn.h index 2bff20b240..a35c419b91 100644 --- a/src/Conn.h +++ b/src/Conn.h @@ -53,13 +53,19 @@ enum ConnEventToFlag : uint8_t { NUM_EVENTS_TO_FLAG, }; +// Deprecated without replacement: remove in v8.1. +// XXX using [[deprecated]] for the whole struct leads to hard errors on FreeBSD/MacOS. +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wdeprecated-declarations" struct ConnTuple { - IPAddr src_addr; - IPAddr dst_addr; - uint32_t src_port = 0; - uint32_t dst_port = 0; - uint16_t proto = UNKNOWN_IP_PROTO; - bool is_one_way = false; // if true, don't canonicalize order +#pragma GCC diagnostic pop + [[deprecated("Remove in v8.1: Switch to new conn_key framework")]] IPAddr src_addr; + [[deprecated("Remove in v8.1: Switch to new conn_key framework")]] IPAddr dst_addr; + [[deprecated("Remove in v8.1: Switch to new conn_key framework")]] uint32_t src_port = 0; + [[deprecated("Remove in v8.1: Switch to new conn_key framework")]] uint32_t dst_port = 0; + [[deprecated("Remove in v8.1: Switch to new conn_key framework")]] uint16_t proto = UNKNOWN_IP_PROTO; + [[deprecated("Remove in v8.1: Switch to new conn_key framework")]] bool is_one_way = + false; // if true, don't canonicalize order }; static inline int addr_port_canon_lt(const IPAddr& addr1, uint32_t p1, const IPAddr& addr2, uint32_t p2) { @@ -68,7 +74,7 @@ static inline int addr_port_canon_lt(const IPAddr& addr1, uint32_t p1, const IPA class Connection final : public session::Session { public: - Connection(zeek::IPBasedConnKeyPtr k, const zeek::ConnTuple& ct, double t, uint32_t flow, const Packet* pkt); + Connection(zeek::IPBasedConnKeyPtr k, double t, uint32_t flow, const Packet* pkt); [[deprecated("Remove in v8.1. Switch to ConnKey factories and the new zeek::ConnKey tree.")]] Connection(const detail::ConnKey& k, double t, const ConnTuple* id, uint32_t flow, const Packet* pkt); diff --git a/src/IPAddr.cc b/src/IPAddr.cc index d8a777f59d..463896b4f8 100644 --- a/src/IPAddr.cc +++ b/src/IPAddr.cc @@ -23,12 +23,12 @@ ConnKey::ConnKey(const IPAddr& src, const IPAddr& dst, uint16_t src_port, uint16 Init(src, dst, src_port, dst_port, proto, one_way); } +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wdeprecated-declarations" ConnKey::ConnKey(const ConnTuple& id) { Init(id.src_addr, id.dst_addr, id.src_port, id.dst_port, id.proto, id.is_one_way); } -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wdeprecated-declarations" ConnKey& ConnKey::operator=(const ConnKey& rhs) { if ( this == &rhs ) return *this; diff --git a/src/analyzer/Analyzer.cc b/src/analyzer/Analyzer.cc index fc73a468f7..006d2d34fb 100644 --- a/src/analyzer/Analyzer.cc +++ b/src/analyzer/Analyzer.cc @@ -807,9 +807,8 @@ TEST_SUITE("Analyzer management") { REQUIRE(zeek::analyzer_mgr); zeek::Packet p; - zeek::ConnTuple ct; zeek::IPBasedConnKeyPtr kp = std::make_unique(); - auto conn = std::make_unique(std::move(kp), ct, 0, 0, &p); + auto conn = std::make_unique(std::move(kp), 0, 0, &p); auto* tcp = new zeek::packet_analysis::TCP::TCPSessionAdapter(conn.get()); conn->SetSessionAdapter(tcp, nullptr); @@ -840,9 +839,8 @@ TEST_SUITE("Analyzer management") { REQUIRE(zeek::analyzer_mgr); zeek::Packet p; - zeek::ConnTuple ct; zeek::IPBasedConnKeyPtr kp = std::make_unique(); - auto conn = std::make_unique(std::move(kp), ct, 0, 0, &p); + auto conn = std::make_unique(std::move(kp), 0, 0, &p); auto ssh = zeek::analyzer_mgr->InstantiateAnalyzer("SSH", conn.get()); REQUIRE(ssh); diff --git a/src/analyzer/protocol/smtp/BDAT.cc b/src/analyzer/protocol/smtp/BDAT.cc index 56fe42f4bb..df86fbbc41 100644 --- a/src/analyzer/protocol/smtp/BDAT.cc +++ b/src/analyzer/protocol/smtp/BDAT.cc @@ -328,9 +328,8 @@ private: TEST_CASE("line forward testing") { zeek::Packet p; - zeek::ConnTuple ct; zeek::IPBasedConnKeyPtr kp = std::make_unique(); - auto conn = std::make_unique(std::move(kp), ct, 0, 0, &p); + auto conn = std::make_unique(std::move(kp), 0, 0, &p); auto smtp_analyzer = std::unique_ptr(zeek::analyzer_mgr->InstantiateAnalyzer("SMTP", conn.get())); auto mail = std::make_unique(smtp_analyzer.get()); diff --git a/src/packet_analysis/protocol/icmp/ICMP.cc b/src/packet_analysis/protocol/icmp/ICMP.cc index 5c40363acd..d1ba857801 100644 --- a/src/packet_analysis/protocol/icmp/ICMP.cc +++ b/src/packet_analysis/protocol/icmp/ICMP.cc @@ -28,24 +28,25 @@ SessionAdapter* ICMPAnalyzer::MakeSessionAdapter(Connection* conn) { return root; } -bool ICMPAnalyzer::BuildConnTuple(size_t len, const uint8_t* data, Packet* packet, ConnTuple& tuple) { +bool ICMPAnalyzer::InitConnKey(size_t len, const uint8_t* data, Packet* packet, IPBasedConnKey& key) { if ( ! CheckHeaderTrunc(ICMP_MINLEN, len, packet) ) return false; - tuple.src_addr = packet->ip_hdr->SrcAddr(); - tuple.dst_addr = packet->ip_hdr->DstAddr(); - tuple.proto = packet->proto; - const struct icmp* icmpp = (const struct icmp*)data; - tuple.src_port = htons(icmpp->icmp_type); + + uint32_t icmp_counter_type = 0; + bool is_one_way = false; if ( packet->proto == IPPROTO_ICMP ) - tuple.dst_port = htons(ICMP4_counterpart(icmpp->icmp_type, icmpp->icmp_code, tuple.is_one_way)); + icmp_counter_type = ICMP4_counterpart(icmpp->icmp_type, icmpp->icmp_code, is_one_way); else if ( packet->proto == IPPROTO_ICMPV6 ) - tuple.dst_port = htons(ICMP6_counterpart(icmpp->icmp_type, icmpp->icmp_code, tuple.is_one_way)); + icmp_counter_type = ICMP6_counterpart(icmpp->icmp_type, icmpp->icmp_code, is_one_way); else reporter->InternalError("Reached ICMP packet analyzer with unknown packet protocol %x", packet->proto); + key.InitTuple(packet->ip_hdr->SrcAddr(), htons(icmpp->icmp_type), packet->ip_hdr->DstAddr(), + htons(icmp_counter_type), packet->proto, is_one_way); + return true; } diff --git a/src/packet_analysis/protocol/icmp/ICMP.h b/src/packet_analysis/protocol/icmp/ICMP.h index 28c469937f..99dbf4248a 100644 --- a/src/packet_analysis/protocol/icmp/ICMP.h +++ b/src/packet_analysis/protocol/icmp/ICMP.h @@ -29,10 +29,7 @@ public: packet_analysis::IP::SessionAdapter* MakeSessionAdapter(Connection* conn) override; protected: - /** - * Parse the header from the packet into a ConnTuple object. - */ - bool BuildConnTuple(size_t len, const uint8_t* data, Packet* packet, ConnTuple& tuple) override; + bool InitConnKey(size_t len, const uint8_t* data, Packet* packet, IPBasedConnKey& key) override; void DeliverPacket(Connection* c, double t, bool is_orig, int remaining, Packet* pkt) override; diff --git a/src/packet_analysis/protocol/ip/IPBasedAnalyzer.cc b/src/packet_analysis/protocol/ip/IPBasedAnalyzer.cc index bfee8c3a04..9773d4817e 100644 --- a/src/packet_analysis/protocol/ip/IPBasedAnalyzer.cc +++ b/src/packet_analysis/protocol/ip/IPBasedAnalyzer.cc @@ -24,10 +24,6 @@ IPBasedAnalyzer::~IPBasedAnalyzer() { } bool IPBasedAnalyzer::AnalyzePacket(size_t len, const uint8_t* data, Packet* pkt) { - ConnTuple tuple; - if ( ! BuildConnTuple(len, data, pkt, tuple) ) - return false; - static IPBasedConnKeyPtr key; // Note, this is static for reuse: if ( ! key ) { ConnKeyPtr ck = conn_key_mgr->GetFactory().NewConnKey(); @@ -39,19 +35,28 @@ bool IPBasedAnalyzer::AnalyzePacket(size_t len, const uint8_t* data, Packet* pkt key = IPBasedConnKeyPtr(static_cast(ck.release())); } - // Initialize the key with the IP conn tuple and the packet as additional context. - // - // Custom IPConnKey implementations can fiddle with the Key through - // the DoInit(const Packet& pkt) hook called at this point. - key->InitTuple(tuple); + // Deprecated: remove ConnTuple use in 8.1 and only use InitConnKey(). +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wdeprecated-declarations" + ConnTuple tuple; + if ( BuildConnTuple(len, data, pkt, tuple) ) { + key->InitTuple(tuple.src_addr, tuple.src_port, tuple.dst_addr, tuple.dst_port, pkt->proto); +#pragma GCC diagnostic pop + } + else if ( ! InitConnKey(len, data, pkt, *key) ) { + return false; + } + key->Init(*pkt); const std::shared_ptr& ip_hdr = pkt->ip_hdr; + auto src_addr = key->SrcAddr(); + auto src_port = key->SrcPort(); Connection* conn = session_mgr->FindConnection(*key); if ( ! conn ) { - conn = NewConn(tuple, std::move(key), pkt); + conn = NewConn(std::move(key), pkt); if ( conn ) session_mgr->Insert(conn, false); } @@ -60,7 +65,7 @@ bool IPBasedAnalyzer::AnalyzePacket(size_t len, const uint8_t* data, Packet* pkt conn->Event(connection_reused, nullptr); session_mgr->Remove(conn); - conn = NewConn(tuple, std::move(key), pkt); + conn = NewConn(std::move(key), pkt); if ( conn ) session_mgr->Insert(conn, false); } @@ -76,7 +81,7 @@ bool IPBasedAnalyzer::AnalyzePacket(size_t len, const uint8_t* data, Packet* pkt // get logged, which means we can mark this packet as having been processed. pkt->processed = true; - bool is_orig = (tuple.src_addr == conn->OrigAddr()) && (tuple.src_port == conn->OrigPort()); + bool is_orig = (src_addr == conn->OrigAddr()) && (src_port == conn->OrigPort()); pkt->is_orig = is_orig; conn->CheckFlowLabel(is_orig, ip_hdr->FlowLabel()); @@ -159,19 +164,18 @@ bool IPBasedAnalyzer::IsLikelyServerPort(uint32_t port) const { return port_cache.find(port) != port_cache.end(); } -zeek::Connection* IPBasedAnalyzer::NewConn(const ConnTuple& id, IPBasedConnKeyPtr key, const Packet* pkt) { - int src_h = ntohs(id.src_port); - int dst_h = ntohs(id.dst_port); +zeek::Connection* IPBasedAnalyzer::NewConn(IPBasedConnKeyPtr key, const Packet* pkt) { + auto src_p = ntohs(key->SrcPort()); + auto dst_p = ntohs(key->DstPort()); bool flip = false; - if ( ! WantConnection(src_h, dst_h, pkt->ip_hdr->Payload(), flip) ) + if ( ! WantConnection(src_p, dst_p, pkt->ip_hdr->Payload(), flip) ) return nullptr; - Connection* conn = - new Connection(std::move(key), id, run_state::processing_start_time, pkt->ip_hdr->FlowLabel(), pkt); + Connection* conn = new Connection(std::move(key), run_state::processing_start_time, pkt->ip_hdr->FlowLabel(), pkt); conn->SetTransport(transport); - if ( flip && ! id.dst_addr.IsBroadcast() ) + if ( flip && ! conn->RespAddr().IsBroadcast() ) conn->FlipRoles(); BuildSessionAnalyzerTree(conn); diff --git a/src/packet_analysis/protocol/ip/IPBasedAnalyzer.h b/src/packet_analysis/protocol/ip/IPBasedAnalyzer.h index ecaaa0970a..bdbf672993 100644 --- a/src/packet_analysis/protocol/ip/IPBasedAnalyzer.h +++ b/src/packet_analysis/protocol/ip/IPBasedAnalyzer.h @@ -98,10 +98,29 @@ protected: */ IPBasedAnalyzer(const char* name, TransportProto proto, uint32_t mask, bool report_unknown_protocols); + /** + * Initialize the given ConnKey from the packet header & data. + * + * @param len Remaining length of data. + * @param data Remaining packet data. + * @param packet The packet being processed. + * @param key The ConnKey instance to initialize. + * + * @return True if initialization succeeded, false otherwise (e.g. because + * there wasn't enough data available). + */ + virtual bool InitConnKey(size_t len, const uint8_t* data, Packet* packet, IPBasedConnKey& key) { + // Given deprecation of BuildConnTuple below, make this pure virtual in 8.1. + return false; + } + /** * Parse the header from the packet into a ConnTuple object. */ - virtual bool BuildConnTuple(size_t len, const uint8_t* data, Packet* packet, ConnTuple& tuple) = 0; + [[deprecated("Remove in v8.1. Switch to InitConnKey() and key-only initialization.")]] + virtual bool BuildConnTuple(size_t len, const uint8_t* data, Packet* packet, ConnTuple& tuple) { + return false; + } /** * Continues process of packet after the connection has been inserted into the @@ -180,12 +199,10 @@ private: /** * Creates a new Connection object from data gleaned from the current packet. * - * @param id A connection ID generated from the packet data. This should have been - * passed in from a child analyzer. - * @param key A connection ID key generated from the ID. - * @param pkt The packet associated with the new connection. + * @param key A ConnKey with common 5-tuple information. + * @param pkt The packet associated with the new connection, for additional connection info. */ - zeek::Connection* NewConn(const ConnTuple& id, IPBasedConnKeyPtr key, const Packet* pkt); + zeek::Connection* NewConn(IPBasedConnKeyPtr key, const Packet* pkt); void BuildSessionAnalyzerTree(Connection* conn); diff --git a/src/packet_analysis/protocol/ip/conn_key/IPBasedConnKey.cc b/src/packet_analysis/protocol/ip/conn_key/IPBasedConnKey.cc index ca4ac01e5e..49a44f9492 100644 --- a/src/packet_analysis/protocol/ip/conn_key/IPBasedConnKey.cc +++ b/src/packet_analysis/protocol/ip/conn_key/IPBasedConnKey.cc @@ -1,3 +1,30 @@ // See the file "COPYING" in the main distribution directory for copyright. #include "zeek/packet_analysis/protocol/ip/conn_key/IPBasedConnKey.h" + +#include "zeek/Conn.h" + +using namespace zeek; +using namespace zeek::packet_analysis::IP; + +void IPBasedConnKey::InitTuple(const IPAddr& src_addr, uint32_t src_port, const IPAddr& dst_addr, uint32_t dst_port, + uint16_t proto, bool is_one_way) { + auto& tuple = PackedTuple(); + + if ( is_one_way || addr_port_canon_lt(src_addr, src_port, dst_addr, dst_port) ) { + src_addr.CopyIPv6(&tuple.ip1); + dst_addr.CopyIPv6(&tuple.ip2); + tuple.port1 = src_port; + tuple.port2 = dst_port; + flipped = false; + } + else { + dst_addr.CopyIPv6(&tuple.ip1); + src_addr.CopyIPv6(&tuple.ip2); + tuple.port1 = dst_port; + tuple.port2 = src_port; + flipped = true; + } + + tuple.proto = proto; +} diff --git a/src/packet_analysis/protocol/ip/conn_key/IPBasedConnKey.h b/src/packet_analysis/protocol/ip/conn_key/IPBasedConnKey.h index 4202b3420d..06e4e440cc 100644 --- a/src/packet_analysis/protocol/ip/conn_key/IPBasedConnKey.h +++ b/src/packet_analysis/protocol/ip/conn_key/IPBasedConnKey.h @@ -34,7 +34,34 @@ struct PackedConnTuple { */ class IPBasedConnKey : public zeek::ConnKey { public: - void InitTuple(const ConnTuple& ct) { InitPackedTuple(ct); } + /** + * Initializes the key to the given 5-tuple. This canonicalizes the + * packed tuple storage, including potential endpoint flips for + * consistent connection lookups regardless of directionality. + */ + void InitTuple(const IPAddr& src_addr, uint32_t src_port, const IPAddr& dst_addr, uint32_t dst_port, uint16_t proto, + bool is_one_way = false); + + /** + * The source address the key got initialized with. + */ + IPAddr SrcAddr() const { return flipped ? IPAddr(PackedTuple().ip2) : IPAddr(PackedTuple().ip1); } + /** + * The destination address the key got initialized with. + */ + IPAddr DstAddr() const { return flipped ? IPAddr(PackedTuple().ip1) : IPAddr(PackedTuple().ip2); } + /** + * The source port the key got initialized with. + */ + uint16_t SrcPort() const { return flipped ? PackedTuple().port2 : PackedTuple().port1; } + /** + * The destination port the key got initialized with. + */ + uint16_t DstPort() const { return flipped ? PackedTuple().port1 : PackedTuple().port2; } + /** + * The IP protocol the key got initialized with. + */ + uint16_t Proto() const { return PackedTuple().proto; } /** * Return a modifiable reference to the embedded PackedConnTuple. @@ -56,28 +83,8 @@ public: */ virtual const detail::PackedConnTuple& PackedTuple() const = 0; -private: - /** - * Initialize a packed tuple from a ConnTuple instance. - */ - void InitPackedTuple(const ConnTuple& ct) { - auto& tuple = PackedTuple(); - - if ( ct.is_one_way || addr_port_canon_lt(ct.src_addr, ct.src_port, ct.dst_addr, ct.dst_port) ) { - ct.src_addr.CopyIPv6(&tuple.ip1); - ct.dst_addr.CopyIPv6(&tuple.ip2); - tuple.port1 = ct.src_port; - tuple.port2 = ct.dst_port; - } - else { - ct.dst_addr.CopyIPv6(&tuple.ip1); - ct.src_addr.CopyIPv6(&tuple.ip2); - tuple.port1 = ct.dst_port; - tuple.port2 = ct.src_port; - } - - tuple.proto = ct.proto; - } +protected: + bool flipped = false; }; using IPBasedConnKeyPtr = std::unique_ptr; diff --git a/src/packet_analysis/protocol/ip/conn_key/fivetuple/Factory.cc b/src/packet_analysis/protocol/ip/conn_key/fivetuple/Factory.cc index fbda9443d0..47dd7e3c78 100644 --- a/src/packet_analysis/protocol/ip/conn_key/fivetuple/Factory.cc +++ b/src/packet_analysis/protocol/ip/conn_key/fivetuple/Factory.cc @@ -65,7 +65,7 @@ zeek::expected Factory::DoConnKeyFromVal(const ze "invalid connection ID record encountered: the proto field has the \"unknown\" 65535 value. " "Did you forget to set it?"); - ick->InitTuple(ConnTuple{orig_addr, resp_addr, htons(orig_portv->Port()), htons(resp_portv->Port()), proto16_t}); + ick->InitTuple(orig_addr, htons(orig_portv->Port()), resp_addr, htons(resp_portv->Port()), proto16_t); // Asserting here on the absence of errors can fail btests. diff --git a/src/packet_analysis/protocol/tcp/TCP.cc b/src/packet_analysis/protocol/tcp/TCP.cc index 0b78967a89..8780f58e86 100644 --- a/src/packet_analysis/protocol/tcp/TCP.cc +++ b/src/packet_analysis/protocol/tcp/TCP.cc @@ -26,21 +26,13 @@ SessionAdapter* TCPAnalyzer::MakeSessionAdapter(Connection* conn) { zeek::analyzer::pia::PIA* TCPAnalyzer::MakePIA(Connection* conn) { return new analyzer::pia::PIA_TCP(conn); } -bool TCPAnalyzer::BuildConnTuple(size_t len, const uint8_t* data, Packet* packet, ConnTuple& tuple) { +bool TCPAnalyzer::InitConnKey(size_t len, const uint8_t* data, Packet* packet, IPBasedConnKey& key) { uint32_t min_hdr_len = sizeof(struct tcphdr); if ( ! CheckHeaderTrunc(min_hdr_len, len, packet) ) return false; - tuple.src_addr = packet->ip_hdr->SrcAddr(); - tuple.dst_addr = packet->ip_hdr->DstAddr(); - - data = packet->ip_hdr->Payload(); - - const struct tcphdr* tp = (const struct tcphdr*)data; - tuple.src_port = tp->th_sport; - tuple.dst_port = tp->th_dport; - tuple.is_one_way = false; - tuple.proto = packet->proto; + const struct tcphdr* tp = (const struct tcphdr*)packet->ip_hdr->Payload(); + key.InitTuple(packet->ip_hdr->SrcAddr(), tp->th_sport, packet->ip_hdr->DstAddr(), tp->th_dport, packet->proto); return true; } diff --git a/src/packet_analysis/protocol/tcp/TCP.h b/src/packet_analysis/protocol/tcp/TCP.h index f946e55e9c..531fd46b3b 100644 --- a/src/packet_analysis/protocol/tcp/TCP.h +++ b/src/packet_analysis/protocol/tcp/TCP.h @@ -35,10 +35,7 @@ public: } protected: - /** - * Parse the header from the packet into a ConnTuple object. - */ - bool BuildConnTuple(size_t len, const uint8_t* data, Packet* packet, ConnTuple& tuple) override; + bool InitConnKey(size_t len, const uint8_t* data, Packet* packet, IPBasedConnKey& key) override; void DeliverPacket(Connection* c, double t, bool is_orig, int remaining, Packet* pkt) override; diff --git a/src/packet_analysis/protocol/udp/UDP.cc b/src/packet_analysis/protocol/udp/UDP.cc index 4e6f549de1..bebfabd3fb 100644 --- a/src/packet_analysis/protocol/udp/UDP.cc +++ b/src/packet_analysis/protocol/udp/UDP.cc @@ -53,19 +53,13 @@ bool UDPAnalyzer::WantConnection(uint16_t src_port, uint16_t dst_port, const u_c return true; } -bool UDPAnalyzer::BuildConnTuple(size_t len, const uint8_t* data, Packet* packet, ConnTuple& tuple) { +bool UDPAnalyzer::InitConnKey(size_t len, const uint8_t* data, Packet* packet, IPBasedConnKey& key) { uint32_t min_hdr_len = sizeof(struct udphdr); if ( ! CheckHeaderTrunc(min_hdr_len, len, packet) ) return false; - tuple.src_addr = packet->ip_hdr->SrcAddr(); - tuple.dst_addr = packet->ip_hdr->DstAddr(); - const struct udphdr* up = (const struct udphdr*)packet->ip_hdr->Payload(); - tuple.src_port = up->uh_sport; - tuple.dst_port = up->uh_dport; - tuple.is_one_way = false; - tuple.proto = packet->proto; + key.InitTuple(packet->ip_hdr->SrcAddr(), up->uh_sport, packet->ip_hdr->DstAddr(), up->uh_dport, packet->proto); return true; } diff --git a/src/packet_analysis/protocol/udp/UDP.h b/src/packet_analysis/protocol/udp/UDP.h index fb53240838..2623ab0a5b 100644 --- a/src/packet_analysis/protocol/udp/UDP.h +++ b/src/packet_analysis/protocol/udp/UDP.h @@ -24,10 +24,7 @@ public: void Initialize() override; protected: - /** - * Parse the header from the packet into a ConnTuple object. - */ - bool BuildConnTuple(size_t len, const uint8_t* data, Packet* packet, ConnTuple& tuple) override; + bool InitConnKey(size_t len, const uint8_t* data, Packet* packet, IPBasedConnKey& key) override; void DeliverPacket(Connection* c, double t, bool is_orig, int remaining, Packet* pkt) override; diff --git a/src/packet_analysis/protocol/unknown_ip_transport/UnknownIPTransport.cc b/src/packet_analysis/protocol/unknown_ip_transport/UnknownIPTransport.cc index ffe29cddb0..eea34dc8dc 100644 --- a/src/packet_analysis/protocol/unknown_ip_transport/UnknownIPTransport.cc +++ b/src/packet_analysis/protocol/unknown_ip_transport/UnknownIPTransport.cc @@ -33,10 +33,8 @@ SessionAdapter* UnknownIPTransportAnalyzer::MakeSessionAdapter(Connection* conn) return root; } -bool UnknownIPTransportAnalyzer::BuildConnTuple(size_t len, const uint8_t* data, Packet* packet, ConnTuple& tuple) { - tuple.src_addr = packet->ip_hdr->SrcAddr(); - tuple.dst_addr = packet->ip_hdr->DstAddr(); - tuple.proto = packet->proto; +bool UnknownIPTransportAnalyzer::InitConnKey(size_t len, const uint8_t* data, Packet* packet, IPBasedConnKey& key) { + key.InitTuple(packet->ip_hdr->SrcAddr(), 0, packet->ip_hdr->DstAddr(), 0, packet->proto); return true; } diff --git a/src/packet_analysis/protocol/unknown_ip_transport/UnknownIPTransport.h b/src/packet_analysis/protocol/unknown_ip_transport/UnknownIPTransport.h index bd06177dfa..bd3c577a33 100644 --- a/src/packet_analysis/protocol/unknown_ip_transport/UnknownIPTransport.h +++ b/src/packet_analysis/protocol/unknown_ip_transport/UnknownIPTransport.h @@ -23,10 +23,7 @@ public: packet_analysis::IP::SessionAdapter* MakeSessionAdapter(Connection* conn) override; protected: - /** - * Parse the header from the packet into a ConnTuple object. - */ - bool BuildConnTuple(size_t len, const uint8_t* data, Packet* packet, ConnTuple& tuple) override; + bool InitConnKey(size_t len, const uint8_t* data, Packet* packet, IPBasedConnKey& key) override; void DeliverPacket(Connection* c, double t, bool is_orig, int remaining, Packet* pkt) override; }; From 29b0f844c04c1c7116b9cffb8d5ac12996324d7c Mon Sep 17 00:00:00 2001 From: Christian Kreibich Date: Fri, 13 Jun 2025 17:10:05 -0700 Subject: [PATCH 08/10] Add a VLAN-aware flow tuple implementation. This is a first "real" implementation of a custom tuple, adding additional fields over the standard five-tuple. Includes test cases. --- .../frameworks/conn_key/vlan_fivetuple.zeek | 14 ++ scripts/test-all-policy.zeek | 1 + scripts/zeekygen/__load__.zeek | 1 + .../protocol/ip/conn_key/CMakeLists.txt | 1 + .../ip/conn_key/vlan_fivetuple/CMakeLists.txt | 3 + .../ip/conn_key/vlan_fivetuple/Factory.cc | 129 ++++++++++++++++++ .../ip/conn_key/vlan_fivetuple/Factory.h | 33 +++++ .../ip/conn_key/vlan_fivetuple/Plugin.cc | 24 ++++ .../conn.log.cut | 5 + .../conn.log.cut | 5 + .../conn.log.cut | 5 + .../conn.log.cut | 5 + .../conn.log.cut | 5 + .../conn.log.cut | 3 + testing/btest/Traces/vlan-collisions.pcap | Bin 0 -> 19125 bytes .../frameworks/conn_key/vlan_fivetuple.zeek | 56 ++++++++ 16 files changed, 290 insertions(+) create mode 100644 scripts/policy/frameworks/conn_key/vlan_fivetuple.zeek create mode 100644 src/packet_analysis/protocol/ip/conn_key/vlan_fivetuple/CMakeLists.txt create mode 100644 src/packet_analysis/protocol/ip/conn_key/vlan_fivetuple/Factory.cc create mode 100644 src/packet_analysis/protocol/ip/conn_key/vlan_fivetuple/Factory.h create mode 100644 src/packet_analysis/protocol/ip/conn_key/vlan_fivetuple/Plugin.cc create mode 100644 testing/btest/Baseline/scripts.policy.frameworks.conn_key.vlan_fivetuple-2/conn.log.cut create mode 100644 testing/btest/Baseline/scripts.policy.frameworks.conn_key.vlan_fivetuple-3/conn.log.cut create mode 100644 testing/btest/Baseline/scripts.policy.frameworks.conn_key.vlan_fivetuple-4/conn.log.cut create mode 100644 testing/btest/Baseline/scripts.policy.frameworks.conn_key.vlan_fivetuple-5/conn.log.cut create mode 100644 testing/btest/Baseline/scripts.policy.frameworks.conn_key.vlan_fivetuple-6/conn.log.cut create mode 100644 testing/btest/Baseline/scripts.policy.frameworks.conn_key.vlan_fivetuple/conn.log.cut create mode 100644 testing/btest/Traces/vlan-collisions.pcap create mode 100644 testing/btest/scripts/policy/frameworks/conn_key/vlan_fivetuple.zeek diff --git a/scripts/policy/frameworks/conn_key/vlan_fivetuple.zeek b/scripts/policy/frameworks/conn_key/vlan_fivetuple.zeek new file mode 100644 index 0000000000..ebef3f1ec0 --- /dev/null +++ b/scripts/policy/frameworks/conn_key/vlan_fivetuple.zeek @@ -0,0 +1,14 @@ +##! This script adapts Zeek's connection key to include 802.1Q VLAN and +##! Q-in-Q tags, when available. Zeek normally ignores VLAN tags for connection +##! lookups; this change makes it factor them in and also makes those VLAN tags +##! part of the :zeek:see:`conn_id` record. + +redef record conn_id += { + ## The outer VLAN for this connection, if applicable. + vlan: int &log &optional; + + ## The inner VLAN for this connection, if applicable. + inner_vlan: int &log &optional; +}; + +redef ConnKey::factory = ConnKey::CONNKEY_VLAN_FIVETUPLE; diff --git a/scripts/test-all-policy.zeek b/scripts/test-all-policy.zeek index 6b63659cc8..cc22fd4e55 100644 --- a/scripts/test-all-policy.zeek +++ b/scripts/test-all-policy.zeek @@ -114,6 +114,7 @@ @load protocols/conn/mac-logging.zeek @load protocols/conn/vlan-logging.zeek @load protocols/conn/weirds.zeek +#@load frameworks/conn_key/vlan_fivetuple.zeek #@load protocols/conn/speculative-service.zeek @load protocols/dhcp/msg-orig.zeek @load protocols/dhcp/software.zeek diff --git a/scripts/zeekygen/__load__.zeek b/scripts/zeekygen/__load__.zeek index c5717b310e..d392c027b6 100644 --- a/scripts/zeekygen/__load__.zeek +++ b/scripts/zeekygen/__load__.zeek @@ -2,6 +2,7 @@ # Scripts which are commented out in test-all-policy.zeek. @load frameworks/analyzer/deprecated-dpd-log.zeek +@load frameworks/conn_key/vlan_fivetuple.zeek # Remove in v8.1: replaced by frameworks/analyzer/detect-protocols.zeek @pragma push ignore-deprecations diff --git a/src/packet_analysis/protocol/ip/conn_key/CMakeLists.txt b/src/packet_analysis/protocol/ip/conn_key/CMakeLists.txt index 2a780c21df..95a2dd90b9 100644 --- a/src/packet_analysis/protocol/ip/conn_key/CMakeLists.txt +++ b/src/packet_analysis/protocol/ip/conn_key/CMakeLists.txt @@ -1,3 +1,4 @@ zeek_add_subdir_library(connkey-ip SOURCES IPBasedConnKey.cc) add_subdirectory(fivetuple) +add_subdirectory(vlan_fivetuple) diff --git a/src/packet_analysis/protocol/ip/conn_key/vlan_fivetuple/CMakeLists.txt b/src/packet_analysis/protocol/ip/conn_key/vlan_fivetuple/CMakeLists.txt new file mode 100644 index 0000000000..bc4c11d944 --- /dev/null +++ b/src/packet_analysis/protocol/ip/conn_key/vlan_fivetuple/CMakeLists.txt @@ -0,0 +1,3 @@ +zeek_add_plugin( + Zeek Conntuple_VLAN + SOURCES Factory.cc Plugin.cc) diff --git a/src/packet_analysis/protocol/ip/conn_key/vlan_fivetuple/Factory.cc b/src/packet_analysis/protocol/ip/conn_key/vlan_fivetuple/Factory.cc new file mode 100644 index 0000000000..a9d5c32880 --- /dev/null +++ b/src/packet_analysis/protocol/ip/conn_key/vlan_fivetuple/Factory.cc @@ -0,0 +1,129 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#include "zeek/packet_analysis/protocol/ip/conn_key/vlan_fivetuple/Factory.h" + +#include + +#include "zeek/ID.h" +#include "zeek/Val.h" +#include "zeek/iosource/Packet.h" +#include "zeek/packet_analysis/protocol/ip/conn_key/IPBasedConnKey.h" +#include "zeek/packet_analysis/protocol/ip/conn_key/fivetuple/Factory.h" +#include "zeek/util-types.h" + +namespace zeek::conn_key::vlan_fivetuple { + +class IPVlanConnKey : public zeek::IPBasedConnKey { +public: + /** + * Constructor. + * + * Fill any holes in the key struct as we use the full tuple as a key. + */ + IPVlanConnKey() { memset(static_cast(&key), 0, sizeof(key)); } + + /** + * @copydoc + */ + detail::PackedConnTuple& PackedTuple() override { return key.tuple; } + + /** + * @copydoc + */ + const detail::PackedConnTuple& PackedTuple() const override { return key.tuple; } + +protected: + zeek::session::detail::Key DoSessionKey() const override { + return {reinterpret_cast(&key), sizeof(key), session::detail::Key::CONNECTION_KEY_TYPE}; + } + + void DoPopulateConnIdVal(RecordVal& conn_id) override { + if ( conn_id.NumFields() <= 5 ) + return; + + // Nothing to do if we have no VLAN tags at all. + if ( key.vlan == 0 && key.inner_vlan == 0 ) + return; + + auto [vlan_offset, inner_vlan_offset] = GetConnIdFieldOffsets(); + + if ( key.vlan && vlan_offset >= 0 ) + conn_id.Assign(vlan_offset, static_cast(key.vlan)); + if ( key.inner_vlan && inner_vlan_offset >= 0 ) + conn_id.Assign(inner_vlan_offset, static_cast(key.inner_vlan)); + }; + + std::pair GetConnIdFieldOffsets() { + static int vlan_offset = -2, inner_vlan_offset = -2; + + if ( vlan_offset == -2 && inner_vlan_offset == -2 ) { + vlan_offset = id::conn_id->FieldOffset("vlan"); + if ( vlan_offset < 0 || id::conn_id->GetFieldType(vlan_offset)->Tag() != TYPE_INT ) + vlan_offset = -1; + + inner_vlan_offset = id::conn_id->FieldOffset("inner_vlan"); + if ( inner_vlan_offset < 0 || id::conn_id->GetFieldType(inner_vlan_offset)->Tag() != TYPE_INT ) + inner_vlan_offset = -1; + } + + return {vlan_offset, inner_vlan_offset}; + } + +protected: + void DoInit(const Packet& pkt) override { + key.vlan = pkt.vlan; + key.inner_vlan = pkt.inner_vlan; + } + +private: + friend class Factory; + + // Key bytes: + struct { + struct detail::PackedConnTuple tuple; + // Add 802.1Q vlan tags to connection tuples. The tag representation + // here is as in the Packet class (where it's oddly 32-bit), since + // that's where we learn the tag values from. 0 indicates absence. + uint32_t vlan; + uint32_t inner_vlan; + } __attribute__((packed, aligned)) key; +}; + +zeek::ConnKeyPtr Factory::DoNewConnKey() const { return std::make_unique(); } + +zeek::expected Factory::DoConnKeyFromVal(const zeek::Val& v) const { + auto ck = zeek::conn_key::fivetuple::Factory::DoConnKeyFromVal(v); + + if ( ! ck.has_value() ) + return ck; + + auto* k = static_cast(ck.value().get()); + auto rt = v.GetType()->AsRecordType(); + auto vl = v.AsRecordVal(); + + int vlan_offset, inner_vlan_offset; + if ( rt == id::conn_id ) { + std::tie(vlan_offset, inner_vlan_offset) = k->GetConnIdFieldOffsets(); + } + else { + // We don't know what we've been passed. + vlan_offset = rt->FieldOffset("vlan"); + inner_vlan_offset = rt->FieldOffset("inner_vlan"); + } + + if ( vlan_offset < 0 || inner_vlan_offset < 0 ) + return zeek::unexpected{"missing vlan or inner_vlan field"}; + + if ( rt->GetFieldType(vlan_offset)->Tag() != TYPE_INT || rt->GetFieldType(inner_vlan_offset)->Tag() != TYPE_INT ) + return zeek::unexpected{"vlan or inner_vlan field not of type int"}; + + if ( vl->HasField(vlan_offset) ) + k->key.vlan = vl->GetFieldAs(vlan_offset); + + if ( vl->HasField(inner_vlan_offset) ) + k->key.inner_vlan = vl->GetFieldAs(inner_vlan_offset); + + return ck; +} + +} // namespace zeek::conn_key::vlan_fivetuple diff --git a/src/packet_analysis/protocol/ip/conn_key/vlan_fivetuple/Factory.h b/src/packet_analysis/protocol/ip/conn_key/vlan_fivetuple/Factory.h new file mode 100644 index 0000000000..6c941c029e --- /dev/null +++ b/src/packet_analysis/protocol/ip/conn_key/vlan_fivetuple/Factory.h @@ -0,0 +1,33 @@ +// See the file "COPYING" in the main distribution directory for copyright. +#pragma once + +#include "zeek/ConnKey.h" +#include "zeek/packet_analysis/protocol/ip/conn_key/fivetuple/Factory.h" + +namespace zeek::conn_key::vlan_fivetuple { + +class Factory : public zeek::conn_key::fivetuple::Factory { +public: + static zeek::conn_key::FactoryPtr Instantiate() { return std::make_unique(); } + +private: + /** + * Instantiates a clean ConnKey derivative and returns it. + * + * @return A unique pointer to the ConnKey instance. + */ + zeek::ConnKeyPtr DoNewConnKey() const override; + + /** + * Instantiates a filled-in ConnKey derivative from a script-layer + * record, usually a conn_id instance. Implementations are free to + * implement this liberally, i.e. the input does not _have_ to be a + * conn_id. + * + * @param v The script-layer value providing key input. + * @return A unique pointer to the ConnKey instance, or an error message. + */ + zeek::expected DoConnKeyFromVal(const zeek::Val& v) const override; +}; + +} // namespace zeek::conn_key::vlan_fivetuple diff --git a/src/packet_analysis/protocol/ip/conn_key/vlan_fivetuple/Plugin.cc b/src/packet_analysis/protocol/ip/conn_key/vlan_fivetuple/Plugin.cc new file mode 100644 index 0000000000..c8b443995c --- /dev/null +++ b/src/packet_analysis/protocol/ip/conn_key/vlan_fivetuple/Plugin.cc @@ -0,0 +1,24 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#include "zeek/plugin/Plugin.h" + +#include "zeek/conn_key/Component.h" +#include "zeek/packet_analysis/protocol/ip/conn_key/vlan_fivetuple/Factory.h" + +namespace zeek::plugin::Zeek_ConnKey_VLAN { + +class Plugin : public zeek::plugin::Plugin { +public: + zeek::plugin::Configuration Configure() override { + AddComponent(new conn_key::Component("VLAN_FIVETUPLE", zeek::conn_key::vlan_fivetuple::Factory::Instantiate)); + + zeek::plugin::Configuration config; + config.name = "Zeek::ConnKey_Vlan_Fivetuple"; + config.description = "ConnKey factory for 802.1Q VLAN/Q-in-Q + IP/port/proto five-tuples"; + return config; + } +}; + +Plugin plugin; + +} // namespace zeek::plugin::Zeek_ConnKey_VLAN diff --git a/testing/btest/Baseline/scripts.policy.frameworks.conn_key.vlan_fivetuple-2/conn.log.cut b/testing/btest/Baseline/scripts.policy.frameworks.conn_key.vlan_fivetuple-2/conn.log.cut new file mode 100644 index 0000000000..0326efbaec --- /dev/null +++ b/testing/btest/Baseline/scripts.policy.frameworks.conn_key.vlan_fivetuple-2/conn.log.cut @@ -0,0 +1,5 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +ts uid id.orig_h id.orig_p id.resp_h id.resp_p id.vlan id.inner_vlan orig_pkts resp_pkts service +XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 141.142.228.5 59856 192.150.187.43 80 - - 7 7 http +XXXXXXXXXX.XXXXXX C4J4Th3PJpwUYZZ6gc 141.142.228.5 59856 192.150.187.43 80 10 20 7 7 http +XXXXXXXXXX.XXXXXX ClEkJM2Vm5giqnMf4h 141.142.228.5 59856 192.150.187.43 80 42 - 7 7 http diff --git a/testing/btest/Baseline/scripts.policy.frameworks.conn_key.vlan_fivetuple-3/conn.log.cut b/testing/btest/Baseline/scripts.policy.frameworks.conn_key.vlan_fivetuple-3/conn.log.cut new file mode 100644 index 0000000000..bb15ef37f3 --- /dev/null +++ b/testing/btest/Baseline/scripts.policy.frameworks.conn_key.vlan_fivetuple-3/conn.log.cut @@ -0,0 +1,5 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +ts uid id.orig_h id.orig_p id.resp_h id.resp_p orig_pkts resp_pkts service +XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 141.142.228.5 59856 192.150.187.43 80 7 7 http +XXXXXXXXXX.XXXXXX C4J4Th3PJpwUYZZ6gc 141.142.228.5 59856 192.150.187.43 80 7 7 http +XXXXXXXXXX.XXXXXX ClEkJM2Vm5giqnMf4h 141.142.228.5 59856 192.150.187.43 80 7 7 http diff --git a/testing/btest/Baseline/scripts.policy.frameworks.conn_key.vlan_fivetuple-4/conn.log.cut b/testing/btest/Baseline/scripts.policy.frameworks.conn_key.vlan_fivetuple-4/conn.log.cut new file mode 100644 index 0000000000..0326efbaec --- /dev/null +++ b/testing/btest/Baseline/scripts.policy.frameworks.conn_key.vlan_fivetuple-4/conn.log.cut @@ -0,0 +1,5 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +ts uid id.orig_h id.orig_p id.resp_h id.resp_p id.vlan id.inner_vlan orig_pkts resp_pkts service +XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 141.142.228.5 59856 192.150.187.43 80 - - 7 7 http +XXXXXXXXXX.XXXXXX C4J4Th3PJpwUYZZ6gc 141.142.228.5 59856 192.150.187.43 80 10 20 7 7 http +XXXXXXXXXX.XXXXXX ClEkJM2Vm5giqnMf4h 141.142.228.5 59856 192.150.187.43 80 42 - 7 7 http diff --git a/testing/btest/Baseline/scripts.policy.frameworks.conn_key.vlan_fivetuple-5/conn.log.cut b/testing/btest/Baseline/scripts.policy.frameworks.conn_key.vlan_fivetuple-5/conn.log.cut new file mode 100644 index 0000000000..0326efbaec --- /dev/null +++ b/testing/btest/Baseline/scripts.policy.frameworks.conn_key.vlan_fivetuple-5/conn.log.cut @@ -0,0 +1,5 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +ts uid id.orig_h id.orig_p id.resp_h id.resp_p id.vlan id.inner_vlan orig_pkts resp_pkts service +XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 141.142.228.5 59856 192.150.187.43 80 - - 7 7 http +XXXXXXXXXX.XXXXXX C4J4Th3PJpwUYZZ6gc 141.142.228.5 59856 192.150.187.43 80 10 20 7 7 http +XXXXXXXXXX.XXXXXX ClEkJM2Vm5giqnMf4h 141.142.228.5 59856 192.150.187.43 80 42 - 7 7 http diff --git a/testing/btest/Baseline/scripts.policy.frameworks.conn_key.vlan_fivetuple-6/conn.log.cut b/testing/btest/Baseline/scripts.policy.frameworks.conn_key.vlan_fivetuple-6/conn.log.cut new file mode 100644 index 0000000000..59bbcde6f1 --- /dev/null +++ b/testing/btest/Baseline/scripts.policy.frameworks.conn_key.vlan_fivetuple-6/conn.log.cut @@ -0,0 +1,5 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +ts uid id.orig_h id.orig_p id.resp_h id.resp_p id.vlan id.inner_vlan orig_pkts resp_pkts service +XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 141.142.228.5 59856 192.150.187.43 80 - - 7 7 http +XXXXXXXXXX.XXXXXX C4J4Th3PJpwUYZZ6gc 141.142.228.5 59856 192.150.187.43 80 - - 7 7 http +XXXXXXXXXX.XXXXXX ClEkJM2Vm5giqnMf4h 141.142.228.5 59856 192.150.187.43 80 - - 7 7 http diff --git a/testing/btest/Baseline/scripts.policy.frameworks.conn_key.vlan_fivetuple/conn.log.cut b/testing/btest/Baseline/scripts.policy.frameworks.conn_key.vlan_fivetuple/conn.log.cut new file mode 100644 index 0000000000..6a3949becc --- /dev/null +++ b/testing/btest/Baseline/scripts.policy.frameworks.conn_key.vlan_fivetuple/conn.log.cut @@ -0,0 +1,3 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +ts uid id.orig_h id.orig_p id.resp_h id.resp_p orig_pkts resp_pkts service +XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 141.142.228.5 59856 192.150.187.43 80 21 21 http diff --git a/testing/btest/Traces/vlan-collisions.pcap b/testing/btest/Traces/vlan-collisions.pcap new file mode 100644 index 0000000000000000000000000000000000000000..7c1ea4c6f989261dc1b8cbb90ed6cbad6c976b55 GIT binary patch literal 19125 zcmeHPTWlOx86LYyTNa$8sDkp+o~A8v+|1s*zGTz1b$m1T>y+@nix(B83qQR0x$R&3(`b*fAD`(` zHvQo{=RQ9D^63-jp1n~SSCsqTf9!tc{_AV+KKj@P>t8wX>$^XCTbcgv?fD|z{LAi( z7k9j(DBbJUztnZzb?dsiZcKe??|Y2?wyxWkp1om%GJ~J#ib{V(d7|&LSz%vkgR&RG z>GC!A*OAii!4{Ov_1`R$?z9JG`lGi`o;q_c_ORpGFAhJEp7!-$*L`Cu^W9xvC;hRP z8^|)w!41kTNTn;PyOdmb>lw1l(wR2P?11$9mNv1w8#`cQ-y&s_OcHN6v_W|j!suEd z@#Vcwk;LwO9VOnv?q0VPmpD0IVj10TTBgnQ%;@fs{gdOf={d(%d1E2%EqH8qsWhF* zrE@*0gRXGYk&3XqB0F3W9{sk!dUftJ4J(&V=hNA(J*g2*6Ls9tm+9+C?Y3Qfgt)Zr zRC-dQwq*&;Gi=9);h3n?e#@A1Od1DK! zQD3*v^ndyL)mt_y6HuS7_#OhRuF2imX&^Y-8CWHpLzMCD+gLxniNeh~G?3m3mUM!Z|9OA{(i5 zttvA4bRpf(dMAW#I~-xMkGq~aW$Q-S5IW*X=-XI+I~!?KNLZfb`-_F`#e5)ayu>R- zwmFv{%&7%!cqrRHFw6(bT6S|7aOw=VD#AsuEqcP0q2?h{)qTRMc-0~+3}pv;Qp-VC zWS&tIcEj5_u#MID!p>YayLx~kaHX`;VjVm8#DbTpo7}K=Fs;fRS9m)QmL}9;isV!_ zogZZ1mb*~1!)m_3as$O&wwNt2H9M5erc#Xcu?b_ruqv!k*I`T*Qb_x4tXVa*D%0$S zsk6GF%`>kmSktj_s|Fn$+hk?K6lvBwW6v2Do3(2-;cSh`oSdFyu2Hc_zNB)MLW2nd@<{;WFD| z!f|Ye_3p*TS)(cDp&V8#v{oC!0xXv=4rBqBs7YqH;R;qWETh(_F-Lr>VK_o(qf>ld zFks0AZ_?`d^dLbgx&T>pMAw&hO!w)+6rNxrnr~^E zQ(+|7C0{7QKyYT%E}Upx)f!%~;hJz=ULntnamx@U;)5?r*I@Vk5t%vII$z8U7V#k> zv(yxP-X*;q=*?jX-**5--GpU1vqTg26|4;NhW;fmxM?7Ytw02pjcz5EWmvNSC~oht zEXh1j7uK%XG3GYvb=x7(w41oK;{`+mxdUohu&%K{)@;{+l#-gMY)=_T!F>Y0usBS` zH~>{a^0Nps1ab{XCSj&w3B-ZG15H-vE=*E&>_(+3MKQ)r3IPbhiX*xMt3%oyg9Afp z3IfJdHX~~GQ3(aRCfMZugN9pUWyh{D-kfhm1mLp#aDVGz{2CCk>y)IWrHLjOc%lwZ zBj65}MrqyUX9tJdpY`wcH|vTzcen@Z1|rpo_~V7`t;YjO2yfJiFpas$T-rrda9udk zI4+C>62#QRQ~Zn#Jh7b6>pm+q|@E&xer@kFgI$(7t3=j9e?+F4g_CKug| zzgo7~Pf^xl`+bJznTXtQQJ6@Lhj|C;2k2yENcxx9_IPEb14xIK23Zv;GdEN$fLsTn z8U*GHB6y3-8pKvCU@aNO+^IBb$e975!Q(oDvBQew@?DJz?dHl&MGv)El_DH@%0?W@ z@Hae{A+hL&j{mvBEU(;bh&Hea*0?9}6)f0kCk!b#LWn`F|{#e{-Dq&p#eA|I+v4 z%{L)8JO!6+lMxUKE6l0^HaScyw4gD3b zn*(1>*~bjigc1$~vh-KrzM$}g*TjPKp-b%GHs&5eHxGYy@KPXCH(XDGkyMlE+@b13 zfW8bwfo?b*o|OeJJt>($KJJH(X4laz0qPA$6DyH1TchI0qe7CFBl2oF5Pl=HX?VP1 zc&ZfIVQHiZdX#L5;yGYeH!MY&Y1=4FVOGOI6ChCiN{*-t2YFN$gq>^A%V2@n7@|@B zlA4@t36QmgA>%p=tRnkl0?8KL{33$`aG~@PI!Ea6hS;gKZQ4RglNE3V=%O1 z%r#`i7ZOK%Eej3$7)73+Tw=hE*%`*u9D$xL^pc16c#iVR>XvC0taG}?QFkAv&Wg4` zq;J}lfF`!#cfdLdmMtD_Y!NwJptF(I;nrdeq1hSWA-~CH5K@qv+^XF~`~feh`eow; zSg@;WNMFdc{DyKK^`$GvWUbbri6c*y3=7tgy|oNHza$47xQXIo zbOePoqKld*376T~eKM@2YVscBM%#jRvjWE04KoETc3{cPK}|qY!p#6F`bzjM+7{&V zc)FdsSAkmVHkr;kHujc2JvuTi`2aN;0)6Dta_bEvdc5kG&#Y@sT_|@!-s!{gWsqacf=U z^{p#6yvMJt80sn16G(V+XaH2qOplayue|@T?aXs@d(>2d2__sNs3Y}90Mt`pJOG{n zvUEWOVAw#>6yeV9$7VF-(b?4X(}&`1lNLxCFgRxJ-NQw8AT zpR^2<1hfiL2yD{A7;d0YCWvv1n~Rv11WXXc7S?X9Tu^BoGg!l+xVeirQ z{q#|HZ1M|1T$-o@nHr3l@ce?m(p?gJW0IyhaDx}#FQhiv?RV5>iEKs{G%T_{)vWn>{wvULwmBq zxhMUjA3FYt+BLSqrykssgt{GZm@-G?~Vc#%f#`|eqm7`3S{apXxlfcX6H+YTUJ zKY8lHS$yL{Inv)cfcOLl5Fe7n_kP;d@!KKsUv-O$rr9D10>__=vPn3&B?es z8Fwe|tuy&aGQb6l zyXjhO3Y3h!ld-q|rS5BB?0wrWmItcQ5p8trsT9^S`o8aqB(G)s{pXjiMCY}Pz`t;@ zt91wv5C8BG;P@KG;7>=S#%&x*UCSu^8?Sa>$tZjc`u;5conn.log.cut +# @TEST-EXEC: btest-diff conn.log.cut + +# Default operation: Zeek isn't VLAN-aware, a single conn.log entry results. + +# @TEST-START-NEXT + +# Switch to VLAN-aware flow tuples: multiple conn.log entries with full +# information. + +@load frameworks/conn_key/vlan_fivetuple + +# @TEST-START-NEXT + +# Leave out the conn_id redef: Zeek still distinguishes flows so multiple +# conn.log entries result, but conn.log doesn't show the VLAN fields. + +redef ConnKey::factory = ConnKey::CONNKEY_VLAN_FIVETUPLE; + +# @TEST-START-NEXT + +# Add an extra field before the VLAN ones, to throw off any fixed-offset code. + +redef record conn_id += { + foo: int &default=1; +}; + +@load frameworks/conn_key/vlan_fivetuple + +# @TEST-START-NEXT + +# Add the right fields, but in the wrong order. (zeek-cut obscures the difference.) + +redef record conn_id += { + inner_vlan: int &log &optional; + vlan: int &log &optional; +}; + +redef ConnKey::factory = ConnKey::CONNKEY_VLAN_FIVETUPLE; + +# @TEST-START-NEXT + +# Add the right fields, but with the wrong types. + +redef record conn_id += { + vlan: string &log &optional; + inner_vlan: string &log &optional; +}; + +redef ConnKey::factory = ConnKey::CONNKEY_VLAN_FIVETUPLE; From a040f550f43786c786fc7827b702a67595fb18b0 Mon Sep 17 00:00:00 2001 From: Christian Kreibich Date: Fri, 13 Jun 2025 17:42:25 -0700 Subject: [PATCH 09/10] NEWS updates for pluggable connection tuples. --- NEWS | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/NEWS b/NEWS index 7cec498a6c..4775ceb03c 100644 --- a/NEWS +++ b/NEWS @@ -73,6 +73,29 @@ Breaking Changes New Functionality ----------------- +- Zeek now supports pluggable and customizable connection tracking. The default + behavior remains unchanged and uses a connection's five tuple based on the + IP/port pairs and proto field. Zeek 8 ships with one additional implementation, + to factor VLAN tags into the connection tracking. To switch to VLAN-aware + connection tracking: + + @load frameworks/conn_key/vlan_fivetuple + + This results in two additional fields in the conn_id record, showing any VLAN + tags involved in the flow. (Accordingly, every log using conn_id reflects the + change as well as these fields have the ``&log`` attribute.) + + This feature does not automatically provide a notion of endpoint that + corresponds with the effective flow tuple. For example, applications tracking + endpoints by IP address do not somehow become VLAN-aware when enabling + VLAN-aware tracking. + + Users may add their own plugins (for example via a zkg package) to provide + alternative implementations. This involves implementing a factory for + connection "keys" that factor in additional flow information. See the VLAN + implementation in the ``src/packet_analysis/protocol/ip/conn_key/vlan_fivetuple`` + directory for an example. + - Generic event metadata support. A new ``EventMetadata`` module was added allowing to register generic event metadata types and accessing the current event's metadata using the functions ``current()`` and ``current_all()`` of this module. @@ -234,6 +257,11 @@ Deprecated Functionality and will lead to compile time warnings. Use ``EventMgr::Enqueue(detail::MetadataVectorPtr meta, ...)`` for populating ``meta`` accordingly. +- For plugin authors: in the core, the constructor for Connection instances has + been deprecated in favor of a new one to support pluggable connection + tuples. The ConnTuple struct, used by this deprecated Connection constructor, + is now deprecated as well. + Zeek 7.2.0 ========== From e7b1b174f09446a583abfea5e96586f41d9695fc Mon Sep 17 00:00:00 2001 From: Arne Welzel Date: Fri, 20 Jun 2025 09:42:01 +0200 Subject: [PATCH 10/10] btest/plugins: Add test for custom ConnKey factory This just counts DoInits() and adds that information to the conn_id record, but without including it into the hash. Mostly for smoke testing. --- .../Baseline/plugins.connkey/conn.log.cut | 7 +++ testing/btest/Baseline/plugins.connkey/output | 15 +++++++ .../plugins/connkey-plugin/.btest-ignore | 0 .../plugins/connkey-plugin/CMakeLists.txt | 15 +++++++ .../btest/plugins/connkey-plugin/src/Foo.cc | 45 +++++++++++++++++++ .../btest/plugins/connkey-plugin/src/Foo.h | 25 +++++++++++ .../plugins/connkey-plugin/src/Plugin.cc | 24 ++++++++++ .../btest/plugins/connkey-plugin/src/Plugin.h | 15 +++++++ testing/btest/plugins/connkey.zeek | 16 +++++++ 9 files changed, 162 insertions(+) create mode 100644 testing/btest/Baseline/plugins.connkey/conn.log.cut create mode 100644 testing/btest/Baseline/plugins.connkey/output create mode 100644 testing/btest/plugins/connkey-plugin/.btest-ignore create mode 100644 testing/btest/plugins/connkey-plugin/CMakeLists.txt create mode 100644 testing/btest/plugins/connkey-plugin/src/Foo.cc create mode 100644 testing/btest/plugins/connkey-plugin/src/Foo.h create mode 100644 testing/btest/plugins/connkey-plugin/src/Plugin.cc create mode 100644 testing/btest/plugins/connkey-plugin/src/Plugin.h create mode 100644 testing/btest/plugins/connkey.zeek diff --git a/testing/btest/Baseline/plugins.connkey/conn.log.cut b/testing/btest/Baseline/plugins.connkey/conn.log.cut new file mode 100644 index 0000000000..93f6eb17ff --- /dev/null +++ b/testing/btest/Baseline/plugins.connkey/conn.log.cut @@ -0,0 +1,7 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +ts uid id.orig_h id.orig_p id.resp_h id.resp_p id.inits proto service orig_pkts resp_pkts +XXXXXXXXXX.XXXXXX ClEkJM2Vm5giqnMf4h 141.142.220.235 37604 199.233.217.249 56666 1 tcp ftp-data 4 4 +XXXXXXXXXX.XXXXXX C4J4Th3PJpwUYZZ6gc 141.142.220.235 59378 199.233.217.249 56667 22 tcp ftp-data 4 4 +XXXXXXXXXX.XXXXXX CtPZjS20MLrsMUOJi2 199.233.217.249 61920 141.142.220.235 33582 40 tcp ftp-data 5 3 +XXXXXXXXXX.XXXXXX CUM0KZ3MLUfNB0cl11 199.233.217.249 61918 141.142.220.235 37835 60 tcp ftp-data 5 3 +XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 141.142.220.235 50003 199.233.217.249 21 0 tcp ftp 38 25 diff --git a/testing/btest/Baseline/plugins.connkey/output b/testing/btest/Baseline/plugins.connkey/output new file mode 100644 index 0000000000..92385ac789 --- /dev/null +++ b/testing/btest/Baseline/plugins.connkey/output @@ -0,0 +1,15 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +Demo::Foo - A Foo ConnKey factory (dynamic, version 1.0.0) + [ConnKey Factory] Foo (CONNKEY_FOO, enabled) + +=== +DoNewConnKey (0 key all_inits) +DoNewConnKey (1 key all_inits) +DoConnKeyFromVal for [orig_h=141.142.220.235, orig_p=50003/tcp, resp_h=199.233.217.249, resp_p=21/tcp, proto=6, inits=0] +DoNewConnKey (2 key all_inits) +DoConnKeyFromVal for [orig_h=141.142.220.235, orig_p=50003/tcp, resp_h=199.233.217.249, resp_p=21/tcp, proto=6, inits=0] +DoNewConnKey (6 key all_inits) +DoNewConnKey (22 key all_inits) +DoNewConnKey (40 key all_inits) +DoNewConnKey (60 key all_inits) +DoNewConnKey (78 key all_inits) diff --git a/testing/btest/plugins/connkey-plugin/.btest-ignore b/testing/btest/plugins/connkey-plugin/.btest-ignore new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testing/btest/plugins/connkey-plugin/CMakeLists.txt b/testing/btest/plugins/connkey-plugin/CMakeLists.txt new file mode 100644 index 0000000000..5945ecac62 --- /dev/null +++ b/testing/btest/plugins/connkey-plugin/CMakeLists.txt @@ -0,0 +1,15 @@ +project(Zeek-Plugin-Demo-Foo) + +cmake_minimum_required(VERSION 3.15) + +if (NOT ZEEK_DIST) + message(FATAL_ERROR "ZEEK_DIST not set") +endif () + +set(CMAKE_MODULE_PATH ${ZEEK_DIST}/cmake) + +include(ZeekPlugin) + +zeek_add_plugin( + Demo Foo + SOURCES src/Plugin.cc src/Foo.cc) diff --git a/testing/btest/plugins/connkey-plugin/src/Foo.cc b/testing/btest/plugins/connkey-plugin/src/Foo.cc new file mode 100644 index 0000000000..4ab9cfdaa6 --- /dev/null +++ b/testing/btest/plugins/connkey-plugin/src/Foo.cc @@ -0,0 +1,45 @@ + +#include "Foo.h" + +#include +#include + +#include "zeek/Desc.h" +#include "zeek/Val.h" +#include "zeek/iosource/Packet.h" +#include "zeek/packet_analysis/protocol/ip/conn_key/IPBasedConnKey.h" +#include "zeek/session/Key.h" + +using namespace btest::plugin::Demo_Foo; + +namespace { + +// Just track how often DoInit() was called for baselining. +int all_inits = 0; + +class MyConnKey : public zeek::IPConnKey { +public: + MyConnKey(int inits) : zeek::IPConnKey(), inits(inits) {} + + void DoInit(const zeek::Packet& pkt) override { ++all_inits; } + + void DoPopulateConnIdVal(zeek::RecordVal& rv) override { + static int offset = rv.GetType()->FieldOffset("inits"); + rv.Assign(offset, zeek::make_intrusive(inits)); + } + +private: + int inits; +}; + +} // namespace + +zeek::ConnKeyPtr FooFactory::DoNewConnKey() const { + std::printf("DoNewConnKey (%d key all_inits)\n", all_inits); + return std::make_unique(all_inits); +} +zeek::expected FooFactory::DoConnKeyFromVal(const zeek::Val& v) const { + std::printf("DoConnKeyFromVal for %s\n", zeek::obj_desc_short(&v).c_str()); + return zeek::conn_key::fivetuple::Factory::DoConnKeyFromVal(v); +} +zeek::conn_key::FactoryPtr FooFactory::Instantiate() { return std::make_unique(); } diff --git a/testing/btest/plugins/connkey-plugin/src/Foo.h b/testing/btest/plugins/connkey-plugin/src/Foo.h new file mode 100644 index 0000000000..c1c5fa79bf --- /dev/null +++ b/testing/btest/plugins/connkey-plugin/src/Foo.h @@ -0,0 +1,25 @@ +#pragma once + +#include "zeek/IntrusivePtr.h" +#include "zeek/conn_key/Factory.h" +#include "zeek/packet_analysis/protocol/ip/conn_key/fivetuple/Factory.h" + +namespace zeek { +class Val; +using ValPtr = zeek::IntrusivePtr; +} // namespace zeek + +namespace btest::plugin::Demo_Foo { + +class FooFactory : public zeek::conn_key::fivetuple::Factory { +public: + static zeek::conn_key::FactoryPtr Instantiate(); + +protected: + zeek::ConnKeyPtr DoNewConnKey() const override; + zeek::expected DoConnKeyFromVal(const zeek::Val& v) const override; + +private: +}; + +} // namespace btest::plugin::Demo_Foo diff --git a/testing/btest/plugins/connkey-plugin/src/Plugin.cc b/testing/btest/plugins/connkey-plugin/src/Plugin.cc new file mode 100644 index 0000000000..0bccf92ee3 --- /dev/null +++ b/testing/btest/plugins/connkey-plugin/src/Plugin.cc @@ -0,0 +1,24 @@ + +#include "Plugin.h" + +#include "zeek/conn_key/Component.h" + +#include "Foo.h" + +namespace btest::plugin::Demo_Foo { +Plugin plugin; +} + +using namespace btest::plugin::Demo_Foo; + +zeek::plugin::Configuration Plugin::Configure() { + AddComponent(new zeek::conn_key::Component("Foo", btest::plugin::Demo_Foo::FooFactory::Instantiate)); + + zeek::plugin::Configuration config; + config.name = "Demo::Foo"; + config.description = "A Foo ConnKey factory"; + config.version.major = 1; + config.version.minor = 0; + config.version.patch = 0; + return config; +} diff --git a/testing/btest/plugins/connkey-plugin/src/Plugin.h b/testing/btest/plugins/connkey-plugin/src/Plugin.h new file mode 100644 index 0000000000..a4f6c4e808 --- /dev/null +++ b/testing/btest/plugins/connkey-plugin/src/Plugin.h @@ -0,0 +1,15 @@ + +#pragma once + +#include "zeek/plugin/Plugin.h" + +namespace btest::plugin::Demo_Foo { + +class Plugin : public zeek::plugin::Plugin { +protected: + zeek::plugin::Configuration Configure() override; +}; + +extern Plugin plugin; + +} // namespace btest::plugin::Demo_Foo diff --git a/testing/btest/plugins/connkey.zeek b/testing/btest/plugins/connkey.zeek new file mode 100644 index 0000000000..eb643f8e88 --- /dev/null +++ b/testing/btest/plugins/connkey.zeek @@ -0,0 +1,16 @@ +# @TEST-EXEC: ${DIST}/auxil/zeek-aux/plugin-support/init-plugin -u . Demo Foo +# @TEST-EXEC: cp -r %DIR/connkey-plugin/* . +# @TEST-EXEC: ./configure --zeek-dist=${DIST} && make +# @TEST-EXEC: ZEEK_PLUGIN_PATH=`pwd` zeek -NN Demo::Foo >>output +# @TEST-EXEC: echo === >>output +# @TEST-EXEC: ZEEK_PLUGIN_PATH=`pwd` zeek -r $TRACES/ftp/ipv4.trace %INPUT >>output +# @TEST-EXEC: zeek-cut -m ts uid id.orig_h id.orig_p id.resp_h id.resp_p id.inits proto service orig_pkts resp_pkts < conn.log > conn.log.cut +# @TEST-EXEC: btest-diff conn.log.cut +# @TEST-EXEC: btest-diff output + + +redef ConnKey::factory = ConnKey::CONNKEY_FOO; + +redef record conn_id += { + inits: int &log &default=-1; # Number of inits happened until the key was created. Not part of the hash, just metadata. +};