Establish plugin infrastructure for ConnKey factories.

ConnKey factories are intermediaries that encapsulate the details of how to
instantiate ConnKeys, which codify the hash input for connection lookups.

scripts/base/init-bare.zeek

@@ -629,6 +629,19 @@ export {
 	const add_missing_remote_network_timestamp: bool = F &redef;
 }
 
+module ConnKey;
+
+export {
+	## The connection key factory to use for Zeek's internal connection
+	## tracking. This is a ``ConnKey::Tag`` plugin component enum value,
+	## and the default is Zeek's traditional 5-tuple-tracking based on
+	## IP/port endpoint pairs, plus transport protocol. Plugins can provide
+	## their own implementation. You'll usually not adjust this value in
+	## isolation, but with a corresponding redef of the :zeek:type:`conn_id`
+	## record to represent additional connection tuple members.
+	const factory = ConnKey::CONNKEY_FIVETUPLE &redef;
+}
+
 module FTP;
 
 export {

src/CMakeLists.txt

@@ -197,6 +197,7 @@ gen_zam_target(${GEN_ZAM_SRC_DIR})
 option(USE_SQLITE "Should Zeek use SQLite?" ON)
 
 add_subdirectory(analyzer)
+add_subdirectory(conn_key)
 add_subdirectory(cluster)
 add_subdirectory(packet_analysis)
 add_subdirectory(broker)

src/conn_key/CMakeLists.txt

@@ -0,0 +1 @@
+zeek_add_subdir_library(connkey SOURCES Factory.h Component.cc Manager.cc)

src/conn_key/Component.cc

@@ -0,0 +1,27 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#include "zeek/conn_key/Component.h"
+
+#include "zeek/Desc.h"
+#include "zeek/conn_key/Manager.h"
+
+using namespace zeek::conn_key;
+
+Component::Component(const std::string& name, factory_callback arg_factory, Tag::subtype_t arg_subtype)
+    : plugin::Component(plugin::component::CONNKEY, name, arg_subtype, conn_key_mgr->GetTagType()),
+      factory(std::move(arg_factory)) {}
+
+void Component::Initialize() {
+    InitializeTag();
+    conn_key_mgr->RegisterComponent(this, "CONNKEY_");
+}
+
+void Component::DoDescribe(ODesc* d) const {
+    if ( factory ) {
+        d->Add("CONNKEY_");
+        d->Add(CanonicalName());
+        d->Add(", ");
+    }
+
+    d->Add(Enabled() ? "enabled" : "disabled");
+}

src/conn_key/Component.h

@@ -0,0 +1,45 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#pragma once
+
+#include <functional>
+#include <memory>
+
+#include "zeek/Tag.h"
+#include "zeek/plugin/Component.h"
+
+namespace zeek::conn_key {
+
+class Factory;
+using FactoryPtr = std::unique_ptr<Factory>;
+
+class Component : public plugin::Component {
+public:
+    using factory_callback = std::function<FactoryPtr()>;
+
+    Component(const std::string& name, factory_callback factory, zeek::Tag::subtype_t subtype = 0);
+    ~Component() override = default;
+
+    /**
+     * Initialization function. This function has to be called before any
+     * plugin component functionality is used; it is used to add the
+     * plugin component to the list of components and to initialize tags
+     */
+    void Initialize() override;
+
+    /**
+     * Returns the analyzer's factory function.
+     */
+    factory_callback Factory() const { return factory; }
+
+protected:
+    /**
+     * Overridden from plugin::Component.
+     */
+    void DoDescribe(ODesc* d) const override;
+
+private:
+    factory_callback factory; // The tuple factory's factory callback.
+};
+
+} // namespace zeek::conn_key

src/conn_key/Factory.h

@@ -0,0 +1,62 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+#pragma once
+
+#include "zeek/ConnKey.h"
+#include "zeek/util-types.h"
+
+namespace zeek {
+
+class Packet;
+class RecordVal;
+using RecordValPtr = IntrusivePtr<RecordVal>;
+
+namespace conn_key {
+
+class Factory;
+using FactoryPtr = std::unique_ptr<Factory>;
+
+/**
+ * ConnKey factories instantiate derivatives of ConnKeys, to provide pluggable flow hashing.
+ */
+class Factory {
+public:
+    virtual ~Factory() = default;
+
+    /**
+     * Instantiates a clean ConnKey derivative and returns it.
+     *
+     * @return A unique pointer to the ConnKey instance.
+     */
+    zeek::ConnKeyPtr NewConnKey() const { return DoNewConnKey(); }
+
+    /**
+     * Instantiates a filled-in ConnKey derivative from a script-layer
+     * record, usually a conn_id instance. Implementations are free to
+     * implement this liberally, i.e. the input does not _have_ to be a
+     * conn_id.
+     *
+     * @param v The script-layer value providing key input.
+     * @return A unique pointer to the ConnKey instance, or an error message.
+     */
+    zeek::expected<zeek::ConnKeyPtr, std::string> ConnKeyFromVal(const zeek::Val& v) const {
+        return DoConnKeyFromVal(v);
+    }
+
+protected:
+    /**
+     * Hook for Factory::NewConnKey.
+     *
+     * @return A unique pointer to the ConnKey instance.
+     */
+    virtual zeek::ConnKeyPtr DoNewConnKey() const = 0;
+
+    /**
+     * Hook for Factory::ConnKeyFromVal.
+     *
+     * @return A unique pointer to the ConnKey instance, or an error message.
+     */
+    virtual zeek::expected<zeek::ConnKeyPtr, std::string> DoConnKeyFromVal(const zeek::Val& v) const = 0;
+};
+
+} // namespace conn_key
+} // namespace zeek

src/conn_key/Manager.cc

@@ -0,0 +1,39 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#include "zeek/conn_key/Manager.h"
+
+#include "zeek/conn_key/Component.h"
+
+using namespace zeek::conn_key;
+
+Manager::Manager() : plugin::ComponentManager<zeek::conn_key::Component>("ConnKey", "Tag") {}
+
+void Manager::InitPostScript() {
+    const auto& factory_val = id::find_val<zeek::EnumVal>("ConnKey::factory");
+    factory = InstantiateFactory(factory_val);
+}
+
+FactoryPtr Manager::InstantiateFactory(const zeek::EnumValPtr& tag) {
+    Component* c = Lookup(tag);
+
+    if ( ! c ) {
+        reporter->FatalError(
+            "request to instantiate unknown connection tuple factory %s, please review ConnTuple::factory value",
+            tag->GetType()->AsEnumType()->Lookup(tag->Get()));
+    }
+
+    if ( ! c->Factory() ) {
+        reporter->FatalError("factory %s cannot be instantiated dynamically", GetComponentName(tag).c_str());
+    }
+
+    FactoryPtr factory = c->Factory()();
+
+    if ( ! factory ) {
+        reporter->FatalError("factory instantiation failed");
+    }
+
+    // Could add validation of actual tag vs obtained one here, as we do e.g. in
+    // the packet_analysis Manager.
+
+    return factory;
+}

src/conn_key/Manager.h

@@ -0,0 +1,57 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#pragma once
+
+#include "zeek/Tag.h"
+#include "zeek/conn_key/Component.h"
+#include "zeek/conn_key/Factory.h"
+#include "zeek/plugin/Component.h"
+#include "zeek/plugin/ComponentManager.h"
+
+namespace zeek {
+
+namespace conn_key {
+
+/**
+ * This component manager is for registration of pluggable ConnKey factories
+ * that provide a zeek::plugin::component::CONNKEY component.
+ */
+class Manager : public plugin::ComponentManager<conn_key::Component> {
+public:
+    /**
+     * Constructor.
+     */
+    Manager();
+
+    /**
+     * Destructor.
+     */
+    ~Manager() = default;
+
+    /**
+     * Hook called during Zeek's startup sequence at InitPostScript() time.
+     */
+    void InitPostScript();
+
+    /**
+     * Return the instantiated Factory selected by the @c ConnKey::factory script-level variable.
+     *
+     * @return A reference to the selected see Factory.
+     */
+    Factory& GetFactory() { return *factory; }
+
+private:
+    /**
+     * @return A pointer to a Factory given @arg tag.
+     */
+    FactoryPtr InstantiateFactory(const EnumValPtr& tag);
+
+    FactoryPtr factory;
+};
+
+} // namespace conn_key
+
+extern zeek::conn_key::Manager* conn_key_mgr;
+
+
+} // namespace zeek

src/plugin/Component.cc

@@ -52,6 +52,8 @@ void Component::Describe(ODesc* d) const {
 
         case component::STORAGE_SERIALIZER: d->Add("Storage Serializer"); break;
 
+        case component::CONNKEY: d->Add("ConnKey Factory"); break;
+
         default:
             reporter->InternalWarning("unknown component type in plugin::Component::Describe");
             d->Add("<unknown component type>");

src/plugin/Component.h

@@ -38,6 +38,7 @@ enum Type : uint8_t {
     LOG_SERIALIZER,     /// A serializer for log batches, used by cluster backends.
     STORAGE_BACKEND,    /// A backend for the storage framework.
     STORAGE_SERIALIZER, /// A serializer for the storage framework.
+    CONNKEY,            /// A factory for connection keys.
 };
 
 } // namespace component

src/zeek-setup.cc

@@ -54,6 +54,7 @@
 #include "zeek/broker/Manager.h"
 #include "zeek/cluster/Backend.h"
 #include "zeek/cluster/Manager.h"
+#include "zeek/conn_key/Manager.h"
 #include "zeek/file_analysis/Manager.h"
 #include "zeek/input.h"
 #include "zeek/input/Manager.h"
@@ -161,6 +162,7 @@ void do_ssl_deinit() {
 
 zeek::ValManager* zeek::val_mgr = nullptr;
 zeek::packet_analysis::Manager* zeek::packet_mgr = nullptr;
+zeek::conn_key::Manager* zeek::conn_key_mgr = nullptr;
 zeek::analyzer::Manager* zeek::analyzer_mgr = nullptr;
 zeek::plugin::Manager* zeek::plugin_mgr = nullptr;
 
@@ -408,6 +410,7 @@ static void terminate_zeek() {
 
     delete zeekygen_mgr;
     delete packet_mgr;
+    delete conn_key_mgr;
     delete analyzer_mgr;
     delete file_mgr;
     delete cluster::manager;
@@ -690,6 +693,7 @@ SetupResult setup(int argc, char** argv, Options* zopts) {
     iosource_mgr = new iosource::Manager();
     event_registry = new EventRegistry();
     packet_mgr = new packet_analysis::Manager();
+    conn_key_mgr = new conn_key::Manager();
     analyzer_mgr = new analyzer::Manager();
     log_mgr = new logging::Manager();
     input_mgr = new input::Manager();
@@ -838,6 +842,7 @@ SetupResult setup(int argc, char** argv, Options* zopts) {
 
         RecordType::InitPostScript();
 
+        conn_key_mgr->InitPostScript();
         telemetry_mgr->InitPostScript();
         thread_mgr->InitPostScript();
         iosource_mgr->InitPostScript();

src/zeekygen/ScriptInfo.cc

@@ -309,6 +309,10 @@ void ScriptInfo::DoInitPostScript() {
         const auto& id = zeek::detail::global_scope()->Find("Input::Reader");
         types.push_back(new IdentifierInfo(id, this));
     }
+    else if ( name == "base/init-bare.zeek" ) {
+        const auto& id = zeek::detail::global_scope()->Find("ConnKey::Tag");
+        types.push_back(new IdentifierInfo(id, this));
+    }
     else if ( name == "base/frameworks/logging/main.zeek" ) {
         const auto& id = zeek::detail::global_scope()->Find("Log::Writer");
         types.push_back(new IdentifierInfo(id, this));