Establish plugin infrastructure for connection tuple "builders".

Builders are intermediaries that encapsulate the details of how to instantiate
connection tuples & keys. By virtualizing those data structures, builder
implementations can adapt Zeek's notion of connection tuples.

scripts/base/init-bare.zeek

@@ -579,6 +579,19 @@ const io_poll_interval_live = 10 &redef;
 ## while testing, but should be used sparingly.
 const running_under_test: bool = F &redef;
 
+module ConnTuple;
+
+export {
+	## The connection tuple builder to use for Zeek's internal flow
+	## tracking. This is a ``ConnTuple::Tag`` plugin component enum value,
+	## and the default is 5-tuple-tracking based on IP/port endpoint pairs,
+	## plus transport protocol. Plugins can provide their own
+	## implementation. You'll usually not adjust this value in isolation,
+	## but with a corresponding redef of the :zeek:type:`conn_id` record to
+	## represent additional tuple members.
+	const builder = ConnTuple::CONNTUPLE_FIVETUPLE &redef;
+}
+
 module FTP;
 
 export {

src/CMakeLists.txt

@@ -194,6 +194,7 @@ gen_zam_target(${GEN_ZAM_SRC_DIR})
 option(USE_SQLITE "Should Zeek use SQLite?" ON)
 
 add_subdirectory(analyzer)
+add_subdirectory(conntuple)
 add_subdirectory(cluster)
 add_subdirectory(packet_analysis)
 add_subdirectory(broker)

src/conntuple/Builder.cc

@@ -0,0 +1,23 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#include "zeek/conntuple/Builder.h"
+
+#include "zeek/Conn.h"
+#include "zeek/ID.h"
+#include "zeek/IPAddr.h"
+#include "zeek/session/Session.h"
+
+namespace zeek::conntuple {
+
+Builder::Builder() {}
+Builder::~Builder() {}
+
+ConnTuplePtr Builder::GetTuple(const Packet* pkt) { return std::make_shared<ConnTuple>(); }
+
+zeek::detail::ConnKeyPtr Builder::GetKey(const ConnTuple& tuple) {
+    return std::make_shared<zeek::detail::ConnKey>(tuple);
+}
+
+zeek::detail::ConnKeyPtr Builder::GetKey(Val* v) { return std::make_shared<zeek::detail::ConnKey>(v); }
+
+} // namespace zeek::conntuple

src/conntuple/Builder.h

@@ -0,0 +1,35 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+#pragma once
+
+#include "zeek/Conn.h"
+#include "zeek/IPAddr.h"
+#include "zeek/conntuple/Manager.h"
+
+namespace zeek {
+
+class Packet;
+class RecordVal;
+using RecordValPtr = IntrusivePtr<RecordVal>;
+
+namespace conntuple {
+
+class Builder;
+using BuilderPtr = std::unique_ptr<Builder>;
+
+class Builder {
+public:
+    Builder();
+    virtual ~Builder();
+
+    virtual ConnTuplePtr GetTuple(const Packet* pkt);
+
+    virtual zeek::detail::ConnKeyPtr GetKey(const ConnTuple& tuple);
+    virtual zeek::detail::ConnKeyPtr GetKey(Val* v);
+
+    virtual void FillConnIdVal(detail::ConnKeyPtr key, RecordValPtr& tuple) {};
+
+    static zeek::conntuple::BuilderPtr Instantiate() { return std::make_unique<Builder>(); }
+};
+
+} // namespace conntuple
+} // namespace zeek

src/conntuple/CMakeLists.txt

@@ -0,0 +1,11 @@
+zeek_add_subdir_library(
+    conntuple
+    INCLUDE_DIRS
+    ${CMAKE_CURRENT_SOURCE_DIR}
+    ${CMAKE_CURRENT_BINARY_DIR}
+    SOURCES
+    Builder.cc
+    Component.cc
+    Manager.cc)
+
+add_subdirectory(fivetuple)

src/conntuple/Component.cc

@@ -0,0 +1,28 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#include "zeek/conntuple/Component.h"
+
+#include "zeek/Desc.h"
+#include "zeek/conntuple/Builder.h"
+#include "zeek/conntuple/Manager.h"
+
+using namespace zeek::conntuple;
+
+Component::Component(const std::string& name, factory_callback arg_factory, Tag::subtype_t arg_subtype)
+    : plugin::Component(plugin::component::CONNTUPLE, name, arg_subtype, conntuple_mgr->GetTagType()),
+      factory(std::move(arg_factory)) {}
+
+void Component::Initialize() {
+    InitializeTag();
+    conntuple_mgr->RegisterComponent(this, "CONNTUPLE_");
+}
+
+void Component::DoDescribe(ODesc* d) const {
+    if ( factory ) {
+        d->Add("CONNTUPLE_");
+        d->Add(CanonicalName());
+        d->Add(", ");
+    }
+
+    d->Add(Enabled() ? "enabled" : "disabled");
+}

src/conntuple/Component.h

@@ -0,0 +1,46 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#pragma once
+
+#include "zeek/zeek-config.h"
+
+#include <functional>
+
+#include "zeek/Tag.h"
+#include "zeek/plugin/Component.h"
+
+namespace zeek::conntuple {
+
+class Builder;
+using BuilderPtr = std::unique_ptr<Builder>;
+
+class Component : public plugin::Component {
+public:
+    using factory_callback = std::function<BuilderPtr()>;
+
+    Component(const std::string& name, factory_callback factory, zeek::Tag::subtype_t subtype = 0);
+    ~Component() override = default;
+
+    /**
+     * Initialization function. This function has to be called before any
+     * plugin component functionality is used; it is used to add the
+     * plugin component to the list of components and to initialize tags
+     */
+    void Initialize() override;
+
+    /**
+     * Returns the analyzer's factory function.
+     */
+    factory_callback Factory() const { return factory; }
+
+protected:
+    /**
+     * Overridden from plugin::Component.
+     */
+    void DoDescribe(ODesc* d) const override;
+
+private:
+    factory_callback factory; // The tuple builder's factory callback.
+};
+
+} // namespace zeek::conntuple

src/conntuple/Manager.cc

@@ -0,0 +1,37 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#include "zeek/conntuple/Manager.h"
+
+using namespace zeek::conntuple;
+
+Manager::Manager() : plugin::ComponentManager<conntuple::Component>("ConnTuple", "Tag") {}
+
+void Manager::InitPostScript() {
+    const auto& builder_val = id::find_val<zeek::EnumVal>("ConnTuple::builder");
+    builder = InstantiateBuilder(builder_val);
+}
+
+BuilderPtr Manager::InstantiateBuilder(const zeek::EnumValPtr& tag) {
+    Component* c = Lookup(tag);
+
+    if ( ! c ) {
+        reporter->FatalError(
+            "request to instantiate unknown connection tuple builder %s, please review ConnTuple::builder value",
+            tag->GetType()->AsEnumType()->Lookup(tag->Get()));
+    }
+
+    if ( ! c->Factory() ) {
+        reporter->FatalError("builder %s cannot be instantiated dynamically", GetComponentName(tag).c_str());
+    }
+
+    BuilderPtr builder = c->Factory()();
+
+    if ( ! builder ) {
+        reporter->FatalError("builder instantiation failed");
+    }
+
+    // Could add validation of actual tag vs obtained one here, as we do e.g. in
+    // the packet_analysis Manager.
+
+    return builder;
+}

src/conntuple/Manager.h

@@ -0,0 +1,33 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#pragma once
+
+#include "zeek/Tag.h"
+#include "zeek/conntuple/Builder.h"
+#include "zeek/conntuple/Component.h"
+#include "zeek/plugin/ComponentManager.h"
+
+namespace zeek {
+
+namespace conntuple {
+
+class Manager : public plugin::ComponentManager<Component> {
+public:
+    Manager();
+    ~Manager() {};
+
+    void InitPostScript();
+
+    Builder& GetBuilder() { return *builder; }
+
+private:
+    BuilderPtr InstantiateBuilder(const EnumValPtr& tag);
+
+    BuilderPtr builder; // The currently active conn tuple builder.
+};
+
+} // namespace conntuple
+
+extern zeek::conntuple::Manager* conntuple_mgr;
+
+} // namespace zeek

src/conntuple/fivetuple/CMakeLists.txt

@@ -0,0 +1,3 @@
+zeek_add_plugin(
+    Zeek Conntuple_Fivetuple
+    SOURCES Plugin.cc)

src/conntuple/fivetuple/Plugin.cc

@@ -0,0 +1,25 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#include "zeek/plugin/Plugin.h"
+
+#include "zeek/conntuple/Builder.h"
+#include "zeek/conntuple/Component.h"
+
+namespace zeek::plugin::Zeek_Conntuple_Fivetuple {
+
+class Plugin : public zeek::plugin::Plugin {
+public:
+    zeek::plugin::Configuration Configure() {
+        // The conntuple::Builder already has the default five-tuple behavior.
+        AddComponent(new conntuple::Component("Fivetuple", zeek::conntuple::Builder::Instantiate));
+
+        zeek::plugin::Configuration config;
+        config.name = "Zeek::Conntuple_Fivetuple";
+        config.description = "Conntuple builder for Zeek's default five-tuples";
+        return config;
+    }
+};
+
+Plugin plugin;
+
+} // namespace zeek::plugin::Zeek_Conntuple_Fivetuple

src/plugin/Component.cc

@@ -47,6 +47,8 @@ void Component::Describe(ODesc* d) const {
 
         case component::STORAGE_BACKEND: d->Add("Storage Backend"); break;
 
+        case component::CONNTUPLE: d->Add("Conntuple Builder"); break;
+
         default:
             reporter->InternalWarning("unknown component type in plugin::Component::Describe");
             d->Add("<unknown component type>");

src/plugin/Component.h

@@ -34,6 +34,7 @@ enum Type {
     EVENT_SERIALIZER, /// A serializer for events, used by cluster backends.
     LOG_SERIALIZER,   /// A serializer for log batches, used by cluster backends.
     STORAGE_BACKEND,  /// A backend for the storage framework.
+    CONNTUPLE,        /// A builder for connection tuples.
 };
 
 } // namespace component

src/zeek-setup.cc

@@ -52,6 +52,7 @@
 #include "zeek/broker/Manager.h"
 #include "zeek/cluster/Backend.h"
 #include "zeek/cluster/Manager.h"
+#include "zeek/conntuple/Manager.h"
 #include "zeek/file_analysis/Manager.h"
 #include "zeek/input.h"
 #include "zeek/input/Manager.h"
@@ -159,6 +160,7 @@ void do_ssl_deinit() {
 
 zeek::ValManager* zeek::val_mgr = nullptr;
 zeek::packet_analysis::Manager* zeek::packet_mgr = nullptr;
+zeek::conntuple::Manager* zeek::conntuple_mgr = nullptr;
 zeek::analyzer::Manager* zeek::analyzer_mgr = nullptr;
 zeek::plugin::Manager* zeek::plugin_mgr = nullptr;
 
@@ -400,6 +402,7 @@ static void terminate_zeek() {
 
     delete zeekygen_mgr;
     delete packet_mgr;
+    delete conntuple_mgr;
     delete analyzer_mgr;
     delete file_mgr;
     delete cluster::manager;
@@ -678,6 +681,7 @@ SetupResult setup(int argc, char** argv, Options* zopts) {
     iosource_mgr = new iosource::Manager();
     event_registry = new EventRegistry();
     packet_mgr = new packet_analysis::Manager();
+    conntuple_mgr = new conntuple::Manager();
     analyzer_mgr = new analyzer::Manager();
     log_mgr = new logging::Manager();
     input_mgr = new input::Manager();
@@ -826,6 +830,7 @@ SetupResult setup(int argc, char** argv, Options* zopts) {
 
         RecordType::InitPostScript();
 
+        conntuple_mgr->InitPostScript();
         telemetry_mgr->InitPostScript();
         thread_mgr->InitPostScript();
         iosource_mgr->InitPostScript();