Add a VLAN-aware flow tuple implementation.

This is a first "real" implementation of a custom tuple, adding additional
fields over the standard five-tuple.

Includes test cases.

scripts/policy/frameworks/conn_key/vlan_fivetuple.zeek

@@ -0,0 +1,14 @@
+##! This script adapts Zeek's connection key to include 802.1Q VLAN and
+##! Q-in-Q tags, when available. Zeek normally ignores VLAN tags for connection
+##! lookups; this change makes it factor them in and also makes those VLAN tags
+##! part of the :zeek:see:`conn_id` record.
+
+redef record conn_id += {
+	## The outer VLAN for this connection, if applicable.
+	vlan: int      &log &optional;
+
+	## The inner VLAN for this connection, if applicable.
+	inner_vlan: int      &log &optional;
+};
+
+redef ConnKey::factory = ConnKey::CONNKEY_VLAN_FIVETUPLE;

scripts/test-all-policy.zeek

@@ -114,6 +114,7 @@
 @load protocols/conn/mac-logging.zeek
 @load protocols/conn/vlan-logging.zeek
 @load protocols/conn/weirds.zeek
+#@load frameworks/conn_key/vlan_fivetuple.zeek
 #@load protocols/conn/speculative-service.zeek
 @load protocols/dhcp/msg-orig.zeek
 @load protocols/dhcp/software.zeek

scripts/zeekygen/__load__.zeek

@@ -2,6 +2,7 @@
 
 # Scripts which are commented out in test-all-policy.zeek.
 @load frameworks/analyzer/deprecated-dpd-log.zeek
+@load frameworks/conn_key/vlan_fivetuple.zeek
 
 # Remove in v8.1: replaced by frameworks/analyzer/detect-protocols.zeek
 @pragma push ignore-deprecations

src/packet_analysis/protocol/ip/conn_key/CMakeLists.txt

@@ -1,3 +1,4 @@
 zeek_add_subdir_library(connkey-ip SOURCES IPBasedConnKey.cc)
 
 add_subdirectory(fivetuple)
+add_subdirectory(vlan_fivetuple)

src/packet_analysis/protocol/ip/conn_key/vlan_fivetuple/CMakeLists.txt

@@ -0,0 +1,3 @@
+zeek_add_plugin(
+    Zeek Conntuple_VLAN
+    SOURCES Factory.cc Plugin.cc)

src/packet_analysis/protocol/ip/conn_key/vlan_fivetuple/Factory.cc

@@ -0,0 +1,129 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#include "zeek/packet_analysis/protocol/ip/conn_key/vlan_fivetuple/Factory.h"
+
+#include <memory>
+
+#include "zeek/ID.h"
+#include "zeek/Val.h"
+#include "zeek/iosource/Packet.h"
+#include "zeek/packet_analysis/protocol/ip/conn_key/IPBasedConnKey.h"
+#include "zeek/packet_analysis/protocol/ip/conn_key/fivetuple/Factory.h"
+#include "zeek/util-types.h"
+
+namespace zeek::conn_key::vlan_fivetuple {
+
+class IPVlanConnKey : public zeek::IPBasedConnKey {
+public:
+    /**
+     * Constructor.
+     *
+     * Fill any holes in the key struct as we use the full tuple as a key.
+     */
+    IPVlanConnKey() { memset(static_cast<void*>(&key), 0, sizeof(key)); }
+
+    /**
+     * @copydoc
+     */
+    detail::PackedConnTuple& PackedTuple() override { return key.tuple; }
+
+    /**
+     * @copydoc
+     */
+    const detail::PackedConnTuple& PackedTuple() const override { return key.tuple; }
+
+protected:
+    zeek::session::detail::Key DoSessionKey() const override {
+        return {reinterpret_cast<const void*>(&key), sizeof(key), session::detail::Key::CONNECTION_KEY_TYPE};
+    }
+
+    void DoPopulateConnIdVal(RecordVal& conn_id) override {
+        if ( conn_id.NumFields() <= 5 )
+            return;
+
+        // Nothing to do if we have no VLAN tags at all.
+        if ( key.vlan == 0 && key.inner_vlan == 0 )
+            return;
+
+        auto [vlan_offset, inner_vlan_offset] = GetConnIdFieldOffsets();
+
+        if ( key.vlan && vlan_offset >= 0 )
+            conn_id.Assign(vlan_offset, static_cast<int>(key.vlan));
+        if ( key.inner_vlan && inner_vlan_offset >= 0 )
+            conn_id.Assign(inner_vlan_offset, static_cast<int>(key.inner_vlan));
+    };
+
+    std::pair<int, int> GetConnIdFieldOffsets() {
+        static int vlan_offset = -2, inner_vlan_offset = -2;
+
+        if ( vlan_offset == -2 && inner_vlan_offset == -2 ) {
+            vlan_offset = id::conn_id->FieldOffset("vlan");
+            if ( vlan_offset < 0 || id::conn_id->GetFieldType(vlan_offset)->Tag() != TYPE_INT )
+                vlan_offset = -1;
+
+            inner_vlan_offset = id::conn_id->FieldOffset("inner_vlan");
+            if ( inner_vlan_offset < 0 || id::conn_id->GetFieldType(inner_vlan_offset)->Tag() != TYPE_INT )
+                inner_vlan_offset = -1;
+        }
+
+        return {vlan_offset, inner_vlan_offset};
+    }
+
+protected:
+    void DoInit(const Packet& pkt) override {
+        key.vlan = pkt.vlan;
+        key.inner_vlan = pkt.inner_vlan;
+    }
+
+private:
+    friend class Factory;
+
+    // Key bytes:
+    struct {
+        struct detail::PackedConnTuple tuple;
+        // Add 802.1Q vlan tags to connection tuples. The tag representation
+        // here is as in the Packet class (where it's oddly 32-bit), since
+        // that's where we learn the tag values from. 0 indicates absence.
+        uint32_t vlan;
+        uint32_t inner_vlan;
+    } __attribute__((packed, aligned)) key;
+};
+
+zeek::ConnKeyPtr Factory::DoNewConnKey() const { return std::make_unique<IPVlanConnKey>(); }
+
+zeek::expected<zeek::ConnKeyPtr, std::string> Factory::DoConnKeyFromVal(const zeek::Val& v) const {
+    auto ck = zeek::conn_key::fivetuple::Factory::DoConnKeyFromVal(v);
+
+    if ( ! ck.has_value() )
+        return ck;
+
+    auto* k = static_cast<IPVlanConnKey*>(ck.value().get());
+    auto rt = v.GetType()->AsRecordType();
+    auto vl = v.AsRecordVal();
+
+    int vlan_offset, inner_vlan_offset;
+    if ( rt == id::conn_id ) {
+        std::tie(vlan_offset, inner_vlan_offset) = k->GetConnIdFieldOffsets();
+    }
+    else {
+        // We don't know what we've been passed.
+        vlan_offset = rt->FieldOffset("vlan");
+        inner_vlan_offset = rt->FieldOffset("inner_vlan");
+    }
+
+    if ( vlan_offset < 0 || inner_vlan_offset < 0 )
+        return zeek::unexpected<std::string>{"missing vlan or inner_vlan field"};
+
+    if ( rt->GetFieldType(vlan_offset)->Tag() != TYPE_INT || rt->GetFieldType(inner_vlan_offset)->Tag() != TYPE_INT )
+        return zeek::unexpected<std::string>{"vlan or inner_vlan field not of type int"};
+
+    if ( vl->HasField(vlan_offset) )
+        k->key.vlan = vl->GetFieldAs<zeek::IntVal>(vlan_offset);
+
+    if ( vl->HasField(inner_vlan_offset) )
+        k->key.inner_vlan = vl->GetFieldAs<zeek::IntVal>(inner_vlan_offset);
+
+    return ck;
+}
+
+} // namespace zeek::conn_key::vlan_fivetuple

src/packet_analysis/protocol/ip/conn_key/vlan_fivetuple/Factory.h

@@ -0,0 +1,33 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+#pragma once
+
+#include "zeek/ConnKey.h"
+#include "zeek/packet_analysis/protocol/ip/conn_key/fivetuple/Factory.h"
+
+namespace zeek::conn_key::vlan_fivetuple {
+
+class Factory : public zeek::conn_key::fivetuple::Factory {
+public:
+    static zeek::conn_key::FactoryPtr Instantiate() { return std::make_unique<Factory>(); }
+
+private:
+    /**
+     * Instantiates a clean ConnKey derivative and returns it.
+     *
+     * @return A unique pointer to the ConnKey instance.
+     */
+    zeek::ConnKeyPtr DoNewConnKey() const override;
+
+    /**
+     * Instantiates a filled-in ConnKey derivative from a script-layer
+     * record, usually a conn_id instance. Implementations are free to
+     * implement this liberally, i.e. the input does not _have_ to be a
+     * conn_id.
+     *
+     * @param v The script-layer value providing key input.
+     * @return A unique pointer to the ConnKey instance, or an error message.
+     */
+    zeek::expected<zeek::ConnKeyPtr, std::string> DoConnKeyFromVal(const zeek::Val& v) const override;
+};
+
+} // namespace zeek::conn_key::vlan_fivetuple

src/packet_analysis/protocol/ip/conn_key/vlan_fivetuple/Plugin.cc

@@ -0,0 +1,24 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#include "zeek/plugin/Plugin.h"
+
+#include "zeek/conn_key/Component.h"
+#include "zeek/packet_analysis/protocol/ip/conn_key/vlan_fivetuple/Factory.h"
+
+namespace zeek::plugin::Zeek_ConnKey_VLAN {
+
+class Plugin : public zeek::plugin::Plugin {
+public:
+    zeek::plugin::Configuration Configure() override {
+        AddComponent(new conn_key::Component("VLAN_FIVETUPLE", zeek::conn_key::vlan_fivetuple::Factory::Instantiate));
+
+        zeek::plugin::Configuration config;
+        config.name = "Zeek::ConnKey_Vlan_Fivetuple";
+        config.description = "ConnKey factory for 802.1Q VLAN/Q-in-Q + IP/port/proto five-tuples";
+        return config;
+    }
+};
+
+Plugin plugin;
+
+} // namespace zeek::plugin::Zeek_ConnKey_VLAN

testing/btest/Baseline/scripts.policy.frameworks.conn_key.vlan_fivetuple-2/conn.log.cut

@@ -0,0 +1,5 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	id.vlan	id.inner_vlan	orig_pkts	resp_pkts	service
+XXXXXXXXXX.XXXXXX	CHhAvVGS1DHFjwGM9	141.142.228.5	59856	192.150.187.43	80	-	-	7	7	http
+XXXXXXXXXX.XXXXXX	C4J4Th3PJpwUYZZ6gc	141.142.228.5	59856	192.150.187.43	80	10	20	7	7	http
+XXXXXXXXXX.XXXXXX	ClEkJM2Vm5giqnMf4h	141.142.228.5	59856	192.150.187.43	80	42	-	7	7	http

testing/btest/Baseline/scripts.policy.frameworks.conn_key.vlan_fivetuple-3/conn.log.cut

@@ -0,0 +1,5 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p			orig_pkts	resp_pkts	service
+XXXXXXXXXX.XXXXXX	CHhAvVGS1DHFjwGM9	141.142.228.5	59856	192.150.187.43	80			7	7	http
+XXXXXXXXXX.XXXXXX	C4J4Th3PJpwUYZZ6gc	141.142.228.5	59856	192.150.187.43	80			7	7	http
+XXXXXXXXXX.XXXXXX	ClEkJM2Vm5giqnMf4h	141.142.228.5	59856	192.150.187.43	80			7	7	http

testing/btest/Baseline/scripts.policy.frameworks.conn_key.vlan_fivetuple-4/conn.log.cut

@@ -0,0 +1,5 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	id.vlan	id.inner_vlan	orig_pkts	resp_pkts	service
+XXXXXXXXXX.XXXXXX	CHhAvVGS1DHFjwGM9	141.142.228.5	59856	192.150.187.43	80	-	-	7	7	http
+XXXXXXXXXX.XXXXXX	C4J4Th3PJpwUYZZ6gc	141.142.228.5	59856	192.150.187.43	80	10	20	7	7	http
+XXXXXXXXXX.XXXXXX	ClEkJM2Vm5giqnMf4h	141.142.228.5	59856	192.150.187.43	80	42	-	7	7	http

testing/btest/Baseline/scripts.policy.frameworks.conn_key.vlan_fivetuple-5/conn.log.cut

@@ -0,0 +1,5 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	id.vlan	id.inner_vlan	orig_pkts	resp_pkts	service
+XXXXXXXXXX.XXXXXX	CHhAvVGS1DHFjwGM9	141.142.228.5	59856	192.150.187.43	80	-	-	7	7	http
+XXXXXXXXXX.XXXXXX	C4J4Th3PJpwUYZZ6gc	141.142.228.5	59856	192.150.187.43	80	10	20	7	7	http
+XXXXXXXXXX.XXXXXX	ClEkJM2Vm5giqnMf4h	141.142.228.5	59856	192.150.187.43	80	42	-	7	7	http

testing/btest/Baseline/scripts.policy.frameworks.conn_key.vlan_fivetuple-6/conn.log.cut

@@ -0,0 +1,5 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	id.vlan	id.inner_vlan	orig_pkts	resp_pkts	service
+XXXXXXXXXX.XXXXXX	CHhAvVGS1DHFjwGM9	141.142.228.5	59856	192.150.187.43	80	-	-	7	7	http
+XXXXXXXXXX.XXXXXX	C4J4Th3PJpwUYZZ6gc	141.142.228.5	59856	192.150.187.43	80	-	-	7	7	http
+XXXXXXXXXX.XXXXXX	ClEkJM2Vm5giqnMf4h	141.142.228.5	59856	192.150.187.43	80	-	-	7	7	http

testing/btest/Baseline/scripts.policy.frameworks.conn_key.vlan_fivetuple/conn.log.cut

@@ -0,0 +1,3 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p			orig_pkts	resp_pkts	service
+XXXXXXXXXX.XXXXXX	CHhAvVGS1DHFjwGM9	141.142.228.5	59856	192.150.187.43	80			21	21	http

testing/btest/scripts/policy/frameworks/conn_key/vlan_fivetuple.zeek

@@ -0,0 +1,56 @@
+# @TEST-DOC: Verify VLAN-aware flow tuples on colliding traffic.
+#
+# The test pcap has 3 overlapping healthy TCP connections, each with different VLAN tagging: none, one VLAN tag, two VLAN tags.
+# To create: tcprewrite --enet-vlan=add --enet-vlan-tag 20 --enet-vlan-cfi=1 --enet-vlan-pri=2 -i in.pcap -o out.pcap
+#
+# @TEST-EXEC: zeek -r $TRACES/vlan-collisions.pcap %INPUT
+# @TEST-EXEC: zeek-cut -m ts uid id.orig_h id.orig_p id.resp_h id.resp_p id.vlan id.inner_vlan orig_pkts resp_pkts service <conn.log >conn.log.cut
+# @TEST-EXEC: btest-diff conn.log.cut
+
+# Default operation: Zeek isn't VLAN-aware, a single conn.log entry results.
+
+# @TEST-START-NEXT
+
+# Switch to VLAN-aware flow tuples: multiple conn.log entries with full
+# information.
+
+@load frameworks/conn_key/vlan_fivetuple
+
+# @TEST-START-NEXT
+
+# Leave out the conn_id redef: Zeek still distinguishes flows so multiple
+# conn.log entries result, but conn.log doesn't show the VLAN fields.
+
+redef ConnKey::factory = ConnKey::CONNKEY_VLAN_FIVETUPLE;
+
+# @TEST-START-NEXT
+
+# Add an extra field before the VLAN ones, to throw off any fixed-offset code.
+
+redef record conn_id += {
+	foo: int &default=1;
+};
+
+@load frameworks/conn_key/vlan_fivetuple
+
+# @TEST-START-NEXT
+
+# Add the right fields, but in the wrong order. (zeek-cut obscures the difference.)
+
+redef record conn_id += {
+	inner_vlan: int      &log &optional;
+	vlan: int      &log &optional;
+};
+
+redef ConnKey::factory = ConnKey::CONNKEY_VLAN_FIVETUPLE;
+
+# @TEST-START-NEXT
+
+# Add the right fields, but with the wrong types.
+
+redef record conn_id += {
+	vlan: string      &log &optional;
+	inner_vlan: string      &log &optional;
+};
+
+redef ConnKey::factory = ConnKey::CONNKEY_VLAN_FIVETUPLE;