Add VLAN-aware connection tuples.

Loading policy/protocols/conntuple/vlan adapts Zeek's flow hashing and the
script-layer conn_id record to show VLAN tags when present.

I'm using script-layer ints for the VLAN tag representation for consistency with
what we alrady do elsewhere, but it seems odd since they can never be negative.

I'm currently skipping protocols/conntuple/vlan in test-all-policy since it
otherwise affects the external testsuites -- could revisit if people feel it
should run on these.

scripts/policy/protocols/conntuple/vlan.zeek

@@ -0,0 +1,14 @@
+##! This script adapts Zeek's connection tuples to include 802.1Q VLAN and
+##! Q-in-Q tags, when available. Zeek normally ignores VLAN tags in its flow
+##! lookups; this change makes it factor them in and also makes those VLAN tags
+##! part of the conn_id record.
+
+redef record conn_id += {
+	## The outer VLAN for this connection, if applicable.
+	vlan: int      &log &optional;
+
+	## The inner VLAN for this connection, if applicable.
+	inner_vlan: int      &log &optional;
+};
+
+redef ConnTuple::builder = ConnTuple::CONNTUPLE_VLAN;

scripts/test-all-policy.zeek

@@ -110,6 +110,7 @@
 @load protocols/conn/mac-logging.zeek
 @load protocols/conn/vlan-logging.zeek
 @load protocols/conn/weirds.zeek
+#@load protocols/conntuple/vlan.zeek
 #@load protocols/conn/speculative-service.zeek
 @load protocols/dhcp/msg-orig.zeek
 @load protocols/dhcp/software.zeek

scripts/zeekygen/__load__.zeek

@@ -16,6 +16,7 @@
 @load frameworks/signatures/iso-9660.zeek
 @load policy/misc/dump-events.zeek
 @load policy/protocols/conn/speculative-service.zeek
+@load policy/protocols/conntuple/vlan.zeek
 
 # Remove in v8.1: This script is deprecated and conflicts with detect-sql-injection.zeek
 # @load policy/protocols/http/detect-sqli.zeek

src/conntuple/CMakeLists.txt

@@ -9,3 +9,4 @@ zeek_add_subdir_library(
     Manager.cc)
 
 add_subdirectory(fivetuple)
+add_subdirectory(vlan)

src/conntuple/vlan/Builder.cc

@@ -0,0 +1,108 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#include "zeek/conntuple/vlan/Builder.h"
+
+#include "zeek/Conn.h"
+#include "zeek/ID.h"
+#include "zeek/IPAddr.h"
+#include "zeek/session/Session.h"
+
+namespace zeek::plugin::Zeek_Conntuple_VLAN {
+
+// Add 802.1Q vlan tags to connection tuples. The tag representation here is as
+// in the Packet class, since that's where we learn the tag values from.
+struct VlanConnTuple : public ConnTuple {
+    uint32_t vlan = 0;
+    uint32_t inner_vlan = 0;
+};
+
+class VlanConnKey : public detail::ConnKey {
+public:
+    uint32_t vlan = 0;
+    uint32_t inner_vlan = 0;
+
+    VlanConnKey(const ConnTuple& conn) : detail::ConnKey(conn) {};
+    VlanConnKey(const VlanConnTuple& conn) : detail::ConnKey(conn), vlan(conn.vlan), inner_vlan(conn.inner_vlan) {}
+
+    VlanConnKey(Val* v) : detail::ConnKey(v) {
+        const auto& vt = v->GetType();
+        if ( ! IsRecord(vt->Tag()) )
+            return;
+
+        RecordType* rt = vt->AsRecordType();
+        auto vl = v->As<RecordVal*>();
+
+        // Be safe in case we get here without the expected
+        // conn_id redef from policy/protocols/conntuple/vlan.
+        if ( rt == id::conn_id && vl->HasField(5) && rt->GetFieldType(5)->Tag() == TYPE_INT )
+            vlan = vl->GetField<CountVal>(5)->AsCount();
+        if ( rt == id::conn_id && vl->HasField(6) && rt->GetFieldType(6)->Tag() == TYPE_INT )
+            inner_vlan = vl->GetField<CountVal>(6)->AsCount();
+    }
+
+    size_t PackedSize() const override {
+        // This depends on whether we actually have VLANs.
+        // We can go with the basic 5-tuple if not.
+        size_t result = detail::ConnKey::PackedSize();
+
+        if ( vlan > 0 )
+            result += sizeof(vlan);
+        if ( inner_vlan > 0 )
+            result += sizeof(inner_vlan);
+
+        return result;
+    }
+
+    size_t Pack(uint8_t* data, size_t size) const override {
+        if ( size < PackedSize() )
+            return 0;
+
+        uint8_t* ptr = data;
+
+        ptr += detail::ConnKey::Pack(data, size);
+
+        if ( vlan > 0 ) {
+            memcpy(ptr, &vlan, sizeof(vlan));
+            ptr += sizeof(vlan);
+        }
+        if ( inner_vlan > 0 ) {
+            memcpy(ptr, &inner_vlan, sizeof(inner_vlan));
+            ptr += sizeof(inner_vlan);
+        }
+
+        return ptr - data;
+    }
+};
+
+ConnTuplePtr Builder::GetTuple(const Packet* pkt) {
+    auto res = std::make_shared<VlanConnTuple>();
+    res->vlan = pkt->vlan;
+    res->inner_vlan = pkt->inner_vlan;
+    return res;
+}
+
+zeek::detail::ConnKeyPtr Builder::GetKey(const ConnTuple& tuple) {
+    const VlanConnTuple& vtuple = dynamic_cast<const VlanConnTuple&>(tuple);
+    auto res = std::make_shared<VlanConnKey>(tuple);
+    res->vlan = vtuple.vlan;
+    res->inner_vlan = vtuple.inner_vlan;
+    return res;
+}
+
+zeek::detail::ConnKeyPtr Builder::GetKey(Val* v) { return std::make_shared<VlanConnKey>(v); }
+
+void Builder::FillConnIdVal(detail::ConnKeyPtr key, RecordValPtr& tuple) {
+    if ( tuple->NumFields() <= 5 )
+        return;
+
+    RecordType* rt = tuple->GetType()->AsRecordType();
+    auto vkey = dynamic_cast<VlanConnKey*>(key.get());
+
+    // Assign only if VLAN tags are present and the record has compatible fields:
+    if ( vkey->vlan > 0 && rt->GetFieldType(5)->Tag() == TYPE_INT )
+        tuple->Assign(5, vkey->vlan);
+    if ( vkey->inner_vlan > 0 && tuple->NumFields() >= 6 && rt->GetFieldType(6)->Tag() == TYPE_INT )
+        tuple->Assign(6, vkey->inner_vlan);
+}
+
+} // namespace zeek::plugin::Zeek_Conntuple_VLAN

src/conntuple/vlan/Builder.h

@@ -0,0 +1,20 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#include "zeek/conntuple/Builder.h"
+#include "zeek/plugin/Plugin.h"
+
+namespace zeek::plugin::Zeek_Conntuple_VLAN {
+
+class Builder : public conntuple::Builder {
+public:
+    ConnTuplePtr GetTuple(const Packet* pkt) override;
+
+    zeek::detail::ConnKeyPtr GetKey(const ConnTuple& tuple) override;
+    zeek::detail::ConnKeyPtr GetKey(Val* v) override;
+
+    void FillConnIdVal(detail::ConnKeyPtr key, RecordValPtr& tuple) override;
+
+    static zeek::conntuple::BuilderPtr Instantiate() { return std::make_unique<Builder>(); }
+};
+
+} // namespace zeek::plugin::Zeek_Conntuple_VLAN

src/conntuple/vlan/CMakeLists.txt

@@ -0,0 +1,3 @@
+zeek_add_plugin(
+    Zeek Conntuple_VLAN
+    SOURCES Builder.cc Plugin.cc)

src/conntuple/vlan/Plugin.cc

@@ -0,0 +1,24 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#include "zeek/plugin/Plugin.h"
+
+#include "zeek/conntuple/Component.h"
+#include "zeek/conntuple/vlan/Builder.h"
+
+namespace zeek::plugin::Zeek_Conntuple_VLAN {
+
+class Plugin : public zeek::plugin::Plugin {
+public:
+    zeek::plugin::Configuration Configure() {
+        AddComponent(new conntuple::Component("VLAN", zeek::plugin::Zeek_Conntuple_VLAN::Builder::Instantiate));
+
+        zeek::plugin::Configuration config;
+        config.name = "Zeek::Conntuple_VLAN";
+        config.description = "Conntuple builder for 802.1Q VLAN- and Q-in-Q-aware flows";
+        return config;
+    }
+};
+
+Plugin plugin;
+
+} // namespace zeek::plugin::Zeek_Conntuple_VLAN

testing/btest/Baseline/scripts.policy.protocols.conntuple.vlan/conn.log.cut

@@ -0,0 +1,4 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	id.vlan	id.inner_vlan
+XXXXXXXXXX.XXXXXX	CHhAvVGS1DHFjwGM9	141.142.228.5	59856	192.150.187.43	80	-	-
+XXXXXXXXXX.XXXXXX	ClEkJM2Vm5giqnMf4h	141.142.228.5	59856	192.150.187.43	80	42	-

testing/btest/scripts/policy/protocols/conntuple/vlan.zeek

@@ -0,0 +1,8 @@
+# @TEST-DOC: Verifies that the VLAN-aware conntuple builder correctly distinguishes colliding 5-tuples that only differ in their vlan tagging.
+#
+# @TEST-EXEC: zeek -b -r $TRACES/conntuple/tuple-collision-vlan.pcap %INPUT
+# @TEST-EXEC: zeek-cut -m ts  uid id.orig_h id.orig_p id.resp_h id.resp_p id.vlan id.inner_vlan <conn.log >conn.log.cut
+# @TEST-EXEC: btest-diff conn.log.cut
+
+@load base/protocols/conn
+@load protocols/conntuple/vlan