Merge remote-tracking branch 'origin/topic/awelzel/add-community-id'

* origin/topic/awelzel/add-community-id:
  testing/external: Bump hashes for community_id addition
  NEWS: Add entry for Community ID
  policy: Import zeek-community-id scripts into protocols/conn frameworks/notice
  Add community_id_v1() based on corelight/zeek-community-id

CHANGES

@@ -1,3 +1,30 @@
+6.0.0-dev.410 | 2023-04-24 09:46:59 +0200
+
+  * policy: Import zeek-community-id scripts into protocols/conn frameworks/notice (Christian Kreibich, Corelight)
+
+  * Add community_id_v1() based on corelight/zeek-community-id (Christian Kreibich, Corelight)
+
+    "Community ID" has become an established flow hash for connection correlation
+    across different monitoring and storage systems. Other NSMs have had native
+    and built-in support for Community ID since late 2018. And even though the
+    roots of "Community ID" are very close to Zeek, Zeek itself has never provided
+    out-of-the-box support and instead required users to install an external plugin.
+
+    While we try to make that installation as easy as possible, an external plugin
+    always sets the bar higher for an initial setup and can be intimidating.
+    It also requires a rebuild operation of the plugin during upgrades. Nothing
+    overly complicated, but somewhat unnecessary for such popular functionality.
+
+    This isn't a 1:1 import. The options are parameters and the "verbose"
+    functionality  has been removed. Further, instead of a `connection`
+    record, the new bif works with `conn_id`, allowing computation of the
+    hash with little effort on the command line:
+
+        $ zeek -e 'print community_id_v1([$orig_h=1.2.3.4, $orig_p=1024/tcp, $resp_h=5.6.7.8, $resp_p=80/tcp])'
+        1:RcCrCS5fwYUeIzgDDx64EN3+okU
+
+    Reference: https://github.com/corelight/zeek-community-id/
+
 6.0.0-dev.405 | 2023-04-24 09:23:01 +0200
 
   * file_analysis/Manager: Remove RunState.h include (Arne Welzel, Corelight)

NEWS

@@ -100,6 +100,16 @@ New Functionality
 
   To disable this functionality, pass ``--disable-javascript`` to configure.
 
+- Zeek now provides native "Community ID" support with a new bif called
+  ``community_id_v1()``. Two policy scripts ``protocols/conn/community-id-logging``
+   and ``frameworks/notice/community-id`` extend the respective logs with a
+  ``community_id`` field the same way as the external zeek-community-id plugin
+  provides. A main difference to the external ``hash_conn()`` bif is that the
+  ``community_id_v1()`` takes a ``conn_id`` record instead of a ``connection``.
+
+  Loading the new policy scripts and using the external zeek-community-id
+  plugin at the same time is unsupported.
+
 - Introduce a new command-line option ``-V`` / ``--build-info``. It produces
   verbose output in JSON format about the repository state and any included
   plugins.

VERSION

@@ -1 +1 @@
-6.0.0-dev.405
+6.0.0-dev.410

scripts/base/init-bare.zeek

@@ -2034,6 +2034,7 @@ type gtp_delete_pdp_ctx_response_elements: record {
 
 # Prototypes of Zeek built-in functions.
 @load base/bif/zeek.bif
+@load base/bif/communityid.bif
 @load base/bif/stats.bif
 @load base/bif/reporter.bif
 @load base/bif/strings.bif

scripts/policy/frameworks/notice/community-id.zeek

@@ -0,0 +1,35 @@
+# Source this script in addition to protocols/conn/community-id
+# to add Community ID to notices.
+
+# Only support loading this if the main script is also loaded.
+@load base/protocols/conn
+@load base/frameworks/notice
+
+@ifdef ( CommunityID::seed )
+
+module CommunityID::Notice;
+
+export {
+	# Turn notice support on/off at runtime. When disabled,
+	# this still leaves the `community_id` string in the notice
+	# log, just unset.
+	option enabled: bool = T;
+
+	redef record Notice::Info += {
+		community_id: string &optional &log;
+	};
+}
+
+hook Notice::notice(n: Notice::Info)
+	{
+	if ( CommunityID::Notice::enabled && n?$conn && n$conn?$conn )
+		{
+		local info = n$conn$conn;
+		# This is set during new_connection(), so it should
+		# always be there, but better safe than sorry.
+		if ( info?$community_id )
+			n$community_id = info$community_id;
+		}
+	}
+
+@endif

scripts/policy/protocols/conn/community-id-logging.zeek

@@ -0,0 +1,26 @@
+##! Adds community hash IDs to conn.log.
+@load base/protocols/conn
+
+module CommunityID;
+
+export {
+	# An unsigned 16-bit number to seed our hashing
+	option seed: count = 0;
+
+	# Whether to add a base64 pass over the hash digest.
+	# Enabled by default, since it shortens the output.
+	option do_base64: bool = T;
+
+	# Add the ID string field to the connection log record.
+	redef record Conn::Info += {
+		community_id: string &optional &log;
+	};
+}
+
+module Conn;
+
+event new_connection(c: connection)
+	{
+	Conn::set_conn(c, F);  # likely first to access :-/
+	c$conn$community_id = community_id_v1(c$id, CommunityID::seed, CommunityID::do_base64);
+	}

scripts/site/local.zeek

@@ -90,6 +90,9 @@ redef digest_salt = "Please change this value.";
 # Extend email alerting to include hostnames
 @load policy/frameworks/notice/extend-email/hostnames
 
+# Extend the notice.log with Community ID hashes
+# @load policy/frameworks/notice/community-id
+
 # Enable logging of telemetry data into telemetry.log and
 # telemetry_histogram.log.
 @load frameworks/telemetry/log
@@ -98,6 +101,10 @@ redef digest_salt = "Please change this value.";
 # this might impact performance a bit.
 # @load policy/protocols/ssl/heartbleed
 
+# Uncomment the following line to enable logging of Community ID hashes in
+# the conn.log file.
+# @load policy/protocols/conn/community-id-logging
+
 # Uncomment the following line to enable logging of connection VLANs. Enabling
 # this adds two VLAN fields to the conn.log file.
 # @load policy/protocols/conn/vlan-logging

scripts/test-all-policy.zeek

@@ -66,6 +66,7 @@
 @load frameworks/files/hash-all-files.zeek
 @load frameworks/notice/__load__.zeek
 @load frameworks/notice/actions/drop.zeek
+@load frameworks/notice/community-id.zeek
 @load frameworks/notice/extend-email/hostnames.zeek
 @load files/x509/disable-certificate-events-known-certs.zeek
 @load frameworks/packet-filter/shunt.zeek
@@ -87,6 +88,7 @@
 @load misc/weird-stats.zeek
 @load misc/trim-trace-file.zeek
 @load misc/unknown-protocols.zeek
+@load protocols/conn/community-id-logging.zeek
 @load protocols/conn/known-hosts.zeek
 @load protocols/conn/known-services.zeek
 @load protocols/conn/mac-logging.zeek

src/CMakeLists.txt

@@ -123,6 +123,7 @@ set(SUPERVISOR_SRCS supervisor/Supervisor.cc Pipe.cc)
 
 set(BIF_SRCS
     zeek.bif
+    communityid.bif
     stats.bif
     event.bif
     const.bif

src/Func.cc

@@ -44,6 +44,7 @@
 // break what symbols are available when, which keeps the build from breaking.
 // clang-format off
 #include "zeek.bif.func_h"
+#include "communityid.bif.func_h"
 #include "stats.bif.func_h"
 #include "reporter.bif.func_h"
 #include "strings.bif.func_h"
@@ -53,6 +54,7 @@
 #include "CPP-load.bif.func_h"
 
 #include "zeek.bif.func_def"
+#include "communityid.bif.func_def"
 #include "stats.bif.func_def"
 #include "reporter.bif.func_def"
 #include "strings.bif.func_def"
@@ -1036,6 +1038,7 @@ void init_primary_bifs()
 	var_sizes = id::find_type("var_sizes")->AsTableType();
 
 #include "CPP-load.bif.func_init"
+#include "communityid.bif.func_init"
 #include "option.bif.func_init"
 #include "packet_analysis.bif.func_init"
 #include "reporter.bif.func_init"

src/communityid.bif

@@ -0,0 +1,130 @@
+%%{ // C segment
+#include "zeek/IPAddr.h"
+#include "zeek/Val.h"
+#include "zeek/digest.h"
+#include "zeek/packet_analysis/protocol/icmp/ICMP.h"
+%%}
+
+## Compute the Community ID hash (v1) from a connection identifier.
+##
+## cid: The identifier of the connection for which to compute the community-id.
+##
+## Returns: The Community ID hash of the connection identifier as string.
+##
+function community_id_v1%(cid: conn_id, seed: count &default=0, do_base64: bool &default=T%): string
+%{
+    const auto *cid_rec = cid->AsRecordVal();
+
+    uint16_t hash_seed = htons(seed);
+    const uint32_t *hash_src_addr = 0;
+    const uint32_t *hash_dst_addr = 0;
+    uint8_t hash_proto = 0;
+    uint8_t hash_padbyte = 0;
+    uint16_t hash_src_port = 0;
+    uint16_t hash_dst_port = 0;
+
+    const auto& orig_addr = cid_rec->GetFieldAs<zeek::AddrVal>(0);
+    const auto& orig_port = cid_rec->GetFieldAs<zeek::PortVal>(1);
+    const auto& resp_addr = cid_rec->GetFieldAs<zeek::AddrVal>(2);
+    const auto& resp_port = cid_rec->GetFieldAs<zeek::PortVal>(3);
+
+    bool is_ipv4 = orig_addr.GetBytes(&hash_src_addr) == 1;
+    resp_addr.GetBytes(&hash_dst_addr);
+    TransportProto proto = orig_port->PortType();
+
+    // Zeek's transport protocol aliases different underlying
+    // protocols, particularly IPv4's and v6's ICMP...
+    switch (proto) {
+        case TRANSPORT_TCP:
+            hash_proto = IPPROTO_TCP;
+            break;
+        case TRANSPORT_UDP:
+            hash_proto = IPPROTO_UDP;
+            break;
+        case TRANSPORT_ICMP:
+            if (is_ipv4)
+                hash_proto = IPPROTO_ICMP;
+            else
+                hash_proto = IPPROTO_ICMPV6;
+
+            break;
+        case TRANSPORT_UNKNOWN:
+            emit_builtin_error("CommunityID: unknown transport layer", cid);
+            return zeek::make_intrusive<zeek::StringVal>("");
+        default:
+            emit_builtin_error("CommunityID: unhandled transport layer", cid);
+            return zeek::make_intrusive<zeek::StringVal>("");
+    }
+
+    hash_src_port = htons((uint16_t) orig_port->Port());
+    hash_dst_port = htons((uint16_t) resp_port->Port());
+
+    // XXX: resolve whether we should copy is_one_way into the
+    // Connection instance at construction time, along with the other
+    // ConnID fields (see Conn.cc around line 125).
+    // awelzel: Maybe the is_one_way should be just a helper?
+
+    bool is_one_way = false;
+
+    if (TRANSPORT_ICMP == proto) {
+        if (is_ipv4)
+            zeek::packet_analysis::ICMP::ICMP4_counterpart(ntohs(hash_src_port),
+                                                           ntohs(hash_dst_port),
+                                                           is_one_way);
+        else
+            zeek::packet_analysis::ICMP::ICMP6_counterpart(ntohs(hash_src_port),
+                                                  ntohs(hash_dst_port),
+                                                  is_one_way);
+    }
+
+    if (is_one_way || zeek::addr_port_canon_lt(orig_addr, hash_src_port,
+                                               resp_addr, hash_dst_port)) {
+        // All good, no need to flip
+    } else {
+        // Need to flip endpoints for hashing.
+        std::swap(hash_src_addr, hash_dst_addr);
+        std::swap(hash_src_port, hash_dst_port);
+    }
+
+    auto digest_update = [](EVP_MD_CTX *ctx, const void* data, unsigned long len) {
+        zeek::detail::hash_update(ctx, data, len);
+        return len;
+    };
+
+    int dlen = 0;
+    auto *ctx = zeek::detail::hash_init(zeek::detail::Hash_SHA1);
+
+    dlen += digest_update(ctx, &hash_seed, 2);
+    dlen += digest_update(ctx, hash_src_addr, is_ipv4 ? 4 : 16);
+    dlen += digest_update(ctx, hash_dst_addr, is_ipv4 ? 4 : 16);
+    dlen += digest_update(ctx, &hash_proto, 1);
+    dlen += digest_update(ctx, &hash_padbyte, 1);
+    dlen += digest_update(ctx, &hash_src_port, 2);
+    dlen += digest_update(ctx, &hash_dst_port, 2);
+
+    u_char digest[SHA_DIGEST_LENGTH];
+    zeek::detail::hash_final(ctx, digest);
+
+    // We currently have no real versioning/hash configuration logic,
+    // so we simply prefix "1:" to the hash.
+    std::string ver("1:");
+    zeek::String *res = 0;
+
+    if (do_base64) {
+        char *outbuf = 0;
+        int outlen = 0;
+
+        zeek::detail::Base64Converter enc{nullptr};
+        enc.Encode(SHA_DIGEST_LENGTH, digest, &outlen, &outbuf);
+        res = new zeek::String(ver + std::string(outbuf, outlen));
+        // When given outlen = 0, the Encode() method creates the
+        // buffer it returns as outbuf, so we must delete it.
+        delete[] outbuf;
+    } else {
+        // The following returns a static buffer; no need to delete.
+        const char *ascii_digest = zeek::detail::sha1_digest_print(digest);
+        res = new zeek::String(ver + ascii_digest);
+    }
+
+    return zeek::make_intrusive<zeek::StringVal>(res);
+%}

testing/btest/Baseline/bifs.community_id.run-pcaps/arp.pcap.out

@@ -0,0 +1 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.

testing/btest/Baseline/bifs.community_id.run-pcaps/icmp.pcap.out

@@ -0,0 +1,2 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+[orig_h=192.168.0.89, orig_p=8/icmp, resp_h=192.168.0.1, resp_p=0/icmp], 1:X0snYXpgwiv9TZtqg64sgzUn6Dk=

testing/btest/Baseline/bifs.community_id.run-pcaps/icmp6.pcap.out

@@ -0,0 +1,15 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+[orig_h=3ffe:501:0:1802:260:97ff:feb6:7ff0, orig_p=3/icmp, resp_h=3ffe:507:0:1:200:86ff:fe05:80da, resp_p=0/icmp], 1:bnQKq8A2r//dWnkRW2EYcMhShjc=
+[orig_h=3ffe:501:1800:2345::2, orig_p=3/icmp, resp_h=3ffe:507:0:1:200:86ff:fe05:80da, resp_p=0/icmp], 1:2ObVBgIn28oZvibYZhZMBgh7WdQ=
+[orig_h=3ffe:501:410:0:2c0:dfff:fe47:33e, orig_p=1/icmp, resp_h=3ffe:507:0:1:200:86ff:fe05:80da, resp_p=4/icmp], 1:hLZd0XGWojozrvxqE0dWB1iM6R0=
+[orig_h=3ffe:507:0:1:200:86ff:fe05:80da, orig_p=1/icmp, resp_h=3ffe:501:4819::42, resp_p=4/icmp], 1:jwuBy9UWZK1KUFqJV5cHdVpfrlY=
+[orig_h=3ffe:507:0:1:200:86ff:fe05:80da, orig_p=128/icmp, resp_h=3ffe:501:0:1001::2, resp_p=129/icmp], 1:+TW+HtLHvV1xnGhV1lv7XoJrqQg=
+[orig_h=3ffe:507:0:1:200:86ff:fe05:80da, orig_p=128/icmp, resp_h=3ffe:507:0:1:260:97ff:fe07:69ea, resp_p=129/icmp], 1:GpbEQrKqfWtsfsFiqg8fufoZe5Y=
+[orig_h=3ffe:507:0:1:200:86ff:fe05:80da, orig_p=135/icmp, resp_h=3ffe:507:0:1:260:97ff:fe07:69ea, resp_p=136/icmp], 1:ORxAZfN3ld7Sv73/HQTNnvgxbpY=
+[orig_h=3ffe:507:0:1:200:86ff:fe05:80da, orig_p=135/icmp, resp_h=ff02::1:ff07:69ea, resp_p=136/icmp], 1:MEixa66kuz0OMvlQqnAIzP3n2xg=
+[orig_h=3ffe:507:0:1:260:97ff:fe07:69ea, orig_p=135/icmp, resp_h=3ffe:507:0:1:200:86ff:fe05:80da, resp_p=136/icmp], 1:BtEUCMYecYjJ7spEkVZDiCFaMTY=
+[orig_h=3ffe:507:0:1:260:97ff:fe07:69ea, orig_p=3/icmp, resp_h=3ffe:507:0:1:200:86ff:fe05:80da, resp_p=0/icmp], 1:NdobDX8PQNJbAyfkWxhtL2Pqp5w=
+[orig_h=fe80::200:86ff:fe05:80da, orig_p=133/icmp, resp_h=ff02::2, resp_p=134/icmp], 1:hO+sN4H+MG5MY/8hIrXPqc4ZQz0=
+[orig_h=fe80::200:86ff:fe05:80da, orig_p=135/icmp, resp_h=fe80::260:97ff:fe07:69ea, resp_p=136/icmp], 1:dGHyGvjMfljg6Bppwm3bg0LO8TY=
+[orig_h=fe80::260:97ff:fe07:69ea, orig_p=134/icmp, resp_h=ff02::1, resp_p=133/icmp], 1:pkvHqCL88/tg1k4cPigmZXUtL00=
+[orig_h=fe80::260:97ff:fe07:69ea, orig_p=135/icmp, resp_h=fe80::200:86ff:fe05:80da, resp_p=136/icmp], 1:zavyT/cezQr1fmImYCwYnMXbgck=

testing/btest/Baseline/bifs.community_id.run-pcaps/ipv6.pcap.out

@@ -0,0 +1,2 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+[orig_h=2001:470:e5bf:dead:4957:2174:e82c:4887, orig_p=63943/tcp, resp_h=2607:f8b0:400c:c03::1a, resp_p=25/tcp], 1:/qFaeAR+gFe1KYjMzVDsMv+wgU4=

testing/btest/Baseline/bifs.community_id.run-pcaps/sctp.pcap.out

@@ -0,0 +1 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.

testing/btest/Baseline/bifs.community_id.run-pcaps/tcp.pcap.out

@@ -0,0 +1,2 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+[orig_h=128.232.110.120, orig_p=34855/tcp, resp_h=66.35.250.204, resp_p=80/tcp], 1:LQU9qZlK+B5F3KDmev6m5PMibrg=

testing/btest/Baseline/bifs.community_id.run-pcaps/udp.pcap.out

@@ -0,0 +1,2 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+[orig_h=192.168.1.52, orig_p=54585/udp, resp_h=8.8.8.8, resp_p=53/udp], 1:d/FP5EW3wiY1vCndhwleRRKHowQ=

testing/btest/Baseline/bifs.community_id.v1/out

@@ -0,0 +1,11 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+PASS: expected '1:wCb3OG7yAFWelaUydu0D+125CLM=', got '1:wCb3OG7yAFWelaUydu0D+125CLM=' ([orig_h=1.2.3.4, orig_p=1122/tcp, resp_h=5.6.7.8, resp_p=3344/tcp], seed=0)
+PASS: expected '1:0Mu9InQx6z4ZiCZM/7HXi2WMhOg=', got '1:0Mu9InQx6z4ZiCZM/7HXi2WMhOg=' ([orig_h=1.2.3.4, orig_p=1122/udp, resp_h=5.6.7.8, resp_p=3344/udp], seed=0)
+PASS: expected '1:crodRHL2FEsHjbv3UkRrfbs4bZ0=', got '1:crodRHL2FEsHjbv3UkRrfbs4bZ0=' ([orig_h=1.2.3.4, orig_p=8/icmp, resp_h=5.6.7.8, resp_p=0/icmp], seed=0)
+PASS: expected '1:0bf7hyMJUwt3fMED7z8LIfRpBeo=', got '1:0bf7hyMJUwt3fMED7z8LIfRpBeo=' ([orig_h=fe80:1:203:405:607:809:a0b:c0d, orig_p=128/icmp, resp_h=fe80:1011:1213:1415:1617:1819:1a1b:1c1d, resp_p=129/icmp], seed=0)
+PASS: expected '1:HhA1B+6CoLbiKPEs5nhNYN4XWfk=', got '1:HhA1B+6CoLbiKPEs5nhNYN4XWfk=' ([orig_h=1.2.3.4, orig_p=1122/tcp, resp_h=5.6.7.8, resp_p=3344/tcp], seed=1)
+PASS: expected '1:OShq+iKDAMVouh/4bMxB9Sz4amw=', got '1:OShq+iKDAMVouh/4bMxB9Sz4amw=' ([orig_h=1.2.3.4, orig_p=1122/udp, resp_h=5.6.7.8, resp_p=3344/udp], seed=1)
+PASS: expected '1:9pr4ZGTICiuZoIh90RRYE2RyXpU=', got '1:9pr4ZGTICiuZoIh90RRYE2RyXpU=' ([orig_h=1.2.3.4, orig_p=8/icmp, resp_h=5.6.7.8, resp_p=0/icmp], seed=1)
+PASS: expected '1:IO27GQzPuCtNnwFvjWALMHu5tJE=', got '1:IO27GQzPuCtNnwFvjWALMHu5tJE=' ([orig_h=fe80:1:203:405:607:809:a0b:c0d, orig_p=128/icmp, resp_h=fe80:1011:1213:1415:1617:1819:1a1b:1c1d, resp_p=129/icmp], seed=1)
+PASS: expected '', got '' ([orig_h=1.2.3.4, orig_p=0/unknown, resp_h=5.6.7.8, resp_p=0/unknown], seed=0)
+PASS: expected '', got '' ([orig_h=fe80:1:203:405:607:809:a0b:c0d, orig_p=0/unknown, resp_h=fe80:1011:1213:1415:1617:1819:1a1b:1c1d, resp_p=0/unknown], seed=1)

testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log

@@ -11,6 +11,7 @@ scripts/base/init-bare.zeek
   build/scripts/base/bif/const.bif.zeek
   build/scripts/base/bif/types.bif.zeek
   build/scripts/base/bif/zeek.bif.zeek
+  build/scripts/base/bif/communityid.bif.zeek
   build/scripts/base/bif/stats.bif.zeek
   build/scripts/base/bif/reporter.bif.zeek
   build/scripts/base/bif/strings.bif.zeek

testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log

@@ -11,6 +11,7 @@ scripts/base/init-bare.zeek
   build/scripts/base/bif/const.bif.zeek
   build/scripts/base/bif/types.bif.zeek
   build/scripts/base/bif/zeek.bif.zeek
+  build/scripts/base/bif/communityid.bif.zeek
   build/scripts/base/bif/stats.bif.zeek
   build/scripts/base/bif/reporter.bif.zeek
   build/scripts/base/bif/strings.bif.zeek

testing/btest/Baseline/plugins.hooks/output

@@ -927,6 +927,7 @@
 0.000000   MetaHookPost  LoadFile(0, ./cardinality-counter.bif.zeek, <...>/cardinality-counter.bif.zeek) -> -1
 0.000000   MetaHookPost  LoadFile(0, ./certificate-event-cache, <...>/certificate-event-cache.zeek) -> -1
 0.000000   MetaHookPost  LoadFile(0, ./comm.bif.zeek, <...>/comm.bif.zeek) -> -1
+0.000000   MetaHookPost  LoadFile(0, ./communityid.bif.zeek, <...>/communityid.bif.zeek) -> -1
 0.000000   MetaHookPost  LoadFile(0, ./const-dos-error, <...>/const-dos-error.zeek) -> -1
 0.000000   MetaHookPost  LoadFile(0, ./const-nt-status, <...>/const-nt-status.zeek) -> -1
 0.000000   MetaHookPost  LoadFile(0, ./const.bif.zeek, <...>/const.bif.zeek) -> -1
@@ -1044,6 +1045,7 @@
 0.000000   MetaHookPost  LoadFile(0, base<...>/broker, <...>/broker) -> -1
 0.000000   MetaHookPost  LoadFile(0, base<...>/cluster, <...>/cluster) -> -1
 0.000000   MetaHookPost  LoadFile(0, base<...>/comm.bif, <...>/comm.bif.zeek) -> -1
+0.000000   MetaHookPost  LoadFile(0, base<...>/communityid.bif, <...>/communityid.bif.zeek) -> -1
 0.000000   MetaHookPost  LoadFile(0, base<...>/config, <...>/config) -> -1
 0.000000   MetaHookPost  LoadFile(0, base<...>/conn, <...>/conn) -> -1
 0.000000   MetaHookPost  LoadFile(0, base<...>/conn-ids, <...>/conn-ids.zeek) -> -1
@@ -1315,6 +1317,7 @@
 0.000000   MetaHookPost  LoadFileExtended(0, ./cardinality-counter.bif.zeek, <...>/cardinality-counter.bif.zeek) -> (-1, <no content>)
 0.000000   MetaHookPost  LoadFileExtended(0, ./certificate-event-cache, <...>/certificate-event-cache.zeek) -> (-1, <no content>)
 0.000000   MetaHookPost  LoadFileExtended(0, ./comm.bif.zeek, <...>/comm.bif.zeek) -> (-1, <no content>)
+0.000000   MetaHookPost  LoadFileExtended(0, ./communityid.bif.zeek, <...>/communityid.bif.zeek) -> (-1, <no content>)
 0.000000   MetaHookPost  LoadFileExtended(0, ./const-dos-error, <...>/const-dos-error.zeek) -> (-1, <no content>)
 0.000000   MetaHookPost  LoadFileExtended(0, ./const-nt-status, <...>/const-nt-status.zeek) -> (-1, <no content>)
 0.000000   MetaHookPost  LoadFileExtended(0, ./const.bif.zeek, <...>/const.bif.zeek) -> (-1, <no content>)
@@ -1432,6 +1435,7 @@
 0.000000   MetaHookPost  LoadFileExtended(0, base<...>/broker, <...>/broker) -> (-1, <no content>)
 0.000000   MetaHookPost  LoadFileExtended(0, base<...>/cluster, <...>/cluster) -> (-1, <no content>)
 0.000000   MetaHookPost  LoadFileExtended(0, base<...>/comm.bif, <...>/comm.bif.zeek) -> (-1, <no content>)
+0.000000   MetaHookPost  LoadFileExtended(0, base<...>/communityid.bif, <...>/communityid.bif.zeek) -> (-1, <no content>)
 0.000000   MetaHookPost  LoadFileExtended(0, base<...>/config, <...>/config) -> (-1, <no content>)
 0.000000   MetaHookPost  LoadFileExtended(0, base<...>/conn, <...>/conn) -> (-1, <no content>)
 0.000000   MetaHookPost  LoadFileExtended(0, base<...>/conn-ids, <...>/conn-ids.zeek) -> (-1, <no content>)
@@ -2501,6 +2505,7 @@
 0.000000   MetaHookPre   LoadFile(0, ./cardinality-counter.bif.zeek, <...>/cardinality-counter.bif.zeek)
 0.000000   MetaHookPre   LoadFile(0, ./certificate-event-cache, <...>/certificate-event-cache.zeek)
 0.000000   MetaHookPre   LoadFile(0, ./comm.bif.zeek, <...>/comm.bif.zeek)
+0.000000   MetaHookPre   LoadFile(0, ./communityid.bif.zeek, <...>/communityid.bif.zeek)
 0.000000   MetaHookPre   LoadFile(0, ./const-dos-error, <...>/const-dos-error.zeek)
 0.000000   MetaHookPre   LoadFile(0, ./const-nt-status, <...>/const-nt-status.zeek)
 0.000000   MetaHookPre   LoadFile(0, ./const.bif.zeek, <...>/const.bif.zeek)
@@ -2618,6 +2623,7 @@
 0.000000   MetaHookPre   LoadFile(0, base<...>/broker, <...>/broker)
 0.000000   MetaHookPre   LoadFile(0, base<...>/cluster, <...>/cluster)
 0.000000   MetaHookPre   LoadFile(0, base<...>/comm.bif, <...>/comm.bif.zeek)
+0.000000   MetaHookPre   LoadFile(0, base<...>/communityid.bif, <...>/communityid.bif.zeek)
 0.000000   MetaHookPre   LoadFile(0, base<...>/config, <...>/config)
 0.000000   MetaHookPre   LoadFile(0, base<...>/conn, <...>/conn)
 0.000000   MetaHookPre   LoadFile(0, base<...>/conn-ids, <...>/conn-ids.zeek)
@@ -2889,6 +2895,7 @@
 0.000000   MetaHookPre   LoadFileExtended(0, ./cardinality-counter.bif.zeek, <...>/cardinality-counter.bif.zeek)
 0.000000   MetaHookPre   LoadFileExtended(0, ./certificate-event-cache, <...>/certificate-event-cache.zeek)
 0.000000   MetaHookPre   LoadFileExtended(0, ./comm.bif.zeek, <...>/comm.bif.zeek)
+0.000000   MetaHookPre   LoadFileExtended(0, ./communityid.bif.zeek, <...>/communityid.bif.zeek)
 0.000000   MetaHookPre   LoadFileExtended(0, ./const-dos-error, <...>/const-dos-error.zeek)
 0.000000   MetaHookPre   LoadFileExtended(0, ./const-nt-status, <...>/const-nt-status.zeek)
 0.000000   MetaHookPre   LoadFileExtended(0, ./const.bif.zeek, <...>/const.bif.zeek)
@@ -3006,6 +3013,7 @@
 0.000000   MetaHookPre   LoadFileExtended(0, base<...>/broker, <...>/broker)
 0.000000   MetaHookPre   LoadFileExtended(0, base<...>/cluster, <...>/cluster)
 0.000000   MetaHookPre   LoadFileExtended(0, base<...>/comm.bif, <...>/comm.bif.zeek)
+0.000000   MetaHookPre   LoadFileExtended(0, base<...>/communityid.bif, <...>/communityid.bif.zeek)
 0.000000   MetaHookPre   LoadFileExtended(0, base<...>/config, <...>/config)
 0.000000   MetaHookPre   LoadFileExtended(0, base<...>/conn, <...>/conn)
 0.000000   MetaHookPre   LoadFileExtended(0, base<...>/conn-ids, <...>/conn-ids.zeek)
@@ -4076,6 +4084,7 @@
 0.000000 | HookLoadFile  ./cardinality-counter.bif.zeek <...>/cardinality-counter.bif.zeek
 0.000000 | HookLoadFile  ./certificate-event-cache <...>/certificate-event-cache.zeek
 0.000000 | HookLoadFile  ./comm.bif.zeek <...>/comm.bif.zeek
+0.000000 | HookLoadFile  ./communityid.bif.zeek <...>/communityid.bif.zeek
 0.000000 | HookLoadFile  ./const-dos-error <...>/const-dos-error.zeek
 0.000000 | HookLoadFile  ./const-nt-status <...>/const-nt-status.zeek
 0.000000 | HookLoadFile  ./const.bif.zeek <...>/const.bif.zeek
@@ -4203,6 +4212,7 @@
 0.000000 | HookLoadFile  base<...>/broker <...>/broker
 0.000000 | HookLoadFile  base<...>/cluster <...>/cluster
 0.000000 | HookLoadFile  base<...>/comm.bif <...>/comm.bif.zeek
+0.000000 | HookLoadFile  base<...>/communityid.bif <...>/communityid.bif.zeek
 0.000000 | HookLoadFile  base<...>/config <...>/config
 0.000000 | HookLoadFile  base<...>/conn <...>/conn
 0.000000 | HookLoadFile  base<...>/conn-ids <...>/conn-ids.zeek
@@ -4464,6 +4474,7 @@
 0.000000 | HookLoadFileExtended ./cardinality-counter.bif.zeek <...>/cardinality-counter.bif.zeek
 0.000000 | HookLoadFileExtended ./certificate-event-cache <...>/certificate-event-cache.zeek
 0.000000 | HookLoadFileExtended ./comm.bif.zeek <...>/comm.bif.zeek
+0.000000 | HookLoadFileExtended ./communityid.bif.zeek <...>/communityid.bif.zeek
 0.000000 | HookLoadFileExtended ./const-dos-error <...>/const-dos-error.zeek
 0.000000 | HookLoadFileExtended ./const-nt-status <...>/const-nt-status.zeek
 0.000000 | HookLoadFileExtended ./const.bif.zeek <...>/const.bif.zeek
@@ -4591,6 +4602,7 @@
 0.000000 | HookLoadFileExtended base<...>/broker <...>/broker
 0.000000 | HookLoadFileExtended base<...>/cluster <...>/cluster
 0.000000 | HookLoadFileExtended base<...>/comm.bif <...>/comm.bif.zeek
+0.000000 | HookLoadFileExtended base<...>/communityid.bif <...>/communityid.bif.zeek
 0.000000 | HookLoadFileExtended base<...>/config <...>/config
 0.000000 | HookLoadFileExtended base<...>/conn <...>/conn
 0.000000 | HookLoadFileExtended base<...>/conn-ids <...>/conn-ids.zeek

testing/btest/Baseline/scripts.policy.frameworks.notice.community-id/notice.log.cut

@@ -0,0 +1,3 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+-	-	-	-	-	-	COMMUNITY_ID_INIT	Zeek initializing
+141.142.228.5	59856	192.150.187.43	80	tcp	1:yvyB8h+3dnggTZW0UEITWCst97w=	COMMUNITY_ID_CONN_ESTABLISHED	Connection establishment

testing/btest/Baseline/scripts.policy.protocols.conn.community-id-logging/conn.log.cut

@@ -0,0 +1,2 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+141.142.228.5	59856	192.150.187.43	80	tcp	-	1:yvyB8h+3dnggTZW0UEITWCst97w=

testing/btest/Traces/communityid/README

@@ -0,0 +1 @@
+# Traces imported from the original zeek-community-id repository.

testing/btest/bifs/community_id/run-pcaps.zeek

@@ -0,0 +1,22 @@
+# Test imported from original zeek-community-id repository.
+#
+# Crank through a set of pcaps and verify the Community ID inputs and
+# outputs. Since each output line is triggered by a connection state
+# removal in Zeek, the ordering of sets of those events can change
+# across Zeek releases, and we don't care about the order (just the
+# values involved), we sort the output files.
+
+# @TEST-EXEC: bash %INPUT
+
+set -ex
+
+for pcap in $(cd $TRACES/communityid && ls *.pcap); do
+    zeek -r $TRACES/communityid/$pcap test-community-id-v1.zeek | sort >$pcap.out
+    btest-diff $pcap.out
+done
+
+@TEST-START-FILE test-community-id-v1.zeek
+event connection_state_remove(c: connection) {
+	print c$id, community_id_v1(c$id);
+}
+@TEST-END-FILE

testing/btest/bifs/community_id/v1.zeek

@@ -0,0 +1,29 @@
+# @TEST-EXEC: zeek -b %INPUT >out
+# @TEST-EXEC: btest-diff out
+
+function test_it(cid: conn_id, seed: count, expected: string)
+	{
+	local actual = community_id_v1(cid, seed);
+	local prefix = actual == expected ? "PASS" : "FAIL";
+	print fmt("%s: expected '%s', got '%s' (%s, seed=%d)", prefix, expected, actual, cid, seed);
+	}
+
+event zeek_init()
+	{
+	test_it([$orig_h=1.2.3.4, $orig_p=1122/tcp, $resp_h=5.6.7.8, $resp_p=3344/tcp], 0, "1:wCb3OG7yAFWelaUydu0D+125CLM=");
+	test_it([$orig_h=1.2.3.4, $orig_p=1122/udp, $resp_h=5.6.7.8, $resp_p=3344/udp], 0, "1:0Mu9InQx6z4ZiCZM/7HXi2WMhOg=");
+	test_it([$orig_h=1.2.3.4, $orig_p=8/icmp, $resp_h=5.6.7.8, $resp_p=0/icmp], 0, "1:crodRHL2FEsHjbv3UkRrfbs4bZ0=");
+	test_it([$orig_h=[fe80:0001:0203:0405:0607:0809:0A0B:0C0D], $orig_p=128/icmp,
+	         $resp_h=[fe80:1011:1213:1415:1617:1819:1A1B:1C1D], $resp_p=129/icmp], 0, "1:0bf7hyMJUwt3fMED7z8LIfRpBeo=");
+
+
+	test_it([$orig_h=1.2.3.4, $orig_p=1122/tcp, $resp_h=5.6.7.8, $resp_p=3344/tcp], 1, "1:HhA1B+6CoLbiKPEs5nhNYN4XWfk=");
+	test_it([$orig_h=1.2.3.4, $orig_p=1122/udp, $resp_h=5.6.7.8, $resp_p=3344/udp], 1, "1:OShq+iKDAMVouh/4bMxB9Sz4amw=");
+	test_it([$orig_h=1.2.3.4, $orig_p=8/icmp, $resp_h=5.6.7.8, $resp_p=0/icmp], 1, "1:9pr4ZGTICiuZoIh90RRYE2RyXpU=");
+	test_it([$orig_h=[fe80:0001:0203:0405:0607:0809:0A0B:0C0D], $orig_p=128/icmp,
+	         $resp_h=[fe80:1011:1213:1415:1617:1819:1A1B:1C1D], $resp_p=129/icmp], 1, "1:IO27GQzPuCtNnwFvjWALMHu5tJE=");
+
+	test_it([$orig_h=1.2.3.4, $orig_p=0/unknown, $resp_h=5.6.7.8, $resp_p=0/unknown], 0, "");
+	test_it([$orig_h=[fe80:0001:0203:0405:0607:0809:0A0B:0C0D], $orig_p=0/unknown,
+	         $resp_h=[fe80:1011:1213:1415:1617:1819:1A1B:1C1D], $resp_p=0/unknown], 1, "");
+	}

testing/btest/scripts/policy/frameworks/notice/community-id.zeek

@@ -0,0 +1,31 @@
+# This test verifies Community ID presence in the notice log, when
+# that part of the package is loaded. The test creates one notice
+# without connection state and one with, and verifies that the latter
+# includes the Community ID value for it.
+
+# @TEST-EXEC: zeek -b -r $TRACES/http/get.trace %INPUT
+# @TEST-EXEC: zeek-cut id.orig_h id.orig_p id.resp_h id.resp_p proto community_id note msg < notice.log > notice.log.cut
+# @TEST-EXEC: btest-diff notice.log.cut
+
+@load protocols/conn/community-id-logging
+@load frameworks/notice/community-id
+
+redef enum Notice::Type += {
+	COMMUNITY_ID_INIT,
+	COMMUNITY_ID_CONN_ESTABLISHED,
+};
+
+event zeek_init()
+	{
+	# A notice without connection context
+	NOTICE([$note=COMMUNITY_ID_INIT,
+	        $msg="Zeek initializing"]);
+	}
+
+event connection_established(c: connection)
+	{
+	# A notice with connection context
+	NOTICE([$note=COMMUNITY_ID_CONN_ESTABLISHED,
+	        $msg="Connection establishment",
+	        $conn=c]);
+	}

testing/btest/scripts/policy/protocols/conn/community-id-logging.zeek

@@ -0,0 +1,5 @@
+# @TEST-EXEC: zeek -b -r $TRACES/http/get.trace %INPUT
+# @TEST-EXEC: zeek-cut id.orig_h id.orig_p id.resp_h id.resp_p proto service community_id < conn.log > conn.log.cut
+# @TEST-EXEC: btest-diff conn.log.cut
+
+@load protocols/conn/community-id-logging

testing/external/commit-hash.zeek-testing

@@ -1 +1 @@
-18a9ac00f5b7617e8660d4ba680a25291d2b44f7
+63952e0fc5c88ff44752c586abe721ccb324003e

testing/external/commit-hash.zeek-testing-private

@@ -1 +1 @@
-98e8aee2f09bff7e8138290242274b5ffd834e58
+e16f299e2ddad6a3495113ff7b26120b6312b220