From 99de7b75261165a9b108b0e4fa2d538c550c167f Mon Sep 17 00:00:00 2001 From: Christian Kreibich Date: Fri, 21 Apr 2023 13:43:08 +0200 Subject: [PATCH 1/4] Add community_id_v1() based on corelight/zeek-community-id "Community ID" has become an established flow hash for connection correlation across different monitoring and storage systems. Other NSMs have had native and built-in support for Community ID since late 2018. And even though the roots of "Community ID" are very close to Zeek, Zeek itself has never provided out-of-the-box support and instead required users to install an external plugin. While we try to make that installation as easy as possible, an external plugin always sets the bar higher for an initial setup and can be intimidating. It also requires a rebuild operation of the plugin during upgrades. Nothing overly complicated, but somewhat unnecessary for such popular functionality. This isn't a 1:1 import. The options are parameters and the "verbose" functionality has been removed. Further, instead of a `connection` record, the new bif works with `conn_id`, allowing computation of the hash with little effort on the command line: $ zeek -e 'print community_id_v1([$orig_h=1.2.3.4, $orig_p=1024/tcp, $resp_h=5.6.7.8, $resp_p=80/tcp])' 1:RcCrCS5fwYUeIzgDDx64EN3+okU Reference: https://github.com/corelight/zeek-community-id/ --- scripts/base/init-bare.zeek | 1 + src/CMakeLists.txt | 1 + src/Func.cc | 3 + src/communityid.bif | 130 ++++++++++++++++++ .../bifs.community_id.run-pcaps/arp.pcap.out | 1 + .../bifs.community_id.run-pcaps/icmp.pcap.out | 2 + .../icmp6.pcap.out | 15 ++ .../bifs.community_id.run-pcaps/ipv6.pcap.out | 2 + .../bifs.community_id.run-pcaps/sctp.pcap.out | 1 + .../bifs.community_id.run-pcaps/tcp.pcap.out | 2 + .../bifs.community_id.run-pcaps/udp.pcap.out | 2 + .../btest/Baseline/bifs.community_id.v1/out | 11 ++ .../canonified_loaded_scripts.log | 1 + .../canonified_loaded_scripts.log | 1 + testing/btest/Baseline/plugins.hooks/output | 12 ++ testing/btest/Traces/communityid/README | 1 + testing/btest/Traces/communityid/arp.pcap | Bin 0 -> 444 bytes testing/btest/Traces/communityid/icmp.pcap | Bin 0 -> 1104 bytes testing/btest/Traces/communityid/icmp6.pcap | Bin 0 -> 5356 bytes testing/btest/Traces/communityid/ipv6.pcap | Bin 0 -> 1828 bytes testing/btest/Traces/communityid/sctp.pcap | Bin 0 -> 69024 bytes testing/btest/Traces/communityid/tcp.pcap | Bin 0 -> 1114 bytes testing/btest/Traces/communityid/udp.pcap | Bin 0 -> 372 bytes .../btest/bifs/community_id/run-pcaps.zeek | 22 +++ testing/btest/bifs/community_id/v1.zeek | 29 ++++ 25 files changed, 237 insertions(+) create mode 100644 src/communityid.bif create mode 100644 testing/btest/Baseline/bifs.community_id.run-pcaps/arp.pcap.out create mode 100644 testing/btest/Baseline/bifs.community_id.run-pcaps/icmp.pcap.out create mode 100644 testing/btest/Baseline/bifs.community_id.run-pcaps/icmp6.pcap.out create mode 100644 testing/btest/Baseline/bifs.community_id.run-pcaps/ipv6.pcap.out create mode 100644 testing/btest/Baseline/bifs.community_id.run-pcaps/sctp.pcap.out create mode 100644 testing/btest/Baseline/bifs.community_id.run-pcaps/tcp.pcap.out create mode 100644 testing/btest/Baseline/bifs.community_id.run-pcaps/udp.pcap.out create mode 100644 testing/btest/Baseline/bifs.community_id.v1/out create mode 100644 testing/btest/Traces/communityid/README create mode 100644 testing/btest/Traces/communityid/arp.pcap create mode 100644 testing/btest/Traces/communityid/icmp.pcap create mode 100644 testing/btest/Traces/communityid/icmp6.pcap create mode 100644 testing/btest/Traces/communityid/ipv6.pcap create mode 100644 testing/btest/Traces/communityid/sctp.pcap create mode 100644 testing/btest/Traces/communityid/tcp.pcap create mode 100644 testing/btest/Traces/communityid/udp.pcap create mode 100644 testing/btest/bifs/community_id/run-pcaps.zeek create mode 100644 testing/btest/bifs/community_id/v1.zeek diff --git a/scripts/base/init-bare.zeek b/scripts/base/init-bare.zeek index f3239c11a3..a02f3e38e7 100644 --- a/scripts/base/init-bare.zeek +++ b/scripts/base/init-bare.zeek @@ -2034,6 +2034,7 @@ type gtp_delete_pdp_ctx_response_elements: record { # Prototypes of Zeek built-in functions. @load base/bif/zeek.bif +@load base/bif/communityid.bif @load base/bif/stats.bif @load base/bif/reporter.bif @load base/bif/strings.bif diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 64f0577e42..773a9a6efc 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -123,6 +123,7 @@ set(SUPERVISOR_SRCS supervisor/Supervisor.cc Pipe.cc) set(BIF_SRCS zeek.bif + communityid.bif stats.bif event.bif const.bif diff --git a/src/Func.cc b/src/Func.cc index 7cbfaf764a..f56719ee00 100644 --- a/src/Func.cc +++ b/src/Func.cc @@ -44,6 +44,7 @@ // break what symbols are available when, which keeps the build from breaking. // clang-format off #include "zeek.bif.func_h" +#include "communityid.bif.func_h" #include "stats.bif.func_h" #include "reporter.bif.func_h" #include "strings.bif.func_h" @@ -53,6 +54,7 @@ #include "CPP-load.bif.func_h" #include "zeek.bif.func_def" +#include "communityid.bif.func_def" #include "stats.bif.func_def" #include "reporter.bif.func_def" #include "strings.bif.func_def" @@ -1036,6 +1038,7 @@ void init_primary_bifs() var_sizes = id::find_type("var_sizes")->AsTableType(); #include "CPP-load.bif.func_init" +#include "communityid.bif.func_init" #include "option.bif.func_init" #include "packet_analysis.bif.func_init" #include "reporter.bif.func_init" diff --git a/src/communityid.bif b/src/communityid.bif new file mode 100644 index 0000000000..d94321d0cd --- /dev/null +++ b/src/communityid.bif @@ -0,0 +1,130 @@ +%%{ // C segment +#include "zeek/IPAddr.h" +#include "zeek/Val.h" +#include "zeek/digest.h" +#include "zeek/packet_analysis/protocol/icmp/ICMP.h" +%%} + +## Compute the Community ID hash (v1) from a connection identifier. +## +## cid: The identifier of the connection for which to compute the community-id. +## +## Returns: The Community ID hash of the connection identifier as string. +## +function community_id_v1%(cid: conn_id, seed: count &default=0, do_base64: bool &default=T%): string +%{ + const auto *cid_rec = cid->AsRecordVal(); + + uint16_t hash_seed = htons(seed); + const uint32_t *hash_src_addr = 0; + const uint32_t *hash_dst_addr = 0; + uint8_t hash_proto = 0; + uint8_t hash_padbyte = 0; + uint16_t hash_src_port = 0; + uint16_t hash_dst_port = 0; + + const auto& orig_addr = cid_rec->GetFieldAs(0); + const auto& orig_port = cid_rec->GetFieldAs(1); + const auto& resp_addr = cid_rec->GetFieldAs(2); + const auto& resp_port = cid_rec->GetFieldAs(3); + + bool is_ipv4 = orig_addr.GetBytes(&hash_src_addr) == 1; + resp_addr.GetBytes(&hash_dst_addr); + TransportProto proto = orig_port->PortType(); + + // Zeek's transport protocol aliases different underlying + // protocols, particularly IPv4's and v6's ICMP... + switch (proto) { + case TRANSPORT_TCP: + hash_proto = IPPROTO_TCP; + break; + case TRANSPORT_UDP: + hash_proto = IPPROTO_UDP; + break; + case TRANSPORT_ICMP: + if (is_ipv4) + hash_proto = IPPROTO_ICMP; + else + hash_proto = IPPROTO_ICMPV6; + + break; + case TRANSPORT_UNKNOWN: + emit_builtin_error("CommunityID: unknown transport layer", cid); + return zeek::make_intrusive(""); + default: + emit_builtin_error("CommunityID: unhandled transport layer", cid); + return zeek::make_intrusive(""); + } + + hash_src_port = htons((uint16_t) orig_port->Port()); + hash_dst_port = htons((uint16_t) resp_port->Port()); + + // XXX: resolve whether we should copy is_one_way into the + // Connection instance at construction time, along with the other + // ConnID fields (see Conn.cc around line 125). + // awelzel: Maybe the is_one_way should be just a helper? + + bool is_one_way = false; + + if (TRANSPORT_ICMP == proto) { + if (is_ipv4) + zeek::packet_analysis::ICMP::ICMP4_counterpart(ntohs(hash_src_port), + ntohs(hash_dst_port), + is_one_way); + else + zeek::packet_analysis::ICMP::ICMP6_counterpart(ntohs(hash_src_port), + ntohs(hash_dst_port), + is_one_way); + } + + if (is_one_way || zeek::addr_port_canon_lt(orig_addr, hash_src_port, + resp_addr, hash_dst_port)) { + // All good, no need to flip + } else { + // Need to flip endpoints for hashing. + std::swap(hash_src_addr, hash_dst_addr); + std::swap(hash_src_port, hash_dst_port); + } + + auto digest_update = [](EVP_MD_CTX *ctx, const void* data, unsigned long len) { + zeek::detail::hash_update(ctx, data, len); + return len; + }; + + int dlen = 0; + auto *ctx = zeek::detail::hash_init(zeek::detail::Hash_SHA1); + + dlen += digest_update(ctx, &hash_seed, 2); + dlen += digest_update(ctx, hash_src_addr, is_ipv4 ? 4 : 16); + dlen += digest_update(ctx, hash_dst_addr, is_ipv4 ? 4 : 16); + dlen += digest_update(ctx, &hash_proto, 1); + dlen += digest_update(ctx, &hash_padbyte, 1); + dlen += digest_update(ctx, &hash_src_port, 2); + dlen += digest_update(ctx, &hash_dst_port, 2); + + u_char digest[SHA_DIGEST_LENGTH]; + zeek::detail::hash_final(ctx, digest); + + // We currently have no real versioning/hash configuration logic, + // so we simply prefix "1:" to the hash. + std::string ver("1:"); + zeek::String *res = 0; + + if (do_base64) { + char *outbuf = 0; + int outlen = 0; + + zeek::detail::Base64Converter enc{nullptr}; + enc.Encode(SHA_DIGEST_LENGTH, digest, &outlen, &outbuf); + res = new zeek::String(ver + std::string(outbuf, outlen)); + // When given outlen = 0, the Encode() method creates the + // buffer it returns as outbuf, so we must delete it. + delete[] outbuf; + } else { + // The following returns a static buffer; no need to delete. + const char *ascii_digest = zeek::detail::sha1_digest_print(digest); + res = new zeek::String(ver + ascii_digest); + } + + return zeek::make_intrusive(res); +%} diff --git a/testing/btest/Baseline/bifs.community_id.run-pcaps/arp.pcap.out b/testing/btest/Baseline/bifs.community_id.run-pcaps/arp.pcap.out new file mode 100644 index 0000000000..49d861c74c --- /dev/null +++ b/testing/btest/Baseline/bifs.community_id.run-pcaps/arp.pcap.out @@ -0,0 +1 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. diff --git a/testing/btest/Baseline/bifs.community_id.run-pcaps/icmp.pcap.out b/testing/btest/Baseline/bifs.community_id.run-pcaps/icmp.pcap.out new file mode 100644 index 0000000000..0aca4bba84 --- /dev/null +++ b/testing/btest/Baseline/bifs.community_id.run-pcaps/icmp.pcap.out @@ -0,0 +1,2 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +[orig_h=192.168.0.89, orig_p=8/icmp, resp_h=192.168.0.1, resp_p=0/icmp], 1:X0snYXpgwiv9TZtqg64sgzUn6Dk= diff --git a/testing/btest/Baseline/bifs.community_id.run-pcaps/icmp6.pcap.out b/testing/btest/Baseline/bifs.community_id.run-pcaps/icmp6.pcap.out new file mode 100644 index 0000000000..caa30f231d --- /dev/null +++ b/testing/btest/Baseline/bifs.community_id.run-pcaps/icmp6.pcap.out @@ -0,0 +1,15 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +[orig_h=3ffe:501:0:1802:260:97ff:feb6:7ff0, orig_p=3/icmp, resp_h=3ffe:507:0:1:200:86ff:fe05:80da, resp_p=0/icmp], 1:bnQKq8A2r//dWnkRW2EYcMhShjc= +[orig_h=3ffe:501:1800:2345::2, orig_p=3/icmp, resp_h=3ffe:507:0:1:200:86ff:fe05:80da, resp_p=0/icmp], 1:2ObVBgIn28oZvibYZhZMBgh7WdQ= +[orig_h=3ffe:501:410:0:2c0:dfff:fe47:33e, orig_p=1/icmp, resp_h=3ffe:507:0:1:200:86ff:fe05:80da, resp_p=4/icmp], 1:hLZd0XGWojozrvxqE0dWB1iM6R0= +[orig_h=3ffe:507:0:1:200:86ff:fe05:80da, orig_p=1/icmp, resp_h=3ffe:501:4819::42, resp_p=4/icmp], 1:jwuBy9UWZK1KUFqJV5cHdVpfrlY= +[orig_h=3ffe:507:0:1:200:86ff:fe05:80da, orig_p=128/icmp, resp_h=3ffe:501:0:1001::2, resp_p=129/icmp], 1:+TW+HtLHvV1xnGhV1lv7XoJrqQg= +[orig_h=3ffe:507:0:1:200:86ff:fe05:80da, orig_p=128/icmp, resp_h=3ffe:507:0:1:260:97ff:fe07:69ea, resp_p=129/icmp], 1:GpbEQrKqfWtsfsFiqg8fufoZe5Y= +[orig_h=3ffe:507:0:1:200:86ff:fe05:80da, orig_p=135/icmp, resp_h=3ffe:507:0:1:260:97ff:fe07:69ea, resp_p=136/icmp], 1:ORxAZfN3ld7Sv73/HQTNnvgxbpY= +[orig_h=3ffe:507:0:1:200:86ff:fe05:80da, orig_p=135/icmp, resp_h=ff02::1:ff07:69ea, resp_p=136/icmp], 1:MEixa66kuz0OMvlQqnAIzP3n2xg= +[orig_h=3ffe:507:0:1:260:97ff:fe07:69ea, orig_p=135/icmp, resp_h=3ffe:507:0:1:200:86ff:fe05:80da, resp_p=136/icmp], 1:BtEUCMYecYjJ7spEkVZDiCFaMTY= +[orig_h=3ffe:507:0:1:260:97ff:fe07:69ea, orig_p=3/icmp, resp_h=3ffe:507:0:1:200:86ff:fe05:80da, resp_p=0/icmp], 1:NdobDX8PQNJbAyfkWxhtL2Pqp5w= +[orig_h=fe80::200:86ff:fe05:80da, orig_p=133/icmp, resp_h=ff02::2, resp_p=134/icmp], 1:hO+sN4H+MG5MY/8hIrXPqc4ZQz0= +[orig_h=fe80::200:86ff:fe05:80da, orig_p=135/icmp, resp_h=fe80::260:97ff:fe07:69ea, resp_p=136/icmp], 1:dGHyGvjMfljg6Bppwm3bg0LO8TY= +[orig_h=fe80::260:97ff:fe07:69ea, orig_p=134/icmp, resp_h=ff02::1, resp_p=133/icmp], 1:pkvHqCL88/tg1k4cPigmZXUtL00= +[orig_h=fe80::260:97ff:fe07:69ea, orig_p=135/icmp, resp_h=fe80::200:86ff:fe05:80da, resp_p=136/icmp], 1:zavyT/cezQr1fmImYCwYnMXbgck= diff --git a/testing/btest/Baseline/bifs.community_id.run-pcaps/ipv6.pcap.out b/testing/btest/Baseline/bifs.community_id.run-pcaps/ipv6.pcap.out new file mode 100644 index 0000000000..a80b4bbb38 --- /dev/null +++ b/testing/btest/Baseline/bifs.community_id.run-pcaps/ipv6.pcap.out @@ -0,0 +1,2 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +[orig_h=2001:470:e5bf:dead:4957:2174:e82c:4887, orig_p=63943/tcp, resp_h=2607:f8b0:400c:c03::1a, resp_p=25/tcp], 1:/qFaeAR+gFe1KYjMzVDsMv+wgU4= diff --git a/testing/btest/Baseline/bifs.community_id.run-pcaps/sctp.pcap.out b/testing/btest/Baseline/bifs.community_id.run-pcaps/sctp.pcap.out new file mode 100644 index 0000000000..49d861c74c --- /dev/null +++ b/testing/btest/Baseline/bifs.community_id.run-pcaps/sctp.pcap.out @@ -0,0 +1 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. diff --git a/testing/btest/Baseline/bifs.community_id.run-pcaps/tcp.pcap.out b/testing/btest/Baseline/bifs.community_id.run-pcaps/tcp.pcap.out new file mode 100644 index 0000000000..7a15ba4be8 --- /dev/null +++ b/testing/btest/Baseline/bifs.community_id.run-pcaps/tcp.pcap.out @@ -0,0 +1,2 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +[orig_h=128.232.110.120, orig_p=34855/tcp, resp_h=66.35.250.204, resp_p=80/tcp], 1:LQU9qZlK+B5F3KDmev6m5PMibrg= diff --git a/testing/btest/Baseline/bifs.community_id.run-pcaps/udp.pcap.out b/testing/btest/Baseline/bifs.community_id.run-pcaps/udp.pcap.out new file mode 100644 index 0000000000..b894f78d61 --- /dev/null +++ b/testing/btest/Baseline/bifs.community_id.run-pcaps/udp.pcap.out @@ -0,0 +1,2 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +[orig_h=192.168.1.52, orig_p=54585/udp, resp_h=8.8.8.8, resp_p=53/udp], 1:d/FP5EW3wiY1vCndhwleRRKHowQ= diff --git a/testing/btest/Baseline/bifs.community_id.v1/out b/testing/btest/Baseline/bifs.community_id.v1/out new file mode 100644 index 0000000000..0cbcb5800e --- /dev/null +++ b/testing/btest/Baseline/bifs.community_id.v1/out @@ -0,0 +1,11 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +PASS: expected '1:wCb3OG7yAFWelaUydu0D+125CLM=', got '1:wCb3OG7yAFWelaUydu0D+125CLM=' ([orig_h=1.2.3.4, orig_p=1122/tcp, resp_h=5.6.7.8, resp_p=3344/tcp], seed=0) +PASS: expected '1:0Mu9InQx6z4ZiCZM/7HXi2WMhOg=', got '1:0Mu9InQx6z4ZiCZM/7HXi2WMhOg=' ([orig_h=1.2.3.4, orig_p=1122/udp, resp_h=5.6.7.8, resp_p=3344/udp], seed=0) +PASS: expected '1:crodRHL2FEsHjbv3UkRrfbs4bZ0=', got '1:crodRHL2FEsHjbv3UkRrfbs4bZ0=' ([orig_h=1.2.3.4, orig_p=8/icmp, resp_h=5.6.7.8, resp_p=0/icmp], seed=0) +PASS: expected '1:0bf7hyMJUwt3fMED7z8LIfRpBeo=', got '1:0bf7hyMJUwt3fMED7z8LIfRpBeo=' ([orig_h=fe80:1:203:405:607:809:a0b:c0d, orig_p=128/icmp, resp_h=fe80:1011:1213:1415:1617:1819:1a1b:1c1d, resp_p=129/icmp], seed=0) +PASS: expected '1:HhA1B+6CoLbiKPEs5nhNYN4XWfk=', got '1:HhA1B+6CoLbiKPEs5nhNYN4XWfk=' ([orig_h=1.2.3.4, orig_p=1122/tcp, resp_h=5.6.7.8, resp_p=3344/tcp], seed=1) +PASS: expected '1:OShq+iKDAMVouh/4bMxB9Sz4amw=', got '1:OShq+iKDAMVouh/4bMxB9Sz4amw=' ([orig_h=1.2.3.4, orig_p=1122/udp, resp_h=5.6.7.8, resp_p=3344/udp], seed=1) +PASS: expected '1:9pr4ZGTICiuZoIh90RRYE2RyXpU=', got '1:9pr4ZGTICiuZoIh90RRYE2RyXpU=' ([orig_h=1.2.3.4, orig_p=8/icmp, resp_h=5.6.7.8, resp_p=0/icmp], seed=1) +PASS: expected '1:IO27GQzPuCtNnwFvjWALMHu5tJE=', got '1:IO27GQzPuCtNnwFvjWALMHu5tJE=' ([orig_h=fe80:1:203:405:607:809:a0b:c0d, orig_p=128/icmp, resp_h=fe80:1011:1213:1415:1617:1819:1a1b:1c1d, resp_p=129/icmp], seed=1) +PASS: expected '', got '' ([orig_h=1.2.3.4, orig_p=0/unknown, resp_h=5.6.7.8, resp_p=0/unknown], seed=0) +PASS: expected '', got '' ([orig_h=fe80:1:203:405:607:809:a0b:c0d, orig_p=0/unknown, resp_h=fe80:1011:1213:1415:1617:1819:1a1b:1c1d, resp_p=0/unknown], seed=1) diff --git a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log index 22b3b8b55a..e1d05d14f1 100644 --- a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log @@ -11,6 +11,7 @@ scripts/base/init-bare.zeek build/scripts/base/bif/const.bif.zeek build/scripts/base/bif/types.bif.zeek build/scripts/base/bif/zeek.bif.zeek + build/scripts/base/bif/communityid.bif.zeek build/scripts/base/bif/stats.bif.zeek build/scripts/base/bif/reporter.bif.zeek build/scripts/base/bif/strings.bif.zeek diff --git a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log index 562168d47e..f9c627c29c 100644 --- a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log @@ -11,6 +11,7 @@ scripts/base/init-bare.zeek build/scripts/base/bif/const.bif.zeek build/scripts/base/bif/types.bif.zeek build/scripts/base/bif/zeek.bif.zeek + build/scripts/base/bif/communityid.bif.zeek build/scripts/base/bif/stats.bif.zeek build/scripts/base/bif/reporter.bif.zeek build/scripts/base/bif/strings.bif.zeek diff --git a/testing/btest/Baseline/plugins.hooks/output b/testing/btest/Baseline/plugins.hooks/output index 30dd240bad..3515875061 100644 --- a/testing/btest/Baseline/plugins.hooks/output +++ b/testing/btest/Baseline/plugins.hooks/output @@ -933,6 +933,7 @@ 0.000000 MetaHookPost LoadFile(0, ./cardinality-counter.bif.zeek, <...>/cardinality-counter.bif.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, ./certificate-event-cache, <...>/certificate-event-cache.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, ./comm.bif.zeek, <...>/comm.bif.zeek) -> -1 +0.000000 MetaHookPost LoadFile(0, ./communityid.bif.zeek, <...>/communityid.bif.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, ./const-dos-error, <...>/const-dos-error.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, ./const-nt-status, <...>/const-nt-status.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, ./const.bif.zeek, <...>/const.bif.zeek) -> -1 @@ -1050,6 +1051,7 @@ 0.000000 MetaHookPost LoadFile(0, base<...>/broker, <...>/broker) -> -1 0.000000 MetaHookPost LoadFile(0, base<...>/cluster, <...>/cluster) -> -1 0.000000 MetaHookPost LoadFile(0, base<...>/comm.bif, <...>/comm.bif.zeek) -> -1 +0.000000 MetaHookPost LoadFile(0, base<...>/communityid.bif, <...>/communityid.bif.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, base<...>/config, <...>/config) -> -1 0.000000 MetaHookPost LoadFile(0, base<...>/conn, <...>/conn) -> -1 0.000000 MetaHookPost LoadFile(0, base<...>/conn-ids, <...>/conn-ids.zeek) -> -1 @@ -1321,6 +1323,7 @@ 0.000000 MetaHookPost LoadFileExtended(0, ./cardinality-counter.bif.zeek, <...>/cardinality-counter.bif.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, ./certificate-event-cache, <...>/certificate-event-cache.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, ./comm.bif.zeek, <...>/comm.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./communityid.bif.zeek, <...>/communityid.bif.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, ./const-dos-error, <...>/const-dos-error.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, ./const-nt-status, <...>/const-nt-status.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, ./const.bif.zeek, <...>/const.bif.zeek) -> (-1, ) @@ -1438,6 +1441,7 @@ 0.000000 MetaHookPost LoadFileExtended(0, base<...>/broker, <...>/broker) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, base<...>/cluster, <...>/cluster) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, base<...>/comm.bif, <...>/comm.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/communityid.bif, <...>/communityid.bif.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, base<...>/config, <...>/config) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, base<...>/conn, <...>/conn) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, base<...>/conn-ids, <...>/conn-ids.zeek) -> (-1, ) @@ -2513,6 +2517,7 @@ 0.000000 MetaHookPre LoadFile(0, ./cardinality-counter.bif.zeek, <...>/cardinality-counter.bif.zeek) 0.000000 MetaHookPre LoadFile(0, ./certificate-event-cache, <...>/certificate-event-cache.zeek) 0.000000 MetaHookPre LoadFile(0, ./comm.bif.zeek, <...>/comm.bif.zeek) +0.000000 MetaHookPre LoadFile(0, ./communityid.bif.zeek, <...>/communityid.bif.zeek) 0.000000 MetaHookPre LoadFile(0, ./const-dos-error, <...>/const-dos-error.zeek) 0.000000 MetaHookPre LoadFile(0, ./const-nt-status, <...>/const-nt-status.zeek) 0.000000 MetaHookPre LoadFile(0, ./const.bif.zeek, <...>/const.bif.zeek) @@ -2630,6 +2635,7 @@ 0.000000 MetaHookPre LoadFile(0, base<...>/broker, <...>/broker) 0.000000 MetaHookPre LoadFile(0, base<...>/cluster, <...>/cluster) 0.000000 MetaHookPre LoadFile(0, base<...>/comm.bif, <...>/comm.bif.zeek) +0.000000 MetaHookPre LoadFile(0, base<...>/communityid.bif, <...>/communityid.bif.zeek) 0.000000 MetaHookPre LoadFile(0, base<...>/config, <...>/config) 0.000000 MetaHookPre LoadFile(0, base<...>/conn, <...>/conn) 0.000000 MetaHookPre LoadFile(0, base<...>/conn-ids, <...>/conn-ids.zeek) @@ -2901,6 +2907,7 @@ 0.000000 MetaHookPre LoadFileExtended(0, ./cardinality-counter.bif.zeek, <...>/cardinality-counter.bif.zeek) 0.000000 MetaHookPre LoadFileExtended(0, ./certificate-event-cache, <...>/certificate-event-cache.zeek) 0.000000 MetaHookPre LoadFileExtended(0, ./comm.bif.zeek, <...>/comm.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./communityid.bif.zeek, <...>/communityid.bif.zeek) 0.000000 MetaHookPre LoadFileExtended(0, ./const-dos-error, <...>/const-dos-error.zeek) 0.000000 MetaHookPre LoadFileExtended(0, ./const-nt-status, <...>/const-nt-status.zeek) 0.000000 MetaHookPre LoadFileExtended(0, ./const.bif.zeek, <...>/const.bif.zeek) @@ -3018,6 +3025,7 @@ 0.000000 MetaHookPre LoadFileExtended(0, base<...>/broker, <...>/broker) 0.000000 MetaHookPre LoadFileExtended(0, base<...>/cluster, <...>/cluster) 0.000000 MetaHookPre LoadFileExtended(0, base<...>/comm.bif, <...>/comm.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/communityid.bif, <...>/communityid.bif.zeek) 0.000000 MetaHookPre LoadFileExtended(0, base<...>/config, <...>/config) 0.000000 MetaHookPre LoadFileExtended(0, base<...>/conn, <...>/conn) 0.000000 MetaHookPre LoadFileExtended(0, base<...>/conn-ids, <...>/conn-ids.zeek) @@ -4094,6 +4102,7 @@ 0.000000 | HookLoadFile ./cardinality-counter.bif.zeek <...>/cardinality-counter.bif.zeek 0.000000 | HookLoadFile ./certificate-event-cache <...>/certificate-event-cache.zeek 0.000000 | HookLoadFile ./comm.bif.zeek <...>/comm.bif.zeek +0.000000 | HookLoadFile ./communityid.bif.zeek <...>/communityid.bif.zeek 0.000000 | HookLoadFile ./const-dos-error <...>/const-dos-error.zeek 0.000000 | HookLoadFile ./const-nt-status <...>/const-nt-status.zeek 0.000000 | HookLoadFile ./const.bif.zeek <...>/const.bif.zeek @@ -4221,6 +4230,7 @@ 0.000000 | HookLoadFile base<...>/broker <...>/broker 0.000000 | HookLoadFile base<...>/cluster <...>/cluster 0.000000 | HookLoadFile base<...>/comm.bif <...>/comm.bif.zeek +0.000000 | HookLoadFile base<...>/communityid.bif <...>/communityid.bif.zeek 0.000000 | HookLoadFile base<...>/config <...>/config 0.000000 | HookLoadFile base<...>/conn <...>/conn 0.000000 | HookLoadFile base<...>/conn-ids <...>/conn-ids.zeek @@ -4482,6 +4492,7 @@ 0.000000 | HookLoadFileExtended ./cardinality-counter.bif.zeek <...>/cardinality-counter.bif.zeek 0.000000 | HookLoadFileExtended ./certificate-event-cache <...>/certificate-event-cache.zeek 0.000000 | HookLoadFileExtended ./comm.bif.zeek <...>/comm.bif.zeek +0.000000 | HookLoadFileExtended ./communityid.bif.zeek <...>/communityid.bif.zeek 0.000000 | HookLoadFileExtended ./const-dos-error <...>/const-dos-error.zeek 0.000000 | HookLoadFileExtended ./const-nt-status <...>/const-nt-status.zeek 0.000000 | HookLoadFileExtended ./const.bif.zeek <...>/const.bif.zeek @@ -4609,6 +4620,7 @@ 0.000000 | HookLoadFileExtended base<...>/broker <...>/broker 0.000000 | HookLoadFileExtended base<...>/cluster <...>/cluster 0.000000 | HookLoadFileExtended base<...>/comm.bif <...>/comm.bif.zeek +0.000000 | HookLoadFileExtended base<...>/communityid.bif <...>/communityid.bif.zeek 0.000000 | HookLoadFileExtended base<...>/config <...>/config 0.000000 | HookLoadFileExtended base<...>/conn <...>/conn 0.000000 | HookLoadFileExtended base<...>/conn-ids <...>/conn-ids.zeek diff --git a/testing/btest/Traces/communityid/README b/testing/btest/Traces/communityid/README new file mode 100644 index 0000000000..e644426048 --- /dev/null +++ b/testing/btest/Traces/communityid/README @@ -0,0 +1 @@ +# Traces imported from the original zeek-community-id repository. diff --git a/testing/btest/Traces/communityid/arp.pcap b/testing/btest/Traces/communityid/arp.pcap new file mode 100644 index 0000000000000000000000000000000000000000..cd94d5fe1a9e40a823e25a18dd817683fc8ccbb6 GIT binary patch literal 444 zcmca|c+)~A1{MYwNB}YzftWj_#-V-{4}%Sm4Z{DyfMJ7|o{R+t8v`Q;0~-raA(($) z+HQ~*AONyNAS`531ZdV7c6J6WpdbiC&EgH26T^*U7MKq-!3d%k$l?R(^yFiQyMh6% zh9QAteIqkmCliDZQV)?wm;rSb*g#RBS#w^n!d(GzJ46TEEJkz_fMzj*beggPTQ3xB>uR!&sC6 literal 0 HcmV?d00001 diff --git a/testing/btest/Traces/communityid/icmp.pcap b/testing/btest/Traces/communityid/icmp.pcap new file mode 100644 index 0000000000000000000000000000000000000000..037606d1beff5b19ce4a44395a490a44e623b2f5 GIT binary patch literal 1104 zcmca|c+)~A1{MYw`2U}Qff2|t|IY4rJdT0E3&;jx2A*@mI~Oo0#oN!A$id*sz+iLC zj)9?p@zeDKD;Oeyn303QDTaxGlOZuFIVCkMJtH$KJ0~|Uzo4+FxTLhK94-tp`7zB* z7R*C(GuX8dmm}Q#pRpL^W=4>^foh3zvjx!2*W!^(#taQ-aA;gYgoY!~WG;fC0Ww)A zkya+(FF-OGGc;Tbh~7 YS&rmp%+LV385kNMH$y|iohUa00Mh7JB>(^b literal 0 HcmV?d00001 diff --git a/testing/btest/Traces/communityid/icmp6.pcap b/testing/btest/Traces/communityid/icmp6.pcap new file mode 100644 index 0000000000000000000000000000000000000000..99e7cde9cfb8c1d02193fc8e4d54345be710c25d GIT binary patch literal 5356 zcmc(jZA=_R7{{O8z4xVXq}xbo#Y+_%8;mq4jZ}||LTj{Yt)|shlWT(s5K@i6Y2`x? zA_vk&;|J0YT5a+0lN2S^_}W}jLu%8AiAIg7X>F^ZmcD$j#F%hC&+OdX-0sQIgTN-U zdo#21d;ar0v-5J_A0Iiaf&)_5w{8FhGv`l?SAW{)hOPLU5`+)9oBjldIio+vehE`a zpe8vTr5^w)#FEokBl6(`$!UDQ2O7VmdTY6YCWu5<9Q)ny2);wfEH!O$D1 zyQi=Z*484=t@gs!-cS}R6G(sT^IUNdG_G0XIo;L=UY?2rS!wwqR~)QVkr#0|y(}U3 zfmo3IS59i zUgJuVYfUA$J&{96EqNE9L}jzp$9u0>y|2NE0SZi*ZnKNd+hA-CI#e?{x2!XjB!?D3 z$va?7o^x^~-EWvmUK?8kC5ON$J?`R4wtZkKIltdG^N0;nbV}gndU8Hqsy#Z#8mJ;n zx7kJKFc_Un-I^Jl%g^}Oh#k4Q2ucRPSQhtiB~P6)m24@~7v3lDg3-9g%au5O6!`BJwfUyVNKQz7PrEMh$7- z-R)w}&HWP9zE=8c)>|pon0G=vgQ_BZRlP*Duk*9gTH6!}C_uDz_q*7$c%MYgQ)@7T zsv&)NwM5NR>tHMrP=RP;FV*q7cv*{Soe7bh!4Bfj+h$?lOk z?8S?a{$zO}1aUx8s;a^pt`X9i$RnLr_&84D8`7nAL;E($`;petpg{zGQ(`oJrr>9v zkVNCVF$+zJ!G`h>xIF;vl>pjAYBx;Z4uJEt<_IOA`cjl7EJ=-js-KZkuS(d&;CLIw7jQkMDUTK4(nf#u8zbh}S+e0Y?8gy}X~{5#DY za(&8ksk1KJf)A4}QVm z(MI|6`I4-RTuNKXa`KqP*WU}De0DaA72MCWQA<`P zFMF?loyj==N8zSON5K3t>dxvw)bT44)joFR72$tQO`T9?qrUQ&K4+~*TRH*?)A;@$ zrq-7(O4K~Hew0C7fgZT^8;NS4BV-Sx*KdnoSM+t@IzzO&Q#yOzGbT~<)LNQBU5WH# zKT6a*weE;^1T-L;`h(7%Cx4Qtd1~FBLA?>_oxe)dJhdLdk>voQ4V}^1v+JBh%~R`8 v2DJ?7k(i!i{TBb4@UKAR`IxZ%AIQs7s}@;-SM?n0x0NdL+-fgm{}1OsOUo7J literal 0 HcmV?d00001 diff --git a/testing/btest/Traces/communityid/ipv6.pcap b/testing/btest/Traces/communityid/ipv6.pcap new file mode 100644 index 0000000000000000000000000000000000000000..eeeeb1f27cd21ed4e1b918e6596bd5375a421ff2 GIT binary patch literal 1828 zcmca|c+)~A1{MYcU}0bcax{e}h8z;%Vh97WL70KHcmeyxjKxQOdVXuWn*b6~U~^Dl zWGQ&M|K3{9aK(}rIv(w6>_0X*@bEB$q=7){=Wzx}<^>%FAWj350>dqL1_mY;)*eP? zW)4P17ABA_TQ~Eu3v^S_ZDo-Bd3$=&UsG)c+mZ#c z0caCD$QG?FybNAISAZ}Uzlad+7ek;;a{~kz9C#NoFlYfyzF5o1Pzy8^gt55aosC%c z&jqDS#q|X zk%F#5dU0+^0WTNGF-=`8)br*Aus8F8jsconRnN}g4fG-iV+lY}k^=BTfCPiXic|)M zRFKg>ML6J&#u9=eM2Fx7UopH`9Bl-Qb+DtCrT}9Z4RAd_zbO{PlnFo8LG zUJ?UCKFH{}CQj-(+7#^QXo#aX0ZsM+CT9@FlBUIoPSd6^M>B*oFz|ALTpgcC#q?ng zc6B7g)q8;^a|6q05XO@7u9B2p%s~ZC@E%}xF*VRdEpQS`6e{yeixi4ei^?*SQ*{)g zjSLJ7txU`ftWr&r(yUTa6H}~AEKSX=j10|9tWqtElC4ZEEX-qhxj=>mdq%k`7@JyH z7@3-zLxe1xJVSgveO;jnLL7raLVSWDJXb#tM?Ys*m*5b`kkDXfe-~G%M1W_2tBqU}ZNF0+N~#i73&I1;s*zqKFPV^nJf&IC<}ICf+~#56{tqMDXO^ zhsWQ$o^wB{Shp;oqLe!>Usmcm`r`gIEqYy^5Tgds&-qlNqx&3wOdV_$YF-+x`oyUm zMStf^ANNN2%GJ?yW~dwKFFMl4l=P)5wCSDH_CclSdhsfL#HI8%%N2c_=BkOh!Cbxl zq!_i9em|e)y5@V()s-p?{#U7~)7KB*K)<9^Y*48<#aE~6{gi%v zy!?0a71$TZ-%l-{5$X}wUa5rYd5Q5a{5k5e^4jk@Wlz43-aNm3bKIcJJ2U@`E~j_E zCu_RV)&oGUXgD14c`=!P; z$5SRXv+wh*spJ8&2cPnNM&+3IIa+uiHn?|7R1Pug5& zxyWT&f;MoLR>0&k37`%9zzQ&J%JL)0W$OFa47RcBEyXn#4k%#ybVb~k#Q_DFE;lcX zgf2}!JGjWwWd-&5aX@xmQOjs4&;~#_Smoq_Df%;e$2F?YqZc?4D=t{ z+@?fD^A|X-bp_a6?=JR@Wv6s3E0VEn+?0{8+gMg2V%crToUe6X+4pZ^S!-m@@B=Hr zv^h<*$)-5Qr`(qaqy@ic+Jq`3Z8ASUAK=(U*GNR{f}5P1iVGs?@{||6l5aDim8vP! zgaV|1=^?LDwA_jV3UI7SDDq^+VtYI)*KQ8aqFCkG9*+%XRMqF%y6aX^cdb^ml#Bxk zSlavzN7Or~6l-9Gri(dF%+CsP)&i|&Fst9!^xW$aq8esZFX zU6ct?bB7Pp<+%fOn33a}|Izs2&TC!=2%r_YT zuoEl5ZVH7wZW3LdK$@Md^BtTGx;!&aXS`xF zSJ_OcRAfRwL)`^;X$4FsbQkI__`%U8Zau{=`hrin8K!LP8W$G3Hb-?CY9|zlZbHG4 z@ZZ~gJCO8T9h%Ma)fzm{X5W8~)LC}XgK!#DaY=}b{0D=B-8FPqBpt{eb4JXyc~+Ui z{a&yVD_}a1LF`Dw4y*vvWo?xw?-o_u)o1lPo}75XMndW^@ss1|FzyOri&d+EAB-Q? zVT%=3Uo1Npw6k6J>6SQfEx*YEE(55}=a`?f~ChZGf+R(jzM4P#iHiur= zGRo3sgP=_Z)a7uOhc-72%**eAwot&i zEAgZ!mNj~!Re!ddn`pHaJ?6h-rylOo3YZ@AJJ^DT9~^D={1C}@&avwF(YD9D5qqj| zlZP&wu%`+?IJ!(f7fCKt`OhdU zt(aiJP0md(H*Mx~M^}g%IMh?lmtCI9nS~>4m**YP<>`nz9PZKzn0a|8)Zy@hqs^ag z@}&)ZQaI6Orld`2_Z2@_+H4ZENkLr>cX?>j8Fe}Q;AnF!)|WQ7Xxhw>v?(k3%+1TU z2->8gE{D53v`Ir<4nH{BOuM<+mCrE|o5z8i&`+2)p@ot*S*y-{Yjc~ef;RV}PKUdk zyPk^o#Ii=u%YQTAwf5{T+FR}2S=MJw|QtIVp&vfh(2+P z>hx(cmNh+}+}Xym?IM8% zqs@>^4{gG`eyw%aZ?f$A^?3ZAF}CYhF1mg{M_mqgX$8zK;C-me;RjZLGv!;dd}%|^ zZl@VTo}^9ZtuKzWwAmqOb3e3&0$Kq>n{?FW@B=Hrv?(Ln3<}UEgwm-*ZbRESx9jbk z*^)MCpRX>kw5brZ$v~YBcR6>>>FyiL?$EJpl5|(eiR>clu1evq2T-TOU0MMX%Q8`? z!w-%&Gx|g_FK^j8;gan!@4_BC+~nv&^YYyapFIRakOG`3cN^fzWyA#Q>Yp|9NU-lE z&SmHt>0;*aKK9sQE)Q+?U>6jAU?;ejNc1^yx z{%IS#GDYnA2PRl>msY^c%Rj&b3x064={3ef8_`pyyJYAy7ioeuQFi?T`90g&uHSCa z_3MVZ9PVLKlhaV|sG$QCn)z1S>;yR-s^HnrFVg&$Y}rcLt~Bbk>E3~YYJ<}x2*lN4@p zZkq6iNMhIZOMltH#;%?scJ0Fi3+~binAlZ^2^Rd|XmjtY9@>P@%jpTVG%tTjcKx#U zC(W>3zZ%i?>jiD0fL6eC{d%J=haXr0&Xh0C_N7gVrp=R*Hc7?v4_VsO3feq~It=d8 z3K-h-L7xVGUF@2UdXTQoYoZ%QSjkK6~TAYR+Zo8hymf;bUx;!d+ScGjsR^yP)s`E5Nkbxh#@- z`P?t3&9}MC0c?`OP0mdnizA6$ub)YL!p5$CB6ih7TPUCvFtO`XOt9bwR)AwyYK4b3 z;q&qbbk}dB?D`#V8Cqbwe)~n&uRrQ?xJxTwx_$#tm%|UN0MlmpPG8#4lW}M}=U0+8 zi+ATuv9$R_&}Ja&a=1$?U}*C&>T>vj6=2$&AlmrcU*3&r6M9U8*l75E?19Fy+= literal 0 HcmV?d00001 diff --git a/testing/btest/Traces/communityid/tcp.pcap b/testing/btest/Traces/communityid/tcp.pcap new file mode 100644 index 0000000000000000000000000000000000000000..92a5e9803bc3ae14b887709a853eae886362976a GIT binary patch literal 1114 zcmca|c+)~A1{MYwNB}Yzf%pa6eS5(q>|YSGI7y2tR_l`T_}%3HPPKQL_t3@H!(f6Sid+;FE6zuJ}JLKFFi9& z!6PIDWS{R*CWH^r!^Wfm95zde(ZVKU-{~J^4FZx14+S9h0bQQ~3>R6Di5D&~fqn3b z73c#r6It&xFoHsR-YHxr-gQPb5$MbSeM3D%1tS9k1%Gc|E|i}L765$!PnT#uU@&Q52Kit~0jdwAw81_QlvKFogX#m2 z8wDkq5hh}W@&vFOA7gQ&PJcK)e(vuZoBBmQzz;4V(bE6K>jpd+}cS{TI K#tLA{(*poB##N92 literal 0 HcmV?d00001 diff --git a/testing/btest/Traces/communityid/udp.pcap b/testing/btest/Traces/communityid/udp.pcap new file mode 100644 index 0000000000000000000000000000000000000000..169c924e4acb4785be65e15813dbe406174b3bb4 GIT binary patch literal 372 zcmca|c+)~A1{MYcU}0bca-t*MgTGGaWN-trLD=KRX0`jCyTE{h!Igo*B5f}NLxUg_ z&w&+;CLBO;)sn%KLB*w?ff=YDWCRefrRV3T=cFB29yI~h$pcap.out + btest-diff $pcap.out +done + +@TEST-START-FILE test-community-id-v1.zeek +event connection_state_remove(c: connection) { + print c$id, community_id_v1(c$id); +} +@TEST-END-FILE diff --git a/testing/btest/bifs/community_id/v1.zeek b/testing/btest/bifs/community_id/v1.zeek new file mode 100644 index 0000000000..ac3847ff14 --- /dev/null +++ b/testing/btest/bifs/community_id/v1.zeek @@ -0,0 +1,29 @@ +# @TEST-EXEC: zeek -b %INPUT >out +# @TEST-EXEC: btest-diff out + +function test_it(cid: conn_id, seed: count, expected: string) + { + local actual = community_id_v1(cid, seed); + local prefix = actual == expected ? "PASS" : "FAIL"; + print fmt("%s: expected '%s', got '%s' (%s, seed=%d)", prefix, expected, actual, cid, seed); + } + +event zeek_init() + { + test_it([$orig_h=1.2.3.4, $orig_p=1122/tcp, $resp_h=5.6.7.8, $resp_p=3344/tcp], 0, "1:wCb3OG7yAFWelaUydu0D+125CLM="); + test_it([$orig_h=1.2.3.4, $orig_p=1122/udp, $resp_h=5.6.7.8, $resp_p=3344/udp], 0, "1:0Mu9InQx6z4ZiCZM/7HXi2WMhOg="); + test_it([$orig_h=1.2.3.4, $orig_p=8/icmp, $resp_h=5.6.7.8, $resp_p=0/icmp], 0, "1:crodRHL2FEsHjbv3UkRrfbs4bZ0="); + test_it([$orig_h=[fe80:0001:0203:0405:0607:0809:0A0B:0C0D], $orig_p=128/icmp, + $resp_h=[fe80:1011:1213:1415:1617:1819:1A1B:1C1D], $resp_p=129/icmp], 0, "1:0bf7hyMJUwt3fMED7z8LIfRpBeo="); + + + test_it([$orig_h=1.2.3.4, $orig_p=1122/tcp, $resp_h=5.6.7.8, $resp_p=3344/tcp], 1, "1:HhA1B+6CoLbiKPEs5nhNYN4XWfk="); + test_it([$orig_h=1.2.3.4, $orig_p=1122/udp, $resp_h=5.6.7.8, $resp_p=3344/udp], 1, "1:OShq+iKDAMVouh/4bMxB9Sz4amw="); + test_it([$orig_h=1.2.3.4, $orig_p=8/icmp, $resp_h=5.6.7.8, $resp_p=0/icmp], 1, "1:9pr4ZGTICiuZoIh90RRYE2RyXpU="); + test_it([$orig_h=[fe80:0001:0203:0405:0607:0809:0A0B:0C0D], $orig_p=128/icmp, + $resp_h=[fe80:1011:1213:1415:1617:1819:1A1B:1C1D], $resp_p=129/icmp], 1, "1:IO27GQzPuCtNnwFvjWALMHu5tJE="); + + test_it([$orig_h=1.2.3.4, $orig_p=0/unknown, $resp_h=5.6.7.8, $resp_p=0/unknown], 0, ""); + test_it([$orig_h=[fe80:0001:0203:0405:0607:0809:0A0B:0C0D], $orig_p=0/unknown, + $resp_h=[fe80:1011:1213:1415:1617:1819:1A1B:1C1D], $resp_p=0/unknown], 1, ""); + } From b90351b7e6d24af40bbfcfa4ffa2a5b470b51c7d Mon Sep 17 00:00:00 2001 From: Christian Kreibich Date: Fri, 21 Apr 2023 19:24:02 +0200 Subject: [PATCH 2/4] policy: Import zeek-community-id scripts into protocols/conn frameworks/notice Slightly adapted for indentation. --- .../frameworks/notice/community-id.zeek | 35 +++++++++++++++++++ .../protocols/conn/community-id-logging.zeek | 26 ++++++++++++++ scripts/site/local.zeek | 7 ++++ scripts/test-all-policy.zeek | 2 ++ .../notice.log.cut | 3 ++ .../conn.log.cut | 2 ++ .../frameworks/notice/community-id.zeek | 31 ++++++++++++++++ .../protocols/conn/community-id-logging.zeek | 5 +++ 8 files changed, 111 insertions(+) create mode 100644 scripts/policy/frameworks/notice/community-id.zeek create mode 100644 scripts/policy/protocols/conn/community-id-logging.zeek create mode 100644 testing/btest/Baseline/scripts.policy.frameworks.notice.community-id/notice.log.cut create mode 100644 testing/btest/Baseline/scripts.policy.protocols.conn.community-id-logging/conn.log.cut create mode 100644 testing/btest/scripts/policy/frameworks/notice/community-id.zeek create mode 100644 testing/btest/scripts/policy/protocols/conn/community-id-logging.zeek diff --git a/scripts/policy/frameworks/notice/community-id.zeek b/scripts/policy/frameworks/notice/community-id.zeek new file mode 100644 index 0000000000..5f16b64285 --- /dev/null +++ b/scripts/policy/frameworks/notice/community-id.zeek @@ -0,0 +1,35 @@ +# Source this script in addition to protocols/conn/community-id +# to add Community ID to notices. + +# Only support loading this if the main script is also loaded. +@load base/protocols/conn +@load base/frameworks/notice + +@ifdef ( CommunityID::seed ) + +module CommunityID::Notice; + +export { + # Turn notice support on/off at runtime. When disabled, + # this still leaves the `community_id` string in the notice + # log, just unset. + option enabled: bool = T; + + redef record Notice::Info += { + community_id: string &optional &log; + }; +} + +hook Notice::notice(n: Notice::Info) + { + if ( CommunityID::Notice::enabled && n?$conn && n$conn?$conn ) + { + local info = n$conn$conn; + # This is set during new_connection(), so it should + # always be there, but better safe than sorry. + if ( info?$community_id ) + n$community_id = info$community_id; + } + } + +@endif diff --git a/scripts/policy/protocols/conn/community-id-logging.zeek b/scripts/policy/protocols/conn/community-id-logging.zeek new file mode 100644 index 0000000000..82bb9049a5 --- /dev/null +++ b/scripts/policy/protocols/conn/community-id-logging.zeek @@ -0,0 +1,26 @@ +##! Adds community hash IDs to conn.log. +@load base/protocols/conn + +module CommunityID; + +export { + # An unsigned 16-bit number to seed our hashing + option seed: count = 0; + + # Whether to add a base64 pass over the hash digest. + # Enabled by default, since it shortens the output. + option do_base64: bool = T; + + # Add the ID string field to the connection log record. + redef record Conn::Info += { + community_id: string &optional &log; + }; +} + +module Conn; + +event new_connection(c: connection) + { + Conn::set_conn(c, F); # likely first to access :-/ + c$conn$community_id = community_id_v1(c$id, CommunityID::seed, CommunityID::do_base64); + } diff --git a/scripts/site/local.zeek b/scripts/site/local.zeek index 5c72ca7c93..512b1ea9cc 100644 --- a/scripts/site/local.zeek +++ b/scripts/site/local.zeek @@ -90,6 +90,9 @@ redef digest_salt = "Please change this value."; # Extend email alerting to include hostnames @load policy/frameworks/notice/extend-email/hostnames +# Extend the notice.log with Community ID hashes +# @load policy/frameworks/notice/community-id + # Enable logging of telemetry data into telemetry.log and # telemetry_histogram.log. @load frameworks/telemetry/log @@ -98,6 +101,10 @@ redef digest_salt = "Please change this value."; # this might impact performance a bit. # @load policy/protocols/ssl/heartbleed +# Uncomment the following line to enable logging of Community ID hashes in +# the conn.log file. +# @load policy/protocols/conn/community-id-logging + # Uncomment the following line to enable logging of connection VLANs. Enabling # this adds two VLAN fields to the conn.log file. # @load policy/protocols/conn/vlan-logging diff --git a/scripts/test-all-policy.zeek b/scripts/test-all-policy.zeek index 87dd2e661c..63247006d0 100644 --- a/scripts/test-all-policy.zeek +++ b/scripts/test-all-policy.zeek @@ -64,6 +64,7 @@ @load frameworks/files/hash-all-files.zeek @load frameworks/notice/__load__.zeek @load frameworks/notice/actions/drop.zeek +@load frameworks/notice/community-id.zeek @load frameworks/notice/extend-email/hostnames.zeek @load files/x509/disable-certificate-events-known-certs.zeek @load frameworks/packet-filter/shunt.zeek @@ -85,6 +86,7 @@ @load misc/weird-stats.zeek @load misc/trim-trace-file.zeek @load misc/unknown-protocols.zeek +@load protocols/conn/community-id-logging.zeek @load protocols/conn/known-hosts.zeek @load protocols/conn/known-services.zeek @load protocols/conn/mac-logging.zeek diff --git a/testing/btest/Baseline/scripts.policy.frameworks.notice.community-id/notice.log.cut b/testing/btest/Baseline/scripts.policy.frameworks.notice.community-id/notice.log.cut new file mode 100644 index 0000000000..7227d38c91 --- /dev/null +++ b/testing/btest/Baseline/scripts.policy.frameworks.notice.community-id/notice.log.cut @@ -0,0 +1,3 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +- - - - - - COMMUNITY_ID_INIT Zeek initializing +141.142.228.5 59856 192.150.187.43 80 tcp 1:yvyB8h+3dnggTZW0UEITWCst97w= COMMUNITY_ID_CONN_ESTABLISHED Connection establishment diff --git a/testing/btest/Baseline/scripts.policy.protocols.conn.community-id-logging/conn.log.cut b/testing/btest/Baseline/scripts.policy.protocols.conn.community-id-logging/conn.log.cut new file mode 100644 index 0000000000..b02a020eb9 --- /dev/null +++ b/testing/btest/Baseline/scripts.policy.protocols.conn.community-id-logging/conn.log.cut @@ -0,0 +1,2 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +141.142.228.5 59856 192.150.187.43 80 tcp - 1:yvyB8h+3dnggTZW0UEITWCst97w= diff --git a/testing/btest/scripts/policy/frameworks/notice/community-id.zeek b/testing/btest/scripts/policy/frameworks/notice/community-id.zeek new file mode 100644 index 0000000000..8383017c63 --- /dev/null +++ b/testing/btest/scripts/policy/frameworks/notice/community-id.zeek @@ -0,0 +1,31 @@ +# This test verifies Community ID presence in the notice log, when +# that part of the package is loaded. The test creates one notice +# without connection state and one with, and verifies that the latter +# includes the Community ID value for it. + +# @TEST-EXEC: zeek -b -r $TRACES/http/get.trace %INPUT +# @TEST-EXEC: zeek-cut id.orig_h id.orig_p id.resp_h id.resp_p proto community_id note msg < notice.log > notice.log.cut +# @TEST-EXEC: btest-diff notice.log.cut + +@load protocols/conn/community-id-logging +@load frameworks/notice/community-id + +redef enum Notice::Type += { + COMMUNITY_ID_INIT, + COMMUNITY_ID_CONN_ESTABLISHED, +}; + +event zeek_init() + { + # A notice without connection context + NOTICE([$note=COMMUNITY_ID_INIT, + $msg="Zeek initializing"]); + } + +event connection_established(c: connection) + { + # A notice with connection context + NOTICE([$note=COMMUNITY_ID_CONN_ESTABLISHED, + $msg="Connection establishment", + $conn=c]); + } diff --git a/testing/btest/scripts/policy/protocols/conn/community-id-logging.zeek b/testing/btest/scripts/policy/protocols/conn/community-id-logging.zeek new file mode 100644 index 0000000000..4dd18920d5 --- /dev/null +++ b/testing/btest/scripts/policy/protocols/conn/community-id-logging.zeek @@ -0,0 +1,5 @@ +# @TEST-EXEC: zeek -b -r $TRACES/http/get.trace %INPUT +# @TEST-EXEC: zeek-cut id.orig_h id.orig_p id.resp_h id.resp_p proto service community_id < conn.log > conn.log.cut +# @TEST-EXEC: btest-diff conn.log.cut + +@load protocols/conn/community-id-logging From 547e1b62809d1978dd0e7c000663b360e3109382 Mon Sep 17 00:00:00 2001 From: Arne Welzel Date: Fri, 21 Apr 2023 19:50:48 +0200 Subject: [PATCH 3/4] NEWS: Add entry for Community ID --- NEWS | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/NEWS b/NEWS index f6f29aab36..453f083524 100644 --- a/NEWS +++ b/NEWS @@ -100,6 +100,11 @@ New Functionality To disable this functionality, pass ``--disable-javascript`` to configure. +- Zeek now provides native "Community ID" support with a new bif called + ``community_id_v1()``. Two policy scripts ``protocols/conn/community-id-logging`` + and ``frameworks/notice/community-id`` extend the respective logs with a + community_id similar to what the external zeek-community-id plugin provides. + - Introduce a new command-line option ``-V`` / ``--build-info``. It produces verbose output in JSON format about the repository state and any included plugins. From 0bbd7cab1b9e0c913c21d812fb0931fbcd92063d Mon Sep 17 00:00:00 2001 From: Arne Welzel Date: Fri, 21 Apr 2023 21:03:38 +0200 Subject: [PATCH 4/4] testing/external: Bump hashes for community_id addition --- testing/external/commit-hash.zeek-testing | 2 +- testing/external/commit-hash.zeek-testing-private | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/testing/external/commit-hash.zeek-testing b/testing/external/commit-hash.zeek-testing index 3c219f5e9d..95889c2ecb 100644 --- a/testing/external/commit-hash.zeek-testing +++ b/testing/external/commit-hash.zeek-testing @@ -1 +1 @@ -18a9ac00f5b7617e8660d4ba680a25291d2b44f7 \ No newline at end of file +fbe011cee82b7d95b42cdac604ad006ec60ef823 diff --git a/testing/external/commit-hash.zeek-testing-private b/testing/external/commit-hash.zeek-testing-private index 72bdde067f..b9916461ae 100644 --- a/testing/external/commit-hash.zeek-testing-private +++ b/testing/external/commit-hash.zeek-testing-private @@ -1 +1 @@ -98e8aee2f09bff7e8138290242274b5ffd834e58 +44cc8beefad84ad58efbf1e6bdaf318dc4dcec7a