diff --git a/CHANGES b/CHANGES index 446dd60737..bcb10f0e52 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,30 @@ +6.0.0-dev.410 | 2023-04-24 09:46:59 +0200 + + * policy: Import zeek-community-id scripts into protocols/conn frameworks/notice (Christian Kreibich, Corelight) + + * Add community_id_v1() based on corelight/zeek-community-id (Christian Kreibich, Corelight) + + "Community ID" has become an established flow hash for connection correlation + across different monitoring and storage systems. Other NSMs have had native + and built-in support for Community ID since late 2018. And even though the + roots of "Community ID" are very close to Zeek, Zeek itself has never provided + out-of-the-box support and instead required users to install an external plugin. + + While we try to make that installation as easy as possible, an external plugin + always sets the bar higher for an initial setup and can be intimidating. + It also requires a rebuild operation of the plugin during upgrades. Nothing + overly complicated, but somewhat unnecessary for such popular functionality. + + This isn't a 1:1 import. The options are parameters and the "verbose" + functionality has been removed. Further, instead of a `connection` + record, the new bif works with `conn_id`, allowing computation of the + hash with little effort on the command line: + + $ zeek -e 'print community_id_v1([$orig_h=1.2.3.4, $orig_p=1024/tcp, $resp_h=5.6.7.8, $resp_p=80/tcp])' + 1:RcCrCS5fwYUeIzgDDx64EN3+okU + + Reference: https://github.com/corelight/zeek-community-id/ + 6.0.0-dev.405 | 2023-04-24 09:23:01 +0200 * file_analysis/Manager: Remove RunState.h include (Arne Welzel, Corelight) diff --git a/NEWS b/NEWS index 936009ae89..2ab2ec0567 100644 --- a/NEWS +++ b/NEWS @@ -100,6 +100,16 @@ New Functionality To disable this functionality, pass ``--disable-javascript`` to configure. +- Zeek now provides native "Community ID" support with a new bif called + ``community_id_v1()``. Two policy scripts ``protocols/conn/community-id-logging`` + and ``frameworks/notice/community-id`` extend the respective logs with a + ``community_id`` field the same way as the external zeek-community-id plugin + provides. A main difference to the external ``hash_conn()`` bif is that the + ``community_id_v1()`` takes a ``conn_id`` record instead of a ``connection``. + + Loading the new policy scripts and using the external zeek-community-id + plugin at the same time is unsupported. + - Introduce a new command-line option ``-V`` / ``--build-info``. It produces verbose output in JSON format about the repository state and any included plugins. diff --git a/VERSION b/VERSION index 42d73c4bb1..b378db4f04 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -6.0.0-dev.405 +6.0.0-dev.410 diff --git a/scripts/base/init-bare.zeek b/scripts/base/init-bare.zeek index f3239c11a3..a02f3e38e7 100644 --- a/scripts/base/init-bare.zeek +++ b/scripts/base/init-bare.zeek @@ -2034,6 +2034,7 @@ type gtp_delete_pdp_ctx_response_elements: record { # Prototypes of Zeek built-in functions. @load base/bif/zeek.bif +@load base/bif/communityid.bif @load base/bif/stats.bif @load base/bif/reporter.bif @load base/bif/strings.bif diff --git a/scripts/policy/frameworks/notice/community-id.zeek b/scripts/policy/frameworks/notice/community-id.zeek new file mode 100644 index 0000000000..5f16b64285 --- /dev/null +++ b/scripts/policy/frameworks/notice/community-id.zeek @@ -0,0 +1,35 @@ +# Source this script in addition to protocols/conn/community-id +# to add Community ID to notices. + +# Only support loading this if the main script is also loaded. +@load base/protocols/conn +@load base/frameworks/notice + +@ifdef ( CommunityID::seed ) + +module CommunityID::Notice; + +export { + # Turn notice support on/off at runtime. When disabled, + # this still leaves the `community_id` string in the notice + # log, just unset. + option enabled: bool = T; + + redef record Notice::Info += { + community_id: string &optional &log; + }; +} + +hook Notice::notice(n: Notice::Info) + { + if ( CommunityID::Notice::enabled && n?$conn && n$conn?$conn ) + { + local info = n$conn$conn; + # This is set during new_connection(), so it should + # always be there, but better safe than sorry. + if ( info?$community_id ) + n$community_id = info$community_id; + } + } + +@endif diff --git a/scripts/policy/protocols/conn/community-id-logging.zeek b/scripts/policy/protocols/conn/community-id-logging.zeek new file mode 100644 index 0000000000..82bb9049a5 --- /dev/null +++ b/scripts/policy/protocols/conn/community-id-logging.zeek @@ -0,0 +1,26 @@ +##! Adds community hash IDs to conn.log. +@load base/protocols/conn + +module CommunityID; + +export { + # An unsigned 16-bit number to seed our hashing + option seed: count = 0; + + # Whether to add a base64 pass over the hash digest. + # Enabled by default, since it shortens the output. + option do_base64: bool = T; + + # Add the ID string field to the connection log record. + redef record Conn::Info += { + community_id: string &optional &log; + }; +} + +module Conn; + +event new_connection(c: connection) + { + Conn::set_conn(c, F); # likely first to access :-/ + c$conn$community_id = community_id_v1(c$id, CommunityID::seed, CommunityID::do_base64); + } diff --git a/scripts/site/local.zeek b/scripts/site/local.zeek index 5c72ca7c93..512b1ea9cc 100644 --- a/scripts/site/local.zeek +++ b/scripts/site/local.zeek @@ -90,6 +90,9 @@ redef digest_salt = "Please change this value."; # Extend email alerting to include hostnames @load policy/frameworks/notice/extend-email/hostnames +# Extend the notice.log with Community ID hashes +# @load policy/frameworks/notice/community-id + # Enable logging of telemetry data into telemetry.log and # telemetry_histogram.log. @load frameworks/telemetry/log @@ -98,6 +101,10 @@ redef digest_salt = "Please change this value."; # this might impact performance a bit. # @load policy/protocols/ssl/heartbleed +# Uncomment the following line to enable logging of Community ID hashes in +# the conn.log file. +# @load policy/protocols/conn/community-id-logging + # Uncomment the following line to enable logging of connection VLANs. Enabling # this adds two VLAN fields to the conn.log file. # @load policy/protocols/conn/vlan-logging diff --git a/scripts/test-all-policy.zeek b/scripts/test-all-policy.zeek index 81c16c39bb..389dd1d442 100644 --- a/scripts/test-all-policy.zeek +++ b/scripts/test-all-policy.zeek @@ -66,6 +66,7 @@ @load frameworks/files/hash-all-files.zeek @load frameworks/notice/__load__.zeek @load frameworks/notice/actions/drop.zeek +@load frameworks/notice/community-id.zeek @load frameworks/notice/extend-email/hostnames.zeek @load files/x509/disable-certificate-events-known-certs.zeek @load frameworks/packet-filter/shunt.zeek @@ -87,6 +88,7 @@ @load misc/weird-stats.zeek @load misc/trim-trace-file.zeek @load misc/unknown-protocols.zeek +@load protocols/conn/community-id-logging.zeek @load protocols/conn/known-hosts.zeek @load protocols/conn/known-services.zeek @load protocols/conn/mac-logging.zeek diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 64f0577e42..773a9a6efc 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -123,6 +123,7 @@ set(SUPERVISOR_SRCS supervisor/Supervisor.cc Pipe.cc) set(BIF_SRCS zeek.bif + communityid.bif stats.bif event.bif const.bif diff --git a/src/Func.cc b/src/Func.cc index 7cbfaf764a..f56719ee00 100644 --- a/src/Func.cc +++ b/src/Func.cc @@ -44,6 +44,7 @@ // break what symbols are available when, which keeps the build from breaking. // clang-format off #include "zeek.bif.func_h" +#include "communityid.bif.func_h" #include "stats.bif.func_h" #include "reporter.bif.func_h" #include "strings.bif.func_h" @@ -53,6 +54,7 @@ #include "CPP-load.bif.func_h" #include "zeek.bif.func_def" +#include "communityid.bif.func_def" #include "stats.bif.func_def" #include "reporter.bif.func_def" #include "strings.bif.func_def" @@ -1036,6 +1038,7 @@ void init_primary_bifs() var_sizes = id::find_type("var_sizes")->AsTableType(); #include "CPP-load.bif.func_init" +#include "communityid.bif.func_init" #include "option.bif.func_init" #include "packet_analysis.bif.func_init" #include "reporter.bif.func_init" diff --git a/src/communityid.bif b/src/communityid.bif new file mode 100644 index 0000000000..d94321d0cd --- /dev/null +++ b/src/communityid.bif @@ -0,0 +1,130 @@ +%%{ // C segment +#include "zeek/IPAddr.h" +#include "zeek/Val.h" +#include "zeek/digest.h" +#include "zeek/packet_analysis/protocol/icmp/ICMP.h" +%%} + +## Compute the Community ID hash (v1) from a connection identifier. +## +## cid: The identifier of the connection for which to compute the community-id. +## +## Returns: The Community ID hash of the connection identifier as string. +## +function community_id_v1%(cid: conn_id, seed: count &default=0, do_base64: bool &default=T%): string +%{ + const auto *cid_rec = cid->AsRecordVal(); + + uint16_t hash_seed = htons(seed); + const uint32_t *hash_src_addr = 0; + const uint32_t *hash_dst_addr = 0; + uint8_t hash_proto = 0; + uint8_t hash_padbyte = 0; + uint16_t hash_src_port = 0; + uint16_t hash_dst_port = 0; + + const auto& orig_addr = cid_rec->GetFieldAs(0); + const auto& orig_port = cid_rec->GetFieldAs(1); + const auto& resp_addr = cid_rec->GetFieldAs(2); + const auto& resp_port = cid_rec->GetFieldAs(3); + + bool is_ipv4 = orig_addr.GetBytes(&hash_src_addr) == 1; + resp_addr.GetBytes(&hash_dst_addr); + TransportProto proto = orig_port->PortType(); + + // Zeek's transport protocol aliases different underlying + // protocols, particularly IPv4's and v6's ICMP... + switch (proto) { + case TRANSPORT_TCP: + hash_proto = IPPROTO_TCP; + break; + case TRANSPORT_UDP: + hash_proto = IPPROTO_UDP; + break; + case TRANSPORT_ICMP: + if (is_ipv4) + hash_proto = IPPROTO_ICMP; + else + hash_proto = IPPROTO_ICMPV6; + + break; + case TRANSPORT_UNKNOWN: + emit_builtin_error("CommunityID: unknown transport layer", cid); + return zeek::make_intrusive(""); + default: + emit_builtin_error("CommunityID: unhandled transport layer", cid); + return zeek::make_intrusive(""); + } + + hash_src_port = htons((uint16_t) orig_port->Port()); + hash_dst_port = htons((uint16_t) resp_port->Port()); + + // XXX: resolve whether we should copy is_one_way into the + // Connection instance at construction time, along with the other + // ConnID fields (see Conn.cc around line 125). + // awelzel: Maybe the is_one_way should be just a helper? + + bool is_one_way = false; + + if (TRANSPORT_ICMP == proto) { + if (is_ipv4) + zeek::packet_analysis::ICMP::ICMP4_counterpart(ntohs(hash_src_port), + ntohs(hash_dst_port), + is_one_way); + else + zeek::packet_analysis::ICMP::ICMP6_counterpart(ntohs(hash_src_port), + ntohs(hash_dst_port), + is_one_way); + } + + if (is_one_way || zeek::addr_port_canon_lt(orig_addr, hash_src_port, + resp_addr, hash_dst_port)) { + // All good, no need to flip + } else { + // Need to flip endpoints for hashing. + std::swap(hash_src_addr, hash_dst_addr); + std::swap(hash_src_port, hash_dst_port); + } + + auto digest_update = [](EVP_MD_CTX *ctx, const void* data, unsigned long len) { + zeek::detail::hash_update(ctx, data, len); + return len; + }; + + int dlen = 0; + auto *ctx = zeek::detail::hash_init(zeek::detail::Hash_SHA1); + + dlen += digest_update(ctx, &hash_seed, 2); + dlen += digest_update(ctx, hash_src_addr, is_ipv4 ? 4 : 16); + dlen += digest_update(ctx, hash_dst_addr, is_ipv4 ? 4 : 16); + dlen += digest_update(ctx, &hash_proto, 1); + dlen += digest_update(ctx, &hash_padbyte, 1); + dlen += digest_update(ctx, &hash_src_port, 2); + dlen += digest_update(ctx, &hash_dst_port, 2); + + u_char digest[SHA_DIGEST_LENGTH]; + zeek::detail::hash_final(ctx, digest); + + // We currently have no real versioning/hash configuration logic, + // so we simply prefix "1:" to the hash. + std::string ver("1:"); + zeek::String *res = 0; + + if (do_base64) { + char *outbuf = 0; + int outlen = 0; + + zeek::detail::Base64Converter enc{nullptr}; + enc.Encode(SHA_DIGEST_LENGTH, digest, &outlen, &outbuf); + res = new zeek::String(ver + std::string(outbuf, outlen)); + // When given outlen = 0, the Encode() method creates the + // buffer it returns as outbuf, so we must delete it. + delete[] outbuf; + } else { + // The following returns a static buffer; no need to delete. + const char *ascii_digest = zeek::detail::sha1_digest_print(digest); + res = new zeek::String(ver + ascii_digest); + } + + return zeek::make_intrusive(res); +%} diff --git a/testing/btest/Baseline/bifs.community_id.run-pcaps/arp.pcap.out b/testing/btest/Baseline/bifs.community_id.run-pcaps/arp.pcap.out new file mode 100644 index 0000000000..49d861c74c --- /dev/null +++ b/testing/btest/Baseline/bifs.community_id.run-pcaps/arp.pcap.out @@ -0,0 +1 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. diff --git a/testing/btest/Baseline/bifs.community_id.run-pcaps/icmp.pcap.out b/testing/btest/Baseline/bifs.community_id.run-pcaps/icmp.pcap.out new file mode 100644 index 0000000000..0aca4bba84 --- /dev/null +++ b/testing/btest/Baseline/bifs.community_id.run-pcaps/icmp.pcap.out @@ -0,0 +1,2 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +[orig_h=192.168.0.89, orig_p=8/icmp, resp_h=192.168.0.1, resp_p=0/icmp], 1:X0snYXpgwiv9TZtqg64sgzUn6Dk= diff --git a/testing/btest/Baseline/bifs.community_id.run-pcaps/icmp6.pcap.out b/testing/btest/Baseline/bifs.community_id.run-pcaps/icmp6.pcap.out new file mode 100644 index 0000000000..caa30f231d --- /dev/null +++ b/testing/btest/Baseline/bifs.community_id.run-pcaps/icmp6.pcap.out @@ -0,0 +1,15 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +[orig_h=3ffe:501:0:1802:260:97ff:feb6:7ff0, orig_p=3/icmp, resp_h=3ffe:507:0:1:200:86ff:fe05:80da, resp_p=0/icmp], 1:bnQKq8A2r//dWnkRW2EYcMhShjc= +[orig_h=3ffe:501:1800:2345::2, orig_p=3/icmp, resp_h=3ffe:507:0:1:200:86ff:fe05:80da, resp_p=0/icmp], 1:2ObVBgIn28oZvibYZhZMBgh7WdQ= +[orig_h=3ffe:501:410:0:2c0:dfff:fe47:33e, orig_p=1/icmp, resp_h=3ffe:507:0:1:200:86ff:fe05:80da, resp_p=4/icmp], 1:hLZd0XGWojozrvxqE0dWB1iM6R0= +[orig_h=3ffe:507:0:1:200:86ff:fe05:80da, orig_p=1/icmp, resp_h=3ffe:501:4819::42, resp_p=4/icmp], 1:jwuBy9UWZK1KUFqJV5cHdVpfrlY= +[orig_h=3ffe:507:0:1:200:86ff:fe05:80da, orig_p=128/icmp, resp_h=3ffe:501:0:1001::2, resp_p=129/icmp], 1:+TW+HtLHvV1xnGhV1lv7XoJrqQg= +[orig_h=3ffe:507:0:1:200:86ff:fe05:80da, orig_p=128/icmp, resp_h=3ffe:507:0:1:260:97ff:fe07:69ea, resp_p=129/icmp], 1:GpbEQrKqfWtsfsFiqg8fufoZe5Y= +[orig_h=3ffe:507:0:1:200:86ff:fe05:80da, orig_p=135/icmp, resp_h=3ffe:507:0:1:260:97ff:fe07:69ea, resp_p=136/icmp], 1:ORxAZfN3ld7Sv73/HQTNnvgxbpY= +[orig_h=3ffe:507:0:1:200:86ff:fe05:80da, orig_p=135/icmp, resp_h=ff02::1:ff07:69ea, resp_p=136/icmp], 1:MEixa66kuz0OMvlQqnAIzP3n2xg= +[orig_h=3ffe:507:0:1:260:97ff:fe07:69ea, orig_p=135/icmp, resp_h=3ffe:507:0:1:200:86ff:fe05:80da, resp_p=136/icmp], 1:BtEUCMYecYjJ7spEkVZDiCFaMTY= +[orig_h=3ffe:507:0:1:260:97ff:fe07:69ea, orig_p=3/icmp, resp_h=3ffe:507:0:1:200:86ff:fe05:80da, resp_p=0/icmp], 1:NdobDX8PQNJbAyfkWxhtL2Pqp5w= +[orig_h=fe80::200:86ff:fe05:80da, orig_p=133/icmp, resp_h=ff02::2, resp_p=134/icmp], 1:hO+sN4H+MG5MY/8hIrXPqc4ZQz0= +[orig_h=fe80::200:86ff:fe05:80da, orig_p=135/icmp, resp_h=fe80::260:97ff:fe07:69ea, resp_p=136/icmp], 1:dGHyGvjMfljg6Bppwm3bg0LO8TY= +[orig_h=fe80::260:97ff:fe07:69ea, orig_p=134/icmp, resp_h=ff02::1, resp_p=133/icmp], 1:pkvHqCL88/tg1k4cPigmZXUtL00= +[orig_h=fe80::260:97ff:fe07:69ea, orig_p=135/icmp, resp_h=fe80::200:86ff:fe05:80da, resp_p=136/icmp], 1:zavyT/cezQr1fmImYCwYnMXbgck= diff --git a/testing/btest/Baseline/bifs.community_id.run-pcaps/ipv6.pcap.out b/testing/btest/Baseline/bifs.community_id.run-pcaps/ipv6.pcap.out new file mode 100644 index 0000000000..a80b4bbb38 --- /dev/null +++ b/testing/btest/Baseline/bifs.community_id.run-pcaps/ipv6.pcap.out @@ -0,0 +1,2 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +[orig_h=2001:470:e5bf:dead:4957:2174:e82c:4887, orig_p=63943/tcp, resp_h=2607:f8b0:400c:c03::1a, resp_p=25/tcp], 1:/qFaeAR+gFe1KYjMzVDsMv+wgU4= diff --git a/testing/btest/Baseline/bifs.community_id.run-pcaps/sctp.pcap.out b/testing/btest/Baseline/bifs.community_id.run-pcaps/sctp.pcap.out new file mode 100644 index 0000000000..49d861c74c --- /dev/null +++ b/testing/btest/Baseline/bifs.community_id.run-pcaps/sctp.pcap.out @@ -0,0 +1 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. diff --git a/testing/btest/Baseline/bifs.community_id.run-pcaps/tcp.pcap.out b/testing/btest/Baseline/bifs.community_id.run-pcaps/tcp.pcap.out new file mode 100644 index 0000000000..7a15ba4be8 --- /dev/null +++ b/testing/btest/Baseline/bifs.community_id.run-pcaps/tcp.pcap.out @@ -0,0 +1,2 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +[orig_h=128.232.110.120, orig_p=34855/tcp, resp_h=66.35.250.204, resp_p=80/tcp], 1:LQU9qZlK+B5F3KDmev6m5PMibrg= diff --git a/testing/btest/Baseline/bifs.community_id.run-pcaps/udp.pcap.out b/testing/btest/Baseline/bifs.community_id.run-pcaps/udp.pcap.out new file mode 100644 index 0000000000..b894f78d61 --- /dev/null +++ b/testing/btest/Baseline/bifs.community_id.run-pcaps/udp.pcap.out @@ -0,0 +1,2 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +[orig_h=192.168.1.52, orig_p=54585/udp, resp_h=8.8.8.8, resp_p=53/udp], 1:d/FP5EW3wiY1vCndhwleRRKHowQ= diff --git a/testing/btest/Baseline/bifs.community_id.v1/out b/testing/btest/Baseline/bifs.community_id.v1/out new file mode 100644 index 0000000000..0cbcb5800e --- /dev/null +++ b/testing/btest/Baseline/bifs.community_id.v1/out @@ -0,0 +1,11 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +PASS: expected '1:wCb3OG7yAFWelaUydu0D+125CLM=', got '1:wCb3OG7yAFWelaUydu0D+125CLM=' ([orig_h=1.2.3.4, orig_p=1122/tcp, resp_h=5.6.7.8, resp_p=3344/tcp], seed=0) +PASS: expected '1:0Mu9InQx6z4ZiCZM/7HXi2WMhOg=', got '1:0Mu9InQx6z4ZiCZM/7HXi2WMhOg=' ([orig_h=1.2.3.4, orig_p=1122/udp, resp_h=5.6.7.8, resp_p=3344/udp], seed=0) +PASS: expected '1:crodRHL2FEsHjbv3UkRrfbs4bZ0=', got '1:crodRHL2FEsHjbv3UkRrfbs4bZ0=' ([orig_h=1.2.3.4, orig_p=8/icmp, resp_h=5.6.7.8, resp_p=0/icmp], seed=0) +PASS: expected '1:0bf7hyMJUwt3fMED7z8LIfRpBeo=', got '1:0bf7hyMJUwt3fMED7z8LIfRpBeo=' ([orig_h=fe80:1:203:405:607:809:a0b:c0d, orig_p=128/icmp, resp_h=fe80:1011:1213:1415:1617:1819:1a1b:1c1d, resp_p=129/icmp], seed=0) +PASS: expected '1:HhA1B+6CoLbiKPEs5nhNYN4XWfk=', got '1:HhA1B+6CoLbiKPEs5nhNYN4XWfk=' ([orig_h=1.2.3.4, orig_p=1122/tcp, resp_h=5.6.7.8, resp_p=3344/tcp], seed=1) +PASS: expected '1:OShq+iKDAMVouh/4bMxB9Sz4amw=', got '1:OShq+iKDAMVouh/4bMxB9Sz4amw=' ([orig_h=1.2.3.4, orig_p=1122/udp, resp_h=5.6.7.8, resp_p=3344/udp], seed=1) +PASS: expected '1:9pr4ZGTICiuZoIh90RRYE2RyXpU=', got '1:9pr4ZGTICiuZoIh90RRYE2RyXpU=' ([orig_h=1.2.3.4, orig_p=8/icmp, resp_h=5.6.7.8, resp_p=0/icmp], seed=1) +PASS: expected '1:IO27GQzPuCtNnwFvjWALMHu5tJE=', got '1:IO27GQzPuCtNnwFvjWALMHu5tJE=' ([orig_h=fe80:1:203:405:607:809:a0b:c0d, orig_p=128/icmp, resp_h=fe80:1011:1213:1415:1617:1819:1a1b:1c1d, resp_p=129/icmp], seed=1) +PASS: expected '', got '' ([orig_h=1.2.3.4, orig_p=0/unknown, resp_h=5.6.7.8, resp_p=0/unknown], seed=0) +PASS: expected '', got '' ([orig_h=fe80:1:203:405:607:809:a0b:c0d, orig_p=0/unknown, resp_h=fe80:1011:1213:1415:1617:1819:1a1b:1c1d, resp_p=0/unknown], seed=1) diff --git a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log index 22b3b8b55a..e1d05d14f1 100644 --- a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log @@ -11,6 +11,7 @@ scripts/base/init-bare.zeek build/scripts/base/bif/const.bif.zeek build/scripts/base/bif/types.bif.zeek build/scripts/base/bif/zeek.bif.zeek + build/scripts/base/bif/communityid.bif.zeek build/scripts/base/bif/stats.bif.zeek build/scripts/base/bif/reporter.bif.zeek build/scripts/base/bif/strings.bif.zeek diff --git a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log index 562168d47e..f9c627c29c 100644 --- a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log @@ -11,6 +11,7 @@ scripts/base/init-bare.zeek build/scripts/base/bif/const.bif.zeek build/scripts/base/bif/types.bif.zeek build/scripts/base/bif/zeek.bif.zeek + build/scripts/base/bif/communityid.bif.zeek build/scripts/base/bif/stats.bif.zeek build/scripts/base/bif/reporter.bif.zeek build/scripts/base/bif/strings.bif.zeek diff --git a/testing/btest/Baseline/plugins.hooks/output b/testing/btest/Baseline/plugins.hooks/output index a29ac7419c..e39717caf7 100644 --- a/testing/btest/Baseline/plugins.hooks/output +++ b/testing/btest/Baseline/plugins.hooks/output @@ -927,6 +927,7 @@ 0.000000 MetaHookPost LoadFile(0, ./cardinality-counter.bif.zeek, <...>/cardinality-counter.bif.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, ./certificate-event-cache, <...>/certificate-event-cache.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, ./comm.bif.zeek, <...>/comm.bif.zeek) -> -1 +0.000000 MetaHookPost LoadFile(0, ./communityid.bif.zeek, <...>/communityid.bif.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, ./const-dos-error, <...>/const-dos-error.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, ./const-nt-status, <...>/const-nt-status.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, ./const.bif.zeek, <...>/const.bif.zeek) -> -1 @@ -1044,6 +1045,7 @@ 0.000000 MetaHookPost LoadFile(0, base<...>/broker, <...>/broker) -> -1 0.000000 MetaHookPost LoadFile(0, base<...>/cluster, <...>/cluster) -> -1 0.000000 MetaHookPost LoadFile(0, base<...>/comm.bif, <...>/comm.bif.zeek) -> -1 +0.000000 MetaHookPost LoadFile(0, base<...>/communityid.bif, <...>/communityid.bif.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, base<...>/config, <...>/config) -> -1 0.000000 MetaHookPost LoadFile(0, base<...>/conn, <...>/conn) -> -1 0.000000 MetaHookPost LoadFile(0, base<...>/conn-ids, <...>/conn-ids.zeek) -> -1 @@ -1315,6 +1317,7 @@ 0.000000 MetaHookPost LoadFileExtended(0, ./cardinality-counter.bif.zeek, <...>/cardinality-counter.bif.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, ./certificate-event-cache, <...>/certificate-event-cache.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, ./comm.bif.zeek, <...>/comm.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./communityid.bif.zeek, <...>/communityid.bif.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, ./const-dos-error, <...>/const-dos-error.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, ./const-nt-status, <...>/const-nt-status.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, ./const.bif.zeek, <...>/const.bif.zeek) -> (-1, ) @@ -1432,6 +1435,7 @@ 0.000000 MetaHookPost LoadFileExtended(0, base<...>/broker, <...>/broker) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, base<...>/cluster, <...>/cluster) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, base<...>/comm.bif, <...>/comm.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/communityid.bif, <...>/communityid.bif.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, base<...>/config, <...>/config) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, base<...>/conn, <...>/conn) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, base<...>/conn-ids, <...>/conn-ids.zeek) -> (-1, ) @@ -2501,6 +2505,7 @@ 0.000000 MetaHookPre LoadFile(0, ./cardinality-counter.bif.zeek, <...>/cardinality-counter.bif.zeek) 0.000000 MetaHookPre LoadFile(0, ./certificate-event-cache, <...>/certificate-event-cache.zeek) 0.000000 MetaHookPre LoadFile(0, ./comm.bif.zeek, <...>/comm.bif.zeek) +0.000000 MetaHookPre LoadFile(0, ./communityid.bif.zeek, <...>/communityid.bif.zeek) 0.000000 MetaHookPre LoadFile(0, ./const-dos-error, <...>/const-dos-error.zeek) 0.000000 MetaHookPre LoadFile(0, ./const-nt-status, <...>/const-nt-status.zeek) 0.000000 MetaHookPre LoadFile(0, ./const.bif.zeek, <...>/const.bif.zeek) @@ -2618,6 +2623,7 @@ 0.000000 MetaHookPre LoadFile(0, base<...>/broker, <...>/broker) 0.000000 MetaHookPre LoadFile(0, base<...>/cluster, <...>/cluster) 0.000000 MetaHookPre LoadFile(0, base<...>/comm.bif, <...>/comm.bif.zeek) +0.000000 MetaHookPre LoadFile(0, base<...>/communityid.bif, <...>/communityid.bif.zeek) 0.000000 MetaHookPre LoadFile(0, base<...>/config, <...>/config) 0.000000 MetaHookPre LoadFile(0, base<...>/conn, <...>/conn) 0.000000 MetaHookPre LoadFile(0, base<...>/conn-ids, <...>/conn-ids.zeek) @@ -2889,6 +2895,7 @@ 0.000000 MetaHookPre LoadFileExtended(0, ./cardinality-counter.bif.zeek, <...>/cardinality-counter.bif.zeek) 0.000000 MetaHookPre LoadFileExtended(0, ./certificate-event-cache, <...>/certificate-event-cache.zeek) 0.000000 MetaHookPre LoadFileExtended(0, ./comm.bif.zeek, <...>/comm.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./communityid.bif.zeek, <...>/communityid.bif.zeek) 0.000000 MetaHookPre LoadFileExtended(0, ./const-dos-error, <...>/const-dos-error.zeek) 0.000000 MetaHookPre LoadFileExtended(0, ./const-nt-status, <...>/const-nt-status.zeek) 0.000000 MetaHookPre LoadFileExtended(0, ./const.bif.zeek, <...>/const.bif.zeek) @@ -3006,6 +3013,7 @@ 0.000000 MetaHookPre LoadFileExtended(0, base<...>/broker, <...>/broker) 0.000000 MetaHookPre LoadFileExtended(0, base<...>/cluster, <...>/cluster) 0.000000 MetaHookPre LoadFileExtended(0, base<...>/comm.bif, <...>/comm.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/communityid.bif, <...>/communityid.bif.zeek) 0.000000 MetaHookPre LoadFileExtended(0, base<...>/config, <...>/config) 0.000000 MetaHookPre LoadFileExtended(0, base<...>/conn, <...>/conn) 0.000000 MetaHookPre LoadFileExtended(0, base<...>/conn-ids, <...>/conn-ids.zeek) @@ -4076,6 +4084,7 @@ 0.000000 | HookLoadFile ./cardinality-counter.bif.zeek <...>/cardinality-counter.bif.zeek 0.000000 | HookLoadFile ./certificate-event-cache <...>/certificate-event-cache.zeek 0.000000 | HookLoadFile ./comm.bif.zeek <...>/comm.bif.zeek +0.000000 | HookLoadFile ./communityid.bif.zeek <...>/communityid.bif.zeek 0.000000 | HookLoadFile ./const-dos-error <...>/const-dos-error.zeek 0.000000 | HookLoadFile ./const-nt-status <...>/const-nt-status.zeek 0.000000 | HookLoadFile ./const.bif.zeek <...>/const.bif.zeek @@ -4203,6 +4212,7 @@ 0.000000 | HookLoadFile base<...>/broker <...>/broker 0.000000 | HookLoadFile base<...>/cluster <...>/cluster 0.000000 | HookLoadFile base<...>/comm.bif <...>/comm.bif.zeek +0.000000 | HookLoadFile base<...>/communityid.bif <...>/communityid.bif.zeek 0.000000 | HookLoadFile base<...>/config <...>/config 0.000000 | HookLoadFile base<...>/conn <...>/conn 0.000000 | HookLoadFile base<...>/conn-ids <...>/conn-ids.zeek @@ -4464,6 +4474,7 @@ 0.000000 | HookLoadFileExtended ./cardinality-counter.bif.zeek <...>/cardinality-counter.bif.zeek 0.000000 | HookLoadFileExtended ./certificate-event-cache <...>/certificate-event-cache.zeek 0.000000 | HookLoadFileExtended ./comm.bif.zeek <...>/comm.bif.zeek +0.000000 | HookLoadFileExtended ./communityid.bif.zeek <...>/communityid.bif.zeek 0.000000 | HookLoadFileExtended ./const-dos-error <...>/const-dos-error.zeek 0.000000 | HookLoadFileExtended ./const-nt-status <...>/const-nt-status.zeek 0.000000 | HookLoadFileExtended ./const.bif.zeek <...>/const.bif.zeek @@ -4591,6 +4602,7 @@ 0.000000 | HookLoadFileExtended base<...>/broker <...>/broker 0.000000 | HookLoadFileExtended base<...>/cluster <...>/cluster 0.000000 | HookLoadFileExtended base<...>/comm.bif <...>/comm.bif.zeek +0.000000 | HookLoadFileExtended base<...>/communityid.bif <...>/communityid.bif.zeek 0.000000 | HookLoadFileExtended base<...>/config <...>/config 0.000000 | HookLoadFileExtended base<...>/conn <...>/conn 0.000000 | HookLoadFileExtended base<...>/conn-ids <...>/conn-ids.zeek diff --git a/testing/btest/Baseline/scripts.policy.frameworks.notice.community-id/notice.log.cut b/testing/btest/Baseline/scripts.policy.frameworks.notice.community-id/notice.log.cut new file mode 100644 index 0000000000..7227d38c91 --- /dev/null +++ b/testing/btest/Baseline/scripts.policy.frameworks.notice.community-id/notice.log.cut @@ -0,0 +1,3 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +- - - - - - COMMUNITY_ID_INIT Zeek initializing +141.142.228.5 59856 192.150.187.43 80 tcp 1:yvyB8h+3dnggTZW0UEITWCst97w= COMMUNITY_ID_CONN_ESTABLISHED Connection establishment diff --git a/testing/btest/Baseline/scripts.policy.protocols.conn.community-id-logging/conn.log.cut b/testing/btest/Baseline/scripts.policy.protocols.conn.community-id-logging/conn.log.cut new file mode 100644 index 0000000000..b02a020eb9 --- /dev/null +++ b/testing/btest/Baseline/scripts.policy.protocols.conn.community-id-logging/conn.log.cut @@ -0,0 +1,2 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +141.142.228.5 59856 192.150.187.43 80 tcp - 1:yvyB8h+3dnggTZW0UEITWCst97w= diff --git a/testing/btest/Traces/communityid/README b/testing/btest/Traces/communityid/README new file mode 100644 index 0000000000..e644426048 --- /dev/null +++ b/testing/btest/Traces/communityid/README @@ -0,0 +1 @@ +# Traces imported from the original zeek-community-id repository. diff --git a/testing/btest/Traces/communityid/arp.pcap b/testing/btest/Traces/communityid/arp.pcap new file mode 100644 index 0000000000..cd94d5fe1a Binary files /dev/null and b/testing/btest/Traces/communityid/arp.pcap differ diff --git a/testing/btest/Traces/communityid/icmp.pcap b/testing/btest/Traces/communityid/icmp.pcap new file mode 100644 index 0000000000..037606d1be Binary files /dev/null and b/testing/btest/Traces/communityid/icmp.pcap differ diff --git a/testing/btest/Traces/communityid/icmp6.pcap b/testing/btest/Traces/communityid/icmp6.pcap new file mode 100644 index 0000000000..99e7cde9cf Binary files /dev/null and b/testing/btest/Traces/communityid/icmp6.pcap differ diff --git a/testing/btest/Traces/communityid/ipv6.pcap b/testing/btest/Traces/communityid/ipv6.pcap new file mode 100644 index 0000000000..eeeeb1f27c Binary files /dev/null and b/testing/btest/Traces/communityid/ipv6.pcap differ diff --git a/testing/btest/Traces/communityid/sctp.pcap b/testing/btest/Traces/communityid/sctp.pcap new file mode 100644 index 0000000000..85113c8f61 Binary files /dev/null and b/testing/btest/Traces/communityid/sctp.pcap differ diff --git a/testing/btest/Traces/communityid/tcp.pcap b/testing/btest/Traces/communityid/tcp.pcap new file mode 100644 index 0000000000..92a5e9803b Binary files /dev/null and b/testing/btest/Traces/communityid/tcp.pcap differ diff --git a/testing/btest/Traces/communityid/udp.pcap b/testing/btest/Traces/communityid/udp.pcap new file mode 100644 index 0000000000..169c924e4a Binary files /dev/null and b/testing/btest/Traces/communityid/udp.pcap differ diff --git a/testing/btest/bifs/community_id/run-pcaps.zeek b/testing/btest/bifs/community_id/run-pcaps.zeek new file mode 100644 index 0000000000..cd4391bbec --- /dev/null +++ b/testing/btest/bifs/community_id/run-pcaps.zeek @@ -0,0 +1,22 @@ +# Test imported from original zeek-community-id repository. +# +# Crank through a set of pcaps and verify the Community ID inputs and +# outputs. Since each output line is triggered by a connection state +# removal in Zeek, the ordering of sets of those events can change +# across Zeek releases, and we don't care about the order (just the +# values involved), we sort the output files. + +# @TEST-EXEC: bash %INPUT + +set -ex + +for pcap in $(cd $TRACES/communityid && ls *.pcap); do + zeek -r $TRACES/communityid/$pcap test-community-id-v1.zeek | sort >$pcap.out + btest-diff $pcap.out +done + +@TEST-START-FILE test-community-id-v1.zeek +event connection_state_remove(c: connection) { + print c$id, community_id_v1(c$id); +} +@TEST-END-FILE diff --git a/testing/btest/bifs/community_id/v1.zeek b/testing/btest/bifs/community_id/v1.zeek new file mode 100644 index 0000000000..ac3847ff14 --- /dev/null +++ b/testing/btest/bifs/community_id/v1.zeek @@ -0,0 +1,29 @@ +# @TEST-EXEC: zeek -b %INPUT >out +# @TEST-EXEC: btest-diff out + +function test_it(cid: conn_id, seed: count, expected: string) + { + local actual = community_id_v1(cid, seed); + local prefix = actual == expected ? "PASS" : "FAIL"; + print fmt("%s: expected '%s', got '%s' (%s, seed=%d)", prefix, expected, actual, cid, seed); + } + +event zeek_init() + { + test_it([$orig_h=1.2.3.4, $orig_p=1122/tcp, $resp_h=5.6.7.8, $resp_p=3344/tcp], 0, "1:wCb3OG7yAFWelaUydu0D+125CLM="); + test_it([$orig_h=1.2.3.4, $orig_p=1122/udp, $resp_h=5.6.7.8, $resp_p=3344/udp], 0, "1:0Mu9InQx6z4ZiCZM/7HXi2WMhOg="); + test_it([$orig_h=1.2.3.4, $orig_p=8/icmp, $resp_h=5.6.7.8, $resp_p=0/icmp], 0, "1:crodRHL2FEsHjbv3UkRrfbs4bZ0="); + test_it([$orig_h=[fe80:0001:0203:0405:0607:0809:0A0B:0C0D], $orig_p=128/icmp, + $resp_h=[fe80:1011:1213:1415:1617:1819:1A1B:1C1D], $resp_p=129/icmp], 0, "1:0bf7hyMJUwt3fMED7z8LIfRpBeo="); + + + test_it([$orig_h=1.2.3.4, $orig_p=1122/tcp, $resp_h=5.6.7.8, $resp_p=3344/tcp], 1, "1:HhA1B+6CoLbiKPEs5nhNYN4XWfk="); + test_it([$orig_h=1.2.3.4, $orig_p=1122/udp, $resp_h=5.6.7.8, $resp_p=3344/udp], 1, "1:OShq+iKDAMVouh/4bMxB9Sz4amw="); + test_it([$orig_h=1.2.3.4, $orig_p=8/icmp, $resp_h=5.6.7.8, $resp_p=0/icmp], 1, "1:9pr4ZGTICiuZoIh90RRYE2RyXpU="); + test_it([$orig_h=[fe80:0001:0203:0405:0607:0809:0A0B:0C0D], $orig_p=128/icmp, + $resp_h=[fe80:1011:1213:1415:1617:1819:1A1B:1C1D], $resp_p=129/icmp], 1, "1:IO27GQzPuCtNnwFvjWALMHu5tJE="); + + test_it([$orig_h=1.2.3.4, $orig_p=0/unknown, $resp_h=5.6.7.8, $resp_p=0/unknown], 0, ""); + test_it([$orig_h=[fe80:0001:0203:0405:0607:0809:0A0B:0C0D], $orig_p=0/unknown, + $resp_h=[fe80:1011:1213:1415:1617:1819:1A1B:1C1D], $resp_p=0/unknown], 1, ""); + } diff --git a/testing/btest/scripts/policy/frameworks/notice/community-id.zeek b/testing/btest/scripts/policy/frameworks/notice/community-id.zeek new file mode 100644 index 0000000000..8383017c63 --- /dev/null +++ b/testing/btest/scripts/policy/frameworks/notice/community-id.zeek @@ -0,0 +1,31 @@ +# This test verifies Community ID presence in the notice log, when +# that part of the package is loaded. The test creates one notice +# without connection state and one with, and verifies that the latter +# includes the Community ID value for it. + +# @TEST-EXEC: zeek -b -r $TRACES/http/get.trace %INPUT +# @TEST-EXEC: zeek-cut id.orig_h id.orig_p id.resp_h id.resp_p proto community_id note msg < notice.log > notice.log.cut +# @TEST-EXEC: btest-diff notice.log.cut + +@load protocols/conn/community-id-logging +@load frameworks/notice/community-id + +redef enum Notice::Type += { + COMMUNITY_ID_INIT, + COMMUNITY_ID_CONN_ESTABLISHED, +}; + +event zeek_init() + { + # A notice without connection context + NOTICE([$note=COMMUNITY_ID_INIT, + $msg="Zeek initializing"]); + } + +event connection_established(c: connection) + { + # A notice with connection context + NOTICE([$note=COMMUNITY_ID_CONN_ESTABLISHED, + $msg="Connection establishment", + $conn=c]); + } diff --git a/testing/btest/scripts/policy/protocols/conn/community-id-logging.zeek b/testing/btest/scripts/policy/protocols/conn/community-id-logging.zeek new file mode 100644 index 0000000000..4dd18920d5 --- /dev/null +++ b/testing/btest/scripts/policy/protocols/conn/community-id-logging.zeek @@ -0,0 +1,5 @@ +# @TEST-EXEC: zeek -b -r $TRACES/http/get.trace %INPUT +# @TEST-EXEC: zeek-cut id.orig_h id.orig_p id.resp_h id.resp_p proto service community_id < conn.log > conn.log.cut +# @TEST-EXEC: btest-diff conn.log.cut + +@load protocols/conn/community-id-logging diff --git a/testing/external/commit-hash.zeek-testing b/testing/external/commit-hash.zeek-testing index 3c219f5e9d..275024bf19 100644 --- a/testing/external/commit-hash.zeek-testing +++ b/testing/external/commit-hash.zeek-testing @@ -1 +1 @@ -18a9ac00f5b7617e8660d4ba680a25291d2b44f7 \ No newline at end of file +63952e0fc5c88ff44752c586abe721ccb324003e diff --git a/testing/external/commit-hash.zeek-testing-private b/testing/external/commit-hash.zeek-testing-private index 72bdde067f..8279cfae9b 100644 --- a/testing/external/commit-hash.zeek-testing-private +++ b/testing/external/commit-hash.zeek-testing-private @@ -1 +1 @@ -98e8aee2f09bff7e8138290242274b5ffd834e58 +e16f299e2ddad6a3495113ff7b26120b6312b220