Add community_id_v1() based on corelight/zeek-community-id

"Community ID" has become an established flow hash for connection correlation
across different monitoring and storage systems. Other NSMs have had native
and built-in support for Community ID since late 2018. And even though the
roots of "Community ID" are very close to Zeek, Zeek itself has never provided
out-of-the-box support and instead required users to install an external plugin.

While we try to make that installation as easy as possible, an external plugin
always sets the bar higher for an initial setup and can be intimidating.
It also requires a rebuild operation of the plugin during upgrades. Nothing
overly complicated, but somewhat unnecessary for such popular functionality.

This isn't a 1:1 import. The options are parameters and the "verbose"
functionality  has been removed. Further, instead of a `connection`
record, the new bif works with `conn_id`, allowing computation of the
hash with little effort on the command line:

    $ zeek -e 'print community_id_v1([$orig_h=1.2.3.4, $orig_p=1024/tcp, $resp_h=5.6.7.8, $resp_p=80/tcp])'
    1:RcCrCS5fwYUeIzgDDx64EN3+okU

Reference: https://github.com/corelight/zeek-community-id/
This commit is contained in:
Christian Kreibich 2023-04-21 13:43:08 +02:00 committed by Arne Welzel
parent 379624404c
commit 99de7b7526
25 changed files with 237 additions and 0 deletions

View file

@ -933,6 +933,7 @@
0.000000 MetaHookPost LoadFile(0, ./cardinality-counter.bif.zeek, <...>/cardinality-counter.bif.zeek) -> -1
0.000000 MetaHookPost LoadFile(0, ./certificate-event-cache, <...>/certificate-event-cache.zeek) -> -1
0.000000 MetaHookPost LoadFile(0, ./comm.bif.zeek, <...>/comm.bif.zeek) -> -1
0.000000 MetaHookPost LoadFile(0, ./communityid.bif.zeek, <...>/communityid.bif.zeek) -> -1
0.000000 MetaHookPost LoadFile(0, ./const-dos-error, <...>/const-dos-error.zeek) -> -1
0.000000 MetaHookPost LoadFile(0, ./const-nt-status, <...>/const-nt-status.zeek) -> -1
0.000000 MetaHookPost LoadFile(0, ./const.bif.zeek, <...>/const.bif.zeek) -> -1
@ -1050,6 +1051,7 @@
0.000000 MetaHookPost LoadFile(0, base<...>/broker, <...>/broker) -> -1
0.000000 MetaHookPost LoadFile(0, base<...>/cluster, <...>/cluster) -> -1
0.000000 MetaHookPost LoadFile(0, base<...>/comm.bif, <...>/comm.bif.zeek) -> -1
0.000000 MetaHookPost LoadFile(0, base<...>/communityid.bif, <...>/communityid.bif.zeek) -> -1
0.000000 MetaHookPost LoadFile(0, base<...>/config, <...>/config) -> -1
0.000000 MetaHookPost LoadFile(0, base<...>/conn, <...>/conn) -> -1
0.000000 MetaHookPost LoadFile(0, base<...>/conn-ids, <...>/conn-ids.zeek) -> -1
@ -1321,6 +1323,7 @@
0.000000 MetaHookPost LoadFileExtended(0, ./cardinality-counter.bif.zeek, <...>/cardinality-counter.bif.zeek) -> (-1, <no content>)
0.000000 MetaHookPost LoadFileExtended(0, ./certificate-event-cache, <...>/certificate-event-cache.zeek) -> (-1, <no content>)
0.000000 MetaHookPost LoadFileExtended(0, ./comm.bif.zeek, <...>/comm.bif.zeek) -> (-1, <no content>)
0.000000 MetaHookPost LoadFileExtended(0, ./communityid.bif.zeek, <...>/communityid.bif.zeek) -> (-1, <no content>)
0.000000 MetaHookPost LoadFileExtended(0, ./const-dos-error, <...>/const-dos-error.zeek) -> (-1, <no content>)
0.000000 MetaHookPost LoadFileExtended(0, ./const-nt-status, <...>/const-nt-status.zeek) -> (-1, <no content>)
0.000000 MetaHookPost LoadFileExtended(0, ./const.bif.zeek, <...>/const.bif.zeek) -> (-1, <no content>)
@ -1438,6 +1441,7 @@
0.000000 MetaHookPost LoadFileExtended(0, base<...>/broker, <...>/broker) -> (-1, <no content>)
0.000000 MetaHookPost LoadFileExtended(0, base<...>/cluster, <...>/cluster) -> (-1, <no content>)
0.000000 MetaHookPost LoadFileExtended(0, base<...>/comm.bif, <...>/comm.bif.zeek) -> (-1, <no content>)
0.000000 MetaHookPost LoadFileExtended(0, base<...>/communityid.bif, <...>/communityid.bif.zeek) -> (-1, <no content>)
0.000000 MetaHookPost LoadFileExtended(0, base<...>/config, <...>/config) -> (-1, <no content>)
0.000000 MetaHookPost LoadFileExtended(0, base<...>/conn, <...>/conn) -> (-1, <no content>)
0.000000 MetaHookPost LoadFileExtended(0, base<...>/conn-ids, <...>/conn-ids.zeek) -> (-1, <no content>)
@ -2513,6 +2517,7 @@
0.000000 MetaHookPre LoadFile(0, ./cardinality-counter.bif.zeek, <...>/cardinality-counter.bif.zeek)
0.000000 MetaHookPre LoadFile(0, ./certificate-event-cache, <...>/certificate-event-cache.zeek)
0.000000 MetaHookPre LoadFile(0, ./comm.bif.zeek, <...>/comm.bif.zeek)
0.000000 MetaHookPre LoadFile(0, ./communityid.bif.zeek, <...>/communityid.bif.zeek)
0.000000 MetaHookPre LoadFile(0, ./const-dos-error, <...>/const-dos-error.zeek)
0.000000 MetaHookPre LoadFile(0, ./const-nt-status, <...>/const-nt-status.zeek)
0.000000 MetaHookPre LoadFile(0, ./const.bif.zeek, <...>/const.bif.zeek)
@ -2630,6 +2635,7 @@
0.000000 MetaHookPre LoadFile(0, base<...>/broker, <...>/broker)
0.000000 MetaHookPre LoadFile(0, base<...>/cluster, <...>/cluster)
0.000000 MetaHookPre LoadFile(0, base<...>/comm.bif, <...>/comm.bif.zeek)
0.000000 MetaHookPre LoadFile(0, base<...>/communityid.bif, <...>/communityid.bif.zeek)
0.000000 MetaHookPre LoadFile(0, base<...>/config, <...>/config)
0.000000 MetaHookPre LoadFile(0, base<...>/conn, <...>/conn)
0.000000 MetaHookPre LoadFile(0, base<...>/conn-ids, <...>/conn-ids.zeek)
@ -2901,6 +2907,7 @@
0.000000 MetaHookPre LoadFileExtended(0, ./cardinality-counter.bif.zeek, <...>/cardinality-counter.bif.zeek)
0.000000 MetaHookPre LoadFileExtended(0, ./certificate-event-cache, <...>/certificate-event-cache.zeek)
0.000000 MetaHookPre LoadFileExtended(0, ./comm.bif.zeek, <...>/comm.bif.zeek)
0.000000 MetaHookPre LoadFileExtended(0, ./communityid.bif.zeek, <...>/communityid.bif.zeek)
0.000000 MetaHookPre LoadFileExtended(0, ./const-dos-error, <...>/const-dos-error.zeek)
0.000000 MetaHookPre LoadFileExtended(0, ./const-nt-status, <...>/const-nt-status.zeek)
0.000000 MetaHookPre LoadFileExtended(0, ./const.bif.zeek, <...>/const.bif.zeek)
@ -3018,6 +3025,7 @@
0.000000 MetaHookPre LoadFileExtended(0, base<...>/broker, <...>/broker)
0.000000 MetaHookPre LoadFileExtended(0, base<...>/cluster, <...>/cluster)
0.000000 MetaHookPre LoadFileExtended(0, base<...>/comm.bif, <...>/comm.bif.zeek)
0.000000 MetaHookPre LoadFileExtended(0, base<...>/communityid.bif, <...>/communityid.bif.zeek)
0.000000 MetaHookPre LoadFileExtended(0, base<...>/config, <...>/config)
0.000000 MetaHookPre LoadFileExtended(0, base<...>/conn, <...>/conn)
0.000000 MetaHookPre LoadFileExtended(0, base<...>/conn-ids, <...>/conn-ids.zeek)
@ -4094,6 +4102,7 @@
0.000000 | HookLoadFile ./cardinality-counter.bif.zeek <...>/cardinality-counter.bif.zeek
0.000000 | HookLoadFile ./certificate-event-cache <...>/certificate-event-cache.zeek
0.000000 | HookLoadFile ./comm.bif.zeek <...>/comm.bif.zeek
0.000000 | HookLoadFile ./communityid.bif.zeek <...>/communityid.bif.zeek
0.000000 | HookLoadFile ./const-dos-error <...>/const-dos-error.zeek
0.000000 | HookLoadFile ./const-nt-status <...>/const-nt-status.zeek
0.000000 | HookLoadFile ./const.bif.zeek <...>/const.bif.zeek
@ -4221,6 +4230,7 @@
0.000000 | HookLoadFile base<...>/broker <...>/broker
0.000000 | HookLoadFile base<...>/cluster <...>/cluster
0.000000 | HookLoadFile base<...>/comm.bif <...>/comm.bif.zeek
0.000000 | HookLoadFile base<...>/communityid.bif <...>/communityid.bif.zeek
0.000000 | HookLoadFile base<...>/config <...>/config
0.000000 | HookLoadFile base<...>/conn <...>/conn
0.000000 | HookLoadFile base<...>/conn-ids <...>/conn-ids.zeek
@ -4482,6 +4492,7 @@
0.000000 | HookLoadFileExtended ./cardinality-counter.bif.zeek <...>/cardinality-counter.bif.zeek
0.000000 | HookLoadFileExtended ./certificate-event-cache <...>/certificate-event-cache.zeek
0.000000 | HookLoadFileExtended ./comm.bif.zeek <...>/comm.bif.zeek
0.000000 | HookLoadFileExtended ./communityid.bif.zeek <...>/communityid.bif.zeek
0.000000 | HookLoadFileExtended ./const-dos-error <...>/const-dos-error.zeek
0.000000 | HookLoadFileExtended ./const-nt-status <...>/const-nt-status.zeek
0.000000 | HookLoadFileExtended ./const.bif.zeek <...>/const.bif.zeek
@ -4609,6 +4620,7 @@
0.000000 | HookLoadFileExtended base<...>/broker <...>/broker
0.000000 | HookLoadFileExtended base<...>/cluster <...>/cluster
0.000000 | HookLoadFileExtended base<...>/comm.bif <...>/comm.bif.zeek
0.000000 | HookLoadFileExtended base<...>/communityid.bif <...>/communityid.bif.zeek
0.000000 | HookLoadFileExtended base<...>/config <...>/config
0.000000 | HookLoadFileExtended base<...>/conn <...>/conn
0.000000 | HookLoadFileExtended base<...>/conn-ids <...>/conn-ids.zeek