Telemetry framework: move BIFs to the primary-bif stage

This moves the Telemetry framework's BIF-defined functionalit from the
secondary-BIFs stage to the primary one. That is, this functionality is now
available from the end of init-bare.zeek, not only after the end of
init-frameworks-and-bifs.zeek.

This allows us to use script-layer telemetry in our Zeek's own code that get
pulled in during init-frameworks-and-bifs.

This change splits up the BIF features into functions, constants, and types,
because that's the granularity most workable in Func.cc and NetVar. It also now
defines the Telemetry::MetricsType enum once, not redundantly in BIFs and script
layer.

Due to subtle load ordering issues between the telemetry and cluster frameworks
this pushes the redef stage of Telemetry::metrics_port and address into
base/frameworks/telemetry/options.zeek, which is loaded sufficiently late in
init-frameworks-and-bifs.zeek to sidestep those issues. (When not doing this,
the effect is that the redef in telemetry/main.zeek doesn't yet find the
cluster-provided values, and Zeek does not end up listening on these ports.)

The need to add basic Zeek headers in script_opt/ZAM/ZBody.cc as a side-effect
of this is curious, but looks harmless.

Also includes baseline updates for the usual btests and adds a few doc strings.

(cherry picked from commit 71f7e89974)
This commit is contained in:
Christian Kreibich 2024-10-17 15:25:24 -07:00
parent 5503688758
commit 2ad80f8fb2
20 changed files with 99 additions and 63 deletions

View file

@ -24,6 +24,8 @@ scripts/base/init-bare.zeek
build/scripts/base/bif/plugins/Zeek_SNMP.types.bif.zeek
build/scripts/base/bif/plugins/Zeek_KRB.types.bif.zeek
build/scripts/base/bif/event.bif.zeek
build/scripts/base/bif/telemetry_functions.bif.zeek
build/scripts/base/bif/telemetry_types.bif.zeek
scripts/base/packet-protocols/__load__.zeek
scripts/base/packet-protocols/main.zeek
scripts/base/frameworks/analyzer/main.zeek
@ -146,8 +148,7 @@ scripts/base/init-frameworks-and-bifs.zeek
scripts/base/frameworks/files/magic/__load__.zeek
scripts/base/frameworks/telemetry/options.zeek
build/scripts/base/bif/__load__.zeek
build/scripts/base/bif/consts.bif.zeek
build/scripts/base/bif/telemetry.bif.zeek
build/scripts/base/bif/telemetry_consts.bif.zeek
build/scripts/base/bif/zeekygen.bif.zeek
build/scripts/base/bif/pcap.bif.zeek
build/scripts/base/bif/bloom-filter.bif.zeek

View file

@ -24,6 +24,8 @@ scripts/base/init-bare.zeek
build/scripts/base/bif/plugins/Zeek_SNMP.types.bif.zeek
build/scripts/base/bif/plugins/Zeek_KRB.types.bif.zeek
build/scripts/base/bif/event.bif.zeek
build/scripts/base/bif/telemetry_functions.bif.zeek
build/scripts/base/bif/telemetry_types.bif.zeek
scripts/base/packet-protocols/__load__.zeek
scripts/base/packet-protocols/main.zeek
scripts/base/frameworks/analyzer/main.zeek
@ -146,8 +148,7 @@ scripts/base/init-frameworks-and-bifs.zeek
scripts/base/frameworks/files/magic/__load__.zeek
scripts/base/frameworks/telemetry/options.zeek
build/scripts/base/bif/__load__.zeek
build/scripts/base/bif/consts.bif.zeek
build/scripts/base/bif/telemetry.bif.zeek
build/scripts/base/bif/telemetry_consts.bif.zeek
build/scripts/base/bif/zeekygen.bif.zeek
build/scripts/base/bif/pcap.bif.zeek
build/scripts/base/bif/bloom-filter.bif.zeek

View file

@ -466,7 +466,6 @@
0.000000 MetaHookPost LoadFile(0, ./comm.bif.zeek, <...>/comm.bif.zeek) -> -1
0.000000 MetaHookPost LoadFile(0, ./communityid.bif.zeek, <...>/communityid.bif.zeek) -> -1
0.000000 MetaHookPost LoadFile(0, ./const.bif.zeek, <...>/const.bif.zeek) -> -1
0.000000 MetaHookPost LoadFile(0, ./consts.bif.zeek, <...>/consts.bif.zeek) -> -1
0.000000 MetaHookPost LoadFile(0, ./contents, <...>/contents.zeek) -> -1
0.000000 MetaHookPost LoadFile(0, ./control, <...>/control.zeek) -> -1
0.000000 MetaHookPost LoadFile(0, ./data.bif.zeek, <...>/data.bif.zeek) -> -1
@ -504,7 +503,9 @@
0.000000 MetaHookPost LoadFile(0, ./store.bif.zeek, <...>/store.bif.zeek) -> -1
0.000000 MetaHookPost LoadFile(0, ./strings.bif.zeek, <...>/strings.bif.zeek) -> -1
0.000000 MetaHookPost LoadFile(0, ./supervisor.bif.zeek, <...>/supervisor.bif.zeek) -> -1
0.000000 MetaHookPost LoadFile(0, ./telemetry.bif.zeek, <...>/telemetry.bif.zeek) -> -1
0.000000 MetaHookPost LoadFile(0, ./telemetry_consts.bif.zeek, <...>/telemetry_consts.bif.zeek) -> -1
0.000000 MetaHookPost LoadFile(0, ./telemetry_functions.bif.zeek, <...>/telemetry_functions.bif.zeek) -> -1
0.000000 MetaHookPost LoadFile(0, ./telemetry_types.bif.zeek, <...>/telemetry_types.bif.zeek) -> -1
0.000000 MetaHookPost LoadFile(0, ./thresholds, <...>/thresholds.zeek) -> -1
0.000000 MetaHookPost LoadFile(0, ./top-k.bif.zeek, <...>/top-k.bif.zeek) -> -1
0.000000 MetaHookPost LoadFile(0, ./types, <...>/types.zeek) -> -1
@ -601,6 +602,8 @@
0.000000 MetaHookPost LoadFile(0, base<...>/supervisor, <...>/supervisor) -> -1
0.000000 MetaHookPost LoadFile(0, base<...>/supervisor.bif, <...>/supervisor.bif.zeek) -> -1
0.000000 MetaHookPost LoadFile(0, base<...>/tcp, <...>/tcp) -> -1
0.000000 MetaHookPost LoadFile(0, base<...>/telemetry_functions.bif, <...>/telemetry_functions.bif.zeek) -> -1
0.000000 MetaHookPost LoadFile(0, base<...>/telemetry_types.bif, <...>/telemetry_types.bif.zeek) -> -1
0.000000 MetaHookPost LoadFile(0, base<...>/teredo, <...>/teredo) -> -1
0.000000 MetaHookPost LoadFile(0, base<...>/tunnels, <...>/tunnels) -> -1
0.000000 MetaHookPost LoadFile(0, base<...>/types.bif, <...>/types.bif.zeek) -> -1
@ -762,7 +765,6 @@
0.000000 MetaHookPost LoadFileExtended(0, ./comm.bif.zeek, <...>/comm.bif.zeek) -> (-1, <no content>)
0.000000 MetaHookPost LoadFileExtended(0, ./communityid.bif.zeek, <...>/communityid.bif.zeek) -> (-1, <no content>)
0.000000 MetaHookPost LoadFileExtended(0, ./const.bif.zeek, <...>/const.bif.zeek) -> (-1, <no content>)
0.000000 MetaHookPost LoadFileExtended(0, ./consts.bif.zeek, <...>/consts.bif.zeek) -> (-1, <no content>)
0.000000 MetaHookPost LoadFileExtended(0, ./contents, <...>/contents.zeek) -> (-1, <no content>)
0.000000 MetaHookPost LoadFileExtended(0, ./control, <...>/control.zeek) -> (-1, <no content>)
0.000000 MetaHookPost LoadFileExtended(0, ./data.bif.zeek, <...>/data.bif.zeek) -> (-1, <no content>)
@ -800,7 +802,9 @@
0.000000 MetaHookPost LoadFileExtended(0, ./store.bif.zeek, <...>/store.bif.zeek) -> (-1, <no content>)
0.000000 MetaHookPost LoadFileExtended(0, ./strings.bif.zeek, <...>/strings.bif.zeek) -> (-1, <no content>)
0.000000 MetaHookPost LoadFileExtended(0, ./supervisor.bif.zeek, <...>/supervisor.bif.zeek) -> (-1, <no content>)
0.000000 MetaHookPost LoadFileExtended(0, ./telemetry.bif.zeek, <...>/telemetry.bif.zeek) -> (-1, <no content>)
0.000000 MetaHookPost LoadFileExtended(0, ./telemetry_consts.bif.zeek, <...>/telemetry_consts.bif.zeek) -> (-1, <no content>)
0.000000 MetaHookPost LoadFileExtended(0, ./telemetry_functions.bif.zeek, <...>/telemetry_functions.bif.zeek) -> (-1, <no content>)
0.000000 MetaHookPost LoadFileExtended(0, ./telemetry_types.bif.zeek, <...>/telemetry_types.bif.zeek) -> (-1, <no content>)
0.000000 MetaHookPost LoadFileExtended(0, ./thresholds, <...>/thresholds.zeek) -> (-1, <no content>)
0.000000 MetaHookPost LoadFileExtended(0, ./top-k.bif.zeek, <...>/top-k.bif.zeek) -> (-1, <no content>)
0.000000 MetaHookPost LoadFileExtended(0, ./types, <...>/types.zeek) -> (-1, <no content>)
@ -897,6 +901,8 @@
0.000000 MetaHookPost LoadFileExtended(0, base<...>/supervisor, <...>/supervisor) -> (-1, <no content>)
0.000000 MetaHookPost LoadFileExtended(0, base<...>/supervisor.bif, <...>/supervisor.bif.zeek) -> (-1, <no content>)
0.000000 MetaHookPost LoadFileExtended(0, base<...>/tcp, <...>/tcp) -> (-1, <no content>)
0.000000 MetaHookPost LoadFileExtended(0, base<...>/telemetry_functions.bif, <...>/telemetry_functions.bif.zeek) -> (-1, <no content>)
0.000000 MetaHookPost LoadFileExtended(0, base<...>/telemetry_types.bif, <...>/telemetry_types.bif.zeek) -> (-1, <no content>)
0.000000 MetaHookPost LoadFileExtended(0, base<...>/teredo, <...>/teredo) -> (-1, <no content>)
0.000000 MetaHookPost LoadFileExtended(0, base<...>/tunnels, <...>/tunnels) -> (-1, <no content>)
0.000000 MetaHookPost LoadFileExtended(0, base<...>/types.bif, <...>/types.bif.zeek) -> (-1, <no content>)
@ -1391,7 +1397,6 @@
0.000000 MetaHookPre LoadFile(0, ./comm.bif.zeek, <...>/comm.bif.zeek)
0.000000 MetaHookPre LoadFile(0, ./communityid.bif.zeek, <...>/communityid.bif.zeek)
0.000000 MetaHookPre LoadFile(0, ./const.bif.zeek, <...>/const.bif.zeek)
0.000000 MetaHookPre LoadFile(0, ./consts.bif.zeek, <...>/consts.bif.zeek)
0.000000 MetaHookPre LoadFile(0, ./contents, <...>/contents.zeek)
0.000000 MetaHookPre LoadFile(0, ./control, <...>/control.zeek)
0.000000 MetaHookPre LoadFile(0, ./data.bif.zeek, <...>/data.bif.zeek)
@ -1429,7 +1434,9 @@
0.000000 MetaHookPre LoadFile(0, ./store.bif.zeek, <...>/store.bif.zeek)
0.000000 MetaHookPre LoadFile(0, ./strings.bif.zeek, <...>/strings.bif.zeek)
0.000000 MetaHookPre LoadFile(0, ./supervisor.bif.zeek, <...>/supervisor.bif.zeek)
0.000000 MetaHookPre LoadFile(0, ./telemetry.bif.zeek, <...>/telemetry.bif.zeek)
0.000000 MetaHookPre LoadFile(0, ./telemetry_consts.bif.zeek, <...>/telemetry_consts.bif.zeek)
0.000000 MetaHookPre LoadFile(0, ./telemetry_functions.bif.zeek, <...>/telemetry_functions.bif.zeek)
0.000000 MetaHookPre LoadFile(0, ./telemetry_types.bif.zeek, <...>/telemetry_types.bif.zeek)
0.000000 MetaHookPre LoadFile(0, ./thresholds, <...>/thresholds.zeek)
0.000000 MetaHookPre LoadFile(0, ./top-k.bif.zeek, <...>/top-k.bif.zeek)
0.000000 MetaHookPre LoadFile(0, ./types, <...>/types.zeek)
@ -1526,6 +1533,8 @@
0.000000 MetaHookPre LoadFile(0, base<...>/supervisor, <...>/supervisor)
0.000000 MetaHookPre LoadFile(0, base<...>/supervisor.bif, <...>/supervisor.bif.zeek)
0.000000 MetaHookPre LoadFile(0, base<...>/tcp, <...>/tcp)
0.000000 MetaHookPre LoadFile(0, base<...>/telemetry_functions.bif, <...>/telemetry_functions.bif.zeek)
0.000000 MetaHookPre LoadFile(0, base<...>/telemetry_types.bif, <...>/telemetry_types.bif.zeek)
0.000000 MetaHookPre LoadFile(0, base<...>/teredo, <...>/teredo)
0.000000 MetaHookPre LoadFile(0, base<...>/tunnels, <...>/tunnels)
0.000000 MetaHookPre LoadFile(0, base<...>/types.bif, <...>/types.bif.zeek)
@ -1687,7 +1696,6 @@
0.000000 MetaHookPre LoadFileExtended(0, ./comm.bif.zeek, <...>/comm.bif.zeek)
0.000000 MetaHookPre LoadFileExtended(0, ./communityid.bif.zeek, <...>/communityid.bif.zeek)
0.000000 MetaHookPre LoadFileExtended(0, ./const.bif.zeek, <...>/const.bif.zeek)
0.000000 MetaHookPre LoadFileExtended(0, ./consts.bif.zeek, <...>/consts.bif.zeek)
0.000000 MetaHookPre LoadFileExtended(0, ./contents, <...>/contents.zeek)
0.000000 MetaHookPre LoadFileExtended(0, ./control, <...>/control.zeek)
0.000000 MetaHookPre LoadFileExtended(0, ./data.bif.zeek, <...>/data.bif.zeek)
@ -1725,7 +1733,9 @@
0.000000 MetaHookPre LoadFileExtended(0, ./store.bif.zeek, <...>/store.bif.zeek)
0.000000 MetaHookPre LoadFileExtended(0, ./strings.bif.zeek, <...>/strings.bif.zeek)
0.000000 MetaHookPre LoadFileExtended(0, ./supervisor.bif.zeek, <...>/supervisor.bif.zeek)
0.000000 MetaHookPre LoadFileExtended(0, ./telemetry.bif.zeek, <...>/telemetry.bif.zeek)
0.000000 MetaHookPre LoadFileExtended(0, ./telemetry_consts.bif.zeek, <...>/telemetry_consts.bif.zeek)
0.000000 MetaHookPre LoadFileExtended(0, ./telemetry_functions.bif.zeek, <...>/telemetry_functions.bif.zeek)
0.000000 MetaHookPre LoadFileExtended(0, ./telemetry_types.bif.zeek, <...>/telemetry_types.bif.zeek)
0.000000 MetaHookPre LoadFileExtended(0, ./thresholds, <...>/thresholds.zeek)
0.000000 MetaHookPre LoadFileExtended(0, ./top-k.bif.zeek, <...>/top-k.bif.zeek)
0.000000 MetaHookPre LoadFileExtended(0, ./types, <...>/types.zeek)
@ -1822,6 +1832,8 @@
0.000000 MetaHookPre LoadFileExtended(0, base<...>/supervisor, <...>/supervisor)
0.000000 MetaHookPre LoadFileExtended(0, base<...>/supervisor.bif, <...>/supervisor.bif.zeek)
0.000000 MetaHookPre LoadFileExtended(0, base<...>/tcp, <...>/tcp)
0.000000 MetaHookPre LoadFileExtended(0, base<...>/telemetry_functions.bif, <...>/telemetry_functions.bif.zeek)
0.000000 MetaHookPre LoadFileExtended(0, base<...>/telemetry_types.bif, <...>/telemetry_types.bif.zeek)
0.000000 MetaHookPre LoadFileExtended(0, base<...>/teredo, <...>/teredo)
0.000000 MetaHookPre LoadFileExtended(0, base<...>/tunnels, <...>/tunnels)
0.000000 MetaHookPre LoadFileExtended(0, base<...>/types.bif, <...>/types.bif.zeek)
@ -2317,7 +2329,6 @@
0.000000 | HookLoadFile ./comm.bif.zeek <...>/comm.bif.zeek
0.000000 | HookLoadFile ./communityid.bif.zeek <...>/communityid.bif.zeek
0.000000 | HookLoadFile ./const.bif.zeek <...>/const.bif.zeek
0.000000 | HookLoadFile ./consts.bif.zeek <...>/consts.bif.zeek
0.000000 | HookLoadFile ./contents <...>/contents.zeek
0.000000 | HookLoadFile ./control <...>/control.zeek
0.000000 | HookLoadFile ./data.bif.zeek <...>/data.bif.zeek
@ -2364,7 +2375,9 @@
0.000000 | HookLoadFile ./store.bif.zeek <...>/store.bif.zeek
0.000000 | HookLoadFile ./strings.bif.zeek <...>/strings.bif.zeek
0.000000 | HookLoadFile ./supervisor.bif.zeek <...>/supervisor.bif.zeek
0.000000 | HookLoadFile ./telemetry.bif.zeek <...>/telemetry.bif.zeek
0.000000 | HookLoadFile ./telemetry_consts.bif.zeek <...>/telemetry_consts.bif.zeek
0.000000 | HookLoadFile ./telemetry_functions.bif.zeek <...>/telemetry_functions.bif.zeek
0.000000 | HookLoadFile ./telemetry_types.bif.zeek <...>/telemetry_types.bif.zeek
0.000000 | HookLoadFile ./thresholds <...>/thresholds.zeek
0.000000 | HookLoadFile ./top-k.bif.zeek <...>/top-k.bif.zeek
0.000000 | HookLoadFile ./types <...>/types.zeek
@ -2462,6 +2475,8 @@
0.000000 | HookLoadFile base<...>/supervisor <...>/supervisor
0.000000 | HookLoadFile base<...>/supervisor.bif <...>/supervisor.bif.zeek
0.000000 | HookLoadFile base<...>/tcp <...>/tcp
0.000000 | HookLoadFile base<...>/telemetry_functions.bif <...>/telemetry_functions.bif.zeek
0.000000 | HookLoadFile base<...>/telemetry_types.bif <...>/telemetry_types.bif.zeek
0.000000 | HookLoadFile base<...>/teredo <...>/teredo
0.000000 | HookLoadFile base<...>/tunnels <...>/tunnels
0.000000 | HookLoadFile base<...>/types.bif <...>/types.bif.zeek
@ -2613,7 +2628,6 @@
0.000000 | HookLoadFileExtended ./comm.bif.zeek <...>/comm.bif.zeek
0.000000 | HookLoadFileExtended ./communityid.bif.zeek <...>/communityid.bif.zeek
0.000000 | HookLoadFileExtended ./const.bif.zeek <...>/const.bif.zeek
0.000000 | HookLoadFileExtended ./consts.bif.zeek <...>/consts.bif.zeek
0.000000 | HookLoadFileExtended ./contents <...>/contents.zeek
0.000000 | HookLoadFileExtended ./control <...>/control.zeek
0.000000 | HookLoadFileExtended ./data.bif.zeek <...>/data.bif.zeek
@ -2660,7 +2674,9 @@
0.000000 | HookLoadFileExtended ./store.bif.zeek <...>/store.bif.zeek
0.000000 | HookLoadFileExtended ./strings.bif.zeek <...>/strings.bif.zeek
0.000000 | HookLoadFileExtended ./supervisor.bif.zeek <...>/supervisor.bif.zeek
0.000000 | HookLoadFileExtended ./telemetry.bif.zeek <...>/telemetry.bif.zeek
0.000000 | HookLoadFileExtended ./telemetry_consts.bif.zeek <...>/telemetry_consts.bif.zeek
0.000000 | HookLoadFileExtended ./telemetry_functions.bif.zeek <...>/telemetry_functions.bif.zeek
0.000000 | HookLoadFileExtended ./telemetry_types.bif.zeek <...>/telemetry_types.bif.zeek
0.000000 | HookLoadFileExtended ./thresholds <...>/thresholds.zeek
0.000000 | HookLoadFileExtended ./top-k.bif.zeek <...>/top-k.bif.zeek
0.000000 | HookLoadFileExtended ./types <...>/types.zeek
@ -2758,6 +2774,8 @@
0.000000 | HookLoadFileExtended base<...>/supervisor <...>/supervisor
0.000000 | HookLoadFileExtended base<...>/supervisor.bif <...>/supervisor.bif.zeek
0.000000 | HookLoadFileExtended base<...>/tcp <...>/tcp
0.000000 | HookLoadFileExtended base<...>/telemetry_functions.bif <...>/telemetry_functions.bif.zeek
0.000000 | HookLoadFileExtended base<...>/telemetry_types.bif <...>/telemetry_types.bif.zeek
0.000000 | HookLoadFileExtended base<...>/teredo <...>/teredo
0.000000 | HookLoadFileExtended base<...>/tunnels <...>/tunnels
0.000000 | HookLoadFileExtended base<...>/types.bif <...>/types.bif.zeek