Merge remote-tracking branch 'origin/master' into topic/johanna/dpd-changes

This also includes some test baseline updates, due to recent QUIC
changes.

* origin/master: (39 commits)
  Update doc submodule [nomail] [skip ci]
  Bump cluster testsuite to pull in resilience to agent connection timing [skip ci]
  IPv6 support for detect-external-names and testcase
  Add  `skip_resp_host_port_pairs` option.
  util/init_random_seed: write_file implies deterministic
  external/subdir-btest.cfg: Set OPENSSL_ENABLE_SHA1_SIGNATURES=1
  btest/x509_verify: Drop OpenSSL 1.0 hack
  testing/btest: Use OPENSSL_ENABLE_SHA1_SIGNATURES
  Add ZAM baseline for new scripts.base.protocols.quic.analyzer-confirmations btest
  QUIC/decrypt_crypto: Rename all_data to data
  QUIC: Confirm before forwarding data to SSL
  QUIC: Parse all QUIC packets in a UDP datagram
  QUIC: Only slurp till packet end, not till &eod
  Remove unused SupervisedNode::InitCluster declaration
  Update doc submodule [nomail] [skip ci]
  Bump cluster testsuite to pull in updated Prometheus tests
  Make enc_part value from kerberos response available to scripts
  Management framework: move up addition of agent IPs into deployable cluster configs
  Support multiple instances per host addr in auto metrics generation
  When auto-generating metrics ports for worker nodes, get them more uniform across instances.
  ...
This commit is contained in:
Johanna Amann 2025-02-05 09:30:48 +00:00
commit 6324445d62
96 changed files with 1882 additions and 1265 deletions

View file

@ -83,12 +83,12 @@ export {
const log_batch_interval = 1sec &redef;
## Max number of threads to use for Broker/CAF functionality. The
## ZEEK_BROKER_MAX_THREADS environment variable overrides this setting.
## ``ZEEK_BROKER_MAX_THREADS`` environment variable overrides this setting.
const max_threads = 1 &redef;
## Max number of items we buffer at most per peer. What action to take when
## the buffer reaches its maximum size is determined by
## `peer_overflow_policy`.
## :zeek:see:`Broker::peer_overflow_policy`.
const peer_buffer_size = 2048 &redef;
## Configures how Broker responds to peers that cannot keep up with the
@ -98,10 +98,10 @@ export {
## - drop_oldest: removed the olsted message from the buffer, then append
const peer_overflow_policy = "disconnect" &redef;
## Same as `peer_buffer_size` but for WebSocket clients.
## Same as :zeek:see:`Broker::peer_buffer_size` but for WebSocket clients.
const web_socket_buffer_size = 512 &redef;
## Same as `peer_overflow_policy` but for WebSocket clients.
## Same as :zeek:see:`Broker::peer_overflow_policy` but for WebSocket clients.
const web_socket_overflow_policy = "disconnect" &redef;
## The CAF scheduling policy to use. Available options are "sharing" and

View file

@ -24,10 +24,10 @@ export {
STREAM = 2
};
## The default input reader used. Defaults to `READER_ASCII`.
## The default input reader used. Defaults to :zeek:see:`Input::READER_ASCII`.
option default_reader = READER_ASCII;
## The default reader mode used. Defaults to `MANUAL`.
## The default reader mode used. Defaults to :zeek:see:`Input::MANUAL`.
option default_mode = MANUAL;
## Separator between fields.
@ -60,7 +60,7 @@ export {
# Common definitions for tables and events
## String that allows the reader to find the source of the data.
## For `READER_ASCII`, this is the filename.
## For :zeek:see:`Input::READER_ASCII`, this is the filename.
source: string;
## Reader to use for this stream.
@ -112,7 +112,7 @@ export {
##
## The event is raised like if it had been declared as follows:
## error_ev: function(desc: TableDescription, message: string, level: Reporter::Level) &optional;
## The actual declaration uses the ``any`` type because of deficiencies of the Zeek type system.
## The actual declaration uses the :zeek:type:`any` type because of deficiencies of the Zeek type system.
error_ev: any &optional;
## A key/value table that will be passed to the reader.
@ -126,7 +126,7 @@ export {
# Common definitions for tables and events
## String that allows the reader to find the source.
## For `READER_ASCII`, this is the filename.
## For :zeek:see:`Input::READER_ASCII`, this is the filename.
source: string;
## Reader to use for this stream.
@ -151,8 +151,8 @@ export {
want_record: bool &default=T;
## The event that is raised each time a new line is received from the
## reader. The event will receive an Input::EventDescription record
## as the first argument, an Input::Event enum as the second
## reader. The event will receive an :zeek:see:`Input::EventDescription` record
## as the first argument, an :zeek:see:`Input::Event` enum as the second
## argument, and the fields (as specified in *fields*) as the following
## arguments (this will either be a single record value containing
## all fields, or each field value as a separate argument).
@ -161,12 +161,12 @@ export {
## Error event that is raised when an information, warning or error
## is raised by the input stream. If the level is error, the stream will automatically
## be closed.
## The event receives the Input::EventDescription as the first argument, the
## message as the second argument and the Reporter::Level as the third argument.
## The event receives the :zeek:see:`Input::EventDescription` as the first argument, the
## message as the second argument and the :zeek:see:`Reporter::Level` as the third argument.
##
## The event is raised like it had been declared as follows:
## error_ev: function(desc: EventDescription, message: string, level: Reporter::Level) &optional;
## The actual declaration uses the ``any`` type because of deficiencies of the Zeek type system.
## The actual declaration uses the :zeek:type:`any` type because of deficiencies of the Zeek type system.
error_ev: any &optional;
## A key/value table that will be passed to the reader.
@ -179,7 +179,7 @@ export {
## file analysis framework.
type AnalysisDescription: record {
## String that allows the reader to find the source.
## For `READER_ASCII`, this is the filename.
## For :zeek:see:`Input::READER_ASCII`, this is the filename.
source: string;
## Reader to use for this stream. Compatible readers must be
@ -205,14 +205,14 @@ export {
## Create a new table input stream from a given source.
##
## description: `TableDescription` record describing the source.
## description: :zeek:see:`Input::TableDescription` record describing the source.
##
## Returns: true on success.
global add_table: function(description: Input::TableDescription) : bool;
## Create a new event input stream from a given source.
##
## description: `EventDescription` record describing the source.
## description: :zeek:see:`Input::EventDescription` record describing the source.
##
## Returns: true on success.
global add_event: function(description: Input::EventDescription) : bool;
@ -278,4 +278,3 @@ function force_update(id: string) : bool
{
return __force_update(id);
}

View file

@ -198,12 +198,12 @@ export {
## Default separator for log field scopes when logs are unrolled and
## flattened. This will be the string between field name components.
## For example, setting this to "_" will cause the typical field
## "id.orig_h" to turn into "id_orig_h".
## For example, setting this to ``_`` will cause the typical field
## ``id.orig_h`` to turn into ``id_orig_h``.
const default_scope_sep = "." &redef;
## A prefix for extension fields which can be optionally prefixed
## on all log lines by setting the `ext_func` field in the
## on all log lines by setting the ``ext_func`` field in the
## log filter.
const Log::default_ext_prefix: string = "_" &redef;

View file

@ -203,7 +203,7 @@ export {
## on rotation with :zeek:enum:`Notice::ACTION_ALARM`.
##
## Note that this is overridden by the ZeekControl MailTo option or by
## the `email_dest` field in the :zeek:see:`Notice::Info` record.
## the ``email_dest`` field in the :zeek:see:`Notice::Info` record.
const mail_dest = "" &redef;
## Address that emails will be from.

View file

@ -23,7 +23,7 @@ export {
# doc-options-end
# doc-types-start
## Result type for `Spicy::resource_usage()`. The values reflect resource
## Result type for :zeek:see:`Spicy::resource_usage`. The values reflect resource
## usage as reported by the Spicy runtime system.
type ResourceUsage: record {
user_time : interval; ##< user CPU time of the Zeek process

View file

@ -135,7 +135,7 @@ export {
global node: function(): NodeConfig;
## Hooks into the stdout stream for all supervisor's child processes.
## If a hook terminates with `break`, that will suppress output to the
## If a hook terminates with ``break``, that will suppress output to the
## associated stream.
##
## node: the name of a previously created node via
@ -149,7 +149,7 @@ export {
global stdout_hook: hook(node: string, msg: string);
## Hooks into the stderr stream for all supervisor's child processes.
## If a hook terminates with `break`, that will suppress output to the
## If a hook terminates with ``break``, that will suppress output to the
## associated stream.
##
## node: the name of a previously created node via

View file

@ -42,8 +42,8 @@ export {
global counter_with: function(cf: CounterFamily,
label_values: labels_vector &default=vector()): Counter;
## Increment a :zeek:see:`Telemetry::Counter` by `amount`.
## Using a negative `amount` is an error.
## Increment a :zeek:see:`Telemetry::Counter` by ``amount``.
## Using a negative ``amount`` is an error.
##
## c: The counter instance.
##
@ -52,7 +52,7 @@ export {
## Returns: True if the counter was incremented successfully.
global counter_inc: function(c: Counter, amount: double &default=1.0): bool;
## Helper to set a :zeek:see:`Telemetry::Counter` to the given `value`.
## Helper to set a :zeek:see:`Telemetry::Counter` to the given ``value``.
## This can be useful for mirroring counter metrics in an
## :zeek:see:`Telemetry::sync` hook implementation.
## Setting a value that is less than the current value of the
@ -126,7 +126,7 @@ export {
global gauge_with: function(gf: GaugeFamily,
label_values: labels_vector &default=vector()): Gauge;
## Increment a :zeek:see:`Telemetry::Gauge` by `amount`.
## Increment a :zeek:see:`Telemetry::Gauge` by ``amount``.
##
## g: The gauge instance.
##
@ -135,7 +135,7 @@ export {
## Returns: True if the gauge was incremented successfully.
global gauge_inc: function(g: Gauge, amount: double &default=1.0): bool;
## Decrement a :zeek:see:`Telemetry::Gauge` by `amount`.
## Decrement a :zeek:see:`Telemetry::Gauge` by ``amount``.
##
## g: The gauge instance.
##
@ -144,7 +144,7 @@ export {
## Returns: True if the gauge was incremented successfully.
global gauge_dec: function(g: Gauge, amount: double &default=1.0): bool;
## Helper to set a :zeek:see:`Telemetry::Gauge` to the given `value`.
## Helper to set a :zeek:see:`Telemetry::Gauge` to the given ``value``.
##
## g: The gauge instance.
##
@ -153,7 +153,7 @@ export {
## Returns: True if the gauge value was set successfully.
global gauge_set: function(g: Gauge, value: double): bool;
## Increment a :zeek:see:`Telemetry::Gauge` by the given `amount` through
## Increment a :zeek:see:`Telemetry::Gauge` by the given ``amount`` through
## the :zeek:see:`Telemetry::GaugeFamily`.
## This is a short-cut for :zeek:see:`Telemetry::gauge_inc`.
## Using a negative amount is an error.
@ -169,7 +169,7 @@ export {
label_values: labels_vector &default=vector(),
amount: double &default=1.0): bool;
## Decrement a :zeek:see:`Telemetry::Gauge` by the given `amount` through
## Decrement a :zeek:see:`Telemetry::Gauge` by the given ``amount`` through
## the :zeek:see:`Telemetry::GaugeFamily`.
## This is a short-cut for :zeek:see:`Telemetry::gauge_dec`.
##
@ -184,7 +184,7 @@ export {
label_values: labels_vector &default=vector(),
amount: double &default=1.0): bool;
## Set a :zeek:see:`Telemetry::Gauge` to the given `value` through
## Set a :zeek:see:`Telemetry::Gauge` to the given ``value`` through
## the :zeek:see:`Telemetry::GaugeFamily`.
## This is a short-cut for :zeek:see:`Telemetry::gauge_set`.
##

File diff suppressed because it is too large Load diff

View file

@ -1,6 +1,6 @@
##! Provide information about the currently running Zeek version.
##! The most convenient way to access this are the Version::number
##! and Version::info constants.
##! Provide information about the currently running Zeek version. The most
##! convenient way to access this are the :zeek:see:`Version::number` and
##! :zeek:see:`Version::info` constants.
module Version;
@ -41,11 +41,11 @@ export {
##
## version_string: Zeek version string.
##
## Returns: `VersionDescription` record.
## Returns: :zeek:see:`Version::VersionDescription` record.
global parse: function(version_string: string): VersionDescription;
## Test if the current running version of Zeek is greater or equal to the given version
## string.
## Test if the current running version of Zeek is greater or equal to
## the given version string.
##
## version_string: Version to check against the current running version.
##
@ -118,13 +118,15 @@ function parse(version_string: string): VersionDescription
}
export {
## `VersionDescription` record pertaining to the currently running version of Zeek.
## :zeek:see:`Version::VersionDescription` record pertaining to the currently
## running version of Zeek.
const info = Version::parse(zeek_version());
## version number of the currently running version of Zeek as a numeric representation.
## The format of the number is ABBCC with A being the major version,
## bb being the minor version (2 digits) and CC being the patchlevel (2 digits).
## As an example, Zeek 2.4.1 results in the number 20401
## version number of the currently running version of Zeek as a numeric
## representation. The format of the number is ABBCC with A being the
## major version, bb being the minor version (2 digits) and CC being the
## patchlevel (2 digits). As an example, Zeek 2.4.1 results in the
## number 20401
const number = info$version_number;
}

View file

@ -6,7 +6,7 @@ module Exec;
export {
type Command: record {
## The command line to execute. Use care to avoid injection
## The command line to execute. Use care to avoid injection
## attacks (i.e., if the command uses untrusted/variable data,
## sanitize it with :zeek:see:`safe_shell_quote`).
cmd: string;
@ -34,10 +34,10 @@ export {
};
## Function for running command line programs and getting
## output. This is an asynchronous function which is meant
## to be run with the `when` statement.
## output. This is an asynchronous function which is meant
## to be run with the ``when`` statement.
##
## cmd: The command to run. Use care to avoid injection attacks!
## cmd: The command to run. Use care to avoid injection attacks!
##
## Returns: A record representing the full results from the
## external program execution.

View file

@ -2,9 +2,9 @@
##
## s: The string to search for a number.
##
## get_first: Provide `F` if you would like the last number found.
## get_first: Provide ``F`` if you would like the last number found.
##
## Returns: The request integer from the given string or 0 if
## Returns: The request integer from the given string or ``0`` if
## no integer was found.
function extract_count(s: string, get_first: bool &default=T): count
{

View file

@ -186,24 +186,6 @@ global g_configs: table[ConfigState] of Management::Configuration
function config_deploy_to_agents(config: Management::Configuration, req: Management::Request::Request)
{
# Make any final changes to the configuration we send off.
# If needed, fill in agent IP address info as learned from their peerings.
# XXX this will need revisiting when we support host names.
local instances: set[Management::Instance];
for ( inst in config$instances )
{
if ( inst$name in g_instances_known
&& inst$host == 0.0.0.0
&& g_instances_known[inst$name]$host != 0.0.0.0 )
inst$host = g_instances_known[inst$name]$host;
add instances[inst];
}
config$instances = instances;
for ( name in g_instances )
{
if ( name !in g_instances_ready )
@ -414,27 +396,34 @@ function config_assign_metrics_ports(config: Management::Configuration)
[Supervisor::WORKER] = 3,
};
local p = port_to_count(Management::Controller::auto_assign_metrics_start_port);
local ports_set: set[count];
local instance_metrics_start_port: table[addr] of count;
local instance_ports_set: table[addr] of set[count];
local instance_addr_lookup: table[string] of addr;
local node: Management::Node;
local node_addr: addr;
# Pre-populate agents ports, if we have them:
for ( inst in config$instances )
{
# build instance name -> addr lookup table
instance_addr_lookup[inst$name] = inst$host;
instance_metrics_start_port[inst$host] = port_to_count(Management::Controller::auto_assign_metrics_start_port);
instance_ports_set[inst$host] = {};
if ( inst?$listen_port )
add ports_set[port_to_count(inst$listen_port)];
add instance_ports_set[inst$host][port_to_count(inst$listen_port)];
}
# Pre-populate nodes with pre-defined metrics ports, as well
# as their Broker ports:
for ( node in config$nodes )
node_addr = instance_addr_lookup[node$instance];
{
if ( node?$p )
add ports_set[port_to_count(node$p)];
add instance_ports_set[node_addr][port_to_count(node$p)];
if ( node?$metrics_port )
{
add ports_set[port_to_count(node$metrics_port)];
add instance_ports_set[node_addr][port_to_count(node$metrics_port)];
add new_nodes[node];
}
}
@ -466,17 +455,18 @@ function config_assign_metrics_ports(config: Management::Configuration)
for ( i in nodes )
{
node = nodes[i];
node_addr = instance_addr_lookup[node$instance];
# Find next available port ...
while ( p in ports_set )
++p;
while ( instance_metrics_start_port[node_addr] in instance_ports_set[node_addr] )
++instance_metrics_start_port[node_addr];
node$metrics_port = count_to_port(p, tcp);
node$metrics_port = count_to_port(instance_metrics_start_port[node_addr], tcp);
add new_nodes[node];
add ports_set[p];
add instance_ports_set[node_addr][instance_metrics_start_port[node_addr]];
# ... and consume it.
++p;
++instance_metrics_start_port[node_addr];
}
config$nodes = new_nodes;
@ -1029,6 +1019,27 @@ event Management::Controller::API::stage_configuration_request(reqid: string, co
g_configs[STAGED] = config;
config_copy = copy(config);
# The staged config is preserved as the client sent it to us. For the
# ready-to-deploy version we fill in additional details here.
#
# One such bit of information is that we know the IP addresses of
# instances that connected to the controller from their Broker peering.
#
# XXX this will need revisiting when we support host names.
local instances: set[Management::Instance];
for ( inst in config_copy$instances )
{
if ( inst$name in g_instances_known
&& inst$host == 0.0.0.0
&& g_instances_known[inst$name]$host != 0.0.0.0 )
inst$host = g_instances_known[inst$name]$host;
add instances[inst];
}
config_copy$instances = instances;
if ( Management::Controller::auto_assign_broker_ports )
config_assign_broker_ports(config_copy);
if ( Management::Controller::auto_assign_metrics_ports )

View file

@ -8,19 +8,19 @@ export {
const max_bpf_shunts = 100 &redef;
## Call this function to use BPF to shunt a connection (to prevent the
## data packets from reaching Zeek). For TCP connections, control
## data packets from reaching Zeek). For TCP connections, control
## packets are still allowed through so that Zeek can continue logging
## the connection and it can stop shunting once the connection ends.
global shunt_conn: function(id: conn_id): bool;
## This function will use a BPF expression to shunt traffic between
## the two hosts given in the `conn_id` so that the traffic is never
## the two hosts given in the ``conn_id`` so that the traffic is never
## exposed to Zeek's traffic processing.
global shunt_host_pair: function(id: conn_id): bool;
## Remove shunting for a host pair given as a `conn_id`. The filter
## Remove shunting for a host pair given as a ``conn_id``. The filter
## is not immediately removed. It waits for the occasional filter
## update done by the `PacketFilter` framework.
## update done by the ``PacketFilter`` framework.
global unshunt_host_pair: function(id: conn_id): bool;
## Performs the same function as the :zeek:id:`PacketFilter::unshunt_host_pair`
@ -40,7 +40,7 @@ export {
No_More_Conn_Shunts_Available,
## Limitations in BPF make shunting some connections with BPF
## impossible. This notice encompasses those various cases.
## impossible. This notice encompasses those various cases.
Cannot_BPF_Shunt_Conn,
};
}

View file

@ -1,5 +1,5 @@
##! Implementation of a telemetry.log and telemetry_histogram.log file
##! using metrics accessible via the Telemetry module.
##! Implementation of a :file:`telemetry.log` and :file:`telemetry_histogram.log`
##! file using metrics accessible via the Telemetry module.
@load base/frameworks/telemetry
@ -12,8 +12,9 @@ export {
option log_interval = 60sec;
## Only metrics with prefixes in this set will be included in the
## `telemetry.log` and `telemetry_histogram.log` files by default.
## Setting this option to an empty set includes all prefixes.
## :file:`telemetry.log` and :file:`telemetry_histogram.log` files
## by default. Setting this option to an empty set includes all
## prefixes.
##
## For more fine-grained customization, setting this option to an
## empty set and implementing the :zeek:see:`Telemetry::log_policy`

View file

@ -15,13 +15,19 @@ export {
## **must** be set appropriately for this detection.
External_Name,
};
## Default is to ignore mDNS broadcasts.
option skip_resp_host_port_pairs: set[addr, port] = { [[224.0.0.251, [ff02::fb]], 5353/udp] };
}
event dns_A_reply(c: connection, msg: dns_msg, ans: dns_answer, a: addr) &priority=-3
function detect_external_names(c: connection, msg: dns_msg, ans: dns_answer, a: addr)
{
if ( |Site::local_zones| == 0 )
return;
if ( [c$id$resp_h, c$id$resp_p] in skip_resp_host_port_pairs )
return;
# Check for responses from remote hosts that point at local hosts
# but the name is not considered to be within a "local" zone.
if ( Site::is_local_addr(a) && # referring to a local host
@ -33,3 +39,13 @@ event dns_A_reply(c: connection, msg: dns_msg, ans: dns_answer, a: addr) &priori
$identifier=cat(a,ans$query)]);
}
}
event dns_A_reply(c: connection, msg: dns_msg, ans: dns_answer, a: addr)
{
detect_external_names(c, msg, ans, a);
}
event dns_AAAA_reply(c: connection, msg: dns_msg, ans: dns_answer, a: addr)
{
detect_external_names(c, msg, ans, a);
}