From d1cd409e59491a77a1a0a63ee7ad825fa36986bc Mon Sep 17 00:00:00 2001 From: Christian Kreibich Date: Fri, 20 May 2022 16:42:04 -0700 Subject: [PATCH] Management framework: set defaults for log rotation and persistent state This adds management/persistence.zeek to establish common configuration for log rotation and persistent variable state. Log-writing Zeek processes initially write locally in their working directory, and rotate into subdirectory "log-queue" of the spool. Since agent and controller have no logger, persistence.zeek puts in place compatible configurations for them. Storage folders for Broker-backed tables and clusterized stores default to subdirectories of the new Zeek-level state folder. When setting the ZEEK_MANAGEMENT_TESTING environment variable, persistent state is kept in the local directory, and log rotation remains disabled. This also tweaks @loads a bit in favor of simply loading frameworks/management, which is easier to keep track of. --- .../frameworks/management/__load__.zeek | 1 + .../frameworks/management/agent/config.zeek | 3 +- .../management/controller/config.zeek | 3 +- .../frameworks/management/node/main.zeek | 13 ++++- .../frameworks/management/persistence.zeek | 47 +++++++++++++++++++ scripts/test-all-policy.zeek | 1 + .../Baseline/coverage.bare-mode-errors/errors | 4 +- 7 files changed, 65 insertions(+), 7 deletions(-) create mode 100644 scripts/policy/frameworks/management/persistence.zeek diff --git a/scripts/policy/frameworks/management/__load__.zeek b/scripts/policy/frameworks/management/__load__.zeek index 96192ea366..8dc27de8c4 100644 --- a/scripts/policy/frameworks/management/__load__.zeek +++ b/scripts/policy/frameworks/management/__load__.zeek @@ -6,6 +6,7 @@ @load ./config @load ./log +@load ./persistence @load ./request @load ./types @load ./util diff --git a/scripts/policy/frameworks/management/agent/config.zeek b/scripts/policy/frameworks/management/agent/config.zeek index eb9d7f315e..53688dcbad 100644 --- a/scripts/policy/frameworks/management/agent/config.zeek +++ b/scripts/policy/frameworks/management/agent/config.zeek @@ -1,7 +1,6 @@ ##! Configuration settings for a cluster agent. -@load policy/frameworks/management/config -@load policy/frameworks/management/types +@load policy/frameworks/management # We source the controller configuration to obtain its network coordinates, so # we can default to connecting to it. diff --git a/scripts/policy/frameworks/management/controller/config.zeek b/scripts/policy/frameworks/management/controller/config.zeek index f39d26ba15..2728d01ec3 100644 --- a/scripts/policy/frameworks/management/controller/config.zeek +++ b/scripts/policy/frameworks/management/controller/config.zeek @@ -1,7 +1,6 @@ ##! Configuration settings for the cluster controller. -@load policy/frameworks/management/config -@load policy/frameworks/management/types +@load policy/frameworks/management module Management::Controller; diff --git a/scripts/policy/frameworks/management/node/main.zeek b/scripts/policy/frameworks/management/node/main.zeek index c8649b6054..df65d754b9 100644 --- a/scripts/policy/frameworks/management/node/main.zeek +++ b/scripts/policy/frameworks/management/node/main.zeek @@ -1,10 +1,14 @@ ##! This module provides Management framework functionality present in every ##! cluster node, to allowing Management agents to interact with the nodes. +@load base/frameworks/broker/store @load base/frameworks/cluster +@load base/frameworks/logging/writers/ascii +@load base/misc/installation +@load base/utils/paths +@load policy/frameworks/management @load policy/frameworks/management/agent/config -@load policy/frameworks/management/log @load ./api @load ./config @@ -103,6 +107,13 @@ event Broker::peer_added(peer: Broker::EndpointInfo, msg: string) event zeek_init() { + if ( Broker::table_store_db_directory != "" && ! mkdir(Broker::table_store_db_directory) ) + Management::Log::error(fmt("could not create Broker data store directory '%s'", + Broker::table_store_db_directory)); + if ( Cluster::default_store_dir != "" && ! mkdir(Cluster::default_store_dir) ) + Management::Log::error(fmt("could not create Cluster store directory '%s'", + Cluster::default_store_dir)); + local epi = Management::Agent::endpoint_info(); Broker::peer(epi$network$address, epi$network$bound_port, Management::connect_retry); diff --git a/scripts/policy/frameworks/management/persistence.zeek b/scripts/policy/frameworks/management/persistence.zeek new file mode 100644 index 0000000000..bb5731ced0 --- /dev/null +++ b/scripts/policy/frameworks/management/persistence.zeek @@ -0,0 +1,47 @@ +##! Common adjustments for any kind of Zeek node when we run the Management +##! framework. + +@load base/misc/installation +@load base/utils/paths + +@load ./config + +# For testing, keep persistent state local to the current working directory, +# and disable log rotation. +@if ( getenv("ZEEK_MANAGEMENT_TESTING") != "" ) + +redef Management::spool_dir = "."; +redef Management::state_dir = "."; +redef Log::default_rotation_interval = 0 secs; + +@else + +# For any kind of Zeek process we steer rotated logs awaiting archival into a +# queue directory in the spool. The name "log-queue" matches logger nodes' default +# config with the Supervisor; see base/frameworks/cluster/nodes/logger.zeek. +redef Log::default_rotation_dir = build_path(Management::get_spool_dir(), "log-queue"); + +@if ( getenv("ZEEK_MANAGEMENT_NODE") != "" ) + +# Management agents and controllers don't have loggers, nor their configuration, +# so establish a similar one here: + +function archiver_rotation_format_func(ri: Log::RotationFmtInfo): Log::RotationPath + { + local open_str = strftime(Log::default_rotation_date_format, ri$open); + local close_str = strftime(Log::default_rotation_date_format, ri$close); + local base = fmt("%s__%s__%s__", ri$path, open_str, close_str); + local rval = Log::RotationPath($file_basename=base); + return rval; + } + +redef Log::default_rotation_interval = 1 hrs; +redef Log::enable_local_logging = T; +redef Log::enable_remote_logging = T; +redef Log::rotation_format_func = archiver_rotation_format_func; + +redef LogAscii::enable_leftover_log_rotation = T; + +@endif # ZEEK_MANAGEMENT_NODE + +@endif # ZEEK_MANAGEMENT_TESTING diff --git a/scripts/test-all-policy.zeek b/scripts/test-all-policy.zeek index 0f2de90609..2dc50cd8af 100644 --- a/scripts/test-all-policy.zeek +++ b/scripts/test-all-policy.zeek @@ -24,6 +24,7 @@ @load frameworks/management/__load__.zeek @load frameworks/management/config.zeek @load frameworks/management/log.zeek +@load frameworks/management/persistence.zeek # @load frameworks/management/node/__load__.zeek @load frameworks/management/node/api.zeek @load frameworks/management/node/config.zeek diff --git a/testing/btest/Baseline/coverage.bare-mode-errors/errors b/testing/btest/Baseline/coverage.bare-mode-errors/errors index bc9bd28f83..6e8c09e1bd 100644 --- a/testing/btest/Baseline/coverage.bare-mode-errors/errors +++ b/testing/btest/Baseline/coverage.bare-mode-errors/errors @@ -2,8 +2,8 @@ ### NOTE: This file has been sorted with diff-sort. warning in <...>/extract-certs-pem.zeek, line 1: deprecated script loaded from <...>/__load__.zeek:15 "Remove in v5.1. Use log-certs-base64.zeek instead." warning in <...>/extract-certs-pem.zeek, line 1: deprecated script loaded from command line arguments "Remove in v5.1. Use log-certs-base64.zeek instead." -warning in <...>/log-ocsp.zeek, line 1: deprecated script loaded from <...>/test-all-policy.zeek:65 ("Remove in v5.1. OCSP logging is now enabled by default") -warning in <...>/log-ocsp.zeek, line 1: deprecated script loaded from <...>/test-all-policy.zeek:65 ("Remove in v5.1. OCSP logging is now enabled by default") +warning in <...>/log-ocsp.zeek, line 1: deprecated script loaded from <...>/test-all-policy.zeek:66 ("Remove in v5.1. OCSP logging is now enabled by default") +warning in <...>/log-ocsp.zeek, line 1: deprecated script loaded from <...>/test-all-policy.zeek:66 ("Remove in v5.1. OCSP logging is now enabled by default") warning in <...>/log-ocsp.zeek, line 1: deprecated script loaded from command line arguments ("Remove in v5.1. OCSP logging is now enabled by default") warning in <...>/notary.zeek, line 1: deprecated script loaded from <...>/__load__.zeek:5 ("Remove in v5.1. Please switch to other more modern approaches like SCT validation (validate-sct.zeek).") warning in <...>/notary.zeek, line 1: deprecated script loaded from command line arguments ("Remove in v5.1. Please switch to other more modern approaches like SCT validation (validate-sct.zeek).")