Management framework: set defaults for log rotation and persistent state

This adds management/persistence.zeek to establish common configuration for log
rotation and persistent variable state. Log-writing Zeek processes initially
write locally in their working directory, and rotate into subdirectory
"log-queue" of the spool. Since agent and controller have no logger,
persistence.zeek puts in place compatible configurations for them.

Storage folders for Broker-backed tables and clusterized stores default to
subdirectories of the new Zeek-level state folder.

When setting the ZEEK_MANAGEMENT_TESTING environment variable, persistent state
is kept in the local directory, and log rotation remains disabled.

This also tweaks @loads a bit in favor of simply loading frameworks/management,
which is easier to keep track of.

scripts/policy/frameworks/management/__load__.zeek

@@ -6,6 +6,7 @@
 
 @load ./config
 @load ./log
+@load ./persistence
 @load ./request
 @load ./types
 @load ./util

scripts/policy/frameworks/management/agent/config.zeek

@@ -1,7 +1,6 @@
 ##! Configuration settings for a cluster agent.
 
-@load policy/frameworks/management/config
+@load policy/frameworks/management
-@load policy/frameworks/management/types
 
 # We source the controller configuration to obtain its network coordinates, so
 # we can default to connecting to it.

scripts/policy/frameworks/management/controller/config.zeek

@@ -1,7 +1,6 @@
 ##! Configuration settings for the cluster controller.
 
-@load policy/frameworks/management/config
+@load policy/frameworks/management
-@load policy/frameworks/management/types
 
 module Management::Controller;
 

scripts/policy/frameworks/management/node/main.zeek

@@ -1,10 +1,14 @@
 ##! This module provides Management framework functionality present in every
 ##! cluster node, to allowing Management agents to interact with the nodes.
 
+@load base/frameworks/broker/store
 @load base/frameworks/cluster
+@load base/frameworks/logging/writers/ascii
+@load base/misc/installation
+@load base/utils/paths
 
+@load policy/frameworks/management
 @load policy/frameworks/management/agent/config
-@load policy/frameworks/management/log
 
 @load ./api
 @load ./config
@@ -103,6 +107,13 @@ event Broker::peer_added(peer: Broker::EndpointInfo, msg: string)
 
 event zeek_init()
 	{
+	if ( Broker::table_store_db_directory != "" && ! mkdir(Broker::table_store_db_directory) )
+		Management::Log::error(fmt("could not create Broker data store directory '%s'",
+		    Broker::table_store_db_directory));
+	if ( Cluster::default_store_dir != "" && ! mkdir(Cluster::default_store_dir) )
+		Management::Log::error(fmt("could not create Cluster store directory '%s'",
+		    Cluster::default_store_dir));
+
 	local epi = Management::Agent::endpoint_info();
 
 	Broker::peer(epi$network$address, epi$network$bound_port, Management::connect_retry);

scripts/policy/frameworks/management/persistence.zeek

@@ -0,0 +1,47 @@
+##! Common adjustments for any kind of Zeek node when we run the Management
+##! framework.
+
+@load base/misc/installation
+@load base/utils/paths
+
+@load ./config
+
+# For testing, keep persistent state local to the current working directory,
+# and disable log rotation.
+@if ( getenv("ZEEK_MANAGEMENT_TESTING") != "" )
+
+redef Management::spool_dir = ".";
+redef Management::state_dir = ".";
+redef Log::default_rotation_interval = 0 secs;
+
+@else
+
+# For any kind of Zeek process we steer rotated logs awaiting archival into a
+# queue directory in the spool. The name "log-queue" matches logger nodes' default
+# config with the Supervisor; see base/frameworks/cluster/nodes/logger.zeek.
+redef Log::default_rotation_dir = build_path(Management::get_spool_dir(), "log-queue");
+
+@if ( getenv("ZEEK_MANAGEMENT_NODE") != "" )
+
+# Management agents and controllers don't have loggers, nor their configuration,
+# so establish a similar one here:
+
+function archiver_rotation_format_func(ri: Log::RotationFmtInfo): Log::RotationPath
+	{
+	local open_str = strftime(Log::default_rotation_date_format, ri$open);
+	local close_str = strftime(Log::default_rotation_date_format, ri$close);
+	local base = fmt("%s__%s__%s__", ri$path, open_str, close_str);
+	local rval = Log::RotationPath($file_basename=base);
+	return rval;
+	}
+
+redef Log::default_rotation_interval = 1 hrs;
+redef Log::enable_local_logging = T;
+redef Log::enable_remote_logging = T;
+redef Log::rotation_format_func = archiver_rotation_format_func;
+
+redef LogAscii::enable_leftover_log_rotation = T;
+
+@endif # ZEEK_MANAGEMENT_NODE
+
+@endif # ZEEK_MANAGEMENT_TESTING

scripts/test-all-policy.zeek

@@ -24,6 +24,7 @@
 @load frameworks/management/__load__.zeek
 @load frameworks/management/config.zeek
 @load frameworks/management/log.zeek
+@load frameworks/management/persistence.zeek
 # @load frameworks/management/node/__load__.zeek
 @load frameworks/management/node/api.zeek
 @load frameworks/management/node/config.zeek

testing/btest/Baseline/coverage.bare-mode-errors/errors

@@ -2,8 +2,8 @@
 ### NOTE: This file has been sorted with diff-sort.
 warning in <...>/extract-certs-pem.zeek, line 1: deprecated script loaded from <...>/__load__.zeek:15 "Remove in v5.1. Use log-certs-base64.zeek instead."
 warning in <...>/extract-certs-pem.zeek, line 1: deprecated script loaded from command line arguments "Remove in v5.1. Use log-certs-base64.zeek instead."
-warning in <...>/log-ocsp.zeek, line 1: deprecated script loaded from <...>/test-all-policy.zeek:65 ("Remove in v5.1. OCSP logging is now enabled by default")
+warning in <...>/log-ocsp.zeek, line 1: deprecated script loaded from <...>/test-all-policy.zeek:66 ("Remove in v5.1. OCSP logging is now enabled by default")
-warning in <...>/log-ocsp.zeek, line 1: deprecated script loaded from <...>/test-all-policy.zeek:65 ("Remove in v5.1. OCSP logging is now enabled by default")
+warning in <...>/log-ocsp.zeek, line 1: deprecated script loaded from <...>/test-all-policy.zeek:66 ("Remove in v5.1. OCSP logging is now enabled by default")
 warning in <...>/log-ocsp.zeek, line 1: deprecated script loaded from command line arguments ("Remove in v5.1. OCSP logging is now enabled by default")
 warning in <...>/notary.zeek, line 1: deprecated script loaded from <...>/__load__.zeek:5 ("Remove in v5.1. Please switch to other more modern approaches like SCT validation (validate-sct.zeek).")
 warning in <...>/notary.zeek, line 1: deprecated script loaded from command line arguments ("Remove in v5.1. Please switch to other more modern approaches like SCT validation (validate-sct.zeek).")