Merge branch 'topic/christian/management-schedule-archiver'

* topic/christian/management-schedule-archiver:
  Management framework: bump external cluster testsuite
  Management framework: make agents support zeek-archiver invocations
  Management framework: fix module naming typo

CHANGES

@@ -1,3 +1,11 @@
+5.1.0-dev.52 | 2022-06-16 11:11:33 -0700
+
+  * Management framework: bump external cluster testsuite (Christian Kreibich, Corelight)
+
+  * Management framework: make agents support zeek-archiver invocations (Christian Kreibich, Corelight)
+
+  * Management framework: fix module naming typo (Christian Kreibich, Corelight)
+
 5.1.0-dev.48 | 2022-06-16 10:08:14 -0700
 
   * Use ccache for docs gen github workflow (Tim Wojtulewicz, Corelight)

VERSION

@@ -1 +1 @@
-5.1.0-dev.48
+5.1.0-dev.52

scripts/policy/frameworks/management/agent/config.zeek

@@ -1,5 +1,6 @@
 ##! Configuration settings for a cluster agent.
 
+@load base/misc/installation
 @load policy/frameworks/management
 
 # We source the controller configuration to obtain its network coordinates, so
@@ -44,6 +45,26 @@ export {
 	## The fallback listen port if :zeek:see:`Management::Agent::listen_port` remains empty.
 	const default_port = 2151/tcp &redef;
 
+	## Whether the agent should periodically invoke zeek-archiver to
+	## finalize logs.
+	const archive_logs = T &redef;
+
+	## The archival interval to use. When 0, it defaults to the log rotation
+	## interval.
+	const archive_interval = 0 sec &redef;
+
+	## The archival command. When empty, defaults to the zeek-archiver
+	## installed with the Zeek distribution. Whatever the command, the
+	## agent will invoke it like zeek-archiver, so take a look at its
+	## command-line arguments if you're planning to put in place a
+	## substitute. Archival happens from the
+	## :zeek:see:`Log::default_rotation_dir` to
+	## :zeek:see:`Management::Agent::archive_dir`.
+	const archive_cmd = "" &redef;
+
+	## The destination interval for archived logs.
+	const archive_dir = Installation::log_dir &redef;
+
 	## The agent's Broker topic prefix. For its own communication, the agent
 	## suffixes this with "/<name>", based on :zeek:see:`Management::Agent::get_name`.
 	const topic_prefix = "zeek/management/agent" &redef;

scripts/policy/frameworks/management/agent/main.zeek

@@ -15,7 +15,7 @@
 @load ./api
 @load ./config
 
-module Mangement::Agent::Runtime;
+module Management::Agent::Runtime;
 
 # This export is mainly to appease Zeekygen's need to understand redefs of the
 # Request record below. Without it, it fails to establish link targets for the
@@ -43,6 +43,17 @@ export {
 		## Request state for every node managed by this agent.
 		requests: set[string] &default=set();
 	};
+
+	# When Management::Agent::archive_logs is T (the default) and the
+	# logging configuration doesn't permanently prevent archival
+	# (e.g. because log rotation isn't configured), the agent triggers this
+	# event each Management::Controller::archive_interval to initiate log
+	# archival.
+	#
+	# run_archival: whether to actually invoke the archiver or just
+	#     ensure (re-)scheduling.
+	#
+	global trigger_log_archival: event(run_archival: bool &default=T);
 }
 
 # We need to go out of our way here to avoid colliding record field names with
@@ -142,6 +153,52 @@ function send_set_configuration_response(req: Management::Request::Request)
 		g_config_reqid_pending = "";
 	}
 
+event Management::Agent::Runtime::trigger_log_archival(run_archival: bool)
+	{
+	# This is currently final, but could be considered dynamically in the
+	# future if we make this an option.
+	if ( Management::Agent::archive_logs == F )
+		return;
+
+	local ival = Management::Agent::archive_interval;
+
+	# Fall back to the default rotation interval when not set explicitly:
+	if ( ival == 0 secs )
+		ival = Log::default_rotation_interval;
+
+	# Without a default rotation interval individual log streams might still
+	# have rotation enabled, and we could scan all filters to determine
+	# their rotation configuration. But it's not clear that this is
+	# intuitive or needed, since it's uncommon to want rotation for only
+	# some logs. So we simply don't proceed if it's not configured.
+	if ( ival == 0 secs )
+		return;
+
+	local cmd = Management::Agent::archive_cmd;
+
+	if ( cmd == "" )
+		{
+		cmd = join_string_vec(vector(Installation::root_dir, "bin"), "/");
+		cmd = build_path_compressed(cmd, "zeek-archiver");
+		}
+
+	# The logging framework creates the rotation directory on demand, so
+	# only trigger archival when it exists. Don't warn when it does not:
+	# this will often be expected, since in larger clusters many instances
+	# may not run loggers.
+	if ( run_archival && file_size(Log::default_rotation_dir) > 0 )
+		{
+		cmd = fmt("%s -1 %s %s",
+		    cmd, Log::default_rotation_dir,
+		    Management::Agent::archive_dir);
+
+		Management::Log::info(fmt("triggering log archival via '%s'", cmd));
+		system(cmd);
+		}
+
+	schedule ival { Management::Agent::Runtime::trigger_log_archival() };
+	}
+
 event Management::Supervisor::API::notify_node_exit(node: string, outputs: Management::NodeOutputs)
 	{
 	if ( node in g_nodes )
@@ -737,5 +794,8 @@ event zeek_init()
 	# If the controller connects to us, it also uses this port.
 	Broker::listen(cat(epi$network$address), epi$network$bound_port);
 
+	if ( Management::Agent::archive_logs )
+		schedule 0 secs { Management::Agent::Runtime::trigger_log_archival(F) };
+
 	Management::Log::info(fmt("agent is live, Broker ID %s", Broker::node_id()));
 	}

testing/external/commit-hash.zeek-testing-cluster

@@ -1 +1 @@
-a1c8c09c8c661a1ea9299e0356f3652502b8dcd2
+fa91ac8b028034bbbf22acca00a059cbc6d90829