diff --git a/CHANGES b/CHANGES index 6c02174371..d673944ef0 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,11 @@ +5.1.0-dev.52 | 2022-06-16 11:11:33 -0700 + + * Management framework: bump external cluster testsuite (Christian Kreibich, Corelight) + + * Management framework: make agents support zeek-archiver invocations (Christian Kreibich, Corelight) + + * Management framework: fix module naming typo (Christian Kreibich, Corelight) + 5.1.0-dev.48 | 2022-06-16 10:08:14 -0700 * Use ccache for docs gen github workflow (Tim Wojtulewicz, Corelight) diff --git a/VERSION b/VERSION index ad2fc867cc..425f67ab9a 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -5.1.0-dev.48 +5.1.0-dev.52 diff --git a/scripts/policy/frameworks/management/agent/config.zeek b/scripts/policy/frameworks/management/agent/config.zeek index ae9266c57a..e6cd13bb9d 100644 --- a/scripts/policy/frameworks/management/agent/config.zeek +++ b/scripts/policy/frameworks/management/agent/config.zeek @@ -1,5 +1,6 @@ ##! Configuration settings for a cluster agent. +@load base/misc/installation @load policy/frameworks/management # We source the controller configuration to obtain its network coordinates, so @@ -44,6 +45,26 @@ export { ## The fallback listen port if :zeek:see:`Management::Agent::listen_port` remains empty. const default_port = 2151/tcp &redef; + ## Whether the agent should periodically invoke zeek-archiver to + ## finalize logs. + const archive_logs = T &redef; + + ## The archival interval to use. When 0, it defaults to the log rotation + ## interval. + const archive_interval = 0 sec &redef; + + ## The archival command. When empty, defaults to the zeek-archiver + ## installed with the Zeek distribution. Whatever the command, the + ## agent will invoke it like zeek-archiver, so take a look at its + ## command-line arguments if you're planning to put in place a + ## substitute. Archival happens from the + ## :zeek:see:`Log::default_rotation_dir` to + ## :zeek:see:`Management::Agent::archive_dir`. + const archive_cmd = "" &redef; + + ## The destination interval for archived logs. + const archive_dir = Installation::log_dir &redef; + ## The agent's Broker topic prefix. For its own communication, the agent ## suffixes this with "/", based on :zeek:see:`Management::Agent::get_name`. const topic_prefix = "zeek/management/agent" &redef; diff --git a/scripts/policy/frameworks/management/agent/main.zeek b/scripts/policy/frameworks/management/agent/main.zeek index 6105edb8dd..4757f5a3ec 100644 --- a/scripts/policy/frameworks/management/agent/main.zeek +++ b/scripts/policy/frameworks/management/agent/main.zeek @@ -15,7 +15,7 @@ @load ./api @load ./config -module Mangement::Agent::Runtime; +module Management::Agent::Runtime; # This export is mainly to appease Zeekygen's need to understand redefs of the # Request record below. Without it, it fails to establish link targets for the @@ -43,6 +43,17 @@ export { ## Request state for every node managed by this agent. requests: set[string] &default=set(); }; + + # When Management::Agent::archive_logs is T (the default) and the + # logging configuration doesn't permanently prevent archival + # (e.g. because log rotation isn't configured), the agent triggers this + # event each Management::Controller::archive_interval to initiate log + # archival. + # + # run_archival: whether to actually invoke the archiver or just + # ensure (re-)scheduling. + # + global trigger_log_archival: event(run_archival: bool &default=T); } # We need to go out of our way here to avoid colliding record field names with @@ -142,6 +153,52 @@ function send_set_configuration_response(req: Management::Request::Request) g_config_reqid_pending = ""; } +event Management::Agent::Runtime::trigger_log_archival(run_archival: bool) + { + # This is currently final, but could be considered dynamically in the + # future if we make this an option. + if ( Management::Agent::archive_logs == F ) + return; + + local ival = Management::Agent::archive_interval; + + # Fall back to the default rotation interval when not set explicitly: + if ( ival == 0 secs ) + ival = Log::default_rotation_interval; + + # Without a default rotation interval individual log streams might still + # have rotation enabled, and we could scan all filters to determine + # their rotation configuration. But it's not clear that this is + # intuitive or needed, since it's uncommon to want rotation for only + # some logs. So we simply don't proceed if it's not configured. + if ( ival == 0 secs ) + return; + + local cmd = Management::Agent::archive_cmd; + + if ( cmd == "" ) + { + cmd = join_string_vec(vector(Installation::root_dir, "bin"), "/"); + cmd = build_path_compressed(cmd, "zeek-archiver"); + } + + # The logging framework creates the rotation directory on demand, so + # only trigger archival when it exists. Don't warn when it does not: + # this will often be expected, since in larger clusters many instances + # may not run loggers. + if ( run_archival && file_size(Log::default_rotation_dir) > 0 ) + { + cmd = fmt("%s -1 %s %s", + cmd, Log::default_rotation_dir, + Management::Agent::archive_dir); + + Management::Log::info(fmt("triggering log archival via '%s'", cmd)); + system(cmd); + } + + schedule ival { Management::Agent::Runtime::trigger_log_archival() }; + } + event Management::Supervisor::API::notify_node_exit(node: string, outputs: Management::NodeOutputs) { if ( node in g_nodes ) @@ -737,5 +794,8 @@ event zeek_init() # If the controller connects to us, it also uses this port. Broker::listen(cat(epi$network$address), epi$network$bound_port); + if ( Management::Agent::archive_logs ) + schedule 0 secs { Management::Agent::Runtime::trigger_log_archival(F) }; + Management::Log::info(fmt("agent is live, Broker ID %s", Broker::node_id())); } diff --git a/testing/external/commit-hash.zeek-testing-cluster b/testing/external/commit-hash.zeek-testing-cluster index d9b7621df3..091d8d272e 100644 --- a/testing/external/commit-hash.zeek-testing-cluster +++ b/testing/external/commit-hash.zeek-testing-cluster @@ -1 +1 @@ -a1c8c09c8c661a1ea9299e0356f3652502b8dcd2 +fa91ac8b028034bbbf22acca00a059cbc6d90829