Mirror/zeek
1
0
Fork 0
mirror of https://github.com/zeek/zeek.git synced 2025-10-02 06:38:20 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Implement leftover log rotation/archival for supervised nodes

Browse source 
This helps prevent a node from being killed/crashing in the middle
of writing a log, restarting, and eventually clobbering that log
file that never underwent the rotation/archival process.

The old `archive-log` and `post-terminate` scripts as used by
ZeekControl previously implemented this behavior, but the new logic is
entirely in the ASCII writer.  It uses ".shadow" log files stored
alongside the real log to help detect such scenarios and rotate them
correctly upon the next startup of the Zeek process.
This commit is contained in:
Jon Siwek 2020-06-24 14:28:54 -07:00
parent a46e24091a
commit 11949ce37a
11 changed files with 523 additions and 17 deletions
Download patch file Download diff file Expand all files Collapse all files

112
testing/btest/supervisor/config-cluster-leftover-log-archival.zeek Normal file
View file

@ -0,0 +1,112 @@
# @TEST-PORT: SUPERVISOR_PORT
# @TEST-PORT: LOGGER_PORT
# Test default leftover log rotation/archival behavior
# @TEST-EXEC: btest-bg-run zeek zeek -j -b %INPUT
# @TEST-EXEC: btest-bg-wait 45
# @TEST-EXEC: cp zeek/logger-1/test*.log test.default.log
# @TEST-EXEC: btest-diff test.default.log
# @TEST-EXEC: rm -rf ./zeek
# Test leftover log rotation/archival behavior with custom postprocessor func
# @TEST-EXEC: btest-bg-run zeek zeek -j -b %INPUT use_custom_postproc=T
# @TEST-EXEC: btest-bg-wait 45
# @TEST-EXEC: cp zeek/logger-1/test*.log test.postproc.log
# @TEST-EXEC: btest-diff test.postproc.log
# @TEST-EXEC: btest-diff zeek/logger-1/postproc.out
# @TEST-EXEC: rm -rf ./zeek
@load base/frameworks/cluster
option use_custom_postproc = F;
# JSON for log file brevity.
redef LogAscii::use_json=T;
global topic = "test-topic";
module Test;
export {
redef enum Log::ID += { LOG };
type Log: record {
s: string;
} &log;
}
module GLOBAL;
module LogAscii;
export {
function my_rotation_postprocessor(info: Log::RotationInfo) : bool
{
local f = open("postproc.out");
print f, "running my rotation postprocessor";
close(f);
return LogAscii::default_rotation_postprocessor_func(info);
}
}
module GLOBAL;
event zeek_init()
{
Log::create_stream(Test::LOG, [$columns=Test::Log]);
if ( use_custom_postproc )
{
local df = Log::get_filter(Test::LOG, "default");
df$postprocessor = LogAscii::my_rotation_postprocessor;
Log::add_filter(Test::LOG, df);
}
if ( Supervisor::is_supervisor() )
{
Broker::subscribe(topic);
Broker::listen("127.0.0.1", to_port(getenv("SUPERVISOR_PORT")));
Broker::peer("127.0.0.1", to_port(getenv("LOGGER_PORT")));
local cluster: table[string] of Supervisor::ClusterEndpoint;
cluster["logger-1"] = [$role=Supervisor::LOGGER, $host=127.0.0.1,
$p=to_port(getenv("LOGGER_PORT"))];
for ( n, ep in cluster )
{
local sn = Supervisor::NodeConfig($name = n);
sn$cluster = cluster;
sn$directory = n;
# Hard to test the full process of a kill/crash leaving these
# leftover files, so just fake them.
mkdir(sn$directory);
local f = open(fmt("%s/test.log", sn$directory));
print f, "{\"s\":\"leftover test\"}";
close(f);
local sf = open(fmt("%s/.shadow.test.log", sn$directory));
print sf, ".log";
if ( use_custom_postproc )
print sf, "LogAscii::my_rotation_postprocessor";
else
print sf, "";
close(sf);
local res = Supervisor::create(sn);
if ( res != "" )
print fmt("failed to create node %s: %s", n, res);
}
}
else
{
Broker::subscribe(topic);
Broker::peer("127.0.0.1", to_port(getenv("SUPERVISOR_PORT")));
}
}
event Broker::peer_added(endpoint: Broker::EndpointInfo, msg: string)
{
if ( Supervisor::is_supervisor() )
terminate();
}