Mirror/zeek
1
0
Fork 0
mirror of https://github.com/zeek/zeek.git synced 2025-10-02 06:38:20 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Add Log::rotation_format_func and Log::default_rotation_dir options

Browse source 
These may be redefined to customize log rotation path prefixes,
including use of a directory.  File extensions are still up to
individual log writers to add themselves during the actual rotation.

These new also allow for some simplication to the default
ASCII postprocessor function: it eliminates the need for it doing an
extra/awkward rename() operation that only changes the timestamp format.

This also teaches the supervisor framework to use these new options
to rotate ascii logs into a log-queue/ directory with a specific
file name format (intended for an external archiver process to
monitor separately).
This commit is contained in:
Jon Siwek 2020-06-27 22:43:37 -07:00
parent 6e67a40d24
commit a06ef66edc
21 changed files with 510 additions and 118 deletions
Download patch file Download diff file Expand all files Collapse all files

16
testing/btest/supervisor/config-cluster-leftover-log-archival.zeek
View file

@ -4,16 +4,14 @@
# Test default leftover log rotation/archival behavior
# @TEST-EXEC: btest-bg-run zeek zeek -j -b %INPUT
# @TEST-EXEC: btest-bg-wait 45
# @TEST-EXEC: cp zeek/logger-1/test*.log test.default.log
# @TEST-EXEC: cp zeek/logger-1/log-queue/test*.log test.default.log
# @TEST-EXEC: btest-diff test.default.log
# @TEST-EXEC: rm -rf ./zeek
# Test leftover log rotation/archival behavior with custom postprocessor func
# @TEST-EXEC: btest-bg-run zeek zeek -j -b %INPUT use_custom_postproc=T
# @TEST-EXEC: btest-bg-wait 45
# @TEST-EXEC: cp zeek/logger-1/test*.log test.postproc.log
# @TEST-EXEC: cp zeek/logger-1/log-queue/test*.log test.postproc.log
# @TEST-EXEC: btest-diff test.postproc.log
# @TEST-EXEC: btest-diff zeek/logger-1/postproc.out
# @TEST-EXEC: rm -rf ./zeek
@ -37,17 +35,13 @@ export {
}
module GLOBAL;
module LogAscii;
export {
function my_rotation_postprocessor(info: Log::RotationInfo) : bool
{
local f = open("postproc.out");
print f, "running my rotation postprocessor";
close(f);
return LogAscii::default_rotation_postprocessor_func(info);
return T;
}
}
module GLOBAL;
event zeek_init()
{
@ -56,7 +50,7 @@ event zeek_init()
if ( use_custom_postproc )
{
local df = Log::get_filter(Test::LOG, "default");
df$postprocessor = LogAscii::my_rotation_postprocessor;
df$postprocessor = my_rotation_postprocessor;
Log::add_filter(Test::LOG, df);
}
@ -86,7 +80,7 @@ event zeek_init()
print sf, ".log";
if ( use_custom_postproc )
print sf, "LogAscii::my_rotation_postprocessor";
print sf, "my_rotation_postprocessor";
else
print sf, "";

102
testing/btest/supervisor/config-cluster-log-archival.zeek Normal file
View file

@ -0,0 +1,102 @@
# @TEST-PORT: SUPERVISOR_PORT
# @TEST-PORT: LOGGER_PORT
# Test default log rotation/archival behavior (rotate into log-queue dir)
# @TEST-EXEC: btest-bg-run zeek zeek -j -b %INPUT
# @TEST-EXEC: btest-bg-wait 45
# @TEST-EXEC: cp zeek/logger-1/log-queue/test*.log test.default.log
# @TEST-EXEC: btest-diff test.default.log
# @TEST-EXEC: rm -rf ./zeek
# Test rotation/archival behavior with in-flight compression
# @TEST-EXEC: btest-bg-run zeek zeek -j -b LogAscii::gzip_level=1 %INPUT
# @TEST-EXEC: btest-bg-wait 45
# @TEST-EXEC: gunzip -c zeek/logger-1/log-queue/test*.log.gz > test.zip-in-flight.log
# @TEST-EXEC: btest-diff test.zip-in-flight.log
# @TEST-EXEC: rm -rf ./zeek
# Test rotation/archival behavior with in-flight compression + custom file extension
# @TEST-EXEC: btest-bg-run zeek zeek -j -b LogAscii::gzip_level=1 LogAscii::gzip_file_extension="mygz" %INPUT
# @TEST-EXEC: btest-bg-wait 45
# @TEST-EXEC: cp zeek/logger-1/log-queue/test*.log.mygz test.log.gz
# @TEST-EXEC: gunzip -c test.log.gz > test.zip-in-flight-custom-ext.log
# @TEST-EXEC: btest-diff test.zip-in-flight-custom-ext.log
# @TEST-EXEC: rm -rf ./zeek
# Test rotation/archival behavior with a custom rotation dir
# @TEST-EXEC: btest-bg-run zeek zeek -j -b %INPUT Log::default_rotation_dir=my-logs
# @TEST-EXEC: btest-bg-wait 45
# @TEST-EXEC: cp zeek/logger-1/my-logs/test*.log test.custom-dir.log
# @TEST-EXEC: btest-diff test.custom-dir.log
# @TEST-EXEC: rm -rf ./zeek
@load base/frameworks/cluster
# JSON for log file brevity.
redef LogAscii::use_json=T;
global topic = "test-topic";
module Test;
export {
redef enum Log::ID += { LOG };
type Log: record {
s: string;
} &log;
}
module GLOBAL;
event pong()
{
terminate();
}
event ping()
{
Log::write(Test::LOG, [$s="test"]);
Broker::publish(topic, pong);
}
event zeek_init()
{
Log::create_stream(Test::LOG, [$columns=Test::Log]);
if ( Supervisor::is_supervisor() )
{
Broker::subscribe(topic);
Broker::listen("127.0.0.1", to_port(getenv("SUPERVISOR_PORT")));
Broker::peer("127.0.0.1", to_port(getenv("LOGGER_PORT")));
local cluster: table[string] of Supervisor::ClusterEndpoint;
cluster["logger-1"] = [$role=Supervisor::LOGGER, $host=127.0.0.1,
$p=to_port(getenv("LOGGER_PORT"))];
for ( n, ep in cluster )
{
local sn = Supervisor::NodeConfig($name = n);
sn$cluster = cluster;
sn$directory = n;
local res = Supervisor::create(sn);
if ( res != "" )
print fmt("failed to create node %s: %s", n, res);
}
}
else
{
Broker::subscribe(topic);
Broker::peer("127.0.0.1", to_port(getenv("SUPERVISOR_PORT")));
}
}
event Broker::peer_added(endpoint: Broker::EndpointInfo, msg: string)
{
if ( Supervisor::is_supervisor() )
Broker::publish(topic, ping);
}
event Broker::peer_lost(endpoint: Broker::EndpointInfo, msg: string)
{
}