mirror of
https://github.com/zeek/zeek.git
synced 2025-10-05 16:18:19 +00:00
Fix differing log filters of streams from writing to same writer/path.
Since WriterFrontend objects are looked up internally by writer type and path, and they also expect to write consistent field arguments, it could be the case that more than one filter of a given stream attempts to write to the same path (derived either from $path or $path_func fields of the filter) with the same writer type. This won't work, so now WriterFrontend objects are bound to the filter that instantiated them so that we can warn about other filters attempting to write to the conflicting writer/path and the write can be skipped. Remote logs don't appear to suffer the same issue due to pre-filtering. Addresses #842.
This commit is contained in:
Jon Siwek
parent
91522e7836
commit
2fafadd930
6 changed files with 78 additions and 6 deletions
|
|
@ -86,6 +86,7 @@ struct Manager::WriterInfo {
|
||||||
Func* postprocessor;
|
Func* postprocessor;
|
||||||
WriterFrontend* writer;
|
WriterFrontend* writer;
|
||||||
WriterBackend::WriterInfo* info;
|
WriterBackend::WriterInfo* info;
|
||||||
|
string instantiating_filter;
|
||||||
};
|
};
|
||||||
|
|
||||||
struct Manager::Stream {
|
struct Manager::Stream {
|
||||||
|
|
@ -764,8 +765,18 @@ bool Manager::Write(EnumVal* id, RecordVal* columns)
|
||||||
WriterFrontend* writer = 0;
|
WriterFrontend* writer = 0;
|
||||||
|
|
||||||
if ( w != stream->writers.end() )
|
if ( w != stream->writers.end() )
|
||||||
|
{
|
||||||
|
if ( w->second->instantiating_filter != filter->name )
|
||||||
|
{
|
||||||
|
reporter->Warning("Skipping write to filter '%s' on path '%s'"
|
||||||
|
" because filter '%s' has already instantiated the same"
|
||||||
|
" writer type for that path", filter->name.c_str(),
|
||||||
|
filter->path.c_str(), w->second->instantiating_filter.c_str());
|
||||||
|
continue;
|
||||||
|
}
|
||||||
// We know this writer already.
|
// We know this writer already.
|
||||||
writer = w->second->writer;
|
writer = w->second->writer;
|
||||||
|
}
|
||||||
|
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
|
|
@ -800,7 +811,7 @@ bool Manager::Write(EnumVal* id, RecordVal* columns)
|
||||||
|
|
||||||
writer = CreateWriter(stream->id, filter->writer,
|
writer = CreateWriter(stream->id, filter->writer,
|
||||||
info, filter->num_fields,
|
info, filter->num_fields,
|
||||||
arg_fields, filter->local, filter->remote);
|
arg_fields, filter->local, filter->remote, filter->name);
|
||||||
|
|
||||||
if ( ! writer )
|
if ( ! writer )
|
||||||
{
|
{
|
||||||
|
|
@ -999,7 +1010,8 @@ threading::Value** Manager::RecordToFilterVals(Stream* stream, Filter* filter,
|
||||||
}
|
}
|
||||||
|
|
||||||
WriterFrontend* Manager::CreateWriter(EnumVal* id, EnumVal* writer, WriterBackend::WriterInfo* info,
|
WriterFrontend* Manager::CreateWriter(EnumVal* id, EnumVal* writer, WriterBackend::WriterInfo* info,
|
||||||
int num_fields, const threading::Field* const* fields, bool local, bool remote)
|
int num_fields, const threading::Field* const* fields, bool local, bool remote,
|
||||||
|
const string& instantiating_filter)
|
||||||
{
|
{
|
||||||
Stream* stream = FindStream(id);
|
Stream* stream = FindStream(id);
|
||||||
|
|
||||||
|
|
@ -1023,6 +1035,7 @@ WriterFrontend* Manager::CreateWriter(EnumVal* id, EnumVal* writer, WriterBacken
|
||||||
winfo->interval = 0;
|
winfo->interval = 0;
|
||||||
winfo->postprocessor = 0;
|
winfo->postprocessor = 0;
|
||||||
winfo->info = info;
|
winfo->info = info;
|
||||||
|
winfo->instantiating_filter = instantiating_filter;
|
||||||
|
|
||||||
// Search for a corresponding filter for the writer/path pair and use its
|
// Search for a corresponding filter for the writer/path pair and use its
|
||||||
// rotation settings. If no matching filter is found, fall back on
|
// rotation settings. If no matching filter is found, fall back on
|
||||||
|
|
|
||||||
|
|
@ -165,7 +165,7 @@ protected:
|
||||||
// Takes ownership of fields and info.
|
// Takes ownership of fields and info.
|
||||||
WriterFrontend* CreateWriter(EnumVal* id, EnumVal* writer, WriterBackend::WriterInfo* info,
|
WriterFrontend* CreateWriter(EnumVal* id, EnumVal* writer, WriterBackend::WriterInfo* info,
|
||||||
int num_fields, const threading::Field* const* fields,
|
int num_fields, const threading::Field* const* fields,
|
||||||
bool local, bool remote);
|
bool local, bool remote, const string& instantiating_filter="");
|
||||||
|
|
||||||
// Takes ownership of values..
|
// Takes ownership of values..
|
||||||
bool Write(EnumVal* id, EnumVal* writer, string path,
|
bool Write(EnumVal* id, EnumVal* writer, string path,
|
||||||
|
|
|
||||||
|
|
@ -201,7 +201,6 @@ bool WriterBackend::Write(int arg_num_fields, int num_writes, Value*** vals)
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
#ifdef DEBUG
|
|
||||||
// Double-check all the types match.
|
// Double-check all the types match.
|
||||||
for ( int j = 0; j < num_writes; j++ )
|
for ( int j = 0; j < num_writes; j++ )
|
||||||
{
|
{
|
||||||
|
|
@ -209,17 +208,17 @@ bool WriterBackend::Write(int arg_num_fields, int num_writes, Value*** vals)
|
||||||
{
|
{
|
||||||
if ( vals[j][i]->type != fields[i]->type )
|
if ( vals[j][i]->type != fields[i]->type )
|
||||||
{
|
{
|
||||||
|
#ifdef DEBUG
|
||||||
const char* msg = Fmt("Field type doesn't match in WriterBackend::Write() (%d vs. %d)",
|
const char* msg = Fmt("Field type doesn't match in WriterBackend::Write() (%d vs. %d)",
|
||||||
vals[j][i]->type, fields[i]->type);
|
vals[j][i]->type, fields[i]->type);
|
||||||
Debug(DBG_LOGGING, msg);
|
Debug(DBG_LOGGING, msg);
|
||||||
|
#endif
|
||||||
DisableFrontend();
|
DisableFrontend();
|
||||||
DeleteVals(num_writes, vals);
|
DeleteVals(num_writes, vals);
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
#endif
|
|
||||||
|
|
||||||
bool success = true;
|
bool success = true;
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,23 @@
|
||||||
|
#separator \x09
|
||||||
|
#set_separator ,
|
||||||
|
#empty_field (empty)
|
||||||
|
#unset_field -
|
||||||
|
#path http
|
||||||
|
#start 2011-03-18-19-06-08
|
||||||
|
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file
|
||||||
|
#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string file
|
||||||
|
1300475168.784020 j4u32Pc5bif 141.142.220.118 48649 208.80.152.118 80 1 GET bits.wikimedia.org /skins-1.5/monobook/main.css http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified - - - (empty) - - - - - -
|
||||||
|
1300475168.916018 VW0XPVINV8a 141.142.220.118 49997 208.80.152.3 80 1 GET upload.wikimedia.org /wikipedia/commons/6/63/Wikipedia-logo.png http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified - - - (empty) - - - - - -
|
||||||
|
1300475168.916183 3PKsZ2Uye21 141.142.220.118 49996 208.80.152.3 80 1 GET upload.wikimedia.org /wikipedia/commons/thumb/b/bb/Wikipedia_wordmark.svg/174px-Wikipedia_wordmark.svg.png http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified - - - (empty) - - - - - -
|
||||||
|
1300475168.918358 GSxOnSLghOa 141.142.220.118 49998 208.80.152.3 80 1 GET upload.wikimedia.org /wikipedia/commons/b/bd/Bookshelf-40x201_6.png http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified - - - (empty) - - - - - -
|
||||||
|
1300475168.952307 Tw8jXtpTGu6 141.142.220.118 50000 208.80.152.3 80 1 GET upload.wikimedia.org /wikipedia/commons/thumb/8/8a/Wikinews-logo.png/35px-Wikinews-logo.png http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified - - - (empty) - - - - - -
|
||||||
|
1300475168.952296 P654jzLoe3a 141.142.220.118 49999 208.80.152.3 80 1 GET upload.wikimedia.org /wikipedia/commons/4/4a/Wiktionary-logo-en-35px.png http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified - - - (empty) - - - - - -
|
||||||
|
1300475168.954820 0Q4FH8sESw5 141.142.220.118 50001 208.80.152.3 80 1 GET upload.wikimedia.org /wikipedia/commons/thumb/f/fa/Wikiquote-logo.svg/35px-Wikiquote-logo.svg.png http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified - - - (empty) - - - - - -
|
||||||
|
1300475168.962687 i2rO3KD1Syg 141.142.220.118 35642 208.80.152.2 80 1 GET meta.wikimedia.org /images/wikimedia-button.png http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified - - - (empty) - - - - - -
|
||||||
|
1300475168.975934 VW0XPVINV8a 141.142.220.118 49997 208.80.152.3 80 2 GET upload.wikimedia.org /wikipedia/commons/thumb/f/fa/Wikibooks-logo.svg/35px-Wikibooks-logo.svg.png http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified - - - (empty) - - - - - -
|
||||||
|
1300475168.976436 3PKsZ2Uye21 141.142.220.118 49996 208.80.152.3 80 2 GET upload.wikimedia.org /wikipedia/commons/thumb/d/df/Wikispecies-logo.svg/35px-Wikispecies-logo.svg.png http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified - - - (empty) - - - - - -
|
||||||
|
1300475168.979264 GSxOnSLghOa 141.142.220.118 49998 208.80.152.3 80 2 GET upload.wikimedia.org /wikipedia/commons/thumb/4/4c/Wikisource-logo.svg/35px-Wikisource-logo.svg.png http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified - - - (empty) - - - - - -
|
||||||
|
1300475169.014619 Tw8jXtpTGu6 141.142.220.118 50000 208.80.152.3 80 2 GET upload.wikimedia.org /wikipedia/commons/thumb/4/4a/Commons-logo.svg/35px-Commons-logo.svg.png http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified - - - (empty) - - - - - -
|
||||||
|
1300475169.014593 P654jzLoe3a 141.142.220.118 49999 208.80.152.3 80 2 GET upload.wikimedia.org /wikipedia/commons/thumb/9/91/Wikiversity-logo.svg/35px-Wikiversity-logo.svg.png http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified - - - (empty) - - - - - -
|
||||||
|
1300475169.014927 0Q4FH8sESw5 141.142.220.118 50001 208.80.152.3 80 2 GET upload.wikimedia.org /wikipedia/commons/thumb/7/75/Wikimedia_Community_Logo.svg/35px-Wikimedia_Community_Logo.svg.png http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified - - - (empty) - - - - - -
|
||||||
|
#end 2011-03-18-19-06-13
|
||||||
|
|
@ -0,0 +1,23 @@
|
||||||
|
#separator \x09
|
||||||
|
#set_separator ,
|
||||||
|
#empty_field (empty)
|
||||||
|
#unset_field -
|
||||||
|
#path reporter
|
||||||
|
#start 2011-03-18-19-06-08
|
||||||
|
#fields ts level message location
|
||||||
|
#types time enum string string
|
||||||
|
1300475168.843894 Reporter::WARNING Skipping write to filter 'host-only' on path 'http' because filter 'default' has already instantiated the same writer type for that path (empty)
|
||||||
|
1300475168.975800 Reporter::WARNING Skipping write to filter 'host-only' on path 'http' because filter 'default' has already instantiated the same writer type for that path (empty)
|
||||||
|
1300475168.976327 Reporter::WARNING Skipping write to filter 'host-only' on path 'http' because filter 'default' has already instantiated the same writer type for that path (empty)
|
||||||
|
1300475168.979160 Reporter::WARNING Skipping write to filter 'host-only' on path 'http' because filter 'default' has already instantiated the same writer type for that path (empty)
|
||||||
|
1300475169.012666 Reporter::WARNING Skipping write to filter 'host-only' on path 'http' because filter 'default' has already instantiated the same writer type for that path (empty)
|
||||||
|
1300475169.012730 Reporter::WARNING Skipping write to filter 'host-only' on path 'http' because filter 'default' has already instantiated the same writer type for that path (empty)
|
||||||
|
1300475169.014860 Reporter::WARNING Skipping write to filter 'host-only' on path 'http' because filter 'default' has already instantiated the same writer type for that path (empty)
|
||||||
|
1300475169.022665 Reporter::WARNING Skipping write to filter 'host-only' on path 'http' because filter 'default' has already instantiated the same writer type for that path (empty)
|
||||||
|
1300475169.036294 Reporter::WARNING Skipping write to filter 'host-only' on path 'http' because filter 'default' has already instantiated the same writer type for that path (empty)
|
||||||
|
1300475169.036798 Reporter::WARNING Skipping write to filter 'host-only' on path 'http' because filter 'default' has already instantiated the same writer type for that path (empty)
|
||||||
|
1300475169.039923 Reporter::WARNING Skipping write to filter 'host-only' on path 'http' because filter 'default' has already instantiated the same writer type for that path (empty)
|
||||||
|
1300475169.074793 Reporter::WARNING Skipping write to filter 'host-only' on path 'http' because filter 'default' has already instantiated the same writer type for that path (empty)
|
||||||
|
1300475169.074938 Reporter::WARNING Skipping write to filter 'host-only' on path 'http' because filter 'default' has already instantiated the same writer type for that path (empty)
|
||||||
|
1300475169.075065 Reporter::WARNING Skipping write to filter 'host-only' on path 'http' because filter 'default' has already instantiated the same writer type for that path (empty)
|
||||||
|
#end 2011-03-18-19-06-13
|
||||||
|
|
@ -0,0 +1,14 @@
|
||||||
|
# @TEST-EXEC: bro -C -r $TRACES/wikipedia.trace %INPUT
|
||||||
|
# @TEST-EXEC: btest-diff reporter.log
|
||||||
|
# @TEST-EXEC: btest-diff http.log
|
||||||
|
|
||||||
|
@load base/protocols/http
|
||||||
|
|
||||||
|
event bro_init()
|
||||||
|
{
|
||||||
|
# Both the default filter for the http stream and this new one will
|
||||||
|
# attempt to have the same writer write to path "http", which will
|
||||||
|
# be reported as a warning and the write skipped.
|
||||||
|
local filter: Log::Filter = [$name="host-only", $include=set("host")];
|
||||||
|
Log::add_filter(HTTP::LOG, filter);
|
||||||
|
}
|
||||||
Loading…
Add table
Add a link
Reference in a new issue