Mirror/zeek
1
0
Fork 0
mirror of https://github.com/zeek/zeek.git synced 2025-10-09 18:18:19 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

Changing semantics of Broker's remote logging to match old communication framework.

Browse source 
Broker had changed the semantics of remote logging: it sent over the
original Bro record containing the values to be logged, which on the
receiving side would then pass through the logging framework normally,
including triggering filters and events. The old communication system
however special-cases logs: it sends already processed log entries,
just as they go into the log files, and without any receiver-side
filtering etc. This more efficient as it short-cuts the processing
path, and also avoids the more expensive Val serialization. It also
lets the sender determine the specifics of what gets logged (and how).

This commit changes Broker over to now use the same semantics as the
old communication system.

TODOs:
     - The new Broker code doesn't have consistent #ifdefs yet.

     - Right now, when a new log receiver connects, all existing logs
     are broadcasted out again to all current clients. That doesn't so
     any harm, but is unncessary. Need to add a way to send the
     existing logs to just the new client.
This commit is contained in:
Robin Sommer 2017-01-31 21:51:31 -08:00
parent 0dd0bfb5bb
commit a5e9a535a5
25 changed files with 1178 additions and 159 deletions
Download patch file Download diff file Expand all files Collapse all files

65
src/logging/Manager.cc
View file

@ -919,12 +919,6 @@ bool Manager::Write(EnumVal* id, RecordVal* columns)
#endif
}
#ifdef ENABLE_BROKER
if ( stream->enable_remote &&
! broker_mgr->Log(id, columns, stream->columns, stream->remote_flags) )
stream->enable_remote = false;
#endif
Unref(columns);
return true;
@ -1121,23 +1115,46 @@ threading::Value** Manager::RecordToFilterVals(Stream* stream, Filter* filter,
return vals;
}
bool Manager::CreateWriterForRemoteLog(EnumVal* id, EnumVal* writer, WriterBackend::WriterInfo* info,
int num_fields, const threading::Field* const* fields)
{
return CreateWriter(id, writer, info, num_fields, fields, true, false, true);
}
static void delete_info_and_fields(WriterBackend::WriterInfo* info, int num_fields, const threading::Field* const* fields)
{
for ( int i = 0; i < num_fields; i++ )
delete fields[i];
delete [] fields;
delete info;
}
WriterFrontend* Manager::CreateWriter(EnumVal* id, EnumVal* writer, WriterBackend::WriterInfo* info,
int num_fields, const threading::Field* const* fields, bool local, bool remote, bool from_remote,
int num_fields, const threading::Field* const* fields, bool local, bool remote, bool from_remote,
const string& instantiating_filter)
{
WriterFrontend* result = 0;
Stream* stream = FindStream(id);
if ( ! stream )
{
// Don't know this stream.
delete_info_and_fields(info, num_fields, fields);
return 0;
}
Stream::WriterMap::iterator w =
stream->writers.find(Stream::WriterPathPair(writer->AsEnum(), info->path));
if ( w != stream->writers.end() )
{
// If we already have a writer for this. That's fine, we just
// return it.
delete_info_and_fields(info, num_fields, fields);
return w->second->writer;
}
WriterInfo* winfo = new WriterInfo;
winfo->type = writer->Ref()->AsEnumVal();
@ -1190,7 +1207,7 @@ WriterFrontend* Manager::CreateWriter(EnumVal* id, EnumVal* writer, WriterBacken
winfo->info->rotation_interval = winfo->interval;
winfo->info->rotation_base = parse_rotate_base_time(base_time);
winfo->writer = new WriterFrontend(*winfo->info, id, writer, local, remote);
winfo->writer = new WriterFrontend(*winfo->info, id, writer, local, remote, stream->remote_flags);
winfo->writer->Init(num_fields, fields);
InstallRotationTimer(winfo);
@ -1207,8 +1224,8 @@ void Manager::DeleteVals(int num_fields, threading::Value** vals)
delete [] vals;
}
bool Manager::Write(EnumVal* id, EnumVal* writer, string path, int num_fields,
threading::Value** vals)
bool Manager::WriteFromRemote(EnumVal* id, EnumVal* writer, string path, int num_fields,
threading::Value** vals)
{
Stream* stream = FindStream(id);
@ -1280,6 +1297,34 @@ void Manager::SendAllWritersTo(RemoteSerializer::PeerID peer)
}
}
void Manager::SendAllWritersTo(const string& peer)
{
#ifdef ENABLE_BROKER
for ( vector<Stream *>::iterator s = streams.begin(); s != streams.end(); ++s )
{
Stream* stream = (*s);
if ( ! stream )
continue;
for ( Stream::WriterMap::iterator i = stream->writers.begin();
i != stream->writers.end(); i++ )
{
WriterFrontend* writer = i->second->writer;
EnumVal writer_val(i->first.first, internal_type("Log::Writer")->AsEnumType());
broker_mgr->CreateLog((*s)->id,
&writer_val,
*i->second->info,
writer->NumFields(),
writer->Fields(),
stream->remote_flags,
peer);
}
}
#endif
}
bool Manager::SetBuf(EnumVal* id, bool enabled)
{
Stream* stream = FindStream(id);

50
src/logging/Manager.h
View file

@ -129,6 +129,52 @@ public:
*/
bool Write(EnumVal* id, RecordVal* columns);
/**
* Create a new log writer frontend. This is exposed so that the
* communication system can recreated remote log streams locally.
*
* @param stream The enum value corresponding the log stream.
*
* @param writer The enum value corresponding the desired log writer.
*
* @param info A fully initialized object defining the
* characteristics of the backend writer instance. The method takes
* ownership of this.
*
* @param num_fields The number of log fields to write.
*
* @param vals An arry of log fields to write, of size num_fields.
* The method takes ownership of the arry.
*
* @return Returns true if the writer was successfully created.
*/
bool CreateWriterForRemoteLog(EnumVal* id, EnumVal* writer, WriterBackend::WriterInfo* info,
int num_fields, const threading::Field* const* fields);
/**
* Writes out log entries that have already passed through all
* filters, and have raised any events. This is meant called for logs
* received alrready processed from remote.
*
* @param stream The enum value corresponding the log stream.
*
* @param writer The enum value corresponding the desired log writer.
*
* @param path The path of the target log stream to write to.
*
* @param num_fields The number of log values to write.
*
* @param vals An arry of log values to write, of size num_fields.
* The method takes ownership of the arry.
*/
bool WriteFromRemote(EnumVal* stream, EnumVal* writer, string path,
int num_fields, threading::Value** vals);
/**
* Announces all instantiated writers to a given Broker peer.
*/
void SendAllWritersTo(const string& peer);
/**
* Sets log streams buffering state. This adjusts all associated
* writers to the new state.
@ -203,10 +249,6 @@ protected:
int num_fields, const threading::Field* const* fields,
bool local, bool remote, bool from_remote, const string& instantiating_filter="");
// Takes ownership of values..
bool Write(EnumVal* id, EnumVal* writer, string path,
int num_fields, threading::Value** vals);
// Announces all instantiated writers to peer.
void SendAllWritersTo(RemoteSerializer::PeerID peer);

57
src/logging/WriterBackend.cc
View file

@ -119,6 +119,63 @@ bool WriterBackend::WriterInfo::Write(SerializationFormat* fmt) const
return true;
}
broker::data WriterBackend::WriterInfo::ToBroker() const
{
auto bpath = broker::record::field(path);
auto brotation_base = broker::record::field(rotation_base);
auto brotation_interval = broker::record::field(rotation_interval);
auto bnetwork_time = broker::record::field(network_time);
auto t = broker::table();
for ( config_map::const_iterator i = config.begin(); i != config.end(); ++i )
{
auto key = std::string(i->first);
auto value = std::string(i->second);
t.insert(std::make_pair(key, value));
}
auto bconfig = broker::record::field(move(t));
return move(broker::record({bpath, brotation_base, brotation_interval, bnetwork_time, bconfig}));
}
bool WriterBackend::WriterInfo::FromBroker(broker::data d)
{
auto r = broker::get<broker::record>(d);
if ( ! r )
return false;
auto bpath = broker::get<std::string>(*r->get(0));
auto brotation_base = broker::get<double>(*r->get(1));
auto brotation_interval = broker::get<double>(*r->get(2));
auto bnetwork_time = broker::get<double>(*r->get(3));
auto bconfig = broker::get<broker::table>(*r->get(4));
if ( ! (bpath && brotation_base && brotation_interval && bnetwork_time && bconfig) )
return false;
path = copy_string(bpath->c_str());
rotation_base = *brotation_base;
rotation_interval = *brotation_interval;
network_time = *bnetwork_time;
for ( auto i : *bconfig )
{
auto k = broker::get<std::string>(i.first);
auto v = broker::get<std::string>(i.second);
if ( ! (k && v) )
return false;
auto p = std::make_pair(copy_string(k->c_str()), copy_string(v->c_str()));
config.insert(p);
}
return true;
}
WriterBackend::WriterBackend(WriterFrontend* arg_frontend) : MsgThread()
{
num_fields = 0;

11
src/logging/WriterBackend.h
View file

@ -6,6 +6,7 @@
#define LOGGING_WRITERBACKEND_H
#include "threading/MsgThread.h"
#include "broker/Data.h"
#include "Component.h"
@ -110,15 +111,15 @@ public:
}
}
private:
const WriterInfo& operator=(const WriterInfo& other); // Disable.
friend class ::RemoteSerializer;
// Note, these need to be adapted when changing the struct's
// fields. They serialize/deserialize the struct.
bool Read(SerializationFormat* fmt);
bool Write(SerializationFormat* fmt) const;
broker::data ToBroker() const;
bool FromBroker(broker::data d);
private:
const WriterInfo& operator=(const WriterInfo& other); // Disable.
};
/**

27
src/logging/WriterFrontend.cc
View file

@ -2,6 +2,10 @@
#include "Net.h"
#include "threading/SerialTypes.h"
#ifdef ENABLE_BROKER
#include "broker/Manager.h"
#endif
#include "Manager.h"
#include "WriterFrontend.h"
#include "WriterBackend.h"
@ -97,7 +101,7 @@ private:
using namespace logging;
WriterFrontend::WriterFrontend(const WriterBackend::WriterInfo& arg_info, EnumVal* arg_stream, EnumVal* arg_writer, bool arg_local, bool arg_remote)
WriterFrontend::WriterFrontend(const WriterBackend::WriterInfo& arg_info, EnumVal* arg_stream, EnumVal* arg_writer, bool arg_local, bool arg_remote, int arg_remote_flags)
{
stream = arg_stream;
writer = arg_writer;
@ -108,6 +112,7 @@ WriterFrontend::WriterFrontend(const WriterBackend::WriterInfo& arg_info, EnumVa
buf = true;
local = arg_local;
remote = arg_remote;
remote_flags = arg_remote_flags;
write_buffer = 0;
write_buffer_pos = 0;
info = new WriterBackend::WriterInfo(arg_info);
@ -167,12 +172,23 @@ void WriterFrontend::Init(int arg_num_fields, const Field* const * arg_fields)
backend->SendIn(new InitMessage(backend, arg_num_fields, arg_fields));
if ( remote )
{
remote_serializer->SendLogCreateWriter(stream,
writer,
*info,
arg_num_fields,
arg_fields);
#ifdef ENABLE_BROKER
broker_mgr->CreateLog(stream,
writer,
*info,
arg_num_fields,
arg_fields,
remote_flags);
#endif
}
}
void WriterFrontend::Write(int arg_num_fields, Value** vals)
@ -191,12 +207,21 @@ void WriterFrontend::Write(int arg_num_fields, Value** vals)
}
if ( remote )
{
remote_serializer->SendLogWrite(stream,
writer,
info->path,
num_fields,
vals);
broker_mgr->Log(stream,
writer,
info->path,
num_fields,
vals,
remote_flags);
}
if ( ! backend )
{
DeleteVals(arg_num_fields, vals);

8
src/logging/WriterFrontend.h
View file

@ -38,12 +38,15 @@ public:
*
* local: If true, the writer will instantiate a local backend.
*
* remote: If true, the writer will forward all data to remote
* remote: If true, the writer will forward logs to remote
* clients.
*
* remote_flags: Broker flags controlling where remote logs are
* propagated to.
*
* Frontends must only be instantiated by the main thread.
*/
WriterFrontend(const WriterBackend::WriterInfo& info, EnumVal* stream, EnumVal* writer, bool local, bool remote);
WriterFrontend(const WriterBackend::WriterInfo& info, EnumVal* stream, EnumVal* writer, bool local, bool remote, int arg_remote_flags);
/**
* Destructor.
@ -214,6 +217,7 @@ protected:
bool buf; // True if buffering is enabled (default).
bool local; // True if logging locally.
bool remote; // True if loggin remotely.
int remote_flags; // Broker propagation flags.
const char* name; // Descriptive name of the
WriterBackend::WriterInfo* info; // The writer information.