Converting log writers and input readers to plugins.

This commit is contained in:
Robin Sommer 2014-07-12 19:09:46 -07:00
parent 6d9e261384
commit f4cbcb9b03
109 changed files with 1177 additions and 495 deletions

View file

@ -0,0 +1,22 @@
include(BroSubdir)
include_directories(BEFORE
${CMAKE_CURRENT_SOURCE_DIR}
${CMAKE_CURRENT_BINARY_DIR}
)
add_subdirectory(writers)
set(logging_SRCS
Component.cc
Manager.cc
WriterBackend.cc
WriterFrontend.cc
Tag.cc
)
bif_target(logging.bif)
bro_add_subdir_library(logging ${logging_SRCS} ${BIF_OUTPUT_CC})

29
src/logging/Component.cc Normal file
View file

@ -0,0 +1,29 @@
// See the file "COPYING" in the main distribution directory for copyright.
#include "Component.h"
#include "Manager.h"
#include "../Desc.h"
#include "../util.h"
using namespace logging;
Component::Component(const std::string& name, factory_callback arg_factory)
: plugin::Component(plugin::component::WRITER, name)
{
factory = arg_factory;
log_mgr->RegisterComponent(this, "WRITER_");
}
Component::~Component()
{
}
void Component::DoDescribe(ODesc* d) const
{
d->Add("Log::WRITER_");
d->Add(CanonicalName());
}

59
src/logging/Component.h Normal file
View file

@ -0,0 +1,59 @@
// See the file "COPYING" in the main distribution directory for copyright.
#ifndef LOGGING_COMPONENT_H
#define LOGGING_COMPONENT_H
#include "Tag.h"
#include "plugin/Component.h"
#include "plugin/TaggedComponent.h"
namespace logging {
class WriterFrontend;
class WriterBackend;
/**
* Component description for plugins providing log writers.
*/
class Component : public plugin::Component,
public plugin::TaggedComponent<logging::Tag> {
public:
typedef WriterBackend* (*factory_callback)(WriterFrontend* frontend);
/**
* Constructor.
*
* @param name The name of the provided writer. This name is used
* across the system to identify the writer.
*
* @param factory A factory function to instantiate instances of the
* writers's class, which must be derived directly or indirectly from
* logging::WriterBackend. This is typically a static \c Instatiate()
* method inside the class that just allocates and returns a new
* instance.
*/
Component(const std::string& name, factory_callback factory);
/**
* Destructor.
*/
~Component();
/**
* Returns the writer's factory function.
*/
factory_callback Factory() const { return factory; }
protected:
/**
* Overriden from plugin::Component.
*/
virtual void DoDescribe(ODesc* d) const;
private:
factory_callback factory;
};
}
#endif

View file

@ -14,48 +14,10 @@
#include "Manager.h"
#include "WriterFrontend.h"
#include "WriterBackend.h"
#include "writers/Ascii.h"
#include "writers/None.h"
#ifdef USE_ELASTICSEARCH
#include "writers/ElasticSearch.h"
#endif
#ifdef USE_DATASERIES
#include "writers/DataSeries.h"
#endif
#include "writers/SQLite.h"
#include "logging.bif.h"
using namespace logging;
// Structure describing a log writer type.
struct WriterDefinition {
bro_int_t type; // The type.
const char *name; // Descriptive name for error messages.
bool (*init)(); // An optional one-time initialization function.
WriterBackend* (*factory)(WriterFrontend* frontend); // A factory function creating instances.
};
// Static table defining all availabel log writers.
WriterDefinition log_writers[] = {
{ BifEnum::Log::WRITER_NONE, "None", 0, writer::None::Instantiate },
{ BifEnum::Log::WRITER_ASCII, "Ascii", 0, writer::Ascii::Instantiate },
{ BifEnum::Log::WRITER_SQLITE, "SQLite", 0, writer::SQLite::Instantiate },
#ifdef USE_ELASTICSEARCH
{ BifEnum::Log::WRITER_ELASTICSEARCH, "ElasticSearch", 0, writer::ElasticSearch::Instantiate },
#endif
#ifdef USE_DATASERIES
{ BifEnum::Log::WRITER_DATASERIES, "DataSeries", 0, writer::DataSeries::Instantiate },
#endif
// End marker, don't touch.
{ BifEnum::Log::WRITER_DEFAULT, "None", 0, (WriterBackend* (*)(WriterFrontend* frontend))0 }
};
struct Manager::Filter {
string name;
EnumVal* id;
@ -142,6 +104,7 @@ Manager::Stream::~Stream()
}
Manager::Manager()
: plugin::ComponentManager<logging::Tag, logging::Component>("Log", "Writer")
{
rotations_pending = 0;
}
@ -152,64 +115,17 @@ Manager::~Manager()
delete *s;
}
list<string> Manager::SupportedFormats()
WriterBackend* Manager::CreateBackend(WriterFrontend* frontend, EnumVal* tag)
{
list<string> formats;
Component* c = Lookup(tag);
for ( WriterDefinition* ld = log_writers; ld->type != BifEnum::Log::WRITER_DEFAULT; ++ld )
formats.push_back(ld->name);
return formats;
}
WriterBackend* Manager::CreateBackend(WriterFrontend* frontend, bro_int_t type)
{
WriterDefinition* ld = log_writers;
while ( true )
if ( ! c )
{
if ( ld->type == BifEnum::Log::WRITER_DEFAULT )
{
reporter->Error("unknown writer type requested");
return 0;
}
if ( ld->type != type )
{
// Not the right one.
++ld;
continue;
}
// If the writer has an init function, call it.
if ( ld->init )
{
if ( (*ld->init)() )
// Clear the init function so that we won't
// call it again later.
ld->init = 0;
else
{
// Init failed, disable by deleting factory
// function.
ld->factory = 0;
reporter->Error("initialization of writer %s failed", ld->name);
return 0;
}
}
if ( ! ld->factory )
// Oops, we can't instantiate this guy.
return 0;
// All done.
break;
reporter->Error("unknown writer type requested");
return 0;
}
assert(ld->factory);
WriterBackend* backend = (*ld->factory)(frontend);
WriterBackend* backend = (*c->Factory())(frontend);
assert(backend);
return backend;
@ -1234,7 +1150,7 @@ void Manager::SendAllWritersTo(RemoteSerializer::PeerID peer)
{
WriterFrontend* writer = i->second->writer;
EnumVal writer_val(i->first.first, BifType::Enum::Log::Writer);
EnumVal writer_val(i->first.first, internal_type("Log::Writer")->AsEnumType());
remote_serializer->SendLogCreateWriter(peer, (*s)->id,
&writer_val,
*i->second->info,

View file

@ -6,9 +6,12 @@
#define LOGGING_MANAGER_H
#include "../Val.h"
#include "../Tag.h"
#include "../EventHandler.h"
#include "../RemoteSerializer.h"
#include "../plugin/ComponentManager.h"
#include "Component.h"
#include "WriterBackend.h"
class SerializationFormat;
@ -23,7 +26,7 @@ class RotationFinishedMessage;
/**
* Singleton class for managing log streams.
*/
class Manager {
class Manager : public plugin::ComponentManager<Tag, Component> {
public:
/**
* Constructor.
@ -154,11 +157,6 @@ public:
*/
void Terminate();
/**
* Returns a list of supported output formats.
*/
static list<string> SupportedFormats();
protected:
friend class WriterFrontend;
friend class RotationFinishedMessage;
@ -168,7 +166,7 @@ protected:
// Instantiates a new WriterBackend of the given type (note that
// doing so creates a new thread!).
WriterBackend* CreateBackend(WriterFrontend* frontend, bro_int_t type);
WriterBackend* CreateBackend(WriterFrontend* frontend, EnumVal* tag);
//// Function also used by the RemoteSerializer.

22
src/logging/Tag.cc Normal file
View file

@ -0,0 +1,22 @@
// See the file "COPYING" in the main distribution directory for copyright.
#include "Tag.h"
#include "Manager.h"
logging::Tag logging::Tag::Error;
logging::Tag::Tag(type_t type, subtype_t subtype)
: ::Tag(log_mgr->GetTagEnumType(), type, subtype)
{
}
logging::Tag& logging::Tag::operator=(const logging::Tag& other)
{
::Tag::operator=(other);
return *this;
}
EnumVal* logging::Tag::AsEnumVal() const
{
return ::Tag::AsEnumVal(log_mgr->GetTagEnumType());
}

116
src/logging/Tag.h Normal file
View file

@ -0,0 +1,116 @@
// See the file "COPYING" in the main distribution directory for copyright.
#ifndef LOGGING_TAG_H
#define LOGGING_TAG_H
#include "config.h"
#include "util.h"
#include "../Tag.h"
#include "plugin/TaggedComponent.h"
#include "plugin/ComponentManager.h"
class EnumVal;
namespace logging {
class Manager;
class Component;
/**
* Class to identify a writer type.
*
* The script-layer analogue is Log::Writer.
*/
class Tag : public ::Tag {
public:
/*
* Copy constructor.
*/
Tag(const Tag& other) : ::Tag(other) {}
/**
* Default constructor. This initializes the tag with an error value
* that will make \c operator \c bool return false.
*/
Tag() : ::Tag() {}
/**
* Destructor.
*/
~Tag() {}
/**
* Returns false if the tag represents an error value rather than a
* legal writer type.
* TODO: make this conversion operator "explicit" (C++11) or use a
* "safe bool" idiom (not necessary if "explicit" is available),
* otherwise this may allow nonsense/undesired comparison operations.
*/
operator bool() const { return *this != Tag(); }
/**
* Assignment operator.
*/
Tag& operator=(const Tag& other);
/**
* Compares two tags for equality.
*/
bool operator==(const Tag& other) const
{
return ::Tag::operator==(other);
}
/**
* Compares two tags for inequality.
*/
bool operator!=(const Tag& other) const
{
return ::Tag::operator!=(other);
}
/**
* Compares two tags for less-than relationship.
*/
bool operator<(const Tag& other) const
{
return ::Tag::operator<(other);
}
/**
* Returns the \c Log::Writer enum that corresponds to this tag.
* The returned value does not have its ref-count increased.
*
* @param etype the script-layer enum type associated with the tag.
*/
EnumVal* AsEnumVal() const;
static Tag Error;
protected:
friend class plugin::ComponentManager<Tag, Component>;
friend class plugin::TaggedComponent<Tag>;
/**
* Constructor.
*
* @param type The main type. Note that the \a logging::Manager
* manages the value space internally, so noone else should assign
* any main types.
*
* @param subtype The sub type, which is left to an writer for
* interpretation. By default it's set to zero.
*/
Tag(type_t type, subtype_t subtype = 0);
/**
* Constructor.
*
* @param val An enum value of script type \c Log::Writer.
*/
Tag(EnumVal* val) : ::Tag(val) {}
};
}
#endif

View file

@ -7,6 +7,8 @@
#include "threading/MsgThread.h"
#include "Component.h"
class RemoteSerializer;
namespace logging {

View file

@ -120,7 +120,7 @@ WriterFrontend::WriterFrontend(const WriterBackend::WriterInfo& arg_info, EnumVa
if ( local )
{
backend = log_mgr->CreateBackend(this, writer->AsEnum());
backend = log_mgr->CreateBackend(this, writer);
if ( backend )
backend->Start();

65
src/logging/logging.bif Normal file
View file

@ -0,0 +1,65 @@
##! Internal functions and types used by the logging framework.
module Log;
%%{
#include "logging/Manager.h"
%%}
type Filter: record;
type Stream: record;
type RotationInfo: record;
function Log::__create_stream%(id: Log::ID, stream: Log::Stream%) : bool
%{
bool result = log_mgr->CreateStream(id->AsEnumVal(), stream->AsRecordVal());
return new Val(result, TYPE_BOOL);
%}
function Log::__remove_stream%(id: Log::ID%) : bool
%{
bool result = log_mgr->RemoveStream(id->AsEnumVal());
return new Val(result, TYPE_BOOL);
%}
function Log::__enable_stream%(id: Log::ID%) : bool
%{
bool result = log_mgr->EnableStream(id->AsEnumVal());
return new Val(result, TYPE_BOOL);
%}
function Log::__disable_stream%(id: Log::ID%) : bool
%{
bool result = log_mgr->DisableStream(id->AsEnumVal());
return new Val(result, TYPE_BOOL);
%}
function Log::__add_filter%(id: Log::ID, filter: Log::Filter%) : bool
%{
bool result = log_mgr->AddFilter(id->AsEnumVal(), filter->AsRecordVal());
return new Val(result, TYPE_BOOL);
%}
function Log::__remove_filter%(id: Log::ID, name: string%) : bool
%{
bool result = log_mgr->RemoveFilter(id->AsEnumVal(), name);
return new Val(result, TYPE_BOOL);
%}
function Log::__write%(id: Log::ID, columns: any%) : bool
%{
bool result = log_mgr->Write(id->AsEnumVal(), columns->AsRecordVal());
return new Val(result, TYPE_BOOL);
%}
function Log::__set_buf%(id: Log::ID, buffered: bool%): bool
%{
bool result = log_mgr->SetBuf(id->AsEnumVal(), buffered);
return new Val(result, TYPE_BOOL);
%}
function Log::__flush%(id: Log::ID%): bool
%{
bool result = log_mgr->Flush(id->AsEnumVal());
return new Val(result, TYPE_BOOL);
%}

View file

@ -0,0 +1,6 @@
add_subdirectory(ascii)
add_subdirectory(dataseries)
add_subdirectory(elasticsearch)
add_subdirectory(none)
add_subdirectory(sqlite)

View file

@ -5,10 +5,10 @@
#include <fcntl.h>
#include <unistd.h>
#include "NetVar.h"
#include "threading/SerialTypes.h"
#include "Ascii.h"
#include "ascii.bif.h"
using namespace logging::writer;
using namespace threading;

View file

@ -5,7 +5,7 @@
#ifndef LOGGING_WRITER_ASCII_H
#define LOGGING_WRITER_ASCII_H
#include "../WriterBackend.h"
#include "logging/WriterBackend.h"
#include "threading/formatters/Ascii.h"
#include "threading/formatters/JSON.h"
@ -16,9 +16,10 @@ public:
Ascii(WriterFrontend* frontend);
~Ascii();
static string LogExt();
static WriterBackend* Instantiate(WriterFrontend* frontend)
{ return new Ascii(frontend); }
static string LogExt();
protected:
virtual bool DoInit(const WriterInfo& info, int num_fields,

View file

@ -0,0 +1,9 @@
include(BroPlugin)
include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR})
bro_plugin_begin(Bro AsciiWriter)
bro_plugin_cc(Ascii.cc Plugin.cc)
bro_plugin_bif(ascii.bif)
bro_plugin_end()

View file

@ -0,0 +1,25 @@
// See the file in the main distribution directory for copyright.
#include "plugin/Plugin.h"
#include "Ascii.h"
namespace plugin {
namespace Bro_AsciiWriter {
class Plugin : public plugin::Plugin {
public:
plugin::Configuration Configure()
{
AddComponent(new ::logging::Component("Ascii", ::logging::writer::Ascii::Instantiate));
plugin::Configuration config;
config.name = "Bro::AsciiWriter";
config.description = "ASCII log writer";
return config;
}
} plugin;
}
}

View file

@ -0,0 +1,14 @@
# Options for the ASCII writer.
module LogAscii;
const output_to_stdout: bool;
const include_meta: bool;
const meta_prefix: string;
const separator: string;
const set_separator: string;
const empty_field: string;
const unset_field: string;
const use_json: bool;
const json_timestamps: JSON::TimestampFormat;

View file

@ -0,0 +1,19 @@
include(BroPlugin)
find_package(Lintel)
find_package(DataSeries)
find_package(LibXML2)
if (NOT DISABLE_DATASERIES AND
LINTEL_FOUND AND DATASERIES_FOUND AND LIBXML2_FOUND)
include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR})
bro_plugin_begin(Bro DataSeriesWriter)
bro_plugin_cc(DataSeries.cc Plugin.cc)
bro_plugin_bif(dataseries.bif)
bro_plugin_end()
endif()

View file

@ -2,8 +2,6 @@
#include "config.h"
#ifdef USE_DATASERIES
#include <map>
#include <string>
#include <errno.h>
@ -14,6 +12,7 @@
#include "threading/SerialTypes.h"
#include "DataSeries.h"
#include "dataseries.bif.h"
using namespace logging;
using namespace writer;
@ -458,5 +457,3 @@ bool DataSeries::DoHeartbeat(double network_time, double current_time)
{
return true;
}
#endif /* USE_DATASERIES */

View file

@ -11,8 +11,7 @@
#include <DataSeries/DataSeriesModule.hpp>
#include <DataSeries/GeneralField.hpp>
#include "../WriterBackend.h"
#include "threading/formatters/Ascii.h"
#include "logging/WriterBackend.h"
namespace logging { namespace writer {

View file

@ -0,0 +1,25 @@
// See the file in the main distribution directory for copyright.
#include "plugin/Plugin.h"
#include "DataSeries.h"
namespace plugin {
namespace Bro_DataSeriesWriter {
class Plugin : public plugin::Plugin {
public:
plugin::Configuration Configure()
{
AddComponent(new ::logging::Component("DataSeries", ::logging::writer::DataSeries::Instantiate));
plugin::Configuration config;
config.name = "Bro::DataSeriesWriter";
config.description = "DataSeries log writer";
return config;
}
} plugin;
}
}

View file

@ -0,0 +1,10 @@
# Options for the DataSeries writer.
module LogDataSeries;
const compression: string;
const extent_size: count;
const dump_schema: bool;
const use_integer_for_time: bool;
const num_threads: count;

View file

@ -0,0 +1,15 @@
include(BroPlugin)
find_package(LibCURL)
if (NOT DISABLE_ELASTICSEARCH AND LIBCURL_FOUND)
include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR})
bro_plugin_begin(Bro ElasticSearchWriter)
bro_plugin_cc(ElasticSearch.cc Plugin.cc)
bro_plugin_bif(elasticsearch.bif)
bro_plugin_link_library(${LibCURL_LIBRARIES})
bro_plugin_end()
endif()

View file

@ -6,21 +6,18 @@
#include "config.h"
#ifdef USE_ELASTICSEARCH
#include "util.h" // Needs to come first for stdint.h
#include <string>
#include <errno.h>
#include "BroString.h"
#include "NetVar.h"
#include "threading/SerialTypes.h"
#include <curl/curl.h>
#include <curl/easy.h>
#include "BroString.h"
#include "threading/SerialTypes.h"
#include "ElasticSearch.h"
#include "elasticsearch.bif.h"
using namespace logging;
using namespace writer;
@ -291,5 +288,3 @@ bool ElasticSearch::HTTPSend(CURL *handle)
// The "successful" return happens above
return false;
}
#endif

View file

@ -9,8 +9,9 @@
#define LOGGING_WRITER_ELASTICSEARCH_H
#include <curl/curl.h>
#include "logging/WriterBackend.h"
#include "threading/formatters/JSON.h"
#include "../WriterBackend.h"
namespace logging { namespace writer {
@ -19,9 +20,10 @@ public:
ElasticSearch(WriterFrontend* frontend);
~ElasticSearch();
static string LogExt();
static WriterBackend* Instantiate(WriterFrontend* frontend)
{ return new ElasticSearch(frontend); }
static string LogExt();
protected:
// Overidden from WriterBackend.

View file

@ -0,0 +1,37 @@
// See the file in the main distribution directory for copyright.
#include <curl/curl.h>
#include "plugin/Plugin.h"
#include "ElasticSearch.h"
namespace plugin {
namespace Bro_ElasticSearchWriter {
class Plugin : public plugin::Plugin {
public:
plugin::Configuration Configure()
{
AddComponent(new ::logging::Component("ElasticSearch", ::logging::writer::ElasticSearch::Instantiate));
plugin::Configuration config;
config.name = "Bro::ElasticSearchWriter";
config.description = "ElasticSearch log writer";
return config;
}
virtual void InitPreScript()
{
curl_global_init(CURL_GLOBAL_ALL);
}
virtual void Done()
{
curl_global_cleanup();
}
} plugin;
}
}

View file

@ -0,0 +1,14 @@
# Options for the ElasticSearch writer.
module LogElasticSearch;
const cluster_name: string;
const server_host: string;
const server_port: count;
const index_prefix: string;
const type_prefix: string;
const transfer_timeout: interval;
const max_batch_size: count;
const max_batch_interval: interval;
const max_byte_size: count;

View file

@ -0,0 +1,9 @@
include(BroPlugin)
include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR})
bro_plugin_begin(Bro NoneWriter)
bro_plugin_cc(None.cc Plugin.cc)
bro_plugin_bif(none.bif)
bro_plugin_end()

View file

@ -2,7 +2,7 @@
#include <algorithm>
#include "None.h"
#include "NetVar.h"
#include "none.bif.h"
using namespace logging;
using namespace writer;

View file

@ -5,7 +5,7 @@
#ifndef LOGGING_WRITER_NONE_H
#define LOGGING_WRITER_NONE_H
#include "../WriterBackend.h"
#include "logging/WriterBackend.h"
namespace logging { namespace writer {

View file

@ -0,0 +1,25 @@
// See the file in the main distribution directory for copyright.
#include "plugin/Plugin.h"
#include "None.h"
namespace plugin {
namespace Bro_NoneWriter {
class Plugin : public plugin::Plugin {
public:
plugin::Configuration Configure()
{
AddComponent(new ::logging::Component("None", ::logging::writer::None::Instantiate));
plugin::Configuration config;
config.name = "Bro::NoneWriter";
config.description = "None log writer (primarily for debugging)";
return config;
}
} plugin;
}
}

View file

@ -0,0 +1,6 @@
# Options for the None writer.
module LogNone;
const debug: bool;

View file

@ -0,0 +1,9 @@
include(BroPlugin)
include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR})
bro_plugin_begin(Bro SQLiteWriter)
bro_plugin_cc(SQLite.cc Plugin.cc)
bro_plugin_bif(sqlite.bif)
bro_plugin_end()

View file

@ -0,0 +1,25 @@
// See the file in the main distribution directory for copyright.
#include "plugin/Plugin.h"
#include "SQLite.h"
namespace plugin {
namespace Bro_SQLiteWriter {
class Plugin : public plugin::Plugin {
public:
plugin::Configuration Configure()
{
AddComponent(new ::logging::Component("SQLite", ::logging::writer::SQLite::Instantiate));
plugin::Configuration config;
config.name = "Bro::SQLiteWriter";
config.description = "SQLite log writer";
return config;
}
} plugin;
}
}

View file

@ -6,10 +6,10 @@
#include <errno.h>
#include <vector>
#include "../../NetVar.h"
#include "../../threading/SerialTypes.h"
#include "threading/SerialTypes.h"
#include "SQLite.h"
#include "sqlite.bif.h"
using namespace logging;
using namespace writer;

View file

@ -7,8 +7,7 @@
#include "config.h"
#include "../WriterBackend.h"
#include "logging/WriterBackend.h"
#include "threading/formatters/Ascii.h"
#include "3rdparty/sqlite3.h"

View file

@ -0,0 +1,9 @@
# Options for the SQLite writer
module LogSQLite;
const set_separator: string;
const empty_field: string;
const unset_field: string;