Mirror/zeek
1
0
Fork 0
mirror of https://github.com/zeek/zeek.git synced 2025-10-02 06:38:20 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Converting log writers and input readers to plugins.

Browse source
This commit is contained in:
Robin Sommer 2014-07-12 19:09:46 -07:00
parent 6d9e261384
commit f4cbcb9b03
109 changed files with 1177 additions and 495 deletions
Download patch file Download diff file Expand all files Collapse all files

27
CMakeLists.txt
View file

@ -127,33 +127,6 @@ if (GOOGLEPERFTOOLS_FOUND)
endif ()
endif ()
set(USE_DATASERIES false)
find_package(Lintel)
find_package(DataSeries)
find_package(LibXML2)
if (NOT DISABLE_DATASERIES AND
LINTEL_FOUND AND DATASERIES_FOUND AND LIBXML2_FOUND)
set(USE_DATASERIES true)
include_directories(BEFORE ${Lintel_INCLUDE_DIR})
include_directories(BEFORE ${DataSeries_INCLUDE_DIR})
include_directories(BEFORE ${LibXML2_INCLUDE_DIR})
list(APPEND OPTLIBS ${Lintel_LIBRARIES})
list(APPEND OPTLIBS ${DataSeries_LIBRARIES})
list(APPEND OPTLIBS ${LibXML2_LIBRARIES})
endif()
set(USE_ELASTICSEARCH false)
set(USE_CURL false)
find_package(LibCURL)
if (NOT DISABLE_ELASTICSEARCH AND LIBCURL_FOUND)
set(USE_ELASTICSEARCH true)
set(USE_CURL true)
include_directories(BEFORE ${LibCURL_INCLUDE_DIR})
list(APPEND OPTLIBS ${LibCURL_LIBRARIES})
endif()
if (ENABLE_PERFTOOLS_DEBUG OR ENABLE_PERFTOOLS)
# Just a no op to prevent CMake from complaining about manually-specified
# ENABLE_PERFTOOLS_DEBUG or ENABLE_PERFTOOLS not being used if google

2
cmake

@ -1 +1 @@
Subproject commit d09122d3275baab8a3915fc7f87895ba0377241f
Subproject commit e9cfa2bec050d5b9dc9a67a72a9675867211f2f5

11
scripts/base/frameworks/input/main.bro
View file

@ -4,6 +4,17 @@
module Input;
export {
type Event: enum {
EVENT_NEW = 0,
EVENT_CHANGED = 1,
EVENT_REMOVED = 2,
};
type Mode: enum {
MANUAL = 0,
REREAD = 1,
STREAM = 2
};
## The default input reader used. Defaults to `READER_ASCII`.
const default_reader = READER_ASCII &redef;

10
scripts/base/frameworks/logging/main.bro
View file

@ -5,9 +5,15 @@
module Log;
# Log::ID and Log::Writer are defined in types.bif due to circular dependencies.
export {
## Type that defines a ID unique for each log stream. Scripts creating new log
## streams need to redef this enum to add their own specific log ID. The log ID
## implicitly determines the default name of the generated log file.
type Log::ID: enum {
## Dummy place-holder.
UNKNOWN
};
## If true, local logging is by default enabled for all filters.
const enable_local_logging = T &redef;

2
scripts/base/frameworks/logging/writers/dataseries.bro
View file

@ -57,4 +57,4 @@ function default_rotation_postprocessor_func(info: Log::RotationInfo) : bool
return Log::run_rotation_postprocessor_cmd(info, dst);
}
redef Log::default_rotation_postprocessors += { [Log::WRITER_DATASERIES] = default_rotation_postprocessor_func };
# redef Log::default_rotation_postprocessors += { [Log::WRITER_DATASERIES] = default_rotation_postprocessor_func };

2
scripts/base/frameworks/logging/writers/none.bro
View file

@ -13,5 +13,5 @@ function default_rotation_postprocessor_func(info: Log::RotationInfo) : bool
return T;
}
redef Log::default_rotation_postprocessors += { [Log::WRITER_NONE] = default_rotation_postprocessor_func };
# redef Log::default_rotation_postprocessors += { [Log::WRITER_NONE] = default_rotation_postprocessor_func };

6
scripts/base/init-bare.bro
View file

@ -3363,8 +3363,6 @@ const global_hash_seed: string = "" &redef;
## The maximum is currently 128 bits.
const bits_per_uid: count = 96 &redef;
# Load BiFs defined by plugins.
@load base/bif/plugins
# Load these frameworks here because they use fairly deep integration with
# BiFs and script-land defined types.
@ -3374,3 +3372,7 @@ const bits_per_uid: count = 96 &redef;
@load base/frameworks/files
@load base/bif
# Load BiFs defined by plugins.
@load base/bif/plugins

22
src/CMakeLists.txt
View file

@ -118,8 +118,6 @@ include(BifCl)
set(BIF_SRCS
bro.bif
logging.bif
input.bif
event.bif
const.bif
types.bif
@ -155,6 +153,8 @@ list(APPEND BINPAC_OUTPUTS "${BINPAC_OUTPUT_CC}")
set(bro_SUBDIR_LIBS CACHE INTERNAL "subdir libraries" FORCE)
set(bro_PLUGIN_LIBS CACHE INTERNAL "plugin libraries" FORCE)
add_subdirectory(input)
add_subdirectory(logging)
add_subdirectory(analyzer)
add_subdirectory(file_analysis)
add_subdirectory(probabilistic)
@ -347,24 +347,6 @@ set(bro_SRCS
threading/formatters/Ascii.cc
threading/formatters/JSON.cc
logging/Manager.cc
logging/WriterBackend.cc
logging/WriterFrontend.cc
logging/writers/Ascii.cc
logging/writers/DataSeries.cc
logging/writers/SQLite.cc
logging/writers/ElasticSearch.cc
logging/writers/None.cc
input/Manager.cc
input/ReaderBackend.cc
input/ReaderFrontend.cc
input/readers/Ascii.cc
input/readers/Raw.cc
input/readers/Benchmark.cc
input/readers/Binary.cc
input/readers/SQLite.cc
3rdparty/sqlite3.c
plugin/Component.cc

8
src/Func.cc
View file

@ -277,7 +277,7 @@ int BroFunc::IsPure() const
return 1;
}
Val* BroFunc::HandlePluginResult(Val* plugin_resul, tval_list* args)
Val* BroFunc::HandlePluginResult(Val* plugin_result, val_list* args) const
{
// Helper function factoring out this code from BroFunc:Call() for better
// readability.
@ -600,14 +600,10 @@ void builtin_error(const char* msg, BroObj* arg)
}
#include "bro.bif.func_h"
#include "logging.bif.func_h"
#include "input.bif.func_h"
#include "reporter.bif.func_h"
#include "strings.bif.func_h"
#include "bro.bif.func_def"
#include "logging.bif.func_def"
#include "input.bif.func_def"
#include "reporter.bif.func_def"
#include "strings.bif.func_def"
@ -623,8 +619,6 @@ void init_builtin_funcs()
gap_info = internal_type("gap_info")->AsRecordType();
#include "bro.bif.func_init"
#include "logging.bif.func_init"
#include "input.bif.func_init"
#include "reporter.bif.func_init"
#include "strings.bif.func_init"

2
src/Func.h
View file

@ -100,7 +100,7 @@ public:
protected:
BroFunc() : Func(BRO_FUNC) {}
Stmt* AddInits(Stmt* body, id_list* inits);
Val* HandlePluginResult(Val* plugin_result, val_list* args);
Val* HandlePluginResult(Val* plugin_result, val_list* args) const;
DECLARE_SERIAL(BroFunc);

4
src/NetVar.cc
View file

@ -245,8 +245,6 @@ bro_uint_t bits_per_uid;
#include "const.bif.netvar_def"
#include "types.bif.netvar_def"
#include "event.bif.netvar_def"
#include "logging.bif.netvar_def"
#include "input.bif.netvar_def"
#include "reporter.bif.netvar_def"
void init_event_handlers()
@ -311,8 +309,6 @@ void init_net_var()
{
#include "const.bif.netvar_init"
#include "types.bif.netvar_init"
#include "logging.bif.netvar_init"
#include "input.bif.netvar_init"
#include "reporter.bif.netvar_init"
conn_id = internal_type("conn_id")->AsRecordType();

2
src/NetVar.h
View file

@ -255,8 +255,6 @@ extern void init_net_var();
#include "const.bif.netvar_h"
#include "types.bif.netvar_h"
#include "event.bif.netvar_h"
#include "logging.bif.netvar_h"
#include "input.bif.netvar_h"
#include "reporter.bif.netvar_h"
#endif

9
src/RemoteSerializer.cc
View file

@ -192,6 +192,7 @@
#include "logging/Manager.h"
#include "IPAddr.h"
#include "bro_inet_ntop.h"
#include "logging/logging.bif.h"
extern "C" {
#include "setsignal.h"
@ -2723,8 +2724,8 @@ bool RemoteSerializer::ProcessLogCreateWriter()
fmt.EndRead();
id_val = new EnumVal(id, BifType::Enum::Log::ID);
writer_val = new EnumVal(writer, BifType::Enum::Log::Writer);
id_val = new EnumVal(id, internal_type("Log::ID")->AsEnumType());
writer_val = new EnumVal(writer, internal_type("Log::Writer")->AsEnumType());
if ( ! log_mgr->CreateWriter(id_val, writer_val, info, num_fields, fields,
true, false, true) )
@ -2796,8 +2797,8 @@ bool RemoteSerializer::ProcessLogWrite()
}
}
id_val = new EnumVal(id, BifType::Enum::Log::ID);
writer_val = new EnumVal(writer, BifType::Enum::Log::Writer);
id_val = new EnumVal(id, internal_type("Log::ID")->AsEnumType());
writer_val = new EnumVal(writer, internal_type("Log::Writer")->AsEnumType());
success = log_mgr->Write(id_val, writer_val, path, num_fields, vals);

9
src/Type.cc
View file

@ -1475,6 +1475,12 @@ void EnumType::CheckAndAddName(const string& module_name, const char* name,
broxygen_mgr->Identifier(id);
}
else
{
// We allow double-definitions if matching exactly. This is
// so that we can define an enum both in a *.bif and *.bro to
// avoid cyclic dependencies.
if ( id->Name() != make_full_var_name(module_name.c_str(), name)
|| (id->HasVal() && val != id->ID_Val()->AsEnum()) )
{
Unref(id);
reporter->Error("identifier or enumerator value in enumerated type definition already exists");
@ -1482,6 +1488,9 @@ void EnumType::CheckAndAddName(const string& module_name, const char* name,
return;
}
Unref(id);
}
AddNameInternal(module_name, name, val, is_export);
set<BroType*> types = BroType::GetAliases(GetName());

8
src/Val.cc
View file

@ -2920,7 +2920,15 @@ void EnumVal::ValDescribe(ODesc* d) const
const char* ename = type->AsEnumType()->Lookup(val.int_val);
if ( ! ename )
{
EnumType::enum_name_list l = type->AsEnumType()->Names();
for ( EnumType::enum_name_list::const_iterator iter = l.begin();
iter != l.end(); ++iter )
fprintf(stderr, "%s -> %lld\n", iter->first.c_str(), iter->second);
ename = "<undefined>";
}
d->Add(ename);
}

2
src/analyzer/Component.cc
View file

@ -28,7 +28,7 @@ void Component::DoDescribe(ODesc* d) const
if ( factory )
{
d->Add("ANALYZER_");
d->Add(canon_name);
d->Add(CanonicalName());
d->Add(", ");
}

2
src/analyzer/Manager.cc
View file

@ -60,7 +60,7 @@ bool Manager::ConnIndex::operator<(const ConnIndex& other) const
}
Manager::Manager()
: plugin::ComponentManager<analyzer::Tag, analyzer::Component>("Analyzer")
: plugin::ComponentManager<analyzer::Tag, analyzer::Component>("Analyzer", "Tag")
{
}

4
src/analyzer/Manager.h
View file

@ -45,10 +45,6 @@ namespace analyzer {
* sets up their initial analyzer tree, including adding the right \c PIA,
* respecting well-known ports, and tracking any analyzers specifically
* scheduled for individidual connections.
*
* Note that we keep the public interface of this class free of std::*
* classes. This allows to external analyzer code to potentially use a
* different C++ standard library.
*/
class Manager : public plugin::ComponentManager<Tag, Component> {
public:

2
src/file_analysis/Component.cc
View file

@ -26,6 +26,6 @@ void Component::DoDescribe(ODesc* d) const
if ( factory )
{
d->Add("ANALYZER_");
d->Add(canon_name);
d->Add(CanonicalName());
}
}

2
src/file_analysis/Manager.cc
View file

@ -22,7 +22,7 @@ string Manager::salt;
Manager::Manager()
: plugin::ComponentManager<file_analysis::Tag,
file_analysis::Component>("Files"),
file_analysis::Component>("Files", "Tag"),
id_map(), ignored(), current_file_id(), magic_state()
{
}

22
src/input/CMakeLists.txt Normal file
View file

@ -0,0 +1,22 @@
include(BroSubdir)
include_directories(BEFORE
${CMAKE_CURRENT_SOURCE_DIR}
${CMAKE_CURRENT_BINARY_DIR}
)
add_subdirectory(readers)
set(input_SRCS
Component.cc
Manager.cc
ReaderBackend.cc
ReaderFrontend.cc
Tag.cc
)
bif_target(input.bif)
bro_add_subdir_library(input ${input_SRCS} ${BIF_OUTPUT_CC})

28
src/input/Component.cc Normal file
View file

@ -0,0 +1,28 @@
// See the file "COPYING" in the main distribution directory for copyright.
#include "Component.h"
#include "Manager.h"
#include "../Desc.h"
#include "../util.h"
using namespace input;
Component::Component(const std::string& name, factory_callback arg_factory)
: plugin::Component(plugin::component::WRITER, name)
{
factory = arg_factory;
input_mgr->RegisterComponent(this, "READER_");
}
Component::~Component()
{
}
void Component::DoDescribe(ODesc* d) const
{
d->Add("Input::READER_");
d->Add(CanonicalName());
}

59
src/input/Component.h Normal file
View file

@ -0,0 +1,59 @@
// See the file "COPYING" in the main distribution directory for copyright.
#ifndef INPUT_COMPONENT_H
#define INPUT_COMPONENT_H
#include "Tag.h"
#include "plugin/Component.h"
#include "plugin/TaggedComponent.h"
namespace input {
class ReaderFrontend;
class ReaderBackend;
/**
* Component description for plugins providing log readers.
*/
class Component : public plugin::Component,
public plugin::TaggedComponent<input::Tag> {
public:
typedef ReaderBackend* (*factory_callback)(ReaderFrontend* frontend);
/**
* Constructor.
*
* @param name The name of the provided reader. This name is used
* across the system to identify the reader.
*
* @param factory A factory function to instantiate instances of the
* readers's class, which must be derived directly or indirectly from
* input::ReaderBackend. This is typically a static \c Instatiate()
* method inside the class that just allocates and returns a new
* instance.
*/
Component(const std::string& name, factory_callback factory);
/**
* Destructor.
*/
~Component();
/**
* Returns the reader's factory function.
*/
factory_callback Factory() const { return factory; }
protected:
/**
* Overriden from plugin::Component.
*/
virtual void DoDescribe(ODesc* d) const;
private:
factory_callback factory;
};
}
#endif

73
src/input/Manager.cc
View file

@ -5,11 +5,7 @@
#include "Manager.h"
#include "ReaderFrontend.h"
#include "ReaderBackend.h"
#include "readers/Ascii.h"
#include "readers/Raw.h"
#include "readers/Benchmark.h"
#include "readers/Binary.h"
#include "readers/SQLite.h"
#include "input.bif.h"
#include "Event.h"
#include "EventHandler.h"
@ -24,24 +20,6 @@ using namespace input;
using threading::Value;
using threading::Field;
struct ReaderDefinition {
bro_int_t type; // The reader type.
const char *name; // Descriptive name for error messages.
bool (*init)(); // Optional one-time initializing function.
ReaderBackend* (*factory)(ReaderFrontend* frontend); // Factory function for creating instances.
};
ReaderDefinition input_readers[] = {
{ BifEnum::Input::READER_ASCII, "Ascii", 0, reader::Ascii::Instantiate },
{ BifEnum::Input::READER_RAW, "Raw", 0, reader::Raw::Instantiate },
{ BifEnum::Input::READER_BENCHMARK, "Benchmark", 0, reader::Benchmark::Instantiate },
{ BifEnum::Input::READER_BINARY, "Binary", 0, reader::Binary::Instantiate },
{ BifEnum::Input::READER_SQLITE, "SQLite", 0, reader::SQLite::Instantiate },
// End marker
{ BifEnum::Input::READER_DEFAULT, "None", 0, (ReaderBackend* (*)(ReaderFrontend* frontend))0 }
};
static void delete_value_ptr_array(Value** vals, int num_fields)
{
for ( int i = 0; i < num_fields; ++i )
@ -215,6 +193,7 @@ Manager::AnalysisStream::~AnalysisStream()
}
Manager::Manager()
: plugin::ComponentManager<input::Tag, input::Component>("Input", "Reader")
{
end_of_data = internal_handler("Input::end_of_data");
}
@ -229,55 +208,17 @@ Manager::~Manager()
}
ReaderBackend* Manager::CreateBackend(ReaderFrontend* frontend, bro_int_t type)
ReaderBackend* Manager::CreateBackend(ReaderFrontend* frontend, EnumVal* tag)
{
ReaderDefinition* ir = input_readers;
Component* c = Lookup(tag);
while ( true )
{
if ( ir->type == BifEnum::Input::READER_DEFAULT )
if ( ! c )
{
reporter->Error("The reader that was requested was not found and could not be initialized.");
return 0;
}
if ( ir->type != type )
{
// no, didn't find the right one...
++ir;
continue;
}
// call init function of writer if presnt
if ( ir->init )
{
if ( (*ir->init)() )
{
//clear it to be not called again
ir->init = 0;
}
else {
// ohok. init failed, kill factory for all eternity
ir->factory = 0;
DBG_LOG(DBG_LOGGING, "Failed to init input class %s", ir->name);
return 0;
}
}
if ( ! ir->factory )
// no factory?
return 0;
// all done. break.
break;
}
assert(ir->factory);
ReaderBackend* backend = (*ir->factory)(frontend);
ReaderBackend* backend = (*c->Factory())(frontend);
assert(backend);
return backend;
@ -286,8 +227,6 @@ ReaderBackend* Manager::CreateBackend(ReaderFrontend* frontend, bro_int_t type)
// Create a new input reader object to be used at whomevers leisure lateron.
bool Manager::CreateStream(Stream* info, RecordVal* description)
{
ReaderDefinition* ir = input_readers;
RecordType* rtype = description->Type()->AsRecordType();
if ( ! ( same_type(rtype, BifType::Record::Input::TableDescription, 0)
|| same_type(rtype, BifType::Record::Input::EventDescription, 0)

6
src/input/Manager.h
View file

@ -10,6 +10,8 @@
#include "RemoteSerializer.h"
#include "Val.h"
#include "Component.h"
#include <map>
namespace input {
@ -20,7 +22,7 @@ class ReaderBackend;
/**
* Singleton class for managing input streams.
*/
class Manager {
class Manager : public plugin::ComponentManager<Tag, Component> {
public:
/**
* Constructor.
@ -131,7 +133,7 @@ protected:
// Instantiates a new ReaderBackend of the given type (note that
// doing so creates a new thread!).
ReaderBackend* CreateBackend(ReaderFrontend* frontend, bro_int_t type);
ReaderBackend* CreateBackend(ReaderFrontend* frontend, EnumVal* tag);
// Function called from the ReaderBackend to notify the manager that
// a stream has been removed or a stream has been closed. Used to

2
src/input/ReaderBackend.cc
View file

@ -156,7 +156,7 @@ public:
}
};
using namespace logging;
using namespace input;
ReaderBackend::ReaderBackend(ReaderFrontend* arg_frontend) : MsgThread()
{

2
src/input/ReaderBackend.h
View file

@ -8,6 +8,8 @@
#include "threading/SerialTypes.h"
#include "threading/MsgThread.h"
#include "Component.h"
namespace input {
/**

2
src/input/ReaderFrontend.cc
View file

@ -44,7 +44,7 @@ ReaderFrontend::ReaderFrontend(const ReaderBackend::ReaderInfo& arg_info, EnumVa
const char* t = type->Type()->AsEnumType()->Lookup(type->InternalInt());
name = copy_string(fmt("%s/%s", arg_info.source, t));
backend = input_mgr->CreateBackend(this, type->InternalInt());
backend = input_mgr->CreateBackend(this, type);
assert(backend);
backend->Start();
}

22
src/input/Tag.cc Normal file
View file

@ -0,0 +1,22 @@
// See the file "COPYING" in the main distribution directory for copyright.
#include "Tag.h"
#include "Manager.h"
input::Tag input::Tag::Error;
input::Tag::Tag(type_t type, subtype_t subtype)
: ::Tag(input_mgr->GetTagEnumType(), type, subtype)
{
}
input::Tag& input::Tag::operator=(const input::Tag& other)
{
::Tag::operator=(other);
return *this;
}
EnumVal* input::Tag::AsEnumVal() const
{
return ::Tag::AsEnumVal(input_mgr->GetTagEnumType());
}

116
src/input/Tag.h Normal file
View file

@ -0,0 +1,116 @@
// See the file "COPYING" in the main distribution directory for copyright.
#ifndef INPUT_TAG_H
#define INPUT_TAG_H
#include "config.h"
#include "util.h"
#include "../Tag.h"
#include "plugin/TaggedComponent.h"
#include "plugin/ComponentManager.h"
class EnumVal;
namespace input {
class Manager;
class Component;
/**
* Class to identify a reader type.
*
* The script-layer analogue is Log::Writer.
*/
class Tag : public ::Tag {
public:
/*
* Copy constructor.
*/
Tag(const Tag& other) : ::Tag(other) {}
/**
* Default constructor. This initializes the tag with an error value
* that will make \c operator \c bool return false.
*/
Tag() : ::Tag() {}
/**
* Destructor.
*/
~Tag() {}
/**
* Returns false if the tag represents an error value rather than a
* legal reader type.
* TODO: make this conversion operator "explicit" (C++11) or use a
* "safe bool" idiom (not necessary if "explicit" is available),
* otherwise this may allow nonsense/undesired comparison operations.
*/
operator bool() const { return *this != Tag(); }
/**
* Assignment operator.
*/
Tag& operator=(const Tag& other);
/**
* Compares two tags for equality.
*/
bool operator==(const Tag& other) const
{
return ::Tag::operator==(other);
}
/**
* Compares two tags for inequality.
*/
bool operator!=(const Tag& other) const
{
return ::Tag::operator!=(other);
}
/**
* Compares two tags for less-than relationship.
*/
bool operator<(const Tag& other) const
{
return ::Tag::operator<(other);
}
/**
* Returns the \c Log::Writer enum that corresponds to this tag.
* The returned value does not have its ref-count increased.
*
* @param etype the script-layer enum type associated with the tag.
*/
EnumVal* AsEnumVal() const;
static Tag Error;
protected:
friend class plugin::ComponentManager<Tag, Component>;
friend class plugin::TaggedComponent<Tag>;
/**
* Constructor.
*
* @param type The main type. Note that the \a input::Manager
* manages the value space internally, so noone else should assign
* any main types.
*
* @param subtype The sub type, which is left to an reader for
* interpretation. By default it's set to zero.
*/
Tag(type_t type, subtype_t subtype = 0);
/**
* Constructor.
*
* @param val An enum value of script type \c Log::Writer.
*/
Tag(EnumVal* val) : ::Tag(val) {}
};
}
#endif

34
src/input.bif → src/input/input.bif
View file

@ -4,9 +4,14 @@ module Input;
%%{
#include "input/Manager.h"
#include "NetVar.h"
%%}
enum Event %{
EVENT_NEW = 0,
EVENT_CHANGED = 1,
EVENT_REMOVED = 2,
%}
type TableDescription: record;
type EventDescription: record;
type AnalysisDescription: record;
@ -45,30 +50,3 @@ function Input::__force_update%(id: string%) : bool
const accept_unsupported_types: bool;
# Options for Ascii Reader
module InputAscii;
const separator: string;
const set_separator: string;
const empty_field: string;
const unset_field: string;
module InputRaw;
const record_separator: string;
module InputBenchmark;
const factor: double;
const spread: count;
const autospread: double;
const addfactor: count;
const stopspreadat: count;
const timedspread: double;
module InputBinary;
const chunk_size: count;
module InputSQLite;
const set_separator: string;
const unset_field: string;
const empty_field: string;

6
src/input/readers/CMakeLists.txt Normal file
View file

@ -0,0 +1,6 @@
add_subdirectory(ascii)
add_subdirectory(benchmark)
add_subdirectory(binary)
add_subdirectory(raw)
add_subdirectory(sqlite)

10
src/input/readers/Ascii.cc → src/input/readers/ascii/Ascii.cc
View file

@ -1,18 +1,18 @@
// See the file "COPYING" in the main distribution directory for copyright.
#include "Ascii.h"
#include "NetVar.h"
#include <fstream>
#include <sstream>
#include "../../threading/SerialTypes.h"
#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>
#include <errno.h>
#include "Ascii.h"
#include "ascii.bif.h"
#include "threading/SerialTypes.h"
using namespace input::reader;
using namespace threading;
using threading::Value;

2
src/input/readers/Ascii.h → src/input/readers/ascii/Ascii.h
View file

@ -6,7 +6,7 @@
#include <iostream>
#include <vector>
#include "../ReaderBackend.h"
#include "input/ReaderBackend.h"
#include "threading/formatters/Ascii.h"
namespace input { namespace reader {

9
src/input/readers/ascii/CMakeLists.txt Normal file
View file

@ -0,0 +1,9 @@
include(BroPlugin)
include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR})
bro_plugin_begin(Bro AsciiReader)
bro_plugin_cc(Ascii.cc Plugin.cc)
bro_plugin_bif(ascii.bif)
bro_plugin_end()

24
src/input/readers/ascii/Plugin.cc Normal file
View file

@ -0,0 +1,24 @@
// See the file in the main distribution directory for copyright.
#include "plugin/Plugin.h"
#include "Ascii.h"
namespace plugin {
namespace Bro_AsciiReader {
class Plugin : public plugin::Plugin {
public:
plugin::Configuration Configure()
{
AddComponent(new ::input::Component("Ascii", ::input::reader::Ascii::Instantiate));
plugin::Configuration config;
config.name = "Bro::AsciiReader";
config.description = "ASCII input reader";
return config;
}
} plugin;
}
}

7
src/input/readers/ascii/ascii.bif Normal file
View file

@ -0,0 +1,7 @@
module InputAscii;
const separator: string;
const set_separator: string;
const empty_field: string;
const unset_field: string;

11
src/input/readers/Benchmark.cc → src/input/readers/benchmark/Benchmark.cc
View file

@ -1,16 +1,15 @@
// See the file "COPYING" in the main distribution directory for copyright.
#include "Benchmark.h"
#include "NetVar.h"
#include "../../threading/SerialTypes.h"
#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>
#include <errno.h>
#include "../../threading/Manager.h"
#include "Benchmark.h"
#include "benchmark.bif.h"
#include "threading/SerialTypes.h"
#include "threading/Manager.h"
using namespace input::reader;
using threading::Value;

2
src/input/readers/Benchmark.h → src/input/readers/benchmark/Benchmark.h
View file

@ -3,7 +3,7 @@
#ifndef INPUT_READERS_BENCHMARK_H
#define INPUT_READERS_BENCHMARK_H
#include "../ReaderBackend.h"
#include "input/ReaderBackend.h"
#include "threading/formatters/Ascii.h"
namespace input { namespace reader {

9
src/input/readers/benchmark/CMakeLists.txt Normal file
View file

@ -0,0 +1,9 @@
include(BroPlugin)
include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR})
bro_plugin_begin(Bro BenchmarkReader)
bro_plugin_cc(Benchmark.cc Plugin.cc)
bro_plugin_bif(benchmark.bif)
bro_plugin_end()

24
src/input/readers/benchmark/Plugin.cc Normal file
View file

@ -0,0 +1,24 @@
// See the file in the main distribution directory for copyright.
#include "plugin/Plugin.h"
#include "Benchmark.h"
namespace plugin {
namespace Bro_BenchmarkReader {
class Plugin : public plugin::Plugin {
public:
plugin::Configuration Configure()
{
AddComponent(new ::input::Component("Benchmark", ::input::reader::Benchmark::Instantiate));
plugin::Configuration config;
config.name = "Bro::BenchmarkReader";
config.description = "Benchmark input reader";
return config;
}
} plugin;
}
}

9
src/input/readers/benchmark/benchmark.bif Normal file
View file

@ -0,0 +1,9 @@
module InputBenchmark;
const factor: double;
const spread: count;
const autospread: double;
const addfactor: count;
const stopspreadat: count;
const timedspread: double;

4
src/input/readers/Binary.cc → src/input/readers/binary/Binary.cc
View file

@ -3,9 +3,9 @@
#include <sys/stat.h>
#include "Binary.h"
#include "NetVar.h"
#include "binary.bif.h"
#include "../../threading/SerialTypes.h"
#include "threading/SerialTypes.h"
using namespace input::reader;
using threading::Value;

2
src/input/readers/Binary.h → src/input/readers/binary/Binary.h
View file

@ -5,7 +5,7 @@
#include <fstream>
#include "../ReaderBackend.h"
#include "input/ReaderBackend.h"
namespace input { namespace reader {

9
src/input/readers/binary/CMakeLists.txt Normal file
View file

@ -0,0 +1,9 @@
include(BroPlugin)
include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR})
bro_plugin_begin(Bro BinaryReader)
bro_plugin_cc(Binary.cc Plugin.cc)
bro_plugin_bif(binary.bif)
bro_plugin_end()

24
src/input/readers/binary/Plugin.cc Normal file
View file

@ -0,0 +1,24 @@
// See the file in the main distribution directory for copyright.
#include "plugin/Plugin.h"
#include "Binary.h"
namespace plugin {
namespace Bro_BinaryReader {
class Plugin : public plugin::Plugin {
public:
plugin::Configuration Configure()
{
AddComponent(new ::input::Component("Binary", ::input::reader::Binary::Instantiate));
plugin::Configuration config;
config.name = "Bro::BinaryReader";
config.description = "Binary input reader";
return config;
}
} plugin;
}
}

4
src/input/readers/binary/binary.bif Normal file
View file

@ -0,0 +1,4 @@
module InputBinary;
const chunk_size: count;

9
src/input/readers/raw/CMakeLists.txt Normal file
View file

@ -0,0 +1,9 @@
include(BroPlugin)
include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR})
bro_plugin_begin(Bro RawReader)
bro_plugin_cc(Raw.cc Plugin.cc)
bro_plugin_bif(raw.bif)
bro_plugin_end()

43
src/input/readers/raw/Plugin.cc Normal file
View file

@ -0,0 +1,43 @@
// See the file in the main distribution directory for copyright.
#include "Plugin.h"
namespace plugin { namespace Bro_RawReader { Plugin plugin; } }
using namespace plugin::Bro_RawReader;
Plugin::Plugin()
{
init = false;
}
plugin::Configuration Plugin::Configure()
{
AddComponent(new ::input::Component("Raw", ::input::reader::Raw::Instantiate));
plugin::Configuration config;
config.name = "Bro::RawReader";
config.description = "Raw input reader";
return config;
}
void Plugin::InitPreScript()
{
if ( pthread_mutex_init(&fork_mutex, 0) != 0 )
reporter->FatalError("cannot initialize raw reader's mutex");
init = true;
}
void Plugin::Done()
{
pthread_mutex_destroy(&fork_mutex);
init = false;
}
pthread_mutex_t* Plugin::ForkMutex()
{
assert(init);
return &fork_mutex;
}

30
src/input/readers/raw/Plugin.h Normal file
View file

@ -0,0 +1,30 @@
// See the file in the main distribution directory for copyright.
#include "plugin/Plugin.h"
#include "Raw.h"
namespace plugin {
namespace Bro_RawReader {
class Plugin : public plugin::Plugin {
public:
Plugin();
plugin::Configuration Configure();
virtual void InitPreScript();
virtual void Done();
pthread_mutex_t * ForkMutex();
private:
bool init;
pthread_mutex_t fork_mutex;
};
extern Plugin plugin;
}
}

21
src/input/readers/Raw.cc → src/input/readers/raw/Raw.cc
View file

@ -1,10 +1,5 @@
// See the file "COPYING" in the main distribution directory for copyright.
#include "Raw.h"
#include "NetVar.h"
#include "../../threading/SerialTypes.h"
#include <sys/types.h>
#include <sys/wait.h>
#include <sys/stat.h>
@ -14,6 +9,12 @@
#include <errno.h>
#include <signal.h>
#include "Raw.h"
#include "Plugin.h"
#include "raw.bif.h"
#include "threading/SerialTypes.h"
extern "C" {
#include "setsignal.h"
}
@ -23,12 +24,6 @@ using threading::Value;
using threading::Field;
const int Raw::block_size = 4096; // how big do we expect our chunks of data to be.
pthread_mutex_t Raw::fork_mutex;
bool Raw::ClassInit()
{
return pthread_mutex_init(&fork_mutex, 0) == 0;
}
Raw::Raw(ReaderFrontend *frontend) : ReaderBackend(frontend)
{
@ -109,7 +104,7 @@ bool Raw::SetFDFlags(int fd, int cmd, int flags)
bool Raw::LockForkMutex()
{
int res = pthread_mutex_lock(&fork_mutex);
int res = pthread_mutex_lock(plugin::Bro_RawReader::plugin.ForkMutex());
if ( res == 0 )
return true;
@ -119,7 +114,7 @@ bool Raw::LockForkMutex()
bool Raw::UnlockForkMutex()
{
int res = pthread_mutex_unlock(&fork_mutex);
int res = pthread_mutex_unlock(plugin::Bro_RawReader::plugin.ForkMutex());
if ( res == 0 )
return true;

4
src/input/readers/Raw.h → src/input/readers/raw/Raw.h
View file

@ -6,7 +6,7 @@
#include <vector>
#include <pthread.h>
#include "../ReaderBackend.h"
#include "input/ReaderBackend.h"
namespace input { namespace reader {
@ -21,8 +21,6 @@ public:
static ReaderBackend* Instantiate(ReaderFrontend* frontend) { return new Raw(frontend); }
static bool ClassInit();
protected:
virtual bool DoInit(const ReaderInfo& info, int arg_num_fields, const threading::Field* const* fields);
virtual void DoClose();

4
src/input/readers/raw/raw.bif Normal file
View file

@ -0,0 +1,4 @@
module InputRaw;
const record_separator: string;

9
src/input/readers/sqlite/CMakeLists.txt Normal file
View file

@ -0,0 +1,9 @@
include(BroPlugin)
include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR})
bro_plugin_begin(Bro SQLiteReader)
bro_plugin_cc(SQLite.cc Plugin.cc)
bro_plugin_bif(sqlite.bif)
bro_plugin_end()

24
src/input/readers/sqlite/Plugin.cc Normal file
View file

@ -0,0 +1,24 @@
// See the file in the main distribution directory for copyright.
#include "plugin/Plugin.h"
#include "SQLite.h"
namespace plugin {
namespace Bro_SQLiteReader {
class Plugin : public plugin::Plugin {
public:
plugin::Configuration Configure()
{
AddComponent(new ::input::Component("SQLite", ::input::reader::SQLite::Instantiate));
plugin::Configuration config;
config.name = "Bro::SQLiteReader";
config.description = "SQLite input reader";
return config;
}
} plugin;
}
}

10
src/input/readers/SQLite.cc → src/input/readers/sqlite/SQLite.cc
View file

@ -2,16 +2,18 @@
#include "config.h"
#include "SQLite.h"
#include "NetVar.h"
#include <fstream>
#include <sstream>
#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>
#include "../../threading/SerialTypes.h"
#include "SQLite.h"
#include "sqlite.bif.h"
#include "logging/writers/sqlite/sqlite.bif.h"
#include "logging/writers/ascii/ascii.bif.h"
#include "threading/SerialTypes.h"
using namespace input::reader;
using threading::Value;

3
src/input/readers/SQLite.h → src/input/readers/sqlite/SQLite.h
View file

@ -8,8 +8,7 @@
#include <iostream>
#include <vector>
#include "../ReaderBackend.h"
#include "input/ReaderBackend.h"
#include "threading/formatters/Ascii.h"
#include "3rdparty/sqlite3.h"

6
src/input/readers/sqlite/sqlite.bif Normal file
View file

@ -0,0 +1,6 @@
module InputSQLite;
const set_separator: string;
const unset_field: string;
const empty_field: string;

22
src/logging/CMakeLists.txt Normal file
View file

@ -0,0 +1,22 @@
include(BroSubdir)
include_directories(BEFORE
${CMAKE_CURRENT_SOURCE_DIR}
${CMAKE_CURRENT_BINARY_DIR}
)
add_subdirectory(writers)
set(logging_SRCS
Component.cc
Manager.cc
WriterBackend.cc
WriterFrontend.cc
Tag.cc
)
bif_target(logging.bif)
bro_add_subdir_library(logging ${logging_SRCS} ${BIF_OUTPUT_CC})

29
src/logging/Component.cc Normal file
View file

@ -0,0 +1,29 @@
// See the file "COPYING" in the main distribution directory for copyright.
#include "Component.h"
#include "Manager.h"
#include "../Desc.h"
#include "../util.h"
using namespace logging;
Component::Component(const std::string& name, factory_callback arg_factory)
: plugin::Component(plugin::component::WRITER, name)
{
factory = arg_factory;
log_mgr->RegisterComponent(this, "WRITER_");
}
Component::~Component()
{
}
void Component::DoDescribe(ODesc* d) const
{
d->Add("Log::WRITER_");
d->Add(CanonicalName());
}

59
src/logging/Component.h Normal file
View file

@ -0,0 +1,59 @@
// See the file "COPYING" in the main distribution directory for copyright.
#ifndef LOGGING_COMPONENT_H
#define LOGGING_COMPONENT_H
#include "Tag.h"
#include "plugin/Component.h"
#include "plugin/TaggedComponent.h"
namespace logging {
class WriterFrontend;
class WriterBackend;
/**
* Component description for plugins providing log writers.
*/
class Component : public plugin::Component,
public plugin::TaggedComponent<logging::Tag> {
public:
typedef WriterBackend* (*factory_callback)(WriterFrontend* frontend);
/**
* Constructor.
*
* @param name The name of the provided writer. This name is used
* across the system to identify the writer.
*
* @param factory A factory function to instantiate instances of the
* writers's class, which must be derived directly or indirectly from
* logging::WriterBackend. This is typically a static \c Instatiate()
* method inside the class that just allocates and returns a new
* instance.
*/
Component(const std::string& name, factory_callback factory);
/**
* Destructor.
*/
~Component();
/**
* Returns the writer's factory function.
*/
factory_callback Factory() const { return factory; }
protected:
/**
* Overriden from plugin::Component.
*/
virtual void DoDescribe(ODesc* d) const;
private:
factory_callback factory;
};
}
#endif

98
src/logging/Manager.cc
View file

@ -14,48 +14,10 @@
#include "Manager.h"
#include "WriterFrontend.h"
#include "WriterBackend.h"
#include "writers/Ascii.h"
#include "writers/None.h"
#ifdef USE_ELASTICSEARCH
#include "writers/ElasticSearch.h"
#endif
#ifdef USE_DATASERIES
#include "writers/DataSeries.h"
#endif
#include "writers/SQLite.h"
#include "logging.bif.h"
using namespace logging;
// Structure describing a log writer type.
struct WriterDefinition {
bro_int_t type; // The type.
const char *name; // Descriptive name for error messages.
bool (*init)(); // An optional one-time initialization function.
WriterBackend* (*factory)(WriterFrontend* frontend); // A factory function creating instances.
};
// Static table defining all availabel log writers.
WriterDefinition log_writers[] = {
{ BifEnum::Log::WRITER_NONE, "None", 0, writer::None::Instantiate },
{ BifEnum::Log::WRITER_ASCII, "Ascii", 0, writer::Ascii::Instantiate },
{ BifEnum::Log::WRITER_SQLITE, "SQLite", 0, writer::SQLite::Instantiate },
#ifdef USE_ELASTICSEARCH
{ BifEnum::Log::WRITER_ELASTICSEARCH, "ElasticSearch", 0, writer::ElasticSearch::Instantiate },
#endif
#ifdef USE_DATASERIES
{ BifEnum::Log::WRITER_DATASERIES, "DataSeries", 0, writer::DataSeries::Instantiate },
#endif
// End marker, don't touch.
{ BifEnum::Log::WRITER_DEFAULT, "None", 0, (WriterBackend* (*)(WriterFrontend* frontend))0 }
};
struct Manager::Filter {
string name;
EnumVal* id;
@ -142,6 +104,7 @@ Manager::Stream::~Stream()
}
Manager::Manager()
: plugin::ComponentManager<logging::Tag, logging::Component>("Log", "Writer")
{
rotations_pending = 0;
}
@ -152,64 +115,17 @@ Manager::~Manager()
delete *s;
}
list<string> Manager::SupportedFormats()
WriterBackend* Manager::CreateBackend(WriterFrontend* frontend, EnumVal* tag)
{
list<string> formats;
Component* c = Lookup(tag);
for ( WriterDefinition* ld = log_writers; ld->type != BifEnum::Log::WRITER_DEFAULT; ++ld )
formats.push_back(ld->name);
return formats;
}
WriterBackend* Manager::CreateBackend(WriterFrontend* frontend, bro_int_t type)
{
WriterDefinition* ld = log_writers;
while ( true )
{
if ( ld->type == BifEnum::Log::WRITER_DEFAULT )
if ( ! c )
{
reporter->Error("unknown writer type requested");
return 0;
}
if ( ld->type != type )
{
// Not the right one.
++ld;
continue;
}
// If the writer has an init function, call it.
if ( ld->init )
{
if ( (*ld->init)() )
// Clear the init function so that we won't
// call it again later.
ld->init = 0;
else
{
// Init failed, disable by deleting factory
// function.
ld->factory = 0;
reporter->Error("initialization of writer %s failed", ld->name);
return 0;
}
}
if ( ! ld->factory )
// Oops, we can't instantiate this guy.
return 0;
// All done.
break;
}
assert(ld->factory);
WriterBackend* backend = (*ld->factory)(frontend);
WriterBackend* backend = (*c->Factory())(frontend);
assert(backend);
return backend;
@ -1234,7 +1150,7 @@ void Manager::SendAllWritersTo(RemoteSerializer::PeerID peer)
{
WriterFrontend* writer = i->second->writer;
EnumVal writer_val(i->first.first, BifType::Enum::Log::Writer);
EnumVal writer_val(i->first.first, internal_type("Log::Writer")->AsEnumType());
remote_serializer->SendLogCreateWriter(peer, (*s)->id,
&writer_val,
*i->second->info,

12
src/logging/Manager.h
View file

@ -6,9 +6,12 @@
#define LOGGING_MANAGER_H
#include "../Val.h"
#include "../Tag.h"
#include "../EventHandler.h"
#include "../RemoteSerializer.h"
#include "../plugin/ComponentManager.h"
#include "Component.h"
#include "WriterBackend.h"
class SerializationFormat;
@ -23,7 +26,7 @@ class RotationFinishedMessage;
/**
* Singleton class for managing log streams.
*/
class Manager {
class Manager : public plugin::ComponentManager<Tag, Component> {
public:
/**
* Constructor.
@ -154,11 +157,6 @@ public:
*/
void Terminate();
/**
* Returns a list of supported output formats.
*/
static list<string> SupportedFormats();
protected:
friend class WriterFrontend;
friend class RotationFinishedMessage;
@ -168,7 +166,7 @@ protected:
// Instantiates a new WriterBackend of the given type (note that
// doing so creates a new thread!).
WriterBackend* CreateBackend(WriterFrontend* frontend, bro_int_t type);
WriterBackend* CreateBackend(WriterFrontend* frontend, EnumVal* tag);
//// Function also used by the RemoteSerializer.

22
src/logging/Tag.cc Normal file
View file

@ -0,0 +1,22 @@
// See the file "COPYING" in the main distribution directory for copyright.
#include "Tag.h"
#include "Manager.h"
logging::Tag logging::Tag::Error;
logging::Tag::Tag(type_t type, subtype_t subtype)
: ::Tag(log_mgr->GetTagEnumType(), type, subtype)
{
}
logging::Tag& logging::Tag::operator=(const logging::Tag& other)
{
::Tag::operator=(other);
return *this;
}
EnumVal* logging::Tag::AsEnumVal() const
{
return ::Tag::AsEnumVal(log_mgr->GetTagEnumType());
}

116
src/logging/Tag.h Normal file
View file

@ -0,0 +1,116 @@
// See the file "COPYING" in the main distribution directory for copyright.
#ifndef LOGGING_TAG_H
#define LOGGING_TAG_H
#include "config.h"
#include "util.h"
#include "../Tag.h"
#include "plugin/TaggedComponent.h"
#include "plugin/ComponentManager.h"
class EnumVal;
namespace logging {
class Manager;
class Component;
/**
* Class to identify a writer type.
*
* The script-layer analogue is Log::Writer.
*/
class Tag : public ::Tag {
public:
/*
* Copy constructor.
*/
Tag(const Tag& other) : ::Tag(other) {}
/**
* Default constructor. This initializes the tag with an error value
* that will make \c operator \c bool return false.
*/
Tag() : ::Tag() {}
/**
* Destructor.
*/
~Tag() {}
/**
* Returns false if the tag represents an error value rather than a
* legal writer type.
* TODO: make this conversion operator "explicit" (C++11) or use a
* "safe bool" idiom (not necessary if "explicit" is available),
* otherwise this may allow nonsense/undesired comparison operations.
*/
operator bool() const { return *this != Tag(); }
/**
* Assignment operator.
*/
Tag& operator=(const Tag& other);
/**
* Compares two tags for equality.
*/
bool operator==(const Tag& other) const
{
return ::Tag::operator==(other);
}
/**
* Compares two tags for inequality.
*/
bool operator!=(const Tag& other) const
{
return ::Tag::operator!=(other);
}
/**
* Compares two tags for less-than relationship.
*/
bool operator<(const Tag& other) const
{
return ::Tag::operator<(other);
}
/**
* Returns the \c Log::Writer enum that corresponds to this tag.
* The returned value does not have its ref-count increased.
*
* @param etype the script-layer enum type associated with the tag.
*/
EnumVal* AsEnumVal() const;
static Tag Error;
protected:
friend class plugin::ComponentManager<Tag, Component>;
friend class plugin::TaggedComponent<Tag>;
/**
* Constructor.
*
* @param type The main type. Note that the \a logging::Manager
* manages the value space internally, so noone else should assign
* any main types.
*
* @param subtype The sub type, which is left to an writer for
* interpretation. By default it's set to zero.
*/
Tag(type_t type, subtype_t subtype = 0);
/**
* Constructor.
*
* @param val An enum value of script type \c Log::Writer.
*/
Tag(EnumVal* val) : ::Tag(val) {}
};
}
#endif

2
src/logging/WriterBackend.h
View file

@ -7,6 +7,8 @@
#include "threading/MsgThread.h"
#include "Component.h"
class RemoteSerializer;
namespace logging {

2
src/logging/WriterFrontend.cc
View file

@ -120,7 +120,7 @@ WriterFrontend::WriterFrontend(const WriterBackend::WriterInfo& arg_info, EnumVa
if ( local )
{
backend = log_mgr->CreateBackend(this, writer->AsEnum());
backend = log_mgr->CreateBackend(this, writer);
if ( backend )
backend->Start();

54
src/logging.bif → src/logging/logging.bif
View file

@ -3,8 +3,6 @@
module Log;
%%{
#include "NetVar.h"
#include "logging/Manager.h"
%%}
@ -65,55 +63,3 @@ function Log::__flush%(id: Log::ID%): bool
bool result = log_mgr->Flush(id->AsEnumVal());
return new Val(result, TYPE_BOOL);
%}
# Options for the ASCII writer.
module LogAscii;
const output_to_stdout: bool;
const include_meta: bool;
const meta_prefix: string;
const separator: string;
const set_separator: string;
const empty_field: string;
const unset_field: string;
const use_json: bool;
const json_timestamps: JSON::TimestampFormat;
# Options for the DataSeries writer.
module LogDataSeries;
const compression: string;
const extent_size: count;
const dump_schema: bool;
const use_integer_for_time: bool;
const num_threads: count;
# Options for the SQLite writer
module LogSQLite;
const set_separator: string;
const empty_field: string;
const unset_field: string;
# Options for the ElasticSearch writer.
module LogElasticSearch;
const cluster_name: string;
const server_host: string;
const server_port: count;
const index_prefix: string;
const type_prefix: string;
const transfer_timeout: interval;
const max_batch_size: count;
const max_batch_interval: interval;
const max_byte_size: count;
# Options for the None writer.
module LogNone;
const debug: bool;

6
src/logging/writers/CMakeLists.txt Normal file
View file

@ -0,0 +1,6 @@
add_subdirectory(ascii)
add_subdirectory(dataseries)
add_subdirectory(elasticsearch)
add_subdirectory(none)
add_subdirectory(sqlite)

2
src/logging/writers/Ascii.cc → src/logging/writers/ascii/Ascii.cc
View file

@ -5,10 +5,10 @@
#include <fcntl.h>
#include <unistd.h>
#include "NetVar.h"
#include "threading/SerialTypes.h"
#include "Ascii.h"
#include "ascii.bif.h"
using namespace logging::writer;
using namespace threading;

5
src/logging/writers/Ascii.h → src/logging/writers/ascii/Ascii.h
View file

@ -5,7 +5,7 @@
#ifndef LOGGING_WRITER_ASCII_H
#define LOGGING_WRITER_ASCII_H
#include "../WriterBackend.h"
#include "logging/WriterBackend.h"
#include "threading/formatters/Ascii.h"
#include "threading/formatters/JSON.h"
@ -16,9 +16,10 @@ public:
Ascii(WriterFrontend* frontend);
~Ascii();
static string LogExt();
static WriterBackend* Instantiate(WriterFrontend* frontend)
{ return new Ascii(frontend); }
static string LogExt();
protected:
virtual bool DoInit(const WriterInfo& info, int num_fields,

9
src/logging/writers/ascii/CMakeLists.txt Normal file
View file

@ -0,0 +1,9 @@
include(BroPlugin)
include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR})
bro_plugin_begin(Bro AsciiWriter)
bro_plugin_cc(Ascii.cc Plugin.cc)
bro_plugin_bif(ascii.bif)
bro_plugin_end()

25
src/logging/writers/ascii/Plugin.cc Normal file
View file

@ -0,0 +1,25 @@
// See the file in the main distribution directory for copyright.
#include "plugin/Plugin.h"
#include "Ascii.h"
namespace plugin {
namespace Bro_AsciiWriter {
class Plugin : public plugin::Plugin {
public:
plugin::Configuration Configure()
{
AddComponent(new ::logging::Component("Ascii", ::logging::writer::Ascii::Instantiate));
plugin::Configuration config;
config.name = "Bro::AsciiWriter";
config.description = "ASCII log writer";
return config;
}
} plugin;
}
}

14
src/logging/writers/ascii/ascii.bif Normal file
View file

@ -0,0 +1,14 @@
# Options for the ASCII writer.
module LogAscii;
const output_to_stdout: bool;
const include_meta: bool;
const meta_prefix: string;
const separator: string;
const set_separator: string;
const empty_field: string;
const unset_field: string;
const use_json: bool;
const json_timestamps: JSON::TimestampFormat;

19
src/logging/writers/dataseries/CMakeLists.txt Normal file
View file

@ -0,0 +1,19 @@
include(BroPlugin)
find_package(Lintel)
find_package(DataSeries)
find_package(LibXML2)
if (NOT DISABLE_DATASERIES AND
LINTEL_FOUND AND DATASERIES_FOUND AND LIBXML2_FOUND)
include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR})
bro_plugin_begin(Bro DataSeriesWriter)
bro_plugin_cc(DataSeries.cc Plugin.cc)
bro_plugin_bif(dataseries.bif)
bro_plugin_end()
endif()

5
src/logging/writers/DataSeries.cc → src/logging/writers/dataseries/DataSeries.cc
View file

@ -2,8 +2,6 @@
#include "config.h"
#ifdef USE_DATASERIES
#include <map>
#include <string>
#include <errno.h>
@ -14,6 +12,7 @@
#include "threading/SerialTypes.h"
#include "DataSeries.h"
#include "dataseries.bif.h"
using namespace logging;
using namespace writer;
@ -458,5 +457,3 @@ bool DataSeries::DoHeartbeat(double network_time, double current_time)
{
return true;
}
#endif /* USE_DATASERIES */

3
src/logging/writers/DataSeries.h → src/logging/writers/dataseries/DataSeries.h
View file

@ -11,8 +11,7 @@
#include <DataSeries/DataSeriesModule.hpp>
#include <DataSeries/GeneralField.hpp>
#include "../WriterBackend.h"
#include "threading/formatters/Ascii.h"
#include "logging/WriterBackend.h"
namespace logging { namespace writer {

25
src/logging/writers/dataseries/Plugin.cc Normal file
View file

@ -0,0 +1,25 @@
// See the file in the main distribution directory for copyright.
#include "plugin/Plugin.h"
#include "DataSeries.h"
namespace plugin {
namespace Bro_DataSeriesWriter {
class Plugin : public plugin::Plugin {
public:
plugin::Configuration Configure()
{
AddComponent(new ::logging::Component("DataSeries", ::logging::writer::DataSeries::Instantiate));
plugin::Configuration config;
config.name = "Bro::DataSeriesWriter";
config.description = "DataSeries log writer";
return config;
}
} plugin;
}
}

10
src/logging/writers/dataseries/dataseries.bif Normal file
View file

@ -0,0 +1,10 @@
# Options for the DataSeries writer.
module LogDataSeries;
const compression: string;
const extent_size: count;
const dump_schema: bool;
const use_integer_for_time: bool;
const num_threads: count;

15
src/logging/writers/elasticsearch/CMakeLists.txt Normal file
View file

@ -0,0 +1,15 @@
include(BroPlugin)
find_package(LibCURL)
if (NOT DISABLE_ELASTICSEARCH AND LIBCURL_FOUND)
include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR})
bro_plugin_begin(Bro ElasticSearchWriter)
bro_plugin_cc(ElasticSearch.cc Plugin.cc)
bro_plugin_bif(elasticsearch.bif)
bro_plugin_link_library(${LibCURL_LIBRARIES})
bro_plugin_end()
endif()

13
src/logging/writers/ElasticSearch.cc → src/logging/writers/elasticsearch/ElasticSearch.cc
View file

@ -6,21 +6,18 @@
#include "config.h"
#ifdef USE_ELASTICSEARCH
#include "util.h" // Needs to come first for stdint.h
#include <string>
#include <errno.h>
#include "BroString.h"
#include "NetVar.h"
#include "threading/SerialTypes.h"
#include <curl/curl.h>
#include <curl/easy.h>
#include "BroString.h"
#include "threading/SerialTypes.h"
#include "ElasticSearch.h"
#include "elasticsearch.bif.h"
using namespace logging;
using namespace writer;
@ -291,5 +288,3 @@ bool ElasticSearch::HTTPSend(CURL *handle)
// The "successful" return happens above
return false;
}
#endif

6
src/logging/writers/ElasticSearch.h → src/logging/writers/elasticsearch/ElasticSearch.h
View file

@ -9,8 +9,9 @@
#define LOGGING_WRITER_ELASTICSEARCH_H
#include <curl/curl.h>
#include "logging/WriterBackend.h"
#include "threading/formatters/JSON.h"
#include "../WriterBackend.h"
namespace logging { namespace writer {
@ -19,9 +20,10 @@ public:
ElasticSearch(WriterFrontend* frontend);
~ElasticSearch();
static string LogExt();
static WriterBackend* Instantiate(WriterFrontend* frontend)
{ return new ElasticSearch(frontend); }
static string LogExt();
protected:
// Overidden from WriterBackend.

37
src/logging/writers/elasticsearch/Plugin.cc Normal file
View file

@ -0,0 +1,37 @@
// See the file in the main distribution directory for copyright.
#include <curl/curl.h>
#include "plugin/Plugin.h"
#include "ElasticSearch.h"
namespace plugin {
namespace Bro_ElasticSearchWriter {
class Plugin : public plugin::Plugin {
public:
plugin::Configuration Configure()
{
AddComponent(new ::logging::Component("ElasticSearch", ::logging::writer::ElasticSearch::Instantiate));
plugin::Configuration config;
config.name = "Bro::ElasticSearchWriter";
config.description = "ElasticSearch log writer";
return config;
}
virtual void InitPreScript()
{
curl_global_init(CURL_GLOBAL_ALL);
}
virtual void Done()
{
curl_global_cleanup();
}
} plugin;
}
}

14
src/logging/writers/elasticsearch/elasticsearch.bif Normal file
View file

@ -0,0 +1,14 @@
# Options for the ElasticSearch writer.
module LogElasticSearch;
const cluster_name: string;
const server_host: string;
const server_port: count;
const index_prefix: string;
const type_prefix: string;
const transfer_timeout: interval;
const max_batch_size: count;
const max_batch_interval: interval;
const max_byte_size: count;

9
src/logging/writers/none/CMakeLists.txt Normal file
View file

@ -0,0 +1,9 @@
include(BroPlugin)
include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR})
bro_plugin_begin(Bro NoneWriter)
bro_plugin_cc(None.cc Plugin.cc)
bro_plugin_bif(none.bif)
bro_plugin_end()

2
src/logging/writers/None.cc → src/logging/writers/none/None.cc
View file

@ -2,7 +2,7 @@
#include <algorithm>
#include "None.h"
#include "NetVar.h"
#include "none.bif.h"
using namespace logging;
using namespace writer;

2
src/logging/writers/None.h → src/logging/writers/none/None.h
View file

@ -5,7 +5,7 @@
#ifndef LOGGING_WRITER_NONE_H
#define LOGGING_WRITER_NONE_H
#include "../WriterBackend.h"
#include "logging/WriterBackend.h"
namespace logging { namespace writer {

25
src/logging/writers/none/Plugin.cc Normal file
View file

@ -0,0 +1,25 @@
// See the file in the main distribution directory for copyright.
#include "plugin/Plugin.h"
#include "None.h"
namespace plugin {
namespace Bro_NoneWriter {
class Plugin : public plugin::Plugin {
public:
plugin::Configuration Configure()
{
AddComponent(new ::logging::Component("None", ::logging::writer::None::Instantiate));
plugin::Configuration config;
config.name = "Bro::NoneWriter";
config.description = "None log writer (primarily for debugging)";
return config;
}
} plugin;
}
}

6
src/logging/writers/none/none.bif Normal file
View file

@ -0,0 +1,6 @@
# Options for the None writer.
module LogNone;
const debug: bool;

9
src/logging/writers/sqlite/CMakeLists.txt Normal file
View file

@ -0,0 +1,9 @@
include(BroPlugin)
include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR})
bro_plugin_begin(Bro SQLiteWriter)
bro_plugin_cc(SQLite.cc Plugin.cc)
bro_plugin_bif(sqlite.bif)
bro_plugin_end()

25
src/logging/writers/sqlite/Plugin.cc Normal file
View file

@ -0,0 +1,25 @@
// See the file in the main distribution directory for copyright.
#include "plugin/Plugin.h"
#include "SQLite.h"
namespace plugin {
namespace Bro_SQLiteWriter {
class Plugin : public plugin::Plugin {
public:
plugin::Configuration Configure()
{
AddComponent(new ::logging::Component("SQLite", ::logging::writer::SQLite::Instantiate));
plugin::Configuration config;
config.name = "Bro::SQLiteWriter";
config.description = "SQLite log writer";
return config;
}
} plugin;
}
}

4
src/logging/writers/SQLite.cc → src/logging/writers/sqlite/SQLite.cc
View file

@ -6,10 +6,10 @@
#include <errno.h>
#include <vector>
#include "../../NetVar.h"
#include "../../threading/SerialTypes.h"
#include "threading/SerialTypes.h"
#include "SQLite.h"
#include "sqlite.bif.h"
using namespace logging;
using namespace writer;

3
src/logging/writers/SQLite.h → src/logging/writers/sqlite/SQLite.h
View file

@ -7,8 +7,7 @@
#include "config.h"
#include "../WriterBackend.h"
#include "logging/WriterBackend.h"
#include "threading/formatters/Ascii.h"
#include "3rdparty/sqlite3.h"

9
src/logging/writers/sqlite/sqlite.bif Normal file
View file

@ -0,0 +1,9 @@
# Options for the SQLite writer
module LogSQLite;
const set_separator: string;
const empty_field: string;
const unset_field: string;

37
src/main.cc
View file

@ -12,10 +12,6 @@
#include <getopt.h>
#endif
#ifdef USE_CURL
#include <curl/curl.h>
#endif
#ifdef USE_IDMEF
extern "C" {
#include <libidmef/idmefxml.h>
@ -54,8 +50,8 @@ extern "C" void OPENSSL_add_all_algorithms_conf(void);
#include "threading/Manager.h"
#include "input/Manager.h"
#include "logging/Manager.h"
#include "logging/writers/Ascii.h"
#include "input/readers/Raw.h"
#include "logging/writers/ascii/Ascii.h"
#include "input/readers/raw/Raw.h"
#include "analyzer/Manager.h"
#include "analyzer/Tag.h"
#include "plugin/Manager.h"
@ -227,25 +223,6 @@ void usage()
fprintf(stderr, " $BRO_PROFILER_FILE | Output file for script execution statistics (not set)\n");
fprintf(stderr, " $BRO_DISABLE_BROXYGEN | Disable Broxygen documentation support (%s)\n", getenv("BRO_DISABLE_BROXYGEN") ? "set" : "not set");
fprintf(stderr, "\n");
fprintf(stderr, " Supported log formats: ");
bool first = true;
list<string> fmts = logging::Manager::SupportedFormats();
for ( list<string>::const_iterator i = fmts.begin(); i != fmts.end(); ++i )
{
if ( *i == "None" )
// Skip, it's uninteresting.
continue;
if ( ! first )
fprintf(stderr, ",");
fprintf(stderr, "%s", (*i).c_str());
first = false;
}
fprintf(stderr, "\n");
exit(1);
@ -813,10 +790,6 @@ int main(int argc, char** argv)
SSL_library_init();
SSL_load_error_strings();
#ifdef USE_CURL
curl_global_init(CURL_GLOBAL_ALL);
#endif
int r = sqlite3_initialize();
if ( r != SQLITE_OK )
@ -897,8 +870,6 @@ int main(int argc, char** argv)
init_event_handlers();
input::reader::Raw::ClassInit();
md5_type = new OpaqueType("md5");
sha1_type = new OpaqueType("sha1");
sha256_type = new OpaqueType("sha256");
@ -1252,10 +1223,6 @@ int main(int argc, char** argv)
done_with_network();
net_delete();
#ifdef USE_CURL
curl_global_cleanup();
#endif
terminate_bro();
sqlite3_shutdown();

18
src/plugin/ComponentManager.h
View file

@ -27,13 +27,16 @@ class ComponentManager {
public:
/**
* Constructor creates a new enum type called a "Tag" to associate with
* Constructor creates a new enum type to associate with
* a component.
*
* @param module The script-layer module in which to install the "Tag" ID
* @param module The script-layer module in which to install the ID
* representing an enum type.
*
* @param local_id The local part of the ID of the new enum type
* (e.g., "Tag").
*/
ComponentManager(const string& module);
ComponentManager(const string& module, const string& local_id);
/**
* @return The script-layer module in which the component's "Tag" ID lives.
@ -125,13 +128,15 @@ private:
};
template <class T, class C>
ComponentManager<T, C>::ComponentManager(const string& arg_module)
ComponentManager<T, C>::ComponentManager(const string& arg_module, const string& local_id)
: module(arg_module)
{
tag_enum_type = new EnumType(module + "::Tag");
::ID* id = install_ID("Tag", module.c_str(), true, true);
tag_enum_type = new EnumType(module + "::" + local_id);
::ID* id = install_ID(local_id.c_str(), module.c_str(), true, true);
add_type(id, tag_enum_type, 0);
broxygen_mgr->Identifier(id);
// fprintf(stderr, "Enum: %s\n", id->Name());
}
template <class T, class C>
@ -241,6 +246,7 @@ void ComponentManager<T, C>::RegisterComponent(C* component,
string id = fmt("%s%s", prefix.c_str(), cname.c_str());
tag_enum_type->AddName(module, id.c_str(),
component->Tag().AsEnumVal()->InternalInt(), true);
// fprintf(stderr, "Enum item: %s/%s\n", module.c_str(), id.c_str());
}
} // namespace plugin

38
src/types.bif
View file

@ -163,21 +163,6 @@ type ModbusHeaders: record;
type ModbusCoils: vector;
type ModbusRegisters: vector;
module Log;
enum Writer %{
WRITER_DEFAULT,
WRITER_NONE,
WRITER_ASCII,
WRITER_DATASERIES,
WRITER_SQLITE,
WRITER_ELASTICSEARCH,
%}
enum ID %{
Unknown,
%}
module Tunnel;
enum Type %{
NONE,
@ -191,29 +176,6 @@ enum Type %{
type EncapsulatingConn: record;
module Input;
enum Reader %{
READER_DEFAULT,
READER_ASCII,
READER_RAW,
READER_BENCHMARK,
READER_BINARY,
READER_SQLITE,
%}
enum Event %{
EVENT_NEW,
EVENT_CHANGED,
EVENT_REMOVED,
%}
enum Mode %{
MANUAL = 0,
REREAD = 1,
STREAM = 2,
%}
module GLOBAL;
type gtpv1_hdr: record;

2
testing/btest/core/leaks/dataseries-rotate.bro
View file

@ -1,5 +1,5 @@
#
# @TEST-REQUIRES: has-writer DataSeries && which ds2txt
# @TEST-REQUIRES: has-writer Bro::DataSeriesWriter && which ds2txt
# @TEST-REQUIRES: bro --help 2>&1 | grep -q mem-leaks
#
# @TEST-GROUP: leaks

2
testing/btest/core/leaks/dataseries.bro
View file

@ -1,6 +1,6 @@
# Needs perftools support.
#
# @TEST-REQUIRES: has-writer DataSeries && which ds2txt
# @TEST-REQUIRES: has-writer Bro::DataSeriesWriter && which ds2txt
# @TEST-REQUIRES: bro --help 2>&1 | grep -q mem-leaks
#
# @TEST-GROUP: leaks

2
testing/btest/scripts/base/frameworks/logging/dataseries/options.bro
View file

@ -1,5 +1,5 @@
#
# @TEST-REQUIRES: has-writer DataSeries && which ds2txt
# @TEST-REQUIRES: has-writer Bro::DataSeriesWriter && which ds2txt
# @TEST-GROUP: dataseries
#
# @TEST-EXEC: bro -b %INPUT Log::default_writer=Log::WRITER_DATASERIES

Some files were not shown because too many files have changed in this diff Show more