Mirror/zeek
1
0
Fork 0
mirror of https://github.com/zeek/zeek.git synced 2025-10-08 09:38:19 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

Reformat the world

Browse source
This commit is contained in:
Tim Wojtulewicz 2021-09-16 15:35:39 -07:00
parent 194cb24547
commit b2f171ec69
714 changed files with 35149 additions and 35203 deletions
Download patch file Download diff file Expand all files Collapse all files

11
src/input/Component.cc
View file

@ -1,12 +1,13 @@
// See the file "COPYING" in the main distribution directory for copyright.
#include "zeek/input/Component.h"
#include "zeek/input/Manager.h"
#include "zeek/Desc.h"
#include "zeek/input/Manager.h"
#include "zeek/util.h"
namespace zeek::input {
namespace zeek::input
{
Component::Component(const std::string& name, factory_callback arg_factory)
: plugin::Component(plugin::component::READER, name)
@ -20,9 +21,7 @@ void Component::Initialize()
input_mgr->RegisterComponent(this, "READER_");
}
Component::~Component()
{
}
Component::~Component() { }
void Component::DoDescribe(ODesc* d) const
{
@ -30,4 +29,4 @@ void Component::DoDescribe(ODesc* d) const
d->Add(CanonicalName());
}
} // namespace zeek::input
} // namespace zeek::input

19
src/input/Component.h
View file

@ -2,11 +2,12 @@
#pragma once
#include "zeek/input/Tag.h"
#include "zeek/plugin/Component.h"
#include "zeek/plugin/TaggedComponent.h"
#include "zeek/input/Tag.h"
namespace zeek::input {
namespace zeek::input
{
class ReaderFrontend;
class ReaderBackend;
@ -14,8 +15,8 @@ class ReaderBackend;
/**
* Component description for plugins providing log readers.
*/
class Component : public plugin::Component,
public plugin::TaggedComponent<Tag> {
class Component : public plugin::Component, public plugin::TaggedComponent<Tag>
{
public:
typedef ReaderBackend* (*factory_callback)(ReaderFrontend* frontend);
@ -48,16 +49,16 @@ public:
/**
* Returns the reader's factory function.
*/
factory_callback Factory() const { return factory; }
factory_callback Factory() const { return factory; }
protected:
/**
* Overriden from plugin::Component.
*/
* Overriden from plugin::Component.
*/
void DoDescribe(ODesc* d) const override;
private:
factory_callback factory;
};
};
} // namespace zeek::input
} // namespace zeek::input

1363
src/input/Manager.cc
View file

File diff suppressed because it is too large Load diff

85
src/input/Manager.h
View file

@ -6,17 +6,19 @@
#include <map>
#include "zeek/input/Component.h"
#include "zeek/EventHandler.h"
#include "zeek/input/Component.h"
#include "zeek/input/Tag.h"
#include "zeek/plugin/ComponentManager.h"
#include "zeek/threading/SerialTypes.h"
#include "zeek/input/Tag.h"
namespace zeek {
namespace zeek
{
class RecordVal;
namespace input {
namespace input
{
class ReaderFrontend;
class ReaderBackend;
@ -24,7 +26,8 @@ class ReaderBackend;
/**
* Singleton class for managing input streams.
*/
class Manager : public plugin::ComponentManager<Tag, Component> {
class Manager : public plugin::ComponentManager<Tag, Component>
{
public:
/**
* Constructor.
@ -83,7 +86,7 @@ public:
* This method corresponds directly to the internal BiF defined in
* input.bif, which just forwards here.
*/
bool ForceUpdate(const std::string &id);
bool ForceUpdate(const std::string& id);
/**
* Deletes an existing input stream.
@ -93,7 +96,7 @@ public:
* This method corresponds directly to the internal BiF defined in
* input.bif, which just forwards here.
*/
bool RemoveStream(const std::string &id);
bool RemoveStream(const std::string& id);
/**
* Signals the manager to shutdown at Bro's termination.
@ -113,7 +116,7 @@ public:
*
* @return True if the type is compatible with the input framework.
*/
static bool IsCompatibleType(Type* t, bool atomic_only=false);
static bool IsCompatibleType(Type* t, bool atomic_only = false);
protected:
friend class ReaderFrontend;
@ -130,9 +133,9 @@ protected:
// For readers to write to input stream in direct mode (reporting
// new/deleted values directly). Functions take ownership of
// threading::Value fields.
void Put(ReaderFrontend* reader, threading::Value* *vals);
void Put(ReaderFrontend* reader, threading::Value** vals);
void Clear(ReaderFrontend* reader);
bool Delete(ReaderFrontend* reader, threading::Value* *vals);
bool Delete(ReaderFrontend* reader, threading::Value** vals);
// Trigger sending the End-of-Data event when the input source has
// finished reading. Just use in direct mode.
void SendEndOfData(ReaderFrontend* reader);
@ -140,7 +143,7 @@ protected:
// For readers to write to input stream in indirect mode (manager is
// monitoring new/deleted values) Functions take ownership of
// threading::Value fields.
void SendEntry(ReaderFrontend* reader, threading::Value* *vals);
void SendEntry(ReaderFrontend* reader, threading::Value** vals);
void EndCurrentSend(ReaderFrontend* reader);
// Instantiates a new ReaderBackend of the given type (note that
@ -187,56 +190,62 @@ private:
// Check if the types of the error_ev event are correct. If table is
// true, check for tablestream type, otherwhise check for eventstream
// type.
bool CheckErrorEventTypes(const std::string& stream_name, const Func* error_event, bool table) const;
bool CheckErrorEventTypes(const std::string& stream_name, const Func* error_event,
bool table) const;
// SendEntry implementation for Table stream.
int SendEntryTable(Stream* i, const threading::Value* const *vals);
int SendEntryTable(Stream* i, const threading::Value* const* vals);
// Put implementation for Table stream.
int PutTable(Stream* i, const threading::Value* const *vals);
int PutTable(Stream* i, const threading::Value* const* vals);
// SendEntry and Put implementation for Event stream.
int SendEventStreamEvent(Stream* i, EnumVal* type, const threading::Value* const *vals);
int SendEventStreamEvent(Stream* i, EnumVal* type, const threading::Value* const* vals);
// Check if a record is made up of compatible types and return a list
// of all fields that are in the record in order. Recursively unrolls
// records
bool UnrollRecordType(std::vector<threading::Field*> *fields, const RecordType *rec, const std::string& nameprepend, bool allow_file_func) const;
bool UnrollRecordType(std::vector<threading::Field*>* fields, const RecordType* rec,
const std::string& nameprepend, bool allow_file_func) const;
// Send events
void SendEvent(EventHandlerPtr ev, const int numvals, ...) const;
void SendEvent(EventHandlerPtr ev, std::list<Val*> events) const;
// Implementation of SendEndOfData (send end_of_data event).
void SendEndOfData(const Stream *i);
void SendEndOfData(const Stream* i);
// Call predicate function and return result.
bool CallPred(Func* pred_func, const int numvals, ...) const;
// Get a hashkey for a set of threading::Values.
zeek::detail::HashKey* HashValues(const int num_elements, const threading::Value* const *vals) const;
zeek::detail::HashKey* HashValues(const int num_elements,
const threading::Value* const* vals) const;
// Get the memory used by a specific value.
int GetValueLength(const threading::Value* val) const;
// Copies the raw data in a specific threading::Value to position
// startpos.
int CopyValue(char *data, const int startpos, const threading::Value* val) const;
int CopyValue(char* data, const int startpos, const threading::Value* val) const;
// Convert Threading::Value to an internal Bro Type (works with Records).
Val* ValueToVal(const Stream* i, const threading::Value* val, Type* request_type, bool& have_error) const;
Val* ValueToVal(const Stream* i, const threading::Value* val, Type* request_type,
bool& have_error) const;
// Convert Threading::Value to an internal Bro list type.
Val* ValueToIndexVal(const Stream* i, int num_fields, const RecordType* type, const threading::Value* const *vals, bool& have_error) const;
Val* ValueToIndexVal(const Stream* i, int num_fields, const RecordType* type,
const threading::Value* const* vals, bool& have_error) const;
// Converts a threading::value to a record type. Mostly used by
// ValueToVal.
RecordVal* ValueToRecordVal(const Stream* i, const threading::Value* const *vals, RecordType *request_type, int* position, bool& have_error) const;
RecordVal* ValueToRecordVal(const Stream* i, const threading::Value* const* vals,
RecordType* request_type, int* position, bool& have_error) const;
Val* RecordValToIndexVal(RecordVal *r) const;
Val* RecordValToIndexVal(RecordVal* r) const;
// Converts a Bro ListVal to a RecordVal given the record type.
RecordVal* ListValToRecordVal(ListVal* list, RecordType *request_type, int* position) const;
RecordVal* ListValToRecordVal(ListVal* list, RecordType* request_type, int* position) const;
// Internally signal errors, warnings, etc.
// These are sent on to input scriptland and reporter.log
@ -244,22 +253,34 @@ private:
void Warning(const Stream* i, const char* fmt, ...) const __attribute__((format(printf, 3, 4)));
void Error(const Stream* i, const char* fmt, ...) const __attribute__((format(printf, 3, 4)));
enum class ErrorType { INFO, WARNING, ERROR };
void ErrorHandler(const Stream* i, ErrorType et, bool reporter_send, const char* fmt, ...) const __attribute__((format(printf, 5, 6)));
void ErrorHandler(const Stream* i, ErrorType et, bool reporter_send, const char* fmt, va_list ap) const __attribute__((format(printf, 5, 0)));
enum class ErrorType
{
INFO,
WARNING,
ERROR
};
void ErrorHandler(const Stream* i, ErrorType et, bool reporter_send, const char* fmt, ...) const
__attribute__((format(printf, 5, 6)));
void ErrorHandler(const Stream* i, ErrorType et, bool reporter_send, const char* fmt,
va_list ap) const __attribute__((format(printf, 5, 0)));
Stream* FindStream(const std::string &name) const;
Stream* FindStream(const std::string& name) const;
Stream* FindStream(ReaderFrontend* reader) const;
enum StreamType { TABLE_STREAM, EVENT_STREAM, ANALYSIS_STREAM };
enum StreamType
{
TABLE_STREAM,
EVENT_STREAM,
ANALYSIS_STREAM
};
std::map<ReaderFrontend*, Stream*> readers;
EventHandlerPtr end_of_data;
};
};
} // namespace input
} // namespace input
extern input::Manager* input_mgr;
} // namespace zeek
} // namespace zeek

161
src/input/ReaderBackend.cc
View file

@ -1,19 +1,23 @@
// See the file "COPYING" in the main distribution directory for copyright.
#include "zeek/input/ReaderBackend.h"
#include "zeek/input/ReaderFrontend.h"
#include "zeek/input/Manager.h"
#include "zeek/input/ReaderFrontend.h"
using zeek::threading::Value;
using zeek::threading::Field;
using zeek::threading::Value;
namespace zeek::input {
namespace zeek::input
{
class PutMessage final : public threading::OutputMessage<ReaderFrontend> {
class PutMessage final : public threading::OutputMessage<ReaderFrontend>
{
public:
PutMessage(ReaderFrontend* reader, Value* *val)
: threading::OutputMessage<ReaderFrontend>("Put", reader),
val(val) {}
PutMessage(ReaderFrontend* reader, Value** val)
: threading::OutputMessage<ReaderFrontend>("Put", reader), val(val)
{
}
bool Process() override
{
@ -22,28 +26,29 @@ public:
}
private:
Value* *val;
};
Value** val;
};
class DeleteMessage final : public threading::OutputMessage<ReaderFrontend> {
class DeleteMessage final : public threading::OutputMessage<ReaderFrontend>
{
public:
DeleteMessage(ReaderFrontend* reader, Value* *val)
: threading::OutputMessage<ReaderFrontend>("Delete", reader),
val(val) {}
bool Process() override
DeleteMessage(ReaderFrontend* reader, Value** val)
: threading::OutputMessage<ReaderFrontend>("Delete", reader), val(val)
{
return input_mgr->Delete(Object(), val);
}
private:
Value* *val;
};
bool Process() override { return input_mgr->Delete(Object(), val); }
class ClearMessage final : public threading::OutputMessage<ReaderFrontend> {
private:
Value** val;
};
class ClearMessage final : public threading::OutputMessage<ReaderFrontend>
{
public:
ClearMessage(ReaderFrontend* reader)
: threading::OutputMessage<ReaderFrontend>("Clear", reader) {}
ClearMessage(ReaderFrontend* reader) : threading::OutputMessage<ReaderFrontend>("Clear", reader)
{
}
bool Process() override
{
@ -52,33 +57,41 @@ public:
}
private:
};
};
class ReaderErrorMessage final : public threading::OutputMessage<ReaderFrontend>
{
{
public:
enum Type {
INFO, WARNING, ERROR
};
enum Type
{
INFO,
WARNING,
ERROR
};
ReaderErrorMessage(ReaderFrontend* reader, Type arg_type, const char* arg_msg)
: threading::OutputMessage<ReaderFrontend>("ReaderErrorMessage", reader)
{ type = arg_type; msg = util::copy_string(arg_msg); }
{
type = arg_type;
msg = util::copy_string(arg_msg);
}
~ReaderErrorMessage() override { delete [] msg; }
~ReaderErrorMessage() override { delete[] msg; }
bool Process() override;
private:
const char* msg;
Type type;
};
};
class SendEntryMessage final : public threading::OutputMessage<ReaderFrontend> {
class SendEntryMessage final : public threading::OutputMessage<ReaderFrontend>
{
public:
SendEntryMessage(ReaderFrontend* reader, Value* *val)
: threading::OutputMessage<ReaderFrontend>("SendEntry", reader),
val(val) { }
SendEntryMessage(ReaderFrontend* reader, Value** val)
: threading::OutputMessage<ReaderFrontend>("SendEntry", reader), val(val)
{
}
bool Process() override
{
@ -87,13 +100,16 @@ public:
}
private:
Value* *val;
};
Value** val;
};
class EndCurrentSendMessage final : public threading::OutputMessage<ReaderFrontend> {
class EndCurrentSendMessage final : public threading::OutputMessage<ReaderFrontend>
{
public:
EndCurrentSendMessage(ReaderFrontend* reader)
: threading::OutputMessage<ReaderFrontend>("EndCurrentSend", reader) {}
: threading::OutputMessage<ReaderFrontend>("EndCurrentSend", reader)
{
}
bool Process() override
{
@ -102,12 +118,15 @@ public:
}
private:
};
};
class EndOfDataMessage final : public threading::OutputMessage<ReaderFrontend> {
class EndOfDataMessage final : public threading::OutputMessage<ReaderFrontend>
{
public:
EndOfDataMessage(ReaderFrontend* reader)
: threading::OutputMessage<ReaderFrontend>("EndOfData", reader) {}
: threading::OutputMessage<ReaderFrontend>("EndOfData", reader)
{
}
bool Process() override
{
@ -116,12 +135,15 @@ public:
}
private:
};
};
class ReaderClosedMessage final : public threading::OutputMessage<ReaderFrontend> {
class ReaderClosedMessage final : public threading::OutputMessage<ReaderFrontend>
{
public:
ReaderClosedMessage(ReaderFrontend* reader)
: threading::OutputMessage<ReaderFrontend>("ReaderClosed", reader) {}
: threading::OutputMessage<ReaderFrontend>("ReaderClosed", reader)
{
}
bool Process() override
{
@ -130,13 +152,15 @@ public:
}
private:
};
};
class DisableMessage final : public threading::OutputMessage<ReaderFrontend>
{
{
public:
DisableMessage(ReaderFrontend* writer)
: threading::OutputMessage<ReaderFrontend>("Disable", writer) {}
: threading::OutputMessage<ReaderFrontend>("Disable", writer)
{
}
bool Process() override
{
@ -149,29 +173,29 @@ public:
input_mgr->RemoveStream(Object());
return true;
}
};
};
bool ReaderErrorMessage::Process()
{
switch ( type ) {
switch ( type )
{
case INFO:
input_mgr->Info(Object(), msg);
break;
case INFO:
input_mgr->Info(Object(), msg);
break;
case WARNING:
input_mgr->Warning(Object(), msg);
break;
case WARNING:
input_mgr->Warning(Object(), msg);
break;
case ERROR:
input_mgr->Error(Object(), msg);
break;
}
case ERROR:
input_mgr->Error(Object(), msg);
break;
}
return true;
}
using namespace input;
ReaderBackend::ReaderBackend(ReaderFrontend* arg_frontend) : MsgThread()
@ -190,12 +214,12 @@ ReaderBackend::~ReaderBackend()
delete info;
}
void ReaderBackend::Put(Value* *val)
void ReaderBackend::Put(Value** val)
{
SendOut(new PutMessage(frontend, val));
}
void ReaderBackend::Delete(Value* *val)
void ReaderBackend::Delete(Value** val)
{
SendOut(new DeleteMessage(frontend, val));
}
@ -215,13 +239,12 @@ void ReaderBackend::EndOfData()
SendOut(new EndOfDataMessage(frontend));
}
void ReaderBackend::SendEntry(Value* *vals)
void ReaderBackend::SendEntry(Value** vals)
{
SendOut(new SendEntryMessage(frontend, vals));
}
bool ReaderBackend::Init(const int arg_num_fields,
const threading::Field* const* arg_fields)
bool ReaderBackend::Init(const int arg_num_fields, const threading::Field* const* arg_fields)
{
if ( Failed() )
return true;
@ -256,9 +279,9 @@ bool ReaderBackend::OnFinish(double network_time)
if ( fields )
{
for ( unsigned int i = 0; i < num_fields; i++ )
delete(fields[i]);
delete (fields[i]);
delete [] (fields);
delete[](fields);
fields = nullptr;
}
@ -307,7 +330,7 @@ void ReaderBackend::Info(const char* msg)
MsgThread::Info(msg);
}
void ReaderBackend::FailWarn(bool is_error, const char *msg, bool suppress_future)
void ReaderBackend::FailWarn(bool is_error, const char* msg, bool suppress_future)
{
if ( is_error )
Error(msg);
@ -340,4 +363,4 @@ void ReaderBackend::Error(const char* msg)
DisableFrontend();
}
} // namespace zeek::input
} // namespace zeek::input

50
src/input/ReaderBackend.h
View file

@ -3,20 +3,20 @@
#pragma once
#include "zeek/ZeekString.h"
#include "zeek/threading/SerialTypes.h"
#include "zeek/threading/MsgThread.h"
#include "zeek/input/Component.h"
#include "zeek/threading/MsgThread.h"
#include "zeek/threading/SerialTypes.h"
namespace zeek::input {
namespace zeek::input
{
class ReaderFrontend;
/**
* The modes a reader can be in.
*/
enum ReaderMode {
enum ReaderMode
{
/**
* Manual refresh reader mode. The reader will read the file once,
* and send all read data back to the manager. After that, no automatic
@ -41,7 +41,7 @@ enum ReaderMode {
/** Internal dummy mode for initialization. */
MODE_NONE
};
};
/**
* Base class for reader implementation. When the input:Manager creates a new
@ -52,7 +52,8 @@ enum ReaderMode {
* All methods must be called only from the corresponding child thread (the
* constructor is the one exception.)
*/
class ReaderBackend : public threading::MsgThread {
class ReaderBackend : public threading::MsgThread
{
public:
/**
* Constructor.
@ -115,23 +116,25 @@ public:
name = other.name ? util::copy_string(other.name) : nullptr;
mode = other.mode;
for ( config_map::const_iterator i = other.config.begin(); i != other.config.end(); i++ )
config.insert(std::make_pair(util::copy_string(i->first), util::copy_string(i->second)));
for ( config_map::const_iterator i = other.config.begin(); i != other.config.end();
i++ )
config.insert(
std::make_pair(util::copy_string(i->first), util::copy_string(i->second)));
}
~ReaderInfo()
{
delete [] source;
delete [] name;
delete[] source;
delete[] name;
for ( config_map::iterator i = config.begin(); i != config.end(); i++ )
{
delete [] i->first;
delete [] i->second;
delete[] i->first;
delete[] i->second;
}
}
private:
private:
const ReaderInfo& operator=(const ReaderInfo& other); // Disable.
};
@ -172,17 +175,17 @@ public:
/**
* Returns the log fields as passed into the constructor.
*/
const threading::Field* const * Fields() const { return fields; }
const threading::Field* const* Fields() const { return fields; }
/**
* Returns the additional reader information into the constructor.
*/
const ReaderInfo& Info() const { return *info; }
const ReaderInfo& Info() const { return *info; }
/**
* Returns the number of log fields as passed into the constructor.
*/
int NumFields() const { return num_fields; }
int NumFields() const { return num_fields; }
/**
* Convenience function that calls Warning or Error, depending on the
@ -198,7 +201,7 @@ public:
* @param suppress_future If set to true, future warnings are suppressed
* until StopWarningSuppression is called.
*/
void FailWarn(bool is_error, const char *msg, bool suppress_future = false);
void FailWarn(bool is_error, const char* msg, bool suppress_future = false);
inline void StopWarningSuppression() { suppress_warnings = false; };
@ -246,7 +249,8 @@ protected:
* provides accessor methods to get them later, and they are passed
* in here only for convinience.
*/
virtual bool DoInit(const ReaderInfo& info, int arg_num_fields, const threading::Field* const* fields) = 0;
virtual bool DoInit(const ReaderInfo& info, int arg_num_fields,
const threading::Field* const* fields) = 0;
/**
* Reader-specific method implementing input finalization at
@ -356,12 +360,12 @@ private:
ReaderInfo* info;
unsigned int num_fields;
const threading::Field* const * fields; // raw mapping
const threading::Field* const* fields; // raw mapping
bool disabled;
// this is an internal indicator in case the read is currently in a failed state
// it's used to suppress duplicate error messages.
bool suppress_warnings = false;
};
};
} // namespace zeek::input
} // namespace zeek::input

33
src/input/ReaderFrontend.cc
View file

@ -5,35 +5,35 @@
#include "zeek/input/Manager.h"
#include "zeek/input/ReaderBackend.h"
namespace zeek::input {
namespace zeek::input
{
class InitMessage final : public threading::InputMessage<ReaderBackend>
{
{
public:
InitMessage(ReaderBackend* backend,
const int num_fields, const threading::Field* const* fields)
: threading::InputMessage<ReaderBackend>("Init", backend),
num_fields(num_fields), fields(fields) { }
bool Process() override
InitMessage(ReaderBackend* backend, const int num_fields, const threading::Field* const* fields)
: threading::InputMessage<ReaderBackend>("Init", backend), num_fields(num_fields),
fields(fields)
{
return Object()->Init(num_fields, fields);
}
bool Process() override { return Object()->Init(num_fields, fields); }
private:
const int num_fields;
const threading::Field* const* fields;
};
};
class UpdateMessage final : public threading::InputMessage<ReaderBackend>
{
{
public:
UpdateMessage(ReaderBackend* backend)
: threading::InputMessage<ReaderBackend>("Update", backend)
{ }
{
}
bool Process() override { return Object()->Update(); }
};
};
ReaderFrontend::ReaderFrontend(const ReaderBackend::ReaderInfo& arg_info, EnumVal* type)
{
@ -59,12 +59,11 @@ void ReaderFrontend::Stop()
ReaderFrontend::~ReaderFrontend()
{
delete [] name;
delete[] name;
delete info;
}
void ReaderFrontend::Init(const int arg_num_fields,
const threading::Field* const* arg_fields)
void ReaderFrontend::Init(const int arg_num_fields, const threading::Field* const* arg_fields)
{
if ( disabled )
return;
@ -98,4 +97,4 @@ const char* ReaderFrontend::Name() const
return name;
}
} // namespace zeek::input
} // namespace zeek::input

43
src/input/ReaderFrontend.h
View file

@ -5,11 +5,13 @@
#include "zeek/input/ReaderBackend.h"
#include "zeek/threading/SerialTypes.h"
namespace zeek {
namespace zeek
{
class EnumVal;
namespace input {
namespace input
{
class Manager;
@ -21,7 +23,8 @@ class Manager;
* spawns a new thread, and it receives messages from the frontend that
* correspond to method called by the manager.
*/
class ReaderFrontend {
class ReaderFrontend
{
public:
/**
* Constructor.
@ -89,13 +92,13 @@ public:
*
* This method must only be called from the main thread.
*/
void SetDisable() { disabled = true; }
void SetDisable() { disabled = true; }
/**
* Returns true if the reader frontend has been disabled with
* SetDisable().
*/
bool Disabled() { return disabled; }
bool Disabled() { return disabled; }
/**
* Returns a descriptive name for the reader, including the type of
@ -108,30 +111,34 @@ public:
/**
* Returns the additional reader information passed into the constructor.
*/
const ReaderBackend::ReaderInfo& Info() const { assert(info); return *info; }
const ReaderBackend::ReaderInfo& Info() const
{
assert(info);
return *info;
}
/**
* Returns the number of log fields as passed into the constructor.
*/
int NumFields() const { return num_fields; }
int NumFields() const { return num_fields; }
/**
* Returns the log fields as passed into the constructor.
*/
const threading::Field* const * Fields() const { return fields; }
const threading::Field* const* Fields() const { return fields; }
protected:
friend class Manager;
private:
ReaderBackend* backend; // The backend we have instanatiated.
ReaderBackend::ReaderInfo* info; // Meta information.
const threading::Field* const* fields; // The input fields.
int num_fields; // Information as passed to Init().
bool disabled; // True if disabled.
bool initialized; // True if initialized.
const char* name; // Descriptive name.
};
ReaderBackend* backend; // The backend we have instanatiated.
ReaderBackend::ReaderInfo* info; // Meta information.
const threading::Field* const* fields; // The input fields.
int num_fields; // Information as passed to Init().
bool disabled; // True if disabled.
bool initialized; // True if initialized.
const char* name; // Descriptive name.
};
} // namespace input
} // namespace zeek
} // namespace input
} // namespace zeek

15
src/input/Tag.cc
View file

@ -1,16 +1,15 @@
// See the file "COPYING" in the main distribution directory for copyright.
#include "zeek/input/Tag.h"
#include "zeek/input/Manager.h"
namespace zeek::input {
namespace zeek::input
{
const Tag Tag::Error;
Tag::Tag(type_t type, subtype_t subtype)
: zeek::Tag(input_mgr->GetTagType(), type, subtype)
{
}
Tag::Tag(type_t type, subtype_t subtype) : zeek::Tag(input_mgr->GetTagType(), type, subtype) { }
Tag& Tag::operator=(const Tag& other)
{
@ -23,8 +22,6 @@ const EnumValPtr& Tag::AsVal() const
return zeek::Tag::AsVal(input_mgr->GetTagType());
}
Tag::Tag(EnumValPtr val)
: zeek::Tag(std::move(val))
{ }
Tag::Tag(EnumValPtr val) : zeek::Tag(std::move(val)) { }
} // namespace zeek::input
} // namespace zeek::input

45
src/input/Tag.h
View file

@ -2,21 +2,24 @@
#pragma once
#include "zeek/zeek-config.h"
#include "zeek/Tag.h"
#include "zeek/zeek-config.h"
namespace zeek {
namespace zeek
{
class EnumVal;
namespace plugin {
namespace plugin
{
template <class T> class TaggedComponent;
template <class T, class C> class ComponentManager;
} // namespace plugin
} // namespace plugin
namespace input {
namespace input
{
class Manager;
class Component;
@ -26,29 +29,30 @@ class Component;
*
* The script-layer analogue is Input::Reader.
*/
class Tag : public zeek::Tag {
class Tag : public zeek::Tag
{
public:
/*
* Copy constructor.
*/
Tag(const Tag& other) : zeek::Tag(other) {}
Tag(const Tag& other) : zeek::Tag(other) { }
/**
* Default constructor. This initializes the tag with an error value
* that will make \c operator \c bool return false.
*/
Tag() : zeek::Tag() {}
Tag() : zeek::Tag() { }
/**
* Destructor.
*/
~Tag() {}
~Tag() { }
/**
* Returns false if the tag represents an error value rather than a
* legal reader type.
*/
explicit operator bool() const { return *this != Error; }
explicit operator bool() const { return *this != Error; }
/**
* Assignment operator.
@ -58,26 +62,17 @@ public:
/**
* Compares two tags for equality.
*/
bool operator==(const Tag& other) const
{
return zeek::Tag::operator==(other);
}
bool operator==(const Tag& other) const { return zeek::Tag::operator==(other); }
/**
* Compares two tags for inequality.
*/
bool operator!=(const Tag& other) const
{
return zeek::Tag::operator!=(other);
}
bool operator!=(const Tag& other) const { return zeek::Tag::operator!=(other); }
/**
* Compares two tags for less-than relationship.
*/
bool operator<(const Tag& other) const
{
return zeek::Tag::operator<(other);
}
bool operator<(const Tag& other) const { return zeek::Tag::operator<(other); }
/**
* Returns the \c Input::Reader enum that corresponds to this tag.
@ -111,7 +106,7 @@ protected:
* @param val An enum value of script type \c Input::Reader.
*/
explicit Tag(EnumValPtr val);
};
};
} // namespace input
} // namespace zeek
} // namespace input
} // namespace zeek

188
src/input/readers/ascii/Ascii.cc
View file

@ -2,21 +2,21 @@
#include "zeek/input/readers/ascii/Ascii.h"
#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>
#include <errno.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <unistd.h>
#include <sstream>
#include "zeek/input/readers/ascii/ascii.bif.h"
#include "zeek/threading/SerialTypes.h"
#include "zeek/input/readers/ascii/ascii.bif.h"
using namespace std;
using zeek::threading::Value;
using zeek::threading::Field;
using zeek::threading::Value;
namespace zeek::input::reader::detail {
namespace zeek::input::reader::detail
{
FieldMapping::FieldMapping(const string& arg_name, const TypeTag& arg_type, int arg_position)
: name(arg_name), type(arg_type), subtype(TYPE_ERROR)
@ -27,7 +27,7 @@ FieldMapping::FieldMapping(const string& arg_name, const TypeTag& arg_type, int
}
FieldMapping::FieldMapping(const string& arg_name, const TypeTag& arg_type,
const TypeTag& arg_subtype, int arg_position)
const TypeTag& arg_subtype, int arg_position)
: name(arg_name), type(arg_type), subtype(arg_subtype)
{
position = arg_position;
@ -47,7 +47,7 @@ FieldMapping FieldMapping::subType()
return FieldMapping(name, subtype, position);
}
Ascii::Ascii(ReaderFrontend *frontend) : ReaderBackend(frontend)
Ascii::Ascii(ReaderFrontend* frontend) : ReaderBackend(frontend)
{
mtime = 0;
ino = 0;
@ -55,38 +55,35 @@ Ascii::Ascii(ReaderFrontend *frontend) : ReaderBackend(frontend)
fail_on_invalid_lines = false;
}
Ascii::~Ascii()
{
}
Ascii::~Ascii() { }
void Ascii::DoClose()
{
}
void Ascii::DoClose() { }
bool Ascii::DoInit(const ReaderInfo& info, int num_fields, const Field* const* fields)
{
StopWarningSuppression();
separator.assign( (const char*) BifConst::InputAscii::separator->Bytes(),
separator.assign((const char*)BifConst::InputAscii::separator->Bytes(),
BifConst::InputAscii::separator->Len());
set_separator.assign( (const char*) BifConst::InputAscii::set_separator->Bytes(),
set_separator.assign((const char*)BifConst::InputAscii::set_separator->Bytes(),
BifConst::InputAscii::set_separator->Len());
empty_field.assign( (const char*) BifConst::InputAscii::empty_field->Bytes(),
empty_field.assign((const char*)BifConst::InputAscii::empty_field->Bytes(),
BifConst::InputAscii::empty_field->Len());
unset_field.assign( (const char*) BifConst::InputAscii::unset_field->Bytes(),
unset_field.assign((const char*)BifConst::InputAscii::unset_field->Bytes(),
BifConst::InputAscii::unset_field->Len());
fail_on_invalid_lines = BifConst::InputAscii::fail_on_invalid_lines;
fail_on_file_problem = BifConst::InputAscii::fail_on_file_problem;
path_prefix.assign((const char*) BifConst::InputAscii::path_prefix->Bytes(),
path_prefix.assign((const char*)BifConst::InputAscii::path_prefix->Bytes(),
BifConst::InputAscii::path_prefix->Len());
// Set per-filter configuration options.
for ( ReaderInfo::config_map::const_iterator i = info.config.begin(); i != info.config.end(); i++ )
for ( ReaderInfo::config_map::const_iterator i = info.config.begin(); i != info.config.end();
i++ )
{
if ( strcmp(i->first, "separator") == 0 )
separator.assign(i->second);
@ -113,13 +110,13 @@ bool Ascii::DoInit(const ReaderInfo& info, int num_fields, const Field* const* f
if ( set_separator.size() != 1 )
Error("set_separator length has to be 1. Separator will be truncated.");
threading::formatter::Ascii::SeparatorInfo sep_info(separator, set_separator, unset_field, empty_field);
threading::formatter::Ascii::SeparatorInfo sep_info(separator, set_separator, unset_field,
empty_field);
formatter = unique_ptr<threading::Formatter>(new threading::formatter::Ascii(this, sep_info));
return DoUpdate();
}
bool Ascii::OpenFile()
{
if ( file.is_open() )
@ -152,7 +149,8 @@ bool Ascii::OpenFile()
if ( ReadHeader(false) == false )
{
FailWarn(fail_on_file_problem, Fmt("Init: cannot open %s; problem reading file header", fname.c_str()), true);
FailWarn(fail_on_file_problem,
Fmt("Init: cannot open %s; problem reading file header", fname.c_str()), true);
file.close();
return ! fail_on_file_problem;
@ -172,8 +170,10 @@ bool Ascii::ReadHeader(bool useCached)
{
if ( ! GetLine(line) )
{
FailWarn(fail_on_file_problem, Fmt("Could not read input data file %s; first line could not be read",
fname.c_str()), true);
FailWarn(fail_on_file_problem,
Fmt("Could not read input data file %s; first line could not be read",
fname.c_str()),
true);
return false;
}
@ -185,11 +185,11 @@ bool Ascii::ReadHeader(bool useCached)
// construct list of field names.
istringstream splitstream(line);
int pos=0;
int pos = 0;
while ( splitstream )
{
string s;
if ( ! getline(splitstream, s, separator[0]))
if ( ! getline(splitstream, s, separator[0]) )
break;
ifields[s] = pos;
@ -208,15 +208,18 @@ bool Ascii::ReadHeader(bool useCached)
{
if ( field->optional )
{
// we do not really need this field. mark it as not present and always send an undef back.
// we do not really need this field. mark it as not present and always send an undef
// back.
FieldMapping f(field->name, field->type, field->subtype, -1);
f.present = false;
columnMap.push_back(f);
continue;
}
FailWarn(fail_on_file_problem, Fmt("Did not find requested field %s in input data file %s.",
field->name, fname.c_str()), true);
FailWarn(fail_on_file_problem,
Fmt("Did not find requested field %s in input data file %s.", field->name,
fname.c_str()),
true);
return false;
}
@ -228,8 +231,10 @@ bool Ascii::ReadHeader(bool useCached)
map<string, uint32_t>::iterator fit2 = ifields.find(field->secondary_name);
if ( fit2 == ifields.end() )
{
FailWarn(fail_on_file_problem, Fmt("Could not find requested port type field %s in input data file %s.",
field->secondary_name, fname.c_str()), true);
FailWarn(fail_on_file_problem,
Fmt("Could not find requested port type field %s in input data file %s.",
field->secondary_name, fname.c_str()),
true);
return false;
}
@ -257,7 +262,7 @@ bool Ascii::GetLine(string& str)
if ( str[0] != '#' )
return true;
if ( ( str.length() > 8 ) && ( str.compare(0,7, "#fields") == 0 ) && ( str[7] == separator[0] ) )
if ( (str.length() > 8) && (str.compare(0, 7, "#fields") == 0) && (str[7] == separator[0]) )
{
str = str.substr(8);
return true;
@ -273,63 +278,64 @@ bool Ascii::DoUpdate()
if ( ! OpenFile() )
return ! fail_on_file_problem;
switch ( Info().mode ) {
switch ( Info().mode )
{
case MODE_REREAD:
{
// check if the file has changed
struct stat sb;
if ( stat(fname.c_str(), &sb) == -1 )
{
FailWarn(fail_on_file_problem, Fmt("Could not get stat for %s", fname.c_str()), true);
// check if the file has changed
struct stat sb;
if ( stat(fname.c_str(), &sb) == -1 )
{
FailWarn(fail_on_file_problem, Fmt("Could not get stat for %s", fname.c_str()),
true);
file.close();
return ! fail_on_file_problem;
file.close();
return ! fail_on_file_problem;
}
if ( sb.st_ino == ino && sb.st_mtime == mtime )
// no change
return true;
// Warn again in case of trouble if the file changes. The comparison to 0
// is to suppress an extra warning that we'd otherwise get on the initial
// inode assignment.
if ( ino != 0 )
StopWarningSuppression();
mtime = sb.st_mtime;
ino = sb.st_ino;
// File changed. Fall through to re-read.
}
if ( sb.st_ino == ino && sb.st_mtime == mtime )
// no change
return true;
// Warn again in case of trouble if the file changes. The comparison to 0
// is to suppress an extra warning that we'd otherwise get on the initial
// inode assignment.
if ( ino != 0 )
StopWarningSuppression();
mtime = sb.st_mtime;
ino = sb.st_ino;
// File changed. Fall through to re-read.
}
case MODE_MANUAL:
case MODE_STREAM:
{
// dirty, fix me. (well, apparently after trying seeking, etc
// - this is not that bad)
if ( file.is_open() )
{
if ( Info().mode == MODE_STREAM )
// dirty, fix me. (well, apparently after trying seeking, etc
// - this is not that bad)
if ( file.is_open() )
{
file.clear(); // remove end of file evil bits
if ( ! ReadHeader(true) )
if ( Info().mode == MODE_STREAM )
{
return ! fail_on_file_problem; // header reading failed
file.clear(); // remove end of file evil bits
if ( ! ReadHeader(true) )
{
return ! fail_on_file_problem; // header reading failed
}
break;
}
break;
file.close();
}
file.close();
OpenFile();
break;
}
OpenFile();
break;
}
default:
assert(false);
}
string line;
@ -359,9 +365,8 @@ bool Ascii::DoUpdate()
Value** fields = new Value*[NumFields()];
int fpos = 0;
for ( vector<FieldMapping>::iterator fit = columnMap.begin();
fit != columnMap.end();
fit++ )
for ( vector<FieldMapping>::iterator fit = columnMap.begin(); fit != columnMap.end();
fit++ )
{
if ( ! fit->present )
@ -372,19 +377,21 @@ bool Ascii::DoUpdate()
continue;
}
assert(fit->position >= 0 );
assert(fit->position >= 0);
if ( (*fit).position > pos || (*fit).secondary_position > pos )
{
FailWarn(fail_on_invalid_lines, Fmt("Not enough fields in line '%s' of %s. Found %d fields, want positions %d and %d",
line.c_str(), fname.c_str(), pos, (*fit).position, (*fit).secondary_position));
FailWarn(fail_on_invalid_lines, Fmt("Not enough fields in line '%s' of %s. Found "
"%d fields, want positions %d and %d",
line.c_str(), fname.c_str(), pos,
(*fit).position, (*fit).secondary_position));
if ( fail_on_invalid_lines )
{
for ( int i = 0; i < fpos; i++ )
delete fields[i];
delete [] fields;
delete[] fields;
return false;
}
@ -395,11 +402,13 @@ bool Ascii::DoUpdate()
}
}
Value* val = formatter->ParseValue(stringfields[(*fit).position], (*fit).name, (*fit).type, (*fit).subtype);
Value* val = formatter->ParseValue(stringfields[(*fit).position], (*fit).name,
(*fit).type, (*fit).subtype);
if ( ! val )
{
Warning(Fmt("Could not convert line '%s' of %s to Val. Ignoring line.", line.c_str(), fname.c_str()));
Warning(Fmt("Could not convert line '%s' of %s to Val. Ignoring line.",
line.c_str(), fname.c_str()));
error = true;
break;
}
@ -407,10 +416,11 @@ bool Ascii::DoUpdate()
if ( (*fit).secondary_position != -1 )
{
// we have a port definition :)
assert(val->type == TYPE_PORT );
assert(val->type == TYPE_PORT);
// Error(Fmt("Got type %d != PORT with secondary position!", val->type));
val->val.port_val.proto = formatter->ParseProto(stringfields[(*fit).secondary_position]);
val->val.port_val.proto =
formatter->ParseProto(stringfields[(*fit).secondary_position]);
}
fields[fpos] = val;
@ -427,12 +437,12 @@ bool Ascii::DoUpdate()
for ( int i = 0; i < fpos; i++ )
delete fields[i];
delete [] fields;
delete[] fields;
continue;
}
//printf("fpos: %d, second.num_fields: %d\n", fpos, (*it).second.num_fields);
assert ( fpos == NumFields() );
// printf("fpos: %d, second.num_fields: %d\n", fpos, (*it).second.num_fields);
assert(fpos == NumFields());
if ( Info().mode == MODE_STREAM )
Put(fields);
@ -460,7 +470,7 @@ bool Ascii::DoHeartbeat(double network_time, double current_time)
case MODE_REREAD:
case MODE_STREAM:
Update(); // Call Update, not DoUpdate, because Update
// checks the "disabled" flag.
// checks the "disabled" flag.
break;
default:
@ -470,4 +480,4 @@ bool Ascii::DoHeartbeat(double network_time, double current_time)
return true;
}
} // namespace zeek::input::reader::detail
} // namespace zeek::input::reader::detail

31
src/input/readers/ascii/Ascii.h
View file

@ -3,18 +3,20 @@
#pragma once
#include <sys/types.h>
#include <iostream>
#include <vector>
#include <fstream>
#include <iostream>
#include <memory>
#include <vector>
#include "zeek/input/ReaderBackend.h"
#include "zeek/threading/formatters/Ascii.h"
namespace zeek::input::reader::detail {
namespace zeek::input::reader::detail
{
// Description for input field mapping.
struct FieldMapping {
struct FieldMapping
{
std::string name;
TypeTag type;
TypeTag subtype; // internal type for sets and vectors
@ -23,18 +25,24 @@ struct FieldMapping {
bool present;
FieldMapping(const std::string& arg_name, const TypeTag& arg_type, int arg_position);
FieldMapping(const std::string& arg_name, const TypeTag& arg_type, const TypeTag& arg_subtype, int arg_position);
FieldMapping(const std::string& arg_name, const TypeTag& arg_type, const TypeTag& arg_subtype,
int arg_position);
FieldMapping(const FieldMapping& arg);
FieldMapping() { position = -1; secondary_position = -1; }
FieldMapping()
{
position = -1;
secondary_position = -1;
}
FieldMapping subType();
};
};
/**
* Reader for structured ASCII files.
*/
class Ascii : public ReaderBackend {
class Ascii : public ReaderBackend
{
public:
explicit Ascii(ReaderFrontend* frontend);
~Ascii() override;
@ -48,7 +56,8 @@ public:
static ReaderBackend* Instantiate(ReaderFrontend* frontend) { return new Ascii(frontend); }
protected:
bool DoInit(const ReaderInfo& info, int arg_num_fields, const threading::Field* const* fields) override;
bool DoInit(const ReaderInfo& info, int arg_num_fields,
const threading::Field* const* fields) override;
void DoClose() override;
bool DoUpdate() override;
bool DoHeartbeat(double network_time, double current_time) override;
@ -83,6 +92,6 @@ private:
std::string path_prefix;
std::unique_ptr<threading::Formatter> formatter;
};
};
} // namespace zeek::input::reader::detail
} // namespace zeek::input::reader::detail

14
src/input/readers/ascii/Plugin.cc
View file

@ -1,21 +1,25 @@
// See the file in the main distribution directory for copyright.
#include "zeek/plugin/Plugin.h"
#include "zeek/input/readers/ascii/Ascii.h"
namespace zeek::plugin::detail::Zeek_AsciiReader {
namespace zeek::plugin::detail::Zeek_AsciiReader
{
class Plugin : public zeek::plugin::Plugin {
class Plugin : public zeek::plugin::Plugin
{
public:
zeek::plugin::Configuration Configure() override
{
AddComponent(new zeek::input::Component("Ascii", zeek::input::reader::detail::Ascii::Instantiate));
AddComponent(
new zeek::input::Component("Ascii", zeek::input::reader::detail::Ascii::Instantiate));
zeek::plugin::Configuration config;
config.name = "Zeek::AsciiReader";
config.description = "ASCII input reader";
return config;
}
} plugin;
} plugin;
} // namespace zeek::plugin::detail::Zeek_AsciiReader
} // namespace zeek::plugin::detail::Zeek_AsciiReader

258
src/input/readers/benchmark/Benchmark.cc
View file

@ -2,22 +2,22 @@
#include "zeek/input/readers/benchmark/Benchmark.h"
#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>
#include <errno.h>
#include "zeek/threading/SerialTypes.h"
#include "zeek/threading/Manager.h"
#include <sys/stat.h>
#include <sys/types.h>
#include <unistd.h>
#include "zeek/input/readers/benchmark/benchmark.bif.h"
#include "zeek/threading/Manager.h"
#include "zeek/threading/SerialTypes.h"
using zeek::threading::Value;
using zeek::threading::Field;
using zeek::threading::Value;
namespace zeek::input::reader::detail {
namespace zeek::input::reader::detail
{
Benchmark::Benchmark(ReaderFrontend *frontend) : ReaderBackend(frontend)
Benchmark::Benchmark(ReaderFrontend* frontend) : ReaderBackend(frontend)
{
num_lines = 0;
multiplication_factor = double(BifConst::InputBenchmark::factor);
@ -40,16 +40,15 @@ Benchmark::~Benchmark()
delete ascii;
}
void Benchmark::DoClose()
{
}
void Benchmark::DoClose() { }
bool Benchmark::DoInit(const ReaderInfo& info, int num_fields, const threading::Field* const* fields)
bool Benchmark::DoInit(const ReaderInfo& info, int num_fields,
const threading::Field* const* fields)
{
num_lines = atoi(info.source);
if ( autospread != 0.0 )
autospread_time = (int) ( (double) 1000000 / (autospread * (double) num_lines) );
autospread_time = (int)((double)1000000 / (autospread * (double)num_lines));
heartbeatstarttime = CurrTime();
DoUpdate();
@ -61,12 +60,11 @@ std::string Benchmark::RandomString(const int len)
{
std::string s(len, ' ');
static const char values[] =
"0123456789!@#$%^&*()-_=+{}[]\\|"
"ABCDEFGHIJKLMNOPQRSTUVWXYZ"
"abcdefghijklmnopqrstuvwxyz";
static const char values[] = "0123456789!@#$%^&*()-_=+{}[]\\|"
"ABCDEFGHIJKLMNOPQRSTUVWXYZ"
"abcdefghijklmnopqrstuvwxyz";
for (int i = 0; i < len; ++i)
for ( int i = 0; i < len; ++i )
s[i] = values[random() / (RAND_MAX / sizeof(values))];
return s;
@ -75,14 +73,14 @@ std::string Benchmark::RandomString(const int len)
double Benchmark::CurrTime()
{
struct timeval tv;
if ( gettimeofday(&tv, 0) != 0 ) {
if ( gettimeofday(&tv, 0) != 0 )
{
FatalError(Fmt("Could not get time: %d", errno));
}
}
return double(tv.tv_sec) + double(tv.tv_usec) / 1e6;
}
// read the entire file and send appropriate thingies back to InputMgr
bool Benchmark::DoUpdate()
{
@ -90,10 +88,10 @@ bool Benchmark::DoUpdate()
for ( int i = 0; i < linestosend; i++ )
{
threading::Value** field = new threading::Value*[NumFields()];
for (int j = 0; j < NumFields(); j++ )
for ( int j = 0; j < NumFields(); j++ )
field[j] = EntryToVal(Fields()[j]->type, Fields()[j]->subtype);
if ( Info().mode == MODE_STREAM )
if ( Info().mode == MODE_STREAM )
// do not do tracking, spread out elements over the second that we have...
Put(field);
else
@ -105,7 +103,7 @@ bool Benchmark::DoUpdate()
usleep(spread);
if ( autospread_time != 0 )
usleep( autospread_time );
usleep(autospread_time);
}
if ( timedspread != 0.0 )
@ -113,17 +111,16 @@ bool Benchmark::DoUpdate()
double diff;
do
diff = CurrTime() - heartbeatstarttime;
while ( diff/heartbeat_interval < i/(linestosend
+ (linestosend * timedspread) ) );
while ( diff / heartbeat_interval <
i / (linestosend + (linestosend * timedspread)) );
}
}
}
if ( Info().mode != MODE_STREAM )
EndCurrentSend();
return true;
}
}
threading::Value* Benchmark::EntryToVal(TypeTag type, TypeTag subtype)
{
@ -131,117 +128,116 @@ threading::Value* Benchmark::EntryToVal(TypeTag type, TypeTag subtype)
// basically construct something random from the fields that we want.
switch ( type ) {
case TYPE_ENUM:
assert(false); // no enums, please.
case TYPE_STRING:
switch ( type )
{
std::string rnd = RandomString(10);
val->val.string_val.data = util::copy_string(rnd.c_str());
val->val.string_val.length = rnd.size();
break;
}
case TYPE_ENUM:
assert(false); // no enums, please.
case TYPE_BOOL:
val->val.int_val = 1; // we never lie.
break;
case TYPE_INT:
val->val.int_val = random();
break;
case TYPE_TIME:
val->val.double_val = CurrTime();
break;
case TYPE_DOUBLE:
case TYPE_INTERVAL:
val->val.double_val = random();
break;
case TYPE_COUNT:
val->val.uint_val = random();
break;
case TYPE_PORT:
val->val.port_val.port = random() / (RAND_MAX / 60000);
val->val.port_val.proto = TRANSPORT_UNKNOWN;
break;
case TYPE_SUBNET:
{
val->val.subnet_val.prefix = ascii->ParseAddr("192.168.17.1");
val->val.subnet_val.length = 16;
}
break;
case TYPE_ADDR:
val->val.addr_val = ascii->ParseAddr("192.168.17.1");
break;
case TYPE_TABLE:
case TYPE_VECTOR:
// First - common initialization
// Then - initialization for table.
// Then - initialization for vector.
// Then - common stuff
{
// how many entries do we have...
unsigned int length = random() / (RAND_MAX / 15);
threading::Value** lvals = new threading::Value* [length];
if ( type == TYPE_TABLE )
{
val->val.set_val.vals = lvals;
val->val.set_val.size = length;
}
else if ( type == TYPE_VECTOR )
{
val->val.vector_val.vals = lvals;
val->val.vector_val.size = length;
}
else
assert(false);
if ( length == 0 )
break; //empty
for ( unsigned int pos = 0; pos < length; pos++ )
{
threading::Value* newval = EntryToVal(subtype, TYPE_ENUM);
if ( newval == nullptr )
case TYPE_STRING:
{
Error("Error while reading set");
delete val;
return nullptr;
std::string rnd = RandomString(10);
val->val.string_val.data = util::copy_string(rnd.c_str());
val->val.string_val.length = rnd.size();
break;
}
lvals[pos] = newval;
}
break;
case TYPE_BOOL:
val->val.int_val = 1; // we never lie.
break;
case TYPE_INT:
val->val.int_val = random();
break;
case TYPE_TIME:
val->val.double_val = CurrTime();
break;
case TYPE_DOUBLE:
case TYPE_INTERVAL:
val->val.double_val = random();
break;
case TYPE_COUNT:
val->val.uint_val = random();
break;
case TYPE_PORT:
val->val.port_val.port = random() / (RAND_MAX / 60000);
val->val.port_val.proto = TRANSPORT_UNKNOWN;
break;
case TYPE_SUBNET:
{
val->val.subnet_val.prefix = ascii->ParseAddr("192.168.17.1");
val->val.subnet_val.length = 16;
}
break;
case TYPE_ADDR:
val->val.addr_val = ascii->ParseAddr("192.168.17.1");
break;
case TYPE_TABLE:
case TYPE_VECTOR:
// First - common initialization
// Then - initialization for table.
// Then - initialization for vector.
// Then - common stuff
{
// how many entries do we have...
unsigned int length = random() / (RAND_MAX / 15);
threading::Value** lvals = new threading::Value*[length];
if ( type == TYPE_TABLE )
{
val->val.set_val.vals = lvals;
val->val.set_val.size = length;
}
else if ( type == TYPE_VECTOR )
{
val->val.vector_val.vals = lvals;
val->val.vector_val.size = length;
}
else
assert(false);
if ( length == 0 )
break; // empty
for ( unsigned int pos = 0; pos < length; pos++ )
{
threading::Value* newval = EntryToVal(subtype, TYPE_ENUM);
if ( newval == nullptr )
{
Error("Error while reading set");
delete val;
return nullptr;
}
lvals[pos] = newval;
}
break;
}
default:
Error(Fmt("unsupported field format %d", type));
delete val;
return nullptr;
}
default:
Error(Fmt("unsupported field format %d", type));
delete val;
return nullptr;
}
return val;
}
bool Benchmark::DoHeartbeat(double network_time, double current_time)
{
num_lines = (int) ( (double) num_lines*multiplication_factor);
{
num_lines = (int)((double)num_lines * multiplication_factor);
num_lines += add;
heartbeatstarttime = CurrTime();
switch ( Info().mode ) {
switch ( Info().mode )
{
case MODE_MANUAL:
// yay, we do nothing :)
break;
@ -262,7 +258,7 @@ bool Benchmark::DoHeartbeat(double network_time, double current_time)
if ( autospread != 0.0 )
// because executing this in every loop is apparently too expensive.
autospread_time = (int) ( (double) 1000000 / (autospread * (double) num_lines) );
autospread_time = (int)((double)1000000 / (autospread * (double)num_lines));
Update(); // call update and not DoUpdate, because update actually checks disabled.
@ -271,9 +267,9 @@ bool Benchmark::DoHeartbeat(double network_time, double current_time)
default:
assert(false);
}
}
return true;
}
}
} // namespace zeek::input::reader::detail
} // namespace zeek::input::reader::detail

13
src/input/readers/benchmark/Benchmark.h
View file

@ -5,12 +5,14 @@
#include "zeek/input/ReaderBackend.h"
#include "zeek/threading/formatters/Ascii.h"
namespace zeek::input::reader::detail {
namespace zeek::input::reader::detail
{
/**
* A benchmark reader to measure performance of the input framework.
*/
class Benchmark : public ReaderBackend {
class Benchmark : public ReaderBackend
{
public:
explicit Benchmark(ReaderFrontend* frontend);
~Benchmark() override;
@ -18,7 +20,8 @@ public:
static ReaderBackend* Instantiate(ReaderFrontend* frontend) { return new Benchmark(frontend); }
protected:
bool DoInit(const ReaderInfo& info, int arg_num_fields, const threading::Field* const* fields) override;
bool DoInit(const ReaderInfo& info, int arg_num_fields,
const threading::Field* const* fields) override;
void DoClose() override;
bool DoUpdate() override;
bool DoHeartbeat(double network_time, double current_time) override;
@ -40,6 +43,6 @@ private:
double heartbeat_interval;
threading::formatter::Ascii* ascii;
};
};
} // namespace zeek::input::reader
} // namespace zeek::input::reader

14
src/input/readers/benchmark/Plugin.cc
View file

@ -1,21 +1,25 @@
// See the file in the main distribution directory for copyright.
#include "zeek/plugin/Plugin.h"
#include "zeek/input/readers/benchmark/Benchmark.h"
namespace zeek::plugin::detail::Zeek_BenchmarkReader {
namespace zeek::plugin::detail::Zeek_BenchmarkReader
{
class Plugin : public zeek::plugin::Plugin {
class Plugin : public zeek::plugin::Plugin
{
public:
zeek::plugin::Configuration Configure() override
{
AddComponent(new zeek::input::Component("Benchmark", zeek::input::reader::detail::Benchmark::Instantiate));
AddComponent(new zeek::input::Component(
"Benchmark", zeek::input::reader::detail::Benchmark::Instantiate));
zeek::plugin::Configuration config;
config.name = "Zeek::BenchmarkReader";
config.description = "Benchmark input reader";
return config;
}
} plugin;
} plugin;
} // namespace zeek::plugin::detail::Zeek_BenchmarkReader
} // namespace zeek::plugin::detail::Zeek_BenchmarkReader

99
src/input/readers/binary/Binary.cc
View file

@ -4,19 +4,19 @@
#include <sys/stat.h>
#include "zeek/input/readers/binary/binary.bif.h"
#include "zeek/threading/SerialTypes.h"
#include "zeek/input/readers/binary/binary.bif.h"
using namespace std;
using zeek::threading::Value;
using zeek::threading::Field;
using zeek::threading::Value;
namespace zeek::input::reader::detail {
namespace zeek::input::reader::detail
{
streamsize Binary::chunk_size = 0;
Binary::Binary(ReaderFrontend *frontend)
Binary::Binary(ReaderFrontend* frontend)
: ReaderBackend(frontend), in(nullptr), mtime(0), ino(0), firstrun(true)
{
if ( ! chunk_size )
@ -56,8 +56,7 @@ bool Binary::CloseInput()
{
if ( ! in || ! in->is_open() )
{
InternalWarning(Fmt("Trying to close closed file for stream %s",
fname.c_str()));
InternalWarning(Fmt("Trying to close closed file for stream %s", fname.c_str()));
return false;
}
@ -76,15 +75,14 @@ bool Binary::CloseInput()
return true;
}
bool Binary::DoInit(const ReaderInfo& info, int num_fields,
const Field* const* fields)
bool Binary::DoInit(const ReaderInfo& info, int num_fields, const Field* const* fields)
{
in = nullptr;
mtime = 0;
ino = 0;
firstrun = true;
path_prefix.assign((const char*) BifConst::InputBinary::path_prefix->Bytes(),
path_prefix.assign((const char*)BifConst::InputBinary::path_prefix->Bytes(),
BifConst::InputBinary::path_prefix->Len());
if ( ! info.source || strlen(info.source) == 0 )
@ -160,7 +158,7 @@ streamsize Binary::GetChunk(char** chunk)
if ( ! bytes_read )
{
delete [] *chunk;
delete[] * chunk;
*chunk = nullptr;
return 0;
}
@ -198,47 +196,49 @@ bool Binary::DoUpdate()
else
{
switch ( Info().mode ) {
case MODE_REREAD:
switch ( Info().mode )
{
switch ( UpdateModificationTime() ) {
case -1:
return false; // error
case 0:
return true; // no change
case 1:
break; // file changed. reread.
case MODE_REREAD:
{
switch ( UpdateModificationTime() )
{
case -1:
return false; // error
case 0:
return true; // no change
case 1:
break; // file changed. reread.
default:
assert(false);
}
// fallthrough
}
case MODE_MANUAL:
case MODE_STREAM:
if ( Info().mode == MODE_STREAM && in )
{
in->clear(); // remove end of file evil bits
break;
}
CloseInput();
if ( ! OpenInput() )
return false;
break;
default:
assert(false);
}
// fallthrough
}
case MODE_MANUAL:
case MODE_STREAM:
if ( Info().mode == MODE_STREAM && in )
{
in->clear(); // remove end of file evil bits
break;
}
CloseInput();
if ( ! OpenInput() )
return false;
break;
default:
assert(false);
}
}
char* chunk = nullptr;
streamsize size = 0;
while ( (size = GetChunk(&chunk)) )
{
assert (NumFields() == 1);
assert(NumFields() == 1);
Value** fields = new Value*[1];
@ -266,7 +266,8 @@ bool Binary::DoUpdate()
bool Binary::DoHeartbeat(double network_time, double current_time)
{
switch ( Info().mode ) {
switch ( Info().mode )
{
case MODE_MANUAL:
// yay, we do nothing :)
break;
@ -274,19 +275,19 @@ bool Binary::DoHeartbeat(double network_time, double current_time)
case MODE_REREAD:
case MODE_STREAM:
#ifdef DEBUG
Debug(DBG_INPUT, "Starting Heartbeat update");
Debug(DBG_INPUT, "Starting Heartbeat update");
#endif
Update(); // call update and not DoUpdate, because update
// checks disabled.
Update(); // call update and not DoUpdate, because update
// checks disabled.
#ifdef DEBUG
Debug(DBG_INPUT, "Finished with heartbeat update");
Debug(DBG_INPUT, "Finished with heartbeat update");
#endif
break;
default:
assert(false);
}
}
return true;
}
} // namespace zeek::input::reader::detail
} // namespace zeek::input::reader::detail

15
src/input/readers/binary/Binary.h
View file

@ -2,23 +2,24 @@
#pragma once
#include <fstream>
#include <sys/types.h>
#include <fstream>
#include "zeek/input/ReaderBackend.h"
namespace zeek::input::reader::detail {
namespace zeek::input::reader::detail
{
/**
* Binary mode file reader.
*/
class Binary : public ReaderBackend {
class Binary : public ReaderBackend
{
public:
explicit Binary(ReaderFrontend* frontend);
~Binary() override;
static ReaderBackend* Instantiate(ReaderFrontend* frontend)
{ return new Binary(frontend); }
static ReaderBackend* Instantiate(ReaderFrontend* frontend) { return new Binary(frontend); }
protected:
bool DoInit(const ReaderInfo& info, int arg_num_fields,
@ -42,6 +43,6 @@ private:
// options set from the script-level.
static std::streamsize chunk_size;
std::string path_prefix;
};
};
} // namespace zeek::input::reader::detail
} // namespace zeek::input::reader::detail

14
src/input/readers/binary/Plugin.cc
View file

@ -1,21 +1,25 @@
// See the file in the main distribution directory for copyright.
#include "zeek/plugin/Plugin.h"
#include "zeek/input/readers/binary/Binary.h"
namespace zeek::plugin::detail::Zeek_BinaryReader {
namespace zeek::plugin::detail::Zeek_BinaryReader
{
class Plugin : public zeek::plugin::Plugin {
class Plugin : public zeek::plugin::Plugin
{
public:
zeek::plugin::Configuration Configure() override
{
AddComponent(new zeek::input::Component("Binary", zeek::input::reader::detail::Binary::Instantiate));
AddComponent(
new zeek::input::Component("Binary", zeek::input::reader::detail::Binary::Instantiate));
zeek::plugin::Configuration config;
config.name = "Zeek::BinaryReader";
config.description = "Binary input reader";
return config;
}
} plugin;
} plugin;
} // namespace zeek::plugin::detail::Zeek_BinaryReader
} // namespace zeek::plugin::detail::Zeek_BinaryReader

126
src/input/readers/config/Config.cc
View file

@ -2,26 +2,26 @@
#include "zeek/input/readers/config/Config.h"
#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>
#include <errno.h>
#include <regex.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <unistd.h>
#include <sstream>
#include <unordered_set>
#include "zeek/Desc.h"
#include "zeek/input/Manager.h"
#include "zeek/input/readers/config/config.bif.h"
#include "zeek/threading/SerialTypes.h"
#include "zeek/input/readers/config/config.bif.h"
using zeek::threading::Value;
using zeek::threading::Field;
using zeek::threading::Value;
namespace zeek::input::reader::detail {
namespace zeek::input::reader::detail
{
Config::Config(ReaderFrontend *frontend) : ReaderBackend(frontend)
Config::Config(ReaderFrontend* frontend) : ReaderBackend(frontend)
{
mtime = 0;
ino = 0;
@ -54,27 +54,23 @@ Config::Config(ReaderFrontend *frontend) : ReaderBackend(frontend)
}
}
Config::~Config()
{
}
Config::~Config() { }
void Config::DoClose()
{
}
void Config::DoClose() { }
bool Config::DoInit(const ReaderInfo& info, int num_fields, const Field* const* fields)
{
fail_on_file_problem = BifConst::InputConfig::fail_on_file_problem;
set_separator.assign( (const char*) BifConst::InputConfig::set_separator->Bytes(),
set_separator.assign((const char*)BifConst::InputConfig::set_separator->Bytes(),
BifConst::InputConfig::set_separator->Len());
empty_field.assign( (const char*) BifConst::InputConfig::empty_field->Bytes(),
empty_field.assign((const char*)BifConst::InputConfig::empty_field->Bytes(),
BifConst::InputConfig::empty_field->Len());
threading::formatter::Ascii::SeparatorInfo sep_info("\t", set_separator, "", empty_field);
formatter = std::unique_ptr<threading::Formatter>(
new threading::formatter::Ascii(this, sep_info));
formatter =
std::unique_ptr<threading::Formatter>(new threading::formatter::Ascii(this, sep_info));
return DoUpdate();
}
@ -119,55 +115,57 @@ bool Config::DoUpdate()
if ( ! OpenFile() )
return ! fail_on_file_problem;
switch ( Info().mode ) {
switch ( Info().mode )
{
case MODE_REREAD:
{
// check if the file has changed
struct stat sb;
if ( stat(Info().source, &sb) == -1 )
{
FailWarn(fail_on_file_problem, Fmt("Could not get stat for %s", Info().source), true);
// check if the file has changed
struct stat sb;
if ( stat(Info().source, &sb) == -1 )
{
FailWarn(fail_on_file_problem, Fmt("Could not get stat for %s", Info().source),
true);
file.close();
return ! fail_on_file_problem;
file.close();
return ! fail_on_file_problem;
}
if ( sb.st_ino == ino && sb.st_mtime == mtime )
// no change
return true;
// Warn again in case of trouble if the file changes. The comparison to 0
// is to suppress an extra warning that we'd otherwise get on the initial
// inode assignment.
if ( ino != 0 )
StopWarningSuppression();
mtime = sb.st_mtime;
ino = sb.st_ino;
// File changed. Fall through to re-read.
}
if ( sb.st_ino == ino && sb.st_mtime == mtime )
// no change
return true;
// Warn again in case of trouble if the file changes. The comparison to 0
// is to suppress an extra warning that we'd otherwise get on the initial
// inode assignment.
if ( ino != 0 )
StopWarningSuppression();
mtime = sb.st_mtime;
ino = sb.st_ino;
// File changed. Fall through to re-read.
}
case MODE_MANUAL:
case MODE_STREAM:
{
// dirty, fix me. (well, apparently after trying seeking, etc
// - this is not that bad)
if ( file.is_open() )
{
if ( Info().mode == MODE_STREAM )
// dirty, fix me. (well, apparently after trying seeking, etc
// - this is not that bad)
if ( file.is_open() )
{
file.clear(); // remove end of file evil bits
break;
if ( Info().mode == MODE_STREAM )
{
file.clear(); // remove end of file evil bits
break;
}
file.close();
}
file.close();
OpenFile();
break;
}
OpenFile();
break;
}
default:
assert(false);
}
@ -184,7 +182,8 @@ bool Config::DoUpdate()
}
regex_t re;
if ( regcomp(&re, "^([^[:blank:]]+)[[:blank:]]+([^[:blank:]](.*[^[:blank:]])?)?[[:blank:]]*$", REG_EXTENDED) )
if ( regcomp(&re, "^([^[:blank:]]+)[[:blank:]]+([^[:blank:]](.*[^[:blank:]])?)?[[:blank:]]*$",
REG_EXTENDED) )
{
Error(Fmt("Failed to compile regex."));
return true;
@ -195,7 +194,8 @@ bool Config::DoUpdate()
regmatch_t match[3];
if ( regexec(&re, line.c_str(), 3, match, 0) )
{
Warning(Fmt("Could not parse '%s'; line has invalid format. Ignoring line.", line.c_str()));
Warning(
Fmt("Could not parse '%s'; line has invalid format. Ignoring line.", line.c_str()));
continue;
}
@ -215,12 +215,14 @@ bool Config::DoUpdate()
{
ODesc d;
std::get<2>((*typeit).second)->GetType()->Describe(&d);
Warning(Fmt("Option '%s' has type '%s', which is not supported for file input. Ignoring line.",
key.c_str(), d.Description()));
Warning(Fmt(
"Option '%s' has type '%s', which is not supported for file input. Ignoring line.",
key.c_str(), d.Description()));
continue;
}
Value* eventval = formatter->ParseValue(value, key, std::get<0>((*typeit).second), std::get<1>((*typeit).second));
Value* eventval = formatter->ParseValue(value, key, std::get<0>((*typeit).second),
std::get<1>((*typeit).second));
if ( ! eventval )
{
Warning(Fmt("Could not convert line '%s' to value. Ignoring line.", line.c_str()));
@ -258,7 +260,7 @@ bool Config::DoUpdate()
val->val.string_val.data = util::copy_string(value.c_str());
fields[1] = val;
if ( Info().mode == MODE_STREAM )
if ( Info().mode == MODE_STREAM )
Put(fields);
else
SendEntry(fields);
@ -304,7 +306,7 @@ bool Config::DoHeartbeat(double network_time, double current_time)
case MODE_REREAD:
case MODE_STREAM:
Update(); // Call Update, not DoUpdate, because Update
// checks the "disabled" flag.
// checks the "disabled" flag.
break;
default:
@ -314,4 +316,4 @@ bool Config::DoHeartbeat(double network_time, double current_time)
return true;
}
} // namespace zeek::input::reader::detail
} // namespace zeek::input::reader::detail

17
src/input/readers/config/Config.h
View file

@ -3,22 +3,24 @@
#pragma once
#include <sys/types.h>
#include <iostream>
#include <vector>
#include <fstream>
#include <iostream>
#include <memory>
#include <unordered_map>
#include <vector>
#include "zeek/ID.h"
#include "zeek/input/ReaderBackend.h"
#include "zeek/threading/formatters/Ascii.h"
namespace zeek::input::reader::detail {
namespace zeek::input::reader::detail
{
/**
* Reader for Configuration files.
*/
class Config : public ReaderBackend {
class Config : public ReaderBackend
{
public:
explicit Config(ReaderFrontend* frontend);
~Config() override;
@ -32,7 +34,8 @@ public:
static ReaderBackend* Instantiate(ReaderFrontend* frontend) { return new Config(frontend); }
protected:
bool DoInit(const ReaderInfo& info, int arg_num_fields, const threading::Field* const* fields) override;
bool DoInit(const ReaderInfo& info, int arg_num_fields,
const threading::Field* const* fields) override;
void DoClose() override;
bool DoUpdate() override;
bool DoHeartbeat(double network_time, double current_time) override;
@ -53,6 +56,6 @@ private:
std::unique_ptr<threading::Formatter> formatter;
std::unordered_map<std::string, std::tuple<TypeTag, TypeTag, zeek::detail::IDPtr>> option_types;
std::unordered_map<std::string, std::string> option_values;
};
};
} // namespace zeek::input::reader::detail
} // namespace zeek::input::reader::detail

14
src/input/readers/config/Plugin.cc
View file

@ -1,21 +1,25 @@
// See the file in the main distribution directory for copyright.
#include "zeek/plugin/Plugin.h"
#include "zeek/input/readers/config/Config.h"
namespace zeek::plugin::detail::Zeek_ConfigReader {
namespace zeek::plugin::detail::Zeek_ConfigReader
{
class Plugin : public zeek::plugin::Plugin {
class Plugin : public zeek::plugin::Plugin
{
public:
zeek::plugin::Configuration Configure() override
{
AddComponent(new zeek::input::Component("Config", zeek::input::reader::detail::Config::Instantiate));
AddComponent(
new zeek::input::Component("Config", zeek::input::reader::detail::Config::Instantiate));
zeek::plugin::Configuration config;
config.name = "Zeek::ConfigReader";
config.description = "Configuration file input reader";
return config;
}
} plugin;
} plugin;
} // namespace zeek::plugin::detail::Zeek_ConfigReader
} // namespace zeek::plugin::detail::Zeek_ConfigReader

17
src/input/readers/raw/Plugin.cc
View file

@ -2,13 +2,12 @@
#include "zeek/input/readers/raw/Plugin.h"
namespace zeek::plugin::detail::Zeek_RawReader {
namespace zeek::plugin::detail::Zeek_RawReader
{
Plugin plugin;
Plugin::Plugin()
{
}
Plugin::Plugin() { }
zeek::plugin::Configuration Plugin::Configure()
{
@ -20,17 +19,13 @@ zeek::plugin::Configuration Plugin::Configure()
return config;
}
void Plugin::InitPreScript()
{
}
void Plugin::InitPreScript() { }
void Plugin::Done()
{
}
void Plugin::Done() { }
std::unique_lock<std::mutex> Plugin::ForkMutex()
{
return std::unique_lock<std::mutex>(fork_mutex, std::defer_lock);
}
} // namespace zeek::plugin::detail::Zeek_RawReader
} // namespace zeek::plugin::detail::Zeek_RawReader

13
src/input/readers/raw/Plugin.h
View file

@ -4,12 +4,14 @@
#include <mutex>
#include "zeek/plugin/Plugin.h"
#include "zeek/input/readers/raw/Raw.h"
#include "zeek/plugin/Plugin.h"
namespace zeek::plugin::detail::Zeek_RawReader {
namespace zeek::plugin::detail::Zeek_RawReader
{
class Plugin : public plugin::Plugin {
class Plugin : public plugin::Plugin
{
public:
Plugin();
@ -22,9 +24,8 @@ public:
private:
std::mutex fork_mutex;
};
};
extern Plugin plugin;
} // namespace zeek::plugin::detail::Zeek_RawReader
} // namespace zeek::plugin::detail::Zeek_RawReader

209
src/input/readers/raw/Raw.cc
View file

@ -2,33 +2,35 @@
#include "zeek/input/readers/raw/Raw.h"
#include <errno.h>
#include <fcntl.h>
#include <signal.h>
#include <stdio.h>
#include <stdlib.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <sys/wait.h>
#include <sys/stat.h>
#include <unistd.h>
#include <stdio.h>
#include <fcntl.h>
#include <errno.h>
#include <signal.h>
#include <stdlib.h>
#include "zeek/input/readers/raw/Plugin.h"
#include "zeek/input/readers/raw/raw.bif.h"
#include "zeek/threading/SerialTypes.h"
#include "zeek/input/readers/raw/raw.bif.h"
extern "C" {
extern "C"
{
#include "zeek/setsignal.h"
}
}
using zeek::threading::Value;
using zeek::threading::Field;
using zeek::threading::Value;
namespace zeek::input::reader::detail {
namespace zeek::input::reader::detail
{
const int Raw::block_size = 4096; // how big do we expect our chunks of data to be.
Raw::Raw(ReaderFrontend *frontend) : ReaderBackend(frontend), file(nullptr, fclose), stderrfile(nullptr, fclose)
Raw::Raw(ReaderFrontend* frontend)
: ReaderBackend(frontend), file(nullptr, fclose), stderrfile(nullptr, fclose)
{
execute = false;
firstrun = true;
@ -36,8 +38,8 @@ Raw::Raw(ReaderFrontend *frontend) : ReaderBackend(frontend), file(nullptr, fclo
ino = 0;
forcekill = false;
offset = 0;
separator.assign( (const char*) BifConst::InputRaw::record_separator->Bytes(),
BifConst::InputRaw::record_separator->Len());
separator.assign((const char*)BifConst::InputRaw::record_separator->Bytes(),
BifConst::InputRaw::record_separator->Len());
sep_length = BifConst::InputRaw::record_separator->Len();
@ -98,7 +100,6 @@ bool Raw::SetFDFlags(int fd, int cmd, int flags)
return false;
}
std::unique_lock<std::mutex> Raw::AcquireForkMutex()
{
auto lock = plugin::detail::Zeek_RawReader::plugin.ForkMutex();
@ -128,7 +129,7 @@ bool Raw::Execute()
// w/ just 2 threads ~33% of the time).
auto lock = AcquireForkMutex();
if ( pipe(pipes) != 0 || pipe(pipes+2) || pipe(pipes+4) )
if ( pipe(pipes) != 0 || pipe(pipes + 2) || pipe(pipes + 4) )
{
Error(Fmt("Could not open pipe: %d", errno));
return false;
@ -176,7 +177,7 @@ bool Raw::Execute()
// makes sense over pthread_sigmask() here.
sigprocmask(SIG_UNBLOCK, &mask, 0);
execl("/bin/sh", "sh", "-c", fname.c_str(), (char*) NULL);
execl("/bin/sh", "sh", "-c", fname.c_str(), (char*)NULL);
fprintf(stderr, "Exec failed :(......\n");
_exit(255);
}
@ -243,7 +244,7 @@ bool Raw::Execute()
else
ClosePipeEnd(stderr_in);
file = std::unique_ptr<FILE, int(*)(FILE*)>(fdopen(pipes[stdout_in], "r"), fclose);
file = std::unique_ptr<FILE, int (*)(FILE*)>(fdopen(pipes[stdout_in], "r"), fclose);
if ( ! file )
{
@ -255,7 +256,8 @@ bool Raw::Execute()
if ( use_stderr )
{
stderrfile = std::unique_ptr<FILE, int(*)(FILE*)>(fdopen(pipes[stderr_in], "r"), fclose);
stderrfile =
std::unique_ptr<FILE, int (*)(FILE*)>(fdopen(pipes[stderr_in], "r"), fclose);
if ( ! stderrfile )
{
@ -277,7 +279,7 @@ bool Raw::OpenInput()
else
{
file = std::unique_ptr<FILE, int(*)(FILE*)>(fopen(fname.c_str(), "r"), fclose);
file = std::unique_ptr<FILE, int (*)(FILE*)>(fopen(fname.c_str(), "r"), fclose);
if ( ! file )
{
Error(Fmt("Init: cannot open %s", fname.c_str()));
@ -288,18 +290,18 @@ bool Raw::OpenInput()
Warning(Fmt("Init: cannot set close-on-exec for %s", fname.c_str()));
}
if ( offset )
{
int whence = (offset >= 0) ? SEEK_SET : SEEK_END;
int64_t pos = (offset >= 0) ? offset : offset + 1; // we want -1 to be the end of the file
if ( offset )
{
int whence = (offset >= 0) ? SEEK_SET : SEEK_END;
int64_t pos = (offset >= 0) ? offset : offset + 1; // we want -1 to be the end of the file
if ( fseek(file.get(), pos, whence) < 0 )
{
char buf[256];
util::zeek_strerror_r(errno, buf, sizeof(buf));
Error(Fmt("Seek failed in init: %s", buf));
}
if ( fseek(file.get(), pos, whence) < 0 )
{
char buf[256];
util::zeek_strerror_r(errno, buf, sizeof(buf));
Error(Fmt("Seek failed in init: %s", buf));
}
}
return true;
}
@ -308,8 +310,7 @@ bool Raw::CloseInput()
{
if ( ! file )
{
InternalWarning(Fmt("Trying to close closed file for stream %s",
fname.c_str()));
InternalWarning(Fmt("Trying to close closed file for stream %s", fname.c_str()));
return false;
}
#ifdef DEBUG
@ -323,7 +324,7 @@ bool Raw::CloseInput()
if ( execute )
{
for ( int i = 0; i < 6; i ++ )
for ( int i = 0; i < 6; i++ )
ClosePipeEnd(i);
}
@ -358,7 +359,8 @@ bool Raw::DoInit(const ReaderInfo& info, int num_fields, const Field* const* fie
fname = source.substr(0, fname.length() - 1);
}
ReaderInfo::config_map::const_iterator it = info.config.find("stdin"); // data that is sent to the child process
ReaderInfo::config_map::const_iterator it =
info.config.find("stdin"); // data that is sent to the child process
if ( it != info.config.end() )
{
stdin_string = it->second;
@ -379,23 +381,27 @@ bool Raw::DoInit(const ReaderInfo& info, int num_fields, const Field* const* fie
}
it = info.config.find("offset"); // we want to seek to a given offset inside the file
if ( it != info.config.end() && ! execute && (Info().mode == MODE_STREAM ||
Info().mode == MODE_MANUAL) )
if ( it != info.config.end() && ! execute &&
(Info().mode == MODE_STREAM || Info().mode == MODE_MANUAL) )
{
std::string offset_s = it->second;
offset = strtoll(offset_s.c_str(), 0, 10);
}
else if ( it != info.config.end() )
{
Error("Offset only is supported for MODE_STREAM and MODE_MANUAL; it is also not supported when executing a command");
Error("Offset only is supported for MODE_STREAM and MODE_MANUAL; it is also not supported "
"when executing a command");
return false;
}
if ( num_fields != want_fields )
{
Error(Fmt("Filter for raw reader contains wrong number of fields -- got %d, expected %d. "
"Filters for the raw reader contain one string field when used in normal mode and one string and one bool fields when using execute mode with stderr capuring. "
"Filter ignored.", num_fields, want_fields));
Error(
Fmt("Filter for raw reader contains wrong number of fields -- got %d, expected %d. "
"Filters for the raw reader contain one string field when used in normal mode and "
"one string and one bool fields when using execute mode with stderr capuring. "
"Filter ignored.",
num_fields, want_fields));
return false;
}
@ -417,7 +423,6 @@ bool Raw::DoInit(const ReaderInfo& info, int num_fields, const Field* const* fie
return false;
}
result = OpenInput();
if ( result == false )
@ -449,9 +454,9 @@ int64_t Raw::GetLine(FILE* arg_file)
for ( ;; )
{
size_t readbytes = fread(buf.get()+bufpos+offset, 1, block_size-bufpos, arg_file);
size_t readbytes = fread(buf.get() + bufpos + offset, 1, block_size - bufpos, arg_file);
pos += bufpos + readbytes;
//printf("Pos: %d\n", pos);
// printf("Pos: %d\n", pos);
bufpos = offset = 0; // read full block size in next read...
if ( pos == 0 && errno != 0 )
@ -460,8 +465,8 @@ int64_t Raw::GetLine(FILE* arg_file)
// researching everything each time is a bit... cpu-intensive. But otherwhise we have
// to deal with situations where the separator is multi-character and split over multiple
// reads...
int found = util::strstr_n(pos, (unsigned char*) buf.get(),
separator.size(), (unsigned char*) separator.c_str());
int found = util::strstr_n(pos, (unsigned char*)buf.get(), separator.size(),
(unsigned char*)separator.c_str());
if ( found == -1 )
{
@ -479,11 +484,13 @@ int64_t Raw::GetLine(FILE* arg_file)
}
repeats++;
// bah, we cannot use realloc because we would have to change the delete in the manager to a free.
std::unique_ptr<char[]> newbuf = std::unique_ptr<char[]>(new char[block_size*repeats]);
memcpy(newbuf.get(), buf.get(), block_size*(repeats-1));
// bah, we cannot use realloc because we would have to change the delete in the manager
// to a free.
std::unique_ptr<char[]> newbuf =
std::unique_ptr<char[]>(new char[block_size * repeats]);
memcpy(newbuf.get(), buf.get(), block_size * (repeats - 1));
buf = std::move(newbuf);
offset = block_size*(repeats-1);
offset = block_size * (repeats - 1);
}
else
{
@ -499,7 +506,6 @@ int64_t Raw::GetLine(FILE* arg_file)
return found;
}
}
if ( errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR )
@ -525,7 +531,9 @@ void Raw::WriteToStdin()
if ( errno != 0 && errno != EAGAIN && errno != EWOULDBLOCK )
{
Error(Fmt("Writing to child process stdin failed: %d. Stopping writing at position %" PRIu64, errno, pos));
Error(
Fmt("Writing to child process stdin failed: %d. Stopping writing at position %" PRIu64,
errno, pos));
stdin_towrite = 0;
}
@ -534,11 +542,11 @@ void Raw::WriteToStdin()
if ( Info().mode == MODE_MANUAL && stdin_towrite != 0 )
{
Error(Fmt("Could not write whole string to stdin of child process in one go. Please use STREAM mode to pass more data to child."));
Error(Fmt("Could not write whole string to stdin of child process in one go. Please use "
"STREAM mode to pass more data to child."));
}
}
// read the entire file and send appropriate thingies back to InputMgr
bool Raw::DoUpdate()
{
@ -547,57 +555,58 @@ bool Raw::DoUpdate()
else
{
switch ( Info().mode ) {
case MODE_REREAD:
switch ( Info().mode )
{
assert(childpid == -1); // mode may not be used to execute child programs
// check if the file has changed
struct stat sb;
if ( stat(fname.c_str(), &sb) == -1 )
{
Error(Fmt("Could not get stat for %s", fname.c_str()));
return false;
}
case MODE_REREAD:
{
assert(childpid == -1); // mode may not be used to execute child programs
// check if the file has changed
struct stat sb;
if ( stat(fname.c_str(), &sb) == -1 )
{
Error(Fmt("Could not get stat for %s", fname.c_str()));
return false;
}
if ( sb.st_ino == ino && sb.st_mtime == mtime )
// no change
return true;
if ( sb.st_ino == ino && sb.st_mtime == mtime )
// no change
return true;
mtime = sb.st_mtime;
ino = sb.st_ino;
// file changed. reread.
//
// fallthrough
}
mtime = sb.st_mtime;
ino = sb.st_ino;
// file changed. reread.
//
// fallthrough
}
case MODE_MANUAL:
case MODE_STREAM:
if ( Info().mode == MODE_STREAM && file )
{
clearerr(file.get()); // remove end of file evil bits
break;
}
CloseInput();
if ( ! OpenInput() )
return false;
case MODE_MANUAL:
case MODE_STREAM:
if ( Info().mode == MODE_STREAM && file )
{
clearerr(file.get()); // remove end of file evil bits
break;
}
CloseInput();
if ( ! OpenInput() )
return false;
break;
default:
assert(false);
}
default:
assert(false);
}
}
std::string line;
assert ( (NumFields() == 1 && !use_stderr) || (NumFields() == 2 && use_stderr));
assert((NumFields() == 1 && ! use_stderr) || (NumFields() == 2 && use_stderr));
for ( ;; )
{
if ( stdin_towrite > 0 )
WriteToStdin();
int64_t length = GetLine(file.get());
//printf("Read %lld bytes\n", length);
// printf("Read %lld bytes\n", length);
if ( length == -3 )
return false;
@ -606,7 +615,8 @@ bool Raw::DoUpdate()
// no data ready or eof
break;
Value** fields = new Value*[2]; // just always reserve 2. This means that our [] is too long by a count of 1 if not using stderr. But who cares...
Value** fields = new Value*[2]; // just always reserve 2. This means that our [] is too long
// by a count of 1 if not using stderr. But who cares...
// filter has exactly one text field. convert to it.
Value* val = new Value(TYPE_STRING, true);
@ -629,7 +639,7 @@ bool Raw::DoUpdate()
for ( ;; )
{
int64_t length = GetLine(stderrfile.get());
//printf("Read stderr %lld bytes\n", length);
// printf("Read stderr %lld bytes\n", length);
if ( length == -3 )
return false;
@ -649,7 +659,7 @@ bool Raw::DoUpdate()
}
}
if ( ( Info().mode == MODE_MANUAL ) || ( Info().mode == MODE_REREAD ) )
if ( (Info().mode == MODE_MANUAL) || (Info().mode == MODE_REREAD) )
// done with the current data source
EndCurrentSend();
@ -698,8 +708,6 @@ bool Raw::DoUpdate()
return false;
}
#ifdef DEBUG
Debug(DBG_INPUT, "DoUpdate finished successfully");
#endif
@ -709,7 +717,8 @@ bool Raw::DoUpdate()
bool Raw::DoHeartbeat(double network_time, double current_time)
{
switch ( Info().mode ) {
switch ( Info().mode )
{
case MODE_MANUAL:
// yay, we do nothing :)
break;
@ -719,17 +728,17 @@ bool Raw::DoHeartbeat(double network_time, double current_time)
#ifdef DEBUG
Debug(DBG_INPUT, "Starting Heartbeat update");
#endif
Update(); // call update and not DoUpdate, because update
// checks disabled.
Update(); // call update and not DoUpdate, because update
// checks disabled.
#ifdef DEBUG
Debug(DBG_INPUT, "Finished with heartbeat update");
#endif
break;
default:
assert(false);
}
}
return true;
}
} // namespace zeek::input::reader::detail
} // namespace zeek::input::reader::detail

24
src/input/readers/raw/Raw.h
View file

@ -3,19 +3,21 @@
#pragma once
#include <sys/types.h>
#include <vector>
#include <memory>
#include <mutex>
#include <vector>
#include "zeek/input/ReaderBackend.h"
namespace zeek::input::reader::detail {
namespace zeek::input::reader::detail
{
/**
* A reader that returns a file (or the output of a command) as a single
* blob.
*/
class Raw : public ReaderBackend {
class Raw : public ReaderBackend
{
public:
explicit Raw(ReaderFrontend* frontend);
~Raw() override;
@ -29,7 +31,8 @@ public:
static ReaderBackend* Instantiate(ReaderFrontend* frontend) { return new Raw(frontend); }
protected:
bool DoInit(const ReaderInfo& info, int arg_num_fields, const threading::Field* const* fields) override;
bool DoInit(const ReaderInfo& info, int arg_num_fields,
const threading::Field* const* fields) override;
void DoClose() override;
bool DoUpdate() override;
bool DoHeartbeat(double network_time, double current_time) override;
@ -46,8 +49,8 @@ private:
void WriteToStdin();
std::string fname; // Source with a potential "|" removed.
std::unique_ptr<FILE, int(*)(FILE*)> file;
std::unique_ptr<FILE, int(*)(FILE*)> stderrfile;
std::unique_ptr<FILE, int (*)(FILE*)> file;
std::unique_ptr<FILE, int (*)(FILE*)> stderrfile;
bool execute;
bool firstrun;
time_t mtime;
@ -77,16 +80,17 @@ private:
int pipes[6];
pid_t childpid;
enum IoChannels {
enum IoChannels
{
stdout_in = 0,
stdout_out = 1,
stdin_in = 2,
stdin_out = 3,
stderr_in = 4,
stderr_out = 5
};
};
static const int block_size;
};
};
} // namespace zeek::input::reader::detail
} // namespace zeek::input::reader::detail

14
src/input/readers/sqlite/Plugin.cc
View file

@ -1,21 +1,25 @@
// See the file in the main distribution directory for copyright.
#include "zeek/plugin/Plugin.h"
#include "zeek/input/readers/sqlite/SQLite.h"
namespace zeek::plugin::detail::Zeek_SQLiteReader {
namespace zeek::plugin::detail::Zeek_SQLiteReader
{
class Plugin : public zeek::plugin::Plugin {
class Plugin : public zeek::plugin::Plugin
{
public:
zeek::plugin::Configuration Configure() override
{
AddComponent(new zeek::input::Component("SQLite", zeek::input::reader::detail::SQLite::Instantiate));
AddComponent(
new zeek::input::Component("SQLite", zeek::input::reader::detail::SQLite::Instantiate));
zeek::plugin::Configuration config;
config.name = "Zeek::SQLiteReader";
config.description = "SQLite input reader";
return config;
}
} plugin;
} plugin;
} // namespae zeek::plugin::detail::Zeek_SQLiteReader
} // namespae zeek::plugin::detail::Zeek_SQLiteReader

319
src/input/readers/sqlite/SQLite.cc
View file

@ -1,46 +1,40 @@
// See the file "COPYING" in the main distribution directory for copyright.
#include "zeek/zeek-config.h"
#include "zeek/input/readers/sqlite/SQLite.h"
#include <sys/types.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <unistd.h>
#include <fstream>
#include <sstream>
#include "zeek/threading/SerialTypes.h"
#include "zeek/input/readers/sqlite/sqlite.bif.h"
#include "zeek/logging/writers/sqlite/sqlite.bif.h"
#include "zeek/logging/writers/ascii/ascii.bif.h"
#include "zeek/logging/writers/sqlite/sqlite.bif.h"
#include "zeek/threading/SerialTypes.h"
#include "zeek/zeek-config.h"
using zeek::threading::Value;
using zeek::threading::Field;
using zeek::threading::Value;
namespace zeek::input::reader::detail {
SQLite::SQLite(ReaderFrontend *frontend)
: ReaderBackend(frontend),
fields(), num_fields(), mode(), started(), query(), db(), st()
namespace zeek::input::reader::detail
{
set_separator.assign(
(const char*) BifConst::LogSQLite::set_separator->Bytes(),
BifConst::InputSQLite::set_separator->Len()
);
unset_field.assign(
(const char*) BifConst::LogSQLite::unset_field->Bytes(),
BifConst::InputSQLite::unset_field->Len()
);
SQLite::SQLite(ReaderFrontend* frontend)
: ReaderBackend(frontend), fields(), num_fields(), mode(), started(), query(), db(), st()
{
set_separator.assign((const char*)BifConst::LogSQLite::set_separator->Bytes(),
BifConst::InputSQLite::set_separator->Len());
empty_field.assign(
(const char*) BifConst::LogAscii::empty_field->Bytes(),
BifConst::InputSQLite::empty_field->Len()
);
unset_field.assign((const char*)BifConst::LogSQLite::unset_field->Bytes(),
BifConst::InputSQLite::unset_field->Len());
io = new threading::formatter::Ascii(this, threading::formatter::Ascii::SeparatorInfo(
std::string(), set_separator, unset_field, empty_field));
empty_field.assign((const char*)BifConst::LogAscii::empty_field->Bytes(),
BifConst::InputSQLite::empty_field->Len());
io = new threading::formatter::Ascii(
this, threading::formatter::Ascii::SeparatorInfo(std::string(), set_separator, unset_field,
empty_field));
}
SQLite::~SQLite()
@ -72,11 +66,13 @@ bool SQLite::checkError(int code)
return false;
}
bool SQLite::DoInit(const ReaderInfo& info, int arg_num_fields, const threading::Field* const* arg_fields)
bool SQLite::DoInit(const ReaderInfo& info, int arg_num_fields,
const threading::Field* const* arg_fields)
{
if ( sqlite3_threadsafe() == 0 )
{
Error("SQLite reports that it is not threadsafe. Zeek needs a threadsafe version of SQLite. Aborting");
Error("SQLite reports that it is not threadsafe. Zeek needs a threadsafe version of "
"SQLite. Aborting");
return false;
}
@ -101,26 +97,22 @@ bool SQLite::DoInit(const ReaderInfo& info, int arg_num_fields, const threading:
ReaderInfo::config_map::const_iterator it = info.config.find("query");
if ( it == info.config.end() )
{
Error(Fmt("No query specified when setting up SQLite data source %s. Aborting.", info.source));
Error(Fmt("No query specified when setting up SQLite data source %s. Aborting.",
info.source));
return false;
}
else
query = it->second;
if ( checkError(sqlite3_open_v2(
fullpath.c_str(),
&db,
SQLITE_OPEN_READWRITE |
SQLITE_OPEN_NOMUTEX
,
NULL)) )
if ( checkError(sqlite3_open_v2(fullpath.c_str(), &db,
SQLITE_OPEN_READWRITE | SQLITE_OPEN_NOMUTEX, NULL)) )
return false;
num_fields = arg_num_fields;
fields = arg_fields;
// create the prepared select statement that we will re-use forever...
if ( checkError(sqlite3_prepare_v2( db, query.c_str(), query.size()+1, &st, NULL )) )
if ( checkError(sqlite3_prepare_v2(db, query.c_str(), query.size() + 1, &st, NULL)) )
{
return false;
}
@ -132,129 +124,129 @@ bool SQLite::DoInit(const ReaderInfo& info, int arg_num_fields, const threading:
// pos = field position
// subpos = subfield position, only used for port-field
Value* SQLite::EntryToVal(sqlite3_stmt *st, const threading::Field *field, int pos, int subpos)
Value* SQLite::EntryToVal(sqlite3_stmt* st, const threading::Field* field, int pos, int subpos)
{
if ( sqlite3_column_type(st, pos ) == SQLITE_NULL )
if ( sqlite3_column_type(st, pos) == SQLITE_NULL )
return new Value(field->type, field->subtype, false);
Value* val = new Value(field->type, true);
switch ( field->type ) {
case TYPE_ENUM:
case TYPE_STRING:
switch ( field->type )
{
const char *text = (const char*) sqlite3_column_text(st, pos);
int length = sqlite3_column_bytes(st, pos);
char *out = new char[length];
memcpy(out, text, length);
val->val.string_val.length = length;
val->val.string_val.data = out;
break;
}
case TYPE_BOOL:
{
if ( sqlite3_column_type(st, pos) != SQLITE_INTEGER )
{
Error("Invalid data type for boolean - expected Integer");
delete val;
return nullptr;
}
int res = sqlite3_column_int(st, pos);
if ( res == 0 || res == 1 )
val->val.int_val = res;
else
{
Error(Fmt("Invalid value for boolean: %d", res));
delete val;
return nullptr;
}
break;
}
case TYPE_INT:
val->val.int_val = sqlite3_column_int64(st, pos);
break;
case TYPE_DOUBLE:
case TYPE_TIME:
case TYPE_INTERVAL:
val->val.double_val = sqlite3_column_double(st, pos);
break;
case TYPE_COUNT:
val->val.uint_val = sqlite3_column_int64(st, pos);
break;
case TYPE_PORT:
{
val->val.port_val.port = sqlite3_column_int(st, pos);
val->val.port_val.proto = TRANSPORT_UNKNOWN;
if ( subpos != -1 )
{
const char *text = (const char*) sqlite3_column_text(st, subpos);
if ( text == 0 )
Error("Port protocol definition did not contain text");
else
case TYPE_ENUM:
case TYPE_STRING:
{
std::string s(text, sqlite3_column_bytes(st, subpos));
val->val.port_val.proto = io->ParseProto(s);
const char* text = (const char*)sqlite3_column_text(st, pos);
int length = sqlite3_column_bytes(st, pos);
char* out = new char[length];
memcpy(out, text, length);
val->val.string_val.length = length;
val->val.string_val.data = out;
break;
}
}
break;
case TYPE_BOOL:
{
if ( sqlite3_column_type(st, pos) != SQLITE_INTEGER )
{
Error("Invalid data type for boolean - expected Integer");
delete val;
return nullptr;
}
int res = sqlite3_column_int(st, pos);
if ( res == 0 || res == 1 )
val->val.int_val = res;
else
{
Error(Fmt("Invalid value for boolean: %d", res));
delete val;
return nullptr;
}
break;
}
case TYPE_INT:
val->val.int_val = sqlite3_column_int64(st, pos);
break;
case TYPE_DOUBLE:
case TYPE_TIME:
case TYPE_INTERVAL:
val->val.double_val = sqlite3_column_double(st, pos);
break;
case TYPE_COUNT:
val->val.uint_val = sqlite3_column_int64(st, pos);
break;
case TYPE_PORT:
{
val->val.port_val.port = sqlite3_column_int(st, pos);
val->val.port_val.proto = TRANSPORT_UNKNOWN;
if ( subpos != -1 )
{
const char* text = (const char*)sqlite3_column_text(st, subpos);
if ( text == 0 )
Error("Port protocol definition did not contain text");
else
{
std::string s(text, sqlite3_column_bytes(st, subpos));
val->val.port_val.proto = io->ParseProto(s);
}
}
break;
}
case TYPE_SUBNET:
{
const char* text = (const char*)sqlite3_column_text(st, pos);
std::string s(text, sqlite3_column_bytes(st, pos));
int pos = s.find('/');
int width = atoi(s.substr(pos + 1).c_str());
std::string addr = s.substr(0, pos);
val->val.subnet_val.prefix = io->ParseAddr(addr);
val->val.subnet_val.length = width;
break;
}
case TYPE_ADDR:
{
const char* text = (const char*)sqlite3_column_text(st, pos);
std::string s(text, sqlite3_column_bytes(st, pos));
val->val.addr_val = io->ParseAddr(s);
break;
}
case TYPE_TABLE:
case TYPE_VECTOR:
{
const char* text = (const char*)sqlite3_column_text(st, pos);
std::string s(text, sqlite3_column_bytes(st, pos));
delete val;
val = io->ParseValue(s, "", field->type, field->subtype);
break;
}
default:
Error(Fmt("unsupported field format %d", field->type));
delete val;
return nullptr;
}
case TYPE_SUBNET:
{
const char *text = (const char*) sqlite3_column_text(st, pos);
std::string s(text, sqlite3_column_bytes(st, pos));
int pos = s.find('/');
int width = atoi(s.substr(pos+1).c_str());
std::string addr = s.substr(0, pos);
val->val.subnet_val.prefix = io->ParseAddr(addr);
val->val.subnet_val.length = width;
break;
}
case TYPE_ADDR:
{
const char *text = (const char*) sqlite3_column_text(st, pos);
std::string s(text, sqlite3_column_bytes(st, pos));
val->val.addr_val = io->ParseAddr(s);
break;
}
case TYPE_TABLE:
case TYPE_VECTOR:
{
const char *text = (const char*) sqlite3_column_text(st, pos);
std::string s(text, sqlite3_column_bytes(st, pos));
delete val;
val = io->ParseValue(s, "", field->type, field->subtype);
break;
}
default:
Error(Fmt("unsupported field format %d", field->type));
delete val;
return nullptr;
}
return val;
}
bool SQLite::DoUpdate()
{
int numcolumns = sqlite3_column_count(st);
int *mapping = new int [num_fields];
int *submapping = new int [num_fields];
int* mapping = new int[num_fields];
int* submapping = new int[num_fields];
// first set them all to -1
for ( unsigned int i = 0; i < num_fields; ++i )
@ -265,7 +257,7 @@ bool SQLite::DoUpdate()
for ( int i = 0; i < numcolumns; ++i )
{
const char *name = sqlite3_column_name(st, i);
const char* name = sqlite3_column_name(st, i);
for ( unsigned j = 0; j < num_fields; j++ )
{
@ -273,23 +265,28 @@ bool SQLite::DoUpdate()
{
if ( mapping[j] != -1 )
{
Error(Fmt("SQLite statement returns several columns with name %s! Cannot decide which to choose, aborting", name));
delete [] mapping;
delete [] submapping;
Error(Fmt("SQLite statement returns several columns with name %s! Cannot "
"decide which to choose, aborting",
name));
delete[] mapping;
delete[] submapping;
return false;
}
mapping[j] = i;
}
if ( fields[j]->secondary_name != nullptr && strcmp(fields[j]->secondary_name, name) == 0 )
if ( fields[j]->secondary_name != nullptr &&
strcmp(fields[j]->secondary_name, name) == 0 )
{
assert(fields[j]->type == TYPE_PORT);
if ( submapping[j] != -1 )
{
Error(Fmt("SQLite statement returns several columns with name %s! Cannot decide which to choose, aborting", name));
delete [] mapping;
delete [] submapping;
Error(Fmt("SQLite statement returns several columns with name %s! Cannot "
"decide which to choose, aborting",
name));
delete[] mapping;
delete[] submapping;
return false;
}
@ -303,18 +300,18 @@ bool SQLite::DoUpdate()
if ( mapping[i] == -1 )
{
Error(Fmt("Required field %s not found after SQLite statement", fields[i]->name));
delete [] mapping;
delete [] submapping;
delete[] mapping;
delete[] submapping;
return false;
}
}
int errorcode;
while ( ( errorcode = sqlite3_step(st)) == SQLITE_ROW )
while ( (errorcode = sqlite3_step(st)) == SQLITE_ROW )
{
Value** ofields = new Value*[num_fields];
for ( unsigned int j = 0; j < num_fields; ++j)
for ( unsigned int j = 0; j < num_fields; ++j )
{
ofields[j] = EntryToVal(st, fields[j], mapping[j], submapping[j]);
if ( ! ofields[j] )
@ -322,9 +319,9 @@ bool SQLite::DoUpdate()
for ( unsigned int k = 0; k < j; ++k )
delete ofields[k];
delete [] ofields;
delete [] mapping;
delete [] submapping;
delete[] ofields;
delete[] mapping;
delete[] submapping;
return false;
}
}
@ -332,8 +329,8 @@ bool SQLite::DoUpdate()
SendEntry(ofields);
}
delete [] mapping;
delete [] submapping;
delete[] mapping;
delete[] submapping;
if ( checkError(errorcode) ) // check the last error code returned by sqlite
return false;
@ -346,4 +343,4 @@ bool SQLite::DoUpdate()
return true;
}
} // namespace zeek::input::reader::detail
} // namespace zeek::input::reader::detail

27
src/input/readers/sqlite/SQLite.h
View file

@ -2,18 +2,19 @@
#pragma once
#include "zeek/zeek-config.h"
#include <iostream>
#include <vector>
#include "zeek/3rdparty/sqlite3.h"
#include "zeek/3rdparty/sqlite3.h"
#include "zeek/input/ReaderBackend.h"
#include "zeek/threading/formatters/Ascii.h"
#include "zeek/zeek-config.h"
namespace zeek::input::reader::detail {
namespace zeek::input::reader::detail
{
class SQLite : public ReaderBackend {
class SQLite : public ReaderBackend
{
public:
explicit SQLite(ReaderFrontend* frontend);
~SQLite() override;
@ -21,7 +22,8 @@ public:
static ReaderBackend* Instantiate(ReaderFrontend* frontend) { return new SQLite(frontend); }
protected:
bool DoInit(const ReaderInfo& info, int arg_num_fields, const threading::Field* const* arg_fields) override;
bool DoInit(const ReaderInfo& info, int arg_num_fields,
const threading::Field* const* arg_fields) override;
void DoClose() override;
bool DoUpdate() override;
bool DoHeartbeat(double network_time, double current_time) override { return true; }
@ -29,20 +31,21 @@ protected:
private:
bool checkError(int code);
threading::Value* EntryToVal(sqlite3_stmt *st, const threading::Field *field, int pos, int subpos);
threading::Value* EntryToVal(sqlite3_stmt* st, const threading::Field* field, int pos,
int subpos);
const threading::Field* const * fields; // raw mapping
const threading::Field* const* fields; // raw mapping
unsigned int num_fields;
int mode;
bool started;
std::string query;
sqlite3 *db;
sqlite3_stmt *st;
sqlite3* db;
sqlite3_stmt* st;
threading::formatter::Ascii* io;
std::string set_separator;
std::string unset_field;
std::string empty_field;
};
};
} // namespace zeek::input::reader
} // namespace zeek::input::reader