Mirror/zeek
1
0
Fork 0
mirror of https://github.com/zeek/zeek.git synced 2025-10-03 15:18:20 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

Refactored formatters and updated the the writers a bit.

Browse source 
- Formatters have been abstracted similarly to readers and writers now.
 - The Ascii writer has a new option for writing out logs as JSON.
 - The Ascii writer now has all options availble as per-filter
   options as well as global.
This commit is contained in:
Seth Hall 2014-03-10 10:42:59 -04:00
parent 83ec05bb4a
commit a56c343715
25 changed files with 750 additions and 428 deletions
Download patch file Download diff file Expand all files Collapse all files

21
scripts/base/frameworks/logging/writers/ascii.bro
View file

@ -17,27 +17,38 @@ module LogAscii;
export {
## If true, output everything to stdout rather than
## into files. This is primarily for debugging purposes.
## This is also available as a per-filter $config option.
const output_to_stdout = F &redef;
## If true, the default option will be to write logs in a JSON format.
## This is also available as a per-filter $config option.
const use_json = F &redef;
## If true, include lines with log meta information such as column names
## with types, the values of ASCII logging options that are in use, and
## the time when the file was opened and closed (the latter at the end).
## If writing in JSON format, this is implicitly disabled.
const include_meta = T &redef;
## Prefix for lines with meta information.
## Prefix for lines with meta information. This is also available as a
## per-filter $config option.
const meta_prefix = "#" &redef;
## Separator between fields.
## Separator between fields. This is also available as a per-filter
## $config option.
const separator = Log::separator &redef;
## Separator between set elements.
## Separator between set elements. This is also available as a
## per-filter $config option.
const set_separator = Log::set_separator &redef;
## String to use for empty fields. This should be different from
## *unset_field* to make the output unambiguous.
## *unset_field* to make the output unambiguous. This is also
## available as a per-filter $config option.
const empty_field = Log::empty_field &redef;
## String to use for an unset &optional field.
## String to use for an unset &optional field. This is also
## available as a per-filter $config option.
const unset_field = Log::unset_field &redef;
}

4
scripts/policy/tuning/json-logs.bro Normal file
View file

@ -0,0 +1,4 @@
##! Loading this script will cause all logs to be written
##! out as JSON by default.
redef LogAscii::use_json=T;

1
scripts/test-all-policy.bro
View file

@ -93,6 +93,7 @@
@load tuning/defaults/extracted_file_limits.bro
@load tuning/defaults/packet-fragments.bro
@load tuning/defaults/warnings.bro
@load tuning/json-logs.bro
@load tuning/logs-to-elasticsearch.bro
@load tuning/track-all-assets.bro

4
src/CMakeLists.txt
View file

@ -335,11 +335,13 @@ set(bro_SRCS
strsep.c
modp_numtoa.c
threading/AsciiFormatter.cc
threading/BasicThread.cc
threading/Formatter.cc
threading/Manager.cc
threading/MsgThread.cc
threading/SerialTypes.cc
threading/formatters/Ascii.cc
threading/formatters/JSON.cc
logging/Manager.cc
logging/WriterBackend.cc

16
src/input/readers/Ascii.cc
View file

@ -69,13 +69,13 @@ Ascii::Ascii(ReaderFrontend *frontend) : ReaderBackend(frontend)
unset_field.assign( (const char*) BifConst::InputAscii::unset_field->Bytes(),
BifConst::InputAscii::unset_field->Len());
ascii = new AsciiFormatter(this, AsciiFormatter::SeparatorInfo(set_separator, unset_field, empty_field));
formatter = new threading::formatter::Ascii(this, threading::formatter::Ascii::SeparatorInfo(string(), set_separator, unset_field, empty_field));
}
Ascii::~Ascii()
{
DoClose();
delete ascii;
delete formatter;
}
void Ascii::DoClose()
@ -173,7 +173,6 @@ bool Ascii::ReadHeader(bool useCached)
return false;
}
FieldMapping f(field->name, field->type, field->subtype, ifields[field->name]);
if ( field->secondary_name && strlen(field->secondary_name) != 0 )
@ -282,7 +281,7 @@ bool Ascii::DoUpdate()
file->sync();
while ( GetLine(line ) )
while ( GetLine(line) )
{
// split on tabs
bool error = false;
@ -332,7 +331,7 @@ bool Ascii::DoUpdate()
return false;
}
Value* val = ascii->ParseValue(stringfields[(*fit).position], (*fit).name, (*fit).type, (*fit).subtype);
Value* val = formatter->ParseValue(stringfields[(*fit).position], (*fit).name, (*fit).type, (*fit).subtype);
if ( val == 0 )
{
@ -347,7 +346,7 @@ bool Ascii::DoUpdate()
assert(val->type == TYPE_PORT );
// Error(Fmt("Got type %d != PORT with secondary position!", val->type));
val->val.port_val.proto = ascii->ParseProto(stringfields[(*fit).secondary_position]);
val->val.port_val.proto = formatter->ParseProto(stringfields[(*fit).secondary_position]);
}
fields[fpos] = val;
@ -384,8 +383,9 @@ bool Ascii::DoUpdate()
}
bool Ascii::DoHeartbeat(double network_time, double current_time)
{
switch ( Info().mode ) {
{
switch ( Info().mode )
{
case MODE_MANUAL:
// yay, we do nothing :)
break;

4
src/input/readers/Ascii.h
View file

@ -7,7 +7,7 @@
#include <vector>
#include "../ReaderBackend.h"
#include "threading/AsciiFormatter.h"
#include "threading/formatters/Ascii.h"
namespace input { namespace reader {
@ -64,7 +64,7 @@ private:
string empty_field;
string unset_field;
AsciiFormatter* ascii;
threading::formatter::Formatter* formatter;
};

2
src/input/readers/Benchmark.cc
View file

@ -29,7 +29,7 @@ Benchmark::Benchmark(ReaderFrontend *frontend) : ReaderBackend(frontend)
heartbeatstarttime = 0;
heartbeat_interval = double(BifConst::Threading::heartbeat_interval);
ascii = new AsciiFormatter(this, AsciiFormatter::SeparatorInfo());
ascii = new threading::formatter::Ascii(this, threading::formatter::Ascii::SeparatorInfo());
}
Benchmark::~Benchmark()

4
src/input/readers/Benchmark.h
View file

@ -4,7 +4,7 @@
#define INPUT_READERS_BENCHMARK_H
#include "../ReaderBackend.h"
#include "threading/AsciiFormatter.h"
#include "threading/formatters/Ascii.h"
namespace input { namespace reader {
@ -40,7 +40,7 @@ private:
double timedspread;
double heartbeat_interval;
AsciiFormatter* ascii;
threading::formatter::Ascii* ascii;
};

2
src/input/readers/SQLite.cc
View file

@ -36,7 +36,7 @@ SQLite::SQLite(ReaderFrontend *frontend)
BifConst::InputSQLite::empty_field->Len()
);
io = new AsciiFormatter(this, AsciiFormatter::SeparatorInfo(set_separator, unset_field, empty_field));
io = new threading::formatter::Ascii(this, threading::formatter::Ascii::SeparatorInfo(string(), set_separator, unset_field, empty_field));
}
SQLite::~SQLite()

4
src/input/readers/SQLite.h
View file

@ -10,7 +10,7 @@
#include "../ReaderBackend.h"
#include "threading/AsciiFormatter.h"
#include "threading/formatters/Ascii.h"
#include "3rdparty/sqlite3.h"
namespace input { namespace reader {
@ -40,7 +40,7 @@ private:
string query;
sqlite3 *db;
sqlite3_stmt *st;
AsciiFormatter* io;
threading::formatter::Ascii* io;
string set_separator;
string unset_field;

1
src/logging.bif
View file

@ -77,6 +77,7 @@ const separator: string;
const set_separator: string;
const empty_field: string;
const unset_field: string;
const use_json: bool;
# Options for the DataSeries writer.

177
src/logging/writers/Ascii.cc
View file

@ -10,8 +10,7 @@
#include "Ascii.h"
using namespace logging;
using namespace writer;
using namespace logging::writer;
using threading::Value;
using threading::Field;
@ -20,9 +19,46 @@ Ascii::Ascii(WriterFrontend* frontend) : WriterBackend(frontend)
fd = 0;
ascii_done = false;
tsv = false;
}
Ascii::~Ascii()
{
if ( ! ascii_done )
{
fprintf(stderr, "internal error: finish missing\n");
abort();
}
delete formatter;
}
bool Ascii::WriteHeaderField(const string& key, const string& val)
{
string str = meta_prefix + key + separator + val + "\n";
return safe_write(fd, str.c_str(), str.length());
}
void Ascii::CloseFile(double t)
{
if ( ! fd )
return;
if ( include_meta && ! tsv )
WriteHeaderField("close", Timestamp(0));
safe_close(fd);
fd = 0;
}
bool Ascii::DoInit(const WriterInfo& info, int num_fields, const Field* const * fields)
{
assert(! fd);
// Set some default values.
output_to_stdout = BifConst::LogAscii::output_to_stdout;
include_meta = BifConst::LogAscii::include_meta;
use_json = BifConst::LogAscii::use_json;
separator.assign(
(const char*) BifConst::LogAscii::separator->Bytes(),
@ -49,48 +85,81 @@ Ascii::Ascii(WriterFrontend* frontend) : WriterBackend(frontend)
BifConst::LogAscii::meta_prefix->Len()
);
// Set per-filter configuration options.
for ( WriterInfo::config_map::const_iterator i = info.config.begin(); i != info.config.end(); i++ )
{
if ( strcmp(i->first, "tsv") == 0 )
{
if ( strcmp(i->second, "T") == 0 )
tsv = true;
else if ( strcmp(i->second, "F") == 0 )
tsv = false;
else
{
Error("invalid value for 'tsv', must be a string and either \"T\" or \"F\"");
return false;
}
}
else if ( strcmp(i->first, "use_json") == 0 )
{
if ( strcmp(i->second, "T") == 0 )
use_json = true;
else if ( strcmp(i->second, "F") == 0 )
use_json = false;
else
{
Error("invalid value for 'use_json', must be a string and either \"T\" or \"F\"");
return false;
}
}
else if ( strcmp(i->first, "output_to_stdout") == 0 )
{
if ( strcmp(i->second, "T") == 0 )
output_to_stdout = true;
else if ( strcmp(i->second, "F") == 0 )
output_to_stdout = false;
else
{
Error("invalid value for 'output_to_stdout', must be a string and either \"T\" or \"F\"");
return false;
}
}
else if ( strcmp(i->first, "separator") == 0 )
separator.assign(i->second);
else if ( strcmp(i->first, "set_separator") == 0 )
set_separator.assign(i->second);
else if ( strcmp(i->first, "empty_field") == 0 )
empty_field.assign(i->second);
else if ( strcmp(i->first, "unset_field") == 0 )
unset_field.assign(i->second);
else if ( strcmp(i->first, "meta_prefix") == 0 )
meta_prefix.assign(i->second);
}
if ( use_json )
{
// Write out JSON formatted logs.
formatter = new threading::formatter::JSON(this);
// Using JSON implicitly turns off the header meta fields.
include_meta = false;
}
else
{
// Use the default "Bro logs" format.
desc.EnableEscaping();
desc.AddEscapeSequence(separator);
ascii = new AsciiFormatter(this, AsciiFormatter::SeparatorInfo(set_separator, unset_field, empty_field));
formatter = new threading::formatter::Ascii(this, threading::formatter::Ascii::SeparatorInfo(separator, set_separator, unset_field, empty_field));
}
Ascii::~Ascii()
{
if ( ! ascii_done )
{
fprintf(stderr, "internal error: finish missing\n");
abort();
}
delete ascii;
}
bool Ascii::WriteHeaderField(const string& key, const string& val)
{
string str = meta_prefix + key + separator + val + "\n";
return safe_write(fd, str.c_str(), str.length());
}
void Ascii::CloseFile(double t)
{
if ( ! fd )
return;
if ( include_meta && ! tsv )
WriteHeaderField("close", Timestamp(0));
safe_close(fd);
fd = 0;
}
bool Ascii::DoInit(const WriterInfo& info, int num_fields, const Field* const * fields)
{
assert(! fd);
string path = info.path;
if ( output_to_stdout )
path = "/dev/stdout";
@ -106,24 +175,6 @@ bool Ascii::DoInit(const WriterInfo& info, int num_fields, const Field* const *
return false;
}
for ( WriterInfo::config_map::const_iterator i = info.config.begin(); i != info.config.end(); i++ )
{
if ( strcmp(i->first, "tsv") == 0 )
{
if ( strcmp(i->second, "T") == 0 )
tsv = true;
else if ( strcmp(i->second, "F") == 0 )
tsv = false;
else
{
Error("invalid value for 'tsv', must be a string and either \"T\" or \"F\"");
return false;
}
}
}
if ( include_meta )
{
string names;
@ -208,17 +259,9 @@ bool Ascii::DoWrite(int num_fields, const Field* const * fields,
DoInit(Info(), NumFields(), Fields());
desc.Clear();
for ( int i = 0; i < num_fields; i++ )
{
if ( i > 0 )
desc.AddRaw(separator);
if ( ! ascii->Describe(&desc, vals[i], fields[i]->name) )
if ( ! formatter->Describe(&desc, num_fields, fields, vals) )
return false;
}
desc.AddRaw("\n", 1);
desc.AddRaw("\n");
const char* bytes = (const char*)desc.Bytes();
int len = desc.Len();

6
src/logging/writers/Ascii.h
View file

@ -6,7 +6,8 @@
#define LOGGING_WRITER_ASCII_H
#include "../WriterBackend.h"
#include "threading/AsciiFormatter.h"
#include "threading/formatters/Ascii.h"
#include "threading/formatters/JSON.h"
namespace logging { namespace writer {
@ -46,6 +47,7 @@ private:
bool output_to_stdout;
bool include_meta;
bool tsv;
bool use_json;
string separator;
string set_separator;
@ -53,7 +55,7 @@ private:
string unset_field;
string meta_prefix;
AsciiFormatter* ascii;
threading::formatter::Formatter* formatter;
};
}

140
src/logging/writers/ElasticSearch.cc
View file

@ -16,7 +16,6 @@
#include "BroString.h"
#include "NetVar.h"
#include "threading/SerialTypes.h"
#include "threading/AsciiFormatter.h"
#include <curl/curl.h>
#include <curl/easy.h>
@ -53,13 +52,13 @@ ElasticSearch::ElasticSearch(WriterFrontend* frontend) : WriterBackend(frontend)
curl_handle = HTTPSetup();
ascii = new AsciiFormatter(this, AsciiFormatter::SeparatorInfo());
json = new threading::formatter::JSON(this);
}
ElasticSearch::~ElasticSearch()
{
delete [] cluster_name;
delete ascii;
delete json;
}
bool ElasticSearch::DoInit(const WriterInfo& info, int num_fields, const threading::Field* const* fields)
@ -98,133 +97,7 @@ bool ElasticSearch::BatchIndex()
return true;
}
bool ElasticSearch::AddValueToBuffer(ODesc* b, Value* val)
{
switch ( val->type )
{
// ES treats 0 as false and any other value as true so bool types go here.
case TYPE_BOOL:
case TYPE_INT:
b->Add(val->val.int_val);
break;
case TYPE_COUNT:
case TYPE_COUNTER:
{
// ElasticSearch doesn't seem to support unsigned 64bit ints.
if ( val->val.uint_val >= INT64_MAX )
{
Error(Fmt("count value too large: %" PRIu64, val->val.uint_val));
b->AddRaw("null", 4);
}
else
b->Add(val->val.uint_val);
break;
}
case TYPE_PORT:
b->Add(val->val.port_val.port);
break;
case TYPE_SUBNET:
b->AddRaw("\"", 1);
b->Add(ascii->Render(val->val.subnet_val));
b->AddRaw("\"", 1);
break;
case TYPE_ADDR:
b->AddRaw("\"", 1);
b->Add(ascii->Render(val->val.addr_val));
b->AddRaw("\"", 1);
break;
case TYPE_DOUBLE:
case TYPE_INTERVAL:
b->Add(val->val.double_val);
break;
case TYPE_TIME:
{
// ElasticSearch uses milliseconds for timestamps and json only
// supports signed ints (uints can be too large).
uint64_t ts = (uint64_t) (val->val.double_val * 1000);
if ( ts >= INT64_MAX )
{
Error(Fmt("time value too large: %" PRIu64, ts));
b->AddRaw("null", 4);
}
else
b->Add(ts);
break;
}
case TYPE_ENUM:
case TYPE_STRING:
case TYPE_FILE:
case TYPE_FUNC:
{
b->AddRaw("\"", 1);
for ( int i = 0; i < val->val.string_val.length; ++i )
{
char c = val->val.string_val.data[i];
// 2byte Unicode escape special characters.
if ( c < 32 || c > 126 || c == '\n' || c == '"' || c == '\'' || c == '\\' || c == '&' )
{
static const char hex_chars[] = "0123456789abcdef";
b->AddRaw("\\u00", 4);
b->AddRaw(&hex_chars[(c & 0xf0) >> 4], 1);
b->AddRaw(&hex_chars[c & 0x0f], 1);
}
else
b->AddRaw(&c, 1);
}
b->AddRaw("\"", 1);
break;
}
case TYPE_TABLE:
{
b->AddRaw("[", 1);
for ( int j = 0; j < val->val.set_val.size; j++ )
{
if ( j > 0 )
b->AddRaw(",", 1);
AddValueToBuffer(b, val->val.set_val.vals[j]);
}
b->AddRaw("]", 1);
break;
}
case TYPE_VECTOR:
{
b->AddRaw("[", 1);
for ( int j = 0; j < val->val.vector_val.size; j++ )
{
if ( j > 0 )
b->AddRaw(",", 1);
AddValueToBuffer(b, val->val.vector_val.vals[j]);
}
b->AddRaw("]", 1);
break;
}
default:
return false;
}
return true;
}
bool ElasticSearch::AddFieldToBuffer(ODesc *b, Value* val, const Field* field)
{
if ( ! val->present )
return false;
b->AddRaw("\"", 1);
b->Add(field->name);
b->AddRaw("\":", 2);
AddValueToBuffer(b, val);
return true;
}
bool ElasticSearch::DoWrite(int num_fields, const Field* const * fields,
Value** vals)
@ -239,14 +112,7 @@ bool ElasticSearch::DoWrite(int num_fields, const Field* const * fields,
buffer.Add(Info().path);
buffer.AddRaw("\"}}\n", 4);
buffer.AddRaw("{", 1);
for ( int i = 0; i < num_fields; i++ )
{
if ( i > 0 && buffer.Bytes()[buffer.Len()] != ',' && vals[i]->present )
buffer.AddRaw(",", 1);
AddFieldToBuffer(&buffer, vals[i], fields[i]);
}
buffer.AddRaw("}\n", 2);
json->Describe(&buffer, num_fields, fields, vals);
counter++;
if ( counter >= BifConst::LogElasticSearch::max_batch_size ||

3
src/logging/writers/ElasticSearch.h
View file

@ -9,6 +9,7 @@
#define LOGGING_WRITER_ELASTICSEARCH_H
#include <curl/curl.h>
#include "threading/formatters/JSON.h"
#include "../WriterBackend.h"
namespace logging { namespace writer {
@ -73,7 +74,7 @@ private:
uint64 batch_size;
AsciiFormatter* ascii;
threading::formatter::JSON* json;
};
}

2
src/logging/writers/SQLite.cc
View file

@ -35,7 +35,7 @@ SQLite::SQLite(WriterFrontend* frontend)
BifConst::LogSQLite::empty_field->Len()
);
io = new AsciiFormatter(this, AsciiFormatter::SeparatorInfo(set_separator, unset_field, empty_field));
io = new threading::formatter::Ascii(this, threading::formatter::Ascii::SeparatorInfo(string(), set_separator, unset_field, empty_field));
}
SQLite::~SQLite()

4
src/logging/writers/SQLite.h
View file

@ -9,7 +9,7 @@
#include "../WriterBackend.h"
#include "threading/AsciiFormatter.h"
#include "threading/formatters/Ascii.h"
#include "3rdparty/sqlite3.h"
namespace logging { namespace writer {
@ -51,7 +51,7 @@ private:
string unset_field;
string empty_field;
AsciiFormatter* io;
threading::formatter::Ascii* io;
};
}

149
src/threading/Formatter.cc Normal file
View file

@ -0,0 +1,149 @@
// See the file "COPYING" in the main distribution directory for copyright.
#include "config.h"
#include <sstream>
#include <errno.h>
#include "Formatter.h"
#include "bro_inet_ntop.h"
using namespace threading;
using namespace formatter;
using threading::Value;
using threading::Field;
Formatter::Formatter(threading::MsgThread* t)
{
thread = t;
}
Formatter::~Formatter()
{
}
bool Formatter::CheckNumberError(const string& s, const char* end) const
{
// Do this check first, before executing s.c_str() or similar.
// otherwise the value to which *end is pointing at the moment might
// be gone ...
bool endnotnull = (*end != '\0');
if ( s.length() == 0 )
{
thread->Error("Got empty string for number field");
return true;
}
if ( end == s.c_str() ) {
thread->Error(thread->Fmt("String '%s' contained no parseable number", s.c_str()));
return true;
}
if ( endnotnull )
thread->Warning(thread->Fmt("Number '%s' contained non-numeric trailing characters. Ignored trailing characters '%s'", s.c_str(), end));
if ( errno == EINVAL )
{
thread->Error(thread->Fmt("String '%s' could not be converted to a number", s.c_str()));
return true;
}
else if ( errno == ERANGE )
{
thread->Error(thread->Fmt("Number '%s' out of supported range.", s.c_str()));
return true;
}
return false;
}
string Formatter::Render(const threading::Value::addr_t& addr) const
{
if ( addr.family == IPv4 )
{
char s[INET_ADDRSTRLEN];
if ( ! bro_inet_ntop(AF_INET, &addr.in.in4, s, INET_ADDRSTRLEN) )
return "<bad IPv4 address conversion>";
else
return s;
}
else
{
char s[INET6_ADDRSTRLEN];
if ( ! bro_inet_ntop(AF_INET6, &addr.in.in6, s, INET6_ADDRSTRLEN) )
return "<bad IPv6 address conversion>";
else
return s;
}
}
TransportProto Formatter::ParseProto(const string &proto) const
{
if ( proto == "unknown" )
return TRANSPORT_UNKNOWN;
else if ( proto == "tcp" )
return TRANSPORT_TCP;
else if ( proto == "udp" )
return TRANSPORT_UDP;
else if ( proto == "icmp" )
return TRANSPORT_ICMP;
thread->Error(thread->Fmt("Tried to parse invalid/unknown protocol: %s", proto.c_str()));
return TRANSPORT_UNKNOWN;
}
// More or less verbose copy from IPAddr.cc -- which uses reporter.
threading::Value::addr_t Formatter::ParseAddr(const string &s) const
{
threading::Value::addr_t val;
if ( s.find(':') == std::string::npos ) // IPv4.
{
val.family = IPv4;
if ( inet_aton(s.c_str(), &(val.in.in4)) <= 0 )
{
thread->Error(thread->Fmt("Bad address: %s", s.c_str()));
memset(&val.in.in4.s_addr, 0, sizeof(val.in.in4.s_addr));
}
}
else
{
val.family = IPv6;
if ( inet_pton(AF_INET6, s.c_str(), val.in.in6.s6_addr) <=0 )
{
thread->Error(thread->Fmt("Bad address: %s", s.c_str()));
memset(val.in.in6.s6_addr, 0, sizeof(val.in.in6.s6_addr));
}
}
return val;
}
string Formatter::Render(const threading::Value::subnet_t& subnet) const
{
char l[16];
if ( subnet.prefix.family == IPv4 )
modp_uitoa10(subnet.length - 96, l);
else
modp_uitoa10(subnet.length, l);
string s = Render(subnet.prefix) + "/" + l;
return s;
}
string Formatter::Render(double d) const
{
char buf[256];
modp_dtoa(d, buf, 6);
return buf;
}

114
src/threading/AsciiFormatter.h → src/threading/Formatter.h
View file

@ -1,42 +1,20 @@
// See the file "COPYING" in the main distribution directory for copyright.
#ifndef THREADING_ASCII_FORMATTER_H
#define THREADING_ASCII_FORMATTER_H
#ifndef THREADING_FORMATTER_H
#define THREADING_FORMATTER_H
#include "../Desc.h"
#include "MsgThread.h"
namespace threading { namespace formatter {
/**
* A thread-safe class for converting values into a readable ASCII
* representation, and vice versa. This is a utility class that factors out
* common rendering/parsing code needed by a number of input/output threads.
* A thread-safe class for converting values into some textual format.
* This is a base class that implements the interface forcommon
* rendering/parsing code needed by a number of input/output threads.
*/
class AsciiFormatter {
class Formatter {
public:
/**
* A struct to pass the necessary configuration values to the
* AsciiFormatter module on initialization.
*/
struct SeparatorInfo
{
string set_separator; // Separator between set elements.
string unset_field; // String marking an unset field.
string empty_field; // String marking an empty (but set) field.
/**
* Constructor that leaves separators etc unset to dummy
* values. Useful if you use only methods that don't need any
* of them, like StringToAddr, etc.
*/
SeparatorInfo();
/**
* Constructor that defines all the configuration options.
* Use if you need either ValToODesc or EntryToVal.
*/
SeparatorInfo(const string& set_separator, const string& unset_field, const string& empty_field);
};
/**
* Constructor.
*
@ -44,31 +22,69 @@ public:
* some of the thread's methods, e.g., for error reporting and
* internal formatting.
*
* @param info SeparatorInfo structure defining the necessary
* separators.
*/
AsciiFormatter(threading::MsgThread* t, const SeparatorInfo info);
Formatter(threading::MsgThread* t);
/**
* Destructor.
*/
~AsciiFormatter();
virtual ~Formatter();
/**
* Convert a threading value into a corresponding ASCII.
* representation.
* Convert a list of threading values into an implementation specific representation.
*
* @param desc The ODesc object to write to.
*
* @param num_fields The number of fields in the logging record.
*
* @param fields Information about the fields for each of the given log values.
*
* @param vals The field values.
*
* @return Returns true on success, false on error. Errors are also
* flagged via the reporter.
*/
virtual bool Describe(ODesc* desc, int num_fields, const threading::Field* const * fields,
threading::Value** vals) const = 0;
/**
* Convert a threading value into an implementation specific representation.
*
* @param desc The ODesc object to write to.
*
* @param val the Value to render to the ODesc object.
*
* @param The name of a field associated with the value. Used only
* for error reporting.
* @return Returns true on success, false on error. Errors are also
* flagged via the reporter.
*/
virtual bool Describe(ODesc* desc, threading::Value* val) const = 0;
/**
* Convert a threading value into an implementation specific representation.
*
* @param desc The ODesc object to write to.
*
* @param val the Value to render to the ODesc object.
*
* @param The name of a field associated with the value.
*
* @return Returns true on success, false on error. Errors are also
* flagged via the reporter.
*/
bool Describe(ODesc* desc, threading::Value* val, const string& name) const;
virtual bool Describe(ODesc* desc, threading::Value* val, const string& name) const = 0;
/**
* Convert a representation of a field into a value.
*
* @param s The string to parse.
*
* @param The name of a field associated with the value. Used only
* for error reporting.
*
* @return The new value, or null on error. Errors are also flagged
* via the reporter.
*/
virtual threading::Value* ParseValue(string s, string name, TypeTag type, TypeTag subtype = TYPE_ERROR) const = 0;
/**
* Convert an IP address into a string.
@ -98,19 +114,6 @@ public:
*/
string Render(double d) const;
/**
* Convert the ASCII representation of a field into a value.
*
* @param s The string to parse.
*
* @param The name of a field associated with the value. Used only
* for error reporting.
*
* @return The new value, or null on error. Errors are also flagged
* via the reporter.
*/
threading::Value* ParseValue(string s, string name, TypeTag type, TypeTag subtype = TYPE_ERROR) const;
/**
* Convert a string into a TransportProto. The string must be one of
* \c tcp, \c udp, \c icmp, or \c unknown.
@ -132,11 +135,12 @@ public:
*/
threading::Value::addr_t ParseAddr(const string &addr) const;
private:
protected:
bool CheckNumberError(const string& s, const char * end) const;
SeparatorInfo separators;
threading::MsgThread* thread;
};
#endif /* THREADING_ASCII_FORMATTER_H */
}}
#endif /* THREADING_FORMATTER_H */

176
src/threading/AsciiFormatter.cc → src/threading/formatters/Ascii.cc
View file

@ -5,35 +5,59 @@
#include <sstream>
#include <errno.h>
#include "AsciiFormatter.h"
#include "bro_inet_ntop.h"
#include "./Ascii.h"
AsciiFormatter::SeparatorInfo::SeparatorInfo()
using namespace threading::formatter;
Ascii::SeparatorInfo::SeparatorInfo()
{
this->separator = "SHOULD_NOT_BE_USED";
this->set_separator = "SHOULD_NOT_BE_USED";
this->unset_field = "SHOULD_NOT_BE_USED";
this->empty_field = "SHOULD_NOT_BE_USED";
}
AsciiFormatter::SeparatorInfo::SeparatorInfo(const string & set_separator,
const string & unset_field, const string & empty_field)
Ascii::SeparatorInfo::SeparatorInfo(const string & separator,
const string & set_separator,
const string & unset_field,
const string & empty_field)
{
this->separator = separator;
this->set_separator = set_separator;
this->unset_field = unset_field;
this->empty_field = empty_field;
}
AsciiFormatter::AsciiFormatter(threading::MsgThread* t, const SeparatorInfo info)
Ascii::Ascii(threading::MsgThread* t, const SeparatorInfo info) : Formatter(t)
{
thread = t;
this->separators = info;
}
AsciiFormatter::~AsciiFormatter()
Ascii::~Ascii()
{
}
bool AsciiFormatter::Describe(ODesc* desc, threading::Value* val, const string& name) const
bool Ascii::Describe(ODesc* desc, int num_fields, const threading::Field* const * fields,
threading::Value** vals) const
{
for ( int i = 0; i < num_fields; i++ )
{
if ( i > 0 )
desc->AddRaw(separators.separator);
if ( ! Describe(desc, vals[i], fields[i]->name) )
return false;
}
return true;
}
bool Ascii::Describe(ODesc* desc, threading::Value* val, const string& name) const
{
return Describe(desc, val);
}
bool Ascii::Describe(ODesc* desc, threading::Value* val) const
{
if ( ! val->present )
{
@ -133,7 +157,7 @@ bool AsciiFormatter::Describe(ODesc* desc, threading::Value* val, const string&
if ( j > 0 )
desc->AddRaw(separators.set_separator);
if ( ! Describe(desc, val->val.set_val.vals[j], name) )
if ( ! Describe(desc, val->val.set_val.vals[j]) )
{
desc->RemoveEscapeSequence(separators.set_separator);
return false;
@ -158,7 +182,7 @@ bool AsciiFormatter::Describe(ODesc* desc, threading::Value* val, const string&
if ( j > 0 )
desc->AddRaw(separators.set_separator);
if ( ! Describe(desc, val->val.vector_val.vals[j], name) )
if ( ! Describe(desc, val->val.vector_val.vals[j]) )
{
desc->RemoveEscapeSequence(separators.set_separator);
return false;
@ -170,7 +194,7 @@ bool AsciiFormatter::Describe(ODesc* desc, threading::Value* val, const string&
}
default:
thread->Error(thread->Fmt("unsupported field format %d for %s", val->type, name.c_str()));
thread->Error(thread->Fmt("Ascii writer unsupported field format %d", val->type));
return false;
}
@ -178,7 +202,7 @@ bool AsciiFormatter::Describe(ODesc* desc, threading::Value* val, const string&
}
threading::Value* AsciiFormatter::ParseValue(string s, string name, TypeTag type, TypeTag subtype) const
threading::Value* Ascii::ParseValue(string s, string name, TypeTag type, TypeTag subtype) const
{
if ( s.compare(separators.unset_field) == 0 ) // field is not set...
return new threading::Value(type, false);
@ -381,129 +405,3 @@ parse_error:
delete val;
return 0;
}
bool AsciiFormatter::CheckNumberError(const string& s, const char* end) const
{
// Do this check first, before executing s.c_str() or similar.
// otherwise the value to which *end is pointing at the moment might
// be gone ...
bool endnotnull = (*end != '\0');
if ( s.length() == 0 )
{
thread->Error("Got empty string for number field");
return true;
}
if ( end == s.c_str() ) {
thread->Error(thread->Fmt("String '%s' contained no parseable number", s.c_str()));
return true;
}
if ( endnotnull )
thread->Warning(thread->Fmt("Number '%s' contained non-numeric trailing characters. Ignored trailing characters '%s'", s.c_str(), end));
if ( errno == EINVAL )
{
thread->Error(thread->Fmt("String '%s' could not be converted to a number", s.c_str()));
return true;
}
else if ( errno == ERANGE )
{
thread->Error(thread->Fmt("Number '%s' out of supported range.", s.c_str()));
return true;
}
return false;
}
string AsciiFormatter::Render(const threading::Value::addr_t& addr) const
{
if ( addr.family == IPv4 )
{
char s[INET_ADDRSTRLEN];
if ( ! bro_inet_ntop(AF_INET, &addr.in.in4, s, INET_ADDRSTRLEN) )
return "<bad IPv4 address conversion>";
else
return s;
}
else
{
char s[INET6_ADDRSTRLEN];
if ( ! bro_inet_ntop(AF_INET6, &addr.in.in6, s, INET6_ADDRSTRLEN) )
return "<bad IPv6 address conversion>";
else
return s;
}
}
TransportProto AsciiFormatter::ParseProto(const string &proto) const
{
if ( proto == "unknown" )
return TRANSPORT_UNKNOWN;
else if ( proto == "tcp" )
return TRANSPORT_TCP;
else if ( proto == "udp" )
return TRANSPORT_UDP;
else if ( proto == "icmp" )
return TRANSPORT_ICMP;
thread->Error(thread->Fmt("Tried to parse invalid/unknown protocol: %s", proto.c_str()));
return TRANSPORT_UNKNOWN;
}
// More or less verbose copy from IPAddr.cc -- which uses reporter.
threading::Value::addr_t AsciiFormatter::ParseAddr(const string &s) const
{
threading::Value::addr_t val;
if ( s.find(':') == std::string::npos ) // IPv4.
{
val.family = IPv4;
if ( inet_aton(s.c_str(), &(val.in.in4)) <= 0 )
{
thread->Error(thread->Fmt("Bad address: %s", s.c_str()));
memset(&val.in.in4.s_addr, 0, sizeof(val.in.in4.s_addr));
}
}
else
{
val.family = IPv6;
if ( inet_pton(AF_INET6, s.c_str(), val.in.in6.s6_addr) <=0 )
{
thread->Error(thread->Fmt("Bad address: %s", s.c_str()));
memset(val.in.in6.s6_addr, 0, sizeof(val.in.in6.s6_addr));
}
}
return val;
}
string AsciiFormatter::Render(const threading::Value::subnet_t& subnet) const
{
char l[16];
if ( subnet.prefix.family == IPv4 )
modp_uitoa10(subnet.length - 96, l);
else
modp_uitoa10(subnet.length, l);
string s = Render(subnet.prefix) + "/" + l;
return s;
}
string AsciiFormatter::Render(double d) const
{
char buf[256];
modp_dtoa(d, buf, 6);
return buf;
}

62
src/threading/formatters/Ascii.h Normal file
View file

@ -0,0 +1,62 @@
// See the file "COPYING" in the main distribution directory for copyright.
#ifndef THREADING_FORMATTERS_ASCII_H
#define THREADING_FORMATTERS_ASCII_H
#include "../Formatter.h"
namespace threading { namespace formatter {
class Ascii : public Formatter {
public:
/**
* A struct to pass the necessary configuration values to the
* Ascii module on initialization.
*/
struct SeparatorInfo
{
string separator; // Separator between columns
string set_separator; // Separator between set elements.
string unset_field; // String marking an unset field.
string empty_field; // String marking an empty (but set) field.
/**
* Constructor that leaves separators etc unset to dummy
* values. Useful if you use only methods that don't need any
* of them, like StringToAddr, etc.
*/
SeparatorInfo();
/**
* Constructor that defines all the configuration options.
* Use if you need either ValToODesc or EntryToVal.
*/
SeparatorInfo(const string& separator, const string& set_separator, const string& unset_field, const string& empty_field);
};
/**
* Constructor.
*
* @param t The thread that uses this class instance. The class uses
* some of the thread's methods, e.g., for error reporting and
* internal formatting.
*
* @param info SeparatorInfo structure defining the necessary
* separators.
*/
Ascii(threading::MsgThread* t, const SeparatorInfo info);
virtual ~Ascii();
virtual bool Describe(ODesc* desc, threading::Value* val) const;
virtual bool Describe(ODesc* desc, threading::Value* val, const string& name) const;
virtual bool Describe(ODesc* desc, int num_fields, const threading::Field* const * fields,
threading::Value** vals) const;
virtual threading::Value* ParseValue(string s, string name, TypeTag type, TypeTag subtype = TYPE_ERROR) const;
private:
SeparatorInfo separators;
};
}}
#endif /* THREADING_FORMATTERS_ASCII_H */

177
src/threading/formatters/JSON.cc Normal file
View file

@ -0,0 +1,177 @@
// See the file "COPYING" in the main distribution directory for copyright.
#include "config.h"
#include <sstream>
#include <errno.h>
#include "./JSON.h"
using namespace threading::formatter;
JSON::JSON(MsgThread* t) : Formatter(t)
{
}
JSON::~JSON()
{
}
bool JSON::Describe(ODesc* desc, int num_fields, const Field* const * fields,
Value** vals) const
{
desc->AddRaw("{");
for ( int i = 0; i < num_fields; i++ )
{
if ( i > 0 &&
desc->Bytes()[desc->Len()] != ',' && vals[i]->present )
{
desc->AddRaw(",");
}
if ( ! Describe(desc, vals[i], fields[i]->name) )
return false;
}
desc->AddRaw("}");
return true;
}
bool JSON::Describe(ODesc* desc, Value* val, const string& name) const
{
if ( ! val->present )
return true;
desc->AddRaw("\"", 1);
desc->Add(name);
desc->AddRaw("\":", 2);
return Describe(desc, val);
}
bool JSON::Describe(ODesc* desc, Value* val) const
{
if ( ! val->present )
return true;
switch ( val->type )
{
case TYPE_BOOL:
desc->AddRaw(val->val.int_val == 0 ? "false" : "true");
break;
case TYPE_INT:
desc->Add(val->val.int_val);
break;
case TYPE_COUNT:
case TYPE_COUNTER:
{
// JSON doesn't support unsigned 64bit ints.
if ( val->val.uint_val >= INT64_MAX )
{
thread->Error(thread->Fmt("count value too large for JSON: %" PRIu64, val->val.uint_val));
desc->AddRaw("null", 4);
}
else
desc->Add(val->val.uint_val);
break;
}
case TYPE_PORT:
desc->Add(val->val.port_val.port);
break;
case TYPE_SUBNET:
desc->AddRaw("\"", 1);
desc->Add(Render(val->val.subnet_val));
desc->AddRaw("\"", 1);
break;
case TYPE_ADDR:
desc->AddRaw("\"", 1);
desc->Add(Render(val->val.addr_val));
desc->AddRaw("\"", 1);
break;
case TYPE_DOUBLE:
case TYPE_INTERVAL:
desc->Add(val->val.double_val);
break;
case TYPE_TIME:
{
// ElasticSearch uses milliseconds for timestamps and json only
// supports signed ints (uints can be too large).
uint64_t ts = (uint64_t) (val->val.double_val * 1000);
if ( ts >= INT64_MAX )
{
thread->Error(thread->Fmt("time value too large for JSON: %" PRIu64, ts));
desc->AddRaw("null", 4);
}
else
desc->Add(ts);
break;
}
case TYPE_ENUM:
case TYPE_STRING:
case TYPE_FILE:
case TYPE_FUNC:
{
desc->AddRaw("\"", 1);
for ( int i = 0; i < val->val.string_val.length; ++i )
{
char c = val->val.string_val.data[i];
// 2byte Unicode escape special characters.
if ( c < 32 || c > 126 || c == '\n' || c == '"' || c == '\'' || c == '\\' || c == '&' )
{
static const char hex_chars[] = "0123456789abcdef";
desc->AddRaw("\\u00", 4);
desc->AddRaw(&hex_chars[(c & 0xf0) >> 4], 1);
desc->AddRaw(&hex_chars[c & 0x0f], 1);
}
else
desc->AddRaw(&c, 1);
}
desc->AddRaw("\"", 1);
break;
}
case TYPE_TABLE:
{
desc->AddRaw("[", 1);
for ( int j = 0; j < val->val.set_val.size; j++ )
{
if ( j > 0 )
desc->AddRaw(",", 1);
Describe(desc, val->val.set_val.vals[j]);
}
desc->AddRaw("]", 1);
break;
}
case TYPE_VECTOR:
{
desc->AddRaw("[", 1);
for ( int j = 0; j < val->val.vector_val.size; j++ )
{
if ( j > 0 )
desc->AddRaw(",", 1);
Describe(desc, val->val.vector_val.vals[j]);
}
desc->AddRaw("]", 1);
break;
}
default:
return false;
}
return true;
}
threading::Value* JSON::ParseValue(string s, string name, TypeTag type, TypeTag subtype) const
{
thread->Error("JSON formatter does not support parsing yet.");
return NULL;
}

30
src/threading/formatters/JSON.h Normal file
View file

@ -0,0 +1,30 @@
// See the file "COPYING" in the main distribution directory for copyright.
#ifndef THREADING_FORMATTERS_JSON_H
#define THREADING_FORMATTERS_JSON_H
#include "../Formatter.h"
namespace threading { namespace formatter {
/**
* A thread-safe class for converting values into a JSON representation
* and vice versa.
*/
class JSON : public Formatter {
public:
JSON(threading::MsgThread* t);
virtual ~JSON();
virtual bool Describe(ODesc* desc, threading::Value* val) const;
virtual bool Describe(ODesc* desc, threading::Value* val, const string& name) const;
virtual bool Describe(ODesc* desc, int num_fields, const threading::Field* const * fields,
threading::Value** vals) const;
virtual threading::Value* ParseValue(string s, string name, TypeTag type, TypeTag subtype = TYPE_ERROR) const;
private:
};
}}
#endif /* THREADING_FORMATTERS_JSON_H */

1
testing/btest/Baseline/scripts.base.frameworks.logging.ascii-json/ssh.log Normal file
View file

@ -0,0 +1 @@
{"b":true,"i":-42,"e":"SSH::LOG","c":21,"p":123,"sn":"10.0.0.0/24","a":"1.2.3.4","d":3.14,"t":1394462315468,"iv":100.0,"s":"hurz","sc":[2,4,1,3],"ss":["CC","AA","BB"],"se":[],"vc":[10,20,30],"ve":[],"f":"SSH::foo\u000a{ \u000aif (0 < SSH::i) \u000a\u0009return (Foo);\u000aelse\u000a\u0009return (Bar);\u000a\u000a}"}

70
testing/btest/scripts/base/frameworks/logging/ascii-json.bro Normal file
View file

@ -0,0 +1,70 @@
#
# @TEST-EXEC: bro -b %INPUT
# @TEST-EXEC: btest-diff ssh.log
#
# Testing all possible types.
redef LogAscii::use_json = T;
module SSH;
export {
redef enum Log::ID += { LOG };
type Log: record {
b: bool;
i: int;
e: Log::ID;
c: count;
p: port;
sn: subnet;
a: addr;
d: double;
t: time;
iv: interval;
s: string;
sc: set[count];
ss: set[string];
se: set[string];
vc: vector of count;
ve: vector of string;
f: function(i: count) : string;
} &log;
}
function foo(i : count) : string
{
if ( i > 0 )
return "Foo";
else
return "Bar";
}
event bro_init()
{
Log::create_stream(SSH::LOG, [$columns=Log]);
local empty_set: set[string];
local empty_vector: vector of string;
Log::write(SSH::LOG, [
$b=T,
$i=-42,
$e=SSH::LOG,
$c=21,
$p=123/tcp,
$sn=10.0.0.1/24,
$a=1.2.3.4,
$d=3.14,
$t=network_time(),
$iv=100secs,
$s="hurz",
$sc=set(1,2,3,4),
$ss=set("AA", "BB", "CC"),
$se=empty_set,
$vc=vector(10, 20, 30),
$ve=empty_vector,
$f=foo
]);
}