Mirror/zeek
1
0
Fork 0
mirror of https://github.com/zeek/zeek.git synced 2025-10-04 07:38:19 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

Merge branch 'topic/robin/master-test'

Browse source 
* topic/robin/master-test: (60 commits)
  Script fix for Linux.
  Updating test base line.
  Another small change to MsgThread API.
  Bug fix for BasicThread.
  make version_ok return true for TLSv12
  Sed usage in canonifier script didn't work on non-Linux systems.
  Changing HTTP DPD port 3138 to 3128.
  Temporarily removing tuning/logs-to-elasticsearch.bro from the test-all-policy.
  More documentation updates.
  Revert "Fixing calc_next_rotate to use UTC based time functions."
  Some documentation updates for elasticsearch plugin.
  Give configure a --disable-perftools option.
  Updating tests for the #start/#end change.
  Further threading and API restructuring for logging and input frameworks.
  Reworking forceful thread termination.
  Moving the ASCII writer over to use UNIX I/O rather than stdio.
  Further reworking the thread API.
  Reworking thread termination logic.
  If a thread doesn't terminate, we log that but not longer proceed (because it could hang later still).
  Removing the thread kill functionality.
  ...
This commit is contained in:
Robin Sommer 2012-07-23 16:06:34 -07:00
commit 24aea295fa
176 changed files with 2238 additions and 771 deletions
Download patch file Download diff file Expand all files Collapse all files

16
CMakeLists.txt
View file

@ -91,7 +91,9 @@ endif ()
set(USE_PERFTOOLS false)
set(USE_PERFTOOLS_DEBUG false)
if (NOT DISABLE_PERFTOOLS)
find_package(GooglePerftools)
endif ()
if (GOOGLEPERFTOOLS_FOUND)
include_directories(BEFORE ${GooglePerftools_INCLUDE_DIR})
@ -122,6 +124,17 @@ if (LINTEL_FOUND AND DATASERIES_FOUND AND LIBXML2_FOUND)
list(APPEND OPTLIBS ${LibXML2_LIBRARIES})
endif()
set(USE_ELASTICSEARCH false)
set(USE_CURL false)
find_package(CURL)
if (CURL_FOUND)
set(USE_ELASTICSEARCH true)
set(USE_CURL true)
include_directories(BEFORE ${CURL_INCLUDE_DIR})
list(APPEND OPTLIBS ${CURL_LIBRARIES})
endif()
if (ENABLE_PERFTOOLS_DEBUG)
# Just a no op to prevent CMake from complaining about manually-specified
# ENABLE_PERFTOOLS_DEBUG not being used if google perftools weren't found
@ -213,7 +226,10 @@ message(
"\nGeoIP: ${USE_GEOIP}"
"\nGoogle perftools: ${USE_PERFTOOLS}"
"\n debugging: ${USE_PERFTOOLS_DEBUG}"
"\ncURL: ${USE_CURL}"
"\n"
"\nDataSeries: ${USE_DATASERIES}"
"\nElasticSearch: ${USE_ELASTICSEARCH}"
"\n"
"\n================================================================\n"
)

5
NEWS
View file

@ -140,6 +140,11 @@ the full set.
Bro now supports decapsulating tunnels directly for protocols it
understands.
- ASCII logs now record the time when they were opened/closed at the
beginning and end of the file, respectively. The options
LogAscii::header_prefix and LogAscii::include_header have been
renamed to LogAscii::meta_prefix and LogAscii::include_meta,
respectively.
Bro 2.0
-------

6
config.h.in
View file

@ -114,9 +114,15 @@
/* Analyze Mobile IPv6 traffic */
#cmakedefine ENABLE_MOBILE_IPV6
/* Use libCurl. */
#cmakedefine USE_CURL
/* Use the DataSeries writer. */
#cmakedefine USE_DATASERIES
/* Use the ElasticSearch writer. */
#cmakedefine USE_ELASTICSEARCH
/* Version number of package */
#define VERSION "@VERSION@"

5
configure vendored
View file

@ -33,6 +33,7 @@ Usage: $0 [OPTION]... [VAR=VALUE]...
--disable-broccoli don't build or install the Broccoli library
--disable-broctl don't install Broctl
--disable-auxtools don't build or install auxiliary tools
--disable-perftools don't try to build python with Google Perftools
--disable-python don't try to build python bindings for broccoli
--disable-ruby don't try to build ruby bindings for broccoli
@ -105,6 +106,7 @@ append_cache_entry INSTALL_BROCCOLI BOOL true
append_cache_entry INSTALL_BROCTL BOOL true
append_cache_entry CPACK_SOURCE_IGNORE_FILES STRING
append_cache_entry ENABLE_MOBILE_IPV6 BOOL false
append_cache_entry DISABLE_PERFTOOLS BOOL false
# parse arguments
while [ $# -ne 0 ]; do
@ -156,6 +158,9 @@ while [ $# -ne 0 ]; do
--disable-auxtools)
append_cache_entry INSTALL_AUX_TOOLS BOOL false
;;
--disable-perftools)
append_cache_entry DISABLE_PERFTOOLS BOOL true
;;
--disable-python)
append_cache_entry DISABLE_PYTHON_BINDINGS BOOL true
;;

89
doc/logging-elasticsearch.rst Normal file
View file

@ -0,0 +1,89 @@
=========================================
Indexed Logging Output with ElasticSearch
=========================================
.. rst-class:: opening
Bro's default ASCII log format is not exactly the most efficient
way for searching large volumes of data. ElasticSearch
is a new data storage technology for dealing with tons of data.
It's also a search engine built on top of Apache's Lucene
project. It scales very well, both for distributed indexing and
distributed searching.
.. contents::
Warning
-------
This writer plugin is still in testing and is not yet recommended for
production use! The approach to how logs are handled in the plugin is "fire
and forget" at this time, there is no error handling if the server fails to
respond successfully to the insertion request.
Installing ElasticSearch
------------------------
Download the latest version from: <http://www.elasticsearch.org/download/>.
Once extracted, start ElasticSearch with::
# ./bin/elasticsearch
For more detailed information, refer to the ElasticSearch installation
documentation: http://www.elasticsearch.org/guide/reference/setup/installation.html
Compiling Bro with ElasticSearch Support
----------------------------------------
First, ensure that you have libcurl installed the run configure.::
# ./configure
[...]
====================| Bro Build Summary |=====================
[...]
cURL: true
[...]
ElasticSearch: true
[...]
================================================================
Activating ElasticSearch
------------------------
The easiest way to enable ElasticSearch output is to load the tuning/logs-to-
elasticsearch.bro script. If you are using BroControl, the following line in
local.bro will enable it.
.. console::
@load tuning/logs-to-elasticsearch
With that, Bro will now write most of its logs into ElasticSearch in addition
to maintaining the Ascii logs like it would do by default. That script has
some tunable options for choosing which logs to send to ElasticSearch, refer
to the autogenerated script documentation for those options.
There is an interface being written specifically to integrate with the data
that Bro outputs into ElasticSearch named Brownian. It can be found here::
https://github.com/grigorescu/Brownian
Tuning
------
A common problem encountered with ElasticSearch is too many files being held
open. The ElasticSearch website has some suggestions on how to increase the
open file limit.
- http://www.elasticsearch.org/tutorials/2011/04/06/too-many-open-files.html
TODO
----
Lots.
- Perform multicast discovery for server.
- Better error detection.
- Better defaults (don't index loaded-plugins, for instance).
-

2
doc/scripts/DocSourcesList.cmake
View file

@ -42,6 +42,7 @@ rest_target(${psd} base/frameworks/logging/postprocessors/scp.bro)
rest_target(${psd} base/frameworks/logging/postprocessors/sftp.bro)
rest_target(${psd} base/frameworks/logging/writers/ascii.bro)
rest_target(${psd} base/frameworks/logging/writers/dataseries.bro)
rest_target(${psd} base/frameworks/logging/writers/elasticsearch.bro)
rest_target(${psd} base/frameworks/logging/writers/none.bro)
rest_target(${psd} base/frameworks/metrics/cluster.bro)
rest_target(${psd} base/frameworks/metrics/main.bro)
@ -145,6 +146,7 @@ rest_target(${psd} policy/protocols/ssl/known-certs.bro)
rest_target(${psd} policy/protocols/ssl/validate-certs.bro)
rest_target(${psd} policy/tuning/defaults/packet-fragments.bro)
rest_target(${psd} policy/tuning/defaults/warnings.bro)
rest_target(${psd} policy/tuning/logs-to-elasticsearch.bro)
rest_target(${psd} policy/tuning/track-all-assets.bro)
rest_target(${psd} site/local-manager.bro)
rest_target(${psd} site/local-proxy.bro)

1
scripts/base/frameworks/logging/__load__.bro
View file

@ -2,4 +2,5 @@
@load ./postprocessors
@load ./writers/ascii
@load ./writers/dataseries
@load ./writers/elasticsearch
@load ./writers/none

11
scripts/base/frameworks/logging/writers/ascii.bro
View file

@ -8,12 +8,13 @@ export {
## into files. This is primarily for debugging purposes.
const output_to_stdout = F &redef;
## If true, include a header line with column names and description
## of the other ASCII logging options that were used.
const include_header = T &redef;
## If true, include lines with log meta information such as column names with
## types, the values of ASCII logging options that in use, and the time when the
## file was opened and closes (the latter at the end).
const include_meta = T &redef;
## Prefix for the header line if included.
const header_prefix = "#" &redef;
## Prefix for lines with meta information.
const meta_prefix = "#" &redef;
## Separator between fields.
const separator = "\t" &redef;

46
scripts/base/frameworks/logging/writers/elasticsearch.bro Normal file
View file

@ -0,0 +1,46 @@
##! Log writer for sending logs to an ElasticSearch server.
##!
##! Note: This module is in testing and is not yet considered stable!
##!
##! There is one known memory issue. If your elasticsearch server is
##! running slowly and taking too long to return from bulk insert
##! requests, the message queue to the writer thread will continue
##! growing larger and larger giving the appearance of a memory leak.
module LogElasticSearch;
export {
## Name of the ES cluster
const cluster_name = "elasticsearch" &redef;
## ES Server
const server_host = "127.0.0.1" &redef;
## ES Port
const server_port = 9200 &redef;
## Name of the ES index
const index_prefix = "bro" &redef;
## The ES type prefix comes before the name of the related log.
## e.g. prefix = "bro_" would create types of bro_dns, bro_software, etc.
const type_prefix = "" &redef;
## The time before an ElasticSearch transfer will timeout.
## This is not working!
const transfer_timeout = 2secs;
## The batch size is the number of messages that will be queued up before
## they are sent to be bulk indexed.
const max_batch_size = 1000 &redef;
## The maximum amount of wall-clock time that is allowed to pass without
## finishing a bulk log send. This represents the maximum delay you
## would like to have with your logs before they are sent to ElasticSearch.
const max_batch_interval = 1min &redef;
## The maximum byte size for a buffered JSON string to send to the bulk
## insert API.
const max_byte_size = 1024 * 1024 &redef;
}

4
scripts/base/protocols/dns/main.bro
View file

@ -162,11 +162,11 @@ function set_session(c: connection, msg: dns_msg, is_query: bool)
c$dns = c$dns_state$pending[msg$id];
if ( ! is_query )
{
c$dns$rcode = msg$rcode;
c$dns$rcode_name = base_errors[msg$rcode];
if ( ! is_query )
{
if ( ! c$dns?$total_answers )
c$dns$total_answers = msg$num_answers;

2
scripts/base/protocols/http/main.bro
View file

@ -114,7 +114,7 @@ event bro_init() &priority=5
# DPD configuration.
const ports = {
80/tcp, 81/tcp, 631/tcp, 1080/tcp, 3138/tcp,
80/tcp, 81/tcp, 631/tcp, 1080/tcp, 3128/tcp,
8000/tcp, 8080/tcp, 8888/tcp,
};
redef dpd_config += {

14
scripts/base/protocols/ssl/mozilla-ca-list.bro
View file

File diff suppressed because one or more lines are too long

45
scripts/policy/tuning/logs-to-elasticsearch.bro Normal file
View file

@ -0,0 +1,45 @@
##! Load this script to enable global log output to an ElasticSearch database.
module LogElasticSearch;
export {
## An elasticsearch specific rotation interval.
const rotation_interval = 24hr &redef;
## Optionally ignore any :bro:enum:`Log::ID` from being sent to
## ElasticSearch with this script.
const excluded_log_ids: set[string] = set("Communication::LOG") &redef;
## If you want to explicitly only send certain :bro:enum:`Log::ID`
## streams, add them to this set. If the set remains empty, all will
## be sent. The :bro:id:`excluded_log_ids` option will remain in
## effect as well.
const send_logs: set[string] = set() &redef;
}
module Log;
event bro_init() &priority=-5
{
local my_filters: table[ID, string] of Filter = table();
for ( [id, name] in filters )
{
local filter = filters[id, name];
if ( fmt("%s", id) in LogElasticSearch::excluded_log_ids ||
(|LogElasticSearch::send_logs| > 0 && fmt("%s", id) !in LogElasticSearch::send_logs) )
next;
filter$name = cat(name, "-es");
filter$writer = Log::WRITER_ELASTICSEARCH;
filter$interv = LogElasticSearch::rotation_interval;
my_filters[id, name] = filter;
}
# This had to be done separately to avoid an ever growing filters list
# where the for loop would never end.
for ( [id, name] in my_filters )
{
Log::add_filter(id, filter);
}
}

1
scripts/test-all-policy.bro
View file

@ -60,4 +60,5 @@
@load tuning/defaults/__load__.bro
@load tuning/defaults/packet-fragments.bro
@load tuning/defaults/warnings.bro
# @load tuning/logs-to-elasticsearch.bro
@load tuning/track-all-assets.bro

1
src/CMakeLists.txt
View file

@ -428,6 +428,7 @@ set(bro_SRCS
logging/WriterFrontend.cc
logging/writers/Ascii.cc
logging/writers/DataSeries.cc
logging/writers/ElasticSearch.cc
logging/writers/None.cc
input/Manager.cc

4
src/RemoteSerializer.cc
View file

@ -2692,12 +2692,12 @@ bool RemoteSerializer::ProcessLogCreateWriter()
int id, writer;
int num_fields;
logging::WriterBackend::WriterInfo info;
logging::WriterBackend::WriterInfo* info = new logging::WriterBackend::WriterInfo();
bool success = fmt.Read(&id, "id") &&
fmt.Read(&writer, "writer") &&
fmt.Read(&num_fields, "num_fields") &&
info.Read(&fmt);
info->Read(&fmt);
if ( ! success )
goto error;

70
src/input/Manager.cc
View file

@ -71,7 +71,7 @@ declare(PDict, InputHash);
class Manager::Stream {
public:
string name;
ReaderBackend::ReaderInfo info;
ReaderBackend::ReaderInfo* info;
bool removed;
StreamType stream_type; // to distinguish between event and table streams
@ -257,7 +257,6 @@ ReaderBackend* Manager::CreateBackend(ReaderFrontend* frontend, bro_int_t type)
assert(ir->factory);
frontend->SetTypeName(ir->name);
ReaderBackend* backend = (*ir->factory)(frontend);
assert(backend);
@ -291,9 +290,6 @@ bool Manager::CreateStream(Stream* info, RecordVal* description)
EnumVal* reader = description->LookupWithDefault(rtype->FieldOffset("reader"))->AsEnumVal();
ReaderFrontend* reader_obj = new ReaderFrontend(reader->InternalInt());
assert(reader_obj);
// get the source ...
Val* sourceval = description->LookupWithDefault(rtype->FieldOffset("source"));
assert ( sourceval != 0 );
@ -301,21 +297,22 @@ bool Manager::CreateStream(Stream* info, RecordVal* description)
string source((const char*) bsource->Bytes(), bsource->Len());
Unref(sourceval);
EnumVal* mode = description->LookupWithDefault(rtype->FieldOffset("mode"))->AsEnumVal();
Val* config = description->LookupWithDefault(rtype->FieldOffset("config"));
ReaderBackend::ReaderInfo* rinfo = new ReaderBackend::ReaderInfo();
rinfo->source = copy_string(source.c_str());
EnumVal* mode = description->LookupWithDefault(rtype->FieldOffset("mode"))->AsEnumVal();
switch ( mode->InternalInt() )
{
case 0:
info->info.mode = MODE_MANUAL;
rinfo->mode = MODE_MANUAL;
break;
case 1:
info->info.mode = MODE_REREAD;
rinfo->mode = MODE_REREAD;
break;
case 2:
info->info.mode = MODE_STREAM;
rinfo->mode = MODE_STREAM;
break;
default:
@ -324,12 +321,16 @@ bool Manager::CreateStream(Stream* info, RecordVal* description)
Unref(mode);
Val* config = description->LookupWithDefault(rtype->FieldOffset("config"));
ReaderFrontend* reader_obj = new ReaderFrontend(*rinfo, reader);
assert(reader_obj);
info->reader = reader_obj;
info->type = reader->AsEnumVal(); // ref'd by lookupwithdefault
info->name = name;
info->config = config->AsTableVal(); // ref'd by LookupWithDefault
info->info.source = source;
info->info = rinfo;
Ref(description);
info->description = description;
@ -344,7 +345,7 @@ bool Manager::CreateStream(Stream* info, RecordVal* description)
ListVal* index = info->config->RecoverIndex(k);
string key = index->Index(0)->AsString()->CheckString();
string value = v->Value()->AsString()->CheckString();
info->info.config.insert(std::make_pair(key, value));
info->info->config.insert(std::make_pair(copy_string(key.c_str()), copy_string(value.c_str())));
Unref(index);
delete k;
}
@ -475,7 +476,7 @@ bool Manager::CreateEventStream(RecordVal* fval)
assert(stream->reader);
stream->reader->Init(stream->info, stream->num_fields, logf );
stream->reader->Init(stream->num_fields, logf );
readers[stream->reader] = stream;
@ -652,7 +653,7 @@ bool Manager::CreateTableStream(RecordVal* fval)
assert(stream->reader);
stream->reader->Init(stream->info, fieldsV.size(), fields );
stream->reader->Init(fieldsV.size(), fields );
readers[stream->reader] = stream;
@ -726,8 +727,6 @@ bool Manager::RemoveStream(Stream *i)
i->removed = true;
i->reader->Close();
DBG_LOG(DBG_INPUT, "Successfully queued removal of stream %s",
i->name.c_str());
@ -793,17 +792,19 @@ bool Manager::UnrollRecordType(vector<Field*> *fields,
else
{
Field* field = new Field();
field->name = nameprepend + rec->FieldName(i);
field->type = rec->FieldType(i)->Tag();
string name = nameprepend + rec->FieldName(i);
const char* secondary = 0;
TypeTag ty = rec->FieldType(i)->Tag();
TypeTag st = TYPE_VOID;
bool optional = false;
if ( field->type == TYPE_TABLE )
field->subtype = rec->FieldType(i)->AsSetType()->Indices()->PureType()->Tag();
if ( ty == TYPE_TABLE )
st = rec->FieldType(i)->AsSetType()->Indices()->PureType()->Tag();
else if ( field->type == TYPE_VECTOR )
field->subtype = rec->FieldType(i)->AsVectorType()->YieldType()->Tag();
else if ( ty == TYPE_VECTOR )
st = rec->FieldType(i)->AsVectorType()->YieldType()->Tag();
else if ( field->type == TYPE_PORT &&
else if ( ty == TYPE_PORT &&
rec->FieldDecl(i)->FindAttr(ATTR_TYPE_COLUMN) )
{
// we have an annotation for the second column
@ -813,12 +814,13 @@ bool Manager::UnrollRecordType(vector<Field*> *fields,
assert(c);
assert(c->Type()->Tag() == TYPE_STRING);
field->secondary_name = c->AsStringVal()->AsString()->CheckString();
secondary = c->AsStringVal()->AsString()->CheckString();
}
if ( rec->FieldDecl(i)->FindAttr(ATTR_OPTIONAL ) )
field->optional = true;
optional = true;
Field* field = new Field(name.c_str(), secondary, ty, st, optional);
fields->push_back(field);
}
}
@ -1232,7 +1234,7 @@ void Manager::EndCurrentSend(ReaderFrontend* reader)
#endif
// Send event that the current update is indeed finished.
SendEvent(update_finished, 2, new StringVal(i->name.c_str()), new StringVal(i->info.source.c_str()));
SendEvent(update_finished, 2, new StringVal(i->name.c_str()), new StringVal(i->info->source));
}
void Manager::Put(ReaderFrontend* reader, Value* *vals)
@ -1709,7 +1711,7 @@ int Manager::GetValueLength(const Value* val) {
case TYPE_STRING:
case TYPE_ENUM:
{
length += val->val.string_val->size();
length += val->val.string_val.length;
break;
}
@ -1808,8 +1810,8 @@ int Manager::CopyValue(char *data, const int startpos, const Value* val)
case TYPE_STRING:
case TYPE_ENUM:
{
memcpy(data+startpos, val->val.string_val->c_str(), val->val.string_val->length());
return val->val.string_val->size();
memcpy(data+startpos, val->val.string_val.data, val->val.string_val.length);
return val->val.string_val.length;
}
case TYPE_ADDR:
@ -1957,7 +1959,7 @@ Val* Manager::ValueToVal(const Value* val, BroType* request_type)
case TYPE_STRING:
{
BroString *s = new BroString(*(val->val.string_val));
BroString *s = new BroString((const u_char*)val->val.string_val.data, val->val.string_val.length, 0);
return new StringVal(s);
}
@ -2041,8 +2043,8 @@ Val* Manager::ValueToVal(const Value* val, BroType* request_type)
case TYPE_ENUM: {
// well, this is kind of stupid, because EnumType just mangles the module name and the var name together again...
// but well
string module = extract_module_name(val->val.string_val->c_str());
string var = extract_var_name(val->val.string_val->c_str());
string module = extract_module_name(val->val.string_val.data);
string var = extract_var_name(val->val.string_val.data);
bro_int_t index = request_type->AsEnumType()->Lookup(module, var.c_str());
if ( index == -1 )
reporter->InternalError("Value not found in enum mappimg. Module: %s, var: %s",

30
src/input/ReaderBackend.cc
View file

@ -56,22 +56,24 @@ private:
class SendEventMessage : public threading::OutputMessage<ReaderFrontend> {
public:
SendEventMessage(ReaderFrontend* reader, const string& name, const int num_vals, Value* *val)
SendEventMessage(ReaderFrontend* reader, const char* name, const int num_vals, Value* *val)
: threading::OutputMessage<ReaderFrontend>("SendEvent", reader),
name(name), num_vals(num_vals), val(val) {}
name(copy_string(name)), num_vals(num_vals), val(val) {}
virtual ~SendEventMessage() { delete [] name; }
virtual bool Process()
{
bool success = input_mgr->SendEvent(name, num_vals, val);
if ( ! success )
reporter->Error("SendEvent for event %s failed", name.c_str());
reporter->Error("SendEvent for event %s failed", name);
return true; // We do not want to die if sendEvent fails because the event did not return.
}
private:
const string name;
const char* name;
const int num_vals;
Value* *val;
};
@ -146,12 +148,14 @@ ReaderBackend::ReaderBackend(ReaderFrontend* arg_frontend) : MsgThread()
{
disabled = true; // disabled will be set correcty in init.
frontend = arg_frontend;
info = new ReaderInfo(frontend->Info());
SetName(frontend->Name());
}
ReaderBackend::~ReaderBackend()
{
delete info;
}
void ReaderBackend::Put(Value* *val)
@ -169,7 +173,7 @@ void ReaderBackend::Clear()
SendOut(new ClearMessage(frontend));
}
void ReaderBackend::SendEvent(const string& name, const int num_vals, Value* *vals)
void ReaderBackend::SendEvent(const char* name, const int num_vals, Value* *vals)
{
SendOut(new SendEventMessage(frontend, name, num_vals, vals));
}
@ -184,17 +188,14 @@ void ReaderBackend::SendEntry(Value* *vals)
SendOut(new SendEntryMessage(frontend, vals));
}
bool ReaderBackend::Init(const ReaderInfo& arg_info, const int arg_num_fields,
bool ReaderBackend::Init(const int arg_num_fields,
const threading::Field* const* arg_fields)
{
info = arg_info;
num_fields = arg_num_fields;
fields = arg_fields;
SetName("InputReader/"+info.source);
// disable if DoInit returns error.
int success = DoInit(arg_info, arg_num_fields, arg_fields);
int success = DoInit(*info, arg_num_fields, arg_fields);
if ( ! success )
{
@ -207,7 +208,7 @@ bool ReaderBackend::Init(const ReaderInfo& arg_info, const int arg_num_fields,
return success;
}
void ReaderBackend::Close()
bool ReaderBackend::OnFinish(double network_time)
{
DoClose();
disabled = true; // frontend disables itself when it gets the Close-message.
@ -221,6 +222,8 @@ void ReaderBackend::Close()
delete [] (fields);
fields = 0;
}
return true;
}
bool ReaderBackend::Update()
@ -243,10 +246,9 @@ void ReaderBackend::DisableFrontend()
SendOut(new DisableMessage(frontend));
}
bool ReaderBackend::DoHeartbeat(double network_time, double current_time)
bool ReaderBackend::OnHeartbeat(double network_time, double current_time)
{
MsgThread::DoHeartbeat(network_time, current_time);
return true;
return DoHeartbeat(network_time, current_time);
}
TransportProto ReaderBackend::StringToProto(const string &proto)

74
src/input/ReaderBackend.h
View file

@ -34,7 +34,10 @@ enum ReaderMode {
* for new appended data. When new data is appended is has to be sent
* using the Put api functions.
*/
MODE_STREAM
MODE_STREAM,
/** Internal dummy mode for initialization. */
MODE_NONE
};
class ReaderFrontend;
@ -70,14 +73,17 @@ public:
*/
struct ReaderInfo
{
typedef std::map<string, string> config_map;
// Structure takes ownership of the strings.
typedef std::map<const char*, const char*> config_map;
/**
* A string left to the interpretation of the reader
* implementation; it corresponds to the value configured on
* the script-level for the logging filter.
*
* Structure takes ownership of the string.
*/
string source;
const char* source;
/**
* A map of key/value pairs corresponding to the relevant
@ -89,6 +95,35 @@ public:
* The opening mode for the input source.
*/
ReaderMode mode;
ReaderInfo()
{
source = 0;
mode = MODE_NONE;
}
ReaderInfo(const ReaderInfo& other)
{
source = other.source ? copy_string(other.source) : 0;
mode = other.mode;
for ( config_map::const_iterator i = other.config.begin(); i != other.config.end(); i++ )
config.insert(std::make_pair(copy_string(i->first), copy_string(i->second)));
}
~ReaderInfo()
{
delete [] source;
for ( config_map::iterator i = config.begin(); i != config.end(); i++ )
{
delete [] i->first;
delete [] i->second;
}
}
private:
const ReaderInfo& operator=(const ReaderInfo& other); // Disable.
};
/**
@ -106,16 +141,7 @@ public:
*
* @return False if an error occured.
*/
bool Init(const ReaderInfo& info, int num_fields, const threading::Field* const* fields);
/**
* Finishes reading from this input stream in a regular fashion. Must
* not be called if an error has been indicated earlier. After
* calling this, no further reading from the stream can be performed.
*
* @return False if an error occured.
*/
void Close();
bool Init(int num_fields, const threading::Field* const* fields);
/**
* Force trigger an update of the input stream. The action that will
@ -142,13 +168,16 @@ public:
/**
* Returns the additional reader information into the constructor.
*/
const ReaderInfo& Info() const { return info; }
const ReaderInfo& Info() const { return *info; }
/**
* Returns the number of log fields as passed into the constructor.
*/
int NumFields() const { return num_fields; }
// Overridden from MsgThread.
virtual bool OnHeartbeat(double network_time, double current_time);
virtual bool OnFinish(double network_time);
protected:
// Methods that have to be overwritten by the individual readers
@ -200,6 +229,11 @@ protected:
*/
virtual bool DoUpdate() = 0;
/**
* Triggered by regular heartbeat messages from the main thread.
*/
virtual bool DoHeartbeat(double network_time, double current_time) = 0;
/**
* Method allowing a reader to send a specified Bro event. Vals must
* match the values expected by the bro event.
@ -210,7 +244,7 @@ protected:
*
* @param vals the values to be given to the event
*/
void SendEvent(const string& name, const int num_vals, threading::Value* *vals);
void SendEvent(const char* name, const int num_vals, threading::Value* *vals);
// Content-sending-functions (simple mode). Include table-specific
// functionality that simply is not used if we have no table.
@ -271,14 +305,6 @@ protected:
*/
void EndCurrentSend();
/**
* Triggered by regular heartbeat messages from the main thread.
*
* This method can be overridden but once must call
* ReaderBackend::DoHeartbeat().
*/
virtual bool DoHeartbeat(double network_time, double current_time);
/**
* Convert a string into a TransportProto. This is just a utility
* function for Readers.
@ -300,7 +326,7 @@ private:
// from this class, it's running in a different thread!
ReaderFrontend* frontend;
ReaderInfo info;
ReaderInfo* info;
unsigned int num_fields;
const threading::Field* const * fields; // raw mapping

56
src/input/ReaderFrontend.cc
View file

@ -11,18 +11,17 @@ namespace input {
class InitMessage : public threading::InputMessage<ReaderBackend>
{
public:
InitMessage(ReaderBackend* backend, const ReaderBackend::ReaderInfo& info,
InitMessage(ReaderBackend* backend,
const int num_fields, const threading::Field* const* fields)
: threading::InputMessage<ReaderBackend>("Init", backend),
info(info), num_fields(num_fields), fields(fields) { }
num_fields(num_fields), fields(fields) { }
virtual bool Process()
{
return Object()->Init(info, num_fields, fields);
return Object()->Init(num_fields, fields);
}
private:
const ReaderBackend::ReaderInfo info;
const int num_fields;
const threading::Field* const* fields;
};
@ -37,32 +36,26 @@ public:
virtual bool Process() { return Object()->Update(); }
};
class CloseMessage : public threading::InputMessage<ReaderBackend>
{
public:
CloseMessage(ReaderBackend* backend)
: threading::InputMessage<ReaderBackend>("Close", backend)
{ }
virtual bool Process() { Object()->Close(); return true; }
};
ReaderFrontend::ReaderFrontend(bro_int_t type)
ReaderFrontend::ReaderFrontend(const ReaderBackend::ReaderInfo& arg_info, EnumVal* type)
{
disabled = initialized = false;
ty_name = "<not set>";
backend = input_mgr->CreateBackend(this, type);
info = new ReaderBackend::ReaderInfo(arg_info);
const char* t = type->Type()->AsEnumType()->Lookup(type->InternalInt());
name = copy_string(fmt("%s/%s", arg_info.source, t));
backend = input_mgr->CreateBackend(this, type->InternalInt());
assert(backend);
backend->Start();
}
ReaderFrontend::~ReaderFrontend()
{
delete [] name;
delete info;
}
void ReaderFrontend::Init(const ReaderBackend::ReaderInfo& arg_info, const int arg_num_fields,
void ReaderFrontend::Init(const int arg_num_fields,
const threading::Field* const* arg_fields)
{
if ( disabled )
@ -71,12 +64,11 @@ void ReaderFrontend::Init(const ReaderBackend::ReaderInfo& arg_info, const int a
if ( initialized )
reporter->InternalError("reader initialize twice");
info = arg_info;
num_fields = arg_num_fields;
fields = arg_fields;
initialized = true;
backend->SendIn(new InitMessage(backend, info, num_fields, fields));
backend->SendIn(new InitMessage(backend, num_fields, fields));
}
void ReaderFrontend::Update()
@ -93,27 +85,9 @@ void ReaderFrontend::Update()
backend->SendIn(new UpdateMessage(backend));
}
void ReaderFrontend::Close()
const char* ReaderFrontend::Name() const
{
if ( disabled )
return;
if ( ! initialized )
{
reporter->Error("Tried to call finish on uninitialized reader");
return;
}
disabled = true;
backend->SendIn(new CloseMessage(backend));
}
string ReaderFrontend::Name() const
{
if ( ! info.source.size() )
return ty_name;
return ty_name + "/" + info.source;
return name;
}
}

27
src/input/ReaderFrontend.h
View file

@ -4,10 +4,11 @@
#define INPUT_READERFRONTEND_H
#include "ReaderBackend.h"
#include "threading/MsgThread.h"
#include "threading/SerialTypes.h"
#include "Val.h"
namespace input {
class Manager;
@ -25,6 +26,8 @@ public:
/**
* Constructor.
*
* info: The meta information struct for the writer.
*
* type: The backend writer type, with the value corresponding to the
* script-level \c Input::Reader enum (e.g., \a READER_ASCII). The
* frontend will internally instantiate a ReaderBackend of the
@ -32,7 +35,7 @@ public:
*
* Frontends must only be instantiated by the main thread.
*/
ReaderFrontend(bro_int_t type);
ReaderFrontend(const ReaderBackend::ReaderInfo& info, EnumVal* type);
/**
* Destructor.
@ -52,7 +55,7 @@ public:
*
* This method must only be called from the main thread.
*/
void Init(const ReaderBackend::ReaderInfo& info, const int arg_num_fields, const threading::Field* const* fields);
void Init(const int arg_num_fields, const threading::Field* const* fields);
/**
* Force an update of the current input source. Actual action depends
@ -100,12 +103,12 @@ public:
*
* This method is safe to call from any thread.
*/
string Name() const;
const char* Name() const;
/**
* Returns the additional reader information passed into the constructor.
*/
const ReaderBackend::ReaderInfo& Info() const { return info; }
const ReaderBackend::ReaderInfo& Info() const { assert(info); return *info; }
/**
* Returns the number of log fields as passed into the constructor.
@ -120,24 +123,14 @@ public:
protected:
friend class Manager;
/**
* Returns the name of the backend's type.
*/
const string& TypeName() const { return ty_name; }
/**
* Sets the name of the backend's type.
*/
void SetTypeName(const string& name) { ty_name = name; }
private:
ReaderBackend* backend; // The backend we have instanatiated.
ReaderBackend::ReaderInfo info; // Meta information as passed to Init().
ReaderBackend::ReaderInfo* info; // Meta information.
const threading::Field* const* fields; // The input fields.
int num_fields; // Information as passed to Init().
string ty_name; // Backend type, set by manager.
bool disabled; // True if disabled.
bool initialized; // True if initialized.
const char* name; // Descriptive name.
};
}

25
src/input/readers/Ascii.cc
View file

@ -87,10 +87,10 @@ bool Ascii::DoInit(const ReaderInfo& info, int num_fields, const Field* const* f
{
mtime = 0;
file = new ifstream(info.source.c_str());
file = new ifstream(info.source);
if ( ! file->is_open() )
{
Error(Fmt("Init: cannot open %s", info.source.c_str()));
Error(Fmt("Init: cannot open %s", info.source));
delete(file);
file = 0;
return false;
@ -98,7 +98,7 @@ bool Ascii::DoInit(const ReaderInfo& info, int num_fields, const Field* const* f
if ( ReadHeader(false) == false )
{
Error(Fmt("Init: cannot open %s; headers are incorrect", info.source.c_str()));
Error(Fmt("Init: cannot open %s; headers are incorrect", info.source));
file->close();
delete(file);
file = 0;
@ -164,20 +164,20 @@ bool Ascii::ReadHeader(bool useCached)
}
Error(Fmt("Did not find requested field %s in input data file %s.",
field->name.c_str(), Info().source.c_str()));
field->name, Info().source));
return false;
}
FieldMapping f(field->name, field->type, field->subtype, ifields[field->name]);
if ( field->secondary_name != "" )
if ( field->secondary_name && strlen(field->secondary_name) != 0 )
{
map<string, uint32_t>::iterator fit2 = ifields.find(field->secondary_name);
if ( fit2 == ifields.end() )
{
Error(Fmt("Could not find requested port type field %s in input data file.",
field->secondary_name.c_str()));
field->secondary_name));
return false;
}
@ -220,7 +220,8 @@ Value* Ascii::EntryToVal(string s, FieldMapping field)
switch ( field.type ) {
case TYPE_ENUM:
case TYPE_STRING:
val->val.string_val = new string(s);
val->val.string_val.length = s.size();
val->val.string_val.data = copy_string(s.c_str());
break;
case TYPE_BOOL:
@ -367,9 +368,9 @@ bool Ascii::DoUpdate()
{
// check if the file has changed
struct stat sb;
if ( stat(Info().source.c_str(), &sb) == -1 )
if ( stat(Info().source, &sb) == -1 )
{
Error(Fmt("Could not get stat for %s", Info().source.c_str()));
Error(Fmt("Could not get stat for %s", Info().source));
return false;
}
@ -403,10 +404,10 @@ bool Ascii::DoUpdate()
file = 0;
}
file = new ifstream(Info().source.c_str());
file = new ifstream(Info().source);
if ( ! file->is_open() )
{
Error(Fmt("cannot open %s", Info().source.c_str()));
Error(Fmt("cannot open %s", Info().source));
return false;
}
@ -506,8 +507,6 @@ bool Ascii::DoUpdate()
bool Ascii::DoHeartbeat(double network_time, double current_time)
{
ReaderBackend::DoHeartbeat(network_time, current_time);
switch ( Info().mode ) {
case MODE_MANUAL:
// yay, we do nothing :)

9
src/input/readers/Benchmark.cc
View file

@ -38,7 +38,7 @@ void Benchmark::DoClose()
bool Benchmark::DoInit(const ReaderInfo& info, int num_fields, const Field* const* fields)
{
num_lines = atoi(info.source.c_str());
num_lines = atoi(info.source);
if ( autospread != 0.0 )
autospread_time = (int) ( (double) 1000000 / (autospread * (double) num_lines) );
@ -126,8 +126,12 @@ threading::Value* Benchmark::EntryToVal(TypeTag type, TypeTag subtype)
assert(false); // no enums, please.
case TYPE_STRING:
val->val.string_val = new string(RandomString(10));
{
string rnd = RandomString(10);
val->val.string_val.data = copy_string(rnd.c_str());
val->val.string_val.length = rnd.size();
break;
}
case TYPE_BOOL:
val->val.int_val = 1; // we never lie.
@ -222,7 +226,6 @@ threading::Value* Benchmark::EntryToVal(TypeTag type, TypeTag subtype)
bool Benchmark::DoHeartbeat(double network_time, double current_time)
{
ReaderBackend::DoHeartbeat(network_time, current_time);
num_lines = (int) ( (double) num_lines*multiplication_factor);
num_lines += add;
heartbeatstarttime = CurrTime();

12
src/input/readers/Raw.cc
View file

@ -108,7 +108,7 @@ bool Raw::DoInit(const ReaderInfo& info, int num_fields, const Field* const* fie
firstrun = true;
bool result;
if ( info.source.length() == 0 )
if ( ! info.source || strlen(info.source) == 0 )
{
Error("No source path provided");
return false;
@ -129,11 +129,12 @@ bool Raw::DoInit(const ReaderInfo& info, int num_fields, const Field* const* fie
}
// do Initialization
char last = info.source[info.source.length()-1];
string source = string(info.source);
char last = info.source[source.length() - 1];
if ( last == '|' )
{
execute = true;
fname = info.source.substr(0, fname.length() - 1);
fname = source.substr(0, fname.length() - 1);
if ( (info.mode != MODE_MANUAL) )
{
@ -237,7 +238,8 @@ bool Raw::DoUpdate()
// filter has exactly one text field. convert to it.
Value* val = new Value(TYPE_STRING, true);
val->val.string_val = new string(line);
val->val.string_val.data = copy_string(line.c_str());
val->val.string_val.length = line.size();
fields[0] = val;
Put(fields);
@ -252,8 +254,6 @@ bool Raw::DoUpdate()
bool Raw::DoHeartbeat(double network_time, double current_time)
{
ReaderBackend::DoHeartbeat(network_time, current_time);
switch ( Info().mode ) {
case MODE_MANUAL:
// yay, we do nothing :)

18
src/logging.bif
View file

@ -65,8 +65,8 @@ function Log::__flush%(id: Log::ID%): bool
module LogAscii;
const output_to_stdout: bool;
const include_header: bool;
const header_prefix: string;
const include_meta: bool;
const meta_prefix: string;
const separator: string;
const set_separator: string;
const empty_field: string;
@ -82,6 +82,20 @@ const dump_schema: bool;
const use_integer_for_time: bool;
const num_threads: count;
# Options for the ElasticSearch writer.
module LogElasticSearch;
const cluster_name: string;
const server_host: string;
const server_port: count;
const index_prefix: string;
const type_prefix: string;
const transfer_timeout: interval;
const max_batch_size: count;
const max_batch_interval: interval;
const max_byte_size: count;
# Options for the None writer.
module LogNone;

99
src/logging/Manager.cc
View file

@ -6,6 +6,7 @@
#include "../EventHandler.h"
#include "../NetVar.h"
#include "../Net.h"
#include "../Type.h"
#include "threading/Manager.h"
#include "threading/SerialTypes.h"
@ -17,6 +18,10 @@
#include "writers/Ascii.h"
#include "writers/None.h"
#ifdef USE_ELASTICSEARCH
#include "writers/ElasticSearch.h"
#endif
#ifdef USE_DATASERIES
#include "writers/DataSeries.h"
#endif
@ -35,6 +40,11 @@ struct WriterDefinition {
WriterDefinition log_writers[] = {
{ BifEnum::Log::WRITER_NONE, "None", 0, writer::None::Instantiate },
{ BifEnum::Log::WRITER_ASCII, "Ascii", 0, writer::Ascii::Instantiate },
#ifdef USE_ELASTICSEARCH
{ BifEnum::Log::WRITER_ELASTICSEARCH, "ElasticSearch", 0, writer::ElasticSearch::Instantiate },
#endif
#ifdef USE_DATASERIES
{ BifEnum::Log::WRITER_DATASERIES, "DataSeries", 0, writer::DataSeries::Instantiate },
#endif
@ -75,7 +85,7 @@ struct Manager::WriterInfo {
double interval;
Func* postprocessor;
WriterFrontend* writer;
WriterBackend::WriterInfo info;
WriterBackend::WriterInfo* info;
};
struct Manager::Stream {
@ -118,6 +128,7 @@ Manager::Stream::~Stream()
Unref(winfo->type);
delete winfo->writer;
delete winfo->info;
delete winfo;
}
@ -193,7 +204,6 @@ WriterBackend* Manager::CreateBackend(WriterFrontend* frontend, bro_int_t type)
assert(ld->factory);
frontend->ty_name = ld->name;
WriterBackend* backend = (*ld->factory)(frontend);
assert(backend);
@ -476,18 +486,17 @@ bool Manager::TraverseRecord(Stream* stream, Filter* filter, RecordType* rt,
return false;
}
threading::Field* field = new threading::Field();
field->name = new_path;
field->type = t->Tag();
field->optional = rt->FieldDecl(i)->FindAttr(ATTR_OPTIONAL);
TypeTag st = TYPE_VOID;
if ( field->type == TYPE_TABLE )
field->subtype = t->AsSetType()->Indices()->PureType()->Tag();
if ( t->Tag() == TYPE_TABLE )
st = t->AsSetType()->Indices()->PureType()->Tag();
else if ( field->type == TYPE_VECTOR )
field->subtype = t->AsVectorType()->YieldType()->Tag();
else if ( t->Tag() == TYPE_VECTOR )
st = t->AsVectorType()->YieldType()->Tag();
filter->fields[filter->num_fields - 1] = field;
bool optional = rt->FieldDecl(i)->FindAttr(ATTR_OPTIONAL);
filter->fields[filter->num_fields - 1] = new threading::Field(new_path.c_str(), 0, t->Tag(), st, optional);
}
return true;
@ -594,7 +603,7 @@ bool Manager::AddFilter(EnumVal* id, RecordVal* fval)
{
threading::Field* field = filter->fields[i];
DBG_LOG(DBG_LOGGING, " field %10s: %s",
field->name.c_str(), type_name(field->type));
field->name, type_name(field->type));
}
#endif
@ -769,8 +778,9 @@ bool Manager::Write(EnumVal* id, RecordVal* columns)
for ( int j = 0; j < filter->num_fields; ++j )
arg_fields[j] = new threading::Field(*filter->fields[j]);
WriterBackend::WriterInfo info;
info.path = path;
WriterBackend::WriterInfo* info = new WriterBackend::WriterInfo;
info->path = copy_string(path.c_str());
info->network_time = network_time;
HashKey* k;
IterCookie* c = filter->config->AsTable()->InitForIteration();
@ -781,7 +791,7 @@ bool Manager::Write(EnumVal* id, RecordVal* columns)
ListVal* index = filter->config->RecoverIndex(k);
string key = index->Index(0)->AsString()->CheckString();
string value = v->Value()->AsString()->CheckString();
info.config.insert(std::make_pair(key, value));
info->config.insert(std::make_pair(copy_string(key.c_str()), copy_string(value.c_str())));
Unref(index);
delete k;
}
@ -843,11 +853,16 @@ threading::Value* Manager::ValToLogVal(Val* val, BroType* ty)
val->Type()->AsEnumType()->Lookup(val->InternalInt());
if ( s )
lval->val.string_val = new string(s);
{
lval->val.string_val.data = copy_string(s);
lval->val.string_val.length = strlen(s);
}
else
{
val->Type()->Error("enum type does not contain value", val);
lval->val.string_val = new string();
lval->val.string_val.data = copy_string("");
lval->val.string_val.length = 0;
}
break;
}
@ -879,15 +894,20 @@ threading::Value* Manager::ValToLogVal(Val* val, BroType* ty)
case TYPE_STRING:
{
const BroString* s = val->AsString();
lval->val.string_val =
new string((const char*) s->Bytes(), s->Len());
char* buf = new char[s->Len()];
memcpy(buf, s->Bytes(), s->Len());
lval->val.string_val.data = buf;
lval->val.string_val.length = s->Len();
break;
}
case TYPE_FILE:
{
const BroFile* f = val->AsFile();
lval->val.string_val = new string(f->Name());
string s = f->Name();
lval->val.string_val.data = copy_string(s.c_str());
lval->val.string_val.length = s.size();
break;
}
@ -896,7 +916,9 @@ threading::Value* Manager::ValToLogVal(Val* val, BroType* ty)
ODesc d;
const Func* f = val->AsFunc();
f->Describe(&d);
lval->val.string_val = new string(d.Description());
const char* s = d.Description();
lval->val.string_val.data = copy_string(s);
lval->val.string_val.length = strlen(s);
break;
}
@ -976,7 +998,7 @@ threading::Value** Manager::RecordToFilterVals(Stream* stream, Filter* filter,
return vals;
}
WriterFrontend* Manager::CreateWriter(EnumVal* id, EnumVal* writer, const WriterBackend::WriterInfo& info,
WriterFrontend* Manager::CreateWriter(EnumVal* id, EnumVal* writer, WriterBackend::WriterInfo* info,
int num_fields, const threading::Field* const* fields, bool local, bool remote)
{
Stream* stream = FindStream(id);
@ -986,7 +1008,7 @@ WriterFrontend* Manager::CreateWriter(EnumVal* id, EnumVal* writer, const Writer
return 0;
Stream::WriterMap::iterator w =
stream->writers.find(Stream::WriterPathPair(writer->AsEnum(), info.path));
stream->writers.find(Stream::WriterPathPair(writer->AsEnum(), info->path));
if ( w != stream->writers.end() )
// If we already have a writer for this. That's fine, we just
@ -1012,7 +1034,7 @@ WriterFrontend* Manager::CreateWriter(EnumVal* id, EnumVal* writer, const Writer
{
Filter* f = *it;
if ( f->writer->AsEnum() == writer->AsEnum() &&
f->path == info.path )
f->path == info->path )
{
found_filter_match = true;
winfo->interval = f->interval;
@ -1029,7 +1051,7 @@ WriterFrontend* Manager::CreateWriter(EnumVal* id, EnumVal* writer, const Writer
}
stream->writers.insert(
Stream::WriterMap::value_type(Stream::WriterPathPair(writer->AsEnum(), info.path),
Stream::WriterMap::value_type(Stream::WriterPathPair(writer->AsEnum(), info->path),
winfo));
// Still need to set the WriterInfo's rotation parameters, which we
@ -1037,11 +1059,11 @@ WriterFrontend* Manager::CreateWriter(EnumVal* id, EnumVal* writer, const Writer
const char* base_time = log_rotate_base_time ?
log_rotate_base_time->AsString()->CheckString() : 0;
winfo->info.rotation_interval = winfo->interval;
winfo->info.rotation_base = parse_rotate_base_time(base_time);
winfo->info->rotation_interval = winfo->interval;
winfo->info->rotation_base = parse_rotate_base_time(base_time);
winfo->writer = new WriterFrontend(id, writer, local, remote);
winfo->writer->Init(winfo->info, num_fields, fields);
winfo->writer = new WriterFrontend(*winfo->info, id, writer, local, remote);
winfo->writer->Init(num_fields, fields);
InstallRotationTimer(winfo);
@ -1123,7 +1145,7 @@ void Manager::SendAllWritersTo(RemoteSerializer::PeerID peer)
EnumVal writer_val(i->first.first, BifType::Enum::Log::Writer);
remote_serializer->SendLogCreateWriter(peer, (*s)->id,
&writer_val,
i->second->info,
*i->second->info,
writer->NumFields(),
writer->Fields());
}
@ -1156,7 +1178,7 @@ bool Manager::Flush(EnumVal* id)
for ( Stream::WriterMap::iterator i = stream->writers.begin();
i != stream->writers.end(); i++ )
i->second->writer->Flush();
i->second->writer->Flush(network_time);
RemoveDisabledWriters(stream);
@ -1259,14 +1281,14 @@ void Manager::InstallRotationTimer(WriterInfo* winfo)
timer_mgr->Add(winfo->rotation_timer);
DBG_LOG(DBG_LOGGING, "Scheduled rotation timer for %s to %.6f",
winfo->writer->Name().c_str(), winfo->rotation_timer->Time());
winfo->writer->Name(), winfo->rotation_timer->Time());
}
}
void Manager::Rotate(WriterInfo* winfo)
{
DBG_LOG(DBG_LOGGING, "Rotating %s at %.6f",
winfo->writer->Name().c_str(), network_time);
winfo->writer->Name(), network_time);
// Build a temporary path for the writer to move the file to.
struct tm tm;
@ -1277,15 +1299,14 @@ void Manager::Rotate(WriterInfo* winfo)
localtime_r(&teatime, &tm);
strftime(buf, sizeof(buf), date_fmt, &tm);
string tmp = string(fmt("%s-%s", winfo->writer->Info().path.c_str(), buf));
// Trigger the rotation.
const char* tmp = fmt("%s-%s", winfo->writer->Info().path, buf);
winfo->writer->Rotate(tmp, winfo->open_time, network_time, terminating);
++rotations_pending;
}
bool Manager::FinishedRotation(WriterFrontend* writer, string new_name, string old_name,
bool Manager::FinishedRotation(WriterFrontend* writer, const char* new_name, const char* old_name,
double open, double close, bool terminating)
{
--rotations_pending;
@ -1295,7 +1316,7 @@ bool Manager::FinishedRotation(WriterFrontend* writer, string new_name, string o
return true;
DBG_LOG(DBG_LOGGING, "Finished rotating %s at %.6f, new name %s",
writer->Name().c_str(), network_time, new_name.c_str());
writer->Name(), network_time, new_name);
WriterInfo* winfo = FindWriter(writer);
if ( ! winfo )
@ -1304,8 +1325,8 @@ bool Manager::FinishedRotation(WriterFrontend* writer, string new_name, string o
// Create the RotationInfo record.
RecordVal* info = new RecordVal(BifType::Record::Log::RotationInfo);
info->Assign(0, winfo->type->Ref());
info->Assign(1, new StringVal(new_name.c_str()));
info->Assign(2, new StringVal(winfo->writer->Info().path.c_str()));
info->Assign(1, new StringVal(new_name));
info->Assign(2, new StringVal(winfo->writer->Info().path));
info->Assign(3, new Val(open, TYPE_TIME));
info->Assign(4, new Val(close, TYPE_TIME));
info->Assign(5, new Val(terminating, TYPE_BOOL));

6
src/logging/Manager.h
View file

@ -162,8 +162,8 @@ protected:
//// Function also used by the RemoteSerializer.
// Takes ownership of fields.
WriterFrontend* CreateWriter(EnumVal* id, EnumVal* writer, const WriterBackend::WriterInfo& info,
// Takes ownership of fields and info.
WriterFrontend* CreateWriter(EnumVal* id, EnumVal* writer, WriterBackend::WriterInfo* info,
int num_fields, const threading::Field* const* fields,
bool local, bool remote);
@ -175,7 +175,7 @@ protected:
void SendAllWritersTo(RemoteSerializer::PeerID peer);
// Signals that a file has been rotated.
bool FinishedRotation(WriterFrontend* writer, string new_name, string old_name,
bool FinishedRotation(WriterFrontend* writer, const char* new_name, const char* old_name,
double open, double close, bool terminating);
// Deletes the values as passed into Write().

54
src/logging/WriterBackend.cc
View file

@ -18,20 +18,26 @@ namespace logging {
class RotationFinishedMessage : public threading::OutputMessage<WriterFrontend>
{
public:
RotationFinishedMessage(WriterFrontend* writer, string new_name, string old_name,
RotationFinishedMessage(WriterFrontend* writer, const char* new_name, const char* old_name,
double open, double close, bool terminating)
: threading::OutputMessage<WriterFrontend>("RotationFinished", writer),
new_name(new_name), old_name(old_name), open(open),
new_name(copy_string(new_name)), old_name(copy_string(old_name)), open(open),
close(close), terminating(terminating) { }
virtual ~RotationFinishedMessage()
{
delete [] new_name;
delete [] old_name;
}
virtual bool Process()
{
return log_mgr->FinishedRotation(Object(), new_name, old_name, open, close, terminating);
}
private:
string new_name;
string old_name;
const char* new_name;
const char* old_name;
double open;
double close;
bool terminating;
@ -65,12 +71,16 @@ bool WriterBackend::WriterInfo::Read(SerializationFormat* fmt)
{
int size;
if ( ! (fmt->Read(&path, "path") &&
string tmp_path;
if ( ! (fmt->Read(&tmp_path, "path") &&
fmt->Read(&rotation_base, "rotation_base") &&
fmt->Read(&rotation_interval, "rotation_interval") &&
fmt->Read(&size, "config_size")) )
return false;
path = copy_string(tmp_path.c_str());
config.clear();
while ( size )
@ -81,7 +91,7 @@ bool WriterBackend::WriterInfo::Read(SerializationFormat* fmt)
if ( ! (fmt->Read(&value, "config-value") && fmt->Read(&value, "config-key")) )
return false;
config.insert(std::make_pair(value, key));
config.insert(std::make_pair(copy_string(value.c_str()), copy_string(key.c_str())));
}
return true;
@ -113,8 +123,7 @@ WriterBackend::WriterBackend(WriterFrontend* arg_frontend) : MsgThread()
fields = 0;
buffering = true;
frontend = arg_frontend;
info.path = "<path not yet set>";
info = new WriterInfo(frontend->Info());
SetName(frontend->Name());
}
@ -128,6 +137,8 @@ WriterBackend::~WriterBackend()
delete [] fields;
}
delete info;
}
void WriterBackend::DeleteVals(int num_writes, Value*** vals)
@ -144,7 +155,7 @@ void WriterBackend::DeleteVals(int num_writes, Value*** vals)
delete [] vals;
}
bool WriterBackend::FinishedRotation(string new_name, string old_name,
bool WriterBackend::FinishedRotation(const char* new_name, const char* old_name,
double open, double close, bool terminating)
{
SendOut(new RotationFinishedMessage(frontend, new_name, old_name, open, close, terminating));
@ -156,17 +167,12 @@ void WriterBackend::DisableFrontend()
SendOut(new DisableMessage(frontend));
}
bool WriterBackend::Init(const WriterInfo& arg_info, int arg_num_fields, const Field* const* arg_fields, const string& frontend_name)
bool WriterBackend::Init(int arg_num_fields, const Field* const* arg_fields)
{
info = arg_info;
num_fields = arg_num_fields;
fields = arg_fields;
string name = Fmt("%s/%s", info.path.c_str(), frontend_name.c_str());
SetName(name);
if ( ! DoInit(arg_info, arg_num_fields, arg_fields) )
if ( ! DoInit(*info, arg_num_fields, arg_fields) )
{
DisableFrontend();
return false;
@ -248,7 +254,7 @@ bool WriterBackend::SetBuf(bool enabled)
return true;
}
bool WriterBackend::Rotate(string rotated_path, double open,
bool WriterBackend::Rotate(const char* rotated_path, double open,
double close, bool terminating)
{
if ( ! DoRotate(rotated_path, open, close, terminating) )
@ -260,9 +266,9 @@ bool WriterBackend::Rotate(string rotated_path, double open,
return true;
}
bool WriterBackend::Flush()
bool WriterBackend::Flush(double network_time)
{
if ( ! DoFlush() )
if ( ! DoFlush(network_time) )
{
DisableFrontend();
return false;
@ -271,13 +277,15 @@ bool WriterBackend::Flush()
return true;
}
bool WriterBackend::DoHeartbeat(double network_time, double current_time)
bool WriterBackend::OnFinish(double network_time)
{
MsgThread::DoHeartbeat(network_time, current_time);
return DoFinish(network_time);
}
bool WriterBackend::OnHeartbeat(double network_time, double current_time)
{
SendOut(new FlushWriteBufferMessage(frontend));
return true;
return DoHeartbeat(network_time, current_time);
}
string WriterBackend::Render(const threading::Value::addr_t& addr) const

83
src/logging/WriterBackend.h
View file

@ -48,14 +48,17 @@ public:
*/
struct WriterInfo
{
typedef std::map<string, string> config_map;
// Structure takes ownership of these strings.
typedef std::map<const char*, const char*> config_map;
/**
* A string left to the interpretation of the writer
* implementation; it corresponds to the 'path' value configured
* on the script-level for the logging filter.
*
* Structure takes ownership of string.
*/
string path;
const char* path;
/**
* The rotation interval as configured for this writer.
@ -67,13 +70,47 @@ public:
*/
double rotation_base;
/**
* The network time when the writer is created.
*/
double network_time;
/**
* A map of key/value pairs corresponding to the relevant
* filter's "config" table.
*/
std::map<string, string> config;
config_map config;
WriterInfo()
{
path = 0;
}
WriterInfo(const WriterInfo& other)
{
path = other.path ? copy_string(other.path) : 0;
rotation_interval = other.rotation_interval;
rotation_base = other.rotation_base;
network_time = other.network_time;
for ( config_map::const_iterator i = other.config.begin(); i != other.config.end(); i++ )
config.insert(std::make_pair(copy_string(i->first), copy_string(i->second)));
}
~WriterInfo()
{
delete [] path;
for ( config_map::iterator i = config.begin(); i != config.end(); i++ )
{
delete [] i->first;
delete [] i->second;
}
}
private:
const WriterInfo& operator=(const WriterInfo& other); // Disable.
friend class ::RemoteSerializer;
// Note, these need to be adapted when changing the struct's
@ -85,7 +122,6 @@ public:
/**
* One-time initialization of the writer to define the logged fields.
*
* @param info Meta information for the writer.
* @param num_fields
*
* @param fields An array of size \a num_fields with the log fields.
@ -95,7 +131,7 @@ public:
*
* @return False if an error occured.
*/
bool Init(const WriterInfo& info, int num_fields, const threading::Field* const* fields, const string& frontend_name);
bool Init(int num_fields, const threading::Field* const* fields);
/**
* Writes one log entry.
@ -129,9 +165,11 @@ public:
* Flushes any currently buffered output, assuming the writer
* supports that. (If not, it will be ignored).
*
* @param network_time The network time when the flush was triggered.
*
* @return False if an error occured.
*/
bool Flush();
bool Flush(double network_time);
/**
* Triggers rotation, if the writer supports that. (If not, it will
@ -139,7 +177,7 @@ public:
*
* @return False if an error occured.
*/
bool Rotate(string rotated_path, double open, double close, bool terminating);
bool Rotate(const char* rotated_path, double open, double close, bool terminating);
/**
* Disables the frontend that has instantiated this backend. Once
@ -150,7 +188,7 @@ public:
/**
* Returns the additional writer information passed into the constructor.
*/
const WriterInfo& Info() const { return info; }
const WriterInfo& Info() const { return *info; }
/**
* Returns the number of log fields as passed into the constructor.
@ -186,7 +224,7 @@ public:
* @param terminating: True if the original rotation request occured
* due to the main Bro process shutting down.
*/
bool FinishedRotation(string new_name, string old_name,
bool FinishedRotation(const char* new_name, const char* old_name,
double open, double close, bool terminating);
/** Helper method to render an IP address as a string.
@ -213,6 +251,10 @@ public:
*/
string Render(double d) const;
// Overridden from MsgThread.
virtual bool OnHeartbeat(double network_time, double current_time);
virtual bool OnFinish(double network_time);
protected:
friend class FinishMessage;
@ -272,8 +314,10 @@ protected:
* will then be disabled and eventually deleted. When returning
* false, an implementation should also call Error() to indicate what
* happened.
*
* @param network_time The network time when the flush was triggered.
*/
virtual bool DoFlush() = 0;
virtual bool DoFlush(double network_time) = 0;
/**
* Writer-specific method implementing log rotation. Most directly
@ -309,25 +353,24 @@ protected:
* due the main Bro prcoess terminating (and not because we've
* reached a regularly scheduled time for rotation).
*/
virtual bool DoRotate(string rotated_path, double open, double close,
virtual bool DoRotate(const char* rotated_path, double open, double close,
bool terminating) = 0;
/**
* Writer-specific method called just before the threading system is
* going to shutdown.
* going to shutdown. It is assumed that once this messages returns,
* the thread can be safely terminated.
*
* This method can be overridden but one must call
* WriterBackend::DoFinish().
* @param network_time The network time when the finish is triggered.
*/
virtual bool DoFinish() { return MsgThread::DoFinish(); }
virtual bool DoFinish(double network_time) = 0;
/**
* Triggered by regular heartbeat messages from the main thread.
*
* This method can be overridden but one must call
* WriterBackend::DoHeartbeat().
* This method can be overridden. Default implementation does
* nothing.
*/
virtual bool DoHeartbeat(double network_time, double current_time);
virtual bool DoHeartbeat(double network_time, double current_time) = 0;
private:
/**
@ -339,7 +382,7 @@ private:
// this class, it's running in a different thread!
WriterFrontend* frontend;
WriterInfo info; // Meta information as passed to Init().
const WriterInfo* info; // Meta information.
int num_fields; // Number of log fields.
const threading::Field* const* fields; // Log fields.
bool buffering; // True if buffering is enabled.

82
src/logging/WriterFrontend.cc
View file

@ -16,35 +16,36 @@ namespace logging {
class InitMessage : public threading::InputMessage<WriterBackend>
{
public:
InitMessage(WriterBackend* backend, const WriterBackend::WriterInfo& info, const int num_fields, const Field* const* fields, const string& frontend_name)
InitMessage(WriterBackend* backend, const int num_fields, const Field* const* fields)
: threading::InputMessage<WriterBackend>("Init", backend),
info(info), num_fields(num_fields), fields(fields),
frontend_name(frontend_name) { }
num_fields(num_fields), fields(fields)
{}
virtual bool Process() { return Object()->Init(info, num_fields, fields, frontend_name); }
virtual bool Process() { return Object()->Init(num_fields, fields); }
private:
WriterBackend::WriterInfo info;
const int num_fields;
const Field * const* fields;
const string frontend_name;
};
class RotateMessage : public threading::InputMessage<WriterBackend>
{
public:
RotateMessage(WriterBackend* backend, WriterFrontend* frontend, const string rotated_path, const double open,
RotateMessage(WriterBackend* backend, WriterFrontend* frontend, const char* rotated_path, const double open,
const double close, const bool terminating)
: threading::InputMessage<WriterBackend>("Rotate", backend),
frontend(frontend),
rotated_path(rotated_path), open(open),
rotated_path(copy_string(rotated_path)), open(open),
close(close), terminating(terminating) { }
virtual ~RotateMessage() { delete [] rotated_path; }
virtual bool Process() { return Object()->Rotate(rotated_path, open, close, terminating); }
private:
WriterFrontend* frontend;
const string rotated_path;
const char* rotated_path;
const double open;
const double close;
const bool terminating;
@ -81,19 +82,13 @@ private:
class FlushMessage : public threading::InputMessage<WriterBackend>
{
public:
FlushMessage(WriterBackend* backend)
: threading::InputMessage<WriterBackend>("Flush", backend) {}
FlushMessage(WriterBackend* backend, double network_time)
: threading::InputMessage<WriterBackend>("Flush", backend),
network_time(network_time) {}
virtual bool Process() { return Object()->Flush(); }
};
class FinishMessage : public threading::InputMessage<WriterBackend>
{
public:
FinishMessage(WriterBackend* backend)
: threading::InputMessage<WriterBackend>("Finish", backend) {}
virtual bool Process() { return Object()->DoFinish(); }
virtual bool Process() { return Object()->Flush(network_time); }
private:
double network_time;
};
}
@ -102,7 +97,7 @@ public:
using namespace logging;
WriterFrontend::WriterFrontend(EnumVal* arg_stream, EnumVal* arg_writer, bool arg_local, bool arg_remote)
WriterFrontend::WriterFrontend(const WriterBackend::WriterInfo& arg_info, EnumVal* arg_stream, EnumVal* arg_writer, bool arg_local, bool arg_remote)
{
stream = arg_stream;
writer = arg_writer;
@ -115,7 +110,10 @@ WriterFrontend::WriterFrontend(EnumVal* arg_stream, EnumVal* arg_writer, bool ar
remote = arg_remote;
write_buffer = 0;
write_buffer_pos = 0;
ty_name = "<not set>";
info = new WriterBackend::WriterInfo(arg_info);
const char* w = arg_writer->Type()->AsEnumType()->Lookup(arg_stream->InternalInt());
name = copy_string(fmt("%s/%s", arg_info.path, w));
if ( local )
{
@ -133,26 +131,16 @@ WriterFrontend::~WriterFrontend()
{
Unref(stream);
Unref(writer);
}
string WriterFrontend::Name() const
{
if ( ! info.path.size() )
return ty_name;
return ty_name + "/" + info.path;
delete info;
}
void WriterFrontend::Stop()
{
FlushWriteBuffer();
SetDisable();
if ( backend )
backend->Stop();
}
void WriterFrontend::Init(const WriterBackend::WriterInfo& arg_info, int arg_num_fields, const Field* const * arg_fields)
void WriterFrontend::Init(int arg_num_fields, const Field* const * arg_fields)
{
if ( disabled )
return;
@ -160,19 +148,18 @@ void WriterFrontend::Init(const WriterBackend::WriterInfo& arg_info, int arg_num
if ( initialized )
reporter->InternalError("writer initialize twice");
info = arg_info;
num_fields = arg_num_fields;
fields = arg_fields;
initialized = true;
if ( backend )
backend->SendIn(new InitMessage(backend, arg_info, arg_num_fields, arg_fields, Name()));
backend->SendIn(new InitMessage(backend, arg_num_fields, arg_fields));
if ( remote )
remote_serializer->SendLogCreateWriter(stream,
writer,
arg_info,
*info,
arg_num_fields,
arg_fields);
@ -186,7 +173,7 @@ void WriterFrontend::Write(int num_fields, Value** vals)
if ( remote )
remote_serializer->SendLogWrite(stream,
writer,
info.path,
info->path,
num_fields,
vals);
@ -240,7 +227,7 @@ void WriterFrontend::SetBuf(bool enabled)
FlushWriteBuffer();
}
void WriterFrontend::Flush()
void WriterFrontend::Flush(double network_time)
{
if ( disabled )
return;
@ -248,10 +235,10 @@ void WriterFrontend::Flush()
FlushWriteBuffer();
if ( backend )
backend->SendIn(new FlushMessage(backend));
backend->SendIn(new FlushMessage(backend, network_time));
}
void WriterFrontend::Rotate(string rotated_path, double open, double close, bool terminating)
void WriterFrontend::Rotate(const char* rotated_path, double open, double close, bool terminating)
{
if ( disabled )
return;
@ -266,17 +253,6 @@ void WriterFrontend::Rotate(string rotated_path, double open, double close, bool
log_mgr->FinishedRotation(0, "", rotated_path, open, close, terminating);
}
void WriterFrontend::Finish()
{
if ( disabled )
return;
FlushWriteBuffer();
if ( backend )
backend->SendIn(new FinishMessage(backend));
}
void WriterFrontend::DeleteVals(Value** vals)
{
// Note this code is duplicated in Manager::DeleteVals().

28
src/logging/WriterFrontend.h
View file

@ -32,6 +32,10 @@ public:
* frontend will internally instantiate a WriterBackend of the
* corresponding type.
*
* info: The meta information struct for the writer.
*
* writer_name: A descriptive name for the writer's type.
*
* local: If true, the writer will instantiate a local backend.
*
* remote: If true, the writer will forward all data to remote
@ -39,7 +43,7 @@ public:
*
* Frontends must only be instantiated by the main thread.
*/
WriterFrontend(EnumVal* stream, EnumVal* writer, bool local, bool remote);
WriterFrontend(const WriterBackend::WriterInfo& info, EnumVal* stream, EnumVal* writer, bool local, bool remote);
/**
* Destructor.
@ -50,7 +54,7 @@ public:
/**
* Stops all output to this writer. Calling this methods disables all
* message forwarding to the backend and stops the backend thread.
* message forwarding to the backend.
*
* This method must only be called from the main thread.
*/
@ -68,7 +72,7 @@ public:
*
* This method must only be called from the main thread.
*/
void Init(const WriterBackend::WriterInfo& info, int num_fields, const threading::Field* const* fields);
void Init(int num_fields, const threading::Field* const* fields);
/**
* Write out a record.
@ -114,8 +118,10 @@ public:
* message back that will asynchronously call Disable().
*
* This method must only be called from the main thread.
*
* @param network_time The network time when the flush was triggered.
*/
void Flush();
void Flush(double network_time);
/**
* Triggers log rotation.
@ -128,7 +134,7 @@ public:
*
* This method must only be called from the main thread.
*/
void Rotate(string rotated_path, double open, double close, bool terminating);
void Rotate(const char* rotated_path, double open, double close, bool terminating);
/**
* Finalizes writing to this tream.
@ -138,8 +144,10 @@ public:
* sends a message back that will asynchronously call Disable().
*
* This method must only be called from the main thread.
*
* @param network_time The network time when the finish was triggered.
*/
void Finish();
void Finish(double network_time);
/**
* Explicitly triggers a transfer of all potentially buffered Write()
@ -171,7 +179,7 @@ public:
/**
* Returns the additional writer information as passed into the constructor.
*/
const WriterBackend::WriterInfo& Info() const { return info; }
const WriterBackend::WriterInfo& Info() const { return *info; }
/**
* Returns the number of log fields as passed into the constructor.
@ -184,7 +192,7 @@ public:
*
* This method is safe to call from any thread.
*/
string Name() const;
const char* Name() const { return name; }
/**
* Returns the log fields as passed into the constructor.
@ -206,8 +214,8 @@ protected:
bool local; // True if logging locally.
bool remote; // True if loggin remotely.
string ty_name; // Name of the backend type. Set by the manager.
WriterBackend::WriterInfo info; // The writer information.
const char* name; // Descriptive name of the
WriterBackend::WriterInfo* info; // The writer information.
int num_fields; // The number of log fields.
const threading::Field* const* fields; // The log fields.

174
src/logging/writers/Ascii.cc
View file

@ -2,6 +2,7 @@
#include <string>
#include <errno.h>
#include <fcntl.h>
#include "NetVar.h"
#include "threading/SerialTypes.h"
@ -15,10 +16,11 @@ using threading::Field;
Ascii::Ascii(WriterFrontend* frontend) : WriterBackend(frontend)
{
file = 0;
fd = 0;
ascii_done = false;
output_to_stdout = BifConst::LogAscii::output_to_stdout;
include_header = BifConst::LogAscii::include_header;
include_meta = BifConst::LogAscii::include_meta;
separator_len = BifConst::LogAscii::separator->Len();
separator = new char[separator_len];
@ -40,10 +42,10 @@ Ascii::Ascii(WriterFrontend* frontend) : WriterBackend(frontend)
memcpy(unset_field, BifConst::LogAscii::unset_field->Bytes(),
unset_field_len);
header_prefix_len = BifConst::LogAscii::header_prefix->Len();
header_prefix = new char[header_prefix_len];
memcpy(header_prefix, BifConst::LogAscii::header_prefix->Bytes(),
header_prefix_len);
meta_prefix_len = BifConst::LogAscii::meta_prefix->Len();
meta_prefix = new char[meta_prefix_len];
memcpy(meta_prefix, BifConst::LogAscii::meta_prefix->Bytes(),
meta_prefix_len);
desc.EnableEscaping();
desc.AddEscapeSequence(separator, separator_len);
@ -51,26 +53,46 @@ Ascii::Ascii(WriterFrontend* frontend) : WriterBackend(frontend)
Ascii::~Ascii()
{
if ( file )
fclose(file);
if ( ! ascii_done )
{
fprintf(stderr, "internal error: finish missing\n");
abort();
}
delete [] separator;
delete [] set_separator;
delete [] empty_field;
delete [] unset_field;
delete [] header_prefix;
delete [] meta_prefix;
}
bool Ascii::WriteHeaderField(const string& key, const string& val)
{
string str = string(header_prefix, header_prefix_len) +
string str = string(meta_prefix, meta_prefix_len) +
key + string(separator, separator_len) + val + "\n";
return (fwrite(str.c_str(), str.length(), 1, file) == 1);
return safe_write(fd, str.c_str(), str.length());
}
void Ascii::CloseFile(double t)
{
if ( ! fd )
return;
if ( include_meta )
{
string ts = t ? Timestamp(t) : string("<abnormal termination>");
WriteHeaderField("end", ts);
}
close(fd);
fd = 0;
}
bool Ascii::DoInit(const WriterInfo& info, int num_fields, const Field* const * fields)
{
assert(! fd);
string path = info.path;
if ( output_to_stdout )
@ -78,34 +100,39 @@ bool Ascii::DoInit(const WriterInfo& info, int num_fields, const Field* const *
fname = IsSpecial(path) ? path : path + "." + LogExt();
if ( ! (file = fopen(fname.c_str(), "w")) )
fd = open(fname.c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0777);
if ( fd < 0 )
{
Error(Fmt("cannot open %s: %s", fname.c_str(),
strerror(errno)));
Strerror(errno)));
fd = 0;
return false;
}
if ( include_header )
if ( include_meta )
{
string names;
string types;
string str = string(header_prefix, header_prefix_len)
string str = string(meta_prefix, meta_prefix_len)
+ "separator " // Always use space as separator here.
+ get_escaped_string(string(separator, separator_len), false)
+ "\n";
if( fwrite(str.c_str(), str.length(), 1, file) != 1 )
if ( ! safe_write(fd, str.c_str(), str.length()) )
goto write_error;
string ts = Timestamp(info.network_time);
if ( ! (WriteHeaderField("set_separator", get_escaped_string(
string(set_separator, set_separator_len), false)) &&
WriteHeaderField("empty_field", get_escaped_string(
string(empty_field, empty_field_len), false)) &&
WriteHeaderField("unset_field", get_escaped_string(
string(unset_field, unset_field_len), false)) &&
WriteHeaderField("path", get_escaped_string(path, false))) )
WriteHeaderField("path", get_escaped_string(path, false)) &&
WriteHeaderField("start", ts)) )
goto write_error;
for ( int i = 0; i < num_fields; ++i )
@ -116,8 +143,8 @@ bool Ascii::DoInit(const WriterInfo& info, int num_fields, const Field* const *
types += string(separator, separator_len);
}
names += fields[i]->name;
types += fields[i]->TypeName();
names += string(fields[i]->name);
types += fields[i]->TypeName().c_str();
}
if ( ! (WriteHeaderField("fields", names)
@ -128,21 +155,32 @@ bool Ascii::DoInit(const WriterInfo& info, int num_fields, const Field* const *
return true;
write_error:
Error(Fmt("error writing to %s: %s", fname.c_str(), strerror(errno)));
Error(Fmt("error writing to %s: %s", fname.c_str(), Strerror(errno)));
return false;
}
bool Ascii::DoFlush()
bool Ascii::DoFlush(double network_time)
{
fflush(file);
fsync(fd);
return true;
}
bool Ascii::DoFinish()
bool Ascii::DoFinish(double network_time)
{
return WriterBackend::DoFinish();
if ( ascii_done )
{
fprintf(stderr, "internal error: duplicate finish\n");
abort();
}
ascii_done = true;
CloseFile(network_time);
return true;
}
bool Ascii::DoWriteOne(ODesc* desc, Value* val, const Field* field)
{
if ( ! val->present )
@ -198,8 +236,8 @@ bool Ascii::DoWriteOne(ODesc* desc, Value* val, const Field* field)
case TYPE_FILE:
case TYPE_FUNC:
{
int size = val->val.string_val->size();
const char* data = val->val.string_val->data();
int size = val->val.string_val.length;
const char* data = val->val.string_val.data;
if ( ! size )
{
@ -280,8 +318,7 @@ bool Ascii::DoWriteOne(ODesc* desc, Value* val, const Field* field)
}
default:
Error(Fmt("unsupported field format %d for %s", val->type,
field->name.c_str()));
Error(Fmt("unsupported field format %d for %s", val->type, field->name));
return false;
}
@ -291,7 +328,7 @@ bool Ascii::DoWriteOne(ODesc* desc, Value* val, const Field* field)
bool Ascii::DoWrite(int num_fields, const Field* const * fields,
Value** vals)
{
if ( ! file )
if ( ! fd )
DoInit(Info(), NumFields(), Fields());
desc.Clear();
@ -307,31 +344,47 @@ bool Ascii::DoWrite(int num_fields, const Field* const * fields,
desc.AddRaw("\n", 1);
if ( fwrite(desc.Bytes(), desc.Len(), 1, file) != 1 )
const char* bytes = (const char*)desc.Bytes();
int len = desc.Len();
if ( strncmp(bytes, meta_prefix, meta_prefix_len) == 0 )
{
Error(Fmt("error writing to %s: %s", fname.c_str(), strerror(errno)));
// It would so escape the first character.
char buf[16];
snprintf(buf, sizeof(buf), "\\x%02x", bytes[0]);
if ( ! safe_write(fd, buf, strlen(buf)) )
goto write_error;
++bytes;
--len;
}
if ( ! safe_write(fd, bytes, len) )
goto write_error;
if ( IsBuf() )
fsync(fd);
return true;
write_error:
Error(Fmt("error writing to %s: %s", fname.c_str(), Strerror(errno)));
return false;
}
if ( IsBuf() )
fflush(file);
return true;
}
bool Ascii::DoRotate(string rotated_path, double open, double close, bool terminating)
bool Ascii::DoRotate(const char* rotated_path, double open, double close, bool terminating)
{
// Don't rotate special files or if there's not one currently open.
if ( ! file || IsSpecial(Info().path) )
if ( ! fd || IsSpecial(Info().path) )
return true;
fclose(file);
file = 0;
CloseFile(close);
string nname = rotated_path + "." + LogExt();
string nname = string(rotated_path) + "." + LogExt();
rename(fname.c_str(), nname.c_str());
if ( ! FinishedRotation(nname, fname, open, close, terminating) )
if ( ! FinishedRotation(nname.c_str(), fname.c_str(), open, close, terminating) )
{
Error(Fmt("error rotating %s to %s", fname.c_str(), nname.c_str()));
return false;
@ -346,9 +399,40 @@ bool Ascii::DoSetBuf(bool enabled)
return true;
}
bool Ascii::DoHeartbeat(double network_time, double current_time)
{
// Nothing to do.
return true;
}
string Ascii::LogExt()
{
const char* ext = getenv("BRO_LOG_SUFFIX");
if ( ! ext ) ext = "log";
if ( ! ext )
ext = "log";
return ext;
}
string Ascii::Timestamp(double t)
{
#if 1
return "2012-01-01-00-00-00";
#else
// Using the version below leads to occasional crashes at least on Mac OS.
// Not sure why, all the function should be thread-safe ...
time_t teatime = time_t(t);
struct tm tmbuf;
struct tm* tm = localtime_r(&teatime, &tmbuf);
char tmp[128];
const char* const date_fmt = "%Y-%m-%d-%H-%M-%S";
strftime(tmp, sizeof(tmp), date_fmt, tm);
return tmp;
#endif
}

18
src/logging/writers/Ascii.h
View file

@ -24,23 +24,27 @@ protected:
virtual bool DoWrite(int num_fields, const threading::Field* const* fields,
threading::Value** vals);
virtual bool DoSetBuf(bool enabled);
virtual bool DoRotate(string rotated_path, double open,
virtual bool DoRotate(const char* rotated_path, double open,
double close, bool terminating);
virtual bool DoFlush();
virtual bool DoFinish();
virtual bool DoFlush(double network_time);
virtual bool DoFinish(double network_time);
virtual bool DoHeartbeat(double network_time, double current_time);
private:
bool IsSpecial(string path) { return path.find("/dev/") == 0; }
bool DoWriteOne(ODesc* desc, threading::Value* val, const threading::Field* field);
bool WriteHeaderField(const string& key, const string& value);
void CloseFile(double t);
string Timestamp(double t);
FILE* file;
int fd;
string fname;
ODesc desc;
bool ascii_done;
// Options set from the script-level.
bool output_to_stdout;
bool include_header;
bool include_meta;
char* separator;
int separator_len;
@ -54,8 +58,8 @@ private:
char* unset_field;
int unset_field_len;
char* header_prefix;
int header_prefix_len;
char* meta_prefix;
int meta_prefix_len;
};
}

29
src/logging/writers/DataSeries.cc
View file

@ -78,10 +78,10 @@ std::string DataSeries::LogValueToString(threading::Value *val)
case TYPE_STRING:
case TYPE_FILE:
case TYPE_FUNC:
if ( ! val->val.string_val->size() )
if ( ! val->val.string_val.length )
return "";
return string(val->val.string_val->data(), val->val.string_val->size());
return string(val->val.string_val.data, val->val.string_val.length);
case TYPE_TABLE:
{
@ -302,7 +302,8 @@ bool DataSeries::DoInit(const WriterInfo& info, int num_fields, const threading:
if( ds_dump_schema )
{
FILE* pFile = fopen ( string(info.path + ".ds.xml").c_str() , "wb" );
string name = string(info.path) + ".ds.xml";
FILE* pFile = fopen(name.c_str(), "wb" );
if( pFile )
{
@ -311,7 +312,7 @@ bool DataSeries::DoInit(const WriterInfo& info, int num_fields, const threading:
}
else
Error(Fmt("cannot dump schema: %s", strerror(errno)));
Error(Fmt("cannot dump schema: %s", Strerror(errno)));
}
compress_type = Extent::compress_all;
@ -343,7 +344,7 @@ bool DataSeries::DoInit(const WriterInfo& info, int num_fields, const threading:
return OpenLog(info.path);
}
bool DataSeries::DoFlush()
bool DataSeries::DoFlush(double network_time)
{
// Flushing is handled by DataSeries automatically, so this function
// doesn't do anything.
@ -366,11 +367,10 @@ void DataSeries::CloseLog()
log_file = 0;
}
bool DataSeries::DoFinish()
bool DataSeries::DoFinish(double network_time)
{
CloseLog();
return WriterBackend::DoFinish();
return true;
}
bool DataSeries::DoWrite(int num_fields, const threading::Field* const * fields,
@ -395,17 +395,17 @@ bool DataSeries::DoWrite(int num_fields, const threading::Field* const * fields,
return true;
}
bool DataSeries::DoRotate(string rotated_path, double open, double close, bool terminating)
bool DataSeries::DoRotate(const char* rotated_path, double open, double close, bool terminating)
{
// Note that if DS files are rotated too often, the aggregate log
// size will be (much) larger.
CloseLog();
string dsname = Info().path + ".ds";
string nname = rotated_path + ".ds";
string dsname = string(Info().path) + ".ds";
string nname = string(rotated_path) + ".ds";
rename(dsname.c_str(), nname.c_str());
if ( ! FinishedRotation(nname, dsname, open, close, terminating) )
if ( ! FinishedRotation(nname.c_str(), dsname.c_str(), open, close, terminating) )
{
Error(Fmt("error rotating %s to %s", dsname.c_str(), nname.c_str()));
return false;
@ -420,4 +420,9 @@ bool DataSeries::DoSetBuf(bool enabled)
return true;
}
bool DataSeries::DoHeartbeat(double network_time, double current_time)
{
return true;
}
#endif /* USE_DATASERIES */

7
src/logging/writers/DataSeries.h
View file

@ -32,10 +32,11 @@ protected:
virtual bool DoWrite(int num_fields, const threading::Field* const* fields,
threading::Value** vals);
virtual bool DoSetBuf(bool enabled);
virtual bool DoRotate(string rotated_path, double open,
virtual bool DoRotate(const char* rotated_path, double open,
double close, bool terminating);
virtual bool DoFlush();
virtual bool DoFinish();
virtual bool DoFlush(double network_time);
virtual bool DoFinish(double network_time);
virtual bool DoHeartbeat(double network_time, double current_time);
private:
static const size_t ROW_MIN = 2048; // Minimum extent size.

415
src/logging/writers/ElasticSearch.cc Normal file
View file

@ -0,0 +1,415 @@
// See the file "COPYING" in the main distribution directory for copyright.
//
// This is experimental code that is not yet ready for production usage.
//
#include "config.h"
#ifdef USE_ELASTICSEARCH
#include <string>
#include <errno.h>
#include "util.h"
#include "BroString.h"
#include "NetVar.h"
#include "threading/SerialTypes.h"
#include <curl/curl.h>
#include <curl/easy.h>
#include "ElasticSearch.h"
using namespace logging;
using namespace writer;
using threading::Value;
using threading::Field;
ElasticSearch::ElasticSearch(WriterFrontend* frontend) : WriterBackend(frontend)
{
cluster_name_len = BifConst::LogElasticSearch::cluster_name->Len();
cluster_name = new char[cluster_name_len + 1];
memcpy(cluster_name, BifConst::LogElasticSearch::cluster_name->Bytes(), cluster_name_len);
cluster_name[cluster_name_len] = 0;
index_prefix = string((const char*) BifConst::LogElasticSearch::index_prefix->Bytes(), BifConst::LogElasticSearch::index_prefix->Len());
es_server = string(Fmt("http://%s:%d", BifConst::LogElasticSearch::server_host->Bytes(),
(int) BifConst::LogElasticSearch::server_port));
bulk_url = string(Fmt("%s/_bulk", es_server.c_str()));
http_headers = curl_slist_append(NULL, "Content-Type: text/json; charset=utf-8");
buffer.Clear();
counter = 0;
current_index = string();
prev_index = string();
last_send = current_time();
failing = false;
transfer_timeout = BifConst::LogElasticSearch::transfer_timeout * 1000;
curl_handle = HTTPSetup();
}
ElasticSearch::~ElasticSearch()
{
delete [] cluster_name;
}
bool ElasticSearch::DoInit(const WriterInfo& info, int num_fields, const threading::Field* const* fields)
{
return true;
}
bool ElasticSearch::DoFlush(double network_time)
{
BatchIndex();
return true;
}
bool ElasticSearch::DoFinish(double network_time)
{
BatchIndex();
curl_slist_free_all(http_headers);
curl_easy_cleanup(curl_handle);
return true;
}
bool ElasticSearch::BatchIndex()
{
curl_easy_reset(curl_handle);
curl_easy_setopt(curl_handle, CURLOPT_URL, bulk_url.c_str());
curl_easy_setopt(curl_handle, CURLOPT_POST, 1);
curl_easy_setopt(curl_handle, CURLOPT_POSTFIELDSIZE_LARGE, (curl_off_t)buffer.Len());
curl_easy_setopt(curl_handle, CURLOPT_POSTFIELDS, buffer.Bytes());
failing = ! HTTPSend(curl_handle);
// We are currently throwing the data out regardless of if the send failed. Fire and forget!
buffer.Clear();
counter = 0;
last_send = current_time();
return true;
}
bool ElasticSearch::AddValueToBuffer(ODesc* b, Value* val)
{
switch ( val->type )
{
// ES treats 0 as false and any other value as true so bool types go here.
case TYPE_BOOL:
case TYPE_INT:
b->Add(val->val.int_val);
break;
case TYPE_COUNT:
case TYPE_COUNTER:
{
// ElasticSearch doesn't seem to support unsigned 64bit ints.
if ( val->val.uint_val >= INT64_MAX )
{
Error(Fmt("count value too large: %" PRIu64, val->val.uint_val));
b->AddRaw("null", 4);
}
else
b->Add(val->val.uint_val);
break;
}
case TYPE_PORT:
b->Add(val->val.port_val.port);
break;
case TYPE_SUBNET:
b->AddRaw("\"", 1);
b->Add(Render(val->val.subnet_val));
b->AddRaw("\"", 1);
break;
case TYPE_ADDR:
b->AddRaw("\"", 1);
b->Add(Render(val->val.addr_val));
b->AddRaw("\"", 1);
break;
case TYPE_DOUBLE:
case TYPE_INTERVAL:
b->Add(val->val.double_val);
break;
case TYPE_TIME:
{
// ElasticSearch uses milliseconds for timestamps and json only
// supports signed ints (uints can be too large).
uint64_t ts = (uint64_t) (val->val.double_val * 1000);
if ( ts >= INT64_MAX )
{
Error(Fmt("time value too large: %" PRIu64, ts));
b->AddRaw("null", 4);
}
else
b->Add(ts);
break;
}
case TYPE_ENUM:
case TYPE_STRING:
case TYPE_FILE:
case TYPE_FUNC:
{
b->AddRaw("\"", 1);
for ( int i = 0; i < val->val.string_val.length; ++i )
{
char c = val->val.string_val.data[i];
// 2byte Unicode escape special characters.
if ( c < 32 || c > 126 || c == '\n' || c == '"' || c == '\'' || c == '\\' || c == '&' )
{
static const char hex_chars[] = "0123456789abcdef";
b->AddRaw("\\u00", 4);
b->AddRaw(&hex_chars[(c & 0xf0) >> 4], 1);
b->AddRaw(&hex_chars[c & 0x0f], 1);
}
else
b->AddRaw(&c, 1);
}
b->AddRaw("\"", 1);
break;
}
case TYPE_TABLE:
{
b->AddRaw("[", 1);
for ( int j = 0; j < val->val.set_val.size; j++ )
{
if ( j > 0 )
b->AddRaw(",", 1);
AddValueToBuffer(b, val->val.set_val.vals[j]);
}
b->AddRaw("]", 1);
break;
}
case TYPE_VECTOR:
{
b->AddRaw("[", 1);
for ( int j = 0; j < val->val.vector_val.size; j++ )
{
if ( j > 0 )
b->AddRaw(",", 1);
AddValueToBuffer(b, val->val.vector_val.vals[j]);
}
b->AddRaw("]", 1);
break;
}
default:
return false;
}
return true;
}
bool ElasticSearch::AddFieldToBuffer(ODesc *b, Value* val, const Field* field)
{
if ( ! val->present )
return false;
b->AddRaw("\"", 1);
b->Add(field->name);
b->AddRaw("\":", 2);
AddValueToBuffer(b, val);
return true;
}
bool ElasticSearch::DoWrite(int num_fields, const Field* const * fields,
Value** vals)
{
if ( current_index.empty() )
UpdateIndex(network_time, Info().rotation_interval, Info().rotation_base);
// Our action line looks like:
buffer.AddRaw("{\"index\":{\"_index\":\"", 20);
buffer.Add(current_index);
buffer.AddRaw("\",\"_type\":\"", 11);
buffer.Add(Info().path);
buffer.AddRaw("\"}}\n", 4);
buffer.AddRaw("{", 1);
for ( int i = 0; i < num_fields; i++ )
{
if ( i > 0 && buffer.Bytes()[buffer.Len()] != ',' && vals[i]->present )
buffer.AddRaw(",", 1);
AddFieldToBuffer(&buffer, vals[i], fields[i]);
}
buffer.AddRaw("}\n", 2);
counter++;
if ( counter >= BifConst::LogElasticSearch::max_batch_size ||
uint(buffer.Len()) >= BifConst::LogElasticSearch::max_byte_size )
BatchIndex();
return true;
}
bool ElasticSearch::UpdateIndex(double now, double rinterval, double rbase)
{
if ( rinterval == 0 )
{
// if logs aren't being rotated, don't use a rotation oriented index name.
current_index = index_prefix;
}
else
{
double nr = calc_next_rotate(now, rinterval, rbase);
double interval_beginning = now - (rinterval - nr);
struct tm tm;
char buf[128];
time_t teatime = (time_t)interval_beginning;
localtime_r(&teatime, &tm);
strftime(buf, sizeof(buf), "%Y%m%d%H%M", &tm);
prev_index = current_index;
current_index = index_prefix + "-" + buf;
// Send some metadata about this index.
buffer.AddRaw("{\"index\":{\"_index\":\"@", 21);
buffer.Add(index_prefix);
buffer.AddRaw("-meta\",\"_type\":\"index\",\"_id\":\"", 30);
buffer.Add(current_index);
buffer.AddRaw("-", 1);
buffer.Add(Info().rotation_base);
buffer.AddRaw("-", 1);
buffer.Add(Info().rotation_interval);
buffer.AddRaw("\"}}\n{\"name\":\"", 13);
buffer.Add(current_index);
buffer.AddRaw("\",\"start\":", 10);
buffer.Add(interval_beginning);
buffer.AddRaw(",\"end\":", 7);
buffer.Add(interval_beginning+rinterval);
buffer.AddRaw("}\n", 2);
}
//printf("%s - prev:%s current:%s\n", Info().path.c_str(), prev_index.c_str(), current_index.c_str());
return true;
}
bool ElasticSearch::DoRotate(const char* rotated_path, double open, double close, bool terminating)
{
// Update the currently used index to the new rotation interval.
UpdateIndex(close, Info().rotation_interval, Info().rotation_base);
// Only do this stuff if there was a previous index.
if ( ! prev_index.empty() )
{
// FIXME: I think this section is taking too long and causing the thread to die.
// Compress the previous index
//curl_easy_reset(curl_handle);
//curl_easy_setopt(curl_handle, CURLOPT_URL, Fmt("%s/%s/_settings", es_server.c_str(), prev_index.c_str()));
//curl_easy_setopt(curl_handle, CURLOPT_CUSTOMREQUEST, "PUT");
//curl_easy_setopt(curl_handle, CURLOPT_POSTFIELDS, "{\"index\":{\"store.compress.stored\":\"true\"}}");
//curl_easy_setopt(curl_handle, CURLOPT_POSTFIELDSIZE_LARGE, (curl_off_t) 42);
//HTTPSend(curl_handle);
// Optimize the previous index.
// TODO: make this into variables.
//curl_easy_reset(curl_handle);
//curl_easy_setopt(curl_handle, CURLOPT_URL, Fmt("%s/%s/_optimize?max_num_segments=1&wait_for_merge=false", es_server.c_str(), prev_index.c_str()));
//HTTPSend(curl_handle);
}
if ( ! FinishedRotation(current_index.c_str(), prev_index.c_str(), open, close, terminating) )
{
Error(Fmt("error rotating %s to %s", prev_index.c_str(), current_index.c_str()));
}
return true;
}
bool ElasticSearch::DoSetBuf(bool enabled)
{
// Nothing to do.
return true;
}
bool ElasticSearch::DoHeartbeat(double network_time, double current_time)
{
if ( last_send > 0 && buffer.Len() > 0 &&
current_time-last_send > BifConst::LogElasticSearch::max_batch_interval )
{
BatchIndex();
}
return true;
}
CURL* ElasticSearch::HTTPSetup()
{
CURL* handle = curl_easy_init();
if ( ! handle )
{
Error("cURL did not initialize correctly.");
return 0;
}
return handle;
}
bool ElasticSearch::HTTPReceive(void* ptr, int size, int nmemb, void* userdata)
{
//TODO: Do some verification on the result?
return true;
}
bool ElasticSearch::HTTPSend(CURL *handle)
{
curl_easy_setopt(handle, CURLOPT_HTTPHEADER, http_headers);
curl_easy_setopt(handle, CURLOPT_WRITEFUNCTION, &logging::writer::ElasticSearch::HTTPReceive); // This gets called with the result.
// HTTP 1.1 likes to use chunked encoded transfers, which aren't good for speed.
// The best (only?) way to disable that is to just use HTTP 1.0
curl_easy_setopt(handle, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_0);
//curl_easy_setopt(handle, CURLOPT_TIMEOUT_MS, transfer_timeout);
CURLcode return_code = curl_easy_perform(handle);
switch ( return_code )
{
case CURLE_COULDNT_CONNECT:
case CURLE_COULDNT_RESOLVE_HOST:
case CURLE_WRITE_ERROR:
case CURLE_RECV_ERROR:
{
if ( ! failing )
Error(Fmt("ElasticSearch server may not be accessible."));
}
case CURLE_OPERATION_TIMEDOUT:
{
if ( ! failing )
Warning(Fmt("HTTP operation with elasticsearch server timed out at %" PRIu64 " msecs.", transfer_timeout));
}
case CURLE_OK:
{
uint http_code = 0;
curl_easy_getinfo(curl_handle, CURLINFO_RESPONSE_CODE, &http_code);
if ( http_code == 200 )
// Hopefully everything goes through here.
return true;
else if ( ! failing )
Error(Fmt("Received a non-successful status code back from ElasticSearch server, check the elasticsearch server log."));
}
default:
{
}
}
// The "successful" return happens above
return false;
}
#endif

81
src/logging/writers/ElasticSearch.h Normal file
View file

@ -0,0 +1,81 @@
// See the file "COPYING" in the main distribution directory for copyright.
//
// Log writer for writing to an ElasticSearch database
//
// This is experimental code that is not yet ready for production usage.
//
#ifndef LOGGING_WRITER_ELASTICSEARCH_H
#define LOGGING_WRITER_ELASTICSEARCH_H
#include <curl/curl.h>
#include "../WriterBackend.h"
namespace logging { namespace writer {
class ElasticSearch : public WriterBackend {
public:
ElasticSearch(WriterFrontend* frontend);
~ElasticSearch();
static WriterBackend* Instantiate(WriterFrontend* frontend)
{ return new ElasticSearch(frontend); }
static string LogExt();
protected:
// Overidden from WriterBackend.
virtual bool DoInit(const WriterInfo& info, int num_fields,
const threading::Field* const* fields);
virtual bool DoWrite(int num_fields, const threading::Field* const* fields,
threading::Value** vals);
virtual bool DoSetBuf(bool enabled);
virtual bool DoRotate(const char* rotated_path, double open,
double close, bool terminating);
virtual bool DoFlush(double network_time);
virtual bool DoFinish(double network_time);
virtual bool DoHeartbeat(double network_time, double current_time);
private:
bool AddFieldToBuffer(ODesc *b, threading::Value* val, const threading::Field* field);
bool AddValueToBuffer(ODesc *b, threading::Value* val);
bool BatchIndex();
bool SendMappings();
bool UpdateIndex(double now, double rinterval, double rbase);
CURL* HTTPSetup();
bool HTTPReceive(void* ptr, int size, int nmemb, void* userdata);
bool HTTPSend(CURL *handle);
// Buffers, etc.
ODesc buffer;
uint64 counter;
double last_send;
string current_index;
string prev_index;
CURL* curl_handle;
// From scripts
char* cluster_name;
int cluster_name_len;
string es_server;
string bulk_url;
struct curl_slist *http_headers;
string path;
string index_prefix;
uint64 transfer_timeout;
bool failing;
uint64 batch_size;
};
}
}
#endif

21
src/logging/writers/None.cc
View file

@ -1,4 +1,6 @@
#include <algorithm>
#include "None.h"
#include "NetVar.h"
@ -15,8 +17,17 @@ bool None::DoInit(const WriterInfo& info, int num_fields,
std::cout << " rotation_interval=" << info.rotation_interval << std::endl;
std::cout << " rotation_base=" << info.rotation_base << std::endl;
for ( std::map<string,string>::const_iterator i = info.config.begin(); i != info.config.end(); i++ )
std::cout << " config[" << i->first << "] = " << i->second << std::endl;
// Output the config sorted by keys.
std::vector<std::pair<string, string> > keys;
for ( WriterInfo::config_map::const_iterator i = info.config.begin(); i != info.config.end(); i++ )
keys.push_back(std::make_pair(i->first, i->second));
std::sort(keys.begin(), keys.end());
for ( std::vector<std::pair<string,string> >::const_iterator i = keys.begin(); i != keys.end(); i++ )
std::cout << " config[" << (*i).first << "] = " << (*i).second << std::endl;
for ( int i = 0; i < num_fields; i++ )
{
@ -31,11 +42,11 @@ bool None::DoInit(const WriterInfo& info, int num_fields,
return true;
}
bool None::DoRotate(string rotated_path, double open, double close, bool terminating)
bool None::DoRotate(const char* rotated_path, double open, double close, bool terminating)
{
if ( ! FinishedRotation(string("/dev/null"), Info().path, open, close, terminating))
if ( ! FinishedRotation("/dev/null", Info().path, open, close, terminating))
{
Error(Fmt("error rotating %s", Info().path.c_str()));
Error(Fmt("error rotating %s", Info().path));
return false;
}

7
src/logging/writers/None.h
View file

@ -24,10 +24,11 @@ protected:
virtual bool DoWrite(int num_fields, const threading::Field* const* fields,
threading::Value** vals) { return true; }
virtual bool DoSetBuf(bool enabled) { return true; }
virtual bool DoRotate(string rotated_path, double open,
virtual bool DoRotate(const char* rotated_path, double open,
double close, bool terminating);
virtual bool DoFlush() { return true; }
virtual bool DoFinish() { WriterBackend::DoFinish(); return true; }
virtual bool DoFlush(double network_time) { return true; }
virtual bool DoFinish(double network_time) { return true; }
virtual bool DoHeartbeat(double network_time, double current_time) { return true; }
};
}

18
src/main.cc
View file

@ -12,6 +12,10 @@
#include <getopt.h>
#endif
#ifdef USE_CURL
#include <curl/curl.h>
#endif
#ifdef USE_IDMEF
extern "C" {
#include <libidmef/idmefxml.h>
@ -361,12 +365,6 @@ RETSIGTYPE sig_handler(int signo)
set_processing_status("TERMINATING", "sig_handler");
signal_val = signo;
if ( thread_mgr->Terminating() && (signal_val == SIGTERM || signal_val == SIGINT) )
// If the thread manager is already terminating (i.e.,
// waiting for child threads to exit), another term signal
// will send the threads a kill.
thread_mgr->KillThreads();
return RETSIGVAL;
}
@ -718,6 +716,10 @@ int main(int argc, char** argv)
SSL_library_init();
SSL_load_error_strings();
#ifdef USE_CURL
curl_global_init(CURL_GLOBAL_ALL);
#endif
// FIXME: On systems that don't provide /dev/urandom, OpenSSL doesn't
// seed the PRNG. We should do this here (but at least Linux, FreeBSD
// and Solaris provide /dev/urandom).
@ -1068,6 +1070,10 @@ int main(int argc, char** argv)
done_with_network();
net_delete();
#ifdef USE_CURL
curl_global_cleanup();
#endif
terminate_bro();
// Close files after net_delete(), because net_delete()

1
src/ssl-analyzer.pac
View file

@ -93,6 +93,7 @@ function version_ok(vers : uint16) : bool
case SSLv30:
case TLSv10:
case TLSv11:
case TLSv12:
return true;
default:

3
src/ssl-defs.pac
View file

@ -22,5 +22,6 @@ enum SSLVersions {
SSLv20 = 0x0002,
SSLv30 = 0x0300,
TLSv10 = 0x0301,
TLSv11 = 0x0302
TLSv11 = 0x0302,
TLSv12 = 0x0303
};

137
src/threading/BasicThread.cc
View file

@ -12,18 +12,23 @@
using namespace threading;
static const int STD_FMT_BUF_LEN = 2048;
uint64_t BasicThread::thread_counter = 0;
BasicThread::BasicThread()
{
started = false;
terminating = false;
killed = false;
pthread = 0;
buf_len = 2048;
buf_len = STD_FMT_BUF_LEN;
buf = (char*) malloc(buf_len);
name = Fmt("thread-%d", ++thread_counter);
strerr_buffer = 0;
name = copy_string(fmt("thread-%" PRIu64, ++thread_counter));
thread_mgr->AddThread(this);
}
@ -32,31 +37,42 @@ BasicThread::~BasicThread()
{
if ( buf )
free(buf);
delete [] name;
delete [] strerr_buffer;
}
void BasicThread::SetName(const string& arg_name)
void BasicThread::SetName(const char* arg_name)
{
// Slight race condition here with reader threads, but shouldn't matter.
name = arg_name;
delete [] name;
name = copy_string(arg_name);
}
void BasicThread::SetOSName(const string& name)
void BasicThread::SetOSName(const char* arg_name)
{
#ifdef HAVE_LINUX
prctl(PR_SET_NAME, name.c_str(), 0, 0, 0);
prctl(PR_SET_NAME, arg_name, 0, 0, 0);
#endif
#ifdef __APPLE__
pthread_setname_np(name.c_str());
pthread_setname_np(arg_name);
#endif
#ifdef FREEBSD
pthread_set_name_np(pthread_self(), name, name.c_str());
pthread_set_name_np(pthread_self(), arg_name, arg_name);
#endif
}
const char* BasicThread::Fmt(const char* format, ...)
{
if ( buf_len > 10 * STD_FMT_BUF_LEN )
{
// Shrink back to normal.
buf = (char*) safe_realloc(buf, STD_FMT_BUF_LEN);
buf_len = STD_FMT_BUF_LEN;
}
va_list al;
va_start(al, format);
int n = safe_vsnprintf(buf, buf_len, format, al);
@ -64,46 +80,56 @@ const char* BasicThread::Fmt(const char* format, ...)
if ( (unsigned int) n >= buf_len )
{ // Not enough room, grow the buffer.
int tmp_len = n + 32;
char* tmp = (char*) malloc(tmp_len);
buf_len = n + 32;
buf = (char*) safe_realloc(buf, buf_len);
// Is it portable to restart?
va_start(al, format);
n = safe_vsnprintf(tmp, tmp_len, format, al);
n = safe_vsnprintf(buf, buf_len, format, al);
va_end(al);
free(tmp);
}
return buf;
}
const char* BasicThread::Strerror(int err)
{
if ( ! strerr_buffer )
strerr_buffer = new char[256];
strerror_r(err, strerr_buffer, 256);
return strerr_buffer;
}
void BasicThread::Start()
{
if ( started )
return;
int err = pthread_mutex_init(&terminate, 0);
if ( err != 0 )
reporter->FatalError("Cannot create terminate mutex for thread %s: %s", name.c_str(), strerror(err));
// We use this like a binary semaphore and acquire it immediately.
err = pthread_mutex_lock(&terminate);
if ( err != 0 )
reporter->FatalError("Cannot aquire terminate mutex for thread %s: %s", name.c_str(), strerror(err));
err = pthread_create(&pthread, 0, BasicThread::launcher, this);
if ( err != 0 )
reporter->FatalError("Cannot create thread %s:%s", name.c_str(), strerror(err));
DBG_LOG(DBG_THREADING, "Started thread %s", name.c_str());
started = true;
int err = pthread_create(&pthread, 0, BasicThread::launcher, this);
if ( err != 0 )
reporter->FatalError("Cannot create thread %s: %s", name, Strerror(err));
DBG_LOG(DBG_THREADING, "Started thread %s", name);
OnStart();
}
void BasicThread::PrepareStop()
{
if ( ! started )
return;
if ( terminating )
return;
DBG_LOG(DBG_THREADING, "Preparing thread %s to terminate ...", name);
OnPrepareStop();
}
void BasicThread::Stop()
{
if ( ! started )
@ -112,17 +138,11 @@ void BasicThread::Stop()
if ( terminating )
return;
DBG_LOG(DBG_THREADING, "Signaling thread %s to terminate ...", name.c_str());
// Signal that it's ok for the thread to exit now by unlocking the
// mutex.
int err = pthread_mutex_unlock(&terminate);
if ( err != 0 )
reporter->FatalError("Failure flagging terminate condition for thread %s: %s", name.c_str(), strerror(err));
terminating = true;
DBG_LOG(DBG_THREADING, "Signaling thread %s to terminate ...", name);
OnStop();
terminating = true;
}
void BasicThread::Join()
@ -130,30 +150,34 @@ void BasicThread::Join()
if ( ! started )
return;
if ( ! terminating )
Stop();
assert(terminating);
DBG_LOG(DBG_THREADING, "Joining thread %s ...", name.c_str());
DBG_LOG(DBG_THREADING, "Joining thread %s ...", name);
if ( pthread_join(pthread, 0) != 0 )
reporter->FatalError("Failure joining thread %s", name.c_str());
if ( pthread && pthread_join(pthread, 0) != 0 )
reporter->FatalError("Failure joining thread %s", name);
pthread_mutex_destroy(&terminate);
DBG_LOG(DBG_THREADING, "Done with thread %s", name.c_str());
DBG_LOG(DBG_THREADING, "Joined with thread %s", name);
pthread = 0;
}
void BasicThread::Kill()
{
if ( ! (started && pthread) )
return;
// We don't *really* kill the thread here because that leads to race
// conditions. Instead we set a flag that parts of the the code need
// to check and get out of any loops they might be in.
terminating = true;
killed = true;
OnKill();
}
// I believe this is safe to call from a signal handler ... Not error
// checking so that killing doesn't bail out if we have already
// terminated.
pthread_kill(pthread, SIGKILL);
void BasicThread::Done()
{
DBG_LOG(DBG_THREADING, "Thread %s has finished", name);
terminating = true;
killed = true;
}
void* BasicThread::launcher(void *arg)
@ -173,15 +197,12 @@ void* BasicThread::launcher(void *arg)
sigdelset(&mask_set, SIGSEGV);
sigdelset(&mask_set, SIGBUS);
int res = pthread_sigmask(SIG_BLOCK, &mask_set, 0);
assert(res == 0); //
assert(res == 0);
// Run thread's main function.
thread->Run();
// Wait until somebody actually wants us to terminate.
if ( pthread_mutex_lock(&thread->terminate) != 0 )
reporter->FatalError("Failure acquiring terminate mutex at end of thread %s", thread->Name().c_str());
thread->Done();
return 0;
}

66
src/threading/BasicThread.h
View file

@ -5,7 +5,6 @@
#include <pthread.h>
#include <semaphore.h>
#include "Queue.h"
#include "util.h"
using namespace std;
@ -42,22 +41,25 @@ public:
*
* This method is safe to call from any thread.
*/
const string& Name() const { return name; }
const char* Name() const { return name; }
/**
* Sets a descriptive name for the thread. This should be a string
* that's useful in output presented to the user and uniquely
* identifies the thread.
*
* This method must be called only from the thread itself.
* This method must be called only from main thread at initialization
* time.
*/
void SetName(const string& name);
void SetName(const char* name);
/**
* Set the name shown by the OS as the thread's description. Not
* supported on all OSs.
*
* Must be called only from the child thread.
*/
void SetOSName(const string& name);
void SetOSName(const char* name);
/**
* Starts the thread. Calling this methods will spawn a new OS thread
@ -68,6 +70,18 @@ public:
*/
void Start();
/**
* Signals the thread to prepare for stopping. This must be called
* before Stop() and allows the thread to trigger shutting down
* without yet blocking for doing so.
*
* Calling this method has no effect if Start() hasn't been executed
* yet.
*
* Only Bro's main thread must call this method.
*/
void PrepareStop();
/**
* Signals the thread to stop. The method lets Terminating() now
* return true. It does however not force the thread to terminate.
@ -88,6 +102,13 @@ public:
*/
bool Terminating() const { return terminating; }
/**
* Returns true if Kill() has been called.
*
* This method is safe to call from any thread.
*/
bool Killed() const { return killed; }
/**
* A version of fmt() that the thread can safely use.
*
@ -96,6 +117,14 @@ public:
*/
const char* Fmt(const char* format, ...);
/**
* A version of strerror() that the thread can safely use. This is
* essentially a wrapper around strerror_r(). Note that it keeps a
* single buffer per thread internally so the result remains valid
* only until the next call.
*/
const char* Strerror(int err);
protected:
friend class Manager;
@ -116,12 +145,24 @@ protected:
virtual void OnStart() {}
/**
* Executed with Stop(). This is a hook into stopping the thread. It
* will be called from Bro's main thread after the thread has been
* signaled to stop.
* Executed with PrepareStop() (and before OnStop()). This is a hook
* into preparing the thread for stopping. It will be called from
* Bro's main thread before the thread has been signaled to stop.
*/
virtual void OnPrepareStop() {}
/**
* Executed with Stop() (and after OnPrepareStop()). This is a hook
* into stopping the thread. It will be called from Bro's main thread
* after the thread has been signaled to stop.
*/
virtual void OnStop() {}
/**
* Executed with Kill(). This is a hook into killing the thread.
*/
virtual void OnKill() {}
/**
* Destructor. This will be called by the manager.
*
@ -145,14 +186,18 @@ protected:
*/
void Kill();
/** Called by child thread's launcher when it's done processing. */
void Done();
private:
// pthread entry function.
static void* launcher(void *arg);
string name;
const char* name;
pthread_t pthread;
bool started; // Set to to true once running.
bool terminating; // Set to to true to signal termination.
bool killed; // Set to true once forcefully killed.
// Used as a semaphore to tell the pthread thread when it may
// terminate.
@ -162,6 +207,9 @@ private:
char* buf;
unsigned int buf_len;
// For implementating Strerror().
char* strerr_buffer;
static uint64_t thread_counter;
};

33
src/threading/Manager.cc
View file

@ -30,6 +30,10 @@ void Manager::Terminate()
do Process(); while ( did_process );
// Signal all to stop.
for ( all_thread_list::iterator i = all_threads.begin(); i != all_threads.end(); i++ )
(*i)->PrepareStop();
for ( all_thread_list::iterator i = all_threads.begin(); i != all_threads.end(); i++ )
(*i)->Stop();
@ -48,24 +52,16 @@ void Manager::Terminate()
terminating = false;
}
void Manager::KillThreads()
{
DBG_LOG(DBG_THREADING, "Killing threads ...");
for ( all_thread_list::iterator i = all_threads.begin(); i != all_threads.end(); i++ )
(*i)->Kill();
}
void Manager::AddThread(BasicThread* thread)
{
DBG_LOG(DBG_THREADING, "Adding thread %s ...", thread->Name().c_str());
DBG_LOG(DBG_THREADING, "Adding thread %s ...", thread->Name());
all_threads.push_back(thread);
idle = false;
}
void Manager::AddMsgThread(MsgThread* thread)
{
DBG_LOG(DBG_THREADING, "%s is a MsgThread ...", thread->Name().c_str());
DBG_LOG(DBG_THREADING, "%s is a MsgThread ...", thread->Name());
msg_threads.push_back(thread);
}
@ -91,6 +87,14 @@ double Manager::NextTimestamp(double* network_time)
return -1.0;
}
void Manager::KillThreads()
{
DBG_LOG(DBG_THREADING, "Killing threads ...");
for ( all_thread_list::iterator i = all_threads.begin(); i != all_threads.end(); i++ )
(*i)->Kill();
}
void Manager::Process()
{
bool do_beat = false;
@ -114,6 +118,12 @@ void Manager::Process()
{
Message* msg = t->RetrieveOut();
if ( ! msg )
{
assert(t->Killed());
break;
}
if ( msg->Process() )
{
if ( network_time )
@ -122,8 +132,7 @@ void Manager::Process()
else
{
string s = msg->Name() + " failed, terminating thread";
reporter->Error("%s", s.c_str());
reporter->Error("%s failed, terminating thread", msg->Name());
t->Stop();
}

16
src/threading/Manager.h
View file

@ -49,15 +49,6 @@ public:
*/
bool Terminating() const { return terminating; }
/**
* Immediately kills all child threads. It does however not yet join
* them, one still needs to call Terminate() for that.
*
* This method is safe to call from a signal handler, and can in fact
* be called while Terminate() is already in progress.
*/
void KillThreads();
typedef std::list<std::pair<string, MsgThread::Stats> > msg_stats_list;
/**
@ -115,6 +106,13 @@ protected:
*/
virtual double NextTimestamp(double* network_time);
/**
* Kills all thread immediately. Note that this may cause race conditions
* if a child thread currently holds a lock that might block somebody
* else.
*/
virtual void KillThreads();
/**
* Part of the IOSource interface.
*/

169
src/threading/MsgThread.cc
View file

@ -5,6 +5,7 @@
#include "Manager.h"
#include <unistd.h>
#include <signal.h>
using namespace threading;
@ -16,19 +17,17 @@ namespace threading {
class FinishMessage : public InputMessage<MsgThread>
{
public:
FinishMessage(MsgThread* thread) : InputMessage<MsgThread>("Finish", thread) { }
FinishMessage(MsgThread* thread, double network_time) : InputMessage<MsgThread>("Finish", thread),
network_time(network_time) { }
virtual bool Process() { return Object()->DoFinish(); }
};
virtual bool Process() {
bool result = Object()->OnFinish(network_time);
Object()->Finished();
return result;
}
// A dummy message that's only purpose is unblock the current read operation
// so that the child's Run() methods can check the termination status.
class UnblockMessage : public InputMessage<MsgThread>
{
public:
UnblockMessage(MsgThread* thread) : InputMessage<MsgThread>("Unblock", thread) { }
virtual bool Process() { return true; }
private:
double network_time;
};
/// Sends a heartbeat to the child thread.
@ -39,7 +38,10 @@ public:
: InputMessage<MsgThread>("Heartbeat", thread)
{ network_time = arg_network_time; current_time = arg_current_time; }
virtual bool Process() { return Object()->DoHeartbeat(network_time, current_time); }
virtual bool Process() {
Object()->HeartbeatInChild();
return Object()->OnHeartbeat(network_time, current_time);
}
private:
double network_time;
@ -55,14 +57,16 @@ public:
INTERNAL_WARNING, INTERNAL_ERROR
};
ReporterMessage(Type arg_type, MsgThread* thread, const string& arg_msg)
ReporterMessage(Type arg_type, MsgThread* thread, const char* arg_msg)
: OutputMessage<MsgThread>("ReporterMessage", thread)
{ type = arg_type; msg = arg_msg; }
{ type = arg_type; msg = copy_string(arg_msg); }
~ReporterMessage() { delete [] msg; }
virtual bool Process();
private:
string msg;
const char* msg;
Type type;
};
@ -71,18 +75,19 @@ private:
class DebugMessage : public OutputMessage<MsgThread>
{
public:
DebugMessage(DebugStream arg_stream, MsgThread* thread, const string& arg_msg)
DebugMessage(DebugStream arg_stream, MsgThread* thread, const char* arg_msg)
: OutputMessage<MsgThread>("DebugMessage", thread)
{ stream = arg_stream; msg = arg_msg; }
{ stream = arg_stream; msg = copy_string(arg_msg); }
virtual ~DebugMessage() { delete [] msg; }
virtual bool Process()
{
string s = Object()->Name() + ": " + msg;
debug_logger.Log(stream, "%s", s.c_str());
debug_logger.Log(stream, "%s: %s", Object()->Name(), msg);
return true;
}
private:
string msg;
const char* msg;
DebugStream stream;
};
#endif
@ -93,41 +98,39 @@ private:
Message::~Message()
{
delete [] name;
}
bool ReporterMessage::Process()
{
string s = Object()->Name() + ": " + msg;
const char* cmsg = s.c_str();
switch ( type ) {
case INFO:
reporter->Info("%s", cmsg);
reporter->Info("%s: %s", Object()->Name(), msg);
break;
case WARNING:
reporter->Warning("%s", cmsg);
reporter->Warning("%s: %s", Object()->Name(), msg);
break;
case ERROR:
reporter->Error("%s", cmsg);
reporter->Error("%s: %s", Object()->Name(), msg);
break;
case FATAL_ERROR:
reporter->FatalError("%s", cmsg);
reporter->FatalError("%s: %s", Object()->Name(), msg);
break;
case FATAL_ERROR_WITH_CORE:
reporter->FatalErrorWithCore("%s", cmsg);
reporter->FatalErrorWithCore("%s: %s", Object()->Name(), msg);
break;
case INTERNAL_WARNING:
reporter->InternalWarning("%s", cmsg);
reporter->InternalWarning("%s: %s", Object()->Name(), msg);
break;
case INTERNAL_ERROR :
reporter->InternalError("%s", cmsg);
reporter->InternalError("%s: %s", Object()->Name(), msg);
break;
default:
@ -137,32 +140,74 @@ bool ReporterMessage::Process()
return true;
}
MsgThread::MsgThread() : BasicThread()
MsgThread::MsgThread() : BasicThread(), queue_in(this, 0), queue_out(0, this)
{
cnt_sent_in = cnt_sent_out = 0;
finished = false;
thread_mgr->AddMsgThread(this);
}
// Set by Bro's main signal handler.
extern int signal_val;
void MsgThread::OnPrepareStop()
{
if ( finished || Killed() )
return;
// Signal thread to terminate and wait until it has acknowledged.
SendIn(new FinishMessage(this, network_time), true);
}
void MsgThread::OnStop()
{
// Signal thread to terminate and wait until it has acknowledged.
SendIn(new FinishMessage(this), true);
int signal_count = 0;
int old_signal_val = signal_val;
signal_val = 0;
int cnt = 0;
while ( ! finished )
uint64_t last_size = 0;
uint64_t cur_size = 0;
while ( ! (finished || Killed() ) )
{
if ( ++cnt > 1000 ) // Insurance against broken threads ...
// Terminate if we get another kill signal.
if ( signal_val == SIGTERM || signal_val == SIGINT )
{
reporter->Warning("thread %s didn't finish in time", Name().c_str());
break;
++signal_count;
if ( signal_count == 1 )
{
// Abort all threads here so that we won't hang next
// on another one.
fprintf(stderr, "received signal while waiting for thread %s, aborting all ...\n", Name());
thread_mgr->KillThreads();
}
else
{
// More than one signal. Abort processing
// right away. on another one.
fprintf(stderr, "received another signal while waiting for thread %s, aborting processing\n", Name());
exit(1);
}
signal_val = 0;
}
queue_in.WakeUp();
usleep(1000);
}
// One more message to make sure the current queue read operation unblocks.
SendIn(new UnblockMessage(this), true);
signal_val = old_signal_val;
}
void MsgThread::OnKill()
{
// Send a message to unblock the reader if its currently waiting for
// input. This is just an optimization to make it terminate more
// quickly, even without the message it will eventually time out.
queue_in.WakeUp();
}
void MsgThread::Heartbeat()
@ -170,25 +215,20 @@ void MsgThread::Heartbeat()
SendIn(new HeartbeatMessage(this, network_time, current_time()));
}
bool MsgThread::DoHeartbeat(double network_time, double current_time)
void MsgThread::HeartbeatInChild()
{
string n = Name();
n = Fmt("bro: %s (%" PRIu64 "/%" PRIu64 ")", n.c_str(),
string n = Fmt("bro: %s (%" PRIu64 "/%" PRIu64 ")", Name(),
cnt_sent_in - queue_in.Size(),
cnt_sent_out - queue_out.Size());
SetOSName(n.c_str());
return true;
}
bool MsgThread::DoFinish()
void MsgThread::Finished()
{
// This is thread-safe "enough", we're the only one ever writing
// there.
finished = true;
return true;
}
void MsgThread::Info(const char* msg)
@ -245,7 +285,7 @@ void MsgThread::SendIn(BasicInputMessage* msg, bool force)
return;
}
DBG_LOG(DBG_THREADING, "Sending '%s' to %s ...", msg->Name().c_str(), Name().c_str());
DBG_LOG(DBG_THREADING, "Sending '%s' to %s ...", msg->Name(), Name());
queue_in.Put(msg);
++cnt_sent_in;
@ -268,9 +308,10 @@ void MsgThread::SendOut(BasicOutputMessage* msg, bool force)
BasicOutputMessage* MsgThread::RetrieveOut()
{
BasicOutputMessage* msg = queue_out.Get();
assert(msg);
if ( ! msg )
return 0;
DBG_LOG(DBG_THREADING, "Retrieved '%s' from %s", msg->Name().c_str(), Name().c_str());
DBG_LOG(DBG_THREADING, "Retrieved '%s' from %s", msg->Name(), Name());
return msg;
}
@ -278,10 +319,12 @@ BasicOutputMessage* MsgThread::RetrieveOut()
BasicInputMessage* MsgThread::RetrieveIn()
{
BasicInputMessage* msg = queue_in.Get();
assert(msg);
if ( ! msg )
return 0;
#ifdef DEBUG
string s = Fmt("Retrieved '%s' in %s", msg->Name().c_str(), Name().c_str());
string s = Fmt("Retrieved '%s' in %s", msg->Name(), Name());
Debug(DBG_THREADING, s.c_str());
#endif
@ -290,27 +333,33 @@ BasicInputMessage* MsgThread::RetrieveIn()
void MsgThread::Run()
{
while ( true )
while ( ! (finished || Killed() ) )
{
// When requested to terminate, we only do so when
// all input has been processed.
if ( Terminating() && ! queue_in.Ready() )
break;
BasicInputMessage* msg = RetrieveIn();
if ( ! msg )
continue;
bool result = msg->Process();
if ( ! result )
{
string s = msg->Name() + " failed, terminating thread (MsgThread)";
string s = Fmt("%s failed, terminating thread (MsgThread)", Name());
Error(s.c_str());
Stop();
break;
}
delete msg;
}
// In case we haven't send the finish method yet, do it now. Reading
// global network_time here should be fine, it isn't changing
// anymore.
if ( ! finished )
{
OnFinish(network_time);
Finished();
}
}
void MsgThread::GetStats(Stats* stats)

54
src/threading/MsgThread.h
View file

@ -189,39 +189,42 @@ protected:
*
* This is method is called regularly by the threading::Manager.
*
* Can be overriden in derived classed to hook into the heart beat,
* but must call the parent implementation. Note that this method is
* always called by the main thread and must not access data of the
* child thread directly. See DoHeartbeat() if you want to do
* something on the child-side.
* Can be overriden in derived classed to hook into the heart beat
* sending, but must call the parent implementation. Note that this
* method is always called by the main thread and must not access
* data of the child thread directly. Implement OnHeartbeat() if you
* want to do something on the child-side.
*/
virtual void Heartbeat();
/**
* Overriden from BasicThread.
*
/** Internal heartbeat processing. Called from child.
*/
virtual void Run();
virtual void OnStop();
void HeartbeatInChild();
/**
* Regulatly triggered for execution in the child thread.
*
* When overriding, one must call the parent class' implementation.
*
* network_time: The network_time when the heartbeat was trigger by
* the main thread.
*
* current_time: Wall clock when the heartbeat was trigger by the
* main thread.
*/
virtual bool DoHeartbeat(double network_time, double current_time);
virtual bool OnHeartbeat(double network_time, double current_time) = 0;
/** Triggered for execution in the child thread just before shutting threads down.
* The child thread should finish its operations and then *must*
* call this class' implementation.
* The child thread should finish its operations.
*/
virtual bool DoFinish();
virtual bool OnFinish(double network_time) = 0;
/**
* Overriden from BasicThread.
*
*/
virtual void Run();
virtual void OnStop();
virtual void OnPrepareStop();
virtual void OnKill();
private:
/**
@ -280,6 +283,10 @@ private:
*/
bool MightHaveOut() { return queue_out.MaybeReady(); }
/** Flags that the child process has finished processing. Called from child.
*/
void Finished();
Queue<BasicInputMessage *> queue_in;
Queue<BasicOutputMessage *> queue_out;
@ -305,7 +312,7 @@ public:
* what's passed into the constructor and used mainly for debugging
* purposes.
*/
const string& Name() const { return name; }
const char* Name() const { return name; }
/**
* Callback that must be overriden for processing a message.
@ -319,10 +326,11 @@ protected:
* @param arg_name A descriptive name for the type of message. Used
* mainly for debugging purposes.
*/
Message(const string& arg_name) { name = arg_name; }
Message(const char* arg_name)
{ name = copy_string(arg_name); }
private:
string name;
const char* name;
};
/**
@ -337,7 +345,7 @@ protected:
* @param name A descriptive name for the type of message. Used
* mainly for debugging purposes.
*/
BasicInputMessage(const string& name) : Message(name) {}
BasicInputMessage(const char* name) : Message(name) {}
};
/**
@ -352,7 +360,7 @@ protected:
* @param name A descriptive name for the type of message. Used
* mainly for debugging purposes.
*/
BasicOutputMessage(const string& name) : Message(name) {}
BasicOutputMessage(const char* name) : Message(name) {}
};
/**
@ -377,7 +385,7 @@ protected:
*
* @param arg_object: An object to store with the message.
*/
InputMessage(const string& name, O* arg_object) : BasicInputMessage(name)
InputMessage(const char* name, O* arg_object) : BasicInputMessage(name)
{ object = arg_object; }
private:
@ -406,7 +414,7 @@ protected:
*
* @param arg_object An object to store with the message.
*/
OutputMessage(const string& name, O* arg_object) : BasicOutputMessage(name)
OutputMessage(const char* name, O* arg_object) : BasicOutputMessage(name)
{ object = arg_object; }
private:

53
src/threading/Queue.h
View file

@ -1,4 +1,3 @@
#ifndef THREADING_QUEUE_H
#define THREADING_QUEUE_H
@ -6,8 +5,10 @@
#include <queue>
#include <deque>
#include <stdint.h>
#include <sys/time.h>
#include "Reporter.h"
#include "BasicThread.h"
#undef Queue // Defined elsewhere unfortunately.
@ -30,8 +31,12 @@ class Queue
public:
/**
* Constructor.
*
* reader, writer: The corresponding threads. This is for checking
* whether they have terminated so that we can abort I/O opeations.
* Can be left null for the main thread.
*/
Queue();
Queue(BasicThread* arg_reader, BasicThread* arg_writer);
/**
* Destructor.
@ -39,7 +44,9 @@ public:
~Queue();
/**
* Retrieves one elment.
* Retrieves one element. This may block for a little while of no
* input is available and eventually return with a null element if
* nothing shows up.
*/
T Get();
@ -60,6 +67,11 @@ public:
*/
bool MaybeReady() { return ( ( read_ptr - write_ptr) != 0 ); }
/** Wake up the reader if it's currently blocked for input. This is
primarily to give it a chance to check termination quickly.
**/
void WakeUp();
/**
* Returns the number of queued items not yet retrieved.
*/
@ -91,6 +103,9 @@ private:
int read_ptr; // Where the next operation will read from
int write_ptr; // Where the next operation will write to
BasicThread* reader;
BasicThread* writer;
// Statistics.
uint64_t num_reads;
uint64_t num_writes;
@ -109,18 +124,20 @@ inline static void safe_unlock(pthread_mutex_t* mutex)
}
template<typename T>
inline Queue<T>::Queue()
inline Queue<T>::Queue(BasicThread* arg_reader, BasicThread* arg_writer)
{
read_ptr = 0;
write_ptr = 0;
num_reads = num_writes = 0;
reader = arg_reader;
writer = arg_writer;
for( int i = 0; i < NUM_QUEUES; ++i )
{
if ( pthread_cond_init(&has_data[i], NULL) != 0 )
if ( pthread_cond_init(&has_data[i], 0) != 0 )
reporter->FatalError("cannot init queue condition variable");
if ( pthread_mutex_init(&mutex[i], NULL) != 0 )
if ( pthread_mutex_init(&mutex[i], 0) != 0 )
reporter->FatalError("cannot init queue mutex");
}
}
@ -138,12 +155,23 @@ inline Queue<T>::~Queue()
template<typename T>
inline T Queue<T>::Get()
{
if ( (reader && reader->Killed()) || (writer && writer->Killed()) )
return 0;
safe_lock(&mutex[read_ptr]);
int old_read_ptr = read_ptr;
if ( messages[read_ptr].empty() )
pthread_cond_wait(&has_data[read_ptr], &mutex[read_ptr]);
{
struct timespec ts;
ts.tv_sec = time(0) + 5;
ts.tv_nsec = 0;
pthread_cond_timedwait(&has_data[read_ptr], &mutex[read_ptr], &ts);
safe_unlock(&mutex[read_ptr]);
return 0;
}
T data = messages[read_ptr].front();
messages[read_ptr].pop();
@ -222,6 +250,17 @@ inline void Queue<T>::GetStats(Stats* stats)
safe_unlock(&mutex[i]);
}
template<typename T>
inline void Queue<T>::WakeUp()
{
for ( int i = 0; i < NUM_QUEUES; i++ )
{
safe_lock(&mutex[i]);
pthread_cond_signal(&has_data[i]);
safe_unlock(&mutex[i]);
}
}
}

48
src/threading/SerialTypes.cc
View file

@ -11,23 +11,54 @@ bool Field::Read(SerializationFormat* fmt)
{
int t;
int st;
string tmp_name;
bool have_2nd;
bool success = (fmt->Read(&name, "name")
&& fmt->Read(&secondary_name, "secondary_name")
if ( ! fmt->Read(&have_2nd, "have_2nd") )
return false;
if ( have_2nd )
{
string tmp_secondary_name;
if ( ! fmt->Read(&tmp_secondary_name, "secondary_name") )
return false;
secondary_name = copy_string(tmp_secondary_name.c_str());
}
else
secondary_name = 0;
bool success = (fmt->Read(&tmp_name, "name")
&& fmt->Read(&t, "type")
&& fmt->Read(&st, "subtype")
&& fmt->Read(&optional, "optional"));
if ( ! success )
return false;
name = copy_string(tmp_name.c_str());
type = (TypeTag) t;
subtype = (TypeTag) st;
return success;
return true;
}
bool Field::Write(SerializationFormat* fmt) const
{
assert(name);
if ( secondary_name )
{
if ( ! (fmt->Write(true, "have_2nd")
&& fmt->Write(secondary_name, "secondary_name")) )
return false;
}
else
if ( ! fmt->Write(false, "have_2nd") )
return false;
return (fmt->Write(name, "name")
&& fmt->Write(secondary_name, "secondary_name")
&& fmt->Write((int)type, "type")
&& fmt->Write((int)subtype, "subtype"),
fmt->Write(optional, "optional"));
@ -51,7 +82,7 @@ Value::~Value()
{
if ( (type == TYPE_ENUM || type == TYPE_STRING || type == TYPE_FILE || type == TYPE_FUNC)
&& present )
delete val.string_val;
delete [] val.string_val.data;
if ( type == TYPE_TABLE && present )
{
@ -224,10 +255,7 @@ bool Value::Read(SerializationFormat* fmt)
case TYPE_STRING:
case TYPE_FILE:
case TYPE_FUNC:
{
val.string_val = new string;
return fmt->Read(val.string_val, "string");
}
return fmt->Read(&val.string_val.data, &val.string_val.length, "string");
case TYPE_TABLE:
{
@ -339,7 +367,7 @@ bool Value::Write(SerializationFormat* fmt) const
case TYPE_STRING:
case TYPE_FILE:
case TYPE_FUNC:
return fmt->Write(*val.string_val, "string");
return fmt->Write(val.string_val.data, val.string_val.length, "string");
case TYPE_TABLE:
{

32
src/threading/SerialTypes.h
View file

@ -12,6 +12,7 @@
using namespace std;
class SerializationFormat;
class RemoteSerializer;
namespace threading {
@ -19,10 +20,10 @@ namespace threading {
* Definition of a log file, i.e., one column of a log stream.
*/
struct Field {
string name; //! Name of the field.
const char* name; //! Name of the field.
//! Needed by input framework. Port fields have two names (one for the
//! port, one for the type), and this specifies the secondary name.
string secondary_name;
const char* secondary_name;
TypeTag type; //! Type of the field.
TypeTag subtype; //! Inner type for sets.
bool optional; //! True if field is optional.
@ -30,13 +31,24 @@ struct Field {
/**
* Constructor.
*/
Field() { subtype = TYPE_VOID; optional = false; }
Field(const char* name, const char* secondary_name, TypeTag type, TypeTag subtype, bool optional)
: name(name ? copy_string(name) : 0),
secondary_name(secondary_name ? copy_string(secondary_name) : 0),
type(type), subtype(subtype), optional(optional) { }
/**
* Copy constructor.
*/
Field(const Field& other)
: name(other.name), type(other.type), subtype(other.subtype), optional(other.optional) { }
: name(other.name ? copy_string(other.name) : 0),
secondary_name(other.secondary_name ? copy_string(other.secondary_name) : 0),
type(other.type), subtype(other.subtype), optional(other.optional) { }
~Field()
{
delete [] name;
delete [] secondary_name;
}
/**
* Unserializes a field.
@ -63,6 +75,12 @@ struct Field {
* thread-safe.
*/
string TypeName() const;
private:
friend class ::RemoteSerializer;
// Force usage of constructor above.
Field() {};
};
/**
@ -102,7 +120,11 @@ struct Value {
vec_t vector_val;
addr_t addr_val;
subnet_t subnet_val;
string* string_val;
struct {
char* data;
int length;
} string_val;
} val;
/**

1
src/types.bif
View file

@ -163,6 +163,7 @@ enum Writer %{
WRITER_NONE,
WRITER_ASCII,
WRITER_DATASERIES,
WRITER_ELASTICSEARCH,
%}
enum ID %{

24
src/util.cc
View file

@ -1290,6 +1290,30 @@ uint64 calculate_unique_id(size_t pool)
return HashKey::HashBytes(&(uid_pool[pool].key), sizeof(uid_pool[pool].key));
}
bool safe_write(int fd, const char* data, int len)
{
while ( len > 0 )
{
int n = write(fd, data, len);
if ( n < 0 )
{
if ( errno == EINTR )
continue;
fprintf(stderr, "safe_write error: %d\n", errno);
abort();
return false;
}
data += n;
len -= n;
}
return true;
}
void out_of_memory(const char* where)
{
reporter->FatalError("out of memory in %s.\n", where);

7
src/util.h
View file

@ -13,6 +13,7 @@
// Expose C99 functionality from inttypes.h, which would otherwise not be
// available in C++.
#define __STDC_FORMAT_MACROS
#define __STDC_LIMIT_MACROS
#include <inttypes.h>
#if __STDC__
@ -289,6 +290,11 @@ inline size_t pad_size(size_t size)
#define padded_sizeof(x) (pad_size(sizeof(x)))
// Like write() but handles interrupted system calls by restarting. Returns
// true if the write was successful, otherwise sets errno. This function is
// thread-safe as long as no two threads write to the same descriptor.
extern bool safe_write(int fd, const char* data, int len);
extern void out_of_memory(const char* where);
inline void* safe_realloc(void* ptr, size_t size)
@ -338,4 +344,5 @@ inline int safe_vsnprintf(char* str, size_t size, const char* format, va_list al
// handed out by malloc.
extern void get_memory_usage(unsigned int* total,
unsigned int* malloced);
#endif

2
testing/btest/Baseline/bifs.to_double/out
View file

@ -3,4 +3,4 @@
-60.0
3600.0
86400.0
1337982322.762159
1342748947.655087

20
testing/btest/Baseline/core.checksums/bad.out
View file

@ -3,81 +3,101 @@
#empty_field (empty)
#unset_field -
#path weird
#start 2012-03-26-18-03-01
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p name addl notice peer
#types time string addr port addr port string string bool string
1332784981.078396 - - - - - bad_IP_checksum - F bro
#end 2012-03-26-18-03-01
#separator \x09
#set_separator ,
#empty_field (empty)
#unset_field -
#path weird
#start 2012-03-26-18-01-25
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p name addl notice peer
#types time string addr port addr port string string bool string
1332784885.686428 UWkUyAuUGXf 127.0.0.1 30000 127.0.0.1 80 bad_TCP_checksum - F bro
#end 2012-03-26-18-01-25
#separator \x09
#set_separator ,
#empty_field (empty)
#unset_field -
#path weird
#start 2012-03-26-18-02-13
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p name addl notice peer
#types time string addr port addr port string string bool string
1332784933.501023 UWkUyAuUGXf 127.0.0.1 30000 127.0.0.1 13000 bad_UDP_checksum - F bro
#end 2012-03-26-18-02-13
#separator \x09
#set_separator ,
#empty_field (empty)
#unset_field -
#path weird
#start 2012-04-10-16-29-23
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p name addl notice peer
#types time string addr port addr port string string bool string
1334075363.536871 UWkUyAuUGXf 192.168.1.100 8 192.168.1.101 0 bad_ICMP_checksum - F bro
#end 2012-04-10-16-29-23
#separator \x09
#set_separator ,
#empty_field (empty)
#unset_field -
#path weird
#start 2012-03-26-18-06-50
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p name addl notice peer
#types time string addr port addr port string string bool string
1332785210.013051 - - - - - routing0_hdr - F bro
1332785210.013051 UWkUyAuUGXf 2001:4f8:4:7:2e0:81ff:fe52:ffff 30000 2001:78:1:32::2 80 bad_TCP_checksum - F bro
#end 2012-03-26-18-06-50
#separator \x09
#set_separator ,
#empty_field (empty)
#unset_field -
#path weird
#start 2012-03-26-17-23-00
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p name addl notice peer
#types time string addr port addr port string string bool string
1332782580.798420 - - - - - routing0_hdr - F bro
1332782580.798420 UWkUyAuUGXf 2001:4f8:4:7:2e0:81ff:fe52:ffff 30000 2001:78:1:32::2 13000 bad_UDP_checksum - F bro
#end 2012-03-26-17-23-00
#separator \x09
#set_separator ,
#empty_field (empty)
#unset_field -
#path weird
#start 2012-04-10-16-25-11
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p name addl notice peer
#types time string addr port addr port string string bool string
1334075111.800086 - - - - - routing0_hdr - F bro
1334075111.800086 UWkUyAuUGXf 2001:4f8:4:7:2e0:81ff:fe52:ffff 128 2001:78:1:32::1 129 bad_ICMP_checksum - F bro
#end 2012-04-10-16-25-11
#separator \x09
#set_separator ,
#empty_field (empty)
#unset_field -
#path weird
#start 2012-03-26-18-07-30
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p name addl notice peer
#types time string addr port addr port string string bool string
1332785250.469132 UWkUyAuUGXf 2001:4f8:4:7:2e0:81ff:fe52:ffff 30000 2001:4f8:4:7:2e0:81ff:fe52:9a6b 80 bad_TCP_checksum - F bro
#end 2012-03-26-18-07-30
#separator \x09
#set_separator ,
#empty_field (empty)
#unset_field -
#path weird
#start 2012-03-26-17-02-22
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p name addl notice peer
#types time string addr port addr port string string bool string
1332781342.923813 UWkUyAuUGXf 2001:4f8:4:7:2e0:81ff:fe52:ffff 30000 2001:4f8:4:7:2e0:81ff:fe52:9a6b 13000 bad_UDP_checksum - F bro
#end 2012-03-26-17-02-22
#separator \x09
#set_separator ,
#empty_field (empty)
#unset_field -
#path weird
#start 2012-04-10-16-22-19
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p name addl notice peer
#types time string addr port addr port string string bool string
1334074939.467194 UWkUyAuUGXf 2001:4f8:4:7:2e0:81ff:fe52:ffff 128 2001:4f8:4:7:2e0:81ff:fe52:9a6b 129 bad_ICMP_checksum - F bro
#end 2012-04-10-16-22-19

14
testing/btest/Baseline/core.checksums/good.out
View file

@ -3,54 +3,68 @@
#empty_field (empty)
#unset_field -
#path weird
#start 2012-04-10-16-22-19
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p name addl notice peer
#types time string addr port addr port string string bool string
1334074939.467194 UWkUyAuUGXf 2001:4f8:4:7:2e0:81ff:fe52:ffff 128 2001:4f8:4:7:2e0:81ff:fe52:9a6b 129 bad_ICMP_checksum - F bro
#end 2012-04-10-16-22-19
#separator \x09
#set_separator ,
#empty_field (empty)
#unset_field -
#path weird
#start 2012-03-26-18-05-25
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p name addl notice peer
#types time string addr port addr port string string bool string
1332785125.596793 - - - - - routing0_hdr - F bro
#end 2012-03-26-18-05-25
#separator \x09
#set_separator ,
#empty_field (empty)
#unset_field -
#path weird
#start 2012-03-26-17-21-48
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p name addl notice peer
#types time string addr port addr port string string bool string
1332782508.592037 - - - - - routing0_hdr - F bro
#end 2012-03-26-17-21-48
#separator \x09
#set_separator ,
#empty_field (empty)
#unset_field -
#path weird
#start 2012-04-10-16-23-47
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p name addl notice peer
#types time string addr port addr port string string bool string
1334075027.053380 - - - - - routing0_hdr - F bro
#end 2012-04-10-16-23-47
#separator \x09
#set_separator ,
#empty_field (empty)
#unset_field -
#path weird
#start 2012-04-10-16-23-47
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p name addl notice peer
#types time string addr port addr port string string bool string
1334075027.053380 - - - - - routing0_hdr - F bro
#end 2012-04-10-16-23-47
#separator \x09
#set_separator ,
#empty_field (empty)
#unset_field -
#path weird
#start 2012-04-10-16-23-47
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p name addl notice peer
#types time string addr port addr port string string bool string
1334075027.053380 - - - - - routing0_hdr - F bro
#end 2012-04-10-16-23-47
#separator \x09
#set_separator ,
#empty_field (empty)
#unset_field -
#path weird
#start 2012-04-10-16-23-47
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p name addl notice peer
#types time string addr port addr port string string bool string
1334075027.053380 - - - - - routing0_hdr - F bro
#end 2012-04-10-16-23-47

2
testing/btest/Baseline/core.disable-mobile-ipv6/weird.log
View file

@ -3,6 +3,8 @@
#empty_field (empty)
#unset_field -
#path weird
#start 2012-04-05-21-56-51
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p name addl notice peer
#types time string addr port addr port string string bool string
1333663011.602839 - - - - - unknown_protocol_135 - F bro
#end 2012-04-05-21-56-51

20
testing/btest/Baseline/core.expr-exception/reporter.log
View file

@ -3,14 +3,16 @@
#empty_field (empty)
#unset_field -
#path reporter
#start 2011-03-18-19-06-08
#fields ts level message location
#types time enum string string
1300475168.783842 Reporter::ERROR field value missing [c$ftp] /home/jsiwek/bro/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 10
1300475168.915940 Reporter::ERROR field value missing [c$ftp] /home/jsiwek/bro/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 10
1300475168.916118 Reporter::ERROR field value missing [c$ftp] /home/jsiwek/bro/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 10
1300475168.918295 Reporter::ERROR field value missing [c$ftp] /home/jsiwek/bro/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 10
1300475168.952193 Reporter::ERROR field value missing [c$ftp] /home/jsiwek/bro/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 10
1300475168.952228 Reporter::ERROR field value missing [c$ftp] /home/jsiwek/bro/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 10
1300475168.954761 Reporter::ERROR field value missing [c$ftp] /home/jsiwek/bro/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 10
1300475168.962628 Reporter::ERROR field value missing [c$ftp] /home/jsiwek/bro/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 10
1300475169.780331 Reporter::ERROR field value missing [c$ftp] /home/jsiwek/bro/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 10
1300475168.783842 Reporter::ERROR field value missing [c$ftp] /da/home/robin/bro/master/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 10
1300475168.915940 Reporter::ERROR field value missing [c$ftp] /da/home/robin/bro/master/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 10
1300475168.916118 Reporter::ERROR field value missing [c$ftp] /da/home/robin/bro/master/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 10
1300475168.918295 Reporter::ERROR field value missing [c$ftp] /da/home/robin/bro/master/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 10
1300475168.952193 Reporter::ERROR field value missing [c$ftp] /da/home/robin/bro/master/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 10
1300475168.952228 Reporter::ERROR field value missing [c$ftp] /da/home/robin/bro/master/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 10
1300475168.954761 Reporter::ERROR field value missing [c$ftp] /da/home/robin/bro/master/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 10
1300475168.962628 Reporter::ERROR field value missing [c$ftp] /da/home/robin/bro/master/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 10
1300475169.780331 Reporter::ERROR field value missing [c$ftp] /da/home/robin/bro/master/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 10
#end 2011-03-18-19-06-13

2
testing/btest/Baseline/core.ipv6-frag/dns.log
View file

@ -3,7 +3,9 @@
#empty_field (empty)
#unset_field -
#path dns
#start 2012-03-07-01-37-58
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto trans_id query qclass qclass_name qtype qtype_name rcode rcode_name AA TC RD RA Z answers TTLs
#types time string addr port addr port enum count string count string count string count string bool bool bool bool count vector[string] vector[interval]
1331084278.438444 UWkUyAuUGXf 2001:470:1f11:81f:d138:5f55:6d4:1fe2 51850 2607:f740:b::f93 53 udp 3903 txtpadding_323.n1.netalyzr.icsi.berkeley.edu 1 C_INTERNET 16 TXT 0 NOERROR T F T F 0 This TXT record should be ignored 1.000000
1331084293.592245 arKYeMETxOg 2001:470:1f11:81f:d138:5f55:6d4:1fe2 51851 2607:f740:b::f93 53 udp 40849 txtpadding_3230.n1.netalyzr.icsi.berkeley.edu 1 C_INTERNET 16 TXT 0 NOERROR T F T F 0 This TXT record should be ignored 1.000000
#end 2012-03-07-01-38-18

2
testing/btest/Baseline/core.print-bpf-filters/conn.log
View file

@ -3,6 +3,8 @@
#empty_field (empty)
#unset_field -
#path conn
#start 2005-10-07-23-23-57
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto service duration orig_bytes resp_bytes conn_state local_orig missed_bytes history orig_pkts orig_ip_bytes resp_pkts resp_ip_bytes tunnel_parents
#types time string addr port addr port enum string interval count count string bool count string count count count count table[string]
1128727435.450898 UWkUyAuUGXf 141.42.64.125 56730 125.190.109.199 80 tcp http 1.733303 98 9417 SF - 0 ShADdFaf 12 730 10 9945 (empty)
#end 2005-10-07-23-23-57

16
testing/btest/Baseline/core.print-bpf-filters/output
View file

@ -3,30 +3,38 @@
#empty_field (empty)
#unset_field -
#path packet_filter
#start 1970-01-01-00-00-00
#fields ts node filter init success
#types time string string bool bool
1340229717.179155 - ip or not ip T T
1342748953.570646 - ip or not ip T T
#end <abnormal termination>
#separator \x09
#set_separator ,
#empty_field (empty)
#unset_field -
#path packet_filter
#start 1970-01-01-00-00-00
#fields ts node filter init success
#types time string string bool bool
1340229717.462355 - (((((((((((((((((((((((((port 53) or (tcp port 989)) or (tcp port 443)) or (port 6669)) or (udp and port 5353)) or (port 6668)) or (tcp port 1080)) or (udp and port 5355)) or (tcp port 22)) or (tcp port 995)) or (port 21)) or (tcp port 25 or tcp port 587)) or (port 6667)) or (tcp port 614)) or (tcp port 990)) or (udp port 137)) or (tcp port 993)) or (tcp port 5223)) or (port 514)) or (tcp port 585)) or (tcp port 992)) or (tcp port 563)) or (tcp port 994)) or (tcp port 636)) or (tcp and port (80 or 81 or 631 or 1080 or 3138 or 8000 or 8080 or 8888))) or (port 6666) T T
1342748953.898675 - (((((((((((((((((((((((((port 53) or (tcp port 989)) or (tcp port 443)) or (port 6669)) or (udp and port 5353)) or (port 6668)) or (tcp port 1080)) or (udp and port 5355)) or (tcp port 22)) or (tcp port 995)) or (port 21)) or (tcp port 25 or tcp port 587)) or (port 6667)) or (tcp port 614)) or (tcp port 990)) or (udp port 137)) or (tcp port 993)) or (tcp port 5223)) or (port 514)) or (tcp port 585)) or (tcp port 992)) or (tcp port 563)) or (tcp port 994)) or (tcp port 636)) or (tcp and port (80 or 81 or 631 or 1080 or 3138 or 8000 or 8080 or 8888))) or (port 6666) T T
#end <abnormal termination>
#separator \x09
#set_separator ,
#empty_field (empty)
#unset_field -
#path packet_filter
#start 1970-01-01-00-00-00
#fields ts node filter init success
#types time string string bool bool
1340229717.733007 - port 42 T T
1342748954.278211 - port 42 T T
#end <abnormal termination>
#separator \x09
#set_separator ,
#empty_field (empty)
#unset_field -
#path packet_filter
#start 1970-01-01-00-00-00
#fields ts node filter init success
#types time string string bool bool
1340229718.001009 - port 56730 T T
1342748954.883780 - port 56730 T T
#end 2005-10-07-23-23-57

2
testing/btest/Baseline/core.reporter-error-in-handler/output
View file

@ -1,2 +1,2 @@
error in /Users/robin/bro/master/testing/btest/.tmp/core.reporter-error-in-handler/reporter-error-in-handler.bro, line 22: no such index (a[2])
error in /da/home/robin/bro/master/testing/btest/.tmp/core.reporter-error-in-handler/reporter-error-in-handler.bro, line 22: no such index (a[2])
1st error printed on script level

2
testing/btest/Baseline/core.reporter-fmt-strings/output
View file

@ -1 +1 @@
error in /Users/robin/bro/master/testing/btest/.tmp/core.reporter-fmt-strings/reporter-fmt-strings.bro, line 9: not an event (dont_interpret_this(%s))
error in /da/home/robin/bro/master/testing/btest/.tmp/core.reporter-fmt-strings/reporter-fmt-strings.bro, line 9: not an event (dont_interpret_this(%s))

2
testing/btest/Baseline/core.reporter-parse-error/output
View file

@ -1 +1 @@
error in /Users/robin/bro/master/testing/btest/.tmp/core.reporter-parse-error/reporter-parse-error.bro, line 7: unknown identifier TESTFAILURE, at or near "TESTFAILURE"
error in /da/home/robin/bro/master/testing/btest/.tmp/core.reporter-parse-error/reporter-parse-error.bro, line 7: unknown identifier TESTFAILURE, at or near "TESTFAILURE"

2
testing/btest/Baseline/core.reporter-runtime-error/output
View file

@ -1 +1 @@
error in /Users/robin/bro/master/testing/btest/.tmp/core.reporter-runtime-error/reporter-runtime-error.bro, line 12: no such index (a[1])
error in /da/home/robin/bro/master/testing/btest/.tmp/core.reporter-runtime-error/reporter-runtime-error.bro, line 12: no such index (a[1])

6
testing/btest/Baseline/core.reporter-type-mismatch/output
View file

@ -1,3 +1,3 @@
error in string and /Users/robin/bro/master/testing/btest/.tmp/core.reporter-type-mismatch/reporter-type-mismatch.bro, line 11: arithmetic mixed with non-arithmetic (string and 42)
error in /Users/robin/bro/master/testing/btest/.tmp/core.reporter-type-mismatch/reporter-type-mismatch.bro, line 11 and string: type mismatch (42 and string)
error in /Users/robin/bro/master/testing/btest/.tmp/core.reporter-type-mismatch/reporter-type-mismatch.bro, line 11: argument type mismatch in event invocation (foo(42))
error in string and /da/home/robin/bro/master/testing/btest/.tmp/core.reporter-type-mismatch/reporter-type-mismatch.bro, line 11: arithmetic mixed with non-arithmetic (string and 42)
error in /da/home/robin/bro/master/testing/btest/.tmp/core.reporter-type-mismatch/reporter-type-mismatch.bro, line 11 and string: type mismatch (42 and string)
error in /da/home/robin/bro/master/testing/btest/.tmp/core.reporter-type-mismatch/reporter-type-mismatch.bro, line 11: argument type mismatch in event invocation (foo(42))

12
testing/btest/Baseline/core.reporter/logger-test.log
View file

@ -1,6 +1,6 @@
reporter_info|init test-info|/Users/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 8|0.000000
reporter_warning|init test-warning|/Users/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 9|0.000000
reporter_error|init test-error|/Users/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 10|0.000000
reporter_info|done test-info|/Users/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 15|0.000000
reporter_warning|done test-warning|/Users/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 16|0.000000
reporter_error|done test-error|/Users/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 17|0.000000
reporter_info|init test-info|/da/home/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 8|0.000000
reporter_warning|init test-warning|/da/home/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 9|0.000000
reporter_error|init test-error|/da/home/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 10|0.000000
reporter_info|done test-info|/da/home/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 15|0.000000
reporter_warning|done test-warning|/da/home/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 16|0.000000
reporter_error|done test-error|/da/home/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 17|0.000000

6
testing/btest/Baseline/core.reporter/output
View file

@ -1,3 +1,3 @@
/Users/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 52: pre test-info
warning in /Users/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 53: pre test-warning
error in /Users/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 54: pre test-error
/da/home/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 52: pre test-info
warning in /da/home/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 53: pre test-warning
error in /da/home/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 54: pre test-error

8
testing/btest/Baseline/core.truncation/output
View file

@ -3,30 +3,38 @@
#empty_field (empty)
#unset_field -
#path weird
#start 2012-04-11-16-01-35
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p name addl notice peer
#types time string addr port addr port string string bool string
1334160095.895421 - - - - - truncated_IP - F bro
#end 2012-04-11-16-01-35
#separator \x09
#set_separator ,
#empty_field (empty)
#unset_field -
#path weird
#start 2012-04-11-14-57-21
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p name addl notice peer
#types time string addr port addr port string string bool string
1334156241.519125 - - - - - truncated_IP - F bro
#end 2012-04-11-14-57-21
#separator \x09
#set_separator ,
#empty_field (empty)
#unset_field -
#path weird
#start 2012-04-10-21-50-48
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p name addl notice peer
#types time string addr port addr port string string bool string
1334094648.590126 - - - - - truncated_IP - F bro
#end 2012-04-10-21-50-48
#separator \x09
#set_separator ,
#empty_field (empty)
#unset_field -
#path weird
#start 2012-05-29-22-02-34
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p name addl notice peer
#types time string addr port addr port string string bool string
1338328954.078361 - - - - - internally_truncated_header - F bro
#end 2012-05-29-22-02-34

2
testing/btest/Baseline/core.tunnels.ayiya/conn.log
View file

@ -3,6 +3,7 @@
#empty_field (empty)
#unset_field -
#path conn
#start 2009-11-08-04-41-57
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto service duration orig_bytes resp_bytes conn_state local_orig missed_bytes history orig_pkts orig_ip_bytes resp_pkts resp_ip_bytes tunnel_parents
#types time string addr port addr port enum string interval count count string bool count string count count count count table[string]
1257655301.595604 5OKnoww6xl4 2001:4978:f:4c::2 53382 2001:4860:b002::68 80 tcp http 2.101052 2981 4665 S1 - 0 ShADad 10 3605 11 5329 k6kgXLOoSKl
@ -13,3 +14,4 @@
1257655296.585188 TEfuqmmG4bh fe80::216:cbff:fe9a:4cb9 131 ff02::1:ff00:2 130 icmp - 0.919988 32 0 OTH - 0 - 2 144 0 0 k6kgXLOoSKl
1257655296.585151 j4u32Pc5bif fe80::216:cbff:fe9a:4cb9 131 ff02::2:f901:d225 130 icmp - 0.719947 32 0 OTH - 0 - 2 144 0 0 k6kgXLOoSKl
1257655296.585034 nQcgTWjvg4c fe80::216:cbff:fe9a:4cb9 131 ff02::1:ff9a:4cb9 130 icmp - 4.922880 32 0 OTH - 0 - 2 144 0 0 k6kgXLOoSKl
#end 2009-11-08-04-41-57

2
testing/btest/Baseline/core.tunnels.ayiya/http.log
View file

@ -3,8 +3,10 @@
#empty_field (empty)
#unset_field -
#path http
#start 2009-11-08-04-41-41
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file
#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string file
1257655301.652206 5OKnoww6xl4 2001:4978:f:4c::2 53382 2001:4860:b002::68 80 1 GET ipv6.google.com / - Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.5; en; rv:1.9.0.15pre) Gecko/2009091516 Camino/2.0b4 (like Firefox/3.0.15pre) 0 10102 200 OK - - - (empty) - - - text/html - -
1257655302.514424 5OKnoww6xl4 2001:4978:f:4c::2 53382 2001:4860:b002::68 80 2 GET ipv6.google.com /csi?v=3&s=webhp&action=&tran=undefined&e=17259,19771,21517,21766,21887,22212&ei=BUz2Su7PMJTglQfz3NzCAw&rt=prt.77,xjs.565,ol.645 http://ipv6.google.com/ Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.5; en; rv:1.9.0.15pre) Gecko/2009091516 Camino/2.0b4 (like Firefox/3.0.15pre) 0 0 204 No Content - - - (empty) - - - - - -
1257655303.603569 5OKnoww6xl4 2001:4978:f:4c::2 53382 2001:4860:b002::68 80 3 GET ipv6.google.com /gen_204?atyp=i&ct=fade&cad=1254&ei=BUz2Su7PMJTglQfz3NzCAw&zx=1257655303600 http://ipv6.google.com/ Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.5; en; rv:1.9.0.15pre) Gecko/2009091516 Camino/2.0b4 (like Firefox/3.0.15pre) 0 0 204 No Content - - - (empty) - - - - - -
#end 2009-11-08-04-41-57

2
testing/btest/Baseline/core.tunnels.ayiya/tunnel.log
View file

@ -3,9 +3,11 @@
#empty_field (empty)
#unset_field -
#path tunnel
#start 2009-11-08-04-41-33
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p tunnel_type action
#types time string addr port addr port enum enum
1257655293.629048 UWkUyAuUGXf 192.168.3.101 53796 216.14.98.22 5072 Tunnel::AYIYA Tunnel::DISCOVER
1257655296.585034 k6kgXLOoSKl 192.168.3.101 53859 216.14.98.22 5072 Tunnel::AYIYA Tunnel::DISCOVER
1257655317.464035 k6kgXLOoSKl 192.168.3.101 53859 216.14.98.22 5072 Tunnel::AYIYA Tunnel::CLOSE
1257655317.464035 UWkUyAuUGXf 192.168.3.101 53796 216.14.98.22 5072 Tunnel::AYIYA Tunnel::CLOSE
#end 2009-11-08-04-41-57

2
testing/btest/Baseline/core.tunnels.false-teredo/dpd.log
View file

@ -3,6 +3,7 @@
#empty_field (empty)
#unset_field -
#path dpd
#start 2009-11-18-17-59-51
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto analyzer failure_reason
#types time string addr port addr port enum string string
1258567191.486869 UWkUyAuUGXf 192.168.1.105 57696 192.168.1.1 53 udp TEREDO Teredo payload length [c\x1d\x81\x80\x00\x01\x00\x02\x00\x02\x00\x00\x04amch\x0equestionmarket\x03com\x00\x00\x01\x00...]
@ -11,3 +12,4 @@
1258581768.898165 TEfuqmmG4bh 192.168.1.104 50798 192.168.1.1 53 udp TEREDO Teredo payload length [o\xe3\x81\x80\x00\x01\x00\x02\x00\x04\x00\x04\x03www\x0fnashuatelegraph\x03com\x00\x00\x01\x00...]
1258584478.989528 FrJExwHcSal 192.168.1.104 64963 192.168.1.1 53 udp TEREDO Teredo payload length [e\xbd\x81\x80\x00\x01\x00\x08\x00\x06\x00\x06\x08wellness\x05blogs\x04time\x03com\x00\x00\x01\x00...]
1258600683.934672 5OKnoww6xl4 192.168.1.103 59838 192.168.1.1 53 udp TEREDO Teredo payload length [h\xf0\x81\x80\x00\x01\x00\x01\x00\x02\x00\x00\x06update\x0csanasecurity\x03com\x00\x00\x01\x00...]
#end 2009-11-19-03-18-03

2
testing/btest/Baseline/core.tunnels.false-teredo/weird.log
View file

@ -3,6 +3,7 @@
#empty_field (empty)
#unset_field -
#path weird
#start 2009-11-18-17-59-51
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p name addl notice peer
#types time string addr port addr port string string bool string
1258567191.405770 - - - - - truncated_header_in_tunnel - F bro
@ -11,3 +12,4 @@
1258581768.568451 - - - - - truncated_header_in_tunnel - F bro
1258584478.859853 - - - - - truncated_header_in_tunnel - F bro
1258600683.934458 - - - - - truncated_header_in_tunnel - F bro
#end 2009-11-19-03-18-03

2
testing/btest/Baseline/core.tunnels.teredo/conn.log
View file

@ -3,6 +3,7 @@
#empty_field (empty)
#unset_field -
#path conn
#start 2008-05-16-15-50-57
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto service duration orig_bytes resp_bytes conn_state local_orig missed_bytes history orig_pkts orig_ip_bytes resp_pkts resp_ip_bytes tunnel_parents
#types time string addr port addr port enum string interval count count string bool count string count count count count table[string]
1210953047.736921 arKYeMETxOg 192.168.2.16 1576 75.126.130.163 80 tcp - 0.000357 0 0 SHR - 0 fA 1 40 1 40 (empty)
@ -26,3 +27,4 @@
1210953052.324629 FrJExwHcSal fe80::8000:f227:bec8:61af 134 fe80::8000:ffff:ffff:fffd 133 icmp - - - - OTH - 0 - 1 88 0 0 TEfuqmmG4bh
1210953060.829303 qCaWGmzFtM5 2001:0:4137:9e50:8000:f12a:b9c8:2815 128 2001:4860:0:2001::68 129 icmp - 0.463615 4 4 OTH - 0 - 1 52 1 52 GSxOnSLghOa,nQcgTWjvg4c
1210953052.202579 j4u32Pc5bif fe80::8000:ffff:ffff:fffd 133 ff02::2 134 icmp - - - - OTH - 0 - 1 64 0 0 nQcgTWjvg4c
#end 2008-05-16-15-51-16

2
testing/btest/Baseline/core.tunnels.teredo/http.log
View file

@ -3,9 +3,11 @@
#empty_field (empty)
#unset_field -
#path http
#start 2008-05-16-15-50-58
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file
#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string file
1210953057.917183 3PKsZ2Uye21 192.168.2.16 1578 75.126.203.78 80 1 POST download913.avast.com /cgi-bin/iavs4stats.cgi - Syncer/4.80 (av_pro-1169;f) 589 0 204 <empty> - - - (empty) - - - text/plain - -
1210953061.585996 70MGiRM1Qf4 2001:0:4137:9e50:8000:f12a:b9c8:2815 1286 2001:4860:0:2001::68 80 1 GET ipv6.google.com / - Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9b5) Gecko/2008032620 Firefox/3.0b5 0 6640 200 OK - - - (empty) - - - text/html - -
1210953073.381474 70MGiRM1Qf4 2001:0:4137:9e50:8000:f12a:b9c8:2815 1286 2001:4860:0:2001::68 80 2 GET ipv6.google.com /search?hl=en&q=Wireshark+!&btnG=Google+Search http://ipv6.google.com/ Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9b5) Gecko/2008032620 Firefox/3.0b5 0 25119 200 OK - - - (empty) - - - text/html - -
1210953074.674817 c4Zw9TmAE05 192.168.2.16 1580 67.228.110.120 80 1 GET www.wireshark.org / http://ipv6.google.com/search?hl=en&q=Wireshark+%21&btnG=Google+Search Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9b5) Gecko/2008032620 Firefox/3.0b5 0 11845 200 OK - - - (empty) - - - text/xml - -
#end 2008-05-16-15-51-16

2
testing/btest/Baseline/core.tunnels.teredo/tunnel.log
View file

@ -3,6 +3,7 @@
#empty_field (empty)
#unset_field -
#path tunnel
#start 2008-05-16-15-50-52
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p tunnel_type action
#types time string addr port addr port enum enum
1210953052.202579 nQcgTWjvg4c 192.168.2.16 3797 65.55.158.80 3544 Tunnel::TEREDO Tunnel::DISCOVER
@ -11,3 +12,4 @@
1210953076.058333 nQcgTWjvg4c 192.168.2.16 3797 65.55.158.80 3544 Tunnel::TEREDO Tunnel::CLOSE
1210953076.058333 GSxOnSLghOa 192.168.2.16 3797 83.170.1.38 32900 Tunnel::TEREDO Tunnel::CLOSE
1210953076.058333 TEfuqmmG4bh 192.168.2.16 3797 65.55.158.81 3544 Tunnel::TEREDO Tunnel::CLOSE
#end 2008-05-16-15-51-16

2
testing/btest/Baseline/core.tunnels.teredo_bubble_with_payload/conn.log
View file

@ -3,6 +3,7 @@
#empty_field (empty)
#unset_field -
#path conn
#start 2012-06-19-17-39-37
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto service duration orig_bytes resp_bytes conn_state local_orig missed_bytes history orig_pkts orig_ip_bytes resp_pkts resp_ip_bytes tunnel_parents
#types time string addr port addr port enum string interval count count string bool count string count count count count table[string]
1340127577.354166 FrJExwHcSal 2001:0:4137:9e50:8000:f12a:b9c8:2815 1286 2001:4860:0:2001::68 80 tcp http 0.052829 1675 10467 S1 - 0 ShADad 10 2279 12 11191 j4u32Pc5bif
@ -12,3 +13,4 @@
1340127577.339015 nQcgTWjvg4c fe80::8000:f227:bec8:61af 134 fe80::8000:ffff:ffff:fffd 133 icmp - - - - OTH - 0 - 1 88 0 0 k6kgXLOoSKl
1340127577.343969 TEfuqmmG4bh 2001:0:4137:9e50:8000:f12a:b9c8:2815 128 2001:4860:0:2001::68 129 icmp - 0.007778 4 4 OTH - 0 - 1 52 1 52 UWkUyAuUGXf,j4u32Pc5bif
1340127577.336558 arKYeMETxOg fe80::8000:ffff:ffff:fffd 133 ff02::2 134 icmp - - - - OTH - 0 - 1 64 0 0 UWkUyAuUGXf
#end 2012-06-19-17-39-37

2
testing/btest/Baseline/core.tunnels.teredo_bubble_with_payload/http.log
View file

@ -3,7 +3,9 @@
#empty_field (empty)
#unset_field -
#path http
#start 2012-06-19-17-39-37
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file
#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string file
1340127577.361683 FrJExwHcSal 2001:0:4137:9e50:8000:f12a:b9c8:2815 1286 2001:4860:0:2001::68 80 1 GET ipv6.google.com / - Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9b5) Gecko/2008032620 Firefox/3.0b5 0 6640 200 OK - - - (empty) - - - text/html - -
1340127577.379360 FrJExwHcSal 2001:0:4137:9e50:8000:f12a:b9c8:2815 1286 2001:4860:0:2001::68 80 2 GET ipv6.google.com /search?hl=en&q=Wireshark+!&btnG=Google+Search http://ipv6.google.com/ Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9b5) Gecko/2008032620 Firefox/3.0b5 0 25119 200 OK - - - (empty) - - - text/html - -
#end 2012-06-19-17-39-37

2
testing/btest/Baseline/core.tunnels.teredo_bubble_with_payload/tunnel.log
View file

@ -3,6 +3,7 @@
#empty_field (empty)
#unset_field -
#path tunnel
#start 2012-06-19-17-39-37
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p tunnel_type action
#types time string addr port addr port enum enum
1340127577.336558 UWkUyAuUGXf 192.168.2.16 3797 65.55.158.80 3544 Tunnel::TEREDO Tunnel::DISCOVER
@ -11,3 +12,4 @@
1340127577.406995 UWkUyAuUGXf 192.168.2.16 3797 65.55.158.80 3544 Tunnel::TEREDO Tunnel::CLOSE
1340127577.406995 j4u32Pc5bif 192.168.2.16 3797 83.170.1.38 32900 Tunnel::TEREDO Tunnel::CLOSE
1340127577.406995 k6kgXLOoSKl 192.168.2.16 3797 65.55.158.81 3544 Tunnel::TEREDO Tunnel::CLOSE
#end 2012-06-19-17-39-37

2
testing/btest/Baseline/core.tunnels.teredo_bubble_with_payload/weird.log
View file

@ -3,7 +3,9 @@
#empty_field (empty)
#unset_field -
#path weird
#start 2012-06-19-17-39-37
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p name addl notice peer
#types time string addr port addr port string string bool string
1340127577.346849 UWkUyAuUGXf 192.168.2.16 3797 65.55.158.80 3544 Teredo_bubble_with_payload - F bro
1340127577.349292 j4u32Pc5bif 192.168.2.16 3797 83.170.1.38 32900 Teredo_bubble_with_payload - F bro
#end 2012-06-19-17-39-37

2
testing/btest/Baseline/core.vlan-mpls/conn.log
View file

@ -3,8 +3,10 @@
#empty_field (empty)
#unset_field -
#path conn
#start 2005-10-07-23-23-55
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto service duration orig_bytes resp_bytes conn_state local_orig missed_bytes history orig_pkts orig_ip_bytes resp_pkts resp_ip_bytes tunnel_parents
#types time string addr port addr port enum string interval count count string bool count string count count count count table[string]
952109346.874907 UWkUyAuUGXf 10.1.2.1 11001 10.34.0.1 23 tcp - 2.102560 26 0 SH - 0 SADF 11 470 0 0 (empty)
1128727435.450898 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 tcp http 1.733303 98 9417 SF - 0 ShADdFaf 12 730 10 9945 (empty)
1278600802.069419 k6kgXLOoSKl 10.20.80.1 50343 10.0.0.15 80 tcp - 0.004152 9 3429 SF - 0 ShADadfF 7 381 7 3801 (empty)
#end 2010-07-08-14-53-22

3
testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log
View file

@ -3,6 +3,7 @@
#empty_field (empty)
#unset_field -
#path loaded_scripts
#start 2012-07-20-14-34-11
#fields name
#types string
scripts/base/init-bare.bro
@ -20,6 +21,7 @@ scripts/base/init-bare.bro
scripts/base/frameworks/logging/./postprocessors/./sftp.bro
scripts/base/frameworks/logging/./writers/ascii.bro
scripts/base/frameworks/logging/./writers/dataseries.bro
scripts/base/frameworks/logging/./writers/elasticsearch.bro
scripts/base/frameworks/logging/./writers/none.bro
scripts/base/frameworks/input/__load__.bro
scripts/base/frameworks/input/./main.bro
@ -28,3 +30,4 @@ scripts/base/init-bare.bro
scripts/base/frameworks/input/./readers/raw.bro
scripts/base/frameworks/input/./readers/benchmark.bro
scripts/policy/misc/loaded-scripts.bro
#end 2012-07-20-14-34-11

10
testing/btest/Baseline/coverage.coverage-blacklist/output
View file

@ -1,5 +1,5 @@
1 /Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/coverage.coverage-blacklist/coverage-blacklist.bro, line 13 print cover me;
1 /Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/coverage.coverage-blacklist/coverage-blacklist.bro, line 17 print always executed;
0 /Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/coverage.coverage-blacklist/coverage-blacklist.bro, line 26 print also impossible, but included in code coverage analysis;
1 /Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/coverage.coverage-blacklist/coverage-blacklist.bro, line 29 print success;
1 /Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/coverage.coverage-blacklist/coverage-blacklist.bro, line 5 print first;
1 /da/home/robin/bro/master/testing/btest/.tmp/coverage.coverage-blacklist/coverage-blacklist.bro, line 13 print cover me;
1 /da/home/robin/bro/master/testing/btest/.tmp/coverage.coverage-blacklist/coverage-blacklist.bro, line 17 print always executed;
0 /da/home/robin/bro/master/testing/btest/.tmp/coverage.coverage-blacklist/coverage-blacklist.bro, line 26 print also impossible, but included in code coverage analysis;
1 /da/home/robin/bro/master/testing/btest/.tmp/coverage.coverage-blacklist/coverage-blacklist.bro, line 29 print success;
1 /da/home/robin/bro/master/testing/btest/.tmp/coverage.coverage-blacklist/coverage-blacklist.bro, line 5 print first;

3
testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log
View file

@ -3,6 +3,7 @@
#empty_field (empty)
#unset_field -
#path loaded_scripts
#start 2012-07-20-14-34-40
#fields name
#types string
scripts/base/init-bare.bro
@ -20,6 +21,7 @@ scripts/base/init-bare.bro
scripts/base/frameworks/logging/./postprocessors/./sftp.bro
scripts/base/frameworks/logging/./writers/ascii.bro
scripts/base/frameworks/logging/./writers/dataseries.bro
scripts/base/frameworks/logging/./writers/elasticsearch.bro
scripts/base/frameworks/logging/./writers/none.bro
scripts/base/frameworks/input/__load__.bro
scripts/base/frameworks/input/./main.bro
@ -108,3 +110,4 @@ scripts/base/init-default.bro
scripts/base/protocols/syslog/./consts.bro
scripts/base/protocols/syslog/./main.bro
scripts/policy/misc/loaded-scripts.bro
#end 2012-07-20-14-34-40

6
testing/btest/Baseline/istate.broccoli/bro.log
View file

@ -1,3 +1,3 @@
ping received, seq 0, 1324314397.698781 at src, 1324314397.699240 at dest,
ping received, seq 1, 1324314398.698905 at src, 1324314398.699094 at dest,
ping received, seq 2, 1324314399.699012 at src, 1324314399.699231 at dest,
ping received, seq 0, 1342749173.594568 at src, 1342749173.637317 at dest,
ping received, seq 1, 1342749174.594948 at src, 1342749174.596551 at dest,
ping received, seq 2, 1342749175.595486 at src, 1342749175.596581 at dest,

4
testing/btest/Baseline/istate.events-ssl/receiver.http.log
View file

@ -3,6 +3,8 @@
#empty_field (empty)
#unset_field -
#path http
#start 2012-07-20-01-53-03
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file
#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string file
1336588614.060989 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - (empty) - - - text/html - -
1342749182.906082 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - (empty) - - - text/html - -
#end 2012-07-20-01-53-04

4
testing/btest/Baseline/istate.events-ssl/sender.http.log
View file

@ -3,6 +3,8 @@
#empty_field (empty)
#unset_field -
#path http
#start 2012-07-20-01-53-03
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file
#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string file
1336588614.060989 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - (empty) - - - text/html - -
1342749182.906082 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - (empty) - - - text/html - -
#end 2012-07-20-01-53-04

4
testing/btest/Baseline/istate.events/receiver.http.log
View file

@ -3,6 +3,8 @@
#empty_field (empty)
#unset_field -
#path http
#start 2012-07-20-01-53-12
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file
#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string file
1336587178.164598 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - (empty) - - - text/html - -
1342749191.765740 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - (empty) - - - text/html - -
#end 2012-07-20-01-53-13

4
testing/btest/Baseline/istate.events/sender.http.log
View file

@ -3,6 +3,8 @@
#empty_field (empty)
#unset_field -
#path http
#start 2012-07-20-01-53-12
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file
#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string file
1336587178.164598 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - (empty) - - - text/html - -
1342749191.765740 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - (empty) - - - text/html - -
#end 2012-07-20-01-53-13

2
testing/btest/Baseline/istate.pybroccoli/bro..stdout
View file

@ -1,7 +1,7 @@
==== atomic
-10
2
1336411585.166009
1342749196.619505
2.0 mins
F
1.5

6
testing/btest/Baseline/istate.pybroccoli/python..stdout.filtered
View file

@ -1,7 +1,7 @@
==== atomic a 1 ====
-4L -4
42 42
1336411585.1711
1342749196.6624
60.0
True True
3.14
@ -14,7 +14,7 @@ True True
==== atomic a 2 ====
-10L -10
2 2
1336411585.1660
1342749196.6195
120.0
False False
1.5
@ -27,7 +27,7 @@ False False
==== atomic b 2 ====
-10L -10
<broccoli.count instance at > 2
<broccoli.time instance at > 1336411585.1660
<broccoli.time instance at > 1342749196.6195
<broccoli.interval instance at > 120.0
False False
1.5

2
testing/btest/Baseline/language.wrong-delete-field/output
View file

@ -1 +1 @@
error in /Users/robin/bro/master/testing/btest/.tmp/language.wrong-delete-field/wrong-delete-field.bro, line 10: illegal delete statement (delete x$a)
error in /da/home/robin/bro/master/testing/btest/.tmp/language.wrong-delete-field/wrong-delete-field.bro, line 10: illegal delete statement (delete x$a)

32
testing/btest/Baseline/scripts.base.frameworks.communication.communication_log_baseline/send.log
View file

@ -3,20 +3,22 @@
#empty_field (empty)
#unset_field -
#path communication
#start 2012-07-20-01-49-40
#fields ts peer src_name connected_peer_desc connected_peer_addr connected_peer_port level message
#types time string string string addr port string string
1340904724.781527 bro parent - - - info [#1/127.0.0.1:47757] added peer
1340904724.784954 bro child - - - info [#1/127.0.0.1:47757] connected
1340904724.786168 bro parent - - - info [#1/127.0.0.1:47757] peer connected
1340904724.786168 bro parent - - - info [#1/127.0.0.1:47757] phase: version
1340904724.786168 bro script - - - info connection established
1340904724.786168 bro script - - - info requesting events matching /^?(NOTHING)$?/
1340904724.786168 bro script - - - info accepting state
1340904724.787645 bro parent - - - info [#1/127.0.0.1:47757] phase: handshake
1340904724.787645 bro parent - - - info warning: no events to request
1340904724.788857 bro parent - - - info [#1/127.0.0.1:47757] peer_description is bro
1340904724.829480 bro parent - - - info [#1/127.0.0.1:47757] peer supports keep-in-cache; using that
1340904724.829480 bro parent - - - info [#1/127.0.0.1:47757] phase: running
1340904724.829480 bro parent - - - info terminating...
1340904724.832952 bro child - - - info terminating
1340904724.834082 bro parent - - - info [#1/127.0.0.1:47757] closing connection
1342748980.737451 bro parent - - - info [#1/127.0.0.1:47757] added peer
1342748980.747149 bro child - - - info [#1/127.0.0.1:47757] connected
1342748980.748489 bro parent - - - info [#1/127.0.0.1:47757] peer connected
1342748980.748489 bro parent - - - info [#1/127.0.0.1:47757] phase: version
1342748980.750749 bro script - - - info connection established
1342748980.750749 bro script - - - info requesting events matching /^?(NOTHING)$?/
1342748980.750749 bro script - - - info accepting state
1342748980.752225 bro parent - - - info [#1/127.0.0.1:47757] phase: handshake
1342748980.752225 bro parent - - - info warning: no events to request
1342748980.753384 bro parent - - - info [#1/127.0.0.1:47757] peer_description is bro
1342748980.793108 bro parent - - - info [#1/127.0.0.1:47757] peer supports keep-in-cache; using that
1342748980.793108 bro parent - - - info [#1/127.0.0.1:47757] phase: running
1342748980.793108 bro parent - - - info terminating...
1342748980.796454 bro child - - - info terminating
1342748980.797536 bro parent - - - info [#1/127.0.0.1:47757] closing connection
#end 2012-07-20-01-49-40

6
testing/btest/Baseline/scripts.base.frameworks.logging.adapt-filter/ssh-new-default.log
View file

@ -3,7 +3,9 @@
#empty_field (empty)
#unset_field -
#path ssh-new-default
#start 2012-07-20-01-49-19
#fields t id.orig_h id.orig_p id.resp_h id.resp_p status country
#types time addr port addr port string string
1324314313.140603 1.2.3.4 1234 2.3.4.5 80 success unknown
1324314313.140603 1.2.3.4 1234 2.3.4.5 80 failure US
1342748959.430282 1.2.3.4 1234 2.3.4.5 80 success unknown
1342748959.430282 1.2.3.4 1234 2.3.4.5 80 failure US
#end 2012-07-20-01-49-19

2
testing/btest/Baseline/scripts.base.frameworks.logging.ascii-binary/ssh.log
View file

@ -3,8 +3,10 @@
#empty_field|(empty)
#unset_field|-
#path|ssh
#start|2012-07-20-01-49-19
#fields|data|data2
#types|string|string
abc\x0a\xffdef|DATA2
abc\x7c\xffdef|DATA2
abc\xff\x7cdef|DATA2
#end|2012-07-20-01-49-19

12
testing/btest/Baseline/scripts.base.frameworks.logging.ascii-empty/ssh-filtered.log Normal file
View file

@ -0,0 +1,12 @@
PREFIX<>separator |
PREFIX<>set_separator|,
PREFIX<>empty_field|EMPTY
PREFIX<>unset_field|NOT-SET
PREFIX<>path|ssh
PREFIX<>fields|t|id.orig_h|id.orig_p|id.resp_h|id.resp_p|status|country|b
PREFIX<>types|time|addr|port|addr|port|string|string|bool
1342748959.659721|1.2.3.4|1234|2.3.4.5|80|success|unknown|NOT-SET
1342748959.659721|1.2.3.4|1234|2.3.4.5|80|NOT-SET|US|NOT-SET
1342748959.659721|1.2.3.4|1234|2.3.4.5|80|failure|UK|NOT-SET
1342748959.659721|1.2.3.4|1234|2.3.4.5|80|NOT-SET|BR|NOT-SET
1342748959.659721|1.2.3.4|1234|2.3.4.5|80|failure|EMPTY|T

12
testing/btest/Baseline/scripts.base.frameworks.logging.ascii-empty/ssh.log
View file

@ -1,12 +0,0 @@
PREFIX<>separator |
PREFIX<>set_separator|,
PREFIX<>empty_field|EMPTY
PREFIX<>unset_field|NOT-SET
PREFIX<>path|ssh
PREFIX<>fields|t|id.orig_h|id.orig_p|id.resp_h|id.resp_p|status|country|b
PREFIX<>types|time|addr|port|addr|port|string|string|bool
1324314313.345323|1.2.3.4|1234|2.3.4.5|80|success|unknown|NOT-SET
1324314313.345323|1.2.3.4|1234|2.3.4.5|80|NOT-SET|US|NOT-SET
1324314313.345323|1.2.3.4|1234|2.3.4.5|80|failure|UK|NOT-SET
1324314313.345323|1.2.3.4|1234|2.3.4.5|80|NOT-SET|BR|NOT-SET
1324314313.345323|1.2.3.4|1234|2.3.4.5|80|failure|EMPTY|T

2
testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape-notset-str/test.log
View file

@ -3,6 +3,8 @@
#empty_field (empty)
#unset_field -
#path test
#start 2012-07-20-01-49-19
#fields x y z
#types string string string
\x2d - (empty)
#end 2012-07-20-01-49-19

Some files were not shown because too many files have changed in this diff Show more