mirror of
https://github.com/zeek/zeek.git
synced 2025-10-04 15:48:19 +00:00
Further threading and API restructuring for logging and input
frameworks. There were a number of cases that weren't thread-safe. In particular, we don't use std::string anymore for anything that's passed between threads (but instead plain old const char*, with manual memmory managmenet). This is still a check-point commit, I'll do more testing.
This commit is contained in:
Robin Sommer
parent
490859cfef
commit
87e10b5f97
31 changed files with 692 additions and 381 deletions
|
|
@ -6,6 +6,7 @@
|
|||
#include "../EventHandler.h"
|
||||
#include "../NetVar.h"
|
||||
#include "../Net.h"
|
||||
#include "../Type.h"
|
||||
|
||||
#include "threading/Manager.h"
|
||||
#include "threading/SerialTypes.h"
|
||||
|
|
@ -75,7 +76,7 @@ struct Manager::WriterInfo {
|
|||
double interval;
|
||||
Func* postprocessor;
|
||||
WriterFrontend* writer;
|
||||
WriterBackend::WriterInfo info;
|
||||
WriterBackend::WriterInfo* info;
|
||||
};
|
||||
|
||||
struct Manager::Stream {
|
||||
|
|
@ -118,6 +119,7 @@ Manager::Stream::~Stream()
|
|||
|
||||
Unref(winfo->type);
|
||||
delete winfo->writer;
|
||||
delete winfo->info;
|
||||
delete winfo;
|
||||
}
|
||||
|
||||
|
|
@ -193,7 +195,6 @@ WriterBackend* Manager::CreateBackend(WriterFrontend* frontend, bro_int_t type)
|
|||
|
||||
assert(ld->factory);
|
||||
|
||||
frontend->ty_name = ld->name;
|
||||
WriterBackend* backend = (*ld->factory)(frontend);
|
||||
assert(backend);
|
||||
|
||||
|
|
@ -476,18 +477,17 @@ bool Manager::TraverseRecord(Stream* stream, Filter* filter, RecordType* rt,
|
|||
return false;
|
||||
}
|
||||
|
||||
threading::Field* field = new threading::Field();
|
||||
field->name = new_path;
|
||||
field->type = t->Tag();
|
||||
field->optional = rt->FieldDecl(i)->FindAttr(ATTR_OPTIONAL);
|
||||
TypeTag st = TYPE_VOID;
|
||||
|
||||
if ( field->type == TYPE_TABLE )
|
||||
field->subtype = t->AsSetType()->Indices()->PureType()->Tag();
|
||||
if ( t->Tag() == TYPE_TABLE )
|
||||
st = t->AsSetType()->Indices()->PureType()->Tag();
|
||||
|
||||
else if ( field->type == TYPE_VECTOR )
|
||||
field->subtype = t->AsVectorType()->YieldType()->Tag();
|
||||
else if ( t->Tag() == TYPE_VECTOR )
|
||||
st = t->AsVectorType()->YieldType()->Tag();
|
||||
|
||||
filter->fields[filter->num_fields - 1] = field;
|
||||
bool optional = rt->FieldDecl(i)->FindAttr(ATTR_OPTIONAL);
|
||||
|
||||
filter->fields[filter->num_fields - 1] = new threading::Field(new_path.c_str(), 0, t->Tag(), st, optional);
|
||||
}
|
||||
|
||||
return true;
|
||||
|
|
@ -594,7 +594,7 @@ bool Manager::AddFilter(EnumVal* id, RecordVal* fval)
|
|||
{
|
||||
threading::Field* field = filter->fields[i];
|
||||
DBG_LOG(DBG_LOGGING, " field %10s: %s",
|
||||
field->name.c_str(), type_name(field->type));
|
||||
field->name, type_name(field->type));
|
||||
}
|
||||
#endif
|
||||
|
||||
|
|
@ -769,9 +769,9 @@ bool Manager::Write(EnumVal* id, RecordVal* columns)
|
|||
for ( int j = 0; j < filter->num_fields; ++j )
|
||||
arg_fields[j] = new threading::Field(*filter->fields[j]);
|
||||
|
||||
WriterBackend::WriterInfo info;
|
||||
info.path = path;
|
||||
info.network_time = network_time;
|
||||
WriterBackend::WriterInfo* info = new WriterBackend::WriterInfo;
|
||||
info->path = copy_string(path.c_str());
|
||||
info->network_time = network_time;
|
||||
|
||||
HashKey* k;
|
||||
IterCookie* c = filter->config->AsTable()->InitForIteration();
|
||||
|
|
@ -782,7 +782,7 @@ bool Manager::Write(EnumVal* id, RecordVal* columns)
|
|||
ListVal* index = filter->config->RecoverIndex(k);
|
||||
string key = index->Index(0)->AsString()->CheckString();
|
||||
string value = v->Value()->AsString()->CheckString();
|
||||
info.config.insert(std::make_pair(key, value));
|
||||
info->config.insert(std::make_pair(copy_string(key.c_str()), copy_string(value.c_str())));
|
||||
Unref(index);
|
||||
delete k;
|
||||
}
|
||||
|
|
@ -844,11 +844,16 @@ threading::Value* Manager::ValToLogVal(Val* val, BroType* ty)
|
|||
val->Type()->AsEnumType()->Lookup(val->InternalInt());
|
||||
|
||||
if ( s )
|
||||
lval->val.string_val = new string(s);
|
||||
{
|
||||
lval->val.string_val.data = copy_string(s);
|
||||
lval->val.string_val.length = strlen(s);
|
||||
}
|
||||
|
||||
else
|
||||
{
|
||||
val->Type()->Error("enum type does not contain value", val);
|
||||
lval->val.string_val = new string();
|
||||
lval->val.string_val.data = copy_string("");
|
||||
lval->val.string_val.length = 0;
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
|
@ -880,15 +885,20 @@ threading::Value* Manager::ValToLogVal(Val* val, BroType* ty)
|
|||
case TYPE_STRING:
|
||||
{
|
||||
const BroString* s = val->AsString();
|
||||
lval->val.string_val =
|
||||
new string((const char*) s->Bytes(), s->Len());
|
||||
char* buf = new char[s->Len()];
|
||||
memcpy(buf, s->Bytes(), s->Len());
|
||||
|
||||
lval->val.string_val.data = buf;
|
||||
lval->val.string_val.length = s->Len();
|
||||
break;
|
||||
}
|
||||
|
||||
case TYPE_FILE:
|
||||
{
|
||||
const BroFile* f = val->AsFile();
|
||||
lval->val.string_val = new string(f->Name());
|
||||
string s = f->Name();
|
||||
lval->val.string_val.data = copy_string(s.c_str());
|
||||
lval->val.string_val.length = s.size();
|
||||
break;
|
||||
}
|
||||
|
||||
|
|
@ -897,7 +907,9 @@ threading::Value* Manager::ValToLogVal(Val* val, BroType* ty)
|
|||
ODesc d;
|
||||
const Func* f = val->AsFunc();
|
||||
f->Describe(&d);
|
||||
lval->val.string_val = new string(d.Description());
|
||||
const char* s = d.Description();
|
||||
lval->val.string_val.data = copy_string(s);
|
||||
lval->val.string_val.length = strlen(s);
|
||||
break;
|
||||
}
|
||||
|
||||
|
|
@ -977,7 +989,7 @@ threading::Value** Manager::RecordToFilterVals(Stream* stream, Filter* filter,
|
|||
return vals;
|
||||
}
|
||||
|
||||
WriterFrontend* Manager::CreateWriter(EnumVal* id, EnumVal* writer, const WriterBackend::WriterInfo& info,
|
||||
WriterFrontend* Manager::CreateWriter(EnumVal* id, EnumVal* writer, WriterBackend::WriterInfo* info,
|
||||
int num_fields, const threading::Field* const* fields, bool local, bool remote)
|
||||
{
|
||||
Stream* stream = FindStream(id);
|
||||
|
|
@ -987,7 +999,7 @@ WriterFrontend* Manager::CreateWriter(EnumVal* id, EnumVal* writer, const Writer
|
|||
return 0;
|
||||
|
||||
Stream::WriterMap::iterator w =
|
||||
stream->writers.find(Stream::WriterPathPair(writer->AsEnum(), info.path));
|
||||
stream->writers.find(Stream::WriterPathPair(writer->AsEnum(), info->path));
|
||||
|
||||
if ( w != stream->writers.end() )
|
||||
// If we already have a writer for this. That's fine, we just
|
||||
|
|
@ -1013,7 +1025,7 @@ WriterFrontend* Manager::CreateWriter(EnumVal* id, EnumVal* writer, const Writer
|
|||
{
|
||||
Filter* f = *it;
|
||||
if ( f->writer->AsEnum() == writer->AsEnum() &&
|
||||
f->path == info.path )
|
||||
f->path == info->path )
|
||||
{
|
||||
found_filter_match = true;
|
||||
winfo->interval = f->interval;
|
||||
|
|
@ -1030,7 +1042,7 @@ WriterFrontend* Manager::CreateWriter(EnumVal* id, EnumVal* writer, const Writer
|
|||
}
|
||||
|
||||
stream->writers.insert(
|
||||
Stream::WriterMap::value_type(Stream::WriterPathPair(writer->AsEnum(), info.path),
|
||||
Stream::WriterMap::value_type(Stream::WriterPathPair(writer->AsEnum(), info->path),
|
||||
winfo));
|
||||
|
||||
// Still need to set the WriterInfo's rotation parameters, which we
|
||||
|
|
@ -1038,11 +1050,11 @@ WriterFrontend* Manager::CreateWriter(EnumVal* id, EnumVal* writer, const Writer
|
|||
const char* base_time = log_rotate_base_time ?
|
||||
log_rotate_base_time->AsString()->CheckString() : 0;
|
||||
|
||||
winfo->info.rotation_interval = winfo->interval;
|
||||
winfo->info.rotation_base = parse_rotate_base_time(base_time);
|
||||
winfo->info->rotation_interval = winfo->interval;
|
||||
winfo->info->rotation_base = parse_rotate_base_time(base_time);
|
||||
|
||||
winfo->writer = new WriterFrontend(id, writer, local, remote);
|
||||
winfo->writer->Init(winfo->info, num_fields, fields);
|
||||
winfo->writer = new WriterFrontend(*winfo->info, id, writer, local, remote);
|
||||
winfo->writer->Init(num_fields, fields);
|
||||
|
||||
InstallRotationTimer(winfo);
|
||||
|
||||
|
|
@ -1124,7 +1136,7 @@ void Manager::SendAllWritersTo(RemoteSerializer::PeerID peer)
|
|||
EnumVal writer_val(i->first.first, BifType::Enum::Log::Writer);
|
||||
remote_serializer->SendLogCreateWriter(peer, (*s)->id,
|
||||
&writer_val,
|
||||
i->second->info,
|
||||
*i->second->info,
|
||||
writer->NumFields(),
|
||||
writer->Fields());
|
||||
}
|
||||
|
|
@ -1260,14 +1272,14 @@ void Manager::InstallRotationTimer(WriterInfo* winfo)
|
|||
timer_mgr->Add(winfo->rotation_timer);
|
||||
|
||||
DBG_LOG(DBG_LOGGING, "Scheduled rotation timer for %s to %.6f",
|
||||
winfo->writer->Name().c_str(), winfo->rotation_timer->Time());
|
||||
winfo->writer->Name(), winfo->rotation_timer->Time());
|
||||
}
|
||||
}
|
||||
|
||||
void Manager::Rotate(WriterInfo* winfo)
|
||||
{
|
||||
DBG_LOG(DBG_LOGGING, "Rotating %s at %.6f",
|
||||
winfo->writer->Name().c_str(), network_time);
|
||||
winfo->writer->Name(), network_time);
|
||||
|
||||
// Build a temporary path for the writer to move the file to.
|
||||
struct tm tm;
|
||||
|
|
@ -1278,15 +1290,14 @@ void Manager::Rotate(WriterInfo* winfo)
|
|||
localtime_r(&teatime, &tm);
|
||||
strftime(buf, sizeof(buf), date_fmt, &tm);
|
||||
|
||||
string tmp = string(fmt("%s-%s", winfo->writer->Info().path.c_str(), buf));
|
||||
|
||||
// Trigger the rotation.
|
||||
const char* tmp = fmt("%s-%s", winfo->writer->Info().path, buf);
|
||||
winfo->writer->Rotate(tmp, winfo->open_time, network_time, terminating);
|
||||
|
||||
++rotations_pending;
|
||||
}
|
||||
|
||||
bool Manager::FinishedRotation(WriterFrontend* writer, string new_name, string old_name,
|
||||
bool Manager::FinishedRotation(WriterFrontend* writer, const char* new_name, const char* old_name,
|
||||
double open, double close, bool terminating)
|
||||
{
|
||||
--rotations_pending;
|
||||
|
|
@ -1296,7 +1307,7 @@ bool Manager::FinishedRotation(WriterFrontend* writer, string new_name, string o
|
|||
return true;
|
||||
|
||||
DBG_LOG(DBG_LOGGING, "Finished rotating %s at %.6f, new name %s",
|
||||
writer->Name().c_str(), network_time, new_name.c_str());
|
||||
writer->Name(), network_time, new_name);
|
||||
|
||||
WriterInfo* winfo = FindWriter(writer);
|
||||
if ( ! winfo )
|
||||
|
|
@ -1305,8 +1316,8 @@ bool Manager::FinishedRotation(WriterFrontend* writer, string new_name, string o
|
|||
// Create the RotationInfo record.
|
||||
RecordVal* info = new RecordVal(BifType::Record::Log::RotationInfo);
|
||||
info->Assign(0, winfo->type->Ref());
|
||||
info->Assign(1, new StringVal(new_name.c_str()));
|
||||
info->Assign(2, new StringVal(winfo->writer->Info().path.c_str()));
|
||||
info->Assign(1, new StringVal(new_name));
|
||||
info->Assign(2, new StringVal(winfo->writer->Info().path));
|
||||
info->Assign(3, new Val(open, TYPE_TIME));
|
||||
info->Assign(4, new Val(close, TYPE_TIME));
|
||||
info->Assign(5, new Val(terminating, TYPE_BOOL));
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue