mirror of
https://github.com/zeek/zeek.git
synced 2025-10-04 15:48:19 +00:00
Merge branch 'topic/robin/master-test'
* topic/robin/master-test: (60 commits) Script fix for Linux. Updating test base line. Another small change to MsgThread API. Bug fix for BasicThread. make version_ok return true for TLSv12 Sed usage in canonifier script didn't work on non-Linux systems. Changing HTTP DPD port 3138 to 3128. Temporarily removing tuning/logs-to-elasticsearch.bro from the test-all-policy. More documentation updates. Revert "Fixing calc_next_rotate to use UTC based time functions." Some documentation updates for elasticsearch plugin. Give configure a --disable-perftools option. Updating tests for the #start/#end change. Further threading and API restructuring for logging and input frameworks. Reworking forceful thread termination. Moving the ASCII writer over to use UNIX I/O rather than stdio. Further reworking the thread API. Reworking thread termination logic. If a thread doesn't terminate, we log that but not longer proceed (because it could hang later still). Removing the thread kill functionality. ...
This commit is contained in:
Robin Sommer
commit
24aea295fa
176 changed files with 2238 additions and 771 deletions
|
|
@ -5,6 +5,7 @@
|
|||
#include "Manager.h"
|
||||
|
||||
#include <unistd.h>
|
||||
#include <signal.h>
|
||||
|
||||
using namespace threading;
|
||||
|
||||
|
|
@ -16,19 +17,17 @@ namespace threading {
|
|||
class FinishMessage : public InputMessage<MsgThread>
|
||||
{
|
||||
public:
|
||||
FinishMessage(MsgThread* thread) : InputMessage<MsgThread>("Finish", thread) { }
|
||||
FinishMessage(MsgThread* thread, double network_time) : InputMessage<MsgThread>("Finish", thread),
|
||||
network_time(network_time) { }
|
||||
|
||||
virtual bool Process() { return Object()->DoFinish(); }
|
||||
};
|
||||
virtual bool Process() {
|
||||
bool result = Object()->OnFinish(network_time);
|
||||
Object()->Finished();
|
||||
return result;
|
||||
}
|
||||
|
||||
// A dummy message that's only purpose is unblock the current read operation
|
||||
// so that the child's Run() methods can check the termination status.
|
||||
class UnblockMessage : public InputMessage<MsgThread>
|
||||
{
|
||||
public:
|
||||
UnblockMessage(MsgThread* thread) : InputMessage<MsgThread>("Unblock", thread) { }
|
||||
|
||||
virtual bool Process() { return true; }
|
||||
private:
|
||||
double network_time;
|
||||
};
|
||||
|
||||
/// Sends a heartbeat to the child thread.
|
||||
|
|
@ -39,7 +38,10 @@ public:
|
|||
: InputMessage<MsgThread>("Heartbeat", thread)
|
||||
{ network_time = arg_network_time; current_time = arg_current_time; }
|
||||
|
||||
virtual bool Process() { return Object()->DoHeartbeat(network_time, current_time); }
|
||||
virtual bool Process() {
|
||||
Object()->HeartbeatInChild();
|
||||
return Object()->OnHeartbeat(network_time, current_time);
|
||||
}
|
||||
|
||||
private:
|
||||
double network_time;
|
||||
|
|
@ -55,14 +57,16 @@ public:
|
|||
INTERNAL_WARNING, INTERNAL_ERROR
|
||||
};
|
||||
|
||||
ReporterMessage(Type arg_type, MsgThread* thread, const string& arg_msg)
|
||||
ReporterMessage(Type arg_type, MsgThread* thread, const char* arg_msg)
|
||||
: OutputMessage<MsgThread>("ReporterMessage", thread)
|
||||
{ type = arg_type; msg = arg_msg; }
|
||||
{ type = arg_type; msg = copy_string(arg_msg); }
|
||||
|
||||
~ReporterMessage() { delete [] msg; }
|
||||
|
||||
virtual bool Process();
|
||||
|
||||
private:
|
||||
string msg;
|
||||
const char* msg;
|
||||
Type type;
|
||||
};
|
||||
|
||||
|
|
@ -71,18 +75,19 @@ private:
|
|||
class DebugMessage : public OutputMessage<MsgThread>
|
||||
{
|
||||
public:
|
||||
DebugMessage(DebugStream arg_stream, MsgThread* thread, const string& arg_msg)
|
||||
DebugMessage(DebugStream arg_stream, MsgThread* thread, const char* arg_msg)
|
||||
: OutputMessage<MsgThread>("DebugMessage", thread)
|
||||
{ stream = arg_stream; msg = arg_msg; }
|
||||
{ stream = arg_stream; msg = copy_string(arg_msg); }
|
||||
|
||||
virtual ~DebugMessage() { delete [] msg; }
|
||||
|
||||
virtual bool Process()
|
||||
{
|
||||
string s = Object()->Name() + ": " + msg;
|
||||
debug_logger.Log(stream, "%s", s.c_str());
|
||||
debug_logger.Log(stream, "%s: %s", Object()->Name(), msg);
|
||||
return true;
|
||||
}
|
||||
private:
|
||||
string msg;
|
||||
const char* msg;
|
||||
DebugStream stream;
|
||||
};
|
||||
#endif
|
||||
|
|
@ -93,41 +98,39 @@ private:
|
|||
|
||||
Message::~Message()
|
||||
{
|
||||
delete [] name;
|
||||
}
|
||||
|
||||
bool ReporterMessage::Process()
|
||||
{
|
||||
string s = Object()->Name() + ": " + msg;
|
||||
const char* cmsg = s.c_str();
|
||||
|
||||
switch ( type ) {
|
||||
|
||||
case INFO:
|
||||
reporter->Info("%s", cmsg);
|
||||
reporter->Info("%s: %s", Object()->Name(), msg);
|
||||
break;
|
||||
|
||||
case WARNING:
|
||||
reporter->Warning("%s", cmsg);
|
||||
reporter->Warning("%s: %s", Object()->Name(), msg);
|
||||
break;
|
||||
|
||||
case ERROR:
|
||||
reporter->Error("%s", cmsg);
|
||||
reporter->Error("%s: %s", Object()->Name(), msg);
|
||||
break;
|
||||
|
||||
case FATAL_ERROR:
|
||||
reporter->FatalError("%s", cmsg);
|
||||
reporter->FatalError("%s: %s", Object()->Name(), msg);
|
||||
break;
|
||||
|
||||
case FATAL_ERROR_WITH_CORE:
|
||||
reporter->FatalErrorWithCore("%s", cmsg);
|
||||
reporter->FatalErrorWithCore("%s: %s", Object()->Name(), msg);
|
||||
break;
|
||||
|
||||
case INTERNAL_WARNING:
|
||||
reporter->InternalWarning("%s", cmsg);
|
||||
reporter->InternalWarning("%s: %s", Object()->Name(), msg);
|
||||
break;
|
||||
|
||||
case INTERNAL_ERROR :
|
||||
reporter->InternalError("%s", cmsg);
|
||||
reporter->InternalError("%s: %s", Object()->Name(), msg);
|
||||
break;
|
||||
|
||||
default:
|
||||
|
|
@ -137,32 +140,74 @@ bool ReporterMessage::Process()
|
|||
return true;
|
||||
}
|
||||
|
||||
MsgThread::MsgThread() : BasicThread()
|
||||
MsgThread::MsgThread() : BasicThread(), queue_in(this, 0), queue_out(0, this)
|
||||
{
|
||||
cnt_sent_in = cnt_sent_out = 0;
|
||||
finished = false;
|
||||
thread_mgr->AddMsgThread(this);
|
||||
}
|
||||
|
||||
// Set by Bro's main signal handler.
|
||||
extern int signal_val;
|
||||
|
||||
void MsgThread::OnPrepareStop()
|
||||
{
|
||||
if ( finished || Killed() )
|
||||
return;
|
||||
|
||||
// Signal thread to terminate and wait until it has acknowledged.
|
||||
SendIn(new FinishMessage(this, network_time), true);
|
||||
}
|
||||
|
||||
void MsgThread::OnStop()
|
||||
{
|
||||
// Signal thread to terminate and wait until it has acknowledged.
|
||||
SendIn(new FinishMessage(this), true);
|
||||
int signal_count = 0;
|
||||
int old_signal_val = signal_val;
|
||||
signal_val = 0;
|
||||
|
||||
int cnt = 0;
|
||||
while ( ! finished )
|
||||
uint64_t last_size = 0;
|
||||
uint64_t cur_size = 0;
|
||||
|
||||
while ( ! (finished || Killed() ) )
|
||||
{
|
||||
if ( ++cnt > 1000 ) // Insurance against broken threads ...
|
||||
// Terminate if we get another kill signal.
|
||||
if ( signal_val == SIGTERM || signal_val == SIGINT )
|
||||
{
|
||||
reporter->Warning("thread %s didn't finish in time", Name().c_str());
|
||||
break;
|
||||
++signal_count;
|
||||
|
||||
if ( signal_count == 1 )
|
||||
{
|
||||
// Abort all threads here so that we won't hang next
|
||||
// on another one.
|
||||
fprintf(stderr, "received signal while waiting for thread %s, aborting all ...\n", Name());
|
||||
thread_mgr->KillThreads();
|
||||
}
|
||||
else
|
||||
{
|
||||
// More than one signal. Abort processing
|
||||
// right away. on another one.
|
||||
fprintf(stderr, "received another signal while waiting for thread %s, aborting processing\n", Name());
|
||||
exit(1);
|
||||
}
|
||||
|
||||
signal_val = 0;
|
||||
}
|
||||
|
||||
queue_in.WakeUp();
|
||||
|
||||
usleep(1000);
|
||||
}
|
||||
|
||||
// One more message to make sure the current queue read operation unblocks.
|
||||
SendIn(new UnblockMessage(this), true);
|
||||
signal_val = old_signal_val;
|
||||
}
|
||||
|
||||
void MsgThread::OnKill()
|
||||
{
|
||||
// Send a message to unblock the reader if its currently waiting for
|
||||
// input. This is just an optimization to make it terminate more
|
||||
// quickly, even without the message it will eventually time out.
|
||||
queue_in.WakeUp();
|
||||
}
|
||||
|
||||
void MsgThread::Heartbeat()
|
||||
|
|
@ -170,25 +215,20 @@ void MsgThread::Heartbeat()
|
|||
SendIn(new HeartbeatMessage(this, network_time, current_time()));
|
||||
}
|
||||
|
||||
bool MsgThread::DoHeartbeat(double network_time, double current_time)
|
||||
void MsgThread::HeartbeatInChild()
|
||||
{
|
||||
string n = Name();
|
||||
|
||||
n = Fmt("bro: %s (%" PRIu64 "/%" PRIu64 ")", n.c_str(),
|
||||
string n = Fmt("bro: %s (%" PRIu64 "/%" PRIu64 ")", Name(),
|
||||
cnt_sent_in - queue_in.Size(),
|
||||
cnt_sent_out - queue_out.Size());
|
||||
|
||||
SetOSName(n.c_str());
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
bool MsgThread::DoFinish()
|
||||
void MsgThread::Finished()
|
||||
{
|
||||
// This is thread-safe "enough", we're the only one ever writing
|
||||
// there.
|
||||
finished = true;
|
||||
return true;
|
||||
}
|
||||
|
||||
void MsgThread::Info(const char* msg)
|
||||
|
|
@ -245,7 +285,7 @@ void MsgThread::SendIn(BasicInputMessage* msg, bool force)
|
|||
return;
|
||||
}
|
||||
|
||||
DBG_LOG(DBG_THREADING, "Sending '%s' to %s ...", msg->Name().c_str(), Name().c_str());
|
||||
DBG_LOG(DBG_THREADING, "Sending '%s' to %s ...", msg->Name(), Name());
|
||||
|
||||
queue_in.Put(msg);
|
||||
++cnt_sent_in;
|
||||
|
|
@ -268,9 +308,10 @@ void MsgThread::SendOut(BasicOutputMessage* msg, bool force)
|
|||
BasicOutputMessage* MsgThread::RetrieveOut()
|
||||
{
|
||||
BasicOutputMessage* msg = queue_out.Get();
|
||||
assert(msg);
|
||||
if ( ! msg )
|
||||
return 0;
|
||||
|
||||
DBG_LOG(DBG_THREADING, "Retrieved '%s' from %s", msg->Name().c_str(), Name().c_str());
|
||||
DBG_LOG(DBG_THREADING, "Retrieved '%s' from %s", msg->Name(), Name());
|
||||
|
||||
return msg;
|
||||
}
|
||||
|
|
@ -278,10 +319,12 @@ BasicOutputMessage* MsgThread::RetrieveOut()
|
|||
BasicInputMessage* MsgThread::RetrieveIn()
|
||||
{
|
||||
BasicInputMessage* msg = queue_in.Get();
|
||||
assert(msg);
|
||||
|
||||
if ( ! msg )
|
||||
return 0;
|
||||
|
||||
#ifdef DEBUG
|
||||
string s = Fmt("Retrieved '%s' in %s", msg->Name().c_str(), Name().c_str());
|
||||
string s = Fmt("Retrieved '%s' in %s", msg->Name(), Name());
|
||||
Debug(DBG_THREADING, s.c_str());
|
||||
#endif
|
||||
|
||||
|
|
@ -290,27 +333,33 @@ BasicInputMessage* MsgThread::RetrieveIn()
|
|||
|
||||
void MsgThread::Run()
|
||||
{
|
||||
while ( true )
|
||||
while ( ! (finished || Killed() ) )
|
||||
{
|
||||
// When requested to terminate, we only do so when
|
||||
// all input has been processed.
|
||||
if ( Terminating() && ! queue_in.Ready() )
|
||||
break;
|
||||
|
||||
BasicInputMessage* msg = RetrieveIn();
|
||||
|
||||
if ( ! msg )
|
||||
continue;
|
||||
|
||||
bool result = msg->Process();
|
||||
|
||||
if ( ! result )
|
||||
{
|
||||
string s = msg->Name() + " failed, terminating thread (MsgThread)";
|
||||
string s = Fmt("%s failed, terminating thread (MsgThread)", Name());
|
||||
Error(s.c_str());
|
||||
Stop();
|
||||
break;
|
||||
}
|
||||
|
||||
delete msg;
|
||||
}
|
||||
|
||||
// In case we haven't send the finish method yet, do it now. Reading
|
||||
// global network_time here should be fine, it isn't changing
|
||||
// anymore.
|
||||
if ( ! finished )
|
||||
{
|
||||
OnFinish(network_time);
|
||||
Finished();
|
||||
}
|
||||
}
|
||||
|
||||
void MsgThread::GetStats(Stats* stats)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue