Merge branch 'topic/robin/master-test'

* topic/robin/master-test: (60 commits)
  Script fix for Linux.
  Updating test base line.
  Another small change to MsgThread API.
  Bug fix for BasicThread.
  make version_ok return true for TLSv12
  Sed usage in canonifier script didn't work on non-Linux systems.
  Changing HTTP DPD port 3138 to 3128.
  Temporarily removing tuning/logs-to-elasticsearch.bro from the test-all-policy.
  More documentation updates.
  Revert "Fixing calc_next_rotate to use UTC based time functions."
  Some documentation updates for elasticsearch plugin.
  Give configure a --disable-perftools option.
  Updating tests for the #start/#end change.
  Further threading and API restructuring for logging and input frameworks.
  Reworking forceful thread termination.
  Moving the ASCII writer over to use UNIX I/O rather than stdio.
  Further reworking the thread API.
  Reworking thread termination logic.
  If a thread doesn't terminate, we log that but not longer proceed (because it could hang later still).
  Removing the thread kill functionality.
  ...
This commit is contained in:
Robin Sommer 2012-07-23 16:06:34 -07:00
commit 24aea295fa
176 changed files with 2238 additions and 771 deletions

View file

@ -12,51 +12,67 @@
using namespace threading;
static const int STD_FMT_BUF_LEN = 2048;
uint64_t BasicThread::thread_counter = 0;
BasicThread::BasicThread()
{
started = false;
terminating = false;
killed = false;
pthread = 0;
buf_len = 2048;
buf_len = STD_FMT_BUF_LEN;
buf = (char*) malloc(buf_len);
name = Fmt("thread-%d", ++thread_counter);
strerr_buffer = 0;
name = copy_string(fmt("thread-%" PRIu64, ++thread_counter));
thread_mgr->AddThread(this);
}
BasicThread::~BasicThread()
{
if ( buf )
if ( buf )
free(buf);
delete [] name;
delete [] strerr_buffer;
}
void BasicThread::SetName(const string& arg_name)
void BasicThread::SetName(const char* arg_name)
{
// Slight race condition here with reader threads, but shouldn't matter.
name = arg_name;
delete [] name;
name = copy_string(arg_name);
}
void BasicThread::SetOSName(const string& name)
void BasicThread::SetOSName(const char* arg_name)
{
#ifdef HAVE_LINUX
prctl(PR_SET_NAME, name.c_str(), 0, 0, 0);
prctl(PR_SET_NAME, arg_name, 0, 0, 0);
#endif
#ifdef __APPLE__
pthread_setname_np(name.c_str());
pthread_setname_np(arg_name);
#endif
#ifdef FREEBSD
pthread_set_name_np(pthread_self(), name, name.c_str());
pthread_set_name_np(pthread_self(), arg_name, arg_name);
#endif
}
const char* BasicThread::Fmt(const char* format, ...)
{
if ( buf_len > 10 * STD_FMT_BUF_LEN )
{
// Shrink back to normal.
buf = (char*) safe_realloc(buf, STD_FMT_BUF_LEN);
buf_len = STD_FMT_BUF_LEN;
}
va_list al;
va_start(al, format);
int n = safe_vsnprintf(buf, buf_len, format, al);
@ -64,46 +80,56 @@ const char* BasicThread::Fmt(const char* format, ...)
if ( (unsigned int) n >= buf_len )
{ // Not enough room, grow the buffer.
int tmp_len = n + 32;
char* tmp = (char*) malloc(tmp_len);
buf_len = n + 32;
buf = (char*) safe_realloc(buf, buf_len);
// Is it portable to restart?
va_start(al, format);
n = safe_vsnprintf(tmp, tmp_len, format, al);
n = safe_vsnprintf(buf, buf_len, format, al);
va_end(al);
free(tmp);
}
return buf;
}
const char* BasicThread::Strerror(int err)
{
if ( ! strerr_buffer )
strerr_buffer = new char[256];
strerror_r(err, strerr_buffer, 256);
return strerr_buffer;
}
void BasicThread::Start()
{
if ( started )
return;
int err = pthread_mutex_init(&terminate, 0);
if ( err != 0 )
reporter->FatalError("Cannot create terminate mutex for thread %s: %s", name.c_str(), strerror(err));
// We use this like a binary semaphore and acquire it immediately.
err = pthread_mutex_lock(&terminate);
if ( err != 0 )
reporter->FatalError("Cannot aquire terminate mutex for thread %s: %s", name.c_str(), strerror(err));
err = pthread_create(&pthread, 0, BasicThread::launcher, this);
if ( err != 0 )
reporter->FatalError("Cannot create thread %s:%s", name.c_str(), strerror(err));
DBG_LOG(DBG_THREADING, "Started thread %s", name.c_str());
started = true;
int err = pthread_create(&pthread, 0, BasicThread::launcher, this);
if ( err != 0 )
reporter->FatalError("Cannot create thread %s: %s", name, Strerror(err));
DBG_LOG(DBG_THREADING, "Started thread %s", name);
OnStart();
}
void BasicThread::PrepareStop()
{
if ( ! started )
return;
if ( terminating )
return;
DBG_LOG(DBG_THREADING, "Preparing thread %s to terminate ...", name);
OnPrepareStop();
}
void BasicThread::Stop()
{
if ( ! started )
@ -112,17 +138,11 @@ void BasicThread::Stop()
if ( terminating )
return;
DBG_LOG(DBG_THREADING, "Signaling thread %s to terminate ...", name.c_str());
// Signal that it's ok for the thread to exit now by unlocking the
// mutex.
int err = pthread_mutex_unlock(&terminate);
if ( err != 0 )
reporter->FatalError("Failure flagging terminate condition for thread %s: %s", name.c_str(), strerror(err));
terminating = true;
DBG_LOG(DBG_THREADING, "Signaling thread %s to terminate ...", name);
OnStop();
terminating = true;
}
void BasicThread::Join()
@ -130,30 +150,34 @@ void BasicThread::Join()
if ( ! started )
return;
if ( ! terminating )
Stop();
assert(terminating);
DBG_LOG(DBG_THREADING, "Joining thread %s ...", name.c_str());
DBG_LOG(DBG_THREADING, "Joining thread %s ...", name);
if ( pthread_join(pthread, 0) != 0 )
reporter->FatalError("Failure joining thread %s", name.c_str());
if ( pthread && pthread_join(pthread, 0) != 0 )
reporter->FatalError("Failure joining thread %s", name);
pthread_mutex_destroy(&terminate);
DBG_LOG(DBG_THREADING, "Done with thread %s", name.c_str());
DBG_LOG(DBG_THREADING, "Joined with thread %s", name);
pthread = 0;
}
void BasicThread::Kill()
{
if ( ! (started && pthread) )
return;
// We don't *really* kill the thread here because that leads to race
// conditions. Instead we set a flag that parts of the the code need
// to check and get out of any loops they might be in.
terminating = true;
killed = true;
OnKill();
}
// I believe this is safe to call from a signal handler ... Not error
// checking so that killing doesn't bail out if we have already
// terminated.
pthread_kill(pthread, SIGKILL);
void BasicThread::Done()
{
DBG_LOG(DBG_THREADING, "Thread %s has finished", name);
terminating = true;
killed = true;
}
void* BasicThread::launcher(void *arg)
@ -173,15 +197,12 @@ void* BasicThread::launcher(void *arg)
sigdelset(&mask_set, SIGSEGV);
sigdelset(&mask_set, SIGBUS);
int res = pthread_sigmask(SIG_BLOCK, &mask_set, 0);
assert(res == 0); //
assert(res == 0);
// Run thread's main function.
thread->Run();
// Wait until somebody actually wants us to terminate.
if ( pthread_mutex_lock(&thread->terminate) != 0 )
reporter->FatalError("Failure acquiring terminate mutex at end of thread %s", thread->Name().c_str());
thread->Done();
return 0;
}

View file

@ -5,7 +5,6 @@
#include <pthread.h>
#include <semaphore.h>
#include "Queue.h"
#include "util.h"
using namespace std;
@ -42,22 +41,25 @@ public:
*
* This method is safe to call from any thread.
*/
const string& Name() const { return name; }
const char* Name() const { return name; }
/**
* Sets a descriptive name for the thread. This should be a string
* that's useful in output presented to the user and uniquely
* identifies the thread.
*
* This method must be called only from the thread itself.
* This method must be called only from main thread at initialization
* time.
*/
void SetName(const string& name);
void SetName(const char* name);
/**
* Set the name shown by the OS as the thread's description. Not
* supported on all OSs.
*
* Must be called only from the child thread.
*/
void SetOSName(const string& name);
void SetOSName(const char* name);
/**
* Starts the thread. Calling this methods will spawn a new OS thread
@ -68,6 +70,18 @@ public:
*/
void Start();
/**
* Signals the thread to prepare for stopping. This must be called
* before Stop() and allows the thread to trigger shutting down
* without yet blocking for doing so.
*
* Calling this method has no effect if Start() hasn't been executed
* yet.
*
* Only Bro's main thread must call this method.
*/
void PrepareStop();
/**
* Signals the thread to stop. The method lets Terminating() now
* return true. It does however not force the thread to terminate.
@ -88,6 +102,13 @@ public:
*/
bool Terminating() const { return terminating; }
/**
* Returns true if Kill() has been called.
*
* This method is safe to call from any thread.
*/
bool Killed() const { return killed; }
/**
* A version of fmt() that the thread can safely use.
*
@ -96,6 +117,14 @@ public:
*/
const char* Fmt(const char* format, ...);
/**
* A version of strerror() that the thread can safely use. This is
* essentially a wrapper around strerror_r(). Note that it keeps a
* single buffer per thread internally so the result remains valid
* only until the next call.
*/
const char* Strerror(int err);
protected:
friend class Manager;
@ -116,12 +145,24 @@ protected:
virtual void OnStart() {}
/**
* Executed with Stop(). This is a hook into stopping the thread. It
* will be called from Bro's main thread after the thread has been
* signaled to stop.
* Executed with PrepareStop() (and before OnStop()). This is a hook
* into preparing the thread for stopping. It will be called from
* Bro's main thread before the thread has been signaled to stop.
*/
virtual void OnPrepareStop() {}
/**
* Executed with Stop() (and after OnPrepareStop()). This is a hook
* into stopping the thread. It will be called from Bro's main thread
* after the thread has been signaled to stop.
*/
virtual void OnStop() {}
/**
* Executed with Kill(). This is a hook into killing the thread.
*/
virtual void OnKill() {}
/**
* Destructor. This will be called by the manager.
*
@ -145,14 +186,18 @@ protected:
*/
void Kill();
/** Called by child thread's launcher when it's done processing. */
void Done();
private:
// pthread entry function.
static void* launcher(void *arg);
string name;
const char* name;
pthread_t pthread;
bool started; // Set to to true once running.
bool terminating; // Set to to true to signal termination.
bool killed; // Set to true once forcefully killed.
// Used as a semaphore to tell the pthread thread when it may
// terminate.
@ -162,6 +207,9 @@ private:
char* buf;
unsigned int buf_len;
// For implementating Strerror().
char* strerr_buffer;
static uint64_t thread_counter;
};

View file

@ -30,6 +30,10 @@ void Manager::Terminate()
do Process(); while ( did_process );
// Signal all to stop.
for ( all_thread_list::iterator i = all_threads.begin(); i != all_threads.end(); i++ )
(*i)->PrepareStop();
for ( all_thread_list::iterator i = all_threads.begin(); i != all_threads.end(); i++ )
(*i)->Stop();
@ -48,24 +52,16 @@ void Manager::Terminate()
terminating = false;
}
void Manager::KillThreads()
{
DBG_LOG(DBG_THREADING, "Killing threads ...");
for ( all_thread_list::iterator i = all_threads.begin(); i != all_threads.end(); i++ )
(*i)->Kill();
}
void Manager::AddThread(BasicThread* thread)
{
DBG_LOG(DBG_THREADING, "Adding thread %s ...", thread->Name().c_str());
DBG_LOG(DBG_THREADING, "Adding thread %s ...", thread->Name());
all_threads.push_back(thread);
idle = false;
}
void Manager::AddMsgThread(MsgThread* thread)
{
DBG_LOG(DBG_THREADING, "%s is a MsgThread ...", thread->Name().c_str());
DBG_LOG(DBG_THREADING, "%s is a MsgThread ...", thread->Name());
msg_threads.push_back(thread);
}
@ -91,6 +87,14 @@ double Manager::NextTimestamp(double* network_time)
return -1.0;
}
void Manager::KillThreads()
{
DBG_LOG(DBG_THREADING, "Killing threads ...");
for ( all_thread_list::iterator i = all_threads.begin(); i != all_threads.end(); i++ )
(*i)->Kill();
}
void Manager::Process()
{
bool do_beat = false;
@ -114,6 +118,12 @@ void Manager::Process()
{
Message* msg = t->RetrieveOut();
if ( ! msg )
{
assert(t->Killed());
break;
}
if ( msg->Process() )
{
if ( network_time )
@ -122,10 +132,9 @@ void Manager::Process()
else
{
string s = msg->Name() + " failed, terminating thread";
reporter->Error("%s", s.c_str());
reporter->Error("%s failed, terminating thread", msg->Name());
t->Stop();
}
}
delete msg;
}

View file

@ -49,15 +49,6 @@ public:
*/
bool Terminating() const { return terminating; }
/**
* Immediately kills all child threads. It does however not yet join
* them, one still needs to call Terminate() for that.
*
* This method is safe to call from a signal handler, and can in fact
* be called while Terminate() is already in progress.
*/
void KillThreads();
typedef std::list<std::pair<string, MsgThread::Stats> > msg_stats_list;
/**
@ -115,6 +106,13 @@ protected:
*/
virtual double NextTimestamp(double* network_time);
/**
* Kills all thread immediately. Note that this may cause race conditions
* if a child thread currently holds a lock that might block somebody
* else.
*/
virtual void KillThreads();
/**
* Part of the IOSource interface.
*/

View file

@ -5,6 +5,7 @@
#include "Manager.h"
#include <unistd.h>
#include <signal.h>
using namespace threading;
@ -16,19 +17,17 @@ namespace threading {
class FinishMessage : public InputMessage<MsgThread>
{
public:
FinishMessage(MsgThread* thread) : InputMessage<MsgThread>("Finish", thread) { }
FinishMessage(MsgThread* thread, double network_time) : InputMessage<MsgThread>("Finish", thread),
network_time(network_time) { }
virtual bool Process() { return Object()->DoFinish(); }
};
virtual bool Process() {
bool result = Object()->OnFinish(network_time);
Object()->Finished();
return result;
}
// A dummy message that's only purpose is unblock the current read operation
// so that the child's Run() methods can check the termination status.
class UnblockMessage : public InputMessage<MsgThread>
{
public:
UnblockMessage(MsgThread* thread) : InputMessage<MsgThread>("Unblock", thread) { }
virtual bool Process() { return true; }
private:
double network_time;
};
/// Sends a heartbeat to the child thread.
@ -39,7 +38,10 @@ public:
: InputMessage<MsgThread>("Heartbeat", thread)
{ network_time = arg_network_time; current_time = arg_current_time; }
virtual bool Process() { return Object()->DoHeartbeat(network_time, current_time); }
virtual bool Process() {
Object()->HeartbeatInChild();
return Object()->OnHeartbeat(network_time, current_time);
}
private:
double network_time;
@ -55,14 +57,16 @@ public:
INTERNAL_WARNING, INTERNAL_ERROR
};
ReporterMessage(Type arg_type, MsgThread* thread, const string& arg_msg)
ReporterMessage(Type arg_type, MsgThread* thread, const char* arg_msg)
: OutputMessage<MsgThread>("ReporterMessage", thread)
{ type = arg_type; msg = arg_msg; }
{ type = arg_type; msg = copy_string(arg_msg); }
~ReporterMessage() { delete [] msg; }
virtual bool Process();
private:
string msg;
const char* msg;
Type type;
};
@ -71,18 +75,19 @@ private:
class DebugMessage : public OutputMessage<MsgThread>
{
public:
DebugMessage(DebugStream arg_stream, MsgThread* thread, const string& arg_msg)
DebugMessage(DebugStream arg_stream, MsgThread* thread, const char* arg_msg)
: OutputMessage<MsgThread>("DebugMessage", thread)
{ stream = arg_stream; msg = arg_msg; }
{ stream = arg_stream; msg = copy_string(arg_msg); }
virtual ~DebugMessage() { delete [] msg; }
virtual bool Process()
{
string s = Object()->Name() + ": " + msg;
debug_logger.Log(stream, "%s", s.c_str());
debug_logger.Log(stream, "%s: %s", Object()->Name(), msg);
return true;
}
private:
string msg;
const char* msg;
DebugStream stream;
};
#endif
@ -93,41 +98,39 @@ private:
Message::~Message()
{
delete [] name;
}
bool ReporterMessage::Process()
{
string s = Object()->Name() + ": " + msg;
const char* cmsg = s.c_str();
switch ( type ) {
case INFO:
reporter->Info("%s", cmsg);
reporter->Info("%s: %s", Object()->Name(), msg);
break;
case WARNING:
reporter->Warning("%s", cmsg);
reporter->Warning("%s: %s", Object()->Name(), msg);
break;
case ERROR:
reporter->Error("%s", cmsg);
reporter->Error("%s: %s", Object()->Name(), msg);
break;
case FATAL_ERROR:
reporter->FatalError("%s", cmsg);
reporter->FatalError("%s: %s", Object()->Name(), msg);
break;
case FATAL_ERROR_WITH_CORE:
reporter->FatalErrorWithCore("%s", cmsg);
reporter->FatalErrorWithCore("%s: %s", Object()->Name(), msg);
break;
case INTERNAL_WARNING:
reporter->InternalWarning("%s", cmsg);
reporter->InternalWarning("%s: %s", Object()->Name(), msg);
break;
case INTERNAL_ERROR :
reporter->InternalError("%s", cmsg);
reporter->InternalError("%s: %s", Object()->Name(), msg);
break;
default:
@ -137,32 +140,74 @@ bool ReporterMessage::Process()
return true;
}
MsgThread::MsgThread() : BasicThread()
MsgThread::MsgThread() : BasicThread(), queue_in(this, 0), queue_out(0, this)
{
cnt_sent_in = cnt_sent_out = 0;
finished = false;
thread_mgr->AddMsgThread(this);
}
// Set by Bro's main signal handler.
extern int signal_val;
void MsgThread::OnPrepareStop()
{
if ( finished || Killed() )
return;
// Signal thread to terminate and wait until it has acknowledged.
SendIn(new FinishMessage(this, network_time), true);
}
void MsgThread::OnStop()
{
// Signal thread to terminate and wait until it has acknowledged.
SendIn(new FinishMessage(this), true);
int signal_count = 0;
int old_signal_val = signal_val;
signal_val = 0;
int cnt = 0;
while ( ! finished )
uint64_t last_size = 0;
uint64_t cur_size = 0;
while ( ! (finished || Killed() ) )
{
if ( ++cnt > 1000 ) // Insurance against broken threads ...
// Terminate if we get another kill signal.
if ( signal_val == SIGTERM || signal_val == SIGINT )
{
reporter->Warning("thread %s didn't finish in time", Name().c_str());
break;
++signal_count;
if ( signal_count == 1 )
{
// Abort all threads here so that we won't hang next
// on another one.
fprintf(stderr, "received signal while waiting for thread %s, aborting all ...\n", Name());
thread_mgr->KillThreads();
}
else
{
// More than one signal. Abort processing
// right away. on another one.
fprintf(stderr, "received another signal while waiting for thread %s, aborting processing\n", Name());
exit(1);
}
signal_val = 0;
}
queue_in.WakeUp();
usleep(1000);
}
// One more message to make sure the current queue read operation unblocks.
SendIn(new UnblockMessage(this), true);
signal_val = old_signal_val;
}
void MsgThread::OnKill()
{
// Send a message to unblock the reader if its currently waiting for
// input. This is just an optimization to make it terminate more
// quickly, even without the message it will eventually time out.
queue_in.WakeUp();
}
void MsgThread::Heartbeat()
@ -170,25 +215,20 @@ void MsgThread::Heartbeat()
SendIn(new HeartbeatMessage(this, network_time, current_time()));
}
bool MsgThread::DoHeartbeat(double network_time, double current_time)
void MsgThread::HeartbeatInChild()
{
string n = Name();
n = Fmt("bro: %s (%" PRIu64 "/%" PRIu64 ")", n.c_str(),
string n = Fmt("bro: %s (%" PRIu64 "/%" PRIu64 ")", Name(),
cnt_sent_in - queue_in.Size(),
cnt_sent_out - queue_out.Size());
SetOSName(n.c_str());
return true;
}
bool MsgThread::DoFinish()
void MsgThread::Finished()
{
// This is thread-safe "enough", we're the only one ever writing
// there.
finished = true;
return true;
}
void MsgThread::Info(const char* msg)
@ -245,7 +285,7 @@ void MsgThread::SendIn(BasicInputMessage* msg, bool force)
return;
}
DBG_LOG(DBG_THREADING, "Sending '%s' to %s ...", msg->Name().c_str(), Name().c_str());
DBG_LOG(DBG_THREADING, "Sending '%s' to %s ...", msg->Name(), Name());
queue_in.Put(msg);
++cnt_sent_in;
@ -268,9 +308,10 @@ void MsgThread::SendOut(BasicOutputMessage* msg, bool force)
BasicOutputMessage* MsgThread::RetrieveOut()
{
BasicOutputMessage* msg = queue_out.Get();
assert(msg);
if ( ! msg )
return 0;
DBG_LOG(DBG_THREADING, "Retrieved '%s' from %s", msg->Name().c_str(), Name().c_str());
DBG_LOG(DBG_THREADING, "Retrieved '%s' from %s", msg->Name(), Name());
return msg;
}
@ -278,10 +319,12 @@ BasicOutputMessage* MsgThread::RetrieveOut()
BasicInputMessage* MsgThread::RetrieveIn()
{
BasicInputMessage* msg = queue_in.Get();
assert(msg);
if ( ! msg )
return 0;
#ifdef DEBUG
string s = Fmt("Retrieved '%s' in %s", msg->Name().c_str(), Name().c_str());
string s = Fmt("Retrieved '%s' in %s", msg->Name(), Name());
Debug(DBG_THREADING, s.c_str());
#endif
@ -290,27 +333,33 @@ BasicInputMessage* MsgThread::RetrieveIn()
void MsgThread::Run()
{
while ( true )
while ( ! (finished || Killed() ) )
{
// When requested to terminate, we only do so when
// all input has been processed.
if ( Terminating() && ! queue_in.Ready() )
break;
BasicInputMessage* msg = RetrieveIn();
if ( ! msg )
continue;
bool result = msg->Process();
if ( ! result )
{
string s = msg->Name() + " failed, terminating thread (MsgThread)";
string s = Fmt("%s failed, terminating thread (MsgThread)", Name());
Error(s.c_str());
Stop();
break;
}
delete msg;
}
// In case we haven't send the finish method yet, do it now. Reading
// global network_time here should be fine, it isn't changing
// anymore.
if ( ! finished )
{
OnFinish(network_time);
Finished();
}
}
void MsgThread::GetStats(Stats* stats)

View file

@ -189,39 +189,42 @@ protected:
*
* This is method is called regularly by the threading::Manager.
*
* Can be overriden in derived classed to hook into the heart beat,
* but must call the parent implementation. Note that this method is
* always called by the main thread and must not access data of the
* child thread directly. See DoHeartbeat() if you want to do
* something on the child-side.
* Can be overriden in derived classed to hook into the heart beat
* sending, but must call the parent implementation. Note that this
* method is always called by the main thread and must not access
* data of the child thread directly. Implement OnHeartbeat() if you
* want to do something on the child-side.
*/
virtual void Heartbeat();
/**
* Overriden from BasicThread.
*
/** Internal heartbeat processing. Called from child.
*/
virtual void Run();
virtual void OnStop();
void HeartbeatInChild();
/**
* Regulatly triggered for execution in the child thread.
*
* When overriding, one must call the parent class' implementation.
*
* network_time: The network_time when the heartbeat was trigger by
* the main thread.
*
* current_time: Wall clock when the heartbeat was trigger by the
* main thread.
*/
virtual bool DoHeartbeat(double network_time, double current_time);
virtual bool OnHeartbeat(double network_time, double current_time) = 0;
/** Triggered for execution in the child thread just before shutting threads down.
* The child thread should finish its operations and then *must*
* call this class' implementation.
* The child thread should finish its operations.
*/
virtual bool DoFinish();
virtual bool OnFinish(double network_time) = 0;
/**
* Overriden from BasicThread.
*
*/
virtual void Run();
virtual void OnStop();
virtual void OnPrepareStop();
virtual void OnKill();
private:
/**
@ -280,6 +283,10 @@ private:
*/
bool MightHaveOut() { return queue_out.MaybeReady(); }
/** Flags that the child process has finished processing. Called from child.
*/
void Finished();
Queue<BasicInputMessage *> queue_in;
Queue<BasicOutputMessage *> queue_out;
@ -305,7 +312,7 @@ public:
* what's passed into the constructor and used mainly for debugging
* purposes.
*/
const string& Name() const { return name; }
const char* Name() const { return name; }
/**
* Callback that must be overriden for processing a message.
@ -319,10 +326,11 @@ protected:
* @param arg_name A descriptive name for the type of message. Used
* mainly for debugging purposes.
*/
Message(const string& arg_name) { name = arg_name; }
Message(const char* arg_name)
{ name = copy_string(arg_name); }
private:
string name;
const char* name;
};
/**
@ -337,7 +345,7 @@ protected:
* @param name A descriptive name for the type of message. Used
* mainly for debugging purposes.
*/
BasicInputMessage(const string& name) : Message(name) {}
BasicInputMessage(const char* name) : Message(name) {}
};
/**
@ -352,7 +360,7 @@ protected:
* @param name A descriptive name for the type of message. Used
* mainly for debugging purposes.
*/
BasicOutputMessage(const string& name) : Message(name) {}
BasicOutputMessage(const char* name) : Message(name) {}
};
/**
@ -377,7 +385,7 @@ protected:
*
* @param arg_object: An object to store with the message.
*/
InputMessage(const string& name, O* arg_object) : BasicInputMessage(name)
InputMessage(const char* name, O* arg_object) : BasicInputMessage(name)
{ object = arg_object; }
private:
@ -406,7 +414,7 @@ protected:
*
* @param arg_object An object to store with the message.
*/
OutputMessage(const string& name, O* arg_object) : BasicOutputMessage(name)
OutputMessage(const char* name, O* arg_object) : BasicOutputMessage(name)
{ object = arg_object; }
private:

View file

@ -1,4 +1,3 @@
#ifndef THREADING_QUEUE_H
#define THREADING_QUEUE_H
@ -6,8 +5,10 @@
#include <queue>
#include <deque>
#include <stdint.h>
#include <sys/time.h>
#include "Reporter.h"
#include "BasicThread.h"
#undef Queue // Defined elsewhere unfortunately.
@ -30,8 +31,12 @@ class Queue
public:
/**
* Constructor.
*
* reader, writer: The corresponding threads. This is for checking
* whether they have terminated so that we can abort I/O opeations.
* Can be left null for the main thread.
*/
Queue();
Queue(BasicThread* arg_reader, BasicThread* arg_writer);
/**
* Destructor.
@ -39,7 +44,9 @@ public:
~Queue();
/**
* Retrieves one elment.
* Retrieves one element. This may block for a little while of no
* input is available and eventually return with a null element if
* nothing shows up.
*/
T Get();
@ -60,6 +67,11 @@ public:
*/
bool MaybeReady() { return ( ( read_ptr - write_ptr) != 0 ); }
/** Wake up the reader if it's currently blocked for input. This is
primarily to give it a chance to check termination quickly.
**/
void WakeUp();
/**
* Returns the number of queued items not yet retrieved.
*/
@ -91,6 +103,9 @@ private:
int read_ptr; // Where the next operation will read from
int write_ptr; // Where the next operation will write to
BasicThread* reader;
BasicThread* writer;
// Statistics.
uint64_t num_reads;
uint64_t num_writes;
@ -109,18 +124,20 @@ inline static void safe_unlock(pthread_mutex_t* mutex)
}
template<typename T>
inline Queue<T>::Queue()
inline Queue<T>::Queue(BasicThread* arg_reader, BasicThread* arg_writer)
{
read_ptr = 0;
write_ptr = 0;
num_reads = num_writes = 0;
reader = arg_reader;
writer = arg_writer;
for( int i = 0; i < NUM_QUEUES; ++i )
{
if ( pthread_cond_init(&has_data[i], NULL) != 0 )
if ( pthread_cond_init(&has_data[i], 0) != 0 )
reporter->FatalError("cannot init queue condition variable");
if ( pthread_mutex_init(&mutex[i], NULL) != 0 )
if ( pthread_mutex_init(&mutex[i], 0) != 0 )
reporter->FatalError("cannot init queue mutex");
}
}
@ -138,12 +155,23 @@ inline Queue<T>::~Queue()
template<typename T>
inline T Queue<T>::Get()
{
if ( (reader && reader->Killed()) || (writer && writer->Killed()) )
return 0;
safe_lock(&mutex[read_ptr]);
int old_read_ptr = read_ptr;
if ( messages[read_ptr].empty() )
pthread_cond_wait(&has_data[read_ptr], &mutex[read_ptr]);
{
struct timespec ts;
ts.tv_sec = time(0) + 5;
ts.tv_nsec = 0;
pthread_cond_timedwait(&has_data[read_ptr], &mutex[read_ptr], &ts);
safe_unlock(&mutex[read_ptr]);
return 0;
}
T data = messages[read_ptr].front();
messages[read_ptr].pop();
@ -222,6 +250,17 @@ inline void Queue<T>::GetStats(Stats* stats)
safe_unlock(&mutex[i]);
}
template<typename T>
inline void Queue<T>::WakeUp()
{
for ( int i = 0; i < NUM_QUEUES; i++ )
{
safe_lock(&mutex[i]);
pthread_cond_signal(&has_data[i]);
safe_unlock(&mutex[i]);
}
}
}

View file

@ -11,23 +11,54 @@ bool Field::Read(SerializationFormat* fmt)
{
int t;
int st;
string tmp_name;
bool have_2nd;
bool success = (fmt->Read(&name, "name")
&& fmt->Read(&secondary_name, "secondary_name")
if ( ! fmt->Read(&have_2nd, "have_2nd") )
return false;
if ( have_2nd )
{
string tmp_secondary_name;
if ( ! fmt->Read(&tmp_secondary_name, "secondary_name") )
return false;
secondary_name = copy_string(tmp_secondary_name.c_str());
}
else
secondary_name = 0;
bool success = (fmt->Read(&tmp_name, "name")
&& fmt->Read(&t, "type")
&& fmt->Read(&st, "subtype")
&& fmt->Read(&optional, "optional"));
if ( ! success )
return false;
name = copy_string(tmp_name.c_str());
type = (TypeTag) t;
subtype = (TypeTag) st;
return success;
return true;
}
bool Field::Write(SerializationFormat* fmt) const
{
assert(name);
if ( secondary_name )
{
if ( ! (fmt->Write(true, "have_2nd")
&& fmt->Write(secondary_name, "secondary_name")) )
return false;
}
else
if ( ! fmt->Write(false, "have_2nd") )
return false;
return (fmt->Write(name, "name")
&& fmt->Write(secondary_name, "secondary_name")
&& fmt->Write((int)type, "type")
&& fmt->Write((int)subtype, "subtype"),
fmt->Write(optional, "optional"));
@ -51,7 +82,7 @@ Value::~Value()
{
if ( (type == TYPE_ENUM || type == TYPE_STRING || type == TYPE_FILE || type == TYPE_FUNC)
&& present )
delete val.string_val;
delete [] val.string_val.data;
if ( type == TYPE_TABLE && present )
{
@ -224,10 +255,7 @@ bool Value::Read(SerializationFormat* fmt)
case TYPE_STRING:
case TYPE_FILE:
case TYPE_FUNC:
{
val.string_val = new string;
return fmt->Read(val.string_val, "string");
}
return fmt->Read(&val.string_val.data, &val.string_val.length, "string");
case TYPE_TABLE:
{
@ -339,7 +367,7 @@ bool Value::Write(SerializationFormat* fmt) const
case TYPE_STRING:
case TYPE_FILE:
case TYPE_FUNC:
return fmt->Write(*val.string_val, "string");
return fmt->Write(val.string_val.data, val.string_val.length, "string");
case TYPE_TABLE:
{

View file

@ -12,6 +12,7 @@
using namespace std;
class SerializationFormat;
class RemoteSerializer;
namespace threading {
@ -19,10 +20,10 @@ namespace threading {
* Definition of a log file, i.e., one column of a log stream.
*/
struct Field {
string name; //! Name of the field.
const char* name; //! Name of the field.
//! Needed by input framework. Port fields have two names (one for the
//! port, one for the type), and this specifies the secondary name.
string secondary_name;
const char* secondary_name;
TypeTag type; //! Type of the field.
TypeTag subtype; //! Inner type for sets.
bool optional; //! True if field is optional.
@ -30,13 +31,24 @@ struct Field {
/**
* Constructor.
*/
Field() { subtype = TYPE_VOID; optional = false; }
Field(const char* name, const char* secondary_name, TypeTag type, TypeTag subtype, bool optional)
: name(name ? copy_string(name) : 0),
secondary_name(secondary_name ? copy_string(secondary_name) : 0),
type(type), subtype(subtype), optional(optional) { }
/**
* Copy constructor.
*/
Field(const Field& other)
: name(other.name), type(other.type), subtype(other.subtype), optional(other.optional) { }
: name(other.name ? copy_string(other.name) : 0),
secondary_name(other.secondary_name ? copy_string(other.secondary_name) : 0),
type(other.type), subtype(other.subtype), optional(other.optional) { }
~Field()
{
delete [] name;
delete [] secondary_name;
}
/**
* Unserializes a field.
@ -63,6 +75,12 @@ struct Field {
* thread-safe.
*/
string TypeName() const;
private:
friend class ::RemoteSerializer;
// Force usage of constructor above.
Field() {};
};
/**
@ -102,7 +120,11 @@ struct Value {
vec_t vector_val;
addr_t addr_val;
subnet_t subnet_val;
string* string_val;
struct {
char* data;
int length;
} string_val;
} val;
/**
@ -147,7 +169,7 @@ struct Value {
static bool IsCompatibleType(BroType* t, bool atomic_only=false);
private:
friend class ::IPAddr;
friend class ::IPAddr;
Value(const Value& other) { } // Disabled.
};