Reworking forceful thread termination.

Ctrl-C now kills a thread even if it hangs at termination. And readded
a (rather long) timeout to kill threads automatically that don't
shutdown.
This commit is contained in:
Robin Sommer 2012-07-17 19:36:30 -07:00
parent e90918aa50
commit 490859cfef
6 changed files with 50 additions and 12 deletions

View file

@ -162,9 +162,7 @@ bool WriterBackend::Init(const WriterInfo& arg_info, int arg_num_fields, const F
num_fields = arg_num_fields; num_fields = arg_num_fields;
fields = arg_fields; fields = arg_fields;
string name = Fmt("%s/%s", info.path.c_str(), frontend_name.c_str()); SetName(frontend->Name());
SetName(name);
if ( ! DoInit(arg_info, arg_num_fields, arg_fields) ) if ( ! DoInit(arg_info, arg_num_fields, arg_fields) )
{ {

View file

@ -169,6 +169,7 @@ bool Ascii::DoFinish(double network_time)
ascii_done = true; ascii_done = true;
CloseFile(network_time); CloseFile(network_time);
return true; return true;
} }

View file

@ -125,7 +125,7 @@ void BasicThread::Join()
DBG_LOG(DBG_THREADING, "Joining thread %s ...", name.c_str()); DBG_LOG(DBG_THREADING, "Joining thread %s ...", name.c_str());
if ( pthread_join(pthread, 0) != 0 ) if ( pthread && pthread_join(pthread, 0) != 0 )
reporter->FatalError("Failure joining thread %s", name.c_str()); reporter->FatalError("Failure joining thread %s", name.c_str());
DBG_LOG(DBG_THREADING, "Done with thread %s", name.c_str()); DBG_LOG(DBG_THREADING, "Done with thread %s", name.c_str());
@ -135,13 +135,13 @@ void BasicThread::Join()
void BasicThread::Kill() void BasicThread::Kill()
{ {
terminating = true;
if ( ! (started && pthread) ) if ( ! (started && pthread) )
return; return;
// I believe this is safe to call from a signal handler ... Not error pthread = 0;
// checking so that killing doesn't bail out if we have already pthread_kill(pthread, SIGTERM);
// terminated.
pthread_kill(pthread, SIGKILL);
} }
void* BasicThread::launcher(void *arg) void* BasicThread::launcher(void *arg)

View file

@ -83,6 +83,14 @@ double Manager::NextTimestamp(double* network_time)
return -1.0; return -1.0;
} }
void Manager::KillThreads()
{
DBG_LOG(DBG_THREADING, "Killing threads ...");
for ( all_thread_list::iterator i = all_threads.begin(); i != all_threads.end(); i++ )
(*i)->Kill();
}
void Manager::Process() void Manager::Process()
{ {
bool do_beat = false; bool do_beat = false;

View file

@ -106,6 +106,13 @@ protected:
*/ */
virtual double NextTimestamp(double* network_time); virtual double NextTimestamp(double* network_time);
/**
* Kills all thread immediately. Note that this may cause race conditions
* if a child thread currently holds a lock that might block somebody
* else.
*/
virtual void KillThreads();
/** /**
* Part of the IOSource interface. * Part of the IOSource interface.
*/ */

View file

@ -156,6 +156,9 @@ MsgThread::MsgThread() : BasicThread()
thread_mgr->AddMsgThread(this); thread_mgr->AddMsgThread(this);
} }
// Set by Bro's main signal handler.
extern int signal_val;
void MsgThread::OnStop() void MsgThread::OnStop()
{ {
if ( stopped ) if ( stopped )
@ -164,13 +167,31 @@ void MsgThread::OnStop()
// Signal thread to terminate and wait until it has acknowledged. // Signal thread to terminate and wait until it has acknowledged.
SendIn(new FinishMessage(this, network_time), true); SendIn(new FinishMessage(this, network_time), true);
int old_signal_val = signal_val;
signal_val = 0;
int cnt = 0; int cnt = 0;
bool aborted = 0;
while ( ! finished ) while ( ! finished )
{ {
if ( ++cnt % 2000 == 0 ) // Insurance against broken threads ... // Terminate if we get another kill signal.
if ( signal_val == SIGTERM || signal_val == SIGINT )
{ {
reporter->Warning("thread %s has not yet terminated ...", Name().c_str()); // Abort all threads here so that we won't hang next
fprintf(stderr, "warning: thread %s has not yet terminated ...", Name().c_str()); // on another one.
fprintf(stderr, "received signal while waiting for thread %s, aborting all ...\n", Name().c_str());
thread_mgr->KillThreads();
aborted = true;
break;
}
if ( ++cnt % 10000 == 0 ) // Insurance against broken threads ...
{
fprintf(stderr, "killing thread %s ...\n", Name().c_str());
Kill();
aborted = true;
break;
} }
usleep(1000); usleep(1000);
@ -178,8 +199,11 @@ void MsgThread::OnStop()
Finished(); Finished();
signal_val = old_signal_val;
// One more message to make sure the current queue read operation unblocks. // One more message to make sure the current queue read operation unblocks.
SendIn(new UnblockMessage(this), true); if ( ! aborted )
SendIn(new UnblockMessage(this), true);
} }
void MsgThread::Heartbeat() void MsgThread::Heartbeat()