Improved signal handling.

Sending SIGTERM triggers a normal shutdown of all threads that waits
until they have processed their remaining data. However, sending a 2nd
SIGTERM while waiting for them to finish will immediately kill them
all.

src/main.cc

@@ -333,6 +333,13 @@ RETSIGTYPE sig_handler(int signo)
 	{
 	set_processing_status("TERMINATING", "sig_handler");
 	signal_val = signo;
+
+	if ( thread_mgr->Terminating() && (signal_val == SIGTERM || signal_val == SIGINT) )
+		// If the thread manager is already terminating (i.e.,
+		// waiting for child threads to exit), another term signal
+		// will send the threads a kill.
+		thread_mgr->KillThreads();
+
 	return RETSIGVAL;
 	}
 

src/threading/BasicThread.cc

@@ -113,6 +113,17 @@ void BasicThread::Join()
 	pthread = 0;
 	}
 
+void BasicThread::Kill()
+	{
+	if ( ! (started && pthread) )
+		return;
+
+	// I believe this is safe to call from a signal handler ... Not error
+	// checking so that killing doesn't bail out if we have already
+	// terminated.
+	pthread_kill(pthread, SIGKILL);
+	}
+
 void* BasicThread::launcher(void *arg)
 	{
 	BasicThread* thread = (BasicThread *)arg;

src/threading/BasicThread.h

@@ -121,6 +121,15 @@ protected:
 	 */
 	void Join();
 
+	/**
+	 * Kills the thread immediately. One still needs to call Join()
+	 * afterwards.
+	 *
+	 * This is called from the threading::Manager and safe to execute
+	 * during a signal handler.
+	 */
+	void Kill();
+
 private:
 	// pthread entry function.
 	static void* launcher(void *arg);

src/threading/Manager.cc

@@ -9,6 +9,7 @@ Manager::Manager()
 
 	did_process = false;
 	next_beat = 0;
+	terminating = false;
 	}
 
 Manager::~Manager()
@@ -21,6 +22,8 @@ void Manager::Terminate()
 	{
 	DBG_LOG(DBG_THREADING, "Terminating thread manager ...");
 
+	terminating = true;
+
 	// First process remaining thread output for the message threads.
 	do Process(); while ( did_process );
 
@@ -37,6 +40,16 @@ void Manager::Terminate()
 
 	all_threads.clear();
 	msg_threads.clear();
+
+	terminating = false;
+	}
+
+void Manager::KillThreads()
+	{
+	DBG_LOG(DBG_THREADING, "Killing threads ...");
+
+	for ( all_thread_list::iterator i = all_threads.begin(); i != all_threads.end(); i++ )
+		(*i)->Kill();
 	}
 
 void Manager::AddThread(BasicThread* thread)

src/threading/Manager.h

@@ -43,6 +43,21 @@ public:
 	 */
 	void Terminate();
 
+	/**
+	 * Returns True if we are currently in Terminate() waiting for 
+	 * threads to exit.
+	 */
+	bool Terminating() const	{ return terminating; }
+
+	/**
+	 * Immediately kills all child threads. It does however not yet join
+	 * them, one still needs to call Terminate() for that.
+	 *
+	 * This method is safe to call from a signal handler, and can in fact
+	 * be called while Terminate() is already in progress.
+	 */
+	void KillThreads();
+
 	typedef std::list<std::pair<string, MsgThread::Stats> > msg_stats_list;
 
 	/**
@@ -115,6 +130,7 @@ private:
 
 	bool did_process;	// True if the last Process() found some work to do.
 	double next_beat;	// Timestamp when the next heartbeat will be sent.
+	bool terminating;	// True if we are in Terminate().
 
 	msg_stats_list stats;
 };