Reworking thread termination logic.

Turns out the finish methods weren't called correctly, caused by a
mess up with method names which all sounded too similar and the wrong
one ended up being called. I've reworked this by changing the
thread/writer/reader interfaces, which actually also simplifies them
by getting rid of the requirement for writer backends to call their
parent methods (i.e., less opportunity for errors).

This commit also includes the following (because I noticed the problem
above when working on some of these):

     - The ASCII log writer now includes "#start <timestamp>" and
      "#end <timestamp> lines in the each file. The latter supersedes
      Bernhard's "EOF" patch.

      This required a number of tests updates. The standard canonifier
      removes the timestamps, but some tests compare files directly,
      which doesn't work if they aren't printing out the same
      timestamps (like the comm tests).

     - The above required yet another change to the writer API to
       network_time to methods.

     - Renamed ASCII logger "header" options to "meta".

     - Fixes #763 "Escape # when first character in log file line".

All btests pass for me on Linux FC15. Will try MacOS next.

src/threading/BasicThread.cc

@@ -78,24 +78,22 @@ const char* BasicThread::Fmt(const char* format, ...)
 	return buf;
 	}
 
+const char* BasicThread::Strerror(int err)
+	{
+	static char buf[128] = "<not set>";
+	strerror_r(err, buf, sizeof(buf));
+	return buf;
+	}
+
 void BasicThread::Start()
 	{
 
 	if ( started )
 		return;
 
-	int err = pthread_mutex_init(&terminate, 0);
-	if ( err != 0  )
-		reporter->FatalError("Cannot create terminate mutex for thread %s: %s", name.c_str(), strerror(err));
-
-	// We use this like a binary semaphore and acquire it immediately.
-	err = pthread_mutex_lock(&terminate);
+	int err = pthread_create(&pthread, 0, BasicThread::launcher, this);
 	if ( err != 0 )
-		reporter->FatalError("Cannot aquire terminate mutex for thread %s: %s", name.c_str(), strerror(err));
-
-	err = pthread_create(&pthread, 0, BasicThread::launcher, this);
-	if ( err != 0 )
-		reporter->FatalError("Cannot create thread %s:%s", name.c_str(), strerror(err));
+		reporter->FatalError("Cannot create thread %s:%s", name.c_str(), Strerror(err));
 
 	DBG_LOG(DBG_THREADING, "Started thread %s", name.c_str());
 
@@ -114,12 +112,6 @@ void BasicThread::Stop()
 
 	DBG_LOG(DBG_THREADING, "Signaling thread %s to terminate ...", name.c_str());
 
-	// Signal that it's ok for the thread to exit now by unlocking the
-	// mutex.
-	int err = pthread_mutex_unlock(&terminate);
-	if ( err != 0 )
-		reporter->FatalError("Failure flagging terminate condition for thread %s: %s", name.c_str(), strerror(err));
-
 	terminating = true;
 
 	OnStop();
@@ -130,16 +122,13 @@ void BasicThread::Join()
 	if ( ! started )
 		return;
 
-	if ( ! terminating )
-		Stop();
+	assert(terminating);
 
 	DBG_LOG(DBG_THREADING, "Joining thread %s ...", name.c_str());
 
 	if ( pthread_join(pthread, 0) != 0  )
 		reporter->FatalError("Failure joining thread %s", name.c_str());
 
-	pthread_mutex_destroy(&terminate);
-
 	DBG_LOG(DBG_THREADING, "Done with thread %s", name.c_str());
 
 	pthread = 0;
@@ -178,10 +167,6 @@ void* BasicThread::launcher(void *arg)
 	// Run thread's main function.
 	thread->Run();
 
-	// Wait until somebody actually wants us to terminate.
-	if ( pthread_mutex_lock(&thread->terminate) != 0 )
-		reporter->FatalError("Failure acquiring terminate mutex at end of thread %s", thread->Name().c_str());
-
 	return 0;
 	}
 

src/threading/BasicThread.h

@@ -96,6 +96,14 @@ public:
 	 */
 	const char* Fmt(const char* format, ...);
 
+	/**
+	 * A version of strerror() that the thread can safely use. This is
+	 * essentially a wrapper around strerror_r(). Note that it keeps a
+	 * single static buffer internally so the result remains valid only
+	 * until the next call.
+	 */
+	const char* Strerror(int err);
+
 protected:
 	friend class Manager;
 

src/threading/MsgThread.cc

@@ -16,9 +16,17 @@ namespace threading  {
 class FinishMessage : public InputMessage<MsgThread>
 {
 public:
-	FinishMessage(MsgThread* thread) : InputMessage<MsgThread>("Finish", thread)	{ }
+	FinishMessage(MsgThread* thread, double network_time) : InputMessage<MsgThread>("Finish", thread),
+		network_time(network_time) { }
 
-	virtual bool Process()	{ return Object()->DoFinish(); }
+	virtual bool Process()	{
+		bool result = Object()->OnFinish(network_time);
+		Object()->Finished();
+		return result;
+	}
+
+private:
+	double network_time;
 };
 
 // A dummy message that's only purpose is unblock the current read operation
@@ -39,7 +47,10 @@ public:
 		: InputMessage<MsgThread>("Heartbeat", thread)
 		{ network_time = arg_network_time; current_time = arg_current_time; }
 
-	virtual bool Process()	{ return Object()->DoHeartbeat(network_time, current_time); }
+	virtual bool Process()	{
+		Object()->HeartbeatInChild();
+		return Object()->OnHeartbeat(network_time, current_time);
+	}
 
 private:
 	double network_time;
@@ -146,8 +157,11 @@ MsgThread::MsgThread() : BasicThread()
 
 void MsgThread::OnStop()
 	{
+	if ( finished )
+		return;
+
 	// Signal thread to terminate and wait until it has acknowledged.
-	SendIn(new FinishMessage(this), true);
+	SendIn(new FinishMessage(this, network_time), true);
 
 	int cnt = 0;
 	while ( ! finished )
@@ -161,6 +175,8 @@ void MsgThread::OnStop()
 		usleep(1000);
 		}
 
+	Finished();
+
 	// One more message to make sure the current queue read operation unblocks.
 	SendIn(new UnblockMessage(this), true);
 	}
@@ -170,7 +186,7 @@ void MsgThread::Heartbeat()
 	SendIn(new HeartbeatMessage(this, network_time, current_time()));
 	}
 
-bool MsgThread::DoHeartbeat(double network_time, double current_time)
+void MsgThread::HeartbeatInChild()
 	{
 	string n = Name();
 
@@ -179,16 +195,13 @@ bool MsgThread::DoHeartbeat(double network_time, double current_time)
 		cnt_sent_out - queue_out.Size());
 
 	SetOSName(n.c_str());
-
-	return true;
 	}
 
-bool MsgThread::DoFinish()
+void MsgThread::Finished()
 	{
 	// This is thread-safe "enough", we're the only one ever writing
 	// there.
 	finished = true;
-	return true;
 	}
 
 void MsgThread::Info(const char* msg)

src/threading/MsgThread.h

@@ -189,39 +189,45 @@ protected:
 	 *
 	 * This is method is called regularly by the threading::Manager.
 	 *
-	 * Can be overriden in derived classed to hook into the heart beat,
-	 * but must call the parent implementation. Note that this method is
-	 * always called by the main thread and must not access data of the
-	 * child thread directly. See DoHeartbeat() if you want to do
-	 * something on the child-side.
+	 * Can be overriden in derived classed to hook into the heart beat
+	 * sending, but must call the parent implementation. Note that this
+	 * method is always called by the main thread and must not access
+	 * data of the child thread directly. Implement OnHeartbeat() if you
+	 * want to do something on the child-side.
 	 */
 	virtual void Heartbeat();
 
-	/**
-	 * Overriden from BasicThread.
-	 *
+	/** Flags that the child process has finished processing. Called from child.
 	 */
-	virtual void Run();
-	virtual void OnStop();
+	void Finished();
+
+	/** Internal heartbeat processing. Called from child.
+	 */
+	void HeartbeatInChild();
 
 	/**
 	 * Regulatly triggered for execution in the child thread.
 	 *
-	 * When overriding, one must call the parent class' implementation.
-	 *
 	 * network_time: The network_time when the heartbeat was trigger by
 	 * the main thread.
 	 *
 	 * current_time: Wall clock when the heartbeat was trigger by the
 	 * main thread.
 	 */
-	virtual bool DoHeartbeat(double network_time, double current_time);
+	virtual bool OnHeartbeat(double network_time, double current_time) = 0;
 
 	/** Triggered for execution in the child thread just before shutting threads down.
 	 *  The child thread should finish its operations and then *must*
 	 *  call this class' implementation.
 	 */
-	virtual bool DoFinish();
+	virtual bool OnFinish(double network_time) = 0;
+
+	/**
+	 * Overriden from BasicThread.
+	 *
+	 */
+	virtual void Run();
+	virtual void OnStop();
 
 private:
 	/**