Merge branch 'master' into topic/jsiwek/autodoc-fixes

Conflicts:
	scripts/CMakeLists.txt
	scripts/base/frameworks/cluster/setup-connections.bro
	scripts/base/frameworks/communication/__load__.bro
	scripts/base/frameworks/metrics/conn-example.bro
	scripts/base/frameworks/metrics/http-example.bro
	scripts/site/local.bro

src/Expr.cc

@@ -231,7 +231,6 @@ bool Expr::DoUnserialize(UnserialInfo* info)
 NameExpr::NameExpr(ID* arg_id) : Expr(EXPR_NAME)
 	{
 	id = arg_id;
-	ReferenceID();
 	SetType(id->Type()->Ref());
 
 	EventHandler* h = event_registry->Lookup(id->Name());
@@ -244,29 +243,6 @@ NameExpr::~NameExpr()
 	Unref(id);
 	}
 
-void NameExpr::ReferenceID()
-	{
-	// ### This is a hack. We check whether one of the remote serializer's
-	// built-in functions is referenced. If so, we activate the serializer.
-	// A better solution would be to either (1) a generic mechanism in
-	// which have (internal) attributes associated with identifiers and
-	// as we see references to the identifiers, we do bookkeeping
-	// associated with their attribute (so in this case the attribute
-	// would be "flag that inter-Bro communication is being used"),
-	// or (2) after the parse is done, we'd query whether these
-	// particular identifiers were seen, rather than doing the test
-	// here for every NameExpr we create.
-	if ( id->Type()->Tag() == TYPE_FUNC )
-		{
-		const char* const* builtins = remote_serializer->GetBuiltins();
-		while( *builtins )
-			{
-			if ( streq(id->Name(), *builtins++) )
-				using_communication = true;
-			}
-		}
-	}
-
 Expr* NameExpr::Simplify(SimplifyType simp_type)
 	{
 	if ( simp_type != SIMPLIFY_LHS && id->IsConst() )
@@ -393,8 +369,6 @@ bool NameExpr::DoUnserialize(UnserialInfo* info)
 	if ( ! id )
 		return false;
 
-	ReferenceID();
-
 	return true;
 	}
 

src/Expr.h

@@ -217,7 +217,6 @@ protected:
 	friend class Expr;
 	NameExpr()	{ id = 0; }
 
-	void ReferenceID();
 	void ExprDescribe(ODesc* d) const;
 
 	DECLARE_SERIAL(NameExpr);

src/LogMgr.cc

@@ -433,6 +433,25 @@ LogMgr::Stream* LogMgr::FindStream(EnumVal* id)
 	return streams[idx];
 	}
 
+LogMgr::WriterInfo* LogMgr::FindWriter(LogWriter* writer)
+	{
+	for ( vector<Stream *>::iterator s = streams.begin(); s != streams.end(); ++s )
+		{
+		if ( ! *s )
+			continue;
+
+		for ( Stream::WriterMap::iterator i = (*s)->writers.begin(); i != (*s)->writers.end(); i++ )
+			{
+			WriterInfo* winfo = i->second;
+
+			if ( winfo->writer == writer )
+				return winfo;
+			}
+		}
+
+	return 0;
+	}
+
 void LogMgr::RemoveDisabledWriters(Stream* stream)
 	{
 	list<Stream::WriterPathPair> disabled;
@@ -1411,6 +1430,8 @@ void LogMgr::InstallRotationTimer(WriterInfo* winfo)
 	RecordVal* rc =
 		LookupRotationControl(winfo->type, winfo->writer->Path());
 
+	assert(rc);
+
 	int idx = rc->Type()->AsRecordType()->FieldOffset("interv");
 	double rotation_interval = rc->LookupWithDefault(idx)->AsInterval();
 
@@ -1448,34 +1469,63 @@ void LogMgr::Rotate(WriterInfo* winfo)
 	DBG_LOG(DBG_LOGGING, "Rotating %s at %.6f",
 		winfo->writer->Path().c_str(), network_time);
 
-	// Create the RotationInfo record.
-	RecordVal* info = new RecordVal(BifType::Record::Log::RotationInfo);
-	info->Assign(0, winfo->type->Ref());
-	info->Assign(1, new StringVal(winfo->writer->Path().c_str()));
-	info->Assign(2, new Val(winfo->open_time, TYPE_TIME));
-	info->Assign(3, new Val(network_time, TYPE_TIME));
+	// Build a temporary path for the writer to move the file to.
+	struct tm tm;
+	char buf[128];
+	const char* const date_fmt = "%y-%m-%d_%H.%M.%S";
+	time_t teatime = (time_t)winfo->open_time;
 
-	// Call the function building us the new path.
+	localtime_r(&teatime, &tm);
+	strftime(buf, sizeof(buf), date_fmt, &tm);
 
-	Func* rotation_path_func =
-		internal_func("Log::default_rotation_path_func");
+	string tmp = string(fmt("%s-%s", winfo->writer->Path().c_str(), buf));
+
+	// Trigger the rotation.
+	winfo->writer->Rotate(tmp, winfo->open_time, network_time, terminating);
+	}
+
+bool LogMgr::FinishedRotation(LogWriter* writer, string new_name, string old_name,
+		      double open, double close, bool terminating)
+	{
+	DBG_LOG(DBG_LOGGING, "Finished rotating %s at %.6f, new name %s",
+		writer->Path().c_str(), network_time, new_name.c_str());
+
+	WriterInfo* winfo = FindWriter(writer);
+	assert(winfo);
 
 	RecordVal* rc =
 		LookupRotationControl(winfo->type, winfo->writer->Path());
 
+	assert(rc);
+
+	// Create the RotationInfo record.
+	RecordVal* info = new RecordVal(BifType::Record::Log::RotationInfo);
+	info->Assign(0, winfo->type->Ref());
+	info->Assign(1, new StringVal(new_name.c_str()));
+	info->Assign(2, new StringVal(winfo->writer->Path().c_str()));
+	info->Assign(3, new Val(open, TYPE_TIME));
+	info->Assign(4, new Val(close, TYPE_TIME));
+	info->Assign(5, new Val(terminating, TYPE_BOOL));
+
 	int idx = rc->Type()->AsRecordType()->FieldOffset("postprocessor");
+	assert(idx >= 0);
 
-	string rotation_postprocessor =
-		rc->LookupWithDefault(idx)->AsString()->CheckString();
+	Val* func = rc->Lookup(idx);
+	if ( ! func )
+		{
+		ID* id = global_scope()->Lookup("Log::__default_rotation_postprocessor");
+		assert(id);
+		func = id->ID_Val();
+		}
 
+	assert(func);
+
+	// Call the postprocessor function.
 	val_list vl(1);
 	vl.append(info);
-	Val* result = rotation_path_func->Call(&vl);
-	string new_path = result->AsString()->CheckString();
-	Unref(result);
-
-	winfo->writer->Rotate(new_path, rotation_postprocessor,
-			      winfo->open_time, network_time, terminating);
+	Val* v = func->AsFunc()->Call(&vl);
+	int result = v->AsBool();
+	Unref(v);
+	return result;
 	}
 
-

src/LogMgr.h

@@ -103,6 +103,10 @@ protected:
 
 	//// Functions safe to use by writers.
 
+	// Signals that a file has been rotated.
+	bool FinishedRotation(LogWriter* writer, string new_name, string old_name,
+			      double open, double close, bool terminating);
+
 	// Reports an error for the given writer.
 	void Error(LogWriter* writer, const char* msg);
 
@@ -127,6 +131,7 @@ private:
 	void Rotate(WriterInfo* info);
 	RecordVal* LookupRotationControl(EnumVal* writer, string path);
 	Filter* FindFilter(EnumVal* id, StringVal* filter);
+	WriterInfo* FindWriter(LogWriter* writer);
 
 	vector<Stream *> streams;	// Indexed by stream enum.
 };

src/LogWriter.cc

@@ -89,10 +89,10 @@ bool LogWriter::SetBuf(bool enabled)
 	return true;
 	}
 
-bool LogWriter::Rotate(string rotated_path, string postprocessor, double open,
+bool LogWriter::Rotate(string rotated_path, double open,
 		       double close, bool terminating)
 	{
-	if ( ! DoRotate(rotated_path, postprocessor, open, close, terminating) )
+	if ( ! DoRotate(rotated_path, open, close, terminating) )
 		{
 		disabled = true;
 		return false;
@@ -151,42 +151,8 @@ void LogWriter::DeleteVals(LogVal** vals)
 	log_mgr->DeleteVals(num_fields, vals);
 	}
 
-bool LogWriter::RunPostProcessor(string fname, string postprocessor,
-				 string old_name, double open, double close,
-				 bool terminating)
+bool LogWriter::FinishedRotation(string new_name, string old_name, double open,
+				 double close, bool terminating)
 	{
-	// This function operates in a way that is backwards-compatible with
-	// the old Bro log rotation scheme.
-
-	if ( ! postprocessor.size() )
-		return true;
-
-	const char* const fmt = "%y-%m-%d_%H.%M.%S";
-
-	struct tm tm1;
-	struct tm tm2;
-
-	time_t tt1 = (time_t)open;
-	time_t tt2 = (time_t)close;
-
-	localtime_r(&tt1, &tm1);
-	localtime_r(&tt2, &tm2);
-
-	char buf1[128];
-	char buf2[128];
-
-	strftime(buf1, sizeof(buf1), fmt, &tm1);
-	strftime(buf2, sizeof(buf2), fmt, &tm2);
-
-	string cmd = postprocessor;
-	cmd += " " + fname;
-	cmd += " " + old_name;
-	cmd += " " + string(buf1);
-	cmd += " " + string(buf2);
-	cmd += " " + string(terminating ? "1" : "0");
-	cmd += " &";
-
-	system(cmd.c_str());
-
-	return true;
+	return log_mgr->FinishedRotation(this, new_name, old_name, open, close, terminating);
 	}

src/LogWriter.h

@@ -60,8 +60,7 @@ public:
 
 	// Triggers rotation, if the writer supports that. (If not, it will
 	// be ignored).
-	bool Rotate(string rotated_path, string postprocessor, double open,
-		    double close, bool terminating);
+	bool Rotate(string rotated_path, double open, double close, bool terminating);
 
 	// Finishes writing to this logger regularly. Must not be called if
 	// an error has been indicated earlier. After calling this, no
@@ -77,7 +76,6 @@ public:
 	const LogField* const * Fields() const	{ return fields; }
 
 protected:
-
 	// Methods for writers to override. If any of these returs false, it
 	// will be assumed that a fatal error has occured that prevents the
 	// writer from further operation. It will then be disabled and
@@ -116,6 +114,10 @@ protected:
 	// applies to writers writing into files, which should then close the
 	// current file and open a new one.  However, a writer may also
 	// trigger other apppropiate actions if semantics are similar.
+	// 
+	// Once rotation has finished, the implementation should call
+	// RotationDone() to signal the log manager that potential
+	// postprocessors can now run.
 	//
 	// "rotate_path" reflects the path to where the rotated output is to
 	// be moved, with specifics depending on the writer. It should
@@ -123,12 +125,7 @@ protected:
 	// as passed into DoInit(). As an example, for file-based output, 
 	// "rotate_path" could be the original filename extended with a
 	// timestamp indicating the time of the rotation.
-
-	// "postprocessor" is the name of a command to execute on the rotated
-	// file. If empty, no postprocessing should take place; if given but
-	// the writer doesn't support postprocessing, it can be ignored (but
-	// the method must still return true in that case).
-
+	// 
 	// "open" and "close" are the network time's when the *current* file
 	// was opened and closed, respectively.
 	//
@@ -138,8 +135,8 @@ protected:
 	//
 	// A writer may ignore rotation requests if it doesn't fit with its
 	// semantics (but must still return true in that case).
-	virtual bool DoRotate(string rotated_path, string postprocessor,
-			      double open, double close, bool terminating) = 0;
+	virtual bool DoRotate(string rotated_path, double open, double close,
+			      bool terminating) = 0;
 
 	// Called once on termination. Not called when any of the other
 	// methods has previously signaled an error, i.e., executing this
@@ -157,11 +154,18 @@ protected:
 	// Reports an error to the user.
 	void Error(const char *msg);
 
-	// Runs a post-processor on the given file. Parameters correspond to
-	// those of DoRotate(). 
-	bool RunPostProcessor(string fname, string postprocessor,
-			      string old_name, double open, double close,
-			      bool terminating);
+	// Signals to the log manager that a file has been rotated.
+	//
+	// new_name: The filename of the rotated file. old_name: The filename
+	// of the origina file.
+	//
+	// open/close: The timestamps when the original file was opened and
+	// closed, respectively.
+	//
+	// terminating: True if rotation request occured due to the main Bro
+	// process shutting down.
+	bool FinishedRotation(string new_name, string old_name, double open,
+			      double close, bool terminating);
 
 private:
 	friend class LogMgr;

src/LogWriterAscii.cc

@@ -242,7 +242,7 @@ bool LogWriterAscii::DoWrite(int num_fields, const LogField* const * fields,
 	return true;
 	}
 
-bool LogWriterAscii::DoRotate(string rotated_path, string postprocessor, double open,
+bool LogWriterAscii::DoRotate(string rotated_path, double open,
 			      double close, bool terminating)
 	{
 	if ( IsSpecial(Path()) )
@@ -254,10 +254,8 @@ bool LogWriterAscii::DoRotate(string rotated_path, string postprocessor, double
 	string nname = rotated_path + ".log";
 	rename(fname.c_str(), nname.c_str());
 
-	if ( postprocessor.size() &&
-	     ! RunPostProcessor(nname, postprocessor, fname.c_str(),
-				open, close, terminating) )
-		return false;
+	if ( ! FinishedRotation(nname, fname, open, close, terminating) )
+		Error(Fmt("error rotating %s to %s", fname.c_str(), nname.c_str()));
 
 	return DoInit(Path(), NumFields(), Fields());
 	}

src/LogWriterAscii.h

@@ -20,8 +20,8 @@ protected:
 	virtual bool DoWrite(int num_fields, const LogField* const * fields,
 			     LogVal** vals);
 	virtual bool DoSetBuf(bool enabled);
-	virtual bool DoRotate(string rotated_path, string postprocessr,
-			      double open, double close, bool terminating);
+	virtual bool DoRotate(string rotated_path, double open, double close,
+			      bool terminating);
 	virtual bool DoFlush();
 	virtual void DoFinish();
 

src/RSH.cc

@@ -15,7 +15,7 @@ Contents_Rsh_Analyzer::Contents_Rsh_Analyzer(Connection* conn, bool orig,
 						Rsh_Analyzer* arg_analyzer)
 : ContentLine_Analyzer(AnalyzerTag::Contents_Rsh, conn, orig)
 	{
-	num_bytes_to_scan = num_bytes_to_scan = 0;
+	num_bytes_to_scan = 0;
 	analyzer = arg_analyzer;
 
 	if ( orig )

src/RemoteSerializer.cc

@@ -3060,13 +3060,6 @@ bool RemoteSerializer::IsActive()
 	return false;
 	}
 
-
-const char* const* RemoteSerializer::GetBuiltins() const
-	{
-	static const char* builtins[] = { "connect", "listen", 0 };
-	return builtins;
-	}
-
 void RemoteSerializer::ReportError(const char* msg)
 	{
 	if ( current_peer && current_peer->phase != Peer::SETUP )

src/RemoteSerializer.h

@@ -128,10 +128,6 @@ public:
 	// Log some statistics.
 	void LogStats();
 
-	// Return a 0-terminated array of built-in functions which,
-	// when referenced, trigger the remote serializer's initialization.
-	const char* const* GetBuiltins() const;
-
 	// Tries to sent out all remaining data.
 	// FIXME: Do we still need this?
 	void Finish();

src/bro.bif

@@ -3632,7 +3632,27 @@ function piped_exec%(program: string, to_write: string%): bool
 		}
 
 	fprintf(f, "%s", to_write->CheckString());
-    pclose(f);
+	pclose(f);
 
-    return new Val(true, TYPE_BOOL);
+	return new Val(true, TYPE_BOOL);
+	%}
+
+## Enables the communication system. Note that by default,
+## communication is off until explicitly enabled, and all other calls
+## to communication-related BiFs' will be ignored until done so.
+function enable_communication%(%): any
+	%{
+	if ( bro_start_network_time != 0.0 )
+		{
+		builtin_error("communication must be enabled in bro_init");
+		return 0;
+		}
+
+	if ( using_communication )
+		// Ignore duplicate calls.
+		return 0;
+
+	using_communication = 1;
+	remote_serializer->Init();
+	return 0;
 	%}

src/main.cc

@@ -880,9 +880,6 @@ int main(int argc, char** argv)
 		exit(0);
 		}
 
-	if ( using_communication )
-		remote_serializer->Init();
-
 	persistence_serializer->SetDir((const char *)state_dir->AsString()->CheckString());
 
 	// Print the ID.