Implement majority of Broxygen features delegated to Bro.

Still have to update the Sphinx integration.

src/BroDoc.cc

@@ -12,6 +12,7 @@
 #include "analyzer/Manager.h"
 #include "analyzer/Component.h"
 #include "file_analysis/Manager.h"
+#include "broxygen/Manager.h"
 
 BroDoc::BroDoc(const std::string& rel, const std::string& abs)
 	{
@@ -565,33 +566,28 @@ static void WritePluginBifItems(FILE* f, const plugin::Plugin* p,
 
 	for ( it = bifitems.begin(); it != bifitems.end(); ++it )
 		{
-		BroDocObj* o = doc_ids[it->GetID()];
+		broxygen::IdentifierDocument* doc = broxygen_mgr->GetIdentifierDoc(
+		                                        it->GetID());
 
-		if ( o )
+		if ( doc )
-			o->WriteReST(f);
+			fprintf(f, "%s\n\n", doc->ReStructuredText().c_str());
 		else
-			reporter->Warning("No docs for ID: %s\n", it->GetID());
+			reporter->InternalWarning("Broxygen ID lookup failed: %s\n",
+			                          it->GetID());
 		}
 	}
 
-static void WriteAnalyzerTagDefn(FILE* f, EnumType* e, const string& module)
+static void WriteAnalyzerTagDefn(FILE* f, const string& module)
 	{
-	/* TODO
 	string tag_id = module + "::Tag";
-	e = new CommentedEnumType(e);
-	e->SetTypeID(copy_string(tag_id.c_str()));
 
-	ID* dummy_id = new ID(tag_id.c_str(), SCOPE_GLOBAL, true);
+	broxygen::IdentifierDocument* doc = broxygen_mgr->GetIdentifierDoc(tag_id);
-	dummy_id->SetType(e);
-	dummy_id->MakeType();
 
-	list<string>* r = new list<string>();
+	if ( ! doc )
-	r->push_back("Unique identifiers for analyzers.");
+		reporter->InternalError("Broxygen failed analyzer tag lookup: %s",
+		                        tag_id.c_str());
 
-	BroDocObj bdo(dummy_id, r, true);
+	fprintf(f, "%s\n", doc->ReStructuredText().c_str());
-
-	bdo.WriteReST(f);
-	*/
 	}
 
 static bool ComponentsMatch(const plugin::Plugin* p, plugin::component::Type t,
@@ -610,16 +606,14 @@ static bool ComponentsMatch(const plugin::Plugin* p, plugin::component::Type t,
 	return true;
 	}
 
-void CreateProtoAnalyzerDoc(const char* filename)
+void CreateProtoAnalyzerDoc(FILE* f)
 	{
-	FILE* f = fopen(filename, "w");
-
 	fprintf(f, "Protocol Analyzers\n");
-	fprintf(f, "==================\n\n\n");
+	fprintf(f, "==================\n\n");
 	fprintf(f, ".. contents::\n");
 	fprintf(f, "     :depth: 1\n\n");
 
-	WriteAnalyzerTagDefn(f, analyzer_mgr->GetTagEnumType(), "Analyzer");
+	WriteAnalyzerTagDefn(f, "Analyzer");
 
 	plugin::Manager::plugin_list plugins = plugin_mgr->Plugins();
 	plugin::Manager::plugin_list::const_iterator it;
@@ -642,16 +636,14 @@ void CreateProtoAnalyzerDoc(const char* filename)
 	fclose(f);
 	}
 
-void CreateFileAnalyzerDoc(const char* filename)
+void CreateFileAnalyzerDoc(FILE* f)
 	{
-	FILE* f = fopen(filename, "w");
-
 	fprintf(f, "File Analyzers\n");
 	fprintf(f, "==============\n\n");
 	fprintf(f, ".. contents::\n");
 	fprintf(f, "     :depth: 1\n\n");
 
-	WriteAnalyzerTagDefn(f, file_mgr->GetTagEnumType(), "Files");
+	WriteAnalyzerTagDefn(f, "Files");
 
 	plugin::Manager::plugin_list plugins = plugin_mgr->Plugins();
 	plugin::Manager::plugin_list::const_iterator it;

src/BroDoc.h

@@ -409,14 +409,14 @@ private:
 
 /**
  * Writes out plugin index documentation for all analyzer plugins.
- * @param filename the name of the file to write.
+ * @param f an open file stream to write docs into.
  */
-void CreateProtoAnalyzerDoc(const char* filename);
+void CreateProtoAnalyzerDoc(FILE* f);
 
 /**
  * Writes out plugin index documentation for all file analyzer plugins.
- * @param filename the name of the file to write.
+ * @param f an open file stream to write docs into.
  */
-void CreateFileAnalyzerDoc(const char* filename);
+void CreateFileAnalyzerDoc(FILE* f);
 
 #endif

src/Desc.h

@@ -69,6 +69,7 @@ public:
 	void PopIndent();
 	void PopIndentNoNL();
 	int GetIndentLevel() const	{ return indent_level; }
+	void ClearIndentLevel() { indent_level = 0; }
 
 	int IndentSpaces() const	{ return indent_with_spaces; }
 	void SetIndentSpaces(int i)	{ indent_with_spaces = i; }

src/ID.cc

@@ -14,6 +14,7 @@
 #include "PersistenceSerializer.h"
 #include "Scope.h"
 #include "Traverse.h"
+#include "broxygen/Manager.h"
 
 ID::ID(const char* arg_name, IDScope arg_scope, bool arg_is_export)
 	{
@@ -618,7 +619,6 @@ void ID::DescribeExtended(ODesc* d) const
 
 void ID::DescribeReSTShort(ODesc* d) const
 	{
-	/* TODO
 	if ( is_type )
 		d->Add(":bro:type:`");
 	else
@@ -632,8 +632,8 @@ void ID::DescribeReSTShort(ODesc* d) const
 		d->Add(": ");
 		d->Add(":bro:type:`");
 
-		if ( ! is_type && type->GetTypeID() )
+		if ( ! is_type && ! type->GetName().empty() )
-			d->Add(type->GetTypeID());
+			d->Add(type->GetName().c_str());
 		else
 			{
 			TypeTag t = type->Tag();
@@ -644,14 +644,14 @@ void ID::DescribeReSTShort(ODesc* d) const
 				break;
 
 			case TYPE_FUNC:
-				d->Add(type->AsFuncType()->FlavorString());
+				d->Add(type->AsFuncType()->FlavorString().c_str());
 				break;
 
 			case TYPE_ENUM:
 				if ( is_type )
 					d->Add(type_name(t));
 				else
-					d->Add(type->AsEnumType()->Name().c_str());
+					d->Add(broxygen_mgr->GetEnumTypeName(Name()).c_str());
 				break;
 
 			default:
@@ -668,12 +668,11 @@ void ID::DescribeReSTShort(ODesc* d) const
 		d->SP();
 		attrs->DescribeReST(d);
 		}
-		*/
 	}
 
-void ID::DescribeReST(ODesc* d, bool is_role) const
+void ID::DescribeReST(ODesc* d, bool roles_only) const
 	{
-	if ( is_role )
+	if ( roles_only )
 		{
 		if ( is_type )
 			d->Add(":bro:type:`");
@@ -705,7 +704,7 @@ void ID::DescribeReST(ODesc* d, bool is_role) const
 			d->Add("`");
 			}
 		else
-			type->DescribeReST(d);
+			type->DescribeReST(d, roles_only);
 
 		d->NL();
 		}

src/ID.h

@@ -84,7 +84,7 @@ public:
 	// Adds type and value to description.
 	void DescribeExtended(ODesc* d) const;
 	// Produces a description that's reST-ready.
-	void DescribeReST(ODesc* d, bool is_role=false) const;
+	void DescribeReST(ODesc* d, bool roles_only = false) const;
 	void DescribeReSTShort(ODesc* d) const;
 
 	bool Serialize(SerialInfo* info) const;

src/Type.cc

@@ -117,16 +117,17 @@ BroType* BroType::Clone() const
 	sinfo.cache = false;
 
 	this->Serialize(&sinfo);
-	char* data;
+	char* data = 0;
 	uint32 len = form->EndWrite(&data);
 	form->StartRead(data, len);
 
 	UnserialInfo uinfo(&ss);
 	uinfo.cache = false;
-	BroType* rval = this->Unserialize(&uinfo);
+
+	BroType* rval = this->Unserialize(&uinfo, false);
+	assert(rval != this);
 
 	delete [] data;
-
 	return rval;
 	}
 
@@ -173,11 +174,9 @@ void BroType::Describe(ODesc* d) const
 		}
 	}
 
-void BroType::DescribeReST(ODesc* d) const
+void BroType::DescribeReST(ODesc* d, bool roles_only) const
 	{
-	d->Add(":bro:type:`");
+	d->Add(fmt(":bro:type:`%s`", type_name(Tag())));
-	d->Add(type_name(Tag()));
-	d->Add("`");
 	}
 
 void BroType::SetError()
@@ -200,7 +199,7 @@ bool BroType::Serialize(SerialInfo* info) const
 	return ret;
 	}
 
-BroType* BroType::Unserialize(UnserialInfo* info, TypeTag want)
+BroType* BroType::Unserialize(UnserialInfo* info, bool use_existing)
 	{
 	// To avoid external Broccoli clients needing to always send full type
 	// objects, we allow them to give us only the name of a type. To
@@ -234,8 +233,8 @@ BroType* BroType::Unserialize(UnserialInfo* info, TypeTag want)
 
 	BroType* t = (BroType*) SerialObj::Unserialize(info, SER_BRO_TYPE);
 
-	if ( ! t )
+	if ( ! t || ! use_existing )
-		return 0;
+		return t;
 
 	if ( ! t->name.empty() )
 		{
@@ -458,7 +457,7 @@ void IndexType::Describe(ODesc* d) const
 		}
 	}
 
-void IndexType::DescribeReST(ODesc* d) const
+void IndexType::DescribeReST(ODesc* d, bool roles_only) const
 	{
 	d->Add(":bro:type:`");
 
@@ -484,7 +483,7 @@ void IndexType::DescribeReST(ODesc* d) const
 			d->Add("`");
 			}
 		else
-			t->DescribeReST(d);
+			t->DescribeReST(d, roles_only);
 		}
 
 	d->Add("]");
@@ -500,7 +499,7 @@ void IndexType::DescribeReST(ODesc* d) const
 			d->Add("`");
 			}
 		else
-			yield_type->DescribeReST(d);
+			yield_type->DescribeReST(d, roles_only);
 		}
 	}
 
@@ -775,7 +774,7 @@ void FuncType::Describe(ODesc* d) const
 		}
 	}
 
-void FuncType::DescribeReST(ODesc* d) const
+void FuncType::DescribeReST(ODesc* d, bool roles_only) const
 	{
 	d->Add(":bro:type:`");
 	d->Add(FlavorString());
@@ -795,7 +794,7 @@ void FuncType::DescribeReST(ODesc* d) const
 			d->Add("`");
 			}
 		else
-			yield->DescribeReST(d);
+			yield->DescribeReST(d, roles_only);
 		}
 	}
 
@@ -927,7 +926,7 @@ TypeDecl* TypeDecl::Unserialize(UnserialInfo* info)
 	return t;
 	}
 
-void TypeDecl::DescribeReST(ODesc* d) const
+void TypeDecl::DescribeReST(ODesc* d, bool roles_only) const
 	{
 	d->Add(id);
 	d->Add(": ");
@@ -939,7 +938,7 @@ void TypeDecl::DescribeReST(ODesc* d) const
 		d->Add("`");
 		}
 	else
-		type->DescribeReST(d);
+		type->DescribeReST(d, roles_only);
 
 	if ( attrs )
 		{
@@ -1037,9 +1036,13 @@ void RecordType::Describe(ODesc* d) const
 		}
 	}
 
-void RecordType::DescribeReST(ODesc* d) const
+void RecordType::DescribeReST(ODesc* d, bool roles_only) const
 	{
 	d->Add(":bro:type:`record`");
+
+	if ( num_fields == 0 )
+		return;
+
 	d->NL();
 	DescribeFieldsReST(d, false);
 	}
@@ -1136,7 +1139,53 @@ void RecordType::DescribeFieldsReST(ODesc* d, bool func_args) const
 				}
 			}
 
-		FieldDecl(i)->DescribeReST(d);
+		const TypeDecl* td = FieldDecl(i);
+		td->DescribeReST(d);
+
+		if ( func_args )
+			continue;
+
+		using broxygen::IdentifierDocument;
+		IdentifierDocument* doc = broxygen_mgr->GetIdentifierDoc(GetName());
+
+		if ( ! doc )
+			{
+			reporter->InternalWarning("Failed to lookup record doc: %s",
+			                          GetName().c_str());
+			continue;
+			}
+
+		string field_from_script = doc->GetDeclaringScriptForField(td->id);
+		string type_from_script;
+
+		if ( doc->GetDeclaringScript() )
+			type_from_script = doc->GetDeclaringScript()->Name();
+
+		if ( ! field_from_script.empty() &&
+		     field_from_script != type_from_script )
+			{
+			d->PushIndent();
+			d->Add(fmt("(from ``redef`` in :doc:`%s`)",
+			           field_from_script.c_str()));
+			d->PopIndentNoNL();
+			}
+
+		vector<string> cmnts = doc->GetFieldComments(td->id);
+
+		if ( cmnts.empty() )
+			continue;
+
+		d->PushIndent();
+
+		for ( size_t i = 0; i < cmnts.size(); ++i )
+			{
+			if ( i > 0 )
+				d->NL();
+
+			d->Add(cmnts[i].c_str());
+			}
+
+		d->PopIndentNoNL();
 		}
 
 	if ( ! func_args )
@@ -1415,10 +1464,82 @@ const char* EnumType::Lookup(bro_int_t value)
 	return 0;
 	}
 
-void EnumType::DescribeReST(ODesc* d) const
+void EnumType::DescribeReST(ODesc* d, bool roles_only) const
 	{
-	// TODO: this probably goes away
 	d->Add(":bro:type:`enum`");
+
+	// Create temporary, reverse name map so that enums can be documented
+	// in ascending order of their actual integral value instead of by name.
+	typedef map< bro_int_t, const char* > RevNameMap;
+
+	RevNameMap rev;
+
+	for ( NameMap::const_iterator it = names.begin(); it != names.end(); ++it )
+		rev[it->second] = it->first;
+
+	for ( RevNameMap::const_iterator it = rev.begin(); it != rev.end(); ++it )
+		{
+		d->NL();
+		d->PushIndent();
+
+		if ( roles_only )
+			d->Add(fmt(":bro:enum:`%s`", it->second));
+		else
+			d->Add(fmt(".. bro:enum:: %s %s", it->second, GetName().c_str()));
+
+		using broxygen::IdentifierDocument;
+		IdentifierDocument* doc = broxygen_mgr->GetIdentifierDoc(it->second);
+
+		if ( ! doc )
+			{
+			reporter->InternalWarning("Enum %s documentation lookup failure",
+			                          it->second);
+			continue;
+			}
+
+		string enum_from_script;
+		string type_from_script;
+
+		if ( doc->GetDeclaringScript() )
+			enum_from_script = doc->GetDeclaringScript()->Name();
+
+		IdentifierDocument* type_doc = broxygen_mgr->GetIdentifierDoc(GetName());
+
+		if ( type_doc && type_doc->GetDeclaringScript() )
+			type_from_script = type_doc->GetDeclaringScript()->Name();
+
+		if ( ! enum_from_script.empty() &&
+		     enum_from_script != type_from_script )
+			{
+			d->NL();
+			d->PushIndent();
+			d->Add(fmt("(from ``redef`` in :doc:`%s`)",
+			           enum_from_script.c_str()));
+			d->PopIndentNoNL();
+			}
+
+		vector<string> cmnts = doc->GetComments();
+
+		if ( cmnts.empty() )
+			{
+			d->PopIndentNoNL();
+			continue;
+			}
+
+		d->NL();
+		d->PushIndent();
+
+		for ( size_t i = 0; i < cmnts.size(); ++i )
+			{
+			if ( i > 0 )
+				d->NL();
+
+			d->Add(cmnts[i].c_str());
+			}
+
+		d->PopIndentNoNL();
+		d->PopIndentNoNL();
+		}
 	}
 
 IMPLEMENT_SERIAL(EnumType, SER_ENUM_TYPE);

src/Type.h

@@ -227,12 +227,12 @@ public:
 	BroType* Ref()		{ ::Ref(this); return this; }
 
 	virtual void Describe(ODesc* d) const;
-	virtual void DescribeReST(ODesc* d) const;
+	virtual void DescribeReST(ODesc* d, bool roles_only = false) const;
 
 	virtual unsigned MemoryAllocation() const;
 
 	bool Serialize(SerialInfo* info) const;
-	static BroType* Unserialize(UnserialInfo* info, TypeTag want = TYPE_ANY);
+	static BroType* Unserialize(UnserialInfo* info, bool use_existing = true);
 
 	void SetName(const string& arg_name) { name = arg_name; }
 	string GetName() const { return name; }
@@ -305,7 +305,7 @@ public:
 	BroType* YieldType();
 
 	void Describe(ODesc* d) const;
-	void DescribeReST(ODesc* d) const;
+	void DescribeReST(ODesc* d, bool roles_only = false) const;
 
 	// Returns true if this table is solely indexed by subnet.
 	bool IsSubNetIndex() const;
@@ -379,7 +379,7 @@ public:
 	TypeList* ArgTypes() const	{ return arg_types; }
 
 	void Describe(ODesc* d) const;
-	void DescribeReST(ODesc* d) const;
+	void DescribeReST(ODesc* d, bool roles_only = false) const;
 
 protected:
 	FuncType()	{ args = 0; arg_types = 0; yield = 0; flavor = FUNC_FLAVOR_FUNCTION; }
@@ -416,7 +416,7 @@ public:
 	bool Serialize(SerialInfo* info) const;
 	static TypeDecl* Unserialize(UnserialInfo* info);
 
-	virtual void DescribeReST(ODesc* d) const;
+	virtual void DescribeReST(ODesc* d, bool roles_only = false) const;
 
 	BroType* type;
 	Attributes* attrs;
@@ -454,7 +454,7 @@ public:
 	const char* AddFields(type_decl_list* types, attr_list* attr);
 
 	void Describe(ODesc* d) const;
-	void DescribeReST(ODesc* d) const;
+	void DescribeReST(ODesc* d, bool roles_only = false) const;
 	void DescribeFields(ODesc* d) const;
 	void DescribeFieldsReST(ODesc* d, bool func_args) const;
 
@@ -527,7 +527,7 @@ public:
 	bro_int_t Lookup(const string& module_name, const char* name);
 	const char* Lookup(bro_int_t value); // Returns 0 if not found
 
-	void DescribeReST(ODesc* d) const;
+	void DescribeReST(ODesc* d, bool roles_only = false) const;
 
 protected:
 	DECLARE_SERIAL(EnumType)

src/Var.cc

@@ -261,17 +261,17 @@ extern Expr* add_and_assign_local(ID* id, Expr* init, Val* val)
 
 void add_type(ID* id, BroType* t, attr_list* attr)
 	{
-	string new_type_name(id->Name());
+	string new_type_name = id->Name();
-	string old_type_name(t->GetName());
+	string old_type_name = t->GetName();
 	BroType* tnew = 0;
 
 	if ( (t->Tag() == TYPE_RECORD || t->Tag() == TYPE_ENUM) &&
-	     ! old_type_name.empty() )
+	     old_type_name.empty() )
+		// An extensible type (record/enum) being declared for first time.
+		tnew = t;
+	else
 		// Clone the type to preserve type name aliasing.
 		tnew = t->Clone();
-	else
-		// An extensible types (record/enum) being declared for first time.
-		tnew = t;
 
 	type_aliases[new_type_name].insert(tnew);
 

src/broxygen/Configuration.cc

@@ -1,4 +1,5 @@
 #include "Configuration.h"
+#include "Manager.h"
 
 #include "util.h"
 #include "Reporter.h"
@@ -7,6 +8,8 @@
 #include <vector>
 #include <algorithm>
 #include <map>
+#include <cstdio>
+#include <sys/stat.h>
 
 using namespace broxygen;
 using namespace std;
@@ -16,12 +19,12 @@ typedef map<string, Target::factory_fn> target_factory_map;
 static target_factory_map create_target_factory_map()
 	{
 	target_factory_map rval;
-	rval["package_index"] =  &PackageTarget::Instantiate;
+	rval["package_index"] =  &PackageIndexTarget::Instantiate;
 	rval["package"] =        &PackageTarget::Instantiate;
 	rval["proto_analyzer"] = &ProtoAnalyzerTarget::Instantiate;
 	rval["file_analyzer"] =  &FileAnalyzerTarget::Instantiate;
-	rval["script_summary"] = &ScriptTarget::Instantiate;
+	rval["script_summary"] = &ScriptSummaryTarget::Instantiate;
-	rval["script_index"] =   &ScriptTarget::Instantiate;
+	rval["script_index"] =   &ScriptIndexTarget::Instantiate;
 	rval["script"] =         &ScriptTarget::Instantiate;
 	rval["identifier"] =     &IdentifierTarget::Instantiate;
 	return rval;
@@ -29,41 +32,300 @@ static target_factory_map create_target_factory_map()
 
 static target_factory_map target_instantiators = create_target_factory_map();
 
-bool Target::MatchesPattern(Document* doc) const
+struct TargetFile {
+	TargetFile(const string& arg_name)
+		: name(arg_name), f()
 		{
-	// TODO: prefix matching or full regex?
+		if ( name.find('/') != string::npos )
+			{
+			string dir = SafeDirname(name).result;
 
-	if ( doc->Name() == pattern )
+			if ( ! ensure_intermediate_dirs(dir.c_str()) )
-		{
+				reporter->FatalError("Broxygen failed to make dir %s",
-		DBG_LOG(DBG_BROXYGEN, "Doc '%s'' matched pattern for target '%s'",
+				                     dir.c_str());
-		        doc->Name().c_str(), name.c_str());
-		return true;
 			}
 
-	return false;
+		f = fopen(name.c_str(), "w");
+
+		if ( ! f )
+			reporter->FatalError("Broxygen failed to open '%s' for writing: %s",
+			                     name.c_str(), strerror(errno));
 		}
 
+	~TargetFile()
+		{
+		if ( f )
+			fclose(f);
+
+		DBG_LOG(DBG_BROXYGEN, "Wrote out-of-date target '%s'", name.c_str());
+		}
+
+	string name;
+	FILE* f;
+};
+
 template<class T>
-void filter_matching_docs(const std::vector<Document*>& filter_from, Target* t)
+static vector<T*> filter_matching_docs(const vector<Document*>& from, Target* t)
 	{
-	for ( size_t i = 0; i < filter_from.size(); ++i )
+	vector<T*> rval;
+
+	for ( size_t i = 0; i < from.size(); ++i )
 		{
-		T* d = dynamic_cast<T*>(filter_from[i]);
+		T* d = dynamic_cast<T*>(from[i]);
 
 		if ( ! d )
 			continue;
 
 		if ( t->MatchesPattern(d) )
-			t->AddDependency(d);
-		}
-	}
-
-void IdentifierTarget::DoFindDependencies(const std::vector<Document*>& docs)
 			{
-	filter_matching_docs<IdentifierDocument>(docs, this);
+			DBG_LOG(DBG_BROXYGEN, "Doc '%s' matched pattern for target '%s'",
+			        d->Name().c_str(), t->Name().c_str());
+			rval.push_back(d);
+			}
 		}
 
-Config::Config(const string& file, const string& delim)
+	return rval;
+	}
+
+Target::Target(const string& arg_name, const string& arg_pattern)
+    : name(arg_name), pattern(arg_pattern), prefix()
+	{
+	size_t pos = pattern.find('*');
+
+	if ( pos == 0 || pos == string::npos )
+		return;
+
+	prefix = pattern.substr(0, pos);
+	}
+
+bool Target::MatchesPattern(Document* doc) const
+	{
+	if ( pattern == "*" )
+		return true;
+
+	if ( prefix.empty() )
+		return doc->Name() == pattern;
+
+	return ! strncmp(doc->Name().c_str(), prefix.c_str(), prefix.size());
+	}
+
+void AnalyzerTarget::DoFindDependencies(const std::vector<Document *>& docs)
+	{
+	// TODO: really should add to dependency list the tag type's ID and
+	// all bif items for matching analyzer plugins, but that's all dependent
+	// on the bro binary itself, so I'm cheating.
+	}
+
+void AnalyzerTarget::DoGenerate() const
+	{
+	if ( broxygen_mgr->IsUpToDate(Name(), vector<Document*>()) )
+		return;
+
+	if ( Pattern() != "*" )
+		reporter->InternalWarning("Broxygen only implements analyzer target"
+		                          " pattern '*'");
+
+	TargetFile file(Name());
+	doc_creator_callback(file.f);
+	}
+
+void PackageTarget::DoFindDependencies(const vector<Document*>& docs)
+	{
+	pkg_deps = filter_matching_docs<PackageDocument>(docs, this);
+
+	if ( pkg_deps.empty() )
+		reporter->FatalError("No match for Broxygen target '%s' pattern '%s'",
+		                     Name().c_str(), Pattern().c_str());
+
+	for ( size_t i = 0; i < docs.size(); ++i )
+		{
+		ScriptDocument* script = dynamic_cast<ScriptDocument*>(docs[i]);
+
+		if ( ! script )
+			continue;
+
+		for ( size_t j = 0; j < pkg_deps.size(); ++j )
+			{
+			if ( strncmp(script->Name().c_str(), pkg_deps[j]->Name().c_str(),
+			             pkg_deps[j]->Name().size()))
+				continue;
+
+			DBG_LOG(DBG_BROXYGEN, "Script %s associated with package %s",
+			        script->Name().c_str(), pkg_deps[j]->Name().c_str());
+			pkg_manifest[pkg_deps[j]].push_back(script);
+			script_deps.push_back(script);
+			}
+		}
+	}
+
+void PackageTarget::DoGenerate() const
+	{
+	if ( broxygen_mgr->IsUpToDate(Name(), script_deps) )
+		return;
+
+	TargetFile file(Name());
+
+	for ( manifest_t::const_iterator it = pkg_manifest.begin();
+	      it != pkg_manifest.end(); ++it )
+		{
+		fprintf(file.f, "Package: %s\n\n", it->first->Name().c_str());
+
+		for ( size_t i = 0; i < it->second.size(); ++i )
+			{
+			fprintf(file.f, "   :doc:`%s`\n", it->second[i]->Name().c_str());
+
+			vector<string> cmnts = it->second[i]->GetComments();
+
+			for ( size_t j = 0; j < cmnts.size(); ++j )
+				fprintf(file.f, "      %s\n", cmnts[j].c_str());
+
+			fprintf(file.f, "\n");
+			}
+		}
+	}
+
+void PackageIndexTarget::DoFindDependencies(const vector<Document*>& docs)
+	{
+	pkg_deps = filter_matching_docs<PackageDocument>(docs, this);
+
+	if ( pkg_deps.empty() )
+		reporter->FatalError("No match for Broxygen target '%s' pattern '%s'",
+		                     Name().c_str(), Pattern().c_str());
+	}
+
+void PackageIndexTarget::DoGenerate() const
+	{
+	if ( broxygen_mgr->IsUpToDate(Name(), pkg_deps) )
+		return;
+
+	TargetFile file(Name());
+
+	for ( size_t i = 0; i < pkg_deps.size(); ++i )
+		fprintf(file.f, "%s\n", pkg_deps[i]->ReStructuredText().c_str());
+	}
+
+void ScriptTarget::DoFindDependencies(const vector<Document*>& docs)
+    {
+	script_deps = filter_matching_docs<ScriptDocument>(docs, this);
+
+	if ( script_deps.empty() )
+		reporter->FatalError("No match for Broxygen target '%s' pattern '%s'",
+		                     Name().c_str(), Pattern().c_str());
+	}
+
+void ScriptTarget::DoGenerate() const
+    {
+	if ( Name()[Name().size() - 1] == '/' )
+		{
+		// Target name is a dir, matching scripts are written within that dir
+		// with a dir tree that parallels the script's BROPATH location.
+
+		for ( size_t i = 0; i < script_deps.size(); ++i )
+			{
+			string target_filename = Name() + script_deps[i]->Name();
+			size_t pos = target_filename.rfind(".bro");
+
+			if ( pos == target_filename.size() - 4 )
+				target_filename.replace(pos, 4, ".rst");
+			else
+				target_filename += ".rst";
+
+			vector<ScriptDocument*> dep;
+			dep.push_back(script_deps[i]);
+
+			if ( broxygen_mgr->IsUpToDate(target_filename, dep) )
+				continue;
+
+			TargetFile file(target_filename);
+
+			fprintf(file.f, "%s\n", script_deps[i]->ReStructuredText().c_str());
+			}
+
+		// TODO: could possibly take inventory of files in the dir beforehand,
+		// track all files written, then compare afterwards in order to remove
+		// stale files.
+		return;
+		}
+
+	// Target is a single file, all matching scripts get written there.
+
+	if ( broxygen_mgr->IsUpToDate(Name(), script_deps) )
+		return;
+
+	TargetFile file(Name());
+
+	for ( size_t i = 0; i < script_deps.size(); ++i )
+		fprintf(file.f, "%s\n", script_deps[i]->ReStructuredText().c_str());
+	}
+
+void ScriptSummaryTarget::DoGenerate() const
+	{
+	if ( broxygen_mgr->IsUpToDate(Name(), script_deps) )
+		return;
+
+	TargetFile file(Name());
+
+	for ( size_t i = 0; i < script_deps.size(); ++i )
+		{
+		ScriptDocument* d = dynamic_cast<ScriptDocument*>(script_deps[i]);
+
+		if ( ! d )
+			continue;
+
+		fprintf(file.f, ":doc:`/scripts/%s`\n", d->Name().c_str());
+
+		vector<string> cmnts = d->GetComments();
+
+		for ( size_t i = 0; i < cmnts.size(); ++i )
+			fprintf(file.f, "    %s\n", cmnts[i].c_str());
+
+		fprintf(file.f, "\n");
+		}
+	}
+
+void ScriptIndexTarget::DoGenerate() const
+	{
+	if ( broxygen_mgr->IsUpToDate(Name(), script_deps) )
+		return;
+
+	TargetFile file(Name());
+
+	fprintf(file.f, ".. toctree::\n");
+	fprintf(file.f, "   :maxdepth: 1\n\n");
+
+	for ( size_t i = 0; i < script_deps.size(); ++i )
+		{
+		ScriptDocument* d = dynamic_cast<ScriptDocument*>(script_deps[i]);
+
+		if ( ! d )
+			continue;
+
+		fprintf(file.f, "   %s <%s>\n", d->Name().c_str(), d->Name().c_str());
+		}
+	}
+
+void IdentifierTarget::DoFindDependencies(const vector<Document*>& docs)
+	{
+	id_deps = filter_matching_docs<IdentifierDocument>(docs, this);
+
+	if ( id_deps.empty() )
+		reporter->FatalError("No match for Broxygen target '%s' pattern '%s'",
+		                     Name().c_str(), Pattern().c_str());
+	}
+
+void IdentifierTarget::DoGenerate() const
+	{
+	if ( broxygen_mgr->IsUpToDate(Name(), id_deps) )
+		return;
+
+	TargetFile file(Name());
+
+	for ( size_t i = 0; i < id_deps.size(); ++i )
+		fprintf(file.f, "%s\n\n", id_deps[i]->ReStructuredText().c_str());
+	}
+
+Config::Config(const string& arg_file, const string& delim)
+	: file(arg_file), targets()
 	{
 	if ( file.empty() )
 		return;
@@ -71,53 +333,71 @@ Config::Config(const string& file, const string& delim)
 	ifstream f(file.c_str());
 
 	if ( ! f.is_open() )
-		reporter->InternalError("failed to open Broxygen config file %s: %s",
+		reporter->FatalError("failed to open Broxygen config file '%s': %s",
 		                     file.c_str(), strerror(errno));
 
 	string line;
+	unsigned int line_number = 0;
 
 	while ( getline(f, line) )
 		{
+		++line_number;
 		vector<string> tokens;
 		tokenize_string(line, delim, &tokens);
 		tokens.erase(remove(tokens.begin(), tokens.end(), ""), tokens.end());
 
+		if ( tokens.empty() )
+			// Blank line.
+			continue;
+
+		if ( ! tokens[0].empty() && tokens[0][0] == '#' )
+			// Comment
+			continue;
+
 		if ( tokens.size() != 3 )
-			reporter->InternalError("malformed Broxygen target: %s",
+			reporter->FatalError("malformed Broxygen target in %s:%u: %s",
-			                        line.c_str());
+			                     file.c_str(), line_number, line.c_str());
 
 		target_factory_map::const_iterator it =
 		        target_instantiators.find(tokens[0]);
 
 		if ( it == target_instantiators.end() )
-			reporter->InternalError("unkown Broxygen target type: %s",
+			reporter->FatalError("unkown Broxygen target type: %s",
 			                     tokens[0].c_str());
 
 		targets.push_back(it->second(tokens[1], tokens[2]));
 		}
 
 	if ( f.bad() )
-		reporter->InternalError("error reading Broxygen config file %s: %s",
+		reporter->InternalError("error reading Broxygen config file '%s': %s",
 		                        file.c_str(), strerror(errno));
 	}
 
 Config::~Config()
 	{
-	for ( target_list::const_iterator it = targets.begin();
+	for ( size_t i = 0; i < targets.size(); ++i )
-	      it != targets.end(); ++it )
+		delete targets[i];
-		delete *it;
 	}
 
 void Config::FindDependencies(const vector<Document*>& docs)
 	{
-	for ( target_list::const_iterator it = targets.begin();
+	for ( size_t i = 0; i < targets.size(); ++i )
-	      it != targets.end(); ++it )
+		targets[i]->FindDependencies(docs);
-		(*it)->FindDependencies(docs);
 	}
 
 void Config::GenerateDocs() const
 	{
-	for ( target_list::const_iterator it = targets.begin();
+	for ( size_t i = 0; i < targets.size(); ++i )
-	      it != targets.end(); ++it )
+		targets[i]->Generate();
-		(*it)->Generate();
+	}
+
+time_t Config::GetModificationTime() const
+	{
+	struct stat s;
+
+	if ( stat(file.c_str(), &s) < 0 )
+		reporter->InternalError("Broxygen can't stat config file %s: %s",
+		                        file.c_str(), strerror(errno));
+
+	return s.st_mtime;
 	}

src/broxygen/Configuration.h

@@ -2,10 +2,11 @@
 #define BROXYGEN_CONFIGURATION_H
 
 #include "Document.h"
+#include "BroDoc.h"
 
 #include <string>
 #include <vector>
-#include <list>
+#include <map>
 
 namespace broxygen {
 
@@ -16,7 +17,8 @@ public:
 
 	typedef Target* (*factory_fn)(const std::string&, const std::string&);
 
-	virtual ~Target() { }
+	virtual ~Target()
+		{ }
 
 	void FindDependencies(const std::vector<Document*>& docs)
 		{ DoFindDependencies(docs); }
@@ -26,27 +28,47 @@ public:
 
 	bool MatchesPattern(Document* doc) const;
 
-	void AddDependency(Document* doc)
+	std::string Name() const
-		{ dependencies.push_back(doc); }
+		{ return name; }
+
+	std::string Pattern() const
+		{ return pattern; }
 
 protected:
 
-	Target(const std::string& arg_name, const std::string& arg_pattern)
+	Target(const std::string& arg_name, const std::string& arg_pattern);
-		: name(arg_name), pattern(arg_pattern)
-		{ }
-
-	std::string name;
-	std::string pattern;
-	std::list<Document*> dependencies;
 
 private:
 
 	virtual void DoFindDependencies(const std::vector<Document*>& docs) = 0;
 
 	virtual void DoGenerate() const = 0;
+
+	std::string name;
+	std::string pattern;
+	std::string prefix;
 };
 
-class ProtoAnalyzerTarget : public Target {
+class AnalyzerTarget : public Target {
+protected:
+
+	typedef void (*doc_creator_fn)(FILE*);
+
+	AnalyzerTarget(const std::string& name, const std::string& pattern,
+	               doc_creator_fn cb)
+		: Target(name, pattern), doc_creator_callback(cb)
+		{ }
+
+private:
+
+	void DoFindDependencies(const std::vector<Document*>& docs);
+
+	void DoGenerate() const;
+
+	doc_creator_fn doc_creator_callback;
+};
+
+class ProtoAnalyzerTarget : public AnalyzerTarget {
 public:
 
 	static Target* Instantiate(const std::string& name,
@@ -56,17 +78,11 @@ public:
 private:
 
 	ProtoAnalyzerTarget(const std::string& name, const std::string& pattern)
-		: Target(name, pattern)
+		: AnalyzerTarget(name, pattern, &CreateProtoAnalyzerDoc)
 		{ }
-
-	void DoFindDependencies(const std::vector<Document*>& docs)
-		{ /* TODO */ }
-
-	void DoGenerate() const
-		{ /* TODO */ }
 };
 
-class FileAnalyzerTarget : public Target {
+class FileAnalyzerTarget : public AnalyzerTarget {
 public:
 
 	static Target* Instantiate(const std::string& name,
@@ -76,14 +92,8 @@ public:
 private:
 
 	FileAnalyzerTarget(const std::string& name, const std::string& pattern)
-		: Target(name, pattern)
+		: AnalyzerTarget(name, pattern, &CreateFileAnalyzerDoc)
 		{ }
-
-	void DoFindDependencies(const std::vector<Document*>& docs)
-		{ /* TODO */ }
-
-	void DoGenerate() const
-		{ /* TODO */ }
 };
 
 class PackageTarget : public Target {
@@ -96,14 +106,37 @@ public:
 private:
 
 	PackageTarget(const std::string& name, const std::string& pattern)
-		: Target(name, pattern)
+		: Target(name, pattern), pkg_deps(), script_deps(), pkg_manifest()
 		{ }
 
-	void DoFindDependencies(const std::vector<Document*>& docs)
+	void DoFindDependencies(const std::vector<Document*>& docs);
-		{ /* TODO */ }
 
-	void DoGenerate() const
+	void DoGenerate() const;
-		{ /* TODO */ }
+
+	std::vector<PackageDocument*> pkg_deps;
+	std::vector<ScriptDocument*> script_deps;
+	typedef std::map<PackageDocument*,std::vector<ScriptDocument*> > manifest_t;
+	manifest_t pkg_manifest;
+};
+
+class PackageIndexTarget : public Target {
+public:
+
+	static Target* Instantiate(const std::string& name,
+	                           const std::string& pattern)
+		{ return new PackageIndexTarget(name, pattern); }
+
+private:
+
+	PackageIndexTarget(const std::string& name, const std::string& pattern)
+		: Target(name, pattern), pkg_deps()
+		{ }
+
+	void DoFindDependencies(const std::vector<Document*>& docs);
+
+	void DoGenerate() const;
+
+	std::vector<PackageDocument*> pkg_deps;
 };
 
 class ScriptTarget : public Target {
@@ -113,17 +146,51 @@ public:
 	                           const std::string& pattern)
 		{ return new ScriptTarget(name, pattern); }
 
-private:
+protected:
 
 	ScriptTarget(const std::string& name, const std::string& pattern)
-		: Target(name, pattern)
+		: Target(name, pattern), script_deps()
 		{ }
 
-	void DoFindDependencies(const std::vector<Document*>& docs)
+	std::vector<ScriptDocument*> script_deps;
-		{ /* TODO */ }
 
-	void DoGenerate() const
+private:
-		{ /* TODO */ }
+
+	void DoFindDependencies(const std::vector<Document*>& docs);
+
+	void DoGenerate() const;
+};
+
+class ScriptSummaryTarget : public ScriptTarget {
+public:
+
+	static Target* Instantiate(const std::string& name,
+	                           const std::string& pattern)
+		{ return new ScriptSummaryTarget(name, pattern); }
+
+private:
+
+	ScriptSummaryTarget(const std::string& name, const std::string& pattern)
+		: ScriptTarget(name, pattern)
+		{ }
+
+	void DoGenerate() const /* override */;
+};
+
+class ScriptIndexTarget : public ScriptTarget {
+public:
+
+	static Target* Instantiate(const std::string& name,
+	                           const std::string& pattern)
+		{ return new ScriptIndexTarget(name, pattern); }
+
+private:
+
+	ScriptIndexTarget(const std::string& name, const std::string& pattern)
+		: ScriptTarget(name, pattern)
+		{ }
+
+	void DoGenerate() const /* override */;
 };
 
 class IdentifierTarget : public Target {
@@ -136,13 +203,14 @@ public:
 private:
 
 	IdentifierTarget(const std::string& name, const std::string& pattern)
-		: Target(name, pattern)
+		: Target(name, pattern), id_deps()
 		{ }
 
 	void DoFindDependencies(const std::vector<Document*>& docs);
 
-	void DoGenerate() const
+	void DoGenerate() const;
-		{ /* TODO */ }
+
+	std::vector<IdentifierDocument*> id_deps;
 };
 
 class Config {
@@ -156,11 +224,12 @@ public:
 
 	void GenerateDocs() const;
 
+	time_t GetModificationTime() const;
+
 private:
 
-	typedef std::list<Target*> target_list;
+	std::string file;
-
+	std::vector<Target*> targets;
-	target_list targets;
 };
 
 

src/broxygen/Document.cc

@@ -1,38 +1,134 @@
 #include "Document.h"
+#include "Manager.h"
 
 #include "util.h"
 #include "Val.h"
+#include "Desc.h"
+#include "Reporter.h"
+
+#include <fstream>
+#include <sys/stat.h>
 
 using namespace broxygen;
 using namespace std;
 
-static string ImplodeStringVec(const vector<string>& v)
+static bool is_public_api(const ID* id)
 	{
-	string rval;
+	return (id->Scope() == SCOPE_GLOBAL) ||
-
+	       (id->Scope() == SCOPE_MODULE && id->IsExport());
-	for ( size_t i = 0; i < v.size(); ++i )
-		{
-		if ( i > 0 )
-			rval += '\n';
-
-		rval += v[i];
 	}
 
-	return rval;
+static bool prettify_params(string& s)
+	{
+	size_t identifier_start_pos = 0;
+	bool in_identifier = false;
+	string identifier;
+
+	for ( size_t i = 0; i < s.size(); ++i )
+		{
+		char next = s[i];
+
+		if ( ! in_identifier )
+			{
+			// Pass by leading whitespace.
+			if ( isspace(next) )
+				continue;
+
+			// Only allow alphabetic and '_' as first char of identifier.
+			if ( isalpha(next) || next == '_' )
+				{
+				identifier_start_pos = i;
+				identifier += next;
+				in_identifier = true;
+				continue;
+				}
+
+			// Don't need to change anything.
+			return false;
+			}
+
+		// All other characters of identifier are alphanumeric or '_'.
+		if ( isalnum(next) || next == '_' )
+			{
+			identifier += next;
+			continue;
+			}
+
+		if ( next == ':' )
+			{
+			if ( i + 1 < s.size() && s[i + 1] == ':' )
+				{
+				// It's part of an identifier's namespace scoping.
+				identifier += next;
+				identifier += s[i + 1];
+				++i;
+				continue;
+				}
+
+			// Prettify function param/return value reST markup.
+			string subst;
+
+			if ( identifier == "Returns" )
+				subst = ":returns";
+			else
+				subst = ":param " + identifier;
+
+			s.replace(identifier_start_pos, identifier.size(), subst);
+			return true;
+			}
+
+		// Don't need to change anything.
+		return false;
+		}
+
+	return false;
+	}
+
+static string make_heading(const string& heading, char underline)
+	{
+	return heading + "\n" + string(heading.size(), underline) + "\n";
 	}
 
 PackageDocument::PackageDocument(const string& arg_name)
     : Document(),
-      pkg_name(arg_name)
+      pkg_name(arg_name), readme()
 	{
-	// TODO: probably need to determine modification times of all files
+	string readme_file = find_file(pkg_name + "/README", bro_path());
-	//       within the directory, recursively
+
+	if ( readme_file.empty() )
+		return;
+
+	ifstream f(readme_file.c_str());
+
+	if ( ! f.is_open() )
+		reporter->InternalWarning("Broxygen failed to open '%s': %s",
+		                          readme_file.c_str(), strerror(errno));
+
+	string line;
+
+	while ( getline(f, line) )
+		readme.push_back(line);
+
+	if ( f.bad() )
+		reporter->InternalWarning("Broxygen error reading '%s': %s",
+		                          readme_file.c_str(), strerror(errno));
 	}
 
-IdentifierDocument::IdentifierDocument(ID* arg_id)
+string PackageDocument::DoReStructuredText(bool roles_only) const
+	{
+	string rval = fmt(":doc:`%s <%s/index>`\n\n", pkg_name.c_str(),
+	                  pkg_name.c_str());
+
+	for ( size_t i = 0; i < readme.size(); ++i )
+		rval += "   " + readme[i] + "\n";
+
+	return rval;
+	}
+
+IdentifierDocument::IdentifierDocument(ID* arg_id, ScriptDocument* script)
     : Document(),
       comments(), id(arg_id), initial_val_desc(), redefs(), fields(),
-      last_field_seen()
+      last_field_seen(), declaring_script(script)
 	{
 	Ref(id);
 
@@ -86,36 +182,539 @@ void IdentifierDocument::AddRecordField(const TypeDecl* field,
 	last_field_seen = rf;
 	}
 
-string IdentifierDocument::GetComments() const
+vector<string> IdentifierDocument::GetComments() const
 	{
-	return ImplodeStringVec(comments);
+	return comments;
 	}
 
-string IdentifierDocument::GetFieldComments(const string& field) const
+vector<string> IdentifierDocument::GetFieldComments(const string& field) const
 	{
 	record_field_map::const_iterator it = fields.find(field);
 
 	if ( it == fields.end() )
-		return string();
+		return vector<string>();
 
-	return ImplodeStringVec(it->second->comments);
+	return it->second->comments;
 	}
 
-ScriptDocument::ScriptDocument(const string& arg_name)
+list<IdentifierDocument::Redefinition>
+IdentifierDocument::GetRedefs(const string& from_script) const
+	{
+	list<Redefinition> rval;
+
+	for ( redef_list::const_iterator it = redefs.begin(); it != redefs.end();
+	      ++it )
+		{
+		if ( from_script == (*it)->from_script )
+			rval.push_back(*(*it));
+		}
+
+	return rval;
+	}
+
+string IdentifierDocument::GetDeclaringScriptForField(const string& field) const
+	{
+	record_field_map::const_iterator it = fields.find(field);
+
+	if ( it == fields.end() )
+		return "";
+
+	return it->second->from_script;
+	}
+
+string IdentifierDocument::DoReStructuredText(bool roles_only) const
+	{
+	ODesc d;
+	d.SetIndentSpaces(3);
+	d.SetQuotes(true);
+	id->DescribeReST(&d, roles_only);
+
+	if ( comments.empty() )
+		return d.Description();
+
+	d.ClearIndentLevel();
+	d.PushIndent();
+
+	for ( size_t i = 0; i < comments.size(); ++i )
+		{
+		if ( i > 0 )
+			d.NL();
+
+		if ( IsFunc(id->Type()->Tag()) )
+			{
+			string s = comments[i];
+
+			if ( prettify_params(s) )
+				d.NL();
+
+			d.Add(s.c_str());
+			}
+		else
+			d.Add(comments[i].c_str());
+		}
+
+	return d.Description();
+	}
+
+ScriptDocument::ScriptDocument(const string& arg_name, const string& arg_path)
     : Document(),
-      name(arg_name),
+      name(arg_name), path(arg_path),
       is_pkg_loader(SafeBasename(name).result == PACKAGE_LOADER),
-      dependencies(), module_usages(), comments(), identifier_docs(), redefs()
+      dependencies(), module_usages(), comments(), identifier_docs(),
+      options(), constants(), state_vars(), types(), events(), hooks(),
+      functions(), redefs()
 	{
 	}
 
 void ScriptDocument::AddIdentifierDoc(IdentifierDocument* doc)
 	{
 	identifier_docs[doc->Name()] = doc;
-	// TODO: sort things (e.g. function flavor, state var vs. option var)
 	}
 
-string ScriptDocument::GetComments() const
+void ScriptDocument::DoInitPostScript()
 	{
-	return ImplodeStringVec(comments);
+	for ( id_doc_map::const_iterator it = identifier_docs.begin();
+	      it != identifier_docs.end(); ++it )
+		{
+		IdentifierDocument* doc = it->second;
+		ID* id = doc->GetID();
+
+		if ( ! is_public_api(id) )
+			continue;
+
+		if ( id->AsType() )
+			{
+			types.push_back(doc);
+			DBG_LOG(DBG_BROXYGEN, "Filter id '%s' in '%s' as a type",
+			        id->Name(), name.c_str());
+			continue;
+			}
+
+		if ( IsFunc(id->Type()->Tag()) )
+			{
+			switch ( id->Type()->AsFuncType()->Flavor() ) {
+			case FUNC_FLAVOR_HOOK:
+				DBG_LOG(DBG_BROXYGEN, "Filter id '%s' in '%s' as a hook",
+				        id->Name(), name.c_str());
+				hooks.push_back(doc);
+				break;
+			case FUNC_FLAVOR_EVENT:
+				DBG_LOG(DBG_BROXYGEN, "Filter id '%s' in '%s' as a event",
+				        id->Name(), name.c_str());
+				events.push_back(doc);
+				break;
+			case FUNC_FLAVOR_FUNCTION:
+				DBG_LOG(DBG_BROXYGEN, "Filter id '%s' in '%s' as a function",
+				        id->Name(), name.c_str());
+				functions.push_back(doc);
+				break;
+			default:
+				reporter->InternalError("Invalid function flavor");
+				break;
+			}
+
+			continue;
+			}
+
+		if ( id->IsConst() )
+			{
+			if ( id->FindAttr(ATTR_REDEF) )
+				{
+				DBG_LOG(DBG_BROXYGEN, "Filter id '%s' in '%s' as an option",
+				        id->Name(), name.c_str());
+				options.push_back(doc);
+				}
+			else
+				{
+				DBG_LOG(DBG_BROXYGEN, "Filter id '%s' in '%s' as a constant",
+				        id->Name(), name.c_str());
+				constants.push_back(doc);
+				}
+
+			continue;
+			}
+
+		if ( id->Type()->Tag() == TYPE_ENUM )
+			// Enums are always referenced/documented from the type's
+			// documentation.
+			continue;
+
+		DBG_LOG(DBG_BROXYGEN, "Filter id '%s' in '%s' as a state variable",
+		        id->Name(), name.c_str());
+		state_vars.push_back(doc);
+		}
+	}
+
+vector<string> ScriptDocument::GetComments() const
+	{
+	return comments;
+	}
+
+static size_t end_of_first_sentence(const string& s)
+	{
+	size_t rval = 0;
+
+	while ( (rval = s.find_first_of('.', rval)) != string::npos )
+		{
+		if ( rval == s.size() - 1 )
+			// Period is at end of string.
+			return rval;
+
+		if ( isspace(s[rval + 1]) )
+			// Period has a space after it.
+			return rval;
+
+		// Period has some non-space character after it, keep looking.
+		++rval;
+		}
+
+	return rval;
+	}
+
+static bool is_all_whitespace(const string& s)
+	{
+	for ( size_t i = 0; i < s.size(); ++i )
+		if ( ! isspace(s[i]) )
+			return false;
+
+	return true;
+	}
+
+static vector<string> summary_comment(const vector<string>& cmnts)
+	{
+	vector<string> rval;
+
+	for ( size_t i = 0; i < cmnts.size(); ++i )
+		{
+		size_t end = end_of_first_sentence(cmnts[i]);
+
+		if ( end == string::npos )
+			{
+			if ( is_all_whitespace(cmnts[i]) )
+				break;
+
+			rval.push_back(cmnts[i]);
+			}
+		else
+			{
+			rval.push_back(cmnts[i].substr(0, end + 1));
+			break;
+			}
+		}
+
+	return rval;
+	}
+
+class ReStructuredTextTable {
+public:
+
+	ReStructuredTextTable(size_t arg_num_cols)
+		: num_cols(arg_num_cols), rows(), longest_row_in_column()
+		{
+		for ( size_t i = 0; i < num_cols; ++i )
+			longest_row_in_column.push_back(1);
+		}
+
+	void AddRow(const vector<string>& new_row)
+		{
+		assert(new_row.size() == num_cols);
+		rows.push_back(new_row);
+
+		for ( size_t i = 0; i < new_row.size(); ++i )
+			if ( new_row[i].size() > longest_row_in_column[i] )
+				longest_row_in_column[i] = new_row[i].size();
+		}
+
+	static string MakeBorder(const vector<size_t> col_sizes, char border)
+		{
+		string rval;
+
+		for ( size_t i = 0; i < col_sizes.size(); ++i )
+			{
+			if ( i > 0 )
+				rval += " ";
+
+			rval += string(col_sizes[i], border);
+			}
+
+		rval += "\n";
+		return rval;
+		}
+
+	string AsString(char border) const
+		{
+		string rval = MakeBorder(longest_row_in_column, border);
+
+		for ( size_t row = 0; row < rows.size(); ++row )
+			{
+			for ( size_t col = 0; col < num_cols; ++col )
+				{
+				if ( col > 0 )
+					{
+					size_t last = rows[row][col - 1].size();
+					size_t longest = longest_row_in_column[col - 1];
+					size_t whitespace = longest - last + 1;
+					rval += string(whitespace, ' ');
+					}
+
+				rval += rows[row][col];
+				}
+
+			rval += "\n";
+			}
+
+		rval += MakeBorder(longest_row_in_column, border);
+		return rval;
+		}
+
+
+private:
+
+	size_t num_cols;
+	vector<vector<string> > rows;
+	vector<size_t> longest_row_in_column;
+};
+
+static void add_summary_rows(const ODesc& id_desc, const vector<string>& cmnts,
+                             ReStructuredTextTable* table)
+	{
+	vector<string> row;
+	row.push_back(id_desc.Description());
+
+	if ( cmnts.empty() )
+		{
+		row.push_back("");
+		table->AddRow(row);
+		return;
+		}
+
+	row.push_back(cmnts[0]);
+	table->AddRow(row);
+
+	for ( size_t i = 1; i < cmnts.size(); ++i )
+		{
+		row.clear();
+		row.push_back("");
+		row.push_back(cmnts[i]);
+		table->AddRow(row);
+		}
+	}
+
+static string make_summary(const string& heading, char underline, char border,
+                           const list<IdentifierDocument*>& id_list)
+	{
+	if ( id_list.empty() )
+		return "";
+
+	ReStructuredTextTable table(2);
+
+	for ( list<IdentifierDocument*>::const_iterator it = id_list.begin();
+	      it != id_list.end(); ++it )
+		{
+		ID* id = (*it)->GetID();
+		ODesc d;
+		d.SetQuotes(1);
+		id->DescribeReSTShort(&d);
+		add_summary_rows(d, summary_comment((*it)->GetComments()), &table);
+		}
+
+	return make_heading(heading, underline) + table.AsString(border) + "\n";
+	}
+
+static string make_redef_summary(const string& heading, char underline,
+                                 char border, const string& from_script,
+                                 const set<IdentifierDocument*>& id_set)
+	{
+	if ( id_set.empty() )
+		return "";
+
+	ReStructuredTextTable table(2);
+
+	for ( set<IdentifierDocument*>::const_iterator it = id_set.begin();
+	      it != id_set.end(); ++it )
+		{
+		ID* id = (*it)->GetID();
+		ODesc d;
+		d.SetQuotes(1);
+		id->DescribeReSTShort(&d);
+
+		typedef list<IdentifierDocument::Redefinition> redef_list;
+		redef_list redefs = (*it)->GetRedefs(from_script);
+
+		for ( redef_list::const_iterator iit = redefs.begin();
+		      iit != redefs.end(); ++iit )
+			add_summary_rows(d, summary_comment(iit->comments), &table);
+		}
+
+	return make_heading(heading, underline) + table.AsString(border) + "\n";
+	}
+
+static string make_details(const string& heading, char underline,
+                           const list<IdentifierDocument*>& id_list)
+	{
+	if ( id_list.empty() )
+		return "";
+
+	string rval = make_heading(heading, underline);
+
+	for ( list<IdentifierDocument*>::const_iterator it = id_list.begin();
+	      it != id_list.end(); ++it )
+		{
+		rval += (*it)->ReStructuredText();
+		rval += "\n\n";
+		}
+
+	return rval;
+	}
+
+static string make_redef_details(const string& heading, char underline,
+                                 const set<IdentifierDocument*>& id_set)
+	{
+	if ( id_set.empty() )
+		return "";
+
+	string rval = make_heading(heading, underline);
+
+	for ( set<IdentifierDocument*>::const_iterator it = id_set.begin();
+	      it != id_set.end(); ++it )
+		{
+		rval += (*it)->ReStructuredText(true);
+		rval += "\n\n";
+		}
+
+	return rval;
+	}
+
+string ScriptDocument::DoReStructuredText(bool roles_only) const
+	{
+	string rval;
+
+	rval += ":tocdepth: 3\n\n";
+	rval += make_heading(name, '=');
+
+	for ( string_set::const_iterator it = module_usages.begin();
+	      it != module_usages.end(); ++it )
+		rval += ".. bro:namespace:: " + *it + "\n";
+
+	rval += "\n";
+
+	for ( size_t i = 0; i < comments.size(); ++i )
+		rval += comments[i] + "\n";
+
+	rval += "\n";
+
+	if ( ! module_usages.empty() )
+		{
+		rval += module_usages.size() > 1 ? ":Namespaces: " : ":Namespace: ";
+
+		for ( string_set::const_iterator it = module_usages.begin();
+		      it != module_usages.end(); ++it )
+			{
+			if ( it != module_usages.begin() )
+				rval += ", ";
+
+			rval += *it;
+			}
+
+		rval += "\n";
+		}
+
+	if ( ! dependencies.empty() )
+		{
+		rval += ":Imports: ";
+
+		for ( string_set::const_iterator it = dependencies.begin();
+		      it != dependencies.end(); ++it )
+			{
+			if ( it != dependencies.begin() )
+				rval += ", ";
+
+			rval += fmt(":doc:`%s </scripts/%s>`", it->c_str(), it->c_str());
+			// TODO linking to packages is a bit different?
+			}
+
+		rval += "\n";
+		}
+
+	// TODO: make this an absolute path?
+	rval += fmt(":Source File: :download:`%s`\n", name.c_str());
+	rval += "\n";
+	rval += make_heading("Summary", '~');
+	rval += make_summary("Options", '#', '=', options);
+	rval += make_summary("Constants", '#', '=', constants);
+	rval += make_summary("State Variables", '#', '=', state_vars);
+	rval += make_summary("Types", '#', '=', types);
+	rval += make_redef_summary("Redefinitions", '#', '=', name, redefs);
+	rval += make_summary("Events", '#', '=', events);
+	rval += make_summary("Hooks", '#', '=', hooks);
+	rval += make_summary("Functions", '#', '=', functions);
+	rval += "\n";
+	rval += make_heading("Detailed Interface", '~');
+	rval += make_details("Options", '#', options);
+	rval += make_details("Constants", '#', constants);
+	rval += make_details("State Variables", '#', state_vars);
+	rval += make_details("Types", '#', types);
+	//rval += make_redef_details("Redefinitions", '#', redefs);
+	rval += make_details("Events", '#', events);
+	rval += make_details("Hooks", '#', hooks);
+	rval += make_details("Functions", '#', functions);
+
+	return rval;
+	}
+
+static time_t get_mtime(const string& filename)
+	{
+	struct stat s;
+
+	if ( stat(filename.c_str(), &s) < 0 )
+		reporter->InternalError("Broxygen failed to stat file '%s': %s",
+		                        filename.c_str(), strerror(errno));
+
+	return s.st_mtime;
+	}
+
+time_t IdentifierDocument::DoGetModificationTime() const
+	{
+	// Could probably get away with just checking the set of scripts that
+	// contributed to the ID declaration/redefinitions, but this is easier...
+	return declaring_script->GetModificationTime();
+	}
+
+time_t ScriptDocument::DoGetModificationTime() const
+	{
+	time_t most_recent = get_mtime(path);
+
+	for ( string_set::const_iterator it = dependencies.begin();
+	      it != dependencies.end(); ++it )
+		{
+		Document* doc = broxygen_mgr->GetScriptDoc(*it);
+
+		if ( ! doc )
+			{
+			string pkg_name = *it + "/" + PACKAGE_LOADER;
+			doc = broxygen_mgr->GetScriptDoc(pkg_name);
+
+			if ( ! doc )
+				reporter->InternalWarning("Broxygen failed to get mtime of %s",
+				                          it->c_str());
+			continue;
+			}
+
+		time_t dep_mtime = doc->GetModificationTime();
+
+		if ( dep_mtime > most_recent )
+			most_recent = dep_mtime;
+		}
+
+	return most_recent;
+	}
+
+time_t PackageDocument::DoGetModificationTime() const
+	{
+	string readme_file = find_file(pkg_name + "/README", bro_path());
+
+	if ( readme_file.empty() )
+		return 0;
+
+	return get_mtime(readme_file);
 	}

src/broxygen/Document.h

@@ -7,7 +7,7 @@
 #include <vector>
 #include <set>
 #include <map>
-#include <time.h>
+#include <ctime>
 
 #include "ID.h"
 #include "Type.h"
@@ -32,10 +32,19 @@ public:
 	std::string Name() const
 		{ return DoName(); }
 
+	std::string ReStructuredText(bool roles_only = false) const
+		{ return DoReStructuredText(roles_only); }
+
+	void InitPostScript()
+		{ return DoInitPostScript(); }
+
 private:
 
 	virtual time_t DoGetModificationTime() const = 0;
 	virtual std::string DoName() const = 0;
+	virtual std::string DoReStructuredText(bool roles_only) const = 0;
+	virtual void DoInitPostScript()
+		{ }
 };
 
 class PackageDocument : public Document {
@@ -44,28 +53,29 @@ public:
 
 	PackageDocument(const std::string& name);
 
-	// TODO: can be comments from package README
+	std::vector<std::string> GetReadme() const
-	std::string GetReadme() const
+		{ return readme; }
-		{ return std::string(); }
 
 private:
 
-	// TODO
+	time_t DoGetModificationTime() const;
-	time_t DoGetModificationTime() const
-		{ return 0; }
 
 	std::string DoName() const
 		{ return pkg_name; }
 
+	std::string DoReStructuredText(bool roles_only) const;
+
 	std::string pkg_name;
+	std::vector<std::string> readme;
 };
 
 
-class IdentifierDocument : public Document {
+class ScriptDocument;
 
+class IdentifierDocument : public Document {
 public:
 
-	IdentifierDocument(ID* id);
+	IdentifierDocument(ID* id, ScriptDocument* script);
 
 	~IdentifierDocument();
 
@@ -88,24 +98,31 @@ public:
 	ID* GetID() const
 		{ return id; }
 
-	std::string GetComments() const;
+	ScriptDocument* GetDeclaringScript() const
+	    { return declaring_script; }
 
-	std::string GetFieldComments(const std::string& field) const;
+	std::string GetDeclaringScriptForField(const std::string& field) const;
+
+	std::vector<std::string> GetComments() const;
+
+	std::vector<std::string> GetFieldComments(const std::string& field) const;
+
+	struct Redefinition {
+		std::string from_script;
+		std::string new_val_desc;
+		std::vector<std::string> comments;
+	};
+
+	std::list<Redefinition> GetRedefs(const std::string& from_script) const;
 
 private:
 
-	// TODO
+	time_t DoGetModificationTime() const;
-	time_t DoGetModificationTime() const
-		{ return 0; }
 
 	std::string DoName() const
 		{ return id->Name(); }
 
-	struct Redefinition {
+	std::string DoReStructuredText(bool roles_only) const;
-		std::string from_script;
-		string new_val_desc;
-		std::vector<std::string> comments;
-	};
 
 	struct RecordField {
 		~RecordField()
@@ -121,17 +138,18 @@ private:
 
 	std::vector<std::string> comments;
 	ID* id;
-	string initial_val_desc;
+	std::string initial_val_desc;
 	redef_list redefs;
 	record_field_map fields;
 	RecordField* last_field_seen;
+	ScriptDocument* declaring_script;
 };
 
 class ScriptDocument : public Document {
 
 public:
 
-	ScriptDocument(const std::string& name);
+	ScriptDocument(const std::string& name, const std::string& path);
 
 	void AddComment(const std::string& comment)
 		{ comments.push_back(comment); }
@@ -145,32 +163,44 @@ public:
 	void AddIdentifierDoc(IdentifierDocument* doc);
 
 	void AddRedef(IdentifierDocument* doc)
-		{ redefs.push_back(doc); }
+		{ redefs.insert(doc); }
 
 	bool IsPkgLoader() const
 		{ return is_pkg_loader; }
 
-	std::string GetComments() const;
+	std::vector<std::string> GetComments() const;
 
 private:
 
-	typedef std::map<std::string, IdentifierDocument*> IdentifierDocMap;
+	typedef std::map<std::string, IdentifierDocument*> id_doc_map;
-	typedef std::list<IdentifierDocument*> IdentifierDocList;
+	typedef std::list<IdentifierDocument*> id_doc_list;
+	typedef std::set<std::string> string_set;
+	typedef std::set<IdentifierDocument*> doc_set;
 
-	// TODO
+	time_t DoGetModificationTime() const;
-	time_t DoGetModificationTime() const
-		{ return 0; }
 
 	std::string DoName() const
 		{ return name; }
 
+	std::string DoReStructuredText(bool roles_only) const;
+
+	void DoInitPostScript() /* override */;
+
 	std::string name;
+	std::string path;
 	bool is_pkg_loader;
-	std::set<std::string> dependencies;
+	string_set dependencies;
-	std::set<std::string> module_usages;
+	string_set module_usages;
 	std::vector<std::string> comments;
-	IdentifierDocMap identifier_docs;
+	id_doc_map identifier_docs;
-	IdentifierDocList redefs;
+	id_doc_list options;
+	id_doc_list constants;
+	id_doc_list state_vars;
+	id_doc_list types;
+	id_doc_list events;
+	id_doc_list hooks;
+	id_doc_list functions;
+	doc_set redefs;
 };
 
 

src/broxygen/Manager.cc

@@ -1,8 +1,8 @@
 #include "Manager.h"
-#include "Reporter.h"
 #include "util.h"
 
 #include <utility>
+#include <cstdlib>
 
 using namespace broxygen;
 using namespace std;
@@ -25,71 +25,22 @@ static string RemoveLeadingSpace(const string& s)
 	return rval;
 	}
 
-static string PrettifyParams(const string& s)
+Manager::Manager(const string& arg_config, const string& bro_command)
-	{
-	size_t identifier_start_pos = 0;
-	bool in_identifier = false;
-	string identifier;
-
-	for ( size_t i = 0; i < s.size(); ++i )
-		{
-		char next = s[i];
-
-		if ( ! in_identifier )
-			{
-			// Pass by leading whitespace.
-			if ( isspace(next) )
-				continue;
-
-			// Only allow alphabetic and '_' as first char of identifier.
-			if ( isalpha(next) || next == '_' )
-				{
-				identifier_start_pos = i;
-				identifier += next;
-				in_identifier = true;
-				continue;
-				}
-
-			// Don't need to change anything.
-			return s;
-			}
-
-		// All other character of identifier are alphanumeric or '_'.
-		if ( isalnum(next) || next == '_' )
-			{
-			identifier += next;
-			continue;
-			}
-
-		// Prettify param and return value docs for a function's reST markup.
-		if ( next == ':' )
-			{
-			string rval = s;
-			string subst;
-
-			if ( identifier == "Returns" )
-				subst = "\n:returns";
-			else
-				subst = "\n:param " + identifier;
-
-			rval.replace(identifier_start_pos, identifier.size(), subst);
-			return rval;
-			}
-
-		// Don't need to change anything.
-		return s;
-		}
-
-	return s;
-	}
-
-Manager::Manager(const string& arg_config)
     : disabled(), comment_buffer(), comment_buffer_map(), packages(), scripts(),
       identifiers(), all_docs(), last_identifier_seen(), incomplete_type(),
-      config(arg_config)
+      enum_mappings(), config(arg_config), bro_mtime()
 	{
 	if ( getenv("BRO_DISABLE_BROXYGEN") )
 		disabled = true;
+
+	string path_to_bro = find_file(bro_command, getenv("PATH"));
+	struct stat s;
+
+	if ( path_to_bro.empty() || stat(path_to_bro.c_str(), &s) < 0 )
+		reporter->InternalError("Broxygen can't get mtime of bro binary %s: %s",
+		                        path_to_bro.c_str(), strerror(errno));
+
+	bro_mtime = s.st_mtime;
 	}
 
 Manager::~Manager()
@@ -102,7 +53,6 @@ void Manager::InitPreScript()
 	{
 	if ( disabled )
 		return;
-	// TODO: create file/proto analyzer doc
 	}
 
 void Manager::InitPostScript()
@@ -110,6 +60,9 @@ void Manager::InitPostScript()
 	if ( disabled )
 		return;
 
+	for ( size_t i = 0; i < all_docs.size(); ++i )
+		all_docs[i]->InitPostScript();
+
 	config.FindDependencies(all_docs);
 	}
 
@@ -134,7 +87,7 @@ void Manager::File(const string& path)
 		return;
 		}
 
-	ScriptDocument* doc = new ScriptDocument(name);
+	ScriptDocument* doc = new ScriptDocument(name, path);
 	scripts.map[name] = doc;
 	all_docs.push_back(doc);
 	DBG_LOG(DBG_BROXYGEN, "Made ScriptDocument %s", name.c_str());
@@ -209,7 +162,7 @@ void Manager::ModuleUsage(const string& path, const string& module)
 
 IdentifierDocument* Manager::CreateIdentifierDoc(ID* id, ScriptDocument* script)
 	{
-	IdentifierDocument* rval = new IdentifierDocument(id);
+	IdentifierDocument* rval = new IdentifierDocument(id, script);
 
 	rval->AddComments(comment_buffer);
 	comment_buffer.clear();
@@ -226,7 +179,10 @@ IdentifierDocument* Manager::CreateIdentifierDoc(ID* id, ScriptDocument* script)
 	all_docs.push_back(rval);
 	identifiers.map[id->Name()] = rval;
 	last_identifier_seen = rval;
+
+	if ( script )
 		script->AddIdentifierDoc(rval);
+
 	return rval;
 	}
 
@@ -256,18 +212,9 @@ void Manager::StartType(ID* id)
 	        id->Name(), script.c_str());
 	}
 
-void Manager::StartRedef(ID* id)
+static bool IsEnumType(ID* id)
 	{
-	if ( disabled )
+	return id->AsType() ? id->AsType()->Tag() == TYPE_ENUM : false;
-		return;
-
-	IdentifierDocument* id_doc = identifiers.GetDocument(id->Name());
-
-	if ( id_doc )
-		last_identifier_seen = id_doc;
-	else
-		DbgAndWarn(fmt("Broxygen redef tracking unknown identifier: %s",
-		               id->Name()));
 	}
 
 void Manager::Identifier(ID* id)
@@ -275,7 +222,9 @@ void Manager::Identifier(ID* id)
 	if ( disabled )
 		return;
 
-	if ( incomplete_type && incomplete_type->Name() == id->Name() )
+	if ( incomplete_type )
+		{
+		if ( incomplete_type->Name() == id->Name() )
 			{
 			DBG_LOG(DBG_BROXYGEN, "Finished document for type %s", id->Name());
 			incomplete_type->CompletedTypeDecl();
@@ -283,13 +232,8 @@ void Manager::Identifier(ID* id)
 			return;
 			}
 
-	if ( id->GetLocationInfo() == &no_location )
+		if ( IsEnumType(incomplete_type->GetID()) )
-		{
+			enum_mappings[id->Name()] = incomplete_type->GetID()->Name();
-		// Internally-created identifier (e.g. file/proto analyzer enum tags).
-		// Can be ignored here as they need to be documented via other means.
-		DBG_LOG(DBG_BROXYGEN, "Skip documenting identifier %s: no location",
-		        id->Name());
-		return;
 		}
 
 	IdentifierDocument* id_doc = identifiers.GetDocument(id->Name());
@@ -308,6 +252,16 @@ void Manager::Identifier(ID* id)
 		return;
 		}
 
+	if ( id->GetLocationInfo() == &no_location )
+		{
+		// Internally-created identifier (e.g. file/proto analyzer enum tags).
+		// Handled specially since they don't have a script location.
+		DBG_LOG(DBG_BROXYGEN, "Made internal IdentifierDocument %s",
+		        id->Name());
+		CreateIdentifierDoc(id, 0);
+		return;
+		}
+
 	string script = without_bropath_component(id->GetLocationInfo()->filename);
 	ScriptDocument* script_doc = scripts.GetDocument(script);
 
@@ -428,32 +382,8 @@ void Manager::PostComment(const string& comment, const string& id_hint)
 		comment_buffer_map[id_hint].push_back(RemoveLeadingSpace(comment));
 	}
 
-StringVal* Manager::GetIdentifierComments(const string& name) const
+string Manager::GetEnumTypeName(const string& id) const
 	{
-	IdentifierDocument* d = identifiers.GetDocument(name);
+	map<string, string>::const_iterator it = enum_mappings.find(id);
-	return new StringVal(d ? d->GetComments() : "");
+	return it == enum_mappings.end() ? "" : it->second;
-	}
-
-StringVal* Manager::GetScriptComments(const string& name) const
-	{
-	ScriptDocument* d = scripts.GetDocument(name);
-	return new StringVal(d ? d->GetComments() : "");
-	}
-
-StringVal* Manager::GetPackageReadme(const string& name) const
-	{
-	PackageDocument* d = packages.GetDocument(name);
-	return new StringVal(d ? d->GetReadme() : "");
-	}
-
-StringVal* Manager::GetRecordFieldComments(const string& name) const
-	{
-	size_t i = name.find('$');
-
-	if ( i > name.size() - 2 )
-		// '$' is last char in string or not found.
-		return new StringVal("");
-
-	IdentifierDocument* d = identifiers.GetDocument(name.substr(0, i));
-	return new StringVal(d ? d->GetFieldComments(name.substr(i + 1)) : "");
 	}

src/broxygen/Manager.h

@@ -3,6 +3,8 @@
 
 #include "Configuration.h"
 #include "Document.h"
+
+#include "Reporter.h"
 #include "ID.h"
 #include "Type.h"
 #include "Val.h"
@@ -10,6 +12,8 @@
 #include <string>
 #include <vector>
 #include <map>
+#include <ctime>
+#include <sys/stat.h>
 
 namespace broxygen {
 
@@ -32,7 +36,7 @@ class Manager {
 
 public:
 
-	Manager(const std::string& config);
+	Manager(const std::string& config, const std::string& bro_command);
 
 	~Manager();
 
@@ -50,8 +54,6 @@ public:
 
 	void StartType(ID* id);
 
-	void StartRedef(ID* id);
-
 	void Identifier(ID* id);
 
 	void RecordField(const ID* id, const TypeDecl* field,
@@ -66,13 +68,20 @@ public:
 	void PostComment(const std::string& comment,
 	                 const std::string& identifier_hint = "");
 
-	StringVal* GetIdentifierComments(const std::string& name) const;
+	std::string GetEnumTypeName(const std::string& id) const;
 
-	StringVal* GetScriptComments(const std::string& name) const;
+	IdentifierDocument* GetIdentifierDoc(const std::string& name) const
+	    { return identifiers.GetDocument(name); }
 
-	StringVal* GetPackageReadme(const std::string& name) const;
+	ScriptDocument* GetScriptDoc(const std::string& name) const
+	    { return scripts.GetDocument(name); }
 
-	StringVal* GetRecordFieldComments(const std::string& name) const;
+	PackageDocument* GetPackageDoc(const std::string& name) const
+	    { return packages.GetDocument(name); }
+
+	template <class T>
+	bool IsUpToDate(const std::string& target_file,
+	                const std::vector<T*>& dependencies) const;
 
 private:
 
@@ -90,9 +99,40 @@ private:
 	std::vector<Document*> all_docs;
 	IdentifierDocument* last_identifier_seen;
 	IdentifierDocument* incomplete_type;
+	std::map<std::string, std::string> enum_mappings;
 	Config config;
+	time_t bro_mtime;
 };
 
+template <class T>
+bool Manager::IsUpToDate(const string& target_file,
+                         const vector<T*>& dependencies) const
+	{
+	struct stat s;
+
+	if ( stat(target_file.c_str(), &s) < 0 )
+		{
+		if ( errno == ENOENT )
+			// Doesn't exist.
+			return false;
+
+		reporter->InternalError("Broxygen failed to stat target file '%s': %s",
+		                        target_file.c_str(), strerror(errno));
+		}
+
+	if ( difftime(bro_mtime, s.st_mtime) > 0 )
+		return false;
+
+	if ( difftime(config.GetModificationTime(), s.st_mtime) > 0 )
+		return false;
+
+	for ( size_t i = 0; i < dependencies.size(); ++i )
+		if ( difftime(dependencies[i]->GetModificationTime(), s.st_mtime) > 0 )
+			return false;
+
+	return true;
+	}
+
 } // namespace broxygen
 
 extern broxygen::Manager* broxygen_mgr;

src/broxygen/broxygen.bif

@@ -2,26 +2,65 @@
 
 %%{
 #include "broxygen/Manager.h"
+#include "util.h"
+
+static StringVal* comments_to_val(const vector<string>& comments)
+	{
+	return new StringVal(implode_string_vector(comments));
+	}
 %%}
 
 # TODO: documentation
 
 function get_identifier_comments%(name: string%): string
 	%{
-	return broxygen_mgr->GetIdentifierComments(name->CheckString());
+	using namespace broxygen;
+	IdentifierDocument* d = broxygen_mgr->GetIdentifierDoc(name->CheckString());
+
+	if ( ! d )
+		return new StringVal("");
+
+	return comments_to_val(d->GetComments());
 	%}
 
 function get_script_comments%(name: string%): string
 	%{
-	return broxygen_mgr->GetScriptComments(name->CheckString());
+	using namespace broxygen;
+	ScriptDocument* d = broxygen_mgr->GetScriptDoc(name->CheckString());
+
+	if ( ! d )
+		return new StringVal("");
+
+	return comments_to_val(d->GetComments());
 	%}
 
 function get_package_readme%(name: string%): string
 	%{
-	return broxygen_mgr->GetPackageReadme(name->CheckString());
+	using namespace broxygen;
+	PackageDocument* d = broxygen_mgr->GetPackageDoc(name->CheckString());
+
+	if ( ! d )
+		return new StringVal("");
+
+	return comments_to_val(d->GetReadme());
 	%}
 
 function get_record_field_comments%(name: string%): string
 	%{
-	return broxygen_mgr->GetRecordFieldComments(name->CheckString());
+	using namespace broxygen;
+	string accessor = name->CheckString();
+	size_t i = accessor.find('$');
+
+	if ( i > accessor.size() - 2 )
+		return new StringVal("");
+
+	string id = accessor.substr(0, i);
+
+	IdentifierDocument* d = broxygen_mgr->GetIdentifierDoc(id);
+
+	if ( ! d )
+		return new StringVal("");
+
+	string field = accessor.substr(i + 1);
+	return comments_to_val(d->GetFieldComments(field));
 	%}

src/main.cc

@@ -762,7 +762,6 @@ int main(int argc, char** argv)
 
 	reporter = new Reporter();
 	thread_mgr = new threading::Manager();
-	broxygen_mgr = new broxygen::Manager(broxygen_config);
 
 #ifdef DEBUG
 	if ( debug_streams )
@@ -809,6 +808,8 @@ int main(int argc, char** argv)
 	timer_mgr = new PQ_TimerMgr("<GLOBAL>");
 	// timer_mgr = new CQ_TimerMgr();
 
+	broxygen_mgr = new broxygen::Manager(broxygen_config, bro_argv[0]);
+
 	add_input_file("base/init-bare.bro");
 	if ( ! bare_mode )
 		add_input_file("base/init-default.bro");

src/parse.y

@@ -1022,14 +1022,15 @@ decl:
 			broxygen_mgr->Redef($2, ::filename);
 			}
 
-	|	TOK_REDEF TOK_ENUM global_id TOK_ADD_TO
+	|	TOK_REDEF TOK_ENUM global_id TOK_ADD_TO '{'
-		'{' { parser_redef_enum($3); } enum_body '}' ';'
+			{ parser_redef_enum($3); broxygen_mgr->Redef($3, ::filename); }
+		enum_body '}' ';'
 			{
 			// Broxygen already grabbed new enum IDs as the type created them.
 			}
 
 	|	TOK_REDEF TOK_RECORD global_id
-			{ cur_decl_type_id = $3; broxygen_mgr->StartRedef($3); }
+			{ cur_decl_type_id = $3; broxygen_mgr->Redef($3, ::filename); }
 		TOK_ADD_TO '{'
 			{ ++in_record; }
 		type_decl_list

src/plugin/ComponentManager.h

@@ -10,6 +10,7 @@
 #include "Var.h"
 #include "Val.h"
 #include "Reporter.h"
+#include "broxygen/Manager.h"
 
 namespace plugin {
 
@@ -133,6 +134,7 @@ ComponentManager<T, C>::ComponentManager(const string& arg_module)
 	tag_enum_type = new EnumType();
 	::ID* id = install_ID("Tag", module.c_str(), true, true);
 	add_type(id, tag_enum_type, 0);
+	broxygen_mgr->Identifier(id);
 	}
 
 template <class T, class C>

src/scan.l

@@ -145,6 +145,7 @@ ESCSEQ	(\\([^\n]|[0-7]+|x[[:xdigit:]]+))
 }
 
 ##.* {
+	if ( yytext[2] != '#' )
 		broxygen_mgr->PreComment(yytext + 2);
 }
 

src/util.cc

@@ -590,6 +590,33 @@ const char* fmt_access_time(double t)
 	return buf;
 	}
 
+bool ensure_intermediate_dirs(const char* dirname)
+	{
+	if ( ! dirname || strlen(dirname) == 0 )
+		return false;
+
+	bool absolute = dirname[0] == '/';
+	string path = normalize_path(dirname);
+
+	vector<string> path_components;
+	tokenize_string(path, "/", &path_components);
+
+	string current_dir;
+
+	for ( size_t i = 0; i < path_components.size(); ++i )
+		{
+		if ( i > 0 || absolute )
+			current_dir += "/";
+
+		current_dir += path_components[i];
+
+		if ( ! ensure_dir(current_dir.c_str()) )
+			return false;
+		}
+
+	return true;
+	}
+
 bool ensure_dir(const char *dirname)
 	{
 	struct stat st;
@@ -976,6 +1003,22 @@ void SafePathOp::DoFunc(PathOpFn fn, const string& path, bool error_aborts)
 	delete [] tmp;
 	}
 
+string implode_string_vector(const std::vector<std::string>& v,
+                             const std::string& delim)
+	{
+	string rval;
+
+	for ( size_t i = 0; i < v.size(); ++i )
+		{
+		if ( i > 0 )
+			rval += delim;
+
+		rval += v[i];
+		}
+
+	return rval;
+	}
+
 string flatten_script_name(const string& name, const string& prefix)
 	{
 	string rval = prefix;

src/util.h

@@ -150,6 +150,7 @@ extern const char* fmt(const char* format, ...)
 	myattribute((format (printf, 1, 2)));
 extern const char* fmt_access_time(double time);
 
+extern bool ensure_intermediate_dirs(const char* dirname);
 extern bool ensure_dir(const char *dirname);
 
 // Returns true if path exists and is a directory.
@@ -251,6 +252,9 @@ public:
 		: SafePathOp(&basename, path, error_aborts) { }
 };
 
+std::string implode_string_vector(const std::vector<std::string>& v,
+                                  const std::string& delim = "\n");
+
 /**
  * Flatten a script name by replacing '/' path separators with '.'.
  * @param file A path to a Bro script.  If it is a __load__.bro, that part