Add BIF interface for retrieving comments/docs.

The new BIFs:
    - get_identifier_comments
    - get_script_comments
    - get_package_readme
    - get_record_field_comments

doc/scripts/DocSourcesList.cmake

@@ -19,6 +19,7 @@ rest_target(${psd} base/init-bare.bro internal)
 rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/analyzer.bif.bro)
 rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/bloom-filter.bif.bro)
 rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/bro.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/broxygen.bif.bro)
 rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/cardinality-counter.bif.bro)
 rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/const.bif.bro)
 rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/event.bif.bro)

src/CMakeLists.txt

@@ -154,6 +154,7 @@ set(bro_PLUGIN_LIBS CACHE INTERNAL "plugin libraries" FORCE)
 add_subdirectory(analyzer)
 add_subdirectory(file_analysis)
 add_subdirectory(probabilistic)
+add_subdirectory(broxygen)
 
 set(bro_SUBDIRS
     ${bro_SUBDIR_LIBS}
@@ -370,9 +371,6 @@ set(bro_SRCS
     plugin/Plugin.cc
     plugin/Macros.h
 
-    broxygen/Manager.cc
-    broxygen/Document.cc
-
     nb_dns.c
     digest.h
 )

src/broxygen/CMakeLists.txt

@@ -0,0 +1,16 @@
+include(BroSubdir)
+
+include_directories(BEFORE
+                    ${CMAKE_CURRENT_SOURCE_DIR}
+                    ${CMAKE_CURRENT_BINARY_DIR}
+)
+
+set(broxygen_SRCS
+    Manager.cc
+    Document.cc
+)
+
+bif_target(broxygen.bif)
+bro_add_subdir_library(broxygen ${broxygen_SRCS})
+
+add_dependencies(bro_broxygen generate_outputs)

src/broxygen/Document.cc

@@ -1,13 +1,29 @@
 #include "Document.h"
 
 #include "util.h"
+#include "Val.h"
 
 using namespace broxygen;
 using namespace std;
 
+static string ImplodeStringVec(const vector<string>& v)
+	{
+	string rval;
+
+	for ( size_t i = 0; i < v.size(); ++i )
+		{
+		if ( i > 0 )
+			rval += '\n';
+
+		rval += v[i];
+		}
+
+	return rval;
+	}
+
 PackageDocument::PackageDocument(const string& arg_name)
     : Document(),
-      pkg_loader_name(arg_name)
+      pkg_name(arg_name)
 	{
 	// TODO: probably need to determine modification times of all files
 	//       within the directory, recursively
@@ -36,8 +52,9 @@ IdentifierDocument::~IdentifierDocument()
 	      ++it )
 		delete *it;
 
-	for ( size_t i = 0; i < fields.size(); ++i )
+	for ( RecordFieldMap::const_iterator it = fields.begin();
-		delete fields[i];
+	      it != fields.end(); ++it )
+		delete it->second;
 	}
 
 void IdentifierDocument::AddRedef(const string& script,
@@ -59,21 +76,36 @@ void IdentifierDocument::AddRedef(const string& script,
 
 void IdentifierDocument::AddRecordField(const TypeDecl* field,
                                         const string& script,
-                                        std::vector<string>& comments)
+                                        vector<string>& comments)
 	{
 	RecordField* rf = new RecordField();
 	rf->field = new TypeDecl(*field);
 	rf->from_script = script;
 	rf->comments = comments;
-	fields.push_back(rf);
+	fields[rf->field->id] = rf;
 	last_field_seen = rf;
 	}
 
+string IdentifierDocument::GetComments() const
+	{
+	return ImplodeStringVec(comments);
+	}
+
+string IdentifierDocument::GetFieldComments(const string& field) const
+	{
+	RecordFieldMap::const_iterator it = fields.find(field);
+
+	if ( it == fields.end() )
+		return string();
+
+	return ImplodeStringVec(it->second->comments);
+	}
+
 ScriptDocument::ScriptDocument(const string& arg_name)
     : Document(),
       name(arg_name),
       is_pkg_loader(safe_basename(name) == PACKAGE_LOADER),
-      dependencies(), module_usages(), comments(), identifier_docs()
+      dependencies(), module_usages(), comments(), identifier_docs(), redefs()
 	{
 	}
 
@@ -82,3 +114,8 @@ void ScriptDocument::AddIdentifierDoc(IdentifierDocument* doc)
 	identifier_docs[doc->Name()] = doc;
 	// TODO: sort things (e.g. function flavor, state var vs. option var)
 	}
+
+string ScriptDocument::GetComments() const
+	{
+	return ImplodeStringVec(comments);
+	}

src/broxygen/Document.h

@@ -10,7 +10,6 @@
 #include <time.h>
 
 #include "ID.h"
-#include "Val.h"
 #include "Type.h"
 
 namespace broxygen {
@@ -30,9 +29,13 @@ public:
 	time_t GetModificationTime() const
 		{ return DoGetModificationTime(); }
 
+	std::string Name() const
+		{ return DoName(); }
+
 private:
 
 	virtual time_t DoGetModificationTime() const = 0;
+	virtual std::string DoName() const = 0;
 };
 
 class PackageDocument : public Document {
@@ -41,13 +44,20 @@ public:
 
 	PackageDocument(const std::string& name);
 
+	// TODO: can be comments from package README
+	std::string GetReadme() const
+		{ return std::string(); }
+
 private:
 
 	// TODO
 	time_t DoGetModificationTime() const
 		{ return 0; }
 
-	std::string pkg_loader_name;
+	std::string DoName() const
+		{ return pkg_name; }
+
+	std::string pkg_name;
 };
 
 
@@ -64,7 +74,7 @@ public:
 		                  : comments.push_back(comment); }
 
 	void AddComments(const std::vector<std::string>& cmtns)
-		{ comments.insert(comments.end(), cmtns.begin(), cmtns.end() ); }
+		{ comments.insert(comments.end(), cmtns.begin(), cmtns.end()); }
 
 	void AddRedef(const std::string& from_script,
 	              const std::vector<std::string>& comments);
@@ -72,14 +82,25 @@ public:
 	void AddRecordField(const TypeDecl* field, const std::string& script,
 	                    std::vector<std::string>& comments);
 
-	string Name() const
+	void CompletedTypeDecl()
-		{ return id->Name(); }
+		{ last_field_seen = 0; }
 
 	ID* GetID() const
 		{ return id; }
 
+	std::string GetComments() const;
+
+	std::string GetFieldComments(const std::string& field) const;
+
 private:
 
+	// TODO
+	time_t DoGetModificationTime() const
+		{ return 0; }
+
+	std::string DoName() const
+		{ return id->Name(); }
+
 	struct Redefinition {
 		std::string from_script;
 		string new_val_desc;
@@ -96,16 +117,13 @@ private:
 	};
 
 	typedef std::list<Redefinition*> RedefList;
-
+	typedef std::map<std::string, RecordField*> RecordFieldMap;
-	// TODO
-	time_t DoGetModificationTime() const
-		{ return 0; }
 
 	std::vector<std::string> comments;
 	ID* id;
 	string initial_val_desc;
 	RedefList redefs;
-	std::vector<RecordField*> fields;
+	RecordFieldMap fields;
 	RecordField* last_field_seen;
 };
 
@@ -126,23 +144,33 @@ public:
 
 	void AddIdentifierDoc(IdentifierDocument* doc);
 
+	void AddRedef(IdentifierDocument* doc)
+		{ redefs.push_back(doc); }
+
 	bool IsPkgLoader() const
 		{ return is_pkg_loader; }
 
+	std::string GetComments() const;
+
 private:
 
 	typedef std::map<std::string, IdentifierDocument*> IdentifierDocMap;
+	typedef std::list<IdentifierDocument*> IdentifierDocList;
 
 	// TODO
 	time_t DoGetModificationTime() const
 		{ return 0; }
 
+	std::string DoName() const
+		{ return name; }
+
 	std::string name;
 	bool is_pkg_loader;
 	std::set<std::string> dependencies;
 	std::set<std::string> module_usages;
 	std::vector<std::string> comments;
 	IdentifierDocMap identifier_docs;
+	IdentifierDocList redefs;
 };
 
 

src/broxygen/Manager.cc

@@ -84,8 +84,8 @@ static string PrettifyParams(const string& s)
 	}
 
 Manager::Manager(const string& config)
-    : disabled(), comment_buffer(), packages(), scripts(), identifiers(),
+    : disabled(), comment_buffer(), comment_buffer_map(), packages(), scripts(),
-      all_docs(), last_doc_seen(), incomplete_type()
+      identifiers(), all_docs(), last_identifier_seen(), incomplete_type()
 	{
 	if ( getenv("BRO_DISABLE_BROXYGEN") )
 		disabled = true;
@@ -94,9 +94,8 @@ Manager::Manager(const string& config)
 
 Manager::~Manager()
 	{
-	for ( DocSet::const_iterator it = all_docs.begin(); it != all_docs.end();
+	for ( size_t i = 0; i < all_docs.size(); ++i )
-	      ++it )
+		delete all_docs[i];
-		delete *it;
 	}
 
 void Manager::InitPreScript()
@@ -129,27 +128,31 @@ void Manager::File(const string& path)
 
 	string name = without_bropath_component(path);
 
-	if ( scripts.find(name) != scripts.end() )
+	if ( scripts.GetDocument(name) )
 		{
 		DbgAndWarn(fmt("Duplicate script documentation: %s", name.c_str()));
 		return;
 		}
 
 	ScriptDocument* doc = new ScriptDocument(name);
-	scripts[name] = doc;
+	scripts.map[name] = doc;
-	RegisterDoc(doc);
+	all_docs.push_back(doc);
 	DBG_LOG(DBG_BROXYGEN, "Made ScriptDocument %s", name.c_str());
 
 	if ( ! doc->IsPkgLoader() )
 		return;
 
-	if ( packages.find(name) != packages.end() )
+	name = safe_dirname(name);
+
+	if ( packages.GetDocument(name) )
 		{
 		DbgAndWarn(fmt("Duplicate package documentation: %s", name.c_str()));
 		return;
 		}
 
-	packages[name] = new PackageDocument(name);
+	PackageDocument* pkgdoc = new PackageDocument(name);
+	packages.map[name] = pkgdoc;
+	all_docs.push_back(pkgdoc);
 	DBG_LOG(DBG_BROXYGEN, "Made PackageDocument %s", name.c_str());
 	}
 
@@ -166,23 +169,22 @@ void Manager::ScriptDependency(const string& path, const string& dep)
 
 	string name = without_bropath_component(path);
 	string depname = without_bropath_component(dep);
-	ScriptMap::const_iterator it = scripts.find(name);
+	ScriptDocument* script_doc = scripts.GetDocument(name);
 
-	if ( it == scripts.end() )
+	if ( ! script_doc )
 		{
 		DbgAndWarn(fmt("Failed to add script doc dependency %s for %s",
 		               depname.c_str(), name.c_str()));
 		return;
 		}
 
-	it->second->AddDependency(depname);
+	script_doc->AddDependency(depname);
 	DBG_LOG(DBG_BROXYGEN, "Added script dependency %s for %s",
 	        depname.c_str(), name.c_str());
 
-	for ( CommentBuffer::const_iterator it = comment_buffer.begin();
+	for ( size_t i = 0; i < comment_buffer.size(); ++i )
-	      it != comment_buffer.end(); ++it )
 		DbgAndWarn(fmt("Discarded extraneous Broxygen comment: %s",
-		               it->c_str()));
+		               comment_buffer[i].c_str()));
 	}
 
 void Manager::ModuleUsage(const string& path, const string& module)
@@ -191,19 +193,42 @@ void Manager::ModuleUsage(const string& path, const string& module)
 		return;
 
 	string name = without_bropath_component(path);
-	ScriptMap::const_iterator it = scripts.find(name);
+	ScriptDocument* script_doc = scripts.GetDocument(name);
 
-	if ( it == scripts.end() )
+	if ( ! script_doc )
 		{
 		DbgAndWarn(fmt("Failed to add module usage %s in %s",
 		               module.c_str(), name.c_str()));
 		return;
 		}
 
+	script_doc->AddModule(module);
 	DBG_LOG(DBG_BROXYGEN, "Added module usage %s in %s",
 	        module.c_str(), name.c_str());
 	}
 
+IdentifierDocument* Manager::CreateIdentifierDoc(ID* id, ScriptDocument* script)
+	{
+	IdentifierDocument* rval = new IdentifierDocument(id);
+
+	rval->AddComments(comment_buffer);
+	comment_buffer.clear();
+
+	CommentBufferMap::const_iterator it = comment_buffer_map.find(id->Name());
+
+	if ( it != comment_buffer_map.end() )
+		{
+		rval->AddComments(it->second);
+		comment_buffer_map.erase(it);
+		}
+
+	all_docs.push_back(rval);
+	identifiers.map[id->Name()] = rval;
+	last_identifier_seen = rval;
+	script->AddIdentifierDoc(rval);
+	return rval;
+	}
+
 void Manager::StartType(ID* id)
 	{
 	if ( disabled )
@@ -216,26 +241,34 @@ void Manager::StartType(ID* id)
 		}
 
 	string script = without_bropath_component(id->GetLocationInfo()->filename);
-	ScriptMap::const_iterator sit = scripts.find(script);
+	ScriptDocument* script_doc = scripts.GetDocument(script);
 
-	if ( sit == scripts.end() )
+	if ( ! script_doc )
 		{
 		DbgAndWarn(fmt("Can't document identifier %s, lookup of %s failed",
 		               id->Name(), script.c_str()));
 		return;
 		}
 
-	IdentifierDocument* doc = new IdentifierDocument(id);
+	incomplete_type = CreateIdentifierDoc(id, script_doc);
-	doc->AddComments(comment_buffer);
-	comment_buffer.clear();
-	identifiers[id->Name()] = doc;
-	RegisterDoc(doc);
-	sit->second->AddIdentifierDoc(doc);
-	incomplete_type = doc;
 	DBG_LOG(DBG_BROXYGEN, "Made IdentifierDocument (incomplete) %s, in %s",
 	        id->Name(), script.c_str());
 	}
 
+void Manager::StartRedef(ID* id)
+	{
+	if ( disabled )
+		return;
+
+	IdentifierDocument* id_doc = identifiers.GetDocument(id->Name());
+
+	if ( id_doc )
+		last_identifier_seen = id_doc;
+	else
+		DbgAndWarn(fmt("Broxygen redef tracking unknown identifier: %s",
+		               id->Name()));
+	}
+
 void Manager::Identifier(ID* id)
 	{
 	if ( disabled )
@@ -244,6 +277,7 @@ void Manager::Identifier(ID* id)
 	if ( incomplete_type && incomplete_type->Name() == id->Name() )
 		{
 		DBG_LOG(DBG_BROXYGEN, "Finished document for type %s", id->Name());
+		incomplete_type->CompletedTypeDecl();
 		incomplete_type = 0;
 		return;
 		}
@@ -257,14 +291,14 @@ void Manager::Identifier(ID* id)
 		return;
 		}
 
-	IdentifierMap::const_iterator iit = identifiers.find(id->Name());
+	IdentifierDocument* id_doc = identifiers.GetDocument(id->Name());
 
-	if ( iit != identifiers.end() )
+	if ( id_doc )
 		{
-		if ( IsFunc(iit->second->GetID()->Type()->Tag()) )
+		if ( IsFunc(id_doc->GetID()->Type()->Tag()) )
 			{
 			// Function may already been seen (declaration versus body).
-			iit->second->AddComments(comment_buffer);
+			id_doc->AddComments(comment_buffer);
 			comment_buffer.clear();
 			return;
 			}
@@ -274,21 +308,16 @@ void Manager::Identifier(ID* id)
 		}
 
 	string script = without_bropath_component(id->GetLocationInfo()->filename);
-	ScriptMap::const_iterator sit = scripts.find(script);
+	ScriptDocument* script_doc = scripts.GetDocument(script);
 
-	if ( sit == scripts.end() )
+	if ( ! script_doc )
 		{
 		DbgAndWarn(fmt("Can't document identifier %s, lookup of %s failed",
 		               id->Name(), script.c_str()));
 		return;
 		}
 
-	IdentifierDocument* doc = new IdentifierDocument(id);
+	CreateIdentifierDoc(id, script_doc);
-	doc->AddComments(comment_buffer);
-	comment_buffer.clear();
-	identifiers[id->Name()] = doc;
-	RegisterDoc(doc);
-	sit->second->AddIdentifierDoc(doc);
 	DBG_LOG(DBG_BROXYGEN, "Made IdentifierDocument %s, in script %s",
 	        id->Name(), script.c_str());
 	}
@@ -299,36 +328,16 @@ void Manager::RecordField(const ID* id, const TypeDecl* field,
 	if ( disabled )
 		return;
 
-	IdentifierDocument* idd = 0;
+	IdentifierDocument* idd = identifiers.GetDocument(id->Name());
 
-	if ( incomplete_type )
+	if ( ! idd )
-		{
-		if ( incomplete_type->Name() != id->Name() )
-			{
-			DbgAndWarn(fmt("Can't document record field %s in record %s, "
-			               "expected record %s", field->id, id->Name(),
-			               incomplete_type->Name().c_str()));
-			return;
-			}
-
-		idd = incomplete_type;
-		}
-	else
-		{
-		IdentifierMap::const_iterator it = identifiers.find(id->Name());
-
-		if ( it == identifiers.end() )
 		{
 		DbgAndWarn(fmt("Can't document record field %s, unknown record: %s",
 		               field->id, id->Name()));
 		return;
 		}
 
-		idd = it->second;
-		}
-
 	string script = without_bropath_component(path);
-
 	idd->AddRecordField(field, script, comment_buffer);
 	comment_buffer.clear();
 	DBG_LOG(DBG_BROXYGEN, "Document record field %s, identifier %s, script %s",
@@ -344,9 +353,9 @@ void Manager::Redef(const ID* id, const string& path)
 		// This is a redef defined on the command line.
 		return;
 
-	IdentifierMap::const_iterator iit = identifiers.find(id->Name());
+	IdentifierDocument* id_doc = identifiers.GetDocument(id->Name());
 
-	if ( iit == identifiers.end() )
+	if ( ! id_doc )
 		{
 		DbgAndWarn(fmt("Can't document redef of %s, identifier lookup failed",
 		               id->Name()));
@@ -354,18 +363,20 @@ void Manager::Redef(const ID* id, const string& path)
 		}
 
 	string from_script = without_bropath_component(path);
-	ScriptMap::const_iterator sit = scripts.find(from_script);
+	ScriptDocument* script_doc = scripts.GetDocument(from_script);
 
-	if ( sit == scripts.end() )
+	if ( ! script_doc )
 		{
 		DbgAndWarn(fmt("Can't document redef of %s, lookup of %s failed",
 		               id->Name(), from_script.c_str()));
 		return;
 		}
 
-	iit->second->AddRedef(from_script, comment_buffer);
+	id_doc->AddRedef(from_script, comment_buffer);
+	script_doc->AddRedef(id_doc);
 	comment_buffer.clear();
-	DBG_LOG(DBG_BROXYGEN, "Added redef of %s to %s",
+	last_identifier_seen = id_doc;
+	DBG_LOG(DBG_BROXYGEN, "Added redef of %s from %s",
 	        id->Name(), from_script.c_str());
 	}
 
@@ -375,16 +386,13 @@ void Manager::SummaryComment(const string& script, const string& comment)
 		return;
 
 	string name = without_bropath_component(script);
-	ScriptMap::const_iterator it = scripts.find(name);
+	ScriptDocument* doc = scripts.GetDocument(name);
 
-	if ( it == scripts.end() )
+	if ( doc )
-		{
+		doc->AddComment(RemoveLeadingSpace(comment));
+	else
 		DbgAndWarn(fmt("Lookup of script %s failed for summary comment %s",
 		               name.c_str(), comment.c_str()));
-		return;
-		}
-
-	it->second->AddComment(RemoveLeadingSpace(comment));
 	}
 
 void Manager::PreComment(const string& comment)
@@ -392,31 +400,59 @@ void Manager::PreComment(const string& comment)
 	if ( disabled )
 		return;
 
-	comment_buffer.push_back(PrettifyParams(RemoveLeadingSpace(comment)));
+	comment_buffer.push_back(RemoveLeadingSpace(comment));
 	}
 
-void Manager::PostComment(const string& comment)
+void Manager::PostComment(const string& comment, const string& id_hint)
 	{
 	if ( disabled )
 		return;
 
-	IdentifierDocument* doc = dynamic_cast<IdentifierDocument*>(last_doc_seen);
+	if ( id_hint.empty() )
-
-	if ( ! doc )
 		{
-		DbgAndWarn(fmt("Discarded comment not associated w/ an identifier %s",
+		if ( last_identifier_seen )
+			last_identifier_seen->AddComment(RemoveLeadingSpace(comment));
+		else
+			DbgAndWarn(fmt("Discarded unassociated Broxygen comment %s",
 			               comment.c_str()));
+
 		return;
 		}
 
-	doc->AddComment(RemoveLeadingSpace(comment));
+	if ( last_identifier_seen &&
+	     last_identifier_seen->Name() == id_hint )
+		last_identifier_seen->AddComment(RemoveLeadingSpace(comment));
+	else
+		// Assume identifier it's associated w/ is coming later.
+		comment_buffer_map[id_hint].push_back(RemoveLeadingSpace(comment));
 	}
 
-void Manager::RegisterDoc(Document* d)
+StringVal* Manager::GetIdentifierComments(const string& name) const
 	{
-	if ( ! d )
+	IdentifierDocument* d = identifiers.GetDocument(name);
-		return;
+	return new StringVal(d ? d->GetComments() : "");
-
+	}
-	all_docs.insert(d);
+
-	last_doc_seen = d;
+StringVal* Manager::GetScriptComments(const string& name) const
+	{
+	ScriptDocument* d = scripts.GetDocument(name);
+	return new StringVal(d ? d->GetComments() : "");
+	}
+
+StringVal* Manager::GetPackageReadme(const string& name) const
+	{
+	PackageDocument* d = packages.GetDocument(name);
+	return new StringVal(d ? d->GetReadme() : "");
+	}
+
+StringVal* Manager::GetRecordFieldComments(const string& name) const
+	{
+	size_t i = name.find('$');
+
+	if ( i > name.size() - 2 )
+		// '$' is last char in string or not found.
+		return new StringVal("");
+
+	IdentifierDocument* d = identifiers.GetDocument(name.substr(0, i));
+	return new StringVal(d ? d->GetFieldComments(name.substr(i + 1)) : "");
 	}

src/broxygen/Manager.h

@@ -4,16 +4,29 @@
 #include "Document.h"
 #include "ID.h"
 #include "Type.h"
+#include "Val.h"
 
 #include <string>
-#include <map>
-#include <set>
 #include <vector>
+#include <map>
 
 namespace broxygen {
 
 // TODO: documentation...
 
+template<class T>
+struct DocumentMap {
+	typedef std::map<std::string, T*> MapType;
+
+	T* GetDocument(const std::string& name) const
+		{
+		typename MapType::const_iterator it = map.find(name);
+		return it == map.end() ? 0 : it->second;
+		}
+
+	MapType map;
+};
+
 class Manager {
 
 public:
@@ -36,6 +49,8 @@ public:
 
 	void StartType(ID* id);
 
+	void StartRedef(ID* id);
+
 	void Identifier(ID* id);
 
 	void RecordField(const ID* id, const TypeDecl* field,
@@ -47,25 +62,32 @@ public:
 
 	void PreComment(const std::string& comment);
 
-	void PostComment(const std::string& comment);
+	void PostComment(const std::string& comment,
+	                 const std::string& identifier_hint = "");
+
+	StringVal* GetIdentifierComments(const std::string& name) const;
+
+	StringVal* GetScriptComments(const std::string& name) const;
+
+	StringVal* GetPackageReadme(const std::string& name) const;
+
+	StringVal* GetRecordFieldComments(const std::string& name) const;
 
 private:
 
 	typedef std::vector<std::string> CommentBuffer;
-	typedef std::map<std::string, PackageDocument*> PackageMap;
+	typedef std::map<std::string, CommentBuffer> CommentBufferMap;
-	typedef std::map<std::string, ScriptDocument*> ScriptMap;
-	typedef std::map<std::string, IdentifierDocument*> IdentifierMap;
-	typedef std::set<Document*> DocSet;
 
-	void RegisterDoc(Document* d);
+	IdentifierDocument* CreateIdentifierDoc(ID* id, ScriptDocument* script);
 
 	bool disabled;
-	CommentBuffer comment_buffer;
+	CommentBuffer comment_buffer; // For whatever next identifier that comes in.
-	PackageMap packages;
+	CommentBufferMap comment_buffer_map; // For a particular identifier.
-	ScriptMap scripts;
+	DocumentMap<PackageDocument> packages;
-	IdentifierMap identifiers;
+	DocumentMap<ScriptDocument> scripts;
-	DocSet all_docs;
+	DocumentMap<IdentifierDocument> identifiers;
-	Document* last_doc_seen;
+	std::vector<Document*> all_docs;
+	IdentifierDocument* last_identifier_seen;
 	IdentifierDocument* incomplete_type;
 };
 

src/broxygen/broxygen.bif

@@ -0,0 +1,27 @@
+##! Functions for querying script, package, or variable documentation.
+
+%%{
+#include "broxygen/Manager.h"
+%%}
+
+# TODO: documentation
+
+function get_identifier_comments%(name: string%): string
+	%{
+	return broxygen_mgr->GetIdentifierComments(name->CheckString());
+	%}
+
+function get_script_comments%(name: string%): string
+	%{
+	return broxygen_mgr->GetScriptComments(name->CheckString());
+	%}
+
+function get_package_readme%(name: string%): string
+	%{
+	return broxygen_mgr->GetPackageReadme(name->CheckString());
+	%}
+
+function get_record_field_comments%(name: string%): string
+	%{
+	return broxygen_mgr->GetRecordFieldComments(name->CheckString());
+	%}

src/parse.y

@@ -1028,8 +1028,13 @@ decl:
 			// Broxygen already grabbed new enum IDs as the type created them.
 			}
 
-	|	TOK_REDEF TOK_RECORD global_id { cur_decl_type_id = $3; } TOK_ADD_TO
+	|	TOK_REDEF TOK_RECORD global_id
-			'{' { ++in_record; } type_decl_list { --in_record; } '}' opt_attr ';'
+			{ cur_decl_type_id = $3; broxygen_mgr->StartRedef($3); }
+		TOK_ADD_TO '{'
+			{ ++in_record; }
+		type_decl_list
+			{ --in_record; }
+		'}' opt_attr ';'
 			{
 			cur_decl_type_id = 0;
 

src/scan.l

@@ -31,6 +31,7 @@
 #include "broxygen/Manager.h"
 
 extern YYLTYPE yylloc;	// holds start line and column of token
+extern EnumType* cur_enum_type;
 
 // Track the @if... depth.
 ptr_compat_int current_depth = 0;
@@ -39,6 +40,7 @@ int_list if_stack;
 
 int line_number = 1;
 const char* filename = 0; // Absolute path of file currently being parsed.
+static const char* last_id_tok = 0;
 
 char last_tok[128];
 
@@ -137,7 +139,9 @@ ESCSEQ	(\\([^\n]|[0-7]+|x[[:xdigit:]]+))
 	}
 
 ##<.* {
-	broxygen_mgr->PostComment(yytext + 3);
+	string hint(cur_enum_type && last_id_tok ?
+	            make_full_var_name(current_module.c_str(), last_id_tok) : "");
+	broxygen_mgr->PostComment(yytext + 3, hint);
 }
 
 ##.* {
@@ -370,6 +374,7 @@ F	RET_CONST(new Val(false, TYPE_BOOL))
 
 {ID}	{
 	yylval.str = copy_string(yytext);
+	last_id_tok = yylval.str;
 	return TOK_ID;
 	}
 

testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log

@@ -3,7 +3,7 @@
 #empty_field	(empty)
 #unset_field	-
 #path	loaded_scripts
-#open	2013-08-31-17-46-55
+#open	2013-10-30-16-52-11
 #fields	name
 #types	string
 scripts/base/init-bare.bro
@@ -96,8 +96,9 @@ scripts/base/init-bare.bro
         scripts/base/utils/patterns.bro
   build/scripts/base/bif/__load__.bro
     build/scripts/base/bif/bloom-filter.bif.bro
+    build/scripts/base/bif/broxygen.bif.bro
     build/scripts/base/bif/cardinality-counter.bif.bro
     build/scripts/base/bif/top-k.bif.bro
 scripts/policy/misc/loaded-scripts.bro
   scripts/base/utils/paths.bro
-#close	2013-08-31-17-46-55
+#close	2013-10-30-16-52-11

testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log

@@ -3,7 +3,7 @@
 #empty_field	(empty)
 #unset_field	-
 #path	loaded_scripts
-#open	2013-08-31-17-46-56
+#open	2013-10-30-16-52-28
 #fields	name
 #types	string
 scripts/base/init-bare.bro
@@ -96,6 +96,7 @@ scripts/base/init-bare.bro
         scripts/base/utils/patterns.bro
   build/scripts/base/bif/__load__.bro
     build/scripts/base/bif/bloom-filter.bif.bro
+    build/scripts/base/bif/broxygen.bif.bro
     build/scripts/base/bif/cardinality-counter.bif.bro
     build/scripts/base/bif/top-k.bif.bro
 scripts/base/init-default.bro
@@ -220,4 +221,4 @@ scripts/base/init-default.bro
     scripts/base/files/unified2/main.bro
   scripts/base/misc/find-checksum-offloading.bro
 scripts/policy/misc/loaded-scripts.bro
-#close	2013-08-31-17-46-56
+#close	2013-10-30-16-52-28

testing/btest/Baseline/doc.broxygen.comment_retrieval_bifs/out

@@ -0,0 +1,70 @@
+This is a test script.
+With some summary comments.
+myvar:
+    Hello world.  This is an option.
+    With some more description here.
+    And here.
+    Maybe just one more.
+print_lines:
+    This function prints a string line by line.
+    
+    lines: A string to print line by line, w/ lines delimited by newline chars.
+    And some more comments on the function implementation.
+mytype:
+    This is an alias for count.
+myrecord:
+    My record type.
+myrecord$aaa:
+    The first field.
+    Does something...
+    Done w/ aaa.
+myrecord$bbb:
+    The second field.
+    Done w/ bbb.
+    No really, done w/ bbb.
+myrecord$ccc:
+    Third field.
+    Done w/ ccc.
+myrecord$ddd:
+    Fourth field.
+    Done w/ ddd.
+myrecord$eee:
+    First redef'd field.
+    With two lines of comments.
+    And two post-notation comments.
+    Done w/ eee.
+myrecord$fff:
+    Second redef'd field.
+    Done w/ fff.
+myrecord$ggg:
+    Third redef'd field.
+    Done w/ ggg.
+myenum:
+    My enum type;
+FIRST:
+    First enum value.
+    I know, the name isn't clever.
+    Done w/ first.
+SECOND:
+    Second enum value.
+    Done w/ second.
+THIRD:
+    Third enum value.
+    Done w/ third.
+    Done w/ third again.
+FORTH:
+    SIC.
+    It's a programming language.
+    Using Reverse Polish Notation.
+    Done w/ forth.
+FIFTH:
+    First redef'd enum val.
+    Done w/ fifth.
+SIXTH:
+    Second redef'd enum val.
+    Done w/ sixth.
+SEVENTH:
+    Third redef'd enum val.
+    Lucky number seven.
+    Still works with comma.
+    Done w/ seventh.

testing/btest/doc/broxygen/comment_retrieval_bifs.bro

@@ -0,0 +1,111 @@
+# @TEST-EXEC: bro -b %INPUT >out
+# @TEST-EXEC: btest-diff out
+
+##! This is a test script.
+##! With some summary comments.
+
+## Hello world.  This is an option.
+## With some more description here.
+## And here.
+const myvar = 7 &redef;  ##< Maybe just one more.
+
+## This function prints a string line by line.
+##
+## lines: A string to print line by line, w/ lines delimited by newline chars.
+global print_lines: function(lines: string, prefix: string &default="");
+
+## And some more comments on the function implementation.
+function print_lines(lines: string, prefix: string)
+	{
+	local v: vector of string;
+	local line_table = split(lines, /\n/);
+
+	for ( i in line_table )
+		v[i] = line_table[i];
+
+	for ( i in v )
+		print fmt("%s%s", prefix, v[i]);
+	}
+
+function print_comments(name: string, func: function(name: string): string)
+	{
+	print fmt("%s:", name);
+	print_lines(func(name), "    ");
+	}
+
+## This is an alias for count.
+type mytype: count;
+
+## My record type.
+type myrecord: record {
+	## The first field.
+	## Does something...
+	aaa: count;    ##< Done w/ aaa.
+	## The second field.
+	bbb: string;   ##< Done w/ bbb.
+	               ##< No really, done w/ bbb.
+	## Third field.
+	ccc: int;      ##< Done w/ ccc.
+	## Fourth field.
+	ddd: interval; ##< Done w/ ddd.
+};
+
+
+## My enum type;
+type myenum: enum {
+	## First enum value.
+	## I know, the name isn't clever.
+	FIRST,  ##< Done w/ first.
+	## Second enum value.
+	SECOND, ##< Done w/ second.
+	## Third enum value.
+	THIRD,  ##< Done w/ third.
+	        ##< Done w/ third again.
+	## SIC.
+	## It's a programming language.
+	FORTH  ##< Using Reverse Polish Notation.
+	       ##< Done w/ forth.
+};
+
+redef record myrecord += {
+	## First redef'd field.
+	## With two lines of comments.
+	eee: count &optional; ##< And two post-notation comments.
+	                      ##< Done w/ eee.
+	## Second redef'd field.
+	fff: count &optional; ##< Done w/ fff.
+	## Third redef'd field.
+	ggg: count &optional; ##< Done w/ ggg.
+};
+
+redef enum myenum += {
+	## First redef'd enum val.
+	FIFTH, ##< Done w/ fifth.
+	## Second redef'd enum val.
+	SIXTH, ##< Done w/ sixth.
+	## Third redef'd enum val.
+	## Lucky number seven.
+	SEVENTH, ##< Still works with comma.
+	         ##< Done w/ seventh.
+};
+
+print_lines(get_script_comments(@DIR + "/" + @FILENAME));
+print_comments("myvar", get_identifier_comments);
+print_comments("print_lines", get_identifier_comments);
+print_comments("mytype", get_identifier_comments);
+print_comments("myrecord", get_identifier_comments);
+print_comments("myrecord$aaa", get_record_field_comments);
+print_comments("myrecord$bbb", get_record_field_comments);
+print_comments("myrecord$ccc", get_record_field_comments);
+print_comments("myrecord$ddd", get_record_field_comments);
+print_comments("myrecord$eee", get_record_field_comments);
+print_comments("myrecord$fff", get_record_field_comments);
+print_comments("myrecord$ggg", get_record_field_comments);
+print_comments("myenum", get_identifier_comments);
+print_comments("FIRST", get_identifier_comments);
+print_comments("SECOND", get_identifier_comments);
+print_comments("THIRD", get_identifier_comments);
+print_comments("FORTH", get_identifier_comments);
+print_comments("FIFTH", get_identifier_comments);
+print_comments("SIXTH", get_identifier_comments);
+print_comments("SEVENTH", get_identifier_comments);