Mirror/zeek
1
0
Fork 0
mirror of https://github.com/zeek/zeek.git synced 2025-10-06 16:48:19 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

Implement majority of Broxygen features delegated to Bro.

Browse source 
Still have to update the Sphinx integration.
This commit is contained in:
Jon Siwek 2013-11-14 14:00:51 -06:00
parent bdd359d58c
commit 4f6d01000a
21 changed files with 1494 additions and 342 deletions
Download patch file Download diff file Expand all files Collapse all files

46
src/BroDoc.cc
View file

@ -12,6 +12,7 @@
#include "analyzer/Manager.h"
#include "analyzer/Component.h"
#include "file_analysis/Manager.h"
#include "broxygen/Manager.h"
BroDoc::BroDoc(const std::string& rel, const std::string& abs)
{
@ -565,33 +566,28 @@ static void WritePluginBifItems(FILE* f, const plugin::Plugin* p,
for ( it = bifitems.begin(); it != bifitems.end(); ++it )
{
BroDocObj* o = doc_ids[it->GetID()];
broxygen::IdentifierDocument* doc = broxygen_mgr->GetIdentifierDoc(
it->GetID());
if ( o )
o->WriteReST(f);
if ( doc )
fprintf(f, "%s\n\n", doc->ReStructuredText().c_str());
else
reporter->Warning("No docs for ID: %s\n", it->GetID());
reporter->InternalWarning("Broxygen ID lookup failed: %s\n",
it->GetID());
}
}
static void WriteAnalyzerTagDefn(FILE* f, EnumType* e, const string& module)
static void WriteAnalyzerTagDefn(FILE* f, const string& module)
{
/* TODO
string tag_id= module + "::Tag";
e = new CommentedEnumType(e);
e->SetTypeID(copy_string(tag_id.c_str()));
string tag_id = module + "::Tag";
ID* dummy_id = new ID(tag_id.c_str(), SCOPE_GLOBAL, true);
dummy_id->SetType(e);
dummy_id->MakeType();
broxygen::IdentifierDocument* doc = broxygen_mgr->GetIdentifierDoc(tag_id);
list<string>* r = new list<string>();
r->push_back("Unique identifiers for analyzers.");
if ( ! doc )
reporter->InternalError("Broxygen failed analyzer tag lookup: %s",
tag_id.c_str());
BroDocObj bdo(dummy_id, r, true);
bdo.WriteReST(f);
*/
fprintf(f, "%s\n", doc->ReStructuredText().c_str());
}
static bool ComponentsMatch(const plugin::Plugin* p, plugin::component::Type t,
@ -610,16 +606,14 @@ static bool ComponentsMatch(const plugin::Plugin* p, plugin::component::Type t,
return true;
}
void CreateProtoAnalyzerDoc(const char* filename)
void CreateProtoAnalyzerDoc(FILE* f)
{
FILE* f = fopen(filename, "w");
fprintf(f, "Protocol Analyzers\n");
fprintf(f, "==================\n\n\n");
fprintf(f, "==================\n\n");
fprintf(f, ".. contents::\n");
fprintf(f, " :depth: 1\n\n");
WriteAnalyzerTagDefn(f, analyzer_mgr->GetTagEnumType(), "Analyzer");
WriteAnalyzerTagDefn(f, "Analyzer");
plugin::Manager::plugin_list plugins = plugin_mgr->Plugins();
plugin::Manager::plugin_list::const_iterator it;
@ -642,16 +636,14 @@ void CreateProtoAnalyzerDoc(const char* filename)
fclose(f);
}
void CreateFileAnalyzerDoc(const char* filename)
void CreateFileAnalyzerDoc(FILE* f)
{
FILE* f = fopen(filename, "w");
fprintf(f, "File Analyzers\n");
fprintf(f, "==============\n\n");
fprintf(f, ".. contents::\n");
fprintf(f, " :depth: 1\n\n");
WriteAnalyzerTagDefn(f, file_mgr->GetTagEnumType(), "Files");
WriteAnalyzerTagDefn(f, "Files");
plugin::Manager::plugin_list plugins = plugin_mgr->Plugins();
plugin::Manager::plugin_list::const_iterator it;

8
src/BroDoc.h
View file

@ -409,14 +409,14 @@ private:
/**
* Writes out plugin index documentation for all analyzer plugins.
* @param filename the name of the file to write.
* @param f an open file stream to write docs into.
*/
void CreateProtoAnalyzerDoc(const char* filename);
void CreateProtoAnalyzerDoc(FILE* f);
/**
* Writes out plugin index documentation for all file analyzer plugins.
* @param filename the name of the file to write.
* @param f an open file stream to write docs into.
*/
void CreateFileAnalyzerDoc(const char* filename);
void CreateFileAnalyzerDoc(FILE* f);
#endif

1
src/Desc.h
View file

@ -69,6 +69,7 @@ public:
void PopIndent();
void PopIndentNoNL();
int GetIndentLevel() const { return indent_level; }
void ClearIndentLevel() { indent_level = 0; }
int IndentSpaces() const { return indent_with_spaces; }
void SetIndentSpaces(int i) { indent_with_spaces = i; }

17
src/ID.cc
View file

@ -14,6 +14,7 @@
#include "PersistenceSerializer.h"
#include "Scope.h"
#include "Traverse.h"
#include "broxygen/Manager.h"
ID::ID(const char* arg_name, IDScope arg_scope, bool arg_is_export)
{
@ -618,7 +619,6 @@ void ID::DescribeExtended(ODesc* d) const
void ID::DescribeReSTShort(ODesc* d) const
{
/* TODO
if ( is_type )
d->Add(":bro:type:`");
else
@ -632,8 +632,8 @@ void ID::DescribeReSTShort(ODesc* d) const
d->Add(": ");
d->Add(":bro:type:`");
if ( ! is_type && type->GetTypeID() )
d->Add(type->GetTypeID());
if ( ! is_type && ! type->GetName().empty() )
d->Add(type->GetName().c_str());
else
{
TypeTag t = type->Tag();
@ -644,14 +644,14 @@ void ID::DescribeReSTShort(ODesc* d) const
break;
case TYPE_FUNC:
d->Add(type->AsFuncType()->FlavorString());
d->Add(type->AsFuncType()->FlavorString().c_str());
break;
case TYPE_ENUM:
if ( is_type )
d->Add(type_name(t));
else
d->Add(type->AsEnumType()->Name().c_str());
d->Add(broxygen_mgr->GetEnumTypeName(Name()).c_str());
break;
default:
@ -668,12 +668,11 @@ void ID::DescribeReSTShort(ODesc* d) const
d->SP();
attrs->DescribeReST(d);
}
*/
}
void ID::DescribeReST(ODesc* d, bool is_role) const
void ID::DescribeReST(ODesc* d, bool roles_only) const
{
if ( is_role )
if ( roles_only )
{
if ( is_type )
d->Add(":bro:type:`");
@ -705,7 +704,7 @@ void ID::DescribeReST(ODesc* d, bool is_role) const
d->Add("`");
}
else
type->DescribeReST(d);
type->DescribeReST(d, roles_only);
d->NL();
}

2
src/ID.h
View file

@ -84,7 +84,7 @@ public:
// Adds type and value to description.
void DescribeExtended(ODesc* d) const;
// Produces a description that's reST-ready.
void DescribeReST(ODesc* d, bool is_role=false) const;
void DescribeReST(ODesc* d, bool roles_only = false) const;
void DescribeReSTShort(ODesc* d) const;
bool Serialize(SerialInfo* info) const;

163
src/Type.cc
View file

@ -117,16 +117,17 @@ BroType* BroType::Clone() const
sinfo.cache = false;
this->Serialize(&sinfo);
char* data;
char* data = 0;
uint32 len = form->EndWrite(&data);
form->StartRead(data, len);
UnserialInfo uinfo(&ss);
uinfo.cache = false;
BroType* rval = this->Unserialize(&uinfo);
BroType* rval = this->Unserialize(&uinfo, false);
assert(rval != this);
delete [] data;
return rval;
}
@ -173,11 +174,9 @@ void BroType::Describe(ODesc* d) const
}
}
void BroType::DescribeReST(ODesc* d) const
void BroType::DescribeReST(ODesc* d, bool roles_only) const
{
d->Add(":bro:type:`");
d->Add(type_name(Tag()));
d->Add("`");
d->Add(fmt(":bro:type:`%s`", type_name(Tag())));
}
void BroType::SetError()
@ -200,7 +199,7 @@ bool BroType::Serialize(SerialInfo* info) const
return ret;
}
BroType* BroType::Unserialize(UnserialInfo* info, TypeTag want)
BroType* BroType::Unserialize(UnserialInfo* info, bool use_existing)
{
// To avoid external Broccoli clients needing to always send full type
// objects, we allow them to give us only the name of a type. To
@ -234,8 +233,8 @@ BroType* BroType::Unserialize(UnserialInfo* info, TypeTag want)
BroType* t = (BroType*) SerialObj::Unserialize(info, SER_BRO_TYPE);
if ( ! t )
return 0;
if ( ! t || ! use_existing )
return t;
if ( ! t->name.empty() )
{
@ -458,7 +457,7 @@ void IndexType::Describe(ODesc* d) const
}
}
void IndexType::DescribeReST(ODesc* d) const
void IndexType::DescribeReST(ODesc* d, bool roles_only) const
{
d->Add(":bro:type:`");
@ -484,7 +483,7 @@ void IndexType::DescribeReST(ODesc* d) const
d->Add("`");
}
else
t->DescribeReST(d);
t->DescribeReST(d, roles_only);
}
d->Add("]");
@ -500,7 +499,7 @@ void IndexType::DescribeReST(ODesc* d) const
d->Add("`");
}
else
yield_type->DescribeReST(d);
yield_type->DescribeReST(d, roles_only);
}
}
@ -775,7 +774,7 @@ void FuncType::Describe(ODesc* d) const
}
}
void FuncType::DescribeReST(ODesc* d) const
void FuncType::DescribeReST(ODesc* d, bool roles_only) const
{
d->Add(":bro:type:`");
d->Add(FlavorString());
@ -795,7 +794,7 @@ void FuncType::DescribeReST(ODesc* d) const
d->Add("`");
}
else
yield->DescribeReST(d);
yield->DescribeReST(d, roles_only);
}
}
@ -927,7 +926,7 @@ TypeDecl* TypeDecl::Unserialize(UnserialInfo* info)
return t;
}
void TypeDecl::DescribeReST(ODesc* d) const
void TypeDecl::DescribeReST(ODesc* d, bool roles_only) const
{
d->Add(id);
d->Add(": ");
@ -939,7 +938,7 @@ void TypeDecl::DescribeReST(ODesc* d) const
d->Add("`");
}
else
type->DescribeReST(d);
type->DescribeReST(d, roles_only);
if ( attrs )
{
@ -1037,9 +1036,13 @@ void RecordType::Describe(ODesc* d) const
}
}
void RecordType::DescribeReST(ODesc* d) const
void RecordType::DescribeReST(ODesc* d, bool roles_only) const
{
d->Add(":bro:type:`record`");
if ( num_fields == 0 )
return;
d->NL();
DescribeFieldsReST(d, false);
}
@ -1136,7 +1139,53 @@ void RecordType::DescribeFieldsReST(ODesc* d, bool func_args) const
}
}
FieldDecl(i)->DescribeReST(d);
const TypeDecl* td = FieldDecl(i);
td->DescribeReST(d);
if ( func_args )
continue;
using broxygen::IdentifierDocument;
IdentifierDocument* doc = broxygen_mgr->GetIdentifierDoc(GetName());
if ( ! doc )
{
reporter->InternalWarning("Failed to lookup record doc: %s",
GetName().c_str());
continue;
}
string field_from_script = doc->GetDeclaringScriptForField(td->id);
string type_from_script;
if ( doc->GetDeclaringScript() )
type_from_script = doc->GetDeclaringScript()->Name();
if ( ! field_from_script.empty() &&
field_from_script != type_from_script )
{
d->PushIndent();
d->Add(fmt("(from ``redef`` in :doc:`%s`)",
field_from_script.c_str()));
d->PopIndentNoNL();
}
vector<string> cmnts = doc->GetFieldComments(td->id);
if ( cmnts.empty() )
continue;
d->PushIndent();
for ( size_t i = 0; i < cmnts.size(); ++i )
{
if ( i > 0 )
d->NL();
d->Add(cmnts[i].c_str());
}
d->PopIndentNoNL();
}
if ( ! func_args )
@ -1415,10 +1464,82 @@ const char* EnumType::Lookup(bro_int_t value)
return 0;
}
void EnumType::DescribeReST(ODesc* d) const
void EnumType::DescribeReST(ODesc* d, bool roles_only) const
{
// TODO: this probably goes away
d->Add(":bro:type:`enum`");
// Create temporary, reverse name map so that enums can be documented
// in ascending order of their actual integral value instead of by name.
typedef map< bro_int_t, const char* > RevNameMap;
RevNameMap rev;
for ( NameMap::const_iterator it = names.begin(); it != names.end(); ++it )
rev[it->second] = it->first;
for ( RevNameMap::const_iterator it = rev.begin(); it != rev.end(); ++it )
{
d->NL();
d->PushIndent();
if ( roles_only )
d->Add(fmt(":bro:enum:`%s`", it->second));
else
d->Add(fmt(".. bro:enum:: %s %s", it->second, GetName().c_str()));
using broxygen::IdentifierDocument;
IdentifierDocument* doc = broxygen_mgr->GetIdentifierDoc(it->second);
if ( ! doc )
{
reporter->InternalWarning("Enum %s documentation lookup failure",
it->second);
continue;
}
string enum_from_script;
string type_from_script;
if ( doc->GetDeclaringScript() )
enum_from_script = doc->GetDeclaringScript()->Name();
IdentifierDocument* type_doc = broxygen_mgr->GetIdentifierDoc(GetName());
if ( type_doc && type_doc->GetDeclaringScript() )
type_from_script = type_doc->GetDeclaringScript()->Name();
if ( ! enum_from_script.empty() &&
enum_from_script != type_from_script )
{
d->NL();
d->PushIndent();
d->Add(fmt("(from ``redef`` in :doc:`%s`)",
enum_from_script.c_str()));
d->PopIndentNoNL();
}
vector<string> cmnts = doc->GetComments();
if ( cmnts.empty() )
{
d->PopIndentNoNL();
continue;
}
d->NL();
d->PushIndent();
for ( size_t i = 0; i < cmnts.size(); ++i )
{
if ( i > 0 )
d->NL();
d->Add(cmnts[i].c_str());
}
d->PopIndentNoNL();
d->PopIndentNoNL();
}
}
IMPLEMENT_SERIAL(EnumType, SER_ENUM_TYPE);

14
src/Type.h
View file

@ -227,12 +227,12 @@ public:
BroType* Ref() { ::Ref(this); return this; }
virtual void Describe(ODesc* d) const;
virtual void DescribeReST(ODesc* d) const;
virtual void DescribeReST(ODesc* d, bool roles_only = false) const;
virtual unsigned MemoryAllocation() const;
bool Serialize(SerialInfo* info) const;
static BroType* Unserialize(UnserialInfo* info, TypeTag want = TYPE_ANY);
static BroType* Unserialize(UnserialInfo* info, bool use_existing = true);
void SetName(const string& arg_name) { name = arg_name; }
string GetName() const { return name; }
@ -305,7 +305,7 @@ public:
BroType* YieldType();
void Describe(ODesc* d) const;
void DescribeReST(ODesc* d) const;
void DescribeReST(ODesc* d, bool roles_only = false) const;
// Returns true if this table is solely indexed by subnet.
bool IsSubNetIndex() const;
@ -379,7 +379,7 @@ public:
TypeList* ArgTypes() const { return arg_types; }
void Describe(ODesc* d) const;
void DescribeReST(ODesc* d) const;
void DescribeReST(ODesc* d, bool roles_only = false) const;
protected:
FuncType() { args = 0; arg_types = 0; yield = 0; flavor = FUNC_FLAVOR_FUNCTION; }
@ -416,7 +416,7 @@ public:
bool Serialize(SerialInfo* info) const;
static TypeDecl* Unserialize(UnserialInfo* info);
virtual void DescribeReST(ODesc* d) const;
virtual void DescribeReST(ODesc* d, bool roles_only = false) const;
BroType* type;
Attributes* attrs;
@ -454,7 +454,7 @@ public:
const char* AddFields(type_decl_list* types, attr_list* attr);
void Describe(ODesc* d) const;
void DescribeReST(ODesc* d) const;
void DescribeReST(ODesc* d, bool roles_only = false) const;
void DescribeFields(ODesc* d) const;
void DescribeFieldsReST(ODesc* d, bool func_args) const;
@ -527,7 +527,7 @@ public:
bro_int_t Lookup(const string& module_name, const char* name);
const char* Lookup(bro_int_t value); // Returns 0 if not found
void DescribeReST(ODesc* d) const;
void DescribeReST(ODesc* d, bool roles_only = false) const;
protected:
DECLARE_SERIAL(EnumType)

12
src/Var.cc
View file

@ -261,17 +261,17 @@ extern Expr* add_and_assign_local(ID* id, Expr* init, Val* val)
void add_type(ID* id, BroType* t, attr_list* attr)
{
string new_type_name(id->Name());
string old_type_name(t->GetName());
string new_type_name = id->Name();
string old_type_name = t->GetName();
BroType* tnew = 0;
if ( (t->Tag() == TYPE_RECORD || t->Tag() == TYPE_ENUM) &&
! old_type_name.empty() )
old_type_name.empty() )
// An extensible type (record/enum) being declared for first time.
tnew = t;
else
// Clone the type to preserve type name aliasing.
tnew = t->Clone();
else
// An extensible types (record/enum) being declared for first time.
tnew = t;
type_aliases[new_type_name].insert(tnew);

350
src/broxygen/Configuration.cc
View file

@ -1,4 +1,5 @@
#include "Configuration.h"
#include "Manager.h"
#include "util.h"
#include "Reporter.h"
@ -7,6 +8,8 @@
#include <vector>
#include <algorithm>
#include <map>
#include <cstdio>
#include <sys/stat.h>
using namespace broxygen;
using namespace std;
@ -16,12 +19,12 @@ typedef map<string, Target::factory_fn> target_factory_map;
static target_factory_map create_target_factory_map()
{
target_factory_map rval;
rval["package_index"] = &PackageTarget::Instantiate;
rval["package_index"] = &PackageIndexTarget::Instantiate;
rval["package"] = &PackageTarget::Instantiate;
rval["proto_analyzer"] = &ProtoAnalyzerTarget::Instantiate;
rval["file_analyzer"] = &FileAnalyzerTarget::Instantiate;
rval["script_summary"] = &ScriptTarget::Instantiate;
rval["script_index"] = &ScriptTarget::Instantiate;
rval["script_summary"] = &ScriptSummaryTarget::Instantiate;
rval["script_index"] = &ScriptIndexTarget::Instantiate;
rval["script"] = &ScriptTarget::Instantiate;
rval["identifier"] = &IdentifierTarget::Instantiate;
return rval;
@ -29,41 +32,300 @@ static target_factory_map create_target_factory_map()
static target_factory_map target_instantiators = create_target_factory_map();
bool Target::MatchesPattern(Document* doc) const
struct TargetFile {
TargetFile(const string& arg_name)
: name(arg_name), f()
{
// TODO: prefix matching or full regex?
if ( name.find('/') != string::npos )
{
string dir = SafeDirname(name).result;
if ( doc->Name() == pattern )
{
DBG_LOG(DBG_BROXYGEN, "Doc '%s'' matched pattern for target '%s'",
doc->Name().c_str(), name.c_str());
return true;
if ( ! ensure_intermediate_dirs(dir.c_str()) )
reporter->FatalError("Broxygen failed to make dir %s",
dir.c_str());
}
return false;
f = fopen(name.c_str(), "w");
if ( ! f )
reporter->FatalError("Broxygen failed to open '%s' for writing: %s",
name.c_str(), strerror(errno));
}
~TargetFile()
{
if ( f )
fclose(f);
DBG_LOG(DBG_BROXYGEN, "Wrote out-of-date target '%s'", name.c_str());
}
string name;
FILE* f;
};
template<class T>
void filter_matching_docs(const std::vector<Document*>& filter_from, Target* t)
static vector<T*> filter_matching_docs(const vector<Document*>& from, Target* t)
{
for ( size_t i = 0; i < filter_from.size(); ++i )
vector<T*> rval;
for ( size_t i = 0; i < from.size(); ++i )
{
T* d = dynamic_cast<T*>(filter_from[i]);
T* d = dynamic_cast<T*>(from[i]);
if ( ! d )
continue;
if ( t->MatchesPattern(d) )
t->AddDependency(d);
}
}
void IdentifierTarget::DoFindDependencies(const std::vector<Document*>& docs)
{
filter_matching_docs<IdentifierDocument>(docs, this);
DBG_LOG(DBG_BROXYGEN, "Doc '%s' matched pattern for target '%s'",
d->Name().c_str(), t->Name().c_str());
rval.push_back(d);
}
}
Config::Config(const string& file, const string& delim)
return rval;
}
Target::Target(const string& arg_name, const string& arg_pattern)
: name(arg_name), pattern(arg_pattern), prefix()
{
size_t pos = pattern.find('*');
if ( pos == 0 || pos == string::npos )
return;
prefix = pattern.substr(0, pos);
}
bool Target::MatchesPattern(Document* doc) const
{
if ( pattern == "*" )
return true;
if ( prefix.empty() )
return doc->Name() == pattern;
return ! strncmp(doc->Name().c_str(), prefix.c_str(), prefix.size());
}
void AnalyzerTarget::DoFindDependencies(const std::vector<Document *>& docs)
{
// TODO: really should add to dependency list the tag type's ID and
// all bif items for matching analyzer plugins, but that's all dependent
// on the bro binary itself, so I'm cheating.
}
void AnalyzerTarget::DoGenerate() const
{
if ( broxygen_mgr->IsUpToDate(Name(), vector<Document*>()) )
return;
if ( Pattern() != "*" )
reporter->InternalWarning("Broxygen only implements analyzer target"
" pattern '*'");
TargetFile file(Name());
doc_creator_callback(file.f);
}
void PackageTarget::DoFindDependencies(const vector<Document*>& docs)
{
pkg_deps = filter_matching_docs<PackageDocument>(docs, this);
if ( pkg_deps.empty() )
reporter->FatalError("No match for Broxygen target '%s' pattern '%s'",
Name().c_str(), Pattern().c_str());
for ( size_t i = 0; i < docs.size(); ++i )
{
ScriptDocument* script = dynamic_cast<ScriptDocument*>(docs[i]);
if ( ! script )
continue;
for ( size_t j = 0; j < pkg_deps.size(); ++j )
{
if ( strncmp(script->Name().c_str(), pkg_deps[j]->Name().c_str(),
pkg_deps[j]->Name().size()))
continue;
DBG_LOG(DBG_BROXYGEN, "Script %s associated with package %s",
script->Name().c_str(), pkg_deps[j]->Name().c_str());
pkg_manifest[pkg_deps[j]].push_back(script);
script_deps.push_back(script);
}
}
}
void PackageTarget::DoGenerate() const
{
if ( broxygen_mgr->IsUpToDate(Name(), script_deps) )
return;
TargetFile file(Name());
for ( manifest_t::const_iterator it = pkg_manifest.begin();
it != pkg_manifest.end(); ++it )
{
fprintf(file.f, "Package: %s\n\n", it->first->Name().c_str());
for ( size_t i = 0; i < it->second.size(); ++i )
{
fprintf(file.f, " :doc:`%s`\n", it->second[i]->Name().c_str());
vector<string> cmnts = it->second[i]->GetComments();
for ( size_t j = 0; j < cmnts.size(); ++j )
fprintf(file.f, " %s\n", cmnts[j].c_str());
fprintf(file.f, "\n");
}
}
}
void PackageIndexTarget::DoFindDependencies(const vector<Document*>& docs)
{
pkg_deps = filter_matching_docs<PackageDocument>(docs, this);
if ( pkg_deps.empty() )
reporter->FatalError("No match for Broxygen target '%s' pattern '%s'",
Name().c_str(), Pattern().c_str());
}
void PackageIndexTarget::DoGenerate() const
{
if ( broxygen_mgr->IsUpToDate(Name(), pkg_deps) )
return;
TargetFile file(Name());
for ( size_t i = 0; i < pkg_deps.size(); ++i )
fprintf(file.f, "%s\n", pkg_deps[i]->ReStructuredText().c_str());
}
void ScriptTarget::DoFindDependencies(const vector<Document*>& docs)
{
script_deps = filter_matching_docs<ScriptDocument>(docs, this);
if ( script_deps.empty() )
reporter->FatalError("No match for Broxygen target '%s' pattern '%s'",
Name().c_str(), Pattern().c_str());
}
void ScriptTarget::DoGenerate() const
{
if ( Name()[Name().size() - 1] == '/' )
{
// Target name is a dir, matching scripts are written within that dir
// with a dir tree that parallels the script's BROPATH location.
for ( size_t i = 0; i < script_deps.size(); ++i )
{
string target_filename = Name() + script_deps[i]->Name();
size_t pos = target_filename.rfind(".bro");
if ( pos == target_filename.size() - 4 )
target_filename.replace(pos, 4, ".rst");
else
target_filename += ".rst";
vector<ScriptDocument*> dep;
dep.push_back(script_deps[i]);
if ( broxygen_mgr->IsUpToDate(target_filename, dep) )
continue;
TargetFile file(target_filename);
fprintf(file.f, "%s\n", script_deps[i]->ReStructuredText().c_str());
}
// TODO: could possibly take inventory of files in the dir beforehand,
// track all files written, then compare afterwards in order to remove
// stale files.
return;
}
// Target is a single file, all matching scripts get written there.
if ( broxygen_mgr->IsUpToDate(Name(), script_deps) )
return;
TargetFile file(Name());
for ( size_t i = 0; i < script_deps.size(); ++i )
fprintf(file.f, "%s\n", script_deps[i]->ReStructuredText().c_str());
}
void ScriptSummaryTarget::DoGenerate() const
{
if ( broxygen_mgr->IsUpToDate(Name(), script_deps) )
return;
TargetFile file(Name());
for ( size_t i = 0; i < script_deps.size(); ++i )
{
ScriptDocument* d = dynamic_cast<ScriptDocument*>(script_deps[i]);
if ( ! d )
continue;
fprintf(file.f, ":doc:`/scripts/%s`\n", d->Name().c_str());
vector<string> cmnts = d->GetComments();
for ( size_t i = 0; i < cmnts.size(); ++i )
fprintf(file.f, " %s\n", cmnts[i].c_str());
fprintf(file.f, "\n");
}
}
void ScriptIndexTarget::DoGenerate() const
{
if ( broxygen_mgr->IsUpToDate(Name(), script_deps) )
return;
TargetFile file(Name());
fprintf(file.f, ".. toctree::\n");
fprintf(file.f, " :maxdepth: 1\n\n");
for ( size_t i = 0; i < script_deps.size(); ++i )
{
ScriptDocument* d = dynamic_cast<ScriptDocument*>(script_deps[i]);
if ( ! d )
continue;
fprintf(file.f, " %s <%s>\n", d->Name().c_str(), d->Name().c_str());
}
}
void IdentifierTarget::DoFindDependencies(const vector<Document*>& docs)
{
id_deps = filter_matching_docs<IdentifierDocument>(docs, this);
if ( id_deps.empty() )
reporter->FatalError("No match for Broxygen target '%s' pattern '%s'",
Name().c_str(), Pattern().c_str());
}
void IdentifierTarget::DoGenerate() const
{
if ( broxygen_mgr->IsUpToDate(Name(), id_deps) )
return;
TargetFile file(Name());
for ( size_t i = 0; i < id_deps.size(); ++i )
fprintf(file.f, "%s\n\n", id_deps[i]->ReStructuredText().c_str());
}
Config::Config(const string& arg_file, const string& delim)
: file(arg_file), targets()
{
if ( file.empty() )
return;
@ -71,53 +333,71 @@ Config::Config(const string& file, const string& delim)
ifstream f(file.c_str());
if ( ! f.is_open() )
reporter->InternalError("failed to open Broxygen config file %s: %s",
reporter->FatalError("failed to open Broxygen config file '%s': %s",
file.c_str(), strerror(errno));
string line;
unsigned int line_number = 0;
while ( getline(f, line) )
{
++line_number;
vector<string> tokens;
tokenize_string(line, delim, &tokens);
tokens.erase(remove(tokens.begin(), tokens.end(), ""), tokens.end());
if ( tokens.empty() )
// Blank line.
continue;
if ( ! tokens[0].empty() && tokens[0][0] == '#' )
// Comment
continue;
if ( tokens.size() != 3 )
reporter->InternalError("malformed Broxygen target: %s",
line.c_str());
reporter->FatalError("malformed Broxygen target in %s:%u: %s",
file.c_str(), line_number, line.c_str());
target_factory_map::const_iterator it =
target_instantiators.find(tokens[0]);
if ( it == target_instantiators.end() )
reporter->InternalError("unkown Broxygen target type: %s",
reporter->FatalError("unkown Broxygen target type: %s",
tokens[0].c_str());
targets.push_back(it->second(tokens[1], tokens[2]));
}
if ( f.bad() )
reporter->InternalError("error reading Broxygen config file %s: %s",
reporter->InternalError("error reading Broxygen config file '%s': %s",
file.c_str(), strerror(errno));
}
Config::~Config()
{
for ( target_list::const_iterator it = targets.begin();
it != targets.end(); ++it )
delete *it;
for ( size_t i = 0; i < targets.size(); ++i )
delete targets[i];
}
void Config::FindDependencies(const vector<Document*>& docs)
{
for ( target_list::const_iterator it = targets.begin();
it != targets.end(); ++it )
(*it)->FindDependencies(docs);
for ( size_t i = 0; i < targets.size(); ++i )
targets[i]->FindDependencies(docs);
}
void Config::GenerateDocs() const
{
for ( target_list::const_iterator it = targets.begin();
it != targets.end(); ++it )
(*it)->Generate();
for ( size_t i = 0; i < targets.size(); ++i )
targets[i]->Generate();
}
time_t Config::GetModificationTime() const
{
struct stat s;
if ( stat(file.c_str(), &s) < 0 )
reporter->InternalError("Broxygen can't stat config file %s: %s",
file.c_str(), strerror(errno));
return s.st_mtime;
}

157
src/broxygen/Configuration.h
View file

@ -2,10 +2,11 @@
#define BROXYGEN_CONFIGURATION_H
#include "Document.h"
#include "BroDoc.h"
#include <string>
#include <vector>
#include <list>
#include <map>
namespace broxygen {
@ -16,7 +17,8 @@ public:
typedef Target* (*factory_fn)(const std::string&, const std::string&);
virtual ~Target() { }
virtual ~Target()
{ }
void FindDependencies(const std::vector<Document*>& docs)
{ DoFindDependencies(docs); }
@ -26,27 +28,47 @@ public:
bool MatchesPattern(Document* doc) const;
void AddDependency(Document* doc)
{ dependencies.push_back(doc); }
std::string Name() const
{ return name; }
std::string Pattern() const
{ return pattern; }
protected:
Target(const std::string& arg_name, const std::string& arg_pattern)
: name(arg_name), pattern(arg_pattern)
{ }
std::string name;
std::string pattern;
std::list<Document*> dependencies;
Target(const std::string& arg_name, const std::string& arg_pattern);
private:
virtual void DoFindDependencies(const std::vector<Document*>& docs) = 0;
virtual void DoGenerate() const = 0;
std::string name;
std::string pattern;
std::string prefix;
};
class ProtoAnalyzerTarget : public Target {
class AnalyzerTarget : public Target {
protected:
typedef void (*doc_creator_fn)(FILE*);
AnalyzerTarget(const std::string& name, const std::string& pattern,
doc_creator_fn cb)
: Target(name, pattern), doc_creator_callback(cb)
{ }
private:
void DoFindDependencies(const std::vector<Document*>& docs);
void DoGenerate() const;
doc_creator_fn doc_creator_callback;
};
class ProtoAnalyzerTarget : public AnalyzerTarget {
public:
static Target* Instantiate(const std::string& name,
@ -56,17 +78,11 @@ public:
private:
ProtoAnalyzerTarget(const std::string& name, const std::string& pattern)
: Target(name, pattern)
: AnalyzerTarget(name, pattern, &CreateProtoAnalyzerDoc)
{ }
void DoFindDependencies(const std::vector<Document*>& docs)
{ /* TODO */ }
void DoGenerate() const
{ /* TODO */ }
};
class FileAnalyzerTarget : public Target {
class FileAnalyzerTarget : public AnalyzerTarget {
public:
static Target* Instantiate(const std::string& name,
@ -76,14 +92,8 @@ public:
private:
FileAnalyzerTarget(const std::string& name, const std::string& pattern)
: Target(name, pattern)
: AnalyzerTarget(name, pattern, &CreateFileAnalyzerDoc)
{ }
void DoFindDependencies(const std::vector<Document*>& docs)
{ /* TODO */ }
void DoGenerate() const
{ /* TODO */ }
};
class PackageTarget : public Target {
@ -96,14 +106,37 @@ public:
private:
PackageTarget(const std::string& name, const std::string& pattern)
: Target(name, pattern)
: Target(name, pattern), pkg_deps(), script_deps(), pkg_manifest()
{ }
void DoFindDependencies(const std::vector<Document*>& docs)
{ /* TODO */ }
void DoFindDependencies(const std::vector<Document*>& docs);
void DoGenerate() const
{ /* TODO */ }
void DoGenerate() const;
std::vector<PackageDocument*> pkg_deps;
std::vector<ScriptDocument*> script_deps;
typedef std::map<PackageDocument*,std::vector<ScriptDocument*> > manifest_t;
manifest_t pkg_manifest;
};
class PackageIndexTarget : public Target {
public:
static Target* Instantiate(const std::string& name,
const std::string& pattern)
{ return new PackageIndexTarget(name, pattern); }
private:
PackageIndexTarget(const std::string& name, const std::string& pattern)
: Target(name, pattern), pkg_deps()
{ }
void DoFindDependencies(const std::vector<Document*>& docs);
void DoGenerate() const;
std::vector<PackageDocument*> pkg_deps;
};
class ScriptTarget : public Target {
@ -113,17 +146,51 @@ public:
const std::string& pattern)
{ return new ScriptTarget(name, pattern); }
private:
protected:
ScriptTarget(const std::string& name, const std::string& pattern)
: Target(name, pattern)
: Target(name, pattern), script_deps()
{ }
void DoFindDependencies(const std::vector<Document*>& docs)
{ /* TODO */ }
std::vector<ScriptDocument*> script_deps;
void DoGenerate() const
{ /* TODO */ }
private:
void DoFindDependencies(const std::vector<Document*>& docs);
void DoGenerate() const;
};
class ScriptSummaryTarget : public ScriptTarget {
public:
static Target* Instantiate(const std::string& name,
const std::string& pattern)
{ return new ScriptSummaryTarget(name, pattern); }
private:
ScriptSummaryTarget(const std::string& name, const std::string& pattern)
: ScriptTarget(name, pattern)
{ }
void DoGenerate() const /* override */;
};
class ScriptIndexTarget : public ScriptTarget {
public:
static Target* Instantiate(const std::string& name,
const std::string& pattern)
{ return new ScriptIndexTarget(name, pattern); }
private:
ScriptIndexTarget(const std::string& name, const std::string& pattern)
: ScriptTarget(name, pattern)
{ }
void DoGenerate() const /* override */;
};
class IdentifierTarget : public Target {
@ -136,13 +203,14 @@ public:
private:
IdentifierTarget(const std::string& name, const std::string& pattern)
: Target(name, pattern)
: Target(name, pattern), id_deps()
{ }
void DoFindDependencies(const std::vector<Document*>& docs);
void DoGenerate() const
{ /* TODO */ }
void DoGenerate() const;
std::vector<IdentifierDocument*> id_deps;
};
class Config {
@ -156,11 +224,12 @@ public:
void GenerateDocs() const;
time_t GetModificationTime() const;
private:
typedef std::list<Target*> target_list;
target_list targets;
std::string file;
std::vector<Target*> targets;
};

651
src/broxygen/Document.cc
View file

@ -1,38 +1,134 @@
#include "Document.h"
#include "Manager.h"
#include "util.h"
#include "Val.h"
#include "Desc.h"
#include "Reporter.h"
#include <fstream>
#include <sys/stat.h>
using namespace broxygen;
using namespace std;
static string ImplodeStringVec(const vector<string>& v)
static bool is_public_api(const ID* id)
{
string rval;
for ( size_t i = 0; i < v.size(); ++i )
{
if ( i > 0 )
rval += '\n';
rval += v[i];
return (id->Scope() == SCOPE_GLOBAL) ||
(id->Scope() == SCOPE_MODULE && id->IsExport());
}
return rval;
static bool prettify_params(string& s)
{
size_t identifier_start_pos = 0;
bool in_identifier = false;
string identifier;
for ( size_t i = 0; i < s.size(); ++i )
{
char next = s[i];
if ( ! in_identifier )
{
// Pass by leading whitespace.
if ( isspace(next) )
continue;
// Only allow alphabetic and '_' as first char of identifier.
if ( isalpha(next) || next == '_' )
{
identifier_start_pos = i;
identifier += next;
in_identifier = true;
continue;
}
// Don't need to change anything.
return false;
}
// All other characters of identifier are alphanumeric or '_'.
if ( isalnum(next) || next == '_' )
{
identifier += next;
continue;
}
if ( next == ':' )
{
if ( i + 1 < s.size() && s[i + 1] == ':' )
{
// It's part of an identifier's namespace scoping.
identifier += next;
identifier += s[i + 1];
++i;
continue;
}
// Prettify function param/return value reST markup.
string subst;
if ( identifier == "Returns" )
subst = ":returns";
else
subst = ":param " + identifier;
s.replace(identifier_start_pos, identifier.size(), subst);
return true;
}
// Don't need to change anything.
return false;
}
return false;
}
static string make_heading(const string& heading, char underline)
{
return heading + "\n" + string(heading.size(), underline) + "\n";
}
PackageDocument::PackageDocument(const string& arg_name)
: Document(),
pkg_name(arg_name)
pkg_name(arg_name), readme()
{
// TODO: probably need to determine modification times of all files
// within the directory, recursively
string readme_file = find_file(pkg_name + "/README", bro_path());
if ( readme_file.empty() )
return;
ifstream f(readme_file.c_str());
if ( ! f.is_open() )
reporter->InternalWarning("Broxygen failed to open '%s': %s",
readme_file.c_str(), strerror(errno));
string line;
while ( getline(f, line) )
readme.push_back(line);
if ( f.bad() )
reporter->InternalWarning("Broxygen error reading '%s': %s",
readme_file.c_str(), strerror(errno));
}
IdentifierDocument::IdentifierDocument(ID* arg_id)
string PackageDocument::DoReStructuredText(bool roles_only) const
{
string rval = fmt(":doc:`%s <%s/index>`\n\n", pkg_name.c_str(),
pkg_name.c_str());
for ( size_t i = 0; i < readme.size(); ++i )
rval += " " + readme[i] + "\n";
return rval;
}
IdentifierDocument::IdentifierDocument(ID* arg_id, ScriptDocument* script)
: Document(),
comments(), id(arg_id), initial_val_desc(), redefs(), fields(),
last_field_seen()
last_field_seen(), declaring_script(script)
{
Ref(id);
@ -86,36 +182,539 @@ void IdentifierDocument::AddRecordField(const TypeDecl* field,
last_field_seen = rf;
}
string IdentifierDocument::GetComments() const
vector<string> IdentifierDocument::GetComments() const
{
return ImplodeStringVec(comments);
return comments;
}
string IdentifierDocument::GetFieldComments(const string& field) const
vector<string> IdentifierDocument::GetFieldComments(const string& field) const
{
record_field_map::const_iterator it = fields.find(field);
if ( it == fields.end() )
return string();
return vector<string>();
return ImplodeStringVec(it->second->comments);
return it->second->comments;
}
ScriptDocument::ScriptDocument(const string& arg_name)
list<IdentifierDocument::Redefinition>
IdentifierDocument::GetRedefs(const string& from_script) const
{
list<Redefinition> rval;
for ( redef_list::const_iterator it = redefs.begin(); it != redefs.end();
++it )
{
if ( from_script == (*it)->from_script )
rval.push_back(*(*it));
}
return rval;
}
string IdentifierDocument::GetDeclaringScriptForField(const string& field) const
{
record_field_map::const_iterator it = fields.find(field);
if ( it == fields.end() )
return "";
return it->second->from_script;
}
string IdentifierDocument::DoReStructuredText(bool roles_only) const
{
ODesc d;
d.SetIndentSpaces(3);
d.SetQuotes(true);
id->DescribeReST(&d, roles_only);
if ( comments.empty() )
return d.Description();
d.ClearIndentLevel();
d.PushIndent();
for ( size_t i = 0; i < comments.size(); ++i )
{
if ( i > 0 )
d.NL();
if ( IsFunc(id->Type()->Tag()) )
{
string s = comments[i];
if ( prettify_params(s) )
d.NL();
d.Add(s.c_str());
}
else
d.Add(comments[i].c_str());
}
return d.Description();
}
ScriptDocument::ScriptDocument(const string& arg_name, const string& arg_path)
: Document(),
name(arg_name),
name(arg_name), path(arg_path),
is_pkg_loader(SafeBasename(name).result == PACKAGE_LOADER),
dependencies(), module_usages(), comments(), identifier_docs(), redefs()
dependencies(), module_usages(), comments(), identifier_docs(),
options(), constants(), state_vars(), types(), events(), hooks(),
functions(), redefs()
{
}
void ScriptDocument::AddIdentifierDoc(IdentifierDocument* doc)
{
identifier_docs[doc->Name()] = doc;
// TODO: sort things (e.g. function flavor, state var vs. option var)
}
string ScriptDocument::GetComments() const
void ScriptDocument::DoInitPostScript()
{
return ImplodeStringVec(comments);
for ( id_doc_map::const_iterator it = identifier_docs.begin();
it != identifier_docs.end(); ++it )
{
IdentifierDocument* doc = it->second;
ID* id = doc->GetID();
if ( ! is_public_api(id) )
continue;
if ( id->AsType() )
{
types.push_back(doc);
DBG_LOG(DBG_BROXYGEN, "Filter id '%s' in '%s' as a type",
id->Name(), name.c_str());
continue;
}
if ( IsFunc(id->Type()->Tag()) )
{
switch ( id->Type()->AsFuncType()->Flavor() ) {
case FUNC_FLAVOR_HOOK:
DBG_LOG(DBG_BROXYGEN, "Filter id '%s' in '%s' as a hook",
id->Name(), name.c_str());
hooks.push_back(doc);
break;
case FUNC_FLAVOR_EVENT:
DBG_LOG(DBG_BROXYGEN, "Filter id '%s' in '%s' as a event",
id->Name(), name.c_str());
events.push_back(doc);
break;
case FUNC_FLAVOR_FUNCTION:
DBG_LOG(DBG_BROXYGEN, "Filter id '%s' in '%s' as a function",
id->Name(), name.c_str());
functions.push_back(doc);
break;
default:
reporter->InternalError("Invalid function flavor");
break;
}
continue;
}
if ( id->IsConst() )
{
if ( id->FindAttr(ATTR_REDEF) )
{
DBG_LOG(DBG_BROXYGEN, "Filter id '%s' in '%s' as an option",
id->Name(), name.c_str());
options.push_back(doc);
}
else
{
DBG_LOG(DBG_BROXYGEN, "Filter id '%s' in '%s' as a constant",
id->Name(), name.c_str());
constants.push_back(doc);
}
continue;
}
if ( id->Type()->Tag() == TYPE_ENUM )
// Enums are always referenced/documented from the type's
// documentation.
continue;
DBG_LOG(DBG_BROXYGEN, "Filter id '%s' in '%s' as a state variable",
id->Name(), name.c_str());
state_vars.push_back(doc);
}
}
vector<string> ScriptDocument::GetComments() const
{
return comments;
}
static size_t end_of_first_sentence(const string& s)
{
size_t rval = 0;
while ( (rval = s.find_first_of('.', rval)) != string::npos )
{
if ( rval == s.size() - 1 )
// Period is at end of string.
return rval;
if ( isspace(s[rval + 1]) )
// Period has a space after it.
return rval;
// Period has some non-space character after it, keep looking.
++rval;
}
return rval;
}
static bool is_all_whitespace(const string& s)
{
for ( size_t i = 0; i < s.size(); ++i )
if ( ! isspace(s[i]) )
return false;
return true;
}
static vector<string> summary_comment(const vector<string>& cmnts)
{
vector<string> rval;
for ( size_t i = 0; i < cmnts.size(); ++i )
{
size_t end = end_of_first_sentence(cmnts[i]);
if ( end == string::npos )
{
if ( is_all_whitespace(cmnts[i]) )
break;
rval.push_back(cmnts[i]);
}
else
{
rval.push_back(cmnts[i].substr(0, end + 1));
break;
}
}
return rval;
}
class ReStructuredTextTable {
public:
ReStructuredTextTable(size_t arg_num_cols)
: num_cols(arg_num_cols), rows(), longest_row_in_column()
{
for ( size_t i = 0; i < num_cols; ++i )
longest_row_in_column.push_back(1);
}
void AddRow(const vector<string>& new_row)
{
assert(new_row.size() == num_cols);
rows.push_back(new_row);
for ( size_t i = 0; i < new_row.size(); ++i )
if ( new_row[i].size() > longest_row_in_column[i] )
longest_row_in_column[i] = new_row[i].size();
}
static string MakeBorder(const vector<size_t> col_sizes, char border)
{
string rval;
for ( size_t i = 0; i < col_sizes.size(); ++i )
{
if ( i > 0 )
rval += " ";
rval += string(col_sizes[i], border);
}
rval += "\n";
return rval;
}
string AsString(char border) const
{
string rval = MakeBorder(longest_row_in_column, border);
for ( size_t row = 0; row < rows.size(); ++row )
{
for ( size_t col = 0; col < num_cols; ++col )
{
if ( col > 0 )
{
size_t last = rows[row][col - 1].size();
size_t longest = longest_row_in_column[col - 1];
size_t whitespace = longest - last + 1;
rval += string(whitespace, ' ');
}
rval += rows[row][col];
}
rval += "\n";
}
rval += MakeBorder(longest_row_in_column, border);
return rval;
}
private:
size_t num_cols;
vector<vector<string> > rows;
vector<size_t> longest_row_in_column;
};
static void add_summary_rows(const ODesc& id_desc, const vector<string>& cmnts,
ReStructuredTextTable* table)
{
vector<string> row;
row.push_back(id_desc.Description());
if ( cmnts.empty() )
{
row.push_back("");
table->AddRow(row);
return;
}
row.push_back(cmnts[0]);
table->AddRow(row);
for ( size_t i = 1; i < cmnts.size(); ++i )
{
row.clear();
row.push_back("");
row.push_back(cmnts[i]);
table->AddRow(row);
}
}
static string make_summary(const string& heading, char underline, char border,
const list<IdentifierDocument*>& id_list)
{
if ( id_list.empty() )
return "";
ReStructuredTextTable table(2);
for ( list<IdentifierDocument*>::const_iterator it = id_list.begin();
it != id_list.end(); ++it )
{
ID* id = (*it)->GetID();
ODesc d;
d.SetQuotes(1);
id->DescribeReSTShort(&d);
add_summary_rows(d, summary_comment((*it)->GetComments()), &table);
}
return make_heading(heading, underline) + table.AsString(border) + "\n";
}
static string make_redef_summary(const string& heading, char underline,
char border, const string& from_script,
const set<IdentifierDocument*>& id_set)
{
if ( id_set.empty() )
return "";
ReStructuredTextTable table(2);
for ( set<IdentifierDocument*>::const_iterator it = id_set.begin();
it != id_set.end(); ++it )
{
ID* id = (*it)->GetID();
ODesc d;
d.SetQuotes(1);
id->DescribeReSTShort(&d);
typedef list<IdentifierDocument::Redefinition> redef_list;
redef_list redefs = (*it)->GetRedefs(from_script);
for ( redef_list::const_iterator iit = redefs.begin();
iit != redefs.end(); ++iit )
add_summary_rows(d, summary_comment(iit->comments), &table);
}
return make_heading(heading, underline) + table.AsString(border) + "\n";
}
static string make_details(const string& heading, char underline,
const list<IdentifierDocument*>& id_list)
{
if ( id_list.empty() )
return "";
string rval = make_heading(heading, underline);
for ( list<IdentifierDocument*>::const_iterator it = id_list.begin();
it != id_list.end(); ++it )
{
rval += (*it)->ReStructuredText();
rval += "\n\n";
}
return rval;
}
static string make_redef_details(const string& heading, char underline,
const set<IdentifierDocument*>& id_set)
{
if ( id_set.empty() )
return "";
string rval = make_heading(heading, underline);
for ( set<IdentifierDocument*>::const_iterator it = id_set.begin();
it != id_set.end(); ++it )
{
rval += (*it)->ReStructuredText(true);
rval += "\n\n";
}
return rval;
}
string ScriptDocument::DoReStructuredText(bool roles_only) const
{
string rval;
rval += ":tocdepth: 3\n\n";
rval += make_heading(name, '=');
for ( string_set::const_iterator it = module_usages.begin();
it != module_usages.end(); ++it )
rval += ".. bro:namespace:: " + *it + "\n";
rval += "\n";
for ( size_t i = 0; i < comments.size(); ++i )
rval += comments[i] + "\n";
rval += "\n";
if ( ! module_usages.empty() )
{
rval += module_usages.size() > 1 ? ":Namespaces: " : ":Namespace: ";
for ( string_set::const_iterator it = module_usages.begin();
it != module_usages.end(); ++it )
{
if ( it != module_usages.begin() )
rval += ", ";
rval += *it;
}
rval += "\n";
}
if ( ! dependencies.empty() )
{
rval += ":Imports: ";
for ( string_set::const_iterator it = dependencies.begin();
it != dependencies.end(); ++it )
{
if ( it != dependencies.begin() )
rval += ", ";
rval += fmt(":doc:`%s </scripts/%s>`", it->c_str(), it->c_str());
// TODO linking to packages is a bit different?
}
rval += "\n";
}
// TODO: make this an absolute path?
rval += fmt(":Source File: :download:`%s`\n", name.c_str());
rval += "\n";
rval += make_heading("Summary", '~');
rval += make_summary("Options", '#', '=', options);
rval += make_summary("Constants", '#', '=', constants);
rval += make_summary("State Variables", '#', '=', state_vars);
rval += make_summary("Types", '#', '=', types);
rval += make_redef_summary("Redefinitions", '#', '=', name, redefs);
rval += make_summary("Events", '#', '=', events);
rval += make_summary("Hooks", '#', '=', hooks);
rval += make_summary("Functions", '#', '=', functions);
rval += "\n";
rval += make_heading("Detailed Interface", '~');
rval += make_details("Options", '#', options);
rval += make_details("Constants", '#', constants);
rval += make_details("State Variables", '#', state_vars);
rval += make_details("Types", '#', types);
//rval += make_redef_details("Redefinitions", '#', redefs);
rval += make_details("Events", '#', events);
rval += make_details("Hooks", '#', hooks);
rval += make_details("Functions", '#', functions);
return rval;
}
static time_t get_mtime(const string& filename)
{
struct stat s;
if ( stat(filename.c_str(), &s) < 0 )
reporter->InternalError("Broxygen failed to stat file '%s': %s",
filename.c_str(), strerror(errno));
return s.st_mtime;
}
time_t IdentifierDocument::DoGetModificationTime() const
{
// Could probably get away with just checking the set of scripts that
// contributed to the ID declaration/redefinitions, but this is easier...
return declaring_script->GetModificationTime();
}
time_t ScriptDocument::DoGetModificationTime() const
{
time_t most_recent = get_mtime(path);
for ( string_set::const_iterator it = dependencies.begin();
it != dependencies.end(); ++it )
{
Document* doc = broxygen_mgr->GetScriptDoc(*it);
if ( ! doc )
{
string pkg_name = *it + "/" + PACKAGE_LOADER;
doc = broxygen_mgr->GetScriptDoc(pkg_name);
if ( ! doc )
reporter->InternalWarning("Broxygen failed to get mtime of %s",
it->c_str());
continue;
}
time_t dep_mtime = doc->GetModificationTime();
if ( dep_mtime > most_recent )
most_recent = dep_mtime;
}
return most_recent;
}
time_t PackageDocument::DoGetModificationTime() const
{
string readme_file = find_file(pkg_name + "/README", bro_path());
if ( readme_file.empty() )
return 0;
return get_mtime(readme_file);
}

94
src/broxygen/Document.h
View file

@ -7,7 +7,7 @@
#include <vector>
#include <set>
#include <map>
#include <time.h>
#include <ctime>
#include "ID.h"
#include "Type.h"
@ -32,10 +32,19 @@ public:
std::string Name() const
{ return DoName(); }
std::string ReStructuredText(bool roles_only = false) const
{ return DoReStructuredText(roles_only); }
void InitPostScript()
{ return DoInitPostScript(); }
private:
virtual time_t DoGetModificationTime() const = 0;
virtual std::string DoName() const = 0;
virtual std::string DoReStructuredText(bool roles_only) const = 0;
virtual void DoInitPostScript()
{ }
};
class PackageDocument : public Document {
@ -44,28 +53,29 @@ public:
PackageDocument(const std::string& name);
// TODO: can be comments from package README
std::string GetReadme() const
{ return std::string(); }
std::vector<std::string> GetReadme() const
{ return readme; }
private:
// TODO
time_t DoGetModificationTime() const
{ return 0; }
time_t DoGetModificationTime() const;
std::string DoName() const
{ return pkg_name; }
std::string DoReStructuredText(bool roles_only) const;
std::string pkg_name;
std::vector<std::string> readme;
};
class IdentifierDocument : public Document {
class ScriptDocument;
class IdentifierDocument : public Document {
public:
IdentifierDocument(ID* id);
IdentifierDocument(ID* id, ScriptDocument* script);
~IdentifierDocument();
@ -88,24 +98,31 @@ public:
ID* GetID() const
{ return id; }
std::string GetComments() const;
ScriptDocument* GetDeclaringScript() const
{ return declaring_script; }
std::string GetFieldComments(const std::string& field) const;
std::string GetDeclaringScriptForField(const std::string& field) const;
std::vector<std::string> GetComments() const;
std::vector<std::string> GetFieldComments(const std::string& field) const;
struct Redefinition {
std::string from_script;
std::string new_val_desc;
std::vector<std::string> comments;
};
std::list<Redefinition> GetRedefs(const std::string& from_script) const;
private:
// TODO
time_t DoGetModificationTime() const
{ return 0; }
time_t DoGetModificationTime() const;
std::string DoName() const
{ return id->Name(); }
struct Redefinition {
std::string from_script;
string new_val_desc;
std::vector<std::string> comments;
};
std::string DoReStructuredText(bool roles_only) const;
struct RecordField {
~RecordField()
@ -121,17 +138,18 @@ private:
std::vector<std::string> comments;
ID* id;
string initial_val_desc;
std::string initial_val_desc;
redef_list redefs;
record_field_map fields;
RecordField* last_field_seen;
ScriptDocument* declaring_script;
};
class ScriptDocument : public Document {
public:
ScriptDocument(const std::string& name);
ScriptDocument(const std::string& name, const std::string& path);
void AddComment(const std::string& comment)
{ comments.push_back(comment); }
@ -145,32 +163,44 @@ public:
void AddIdentifierDoc(IdentifierDocument* doc);
void AddRedef(IdentifierDocument* doc)
{ redefs.push_back(doc); }
{ redefs.insert(doc); }
bool IsPkgLoader() const
{ return is_pkg_loader; }
std::string GetComments() const;
std::vector<std::string> GetComments() const;
private:
typedef std::map<std::string, IdentifierDocument*> IdentifierDocMap;
typedef std::list<IdentifierDocument*> IdentifierDocList;
typedef std::map<std::string, IdentifierDocument*> id_doc_map;
typedef std::list<IdentifierDocument*> id_doc_list;
typedef std::set<std::string> string_set;
typedef std::set<IdentifierDocument*> doc_set;
// TODO
time_t DoGetModificationTime() const
{ return 0; }
time_t DoGetModificationTime() const;
std::string DoName() const
{ return name; }
std::string DoReStructuredText(bool roles_only) const;
void DoInitPostScript() /* override */;
std::string name;
std::string path;
bool is_pkg_loader;
std::set<std::string> dependencies;
std::set<std::string> module_usages;
string_set dependencies;
string_set module_usages;
std::vector<std::string> comments;
IdentifierDocMap identifier_docs;
IdentifierDocList redefs;
id_doc_map identifier_docs;
id_doc_list options;
id_doc_list constants;
id_doc_list state_vars;
id_doc_list types;
id_doc_list events;
id_doc_list hooks;
id_doc_list functions;
doc_set redefs;
};

150
src/broxygen/Manager.cc
View file

@ -1,8 +1,8 @@
#include "Manager.h"
#include "Reporter.h"
#include "util.h"
#include <utility>
#include <cstdlib>
using namespace broxygen;
using namespace std;
@ -25,71 +25,22 @@ static string RemoveLeadingSpace(const string& s)
return rval;
}
static string PrettifyParams(const string& s)
{
size_t identifier_start_pos = 0;
bool in_identifier = false;
string identifier;
for ( size_t i = 0; i < s.size(); ++i )
{
char next = s[i];
if ( ! in_identifier )
{
// Pass by leading whitespace.
if ( isspace(next) )
continue;
// Only allow alphabetic and '_' as first char of identifier.
if ( isalpha(next) || next == '_' )
{
identifier_start_pos = i;
identifier += next;
in_identifier = true;
continue;
}
// Don't need to change anything.
return s;
}
// All other character of identifier are alphanumeric or '_'.
if ( isalnum(next) || next == '_' )
{
identifier += next;
continue;
}
// Prettify param and return value docs for a function's reST markup.
if ( next == ':' )
{
string rval = s;
string subst;
if ( identifier == "Returns" )
subst = "\n:returns";
else
subst = "\n:param " + identifier;
rval.replace(identifier_start_pos, identifier.size(), subst);
return rval;
}
// Don't need to change anything.
return s;
}
return s;
}
Manager::Manager(const string& arg_config)
Manager::Manager(const string& arg_config, const string& bro_command)
: disabled(), comment_buffer(), comment_buffer_map(), packages(), scripts(),
identifiers(), all_docs(), last_identifier_seen(), incomplete_type(),
config(arg_config)
enum_mappings(), config(arg_config), bro_mtime()
{
if ( getenv("BRO_DISABLE_BROXYGEN") )
disabled = true;
string path_to_bro = find_file(bro_command, getenv("PATH"));
struct stat s;
if ( path_to_bro.empty() || stat(path_to_bro.c_str(), &s) < 0 )
reporter->InternalError("Broxygen can't get mtime of bro binary %s: %s",
path_to_bro.c_str(), strerror(errno));
bro_mtime = s.st_mtime;
}
Manager::~Manager()
@ -102,7 +53,6 @@ void Manager::InitPreScript()
{
if ( disabled )
return;
// TODO: create file/proto analyzer doc
}
void Manager::InitPostScript()
@ -110,6 +60,9 @@ void Manager::InitPostScript()
if ( disabled )
return;
for ( size_t i = 0; i < all_docs.size(); ++i )
all_docs[i]->InitPostScript();
config.FindDependencies(all_docs);
}
@ -134,7 +87,7 @@ void Manager::File(const string& path)
return;
}
ScriptDocument* doc = new ScriptDocument(name);
ScriptDocument* doc = new ScriptDocument(name, path);
scripts.map[name] = doc;
all_docs.push_back(doc);
DBG_LOG(DBG_BROXYGEN, "Made ScriptDocument %s", name.c_str());
@ -209,7 +162,7 @@ void Manager::ModuleUsage(const string& path, const string& module)
IdentifierDocument* Manager::CreateIdentifierDoc(ID* id, ScriptDocument* script)
{
IdentifierDocument* rval = new IdentifierDocument(id);
IdentifierDocument* rval = new IdentifierDocument(id, script);
rval->AddComments(comment_buffer);
comment_buffer.clear();
@ -226,7 +179,10 @@ IdentifierDocument* Manager::CreateIdentifierDoc(ID* id, ScriptDocument* script)
all_docs.push_back(rval);
identifiers.map[id->Name()] = rval;
last_identifier_seen = rval;
if ( script )
script->AddIdentifierDoc(rval);
return rval;
}
@ -256,18 +212,9 @@ void Manager::StartType(ID* id)
id->Name(), script.c_str());
}
void Manager::StartRedef(ID* id)
static bool IsEnumType(ID* id)
{
if ( disabled )
return;
IdentifierDocument* id_doc = identifiers.GetDocument(id->Name());
if ( id_doc )
last_identifier_seen = id_doc;
else
DbgAndWarn(fmt("Broxygen redef tracking unknown identifier: %s",
id->Name()));
return id->AsType() ? id->AsType()->Tag() == TYPE_ENUM : false;
}
void Manager::Identifier(ID* id)
@ -275,7 +222,9 @@ void Manager::Identifier(ID* id)
if ( disabled )
return;
if ( incomplete_type && incomplete_type->Name() == id->Name() )
if ( incomplete_type )
{
if ( incomplete_type->Name() == id->Name() )
{
DBG_LOG(DBG_BROXYGEN, "Finished document for type %s", id->Name());
incomplete_type->CompletedTypeDecl();
@ -283,13 +232,8 @@ void Manager::Identifier(ID* id)
return;
}
if ( id->GetLocationInfo() == &no_location )
{
// Internally-created identifier (e.g. file/proto analyzer enum tags).
// Can be ignored here as they need to be documented via other means.
DBG_LOG(DBG_BROXYGEN, "Skip documenting identifier %s: no location",
id->Name());
return;
if ( IsEnumType(incomplete_type->GetID()) )
enum_mappings[id->Name()] = incomplete_type->GetID()->Name();
}
IdentifierDocument* id_doc = identifiers.GetDocument(id->Name());
@ -308,6 +252,16 @@ void Manager::Identifier(ID* id)
return;
}
if ( id->GetLocationInfo() == &no_location )
{
// Internally-created identifier (e.g. file/proto analyzer enum tags).
// Handled specially since they don't have a script location.
DBG_LOG(DBG_BROXYGEN, "Made internal IdentifierDocument %s",
id->Name());
CreateIdentifierDoc(id, 0);
return;
}
string script = without_bropath_component(id->GetLocationInfo()->filename);
ScriptDocument* script_doc = scripts.GetDocument(script);
@ -428,32 +382,8 @@ void Manager::PostComment(const string& comment, const string& id_hint)
comment_buffer_map[id_hint].push_back(RemoveLeadingSpace(comment));
}
StringVal* Manager::GetIdentifierComments(const string& name) const
string Manager::GetEnumTypeName(const string& id) const
{
IdentifierDocument* d = identifiers.GetDocument(name);
return new StringVal(d ? d->GetComments() : "");
}
StringVal* Manager::GetScriptComments(const string& name) const
{
ScriptDocument* d = scripts.GetDocument(name);
return new StringVal(d ? d->GetComments() : "");
}
StringVal* Manager::GetPackageReadme(const string& name) const
{
PackageDocument* d = packages.GetDocument(name);
return new StringVal(d ? d->GetReadme() : "");
}
StringVal* Manager::GetRecordFieldComments(const string& name) const
{
size_t i = name.find('$');
if ( i > name.size() - 2 )
// '$' is last char in string or not found.
return new StringVal("");
IdentifierDocument* d = identifiers.GetDocument(name.substr(0, i));
return new StringVal(d ? d->GetFieldComments(name.substr(i + 1)) : "");
map<string, string>::const_iterator it = enum_mappings.find(id);
return it == enum_mappings.end() ? "" : it->second;
}

54
src/broxygen/Manager.h
View file

@ -3,6 +3,8 @@
#include "Configuration.h"
#include "Document.h"
#include "Reporter.h"
#include "ID.h"
#include "Type.h"
#include "Val.h"
@ -10,6 +12,8 @@
#include <string>
#include <vector>
#include <map>
#include <ctime>
#include <sys/stat.h>
namespace broxygen {
@ -32,7 +36,7 @@ class Manager {
public:
Manager(const std::string& config);
Manager(const std::string& config, const std::string& bro_command);
~Manager();
@ -50,8 +54,6 @@ public:
void StartType(ID* id);
void StartRedef(ID* id);
void Identifier(ID* id);
void RecordField(const ID* id, const TypeDecl* field,
@ -66,13 +68,20 @@ public:
void PostComment(const std::string& comment,
const std::string& identifier_hint = "");
StringVal* GetIdentifierComments(const std::string& name) const;
std::string GetEnumTypeName(const std::string& id) const;
StringVal* GetScriptComments(const std::string& name) const;
IdentifierDocument* GetIdentifierDoc(const std::string& name) const
{ return identifiers.GetDocument(name); }
StringVal* GetPackageReadme(const std::string& name) const;
ScriptDocument* GetScriptDoc(const std::string& name) const
{ return scripts.GetDocument(name); }
StringVal* GetRecordFieldComments(const std::string& name) const;
PackageDocument* GetPackageDoc(const std::string& name) const
{ return packages.GetDocument(name); }
template <class T>
bool IsUpToDate(const std::string& target_file,
const std::vector<T*>& dependencies) const;
private:
@ -90,9 +99,40 @@ private:
std::vector<Document*> all_docs;
IdentifierDocument* last_identifier_seen;
IdentifierDocument* incomplete_type;
std::map<std::string, std::string> enum_mappings;
Config config;
time_t bro_mtime;
};
template <class T>
bool Manager::IsUpToDate(const string& target_file,
const vector<T*>& dependencies) const
{
struct stat s;
if ( stat(target_file.c_str(), &s) < 0 )
{
if ( errno == ENOENT )
// Doesn't exist.
return false;
reporter->InternalError("Broxygen failed to stat target file '%s': %s",
target_file.c_str(), strerror(errno));
}
if ( difftime(bro_mtime, s.st_mtime) > 0 )
return false;
if ( difftime(config.GetModificationTime(), s.st_mtime) > 0 )
return false;
for ( size_t i = 0; i < dependencies.size(); ++i )
if ( difftime(dependencies[i]->GetModificationTime(), s.st_mtime) > 0 )
return false;
return true;
}
} // namespace broxygen
extern broxygen::Manager* broxygen_mgr;

47
src/broxygen/broxygen.bif
View file

@ -2,26 +2,65 @@
%%{
#include "broxygen/Manager.h"
#include "util.h"
static StringVal* comments_to_val(const vector<string>& comments)
{
return new StringVal(implode_string_vector(comments));
}
%%}
# TODO: documentation
function get_identifier_comments%(name: string%): string
%{
return broxygen_mgr->GetIdentifierComments(name->CheckString());
using namespace broxygen;
IdentifierDocument* d = broxygen_mgr->GetIdentifierDoc(name->CheckString());
if ( ! d )
return new StringVal("");
return comments_to_val(d->GetComments());
%}
function get_script_comments%(name: string%): string
%{
return broxygen_mgr->GetScriptComments(name->CheckString());
using namespace broxygen;
ScriptDocument* d = broxygen_mgr->GetScriptDoc(name->CheckString());
if ( ! d )
return new StringVal("");
return comments_to_val(d->GetComments());
%}
function get_package_readme%(name: string%): string
%{
return broxygen_mgr->GetPackageReadme(name->CheckString());
using namespace broxygen;
PackageDocument* d = broxygen_mgr->GetPackageDoc(name->CheckString());
if ( ! d )
return new StringVal("");
return comments_to_val(d->GetReadme());
%}
function get_record_field_comments%(name: string%): string
%{
return broxygen_mgr->GetRecordFieldComments(name->CheckString());
using namespace broxygen;
string accessor = name->CheckString();
size_t i = accessor.find('$');
if ( i > accessor.size() - 2 )
return new StringVal("");
string id = accessor.substr(0, i);
IdentifierDocument* d = broxygen_mgr->GetIdentifierDoc(id);
if ( ! d )
return new StringVal("");
string field = accessor.substr(i + 1);
return comments_to_val(d->GetFieldComments(field));
%}

3
src/main.cc
View file

@ -762,7 +762,6 @@ int main(int argc, char** argv)
reporter = new Reporter();
thread_mgr = new threading::Manager();
broxygen_mgr = new broxygen::Manager(broxygen_config);
#ifdef DEBUG
if ( debug_streams )
@ -809,6 +808,8 @@ int main(int argc, char** argv)
timer_mgr = new PQ_TimerMgr("<GLOBAL>");
// timer_mgr = new CQ_TimerMgr();
broxygen_mgr = new broxygen::Manager(broxygen_config, bro_argv[0]);
add_input_file("base/init-bare.bro");
if ( ! bare_mode )
add_input_file("base/init-default.bro");

7
src/parse.y
View file

@ -1022,14 +1022,15 @@ decl:
broxygen_mgr->Redef($2, ::filename);
}
| TOK_REDEF TOK_ENUM global_id TOK_ADD_TO
'{' { parser_redef_enum($3); } enum_body '}' ';'
| TOK_REDEF TOK_ENUM global_id TOK_ADD_TO '{'
{ parser_redef_enum($3); broxygen_mgr->Redef($3, ::filename); }
enum_body '}' ';'
{
// Broxygen already grabbed new enum IDs as the type created them.
}
| TOK_REDEF TOK_RECORD global_id
{ cur_decl_type_id = $3; broxygen_mgr->StartRedef($3); }
{ cur_decl_type_id = $3; broxygen_mgr->Redef($3, ::filename); }
TOK_ADD_TO '{'
{ ++in_record; }
type_decl_list

2
src/plugin/ComponentManager.h
View file

@ -10,6 +10,7 @@
#include "Var.h"
#include "Val.h"
#include "Reporter.h"
#include "broxygen/Manager.h"
namespace plugin {
@ -133,6 +134,7 @@ ComponentManager<T, C>::ComponentManager(const string& arg_module)
tag_enum_type = new EnumType();
::ID* id = install_ID("Tag", module.c_str(), true, true);
add_type(id, tag_enum_type, 0);
broxygen_mgr->Identifier(id);
}
template <class T, class C>

1
src/scan.l
View file

@ -145,6 +145,7 @@ ESCSEQ (\\([^\n]|[0-7]+|x[[:xdigit:]]+))
}
##.* {
if ( yytext[2] != '#' )
broxygen_mgr->PreComment(yytext + 2);
}

43
src/util.cc
View file

@ -590,6 +590,33 @@ const char* fmt_access_time(double t)
return buf;
}
bool ensure_intermediate_dirs(const char* dirname)
{
if ( ! dirname || strlen(dirname) == 0 )
return false;
bool absolute = dirname[0] == '/';
string path = normalize_path(dirname);
vector<string> path_components;
tokenize_string(path, "/", &path_components);
string current_dir;
for ( size_t i = 0; i < path_components.size(); ++i )
{
if ( i > 0 || absolute )
current_dir += "/";
current_dir += path_components[i];
if ( ! ensure_dir(current_dir.c_str()) )
return false;
}
return true;
}
bool ensure_dir(const char *dirname)
{
struct stat st;
@ -976,6 +1003,22 @@ void SafePathOp::DoFunc(PathOpFn fn, const string& path, bool error_aborts)
delete [] tmp;
}
string implode_string_vector(const std::vector<std::string>& v,
const std::string& delim)
{
string rval;
for ( size_t i = 0; i < v.size(); ++i )
{
if ( i > 0 )
rval += delim;
rval += v[i];
}
return rval;
}
string flatten_script_name(const string& name, const string& prefix)
{
string rval = prefix;

4
src/util.h
View file

@ -150,6 +150,7 @@ extern const char* fmt(const char* format, ...)
myattribute((format (printf, 1, 2)));
extern const char* fmt_access_time(double time);
extern bool ensure_intermediate_dirs(const char* dirname);
extern bool ensure_dir(const char *dirname);
// Returns true if path exists and is a directory.
@ -251,6 +252,9 @@ public:
: SafePathOp(&basename, path, error_aborts) { }
};
std::string implode_string_vector(const std::vector<std::string>& v,
const std::string& delim = "\n");
/**
* Flatten a script name by replacing '/' path separators with '.'.
* @param file A path to a Bro script. If it is a __load__.bro, that part