diff --git a/src/BroDoc.cc b/src/BroDoc.cc index ecdab83103..a7b7580fa6 100644 --- a/src/BroDoc.cc +++ b/src/BroDoc.cc @@ -12,6 +12,7 @@ #include "analyzer/Manager.h" #include "analyzer/Component.h" #include "file_analysis/Manager.h" +#include "broxygen/Manager.h" BroDoc::BroDoc(const std::string& rel, const std::string& abs) { @@ -565,33 +566,28 @@ static void WritePluginBifItems(FILE* f, const plugin::Plugin* p, for ( it = bifitems.begin(); it != bifitems.end(); ++it ) { - BroDocObj* o = doc_ids[it->GetID()]; + broxygen::IdentifierDocument* doc = broxygen_mgr->GetIdentifierDoc( + it->GetID()); - if ( o ) - o->WriteReST(f); + if ( doc ) + fprintf(f, "%s\n\n", doc->ReStructuredText().c_str()); else - reporter->Warning("No docs for ID: %s\n", it->GetID()); + reporter->InternalWarning("Broxygen ID lookup failed: %s\n", + it->GetID()); } } -static void WriteAnalyzerTagDefn(FILE* f, EnumType* e, const string& module) +static void WriteAnalyzerTagDefn(FILE* f, const string& module) { - /* TODO - string tag_id= module + "::Tag"; - e = new CommentedEnumType(e); - e->SetTypeID(copy_string(tag_id.c_str())); + string tag_id = module + "::Tag"; - ID* dummy_id = new ID(tag_id.c_str(), SCOPE_GLOBAL, true); - dummy_id->SetType(e); - dummy_id->MakeType(); + broxygen::IdentifierDocument* doc = broxygen_mgr->GetIdentifierDoc(tag_id); - list* r = new list(); - r->push_back("Unique identifiers for analyzers."); + if ( ! doc ) + reporter->InternalError("Broxygen failed analyzer tag lookup: %s", + tag_id.c_str()); - BroDocObj bdo(dummy_id, r, true); - - bdo.WriteReST(f); - */ + fprintf(f, "%s\n", doc->ReStructuredText().c_str()); } static bool ComponentsMatch(const plugin::Plugin* p, plugin::component::Type t, @@ -610,16 +606,14 @@ static bool ComponentsMatch(const plugin::Plugin* p, plugin::component::Type t, return true; } -void CreateProtoAnalyzerDoc(const char* filename) +void CreateProtoAnalyzerDoc(FILE* f) { - FILE* f = fopen(filename, "w"); - fprintf(f, "Protocol Analyzers\n"); - fprintf(f, "==================\n\n\n"); + fprintf(f, "==================\n\n"); fprintf(f, ".. contents::\n"); fprintf(f, " :depth: 1\n\n"); - WriteAnalyzerTagDefn(f, analyzer_mgr->GetTagEnumType(), "Analyzer"); + WriteAnalyzerTagDefn(f, "Analyzer"); plugin::Manager::plugin_list plugins = plugin_mgr->Plugins(); plugin::Manager::plugin_list::const_iterator it; @@ -642,16 +636,14 @@ void CreateProtoAnalyzerDoc(const char* filename) fclose(f); } -void CreateFileAnalyzerDoc(const char* filename) +void CreateFileAnalyzerDoc(FILE* f) { - FILE* f = fopen(filename, "w"); - fprintf(f, "File Analyzers\n"); fprintf(f, "==============\n\n"); fprintf(f, ".. contents::\n"); fprintf(f, " :depth: 1\n\n"); - WriteAnalyzerTagDefn(f, file_mgr->GetTagEnumType(), "Files"); + WriteAnalyzerTagDefn(f, "Files"); plugin::Manager::plugin_list plugins = plugin_mgr->Plugins(); plugin::Manager::plugin_list::const_iterator it; diff --git a/src/BroDoc.h b/src/BroDoc.h index 081df698d9..926713d01e 100644 --- a/src/BroDoc.h +++ b/src/BroDoc.h @@ -409,14 +409,14 @@ private: /** * Writes out plugin index documentation for all analyzer plugins. - * @param filename the name of the file to write. + * @param f an open file stream to write docs into. */ -void CreateProtoAnalyzerDoc(const char* filename); +void CreateProtoAnalyzerDoc(FILE* f); /** * Writes out plugin index documentation for all file analyzer plugins. - * @param filename the name of the file to write. + * @param f an open file stream to write docs into. */ -void CreateFileAnalyzerDoc(const char* filename); +void CreateFileAnalyzerDoc(FILE* f); #endif diff --git a/src/Desc.h b/src/Desc.h index c16c00cf13..27dc326ff0 100644 --- a/src/Desc.h +++ b/src/Desc.h @@ -69,6 +69,7 @@ public: void PopIndent(); void PopIndentNoNL(); int GetIndentLevel() const { return indent_level; } + void ClearIndentLevel() { indent_level = 0; } int IndentSpaces() const { return indent_with_spaces; } void SetIndentSpaces(int i) { indent_with_spaces = i; } diff --git a/src/ID.cc b/src/ID.cc index 8d30d10c27..aa965b880e 100644 --- a/src/ID.cc +++ b/src/ID.cc @@ -14,6 +14,7 @@ #include "PersistenceSerializer.h" #include "Scope.h" #include "Traverse.h" +#include "broxygen/Manager.h" ID::ID(const char* arg_name, IDScope arg_scope, bool arg_is_export) { @@ -618,7 +619,6 @@ void ID::DescribeExtended(ODesc* d) const void ID::DescribeReSTShort(ODesc* d) const { - /* TODO if ( is_type ) d->Add(":bro:type:`"); else @@ -632,8 +632,8 @@ void ID::DescribeReSTShort(ODesc* d) const d->Add(": "); d->Add(":bro:type:`"); - if ( ! is_type && type->GetTypeID() ) - d->Add(type->GetTypeID()); + if ( ! is_type && ! type->GetName().empty() ) + d->Add(type->GetName().c_str()); else { TypeTag t = type->Tag(); @@ -644,14 +644,14 @@ void ID::DescribeReSTShort(ODesc* d) const break; case TYPE_FUNC: - d->Add(type->AsFuncType()->FlavorString()); + d->Add(type->AsFuncType()->FlavorString().c_str()); break; case TYPE_ENUM: if ( is_type ) d->Add(type_name(t)); else - d->Add(type->AsEnumType()->Name().c_str()); + d->Add(broxygen_mgr->GetEnumTypeName(Name()).c_str()); break; default: @@ -668,12 +668,11 @@ void ID::DescribeReSTShort(ODesc* d) const d->SP(); attrs->DescribeReST(d); } - */ } -void ID::DescribeReST(ODesc* d, bool is_role) const +void ID::DescribeReST(ODesc* d, bool roles_only) const { - if ( is_role ) + if ( roles_only ) { if ( is_type ) d->Add(":bro:type:`"); @@ -705,7 +704,7 @@ void ID::DescribeReST(ODesc* d, bool is_role) const d->Add("`"); } else - type->DescribeReST(d); + type->DescribeReST(d, roles_only); d->NL(); } diff --git a/src/ID.h b/src/ID.h index 57e1222511..59b397fa90 100644 --- a/src/ID.h +++ b/src/ID.h @@ -84,7 +84,7 @@ public: // Adds type and value to description. void DescribeExtended(ODesc* d) const; // Produces a description that's reST-ready. - void DescribeReST(ODesc* d, bool is_role=false) const; + void DescribeReST(ODesc* d, bool roles_only = false) const; void DescribeReSTShort(ODesc* d) const; bool Serialize(SerialInfo* info) const; diff --git a/src/Type.cc b/src/Type.cc index 417b73fd2f..83f239bbfe 100644 --- a/src/Type.cc +++ b/src/Type.cc @@ -117,16 +117,17 @@ BroType* BroType::Clone() const sinfo.cache = false; this->Serialize(&sinfo); - char* data; + char* data = 0; uint32 len = form->EndWrite(&data); form->StartRead(data, len); UnserialInfo uinfo(&ss); uinfo.cache = false; - BroType* rval = this->Unserialize(&uinfo); + + BroType* rval = this->Unserialize(&uinfo, false); + assert(rval != this); delete [] data; - return rval; } @@ -173,11 +174,9 @@ void BroType::Describe(ODesc* d) const } } -void BroType::DescribeReST(ODesc* d) const +void BroType::DescribeReST(ODesc* d, bool roles_only) const { - d->Add(":bro:type:`"); - d->Add(type_name(Tag())); - d->Add("`"); + d->Add(fmt(":bro:type:`%s`", type_name(Tag()))); } void BroType::SetError() @@ -200,7 +199,7 @@ bool BroType::Serialize(SerialInfo* info) const return ret; } -BroType* BroType::Unserialize(UnserialInfo* info, TypeTag want) +BroType* BroType::Unserialize(UnserialInfo* info, bool use_existing) { // To avoid external Broccoli clients needing to always send full type // objects, we allow them to give us only the name of a type. To @@ -234,8 +233,8 @@ BroType* BroType::Unserialize(UnserialInfo* info, TypeTag want) BroType* t = (BroType*) SerialObj::Unserialize(info, SER_BRO_TYPE); - if ( ! t ) - return 0; + if ( ! t || ! use_existing ) + return t; if ( ! t->name.empty() ) { @@ -458,7 +457,7 @@ void IndexType::Describe(ODesc* d) const } } -void IndexType::DescribeReST(ODesc* d) const +void IndexType::DescribeReST(ODesc* d, bool roles_only) const { d->Add(":bro:type:`"); @@ -484,7 +483,7 @@ void IndexType::DescribeReST(ODesc* d) const d->Add("`"); } else - t->DescribeReST(d); + t->DescribeReST(d, roles_only); } d->Add("]"); @@ -500,7 +499,7 @@ void IndexType::DescribeReST(ODesc* d) const d->Add("`"); } else - yield_type->DescribeReST(d); + yield_type->DescribeReST(d, roles_only); } } @@ -775,7 +774,7 @@ void FuncType::Describe(ODesc* d) const } } -void FuncType::DescribeReST(ODesc* d) const +void FuncType::DescribeReST(ODesc* d, bool roles_only) const { d->Add(":bro:type:`"); d->Add(FlavorString()); @@ -795,7 +794,7 @@ void FuncType::DescribeReST(ODesc* d) const d->Add("`"); } else - yield->DescribeReST(d); + yield->DescribeReST(d, roles_only); } } @@ -927,7 +926,7 @@ TypeDecl* TypeDecl::Unserialize(UnserialInfo* info) return t; } -void TypeDecl::DescribeReST(ODesc* d) const +void TypeDecl::DescribeReST(ODesc* d, bool roles_only) const { d->Add(id); d->Add(": "); @@ -939,7 +938,7 @@ void TypeDecl::DescribeReST(ODesc* d) const d->Add("`"); } else - type->DescribeReST(d); + type->DescribeReST(d, roles_only); if ( attrs ) { @@ -1037,9 +1036,13 @@ void RecordType::Describe(ODesc* d) const } } -void RecordType::DescribeReST(ODesc* d) const +void RecordType::DescribeReST(ODesc* d, bool roles_only) const { d->Add(":bro:type:`record`"); + + if ( num_fields == 0 ) + return; + d->NL(); DescribeFieldsReST(d, false); } @@ -1136,7 +1139,53 @@ void RecordType::DescribeFieldsReST(ODesc* d, bool func_args) const } } - FieldDecl(i)->DescribeReST(d); + const TypeDecl* td = FieldDecl(i); + td->DescribeReST(d); + + if ( func_args ) + continue; + + using broxygen::IdentifierDocument; + IdentifierDocument* doc = broxygen_mgr->GetIdentifierDoc(GetName()); + + if ( ! doc ) + { + reporter->InternalWarning("Failed to lookup record doc: %s", + GetName().c_str()); + continue; + } + + string field_from_script = doc->GetDeclaringScriptForField(td->id); + string type_from_script; + + if ( doc->GetDeclaringScript() ) + type_from_script = doc->GetDeclaringScript()->Name(); + + if ( ! field_from_script.empty() && + field_from_script != type_from_script ) + { + d->PushIndent(); + d->Add(fmt("(from ``redef`` in :doc:`%s`)", + field_from_script.c_str())); + d->PopIndentNoNL(); + } + + vector cmnts = doc->GetFieldComments(td->id); + + if ( cmnts.empty() ) + continue; + + d->PushIndent(); + + for ( size_t i = 0; i < cmnts.size(); ++i ) + { + if ( i > 0 ) + d->NL(); + + d->Add(cmnts[i].c_str()); + } + + d->PopIndentNoNL(); } if ( ! func_args ) @@ -1415,10 +1464,82 @@ const char* EnumType::Lookup(bro_int_t value) return 0; } -void EnumType::DescribeReST(ODesc* d) const +void EnumType::DescribeReST(ODesc* d, bool roles_only) const { - // TODO: this probably goes away d->Add(":bro:type:`enum`"); + + // Create temporary, reverse name map so that enums can be documented + // in ascending order of their actual integral value instead of by name. + typedef map< bro_int_t, const char* > RevNameMap; + + RevNameMap rev; + + for ( NameMap::const_iterator it = names.begin(); it != names.end(); ++it ) + rev[it->second] = it->first; + + for ( RevNameMap::const_iterator it = rev.begin(); it != rev.end(); ++it ) + { + d->NL(); + d->PushIndent(); + + if ( roles_only ) + d->Add(fmt(":bro:enum:`%s`", it->second)); + else + d->Add(fmt(".. bro:enum:: %s %s", it->second, GetName().c_str())); + + using broxygen::IdentifierDocument; + IdentifierDocument* doc = broxygen_mgr->GetIdentifierDoc(it->second); + + if ( ! doc ) + { + reporter->InternalWarning("Enum %s documentation lookup failure", + it->second); + continue; + } + + string enum_from_script; + string type_from_script; + + if ( doc->GetDeclaringScript() ) + enum_from_script = doc->GetDeclaringScript()->Name(); + + IdentifierDocument* type_doc = broxygen_mgr->GetIdentifierDoc(GetName()); + + if ( type_doc && type_doc->GetDeclaringScript() ) + type_from_script = type_doc->GetDeclaringScript()->Name(); + + if ( ! enum_from_script.empty() && + enum_from_script != type_from_script ) + { + d->NL(); + d->PushIndent(); + d->Add(fmt("(from ``redef`` in :doc:`%s`)", + enum_from_script.c_str())); + d->PopIndentNoNL(); + } + + vector cmnts = doc->GetComments(); + + if ( cmnts.empty() ) + { + d->PopIndentNoNL(); + continue; + } + + d->NL(); + d->PushIndent(); + + for ( size_t i = 0; i < cmnts.size(); ++i ) + { + if ( i > 0 ) + d->NL(); + + d->Add(cmnts[i].c_str()); + } + + d->PopIndentNoNL(); + d->PopIndentNoNL(); + } } IMPLEMENT_SERIAL(EnumType, SER_ENUM_TYPE); diff --git a/src/Type.h b/src/Type.h index be27426546..3e774f373a 100644 --- a/src/Type.h +++ b/src/Type.h @@ -227,12 +227,12 @@ public: BroType* Ref() { ::Ref(this); return this; } virtual void Describe(ODesc* d) const; - virtual void DescribeReST(ODesc* d) const; + virtual void DescribeReST(ODesc* d, bool roles_only = false) const; virtual unsigned MemoryAllocation() const; bool Serialize(SerialInfo* info) const; - static BroType* Unserialize(UnserialInfo* info, TypeTag want = TYPE_ANY); + static BroType* Unserialize(UnserialInfo* info, bool use_existing = true); void SetName(const string& arg_name) { name = arg_name; } string GetName() const { return name; } @@ -305,7 +305,7 @@ public: BroType* YieldType(); void Describe(ODesc* d) const; - void DescribeReST(ODesc* d) const; + void DescribeReST(ODesc* d, bool roles_only = false) const; // Returns true if this table is solely indexed by subnet. bool IsSubNetIndex() const; @@ -379,7 +379,7 @@ public: TypeList* ArgTypes() const { return arg_types; } void Describe(ODesc* d) const; - void DescribeReST(ODesc* d) const; + void DescribeReST(ODesc* d, bool roles_only = false) const; protected: FuncType() { args = 0; arg_types = 0; yield = 0; flavor = FUNC_FLAVOR_FUNCTION; } @@ -416,7 +416,7 @@ public: bool Serialize(SerialInfo* info) const; static TypeDecl* Unserialize(UnserialInfo* info); - virtual void DescribeReST(ODesc* d) const; + virtual void DescribeReST(ODesc* d, bool roles_only = false) const; BroType* type; Attributes* attrs; @@ -454,7 +454,7 @@ public: const char* AddFields(type_decl_list* types, attr_list* attr); void Describe(ODesc* d) const; - void DescribeReST(ODesc* d) const; + void DescribeReST(ODesc* d, bool roles_only = false) const; void DescribeFields(ODesc* d) const; void DescribeFieldsReST(ODesc* d, bool func_args) const; @@ -527,7 +527,7 @@ public: bro_int_t Lookup(const string& module_name, const char* name); const char* Lookup(bro_int_t value); // Returns 0 if not found - void DescribeReST(ODesc* d) const; + void DescribeReST(ODesc* d, bool roles_only = false) const; protected: DECLARE_SERIAL(EnumType) diff --git a/src/Var.cc b/src/Var.cc index 821c9e207b..eb03f2f912 100644 --- a/src/Var.cc +++ b/src/Var.cc @@ -261,17 +261,17 @@ extern Expr* add_and_assign_local(ID* id, Expr* init, Val* val) void add_type(ID* id, BroType* t, attr_list* attr) { - string new_type_name(id->Name()); - string old_type_name(t->GetName()); + string new_type_name = id->Name(); + string old_type_name = t->GetName(); BroType* tnew = 0; if ( (t->Tag() == TYPE_RECORD || t->Tag() == TYPE_ENUM) && - ! old_type_name.empty() ) + old_type_name.empty() ) + // An extensible type (record/enum) being declared for first time. + tnew = t; + else // Clone the type to preserve type name aliasing. tnew = t->Clone(); - else - // An extensible types (record/enum) being declared for first time. - tnew = t; type_aliases[new_type_name].insert(tnew); diff --git a/src/broxygen/Configuration.cc b/src/broxygen/Configuration.cc index cd8ee7be56..d88aa1920c 100644 --- a/src/broxygen/Configuration.cc +++ b/src/broxygen/Configuration.cc @@ -1,4 +1,5 @@ #include "Configuration.h" +#include "Manager.h" #include "util.h" #include "Reporter.h" @@ -7,6 +8,8 @@ #include #include #include +#include +#include using namespace broxygen; using namespace std; @@ -16,12 +19,12 @@ typedef map target_factory_map; static target_factory_map create_target_factory_map() { target_factory_map rval; - rval["package_index"] = &PackageTarget::Instantiate; + rval["package_index"] = &PackageIndexTarget::Instantiate; rval["package"] = &PackageTarget::Instantiate; rval["proto_analyzer"] = &ProtoAnalyzerTarget::Instantiate; rval["file_analyzer"] = &FileAnalyzerTarget::Instantiate; - rval["script_summary"] = &ScriptTarget::Instantiate; - rval["script_index"] = &ScriptTarget::Instantiate; + rval["script_summary"] = &ScriptSummaryTarget::Instantiate; + rval["script_index"] = &ScriptIndexTarget::Instantiate; rval["script"] = &ScriptTarget::Instantiate; rval["identifier"] = &IdentifierTarget::Instantiate; return rval; @@ -29,41 +32,300 @@ static target_factory_map create_target_factory_map() static target_factory_map target_instantiators = create_target_factory_map(); -bool Target::MatchesPattern(Document* doc) const - { - // TODO: prefix matching or full regex? - - if ( doc->Name() == pattern ) +struct TargetFile { + TargetFile(const string& arg_name) + : name(arg_name), f() { - DBG_LOG(DBG_BROXYGEN, "Doc '%s'' matched pattern for target '%s'", - doc->Name().c_str(), name.c_str()); - return true; + if ( name.find('/') != string::npos ) + { + string dir = SafeDirname(name).result; + + if ( ! ensure_intermediate_dirs(dir.c_str()) ) + reporter->FatalError("Broxygen failed to make dir %s", + dir.c_str()); + } + + f = fopen(name.c_str(), "w"); + + if ( ! f ) + reporter->FatalError("Broxygen failed to open '%s' for writing: %s", + name.c_str(), strerror(errno)); } - return false; - } + ~TargetFile() + { + if ( f ) + fclose(f); + + DBG_LOG(DBG_BROXYGEN, "Wrote out-of-date target '%s'", name.c_str()); + } + + string name; + FILE* f; +}; template -void filter_matching_docs(const std::vector& filter_from, Target* t) +static vector filter_matching_docs(const vector& from, Target* t) { - for ( size_t i = 0; i < filter_from.size(); ++i ) + vector rval; + + for ( size_t i = 0; i < from.size(); ++i ) { - T* d = dynamic_cast(filter_from[i]); + T* d = dynamic_cast(from[i]); if ( ! d ) continue; if ( t->MatchesPattern(d) ) - t->AddDependency(d); + { + DBG_LOG(DBG_BROXYGEN, "Doc '%s' matched pattern for target '%s'", + d->Name().c_str(), t->Name().c_str()); + rval.push_back(d); + } + } + + return rval; + } + +Target::Target(const string& arg_name, const string& arg_pattern) + : name(arg_name), pattern(arg_pattern), prefix() + { + size_t pos = pattern.find('*'); + + if ( pos == 0 || pos == string::npos ) + return; + + prefix = pattern.substr(0, pos); + } + +bool Target::MatchesPattern(Document* doc) const + { + if ( pattern == "*" ) + return true; + + if ( prefix.empty() ) + return doc->Name() == pattern; + + return ! strncmp(doc->Name().c_str(), prefix.c_str(), prefix.size()); + } + +void AnalyzerTarget::DoFindDependencies(const std::vector& docs) + { + // TODO: really should add to dependency list the tag type's ID and + // all bif items for matching analyzer plugins, but that's all dependent + // on the bro binary itself, so I'm cheating. + } + +void AnalyzerTarget::DoGenerate() const + { + if ( broxygen_mgr->IsUpToDate(Name(), vector()) ) + return; + + if ( Pattern() != "*" ) + reporter->InternalWarning("Broxygen only implements analyzer target" + " pattern '*'"); + + TargetFile file(Name()); + doc_creator_callback(file.f); + } + +void PackageTarget::DoFindDependencies(const vector& docs) + { + pkg_deps = filter_matching_docs(docs, this); + + if ( pkg_deps.empty() ) + reporter->FatalError("No match for Broxygen target '%s' pattern '%s'", + Name().c_str(), Pattern().c_str()); + + for ( size_t i = 0; i < docs.size(); ++i ) + { + ScriptDocument* script = dynamic_cast(docs[i]); + + if ( ! script ) + continue; + + for ( size_t j = 0; j < pkg_deps.size(); ++j ) + { + if ( strncmp(script->Name().c_str(), pkg_deps[j]->Name().c_str(), + pkg_deps[j]->Name().size())) + continue; + + DBG_LOG(DBG_BROXYGEN, "Script %s associated with package %s", + script->Name().c_str(), pkg_deps[j]->Name().c_str()); + pkg_manifest[pkg_deps[j]].push_back(script); + script_deps.push_back(script); + } } } -void IdentifierTarget::DoFindDependencies(const std::vector& docs) +void PackageTarget::DoGenerate() const { - filter_matching_docs(docs, this); + if ( broxygen_mgr->IsUpToDate(Name(), script_deps) ) + return; + + TargetFile file(Name()); + + for ( manifest_t::const_iterator it = pkg_manifest.begin(); + it != pkg_manifest.end(); ++it ) + { + fprintf(file.f, "Package: %s\n\n", it->first->Name().c_str()); + + for ( size_t i = 0; i < it->second.size(); ++i ) + { + fprintf(file.f, " :doc:`%s`\n", it->second[i]->Name().c_str()); + + vector cmnts = it->second[i]->GetComments(); + + for ( size_t j = 0; j < cmnts.size(); ++j ) + fprintf(file.f, " %s\n", cmnts[j].c_str()); + + fprintf(file.f, "\n"); + } + } } -Config::Config(const string& file, const string& delim) +void PackageIndexTarget::DoFindDependencies(const vector& docs) + { + pkg_deps = filter_matching_docs(docs, this); + + if ( pkg_deps.empty() ) + reporter->FatalError("No match for Broxygen target '%s' pattern '%s'", + Name().c_str(), Pattern().c_str()); + } + +void PackageIndexTarget::DoGenerate() const + { + if ( broxygen_mgr->IsUpToDate(Name(), pkg_deps) ) + return; + + TargetFile file(Name()); + + for ( size_t i = 0; i < pkg_deps.size(); ++i ) + fprintf(file.f, "%s\n", pkg_deps[i]->ReStructuredText().c_str()); + } + +void ScriptTarget::DoFindDependencies(const vector& docs) + { + script_deps = filter_matching_docs(docs, this); + + if ( script_deps.empty() ) + reporter->FatalError("No match for Broxygen target '%s' pattern '%s'", + Name().c_str(), Pattern().c_str()); + } + +void ScriptTarget::DoGenerate() const + { + if ( Name()[Name().size() - 1] == '/' ) + { + // Target name is a dir, matching scripts are written within that dir + // with a dir tree that parallels the script's BROPATH location. + + for ( size_t i = 0; i < script_deps.size(); ++i ) + { + string target_filename = Name() + script_deps[i]->Name(); + size_t pos = target_filename.rfind(".bro"); + + if ( pos == target_filename.size() - 4 ) + target_filename.replace(pos, 4, ".rst"); + else + target_filename += ".rst"; + + vector dep; + dep.push_back(script_deps[i]); + + if ( broxygen_mgr->IsUpToDate(target_filename, dep) ) + continue; + + TargetFile file(target_filename); + + fprintf(file.f, "%s\n", script_deps[i]->ReStructuredText().c_str()); + } + + // TODO: could possibly take inventory of files in the dir beforehand, + // track all files written, then compare afterwards in order to remove + // stale files. + return; + } + + // Target is a single file, all matching scripts get written there. + + if ( broxygen_mgr->IsUpToDate(Name(), script_deps) ) + return; + + TargetFile file(Name()); + + for ( size_t i = 0; i < script_deps.size(); ++i ) + fprintf(file.f, "%s\n", script_deps[i]->ReStructuredText().c_str()); + } + +void ScriptSummaryTarget::DoGenerate() const + { + if ( broxygen_mgr->IsUpToDate(Name(), script_deps) ) + return; + + TargetFile file(Name()); + + for ( size_t i = 0; i < script_deps.size(); ++i ) + { + ScriptDocument* d = dynamic_cast(script_deps[i]); + + if ( ! d ) + continue; + + fprintf(file.f, ":doc:`/scripts/%s`\n", d->Name().c_str()); + + vector cmnts = d->GetComments(); + + for ( size_t i = 0; i < cmnts.size(); ++i ) + fprintf(file.f, " %s\n", cmnts[i].c_str()); + + fprintf(file.f, "\n"); + } + } + +void ScriptIndexTarget::DoGenerate() const + { + if ( broxygen_mgr->IsUpToDate(Name(), script_deps) ) + return; + + TargetFile file(Name()); + + fprintf(file.f, ".. toctree::\n"); + fprintf(file.f, " :maxdepth: 1\n\n"); + + for ( size_t i = 0; i < script_deps.size(); ++i ) + { + ScriptDocument* d = dynamic_cast(script_deps[i]); + + if ( ! d ) + continue; + + fprintf(file.f, " %s <%s>\n", d->Name().c_str(), d->Name().c_str()); + } + } + +void IdentifierTarget::DoFindDependencies(const vector& docs) + { + id_deps = filter_matching_docs(docs, this); + + if ( id_deps.empty() ) + reporter->FatalError("No match for Broxygen target '%s' pattern '%s'", + Name().c_str(), Pattern().c_str()); + } + +void IdentifierTarget::DoGenerate() const + { + if ( broxygen_mgr->IsUpToDate(Name(), id_deps) ) + return; + + TargetFile file(Name()); + + for ( size_t i = 0; i < id_deps.size(); ++i ) + fprintf(file.f, "%s\n\n", id_deps[i]->ReStructuredText().c_str()); + } + +Config::Config(const string& arg_file, const string& delim) + : file(arg_file), targets() { if ( file.empty() ) return; @@ -71,53 +333,71 @@ Config::Config(const string& file, const string& delim) ifstream f(file.c_str()); if ( ! f.is_open() ) - reporter->InternalError("failed to open Broxygen config file %s: %s", - file.c_str(), strerror(errno)); + reporter->FatalError("failed to open Broxygen config file '%s': %s", + file.c_str(), strerror(errno)); string line; + unsigned int line_number = 0; while ( getline(f, line) ) { + ++line_number; vector tokens; tokenize_string(line, delim, &tokens); tokens.erase(remove(tokens.begin(), tokens.end(), ""), tokens.end()); + if ( tokens.empty() ) + // Blank line. + continue; + + if ( ! tokens[0].empty() && tokens[0][0] == '#' ) + // Comment + continue; + if ( tokens.size() != 3 ) - reporter->InternalError("malformed Broxygen target: %s", - line.c_str()); + reporter->FatalError("malformed Broxygen target in %s:%u: %s", + file.c_str(), line_number, line.c_str()); target_factory_map::const_iterator it = target_instantiators.find(tokens[0]); if ( it == target_instantiators.end() ) - reporter->InternalError("unkown Broxygen target type: %s", - tokens[0].c_str()); + reporter->FatalError("unkown Broxygen target type: %s", + tokens[0].c_str()); targets.push_back(it->second(tokens[1], tokens[2])); } if ( f.bad() ) - reporter->InternalError("error reading Broxygen config file %s: %s", + reporter->InternalError("error reading Broxygen config file '%s': %s", file.c_str(), strerror(errno)); } Config::~Config() { - for ( target_list::const_iterator it = targets.begin(); - it != targets.end(); ++it ) - delete *it; + for ( size_t i = 0; i < targets.size(); ++i ) + delete targets[i]; } void Config::FindDependencies(const vector& docs) { - for ( target_list::const_iterator it = targets.begin(); - it != targets.end(); ++it ) - (*it)->FindDependencies(docs); + for ( size_t i = 0; i < targets.size(); ++i ) + targets[i]->FindDependencies(docs); } void Config::GenerateDocs() const { - for ( target_list::const_iterator it = targets.begin(); - it != targets.end(); ++it ) - (*it)->Generate(); + for ( size_t i = 0; i < targets.size(); ++i ) + targets[i]->Generate(); + } + +time_t Config::GetModificationTime() const + { + struct stat s; + + if ( stat(file.c_str(), &s) < 0 ) + reporter->InternalError("Broxygen can't stat config file %s: %s", + file.c_str(), strerror(errno)); + + return s.st_mtime; } diff --git a/src/broxygen/Configuration.h b/src/broxygen/Configuration.h index 2aef6f9806..6b5c45ed45 100644 --- a/src/broxygen/Configuration.h +++ b/src/broxygen/Configuration.h @@ -2,10 +2,11 @@ #define BROXYGEN_CONFIGURATION_H #include "Document.h" +#include "BroDoc.h" #include #include -#include +#include namespace broxygen { @@ -16,7 +17,8 @@ public: typedef Target* (*factory_fn)(const std::string&, const std::string&); - virtual ~Target() { } + virtual ~Target() + { } void FindDependencies(const std::vector& docs) { DoFindDependencies(docs); } @@ -26,27 +28,47 @@ public: bool MatchesPattern(Document* doc) const; - void AddDependency(Document* doc) - { dependencies.push_back(doc); } + std::string Name() const + { return name; } + + std::string Pattern() const + { return pattern; } protected: - Target(const std::string& arg_name, const std::string& arg_pattern) - : name(arg_name), pattern(arg_pattern) - { } - - std::string name; - std::string pattern; - std::list dependencies; + Target(const std::string& arg_name, const std::string& arg_pattern); private: virtual void DoFindDependencies(const std::vector& docs) = 0; virtual void DoGenerate() const = 0; + + std::string name; + std::string pattern; + std::string prefix; }; -class ProtoAnalyzerTarget : public Target { +class AnalyzerTarget : public Target { +protected: + + typedef void (*doc_creator_fn)(FILE*); + + AnalyzerTarget(const std::string& name, const std::string& pattern, + doc_creator_fn cb) + : Target(name, pattern), doc_creator_callback(cb) + { } + +private: + + void DoFindDependencies(const std::vector& docs); + + void DoGenerate() const; + + doc_creator_fn doc_creator_callback; +}; + +class ProtoAnalyzerTarget : public AnalyzerTarget { public: static Target* Instantiate(const std::string& name, @@ -56,17 +78,11 @@ public: private: ProtoAnalyzerTarget(const std::string& name, const std::string& pattern) - : Target(name, pattern) + : AnalyzerTarget(name, pattern, &CreateProtoAnalyzerDoc) { } - - void DoFindDependencies(const std::vector& docs) - { /* TODO */ } - - void DoGenerate() const - { /* TODO */ } }; -class FileAnalyzerTarget : public Target { +class FileAnalyzerTarget : public AnalyzerTarget { public: static Target* Instantiate(const std::string& name, @@ -76,14 +92,8 @@ public: private: FileAnalyzerTarget(const std::string& name, const std::string& pattern) - : Target(name, pattern) + : AnalyzerTarget(name, pattern, &CreateFileAnalyzerDoc) { } - - void DoFindDependencies(const std::vector& docs) - { /* TODO */ } - - void DoGenerate() const - { /* TODO */ } }; class PackageTarget : public Target { @@ -96,14 +106,37 @@ public: private: PackageTarget(const std::string& name, const std::string& pattern) - : Target(name, pattern) + : Target(name, pattern), pkg_deps(), script_deps(), pkg_manifest() { } - void DoFindDependencies(const std::vector& docs) - { /* TODO */ } + void DoFindDependencies(const std::vector& docs); - void DoGenerate() const - { /* TODO */ } + void DoGenerate() const; + + std::vector pkg_deps; + std::vector script_deps; + typedef std::map > manifest_t; + manifest_t pkg_manifest; +}; + +class PackageIndexTarget : public Target { +public: + + static Target* Instantiate(const std::string& name, + const std::string& pattern) + { return new PackageIndexTarget(name, pattern); } + +private: + + PackageIndexTarget(const std::string& name, const std::string& pattern) + : Target(name, pattern), pkg_deps() + { } + + void DoFindDependencies(const std::vector& docs); + + void DoGenerate() const; + + std::vector pkg_deps; }; class ScriptTarget : public Target { @@ -113,17 +146,51 @@ public: const std::string& pattern) { return new ScriptTarget(name, pattern); } -private: +protected: ScriptTarget(const std::string& name, const std::string& pattern) - : Target(name, pattern) + : Target(name, pattern), script_deps() { } - void DoFindDependencies(const std::vector& docs) - { /* TODO */ } + std::vector script_deps; - void DoGenerate() const - { /* TODO */ } +private: + + void DoFindDependencies(const std::vector& docs); + + void DoGenerate() const; +}; + +class ScriptSummaryTarget : public ScriptTarget { +public: + + static Target* Instantiate(const std::string& name, + const std::string& pattern) + { return new ScriptSummaryTarget(name, pattern); } + +private: + + ScriptSummaryTarget(const std::string& name, const std::string& pattern) + : ScriptTarget(name, pattern) + { } + + void DoGenerate() const /* override */; +}; + +class ScriptIndexTarget : public ScriptTarget { +public: + + static Target* Instantiate(const std::string& name, + const std::string& pattern) + { return new ScriptIndexTarget(name, pattern); } + +private: + + ScriptIndexTarget(const std::string& name, const std::string& pattern) + : ScriptTarget(name, pattern) + { } + + void DoGenerate() const /* override */; }; class IdentifierTarget : public Target { @@ -136,13 +203,14 @@ public: private: IdentifierTarget(const std::string& name, const std::string& pattern) - : Target(name, pattern) + : Target(name, pattern), id_deps() { } void DoFindDependencies(const std::vector& docs); - void DoGenerate() const - { /* TODO */ } + void DoGenerate() const; + + std::vector id_deps; }; class Config { @@ -156,11 +224,12 @@ public: void GenerateDocs() const; + time_t GetModificationTime() const; + private: - typedef std::list target_list; - - target_list targets; + std::string file; + std::vector targets; }; diff --git a/src/broxygen/Document.cc b/src/broxygen/Document.cc index dbac11f234..2619755b35 100644 --- a/src/broxygen/Document.cc +++ b/src/broxygen/Document.cc @@ -1,38 +1,134 @@ #include "Document.h" +#include "Manager.h" #include "util.h" #include "Val.h" +#include "Desc.h" +#include "Reporter.h" + +#include +#include using namespace broxygen; using namespace std; -static string ImplodeStringVec(const vector& v) +static bool is_public_api(const ID* id) { - string rval; + return (id->Scope() == SCOPE_GLOBAL) || + (id->Scope() == SCOPE_MODULE && id->IsExport()); + } - for ( size_t i = 0; i < v.size(); ++i ) +static bool prettify_params(string& s) + { + size_t identifier_start_pos = 0; + bool in_identifier = false; + string identifier; + + for ( size_t i = 0; i < s.size(); ++i ) { - if ( i > 0 ) - rval += '\n'; + char next = s[i]; - rval += v[i]; + if ( ! in_identifier ) + { + // Pass by leading whitespace. + if ( isspace(next) ) + continue; + + // Only allow alphabetic and '_' as first char of identifier. + if ( isalpha(next) || next == '_' ) + { + identifier_start_pos = i; + identifier += next; + in_identifier = true; + continue; + } + + // Don't need to change anything. + return false; + } + + // All other characters of identifier are alphanumeric or '_'. + if ( isalnum(next) || next == '_' ) + { + identifier += next; + continue; + } + + if ( next == ':' ) + { + if ( i + 1 < s.size() && s[i + 1] == ':' ) + { + // It's part of an identifier's namespace scoping. + identifier += next; + identifier += s[i + 1]; + ++i; + continue; + } + + // Prettify function param/return value reST markup. + string subst; + + if ( identifier == "Returns" ) + subst = ":returns"; + else + subst = ":param " + identifier; + + s.replace(identifier_start_pos, identifier.size(), subst); + return true; + } + + // Don't need to change anything. + return false; } - return rval; + return false; + } + +static string make_heading(const string& heading, char underline) + { + return heading + "\n" + string(heading.size(), underline) + "\n"; } PackageDocument::PackageDocument(const string& arg_name) : Document(), - pkg_name(arg_name) + pkg_name(arg_name), readme() { - // TODO: probably need to determine modification times of all files - // within the directory, recursively + string readme_file = find_file(pkg_name + "/README", bro_path()); + + if ( readme_file.empty() ) + return; + + ifstream f(readme_file.c_str()); + + if ( ! f.is_open() ) + reporter->InternalWarning("Broxygen failed to open '%s': %s", + readme_file.c_str(), strerror(errno)); + + string line; + + while ( getline(f, line) ) + readme.push_back(line); + + if ( f.bad() ) + reporter->InternalWarning("Broxygen error reading '%s': %s", + readme_file.c_str(), strerror(errno)); } -IdentifierDocument::IdentifierDocument(ID* arg_id) +string PackageDocument::DoReStructuredText(bool roles_only) const + { + string rval = fmt(":doc:`%s <%s/index>`\n\n", pkg_name.c_str(), + pkg_name.c_str()); + + for ( size_t i = 0; i < readme.size(); ++i ) + rval += " " + readme[i] + "\n"; + + return rval; + } + +IdentifierDocument::IdentifierDocument(ID* arg_id, ScriptDocument* script) : Document(), comments(), id(arg_id), initial_val_desc(), redefs(), fields(), - last_field_seen() + last_field_seen(), declaring_script(script) { Ref(id); @@ -86,36 +182,539 @@ void IdentifierDocument::AddRecordField(const TypeDecl* field, last_field_seen = rf; } -string IdentifierDocument::GetComments() const +vector IdentifierDocument::GetComments() const { - return ImplodeStringVec(comments); + return comments; } -string IdentifierDocument::GetFieldComments(const string& field) const +vector IdentifierDocument::GetFieldComments(const string& field) const { record_field_map::const_iterator it = fields.find(field); if ( it == fields.end() ) - return string(); + return vector(); - return ImplodeStringVec(it->second->comments); + return it->second->comments; } -ScriptDocument::ScriptDocument(const string& arg_name) +list +IdentifierDocument::GetRedefs(const string& from_script) const + { + list rval; + + for ( redef_list::const_iterator it = redefs.begin(); it != redefs.end(); + ++it ) + { + if ( from_script == (*it)->from_script ) + rval.push_back(*(*it)); + } + + return rval; + } + +string IdentifierDocument::GetDeclaringScriptForField(const string& field) const + { + record_field_map::const_iterator it = fields.find(field); + + if ( it == fields.end() ) + return ""; + + return it->second->from_script; + } + +string IdentifierDocument::DoReStructuredText(bool roles_only) const + { + ODesc d; + d.SetIndentSpaces(3); + d.SetQuotes(true); + id->DescribeReST(&d, roles_only); + + if ( comments.empty() ) + return d.Description(); + + d.ClearIndentLevel(); + d.PushIndent(); + + for ( size_t i = 0; i < comments.size(); ++i ) + { + if ( i > 0 ) + d.NL(); + + if ( IsFunc(id->Type()->Tag()) ) + { + string s = comments[i]; + + if ( prettify_params(s) ) + d.NL(); + + d.Add(s.c_str()); + } + else + d.Add(comments[i].c_str()); + } + + return d.Description(); + } + +ScriptDocument::ScriptDocument(const string& arg_name, const string& arg_path) : Document(), - name(arg_name), + name(arg_name), path(arg_path), is_pkg_loader(SafeBasename(name).result == PACKAGE_LOADER), - dependencies(), module_usages(), comments(), identifier_docs(), redefs() + dependencies(), module_usages(), comments(), identifier_docs(), + options(), constants(), state_vars(), types(), events(), hooks(), + functions(), redefs() { } void ScriptDocument::AddIdentifierDoc(IdentifierDocument* doc) { identifier_docs[doc->Name()] = doc; - // TODO: sort things (e.g. function flavor, state var vs. option var) } -string ScriptDocument::GetComments() const +void ScriptDocument::DoInitPostScript() { - return ImplodeStringVec(comments); + for ( id_doc_map::const_iterator it = identifier_docs.begin(); + it != identifier_docs.end(); ++it ) + { + IdentifierDocument* doc = it->second; + ID* id = doc->GetID(); + + if ( ! is_public_api(id) ) + continue; + + if ( id->AsType() ) + { + types.push_back(doc); + DBG_LOG(DBG_BROXYGEN, "Filter id '%s' in '%s' as a type", + id->Name(), name.c_str()); + continue; + } + + if ( IsFunc(id->Type()->Tag()) ) + { + switch ( id->Type()->AsFuncType()->Flavor() ) { + case FUNC_FLAVOR_HOOK: + DBG_LOG(DBG_BROXYGEN, "Filter id '%s' in '%s' as a hook", + id->Name(), name.c_str()); + hooks.push_back(doc); + break; + case FUNC_FLAVOR_EVENT: + DBG_LOG(DBG_BROXYGEN, "Filter id '%s' in '%s' as a event", + id->Name(), name.c_str()); + events.push_back(doc); + break; + case FUNC_FLAVOR_FUNCTION: + DBG_LOG(DBG_BROXYGEN, "Filter id '%s' in '%s' as a function", + id->Name(), name.c_str()); + functions.push_back(doc); + break; + default: + reporter->InternalError("Invalid function flavor"); + break; + } + + continue; + } + + if ( id->IsConst() ) + { + if ( id->FindAttr(ATTR_REDEF) ) + { + DBG_LOG(DBG_BROXYGEN, "Filter id '%s' in '%s' as an option", + id->Name(), name.c_str()); + options.push_back(doc); + } + else + { + DBG_LOG(DBG_BROXYGEN, "Filter id '%s' in '%s' as a constant", + id->Name(), name.c_str()); + constants.push_back(doc); + } + + continue; + } + + if ( id->Type()->Tag() == TYPE_ENUM ) + // Enums are always referenced/documented from the type's + // documentation. + continue; + + DBG_LOG(DBG_BROXYGEN, "Filter id '%s' in '%s' as a state variable", + id->Name(), name.c_str()); + state_vars.push_back(doc); + } + } + +vector ScriptDocument::GetComments() const + { + return comments; + } + +static size_t end_of_first_sentence(const string& s) + { + size_t rval = 0; + + while ( (rval = s.find_first_of('.', rval)) != string::npos ) + { + if ( rval == s.size() - 1 ) + // Period is at end of string. + return rval; + + if ( isspace(s[rval + 1]) ) + // Period has a space after it. + return rval; + + // Period has some non-space character after it, keep looking. + ++rval; + } + + return rval; + } + +static bool is_all_whitespace(const string& s) + { + for ( size_t i = 0; i < s.size(); ++i ) + if ( ! isspace(s[i]) ) + return false; + + return true; + } + +static vector summary_comment(const vector& cmnts) + { + vector rval; + + for ( size_t i = 0; i < cmnts.size(); ++i ) + { + size_t end = end_of_first_sentence(cmnts[i]); + + if ( end == string::npos ) + { + if ( is_all_whitespace(cmnts[i]) ) + break; + + rval.push_back(cmnts[i]); + } + else + { + rval.push_back(cmnts[i].substr(0, end + 1)); + break; + } + } + + return rval; + } + +class ReStructuredTextTable { +public: + + ReStructuredTextTable(size_t arg_num_cols) + : num_cols(arg_num_cols), rows(), longest_row_in_column() + { + for ( size_t i = 0; i < num_cols; ++i ) + longest_row_in_column.push_back(1); + } + + void AddRow(const vector& new_row) + { + assert(new_row.size() == num_cols); + rows.push_back(new_row); + + for ( size_t i = 0; i < new_row.size(); ++i ) + if ( new_row[i].size() > longest_row_in_column[i] ) + longest_row_in_column[i] = new_row[i].size(); + } + + static string MakeBorder(const vector col_sizes, char border) + { + string rval; + + for ( size_t i = 0; i < col_sizes.size(); ++i ) + { + if ( i > 0 ) + rval += " "; + + rval += string(col_sizes[i], border); + } + + rval += "\n"; + return rval; + } + + string AsString(char border) const + { + string rval = MakeBorder(longest_row_in_column, border); + + for ( size_t row = 0; row < rows.size(); ++row ) + { + for ( size_t col = 0; col < num_cols; ++col ) + { + if ( col > 0 ) + { + size_t last = rows[row][col - 1].size(); + size_t longest = longest_row_in_column[col - 1]; + size_t whitespace = longest - last + 1; + rval += string(whitespace, ' '); + } + + rval += rows[row][col]; + } + + rval += "\n"; + } + + rval += MakeBorder(longest_row_in_column, border); + return rval; + } + + +private: + + size_t num_cols; + vector > rows; + vector longest_row_in_column; +}; + +static void add_summary_rows(const ODesc& id_desc, const vector& cmnts, + ReStructuredTextTable* table) + { + vector row; + row.push_back(id_desc.Description()); + + if ( cmnts.empty() ) + { + row.push_back(""); + table->AddRow(row); + return; + } + + row.push_back(cmnts[0]); + table->AddRow(row); + + for ( size_t i = 1; i < cmnts.size(); ++i ) + { + row.clear(); + row.push_back(""); + row.push_back(cmnts[i]); + table->AddRow(row); + } + } + +static string make_summary(const string& heading, char underline, char border, + const list& id_list) + { + if ( id_list.empty() ) + return ""; + + ReStructuredTextTable table(2); + + for ( list::const_iterator it = id_list.begin(); + it != id_list.end(); ++it ) + { + ID* id = (*it)->GetID(); + ODesc d; + d.SetQuotes(1); + id->DescribeReSTShort(&d); + add_summary_rows(d, summary_comment((*it)->GetComments()), &table); + } + + return make_heading(heading, underline) + table.AsString(border) + "\n"; + } + +static string make_redef_summary(const string& heading, char underline, + char border, const string& from_script, + const set& id_set) + { + if ( id_set.empty() ) + return ""; + + ReStructuredTextTable table(2); + + for ( set::const_iterator it = id_set.begin(); + it != id_set.end(); ++it ) + { + ID* id = (*it)->GetID(); + ODesc d; + d.SetQuotes(1); + id->DescribeReSTShort(&d); + + typedef list redef_list; + redef_list redefs = (*it)->GetRedefs(from_script); + + for ( redef_list::const_iterator iit = redefs.begin(); + iit != redefs.end(); ++iit ) + add_summary_rows(d, summary_comment(iit->comments), &table); + } + + return make_heading(heading, underline) + table.AsString(border) + "\n"; + } + +static string make_details(const string& heading, char underline, + const list& id_list) + { + if ( id_list.empty() ) + return ""; + + string rval = make_heading(heading, underline); + + for ( list::const_iterator it = id_list.begin(); + it != id_list.end(); ++it ) + { + rval += (*it)->ReStructuredText(); + rval += "\n\n"; + } + + return rval; + } + +static string make_redef_details(const string& heading, char underline, + const set& id_set) + { + if ( id_set.empty() ) + return ""; + + string rval = make_heading(heading, underline); + + for ( set::const_iterator it = id_set.begin(); + it != id_set.end(); ++it ) + { + rval += (*it)->ReStructuredText(true); + rval += "\n\n"; + } + + return rval; + } + +string ScriptDocument::DoReStructuredText(bool roles_only) const + { + string rval; + + rval += ":tocdepth: 3\n\n"; + rval += make_heading(name, '='); + + for ( string_set::const_iterator it = module_usages.begin(); + it != module_usages.end(); ++it ) + rval += ".. bro:namespace:: " + *it + "\n"; + + rval += "\n"; + + for ( size_t i = 0; i < comments.size(); ++i ) + rval += comments[i] + "\n"; + + rval += "\n"; + + if ( ! module_usages.empty() ) + { + rval += module_usages.size() > 1 ? ":Namespaces: " : ":Namespace: "; + + for ( string_set::const_iterator it = module_usages.begin(); + it != module_usages.end(); ++it ) + { + if ( it != module_usages.begin() ) + rval += ", "; + + rval += *it; + } + + rval += "\n"; + } + + if ( ! dependencies.empty() ) + { + rval += ":Imports: "; + + for ( string_set::const_iterator it = dependencies.begin(); + it != dependencies.end(); ++it ) + { + if ( it != dependencies.begin() ) + rval += ", "; + + rval += fmt(":doc:`%s `", it->c_str(), it->c_str()); + // TODO linking to packages is a bit different? + } + + rval += "\n"; + } + + // TODO: make this an absolute path? + rval += fmt(":Source File: :download:`%s`\n", name.c_str()); + rval += "\n"; + rval += make_heading("Summary", '~'); + rval += make_summary("Options", '#', '=', options); + rval += make_summary("Constants", '#', '=', constants); + rval += make_summary("State Variables", '#', '=', state_vars); + rval += make_summary("Types", '#', '=', types); + rval += make_redef_summary("Redefinitions", '#', '=', name, redefs); + rval += make_summary("Events", '#', '=', events); + rval += make_summary("Hooks", '#', '=', hooks); + rval += make_summary("Functions", '#', '=', functions); + rval += "\n"; + rval += make_heading("Detailed Interface", '~'); + rval += make_details("Options", '#', options); + rval += make_details("Constants", '#', constants); + rval += make_details("State Variables", '#', state_vars); + rval += make_details("Types", '#', types); + //rval += make_redef_details("Redefinitions", '#', redefs); + rval += make_details("Events", '#', events); + rval += make_details("Hooks", '#', hooks); + rval += make_details("Functions", '#', functions); + + return rval; + } + +static time_t get_mtime(const string& filename) + { + struct stat s; + + if ( stat(filename.c_str(), &s) < 0 ) + reporter->InternalError("Broxygen failed to stat file '%s': %s", + filename.c_str(), strerror(errno)); + + return s.st_mtime; + } + +time_t IdentifierDocument::DoGetModificationTime() const + { + // Could probably get away with just checking the set of scripts that + // contributed to the ID declaration/redefinitions, but this is easier... + return declaring_script->GetModificationTime(); + } + +time_t ScriptDocument::DoGetModificationTime() const + { + time_t most_recent = get_mtime(path); + + for ( string_set::const_iterator it = dependencies.begin(); + it != dependencies.end(); ++it ) + { + Document* doc = broxygen_mgr->GetScriptDoc(*it); + + if ( ! doc ) + { + string pkg_name = *it + "/" + PACKAGE_LOADER; + doc = broxygen_mgr->GetScriptDoc(pkg_name); + + if ( ! doc ) + reporter->InternalWarning("Broxygen failed to get mtime of %s", + it->c_str()); + continue; + } + + time_t dep_mtime = doc->GetModificationTime(); + + if ( dep_mtime > most_recent ) + most_recent = dep_mtime; + } + + return most_recent; + } + +time_t PackageDocument::DoGetModificationTime() const + { + string readme_file = find_file(pkg_name + "/README", bro_path()); + + if ( readme_file.empty() ) + return 0; + + return get_mtime(readme_file); } diff --git a/src/broxygen/Document.h b/src/broxygen/Document.h index ec9212bfba..6046369727 100644 --- a/src/broxygen/Document.h +++ b/src/broxygen/Document.h @@ -7,7 +7,7 @@ #include #include #include -#include +#include #include "ID.h" #include "Type.h" @@ -32,10 +32,19 @@ public: std::string Name() const { return DoName(); } + std::string ReStructuredText(bool roles_only = false) const + { return DoReStructuredText(roles_only); } + + void InitPostScript() + { return DoInitPostScript(); } + private: virtual time_t DoGetModificationTime() const = 0; virtual std::string DoName() const = 0; + virtual std::string DoReStructuredText(bool roles_only) const = 0; + virtual void DoInitPostScript() + { } }; class PackageDocument : public Document { @@ -44,28 +53,29 @@ public: PackageDocument(const std::string& name); - // TODO: can be comments from package README - std::string GetReadme() const - { return std::string(); } + std::vector GetReadme() const + { return readme; } private: - // TODO - time_t DoGetModificationTime() const - { return 0; } + time_t DoGetModificationTime() const; std::string DoName() const { return pkg_name; } + std::string DoReStructuredText(bool roles_only) const; + std::string pkg_name; + std::vector readme; }; -class IdentifierDocument : public Document { +class ScriptDocument; +class IdentifierDocument : public Document { public: - IdentifierDocument(ID* id); + IdentifierDocument(ID* id, ScriptDocument* script); ~IdentifierDocument(); @@ -88,24 +98,31 @@ public: ID* GetID() const { return id; } - std::string GetComments() const; + ScriptDocument* GetDeclaringScript() const + { return declaring_script; } - std::string GetFieldComments(const std::string& field) const; + std::string GetDeclaringScriptForField(const std::string& field) const; + + std::vector GetComments() const; + + std::vector GetFieldComments(const std::string& field) const; + + struct Redefinition { + std::string from_script; + std::string new_val_desc; + std::vector comments; + }; + + std::list GetRedefs(const std::string& from_script) const; private: - // TODO - time_t DoGetModificationTime() const - { return 0; } + time_t DoGetModificationTime() const; std::string DoName() const { return id->Name(); } - struct Redefinition { - std::string from_script; - string new_val_desc; - std::vector comments; - }; + std::string DoReStructuredText(bool roles_only) const; struct RecordField { ~RecordField() @@ -121,17 +138,18 @@ private: std::vector comments; ID* id; - string initial_val_desc; + std::string initial_val_desc; redef_list redefs; record_field_map fields; RecordField* last_field_seen; + ScriptDocument* declaring_script; }; class ScriptDocument : public Document { public: - ScriptDocument(const std::string& name); + ScriptDocument(const std::string& name, const std::string& path); void AddComment(const std::string& comment) { comments.push_back(comment); } @@ -145,32 +163,44 @@ public: void AddIdentifierDoc(IdentifierDocument* doc); void AddRedef(IdentifierDocument* doc) - { redefs.push_back(doc); } + { redefs.insert(doc); } bool IsPkgLoader() const { return is_pkg_loader; } - std::string GetComments() const; + std::vector GetComments() const; private: - typedef std::map IdentifierDocMap; - typedef std::list IdentifierDocList; + typedef std::map id_doc_map; + typedef std::list id_doc_list; + typedef std::set string_set; + typedef std::set doc_set; - // TODO - time_t DoGetModificationTime() const - { return 0; } + time_t DoGetModificationTime() const; std::string DoName() const { return name; } + std::string DoReStructuredText(bool roles_only) const; + + void DoInitPostScript() /* override */; + std::string name; + std::string path; bool is_pkg_loader; - std::set dependencies; - std::set module_usages; + string_set dependencies; + string_set module_usages; std::vector comments; - IdentifierDocMap identifier_docs; - IdentifierDocList redefs; + id_doc_map identifier_docs; + id_doc_list options; + id_doc_list constants; + id_doc_list state_vars; + id_doc_list types; + id_doc_list events; + id_doc_list hooks; + id_doc_list functions; + doc_set redefs; }; diff --git a/src/broxygen/Manager.cc b/src/broxygen/Manager.cc index 707481d86a..68e4c07b98 100644 --- a/src/broxygen/Manager.cc +++ b/src/broxygen/Manager.cc @@ -1,8 +1,8 @@ #include "Manager.h" -#include "Reporter.h" #include "util.h" #include +#include using namespace broxygen; using namespace std; @@ -25,71 +25,22 @@ static string RemoveLeadingSpace(const string& s) return rval; } -static string PrettifyParams(const string& s) - { - size_t identifier_start_pos = 0; - bool in_identifier = false; - string identifier; - - for ( size_t i = 0; i < s.size(); ++i ) - { - char next = s[i]; - - if ( ! in_identifier ) - { - // Pass by leading whitespace. - if ( isspace(next) ) - continue; - - // Only allow alphabetic and '_' as first char of identifier. - if ( isalpha(next) || next == '_' ) - { - identifier_start_pos = i; - identifier += next; - in_identifier = true; - continue; - } - - // Don't need to change anything. - return s; - } - - // All other character of identifier are alphanumeric or '_'. - if ( isalnum(next) || next == '_' ) - { - identifier += next; - continue; - } - - // Prettify param and return value docs for a function's reST markup. - if ( next == ':' ) - { - string rval = s; - string subst; - - if ( identifier == "Returns" ) - subst = "\n:returns"; - else - subst = "\n:param " + identifier; - - rval.replace(identifier_start_pos, identifier.size(), subst); - return rval; - } - - // Don't need to change anything. - return s; - } - - return s; - } - -Manager::Manager(const string& arg_config) +Manager::Manager(const string& arg_config, const string& bro_command) : disabled(), comment_buffer(), comment_buffer_map(), packages(), scripts(), identifiers(), all_docs(), last_identifier_seen(), incomplete_type(), - config(arg_config) + enum_mappings(), config(arg_config), bro_mtime() { if ( getenv("BRO_DISABLE_BROXYGEN") ) disabled = true; + + string path_to_bro = find_file(bro_command, getenv("PATH")); + struct stat s; + + if ( path_to_bro.empty() || stat(path_to_bro.c_str(), &s) < 0 ) + reporter->InternalError("Broxygen can't get mtime of bro binary %s: %s", + path_to_bro.c_str(), strerror(errno)); + + bro_mtime = s.st_mtime; } Manager::~Manager() @@ -102,7 +53,6 @@ void Manager::InitPreScript() { if ( disabled ) return; - // TODO: create file/proto analyzer doc } void Manager::InitPostScript() @@ -110,6 +60,9 @@ void Manager::InitPostScript() if ( disabled ) return; + for ( size_t i = 0; i < all_docs.size(); ++i ) + all_docs[i]->InitPostScript(); + config.FindDependencies(all_docs); } @@ -134,7 +87,7 @@ void Manager::File(const string& path) return; } - ScriptDocument* doc = new ScriptDocument(name); + ScriptDocument* doc = new ScriptDocument(name, path); scripts.map[name] = doc; all_docs.push_back(doc); DBG_LOG(DBG_BROXYGEN, "Made ScriptDocument %s", name.c_str()); @@ -209,7 +162,7 @@ void Manager::ModuleUsage(const string& path, const string& module) IdentifierDocument* Manager::CreateIdentifierDoc(ID* id, ScriptDocument* script) { - IdentifierDocument* rval = new IdentifierDocument(id); + IdentifierDocument* rval = new IdentifierDocument(id, script); rval->AddComments(comment_buffer); comment_buffer.clear(); @@ -226,7 +179,10 @@ IdentifierDocument* Manager::CreateIdentifierDoc(ID* id, ScriptDocument* script) all_docs.push_back(rval); identifiers.map[id->Name()] = rval; last_identifier_seen = rval; - script->AddIdentifierDoc(rval); + + if ( script ) + script->AddIdentifierDoc(rval); + return rval; } @@ -256,18 +212,9 @@ void Manager::StartType(ID* id) id->Name(), script.c_str()); } -void Manager::StartRedef(ID* id) +static bool IsEnumType(ID* id) { - if ( disabled ) - return; - - IdentifierDocument* id_doc = identifiers.GetDocument(id->Name()); - - if ( id_doc ) - last_identifier_seen = id_doc; - else - DbgAndWarn(fmt("Broxygen redef tracking unknown identifier: %s", - id->Name())); + return id->AsType() ? id->AsType()->Tag() == TYPE_ENUM : false; } void Manager::Identifier(ID* id) @@ -275,21 +222,18 @@ void Manager::Identifier(ID* id) if ( disabled ) return; - if ( incomplete_type && incomplete_type->Name() == id->Name() ) + if ( incomplete_type ) { - DBG_LOG(DBG_BROXYGEN, "Finished document for type %s", id->Name()); - incomplete_type->CompletedTypeDecl(); - incomplete_type = 0; - return; - } + if ( incomplete_type->Name() == id->Name() ) + { + DBG_LOG(DBG_BROXYGEN, "Finished document for type %s", id->Name()); + incomplete_type->CompletedTypeDecl(); + incomplete_type = 0; + return; + } - if ( id->GetLocationInfo() == &no_location ) - { - // Internally-created identifier (e.g. file/proto analyzer enum tags). - // Can be ignored here as they need to be documented via other means. - DBG_LOG(DBG_BROXYGEN, "Skip documenting identifier %s: no location", - id->Name()); - return; + if ( IsEnumType(incomplete_type->GetID()) ) + enum_mappings[id->Name()] = incomplete_type->GetID()->Name(); } IdentifierDocument* id_doc = identifiers.GetDocument(id->Name()); @@ -308,6 +252,16 @@ void Manager::Identifier(ID* id) return; } + if ( id->GetLocationInfo() == &no_location ) + { + // Internally-created identifier (e.g. file/proto analyzer enum tags). + // Handled specially since they don't have a script location. + DBG_LOG(DBG_BROXYGEN, "Made internal IdentifierDocument %s", + id->Name()); + CreateIdentifierDoc(id, 0); + return; + } + string script = without_bropath_component(id->GetLocationInfo()->filename); ScriptDocument* script_doc = scripts.GetDocument(script); @@ -428,32 +382,8 @@ void Manager::PostComment(const string& comment, const string& id_hint) comment_buffer_map[id_hint].push_back(RemoveLeadingSpace(comment)); } -StringVal* Manager::GetIdentifierComments(const string& name) const +string Manager::GetEnumTypeName(const string& id) const { - IdentifierDocument* d = identifiers.GetDocument(name); - return new StringVal(d ? d->GetComments() : ""); - } - -StringVal* Manager::GetScriptComments(const string& name) const - { - ScriptDocument* d = scripts.GetDocument(name); - return new StringVal(d ? d->GetComments() : ""); - } - -StringVal* Manager::GetPackageReadme(const string& name) const - { - PackageDocument* d = packages.GetDocument(name); - return new StringVal(d ? d->GetReadme() : ""); - } - -StringVal* Manager::GetRecordFieldComments(const string& name) const - { - size_t i = name.find('$'); - - if ( i > name.size() - 2 ) - // '$' is last char in string or not found. - return new StringVal(""); - - IdentifierDocument* d = identifiers.GetDocument(name.substr(0, i)); - return new StringVal(d ? d->GetFieldComments(name.substr(i + 1)) : ""); + map::const_iterator it = enum_mappings.find(id); + return it == enum_mappings.end() ? "" : it->second; } diff --git a/src/broxygen/Manager.h b/src/broxygen/Manager.h index 67f7e543af..42b9ea828a 100644 --- a/src/broxygen/Manager.h +++ b/src/broxygen/Manager.h @@ -3,6 +3,8 @@ #include "Configuration.h" #include "Document.h" + +#include "Reporter.h" #include "ID.h" #include "Type.h" #include "Val.h" @@ -10,6 +12,8 @@ #include #include #include +#include +#include namespace broxygen { @@ -32,7 +36,7 @@ class Manager { public: - Manager(const std::string& config); + Manager(const std::string& config, const std::string& bro_command); ~Manager(); @@ -50,8 +54,6 @@ public: void StartType(ID* id); - void StartRedef(ID* id); - void Identifier(ID* id); void RecordField(const ID* id, const TypeDecl* field, @@ -66,13 +68,20 @@ public: void PostComment(const std::string& comment, const std::string& identifier_hint = ""); - StringVal* GetIdentifierComments(const std::string& name) const; + std::string GetEnumTypeName(const std::string& id) const; - StringVal* GetScriptComments(const std::string& name) const; + IdentifierDocument* GetIdentifierDoc(const std::string& name) const + { return identifiers.GetDocument(name); } - StringVal* GetPackageReadme(const std::string& name) const; + ScriptDocument* GetScriptDoc(const std::string& name) const + { return scripts.GetDocument(name); } - StringVal* GetRecordFieldComments(const std::string& name) const; + PackageDocument* GetPackageDoc(const std::string& name) const + { return packages.GetDocument(name); } + + template + bool IsUpToDate(const std::string& target_file, + const std::vector& dependencies) const; private: @@ -90,9 +99,40 @@ private: std::vector all_docs; IdentifierDocument* last_identifier_seen; IdentifierDocument* incomplete_type; + std::map enum_mappings; Config config; + time_t bro_mtime; }; +template +bool Manager::IsUpToDate(const string& target_file, + const vector& dependencies) const + { + struct stat s; + + if ( stat(target_file.c_str(), &s) < 0 ) + { + if ( errno == ENOENT ) + // Doesn't exist. + return false; + + reporter->InternalError("Broxygen failed to stat target file '%s': %s", + target_file.c_str(), strerror(errno)); + } + + if ( difftime(bro_mtime, s.st_mtime) > 0 ) + return false; + + if ( difftime(config.GetModificationTime(), s.st_mtime) > 0 ) + return false; + + for ( size_t i = 0; i < dependencies.size(); ++i ) + if ( difftime(dependencies[i]->GetModificationTime(), s.st_mtime) > 0 ) + return false; + + return true; + } + } // namespace broxygen extern broxygen::Manager* broxygen_mgr; diff --git a/src/broxygen/broxygen.bif b/src/broxygen/broxygen.bif index a241b2c985..0d346411d0 100644 --- a/src/broxygen/broxygen.bif +++ b/src/broxygen/broxygen.bif @@ -2,26 +2,65 @@ %%{ #include "broxygen/Manager.h" +#include "util.h" + +static StringVal* comments_to_val(const vector& comments) + { + return new StringVal(implode_string_vector(comments)); + } %%} # TODO: documentation function get_identifier_comments%(name: string%): string %{ - return broxygen_mgr->GetIdentifierComments(name->CheckString()); + using namespace broxygen; + IdentifierDocument* d = broxygen_mgr->GetIdentifierDoc(name->CheckString()); + + if ( ! d ) + return new StringVal(""); + + return comments_to_val(d->GetComments()); %} function get_script_comments%(name: string%): string %{ - return broxygen_mgr->GetScriptComments(name->CheckString()); + using namespace broxygen; + ScriptDocument* d = broxygen_mgr->GetScriptDoc(name->CheckString()); + + if ( ! d ) + return new StringVal(""); + + return comments_to_val(d->GetComments()); %} function get_package_readme%(name: string%): string %{ - return broxygen_mgr->GetPackageReadme(name->CheckString()); + using namespace broxygen; + PackageDocument* d = broxygen_mgr->GetPackageDoc(name->CheckString()); + + if ( ! d ) + return new StringVal(""); + + return comments_to_val(d->GetReadme()); %} function get_record_field_comments%(name: string%): string %{ - return broxygen_mgr->GetRecordFieldComments(name->CheckString()); + using namespace broxygen; + string accessor = name->CheckString(); + size_t i = accessor.find('$'); + + if ( i > accessor.size() - 2 ) + return new StringVal(""); + + string id = accessor.substr(0, i); + + IdentifierDocument* d = broxygen_mgr->GetIdentifierDoc(id); + + if ( ! d ) + return new StringVal(""); + + string field = accessor.substr(i + 1); + return comments_to_val(d->GetFieldComments(field)); %} diff --git a/src/main.cc b/src/main.cc index 2ab25460d4..cac1acd7b1 100644 --- a/src/main.cc +++ b/src/main.cc @@ -762,7 +762,6 @@ int main(int argc, char** argv) reporter = new Reporter(); thread_mgr = new threading::Manager(); - broxygen_mgr = new broxygen::Manager(broxygen_config); #ifdef DEBUG if ( debug_streams ) @@ -809,6 +808,8 @@ int main(int argc, char** argv) timer_mgr = new PQ_TimerMgr(""); // timer_mgr = new CQ_TimerMgr(); + broxygen_mgr = new broxygen::Manager(broxygen_config, bro_argv[0]); + add_input_file("base/init-bare.bro"); if ( ! bare_mode ) add_input_file("base/init-default.bro"); diff --git a/src/parse.y b/src/parse.y index d428452d66..840ccd527d 100644 --- a/src/parse.y +++ b/src/parse.y @@ -1022,14 +1022,15 @@ decl: broxygen_mgr->Redef($2, ::filename); } - | TOK_REDEF TOK_ENUM global_id TOK_ADD_TO - '{' { parser_redef_enum($3); } enum_body '}' ';' + | TOK_REDEF TOK_ENUM global_id TOK_ADD_TO '{' + { parser_redef_enum($3); broxygen_mgr->Redef($3, ::filename); } + enum_body '}' ';' { // Broxygen already grabbed new enum IDs as the type created them. } | TOK_REDEF TOK_RECORD global_id - { cur_decl_type_id = $3; broxygen_mgr->StartRedef($3); } + { cur_decl_type_id = $3; broxygen_mgr->Redef($3, ::filename); } TOK_ADD_TO '{' { ++in_record; } type_decl_list diff --git a/src/plugin/ComponentManager.h b/src/plugin/ComponentManager.h index dab98fcad0..7561764035 100644 --- a/src/plugin/ComponentManager.h +++ b/src/plugin/ComponentManager.h @@ -10,6 +10,7 @@ #include "Var.h" #include "Val.h" #include "Reporter.h" +#include "broxygen/Manager.h" namespace plugin { @@ -133,6 +134,7 @@ ComponentManager::ComponentManager(const string& arg_module) tag_enum_type = new EnumType(); ::ID* id = install_ID("Tag", module.c_str(), true, true); add_type(id, tag_enum_type, 0); + broxygen_mgr->Identifier(id); } template diff --git a/src/scan.l b/src/scan.l index 3dc68a52cc..5b80350111 100644 --- a/src/scan.l +++ b/src/scan.l @@ -145,7 +145,8 @@ ESCSEQ (\\([^\n]|[0-7]+|x[[:xdigit:]]+)) } ##.* { - broxygen_mgr->PreComment(yytext + 2); + if ( yytext[2] != '#' ) + broxygen_mgr->PreComment(yytext + 2); } #{OWS}@no-test.* return TOK_NO_TEST; diff --git a/src/util.cc b/src/util.cc index 8f85d6fca4..c526527648 100644 --- a/src/util.cc +++ b/src/util.cc @@ -590,6 +590,33 @@ const char* fmt_access_time(double t) return buf; } +bool ensure_intermediate_dirs(const char* dirname) + { + if ( ! dirname || strlen(dirname) == 0 ) + return false; + + bool absolute = dirname[0] == '/'; + string path = normalize_path(dirname); + + vector path_components; + tokenize_string(path, "/", &path_components); + + string current_dir; + + for ( size_t i = 0; i < path_components.size(); ++i ) + { + if ( i > 0 || absolute ) + current_dir += "/"; + + current_dir += path_components[i]; + + if ( ! ensure_dir(current_dir.c_str()) ) + return false; + } + + return true; + } + bool ensure_dir(const char *dirname) { struct stat st; @@ -976,6 +1003,22 @@ void SafePathOp::DoFunc(PathOpFn fn, const string& path, bool error_aborts) delete [] tmp; } +string implode_string_vector(const std::vector& v, + const std::string& delim) + { + string rval; + + for ( size_t i = 0; i < v.size(); ++i ) + { + if ( i > 0 ) + rval += delim; + + rval += v[i]; + } + + return rval; + } + string flatten_script_name(const string& name, const string& prefix) { string rval = prefix; diff --git a/src/util.h b/src/util.h index 1dc47db68a..815534a6a6 100644 --- a/src/util.h +++ b/src/util.h @@ -150,6 +150,7 @@ extern const char* fmt(const char* format, ...) myattribute((format (printf, 1, 2))); extern const char* fmt_access_time(double time); +extern bool ensure_intermediate_dirs(const char* dirname); extern bool ensure_dir(const char *dirname); // Returns true if path exists and is a directory. @@ -251,6 +252,9 @@ public: : SafePathOp(&basename, path, error_aborts) { } }; +std::string implode_string_vector(const std::vector& v, + const std::string& delim = "\n"); + /** * Flatten a script name by replacing '/' path separators with '.'. * @param file A path to a Bro script. If it is a __load__.bro, that part