diff --git a/src/BroDoc.cc b/src/BroDoc.cc index 6953680df0..5733c586bf 100644 --- a/src/BroDoc.cc +++ b/src/BroDoc.cc @@ -556,6 +556,7 @@ static void WritePluginBifItems(FILE* f, const plugin::Plugin* p, static void WriteAnalyzerTagDefn(FILE* f, EnumType* e, const string& module) { + /* TODO string tag_id= module + "::Tag"; e = new CommentedEnumType(e); e->SetTypeID(copy_string(tag_id.c_str())); @@ -570,6 +571,7 @@ static void WriteAnalyzerTagDefn(FILE* f, EnumType* e, const string& module) BroDocObj bdo(dummy_id, r, true); bdo.WriteReST(f); + */ } static bool ComponentsMatch(const plugin::Plugin* p, plugin::component::Type t, diff --git a/src/Brofiler.h b/src/Brofiler.h index 22e5808bf6..88ce434070 100644 --- a/src/Brofiler.h +++ b/src/Brofiler.h @@ -78,4 +78,6 @@ private: }; }; +extern Brofiler brofiler; + #endif /* BROFILER_H_ */ diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index c881cc4df1..4145984f3b 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -370,6 +370,8 @@ set(bro_SRCS plugin/Plugin.cc plugin/Macros.h + broxygen/Manager.cc + nb_dns.c digest.h ) diff --git a/src/Expr.cc b/src/Expr.cc index 0eaa7ce918..303b3cf5c7 100644 --- a/src/Expr.cc +++ b/src/Expr.cc @@ -5111,6 +5111,7 @@ BroType* ListExpr::InitType() const types->append(td); } + return new RecordType(types); } diff --git a/src/ID.cc b/src/ID.cc index a6e592146b..ed561016ff 100644 --- a/src/ID.cc +++ b/src/ID.cc @@ -619,6 +619,7 @@ void ID::DescribeExtended(ODesc* d) const void ID::DescribeReSTShort(ODesc* d) const { + /* TODO if ( is_type ) d->Add(":bro:type:`"); else @@ -668,6 +669,7 @@ void ID::DescribeReSTShort(ODesc* d) const d->SP(); attrs->DescribeReST(d); } + */ } void ID::DescribeReST(ODesc* d, bool is_role) const @@ -697,10 +699,10 @@ void ID::DescribeReST(ODesc* d, bool is_role) const { d->Add(":Type: "); - if ( ! is_type && type->GetTypeID() ) + if ( ! is_type && ! type->GetName().empty() ) { d->Add(":bro:type:`"); - d->Add(type->GetTypeID()); + d->Add(type->GetName()); d->Add("`"); } else diff --git a/src/Serializer.h b/src/Serializer.h index 72e0723880..c1af1a2c4f 100644 --- a/src/Serializer.h +++ b/src/Serializer.h @@ -125,7 +125,7 @@ protected: // This will be increased whenever there is an incompatible change // in the data format. - static const uint32 DATA_FORMAT_VERSION = 23; + static const uint32 DATA_FORMAT_VERSION = 24; ChunkedIO* io; diff --git a/src/Type.cc b/src/Type.cc index 2b9faa8018..a659dc473b 100644 --- a/src/Type.cc +++ b/src/Type.cc @@ -8,13 +8,12 @@ #include "Scope.h" #include "Serializer.h" #include "Reporter.h" +#include "broxygen/Manager.h" #include #include #include -extern int generate_documentation; - // Note: This function must be thread-safe. const char* type_name(TypeTag t) { @@ -47,7 +46,6 @@ BroType::BroType(TypeTag t, bool arg_base_type) tag = t; is_network_order = 0; base_type = arg_base_type; - type_id = 0; switch ( tag ) { case TYPE_VOID: @@ -110,10 +108,26 @@ BroType::BroType(TypeTag t, bool arg_base_type) } -BroType::~BroType() +BroType* BroType::Clone() const { - if ( type_id ) - delete [] type_id; + SerializationFormat* form = new BinarySerializationFormat(); + form->StartWrite(); + CloneSerializer ss(form); + SerialInfo sinfo(&ss); + sinfo.cache = false; + + this->Serialize(&sinfo); + char* data; + uint32 len = form->EndWrite(&data); + form->StartRead(data, len); + + UnserialInfo uinfo(&ss); + uinfo.cache = false; + BroType* rval = this->Unserialize(&uinfo); + + delete [] data; + + return rval; } int BroType::MatchesIndex(ListExpr*& index) const @@ -222,9 +236,21 @@ BroType* BroType::Unserialize(UnserialInfo* info, TypeTag want) if ( ! t ) return 0; - // For base types, we return our current instance - // if not in "documentation mode". - if ( t->base_type && ! generate_documentation ) + if ( ! t->name.empty() ) + { + // Avoid creating a new type if it's known by name. + // Also avoids loss of base type name alias (from condition below). + ID* id = global_scope()->Lookup(t->name.c_str()); + BroType* t2 = id ? id->AsType() : 0; + + if ( t2 ) + { + Unref(t); + return t2->Ref(); + } + } + + if ( t->base_type ) { BroType* t2 = ::base_type(TypeTag(t->tag)); Unref(t); @@ -247,21 +273,10 @@ bool BroType::DoSerialize(SerialInfo* info) const if ( ! (SERIALIZE(char(tag)) && SERIALIZE(char(internal_tag))) ) return false; - if ( ! (SERIALIZE(is_network_order) && SERIALIZE(base_type) && - // Serialize the former "bool is_global_attributes_type" for - // backwards compatibility. - SERIALIZE(false)) ) + if ( ! (SERIALIZE(is_network_order) && SERIALIZE(base_type)) ) return false; - // Likewise, serialize the former optional "RecordType* attributes_type" - // for backwards compatibility. - void* null = NULL; - SERIALIZE(null); - - if ( generate_documentation ) - { - SERIALIZE_OPTIONAL_STR(type_id); - } + SERIALIZE_STR(name.c_str(), name.size()); info->s->WriteCloseTag("Type"); @@ -279,24 +294,15 @@ bool BroType::DoUnserialize(UnserialInfo* info) tag = (TypeTag) c1; internal_tag = (InternalTypeTag) c2; - bool not_used; - - if ( ! (UNSERIALIZE(&is_network_order) && UNSERIALIZE(&base_type) - // Unerialize the former "bool is_global_attributes_type" for - // backwards compatibility. - && UNSERIALIZE(¬_used)) ) + if ( ! (UNSERIALIZE(&is_network_order) && UNSERIALIZE(&base_type)) ) return 0; - BroType* not_used_either; + const char* n; + if ( ! UNSERIALIZE_STR(&n, 0) ) + return false; - // Likewise, unserialize the former optional "RecordType* - // attributes_type" for backwards compatibility. - UNSERIALIZE_OPTIONAL(not_used_either, BroType::Unserialize(info, TYPE_RECORD)); - - if ( generate_documentation ) - { - UNSERIALIZE_OPTIONAL_STR(type_id); - } + name = n; + delete [] n; return true; } @@ -470,10 +476,10 @@ void IndexType::DescribeReST(ODesc* d) const const BroType* t = (*IndexTypes())[i]; - if ( t->GetTypeID() ) + if ( ! t->GetName().empty() ) { d->Add(":bro:type:`"); - d->Add(t->GetTypeID()); + d->Add(t->GetName()); d->Add("`"); } else @@ -486,10 +492,10 @@ void IndexType::DescribeReST(ODesc* d) const { d->Add(" of "); - if ( yield_type->GetTypeID() ) + if ( ! yield_type->GetName().empty() ) { d->Add(":bro:type:`"); - d->Add(yield_type->GetTypeID()); + d->Add(yield_type->GetName()); d->Add("`"); } else @@ -781,10 +787,10 @@ void FuncType::DescribeReST(ODesc* d) const { d->AddSP(" :"); - if ( yield->GetTypeID() ) + if ( ! yield->GetName().empty() ) { d->Add(":bro:type:`"); - d->Add(yield->GetTypeID()); + d->Add(yield->GetName()); d->Add("`"); } else @@ -873,6 +879,17 @@ TypeDecl::TypeDecl(BroType* t, const char* i, attr_list* arg_attrs, bool in_reco id = i; } +TypeDecl::TypeDecl(const TypeDecl& other) + { + type = other.type->Ref(); + attrs = other.attrs; + + if ( attrs ) + ::Ref(attrs); + + id = copy_string(other.id); + } + TypeDecl::~TypeDecl() { Unref(type); @@ -914,10 +931,10 @@ void TypeDecl::DescribeReST(ODesc* d) const d->Add(id); d->Add(": "); - if ( type->GetTypeID() ) + if ( ! type->GetName().empty() ) { d->Add(":bro:type:`"); - d->Add(type->GetTypeID()); + d->Add(type->GetName()); d->Add("`"); } else @@ -930,37 +947,6 @@ void TypeDecl::DescribeReST(ODesc* d) const } } -CommentedTypeDecl::CommentedTypeDecl(BroType* t, const char* i, - attr_list* attrs, bool in_record, std::list* cmnt_list) - : TypeDecl(t, i, attrs, in_record) - { - comments = cmnt_list; - } - -CommentedTypeDecl::~CommentedTypeDecl() - { - if ( comments ) delete comments; - } - -void CommentedTypeDecl::DescribeReST(ODesc* d) const - { - TypeDecl::DescribeReST(d); - - if ( comments ) - { - d->PushIndent(); - std::list::const_iterator i; - - for ( i = comments->begin(); i != comments->end(); ++i) - { - if ( i != comments->begin() ) d->NL(); - d->Add(i->c_str()); - } - - d->PopIndentNoNL(); - } - } - RecordType::RecordType(type_decl_list* arg_types) : BroType(TYPE_RECORD) { types = arg_types; @@ -1328,38 +1314,12 @@ bool OpaqueType::DoUnserialize(UnserialInfo* info) return true; } -EnumType::EnumType(const string& arg_name) -: BroType(TYPE_ENUM) - { - name = arg_name; - counter = 0; - } - -EnumType::EnumType(EnumType* e) -: BroType(TYPE_ENUM) - { - name = e->name; - counter = e->counter; - - for ( NameMap::iterator it = e->names.begin(); it != e->names.end(); ++it ) - names[copy_string(it->first)] = it->second; - } - EnumType::~EnumType() { for ( NameMap::iterator iter = names.begin(); iter != names.end(); ++iter ) delete [] iter->first; } -CommentedEnumType::~CommentedEnumType() - { - for ( CommentMap::iterator iter = comments.begin(); iter != comments.end(); ++iter ) - { - delete [] iter->first; - delete iter->second; - } - } - // Note, we use reporter->Error() here (not Error()) to include the current script // location in the error message, rather than the one where the type was // originally defined. @@ -1372,7 +1332,7 @@ void EnumType::AddName(const string& module_name, const char* name, bool is_expo SetError(); return; } - AddNameInternal(module_name, name, counter, is_export); + CheckAndAddName(module_name, name, counter, is_export); counter++; } @@ -1386,32 +1346,12 @@ void EnumType::AddName(const string& module_name, const char* name, bro_int_t va return; } counter = -1; - AddNameInternal(module_name, name, val, is_export); + CheckAndAddName(module_name, name, val, is_export); } -void CommentedEnumType::AddComment(const string& module_name, const char* name, - std::list* new_comments) +void EnumType::CheckAndAddName(const string& module_name, const char* name, + bro_int_t val, bool is_export) { - if ( ! new_comments ) - return; - - string fullname = make_full_var_name(module_name.c_str(), name); - - CommentMap::iterator it = comments.find(fullname.c_str()); - - if ( it == comments.end() ) - comments[copy_string(fullname.c_str())] = new_comments; - else - { - list* prev_comments = comments[fullname.c_str()]; - prev_comments->splice(prev_comments->end(), *new_comments); - delete new_comments; - } - } - -void EnumType::AddNameInternal(const string& module_name, const char* name, bro_int_t val, bool is_export) - { - ID *id; if ( Lookup(val) ) { reporter->Error("enumerator value in enumerated type definition already exists"); @@ -1419,12 +1359,14 @@ void EnumType::AddNameInternal(const string& module_name, const char* name, bro_ return; } - id = lookup_ID(name, module_name.c_str()); + ID* id = lookup_ID(name, module_name.c_str()); + if ( ! id ) { id = install_ID(name, module_name.c_str(), true, is_export); id->SetType(this->Ref()); id->SetEnumConst(); + broxygen_mgr->Identifier(id); } else { @@ -1433,11 +1375,19 @@ void EnumType::AddNameInternal(const string& module_name, const char* name, bro_ return; } - string fullname = make_full_var_name(module_name.c_str(), name); - names[copy_string(fullname.c_str())] = val; + AddNameInternal(module_name, name, val, is_export); + + set types = type_aliases[GetName()]; + set::const_iterator it; + + for ( it = types.begin(); it != types.end(); ++it ) + if ( *it != this ) + (*it)->AsEnumType()->AddNameInternal(module_name, name, val, + is_export); } -void CommentedEnumType::AddNameInternal(const string& module_name, const char* name, bro_int_t val, bool is_export) +void EnumType::AddNameInternal(const string& module_name, const char* name, + bro_int_t val, bool is_export) { string fullname = make_full_var_name(module_name.c_str(), name); names[copy_string(fullname.c_str())] = val; @@ -1466,54 +1416,8 @@ const char* EnumType::Lookup(bro_int_t value) void EnumType::DescribeReST(ODesc* d) const { - d->Add(":bro:type:`"); - d->Add(name.c_str()); - d->Add("`"); - } - -void CommentedEnumType::DescribeReST(ODesc* d) const - { - // create temporary, reverse name map so that enums can be documented - // in ascending order of their actual integral value instead of by name - typedef std::map< bro_int_t, const char* > RevNameMap; - RevNameMap rev; - for ( NameMap::const_iterator it = names.begin(); it != names.end(); ++it ) - rev[it->second] = it->first; - - d->Add(":bro:type:`"); - d->Add(type_name(Tag())); - d->Add("`"); - d->PushIndent(); - d->NL(); - - for ( RevNameMap::const_iterator it = rev.begin(); it != rev.end(); ++it ) - { - if ( it != rev.begin() ) - { - d->NL(); - d->NL(); - } - - d->Add(".. bro:enum:: "); - d->AddSP(it->second); - d->Add(GetTypeID()); - - CommentMap::const_iterator cmnt_it = comments.find(it->second); - if ( cmnt_it != comments.end() ) - { - d->PushIndent(); - d->NL(); - std::list::const_iterator i; - const std::list* cmnt_list = cmnt_it->second; - for ( i = cmnt_list->begin(); i != cmnt_list->end(); ++i) - { - if ( i != cmnt_list->begin() ) d->NL(); - d->Add(i->c_str()); - } - d->PopIndentNoNL(); - } - } - d->PopIndentNoNL(); + // TODO: this probably goes away + d->Add(":bro:type:`enum`"); } IMPLEMENT_SERIAL(EnumType, SER_ENUM_TYPE); diff --git a/src/Type.h b/src/Type.h index a6163d5152..be27426546 100644 --- a/src/Type.h +++ b/src/Type.h @@ -4,7 +4,7 @@ #define type_h #include -#include +#include #include #include "Obj.h" @@ -73,7 +73,9 @@ const int MATCHES_INDEX_VECTOR = 2; class BroType : public BroObj { public: BroType(TypeTag tag, bool base_type = false); - ~BroType(); + ~BroType() { } + + BroType* Clone() const; TypeTag Tag() const { return tag; } InternalTypeTag InternalType() const { return internal_tag; } @@ -232,11 +234,11 @@ public: bool Serialize(SerialInfo* info) const; static BroType* Unserialize(UnserialInfo* info, TypeTag want = TYPE_ANY); - void SetTypeID(const char* id) { type_id = id; } - const char* GetTypeID() const { return type_id; } + void SetName(const string& arg_name) { name = arg_name; } + string GetName() const { return name; } protected: - BroType() { type_id = 0; } + BroType() { } void SetError(); @@ -247,10 +249,7 @@ private: InternalTypeTag internal_tag; bool is_network_order; bool base_type; - - // This type_id field is only used by the documentation framework to - // track the names of declared types. - const char* type_id; + string name; }; class TypeList : public BroType { @@ -408,6 +407,7 @@ protected: class TypeDecl { public: TypeDecl(BroType* t, const char* i, attr_list* attrs = 0, bool in_record = false); + TypeDecl(const TypeDecl& other); virtual ~TypeDecl(); const Attr* FindAttr(attr_tag a) const @@ -423,17 +423,6 @@ public: const char* id; }; -class CommentedTypeDecl : public TypeDecl { -public: - CommentedTypeDecl(BroType* t, const char* i, attr_list* attrs = 0, - bool in_record = false, std::list* cmnt_list = 0); - virtual ~CommentedTypeDecl(); - - void DescribeReST(ODesc* d) const; - - std::list* comments; -}; - class RecordType : public BroType { public: RecordType(type_decl_list* types); @@ -522,8 +511,7 @@ protected: class EnumType : public BroType { public: - EnumType(const string& arg_name); - EnumType(EnumType* e); + EnumType() : BroType(TYPE_ENUM) { counter = 0; } ~EnumType(); // The value of this name is next internal counter value, starting @@ -539,17 +527,17 @@ public: bro_int_t Lookup(const string& module_name, const char* name); const char* Lookup(bro_int_t value); // Returns 0 if not found - string Name() const { return name; } - void DescribeReST(ODesc* d) const; protected: - EnumType() { counter = 0; } DECLARE_SERIAL(EnumType) - virtual void AddNameInternal(const string& module_name, + void AddNameInternal(const string& module_name, const char* name, bro_int_t val, bool is_export); + void CheckAndAddName(const string& module_name, + const char* name, bro_int_t val, bool is_export); + typedef std::map< const char*, bro_int_t, ltstr > NameMap; NameMap names; @@ -560,31 +548,6 @@ protected: // as a flag to prevent mixing of auto-increment and explicit // enumerator specifications. bro_int_t counter; - - // The name of the enum type is stored for documentation purposes. - string name; -}; - -class CommentedEnumType: public EnumType { -public: - CommentedEnumType(const string& arg_name) : EnumType(arg_name) {} - CommentedEnumType(EnumType* e) : EnumType(e) {} - ~CommentedEnumType(); - - void DescribeReST(ODesc* d) const; - void AddComment(const string& module_name, const char* name, - std::list* comments); - -protected: - // This overriden method does not install the given ID name into a - // scope and it also does not do any kind of checking that the - // provided name already exists. - void AddNameInternal(const string& module_name, const char* name, - bro_int_t val, bool is_export); - - // Comments are only filled when in "documentation mode". - typedef std::map< const char*, std::list*, ltstr > CommentMap; - CommentMap comments; }; class VectorType : public BroType { @@ -609,6 +572,9 @@ protected: BroType* yield_type; }; +typedef std::map > TypeAliasMap; +extern TypeAliasMap type_aliases; + extern OpaqueType* md5_type; extern OpaqueType* sha1_type; extern OpaqueType* sha256_type; diff --git a/src/Var.cc b/src/Var.cc index d384fedc74..821c9e207b 100644 --- a/src/Var.cc +++ b/src/Var.cc @@ -10,8 +10,6 @@ #include "RemoteSerializer.h" #include "EventRegistry.h" -extern int generate_documentation; - static Val* init_val(Expr* init, const BroType* t, Val* aggr) { return init->InitVal(t, aggr); @@ -261,61 +259,26 @@ extern Expr* add_and_assign_local(ID* id, Expr* init, Val* val) return new AssignExpr(new NameExpr(id), init, 0, val); } -void add_type(ID* id, BroType* t, attr_list* attr, int /* is_event */) +void add_type(ID* id, BroType* t, attr_list* attr) { - BroType* tnew = t; + string new_type_name(id->Name()); + string old_type_name(t->GetName()); + BroType* tnew = 0; - // In "documentation mode", we'd like to to be able to associate - // an identifier name with a declared type. Dealing with declared - // types that are "aliases" to a builtin type requires that the BroType - // is cloned before setting the identifier name that resolves to it. - // And still this is not enough to document cases where the declared type - // is an alias for another declared type -- but that's not a natural/common - // practice. If documenting that corner case is desired, one way - // is to add an ID* to class ID that tracks aliases and set it here if - // t->GetTypeID() is true. - if ( generate_documentation ) - { - switch ( t->Tag() ) { - // Only "shallow" copy types that may contain records because - // we want to be able to see additions to the original record type's - // list of fields - case TYPE_RECORD: - tnew = new RecordType(t->AsRecordType()->Types()); - break; - case TYPE_TABLE: - tnew = new TableType(t->AsTableType()->Indices(), - t->AsTableType()->YieldType()); - break; - case TYPE_VECTOR: - tnew = new VectorType(t->AsVectorType()->YieldType()); - break; - case TYPE_FUNC: - tnew = new FuncType(t->AsFuncType()->Args(), - t->AsFuncType()->YieldType(), - t->AsFuncType()->Flavor()); - break; - default: - SerializationFormat* form = new BinarySerializationFormat(); - form->StartWrite(); - CloneSerializer ss(form); - SerialInfo sinfo(&ss); - sinfo.cache = false; + if ( (t->Tag() == TYPE_RECORD || t->Tag() == TYPE_ENUM) && + ! old_type_name.empty() ) + // Clone the type to preserve type name aliasing. + tnew = t->Clone(); + else + // An extensible types (record/enum) being declared for first time. + tnew = t; - t->Serialize(&sinfo); - char* data; - uint32 len = form->EndWrite(&data); - form->StartRead(data, len); + type_aliases[new_type_name].insert(tnew); - UnserialInfo uinfo(&ss); - uinfo.cache = false; - tnew = t->Unserialize(&uinfo); + if ( new_type_name != old_type_name && ! old_type_name.empty() ) + type_aliases[old_type_name].insert(tnew); - delete [] data; - } - - tnew->SetTypeID(copy_string(id->Name())); - } + tnew->SetName(id->Name()); id->SetType(tnew); id->MakeType(); diff --git a/src/Var.h b/src/Var.h index 8b9866ed2d..bcdd45dad2 100644 --- a/src/Var.h +++ b/src/Var.h @@ -18,7 +18,7 @@ extern Stmt* add_local(ID* id, BroType* t, init_class c, Expr* init, attr_list* attr, decl_type dt); extern Expr* add_and_assign_local(ID* id, Expr* init, Val* val = 0); -extern void add_type(ID* id, BroType* t, attr_list* attr, int is_event); +extern void add_type(ID* id, BroType* t, attr_list* attr); extern void begin_func(ID* id, const char* module_name, function_flavor flavor, int is_redef, FuncType* t); diff --git a/src/broxygen/Manager.cc b/src/broxygen/Manager.cc new file mode 100644 index 0000000000..30c9008787 --- /dev/null +++ b/src/broxygen/Manager.cc @@ -0,0 +1,88 @@ +#include "Manager.h" + +using namespace broxygen; + +Manager::Manager(const std::string& config) + { + // TODO + } + +void Manager::GenerateDocs() const + { + // TODO + + // may be a no-op if no config + + // does the old canon_doc_func_param stuff happen here now on the fly + // for functions we're about to document? + } + +void Manager::File(const std::string& path) + { + // TODO + // determine bropath subpath + // can be a file or directory? + } + +void Manager::ScriptDependency(const std::string& path, const std::string& dep) + { + // TODO: + // need anything from BroDoc::AddImport? + // warn about unconsumed comments (and discard any) + } + +void Manager::ModuleUsage(const std::string& path, const std::string& module) + { + // TODO lookup script and add module to a set + } + +void Manager::Identifier(const ID *id) + { + // TODO: lookup script to associate w/ by GetLocationInfo()->filename + // do different things depending on Type? (eg function flavor versus state) + // do different things based on redef attr + const ? + // consume any buffered comments and associate w/ id + // deal w/ type aliasing + // special enum or record handing? + // if it's a function we may already have added it (decl versus impl) + } + +void Manager::RecordField(const ID *id, const TypeDecl *field, + const std::string& path) + { + // TODO: consume comments + // redef is implicit -- script path of field will differ from ID/type's + } + +void Manager::Redef(const ID* id, const string& path) + { + // TODO: lookup script w/ 'path' to associate the id in as redef'd + // consume any buffered comments and associate w/ redef'd id + // can sort notices here + } + +void Manager::SummaryComment(const std::string& script, + const std::string& comment) + { + // TODO + // canon_doc_comment ? + } + +void Manager::PreComment(const std::string& comment) + { + // TODO + // canon_doc_comment + } + +void Manager::PostComment(const std::string& comment) + { + // TODO this gets associated with the last thing registered + // canon_doc_comment + } + + +// TODO: "canon_doc_comment" means treat "##Text" and "## Text" the same +// so that a single space doesn't generate an indentation level. + + +// TODO: creating proto/file analyzer docs diff --git a/src/broxygen/Manager.h b/src/broxygen/Manager.h new file mode 100644 index 0000000000..d0665c307d --- /dev/null +++ b/src/broxygen/Manager.h @@ -0,0 +1,43 @@ +#ifndef BROXYGEN_MANAGER_H +#define BROXYGEN_MANAGER_H + +#include + +#include "ID.h" +#include "Type.h" + +namespace broxygen { + +class Manager { + +public: + + Manager(const std::string& config); + + void GenerateDocs() const; + + void File(const std::string& path); + + void ScriptDependency(const std::string& path, const std::string& dep); + + void ModuleUsage(const std::string& path, const std::string& module); + + void Identifier(const ID* id); + + void RecordField(const ID* id, const TypeDecl* field, + const std::string& path); + + void Redef(const ID* id, const std::string& path); + + void SummaryComment(const std::string& path, const std::string& comment); + + void PreComment(const std::string& comment); + + void PostComment(const std::string& comment); +}; + +} // namespace broxygen + +extern broxygen::Manager* broxygen_mgr; + +#endif diff --git a/src/main.cc b/src/main.cc index 313e1a40b0..bdb6499445 100644 --- a/src/main.cc +++ b/src/main.cc @@ -61,8 +61,8 @@ extern "C" void OPENSSL_add_all_algorithms_conf(void); #include "analyzer/Manager.h" #include "analyzer/Tag.h" #include "plugin/Manager.h" - #include "file_analysis/Manager.h" +#include "broxygen/Manager.h" #include "binpac_bro.h" @@ -100,6 +100,7 @@ input::Manager* input_mgr = 0; plugin::Manager* plugin_mgr = 0; analyzer::Manager* analyzer_mgr = 0; file_analysis::Manager* file_mgr = 0; +broxygen::Manager* broxygen_mgr = 0; Stmt* stmts; EventHandlerPtr net_done = 0; RuleMatcher* rule_matcher = 0; @@ -116,7 +117,6 @@ int signal_val = 0; int optimize = 0; int do_notice_analysis = 0; int rule_bench = 0; -int generate_documentation = 0; SecondaryPath* secondary_path = 0; extern char version[]; char* command_line_policy = 0; @@ -124,6 +124,8 @@ vector params; char* proc_status_file = 0; int snaplen = 0; // this gets set from the scripting-layer's value +TypeAliasMap type_aliases; + OpaqueType* md5_type = 0; OpaqueType* sha1_type = 0; OpaqueType* sha256_type = 0; @@ -132,8 +134,6 @@ OpaqueType* cardinality_type = 0; OpaqueType* topk_type = 0; OpaqueType* bloomfilter_type = 0; -extern std::list docs_generated; - // Keep copy of command line int bro_argc; char** bro_argv; @@ -203,7 +203,7 @@ void usage() fprintf(stderr, " -T|--re-level | set 'RE_level' for rules\n"); fprintf(stderr, " -U|--status-file | Record process status in file\n"); fprintf(stderr, " -W|--watchdog | activate watchdog timer\n"); - fprintf(stderr, " -Z|--doc-scripts | generate documentation for all loaded scripts\n"); + fprintf(stderr, " -X|--broxygen | generate documentation based on config file\n"); #ifdef USE_PERFTOOLS_DEBUG fprintf(stderr, " -m|--mem-leaks | show leaks [perftools]\n"); @@ -373,6 +373,7 @@ void terminate_bro() plugin_mgr->FinishPlugins(); + delete broxygen_mgr; delete timer_mgr; delete dns_mgr; delete persistence_serializer; @@ -473,7 +474,7 @@ int main(int argc, char** argv) {"filter", required_argument, 0, 'f'}, {"help", no_argument, 0, 'h'}, {"iface", required_argument, 0, 'i'}, - {"doc-scripts", no_argument, 0, 'Z'}, + {"broxygen", required_argument, 0, 'X'}, {"prefix", required_argument, 0, 'p'}, {"readfile", required_argument, 0, 'r'}, {"flowfile", required_argument, 0, 'y'}, @@ -532,7 +533,7 @@ int main(int argc, char** argv) if ( p ) add_to_name_list(p, ':', prefixes); - string active_file; + string broxygen_config; #ifdef USE_IDMEF string libidmef_dtd_path = "idmef-message.dtd"; @@ -545,7 +546,7 @@ int main(int argc, char** argv) opterr = 0; char opts[256]; - safe_strncpy(opts, "B:D:e:f:I:i:K:l:n:p:R:r:s:T:t:U:w:x:X:y:Y:z:CFGLNOPSWbdghvZ", + safe_strncpy(opts, "B:D:e:f:I:i:K:l:n:p:R:r:s:T:t:U:w:x:X:y:Y:z:CFGLNOPSWbdghv", sizeof(opts)); #ifdef USE_PERFTOOLS_DEBUG @@ -727,8 +728,8 @@ int main(int argc, char** argv) break; #endif - case 'Z': - generate_documentation = 1; + case 'X': + broxygen_config = optarg; break; #ifdef USE_IDMEF @@ -760,6 +761,7 @@ int main(int argc, char** argv) reporter = new Reporter(); thread_mgr = new threading::Manager(); + broxygen_mgr = new broxygen::Manager(broxygen_config); #ifdef DEBUG if ( debug_streams ) @@ -888,23 +890,6 @@ int main(int argc, char** argv) } #endif - if ( generate_documentation ) - { - CreateProtoAnalyzerDoc("proto-analyzers.rst"); - CreateFileAnalyzerDoc("file-analyzers.rst"); - - std::list::iterator it; - - for ( it = docs_generated.begin(); it != docs_generated.end(); ++it ) - (*it)->WriteDocFile(); - - for ( it = docs_generated.begin(); it != docs_generated.end(); ++it ) - delete *it; - - terminate_bro(); - return 0; - } - if ( reporter->Errors() > 0 ) { delete dns_mgr; @@ -915,6 +900,8 @@ int main(int argc, char** argv) init_general_global_var(); + broxygen_mgr->GenerateDocs(); + if ( user_pcap_filter ) { ID* id = global_scope()->Lookup("cmd_line_bpf_filter"); diff --git a/src/parse.y b/src/parse.y index 98df0de2a3..ed79808c78 100644 --- a/src/parse.y +++ b/src/parse.y @@ -2,7 +2,7 @@ // See the file "COPYING" in the main distribution directory for copyright. %} -%expect 85 +%expect 75 %token TOK_ADD TOK_ADD_TO TOK_ADDR TOK_ANY %token TOK_ATENDIF TOK_ATELSE TOK_ATIF TOK_ATIFDEF TOK_ATIFNDEF @@ -28,8 +28,6 @@ %token TOK_DEBUG -%token TOK_DOC TOK_POST_DOC - %token TOK_NO_TEST %nonassoc TOK_HOOK @@ -47,8 +45,7 @@ %left '$' '[' ']' '(' ')' TOK_HAS_FIELD TOK_HAS_ATTR %type opt_no_test opt_no_test_block -%type TOK_ID TOK_PATTERN_TEXT single_pattern TOK_DOC TOK_POST_DOC -%type opt_doc_list opt_post_doc_list +%type TOK_ID TOK_PATTERN_TEXT single_pattern %type local_id global_id def_global_id event_id global_or_event_id resolve_id begin_func %type local_id_list %type init_class @@ -83,17 +80,13 @@ #include "RE.h" #include "Scope.h" #include "Reporter.h" -#include "BroDoc.h" -#include "BroDocObj.h" #include "Brofiler.h" +#include "broxygen/Manager.h" -#include +#include #include -extern Brofiler brofiler; -extern BroDoc* current_reST_doc; -extern int generate_documentation; -extern std::list* reST_doc_comments; +extern const char* filename; // Absolute path of file currently being parsed. YYLTYPE GetCurrentLocation(); extern int yyerror(const char[]); @@ -127,24 +120,13 @@ bool defining_global_ID = false; ID* func_id = 0; EnumType *cur_enum_type = 0; -CommentedEnumType *cur_enum_type_doc = 0; -const char* cur_enum_elem_id = 0; - -type_decl_list* fake_type_decl_list = 0; -TypeDecl* last_fake_type_decl = 0; - static ID* cur_decl_type_id = 0; static void parser_new_enum (void) { /* Starting a new enum definition. */ assert(cur_enum_type == NULL); - cur_enum_type = new EnumType(cur_decl_type_id->Name()); - - // For documentation purposes, a separate type object is created - // in order to avoid overlap that can be caused by redefs. - if ( generate_documentation ) - cur_enum_type_doc = new CommentedEnumType(cur_decl_type_id->Name()); + cur_enum_type = new EnumType(); } static void parser_redef_enum (ID *id) @@ -160,53 +142,75 @@ static void parser_redef_enum (ID *id) if ( ! cur_enum_type ) id->Error("not an enum"); } - - if ( generate_documentation ) - cur_enum_type_doc = new CommentedEnumType(id->Name()); } -static void add_enum_comment (std::list* comments) +static type_decl_list* copy_type_decl_list(type_decl_list* tdl) { - cur_enum_type_doc->AddComment(current_module, cur_enum_elem_id, comments); - } + if ( ! tdl ) + return 0; -static ID* create_dummy_id (ID* id, BroType* type) - { - ID* fake_id = new ID(copy_string(id->Name()), (IDScope) id->Scope(), - is_export); + type_decl_list* rval = new type_decl_list(); - fake_id->SetType(type->Ref()); - - if ( id->AsType() ) + loop_over_list(*tdl, i) { - type->SetTypeID(copy_string(id->Name())); - fake_id->MakeType(); + TypeDecl* td = (*tdl)[i]; + rval->append(new TypeDecl(*td)); } - return fake_id; + return rval; } -static std::list* concat_opt_docs (std::list* pre, - std::list* post) +static attr_list* copy_attr_list(attr_list* al) { - if ( ! pre && ! post ) return 0; + if ( ! al ) + return 0; - if ( pre && ! post ) return pre; + attr_list* rval = new attr_list(); - if ( ! pre && post ) return post; + loop_over_list(*al, i) + { + Attr* a = (*al)[i]; + ::Ref(a); + rval->append(a); + } - pre->splice(pre->end(), *post); - delete post; - - return pre; + return rval; } +static void extend_record(ID* id, type_decl_list* fields, attr_list* attrs) + { + set types = type_aliases[id->Name()]; + + if ( types.empty() ) + { + id->Error("failed to redef record: no types found in alias map"); + return; + } + + for ( set::const_iterator it = types.begin(); it != types.end(); ) + { + RecordType* add_to = (*it)->AsRecordType(); + const char* error = 0; + ++it; + + if ( it == types.end() ) + error = add_to->AddFields(fields, attrs); + else + error = add_to->AddFields(copy_type_decl_list(fields), + copy_attr_list(attrs)); + + if ( error ) + { + id->Error(error); + break; + } + } + } %} %union { bool b; char* str; - std::list* str_l; ID* id; id_list* id_l; init_class ic; @@ -699,46 +703,24 @@ single_pattern: ; enum_body: - enum_body_list opt_post_doc_list + enum_body_list { $$ = cur_enum_type; - - if ( generate_documentation ) - { - add_enum_comment($2); - cur_enum_elem_id = 0; - } - cur_enum_type = NULL; } - | enum_body_list ',' opt_post_doc_list + | enum_body_list ',' { $$ = cur_enum_type; - - if ( generate_documentation ) - { - add_enum_comment($3); - cur_enum_elem_id = 0; - } - cur_enum_type = NULL; } ; enum_body_list: - enum_body_elem opt_post_doc_list - { - if ( generate_documentation ) - add_enum_comment($2); - } + enum_body_elem - | enum_body_list ',' opt_post_doc_list - { - if ( generate_documentation ) - add_enum_comment($3); - } enum_body_elem -; + | enum_body_list ',' enum_body_elem + ; enum_body_elem: /* TODO: We could also define this as TOK_ID '=' expr, (or @@ -746,25 +728,19 @@ enum_body_elem: error messages if someboy tries to use constant variables as enumerator. */ - opt_doc_list TOK_ID '=' TOK_CONSTANT + TOK_ID '=' TOK_CONSTANT { - set_location(@2, @4); + set_location(@1, @3); assert(cur_enum_type); - if ( $4->Type()->Tag() != TYPE_COUNT ) + if ( $3->Type()->Tag() != TYPE_COUNT ) reporter->Error("enumerator is not a count constant"); else - cur_enum_type->AddName(current_module, $2, $4->InternalUnsigned(), is_export); - - if ( generate_documentation ) - { - cur_enum_type_doc->AddName(current_module, $2, $4->InternalUnsigned(), is_export); - cur_enum_elem_id = $2; - add_enum_comment($1); - } + cur_enum_type->AddName(current_module, $1, + $3->InternalUnsigned(), is_export); } - | opt_doc_list TOK_ID '=' '-' TOK_CONSTANT + | TOK_ID '=' '-' TOK_CONSTANT { /* We only accept counts as enumerator, but we want to return a nice error message if users triy to use a negative integer (will also @@ -773,18 +749,11 @@ enum_body_elem: reporter->Error("enumerator is not a count constant"); } - | opt_doc_list TOK_ID + | TOK_ID { - set_location(@2); + set_location(@1); assert(cur_enum_type); - cur_enum_type->AddName(current_module, $2, is_export); - - if ( generate_documentation ) - { - cur_enum_type_doc->AddName(current_module, $2, is_export); - cur_enum_elem_id = $2; - add_enum_comment($1); - } + cur_enum_type->AddName(current_module, $1, is_export); } ; @@ -872,12 +841,11 @@ type: } | TOK_RECORD '{' - { ++in_record; do_doc_token_start(); } + { ++in_record; } type_decl_list { --in_record; } '}' { - do_doc_token_stop(); set_location(@1, @5); $$ = new RecordType($4); } @@ -889,9 +857,8 @@ type: $$ = 0; } - | TOK_ENUM '{' { set_location(@1); parser_new_enum(); do_doc_token_start(); } enum_body '}' + | TOK_ENUM '{' { set_location(@1); parser_new_enum(); } enum_body '}' { - do_doc_token_stop(); set_location(@1, @5); $4->UpdateLocationEndInfo(@5); $$ = $4; @@ -983,45 +950,21 @@ type_decl_list: type_decl_list type_decl { $1->append($2); - - if ( generate_documentation && last_fake_type_decl ) - { - fake_type_decl_list->append(last_fake_type_decl); - last_fake_type_decl = 0; - } } | { $$ = new type_decl_list(); - - if ( generate_documentation ) - fake_type_decl_list = new type_decl_list(); } ; type_decl: - opt_doc_list TOK_ID ':' type opt_attr ';' opt_post_doc_list + TOK_ID ':' type opt_attr ';' { - set_location(@2, @6); + set_location(@1, @4); + $$ = new TypeDecl($3, $1, $4, (in_record > 0)); - if ( generate_documentation ) - { - // TypeDecl ctor deletes the attr list, so make a copy - attr_list* a = $5; - attr_list* a_copy = 0; - - if ( a ) - { - a_copy = new attr_list; - loop_over_list(*a, i) - a_copy->append((*a)[i]); - } - - last_fake_type_decl = new CommentedTypeDecl( - $4, $2, a_copy, (in_record > 0), concat_opt_docs($1, $7)); - } - - $$ = new TypeDecl($4, $2, $5, (in_record > 0)); + if ( in_record > 0 ) + broxygen_mgr->RecordField(cur_decl_type_id, $$, ::filename); } ; @@ -1055,9 +998,7 @@ decl: TOK_MODULE TOK_ID ';' { current_module = $2; - - if ( generate_documentation ) - current_reST_doc->AddModule(current_module); + broxygen_mgr->ModuleUsage(::filename, current_module); } | TOK_EXPORT '{' { is_export = true; } decl_list '}' @@ -1066,171 +1007,43 @@ decl: | TOK_GLOBAL def_global_id opt_type init_class opt_init opt_attr ';' { add_global($2, $3, $4, $5, $6, VAR_REGULAR); - - if ( generate_documentation ) - { - ID* id = $2; - if ( id->Type()->Tag() == TYPE_FUNC ) - { - switch ( id->Type()->AsFuncType()->Flavor() ) { - - case FUNC_FLAVOR_FUNCTION: - current_reST_doc->AddFunction( - new BroDocObj(id, reST_doc_comments)); - break; - - case FUNC_FLAVOR_EVENT: - current_reST_doc->AddEvent( - new BroDocObj(id, reST_doc_comments)); - break; - - case FUNC_FLAVOR_HOOK: - current_reST_doc->AddHook( - new BroDocObj(id, reST_doc_comments)); - break; - - default: - reporter->InternalError("invalid function flavor"); - break; - } - } - - else - { - current_reST_doc->AddStateVar( - new BroDocObj(id, reST_doc_comments)); - } - } + broxygen_mgr->Identifier($2); } | TOK_CONST def_global_id opt_type init_class opt_init opt_attr ';' { add_global($2, $3, $4, $5, $6, VAR_CONST); - - if ( generate_documentation ) - { - if ( $2->FindAttr(ATTR_REDEF) ) - current_reST_doc->AddOption( - new BroDocObj($2, reST_doc_comments)); - else - current_reST_doc->AddConstant( - new BroDocObj($2, reST_doc_comments)); - } + broxygen_mgr->Identifier($2); } | TOK_REDEF global_id opt_type init_class opt_init opt_attr ';' { add_global($2, $3, $4, $5, $6, VAR_REDEF); - - if ( generate_documentation && - ! streq("capture_filters", $2->Name()) ) - { - ID* fake_id = create_dummy_id($2, $2->Type()); - BroDocObj* o = new BroDocObj(fake_id, reST_doc_comments, true); - o->SetRole(true); - current_reST_doc->AddRedef(o); - } + broxygen_mgr->Redef($2, ::filename); } | TOK_REDEF TOK_ENUM global_id TOK_ADD_TO - '{' { parser_redef_enum($3); do_doc_token_start(); } enum_body '}' ';' + '{' { parser_redef_enum($3); } enum_body '}' ';' { - do_doc_token_stop(); - - if ( generate_documentation ) - { - ID* fake_id = create_dummy_id($3, cur_enum_type_doc); - cur_enum_type_doc = 0; - BroDocObj* o = new BroDocObj(fake_id, reST_doc_comments, true); - o->SetRole(true); - - if ( extract_module_name(fake_id->Name()) == "Notice" && - extract_var_name(fake_id->Name()) == "Type" ) - current_reST_doc->AddNotice(o); - else - current_reST_doc->AddRedef(o); - } + // Broxygen already grabbed new enum IDs as the type created them. } - | TOK_REDEF TOK_RECORD global_id TOK_ADD_TO - '{' { ++in_record; do_doc_token_start(); } - type_decl_list - { --in_record; do_doc_token_stop(); } '}' opt_attr ';' + | TOK_REDEF TOK_RECORD global_id { cur_decl_type_id = $3; } TOK_ADD_TO + '{' { ++in_record; } type_decl_list { --in_record; } '}' opt_attr ';' { + cur_decl_type_id = 0; + if ( ! $3->Type() ) $3->Error("unknown identifier"); else - { - RecordType* add_to = $3->Type()->AsRecordType(); - if ( ! add_to ) - $3->Error("not a record type"); - else - { - const char* error = add_to->AddFields($7, $10); - if ( error ) - $3->Error(error); - else if ( generate_documentation ) - { - if ( fake_type_decl_list ) - { - BroType* fake_record = - new RecordType(fake_type_decl_list); - ID* fake = create_dummy_id($3, fake_record); - fake_type_decl_list = 0; - BroDocObj* o = - new BroDocObj(fake, reST_doc_comments, true); - o->SetRole(true); - current_reST_doc->AddRedef(o); - } - else - { - fprintf(stderr, "Warning: doc mode did not process " - "record extension for '%s', CommentedTypeDecl" - "list unavailable.\n", $3->Name()); - } - } - } - } + extend_record($3, $8, $11); } | TOK_TYPE global_id ':' { cur_decl_type_id = $2; } type opt_attr ';' { cur_decl_type_id = 0; - add_type($2, $5, $6, 0); - - if ( generate_documentation ) - { - TypeTag t = $2->AsType()->Tag(); - if ( t == TYPE_ENUM && cur_enum_type_doc ) - { - ID* fake = create_dummy_id($2, cur_enum_type_doc); - cur_enum_type_doc = 0; - current_reST_doc->AddType( - new BroDocObj(fake, reST_doc_comments, true)); - } - - else if ( t == TYPE_RECORD && fake_type_decl_list ) - { - BroType* fake_record = new RecordType(fake_type_decl_list); - ID* fake = create_dummy_id($2, fake_record); - fake_type_decl_list = 0; - current_reST_doc->AddType( - new BroDocObj(fake, reST_doc_comments, true)); - } - - else - current_reST_doc->AddType( - new BroDocObj($2, reST_doc_comments)); - } - } - - | TOK_EVENT event_id ':' type_list opt_attr ';' - { - add_type($2, $4, $5, 1); - - if ( generate_documentation ) - current_reST_doc->AddEvent( - new BroDocObj($2, reST_doc_comments)); + add_type($2, $5, $6); + broxygen_mgr->Identifier($2); } | func_hdr func_body @@ -1258,18 +1071,13 @@ func_hdr: begin_func($2, current_module.c_str(), FUNC_FLAVOR_FUNCTION, 0, $3); $$ = $3; - if ( generate_documentation ) - current_reST_doc->AddFunction( - new BroDocObj($2, reST_doc_comments)); + broxygen_mgr->Identifier($2); } | TOK_EVENT event_id func_params { begin_func($2, current_module.c_str(), FUNC_FLAVOR_EVENT, 0, $3); $$ = $3; - if ( generate_documentation ) - current_reST_doc->AddEventHandler( - new BroDocObj($2, reST_doc_comments)); } | TOK_HOOK def_global_id func_params { @@ -1278,9 +1086,6 @@ func_hdr: begin_func($2, current_module.c_str(), FUNC_FLAVOR_HOOK, 0, $3); $$ = $3; - if ( generate_documentation ) - current_reST_doc->AddHookHandler( - new BroDocObj($2, reST_doc_comments)); } | TOK_REDEF TOK_EVENT event_id func_params { @@ -1729,40 +1534,6 @@ resolve_id: } ; -opt_post_doc_list: - opt_post_doc_list TOK_POST_DOC - { - $1->push_back($2); - $$ = $1; - } - | - TOK_POST_DOC - { - $$ = new std::list(); - $$->push_back($1); - delete [] $1; - } - | - { $$ = 0; } - ; - -opt_doc_list: - opt_doc_list TOK_DOC - { - $1->push_back($2); - $$ = $1; - } - | - TOK_DOC - { - $$ = new std::list(); - $$->push_back($1); - delete [] $1; - } - | - { $$ = 0; } - ; - opt_no_test: TOK_NO_TEST { $$ = true; } @@ -1788,10 +1559,6 @@ int yyerror(const char msg[]) else sprintf(msgbuf, "%s, at or near \"%s\"", msg, last_tok); - if ( generate_documentation ) - strcat(msgbuf, "\nDocumentation mode is enabled: " - "remember to check syntax of ## style comments\n"); - if ( in_debug ) g_curr_debug_error = copy_string(msg); diff --git a/src/plugin/ComponentManager.h b/src/plugin/ComponentManager.h index 16f9d80743..7d90f93bf1 100644 --- a/src/plugin/ComponentManager.h +++ b/src/plugin/ComponentManager.h @@ -130,9 +130,9 @@ template ComponentManager::ComponentManager(const string& arg_module) : module(arg_module) { - tag_enum_type = new EnumType(module + "::Tag"); + tag_enum_type = new EnumType(); ::ID* id = install_ID("Tag", module.c_str(), true, true); - add_type(id, tag_enum_type, 0, 0); + add_type(id, tag_enum_type, 0); } template diff --git a/src/scan.l b/src/scan.l index 636ec5b251..b126b2ee1f 100644 --- a/src/scan.l +++ b/src/scan.l @@ -23,16 +23,15 @@ #include "Debug.h" #include "PolicyFile.h" #include "broparse.h" -#include "BroDoc.h" #include "Reporter.h" #include "RE.h" #include "Net.h" #include "analyzer/Analyzer.h" +#include "broxygen/Manager.h" extern YYLTYPE yylloc; // holds start line and column of token extern int print_loaded_scripts; -extern int generate_documentation; // Track the @if... depth. ptr_compat_int current_depth = 0; @@ -40,10 +39,7 @@ ptr_compat_int current_depth = 0; int_list if_stack; int line_number = 1; -const char* filename = 0; -BroDoc* current_reST_doc = 0; -static BroDoc* last_reST_doc = 0; -string current_scanned_file_path; +const char* filename = 0; // Absolute path of file currently being parsed. char last_tok[128]; @@ -56,41 +52,15 @@ char last_tok[128]; if ( ((result = fread(buf, 1, max_size, yyin)) == 0) && ferror(yyin) ) \ reporter->Error("read failed with \"%s\"", strerror(errno)); -// reST documents that we've created (or have at least opened so far). -std::list docs_generated; - -// reST comments (those starting with ##) seen so far. -std::list* reST_doc_comments = 0; - -// Print current contents of reST_doc_comments list to stderr. -void print_current_reST_doc_comments(); - -// Delete the reST_doc_comments list object. -void clear_reST_doc_comments(); - -// Adds changes to capture_filter to the current script's reST documentation. -static void check_capture_filter_changes(); - -static const char* canon_doc_comment(const char* comment) +static string get_dirname(const char* path) { - // "##Text" and "## Text" are treated the same in order to be able - // to still preserve indentation level, but not unintentionally - // signify an indentation level for all the text when using - // the "## Text" style. - return ( comment[0] == ' ' ) ? comment + 1 : comment; - } + if ( ! path ) + return ""; -static std::string canon_doc_func_param(const char* id_start) - { - std::string id_name(id_start, strcspn(id_start, ":")); - const char* comment = id_start + id_name.size() + 1; - std::string doc; - - if ( id_name == "Returns" ) - doc.append(":returns:").append(comment); - else - doc.append(":param ").append(id_name).append(":").append(comment); - return doc; + char* tmp = copy_string(path); + string rval = dirname(tmp); + delete[] tmp; + return rval; } static ino_t get_inode_num(FILE* f, const char* filename) @@ -99,7 +69,8 @@ static ino_t get_inode_num(FILE* f, const char* filename) if ( fstat(fileno(f), &b) ) { - reporter->Error("failed to fstat fd of %s\n", filename); + reporter->Error("failed to fstat fd of %s: %s\n", filename, + strerror(errno)); exit(1); } @@ -116,8 +87,6 @@ public: const char* name; int line; int level; - BroDoc* doc; - string path; }; // A stack of input buffers we're scanning. file_stack[len-1] is the @@ -141,7 +110,6 @@ static int load_files(const char* file); %x RE %x IGNORE -%s DOC OWS [ \t]* WS [ \t]+ @@ -159,63 +127,15 @@ ESCSEQ (\\([^\n]|[0-7]+|x[[:xdigit:]]+)) %% ##!.* { - // Add this format of comments to the script documentation's "summary". - if ( generate_documentation ) - current_reST_doc->AddSummary(canon_doc_comment(yytext + 3)); + broxygen_mgr->SummaryComment(::filename, yytext + 3); } -##<.* { - yylval.str = copy_string(canon_doc_comment(yytext + 3)); - return TOK_POST_DOC; -} - -##{OWS}{ID}:{WS}.* { - const char* id_start = skip_whitespace(yytext + 2); - yylval.str = copy_string(canon_doc_func_param(id_start).c_str()); - return TOK_DOC; -} - -##.* { - if ( yytext[2] != '#' ) - { - yylval.str = copy_string(canon_doc_comment(yytext + 2)); - return TOK_DOC; - } -} - -##{OWS}{ID}:{WS}.* { - if ( generate_documentation ) - { - // Comment is documenting either a function parameter or return type, - // so appropriate reST markup substitutions are automatically made - // in order to distinguish them from other comments. - if ( ! reST_doc_comments ) - reST_doc_comments = new std::list(); - - // always insert a blank line so that this param/return markup - // 1) doesn't show up in the summary section in the case that it's - // the first comment for the function/event - // 2) has a blank line between it and non-field-list reST markup, - // which is required for correct HTML rendering by Sphinx - reST_doc_comments->push_back(""); - const char* id_start = skip_whitespace(yytext + 2); - reST_doc_comments->push_back(canon_doc_func_param(id_start)); - } -} - ##<.* { - if ( generate_documentation && BroDocObj::last ) - BroDocObj::last->AddDocString(canon_doc_comment(yytext + 3)); + broxygen_mgr->PostComment(yytext + 3); } ##.* { - if ( generate_documentation && (yytext[2] != '#') ) - { - if ( ! reST_doc_comments ) - reST_doc_comments = new std::list(); - - reST_doc_comments->push_back(canon_doc_comment(yytext + 2)); - } + broxygen_mgr->PreComment(yytext + 2); } #{OWS}@no-test.* return TOK_NO_TEST; @@ -224,7 +144,7 @@ ESCSEQ (\\([^\n]|[0-7]+|x[[:xdigit:]]+)) {WS} /* eat whitespace */ -\n { +\n { ++line_number; ++yylloc.first_line; ++yylloc.last_line; @@ -345,7 +265,7 @@ when return TOK_WHEN; @DEBUG return TOK_DEBUG; // marks input for debugger @DIR { - string rval = current_scanned_file_path; + string rval = get_dirname(::filename); if ( ! rval.empty() && rval[0] == '.' ) { @@ -374,25 +294,15 @@ when return TOK_WHEN; @load{WS}{FILE} { const char* new_file = skip_whitespace(yytext + 5); // Skip "@load". - if ( generate_documentation ) - { - current_reST_doc->AddImport(new_file); - - if ( reST_doc_comments ) - { - fprintf(stderr, "Warning: unconsumed reST documentation is being " - "discarded before doing '@load %s' in %s:\n", - new_file, current_reST_doc->GetSourceFileName()); - clear_reST_doc_comments(); - } - } + broxygen_mgr->ScriptDependency(::filename, new_file); (void) load_files(new_file); } @load-sigs{WS}{FILE} { const char* new_sig_file = skip_whitespace(yytext + 10); const char* full_filename = 0; - FILE* f = search_for_file(new_sig_file, "sig", &full_filename, false, 0); + FILE* f = search_for_file(new_sig_file, "sig", &full_filename, false, 0, + get_dirname(::filename)); if ( f ) { @@ -411,7 +321,8 @@ when return TOK_WHEN; // All we have to do is pretend we've already scanned it. const char* full_filename; - FILE* f = search_for_file(new_file, "bro", &full_filename, true, 0); + FILE* f = search_for_file(new_file, "bro", &full_filename, true, 0, + get_dirname(::filename)); if ( f ) { @@ -603,7 +514,7 @@ static int load_files(const char* orig_file) else { - f = search_for_file(orig_file, "bro", &full_filename, true, &bropath_subpath); + f = search_for_file(orig_file, "bro", &full_filename, true, &bropath_subpath, get_dirname(::filename)); bropath_subpath_delete = bropath_subpath; // This will be deleted. } @@ -666,15 +577,7 @@ static int load_files(const char* orig_file) else file_stack.append(new FileInfo); - char* tmp = copy_string(full_filename); - current_scanned_file_path = dirname(tmp); - delete [] tmp; - - if ( generate_documentation ) - { - current_reST_doc = new BroDoc(bropath_subpath, full_filename); - docs_generated.push_back(current_reST_doc); - } + broxygen_mgr->File(full_filename); delete [] bropath_subpath_delete; @@ -776,28 +679,11 @@ void do_atendif() --current_depth; } -void do_doc_token_start() - { - if ( generate_documentation ) - BEGIN(DOC); - } - -void do_doc_token_stop() - { - if ( generate_documentation ) - BEGIN(INITIAL); - } - // Be careful to never delete things from this list, as the strings // are referred to (in order to save the locations of tokens and statements, // for error reporting and debugging). static name_list input_files; -const char* get_current_input_filename() - { - return ::filename; - } - void add_input_file(const char* file) { if ( ! file ) @@ -852,8 +738,6 @@ int yywrap() // Stack is now empty. while ( input_files.length() > 0 ) { - check_capture_filter_changes(); - if ( load_files(input_files[0]) ) { // Don't delete the filename - it's pointed to by @@ -867,8 +751,6 @@ int yywrap() (void) input_files.remove_nth(0); } - check_capture_filter_changes(); - // For each file scanned so far, and for each @prefix, look for a // prefixed and flattened version of the loaded file in BROPATH. The // flattening involves taking the path in BROPATH in which the @@ -893,7 +775,8 @@ int yywrap() string s; s = dot_canon(it->subpath.c_str(), it->name.c_str(), prefixes[i]); - FILE* f = search_for_file(s.c_str(), "bro", 0, false, 0); + FILE* f = search_for_file(s.c_str(), "bro", 0, false, 0, + get_dirname(::filename)); //printf("====== prefix search ======\n"); //printf("File : %s\n", it->name.c_str()); @@ -977,9 +860,6 @@ int yywrap() return 0; } - if ( generate_documentation ) - clear_reST_doc_comments(); - // Otherwise, we are done. return 1; } @@ -990,8 +870,6 @@ FileInfo::FileInfo(string arg_restore_module) restore_module = arg_restore_module; name = ::filename; line = ::line_number; - doc = ::current_reST_doc; - path = current_scanned_file_path; } FileInfo::~FileInfo() @@ -1002,56 +880,7 @@ FileInfo::~FileInfo() yy_switch_to_buffer(buffer_state); yylloc.filename = filename = name; yylloc.first_line = yylloc.last_line = line_number = line; - last_reST_doc = current_reST_doc; - current_reST_doc = doc; - current_scanned_file_path = path; if ( restore_module != "" ) current_module = restore_module; } - -static void check_capture_filter_changes() - { - if ( ! generate_documentation ) - return; - - // Lookup the "capture_filters" identifier, if it has any defined - // value, add it to the script's reST documentation, and finally - // clear the table so it doesn't taint the documentation for - // subsequent scripts. - - ID* capture_filters = global_scope()->Lookup("capture_filters"); - - if ( capture_filters ) - { - ODesc desc; - desc.SetIndentSpaces(4); - capture_filters->ID_Val()->Describe(&desc); - last_reST_doc->SetPacketFilter(desc.Description()); - capture_filters->ID_Val()->AsTableVal()->RemoveAll(); - } - } - -void print_current_reST_doc_comments() - { - if ( ! reST_doc_comments ) - return; - - std::list::iterator it; - - for ( it = reST_doc_comments->begin(); it != reST_doc_comments->end(); ++it ) - fprintf(stderr, "##%s\n", it->c_str()); - } - -void clear_reST_doc_comments() - { - if ( ! reST_doc_comments ) - return; - - fprintf(stderr, "Warning: %zu unconsumed reST comments:\n", - reST_doc_comments->size()); - - print_current_reST_doc_comments(); - delete reST_doc_comments; - reST_doc_comments = 0; - } diff --git a/src/util.cc b/src/util.cc index f26b3fb0c2..d06c6aff96 100644 --- a/src/util.cc +++ b/src/util.cc @@ -1060,11 +1060,9 @@ void get_script_subpath(const std::string& full_filename, const char** subpath) *subpath = normalize_path(my_subpath.c_str()); } -extern string current_scanned_file_path; - FILE* search_for_file(const char* filename, const char* ext, const char** full_filename, bool load_pkgs, - const char** bropath_subpath) + const char** bropath_subpath, string prepend_to_search_path) { // If the file is a literal absolute path we don't have to search, // just return the result of trying to open it. If the file is @@ -1088,9 +1086,9 @@ FILE* search_for_file(const char* filename, const char* ext, // Prepend the currently loading script's path to BROPATH so that // @loads can be referenced relatively. - if ( current_scanned_file_path != "" && filename[0] == '.' ) + if ( ! prepend_to_search_path.empty() && filename[0] == '.' ) safe_snprintf(path, sizeof(path), "%s:%s", - current_scanned_file_path.c_str(), bro_path()); + prepend_to_search_path.c_str(), bro_path()); else safe_strncpy(path, bro_path(), sizeof(path)); diff --git a/src/util.h b/src/util.h index fcdfd6d499..e7ac4d33af 100644 --- a/src/util.h +++ b/src/util.h @@ -209,7 +209,8 @@ std::string dot_canon(std::string path, std::string file, std::string prefix = " const char* normalize_path(const char* path); void get_script_subpath(const std::string& full_filename, const char** subpath); extern FILE* search_for_file(const char* filename, const char* ext, - const char** full_filename, bool load_pkgs, const char** bropath_subpath); + const char** full_filename, bool load_pkgs, const char** bropath_subpath, + std::string prepend_to_search_path = ""); // Renames the given file to a new temporary name, and opens a new file with // the original name. Returns new file or NULL on error. Inits rotate_info if