From 5a857a6dfc809a23237f983d8e19c559dc382331 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Thu, 3 Oct 2013 10:42:04 -0500 Subject: [PATCH 01/40] Initial skeleton of new Broxygen infrastructure. Doesn't generate any docs, but it's hooked in to all places needed to gather the necessary stuff w/ significantly less coupling than before. The gathering now always occurs unconditionally to make documentation available at runtime and a command line switch (-X) only toggles whether to output docs to disk (reST format). Should also improve the treatment of type name aliasing which wasn't a big problem in practice before, but I think it's more correct now: there's now a distinct BroType for each alias, but extensible types (record/enum) will automatically update the types for aliases on redef. Other misc refactoring of note: - Removed a redundant/unused way of declaring event types. - Changed type serialization format/process to preserve type name information and remove compatibility code (since broccoli will have be updated anyway). --- src/BroDoc.cc | 2 + src/Brofiler.h | 2 + src/CMakeLists.txt | 2 + src/Expr.cc | 1 + src/ID.cc | 6 +- src/Serializer.h | 2 +- src/Type.cc | 260 +++++++-------------- src/Type.h | 68 ++---- src/Var.cc | 67 ++---- src/Var.h | 2 +- src/broxygen/Manager.cc | 88 ++++++++ src/broxygen/Manager.h | 43 ++++ src/main.cc | 41 ++-- src/parse.y | 411 ++++++++-------------------------- src/plugin/ComponentManager.h | 4 +- src/scan.l | 221 +++--------------- src/util.cc | 8 +- src/util.h | 3 +- 18 files changed, 393 insertions(+), 838 deletions(-) create mode 100644 src/broxygen/Manager.cc create mode 100644 src/broxygen/Manager.h diff --git a/src/BroDoc.cc b/src/BroDoc.cc index 6953680df0..5733c586bf 100644 --- a/src/BroDoc.cc +++ b/src/BroDoc.cc @@ -556,6 +556,7 @@ static void WritePluginBifItems(FILE* f, const plugin::Plugin* p, static void WriteAnalyzerTagDefn(FILE* f, EnumType* e, const string& module) { + /* TODO string tag_id= module + "::Tag"; e = new CommentedEnumType(e); e->SetTypeID(copy_string(tag_id.c_str())); @@ -570,6 +571,7 @@ static void WriteAnalyzerTagDefn(FILE* f, EnumType* e, const string& module) BroDocObj bdo(dummy_id, r, true); bdo.WriteReST(f); + */ } static bool ComponentsMatch(const plugin::Plugin* p, plugin::component::Type t, diff --git a/src/Brofiler.h b/src/Brofiler.h index 22e5808bf6..88ce434070 100644 --- a/src/Brofiler.h +++ b/src/Brofiler.h @@ -78,4 +78,6 @@ private: }; }; +extern Brofiler brofiler; + #endif /* BROFILER_H_ */ diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index c881cc4df1..4145984f3b 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -370,6 +370,8 @@ set(bro_SRCS plugin/Plugin.cc plugin/Macros.h + broxygen/Manager.cc + nb_dns.c digest.h ) diff --git a/src/Expr.cc b/src/Expr.cc index 0eaa7ce918..303b3cf5c7 100644 --- a/src/Expr.cc +++ b/src/Expr.cc @@ -5111,6 +5111,7 @@ BroType* ListExpr::InitType() const types->append(td); } + return new RecordType(types); } diff --git a/src/ID.cc b/src/ID.cc index a6e592146b..ed561016ff 100644 --- a/src/ID.cc +++ b/src/ID.cc @@ -619,6 +619,7 @@ void ID::DescribeExtended(ODesc* d) const void ID::DescribeReSTShort(ODesc* d) const { + /* TODO if ( is_type ) d->Add(":bro:type:`"); else @@ -668,6 +669,7 @@ void ID::DescribeReSTShort(ODesc* d) const d->SP(); attrs->DescribeReST(d); } + */ } void ID::DescribeReST(ODesc* d, bool is_role) const @@ -697,10 +699,10 @@ void ID::DescribeReST(ODesc* d, bool is_role) const { d->Add(":Type: "); - if ( ! is_type && type->GetTypeID() ) + if ( ! is_type && ! type->GetName().empty() ) { d->Add(":bro:type:`"); - d->Add(type->GetTypeID()); + d->Add(type->GetName()); d->Add("`"); } else diff --git a/src/Serializer.h b/src/Serializer.h index 72e0723880..c1af1a2c4f 100644 --- a/src/Serializer.h +++ b/src/Serializer.h @@ -125,7 +125,7 @@ protected: // This will be increased whenever there is an incompatible change // in the data format. - static const uint32 DATA_FORMAT_VERSION = 23; + static const uint32 DATA_FORMAT_VERSION = 24; ChunkedIO* io; diff --git a/src/Type.cc b/src/Type.cc index 2b9faa8018..a659dc473b 100644 --- a/src/Type.cc +++ b/src/Type.cc @@ -8,13 +8,12 @@ #include "Scope.h" #include "Serializer.h" #include "Reporter.h" +#include "broxygen/Manager.h" #include #include #include -extern int generate_documentation; - // Note: This function must be thread-safe. const char* type_name(TypeTag t) { @@ -47,7 +46,6 @@ BroType::BroType(TypeTag t, bool arg_base_type) tag = t; is_network_order = 0; base_type = arg_base_type; - type_id = 0; switch ( tag ) { case TYPE_VOID: @@ -110,10 +108,26 @@ BroType::BroType(TypeTag t, bool arg_base_type) } -BroType::~BroType() +BroType* BroType::Clone() const { - if ( type_id ) - delete [] type_id; + SerializationFormat* form = new BinarySerializationFormat(); + form->StartWrite(); + CloneSerializer ss(form); + SerialInfo sinfo(&ss); + sinfo.cache = false; + + this->Serialize(&sinfo); + char* data; + uint32 len = form->EndWrite(&data); + form->StartRead(data, len); + + UnserialInfo uinfo(&ss); + uinfo.cache = false; + BroType* rval = this->Unserialize(&uinfo); + + delete [] data; + + return rval; } int BroType::MatchesIndex(ListExpr*& index) const @@ -222,9 +236,21 @@ BroType* BroType::Unserialize(UnserialInfo* info, TypeTag want) if ( ! t ) return 0; - // For base types, we return our current instance - // if not in "documentation mode". - if ( t->base_type && ! generate_documentation ) + if ( ! t->name.empty() ) + { + // Avoid creating a new type if it's known by name. + // Also avoids loss of base type name alias (from condition below). + ID* id = global_scope()->Lookup(t->name.c_str()); + BroType* t2 = id ? id->AsType() : 0; + + if ( t2 ) + { + Unref(t); + return t2->Ref(); + } + } + + if ( t->base_type ) { BroType* t2 = ::base_type(TypeTag(t->tag)); Unref(t); @@ -247,21 +273,10 @@ bool BroType::DoSerialize(SerialInfo* info) const if ( ! (SERIALIZE(char(tag)) && SERIALIZE(char(internal_tag))) ) return false; - if ( ! (SERIALIZE(is_network_order) && SERIALIZE(base_type) && - // Serialize the former "bool is_global_attributes_type" for - // backwards compatibility. - SERIALIZE(false)) ) + if ( ! (SERIALIZE(is_network_order) && SERIALIZE(base_type)) ) return false; - // Likewise, serialize the former optional "RecordType* attributes_type" - // for backwards compatibility. - void* null = NULL; - SERIALIZE(null); - - if ( generate_documentation ) - { - SERIALIZE_OPTIONAL_STR(type_id); - } + SERIALIZE_STR(name.c_str(), name.size()); info->s->WriteCloseTag("Type"); @@ -279,24 +294,15 @@ bool BroType::DoUnserialize(UnserialInfo* info) tag = (TypeTag) c1; internal_tag = (InternalTypeTag) c2; - bool not_used; - - if ( ! (UNSERIALIZE(&is_network_order) && UNSERIALIZE(&base_type) - // Unerialize the former "bool is_global_attributes_type" for - // backwards compatibility. - && UNSERIALIZE(¬_used)) ) + if ( ! (UNSERIALIZE(&is_network_order) && UNSERIALIZE(&base_type)) ) return 0; - BroType* not_used_either; + const char* n; + if ( ! UNSERIALIZE_STR(&n, 0) ) + return false; - // Likewise, unserialize the former optional "RecordType* - // attributes_type" for backwards compatibility. - UNSERIALIZE_OPTIONAL(not_used_either, BroType::Unserialize(info, TYPE_RECORD)); - - if ( generate_documentation ) - { - UNSERIALIZE_OPTIONAL_STR(type_id); - } + name = n; + delete [] n; return true; } @@ -470,10 +476,10 @@ void IndexType::DescribeReST(ODesc* d) const const BroType* t = (*IndexTypes())[i]; - if ( t->GetTypeID() ) + if ( ! t->GetName().empty() ) { d->Add(":bro:type:`"); - d->Add(t->GetTypeID()); + d->Add(t->GetName()); d->Add("`"); } else @@ -486,10 +492,10 @@ void IndexType::DescribeReST(ODesc* d) const { d->Add(" of "); - if ( yield_type->GetTypeID() ) + if ( ! yield_type->GetName().empty() ) { d->Add(":bro:type:`"); - d->Add(yield_type->GetTypeID()); + d->Add(yield_type->GetName()); d->Add("`"); } else @@ -781,10 +787,10 @@ void FuncType::DescribeReST(ODesc* d) const { d->AddSP(" :"); - if ( yield->GetTypeID() ) + if ( ! yield->GetName().empty() ) { d->Add(":bro:type:`"); - d->Add(yield->GetTypeID()); + d->Add(yield->GetName()); d->Add("`"); } else @@ -873,6 +879,17 @@ TypeDecl::TypeDecl(BroType* t, const char* i, attr_list* arg_attrs, bool in_reco id = i; } +TypeDecl::TypeDecl(const TypeDecl& other) + { + type = other.type->Ref(); + attrs = other.attrs; + + if ( attrs ) + ::Ref(attrs); + + id = copy_string(other.id); + } + TypeDecl::~TypeDecl() { Unref(type); @@ -914,10 +931,10 @@ void TypeDecl::DescribeReST(ODesc* d) const d->Add(id); d->Add(": "); - if ( type->GetTypeID() ) + if ( ! type->GetName().empty() ) { d->Add(":bro:type:`"); - d->Add(type->GetTypeID()); + d->Add(type->GetName()); d->Add("`"); } else @@ -930,37 +947,6 @@ void TypeDecl::DescribeReST(ODesc* d) const } } -CommentedTypeDecl::CommentedTypeDecl(BroType* t, const char* i, - attr_list* attrs, bool in_record, std::list* cmnt_list) - : TypeDecl(t, i, attrs, in_record) - { - comments = cmnt_list; - } - -CommentedTypeDecl::~CommentedTypeDecl() - { - if ( comments ) delete comments; - } - -void CommentedTypeDecl::DescribeReST(ODesc* d) const - { - TypeDecl::DescribeReST(d); - - if ( comments ) - { - d->PushIndent(); - std::list::const_iterator i; - - for ( i = comments->begin(); i != comments->end(); ++i) - { - if ( i != comments->begin() ) d->NL(); - d->Add(i->c_str()); - } - - d->PopIndentNoNL(); - } - } - RecordType::RecordType(type_decl_list* arg_types) : BroType(TYPE_RECORD) { types = arg_types; @@ -1328,38 +1314,12 @@ bool OpaqueType::DoUnserialize(UnserialInfo* info) return true; } -EnumType::EnumType(const string& arg_name) -: BroType(TYPE_ENUM) - { - name = arg_name; - counter = 0; - } - -EnumType::EnumType(EnumType* e) -: BroType(TYPE_ENUM) - { - name = e->name; - counter = e->counter; - - for ( NameMap::iterator it = e->names.begin(); it != e->names.end(); ++it ) - names[copy_string(it->first)] = it->second; - } - EnumType::~EnumType() { for ( NameMap::iterator iter = names.begin(); iter != names.end(); ++iter ) delete [] iter->first; } -CommentedEnumType::~CommentedEnumType() - { - for ( CommentMap::iterator iter = comments.begin(); iter != comments.end(); ++iter ) - { - delete [] iter->first; - delete iter->second; - } - } - // Note, we use reporter->Error() here (not Error()) to include the current script // location in the error message, rather than the one where the type was // originally defined. @@ -1372,7 +1332,7 @@ void EnumType::AddName(const string& module_name, const char* name, bool is_expo SetError(); return; } - AddNameInternal(module_name, name, counter, is_export); + CheckAndAddName(module_name, name, counter, is_export); counter++; } @@ -1386,32 +1346,12 @@ void EnumType::AddName(const string& module_name, const char* name, bro_int_t va return; } counter = -1; - AddNameInternal(module_name, name, val, is_export); + CheckAndAddName(module_name, name, val, is_export); } -void CommentedEnumType::AddComment(const string& module_name, const char* name, - std::list* new_comments) +void EnumType::CheckAndAddName(const string& module_name, const char* name, + bro_int_t val, bool is_export) { - if ( ! new_comments ) - return; - - string fullname = make_full_var_name(module_name.c_str(), name); - - CommentMap::iterator it = comments.find(fullname.c_str()); - - if ( it == comments.end() ) - comments[copy_string(fullname.c_str())] = new_comments; - else - { - list* prev_comments = comments[fullname.c_str()]; - prev_comments->splice(prev_comments->end(), *new_comments); - delete new_comments; - } - } - -void EnumType::AddNameInternal(const string& module_name, const char* name, bro_int_t val, bool is_export) - { - ID *id; if ( Lookup(val) ) { reporter->Error("enumerator value in enumerated type definition already exists"); @@ -1419,12 +1359,14 @@ void EnumType::AddNameInternal(const string& module_name, const char* name, bro_ return; } - id = lookup_ID(name, module_name.c_str()); + ID* id = lookup_ID(name, module_name.c_str()); + if ( ! id ) { id = install_ID(name, module_name.c_str(), true, is_export); id->SetType(this->Ref()); id->SetEnumConst(); + broxygen_mgr->Identifier(id); } else { @@ -1433,11 +1375,19 @@ void EnumType::AddNameInternal(const string& module_name, const char* name, bro_ return; } - string fullname = make_full_var_name(module_name.c_str(), name); - names[copy_string(fullname.c_str())] = val; + AddNameInternal(module_name, name, val, is_export); + + set types = type_aliases[GetName()]; + set::const_iterator it; + + for ( it = types.begin(); it != types.end(); ++it ) + if ( *it != this ) + (*it)->AsEnumType()->AddNameInternal(module_name, name, val, + is_export); } -void CommentedEnumType::AddNameInternal(const string& module_name, const char* name, bro_int_t val, bool is_export) +void EnumType::AddNameInternal(const string& module_name, const char* name, + bro_int_t val, bool is_export) { string fullname = make_full_var_name(module_name.c_str(), name); names[copy_string(fullname.c_str())] = val; @@ -1466,54 +1416,8 @@ const char* EnumType::Lookup(bro_int_t value) void EnumType::DescribeReST(ODesc* d) const { - d->Add(":bro:type:`"); - d->Add(name.c_str()); - d->Add("`"); - } - -void CommentedEnumType::DescribeReST(ODesc* d) const - { - // create temporary, reverse name map so that enums can be documented - // in ascending order of their actual integral value instead of by name - typedef std::map< bro_int_t, const char* > RevNameMap; - RevNameMap rev; - for ( NameMap::const_iterator it = names.begin(); it != names.end(); ++it ) - rev[it->second] = it->first; - - d->Add(":bro:type:`"); - d->Add(type_name(Tag())); - d->Add("`"); - d->PushIndent(); - d->NL(); - - for ( RevNameMap::const_iterator it = rev.begin(); it != rev.end(); ++it ) - { - if ( it != rev.begin() ) - { - d->NL(); - d->NL(); - } - - d->Add(".. bro:enum:: "); - d->AddSP(it->second); - d->Add(GetTypeID()); - - CommentMap::const_iterator cmnt_it = comments.find(it->second); - if ( cmnt_it != comments.end() ) - { - d->PushIndent(); - d->NL(); - std::list::const_iterator i; - const std::list* cmnt_list = cmnt_it->second; - for ( i = cmnt_list->begin(); i != cmnt_list->end(); ++i) - { - if ( i != cmnt_list->begin() ) d->NL(); - d->Add(i->c_str()); - } - d->PopIndentNoNL(); - } - } - d->PopIndentNoNL(); + // TODO: this probably goes away + d->Add(":bro:type:`enum`"); } IMPLEMENT_SERIAL(EnumType, SER_ENUM_TYPE); diff --git a/src/Type.h b/src/Type.h index a6163d5152..be27426546 100644 --- a/src/Type.h +++ b/src/Type.h @@ -4,7 +4,7 @@ #define type_h #include -#include +#include #include #include "Obj.h" @@ -73,7 +73,9 @@ const int MATCHES_INDEX_VECTOR = 2; class BroType : public BroObj { public: BroType(TypeTag tag, bool base_type = false); - ~BroType(); + ~BroType() { } + + BroType* Clone() const; TypeTag Tag() const { return tag; } InternalTypeTag InternalType() const { return internal_tag; } @@ -232,11 +234,11 @@ public: bool Serialize(SerialInfo* info) const; static BroType* Unserialize(UnserialInfo* info, TypeTag want = TYPE_ANY); - void SetTypeID(const char* id) { type_id = id; } - const char* GetTypeID() const { return type_id; } + void SetName(const string& arg_name) { name = arg_name; } + string GetName() const { return name; } protected: - BroType() { type_id = 0; } + BroType() { } void SetError(); @@ -247,10 +249,7 @@ private: InternalTypeTag internal_tag; bool is_network_order; bool base_type; - - // This type_id field is only used by the documentation framework to - // track the names of declared types. - const char* type_id; + string name; }; class TypeList : public BroType { @@ -408,6 +407,7 @@ protected: class TypeDecl { public: TypeDecl(BroType* t, const char* i, attr_list* attrs = 0, bool in_record = false); + TypeDecl(const TypeDecl& other); virtual ~TypeDecl(); const Attr* FindAttr(attr_tag a) const @@ -423,17 +423,6 @@ public: const char* id; }; -class CommentedTypeDecl : public TypeDecl { -public: - CommentedTypeDecl(BroType* t, const char* i, attr_list* attrs = 0, - bool in_record = false, std::list* cmnt_list = 0); - virtual ~CommentedTypeDecl(); - - void DescribeReST(ODesc* d) const; - - std::list* comments; -}; - class RecordType : public BroType { public: RecordType(type_decl_list* types); @@ -522,8 +511,7 @@ protected: class EnumType : public BroType { public: - EnumType(const string& arg_name); - EnumType(EnumType* e); + EnumType() : BroType(TYPE_ENUM) { counter = 0; } ~EnumType(); // The value of this name is next internal counter value, starting @@ -539,17 +527,17 @@ public: bro_int_t Lookup(const string& module_name, const char* name); const char* Lookup(bro_int_t value); // Returns 0 if not found - string Name() const { return name; } - void DescribeReST(ODesc* d) const; protected: - EnumType() { counter = 0; } DECLARE_SERIAL(EnumType) - virtual void AddNameInternal(const string& module_name, + void AddNameInternal(const string& module_name, const char* name, bro_int_t val, bool is_export); + void CheckAndAddName(const string& module_name, + const char* name, bro_int_t val, bool is_export); + typedef std::map< const char*, bro_int_t, ltstr > NameMap; NameMap names; @@ -560,31 +548,6 @@ protected: // as a flag to prevent mixing of auto-increment and explicit // enumerator specifications. bro_int_t counter; - - // The name of the enum type is stored for documentation purposes. - string name; -}; - -class CommentedEnumType: public EnumType { -public: - CommentedEnumType(const string& arg_name) : EnumType(arg_name) {} - CommentedEnumType(EnumType* e) : EnumType(e) {} - ~CommentedEnumType(); - - void DescribeReST(ODesc* d) const; - void AddComment(const string& module_name, const char* name, - std::list* comments); - -protected: - // This overriden method does not install the given ID name into a - // scope and it also does not do any kind of checking that the - // provided name already exists. - void AddNameInternal(const string& module_name, const char* name, - bro_int_t val, bool is_export); - - // Comments are only filled when in "documentation mode". - typedef std::map< const char*, std::list*, ltstr > CommentMap; - CommentMap comments; }; class VectorType : public BroType { @@ -609,6 +572,9 @@ protected: BroType* yield_type; }; +typedef std::map > TypeAliasMap; +extern TypeAliasMap type_aliases; + extern OpaqueType* md5_type; extern OpaqueType* sha1_type; extern OpaqueType* sha256_type; diff --git a/src/Var.cc b/src/Var.cc index d384fedc74..821c9e207b 100644 --- a/src/Var.cc +++ b/src/Var.cc @@ -10,8 +10,6 @@ #include "RemoteSerializer.h" #include "EventRegistry.h" -extern int generate_documentation; - static Val* init_val(Expr* init, const BroType* t, Val* aggr) { return init->InitVal(t, aggr); @@ -261,61 +259,26 @@ extern Expr* add_and_assign_local(ID* id, Expr* init, Val* val) return new AssignExpr(new NameExpr(id), init, 0, val); } -void add_type(ID* id, BroType* t, attr_list* attr, int /* is_event */) +void add_type(ID* id, BroType* t, attr_list* attr) { - BroType* tnew = t; + string new_type_name(id->Name()); + string old_type_name(t->GetName()); + BroType* tnew = 0; - // In "documentation mode", we'd like to to be able to associate - // an identifier name with a declared type. Dealing with declared - // types that are "aliases" to a builtin type requires that the BroType - // is cloned before setting the identifier name that resolves to it. - // And still this is not enough to document cases where the declared type - // is an alias for another declared type -- but that's not a natural/common - // practice. If documenting that corner case is desired, one way - // is to add an ID* to class ID that tracks aliases and set it here if - // t->GetTypeID() is true. - if ( generate_documentation ) - { - switch ( t->Tag() ) { - // Only "shallow" copy types that may contain records because - // we want to be able to see additions to the original record type's - // list of fields - case TYPE_RECORD: - tnew = new RecordType(t->AsRecordType()->Types()); - break; - case TYPE_TABLE: - tnew = new TableType(t->AsTableType()->Indices(), - t->AsTableType()->YieldType()); - break; - case TYPE_VECTOR: - tnew = new VectorType(t->AsVectorType()->YieldType()); - break; - case TYPE_FUNC: - tnew = new FuncType(t->AsFuncType()->Args(), - t->AsFuncType()->YieldType(), - t->AsFuncType()->Flavor()); - break; - default: - SerializationFormat* form = new BinarySerializationFormat(); - form->StartWrite(); - CloneSerializer ss(form); - SerialInfo sinfo(&ss); - sinfo.cache = false; + if ( (t->Tag() == TYPE_RECORD || t->Tag() == TYPE_ENUM) && + ! old_type_name.empty() ) + // Clone the type to preserve type name aliasing. + tnew = t->Clone(); + else + // An extensible types (record/enum) being declared for first time. + tnew = t; - t->Serialize(&sinfo); - char* data; - uint32 len = form->EndWrite(&data); - form->StartRead(data, len); + type_aliases[new_type_name].insert(tnew); - UnserialInfo uinfo(&ss); - uinfo.cache = false; - tnew = t->Unserialize(&uinfo); + if ( new_type_name != old_type_name && ! old_type_name.empty() ) + type_aliases[old_type_name].insert(tnew); - delete [] data; - } - - tnew->SetTypeID(copy_string(id->Name())); - } + tnew->SetName(id->Name()); id->SetType(tnew); id->MakeType(); diff --git a/src/Var.h b/src/Var.h index 8b9866ed2d..bcdd45dad2 100644 --- a/src/Var.h +++ b/src/Var.h @@ -18,7 +18,7 @@ extern Stmt* add_local(ID* id, BroType* t, init_class c, Expr* init, attr_list* attr, decl_type dt); extern Expr* add_and_assign_local(ID* id, Expr* init, Val* val = 0); -extern void add_type(ID* id, BroType* t, attr_list* attr, int is_event); +extern void add_type(ID* id, BroType* t, attr_list* attr); extern void begin_func(ID* id, const char* module_name, function_flavor flavor, int is_redef, FuncType* t); diff --git a/src/broxygen/Manager.cc b/src/broxygen/Manager.cc new file mode 100644 index 0000000000..30c9008787 --- /dev/null +++ b/src/broxygen/Manager.cc @@ -0,0 +1,88 @@ +#include "Manager.h" + +using namespace broxygen; + +Manager::Manager(const std::string& config) + { + // TODO + } + +void Manager::GenerateDocs() const + { + // TODO + + // may be a no-op if no config + + // does the old canon_doc_func_param stuff happen here now on the fly + // for functions we're about to document? + } + +void Manager::File(const std::string& path) + { + // TODO + // determine bropath subpath + // can be a file or directory? + } + +void Manager::ScriptDependency(const std::string& path, const std::string& dep) + { + // TODO: + // need anything from BroDoc::AddImport? + // warn about unconsumed comments (and discard any) + } + +void Manager::ModuleUsage(const std::string& path, const std::string& module) + { + // TODO lookup script and add module to a set + } + +void Manager::Identifier(const ID *id) + { + // TODO: lookup script to associate w/ by GetLocationInfo()->filename + // do different things depending on Type? (eg function flavor versus state) + // do different things based on redef attr + const ? + // consume any buffered comments and associate w/ id + // deal w/ type aliasing + // special enum or record handing? + // if it's a function we may already have added it (decl versus impl) + } + +void Manager::RecordField(const ID *id, const TypeDecl *field, + const std::string& path) + { + // TODO: consume comments + // redef is implicit -- script path of field will differ from ID/type's + } + +void Manager::Redef(const ID* id, const string& path) + { + // TODO: lookup script w/ 'path' to associate the id in as redef'd + // consume any buffered comments and associate w/ redef'd id + // can sort notices here + } + +void Manager::SummaryComment(const std::string& script, + const std::string& comment) + { + // TODO + // canon_doc_comment ? + } + +void Manager::PreComment(const std::string& comment) + { + // TODO + // canon_doc_comment + } + +void Manager::PostComment(const std::string& comment) + { + // TODO this gets associated with the last thing registered + // canon_doc_comment + } + + +// TODO: "canon_doc_comment" means treat "##Text" and "## Text" the same +// so that a single space doesn't generate an indentation level. + + +// TODO: creating proto/file analyzer docs diff --git a/src/broxygen/Manager.h b/src/broxygen/Manager.h new file mode 100644 index 0000000000..d0665c307d --- /dev/null +++ b/src/broxygen/Manager.h @@ -0,0 +1,43 @@ +#ifndef BROXYGEN_MANAGER_H +#define BROXYGEN_MANAGER_H + +#include + +#include "ID.h" +#include "Type.h" + +namespace broxygen { + +class Manager { + +public: + + Manager(const std::string& config); + + void GenerateDocs() const; + + void File(const std::string& path); + + void ScriptDependency(const std::string& path, const std::string& dep); + + void ModuleUsage(const std::string& path, const std::string& module); + + void Identifier(const ID* id); + + void RecordField(const ID* id, const TypeDecl* field, + const std::string& path); + + void Redef(const ID* id, const std::string& path); + + void SummaryComment(const std::string& path, const std::string& comment); + + void PreComment(const std::string& comment); + + void PostComment(const std::string& comment); +}; + +} // namespace broxygen + +extern broxygen::Manager* broxygen_mgr; + +#endif diff --git a/src/main.cc b/src/main.cc index 313e1a40b0..bdb6499445 100644 --- a/src/main.cc +++ b/src/main.cc @@ -61,8 +61,8 @@ extern "C" void OPENSSL_add_all_algorithms_conf(void); #include "analyzer/Manager.h" #include "analyzer/Tag.h" #include "plugin/Manager.h" - #include "file_analysis/Manager.h" +#include "broxygen/Manager.h" #include "binpac_bro.h" @@ -100,6 +100,7 @@ input::Manager* input_mgr = 0; plugin::Manager* plugin_mgr = 0; analyzer::Manager* analyzer_mgr = 0; file_analysis::Manager* file_mgr = 0; +broxygen::Manager* broxygen_mgr = 0; Stmt* stmts; EventHandlerPtr net_done = 0; RuleMatcher* rule_matcher = 0; @@ -116,7 +117,6 @@ int signal_val = 0; int optimize = 0; int do_notice_analysis = 0; int rule_bench = 0; -int generate_documentation = 0; SecondaryPath* secondary_path = 0; extern char version[]; char* command_line_policy = 0; @@ -124,6 +124,8 @@ vector params; char* proc_status_file = 0; int snaplen = 0; // this gets set from the scripting-layer's value +TypeAliasMap type_aliases; + OpaqueType* md5_type = 0; OpaqueType* sha1_type = 0; OpaqueType* sha256_type = 0; @@ -132,8 +134,6 @@ OpaqueType* cardinality_type = 0; OpaqueType* topk_type = 0; OpaqueType* bloomfilter_type = 0; -extern std::list docs_generated; - // Keep copy of command line int bro_argc; char** bro_argv; @@ -203,7 +203,7 @@ void usage() fprintf(stderr, " -T|--re-level | set 'RE_level' for rules\n"); fprintf(stderr, " -U|--status-file | Record process status in file\n"); fprintf(stderr, " -W|--watchdog | activate watchdog timer\n"); - fprintf(stderr, " -Z|--doc-scripts | generate documentation for all loaded scripts\n"); + fprintf(stderr, " -X|--broxygen | generate documentation based on config file\n"); #ifdef USE_PERFTOOLS_DEBUG fprintf(stderr, " -m|--mem-leaks | show leaks [perftools]\n"); @@ -373,6 +373,7 @@ void terminate_bro() plugin_mgr->FinishPlugins(); + delete broxygen_mgr; delete timer_mgr; delete dns_mgr; delete persistence_serializer; @@ -473,7 +474,7 @@ int main(int argc, char** argv) {"filter", required_argument, 0, 'f'}, {"help", no_argument, 0, 'h'}, {"iface", required_argument, 0, 'i'}, - {"doc-scripts", no_argument, 0, 'Z'}, + {"broxygen", required_argument, 0, 'X'}, {"prefix", required_argument, 0, 'p'}, {"readfile", required_argument, 0, 'r'}, {"flowfile", required_argument, 0, 'y'}, @@ -532,7 +533,7 @@ int main(int argc, char** argv) if ( p ) add_to_name_list(p, ':', prefixes); - string active_file; + string broxygen_config; #ifdef USE_IDMEF string libidmef_dtd_path = "idmef-message.dtd"; @@ -545,7 +546,7 @@ int main(int argc, char** argv) opterr = 0; char opts[256]; - safe_strncpy(opts, "B:D:e:f:I:i:K:l:n:p:R:r:s:T:t:U:w:x:X:y:Y:z:CFGLNOPSWbdghvZ", + safe_strncpy(opts, "B:D:e:f:I:i:K:l:n:p:R:r:s:T:t:U:w:x:X:y:Y:z:CFGLNOPSWbdghv", sizeof(opts)); #ifdef USE_PERFTOOLS_DEBUG @@ -727,8 +728,8 @@ int main(int argc, char** argv) break; #endif - case 'Z': - generate_documentation = 1; + case 'X': + broxygen_config = optarg; break; #ifdef USE_IDMEF @@ -760,6 +761,7 @@ int main(int argc, char** argv) reporter = new Reporter(); thread_mgr = new threading::Manager(); + broxygen_mgr = new broxygen::Manager(broxygen_config); #ifdef DEBUG if ( debug_streams ) @@ -888,23 +890,6 @@ int main(int argc, char** argv) } #endif - if ( generate_documentation ) - { - CreateProtoAnalyzerDoc("proto-analyzers.rst"); - CreateFileAnalyzerDoc("file-analyzers.rst"); - - std::list::iterator it; - - for ( it = docs_generated.begin(); it != docs_generated.end(); ++it ) - (*it)->WriteDocFile(); - - for ( it = docs_generated.begin(); it != docs_generated.end(); ++it ) - delete *it; - - terminate_bro(); - return 0; - } - if ( reporter->Errors() > 0 ) { delete dns_mgr; @@ -915,6 +900,8 @@ int main(int argc, char** argv) init_general_global_var(); + broxygen_mgr->GenerateDocs(); + if ( user_pcap_filter ) { ID* id = global_scope()->Lookup("cmd_line_bpf_filter"); diff --git a/src/parse.y b/src/parse.y index 98df0de2a3..ed79808c78 100644 --- a/src/parse.y +++ b/src/parse.y @@ -2,7 +2,7 @@ // See the file "COPYING" in the main distribution directory for copyright. %} -%expect 85 +%expect 75 %token TOK_ADD TOK_ADD_TO TOK_ADDR TOK_ANY %token TOK_ATENDIF TOK_ATELSE TOK_ATIF TOK_ATIFDEF TOK_ATIFNDEF @@ -28,8 +28,6 @@ %token TOK_DEBUG -%token TOK_DOC TOK_POST_DOC - %token TOK_NO_TEST %nonassoc TOK_HOOK @@ -47,8 +45,7 @@ %left '$' '[' ']' '(' ')' TOK_HAS_FIELD TOK_HAS_ATTR %type opt_no_test opt_no_test_block -%type TOK_ID TOK_PATTERN_TEXT single_pattern TOK_DOC TOK_POST_DOC -%type opt_doc_list opt_post_doc_list +%type TOK_ID TOK_PATTERN_TEXT single_pattern %type local_id global_id def_global_id event_id global_or_event_id resolve_id begin_func %type local_id_list %type init_class @@ -83,17 +80,13 @@ #include "RE.h" #include "Scope.h" #include "Reporter.h" -#include "BroDoc.h" -#include "BroDocObj.h" #include "Brofiler.h" +#include "broxygen/Manager.h" -#include +#include #include -extern Brofiler brofiler; -extern BroDoc* current_reST_doc; -extern int generate_documentation; -extern std::list* reST_doc_comments; +extern const char* filename; // Absolute path of file currently being parsed. YYLTYPE GetCurrentLocation(); extern int yyerror(const char[]); @@ -127,24 +120,13 @@ bool defining_global_ID = false; ID* func_id = 0; EnumType *cur_enum_type = 0; -CommentedEnumType *cur_enum_type_doc = 0; -const char* cur_enum_elem_id = 0; - -type_decl_list* fake_type_decl_list = 0; -TypeDecl* last_fake_type_decl = 0; - static ID* cur_decl_type_id = 0; static void parser_new_enum (void) { /* Starting a new enum definition. */ assert(cur_enum_type == NULL); - cur_enum_type = new EnumType(cur_decl_type_id->Name()); - - // For documentation purposes, a separate type object is created - // in order to avoid overlap that can be caused by redefs. - if ( generate_documentation ) - cur_enum_type_doc = new CommentedEnumType(cur_decl_type_id->Name()); + cur_enum_type = new EnumType(); } static void parser_redef_enum (ID *id) @@ -160,53 +142,75 @@ static void parser_redef_enum (ID *id) if ( ! cur_enum_type ) id->Error("not an enum"); } - - if ( generate_documentation ) - cur_enum_type_doc = new CommentedEnumType(id->Name()); } -static void add_enum_comment (std::list* comments) +static type_decl_list* copy_type_decl_list(type_decl_list* tdl) { - cur_enum_type_doc->AddComment(current_module, cur_enum_elem_id, comments); - } + if ( ! tdl ) + return 0; -static ID* create_dummy_id (ID* id, BroType* type) - { - ID* fake_id = new ID(copy_string(id->Name()), (IDScope) id->Scope(), - is_export); + type_decl_list* rval = new type_decl_list(); - fake_id->SetType(type->Ref()); - - if ( id->AsType() ) + loop_over_list(*tdl, i) { - type->SetTypeID(copy_string(id->Name())); - fake_id->MakeType(); + TypeDecl* td = (*tdl)[i]; + rval->append(new TypeDecl(*td)); } - return fake_id; + return rval; } -static std::list* concat_opt_docs (std::list* pre, - std::list* post) +static attr_list* copy_attr_list(attr_list* al) { - if ( ! pre && ! post ) return 0; + if ( ! al ) + return 0; - if ( pre && ! post ) return pre; + attr_list* rval = new attr_list(); - if ( ! pre && post ) return post; + loop_over_list(*al, i) + { + Attr* a = (*al)[i]; + ::Ref(a); + rval->append(a); + } - pre->splice(pre->end(), *post); - delete post; - - return pre; + return rval; } +static void extend_record(ID* id, type_decl_list* fields, attr_list* attrs) + { + set types = type_aliases[id->Name()]; + + if ( types.empty() ) + { + id->Error("failed to redef record: no types found in alias map"); + return; + } + + for ( set::const_iterator it = types.begin(); it != types.end(); ) + { + RecordType* add_to = (*it)->AsRecordType(); + const char* error = 0; + ++it; + + if ( it == types.end() ) + error = add_to->AddFields(fields, attrs); + else + error = add_to->AddFields(copy_type_decl_list(fields), + copy_attr_list(attrs)); + + if ( error ) + { + id->Error(error); + break; + } + } + } %} %union { bool b; char* str; - std::list* str_l; ID* id; id_list* id_l; init_class ic; @@ -699,46 +703,24 @@ single_pattern: ; enum_body: - enum_body_list opt_post_doc_list + enum_body_list { $$ = cur_enum_type; - - if ( generate_documentation ) - { - add_enum_comment($2); - cur_enum_elem_id = 0; - } - cur_enum_type = NULL; } - | enum_body_list ',' opt_post_doc_list + | enum_body_list ',' { $$ = cur_enum_type; - - if ( generate_documentation ) - { - add_enum_comment($3); - cur_enum_elem_id = 0; - } - cur_enum_type = NULL; } ; enum_body_list: - enum_body_elem opt_post_doc_list - { - if ( generate_documentation ) - add_enum_comment($2); - } + enum_body_elem - | enum_body_list ',' opt_post_doc_list - { - if ( generate_documentation ) - add_enum_comment($3); - } enum_body_elem -; + | enum_body_list ',' enum_body_elem + ; enum_body_elem: /* TODO: We could also define this as TOK_ID '=' expr, (or @@ -746,25 +728,19 @@ enum_body_elem: error messages if someboy tries to use constant variables as enumerator. */ - opt_doc_list TOK_ID '=' TOK_CONSTANT + TOK_ID '=' TOK_CONSTANT { - set_location(@2, @4); + set_location(@1, @3); assert(cur_enum_type); - if ( $4->Type()->Tag() != TYPE_COUNT ) + if ( $3->Type()->Tag() != TYPE_COUNT ) reporter->Error("enumerator is not a count constant"); else - cur_enum_type->AddName(current_module, $2, $4->InternalUnsigned(), is_export); - - if ( generate_documentation ) - { - cur_enum_type_doc->AddName(current_module, $2, $4->InternalUnsigned(), is_export); - cur_enum_elem_id = $2; - add_enum_comment($1); - } + cur_enum_type->AddName(current_module, $1, + $3->InternalUnsigned(), is_export); } - | opt_doc_list TOK_ID '=' '-' TOK_CONSTANT + | TOK_ID '=' '-' TOK_CONSTANT { /* We only accept counts as enumerator, but we want to return a nice error message if users triy to use a negative integer (will also @@ -773,18 +749,11 @@ enum_body_elem: reporter->Error("enumerator is not a count constant"); } - | opt_doc_list TOK_ID + | TOK_ID { - set_location(@2); + set_location(@1); assert(cur_enum_type); - cur_enum_type->AddName(current_module, $2, is_export); - - if ( generate_documentation ) - { - cur_enum_type_doc->AddName(current_module, $2, is_export); - cur_enum_elem_id = $2; - add_enum_comment($1); - } + cur_enum_type->AddName(current_module, $1, is_export); } ; @@ -872,12 +841,11 @@ type: } | TOK_RECORD '{' - { ++in_record; do_doc_token_start(); } + { ++in_record; } type_decl_list { --in_record; } '}' { - do_doc_token_stop(); set_location(@1, @5); $$ = new RecordType($4); } @@ -889,9 +857,8 @@ type: $$ = 0; } - | TOK_ENUM '{' { set_location(@1); parser_new_enum(); do_doc_token_start(); } enum_body '}' + | TOK_ENUM '{' { set_location(@1); parser_new_enum(); } enum_body '}' { - do_doc_token_stop(); set_location(@1, @5); $4->UpdateLocationEndInfo(@5); $$ = $4; @@ -983,45 +950,21 @@ type_decl_list: type_decl_list type_decl { $1->append($2); - - if ( generate_documentation && last_fake_type_decl ) - { - fake_type_decl_list->append(last_fake_type_decl); - last_fake_type_decl = 0; - } } | { $$ = new type_decl_list(); - - if ( generate_documentation ) - fake_type_decl_list = new type_decl_list(); } ; type_decl: - opt_doc_list TOK_ID ':' type opt_attr ';' opt_post_doc_list + TOK_ID ':' type opt_attr ';' { - set_location(@2, @6); + set_location(@1, @4); + $$ = new TypeDecl($3, $1, $4, (in_record > 0)); - if ( generate_documentation ) - { - // TypeDecl ctor deletes the attr list, so make a copy - attr_list* a = $5; - attr_list* a_copy = 0; - - if ( a ) - { - a_copy = new attr_list; - loop_over_list(*a, i) - a_copy->append((*a)[i]); - } - - last_fake_type_decl = new CommentedTypeDecl( - $4, $2, a_copy, (in_record > 0), concat_opt_docs($1, $7)); - } - - $$ = new TypeDecl($4, $2, $5, (in_record > 0)); + if ( in_record > 0 ) + broxygen_mgr->RecordField(cur_decl_type_id, $$, ::filename); } ; @@ -1055,9 +998,7 @@ decl: TOK_MODULE TOK_ID ';' { current_module = $2; - - if ( generate_documentation ) - current_reST_doc->AddModule(current_module); + broxygen_mgr->ModuleUsage(::filename, current_module); } | TOK_EXPORT '{' { is_export = true; } decl_list '}' @@ -1066,171 +1007,43 @@ decl: | TOK_GLOBAL def_global_id opt_type init_class opt_init opt_attr ';' { add_global($2, $3, $4, $5, $6, VAR_REGULAR); - - if ( generate_documentation ) - { - ID* id = $2; - if ( id->Type()->Tag() == TYPE_FUNC ) - { - switch ( id->Type()->AsFuncType()->Flavor() ) { - - case FUNC_FLAVOR_FUNCTION: - current_reST_doc->AddFunction( - new BroDocObj(id, reST_doc_comments)); - break; - - case FUNC_FLAVOR_EVENT: - current_reST_doc->AddEvent( - new BroDocObj(id, reST_doc_comments)); - break; - - case FUNC_FLAVOR_HOOK: - current_reST_doc->AddHook( - new BroDocObj(id, reST_doc_comments)); - break; - - default: - reporter->InternalError("invalid function flavor"); - break; - } - } - - else - { - current_reST_doc->AddStateVar( - new BroDocObj(id, reST_doc_comments)); - } - } + broxygen_mgr->Identifier($2); } | TOK_CONST def_global_id opt_type init_class opt_init opt_attr ';' { add_global($2, $3, $4, $5, $6, VAR_CONST); - - if ( generate_documentation ) - { - if ( $2->FindAttr(ATTR_REDEF) ) - current_reST_doc->AddOption( - new BroDocObj($2, reST_doc_comments)); - else - current_reST_doc->AddConstant( - new BroDocObj($2, reST_doc_comments)); - } + broxygen_mgr->Identifier($2); } | TOK_REDEF global_id opt_type init_class opt_init opt_attr ';' { add_global($2, $3, $4, $5, $6, VAR_REDEF); - - if ( generate_documentation && - ! streq("capture_filters", $2->Name()) ) - { - ID* fake_id = create_dummy_id($2, $2->Type()); - BroDocObj* o = new BroDocObj(fake_id, reST_doc_comments, true); - o->SetRole(true); - current_reST_doc->AddRedef(o); - } + broxygen_mgr->Redef($2, ::filename); } | TOK_REDEF TOK_ENUM global_id TOK_ADD_TO - '{' { parser_redef_enum($3); do_doc_token_start(); } enum_body '}' ';' + '{' { parser_redef_enum($3); } enum_body '}' ';' { - do_doc_token_stop(); - - if ( generate_documentation ) - { - ID* fake_id = create_dummy_id($3, cur_enum_type_doc); - cur_enum_type_doc = 0; - BroDocObj* o = new BroDocObj(fake_id, reST_doc_comments, true); - o->SetRole(true); - - if ( extract_module_name(fake_id->Name()) == "Notice" && - extract_var_name(fake_id->Name()) == "Type" ) - current_reST_doc->AddNotice(o); - else - current_reST_doc->AddRedef(o); - } + // Broxygen already grabbed new enum IDs as the type created them. } - | TOK_REDEF TOK_RECORD global_id TOK_ADD_TO - '{' { ++in_record; do_doc_token_start(); } - type_decl_list - { --in_record; do_doc_token_stop(); } '}' opt_attr ';' + | TOK_REDEF TOK_RECORD global_id { cur_decl_type_id = $3; } TOK_ADD_TO + '{' { ++in_record; } type_decl_list { --in_record; } '}' opt_attr ';' { + cur_decl_type_id = 0; + if ( ! $3->Type() ) $3->Error("unknown identifier"); else - { - RecordType* add_to = $3->Type()->AsRecordType(); - if ( ! add_to ) - $3->Error("not a record type"); - else - { - const char* error = add_to->AddFields($7, $10); - if ( error ) - $3->Error(error); - else if ( generate_documentation ) - { - if ( fake_type_decl_list ) - { - BroType* fake_record = - new RecordType(fake_type_decl_list); - ID* fake = create_dummy_id($3, fake_record); - fake_type_decl_list = 0; - BroDocObj* o = - new BroDocObj(fake, reST_doc_comments, true); - o->SetRole(true); - current_reST_doc->AddRedef(o); - } - else - { - fprintf(stderr, "Warning: doc mode did not process " - "record extension for '%s', CommentedTypeDecl" - "list unavailable.\n", $3->Name()); - } - } - } - } + extend_record($3, $8, $11); } | TOK_TYPE global_id ':' { cur_decl_type_id = $2; } type opt_attr ';' { cur_decl_type_id = 0; - add_type($2, $5, $6, 0); - - if ( generate_documentation ) - { - TypeTag t = $2->AsType()->Tag(); - if ( t == TYPE_ENUM && cur_enum_type_doc ) - { - ID* fake = create_dummy_id($2, cur_enum_type_doc); - cur_enum_type_doc = 0; - current_reST_doc->AddType( - new BroDocObj(fake, reST_doc_comments, true)); - } - - else if ( t == TYPE_RECORD && fake_type_decl_list ) - { - BroType* fake_record = new RecordType(fake_type_decl_list); - ID* fake = create_dummy_id($2, fake_record); - fake_type_decl_list = 0; - current_reST_doc->AddType( - new BroDocObj(fake, reST_doc_comments, true)); - } - - else - current_reST_doc->AddType( - new BroDocObj($2, reST_doc_comments)); - } - } - - | TOK_EVENT event_id ':' type_list opt_attr ';' - { - add_type($2, $4, $5, 1); - - if ( generate_documentation ) - current_reST_doc->AddEvent( - new BroDocObj($2, reST_doc_comments)); + add_type($2, $5, $6); + broxygen_mgr->Identifier($2); } | func_hdr func_body @@ -1258,18 +1071,13 @@ func_hdr: begin_func($2, current_module.c_str(), FUNC_FLAVOR_FUNCTION, 0, $3); $$ = $3; - if ( generate_documentation ) - current_reST_doc->AddFunction( - new BroDocObj($2, reST_doc_comments)); + broxygen_mgr->Identifier($2); } | TOK_EVENT event_id func_params { begin_func($2, current_module.c_str(), FUNC_FLAVOR_EVENT, 0, $3); $$ = $3; - if ( generate_documentation ) - current_reST_doc->AddEventHandler( - new BroDocObj($2, reST_doc_comments)); } | TOK_HOOK def_global_id func_params { @@ -1278,9 +1086,6 @@ func_hdr: begin_func($2, current_module.c_str(), FUNC_FLAVOR_HOOK, 0, $3); $$ = $3; - if ( generate_documentation ) - current_reST_doc->AddHookHandler( - new BroDocObj($2, reST_doc_comments)); } | TOK_REDEF TOK_EVENT event_id func_params { @@ -1729,40 +1534,6 @@ resolve_id: } ; -opt_post_doc_list: - opt_post_doc_list TOK_POST_DOC - { - $1->push_back($2); - $$ = $1; - } - | - TOK_POST_DOC - { - $$ = new std::list(); - $$->push_back($1); - delete [] $1; - } - | - { $$ = 0; } - ; - -opt_doc_list: - opt_doc_list TOK_DOC - { - $1->push_back($2); - $$ = $1; - } - | - TOK_DOC - { - $$ = new std::list(); - $$->push_back($1); - delete [] $1; - } - | - { $$ = 0; } - ; - opt_no_test: TOK_NO_TEST { $$ = true; } @@ -1788,10 +1559,6 @@ int yyerror(const char msg[]) else sprintf(msgbuf, "%s, at or near \"%s\"", msg, last_tok); - if ( generate_documentation ) - strcat(msgbuf, "\nDocumentation mode is enabled: " - "remember to check syntax of ## style comments\n"); - if ( in_debug ) g_curr_debug_error = copy_string(msg); diff --git a/src/plugin/ComponentManager.h b/src/plugin/ComponentManager.h index 16f9d80743..7d90f93bf1 100644 --- a/src/plugin/ComponentManager.h +++ b/src/plugin/ComponentManager.h @@ -130,9 +130,9 @@ template ComponentManager::ComponentManager(const string& arg_module) : module(arg_module) { - tag_enum_type = new EnumType(module + "::Tag"); + tag_enum_type = new EnumType(); ::ID* id = install_ID("Tag", module.c_str(), true, true); - add_type(id, tag_enum_type, 0, 0); + add_type(id, tag_enum_type, 0); } template diff --git a/src/scan.l b/src/scan.l index 636ec5b251..b126b2ee1f 100644 --- a/src/scan.l +++ b/src/scan.l @@ -23,16 +23,15 @@ #include "Debug.h" #include "PolicyFile.h" #include "broparse.h" -#include "BroDoc.h" #include "Reporter.h" #include "RE.h" #include "Net.h" #include "analyzer/Analyzer.h" +#include "broxygen/Manager.h" extern YYLTYPE yylloc; // holds start line and column of token extern int print_loaded_scripts; -extern int generate_documentation; // Track the @if... depth. ptr_compat_int current_depth = 0; @@ -40,10 +39,7 @@ ptr_compat_int current_depth = 0; int_list if_stack; int line_number = 1; -const char* filename = 0; -BroDoc* current_reST_doc = 0; -static BroDoc* last_reST_doc = 0; -string current_scanned_file_path; +const char* filename = 0; // Absolute path of file currently being parsed. char last_tok[128]; @@ -56,41 +52,15 @@ char last_tok[128]; if ( ((result = fread(buf, 1, max_size, yyin)) == 0) && ferror(yyin) ) \ reporter->Error("read failed with \"%s\"", strerror(errno)); -// reST documents that we've created (or have at least opened so far). -std::list docs_generated; - -// reST comments (those starting with ##) seen so far. -std::list* reST_doc_comments = 0; - -// Print current contents of reST_doc_comments list to stderr. -void print_current_reST_doc_comments(); - -// Delete the reST_doc_comments list object. -void clear_reST_doc_comments(); - -// Adds changes to capture_filter to the current script's reST documentation. -static void check_capture_filter_changes(); - -static const char* canon_doc_comment(const char* comment) +static string get_dirname(const char* path) { - // "##Text" and "## Text" are treated the same in order to be able - // to still preserve indentation level, but not unintentionally - // signify an indentation level for all the text when using - // the "## Text" style. - return ( comment[0] == ' ' ) ? comment + 1 : comment; - } + if ( ! path ) + return ""; -static std::string canon_doc_func_param(const char* id_start) - { - std::string id_name(id_start, strcspn(id_start, ":")); - const char* comment = id_start + id_name.size() + 1; - std::string doc; - - if ( id_name == "Returns" ) - doc.append(":returns:").append(comment); - else - doc.append(":param ").append(id_name).append(":").append(comment); - return doc; + char* tmp = copy_string(path); + string rval = dirname(tmp); + delete[] tmp; + return rval; } static ino_t get_inode_num(FILE* f, const char* filename) @@ -99,7 +69,8 @@ static ino_t get_inode_num(FILE* f, const char* filename) if ( fstat(fileno(f), &b) ) { - reporter->Error("failed to fstat fd of %s\n", filename); + reporter->Error("failed to fstat fd of %s: %s\n", filename, + strerror(errno)); exit(1); } @@ -116,8 +87,6 @@ public: const char* name; int line; int level; - BroDoc* doc; - string path; }; // A stack of input buffers we're scanning. file_stack[len-1] is the @@ -141,7 +110,6 @@ static int load_files(const char* file); %x RE %x IGNORE -%s DOC OWS [ \t]* WS [ \t]+ @@ -159,63 +127,15 @@ ESCSEQ (\\([^\n]|[0-7]+|x[[:xdigit:]]+)) %% ##!.* { - // Add this format of comments to the script documentation's "summary". - if ( generate_documentation ) - current_reST_doc->AddSummary(canon_doc_comment(yytext + 3)); + broxygen_mgr->SummaryComment(::filename, yytext + 3); } -##<.* { - yylval.str = copy_string(canon_doc_comment(yytext + 3)); - return TOK_POST_DOC; -} - -##{OWS}{ID}:{WS}.* { - const char* id_start = skip_whitespace(yytext + 2); - yylval.str = copy_string(canon_doc_func_param(id_start).c_str()); - return TOK_DOC; -} - -##.* { - if ( yytext[2] != '#' ) - { - yylval.str = copy_string(canon_doc_comment(yytext + 2)); - return TOK_DOC; - } -} - -##{OWS}{ID}:{WS}.* { - if ( generate_documentation ) - { - // Comment is documenting either a function parameter or return type, - // so appropriate reST markup substitutions are automatically made - // in order to distinguish them from other comments. - if ( ! reST_doc_comments ) - reST_doc_comments = new std::list(); - - // always insert a blank line so that this param/return markup - // 1) doesn't show up in the summary section in the case that it's - // the first comment for the function/event - // 2) has a blank line between it and non-field-list reST markup, - // which is required for correct HTML rendering by Sphinx - reST_doc_comments->push_back(""); - const char* id_start = skip_whitespace(yytext + 2); - reST_doc_comments->push_back(canon_doc_func_param(id_start)); - } -} - ##<.* { - if ( generate_documentation && BroDocObj::last ) - BroDocObj::last->AddDocString(canon_doc_comment(yytext + 3)); + broxygen_mgr->PostComment(yytext + 3); } ##.* { - if ( generate_documentation && (yytext[2] != '#') ) - { - if ( ! reST_doc_comments ) - reST_doc_comments = new std::list(); - - reST_doc_comments->push_back(canon_doc_comment(yytext + 2)); - } + broxygen_mgr->PreComment(yytext + 2); } #{OWS}@no-test.* return TOK_NO_TEST; @@ -224,7 +144,7 @@ ESCSEQ (\\([^\n]|[0-7]+|x[[:xdigit:]]+)) {WS} /* eat whitespace */ -\n { +\n { ++line_number; ++yylloc.first_line; ++yylloc.last_line; @@ -345,7 +265,7 @@ when return TOK_WHEN; @DEBUG return TOK_DEBUG; // marks input for debugger @DIR { - string rval = current_scanned_file_path; + string rval = get_dirname(::filename); if ( ! rval.empty() && rval[0] == '.' ) { @@ -374,25 +294,15 @@ when return TOK_WHEN; @load{WS}{FILE} { const char* new_file = skip_whitespace(yytext + 5); // Skip "@load". - if ( generate_documentation ) - { - current_reST_doc->AddImport(new_file); - - if ( reST_doc_comments ) - { - fprintf(stderr, "Warning: unconsumed reST documentation is being " - "discarded before doing '@load %s' in %s:\n", - new_file, current_reST_doc->GetSourceFileName()); - clear_reST_doc_comments(); - } - } + broxygen_mgr->ScriptDependency(::filename, new_file); (void) load_files(new_file); } @load-sigs{WS}{FILE} { const char* new_sig_file = skip_whitespace(yytext + 10); const char* full_filename = 0; - FILE* f = search_for_file(new_sig_file, "sig", &full_filename, false, 0); + FILE* f = search_for_file(new_sig_file, "sig", &full_filename, false, 0, + get_dirname(::filename)); if ( f ) { @@ -411,7 +321,8 @@ when return TOK_WHEN; // All we have to do is pretend we've already scanned it. const char* full_filename; - FILE* f = search_for_file(new_file, "bro", &full_filename, true, 0); + FILE* f = search_for_file(new_file, "bro", &full_filename, true, 0, + get_dirname(::filename)); if ( f ) { @@ -603,7 +514,7 @@ static int load_files(const char* orig_file) else { - f = search_for_file(orig_file, "bro", &full_filename, true, &bropath_subpath); + f = search_for_file(orig_file, "bro", &full_filename, true, &bropath_subpath, get_dirname(::filename)); bropath_subpath_delete = bropath_subpath; // This will be deleted. } @@ -666,15 +577,7 @@ static int load_files(const char* orig_file) else file_stack.append(new FileInfo); - char* tmp = copy_string(full_filename); - current_scanned_file_path = dirname(tmp); - delete [] tmp; - - if ( generate_documentation ) - { - current_reST_doc = new BroDoc(bropath_subpath, full_filename); - docs_generated.push_back(current_reST_doc); - } + broxygen_mgr->File(full_filename); delete [] bropath_subpath_delete; @@ -776,28 +679,11 @@ void do_atendif() --current_depth; } -void do_doc_token_start() - { - if ( generate_documentation ) - BEGIN(DOC); - } - -void do_doc_token_stop() - { - if ( generate_documentation ) - BEGIN(INITIAL); - } - // Be careful to never delete things from this list, as the strings // are referred to (in order to save the locations of tokens and statements, // for error reporting and debugging). static name_list input_files; -const char* get_current_input_filename() - { - return ::filename; - } - void add_input_file(const char* file) { if ( ! file ) @@ -852,8 +738,6 @@ int yywrap() // Stack is now empty. while ( input_files.length() > 0 ) { - check_capture_filter_changes(); - if ( load_files(input_files[0]) ) { // Don't delete the filename - it's pointed to by @@ -867,8 +751,6 @@ int yywrap() (void) input_files.remove_nth(0); } - check_capture_filter_changes(); - // For each file scanned so far, and for each @prefix, look for a // prefixed and flattened version of the loaded file in BROPATH. The // flattening involves taking the path in BROPATH in which the @@ -893,7 +775,8 @@ int yywrap() string s; s = dot_canon(it->subpath.c_str(), it->name.c_str(), prefixes[i]); - FILE* f = search_for_file(s.c_str(), "bro", 0, false, 0); + FILE* f = search_for_file(s.c_str(), "bro", 0, false, 0, + get_dirname(::filename)); //printf("====== prefix search ======\n"); //printf("File : %s\n", it->name.c_str()); @@ -977,9 +860,6 @@ int yywrap() return 0; } - if ( generate_documentation ) - clear_reST_doc_comments(); - // Otherwise, we are done. return 1; } @@ -990,8 +870,6 @@ FileInfo::FileInfo(string arg_restore_module) restore_module = arg_restore_module; name = ::filename; line = ::line_number; - doc = ::current_reST_doc; - path = current_scanned_file_path; } FileInfo::~FileInfo() @@ -1002,56 +880,7 @@ FileInfo::~FileInfo() yy_switch_to_buffer(buffer_state); yylloc.filename = filename = name; yylloc.first_line = yylloc.last_line = line_number = line; - last_reST_doc = current_reST_doc; - current_reST_doc = doc; - current_scanned_file_path = path; if ( restore_module != "" ) current_module = restore_module; } - -static void check_capture_filter_changes() - { - if ( ! generate_documentation ) - return; - - // Lookup the "capture_filters" identifier, if it has any defined - // value, add it to the script's reST documentation, and finally - // clear the table so it doesn't taint the documentation for - // subsequent scripts. - - ID* capture_filters = global_scope()->Lookup("capture_filters"); - - if ( capture_filters ) - { - ODesc desc; - desc.SetIndentSpaces(4); - capture_filters->ID_Val()->Describe(&desc); - last_reST_doc->SetPacketFilter(desc.Description()); - capture_filters->ID_Val()->AsTableVal()->RemoveAll(); - } - } - -void print_current_reST_doc_comments() - { - if ( ! reST_doc_comments ) - return; - - std::list::iterator it; - - for ( it = reST_doc_comments->begin(); it != reST_doc_comments->end(); ++it ) - fprintf(stderr, "##%s\n", it->c_str()); - } - -void clear_reST_doc_comments() - { - if ( ! reST_doc_comments ) - return; - - fprintf(stderr, "Warning: %zu unconsumed reST comments:\n", - reST_doc_comments->size()); - - print_current_reST_doc_comments(); - delete reST_doc_comments; - reST_doc_comments = 0; - } diff --git a/src/util.cc b/src/util.cc index f26b3fb0c2..d06c6aff96 100644 --- a/src/util.cc +++ b/src/util.cc @@ -1060,11 +1060,9 @@ void get_script_subpath(const std::string& full_filename, const char** subpath) *subpath = normalize_path(my_subpath.c_str()); } -extern string current_scanned_file_path; - FILE* search_for_file(const char* filename, const char* ext, const char** full_filename, bool load_pkgs, - const char** bropath_subpath) + const char** bropath_subpath, string prepend_to_search_path) { // If the file is a literal absolute path we don't have to search, // just return the result of trying to open it. If the file is @@ -1088,9 +1086,9 @@ FILE* search_for_file(const char* filename, const char* ext, // Prepend the currently loading script's path to BROPATH so that // @loads can be referenced relatively. - if ( current_scanned_file_path != "" && filename[0] == '.' ) + if ( ! prepend_to_search_path.empty() && filename[0] == '.' ) safe_snprintf(path, sizeof(path), "%s:%s", - current_scanned_file_path.c_str(), bro_path()); + prepend_to_search_path.c_str(), bro_path()); else safe_strncpy(path, bro_path(), sizeof(path)); diff --git a/src/util.h b/src/util.h index fcdfd6d499..e7ac4d33af 100644 --- a/src/util.h +++ b/src/util.h @@ -209,7 +209,8 @@ std::string dot_canon(std::string path, std::string file, std::string prefix = " const char* normalize_path(const char* path); void get_script_subpath(const std::string& full_filename, const char** subpath); extern FILE* search_for_file(const char* filename, const char* ext, - const char** full_filename, bool load_pkgs, const char** bropath_subpath); + const char** full_filename, bool load_pkgs, const char** bropath_subpath, + std::string prepend_to_search_path = ""); // Renames the given file to a new temporary name, and opens a new file with // the original name. Returns new file or NULL on error. Inits rotate_info if From 90477df973ae34896e265c4def5547fc6540f6c3 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Mon, 7 Oct 2013 15:01:03 -0500 Subject: [PATCH 02/40] Refactor search_for_file() util function. It was getting too bloated and allocated memory in ways that were difficult to understand how to manage. Separated out primarily in to new find_file() and open_file()/open_package() functions. Also renamed other util functions for path-related things. --- src/BroDoc.cc | 2 + src/Debug.cc | 10 +- src/Net.h | 8 +- src/OSFinger.cc | 3 +- src/RuleMatcher.cc | 2 +- src/broxygen/Manager.cc | 2 +- src/scan.l | 252 ++++++++++++++++------------------ src/util.cc | 292 ++++++++++++++++++++-------------------- src/util.h | 64 ++++++++- 9 files changed, 331 insertions(+), 304 deletions(-) diff --git a/src/BroDoc.cc b/src/BroDoc.cc index 05d9d8f562..2c3d6f6d77 100644 --- a/src/BroDoc.cc +++ b/src/BroDoc.cc @@ -85,6 +85,7 @@ BroDoc::~BroDoc() void BroDoc::AddImport(const std::string& s) { + /* std::string lname(s); // First strip any .bro extension. size_t ext_pos = lname.find(".bro"); @@ -142,6 +143,7 @@ void BroDoc::AddImport(const std::string& s) delete [] full_filename; delete [] subpath; + */ } void BroDoc::SetPacketFilter(const std::string& s) diff --git a/src/Debug.cc b/src/Debug.cc index f4ac8c2fdf..94b8abf952 100644 --- a/src/Debug.cc +++ b/src/Debug.cc @@ -342,18 +342,16 @@ vector parse_location_string(const string& s) if ( ! sscanf(line_string.c_str(), "%d", &plr.line) ) plr.type = plrUnknown; - FILE* throwaway = search_for_file(filename.c_str(), "bro", - &full_filename, true, 0); - if ( ! throwaway ) + string path(find_file(filename, bro_path(), "bro")); + + if ( path.empty() ) { debug_msg("No such policy file: %s.\n", filename.c_str()); plr.type = plrUnknown; return result; } - fclose(throwaway); - - loc_filename = full_filename; + loc_filename = copy_string(path.c_str()); plr.type = plrFileAndLine; } } diff --git a/src/Net.h b/src/Net.h index 5b959d1688..07c856d1dd 100644 --- a/src/Net.h +++ b/src/Net.h @@ -4,6 +4,7 @@ #define net_h #include "net_util.h" +#include "util.h" #include "BPF_Program.h" #include "List.h" #include "PktSrc.h" @@ -97,15 +98,14 @@ struct ScannedFile { ino_t inode; int include_level; string name; - string subpath; // Path in BROPATH's policy/ containing the file. bool skipped; // This ScannedFile was @unload'd. bool prefixes_checked; // If loading prefixes for this file has been tried. - ScannedFile(ino_t arg_inode, int arg_include_level, string arg_name, - string arg_subpath = "", bool arg_skipped = false, + ScannedFile(ino_t arg_inode, int arg_include_level, const string& arg_name, + bool arg_skipped = false, bool arg_prefixes_checked = false) : inode(arg_inode), include_level(arg_include_level), - name(arg_name), subpath(arg_subpath), skipped(arg_skipped), + name(arg_name), skipped(arg_skipped), prefixes_checked(arg_prefixes_checked) { } }; diff --git a/src/OSFinger.cc b/src/OSFinger.cc index 3368a8e40c..bcb00e324b 100644 --- a/src/OSFinger.cc +++ b/src/OSFinger.cc @@ -294,7 +294,8 @@ void OSFingerprint::load_config(const char* file) uint32 ln=0; char buf[MAXLINE]; char* p; - FILE* c = search_for_file(file, "osf", 0, false, 0); + + FILE* c = open_file(find_file(file, bro_path(), "osf")); if (!c) { diff --git a/src/RuleMatcher.cc b/src/RuleMatcher.cc index ed33db6792..84ff5af774 100644 --- a/src/RuleMatcher.cc +++ b/src/RuleMatcher.cc @@ -226,7 +226,7 @@ bool RuleMatcher::ReadFiles(const name_list& files) for ( int i = 0; i < files.length(); ++i ) { - rules_in = search_for_file(files[i], "sig", 0, false, 0); + rules_in = open_file(find_file(files[i], bro_path(), "sig")); if ( ! rules_in ) { reporter->Error("Can't open signature file %s", files[i]); diff --git a/src/broxygen/Manager.cc b/src/broxygen/Manager.cc index 30c9008787..ee995d8b4a 100644 --- a/src/broxygen/Manager.cc +++ b/src/broxygen/Manager.cc @@ -20,8 +20,8 @@ void Manager::GenerateDocs() const void Manager::File(const std::string& path) { // TODO - // determine bropath subpath // can be a file or directory? + // determine path within BROPATH } void Manager::ScriptDependency(const std::string& path, const std::string& dep) diff --git a/src/scan.l b/src/scan.l index b126b2ee1f..899fa77b66 100644 --- a/src/scan.l +++ b/src/scan.l @@ -31,7 +31,6 @@ #include "broxygen/Manager.h" extern YYLTYPE yylloc; // holds start line and column of token -extern int print_loaded_scripts; // Track the @if... depth. ptr_compat_int current_depth = 0; @@ -52,31 +51,38 @@ char last_tok[128]; if ( ((result = fread(buf, 1, max_size, yyin)) == 0) && ferror(yyin) ) \ reporter->Error("read failed with \"%s\"", strerror(errno)); -static string get_dirname(const char* path) +static string find_relative_file(const string& filename, const string& ext) { - if ( ! path ) - return ""; + if ( filename.empty() ) + return string(); - char* tmp = copy_string(path); - string rval = dirname(tmp); - delete[] tmp; - return rval; + if ( filename[0] == '.' ) + return find_file(filename, safe_dirname(::filename), ext); + else + return find_file(filename, bro_path(), ext); } -static ino_t get_inode_num(FILE* f, const char* filename) +static ino_t get_inode_num(FILE* f, const string& path) { struct stat b; if ( fstat(fileno(f), &b) ) - { - reporter->Error("failed to fstat fd of %s: %s\n", filename, - strerror(errno)); - exit(1); - } + reporter->FatalError("fstat of %s failed: %s\n", path.c_str(), + strerror(errno)); return b.st_ino; } +static ino_t get_inode_num(const string& path) + { + FILE* f = open_file(path); + + if ( ! f ) + reporter->FatalError("failed to open %s\n", path.c_str()); + + return get_inode_num(f, path); + } + class FileInfo { public: FileInfo(string restore_module = ""); @@ -265,7 +271,7 @@ when return TOK_WHEN; @DEBUG return TOK_DEBUG; // marks input for debugger @DIR { - string rval = get_dirname(::filename); + string rval(safe_dirname(::filename)); if ( ! rval.empty() && rval[0] == '.' ) { @@ -299,42 +305,29 @@ when return TOK_WHEN; } @load-sigs{WS}{FILE} { - const char* new_sig_file = skip_whitespace(yytext + 10); - const char* full_filename = 0; - FILE* f = search_for_file(new_sig_file, "sig", &full_filename, false, 0, - get_dirname(::filename)); + const char* file = skip_whitespace(yytext + 10); + string path(find_relative_file(file, "sig")); - if ( f ) - { - sig_files.push_back(full_filename); - fclose(f); - delete [] full_filename; - } - else + if ( path.empty() ) reporter->Error("failed to find file associated with @load-sigs %s", - new_sig_file); + file); + else + sig_files.push_back(copy_string(path.c_str())); } @unload{WS}{FILE} { // Skip "@unload". - const char* new_file = skip_whitespace(yytext + 7); - - // All we have to do is pretend we've already scanned it. - const char* full_filename; - FILE* f = search_for_file(new_file, "bro", &full_filename, true, 0, - get_dirname(::filename)); - - if ( f ) - { - ScannedFile sf(get_inode_num(f, full_filename), file_stack.length(), full_filename, "", true); - files_scanned.push_back(sf); - - fclose(f); - delete [] full_filename; - } + const char* file = skip_whitespace(yytext + 7); + string path(find_relative_file(file, "bro")); + if ( path.empty() ) + reporter->Error("failed find file associated with @unload %s", file); else - reporter->Error("failed find file associated with @unload %s", new_file); + { + // All we have to do is pretend we've already scanned it. + ScannedFile sf(get_inode_num(path), file_stack.length(), path, true); + files_scanned.push_back(sf); + } } @prefixes{WS}("+"?)={WS}{PREFIX} { @@ -488,22 +481,35 @@ YYLTYPE GetCurrentLocation() return currloc; } + +static bool already_scanned(ino_t i) + { + list::const_iterator it; + + for ( it = files_scanned.begin(); it != files_scanned.end(); ++it ) + if ( it->inode == i ) + return true; + + return false; + } + +static bool already_scanned(const string& path) + { + return already_scanned(get_inode_num(path)); + } + static int load_files(const char* orig_file) { // Whether we pushed on a FileInfo that will restore the // current module after the final file has been scanned. bool did_module_restore = false; - - const char* full_filename = ""; - const char* bropath_subpath = ""; - const char* bropath_subpath_delete = 0; - FILE* f; + string file_path; + FILE* f = 0; if ( streq(orig_file, "-") ) { f = stdin; - full_filename = ""; - bropath_subpath = ""; + file_path = ""; if ( g_policy_debug ) { @@ -514,90 +520,62 @@ static int load_files(const char* orig_file) else { - f = search_for_file(orig_file, "bro", &full_filename, true, &bropath_subpath, get_dirname(::filename)); - bropath_subpath_delete = bropath_subpath; // This will be deleted. - } + file_path = find_relative_file(orig_file, "bro"); - if ( f ) - { - ino_t i = get_inode_num(f, full_filename); - std::list::const_iterator it; + if ( file_path.empty() ) + reporter->FatalError("can't find %s", orig_file); - for ( it = files_scanned.begin(); it != files_scanned.end(); ++it ) - { - if ( it->inode == i ) - { - if ( f != stdin ) - { - fclose(f); - delete [] full_filename; - delete [] bropath_subpath_delete; - } - return 0; - } - } - - ScannedFile sf(i, file_stack.length(), full_filename, bropath_subpath); - files_scanned.push_back(sf); - - if ( g_policy_debug ) - { - // Add the filename to the file mapping - // table (Debug.h). - Filemap* map = new Filemap; - - // Make sure it wasn't already read in. - HashKey* key = new HashKey(full_filename); - if ( g_dbgfilemaps.Lookup(key) ) - { - // reporter->Warning("Not re-reading policy file; check BRO_PREFIXES:", full_filename); - fclose(f); - delete key; - return 0; - } - else - { - g_dbgfilemaps.Insert(key, map); - } - - if ( full_filename ) - LoadPolicyFileText(full_filename); - } - - // Remember where we were. If this is the first - // file being pushed on the stack, i.e., the *last* - // one that will be processed, then we want to - // restore the module scope in which this @load - // was done when we're finished processing it. - if ( ! did_module_restore ) - { - file_stack.append(new FileInfo(current_module)); - did_module_restore = true; - } + if ( is_dir(file_path.c_str()) ) + f = open_package(file_path); else - file_stack.append(new FileInfo); + f = open_file(file_path); - broxygen_mgr->File(full_filename); - - delete [] bropath_subpath_delete; - - // "orig_file", could be an alias for yytext, which is ephemeral - // and will be zapped after the yy_switch_to_buffer() below. - yy_switch_to_buffer(yy_create_buffer(f, YY_BUF_SIZE)); - - yylloc.first_line = yylloc.last_line = line_number = 1; - - // Don't delete the old filename - it's pointed to by - // every BroObj created when parsing it. - yylloc.filename = filename = full_filename; + if ( ! f ) + reporter->FatalError("can't open %s", file_path.c_str()); } - else + ino_t i = get_inode_num(f, file_path); + + if ( already_scanned(i) ) + return 0; + + ScannedFile sf(i, file_stack.length(), file_path); + files_scanned.push_back(sf); + + if ( g_policy_debug && ! file_path.empty() ) { - reporter->Error("can't open %s", full_filename); - exit(1); + // Add the filename to the file mapping table (Debug.h). + Filemap* map = new Filemap; + HashKey* key = new HashKey(file_path.c_str()); + g_dbgfilemaps.Insert(key, map); + LoadPolicyFileText(file_path.c_str()); } + // Remember where we were. If this is the first + // file being pushed on the stack, i.e., the *last* + // one that will be processed, then we want to + // restore the module scope in which this @load + // was done when we're finished processing it. + if ( ! did_module_restore ) + { + file_stack.append(new FileInfo(current_module)); + did_module_restore = true; + } + else + file_stack.append(new FileInfo); + + broxygen_mgr->File(file_path); + + // "orig_file", could be an alias for yytext, which is ephemeral + // and will be zapped after the yy_switch_to_buffer() below. + yy_switch_to_buffer(yy_create_buffer(f, YY_BUF_SIZE)); + + yylloc.first_line = yylloc.last_line = line_number = 1; + + // Don't delete the old filename - it's pointed to by + // every BroObj created when parsing it. + yylloc.filename = filename = copy_string(file_path.c_str()); + return 1; } @@ -773,24 +751,22 @@ int yywrap() if ( ! prefixes[i][0] ) continue; - string s; - s = dot_canon(it->subpath.c_str(), it->name.c_str(), prefixes[i]); - FILE* f = search_for_file(s.c_str(), "bro", 0, false, 0, - get_dirname(::filename)); + string sub(find_dir_in_bropath(it->name)); + string flat(flatten_script_name(sub, it->name, prefixes[i])); + string path(find_relative_file(flat, "bro")); + + if ( ! path.empty() ) + { + add_input_file(path.c_str()); + found_prefixed_files = true; + } //printf("====== prefix search ======\n"); //printf("File : %s\n", it->name.c_str()); - //printf("Path : %s\n", it->subpath.c_str()); - //printf("Dotted: %s\n", s.c_str()); + //printf("Path : %s\n", sub.c_str()); + //printf("Dotted: %s\n", flat.c_str()); //printf("Found : %s\n", f ? "T" : "F"); //printf("===========================\n"); - - if ( f ) - { - add_input_file(s.c_str()); - found_prefixed_files = true; - fclose(f); - } } } diff --git a/src/util.cc b/src/util.cc index ad6d0368a4..1f087a697a 100644 --- a/src/util.cc +++ b/src/util.cc @@ -917,90 +917,127 @@ string bro_prefixes() const char* PACKAGE_LOADER = "__load__.bro"; -// If filename is pointing to a directory that contains a file called -// PACKAGE_LOADER, returns the files path. Otherwise returns filename itself. -// In both cases, the returned string is newly allocated. -static const char* check_for_dir(const char* filename, bool load_pkgs) +FILE* open_file(const string& path, const string& mode) { - if ( load_pkgs && is_dir(filename) ) - { - char init_filename_buf[1024]; - safe_snprintf(init_filename_buf, sizeof(init_filename_buf), - "%s/%s", filename, PACKAGE_LOADER); + if ( path.empty() ) + return 0; - if ( access(init_filename_buf, R_OK) == 0 ) - return copy_string(init_filename_buf); - } + FILE* rval = fopen(path.c_str(), mode.c_str()); - return copy_string(filename); - } - -static FILE* open_file(const char* filename, const char** full_filename, bool load_pkgs) - { - filename = check_for_dir(filename, load_pkgs); - - if ( full_filename ) - *full_filename = copy_string(filename); - - FILE* f = fopen(filename, "r"); - - if ( ! f ) + if ( ! rval ) { char buf[256]; strerror_r(errno, buf, sizeof(buf)); reporter->Error("Failed to open file %s: %s", filename, buf); } - delete [] filename; - - return f; + return rval; } -// Canonicalizes a given 'file' that lives in 'path' into a flattened, -// dotted format. If the optional 'prefix' argument is given, it is -// prepended to the dotted-format, separated by another dot. -// If 'file' is __load__.bro, that part is discarded when constructing -// the final dotted-format. -string dot_canon(string path, string file, string prefix) +static bool can_read(const string& path) + { + return access(path.c_str(), R_OK) == 0; + } + +FILE* open_package(string& path, const string& mode) + { + string arg_path(path); + path.append("/").append(PACKAGE_LOADER); + + if ( can_read(path) ) + return open_file(path, mode); + + reporter->Error("Failed to open package '%s': missing '%s' file", + arg_path.c_str(), PACKAGE_LOADER); + return 0; + } + +string safe_dirname(const char* path) + { + if ( ! path ) + return "."; + return safe_dirname(string(path)); + } + +string safe_dirname(const string& path) + { + char* tmp = copy_string(path.c_str()); + string rval(dirname(tmp)); + delete [] tmp; + return rval; + } + +string safe_basename(const char* path) + { + if ( ! path ) + return "."; + return safe_basename(string(path)); + } + +string safe_basename(const string& path) + { + char* tmp = copy_string(path.c_str()); + string rval(basename(tmp)); + delete [] tmp; + return rval; + } + +string flatten_script_name(const string& dir, const string& file, + const string& prefix) { string dottedform(prefix); + if ( prefix != "" ) dottedform.append("."); - dottedform.append(path); - char* tmp = copy_string(file.c_str()); - char* bname = basename(tmp); - if ( ! streq(bname, PACKAGE_LOADER) ) + + dottedform.append(dir); + string bname(safe_basename(file)); + + if ( bname != string(PACKAGE_LOADER) ) { - if ( path != "" ) + if ( dir != "" ) dottedform.append("."); + dottedform.append(bname); } - delete [] tmp; + size_t n; + while ( (n = dottedform.find("/")) != string::npos ) dottedform.replace(n, 1, "."); + return dottedform; } -// returns a normalized version of a path, removing duplicate slashes, -// extraneous dots that refer to the current directory, and pops as many -// parent directories referred to by "../" as possible -const char* normalize_path(const char* path) +static vector* tokenize_string(string input, const string& delim, + vector* rval) + { + if ( ! rval ) + rval = new vector(); + + size_t n; + + while ( (n = input.find(delim)) != string::npos ) + { + rval->push_back(input.substr(0, n)); + input.erase(0, n + 1); + } + + rval->push_back(input); + return rval; + } + + +string normalize_path(const string& path) { size_t n; - string p(path); vector components, final_components; string new_path; - if ( p[0] == '/' ) + if ( path[0] == '/' ) new_path = "/"; - while ( (n = p.find("/")) != string::npos ) - { - components.push_back(p.substr(0, n)); - p.erase(0, n + 1); - } - components.push_back(p); + tokenize_string(path, "/", &components); vector::const_iterator it; for ( it = components.begin(); it != components.end(); ++it ) @@ -1026,125 +1063,86 @@ const char* normalize_path(const char* path) if ( new_path.size() > 1 && new_path[new_path.size() - 1] == '/' ) new_path.erase(new_path.size() - 1); - return copy_string(new_path.c_str()); + return new_path; } -// Returns the subpath of the root Bro script install/source/build directory in -// which the loaded file is located. If it's not under a subpath of that -// directory (e.g. cwd or custom path) then the full path is returned. -void get_script_subpath(const std::string& full_filename, const char** subpath) +string find_dir_in_bropath(const string& path) { size_t p; - std::string my_subpath(full_filename); + string rval(path); // get the parent directory of file (if not already a directory) - if ( ! is_dir(full_filename.c_str()) ) - { - char* tmp = copy_string(full_filename.c_str()); - my_subpath = dirname(tmp); - delete [] tmp; - } + if ( ! is_dir(path.c_str()) ) + rval = safe_dirname(path); // first check if this is some subpath of the installed scripts root path, // if not check if it's a subpath of the script source root path, // then check if it's a subpath of the build directory (where BIF scripts // will get generated). // If none of those, will just use the given directory. - if ( (p = my_subpath.find(BRO_SCRIPT_INSTALL_PATH)) != std::string::npos ) - my_subpath.erase(0, strlen(BRO_SCRIPT_INSTALL_PATH)); - else if ( (p = my_subpath.find(BRO_SCRIPT_SOURCE_PATH)) != std::string::npos ) - my_subpath.erase(0, strlen(BRO_SCRIPT_SOURCE_PATH)); - else if ( (p = my_subpath.find(BRO_BUILD_SOURCE_PATH)) != std::string::npos ) - my_subpath.erase(0, strlen(BRO_BUILD_SOURCE_PATH)); - else if ( (p = my_subpath.find(BRO_BUILD_SCRIPTS_PATH)) != std::string::npos ) - my_subpath.erase(0, strlen(BRO_BUILD_SCRIPTS_PATH)); + if ( (p = rval.find(BRO_SCRIPT_INSTALL_PATH)) != std::string::npos ) + rval.erase(0, strlen(BRO_SCRIPT_INSTALL_PATH)); + else if ( (p = rval.find(BRO_SCRIPT_SOURCE_PATH)) != std::string::npos ) + rval.erase(0, strlen(BRO_SCRIPT_SOURCE_PATH)); + else if ( (p = rval.find(BRO_BUILD_SOURCE_PATH)) != std::string::npos ) + rval.erase(0, strlen(BRO_BUILD_SOURCE_PATH)); + else if ( (p = rval.find(BRO_BUILD_SCRIPTS_PATH)) != std::string::npos ) + rval.erase(0, strlen(BRO_BUILD_SCRIPTS_PATH)); // if root path found, remove path separators until next path component if ( p != std::string::npos ) - while ( my_subpath.size() && my_subpath[0] == '/' ) - my_subpath.erase(0, 1); + while ( rval.size() && rval[0] == '/' ) + rval.erase(0, 1); - *subpath = normalize_path(my_subpath.c_str()); + return normalize_path(rval); } -FILE* search_for_file(const char* filename, const char* ext, - const char** full_filename, bool load_pkgs, - const char** bropath_subpath, string prepend_to_search_path) +static string find_file_in_path(const string& filename, const string& path, + const string& opt_ext = "") { - // If the file is a literal absolute path we don't have to search, - // just return the result of trying to open it. If the file is - // might be a relative path, check first if it's a real file that - // can be referenced from cwd, else we'll try to search for it based - // on what path the currently-loading script is in as well as the - // standard BROPATH paths. - if ( filename[0] == '/' || - (filename[0] == '.' && access(filename, R_OK) == 0) ) + if ( filename.empty() ) + return string(); + + // If file name is an absolute path, searching within *path* is pointless. + if ( filename[0] == '/' ) { - if ( bropath_subpath ) - { - char* tmp = copy_string(filename); - *bropath_subpath = copy_string(dirname(tmp)); - delete [] tmp; - } - return open_file(filename, full_filename, load_pkgs); - } - - char path[1024], full_filename_buf[1024]; - - // Prepend the currently loading script's path to BROPATH so that - // @loads can be referenced relatively. - if ( ! prepend_to_search_path.empty() && filename[0] == '.' ) - safe_snprintf(path, sizeof(path), "%s:%s", - prepend_to_search_path.c_str(), bro_path()); - else - safe_strncpy(path, bro_path(), sizeof(path)); - - char* dir_beginning = path; - char* dir_ending = path; - int more = *dir_beginning != '\0'; - - while ( more ) - { - while ( *dir_ending && *dir_ending != ':' ) - ++dir_ending; - - if ( *dir_ending == ':' ) - *dir_ending = '\0'; + if ( can_read(filename) ) + return filename; else - more = 0; - - safe_snprintf(full_filename_buf, sizeof(full_filename_buf), - "%s/%s.%s", dir_beginning, filename, ext); - if ( access(full_filename_buf, R_OK) == 0 && - ! is_dir(full_filename_buf) ) - { - if ( bropath_subpath ) - get_script_subpath(full_filename_buf, bropath_subpath); - return open_file(full_filename_buf, full_filename, load_pkgs); - } - - safe_snprintf(full_filename_buf, sizeof(full_filename_buf), - "%s/%s", dir_beginning, filename); - if ( access(full_filename_buf, R_OK) == 0 ) - { - if ( bropath_subpath ) - get_script_subpath(full_filename_buf, bropath_subpath); - return open_file(full_filename_buf, full_filename, load_pkgs); - } - - dir_beginning = ++dir_ending; + return string(); } - if ( full_filename ) - *full_filename = copy_string(filename); - if ( bropath_subpath ) - { - char* tmp = copy_string(filename); - *bropath_subpath = copy_string(dirname(tmp)); - delete [] tmp; - } + string abs_path(path + '/' + filename); - return 0; + if ( ! opt_ext.empty() ) + { + string with_ext(abs_path + '.' + opt_ext); + + if ( can_read(with_ext) ) + return with_ext; + } + + if ( can_read(abs_path) ) + return abs_path; + + return string(); + } + +string find_file(const string& filename, const string& path_set, + const string& opt_ext) + { + vector paths; + tokenize_string(path_set, ":", &paths); + + for ( size_t n = 0; n < paths.size(); ++n ) + { + string f = find_file_in_path(filename, paths[n], opt_ext); + + if ( ! f.empty() ) + return f; + } + + return string(); } FILE* rotate_file(const char* name, RecordVal* rotate_info) diff --git a/src/util.h b/src/util.h index e7ac4d33af..e35a1605e5 100644 --- a/src/util.h +++ b/src/util.h @@ -205,12 +205,64 @@ extern int int_list_cmp(const void* v1, const void* v2); extern const char* bro_path(); extern const char* bro_magic_path(); extern std::string bro_prefixes(); -std::string dot_canon(std::string path, std::string file, std::string prefix = ""); -const char* normalize_path(const char* path); -void get_script_subpath(const std::string& full_filename, const char** subpath); -extern FILE* search_for_file(const char* filename, const char* ext, - const char** full_filename, bool load_pkgs, const char** bropath_subpath, - std::string prepend_to_search_path = ""); + +// Wrappers for dirname(3) that won't modify argument. +std::string safe_dirname(const char* path); +std::string safe_dirname(const std::string& path); + +// Wrappers for basename(3) that won't modify argument. +std::string safe_basename(const char* path); +std::string safe_basename(const std::string& path); + +/** + * Flatten a script name by replacing '/' path separators with '.'. + * @param dir A directory containing \a file. + * @param file A path to a Bro script. If it is a __load__.bro, that part + * is discarded when constructing the flattened the name. + * @param prefix A string to prepend to the flattened script name. + * @return The flattened script name. + */ +std::string flatten_script_name(const std::string& dir, + const std::string& file, + const std::string& prefix = ""); + +/** + * Return a canonical/shortened path string by removing superfluous elements + * (path delimiters, dots referring to CWD or parent dir). + * @param path A filesystem path. + * @return A canonical/shortened version of \a path. + */ +std::string normalize_path(const std::string& path); + +/** + * Locate a file/direcotry within BROPATH. + * @param path A file/directory to locate within BROPATH. + * @return The directory within BROPATH that \a path located or an absolute + * path to \a path if it couldn't be located in BROPATH + */ +std::string find_dir_in_bropath(const std::string& path); + +/** + * Locate a file within a given search path. + * @param filename Name of a file to find. + * @param path_set Colon-delimited set of paths to search for the file. + * @param opt_ext A filename extension/suffix to allow. + * @return Path to the found file, or an empty string if not found. + */ +std::string find_file(const std::string& filename, const std::string& path_set, + const std::string& opt_ext = ""); + +// Wrapper around fopen(3). Emits an error when failing to open. +FILE* open_file(const std::string& path, const std::string& mode = "r"); + +/** Opens a Bro script package. + * @param path Location of a Bro script package (a directory). Will be changed + * to the path of the package's loader script. + * @param mode An fopen(3) mode. + * @return The return value of fopen(3) on the loader script or null if one + * doesn't exist. + */ +FILE* open_package(std::string& path, const std::string& mode = "r"); // Renames the given file to a new temporary name, and opens a new file with // the original name. Returns new file or NULL on error. Inits rotate_info if From f18436640ea9cba8ed33579111defe61b2c8fca4 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Tue, 22 Oct 2013 14:45:47 -0500 Subject: [PATCH 03/40] Flesh out Broxygen doc-gathering skeleton. --- CMakeLists.txt | 1 - bro-path-dev.in | 8 +- src/CMakeLists.txt | 1 + src/DebugLogger.cc | 2 +- src/DebugLogger.h | 1 + src/broxygen/Document.cc | 74 ++++++++ src/broxygen/Document.h | 154 ++++++++++++++++ src/broxygen/Manager.cc | 388 ++++++++++++++++++++++++++++++++++----- src/broxygen/Manager.h | 39 +++- src/main.cc | 2 + src/parse.y | 4 +- src/scan.l | 22 ++- src/util-config.h.in | 3 - src/util.cc | 81 ++++---- src/util.h | 15 +- 15 files changed, 669 insertions(+), 126 deletions(-) create mode 100644 src/broxygen/Document.cc create mode 100644 src/broxygen/Document.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 1c427188ca..88cee2ec29 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -10,7 +10,6 @@ if (NOT BRO_SCRIPT_INSTALL_PATH) # set the default Bro script installation path (user did not specify one) set(BRO_SCRIPT_INSTALL_PATH ${BRO_ROOT_DIR}/share/bro) endif () -set(BRO_SCRIPT_SOURCE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/scripts) # sanitize the Bro script install directory into an absolute path # (CMake is confused by ~ as a representation of home directory) diff --git a/bro-path-dev.in b/bro-path-dev.in index 2c17d057c9..de8b0274b9 100755 --- a/bro-path-dev.in +++ b/bro-path-dev.in @@ -10,10 +10,4 @@ # BROPATH=`./bro-path-dev` ./src/bro # -broPolicies=${BRO_SCRIPT_SOURCE_PATH}:${BRO_SCRIPT_SOURCE_PATH}/policy:${BRO_SCRIPT_SOURCE_PATH}/site - -broGenPolicies=${CMAKE_BINARY_DIR}/scripts - -installedPolicies=${BRO_SCRIPT_INSTALL_PATH}:${BRO_SCRIPT_INSTALL_PATH}/site - -echo .:$broPolicies:$broGenPolicies +echo .:${CMAKE_SOURCE_DIR}/scripts:${CMAKE_SOURCE_DIR}/scripts/policy:${CMAKE_SOURCE_DIR}/scripts/site:${CMAKE_BINARY_DIR}/scripts diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 4145984f3b..dec2ea6680 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -371,6 +371,7 @@ set(bro_SRCS plugin/Macros.h broxygen/Manager.cc + broxygen/Document.cc nb_dns.c digest.h diff --git a/src/DebugLogger.cc b/src/DebugLogger.cc index 95049ef70b..78377eafcf 100644 --- a/src/DebugLogger.cc +++ b/src/DebugLogger.cc @@ -17,7 +17,7 @@ DebugLogger::Stream DebugLogger::streams[NUM_DBGS] = { { "dpd", 0, false }, { "tm", 0, false }, { "logging", 0, false }, {"input", 0, false }, { "threading", 0, false }, { "file_analysis", 0, false }, - { "plugins", 0, false} + { "plugins", 0, false }, { "broxygen", 0, false } }; DebugLogger::DebugLogger(const char* filename) diff --git a/src/DebugLogger.h b/src/DebugLogger.h index c5744642f5..d1f053788e 100644 --- a/src/DebugLogger.h +++ b/src/DebugLogger.h @@ -28,6 +28,7 @@ enum DebugStream { DBG_THREADING, // Threading system DBG_FILE_ANALYSIS, // File analysis DBG_PLUGINS, + DBG_BROXYGEN, NUM_DBGS // Has to be last }; diff --git a/src/broxygen/Document.cc b/src/broxygen/Document.cc new file mode 100644 index 0000000000..ab8aca08aa --- /dev/null +++ b/src/broxygen/Document.cc @@ -0,0 +1,74 @@ +#include "Document.h" + +#include "util.h" + +using namespace broxygen; +using namespace std; + +PackageDocument::PackageDocument(const string& arg_name) + : Document(), + pkg_loader_name(arg_name) + { + // TODO: probably need to determine modification times of all files + // within the directory, recursively + } + +IdentifierDocument::IdentifierDocument(ID* arg_id) + : Document(), + comments(), id(arg_id), initial_val(), redefs(), fields(), + last_field_seen() + { + Ref(id); + + if ( id->ID_Val() ) + initial_val = id->ID_Val()->Clone(); + } + +IdentifierDocument::~IdentifierDocument() + { + Unref(id); + //Unref(initial_val); // TODO: problematic w/ PatternVals + + for ( RedefList::const_iterator it = redefs.begin(); it != redefs.end(); + ++it ) + delete *it; + + for ( size_t i = 0; i < fields.size(); ++i ) + delete fields[i]; + } + +void IdentifierDocument::AddRedef(const string& script, + const vector& comments) + { + Redefinition* redef = new Redefinition(); + redef->from_script = script; + redef->new_val = id->ID_Val() ? id->ID_Val()->Clone() : 0; + redef->comments = comments; + redefs.push_back(redef); + } + +void IdentifierDocument::AddRecordField(const TypeDecl* field, + const string& script, + std::vector& comments) + { + RecordField* rf = new RecordField(); + rf->field = new TypeDecl(*field); + rf->from_script = script; + rf->comments = comments; + fields.push_back(rf); + last_field_seen = rf; + } + +ScriptDocument::ScriptDocument(const string& arg_name) + : Document(), + name(arg_name), + is_pkg_loader(safe_basename(name) == PACKAGE_LOADER), + dependencies(), module_usages(), comments(), identifier_docs() + { + } + +void ScriptDocument::AddIdentifierDoc(IdentifierDocument* doc) + { + identifier_docs[doc->Name()] = doc; + // TODO: sort things (e.g. function flavor, state var vs. option var) + } diff --git a/src/broxygen/Document.h b/src/broxygen/Document.h new file mode 100644 index 0000000000..34742f8069 --- /dev/null +++ b/src/broxygen/Document.h @@ -0,0 +1,154 @@ +#ifndef BROXYGEN_DOCUMENT_H +#define BROXYGEN_DOCUMENT_H + +#include +#include +#include +#include +#include +#include +#include + +#include "ID.h" +#include "Val.h" +#include "Type.h" + +namespace broxygen { + +// TODO: documentation... + +class Document { + +public: + + Document() + { } + + virtual ~Document() + { } + + time_t GetModificationTime() const + { return DoGetModificationTime(); } + +private: + + virtual time_t DoGetModificationTime() const = 0; +}; + +class PackageDocument : public Document { + +public: + + PackageDocument(const std::string& name); + +private: + + // TODO + time_t DoGetModificationTime() const + { return 0; } + + std::string pkg_loader_name; +}; + + +class IdentifierDocument : public Document { + +public: + + IdentifierDocument(ID* id); + + ~IdentifierDocument(); + + void AddComment(const std::string& comment) + { last_field_seen ? last_field_seen->comments.push_back(comment) + : comments.push_back(comment); } + + void AddComments(const std::vector& cmtns) + { comments.insert(comments.end(), cmtns.begin(), cmtns.end() ); } + + void AddRedef(const std::string& from_script, + const std::vector& comments); + + void AddRecordField(const TypeDecl* field, const std::string& script, + std::vector& comments); + + string Name() const + { return id->Name(); } + + ID* GetID() const + { return id; } + +private: + + struct Redefinition { + ~Redefinition() + { Unref(new_val); } + + std::string from_script; + Val* new_val; + std::vector comments; + }; + + struct RecordField { + ~RecordField() + { delete field; } + + TypeDecl* field; + std::string from_script; + std::vector comments; + }; + + typedef std::list RedefList; + + // TODO + time_t DoGetModificationTime() const + { return 0; } + + std::vector comments; + ID* id; + Val* initial_val; + RedefList redefs; + std::vector fields; + RecordField* last_field_seen; +}; + +class ScriptDocument : public Document { + +public: + + ScriptDocument(const std::string& name); + + void AddComment(const std::string& comment) + { comments.push_back(comment); } + + void AddDependency(const std::string& name) + { dependencies.insert(name); } + + void AddModule(const std::string& name) + { module_usages.insert(name); } + + void AddIdentifierDoc(IdentifierDocument* doc); + + bool IsPkgLoader() const + { return is_pkg_loader; } + +private: + + typedef std::map IdentifierDocMap; + + // TODO + time_t DoGetModificationTime() const + { return 0; } + + std::string name; + bool is_pkg_loader; + std::set dependencies; + std::set module_usages; + std::vector comments; + IdentifierDocMap identifier_docs; +}; + + +} // namespace broxygen + +#endif diff --git a/src/broxygen/Manager.cc b/src/broxygen/Manager.cc index ee995d8b4a..8d01c51224 100644 --- a/src/broxygen/Manager.cc +++ b/src/broxygen/Manager.cc @@ -1,88 +1,384 @@ #include "Manager.h" +#include "Reporter.h" +#include "util.h" + +#include using namespace broxygen; +using namespace std; -Manager::Manager(const std::string& config) +static void DbgAndWarn(const char* msg) { - // TODO + reporter->InternalWarning("%s", msg); + DBG_LOG(DBG_BROXYGEN, "%s", msg); + } + +static string RemoveLeadingSpace(const string& s) + { + if ( s.empty() || s[0] != ' ' ) + return s; + + // Treat "##Text" and "## Text" the same, so that a single space doesn't + // cause reST formatting to think the later is indented a level. + string rval = s; + rval.erase(0, 1); + return rval; + } + +static string PrettifyParams(const string& s) + { + size_t identifier_start_pos = 0; + bool in_identifier = false; + string identifier; + + for ( size_t i = 0; i < s.size(); ++i ) + { + char next = s[i]; + + if ( ! in_identifier ) + { + // Pass by leading whitespace. + if ( isspace(next) ) + continue; + + // Only allow alphabetic and '_' as first char of identifier. + if ( isalpha(next) || next == '_' ) + { + identifier_start_pos = i; + identifier += next; + in_identifier = true; + continue; + } + + // Don't need to change anything. + return s; + } + + // All other character of identifier are alphanumeric or '_'. + if ( isalnum(next) || next == '_' ) + { + identifier += next; + continue; + } + + // Prettify param and return value docs for a function's reST markup. + if ( next == ':' ) + { + string rval = s; + string subst; + + if ( identifier == "Returns" ) + subst = "\n:returns"; + else + subst = "\n:param " + identifier; + + rval.replace(identifier_start_pos, identifier.size(), subst); + return rval; + } + + // Don't need to change anything. + return s; + } + + return s; + } + +Manager::Manager(const string& config) + : comment_buffer(), packages(), scripts(), identifiers(), all_docs(), + last_doc_seen(), incomplete_type() + { + // TODO config file stuff + } + +Manager::~Manager() + { + for ( DocSet::const_iterator it = all_docs.begin(); it != all_docs.end(); + ++it ) + delete *it; + } + +void Manager::InitPreScript() + { + // TODO: create file/proto analyzer doc + } + +void Manager::InitPostScript() + { + // TODO: dependency resolution stuff? } void Manager::GenerateDocs() const { // TODO - // may be a no-op if no config - - // does the old canon_doc_func_param stuff happen here now on the fly - // for functions we're about to document? + // may be a no-op if no config file } -void Manager::File(const std::string& path) +void Manager::File(const string& path) { - // TODO - // can be a file or directory? - // determine path within BROPATH + string name = without_bropath_component(path); + + if ( scripts.find(name) != scripts.end() ) + { + DbgAndWarn(fmt("Duplicate script documentation: %s", name.c_str())); + return; + } + + ScriptDocument* doc = new ScriptDocument(name); + scripts[name] = doc; + RegisterDoc(doc); + DBG_LOG(DBG_BROXYGEN, "Made ScriptDocument %s", name.c_str()); + + if ( ! doc->IsPkgLoader() ) + return; + + if ( packages.find(name) != packages.end() ) + { + DbgAndWarn(fmt("Duplicate package documentation: %s", name.c_str())); + return; + } + + packages[name] = new PackageDocument(name); + DBG_LOG(DBG_BROXYGEN, "Made PackageDocument %s", name.c_str()); } -void Manager::ScriptDependency(const std::string& path, const std::string& dep) +void Manager::ScriptDependency(const string& path, const string& dep) { - // TODO: - // need anything from BroDoc::AddImport? - // warn about unconsumed comments (and discard any) + if ( dep.empty() ) + { + DbgAndWarn(fmt("Empty script doc dependency: %s", path.c_str())); + return; + } + + string name = without_bropath_component(path); + string depname = without_bropath_component(dep); + ScriptMap::const_iterator it = scripts.find(name); + + if ( it == scripts.end() ) + { + DbgAndWarn(fmt("Failed to add script doc dependency %s for %s", + depname.c_str(), name.c_str())); + return; + } + + it->second->AddDependency(depname); + DBG_LOG(DBG_BROXYGEN, "Added script dependency %s for %s", + depname.c_str(), name.c_str()); + + for ( CommentBuffer::const_iterator it = comment_buffer.begin(); + it != comment_buffer.end(); ++it ) + DbgAndWarn(fmt("Discarded extraneous Broxygen comment: %s", + it->c_str())); } -void Manager::ModuleUsage(const std::string& path, const std::string& module) +void Manager::ModuleUsage(const string& path, const string& module) { - // TODO lookup script and add module to a set + string name = without_bropath_component(path); + ScriptMap::const_iterator it = scripts.find(name); + + if ( it == scripts.end() ) + { + DbgAndWarn(fmt("Failed to add module usage %s in %s", + module.c_str(), name.c_str())); + return; + } + + DBG_LOG(DBG_BROXYGEN, "Added module usage %s in %s", + module.c_str(), name.c_str()); } -void Manager::Identifier(const ID *id) +void Manager::StartType(ID* id) { - // TODO: lookup script to associate w/ by GetLocationInfo()->filename - // do different things depending on Type? (eg function flavor versus state) - // do different things based on redef attr + const ? - // consume any buffered comments and associate w/ id - // deal w/ type aliasing - // special enum or record handing? - // if it's a function we may already have added it (decl versus impl) + if ( id->GetLocationInfo() == &no_location ) + { + DbgAndWarn(fmt("Can't document %s, no location available", id->Name())); + return; + } + + string script = without_bropath_component(id->GetLocationInfo()->filename); + ScriptMap::const_iterator sit = scripts.find(script); + + if ( sit == scripts.end() ) + { + DbgAndWarn(fmt("Can't document identifier %s, lookup of %s failed", + id->Name(), script.c_str())); + return; + } + + IdentifierDocument* doc = new IdentifierDocument(id); + doc->AddComments(comment_buffer); + comment_buffer.clear(); + identifiers[id->Name()] = doc; + RegisterDoc(doc); + sit->second->AddIdentifierDoc(doc); + incomplete_type = doc; + DBG_LOG(DBG_BROXYGEN, "Made IdentifierDocument (incomplete) %s, in %s", + id->Name(), script.c_str()); } -void Manager::RecordField(const ID *id, const TypeDecl *field, - const std::string& path) +void Manager::Identifier(ID* id) { - // TODO: consume comments - // redef is implicit -- script path of field will differ from ID/type's + if ( incomplete_type && incomplete_type->Name() == id->Name() ) + { + DBG_LOG(DBG_BROXYGEN, "Finished document for type %s", id->Name()); + incomplete_type = 0; + return; + } + + if ( id->GetLocationInfo() == &no_location ) + { + // Internally-created identifier (e.g. file/proto analyzer enum tags). + // Can be ignored here as they need to be documented via other means. + DBG_LOG(DBG_BROXYGEN, "Skip documenting identifier %s: no location", + id->Name()); + return; + } + + IdentifierMap::const_iterator iit = identifiers.find(id->Name()); + + if ( iit != identifiers.end() ) + { + if ( IsFunc(iit->second->GetID()->Type()->Tag()) ) + { + // Function may already been seen (declaration versus body). + iit->second->AddComments(comment_buffer); + comment_buffer.clear(); + return; + } + + DbgAndWarn(fmt("Duplicate identifier documentation: %s", id->Name())); + return; + } + + string script = without_bropath_component(id->GetLocationInfo()->filename); + ScriptMap::const_iterator sit = scripts.find(script); + + if ( sit == scripts.end() ) + { + DbgAndWarn(fmt("Can't document identifier %s, lookup of %s failed", + id->Name(), script.c_str())); + return; + } + + IdentifierDocument* doc = new IdentifierDocument(id); + doc->AddComments(comment_buffer); + comment_buffer.clear(); + identifiers[id->Name()] = doc; + RegisterDoc(doc); + sit->second->AddIdentifierDoc(doc); + DBG_LOG(DBG_BROXYGEN, "Made IdentifierDocument %s, in script %s", + id->Name(), script.c_str()); + } + +void Manager::RecordField(const ID* id, const TypeDecl* field, + const string& path) + { + IdentifierDocument* idd = 0; + + if ( incomplete_type ) + { + if ( incomplete_type->Name() != id->Name() ) + { + DbgAndWarn(fmt("Can't document record field %s in record %s, " + "expected record %s", field->id, id->Name(), + incomplete_type->Name().c_str())); + return; + } + + idd = incomplete_type; + } + else + { + IdentifierMap::const_iterator it = identifiers.find(id->Name()); + + if ( it == identifiers.end() ) + { + DbgAndWarn(fmt("Can't document record field %s, unknown record: %s", + field->id, id->Name())); + return; + } + + idd = it->second; + } + + string script = without_bropath_component(path); + + idd->AddRecordField(field, script, comment_buffer); + comment_buffer.clear(); + DBG_LOG(DBG_BROXYGEN, "Document record field %s, identifier %s, script %s", + field->id, id->Name(), script.c_str()); } void Manager::Redef(const ID* id, const string& path) { - // TODO: lookup script w/ 'path' to associate the id in as redef'd - // consume any buffered comments and associate w/ redef'd id - // can sort notices here + if ( path == "" ) + // This is a redef defined on the command line. + return; + + IdentifierMap::const_iterator iit = identifiers.find(id->Name()); + + if ( iit == identifiers.end() ) + { + DbgAndWarn(fmt("Can't document redef of %s, identifier lookup failed", + id->Name())); + return; + } + + string from_script = without_bropath_component(path); + ScriptMap::const_iterator sit = scripts.find(from_script); + + if ( sit == scripts.end() ) + { + DbgAndWarn(fmt("Can't document redef of %s, lookup of %s failed", + id->Name(), from_script.c_str())); + return; + } + + iit->second->AddRedef(from_script, comment_buffer); + comment_buffer.clear(); + DBG_LOG(DBG_BROXYGEN, "Added redef of %s to %s", + id->Name(), from_script.c_str()); } -void Manager::SummaryComment(const std::string& script, - const std::string& comment) +void Manager::SummaryComment(const string& script, const string& comment) { - // TODO - // canon_doc_comment ? + string name = without_bropath_component(script); + ScriptMap::const_iterator it = scripts.find(name); + + if ( it == scripts.end() ) + { + DbgAndWarn(fmt("Lookup of script %s failed for summary comment %s", + name.c_str(), comment.c_str())); + return; + } + + it->second->AddComment(RemoveLeadingSpace(comment)); } -void Manager::PreComment(const std::string& comment) +void Manager::PreComment(const string& comment) { - // TODO - // canon_doc_comment + comment_buffer.push_back(PrettifyParams(RemoveLeadingSpace(comment))); } -void Manager::PostComment(const std::string& comment) +void Manager::PostComment(const string& comment) { - // TODO this gets associated with the last thing registered - // canon_doc_comment + IdentifierDocument* doc = dynamic_cast(last_doc_seen); + + if ( ! doc ) + { + DbgAndWarn(fmt("Discarded comment not associated w/ an identifier %s", + comment.c_str())); + return; + } + + doc->AddComment(RemoveLeadingSpace(comment)); } +void Manager::RegisterDoc(Document* d) + { + if ( ! d ) + return; -// TODO: "canon_doc_comment" means treat "##Text" and "## Text" the same -// so that a single space doesn't generate an indentation level. - - -// TODO: creating proto/file analyzer docs + all_docs.insert(d); + last_doc_seen = d; + } diff --git a/src/broxygen/Manager.h b/src/broxygen/Manager.h index d0665c307d..10b63af945 100644 --- a/src/broxygen/Manager.h +++ b/src/broxygen/Manager.h @@ -1,19 +1,32 @@ #ifndef BROXYGEN_MANAGER_H #define BROXYGEN_MANAGER_H -#include - +#include "Document.h" #include "ID.h" #include "Type.h" +#include +#include +#include +#include + namespace broxygen { +// TODO: documentation... +// TODO: optimize parse time... maybe an env. option to disable doc collection? + class Manager { public: Manager(const std::string& config); + ~Manager(); + + void InitPreScript(); + + void InitPostScript(); + void GenerateDocs() const; void File(const std::string& path); @@ -22,7 +35,9 @@ public: void ModuleUsage(const std::string& path, const std::string& module); - void Identifier(const ID* id); + void StartType(ID* id); + + void Identifier(ID* id); void RecordField(const ID* id, const TypeDecl* field, const std::string& path); @@ -34,6 +49,24 @@ public: void PreComment(const std::string& comment); void PostComment(const std::string& comment); + +private: + + typedef std::vector CommentBuffer; + typedef std::map PackageMap; + typedef std::map ScriptMap; + typedef std::map IdentifierMap; + typedef std::set DocSet; + + void RegisterDoc(Document* d); + + CommentBuffer comment_buffer; + PackageMap packages; + ScriptMap scripts; + IdentifierMap identifiers; + DocSet all_docs; + Document* last_doc_seen; + IdentifierDocument* incomplete_type; }; } // namespace broxygen diff --git a/src/main.cc b/src/main.cc index f61e8cc1c5..764b508749 100644 --- a/src/main.cc +++ b/src/main.cc @@ -850,6 +850,7 @@ int main(int argc, char** argv) plugin_mgr->InitPreScript(); analyzer_mgr->InitPreScript(); file_mgr->InitPreScript(); + broxygen_mgr->InitPreScript(); if ( events_file ) event_player = new EventPlayer(events_file); @@ -882,6 +883,7 @@ int main(int argc, char** argv) plugin_mgr->InitPostScript(); analyzer_mgr->InitPostScript(); file_mgr->InitPostScript(); + broxygen_mgr->InitPostScript(); if ( print_plugins ) { diff --git a/src/parse.y b/src/parse.y index ed79808c78..75be472853 100644 --- a/src/parse.y +++ b/src/parse.y @@ -1039,7 +1039,9 @@ decl: extend_record($3, $8, $11); } - | TOK_TYPE global_id ':' { cur_decl_type_id = $2; } type opt_attr ';' + | TOK_TYPE global_id ':' + { cur_decl_type_id = $2; broxygen_mgr->StartType($2); } + type opt_attr ';' { cur_decl_type_id = 0; add_type($2, $5, $6); diff --git a/src/scan.l b/src/scan.l index 899fa77b66..3ac256969a 100644 --- a/src/scan.l +++ b/src/scan.l @@ -271,7 +271,7 @@ when return TOK_WHEN; @DEBUG return TOK_DEBUG; // marks input for debugger @DIR { - string rval(safe_dirname(::filename)); + string rval = safe_dirname(::filename); if ( ! rval.empty() && rval[0] == '.' ) { @@ -300,13 +300,15 @@ when return TOK_WHEN; @load{WS}{FILE} { const char* new_file = skip_whitespace(yytext + 5); // Skip "@load". - broxygen_mgr->ScriptDependency(::filename, new_file); + string loader = ::filename; // load_files may change ::filename, save copy + string loading = find_relative_file(new_file, "bro"); (void) load_files(new_file); + broxygen_mgr->ScriptDependency(loader, loading); } @load-sigs{WS}{FILE} { const char* file = skip_whitespace(yytext + 10); - string path(find_relative_file(file, "sig")); + string path = find_relative_file(file, "sig"); if ( path.empty() ) reporter->Error("failed to find file associated with @load-sigs %s", @@ -318,7 +320,7 @@ when return TOK_WHEN; @unload{WS}{FILE} { // Skip "@unload". const char* file = skip_whitespace(yytext + 7); - string path(find_relative_file(file, "bro")); + string path = find_relative_file(file, "bro"); if ( path.empty() ) reporter->Error("failed find file associated with @unload %s", file); @@ -751,9 +753,9 @@ int yywrap() if ( ! prefixes[i][0] ) continue; - string sub(find_dir_in_bropath(it->name)); - string flat(flatten_script_name(sub, it->name, prefixes[i])); - string path(find_relative_file(flat, "bro")); + string canon = without_bropath_component(it->name); + string flat = flatten_script_name(canon, prefixes[i]); + string path = find_relative_file(flat, "bro"); if ( ! path.empty() ) { @@ -763,9 +765,9 @@ int yywrap() //printf("====== prefix search ======\n"); //printf("File : %s\n", it->name.c_str()); - //printf("Path : %s\n", sub.c_str()); - //printf("Dotted: %s\n", flat.c_str()); - //printf("Found : %s\n", f ? "T" : "F"); + //printf("Canon : %s\n", canon.c_str()); + //printf("Flat : %s\n", flat.c_str()); + //printf("Found : %s\n", path.empty() ? "F" : "T"); //printf("===========================\n"); } } diff --git a/src/util-config.h.in b/src/util-config.h.in index ff5e28537a..385c789b97 100644 --- a/src/util-config.h.in +++ b/src/util-config.h.in @@ -1,5 +1,2 @@ #define BRO_SCRIPT_INSTALL_PATH "@BRO_SCRIPT_INSTALL_PATH@" -#define BRO_SCRIPT_SOURCE_PATH "@BRO_SCRIPT_SOURCE_PATH@" -#define BRO_BUILD_SOURCE_PATH "@CMAKE_BINARY_DIR@/src" -#define BRO_BUILD_SCRIPTS_PATH "@CMAKE_BINARY_DIR@/scripts" #define BRO_MAGIC_INSTALL_PATH "@BRO_MAGIC_INSTALL_PATH@" diff --git a/src/util.cc b/src/util.cc index 3973ece37d..bf98bb9ed3 100644 --- a/src/util.cc +++ b/src/util.cc @@ -933,7 +933,7 @@ static bool can_read(const string& path) FILE* open_package(string& path, const string& mode) { - string arg_path(path); + string arg_path = path; path.append("/").append(PACKAGE_LOADER); if ( can_read(path) ) @@ -954,7 +954,7 @@ string safe_dirname(const char* path) string safe_dirname(const string& path) { char* tmp = copy_string(path.c_str()); - string rval(dirname(tmp)); + string rval = dirname(tmp); delete [] tmp; return rval; } @@ -969,36 +969,29 @@ string safe_basename(const char* path) string safe_basename(const string& path) { char* tmp = copy_string(path.c_str()); - string rval(basename(tmp)); + string rval = basename(tmp); delete [] tmp; return rval; } -string flatten_script_name(const string& dir, const string& file, - const string& prefix) +string flatten_script_name(const string& name, const string& prefix) { - string dottedform(prefix); + string rval = prefix; - if ( prefix != "" ) - dottedform.append("."); + if ( ! rval.empty() ) + rval.append("."); - dottedform.append(dir); - string bname(safe_basename(file)); + if ( safe_basename(name) == PACKAGE_LOADER ) + rval.append(safe_dirname(name)); + else + rval.append(name); - if ( bname != string(PACKAGE_LOADER) ) - { - if ( dir != "" ) - dottedform.append("."); + size_t i; - dottedform.append(bname); - } + while ( (i = rval.find('/')) != string::npos ) + rval[i] = '.'; - size_t n; - - while ( (n = dottedform.find("/")) != string::npos ) - dottedform.replace(n, 1, "."); - - return dottedform; + return rval; } static vector* tokenize_string(string input, const string& delim, @@ -1058,35 +1051,31 @@ string normalize_path(const string& path) return new_path; } -string find_dir_in_bropath(const string& path) +string without_bropath_component(const string& path) { - size_t p; - string rval(path); + string rval = normalize_path(path); - // get the parent directory of file (if not already a directory) - if ( ! is_dir(path.c_str()) ) - rval = safe_dirname(path); + vector paths; + tokenize_string(bro_path(), ":", &paths); - // first check if this is some subpath of the installed scripts root path, - // if not check if it's a subpath of the script source root path, - // then check if it's a subpath of the build directory (where BIF scripts - // will get generated). - // If none of those, will just use the given directory. - if ( (p = rval.find(BRO_SCRIPT_INSTALL_PATH)) != std::string::npos ) - rval.erase(0, strlen(BRO_SCRIPT_INSTALL_PATH)); - else if ( (p = rval.find(BRO_SCRIPT_SOURCE_PATH)) != std::string::npos ) - rval.erase(0, strlen(BRO_SCRIPT_SOURCE_PATH)); - else if ( (p = rval.find(BRO_BUILD_SOURCE_PATH)) != std::string::npos ) - rval.erase(0, strlen(BRO_BUILD_SOURCE_PATH)); - else if ( (p = rval.find(BRO_BUILD_SCRIPTS_PATH)) != std::string::npos ) - rval.erase(0, strlen(BRO_BUILD_SCRIPTS_PATH)); + for ( size_t i = 0; i < paths.size(); ++i ) + { + string common = normalize_path(paths[i]); - // if root path found, remove path separators until next path component - if ( p != std::string::npos ) + if ( rval.find(common) != 0 ) + continue; + + // Found the containing directory. + rval.erase(0, common.size()); + + // Remove leading path separators. while ( rval.size() && rval[0] == '/' ) rval.erase(0, 1); - return normalize_path(rval); + return rval; + } + + return rval; } static string find_file_in_path(const string& filename, const string& path, @@ -1104,11 +1093,11 @@ static string find_file_in_path(const string& filename, const string& path, return string(); } - string abs_path(path + '/' + filename); + string abs_path = path + '/' + filename; if ( ! opt_ext.empty() ) { - string with_ext(abs_path + '.' + opt_ext); + string with_ext = abs_path + '.' + opt_ext; if ( can_read(with_ext) ) return with_ext; diff --git a/src/util.h b/src/util.h index e35a1605e5..faca450cd8 100644 --- a/src/util.h +++ b/src/util.h @@ -202,6 +202,8 @@ static const SourceID SOURCE_LOCAL = 0; extern void pinpoint(); extern int int_list_cmp(const void* v1, const void* v2); +extern const char* PACKAGE_LOADER; + extern const char* bro_path(); extern const char* bro_magic_path(); extern std::string bro_prefixes(); @@ -216,14 +218,12 @@ std::string safe_basename(const std::string& path); /** * Flatten a script name by replacing '/' path separators with '.'. - * @param dir A directory containing \a file. * @param file A path to a Bro script. If it is a __load__.bro, that part * is discarded when constructing the flattened the name. * @param prefix A string to prepend to the flattened script name. * @return The flattened script name. */ -std::string flatten_script_name(const std::string& dir, - const std::string& file, +std::string flatten_script_name(const std::string& name, const std::string& prefix = ""); /** @@ -235,12 +235,11 @@ std::string flatten_script_name(const std::string& dir, std::string normalize_path(const std::string& path); /** - * Locate a file/direcotry within BROPATH. - * @param path A file/directory to locate within BROPATH. - * @return The directory within BROPATH that \a path located or an absolute - * path to \a path if it couldn't be located in BROPATH + * Strip the BROPATH component from a path. + * @param path A file/directory path that may be within a BROPATH component. + * @return *path* minus the common BROPATH component (if any) removed. */ -std::string find_dir_in_bropath(const std::string& path); +std::string without_bropath_component(const std::string& path); /** * Locate a file within a given search path. From 52733b0501e5500c1144570fc799080a5f9a1801 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Tue, 22 Oct 2013 16:29:58 -0500 Subject: [PATCH 04/40] Quick optimization to Broxygen doc gathering. So script parsing is only ~2x slower rather than 20x. Turns out cloning Vals is particularly slow. Changed to just get a string description of the Val for initial value and redef tracking. --- src/broxygen/Document.cc | 18 +++++++++++++---- src/broxygen/Document.h | 7 ++----- src/broxygen/Manager.cc | 42 ++++++++++++++++++++++++++++++++++++++-- src/broxygen/Manager.h | 2 +- src/main.cc | 1 + src/parse.y | 2 +- 6 files changed, 59 insertions(+), 13 deletions(-) diff --git a/src/broxygen/Document.cc b/src/broxygen/Document.cc index ab8aca08aa..6b3a7309ce 100644 --- a/src/broxygen/Document.cc +++ b/src/broxygen/Document.cc @@ -15,19 +15,22 @@ PackageDocument::PackageDocument(const string& arg_name) IdentifierDocument::IdentifierDocument(ID* arg_id) : Document(), - comments(), id(arg_id), initial_val(), redefs(), fields(), + comments(), id(arg_id), initial_val_desc(), redefs(), fields(), last_field_seen() { Ref(id); if ( id->ID_Val() ) - initial_val = id->ID_Val()->Clone(); + { + ODesc d; + id->ID_Val()->Describe(&d); + initial_val_desc = d.Description(); + } } IdentifierDocument::~IdentifierDocument() { Unref(id); - //Unref(initial_val); // TODO: problematic w/ PatternVals for ( RedefList::const_iterator it = redefs.begin(); it != redefs.end(); ++it ) @@ -42,7 +45,14 @@ void IdentifierDocument::AddRedef(const string& script, { Redefinition* redef = new Redefinition(); redef->from_script = script; - redef->new_val = id->ID_Val() ? id->ID_Val()->Clone() : 0; + + if ( id->ID_Val() ) + { + ODesc d; + id->ID_Val()->Describe(&d); + redef->new_val_desc = d.Description(); + } + redef->comments = comments; redefs.push_back(redef); } diff --git a/src/broxygen/Document.h b/src/broxygen/Document.h index 34742f8069..9010dd0a38 100644 --- a/src/broxygen/Document.h +++ b/src/broxygen/Document.h @@ -81,11 +81,8 @@ public: private: struct Redefinition { - ~Redefinition() - { Unref(new_val); } - std::string from_script; - Val* new_val; + string new_val_desc; std::vector comments; }; @@ -106,7 +103,7 @@ private: std::vector comments; ID* id; - Val* initial_val; + string initial_val_desc; RedefList redefs; std::vector fields; RecordField* last_field_seen; diff --git a/src/broxygen/Manager.cc b/src/broxygen/Manager.cc index 8d01c51224..c886973269 100644 --- a/src/broxygen/Manager.cc +++ b/src/broxygen/Manager.cc @@ -84,9 +84,11 @@ static string PrettifyParams(const string& s) } Manager::Manager(const string& config) - : comment_buffer(), packages(), scripts(), identifiers(), all_docs(), - last_doc_seen(), incomplete_type() + : disabled(), comment_buffer(), packages(), scripts(), identifiers(), + all_docs(), last_doc_seen(), incomplete_type() { + if ( getenv("BRO_DISABLE_BROXYGEN") ) + disabled = true; // TODO config file stuff } @@ -99,16 +101,22 @@ Manager::~Manager() void Manager::InitPreScript() { + if ( disabled ) + return; // TODO: create file/proto analyzer doc } void Manager::InitPostScript() { + if ( disabled ) + return; // TODO: dependency resolution stuff? } void Manager::GenerateDocs() const { + if ( disabled ) + return; // TODO // may be a no-op if no config file @@ -116,6 +124,9 @@ void Manager::GenerateDocs() const void Manager::File(const string& path) { + if ( disabled ) + return; + string name = without_bropath_component(path); if ( scripts.find(name) != scripts.end() ) @@ -144,6 +155,9 @@ void Manager::File(const string& path) void Manager::ScriptDependency(const string& path, const string& dep) { + if ( disabled ) + return; + if ( dep.empty() ) { DbgAndWarn(fmt("Empty script doc dependency: %s", path.c_str())); @@ -173,6 +187,9 @@ void Manager::ScriptDependency(const string& path, const string& dep) void Manager::ModuleUsage(const string& path, const string& module) { + if ( disabled ) + return; + string name = without_bropath_component(path); ScriptMap::const_iterator it = scripts.find(name); @@ -189,6 +206,9 @@ void Manager::ModuleUsage(const string& path, const string& module) void Manager::StartType(ID* id) { + if ( disabled ) + return; + if ( id->GetLocationInfo() == &no_location ) { DbgAndWarn(fmt("Can't document %s, no location available", id->Name())); @@ -218,6 +238,9 @@ void Manager::StartType(ID* id) void Manager::Identifier(ID* id) { + if ( disabled ) + return; + if ( incomplete_type && incomplete_type->Name() == id->Name() ) { DBG_LOG(DBG_BROXYGEN, "Finished document for type %s", id->Name()); @@ -273,6 +296,9 @@ void Manager::Identifier(ID* id) void Manager::RecordField(const ID* id, const TypeDecl* field, const string& path) { + if ( disabled ) + return; + IdentifierDocument* idd = 0; if ( incomplete_type ) @@ -311,6 +337,9 @@ void Manager::RecordField(const ID* id, const TypeDecl* field, void Manager::Redef(const ID* id, const string& path) { + if ( disabled ) + return; + if ( path == "" ) // This is a redef defined on the command line. return; @@ -342,6 +371,9 @@ void Manager::Redef(const ID* id, const string& path) void Manager::SummaryComment(const string& script, const string& comment) { + if ( disabled ) + return; + string name = without_bropath_component(script); ScriptMap::const_iterator it = scripts.find(name); @@ -357,11 +389,17 @@ void Manager::SummaryComment(const string& script, const string& comment) void Manager::PreComment(const string& comment) { + if ( disabled ) + return; + comment_buffer.push_back(PrettifyParams(RemoveLeadingSpace(comment))); } void Manager::PostComment(const string& comment) { + if ( disabled ) + return; + IdentifierDocument* doc = dynamic_cast(last_doc_seen); if ( ! doc ) diff --git a/src/broxygen/Manager.h b/src/broxygen/Manager.h index 10b63af945..94ba2dec8e 100644 --- a/src/broxygen/Manager.h +++ b/src/broxygen/Manager.h @@ -13,7 +13,6 @@ namespace broxygen { // TODO: documentation... -// TODO: optimize parse time... maybe an env. option to disable doc collection? class Manager { @@ -60,6 +59,7 @@ private: void RegisterDoc(Document* d); + bool disabled; CommentBuffer comment_buffer; PackageMap packages; ScriptMap scripts; diff --git a/src/main.cc b/src/main.cc index 764b508749..2ab25460d4 100644 --- a/src/main.cc +++ b/src/main.cc @@ -227,6 +227,7 @@ void usage() fprintf(stderr, " $BRO_SEED_FILE | file to load seeds from (not set)\n"); fprintf(stderr, " $BRO_LOG_SUFFIX | ASCII log file extension (.%s)\n", logging::writer::Ascii::LogExt().c_str()); fprintf(stderr, " $BRO_PROFILER_FILE | Output file for script execution statistics (not set)\n"); + fprintf(stderr, " $BRO_DISABLE_BROXYGEN | Disable Broxygen documentation support (%s)\n", getenv("BRO_DISABLE_BROXYGEN") ? "set" : "not set"); fprintf(stderr, "\n"); fprintf(stderr, " Supported log formats: "); diff --git a/src/parse.y b/src/parse.y index 75be472853..5dc43a8476 100644 --- a/src/parse.y +++ b/src/parse.y @@ -963,7 +963,7 @@ type_decl: set_location(@1, @4); $$ = new TypeDecl($3, $1, $4, (in_record > 0)); - if ( in_record > 0 ) + if ( in_record > 0 && cur_decl_type_id ) broxygen_mgr->RecordField(cur_decl_type_id, $$, ::filename); } ; From 3a99aaaf0a7d5618553b52b10024fa03eec319b6 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Wed, 30 Oct 2013 16:07:57 -0500 Subject: [PATCH 05/40] Add BIF interface for retrieving comments/docs. The new BIFs: - get_identifier_comments - get_script_comments - get_package_readme - get_record_field_comments --- doc/scripts/DocSourcesList.cmake | 1 + src/CMakeLists.txt | 4 +- src/broxygen/CMakeLists.txt | 16 ++ src/broxygen/Document.cc | 49 +++- src/broxygen/Document.h | 48 +++- src/broxygen/Manager.cc | 218 ++++++++++-------- src/broxygen/Manager.h | 50 ++-- src/broxygen/broxygen.bif | 27 +++ src/parse.y | 9 +- src/scan.l | 7 +- .../canonified_loaded_scripts.log | 5 +- .../canonified_loaded_scripts.log | 5 +- .../doc.broxygen.comment_retrieval_bifs/out | 70 ++++++ .../doc/broxygen/comment_retrieval_bifs.bro | 111 +++++++++ 14 files changed, 489 insertions(+), 131 deletions(-) create mode 100644 src/broxygen/CMakeLists.txt create mode 100644 src/broxygen/broxygen.bif create mode 100644 testing/btest/Baseline/doc.broxygen.comment_retrieval_bifs/out create mode 100644 testing/btest/doc/broxygen/comment_retrieval_bifs.bro diff --git a/doc/scripts/DocSourcesList.cmake b/doc/scripts/DocSourcesList.cmake index 0b67ab1995..6e08ccee76 100644 --- a/doc/scripts/DocSourcesList.cmake +++ b/doc/scripts/DocSourcesList.cmake @@ -19,6 +19,7 @@ rest_target(${psd} base/init-bare.bro internal) rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/analyzer.bif.bro) rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/bloom-filter.bif.bro) rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/bro.bif.bro) +rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/broxygen.bif.bro) rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/cardinality-counter.bif.bro) rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/const.bif.bro) rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/event.bif.bro) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index dec2ea6680..48d651cd93 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -154,6 +154,7 @@ set(bro_PLUGIN_LIBS CACHE INTERNAL "plugin libraries" FORCE) add_subdirectory(analyzer) add_subdirectory(file_analysis) add_subdirectory(probabilistic) +add_subdirectory(broxygen) set(bro_SUBDIRS ${bro_SUBDIR_LIBS} @@ -370,9 +371,6 @@ set(bro_SRCS plugin/Plugin.cc plugin/Macros.h - broxygen/Manager.cc - broxygen/Document.cc - nb_dns.c digest.h ) diff --git a/src/broxygen/CMakeLists.txt b/src/broxygen/CMakeLists.txt new file mode 100644 index 0000000000..c447ea9188 --- /dev/null +++ b/src/broxygen/CMakeLists.txt @@ -0,0 +1,16 @@ +include(BroSubdir) + +include_directories(BEFORE + ${CMAKE_CURRENT_SOURCE_DIR} + ${CMAKE_CURRENT_BINARY_DIR} +) + +set(broxygen_SRCS + Manager.cc + Document.cc +) + +bif_target(broxygen.bif) +bro_add_subdir_library(broxygen ${broxygen_SRCS}) + +add_dependencies(bro_broxygen generate_outputs) diff --git a/src/broxygen/Document.cc b/src/broxygen/Document.cc index 6b3a7309ce..1b9618f66b 100644 --- a/src/broxygen/Document.cc +++ b/src/broxygen/Document.cc @@ -1,13 +1,29 @@ #include "Document.h" #include "util.h" +#include "Val.h" using namespace broxygen; using namespace std; +static string ImplodeStringVec(const vector& v) + { + string rval; + + for ( size_t i = 0; i < v.size(); ++i ) + { + if ( i > 0 ) + rval += '\n'; + + rval += v[i]; + } + + return rval; + } + PackageDocument::PackageDocument(const string& arg_name) : Document(), - pkg_loader_name(arg_name) + pkg_name(arg_name) { // TODO: probably need to determine modification times of all files // within the directory, recursively @@ -36,8 +52,9 @@ IdentifierDocument::~IdentifierDocument() ++it ) delete *it; - for ( size_t i = 0; i < fields.size(); ++i ) - delete fields[i]; + for ( RecordFieldMap::const_iterator it = fields.begin(); + it != fields.end(); ++it ) + delete it->second; } void IdentifierDocument::AddRedef(const string& script, @@ -59,21 +76,36 @@ void IdentifierDocument::AddRedef(const string& script, void IdentifierDocument::AddRecordField(const TypeDecl* field, const string& script, - std::vector& comments) + vector& comments) { RecordField* rf = new RecordField(); rf->field = new TypeDecl(*field); rf->from_script = script; rf->comments = comments; - fields.push_back(rf); + fields[rf->field->id] = rf; last_field_seen = rf; } +string IdentifierDocument::GetComments() const + { + return ImplodeStringVec(comments); + } + +string IdentifierDocument::GetFieldComments(const string& field) const + { + RecordFieldMap::const_iterator it = fields.find(field); + + if ( it == fields.end() ) + return string(); + + return ImplodeStringVec(it->second->comments); + } + ScriptDocument::ScriptDocument(const string& arg_name) : Document(), name(arg_name), is_pkg_loader(safe_basename(name) == PACKAGE_LOADER), - dependencies(), module_usages(), comments(), identifier_docs() + dependencies(), module_usages(), comments(), identifier_docs(), redefs() { } @@ -82,3 +114,8 @@ void ScriptDocument::AddIdentifierDoc(IdentifierDocument* doc) identifier_docs[doc->Name()] = doc; // TODO: sort things (e.g. function flavor, state var vs. option var) } + +string ScriptDocument::GetComments() const + { + return ImplodeStringVec(comments); + } diff --git a/src/broxygen/Document.h b/src/broxygen/Document.h index 9010dd0a38..fae3073ef9 100644 --- a/src/broxygen/Document.h +++ b/src/broxygen/Document.h @@ -10,7 +10,6 @@ #include #include "ID.h" -#include "Val.h" #include "Type.h" namespace broxygen { @@ -30,9 +29,13 @@ public: time_t GetModificationTime() const { return DoGetModificationTime(); } + std::string Name() const + { return DoName(); } + private: virtual time_t DoGetModificationTime() const = 0; + virtual std::string DoName() const = 0; }; class PackageDocument : public Document { @@ -41,13 +44,20 @@ public: PackageDocument(const std::string& name); + // TODO: can be comments from package README + std::string GetReadme() const + { return std::string(); } + private: // TODO time_t DoGetModificationTime() const { return 0; } - std::string pkg_loader_name; + std::string DoName() const + { return pkg_name; } + + std::string pkg_name; }; @@ -64,7 +74,7 @@ public: : comments.push_back(comment); } void AddComments(const std::vector& cmtns) - { comments.insert(comments.end(), cmtns.begin(), cmtns.end() ); } + { comments.insert(comments.end(), cmtns.begin(), cmtns.end()); } void AddRedef(const std::string& from_script, const std::vector& comments); @@ -72,14 +82,25 @@ public: void AddRecordField(const TypeDecl* field, const std::string& script, std::vector& comments); - string Name() const - { return id->Name(); } + void CompletedTypeDecl() + { last_field_seen = 0; } ID* GetID() const { return id; } + std::string GetComments() const; + + std::string GetFieldComments(const std::string& field) const; + private: + // TODO + time_t DoGetModificationTime() const + { return 0; } + + std::string DoName() const + { return id->Name(); } + struct Redefinition { std::string from_script; string new_val_desc; @@ -96,16 +117,13 @@ private: }; typedef std::list RedefList; - - // TODO - time_t DoGetModificationTime() const - { return 0; } + typedef std::map RecordFieldMap; std::vector comments; ID* id; string initial_val_desc; RedefList redefs; - std::vector fields; + RecordFieldMap fields; RecordField* last_field_seen; }; @@ -126,23 +144,33 @@ public: void AddIdentifierDoc(IdentifierDocument* doc); + void AddRedef(IdentifierDocument* doc) + { redefs.push_back(doc); } + bool IsPkgLoader() const { return is_pkg_loader; } + std::string GetComments() const; + private: typedef std::map IdentifierDocMap; + typedef std::list IdentifierDocList; // TODO time_t DoGetModificationTime() const { return 0; } + std::string DoName() const + { return name; } + std::string name; bool is_pkg_loader; std::set dependencies; std::set module_usages; std::vector comments; IdentifierDocMap identifier_docs; + IdentifierDocList redefs; }; diff --git a/src/broxygen/Manager.cc b/src/broxygen/Manager.cc index c886973269..2165b57122 100644 --- a/src/broxygen/Manager.cc +++ b/src/broxygen/Manager.cc @@ -84,8 +84,8 @@ static string PrettifyParams(const string& s) } Manager::Manager(const string& config) - : disabled(), comment_buffer(), packages(), scripts(), identifiers(), - all_docs(), last_doc_seen(), incomplete_type() + : disabled(), comment_buffer(), comment_buffer_map(), packages(), scripts(), + identifiers(), all_docs(), last_identifier_seen(), incomplete_type() { if ( getenv("BRO_DISABLE_BROXYGEN") ) disabled = true; @@ -94,9 +94,8 @@ Manager::Manager(const string& config) Manager::~Manager() { - for ( DocSet::const_iterator it = all_docs.begin(); it != all_docs.end(); - ++it ) - delete *it; + for ( size_t i = 0; i < all_docs.size(); ++i ) + delete all_docs[i]; } void Manager::InitPreScript() @@ -129,27 +128,31 @@ void Manager::File(const string& path) string name = without_bropath_component(path); - if ( scripts.find(name) != scripts.end() ) + if ( scripts.GetDocument(name) ) { DbgAndWarn(fmt("Duplicate script documentation: %s", name.c_str())); return; } ScriptDocument* doc = new ScriptDocument(name); - scripts[name] = doc; - RegisterDoc(doc); + scripts.map[name] = doc; + all_docs.push_back(doc); DBG_LOG(DBG_BROXYGEN, "Made ScriptDocument %s", name.c_str()); if ( ! doc->IsPkgLoader() ) return; - if ( packages.find(name) != packages.end() ) + name = safe_dirname(name); + + if ( packages.GetDocument(name) ) { DbgAndWarn(fmt("Duplicate package documentation: %s", name.c_str())); return; } - packages[name] = new PackageDocument(name); + PackageDocument* pkgdoc = new PackageDocument(name); + packages.map[name] = pkgdoc; + all_docs.push_back(pkgdoc); DBG_LOG(DBG_BROXYGEN, "Made PackageDocument %s", name.c_str()); } @@ -166,23 +169,22 @@ void Manager::ScriptDependency(const string& path, const string& dep) string name = without_bropath_component(path); string depname = without_bropath_component(dep); - ScriptMap::const_iterator it = scripts.find(name); + ScriptDocument* script_doc = scripts.GetDocument(name); - if ( it == scripts.end() ) + if ( ! script_doc ) { DbgAndWarn(fmt("Failed to add script doc dependency %s for %s", depname.c_str(), name.c_str())); return; } - it->second->AddDependency(depname); + script_doc->AddDependency(depname); DBG_LOG(DBG_BROXYGEN, "Added script dependency %s for %s", depname.c_str(), name.c_str()); - for ( CommentBuffer::const_iterator it = comment_buffer.begin(); - it != comment_buffer.end(); ++it ) + for ( size_t i = 0; i < comment_buffer.size(); ++i ) DbgAndWarn(fmt("Discarded extraneous Broxygen comment: %s", - it->c_str())); + comment_buffer[i].c_str())); } void Manager::ModuleUsage(const string& path, const string& module) @@ -191,19 +193,42 @@ void Manager::ModuleUsage(const string& path, const string& module) return; string name = without_bropath_component(path); - ScriptMap::const_iterator it = scripts.find(name); + ScriptDocument* script_doc = scripts.GetDocument(name); - if ( it == scripts.end() ) + if ( ! script_doc ) { DbgAndWarn(fmt("Failed to add module usage %s in %s", module.c_str(), name.c_str())); return; } + script_doc->AddModule(module); DBG_LOG(DBG_BROXYGEN, "Added module usage %s in %s", module.c_str(), name.c_str()); } +IdentifierDocument* Manager::CreateIdentifierDoc(ID* id, ScriptDocument* script) + { + IdentifierDocument* rval = new IdentifierDocument(id); + + rval->AddComments(comment_buffer); + comment_buffer.clear(); + + CommentBufferMap::const_iterator it = comment_buffer_map.find(id->Name()); + + if ( it != comment_buffer_map.end() ) + { + rval->AddComments(it->second); + comment_buffer_map.erase(it); + } + + all_docs.push_back(rval); + identifiers.map[id->Name()] = rval; + last_identifier_seen = rval; + script->AddIdentifierDoc(rval); + return rval; + } + void Manager::StartType(ID* id) { if ( disabled ) @@ -216,26 +241,34 @@ void Manager::StartType(ID* id) } string script = without_bropath_component(id->GetLocationInfo()->filename); - ScriptMap::const_iterator sit = scripts.find(script); + ScriptDocument* script_doc = scripts.GetDocument(script); - if ( sit == scripts.end() ) + if ( ! script_doc ) { DbgAndWarn(fmt("Can't document identifier %s, lookup of %s failed", id->Name(), script.c_str())); return; } - IdentifierDocument* doc = new IdentifierDocument(id); - doc->AddComments(comment_buffer); - comment_buffer.clear(); - identifiers[id->Name()] = doc; - RegisterDoc(doc); - sit->second->AddIdentifierDoc(doc); - incomplete_type = doc; + incomplete_type = CreateIdentifierDoc(id, script_doc); DBG_LOG(DBG_BROXYGEN, "Made IdentifierDocument (incomplete) %s, in %s", id->Name(), script.c_str()); } +void Manager::StartRedef(ID* id) + { + if ( disabled ) + return; + + IdentifierDocument* id_doc = identifiers.GetDocument(id->Name()); + + if ( id_doc ) + last_identifier_seen = id_doc; + else + DbgAndWarn(fmt("Broxygen redef tracking unknown identifier: %s", + id->Name())); + } + void Manager::Identifier(ID* id) { if ( disabled ) @@ -244,6 +277,7 @@ void Manager::Identifier(ID* id) if ( incomplete_type && incomplete_type->Name() == id->Name() ) { DBG_LOG(DBG_BROXYGEN, "Finished document for type %s", id->Name()); + incomplete_type->CompletedTypeDecl(); incomplete_type = 0; return; } @@ -257,14 +291,14 @@ void Manager::Identifier(ID* id) return; } - IdentifierMap::const_iterator iit = identifiers.find(id->Name()); + IdentifierDocument* id_doc = identifiers.GetDocument(id->Name()); - if ( iit != identifiers.end() ) + if ( id_doc ) { - if ( IsFunc(iit->second->GetID()->Type()->Tag()) ) + if ( IsFunc(id_doc->GetID()->Type()->Tag()) ) { // Function may already been seen (declaration versus body). - iit->second->AddComments(comment_buffer); + id_doc->AddComments(comment_buffer); comment_buffer.clear(); return; } @@ -274,21 +308,16 @@ void Manager::Identifier(ID* id) } string script = without_bropath_component(id->GetLocationInfo()->filename); - ScriptMap::const_iterator sit = scripts.find(script); + ScriptDocument* script_doc = scripts.GetDocument(script); - if ( sit == scripts.end() ) + if ( ! script_doc ) { DbgAndWarn(fmt("Can't document identifier %s, lookup of %s failed", id->Name(), script.c_str())); return; } - IdentifierDocument* doc = new IdentifierDocument(id); - doc->AddComments(comment_buffer); - comment_buffer.clear(); - identifiers[id->Name()] = doc; - RegisterDoc(doc); - sit->second->AddIdentifierDoc(doc); + CreateIdentifierDoc(id, script_doc); DBG_LOG(DBG_BROXYGEN, "Made IdentifierDocument %s, in script %s", id->Name(), script.c_str()); } @@ -299,36 +328,16 @@ void Manager::RecordField(const ID* id, const TypeDecl* field, if ( disabled ) return; - IdentifierDocument* idd = 0; + IdentifierDocument* idd = identifiers.GetDocument(id->Name()); - if ( incomplete_type ) + if ( ! idd ) { - if ( incomplete_type->Name() != id->Name() ) - { - DbgAndWarn(fmt("Can't document record field %s in record %s, " - "expected record %s", field->id, id->Name(), - incomplete_type->Name().c_str())); - return; - } - - idd = incomplete_type; - } - else - { - IdentifierMap::const_iterator it = identifiers.find(id->Name()); - - if ( it == identifiers.end() ) - { - DbgAndWarn(fmt("Can't document record field %s, unknown record: %s", - field->id, id->Name())); - return; - } - - idd = it->second; + DbgAndWarn(fmt("Can't document record field %s, unknown record: %s", + field->id, id->Name())); + return; } string script = without_bropath_component(path); - idd->AddRecordField(field, script, comment_buffer); comment_buffer.clear(); DBG_LOG(DBG_BROXYGEN, "Document record field %s, identifier %s, script %s", @@ -344,9 +353,9 @@ void Manager::Redef(const ID* id, const string& path) // This is a redef defined on the command line. return; - IdentifierMap::const_iterator iit = identifiers.find(id->Name()); + IdentifierDocument* id_doc = identifiers.GetDocument(id->Name()); - if ( iit == identifiers.end() ) + if ( ! id_doc ) { DbgAndWarn(fmt("Can't document redef of %s, identifier lookup failed", id->Name())); @@ -354,18 +363,20 @@ void Manager::Redef(const ID* id, const string& path) } string from_script = without_bropath_component(path); - ScriptMap::const_iterator sit = scripts.find(from_script); + ScriptDocument* script_doc = scripts.GetDocument(from_script); - if ( sit == scripts.end() ) + if ( ! script_doc ) { DbgAndWarn(fmt("Can't document redef of %s, lookup of %s failed", id->Name(), from_script.c_str())); return; } - iit->second->AddRedef(from_script, comment_buffer); + id_doc->AddRedef(from_script, comment_buffer); + script_doc->AddRedef(id_doc); comment_buffer.clear(); - DBG_LOG(DBG_BROXYGEN, "Added redef of %s to %s", + last_identifier_seen = id_doc; + DBG_LOG(DBG_BROXYGEN, "Added redef of %s from %s", id->Name(), from_script.c_str()); } @@ -375,16 +386,13 @@ void Manager::SummaryComment(const string& script, const string& comment) return; string name = without_bropath_component(script); - ScriptMap::const_iterator it = scripts.find(name); + ScriptDocument* doc = scripts.GetDocument(name); - if ( it == scripts.end() ) - { + if ( doc ) + doc->AddComment(RemoveLeadingSpace(comment)); + else DbgAndWarn(fmt("Lookup of script %s failed for summary comment %s", name.c_str(), comment.c_str())); - return; - } - - it->second->AddComment(RemoveLeadingSpace(comment)); } void Manager::PreComment(const string& comment) @@ -392,31 +400,59 @@ void Manager::PreComment(const string& comment) if ( disabled ) return; - comment_buffer.push_back(PrettifyParams(RemoveLeadingSpace(comment))); + comment_buffer.push_back(RemoveLeadingSpace(comment)); } -void Manager::PostComment(const string& comment) +void Manager::PostComment(const string& comment, const string& id_hint) { if ( disabled ) return; - IdentifierDocument* doc = dynamic_cast(last_doc_seen); - - if ( ! doc ) + if ( id_hint.empty() ) { - DbgAndWarn(fmt("Discarded comment not associated w/ an identifier %s", - comment.c_str())); + if ( last_identifier_seen ) + last_identifier_seen->AddComment(RemoveLeadingSpace(comment)); + else + DbgAndWarn(fmt("Discarded unassociated Broxygen comment %s", + comment.c_str())); + return; } - doc->AddComment(RemoveLeadingSpace(comment)); + if ( last_identifier_seen && + last_identifier_seen->Name() == id_hint ) + last_identifier_seen->AddComment(RemoveLeadingSpace(comment)); + else + // Assume identifier it's associated w/ is coming later. + comment_buffer_map[id_hint].push_back(RemoveLeadingSpace(comment)); } -void Manager::RegisterDoc(Document* d) +StringVal* Manager::GetIdentifierComments(const string& name) const { - if ( ! d ) - return; - - all_docs.insert(d); - last_doc_seen = d; + IdentifierDocument* d = identifiers.GetDocument(name); + return new StringVal(d ? d->GetComments() : ""); + } + +StringVal* Manager::GetScriptComments(const string& name) const + { + ScriptDocument* d = scripts.GetDocument(name); + return new StringVal(d ? d->GetComments() : ""); + } + +StringVal* Manager::GetPackageReadme(const string& name) const + { + PackageDocument* d = packages.GetDocument(name); + return new StringVal(d ? d->GetReadme() : ""); + } + +StringVal* Manager::GetRecordFieldComments(const string& name) const + { + size_t i = name.find('$'); + + if ( i > name.size() - 2 ) + // '$' is last char in string or not found. + return new StringVal(""); + + IdentifierDocument* d = identifiers.GetDocument(name.substr(0, i)); + return new StringVal(d ? d->GetFieldComments(name.substr(i + 1)) : ""); } diff --git a/src/broxygen/Manager.h b/src/broxygen/Manager.h index 94ba2dec8e..4207c14630 100644 --- a/src/broxygen/Manager.h +++ b/src/broxygen/Manager.h @@ -4,16 +4,29 @@ #include "Document.h" #include "ID.h" #include "Type.h" +#include "Val.h" #include -#include -#include #include +#include namespace broxygen { // TODO: documentation... +template +struct DocumentMap { + typedef std::map MapType; + + T* GetDocument(const std::string& name) const + { + typename MapType::const_iterator it = map.find(name); + return it == map.end() ? 0 : it->second; + } + + MapType map; +}; + class Manager { public: @@ -36,6 +49,8 @@ public: void StartType(ID* id); + void StartRedef(ID* id); + void Identifier(ID* id); void RecordField(const ID* id, const TypeDecl* field, @@ -47,25 +62,32 @@ public: void PreComment(const std::string& comment); - void PostComment(const std::string& comment); + void PostComment(const std::string& comment, + const std::string& identifier_hint = ""); + + StringVal* GetIdentifierComments(const std::string& name) const; + + StringVal* GetScriptComments(const std::string& name) const; + + StringVal* GetPackageReadme(const std::string& name) const; + + StringVal* GetRecordFieldComments(const std::string& name) const; private: typedef std::vector CommentBuffer; - typedef std::map PackageMap; - typedef std::map ScriptMap; - typedef std::map IdentifierMap; - typedef std::set DocSet; + typedef std::map CommentBufferMap; - void RegisterDoc(Document* d); + IdentifierDocument* CreateIdentifierDoc(ID* id, ScriptDocument* script); bool disabled; - CommentBuffer comment_buffer; - PackageMap packages; - ScriptMap scripts; - IdentifierMap identifiers; - DocSet all_docs; - Document* last_doc_seen; + CommentBuffer comment_buffer; // For whatever next identifier that comes in. + CommentBufferMap comment_buffer_map; // For a particular identifier. + DocumentMap packages; + DocumentMap scripts; + DocumentMap identifiers; + std::vector all_docs; + IdentifierDocument* last_identifier_seen; IdentifierDocument* incomplete_type; }; diff --git a/src/broxygen/broxygen.bif b/src/broxygen/broxygen.bif new file mode 100644 index 0000000000..a241b2c985 --- /dev/null +++ b/src/broxygen/broxygen.bif @@ -0,0 +1,27 @@ +##! Functions for querying script, package, or variable documentation. + +%%{ +#include "broxygen/Manager.h" +%%} + +# TODO: documentation + +function get_identifier_comments%(name: string%): string + %{ + return broxygen_mgr->GetIdentifierComments(name->CheckString()); + %} + +function get_script_comments%(name: string%): string + %{ + return broxygen_mgr->GetScriptComments(name->CheckString()); + %} + +function get_package_readme%(name: string%): string + %{ + return broxygen_mgr->GetPackageReadme(name->CheckString()); + %} + +function get_record_field_comments%(name: string%): string + %{ + return broxygen_mgr->GetRecordFieldComments(name->CheckString()); + %} diff --git a/src/parse.y b/src/parse.y index 5dc43a8476..d428452d66 100644 --- a/src/parse.y +++ b/src/parse.y @@ -1028,8 +1028,13 @@ decl: // Broxygen already grabbed new enum IDs as the type created them. } - | TOK_REDEF TOK_RECORD global_id { cur_decl_type_id = $3; } TOK_ADD_TO - '{' { ++in_record; } type_decl_list { --in_record; } '}' opt_attr ';' + | TOK_REDEF TOK_RECORD global_id + { cur_decl_type_id = $3; broxygen_mgr->StartRedef($3); } + TOK_ADD_TO '{' + { ++in_record; } + type_decl_list + { --in_record; } + '}' opt_attr ';' { cur_decl_type_id = 0; diff --git a/src/scan.l b/src/scan.l index 3ac256969a..2477afa645 100644 --- a/src/scan.l +++ b/src/scan.l @@ -31,6 +31,7 @@ #include "broxygen/Manager.h" extern YYLTYPE yylloc; // holds start line and column of token +extern EnumType* cur_enum_type; // Track the @if... depth. ptr_compat_int current_depth = 0; @@ -39,6 +40,7 @@ int_list if_stack; int line_number = 1; const char* filename = 0; // Absolute path of file currently being parsed. +static const char* last_id_tok = 0; char last_tok[128]; @@ -137,7 +139,9 @@ ESCSEQ (\\([^\n]|[0-7]+|x[[:xdigit:]]+)) } ##<.* { - broxygen_mgr->PostComment(yytext + 3); + string hint(cur_enum_type && last_id_tok ? + make_full_var_name(current_module.c_str(), last_id_tok) : ""); + broxygen_mgr->PostComment(yytext + 3, hint); } ##.* { @@ -370,6 +374,7 @@ F RET_CONST(new Val(false, TYPE_BOOL)) {ID} { yylval.str = copy_string(yytext); + last_id_tok = yylval.str; return TOK_ID; } diff --git a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log index 5ee8158ddf..0218611d1c 100644 --- a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log @@ -3,7 +3,7 @@ #empty_field (empty) #unset_field - #path loaded_scripts -#open 2013-08-31-17-46-55 +#open 2013-10-30-16-52-11 #fields name #types string scripts/base/init-bare.bro @@ -96,8 +96,9 @@ scripts/base/init-bare.bro scripts/base/utils/patterns.bro build/scripts/base/bif/__load__.bro build/scripts/base/bif/bloom-filter.bif.bro + build/scripts/base/bif/broxygen.bif.bro build/scripts/base/bif/cardinality-counter.bif.bro build/scripts/base/bif/top-k.bif.bro scripts/policy/misc/loaded-scripts.bro scripts/base/utils/paths.bro -#close 2013-08-31-17-46-55 +#close 2013-10-30-16-52-11 diff --git a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log index a11933c79a..90145d94fb 100644 --- a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log @@ -3,7 +3,7 @@ #empty_field (empty) #unset_field - #path loaded_scripts -#open 2013-08-31-17-46-56 +#open 2013-10-30-16-52-28 #fields name #types string scripts/base/init-bare.bro @@ -96,6 +96,7 @@ scripts/base/init-bare.bro scripts/base/utils/patterns.bro build/scripts/base/bif/__load__.bro build/scripts/base/bif/bloom-filter.bif.bro + build/scripts/base/bif/broxygen.bif.bro build/scripts/base/bif/cardinality-counter.bif.bro build/scripts/base/bif/top-k.bif.bro scripts/base/init-default.bro @@ -220,4 +221,4 @@ scripts/base/init-default.bro scripts/base/files/unified2/main.bro scripts/base/misc/find-checksum-offloading.bro scripts/policy/misc/loaded-scripts.bro -#close 2013-08-31-17-46-56 +#close 2013-10-30-16-52-28 diff --git a/testing/btest/Baseline/doc.broxygen.comment_retrieval_bifs/out b/testing/btest/Baseline/doc.broxygen.comment_retrieval_bifs/out new file mode 100644 index 0000000000..2a01fa0a94 --- /dev/null +++ b/testing/btest/Baseline/doc.broxygen.comment_retrieval_bifs/out @@ -0,0 +1,70 @@ +This is a test script. +With some summary comments. +myvar: + Hello world. This is an option. + With some more description here. + And here. + Maybe just one more. +print_lines: + This function prints a string line by line. + + lines: A string to print line by line, w/ lines delimited by newline chars. + And some more comments on the function implementation. +mytype: + This is an alias for count. +myrecord: + My record type. +myrecord$aaa: + The first field. + Does something... + Done w/ aaa. +myrecord$bbb: + The second field. + Done w/ bbb. + No really, done w/ bbb. +myrecord$ccc: + Third field. + Done w/ ccc. +myrecord$ddd: + Fourth field. + Done w/ ddd. +myrecord$eee: + First redef'd field. + With two lines of comments. + And two post-notation comments. + Done w/ eee. +myrecord$fff: + Second redef'd field. + Done w/ fff. +myrecord$ggg: + Third redef'd field. + Done w/ ggg. +myenum: + My enum type; +FIRST: + First enum value. + I know, the name isn't clever. + Done w/ first. +SECOND: + Second enum value. + Done w/ second. +THIRD: + Third enum value. + Done w/ third. + Done w/ third again. +FORTH: + SIC. + It's a programming language. + Using Reverse Polish Notation. + Done w/ forth. +FIFTH: + First redef'd enum val. + Done w/ fifth. +SIXTH: + Second redef'd enum val. + Done w/ sixth. +SEVENTH: + Third redef'd enum val. + Lucky number seven. + Still works with comma. + Done w/ seventh. diff --git a/testing/btest/doc/broxygen/comment_retrieval_bifs.bro b/testing/btest/doc/broxygen/comment_retrieval_bifs.bro new file mode 100644 index 0000000000..77a6058d71 --- /dev/null +++ b/testing/btest/doc/broxygen/comment_retrieval_bifs.bro @@ -0,0 +1,111 @@ +# @TEST-EXEC: bro -b %INPUT >out +# @TEST-EXEC: btest-diff out + +##! This is a test script. +##! With some summary comments. + +## Hello world. This is an option. +## With some more description here. +## And here. +const myvar = 7 &redef; ##< Maybe just one more. + +## This function prints a string line by line. +## +## lines: A string to print line by line, w/ lines delimited by newline chars. +global print_lines: function(lines: string, prefix: string &default=""); + +## And some more comments on the function implementation. +function print_lines(lines: string, prefix: string) + { + local v: vector of string; + local line_table = split(lines, /\n/); + + for ( i in line_table ) + v[i] = line_table[i]; + + for ( i in v ) + print fmt("%s%s", prefix, v[i]); + } + +function print_comments(name: string, func: function(name: string): string) + { + print fmt("%s:", name); + print_lines(func(name), " "); + } + +## This is an alias for count. +type mytype: count; + +## My record type. +type myrecord: record { + ## The first field. + ## Does something... + aaa: count; ##< Done w/ aaa. + ## The second field. + bbb: string; ##< Done w/ bbb. + ##< No really, done w/ bbb. + ## Third field. + ccc: int; ##< Done w/ ccc. + ## Fourth field. + ddd: interval; ##< Done w/ ddd. +}; + + +## My enum type; +type myenum: enum { + ## First enum value. + ## I know, the name isn't clever. + FIRST, ##< Done w/ first. + ## Second enum value. + SECOND, ##< Done w/ second. + ## Third enum value. + THIRD, ##< Done w/ third. + ##< Done w/ third again. + ## SIC. + ## It's a programming language. + FORTH ##< Using Reverse Polish Notation. + ##< Done w/ forth. +}; + +redef record myrecord += { + ## First redef'd field. + ## With two lines of comments. + eee: count &optional; ##< And two post-notation comments. + ##< Done w/ eee. + ## Second redef'd field. + fff: count &optional; ##< Done w/ fff. + ## Third redef'd field. + ggg: count &optional; ##< Done w/ ggg. +}; + +redef enum myenum += { + ## First redef'd enum val. + FIFTH, ##< Done w/ fifth. + ## Second redef'd enum val. + SIXTH, ##< Done w/ sixth. + ## Third redef'd enum val. + ## Lucky number seven. + SEVENTH, ##< Still works with comma. + ##< Done w/ seventh. +}; + +print_lines(get_script_comments(@DIR + "/" + @FILENAME)); +print_comments("myvar", get_identifier_comments); +print_comments("print_lines", get_identifier_comments); +print_comments("mytype", get_identifier_comments); +print_comments("myrecord", get_identifier_comments); +print_comments("myrecord$aaa", get_record_field_comments); +print_comments("myrecord$bbb", get_record_field_comments); +print_comments("myrecord$ccc", get_record_field_comments); +print_comments("myrecord$ddd", get_record_field_comments); +print_comments("myrecord$eee", get_record_field_comments); +print_comments("myrecord$fff", get_record_field_comments); +print_comments("myrecord$ggg", get_record_field_comments); +print_comments("myenum", get_identifier_comments); +print_comments("FIRST", get_identifier_comments); +print_comments("SECOND", get_identifier_comments); +print_comments("THIRD", get_identifier_comments); +print_comments("FORTH", get_identifier_comments); +print_comments("FIFTH", get_identifier_comments); +print_comments("SIXTH", get_identifier_comments); +print_comments("SEVENTH", get_identifier_comments); From 3046013d6922ecece86d87eac6becff4de03ce3d Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Mon, 4 Nov 2013 11:42:39 -0600 Subject: [PATCH 06/40] Replace safe_basename/safe_dirname w/ SafeBasename/SafeDirname. So errors can be better handled. --- src/broxygen/Document.cc | 2 +- src/broxygen/Manager.cc | 2 +- src/scan.l | 14 +++--------- src/util.cc | 46 +++++++++++++++++++++------------------- src/util.h | 42 ++++++++++++++++++++++++++++++------ 5 files changed, 65 insertions(+), 41 deletions(-) diff --git a/src/broxygen/Document.cc b/src/broxygen/Document.cc index 1b9618f66b..41622aa41f 100644 --- a/src/broxygen/Document.cc +++ b/src/broxygen/Document.cc @@ -104,7 +104,7 @@ string IdentifierDocument::GetFieldComments(const string& field) const ScriptDocument::ScriptDocument(const string& arg_name) : Document(), name(arg_name), - is_pkg_loader(safe_basename(name) == PACKAGE_LOADER), + is_pkg_loader(SafeBasename(name).result == PACKAGE_LOADER), dependencies(), module_usages(), comments(), identifier_docs(), redefs() { } diff --git a/src/broxygen/Manager.cc b/src/broxygen/Manager.cc index 2165b57122..72ef0aab1e 100644 --- a/src/broxygen/Manager.cc +++ b/src/broxygen/Manager.cc @@ -142,7 +142,7 @@ void Manager::File(const string& path) if ( ! doc->IsPkgLoader() ) return; - name = safe_dirname(name); + name = SafeDirname(name).result; if ( packages.GetDocument(name) ) { diff --git a/src/scan.l b/src/scan.l index 2477afa645..3dc68a52cc 100644 --- a/src/scan.l +++ b/src/scan.l @@ -59,7 +59,7 @@ static string find_relative_file(const string& filename, const string& ext) return string(); if ( filename[0] == '.' ) - return find_file(filename, safe_dirname(::filename), ext); + return find_file(filename, SafeDirname(::filename).result, ext); else return find_file(filename, bro_path(), ext); } @@ -275,7 +275,7 @@ when return TOK_WHEN; @DEBUG return TOK_DEBUG; // marks input for debugger @DIR { - string rval = safe_dirname(::filename); + string rval = SafeDirname(::filename).result; if ( ! rval.empty() && rval[0] == '.' ) { @@ -291,15 +291,7 @@ when return TOK_WHEN; } @FILENAME { - char* filename_copy = copy_string(::filename); - const char* bname = basename(filename_copy); - - if ( ! bname ) - reporter->InternalError("basename failed: %s", strerror(errno)); - - StringVal* rval = new StringVal(bname); - delete [] filename_copy; - RET_CONST(rval); + RET_CONST(new StringVal(SafeBasename(::filename).result)); } @load{WS}{FILE} { diff --git a/src/util.cc b/src/util.cc index bf98bb9ed3..eacbf46eb7 100644 --- a/src/util.cc +++ b/src/util.cc @@ -944,34 +944,36 @@ FILE* open_package(string& path, const string& mode) return 0; } -string safe_dirname(const char* path) +SafePathOp::SafePathOp(PathOpFn fn, const char* path, bool error_aborts) { - if ( ! path ) - return "."; - return safe_dirname(string(path)); + DoFunc(fn, path ? path : "", error_aborts); } -string safe_dirname(const string& path) +SafePathOp::SafePathOp(PathOpFn fn, const string& path, bool error_aborts) + { + DoFunc(fn, path, error_aborts); + } + +void SafePathOp::DoFunc(PathOpFn fn, const string& path, bool error_aborts) { char* tmp = copy_string(path.c_str()); - string rval = dirname(tmp); - delete [] tmp; - return rval; - } + char* rval = fn(tmp); -string safe_basename(const char* path) - { - if ( ! path ) - return "."; - return safe_basename(string(path)); - } + if ( rval ) + { + result = rval; + error = false; + } + else + { + if ( error_aborts ) + reporter->InternalError("Path operation failed on %s: %s", + tmp ? tmp : "", strerror(errno)); + else + error = true; + } -string safe_basename(const string& path) - { - char* tmp = copy_string(path.c_str()); - string rval = basename(tmp); delete [] tmp; - return rval; } string flatten_script_name(const string& name, const string& prefix) @@ -981,8 +983,8 @@ string flatten_script_name(const string& name, const string& prefix) if ( ! rval.empty() ) rval.append("."); - if ( safe_basename(name) == PACKAGE_LOADER ) - rval.append(safe_dirname(name)); + if ( SafeBasename(name).result == PACKAGE_LOADER ) + rval.append(SafeDirname(name).result); else rval.append(name); diff --git a/src/util.h b/src/util.h index faca450cd8..17dfb9c217 100644 --- a/src/util.h +++ b/src/util.h @@ -22,6 +22,7 @@ #include #include #include +#include #include "config.h" #if __STDC__ @@ -208,13 +209,42 @@ extern const char* bro_path(); extern const char* bro_magic_path(); extern std::string bro_prefixes(); -// Wrappers for dirname(3) that won't modify argument. -std::string safe_dirname(const char* path); -std::string safe_dirname(const std::string& path); +/** + * Wrapper class for functions like dirname(3) or basename(3) that won't + * modify the path argument and may optionally abort execution on error. + */ +class SafePathOp { +public: -// Wrappers for basename(3) that won't modify argument. -std::string safe_basename(const char* path); -std::string safe_basename(const std::string& path); + typedef char*(*PathOpFn)(char*); + + SafePathOp(PathOpFn fn, const char* path, bool error_aborts = true); + SafePathOp(PathOpFn fn, const std::string& path, bool error_aborts = true); + + std::string result; + bool error; + +private: + + void DoFunc(PathOpFn fn, const std::string& path, bool error_aborts = true); +}; + +class SafeDirname : public SafePathOp { +public: + + SafeDirname(const char* path, bool error_aborts = true) + : SafePathOp(&dirname, path, error_aborts) { } + SafeDirname(const std::string& path, bool error_aborts = true) + : SafePathOp(&dirname, path, error_aborts) { } +}; + +class SafeBasename : public SafePathOp { +public: + SafeBasename(const char* path, bool error_aborts = true) + : SafePathOp(&basename, path, error_aborts) { } + SafeBasename(const std::string& path, bool error_aborts = true) + : SafePathOp(&basename, path, error_aborts) { } +}; /** * Flatten a script name by replacing '/' path separators with '.'. From 1fabbd441c2817a603181f453f82c859917fa3a8 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Tue, 5 Nov 2013 10:54:05 -0600 Subject: [PATCH 07/40] Remove unneeded Broxygen comments in scan.bro. They just duplicated the text from where the events are originally declared and also it's not generally useful to Broxygen-style comment event *handlers* (they're more of an implementation detail of a script, not a user-facing element). --- scripts/policy/misc/scan.bro | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/scripts/policy/misc/scan.bro b/scripts/policy/misc/scan.bro index e458f6c450..d70f8f9e79 100644 --- a/scripts/policy/misc/scan.bro +++ b/scripts/policy/misc/scan.bro @@ -147,11 +147,6 @@ function is_reverse_failed_conn(c: connection): bool return F; } -## Generated for an unsuccessful connection attempt. This -## event is raised when an originator unsuccessfully attempted -## to establish a connection. "Unsuccessful" is defined as at least -## tcp_attempt_delay seconds having elapsed since the originator first sent a -## connection establishment packet to the destination without seeing a reply. event connection_attempt(c: connection) { local is_reverse_scan = F; @@ -161,9 +156,6 @@ event connection_attempt(c: connection) add_sumstats(c$id, is_reverse_scan); } -## Generated for a rejected TCP connection. This event is raised when an -## originator attempted to setup a TCP connection but the responder replied with -## a RST packet denying it. event connection_rejected(c: connection) { local is_reverse_scan = F; @@ -173,9 +165,6 @@ event connection_rejected(c: connection) add_sumstats(c$id, is_reverse_scan); } -## Generated when an endpoint aborted a TCP connection. The event is raised when -## one endpoint of an *established* TCP connection aborted by sending a RST -## packet. event connection_reset(c: connection) { if ( is_failed_conn(c) ) @@ -184,7 +173,6 @@ event connection_reset(c: connection) add_sumstats(c$id, T); } -## Generated for each still-open connection when Bro terminates. event connection_pending(c: connection) { if ( is_failed_conn(c) ) From bdd359d58c6abdb8dce2aedd4199c7a320bcb467 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Tue, 5 Nov 2013 16:40:24 -0600 Subject: [PATCH 08/40] Broxygen can now read a config file specifying particular targets. Though nothing currently gets built as most dependency/outdated checks and doc-generation methods are still skeleton code. --- src/broxygen/CMakeLists.txt | 3 +- src/broxygen/Configuration.cc | 123 +++++++++++++++++++++++++ src/broxygen/Configuration.h | 169 ++++++++++++++++++++++++++++++++++ src/broxygen/Document.cc | 6 +- src/broxygen/Document.h | 8 +- src/broxygen/Manager.cc | 15 +-- src/broxygen/Manager.h | 16 ++-- src/util.cc | 4 +- src/util.h | 5 + 9 files changed, 325 insertions(+), 24 deletions(-) create mode 100644 src/broxygen/Configuration.cc create mode 100644 src/broxygen/Configuration.h diff --git a/src/broxygen/CMakeLists.txt b/src/broxygen/CMakeLists.txt index c447ea9188..4e5c5e5d0d 100644 --- a/src/broxygen/CMakeLists.txt +++ b/src/broxygen/CMakeLists.txt @@ -6,8 +6,9 @@ include_directories(BEFORE ) set(broxygen_SRCS - Manager.cc + Configuration.cc Document.cc + Manager.cc ) bif_target(broxygen.bif) diff --git a/src/broxygen/Configuration.cc b/src/broxygen/Configuration.cc new file mode 100644 index 0000000000..cd8ee7be56 --- /dev/null +++ b/src/broxygen/Configuration.cc @@ -0,0 +1,123 @@ +#include "Configuration.h" + +#include "util.h" +#include "Reporter.h" + +#include +#include +#include +#include + +using namespace broxygen; +using namespace std; + +typedef map target_factory_map; + +static target_factory_map create_target_factory_map() + { + target_factory_map rval; + rval["package_index"] = &PackageTarget::Instantiate; + rval["package"] = &PackageTarget::Instantiate; + rval["proto_analyzer"] = &ProtoAnalyzerTarget::Instantiate; + rval["file_analyzer"] = &FileAnalyzerTarget::Instantiate; + rval["script_summary"] = &ScriptTarget::Instantiate; + rval["script_index"] = &ScriptTarget::Instantiate; + rval["script"] = &ScriptTarget::Instantiate; + rval["identifier"] = &IdentifierTarget::Instantiate; + return rval; + } + +static target_factory_map target_instantiators = create_target_factory_map(); + +bool Target::MatchesPattern(Document* doc) const + { + // TODO: prefix matching or full regex? + + if ( doc->Name() == pattern ) + { + DBG_LOG(DBG_BROXYGEN, "Doc '%s'' matched pattern for target '%s'", + doc->Name().c_str(), name.c_str()); + return true; + } + + return false; + } + +template +void filter_matching_docs(const std::vector& filter_from, Target* t) + { + for ( size_t i = 0; i < filter_from.size(); ++i ) + { + T* d = dynamic_cast(filter_from[i]); + + if ( ! d ) + continue; + + if ( t->MatchesPattern(d) ) + t->AddDependency(d); + } + } + +void IdentifierTarget::DoFindDependencies(const std::vector& docs) + { + filter_matching_docs(docs, this); + } + +Config::Config(const string& file, const string& delim) + { + if ( file.empty() ) + return; + + ifstream f(file.c_str()); + + if ( ! f.is_open() ) + reporter->InternalError("failed to open Broxygen config file %s: %s", + file.c_str(), strerror(errno)); + + string line; + + while ( getline(f, line) ) + { + vector tokens; + tokenize_string(line, delim, &tokens); + tokens.erase(remove(tokens.begin(), tokens.end(), ""), tokens.end()); + + if ( tokens.size() != 3 ) + reporter->InternalError("malformed Broxygen target: %s", + line.c_str()); + + target_factory_map::const_iterator it = + target_instantiators.find(tokens[0]); + + if ( it == target_instantiators.end() ) + reporter->InternalError("unkown Broxygen target type: %s", + tokens[0].c_str()); + + targets.push_back(it->second(tokens[1], tokens[2])); + } + + if ( f.bad() ) + reporter->InternalError("error reading Broxygen config file %s: %s", + file.c_str(), strerror(errno)); + } + +Config::~Config() + { + for ( target_list::const_iterator it = targets.begin(); + it != targets.end(); ++it ) + delete *it; + } + +void Config::FindDependencies(const vector& docs) + { + for ( target_list::const_iterator it = targets.begin(); + it != targets.end(); ++it ) + (*it)->FindDependencies(docs); + } + +void Config::GenerateDocs() const + { + for ( target_list::const_iterator it = targets.begin(); + it != targets.end(); ++it ) + (*it)->Generate(); + } diff --git a/src/broxygen/Configuration.h b/src/broxygen/Configuration.h new file mode 100644 index 0000000000..2aef6f9806 --- /dev/null +++ b/src/broxygen/Configuration.h @@ -0,0 +1,169 @@ +#ifndef BROXYGEN_CONFIGURATION_H +#define BROXYGEN_CONFIGURATION_H + +#include "Document.h" + +#include +#include +#include + +namespace broxygen { + +// TODO: documentation... + +class Target { +public: + + typedef Target* (*factory_fn)(const std::string&, const std::string&); + + virtual ~Target() { } + + void FindDependencies(const std::vector& docs) + { DoFindDependencies(docs); } + + void Generate() const + { DoGenerate(); } + + bool MatchesPattern(Document* doc) const; + + void AddDependency(Document* doc) + { dependencies.push_back(doc); } + +protected: + + Target(const std::string& arg_name, const std::string& arg_pattern) + : name(arg_name), pattern(arg_pattern) + { } + + std::string name; + std::string pattern; + std::list dependencies; + +private: + + virtual void DoFindDependencies(const std::vector& docs) = 0; + + virtual void DoGenerate() const = 0; +}; + +class ProtoAnalyzerTarget : public Target { +public: + + static Target* Instantiate(const std::string& name, + const std::string& pattern) + { return new ProtoAnalyzerTarget(name, pattern); } + +private: + + ProtoAnalyzerTarget(const std::string& name, const std::string& pattern) + : Target(name, pattern) + { } + + void DoFindDependencies(const std::vector& docs) + { /* TODO */ } + + void DoGenerate() const + { /* TODO */ } +}; + +class FileAnalyzerTarget : public Target { +public: + + static Target* Instantiate(const std::string& name, + const std::string& pattern) + { return new FileAnalyzerTarget(name, pattern); } + +private: + + FileAnalyzerTarget(const std::string& name, const std::string& pattern) + : Target(name, pattern) + { } + + void DoFindDependencies(const std::vector& docs) + { /* TODO */ } + + void DoGenerate() const + { /* TODO */ } +}; + +class PackageTarget : public Target { +public: + + static Target* Instantiate(const std::string& name, + const std::string& pattern) + { return new PackageTarget(name, pattern); } + +private: + + PackageTarget(const std::string& name, const std::string& pattern) + : Target(name, pattern) + { } + + void DoFindDependencies(const std::vector& docs) + { /* TODO */ } + + void DoGenerate() const + { /* TODO */ } +}; + +class ScriptTarget : public Target { +public: + + static Target* Instantiate(const std::string& name, + const std::string& pattern) + { return new ScriptTarget(name, pattern); } + +private: + + ScriptTarget(const std::string& name, const std::string& pattern) + : Target(name, pattern) + { } + + void DoFindDependencies(const std::vector& docs) + { /* TODO */ } + + void DoGenerate() const + { /* TODO */ } +}; + +class IdentifierTarget : public Target { +public: + + static Target* Instantiate(const std::string& name, + const std::string& pattern) + { return new IdentifierTarget(name, pattern); } + +private: + + IdentifierTarget(const std::string& name, const std::string& pattern) + : Target(name, pattern) + { } + + void DoFindDependencies(const std::vector& docs); + + void DoGenerate() const + { /* TODO */ } +}; + +class Config { +public: + + Config(const std::string& file, const std::string& delim = "\t"); + + ~Config(); + + void FindDependencies(const std::vector& docs); + + void GenerateDocs() const; + +private: + + typedef std::list target_list; + + target_list targets; +}; + + +} // namespace broxygen + +#endif diff --git a/src/broxygen/Document.cc b/src/broxygen/Document.cc index 41622aa41f..dbac11f234 100644 --- a/src/broxygen/Document.cc +++ b/src/broxygen/Document.cc @@ -48,11 +48,11 @@ IdentifierDocument::~IdentifierDocument() { Unref(id); - for ( RedefList::const_iterator it = redefs.begin(); it != redefs.end(); + for ( redef_list::const_iterator it = redefs.begin(); it != redefs.end(); ++it ) delete *it; - for ( RecordFieldMap::const_iterator it = fields.begin(); + for ( record_field_map::const_iterator it = fields.begin(); it != fields.end(); ++it ) delete it->second; } @@ -93,7 +93,7 @@ string IdentifierDocument::GetComments() const string IdentifierDocument::GetFieldComments(const string& field) const { - RecordFieldMap::const_iterator it = fields.find(field); + record_field_map::const_iterator it = fields.find(field); if ( it == fields.end() ) return string(); diff --git a/src/broxygen/Document.h b/src/broxygen/Document.h index fae3073ef9..ec9212bfba 100644 --- a/src/broxygen/Document.h +++ b/src/broxygen/Document.h @@ -116,14 +116,14 @@ private: std::vector comments; }; - typedef std::list RedefList; - typedef std::map RecordFieldMap; + typedef std::list redef_list; + typedef std::map record_field_map; std::vector comments; ID* id; string initial_val_desc; - RedefList redefs; - RecordFieldMap fields; + redef_list redefs; + record_field_map fields; RecordField* last_field_seen; }; diff --git a/src/broxygen/Manager.cc b/src/broxygen/Manager.cc index 72ef0aab1e..707481d86a 100644 --- a/src/broxygen/Manager.cc +++ b/src/broxygen/Manager.cc @@ -83,13 +83,13 @@ static string PrettifyParams(const string& s) return s; } -Manager::Manager(const string& config) +Manager::Manager(const string& arg_config) : disabled(), comment_buffer(), comment_buffer_map(), packages(), scripts(), - identifiers(), all_docs(), last_identifier_seen(), incomplete_type() + identifiers(), all_docs(), last_identifier_seen(), incomplete_type(), + config(arg_config) { if ( getenv("BRO_DISABLE_BROXYGEN") ) disabled = true; - // TODO config file stuff } Manager::~Manager() @@ -109,16 +109,16 @@ void Manager::InitPostScript() { if ( disabled ) return; - // TODO: dependency resolution stuff? + + config.FindDependencies(all_docs); } void Manager::GenerateDocs() const { if ( disabled ) return; - // TODO - // may be a no-op if no config file + config.GenerateDocs(); } void Manager::File(const string& path) @@ -214,7 +214,8 @@ IdentifierDocument* Manager::CreateIdentifierDoc(ID* id, ScriptDocument* script) rval->AddComments(comment_buffer); comment_buffer.clear(); - CommentBufferMap::const_iterator it = comment_buffer_map.find(id->Name()); + comment_buffer_map_t::const_iterator it = + comment_buffer_map.find(id->Name()); if ( it != comment_buffer_map.end() ) { diff --git a/src/broxygen/Manager.h b/src/broxygen/Manager.h index 4207c14630..67f7e543af 100644 --- a/src/broxygen/Manager.h +++ b/src/broxygen/Manager.h @@ -1,6 +1,7 @@ #ifndef BROXYGEN_MANAGER_H #define BROXYGEN_MANAGER_H +#include "Configuration.h" #include "Document.h" #include "ID.h" #include "Type.h" @@ -16,15 +17,15 @@ namespace broxygen { template struct DocumentMap { - typedef std::map MapType; + typedef std::map map_type; T* GetDocument(const std::string& name) const { - typename MapType::const_iterator it = map.find(name); + typename map_type::const_iterator it = map.find(name); return it == map.end() ? 0 : it->second; } - MapType map; + map_type map; }; class Manager { @@ -75,20 +76,21 @@ public: private: - typedef std::vector CommentBuffer; - typedef std::map CommentBufferMap; + typedef std::vector comment_buffer_t; + typedef std::map comment_buffer_map_t; IdentifierDocument* CreateIdentifierDoc(ID* id, ScriptDocument* script); bool disabled; - CommentBuffer comment_buffer; // For whatever next identifier that comes in. - CommentBufferMap comment_buffer_map; // For a particular identifier. + comment_buffer_t comment_buffer; // For whatever next identifier that comes in. + comment_buffer_map_t comment_buffer_map; // For a particular identifier. DocumentMap packages; DocumentMap scripts; DocumentMap identifiers; std::vector all_docs; IdentifierDocument* last_identifier_seen; IdentifierDocument* incomplete_type; + Config config; }; } // namespace broxygen diff --git a/src/util.cc b/src/util.cc index eacbf46eb7..8f85d6fca4 100644 --- a/src/util.cc +++ b/src/util.cc @@ -996,8 +996,8 @@ string flatten_script_name(const string& name, const string& prefix) return rval; } -static vector* tokenize_string(string input, const string& delim, - vector* rval) +vector* tokenize_string(string input, const string& delim, + vector* rval) { if ( ! rval ) rval = new vector(); diff --git a/src/util.h b/src/util.h index 17dfb9c217..1dc47db68a 100644 --- a/src/util.h +++ b/src/util.h @@ -17,6 +17,7 @@ #include #include +#include #include #include #include @@ -105,6 +106,10 @@ std::string extract_ip_and_len(const std::string& i, int* len); std::string get_unescaped_string(const std::string& str); std::string get_escaped_string(const std::string& str, bool escape_all); +std::vector* tokenize_string(std::string input, + const std::string& delim, + std::vector* rval = 0); + extern char* copy_string(const char* s); extern int streq(const char* s1, const char* s2); From cc314084a882e2bf99ce59661011a097f1bc1961 Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Fri, 8 Nov 2013 16:47:22 -0500 Subject: [PATCH 09/40] Fix the irc_reply event for several server message types. --- src/analyzer/protocol/irc/IRC.cc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/analyzer/protocol/irc/IRC.cc b/src/analyzer/protocol/irc/IRC.cc index 71f591635f..d175e447fe 100644 --- a/src/analyzer/protocol/irc/IRC.cc +++ b/src/analyzer/protocol/irc/IRC.cc @@ -164,7 +164,7 @@ void IRC_Analyzer::DeliverStream(int length, const u_char* line, bool orig) if ( code > 0 ) { switch ( code ) { - // Ignore unimportant messages. + /* case 1: // RPL_WELCOME case 2: // RPL_YOURHOST case 3: // RPL_CREATED @@ -183,7 +183,8 @@ void IRC_Analyzer::DeliverStream(int length, const u_char* line, bool orig) case 376: // RPL_ENDOFMOTD case 331: // RPL_NOTOPIC break; - + */ + // Count of users, services and servers in whole network. case 251: if ( ! irc_network_info ) From 4f6d01000ab706afa4dbdaa5031f741c85bc7908 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Thu, 14 Nov 2013 14:00:51 -0600 Subject: [PATCH 10/40] Implement majority of Broxygen features delegated to Bro. Still have to update the Sphinx integration. --- src/BroDoc.cc | 46 +-- src/BroDoc.h | 8 +- src/Desc.h | 1 + src/ID.cc | 17 +- src/ID.h | 2 +- src/Type.cc | 163 +++++++-- src/Type.h | 14 +- src/Var.cc | 12 +- src/broxygen/Configuration.cc | 352 +++++++++++++++++-- src/broxygen/Configuration.h | 157 ++++++--- src/broxygen/Document.cc | 645 ++++++++++++++++++++++++++++++++-- src/broxygen/Document.h | 94 +++-- src/broxygen/Manager.cc | 162 +++------ src/broxygen/Manager.h | 54 ++- src/broxygen/broxygen.bif | 47 ++- src/main.cc | 3 +- src/parse.y | 7 +- src/plugin/ComponentManager.h | 2 + src/scan.l | 3 +- src/util.cc | 43 +++ src/util.h | 4 + 21 files changed, 1494 insertions(+), 342 deletions(-) diff --git a/src/BroDoc.cc b/src/BroDoc.cc index ecdab83103..a7b7580fa6 100644 --- a/src/BroDoc.cc +++ b/src/BroDoc.cc @@ -12,6 +12,7 @@ #include "analyzer/Manager.h" #include "analyzer/Component.h" #include "file_analysis/Manager.h" +#include "broxygen/Manager.h" BroDoc::BroDoc(const std::string& rel, const std::string& abs) { @@ -565,33 +566,28 @@ static void WritePluginBifItems(FILE* f, const plugin::Plugin* p, for ( it = bifitems.begin(); it != bifitems.end(); ++it ) { - BroDocObj* o = doc_ids[it->GetID()]; + broxygen::IdentifierDocument* doc = broxygen_mgr->GetIdentifierDoc( + it->GetID()); - if ( o ) - o->WriteReST(f); + if ( doc ) + fprintf(f, "%s\n\n", doc->ReStructuredText().c_str()); else - reporter->Warning("No docs for ID: %s\n", it->GetID()); + reporter->InternalWarning("Broxygen ID lookup failed: %s\n", + it->GetID()); } } -static void WriteAnalyzerTagDefn(FILE* f, EnumType* e, const string& module) +static void WriteAnalyzerTagDefn(FILE* f, const string& module) { - /* TODO - string tag_id= module + "::Tag"; - e = new CommentedEnumType(e); - e->SetTypeID(copy_string(tag_id.c_str())); + string tag_id = module + "::Tag"; - ID* dummy_id = new ID(tag_id.c_str(), SCOPE_GLOBAL, true); - dummy_id->SetType(e); - dummy_id->MakeType(); + broxygen::IdentifierDocument* doc = broxygen_mgr->GetIdentifierDoc(tag_id); - list* r = new list(); - r->push_back("Unique identifiers for analyzers."); + if ( ! doc ) + reporter->InternalError("Broxygen failed analyzer tag lookup: %s", + tag_id.c_str()); - BroDocObj bdo(dummy_id, r, true); - - bdo.WriteReST(f); - */ + fprintf(f, "%s\n", doc->ReStructuredText().c_str()); } static bool ComponentsMatch(const plugin::Plugin* p, plugin::component::Type t, @@ -610,16 +606,14 @@ static bool ComponentsMatch(const plugin::Plugin* p, plugin::component::Type t, return true; } -void CreateProtoAnalyzerDoc(const char* filename) +void CreateProtoAnalyzerDoc(FILE* f) { - FILE* f = fopen(filename, "w"); - fprintf(f, "Protocol Analyzers\n"); - fprintf(f, "==================\n\n\n"); + fprintf(f, "==================\n\n"); fprintf(f, ".. contents::\n"); fprintf(f, " :depth: 1\n\n"); - WriteAnalyzerTagDefn(f, analyzer_mgr->GetTagEnumType(), "Analyzer"); + WriteAnalyzerTagDefn(f, "Analyzer"); plugin::Manager::plugin_list plugins = plugin_mgr->Plugins(); plugin::Manager::plugin_list::const_iterator it; @@ -642,16 +636,14 @@ void CreateProtoAnalyzerDoc(const char* filename) fclose(f); } -void CreateFileAnalyzerDoc(const char* filename) +void CreateFileAnalyzerDoc(FILE* f) { - FILE* f = fopen(filename, "w"); - fprintf(f, "File Analyzers\n"); fprintf(f, "==============\n\n"); fprintf(f, ".. contents::\n"); fprintf(f, " :depth: 1\n\n"); - WriteAnalyzerTagDefn(f, file_mgr->GetTagEnumType(), "Files"); + WriteAnalyzerTagDefn(f, "Files"); plugin::Manager::plugin_list plugins = plugin_mgr->Plugins(); plugin::Manager::plugin_list::const_iterator it; diff --git a/src/BroDoc.h b/src/BroDoc.h index 081df698d9..926713d01e 100644 --- a/src/BroDoc.h +++ b/src/BroDoc.h @@ -409,14 +409,14 @@ private: /** * Writes out plugin index documentation for all analyzer plugins. - * @param filename the name of the file to write. + * @param f an open file stream to write docs into. */ -void CreateProtoAnalyzerDoc(const char* filename); +void CreateProtoAnalyzerDoc(FILE* f); /** * Writes out plugin index documentation for all file analyzer plugins. - * @param filename the name of the file to write. + * @param f an open file stream to write docs into. */ -void CreateFileAnalyzerDoc(const char* filename); +void CreateFileAnalyzerDoc(FILE* f); #endif diff --git a/src/Desc.h b/src/Desc.h index c16c00cf13..27dc326ff0 100644 --- a/src/Desc.h +++ b/src/Desc.h @@ -69,6 +69,7 @@ public: void PopIndent(); void PopIndentNoNL(); int GetIndentLevel() const { return indent_level; } + void ClearIndentLevel() { indent_level = 0; } int IndentSpaces() const { return indent_with_spaces; } void SetIndentSpaces(int i) { indent_with_spaces = i; } diff --git a/src/ID.cc b/src/ID.cc index 8d30d10c27..aa965b880e 100644 --- a/src/ID.cc +++ b/src/ID.cc @@ -14,6 +14,7 @@ #include "PersistenceSerializer.h" #include "Scope.h" #include "Traverse.h" +#include "broxygen/Manager.h" ID::ID(const char* arg_name, IDScope arg_scope, bool arg_is_export) { @@ -618,7 +619,6 @@ void ID::DescribeExtended(ODesc* d) const void ID::DescribeReSTShort(ODesc* d) const { - /* TODO if ( is_type ) d->Add(":bro:type:`"); else @@ -632,8 +632,8 @@ void ID::DescribeReSTShort(ODesc* d) const d->Add(": "); d->Add(":bro:type:`"); - if ( ! is_type && type->GetTypeID() ) - d->Add(type->GetTypeID()); + if ( ! is_type && ! type->GetName().empty() ) + d->Add(type->GetName().c_str()); else { TypeTag t = type->Tag(); @@ -644,14 +644,14 @@ void ID::DescribeReSTShort(ODesc* d) const break; case TYPE_FUNC: - d->Add(type->AsFuncType()->FlavorString()); + d->Add(type->AsFuncType()->FlavorString().c_str()); break; case TYPE_ENUM: if ( is_type ) d->Add(type_name(t)); else - d->Add(type->AsEnumType()->Name().c_str()); + d->Add(broxygen_mgr->GetEnumTypeName(Name()).c_str()); break; default: @@ -668,12 +668,11 @@ void ID::DescribeReSTShort(ODesc* d) const d->SP(); attrs->DescribeReST(d); } - */ } -void ID::DescribeReST(ODesc* d, bool is_role) const +void ID::DescribeReST(ODesc* d, bool roles_only) const { - if ( is_role ) + if ( roles_only ) { if ( is_type ) d->Add(":bro:type:`"); @@ -705,7 +704,7 @@ void ID::DescribeReST(ODesc* d, bool is_role) const d->Add("`"); } else - type->DescribeReST(d); + type->DescribeReST(d, roles_only); d->NL(); } diff --git a/src/ID.h b/src/ID.h index 57e1222511..59b397fa90 100644 --- a/src/ID.h +++ b/src/ID.h @@ -84,7 +84,7 @@ public: // Adds type and value to description. void DescribeExtended(ODesc* d) const; // Produces a description that's reST-ready. - void DescribeReST(ODesc* d, bool is_role=false) const; + void DescribeReST(ODesc* d, bool roles_only = false) const; void DescribeReSTShort(ODesc* d) const; bool Serialize(SerialInfo* info) const; diff --git a/src/Type.cc b/src/Type.cc index 417b73fd2f..83f239bbfe 100644 --- a/src/Type.cc +++ b/src/Type.cc @@ -117,16 +117,17 @@ BroType* BroType::Clone() const sinfo.cache = false; this->Serialize(&sinfo); - char* data; + char* data = 0; uint32 len = form->EndWrite(&data); form->StartRead(data, len); UnserialInfo uinfo(&ss); uinfo.cache = false; - BroType* rval = this->Unserialize(&uinfo); + + BroType* rval = this->Unserialize(&uinfo, false); + assert(rval != this); delete [] data; - return rval; } @@ -173,11 +174,9 @@ void BroType::Describe(ODesc* d) const } } -void BroType::DescribeReST(ODesc* d) const +void BroType::DescribeReST(ODesc* d, bool roles_only) const { - d->Add(":bro:type:`"); - d->Add(type_name(Tag())); - d->Add("`"); + d->Add(fmt(":bro:type:`%s`", type_name(Tag()))); } void BroType::SetError() @@ -200,7 +199,7 @@ bool BroType::Serialize(SerialInfo* info) const return ret; } -BroType* BroType::Unserialize(UnserialInfo* info, TypeTag want) +BroType* BroType::Unserialize(UnserialInfo* info, bool use_existing) { // To avoid external Broccoli clients needing to always send full type // objects, we allow them to give us only the name of a type. To @@ -234,8 +233,8 @@ BroType* BroType::Unserialize(UnserialInfo* info, TypeTag want) BroType* t = (BroType*) SerialObj::Unserialize(info, SER_BRO_TYPE); - if ( ! t ) - return 0; + if ( ! t || ! use_existing ) + return t; if ( ! t->name.empty() ) { @@ -458,7 +457,7 @@ void IndexType::Describe(ODesc* d) const } } -void IndexType::DescribeReST(ODesc* d) const +void IndexType::DescribeReST(ODesc* d, bool roles_only) const { d->Add(":bro:type:`"); @@ -484,7 +483,7 @@ void IndexType::DescribeReST(ODesc* d) const d->Add("`"); } else - t->DescribeReST(d); + t->DescribeReST(d, roles_only); } d->Add("]"); @@ -500,7 +499,7 @@ void IndexType::DescribeReST(ODesc* d) const d->Add("`"); } else - yield_type->DescribeReST(d); + yield_type->DescribeReST(d, roles_only); } } @@ -775,7 +774,7 @@ void FuncType::Describe(ODesc* d) const } } -void FuncType::DescribeReST(ODesc* d) const +void FuncType::DescribeReST(ODesc* d, bool roles_only) const { d->Add(":bro:type:`"); d->Add(FlavorString()); @@ -795,7 +794,7 @@ void FuncType::DescribeReST(ODesc* d) const d->Add("`"); } else - yield->DescribeReST(d); + yield->DescribeReST(d, roles_only); } } @@ -927,7 +926,7 @@ TypeDecl* TypeDecl::Unserialize(UnserialInfo* info) return t; } -void TypeDecl::DescribeReST(ODesc* d) const +void TypeDecl::DescribeReST(ODesc* d, bool roles_only) const { d->Add(id); d->Add(": "); @@ -939,7 +938,7 @@ void TypeDecl::DescribeReST(ODesc* d) const d->Add("`"); } else - type->DescribeReST(d); + type->DescribeReST(d, roles_only); if ( attrs ) { @@ -1037,9 +1036,13 @@ void RecordType::Describe(ODesc* d) const } } -void RecordType::DescribeReST(ODesc* d) const +void RecordType::DescribeReST(ODesc* d, bool roles_only) const { d->Add(":bro:type:`record`"); + + if ( num_fields == 0 ) + return; + d->NL(); DescribeFieldsReST(d, false); } @@ -1136,7 +1139,53 @@ void RecordType::DescribeFieldsReST(ODesc* d, bool func_args) const } } - FieldDecl(i)->DescribeReST(d); + const TypeDecl* td = FieldDecl(i); + td->DescribeReST(d); + + if ( func_args ) + continue; + + using broxygen::IdentifierDocument; + IdentifierDocument* doc = broxygen_mgr->GetIdentifierDoc(GetName()); + + if ( ! doc ) + { + reporter->InternalWarning("Failed to lookup record doc: %s", + GetName().c_str()); + continue; + } + + string field_from_script = doc->GetDeclaringScriptForField(td->id); + string type_from_script; + + if ( doc->GetDeclaringScript() ) + type_from_script = doc->GetDeclaringScript()->Name(); + + if ( ! field_from_script.empty() && + field_from_script != type_from_script ) + { + d->PushIndent(); + d->Add(fmt("(from ``redef`` in :doc:`%s`)", + field_from_script.c_str())); + d->PopIndentNoNL(); + } + + vector cmnts = doc->GetFieldComments(td->id); + + if ( cmnts.empty() ) + continue; + + d->PushIndent(); + + for ( size_t i = 0; i < cmnts.size(); ++i ) + { + if ( i > 0 ) + d->NL(); + + d->Add(cmnts[i].c_str()); + } + + d->PopIndentNoNL(); } if ( ! func_args ) @@ -1415,10 +1464,82 @@ const char* EnumType::Lookup(bro_int_t value) return 0; } -void EnumType::DescribeReST(ODesc* d) const +void EnumType::DescribeReST(ODesc* d, bool roles_only) const { - // TODO: this probably goes away d->Add(":bro:type:`enum`"); + + // Create temporary, reverse name map so that enums can be documented + // in ascending order of their actual integral value instead of by name. + typedef map< bro_int_t, const char* > RevNameMap; + + RevNameMap rev; + + for ( NameMap::const_iterator it = names.begin(); it != names.end(); ++it ) + rev[it->second] = it->first; + + for ( RevNameMap::const_iterator it = rev.begin(); it != rev.end(); ++it ) + { + d->NL(); + d->PushIndent(); + + if ( roles_only ) + d->Add(fmt(":bro:enum:`%s`", it->second)); + else + d->Add(fmt(".. bro:enum:: %s %s", it->second, GetName().c_str())); + + using broxygen::IdentifierDocument; + IdentifierDocument* doc = broxygen_mgr->GetIdentifierDoc(it->second); + + if ( ! doc ) + { + reporter->InternalWarning("Enum %s documentation lookup failure", + it->second); + continue; + } + + string enum_from_script; + string type_from_script; + + if ( doc->GetDeclaringScript() ) + enum_from_script = doc->GetDeclaringScript()->Name(); + + IdentifierDocument* type_doc = broxygen_mgr->GetIdentifierDoc(GetName()); + + if ( type_doc && type_doc->GetDeclaringScript() ) + type_from_script = type_doc->GetDeclaringScript()->Name(); + + if ( ! enum_from_script.empty() && + enum_from_script != type_from_script ) + { + d->NL(); + d->PushIndent(); + d->Add(fmt("(from ``redef`` in :doc:`%s`)", + enum_from_script.c_str())); + d->PopIndentNoNL(); + } + + vector cmnts = doc->GetComments(); + + if ( cmnts.empty() ) + { + d->PopIndentNoNL(); + continue; + } + + d->NL(); + d->PushIndent(); + + for ( size_t i = 0; i < cmnts.size(); ++i ) + { + if ( i > 0 ) + d->NL(); + + d->Add(cmnts[i].c_str()); + } + + d->PopIndentNoNL(); + d->PopIndentNoNL(); + } } IMPLEMENT_SERIAL(EnumType, SER_ENUM_TYPE); diff --git a/src/Type.h b/src/Type.h index be27426546..3e774f373a 100644 --- a/src/Type.h +++ b/src/Type.h @@ -227,12 +227,12 @@ public: BroType* Ref() { ::Ref(this); return this; } virtual void Describe(ODesc* d) const; - virtual void DescribeReST(ODesc* d) const; + virtual void DescribeReST(ODesc* d, bool roles_only = false) const; virtual unsigned MemoryAllocation() const; bool Serialize(SerialInfo* info) const; - static BroType* Unserialize(UnserialInfo* info, TypeTag want = TYPE_ANY); + static BroType* Unserialize(UnserialInfo* info, bool use_existing = true); void SetName(const string& arg_name) { name = arg_name; } string GetName() const { return name; } @@ -305,7 +305,7 @@ public: BroType* YieldType(); void Describe(ODesc* d) const; - void DescribeReST(ODesc* d) const; + void DescribeReST(ODesc* d, bool roles_only = false) const; // Returns true if this table is solely indexed by subnet. bool IsSubNetIndex() const; @@ -379,7 +379,7 @@ public: TypeList* ArgTypes() const { return arg_types; } void Describe(ODesc* d) const; - void DescribeReST(ODesc* d) const; + void DescribeReST(ODesc* d, bool roles_only = false) const; protected: FuncType() { args = 0; arg_types = 0; yield = 0; flavor = FUNC_FLAVOR_FUNCTION; } @@ -416,7 +416,7 @@ public: bool Serialize(SerialInfo* info) const; static TypeDecl* Unserialize(UnserialInfo* info); - virtual void DescribeReST(ODesc* d) const; + virtual void DescribeReST(ODesc* d, bool roles_only = false) const; BroType* type; Attributes* attrs; @@ -454,7 +454,7 @@ public: const char* AddFields(type_decl_list* types, attr_list* attr); void Describe(ODesc* d) const; - void DescribeReST(ODesc* d) const; + void DescribeReST(ODesc* d, bool roles_only = false) const; void DescribeFields(ODesc* d) const; void DescribeFieldsReST(ODesc* d, bool func_args) const; @@ -527,7 +527,7 @@ public: bro_int_t Lookup(const string& module_name, const char* name); const char* Lookup(bro_int_t value); // Returns 0 if not found - void DescribeReST(ODesc* d) const; + void DescribeReST(ODesc* d, bool roles_only = false) const; protected: DECLARE_SERIAL(EnumType) diff --git a/src/Var.cc b/src/Var.cc index 821c9e207b..eb03f2f912 100644 --- a/src/Var.cc +++ b/src/Var.cc @@ -261,17 +261,17 @@ extern Expr* add_and_assign_local(ID* id, Expr* init, Val* val) void add_type(ID* id, BroType* t, attr_list* attr) { - string new_type_name(id->Name()); - string old_type_name(t->GetName()); + string new_type_name = id->Name(); + string old_type_name = t->GetName(); BroType* tnew = 0; if ( (t->Tag() == TYPE_RECORD || t->Tag() == TYPE_ENUM) && - ! old_type_name.empty() ) + old_type_name.empty() ) + // An extensible type (record/enum) being declared for first time. + tnew = t; + else // Clone the type to preserve type name aliasing. tnew = t->Clone(); - else - // An extensible types (record/enum) being declared for first time. - tnew = t; type_aliases[new_type_name].insert(tnew); diff --git a/src/broxygen/Configuration.cc b/src/broxygen/Configuration.cc index cd8ee7be56..d88aa1920c 100644 --- a/src/broxygen/Configuration.cc +++ b/src/broxygen/Configuration.cc @@ -1,4 +1,5 @@ #include "Configuration.h" +#include "Manager.h" #include "util.h" #include "Reporter.h" @@ -7,6 +8,8 @@ #include #include #include +#include +#include using namespace broxygen; using namespace std; @@ -16,12 +19,12 @@ typedef map target_factory_map; static target_factory_map create_target_factory_map() { target_factory_map rval; - rval["package_index"] = &PackageTarget::Instantiate; + rval["package_index"] = &PackageIndexTarget::Instantiate; rval["package"] = &PackageTarget::Instantiate; rval["proto_analyzer"] = &ProtoAnalyzerTarget::Instantiate; rval["file_analyzer"] = &FileAnalyzerTarget::Instantiate; - rval["script_summary"] = &ScriptTarget::Instantiate; - rval["script_index"] = &ScriptTarget::Instantiate; + rval["script_summary"] = &ScriptSummaryTarget::Instantiate; + rval["script_index"] = &ScriptIndexTarget::Instantiate; rval["script"] = &ScriptTarget::Instantiate; rval["identifier"] = &IdentifierTarget::Instantiate; return rval; @@ -29,41 +32,300 @@ static target_factory_map create_target_factory_map() static target_factory_map target_instantiators = create_target_factory_map(); -bool Target::MatchesPattern(Document* doc) const - { - // TODO: prefix matching or full regex? - - if ( doc->Name() == pattern ) +struct TargetFile { + TargetFile(const string& arg_name) + : name(arg_name), f() { - DBG_LOG(DBG_BROXYGEN, "Doc '%s'' matched pattern for target '%s'", - doc->Name().c_str(), name.c_str()); - return true; + if ( name.find('/') != string::npos ) + { + string dir = SafeDirname(name).result; + + if ( ! ensure_intermediate_dirs(dir.c_str()) ) + reporter->FatalError("Broxygen failed to make dir %s", + dir.c_str()); + } + + f = fopen(name.c_str(), "w"); + + if ( ! f ) + reporter->FatalError("Broxygen failed to open '%s' for writing: %s", + name.c_str(), strerror(errno)); } - return false; - } + ~TargetFile() + { + if ( f ) + fclose(f); + + DBG_LOG(DBG_BROXYGEN, "Wrote out-of-date target '%s'", name.c_str()); + } + + string name; + FILE* f; +}; template -void filter_matching_docs(const std::vector& filter_from, Target* t) +static vector filter_matching_docs(const vector& from, Target* t) { - for ( size_t i = 0; i < filter_from.size(); ++i ) + vector rval; + + for ( size_t i = 0; i < from.size(); ++i ) { - T* d = dynamic_cast(filter_from[i]); + T* d = dynamic_cast(from[i]); if ( ! d ) continue; if ( t->MatchesPattern(d) ) - t->AddDependency(d); + { + DBG_LOG(DBG_BROXYGEN, "Doc '%s' matched pattern for target '%s'", + d->Name().c_str(), t->Name().c_str()); + rval.push_back(d); + } + } + + return rval; + } + +Target::Target(const string& arg_name, const string& arg_pattern) + : name(arg_name), pattern(arg_pattern), prefix() + { + size_t pos = pattern.find('*'); + + if ( pos == 0 || pos == string::npos ) + return; + + prefix = pattern.substr(0, pos); + } + +bool Target::MatchesPattern(Document* doc) const + { + if ( pattern == "*" ) + return true; + + if ( prefix.empty() ) + return doc->Name() == pattern; + + return ! strncmp(doc->Name().c_str(), prefix.c_str(), prefix.size()); + } + +void AnalyzerTarget::DoFindDependencies(const std::vector& docs) + { + // TODO: really should add to dependency list the tag type's ID and + // all bif items for matching analyzer plugins, but that's all dependent + // on the bro binary itself, so I'm cheating. + } + +void AnalyzerTarget::DoGenerate() const + { + if ( broxygen_mgr->IsUpToDate(Name(), vector()) ) + return; + + if ( Pattern() != "*" ) + reporter->InternalWarning("Broxygen only implements analyzer target" + " pattern '*'"); + + TargetFile file(Name()); + doc_creator_callback(file.f); + } + +void PackageTarget::DoFindDependencies(const vector& docs) + { + pkg_deps = filter_matching_docs(docs, this); + + if ( pkg_deps.empty() ) + reporter->FatalError("No match for Broxygen target '%s' pattern '%s'", + Name().c_str(), Pattern().c_str()); + + for ( size_t i = 0; i < docs.size(); ++i ) + { + ScriptDocument* script = dynamic_cast(docs[i]); + + if ( ! script ) + continue; + + for ( size_t j = 0; j < pkg_deps.size(); ++j ) + { + if ( strncmp(script->Name().c_str(), pkg_deps[j]->Name().c_str(), + pkg_deps[j]->Name().size())) + continue; + + DBG_LOG(DBG_BROXYGEN, "Script %s associated with package %s", + script->Name().c_str(), pkg_deps[j]->Name().c_str()); + pkg_manifest[pkg_deps[j]].push_back(script); + script_deps.push_back(script); + } } } -void IdentifierTarget::DoFindDependencies(const std::vector& docs) +void PackageTarget::DoGenerate() const { - filter_matching_docs(docs, this); + if ( broxygen_mgr->IsUpToDate(Name(), script_deps) ) + return; + + TargetFile file(Name()); + + for ( manifest_t::const_iterator it = pkg_manifest.begin(); + it != pkg_manifest.end(); ++it ) + { + fprintf(file.f, "Package: %s\n\n", it->first->Name().c_str()); + + for ( size_t i = 0; i < it->second.size(); ++i ) + { + fprintf(file.f, " :doc:`%s`\n", it->second[i]->Name().c_str()); + + vector cmnts = it->second[i]->GetComments(); + + for ( size_t j = 0; j < cmnts.size(); ++j ) + fprintf(file.f, " %s\n", cmnts[j].c_str()); + + fprintf(file.f, "\n"); + } + } } -Config::Config(const string& file, const string& delim) +void PackageIndexTarget::DoFindDependencies(const vector& docs) + { + pkg_deps = filter_matching_docs(docs, this); + + if ( pkg_deps.empty() ) + reporter->FatalError("No match for Broxygen target '%s' pattern '%s'", + Name().c_str(), Pattern().c_str()); + } + +void PackageIndexTarget::DoGenerate() const + { + if ( broxygen_mgr->IsUpToDate(Name(), pkg_deps) ) + return; + + TargetFile file(Name()); + + for ( size_t i = 0; i < pkg_deps.size(); ++i ) + fprintf(file.f, "%s\n", pkg_deps[i]->ReStructuredText().c_str()); + } + +void ScriptTarget::DoFindDependencies(const vector& docs) + { + script_deps = filter_matching_docs(docs, this); + + if ( script_deps.empty() ) + reporter->FatalError("No match for Broxygen target '%s' pattern '%s'", + Name().c_str(), Pattern().c_str()); + } + +void ScriptTarget::DoGenerate() const + { + if ( Name()[Name().size() - 1] == '/' ) + { + // Target name is a dir, matching scripts are written within that dir + // with a dir tree that parallels the script's BROPATH location. + + for ( size_t i = 0; i < script_deps.size(); ++i ) + { + string target_filename = Name() + script_deps[i]->Name(); + size_t pos = target_filename.rfind(".bro"); + + if ( pos == target_filename.size() - 4 ) + target_filename.replace(pos, 4, ".rst"); + else + target_filename += ".rst"; + + vector dep; + dep.push_back(script_deps[i]); + + if ( broxygen_mgr->IsUpToDate(target_filename, dep) ) + continue; + + TargetFile file(target_filename); + + fprintf(file.f, "%s\n", script_deps[i]->ReStructuredText().c_str()); + } + + // TODO: could possibly take inventory of files in the dir beforehand, + // track all files written, then compare afterwards in order to remove + // stale files. + return; + } + + // Target is a single file, all matching scripts get written there. + + if ( broxygen_mgr->IsUpToDate(Name(), script_deps) ) + return; + + TargetFile file(Name()); + + for ( size_t i = 0; i < script_deps.size(); ++i ) + fprintf(file.f, "%s\n", script_deps[i]->ReStructuredText().c_str()); + } + +void ScriptSummaryTarget::DoGenerate() const + { + if ( broxygen_mgr->IsUpToDate(Name(), script_deps) ) + return; + + TargetFile file(Name()); + + for ( size_t i = 0; i < script_deps.size(); ++i ) + { + ScriptDocument* d = dynamic_cast(script_deps[i]); + + if ( ! d ) + continue; + + fprintf(file.f, ":doc:`/scripts/%s`\n", d->Name().c_str()); + + vector cmnts = d->GetComments(); + + for ( size_t i = 0; i < cmnts.size(); ++i ) + fprintf(file.f, " %s\n", cmnts[i].c_str()); + + fprintf(file.f, "\n"); + } + } + +void ScriptIndexTarget::DoGenerate() const + { + if ( broxygen_mgr->IsUpToDate(Name(), script_deps) ) + return; + + TargetFile file(Name()); + + fprintf(file.f, ".. toctree::\n"); + fprintf(file.f, " :maxdepth: 1\n\n"); + + for ( size_t i = 0; i < script_deps.size(); ++i ) + { + ScriptDocument* d = dynamic_cast(script_deps[i]); + + if ( ! d ) + continue; + + fprintf(file.f, " %s <%s>\n", d->Name().c_str(), d->Name().c_str()); + } + } + +void IdentifierTarget::DoFindDependencies(const vector& docs) + { + id_deps = filter_matching_docs(docs, this); + + if ( id_deps.empty() ) + reporter->FatalError("No match for Broxygen target '%s' pattern '%s'", + Name().c_str(), Pattern().c_str()); + } + +void IdentifierTarget::DoGenerate() const + { + if ( broxygen_mgr->IsUpToDate(Name(), id_deps) ) + return; + + TargetFile file(Name()); + + for ( size_t i = 0; i < id_deps.size(); ++i ) + fprintf(file.f, "%s\n\n", id_deps[i]->ReStructuredText().c_str()); + } + +Config::Config(const string& arg_file, const string& delim) + : file(arg_file), targets() { if ( file.empty() ) return; @@ -71,53 +333,71 @@ Config::Config(const string& file, const string& delim) ifstream f(file.c_str()); if ( ! f.is_open() ) - reporter->InternalError("failed to open Broxygen config file %s: %s", - file.c_str(), strerror(errno)); + reporter->FatalError("failed to open Broxygen config file '%s': %s", + file.c_str(), strerror(errno)); string line; + unsigned int line_number = 0; while ( getline(f, line) ) { + ++line_number; vector tokens; tokenize_string(line, delim, &tokens); tokens.erase(remove(tokens.begin(), tokens.end(), ""), tokens.end()); + if ( tokens.empty() ) + // Blank line. + continue; + + if ( ! tokens[0].empty() && tokens[0][0] == '#' ) + // Comment + continue; + if ( tokens.size() != 3 ) - reporter->InternalError("malformed Broxygen target: %s", - line.c_str()); + reporter->FatalError("malformed Broxygen target in %s:%u: %s", + file.c_str(), line_number, line.c_str()); target_factory_map::const_iterator it = target_instantiators.find(tokens[0]); if ( it == target_instantiators.end() ) - reporter->InternalError("unkown Broxygen target type: %s", - tokens[0].c_str()); + reporter->FatalError("unkown Broxygen target type: %s", + tokens[0].c_str()); targets.push_back(it->second(tokens[1], tokens[2])); } if ( f.bad() ) - reporter->InternalError("error reading Broxygen config file %s: %s", + reporter->InternalError("error reading Broxygen config file '%s': %s", file.c_str(), strerror(errno)); } Config::~Config() { - for ( target_list::const_iterator it = targets.begin(); - it != targets.end(); ++it ) - delete *it; + for ( size_t i = 0; i < targets.size(); ++i ) + delete targets[i]; } void Config::FindDependencies(const vector& docs) { - for ( target_list::const_iterator it = targets.begin(); - it != targets.end(); ++it ) - (*it)->FindDependencies(docs); + for ( size_t i = 0; i < targets.size(); ++i ) + targets[i]->FindDependencies(docs); } void Config::GenerateDocs() const { - for ( target_list::const_iterator it = targets.begin(); - it != targets.end(); ++it ) - (*it)->Generate(); + for ( size_t i = 0; i < targets.size(); ++i ) + targets[i]->Generate(); + } + +time_t Config::GetModificationTime() const + { + struct stat s; + + if ( stat(file.c_str(), &s) < 0 ) + reporter->InternalError("Broxygen can't stat config file %s: %s", + file.c_str(), strerror(errno)); + + return s.st_mtime; } diff --git a/src/broxygen/Configuration.h b/src/broxygen/Configuration.h index 2aef6f9806..6b5c45ed45 100644 --- a/src/broxygen/Configuration.h +++ b/src/broxygen/Configuration.h @@ -2,10 +2,11 @@ #define BROXYGEN_CONFIGURATION_H #include "Document.h" +#include "BroDoc.h" #include #include -#include +#include namespace broxygen { @@ -16,7 +17,8 @@ public: typedef Target* (*factory_fn)(const std::string&, const std::string&); - virtual ~Target() { } + virtual ~Target() + { } void FindDependencies(const std::vector& docs) { DoFindDependencies(docs); } @@ -26,27 +28,47 @@ public: bool MatchesPattern(Document* doc) const; - void AddDependency(Document* doc) - { dependencies.push_back(doc); } + std::string Name() const + { return name; } + + std::string Pattern() const + { return pattern; } protected: - Target(const std::string& arg_name, const std::string& arg_pattern) - : name(arg_name), pattern(arg_pattern) - { } - - std::string name; - std::string pattern; - std::list dependencies; + Target(const std::string& arg_name, const std::string& arg_pattern); private: virtual void DoFindDependencies(const std::vector& docs) = 0; virtual void DoGenerate() const = 0; + + std::string name; + std::string pattern; + std::string prefix; }; -class ProtoAnalyzerTarget : public Target { +class AnalyzerTarget : public Target { +protected: + + typedef void (*doc_creator_fn)(FILE*); + + AnalyzerTarget(const std::string& name, const std::string& pattern, + doc_creator_fn cb) + : Target(name, pattern), doc_creator_callback(cb) + { } + +private: + + void DoFindDependencies(const std::vector& docs); + + void DoGenerate() const; + + doc_creator_fn doc_creator_callback; +}; + +class ProtoAnalyzerTarget : public AnalyzerTarget { public: static Target* Instantiate(const std::string& name, @@ -56,17 +78,11 @@ public: private: ProtoAnalyzerTarget(const std::string& name, const std::string& pattern) - : Target(name, pattern) + : AnalyzerTarget(name, pattern, &CreateProtoAnalyzerDoc) { } - - void DoFindDependencies(const std::vector& docs) - { /* TODO */ } - - void DoGenerate() const - { /* TODO */ } }; -class FileAnalyzerTarget : public Target { +class FileAnalyzerTarget : public AnalyzerTarget { public: static Target* Instantiate(const std::string& name, @@ -76,14 +92,8 @@ public: private: FileAnalyzerTarget(const std::string& name, const std::string& pattern) - : Target(name, pattern) + : AnalyzerTarget(name, pattern, &CreateFileAnalyzerDoc) { } - - void DoFindDependencies(const std::vector& docs) - { /* TODO */ } - - void DoGenerate() const - { /* TODO */ } }; class PackageTarget : public Target { @@ -96,14 +106,37 @@ public: private: PackageTarget(const std::string& name, const std::string& pattern) - : Target(name, pattern) + : Target(name, pattern), pkg_deps(), script_deps(), pkg_manifest() { } - void DoFindDependencies(const std::vector& docs) - { /* TODO */ } + void DoFindDependencies(const std::vector& docs); - void DoGenerate() const - { /* TODO */ } + void DoGenerate() const; + + std::vector pkg_deps; + std::vector script_deps; + typedef std::map > manifest_t; + manifest_t pkg_manifest; +}; + +class PackageIndexTarget : public Target { +public: + + static Target* Instantiate(const std::string& name, + const std::string& pattern) + { return new PackageIndexTarget(name, pattern); } + +private: + + PackageIndexTarget(const std::string& name, const std::string& pattern) + : Target(name, pattern), pkg_deps() + { } + + void DoFindDependencies(const std::vector& docs); + + void DoGenerate() const; + + std::vector pkg_deps; }; class ScriptTarget : public Target { @@ -113,17 +146,51 @@ public: const std::string& pattern) { return new ScriptTarget(name, pattern); } -private: +protected: ScriptTarget(const std::string& name, const std::string& pattern) - : Target(name, pattern) + : Target(name, pattern), script_deps() { } - void DoFindDependencies(const std::vector& docs) - { /* TODO */ } + std::vector script_deps; - void DoGenerate() const - { /* TODO */ } +private: + + void DoFindDependencies(const std::vector& docs); + + void DoGenerate() const; +}; + +class ScriptSummaryTarget : public ScriptTarget { +public: + + static Target* Instantiate(const std::string& name, + const std::string& pattern) + { return new ScriptSummaryTarget(name, pattern); } + +private: + + ScriptSummaryTarget(const std::string& name, const std::string& pattern) + : ScriptTarget(name, pattern) + { } + + void DoGenerate() const /* override */; +}; + +class ScriptIndexTarget : public ScriptTarget { +public: + + static Target* Instantiate(const std::string& name, + const std::string& pattern) + { return new ScriptIndexTarget(name, pattern); } + +private: + + ScriptIndexTarget(const std::string& name, const std::string& pattern) + : ScriptTarget(name, pattern) + { } + + void DoGenerate() const /* override */; }; class IdentifierTarget : public Target { @@ -136,13 +203,14 @@ public: private: IdentifierTarget(const std::string& name, const std::string& pattern) - : Target(name, pattern) + : Target(name, pattern), id_deps() { } void DoFindDependencies(const std::vector& docs); - void DoGenerate() const - { /* TODO */ } + void DoGenerate() const; + + std::vector id_deps; }; class Config { @@ -156,11 +224,12 @@ public: void GenerateDocs() const; + time_t GetModificationTime() const; + private: - typedef std::list target_list; - - target_list targets; + std::string file; + std::vector targets; }; diff --git a/src/broxygen/Document.cc b/src/broxygen/Document.cc index dbac11f234..2619755b35 100644 --- a/src/broxygen/Document.cc +++ b/src/broxygen/Document.cc @@ -1,38 +1,134 @@ #include "Document.h" +#include "Manager.h" #include "util.h" #include "Val.h" +#include "Desc.h" +#include "Reporter.h" + +#include +#include using namespace broxygen; using namespace std; -static string ImplodeStringVec(const vector& v) +static bool is_public_api(const ID* id) { - string rval; + return (id->Scope() == SCOPE_GLOBAL) || + (id->Scope() == SCOPE_MODULE && id->IsExport()); + } - for ( size_t i = 0; i < v.size(); ++i ) +static bool prettify_params(string& s) + { + size_t identifier_start_pos = 0; + bool in_identifier = false; + string identifier; + + for ( size_t i = 0; i < s.size(); ++i ) { - if ( i > 0 ) - rval += '\n'; + char next = s[i]; - rval += v[i]; + if ( ! in_identifier ) + { + // Pass by leading whitespace. + if ( isspace(next) ) + continue; + + // Only allow alphabetic and '_' as first char of identifier. + if ( isalpha(next) || next == '_' ) + { + identifier_start_pos = i; + identifier += next; + in_identifier = true; + continue; + } + + // Don't need to change anything. + return false; + } + + // All other characters of identifier are alphanumeric or '_'. + if ( isalnum(next) || next == '_' ) + { + identifier += next; + continue; + } + + if ( next == ':' ) + { + if ( i + 1 < s.size() && s[i + 1] == ':' ) + { + // It's part of an identifier's namespace scoping. + identifier += next; + identifier += s[i + 1]; + ++i; + continue; + } + + // Prettify function param/return value reST markup. + string subst; + + if ( identifier == "Returns" ) + subst = ":returns"; + else + subst = ":param " + identifier; + + s.replace(identifier_start_pos, identifier.size(), subst); + return true; + } + + // Don't need to change anything. + return false; } - return rval; + return false; + } + +static string make_heading(const string& heading, char underline) + { + return heading + "\n" + string(heading.size(), underline) + "\n"; } PackageDocument::PackageDocument(const string& arg_name) : Document(), - pkg_name(arg_name) + pkg_name(arg_name), readme() { - // TODO: probably need to determine modification times of all files - // within the directory, recursively + string readme_file = find_file(pkg_name + "/README", bro_path()); + + if ( readme_file.empty() ) + return; + + ifstream f(readme_file.c_str()); + + if ( ! f.is_open() ) + reporter->InternalWarning("Broxygen failed to open '%s': %s", + readme_file.c_str(), strerror(errno)); + + string line; + + while ( getline(f, line) ) + readme.push_back(line); + + if ( f.bad() ) + reporter->InternalWarning("Broxygen error reading '%s': %s", + readme_file.c_str(), strerror(errno)); } -IdentifierDocument::IdentifierDocument(ID* arg_id) +string PackageDocument::DoReStructuredText(bool roles_only) const + { + string rval = fmt(":doc:`%s <%s/index>`\n\n", pkg_name.c_str(), + pkg_name.c_str()); + + for ( size_t i = 0; i < readme.size(); ++i ) + rval += " " + readme[i] + "\n"; + + return rval; + } + +IdentifierDocument::IdentifierDocument(ID* arg_id, ScriptDocument* script) : Document(), comments(), id(arg_id), initial_val_desc(), redefs(), fields(), - last_field_seen() + last_field_seen(), declaring_script(script) { Ref(id); @@ -86,36 +182,539 @@ void IdentifierDocument::AddRecordField(const TypeDecl* field, last_field_seen = rf; } -string IdentifierDocument::GetComments() const +vector IdentifierDocument::GetComments() const { - return ImplodeStringVec(comments); + return comments; } -string IdentifierDocument::GetFieldComments(const string& field) const +vector IdentifierDocument::GetFieldComments(const string& field) const { record_field_map::const_iterator it = fields.find(field); if ( it == fields.end() ) - return string(); + return vector(); - return ImplodeStringVec(it->second->comments); + return it->second->comments; } -ScriptDocument::ScriptDocument(const string& arg_name) +list +IdentifierDocument::GetRedefs(const string& from_script) const + { + list rval; + + for ( redef_list::const_iterator it = redefs.begin(); it != redefs.end(); + ++it ) + { + if ( from_script == (*it)->from_script ) + rval.push_back(*(*it)); + } + + return rval; + } + +string IdentifierDocument::GetDeclaringScriptForField(const string& field) const + { + record_field_map::const_iterator it = fields.find(field); + + if ( it == fields.end() ) + return ""; + + return it->second->from_script; + } + +string IdentifierDocument::DoReStructuredText(bool roles_only) const + { + ODesc d; + d.SetIndentSpaces(3); + d.SetQuotes(true); + id->DescribeReST(&d, roles_only); + + if ( comments.empty() ) + return d.Description(); + + d.ClearIndentLevel(); + d.PushIndent(); + + for ( size_t i = 0; i < comments.size(); ++i ) + { + if ( i > 0 ) + d.NL(); + + if ( IsFunc(id->Type()->Tag()) ) + { + string s = comments[i]; + + if ( prettify_params(s) ) + d.NL(); + + d.Add(s.c_str()); + } + else + d.Add(comments[i].c_str()); + } + + return d.Description(); + } + +ScriptDocument::ScriptDocument(const string& arg_name, const string& arg_path) : Document(), - name(arg_name), + name(arg_name), path(arg_path), is_pkg_loader(SafeBasename(name).result == PACKAGE_LOADER), - dependencies(), module_usages(), comments(), identifier_docs(), redefs() + dependencies(), module_usages(), comments(), identifier_docs(), + options(), constants(), state_vars(), types(), events(), hooks(), + functions(), redefs() { } void ScriptDocument::AddIdentifierDoc(IdentifierDocument* doc) { identifier_docs[doc->Name()] = doc; - // TODO: sort things (e.g. function flavor, state var vs. option var) } -string ScriptDocument::GetComments() const +void ScriptDocument::DoInitPostScript() { - return ImplodeStringVec(comments); + for ( id_doc_map::const_iterator it = identifier_docs.begin(); + it != identifier_docs.end(); ++it ) + { + IdentifierDocument* doc = it->second; + ID* id = doc->GetID(); + + if ( ! is_public_api(id) ) + continue; + + if ( id->AsType() ) + { + types.push_back(doc); + DBG_LOG(DBG_BROXYGEN, "Filter id '%s' in '%s' as a type", + id->Name(), name.c_str()); + continue; + } + + if ( IsFunc(id->Type()->Tag()) ) + { + switch ( id->Type()->AsFuncType()->Flavor() ) { + case FUNC_FLAVOR_HOOK: + DBG_LOG(DBG_BROXYGEN, "Filter id '%s' in '%s' as a hook", + id->Name(), name.c_str()); + hooks.push_back(doc); + break; + case FUNC_FLAVOR_EVENT: + DBG_LOG(DBG_BROXYGEN, "Filter id '%s' in '%s' as a event", + id->Name(), name.c_str()); + events.push_back(doc); + break; + case FUNC_FLAVOR_FUNCTION: + DBG_LOG(DBG_BROXYGEN, "Filter id '%s' in '%s' as a function", + id->Name(), name.c_str()); + functions.push_back(doc); + break; + default: + reporter->InternalError("Invalid function flavor"); + break; + } + + continue; + } + + if ( id->IsConst() ) + { + if ( id->FindAttr(ATTR_REDEF) ) + { + DBG_LOG(DBG_BROXYGEN, "Filter id '%s' in '%s' as an option", + id->Name(), name.c_str()); + options.push_back(doc); + } + else + { + DBG_LOG(DBG_BROXYGEN, "Filter id '%s' in '%s' as a constant", + id->Name(), name.c_str()); + constants.push_back(doc); + } + + continue; + } + + if ( id->Type()->Tag() == TYPE_ENUM ) + // Enums are always referenced/documented from the type's + // documentation. + continue; + + DBG_LOG(DBG_BROXYGEN, "Filter id '%s' in '%s' as a state variable", + id->Name(), name.c_str()); + state_vars.push_back(doc); + } + } + +vector ScriptDocument::GetComments() const + { + return comments; + } + +static size_t end_of_first_sentence(const string& s) + { + size_t rval = 0; + + while ( (rval = s.find_first_of('.', rval)) != string::npos ) + { + if ( rval == s.size() - 1 ) + // Period is at end of string. + return rval; + + if ( isspace(s[rval + 1]) ) + // Period has a space after it. + return rval; + + // Period has some non-space character after it, keep looking. + ++rval; + } + + return rval; + } + +static bool is_all_whitespace(const string& s) + { + for ( size_t i = 0; i < s.size(); ++i ) + if ( ! isspace(s[i]) ) + return false; + + return true; + } + +static vector summary_comment(const vector& cmnts) + { + vector rval; + + for ( size_t i = 0; i < cmnts.size(); ++i ) + { + size_t end = end_of_first_sentence(cmnts[i]); + + if ( end == string::npos ) + { + if ( is_all_whitespace(cmnts[i]) ) + break; + + rval.push_back(cmnts[i]); + } + else + { + rval.push_back(cmnts[i].substr(0, end + 1)); + break; + } + } + + return rval; + } + +class ReStructuredTextTable { +public: + + ReStructuredTextTable(size_t arg_num_cols) + : num_cols(arg_num_cols), rows(), longest_row_in_column() + { + for ( size_t i = 0; i < num_cols; ++i ) + longest_row_in_column.push_back(1); + } + + void AddRow(const vector& new_row) + { + assert(new_row.size() == num_cols); + rows.push_back(new_row); + + for ( size_t i = 0; i < new_row.size(); ++i ) + if ( new_row[i].size() > longest_row_in_column[i] ) + longest_row_in_column[i] = new_row[i].size(); + } + + static string MakeBorder(const vector col_sizes, char border) + { + string rval; + + for ( size_t i = 0; i < col_sizes.size(); ++i ) + { + if ( i > 0 ) + rval += " "; + + rval += string(col_sizes[i], border); + } + + rval += "\n"; + return rval; + } + + string AsString(char border) const + { + string rval = MakeBorder(longest_row_in_column, border); + + for ( size_t row = 0; row < rows.size(); ++row ) + { + for ( size_t col = 0; col < num_cols; ++col ) + { + if ( col > 0 ) + { + size_t last = rows[row][col - 1].size(); + size_t longest = longest_row_in_column[col - 1]; + size_t whitespace = longest - last + 1; + rval += string(whitespace, ' '); + } + + rval += rows[row][col]; + } + + rval += "\n"; + } + + rval += MakeBorder(longest_row_in_column, border); + return rval; + } + + +private: + + size_t num_cols; + vector > rows; + vector longest_row_in_column; +}; + +static void add_summary_rows(const ODesc& id_desc, const vector& cmnts, + ReStructuredTextTable* table) + { + vector row; + row.push_back(id_desc.Description()); + + if ( cmnts.empty() ) + { + row.push_back(""); + table->AddRow(row); + return; + } + + row.push_back(cmnts[0]); + table->AddRow(row); + + for ( size_t i = 1; i < cmnts.size(); ++i ) + { + row.clear(); + row.push_back(""); + row.push_back(cmnts[i]); + table->AddRow(row); + } + } + +static string make_summary(const string& heading, char underline, char border, + const list& id_list) + { + if ( id_list.empty() ) + return ""; + + ReStructuredTextTable table(2); + + for ( list::const_iterator it = id_list.begin(); + it != id_list.end(); ++it ) + { + ID* id = (*it)->GetID(); + ODesc d; + d.SetQuotes(1); + id->DescribeReSTShort(&d); + add_summary_rows(d, summary_comment((*it)->GetComments()), &table); + } + + return make_heading(heading, underline) + table.AsString(border) + "\n"; + } + +static string make_redef_summary(const string& heading, char underline, + char border, const string& from_script, + const set& id_set) + { + if ( id_set.empty() ) + return ""; + + ReStructuredTextTable table(2); + + for ( set::const_iterator it = id_set.begin(); + it != id_set.end(); ++it ) + { + ID* id = (*it)->GetID(); + ODesc d; + d.SetQuotes(1); + id->DescribeReSTShort(&d); + + typedef list redef_list; + redef_list redefs = (*it)->GetRedefs(from_script); + + for ( redef_list::const_iterator iit = redefs.begin(); + iit != redefs.end(); ++iit ) + add_summary_rows(d, summary_comment(iit->comments), &table); + } + + return make_heading(heading, underline) + table.AsString(border) + "\n"; + } + +static string make_details(const string& heading, char underline, + const list& id_list) + { + if ( id_list.empty() ) + return ""; + + string rval = make_heading(heading, underline); + + for ( list::const_iterator it = id_list.begin(); + it != id_list.end(); ++it ) + { + rval += (*it)->ReStructuredText(); + rval += "\n\n"; + } + + return rval; + } + +static string make_redef_details(const string& heading, char underline, + const set& id_set) + { + if ( id_set.empty() ) + return ""; + + string rval = make_heading(heading, underline); + + for ( set::const_iterator it = id_set.begin(); + it != id_set.end(); ++it ) + { + rval += (*it)->ReStructuredText(true); + rval += "\n\n"; + } + + return rval; + } + +string ScriptDocument::DoReStructuredText(bool roles_only) const + { + string rval; + + rval += ":tocdepth: 3\n\n"; + rval += make_heading(name, '='); + + for ( string_set::const_iterator it = module_usages.begin(); + it != module_usages.end(); ++it ) + rval += ".. bro:namespace:: " + *it + "\n"; + + rval += "\n"; + + for ( size_t i = 0; i < comments.size(); ++i ) + rval += comments[i] + "\n"; + + rval += "\n"; + + if ( ! module_usages.empty() ) + { + rval += module_usages.size() > 1 ? ":Namespaces: " : ":Namespace: "; + + for ( string_set::const_iterator it = module_usages.begin(); + it != module_usages.end(); ++it ) + { + if ( it != module_usages.begin() ) + rval += ", "; + + rval += *it; + } + + rval += "\n"; + } + + if ( ! dependencies.empty() ) + { + rval += ":Imports: "; + + for ( string_set::const_iterator it = dependencies.begin(); + it != dependencies.end(); ++it ) + { + if ( it != dependencies.begin() ) + rval += ", "; + + rval += fmt(":doc:`%s `", it->c_str(), it->c_str()); + // TODO linking to packages is a bit different? + } + + rval += "\n"; + } + + // TODO: make this an absolute path? + rval += fmt(":Source File: :download:`%s`\n", name.c_str()); + rval += "\n"; + rval += make_heading("Summary", '~'); + rval += make_summary("Options", '#', '=', options); + rval += make_summary("Constants", '#', '=', constants); + rval += make_summary("State Variables", '#', '=', state_vars); + rval += make_summary("Types", '#', '=', types); + rval += make_redef_summary("Redefinitions", '#', '=', name, redefs); + rval += make_summary("Events", '#', '=', events); + rval += make_summary("Hooks", '#', '=', hooks); + rval += make_summary("Functions", '#', '=', functions); + rval += "\n"; + rval += make_heading("Detailed Interface", '~'); + rval += make_details("Options", '#', options); + rval += make_details("Constants", '#', constants); + rval += make_details("State Variables", '#', state_vars); + rval += make_details("Types", '#', types); + //rval += make_redef_details("Redefinitions", '#', redefs); + rval += make_details("Events", '#', events); + rval += make_details("Hooks", '#', hooks); + rval += make_details("Functions", '#', functions); + + return rval; + } + +static time_t get_mtime(const string& filename) + { + struct stat s; + + if ( stat(filename.c_str(), &s) < 0 ) + reporter->InternalError("Broxygen failed to stat file '%s': %s", + filename.c_str(), strerror(errno)); + + return s.st_mtime; + } + +time_t IdentifierDocument::DoGetModificationTime() const + { + // Could probably get away with just checking the set of scripts that + // contributed to the ID declaration/redefinitions, but this is easier... + return declaring_script->GetModificationTime(); + } + +time_t ScriptDocument::DoGetModificationTime() const + { + time_t most_recent = get_mtime(path); + + for ( string_set::const_iterator it = dependencies.begin(); + it != dependencies.end(); ++it ) + { + Document* doc = broxygen_mgr->GetScriptDoc(*it); + + if ( ! doc ) + { + string pkg_name = *it + "/" + PACKAGE_LOADER; + doc = broxygen_mgr->GetScriptDoc(pkg_name); + + if ( ! doc ) + reporter->InternalWarning("Broxygen failed to get mtime of %s", + it->c_str()); + continue; + } + + time_t dep_mtime = doc->GetModificationTime(); + + if ( dep_mtime > most_recent ) + most_recent = dep_mtime; + } + + return most_recent; + } + +time_t PackageDocument::DoGetModificationTime() const + { + string readme_file = find_file(pkg_name + "/README", bro_path()); + + if ( readme_file.empty() ) + return 0; + + return get_mtime(readme_file); } diff --git a/src/broxygen/Document.h b/src/broxygen/Document.h index ec9212bfba..6046369727 100644 --- a/src/broxygen/Document.h +++ b/src/broxygen/Document.h @@ -7,7 +7,7 @@ #include #include #include -#include +#include #include "ID.h" #include "Type.h" @@ -32,10 +32,19 @@ public: std::string Name() const { return DoName(); } + std::string ReStructuredText(bool roles_only = false) const + { return DoReStructuredText(roles_only); } + + void InitPostScript() + { return DoInitPostScript(); } + private: virtual time_t DoGetModificationTime() const = 0; virtual std::string DoName() const = 0; + virtual std::string DoReStructuredText(bool roles_only) const = 0; + virtual void DoInitPostScript() + { } }; class PackageDocument : public Document { @@ -44,28 +53,29 @@ public: PackageDocument(const std::string& name); - // TODO: can be comments from package README - std::string GetReadme() const - { return std::string(); } + std::vector GetReadme() const + { return readme; } private: - // TODO - time_t DoGetModificationTime() const - { return 0; } + time_t DoGetModificationTime() const; std::string DoName() const { return pkg_name; } + std::string DoReStructuredText(bool roles_only) const; + std::string pkg_name; + std::vector readme; }; -class IdentifierDocument : public Document { +class ScriptDocument; +class IdentifierDocument : public Document { public: - IdentifierDocument(ID* id); + IdentifierDocument(ID* id, ScriptDocument* script); ~IdentifierDocument(); @@ -88,24 +98,31 @@ public: ID* GetID() const { return id; } - std::string GetComments() const; + ScriptDocument* GetDeclaringScript() const + { return declaring_script; } - std::string GetFieldComments(const std::string& field) const; + std::string GetDeclaringScriptForField(const std::string& field) const; + + std::vector GetComments() const; + + std::vector GetFieldComments(const std::string& field) const; + + struct Redefinition { + std::string from_script; + std::string new_val_desc; + std::vector comments; + }; + + std::list GetRedefs(const std::string& from_script) const; private: - // TODO - time_t DoGetModificationTime() const - { return 0; } + time_t DoGetModificationTime() const; std::string DoName() const { return id->Name(); } - struct Redefinition { - std::string from_script; - string new_val_desc; - std::vector comments; - }; + std::string DoReStructuredText(bool roles_only) const; struct RecordField { ~RecordField() @@ -121,17 +138,18 @@ private: std::vector comments; ID* id; - string initial_val_desc; + std::string initial_val_desc; redef_list redefs; record_field_map fields; RecordField* last_field_seen; + ScriptDocument* declaring_script; }; class ScriptDocument : public Document { public: - ScriptDocument(const std::string& name); + ScriptDocument(const std::string& name, const std::string& path); void AddComment(const std::string& comment) { comments.push_back(comment); } @@ -145,32 +163,44 @@ public: void AddIdentifierDoc(IdentifierDocument* doc); void AddRedef(IdentifierDocument* doc) - { redefs.push_back(doc); } + { redefs.insert(doc); } bool IsPkgLoader() const { return is_pkg_loader; } - std::string GetComments() const; + std::vector GetComments() const; private: - typedef std::map IdentifierDocMap; - typedef std::list IdentifierDocList; + typedef std::map id_doc_map; + typedef std::list id_doc_list; + typedef std::set string_set; + typedef std::set doc_set; - // TODO - time_t DoGetModificationTime() const - { return 0; } + time_t DoGetModificationTime() const; std::string DoName() const { return name; } + std::string DoReStructuredText(bool roles_only) const; + + void DoInitPostScript() /* override */; + std::string name; + std::string path; bool is_pkg_loader; - std::set dependencies; - std::set module_usages; + string_set dependencies; + string_set module_usages; std::vector comments; - IdentifierDocMap identifier_docs; - IdentifierDocList redefs; + id_doc_map identifier_docs; + id_doc_list options; + id_doc_list constants; + id_doc_list state_vars; + id_doc_list types; + id_doc_list events; + id_doc_list hooks; + id_doc_list functions; + doc_set redefs; }; diff --git a/src/broxygen/Manager.cc b/src/broxygen/Manager.cc index 707481d86a..68e4c07b98 100644 --- a/src/broxygen/Manager.cc +++ b/src/broxygen/Manager.cc @@ -1,8 +1,8 @@ #include "Manager.h" -#include "Reporter.h" #include "util.h" #include +#include using namespace broxygen; using namespace std; @@ -25,71 +25,22 @@ static string RemoveLeadingSpace(const string& s) return rval; } -static string PrettifyParams(const string& s) - { - size_t identifier_start_pos = 0; - bool in_identifier = false; - string identifier; - - for ( size_t i = 0; i < s.size(); ++i ) - { - char next = s[i]; - - if ( ! in_identifier ) - { - // Pass by leading whitespace. - if ( isspace(next) ) - continue; - - // Only allow alphabetic and '_' as first char of identifier. - if ( isalpha(next) || next == '_' ) - { - identifier_start_pos = i; - identifier += next; - in_identifier = true; - continue; - } - - // Don't need to change anything. - return s; - } - - // All other character of identifier are alphanumeric or '_'. - if ( isalnum(next) || next == '_' ) - { - identifier += next; - continue; - } - - // Prettify param and return value docs for a function's reST markup. - if ( next == ':' ) - { - string rval = s; - string subst; - - if ( identifier == "Returns" ) - subst = "\n:returns"; - else - subst = "\n:param " + identifier; - - rval.replace(identifier_start_pos, identifier.size(), subst); - return rval; - } - - // Don't need to change anything. - return s; - } - - return s; - } - -Manager::Manager(const string& arg_config) +Manager::Manager(const string& arg_config, const string& bro_command) : disabled(), comment_buffer(), comment_buffer_map(), packages(), scripts(), identifiers(), all_docs(), last_identifier_seen(), incomplete_type(), - config(arg_config) + enum_mappings(), config(arg_config), bro_mtime() { if ( getenv("BRO_DISABLE_BROXYGEN") ) disabled = true; + + string path_to_bro = find_file(bro_command, getenv("PATH")); + struct stat s; + + if ( path_to_bro.empty() || stat(path_to_bro.c_str(), &s) < 0 ) + reporter->InternalError("Broxygen can't get mtime of bro binary %s: %s", + path_to_bro.c_str(), strerror(errno)); + + bro_mtime = s.st_mtime; } Manager::~Manager() @@ -102,7 +53,6 @@ void Manager::InitPreScript() { if ( disabled ) return; - // TODO: create file/proto analyzer doc } void Manager::InitPostScript() @@ -110,6 +60,9 @@ void Manager::InitPostScript() if ( disabled ) return; + for ( size_t i = 0; i < all_docs.size(); ++i ) + all_docs[i]->InitPostScript(); + config.FindDependencies(all_docs); } @@ -134,7 +87,7 @@ void Manager::File(const string& path) return; } - ScriptDocument* doc = new ScriptDocument(name); + ScriptDocument* doc = new ScriptDocument(name, path); scripts.map[name] = doc; all_docs.push_back(doc); DBG_LOG(DBG_BROXYGEN, "Made ScriptDocument %s", name.c_str()); @@ -209,7 +162,7 @@ void Manager::ModuleUsage(const string& path, const string& module) IdentifierDocument* Manager::CreateIdentifierDoc(ID* id, ScriptDocument* script) { - IdentifierDocument* rval = new IdentifierDocument(id); + IdentifierDocument* rval = new IdentifierDocument(id, script); rval->AddComments(comment_buffer); comment_buffer.clear(); @@ -226,7 +179,10 @@ IdentifierDocument* Manager::CreateIdentifierDoc(ID* id, ScriptDocument* script) all_docs.push_back(rval); identifiers.map[id->Name()] = rval; last_identifier_seen = rval; - script->AddIdentifierDoc(rval); + + if ( script ) + script->AddIdentifierDoc(rval); + return rval; } @@ -256,18 +212,9 @@ void Manager::StartType(ID* id) id->Name(), script.c_str()); } -void Manager::StartRedef(ID* id) +static bool IsEnumType(ID* id) { - if ( disabled ) - return; - - IdentifierDocument* id_doc = identifiers.GetDocument(id->Name()); - - if ( id_doc ) - last_identifier_seen = id_doc; - else - DbgAndWarn(fmt("Broxygen redef tracking unknown identifier: %s", - id->Name())); + return id->AsType() ? id->AsType()->Tag() == TYPE_ENUM : false; } void Manager::Identifier(ID* id) @@ -275,21 +222,18 @@ void Manager::Identifier(ID* id) if ( disabled ) return; - if ( incomplete_type && incomplete_type->Name() == id->Name() ) + if ( incomplete_type ) { - DBG_LOG(DBG_BROXYGEN, "Finished document for type %s", id->Name()); - incomplete_type->CompletedTypeDecl(); - incomplete_type = 0; - return; - } + if ( incomplete_type->Name() == id->Name() ) + { + DBG_LOG(DBG_BROXYGEN, "Finished document for type %s", id->Name()); + incomplete_type->CompletedTypeDecl(); + incomplete_type = 0; + return; + } - if ( id->GetLocationInfo() == &no_location ) - { - // Internally-created identifier (e.g. file/proto analyzer enum tags). - // Can be ignored here as they need to be documented via other means. - DBG_LOG(DBG_BROXYGEN, "Skip documenting identifier %s: no location", - id->Name()); - return; + if ( IsEnumType(incomplete_type->GetID()) ) + enum_mappings[id->Name()] = incomplete_type->GetID()->Name(); } IdentifierDocument* id_doc = identifiers.GetDocument(id->Name()); @@ -308,6 +252,16 @@ void Manager::Identifier(ID* id) return; } + if ( id->GetLocationInfo() == &no_location ) + { + // Internally-created identifier (e.g. file/proto analyzer enum tags). + // Handled specially since they don't have a script location. + DBG_LOG(DBG_BROXYGEN, "Made internal IdentifierDocument %s", + id->Name()); + CreateIdentifierDoc(id, 0); + return; + } + string script = without_bropath_component(id->GetLocationInfo()->filename); ScriptDocument* script_doc = scripts.GetDocument(script); @@ -428,32 +382,8 @@ void Manager::PostComment(const string& comment, const string& id_hint) comment_buffer_map[id_hint].push_back(RemoveLeadingSpace(comment)); } -StringVal* Manager::GetIdentifierComments(const string& name) const +string Manager::GetEnumTypeName(const string& id) const { - IdentifierDocument* d = identifiers.GetDocument(name); - return new StringVal(d ? d->GetComments() : ""); - } - -StringVal* Manager::GetScriptComments(const string& name) const - { - ScriptDocument* d = scripts.GetDocument(name); - return new StringVal(d ? d->GetComments() : ""); - } - -StringVal* Manager::GetPackageReadme(const string& name) const - { - PackageDocument* d = packages.GetDocument(name); - return new StringVal(d ? d->GetReadme() : ""); - } - -StringVal* Manager::GetRecordFieldComments(const string& name) const - { - size_t i = name.find('$'); - - if ( i > name.size() - 2 ) - // '$' is last char in string or not found. - return new StringVal(""); - - IdentifierDocument* d = identifiers.GetDocument(name.substr(0, i)); - return new StringVal(d ? d->GetFieldComments(name.substr(i + 1)) : ""); + map::const_iterator it = enum_mappings.find(id); + return it == enum_mappings.end() ? "" : it->second; } diff --git a/src/broxygen/Manager.h b/src/broxygen/Manager.h index 67f7e543af..42b9ea828a 100644 --- a/src/broxygen/Manager.h +++ b/src/broxygen/Manager.h @@ -3,6 +3,8 @@ #include "Configuration.h" #include "Document.h" + +#include "Reporter.h" #include "ID.h" #include "Type.h" #include "Val.h" @@ -10,6 +12,8 @@ #include #include #include +#include +#include namespace broxygen { @@ -32,7 +36,7 @@ class Manager { public: - Manager(const std::string& config); + Manager(const std::string& config, const std::string& bro_command); ~Manager(); @@ -50,8 +54,6 @@ public: void StartType(ID* id); - void StartRedef(ID* id); - void Identifier(ID* id); void RecordField(const ID* id, const TypeDecl* field, @@ -66,13 +68,20 @@ public: void PostComment(const std::string& comment, const std::string& identifier_hint = ""); - StringVal* GetIdentifierComments(const std::string& name) const; + std::string GetEnumTypeName(const std::string& id) const; - StringVal* GetScriptComments(const std::string& name) const; + IdentifierDocument* GetIdentifierDoc(const std::string& name) const + { return identifiers.GetDocument(name); } - StringVal* GetPackageReadme(const std::string& name) const; + ScriptDocument* GetScriptDoc(const std::string& name) const + { return scripts.GetDocument(name); } - StringVal* GetRecordFieldComments(const std::string& name) const; + PackageDocument* GetPackageDoc(const std::string& name) const + { return packages.GetDocument(name); } + + template + bool IsUpToDate(const std::string& target_file, + const std::vector& dependencies) const; private: @@ -90,9 +99,40 @@ private: std::vector all_docs; IdentifierDocument* last_identifier_seen; IdentifierDocument* incomplete_type; + std::map enum_mappings; Config config; + time_t bro_mtime; }; +template +bool Manager::IsUpToDate(const string& target_file, + const vector& dependencies) const + { + struct stat s; + + if ( stat(target_file.c_str(), &s) < 0 ) + { + if ( errno == ENOENT ) + // Doesn't exist. + return false; + + reporter->InternalError("Broxygen failed to stat target file '%s': %s", + target_file.c_str(), strerror(errno)); + } + + if ( difftime(bro_mtime, s.st_mtime) > 0 ) + return false; + + if ( difftime(config.GetModificationTime(), s.st_mtime) > 0 ) + return false; + + for ( size_t i = 0; i < dependencies.size(); ++i ) + if ( difftime(dependencies[i]->GetModificationTime(), s.st_mtime) > 0 ) + return false; + + return true; + } + } // namespace broxygen extern broxygen::Manager* broxygen_mgr; diff --git a/src/broxygen/broxygen.bif b/src/broxygen/broxygen.bif index a241b2c985..0d346411d0 100644 --- a/src/broxygen/broxygen.bif +++ b/src/broxygen/broxygen.bif @@ -2,26 +2,65 @@ %%{ #include "broxygen/Manager.h" +#include "util.h" + +static StringVal* comments_to_val(const vector& comments) + { + return new StringVal(implode_string_vector(comments)); + } %%} # TODO: documentation function get_identifier_comments%(name: string%): string %{ - return broxygen_mgr->GetIdentifierComments(name->CheckString()); + using namespace broxygen; + IdentifierDocument* d = broxygen_mgr->GetIdentifierDoc(name->CheckString()); + + if ( ! d ) + return new StringVal(""); + + return comments_to_val(d->GetComments()); %} function get_script_comments%(name: string%): string %{ - return broxygen_mgr->GetScriptComments(name->CheckString()); + using namespace broxygen; + ScriptDocument* d = broxygen_mgr->GetScriptDoc(name->CheckString()); + + if ( ! d ) + return new StringVal(""); + + return comments_to_val(d->GetComments()); %} function get_package_readme%(name: string%): string %{ - return broxygen_mgr->GetPackageReadme(name->CheckString()); + using namespace broxygen; + PackageDocument* d = broxygen_mgr->GetPackageDoc(name->CheckString()); + + if ( ! d ) + return new StringVal(""); + + return comments_to_val(d->GetReadme()); %} function get_record_field_comments%(name: string%): string %{ - return broxygen_mgr->GetRecordFieldComments(name->CheckString()); + using namespace broxygen; + string accessor = name->CheckString(); + size_t i = accessor.find('$'); + + if ( i > accessor.size() - 2 ) + return new StringVal(""); + + string id = accessor.substr(0, i); + + IdentifierDocument* d = broxygen_mgr->GetIdentifierDoc(id); + + if ( ! d ) + return new StringVal(""); + + string field = accessor.substr(i + 1); + return comments_to_val(d->GetFieldComments(field)); %} diff --git a/src/main.cc b/src/main.cc index 2ab25460d4..cac1acd7b1 100644 --- a/src/main.cc +++ b/src/main.cc @@ -762,7 +762,6 @@ int main(int argc, char** argv) reporter = new Reporter(); thread_mgr = new threading::Manager(); - broxygen_mgr = new broxygen::Manager(broxygen_config); #ifdef DEBUG if ( debug_streams ) @@ -809,6 +808,8 @@ int main(int argc, char** argv) timer_mgr = new PQ_TimerMgr(""); // timer_mgr = new CQ_TimerMgr(); + broxygen_mgr = new broxygen::Manager(broxygen_config, bro_argv[0]); + add_input_file("base/init-bare.bro"); if ( ! bare_mode ) add_input_file("base/init-default.bro"); diff --git a/src/parse.y b/src/parse.y index d428452d66..840ccd527d 100644 --- a/src/parse.y +++ b/src/parse.y @@ -1022,14 +1022,15 @@ decl: broxygen_mgr->Redef($2, ::filename); } - | TOK_REDEF TOK_ENUM global_id TOK_ADD_TO - '{' { parser_redef_enum($3); } enum_body '}' ';' + | TOK_REDEF TOK_ENUM global_id TOK_ADD_TO '{' + { parser_redef_enum($3); broxygen_mgr->Redef($3, ::filename); } + enum_body '}' ';' { // Broxygen already grabbed new enum IDs as the type created them. } | TOK_REDEF TOK_RECORD global_id - { cur_decl_type_id = $3; broxygen_mgr->StartRedef($3); } + { cur_decl_type_id = $3; broxygen_mgr->Redef($3, ::filename); } TOK_ADD_TO '{' { ++in_record; } type_decl_list diff --git a/src/plugin/ComponentManager.h b/src/plugin/ComponentManager.h index dab98fcad0..7561764035 100644 --- a/src/plugin/ComponentManager.h +++ b/src/plugin/ComponentManager.h @@ -10,6 +10,7 @@ #include "Var.h" #include "Val.h" #include "Reporter.h" +#include "broxygen/Manager.h" namespace plugin { @@ -133,6 +134,7 @@ ComponentManager::ComponentManager(const string& arg_module) tag_enum_type = new EnumType(); ::ID* id = install_ID("Tag", module.c_str(), true, true); add_type(id, tag_enum_type, 0); + broxygen_mgr->Identifier(id); } template diff --git a/src/scan.l b/src/scan.l index 3dc68a52cc..5b80350111 100644 --- a/src/scan.l +++ b/src/scan.l @@ -145,7 +145,8 @@ ESCSEQ (\\([^\n]|[0-7]+|x[[:xdigit:]]+)) } ##.* { - broxygen_mgr->PreComment(yytext + 2); + if ( yytext[2] != '#' ) + broxygen_mgr->PreComment(yytext + 2); } #{OWS}@no-test.* return TOK_NO_TEST; diff --git a/src/util.cc b/src/util.cc index 8f85d6fca4..c526527648 100644 --- a/src/util.cc +++ b/src/util.cc @@ -590,6 +590,33 @@ const char* fmt_access_time(double t) return buf; } +bool ensure_intermediate_dirs(const char* dirname) + { + if ( ! dirname || strlen(dirname) == 0 ) + return false; + + bool absolute = dirname[0] == '/'; + string path = normalize_path(dirname); + + vector path_components; + tokenize_string(path, "/", &path_components); + + string current_dir; + + for ( size_t i = 0; i < path_components.size(); ++i ) + { + if ( i > 0 || absolute ) + current_dir += "/"; + + current_dir += path_components[i]; + + if ( ! ensure_dir(current_dir.c_str()) ) + return false; + } + + return true; + } + bool ensure_dir(const char *dirname) { struct stat st; @@ -976,6 +1003,22 @@ void SafePathOp::DoFunc(PathOpFn fn, const string& path, bool error_aborts) delete [] tmp; } +string implode_string_vector(const std::vector& v, + const std::string& delim) + { + string rval; + + for ( size_t i = 0; i < v.size(); ++i ) + { + if ( i > 0 ) + rval += delim; + + rval += v[i]; + } + + return rval; + } + string flatten_script_name(const string& name, const string& prefix) { string rval = prefix; diff --git a/src/util.h b/src/util.h index 1dc47db68a..815534a6a6 100644 --- a/src/util.h +++ b/src/util.h @@ -150,6 +150,7 @@ extern const char* fmt(const char* format, ...) myattribute((format (printf, 1, 2))); extern const char* fmt_access_time(double time); +extern bool ensure_intermediate_dirs(const char* dirname); extern bool ensure_dir(const char *dirname); // Returns true if path exists and is a directory. @@ -251,6 +252,9 @@ public: : SafePathOp(&basename, path, error_aborts) { } }; +std::string implode_string_vector(const std::vector& v, + const std::string& delim = "\n"); + /** * Flatten a script name by replacing '/' path separators with '.'. * @param file A path to a Bro script. If it is a __load__.bro, that part From 9967aea52c6da14daacbbc9f1662dfcf344793a1 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Thu, 21 Nov 2013 14:34:32 -0600 Subject: [PATCH 11/40] Integrate new Broxygen functionality into Sphinx. Add a "broxygen" domain Sphinx extension w/ directives to allow on-the-fly documentation to be generated w/ Bro and included in files. This means all autogenerated reST docs are now done by Bro. The odd CMake/Python glue scipts which used to generate some portions are now gone. Bro and the Sphinx extension handle checking for outdated docs themselves. Parallel builds of `make doc` target should now work (mostly because I don't think there's any tasks that can be done in parallel anymore). Overall, this seems to simplify things and make the Broxygen-generated portions of the documentation visible/traceable from the main Sphinx source tree. The one odd thing still is that per-script documentation is rsync'd in to a shadow copy of the Sphinx source tree within the build dir. This is less elegant than using the new broxygen extension to make per-script docs, but rsync is faster and simpler. Simpler as in less code because it seems like, in the best case, I'd need to write a custom Sphinx Builder to be able to get that to even work. --- Makefile | 12 - doc/CMakeLists.txt | 135 ++++---- doc/README | 9 +- doc/bin/group_index_generator.py | 72 ---- doc/broxygen.conf.in | 1 + doc/conf.py.in | 12 +- doc/ext/bro.py | 4 + doc/ext/broxygen.py | 311 ++++++++++++++++++ doc/frameworks/logging-dataseries.rst | 2 +- doc/frameworks/logging.rst | 6 +- doc/frameworks/notice.rst | 2 +- doc/frameworks/signatures.rst | 2 +- doc/index.rst | 2 +- doc/quickstart/index.rst | 2 +- .../builtins.rst | 0 doc/script-reference/file-analyzers.rst | 1 + doc/{scripts => script-reference}/index.rst | 12 +- .../packages.rst | 8 +- doc/script-reference/proto-analyzers.rst | 1 + doc/script-reference/scripts.rst | 5 + doc/scripting/index.rst | 8 +- doc/scripts/CMakeLists.txt | 226 ------------- doc/scripts/DocSourcesList.cmake | 282 ---------------- doc/scripts/README | 44 --- doc/scripts/example.rst | 291 ---------------- doc/scripts/genDocSourcesList.sh | 86 ----- doc/scripts/internal.rst | 5 - doc/scripts/scripts.rst | 8 - scripts/base/frameworks/reporter/main.bro | 2 +- scripts/base/init-bare.bro | 6 +- scripts/base/protocols/dhcp/main.bro | 2 +- scripts/base/protocols/http/__load__.bro | 2 +- scripts/broxygen/README | 4 + scripts/broxygen/__load__.bro | 2 + {doc/scripts => scripts/broxygen}/example.bro | 0 scripts/policy/misc/known-devices.bro | 2 +- scripts/policy/misc/stats.bro | 2 +- src/BroDoc.cc | 4 +- src/Type.cc | 8 +- src/broxygen/Configuration.cc | 126 ++++++- src/broxygen/Configuration.h | 8 + src/broxygen/Document.cc | 15 +- src/const.bif | 2 +- src/event.bif | 2 +- src/reporter.bif | 2 +- testing/btest/Makefile | 5 +- testing/btest/coverage/doc.test | 8 - 47 files changed, 578 insertions(+), 1173 deletions(-) delete mode 100755 doc/bin/group_index_generator.py create mode 100644 doc/broxygen.conf.in create mode 100644 doc/ext/broxygen.py rename doc/{scripts => script-reference}/builtins.rst (100%) create mode 100644 doc/script-reference/file-analyzers.rst rename doc/{scripts => script-reference}/index.rst (50%) rename doc/{scripts => script-reference}/packages.rst (81%) create mode 100644 doc/script-reference/proto-analyzers.rst create mode 100644 doc/script-reference/scripts.rst delete mode 100644 doc/scripts/CMakeLists.txt delete mode 100644 doc/scripts/DocSourcesList.cmake delete mode 100644 doc/scripts/README delete mode 100644 doc/scripts/example.rst delete mode 100755 doc/scripts/genDocSourcesList.sh delete mode 100644 doc/scripts/internal.rst delete mode 100644 doc/scripts/scripts.rst create mode 100644 scripts/broxygen/README create mode 100644 scripts/broxygen/__load__.bro rename {doc/scripts => scripts/broxygen}/example.bro (100%) delete mode 100644 testing/btest/coverage/doc.test diff --git a/Makefile b/Makefile index 4984f6db31..f4b2104c73 100644 --- a/Makefile +++ b/Makefile @@ -29,18 +29,6 @@ doc: configured docclean: configured $(MAKE) -C $(BUILD) $@ -restdoc: configured - $(MAKE) -C $(BUILD) $@ - -restclean: configured - $(MAKE) -C $(BUILD) $@ - -broxygen: configured - $(MAKE) -C $(BUILD) $@ - -broxygenclean: configured - $(MAKE) -C $(BUILD) $@ - dist: @rm -rf $(VERSION_FULL) $(VERSION_FULL).tgz @rm -rf $(VERSION_MIN) $(VERSION_MIN).tgz diff --git a/doc/CMakeLists.txt b/doc/CMakeLists.txt index 373b4643ba..9498556edc 100644 --- a/doc/CMakeLists.txt +++ b/doc/CMakeLists.txt @@ -1,75 +1,86 @@ -set(BIF_SRC_DIR ${PROJECT_SOURCE_DIR}/src) -set(RST_OUTPUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/rest_output) -set(DOC_OUTPUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/out) -set(DOC_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}) -set(DOC_SOURCE_WORKDIR ${CMAKE_CURRENT_BINARY_DIR}/sphinx-sources) +set(BROCCOLI_DOCS_SRC ${CMAKE_BINARY_DIR}/aux/broccoli/doc/html) +set(BROCCOLI_DOCS_DST ${CMAKE_BINARY_DIR}/html/broccoli-api) +set(SPHINX_INPUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/sphinx_input) +set(SPHINX_OUTPUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/sphinx_output) +set(BROXYGEN_SCRIPT_OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/broxygen_script_output) +set(BROXYGEN_CACHE_DIR ${CMAKE_CURRENT_BINARY_DIR}/broxygen_cache) -set(MASTER_POLICY_INDEX ${CMAKE_CURRENT_BINARY_DIR}/scripts/policy_index) -set(MASTER_PACKAGE_INDEX ${CMAKE_CURRENT_BINARY_DIR}/scripts/pkg_index) +# Find out what BROPATH to use when executing bro. +execute_process(COMMAND ${CMAKE_BINARY_DIR}/bro-path-dev + OUTPUT_VARIABLE BROPATH + RESULT_VARIABLE retval + OUTPUT_STRIP_TRAILING_WHITESPACE) +if (NOT ${retval} EQUAL 0) + message(FATAL_ERROR "Problem setting BROPATH") +endif () -file(GLOB_RECURSE DOC_SOURCES FOLLOW_SYMLINKS "*") +set(BROMAGIC ${BRO_MAGIC_SOURCE_PATH}) -# configure the Sphinx config file (expand variables CMake might know about) +# Configure the Sphinx config file (expand variables CMake might know about). configure_file(${CMAKE_CURRENT_SOURCE_DIR}/conf.py.in ${CMAKE_CURRENT_BINARY_DIR}/conf.py @ONLY) -add_subdirectory(scripts) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/broxygen.conf.in + ${CMAKE_CURRENT_BINARY_DIR}/broxygen.conf + @ONLY) -# The "broxygen" target generates reST documentation for any outdated bro -# scripts and then uses Sphinx to generate HTML documentation from the reST -add_custom_target(broxygen - # copy the template documentation to the build directory - # to give as input for sphinx - COMMAND rsync -r --copy-links --times - ${DOC_SOURCE_DIR}/ - ${DOC_SOURCE_WORKDIR} - # copy generated policy script documentation into the - # working copy of the template documentation - COMMAND rsync -r --copy-links --times - ${RST_OUTPUT_DIR}/ - ${DOC_SOURCE_WORKDIR}/scripts - # append to the master index of all policy scripts - COMMAND cat ${MASTER_POLICY_INDEX} >> - ${DOC_SOURCE_WORKDIR}/scripts/scripts.rst - # append to the master index of all policy packages - COMMAND cat ${MASTER_PACKAGE_INDEX} >> - ${DOC_SOURCE_WORKDIR}/scripts/packages.rst - # construct a reST file for each group - COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/bin/group_index_generator.py - ${CMAKE_CURRENT_BINARY_DIR}/scripts/group_list - ${CMAKE_CURRENT_BINARY_DIR}/scripts - ${DOC_SOURCE_WORKDIR}/scripts - # tell sphinx to generate html - COMMAND sphinx-build - -b html - -c ${CMAKE_CURRENT_BINARY_DIR} - -d ${DOC_OUTPUT_DIR}/doctrees - ${DOC_SOURCE_WORKDIR} - ${DOC_OUTPUT_DIR}/html - # create symlink to the html output directory for convenience - COMMAND "${CMAKE_COMMAND}" -E create_symlink - ${DOC_OUTPUT_DIR}/html - ${CMAKE_BINARY_DIR}/html - # copy Broccoli API reference into output dir if it exists - COMMAND test -d ${CMAKE_BINARY_DIR}/aux/broccoli/doc/html && ( rm -rf ${CMAKE_BINARY_DIR}/html/broccoli-api && cp -r ${CMAKE_BINARY_DIR}/aux/broccoli/doc/html ${CMAKE_BINARY_DIR}/html/broccoli-api ) || true - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} - COMMENT "[Sphinx] Generating HTML policy script docs" - # SOURCES just adds stuff to IDE projects as a convenience - SOURCES ${DOC_SOURCES}) +add_custom_target(sphinxdoc + # Copy the template documentation to build directory to use as input tree + # for Sphinx. This is needed because some parts are dynamically generated + # in to that tree by Bro/Broxygen. + COMMAND rsync -q -r --copy-links --times --delete + --filter='protect scripts/*' + ${CMAKE_CURRENT_SOURCE_DIR}/ ${SPHINX_INPUT_DIR} + # Use Bro/Broxygen to dynamically generate reST for all Bro scripts. + COMMAND BROPATH=${BROPATH} + BROMAGIC=${BROMAGIC} + ${CMAKE_BINARY_DIR}/src/bro + -X ${CMAKE_CURRENT_BINARY_DIR}/broxygen.conf + broxygen >/dev/null + # Rsync over the generated reST to the Sphinx source tree in the build dir. + COMMAND rsync -q -r --copy-links --times --delete --filter='protect *.bro' + ${BROXYGEN_SCRIPT_OUTPUT}/ ${SPHINX_INPUT_DIR}/scripts + # Rsync over Bro scripts to the Sphinx source tree in the build dir. + # These are used by :download: references in the generated script docs. + COMMAND rsync -q -r --copy-links --times --delete + --filter='protect /base/bif/*' --filter='protect *.rst' + --filter='include */' --filter='include *.bro' --filter='exclude *' + ${CMAKE_SOURCE_DIR}/scripts/ ${SPHINX_INPUT_DIR}/scripts + # Rsync over Bro scripts created by BIF compiler to the Sphinx source tree. + COMMAND rsync -q -r --copy-links --times --delete + --filter='protect *.rst' --filter='include */' + --filter='include *.bro' --filter='exclude *' + ${CMAKE_BINARY_DIR}/scripts/base/bif/ + ${SPHINX_INPUT_DIR}/scripts/base/bif + # Use Sphinx to build HTML. + COMMAND sphinx-build + -b html + -c ${CMAKE_CURRENT_BINARY_DIR} + -d ${SPHINX_OUTPUT_DIR}/doctrees + ${SPHINX_INPUT_DIR} + ${SPHINX_OUTPUT_DIR}/html + # Create symlink to the html output directory for convenience. + COMMAND "${CMAKE_COMMAND}" -E create_symlink + ${SPHINX_OUTPUT_DIR}/html + ${CMAKE_BINARY_DIR}/html + # Copy Broccoli API reference into output dir if it exists. + COMMAND test -d ${BROCCOLI_DOCS_SRC} && + ( rm -rf ${BROCCOLI_DOCS_DST} && + cp -r ${BROCCOLI_DOCS_SRC} ${BROCCOLI_DOCS_DST} ) || true + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + COMMENT "[Sphinx] Generate HTML documentation in ${CMAKE_BINARY_DIR}/html") -# The "broxygenclean" target removes just the Sphinx input/output directories -# from the build directory. -add_custom_target(broxygenclean - COMMAND "${CMAKE_COMMAND}" -E remove_directory - ${DOC_SOURCE_WORKDIR} - COMMAND "${CMAKE_COMMAND}" -E remove_directory - ${DOC_OUTPUT_DIR} - VERBATIM) +add_dependencies(sphinxdoc bro) -add_dependencies(broxygen restdoc) +add_custom_target(sphinxdoc_clean + COMMAND "${CMAKE_COMMAND}" -E remove_directory ${SPHINX_INPUT_DIR} + COMMAND "${CMAKE_COMMAND}" -E remove_directory ${SPHINX_OUTPUT_DIR} + COMMAND "${CMAKE_COMMAND}" -E remove_directory ${BROXYGEN_SCRIPT_OUTPUT} + COMMAND "${CMAKE_COMMAND}" -E remove_directory ${BROXYGEN_CACHE_DIR} + VERBATIM) add_custom_target(doc) add_custom_target(docclean) -add_dependencies(doc broxygen) -add_dependencies(docclean broxygenclean restclean) +add_dependencies(doc sphinxdoc) +add_dependencies(docclean sphinxdoc_clean) diff --git a/doc/README b/doc/README index 0ba0a8587f..c14a3bfbfb 100644 --- a/doc/README +++ b/doc/README @@ -15,17 +15,16 @@ which adds some reST directives and roles that aid in generating useful index entries and cross-references. Other extensions can be added in a similar fashion. -Either the ``make doc`` or ``make broxygen`` targets in the top-level -Makefile can be used to locally render the reST files into HTML. -Those targets depend on: +The ``make doc`` target in the top-level Makefile can be used to locally +render the reST files into HTML. That target depend on: * Python interpreter >= 2.5 * `Sphinx `_ >= 1.0.1 After completion, HTML documentation is symlinked in ``build/html``. -There's also ``make docclean`` and ``make broxygenclean`` targets to -clean the resulting documentation. +There's also a ``make docclean`` target which deletes any files +created during the documentation build process. Notes for Writing Documentation ------------------------------- diff --git a/doc/bin/group_index_generator.py b/doc/bin/group_index_generator.py deleted file mode 100755 index b41dcb379c..0000000000 --- a/doc/bin/group_index_generator.py +++ /dev/null @@ -1,72 +0,0 @@ -#! /usr/bin/env python - -# This script automatically generates a reST documents that lists -# a collection of Bro scripts that are "grouped" together. -# The summary text (##! comments) of the script is embedded in the list -# -# 1st argument is the file containing list of groups -# 2nd argument is the directory containing ${group}_files lists of -# scripts that belong to the group and ${group}_doc_names lists of -# document names that can be supplied to a reST :doc: role -# 3rd argument is a directory in which write a ${group}.rst file that contains -# reST style references to script docs along with summary text contained -# in original script. If ${group} ends with "index", then the file -# is always clobbered by this script, but for other unique group names, -# this script will append to existing files. - -import sys -import os -import string - -group_list = sys.argv[1] -file_manifest_dir = sys.argv[2] -output_dir = sys.argv[3] - -def make_group_file_index(dir_name, group_name): - group_file = os.path.join(dir_name, group_name + ".rst") - - if not os.path.exists(group_file): - if not os.path.exists(os.path.dirname(group_file)): - os.makedirs(os.path.dirname(group_file)) - - if group_name.endswith("index"): - with open(group_file, 'w') as f_group_file: - f_group_file.write(":orphan:\n\n") - title = "Package Index: %s\n" % os.path.dirname(group_name) - f_group_file.write(title); - for n in range(len(title)): - f_group_file.write("=") - f_group_file.write("\n"); - - return group_file - -with open(group_list, 'r') as f_group_list: - for group in f_group_list.read().splitlines(): - #print group - group_file = make_group_file_index(output_dir, group) - file_manifest = os.path.join(file_manifest_dir, group + "_files") - doc_manifest = os.path.join(file_manifest_dir, group + "_doc_names") - src_files = [] - doc_names = [] - - with open(file_manifest, 'r') as f_file_manifest: - src_files = f_file_manifest.read().splitlines() - - with open(doc_manifest, 'r') as f_doc_manifest: - doc_names = f_doc_manifest.read().splitlines() - - for i in range(len(src_files)): - src_file = src_files[i] - #print "\t" + src_file - summary_comments = [] - with open(src_file, 'r') as f_src_file: - for line in f_src_file: - sum_pos = string.find(line, "##!") - if sum_pos != -1: - summary_comments.append(line[(sum_pos+3):]) - #print summary_comments - - with open(group_file, 'a') as f_group_file: - f_group_file.write("\n:doc:`/scripts/%s`\n" % doc_names[i]) - for line in summary_comments: - f_group_file.write(" " + line) diff --git a/doc/broxygen.conf.in b/doc/broxygen.conf.in new file mode 100644 index 0000000000..b27ba6248d --- /dev/null +++ b/doc/broxygen.conf.in @@ -0,0 +1 @@ +script @BROXYGEN_SCRIPT_OUTPUT@/ * diff --git a/doc/conf.py.in b/doc/conf.py.in index 4cf64f10db..f54ca9d8c6 100644 --- a/doc/conf.py.in +++ b/doc/conf.py.in @@ -17,7 +17,7 @@ extensions = [] # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. -sys.path.insert(0, os.path.abspath('sphinx-sources/ext')) +sys.path.insert(0, os.path.abspath('sphinx_input/ext')) # ----- Begin of BTest configuration. ----- btest = os.path.abspath("@CMAKE_SOURCE_DIR@/aux/btest") @@ -33,6 +33,12 @@ btest_base="@CMAKE_SOURCE_DIR@/testing/btest" btest_tests="doc/sphinx" # ----- End of BTest configuration. ----- +# ----- Begin of Broxygen configuration. ----- +extensions += ["broxygen"] +broxygen_cache="@BROXYGEN_CACHE_DIR@" +os.environ["BROPATH"] = "@BROPATH@" +os.environ["BROMAGIC"] = "@BROMAGIC@" +# ----- End of Broxygen configuration. ----- # -- General configuration ----------------------------------------------------- @@ -47,7 +53,7 @@ os.environ["BRO_SRC_ROOT"] = "@CMAKE_SOURCE_DIR@" os.environ["DOC_ROOT"] = "@CMAKE_SOURCE_DIR@/doc" # Add any paths that contain templates here, relative to this directory. -templates_path = ['sphinx-sources/_templates', 'sphinx-sources/_static'] +templates_path = ['sphinx_input/_templates', 'sphinx_input/_static'] # The suffix of source filenames. source_suffix = '.rst' @@ -141,7 +147,7 @@ html_theme_options = { } # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['sphinx-sources/_static'] +html_static_path = ['sphinx_input/_static'] # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. diff --git a/doc/ext/bro.py b/doc/ext/bro.py index 6ef11c37f6..91f42b4e5a 100644 --- a/doc/ext/bro.py +++ b/doc/ext/bro.py @@ -191,6 +191,10 @@ class BroNotices(Index): def generate(self, docnames=None): content = {} + + if 'notices' not in self.domain.env.domaindata['bro']: + return content, False + for n in self.domain.env.domaindata['bro']['notices']: modname = n[0].split("::")[0] entries = content.setdefault(modname, []) diff --git a/doc/ext/broxygen.py b/doc/ext/broxygen.py new file mode 100644 index 0000000000..d7a4e57c21 --- /dev/null +++ b/doc/ext/broxygen.py @@ -0,0 +1,311 @@ +""" +Broxygen domain for Sphinx. + +Adds directives that allow Sphinx to invoke Bro in order to generate script +reference documentation on the fly. The directives are: + +broxygen:package + - Shows links to all scripts contained within matching package(s). +broxygen:package_index + - An index with links to matching package document(s). +broxygen:script + - Reference for matching script(s) (i.e. everything declared by the script). +broxygen:script_summary + - Shows link to matching script(s) with it's summary-section comments. +broxygen:script_index + - An index with links to all matching scrips. +broxygen:proto_analyzer + - All protocol analyzers and their components (events/bifs, etc.) +broxygen:file_analyzer + - All file analyzers and their components (events/bifs, etc.) +""" + + +from sphinx.domains import Domain, ObjType +from sphinx.locale import l_ +from docutils.parsers.rst.directives.misc import Include + + +App = None + + +def info(msg): + """Use Sphinx builder to output a console message.""" + global App + from sphinx.util.console import blue + App.builder.info(blue(msg)) + + +def pattern_to_filename_component(pattern): + """Replace certain characters in Broxygen config file target pattern. + + Such that it can be used as part of a (sane) filename. + + """ + return pattern.replace("/", ".").replace("*", "star") + + +def ensure_dir(path): + """Should act like ``mkdir -p``.""" + import os + import errno + + try: + os.makedirs(path) + except OSError as e: + if e.errno != errno.EEXIST: + raise + + +def generate_config(env, type, pattern): + """Create a Broxygen config file for a particular target. + + It can be used by Bro to generate reST docs for that target. + + """ + import os + import tempfile + from sphinx.errors import SphinxError + + work_dir = env.config.broxygen_cache + + if not work_dir: + raise SphinxError("broxygen_cache not set in sphinx config file") + + ensure_dir(work_dir) + prefix = "{0}-{1}-".format(type, pattern_to_filename_component(pattern)) + (fd, cfg) = tempfile.mkstemp(suffix=".cfg", prefix=prefix, dir=work_dir) + generated_file = "{0}.rst".format(cfg) + config = "{0}\t{1}\t{2}".format(type, generated_file, pattern) + f = os.fdopen(fd, "w") + f.write(config) + f.close() + return (cfg, generated_file) + + +def generate_target(env, type, pattern): + """Create a Broxygen target and build it. + + For a target which hasn't been referenced by any other script, this function + creates an associated config file then uses Bro w/ it to build the target + and stores the target information in the build environment. + + If a script references a target that's already found in the build + environment the results of the previous built are re-used. + + """ + app_data = env.domaindata["broxygen"] + + if (type, pattern) in app_data["targets"]: + info("Broxygen has cached doc for target '{0} {1}'".format( + type, pattern)) + return app_data["targets"] + + (cfg, gend_file) = generate_config(env, type, pattern) + target = BroxygenTarget(type, pattern, cfg, gend_file) + app_data["targets"][(type, pattern)] = target + build_target(env, target) + info("Broxygen built target '{0} {1}'".format(type, pattern)) + return target + + +def build_target(env, target): + """Invoke a Bro process to build a Broxygen target.""" + import os + import subprocess + + bro_cmd = "bro -X {0} broxygen".format(target.config_file) + cwd = os.getcwd() + os.chdir(os.path.dirname(target.config_file)) + + try: + subprocess.check_output(bro_cmd, stderr=subprocess.STDOUT, shell=True) + except subprocess.CalledProcessError as e: + from sphinx.errors import SphinxError + raise SphinxError( + "Command '{0}' returned non-zero exit status {1}: {2}".format( + e.cmd, e.returncode, e.output)) + finally: + os.chdir(cwd) + + +class BroxygenTarget(object): + + """Some portion of reST documentation that Bro knows how to generate. + + A target is identified by its type and pattern. E.g. type "script" and + pattern "broxygen/example.bro". + + """ + + def __init__(self, type, pattern, config_file, generated_file): + self.type = type + self.pattern = pattern + self.config_file = config_file + self.generated_file = generated_file + self.used_in_docs = set() + + +class BroxygenDirective(Include): + + """Base class for Broxygen directives. + + It can use Bro to generate reST documentation on the fly and embed it in + the document at the location of the directive just like the ``.. include::`` + directive. The only argument is a pattern to identify to Bro which + pieces of documentation it needs to create. + """ + + required_arguments = 1 + has_content = False + + target_type = None + + def run(self): + env = self.state.document.settings.env + info("Broxygen running .. {0}:: {1} in {2}".format( + self.name, self.arguments[0], env.docname)) + target = generate_target(env, self.target_type, self.arguments[0]) + target.used_in_docs.add(env.docname) + self.arguments = [target.generated_file] + return super(BroxygenDirective, self).run() + + +class PackageDirective(BroxygenDirective): + + target_type = "package" + + +class PackageIndexDirective(BroxygenDirective): + + target_type = "package_index" + + +class ScriptDirective(BroxygenDirective): + + target_type = "script" + + +class ScriptSummaryDirective(BroxygenDirective): + + target_type = "script_summary" + + +class ScriptIndexDirective(BroxygenDirective): + + target_type = "script_index" + + +class ProtoAnalyzerDirective(BroxygenDirective): + + target_type = "proto_analyzer" + + +class FileAnalyzerDirective(BroxygenDirective): + + target_type = "file_analyzer" + + +class IdentifierDirective(BroxygenDirective): + + target_type = "identifier" + + +class BroxygenDomain(Domain): + + name = "broxygen" + label = "Broxygen" + + object_types = { + "package": ObjType(l_("package")), + "package_index": ObjType(l_("package_index")), + "script": ObjType(l_("script")), + "script_summary": ObjType(l_("script_summary")), + "script_index": ObjType(l_("script_index")), + "proto_analyzer": ObjType(l_("proto_analyzer")), + "file_analyzer": ObjType(l_("file_analyzer")), + "identifier": ObjType(l_("identifier")), + } + + directives = { + "package": PackageDirective, + "package_index": PackageIndexDirective, + "script": ScriptDirective, + "script_summary": ScriptSummaryDirective, + "script_index": ScriptIndexDirective, + "proto_analyzer": ProtoAnalyzerDirective, + "file_analyzer": FileAnalyzerDirective, + "identifier": IdentifierDirective, + } + + roles = {} + + initial_data = { + "targets": {} + } + + def clear_doc(self, docname): + """Update Broxygen targets referenced in docname. + + If it's the last place the target was referenced, remove it from + the build environment and delete any generated config/reST files + associated with it from the cache. + + """ + import os + + stale_targets = [] + + for (type, pattern), target in self.data["targets"].items(): + if docname in target.used_in_docs: + target.used_in_docs.remove(docname) + + if not target.used_in_docs: + stale_targets.append(target) + + for target in stale_targets: + del self.data["targets"][(target.type, target.pattern)] + os.remove(target.config_file) + os.remove(target.generated_file) + + def get_objects(self): + """No Broxygen-generated content is itself linkable/searchable.""" + return [] + + +def env_get_outdated_hook(app, env, added, changed, removed): + """Check whether to re-read any documents referencing Broxygen targets. + + To do that we have to ask Bro to rebuild each target and compare the + before and after modification times of the generated reST output file. + If Bro changed it, then the document containing the Broxygen directive + needs to be re-read. + + """ + import os + + reread = set() + + for target in app.env.domaindata["broxygen"]["targets"].values(): + before_mtime = os.stat(target.generated_file) + build_target(env, target) + after_mtime = os.stat(target.generated_file) + + if after_mtime > before_mtime: + info("Broxygen target '{0} {1}' outdated".format( + target.type, target.pattern)) + + for docname in target.used_in_docs: + if docname not in removed: + info(" in document: {0}".format(docname)) + reread.add(docname) + + return list(reread) + + +def setup(app): + global App + App = app + app.add_domain(BroxygenDomain) + app.add_config_value("broxygen_cache", None, "env") + app.connect("env-get-outdated", env_get_outdated_hook) diff --git a/doc/frameworks/logging-dataseries.rst b/doc/frameworks/logging-dataseries.rst index 667edcbe14..cc479eae76 100644 --- a/doc/frameworks/logging-dataseries.rst +++ b/doc/frameworks/logging-dataseries.rst @@ -104,7 +104,7 @@ code like this to your ``local.bro``: } Bro's DataSeries writer comes with a few tuning options, see -:doc:`/scripts/base/frameworks/logging/writers/dataseries`. +:doc:`/scripts/base/frameworks/logging/writers/dataseries.bro`. Working with DataSeries ======================= diff --git a/doc/frameworks/logging.rst b/doc/frameworks/logging.rst index eb64df4ec3..47d3338e8a 100644 --- a/doc/frameworks/logging.rst +++ b/doc/frameworks/logging.rst @@ -48,7 +48,7 @@ Basics The data fields that a stream records are defined by a record type specified when it is created. Let's look at the script generating Bro's connection summaries as an example, -:doc:`/scripts/base/protocols/conn/main`. It defines a record +:doc:`/scripts/base/protocols/conn/main.bro`. It defines a record :bro:type:`Conn::Info` that lists all the fields that go into ``conn.log``, each marked with a ``&log`` attribute indicating that it is part of the information written out. To write a log record, the @@ -309,7 +309,7 @@ ASCII Writer Configuration -------------------------- The ASCII writer has a number of options for customizing the format of -its output, see :doc:`/scripts/base/frameworks/logging/writers/ascii`. +its output, see :doc:`/scripts/base/frameworks/logging/writers/ascii.bro`. Adding Streams ============== @@ -369,7 +369,7 @@ save the logged ``Foo::Info`` record into the connection record: } See the existing scripts for how to work with such a new connection -field. A simple example is :doc:`/scripts/base/protocols/syslog/main`. +field. A simple example is :doc:`/scripts/base/protocols/syslog/main.bro`. When you are developing scripts that add data to the :bro:type:`connection` record, care must be given to when and how long data is stored. diff --git a/doc/frameworks/notice.rst b/doc/frameworks/notice.rst index 1f0cc9da11..fe94d7f2ca 100644 --- a/doc/frameworks/notice.rst +++ b/doc/frameworks/notice.rst @@ -283,7 +283,7 @@ information to suppress duplicates for a configurable period of time. The ``$identifier`` field is typically comprised of several pieces of data related to the notice that when combined represent a unique instance of that notice. Here is an example of the script -:doc:`/scripts/policy/protocols/ssl/validate-certs` raising a notice +:doc:`/scripts/policy/protocols/ssl/validate-certs.bro` raising a notice for session negotiations where the certificate or certificate chain did not validate successfully against the available certificate authority certificates. diff --git a/doc/frameworks/signatures.rst b/doc/frameworks/signatures.rst index 915133e178..884dcb8a47 100644 --- a/doc/frameworks/signatures.rst +++ b/doc/frameworks/signatures.rst @@ -46,7 +46,7 @@ signature's event statement (``Found root!``), and data is the last piece of payload which triggered the pattern match. To turn such :bro:id:`signature_match` events into actual alarms, you can -load Bro's :doc:`/scripts/base/frameworks/signatures/main` script. +load Bro's :doc:`/scripts/base/frameworks/signatures/main.bro` script. This script contains a default event handler that raises :bro:enum:`Signatures::Sensitive_Signature` :doc:`Notices ` (as well as others; see the beginning of the script). diff --git a/doc/index.rst b/doc/index.rst index b79f1b67a6..34096694b3 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -15,7 +15,7 @@ Bro Documentation scripting/index.rst frameworks/index.rst cluster/index.rst - scripts/index.rst + script-reference/index.rst components/index.rst * :ref:`General Index ` diff --git a/doc/quickstart/index.rst b/doc/quickstart/index.rst index f570ac92db..df66eb36e8 100644 --- a/doc/quickstart/index.rst +++ b/doc/quickstart/index.rst @@ -197,7 +197,7 @@ the variable's value may not change at run-time, but whose initial value can be modified via the ``redef`` operator at parse-time. So let's continue on our path to modify the behavior for the two SSL -and SSH notices. Looking at :doc:`/scripts/base/frameworks/notice/main`, +and SSH notices. Looking at :doc:`/scripts/base/frameworks/notice/main.bro`, we see that it advertises: .. code:: bro diff --git a/doc/scripts/builtins.rst b/doc/script-reference/builtins.rst similarity index 100% rename from doc/scripts/builtins.rst rename to doc/script-reference/builtins.rst diff --git a/doc/script-reference/file-analyzers.rst b/doc/script-reference/file-analyzers.rst new file mode 100644 index 0000000000..81e71969a4 --- /dev/null +++ b/doc/script-reference/file-analyzers.rst @@ -0,0 +1 @@ +.. broxygen:file_analyzer:: * diff --git a/doc/scripts/index.rst b/doc/script-reference/index.rst similarity index 50% rename from doc/scripts/index.rst rename to doc/script-reference/index.rst index 5a19bd23f0..ccb374bd82 100644 --- a/doc/scripts/index.rst +++ b/doc/script-reference/index.rst @@ -1,5 +1,3 @@ -.. This is a stub doc to which broxygen appends during the build process - ================ Script Reference ================ @@ -7,15 +5,9 @@ Script Reference .. toctree:: :maxdepth: 1 - packages proto-analyzers file-analyzers - notices builtins - Built-in Functions (BIFs) - internal + packages scripts - - - - + Broxygen Example Script diff --git a/doc/scripts/packages.rst b/doc/script-reference/packages.rst similarity index 81% rename from doc/scripts/packages.rst rename to doc/script-reference/packages.rst index 9f01a91e91..57fde4834e 100644 --- a/doc/scripts/packages.rst +++ b/doc/script-reference/packages.rst @@ -1,9 +1,7 @@ -.. This is a stub doc to which broxygen appends during the build process - .. _script-packages: -Bro Script Packages -=================== +Bro Package Index +================= Bro has the following script packages (e.g. collections of related scripts in a common directory). If the package directory contains a ``__load__.bro`` @@ -12,3 +10,5 @@ script, it supports being loaded in mass as a whole directory for convenience. Packages/scripts in the ``base/`` directory are all loaded by default, while ones in ``policy/`` provide functionality and customization options that are more appropriate for users to decide whether they'd like to load it or not. + +.. broxygen:package_index:: * diff --git a/doc/script-reference/proto-analyzers.rst b/doc/script-reference/proto-analyzers.rst new file mode 100644 index 0000000000..85c6ec1f12 --- /dev/null +++ b/doc/script-reference/proto-analyzers.rst @@ -0,0 +1 @@ +.. broxygen:proto_analyzer:: * diff --git a/doc/script-reference/scripts.rst b/doc/script-reference/scripts.rst new file mode 100644 index 0000000000..90bbc5543d --- /dev/null +++ b/doc/script-reference/scripts.rst @@ -0,0 +1,5 @@ +================ +Bro Script Index +================ + +.. broxygen:script_index:: * diff --git a/doc/scripting/index.rst b/doc/scripting/index.rst index c6b479a7af..2118e73715 100644 --- a/doc/scripting/index.rst +++ b/doc/scripting/index.rst @@ -316,7 +316,7 @@ block that variable is available to any other script through the naming convention of ``MODULE::variable_name``. The declaration below is taken from the -:doc:`/scripts/policy/protocols/conn/known-hosts` script and +:doc:`/scripts/policy/protocols/conn/known-hosts.bro` script and declares a variable called ``known_hosts`` as a global set of unique IP addresses within the ``Known`` namespace and exports it for use outside of the ``Known`` namespace. Were we to want to use the @@ -348,7 +348,7 @@ constants are used in Bro scripts as containers for configuration options. For example, the configuration option to log password decrypted from HTTP streams is stored in ``HTTP::default_capture_password`` as shown in the stripped down -excerpt from :doc:`/scripts/base/protocols/http/main` below. +excerpt from :doc:`/scripts/base/protocols/http/main.bro` below. .. btest-include:: ${BRO_SRC_ROOT}/scripts/base/protocols/http/main.bro :lines: 8-10,19-21,120 @@ -1182,7 +1182,7 @@ passing in the ``Notice::Info`` record. The simplest kind of action based on the answer. The hook below adds the :bro:enum:`Notice::ACTION_EMAIL` action for the ``SSH::Interesting_Hostname_Login`` notice raised in the -:doc:`/scripts/policy/protocols/ssh/interesting-hostnames` script. +:doc:`/scripts/policy/protocols/ssh/interesting-hostnames.bro` script. .. btest-include:: ${DOC_ROOT}/scripting/framework_notice_hook_01.bro @@ -1224,7 +1224,7 @@ Bro. .. btest-include:: ${BRO_SRC_ROOT}/scripts/policy/protocols/ssl/expiring-certs.bro :lines: 60-63 -In the :doc:`/scripts/policy/protocols/ssl/expiring-certs` script +In the :doc:`/scripts/policy/protocols/ssl/expiring-certs.bro` script which identifies when SSL certificates are set to expire and raises notices when it crosses a predefined threshold, the call to ``NOTICE`` above also sets the ``$identifier`` entry by concatenating diff --git a/doc/scripts/CMakeLists.txt b/doc/scripts/CMakeLists.txt deleted file mode 100644 index fa234e74f2..0000000000 --- a/doc/scripts/CMakeLists.txt +++ /dev/null @@ -1,226 +0,0 @@ -# find out what BROPATH to use when executing bro -execute_process(COMMAND ${CMAKE_BINARY_DIR}/bro-path-dev - OUTPUT_VARIABLE BROPATH - RESULT_VARIABLE retval - OUTPUT_STRIP_TRAILING_WHITESPACE) -if (NOT ${retval} EQUAL 0) - message(FATAL_ERROR "Problem setting BROPATH") -endif () - -# This macro is used to add a new makefile target for reST policy script -# documentation that can be generated using Bro itself to parse policy scripts. -# It's called like: -# -# rest_target(srcDir broInput [group]) -# -# srcDir: the directory which contains broInput -# broInput: the file name of a bro policy script, any path prefix of this -# argument will be used to derive what path under scripts/ the generated -# documentation will be placed. -# group: optional name of group that the script documentation will belong to. -# If this is not given, the group is automatically set to any path portion -# of the broInput argument. -# -# In addition to adding the makefile target, several CMake variables are set: -# -# MASTER_POLICY_INDEX_TEXT: a running list of policy scripts docs that have -# been generated so far, formatted such that it can be appended to a file -# that ends in a Sphinx toctree directive -# ALL_REST_OUTPUTS: a running list (the CMake list type) of all reST docs -# that are to be generated -# MASTER_GROUP_LIST: a running list (the CMake list type) of all script groups -# MASTER_PKG_LIST: a running list (the CMake list type) of all script groups -# that were defived from the path portion of the broInput argument -# ${group}_files: a running list of files belonging to a given group, from -# which summary text can be extracted at build time -# ${group}_doc_names: a running list of reST style document names that can be -# given to a :doc: role, shared indices with ${group}_files - -macro(REST_TARGET srcDir broInput) - set(absSrcPath ${srcDir}/${broInput}) - get_filename_component(basename ${broInput} NAME) - string(REPLACE .bro "" basename ${basename}) - get_filename_component(extension ${broInput} EXT) - get_filename_component(relDstDir ${broInput} PATH) - - set(sumTextSrc ${absSrcPath}) - set(ogSourceFile ${absSrcPath}) - - if (NOT relDstDir) - set(docName "${basename}") - set(dstDir "${RST_OUTPUT_DIR}") - else () - set(docName "${relDstDir}/${basename}") - set(dstDir "${RST_OUTPUT_DIR}/${relDstDir}") - endif () - - set(restFile "${docName}.rst") - string(REPLACE "/" "^" restFile ${restFile}) - set(restOutput "${dstDir}/${basename}.rst") - - set(MASTER_POLICY_INDEX_TEXT - "${MASTER_POLICY_INDEX_TEXT}\n ${docName} <${docName}>") - list(APPEND ALL_REST_OUTPUTS ${restOutput}) - - if (NOT "${ARGN}" STREQUAL "") - set(group ${ARGN}) - elseif (relDstDir) - set(group ${relDstDir}/index) - # add package index to master package list if not already in it - # and if a __load__.bro exists in the original script directory - list(FIND MASTER_PKG_LIST ${relDstDir} _found) - if (_found EQUAL -1) - if (EXISTS ${CMAKE_SOURCE_DIR}/scripts/${relDstDir}/__load__.bro) - list(APPEND MASTER_PKG_LIST ${relDstDir}) - endif () - endif () - else () - set(group "") - endif () - - if (NOT "${group}" STREQUAL "") - # add group to master group list if not already in it - list(FIND MASTER_GROUP_LIST ${group} _found) - if (_found EQUAL -1) - list(APPEND MASTER_GROUP_LIST ${group}) - if (MASTER_GROUP_LIST_TEXT) - set(MASTER_GROUP_LIST_TEXT "${MASTER_GROUP_LIST_TEXT}\n${group}") - else () - set(MASTER_GROUP_LIST_TEXT "${group}") - endif () - endif () - - list(APPEND ${group}_files ${sumTextSrc}) - list(APPEND ${group}_doc_names ${docName}) - endif () - - add_custom_command(OUTPUT ${restOutput} - # delete any leftover state from previous bro runs - COMMAND "${CMAKE_COMMAND}" - ARGS -E remove_directory .state - # generate the reST documentation using bro - COMMAND BROPATH=${BROPATH}:${srcDir} BROMAGIC=${CMAKE_SOURCE_DIR}/magic/database ${CMAKE_BINARY_DIR}/src/bro - ARGS -b -Z ${broInput} || (rm -rf .state *.log *.rst && exit 1) - # move generated doc into a new directory tree that - # defines the final structure of documents - COMMAND "${CMAKE_COMMAND}" - ARGS -E make_directory ${dstDir} - COMMAND "${CMAKE_COMMAND}" - ARGS -E copy ${restFile} ${restOutput} - # copy the bro or bif script, too - COMMAND "${CMAKE_COMMAND}" - ARGS -E copy ${ogSourceFile} ${dstDir} - # clean up the build directory - COMMAND rm - ARGS -rf .state *.log *.rst - DEPENDS bro - DEPENDS ${absSrcPath} - WORKING_DIRECTORY ${CMAKE_BINARY_DIR} - COMMENT "[Bro] Generating reST docs for ${broInput}" - ) - -endmacro(REST_TARGET) - -# Schedule Bro scripts for which to generate documentation. -include(DocSourcesList.cmake) - -# Macro for generating reST docs that are independent of any particular Bro -# script. -macro(INDEPENDENT_REST_TARGET reST_file) - add_custom_command(OUTPUT ${reST_file} - # delete any leftover state from previous bro runs - COMMAND "${CMAKE_COMMAND}" - ARGS -E remove_directory .state - # generate the reST documentation using bro - COMMAND BROPATH=${BROPATH}:${srcDir} BROMAGIC=${CMAKE_SOURCE_DIR}/magic/database ${CMAKE_BINARY_DIR}/src/bro - ARGS -b -Z base/init-bare.bro || (rm -rf .state *.log *.rst && exit 1) - # move generated doc into a new directory tree that - # defines the final structure of documents - COMMAND "${CMAKE_COMMAND}" - ARGS -E make_directory ${dstDir} - COMMAND "${CMAKE_COMMAND}" - ARGS -E copy ${reST_file} ${dstDir} - # clean up the build directory - COMMAND rm - ARGS -rf .state *.log *.rst - DEPENDS bro - WORKING_DIRECTORY ${CMAKE_BINARY_DIR} - COMMENT "[Bro] Generating reST docs for ${reST_file}" - ) - list(APPEND ALL_REST_OUTPUTS ${reST_file}) -endmacro(INDEPENDENT_REST_TARGET) - -independent_rest_target(proto-analyzers.rst) -independent_rest_target(file-analyzers.rst) - -# create temporary list of all docs to include in the master policy/index file -file(WRITE ${MASTER_POLICY_INDEX} "${MASTER_POLICY_INDEX_TEXT}") - -# create the temporary list of all packages to include in the master -# policy/packages.rst file -set(MASTER_PKG_INDEX_TEXT "") -foreach (pkg ${MASTER_PKG_LIST}) - set(MASTER_PKG_INDEX_TEXT - "${MASTER_PKG_INDEX_TEXT}\n:doc:`${pkg} <${pkg}/index>`\n") - if (EXISTS ${CMAKE_SOURCE_DIR}/scripts/${pkg}/README) - file(STRINGS ${CMAKE_SOURCE_DIR}/scripts/${pkg}/README pkgreadme) - foreach (line ${pkgreadme}) - set(MASTER_PKG_INDEX_TEXT "${MASTER_PKG_INDEX_TEXT}\n ${line}") - endforeach () - set(MASTER_PKG_INDEX_TEXT "${MASTER_PKG_INDEX_TEXT}\n") - endif () -endforeach () -file(WRITE ${MASTER_PACKAGE_INDEX} "${MASTER_PKG_INDEX_TEXT}") - -# create temporary file containing list of all groups -file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/group_list - "${MASTER_GROUP_LIST_TEXT}") - -# create temporary files containing list of each source file in a given group -foreach (group ${MASTER_GROUP_LIST}) - if (EXISTS ${CMAKE_CURRENT_BINARY_DIR}/${group}_files) - file(REMOVE ${CMAKE_CURRENT_BINARY_DIR}/${group}_files) - endif () - if (EXISTS ${CMAKE_CURRENT_BINARY_DIR}/${group}_doc_names) - file(REMOVE ${CMAKE_CURRENT_BINARY_DIR}/${group}_doc_names) - endif () - foreach (src ${${group}_files}) - file(APPEND ${CMAKE_CURRENT_BINARY_DIR}/${group}_files "${src}\n") - endforeach () - foreach (dname ${${group}_doc_names}) - file(APPEND ${CMAKE_CURRENT_BINARY_DIR}/${group}_doc_names "${dname}\n") - endforeach () -endforeach () - -# remove previously generated docs no longer scheduled for generation -if (EXISTS ${RST_OUTPUT_DIR}) - file(GLOB_RECURSE EXISTING_REST_DOCS "${RST_OUTPUT_DIR}/*.rst") - foreach (_doc ${EXISTING_REST_DOCS}) - list(FIND ALL_REST_OUTPUTS ${_doc} _found) - if (_found EQUAL -1) - file(REMOVE ${_doc}) - message(STATUS "Broxygen: remove stale reST doc: ${_doc}") - string(REPLACE .rst .bro _brofile ${_doc}) - if (EXISTS ${_brofile}) - file(REMOVE ${_brofile}) - message(STATUS "Broxygen: remove stale bro source: ${_brofile}") - endif () - endif () - endforeach () -endif () - -# The "restdoc" target uses Bro to parse policy scripts in order to -# generate reST documentation from them. -add_custom_target(restdoc - # create symlink to the reST output directory for convenience - COMMAND "${CMAKE_COMMAND}" -E create_symlink - ${RST_OUTPUT_DIR} - ${CMAKE_BINARY_DIR}/reST - DEPENDS ${ALL_REST_OUTPUTS}) - -# The "restclean" target removes all generated reST documentation from the -# build directory. -add_custom_target(restclean - COMMAND "${CMAKE_COMMAND}" -E remove_directory - ${RST_OUTPUT_DIR} - VERBATIM) diff --git a/doc/scripts/DocSourcesList.cmake b/doc/scripts/DocSourcesList.cmake deleted file mode 100644 index 61682d73b3..0000000000 --- a/doc/scripts/DocSourcesList.cmake +++ /dev/null @@ -1,282 +0,0 @@ -# DO NOT EDIT -# This file is auto-generated from the genDocSourcesList.sh script. -# -# This is a list of Bro script sources for which to generate reST documentation. -# It will be included inline in the CMakeLists.txt found in the same directory -# in order to create Makefile targets that define how to generate reST from -# a given Bro script. -# -# Note: any path prefix of the script (2nd argument of rest_target macro) -# will be used to derive what path under scripts/ the generated documentation -# will be placed. - -set(psd ${PROJECT_SOURCE_DIR}/scripts) - -rest_target(${CMAKE_CURRENT_SOURCE_DIR} example.bro internal) -rest_target(${psd} base/init-default.bro internal) -rest_target(${psd} base/init-bare.bro internal) - -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/analyzer.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/bloom-filter.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/bro.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/broxygen.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/cardinality-counter.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/const.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/event.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/file_analysis.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/input.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/logging.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_ARP.events.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_AYIYA.events.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_BackDoor.events.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_BitTorrent.events.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_ConnSize.events.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_DCE_RPC.events.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_DHCP.events.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_DNP3.events.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_DNS.events.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_FTP.events.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_FTP.functions.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_File.events.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_FileExtract.events.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_FileExtract.functions.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_FileHash.events.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_Finger.events.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_GTPv1.events.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_Gnutella.events.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_HTTP.events.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_HTTP.functions.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_ICMP.events.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_IRC.events.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_Ident.events.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_InterConn.events.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_Login.events.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_Login.functions.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_MIME.events.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_Modbus.events.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_NCP.events.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_NTP.events.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_NetBIOS.events.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_NetBIOS.functions.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_NetFlow.events.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_PIA.events.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_POP3.events.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_RPC.events.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_SMB.events.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_SMTP.events.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_SMTP.functions.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_SOCKS.events.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_SSH.events.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_SSL.events.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_SSL.functions.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_SteppingStone.events.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_Syslog.events.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_TCP.events.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_TCP.functions.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_Teredo.events.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_UDP.events.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_Unified2.events.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_Unified2.types.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_ZIP.events.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/reporter.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/strings.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/top-k.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/types.bif.bro) -rest_target(${psd} base/files/extract/main.bro) -rest_target(${psd} base/files/hash/main.bro) -rest_target(${psd} base/files/unified2/main.bro) -rest_target(${psd} base/frameworks/analyzer/main.bro) -rest_target(${psd} base/frameworks/cluster/main.bro) -rest_target(${psd} base/frameworks/cluster/nodes/manager.bro) -rest_target(${psd} base/frameworks/cluster/nodes/proxy.bro) -rest_target(${psd} base/frameworks/cluster/nodes/worker.bro) -rest_target(${psd} base/frameworks/cluster/setup-connections.bro) -rest_target(${psd} base/frameworks/communication/main.bro) -rest_target(${psd} base/frameworks/control/main.bro) -rest_target(${psd} base/frameworks/dpd/main.bro) -rest_target(${psd} base/frameworks/files/main.bro) -rest_target(${psd} base/frameworks/input/main.bro) -rest_target(${psd} base/frameworks/input/readers/ascii.bro) -rest_target(${psd} base/frameworks/input/readers/benchmark.bro) -rest_target(${psd} base/frameworks/input/readers/binary.bro) -rest_target(${psd} base/frameworks/input/readers/raw.bro) -rest_target(${psd} base/frameworks/input/readers/sqlite.bro) -rest_target(${psd} base/frameworks/intel/cluster.bro) -rest_target(${psd} base/frameworks/intel/input.bro) -rest_target(${psd} base/frameworks/intel/main.bro) -rest_target(${psd} base/frameworks/logging/main.bro) -rest_target(${psd} base/frameworks/logging/postprocessors/scp.bro) -rest_target(${psd} base/frameworks/logging/postprocessors/sftp.bro) -rest_target(${psd} base/frameworks/logging/writers/ascii.bro) -rest_target(${psd} base/frameworks/logging/writers/dataseries.bro) -rest_target(${psd} base/frameworks/logging/writers/elasticsearch.bro) -rest_target(${psd} base/frameworks/logging/writers/none.bro) -rest_target(${psd} base/frameworks/logging/writers/sqlite.bro) -rest_target(${psd} base/frameworks/notice/actions/add-geodata.bro) -rest_target(${psd} base/frameworks/notice/actions/drop.bro) -rest_target(${psd} base/frameworks/notice/actions/email_admin.bro) -rest_target(${psd} base/frameworks/notice/actions/page.bro) -rest_target(${psd} base/frameworks/notice/actions/pp-alarms.bro) -rest_target(${psd} base/frameworks/notice/cluster.bro) -rest_target(${psd} base/frameworks/notice/extend-email/hostnames.bro) -rest_target(${psd} base/frameworks/notice/main.bro) -rest_target(${psd} base/frameworks/notice/non-cluster.bro) -rest_target(${psd} base/frameworks/notice/weird.bro) -rest_target(${psd} base/frameworks/packet-filter/cluster.bro) -rest_target(${psd} base/frameworks/packet-filter/main.bro) -rest_target(${psd} base/frameworks/packet-filter/netstats.bro) -rest_target(${psd} base/frameworks/packet-filter/utils.bro) -rest_target(${psd} base/frameworks/reporter/main.bro) -rest_target(${psd} base/frameworks/signatures/main.bro) -rest_target(${psd} base/frameworks/software/main.bro) -rest_target(${psd} base/frameworks/sumstats/cluster.bro) -rest_target(${psd} base/frameworks/sumstats/main.bro) -rest_target(${psd} base/frameworks/sumstats/non-cluster.bro) -rest_target(${psd} base/frameworks/sumstats/plugins/average.bro) -rest_target(${psd} base/frameworks/sumstats/plugins/hll_unique.bro) -rest_target(${psd} base/frameworks/sumstats/plugins/last.bro) -rest_target(${psd} base/frameworks/sumstats/plugins/max.bro) -rest_target(${psd} base/frameworks/sumstats/plugins/min.bro) -rest_target(${psd} base/frameworks/sumstats/plugins/sample.bro) -rest_target(${psd} base/frameworks/sumstats/plugins/std-dev.bro) -rest_target(${psd} base/frameworks/sumstats/plugins/sum.bro) -rest_target(${psd} base/frameworks/sumstats/plugins/topk.bro) -rest_target(${psd} base/frameworks/sumstats/plugins/unique.bro) -rest_target(${psd} base/frameworks/sumstats/plugins/variance.bro) -rest_target(${psd} base/frameworks/tunnels/main.bro) -rest_target(${psd} base/misc/find-checksum-offloading.bro) -rest_target(${psd} base/protocols/conn/contents.bro) -rest_target(${psd} base/protocols/conn/inactivity.bro) -rest_target(${psd} base/protocols/conn/main.bro) -rest_target(${psd} base/protocols/conn/polling.bro) -rest_target(${psd} base/protocols/dhcp/consts.bro) -rest_target(${psd} base/protocols/dhcp/main.bro) -rest_target(${psd} base/protocols/dhcp/utils.bro) -rest_target(${psd} base/protocols/dnp3/consts.bro) -rest_target(${psd} base/protocols/dnp3/main.bro) -rest_target(${psd} base/protocols/dns/consts.bro) -rest_target(${psd} base/protocols/dns/main.bro) -rest_target(${psd} base/protocols/ftp/files.bro) -rest_target(${psd} base/protocols/ftp/gridftp.bro) -rest_target(${psd} base/protocols/ftp/info.bro) -rest_target(${psd} base/protocols/ftp/main.bro) -rest_target(${psd} base/protocols/ftp/utils-commands.bro) -rest_target(${psd} base/protocols/ftp/utils.bro) -rest_target(${psd} base/protocols/http/entities.bro) -rest_target(${psd} base/protocols/http/files.bro) -rest_target(${psd} base/protocols/http/main.bro) -rest_target(${psd} base/protocols/http/utils.bro) -rest_target(${psd} base/protocols/irc/dcc-send.bro) -rest_target(${psd} base/protocols/irc/files.bro) -rest_target(${psd} base/protocols/irc/main.bro) -rest_target(${psd} base/protocols/modbus/consts.bro) -rest_target(${psd} base/protocols/modbus/main.bro) -rest_target(${psd} base/protocols/smtp/entities.bro) -rest_target(${psd} base/protocols/smtp/files.bro) -rest_target(${psd} base/protocols/smtp/main.bro) -rest_target(${psd} base/protocols/socks/consts.bro) -rest_target(${psd} base/protocols/socks/main.bro) -rest_target(${psd} base/protocols/ssh/main.bro) -rest_target(${psd} base/protocols/ssl/consts.bro) -rest_target(${psd} base/protocols/ssl/main.bro) -rest_target(${psd} base/protocols/ssl/mozilla-ca-list.bro) -rest_target(${psd} base/protocols/syslog/consts.bro) -rest_target(${psd} base/protocols/syslog/main.bro) -rest_target(${psd} base/utils/active-http.bro) -rest_target(${psd} base/utils/addrs.bro) -rest_target(${psd} base/utils/conn-ids.bro) -rest_target(${psd} base/utils/dir.bro) -rest_target(${psd} base/utils/directions-and-hosts.bro) -rest_target(${psd} base/utils/exec.bro) -rest_target(${psd} base/utils/files.bro) -rest_target(${psd} base/utils/numbers.bro) -rest_target(${psd} base/utils/paths.bro) -rest_target(${psd} base/utils/patterns.bro) -rest_target(${psd} base/utils/queue.bro) -rest_target(${psd} base/utils/site.bro) -rest_target(${psd} base/utils/strings.bro) -rest_target(${psd} base/utils/thresholds.bro) -rest_target(${psd} base/utils/time.bro) -rest_target(${psd} base/utils/urls.bro) -rest_target(${psd} policy/frameworks/communication/listen.bro) -rest_target(${psd} policy/frameworks/control/controllee.bro) -rest_target(${psd} policy/frameworks/control/controller.bro) -rest_target(${psd} policy/frameworks/dpd/detect-protocols.bro) -rest_target(${psd} policy/frameworks/dpd/packet-segment-logging.bro) -rest_target(${psd} policy/frameworks/files/detect-MHR.bro) -rest_target(${psd} policy/frameworks/files/hash-all-files.bro) -rest_target(${psd} policy/frameworks/intel/do_notice.bro) -rest_target(${psd} policy/frameworks/intel/seen/conn-established.bro) -rest_target(${psd} policy/frameworks/intel/seen/dns.bro) -rest_target(${psd} policy/frameworks/intel/seen/file-hashes.bro) -rest_target(${psd} policy/frameworks/intel/seen/file-names.bro) -rest_target(${psd} policy/frameworks/intel/seen/http-headers.bro) -rest_target(${psd} policy/frameworks/intel/seen/http-url.bro) -rest_target(${psd} policy/frameworks/intel/seen/smtp-url-extraction.bro) -rest_target(${psd} policy/frameworks/intel/seen/smtp.bro) -rest_target(${psd} policy/frameworks/intel/seen/ssl.bro) -rest_target(${psd} policy/frameworks/intel/seen/where-locations.bro) -rest_target(${psd} policy/frameworks/packet-filter/shunt.bro) -rest_target(${psd} policy/frameworks/software/version-changes.bro) -rest_target(${psd} policy/frameworks/software/vulnerable.bro) -rest_target(${psd} policy/integration/barnyard2/main.bro) -rest_target(${psd} policy/integration/barnyard2/types.bro) -rest_target(${psd} policy/integration/collective-intel/main.bro) -rest_target(${psd} policy/misc/app-stats/main.bro) -rest_target(${psd} policy/misc/app-stats/plugins/facebook.bro) -rest_target(${psd} policy/misc/app-stats/plugins/gmail.bro) -rest_target(${psd} policy/misc/app-stats/plugins/google.bro) -rest_target(${psd} policy/misc/app-stats/plugins/netflix.bro) -rest_target(${psd} policy/misc/app-stats/plugins/pandora.bro) -rest_target(${psd} policy/misc/app-stats/plugins/youtube.bro) -rest_target(${psd} policy/misc/capture-loss.bro) -rest_target(${psd} policy/misc/detect-traceroute/main.bro) -rest_target(${psd} policy/misc/known-devices.bro) -rest_target(${psd} policy/misc/load-balancing.bro) -rest_target(${psd} policy/misc/loaded-scripts.bro) -rest_target(${psd} policy/misc/profiling.bro) -rest_target(${psd} policy/misc/scan.bro) -rest_target(${psd} policy/misc/stats.bro) -rest_target(${psd} policy/misc/trim-trace-file.bro) -rest_target(${psd} policy/protocols/conn/known-hosts.bro) -rest_target(${psd} policy/protocols/conn/known-services.bro) -rest_target(${psd} policy/protocols/conn/weirds.bro) -rest_target(${psd} policy/protocols/dhcp/known-devices-and-hostnames.bro) -rest_target(${psd} policy/protocols/dns/auth-addl.bro) -rest_target(${psd} policy/protocols/dns/detect-external-names.bro) -rest_target(${psd} policy/protocols/ftp/detect-bruteforcing.bro) -rest_target(${psd} policy/protocols/ftp/detect.bro) -rest_target(${psd} policy/protocols/ftp/software.bro) -rest_target(${psd} policy/protocols/http/detect-sqli.bro) -rest_target(${psd} policy/protocols/http/detect-webapps.bro) -rest_target(${psd} policy/protocols/http/header-names.bro) -rest_target(${psd} policy/protocols/http/software-browser-plugins.bro) -rest_target(${psd} policy/protocols/http/software.bro) -rest_target(${psd} policy/protocols/http/var-extraction-cookies.bro) -rest_target(${psd} policy/protocols/http/var-extraction-uri.bro) -rest_target(${psd} policy/protocols/modbus/known-masters-slaves.bro) -rest_target(${psd} policy/protocols/modbus/track-memmap.bro) -rest_target(${psd} policy/protocols/smtp/blocklists.bro) -rest_target(${psd} policy/protocols/smtp/detect-suspicious-orig.bro) -rest_target(${psd} policy/protocols/smtp/entities-excerpt.bro) -rest_target(${psd} policy/protocols/smtp/software.bro) -rest_target(${psd} policy/protocols/ssh/detect-bruteforcing.bro) -rest_target(${psd} policy/protocols/ssh/geo-data.bro) -rest_target(${psd} policy/protocols/ssh/interesting-hostnames.bro) -rest_target(${psd} policy/protocols/ssh/software.bro) -rest_target(${psd} policy/protocols/ssl/cert-hash.bro) -rest_target(${psd} policy/protocols/ssl/expiring-certs.bro) -rest_target(${psd} policy/protocols/ssl/extract-certs-pem.bro) -rest_target(${psd} policy/protocols/ssl/known-certs.bro) -rest_target(${psd} policy/protocols/ssl/notary.bro) -rest_target(${psd} policy/protocols/ssl/validate-certs.bro) -rest_target(${psd} policy/tuning/defaults/extracted_file_limits.bro) -rest_target(${psd} policy/tuning/defaults/packet-fragments.bro) -rest_target(${psd} policy/tuning/defaults/warnings.bro) -rest_target(${psd} policy/tuning/logs-to-elasticsearch.bro) -rest_target(${psd} policy/tuning/track-all-assets.bro) -rest_target(${psd} site/local-manager.bro) -rest_target(${psd} site/local-proxy.bro) -rest_target(${psd} site/local-worker.bro) -rest_target(${psd} site/local.bro) -rest_target(${psd} test-all-policy.bro) diff --git a/doc/scripts/README b/doc/scripts/README deleted file mode 100644 index a15812609c..0000000000 --- a/doc/scripts/README +++ /dev/null @@ -1,44 +0,0 @@ -This directory contains scripts and templates that can be used to automate -the generation of Bro script documentation. Several build targets are defined -by CMake and available in the top-level Makefile: - -``restdoc`` - - This target uses Bro to parse policy scripts in order to generate - reStructuredText (reST) documentation from them. The list of scripts - for which to generate reST documentation is defined in the - ``CMakeLists.txt`` file in this directory. Script documentation is - rebuild automatically if the policy script from which it is derived - or the Bro binary becomes out of date - - The resulting output from this target can be found in the CMake - ``build/`` directory inside ``reST`` (a symlink to - ``doc/scripts/rest_output``). - -``restclean`` - - This target removes any reST documentation that has been generated so far. - -The ``genDocSourcesList.sh`` script can be run to automatically generate -``DocSourcesList.cmake``, which is the file CMake uses to define the list -of documentation targets. This script should be run after adding new -Bro script source files, and the changes commited to git. - -If a script shouldn't have documentation generated for it, there's also a -blacklist manifest that can be maintained in the ``genDocSourcesList.sh`` -script. - -The blacklist can also be used if you want to define a certain grouping for -the script's generated docs to belong to (as opposed to the automatic grouping -the happens for script packages/directories). To do that, add the -script's name to the blacklist, then append a ``rest_target()`` to the -``statictext`` variable where the first argument is the source directory -containing the policy script to document, the second argument is the file -name of the policy script, and the third argument is the path/name of a -pre-created reST document in the ``../`` source directory to which the -``make doc`` process can append script documentation references. This -pre-created reST document should also then be linked to from the TOC tree -in ``../index.rst``. - -See ``example.bro`` for an example of how to document a Bro script such that -``make doc`` will be able to produce reST/HTML documentation for it. diff --git a/doc/scripts/example.rst b/doc/scripts/example.rst deleted file mode 100644 index b76b9af59b..0000000000 --- a/doc/scripts/example.rst +++ /dev/null @@ -1,291 +0,0 @@ -.. Automatically generated. Do not edit. - -example.bro -=========== - -:download:`Original Source File ` - -Overview --------- -This is an example script that demonstrates how to document. Comments -of the form ``##!`` are for the script summary. The contents of -these comments are transferred directly into the auto-generated -`reStructuredText `_ -(reST) document's summary section. - -.. tip:: You can embed directives and roles within ``##``-stylized comments. - -:Imports: :doc:`policy/frameworks/software/vulnerable ` - -Summary -~~~~~~~ -Options -####### -============================================================================ ====================================== -:bro:id:`Example::an_option`: :bro:type:`set` :bro:attr:`&redef` add documentation for "an_option" here - -:bro:id:`Example::option_with_init`: :bro:type:`interval` :bro:attr:`&redef` -============================================================================ ====================================== - -State Variables -############### -=========================================================================== ======================================= -:bro:id:`Example::a_var`: :bro:type:`bool` put some documentation for "a_var" here - -:bro:id:`Example::var_with_attr`: :bro:type:`count` :bro:attr:`&persistent` - -:bro:id:`Example::var_without_explicit_type`: :bro:type:`string` -=========================================================================== ======================================= - -Types -##### -====================================================== ========================================================== -:bro:type:`Example::SimpleEnum`: :bro:type:`enum` documentation for "SimpleEnum" - goes here. - -:bro:type:`Example::SimpleRecord`: :bro:type:`record` general documentation for a type "SimpleRecord" - goes here. - -:bro:type:`Example::ComplexRecord`: :bro:type:`record` general documentation for a type "ComplexRecord" goes here - -:bro:type:`Example::Info`: :bro:type:`record` An example record to be used with a logging stream. -====================================================== ========================================================== - -Events -###### -================================================= ============================================================= -:bro:id:`Example::an_event`: :bro:type:`event` Summarize "an_event" here. - -:bro:id:`Example::log_example`: :bro:type:`event` This is a declaration of an example event that can be used in - logging streams and is raised once for each log entry. -================================================= ============================================================= - -Functions -######### -=============================================== ======================================= -:bro:id:`Example::a_function`: :bro:type:`func` Summarize purpose of "a_function" here. -=============================================== ======================================= - -Redefinitions -############# -===================================================== ======================================== -:bro:type:`Log::ID`: :bro:type:`enum` - -:bro:type:`Example::SimpleEnum`: :bro:type:`enum` document the "SimpleEnum" redef here - -:bro:type:`Example::SimpleRecord`: :bro:type:`record` document the record extension redef here -===================================================== ======================================== - -Namespaces -~~~~~~~~~~ -.. bro:namespace:: Example - -Notices -~~~~~~~ -:bro:type:`Notice::Type` - - :Type: :bro:type:`enum` - - .. bro:enum:: Example::Notice_One Notice::Type - - any number of this type of comment - will document "Notice_One" - - .. bro:enum:: Example::Notice_Two Notice::Type - - any number of this type of comment - will document "Notice_Two" - - .. bro:enum:: Example::Notice_Three Notice::Type - - .. bro:enum:: Example::Notice_Four Notice::Type - -Public Interface ----------------- -Options -~~~~~~~ -.. bro:id:: Example::an_option - - :Type: :bro:type:`set` [:bro:type:`addr`, :bro:type:`addr`, :bro:type:`string`] - :Attributes: :bro:attr:`&redef` - :Default: ``{}`` - - add documentation for "an_option" here - -.. bro:id:: Example::option_with_init - - :Type: :bro:type:`interval` - :Attributes: :bro:attr:`&redef` - :Default: ``10.0 msecs`` - -State Variables -~~~~~~~~~~~~~~~ -.. bro:id:: Example::a_var - - :Type: :bro:type:`bool` - - put some documentation for "a_var" here - -.. bro:id:: Example::var_with_attr - - :Type: :bro:type:`count` - :Attributes: :bro:attr:`&persistent` - -.. bro:id:: Example::var_without_explicit_type - - :Type: :bro:type:`string` - :Default: ``"this works"`` - -Types -~~~~~ -.. bro:type:: Example::SimpleEnum - - :Type: :bro:type:`enum` - - .. bro:enum:: Example::ONE Example::SimpleEnum - - and more specific info for "ONE" - can span multiple lines - - .. bro:enum:: Example::TWO Example::SimpleEnum - - or more info like this for "TWO" - can span multiple lines - - .. bro:enum:: Example::THREE Example::SimpleEnum - - documentation for "SimpleEnum" - goes here. - -.. bro:type:: Example::SimpleRecord - - :Type: :bro:type:`record` - - field1: :bro:type:`count` - counts something - - field2: :bro:type:`bool` - toggles something - - general documentation for a type "SimpleRecord" - goes here. - -.. bro:type:: Example::ComplexRecord - - :Type: :bro:type:`record` - - field1: :bro:type:`count` - counts something - - field2: :bro:type:`bool` - toggles something - - field3: :bro:type:`Example::SimpleRecord` - - msg: :bro:type:`string` :bro:attr:`&default` = ``"blah"`` :bro:attr:`&optional` - attributes are self-documenting - - general documentation for a type "ComplexRecord" goes here - -.. bro:type:: Example::Info - - :Type: :bro:type:`record` - - ts: :bro:type:`time` :bro:attr:`&log` - - uid: :bro:type:`string` :bro:attr:`&log` - - status: :bro:type:`count` :bro:attr:`&log` :bro:attr:`&optional` - - An example record to be used with a logging stream. - -Events -~~~~~~ -.. bro:id:: Example::an_event - - :Type: :bro:type:`event` (name: :bro:type:`string`) - - Summarize "an_event" here. - Give more details about "an_event" here. - - :param name: describe the argument here - -.. bro:id:: Example::log_example - - :Type: :bro:type:`event` (rec: :bro:type:`Example::Info`) - - This is a declaration of an example event that can be used in - logging streams and is raised once for each log entry. - -Functions -~~~~~~~~~ -.. bro:id:: Example::a_function - - :Type: :bro:type:`function` (tag: :bro:type:`string`, msg: :bro:type:`string`) : :bro:type:`string` - - Summarize purpose of "a_function" here. - Give more details about "a_function" here. - Separating the documentation of the params/return values with - empty comments is optional, but improves readability of script. - - - :param tag: function arguments can be described - like this - - :param msg: another param - - - :returns: describe the return type here - -Redefinitions -~~~~~~~~~~~~~ -:bro:type:`Log::ID` - - :Type: :bro:type:`enum` - - .. bro:enum:: Example::LOG Log::ID - -:bro:type:`Example::SimpleEnum` - - :Type: :bro:type:`enum` - - .. bro:enum:: Example::FOUR Example::SimpleEnum - - and some documentation for "FOUR" - - .. bro:enum:: Example::FIVE Example::SimpleEnum - - also "FIVE" for good measure - - document the "SimpleEnum" redef here - -:bro:type:`Example::SimpleRecord` - - :Type: :bro:type:`record` - - field_ext: :bro:type:`string` :bro:attr:`&optional` - document the extending field here - (or here) - - document the record extension redef here - -Port Analysis -------------- -:ref:`More Information ` - -SSL:: - - [ports={ - 443/tcp, - 562/tcp - }] - -Packet Filter -------------- -:ref:`More Information ` - -Filters added:: - - [ssl] = tcp port 443, - [nntps] = tcp port 562 - diff --git a/doc/scripts/genDocSourcesList.sh b/doc/scripts/genDocSourcesList.sh deleted file mode 100755 index 31905c68db..0000000000 --- a/doc/scripts/genDocSourcesList.sh +++ /dev/null @@ -1,86 +0,0 @@ -#!/usr/bin/env bash - -# ./genDocSourcesList.sh [output file] -# -# Run this script to a generate file that's used to tell CMake about all the -# possible scripts for which reST documentation can be created. -# -# The optional argument can be used to avoid overwriting the file CMake uses -# by default. -# -# Specific scripts can be blacklisted below when e.g. they currently aren't -# parseable or they just aren't meant to be documented. - -export LC_ALL=C # Make sorting stable. - -blacklist () - { - if [[ "$blacklist" == "" ]]; then - blacklist="$1" - else - blacklist="$blacklist|$1" - fi - } - -# files passed into this function are meant to be temporary workarounds -# because they're not finished or otherwise can't be loaded for some reason -tmp_blacklist () - { - echo "Warning: temporarily blacklisted files named '$1'" 1>&2 - blacklist $1 - } - -blacklist __load__.bro -blacklist test-all.bro -blacklist all.bro -blacklist init-default.bro -blacklist init-bare.bro - -statictext="\ -# DO NOT EDIT -# This file is auto-generated from the "genDocSourcesList.sh" script. -# -# This is a list of Bro script sources for which to generate reST documentation. -# It will be included inline in the CMakeLists.txt found in the same directory -# in order to create Makefile targets that define how to generate reST from -# a given Bro script. -# -# Note: any path prefix of the script (2nd argument of rest_target macro) -# will be used to derive what path under scripts/ the generated documentation -# will be placed. - -set(psd \${PROJECT_SOURCE_DIR}/scripts) - -rest_target(\${CMAKE_CURRENT_SOURCE_DIR} example.bro internal) -rest_target(\${psd} base/init-default.bro internal) -rest_target(\${psd} base/init-bare.bro internal) -" - -if [[ $# -ge 1 ]]; then - outfile=$1 -else - outfile=DocSourcesList.cmake -fi - -thisdir="$( cd "$( dirname "$0" )" && pwd )" -sourcedir=${thisdir}/../.. - -echo "$statictext" > $outfile - -bifs=`( cd ${sourcedir}/build/scripts/base && find . -name \*\.bif.bro | sort )` - -for file in $bifs -do - f=${file:2} - echo "rest_target(\${CMAKE_BINARY_DIR}/scripts base/$f)" >> $outfile -done - -scriptfiles=`( cd ${sourcedir}/scripts && find . -name \*\.bro | sort )` - -for file in $scriptfiles -do - f=${file:2} - if [[ ! $f =~ $blacklist ]]; then - echo "rest_target(\${psd} $f)" >> $outfile - fi -done diff --git a/doc/scripts/internal.rst b/doc/scripts/internal.rst deleted file mode 100644 index a6c10f1cfb..0000000000 --- a/doc/scripts/internal.rst +++ /dev/null @@ -1,5 +0,0 @@ -.. This is a stub doc to which broxygen appends during the build process - -Internal Scripts -================ - diff --git a/doc/scripts/scripts.rst b/doc/scripts/scripts.rst deleted file mode 100644 index 8aceacf329..0000000000 --- a/doc/scripts/scripts.rst +++ /dev/null @@ -1,8 +0,0 @@ -.. This is a stub doc to which broxygen appends during the build process - -=============== -All Bro Scripts -=============== - -.. toctree:: - :maxdepth: 1 diff --git a/scripts/base/frameworks/reporter/main.bro b/scripts/base/frameworks/reporter/main.bro index 3668348ef1..873cb15a45 100644 --- a/scripts/base/frameworks/reporter/main.bro +++ b/scripts/base/frameworks/reporter/main.bro @@ -9,7 +9,7 @@ ##! Note that this framework deals with the handling of internally generated ##! reporter messages, for the interface ##! into actually creating reporter messages from the scripting layer, use -##! the built-in functions in :doc:`/scripts/base/bif/reporter.bif`. +##! the built-in functions in :doc:`/scripts/base/bif/reporter.bif.bro`. module Reporter; diff --git a/scripts/base/init-bare.bro b/scripts/base/init-bare.bro index de26e6a41d..8213126b2f 100644 --- a/scripts/base/init-bare.bro +++ b/scripts/base/init-bare.bro @@ -399,7 +399,7 @@ type NetStats: record { pkts_dropped: count &default=0; ##< Packets reported dropped by the system. ## Packets seen on the link. Note that this may differ ## from *pkts_recvd* because of a potential capture_filter. See - ## :doc:`/scripts/base/frameworks/packet-filter/main`. Depending on the + ## :doc:`/scripts/base/frameworks/packet-filter/main.bro`. Depending on the ## packet capture system, this value may not be available and will then ## be always set to zero. pkts_link: count &default=0; @@ -2768,13 +2768,13 @@ const log_max_size = 0.0 &redef; const log_encryption_key = "" &redef; ## Write profiling info into this file in regular intervals. The easiest way to -## activate profiling is loading :doc:`/scripts/policy/misc/profiling`. +## activate profiling is loading :doc:`/scripts/policy/misc/profiling.bro`. ## ## .. bro:see:: profiling_interval expensive_profiling_multiple segment_profiling global profiling_file: file &redef; ## Update interval for profiling (0 disables). The easiest way to activate -## profiling is loading :doc:`/scripts/policy/misc/profiling`. +## profiling is loading :doc:`/scripts/policy/misc/profiling.bro`. ## ## .. bro:see:: profiling_file expensive_profiling_multiple segment_profiling const profiling_interval = 0 secs &redef; diff --git a/scripts/base/protocols/dhcp/main.bro b/scripts/base/protocols/dhcp/main.bro index e5be50b1ab..c94bae0476 100644 --- a/scripts/base/protocols/dhcp/main.bro +++ b/scripts/base/protocols/dhcp/main.bro @@ -4,7 +4,7 @@ ##! ##! If you'd like to track known DHCP devices and to log the hostname ##! supplied by the client, see -##! :doc:`/scripts/policy/protocols/dhcp/known-devices-and-hostnames`. +##! :doc:`/scripts/policy/protocols/dhcp/known-devices-and-hostnames.bro`. @load ./utils.bro diff --git a/scripts/base/protocols/http/__load__.bro b/scripts/base/protocols/http/__load__.bro index 20fcd5f1ec..42a45efb36 100644 --- a/scripts/base/protocols/http/__load__.bro +++ b/scripts/base/protocols/http/__load__.bro @@ -3,4 +3,4 @@ @load ./utils @load ./files -@load-sigs ./dpd.sig \ No newline at end of file +@load-sigs ./dpd.sig diff --git a/scripts/broxygen/README b/scripts/broxygen/README new file mode 100644 index 0000000000..ac7f522285 --- /dev/null +++ b/scripts/broxygen/README @@ -0,0 +1,4 @@ +This package is loaded during the process which automatically generates +reference documentation for all Bro scripts (i.e. "Broxygen"). Its only +purpose is to provide an easy way to load all known Bro scripts plus any +extra scripts needed or used by the documentation process. diff --git a/scripts/broxygen/__load__.bro b/scripts/broxygen/__load__.bro new file mode 100644 index 0000000000..68a0d878b6 --- /dev/null +++ b/scripts/broxygen/__load__.bro @@ -0,0 +1,2 @@ +@load test-all-policy.bro +@load ./example.bro diff --git a/doc/scripts/example.bro b/scripts/broxygen/example.bro similarity index 100% rename from doc/scripts/example.bro rename to scripts/broxygen/example.bro diff --git a/scripts/policy/misc/known-devices.bro b/scripts/policy/misc/known-devices.bro index 16c5250d1c..8378d589f8 100644 --- a/scripts/policy/misc/known-devices.bro +++ b/scripts/policy/misc/known-devices.bro @@ -7,7 +7,7 @@ ##! ##! This script will not generate any logs on its own, it needs to be ##! supplied with information from elsewhere, such as -##! :doc:`/scripts/policy/protocols/dhcp/known-devices-and-hostnames`. +##! :doc:`/scripts/policy/protocols/dhcp/known-devices-and-hostnames.bro`. module Known; diff --git a/scripts/policy/misc/stats.bro b/scripts/policy/misc/stats.bro index 7e1e4b6689..eb1ddb0202 100644 --- a/scripts/policy/misc/stats.bro +++ b/scripts/policy/misc/stats.bro @@ -1,5 +1,5 @@ ##! Log memory/packet/lag statistics. Differs from -##! :doc:`/scripts/policy/misc/profiling` in that this +##! :doc:`/scripts/policy/misc/profiling.bro` in that this ##! is lighter-weight (much less info, and less load to generate). @load base/frameworks/notice diff --git a/src/BroDoc.cc b/src/BroDoc.cc index a7b7580fa6..1239c4efce 100644 --- a/src/BroDoc.cc +++ b/src/BroDoc.cc @@ -611,7 +611,7 @@ void CreateProtoAnalyzerDoc(FILE* f) fprintf(f, "Protocol Analyzers\n"); fprintf(f, "==================\n\n"); fprintf(f, ".. contents::\n"); - fprintf(f, " :depth: 1\n\n"); + fprintf(f, " :depth: 2\n\n"); WriteAnalyzerTagDefn(f, "Analyzer"); @@ -641,7 +641,7 @@ void CreateFileAnalyzerDoc(FILE* f) fprintf(f, "File Analyzers\n"); fprintf(f, "==============\n\n"); fprintf(f, ".. contents::\n"); - fprintf(f, " :depth: 1\n\n"); + fprintf(f, " :depth: 2\n\n"); WriteAnalyzerTagDefn(f, "Files"); diff --git a/src/Type.cc b/src/Type.cc index 83f239bbfe..8ef6471d67 100644 --- a/src/Type.cc +++ b/src/Type.cc @@ -1165,9 +1165,9 @@ void RecordType::DescribeFieldsReST(ODesc* d, bool func_args) const field_from_script != type_from_script ) { d->PushIndent(); - d->Add(fmt("(from ``redef`` in :doc:`%s`)", + d->Add(fmt("(from ``redef`` in :doc:`/scripts/%s`)", field_from_script.c_str())); - d->PopIndentNoNL(); + d->PopIndent(); } vector cmnts = doc->GetFieldComments(td->id); @@ -1513,9 +1513,9 @@ void EnumType::DescribeReST(ODesc* d, bool roles_only) const { d->NL(); d->PushIndent(); - d->Add(fmt("(from ``redef`` in :doc:`%s`)", + d->Add(fmt("(from ``redef`` in :doc:`/scripts/%s`)", enum_from_script.c_str())); - d->PopIndentNoNL(); + d->PopIndent(); } vector cmnts = doc->GetComments(); diff --git a/src/broxygen/Configuration.cc b/src/broxygen/Configuration.cc index d88aa1920c..9d0c0f7015 100644 --- a/src/broxygen/Configuration.cc +++ b/src/broxygen/Configuration.cc @@ -9,7 +9,10 @@ #include #include #include +#include #include +#include +#include using namespace broxygen; using namespace std; @@ -160,24 +163,38 @@ void PackageTarget::DoFindDependencies(const vector& docs) void PackageTarget::DoGenerate() const { - if ( broxygen_mgr->IsUpToDate(Name(), script_deps) ) + if ( broxygen_mgr->IsUpToDate(Name(), script_deps) && + broxygen_mgr->IsUpToDate(Name(), pkg_deps) ) return; TargetFile file(Name()); + fprintf(file.f, ":orphan:\n\n"); + for ( manifest_t::const_iterator it = pkg_manifest.begin(); it != pkg_manifest.end(); ++it ) { - fprintf(file.f, "Package: %s\n\n", it->first->Name().c_str()); + string header = fmt("Package: %s", it->first->Name().c_str()); + header += "\n" + string(header.size(), '='); + + fprintf(file.f, "%s\n\n", header.c_str()); + + vector readme = it->first->GetReadme(); + + for ( size_t i = 0; i < readme.size(); ++i ) + fprintf(file.f, "%s\n", readme[i].c_str()); + + fprintf(file.f, "\n"); for ( size_t i = 0; i < it->second.size(); ++i ) { - fprintf(file.f, " :doc:`%s`\n", it->second[i]->Name().c_str()); + fprintf(file.f, ":doc:`/scripts/%s`\n", + it->second[i]->Name().c_str()); vector cmnts = it->second[i]->GetComments(); for ( size_t j = 0; j < cmnts.size(); ++j ) - fprintf(file.f, " %s\n", cmnts[j].c_str()); + fprintf(file.f, " %s\n", cmnts[j].c_str()); fprintf(file.f, "\n"); } @@ -211,25 +228,82 @@ void ScriptTarget::DoFindDependencies(const vector& docs) if ( script_deps.empty() ) reporter->FatalError("No match for Broxygen target '%s' pattern '%s'", Name().c_str(), Pattern().c_str()); + + if ( ! IsDir() ) + return; + + for ( size_t i = 0; i < script_deps.size(); ++i ) + { + if ( SafeBasename(script_deps[i]->Name()).result == PACKAGE_LOADER ) + { + string pkg_dir = SafeDirname(script_deps[i]->Name()).result; + string target_file = Name() + pkg_dir + "/index.rst"; + Target* t = PackageTarget::Instantiate(target_file, pkg_dir); + t->FindDependencies(docs); + pkg_deps.push_back(t); + } + } + } + +vector dir_contents_recursive(string dir) + { + vector rval; + struct stat st; + + if ( stat(dir.c_str(), &st) < 0 && errno == ENOENT ) + return rval; + + while ( dir[dir.size() - 1] == '/' ) + dir.erase(dir.size() - 1, 1); + + char* dir_copy = copy_string(dir.c_str()); + char** scan_path = new char*[2]; + scan_path[0] = dir_copy; + scan_path[1] = 0; + + FTS* fts = fts_open(scan_path, FTS_NOCHDIR, 0); + + if ( ! fts ) + { + reporter->Error("fts_open failure: %s", strerror(errno)); + delete [] scan_path; + delete [] dir_copy; + return rval; + } + + FTSENT* n; + + while ( (n = fts_read(fts)) ) + { + if ( n->fts_info & FTS_F ) + rval.push_back(n->fts_path); + } + + if ( errno ) + reporter->Error("fts_read failure: %s", strerror(errno)); + + if ( fts_close(fts) < 0 ) + reporter->Error("fts_close failure: %s", strerror(errno)); + + delete [] scan_path; + delete [] dir_copy; + return rval; } void ScriptTarget::DoGenerate() const { - if ( Name()[Name().size() - 1] == '/' ) + if ( IsDir() ) { // Target name is a dir, matching scripts are written within that dir // with a dir tree that parallels the script's BROPATH location. + set targets; + vector dir_contents = dir_contents_recursive(Name()); + for ( size_t i = 0; i < script_deps.size(); ++i ) { - string target_filename = Name() + script_deps[i]->Name(); - size_t pos = target_filename.rfind(".bro"); - - if ( pos == target_filename.size() - 4 ) - target_filename.replace(pos, 4, ".rst"); - else - target_filename += ".rst"; - + string target_filename = Name() + script_deps[i]->Name() + ".rst"; + targets.insert(target_filename); vector dep; dep.push_back(script_deps[i]); @@ -241,9 +315,26 @@ void ScriptTarget::DoGenerate() const fprintf(file.f, "%s\n", script_deps[i]->ReStructuredText().c_str()); } - // TODO: could possibly take inventory of files in the dir beforehand, - // track all files written, then compare afterwards in order to remove - // stale files. + for ( size_t i = 0; i < pkg_deps.size(); ++i ) + { + targets.insert(pkg_deps[i]->Name()); + pkg_deps[i]->Generate(); + } + + for ( size_t i = 0; i < dir_contents.size(); ++i ) + { + string f = dir_contents[i]; + + if ( targets.find(f) != targets.end() ) + continue; + + if ( unlink(f.c_str()) < 0 ) + reporter->Warning("Failed to unlink %s: %s", f.c_str(), + strerror(errno)); + + DBG_LOG(DBG_BROXYGEN, "Delete stale script file %s", f.c_str()); + } + return; } @@ -300,7 +391,8 @@ void ScriptIndexTarget::DoGenerate() const if ( ! d ) continue; - fprintf(file.f, " %s <%s>\n", d->Name().c_str(), d->Name().c_str()); + fprintf(file.f, " %s \n", d->Name().c_str(), + d->Name().c_str()); } } diff --git a/src/broxygen/Configuration.h b/src/broxygen/Configuration.h index 6b5c45ed45..6ea9f640be 100644 --- a/src/broxygen/Configuration.h +++ b/src/broxygen/Configuration.h @@ -146,6 +146,9 @@ public: const std::string& pattern) { return new ScriptTarget(name, pattern); } + ~ScriptTarget() + { for ( size_t i = 0; i < pkg_deps.size(); ++i ) delete pkg_deps[i]; } + protected: ScriptTarget(const std::string& name, const std::string& pattern) @@ -159,6 +162,11 @@ private: void DoFindDependencies(const std::vector& docs); void DoGenerate() const; + + bool IsDir() const + { return Name()[Name().size() - 1] == '/'; } + + std::vector pkg_deps; }; class ScriptSummaryTarget : public ScriptTarget { diff --git a/src/broxygen/Document.cc b/src/broxygen/Document.cc index 2619755b35..a44a8ecbdc 100644 --- a/src/broxygen/Document.cc +++ b/src/broxygen/Document.cc @@ -116,7 +116,7 @@ PackageDocument::PackageDocument(const string& arg_name) string PackageDocument::DoReStructuredText(bool roles_only) const { - string rval = fmt(":doc:`%s <%s/index>`\n\n", pkg_name.c_str(), + string rval = fmt(":doc:`%s `\n\n", pkg_name.c_str(), pkg_name.c_str()); for ( size_t i = 0; i < readme.size(); ++i ) @@ -629,15 +629,20 @@ string ScriptDocument::DoReStructuredText(bool roles_only) const if ( it != dependencies.begin() ) rval += ", "; - rval += fmt(":doc:`%s `", it->c_str(), it->c_str()); - // TODO linking to packages is a bit different? + string path = find_file(*it, bro_path(), "bro"); + string doc = *it; + + if ( ! path.empty() && is_dir(path.c_str()) ) + // Reference the package. + doc += "/index"; + + rval += fmt(":doc:`%s `", it->c_str(), doc.c_str()); } rval += "\n"; } - // TODO: make this an absolute path? - rval += fmt(":Source File: :download:`%s`\n", name.c_str()); + rval += fmt(":Source File: :download:`/scripts/%s`\n", name.c_str()); rval += "\n"; rval += make_heading("Summary", '~'); rval += make_summary("Options", '#', '=', options); diff --git a/src/const.bif b/src/const.bif index ea84b3363d..a45dfb0a7a 100644 --- a/src/const.bif +++ b/src/const.bif @@ -1,6 +1,6 @@ ##! Declaration of various scripting-layer constants that the Bro core uses ##! internally. Documentation and default values for the scripting-layer -##! variables themselves are found in :doc:`/scripts/base/init-bare`. +##! variables themselves are found in :doc:`/scripts/base/init-bare.bro`. const ignore_keep_alive_rexmit: bool; const skip_http_data: bool; diff --git a/src/event.bif b/src/event.bif index ddadb47f8a..79cf46dd8a 100644 --- a/src/event.bif +++ b/src/event.bif @@ -360,7 +360,7 @@ event content_gap%(c: connection, is_orig: bool, seq: count, length: count%); ## ## .. note:: ## -## Bro comes with a script :doc:`/scripts/policy/misc/capture-loss` that uses +## Bro comes with a script :doc:`/scripts/policy/misc/capture-loss.bro` that uses ## this event to estimate packet loss and report when a predefined threshold ## is exceeded. event gap_report%(dt: interval, info: gap_info%); diff --git a/src/reporter.bif b/src/reporter.bif index c0f83205c3..3cdefc4da4 100644 --- a/src/reporter.bif +++ b/src/reporter.bif @@ -4,7 +4,7 @@ ##! If event handlers do exist, it's assumed they take care of determining ##! how/where to output the messages. ##! -##! See :doc:`/scripts/base/frameworks/reporter/main` for a convenient +##! See :doc:`/scripts/base/frameworks/reporter/main.bro` for a convenient ##! reporter message logging framework. module Reporter; diff --git a/testing/btest/Makefile b/testing/btest/Makefile index 47451fbf27..261c7262b0 100644 --- a/testing/btest/Makefile +++ b/testing/btest/Makefile @@ -21,12 +21,9 @@ cleanup: @rm -f $(DIAG) @rm -f .tmp/script-coverage* -update-doc-sources: - ../../doc/scripts/genDocSourcesList.sh ../../doc/scripts/DocSourcesList.cmake - # Updates the three coverage tests that usually need tweaking when # scripts get added/removed. -update-coverage-tests: update-doc-sources +update-coverage-tests: btest -qU coverage.bare-load-baseline btest -qU coverage.default-load-baseline @echo "Use 'git diff' to check updates look right." diff --git a/testing/btest/coverage/doc.test b/testing/btest/coverage/doc.test deleted file mode 100644 index 074e397d88..0000000000 --- a/testing/btest/coverage/doc.test +++ /dev/null @@ -1,8 +0,0 @@ -# This tests that we're generating bro script documentation for all the -# available bro scripts. If this fails, then the genDocSources.sh needs -# to be run to produce a new DocSourcesList.cmake or genDocSources.sh needs -# to be updated to blacklist undesired scripts. To update, run -# "make update-doc-sources" -# -# @TEST-EXEC: $DIST/doc/scripts/genDocSourcesList.sh -# @TEST-EXEC: cmp $DIST/doc/scripts/DocSourcesList.cmake ./DocSourcesList.cmake From 7e0864468cbc7f8fd629e0e2cb7c983883dff8bb Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Thu, 21 Nov 2013 15:59:07 -0600 Subject: [PATCH 12/40] A couple documentation fixes. - Move notice index wrapper doc to doc/script-reference -- doc/scripts no longer contains any static documentation because that location will be managed by Bro to generate per-script docs. - :doc: references for generated per-script docs now need the ".bro" suffix. (IMO this is better since it directly mirrors the actual script's file name and can't be confused w/ a package). --- doc/frameworks/sumstats.rst | 2 +- doc/script-reference/index.rst | 1 + doc/{scripts => script-reference}/notices.rst | 0 3 files changed, 2 insertions(+), 1 deletion(-) rename doc/{scripts => script-reference}/notices.rst (100%) diff --git a/doc/frameworks/sumstats.rst b/doc/frameworks/sumstats.rst index 057cc87fc1..6ab2f43b35 100644 --- a/doc/frameworks/sumstats.rst +++ b/doc/frameworks/sumstats.rst @@ -87,7 +87,7 @@ Taking the previous example even further, we can implement a simple detection to demonstrate the thresholding functionality. This example is a toy to demonstrate how thresholding works in Sumstats and is not meant to be a real-world functional example, that is left to the -:doc:`/scripts/policy/misc/scan` script that is included with Bro. +:doc:`/scripts/policy/misc/scan.bro` script that is included with Bro. .. btest-include:: ${DOC_ROOT}/frameworks/sumstats-toy-scan.bro diff --git a/doc/script-reference/index.rst b/doc/script-reference/index.rst index ccb374bd82..bd600e4a97 100644 --- a/doc/script-reference/index.rst +++ b/doc/script-reference/index.rst @@ -5,6 +5,7 @@ Script Reference .. toctree:: :maxdepth: 1 + notices proto-analyzers file-analyzers builtins diff --git a/doc/scripts/notices.rst b/doc/script-reference/notices.rst similarity index 100% rename from doc/scripts/notices.rst rename to doc/script-reference/notices.rst From 98dcfc64a8e2d1b865739fe3b16b2e01b7766353 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Fri, 22 Nov 2013 14:18:24 -0600 Subject: [PATCH 13/40] Broxygen doc-related test updates. Fix two regressions. - Fix automatic function parameter documentation formatting for record fields that are functions. - Document redefs in a fixed order. --- scripts/broxygen/example.bro | 265 +++++++-------- src/Type.cc | 13 +- src/broxygen/CMakeLists.txt | 1 + src/broxygen/Document.cc | 79 +---- src/broxygen/Document.h | 10 +- src/broxygen/utils.cc | 70 ++++ src/broxygen/utils.h | 12 + .../autogen-reST-enums.rst | 116 ------- .../doc.autogen-reST-example/example.rst | 301 ------------------ .../autogen-reST-func-params.rst | 58 ---- .../autogen-reST-records.rst | 53 --- .../autogen-reST-type-aliases.rst | 62 ---- .../Baseline/doc.broxygen.all_scripts/.stderr | 0 .../Baseline/doc.broxygen.all_scripts/.stdout | 1 + .../doc.broxygen.enums/autogen-reST-enums.rst | 60 ++++ .../Baseline/doc.broxygen.example/example.rst | 249 +++++++++++++++ .../autogen-reST-func-params.rst | 30 ++ .../autogen-reST-records.rst | 28 ++ .../autogen-reST-type-aliases.rst | 44 +++ testing/btest/doc/autogen-reST-all | 1 - testing/btest/doc/autogen-reST-example | 2 - .../btest/doc/autogen-reST-type-aliases.bro | 15 - testing/btest/doc/broxygen/all_scripts.test | 10 + .../enums.bro} | 8 +- testing/btest/doc/broxygen/example.bro | 8 + .../func-params.bro} | 10 +- .../records.bro} | 12 +- testing/btest/doc/broxygen/type-aliases.bro | 34 ++ testing/btest/doc/record-add.bro | 6 +- testing/btest/doc/record-attr-check.bro | 2 +- 30 files changed, 714 insertions(+), 846 deletions(-) create mode 100644 src/broxygen/utils.cc create mode 100644 src/broxygen/utils.h delete mode 100644 testing/btest/Baseline/doc.autogen-reST-enums/autogen-reST-enums.rst delete mode 100644 testing/btest/Baseline/doc.autogen-reST-example/example.rst delete mode 100644 testing/btest/Baseline/doc.autogen-reST-func-params/autogen-reST-func-params.rst delete mode 100644 testing/btest/Baseline/doc.autogen-reST-records/autogen-reST-records.rst delete mode 100644 testing/btest/Baseline/doc.autogen-reST-type-aliases/autogen-reST-type-aliases.rst create mode 100644 testing/btest/Baseline/doc.broxygen.all_scripts/.stderr create mode 100644 testing/btest/Baseline/doc.broxygen.all_scripts/.stdout create mode 100644 testing/btest/Baseline/doc.broxygen.enums/autogen-reST-enums.rst create mode 100644 testing/btest/Baseline/doc.broxygen.example/example.rst create mode 100644 testing/btest/Baseline/doc.broxygen.func-params/autogen-reST-func-params.rst create mode 100644 testing/btest/Baseline/doc.broxygen.records/autogen-reST-records.rst create mode 100644 testing/btest/Baseline/doc.broxygen.type-aliases/autogen-reST-type-aliases.rst delete mode 100644 testing/btest/doc/autogen-reST-all delete mode 100644 testing/btest/doc/autogen-reST-example delete mode 100644 testing/btest/doc/autogen-reST-type-aliases.bro create mode 100644 testing/btest/doc/broxygen/all_scripts.test rename testing/btest/doc/{autogen-reST-enums.bro => broxygen/enums.bro} (80%) create mode 100644 testing/btest/doc/broxygen/example.bro rename testing/btest/doc/{autogen-reST-func-params.bro => broxygen/func-params.bro} (56%) rename testing/btest/doc/{autogen-reST-records.bro => broxygen/records.bro} (58%) create mode 100644 testing/btest/doc/broxygen/type-aliases.bro diff --git a/scripts/broxygen/example.bro b/scripts/broxygen/example.bro index ff6e4e34d2..65cc5ff1c7 100644 --- a/scripts/broxygen/example.bro +++ b/scripts/broxygen/example.bro @@ -1,20 +1,24 @@ -##! This is an example script that demonstrates documentation features. -##! Comments of the form ``##!`` are for the script summary. The contents of -##! these comments are transferred directly into the auto-generated +##! This is an example script that demonstrates Broxygen-style +##! documentation. It generally will make most sense when viewing +##! the script's raw source code and comparing to the HTML-rendered +##! version. +##! +##! Comments in the from ``##!`` are meant to summarize the script's +##! purpose. They are transferred directly in to the generated ##! `reStructuredText `_ -##! (reST) document's summary section. +##! (reST) document associated with the script. ##! ##! .. tip:: You can embed directives and roles within ``##``-stylized comments. ##! ##! There's also a custom role to reference any identifier node in ##! the Bro Sphinx domain that's good for "see alsos", e.g. ##! -##! See also: :bro:see:`Example::a_var`, :bro:see:`Example::ONE`, -##! :bro:see:`SSH::Info` +##! See also: :bro:see:`BroxygenExample::a_var`, +##! :bro:see:`BroxygenExample::ONE`, :bro:see:`SSH::Info` ##! ##! And a custom directive does the equivalent references: ##! -##! .. bro:see:: Example::a_var Example::ONE SSH::Info +##! .. bro:see:: BroxygenExample::a_var BroxygenExample::ONE SSH::Info # Comments that use a single pound sign (#) are not significant to # a script's auto-generated documentation, but ones that use a @@ -26,7 +30,7 @@ # variable declarations to associate with the last-declared identifier. # # Generally, the auto-doc comments (##) are associated with the -# next declaration/identifier found in the script, but the doc framework +# next declaration/identifier found in the script, but Broxygen # will track/render identifiers regardless of whether they have any # of these special comments associated with them. # @@ -37,193 +41,154 @@ # in the "##"-stylized comments up until the first empty comment # is taken as the summary text for a given identifier. -# @load directives are self-documenting +# @load directives are self-documenting, don't use any ``##`` style +# comments with them. +@load base/frameworks/notice +@load base/protocols/http @load frameworks/software/vulnerable -# "module" statements are self-documenting -module Example; +# "module" statements are self-documenting, don't use any ``##`` style +# comments with them. +module BroxygenExample; -# redefinitions of "capture_filters" are self-documenting and -# go into the generated documentation's "Packet Filter" section -redef capture_filters += { - ["ssl"] = "tcp port 443", - ["nntps"] = "tcp port 562", -}; - -global example_ports = { - 443/tcp, 562/tcp, -} &redef; - - -event bro_init() - { - Analyzer::register_for_ports(Analyzer::ANALYZER_SSL, example_ports); - } - -# redefinitions of "Notice::Type" are self-documenting, but -# more information can be supplied in two different ways +# Redefinitions of "Notice::Type" are self-documenting, but +# more information can be supplied in two different ways. redef enum Notice::Type += { - ## any number of this type of comment - ## will document "Notice_One" - Notice_One, - Notice_Two, ##< any number of this type of comment - ##< will document "Notice_Two" - Notice_Three, - Notice_Four, + ## Any number of this type of comment + ## will document "Broxygen_One". + Broxygen_One, + Broxygen_Two, ##< Any number of this type of comment + ##< will document "BROXYGEN_TWO". + Broxygen_Three, + ## Omitting comments is fine, and so is mixing ``##`` and ``##<``, but + Broxygen_Four, ##< it's probably best to use only one style consistently. }; -# Redef'ing the ID enumeration for logging streams is automatically tracked. -# Comments of the "##" form can be use to further document it, but it's -# better to do all documentation related to logging in the summary section -# as is shown above. +# All redefs are automatically tracked. Comments of the "##" form can be use +# to further document it, but in some cases, like here, they wouldn't be +# ading any interesting information that's not implicit. redef enum Log::ID += { LOG }; -# Anything declared in the export section will show up in the rendered -# documentation's "public interface" section +# Only identifiers declared in an export section will show up in generated docs. export { - # these headings don't mean anything special to the - # doc framework right now, I'm just including them - # to make it more clear to the reader how the doc - # framework will actually categorize a script's identifiers + ## Documentation for the "SimpleEnum" type goes here. + ## It can span multiple lines. + type SimpleEnum: enum { + ## Documentation for particular enum values is added like this. + ## And can also span multiple lines. + ONE, + TWO, ##< Or this style is valid to document the preceding enum value. + THREE, + }; - ############## types ################ + ## Document the "SimpleEnum" redef here with any special info regarding + ## the *redef* itself. + redef enum SimpleEnum += { + FOUR, ##< And some documentation for "FOUR". + ## Also "FIVE". + FIVE + }; - # Note that I'm just mixing the "##" and "##<" - # types of comments in the following declarations - # as a demonstration. Normally, it would be good style - # to pick one and be consistent. + ## General documentation for a type "SimpleRecord" goes here. + ## The way fields can be documented is similar to what's already seen + ## for enums. + type SimpleRecord: record { + ## Counts something. + field1: count; + field2: bool; ##< Toggles something. + }; - ## documentation for "SimpleEnum" - ## goes here. - type SimpleEnum: enum { - ## and more specific info for "ONE" - ## can span multiple lines - ONE, - TWO, ##< or more info like this for "TWO" - ##< can span multiple lines - THREE, - }; + ## Document the record extension *redef* itself here. + redef record SimpleRecord += { + ## Document the extending field like this. + field_ext: string &optional; ##< Or here, like this. + }; - ## document the "SimpleEnum" redef here - redef enum SimpleEnum += { - FOUR, ##< and some documentation for "FOUR" - ## also "FIVE" for good measure - FIVE - }; + ## General documentation for a type "ComplexRecord" goes here. + type ComplexRecord: record { + field1: count; ##< Counts something. + field2: bool; ##< Toggles something. + field3: SimpleRecord; ##< Broxygen automatically tracks types + ##< and cross-references are automatically + ##< inserted in to generated docs. + msg: string &default="blah"; ##< Attributes are self-documenting. + } &redef; - ## general documentation for a type "SimpleRecord" - ## goes here. - type SimpleRecord: record { - ## counts something - field1: count; - field2: bool; ##< toggles something - }; + ## An example record to be used with a logging stream. + ## Nothing special about it. If another script redefs this type + ## to add fields, the generated documentation will show all original + ## fields plus the extensions and the scripts which contributed to it + ## (provided they are also @load'ed). + type Info: record { + ts: time &log; + uid: string &log; + status: count &log &optional; + }; - ## document the record extension redef here - redef record SimpleRecord += { - ## document the extending field here - field_ext: string &optional; ##< (or here) - }; + ## Add documentation for "an_option" here. + ## The type/attribute information is all generated automatically. + const an_option: set[addr, addr, string] &redef; - ## general documentation for a type "ComplexRecord" goes here - type ComplexRecord: record { - field1: count; ##< counts something - field2: bool; ##< toggles something - field3: SimpleRecord; - msg: string &default="blah"; ##< attributes are self-documenting - } &redef; + ## Default initialization will be generated automatically. + const option_with_init = 0.01 secs &redef; ##< More docs can be added here. - ## An example record to be used with a logging stream. - type Info: record { - ts: time &log; - uid: string &log; - status: count &log &optional; - }; + ## Put some documentation for "a_var" here. Any global/non-const that + ## isn't a function/event/hook is classified as a "state variable" + ## in the generated docs. + global a_var: bool; - ############## options ################ - # right now, I'm just defining an option as - # any const with &redef (something that can - # change at parse time, but not at run time. + ## Types are inferred, that information is self-documenting. + global var_without_explicit_type = "this works"; - ## add documentation for "an_option" here - const an_option: set[addr, addr, string] &redef; - - # default initialization will be self-documenting - const option_with_init = 0.01 secs &redef; ##< More docs can be added here. - - ############## state variables ############ - # right now, I'm defining this as any global - # that's not a function/event. doesn't matter - # if &redef attribute is present - - ## put some documentation for "a_var" here - global a_var: bool; - - # attributes are self-documenting - global var_with_attr: count &persistent; - - # it's fine if the type is inferred, that information is self-documenting - global var_without_explicit_type = "this works"; - - ## The first.sentence for the summary text ends here. And this second - ## sentence doesn't show in the short description. - global dummy: string; - - ############## functions/events ############ + ## The first sentence for a particular identifier's summary text ends here. + ## And this second sentence doesn't show in the short description provided + ## by the table of all identifiers declared by this script. + global summary_test: string; ## Summarize purpose of "a_function" here. ## Give more details about "a_function" here. ## Separating the documentation of the params/return values with ## empty comments is optional, but improves readability of script. ## - ## tag: function arguments can be described - ## like this - ## msg: another param + ## tag: Function arguments can be described + ## like this. + ## + ## msg: Another param. ## - ## Returns: describe the return type here + ## Returns: Describe the return type here. global a_function: function(tag: string, msg: string): string; ## Summarize "an_event" here. ## Give more details about "an_event" here. - ## Example::an_event should not be confused as a parameter. - ## name: describe the argument here + ## + ## BroxygenExample::a_function should not be confused as a parameter + ## in the generated docs, but it also doesn't generate a cross-reference + ## link. Use the see role instead: :bro:see:`BroxygenExample::a_function`. + ## + ## name: Describe the argument here. global an_event: event(name: string); - - ## This is a declaration of an example event that can be used in - ## logging streams and is raised once for each log entry. - global log_example: event(rec: Info); } -function filter_func(rec: Info): bool - { - return T; - } - -# this function is documented in the "private interface" section -# of generated documentation and any "##"-stylized comments would also -# be rendered there +# This function isn't exported, so it won't appear anywhere in the generated +# documentation. So using ``##``-style comments is pointless here. function function_without_proto(tag: string): string { return "blah"; } -# this record type is documented in the "private interface" section -# of generated documentation and any "##"-stylized comments would also -# be rendered there +# Same thing goes for types -- it's not exported, so it's considered +# private to this script and comments are only interesting to a person +# who is already reading the raw source for the script (so don't use +# ``##`` comments here. type PrivateRecord: record { field1: bool; field2: count; }; +# Event handlers are also an implementation detail of a script, so they +# don't show up anywhere in the generated documentation. event bro_init() { - Log::create_stream(Example::LOG, [$columns=Info, $ev=log_example]); - Log::add_filter(Example::LOG, [ - $name="example-filter", - $path="example-filter", - $pred=filter_func, - $exclude=set("ts") - ]); } diff --git a/src/Type.cc b/src/Type.cc index 8ef6471d67..1e882f27d6 100644 --- a/src/Type.cc +++ b/src/Type.cc @@ -9,6 +9,7 @@ #include "Serializer.h" #include "Reporter.h" #include "broxygen/Manager.h" +#include "broxygen/utils.h" #include #include @@ -1182,7 +1183,17 @@ void RecordType::DescribeFieldsReST(ODesc* d, bool func_args) const if ( i > 0 ) d->NL(); - d->Add(cmnts[i].c_str()); + if ( IsFunc(td->type->Tag()) ) + { + string s = cmnts[i]; + + if ( broxygen::prettify_params(s) ) + d->NL(); + + d->Add(s.c_str()); + } + else + d->Add(cmnts[i].c_str()); } d->PopIndentNoNL(); diff --git a/src/broxygen/CMakeLists.txt b/src/broxygen/CMakeLists.txt index 4e5c5e5d0d..6c0b240e39 100644 --- a/src/broxygen/CMakeLists.txt +++ b/src/broxygen/CMakeLists.txt @@ -9,6 +9,7 @@ set(broxygen_SRCS Configuration.cc Document.cc Manager.cc + utils.cc ) bif_target(broxygen.bif) diff --git a/src/broxygen/Document.cc b/src/broxygen/Document.cc index a44a8ecbdc..bb64f90555 100644 --- a/src/broxygen/Document.cc +++ b/src/broxygen/Document.cc @@ -1,5 +1,6 @@ #include "Document.h" #include "Manager.h" +#include "utils.h" #include "util.h" #include "Val.h" @@ -18,72 +19,6 @@ static bool is_public_api(const ID* id) (id->Scope() == SCOPE_MODULE && id->IsExport()); } -static bool prettify_params(string& s) - { - size_t identifier_start_pos = 0; - bool in_identifier = false; - string identifier; - - for ( size_t i = 0; i < s.size(); ++i ) - { - char next = s[i]; - - if ( ! in_identifier ) - { - // Pass by leading whitespace. - if ( isspace(next) ) - continue; - - // Only allow alphabetic and '_' as first char of identifier. - if ( isalpha(next) || next == '_' ) - { - identifier_start_pos = i; - identifier += next; - in_identifier = true; - continue; - } - - // Don't need to change anything. - return false; - } - - // All other characters of identifier are alphanumeric or '_'. - if ( isalnum(next) || next == '_' ) - { - identifier += next; - continue; - } - - if ( next == ':' ) - { - if ( i + 1 < s.size() && s[i + 1] == ':' ) - { - // It's part of an identifier's namespace scoping. - identifier += next; - identifier += s[i + 1]; - ++i; - continue; - } - - // Prettify function param/return value reST markup. - string subst; - - if ( identifier == "Returns" ) - subst = ":returns"; - else - subst = ":param " + identifier; - - s.replace(identifier_start_pos, identifier.size(), subst); - return true; - } - - // Don't need to change anything. - return false; - } - - return false; - } - static string make_heading(const string& heading, char underline) { return heading + "\n" + string(heading.size(), underline) + "\n"; @@ -244,7 +179,7 @@ string IdentifierDocument::DoReStructuredText(bool roles_only) const { string s = comments[i]; - if ( prettify_params(s) ) + if ( broxygen::prettify_params(s) ) d.NL(); d.Add(s.c_str()); @@ -523,15 +458,15 @@ static string make_summary(const string& heading, char underline, char border, static string make_redef_summary(const string& heading, char underline, char border, const string& from_script, - const set& id_set) + const id_doc_set& id_set) { if ( id_set.empty() ) return ""; ReStructuredTextTable table(2); - for ( set::const_iterator it = id_set.begin(); - it != id_set.end(); ++it ) + for ( id_doc_set::const_iterator it = id_set.begin(); it != id_set.end(); + ++it ) { ID* id = (*it)->GetID(); ODesc d; @@ -568,14 +503,14 @@ static string make_details(const string& heading, char underline, } static string make_redef_details(const string& heading, char underline, - const set& id_set) + const id_doc_set& id_set) { if ( id_set.empty() ) return ""; string rval = make_heading(heading, underline); - for ( set::const_iterator it = id_set.begin(); + for ( id_doc_set::const_iterator it = id_set.begin(); it != id_set.end(); ++it ) { rval += (*it)->ReStructuredText(true); diff --git a/src/broxygen/Document.h b/src/broxygen/Document.h index 6046369727..10155ab0eb 100644 --- a/src/broxygen/Document.h +++ b/src/broxygen/Document.h @@ -145,6 +145,13 @@ private: ScriptDocument* declaring_script; }; +struct IdDocComp { + bool operator() (IdentifierDocument* lhs, IdentifierDocument* rhs) const + { return lhs->Name() < rhs->Name(); } +}; + +typedef std::set id_doc_set; + class ScriptDocument : public Document { public: @@ -175,7 +182,6 @@ private: typedef std::map id_doc_map; typedef std::list id_doc_list; typedef std::set string_set; - typedef std::set doc_set; time_t DoGetModificationTime() const; @@ -200,7 +206,7 @@ private: id_doc_list events; id_doc_list hooks; id_doc_list functions; - doc_set redefs; + id_doc_set redefs; }; diff --git a/src/broxygen/utils.cc b/src/broxygen/utils.cc new file mode 100644 index 0000000000..ed27219144 --- /dev/null +++ b/src/broxygen/utils.cc @@ -0,0 +1,70 @@ +#include "utils.h" + +using namespace broxygen; +using namespace std; + +bool broxygen::prettify_params(string& s) + { + size_t identifier_start_pos = 0; + bool in_identifier = false; + string identifier; + + for ( size_t i = 0; i < s.size(); ++i ) + { + char next = s[i]; + + if ( ! in_identifier ) + { + // Pass by leading whitespace. + if ( isspace(next) ) + continue; + + // Only allow alphabetic and '_' as first char of identifier. + if ( isalpha(next) || next == '_' ) + { + identifier_start_pos = i; + identifier += next; + in_identifier = true; + continue; + } + + // Don't need to change anything. + return false; + } + + // All other characters of identifier are alphanumeric or '_'. + if ( isalnum(next) || next == '_' ) + { + identifier += next; + continue; + } + + if ( next == ':' ) + { + if ( i + 1 < s.size() && s[i + 1] == ':' ) + { + // It's part of an identifier's namespace scoping. + identifier += next; + identifier += s[i + 1]; + ++i; + continue; + } + + // Prettify function param/return value reST markup. + string subst; + + if ( identifier == "Returns" ) + subst = ":returns"; + else + subst = ":param " + identifier; + + s.replace(identifier_start_pos, identifier.size(), subst); + return true; + } + + // Don't need to change anything. + return false; + } + + return false; + } diff --git a/src/broxygen/utils.h b/src/broxygen/utils.h new file mode 100644 index 0000000000..a8ba239802 --- /dev/null +++ b/src/broxygen/utils.h @@ -0,0 +1,12 @@ +#ifndef BROXYGEN_UTILS_H +#define BROXYGEN_UTILS_H + +#include + +namespace broxygen { + +bool prettify_params(std::string& s); + +} // namespace broxygen + +#endif diff --git a/testing/btest/Baseline/doc.autogen-reST-enums/autogen-reST-enums.rst b/testing/btest/Baseline/doc.autogen-reST-enums/autogen-reST-enums.rst deleted file mode 100644 index c20ea7e602..0000000000 --- a/testing/btest/Baseline/doc.autogen-reST-enums/autogen-reST-enums.rst +++ /dev/null @@ -1,116 +0,0 @@ -.. Automatically generated. Do not edit. - -:tocdepth: 3 - -autogen-reST-enums.bro -====================== - - - - -:Source File: :download:`autogen-reST-enums.bro` - -Summary -~~~~~~~ -Options -####### -==================================================================== ====================================================================== -:bro:id:`test_enum_option`: :bro:type:`TestEnum1` :bro:attr:`&redef` this should reference the TestEnum1 type and not a generic "enum" type -==================================================================== ====================================================================== - -Types -##### -======================================= ======================================== -:bro:type:`TestEnum1`: :bro:type:`enum` There's tons of ways an enum can look... - -:bro:type:`TestEnum2`: :bro:type:`enum` The final comma is optional -======================================= ======================================== - -Redefinitions -############# -======================================= ======================= -:bro:type:`TestEnum1`: :bro:type:`enum` redefs should also work - -:bro:type:`TestEnum1`: :bro:type:`enum` now with a comma -======================================= ======================= - -Detailed Interface -~~~~~~~~~~~~~~~~~~ -Options -####### -.. bro:id:: test_enum_option - - :Type: :bro:type:`TestEnum1` - :Attributes: :bro:attr:`&redef` - :Default: ``ONE`` - - this should reference the TestEnum1 type and not a generic "enum" type - -Types -##### -.. bro:type:: TestEnum1 - - :Type: :bro:type:`enum` - - .. bro:enum:: ONE TestEnum1 - - like this - - .. bro:enum:: TWO TestEnum1 - - or like this - - .. bro:enum:: THREE TestEnum1 - - multiple - comments - and even - more comments - - There's tons of ways an enum can look... - -.. bro:type:: TestEnum2 - - :Type: :bro:type:`enum` - - .. bro:enum:: A TestEnum2 - - like this - - .. bro:enum:: B TestEnum2 - - or like this - - .. bro:enum:: C TestEnum2 - - multiple - comments - and even - more comments - - The final comma is optional - -Redefinitions -############# -:bro:type:`TestEnum1` - - :Type: :bro:type:`enum` - - .. bro:enum:: FOUR TestEnum1 - - adding another - value - - redefs should also work - -:bro:type:`TestEnum1` - - :Type: :bro:type:`enum` - - .. bro:enum:: FIVE TestEnum1 - - adding another - value - - now with a comma - diff --git a/testing/btest/Baseline/doc.autogen-reST-example/example.rst b/testing/btest/Baseline/doc.autogen-reST-example/example.rst deleted file mode 100644 index 64fb4cb06d..0000000000 --- a/testing/btest/Baseline/doc.autogen-reST-example/example.rst +++ /dev/null @@ -1,301 +0,0 @@ -.. Automatically generated. Do not edit. - -:tocdepth: 3 - -example.bro -=========== -.. bro:namespace:: Example - -This is an example script that demonstrates documentation features. -Comments of the form ``##!`` are for the script summary. The contents of -these comments are transferred directly into the auto-generated -`reStructuredText `_ -(reST) document's summary section. - -.. tip:: You can embed directives and roles within ``##``-stylized comments. - -There's also a custom role to reference any identifier node in -the Bro Sphinx domain that's good for "see alsos", e.g. - -See also: :bro:see:`Example::a_var`, :bro:see:`Example::ONE`, -:bro:see:`SSH::Info` - -And a custom directive does the equivalent references: - -.. bro:see:: Example::a_var Example::ONE SSH::Info - -:Namespace: ``Example`` -:Imports: :doc:`policy/frameworks/software/vulnerable ` -:Source File: :download:`example.bro` - -Summary -~~~~~~~ -Options -####### -============================================================================ ====================================== -:bro:id:`Example::an_option`: :bro:type:`set` :bro:attr:`&redef` add documentation for "an_option" here - -:bro:id:`Example::option_with_init`: :bro:type:`interval` :bro:attr:`&redef` More docs can be added here. -============================================================================ ====================================== - -State Variables -############### -=========================================================================== ================================================== -:bro:id:`Example::a_var`: :bro:type:`bool` put some documentation for "a_var" here - -:bro:id:`Example::var_with_attr`: :bro:type:`count` :bro:attr:`&persistent` - -:bro:id:`Example::var_without_explicit_type`: :bro:type:`string` - -:bro:id:`Example::dummy`: :bro:type:`string` The first.sentence for the summary text ends here. -=========================================================================== ================================================== - -Types -##### -====================================================== ========================================================== -:bro:type:`Example::SimpleEnum`: :bro:type:`enum` documentation for "SimpleEnum" - goes here. - -:bro:type:`Example::SimpleRecord`: :bro:type:`record` general documentation for a type "SimpleRecord" - goes here. - -:bro:type:`Example::ComplexRecord`: :bro:type:`record` general documentation for a type "ComplexRecord" goes here - -:bro:type:`Example::Info`: :bro:type:`record` An example record to be used with a logging stream. -====================================================== ========================================================== - -Events -###### -================================================= ============================================================= -:bro:id:`Example::an_event`: :bro:type:`event` Summarize "an_event" here. - -:bro:id:`Example::log_example`: :bro:type:`event` This is a declaration of an example event that can be used in - logging streams and is raised once for each log entry. -================================================= ============================================================= - -Functions -######### -=================================================== ======================================= -:bro:id:`Example::a_function`: :bro:type:`function` Summarize purpose of "a_function" here. -=================================================== ======================================= - -Redefinitions -############# -===================================================== ======================================== -:bro:type:`Log::ID`: :bro:type:`enum` - -:bro:type:`Example::SimpleEnum`: :bro:type:`enum` document the "SimpleEnum" redef here - -:bro:type:`Example::SimpleRecord`: :bro:type:`record` document the record extension redef here -===================================================== ======================================== - -Notices -####### -:bro:type:`Notice::Type` - - :Type: :bro:type:`enum` - - .. bro:enum:: Example::Notice_One Notice::Type - - any number of this type of comment - will document "Notice_One" - - .. bro:enum:: Example::Notice_Two Notice::Type - - any number of this type of comment - will document "Notice_Two" - - .. bro:enum:: Example::Notice_Three Notice::Type - - .. bro:enum:: Example::Notice_Four Notice::Type - -Configuration Changes -##################### -Packet Filter -^^^^^^^^^^^^^ -Loading this script makes the following changes to :bro:see:`capture_filters`. - -Filters added:: - - [ssl] = tcp port 443, - [nntps] = tcp port 562 - -Detailed Interface -~~~~~~~~~~~~~~~~~~ -Options -####### -.. bro:id:: Example::an_option - - :Type: :bro:type:`set` [:bro:type:`addr`, :bro:type:`addr`, :bro:type:`string`] - :Attributes: :bro:attr:`&redef` - :Default: ``{}`` - - add documentation for "an_option" here - -.. bro:id:: Example::option_with_init - - :Type: :bro:type:`interval` - :Attributes: :bro:attr:`&redef` - :Default: ``10.0 msecs`` - - More docs can be added here. - -State Variables -############### -.. bro:id:: Example::a_var - - :Type: :bro:type:`bool` - - put some documentation for "a_var" here - -.. bro:id:: Example::var_with_attr - - :Type: :bro:type:`count` - :Attributes: :bro:attr:`&persistent` - -.. bro:id:: Example::var_without_explicit_type - - :Type: :bro:type:`string` - :Default: ``"this works"`` - -.. bro:id:: Example::dummy - - :Type: :bro:type:`string` - - The first.sentence for the summary text ends here. And this second - sentence doesn't show in the short description. - -Types -##### -.. bro:type:: Example::SimpleEnum - - :Type: :bro:type:`enum` - - .. bro:enum:: Example::ONE Example::SimpleEnum - - and more specific info for "ONE" - can span multiple lines - - .. bro:enum:: Example::TWO Example::SimpleEnum - - or more info like this for "TWO" - can span multiple lines - - .. bro:enum:: Example::THREE Example::SimpleEnum - - documentation for "SimpleEnum" - goes here. - -.. bro:type:: Example::SimpleRecord - - :Type: :bro:type:`record` - - field1: :bro:type:`count` - counts something - - field2: :bro:type:`bool` - toggles something - - general documentation for a type "SimpleRecord" - goes here. - -.. bro:type:: Example::ComplexRecord - - :Type: :bro:type:`record` - - field1: :bro:type:`count` - counts something - - field2: :bro:type:`bool` - toggles something - - field3: :bro:type:`Example::SimpleRecord` - - msg: :bro:type:`string` :bro:attr:`&default` = ``"blah"`` :bro:attr:`&optional` - attributes are self-documenting - - general documentation for a type "ComplexRecord" goes here - -.. bro:type:: Example::Info - - :Type: :bro:type:`record` - - ts: :bro:type:`time` :bro:attr:`&log` - - uid: :bro:type:`string` :bro:attr:`&log` - - status: :bro:type:`count` :bro:attr:`&log` :bro:attr:`&optional` - - An example record to be used with a logging stream. - -Events -###### -.. bro:id:: Example::an_event - - :Type: :bro:type:`event` (name: :bro:type:`string`) - - Summarize "an_event" here. - Give more details about "an_event" here. - Example::an_event should not be confused as a parameter. - - :param name: describe the argument here - -.. bro:id:: Example::log_example - - :Type: :bro:type:`event` (rec: :bro:type:`Example::Info`) - - This is a declaration of an example event that can be used in - logging streams and is raised once for each log entry. - -Functions -######### -.. bro:id:: Example::a_function - - :Type: :bro:type:`function` (tag: :bro:type:`string`, msg: :bro:type:`string`) : :bro:type:`string` - - Summarize purpose of "a_function" here. - Give more details about "a_function" here. - Separating the documentation of the params/return values with - empty comments is optional, but improves readability of script. - - - :param tag: function arguments can be described - like this - - :param msg: another param - - - :returns: describe the return type here - -Redefinitions -############# -:bro:type:`Log::ID` - - :Type: :bro:type:`enum` - - .. bro:enum:: Example::LOG Log::ID - -:bro:type:`Example::SimpleEnum` - - :Type: :bro:type:`enum` - - .. bro:enum:: Example::FOUR Example::SimpleEnum - - and some documentation for "FOUR" - - .. bro:enum:: Example::FIVE Example::SimpleEnum - - also "FIVE" for good measure - - document the "SimpleEnum" redef here - -:bro:type:`Example::SimpleRecord` - - :Type: :bro:type:`record` - - field_ext: :bro:type:`string` :bro:attr:`&optional` - document the extending field here - (or here) - - document the record extension redef here - diff --git a/testing/btest/Baseline/doc.autogen-reST-func-params/autogen-reST-func-params.rst b/testing/btest/Baseline/doc.autogen-reST-func-params/autogen-reST-func-params.rst deleted file mode 100644 index 9739d024e3..0000000000 --- a/testing/btest/Baseline/doc.autogen-reST-func-params/autogen-reST-func-params.rst +++ /dev/null @@ -1,58 +0,0 @@ -.. Automatically generated. Do not edit. - -:tocdepth: 3 - -autogen-reST-func-params.bro -============================ - - - - -:Source File: :download:`autogen-reST-func-params.bro` - -Summary -~~~~~~~ -Types -##### -======================================== = -:bro:type:`test_rec`: :bro:type:`record` -======================================== = - -Functions -######### -========================================= ====================================== -:bro:id:`test_func`: :bro:type:`function` This is a global function declaration. -========================================= ====================================== - -Detailed Interface -~~~~~~~~~~~~~~~~~~ -Types -##### -.. bro:type:: test_rec - - :Type: :bro:type:`record` - - field_func: :bro:type:`function` (i: :bro:type:`int`, j: :bro:type:`int`) : :bro:type:`string` - This is a record field function. - - :param i: First param. - :param j: Second param. - - :returns: A string. - -Functions -######### -.. bro:id:: test_func - - :Type: :bro:type:`function` (i: :bro:type:`int`, j: :bro:type:`int`) : :bro:type:`string` - - This is a global function declaration. - - - :param i: First param. - - :param j: Second param. - - - :returns: A string. - diff --git a/testing/btest/Baseline/doc.autogen-reST-records/autogen-reST-records.rst b/testing/btest/Baseline/doc.autogen-reST-records/autogen-reST-records.rst deleted file mode 100644 index 0344fa265c..0000000000 --- a/testing/btest/Baseline/doc.autogen-reST-records/autogen-reST-records.rst +++ /dev/null @@ -1,53 +0,0 @@ -.. Automatically generated. Do not edit. - -:tocdepth: 3 - -autogen-reST-records.bro -======================== - - - - -:Source File: :download:`autogen-reST-records.bro` - -Summary -~~~~~~~ -Types -##### -============================================ ============================================================ -:bro:type:`SimpleRecord`: :bro:type:`record` - -:bro:type:`TestRecord`: :bro:type:`record` Here's the ways records and record fields can be documented. -============================================ ============================================================ - -Detailed Interface -~~~~~~~~~~~~~~~~~~ -Types -##### -.. bro:type:: SimpleRecord - - :Type: :bro:type:`record` - - field1: :bro:type:`bool` - - field2: :bro:type:`count` - -.. bro:type:: TestRecord - - :Type: :bro:type:`record` - - A: :bro:type:`count` - document ``A`` - - B: :bro:type:`bool` - document ``B`` - - C: :bro:type:`SimpleRecord` - and now ``C`` - is a declared type - - D: :bro:type:`set` [:bro:type:`count`, :bro:type:`bool`] - sets/tables should show the index types - - Here's the ways records and record fields can be documented. - diff --git a/testing/btest/Baseline/doc.autogen-reST-type-aliases/autogen-reST-type-aliases.rst b/testing/btest/Baseline/doc.autogen-reST-type-aliases/autogen-reST-type-aliases.rst deleted file mode 100644 index 96a3b9377d..0000000000 --- a/testing/btest/Baseline/doc.autogen-reST-type-aliases/autogen-reST-type-aliases.rst +++ /dev/null @@ -1,62 +0,0 @@ -.. Automatically generated. Do not edit. - -:tocdepth: 3 - -autogen-reST-type-aliases.bro -============================= - - - - -:Source File: :download:`autogen-reST-type-aliases.bro` - -Summary -~~~~~~~ -State Variables -############### -======================================= ======================================================= -:bro:id:`a`: :bro:type:`TypeAlias` But this should reference a type of ``TypeAlias``. - -:bro:id:`b`: :bro:type:`OtherTypeAlias` And this should reference a type of ``OtherTypeAlias``. -======================================= ======================================================= - -Types -##### -============================================ ========================================================================== -:bro:type:`TypeAlias`: :bro:type:`bool` This is just an alias for a builtin type ``bool``. - -:bro:type:`OtherTypeAlias`: :bro:type:`bool` We decided that creating alias "chains" might not be so useful to document - so this type just creates a cross reference to ``bool``. -============================================ ========================================================================== - -Detailed Interface -~~~~~~~~~~~~~~~~~~ -State Variables -############### -.. bro:id:: a - - :Type: :bro:type:`TypeAlias` - - But this should reference a type of ``TypeAlias``. - -.. bro:id:: b - - :Type: :bro:type:`OtherTypeAlias` - - And this should reference a type of ``OtherTypeAlias``. - -Types -##### -.. bro:type:: TypeAlias - - :Type: :bro:type:`bool` - - This is just an alias for a builtin type ``bool``. - -.. bro:type:: OtherTypeAlias - - :Type: :bro:type:`bool` - - We decided that creating alias "chains" might not be so useful to document - so this type just creates a cross reference to ``bool``. - diff --git a/testing/btest/Baseline/doc.broxygen.all_scripts/.stderr b/testing/btest/Baseline/doc.broxygen.all_scripts/.stderr new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testing/btest/Baseline/doc.broxygen.all_scripts/.stdout b/testing/btest/Baseline/doc.broxygen.all_scripts/.stdout new file mode 100644 index 0000000000..bfc3c033df --- /dev/null +++ b/testing/btest/Baseline/doc.broxygen.all_scripts/.stdout @@ -0,0 +1 @@ +WARNING: No Site::local_nets have been defined. It's usually a good idea to define your local networks. diff --git a/testing/btest/Baseline/doc.broxygen.enums/autogen-reST-enums.rst b/testing/btest/Baseline/doc.broxygen.enums/autogen-reST-enums.rst new file mode 100644 index 0000000000..c98d2792df --- /dev/null +++ b/testing/btest/Baseline/doc.broxygen.enums/autogen-reST-enums.rst @@ -0,0 +1,60 @@ +.. bro:type:: TestEnum1 + + :Type: :bro:type:`enum` + + .. bro:enum:: ONE TestEnum1 + + like this + + .. bro:enum:: TWO TestEnum1 + + or like this + + .. bro:enum:: THREE TestEnum1 + + multiple + comments + and even + more comments + + .. bro:enum:: FOUR TestEnum1 + + adding another + value + + .. bro:enum:: FIVE TestEnum1 + + adding another + value + + There's tons of ways an enum can look... + +.. bro:type:: TestEnum2 + + :Type: :bro:type:`enum` + + .. bro:enum:: A TestEnum2 + + like this + + .. bro:enum:: B TestEnum2 + + or like this + + .. bro:enum:: C TestEnum2 + + multiple + comments + and even + more comments + + The final comma is optional + +.. bro:id:: TestEnumVal + + :Type: :bro:type:`TestEnum1` + :Attributes: :bro:attr:`&redef` + :Default: ``ONE`` + + this should reference the TestEnum1 type and not a generic "enum" type + diff --git a/testing/btest/Baseline/doc.broxygen.example/example.rst b/testing/btest/Baseline/doc.broxygen.example/example.rst new file mode 100644 index 0000000000..089a0518e7 --- /dev/null +++ b/testing/btest/Baseline/doc.broxygen.example/example.rst @@ -0,0 +1,249 @@ +:tocdepth: 3 + +broxygen/example.bro +==================== +.. bro:namespace:: BroxygenExample + +This is an example script that demonstrates Broxygen-style +documentation. It generally will make most sense when viewing +the script's raw source code and comparing to the HTML-rendered +version. + +Comments in the from ``##!`` are meant to summarize the script's +purpose. They are transferred directly in to the generated +`reStructuredText `_ +(reST) document associated with the script. + +.. tip:: You can embed directives and roles within ``##``-stylized comments. + +There's also a custom role to reference any identifier node in +the Bro Sphinx domain that's good for "see alsos", e.g. + +See also: :bro:see:`BroxygenExample::a_var`, +:bro:see:`BroxygenExample::ONE`, :bro:see:`SSH::Info` + +And a custom directive does the equivalent references: + +.. bro:see:: BroxygenExample::a_var BroxygenExample::ONE SSH::Info + +:Namespace: BroxygenExample +:Imports: :doc:`base/frameworks/notice `, :doc:`base/protocols/http `, :doc:`policy/frameworks/software/vulnerable.bro ` +:Source File: :download:`/scripts/broxygen/example.bro` + +Summary +~~~~~~~ +Options +####### +==================================================================================== ======================================================= +:bro:id:`BroxygenExample::an_option`: :bro:type:`set` :bro:attr:`&redef` Add documentation for "an_option" here. +:bro:id:`BroxygenExample::option_with_init`: :bro:type:`interval` :bro:attr:`&redef` Default initialization will be generated automatically. +==================================================================================== ======================================================= + +State Variables +############### +======================================================================== ======================================================================== +:bro:id:`BroxygenExample::a_var`: :bro:type:`bool` Put some documentation for "a_var" here. +:bro:id:`BroxygenExample::summary_test`: :bro:type:`string` The first sentence for a particular identifier's summary text ends here. +:bro:id:`BroxygenExample::var_without_explicit_type`: :bro:type:`string` Types are inferred, that information is self-documenting. +======================================================================== ======================================================================== + +Types +##### +================================================================================= =========================================================== +:bro:type:`BroxygenExample::ComplexRecord`: :bro:type:`record` :bro:attr:`&redef` General documentation for a type "ComplexRecord" goes here. +:bro:type:`BroxygenExample::Info`: :bro:type:`record` An example record to be used with a logging stream. +:bro:type:`BroxygenExample::SimpleEnum`: :bro:type:`enum` Documentation for the "SimpleEnum" type goes here. +:bro:type:`BroxygenExample::SimpleRecord`: :bro:type:`record` General documentation for a type "SimpleRecord" goes here. +================================================================================= =========================================================== + +Redefinitions +############# +============================================================= ==================================================================== +:bro:type:`BroxygenExample::SimpleEnum`: :bro:type:`enum` Document the "SimpleEnum" redef here with any special info regarding + the *redef* itself. +:bro:type:`BroxygenExample::SimpleRecord`: :bro:type:`record` Document the record extension *redef* itself here. +:bro:type:`Log::ID`: :bro:type:`enum` +:bro:type:`Notice::Type`: :bro:type:`enum` +============================================================= ==================================================================== + +Events +###### +====================================================== ========================== +:bro:id:`BroxygenExample::an_event`: :bro:type:`event` Summarize "an_event" here. +====================================================== ========================== + +Functions +######### +=========================================================== ======================================= +:bro:id:`BroxygenExample::a_function`: :bro:type:`function` Summarize purpose of "a_function" here. +=========================================================== ======================================= + + +Detailed Interface +~~~~~~~~~~~~~~~~~~ +Options +####### +.. bro:id:: BroxygenExample::an_option + + :Type: :bro:type:`set` [:bro:type:`addr`, :bro:type:`addr`, :bro:type:`string`] + :Attributes: :bro:attr:`&redef` + :Default: ``{}`` + + Add documentation for "an_option" here. + The type/attribute information is all generated automatically. + +.. bro:id:: BroxygenExample::option_with_init + + :Type: :bro:type:`interval` + :Attributes: :bro:attr:`&redef` + :Default: ``10.0 msecs`` + + Default initialization will be generated automatically. + More docs can be added here. + +State Variables +############### +.. bro:id:: BroxygenExample::a_var + + :Type: :bro:type:`bool` + + Put some documentation for "a_var" here. Any global/non-const that + isn't a function/event/hook is classified as a "state variable" + in the generated docs. + +.. bro:id:: BroxygenExample::summary_test + + :Type: :bro:type:`string` + + The first sentence for a particular identifier's summary text ends here. + And this second sentence doesn't show in the short description provided + by the table of all identifiers declared by this script. + +.. bro:id:: BroxygenExample::var_without_explicit_type + + :Type: :bro:type:`string` + :Default: ``"this works"`` + + Types are inferred, that information is self-documenting. + +Types +##### +.. bro:type:: BroxygenExample::ComplexRecord + + :Type: :bro:type:`record` + + field1: :bro:type:`count` + Counts something. + + field2: :bro:type:`bool` + Toggles something. + + field3: :bro:type:`BroxygenExample::SimpleRecord` + Broxygen automatically tracks types + and cross-references are automatically + inserted in to generated docs. + + msg: :bro:type:`string` :bro:attr:`&default` = ``"blah"`` :bro:attr:`&optional` + Attributes are self-documenting. + :Attributes: :bro:attr:`&redef` + + General documentation for a type "ComplexRecord" goes here. + +.. bro:type:: BroxygenExample::Info + + :Type: :bro:type:`record` + + ts: :bro:type:`time` :bro:attr:`&log` + + uid: :bro:type:`string` :bro:attr:`&log` + + status: :bro:type:`count` :bro:attr:`&log` :bro:attr:`&optional` + + An example record to be used with a logging stream. + Nothing special about it. If another script redefs this type + to add fields, the generated documentation will show all original + fields plus the extensions and the scripts which contributed to it + (provided they are also @load'ed). + +.. bro:type:: BroxygenExample::SimpleEnum + + :Type: :bro:type:`enum` + + .. bro:enum:: BroxygenExample::ONE BroxygenExample::SimpleEnum + + Documentation for particular enum values is added like this. + And can also span multiple lines. + + .. bro:enum:: BroxygenExample::TWO BroxygenExample::SimpleEnum + + Or this style is valid to document the preceding enum value. + + .. bro:enum:: BroxygenExample::THREE BroxygenExample::SimpleEnum + + .. bro:enum:: BroxygenExample::FOUR BroxygenExample::SimpleEnum + + And some documentation for "FOUR". + + .. bro:enum:: BroxygenExample::FIVE BroxygenExample::SimpleEnum + + Also "FIVE". + + Documentation for the "SimpleEnum" type goes here. + It can span multiple lines. + +.. bro:type:: BroxygenExample::SimpleRecord + + :Type: :bro:type:`record` + + field1: :bro:type:`count` + Counts something. + + field2: :bro:type:`bool` + Toggles something. + + field_ext: :bro:type:`string` :bro:attr:`&optional` + Document the extending field like this. + Or here, like this. + + General documentation for a type "SimpleRecord" goes here. + The way fields can be documented is similar to what's already seen + for enums. + +Events +###### +.. bro:id:: BroxygenExample::an_event + + :Type: :bro:type:`event` (name: :bro:type:`string`) + + Summarize "an_event" here. + Give more details about "an_event" here. + + BroxygenExample::a_function should not be confused as a parameter + in the generated docs, but it also doesn't generate a cross-reference + link. Use the see role instead: :bro:see:`BroxygenExample::a_function`. + + + :param name: Describe the argument here. + +Functions +######### +.. bro:id:: BroxygenExample::a_function + + :Type: :bro:type:`function` (tag: :bro:type:`string`, msg: :bro:type:`string`) : :bro:type:`string` + + Summarize purpose of "a_function" here. + Give more details about "a_function" here. + Separating the documentation of the params/return values with + empty comments is optional, but improves readability of script. + + + :param tag: Function arguments can be described + like this. + + + :param msg: Another param. + + + :returns: Describe the return type here. + + diff --git a/testing/btest/Baseline/doc.broxygen.func-params/autogen-reST-func-params.rst b/testing/btest/Baseline/doc.broxygen.func-params/autogen-reST-func-params.rst new file mode 100644 index 0000000000..57f12e27c3 --- /dev/null +++ b/testing/btest/Baseline/doc.broxygen.func-params/autogen-reST-func-params.rst @@ -0,0 +1,30 @@ +.. bro:id:: test_func_params_func + + :Type: :bro:type:`function` (i: :bro:type:`int`, j: :bro:type:`int`) : :bro:type:`string` + + This is a global function declaration. + + + :param i: First param. + + :param j: Second param. + + + :returns: A string. + +.. bro:type:: test_func_params_rec + + :Type: :bro:type:`record` + + field_func: :bro:type:`function` (i: :bro:type:`int`, j: :bro:type:`int`) : :bro:type:`string` + This is a record field function. + + + :param i: First param. + + :param j: Second param. + + + :returns: A string. + + diff --git a/testing/btest/Baseline/doc.broxygen.records/autogen-reST-records.rst b/testing/btest/Baseline/doc.broxygen.records/autogen-reST-records.rst new file mode 100644 index 0000000000..60d80f6b07 --- /dev/null +++ b/testing/btest/Baseline/doc.broxygen.records/autogen-reST-records.rst @@ -0,0 +1,28 @@ +.. bro:type:: TestRecord1 + + :Type: :bro:type:`record` + + field1: :bro:type:`bool` + + field2: :bro:type:`count` + + +.. bro:type:: TestRecord2 + + :Type: :bro:type:`record` + + A: :bro:type:`count` + document ``A`` + + B: :bro:type:`bool` + document ``B`` + + C: :bro:type:`TestRecord1` + and now ``C`` + is a declared type + + D: :bro:type:`set` [:bro:type:`count`, :bro:type:`bool`] + sets/tables should show the index types + + Here's the ways records and record fields can be documented. + diff --git a/testing/btest/Baseline/doc.broxygen.type-aliases/autogen-reST-type-aliases.rst b/testing/btest/Baseline/doc.broxygen.type-aliases/autogen-reST-type-aliases.rst new file mode 100644 index 0000000000..3a26b8adc6 --- /dev/null +++ b/testing/btest/Baseline/doc.broxygen.type-aliases/autogen-reST-type-aliases.rst @@ -0,0 +1,44 @@ +.. bro:type:: BroxygenTest::TypeAlias + + :Type: :bro:type:`bool` + + This is just an alias for a builtin type ``bool``. + +.. bro:type:: BroxygenTest::NotTypeAlias + + :Type: :bro:type:`bool` + + This type should get its own comments, not associated w/ TypeAlias. + +.. bro:type:: BroxygenTest::OtherTypeAlias + + :Type: :bro:type:`bool` + + This cross references ``bool`` in the description of its type + instead of ``TypeAlias`` just because it seems more useful -- + one doesn't have to click through the full type alias chain to + find out what the actual type is... + +.. bro:id:: BroxygenTest::a + + :Type: :bro:type:`BroxygenTest::TypeAlias` + + But this should reference a type of ``TypeAlias``. + +.. bro:id:: BroxygenTest::b + + :Type: :bro:type:`BroxygenTest::OtherTypeAlias` + + And this should reference a type of ``OtherTypeAlias``. + +.. bro:type:: BroxygenTest::MyRecord + + :Type: :bro:type:`record` + + f1: :bro:type:`BroxygenTest::TypeAlias` + + f2: :bro:type:`BroxygenTest::OtherTypeAlias` + + f3: :bro:type:`bool` + + diff --git a/testing/btest/doc/autogen-reST-all b/testing/btest/doc/autogen-reST-all deleted file mode 100644 index d6326c7042..0000000000 --- a/testing/btest/doc/autogen-reST-all +++ /dev/null @@ -1 +0,0 @@ -@TEST-EXEC: cd $BUILD && make restdoc diff --git a/testing/btest/doc/autogen-reST-example b/testing/btest/doc/autogen-reST-example deleted file mode 100644 index ecd314eba6..0000000000 --- a/testing/btest/doc/autogen-reST-example +++ /dev/null @@ -1,2 +0,0 @@ -@TEST-EXEC: bro --doc-scripts $DIST/doc/scripts/example.bro -@TEST-EXEC: btest-diff example.rst diff --git a/testing/btest/doc/autogen-reST-type-aliases.bro b/testing/btest/doc/autogen-reST-type-aliases.bro deleted file mode 100644 index ce8986f8be..0000000000 --- a/testing/btest/doc/autogen-reST-type-aliases.bro +++ /dev/null @@ -1,15 +0,0 @@ -# @TEST-EXEC: bro --doc-scripts %INPUT -# @TEST-EXEC: btest-diff autogen-reST-type-aliases.rst - -## This is just an alias for a builtin type ``bool``. -type TypeAlias: bool; - -## We decided that creating alias "chains" might not be so useful to document -## so this type just creates a cross reference to ``bool``. -type OtherTypeAlias: TypeAlias; - -## But this should reference a type of ``TypeAlias``. -global a: TypeAlias; - -## And this should reference a type of ``OtherTypeAlias``. -global b: OtherTypeAlias; diff --git a/testing/btest/doc/broxygen/all_scripts.test b/testing/btest/doc/broxygen/all_scripts.test new file mode 100644 index 0000000000..cb746df9a9 --- /dev/null +++ b/testing/btest/doc/broxygen/all_scripts.test @@ -0,0 +1,10 @@ +# This test is mostly just checking that there's no errors that result +# from loading all scripts and generated docs for each. + +# @TEST-EXEC: bro -X broxygen.config broxygen +# @TEST-EXEC: btest-diff .stdout +# @TEST-EXEC: btest-diff .stderr + +@TEST-START-FILE broxygen.config +script scripts/ * +@TEST-END-FILE diff --git a/testing/btest/doc/autogen-reST-enums.bro b/testing/btest/doc/broxygen/enums.bro similarity index 80% rename from testing/btest/doc/autogen-reST-enums.bro rename to testing/btest/doc/broxygen/enums.bro index e3cceba22e..eeb393ebfb 100644 --- a/testing/btest/doc/autogen-reST-enums.bro +++ b/testing/btest/doc/broxygen/enums.bro @@ -1,6 +1,10 @@ -# @TEST-EXEC: bro --doc-scripts %INPUT +# @TEST-EXEC: bro -b -X broxygen.config %INPUT # @TEST-EXEC: btest-diff autogen-reST-enums.rst +@TEST-START-FILE broxygen.config +identifier autogen-reST-enums.rst TestEnum* +@TEST-END-FILE + ## There's tons of ways an enum can look... type TestEnum1: enum { ## like this @@ -36,4 +40,4 @@ redef enum TestEnum1 += { }; ## this should reference the TestEnum1 type and not a generic "enum" type -const test_enum_option = ONE &redef; +const TestEnumVal = ONE &redef; diff --git a/testing/btest/doc/broxygen/example.bro b/testing/btest/doc/broxygen/example.bro new file mode 100644 index 0000000000..6129c8d3b8 --- /dev/null +++ b/testing/btest/doc/broxygen/example.bro @@ -0,0 +1,8 @@ +# @TEST-EXEC: bro -X broxygen.config %INPUT +# @TEST-EXEC: btest-diff example.rst + +@TEST-START-FILE broxygen.config +script example.rst broxygen/example.bro +@TEST-END-FILE + +@load broxygen/example.bro diff --git a/testing/btest/doc/autogen-reST-func-params.bro b/testing/btest/doc/broxygen/func-params.bro similarity index 56% rename from testing/btest/doc/autogen-reST-func-params.bro rename to testing/btest/doc/broxygen/func-params.bro index 89cf90ef5a..cdb8760935 100644 --- a/testing/btest/doc/autogen-reST-func-params.bro +++ b/testing/btest/doc/broxygen/func-params.bro @@ -1,15 +1,19 @@ -# @TEST-EXEC: bro --doc-scripts %INPUT +# @TEST-EXEC: bro -b -X broxygen.config %INPUT # @TEST-EXEC: btest-diff autogen-reST-func-params.rst +@TEST-START-FILE broxygen.config +identifier autogen-reST-func-params.rst test_func_params* +@TEST-END-FILE + ## This is a global function declaration. ## ## i: First param. ## j: Second param. ## ## Returns: A string. -global test_func: function(i: int, j: int): string; +global test_func_params_func: function(i: int, j: int): string; -type test_rec: record { +type test_func_params_rec: record { ## This is a record field function. ## ## i: First param. diff --git a/testing/btest/doc/autogen-reST-records.bro b/testing/btest/doc/broxygen/records.bro similarity index 58% rename from testing/btest/doc/autogen-reST-records.bro rename to testing/btest/doc/broxygen/records.bro index fc6bb9f2c0..70dccecc58 100644 --- a/testing/btest/doc/autogen-reST-records.bro +++ b/testing/btest/doc/broxygen/records.bro @@ -1,21 +1,25 @@ -# @TEST-EXEC: bro --doc-scripts %INPUT +# @TEST-EXEC: bro -b -X broxygen.config %INPUT # @TEST-EXEC: btest-diff autogen-reST-records.rst +@TEST-START-FILE broxygen.config +identifier autogen-reST-records.rst TestRecord* +@TEST-END-FILE + # undocumented record -type SimpleRecord: record { +type TestRecord1: record { field1: bool; field2: count; }; ## Here's the ways records and record fields can be documented. -type TestRecord: record { +type TestRecord2: record { ## document ``A`` A: count; B: bool; ##< document ``B`` ## and now ``C`` - C: SimpleRecord; ##< is a declared type + C: TestRecord1; ##< is a declared type ## sets/tables should show the index types D: set[count, bool]; diff --git a/testing/btest/doc/broxygen/type-aliases.bro b/testing/btest/doc/broxygen/type-aliases.bro new file mode 100644 index 0000000000..1ab83cc02a --- /dev/null +++ b/testing/btest/doc/broxygen/type-aliases.bro @@ -0,0 +1,34 @@ +# @TEST-EXEC: bro -b -X broxygen.config %INPUT +# @TEST-EXEC: btest-diff autogen-reST-type-aliases.rst + +@TEST-START-FILE broxygen.config +identifier autogen-reST-type-aliases.rst BroxygenTest::* +@TEST-END-FILE + +module BroxygenTest; + +export { + ## This is just an alias for a builtin type ``bool``. + type TypeAlias: bool; + + ## This type should get its own comments, not associated w/ TypeAlias. + type NotTypeAlias: bool; + + ## This cross references ``bool`` in the description of its type + ## instead of ``TypeAlias`` just because it seems more useful -- + ## one doesn't have to click through the full type alias chain to + ## find out what the actual type is... + type OtherTypeAlias: TypeAlias; + + ## But this should reference a type of ``TypeAlias``. + global a: TypeAlias; + + ## And this should reference a type of ``OtherTypeAlias``. + global b: OtherTypeAlias; + + type MyRecord: record { + f1: TypeAlias; + f2: OtherTypeAlias; + f3: bool; + }; +} diff --git a/testing/btest/doc/record-add.bro b/testing/btest/doc/record-add.bro index a326314093..284ea22959 100644 --- a/testing/btest/doc/record-add.bro +++ b/testing/btest/doc/record-add.bro @@ -1,7 +1,7 @@ -# @TEST-EXEC: bro --doc-scripts %INPUT +# @TEST-EXEC: bro -b %INPUT -# When in doc mode, bro will clone declared types (see add_type() in Var.cc) -# in order to keep track of the identifier name associated with the new type. +# To support documentation of type aliases, Bro clones declared types +# (see add_type() in Var.cc) in order to keep track of type names and aliases. # This test makes sure that the cloning is done in a way that's compatible # with adding fields to a record type -- we want to be sure that cloning # a type that contains record types will correctly see field additions to diff --git a/testing/btest/doc/record-attr-check.bro b/testing/btest/doc/record-attr-check.bro index 33ada44bfd..c7dc74631d 100644 --- a/testing/btest/doc/record-attr-check.bro +++ b/testing/btest/doc/record-attr-check.bro @@ -1,4 +1,4 @@ -# @TEST-EXEC: bro --doc-scripts %INPUT +# @TEST-EXEC: bro -b %INPUT type Tag: enum { SOMETHING From e3f74dc319d32b866b40264cc2ff71ba92fec227 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Fri, 22 Nov 2013 14:37:07 -0600 Subject: [PATCH 14/40] Change Broxygen config file format. " " seems a more natural order for specifying targets. --- doc/broxygen.conf.in | 2 +- doc/ext/broxygen.py | 2 +- src/broxygen/Configuration.cc | 2 +- testing/btest/doc/broxygen/all_scripts.test | 2 +- testing/btest/doc/broxygen/enums.bro | 2 +- testing/btest/doc/broxygen/example.bro | 2 +- testing/btest/doc/broxygen/func-params.bro | 2 +- testing/btest/doc/broxygen/records.bro | 2 +- testing/btest/doc/broxygen/type-aliases.bro | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) diff --git a/doc/broxygen.conf.in b/doc/broxygen.conf.in index b27ba6248d..1e70445f58 100644 --- a/doc/broxygen.conf.in +++ b/doc/broxygen.conf.in @@ -1 +1 @@ -script @BROXYGEN_SCRIPT_OUTPUT@/ * +script * @BROXYGEN_SCRIPT_OUTPUT@/ diff --git a/doc/ext/broxygen.py b/doc/ext/broxygen.py index d7a4e57c21..adea390642 100644 --- a/doc/ext/broxygen.py +++ b/doc/ext/broxygen.py @@ -76,7 +76,7 @@ def generate_config(env, type, pattern): prefix = "{0}-{1}-".format(type, pattern_to_filename_component(pattern)) (fd, cfg) = tempfile.mkstemp(suffix=".cfg", prefix=prefix, dir=work_dir) generated_file = "{0}.rst".format(cfg) - config = "{0}\t{1}\t{2}".format(type, generated_file, pattern) + config = "{0}\t{1}\t{2}".format(type, pattern, generated_file) f = os.fdopen(fd, "w") f.write(config) f.close() diff --git a/src/broxygen/Configuration.cc b/src/broxygen/Configuration.cc index 9d0c0f7015..19dc61aa1a 100644 --- a/src/broxygen/Configuration.cc +++ b/src/broxygen/Configuration.cc @@ -457,7 +457,7 @@ Config::Config(const string& arg_file, const string& delim) reporter->FatalError("unkown Broxygen target type: %s", tokens[0].c_str()); - targets.push_back(it->second(tokens[1], tokens[2])); + targets.push_back(it->second(tokens[2], tokens[1])); } if ( f.bad() ) diff --git a/testing/btest/doc/broxygen/all_scripts.test b/testing/btest/doc/broxygen/all_scripts.test index cb746df9a9..56bb5313bb 100644 --- a/testing/btest/doc/broxygen/all_scripts.test +++ b/testing/btest/doc/broxygen/all_scripts.test @@ -6,5 +6,5 @@ # @TEST-EXEC: btest-diff .stderr @TEST-START-FILE broxygen.config -script scripts/ * +script * scripts/ @TEST-END-FILE diff --git a/testing/btest/doc/broxygen/enums.bro b/testing/btest/doc/broxygen/enums.bro index eeb393ebfb..e8b4c741c2 100644 --- a/testing/btest/doc/broxygen/enums.bro +++ b/testing/btest/doc/broxygen/enums.bro @@ -2,7 +2,7 @@ # @TEST-EXEC: btest-diff autogen-reST-enums.rst @TEST-START-FILE broxygen.config -identifier autogen-reST-enums.rst TestEnum* +identifier TestEnum* autogen-reST-enums.rst @TEST-END-FILE ## There's tons of ways an enum can look... diff --git a/testing/btest/doc/broxygen/example.bro b/testing/btest/doc/broxygen/example.bro index 6129c8d3b8..e7212f3c5f 100644 --- a/testing/btest/doc/broxygen/example.bro +++ b/testing/btest/doc/broxygen/example.bro @@ -2,7 +2,7 @@ # @TEST-EXEC: btest-diff example.rst @TEST-START-FILE broxygen.config -script example.rst broxygen/example.bro +script broxygen/example.bro example.rst @TEST-END-FILE @load broxygen/example.bro diff --git a/testing/btest/doc/broxygen/func-params.bro b/testing/btest/doc/broxygen/func-params.bro index cdb8760935..42d1308151 100644 --- a/testing/btest/doc/broxygen/func-params.bro +++ b/testing/btest/doc/broxygen/func-params.bro @@ -2,7 +2,7 @@ # @TEST-EXEC: btest-diff autogen-reST-func-params.rst @TEST-START-FILE broxygen.config -identifier autogen-reST-func-params.rst test_func_params* +identifier test_func_params* autogen-reST-func-params.rst @TEST-END-FILE ## This is a global function declaration. diff --git a/testing/btest/doc/broxygen/records.bro b/testing/btest/doc/broxygen/records.bro index 70dccecc58..0cc7d27500 100644 --- a/testing/btest/doc/broxygen/records.bro +++ b/testing/btest/doc/broxygen/records.bro @@ -2,7 +2,7 @@ # @TEST-EXEC: btest-diff autogen-reST-records.rst @TEST-START-FILE broxygen.config -identifier autogen-reST-records.rst TestRecord* +identifier TestRecord* autogen-reST-records.rst @TEST-END-FILE # undocumented record diff --git a/testing/btest/doc/broxygen/type-aliases.bro b/testing/btest/doc/broxygen/type-aliases.bro index 1ab83cc02a..28c2cc5568 100644 --- a/testing/btest/doc/broxygen/type-aliases.bro +++ b/testing/btest/doc/broxygen/type-aliases.bro @@ -2,7 +2,7 @@ # @TEST-EXEC: btest-diff autogen-reST-type-aliases.rst @TEST-START-FILE broxygen.config -identifier autogen-reST-type-aliases.rst BroxygenTest::* +identifier BroxygenTest::* autogen-reST-type-aliases.rst @TEST-END-FILE module BroxygenTest; From 27138b893a4db4344313f052c621222fe5be2d2e Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Fri, 22 Nov 2013 14:59:16 -0600 Subject: [PATCH 15/40] Add unit tests for Broxygen config file targets. --- .../Baseline/doc.broxygen.identifier/test.rst | 230 ++++++++++++++++++ .../Baseline/doc.broxygen.package/test.rst | 35 +++ .../doc.broxygen.package_index/test.rst | 7 + .../doc.broxygen.script_index/test.rst | 5 + .../doc.broxygen.script_summary/test.rst | 23 ++ testing/btest/doc/broxygen/identifier.bro | 8 + testing/btest/doc/broxygen/package.bro | 8 + testing/btest/doc/broxygen/package_index.bro | 8 + testing/btest/doc/broxygen/script_index.bro | 8 + testing/btest/doc/broxygen/script_summary.bro | 8 + 10 files changed, 340 insertions(+) create mode 100644 testing/btest/Baseline/doc.broxygen.identifier/test.rst create mode 100644 testing/btest/Baseline/doc.broxygen.package/test.rst create mode 100644 testing/btest/Baseline/doc.broxygen.package_index/test.rst create mode 100644 testing/btest/Baseline/doc.broxygen.script_index/test.rst create mode 100644 testing/btest/Baseline/doc.broxygen.script_summary/test.rst create mode 100644 testing/btest/doc/broxygen/identifier.bro create mode 100644 testing/btest/doc/broxygen/package.bro create mode 100644 testing/btest/doc/broxygen/package_index.bro create mode 100644 testing/btest/doc/broxygen/script_index.bro create mode 100644 testing/btest/doc/broxygen/script_summary.bro diff --git a/testing/btest/Baseline/doc.broxygen.identifier/test.rst b/testing/btest/Baseline/doc.broxygen.identifier/test.rst new file mode 100644 index 0000000000..539a03c92a --- /dev/null +++ b/testing/btest/Baseline/doc.broxygen.identifier/test.rst @@ -0,0 +1,230 @@ +.. bro:id:: BroxygenExample::Broxygen_One + + :Type: :bro:type:`Notice::Type` + + Any number of this type of comment + will document "Broxygen_One". + +.. bro:id:: BroxygenExample::Broxygen_Two + + :Type: :bro:type:`Notice::Type` + + Any number of this type of comment + will document "BROXYGEN_TWO". + +.. bro:id:: BroxygenExample::Broxygen_Three + + :Type: :bro:type:`Notice::Type` + + +.. bro:id:: BroxygenExample::Broxygen_Four + + :Type: :bro:type:`Notice::Type` + + Omitting comments is fine, and so is mixing ``##`` and ``##<``, but + it's probably best to use only one style consistently. + +.. bro:id:: BroxygenExample::LOG + + :Type: :bro:type:`Log::ID` + + +.. bro:type:: BroxygenExample::SimpleEnum + + :Type: :bro:type:`enum` + + .. bro:enum:: BroxygenExample::ONE BroxygenExample::SimpleEnum + + Documentation for particular enum values is added like this. + And can also span multiple lines. + + .. bro:enum:: BroxygenExample::TWO BroxygenExample::SimpleEnum + + Or this style is valid to document the preceding enum value. + + .. bro:enum:: BroxygenExample::THREE BroxygenExample::SimpleEnum + + .. bro:enum:: BroxygenExample::FOUR BroxygenExample::SimpleEnum + + And some documentation for "FOUR". + + .. bro:enum:: BroxygenExample::FIVE BroxygenExample::SimpleEnum + + Also "FIVE". + + Documentation for the "SimpleEnum" type goes here. + It can span multiple lines. + +.. bro:id:: BroxygenExample::ONE + + :Type: :bro:type:`BroxygenExample::SimpleEnum` + + Documentation for particular enum values is added like this. + And can also span multiple lines. + +.. bro:id:: BroxygenExample::TWO + + :Type: :bro:type:`BroxygenExample::SimpleEnum` + + Or this style is valid to document the preceding enum value. + +.. bro:id:: BroxygenExample::THREE + + :Type: :bro:type:`BroxygenExample::SimpleEnum` + + +.. bro:id:: BroxygenExample::FOUR + + :Type: :bro:type:`BroxygenExample::SimpleEnum` + + And some documentation for "FOUR". + +.. bro:id:: BroxygenExample::FIVE + + :Type: :bro:type:`BroxygenExample::SimpleEnum` + + Also "FIVE". + +.. bro:type:: BroxygenExample::SimpleRecord + + :Type: :bro:type:`record` + + field1: :bro:type:`count` + Counts something. + + field2: :bro:type:`bool` + Toggles something. + + field_ext: :bro:type:`string` :bro:attr:`&optional` + Document the extending field like this. + Or here, like this. + + General documentation for a type "SimpleRecord" goes here. + The way fields can be documented is similar to what's already seen + for enums. + +.. bro:type:: BroxygenExample::ComplexRecord + + :Type: :bro:type:`record` + + field1: :bro:type:`count` + Counts something. + + field2: :bro:type:`bool` + Toggles something. + + field3: :bro:type:`BroxygenExample::SimpleRecord` + Broxygen automatically tracks types + and cross-references are automatically + inserted in to generated docs. + + msg: :bro:type:`string` :bro:attr:`&default` = ``"blah"`` :bro:attr:`&optional` + Attributes are self-documenting. + :Attributes: :bro:attr:`&redef` + + General documentation for a type "ComplexRecord" goes here. + +.. bro:type:: BroxygenExample::Info + + :Type: :bro:type:`record` + + ts: :bro:type:`time` :bro:attr:`&log` + + uid: :bro:type:`string` :bro:attr:`&log` + + status: :bro:type:`count` :bro:attr:`&log` :bro:attr:`&optional` + + An example record to be used with a logging stream. + Nothing special about it. If another script redefs this type + to add fields, the generated documentation will show all original + fields plus the extensions and the scripts which contributed to it + (provided they are also @load'ed). + +.. bro:id:: BroxygenExample::an_option + + :Type: :bro:type:`set` [:bro:type:`addr`, :bro:type:`addr`, :bro:type:`string`] + :Attributes: :bro:attr:`&redef` + :Default: ``{}`` + + Add documentation for "an_option" here. + The type/attribute information is all generated automatically. + +.. bro:id:: BroxygenExample::option_with_init + + :Type: :bro:type:`interval` + :Attributes: :bro:attr:`&redef` + :Default: ``10.0 msecs`` + + Default initialization will be generated automatically. + More docs can be added here. + +.. bro:id:: BroxygenExample::a_var + + :Type: :bro:type:`bool` + + Put some documentation for "a_var" here. Any global/non-const that + isn't a function/event/hook is classified as a "state variable" + in the generated docs. + +.. bro:id:: BroxygenExample::var_without_explicit_type + + :Type: :bro:type:`string` + :Default: ``"this works"`` + + Types are inferred, that information is self-documenting. + +.. bro:id:: BroxygenExample::summary_test + + :Type: :bro:type:`string` + + The first sentence for a particular identifier's summary text ends here. + And this second sentence doesn't show in the short description provided + by the table of all identifiers declared by this script. + +.. bro:id:: BroxygenExample::a_function + + :Type: :bro:type:`function` (tag: :bro:type:`string`, msg: :bro:type:`string`) : :bro:type:`string` + + Summarize purpose of "a_function" here. + Give more details about "a_function" here. + Separating the documentation of the params/return values with + empty comments is optional, but improves readability of script. + + + :param tag: Function arguments can be described + like this. + + + :param msg: Another param. + + + :returns: Describe the return type here. + +.. bro:id:: BroxygenExample::an_event + + :Type: :bro:type:`event` (name: :bro:type:`string`) + + Summarize "an_event" here. + Give more details about "an_event" here. + + BroxygenExample::a_function should not be confused as a parameter + in the generated docs, but it also doesn't generate a cross-reference + link. Use the see role instead: :bro:see:`BroxygenExample::a_function`. + + + :param name: Describe the argument here. + +.. bro:id:: BroxygenExample::function_without_proto + + :Type: :bro:type:`function` (tag: :bro:type:`string`) : :bro:type:`string` + + +.. bro:type:: BroxygenExample::PrivateRecord + + :Type: :bro:type:`record` + + field1: :bro:type:`bool` + + field2: :bro:type:`count` + + diff --git a/testing/btest/Baseline/doc.broxygen.package/test.rst b/testing/btest/Baseline/doc.broxygen.package/test.rst new file mode 100644 index 0000000000..d65f563fd5 --- /dev/null +++ b/testing/btest/Baseline/doc.broxygen.package/test.rst @@ -0,0 +1,35 @@ +:orphan: + +Package: broxygen +================= + +This package is loaded during the process which automatically generates +reference documentation for all Bro scripts (i.e. "Broxygen"). Its only +purpose is to provide an easy way to load all known Bro scripts plus any +extra scripts needed or used by the documentation process. + +:doc:`/scripts/broxygen/__load__.bro` + +:doc:`/scripts/broxygen/example.bro` + This is an example script that demonstrates Broxygen-style + documentation. It generally will make most sense when viewing + the script's raw source code and comparing to the HTML-rendered + version. + + Comments in the from ``##!`` are meant to summarize the script's + purpose. They are transferred directly in to the generated + `reStructuredText `_ + (reST) document associated with the script. + + .. tip:: You can embed directives and roles within ``##``-stylized comments. + + There's also a custom role to reference any identifier node in + the Bro Sphinx domain that's good for "see alsos", e.g. + + See also: :bro:see:`BroxygenExample::a_var`, + :bro:see:`BroxygenExample::ONE`, :bro:see:`SSH::Info` + + And a custom directive does the equivalent references: + + .. bro:see:: BroxygenExample::a_var BroxygenExample::ONE SSH::Info + diff --git a/testing/btest/Baseline/doc.broxygen.package_index/test.rst b/testing/btest/Baseline/doc.broxygen.package_index/test.rst new file mode 100644 index 0000000000..f551ab1cd3 --- /dev/null +++ b/testing/btest/Baseline/doc.broxygen.package_index/test.rst @@ -0,0 +1,7 @@ +:doc:`broxygen ` + + This package is loaded during the process which automatically generates + reference documentation for all Bro scripts (i.e. "Broxygen"). Its only + purpose is to provide an easy way to load all known Bro scripts plus any + extra scripts needed or used by the documentation process. + diff --git a/testing/btest/Baseline/doc.broxygen.script_index/test.rst b/testing/btest/Baseline/doc.broxygen.script_index/test.rst new file mode 100644 index 0000000000..dda280facf --- /dev/null +++ b/testing/btest/Baseline/doc.broxygen.script_index/test.rst @@ -0,0 +1,5 @@ +.. toctree:: + :maxdepth: 1 + + broxygen/__load__.bro + broxygen/example.bro diff --git a/testing/btest/Baseline/doc.broxygen.script_summary/test.rst b/testing/btest/Baseline/doc.broxygen.script_summary/test.rst new file mode 100644 index 0000000000..125a579c81 --- /dev/null +++ b/testing/btest/Baseline/doc.broxygen.script_summary/test.rst @@ -0,0 +1,23 @@ +:doc:`/scripts/broxygen/example.bro` + This is an example script that demonstrates Broxygen-style + documentation. It generally will make most sense when viewing + the script's raw source code and comparing to the HTML-rendered + version. + + Comments in the from ``##!`` are meant to summarize the script's + purpose. They are transferred directly in to the generated + `reStructuredText `_ + (reST) document associated with the script. + + .. tip:: You can embed directives and roles within ``##``-stylized comments. + + There's also a custom role to reference any identifier node in + the Bro Sphinx domain that's good for "see alsos", e.g. + + See also: :bro:see:`BroxygenExample::a_var`, + :bro:see:`BroxygenExample::ONE`, :bro:see:`SSH::Info` + + And a custom directive does the equivalent references: + + .. bro:see:: BroxygenExample::a_var BroxygenExample::ONE SSH::Info + diff --git a/testing/btest/doc/broxygen/identifier.bro b/testing/btest/doc/broxygen/identifier.bro new file mode 100644 index 0000000000..3768b0c0c6 --- /dev/null +++ b/testing/btest/doc/broxygen/identifier.bro @@ -0,0 +1,8 @@ +# @TEST-EXEC: bro -b -X broxygen.config %INPUT +# @TEST-EXEC: btest-diff test.rst + +@TEST-START-FILE broxygen.config +identifier BroxygenExample::* test.rst +@TEST-END-FILE + +@load broxygen diff --git a/testing/btest/doc/broxygen/package.bro b/testing/btest/doc/broxygen/package.bro new file mode 100644 index 0000000000..6857d5e646 --- /dev/null +++ b/testing/btest/doc/broxygen/package.bro @@ -0,0 +1,8 @@ +# @TEST-EXEC: bro -b -X broxygen.config %INPUT +# @TEST-EXEC: btest-diff test.rst + +@TEST-START-FILE broxygen.config +package broxygen test.rst +@TEST-END-FILE + +@load broxygen diff --git a/testing/btest/doc/broxygen/package_index.bro b/testing/btest/doc/broxygen/package_index.bro new file mode 100644 index 0000000000..e29479d49f --- /dev/null +++ b/testing/btest/doc/broxygen/package_index.bro @@ -0,0 +1,8 @@ +# @TEST-EXEC: bro -b -X broxygen.config %INPUT +# @TEST-EXEC: btest-diff test.rst + +@TEST-START-FILE broxygen.config +package_index broxygen test.rst +@TEST-END-FILE + +@load broxygen diff --git a/testing/btest/doc/broxygen/script_index.bro b/testing/btest/doc/broxygen/script_index.bro new file mode 100644 index 0000000000..91bb4b756f --- /dev/null +++ b/testing/btest/doc/broxygen/script_index.bro @@ -0,0 +1,8 @@ +# @TEST-EXEC: bro -b -X broxygen.config %INPUT +# @TEST-EXEC: btest-diff test.rst + +@TEST-START-FILE broxygen.config +script_index broxygen/* test.rst +@TEST-END-FILE + +@load broxygen diff --git a/testing/btest/doc/broxygen/script_summary.bro b/testing/btest/doc/broxygen/script_summary.bro new file mode 100644 index 0000000000..9d3cda012b --- /dev/null +++ b/testing/btest/doc/broxygen/script_summary.bro @@ -0,0 +1,8 @@ +# @TEST-EXEC: bro -b -X broxygen.config %INPUT +# @TEST-EXEC: btest-diff test.rst + +@TEST-START-FILE broxygen.config +script_summary broxygen/example.bro test.rst +@TEST-END-FILE + +@load broxygen From 08c7dd3d717bc159f7ad8dcbd754af1b9e7e6a09 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Sat, 28 Sep 2013 16:06:40 -0700 Subject: [PATCH 16/40] Prettyfing Describe() for record types. If a record type has a name and ODesc is set to short, we now print the name instead of the full field list. --- src/Type.cc | 14 ++++++++++---- src/Var.cc | 2 +- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/src/Type.cc b/src/Type.cc index a6d8b90c6c..30deb3cd2a 100644 --- a/src/Type.cc +++ b/src/Type.cc @@ -1038,10 +1038,16 @@ void RecordType::Describe(ODesc* d) const { if ( d->IsReadable() ) { - d->AddSP("record {"); - DescribeFields(d); - d->SP(); - d->Add("}"); + if ( d->IsShort() && GetTypeID() ) + d->Add(GetTypeID()); + + else + { + d->AddSP("record {"); + DescribeFields(d); + d->SP(); + d->Add("}"); + } } else diff --git a/src/Var.cc b/src/Var.cc index d384fedc74..08129bdc3e 100644 --- a/src/Var.cc +++ b/src/Var.cc @@ -314,9 +314,9 @@ void add_type(ID* id, BroType* t, attr_list* attr, int /* is_event */) delete [] data; } - tnew->SetTypeID(copy_string(id->Name())); } + tnew->SetTypeID(copy_string(id->Name())); id->SetType(tnew); id->MakeType(); From de9f03b0bffae6cc89d5a4430bba8db7550e881f Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Fri, 22 Nov 2013 14:17:15 -0800 Subject: [PATCH 17/40] New script misc/dump-events.bro, along with core support, that dumps events Bro is raising in an easily readable form. This is for debugging purposes, obviously. Example, including only SMTP events: > bro -r smtp.trace misc/dump-events.bro DumpEvents::include=/smtp/ [...] 1254722768.219663 smtp_reply [0] c: connection = [id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, [...] [1] is_orig: bool = F [2] code: count = 220 [3] cmd: string = > [4] msg: string = xc90.websitewelcome.com ESMTP Exim 4.69 #1 Mon, 05 Oct 2009 01:05:54 -0500 [5] cont_resp: bool = T 1254722768.219663 smtp_reply [0] c: connection = [id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, [...] [1] is_orig: bool = F [2] code: count = 220 [3] cmd: string = > [4] msg: string = We do not authorize the use of this system to transport unsolicited, [5] cont_resp: bool = T [...] --- scripts/base/init-bare.bro | 18 ++++++++++ scripts/policy/misc/dump-events.bro | 35 +++++++++++++++++++ src/EventHandler.cc | 54 +++++++++++++++++++++++++++++ src/EventHandler.h | 2 ++ src/NetVar.cc | 4 +++ src/NetVar.h | 2 ++ src/event.bif | 11 ++++++ 7 files changed, 126 insertions(+) create mode 100644 scripts/policy/misc/dump-events.bro diff --git a/scripts/base/init-bare.bro b/scripts/base/init-bare.bro index de26e6a41d..1cfc570a6e 100644 --- a/scripts/base/init-bare.bro +++ b/scripts/base/init-bare.bro @@ -529,6 +529,24 @@ type record_field: record { ## directly and then remove this alias. type record_field_table: table[string] of record_field; +## Meta-information about a parameter to a function/event. +## +## .. bro:see:: call_argument_event new_event +type call_argument: record { + name: string; ##< The name of the parameter. + type_name: string; ##< The name of the parameters's type. + default_val: any &optional; ##< The value of the :bro:attr:`&default` attribute if defined. + + ## The value of the parameter as passed into a given call instance. Might be unset + ## in the case a :bro:attr:`&default` attribute is defined. + value: any &optional; +}; + +## Vector type used to capture parameters of a function/event call. +## +## .. bro:see:: call_argument new_event +type call_argument_vector: vector of call_argument; + # todo:: Do we still need these here? Can they move into the packet filter # framework? # diff --git a/scripts/policy/misc/dump-events.bro b/scripts/policy/misc/dump-events.bro new file mode 100644 index 0000000000..a88f980b2e --- /dev/null +++ b/scripts/policy/misc/dump-events.bro @@ -0,0 +1,35 @@ + +module DumpEvents; + +export { + # If true, include event argument in output. + const include_args = T &redef; + + # Only include events matching the given pattern into output. + const include = /.*/ &redef; +} + +event new_event(name: string, args: call_argument_vector) + { + if ( include !in name ) + return; + + if ( ! include_args || |args| == 0 ) + return; + + print fmt("%.6f %s", network_time(), name); + + for ( i in args ) + { + local a = args[i]; + + local proto = fmt("%s: %s", a$name, a$type_name); + + if ( a?$value ) + print fmt(" [%d] %-15s = %s", i, proto, a$value); + else + print fmt(" | %-15s = %s [default]", proto, a$value); + } + + print ""; + } diff --git a/src/EventHandler.cc b/src/EventHandler.cc index 4a74d68a08..43940e0d27 100644 --- a/src/EventHandler.cc +++ b/src/EventHandler.cc @@ -3,6 +3,7 @@ #include "Func.h" #include "Scope.h" #include "RemoteSerializer.h" +#include "NetVar.h" EventHandler::EventHandler(const char* arg_name) { @@ -56,6 +57,9 @@ void EventHandler::Call(val_list* vl, bool no_remote) DEBUG_MSG("Event: %s\n", Name()); #endif + if ( new_event ) + NewEvent(vl); + if ( ! no_remote ) { loop_over_list(receivers, i) @@ -75,6 +79,56 @@ void EventHandler::Call(val_list* vl, bool no_remote) } } +void EventHandler::NewEvent(val_list* vl) + { + if ( ! new_event ) + return; + + if ( this == new_event.Ptr() ) + return; + + RecordType* args = FType()->Args(); + VectorVal* vargs = new VectorVal(call_argument_vector); + + for ( int i = 0; i < args->NumFields(); i++ ) + { + const char* fname = args->FieldName(i); + BroType* ftype = args->FieldType(i); + Val* fdefault = args->FieldDefault(i); + + RecordVal* rec = new RecordVal(call_argument); + + rec->Assign(0, new StringVal(fname)); + + ODesc d; + d.SetShort(); + ftype->Describe(&d); + rec->Assign(1, new StringVal(d.Description())); + + if ( fdefault ) + { + Ref(fdefault); + rec->Assign(2, fdefault); + } + + if ( i < vl->length() && (*vl)[i] ) + { + Val* val = (*vl)[i]; + Ref(val); + rec->Assign(3, val); + } + + vargs->Assign(i, rec); + } + + val_list* mvl = new val_list(2); + mvl->append(new StringVal(name)); + mvl->append(vargs); + + Event* ev = new Event(new_event, mvl); + mgr.Dispatch(ev); + } + void EventHandler::AddRemoteHandler(SourceID peer) { receivers.append(peer); diff --git a/src/EventHandler.h b/src/EventHandler.h index 786d9f94ba..e84f635175 100644 --- a/src/EventHandler.h +++ b/src/EventHandler.h @@ -49,6 +49,8 @@ public: static EventHandler* Unserialize(UnserialInfo* info); private: + void NewEvent(val_list* vl); // Raise new_event() meta event. + const char* name; Func* local; FuncType* type; diff --git a/src/NetVar.cc b/src/NetVar.cc index 7a11c3f2d1..79652112f3 100644 --- a/src/NetVar.cc +++ b/src/NetVar.cc @@ -235,6 +235,8 @@ RecordType* script_id; TableType* id_table; RecordType* record_field; TableType* record_field_table; +RecordType* call_argument; +VectorType* call_argument_vector; StringVal* cmd_line_bpf_filter; @@ -528,4 +530,6 @@ void init_net_var() id_table = internal_type("id_table")->AsTableType(); record_field = internal_type("record_field")->AsRecordType(); record_field_table = internal_type("record_field_table")->AsTableType(); + call_argument_vector = internal_type("call_argument_vector")->AsVectorType(); + call_argument = internal_type("call_argument")->AsRecordType(); } diff --git a/src/NetVar.h b/src/NetVar.h index c30895d5d4..12949c0e55 100644 --- a/src/NetVar.h +++ b/src/NetVar.h @@ -239,6 +239,8 @@ extern RecordType* script_id; extern TableType* id_table; extern RecordType* record_field; extern TableType* record_field_table; +extern RecordType* call_argument; +extern VectorType* call_argument_vector; extern StringVal* cmd_line_bpf_filter; diff --git a/src/event.bif b/src/event.bif index ddadb47f8a..fc2b9c4d5d 100644 --- a/src/event.bif +++ b/src/event.bif @@ -1007,6 +1007,17 @@ event dns_mapping_lost_name%(dm: dns_mapping%); ## dns_mapping_valid event dns_mapping_altered%(dm: dns_mapping, old_addrs: addr_set, new_addrs: addr_set%); +## A meta event generated for each event Bro raises. This will report *all* events, +## even those for which no handler is defined. +## +## Note that handling this meta event is expensive and should be limited to +## debugging purposes. +## +## name: The name of the event. +## +## params: The event's parameters. +event new_event%(name: string, params: call_argument_vector%); + ## Deprecated. Will be removed. event root_backdoor_signature_found%(c: connection%); From e58865af222711e3395d2cd05de3d0faafaeb8b4 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Mon, 25 Nov 2013 14:36:05 -0600 Subject: [PATCH 18/40] Internal Broxygen organization/documentation/polish. --- src/BroDoc.cc | 667 -------------------------- src/BroDoc.h | 422 ---------------- src/BroDocObj.cc | 195 -------- src/BroDocObj.h | 143 ------ src/CMakeLists.txt | 2 - src/Type.cc | 10 +- src/broxygen/CMakeLists.txt | 9 +- src/broxygen/Configuration.cc | 435 +---------------- src/broxygen/Configuration.h | 247 ++-------- src/broxygen/Document.cc | 660 ------------------------- src/broxygen/Document.h | 215 --------- src/broxygen/IdentifierInfo.cc | 146 ++++++ src/broxygen/IdentifierInfo.h | 162 +++++++ src/broxygen/Info.h | 71 +++ src/broxygen/Manager.cc | 106 ++-- src/broxygen/Manager.h | 164 ++++++- src/broxygen/PackageInfo.cc | 55 +++ src/broxygen/PackageInfo.h | 48 ++ src/broxygen/ReStructuredTextTable.cc | 66 +++ src/broxygen/ReStructuredTextTable.h | 51 ++ src/broxygen/ScriptInfo.cc | 361 ++++++++++++++ src/broxygen/ScriptInfo.h | 121 +++++ src/broxygen/Target.cc | 595 +++++++++++++++++++++++ src/broxygen/Target.h | 387 +++++++++++++++ src/broxygen/broxygen.bif | 41 +- src/broxygen/utils.cc | 56 +++ src/broxygen/utils.h | 47 ++ src/main.cc | 1 - src/scan.l | 2 +- 29 files changed, 2461 insertions(+), 3024 deletions(-) delete mode 100644 src/BroDoc.cc delete mode 100644 src/BroDoc.h delete mode 100644 src/BroDocObj.cc delete mode 100644 src/BroDocObj.h delete mode 100644 src/broxygen/Document.cc delete mode 100644 src/broxygen/Document.h create mode 100644 src/broxygen/IdentifierInfo.cc create mode 100644 src/broxygen/IdentifierInfo.h create mode 100644 src/broxygen/Info.h create mode 100644 src/broxygen/PackageInfo.cc create mode 100644 src/broxygen/PackageInfo.h create mode 100644 src/broxygen/ReStructuredTextTable.cc create mode 100644 src/broxygen/ReStructuredTextTable.h create mode 100644 src/broxygen/ScriptInfo.cc create mode 100644 src/broxygen/ScriptInfo.h create mode 100644 src/broxygen/Target.cc create mode 100644 src/broxygen/Target.h diff --git a/src/BroDoc.cc b/src/BroDoc.cc deleted file mode 100644 index 1239c4efce..0000000000 --- a/src/BroDoc.cc +++ /dev/null @@ -1,667 +0,0 @@ -#include -#include -#include -#include -#include -#include - -#include "BroDoc.h" -#include "BroDocObj.h" -#include "util.h" -#include "plugin/Manager.h" -#include "analyzer/Manager.h" -#include "analyzer/Component.h" -#include "file_analysis/Manager.h" -#include "broxygen/Manager.h" - -BroDoc::BroDoc(const std::string& rel, const std::string& abs) - { - size_t f_pos = abs.find_last_of('/'); - - if ( std::string::npos == f_pos ) - source_filename = abs; - else - source_filename = abs.substr(f_pos + 1); - - if ( rel[0] == '/' || rel[0] == '.' ) - { - // The Bro script isn't being loaded via BROPATH, so just use basename - // as the document title. - doc_title = source_filename; - } - else - { - // Keep the relative directory as part of the document title. - if ( rel.size() == 0 || rel[rel.size() - 1] == '/' ) - doc_title = rel + source_filename; - else - doc_title = rel + "/" + source_filename; - } - - downloadable_filename = source_filename; - -#if 0 - size_t ext_pos = downloadable_filename.find(".bif.bro"); - if ( std::string::npos != ext_pos ) - downloadable_filename.erase(ext_pos + 4); -#endif - - reST_filename = doc_title; - size_t ext_pos = reST_filename.find(".bro"); - - if ( std::string::npos == ext_pos ) - reST_filename += ".rst"; - else - reST_filename.replace(ext_pos, 4, ".rst"); - - reST_filename = doc_title.substr(0, ext_pos); - reST_filename += ".rst"; - - // Instead of re-creating the directory hierarchy based on related - // loads, just replace the directory separatories such that the reST - // output will all be placed in a flat directory (the working dir). - std::for_each(reST_filename.begin(), reST_filename.end(), replace_slash()); - - reST_file = fopen(reST_filename.c_str(), "w"); - - if ( ! reST_file ) - fprintf(stderr, "Failed to open %s\n", reST_filename.c_str()); - -#ifdef DOCDEBUG - fprintf(stdout, "Documenting absolute source: %s\n", abs.c_str()); - fprintf(stdout, "\trelative dir: %s\n", rel.c_str()); - fprintf(stdout, "\tdoc title: %s\n", doc_title.c_str()); - fprintf(stdout, "\tbro file: %s\n", source_filename.c_str()); - fprintf(stdout, "\trst file: %s\n", reST_filename.c_str()); -#endif - } - -BroDoc::~BroDoc() - { - if ( reST_file && fclose( reST_file ) ) - fprintf(stderr, "Failed to close %s\n", reST_filename.c_str()); - - FreeBroDocObjPtrList(all); - } - -void BroDoc::AddImport(const std::string& s) - { - /* - std::string lname(s); - // First strip any .bro extension. - size_t ext_pos = lname.find(".bro"); - if ( ext_pos != std::string::npos ) - lname = lname.substr(0, ext_pos); - - const char* full_filename = NULL; - const char* subpath = NULL; - - FILE* f = search_for_file(lname.c_str(), "bro", &full_filename, true, - &subpath); - - if ( f && full_filename && subpath ) - { - char* tmp = copy_string(full_filename); - char* filename = basename(tmp); - extern char* PACKAGE_LOADER; - - if ( streq(filename, PACKAGE_LOADER) ) - { - // link to the package's index - string pkg(subpath); - pkg += "/index"; - imports.push_back(pkg); - } - else - { - if ( subpath[0] == '/' || subpath[0] == '.' ) - { - // it's not a subpath of scripts/, so just add the name of it - // as it's given in the @load directive - imports.push_back(lname); - } - else - { - // combine the base file name of script in the @load directive - // with the subpath of BROPATH's scripts/ directory - string fname(subpath); - char* othertmp = copy_string(lname.c_str()); - fname.append("/").append(basename(othertmp)); - imports.push_back(fname); - delete [] othertmp; - } - } - - delete [] tmp; - } - - else - fprintf(stderr, "Failed to document '@load %s' in file: %s\n", - s.c_str(), reST_filename.c_str()); - - if ( f ) - fclose(f); - - delete [] full_filename; - delete [] subpath; - */ - } - -void BroDoc::SetPacketFilter(const std::string& s) - { - packet_filter = s; - size_t pos1 = s.find("{\n"); - size_t pos2 = s.find("}"); - - if ( pos1 != std::string::npos && pos2 != std::string::npos ) - packet_filter = s.substr(pos1 + 2, pos2 - 2); - - bool has_non_whitespace = false; - - for ( std::string::const_iterator it = packet_filter.begin(); - it != packet_filter.end(); ++it ) - { - if ( *it != ' ' && *it != '\t' && *it != '\n' && *it != '\r' ) - { - has_non_whitespace = true; - break; - } - } - - if ( ! has_non_whitespace ) - packet_filter.clear(); - } - -void BroDoc::WriteDocFile() const - { - WriteToDoc(reST_file, ".. Automatically generated. Do not edit.\n\n"); - - WriteToDoc(reST_file, ":tocdepth: 3\n\n"); - - WriteSectionHeading(reST_file, doc_title.c_str(), '='); - - WriteStringList(reST_file, ".. bro:namespace:: %s\n", modules); - - WriteToDoc(reST_file, "\n"); - - // WriteSectionHeading(reST_file, "Overview", '-'); - WriteStringList(reST_file, "%s\n", summary); - - WriteToDoc(reST_file, "\n"); - - if ( ! modules.empty() ) - { - WriteToDoc(reST_file, ":Namespace%s: ", (modules.size() > 1 ? "s" : "")); - // WriteStringList(reST_file, ":bro:namespace:`%s`", modules); - WriteStringList(reST_file, "``%s``, ", "``%s``", modules); - WriteToDoc(reST_file, "\n"); - } - - if ( ! imports.empty() ) - { - WriteToDoc(reST_file, ":Imports: "); - std::list::const_iterator it; - for ( it = imports.begin(); it != imports.end(); ++it ) - { - if ( it != imports.begin() ) - WriteToDoc(reST_file, ", "); - - string pretty(*it); - size_t pos = pretty.find("/index"); - if ( pos != std::string::npos && pos + 6 == pretty.size() ) - pretty = pretty.substr(0, pos); - WriteToDoc(reST_file, ":doc:`%s `", pretty.c_str(), it->c_str()); - } - WriteToDoc(reST_file, "\n"); - } - - WriteToDoc(reST_file, ":Source File: :download:`%s`\n", - downloadable_filename.c_str()); - - WriteToDoc(reST_file, "\n"); - - WriteInterface("Summary", '~', '#', true, true); - - if ( ! notices.empty() ) - WriteBroDocObjList(reST_file, notices, "Notices", '#'); - - if ( port_analysis.size() || packet_filter.size() ) - WriteSectionHeading(reST_file, "Configuration Changes", '#'); - - if ( ! port_analysis.empty() ) - { - WriteSectionHeading(reST_file, "Port Analysis", '^'); - WriteToDoc(reST_file, "Loading this script makes the following changes to " - ":bro:see:`dpd_config`.\n\n"); - WriteStringList(reST_file, "%s, ", "%s", port_analysis); - } - - if ( ! packet_filter.empty() ) - { - WriteSectionHeading(reST_file, "Packet Filter", '^'); - WriteToDoc(reST_file, "Loading this script makes the following changes to " - ":bro:see:`capture_filters`.\n\n"); - WriteToDoc(reST_file, "Filters added::\n\n"); - WriteToDoc(reST_file, "%s\n", packet_filter.c_str()); - } - - WriteInterface("Detailed Interface", '~', '#', true, false); - -#if 0 // Disabled for now. - BroDocObjList::const_iterator it; - bool hasPrivateIdentifiers = false; - - for ( it = all.begin(); it != all.end(); ++it ) - { - if ( ! IsPublicAPI(*it) ) - { - hasPrivateIdentifiers = true; - break; - } - } - - if ( hasPrivateIdentifiers ) - WriteInterface("Private Interface", '~', '#', false, false); -#endif - } - -void BroDoc::WriteInterface(const char* heading, char underline, - char sub, bool isPublic, bool isShort) const - { - WriteSectionHeading(reST_file, heading, underline); - WriteBroDocObjList(reST_file, options, isPublic, "Options", sub, isShort); - WriteBroDocObjList(reST_file, constants, isPublic, "Constants", sub, isShort); - WriteBroDocObjList(reST_file, state_vars, isPublic, "State Variables", sub, isShort); - WriteBroDocObjList(reST_file, types, isPublic, "Types", sub, isShort); - WriteBroDocObjList(reST_file, events, isPublic, "Events", sub, isShort); - WriteBroDocObjList(reST_file, hooks, isPublic, "Hooks", sub, isShort); - WriteBroDocObjList(reST_file, functions, isPublic, "Functions", sub, isShort); - WriteBroDocObjList(reST_file, redefs, isPublic, "Redefinitions", sub, isShort); - } - -void BroDoc::WriteStringList(FILE* f, const char* format, const char* last_format, - const std::list& l) - { - if ( l.empty() ) - { - WriteToDoc(f, "\n"); - return; - } - - std::list::const_iterator it; - std::list::const_iterator last = l.end(); - last--; - - for ( it = l.begin(); it != last; ++it ) - WriteToDoc(f, format, it->c_str()); - - WriteToDoc(f, last_format, last->c_str()); - } - -void BroDoc::WriteBroDocObjTable(FILE* f, const BroDocObjList& l) - { - int max_id_col = 0; - int max_com_col = 0; - BroDocObjList::const_iterator it; - - for ( it = l.begin(); it != l.end(); ++it ) - { - int c = (*it)->ColumnSize(); - - if ( c > max_id_col ) - max_id_col = c; - - c = (*it)->LongestShortDescLen(); - - if ( c > max_com_col ) - max_com_col = c; - } - - // Start table. - WriteRepeatedChar(f, '=', max_id_col); - WriteToDoc(f, " "); - - if ( max_com_col == 0 ) - WriteToDoc(f, "="); - else - WriteRepeatedChar(f, '=', max_com_col); - - WriteToDoc(f, "\n"); - - for ( it = l.begin(); it != l.end(); ++it ) - { - if ( it != l.begin() ) - WriteToDoc(f, "\n\n"); - (*it)->WriteReSTCompact(f, max_id_col); - } - - // End table. - WriteToDoc(f, "\n"); - WriteRepeatedChar(f, '=', max_id_col); - WriteToDoc(f, " "); - - if ( max_com_col == 0 ) - WriteToDoc(f, "="); - else - WriteRepeatedChar(f, '=', max_com_col); - - WriteToDoc(f, "\n\n"); - } - -void BroDoc::WriteBroDocObjList(FILE* f, const BroDocObjList& l, bool wantPublic, - const char* heading, char underline, bool isShort) - { - if ( l.empty() ) - return; - - BroDocObjList::const_iterator it; - bool (*f_ptr)(const BroDocObj* o) = 0; - - if ( wantPublic ) - f_ptr = IsPublicAPI; - else - f_ptr = IsPrivateAPI; - - it = std::find_if(l.begin(), l.end(), f_ptr); - - if ( it == l.end() ) - return; - - WriteSectionHeading(f, heading, underline); - - BroDocObjList filtered_list; - - while ( it != l.end() ) - { - filtered_list.push_back(*it); - it = find_if(++it, l.end(), f_ptr); - } - - if ( isShort ) - WriteBroDocObjTable(f, filtered_list); - else - WriteBroDocObjList(f, filtered_list); - } - -void BroDoc::WriteBroDocObjList(FILE* f, const BroDocObjMap& m, bool wantPublic, - const char* heading, char underline, bool isShort) - { - BroDocObjMap::const_iterator it; - BroDocObjList l; - - for ( it = m.begin(); it != m.end(); ++it ) - l.push_back(it->second); - - WriteBroDocObjList(f, l, wantPublic, heading, underline, isShort); - } - -void BroDoc::WriteBroDocObjList(FILE* f, const BroDocObjList& l, const char* heading, - char underline) - { - WriteSectionHeading(f, heading, underline); - WriteBroDocObjList(f, l); - } - -void BroDoc::WriteBroDocObjList(FILE* f, const BroDocObjList& l) - { - for ( BroDocObjList::const_iterator it = l.begin(); it != l.end(); ++it ) - (*it)->WriteReST(f); - } - -void BroDoc::WriteBroDocObjList(FILE* f, const BroDocObjMap& m, const char* heading, - char underline) - { - BroDocObjMap::const_iterator it; - BroDocObjList l; - - for ( it = m.begin(); it != m.end(); ++it ) - l.push_back(it->second); - - WriteBroDocObjList(f, l, heading, underline); - } - -void BroDoc::WriteToDoc(FILE* f, const char* format, ...) - { - va_list argp; - va_start(argp, format); - vfprintf(f, format, argp); - va_end(argp); - } - -void BroDoc::WriteSectionHeading(FILE* f, const char* heading, char underline) - { - WriteToDoc(f, "%s\n", heading); - WriteRepeatedChar(f, underline, strlen(heading)); - WriteToDoc(f, "\n"); - } - -void BroDoc::WriteRepeatedChar(FILE* f, char c, size_t n) - { - for ( size_t i = 0; i < n; ++i ) - WriteToDoc(f, "%c", c); - } - -void BroDoc::FreeBroDocObjPtrList(BroDocObjList& l) - { - for ( BroDocObjList::const_iterator it = l.begin(); it != l.end(); ++it ) - delete *it; - - l.clear(); - } - -void BroDoc::AddFunction(BroDocObj* o) - { - BroDocObjMap::const_iterator it = functions.find(o->Name()); - if ( it == functions.end() ) - { - functions[o->Name()] = o; - all.push_back(o); - } - else - functions[o->Name()]->Combine(o); - } - -static void WritePluginSectionHeading(FILE* f, const plugin::Plugin* p) - { - string name = p->Name(); - - fprintf(f, "%s\n", name.c_str()); - for ( size_t i = 0; i < name.size(); ++i ) - fprintf(f, "-"); - fprintf(f, "\n\n"); - - fprintf(f, "%s\n\n", p->Description()); - } - -static void WriteAnalyzerComponent(FILE* f, const analyzer::Component* c) - { - EnumType* atag = analyzer_mgr->GetTagEnumType(); - string tag = fmt("ANALYZER_%s", c->CanonicalName()); - - if ( atag->Lookup("Analyzer", tag.c_str()) < 0 ) - reporter->InternalError("missing analyzer tag for %s", tag.c_str()); - - fprintf(f, ":bro:enum:`Analyzer::%s`\n\n", tag.c_str()); - } - -static void WriteAnalyzerComponent(FILE* f, const file_analysis::Component* c) - { - EnumType* atag = file_mgr->GetTagEnumType(); - string tag = fmt("ANALYZER_%s", c->CanonicalName()); - - if ( atag->Lookup("Files", tag.c_str()) < 0 ) - reporter->InternalError("missing analyzer tag for %s", tag.c_str()); - - fprintf(f, ":bro:enum:`Files::%s`\n\n", tag.c_str()); - } - -static void WritePluginComponents(FILE* f, const plugin::Plugin* p) - { - plugin::Plugin::component_list components = p->Components(); - plugin::Plugin::component_list::const_iterator it; - - fprintf(f, "Components\n"); - fprintf(f, "++++++++++\n\n"); - - for ( it = components.begin(); it != components.end(); ++it ) - { - switch ( (*it)->Type() ) { - case plugin::component::ANALYZER: - { - const analyzer::Component* c = - dynamic_cast(*it); - - if ( c ) - WriteAnalyzerComponent(f, c); - else - reporter->InternalError("component type mismatch"); - } - break; - - case plugin::component::FILE_ANALYZER: - { - const file_analysis::Component* c = - dynamic_cast(*it); - - if ( c ) - WriteAnalyzerComponent(f, c); - else - reporter->InternalError("component type mismatch"); - } - break; - - case plugin::component::READER: - reporter->InternalError("docs for READER component unimplemented"); - - case plugin::component::WRITER: - reporter->InternalError("docs for WRITER component unimplemented"); - - default: - reporter->InternalError("docs for unknown component unimplemented"); - } - } - } - -static void WritePluginBifItems(FILE* f, const plugin::Plugin* p, - plugin::BifItem::Type t, const string& heading) - { - plugin::Plugin::bif_item_list bifitems = p->BifItems(); - plugin::Plugin::bif_item_list::iterator it = bifitems.begin(); - - while ( it != bifitems.end() ) - { - if ( it->GetType() != t ) - it = bifitems.erase(it); - else - ++it; - } - - if ( bifitems.empty() ) - return; - - fprintf(f, "%s\n", heading.c_str()); - for ( size_t i = 0; i < heading.size(); ++i ) - fprintf(f, "+"); - fprintf(f, "\n\n"); - - for ( it = bifitems.begin(); it != bifitems.end(); ++it ) - { - broxygen::IdentifierDocument* doc = broxygen_mgr->GetIdentifierDoc( - it->GetID()); - - if ( doc ) - fprintf(f, "%s\n\n", doc->ReStructuredText().c_str()); - else - reporter->InternalWarning("Broxygen ID lookup failed: %s\n", - it->GetID()); - } - } - -static void WriteAnalyzerTagDefn(FILE* f, const string& module) - { - string tag_id = module + "::Tag"; - - broxygen::IdentifierDocument* doc = broxygen_mgr->GetIdentifierDoc(tag_id); - - if ( ! doc ) - reporter->InternalError("Broxygen failed analyzer tag lookup: %s", - tag_id.c_str()); - - fprintf(f, "%s\n", doc->ReStructuredText().c_str()); - } - -static bool ComponentsMatch(const plugin::Plugin* p, plugin::component::Type t, - bool match_empty = false) - { - plugin::Plugin::component_list components = p->Components(); - plugin::Plugin::component_list::const_iterator it; - - if ( components.empty() ) - return match_empty; - - for ( it = components.begin(); it != components.end(); ++it ) - if ( (*it)->Type() != t ) - return false; - - return true; - } - -void CreateProtoAnalyzerDoc(FILE* f) - { - fprintf(f, "Protocol Analyzers\n"); - fprintf(f, "==================\n\n"); - fprintf(f, ".. contents::\n"); - fprintf(f, " :depth: 2\n\n"); - - WriteAnalyzerTagDefn(f, "Analyzer"); - - plugin::Manager::plugin_list plugins = plugin_mgr->Plugins(); - plugin::Manager::plugin_list::const_iterator it; - - for ( it = plugins.begin(); it != plugins.end(); ++it ) - { - if ( ! ComponentsMatch(*it, plugin::component::ANALYZER, true) ) - continue; - - WritePluginSectionHeading(f, *it); - WritePluginComponents(f, *it); - WritePluginBifItems(f, *it, plugin::BifItem::CONSTANT, - "Options/Constants"); - WritePluginBifItems(f, *it, plugin::BifItem::GLOBAL, "Globals"); - WritePluginBifItems(f, *it, plugin::BifItem::TYPE, "Types"); - WritePluginBifItems(f, *it, plugin::BifItem::EVENT, "Events"); - WritePluginBifItems(f, *it, plugin::BifItem::FUNCTION, "Functions"); - } - - fclose(f); - } - -void CreateFileAnalyzerDoc(FILE* f) - { - fprintf(f, "File Analyzers\n"); - fprintf(f, "==============\n\n"); - fprintf(f, ".. contents::\n"); - fprintf(f, " :depth: 2\n\n"); - - WriteAnalyzerTagDefn(f, "Files"); - - plugin::Manager::plugin_list plugins = plugin_mgr->Plugins(); - plugin::Manager::plugin_list::const_iterator it; - - for ( it = plugins.begin(); it != plugins.end(); ++it ) - { - if ( ! ComponentsMatch(*it, plugin::component::FILE_ANALYZER) ) - continue; - - WritePluginSectionHeading(f, *it); - WritePluginComponents(f, *it); - WritePluginBifItems(f, *it, plugin::BifItem::CONSTANT, - "Options/Constants"); - WritePluginBifItems(f, *it, plugin::BifItem::GLOBAL, "Globals"); - WritePluginBifItems(f, *it, plugin::BifItem::TYPE, "Types"); - WritePluginBifItems(f, *it, plugin::BifItem::EVENT, "Events"); - WritePluginBifItems(f, *it, plugin::BifItem::FUNCTION, "Functions"); - } - - fclose(f); - } diff --git a/src/BroDoc.h b/src/BroDoc.h deleted file mode 100644 index 926713d01e..0000000000 --- a/src/BroDoc.h +++ /dev/null @@ -1,422 +0,0 @@ -#ifndef brodoc_h -#define brodoc_h - -#include -#include -#include -#include - -#include "BroDocObj.h" - -/** - * This class is used to gather all data relevant to the automatic generation - * of a reStructuredText (reST) document from a given Bro script. - */ -class BroDoc { -public: - /** - * BroDoc constructor - * Given a Bro script, opens new file in the current working directory - * that will contain reST documentation generated from the parsing - * of the Bro script. The new reST file will be named similar to - * the filename of the Bro script that generates it, except any - * ".bro" file extension is stripped and ".rst" takes it place. - * If the filename doesn't end in ".bro", then ".rst" is just appended. - * Any '/' characters in the reST file name that result from choice of - * the 'rel' parameter are replaced with '^'. - * @param rel A string representing a subpath of the root Bro script - * source/install directory in which the source file is located. - * It can also be an absolute path, but then the parameter is - * ignored and the document title is just derived from file name - * @param abs The absolute path to the Bro script for which to generate - * documentation. - */ - BroDoc(const std::string& rel, const std::string& abs); - - /** - * BroDoc destructor - * Closes the file that was opened by the constructor and frees up - * memory taken by BroDocObj objects. - */ - virtual ~BroDoc(); - - /** - * Write out full reST documentation for the Bro script that was parsed. - * BroDoc's default implementation of this function will care - * about whether declarations made in the Bro script are part of - * the public versus private interface (whether things are declared in - * the export section). - */ - virtual void WriteDocFile() const; - - /** - * Schedules some summarizing text to be output directly into the reST doc. - * This should be called whenever the scanner sees a line in the Bro script - * starting with "##!" - * @param s The summary text to add to the reST doc. - */ - void AddSummary(const std::string& s) { summary.push_back(s); } - - /** - * Schedules an import (@load) to be documented. - * If the script being loaded has a .bro suffix, it is internally stripped. - * This should be called whenever the scanner sees an @load. - * @param s The name of the imported script. - */ - void AddImport(const std::string& s); - - /** - * Schedules a namespace (module) to be documented. - * This should be called whenever the parser sees a TOK_MODULE. - * @param s The namespace (module) identifier's name. - */ - void AddModule(const std::string& s) { modules.push_back(s); } - - /** - * Sets the way the script changes the "capture_filters" table. - * This is determined by the scanner checking for changes to - * the "capture_filters" table after each of Bro's input scripts - * (given as command line arguments to Bro) are finished being parsed. - * @param s The value "capture_filters" as given by TableVal::Describe() - */ - void SetPacketFilter(const std::string& s); - - /** - * Schedules documentation of a script option. An option is - * defined as any variable in the script that is declared 'const' - * and has the '&redef' attribute. - * @param o A pointer to a BroDocObj which contains the internal - * Bro language representation of the script option and - * also any associated comments about it. - */ - void AddOption(const BroDocObj* o) - { - options.push_back(o); - all.push_back(o); - } - - /** - * Schedules documentation of a script constant. An option is - * defined as any variable in the script that is declared 'const' - * and does *not* have the '&redef' attribute. - * @param o A pointer to a BroDocObj which contains the internal - * Bro language representation of the script constant and - * also any associated comments about it. - */ - void AddConstant(const BroDocObj* o) - { - constants.push_back(o); - all.push_back(o); - } - - /** - * Schedules documentation of a script state variable. A state variable - * is defined as any variable in the script that is declared 'global' - * @param o A pointer to a BroDocObj which contains the internal - * Bro language representation of the script state variable - * and also any associated comments about it. - */ - void AddStateVar(const BroDocObj* o) - { - state_vars.push_back(o); - all.push_back(o); - } - - /** - * Schedules documentation of a type declared by the script. - * @param o A pointer to a BroDocObj which contains the internal - * Bro language representation of the script option and - * also any associated comments about it. - */ - void AddType(const BroDocObj* o) - { - types.push_back(o); - all.push_back(o); - } - - /** - * Schedules documentation of a Notice (enum redef) declared by script - * @param o A pointer to a BroDocObj which contains the internal - * Bro language representation of the Notice and also - * any associated comments about it. - */ - void AddNotice(const BroDocObj* o) - { - notices.push_back(o); - all.push_back(o); - } - - /** - * Schedules documentation of an event declared by the script. - * @param o A pointer to a BroDocObj which contains the internal - * Bro language representation of the script event and - * also any associated comments about it. - */ - void AddEvent(const BroDocObj* o) - { - events.push_back(o); - all.push_back(o); - } - - /** - * Schedules documentation of an event handler declared by the script. - * @param o A pointer to a BroDocObj which contains the internal - * Bro language representation of the script event handler and - * also any associated comments about it. - */ - void AddEventHandler(const BroDocObj* o) - { - event_handlers.push_back(o); - all.push_back(o); - } - - /** - * Schedules documentation of a hook declared by the script. - * @param o A pointer to a BroDocObj which contains the internal - * Bro language representation of the script hook and - * also any associated comments about it. - */ - void AddHook(const BroDocObj* o) - { - hooks.push_back(o); - all.push_back(o); - } - - /** - * Schedules documentation of a hook handler declared by the script. - * @param o A pointer to a BroDocObj which contains the internal - * Bro language representation of the script hook handler and - * also any associated comments about it. - */ - void AddHookHandler(const BroDocObj* o) - { - hook_handlers.push_back(o); - all.push_back(o); - } - - /** - * Schedules documentation of a function declared by the script. - * @param o A pointer to a BroDocObj which contains the internal - * Bro language representation of the script function and - * also any associated comments about it. - */ - void AddFunction(BroDocObj* o); - - /** - * Schedules documentation of a redef done by the script - * @param o A pointer to a BroDocObj which contains the internal - * Bro language representation of the script identifier - * that was redefined and also any associated comments. - */ - void AddRedef(const BroDocObj* o) - { - redefs.push_back(o); - all.push_back(o); - } - - /** - * Gets the name of the Bro script source file for which reST - * documentation is being generated. - * @return A char* to the start of the source file's name. - */ - const char* GetSourceFileName() const - { - return source_filename.c_str(); - } - - /** - * Gets the name of the generated reST documentation file. - * @return A char* to the start of the generated reST file's name. - */ - const char* GetOutputFileName() const - { - return reST_filename.c_str(); - } - - typedef std::list BroDocObjList; - typedef std::map BroDocObjMap; - - /** - * Writes out a table of BroDocObj's to the reST document - * @param f The file to write to. - * @param l A list of BroDocObj pointers - */ - static void WriteBroDocObjTable(FILE* f, const BroDocObjList& l); - - /** - * Writes out given number of characters to reST document - * @param f The file to write to. - * @param c the character to write - * @param n the number of characters to write - */ - static void WriteRepeatedChar(FILE* f, char c, size_t n); - - /** - * A wrapper to fprintf() that always uses the reST document - * for the FILE* argument. - * @param f The file to write to. - * @param format A printf style format string. - */ - static void WriteToDoc(FILE* f, const char* format, ...); - - /** - * Writes out a list of strings to the reST document. - * If the list is empty, prints a newline character. - * @param f The file to write to. - * @param format A printf style format string for elements of the list - * except for the last one in the list - * @param last_format A printf style format string to use for the last - * element of the list - * @param l A reference to a list of strings - */ - static void WriteStringList(FILE* f, const char* format, const char* last_format, - const std::list& l); - - /** - * @see WriteStringList(FILE* f, const char*, const char*, - * const std::list&>) - */ - static void WriteStringList(FILE* f, const char* format, - const std::list& l){ - WriteStringList(f, format, format, l); - } - - /** - * Writes out a list of BroDocObj objects to the reST document - * @param f The file to write to. - * @param l A list of BroDocObj pointers - * @param wantPublic If true, filter out objects that are not declared - * in the global scope. If false, filter out those that are in - * the global scope. - * @param heading The title of the section to create in the reST doc. - * @param underline The character to use to underline the reST - * section heading. - * @param isShort Whether to write the full documentation or a "short" - * version (a single sentence) - */ - static void WriteBroDocObjList(FILE* f, const BroDocObjList& l, bool wantPublic, - const char* heading, char underline, - bool isShort); - - /** - * Wraps the BroDocObjMap into a BroDocObjList and the writes that list - * to the reST document - * @see WriteBroDocObjList(FILE* f, const BroDocObjList&, bool, const char*, char, - bool) - */ - static void WriteBroDocObjList(FILE* f, const BroDocObjMap& m, bool wantPublic, - const char* heading, char underline, - bool isShort); - - /** - * Writes out a list of BroDocObj objects to the reST document - * @param l A list of BroDocObj pointers - * @param heading The title of the section to create in the reST doc. - * @param underline The character to use to underline the reST - * section heading. - */ - static void WriteBroDocObjList(FILE* f, const BroDocObjList& l, const char* heading, - char underline); - - /** - * Writes out a list of BroDocObj objects to the reST document - * @param l A list of BroDocObj pointers - */ - static void WriteBroDocObjList(FILE* f, const BroDocObjList& l); - - /** - * Wraps the BroDocObjMap into a BroDocObjList and the writes that list - * to the reST document - * @see WriteBroDocObjList(FILE* f, const BroDocObjList&, const char*, char) - */ - static void WriteBroDocObjList(FILE* f, const BroDocObjMap& m, const char* heading, - char underline); - - /** - * Writes out a reST section heading - * @param f The file to write to. - * @param heading The title of the heading to create - * @param underline The character to use to underline the section title - * within the reST document - */ - static void WriteSectionHeading(FILE* f, const char* heading, char underline); - -private: - FILE* reST_file; - std::string reST_filename; - std::string source_filename; // points to the basename of source file - std::string downloadable_filename; // file that will be linked for download - std::string doc_title; - std::string packet_filter; - - std::list modules; - std::list summary; - std::list imports; - std::list port_analysis; - - BroDocObjList options; - BroDocObjList constants; - BroDocObjList state_vars; - BroDocObjList types; - BroDocObjList notices; - BroDocObjList events; - BroDocObjList event_handlers; - BroDocObjList hooks; - BroDocObjList hook_handlers; - BroDocObjMap functions; - BroDocObjList redefs; - - BroDocObjList all; - - /** - * Writes out the reST for either the script's public or private interface - * @param heading The title of the interfaces section heading - * @param underline The underline character to use for the interface - * section - * @param subunderline The underline character to use for interface - * sub-sections - * @param isPublic Whether to write out the public or private script - * interface - * @param isShort Whether to write out the full documentation or a "short" - * description (a single sentence) - */ - void WriteInterface(const char* heading, char underline, char subunderline, - bool isPublic, bool isShort) const; - - /** - * Frees memory allocated to BroDocObj's objects in a given list. - * @param a reference to a list of BroDocObj pointers - */ - void FreeBroDocObjPtrList(BroDocObjList& l); - - static bool IsPublicAPI(const BroDocObj* o) - { - return o->IsPublicAPI(); - } - - static bool IsPrivateAPI(const BroDocObj* o) - { - return ! o->IsPublicAPI(); - } - - struct replace_slash { - void operator()(char& c) - { - if ( c == '/' ) c = '^'; - } - }; -}; - -/** - * Writes out plugin index documentation for all analyzer plugins. - * @param f an open file stream to write docs into. - */ -void CreateProtoAnalyzerDoc(FILE* f); - -/** - * Writes out plugin index documentation for all file analyzer plugins. - * @param f an open file stream to write docs into. - */ -void CreateFileAnalyzerDoc(FILE* f); - -#endif diff --git a/src/BroDocObj.cc b/src/BroDocObj.cc deleted file mode 100644 index 063bdb1941..0000000000 --- a/src/BroDocObj.cc +++ /dev/null @@ -1,195 +0,0 @@ -#include -#include -#include -#include "ID.h" -#include "BroDocObj.h" - -map doc_ids = map(); - -BroDocObj* BroDocObj::last = 0; - -BroDocObj::BroDocObj(const ID* id, std::list*& reST, - bool is_fake) - { - last = this; - broID = id; - reST_doc_strings = reST; - reST = 0; - is_fake_id = is_fake; - use_role = 0; - FormulateShortDesc(); - doc_ids[id->Name()] = this; - } - -BroDocObj::~BroDocObj() - { - if ( reST_doc_strings ) - delete reST_doc_strings; - - if ( is_fake_id ) - delete broID; - } - -void BroDocObj::WriteReSTCompact(FILE* file, int max_col) const - { - ODesc desc; - desc.SetQuotes(1); - broID->DescribeReSTShort(&desc); - - fprintf(file, "%s", desc.Description()); - - std::list::const_iterator it; - - for ( it = short_desc.begin(); it != short_desc.end(); ++it ) - { - int start_col; - - if ( it == short_desc.begin() ) - start_col = max_col - desc.Len() + 1; - else - { - start_col = max_col + 1; - fprintf(file, "\n"); - } - - for ( int i = 0; i < start_col; ++i ) - fprintf(file, " "); - - fprintf(file, "%s", it->c_str()); - } - } - -int BroDocObj::LongestShortDescLen() const - { - size_t max = 0; - - std::list::const_iterator it; - - for ( it = short_desc.begin(); it != short_desc.end(); ++it ) - { - if ( it->size() > max ) - max = it->size(); - } - - return max; - } - -static size_t end_of_first_sentence(string s) - { - size_t rval = 0; - - while ( (rval = s.find_first_of('.', rval)) != string::npos ) - { - if ( rval == s.size() - 1 ) - // Period is at end of string. - return rval; - - if ( isspace(s[rval + 1]) ) - // Period has a space after it. - return rval; - - // Period has some non-space character after it, keep looking. - ++rval; - } - - return rval; - } - -void BroDocObj::FormulateShortDesc() - { - if ( ! reST_doc_strings ) - return; - - short_desc.clear(); - std::list::const_iterator it; - - for ( it = reST_doc_strings->begin(); - it != reST_doc_strings->end(); ++it ) - { - // The short description stops at the first sentence or the - // first empty comment. - size_t end = end_of_first_sentence(*it); - - if ( end == string::npos ) - { - std::string::const_iterator s; - bool empty = true; - - for ( s = it->begin(); s != it->end(); ++s ) - { - if ( *s != ' ' && *s != '\t' && *s != '\n' && *s != '\r' ) - { - empty = false; - short_desc.push_back(*it); - break; - } - } - - if ( empty ) - break; - } - else - { - short_desc.push_back(it->substr(0, end + 1)); - break; - } - } - } - -void BroDocObj::WriteReST(FILE* file) const - { - int indent_spaces = 3; - ODesc desc; - desc.SetIndentSpaces(indent_spaces); - desc.SetQuotes(1); - - broID->DescribeReST(&desc, use_role); - - fprintf(file, "%s", desc.Description()); - - if ( HasDocumentation() ) - { - fprintf(file, "\n"); - std::list::const_iterator it; - - for ( it = reST_doc_strings->begin(); - it != reST_doc_strings->end(); ++it) - { - for ( int i = 0; i < indent_spaces; ++i ) - fprintf(file, " "); - - fprintf(file, "%s\n", it->c_str()); - } - } - - fprintf(file, "\n"); - } - -int BroDocObj::ColumnSize() const - { - ODesc desc; - desc.SetQuotes(1); - broID->DescribeReSTShort(&desc); - return desc.Len(); - } - -bool BroDocObj::IsPublicAPI() const - { - return (broID->Scope() == SCOPE_GLOBAL) || - (broID->Scope() == SCOPE_MODULE && broID->IsExport()); - } - -void BroDocObj::Combine(const BroDocObj* o) - { - if ( o->reST_doc_strings ) - { - if ( ! reST_doc_strings ) - reST_doc_strings = new std::list(); - - reST_doc_strings->splice(reST_doc_strings->end(), - *(o->reST_doc_strings)); - } - - delete o; - FormulateShortDesc(); - } diff --git a/src/BroDocObj.h b/src/BroDocObj.h deleted file mode 100644 index ab42dc3c94..0000000000 --- a/src/BroDocObj.h +++ /dev/null @@ -1,143 +0,0 @@ -#ifndef brodocobj_h -#define brodocobj_h - -#include -#include -#include -#include - -#include "ID.h" - -/** - * This class wraps a Bro script identifier, providing methods relevant - * to automatic generation of reStructuredText (reST) documentation for it. - */ -class BroDocObj { -public: - /** - * BroDocObj constructor - * @param id a pointer to an identifier that is to be documented - * @param reST a reference to a pointer of a list of strings that - * represent the reST documentation for the ID. The pointer - * will be set to 0 after this constructor finishes. - * @param is_fake whether the ID* is a dummy just for doc purposes - */ - BroDocObj(const ID* id, std::list*& reST, - bool is_fake = false); - - /** - * BroDocObj destructor - * Deallocates the memory associated with the list of reST strings - */ - ~BroDocObj(); - - /** - * Writes the reST representation of this object which includes - * 1) a reST friendly description of the ID - * 2) "##" or "##<" stylized comments. - * Anything after these style of comments is inserted as-is into - * the reST document. - * @param file The (already opened) file to write the reST to. - */ - void WriteReST(FILE* file) const; - - /** - * Writes a compact version of the ID and associated documentation - * for insertion into a table. - * @param file The (already opened) file to write the reST to. - * @param max_col The maximum length of the first table column - */ - void WriteReSTCompact(FILE* file, int max_col) const; - - /** - * @return the column size required by the reST representation of the ID - */ - int ColumnSize() const; - - /** - * Check whether this documentation is part of the public API. In - * other words, this means that the identifier is declared as part of - * the global scope (has GLOBAL namespace or is exported from another - * namespace). - * @return true if the identifier is part of the script's public API - */ - bool IsPublicAPI() const; - - /** - * Return whether this object has documentation (## comments) - * @return true if the ID has comments associated with it - */ - bool HasDocumentation() const - { - return reST_doc_strings && reST_doc_strings->size() > 0; - } - - /** - * @return whether this object will use reST role (T) or directive (F) - * notation for the wrapped identifier. Roles are usually used - * for cross-referencing. - */ - bool UseRole() const { return use_role; } - - /** - * @param b whether this object will use reST role (T) or directive (F) - * notation for the wrapped identifier. Roles are usually used - * for cross-referencing. - */ - void SetRole(bool b) { use_role = b; } - - /** - * Append any reST documentation strings in a given BroDocObj to this - * object's list and then delete the given BroDocObj - * @param o a pointer to a BroDocObj to subsume - */ - void Combine(const BroDocObj* o); - - /** - * @return the name of the wrapped identifier - */ - const char* Name() const { return broID->Name(); } - - /** - * @return the longest string element of the short description's list of - * strings - */ - int LongestShortDescLen() const; - - /** - * Adds a reST documentation string to this BroDocObj's list. - * @param s the documentation string to append. - */ - void AddDocString(const std::string& s) - { - if ( ! reST_doc_strings ) - reST_doc_strings = new std::list(); - reST_doc_strings->push_back(s); - FormulateShortDesc(); - } - - static BroDocObj* last; - -protected: - std::list* reST_doc_strings; - std::list short_desc; - const ID* broID; - bool is_fake_id; /**< Whether the ID* is a dummy just for doc purposes */ - bool use_role; /**< Whether to use a reST role or directive for the ID */ - - /** - * Set the short_desc member to be a subset of reST_doc_strings. - * Specifically, short_desc will be everything in reST_doc_strings - * up until the first period or first empty string list element found. - */ - void FormulateShortDesc(); - -private: -}; - -/** - * Map identifiers to their broxygen documentation objects. - */ -extern map doc_ids; - -#endif diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 48d651cd93..8e22b504e4 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -250,8 +250,6 @@ set(bro_SRCS Attr.cc Base64.cc BPF_Program.cc - BroDoc.cc - BroDocObj.cc Brofiler.cc BroString.cc CCL.cc diff --git a/src/Type.cc b/src/Type.cc index 1e882f27d6..fa9480becf 100644 --- a/src/Type.cc +++ b/src/Type.cc @@ -1146,8 +1146,8 @@ void RecordType::DescribeFieldsReST(ODesc* d, bool func_args) const if ( func_args ) continue; - using broxygen::IdentifierDocument; - IdentifierDocument* doc = broxygen_mgr->GetIdentifierDoc(GetName()); + using broxygen::IdentifierInfo; + IdentifierInfo* doc = broxygen_mgr->GetIdentifierInfo(GetName()); if ( ! doc ) { @@ -1498,8 +1498,8 @@ void EnumType::DescribeReST(ODesc* d, bool roles_only) const else d->Add(fmt(".. bro:enum:: %s %s", it->second, GetName().c_str())); - using broxygen::IdentifierDocument; - IdentifierDocument* doc = broxygen_mgr->GetIdentifierDoc(it->second); + using broxygen::IdentifierInfo; + IdentifierInfo* doc = broxygen_mgr->GetIdentifierInfo(it->second); if ( ! doc ) { @@ -1514,7 +1514,7 @@ void EnumType::DescribeReST(ODesc* d, bool roles_only) const if ( doc->GetDeclaringScript() ) enum_from_script = doc->GetDeclaringScript()->Name(); - IdentifierDocument* type_doc = broxygen_mgr->GetIdentifierDoc(GetName()); + IdentifierInfo* type_doc = broxygen_mgr->GetIdentifierInfo(GetName()); if ( type_doc && type_doc->GetDeclaringScript() ) type_from_script = type_doc->GetDeclaringScript()->Name(); diff --git a/src/broxygen/CMakeLists.txt b/src/broxygen/CMakeLists.txt index 6c0b240e39..13cfed0496 100644 --- a/src/broxygen/CMakeLists.txt +++ b/src/broxygen/CMakeLists.txt @@ -6,9 +6,14 @@ include_directories(BEFORE ) set(broxygen_SRCS - Configuration.cc - Document.cc Manager.cc + Info.h + PackageInfo.cc + ScriptInfo.cc + IdentifierInfo.cc + Target.cc + Configuration.cc + ReStructuredTextTable.cc utils.cc ) diff --git a/src/broxygen/Configuration.cc b/src/broxygen/Configuration.cc index 19dc61aa1a..bb346935a9 100644 --- a/src/broxygen/Configuration.cc +++ b/src/broxygen/Configuration.cc @@ -1,5 +1,5 @@ #include "Configuration.h" -#include "Manager.h" +#include "utils.h" #include "util.h" #include "Reporter.h" @@ -7,417 +7,26 @@ #include #include #include -#include -#include -#include -#include -#include -#include using namespace broxygen; using namespace std; -typedef map target_factory_map; - -static target_factory_map create_target_factory_map() +static TargetFactory create_target_factory() { - target_factory_map rval; - rval["package_index"] = &PackageIndexTarget::Instantiate; - rval["package"] = &PackageTarget::Instantiate; - rval["proto_analyzer"] = &ProtoAnalyzerTarget::Instantiate; - rval["file_analyzer"] = &FileAnalyzerTarget::Instantiate; - rval["script_summary"] = &ScriptSummaryTarget::Instantiate; - rval["script_index"] = &ScriptIndexTarget::Instantiate; - rval["script"] = &ScriptTarget::Instantiate; - rval["identifier"] = &IdentifierTarget::Instantiate; + TargetFactory rval; + rval.Register("package_index"); + rval.Register("package"); + rval.Register("proto_analyzer"); + rval.Register("file_analyzer"); + rval.Register("script_summary"); + rval.Register("script_index"); + rval.Register("script"); + rval.Register("identifier"); return rval; } -static target_factory_map target_instantiators = create_target_factory_map(); - -struct TargetFile { - TargetFile(const string& arg_name) - : name(arg_name), f() - { - if ( name.find('/') != string::npos ) - { - string dir = SafeDirname(name).result; - - if ( ! ensure_intermediate_dirs(dir.c_str()) ) - reporter->FatalError("Broxygen failed to make dir %s", - dir.c_str()); - } - - f = fopen(name.c_str(), "w"); - - if ( ! f ) - reporter->FatalError("Broxygen failed to open '%s' for writing: %s", - name.c_str(), strerror(errno)); - } - - ~TargetFile() - { - if ( f ) - fclose(f); - - DBG_LOG(DBG_BROXYGEN, "Wrote out-of-date target '%s'", name.c_str()); - } - - string name; - FILE* f; -}; - -template -static vector filter_matching_docs(const vector& from, Target* t) - { - vector rval; - - for ( size_t i = 0; i < from.size(); ++i ) - { - T* d = dynamic_cast(from[i]); - - if ( ! d ) - continue; - - if ( t->MatchesPattern(d) ) - { - DBG_LOG(DBG_BROXYGEN, "Doc '%s' matched pattern for target '%s'", - d->Name().c_str(), t->Name().c_str()); - rval.push_back(d); - } - } - - return rval; - } - -Target::Target(const string& arg_name, const string& arg_pattern) - : name(arg_name), pattern(arg_pattern), prefix() - { - size_t pos = pattern.find('*'); - - if ( pos == 0 || pos == string::npos ) - return; - - prefix = pattern.substr(0, pos); - } - -bool Target::MatchesPattern(Document* doc) const - { - if ( pattern == "*" ) - return true; - - if ( prefix.empty() ) - return doc->Name() == pattern; - - return ! strncmp(doc->Name().c_str(), prefix.c_str(), prefix.size()); - } - -void AnalyzerTarget::DoFindDependencies(const std::vector& docs) - { - // TODO: really should add to dependency list the tag type's ID and - // all bif items for matching analyzer plugins, but that's all dependent - // on the bro binary itself, so I'm cheating. - } - -void AnalyzerTarget::DoGenerate() const - { - if ( broxygen_mgr->IsUpToDate(Name(), vector()) ) - return; - - if ( Pattern() != "*" ) - reporter->InternalWarning("Broxygen only implements analyzer target" - " pattern '*'"); - - TargetFile file(Name()); - doc_creator_callback(file.f); - } - -void PackageTarget::DoFindDependencies(const vector& docs) - { - pkg_deps = filter_matching_docs(docs, this); - - if ( pkg_deps.empty() ) - reporter->FatalError("No match for Broxygen target '%s' pattern '%s'", - Name().c_str(), Pattern().c_str()); - - for ( size_t i = 0; i < docs.size(); ++i ) - { - ScriptDocument* script = dynamic_cast(docs[i]); - - if ( ! script ) - continue; - - for ( size_t j = 0; j < pkg_deps.size(); ++j ) - { - if ( strncmp(script->Name().c_str(), pkg_deps[j]->Name().c_str(), - pkg_deps[j]->Name().size())) - continue; - - DBG_LOG(DBG_BROXYGEN, "Script %s associated with package %s", - script->Name().c_str(), pkg_deps[j]->Name().c_str()); - pkg_manifest[pkg_deps[j]].push_back(script); - script_deps.push_back(script); - } - } - } - -void PackageTarget::DoGenerate() const - { - if ( broxygen_mgr->IsUpToDate(Name(), script_deps) && - broxygen_mgr->IsUpToDate(Name(), pkg_deps) ) - return; - - TargetFile file(Name()); - - fprintf(file.f, ":orphan:\n\n"); - - for ( manifest_t::const_iterator it = pkg_manifest.begin(); - it != pkg_manifest.end(); ++it ) - { - string header = fmt("Package: %s", it->first->Name().c_str()); - header += "\n" + string(header.size(), '='); - - fprintf(file.f, "%s\n\n", header.c_str()); - - vector readme = it->first->GetReadme(); - - for ( size_t i = 0; i < readme.size(); ++i ) - fprintf(file.f, "%s\n", readme[i].c_str()); - - fprintf(file.f, "\n"); - - for ( size_t i = 0; i < it->second.size(); ++i ) - { - fprintf(file.f, ":doc:`/scripts/%s`\n", - it->second[i]->Name().c_str()); - - vector cmnts = it->second[i]->GetComments(); - - for ( size_t j = 0; j < cmnts.size(); ++j ) - fprintf(file.f, " %s\n", cmnts[j].c_str()); - - fprintf(file.f, "\n"); - } - } - } - -void PackageIndexTarget::DoFindDependencies(const vector& docs) - { - pkg_deps = filter_matching_docs(docs, this); - - if ( pkg_deps.empty() ) - reporter->FatalError("No match for Broxygen target '%s' pattern '%s'", - Name().c_str(), Pattern().c_str()); - } - -void PackageIndexTarget::DoGenerate() const - { - if ( broxygen_mgr->IsUpToDate(Name(), pkg_deps) ) - return; - - TargetFile file(Name()); - - for ( size_t i = 0; i < pkg_deps.size(); ++i ) - fprintf(file.f, "%s\n", pkg_deps[i]->ReStructuredText().c_str()); - } - -void ScriptTarget::DoFindDependencies(const vector& docs) - { - script_deps = filter_matching_docs(docs, this); - - if ( script_deps.empty() ) - reporter->FatalError("No match for Broxygen target '%s' pattern '%s'", - Name().c_str(), Pattern().c_str()); - - if ( ! IsDir() ) - return; - - for ( size_t i = 0; i < script_deps.size(); ++i ) - { - if ( SafeBasename(script_deps[i]->Name()).result == PACKAGE_LOADER ) - { - string pkg_dir = SafeDirname(script_deps[i]->Name()).result; - string target_file = Name() + pkg_dir + "/index.rst"; - Target* t = PackageTarget::Instantiate(target_file, pkg_dir); - t->FindDependencies(docs); - pkg_deps.push_back(t); - } - } - } - -vector dir_contents_recursive(string dir) - { - vector rval; - struct stat st; - - if ( stat(dir.c_str(), &st) < 0 && errno == ENOENT ) - return rval; - - while ( dir[dir.size() - 1] == '/' ) - dir.erase(dir.size() - 1, 1); - - char* dir_copy = copy_string(dir.c_str()); - char** scan_path = new char*[2]; - scan_path[0] = dir_copy; - scan_path[1] = 0; - - FTS* fts = fts_open(scan_path, FTS_NOCHDIR, 0); - - if ( ! fts ) - { - reporter->Error("fts_open failure: %s", strerror(errno)); - delete [] scan_path; - delete [] dir_copy; - return rval; - } - - FTSENT* n; - - while ( (n = fts_read(fts)) ) - { - if ( n->fts_info & FTS_F ) - rval.push_back(n->fts_path); - } - - if ( errno ) - reporter->Error("fts_read failure: %s", strerror(errno)); - - if ( fts_close(fts) < 0 ) - reporter->Error("fts_close failure: %s", strerror(errno)); - - delete [] scan_path; - delete [] dir_copy; - return rval; - } - -void ScriptTarget::DoGenerate() const - { - if ( IsDir() ) - { - // Target name is a dir, matching scripts are written within that dir - // with a dir tree that parallels the script's BROPATH location. - - set targets; - vector dir_contents = dir_contents_recursive(Name()); - - for ( size_t i = 0; i < script_deps.size(); ++i ) - { - string target_filename = Name() + script_deps[i]->Name() + ".rst"; - targets.insert(target_filename); - vector dep; - dep.push_back(script_deps[i]); - - if ( broxygen_mgr->IsUpToDate(target_filename, dep) ) - continue; - - TargetFile file(target_filename); - - fprintf(file.f, "%s\n", script_deps[i]->ReStructuredText().c_str()); - } - - for ( size_t i = 0; i < pkg_deps.size(); ++i ) - { - targets.insert(pkg_deps[i]->Name()); - pkg_deps[i]->Generate(); - } - - for ( size_t i = 0; i < dir_contents.size(); ++i ) - { - string f = dir_contents[i]; - - if ( targets.find(f) != targets.end() ) - continue; - - if ( unlink(f.c_str()) < 0 ) - reporter->Warning("Failed to unlink %s: %s", f.c_str(), - strerror(errno)); - - DBG_LOG(DBG_BROXYGEN, "Delete stale script file %s", f.c_str()); - } - - return; - } - - // Target is a single file, all matching scripts get written there. - - if ( broxygen_mgr->IsUpToDate(Name(), script_deps) ) - return; - - TargetFile file(Name()); - - for ( size_t i = 0; i < script_deps.size(); ++i ) - fprintf(file.f, "%s\n", script_deps[i]->ReStructuredText().c_str()); - } - -void ScriptSummaryTarget::DoGenerate() const - { - if ( broxygen_mgr->IsUpToDate(Name(), script_deps) ) - return; - - TargetFile file(Name()); - - for ( size_t i = 0; i < script_deps.size(); ++i ) - { - ScriptDocument* d = dynamic_cast(script_deps[i]); - - if ( ! d ) - continue; - - fprintf(file.f, ":doc:`/scripts/%s`\n", d->Name().c_str()); - - vector cmnts = d->GetComments(); - - for ( size_t i = 0; i < cmnts.size(); ++i ) - fprintf(file.f, " %s\n", cmnts[i].c_str()); - - fprintf(file.f, "\n"); - } - } - -void ScriptIndexTarget::DoGenerate() const - { - if ( broxygen_mgr->IsUpToDate(Name(), script_deps) ) - return; - - TargetFile file(Name()); - - fprintf(file.f, ".. toctree::\n"); - fprintf(file.f, " :maxdepth: 1\n\n"); - - for ( size_t i = 0; i < script_deps.size(); ++i ) - { - ScriptDocument* d = dynamic_cast(script_deps[i]); - - if ( ! d ) - continue; - - fprintf(file.f, " %s \n", d->Name().c_str(), - d->Name().c_str()); - } - } - -void IdentifierTarget::DoFindDependencies(const vector& docs) - { - id_deps = filter_matching_docs(docs, this); - - if ( id_deps.empty() ) - reporter->FatalError("No match for Broxygen target '%s' pattern '%s'", - Name().c_str(), Pattern().c_str()); - } - -void IdentifierTarget::DoGenerate() const - { - if ( broxygen_mgr->IsUpToDate(Name(), id_deps) ) - return; - - TargetFile file(Name()); - - for ( size_t i = 0; i < id_deps.size(); ++i ) - fprintf(file.f, "%s\n\n", id_deps[i]->ReStructuredText().c_str()); - } - Config::Config(const string& arg_file, const string& delim) - : file(arg_file), targets() + : file(arg_file), targets(), target_factory(create_target_factory()) { if ( file.empty() ) return; @@ -450,14 +59,13 @@ Config::Config(const string& arg_file, const string& delim) reporter->FatalError("malformed Broxygen target in %s:%u: %s", file.c_str(), line_number, line.c_str()); - target_factory_map::const_iterator it = - target_instantiators.find(tokens[0]); + Target* target = target_factory.Create(tokens[0], tokens[2], tokens[1]); - if ( it == target_instantiators.end() ) + if ( ! target ) reporter->FatalError("unkown Broxygen target type: %s", tokens[0].c_str()); - targets.push_back(it->second(tokens[2], tokens[1])); + targets.push_back(target); } if ( f.bad() ) @@ -471,10 +79,10 @@ Config::~Config() delete targets[i]; } -void Config::FindDependencies(const vector& docs) +void Config::FindDependencies(const vector& infos) { for ( size_t i = 0; i < targets.size(); ++i ) - targets[i]->FindDependencies(docs); + targets[i]->FindDependencies(infos); } void Config::GenerateDocs() const @@ -485,11 +93,8 @@ void Config::GenerateDocs() const time_t Config::GetModificationTime() const { - struct stat s; + if ( file.empty() ) + return 0; - if ( stat(file.c_str(), &s) < 0 ) - reporter->InternalError("Broxygen can't stat config file %s: %s", - file.c_str(), strerror(errno)); - - return s.st_mtime; + return broxygen::get_mtime(file); } diff --git a/src/broxygen/Configuration.h b/src/broxygen/Configuration.h index 6ea9f640be..0a2f98bcda 100644 --- a/src/broxygen/Configuration.h +++ b/src/broxygen/Configuration.h @@ -1,246 +1,61 @@ #ifndef BROXYGEN_CONFIGURATION_H #define BROXYGEN_CONFIGURATION_H -#include "Document.h" -#include "BroDoc.h" +#include "Info.h" +#include "Target.h" #include #include -#include namespace broxygen { -// TODO: documentation... - -class Target { -public: - - typedef Target* (*factory_fn)(const std::string&, const std::string&); - - virtual ~Target() - { } - - void FindDependencies(const std::vector& docs) - { DoFindDependencies(docs); } - - void Generate() const - { DoGenerate(); } - - bool MatchesPattern(Document* doc) const; - - std::string Name() const - { return name; } - - std::string Pattern() const - { return pattern; } - -protected: - - Target(const std::string& arg_name, const std::string& arg_pattern); - -private: - - virtual void DoFindDependencies(const std::vector& docs) = 0; - - virtual void DoGenerate() const = 0; - - std::string name; - std::string pattern; - std::string prefix; -}; - -class AnalyzerTarget : public Target { -protected: - - typedef void (*doc_creator_fn)(FILE*); - - AnalyzerTarget(const std::string& name, const std::string& pattern, - doc_creator_fn cb) - : Target(name, pattern), doc_creator_callback(cb) - { } - -private: - - void DoFindDependencies(const std::vector& docs); - - void DoGenerate() const; - - doc_creator_fn doc_creator_callback; -}; - -class ProtoAnalyzerTarget : public AnalyzerTarget { -public: - - static Target* Instantiate(const std::string& name, - const std::string& pattern) - { return new ProtoAnalyzerTarget(name, pattern); } - -private: - - ProtoAnalyzerTarget(const std::string& name, const std::string& pattern) - : AnalyzerTarget(name, pattern, &CreateProtoAnalyzerDoc) - { } -}; - -class FileAnalyzerTarget : public AnalyzerTarget { -public: - - static Target* Instantiate(const std::string& name, - const std::string& pattern) - { return new FileAnalyzerTarget(name, pattern); } - -private: - - FileAnalyzerTarget(const std::string& name, const std::string& pattern) - : AnalyzerTarget(name, pattern, &CreateFileAnalyzerDoc) - { } -}; - -class PackageTarget : public Target { -public: - - static Target* Instantiate(const std::string& name, - const std::string& pattern) - { return new PackageTarget(name, pattern); } - -private: - - PackageTarget(const std::string& name, const std::string& pattern) - : Target(name, pattern), pkg_deps(), script_deps(), pkg_manifest() - { } - - void DoFindDependencies(const std::vector& docs); - - void DoGenerate() const; - - std::vector pkg_deps; - std::vector script_deps; - typedef std::map > manifest_t; - manifest_t pkg_manifest; -}; - -class PackageIndexTarget : public Target { -public: - - static Target* Instantiate(const std::string& name, - const std::string& pattern) - { return new PackageIndexTarget(name, pattern); } - -private: - - PackageIndexTarget(const std::string& name, const std::string& pattern) - : Target(name, pattern), pkg_deps() - { } - - void DoFindDependencies(const std::vector& docs); - - void DoGenerate() const; - - std::vector pkg_deps; -}; - -class ScriptTarget : public Target { -public: - - static Target* Instantiate(const std::string& name, - const std::string& pattern) - { return new ScriptTarget(name, pattern); } - - ~ScriptTarget() - { for ( size_t i = 0; i < pkg_deps.size(); ++i ) delete pkg_deps[i]; } - -protected: - - ScriptTarget(const std::string& name, const std::string& pattern) - : Target(name, pattern), script_deps() - { } - - std::vector script_deps; - -private: - - void DoFindDependencies(const std::vector& docs); - - void DoGenerate() const; - - bool IsDir() const - { return Name()[Name().size() - 1] == '/'; } - - std::vector pkg_deps; -}; - -class ScriptSummaryTarget : public ScriptTarget { -public: - - static Target* Instantiate(const std::string& name, - const std::string& pattern) - { return new ScriptSummaryTarget(name, pattern); } - -private: - - ScriptSummaryTarget(const std::string& name, const std::string& pattern) - : ScriptTarget(name, pattern) - { } - - void DoGenerate() const /* override */; -}; - -class ScriptIndexTarget : public ScriptTarget { -public: - - static Target* Instantiate(const std::string& name, - const std::string& pattern) - { return new ScriptIndexTarget(name, pattern); } - -private: - - ScriptIndexTarget(const std::string& name, const std::string& pattern) - : ScriptTarget(name, pattern) - { } - - void DoGenerate() const /* override */; -}; - -class IdentifierTarget : public Target { -public: - - static Target* Instantiate(const std::string& name, - const std::string& pattern) - { return new IdentifierTarget(name, pattern); } - -private: - - IdentifierTarget(const std::string& name, const std::string& pattern) - : Target(name, pattern), id_deps() - { } - - void DoFindDependencies(const std::vector& docs); - - void DoGenerate() const; - - std::vector id_deps; -}; - +/** + * Manages the generation of reStructuredText documents corresponding to + * particular targets that are specified in a config file. The config file + * is a simple list of one target per line, with the target format being + * a tab-delimited list of target-type, target-pattern, and target-output-file. + */ class Config { + public: + /** + * Read a Broxygen configuration file, parsing all targets in it. + * @param file The file containing a list of Broxygen targets. If it's + * an empty string most methods are a no-op. + * @param delim The delimiter between target fields. + */ Config(const std::string& file, const std::string& delim = "\t"); + /** + * Destructor, cleans up targets created when parsing config file. + */ ~Config(); - void FindDependencies(const std::vector& docs); + /** + * Resolves dependency information for each target. + * @param infos All known information objects for documentable things. + */ + void FindDependencies(const std::vector& infos); + /** + * Build each Broxygen target (i.e. write out the reST documents to disk). + */ void GenerateDocs() const; + /** + * @return The modification time of the config file, or 0 if config + * file was specified by an empty string. + */ time_t GetModificationTime() const; private: std::string file; std::vector targets; + TargetFactory target_factory; }; - } // namespace broxygen #endif diff --git a/src/broxygen/Document.cc b/src/broxygen/Document.cc deleted file mode 100644 index bb64f90555..0000000000 --- a/src/broxygen/Document.cc +++ /dev/null @@ -1,660 +0,0 @@ -#include "Document.h" -#include "Manager.h" -#include "utils.h" - -#include "util.h" -#include "Val.h" -#include "Desc.h" -#include "Reporter.h" - -#include -#include - -using namespace broxygen; -using namespace std; - -static bool is_public_api(const ID* id) - { - return (id->Scope() == SCOPE_GLOBAL) || - (id->Scope() == SCOPE_MODULE && id->IsExport()); - } - -static string make_heading(const string& heading, char underline) - { - return heading + "\n" + string(heading.size(), underline) + "\n"; - } - -PackageDocument::PackageDocument(const string& arg_name) - : Document(), - pkg_name(arg_name), readme() - { - string readme_file = find_file(pkg_name + "/README", bro_path()); - - if ( readme_file.empty() ) - return; - - ifstream f(readme_file.c_str()); - - if ( ! f.is_open() ) - reporter->InternalWarning("Broxygen failed to open '%s': %s", - readme_file.c_str(), strerror(errno)); - - string line; - - while ( getline(f, line) ) - readme.push_back(line); - - if ( f.bad() ) - reporter->InternalWarning("Broxygen error reading '%s': %s", - readme_file.c_str(), strerror(errno)); - } - -string PackageDocument::DoReStructuredText(bool roles_only) const - { - string rval = fmt(":doc:`%s `\n\n", pkg_name.c_str(), - pkg_name.c_str()); - - for ( size_t i = 0; i < readme.size(); ++i ) - rval += " " + readme[i] + "\n"; - - return rval; - } - -IdentifierDocument::IdentifierDocument(ID* arg_id, ScriptDocument* script) - : Document(), - comments(), id(arg_id), initial_val_desc(), redefs(), fields(), - last_field_seen(), declaring_script(script) - { - Ref(id); - - if ( id->ID_Val() ) - { - ODesc d; - id->ID_Val()->Describe(&d); - initial_val_desc = d.Description(); - } - } - -IdentifierDocument::~IdentifierDocument() - { - Unref(id); - - for ( redef_list::const_iterator it = redefs.begin(); it != redefs.end(); - ++it ) - delete *it; - - for ( record_field_map::const_iterator it = fields.begin(); - it != fields.end(); ++it ) - delete it->second; - } - -void IdentifierDocument::AddRedef(const string& script, - const vector& comments) - { - Redefinition* redef = new Redefinition(); - redef->from_script = script; - - if ( id->ID_Val() ) - { - ODesc d; - id->ID_Val()->Describe(&d); - redef->new_val_desc = d.Description(); - } - - redef->comments = comments; - redefs.push_back(redef); - } - -void IdentifierDocument::AddRecordField(const TypeDecl* field, - const string& script, - vector& comments) - { - RecordField* rf = new RecordField(); - rf->field = new TypeDecl(*field); - rf->from_script = script; - rf->comments = comments; - fields[rf->field->id] = rf; - last_field_seen = rf; - } - -vector IdentifierDocument::GetComments() const - { - return comments; - } - -vector IdentifierDocument::GetFieldComments(const string& field) const - { - record_field_map::const_iterator it = fields.find(field); - - if ( it == fields.end() ) - return vector(); - - return it->second->comments; - } - -list -IdentifierDocument::GetRedefs(const string& from_script) const - { - list rval; - - for ( redef_list::const_iterator it = redefs.begin(); it != redefs.end(); - ++it ) - { - if ( from_script == (*it)->from_script ) - rval.push_back(*(*it)); - } - - return rval; - } - -string IdentifierDocument::GetDeclaringScriptForField(const string& field) const - { - record_field_map::const_iterator it = fields.find(field); - - if ( it == fields.end() ) - return ""; - - return it->second->from_script; - } - -string IdentifierDocument::DoReStructuredText(bool roles_only) const - { - ODesc d; - d.SetIndentSpaces(3); - d.SetQuotes(true); - id->DescribeReST(&d, roles_only); - - if ( comments.empty() ) - return d.Description(); - - d.ClearIndentLevel(); - d.PushIndent(); - - for ( size_t i = 0; i < comments.size(); ++i ) - { - if ( i > 0 ) - d.NL(); - - if ( IsFunc(id->Type()->Tag()) ) - { - string s = comments[i]; - - if ( broxygen::prettify_params(s) ) - d.NL(); - - d.Add(s.c_str()); - } - else - d.Add(comments[i].c_str()); - } - - return d.Description(); - } - -ScriptDocument::ScriptDocument(const string& arg_name, const string& arg_path) - : Document(), - name(arg_name), path(arg_path), - is_pkg_loader(SafeBasename(name).result == PACKAGE_LOADER), - dependencies(), module_usages(), comments(), identifier_docs(), - options(), constants(), state_vars(), types(), events(), hooks(), - functions(), redefs() - { - } - -void ScriptDocument::AddIdentifierDoc(IdentifierDocument* doc) - { - identifier_docs[doc->Name()] = doc; - } - -void ScriptDocument::DoInitPostScript() - { - for ( id_doc_map::const_iterator it = identifier_docs.begin(); - it != identifier_docs.end(); ++it ) - { - IdentifierDocument* doc = it->second; - ID* id = doc->GetID(); - - if ( ! is_public_api(id) ) - continue; - - if ( id->AsType() ) - { - types.push_back(doc); - DBG_LOG(DBG_BROXYGEN, "Filter id '%s' in '%s' as a type", - id->Name(), name.c_str()); - continue; - } - - if ( IsFunc(id->Type()->Tag()) ) - { - switch ( id->Type()->AsFuncType()->Flavor() ) { - case FUNC_FLAVOR_HOOK: - DBG_LOG(DBG_BROXYGEN, "Filter id '%s' in '%s' as a hook", - id->Name(), name.c_str()); - hooks.push_back(doc); - break; - case FUNC_FLAVOR_EVENT: - DBG_LOG(DBG_BROXYGEN, "Filter id '%s' in '%s' as a event", - id->Name(), name.c_str()); - events.push_back(doc); - break; - case FUNC_FLAVOR_FUNCTION: - DBG_LOG(DBG_BROXYGEN, "Filter id '%s' in '%s' as a function", - id->Name(), name.c_str()); - functions.push_back(doc); - break; - default: - reporter->InternalError("Invalid function flavor"); - break; - } - - continue; - } - - if ( id->IsConst() ) - { - if ( id->FindAttr(ATTR_REDEF) ) - { - DBG_LOG(DBG_BROXYGEN, "Filter id '%s' in '%s' as an option", - id->Name(), name.c_str()); - options.push_back(doc); - } - else - { - DBG_LOG(DBG_BROXYGEN, "Filter id '%s' in '%s' as a constant", - id->Name(), name.c_str()); - constants.push_back(doc); - } - - continue; - } - - if ( id->Type()->Tag() == TYPE_ENUM ) - // Enums are always referenced/documented from the type's - // documentation. - continue; - - DBG_LOG(DBG_BROXYGEN, "Filter id '%s' in '%s' as a state variable", - id->Name(), name.c_str()); - state_vars.push_back(doc); - } - } - -vector ScriptDocument::GetComments() const - { - return comments; - } - -static size_t end_of_first_sentence(const string& s) - { - size_t rval = 0; - - while ( (rval = s.find_first_of('.', rval)) != string::npos ) - { - if ( rval == s.size() - 1 ) - // Period is at end of string. - return rval; - - if ( isspace(s[rval + 1]) ) - // Period has a space after it. - return rval; - - // Period has some non-space character after it, keep looking. - ++rval; - } - - return rval; - } - -static bool is_all_whitespace(const string& s) - { - for ( size_t i = 0; i < s.size(); ++i ) - if ( ! isspace(s[i]) ) - return false; - - return true; - } - -static vector summary_comment(const vector& cmnts) - { - vector rval; - - for ( size_t i = 0; i < cmnts.size(); ++i ) - { - size_t end = end_of_first_sentence(cmnts[i]); - - if ( end == string::npos ) - { - if ( is_all_whitespace(cmnts[i]) ) - break; - - rval.push_back(cmnts[i]); - } - else - { - rval.push_back(cmnts[i].substr(0, end + 1)); - break; - } - } - - return rval; - } - -class ReStructuredTextTable { -public: - - ReStructuredTextTable(size_t arg_num_cols) - : num_cols(arg_num_cols), rows(), longest_row_in_column() - { - for ( size_t i = 0; i < num_cols; ++i ) - longest_row_in_column.push_back(1); - } - - void AddRow(const vector& new_row) - { - assert(new_row.size() == num_cols); - rows.push_back(new_row); - - for ( size_t i = 0; i < new_row.size(); ++i ) - if ( new_row[i].size() > longest_row_in_column[i] ) - longest_row_in_column[i] = new_row[i].size(); - } - - static string MakeBorder(const vector col_sizes, char border) - { - string rval; - - for ( size_t i = 0; i < col_sizes.size(); ++i ) - { - if ( i > 0 ) - rval += " "; - - rval += string(col_sizes[i], border); - } - - rval += "\n"; - return rval; - } - - string AsString(char border) const - { - string rval = MakeBorder(longest_row_in_column, border); - - for ( size_t row = 0; row < rows.size(); ++row ) - { - for ( size_t col = 0; col < num_cols; ++col ) - { - if ( col > 0 ) - { - size_t last = rows[row][col - 1].size(); - size_t longest = longest_row_in_column[col - 1]; - size_t whitespace = longest - last + 1; - rval += string(whitespace, ' '); - } - - rval += rows[row][col]; - } - - rval += "\n"; - } - - rval += MakeBorder(longest_row_in_column, border); - return rval; - } - - -private: - - size_t num_cols; - vector > rows; - vector longest_row_in_column; -}; - -static void add_summary_rows(const ODesc& id_desc, const vector& cmnts, - ReStructuredTextTable* table) - { - vector row; - row.push_back(id_desc.Description()); - - if ( cmnts.empty() ) - { - row.push_back(""); - table->AddRow(row); - return; - } - - row.push_back(cmnts[0]); - table->AddRow(row); - - for ( size_t i = 1; i < cmnts.size(); ++i ) - { - row.clear(); - row.push_back(""); - row.push_back(cmnts[i]); - table->AddRow(row); - } - } - -static string make_summary(const string& heading, char underline, char border, - const list& id_list) - { - if ( id_list.empty() ) - return ""; - - ReStructuredTextTable table(2); - - for ( list::const_iterator it = id_list.begin(); - it != id_list.end(); ++it ) - { - ID* id = (*it)->GetID(); - ODesc d; - d.SetQuotes(1); - id->DescribeReSTShort(&d); - add_summary_rows(d, summary_comment((*it)->GetComments()), &table); - } - - return make_heading(heading, underline) + table.AsString(border) + "\n"; - } - -static string make_redef_summary(const string& heading, char underline, - char border, const string& from_script, - const id_doc_set& id_set) - { - if ( id_set.empty() ) - return ""; - - ReStructuredTextTable table(2); - - for ( id_doc_set::const_iterator it = id_set.begin(); it != id_set.end(); - ++it ) - { - ID* id = (*it)->GetID(); - ODesc d; - d.SetQuotes(1); - id->DescribeReSTShort(&d); - - typedef list redef_list; - redef_list redefs = (*it)->GetRedefs(from_script); - - for ( redef_list::const_iterator iit = redefs.begin(); - iit != redefs.end(); ++iit ) - add_summary_rows(d, summary_comment(iit->comments), &table); - } - - return make_heading(heading, underline) + table.AsString(border) + "\n"; - } - -static string make_details(const string& heading, char underline, - const list& id_list) - { - if ( id_list.empty() ) - return ""; - - string rval = make_heading(heading, underline); - - for ( list::const_iterator it = id_list.begin(); - it != id_list.end(); ++it ) - { - rval += (*it)->ReStructuredText(); - rval += "\n\n"; - } - - return rval; - } - -static string make_redef_details(const string& heading, char underline, - const id_doc_set& id_set) - { - if ( id_set.empty() ) - return ""; - - string rval = make_heading(heading, underline); - - for ( id_doc_set::const_iterator it = id_set.begin(); - it != id_set.end(); ++it ) - { - rval += (*it)->ReStructuredText(true); - rval += "\n\n"; - } - - return rval; - } - -string ScriptDocument::DoReStructuredText(bool roles_only) const - { - string rval; - - rval += ":tocdepth: 3\n\n"; - rval += make_heading(name, '='); - - for ( string_set::const_iterator it = module_usages.begin(); - it != module_usages.end(); ++it ) - rval += ".. bro:namespace:: " + *it + "\n"; - - rval += "\n"; - - for ( size_t i = 0; i < comments.size(); ++i ) - rval += comments[i] + "\n"; - - rval += "\n"; - - if ( ! module_usages.empty() ) - { - rval += module_usages.size() > 1 ? ":Namespaces: " : ":Namespace: "; - - for ( string_set::const_iterator it = module_usages.begin(); - it != module_usages.end(); ++it ) - { - if ( it != module_usages.begin() ) - rval += ", "; - - rval += *it; - } - - rval += "\n"; - } - - if ( ! dependencies.empty() ) - { - rval += ":Imports: "; - - for ( string_set::const_iterator it = dependencies.begin(); - it != dependencies.end(); ++it ) - { - if ( it != dependencies.begin() ) - rval += ", "; - - string path = find_file(*it, bro_path(), "bro"); - string doc = *it; - - if ( ! path.empty() && is_dir(path.c_str()) ) - // Reference the package. - doc += "/index"; - - rval += fmt(":doc:`%s `", it->c_str(), doc.c_str()); - } - - rval += "\n"; - } - - rval += fmt(":Source File: :download:`/scripts/%s`\n", name.c_str()); - rval += "\n"; - rval += make_heading("Summary", '~'); - rval += make_summary("Options", '#', '=', options); - rval += make_summary("Constants", '#', '=', constants); - rval += make_summary("State Variables", '#', '=', state_vars); - rval += make_summary("Types", '#', '=', types); - rval += make_redef_summary("Redefinitions", '#', '=', name, redefs); - rval += make_summary("Events", '#', '=', events); - rval += make_summary("Hooks", '#', '=', hooks); - rval += make_summary("Functions", '#', '=', functions); - rval += "\n"; - rval += make_heading("Detailed Interface", '~'); - rval += make_details("Options", '#', options); - rval += make_details("Constants", '#', constants); - rval += make_details("State Variables", '#', state_vars); - rval += make_details("Types", '#', types); - //rval += make_redef_details("Redefinitions", '#', redefs); - rval += make_details("Events", '#', events); - rval += make_details("Hooks", '#', hooks); - rval += make_details("Functions", '#', functions); - - return rval; - } - -static time_t get_mtime(const string& filename) - { - struct stat s; - - if ( stat(filename.c_str(), &s) < 0 ) - reporter->InternalError("Broxygen failed to stat file '%s': %s", - filename.c_str(), strerror(errno)); - - return s.st_mtime; - } - -time_t IdentifierDocument::DoGetModificationTime() const - { - // Could probably get away with just checking the set of scripts that - // contributed to the ID declaration/redefinitions, but this is easier... - return declaring_script->GetModificationTime(); - } - -time_t ScriptDocument::DoGetModificationTime() const - { - time_t most_recent = get_mtime(path); - - for ( string_set::const_iterator it = dependencies.begin(); - it != dependencies.end(); ++it ) - { - Document* doc = broxygen_mgr->GetScriptDoc(*it); - - if ( ! doc ) - { - string pkg_name = *it + "/" + PACKAGE_LOADER; - doc = broxygen_mgr->GetScriptDoc(pkg_name); - - if ( ! doc ) - reporter->InternalWarning("Broxygen failed to get mtime of %s", - it->c_str()); - continue; - } - - time_t dep_mtime = doc->GetModificationTime(); - - if ( dep_mtime > most_recent ) - most_recent = dep_mtime; - } - - return most_recent; - } - -time_t PackageDocument::DoGetModificationTime() const - { - string readme_file = find_file(pkg_name + "/README", bro_path()); - - if ( readme_file.empty() ) - return 0; - - return get_mtime(readme_file); - } diff --git a/src/broxygen/Document.h b/src/broxygen/Document.h deleted file mode 100644 index 10155ab0eb..0000000000 --- a/src/broxygen/Document.h +++ /dev/null @@ -1,215 +0,0 @@ -#ifndef BROXYGEN_DOCUMENT_H -#define BROXYGEN_DOCUMENT_H - -#include -#include -#include -#include -#include -#include -#include - -#include "ID.h" -#include "Type.h" - -namespace broxygen { - -// TODO: documentation... - -class Document { - -public: - - Document() - { } - - virtual ~Document() - { } - - time_t GetModificationTime() const - { return DoGetModificationTime(); } - - std::string Name() const - { return DoName(); } - - std::string ReStructuredText(bool roles_only = false) const - { return DoReStructuredText(roles_only); } - - void InitPostScript() - { return DoInitPostScript(); } - -private: - - virtual time_t DoGetModificationTime() const = 0; - virtual std::string DoName() const = 0; - virtual std::string DoReStructuredText(bool roles_only) const = 0; - virtual void DoInitPostScript() - { } -}; - -class PackageDocument : public Document { - -public: - - PackageDocument(const std::string& name); - - std::vector GetReadme() const - { return readme; } - -private: - - time_t DoGetModificationTime() const; - - std::string DoName() const - { return pkg_name; } - - std::string DoReStructuredText(bool roles_only) const; - - std::string pkg_name; - std::vector readme; -}; - - -class ScriptDocument; - -class IdentifierDocument : public Document { -public: - - IdentifierDocument(ID* id, ScriptDocument* script); - - ~IdentifierDocument(); - - void AddComment(const std::string& comment) - { last_field_seen ? last_field_seen->comments.push_back(comment) - : comments.push_back(comment); } - - void AddComments(const std::vector& cmtns) - { comments.insert(comments.end(), cmtns.begin(), cmtns.end()); } - - void AddRedef(const std::string& from_script, - const std::vector& comments); - - void AddRecordField(const TypeDecl* field, const std::string& script, - std::vector& comments); - - void CompletedTypeDecl() - { last_field_seen = 0; } - - ID* GetID() const - { return id; } - - ScriptDocument* GetDeclaringScript() const - { return declaring_script; } - - std::string GetDeclaringScriptForField(const std::string& field) const; - - std::vector GetComments() const; - - std::vector GetFieldComments(const std::string& field) const; - - struct Redefinition { - std::string from_script; - std::string new_val_desc; - std::vector comments; - }; - - std::list GetRedefs(const std::string& from_script) const; - -private: - - time_t DoGetModificationTime() const; - - std::string DoName() const - { return id->Name(); } - - std::string DoReStructuredText(bool roles_only) const; - - struct RecordField { - ~RecordField() - { delete field; } - - TypeDecl* field; - std::string from_script; - std::vector comments; - }; - - typedef std::list redef_list; - typedef std::map record_field_map; - - std::vector comments; - ID* id; - std::string initial_val_desc; - redef_list redefs; - record_field_map fields; - RecordField* last_field_seen; - ScriptDocument* declaring_script; -}; - -struct IdDocComp { - bool operator() (IdentifierDocument* lhs, IdentifierDocument* rhs) const - { return lhs->Name() < rhs->Name(); } -}; - -typedef std::set id_doc_set; - -class ScriptDocument : public Document { - -public: - - ScriptDocument(const std::string& name, const std::string& path); - - void AddComment(const std::string& comment) - { comments.push_back(comment); } - - void AddDependency(const std::string& name) - { dependencies.insert(name); } - - void AddModule(const std::string& name) - { module_usages.insert(name); } - - void AddIdentifierDoc(IdentifierDocument* doc); - - void AddRedef(IdentifierDocument* doc) - { redefs.insert(doc); } - - bool IsPkgLoader() const - { return is_pkg_loader; } - - std::vector GetComments() const; - -private: - - typedef std::map id_doc_map; - typedef std::list id_doc_list; - typedef std::set string_set; - - time_t DoGetModificationTime() const; - - std::string DoName() const - { return name; } - - std::string DoReStructuredText(bool roles_only) const; - - void DoInitPostScript() /* override */; - - std::string name; - std::string path; - bool is_pkg_loader; - string_set dependencies; - string_set module_usages; - std::vector comments; - id_doc_map identifier_docs; - id_doc_list options; - id_doc_list constants; - id_doc_list state_vars; - id_doc_list types; - id_doc_list events; - id_doc_list hooks; - id_doc_list functions; - id_doc_set redefs; -}; - - -} // namespace broxygen - -#endif diff --git a/src/broxygen/IdentifierInfo.cc b/src/broxygen/IdentifierInfo.cc new file mode 100644 index 0000000000..870f128171 --- /dev/null +++ b/src/broxygen/IdentifierInfo.cc @@ -0,0 +1,146 @@ +#include "IdentifierInfo.h" +#include "utils.h" + +#include "Desc.h" +#include "Val.h" + +using namespace std; +using namespace broxygen; + +IdentifierInfo::IdentifierInfo(ID* arg_id, ScriptInfo* script) + : Info(), + comments(), id(arg_id), initial_val_desc(), redefs(), fields(), + last_field_seen(), declaring_script(script) + { + Ref(id); + + if ( id->ID_Val() ) + { + ODesc d; + id->ID_Val()->Describe(&d); + initial_val_desc = d.Description(); + } + } + +IdentifierInfo::~IdentifierInfo() + { + Unref(id); + + for ( redef_list::const_iterator it = redefs.begin(); it != redefs.end(); + ++it ) + delete *it; + + for ( record_field_map::const_iterator it = fields.begin(); + it != fields.end(); ++it ) + delete it->second; + } + +void IdentifierInfo::AddRedef(const string& script, + const vector& comments) + { + Redefinition* redef = new Redefinition(); + redef->from_script = script; + + if ( id->ID_Val() ) + { + ODesc d; + id->ID_Val()->Describe(&d); + redef->new_val_desc = d.Description(); + } + + redef->comments = comments; + redefs.push_back(redef); + } + +void IdentifierInfo::AddRecordField(const TypeDecl* field, + const string& script, + vector& comments) + { + RecordField* rf = new RecordField(); + rf->field = new TypeDecl(*field); + rf->from_script = script; + rf->comments = comments; + fields[rf->field->id] = rf; + last_field_seen = rf; + } + +vector IdentifierInfo::GetComments() const + { + return comments; + } + +vector IdentifierInfo::GetFieldComments(const string& field) const + { + record_field_map::const_iterator it = fields.find(field); + + if ( it == fields.end() ) + return vector(); + + return it->second->comments; + } + +list +IdentifierInfo::GetRedefs(const string& from_script) const + { + list rval; + + for ( redef_list::const_iterator it = redefs.begin(); it != redefs.end(); + ++it ) + { + if ( from_script == (*it)->from_script ) + rval.push_back(*(*it)); + } + + return rval; + } + +string IdentifierInfo::GetDeclaringScriptForField(const string& field) const + { + record_field_map::const_iterator it = fields.find(field); + + if ( it == fields.end() ) + return ""; + + return it->second->from_script; + } + +string IdentifierInfo::DoReStructuredText(bool roles_only) const + { + ODesc d; + d.SetIndentSpaces(3); + d.SetQuotes(true); + id->DescribeReST(&d, roles_only); + + if ( comments.empty() ) + return d.Description(); + + d.ClearIndentLevel(); + d.PushIndent(); + + for ( size_t i = 0; i < comments.size(); ++i ) + { + if ( i > 0 ) + d.NL(); + + if ( IsFunc(id->Type()->Tag()) ) + { + string s = comments[i]; + + if ( broxygen::prettify_params(s) ) + d.NL(); + + d.Add(s.c_str()); + } + else + d.Add(comments[i].c_str()); + } + + return d.Description(); + } + +time_t IdentifierInfo::DoGetModificationTime() const + { + // Could probably get away with just checking the set of scripts that + // contributed to the ID declaration/redefinitions, but this is easier... + return declaring_script->GetModificationTime(); + } diff --git a/src/broxygen/IdentifierInfo.h b/src/broxygen/IdentifierInfo.h new file mode 100644 index 0000000000..322f776ee6 --- /dev/null +++ b/src/broxygen/IdentifierInfo.h @@ -0,0 +1,162 @@ +#ifndef BROXYGEN_IDENTIFIERINFO_H +#define BROXYGEN_IDENTIFIERINFO_H + +#include "Info.h" +#include "ScriptInfo.h" + +#include "ID.h" +#include "Type.h" + +#include +#include +#include +#include + +namespace broxygen { + +class ScriptInfo; + +/** + * Information regarding a script-level identifier and its documentation. + */ +class IdentifierInfo : public Info { + +public: + + /** + * Create a new identifier info object. + * @param id The script-level identifier. + * @param script The info object associated with the script in which \a id + * is declared. + */ + IdentifierInfo(ID* id, ScriptInfo* script); + + /** + * Dtor. Releases any references to script-level objects. + */ + ~IdentifierInfo(); + + /** + * Add a comment associated with the identifier. If the identifier is a + * record type and it's in the middle of parsing fields, the comment is + * associated with the last field that was parsed. + * @param comment A string extracted from Broxygen-style comment. + */ + void AddComment(const std::string& comment) + { last_field_seen ? last_field_seen->comments.push_back(comment) + : comments.push_back(comment); } + + /** + * Associate several comments with the identifier. They will be appended + * to the end of the list of any current comments. + * @param cmtns A vector of comments to associate. + */ + void AddComments(const std::vector& cmtns) + { comments.insert(comments.end(), cmtns.begin(), cmtns.end()); } + + /** + * Register a redefinition of the identifier. + * @param from_script The script in which the redef occurred. + * @param comments Comments associated with the redef statement. + */ + void AddRedef(const std::string& from_script, + const std::vector& comments); + + /** + * Register a record field associated with the identifier + * (which is implicitly a record type). + * @param field The name/type information of the field. + * @param script The script in which the field was declared. This may + * differ from the script in which a record type is declared due to redefs. + * @param comments Comments associated with the record field. + */ + void AddRecordField(const TypeDecl* field, const std::string& script, + std::vector& comments); + + /** + * Signals that a record type has been completely parsed. This resets + * internal tracking of the last record field seen so that "##<"-style + * comments are correctly associated. + */ + void CompletedTypeDecl() + { last_field_seen = 0; } + + /** + * @return the script-level ID tracked by this info object. + */ + ID* GetID() const + { return id; } + + /** + * @return The script which declared the script-level identifier. + */ + ScriptInfo* GetDeclaringScript() const + { return declaring_script; } + + /** + * @param field A record field name. + * @return The script which declared the record field name. + */ + std::string GetDeclaringScriptForField(const std::string& field) const; + + /** + * @return All Broxygen comments associated with the identifier. + */ + std::vector GetComments() const; + + /** + * @param field A record field name. + * @return All Broxygen comments associated with the record field. + */ + std::vector GetFieldComments(const std::string& field) const; + + /** + * Tracks useful information related to a redef. + */ + struct Redefinition { + std::string from_script; /**< Name of script doing the redef. */ + std::string new_val_desc; /**< Description of new value bound to ID. */ + std::vector comments; /**< Broxygen comments on redef. */ + }; + + /** + * Get a list of information about redefinitions of the identifier within + * a particular script. + * @param from_script The name of a script in which to look for redefs. + * @return A list of redefs that occurred in \a from_script. + */ + std::list GetRedefs(const std::string& from_script) const; + +private: + + time_t DoGetModificationTime() const; + + std::string DoName() const + { return id->Name(); } + + std::string DoReStructuredText(bool roles_only) const; + + struct RecordField { + ~RecordField() + { delete field; } + + TypeDecl* field; + std::string from_script; + std::vector comments; + }; + + typedef std::list redef_list; + typedef std::map record_field_map; + + std::vector comments; + ID* id; + std::string initial_val_desc; + redef_list redefs; + record_field_map fields; + RecordField* last_field_seen; + ScriptInfo* declaring_script; +}; + +} // namespace broxygen + +#endif diff --git a/src/broxygen/Info.h b/src/broxygen/Info.h new file mode 100644 index 0000000000..cd7282a293 --- /dev/null +++ b/src/broxygen/Info.h @@ -0,0 +1,71 @@ +#ifndef BROXYGEN_INFO_H +#define BROXYGEN_INFO_H + +#include +#include + +namespace broxygen { + +/** + * Abstract base class for any thing that Broxygen can document. + */ +class Info { + +public: + + /** + * Ctor. + */ + Info() + { } + + /** + * Dtor. + */ + virtual ~Info() + { } + + /** + * @return The time any information related to the object was last modified. + */ + time_t GetModificationTime() const + { return DoGetModificationTime(); } + + /** + * @return A unique name for the documentable object. + */ + std::string Name() const + { return DoName(); } + + /** + * Get a reST representation of the object and any associated documentation. + * @param roles_only True if the reST should only use cross-referencing role + * syntax to refer itself instead of using a directive (which declares this + * reST the authoritative "anchor" for cross-references). + * @return A reST representation of the object and associated documentation. + */ + std::string ReStructuredText(bool roles_only = false) const + { return DoReStructuredText(roles_only); } + + /** + * Perform any remaining info gathering/initialization that can only be done + * after all script parsing is complete. + */ + void InitPostScript() + { DoInitPostScript(); } + +private: + + virtual time_t DoGetModificationTime() const = 0; + + virtual std::string DoName() const = 0; + + virtual std::string DoReStructuredText(bool roles_only) const = 0; + + virtual void DoInitPostScript() + { } +}; + +} // namespace broxygen + +#endif diff --git a/src/broxygen/Manager.cc b/src/broxygen/Manager.cc index 68e4c07b98..b923efe61f 100644 --- a/src/broxygen/Manager.cc +++ b/src/broxygen/Manager.cc @@ -27,7 +27,7 @@ static string RemoveLeadingSpace(const string& s) Manager::Manager(const string& arg_config, const string& bro_command) : disabled(), comment_buffer(), comment_buffer_map(), packages(), scripts(), - identifiers(), all_docs(), last_identifier_seen(), incomplete_type(), + identifiers(), all_info(), last_identifier_seen(), incomplete_type(), enum_mappings(), config(arg_config), bro_mtime() { if ( getenv("BRO_DISABLE_BROXYGEN") ) @@ -45,8 +45,8 @@ Manager::Manager(const string& arg_config, const string& bro_command) Manager::~Manager() { - for ( size_t i = 0; i < all_docs.size(); ++i ) - delete all_docs[i]; + for ( size_t i = 0; i < all_info.size(); ++i ) + delete all_info[i]; } void Manager::InitPreScript() @@ -60,10 +60,10 @@ void Manager::InitPostScript() if ( disabled ) return; - for ( size_t i = 0; i < all_docs.size(); ++i ) - all_docs[i]->InitPostScript(); + for ( size_t i = 0; i < all_info.size(); ++i ) + all_info[i]->InitPostScript(); - config.FindDependencies(all_docs); + config.FindDependencies(all_info); } void Manager::GenerateDocs() const @@ -74,39 +74,39 @@ void Manager::GenerateDocs() const config.GenerateDocs(); } -void Manager::File(const string& path) +void Manager::Script(const string& path) { if ( disabled ) return; string name = without_bropath_component(path); - if ( scripts.GetDocument(name) ) + if ( scripts.GetInfo(name) ) { DbgAndWarn(fmt("Duplicate script documentation: %s", name.c_str())); return; } - ScriptDocument* doc = new ScriptDocument(name, path); - scripts.map[name] = doc; - all_docs.push_back(doc); - DBG_LOG(DBG_BROXYGEN, "Made ScriptDocument %s", name.c_str()); + ScriptInfo* info = new ScriptInfo(name, path); + scripts.map[name] = info; + all_info.push_back(info); + DBG_LOG(DBG_BROXYGEN, "Made ScriptInfo %s", name.c_str()); - if ( ! doc->IsPkgLoader() ) + if ( ! info->IsPkgLoader() ) return; name = SafeDirname(name).result; - if ( packages.GetDocument(name) ) + if ( packages.GetInfo(name) ) { DbgAndWarn(fmt("Duplicate package documentation: %s", name.c_str())); return; } - PackageDocument* pkgdoc = new PackageDocument(name); - packages.map[name] = pkgdoc; - all_docs.push_back(pkgdoc); - DBG_LOG(DBG_BROXYGEN, "Made PackageDocument %s", name.c_str()); + PackageInfo* pkginfo = new PackageInfo(name); + packages.map[name] = pkginfo; + all_info.push_back(pkginfo); + DBG_LOG(DBG_BROXYGEN, "Made PackageInfo %s", name.c_str()); } void Manager::ScriptDependency(const string& path, const string& dep) @@ -122,16 +122,16 @@ void Manager::ScriptDependency(const string& path, const string& dep) string name = without_bropath_component(path); string depname = without_bropath_component(dep); - ScriptDocument* script_doc = scripts.GetDocument(name); + ScriptInfo* script_info = scripts.GetInfo(name); - if ( ! script_doc ) + if ( ! script_info ) { DbgAndWarn(fmt("Failed to add script doc dependency %s for %s", depname.c_str(), name.c_str())); return; } - script_doc->AddDependency(depname); + script_info->AddDependency(depname); DBG_LOG(DBG_BROXYGEN, "Added script dependency %s for %s", depname.c_str(), name.c_str()); @@ -146,23 +146,23 @@ void Manager::ModuleUsage(const string& path, const string& module) return; string name = without_bropath_component(path); - ScriptDocument* script_doc = scripts.GetDocument(name); + ScriptInfo* script_info = scripts.GetInfo(name); - if ( ! script_doc ) + if ( ! script_info ) { DbgAndWarn(fmt("Failed to add module usage %s in %s", module.c_str(), name.c_str())); return; } - script_doc->AddModule(module); + script_info->AddModule(module); DBG_LOG(DBG_BROXYGEN, "Added module usage %s in %s", module.c_str(), name.c_str()); } -IdentifierDocument* Manager::CreateIdentifierDoc(ID* id, ScriptDocument* script) +IdentifierInfo* Manager::CreateIdentifierInfo(ID* id, ScriptInfo* script) { - IdentifierDocument* rval = new IdentifierDocument(id, script); + IdentifierInfo* rval = new IdentifierInfo(id, script); rval->AddComments(comment_buffer); comment_buffer.clear(); @@ -176,12 +176,12 @@ IdentifierDocument* Manager::CreateIdentifierDoc(ID* id, ScriptDocument* script) comment_buffer_map.erase(it); } - all_docs.push_back(rval); + all_info.push_back(rval); identifiers.map[id->Name()] = rval; last_identifier_seen = rval; if ( script ) - script->AddIdentifierDoc(rval); + script->AddIdentifierInfo(rval); return rval; } @@ -198,17 +198,17 @@ void Manager::StartType(ID* id) } string script = without_bropath_component(id->GetLocationInfo()->filename); - ScriptDocument* script_doc = scripts.GetDocument(script); + ScriptInfo* script_info = scripts.GetInfo(script); - if ( ! script_doc ) + if ( ! script_info ) { DbgAndWarn(fmt("Can't document identifier %s, lookup of %s failed", id->Name(), script.c_str())); return; } - incomplete_type = CreateIdentifierDoc(id, script_doc); - DBG_LOG(DBG_BROXYGEN, "Made IdentifierDocument (incomplete) %s, in %s", + incomplete_type = CreateIdentifierInfo(id, script_info); + DBG_LOG(DBG_BROXYGEN, "Made IdentifierInfo (incomplete) %s, in %s", id->Name(), script.c_str()); } @@ -236,14 +236,14 @@ void Manager::Identifier(ID* id) enum_mappings[id->Name()] = incomplete_type->GetID()->Name(); } - IdentifierDocument* id_doc = identifiers.GetDocument(id->Name()); + IdentifierInfo* id_info = identifiers.GetInfo(id->Name()); - if ( id_doc ) + if ( id_info ) { - if ( IsFunc(id_doc->GetID()->Type()->Tag()) ) + if ( IsFunc(id_info->GetID()->Type()->Tag()) ) { // Function may already been seen (declaration versus body). - id_doc->AddComments(comment_buffer); + id_info->AddComments(comment_buffer); comment_buffer.clear(); return; } @@ -256,24 +256,24 @@ void Manager::Identifier(ID* id) { // Internally-created identifier (e.g. file/proto analyzer enum tags). // Handled specially since they don't have a script location. - DBG_LOG(DBG_BROXYGEN, "Made internal IdentifierDocument %s", + DBG_LOG(DBG_BROXYGEN, "Made internal IdentifierInfo %s", id->Name()); - CreateIdentifierDoc(id, 0); + CreateIdentifierInfo(id, 0); return; } string script = without_bropath_component(id->GetLocationInfo()->filename); - ScriptDocument* script_doc = scripts.GetDocument(script); + ScriptInfo* script_info = scripts.GetInfo(script); - if ( ! script_doc ) + if ( ! script_info ) { DbgAndWarn(fmt("Can't document identifier %s, lookup of %s failed", id->Name(), script.c_str())); return; } - CreateIdentifierDoc(id, script_doc); - DBG_LOG(DBG_BROXYGEN, "Made IdentifierDocument %s, in script %s", + CreateIdentifierInfo(id, script_info); + DBG_LOG(DBG_BROXYGEN, "Made IdentifierInfo %s, in script %s", id->Name(), script.c_str()); } @@ -283,7 +283,7 @@ void Manager::RecordField(const ID* id, const TypeDecl* field, if ( disabled ) return; - IdentifierDocument* idd = identifiers.GetDocument(id->Name()); + IdentifierInfo* idd = identifiers.GetInfo(id->Name()); if ( ! idd ) { @@ -308,9 +308,9 @@ void Manager::Redef(const ID* id, const string& path) // This is a redef defined on the command line. return; - IdentifierDocument* id_doc = identifiers.GetDocument(id->Name()); + IdentifierInfo* id_info = identifiers.GetInfo(id->Name()); - if ( ! id_doc ) + if ( ! id_info ) { DbgAndWarn(fmt("Can't document redef of %s, identifier lookup failed", id->Name())); @@ -318,19 +318,19 @@ void Manager::Redef(const ID* id, const string& path) } string from_script = without_bropath_component(path); - ScriptDocument* script_doc = scripts.GetDocument(from_script); + ScriptInfo* script_info = scripts.GetInfo(from_script); - if ( ! script_doc ) + if ( ! script_info ) { DbgAndWarn(fmt("Can't document redef of %s, lookup of %s failed", id->Name(), from_script.c_str())); return; } - id_doc->AddRedef(from_script, comment_buffer); - script_doc->AddRedef(id_doc); + id_info->AddRedef(from_script, comment_buffer); + script_info->AddRedef(id_info); comment_buffer.clear(); - last_identifier_seen = id_doc; + last_identifier_seen = id_info; DBG_LOG(DBG_BROXYGEN, "Added redef of %s from %s", id->Name(), from_script.c_str()); } @@ -341,10 +341,10 @@ void Manager::SummaryComment(const string& script, const string& comment) return; string name = without_bropath_component(script); - ScriptDocument* doc = scripts.GetDocument(name); + ScriptInfo* info = scripts.GetInfo(name); - if ( doc ) - doc->AddComment(RemoveLeadingSpace(comment)); + if ( info ) + info->AddComment(RemoveLeadingSpace(comment)); else DbgAndWarn(fmt("Lookup of script %s failed for summary comment %s", name.c_str(), comment.c_str())); diff --git a/src/broxygen/Manager.h b/src/broxygen/Manager.h index 42b9ea828a..bb67cf35cf 100644 --- a/src/broxygen/Manager.h +++ b/src/broxygen/Manager.h @@ -2,7 +2,10 @@ #define BROXYGEN_MANAGER_H #include "Configuration.h" -#include "Document.h" +#include "Info.h" +#include "PackageInfo.h" +#include "ScriptInfo.h" +#include "IdentifierInfo.h" #include "Reporter.h" #include "ID.h" @@ -17,13 +20,20 @@ namespace broxygen { -// TODO: documentation... - +/** + * Map of info objects. Just a wrapper around std::map to improve code + * readability (less typedefs for specific map types and not having to use + * iterators directly to find a particular info object). + */ template -struct DocumentMap { +struct InfoMap { typedef std::map map_type; - T* GetDocument(const std::string& name) const + /** + * @param name Name of an info object to retrieve. + * @return The info object associated with \a name. + */ + T* GetInfo(const std::string& name) const { typename map_type::const_iterator it = map.find(name); return it == map.end() ? 0 : it->second; @@ -32,53 +42,165 @@ struct DocumentMap { map_type map; }; +/** + * Manages all documentation tracking and generation. + */ class Manager { public: + /** + * Ctor. + * @param config Path to a Broxygen config file if documentation is to be + * written to disk. + * @param bro_command The command used to invoke the bro process. + * It's used when checking for out-of-date targets. If the bro binary is + * newer then a target, it needs to be rebuilt. + */ Manager(const std::string& config, const std::string& bro_command); + /** + * Dtor. + */ ~Manager(); + /** + * Do initialization that needs to happen before scripts are parsed. + * Currently nothing outside of what's done in ctor is needed. + */ void InitPreScript(); + /** + * Do initialization that needs to happen after scripts are parsed. + * This is primarly dependency resolution/filtering. + */ void InitPostScript(); + /** + * Builds all Broxygen targets specified by config file and write out + * documentation to disk. + */ void GenerateDocs() const; - void File(const std::string& path); + /** + * Register Bro script for which information/documentation will be gathered. + * @param path Absolute path to Bro script. + */ + void Script(const std::string& path); + /** + * Register Bro script dependency ("@load"). + * @param path Absolute path to a Bro script. + * @param dep Absolute path to a Bro script being "@load"d from script given + * by \a path. + */ void ScriptDependency(const std::string& path, const std::string& dep); + /** + * Register a module usage (script may export identifiers in to the + * module namespace). + * @param path Absolute path to a Bro script. + * @param module The module which script given by \a path is using. + */ void ModuleUsage(const std::string& path, const std::string& module); + /** + * Signal that a record or enum type is now being parsed. + * @param id The record or enum type identifier. + */ void StartType(ID* id); + /** + * Register a script-level identifier for which information/documentation + * will be gathered. + * @param id The script-level identifier. + */ void Identifier(ID* id); + /** + * Register a record-field for which information/documentation will be + * gathered. + * @param id The identifier of the record type which has the field. + * @param field The field name/type information. + * @param path Absolute path to a Bro script in which this field is + * declared. This can be different from the place where the record type + * is declared due to redefs. + */ void RecordField(const ID* id, const TypeDecl* field, const std::string& path); + /** + * Register a redefinition of a particular identifier. + * @param id The identifier being redef'd. + * @param path Absolute path to a Bro script doing the redef. + */ void Redef(const ID* id, const std::string& path); + /** + * Register Broxygen script summary content. + * @param path Absolute path to a Bro script. + * @param comment Broxygen-style summary comment ("##!") to associate with + * script given by \a path. + */ void SummaryComment(const std::string& path, const std::string& comment); + /** + * Register a Broxygen comment ("##") for an upcoming identifier (i.e. + * this content is buffered and consumed by next identifier/field + * declaration. + * @param comment Content of the Broxygen comment. + */ void PreComment(const std::string& comment); + /** + * Register a Broxygen comment ("##<") for the last identifier seen. + * @param comment Content of the Broxygen comment. + * @param identifier_hint Expected name of identifier with which to + * associate \a comment. + */ void PostComment(const std::string& comment, const std::string& identifier_hint = ""); + /** + * @param id Name of script-level enum identifier. + * @return The name of the enum's type. + */ std::string GetEnumTypeName(const std::string& id) const; - IdentifierDocument* GetIdentifierDoc(const std::string& name) const - { return identifiers.GetDocument(name); } + /** + * @param name Name of a script-level identifier. + * @return an identifier info object associated with \a name or a null + * pointer if it's not a known identifier. + */ + IdentifierInfo* GetIdentifierInfo(const std::string& name) const + { return identifiers.GetInfo(name); } - ScriptDocument* GetScriptDoc(const std::string& name) const - { return scripts.GetDocument(name); } + /** + * @param name Name of a Bro script ("normalized" to be a path relative + * to a component within BROPATH). + * @return a script info object associated with \a name or a null pointer + * if it's not a known script name. + */ + ScriptInfo* GetScriptInfo(const std::string& name) const + { return scripts.GetInfo(name); } - PackageDocument* GetPackageDoc(const std::string& name) const - { return packages.GetDocument(name); } + /** + * @param name Nmae of a Bro script package ("normalized" to be a path + * relative to a component within BROPATH). + * @return a package info object assocated with \a name or a null pointer + * if it's not a known package name. + */ + PackageInfo* GetPackageInfo(const std::string& name) const + { return packages.GetInfo(name); } + /** + * Check if a Broxygen target is up-to-date. + * @param target_file output file of a Broxygen target. + * @param dependencies all dependencies of the target. + * @return true if modification time of \a target_file is newer than + * modification time of Bro binary, Broxygen config file, and all + * dependencies, else false. + */ template bool IsUpToDate(const std::string& target_file, const std::vector& dependencies) const; @@ -88,18 +210,18 @@ private: typedef std::vector comment_buffer_t; typedef std::map comment_buffer_map_t; - IdentifierDocument* CreateIdentifierDoc(ID* id, ScriptDocument* script); + IdentifierInfo* CreateIdentifierInfo(ID* id, ScriptInfo* script); bool disabled; - comment_buffer_t comment_buffer; // For whatever next identifier that comes in. + comment_buffer_t comment_buffer; // For whatever next identifier comes in. comment_buffer_map_t comment_buffer_map; // For a particular identifier. - DocumentMap packages; - DocumentMap scripts; - DocumentMap identifiers; - std::vector all_docs; - IdentifierDocument* last_identifier_seen; - IdentifierDocument* incomplete_type; - std::map enum_mappings; + InfoMap packages; + InfoMap scripts; + InfoMap identifiers; + std::vector all_info; + IdentifierInfo* last_identifier_seen; + IdentifierInfo* incomplete_type; + std::map enum_mappings; // enum id -> enum type id Config config; time_t bro_mtime; }; diff --git a/src/broxygen/PackageInfo.cc b/src/broxygen/PackageInfo.cc new file mode 100644 index 0000000000..c01182dfeb --- /dev/null +++ b/src/broxygen/PackageInfo.cc @@ -0,0 +1,55 @@ +#include "PackageInfo.h" +#include "utils.h" + +#include "Reporter.h" + +#include + +using namespace std; +using namespace broxygen; + +PackageInfo::PackageInfo(const string& arg_name) + : Info(), + pkg_name(arg_name), readme() + { + string readme_file = find_file(pkg_name + "/README", bro_path()); + + if ( readme_file.empty() ) + return; + + ifstream f(readme_file.c_str()); + + if ( ! f.is_open() ) + reporter->InternalWarning("Broxygen failed to open '%s': %s", + readme_file.c_str(), strerror(errno)); + + string line; + + while ( getline(f, line) ) + readme.push_back(line); + + if ( f.bad() ) + reporter->InternalWarning("Broxygen error reading '%s': %s", + readme_file.c_str(), strerror(errno)); + } + +string PackageInfo::DoReStructuredText(bool roles_only) const + { + string rval = fmt(":doc:`%s `\n\n", pkg_name.c_str(), + pkg_name.c_str()); + + for ( size_t i = 0; i < readme.size(); ++i ) + rval += " " + readme[i] + "\n"; + + return rval; + } + +time_t PackageInfo::DoGetModificationTime() const + { + string readme_file = find_file(pkg_name + "/README", bro_path()); + + if ( readme_file.empty() ) + return 0; + + return broxygen::get_mtime(readme_file); + } diff --git a/src/broxygen/PackageInfo.h b/src/broxygen/PackageInfo.h new file mode 100644 index 0000000000..fe0e3c29a7 --- /dev/null +++ b/src/broxygen/PackageInfo.h @@ -0,0 +1,48 @@ +#ifndef BROXYGEN_PACKAGEINFO_H +#define BROXYGEN_PACKAGEINFO_H + +#include "Info.h" + +#include +#include + +namespace broxygen { + +/** + * Information about a Bro script package. + */ +class PackageInfo : public Info { + +public: + + /** + * Ctor. + * @param name The name of the Bro script package (relative path from a + * component within BROPATH. + */ + PackageInfo(const std::string& name); + + /** + * @return The content of the package's README file, each line being + * an element in the returned vector. If the package has no README, the + * vector is empty. + */ + std::vector GetReadme() const + { return readme; } + +private: + + time_t DoGetModificationTime() const; + + std::string DoName() const + { return pkg_name; } + + std::string DoReStructuredText(bool roles_only) const; + + std::string pkg_name; + std::vector readme; +}; + +} // namespace broxygen + +#endif diff --git a/src/broxygen/ReStructuredTextTable.cc b/src/broxygen/ReStructuredTextTable.cc new file mode 100644 index 0000000000..c0e3b1920a --- /dev/null +++ b/src/broxygen/ReStructuredTextTable.cc @@ -0,0 +1,66 @@ +#include "ReStructuredTextTable.h" + +#include + +using namespace std; +using namespace broxygen; + +ReStructuredTextTable::ReStructuredTextTable(size_t arg_num_cols) + : num_cols(arg_num_cols), rows(), longest_row_in_column() + { + for ( size_t i = 0; i < num_cols; ++i ) + longest_row_in_column.push_back(1); + } + +void ReStructuredTextTable::AddRow(const vector& new_row) + { + assert(new_row.size() == num_cols); + rows.push_back(new_row); + + for ( size_t i = 0; i < new_row.size(); ++i ) + if ( new_row[i].size() > longest_row_in_column[i] ) + longest_row_in_column[i] = new_row[i].size(); + } + +string ReStructuredTextTable::MakeBorder(const vector col_sizes, + char border) + { + string rval; + + for ( size_t i = 0; i < col_sizes.size(); ++i ) + { + if ( i > 0 ) + rval += " "; + + rval += string(col_sizes[i], border); + } + + rval += "\n"; + return rval; + } + +string ReStructuredTextTable::AsString(char border) const + { + string rval = MakeBorder(longest_row_in_column, border); + + for ( size_t row = 0; row < rows.size(); ++row ) + { + for ( size_t col = 0; col < num_cols; ++col ) + { + if ( col > 0 ) + { + size_t last = rows[row][col - 1].size(); + size_t longest = longest_row_in_column[col - 1]; + size_t whitespace = longest - last + 1; + rval += string(whitespace, ' '); + } + + rval += rows[row][col]; + } + + rval += "\n"; + } + + rval += MakeBorder(longest_row_in_column, border); + return rval; + } diff --git a/src/broxygen/ReStructuredTextTable.h b/src/broxygen/ReStructuredTextTable.h new file mode 100644 index 0000000000..c00c2191fe --- /dev/null +++ b/src/broxygen/ReStructuredTextTable.h @@ -0,0 +1,51 @@ +#ifndef BROXYGEN_RESTTABLE_H +#define BROXYGEN_RESTTABLE_H + +#include +#include + +namespace broxygen { + +/** + * A reST table with arbitrary number of columns. + */ +class ReStructuredTextTable { +public: + + /** + * Create the reST table object. + * @param arg_num_cols The number of columns in the table. + */ + ReStructuredTextTable(size_t arg_num_cols); + + /** + * Add a new content row to the table. + * @param new_row A vector with one element for each column in the table. + */ + void AddRow(const std::vector& new_row); + + /** + * @param col_sizes Vector of column sizes (width in number of characters). + * @param border Character to use for the border. + * @return A border sized appropriated for the table with columns of sizes + * denoted by \a col_sizes. + */ + static std::string MakeBorder(const std::vector col_sizes, + char border); + + /** + * @param border Character to use for the border. + * @return the reST representation of the table and its content. + */ + std::string AsString(char border) const; + +private: + + size_t num_cols; + std::vector > rows; + std::vector longest_row_in_column; +}; + +} // namespace broxygen + +#endif diff --git a/src/broxygen/ScriptInfo.cc b/src/broxygen/ScriptInfo.cc new file mode 100644 index 0000000000..c49edff288 --- /dev/null +++ b/src/broxygen/ScriptInfo.cc @@ -0,0 +1,361 @@ +#include "ScriptInfo.h" +#include "IdentifierInfo.h" +#include "ReStructuredTextTable.h" +#include "utils.h" +#include "Manager.h" + +#include "Reporter.h" +#include "Desc.h" + +using namespace std; +using namespace broxygen; + +bool IdInfoComp::operator ()(const IdentifierInfo* lhs, + const IdentifierInfo* rhs) const + { + return lhs->Name() < rhs->Name(); + } + +static vector summary_comment(const vector& cmnts) + { + vector rval; + + for ( size_t i = 0; i < cmnts.size(); ++i ) + { + size_t end = broxygen::end_of_first_sentence(cmnts[i]); + + if ( end == string::npos ) + { + if ( broxygen::is_all_whitespace(cmnts[i]) ) + break; + + rval.push_back(cmnts[i]); + } + else + { + rval.push_back(cmnts[i].substr(0, end + 1)); + break; + } + } + + return rval; + } + +static void add_summary_rows(const ODesc& id_desc, const vector& cmnts, + ReStructuredTextTable* table) + { + vector row; + row.push_back(id_desc.Description()); + + if ( cmnts.empty() ) + { + row.push_back(""); + table->AddRow(row); + return; + } + + row.push_back(cmnts[0]); + table->AddRow(row); + + for ( size_t i = 1; i < cmnts.size(); ++i ) + { + row.clear(); + row.push_back(""); + row.push_back(cmnts[i]); + table->AddRow(row); + } + } + +static string make_summary(const string& heading, char underline, char border, + const id_info_list& id_list) + { + if ( id_list.empty() ) + return ""; + + ReStructuredTextTable table(2); + + for ( id_info_list::const_iterator it = id_list.begin(); + it != id_list.end(); ++it ) + { + ID* id = (*it)->GetID(); + ODesc d; + d.SetQuotes(1); + id->DescribeReSTShort(&d); + add_summary_rows(d, summary_comment((*it)->GetComments()), &table); + } + + return broxygen::make_heading(heading, underline) + table.AsString(border) + + "\n"; + } + +static string make_redef_summary(const string& heading, char underline, + char border, const string& from_script, + const id_info_set& id_set) + { + if ( id_set.empty() ) + return ""; + + ReStructuredTextTable table(2); + + for ( id_info_set::const_iterator it = id_set.begin(); it != id_set.end(); + ++it ) + { + ID* id = (*it)->GetID(); + ODesc d; + d.SetQuotes(1); + id->DescribeReSTShort(&d); + + typedef list redef_list; + redef_list redefs = (*it)->GetRedefs(from_script); + + for ( redef_list::const_iterator iit = redefs.begin(); + iit != redefs.end(); ++iit ) + add_summary_rows(d, summary_comment(iit->comments), &table); + } + + return broxygen::make_heading(heading, underline) + table.AsString(border) + + "\n"; + } + +static string make_details(const string& heading, char underline, + const id_info_list& id_list) + { + if ( id_list.empty() ) + return ""; + + string rval = broxygen::make_heading(heading, underline); + + for ( id_info_list::const_iterator it = id_list.begin(); + it != id_list.end(); ++it ) + { + rval += (*it)->ReStructuredText(); + rval += "\n\n"; + } + + return rval; + } + +static string make_redef_details(const string& heading, char underline, + const id_info_set& id_set) + { + if ( id_set.empty() ) + return ""; + + string rval = broxygen::make_heading(heading, underline); + + for ( id_info_set::const_iterator it = id_set.begin(); + it != id_set.end(); ++it ) + { + rval += (*it)->ReStructuredText(true); + rval += "\n\n"; + } + + return rval; + } + +ScriptInfo::ScriptInfo(const string& arg_name, const string& arg_path) + : Info(), + name(arg_name), path(arg_path), + is_pkg_loader(SafeBasename(name).result == PACKAGE_LOADER), + dependencies(), module_usages(), comments(), id_info(), + options(), constants(), state_vars(), types(), events(), hooks(), + functions(), redefs() + { + } + +void ScriptInfo::AddIdentifierInfo(IdentifierInfo* info) + { + id_info[info->Name()] = info; + } + +void ScriptInfo::DoInitPostScript() + { + for ( id_info_map::const_iterator it = id_info.begin(); + it != id_info.end(); ++it ) + { + IdentifierInfo* info = it->second; + ID* id = info->GetID(); + + if ( ! broxygen::is_public_api(id) ) + continue; + + if ( id->AsType() ) + { + types.push_back(info); + DBG_LOG(DBG_BROXYGEN, "Filter id '%s' in '%s' as a type", + id->Name(), name.c_str()); + continue; + } + + if ( IsFunc(id->Type()->Tag()) ) + { + switch ( id->Type()->AsFuncType()->Flavor() ) { + case FUNC_FLAVOR_HOOK: + DBG_LOG(DBG_BROXYGEN, "Filter id '%s' in '%s' as a hook", + id->Name(), name.c_str()); + hooks.push_back(info); + break; + case FUNC_FLAVOR_EVENT: + DBG_LOG(DBG_BROXYGEN, "Filter id '%s' in '%s' as a event", + id->Name(), name.c_str()); + events.push_back(info); + break; + case FUNC_FLAVOR_FUNCTION: + DBG_LOG(DBG_BROXYGEN, "Filter id '%s' in '%s' as a function", + id->Name(), name.c_str()); + functions.push_back(info); + break; + default: + reporter->InternalError("Invalid function flavor"); + break; + } + + continue; + } + + if ( id->IsConst() ) + { + if ( id->FindAttr(ATTR_REDEF) ) + { + DBG_LOG(DBG_BROXYGEN, "Filter id '%s' in '%s' as an option", + id->Name(), name.c_str()); + options.push_back(info); + } + else + { + DBG_LOG(DBG_BROXYGEN, "Filter id '%s' in '%s' as a constant", + id->Name(), name.c_str()); + constants.push_back(info); + } + + continue; + } + + if ( id->Type()->Tag() == TYPE_ENUM ) + // Enums are always referenced/documented from the type's + // documentation. + continue; + + DBG_LOG(DBG_BROXYGEN, "Filter id '%s' in '%s' as a state variable", + id->Name(), name.c_str()); + state_vars.push_back(info); + } + } + +vector ScriptInfo::GetComments() const + { + return comments; + } + +string ScriptInfo::DoReStructuredText(bool roles_only) const + { + string rval; + + rval += ":tocdepth: 3\n\n"; + rval += broxygen::make_heading(name, '='); + + for ( string_set::const_iterator it = module_usages.begin(); + it != module_usages.end(); ++it ) + rval += ".. bro:namespace:: " + *it + "\n"; + + rval += "\n"; + + for ( size_t i = 0; i < comments.size(); ++i ) + rval += comments[i] + "\n"; + + rval += "\n"; + + if ( ! module_usages.empty() ) + { + rval += module_usages.size() > 1 ? ":Namespaces: " : ":Namespace: "; + + for ( string_set::const_iterator it = module_usages.begin(); + it != module_usages.end(); ++it ) + { + if ( it != module_usages.begin() ) + rval += ", "; + + rval += *it; + } + + rval += "\n"; + } + + if ( ! dependencies.empty() ) + { + rval += ":Imports: "; + + for ( string_set::const_iterator it = dependencies.begin(); + it != dependencies.end(); ++it ) + { + if ( it != dependencies.begin() ) + rval += ", "; + + string path = find_file(*it, bro_path(), "bro"); + string doc = *it; + + if ( ! path.empty() && is_dir(path.c_str()) ) + // Reference the package. + doc += "/index"; + + rval += fmt(":doc:`%s `", it->c_str(), doc.c_str()); + } + + rval += "\n"; + } + + rval += fmt(":Source File: :download:`/scripts/%s`\n", name.c_str()); + rval += "\n"; + rval += broxygen::make_heading("Summary", '~'); + rval += make_summary("Options", '#', '=', options); + rval += make_summary("Constants", '#', '=', constants); + rval += make_summary("State Variables", '#', '=', state_vars); + rval += make_summary("Types", '#', '=', types); + rval += make_redef_summary("Redefinitions", '#', '=', name, redefs); + rval += make_summary("Events", '#', '=', events); + rval += make_summary("Hooks", '#', '=', hooks); + rval += make_summary("Functions", '#', '=', functions); + rval += "\n"; + rval += broxygen::make_heading("Detailed Interface", '~'); + rval += make_details("Options", '#', options); + rval += make_details("Constants", '#', constants); + rval += make_details("State Variables", '#', state_vars); + rval += make_details("Types", '#', types); + //rval += make_redef_details("Redefinitions", '#', redefs); + rval += make_details("Events", '#', events); + rval += make_details("Hooks", '#', hooks); + rval += make_details("Functions", '#', functions); + + return rval; + } + +time_t ScriptInfo::DoGetModificationTime() const + { + time_t most_recent = broxygen::get_mtime(path); + + for ( string_set::const_iterator it = dependencies.begin(); + it != dependencies.end(); ++it ) + { + Info* info = broxygen_mgr->GetScriptInfo(*it); + + if ( ! info ) + { + string pkg_name = *it + "/" + PACKAGE_LOADER; + info = broxygen_mgr->GetScriptInfo(pkg_name); + + if ( ! info ) + reporter->InternalWarning("Broxygen failed to get mtime of %s", + it->c_str()); + continue; + } + + time_t dep_mtime = info->GetModificationTime(); + + if ( dep_mtime > most_recent ) + most_recent = dep_mtime; + } + + return most_recent; + } + + diff --git a/src/broxygen/ScriptInfo.h b/src/broxygen/ScriptInfo.h new file mode 100644 index 0000000000..3b5cc553de --- /dev/null +++ b/src/broxygen/ScriptInfo.h @@ -0,0 +1,121 @@ +#ifndef BROXYGEN_SCRIPTINFO_H +#define BROXYGEN_SCRIPTINFO_H + +#include "Info.h" +#include "IdentifierInfo.h" + +#include +#include +#include +#include +#include + +namespace broxygen { + +class IdentifierInfo; + +struct IdInfoComp { + bool operator() (const IdentifierInfo* lhs, + const IdentifierInfo* rhs) const; +}; + +typedef std::set id_info_set; +typedef std::list id_info_list; + +/** + * Information about a Bro script. + */ +class ScriptInfo : public Info { + +public: + + /** + * Ctor. + * @param name Name of script: a path relative to a component in BROPATH. + * @param path Absolute path to the script. + */ + ScriptInfo(const std::string& name, const std::string& path); + + /** + * Associate a Broxygen summary comment ("##!") with the script. + * @param comment String extracted from the comment. + */ + void AddComment(const std::string& comment) + { comments.push_back(comment); } + + /** + * Register a dependency on another script. + * @param name Name of a script with this one @loads. This is the + * "normalized" name (a path relative to a component in BROPATH). + */ + void AddDependency(const std::string& name) + { dependencies.insert(name); } + + /** + * Register a module usage (signifying the script may export identifiers + * into that modules namespace). + * @param name The name of the module. + */ + void AddModule(const std::string& name) + { module_usages.insert(name); } + + /** + * Register an identifier declared by this script. + * @param info The identifier info object associated with a script-level + * identifier declared by the script. + */ + void AddIdentifierInfo(IdentifierInfo* info); + + /** + * Register a redef of an identifier done by this script. + * @param info The identifier info object associated with the script-level + * identifier redef'd by the script. + */ + void AddRedef(IdentifierInfo* info) + { redefs.insert(info); } + + /** + * @return Whether the script is a package loader (i.e. "__load__.bro"). + */ + bool IsPkgLoader() const + { return is_pkg_loader; } + + /** + * @return All the scripts Broxygen summary comments. + */ + std::vector GetComments() const; + +private: + + typedef std::map id_info_map; + typedef std::set string_set; + + time_t DoGetModificationTime() const; + + std::string DoName() const + { return name; } + + std::string DoReStructuredText(bool roles_only) const; + + void DoInitPostScript() /* override */; + + std::string name; + std::string path; + bool is_pkg_loader; + string_set dependencies; + string_set module_usages; + std::vector comments; + id_info_map id_info; + id_info_list options; + id_info_list constants; + id_info_list state_vars; + id_info_list types; + id_info_list events; + id_info_list hooks; + id_info_list functions; + id_info_set redefs; +}; + +} // namespace broxygen + +#endif diff --git a/src/broxygen/Target.cc b/src/broxygen/Target.cc new file mode 100644 index 0000000000..0bbc372724 --- /dev/null +++ b/src/broxygen/Target.cc @@ -0,0 +1,595 @@ +#include "Target.h" +#include "Manager.h" + +#include "util.h" +#include "Reporter.h" +#include "plugin/Manager.h" +#include "analyzer/Manager.h" +#include "analyzer/Component.h" +#include "file_analysis/Manager.h" + +#include +#include +#include +#include + +using namespace std; +using namespace broxygen; + +static void write_plugin_section_heading(FILE* f, const plugin::Plugin* p) + { + string name = p->Name(); + + fprintf(f, "%s\n", name.c_str()); + for ( size_t i = 0; i < name.size(); ++i ) + fprintf(f, "-"); + fprintf(f, "\n\n"); + + fprintf(f, "%s\n\n", p->Description()); + } + +static void write_analyzer_component(FILE* f, const analyzer::Component* c) + { + EnumType* atag = analyzer_mgr->GetTagEnumType(); + string tag = fmt("ANALYZER_%s", c->CanonicalName()); + + if ( atag->Lookup("Analyzer", tag.c_str()) < 0 ) + reporter->InternalError("missing analyzer tag for %s", tag.c_str()); + + fprintf(f, ":bro:enum:`Analyzer::%s`\n\n", tag.c_str()); + } + +static void write_analyzer_component(FILE* f, const file_analysis::Component* c) + { + EnumType* atag = file_mgr->GetTagEnumType(); + string tag = fmt("ANALYZER_%s", c->CanonicalName()); + + if ( atag->Lookup("Files", tag.c_str()) < 0 ) + reporter->InternalError("missing analyzer tag for %s", tag.c_str()); + + fprintf(f, ":bro:enum:`Files::%s`\n\n", tag.c_str()); + } + +static void write_plugin_components(FILE* f, const plugin::Plugin* p) + { + plugin::Plugin::component_list components = p->Components(); + plugin::Plugin::component_list::const_iterator it; + + fprintf(f, "Components\n"); + fprintf(f, "++++++++++\n\n"); + + for ( it = components.begin(); it != components.end(); ++it ) + { + switch ( (*it)->Type() ) { + case plugin::component::ANALYZER: + { + const analyzer::Component* c = + dynamic_cast(*it); + + if ( c ) + write_analyzer_component(f, c); + else + reporter->InternalError("component type mismatch"); + } + break; + + case plugin::component::FILE_ANALYZER: + { + const file_analysis::Component* c = + dynamic_cast(*it); + + if ( c ) + write_analyzer_component(f, c); + else + reporter->InternalError("component type mismatch"); + } + break; + + case plugin::component::READER: + reporter->InternalError("docs for READER component unimplemented"); + + case plugin::component::WRITER: + reporter->InternalError("docs for WRITER component unimplemented"); + + default: + reporter->InternalError("docs for unknown component unimplemented"); + } + } + } + +static void write_plugin_bif_items(FILE* f, const plugin::Plugin* p, + plugin::BifItem::Type t, const string& heading) + { + plugin::Plugin::bif_item_list bifitems = p->BifItems(); + plugin::Plugin::bif_item_list::iterator it = bifitems.begin(); + + while ( it != bifitems.end() ) + { + if ( it->GetType() != t ) + it = bifitems.erase(it); + else + ++it; + } + + if ( bifitems.empty() ) + return; + + fprintf(f, "%s\n", heading.c_str()); + for ( size_t i = 0; i < heading.size(); ++i ) + fprintf(f, "+"); + fprintf(f, "\n\n"); + + for ( it = bifitems.begin(); it != bifitems.end(); ++it ) + { + broxygen::IdentifierInfo* doc = broxygen_mgr->GetIdentifierInfo( + it->GetID()); + + if ( doc ) + fprintf(f, "%s\n\n", doc->ReStructuredText().c_str()); + else + reporter->InternalWarning("Broxygen ID lookup failed: %s\n", + it->GetID()); + } + } + +static void WriteAnalyzerTagDefn(FILE* f, const string& module) + { + string tag_id = module + "::Tag"; + + broxygen::IdentifierInfo* doc = broxygen_mgr->GetIdentifierInfo(tag_id); + + if ( ! doc ) + reporter->InternalError("Broxygen failed analyzer tag lookup: %s", + tag_id.c_str()); + + fprintf(f, "%s\n", doc->ReStructuredText().c_str()); + } + +static bool ComponentsMatch(const plugin::Plugin* p, plugin::component::Type t, + bool match_empty = false) + { + plugin::Plugin::component_list components = p->Components(); + plugin::Plugin::component_list::const_iterator it; + + if ( components.empty() ) + return match_empty; + + for ( it = components.begin(); it != components.end(); ++it ) + if ( (*it)->Type() != t ) + return false; + + return true; + } + +template +static vector filter_matches(const vector& from, Target* t) + { + vector rval; + + for ( size_t i = 0; i < from.size(); ++i ) + { + T* d = dynamic_cast(from[i]); + + if ( ! d ) + continue; + + if ( t->MatchesPattern(d) ) + { + DBG_LOG(DBG_BROXYGEN, "'%s' matched pattern for target '%s'", + d->Name().c_str(), t->Name().c_str()); + rval.push_back(d); + } + } + + return rval; + } + +TargetFile::TargetFile(const string& arg_name) + : name(arg_name), f() + { + if ( name.find('/') != string::npos ) + { + string dir = SafeDirname(name).result; + + if ( ! ensure_intermediate_dirs(dir.c_str()) ) + reporter->FatalError("Broxygen failed to make dir %s", + dir.c_str()); + } + + f = fopen(name.c_str(), "w"); + + if ( ! f ) + reporter->FatalError("Broxygen failed to open '%s' for writing: %s", + name.c_str(), strerror(errno)); + } + +TargetFile::~TargetFile() + { + if ( f ) + fclose(f); + + DBG_LOG(DBG_BROXYGEN, "Wrote out-of-date target '%s'", name.c_str()); + } + + +Target::Target(const string& arg_name, const string& arg_pattern) + : name(arg_name), pattern(arg_pattern), prefix() + { + size_t pos = pattern.find('*'); + + if ( pos == 0 || pos == string::npos ) + return; + + prefix = pattern.substr(0, pos); + } + +bool Target::MatchesPattern(Info* info) const + { + if ( pattern == "*" ) + return true; + + if ( prefix.empty() ) + return info->Name() == pattern; + + return ! strncmp(info->Name().c_str(), prefix.c_str(), prefix.size()); + } + +void AnalyzerTarget::DoFindDependencies(const std::vector& infos) + { + // TODO: really should add to dependency list the tag type's ID and + // all bif items for matching analyzer plugins, but that's all dependent + // on the bro binary itself, so I'm cheating. + } + +void AnalyzerTarget::DoGenerate() const + { + if ( broxygen_mgr->IsUpToDate(Name(), vector()) ) + return; + + if ( Pattern() != "*" ) + reporter->InternalWarning("Broxygen only implements analyzer target" + " pattern '*'"); + + TargetFile file(Name()); + CreateAnalyzerDoc(file.f); + } + +void ProtoAnalyzerTarget::DoCreateAnalyzerDoc(FILE* f) const + { + fprintf(f, "Protocol Analyzers\n"); + fprintf(f, "==================\n\n"); + fprintf(f, ".. contents::\n"); + fprintf(f, " :depth: 2\n\n"); + + WriteAnalyzerTagDefn(f, "Analyzer"); + + plugin::Manager::plugin_list plugins = plugin_mgr->Plugins(); + plugin::Manager::plugin_list::const_iterator it; + + for ( it = plugins.begin(); it != plugins.end(); ++it ) + { + if ( ! ComponentsMatch(*it, plugin::component::ANALYZER, true) ) + continue; + + write_plugin_section_heading(f, *it); + write_plugin_components(f, *it); + write_plugin_bif_items(f, *it, plugin::BifItem::CONSTANT, + "Options/Constants"); + write_plugin_bif_items(f, *it, plugin::BifItem::GLOBAL, "Globals"); + write_plugin_bif_items(f, *it, plugin::BifItem::TYPE, "Types"); + write_plugin_bif_items(f, *it, plugin::BifItem::EVENT, "Events"); + write_plugin_bif_items(f, *it, plugin::BifItem::FUNCTION, "Functions"); + } + } + +void FileAnalyzerTarget::DoCreateAnalyzerDoc(FILE* f) const + { + fprintf(f, "File Analyzers\n"); + fprintf(f, "==============\n\n"); + fprintf(f, ".. contents::\n"); + fprintf(f, " :depth: 2\n\n"); + + WriteAnalyzerTagDefn(f, "Files"); + + plugin::Manager::plugin_list plugins = plugin_mgr->Plugins(); + plugin::Manager::plugin_list::const_iterator it; + + for ( it = plugins.begin(); it != plugins.end(); ++it ) + { + if ( ! ComponentsMatch(*it, plugin::component::FILE_ANALYZER) ) + continue; + + write_plugin_section_heading(f, *it); + write_plugin_components(f, *it); + write_plugin_bif_items(f, *it, plugin::BifItem::CONSTANT, + "Options/Constants"); + write_plugin_bif_items(f, *it, plugin::BifItem::GLOBAL, "Globals"); + write_plugin_bif_items(f, *it, plugin::BifItem::TYPE, "Types"); + write_plugin_bif_items(f, *it, plugin::BifItem::EVENT, "Events"); + write_plugin_bif_items(f, *it, plugin::BifItem::FUNCTION, "Functions"); + } + } + +void PackageTarget::DoFindDependencies(const vector& infos) + { + pkg_deps = filter_matches(infos, this); + + if ( pkg_deps.empty() ) + reporter->FatalError("No match for Broxygen target '%s' pattern '%s'", + Name().c_str(), Pattern().c_str()); + + for ( size_t i = 0; i < infos.size(); ++i ) + { + ScriptInfo* script = dynamic_cast(infos[i]); + + if ( ! script ) + continue; + + for ( size_t j = 0; j < pkg_deps.size(); ++j ) + { + if ( strncmp(script->Name().c_str(), pkg_deps[j]->Name().c_str(), + pkg_deps[j]->Name().size())) + continue; + + DBG_LOG(DBG_BROXYGEN, "Script %s associated with package %s", + script->Name().c_str(), pkg_deps[j]->Name().c_str()); + pkg_manifest[pkg_deps[j]].push_back(script); + script_deps.push_back(script); + } + } + } + +void PackageTarget::DoGenerate() const + { + if ( broxygen_mgr->IsUpToDate(Name(), script_deps) && + broxygen_mgr->IsUpToDate(Name(), pkg_deps) ) + return; + + TargetFile file(Name()); + + fprintf(file.f, ":orphan:\n\n"); + + for ( manifest_t::const_iterator it = pkg_manifest.begin(); + it != pkg_manifest.end(); ++it ) + { + string header = fmt("Package: %s", it->first->Name().c_str()); + header += "\n" + string(header.size(), '='); + + fprintf(file.f, "%s\n\n", header.c_str()); + + vector readme = it->first->GetReadme(); + + for ( size_t i = 0; i < readme.size(); ++i ) + fprintf(file.f, "%s\n", readme[i].c_str()); + + fprintf(file.f, "\n"); + + for ( size_t i = 0; i < it->second.size(); ++i ) + { + fprintf(file.f, ":doc:`/scripts/%s`\n", + it->second[i]->Name().c_str()); + + vector cmnts = it->second[i]->GetComments(); + + for ( size_t j = 0; j < cmnts.size(); ++j ) + fprintf(file.f, " %s\n", cmnts[j].c_str()); + + fprintf(file.f, "\n"); + } + } + } + +void PackageIndexTarget::DoFindDependencies(const vector& infos) + { + pkg_deps = filter_matches(infos, this); + + if ( pkg_deps.empty() ) + reporter->FatalError("No match for Broxygen target '%s' pattern '%s'", + Name().c_str(), Pattern().c_str()); + } + +void PackageIndexTarget::DoGenerate() const + { + if ( broxygen_mgr->IsUpToDate(Name(), pkg_deps) ) + return; + + TargetFile file(Name()); + + for ( size_t i = 0; i < pkg_deps.size(); ++i ) + fprintf(file.f, "%s\n", pkg_deps[i]->ReStructuredText().c_str()); + } + +void ScriptTarget::DoFindDependencies(const vector& infos) + { + script_deps = filter_matches(infos, this); + + if ( script_deps.empty() ) + reporter->FatalError("No match for Broxygen target '%s' pattern '%s'", + Name().c_str(), Pattern().c_str()); + + if ( ! IsDir() ) + return; + + for ( size_t i = 0; i < script_deps.size(); ++i ) + { + if ( SafeBasename(script_deps[i]->Name()).result == PACKAGE_LOADER ) + { + string pkg_dir = SafeDirname(script_deps[i]->Name()).result; + string target_file = Name() + pkg_dir + "/index.rst"; + Target* t = new PackageTarget(target_file, pkg_dir); + t->FindDependencies(infos); + pkg_deps.push_back(t); + } + } + } + +vector dir_contents_recursive(string dir) + { + vector rval; + struct stat st; + + if ( stat(dir.c_str(), &st) < 0 && errno == ENOENT ) + return rval; + + while ( dir[dir.size() - 1] == '/' ) + dir.erase(dir.size() - 1, 1); + + char* dir_copy = copy_string(dir.c_str()); + char** scan_path = new char*[2]; + scan_path[0] = dir_copy; + scan_path[1] = 0; + + FTS* fts = fts_open(scan_path, FTS_NOCHDIR, 0); + + if ( ! fts ) + { + reporter->Error("fts_open failure: %s", strerror(errno)); + delete [] scan_path; + delete [] dir_copy; + return rval; + } + + FTSENT* n; + + while ( (n = fts_read(fts)) ) + { + if ( n->fts_info & FTS_F ) + rval.push_back(n->fts_path); + } + + if ( errno ) + reporter->Error("fts_read failure: %s", strerror(errno)); + + if ( fts_close(fts) < 0 ) + reporter->Error("fts_close failure: %s", strerror(errno)); + + delete [] scan_path; + delete [] dir_copy; + return rval; + } + +void ScriptTarget::DoGenerate() const + { + if ( IsDir() ) + { + // Target name is a dir, matching scripts are written within that dir + // with a dir tree that parallels the script's BROPATH location. + + set targets; + vector dir_contents = dir_contents_recursive(Name()); + + for ( size_t i = 0; i < script_deps.size(); ++i ) + { + string target_filename = Name() + script_deps[i]->Name() + ".rst"; + targets.insert(target_filename); + vector dep; + dep.push_back(script_deps[i]); + + if ( broxygen_mgr->IsUpToDate(target_filename, dep) ) + continue; + + TargetFile file(target_filename); + + fprintf(file.f, "%s\n", script_deps[i]->ReStructuredText().c_str()); + } + + for ( size_t i = 0; i < pkg_deps.size(); ++i ) + { + targets.insert(pkg_deps[i]->Name()); + pkg_deps[i]->Generate(); + } + + for ( size_t i = 0; i < dir_contents.size(); ++i ) + { + string f = dir_contents[i]; + + if ( targets.find(f) != targets.end() ) + continue; + + if ( unlink(f.c_str()) < 0 ) + reporter->Warning("Failed to unlink %s: %s", f.c_str(), + strerror(errno)); + + DBG_LOG(DBG_BROXYGEN, "Delete stale script file %s", f.c_str()); + } + + return; + } + + // Target is a single file, all matching scripts get written there. + + if ( broxygen_mgr->IsUpToDate(Name(), script_deps) ) + return; + + TargetFile file(Name()); + + for ( size_t i = 0; i < script_deps.size(); ++i ) + fprintf(file.f, "%s\n", script_deps[i]->ReStructuredText().c_str()); + } + +void ScriptSummaryTarget::DoGenerate() const + { + if ( broxygen_mgr->IsUpToDate(Name(), script_deps) ) + return; + + TargetFile file(Name()); + + for ( size_t i = 0; i < script_deps.size(); ++i ) + { + ScriptInfo* d = dynamic_cast(script_deps[i]); + + if ( ! d ) + continue; + + fprintf(file.f, ":doc:`/scripts/%s`\n", d->Name().c_str()); + + vector cmnts = d->GetComments(); + + for ( size_t i = 0; i < cmnts.size(); ++i ) + fprintf(file.f, " %s\n", cmnts[i].c_str()); + + fprintf(file.f, "\n"); + } + } + +void ScriptIndexTarget::DoGenerate() const + { + if ( broxygen_mgr->IsUpToDate(Name(), script_deps) ) + return; + + TargetFile file(Name()); + + fprintf(file.f, ".. toctree::\n"); + fprintf(file.f, " :maxdepth: 1\n\n"); + + for ( size_t i = 0; i < script_deps.size(); ++i ) + { + ScriptInfo* d = dynamic_cast(script_deps[i]); + + if ( ! d ) + continue; + + fprintf(file.f, " %s \n", d->Name().c_str(), + d->Name().c_str()); + } + } + +void IdentifierTarget::DoFindDependencies(const vector& infos) + { + id_deps = filter_matches(infos, this); + + if ( id_deps.empty() ) + reporter->FatalError("No match for Broxygen target '%s' pattern '%s'", + Name().c_str(), Pattern().c_str()); + } + +void IdentifierTarget::DoGenerate() const + { + if ( broxygen_mgr->IsUpToDate(Name(), id_deps) ) + return; + + TargetFile file(Name()); + + for ( size_t i = 0; i < id_deps.size(); ++i ) + fprintf(file.f, "%s\n\n", id_deps[i]->ReStructuredText().c_str()); + } diff --git a/src/broxygen/Target.h b/src/broxygen/Target.h new file mode 100644 index 0000000000..8fe9d0337e --- /dev/null +++ b/src/broxygen/Target.h @@ -0,0 +1,387 @@ +#ifndef BROXYGEN_TARGET_H +#define BROXYGEN_TARGET_H + +#include "Info.h" +#include "PackageInfo.h" +#include "ScriptInfo.h" +#include "IdentifierInfo.h" + +#include +#include +#include +#include + +namespace broxygen { + +/** + * Helper class to create files in arbitrary file paths and automatically + * close it on destruction. + */ +struct TargetFile { + /** + * Open a file. + * @param arg_name Path to a file to create. It's a fatal error if + * creating it fails. Creating it will also create any intermediate + * directories that don't already exist. + * + */ + TargetFile(const std::string& arg_name); + + /** + * Close the file. + */ + ~TargetFile(); + + std::string name; /**< File name. */ + FILE* f; /**< File stream. */ +}; + +/** + * A Broxygen target abstract base class. A target is generally any portion of + * documentation that Bro can build. It's identified by a type (e.g. script, + * identifier, package), a pattern (e.g. "example.bro", "HTTP::Info"), and + * a path to an output file. + */ +class Target { + +public: + + /** + * Ctor. + * @param arg_name output file name of the target. + * @param arg_pattern pattern of info objects the target depends upon. Only + * exact string and simple prefix matching is currently allowed. + */ + Target(const std::string& arg_name, const std::string& arg_pattern); + + /** + * Dtor. + */ + virtual ~Target() + { } + + /** + * Filter out any dependency information from a set of all known info. + * @param infos All known info objects. + */ + void FindDependencies(const std::vector& infos) + { DoFindDependencies(infos); } + + /** + * Build the target by generating its output file. Implementations may + * not always write to the output file if they determine an existing + * version is already up-to-date. + */ + void Generate() const + { DoGenerate(); } + + /** + * Check if a particular info object matches the target pattern. + * Currently only exact string and simple prefix matching patterns are + * used. E.g. for prefix matching "HTTP::*" or "base/protocols/http*". + * @param info An info object for some thing that is documentable. + * @return true if it matches, else false. + */ + bool MatchesPattern(Info* info) const; + + /** + * @return The output file name of the target. + */ + std::string Name() const + { return name; } + + /** + * @return The pattern string of the target. + */ + std::string Pattern() const + { return pattern; } + +private: + + virtual void DoFindDependencies(const std::vector& infos) = 0; + + virtual void DoGenerate() const = 0; + + std::string name; + std::string pattern; + std::string prefix; +}; + +template +static Target* create_target(const std::string& name, + const std::string& pattern) + { + return new T(name, pattern); + } + +/** + * Factory for creating Target instances. + */ +class TargetFactory { + +public: + + /** + * Register a new target type. + * @param type_name The target type name as it will appear in Broxygen + * config files. + */ + template + void Register(const std::string& type_name) + { + target_creators[type_name] = &create_target; + } + + /** + * Instantiate a target. + * @param type_name The target type name as it appears in Broxygen config + * files. + * @param name The output file name of the target. + * @param pattern The dependency pattern of the target. + * @return A new target instance or a pointer if \a type_name is not + * registered. + */ + Target* Create(const std::string& type_name, + const std::string& name, const std::string& pattern) + { + target_creator_map::const_iterator it = target_creators.find(type_name); + + if ( it == target_creators.end() ) + return 0; + + return it->second(name, pattern); + } + +private: + + typedef Target* (*TargetFactoryFn)(const std::string& name, + const std::string& pattern); + typedef std::map target_creator_map; + target_creator_map target_creators; +}; + +/** + * Target to build analyzer documentation. + */ +class AnalyzerTarget : public Target { +public: + + /** + * Writes out plugin index documentation for all analyzer plugins. + * @param f an open file stream to write docs into. + */ + void CreateAnalyzerDoc(FILE* f) const + { return DoCreateAnalyzerDoc(f); } + +protected: + + typedef void (*doc_creator_fn)(FILE*); + + AnalyzerTarget(const std::string& name, const std::string& pattern) + : Target(name, pattern) + { } + +private: + + void DoFindDependencies(const std::vector& infos); + + void DoGenerate() const; + + virtual void DoCreateAnalyzerDoc(FILE* f) const = 0; +}; + +/** + * Target to build protocol analyzer documentation. + */ +class ProtoAnalyzerTarget : public AnalyzerTarget { +public: + + /** + * Ctor. + * @param name Output file name. + * @param pattern Dependency pattern. + */ + ProtoAnalyzerTarget(const std::string& name, const std::string& pattern) + : AnalyzerTarget(name, pattern) + { } + +private: + + void DoCreateAnalyzerDoc(FILE* f) const; +}; + +/** + * Target to build file analyzer documentation. + */ +class FileAnalyzerTarget : public AnalyzerTarget { +public: + + /** + * Ctor. + * @param name Output file name. + * @param pattern Dependency pattern. + */ + FileAnalyzerTarget(const std::string& name, const std::string& pattern) + : AnalyzerTarget(name, pattern) + { } + +private: + + void DoCreateAnalyzerDoc(FILE* f) const; +}; + +/** + * Target to build package documentation. + */ +class PackageTarget : public Target { +public: + + /** + * Ctor. + * @param name Output file name. + * @param pattern Dependency pattern. + */ + PackageTarget(const std::string& name, const std::string& pattern) + : Target(name, pattern), pkg_deps(), script_deps(), pkg_manifest() + { } + +private: + + void DoFindDependencies(const std::vector& infos); + + void DoGenerate() const; + + std::vector pkg_deps; + std::vector script_deps; + typedef std::map > manifest_t; + manifest_t pkg_manifest; +}; + +/** + * Target to build package index documentation. + */ +class PackageIndexTarget : public Target { +public: + + /** + * Ctor. + * @param name Output file name. + * @param pattern Dependency pattern. + */ + PackageIndexTarget(const std::string& name, const std::string& pattern) + : Target(name, pattern), pkg_deps() + { } + +private: + + void DoFindDependencies(const std::vector& infos); + + void DoGenerate() const; + + std::vector pkg_deps; +}; + +/** + * Target to build script documentation. + */ +class ScriptTarget : public Target { +public: + + /** + * Ctor. + * @param name Output file name or directory. If it's a directory, + * then one document for each script that matches the pattern is written to + * the directory in a directory structure which mirrors the script's path + * relative to a component in BROPATH. + * @param pattern Dependency pattern. + */ + ScriptTarget(const std::string& name, const std::string& pattern) + : Target(name, pattern), script_deps() + { } + + ~ScriptTarget() + { for ( size_t i = 0; i < pkg_deps.size(); ++i ) delete pkg_deps[i]; } + +protected: + + std::vector script_deps; + +private: + + void DoFindDependencies(const std::vector& infos); + + void DoGenerate() const; + + bool IsDir() const + { return Name()[Name().size() - 1] == '/'; } + + std::vector pkg_deps; +}; + +/** + * Target to build script summary documentation. + */ +class ScriptSummaryTarget : public ScriptTarget { +public: + + /** + * Ctor. + * @param name Output file name. + * @param pattern Dependency pattern. + */ + ScriptSummaryTarget(const std::string& name, const std::string& pattern) + : ScriptTarget(name, pattern) + { } + +private: + + void DoGenerate() const /* override */; +}; + +/** + * Target to build script index documentation. + */ +class ScriptIndexTarget : public ScriptTarget { +public: + + /** + * Ctor. + * @param name Output file name. + * @param pattern Dependency pattern. + */ + ScriptIndexTarget(const std::string& name, const std::string& pattern) + : ScriptTarget(name, pattern) + { } + +private: + + void DoGenerate() const /* override */; +}; + +/** + * Target to build identifier documentation. + */ +class IdentifierTarget : public Target { +public: + + /** + * Ctor. + * @param name Output file name. + * @param pattern Dependency pattern. + */ + IdentifierTarget(const std::string& name, const std::string& pattern) + : Target(name, pattern), id_deps() + { } + +private: + + void DoFindDependencies(const std::vector& infos); + + void DoGenerate() const; + + std::vector id_deps; +}; + +} // namespace broxygen + +#endif diff --git a/src/broxygen/broxygen.bif b/src/broxygen/broxygen.bif index 0d346411d0..1229e0317c 100644 --- a/src/broxygen/broxygen.bif +++ b/src/broxygen/broxygen.bif @@ -10,12 +10,17 @@ static StringVal* comments_to_val(const vector& comments) } %%} -# TODO: documentation - +## Retrieve the Broxygen-style comments (``##``) associated with an identifier +## (e.g. a variable or type). +## +## name: a script-level identifier for which to retrieve comments. +## +## Returns: comments associated with *name*. If *name* is not a known +## identifier, an empty string is returned. function get_identifier_comments%(name: string%): string %{ using namespace broxygen; - IdentifierDocument* d = broxygen_mgr->GetIdentifierDoc(name->CheckString()); + IdentifierInfo* d = broxygen_mgr->GetIdentifierInfo(name->CheckString()); if ( ! d ) return new StringVal(""); @@ -23,10 +28,19 @@ function get_identifier_comments%(name: string%): string return comments_to_val(d->GetComments()); %} +## Retrieve the Broxygen-style summary comments (``##!``) associated with +## a Bro script. +## +## name: the name of a Bro script. It must be a relative path to where +## it is located within a particular component of BROPATH and use +## the same file name extension/suffix as the actual file (e.g. ".bro"). +## +## Returns: summary comments associated with script with *name*. If +## *name* is not a known script, an empty string is returned. function get_script_comments%(name: string%): string %{ using namespace broxygen; - ScriptDocument* d = broxygen_mgr->GetScriptDoc(name->CheckString()); + ScriptInfo* d = broxygen_mgr->GetScriptInfo(name->CheckString()); if ( ! d ) return new StringVal(""); @@ -34,10 +48,17 @@ function get_script_comments%(name: string%): string return comments_to_val(d->GetComments()); %} +## Retrieve the contents of a Bro script package's README file. +## +## name: the name of a Bro script package. It must be a relative path +## to where it is located within a particular component of BROPATH. +## +## Returns: contents of the package's README file. If *name* is not a known +## package, an empty string is returned. function get_package_readme%(name: string%): string %{ using namespace broxygen; - PackageDocument* d = broxygen_mgr->GetPackageDoc(name->CheckString()); + PackageInfo* d = broxygen_mgr->GetPackageInfo(name->CheckString()); if ( ! d ) return new StringVal(""); @@ -45,6 +66,14 @@ function get_package_readme%(name: string%): string return comments_to_val(d->GetReadme()); %} +## Retrieve the Broxygen-style comments (``##``) associated with a record field. +## +## name: the name of a record type and a field within it formatted like +## a typical record field access: "$". +## +## Returns: comments associated with the record field. If *name* does +## not point to a known record type or a known field within a record +## type, an empty string is returned. function get_record_field_comments%(name: string%): string %{ using namespace broxygen; @@ -56,7 +85,7 @@ function get_record_field_comments%(name: string%): string string id = accessor.substr(0, i); - IdentifierDocument* d = broxygen_mgr->GetIdentifierDoc(id); + IdentifierInfo* d = broxygen_mgr->GetIdentifierInfo(id); if ( ! d ) return new StringVal(""); diff --git a/src/broxygen/utils.cc b/src/broxygen/utils.cc index ed27219144..5d0b3be91b 100644 --- a/src/broxygen/utils.cc +++ b/src/broxygen/utils.cc @@ -1,5 +1,9 @@ #include "utils.h" +#include "Reporter.h" + +#include + using namespace broxygen; using namespace std; @@ -68,3 +72,55 @@ bool broxygen::prettify_params(string& s) return false; } + +bool broxygen::is_public_api(const ID* id) + { + return (id->Scope() == SCOPE_GLOBAL) || + (id->Scope() == SCOPE_MODULE && id->IsExport()); + } + +time_t broxygen::get_mtime(const string& filename) + { + struct stat s; + + if ( stat(filename.c_str(), &s) < 0 ) + reporter->InternalError("Broxygen failed to stat file '%s': %s", + filename.c_str(), strerror(errno)); + + return s.st_mtime; + } + +string broxygen::make_heading(const string& heading, char underline) + { + return heading + "\n" + string(heading.size(), underline) + "\n"; + } + +size_t broxygen::end_of_first_sentence(const string& s) + { + size_t rval = 0; + + while ( (rval = s.find_first_of('.', rval)) != string::npos ) + { + if ( rval == s.size() - 1 ) + // Period is at end of string. + return rval; + + if ( isspace(s[rval + 1]) ) + // Period has a space after it. + return rval; + + // Period has some non-space character after it, keep looking. + ++rval; + } + + return rval; + } + +bool broxygen::is_all_whitespace(const string& s) + { + for ( size_t i = 0; i < s.size(); ++i ) + if ( ! isspace(s[i]) ) + return false; + + return true; + } diff --git a/src/broxygen/utils.h b/src/broxygen/utils.h index a8ba239802..a11113715a 100644 --- a/src/broxygen/utils.h +++ b/src/broxygen/utils.h @@ -1,12 +1,59 @@ #ifndef BROXYGEN_UTILS_H #define BROXYGEN_UTILS_H +#include "ID.h" + #include namespace broxygen { +/** + * Transform content of a Broxygen comment which may contain function + * parameter or return value documentation to a prettier reST format. + * @param s Content from a Broxygen comment to transform. "id: ..." and + * "Returns: ..." change to ":param id: ..." and ":returns: ...". + * @return Whether any content in \a s was transformed. + */ bool prettify_params(std::string& s); +/** + * Check whether an identifier is part of the "public" interface. + * @param id A script-level identifier. + * @return true if the ID is in the global scope or if it's exported in to + * any modules namespace. + */ +bool is_public_api(const ID* id); + +/** + * Get the modification time of a file or abort if there's an error. + * @param filename Path to a file. + * @return The modification time of \a filename via stat(2). + */ +time_t get_mtime(const std::string& filename); + +/** + * Make a reST-style heading. + * @param heading Content of the heading. + * @param underline Character in which to underline heading content. + * @return underlined heading string. + */ +std::string make_heading(const std::string& heading, char underline); + +/** + * Get the position of the end of the first sentence in a string. + * @param s Any string. + * @return The position which looks like the end of the first sentence in + * \a s or 0 if no such position is found. + */ +size_t end_of_first_sentence(const std::string& s); + +/** + * Check if a string is entirely white space. + * @param s Any string. + * @return True if \a s is nothing but white space, else false. + */ +bool is_all_whitespace(const std::string& s); + } // namespace broxygen #endif diff --git a/src/main.cc b/src/main.cc index cac1acd7b1..d48a931b24 100644 --- a/src/main.cc +++ b/src/main.cc @@ -50,7 +50,6 @@ extern "C" void OPENSSL_add_all_algorithms_conf(void); #include "PersistenceSerializer.h" #include "EventRegistry.h" #include "Stats.h" -#include "BroDoc.h" #include "Brofiler.h" #include "threading/Manager.h" diff --git a/src/scan.l b/src/scan.l index 5b80350111..8cb84b6de3 100644 --- a/src/scan.l +++ b/src/scan.l @@ -564,7 +564,7 @@ static int load_files(const char* orig_file) else file_stack.append(new FileInfo); - broxygen_mgr->File(file_path); + broxygen_mgr->Script(file_path); // "orig_file", could be an alias for yytext, which is ephemeral // and will be zapped after the yy_switch_to_buffer() below. From 25e28c04f0e63922f30779e87328c5218596b22a Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Mon, 25 Nov 2013 15:48:16 -0600 Subject: [PATCH 19/40] Add a Broxygen coverage test. --- scripts/broxygen/__load__.bro | 12 +++++++ .../Baseline/doc.broxygen.all_scripts/.stderr | 1 + .../Baseline/doc.broxygen.all_scripts/.stdout | 1 + testing/btest/coverage/broxygen.sh | 35 +++++++++++++++++++ testing/btest/doc/broxygen/all_scripts.test | 4 +++ 5 files changed, 53 insertions(+) create mode 100644 testing/btest/coverage/broxygen.sh diff --git a/scripts/broxygen/__load__.bro b/scripts/broxygen/__load__.bro index 68a0d878b6..70bf1cf2ef 100644 --- a/scripts/broxygen/__load__.bro +++ b/scripts/broxygen/__load__.bro @@ -1,2 +1,14 @@ @load test-all-policy.bro + +# Scripts which are commented out in test-all-policy.bro. +@load protocols/ssl/notary.bro +@load frameworks/communication/listen.bro +@load frameworks/control/controllee.bro +@load frameworks/control/controller.bro + @load ./example.bro + +event bro_init() + { + terminate(); + } diff --git a/testing/btest/Baseline/doc.broxygen.all_scripts/.stderr b/testing/btest/Baseline/doc.broxygen.all_scripts/.stderr index e69de29bb2..977e8fc37a 100644 --- a/testing/btest/Baseline/doc.broxygen.all_scripts/.stderr +++ b/testing/btest/Baseline/doc.broxygen.all_scripts/.stderr @@ -0,0 +1 @@ +received termination signal diff --git a/testing/btest/Baseline/doc.broxygen.all_scripts/.stdout b/testing/btest/Baseline/doc.broxygen.all_scripts/.stdout index bfc3c033df..f72022e359 100644 --- a/testing/btest/Baseline/doc.broxygen.all_scripts/.stdout +++ b/testing/btest/Baseline/doc.broxygen.all_scripts/.stdout @@ -1 +1,2 @@ +The '' control command is unknown. WARNING: No Site::local_nets have been defined. It's usually a good idea to define your local networks. diff --git a/testing/btest/coverage/broxygen.sh b/testing/btest/coverage/broxygen.sh new file mode 100644 index 0000000000..13bf24bce3 --- /dev/null +++ b/testing/btest/coverage/broxygen.sh @@ -0,0 +1,35 @@ +# This check piggy-backs on the test-all-policy.bro test, assuming that every +# loadable script is referenced there. The only additional check here is +# that the broxygen package should even load scripts that are commented +# out in test-all-policy.bro because the broxygen package is only loaded +# when generated documentation and will terminate has soon as bro_init +# is handled, even if a script will e.g. put Bro into listen mode or otherwise +# cause it to not terminate after scripts are parsed. + +# @TEST-EXEC: bash %INPUT $DIST/scripts/test-all-policy.bro $DIST/scripts/broxygen/__load__.bro + +error_count=0 + +error_msg() + { + error_count=$((error_count+1)) + echo "$@" 1>&2; + } + +if [ $# -ne 2 ]; then + print "incorrect arguments" + exit 1 +fi + +all_loads=$(egrep "#[[:space:]]*@load.*" $1 | sed 's/#[[:space:]]*@load[[:space:]]*//g') +broxygen_loads=$(egrep "@load.*" $2 | sed 's/@load[[:space:]]*//g') + +for f in $all_loads; do + echo "$broxygen_loads" | grep -q $f || error_msg "$f not loaded in broxygen/__load__.bro" +done + +if [ $error_count -gt 0 ]; then + exit 1; +fi + +exit 0 diff --git a/testing/btest/doc/broxygen/all_scripts.test b/testing/btest/doc/broxygen/all_scripts.test index 56bb5313bb..f4d9237adf 100644 --- a/testing/btest/doc/broxygen/all_scripts.test +++ b/testing/btest/doc/broxygen/all_scripts.test @@ -1,6 +1,10 @@ # This test is mostly just checking that there's no errors that result # from loading all scripts and generated docs for each. +# This must be serialized with communication tests because it does load +# listen.bro in order to document it. + +# @TEST-SERIALIZE: comm # @TEST-EXEC: bro -X broxygen.config broxygen # @TEST-EXEC: btest-diff .stdout # @TEST-EXEC: btest-diff .stderr From 9163935aba268c78fa69d71575a99e6876b86d59 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Mon, 25 Nov 2013 16:08:46 -0600 Subject: [PATCH 20/40] Fix Broxygen-related compile errors. --- src/broxygen/Configuration.cc | 1 + src/broxygen/Manager.cc | 3 +-- src/broxygen/Manager.h | 1 + src/broxygen/PackageInfo.cc | 1 + src/broxygen/utils.cc | 1 + 5 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/broxygen/Configuration.cc b/src/broxygen/Configuration.cc index bb346935a9..92f070c5de 100644 --- a/src/broxygen/Configuration.cc +++ b/src/broxygen/Configuration.cc @@ -7,6 +7,7 @@ #include #include #include +#include using namespace broxygen; using namespace std; diff --git a/src/broxygen/Manager.cc b/src/broxygen/Manager.cc index b923efe61f..64c7bf1ef7 100644 --- a/src/broxygen/Manager.cc +++ b/src/broxygen/Manager.cc @@ -167,8 +167,7 @@ IdentifierInfo* Manager::CreateIdentifierInfo(ID* id, ScriptInfo* script) rval->AddComments(comment_buffer); comment_buffer.clear(); - comment_buffer_map_t::const_iterator it = - comment_buffer_map.find(id->Name()); + comment_buffer_map_t::iterator it = comment_buffer_map.find(id->Name()); if ( it != comment_buffer_map.end() ) { diff --git a/src/broxygen/Manager.h b/src/broxygen/Manager.h index bb67cf35cf..8d606357f2 100644 --- a/src/broxygen/Manager.h +++ b/src/broxygen/Manager.h @@ -17,6 +17,7 @@ #include #include #include +#include namespace broxygen { diff --git a/src/broxygen/PackageInfo.cc b/src/broxygen/PackageInfo.cc index c01182dfeb..88bf167ef6 100644 --- a/src/broxygen/PackageInfo.cc +++ b/src/broxygen/PackageInfo.cc @@ -4,6 +4,7 @@ #include "Reporter.h" #include +#include using namespace std; using namespace broxygen; diff --git a/src/broxygen/utils.cc b/src/broxygen/utils.cc index 5d0b3be91b..4d33670f28 100644 --- a/src/broxygen/utils.cc +++ b/src/broxygen/utils.cc @@ -3,6 +3,7 @@ #include "Reporter.h" #include +#include using namespace broxygen; using namespace std; From 5aa04089afb052fa496b22a54f45a53eac457e08 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Tue, 3 Dec 2013 11:18:17 -0600 Subject: [PATCH 21/40] Unit test for broccoli vector support. Broccoli clients can only recv events w/ vectors for now. Also changed ordering of Bro type tag enum -- the addition of opaque types changed the value of the vector type, making broccoli.h's definition out of sync. Probably could have just changed broccoli's definition, but seems more correct to go back to using the same value for vectors as they were before opaques. It's also better in case there's some other location I'm not aware of where the values are replicated. --- src/Type.cc | 41 ++++++++++------- src/Type.h | 44 +++++++++++-------- .../istate.broccoli-vector/bro..stdout | 13 ++++++ .../istate.broccoli-vector/broccoli..stdout | 12 +++++ testing/btest/istate/broccoli-vector.bro | 10 +++++ 5 files changed, 87 insertions(+), 33 deletions(-) create mode 100644 testing/btest/Baseline/istate.broccoli-vector/bro..stdout create mode 100644 testing/btest/Baseline/istate.broccoli-vector/broccoli..stdout create mode 100644 testing/btest/istate/broccoli-vector.bro diff --git a/src/Type.cc b/src/Type.cc index a6d8b90c6c..655b25c644 100644 --- a/src/Type.cc +++ b/src/Type.cc @@ -19,21 +19,32 @@ extern int generate_documentation; const char* type_name(TypeTag t) { static const char* type_names[int(NUM_TYPES)] = { - "void", - "bool", "int", "count", "counter", - "double", "time", "interval", - "string", "pattern", - "enum", - "timer", - "port", "addr", "subnet", - "any", - "table", "union", "record", "types", - "func", - "file", - "opaque", - "vector", - "type", - "error", + "void", // 0 + "bool", // 1 + "int", // 2 + "count", // 3 + "counter", // 4 + "double", // 5 + "time", // 6 + "interval", // 7 + "string", // 8 + "pattern", // 9 + "enum", // 10 + "timer", // 11 + "port", // 12 + "addr", // 13 + "subnet", // 14 + "any", // 15 + "table", // 16 + "union", // 17 + "record", // 18 + "types", // 19 + "func", // 20 + "file", // 21 + "vector", // 22 + "opaque", // 23 + "type", // 24 + "error", // 25 }; if ( int(t) >= NUM_TYPES ) diff --git a/src/Type.h b/src/Type.h index a6163d5152..92b74b8457 100644 --- a/src/Type.h +++ b/src/Type.h @@ -15,24 +15,32 @@ // BRO types. typedef enum { - TYPE_VOID, - TYPE_BOOL, TYPE_INT, TYPE_COUNT, TYPE_COUNTER, TYPE_DOUBLE, - TYPE_TIME, TYPE_INTERVAL, - TYPE_STRING, TYPE_PATTERN, - TYPE_ENUM, - TYPE_TIMER, - TYPE_PORT, TYPE_ADDR, TYPE_SUBNET, - TYPE_ANY, - TYPE_TABLE, - TYPE_UNION, - TYPE_RECORD, - TYPE_LIST, - TYPE_FUNC, - TYPE_FILE, - TYPE_OPAQUE, - TYPE_VECTOR, - TYPE_TYPE, - TYPE_ERROR + TYPE_VOID, // 0 + TYPE_BOOL, // 1 + TYPE_INT, // 2 + TYPE_COUNT, // 3 + TYPE_COUNTER, // 4 + TYPE_DOUBLE, // 5 + TYPE_TIME, // 6 + TYPE_INTERVAL, // 7 + TYPE_STRING, // 8 + TYPE_PATTERN, // 9 + TYPE_ENUM, // 10 + TYPE_TIMER, // 11 + TYPE_PORT, // 12 + TYPE_ADDR, // 13 + TYPE_SUBNET, // 14 + TYPE_ANY, // 15 + TYPE_TABLE, // 16 + TYPE_UNION, // 17 + TYPE_RECORD, // 18 + TYPE_LIST, // 19 + TYPE_FUNC, // 20 + TYPE_FILE, // 21 + TYPE_VECTOR, // 22 + TYPE_OPAQUE, // 23 + TYPE_TYPE, // 24 + TYPE_ERROR // 25 #define NUM_TYPES (int(TYPE_ERROR) + 1) } TypeTag; diff --git a/testing/btest/Baseline/istate.broccoli-vector/bro..stdout b/testing/btest/Baseline/istate.broccoli-vector/bro..stdout new file mode 100644 index 0000000000..5b50dcda66 --- /dev/null +++ b/testing/btest/Baseline/istate.broccoli-vector/bro..stdout @@ -0,0 +1,13 @@ +handshake done with peer +bro_vector([a, b, c, d, e]) +bro_vector([one, two, three]) +broccoli_vector_element(0, a) +broccoli_vector_element(1, b) +broccoli_vector_element(2, c) +broccoli_vector_element(3, d) +broccoli_vector_element(4, e) +broccoli_vector_element(5, additional element) +broccoli_vector_element(0, one) +broccoli_vector_element(1, two) +broccoli_vector_element(2, three) +broccoli_vector_element(3, additional element) diff --git a/testing/btest/Baseline/istate.broccoli-vector/broccoli..stdout b/testing/btest/Baseline/istate.broccoli-vector/broccoli..stdout new file mode 100644 index 0000000000..147384f565 --- /dev/null +++ b/testing/btest/Baseline/istate.broccoli-vector/broccoli..stdout @@ -0,0 +1,12 @@ +Connected to Bro instance at: localhost:47757 +Received bro_vector + 0: a + 1: b + 2: c + 3: d + 4: e +Received bro_vector + 0: one + 1: two + 2: three +Terminating diff --git a/testing/btest/istate/broccoli-vector.bro b/testing/btest/istate/broccoli-vector.bro new file mode 100644 index 0000000000..ce107f45d3 --- /dev/null +++ b/testing/btest/istate/broccoli-vector.bro @@ -0,0 +1,10 @@ +# @TEST-SERIALIZE: comm +# +# @TEST-REQUIRES: test -e $BUILD/aux/broccoli/src/libbroccoli.so || test -e $BUILD/aux/broccoli/src/libbroccoli.dylib +# +# @TEST-EXEC: btest-bg-run bro bro $DIST/aux/broccoli/test/broccoli-vectors.bro +# @TEST-EXEC: sleep 1 +# @TEST-EXEC: btest-bg-run broccoli $BUILD/aux/broccoli/test/broccoli-vectors +# @TEST-EXEC: btest-bg-wait 20 +# @TEST-EXEC: btest-diff bro/.stdout +# @TEST-EXEC: btest-diff broccoli/.stdout From 4014cdc277d0e17b9895b52b81ff1ba7925eefb1 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Tue, 3 Dec 2013 14:39:21 -0600 Subject: [PATCH 22/40] Fix string slice notation, addresses BIT-1097. Slice ranges were not correctly determined for negative indices and also off by one in general (included one more element at the end of the substring than what actually matched the index range). It's now equivalent to Python slice notation. Accessing a string at a single index is also the same as Python except that an out-of-range index returns an empty string instead of throwing an expection. --- src/Expr.cc | 38 +++++++-- .../Baseline/language.string-indexing/out | 84 +++++++++++++++++-- testing/btest/language/string-indexing.bro | 77 +++++++++++++++++ testing/scripts/file-analysis-test.bro | 2 +- 4 files changed, 187 insertions(+), 14 deletions(-) diff --git a/src/Expr.cc b/src/Expr.cc index 907cfc904c..2b55184945 100644 --- a/src/Expr.cc +++ b/src/Expr.cc @@ -3037,6 +3037,16 @@ Val* IndexExpr::Eval(Frame* f) const return result; } +static int get_slice_index(int idx, int len) + { + if ( abs(idx) > len ) + idx = idx > 0 ? len : 0; // Clamp maximum positive/negative indices. + else if ( idx < 0 ) + idx += len; // Map to a positive index. + + return idx; + } + Val* IndexExpr::Fold(Val* v1, Val* v2) const { if ( IsError() ) @@ -3058,16 +3068,30 @@ Val* IndexExpr::Fold(Val* v1, Val* v2) const const ListVal* lv = v2->AsListVal(); const BroString* s = v1->AsString(); int len = s->Len(); - bro_int_t first = lv->Index(0)->AsInt(); - bro_int_t last = lv->Length() > 1 ? lv->Index(1)->AsInt() : first; + BroString* substring = 0; - if ( first < 0 ) - first += len; + if ( lv->Length() == 1 ) + { + bro_int_t idx = lv->Index(0)->AsInt(); - if ( last < 0 ) - last += len; + if ( idx < 0 ) + idx += len; + + // Out-of-range index will return null pointer. + substring = s->GetSubstring(idx, 1); + } + else + { + bro_int_t first = get_slice_index(lv->Index(0)->AsInt(), len); + bro_int_t last = get_slice_index(lv->Index(1)->AsInt(), len); + int substring_len = last - first; + + if ( substring_len < 0 ) + substring = 0; + else + substring = s->GetSubstring(first, substring_len); + } - BroString* substring = s->GetSubstring(first, last - first + 1); return new StringVal(substring ? substring : new BroString("")); } diff --git a/testing/btest/Baseline/language.string-indexing/out b/testing/btest/Baseline/language.string-indexing/out index 3359187d4c..99464302ed 100644 --- a/testing/btest/Baseline/language.string-indexing/out +++ b/testing/btest/Baseline/language.string-indexing/out @@ -1,13 +1,85 @@ 1 -12 -123456 +1 +12345 0123456789 8 -789 -9 -9 -9 +78 + + +0123456789 2 1 +word[-100] = +word[-7] = +word[-6] = +word[-5] = H +word[-4] = e +word[-3] = l +word[-2] = p +word[-1] = A +word[0] = H +word[1] = e +word[2] = l +word[3] = p +word[4] = A +word[5] = +word[6] = +word[7] = +word[100] = + +A +1234 +123 +12 +1 + + + + + + +B + + + + + +C + +0123 + +01 +012345 + +D + + +4 + + +45 + +E + +01234 + +01 +012345 + +F + +234 + + +23 +2345 + +F + + + + + + diff --git a/testing/btest/language/string-indexing.bro b/testing/btest/language/string-indexing.bro index f991b3c5fa..bff37f6bec 100644 --- a/testing/btest/language/string-indexing.bro +++ b/testing/btest/language/string-indexing.bro @@ -1,7 +1,10 @@ # @TEST-EXEC: bro -b %INPUT >out # @TEST-EXEC: btest-diff out +local word = "HelpA"; local s = "0123456789"; +local indices = vector(-100, -7, -6, -5, -4, -3, -2, -1, 0, 1, 2, 3, 4, 5, 6, 7, 100); + print s[1]; print s[1:2]; print s[1:6]; @@ -15,3 +18,77 @@ print s[20:23]; print s[-20:23]; print s[0:5][2]; print s[0:5][1:3][0]; + +s = "012345"; + +for ( i in indices ) + print fmt("word[%s] = %s", indices[i], word[indices[i]]); + +print ""; + +print "A"; +print s[1:-1]; +print s[1:-2]; +print s[1:-3]; +print s[1:-4]; +print s[1:-5]; +print s[1:-6]; +print s[1:-7]; +print s[1:-8]; +print s[1:-9]; + +print ""; + +print "B"; +print s[-1:-1]; +print s[-1:-2]; +print s[-1:-3]; +print s[-1:-4]; + +print ""; + +print "C"; +print s[-100:-99]; +print s[-100:-2]; +print s[-100:0]; +print s[-100:2]; +print s[-100:100]; + +print ""; + +print "D";; +print s[-2:-99]; +print s[-2:-3]; +print s[-2:-1]; +print s[-2:0]; +print s[-2:2]; +print s[-2:100]; + +print ""; + +print "E";; +print s[0:-100]; +print s[0:-1]; +print s[0:0]; +print s[0:2]; +print s[0:100]; + +print ""; + +print "F";; +print s[2:-100]; +print s[2:-1]; +print s[2:0]; +print s[2:1]; +print s[2:4]; +print s[2:100]; + +print ""; + +print "F";; +print s[100:-100]; +print s[100:-1]; +print s[100:0]; +print s[100:1]; +print s[100:4]; +print s[100:100]; diff --git a/testing/scripts/file-analysis-test.bro b/testing/scripts/file-analysis-test.bro index d84fadae5c..957218b48f 100644 --- a/testing/scripts/file-analysis-test.bro +++ b/testing/scripts/file-analysis-test.bro @@ -60,7 +60,7 @@ event file_new(f: fa_file) if ( f?$bof_buffer ) { print "FILE_BOF_BUFFER"; - print f$bof_buffer[0:10]; + print f$bof_buffer[0:11]; } if ( f?$mime_type ) From 2ea6011186e91dff37c639667b1604d837363908 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Tue, 3 Dec 2013 16:51:36 -0600 Subject: [PATCH 23/40] Improve a unit test involving 'when' conditionals. May fix a sporadic failure, in which case it was just not getting enough time to run or lookup_hostname() taking longer than ~3 seconds. Else, the new output should give more hints on what's going wrong. In any case, termination conditions for the test are now explicit. --- .../bro..stderr | 5 -- .../bro..stdout | 8 --- .../bro.output | 13 +++++ .../core/when-interpreter-exceptions.bro | 57 +++++++++++++++++-- 4 files changed, 66 insertions(+), 17 deletions(-) delete mode 100644 testing/btest/Baseline/core.when-interpreter-exceptions/bro..stderr delete mode 100644 testing/btest/Baseline/core.when-interpreter-exceptions/bro..stdout create mode 100644 testing/btest/Baseline/core.when-interpreter-exceptions/bro.output diff --git a/testing/btest/Baseline/core.when-interpreter-exceptions/bro..stderr b/testing/btest/Baseline/core.when-interpreter-exceptions/bro..stderr deleted file mode 100644 index 256600445d..0000000000 --- a/testing/btest/Baseline/core.when-interpreter-exceptions/bro..stderr +++ /dev/null @@ -1,5 +0,0 @@ -1355264617.053514 expression error in /home/jsiwek/bro/testing/btest/.tmp/core.when-interpreter-exceptions/when-interpreter-exceptions.bro, line 32: field value missing [p$ip] -1355264617.053514 expression error in /home/jsiwek/bro/testing/btest/.tmp/core.when-interpreter-exceptions/when-interpreter-exceptions.bro, line 53: field value missing [p$ip] -1355264617.053514 expression error in /home/jsiwek/bro/testing/btest/.tmp/core.when-interpreter-exceptions/when-interpreter-exceptions.bro, line 43: field value missing [p$ip] -1355264617.053514 expression error in /home/jsiwek/bro/testing/btest/.tmp/core.when-interpreter-exceptions/when-interpreter-exceptions.bro, line 16: field value missing [p$ip] -1355264617.053514 received termination signal diff --git a/testing/btest/Baseline/core.when-interpreter-exceptions/bro..stdout b/testing/btest/Baseline/core.when-interpreter-exceptions/bro..stdout deleted file mode 100644 index 49eafc365f..0000000000 --- a/testing/btest/Baseline/core.when-interpreter-exceptions/bro..stdout +++ /dev/null @@ -1,8 +0,0 @@ -timeout g(), F -timeout g(), T -timeout -g() done, no exception, T -localhost resolved -localhost resolved from f(), T -localhost resolved from f(), F -f() done, no exception, T diff --git a/testing/btest/Baseline/core.when-interpreter-exceptions/bro.output b/testing/btest/Baseline/core.when-interpreter-exceptions/bro.output new file mode 100644 index 0000000000..154734f6d3 --- /dev/null +++ b/testing/btest/Baseline/core.when-interpreter-exceptions/bro.output @@ -0,0 +1,13 @@ +1386110869.157209 expression error in /Users/jon/Projects/bro/bro/testing/btest/.tmp/core.when-interpreter-exceptions/when-interpreter-exceptions.bro, line 96: field value missing [p$ip] +1386110869.157209 expression error in /Users/jon/Projects/bro/bro/testing/btest/.tmp/core.when-interpreter-exceptions/when-interpreter-exceptions.bro, line 63: field value missing [p$ip] +1386110869.157209 expression error in /Users/jon/Projects/bro/bro/testing/btest/.tmp/core.when-interpreter-exceptions/when-interpreter-exceptions.bro, line 79: field value missing [p$ip] +1386110869.157209 expression error in /Users/jon/Projects/bro/bro/testing/btest/.tmp/core.when-interpreter-exceptions/when-interpreter-exceptions.bro, line 36: field value missing [p$ip] +1386110869.157209 received termination signal +timeout +timeout g(), F +timeout g(), T +g() done, no exception, T +localhost resolved +localhost resolved from f(), T +localhost resolved from f(), F +f() done, no exception, T diff --git a/testing/btest/core/when-interpreter-exceptions.bro b/testing/btest/core/when-interpreter-exceptions.bro index 9b6f689dcf..5918f80ab5 100644 --- a/testing/btest/core/when-interpreter-exceptions.bro +++ b/testing/btest/core/when-interpreter-exceptions.bro @@ -1,21 +1,49 @@ -# @TEST-EXEC: btest-bg-run bro bro -b --pseudo-realtime -r $TRACES/rotation.trace %INPUT -# @TEST-EXEC: btest-bg-wait -k 3 -# @TEST-EXEC: TEST_DIFF_CANONIFIER="$SCRIPTS/diff-remove-abspath | $SCRIPTS/diff-remove-timestamps | $SCRIPTS/diff-sort" btest-diff bro/.stderr -# @TEST-EXEC: btest-diff bro/.stdout +# @TEST-EXEC: btest-bg-run bro "bro -b --pseudo-realtime -r $TRACES/rotation.trace %INPUT >output 2>&1" +# @TEST-EXEC: btest-bg-wait 15 +# @TEST-EXEC: TEST_DIFF_CANONIFIER="$SCRIPTS/diff-remove-abspath | $SCRIPTS/diff-remove-timestamps | $SCRIPTS/diff-sort" btest-diff bro/output # interpreter exceptions in "when" blocks shouldn't cause termination global p: pkt_hdr; +global c = 0; + +function check_term_condition() + { + ++c; + + #print "check_term_condition", c; + + if ( c == 6 ) + terminate(); + } + +event termination_check() + { + #print "termination_check event"; + check_term_condition(); + } + function f(do_exception: bool): bool { return when ( local addrs = lookup_hostname("localhost") ) { print "localhost resolved from f()", do_exception; + if ( do_exception ) + { + event termination_check(); print p$ip; + } + return T; } + timeout 10 sec + { + print "lookup_hostname in f() timed out unexpectedly"; + } + + check_term_condition(); return F; } @@ -28,10 +56,17 @@ function g(do_exception: bool): bool timeout 0 sec { print "timeout g()", do_exception; + if ( do_exception ) + { + event termination_check(); print p$ip; + } + return T; } + + check_term_condition(); return F; } @@ -40,16 +75,24 @@ event bro_init() when ( local addrs = lookup_hostname("localhost") ) { print "localhost resolved"; + event termination_check(); print p$ip; } + timeout 10 sec + { + print "lookup_hostname timed out unexpectedly"; + check_term_condition(); + } when ( local addrs2 = lookup_hostname("localhost") ) { print "shouldn't get here"; + check_term_condition(); } timeout 0 sec { print "timeout"; + event termination_check(); print p$ip; } @@ -60,8 +103,14 @@ event bro_init() print "g() exception done (shouldn't be printed)", b2; when ( local b3 = f(F) ) + { print "f() done, no exception", b3; + check_term_condition(); + } when ( local b4 = g(F) ) + { print "g() done, no exception", b4; + check_term_condition(); + } } From b7dc03bb822c5a61a0b9867db5a936e2febd8730 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Wed, 4 Dec 2013 11:27:34 -0800 Subject: [PATCH 24/40] Change ciphers in changes ciphers from a set to a vector. This preserves the ordering of the cipher suites the client sent, allowing e.g. better client fingerprinting. --- scripts/base/protocols/ssl/main.bro | 2 +- src/analyzer/protocol/ssl/events.bif | 2 +- src/analyzer/protocol/ssl/ssl-analyzer.pac | 7 +- .../.stdout | 81 +++++++++++++++++++ .../base/protocols/ssl/tls-1.2-ciphers.test | 9 +++ .../base/protocols/ssl/tls-1.2-random.test | 2 +- 6 files changed, 96 insertions(+), 7 deletions(-) create mode 100644 testing/btest/Baseline/scripts.base.protocols.ssl.tls-1.2-ciphers/.stdout create mode 100644 testing/btest/scripts/base/protocols/ssl/tls-1.2-ciphers.test diff --git a/scripts/base/protocols/ssl/main.bro b/scripts/base/protocols/ssl/main.bro index 9eab0ca970..3958a90fa2 100644 --- a/scripts/base/protocols/ssl/main.bro +++ b/scripts/base/protocols/ssl/main.bro @@ -153,7 +153,7 @@ function finish(c: connection) disable_analyzer(c$id, c$ssl$analyzer_id); } -event ssl_client_hello(c: connection, version: count, possible_ts: time, client_random: string, session_id: string, ciphers: count_set) &priority=5 +event ssl_client_hello(c: connection, version: count, possible_ts: time, client_random: string, session_id: string, ciphers: index_vec) &priority=5 { set_session(c); diff --git a/src/analyzer/protocol/ssl/events.bif b/src/analyzer/protocol/ssl/events.bif index b673954e53..01abb87745 100644 --- a/src/analyzer/protocol/ssl/events.bif +++ b/src/analyzer/protocol/ssl/events.bif @@ -26,7 +26,7 @@ ## ## .. bro:see:: ssl_alert ssl_established ssl_extension ssl_server_hello ## ssl_session_ticket_handshake x509_certificate x509_error x509_extension -event ssl_client_hello%(c: connection, version: count, possible_ts: time, client_random: string, session_id: string, ciphers: count_set%); +event ssl_client_hello%(c: connection, version: count, possible_ts: time, client_random: string, session_id: string, ciphers: index_vec%); ## Generated for an SSL/TLS server's initial *hello* message. SSL/TLS sessions ## start with an unencrypted handshake, and Bro extracts as much information out diff --git a/src/analyzer/protocol/ssl/ssl-analyzer.pac b/src/analyzer/protocol/ssl/ssl-analyzer.pac index 4043d1ac89..18d3812742 100644 --- a/src/analyzer/protocol/ssl/ssl-analyzer.pac +++ b/src/analyzer/protocol/ssl/ssl-analyzer.pac @@ -168,19 +168,18 @@ refine connection SSL_Conn += { else std::transform(cipher_suites24->begin(), cipher_suites24->end(), std::back_inserter(*cipher_suites), to_int()); - TableVal* cipher_set = new TableVal(internal_type("count_set")->AsTableType()); + VectorVal* cipher_vec = new VectorVal(internal_type("index_vec")->AsVectorType()); for ( unsigned int i = 0; i < cipher_suites->size(); ++i ) { Val* ciph = new Val((*cipher_suites)[i], TYPE_COUNT); - cipher_set->Assign(ciph, 0); - Unref(ciph); + cipher_vec->Assign(i, ciph); } BifEvent::generate_ssl_client_hello(bro_analyzer(), bro_analyzer()->Conn(), version, ts, new StringVal(client_random.length(), (const char*) client_random.data()), to_string_val(session_id), - cipher_set); + cipher_vec); delete cipher_suites; } diff --git a/testing/btest/Baseline/scripts.base.protocols.ssl.tls-1.2-ciphers/.stdout b/testing/btest/Baseline/scripts.base.protocols.ssl.tls-1.2-ciphers/.stdout new file mode 100644 index 0000000000..2f2781f430 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.protocols.ssl.tls-1.2-ciphers/.stdout @@ -0,0 +1,81 @@ +Got 80 cipher suites +TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 +TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 +TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384 +TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384 +TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA +TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA +TLS_SRP_SHA_DSS_WITH_AES_256_CBC_SHA +TLS_SRP_SHA_RSA_WITH_AES_256_CBC_SHA +TLS_DHE_DSS_WITH_AES_256_GCM_SHA384 +TLS_DHE_RSA_WITH_AES_256_GCM_SHA384 +TLS_DHE_RSA_WITH_AES_256_CBC_SHA256 +TLS_DHE_DSS_WITH_AES_256_CBC_SHA256 +TLS_DHE_RSA_WITH_AES_256_CBC_SHA +TLS_DHE_DSS_WITH_AES_256_CBC_SHA +TLS_DHE_RSA_WITH_CAMELLIA_256_CBC_SHA +TLS_DHE_DSS_WITH_CAMELLIA_256_CBC_SHA +TLS_ECDH_RSA_WITH_AES_256_GCM_SHA384 +TLS_ECDH_ECDSA_WITH_AES_256_GCM_SHA384 +TLS_ECDH_RSA_WITH_AES_256_CBC_SHA384 +TLS_ECDH_ECDSA_WITH_AES_256_CBC_SHA384 +TLS_ECDH_RSA_WITH_AES_256_CBC_SHA +TLS_ECDH_ECDSA_WITH_AES_256_CBC_SHA +TLS_RSA_WITH_AES_256_GCM_SHA384 +TLS_RSA_WITH_AES_256_CBC_SHA256 +TLS_RSA_WITH_AES_256_CBC_SHA +TLS_RSA_WITH_CAMELLIA_256_CBC_SHA +TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA +TLS_ECDHE_ECDSA_WITH_3DES_EDE_CBC_SHA +TLS_SRP_SHA_DSS_WITH_3DES_EDE_CBC_SHA +TLS_SRP_SHA_RSA_WITH_3DES_EDE_CBC_SHA +TLS_DHE_RSA_WITH_3DES_EDE_CBC_SHA +TLS_DHE_DSS_WITH_3DES_EDE_CBC_SHA +TLS_ECDH_RSA_WITH_3DES_EDE_CBC_SHA +TLS_ECDH_ECDSA_WITH_3DES_EDE_CBC_SHA +TLS_RSA_WITH_3DES_EDE_CBC_SHA +TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 +TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 +TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256 +TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256 +TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA +TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA +TLS_SRP_SHA_DSS_WITH_AES_128_CBC_SHA +TLS_SRP_SHA_RSA_WITH_AES_128_CBC_SHA +TLS_DHE_DSS_WITH_AES_128_GCM_SHA256 +TLS_DHE_RSA_WITH_AES_128_GCM_SHA256 +TLS_DHE_RSA_WITH_AES_128_CBC_SHA256 +TLS_DHE_DSS_WITH_AES_128_CBC_SHA256 +TLS_DHE_RSA_WITH_AES_128_CBC_SHA +TLS_DHE_DSS_WITH_AES_128_CBC_SHA +TLS_DHE_RSA_WITH_SEED_CBC_SHA +TLS_DHE_DSS_WITH_SEED_CBC_SHA +TLS_DHE_RSA_WITH_CAMELLIA_128_CBC_SHA +TLS_DHE_DSS_WITH_CAMELLIA_128_CBC_SHA +TLS_ECDH_RSA_WITH_AES_128_GCM_SHA256 +TLS_ECDH_ECDSA_WITH_AES_128_GCM_SHA256 +TLS_ECDH_RSA_WITH_AES_128_CBC_SHA256 +TLS_ECDH_ECDSA_WITH_AES_128_CBC_SHA256 +TLS_ECDH_RSA_WITH_AES_128_CBC_SHA +TLS_ECDH_ECDSA_WITH_AES_128_CBC_SHA +TLS_RSA_WITH_AES_128_GCM_SHA256 +TLS_RSA_WITH_AES_128_CBC_SHA256 +TLS_RSA_WITH_AES_128_CBC_SHA +TLS_RSA_WITH_SEED_CBC_SHA +TLS_RSA_WITH_CAMELLIA_128_CBC_SHA +TLS_RSA_WITH_IDEA_CBC_SHA +TLS_ECDHE_RSA_WITH_RC4_128_SHA +TLS_ECDHE_ECDSA_WITH_RC4_128_SHA +TLS_ECDH_RSA_WITH_RC4_128_SHA +TLS_ECDH_ECDSA_WITH_RC4_128_SHA +TLS_RSA_WITH_RC4_128_SHA +TLS_RSA_WITH_RC4_128_MD5 +TLS_DHE_RSA_WITH_DES_CBC_SHA +TLS_DHE_DSS_WITH_DES_CBC_SHA +TLS_RSA_WITH_DES_CBC_SHA +TLS_DHE_RSA_EXPORT_WITH_DES40_CBC_SHA +TLS_DHE_DSS_EXPORT_WITH_DES40_CBC_SHA +TLS_RSA_EXPORT_WITH_DES40_CBC_SHA +TLS_RSA_EXPORT_WITH_RC2_CBC_40_MD5 +TLS_RSA_EXPORT_WITH_RC4_40_MD5 +TLS_EMPTY_RENEGOTIATION_INFO_SCSV diff --git a/testing/btest/scripts/base/protocols/ssl/tls-1.2-ciphers.test b/testing/btest/scripts/base/protocols/ssl/tls-1.2-ciphers.test new file mode 100644 index 0000000000..be1a2c2c2d --- /dev/null +++ b/testing/btest/scripts/base/protocols/ssl/tls-1.2-ciphers.test @@ -0,0 +1,9 @@ +# @TEST-EXEC: bro -r $TRACES/tls1.2.trace %INPUT +# @TEST-EXEC: btest-diff .stdout + +event ssl_client_hello(c: connection, version: count, possible_ts: time, client_random: string, session_id: string, ciphers: index_vec) + { + print fmt("Got %d cipher suites", |ciphers|); + for ( i in ciphers ) + print SSL::cipher_desc[ciphers[i]]; + } diff --git a/testing/btest/scripts/base/protocols/ssl/tls-1.2-random.test b/testing/btest/scripts/base/protocols/ssl/tls-1.2-random.test index acea4fa131..d2f82c4bc0 100644 --- a/testing/btest/scripts/base/protocols/ssl/tls-1.2-random.test +++ b/testing/btest/scripts/base/protocols/ssl/tls-1.2-random.test @@ -1,7 +1,7 @@ # @TEST-EXEC: bro -r $TRACES/tls1.2.trace %INPUT # @TEST-EXEC: btest-diff .stdout -event ssl_client_hello(c: connection, version: count, possible_ts: time, client_random: string, session_id: string, ciphers: count_set) +event ssl_client_hello(c: connection, version: count, possible_ts: time, client_random: string, session_id: string, ciphers: index_vec) { print client_random; } From 8331d0d09b00cede80a7362af7a18e42ec116a00 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Wed, 4 Dec 2013 12:40:51 -0800 Subject: [PATCH 25/40] Updating tests. --- scripts/test-all-policy.bro | 1 + testing/btest/Baseline/language.record-ceorce-orphan/out | 4 ++-- .../Baseline/scripts.policy.misc.dump-events/all-events.log | 6 +++--- .../Baseline/scripts.policy.misc.dump-events/ssl-events.log | 6 +++--- 4 files changed, 9 insertions(+), 8 deletions(-) diff --git a/scripts/test-all-policy.bro b/scripts/test-all-policy.bro index d6383af38b..05b594946e 100644 --- a/scripts/test-all-policy.bro +++ b/scripts/test-all-policy.bro @@ -48,6 +48,7 @@ @load misc/capture-loss.bro @load misc/detect-traceroute/__load__.bro @load misc/detect-traceroute/main.bro +@load misc/dump-events.bro @load misc/known-devices.bro @load misc/load-balancing.bro @load misc/loaded-scripts.bro diff --git a/testing/btest/Baseline/language.record-ceorce-orphan/out b/testing/btest/Baseline/language.record-ceorce-orphan/out index aa42d13892..59df204af2 100644 --- a/testing/btest/Baseline/language.record-ceorce-orphan/out +++ b/testing/btest/Baseline/language.record-ceorce-orphan/out @@ -1,2 +1,2 @@ -error in /Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/language.record-ceorce-orphan/record-ceorce-orphan.bro, line 19: orphaned field "wtf" in record coercion ((coerce [$a=test, $b=42, $wtf=1.0 sec] to record { a:string; b:count; c:interval; })) -error in /Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/language.record-ceorce-orphan/record-ceorce-orphan.bro, line 21: orphaned field "wtf" in record coercion ((coerce [$a=test, $b=42, $wtf=1.0 sec] to record { a:string; b:count; c:interval; })) +error in /home/robin/bro/master/testing/btest/.tmp/language.record-ceorce-orphan/record-ceorce-orphan.bro, line 19: orphaned field "wtf" in record coercion ((coerce [$a=test, $b=42, $wtf=1.0 sec] to myrec)) +error in /home/robin/bro/master/testing/btest/.tmp/language.record-ceorce-orphan/record-ceorce-orphan.bro, line 21: orphaned field "wtf" in record coercion ((coerce [$a=test, $b=42, $wtf=1.0 sec] to myrec)) diff --git a/testing/btest/Baseline/scripts.policy.misc.dump-events/all-events.log b/testing/btest/Baseline/scripts.policy.misc.dump-events/all-events.log index e3ea885785..8e9e980fbf 100644 --- a/testing/btest/Baseline/scripts.policy.misc.dump-events/all-events.log +++ b/testing/btest/Baseline/scripts.policy.misc.dump-events/all-events.log @@ -14,7 +14,7 @@ [2] possible_ts: time = 0.0 [3] client_random: string = \xe6\xb8\xef\xdf\x91\xcfD\xf7\xea\xe4<\x839\x8f\xdc\xb2 [4] session_id: string = \0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0 - [5] ciphers: set[count] = {^J^I4,^J^I6,^J^I19,^J^I10,^J^I56,^J^I50,^J^I21,^J^I98,^J^I57,^J^I51,^J^I22,^J^I65279,^J^I18,^J^I65278,^J^I100,^J^I53,^J^I9,^J^I5,^J^I47,^J^I3^J} + [5] ciphers: vector of count = [57, 56, 53, 51, 50, 4, 5, 47, 22, 19, 65279, 10, 21, 18, 65278, 9, 100, 98, 3, 6] 1170717505.734145 protocol_confirmation [0] c: connection = [id=[orig_h=192.150.187.164, orig_p=58868/tcp, resp_h=194.127.84.106, resp_p=443/tcp], orig=[size=87, state=4, num_pkts=3, num_bytes_ip=255, flow_label=0], resp=[size=1448, state=4, num_pkts=2, num_bytes_ip=116, flow_label=0], start_time=1170717505.366729, duration=0.367416, service={^J^J}, addl=, hot=0, history=ShADad, uid=CXWv6p3arKYeMETxOg, tunnel=, dpd=, conn=, extract_orig=F, extract_resp=F, dhcp=, dnp3=, dns=, dns_state=, ftp=, ftp_data_reuse=F, ssl=[ts=1170717505.549109, uid=CXWv6p3arKYeMETxOg, id=[orig_h=192.150.187.164, orig_p=58868/tcp, resp_h=194.127.84.106, resp_p=443/tcp], version=, cipher=, server_name=, session_id=, subject=, issuer_subject=, not_valid_before=, not_valid_after=, last_alert=, client_subject=, client_issuer_subject=, cert=, cert_chain=[], client_cert=, client_cert_chain=[], analyzer_id=, delay_tokens=], http=, http_state=, irc=, modbus=, smtp=, smtp_state=, socks=, ssh=, syslog=] @@ -61,7 +61,7 @@ [2] possible_ts: time = 2486404.0 [3] client_random: string = \xa8\xa2\xabs\x9ad\xab\xb4\xe6\x8c\xfc\xfc4p\xffbi\xb1\xa8hXP\x1f\xbb\xd12~\xd8 [4] session_id: string = \xa8\xc1\xc5h^Y$\xe8^J2\xa1]^^? \xbc^?Q>V\xb2^U^C\x9d^MU\xde\xfd\xa5\xa3 \xc0 - [5] ciphers: set[count] = {^J^I4,^J^I6,^J^I19,^J^I10,^J^I56,^J^I50,^J^I21,^J^I98,^J^I57,^J^I51,^J^I22,^J^I65279,^J^I18,^J^I65278,^J^I100,^J^I53,^J^I9,^J^I5,^J^I47,^J^I3^J} + [5] ciphers: vector of count = [57, 56, 53, 51, 50, 4, 5, 47, 22, 19, 65279, 10, 21, 18, 65278, 9, 100, 98, 3, 6] 1170717508.881857 protocol_confirmation [0] c: connection = [id=[orig_h=192.150.187.164, orig_p=58869/tcp, resp_h=194.127.84.106, resp_p=443/tcp], orig=[size=120, state=4, num_pkts=3, num_bytes_ip=288, flow_label=0], resp=[size=1448, state=4, num_pkts=2, num_bytes_ip=116, flow_label=0], start_time=1170717508.515696, duration=0.366161, service={^J^J}, addl=, hot=0, history=ShADad, uid=CjhGID4nQcgTWjvg4c, tunnel=, dpd=, conn=, extract_orig=F, extract_resp=F, dhcp=, dnp3=, dns=, dns_state=, ftp=, ftp_data_reuse=F, ssl=[ts=1170717508.69718, uid=CjhGID4nQcgTWjvg4c, id=[orig_h=192.150.187.164, orig_p=58869/tcp, resp_h=194.127.84.106, resp_p=443/tcp], version=, cipher=, server_name=, session_id=a8c1c5681924e80a32a15d5e7f20bc5e3f513e56b215039d0d55defda5a320c0, subject=, issuer_subject=, not_valid_before=, not_valid_after=, last_alert=, client_subject=, client_issuer_subject=, cert=, cert_chain=[], client_cert=, client_cert_chain=[], analyzer_id=, delay_tokens=], http=, http_state=, irc=, modbus=, smtp=, smtp_state=, socks=, ssh=, syslog=] @@ -108,7 +108,7 @@ [2] possible_ts: time = 2486407.0 [3] client_random: string = $^F^D\xbe/VD\xc8\xdf\xd2\xe5\x1c\xc2\xb3\xa3^Aq\xbdX\x85>\xd7\xc6\xe3\xfc\xd1\x88F [4] session_id: string = \x9eQ\xca\xef@\xad\x85\xf9\xf0=\xbb\x8c\x1f\xdc\x866!\x80\x8c1^Rr\xe1^BB\xcb@k\xf9^W\xbc\xd9 - [5] ciphers: set[count] = {^J^I4,^J^I6,^J^I19,^J^I10,^J^I56,^J^I50,^J^I21,^J^I98,^J^I57,^J^I51,^J^I22,^J^I65279,^J^I18,^J^I65278,^J^I100,^J^I53,^J^I9,^J^I5,^J^I47,^J^I3^J} + [5] ciphers: vector of count = [57, 56, 53, 51, 50, 4, 5, 47, 22, 19, 65279, 10, 21, 18, 65278, 9, 100, 98, 3, 6] 1170717511.908619 protocol_confirmation [0] c: connection = [id=[orig_h=192.150.187.164, orig_p=58870/tcp, resp_h=194.127.84.106, resp_p=443/tcp], orig=[size=120, state=4, num_pkts=3, num_bytes_ip=288, flow_label=0], resp=[size=1448, state=4, num_pkts=2, num_bytes_ip=116, flow_label=0], start_time=1170717511.541455, duration=0.367164, service={^J^J}, addl=, hot=0, history=ShADad, uid=CCvvfg3TEfuqmmG4bh, tunnel=, dpd=, conn=, extract_orig=F, extract_resp=F, dhcp=, dnp3=, dns=, dns_state=, ftp=, ftp_data_reuse=F, ssl=[ts=1170717511.722913, uid=CCvvfg3TEfuqmmG4bh, id=[orig_h=192.150.187.164, orig_p=58870/tcp, resp_h=194.127.84.106, resp_p=443/tcp], version=, cipher=, server_name=, session_id=9e51caef40ad85f9f03dbb8c1fdc863621808c311272e10242cb406bf917bcd9, subject=, issuer_subject=, not_valid_before=, not_valid_after=, last_alert=, client_subject=, client_issuer_subject=, cert=, cert_chain=[], client_cert=, client_cert_chain=[], analyzer_id=, delay_tokens=], http=, http_state=, irc=, modbus=, smtp=, smtp_state=, socks=, ssh=, syslog=] diff --git a/testing/btest/Baseline/scripts.policy.misc.dump-events/ssl-events.log b/testing/btest/Baseline/scripts.policy.misc.dump-events/ssl-events.log index 303e598f7c..1fd4bc81d6 100644 --- a/testing/btest/Baseline/scripts.policy.misc.dump-events/ssl-events.log +++ b/testing/btest/Baseline/scripts.policy.misc.dump-events/ssl-events.log @@ -4,7 +4,7 @@ [2] possible_ts: time = 0.0 [3] client_random: string = \xe6\xb8\xef\xdf\x91\xcfD\xf7\xea\xe4<\x839\x8f\xdc\xb2 [4] session_id: string = \0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0 - [5] ciphers: set[count] = {^J^I4,^J^I6,^J^I19,^J^I10,^J^I56,^J^I50,^J^I21,^J^I98,^J^I57,^J^I51,^J^I22,^J^I65279,^J^I18,^J^I65278,^J^I100,^J^I53,^J^I9,^J^I5,^J^I47,^J^I3^J} + [5] ciphers: vector of count = [57, 56, 53, 51, 50, 4, 5, 47, 22, 19, 65279, 10, 21, 18, 65278, 9, 100, 98, 3, 6] 1170717505.734145 ssl_server_hello [0] c: connection = [id=[orig_h=192.150.187.164, orig_p=58868/tcp, resp_h=194.127.84.106, resp_p=443/tcp], orig=[size=87, state=4, num_pkts=3, num_bytes_ip=255, flow_label=0], resp=[size=1448, state=4, num_pkts=2, num_bytes_ip=116, flow_label=0], start_time=1170717505.366729, duration=0.367416, service={^J^ISSL^J}, addl=, hot=0, history=ShADad, uid=CXWv6p3arKYeMETxOg, tunnel=, dpd=, conn=, extract_orig=F, extract_resp=F, dhcp=, dnp3=, dns=, dns_state=, ftp=, ftp_data_reuse=F, ssl=[ts=1170717505.549109, uid=CXWv6p3arKYeMETxOg, id=[orig_h=192.150.187.164, orig_p=58868/tcp, resp_h=194.127.84.106, resp_p=443/tcp], version=, cipher=, server_name=, session_id=, subject=, issuer_subject=, not_valid_before=, not_valid_after=, last_alert=, client_subject=, client_issuer_subject=, cert=, cert_chain=[], client_cert=, client_cert_chain=[], analyzer_id=3, delay_tokens=], http=, http_state=, irc=, modbus=, smtp=, smtp_state=, socks=, ssh=, syslog=] @@ -24,7 +24,7 @@ [2] possible_ts: time = 2486404.0 [3] client_random: string = \xa8\xa2\xabs\x9ad\xab\xb4\xe6\x8c\xfc\xfc4p\xffbi\xb1\xa8hXP\x1f\xbb\xd12~\xd8 [4] session_id: string = \xa8\xc1\xc5h^Y$\xe8^J2\xa1]^^? \xbc^?Q>V\xb2^U^C\x9d^MU\xde\xfd\xa5\xa3 \xc0 - [5] ciphers: set[count] = {^J^I4,^J^I6,^J^I19,^J^I10,^J^I56,^J^I50,^J^I21,^J^I98,^J^I57,^J^I51,^J^I22,^J^I65279,^J^I18,^J^I65278,^J^I100,^J^I53,^J^I9,^J^I5,^J^I47,^J^I3^J} + [5] ciphers: vector of count = [57, 56, 53, 51, 50, 4, 5, 47, 22, 19, 65279, 10, 21, 18, 65278, 9, 100, 98, 3, 6] 1170717508.881857 ssl_server_hello [0] c: connection = [id=[orig_h=192.150.187.164, orig_p=58869/tcp, resp_h=194.127.84.106, resp_p=443/tcp], orig=[size=120, state=4, num_pkts=3, num_bytes_ip=288, flow_label=0], resp=[size=1448, state=4, num_pkts=2, num_bytes_ip=116, flow_label=0], start_time=1170717508.515696, duration=0.366161, service={^J^ISSL^J}, addl=, hot=0, history=ShADad, uid=CjhGID4nQcgTWjvg4c, tunnel=, dpd=, conn=, extract_orig=F, extract_resp=F, dhcp=, dnp3=, dns=, dns_state=, ftp=, ftp_data_reuse=F, ssl=[ts=1170717508.69718, uid=CjhGID4nQcgTWjvg4c, id=[orig_h=192.150.187.164, orig_p=58869/tcp, resp_h=194.127.84.106, resp_p=443/tcp], version=, cipher=, server_name=, session_id=a8c1c5681924e80a32a15d5e7f20bc5e3f513e56b215039d0d55defda5a320c0, subject=, issuer_subject=, not_valid_before=, not_valid_after=, last_alert=, client_subject=, client_issuer_subject=, cert=, cert_chain=[], client_cert=, client_cert_chain=[], analyzer_id=7, delay_tokens=], http=, http_state=, irc=, modbus=, smtp=, smtp_state=, socks=, ssh=, syslog=] @@ -44,7 +44,7 @@ [2] possible_ts: time = 2486407.0 [3] client_random: string = $^F^D\xbe/VD\xc8\xdf\xd2\xe5\x1c\xc2\xb3\xa3^Aq\xbdX\x85>\xd7\xc6\xe3\xfc\xd1\x88F [4] session_id: string = \x9eQ\xca\xef@\xad\x85\xf9\xf0=\xbb\x8c\x1f\xdc\x866!\x80\x8c1^Rr\xe1^BB\xcb@k\xf9^W\xbc\xd9 - [5] ciphers: set[count] = {^J^I4,^J^I6,^J^I19,^J^I10,^J^I56,^J^I50,^J^I21,^J^I98,^J^I57,^J^I51,^J^I22,^J^I65279,^J^I18,^J^I65278,^J^I100,^J^I53,^J^I9,^J^I5,^J^I47,^J^I3^J} + [5] ciphers: vector of count = [57, 56, 53, 51, 50, 4, 5, 47, 22, 19, 65279, 10, 21, 18, 65278, 9, 100, 98, 3, 6] 1170717511.908619 ssl_server_hello [0] c: connection = [id=[orig_h=192.150.187.164, orig_p=58870/tcp, resp_h=194.127.84.106, resp_p=443/tcp], orig=[size=120, state=4, num_pkts=3, num_bytes_ip=288, flow_label=0], resp=[size=1448, state=4, num_pkts=2, num_bytes_ip=116, flow_label=0], start_time=1170717511.541455, duration=0.367164, service={^J^ISSL^J}, addl=, hot=0, history=ShADad, uid=CCvvfg3TEfuqmmG4bh, tunnel=, dpd=, conn=, extract_orig=F, extract_resp=F, dhcp=, dnp3=, dns=, dns_state=, ftp=, ftp_data_reuse=F, ssl=[ts=1170717511.722913, uid=CCvvfg3TEfuqmmG4bh, id=[orig_h=192.150.187.164, orig_p=58870/tcp, resp_h=194.127.84.106, resp_p=443/tcp], version=, cipher=, server_name=, session_id=9e51caef40ad85f9f03dbb8c1fdc863621808c311272e10242cb406bf917bcd9, subject=, issuer_subject=, not_valid_before=, not_valid_after=, last_alert=, client_subject=, client_issuer_subject=, cert=, cert_chain=[], client_cert=, client_cert_chain=[], analyzer_id=11, delay_tokens=], http=, http_state=, irc=, modbus=, smtp=, smtp_state=, socks=, ssh=, syslog=] From 8515d3aa57254c95a1afad4fc64324b1f2a2755f Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Wed, 4 Dec 2013 15:11:48 -0600 Subject: [PATCH 26/40] Support omission of string slice low/high indices, BIT-1097. Omission of the low index defaults to 0: s = "12345"; s[:3] == "123" Omission of the high index defaults to length of the string: s = "12345"; s[3:] == "45" --- src/parse.y | 17 ++++++--- .../Baseline/language.string-indexing/out | 35 +++++++++++++++++++ testing/btest/language/string-indexing.bro | 8 +++++ 3 files changed, 56 insertions(+), 4 deletions(-) diff --git a/src/parse.y b/src/parse.y index 98df0de2a3..d5ea63e5a8 100644 --- a/src/parse.y +++ b/src/parse.y @@ -55,7 +55,7 @@ %type opt_init %type TOK_CONSTANT %type pattern -%type expr init anonymous_function +%type expr opt_expr init anonymous_function %type event %type stmt stmt_list func_body for_head %type type opt_type enum_body @@ -261,6 +261,13 @@ decl_list: | ; +opt_expr: + expr + { $$ = $1; } + | + { $$ = 0; } + ; + expr: '(' expr ')' { @@ -418,11 +425,13 @@ expr: $$ = new IndexExpr($1, $3); } - | expr '[' expr ':' expr ']' + | expr '[' opt_expr ':' opt_expr ']' { set_location(@1, @6); - ListExpr* le = new ListExpr($3); - le->Append($5); + Expr* low = $3 ? $3 : new ConstExpr(new Val(0, TYPE_COUNT)); + Expr* high = $5 ? $5 : new SizeExpr($1); + ListExpr* le = new ListExpr(low); + le->Append(high); $$ = new IndexExpr($1, le, true); } diff --git a/testing/btest/Baseline/language.string-indexing/out b/testing/btest/Baseline/language.string-indexing/out index 99464302ed..4fe25c8161 100644 --- a/testing/btest/Baseline/language.string-indexing/out +++ b/testing/btest/Baseline/language.string-indexing/out @@ -28,6 +28,41 @@ word[5] = word[6] = word[7] = word[100] = +word[:-100] = +word[:-7] = +word[:-6] = +word[:-5] = +word[:-4] = H +word[:-3] = He +word[:-2] = Hel +word[:-1] = Help +word[:0] = +word[:1] = H +word[:2] = He +word[:3] = Hel +word[:4] = Help +word[:5] = HelpA +word[:6] = HelpA +word[:7] = HelpA +word[:100] = HelpA +word[-100:] = HelpA +word[-7:] = HelpA +word[-6:] = HelpA +word[-5:] = HelpA +word[-4:] = elpA +word[-3:] = lpA +word[-2:] = pA +word[-1:] = A +word[0:] = HelpA +word[1:] = elpA +word[2:] = lpA +word[3:] = pA +word[4:] = A +word[5:] = +word[6:] = +word[7:] = +word[100:] = +HelpA A 1234 diff --git a/testing/btest/language/string-indexing.bro b/testing/btest/language/string-indexing.bro index bff37f6bec..e109eeba80 100644 --- a/testing/btest/language/string-indexing.bro +++ b/testing/btest/language/string-indexing.bro @@ -24,6 +24,14 @@ s = "012345"; for ( i in indices ) print fmt("word[%s] = %s", indices[i], word[indices[i]]); +for ( i in indices ) + print fmt("word[:%s] = %s", indices[i], word[:indices[i]]); + +for ( i in indices ) + print fmt("word[%s:] = %s", indices[i], word[indices[i]:]); + +print word[:]; + print ""; print "A"; From 3c6f82ca7388ecc31e812356fb653dbf074096d3 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Wed, 4 Dec 2013 13:09:42 -0800 Subject: [PATCH 27/40] More test updates. The dump-events.bro generates output that would show in the Broxygen all-script tests. Suppressing that manually in the test for lack of a better idea. --- scripts/broxygen/__load__.bro | 1 + scripts/test-all-policy.bro | 2 +- testing/btest/Baseline/doc.broxygen.all_scripts/.stderr | 2 +- testing/btest/doc/broxygen/all_scripts.test | 2 +- 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/scripts/broxygen/__load__.bro b/scripts/broxygen/__load__.bro index 70bf1cf2ef..8db4a7c1b8 100644 --- a/scripts/broxygen/__load__.bro +++ b/scripts/broxygen/__load__.bro @@ -5,6 +5,7 @@ @load frameworks/communication/listen.bro @load frameworks/control/controllee.bro @load frameworks/control/controller.bro +@load policy/misc/dump-events.bro @load ./example.bro diff --git a/scripts/test-all-policy.bro b/scripts/test-all-policy.bro index 05b594946e..3a0bd17614 100644 --- a/scripts/test-all-policy.bro +++ b/scripts/test-all-policy.bro @@ -48,7 +48,7 @@ @load misc/capture-loss.bro @load misc/detect-traceroute/__load__.bro @load misc/detect-traceroute/main.bro -@load misc/dump-events.bro +# @load misc/dump-events.bro @load misc/known-devices.bro @load misc/load-balancing.bro @load misc/loaded-scripts.bro diff --git a/testing/btest/Baseline/doc.broxygen.all_scripts/.stderr b/testing/btest/Baseline/doc.broxygen.all_scripts/.stderr index 977e8fc37a..d4746c0865 100644 --- a/testing/btest/Baseline/doc.broxygen.all_scripts/.stderr +++ b/testing/btest/Baseline/doc.broxygen.all_scripts/.stderr @@ -1 +1 @@ -received termination signal +, line 1: received termination signal diff --git a/testing/btest/doc/broxygen/all_scripts.test b/testing/btest/doc/broxygen/all_scripts.test index f4d9237adf..61cead160b 100644 --- a/testing/btest/doc/broxygen/all_scripts.test +++ b/testing/btest/doc/broxygen/all_scripts.test @@ -5,7 +5,7 @@ # listen.bro in order to document it. # @TEST-SERIALIZE: comm -# @TEST-EXEC: bro -X broxygen.config broxygen +# @TEST-EXEC: bro -X broxygen.config broxygen DumpEvents::include=/NOTHING_MATCHES/ # @TEST-EXEC: btest-diff .stdout # @TEST-EXEC: btest-diff .stderr From d5a48f04ac589184eef4a4805aa12902958452a2 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Wed, 4 Dec 2013 13:11:08 -0800 Subject: [PATCH 28/40] Tweak to SMTP script to adjust for new string slicing behaviour. Also updating NEWS with a note. --- NEWS | 3 +++ scripts/base/protocols/smtp/main.bro | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/NEWS b/NEWS index 75c63ca30b..524cac14e0 100644 --- a/NEWS +++ b/NEWS @@ -16,6 +16,9 @@ New Functionality Changed Functionality --------------------- +- string slices now exclude the end index (e.g., "123"[1:2] returns + "2"). Generally, Bro's string slices now behave similar to Python. + - ssl_client_hello() now receives a vector of ciphers, instead of a set, to preserve their order. diff --git a/scripts/base/protocols/smtp/main.bro b/scripts/base/protocols/smtp/main.bro index fb3095c121..d3bd8a97b4 100644 --- a/scripts/base/protocols/smtp/main.bro +++ b/scripts/base/protocols/smtp/main.bro @@ -291,7 +291,7 @@ function describe(rec: Info): string { if ( |rec$subject| > 20 ) { - abbrev_subject = rec$subject[0:20] + "..."; + abbrev_subject = rec$subject[0:21] + "..."; } } From 1732fd77a4006cf6b3fb28d7e0a444f999c383d0 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Thu, 5 Dec 2013 07:53:26 -0800 Subject: [PATCH 29/40] Updating CHANGES and VERSION. --- CHANGES | 17 +++++++++++++++++ VERSION | 2 +- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/CHANGES b/CHANGES index 88f65080c9..580e423b18 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,21 @@ +2.2-48 | 2013-12-04 13:45:47 -0800 + + * Support omission of string slice low/high indices, BIT-1097. + + Omission of the low index defaults to 0: + + s = "12345"; s[:3] == "123" + + Omission of the high index defaults to length of the string: + + s = "12345"; s[3:] == "45" (Jon Siwek) + + * Tweak to SMTP script to adjust for new string slicing behaviour. + (Robin Sommer) + + * Test updates. (Robin Sommer) + 2.2-44 | 2013-12-04 12:41:51 -0800 * Fix string slice notation. Addresses BIT-1097. (Jon Siwek) diff --git a/VERSION b/VERSION index 86c05ec5e4..91c700588c 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.2-44 +2.2-48 From 21df25d429136bbe089ff54d3fddfe4294a46057 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Thu, 5 Dec 2013 11:01:44 -0600 Subject: [PATCH 30/40] Fix build on FreeBSD. basename(3)/dirname(3) const-ness may vary w/ platform. --- src/util.cc | 58 +++++++++++++++++++++++++++++++++++++---------------- src/util.h | 34 +++++++++++++++++-------------- 2 files changed, 60 insertions(+), 32 deletions(-) diff --git a/src/util.cc b/src/util.cc index c526527648..8b38246893 100644 --- a/src/util.cc +++ b/src/util.cc @@ -971,35 +971,59 @@ FILE* open_package(string& path, const string& mode) return 0; } -SafePathOp::SafePathOp(PathOpFn fn, const char* path, bool error_aborts) +void SafePathOp::CheckValid(const char* result, const char* path, + bool error_aborts) { - DoFunc(fn, path ? path : "", error_aborts); - } - -SafePathOp::SafePathOp(PathOpFn fn, const string& path, bool error_aborts) - { - DoFunc(fn, path, error_aborts); - } - -void SafePathOp::DoFunc(PathOpFn fn, const string& path, bool error_aborts) - { - char* tmp = copy_string(path.c_str()); - char* rval = fn(tmp); - - if ( rval ) + if ( result ) { - result = rval; + result = result; error = false; } else { if ( error_aborts ) reporter->InternalError("Path operation failed on %s: %s", - tmp ? tmp : "", strerror(errno)); + path ? path : "", strerror(errno)); else error = true; } + } +SafeDirname::SafeDirname(const char* path, bool error_aborts) + : SafePathOp() + { + DoFunc(path ? path : "", error_aborts); + } + +SafeDirname::SafeDirname(const string& path, bool error_aborts) + : SafePathOp() + { + DoFunc(path, error_aborts); + } + +void SafeDirname::DoFunc(const string& path, bool error_aborts) + { + char* tmp = copy_string(path.c_str()); + CheckValid(dirname(tmp), tmp, error_aborts); + delete [] tmp; + } + +SafeBasename::SafeBasename(const char* path, bool error_aborts) + : SafePathOp() + { + DoFunc(path ? path : "", error_aborts); + } + +SafeBasename::SafeBasename(const string& path, bool error_aborts) + : SafePathOp() + { + DoFunc(path, error_aborts); + } + +void SafeBasename::DoFunc(const string& path, bool error_aborts) + { + char* tmp = copy_string(path.c_str()); + CheckValid(basename(tmp), tmp, error_aborts); delete [] tmp; } diff --git a/src/util.h b/src/util.h index 976965aebf..d86f45880d 100644 --- a/src/util.h +++ b/src/util.h @@ -224,34 +224,38 @@ extern std::string bro_prefixes(); class SafePathOp { public: - typedef char*(*PathOpFn)(char*); - - SafePathOp(PathOpFn fn, const char* path, bool error_aborts = true); - SafePathOp(PathOpFn fn, const std::string& path, bool error_aborts = true); - std::string result; bool error; -private: +protected: + + SafePathOp() + { } + + void CheckValid(const char* result, const char* path, bool error_aborts); - void DoFunc(PathOpFn fn, const std::string& path, bool error_aborts = true); }; class SafeDirname : public SafePathOp { public: - SafeDirname(const char* path, bool error_aborts = true) - : SafePathOp(&dirname, path, error_aborts) { } - SafeDirname(const std::string& path, bool error_aborts = true) - : SafePathOp(&dirname, path, error_aborts) { } + SafeDirname(const char* path, bool error_aborts = true); + SafeDirname(const std::string& path, bool error_aborts = true); + +private: + + void DoFunc(const std::string& path, bool error_aborts = true); }; class SafeBasename : public SafePathOp { public: - SafeBasename(const char* path, bool error_aborts = true) - : SafePathOp(&basename, path, error_aborts) { } - SafeBasename(const std::string& path, bool error_aborts = true) - : SafePathOp(&basename, path, error_aborts) { } + + SafeBasename(const char* path, bool error_aborts = true); + SafeBasename(const std::string& path, bool error_aborts = true); + +private: + + void DoFunc(const std::string& path, bool error_aborts = true); }; std::string implode_string_vector(const std::vector& v, From be05d7b4358d0cc0cb1f5f8e5f04a77222d1cfee Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Thu, 5 Dec 2013 11:15:02 -0600 Subject: [PATCH 31/40] FreeBSD build fix addendum: unintended variable shadowing. --- src/util.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/util.cc b/src/util.cc index 8b38246893..ad55e3f75e 100644 --- a/src/util.cc +++ b/src/util.cc @@ -971,12 +971,12 @@ FILE* open_package(string& path, const string& mode) return 0; } -void SafePathOp::CheckValid(const char* result, const char* path, +void SafePathOp::CheckValid(const char* op_result, const char* path, bool error_aborts) { - if ( result ) + if ( op_result ) { - result = result; + result = op_result; error = false; } else From 504d22b91be163ac23b595079762445cf9f5e866 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Thu, 5 Dec 2013 10:53:12 -0800 Subject: [PATCH 32/40] fix memory leak in input framework. If the input framework was used to read event streams and those streams contained records with more than one field, not all elements of the threading Values were cleaned up. The reason for this is, that the SendEventStreamEvent function returned the number of record elements in the outmost record instead of the number of unrolled elements in the whole vector. This number is later used to determine how many objects to delete. Also - add a whole bunch of leak checks for the input framework that would have caught that and should cover quite a number of use-cases. --- src/input/Manager.cc | 2 +- testing/btest/core/leaks/input-basic.bro | 67 ++++++++ .../btest/core/leaks/input-optional-event.bro | 65 ++++++++ .../btest/core/leaks/input-optional-table.bro | 68 ++++++++ testing/btest/core/leaks/input-raw.bro | 68 ++++++++ testing/btest/core/leaks/input-reread.bro | 156 ++++++++++++++++++ testing/btest/core/leaks/input-sqlite.bro | 104 ++++++++++++ 7 files changed, 529 insertions(+), 1 deletion(-) create mode 100644 testing/btest/core/leaks/input-basic.bro create mode 100644 testing/btest/core/leaks/input-optional-event.bro create mode 100644 testing/btest/core/leaks/input-optional-table.bro create mode 100644 testing/btest/core/leaks/input-raw.bro create mode 100644 testing/btest/core/leaks/input-reread.bro create mode 100644 testing/btest/core/leaks/input-sqlite.bro diff --git a/src/input/Manager.cc b/src/input/Manager.cc index ee1e1ef522..cd4976aa64 100644 --- a/src/input/Manager.cc +++ b/src/input/Manager.cc @@ -1468,7 +1468,7 @@ int Manager::SendEventStreamEvent(Stream* i, EnumVal* type, const Value* const * SendEvent(stream->event, out_vals); - return stream->fields->NumFields(); + return stream->num_fields; } int Manager::PutTable(Stream* i, const Value* const *vals) diff --git a/testing/btest/core/leaks/input-basic.bro b/testing/btest/core/leaks/input-basic.bro new file mode 100644 index 0000000000..5a58e0465d --- /dev/null +++ b/testing/btest/core/leaks/input-basic.bro @@ -0,0 +1,67 @@ +# Needs perftools support. +# +# @TEST-GROUP: leaks +# +# @TEST-REQUIRES: bro --help 2>&1 | grep -q mem-leaks +# +# @TEST-EXEC: HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local btest-bg-run bro bro -m -b %INPUT +# @TEST-EXEC: btest-bg-wait 15 + +redef exit_only_after_terminate = T; + +@TEST-START-FILE input.log +#separator \x09 +#path ssh +#fields b i e c p sn a d t iv s sc ss se vc ve ns +#types bool int enum count port subnet addr double time interval string table table table vector vector string +T -42 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY 4242 +@TEST-END-FILE + +@load base/protocols/ssh + +global outfile: file; + +redef InputAscii::empty_field = "EMPTY"; + +module A; + +type Idx: record { + i: int; +}; + +type Val: record { + b: bool; + e: Log::ID; + c: count; + p: port; + sn: subnet; + a: addr; + d: double; + t: time; + iv: interval; + s: string; + ns: string; + sc: set[count]; + ss: set[string]; + se: set[string]; + vc: vector of int; + ve: vector of int; +}; + +global servers: table[int] of Val = table(); + +event bro_init() + { + outfile = open("../out"); + # first read in the old stuff into the table... + Input::add_table([$source="../input.log", $name="ssh", $idx=Idx, $val=Val, $destination=servers]); + } + +event Input::end_of_data(name: string, source:string) + { + print outfile, servers; + print outfile, to_count(servers[-42]$ns); # try to actually use a string. If null-termination is wrong this will fail. + Input::remove("ssh"); + close(outfile); + terminate(); + } diff --git a/testing/btest/core/leaks/input-optional-event.bro b/testing/btest/core/leaks/input-optional-event.bro new file mode 100644 index 0000000000..72e62bb285 --- /dev/null +++ b/testing/btest/core/leaks/input-optional-event.bro @@ -0,0 +1,65 @@ +# Needs perftools support. +# +# @TEST-GROUP: leaks +# +# @TEST-REQUIRES: bro --help 2>&1 | grep -q mem-leaks +# +# @TEST-EXEC: HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local btest-bg-run bro bro -m -b %INPUT +# @TEST-EXEC: btest-bg-wait 15 + +@TEST-START-FILE input.log +#separator \x09 +#path ssh +#fields i b r.a r.b r.c +#types int bool string string string +1 T a b c +2 T a b c +3 F ba bb bc +4 T bb bd - +5 F a b c +6 T a b c +7 T a b c +@TEST-END-FILE + +redef exit_only_after_terminate = T; + +global outfile: file; + +redef InputAscii::empty_field = "EMPTY"; + +module A; + +type Sub: record { + a: string; + aa: string &optional; + b : string; + bb: string &optional; + c: string &optional; + d: string &optional; +}; + +type Val: record { + i: int; + b: bool; + notb: bool &optional; + r: Sub; +}; + +event servers(desc: Input::EventDescription, tpe: Input::Event, item: Val) + { + print outfile, item; + } + +event bro_init() + { + outfile = open("../out"); + # first read in the old stuff into the table... + Input::add_event([$source="../input.log", $name="input", $fields=Val, $ev=servers]); + } + +event Input::end_of_data(name: string, source: string) + { + Input::remove("input"); + close(outfile); + terminate(); + } diff --git a/testing/btest/core/leaks/input-optional-table.bro b/testing/btest/core/leaks/input-optional-table.bro new file mode 100644 index 0000000000..c15589a948 --- /dev/null +++ b/testing/btest/core/leaks/input-optional-table.bro @@ -0,0 +1,68 @@ +# Needs perftools support. +# +# @TEST-GROUP: leaks +# +# @TEST-REQUIRES: bro --help 2>&1 | grep -q mem-leaks +# +# @TEST-EXEC: HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local btest-bg-run bro bro -m -b %INPUT +# @TEST-EXEC: btest-bg-wait 15 + +@TEST-START-FILE input.log +#separator \x09 +#path ssh +#fields i b r.a r.b r.c +#types int bool string string string +1 T a b c +2 T a b c +3 F ba bb bc +4 T bb bd - +5 T a b c +6 F a b c +7 T a b c +@TEST-END-FILE + +redef exit_only_after_terminate = T; + +global outfile: file; + +redef InputAscii::empty_field = "EMPTY"; + +module A; + +type Sub: record { + a: string; + aa: string &optional; + b : string; + bb: string &optional; + c: string &optional; + d: string &optional; +}; + +type Idx: record { + i: int; +}; + +type Val: record { + b: bool; + notb: bool &optional; + r: Sub; +}; + +global servers: table[int] of Val = table(); + +event bro_init() + { + outfile = open("../out"); + # first read in the old stuff into the table... + Input::add_table([$source="../input.log", $name="input", $idx=Idx, $val=Val, $destination=servers, + $pred(typ: Input::Event, left: Idx, right: Val) = { right$notb = !right$b; return T; } + ]); + } + +event Input::end_of_data(name: string, source: string) + { + print outfile, servers; + Input::remove("input"); + close(outfile); + terminate(); + } diff --git a/testing/btest/core/leaks/input-raw.bro b/testing/btest/core/leaks/input-raw.bro new file mode 100644 index 0000000000..7329a7c70f --- /dev/null +++ b/testing/btest/core/leaks/input-raw.bro @@ -0,0 +1,68 @@ +# Needs perftools support. +# +# @TEST-GROUP: leaks +# +# @TEST-REQUIRES: bro --help 2>&1 | grep -q mem-leaks +# +# @TEST-EXEC: cp input1.log input.log +# @TEST-EXEC: HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local btest-bg-run bro bro -m -b %INPUT +# @TEST-EXEC: sleep 5 +# @TEST-EXEC: cat input2.log >> input.log +# @TEST-EXEC: sleep 5 +# @TEST-EXEC: cat input3.log >> input.log +# @TEST-EXEC: btest-bg-wait 10 + +redef exit_only_after_terminate = T; + +@TEST-START-FILE input1.log +sdfkh:KH;fdkncv;ISEUp34:Fkdj;YVpIODhfDF +@TEST-END-FILE + +@TEST-START-FILE input2.log +DSF"DFKJ"SDFKLh304yrsdkfj@#(*U$34jfDJup3UF +q3r3057fdf +@TEST-END-FILE + +@TEST-START-FILE input3.log +sdfs\d + +dfsdf +sdf +3rw43wRRERLlL#RWERERERE. +@TEST-END-FILE + +@load base/frameworks/communication # let network-time run + +module A; + +type Val: record { + s: string; +}; + +global try: count; +global outfile: file; + +event line(description: Input::EventDescription, tpe: Input::Event, s: string) + { + print outfile, description$name; + print outfile, tpe; + print outfile, s; + + try = try + 1; + if ( try == 16 ) + { + print outfile, "done"; + close(outfile); + Input::remove("input"); + Input::remove("tail"); + terminate(); + } + } + +event bro_init() + { + outfile = open("../out"); + try = 0; + Input::add_event([$source="../input.log", $reader=Input::READER_RAW, $mode=Input::STREAM, $name="input", $fields=Val, $ev=line, $want_record=F]); + Input::add_event([$source="tail -f ../input.log |", $reader=Input::READER_RAW, $mode=Input::STREAM, $name="tail", $fields=Val, $ev=line, $want_record=F]); + } diff --git a/testing/btest/core/leaks/input-reread.bro b/testing/btest/core/leaks/input-reread.bro new file mode 100644 index 0000000000..fa37f04ede --- /dev/null +++ b/testing/btest/core/leaks/input-reread.bro @@ -0,0 +1,156 @@ +# Needs perftools support. +# +# @TEST-GROUP: leaks +# +# @TEST-REQUIRES: bro --help 2>&1 | grep -q mem-leaks +# +# @TEST-EXEC: cp input1.log input.log +# @TEST-EXEC: HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local btest-bg-run bro bro -m -b %INPUT +# @TEST-EXEC: sleep 5 +# @TEST-EXEC: cp input2.log input.log +# @TEST-EXEC: sleep 5 +# @TEST-EXEC: cp input3.log input.log +# @TEST-EXEC: sleep 5 +# @TEST-EXEC: cp input4.log input.log +# @TEST-EXEC: sleep 5 +# @TEST-EXEC: cp input5.log input.log +# @TEST-EXEC: btest-bg-wait 15 + +@TEST-START-FILE input1.log +#separator \x09 +#path ssh +#fields b i e c p sn a d t iv s sc ss se vc ve r.a r.b +#types bool int enum count port subnet addr double time interval string table table table vector vector string string +T -42 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY fortytwo - +@TEST-END-FILE +@TEST-START-FILE input2.log +#separator \x09 +#path ssh +#fields b i e c p sn a d t iv s sc ss se vc ve r.a r.b +#types bool int enum count port subnet addr double time interval string table table table vector vector string string +T -42 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY fortytwo - +T -43 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY fortythree 43 +@TEST-END-FILE +@TEST-START-FILE input3.log +#separator \x09 +#path ssh +#fields b i e c p sn a d t iv s sc ss se vc ve r.a r.b +#types bool int enum count port subnet addr double time interval string table table table vector vector string string +T -42 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY fortytwo - +F -43 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY fortythree 43 +@TEST-END-FILE +@TEST-START-FILE input4.log +#separator \x09 +#path ssh +#fields b i e c p sn a d t iv s sc ss se vc ve r.a r.b r.d +#types bool int enum count port subnet addr double time interval string table table table vector vector string string string +T -42 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY fortytwo - - +F -43 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY fortythree 43 - +F -44 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY fortyfour - - +F -45 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY fourtyfive - - +F -46 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY fourtysix - - +F -47 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY fourtyseven - - +F -48 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY fourtyeight 48 f +@TEST-END-FILE +@TEST-START-FILE input5.log +#separator \x09 +#path ssh +#fields b i e c p sn a d t iv s sc ss se vc ve r.a r.b r.d +#types bool int enum count port subnet addr double time interval string table table table vector vector string string string +F -48 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY fourtyeight 48 f +@TEST-END-FILE + +@load base/protocols/ssh +@load base/frameworks/communication # let network-time run + +redef exit_only_after_terminate = T; +redef InputAscii::empty_field = "EMPTY"; + +module A; + +type Sub: record { + a: string; + b: string &optional; + c: string &optional; + d: string &optional; +}; + +type Idx: record { + i: int; +}; + +type Val: record { + b: bool; + e: Log::ID; + c: count; + p: port; + sn: subnet; + a: addr; + d: double; + t: time; + iv: interval; + s: string; + sc: set[count]; + ss: set[string]; + se: set[string]; + vc: vector of int; + ve: vector of int; + r: Sub; +}; + +global servers: table[int] of Val = table(); + +global outfile: file; + +global try: count; + +event servers_ev(description: Input::EventDescription, tpe: Input::Event, item: Val) + { + print outfile, "============EVENT EVENT============"; + print outfile, item; + } + +event line(description: Input::TableDescription, tpe: Input::Event, left: Idx, right: Val) + { + print outfile, "============TABLE EVENT============"; + print outfile, "Left"; + print outfile, left; + print outfile, "Right"; + print outfile, right; + } + +event bro_init() + { + outfile = open("../out"); + try = 0; + # first read in the old stuff into the table... + Input::add_table([$source="../input.log", $mode=Input::REREAD, $name="ssh", $idx=Idx, $val=Val, $destination=servers, $ev=line, + $pred(typ: Input::Event, left: Idx, right: Val) = { + print outfile, "============PREDICATE============"; + print outfile, left; + print outfile, right; + return T; + } + ]); + Input::add_event([$source="../input.log", $mode=Input::REREAD, $name="sshevent", $fields=Val, $ev=servers_ev]); + } + + +event Input::end_of_data(name: string, source: string) + { + if ( name == "ssh" ) { + print outfile, "==========SERVERS============"; + print outfile, servers; + } else { + print outfile, "==========END OF EVENTS EVENTS==========="; + } + + try = try + 1; + if ( try == 10 ) + { + print outfile, "done"; + close(outfile); + Input::remove("input"); + terminate(); + } + } diff --git a/testing/btest/core/leaks/input-sqlite.bro b/testing/btest/core/leaks/input-sqlite.bro new file mode 100644 index 0000000000..0de1069b5e --- /dev/null +++ b/testing/btest/core/leaks/input-sqlite.bro @@ -0,0 +1,104 @@ +# Needs perftools support. +# +# @TEST-GROUP: leaks +# +# @TEST-REQUIRES: bro --help 2>&1 | grep -q mem-leaks +# @TEST-REQUIRES: which sqlite3 +# +# @TEST-EXEC: cat conn.sql | sqlite3 conn.sqlite +# @TEST-EXEC: HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local btest-bg-run bro bro -m -b %INPUT +# @TEST-EXEC: btest-bg-wait 20 + +@TEST-START-FILE conn.sql +PRAGMA foreign_keys=OFF; +BEGIN TRANSACTION; +CREATE TABLE conn ( +'ts' double precision, +'uid' text, +'id.orig_h' text, +'id.orig_p' integer, +'id.resp_h' text, +'id.resp_p' integer, +'proto' text, +'service' text, +'duration' double precision, +'orig_bytes' integer, +'resp_bytes' integer, +'conn_state' text, +'local_orig' boolean, +'missed_bytes' integer, +'history' text, +'orig_pkts' integer, +'orig_ip_bytes' integer, +'resp_pkts' integer, +'resp_ip_bytes' integer, +'tunnel_parents' text +); +INSERT INTO "conn" VALUES(1.30047516709653496744e+09,'dnGM1AdIVyh','141.142.220.202',5353,'224.0.0.251',5353,'udp','dns',NULL,NULL,NULL,'S0',NULL,0,'D',1,73,0,0,'(empty)'); +INSERT INTO "conn" VALUES(1.30047516709701204296e+09,'fv9q7WjEgp1','fe80::217:f2ff:fed7:cf65',5353,'ff02::fb',5353,'udp',NULL,NULL,NULL,NULL,'S0',NULL,0,'D',1,199,0,0,'(empty)'); +INSERT INTO "conn" VALUES(1.30047516709981608392e+09,'0Ox0H56yl88','141.142.220.50',5353,'224.0.0.251',5353,'udp',NULL,NULL,NULL,NULL,'S0',NULL,0,'D',1,179,0,0,'(empty)'); +INSERT INTO "conn" VALUES(1.30047516885389900212e+09,'rvmSc7rDQub','141.142.220.118',43927,'141.142.2.2',53,'udp','dns',4.351139068603515625e-04,38,89,'SF',NULL,0,'Dd',1,66,1,117,'(empty)'); +INSERT INTO "conn" VALUES(1.30047516885437798497e+09,'ogkztouSArh','141.142.220.118',37676,'141.142.2.2',53,'udp','dns',4.20093536376953125e-04,52,99,'SF',NULL,0,'Dd',1,80,1,127,'(empty)'); +INSERT INTO "conn" VALUES(1.30047516885483694076e+09,'0UIDdXFt7Tb','141.142.220.118',40526,'141.142.2.2',53,'udp','dns',3.9196014404296875e-04,38,183,'SF',NULL,0,'Dd',1,66,1,211,'(empty)'); +INSERT INTO "conn" VALUES(1.30047516885795593258e+09,'WqFYV51UIq7','141.142.220.118',32902,'141.142.2.2',53,'udp','dns',3.17096710205078125e-04,38,89,'SF',NULL,0,'Dd',1,66,1,117,'(empty)'); +INSERT INTO "conn" VALUES(1.30047516885830593104e+09,'ylcqZpbz6K2','141.142.220.118',59816,'141.142.2.2',53,'udp','dns',3.430843353271484375e-04,52,99,'SF',NULL,0,'Dd',1,80,1,127,'(empty)'); +INSERT INTO "conn" VALUES(1.30047516885871291159e+09,'blhldTzA7Y6','141.142.220.118',59714,'141.142.2.2',53,'udp','dns',3.750324249267578125e-04,38,183,'SF',NULL,0,'Dd',1,66,1,211,'(empty)'); +INSERT INTO "conn" VALUES(1.30047516889164400098e+09,'Sc34cGJo3Kg','141.142.220.118',58206,'141.142.2.2',53,'udp','dns',3.39031219482421875e-04,38,89,'SF',NULL,0,'Dd',1,66,1,117,'(empty)'); +INSERT INTO "conn" VALUES(1.30047516889203691487e+09,'RzvFrfXSRfk','141.142.220.118',38911,'141.142.2.2',53,'udp','dns',3.349781036376953125e-04,52,99,'SF',NULL,0,'Dd',1,80,1,127,'(empty)'); +INSERT INTO "conn" VALUES(1.30047516889241409298e+09,'GaaFI58mpbe','141.142.220.118',59746,'141.142.2.2',53,'udp','dns',4.208087921142578125e-04,38,183,'SF',NULL,0,'Dd',1,66,1,211,'(empty)'); +INSERT INTO "conn" VALUES(1.30047516889398789407e+09,'tr7M6tvAIQa','141.142.220.118',45000,'141.142.2.2',53,'udp','dns',3.840923309326171875e-04,38,89,'SF',NULL,0,'Dd',1,66,1,117,'(empty)'); +INSERT INTO "conn" VALUES(1.30047516889442205426e+09,'gV0TcSc2pb4','141.142.220.118',48479,'141.142.2.2',53,'udp','dns',3.168582916259765625e-04,52,99,'SF',NULL,0,'Dd',1,80,1,127,'(empty)'); +INSERT INTO "conn" VALUES(1.30047516889478707315e+09,'MOG0z4PYOhk','141.142.220.118',48128,'141.142.2.2',53,'udp','dns',4.22954559326171875e-04,38,183,'SF',NULL,0,'Dd',1,66,1,211,'(empty)'); +INSERT INTO "conn" VALUES(1.30047516890174889565e+09,'PlehgEduUyj','141.142.220.118',56056,'141.142.2.2',53,'udp','dns',4.022121429443359375e-04,36,131,'SF',NULL,0,'Dd',1,64,1,159,'(empty)'); +INSERT INTO "conn" VALUES(1.30047516890219497676e+09,'4eZgk09f2Re','141.142.220.118',55092,'141.142.2.2',53,'udp','dns',3.740787506103515625e-04,36,198,'SF',NULL,0,'Dd',1,64,1,226,'(empty)'); +INSERT INTO "conn" VALUES(1.30047516989943790432e+09,'3xwJPc7mQ9a','141.142.220.44',5353,'224.0.0.251',5353,'udp','dns',NULL,NULL,NULL,'S0',NULL,0,'D',1,85,0,0,'(empty)'); +INSERT INTO "conn" VALUES(1.30047517086238408089e+09,'yxTcvvTKWQ4','141.142.220.226',137,'141.142.220.255',137,'udp','dns',2.61301684379577636718e+00,350,0,'S0',NULL,0,'D',7,546,0,0,'(empty)'); +INSERT INTO "conn" VALUES(1.30047517167537188525e+09,'8bLW3XNfhCj','fe80::3074:17d5:2052:c324',65373,'ff02::1:3',5355,'udp','dns',1.00096225738525390625e-01,66,0,'S0',NULL,0,'D',2,162,0,0,'(empty)'); +INSERT INTO "conn" VALUES(1.30047517167708110807e+09,'rqjhiiRPjEe','141.142.220.226',55131,'224.0.0.252',5355,'udp','dns',1.00020885467529296875e-01,66,0,'S0',NULL,0,'D',2,122,0,0,'(empty)'); +INSERT INTO "conn" VALUES(1.30047517311674904827e+09,'hTPyfL3QSGa','fe80::3074:17d5:2052:c324',54213,'ff02::1:3',5355,'udp','dns',9.980106353759765625e-02,66,0,'S0',NULL,0,'D',2,162,0,0,'(empty)'); +INSERT INTO "conn" VALUES(1.30047517311736202235e+09,'EruUQ9AJRj4','141.142.220.226',55671,'224.0.0.252',5355,'udp','dns',9.98489856719970703125e-02,66,0,'S0',NULL,0,'D',2,122,0,0,'(empty)'); +INSERT INTO "conn" VALUES(1.30047517315367889406e+09,'sw1bKJOMjuk','141.142.220.238',56641,'141.142.220.255',137,'udp','dns',NULL,NULL,NULL,'S0',NULL,0,'D',1,78,0,0,'(empty)'); +INSERT INTO "conn" VALUES(1.30047516872400689127e+09,'NPHCuyWykE7','141.142.220.118',48649,'208.80.152.118',80,'tcp','http',1.19904994964599609375e-01,525,232,'S1',NULL,0,'ShADad',4,741,3,396,'(empty)'); +INSERT INTO "conn" VALUES(1.30047516889293599126e+09,'VapPqRhPgJ4','141.142.220.118',50000,'208.80.152.3',80,'tcp','http',2.29603052139282226562e-01,1148,734,'S1',NULL,0,'ShADad',6,1468,4,950,'(empty)'); +INSERT INTO "conn" VALUES(1.30047516885916304588e+09,'3607hh8C3bc','141.142.220.118',49998,'208.80.152.3',80,'tcp','http',2.15893030166625976562e-01,1130,734,'S1',NULL,0,'ShADad',6,1450,4,950,'(empty)'); +INSERT INTO "conn" VALUES(1.30047516885530495647e+09,'tgYMrIvzDSg','141.142.220.118',49996,'208.80.152.3',80,'tcp','http',2.1850109100341796875e-01,1171,733,'S1',NULL,0,'ShADad',6,1491,4,949,'(empty)'); +INSERT INTO "conn" VALUES(1.30047516889526700977e+09,'xQsjPwNBrXd','141.142.220.118',50001,'208.80.152.3',80,'tcp','http',2.27283954620361328125e-01,1178,734,'S1',NULL,0,'ShADad',6,1498,4,950,'(empty)'); +INSERT INTO "conn" VALUES(1.30047516890263509747e+09,'Ap3GzMI1vM9','141.142.220.118',35642,'208.80.152.2',80,'tcp','http',1.200408935546875e-01,534,412,'S1',NULL,0,'ShADad',4,750,3,576,'(empty)'); +INSERT INTO "conn" VALUES(1300475168.85533,'FTVcgrmNy52','141.142.220.118',49997,'208.80.152.3',80,'tcp','http',2.19720125198364257812e-01,1125,734,'S1',NULL,0,'ShADad',6,1445,4,950,'(empty)'); +INSERT INTO "conn" VALUES(1.30047516978033089643e+09,'1xFx4PGdeq5','141.142.220.235',6705,'173.192.163.128',80,'tcp',NULL,NULL,NULL,NULL,'OTH',NULL,0,'h',0,0,1,48,'(empty)'); +INSERT INTO "conn" VALUES(1.3004751686520030498e+09,'WIG1ud65z22','141.142.220.118',35634,'208.80.152.2',80,'tcp',NULL,6.1328887939453125e-02,463,350,'OTH',NULL,0,'DdA',2,567,1,402,'(empty)'); +INSERT INTO "conn" VALUES(1.3004751688929131031e+09,'o2gAkl4V7sa','141.142.220.118',49999,'208.80.152.3',80,'tcp','http',2.20960855484008789062e-01,1137,733,'S1',NULL,0,'ShADad',6,1457,4,949,'(empty)'); +COMMIT; +@TEST-END-FILE + +@load base/protocols/conn + +redef exit_only_after_terminate = T; +redef Input::accept_unsupported_types = T; + +global outfile: file; + +module A; + +event line(description: Input::EventDescription, tpe: Input::Event, r: Conn::Info) + { + print outfile, r; + print outfile, |r$tunnel_parents|; # to make sure I got empty right + } + +event bro_init() + { + local config_strings: table[string] of string = { + ["query"] = "select * from conn;", + }; + + outfile = open("../out"); + Input::add_event([$source="../conn", $name="conn", $fields=Conn::Info, $ev=line, $want_record=T, $reader=Input::READER_SQLITE, $config=config_strings]); + } + +event Input::end_of_data(name: string, source:string) + { + print outfile, "End of data"; + close(outfile); + terminate(); + } From d3e908af3b8c664adcaf2cad294155a3b6f7e77e Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Thu, 5 Dec 2013 12:56:05 -0600 Subject: [PATCH 33/40] Fix unlikely null ptr deref in broxygen::Manager. --- src/broxygen/Manager.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/broxygen/Manager.cc b/src/broxygen/Manager.cc index aa097f9ee0..405a1019f9 100644 --- a/src/broxygen/Manager.cc +++ b/src/broxygen/Manager.cc @@ -35,7 +35,8 @@ Manager::Manager(const string& arg_config, const string& bro_command) if ( getenv("BRO_DISABLE_BROXYGEN") ) disabled = true; - string path_to_bro = find_file(bro_command, getenv("PATH")); + const char* path = getenv("PATH"); + string path_to_bro = path ? find_file(bro_command, path): ""; struct stat s; if ( path_to_bro.empty() || stat(path_to_bro.c_str(), &s) < 0 ) From 92d2fdd4a6551b05cd678e8ff4e7055514c579b3 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Thu, 5 Dec 2013 13:22:50 -0600 Subject: [PATCH 34/40] Close signature files after done parsing. --- src/RuleMatcher.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/RuleMatcher.cc b/src/RuleMatcher.cc index 6cb93e7ef6..c78e65ea18 100644 --- a/src/RuleMatcher.cc +++ b/src/RuleMatcher.cc @@ -227,6 +227,7 @@ bool RuleMatcher::ReadFiles(const name_list& files) for ( int i = 0; i < files.length(); ++i ) { rules_in = open_file(find_file(files[i], bro_path(), "sig")); + if ( ! rules_in ) { reporter->Error("Can't open signature file %s", files[i]); @@ -236,6 +237,7 @@ bool RuleMatcher::ReadFiles(const name_list& files) rules_line_number = 0; current_rule_file = files[i]; rules_parse(); + fclose(rules_in); } if ( parse_error ) From 6d782e6834f91d5151c4f1312d3322ac2ade595f Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Thu, 5 Dec 2013 15:05:11 -0500 Subject: [PATCH 35/40] Updated software framework to support parsing IE11 user-agent strings. --- scripts/base/frameworks/software/main.bro | 4 +++- .../scripts.base.frameworks.software.version-parsing/output | 2 ++ .../scripts/base/frameworks/software/version-parsing.bro | 6 ++++++ 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/scripts/base/frameworks/software/main.bro b/scripts/base/frameworks/software/main.bro index 5080c9f888..c8f413a8f2 100644 --- a/scripts/base/frameworks/software/main.bro +++ b/scripts/base/frameworks/software/main.bro @@ -209,7 +209,7 @@ function parse_mozilla(unparsed_version: string): Description if ( 2 in parts ) v = parse(parts[2])$version; } - else if ( / MSIE / in unparsed_version ) + else if ( / MSIE |Trident\// in unparsed_version ) { software_name = "MSIE"; if ( /Trident\/4\.0/ in unparsed_version ) @@ -218,6 +218,8 @@ function parse_mozilla(unparsed_version: string): Description v = [$major=9,$minor=0]; else if ( /Trident\/6\.0/ in unparsed_version ) v = [$major=10,$minor=0]; + else if ( /Trident\/7\.0/ in unparsed_version ) + v = [$major=11,$minor=0]; else { parts = split_all(unparsed_version, /MSIE [0-9]{1,2}\.*[0-9]*b?[0-9]*/); diff --git a/testing/btest/Baseline/scripts.base.frameworks.software.version-parsing/output b/testing/btest/Baseline/scripts.base.frameworks.software.version-parsing/output index f172268aa6..77a9f59510 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.software.version-parsing/output +++ b/testing/btest/Baseline/scripts.base.frameworks.software.version-parsing/output @@ -2,6 +2,7 @@ success on: Apache/1.3.19 (Unix) success on: Python-urllib/3.1 success on: Apache success on: Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; InfoPath.3; Creative AutoUpdate v1.40.02) +success on: Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.3; Trident/7.0; .NET4.0E; .NET4.0C) success on: Java/1.6.0_13 success on: Wget/1.11.4 (Red Hat modified) success on: curl/7.15.1 (i486-pc-linux-gnu) libcurl/7.15.1 OpenSSL/0.9.8a zlib/1.2.3 libidn/0.5.18 @@ -38,6 +39,7 @@ success on: Mozilla/5.0 (Linux; U; Android 2.3.3; zh-tw; HTC Pyramid Build/GRI40 success on: wu-2.6.2(1) success on: Opera/9.80 (J2ME/MIDP; Opera Mini/5.0.18741/18.794; U; en) Presto/2.4.15 success on: Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; Tablet PC 2.0; InfoPath.2; InfoPath.3) +success on: Mozilla/5.0 (Windows NT 6.3; Trident/7.0; rv:11.0) like Gecko success on: Flash/10,2,153,1 success on: CacheFlyServe v26b success on: Mozilla/5.0 (Windows; U; Windows NT 5.1; de; rv:1.9.2.11) Gecko/20101013 Lightning/1.0b2 Thunderbird/3.1.5 diff --git a/testing/btest/scripts/base/frameworks/software/version-parsing.bro b/testing/btest/scripts/base/frameworks/software/version-parsing.bro index 2b406f22b8..19a803cafe 100644 --- a/testing/btest/scripts/base/frameworks/software/version-parsing.bro +++ b/testing/btest/scripts/base/frameworks/software/version-parsing.bro @@ -91,6 +91,12 @@ global matched_software: table[string] of Software::Description = { [$name="MSIE", $version=[$major=9,$minor=0], $unparsed_version=""], ["Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0)"] = [$name="MSIE", $version=[$major=10,$minor=0], $unparsed_version=""], + # IE 11 normal mode. + ["Mozilla/5.0 (Windows NT 6.3; Trident/7.0; rv:11.0) like Gecko"] = + [$name="MSIE", $version=[$major=11,$minor=0], $unparsed_version=""], + # IE 11 compatibility mode + ["Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.3; Trident/7.0; .NET4.0E; .NET4.0C)"] = + [$name="MSIE", $version=[$major=11,$minor=0], $unparsed_version=""], ["The Bat! (3.0.1 RC3) Professional"] = [$name="The Bat!", $version=[$major=3,$minor=0,$minor2=1,$addl="RC3"], $unparsed_version=""], # This is an FTP client (found with CLNT command) From 54ebe9845876970717360e5fd949e34a7697d2f6 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Thu, 5 Dec 2013 14:23:32 -0600 Subject: [PATCH 36/40] Fix table &default reference counting for record ctor expressions. --- src/Val.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Val.cc b/src/Val.cc index dbd4863c67..e072914afb 100644 --- a/src/Val.cc +++ b/src/Val.cc @@ -1741,7 +1741,8 @@ Val* TableVal::Default(Val* index) record_promotion_compatible(dtype->AsRecordType(), ytype->AsRecordType()) ) { - Expr* coerce = new RecordCoerceExpr(def_attr->AttrExpr(), ytype->AsRecordType()); + Expr* coerce = new RecordCoerceExpr(def_attr->AttrExpr()->Ref(), + ytype->AsRecordType()); def_val = coerce->Eval(0); Unref(coerce); } From 3c59aa9459795ea77e56e6890a7f8ff38cfe5146 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Thu, 5 Dec 2013 15:04:28 -0800 Subject: [PATCH 37/40] Several fixes for input manager error handling. - First: Due to architectural constraints, it is very hard for the input framework to handle optional records. For an optional record, either the whole record has to be missing, or all non-optional elements of the record have to be defined. This information is not available to input readers after the records have been unrolled into the threading types. Behavior so far was to treat optional records like they are non-optional, without warning. The patch changes this behavior to emit an error on stream- creation (during type-checking) and refusing to open the file. I think this is a better idea - the behavior so far was undocumented and unintuitive. - Second: For table and event streams, reader backend creation was done very early, before actually checking if all arguments are valid. Initialization is moved after the checks now - this makes a number of delete statements unnecessary. Also - I suspect threads of failed input reader instances were not deleted until shutdown - Third: Add a couple more consistency checks, e.g. checking if the destination value of a table has the same type as we need. We did not check everything in all instances, instead we just assigned the things without caring (which works, but is not really desirable). This change also exposed a few bugs in other testcases where table definitions were wrong (did not respect $want_record) - Fourth: Improve error messages and write testcases for all error messages (I think). --- src/input/Manager.cc | 190 +++++++++++------- .../.stderr | 36 ++++ .../scripts.base.frameworks.input.errors/out | 2 + .../scripts/base/frameworks/input/errors.bro | 161 +++++++++++++++ .../frameworks/input/onecolumn-norecord.bro | 2 +- .../frameworks/input/predicate-stream.bro | 2 +- .../scripts/base/frameworks/input/repeat.bro | 2 +- .../base/frameworks/input/tableevent.bro | 4 +- 8 files changed, 320 insertions(+), 79 deletions(-) create mode 100644 testing/btest/Baseline/scripts.base.frameworks.input.errors/.stderr create mode 100644 testing/btest/Baseline/scripts.base.frameworks.input.errors/out create mode 100644 testing/btest/scripts/base/frameworks/input/errors.bro diff --git a/src/input/Manager.cc b/src/input/Manager.cc index ee1e1ef522..afc0b74fb2 100644 --- a/src/input/Manager.cc +++ b/src/input/Manager.cc @@ -391,18 +391,11 @@ bool Manager::CreateEventStream(RecordVal* fval) { reporter->Error("EventDescription argument not of right type"); return false; - } - - EventStream* stream = new EventStream(); - { - bool res = CreateStream(stream, fval); - if ( res == false ) - { - delete stream; - return false; - } - } + } + Val* name_val = fval->Lookup("name", true); + string stream_name = name_val->AsString()->CheckString(); + Unref(name_val); RecordType *fields = fval->Lookup("fields", true)->AsType()->AsTypeType()->Type()->AsRecordType(); @@ -418,8 +411,7 @@ bool Manager::CreateEventStream(RecordVal* fval) if ( etype->Flavor() != FUNC_FLAVOR_EVENT ) { - reporter->Error("stream event is a function, not an event"); - delete stream; + reporter->Error("Input stream %s: Stream event is a function, not an event", stream_name.c_str()); return false; } @@ -427,22 +419,19 @@ bool Manager::CreateEventStream(RecordVal* fval) if ( args->length() < 2 ) { - reporter->Error("event takes not enough arguments"); - delete stream; + reporter->Error("Input stream %s: Event does not take enough arguments", stream_name.c_str()); return false; } if ( ! same_type((*args)[1], BifType::Enum::Input::Event, 0) ) { - reporter->Error("events second attribute must be of type Input::Event"); - delete stream; + reporter->Error("Input stream %s: Event's second attribute must be of type Input::Event", stream_name.c_str()); return false; } if ( ! same_type((*args)[0], BifType::Record::Input::EventDescription, 0) ) { - reporter->Error("events first attribute must be of type Input::EventDescription"); - delete stream; + reporter->Error("Input stream %s: Event's first attribute must be of type Input::EventDescription", stream_name.c_str()); return false; } @@ -450,8 +439,7 @@ bool Manager::CreateEventStream(RecordVal* fval) { if ( args->length() != fields->NumFields() + 2 ) { - reporter->Error("event has wrong number of arguments"); - delete stream; + reporter->Error("Input stream %s: Event has wrong number of arguments", stream_name.c_str()); return false; } @@ -459,8 +447,15 @@ bool Manager::CreateEventStream(RecordVal* fval) { if ( !same_type((*args)[i+2], fields->FieldType(i) ) ) { - reporter->Error("Incompatible type for event"); - delete stream; + ODesc desc1; + ODesc desc2; + (*args)[i+2]->Describe(&desc1); + fields->FieldType(i)->Describe(&desc2); + reporter->Error("Input stream %s: Incompatible type for event in field %d. Need type '%s':%s, got '%s':%s", + stream_name.c_str(), i+3, + type_name(fields->FieldType(i)->Tag()), desc2.Description(), + type_name((*args)[i+2]->Tag()), desc1.Description() + ); return false; } } @@ -471,8 +466,7 @@ bool Manager::CreateEventStream(RecordVal* fval) { if ( args->length() != 3 ) { - reporter->Error("event has wrong number of arguments"); - delete stream; + reporter->Error("Input stream %s: Event has wrong number of arguments", stream_name.c_str()); return false; } @@ -482,10 +476,10 @@ bool Manager::CreateEventStream(RecordVal* fval) ODesc desc2; (*args)[2]->Describe(&desc1); fields->Describe(&desc2); - reporter->Error("Incompatible type '%s':%s for event, which needs type '%s':%s\n", + reporter->Error("Input stream %s: Incompatible type '%s':%s for event, which needs type '%s':%s\n", + stream_name.c_str(), type_name((*args)[2]->Tag()), desc1.Description(), type_name(fields->Tag()), desc2.Description()); - delete stream; return false; } @@ -496,17 +490,26 @@ bool Manager::CreateEventStream(RecordVal* fval) else assert(false); - vector fieldsV; // vector, because UnrollRecordType needs it bool status = (! UnrollRecordType(&fieldsV, fields, "", allow_file_func)); if ( status ) { - reporter->Error("Problem unrolling"); - delete stream; + reporter->Error("Input stream %s: Problem unrolling", stream_name.c_str()); return false; } + + EventStream* stream = new EventStream(); + { + bool res = CreateStream(stream, fval); + if ( res == false ) + { + delete stream; + return false; + } + } + Field** logf = new Field*[fieldsV.size()]; for ( unsigned int i = 0; i < fieldsV.size(); i++ ) @@ -540,15 +543,9 @@ bool Manager::CreateTableStream(RecordVal* fval) return false; } - TableStream* stream = new TableStream(); - { - bool res = CreateStream(stream, fval); - if ( res == false ) - { - delete stream; - return false; - } - } + Val* name_val = fval->Lookup("name", true); + string stream_name = name_val->AsString()->CheckString(); + Unref(name_val); Val* pred = fval->Lookup("pred", true); @@ -571,28 +568,53 @@ bool Manager::CreateTableStream(RecordVal* fval) { if ( j >= num ) { - reporter->Error("Table type has more indexes than index definition"); - delete stream; + reporter->Error("Input stream %s: Table type has more indexes than index definition", stream_name.c_str()); return false; } if ( ! same_type(idx->FieldType(j), (*tl)[j]) ) { - reporter->Error("Table type does not match index type"); - delete stream; + ODesc desc1; + ODesc desc2; + idx->FieldType(j)->Describe(&desc1); + (*tl)[j]->Describe(&desc2); + reporter->Error("Input stream %s: Table type does not match index type. Need type '%s':%s, got '%s':%s", stream_name.c_str(), + type_name(idx->FieldType(j)->Tag()), desc1.Description(), + type_name((*tl)[j]->Tag()), desc2.Description() + ); return false; } } - if ( num != j ) { - reporter->Error("Table has less elements than index definition"); - delete stream; + reporter->Error("Input stream %s: Table has less elements than index definition", stream_name.c_str()); return false; } Val *want_record = fval->Lookup("want_record", true); + { + const BroType* table_yield = dst->Type()->AsTableType()->YieldType(); + const BroType* compare_type = val; + + if ( want_record->InternalInt() == 0 ) + compare_type = val->FieldType(0); + + if ( !same_type(table_yield, compare_type) ) + { + ODesc desc1; + ODesc desc2; + compare_type->Describe(&desc1); + table_yield->Describe(&desc2); + reporter->Error("Input stream %s: Table type does not match value type. Need type '%s', got '%s'", stream_name.c_str(), + desc1.Description(), desc2.Description() + ); + return false; + } + } + + + Val* event_val = fval->Lookup("ev", true); Func* event = event_val ? event_val->AsFunc() : 0; Unref(event_val); @@ -603,8 +625,7 @@ bool Manager::CreateTableStream(RecordVal* fval) if ( etype->Flavor() != FUNC_FLAVOR_EVENT ) { - reporter->Error("stream event is a function, not an event"); - delete stream; + reporter->Error("Input stream %s: Stream event is a function, not an event", stream_name.c_str()); return false; } @@ -612,43 +633,52 @@ bool Manager::CreateTableStream(RecordVal* fval) if ( args->length() != 4 ) { - reporter->Error("Table event must take 4 arguments"); - delete stream; + reporter->Error("Input stream %s: Table event must take 4 arguments", stream_name.c_str()); return false; } if ( ! same_type((*args)[0], BifType::Record::Input::TableDescription, 0) ) { - reporter->Error("table events first attribute must be of type Input::TableDescription"); - delete stream; + reporter->Error("Input stream %s: Table event's first attribute must be of type Input::TableDescription", stream_name.c_str()); return false; } if ( ! same_type((*args)[1], BifType::Enum::Input::Event, 0) ) { - reporter->Error("table events second attribute must be of type Input::Event"); - delete stream; + reporter->Error("Input stream %s: Table event's second attribute must be of type Input::Event", stream_name.c_str()); return false; } if ( ! same_type((*args)[2], idx) ) { - reporter->Error("table events index attributes do not match"); - delete stream; + ODesc desc1; + ODesc desc2; + idx->Describe(&desc1); + (*args)[2]->Describe(&desc2); + reporter->Error("Input stream %s: Table event's index attributes do not match. Need '%s', got '%s'", stream_name.c_str(), + desc1.Description(), desc2.Description()); return false; } if ( want_record->InternalInt() == 1 && ! same_type((*args)[3], val) ) { - reporter->Error("table events value attributes do not match"); - delete stream; + ODesc desc1; + ODesc desc2; + val->Describe(&desc1); + (*args)[3]->Describe(&desc2); + reporter->Error("Input stream %s: Table event's value attributes do not match. Need '%s', got '%s'", stream_name.c_str(), + desc1.Description(), desc2.Description()); return false; } else if ( want_record->InternalInt() == 0 && !same_type((*args)[3], val->FieldType(0) ) ) { - reporter->Error("table events value attribute does not match"); - delete stream; + ODesc desc1; + ODesc desc2; + val->FieldType(0)->Describe(&desc1); + (*args)[3]->Describe(&desc2); + reporter->Error("Input stream %s: Table event's value attribute does not match. Need '%s', got '%s'", stream_name.c_str(), + desc1.Description(), desc2.Description()); return false; } @@ -666,21 +696,36 @@ bool Manager::CreateTableStream(RecordVal* fval) status = status || ! UnrollRecordType(&fieldsV, val, "", BifConst::Input::accept_unsupported_types); int valfields = fieldsV.size() - idxfields; + + if ( (valfields > 1) && (want_record->InternalInt() != 1) ) + { + reporter->Error("Input stream %s: Stream does not want a record (want_record=F), but has more then one value field.", stream_name.c_str()); + return false; + } if ( ! val ) assert(valfields == 0); if ( status ) { - reporter->Error("Problem unrolling"); - delete stream; + reporter->Error("Input stream %s: Problem unrolling", stream_name.c_str()); return false; } + + TableStream* stream = new TableStream(); + { + bool res = CreateStream(stream, fval); + if ( res == false ) + { + delete stream; + return false; + } + } Field** fields = new Field*[fieldsV.size()]; for ( unsigned int i = 0; i < fieldsV.size(); i++ ) fields[i] = fieldsV[i]; - + stream->pred = pred ? pred->AsFunc() : 0; stream->num_idx_fields = idxfields; stream->num_val_fields = valfields; @@ -697,15 +742,6 @@ bool Manager::CreateTableStream(RecordVal* fval) Unref(want_record); // ref'd by lookupwithdefault Unref(pred); - if ( valfields > 1 ) - { - if ( ! stream->want_record ) - { - reporter->Error("Stream %s does not want a record (want_record=F), but has more then one value field. Aborting", stream->name.c_str()); - delete stream; - return false; - } - } assert(stream->reader); @@ -866,6 +902,7 @@ bool Manager::UnrollRecordType(vector *fields, const RecordType *rec, if ( ! IsCompatibleType(rec->FieldType(i)) ) { + string name = nameprepend + rec->FieldName(i); // If the field is a file, function, or opaque // and it is optional, we accept it nevertheless. // This allows importing logfiles containing this @@ -877,12 +914,12 @@ bool Manager::UnrollRecordType(vector *fields, const RecordType *rec, rec->FieldType(i)->Tag() == TYPE_OPAQUE ) && rec->FieldDecl(i)->FindAttr(ATTR_OPTIONAL) ) { - reporter->Info("Encountered incompatible type \"%s\" in type definition for ReaderFrontend. Ignoring optional field.", type_name(rec->FieldType(i)->Tag())); + reporter->Info("Encountered incompatible type \"%s\" in type definition for field \"%s\" in ReaderFrontend. Ignoring optional field.", type_name(rec->FieldType(i)->Tag()), name.c_str()); continue; } } - reporter->Error("Incompatible type \"%s\" in type definition for ReaderFrontend", type_name(rec->FieldType(i)->Tag())); + reporter->Error("Incompatible type \"%s\" in type definition for for field \"%s\" in ReaderFrontend", type_name(rec->FieldType(i)->Tag()), name.c_str()); return false; } @@ -890,6 +927,11 @@ bool Manager::UnrollRecordType(vector *fields, const RecordType *rec, { string prep = nameprepend + rec->FieldName(i) + "."; + if ( rec->FieldDecl(i)->FindAttr(ATTR_OPTIONAL) ) { + reporter->Info("The input framework does not support optional record fields: \"%s\"", rec->FieldName(i)); + return false; + } + if ( !UnrollRecordType(fields, rec->FieldType(i)->AsRecordType(), prep, allow_file_func) ) { return false; diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.errors/.stderr b/testing/btest/Baseline/scripts.base.frameworks.input.errors/.stderr new file mode 100644 index 0000000000..02c1f56ef3 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.errors/.stderr @@ -0,0 +1,36 @@ +error: Incompatible type "file" in type definition for for field "s" in ReaderFrontend +error: Input stream file: Problem unrolling +The input framework does not support optional record fields: "r" +error: Input stream optionalrecord: Problem unrolling +Encountered incompatible type "file" in type definition for field "s" in ReaderFrontend. Ignoring optional field. +error: Incompatible type "file" in type definition for for field "s" in ReaderFrontend +error: Input stream filetable: Problem unrolling +The input framework does not support optional record fields: "r" +error: Input stream optionalrecordtable: Problem unrolling +Encountered incompatible type "file" in type definition for field "s" in ReaderFrontend. Ignoring optional field. +error: Input stream optionalfiletable: Table type does not match value type. Need type 'record { i:int; s:file of string; }', got 'record { i:int; r:record { i:int; s:file of string; }; }' +error: Input stream optionalfiletable2: Table type does not match index type. Need type 'count':count, got 'string':string +error: Input stream optionalfiletable3: Stream event is a function, not an event +error: Input stream optionalfiletable3: Table event must take 4 arguments +error: Input stream optionalfiletable4: Table event's first attribute must be of type Input::TableDescription +error: Input stream optionalfiletable5: Table event's second attribute must be of type Input::Event +error: Input stream optionalfiletable6: Table event's index attributes do not match. Need 'record { c:count; }', got 'record { i:int; r:record { i:int; s:file of string; }; }' +error: Input stream optionalfiletable7: Table event's value attributes do not match. Need 'record { i:int; s:file of string; }', got 'record { i:int; r:record { i:int; s:file of string; }; }' +error: Input stream optionalfiletable8: Stream does not want a record (want_record=F), but has more then one value field. +error: Input stream optionalfiletable9: Table has less elements than index definition +error: Input stream optionalfiletable10: Table type has more indexes than index definition +error: Input stream optionalfiletable11: Table type does not match value type. Need type 'count', got 'int' +error: Input stream optionalfiletable12: Table type does not match value type. Need type 'count', got 'record { i:int; s:string; a:addr; }' +error: Input stream optionalfiletable14: Table type does not match value type. Need type 'int', got 'record { i:int; s:file of string; }' +error: Input stream optionalfiletable15: Table type does not match value type. Need type 'record { c:count; }', got 'record { i:int; s:string; a:addr; }' +error: Input stream event1: Stream event is a function, not an event +error: Input stream event2: Event does not take enough arguments +error: Input stream event3: Event's first attribute must be of type Input::EventDescription +error: Input stream event4: Event's second attribute must be of type Input::Event +error: Input stream event5: Incompatible type 'record':record { i:int; r:record { i:int; s:file of string; }; } for event, which needs type 'record':record { i:int; s:file of string; } + +error: Input stream event6: Event has wrong number of arguments +error: Input stream event7: Incompatible type for event in field 3. Need type 'int':int, got 'record':record { i:int; r:record { i:int; s:file of string; }; } +error: Input stream event8: Incompatible type for event in field 5. Need type 'addr':addr, got 'string':string +error: Input stream event9: Event has wrong number of arguments +received termination signal diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.errors/out b/testing/btest/Baseline/scripts.base.frameworks.input.errors/out new file mode 100644 index 0000000000..dbe0263328 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.errors/out @@ -0,0 +1,2 @@ +optionalfile +[i=-42, s=] diff --git a/testing/btest/scripts/base/frameworks/input/errors.bro b/testing/btest/scripts/base/frameworks/input/errors.bro new file mode 100644 index 0000000000..22b88434cd --- /dev/null +++ b/testing/btest/scripts/base/frameworks/input/errors.bro @@ -0,0 +1,161 @@ +# Test different kinds of errors of the input framework +# +# @TEST-EXEC: bro -b %INPUT +# @TEST-EXEC: btest-diff .stderr +# @TEST-EXEC: btest-diff out + +@TEST-START-FILE input.log +#separator \x09 +#path ssh +#fields b i e c p sn a d t iv s sc ss se vc ve ns +#types bool int enum count port subnet addr double time interval string table table table vector vector string +T -42 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY 4242 +@TEST-END-FILE + +redef Input::accept_unsupported_types = T; + +redef exit_only_after_terminate = T; + +module Test; + +global outfile: file; + +type Idx: record { + c: count; +}; + +type Idx2: record { + c: count; + i: int; +}; + +type FileVal: record { + i: int; + s: file; +}; + +type Val: record { + i: int; + s: string; + a: addr; +}; + +type OptionalRecordVal: record { + i: int; + r: FileVal &optional; +}; + +type OptionalFileVal: record { + i: int; + s: file &optional; +}; + +global file_table: table[count] of FileVal = table(); +global optional_file_table: table[count] of OptionalFileVal = table(); +global record_table: table[count] of OptionalRecordVal = table(); +global string_table: table[string] of OptionalRecordVal = table(); + +global val_table: table[count] of Val = table(); +global val_table2: table[count, int] of Val = table(); +global val_table3: table[count, int] of int = table(); +global val_table4: table[count] of int; + +event line_file(description: Input::EventDescription, tpe: Input::Event, r:FileVal) + { + print outfile, description$name; + print outfile, r; + } + +event optional_line_file(description: Input::EventDescription, tpe: Input::Event, r:OptionalFileVal) + { + print outfile, description$name; + print outfile, r; + } + +event line_record(description: Input::EventDescription, tpe: Input::Event, r: OptionalRecordVal) + { + print outfile, description$name; + print outfile, r; + } + +event event1(description: Input::EventDescription, tpe: Input::Event, r: OptionalRecordVal, r2: OptionalRecordVal) + { + } + +event event2(description: Input::TableDescription, tpe: string, r: OptionalRecordVal, r2: OptionalRecordVal) + { + } + +event event3(description: Input::TableDescription, tpe: Input::Event, r: OptionalRecordVal, r2: OptionalRecordVal) + { + } + +event event4(description: Input::TableDescription, tpe: Input::Event, r: Idx, r2: OptionalRecordVal) + { + } + +event event5(description: Input::EventDescription, tpe: string, r: OptionalRecordVal, r2: OptionalRecordVal) + { + } + +event event6(description: Input::EventDescription, tpe: Input::Event, r: OptionalRecordVal) + { + } + +event event7(description: Input::EventDescription, tpe: Input::Event, r: OptionalRecordVal, r2:OptionalRecordVal) + { + } + +event event8(description: Input::EventDescription, tpe: Input::Event, i: int, s:string, a:string) + { + } + +event event9(description: Input::EventDescription, tpe: Input::Event, i: int, s:string, a:addr, ii: int) + { + } + +event event10(description: Input::TableDescription, tpe: Input::Event, i: Idx, c: count) + { + } + +event kill_me() + { + terminate(); + } + +event bro_init() + { + outfile = open("out"); + Input::add_event([$source="input.log", $name="file", $fields=FileVal, $ev=line_file, $want_record=T]); + Input::add_event([$source="input.log", $name="optionalrecord", $fields=OptionalRecordVal, $ev=line_record, $want_record=T]); + Input::add_event([$source="input.log", $name="optionalfile", $fields=OptionalFileVal, $ev=optional_line_file, $want_record=T]); + Input::add_table([$source="input.log", $name="filetable", $idx=Idx, $val=FileVal, $destination=file_table]); + Input::add_table([$source="input.log", $name="optionalrecordtable", $idx=Idx, $val=OptionalRecordVal, $destination=record_table]); + Input::add_table([$source="input.log", $name="optionalfiletable", $idx=Idx, $val=OptionalFileVal, $destination=optional_file_table]); + Input::add_table([$source="input.log", $name="optionalfiletable", $idx=Idx, $val=OptionalFileVal, $destination=record_table]); + Input::add_table([$source="input.log", $name="optionalfiletable2", $idx=Idx, $val=OptionalFileVal, $destination=string_table]); + Input::add_table([$source="input.log", $name="optionalfiletable3", $idx=Idx, $val=OptionalFileVal, $destination=optional_file_table, $ev=terminate]); + Input::add_table([$source="input.log", $name="optionalfiletable3", $idx=Idx, $val=OptionalFileVal, $destination=optional_file_table, $ev=kill_me]); + Input::add_table([$source="input.log", $name="optionalfiletable4", $idx=Idx, $val=OptionalFileVal, $destination=optional_file_table, $ev=event1]); + Input::add_table([$source="input.log", $name="optionalfiletable5", $idx=Idx, $val=OptionalFileVal, $destination=optional_file_table, $ev=event2]); + Input::add_table([$source="input.log", $name="optionalfiletable6", $idx=Idx, $val=OptionalFileVal, $destination=optional_file_table, $ev=event3]); + Input::add_table([$source="input.log", $name="optionalfiletable7", $idx=Idx, $val=OptionalFileVal, $destination=optional_file_table, $ev=event4]); + Input::add_table([$source="input.log", $name="optionalfiletable8", $idx=Idx, $val=Val, $destination=val_table4, $want_record=F]); + Input::add_table([$source="input.log", $name="optionalfiletable9", $idx=Idx2, $val=Val, $destination=val_table, $want_record=F]); + Input::add_table([$source="input.log", $name="optionalfiletable10", $idx=Idx, $val=Val, $destination=val_table2, $want_record=F]); + Input::add_table([$source="input.log", $name="optionalfiletable11", $idx=Idx2, $val=Idx, $destination=val_table3, $want_record=F]); + Input::add_table([$source="input.log", $name="optionalfiletable12", $idx=Idx2, $val=Idx, $destination=val_table2, $want_record=F]); + Input::add_table([$source="input.log", $name="optionalfiletable14", $idx=Idx, $val=OptionalFileVal, $destination=optional_file_table, $ev=event10, $want_record=F]); + Input::add_table([$source="input.log", $name="optionalfiletable15", $idx=Idx2, $val=Idx, $destination=val_table2, $want_record=T]); + Input::add_event([$source="input.log", $name="event1", $fields=OptionalFileVal, $ev=terminate, $want_record=T]); + Input::add_event([$source="input.log", $name="event2", $fields=OptionalFileVal, $ev=kill_me, $want_record=T]); + Input::add_event([$source="input.log", $name="event3", $fields=OptionalFileVal, $ev=event3, $want_record=T]); + Input::add_event([$source="input.log", $name="event4", $fields=OptionalFileVal, $ev=event5, $want_record=T]); + Input::add_event([$source="input.log", $name="event5", $fields=OptionalFileVal, $ev=event6, $want_record=T]); + Input::add_event([$source="input.log", $name="event6", $fields=OptionalFileVal, $ev=event7, $want_record=T]); + Input::add_event([$source="input.log", $name="event7", $fields=OptionalFileVal, $ev=event7, $want_record=F]); + Input::add_event([$source="input.log", $name="event8", $fields=Val, $ev=event8, $want_record=F]); + Input::add_event([$source="input.log", $name="event9", $fields=Val, $ev=event9, $want_record=F]); + + schedule 3secs { kill_me() }; + } diff --git a/testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro b/testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro index 7d12fd6d3f..c38c4efd85 100644 --- a/testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro +++ b/testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro @@ -26,7 +26,7 @@ type Val: record { b: bool; }; -global servers: table[int] of Val = table(); +global servers: table[int] of bool = table(); event bro_init() { diff --git a/testing/btest/scripts/base/frameworks/input/predicate-stream.bro b/testing/btest/scripts/base/frameworks/input/predicate-stream.bro index 45cdf81059..aac44fb8ee 100644 --- a/testing/btest/scripts/base/frameworks/input/predicate-stream.bro +++ b/testing/btest/scripts/base/frameworks/input/predicate-stream.bro @@ -36,7 +36,7 @@ type Val: record { b: bool; }; -global servers: table[int] of Val = table(); +global servers: table[int] of bool = table(); global ct: int; event line(description: Input::TableDescription, tpe: Input::Event, left: Idx, right: bool) diff --git a/testing/btest/scripts/base/frameworks/input/repeat.bro b/testing/btest/scripts/base/frameworks/input/repeat.bro index d754e10257..5093e30351 100644 --- a/testing/btest/scripts/base/frameworks/input/repeat.bro +++ b/testing/btest/scripts/base/frameworks/input/repeat.bro @@ -27,7 +27,7 @@ type Val: record { b: bool; }; -global destination: table[int] of Val = table(); +global destination: table[int] of bool = table(); const one_to_32: vector of count = {1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32}; diff --git a/testing/btest/scripts/base/frameworks/input/tableevent.bro b/testing/btest/scripts/base/frameworks/input/tableevent.bro index 162b5dfe34..760b19d24f 100644 --- a/testing/btest/scripts/base/frameworks/input/tableevent.bro +++ b/testing/btest/scripts/base/frameworks/input/tableevent.bro @@ -31,7 +31,7 @@ type Val: record { b: bool; }; -global destination: table[int] of Val = table(); +global destination: table[int] of bool = table(); event line(description: Input::TableDescription, tpe: Input::Event, left: Idx, right: bool) { @@ -51,5 +51,5 @@ event bro_init() { try = 0; outfile = open("../out"); - Input::add_table([$source="../input.log", $name="input", $idx=Idx, $val=Val, $destination=destination, $want_record=F,$ev=line]); + Input::add_table([$source="../input.log", $name="input", $idx=Idx, $val=Val, $destination=destination, $want_record=F, $ev=line]); } From dedc39d7848edef4dbc35cc38cabd1e6d7f559c1 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Fri, 6 Dec 2013 09:35:35 -0600 Subject: [PATCH 38/40] Minor Broxygen improvements, addresses BIT-1098. - Internals: move type alias table to private static BroType member. - Sphinx extension: now uses absolute path to bro binary. - reST ouput formatting: remove "param" from function desriptions and change package overview docs so script link+summaries render consistently. --- doc/conf.py.in | 1 + doc/ext/broxygen.py | 8 +++++++- src/Type.cc | 10 +++++----- src/Type.h | 13 ++++++++++--- src/Var.cc | 4 ++-- src/broxygen/Target.cc | 2 +- src/broxygen/utils.cc | 8 +++++++- src/broxygen/utils.h | 8 +++++++- src/main.cc | 2 -- src/parse.y | 2 +- .../btest/Baseline/doc.broxygen.example/example.rst | 6 +++--- .../autogen-reST-func-params.rst | 8 ++++---- .../btest/Baseline/doc.broxygen.identifier/test.rst | 6 +++--- .../btest/Baseline/doc.broxygen.package/test.rst | 2 ++ 14 files changed, 53 insertions(+), 27 deletions(-) diff --git a/doc/conf.py.in b/doc/conf.py.in index f54ca9d8c6..91e16452f3 100644 --- a/doc/conf.py.in +++ b/doc/conf.py.in @@ -35,6 +35,7 @@ btest_tests="doc/sphinx" # ----- Begin of Broxygen configuration. ----- extensions += ["broxygen"] +bro_binary = os.path.abspath("@CMAKE_SOURCE_DIR@/build/src/bro") broxygen_cache="@BROXYGEN_CACHE_DIR@" os.environ["BROPATH"] = "@BROPATH@" os.environ["BROMAGIC"] = "@BROMAGIC@" diff --git a/doc/ext/broxygen.py b/doc/ext/broxygen.py index adea390642..b6b47bb82b 100644 --- a/doc/ext/broxygen.py +++ b/doc/ext/broxygen.py @@ -114,7 +114,12 @@ def build_target(env, target): import os import subprocess - bro_cmd = "bro -X {0} broxygen".format(target.config_file) + path_to_bro = env.config.bro_binary + + if not path_to_bro: + raise SphinxError("'bro' not set in sphinx config file (path to bro)") + + bro_cmd = "{0} -X {1} broxygen".format(path_to_bro, target.config_file) cwd = os.getcwd() os.chdir(os.path.dirname(target.config_file)) @@ -307,5 +312,6 @@ def setup(app): global App App = app app.add_domain(BroxygenDomain) + app.add_config_value("bro_binary", None, "env") app.add_config_value("broxygen_cache", None, "env") app.connect("env-get-outdated", env_get_outdated_hook) diff --git a/src/Type.cc b/src/Type.cc index e8e53fd9a6..340ab973bc 100644 --- a/src/Type.cc +++ b/src/Type.cc @@ -15,6 +15,8 @@ #include #include +BroType::TypeAliasMap BroType::type_aliases; + // Note: This function must be thread-safe. const char* type_name(TypeTag t) { @@ -1183,8 +1185,7 @@ void RecordType::DescribeFieldsReST(ODesc* d, bool func_args) const field_from_script != type_from_script ) { d->PushIndent(); - d->Add(fmt("(from ``redef`` in :doc:`/scripts/%s`)", - field_from_script.c_str())); + d->Add(broxygen::redef_indication(field_from_script).c_str()); d->PopIndent(); } @@ -1455,7 +1456,7 @@ void EnumType::CheckAndAddName(const string& module_name, const char* name, AddNameInternal(module_name, name, val, is_export); - set types = type_aliases[GetName()]; + set types = BroType::GetAliases(GetName()); set::const_iterator it; for ( it = types.begin(); it != types.end(); ++it ) @@ -1541,8 +1542,7 @@ void EnumType::DescribeReST(ODesc* d, bool roles_only) const { d->NL(); d->PushIndent(); - d->Add(fmt("(from ``redef`` in :doc:`/scripts/%s`)", - enum_from_script.c_str())); + d->Add(broxygen::redef_indication(enum_from_script).c_str()); d->PopIndent(); } diff --git a/src/Type.h b/src/Type.h index b68bd9a818..f6328aed4a 100644 --- a/src/Type.h +++ b/src/Type.h @@ -245,6 +245,14 @@ public: void SetName(const string& arg_name) { name = arg_name; } string GetName() const { return name; } + typedef std::map > TypeAliasMap; + + static std::set GetAliases(const std::string& type_name) + { return BroType::type_aliases[type_name]; } + + static void AddAlias(const std::string type_name, BroType* type) + { BroType::type_aliases[type_name].insert(type); } + protected: BroType() { } @@ -258,6 +266,8 @@ private: bool is_network_order; bool base_type; string name; + + static TypeAliasMap type_aliases; }; class TypeList : public BroType { @@ -580,9 +590,6 @@ protected: BroType* yield_type; }; -typedef std::map > TypeAliasMap; -extern TypeAliasMap type_aliases; - extern OpaqueType* md5_type; extern OpaqueType* sha1_type; extern OpaqueType* sha256_type; diff --git a/src/Var.cc b/src/Var.cc index eb03f2f912..1d4f1f4620 100644 --- a/src/Var.cc +++ b/src/Var.cc @@ -273,10 +273,10 @@ void add_type(ID* id, BroType* t, attr_list* attr) // Clone the type to preserve type name aliasing. tnew = t->Clone(); - type_aliases[new_type_name].insert(tnew); + BroType::AddAlias(new_type_name, tnew); if ( new_type_name != old_type_name && ! old_type_name.empty() ) - type_aliases[old_type_name].insert(tnew); + BroType::AddAlias(old_type_name, tnew); tnew->SetName(id->Name()); diff --git a/src/broxygen/Target.cc b/src/broxygen/Target.cc index 692494078d..5518dabc3d 100644 --- a/src/broxygen/Target.cc +++ b/src/broxygen/Target.cc @@ -368,7 +368,7 @@ void PackageTarget::DoGenerate() const for ( size_t i = 0; i < it->second.size(); ++i ) { - fprintf(file.f, ":doc:`/scripts/%s`\n", + fprintf(file.f, ":doc:`/scripts/%s`\n\n", it->second[i]->Name().c_str()); vector cmnts = it->second[i]->GetComments(); diff --git a/src/broxygen/utils.cc b/src/broxygen/utils.cc index 93ec268fc2..93f822b846 100644 --- a/src/broxygen/utils.cc +++ b/src/broxygen/utils.cc @@ -63,7 +63,7 @@ bool broxygen::prettify_params(string& s) if ( identifier == "Returns" ) subst = ":returns"; else - subst = ":param " + identifier; + subst = ":" + identifier; s.replace(identifier_start_pos, identifier.size(), subst); return true; @@ -127,3 +127,9 @@ bool broxygen::is_all_whitespace(const string& s) return true; } + +string broxygen::redef_indication(const string& from_script) + { + return fmt("(present if :doc:`/scripts/%s` is loaded)", + from_script.c_str()); + } diff --git a/src/broxygen/utils.h b/src/broxygen/utils.h index a615de7c09..7e11019a3d 100644 --- a/src/broxygen/utils.h +++ b/src/broxygen/utils.h @@ -13,7 +13,7 @@ namespace broxygen { * Transform content of a Broxygen comment which may contain function * parameter or return value documentation to a prettier reST format. * @param s Content from a Broxygen comment to transform. "id: ..." and - * "Returns: ..." change to ":param id: ..." and ":returns: ...". + * "Returns: ..." change to ":id: ..." and ":returns: ...". * @return Whether any content in \a s was transformed. */ bool prettify_params(std::string& s); @@ -56,6 +56,12 @@ size_t end_of_first_sentence(const std::string& s); */ bool is_all_whitespace(const std::string& s); +/** + * @return a string indicating the script that has redef'd an enum value or + * record field. + */ +std::string redef_indication(const std::string& from_script); + } // namespace broxygen #endif diff --git a/src/main.cc b/src/main.cc index d48a931b24..53a0cb20ee 100644 --- a/src/main.cc +++ b/src/main.cc @@ -123,8 +123,6 @@ vector params; char* proc_status_file = 0; int snaplen = 0; // this gets set from the scripting-layer's value -TypeAliasMap type_aliases; - OpaqueType* md5_type = 0; OpaqueType* sha1_type = 0; OpaqueType* sha256_type = 0; diff --git a/src/parse.y b/src/parse.y index 9e7f2c44a7..941a268c0b 100644 --- a/src/parse.y +++ b/src/parse.y @@ -179,7 +179,7 @@ static attr_list* copy_attr_list(attr_list* al) static void extend_record(ID* id, type_decl_list* fields, attr_list* attrs) { - set types = type_aliases[id->Name()]; + set types = BroType::GetAliases(id->Name()); if ( types.empty() ) { diff --git a/testing/btest/Baseline/doc.broxygen.example/example.rst b/testing/btest/Baseline/doc.broxygen.example/example.rst index 089a0518e7..48289fe466 100644 --- a/testing/btest/Baseline/doc.broxygen.example/example.rst +++ b/testing/btest/Baseline/doc.broxygen.example/example.rst @@ -223,7 +223,7 @@ Events link. Use the see role instead: :bro:see:`BroxygenExample::a_function`. - :param name: Describe the argument here. + :name: Describe the argument here. Functions ######### @@ -237,11 +237,11 @@ Functions empty comments is optional, but improves readability of script. - :param tag: Function arguments can be described + :tag: Function arguments can be described like this. - :param msg: Another param. + :msg: Another param. :returns: Describe the return type here. diff --git a/testing/btest/Baseline/doc.broxygen.func-params/autogen-reST-func-params.rst b/testing/btest/Baseline/doc.broxygen.func-params/autogen-reST-func-params.rst index 57f12e27c3..06f196b73c 100644 --- a/testing/btest/Baseline/doc.broxygen.func-params/autogen-reST-func-params.rst +++ b/testing/btest/Baseline/doc.broxygen.func-params/autogen-reST-func-params.rst @@ -5,9 +5,9 @@ This is a global function declaration. - :param i: First param. + :i: First param. - :param j: Second param. + :j: Second param. :returns: A string. @@ -20,9 +20,9 @@ This is a record field function. - :param i: First param. + :i: First param. - :param j: Second param. + :j: Second param. :returns: A string. diff --git a/testing/btest/Baseline/doc.broxygen.identifier/test.rst b/testing/btest/Baseline/doc.broxygen.identifier/test.rst index 539a03c92a..0c7c44581d 100644 --- a/testing/btest/Baseline/doc.broxygen.identifier/test.rst +++ b/testing/btest/Baseline/doc.broxygen.identifier/test.rst @@ -191,11 +191,11 @@ empty comments is optional, but improves readability of script. - :param tag: Function arguments can be described + :tag: Function arguments can be described like this. - :param msg: Another param. + :msg: Another param. :returns: Describe the return type here. @@ -212,7 +212,7 @@ link. Use the see role instead: :bro:see:`BroxygenExample::a_function`. - :param name: Describe the argument here. + :name: Describe the argument here. .. bro:id:: BroxygenExample::function_without_proto diff --git a/testing/btest/Baseline/doc.broxygen.package/test.rst b/testing/btest/Baseline/doc.broxygen.package/test.rst index d65f563fd5..b96de2148b 100644 --- a/testing/btest/Baseline/doc.broxygen.package/test.rst +++ b/testing/btest/Baseline/doc.broxygen.package/test.rst @@ -10,7 +10,9 @@ extra scripts needed or used by the documentation process. :doc:`/scripts/broxygen/__load__.bro` + :doc:`/scripts/broxygen/example.bro` + This is an example script that demonstrates Broxygen-style documentation. It generally will make most sense when viewing the script's raw source code and comparing to the HTML-rendered From 49a3d5397011be8871ab9792aab92c02baad4404 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Mon, 9 Dec 2013 13:54:16 -0800 Subject: [PATCH 39/40] Updating submodule(s). [nomail] --- CHANGES | 2 +- VERSION | 2 +- aux/broccoli | 2 +- aux/broctl | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGES b/CHANGES index 6c49fcc4a7..3356637c8b 100644 --- a/CHANGES +++ b/CHANGES @@ -1,5 +1,5 @@ -2.2-65 | 2013-12-09 13:23:06 -0800 +2.2-66 | 2013-12-09 13:54:16 -0800 * Fix table &default reference counting for record ctor expressions. (Jon Siwek) diff --git a/VERSION b/VERSION index bbaf734378..7422adecf7 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.2-65 +2.2-66 diff --git a/aux/broccoli b/aux/broccoli index fd28cff404..e02ccc0a27 160000 --- a/aux/broccoli +++ b/aux/broccoli @@ -1 +1 @@ -Subproject commit fd28cff404a9f8a8b58972ac31a331c5004e5f0c +Subproject commit e02ccc0a27e64b147f01e4c7deb5b897864d59d5 diff --git a/aux/broctl b/aux/broctl index 6e01d6972f..2e07720b4f 160000 --- a/aux/broctl +++ b/aux/broctl @@ -1 +1 @@ -Subproject commit 6e01d6972f02d68ee82d05f392d1a00725595b7f +Subproject commit 2e07720b4f129802e07ca99498e2aff4542c737a From 5a67135486d918e002696c26ae3ca7f6f217a620 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Tue, 10 Dec 2013 14:08:09 -0600 Subject: [PATCH 40/40] Fix uninitialized field in basename/dirname util wrapper. Shouldn't cause a problem as it's always set in subclass ctors, just silences a coverity warning. --- src/util.h | 1 + 1 file changed, 1 insertion(+) diff --git a/src/util.h b/src/util.h index d86f45880d..16a5181dbc 100644 --- a/src/util.h +++ b/src/util.h @@ -230,6 +230,7 @@ public: protected: SafePathOp() + : result(), error() { } void CheckValid(const char* result, const char* path, bool error_aborts);