From 30209b56bb29dc8a5af00e305f4fc7f4b9c9fd20 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Fri, 25 Feb 2011 15:30:18 -0600 Subject: [PATCH 01/47] Bro script documentation framework checkpoint * New bro runtime options: -Z or --doc-scripts enables documentation mode * New BroDoc, BroBifDoc, and BroDocObj interfaces to support script documentation * Modifications to the bro scanner (scan.l) to get it to keep track of which script is being scanned/parsed and which document is being generated * Modifications to scan.l and the bro parser (parse.y) to produce/consume script comments denoted with "##" * Documentation is currently generated for the following ** Script author ** Script summary ** @load's ** capture_filters ** modules (namespaces) Most of the remaining framework/infrastructure work should be in extracting the interesting BroObj objects as the parser sees them and better formatting the reST documents. --- src/BroBifDoc.cc | 16 ++++ src/BroBifDoc.h | 21 ++++++ src/BroDoc.cc | 163 ++++++++++++++++++++++++++++++++++++++++ src/BroDoc.h | 183 +++++++++++++++++++++++++++++++++++++++++++++ src/BroDocObj.cc | 35 +++++++++ src/BroDocObj.h | 72 ++++++++++++++++++ src/CMakeLists.txt | 3 + src/main.cc | 9 ++- src/parse.y | 17 ++++- src/scan.l | 128 ++++++++++++++++++++++++++++++- 10 files changed, 644 insertions(+), 3 deletions(-) create mode 100644 src/BroBifDoc.cc create mode 100644 src/BroBifDoc.h create mode 100644 src/BroDoc.cc create mode 100644 src/BroDoc.h create mode 100644 src/BroDocObj.cc create mode 100644 src/BroDocObj.h diff --git a/src/BroBifDoc.cc b/src/BroBifDoc.cc new file mode 100644 index 0000000000..9d358eb04b --- /dev/null +++ b/src/BroBifDoc.cc @@ -0,0 +1,16 @@ +#include +#include +#include + +#include "BroDoc.h" +#include "BroBifDoc.h" + +BroBifDoc::BroBifDoc(const std::string& sourcename) : BroDoc(sourcename) + { + } + +//TODO: this needs to do something different than parent class's version +void BroBifDoc::WriteDocFile() const + { + BroDoc::WriteDocFile(); + } diff --git a/src/BroBifDoc.h b/src/BroBifDoc.h new file mode 100644 index 0000000000..ef2dc5e2d7 --- /dev/null +++ b/src/BroBifDoc.h @@ -0,0 +1,21 @@ +#ifndef brobifdoc_h +#define brobifdoc_h + +#include +#include +#include + +#include "BroDoc.h" + +class BroBifDoc : public BroDoc { +public: + BroBifDoc(const std::string& sourcename); + + void WriteDocFile() const; + +protected: + +private: +}; + +#endif diff --git a/src/BroDoc.cc b/src/BroDoc.cc new file mode 100644 index 0000000000..fb06a32308 --- /dev/null +++ b/src/BroDoc.cc @@ -0,0 +1,163 @@ +#include +#include +#include +#include + +#include "BroDoc.h" +#include "BroDocObj.h" + +BroDoc::BroDoc(const std::string& sourcename) + { + fprintf(stdout, "Documenting source: %s\n", sourcename.c_str()); + + source_filename = sourcename.substr(sourcename.find_last_of('/') + 1); + + size_t ext_pos = source_filename.find_last_of('.'); + std::string ext = source_filename.substr(ext_pos + 1); + if ( ext_pos == std::string::npos || ext != "bro" ) + { + if ( source_filename != "bro.init" ) + { + fprintf(stderr, + "Warning: documenting file without .bro extension: %s\n", + sourcename.c_str()); + } + else + { + // Force the reST documentation file to be "bro.init.rst" + ext_pos = std::string::npos; + } + } + + reST_filename = source_filename.substr(0, ext_pos); + reST_filename += ".rst"; + reST_file = fopen(reST_filename.c_str(), "w"); + + if ( ! reST_file ) + fprintf(stderr, "Failed to open %s", reST_filename.c_str()); + else + fprintf(stdout, "Created reST document: %s\n", reST_filename.c_str()); + } + +BroDoc::~BroDoc() + { + if ( reST_file ) + if ( fclose( reST_file ) ) + fprintf(stderr, "Failed to close %s", reST_filename.c_str()); + FreeBroDocObjPtrList(options); + FreeBroDocObjPtrList(state_vars); + FreeBroDocObjPtrList(types); + FreeBroDocObjPtrList(notices); + FreeBroDocObjPtrList(events); + FreeBroDocObjPtrList(functions); + } + +void BroDoc::SetPacketFilter(const std::string& s) + { + packet_filter = s; + size_t pos1 = s.find("{\n"); + size_t pos2 = s.find("}"); + if ( pos1 != std::string::npos && pos2 != std::string::npos ) + packet_filter = s.substr(pos1 + 2, pos2 - 2); + } + +void BroDoc::WriteDocFile() const + { + WriteToDoc("%s\n", source_filename.c_str()); + for ( size_t i = 0; i < source_filename.length(); ++i ) + WriteToDoc("="); + WriteToDoc("\n\n"); + + WriteSectionHeading("Summary", '-'); + WriteStringList("%s\n", "%s\n\n", summary); + + WriteToDoc(":Author: %s\n", author_name.c_str()); + + WriteToDoc(":Namespaces: "); + WriteStringList("`%s`, ", "`%s`\n", modules); + + WriteToDoc(":Imports:\n"); + WriteStringList(" :bro:script: `%s`\n", + " :bro:script: `%s`\n\n", imports); + + WriteSectionHeading("Public Interface", '-'); + WriteBroDocObjList(options, true, "Options", '~'); + WriteBroDocObjList(state_vars, true, "State Variables", '~'); + WriteBroDocObjList(types, true, "Types", '~'); + WriteBroDocObjList(notices, true, "Notices", '~'); + WriteBroDocObjList(events, true, "Events", '~'); + WriteBroDocObjList(functions, true, "Functions", '~'); + + WriteSectionHeading("Packet Filter", '-'); + WriteToDoc("%s\n", packet_filter.c_str()); + + WriteSectionHeading("Private Interface", '-'); + WriteBroDocObjList(options, false, "Options", '~'); + WriteBroDocObjList(state_vars, false, "State Variables", '~'); + WriteBroDocObjList(types, false, "Types", '~'); + WriteBroDocObjList(notices, false, "Notices", '~'); + WriteBroDocObjList(events, false, "Events", '~'); + WriteBroDocObjList(functions, false, "Functions", '~'); + } + +void BroDoc::WriteStringList(const char* format, + const char* last_format, + const std::list& l) const + { + if ( l.empty() ) return; + std::list::const_iterator it; + std::list::const_iterator last = l.end(); + last--; + for ( it = l.begin(); it != last; ++it ) + WriteToDoc(format, it->c_str()); + WriteToDoc(last_format, last->c_str()); + } + +void BroDoc::WriteBroDocObjList(const std::list& l, + bool exportCond, + const char* heading, + char underline) const + { + WriteSectionHeading(heading, underline); + std::list::const_iterator it; + for ( it = l.begin(); it != l.end(); ++it ) + { + if ( exportCond ) + { + // write out only those in an export section + if ( (*it)->IsPublicAPI() ) + (*it)->WriteReST(reST_file); + } + else + { + // write out only those that have comments and are not exported + if ( !(*it)->IsPublicAPI() && (*it)->HasDocumentation() ) + (*it)->WriteReST(reST_file); + } + } + } + +void BroDoc::WriteToDoc(const char* format, ...) const + { + va_list argp; + va_start(argp, format); + vfprintf(reST_file, format, argp); + va_end(argp); + } + +void BroDoc::WriteSectionHeading(const char* heading, char underline) const + { + WriteToDoc("%s\n", heading); + size_t len = strlen(heading); + for ( size_t i = 0; i < len; ++i ) + WriteToDoc("%c", underline); + WriteToDoc("\n"); + } + +void BroDoc::FreeBroDocObjPtrList(std::list& l) + { + std::list::iterator it; + for ( it = l.begin(); it != l.end(); ++it ) + delete *it; + l.clear(); + } diff --git a/src/BroDoc.h b/src/BroDoc.h new file mode 100644 index 0000000000..683a4cd5a5 --- /dev/null +++ b/src/BroDoc.h @@ -0,0 +1,183 @@ +#ifndef brodoc_h +#define brodoc_h + +#include +#include +#include +#include + +#include "BroDocObj.h" + +class BroDoc { +public: + /** + * BroDoc constructor + * Given a Bro script, opens new file in the current working directory + * that will contain reST documentation generated from the parsing + * of the Bro script. The new reST file will be named similar to + * the filename of the Bro script that generates it, except any + * ".bro" file extension is stripped and ".rst" takes it place. + * If the filename doesn't end in ".bro", then ".rst" is just appended. + * @param sourcename The name of the Bro script for which to generate + * documentation. May contain a path. + */ + BroDoc(const std::string& sourcename); + + /** + * BroDoc destructor + * Closes the file that was opened by the constructor and frees up + * memory taken by BroDocObj objects. + */ + ~BroDoc(); + + /** + * Write out full reST documentation for the Bro script that was parsed. + * BroDoc's default implementation of this function will care + * about whether declarations made in the Bro script are part of + * the public versus private interface (whether things are declared in + * the export section). Things in a script's export section make it + * into the reST output regardless of whether they have ## comments + * but things outside the export section are only output into the reST + * if they have ## comments. + */ + virtual void WriteDocFile() const; + + /** + * Schedules some summarizing text to be output directly into the reST doc. + * This should be called whenever the scanner sees a line in the Bro script + * starting with "##!" + * @param s The summary text to add to the reST doc. + */ + void AddSummary(const std::string& s) { summary.push_back(s); } + + /** + * Schedules an import (@load) to be documented. + * This should be called whenever the scanner sees an @load. + * @param s The name of the imported script. + */ + void AddImport(const std::string& s) { imports.push_back(s); } + + /** + * Schedules a namespace (module) to be documented. + * This should be called whenever the parser sees a TOK_MODULE. + * @param s The namespace (module) identifier's name. + */ + void AddModule(const std::string& s) { modules.push_back(s); } + + /** + * Sets the way the script changes the "capture_filters" table. + * This is determined by the scanner checking for changes to + * the "capture_filters" table after each of Bro's input scripts + * (given as command line arguments to Bro) are finished being parsed. + * @param s The value "capture_filters" as given by TableVal::Describe() + */ + void SetPacketFilter(const std::string& s); + + /** + * Sets the author of the script. + * The scanner should call this when it sees "## Author: ..." + * @param s The name, email, etc. of the script author(s). Must be + * all on one line. + */ + void SetAuthor(const std::string& s) { author_name = s; } + + //TODO: document these better + // the rest of these need to be called from the parser + void AddOption(const BroDocObj* o) { options.push_back(o); } + void AddStateVar(const BroDocObj* o) { state_vars.push_back(o); } + void AddType(const BroDocObj* o) { types.push_back(o); } + void AddNotice(const BroDocObj* o) { notices.push_back(o); } + void AddEvent(const BroDocObj* o) { events.push_back(o); } + void AddFunction(const BroDocObj* o) { functions.push_back(o); } + + /** + * Gets the name of the Bro script source file for which reST + * documentation is being generated. + * @return A char* to the start of the source file's name. + */ + const char* GetSourceFileName() const { return source_filename.c_str(); } + + /** + * Gets the name of the generated reST documentation file. + * @return A char* to the start of the generated reST file's name. + */ + const char* GetOutputFileName() const { return reST_filename.c_str(); } + +protected: + FILE* reST_file; + std::string reST_filename; + std::string source_filename; + std::string author_name; + std::string packet_filter; + + std::list ls; + std::list modules; + std::list summary; + std::list imports; + + std::list options; // identifiers with &redef attr + std::list state_vars; // identifiers without &redef? + std::list types; + std::list notices; + std::list events; + std::list functions; + + /** + * Writes out a list of strings to the reST document. + * @param format A printf style format string for elements of the list + * except for the last one in the list + * @param last_format A printf style format string to use for the last + * element of the list + * @param l A reference to a list of strings + */ + void WriteStringList(const char* format, + const char* last_format, + const std::list& l) const; + + /** + * @see WriteStringList(const char*, const char*, + const std::list&>) + */ + void WriteStringList(const char* format, + const std::list& l) const + { WriteStringList(format, format, l); } + + /** + * Writes out a list of BroDocObj objects to the reST document + * @param l A list of BroDocObj pointers + * @param exportCond If true, filter out objects that are not in an + * export section. If false, filter out those that are in + * an export section. + * @param heading The title of the section to create in the reST doc. + * @param underline The character to use to underline the reST + * section heading. + */ + void WriteBroDocObjList(const std::list& l, + bool exportCond, + const char* heading, + char underline) const; + + /** + * A wrapper to fprintf() that always uses the reST document + * for the FILE* argument. + * @param format A printf style format string. + */ + void WriteToDoc(const char* format, ...) const; + + /** + * Writes out a reST section heading + * @param heading The title of the heading to create + * @param underline The character to use to underline the section title + within the reST document + */ + void WriteSectionHeading(const char* heading, char underline) const; +private: + + /** + * Frees memory allocated to BroDocObj's objects in a given list. + * @param a reference to a list of BroDocObj pointers + */ + void FreeBroDocObjPtrList(std::list& l); +}; + +#endif diff --git a/src/BroDocObj.cc b/src/BroDocObj.cc new file mode 100644 index 0000000000..645c4f1475 --- /dev/null +++ b/src/BroDocObj.cc @@ -0,0 +1,35 @@ +#include +#include +#include +#include "Obj.h" +#include "BroDocObj.h" + +BroDocObj::BroDocObj(const BroObj* obj, + std::list*& reST, + bool exported) + { + broObj = obj; + isExported = exported; + reST_doc_strings = reST; + reST = 0; + } + +BroDocObj::~BroDocObj() + { + delete reST_doc_strings; + } + +void BroDocObj::WriteReST(FILE* file) const + { + if ( reST_doc_strings ) + { + std::list::const_iterator it; + for ( it = reST_doc_strings->begin(); + it != reST_doc_strings->end(); ++it) + fprintf(file, "%s\n", it->c_str()); + } + + ODesc desc; + broObj->Describe(&desc); + fprintf(file, "%s\n", desc.Description()); + } diff --git a/src/BroDocObj.h b/src/BroDocObj.h new file mode 100644 index 0000000000..42bfc8882f --- /dev/null +++ b/src/BroDocObj.h @@ -0,0 +1,72 @@ +#ifndef brodocobj_h +#define brodocobj_h + +#include +#include +#include + +#include "Obj.h" + +class BroDocObj { +public: + /** + * BroDocObj constructor + * @param obj a pointer to a BroObj that is to be documented + * @param reST a reference to a pointer of a list of strings that + represent the reST documentation for the BroObj. The pointer + will be set to 0 after this constructor finishes. + * @param exported whether the BroObj is declared in an export section + */ + BroDocObj(const BroObj* obj, std::list*& reST, bool exported); + + /** + * BroDocObj destructor + * Deallocates the memory associated with the list of reST strings + */ + ~BroDocObj(); + + /** + * writes the reST representation of this object which includes + * 1) any of the "##" comments (stored internally in reST_doc_string) + * To make things easy, I think we should assume that the documenter + * writes their comments such that anything after ## is valid reST + * so that at parse time the ## is just stripped and the remainder + * is scheduled to be inserted as-is into the reST. + * TODO: prepare for some kind of filtering mechanism that transforms + * the reST as written into new reST before being written out. + * This allows for additional custom markup or macros when writing + * pure reST might be inconvenient. + * 2) a reST friendly description of the BroObj + * Could be implemented similar to the virtual BroObj::Describe(ODesc) + * E.g. all subclasses will now need to implement a reSTDescribe(ODesc) + * so that they can describe themselves in terms of the custom reST + * directives/roles that we'll later teach to Sphinx via a "bro domain". + * ID's should be able to implement the reSTDescribe(ODesc) function + * such that their namespace and attributes are output as well. + * @param The (already opened) file to write the reST to. + */ + void WriteReST(FILE* file) const; + + /** + * Check whether this documentation is part of the public API + * (The BroObj declaration was while in an export section). + * @return true if the BroObj was declared in an export section, else false + */ + bool IsPublicAPI() const { return isExported; } + + /** + * Return whether this object has documentation (## comments) + * @return true if the BroObj has comments associated with it + */ + bool HasDocumentation() const { return reST_doc_strings && + reST_doc_strings->size() > 0; } + +protected: + std::list* reST_doc_strings; + const BroObj* broObj; + bool isExported; + +private: +}; + +#endif diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 0f67dc173e..2a4a8e6dee 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -247,6 +247,9 @@ set(bro_SRCS BitTorrent.cc BitTorrentTracker.cc BPF_Program.cc + BroBifDoc.cc + BroDoc.cc + BroDocObj.cc BroString.cc CCL.cc ChunkedIO.cc diff --git a/src/main.cc b/src/main.cc index 82866302fd..38acfaa355 100644 --- a/src/main.cc +++ b/src/main.cc @@ -91,6 +91,7 @@ int optimize = 0; int do_notice_analysis = 0; int rule_bench = 0; int print_loaded_scripts = 0; +int generate_documentation = 0; SecondaryPath* secondary_path = 0; ConnCompressor* conn_compressor = 0; extern char version[]; @@ -145,6 +146,7 @@ void usage() fprintf(stderr, " -h|--help|-? | command line help\n"); fprintf(stderr, " -i|--iface | read from given interface\n"); fprintf(stderr, " -l|--print-scripts | print all loaded scripts\n"); + fprintf(stderr, " -Z|--doc-scripts | generate documentation for all loaded scripts\n"); fprintf(stderr, " -p|--prefix | add given prefix to policy file resolution\n"); fprintf(stderr, " -r|--readfile | read from given tcpdump file\n"); fprintf(stderr, " -y|--flowfile [=] | read from given flow file\n"); @@ -364,6 +366,7 @@ int main(int argc, char** argv) {"help", no_argument, 0, 'h'}, {"iface", required_argument, 0, 'i'}, {"print-scripts", no_argument, 0, 'l'}, + {"doc-scripts", no_argument, 0, 'Z'}, {"prefix", required_argument, 0, 'p'}, {"readfile", required_argument, 0, 'r'}, {"flowfile", required_argument, 0, 'y'}, @@ -440,7 +443,7 @@ int main(int argc, char** argv) opterr = 0; char opts[256]; - safe_strncpy(opts, "A:a:B:D:e:f:I:i:K:n:p:R:r:s:T:t:U:w:x:X:y:Y:z:CFGHLOPSWdghlv", + safe_strncpy(opts, "A:a:B:D:e:f:I:i:K:n:p:R:r:s:T:t:U:w:x:X:y:Y:z:CFGHLOPSWdghlvZ", sizeof(opts)); #ifdef USE_PERFTOOLS @@ -633,6 +636,10 @@ int main(int argc, char** argv) break; #endif + case 'Z': + generate_documentation = 1; + break; + #ifdef USE_IDMEF case 'n': fprintf(stderr, "Using IDMEF XML DTD from %s\n", optarg); diff --git a/src/parse.y b/src/parse.y index 3cf2c07b18..fbbb531050 100644 --- a/src/parse.y +++ b/src/parse.y @@ -73,6 +73,15 @@ #include "DNS.h" #include "RE.h" #include "Scope.h" +#include "BroDoc.h" +#include "BroDocObj.h" + +#include +#include + +extern BroDoc* current_reST_doc; +extern int generate_documentation; +extern std::list* reST_doc_comments; YYLTYPE GetCurrentLocation(); extern int yyerror(const char[]); @@ -785,7 +794,13 @@ formal_args_decl: decl: TOK_MODULE TOK_ID ';' - { current_module = $2; } + { + current_module = $2; + if ( generate_documentation ) + { + current_reST_doc->AddModule(current_module); + } + } | TOK_EXPORT '{' { is_export = true; } decl_list '}' { is_export = false; } diff --git a/src/scan.l b/src/scan.l index 9dc4d828e0..41c9fc08f7 100644 --- a/src/scan.l +++ b/src/scan.l @@ -15,11 +15,16 @@ #include "Debug.h" #include "PolicyFile.h" #include "broparse.h" +#include "BroDoc.h" +#include "BroBifDoc.h" #include +#include +#include extern YYLTYPE yylloc; // holds start line and column of token extern int print_loaded_scripts; +extern int generate_documentation; int nwarn = 0; int nerr = 0; @@ -35,6 +40,8 @@ int_list if_stack; int line_number = 1; int include_level = 0; const char* filename = 0; +BroDoc* current_reST_doc = 0; +static BroDoc* last_reST_doc = 0; char last_tok[128]; @@ -50,6 +57,21 @@ char last_tok[128]; // Files we have already scanned (or are in the process of scanning). static PList(char) files_scanned; +// reST documents that we've created (or have at least opened so far) +static std::list docs_generated; + +// reST comments (those starting with ##) seen so far +std::list* reST_doc_comments = 0; + +// print current contents of reST_doc_comments list to stderr +void print_current_reST_doc_comments(); + +// delete the reST_doc_comments list object +void clear_reST_doc_comments(); + +// adds changes to capture_filter to the current script's reST documentation +static void check_capture_filter_changes(); + class FileInfo { public: FileInfo(string restore_module = ""); @@ -60,6 +82,7 @@ public: const char* name; int line; int level; + BroDoc* doc; }; // A stack of input buffers we're scanning. file_stack[len-1] is the @@ -102,6 +125,30 @@ ESCSEQ (\\([^\n]|[0-7]+|x[[:xdigit:]]+)) %% +##!.* { + // Add this format of comments to the script documentation's "summary" + if ( generate_documentation ) + current_reST_doc->AddSummary(yytext + 3); + } + +##{OWS}Author:.* { + if ( generate_documentation ) + current_reST_doc->SetAuthor( + skip_whitespace( // Skip whitespace after "Author:" + skip_whitespace(yytext + 2) // Skip whitespace after ## + + 7) // Skip "Author:" + ); + } + +##[^#\n].* { + if ( generate_documentation ) + { + if ( ! reST_doc_comments ) + reST_doc_comments = new std::list(); + reST_doc_comments->push_back(yytext + 2); + } +} + #.* /* eat comments */ {WS} /* eat whitespace */ @@ -211,6 +258,17 @@ when return TOK_WHEN; @load{WS}{FILE} { const char* new_file = skip_whitespace(yytext + 5); // Skip "@load". + if ( generate_documentation ) + { + current_reST_doc->AddImport(new_file); + if ( reST_doc_comments ) + { + fprintf(stderr, "Warning: unconsumed reST documentation is being " + "discarded before doing '@load %s' in %s:\n", + new_file, current_reST_doc->GetSourceFileName()); + clear_reST_doc_comments(); + } + } (void) load_files_with_prefix(new_file); } @@ -446,7 +504,6 @@ static int load_files_with_prefix(const char* orig_file) strcpy(new_filename, file); f = search_for_file(new_filename, "bro", &full_filename); - delete [] new_filename; } @@ -496,6 +553,21 @@ static int load_files_with_prefix(const char* orig_file) // Don't delete the old filename - it's pointed to by // every BroObj created when parsing it. yylloc.filename = filename = full_filename; + + if ( generate_documentation ) + { + char* bifExtStart = strstr(full_filename, ".bif.bro"); + BroDoc* reST_doc; + + if ( bifExtStart ) + reST_doc = new BroBifDoc(full_filename); + else + reST_doc = new BroDoc(full_filename); + + docs_generated.push_back(reST_doc); + + current_reST_doc = reST_doc; + } } else @@ -655,6 +727,7 @@ int yywrap() // Stack is now empty. while ( input_files.length() > 0 ) { + check_capture_filter_changes(); if ( load_files_with_prefix(input_files[0]) ) { // Don't delete the filename - it's pointed to by @@ -667,6 +740,7 @@ int yywrap() // if any. (void) input_files.remove_nth(0); } + check_capture_filter_changes(); // Add redef statements for any X=Y command line parameters. if ( params.size() > 0 ) @@ -731,6 +805,18 @@ int yywrap() return 0; } + if ( generate_documentation ) + { + std::list::iterator it; + for ( it = docs_generated.begin(); it != docs_generated.end(); ++it ) + { + (*it)->WriteDocFile(); + delete *it; + } + docs_generated.clear(); + clear_reST_doc_comments(); + } + // Otherwise, we are done. return 1; } @@ -742,6 +828,7 @@ FileInfo::FileInfo(string arg_restore_module) name = ::filename; line = ::line_number; level = ::include_level; + doc = ::current_reST_doc; } FileInfo::~FileInfo() @@ -753,6 +840,8 @@ FileInfo::~FileInfo() yylloc.filename = filename = name; yylloc.first_line = yylloc.last_line = line_number = line; include_level = level; + last_reST_doc = current_reST_doc; + current_reST_doc = doc; if ( restore_module != "" ) current_module = restore_module; @@ -779,3 +868,40 @@ static void report_file() files_reported.append(copy_string(filename)); } +static void check_capture_filter_changes() + { + if ( generate_documentation ) + { + // Lookup the "capture_filters" identifier, if it has any + // defined value, add it to the script's reST documentation, + // and finally clear the table so it doesn't taint the + // documentation for subsequent scripts. + + ID* capture_filters = global_scope()->Lookup("capture_filters"); + if ( capture_filters ) + { + ODesc desc; + capture_filters->ID_Val()->Describe(&desc); + last_reST_doc->SetPacketFilter(desc.Description()); + ( (TableVal*) capture_filters->ID_Val() )->RemoveAll(); + } + } + } + +void print_current_reST_doc_comments() + { + if ( ! reST_doc_comments ) return; + std::list::iterator it; + for (it = reST_doc_comments->begin(); it != reST_doc_comments->end(); ++it) + fprintf(stderr, "##%s\n", it->c_str()); + } + +void clear_reST_doc_comments() + { + if ( ! reST_doc_comments ) return; + fprintf(stderr, "Warning: %lu unconsumed reST comments:\n", + reST_doc_comments->size()); + print_current_reST_doc_comments(); + delete reST_doc_comments; + reST_doc_comments = 0; + } From 0d7ffe998f164cc39abb545a0e5d6a5e6c3a36a6 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Sat, 5 Mar 2011 14:22:06 -0600 Subject: [PATCH 02/47] Debugging output from doc-framework now only in debug mode. Also a minor format fix for documenting empty string lists. --- src/BroDoc.cc | 11 +++++++++-- src/BroDoc.h | 1 + 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/BroDoc.cc b/src/BroDoc.cc index fb06a32308..8d734ab8e6 100644 --- a/src/BroDoc.cc +++ b/src/BroDoc.cc @@ -8,8 +8,9 @@ BroDoc::BroDoc(const std::string& sourcename) { +#ifdef DEBUG fprintf(stdout, "Documenting source: %s\n", sourcename.c_str()); - +#endif source_filename = sourcename.substr(sourcename.find_last_of('/') + 1); size_t ext_pos = source_filename.find_last_of('.'); @@ -35,8 +36,10 @@ BroDoc::BroDoc(const std::string& sourcename) if ( ! reST_file ) fprintf(stderr, "Failed to open %s", reST_filename.c_str()); +#ifdef DEBUG else fprintf(stdout, "Created reST document: %s\n", reST_filename.c_str()); +#endif } BroDoc::~BroDoc() @@ -104,7 +107,11 @@ void BroDoc::WriteStringList(const char* format, const char* last_format, const std::list& l) const { - if ( l.empty() ) return; + if ( l.empty() ) + { + WriteToDoc("\n"); + return; + } std::list::const_iterator it; std::list::const_iterator last = l.end(); last--; diff --git a/src/BroDoc.h b/src/BroDoc.h index 683a4cd5a5..adad10e3db 100644 --- a/src/BroDoc.h +++ b/src/BroDoc.h @@ -124,6 +124,7 @@ protected: /** * Writes out a list of strings to the reST document. + * If the list is empty, prints a newline character. * @param format A printf style format string for elements of the list * except for the last one in the list * @param last_format A printf style format string to use for the last From 1bad6e3a95b563dc2fe5aac28a115e5467d7ca7a Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Sat, 5 Mar 2011 16:11:26 -0600 Subject: [PATCH 03/47] Changes to auto-document dpd_config (port analysis) script settings. --- src/BroDoc.cc | 18 +++++++++++++----- src/BroDoc.h | 10 ++++++++++ src/scan.l | 49 ++++++++++++++++++++++++++++++++++++++++++++++--- 3 files changed, 69 insertions(+), 8 deletions(-) diff --git a/src/BroDoc.cc b/src/BroDoc.cc index 8d734ab8e6..715e8d9cf9 100644 --- a/src/BroDoc.cc +++ b/src/BroDoc.cc @@ -64,6 +64,13 @@ void BroDoc::SetPacketFilter(const std::string& s) packet_filter = s.substr(pos1 + 2, pos2 - 2); } +void BroDoc::AddPortAnalysis(const std::string& analyzer, + const std::string& ports) + { + std::string reST_string = analyzer + "::\n" + ports + "\n"; + port_analysis.push_back(reST_string); + } + void BroDoc::WriteDocFile() const { WriteToDoc("%s\n", source_filename.c_str()); @@ -83,6 +90,12 @@ void BroDoc::WriteDocFile() const WriteStringList(" :bro:script: `%s`\n", " :bro:script: `%s`\n\n", imports); + WriteSectionHeading("Port Analysis", '-'); + WriteStringList("%s", port_analysis); + + WriteSectionHeading("Packet Filter", '-'); + WriteToDoc("%s\n", packet_filter.c_str()); + WriteSectionHeading("Public Interface", '-'); WriteBroDocObjList(options, true, "Options", '~'); WriteBroDocObjList(state_vars, true, "State Variables", '~'); @@ -91,14 +104,9 @@ void BroDoc::WriteDocFile() const WriteBroDocObjList(events, true, "Events", '~'); WriteBroDocObjList(functions, true, "Functions", '~'); - WriteSectionHeading("Packet Filter", '-'); - WriteToDoc("%s\n", packet_filter.c_str()); - WriteSectionHeading("Private Interface", '-'); - WriteBroDocObjList(options, false, "Options", '~'); WriteBroDocObjList(state_vars, false, "State Variables", '~'); WriteBroDocObjList(types, false, "Types", '~'); - WriteBroDocObjList(notices, false, "Notices", '~'); WriteBroDocObjList(events, false, "Events", '~'); WriteBroDocObjList(functions, false, "Functions", '~'); } diff --git a/src/BroDoc.h b/src/BroDoc.h index adad10e3db..37adcc8b70 100644 --- a/src/BroDoc.h +++ b/src/BroDoc.h @@ -73,6 +73,15 @@ public: */ void SetPacketFilter(const std::string& s); + /** + * Schedules documentation of a given set of ports being associated + * with a particular analyzer as a result of the current script + * being loaded -- the way the "dpd_config" table is changed. + * @param analyzer An analyzer that changed the "dpd_config" table. + * @param ports The set of ports assigned to the analyzer in table. + */ + void AddPortAnalysis(const std::string& analyzer, const std::string& ports); + /** * Sets the author of the script. * The scanner should call this when it sees "## Author: ..." @@ -114,6 +123,7 @@ protected: std::list modules; std::list summary; std::list imports; + std::list port_analysis; std::list options; // identifiers with &redef attr std::list state_vars; // identifiers without &redef? diff --git a/src/scan.l b/src/scan.l index 41c9fc08f7..d47986b144 100644 --- a/src/scan.l +++ b/src/scan.l @@ -17,6 +17,8 @@ #include "broparse.h" #include "BroDoc.h" #include "BroBifDoc.h" +#include "Analyzer.h" +#include "AnalyzerTags.h" #include #include @@ -33,8 +35,6 @@ int nruntime = 0; // Track the @if... depth. ptr_compat_int current_depth = 0; -declare(List,ptr_compat_int); -typedef List(ptr_compat_int) int_list; int_list if_stack; int line_number = 1; @@ -72,6 +72,9 @@ void clear_reST_doc_comments(); // adds changes to capture_filter to the current script's reST documentation static void check_capture_filter_changes(); +// adds changes to dpd_config to the current script's reST documentation +static void check_dpd_config_changes(); + class FileInfo { public: FileInfo(string restore_module = ""); @@ -728,6 +731,7 @@ int yywrap() while ( input_files.length() > 0 ) { check_capture_filter_changes(); + check_dpd_config_changes(); if ( load_files_with_prefix(input_files[0]) ) { // Don't delete the filename - it's pointed to by @@ -741,6 +745,7 @@ int yywrap() (void) input_files.remove_nth(0); } check_capture_filter_changes(); + check_dpd_config_changes(); // Add redef statements for any X=Y command line parameters. if ( params.size() > 0 ) @@ -883,11 +888,49 @@ static void check_capture_filter_changes() ODesc desc; capture_filters->ID_Val()->Describe(&desc); last_reST_doc->SetPacketFilter(desc.Description()); - ( (TableVal*) capture_filters->ID_Val() )->RemoveAll(); + capture_filters->ID_Val()->AsTableVal()->RemoveAll(); } } } +static void check_dpd_config_changes() + { + if ( generate_documentation ) + { + // Lookup the "dpd_config" identifier, if it has any + // defined value, add it to the script's documentation, + // and clear the table so that it doesn't taint the + // documentation for subsequent scripts. + ID* dpd_config = global_scope()->Lookup("dpd_config"); + if ( ! dpd_config ) return; + + TableVal* dpd_table = dpd_config->ID_Val()->AsTableVal(); + ListVal* dpd_list = dpd_table->ConvertToList(); + for ( int i = 0; i < dpd_list->Length(); ++i ) + { + Val* key = dpd_list->Index(i); + if ( ! key ) continue; + Val* v = dpd_table->Lookup(key); + if ( ! v ) continue; + + int tag = key->AsListVal()->Index(0)->AsCount(); + ODesc valdesc; + valdesc.PushIndent(); + v->Describe(&valdesc); + + if ( tag < AnalyzerTag::Error || tag > AnalyzerTag::LastAnalyzer ) + { + fprintf(stderr, "Warning: skipped bad analyzer tag: %i\n", tag); + continue; + } + last_reST_doc->AddPortAnalysis( + Analyzer::GetTagName((AnalyzerTag::Tag)tag), + valdesc.Description()); + } + dpd_table->RemoveAll(); + } + } + void print_current_reST_doc_comments() { if ( ! reST_doc_comments ) return; From 1c962cbb895bd402459ec76eafcb1fb1b6a7b5e4 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Wed, 9 Mar 2011 14:53:22 -0600 Subject: [PATCH 04/47] Changes to make declared types track their identifier. Only occurs when in documentation mode. --- src/Type.cc | 8 ++++++-- src/Type.h | 9 ++++++++- src/Var.cc | 39 +++++++++++++++++++++++++++++++++++++-- 3 files changed, 51 insertions(+), 5 deletions(-) diff --git a/src/Type.cc b/src/Type.cc index ce3bbc52af..e5e3103fb8 100644 --- a/src/Type.cc +++ b/src/Type.cc @@ -10,6 +10,8 @@ #include "Scope.h" #include "Serializer.h" +extern int generate_documentation; + const char* type_name(TypeTag t) { static char errbuf[512]; @@ -44,6 +46,7 @@ BroType::BroType(TypeTag t, bool arg_base_type) tag = t; is_network_order = 0; base_type = arg_base_type; + type_id = 0; switch ( tag ) { case TYPE_VOID: @@ -195,8 +198,9 @@ BroType* BroType::Unserialize(UnserialInfo* info, TypeTag want) if ( ! t ) return 0; - // For base types, we return our current instance. - if ( t->base_type ) + // For base types, we return our current instance + // if not in "documentation mode". + if ( t->base_type && ! generate_documentation ) { BroType* t2 = ::base_type(TypeTag(t->tag)); Unref(t); diff --git a/src/Type.h b/src/Type.h index fe4f7a5eb5..6dd3364ad4 100644 --- a/src/Type.h +++ b/src/Type.h @@ -206,8 +206,11 @@ public: bool Serialize(SerialInfo* info) const; static BroType* Unserialize(UnserialInfo* info, TypeTag want = TYPE_ANY); + void SetTypeID(const ID* id) { type_id = id; } + const ID* GetTypeID() const { return type_id; } + protected: - BroType() { } + BroType() { type_id = 0; } void SetError(); @@ -218,6 +221,10 @@ private: InternalTypeTag internal_tag; bool is_network_order; bool base_type; + + // This type_id field is only used by the documentation framework to + // track the names of declared types. + const ID* type_id; }; class TypeList : public BroType { diff --git a/src/Var.cc b/src/Var.cc index 04ef78c14c..324ffb05e8 100644 --- a/src/Var.cc +++ b/src/Var.cc @@ -12,6 +12,8 @@ #include "RemoteSerializer.h" #include "EventRegistry.h" +extern int generate_documentation; + static Val* init_val(Expr* init, const BroType* t, Val* aggr) { return init->InitVal(t, aggr); @@ -217,11 +219,44 @@ extern Expr* add_and_assign_local(ID* id, Expr* init, Val* val) void add_type(ID* id, BroType* t, attr_list* attr, int /* is_event */) { - id->SetType(t); + BroType* tnew = t; + + // In "documentation mode", we'd like to to be able to associate + // an identifier name with a declared type. Dealing with declared + // types that are "aliases" to a builtin type requires that the BroType + // is cloned before setting the identifier name that resolves to it. + // And still this is not enough to document cases where the declared type + // is an alias for another declared type -- but that's not a natural/common + // practice. If documenting that corner case is desired, one way + // is to add an ID* to class ID that tracks aliases and set it here if + // t->GetTypeID() is true. + if ( generate_documentation ) + { + SerializationFormat* form = new BinarySerializationFormat(); + form->StartWrite(); + CloneSerializer ss(form); + SerialInfo sinfo(&ss); + sinfo.cache = false; + + t->Serialize(&sinfo); + char* data; + uint32 len = form->EndWrite(&data); + form->StartRead(data, len); + + UnserialInfo uinfo(&ss); + uinfo.cache = false; + tnew = t->Unserialize(&uinfo); + + delete [] data; + + tnew->SetTypeID(id); + } + + id->SetType(tnew); id->MakeType(); if ( attr ) - id->SetAttrs(new Attributes(attr, t)); + id->SetAttrs(new Attributes(attr, tnew)); } void begin_func(ID* id, const char* module_name, function_flavor flavor, From 1310f2387c3fb9feb0a50119f08c117c90cd41d2 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Wed, 9 Mar 2011 15:10:05 -0600 Subject: [PATCH 05/47] Changing BroDocObj wrapper class to wrap ID instead of BroObj. --- src/BroDocObj.cc | 11 ++++------- src/BroDocObj.h | 31 +++++++++++++------------------ 2 files changed, 17 insertions(+), 25 deletions(-) diff --git a/src/BroDocObj.cc b/src/BroDocObj.cc index 645c4f1475..2ff5d88f5e 100644 --- a/src/BroDocObj.cc +++ b/src/BroDocObj.cc @@ -1,15 +1,12 @@ #include #include #include -#include "Obj.h" +#include "ID.h" #include "BroDocObj.h" -BroDocObj::BroDocObj(const BroObj* obj, - std::list*& reST, - bool exported) +BroDocObj::BroDocObj(const ID* id, std::list*& reST) { - broObj = obj; - isExported = exported; + broID = id; reST_doc_strings = reST; reST = 0; } @@ -30,6 +27,6 @@ void BroDocObj::WriteReST(FILE* file) const } ODesc desc; - broObj->Describe(&desc); + broID->DescribeExtended(&desc); fprintf(file, "%s\n", desc.Description()); } diff --git a/src/BroDocObj.h b/src/BroDocObj.h index 42bfc8882f..0fdbcca8a5 100644 --- a/src/BroDocObj.h +++ b/src/BroDocObj.h @@ -5,19 +5,18 @@ #include #include -#include "Obj.h" +#include "ID.h" class BroDocObj { public: /** * BroDocObj constructor - * @param obj a pointer to a BroObj that is to be documented + * @param id a pointer to an identifier that is to be documented * @param reST a reference to a pointer of a list of strings that - represent the reST documentation for the BroObj. The pointer + represent the reST documentation for the ID. The pointer will be set to 0 after this constructor finishes. - * @param exported whether the BroObj is declared in an export section */ - BroDocObj(const BroObj* obj, std::list*& reST, bool exported); + BroDocObj(const ID* id, std::list*& reST); /** * BroDocObj destructor @@ -36,35 +35,31 @@ public: * the reST as written into new reST before being written out. * This allows for additional custom markup or macros when writing * pure reST might be inconvenient. - * 2) a reST friendly description of the BroObj - * Could be implemented similar to the virtual BroObj::Describe(ODesc) - * E.g. all subclasses will now need to implement a reSTDescribe(ODesc) - * so that they can describe themselves in terms of the custom reST - * directives/roles that we'll later teach to Sphinx via a "bro domain". - * ID's should be able to implement the reSTDescribe(ODesc) function - * such that their namespace and attributes are output as well. + * 2) a reST friendly description of the ID + * Could be implemented similar to the ID::DescribeExtended(ODesc) + * expect with new directives/roles that we'll later teach to Sphinx + * via a "bro domain". * @param The (already opened) file to write the reST to. */ void WriteReST(FILE* file) const; /** * Check whether this documentation is part of the public API - * (The BroObj declaration was while in an export section). - * @return true if the BroObj was declared in an export section, else false + * (The declaration was while in an export section). + * @return true if the ID was declared in an export section, else false */ - bool IsPublicAPI() const { return isExported; } + bool IsPublicAPI() const { return broID->IsExport(); } /** * Return whether this object has documentation (## comments) - * @return true if the BroObj has comments associated with it + * @return true if the ID has comments associated with it */ bool HasDocumentation() const { return reST_doc_strings && reST_doc_strings->size() > 0; } protected: std::list* reST_doc_strings; - const BroObj* broObj; - bool isExported; + const ID* broID; private: }; From 287a3a3cb82643bc81b948ef83ea67a70d58153a Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Wed, 9 Mar 2011 15:47:58 -0600 Subject: [PATCH 06/47] Changes to parser to pass along identifiers to doc framework. --- src/parse.y | 57 ++++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 50 insertions(+), 7 deletions(-) diff --git a/src/parse.y b/src/parse.y index 49fe9c8ff8..5f5b37a526 100644 --- a/src/parse.y +++ b/src/parse.y @@ -849,34 +849,71 @@ decl: { current_module = $2; if ( generate_documentation ) - { current_reST_doc->AddModule(current_module); - } } | TOK_EXPORT '{' { is_export = true; } decl_list '}' { is_export = false; } | TOK_GLOBAL global_id opt_type init_class opt_init opt_attr ';' - { add_global($2, $3, $4, $5, $6, VAR_REGULAR); } + { + add_global($2, $3, $4, $5, $6, VAR_REGULAR); + if ( generate_documentation ) + { + ID* id = $2; + if ( id->Type()->Tag() == TYPE_FUNC ) + if ( id->Type()->AsFuncType()->IsEvent() ) + current_reST_doc->AddEvent( + new BroDocObj(id, reST_doc_comments)); + else + current_reST_doc->AddFunction( + new BroDocObj(id, reST_doc_comments)); + else + current_reST_doc->AddStateVar( + new BroDocObj(id, reST_doc_comments)); + } + } | TOK_CONST global_id opt_type init_class opt_init opt_attr ';' - { add_global($2, $3, $4, $5, $6, VAR_CONST); } + { + add_global($2, $3, $4, $5, $6, VAR_CONST); + if ( generate_documentation ) + if ( $2->FindAttr(ATTR_REDEF) ) + current_reST_doc->AddOption( + new BroDocObj($2, reST_doc_comments)); + } | TOK_REDEF global_id opt_type init_class opt_init opt_attr ';' - { add_global($2, $3, $4, $5, $6, VAR_REDEF); } + { + add_global($2, $3, $4, $5, $6, VAR_REDEF); + if ( generate_documentation ) + { + // TODO: eventually handle type redefs that add record fields + } + } | TOK_REDEF TOK_ENUM global_id TOK_ADD_TO '{' { parser_redef_enum($3); } enum_body '}' ';' - { /* no action */ } + { + /* no action */ + // TODO: handle "Notice" redefinitions for doc framework + } | TOK_TYPE global_id ':' refined_type opt_attr ';' { add_type($2, $4, $5, 0); + if ( generate_documentation ) + current_reST_doc->AddType( + new BroDocObj($2, reST_doc_comments)); } | TOK_EVENT event_id ':' refined_type opt_attr ';' - { add_type($2, $4, $5, 1); } + { + add_type($2, $4, $5, 1); + if ( generate_documentation ) + current_reST_doc->AddEvent( + new BroDocObj($2, reST_doc_comments)); + } | func_hdr func_body { } @@ -903,12 +940,18 @@ func_hdr: begin_func($2, current_module.c_str(), FUNC_FLAVOR_FUNCTION, 0, $3); $$ = $3; + if ( generate_documentation ) + current_reST_doc->AddFunction( + new BroDocObj($2, reST_doc_comments)); } | TOK_EVENT event_id func_params { begin_func($2, current_module.c_str(), FUNC_FLAVOR_EVENT, 0, $3); $$ = $3; + if ( generate_documentation ) + current_reST_doc->AddEvent( + new BroDocObj($2, reST_doc_comments)); } | TOK_REDEF TOK_EVENT event_id func_params { From 15fd5297a33c0cdc7b3066837892a3e0a3d8ac70 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Thu, 10 Mar 2011 16:22:45 -0600 Subject: [PATCH 07/47] Fixes for declared type ID tracking & adding DescribeReST()'s Changed BroType to track a char* instead of an ID* that represents the declared type's identifier. It was also necessary to serialize this information or else it can be lost (e.g. FieldDecl's in RecordType always seem to get serialized at some point). DescribeReST() functions added to many classes to get the output closer to being reST compatible; still needs tweaking for Sphinx (reST->HTML) compatibility. --- src/Attr.cc | 20 ++++++++ src/Attr.h | 2 + src/BroDocObj.cc | 5 +- src/ID.cc | 35 +++++++++++++ src/ID.h | 2 + src/Type.cc | 128 +++++++++++++++++++++++++++++++++++++++++++++++ src/Type.h | 14 ++++-- src/Val.cc | 10 ++++ src/Val.h | 2 + src/Var.cc | 2 +- 10 files changed, 214 insertions(+), 6 deletions(-) diff --git a/src/Attr.cc b/src/Attr.cc index 5a83d0501b..31a01b3070 100644 --- a/src/Attr.cc +++ b/src/Attr.cc @@ -49,6 +49,17 @@ void Attr::Describe(ODesc* d) const } } +void Attr::DescribeReST(ODesc* d) const + { + d->Add(".. bro:attr:: "); + AddTag(d); + if ( expr ) + { + d->Add("="); + expr->Describe(d); + } + } + void Attr::AddTag(ODesc* d) const { if ( d->IsBinary() ) @@ -161,6 +172,15 @@ void Attributes::Describe(ODesc* d) const } } +void Attributes::DescribeReST(ODesc* d) const + { + loop_over_list(*attrs, i) + { + d->NL(); + (*attrs)[i]->DescribeReST(d); + } + } + void Attributes::CheckAttr(Attr* a) { switch ( a->Tag() ) { diff --git a/src/Attr.h b/src/Attr.h index 73fb101841..ccc6b0f5d1 100644 --- a/src/Attr.h +++ b/src/Attr.h @@ -51,6 +51,7 @@ public: { return tag == ATTR_REDEF || tag == ATTR_OPTIONAL; } void Describe(ODesc* d) const; + void DescribeReST(ODesc* d) const; protected: void AddTag(ODesc* d) const; @@ -73,6 +74,7 @@ public: void RemoveAttr(attr_tag t); void Describe(ODesc* d) const; + void DescribeReST(ODesc* d) const; attr_list* Attrs() { return attrs; } diff --git a/src/BroDocObj.cc b/src/BroDocObj.cc index 2ff5d88f5e..a23dec16be 100644 --- a/src/BroDocObj.cc +++ b/src/BroDocObj.cc @@ -27,6 +27,7 @@ void BroDocObj::WriteReST(FILE* file) const } ODesc desc; - broID->DescribeExtended(&desc); - fprintf(file, "%s\n", desc.Description()); + desc.SetQuotes(1); + broID->DescribeReST(&desc); + fprintf(file, "%s\n\n", desc.Description()); } diff --git a/src/ID.cc b/src/ID.cc index c908a8c3c4..03403af86f 100644 --- a/src/ID.cc +++ b/src/ID.cc @@ -607,6 +607,41 @@ void ID::DescribeExtended(ODesc* d) const } } +void ID::DescribeReST(ODesc* d) const + { + d->Add(".. bro:id:: "); + d->Add(name); + d->PushIndent(); + + if ( type ) + { + d->Add(".. bro:type:: "); + if ( ! is_type && type->GetTypeID() ) + { + d->Add("`"); + d->Add(type->GetTypeID()); + d->Add("`"); + } + else + type->DescribeReST(d); + } + + if ( attrs ) + { + attrs->DescribeReST(d); + } + + if ( val && type && + type->InternalType() != TYPE_INTERNAL_OTHER && + type->InternalType() != TYPE_INTERNAL_VOID ) + { + d->NL(); + d->Add(".. bro:val:: "); + val->DescribeReST(d); + } + + } + #ifdef DEBUG void ID::UpdateValID() { diff --git a/src/ID.h b/src/ID.h index f576232b0e..7ebaf27918 100644 --- a/src/ID.h +++ b/src/ID.h @@ -84,6 +84,8 @@ public: void Describe(ODesc* d) const; // Adds type and value to description. void DescribeExtended(ODesc* d) const; + // Produces a description that's reST-ready + void DescribeReST(ODesc* d) const; bool Serialize(SerialInfo* info) const; static ID* Unserialize(UnserialInfo* info); diff --git a/src/Type.cc b/src/Type.cc index e5e3103fb8..cc41bcc5ec 100644 --- a/src/Type.cc +++ b/src/Type.cc @@ -108,6 +108,11 @@ BroType::BroType(TypeTag t, bool arg_base_type) } +BroType::~BroType() + { + if ( type_id ) delete [] type_id; + } + int BroType::MatchesIndex(ListExpr*& /* index */) const { return DOES_NOT_MATCH_INDEX; @@ -142,6 +147,11 @@ void BroType::Describe(ODesc* d) const } } +void BroType::DescribeReST(ODesc* d) const + { + Describe(d); + } + void BroType::SetError() { tag = TYPE_ERROR; @@ -234,6 +244,11 @@ bool BroType::DoSerialize(SerialInfo* info) const void* null = NULL; SERIALIZE(null); + if ( generate_documentation ) + { + SERIALIZE_OPTIONAL_STR(type_id); + } + info->s->WriteCloseTag("Type"); return true; @@ -264,6 +279,11 @@ bool BroType::DoUnserialize(UnserialInfo* info) // attributes_type" for backwards compatibility. UNSERIALIZE_OPTIONAL(not_used_either, BroType::Unserialize(info, TYPE_RECORD)); + if ( generate_documentation ) + { + UNSERIALIZE_OPTIONAL_STR(type_id); + } + return true; } @@ -417,6 +437,44 @@ void IndexType::Describe(ODesc* d) const } } +void IndexType::DescribeReST(ODesc* d) const + { + if ( IsSet() ) + d->Add("set"); + else + d->Add(type_name(Tag())); + + d->Add("["); + loop_over_list(*IndexTypes(), i) + { + if ( i > 0 ) + d->Add(","); + const BroType* t = (*IndexTypes())[i]; + if ( t->GetTypeID() ) + { + d->Add("`"); + d->Add(t->GetTypeID()); + d->Add("`"); + } + else + t->DescribeReST(d); + } + d->Add("]"); + + if ( yield_type ) + { + d->Add(" of "); + if ( yield_type->GetTypeID() ) + { + d->Add("`"); + d->Add(yield_type->GetTypeID()); + d->Add("`"); + } + else + yield_type->DescribeReST(d); + } + } + bool IndexType::IsSubNetIndex() const { const type_list* types = indices->Types(); @@ -651,6 +709,26 @@ void FuncType::Describe(ODesc* d) const } } +void FuncType::DescribeReST(ODesc* d) const + { + d->Add(is_event ? "event" : "function"); + d->Add("("); + args->DescribeFieldsReST(d, true); + d->Add(")"); + if ( yield ) + { + d->AddSP(" :"); + if ( yield->GetTypeID() ) + { + d->Add("`"); + d->Add(yield->GetTypeID()); + d->Add("`"); + } + else + yield->DescribeReST(d); + } + } + IMPLEMENT_SERIAL(FuncType, SER_FUNC_TYPE); bool FuncType::DoSerialize(SerialInfo* info) const @@ -902,6 +980,12 @@ void RecordType::Describe(ODesc* d) const } } +void RecordType::DescribeReST(ODesc* d) const + { + d->Add("record"); + DescribeFieldsReST(d, false); + } + void RecordType::DescribeFields(ODesc* d) const { if ( d->IsReadable() ) @@ -941,6 +1025,37 @@ void RecordType::DescribeFields(ODesc* d) const } } +void RecordType::DescribeFieldsReST(ODesc* d, bool func_args) const + { + if ( ! func_args ) + d->PushIndent(); + + for ( int i = 0; i < num_fields; ++i ) + { + const TypeDecl* td = FieldDecl(i); + if ( ! func_args ) + d->Add(":bro:field: "); + + d->Add(td->id); + d->Add(": "); + d->Add(":bro:type: "); + if ( td->type->GetTypeID() ) + { + d->Add("`"); + d->Add(td->type->GetTypeID()); + d->Add("`"); + } + else + td->type->DescribeReST(d); + + if ( i + 1 != num_fields ) + if ( func_args ) + d->Add(", "); + else + d->NL(); + } + } + IMPLEMENT_SERIAL(RecordType, SER_RECORD_TYPE) bool RecordType::DoSerialize(SerialInfo* info) const @@ -1176,6 +1291,19 @@ const char* EnumType::Lookup(bro_int_t value) return 0; } +void EnumType::DescribeReST(ODesc* d) const + { + d->Add(type_name(Tag())); + d->PushIndent(); + for ( NameMap::const_iterator it = names.begin(); it != names.end(); ) + { + d->Add(".. bro:enum:: "); + d->Add(it->first); + if ( ++it != names.end() ) + d->NL(); + } + } + IMPLEMENT_SERIAL(EnumType, SER_ENUM_TYPE); bool EnumType::DoSerialize(SerialInfo* info) const diff --git a/src/Type.h b/src/Type.h index 6dd3364ad4..89652ddb01 100644 --- a/src/Type.h +++ b/src/Type.h @@ -67,6 +67,7 @@ const int MATCHES_INDEX_VECTOR = 2; class BroType : public BroObj { public: BroType(TypeTag tag, bool base_type = false); + ~BroType(); TypeTag Tag() const { return tag; } InternalTypeTag InternalType() const { return internal_tag; } @@ -200,14 +201,15 @@ public: BroType* Ref() { ::Ref(this); return this; } virtual void Describe(ODesc* d) const; + virtual void DescribeReST(ODesc* d) const; virtual unsigned MemoryAllocation() const; bool Serialize(SerialInfo* info) const; static BroType* Unserialize(UnserialInfo* info, TypeTag want = TYPE_ANY); - void SetTypeID(const ID* id) { type_id = id; } - const ID* GetTypeID() const { return type_id; } + void SetTypeID(const char* id) { type_id = id; } + const char* GetTypeID() const { return type_id; } protected: BroType() { type_id = 0; } @@ -224,7 +226,7 @@ private: // This type_id field is only used by the documentation framework to // track the names of declared types. - const ID* type_id; + const char* type_id; }; class TypeList : public BroType { @@ -280,6 +282,7 @@ public: BroType* YieldType(); void Describe(ODesc* d) const; + void DescribeReST(ODesc* d) const; // Returns true if this table is solely indexed by subnet. bool IsSubNetIndex() const; @@ -354,6 +357,7 @@ public: ID* GetReturnValueID() const; void Describe(ODesc* d) const; + void DescribeReST(ODesc* d) const; protected: FuncType() { args = 0; arg_types = 0; yield = 0; return_value = 0; } @@ -417,7 +421,9 @@ public: int NumFields() const { return num_fields; } void Describe(ODesc* d) const; + void DescribeReST(ODesc* d) const; void DescribeFields(ODesc* d) const; + void DescribeFieldsReST(ODesc* d, bool func_args) const; protected: RecordType() { fields = 0; base = 0; types = 0; } @@ -475,6 +481,8 @@ public: bro_int_t Lookup(const string& module_name, const char* name); const char* Lookup(bro_int_t value); // Returns 0 if not found + void DescribeReST(ODesc* d) const; + protected: DECLARE_SERIAL(EnumType) diff --git a/src/Val.cc b/src/Val.cc index eab7842e74..f4c471b8dc 100644 --- a/src/Val.cc +++ b/src/Val.cc @@ -574,6 +574,11 @@ void Val::Describe(ODesc* d) const Val::ValDescribe(d); } +void Val::DescribeReST(ODesc* d) const + { + ValDescribeReST(d); + } + void Val::ValDescribe(ODesc* d) const { if ( d->IsReadable() && type->Tag() == TYPE_BOOL ) @@ -615,6 +620,11 @@ void Val::ValDescribe(ODesc* d) const } } +void Val::ValDescribeReST(ODesc* d) const + { + ValDescribe(d); + } + MutableVal::~MutableVal() { for ( list::iterator i = aliases.begin(); i != aliases.end(); ++i ) diff --git a/src/Val.h b/src/Val.h index 39be5f0e6a..0d759360a7 100644 --- a/src/Val.h +++ b/src/Val.h @@ -313,6 +313,7 @@ public: } void Describe(ODesc* d) const; + void DescribeReST(ODesc* d) const; bool Serialize(SerialInfo* info) const; static Val* Unserialize(UnserialInfo* info, TypeTag type = TYPE_ANY) @@ -347,6 +348,7 @@ protected: } virtual void ValDescribe(ODesc* d) const; + virtual void ValDescribeReST(ODesc* d) const; Val(TypeTag t) { diff --git a/src/Var.cc b/src/Var.cc index 324ffb05e8..2b94c45461 100644 --- a/src/Var.cc +++ b/src/Var.cc @@ -249,7 +249,7 @@ void add_type(ID* id, BroType* t, attr_list* attr, int /* is_event */) delete [] data; - tnew->SetTypeID(id); + tnew->SetTypeID(copy_string(id->Name())); } id->SetType(tnew); From 4b0eb8127df6bb246e7bea81e926925a1e004862 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Mon, 14 Mar 2011 11:50:46 -0500 Subject: [PATCH 08/47] Changes to add comments per enum or record type field. --- src/SerialObj.h | 23 ++++++++++++++++ src/Type.cc | 71 ++++++++++++++++++++++++++++++++++++++++++++++++- src/Type.h | 9 +++++++ 3 files changed, 102 insertions(+), 1 deletion(-) diff --git a/src/SerialObj.h b/src/SerialObj.h index 9c02b21e5b..4a12d53fe6 100644 --- a/src/SerialObj.h +++ b/src/SerialObj.h @@ -330,6 +330,29 @@ public: dst = 0; \ } +#define UNSERIALIZE_OPTIONAL_STR_DEL(dst, del) \ + { \ + bool has_it; \ + if ( ! info->s->Read(&has_it, "has_" #dst) ) \ + { \ + delete del; \ + return 0; \ + } \ + \ + if ( has_it ) \ + { \ + info->s->Read(&dst, 0, "has_" #dst); \ + if ( ! dst ) \ + { \ + delete del; \ + return 0; \ + } \ + } \ + \ + else \ + dst = 0; \ + } + #define UNSERIALIZE_OPTIONAL_STATIC(dst, unserialize, del) \ { \ bool has_it; \ diff --git a/src/Type.cc b/src/Type.cc index cc41bcc5ec..67b0bb7be9 100644 --- a/src/Type.cc +++ b/src/Type.cc @@ -767,6 +767,7 @@ TypeDecl::TypeDecl(BroType* t, const char* i, attr_list* arg_attrs) type = t; attrs = arg_attrs ? new Attributes(arg_attrs, t) : 0; id = i; + comment = 0; } TypeDecl::~TypeDecl() @@ -774,6 +775,7 @@ TypeDecl::~TypeDecl() Unref(type); Unref(attrs); delete [] id; + if ( comment ) delete [] comment; } bool TypeDecl::Serialize(SerialInfo* info) const @@ -783,7 +785,15 @@ bool TypeDecl::Serialize(SerialInfo* info) const SERIALIZE_OPTIONAL(attrs); - return type->Serialize(info) && SERIALIZE(id); + if ( ! (type->Serialize(info) && SERIALIZE(id)) ) + return false; + + if ( generate_documentation ) + { + SERIALIZE_OPTIONAL_STR(comment); + } + + return true; } TypeDecl* TypeDecl::Unserialize(UnserialInfo* info) @@ -799,6 +809,11 @@ TypeDecl* TypeDecl::Unserialize(UnserialInfo* info) return 0; } + if ( generate_documentation ) + { + UNSERIALIZE_OPTIONAL_STR_DEL(t->comment, t); + } + return t; } @@ -1048,6 +1063,14 @@ void RecordType::DescribeFieldsReST(ODesc* d, bool func_args) const else td->type->DescribeReST(d); + if ( ! func_args ) + if ( td->comment ) + { + d->PushIndent(); + d->Add(td->comment); + d->PopIndent(); + } + if ( i + 1 != num_fields ) if ( func_args ) d->Add(", "); @@ -1211,6 +1234,12 @@ EnumType::~EnumType() { for ( NameMap::iterator iter = names.begin(); iter != names.end(); ++iter ) delete [] iter->first; + + for ( CommentMap::iterator iter = comments.begin(); iter != comments.end(); ++iter ) + { + delete [] iter->first; + delete [] iter->second; + } } // Note, we use error() here (not Error()) to include the current script @@ -1242,6 +1271,12 @@ void EnumType::AddName(const string& module_name, const char* name, bro_int_t va AddNameInternal(module_name, name, val, is_export); } +void EnumType::AddComment(const string& module_name, const char* name, const char* comment) + { + string fullname = make_full_var_name(module_name.c_str(), name); + comments[copy_string(fullname.c_str())] = copy_string(comment); + } + void EnumType::AddNameInternal(const string& module_name, const char* name, bro_int_t val, bool is_export) { ID *id; @@ -1299,6 +1334,13 @@ void EnumType::DescribeReST(ODesc* d) const { d->Add(".. bro:enum:: "); d->Add(it->first); + CommentMap::const_iterator cmnt_it = comments.find(it->first); + if ( cmnt_it != comments.end() ) + { + d->PushIndent(); + d->Add(cmnt_it->second); + d->PopIndent(); + } if ( ++it != names.end() ) d->NL(); } @@ -1315,6 +1357,10 @@ bool EnumType::DoSerialize(SerialInfo* info) const SERIALIZE(false)) ) return false; + if ( generate_documentation ) + if ( ! (SERIALIZE((unsigned int) comments.size())) ) + return false; + for ( NameMap::const_iterator iter = names.begin(); iter != names.end(); ++iter ) { @@ -1322,6 +1368,14 @@ bool EnumType::DoSerialize(SerialInfo* info) const return false; } + if ( generate_documentation ) + for ( CommentMap::const_iterator it = comments.begin(); + it != comments.end(); ++ it ) + { + if ( ! SERIALIZE(it->first) || ! SERIALIZE(it->second) ) + return false; + } + return true; } @@ -1330,6 +1384,7 @@ bool EnumType::DoUnserialize(UnserialInfo* info) DO_UNSERIALIZE(BroType); unsigned int len; + unsigned int cmnt_len; bool dummy; if ( ! UNSERIALIZE(&counter) || ! UNSERIALIZE(&len) || @@ -1337,6 +1392,10 @@ bool EnumType::DoUnserialize(UnserialInfo* info) ! UNSERIALIZE(&dummy) ) return false; + if ( generate_documentation ) + if ( ! UNSERIALIZE(&cmnt_len) ) + return false; + while ( len-- ) { const char* name; @@ -1347,6 +1406,16 @@ bool EnumType::DoUnserialize(UnserialInfo* info) names[name] = val; } + if ( generate_documentation ) + while ( cmnt_len-- ) + { + const char* cmnt; + const char* name; + if ( ! (UNSERIALIZE_STR(&name, 0) && UNSERIALIZE_STR(&cmnt, 0)) ) + return false; + comments[name] = cmnt; + } + return true; } diff --git a/src/Type.h b/src/Type.h index 89652ddb01..1bffa0bbe2 100644 --- a/src/Type.h +++ b/src/Type.h @@ -384,6 +384,9 @@ public: BroType* type; Attributes* attrs; const char* id; + + // comments are only filled when in "documentation mode" + const char* comment; }; class RecordField { @@ -477,6 +480,8 @@ public: // added that aren't likewise explicitly initalized. void AddName(const string& module_name, const char* name, bro_int_t val, bool is_export); + void AddComment(const string& module_name, const char* name, const char* comment); + // -1 indicates not found. bro_int_t Lookup(const string& module_name, const char* name); const char* Lookup(bro_int_t value); // Returns 0 if not found @@ -491,6 +496,10 @@ protected: typedef std::map< const char*, bro_int_t, ltstr > NameMap; NameMap names; + // comments are only filled when in "documentation mode" + typedef std::map< const char*, const char*, ltstr > CommentMap; + CommentMap comments; + // The counter is initialized to 0 and incremented on every implicit // auto-increment name that gets added (thus its > 0 if // auto-increment is used). Once an explicit value has been From c4ca6f098c56e55a5e5f732fe43d41b996c1524c Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Mon, 14 Mar 2011 13:10:49 -0500 Subject: [PATCH 09/47] Changes to scanner and parser to allow record field comments. The scanner can now be told to start/stop producing new token types that assist in documenting record field types (and eventually enums also). TOK_DOC: Produced on "##" style comments; documents the field that follows. TOK_POST_DOC: Produced on "##<" style comments; documents the previous field. --- src/input.h | 2 ++ src/parse.y | 70 +++++++++++++++++++++++++++++++++++++++++++++++------ src/scan.l | 24 +++++++++++++++++- 3 files changed, 87 insertions(+), 9 deletions(-) diff --git a/src/input.h b/src/input.h index 004d366748..5ef64f4a60 100644 --- a/src/input.h +++ b/src/input.h @@ -30,6 +30,8 @@ extern void do_atifdef(const char* id); extern void do_atifndef(const char* id); extern void do_atelse(); extern void do_atendif(); +extern void do_doc_token_start(); +extern void do_doc_token_stop(); extern int line_number; extern const char* filename; diff --git a/src/parse.y b/src/parse.y index 5f5b37a526..a01f655ea4 100644 --- a/src/parse.y +++ b/src/parse.y @@ -3,7 +3,7 @@ // See the file "COPYING" in the main distribution directory for copyright. %} -%expect 71 +%expect 74 %token TOK_ADD TOK_ADD_TO TOK_ADDR TOK_ALARM TOK_ANY %token TOK_ATENDIF TOK_ATELSE TOK_ATIF TOK_ATIFDEF TOK_ATIFNDEF @@ -28,6 +28,8 @@ %token TOK_DEBUG +%token TOK_DOC TOK_POST_DOC + %left ',' '|' %right '=' TOK_ADD_TO TOK_REMOVE_FROM %right '?' ':' TOK_USING @@ -41,7 +43,7 @@ %right '!' %left '$' '[' ']' '(' ')' TOK_HAS_FIELD TOK_HAS_ATTR -%type TOK_ID TOK_PATTERN_TEXT single_pattern +%type TOK_ID TOK_PATTERN_TEXT single_pattern TOK_DOC TOK_POST_DOC opt_doc_list opt_post_doc_list %type local_id global_id event_id global_or_event_id resolve_id begin_func %type local_id_list %type init_class @@ -137,6 +139,31 @@ static void parser_redef_enum (ID *id) } } +static char* concat_opt_docs (const char* pre, const char* post) + { + if ( ! pre && ! post ) + return 0; + + size_t len = 0; + if ( pre ) + len += strlen(pre); + if ( post ) + len += strlen(post); + char* s = new char[len + 1]; + s[0] = '\0'; + if ( pre ) + { + strcat(s, pre); + delete [] pre; + } + if ( post ) + { + strcat(s, post); + delete [] post; + } + return s; + } + %} %union { @@ -713,10 +740,11 @@ type: $$ = new SetType($3, 0); } - | TOK_RECORD '{' type_decl_list '}' + | TOK_RECORD '{' { do_doc_token_start(); } type_decl_list '}' { - set_location(@1, @4); - $$ = new RecordType($3); + do_doc_token_stop(); + set_location(@1, @5); + $$ = new RecordType($4); } | TOK_UNION '{' type_list '}' @@ -811,10 +839,12 @@ type_decl_list: ; type_decl: - TOK_ID ':' type opt_attr ';' + opt_doc_list TOK_ID ':' type opt_attr ';' opt_post_doc_list { - set_location(@1, @5); - $$ = new TypeDecl($3, $1, $4); + set_location(@2, @6); + $$ = new TypeDecl($4, $2, $5); + if ( generate_documentation ) + $$->comment = concat_opt_docs($1, $7); } ; @@ -1366,6 +1396,30 @@ resolve_id: } ; +opt_post_doc_list: + opt_post_doc_list TOK_POST_DOC + { + $$ = concat_opt_docs($1, $2); + } + | + TOK_POST_DOC + { $$ = $1; } + | + { $$ = 0; } + ; + +opt_doc_list: + opt_doc_list TOK_DOC + { + $$ = concat_opt_docs($1, $2); + } + | + TOK_DOC + { $$ = $1; } + | + { $$ = 0; } + ; + %% int yyerror(const char msg[]) diff --git a/src/scan.l b/src/scan.l index d47986b144..4f2a4cf743 100644 --- a/src/scan.l +++ b/src/scan.l @@ -113,6 +113,7 @@ static void report_file(); %x RE %x IGNORE +%s DOC OWS [ \t]* WS [ \t]+ @@ -143,6 +144,17 @@ ESCSEQ (\\([^\n]|[0-7]+|x[[:xdigit:]]+)) ); } +##<.* { + yylval.str = copy_string(yytext + 3); + return TOK_POST_DOC; +} + +##[^#\n].* { + yylval.str = copy_string(yytext + 2); + return TOK_DOC; +} + + ##[^#\n].* { if ( generate_documentation ) { @@ -156,7 +168,7 @@ ESCSEQ (\\([^\n]|[0-7]+|x[[:xdigit:]]+)) {WS} /* eat whitespace */ -\n { +\n { report_file(); ++line_number; ++yylloc.first_line; @@ -664,6 +676,16 @@ void do_atendif() --current_depth; } +void do_doc_token_start() + { + if ( generate_documentation ) BEGIN(DOC); + } + +void do_doc_token_stop() + { + if ( generate_documentation ) BEGIN(INITIAL); + } + // Be careful to never delete things from this list, as the strings // are referred to (in order to save the locations of tokens and statements, // for error reporting and debugging). From f47c8e4d93c8373813d2b3757ca5453156935235 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Mon, 14 Mar 2011 14:13:52 -0500 Subject: [PATCH 10/47] Changes to parser to allow enum type documentation comments. "##" style comments before identifiers and "##<" style after identifiers in the body of an enum type declaration will now show up in the auto-generated reST documentation. --- src/Type.cc | 19 ++++++++++++++- src/parse.y | 67 ++++++++++++++++++++++++++++++++++++++++++----------- 2 files changed, 71 insertions(+), 15 deletions(-) diff --git a/src/Type.cc b/src/Type.cc index 67b0bb7be9..ae2d94046c 100644 --- a/src/Type.cc +++ b/src/Type.cc @@ -1273,8 +1273,25 @@ void EnumType::AddName(const string& module_name, const char* name, bro_int_t va void EnumType::AddComment(const string& module_name, const char* name, const char* comment) { + if ( ! comment ) return; + string fullname = make_full_var_name(module_name.c_str(), name); - comments[copy_string(fullname.c_str())] = copy_string(comment); + + CommentMap::iterator it = comments.find(fullname.c_str()); + + if ( it == comments.end() ) + comments[copy_string(fullname.c_str())] = comment; + else + { + // append to current comments + size_t len = strlen(it->second) + strlen(comment) + 1; + char* s = new char[len]; + sprintf(s, "%s%s", it->second, comment); + s[len - 1] = '\0'; + delete [] it->second; + delete [] comment; + comments[fullname.c_str()] = s; + } } void EnumType::AddNameInternal(const string& module_name, const char* name, bro_int_t val, bool is_export) diff --git a/src/parse.y b/src/parse.y index a01f655ea4..e9ee044c50 100644 --- a/src/parse.y +++ b/src/parse.y @@ -3,7 +3,7 @@ // See the file "COPYING" in the main distribution directory for copyright. %} -%expect 74 +%expect 81 %token TOK_ADD TOK_ADD_TO TOK_ADDR TOK_ALARM TOK_ANY %token TOK_ATENDIF TOK_ATELSE TOK_ATIF TOK_ATIFDEF TOK_ATIFNDEF @@ -116,6 +116,7 @@ bool resolving_global_ID = false; ID* func_id = 0; EnumType *cur_enum_type = 0; +const char* cur_enum_elem_id = 0; static void parser_new_enum (void) { @@ -582,10 +583,12 @@ opt_expr_list: { $$ = new ListExpr(); } ; +/* opt_comma: ',' | ; +*/ pattern: pattern '|' single_pattern @@ -607,17 +610,40 @@ single_pattern: ; enum_body: - enum_body_list opt_comma + enum_body_list opt_post_doc_list { $$ = cur_enum_type; + if ( generate_documentation ) + { + cur_enum_type->AddComment(current_module, cur_enum_elem_id, $2); + cur_enum_elem_id = 0; + } + cur_enum_type = NULL; + } + | enum_body_list ',' opt_post_doc_list + { + $$ = cur_enum_type; + if ( generate_documentation ) + { + cur_enum_type->AddComment(current_module, cur_enum_elem_id, $3); + cur_enum_elem_id = 0; + } cur_enum_type = NULL; } ; enum_body_list: - enum_body_elem /* No action */ - | enum_body_list ',' enum_body_elem /* no action */ - ; + enum_body_elem opt_post_doc_list + { + if ( generate_documentation ) + cur_enum_type->AddComment(current_module, cur_enum_elem_id, $2); + } + | enum_body_list ',' opt_post_doc_list + { + if ( generate_documentation ) + cur_enum_type->AddComment(current_module, cur_enum_elem_id, $3); + } enum_body_elem +; enum_body_elem: /* TODO: We could also define this as TOK_ID '=' expr, (or @@ -625,17 +651,23 @@ enum_body_elem: error messages if someboy tries to use constant variables as enumerator. */ - TOK_ID '=' TOK_CONSTANT + opt_doc_list TOK_ID '=' TOK_CONSTANT { - set_location(@1, @3); + set_location(@2, @4); assert(cur_enum_type); - if ( $3->Type()->Tag() != TYPE_COUNT ) + if ( $4->Type()->Tag() != TYPE_COUNT ) error("enumerator is not a count constant"); else - cur_enum_type->AddName(current_module, $1, $3->InternalUnsigned(), is_export); + cur_enum_type->AddName(current_module, $2, $4->InternalUnsigned(), is_export); + + if ( generate_documentation ) + { + cur_enum_elem_id = $2; + cur_enum_type->AddComment(current_module, cur_enum_elem_id, $1); + } } - | TOK_ID '=' '-' TOK_CONSTANT + | opt_doc_list TOK_ID '=' '-' TOK_CONSTANT { /* We only accept counts as enumerator, but we want to return a nice error message if users triy to use a negative integer (will also @@ -644,11 +676,17 @@ enum_body_elem: error("enumerator is not a count constant"); } - | TOK_ID + | opt_doc_list TOK_ID { - set_location(@1); + set_location(@2); assert(cur_enum_type); - cur_enum_type->AddName(current_module, $1, is_export); + cur_enum_type->AddName(current_module, $2, is_export); + + if ( generate_documentation ) + { + cur_enum_elem_id = $2; + cur_enum_type->AddComment(current_module, cur_enum_elem_id, $1); + } } ; @@ -754,8 +792,9 @@ type: $$ = 0; } - | TOK_ENUM '{' { set_location(@1); parser_new_enum(); } enum_body '}' + | TOK_ENUM '{' { set_location(@1); parser_new_enum(); do_doc_token_start(); } enum_body '}' { + do_doc_token_stop(); set_location(@1, @5); $4->UpdateLocationEndInfo(@5); $$ = $4; From b1dc5d3a1c94a22aa507b8ced78c45d67d99c6fa Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Mon, 14 Mar 2011 14:49:58 -0500 Subject: [PATCH 11/47] Update to auto-generated documentation format. Comments are now clearly marked. Enums are now documented in order of their integral value, not their identifier string. --- src/BroDocObj.cc | 23 +++++++++++++---------- src/Type.cc | 18 ++++++++++++++---- 2 files changed, 27 insertions(+), 14 deletions(-) diff --git a/src/BroDocObj.cc b/src/BroDocObj.cc index a23dec16be..b8f3309f05 100644 --- a/src/BroDocObj.cc +++ b/src/BroDocObj.cc @@ -18,16 +18,19 @@ BroDocObj::~BroDocObj() void BroDocObj::WriteReST(FILE* file) const { - if ( reST_doc_strings ) - { - std::list::const_iterator it; - for ( it = reST_doc_strings->begin(); - it != reST_doc_strings->end(); ++it) - fprintf(file, "%s\n", it->c_str()); - } - ODesc desc; desc.SetQuotes(1); broID->DescribeReST(&desc); - fprintf(file, "%s\n\n", desc.Description()); - } + fprintf(file, "%s\n", desc.Description()); + + if ( HasDocumentation() ) + { + fprintf(file, "\t.. bro:comment::\n"); + std::list::const_iterator it; + for ( it = reST_doc_strings->begin(); + it != reST_doc_strings->end(); ++it) + fprintf(file, "\t\t%s\n", it->c_str()); + } + + fprintf(file, "\n"); + } diff --git a/src/Type.cc b/src/Type.cc index ae2d94046c..56d51e98da 100644 --- a/src/Type.cc +++ b/src/Type.cc @@ -1067,6 +1067,7 @@ void RecordType::DescribeFieldsReST(ODesc* d, bool func_args) const if ( td->comment ) { d->PushIndent(); + d->Add(".. bro:comment:: "); d->Add(td->comment); d->PopIndent(); } @@ -1345,20 +1346,29 @@ const char* EnumType::Lookup(bro_int_t value) void EnumType::DescribeReST(ODesc* d) const { + // create temporary, reverse name map so that enums can be documented + // in ascending order of their actual integral value instead of by name + typedef std::map< bro_int_t, const char* > RevNameMap; + RevNameMap rev; + for ( NameMap::const_iterator it = names.begin(); it != names.end(); ++it ) + rev[it->second] = it->first; + d->Add(type_name(Tag())); d->PushIndent(); - for ( NameMap::const_iterator it = names.begin(); it != names.end(); ) + + for ( RevNameMap::const_iterator it = rev.begin(); it != rev.end(); ) { d->Add(".. bro:enum:: "); - d->Add(it->first); - CommentMap::const_iterator cmnt_it = comments.find(it->first); + d->Add(it->second); + CommentMap::const_iterator cmnt_it = comments.find(it->second); if ( cmnt_it != comments.end() ) { d->PushIndent(); + d->Add(".. bro:comment:: "); d->Add(cmnt_it->second); d->PopIndent(); } - if ( ++it != names.end() ) + if ( ++it != rev.end() ) d->NL(); } } From f67c0892e520270d171238b984fd3daa5b44535a Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Tue, 15 Mar 2011 14:51:50 -0500 Subject: [PATCH 12/47] Revise enum documentation autogeneration. This adds a new subclass of EnumType, CommentedEnumType, and removes any previous changes to EnumType that were done to support the autodoc framework. Dummy CommentedEnumType and ID's are constructed in parallel with the real EnumType ID's during parsing and passed on to the autodoc framework. This allows the generated documentation to track enum redefs, with a special case being the "Notice" enum type. --- src/BroDoc.cc | 11 ++++++-- src/BroDoc.h | 6 +++-- src/BroDocObj.cc | 5 +++- src/BroDocObj.h | 5 +++- src/Type.cc | 40 ++++++++-------------------- src/Type.h | 28 +++++++++++++------- src/parse.y | 68 +++++++++++++++++++++++++++++++++++++++--------- 7 files changed, 107 insertions(+), 56 deletions(-) diff --git a/src/BroDoc.cc b/src/BroDoc.cc index 715e8d9cf9..27895990cf 100644 --- a/src/BroDoc.cc +++ b/src/BroDoc.cc @@ -40,6 +40,7 @@ BroDoc::BroDoc(const std::string& sourcename) else fprintf(stdout, "Created reST document: %s\n", reST_filename.c_str()); #endif + notices = 0; } BroDoc::~BroDoc() @@ -50,9 +51,10 @@ BroDoc::~BroDoc() FreeBroDocObjPtrList(options); FreeBroDocObjPtrList(state_vars); FreeBroDocObjPtrList(types); - FreeBroDocObjPtrList(notices); FreeBroDocObjPtrList(events); FreeBroDocObjPtrList(functions); + FreeBroDocObjPtrList(redefs); + if ( notices ) delete notices; } void BroDoc::SetPacketFilter(const std::string& s) @@ -90,6 +92,10 @@ void BroDoc::WriteDocFile() const WriteStringList(" :bro:script: `%s`\n", " :bro:script: `%s`\n\n", imports); + WriteSectionHeading("Notices", '-'); + if ( notices ) + notices->WriteReST(reST_file); + WriteSectionHeading("Port Analysis", '-'); WriteStringList("%s", port_analysis); @@ -100,15 +106,16 @@ void BroDoc::WriteDocFile() const WriteBroDocObjList(options, true, "Options", '~'); WriteBroDocObjList(state_vars, true, "State Variables", '~'); WriteBroDocObjList(types, true, "Types", '~'); - WriteBroDocObjList(notices, true, "Notices", '~'); WriteBroDocObjList(events, true, "Events", '~'); WriteBroDocObjList(functions, true, "Functions", '~'); + WriteBroDocObjList(redefs, true, "Redefinitions", '~'); WriteSectionHeading("Private Interface", '-'); WriteBroDocObjList(state_vars, false, "State Variables", '~'); WriteBroDocObjList(types, false, "Types", '~'); WriteBroDocObjList(events, false, "Events", '~'); WriteBroDocObjList(functions, false, "Functions", '~'); + WriteBroDocObjList(redefs, false, "Redefinitions", '~'); } void BroDoc::WriteStringList(const char* format, diff --git a/src/BroDoc.h b/src/BroDoc.h index 37adcc8b70..1f5dae78c0 100644 --- a/src/BroDoc.h +++ b/src/BroDoc.h @@ -95,9 +95,10 @@ public: void AddOption(const BroDocObj* o) { options.push_back(o); } void AddStateVar(const BroDocObj* o) { state_vars.push_back(o); } void AddType(const BroDocObj* o) { types.push_back(o); } - void AddNotice(const BroDocObj* o) { notices.push_back(o); } + void AddNotice(const BroDocObj* o) { notices = o; } void AddEvent(const BroDocObj* o) { events.push_back(o); } void AddFunction(const BroDocObj* o) { functions.push_back(o); } + void AddRedef(const BroDocObj* o) { redefs.push_back(o); } /** * Gets the name of the Bro script source file for which reST @@ -128,9 +129,10 @@ protected: std::list options; // identifiers with &redef attr std::list state_vars; // identifiers without &redef? std::list types; - std::list notices; + const BroDocObj* notices; std::list events; std::list functions; + std::list redefs; /** * Writes out a list of strings to the reST document. diff --git a/src/BroDocObj.cc b/src/BroDocObj.cc index b8f3309f05..2b79d9fcdf 100644 --- a/src/BroDocObj.cc +++ b/src/BroDocObj.cc @@ -4,16 +4,19 @@ #include "ID.h" #include "BroDocObj.h" -BroDocObj::BroDocObj(const ID* id, std::list*& reST) +BroDocObj::BroDocObj(const ID* id, std::list*& reST, + bool is_fake) { broID = id; reST_doc_strings = reST; reST = 0; + is_fake_id = is_fake; } BroDocObj::~BroDocObj() { delete reST_doc_strings; + if ( is_fake_id ) delete broID; } void BroDocObj::WriteReST(FILE* file) const diff --git a/src/BroDocObj.h b/src/BroDocObj.h index 0fdbcca8a5..38f3d8f182 100644 --- a/src/BroDocObj.h +++ b/src/BroDocObj.h @@ -15,8 +15,10 @@ public: * @param reST a reference to a pointer of a list of strings that represent the reST documentation for the ID. The pointer will be set to 0 after this constructor finishes. + * @param is_fake whether the ID* is a dummy just for doc purposes */ - BroDocObj(const ID* id, std::list*& reST); + BroDocObj(const ID* id, std::list*& reST, + bool is_fake = false); /** * BroDocObj destructor @@ -60,6 +62,7 @@ public: protected: std::list* reST_doc_strings; const ID* broID; + bool is_fake_id; /**< Whether the ID* is a dummy just for doc purposes */ private: }; diff --git a/src/Type.cc b/src/Type.cc index 56d51e98da..d74d02770c 100644 --- a/src/Type.cc +++ b/src/Type.cc @@ -1235,7 +1235,10 @@ EnumType::~EnumType() { for ( NameMap::iterator iter = names.begin(); iter != names.end(); ++iter ) delete [] iter->first; + } +CommentedEnumType::~CommentedEnumType() + { for ( CommentMap::iterator iter = comments.begin(); iter != comments.end(); ++iter ) { delete [] iter->first; @@ -1272,7 +1275,7 @@ void EnumType::AddName(const string& module_name, const char* name, bro_int_t va AddNameInternal(module_name, name, val, is_export); } -void EnumType::AddComment(const string& module_name, const char* name, const char* comment) +void CommentedEnumType::AddComment(const string& module_name, const char* name, const char* comment) { if ( ! comment ) return; @@ -1323,6 +1326,12 @@ void EnumType::AddNameInternal(const string& module_name, const char* name, bro_ names[copy_string(fullname.c_str())] = val; } +void CommentedEnumType::AddNameInternal(const string& module_name, const char* name, bro_int_t val, bool is_export) + { + string fullname = make_full_var_name(module_name.c_str(), name); + names[copy_string(fullname.c_str())] = val; + } + bro_int_t EnumType::Lookup(const string& module_name, const char* name) { NameMap::iterator pos = @@ -1344,7 +1353,7 @@ const char* EnumType::Lookup(bro_int_t value) return 0; } -void EnumType::DescribeReST(ODesc* d) const +void CommentedEnumType::DescribeReST(ODesc* d) const { // create temporary, reverse name map so that enums can be documented // in ascending order of their actual integral value instead of by name @@ -1384,10 +1393,6 @@ bool EnumType::DoSerialize(SerialInfo* info) const SERIALIZE(false)) ) return false; - if ( generate_documentation ) - if ( ! (SERIALIZE((unsigned int) comments.size())) ) - return false; - for ( NameMap::const_iterator iter = names.begin(); iter != names.end(); ++iter ) { @@ -1395,14 +1400,6 @@ bool EnumType::DoSerialize(SerialInfo* info) const return false; } - if ( generate_documentation ) - for ( CommentMap::const_iterator it = comments.begin(); - it != comments.end(); ++ it ) - { - if ( ! SERIALIZE(it->first) || ! SERIALIZE(it->second) ) - return false; - } - return true; } @@ -1411,7 +1408,6 @@ bool EnumType::DoUnserialize(UnserialInfo* info) DO_UNSERIALIZE(BroType); unsigned int len; - unsigned int cmnt_len; bool dummy; if ( ! UNSERIALIZE(&counter) || ! UNSERIALIZE(&len) || @@ -1419,10 +1415,6 @@ bool EnumType::DoUnserialize(UnserialInfo* info) ! UNSERIALIZE(&dummy) ) return false; - if ( generate_documentation ) - if ( ! UNSERIALIZE(&cmnt_len) ) - return false; - while ( len-- ) { const char* name; @@ -1433,16 +1425,6 @@ bool EnumType::DoUnserialize(UnserialInfo* info) names[name] = val; } - if ( generate_documentation ) - while ( cmnt_len-- ) - { - const char* cmnt; - const char* name; - if ( ! (UNSERIALIZE_STR(&name, 0) && UNSERIALIZE_STR(&cmnt, 0)) ) - return false; - comments[name] = cmnt; - } - return true; } diff --git a/src/Type.h b/src/Type.h index 1bffa0bbe2..7fc89dfcaf 100644 --- a/src/Type.h +++ b/src/Type.h @@ -480,26 +480,18 @@ public: // added that aren't likewise explicitly initalized. void AddName(const string& module_name, const char* name, bro_int_t val, bool is_export); - void AddComment(const string& module_name, const char* name, const char* comment); - // -1 indicates not found. bro_int_t Lookup(const string& module_name, const char* name); const char* Lookup(bro_int_t value); // Returns 0 if not found - void DescribeReST(ODesc* d) const; - protected: DECLARE_SERIAL(EnumType) - void AddNameInternal(const string& module_name, const char* name, bro_int_t val, bool is_export); + virtual void AddNameInternal(const string& module_name, const char* name, bro_int_t val, bool is_export); typedef std::map< const char*, bro_int_t, ltstr > NameMap; NameMap names; - // comments are only filled when in "documentation mode" - typedef std::map< const char*, const char*, ltstr > CommentMap; - CommentMap comments; - // The counter is initialized to 0 and incremented on every implicit // auto-increment name that gets added (thus its > 0 if // auto-increment is used). Once an explicit value has been @@ -509,6 +501,24 @@ protected: bro_int_t counter; }; +class CommentedEnumType: public EnumType { +public: + CommentedEnumType() {} + ~CommentedEnumType(); + void DescribeReST(ODesc* d) const; + void AddComment(const string& module_name, const char* name, const char* comment); + +protected: + // This overriden method does not install the given ID name into a + // scope and it also does not do any kind of checking that the provided + // name already exists. + void AddNameInternal(const string& module_name, const char* name, bro_int_t val, bool is_export); + + // comments are only filled when in "documentation mode" + typedef std::map< const char*, const char*, ltstr > CommentMap; + CommentMap comments; +}; + class VectorType : public BroType { public: VectorType(BroType* t); diff --git a/src/parse.y b/src/parse.y index e9ee044c50..5e4966fd5e 100644 --- a/src/parse.y +++ b/src/parse.y @@ -116,6 +116,7 @@ bool resolving_global_ID = false; ID* func_id = 0; EnumType *cur_enum_type = 0; +CommentedEnumType *cur_enum_type_doc = 0; const char* cur_enum_elem_id = 0; static void parser_new_enum (void) @@ -123,6 +124,10 @@ static void parser_new_enum (void) /* Starting a new enum definition. */ assert(cur_enum_type == NULL); cur_enum_type = new EnumType(); + // for documentation purpose, a separate type object is created + // in order to avoid overlap that can be caused by redefs + if ( generate_documentation ) + cur_enum_type_doc = new CommentedEnumType(); } static void parser_redef_enum (ID *id) @@ -138,6 +143,24 @@ static void parser_redef_enum (ID *id) if ( ! cur_enum_type ) id->Error("not an enum"); } + + if ( generate_documentation ) + cur_enum_type_doc = new CommentedEnumType(); + } + +static void add_enum_comment (const char* comment) + { + cur_enum_type_doc->AddComment(current_module, cur_enum_elem_id, comment); + } + +static ID* create_dummy_id (const char* name, const BroType* type) + { + // normally, install_ID() figures out the right IDScope + // but it doesn't matter for the dummy ID so use SCOPE_GLOBAL + ID* fake_id = new ID(copy_string(name), SCOPE_GLOBAL, is_export); + fake_id->SetType(cur_enum_type_doc); + fake_id->MakeType(); + return fake_id; } static char* concat_opt_docs (const char* pre, const char* post) @@ -615,7 +638,7 @@ enum_body: $$ = cur_enum_type; if ( generate_documentation ) { - cur_enum_type->AddComment(current_module, cur_enum_elem_id, $2); + add_enum_comment($2); cur_enum_elem_id = 0; } cur_enum_type = NULL; @@ -625,7 +648,7 @@ enum_body: $$ = cur_enum_type; if ( generate_documentation ) { - cur_enum_type->AddComment(current_module, cur_enum_elem_id, $3); + add_enum_comment($3); cur_enum_elem_id = 0; } cur_enum_type = NULL; @@ -636,12 +659,12 @@ enum_body_list: enum_body_elem opt_post_doc_list { if ( generate_documentation ) - cur_enum_type->AddComment(current_module, cur_enum_elem_id, $2); + add_enum_comment($2); } | enum_body_list ',' opt_post_doc_list { if ( generate_documentation ) - cur_enum_type->AddComment(current_module, cur_enum_elem_id, $3); + add_enum_comment($3); } enum_body_elem ; @@ -662,8 +685,9 @@ enum_body_elem: if ( generate_documentation ) { + cur_enum_type_doc->AddName(current_module, $2, $4->InternalUnsigned(), is_export); cur_enum_elem_id = $2; - cur_enum_type->AddComment(current_module, cur_enum_elem_id, $1); + add_enum_comment($1); } } @@ -684,8 +708,9 @@ enum_body_elem: if ( generate_documentation ) { + cur_enum_type_doc->AddName(current_module, $2, is_export); cur_enum_elem_id = $2; - cur_enum_type->AddComment(current_module, cur_enum_elem_id, $1); + add_enum_comment($1); } } ; @@ -957,23 +982,42 @@ decl: add_global($2, $3, $4, $5, $6, VAR_REDEF); if ( generate_documentation ) { - // TODO: eventually handle type redefs that add record fields + // TODO: handle more types of redefs, e.g. adding record fields } } | TOK_REDEF TOK_ENUM global_id TOK_ADD_TO - '{' { parser_redef_enum($3); } enum_body '}' ';' + '{' { parser_redef_enum($3); do_doc_token_start(); } enum_body '}' ';' { - /* no action */ - // TODO: handle "Notice" redefinitions for doc framework + do_doc_token_stop(); + if ( generate_documentation ) + { + ID* fake_id = create_dummy_id($3->Name(), cur_enum_type_doc); + cur_enum_type_doc = 0; + BroDocObj* o = new BroDocObj(fake_id, reST_doc_comments, true); + if ( streq(fake_id->Name(), "Notice" ) ) + current_reST_doc->AddNotice(o); + else + current_reST_doc->AddRedef(o); + } } | TOK_TYPE global_id ':' refined_type opt_attr ';' { add_type($2, $4, $5, 0); if ( generate_documentation ) - current_reST_doc->AddType( - new BroDocObj($2, reST_doc_comments)); + { + if ( $2->AsType()->Tag() == TYPE_ENUM && cur_enum_type_doc ) + { + ID* fake = create_dummy_id($2->Name(), cur_enum_type_doc); + cur_enum_type_doc = 0; + current_reST_doc->AddType( + new BroDocObj(fake, reST_doc_comments, true)); + } + else + current_reST_doc->AddType( + new BroDocObj($2, reST_doc_comments)); + } } | TOK_EVENT event_id ':' refined_type opt_attr ';' From dbf2b9996e1b89f752db4bd3be8b4f16a79199e2 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Tue, 15 Mar 2011 15:38:43 -0500 Subject: [PATCH 13/47] Revising BroDoc*.h internal/api documentation. --- src/BroDoc.h | 57 +++++++++++++++++++++++++++++++++++++++++++++---- src/BroDocObj.h | 17 ++++----------- 2 files changed, 57 insertions(+), 17 deletions(-) diff --git a/src/BroDoc.h b/src/BroDoc.h index 1f5dae78c0..fdd01020b2 100644 --- a/src/BroDoc.h +++ b/src/BroDoc.h @@ -90,14 +90,63 @@ public: */ void SetAuthor(const std::string& s) { author_name = s; } - //TODO: document these better - // the rest of these need to be called from the parser + /** + * Schedules documentation of a script option. An option is + * defined as any variable in the script that is declared 'const' + * and has the '&redef' attribute. + * @param o A pointer to a BroDocObj which contains the internal + * Bro language representation of the script option and + * also any associated comments about it. + */ void AddOption(const BroDocObj* o) { options.push_back(o); } + + /** + * Schedules documentation of a script state variable. A state variable + * is defined as any variable in the script that is declared 'global' + * @param o A pointer to a BroDocObj which contains the internal + * Bro language representation of the script state variable + * and also any associated comments about it. + */ void AddStateVar(const BroDocObj* o) { state_vars.push_back(o); } + + /** + * Schedules documentation of a type declared by the script. + * @param o A pointer to a BroDocObj which contains the internal + * Bro language representation of the script option and + * also any associated comments about it. + */ void AddType(const BroDocObj* o) { types.push_back(o); } + + /** + * Schedules documentation of a Notice (enum redef) declared by script + * @param o A pointer to a BroDocObj which contains the internal + * Bro language representation of the Notice and also + * any associated comments about it. + */ void AddNotice(const BroDocObj* o) { notices = o; } + + /** + * Schedules documentation of an event declared by the script. + * @param o A pointer to a BroDocObj which contains the internal + * Bro language representation of the script event and + * also any associated comments about it. + */ void AddEvent(const BroDocObj* o) { events.push_back(o); } + + /** + * Schedules documentation of a function declared by the script. + * @param o A pointer to a BroDocObj which contains the internal + * Bro language representation of the script function and + * also any associated comments about it. + */ void AddFunction(const BroDocObj* o) { functions.push_back(o); } + + /** + * Schedules documentation of a redef done by the script + * @param o A pointer to a BroDocObj which contains the internal + * Bro language representation of the script identifier + * that was redefined and also any associated comments. + */ void AddRedef(const BroDocObj* o) { redefs.push_back(o); } /** @@ -126,8 +175,8 @@ protected: std::list imports; std::list port_analysis; - std::list options; // identifiers with &redef attr - std::list state_vars; // identifiers without &redef? + std::list options; + std::list state_vars; std::list types; const BroDocObj* notices; std::list events; diff --git a/src/BroDocObj.h b/src/BroDocObj.h index 38f3d8f182..e9254027e5 100644 --- a/src/BroDocObj.h +++ b/src/BroDocObj.h @@ -27,20 +27,11 @@ public: ~BroDocObj(); /** - * writes the reST representation of this object which includes - * 1) any of the "##" comments (stored internally in reST_doc_string) - * To make things easy, I think we should assume that the documenter - * writes their comments such that anything after ## is valid reST - * so that at parse time the ## is just stripped and the remainder - * is scheduled to be inserted as-is into the reST. - * TODO: prepare for some kind of filtering mechanism that transforms - * the reST as written into new reST before being written out. - * This allows for additional custom markup or macros when writing - * pure reST might be inconvenient. + * Writes the reST representation of this object which includes + * 1) Any "##" or "##<" stylized comments. + * Anything after these style of comments is inserted as-is into + * the reST document. * 2) a reST friendly description of the ID - * Could be implemented similar to the ID::DescribeExtended(ODesc) - * expect with new directives/roles that we'll later teach to Sphinx - * via a "bro domain". * @param The (already opened) file to write the reST to. */ void WriteReST(FILE* file) const; From 9e13d15f29ac8445a63f9a88bbc5b7a418fc077c Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Tue, 15 Mar 2011 16:48:51 -0500 Subject: [PATCH 14/47] Tabifying BroDoc* sources to make consistent with general style. --- src/BroDoc.cc | 272 ++++++++++++++++---------------- src/BroDoc.h | 398 +++++++++++++++++++++++------------------------ src/BroDocObj.cc | 50 +++--- src/BroDocObj.h | 78 +++++----- 4 files changed, 399 insertions(+), 399 deletions(-) diff --git a/src/BroDoc.cc b/src/BroDoc.cc index 27895990cf..1e845a49fd 100644 --- a/src/BroDoc.cc +++ b/src/BroDoc.cc @@ -7,179 +7,179 @@ #include "BroDocObj.h" BroDoc::BroDoc(const std::string& sourcename) - { + { #ifdef DEBUG - fprintf(stdout, "Documenting source: %s\n", sourcename.c_str()); + fprintf(stdout, "Documenting source: %s\n", sourcename.c_str()); #endif - source_filename = sourcename.substr(sourcename.find_last_of('/') + 1); + source_filename = sourcename.substr(sourcename.find_last_of('/') + 1); - size_t ext_pos = source_filename.find_last_of('.'); - std::string ext = source_filename.substr(ext_pos + 1); - if ( ext_pos == std::string::npos || ext != "bro" ) - { - if ( source_filename != "bro.init" ) - { - fprintf(stderr, - "Warning: documenting file without .bro extension: %s\n", - sourcename.c_str()); - } - else - { - // Force the reST documentation file to be "bro.init.rst" - ext_pos = std::string::npos; - } - } + size_t ext_pos = source_filename.find_last_of('.'); + std::string ext = source_filename.substr(ext_pos + 1); + if ( ext_pos == std::string::npos || ext != "bro" ) + { + if ( source_filename != "bro.init" ) + { + fprintf(stderr, + "Warning: documenting file without .bro extension: %s\n", + sourcename.c_str()); + } + else + { + // Force the reST documentation file to be "bro.init.rst" + ext_pos = std::string::npos; + } + } - reST_filename = source_filename.substr(0, ext_pos); - reST_filename += ".rst"; - reST_file = fopen(reST_filename.c_str(), "w"); + reST_filename = source_filename.substr(0, ext_pos); + reST_filename += ".rst"; + reST_file = fopen(reST_filename.c_str(), "w"); - if ( ! reST_file ) - fprintf(stderr, "Failed to open %s", reST_filename.c_str()); + if ( ! reST_file ) + fprintf(stderr, "Failed to open %s", reST_filename.c_str()); #ifdef DEBUG - else - fprintf(stdout, "Created reST document: %s\n", reST_filename.c_str()); + else + fprintf(stdout, "Created reST document: %s\n", reST_filename.c_str()); #endif - notices = 0; - } + notices = 0; + } BroDoc::~BroDoc() - { - if ( reST_file ) - if ( fclose( reST_file ) ) - fprintf(stderr, "Failed to close %s", reST_filename.c_str()); - FreeBroDocObjPtrList(options); - FreeBroDocObjPtrList(state_vars); - FreeBroDocObjPtrList(types); - FreeBroDocObjPtrList(events); - FreeBroDocObjPtrList(functions); - FreeBroDocObjPtrList(redefs); - if ( notices ) delete notices; - } + { + if ( reST_file ) + if ( fclose( reST_file ) ) + fprintf(stderr, "Failed to close %s", reST_filename.c_str()); + FreeBroDocObjPtrList(options); + FreeBroDocObjPtrList(state_vars); + FreeBroDocObjPtrList(types); + FreeBroDocObjPtrList(events); + FreeBroDocObjPtrList(functions); + FreeBroDocObjPtrList(redefs); + if ( notices ) delete notices; + } void BroDoc::SetPacketFilter(const std::string& s) - { - packet_filter = s; - size_t pos1 = s.find("{\n"); - size_t pos2 = s.find("}"); - if ( pos1 != std::string::npos && pos2 != std::string::npos ) - packet_filter = s.substr(pos1 + 2, pos2 - 2); - } + { + packet_filter = s; + size_t pos1 = s.find("{\n"); + size_t pos2 = s.find("}"); + if ( pos1 != std::string::npos && pos2 != std::string::npos ) + packet_filter = s.substr(pos1 + 2, pos2 - 2); + } void BroDoc::AddPortAnalysis(const std::string& analyzer, const std::string& ports) - { - std::string reST_string = analyzer + "::\n" + ports + "\n"; - port_analysis.push_back(reST_string); - } + { + std::string reST_string = analyzer + "::\n" + ports + "\n"; + port_analysis.push_back(reST_string); + } void BroDoc::WriteDocFile() const - { - WriteToDoc("%s\n", source_filename.c_str()); - for ( size_t i = 0; i < source_filename.length(); ++i ) - WriteToDoc("="); - WriteToDoc("\n\n"); + { + WriteToDoc("%s\n", source_filename.c_str()); + for ( size_t i = 0; i < source_filename.length(); ++i ) + WriteToDoc("="); + WriteToDoc("\n\n"); - WriteSectionHeading("Summary", '-'); - WriteStringList("%s\n", "%s\n\n", summary); + WriteSectionHeading("Summary", '-'); + WriteStringList("%s\n", "%s\n\n", summary); - WriteToDoc(":Author: %s\n", author_name.c_str()); + WriteToDoc(":Author: %s\n", author_name.c_str()); - WriteToDoc(":Namespaces: "); - WriteStringList("`%s`, ", "`%s`\n", modules); + WriteToDoc(":Namespaces: "); + WriteStringList("`%s`, ", "`%s`\n", modules); - WriteToDoc(":Imports:\n"); - WriteStringList(" :bro:script: `%s`\n", - " :bro:script: `%s`\n\n", imports); + WriteToDoc(":Imports:\n"); + WriteStringList("\t:bro:script: `%s`\n", + "\t:bro:script: `%s`\n\n", imports); - WriteSectionHeading("Notices", '-'); - if ( notices ) - notices->WriteReST(reST_file); + WriteSectionHeading("Notices", '-'); + if ( notices ) + notices->WriteReST(reST_file); - WriteSectionHeading("Port Analysis", '-'); - WriteStringList("%s", port_analysis); + WriteSectionHeading("Port Analysis", '-'); + WriteStringList("%s", port_analysis); - WriteSectionHeading("Packet Filter", '-'); - WriteToDoc("%s\n", packet_filter.c_str()); + WriteSectionHeading("Packet Filter", '-'); + WriteToDoc("%s\n", packet_filter.c_str()); - WriteSectionHeading("Public Interface", '-'); - WriteBroDocObjList(options, true, "Options", '~'); - WriteBroDocObjList(state_vars, true, "State Variables", '~'); - WriteBroDocObjList(types, true, "Types", '~'); - WriteBroDocObjList(events, true, "Events", '~'); - WriteBroDocObjList(functions, true, "Functions", '~'); - WriteBroDocObjList(redefs, true, "Redefinitions", '~'); + WriteSectionHeading("Public Interface", '-'); + WriteBroDocObjList(options, true, "Options", '~'); + WriteBroDocObjList(state_vars, true, "State Variables", '~'); + WriteBroDocObjList(types, true, "Types", '~'); + WriteBroDocObjList(events, true, "Events", '~'); + WriteBroDocObjList(functions, true, "Functions", '~'); + WriteBroDocObjList(redefs, true, "Redefinitions", '~'); - WriteSectionHeading("Private Interface", '-'); - WriteBroDocObjList(state_vars, false, "State Variables", '~'); - WriteBroDocObjList(types, false, "Types", '~'); - WriteBroDocObjList(events, false, "Events", '~'); - WriteBroDocObjList(functions, false, "Functions", '~'); - WriteBroDocObjList(redefs, false, "Redefinitions", '~'); - } + WriteSectionHeading("Private Interface", '-'); + WriteBroDocObjList(state_vars, false, "State Variables", '~'); + WriteBroDocObjList(types, false, "Types", '~'); + WriteBroDocObjList(events, false, "Events", '~'); + WriteBroDocObjList(functions, false, "Functions", '~'); + WriteBroDocObjList(redefs, false, "Redefinitions", '~'); + } void BroDoc::WriteStringList(const char* format, const char* last_format, const std::list& l) const - { - if ( l.empty() ) - { - WriteToDoc("\n"); - return; - } - std::list::const_iterator it; - std::list::const_iterator last = l.end(); - last--; - for ( it = l.begin(); it != last; ++it ) - WriteToDoc(format, it->c_str()); - WriteToDoc(last_format, last->c_str()); - } + { + if ( l.empty() ) + { + WriteToDoc("\n"); + return; + } + std::list::const_iterator it; + std::list::const_iterator last = l.end(); + last--; + for ( it = l.begin(); it != last; ++it ) + WriteToDoc(format, it->c_str()); + WriteToDoc(last_format, last->c_str()); + } void BroDoc::WriteBroDocObjList(const std::list& l, bool exportCond, const char* heading, char underline) const - { - WriteSectionHeading(heading, underline); - std::list::const_iterator it; - for ( it = l.begin(); it != l.end(); ++it ) - { - if ( exportCond ) - { - // write out only those in an export section - if ( (*it)->IsPublicAPI() ) - (*it)->WriteReST(reST_file); - } - else - { - // write out only those that have comments and are not exported - if ( !(*it)->IsPublicAPI() && (*it)->HasDocumentation() ) - (*it)->WriteReST(reST_file); - } - } - } + { + WriteSectionHeading(heading, underline); + std::list::const_iterator it; + for ( it = l.begin(); it != l.end(); ++it ) + { + if ( exportCond ) + { + // write out only those in an export section + if ( (*it)->IsPublicAPI() ) + (*it)->WriteReST(reST_file); + } + else + { + // write out only those that have comments and are not exported + if ( !(*it)->IsPublicAPI() && (*it)->HasDocumentation() ) + (*it)->WriteReST(reST_file); + } + } + } void BroDoc::WriteToDoc(const char* format, ...) const - { - va_list argp; - va_start(argp, format); - vfprintf(reST_file, format, argp); - va_end(argp); - } + { + va_list argp; + va_start(argp, format); + vfprintf(reST_file, format, argp); + va_end(argp); + } void BroDoc::WriteSectionHeading(const char* heading, char underline) const - { - WriteToDoc("%s\n", heading); - size_t len = strlen(heading); - for ( size_t i = 0; i < len; ++i ) - WriteToDoc("%c", underline); - WriteToDoc("\n"); - } + { + WriteToDoc("%s\n", heading); + size_t len = strlen(heading); + for ( size_t i = 0; i < len; ++i ) + WriteToDoc("%c", underline); + WriteToDoc("\n"); + } void BroDoc::FreeBroDocObjPtrList(std::list& l) - { - std::list::iterator it; - for ( it = l.begin(); it != l.end(); ++it ) - delete *it; - l.clear(); - } + { + std::list::iterator it; + for ( it = l.begin(); it != l.end(); ++it ) + delete *it; + l.clear(); + } diff --git a/src/BroDoc.h b/src/BroDoc.h index fdd01020b2..30b6fe4426 100644 --- a/src/BroDoc.h +++ b/src/BroDoc.h @@ -10,236 +10,236 @@ class BroDoc { public: - /** - * BroDoc constructor - * Given a Bro script, opens new file in the current working directory - * that will contain reST documentation generated from the parsing - * of the Bro script. The new reST file will be named similar to - * the filename of the Bro script that generates it, except any - * ".bro" file extension is stripped and ".rst" takes it place. - * If the filename doesn't end in ".bro", then ".rst" is just appended. - * @param sourcename The name of the Bro script for which to generate - * documentation. May contain a path. - */ - BroDoc(const std::string& sourcename); + /** + * BroDoc constructor + * Given a Bro script, opens new file in the current working directory + * that will contain reST documentation generated from the parsing + * of the Bro script. The new reST file will be named similar to + * the filename of the Bro script that generates it, except any + * ".bro" file extension is stripped and ".rst" takes it place. + * If the filename doesn't end in ".bro", then ".rst" is just appended. + * @param sourcename The name of the Bro script for which to generate + * documentation. May contain a path. + */ + BroDoc(const std::string& sourcename); - /** - * BroDoc destructor - * Closes the file that was opened by the constructor and frees up - * memory taken by BroDocObj objects. - */ - ~BroDoc(); + /** + * BroDoc destructor + * Closes the file that was opened by the constructor and frees up + * memory taken by BroDocObj objects. + */ + ~BroDoc(); - /** - * Write out full reST documentation for the Bro script that was parsed. - * BroDoc's default implementation of this function will care - * about whether declarations made in the Bro script are part of - * the public versus private interface (whether things are declared in - * the export section). Things in a script's export section make it - * into the reST output regardless of whether they have ## comments - * but things outside the export section are only output into the reST - * if they have ## comments. - */ - virtual void WriteDocFile() const; + /** + * Write out full reST documentation for the Bro script that was parsed. + * BroDoc's default implementation of this function will care + * about whether declarations made in the Bro script are part of + * the public versus private interface (whether things are declared in + * the export section). Things in a script's export section make it + * into the reST output regardless of whether they have ## comments + * but things outside the export section are only output into the reST + * if they have ## comments. + */ + virtual void WriteDocFile() const; - /** - * Schedules some summarizing text to be output directly into the reST doc. - * This should be called whenever the scanner sees a line in the Bro script - * starting with "##!" - * @param s The summary text to add to the reST doc. - */ - void AddSummary(const std::string& s) { summary.push_back(s); } + /** + * Schedules some summarizing text to be output directly into the reST doc. + * This should be called whenever the scanner sees a line in the Bro script + * starting with "##!" + * @param s The summary text to add to the reST doc. + */ + void AddSummary(const std::string& s) { summary.push_back(s); } - /** - * Schedules an import (@load) to be documented. - * This should be called whenever the scanner sees an @load. - * @param s The name of the imported script. - */ - void AddImport(const std::string& s) { imports.push_back(s); } + /** + * Schedules an import (@load) to be documented. + * This should be called whenever the scanner sees an @load. + * @param s The name of the imported script. + */ + void AddImport(const std::string& s) { imports.push_back(s); } - /** - * Schedules a namespace (module) to be documented. - * This should be called whenever the parser sees a TOK_MODULE. - * @param s The namespace (module) identifier's name. - */ - void AddModule(const std::string& s) { modules.push_back(s); } + /** + * Schedules a namespace (module) to be documented. + * This should be called whenever the parser sees a TOK_MODULE. + * @param s The namespace (module) identifier's name. + */ + void AddModule(const std::string& s) { modules.push_back(s); } - /** - * Sets the way the script changes the "capture_filters" table. - * This is determined by the scanner checking for changes to - * the "capture_filters" table after each of Bro's input scripts - * (given as command line arguments to Bro) are finished being parsed. - * @param s The value "capture_filters" as given by TableVal::Describe() - */ - void SetPacketFilter(const std::string& s); + /** + * Sets the way the script changes the "capture_filters" table. + * This is determined by the scanner checking for changes to + * the "capture_filters" table after each of Bro's input scripts + * (given as command line arguments to Bro) are finished being parsed. + * @param s The value "capture_filters" as given by TableVal::Describe() + */ + void SetPacketFilter(const std::string& s); - /** - * Schedules documentation of a given set of ports being associated - * with a particular analyzer as a result of the current script - * being loaded -- the way the "dpd_config" table is changed. - * @param analyzer An analyzer that changed the "dpd_config" table. - * @param ports The set of ports assigned to the analyzer in table. - */ - void AddPortAnalysis(const std::string& analyzer, const std::string& ports); + /** + * Schedules documentation of a given set of ports being associated + * with a particular analyzer as a result of the current script + * being loaded -- the way the "dpd_config" table is changed. + * @param analyzer An analyzer that changed the "dpd_config" table. + * @param ports The set of ports assigned to the analyzer in table. + */ + void AddPortAnalysis(const std::string& analyzer, const std::string& ports); - /** - * Sets the author of the script. - * The scanner should call this when it sees "## Author: ..." - * @param s The name, email, etc. of the script author(s). Must be - * all on one line. - */ - void SetAuthor(const std::string& s) { author_name = s; } + /** + * Sets the author of the script. + * The scanner should call this when it sees "## Author: ..." + * @param s The name, email, etc. of the script author(s). Must be + * all on one line. + */ + void SetAuthor(const std::string& s) { author_name = s; } - /** - * Schedules documentation of a script option. An option is - * defined as any variable in the script that is declared 'const' - * and has the '&redef' attribute. - * @param o A pointer to a BroDocObj which contains the internal - * Bro language representation of the script option and - * also any associated comments about it. - */ - void AddOption(const BroDocObj* o) { options.push_back(o); } + /** + * Schedules documentation of a script option. An option is + * defined as any variable in the script that is declared 'const' + * and has the '&redef' attribute. + * @param o A pointer to a BroDocObj which contains the internal + * Bro language representation of the script option and + * also any associated comments about it. + */ + void AddOption(const BroDocObj* o) { options.push_back(o); } - /** - * Schedules documentation of a script state variable. A state variable - * is defined as any variable in the script that is declared 'global' - * @param o A pointer to a BroDocObj which contains the internal - * Bro language representation of the script state variable - * and also any associated comments about it. - */ - void AddStateVar(const BroDocObj* o) { state_vars.push_back(o); } + /** + * Schedules documentation of a script state variable. A state variable + * is defined as any variable in the script that is declared 'global' + * @param o A pointer to a BroDocObj which contains the internal + * Bro language representation of the script state variable + * and also any associated comments about it. + */ + void AddStateVar(const BroDocObj* o) { state_vars.push_back(o); } - /** - * Schedules documentation of a type declared by the script. - * @param o A pointer to a BroDocObj which contains the internal - * Bro language representation of the script option and - * also any associated comments about it. - */ - void AddType(const BroDocObj* o) { types.push_back(o); } + /** + * Schedules documentation of a type declared by the script. + * @param o A pointer to a BroDocObj which contains the internal + * Bro language representation of the script option and + * also any associated comments about it. + */ + void AddType(const BroDocObj* o) { types.push_back(o); } - /** - * Schedules documentation of a Notice (enum redef) declared by script - * @param o A pointer to a BroDocObj which contains the internal - * Bro language representation of the Notice and also - * any associated comments about it. - */ - void AddNotice(const BroDocObj* o) { notices = o; } + /** + * Schedules documentation of a Notice (enum redef) declared by script + * @param o A pointer to a BroDocObj which contains the internal + * Bro language representation of the Notice and also + * any associated comments about it. + */ + void AddNotice(const BroDocObj* o) { notices = o; } - /** - * Schedules documentation of an event declared by the script. - * @param o A pointer to a BroDocObj which contains the internal - * Bro language representation of the script event and - * also any associated comments about it. - */ - void AddEvent(const BroDocObj* o) { events.push_back(o); } + /** + * Schedules documentation of an event declared by the script. + * @param o A pointer to a BroDocObj which contains the internal + * Bro language representation of the script event and + * also any associated comments about it. + */ + void AddEvent(const BroDocObj* o) { events.push_back(o); } /** - * Schedules documentation of a function declared by the script. - * @param o A pointer to a BroDocObj which contains the internal - * Bro language representation of the script function and - * also any associated comments about it. - */ - void AddFunction(const BroDocObj* o) { functions.push_back(o); } + * Schedules documentation of a function declared by the script. + * @param o A pointer to a BroDocObj which contains the internal + * Bro language representation of the script function and + * also any associated comments about it. + */ + void AddFunction(const BroDocObj* o) { functions.push_back(o); } /** - * Schedules documentation of a redef done by the script - * @param o A pointer to a BroDocObj which contains the internal - * Bro language representation of the script identifier - * that was redefined and also any associated comments. - */ - void AddRedef(const BroDocObj* o) { redefs.push_back(o); } + * Schedules documentation of a redef done by the script + * @param o A pointer to a BroDocObj which contains the internal + * Bro language representation of the script identifier + * that was redefined and also any associated comments. + */ + void AddRedef(const BroDocObj* o) { redefs.push_back(o); } - /** - * Gets the name of the Bro script source file for which reST - * documentation is being generated. - * @return A char* to the start of the source file's name. - */ - const char* GetSourceFileName() const { return source_filename.c_str(); } + /** + * Gets the name of the Bro script source file for which reST + * documentation is being generated. + * @return A char* to the start of the source file's name. + */ + const char* GetSourceFileName() const { return source_filename.c_str(); } - /** - * Gets the name of the generated reST documentation file. - * @return A char* to the start of the generated reST file's name. - */ - const char* GetOutputFileName() const { return reST_filename.c_str(); } + /** + * Gets the name of the generated reST documentation file. + * @return A char* to the start of the generated reST file's name. + */ + const char* GetOutputFileName() const { return reST_filename.c_str(); } protected: - FILE* reST_file; - std::string reST_filename; - std::string source_filename; - std::string author_name; - std::string packet_filter; + FILE* reST_file; + std::string reST_filename; + std::string source_filename; + std::string author_name; + std::string packet_filter; - std::list ls; - std::list modules; - std::list summary; - std::list imports; - std::list port_analysis; + std::list ls; + std::list modules; + std::list summary; + std::list imports; + std::list port_analysis; - std::list options; - std::list state_vars; - std::list types; - const BroDocObj* notices; - std::list events; - std::list functions; - std::list redefs; + std::list options; + std::list state_vars; + std::list types; + const BroDocObj* notices; + std::list events; + std::list functions; + std::list redefs; - /** - * Writes out a list of strings to the reST document. + /** + * Writes out a list of strings to the reST document. * If the list is empty, prints a newline character. - * @param format A printf style format string for elements of the list - * except for the last one in the list - * @param last_format A printf style format string to use for the last - * element of the list - * @param l A reference to a list of strings - */ - void WriteStringList(const char* format, - const char* last_format, - const std::list& l) const; + * @param format A printf style format string for elements of the list + * except for the last one in the list + * @param last_format A printf style format string to use for the last + * element of the list + * @param l A reference to a list of strings + */ + void WriteStringList(const char* format, + const char* last_format, + const std::list& l) const; - /** - * @see WriteStringList(const char*, const char*, - const std::list&>) - */ - void WriteStringList(const char* format, - const std::list& l) const - { WriteStringList(format, format, l); } + /** + * @see WriteStringList(const char*, const char*, + * const std::list&>) + */ + void WriteStringList(const char* format, + const std::list& l) const + { WriteStringList(format, format, l); } - /** - * Writes out a list of BroDocObj objects to the reST document - * @param l A list of BroDocObj pointers - * @param exportCond If true, filter out objects that are not in an - * export section. If false, filter out those that are in - * an export section. - * @param heading The title of the section to create in the reST doc. - * @param underline The character to use to underline the reST - * section heading. - */ - void WriteBroDocObjList(const std::list& l, - bool exportCond, - const char* heading, - char underline) const; + /** + * Writes out a list of BroDocObj objects to the reST document + * @param l A list of BroDocObj pointers + * @param exportCond If true, filter out objects that are not in an + * export section. If false, filter out those that are in + * an export section. + * @param heading The title of the section to create in the reST doc. + * @param underline The character to use to underline the reST + * section heading. + */ + void WriteBroDocObjList(const std::list& l, + bool exportCond, + const char* heading, + char underline) const; - /** - * A wrapper to fprintf() that always uses the reST document - * for the FILE* argument. - * @param format A printf style format string. - */ - void WriteToDoc(const char* format, ...) const; + /** + * A wrapper to fprintf() that always uses the reST document + * for the FILE* argument. + * @param format A printf style format string. + */ + void WriteToDoc(const char* format, ...) const; - /** - * Writes out a reST section heading - * @param heading The title of the heading to create - * @param underline The character to use to underline the section title - within the reST document - */ - void WriteSectionHeading(const char* heading, char underline) const; + /** + * Writes out a reST section heading + * @param heading The title of the heading to create + * @param underline The character to use to underline the section title + * within the reST document + */ + void WriteSectionHeading(const char* heading, char underline) const; private: - /** - * Frees memory allocated to BroDocObj's objects in a given list. - * @param a reference to a list of BroDocObj pointers - */ - void FreeBroDocObjPtrList(std::list& l); + /** + * Frees memory allocated to BroDocObj's objects in a given list. + * @param a reference to a list of BroDocObj pointers + */ + void FreeBroDocObjPtrList(std::list& l); }; #endif diff --git a/src/BroDocObj.cc b/src/BroDocObj.cc index 2b79d9fcdf..d86129faf0 100644 --- a/src/BroDocObj.cc +++ b/src/BroDocObj.cc @@ -6,34 +6,34 @@ BroDocObj::BroDocObj(const ID* id, std::list*& reST, bool is_fake) - { - broID = id; - reST_doc_strings = reST; - reST = 0; - is_fake_id = is_fake; - } + { + broID = id; + reST_doc_strings = reST; + reST = 0; + is_fake_id = is_fake; + } BroDocObj::~BroDocObj() - { - delete reST_doc_strings; - if ( is_fake_id ) delete broID; - } + { + delete reST_doc_strings; + if ( is_fake_id ) delete broID; + } void BroDocObj::WriteReST(FILE* file) const - { - ODesc desc; - desc.SetQuotes(1); - broID->DescribeReST(&desc); - fprintf(file, "%s\n", desc.Description()); + { + ODesc desc; + desc.SetQuotes(1); + broID->DescribeReST(&desc); + fprintf(file, "%s\n", desc.Description()); - if ( HasDocumentation() ) - { - fprintf(file, "\t.. bro:comment::\n"); - std::list::const_iterator it; - for ( it = reST_doc_strings->begin(); - it != reST_doc_strings->end(); ++it) - fprintf(file, "\t\t%s\n", it->c_str()); - } + if ( HasDocumentation() ) + { + fprintf(file, "\t.. bro:comment::\n"); + std::list::const_iterator it; + for ( it = reST_doc_strings->begin(); + it != reST_doc_strings->end(); ++it) + fprintf(file, "\t\t%s\n", it->c_str()); + } - fprintf(file, "\n"); - } + fprintf(file, "\n"); + } diff --git a/src/BroDocObj.h b/src/BroDocObj.h index e9254027e5..3cbe72d7c9 100644 --- a/src/BroDocObj.h +++ b/src/BroDocObj.h @@ -9,51 +9,51 @@ class BroDocObj { public: - /** - * BroDocObj constructor - * @param id a pointer to an identifier that is to be documented - * @param reST a reference to a pointer of a list of strings that - represent the reST documentation for the ID. The pointer - will be set to 0 after this constructor finishes. - * @param is_fake whether the ID* is a dummy just for doc purposes - */ - BroDocObj(const ID* id, std::list*& reST, - bool is_fake = false); + /** + * BroDocObj constructor + * @param id a pointer to an identifier that is to be documented + * @param reST a reference to a pointer of a list of strings that + * represent the reST documentation for the ID. The pointer + * will be set to 0 after this constructor finishes. + * @param is_fake whether the ID* is a dummy just for doc purposes + */ + BroDocObj(const ID* id, std::list*& reST, + bool is_fake = false); - /** - * BroDocObj destructor - * Deallocates the memory associated with the list of reST strings - */ - ~BroDocObj(); + /** + * BroDocObj destructor + * Deallocates the memory associated with the list of reST strings + */ + ~BroDocObj(); - /** - * Writes the reST representation of this object which includes - * 1) Any "##" or "##<" stylized comments. - * Anything after these style of comments is inserted as-is into - * the reST document. - * 2) a reST friendly description of the ID - * @param The (already opened) file to write the reST to. - */ - void WriteReST(FILE* file) const; + /** + * Writes the reST representation of this object which includes + * 1) Any "##" or "##<" stylized comments. + * Anything after these style of comments is inserted as-is into + * the reST document. + * 2) a reST friendly description of the ID + * @param The (already opened) file to write the reST to. + */ + void WriteReST(FILE* file) const; - /** - * Check whether this documentation is part of the public API - * (The declaration was while in an export section). - * @return true if the ID was declared in an export section, else false - */ - bool IsPublicAPI() const { return broID->IsExport(); } + /** + * Check whether this documentation is part of the public API + * (The declaration was while in an export section). + * @return true if the ID was declared in an export section, else false + */ + bool IsPublicAPI() const { return broID->IsExport(); } - /** - * Return whether this object has documentation (## comments) - * @return true if the ID has comments associated with it - */ - bool HasDocumentation() const { return reST_doc_strings && - reST_doc_strings->size() > 0; } + /** + * Return whether this object has documentation (## comments) + * @return true if the ID has comments associated with it + */ + bool HasDocumentation() const { return reST_doc_strings && + reST_doc_strings->size() > 0; } protected: - std::list* reST_doc_strings; - const ID* broID; - bool is_fake_id; /**< Whether the ID* is a dummy just for doc purposes */ + std::list* reST_doc_strings; + const ID* broID; + bool is_fake_id; /**< Whether the ID* is a dummy just for doc purposes */ private: }; From e491caeeb7d962edfb85aab66686f534a88b5240 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Mon, 21 Mar 2011 12:49:54 -0500 Subject: [PATCH 15/47] Add note to auto-generated reST about not editing it directly --- src/BroDoc.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/BroDoc.cc b/src/BroDoc.cc index 1e845a49fd..500db684f9 100644 --- a/src/BroDoc.cc +++ b/src/BroDoc.cc @@ -75,6 +75,8 @@ void BroDoc::AddPortAnalysis(const std::string& analyzer, void BroDoc::WriteDocFile() const { + WriteToDoc(".. Automatically generated. Do not edit.\n\n"); + WriteToDoc("%s\n", source_filename.c_str()); for ( size_t i = 0; i < source_filename.length(); ++i ) WriteToDoc("="); From 5fbcde7344b77e41dd525c99b41e50fb98affc69 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Mon, 21 Mar 2011 13:49:53 -0500 Subject: [PATCH 16/47] Teach Desc class how to (optionally) use spaces for indentation. And (to be consistent with current conventions for reST documentation) update places in the auto-documentation-generation framework where tabs were used in the generated reST. --- src/BroDoc.cc | 5 ++--- src/BroDocObj.cc | 5 +++-- src/Desc.cc | 10 ++++++++-- src/Desc.h | 4 ++++ src/scan.l | 2 ++ 5 files changed, 19 insertions(+), 7 deletions(-) diff --git a/src/BroDoc.cc b/src/BroDoc.cc index 500db684f9..b0ddfe72bd 100644 --- a/src/BroDoc.cc +++ b/src/BroDoc.cc @@ -90,9 +90,8 @@ void BroDoc::WriteDocFile() const WriteToDoc(":Namespaces: "); WriteStringList("`%s`, ", "`%s`\n", modules); - WriteToDoc(":Imports:\n"); - WriteStringList("\t:bro:script: `%s`\n", - "\t:bro:script: `%s`\n\n", imports); + WriteToDoc(":Imports: "); + WriteStringList(":doc:`%s`, ", ":doc:`%s`\n\n", imports); WriteSectionHeading("Notices", '-'); if ( notices ) diff --git a/src/BroDocObj.cc b/src/BroDocObj.cc index d86129faf0..c911817b30 100644 --- a/src/BroDocObj.cc +++ b/src/BroDocObj.cc @@ -22,17 +22,18 @@ BroDocObj::~BroDocObj() void BroDocObj::WriteReST(FILE* file) const { ODesc desc; + desc.SetIndentSpaces(4); desc.SetQuotes(1); broID->DescribeReST(&desc); fprintf(file, "%s\n", desc.Description()); if ( HasDocumentation() ) { - fprintf(file, "\t.. bro:comment::\n"); + fprintf(file, "\n"); std::list::const_iterator it; for ( it = reST_doc_strings->begin(); it != reST_doc_strings->end(); ++it) - fprintf(file, "\t\t%s\n", it->c_str()); + fprintf(file, " %s\n", it->c_str()); } fprintf(file, "\n"); diff --git a/src/Desc.cc b/src/Desc.cc index baf3ad1160..a70d94e188 100644 --- a/src/Desc.cc +++ b/src/Desc.cc @@ -41,6 +41,7 @@ ODesc::ODesc(desc_type t, BroFile* arg_f) want_quotes = 0; do_flush = 1; include_stats = 0; + indent_with_spaces = 0; } ODesc::~ODesc() @@ -179,8 +180,13 @@ void ODesc::AddBytes(const BroString* s) void ODesc::Indent() { - for ( int i = 0; i < indent_level; ++i ) - Add("\t", 0); + if ( indent_with_spaces > 0 ) + for ( int i = 0; i < indent_level; ++i ) + for ( int j = 0; j < indent_with_spaces; ++j ) + Add(" ", 0); + else + for ( int i = 0; i < indent_level; ++i ) + Add("\t", 0); } diff --git a/src/Desc.h b/src/Desc.h index 49b331d51b..2a40cfea11 100644 --- a/src/Desc.h +++ b/src/Desc.h @@ -53,6 +53,9 @@ public: void PopIndent(); int GetIndentLevel() const { return indent_level; } + int IndentSpaces() const { return indent_with_spaces; } + void SetIndentSpaces(int i) { indent_with_spaces = i; } + void Add(const char* s, int do_indent=1); void AddN(const char* s, int len) { AddBytes(s, len); } void Add(int i); @@ -135,6 +138,7 @@ protected: int want_quotes; int do_flush; int include_stats; + int indent_with_spaces; }; #endif diff --git a/src/scan.l b/src/scan.l index 4f2a4cf743..a032d31ea2 100644 --- a/src/scan.l +++ b/src/scan.l @@ -908,6 +908,7 @@ static void check_capture_filter_changes() if ( capture_filters ) { ODesc desc; + desc.SetIndentSpaces(4); capture_filters->ID_Val()->Describe(&desc); last_reST_doc->SetPacketFilter(desc.Description()); capture_filters->ID_Val()->AsTableVal()->RemoveAll(); @@ -937,6 +938,7 @@ static void check_dpd_config_changes() int tag = key->AsListVal()->Index(0)->AsCount(); ODesc valdesc; + valdesc.SetIndentSpaces(4); valdesc.PushIndent(); v->Describe(&valdesc); From e0a77cb794ccc1f3f5565b3b1eebc00bfc5564aa Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Mon, 21 Mar 2011 14:03:11 -0500 Subject: [PATCH 17/47] Auto-doc framework now handles multiple script authors more conveniently. --- src/BroDoc.cc | 3 ++- src/BroDoc.h | 10 ++++------ src/scan.l | 2 +- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/src/BroDoc.cc b/src/BroDoc.cc index b0ddfe72bd..80432570a4 100644 --- a/src/BroDoc.cc +++ b/src/BroDoc.cc @@ -85,7 +85,8 @@ void BroDoc::WriteDocFile() const WriteSectionHeading("Summary", '-'); WriteStringList("%s\n", "%s\n\n", summary); - WriteToDoc(":Author: %s\n", author_name.c_str()); + WriteToDoc(":Author(s): "); + WriteStringList("%s, ", "%s\n", authors); WriteToDoc(":Namespaces: "); WriteStringList("`%s`, ", "`%s`\n", modules); diff --git a/src/BroDoc.h b/src/BroDoc.h index 30b6fe4426..91d51f0dd5 100644 --- a/src/BroDoc.h +++ b/src/BroDoc.h @@ -83,12 +83,11 @@ public: void AddPortAnalysis(const std::string& analyzer, const std::string& ports); /** - * Sets the author of the script. + * Adds a script author. * The scanner should call this when it sees "## Author: ..." - * @param s The name, email, etc. of the script author(s). Must be - * all on one line. + * @param s The name, email, etc. of the script author. */ - void SetAuthor(const std::string& s) { author_name = s; } + void AddAuthor(const std::string& s) { authors.push_back(s); } /** * Schedules documentation of a script option. An option is @@ -166,10 +165,9 @@ protected: FILE* reST_file; std::string reST_filename; std::string source_filename; - std::string author_name; std::string packet_filter; - std::list ls; + std::list authors; std::list modules; std::list summary; std::list imports; diff --git a/src/scan.l b/src/scan.l index a032d31ea2..6a104d8776 100644 --- a/src/scan.l +++ b/src/scan.l @@ -137,7 +137,7 @@ ESCSEQ (\\([^\n]|[0-7]+|x[[:xdigit:]]+)) ##{OWS}Author:.* { if ( generate_documentation ) - current_reST_doc->SetAuthor( + current_reST_doc->AddAuthor( skip_whitespace( // Skip whitespace after "Author:" skip_whitespace(yytext + 2) // Skip whitespace after ## + 7) // Skip "Author:" From 384fa03c2693e8a6c595f0229a1ffcb1a451184f Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Mon, 21 Mar 2011 15:08:39 -0500 Subject: [PATCH 18/47] Revising notice, port analysis, packet filter, auto-generated documentation. They're now all optional sections -- if a given bro script doesn't use the functionality, then the documentation doesn't mention anything about it. --- src/BroDoc.cc | 54 ++++++++++++++++++++++++++++++++++++++++----------- src/BroDoc.h | 15 ++++++++++++-- 2 files changed, 56 insertions(+), 13 deletions(-) diff --git a/src/BroDoc.cc b/src/BroDoc.cc index 80432570a4..1de346987d 100644 --- a/src/BroDoc.cc +++ b/src/BroDoc.cc @@ -40,7 +40,6 @@ BroDoc::BroDoc(const std::string& sourcename) else fprintf(stdout, "Created reST document: %s\n", reST_filename.c_str()); #endif - notices = 0; } BroDoc::~BroDoc() @@ -51,10 +50,10 @@ BroDoc::~BroDoc() FreeBroDocObjPtrList(options); FreeBroDocObjPtrList(state_vars); FreeBroDocObjPtrList(types); + FreeBroDocObjPtrList(notices); FreeBroDocObjPtrList(events); FreeBroDocObjPtrList(functions); FreeBroDocObjPtrList(redefs); - if ( notices ) delete notices; } void BroDoc::SetPacketFilter(const std::string& s) @@ -64,12 +63,24 @@ void BroDoc::SetPacketFilter(const std::string& s) size_t pos2 = s.find("}"); if ( pos1 != std::string::npos && pos2 != std::string::npos ) packet_filter = s.substr(pos1 + 2, pos2 - 2); + + bool has_non_whitespace = false; + for ( std::string::const_iterator it = packet_filter.begin(); + it != packet_filter.end(); ++it ) + if ( *it != ' ' && *it != '\t' && *it != '\n' && *it != '\r' ) + { + has_non_whitespace = true; + break; + } + + if ( ! has_non_whitespace ) + packet_filter.clear(); } void BroDoc::AddPortAnalysis(const std::string& analyzer, const std::string& ports) { - std::string reST_string = analyzer + "::\n" + ports + "\n"; + std::string reST_string = analyzer + "::\n" + ports + "\n\n"; port_analysis.push_back(reST_string); } @@ -92,17 +103,28 @@ void BroDoc::WriteDocFile() const WriteStringList("`%s`, ", "`%s`\n", modules); WriteToDoc(":Imports: "); - WriteStringList(":doc:`%s`, ", ":doc:`%s`\n\n", imports); + WriteStringList(":doc:`%s`, ", ":doc:`%s`\n", imports); - WriteSectionHeading("Notices", '-'); - if ( notices ) - notices->WriteReST(reST_file); + WriteToDoc("\n"); - WriteSectionHeading("Port Analysis", '-'); - WriteStringList("%s", port_analysis); + if ( ! notices.empty() ) + WriteBroDocObjList(notices, "Notices", '-'); - WriteSectionHeading("Packet Filter", '-'); - WriteToDoc("%s\n", packet_filter.c_str()); + if ( ! port_analysis.empty() ) + { + WriteSectionHeading("Port Analysis", '-'); + WriteStringList("%s", port_analysis); + } + + if ( ! packet_filter.empty() ) + { + WriteSectionHeading("Packet Filter", '-'); + WriteToDoc(".. note:: Filters are only relevant when dynamic protocol " + "detection (DPD) is explicitly turned off (Bro release 1.6 " + "enabled DPD by default).\n\n"); + WriteToDoc("Filters added::\n\n"); + WriteToDoc("%s\n", packet_filter.c_str()); + } WriteSectionHeading("Public Interface", '-'); WriteBroDocObjList(options, true, "Options", '~'); @@ -161,6 +183,16 @@ void BroDoc::WriteBroDocObjList(const std::list& l, } } +void BroDoc::WriteBroDocObjList(const std::list& l, + const char* heading, + char underline) const + { + WriteSectionHeading(heading, underline); + std::list::const_iterator it; + for ( it = l.begin(); it != l.end(); ++it ) + (*it)->WriteReST(reST_file); + } + void BroDoc::WriteToDoc(const char* format, ...) const { va_list argp; diff --git a/src/BroDoc.h b/src/BroDoc.h index 91d51f0dd5..a28a07561e 100644 --- a/src/BroDoc.h +++ b/src/BroDoc.h @@ -122,7 +122,7 @@ public: * Bro language representation of the Notice and also * any associated comments about it. */ - void AddNotice(const BroDocObj* o) { notices = o; } + void AddNotice(const BroDocObj* o) { notices.push_back(o); } /** * Schedules documentation of an event declared by the script. @@ -176,7 +176,7 @@ protected: std::list options; std::list state_vars; std::list types; - const BroDocObj* notices; + std::list notices; std::list events; std::list functions; std::list redefs; @@ -217,6 +217,17 @@ protected: const char* heading, char underline) const; + /** + * Writes out a list of BroDocObj objects to the reST document + * @param l A list of BroDocObj pointers + * @param heading The title of the section to create in the reST doc. + * @param underline The character to use to underline the reST + * section heading. + */ + void WriteBroDocObjList(const std::list& l, + const char* heading, + char underline) const; + /** * A wrapper to fprintf() that always uses the reST document * for the FILE* argument. From bbe7c98ab39976523702b36f4588a8bba58808d0 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Tue, 22 Mar 2011 16:05:59 -0500 Subject: [PATCH 19/47] Major reformatting of auto-generated reST documentation. Introduces reST directives and roles in a "bro" domain that Sphinx will be taught to recognize. --- src/Attr.cc | 10 ++++-- src/BroDoc.cc | 10 ++++-- src/BroDocObj.cc | 14 ++++++--- src/BroDocObj.h | 15 +++++++++ src/Desc.cc | 6 ++++ src/Desc.h | 1 + src/ID.cc | 34 ++++++++++++++++----- src/ID.h | 2 +- src/Type.cc | 79 ++++++++++++++++++++++++++++++++---------------- src/Val.cc | 2 ++ src/parse.y | 4 ++- 11 files changed, 132 insertions(+), 45 deletions(-) diff --git a/src/Attr.cc b/src/Attr.cc index 31a01b3070..f7f7f0576a 100644 --- a/src/Attr.cc +++ b/src/Attr.cc @@ -51,12 +51,17 @@ void Attr::Describe(ODesc* d) const void Attr::DescribeReST(ODesc* d) const { - d->Add(".. bro:attr:: "); + d->Add(":bro:attr:`"); AddTag(d); + d->Add("`"); if ( expr ) { + d->SP(); d->Add("="); + d->SP(); + d->Add("``"); expr->Describe(d); + d-> Add("``"); } } @@ -176,7 +181,8 @@ void Attributes::DescribeReST(ODesc* d) const { loop_over_list(*attrs, i) { - d->NL(); + if ( i > 0 ) + d->Add(" "); (*attrs)[i]->DescribeReST(d); } } diff --git a/src/BroDoc.cc b/src/BroDoc.cc index 1de346987d..12b8cce284 100644 --- a/src/BroDoc.cc +++ b/src/BroDoc.cc @@ -99,14 +99,18 @@ void BroDoc::WriteDocFile() const WriteToDoc(":Author(s): "); WriteStringList("%s, ", "%s\n", authors); - WriteToDoc(":Namespaces: "); - WriteStringList("`%s`, ", "`%s`\n", modules); - WriteToDoc(":Imports: "); WriteStringList(":doc:`%s`, ", ":doc:`%s`\n", imports); WriteToDoc("\n"); + if ( ! modules.empty() ) + { + WriteSectionHeading("Namespaces", '-'); + WriteStringList(".. bro:namespace:: %s\n", modules); + WriteToDoc("\n"); + } + if ( ! notices.empty() ) WriteBroDocObjList(notices, "Notices", '-'); diff --git a/src/BroDocObj.cc b/src/BroDocObj.cc index c911817b30..326550e545 100644 --- a/src/BroDocObj.cc +++ b/src/BroDocObj.cc @@ -11,6 +11,7 @@ BroDocObj::BroDocObj(const ID* id, std::list*& reST, reST_doc_strings = reST; reST = 0; is_fake_id = is_fake; + use_role = 0; } BroDocObj::~BroDocObj() @@ -21,11 +22,12 @@ BroDocObj::~BroDocObj() void BroDocObj::WriteReST(FILE* file) const { + int indent_spaces = 3; ODesc desc; - desc.SetIndentSpaces(4); + desc.SetIndentSpaces(indent_spaces); desc.SetQuotes(1); - broID->DescribeReST(&desc); - fprintf(file, "%s\n", desc.Description()); + broID->DescribeReST(&desc, use_role); + fprintf(file, "%s", desc.Description()); if ( HasDocumentation() ) { @@ -33,7 +35,11 @@ void BroDocObj::WriteReST(FILE* file) const std::list::const_iterator it; for ( it = reST_doc_strings->begin(); it != reST_doc_strings->end(); ++it) - fprintf(file, " %s\n", it->c_str()); + { + for ( int i = 0; i < indent_spaces; ++i ) + fprintf(file, " "); + fprintf(file, "%s\n", it->c_str()); + } } fprintf(file, "\n"); diff --git a/src/BroDocObj.h b/src/BroDocObj.h index 3cbe72d7c9..e3fd30518b 100644 --- a/src/BroDocObj.h +++ b/src/BroDocObj.h @@ -50,10 +50,25 @@ public: bool HasDocumentation() const { return reST_doc_strings && reST_doc_strings->size() > 0; } + /** + * @return whether this object will use reST role (T) or directive (F) + * notation for the wrapped identifier. Roles are usually used + * for cross-referencing. + */ + bool UseRole() const { return use_role; } + + /** + * @param b whether this object will use reST role (T) or directive (F) + * notation for the wrapped identifier. Roles are usually used + * for cross-referencing. + */ + void SetRole(bool b) { use_role = b; } + protected: std::list* reST_doc_strings; const ID* broID; bool is_fake_id; /**< Whether the ID* is a dummy just for doc purposes */ + bool use_role; /**< Whether to use a reST role or directive for the ID */ private: }; diff --git a/src/Desc.cc b/src/Desc.cc index a70d94e188..57e67e6420 100644 --- a/src/Desc.cc +++ b/src/Desc.cc @@ -68,6 +68,12 @@ void ODesc::PopIndent() NL(); } +void ODesc::PopIndentNoNL() + { + if ( --indent_level < 0 ) + internal_error("ODesc::PopIndent underflow"); + } + void ODesc::Add(const char* s, int do_indent) { unsigned int n = strlen(s); diff --git a/src/Desc.h b/src/Desc.h index 2a40cfea11..1dc3b5434d 100644 --- a/src/Desc.h +++ b/src/Desc.h @@ -51,6 +51,7 @@ public: void PushIndent(); void PopIndent(); + void PopIndentNoNL(); int GetIndentLevel() const { return indent_level; } int IndentSpaces() const { return indent_with_spaces; } diff --git a/src/ID.cc b/src/ID.cc index 03403af86f..c4cf5a58c1 100644 --- a/src/ID.cc +++ b/src/ID.cc @@ -607,39 +607,57 @@ void ID::DescribeExtended(ODesc* d) const } } -void ID::DescribeReST(ODesc* d) const +void ID::DescribeReST(ODesc* d, bool is_role) const { - d->Add(".. bro:id:: "); - d->Add(name); + if ( is_role ) + { + if ( is_type ) + d->Add(":bro:type:`"); + else + d->Add(":bro:id:`"); + d->Add(name); + d->Add("`"); + } + else + { + if ( is_type ) + d->Add(".. bro:type:: "); + else + d->Add(".. bro:id:: "); + d->Add(name); + } d->PushIndent(); + d->NL(); if ( type ) { - d->Add(".. bro:type:: "); + d->Add(":Type: "); if ( ! is_type && type->GetTypeID() ) { - d->Add("`"); + d->Add(":bro:type:`"); d->Add(type->GetTypeID()); d->Add("`"); } else type->DescribeReST(d); + d->NL(); } if ( attrs ) { + d->Add(":Attributes: "); attrs->DescribeReST(d); + d->NL(); } if ( val && type && type->InternalType() != TYPE_INTERNAL_OTHER && type->InternalType() != TYPE_INTERNAL_VOID ) { - d->NL(); - d->Add(".. bro:val:: "); + d->Add(":Init: "); val->DescribeReST(d); + d->NL(); } - } #ifdef DEBUG diff --git a/src/ID.h b/src/ID.h index 7ebaf27918..90a0862a18 100644 --- a/src/ID.h +++ b/src/ID.h @@ -85,7 +85,7 @@ public: // Adds type and value to description. void DescribeExtended(ODesc* d) const; // Produces a description that's reST-ready - void DescribeReST(ODesc* d) const; + void DescribeReST(ODesc* d, bool is_role=false) const; bool Serialize(SerialInfo* info) const; static ID* Unserialize(UnserialInfo* info); diff --git a/src/Type.cc b/src/Type.cc index d74d02770c..ff5464843a 100644 --- a/src/Type.cc +++ b/src/Type.cc @@ -149,7 +149,9 @@ void BroType::Describe(ODesc* d) const void BroType::DescribeReST(ODesc* d) const { - Describe(d); + d->Add(":bro:type:`"); + d->Add(type_name(Tag())); + d->Add("`"); } void BroType::SetError() @@ -439,20 +441,22 @@ void IndexType::Describe(ODesc* d) const void IndexType::DescribeReST(ODesc* d) const { + d->Add(":bro:type:`"); if ( IsSet() ) d->Add("set"); else d->Add(type_name(Tag())); + d->Add("` "); d->Add("["); loop_over_list(*IndexTypes(), i) { if ( i > 0 ) - d->Add(","); + d->Add(", "); const BroType* t = (*IndexTypes())[i]; if ( t->GetTypeID() ) { - d->Add("`"); + d->Add(":bro:type:`"); d->Add(t->GetTypeID()); d->Add("`"); } @@ -466,7 +470,7 @@ void IndexType::DescribeReST(ODesc* d) const d->Add(" of "); if ( yield_type->GetTypeID() ) { - d->Add("`"); + d->Add(":bro:type:`"); d->Add(yield_type->GetTypeID()); d->Add("`"); } @@ -711,8 +715,10 @@ void FuncType::Describe(ODesc* d) const void FuncType::DescribeReST(ODesc* d) const { + d->Add(":bro:type:`"); d->Add(is_event ? "event" : "function"); - d->Add("("); + d->Add("`"); + d->Add(" ("); args->DescribeFieldsReST(d, true); d->Add(")"); if ( yield ) @@ -720,7 +726,7 @@ void FuncType::DescribeReST(ODesc* d) const d->AddSP(" :"); if ( yield->GetTypeID() ) { - d->Add("`"); + d->Add(":bro:type:`"); d->Add(yield->GetTypeID()); d->Add("`"); } @@ -997,7 +1003,8 @@ void RecordType::Describe(ODesc* d) const void RecordType::DescribeReST(ODesc* d) const { - d->Add("record"); + d->Add(":bro:type:`record`"); + d->NL(); DescribeFieldsReST(d, false); } @@ -1048,36 +1055,46 @@ void RecordType::DescribeFieldsReST(ODesc* d, bool func_args) const for ( int i = 0; i < num_fields; ++i ) { const TypeDecl* td = FieldDecl(i); - if ( ! func_args ) - d->Add(":bro:field: "); + + if ( i > 0 ) + if ( func_args ) + d->Add(", "); + else + { + d->NL(); + d->NL(); + } d->Add(td->id); d->Add(": "); - d->Add(":bro:type: "); if ( td->type->GetTypeID() ) { - d->Add("`"); + d->Add(":bro:type:`"); d->Add(td->type->GetTypeID()); d->Add("`"); } else td->type->DescribeReST(d); + if ( td->attrs ) + { + d->SP(); + td->attrs->DescribeReST(d); + } + if ( ! func_args ) + { if ( td->comment ) { d->PushIndent(); - d->Add(".. bro:comment:: "); d->Add(td->comment); - d->PopIndent(); + d->PopIndentNoNL(); } - - if ( i + 1 != num_fields ) - if ( func_args ) - d->Add(", "); - else - d->NL(); + } } + + if ( ! func_args ) + d->PopIndentNoNL(); } IMPLEMENT_SERIAL(RecordType, SER_RECORD_TYPE) @@ -1362,24 +1379,34 @@ void CommentedEnumType::DescribeReST(ODesc* d) const for ( NameMap::const_iterator it = names.begin(); it != names.end(); ++it ) rev[it->second] = it->first; + d->Add(":bro:type:`"); d->Add(type_name(Tag())); + d->Add("`"); d->PushIndent(); + d->NL(); - for ( RevNameMap::const_iterator it = rev.begin(); it != rev.end(); ) + for ( RevNameMap::const_iterator it = rev.begin(); it != rev.end(); ++it ) { + if ( it != rev.begin() ) + { + d->NL(); + d->NL(); + } + d->Add(".. bro:enum:: "); - d->Add(it->second); + d->AddSP(it->second); + d->Add(GetTypeID()); + CommentMap::const_iterator cmnt_it = comments.find(it->second); if ( cmnt_it != comments.end() ) { d->PushIndent(); - d->Add(".. bro:comment:: "); - d->Add(cmnt_it->second); - d->PopIndent(); - } - if ( ++it != rev.end() ) d->NL(); + d->Add(cmnt_it->second); + d->PopIndentNoNL(); + } } + d->PopIndentNoNL(); } IMPLEMENT_SERIAL(EnumType, SER_ENUM_TYPE); diff --git a/src/Val.cc b/src/Val.cc index f4c471b8dc..05ca0adc65 100644 --- a/src/Val.cc +++ b/src/Val.cc @@ -576,7 +576,9 @@ void Val::Describe(ODesc* d) const void Val::DescribeReST(ODesc* d) const { + d->Add("``"); ValDescribeReST(d); + d->Add("``"); } void Val::ValDescribe(ODesc* d) const diff --git a/src/parse.y b/src/parse.y index 5e4966fd5e..3c537e185e 100644 --- a/src/parse.y +++ b/src/parse.y @@ -153,12 +153,13 @@ static void add_enum_comment (const char* comment) cur_enum_type_doc->AddComment(current_module, cur_enum_elem_id, comment); } -static ID* create_dummy_id (const char* name, const BroType* type) +static ID* create_dummy_id (const char* name, BroType* type) { // normally, install_ID() figures out the right IDScope // but it doesn't matter for the dummy ID so use SCOPE_GLOBAL ID* fake_id = new ID(copy_string(name), SCOPE_GLOBAL, is_export); fake_id->SetType(cur_enum_type_doc); + type->SetTypeID(copy_string(name)); fake_id->MakeType(); return fake_id; } @@ -995,6 +996,7 @@ decl: ID* fake_id = create_dummy_id($3->Name(), cur_enum_type_doc); cur_enum_type_doc = 0; BroDocObj* o = new BroDocObj(fake_id, reST_doc_comments, true); + o->SetRole(true); if ( streq(fake_id->Name(), "Notice" ) ) current_reST_doc->AddNotice(o); else From 037a6906af3601917247bd68a4b8edad0ff4f117 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Wed, 23 Mar 2011 15:50:26 -0500 Subject: [PATCH 20/47] Canonicalize "##" and "## " stylized comments, revise record/enum documentation tracking The documentation framework now sees "##Text" and "## Text" as equivalent documentation comments. This prevents unintentional indentation in the generated reST as a result of the later style, but still allows embedded reST markup that relies on indentation of more than two spaces to work as expected. Comments associated with record fields and enums values are able to span multiple "##"-stylized comments, allowing for more robust reST markup to be embedded. The documentation framework now tracks record fields through a new CommentedTypeDecl subclass of TypeDecl that the parser constructs in parallel with the real TypeDecl. --- src/Type.cc | 123 ++++++++++++++++++++++++++++++---------------------- src/Type.h | 20 +++++++-- src/parse.y | 103 ++++++++++++++++++++++++++++--------------- src/scan.l | 32 ++++++++++---- 4 files changed, 178 insertions(+), 100 deletions(-) diff --git a/src/Type.cc b/src/Type.cc index ff5464843a..6c8035ca31 100644 --- a/src/Type.cc +++ b/src/Type.cc @@ -10,6 +10,10 @@ #include "Scope.h" #include "Serializer.h" +#include +#include +#include + extern int generate_documentation; const char* type_name(TypeTag t) @@ -773,7 +777,6 @@ TypeDecl::TypeDecl(BroType* t, const char* i, attr_list* arg_attrs) type = t; attrs = arg_attrs ? new Attributes(arg_attrs, t) : 0; id = i; - comment = 0; } TypeDecl::~TypeDecl() @@ -781,7 +784,6 @@ TypeDecl::~TypeDecl() Unref(type); Unref(attrs); delete [] id; - if ( comment ) delete [] comment; } bool TypeDecl::Serialize(SerialInfo* info) const @@ -794,11 +796,6 @@ bool TypeDecl::Serialize(SerialInfo* info) const if ( ! (type->Serialize(info) && SERIALIZE(id)) ) return false; - if ( generate_documentation ) - { - SERIALIZE_OPTIONAL_STR(comment); - } - return true; } @@ -815,14 +812,59 @@ TypeDecl* TypeDecl::Unserialize(UnserialInfo* info) return 0; } - if ( generate_documentation ) - { - UNSERIALIZE_OPTIONAL_STR_DEL(t->comment, t); - } - return t; } +void TypeDecl::DescribeReST(ODesc* d) const + { + d->Add(id); + d->Add(": "); + if ( type->GetTypeID() ) + { + d->Add(":bro:type:`"); + d->Add(type->GetTypeID()); + d->Add("`"); + } + else + type->DescribeReST(d); + + if ( attrs ) + { + d->SP(); + attrs->DescribeReST(d); + } + } + +CommentedTypeDecl::CommentedTypeDecl(BroType* t, const char* i, + attr_list* attrs, + std::list* cmnt_list) + : TypeDecl(t, i, attrs) + { + comments = cmnt_list; + } + +CommentedTypeDecl::~CommentedTypeDecl() + { + if ( comments ) delete comments; + } + +void CommentedTypeDecl::DescribeReST(ODesc* d) const + { + TypeDecl::DescribeReST(d); + + if ( comments ) + { + d->PushIndent(); + std::list::const_iterator i; + for ( i = comments->begin(); i != comments->end(); ++i) + { + if ( i != comments->begin() ) d->NL(); + d->Add(i->c_str()); + } + d->PopIndentNoNL(); + } + } + RecordField::RecordField(int arg_base, int arg_offset, int arg_total_offset) { base = arg_base; @@ -1054,8 +1096,6 @@ void RecordType::DescribeFieldsReST(ODesc* d, bool func_args) const for ( int i = 0; i < num_fields; ++i ) { - const TypeDecl* td = FieldDecl(i); - if ( i > 0 ) if ( func_args ) d->Add(", "); @@ -1065,32 +1105,7 @@ void RecordType::DescribeFieldsReST(ODesc* d, bool func_args) const d->NL(); } - d->Add(td->id); - d->Add(": "); - if ( td->type->GetTypeID() ) - { - d->Add(":bro:type:`"); - d->Add(td->type->GetTypeID()); - d->Add("`"); - } - else - td->type->DescribeReST(d); - - if ( td->attrs ) - { - d->SP(); - td->attrs->DescribeReST(d); - } - - if ( ! func_args ) - { - if ( td->comment ) - { - d->PushIndent(); - d->Add(td->comment); - d->PopIndentNoNL(); - } - } + FieldDecl(i)->DescribeReST(d); } if ( ! func_args ) @@ -1259,7 +1274,7 @@ CommentedEnumType::~CommentedEnumType() for ( CommentMap::iterator iter = comments.begin(); iter != comments.end(); ++iter ) { delete [] iter->first; - delete [] iter->second; + delete iter->second; } } @@ -1292,26 +1307,22 @@ void EnumType::AddName(const string& module_name, const char* name, bro_int_t va AddNameInternal(module_name, name, val, is_export); } -void CommentedEnumType::AddComment(const string& module_name, const char* name, const char* comment) +void CommentedEnumType::AddComment(const string& module_name, const char* name, + std::list* new_comments) { - if ( ! comment ) return; + if ( ! new_comments ) return; string fullname = make_full_var_name(module_name.c_str(), name); CommentMap::iterator it = comments.find(fullname.c_str()); if ( it == comments.end() ) - comments[copy_string(fullname.c_str())] = comment; + comments[copy_string(fullname.c_str())] = new_comments; else { - // append to current comments - size_t len = strlen(it->second) + strlen(comment) + 1; - char* s = new char[len]; - sprintf(s, "%s%s", it->second, comment); - s[len - 1] = '\0'; - delete [] it->second; - delete [] comment; - comments[fullname.c_str()] = s; + comments[fullname.c_str()]->splice(comments[fullname.c_str()]->end(), + *new_comments); + delete [] new_comments; } } @@ -1402,7 +1413,13 @@ void CommentedEnumType::DescribeReST(ODesc* d) const { d->PushIndent(); d->NL(); - d->Add(cmnt_it->second); + std::list::const_iterator i; + const std::list* cmnt_list = cmnt_it->second; + for ( i = cmnt_list->begin(); i != cmnt_list->end(); ++i) + { + if ( i != cmnt_list->begin() ) d->NL(); + d->Add(i->c_str()); + } d->PopIndentNoNL(); } } diff --git a/src/Type.h b/src/Type.h index 7fc89dfcaf..087fc29c9c 100644 --- a/src/Type.h +++ b/src/Type.h @@ -6,6 +6,7 @@ #define type_h #include +#include #include #include "Obj.h" @@ -381,12 +382,22 @@ public: bool Serialize(SerialInfo* info) const; static TypeDecl* Unserialize(UnserialInfo* info); + virtual void DescribeReST(ODesc* d) const; + BroType* type; Attributes* attrs; const char* id; +}; - // comments are only filled when in "documentation mode" - const char* comment; +class CommentedTypeDecl : public TypeDecl { +public: + CommentedTypeDecl(BroType* t, const char* i, attr_list* attrs = 0, + std::list* cmnt_list = 0); + ~CommentedTypeDecl(); + + void DescribeReST(ODesc* d) const; + + std::list* comments; }; class RecordField { @@ -506,7 +517,8 @@ public: CommentedEnumType() {} ~CommentedEnumType(); void DescribeReST(ODesc* d) const; - void AddComment(const string& module_name, const char* name, const char* comment); + void AddComment(const string& module_name, const char* name, + std::list* comments); protected: // This overriden method does not install the given ID name into a @@ -515,7 +527,7 @@ protected: void AddNameInternal(const string& module_name, const char* name, bro_int_t val, bool is_export); // comments are only filled when in "documentation mode" - typedef std::map< const char*, const char*, ltstr > CommentMap; + typedef std::map< const char*, std::list*, ltstr > CommentMap; CommentMap comments; }; diff --git a/src/parse.y b/src/parse.y index 3c537e185e..1d2d4f277f 100644 --- a/src/parse.y +++ b/src/parse.y @@ -43,7 +43,8 @@ %right '!' %left '$' '[' ']' '(' ')' TOK_HAS_FIELD TOK_HAS_ATTR -%type TOK_ID TOK_PATTERN_TEXT single_pattern TOK_DOC TOK_POST_DOC opt_doc_list opt_post_doc_list +%type TOK_ID TOK_PATTERN_TEXT single_pattern TOK_DOC TOK_POST_DOC +%type opt_doc_list opt_post_doc_list %type local_id global_id event_id global_or_event_id resolve_id begin_func %type local_id_list %type init_class @@ -119,6 +120,9 @@ EnumType *cur_enum_type = 0; CommentedEnumType *cur_enum_type_doc = 0; const char* cur_enum_elem_id = 0; +type_decl_list* fake_type_decl_list = 0; +TypeDecl* last_fake_type_decl = 0; + static void parser_new_enum (void) { /* Starting a new enum definition. */ @@ -148,9 +152,9 @@ static void parser_redef_enum (ID *id) cur_enum_type_doc = new CommentedEnumType(); } -static void add_enum_comment (const char* comment) +static void add_enum_comment (std::list* comments) { - cur_enum_type_doc->AddComment(current_module, cur_enum_elem_id, comment); + cur_enum_type_doc->AddComment(current_module, cur_enum_elem_id, comments); } static ID* create_dummy_id (const char* name, BroType* type) @@ -158,41 +162,31 @@ static ID* create_dummy_id (const char* name, BroType* type) // normally, install_ID() figures out the right IDScope // but it doesn't matter for the dummy ID so use SCOPE_GLOBAL ID* fake_id = new ID(copy_string(name), SCOPE_GLOBAL, is_export); - fake_id->SetType(cur_enum_type_doc); + fake_id->SetType(type); type->SetTypeID(copy_string(name)); fake_id->MakeType(); return fake_id; } -static char* concat_opt_docs (const char* pre, const char* post) +static std::list* concat_opt_docs (std::list* pre, + std::list* post) { - if ( ! pre && ! post ) - return 0; + if ( ! pre && ! post ) return 0; - size_t len = 0; - if ( pre ) - len += strlen(pre); - if ( post ) - len += strlen(post); - char* s = new char[len + 1]; - s[0] = '\0'; - if ( pre ) - { - strcat(s, pre); - delete [] pre; - } - if ( post ) - { - strcat(s, post); - delete [] post; - } - return s; + if ( pre && ! post ) return pre; + + if ( ! pre && post ) return post; + + pre->splice(pre->end(), *post); + delete post; + return pre; } %} %union { char* str; + std::list* str_l; ID* id; id_list* id_l; init_class ic; @@ -898,18 +892,40 @@ type_list: type_decl_list: type_decl_list type_decl - { $1->append($2); } + { + $1->append($2); + if ( generate_documentation && last_fake_type_decl ) + { + fake_type_decl_list->append(last_fake_type_decl); + last_fake_type_decl = 0; + } + } | - { $$ = new type_decl_list(); } + { + $$ = new type_decl_list(); + if ( generate_documentation ) + fake_type_decl_list = new type_decl_list(); + } ; type_decl: opt_doc_list TOK_ID ':' type opt_attr ';' opt_post_doc_list { set_location(@2, @6); - $$ = new TypeDecl($4, $2, $5); if ( generate_documentation ) - $$->comment = concat_opt_docs($1, $7); + { + attr_list* a = $5; + attr_list* a_copy = 0; + if ( a ) + { + a_copy = new attr_list; + loop_over_list(*a, i) + a_copy->append((*a)[i]); + } + last_fake_type_decl = new CommentedTypeDecl( + $4, $2, a_copy, concat_opt_docs($1, $7)); + } + $$ = new TypeDecl($4, $2, $5); } ; @@ -1009,13 +1025,22 @@ decl: add_type($2, $4, $5, 0); if ( generate_documentation ) { - if ( $2->AsType()->Tag() == TYPE_ENUM && cur_enum_type_doc ) + TypeTag t = $2->AsType()->Tag(); + if ( t == TYPE_ENUM && cur_enum_type_doc ) { ID* fake = create_dummy_id($2->Name(), cur_enum_type_doc); cur_enum_type_doc = 0; current_reST_doc->AddType( new BroDocObj(fake, reST_doc_comments, true)); } + else if ( t == TYPE_RECORD && fake_type_decl_list ) + { + BroType* fake_record = new RecordType(fake_type_decl_list); + ID* fake = create_dummy_id($2->Name(), fake_record); + fake_type_decl_list = 0; + current_reST_doc->AddType( + new BroDocObj(fake, reST_doc_comments, true)); + } else current_reST_doc->AddType( new BroDocObj($2, reST_doc_comments)); @@ -1484,11 +1509,16 @@ resolve_id: opt_post_doc_list: opt_post_doc_list TOK_POST_DOC { - $$ = concat_opt_docs($1, $2); + $1->push_back($2); + $$ = $1; } | TOK_POST_DOC - { $$ = $1; } + { + $$ = new std::list(); + $$->push_back($1); + delete [] $1; + } | { $$ = 0; } ; @@ -1496,11 +1526,16 @@ opt_post_doc_list: opt_doc_list: opt_doc_list TOK_DOC { - $$ = concat_opt_docs($1, $2); + $1->push_back($2); + $$ = $1; } | TOK_DOC - { $$ = $1; } + { + $$ = new std::list(); + $$->push_back($1); + delete [] $1; + } | { $$ = 0; } ; diff --git a/src/scan.l b/src/scan.l index 6a104d8776..53b6afd72a 100644 --- a/src/scan.l +++ b/src/scan.l @@ -75,6 +75,15 @@ static void check_capture_filter_changes(); // adds changes to dpd_config to the current script's reST documentation static void check_dpd_config_changes(); +static const char* canon_doc_comment(const char* comment) + { + // "##Text" and "## Text" are treated the same in order to be able + // to still preserve indentation level, but not unintentionally + // signify an indentation level for all the text when using + // the "## Text" style + return ( comment[0] == ' ' ) ? comment + 1 : comment; + } + class FileInfo { public: FileInfo(string restore_module = ""); @@ -132,7 +141,9 @@ ESCSEQ (\\([^\n]|[0-7]+|x[[:xdigit:]]+)) ##!.* { // Add this format of comments to the script documentation's "summary" if ( generate_documentation ) - current_reST_doc->AddSummary(yytext + 3); + { + current_reST_doc->AddSummary(canon_doc_comment(yytext + 3)); + } } ##{OWS}Author:.* { @@ -145,22 +156,25 @@ ESCSEQ (\\([^\n]|[0-7]+|x[[:xdigit:]]+)) } ##<.* { - yylval.str = copy_string(yytext + 3); - return TOK_POST_DOC; + yylval.str = copy_string(canon_doc_comment(yytext + 3)); + return TOK_POST_DOC; } -##[^#\n].* { - yylval.str = copy_string(yytext + 2); - return TOK_DOC; +##.* { + if ( yytext[2] != '#' ) + { + yylval.str = copy_string(canon_doc_comment(yytext + 2)); + return TOK_DOC; + } } -##[^#\n].* { - if ( generate_documentation ) +##.* { + if ( generate_documentation && (yytext[2] != '#') ) { if ( ! reST_doc_comments ) reST_doc_comments = new std::list(); - reST_doc_comments->push_back(yytext + 2); + reST_doc_comments->push_back(canon_doc_comment(yytext + 2)); } } From 8b79971e2170cee30d452b9717f2be65eb238a2c Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Wed, 23 Mar 2011 16:59:41 -0500 Subject: [PATCH 21/47] Fix auto-generated reST :doc: references to strip .bro file suffixes. It's allowed for a script to "@load example.bro", but Sphinx doesn't want that file extension for the purposes of generating cross-referencing links to other documentation. --- src/BroDoc.cc | 16 ++++++++++++++++ src/BroDoc.h | 3 ++- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/src/BroDoc.cc b/src/BroDoc.cc index 12b8cce284..9eb804432a 100644 --- a/src/BroDoc.cc +++ b/src/BroDoc.cc @@ -56,6 +56,22 @@ BroDoc::~BroDoc() FreeBroDocObjPtrList(redefs); } +void BroDoc::AddImport(const std::string& s) + { + size_t ext_pos = s.find_last_of('.'); + + if ( ext_pos == std::string::npos ) + imports.push_back(s); + else + { + if ( s.substr(ext_pos + 1) == "bro" ) + imports.push_back(s.substr(0, ext_pos)); + else + fprintf(stderr, "Warning: skipped documenting @load of file " + "without .bro extension: %s\n", s.c_str()); + } + } + void BroDoc::SetPacketFilter(const std::string& s) { packet_filter = s; diff --git a/src/BroDoc.h b/src/BroDoc.h index a28a07561e..3ed92d9432 100644 --- a/src/BroDoc.h +++ b/src/BroDoc.h @@ -52,10 +52,11 @@ public: /** * Schedules an import (@load) to be documented. + * If the script being loaded has a .bro suffix, it is internally stripped. * This should be called whenever the scanner sees an @load. * @param s The name of the imported script. */ - void AddImport(const std::string& s) { imports.push_back(s); } + void AddImport(const std::string& s); /** * Schedules a namespace (module) to be documented. From c2f0332b5f073d70939b5a736d9bbb20b077f750 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Thu, 24 Mar 2011 09:51:47 -0500 Subject: [PATCH 22/47] Removing lexical scanner recognition of "## Author:" comments. This functionality is better done manually by the script writer embedding reST into the script summary section (##! comments). This allows flexibility in choosing between different methods to convey the same information (e.g. ":Author: " or the ".. codeauthor:: " directive that Sphinx configurations can recognize). --- src/BroDoc.cc | 3 --- src/BroDoc.h | 8 -------- src/scan.l | 9 --------- 3 files changed, 20 deletions(-) diff --git a/src/BroDoc.cc b/src/BroDoc.cc index 9eb804432a..e6b01c3c3a 100644 --- a/src/BroDoc.cc +++ b/src/BroDoc.cc @@ -112,9 +112,6 @@ void BroDoc::WriteDocFile() const WriteSectionHeading("Summary", '-'); WriteStringList("%s\n", "%s\n\n", summary); - WriteToDoc(":Author(s): "); - WriteStringList("%s, ", "%s\n", authors); - WriteToDoc(":Imports: "); WriteStringList(":doc:`%s`, ", ":doc:`%s`\n", imports); diff --git a/src/BroDoc.h b/src/BroDoc.h index 3ed92d9432..de950083ea 100644 --- a/src/BroDoc.h +++ b/src/BroDoc.h @@ -83,13 +83,6 @@ public: */ void AddPortAnalysis(const std::string& analyzer, const std::string& ports); - /** - * Adds a script author. - * The scanner should call this when it sees "## Author: ..." - * @param s The name, email, etc. of the script author. - */ - void AddAuthor(const std::string& s) { authors.push_back(s); } - /** * Schedules documentation of a script option. An option is * defined as any variable in the script that is declared 'const' @@ -168,7 +161,6 @@ protected: std::string source_filename; std::string packet_filter; - std::list authors; std::list modules; std::list summary; std::list imports; diff --git a/src/scan.l b/src/scan.l index 53b6afd72a..5b82a92788 100644 --- a/src/scan.l +++ b/src/scan.l @@ -146,15 +146,6 @@ ESCSEQ (\\([^\n]|[0-7]+|x[[:xdigit:]]+)) } } -##{OWS}Author:.* { - if ( generate_documentation ) - current_reST_doc->AddAuthor( - skip_whitespace( // Skip whitespace after "Author:" - skip_whitespace(yytext + 2) // Skip whitespace after ## - + 7) // Skip "Author:" - ); - } - ##<.* { yylval.str = copy_string(canon_doc_comment(yytext + 3)); return TOK_POST_DOC; From 2e88c5100c069a5ef03bb6c2a198ad40a5f609e6 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Thu, 24 Mar 2011 12:32:32 -0500 Subject: [PATCH 23/47] Revise autodoc tracking of public vs private script interfaces A bro script's public interface is taken to mean any identifier declared in the global scope that optionally is exported from some namespace/module. Or more simply: ID::IsGlobal() --- src/BroDoc.cc | 13 ++++--------- src/BroDoc.h | 8 ++++---- src/BroDocObj.h | 8 +++++--- src/parse.y | 15 +++++++-------- 4 files changed, 20 insertions(+), 24 deletions(-) diff --git a/src/BroDoc.cc b/src/BroDoc.cc index e6b01c3c3a..2aa949e440 100644 --- a/src/BroDoc.cc +++ b/src/BroDoc.cc @@ -152,6 +152,7 @@ void BroDoc::WriteDocFile() const WriteBroDocObjList(redefs, true, "Redefinitions", '~'); WriteSectionHeading("Private Interface", '-'); + WriteBroDocObjList(options, false, "Options", '~'); WriteBroDocObjList(state_vars, false, "State Variables", '~'); WriteBroDocObjList(types, false, "Types", '~'); WriteBroDocObjList(events, false, "Events", '~'); @@ -177,7 +178,7 @@ void BroDoc::WriteStringList(const char* format, } void BroDoc::WriteBroDocObjList(const std::list& l, - bool exportCond, + bool wantPublic, const char* heading, char underline) const { @@ -185,18 +186,12 @@ void BroDoc::WriteBroDocObjList(const std::list& l, std::list::const_iterator it; for ( it = l.begin(); it != l.end(); ++it ) { - if ( exportCond ) - { - // write out only those in an export section + if ( wantPublic ) if ( (*it)->IsPublicAPI() ) (*it)->WriteReST(reST_file); - } else - { - // write out only those that have comments and are not exported - if ( !(*it)->IsPublicAPI() && (*it)->HasDocumentation() ) + if ( ! (*it)->IsPublicAPI() ) (*it)->WriteReST(reST_file); - } } } diff --git a/src/BroDoc.h b/src/BroDoc.h index de950083ea..7cee9faf1a 100644 --- a/src/BroDoc.h +++ b/src/BroDoc.h @@ -198,15 +198,15 @@ protected: /** * Writes out a list of BroDocObj objects to the reST document * @param l A list of BroDocObj pointers - * @param exportCond If true, filter out objects that are not in an - * export section. If false, filter out those that are in - * an export section. + * @param wantPublic If true, filter out objects that are not declared + * in the global scope. If false, filter out those that are in + * the global scope. * @param heading The title of the section to create in the reST doc. * @param underline The character to use to underline the reST * section heading. */ void WriteBroDocObjList(const std::list& l, - bool exportCond, + bool wantPublic, const char* heading, char underline) const; diff --git a/src/BroDocObj.h b/src/BroDocObj.h index e3fd30518b..601d5f148b 100644 --- a/src/BroDocObj.h +++ b/src/BroDocObj.h @@ -37,11 +37,13 @@ public: void WriteReST(FILE* file) const; /** - * Check whether this documentation is part of the public API - * (The declaration was while in an export section). + * Check whether this documentation is part of the public API. In + * other words, this means that the identifier is declared as part of + * the global scope (has GLOBAL namespace or is exported from another + * namespace). * @return true if the ID was declared in an export section, else false */ - bool IsPublicAPI() const { return broID->IsExport(); } + bool IsPublicAPI() const { return broID->IsGlobal(); } /** * Return whether this object has documentation (## comments) diff --git a/src/parse.y b/src/parse.y index 1d2d4f277f..d561fa8e06 100644 --- a/src/parse.y +++ b/src/parse.y @@ -157,13 +157,12 @@ static void add_enum_comment (std::list* comments) cur_enum_type_doc->AddComment(current_module, cur_enum_elem_id, comments); } -static ID* create_dummy_id (const char* name, BroType* type) +static ID* create_dummy_id (const ID* id, BroType* type) { - // normally, install_ID() figures out the right IDScope - // but it doesn't matter for the dummy ID so use SCOPE_GLOBAL - ID* fake_id = new ID(copy_string(name), SCOPE_GLOBAL, is_export); + ID* fake_id = new ID(copy_string(id->Name()), (IDScope) id->Scope(), + is_export); fake_id->SetType(type); - type->SetTypeID(copy_string(name)); + type->SetTypeID(copy_string(id->Name())); fake_id->MakeType(); return fake_id; } @@ -1009,7 +1008,7 @@ decl: do_doc_token_stop(); if ( generate_documentation ) { - ID* fake_id = create_dummy_id($3->Name(), cur_enum_type_doc); + ID* fake_id = create_dummy_id($3, cur_enum_type_doc); cur_enum_type_doc = 0; BroDocObj* o = new BroDocObj(fake_id, reST_doc_comments, true); o->SetRole(true); @@ -1028,7 +1027,7 @@ decl: TypeTag t = $2->AsType()->Tag(); if ( t == TYPE_ENUM && cur_enum_type_doc ) { - ID* fake = create_dummy_id($2->Name(), cur_enum_type_doc); + ID* fake = create_dummy_id($2, cur_enum_type_doc); cur_enum_type_doc = 0; current_reST_doc->AddType( new BroDocObj(fake, reST_doc_comments, true)); @@ -1036,7 +1035,7 @@ decl: else if ( t == TYPE_RECORD && fake_type_decl_list ) { BroType* fake_record = new RecordType(fake_type_decl_list); - ID* fake = create_dummy_id($2->Name(), fake_record); + ID* fake = create_dummy_id($2, fake_record); fake_type_decl_list = 0; current_reST_doc->AddType( new BroDocObj(fake, reST_doc_comments, true)); From 2490878656eb46ebb162ee7bc351cf711913477f Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Thu, 24 Mar 2011 14:04:30 -0500 Subject: [PATCH 24/47] Another revision for autodoc tracking of public vs private interfaces A script's public API wasn't simply definable as identifiers for which ID::IsGlobal() is true, e.g. an unexported identifier with SCOPE_MODULE will still pass that test and (incorrectly) be considered public API. Also, generated reST now omits empty interface sections. --- src/BroDoc.cc | 30 ++++++++++++++++++++---------- src/BroDoc.h | 3 +++ src/BroDocObj.cc | 6 ++++++ src/BroDocObj.h | 4 ++-- 4 files changed, 31 insertions(+), 12 deletions(-) diff --git a/src/BroDoc.cc b/src/BroDoc.cc index 2aa949e440..d0815abe4a 100644 --- a/src/BroDoc.cc +++ b/src/BroDoc.cc @@ -112,8 +112,11 @@ void BroDoc::WriteDocFile() const WriteSectionHeading("Summary", '-'); WriteStringList("%s\n", "%s\n\n", summary); - WriteToDoc(":Imports: "); - WriteStringList(":doc:`%s`, ", ":doc:`%s`\n", imports); + if ( ! imports.empty() ) + { + WriteToDoc(":Imports: "); + WriteStringList(":doc:`%s`, ", ":doc:`%s`\n", imports); + } WriteToDoc("\n"); @@ -182,16 +185,23 @@ void BroDoc::WriteBroDocObjList(const std::list& l, const char* heading, char underline) const { - WriteSectionHeading(heading, underline); + if ( l.empty() ) return; + std::list::const_iterator it; - for ( it = l.begin(); it != l.end(); ++it ) + bool (*f_ptr)(const BroDocObj* o) = 0; + + if ( wantPublic ) + f_ptr = IsPublicAPI; + else + f_ptr = IsPrivateAPI; + + it = std::find_if(l.begin(), l.end(), f_ptr); + if ( it == l.end() ) return; + WriteSectionHeading(heading, underline); + while ( it != l.end() ) { - if ( wantPublic ) - if ( (*it)->IsPublicAPI() ) - (*it)->WriteReST(reST_file); - else - if ( ! (*it)->IsPublicAPI() ) - (*it)->WriteReST(reST_file); + (*it)->WriteReST(reST_file); + it = find_if(++it, l.end(), f_ptr); } } diff --git a/src/BroDoc.h b/src/BroDoc.h index 7cee9faf1a..04d8206eaa 100644 --- a/src/BroDoc.h +++ b/src/BroDoc.h @@ -242,6 +242,9 @@ private: * @param a reference to a list of BroDocObj pointers */ void FreeBroDocObjPtrList(std::list& l); + + static bool IsPublicAPI(const BroDocObj* o) { return o->IsPublicAPI(); } + static bool IsPrivateAPI(const BroDocObj* o) { return ! o->IsPublicAPI(); } }; #endif diff --git a/src/BroDocObj.cc b/src/BroDocObj.cc index 326550e545..c04dba9a89 100644 --- a/src/BroDocObj.cc +++ b/src/BroDocObj.cc @@ -44,3 +44,9 @@ void BroDocObj::WriteReST(FILE* file) const fprintf(file, "\n"); } + +bool BroDocObj::IsPublicAPI() const + { + return (broID->Scope() == SCOPE_GLOBAL) || + (broID->Scope() == SCOPE_MODULE && broID->IsExport()); + } diff --git a/src/BroDocObj.h b/src/BroDocObj.h index 601d5f148b..fc33372043 100644 --- a/src/BroDocObj.h +++ b/src/BroDocObj.h @@ -41,9 +41,9 @@ public: * other words, this means that the identifier is declared as part of * the global scope (has GLOBAL namespace or is exported from another * namespace). - * @return true if the ID was declared in an export section, else false + * @return true if the identifier is part of the script's public API */ - bool IsPublicAPI() const { return broID->IsGlobal(); } + bool IsPublicAPI() const; /** * Return whether this object has documentation (## comments) From 60a7dc6f55d6881d4a2d776feb6d2532deace735 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Thu, 24 Mar 2011 16:56:25 -0500 Subject: [PATCH 25/47] Autodoc framework now tracks script constants Also, it's starting to attempt to describe more complex types of initial values. --- src/Attr.cc | 11 ++++++++--- src/BroDoc.cc | 29 +++++++++++++++-------------- src/BroDoc.h | 29 +++++++++++++++++++++++++---- src/ID.cc | 22 ++++++++++++++++++---- src/Val.cc | 11 ++++++++--- src/parse.y | 3 +++ 6 files changed, 77 insertions(+), 28 deletions(-) diff --git a/src/Attr.cc b/src/Attr.cc index f7f7f0576a..6e6f6c4ca6 100644 --- a/src/Attr.cc +++ b/src/Attr.cc @@ -59,9 +59,14 @@ void Attr::DescribeReST(ODesc* d) const d->SP(); d->Add("="); d->SP(); - d->Add("``"); - expr->Describe(d); - d-> Add("``"); + if ( expr->Type()->Tag() == TYPE_FUNC ) + d->Add(":bro:type:`func`"); + else + { + d->Add("``"); + expr->Describe(d); + d-> Add("``"); + } } } diff --git a/src/BroDoc.cc b/src/BroDoc.cc index d0815abe4a..23ed3ee189 100644 --- a/src/BroDoc.cc +++ b/src/BroDoc.cc @@ -48,6 +48,7 @@ BroDoc::~BroDoc() if ( fclose( reST_file ) ) fprintf(stderr, "Failed to close %s", reST_filename.c_str()); FreeBroDocObjPtrList(options); + FreeBroDocObjPtrList(constants); FreeBroDocObjPtrList(state_vars); FreeBroDocObjPtrList(types); FreeBroDocObjPtrList(notices); @@ -146,21 +147,21 @@ void BroDoc::WriteDocFile() const WriteToDoc("%s\n", packet_filter.c_str()); } - WriteSectionHeading("Public Interface", '-'); - WriteBroDocObjList(options, true, "Options", '~'); - WriteBroDocObjList(state_vars, true, "State Variables", '~'); - WriteBroDocObjList(types, true, "Types", '~'); - WriteBroDocObjList(events, true, "Events", '~'); - WriteBroDocObjList(functions, true, "Functions", '~'); - WriteBroDocObjList(redefs, true, "Redefinitions", '~'); + WriteInterface("Public Interface", '-', '~', true); + WriteInterface("Private Interface", '-', '~', false); + } - WriteSectionHeading("Private Interface", '-'); - WriteBroDocObjList(options, false, "Options", '~'); - WriteBroDocObjList(state_vars, false, "State Variables", '~'); - WriteBroDocObjList(types, false, "Types", '~'); - WriteBroDocObjList(events, false, "Events", '~'); - WriteBroDocObjList(functions, false, "Functions", '~'); - WriteBroDocObjList(redefs, false, "Redefinitions", '~'); +void BroDoc::WriteInterface(const char* heading, char underline, + char sub, bool isPublic) const + { + WriteSectionHeading(heading, underline); + WriteBroDocObjList(options, isPublic, "Options", sub); + WriteBroDocObjList(constants, isPublic, "Constants", sub); + WriteBroDocObjList(state_vars, isPublic, "State Variables", sub); + WriteBroDocObjList(types, isPublic, "Types", sub); + WriteBroDocObjList(events, isPublic, "Events", sub); + WriteBroDocObjList(functions, isPublic, "Functions", sub); + WriteBroDocObjList(redefs, isPublic, "Redefinitions", sub); } void BroDoc::WriteStringList(const char* format, diff --git a/src/BroDoc.h b/src/BroDoc.h index 04d8206eaa..b1969d0da9 100644 --- a/src/BroDoc.h +++ b/src/BroDoc.h @@ -35,10 +35,7 @@ public: * BroDoc's default implementation of this function will care * about whether declarations made in the Bro script are part of * the public versus private interface (whether things are declared in - * the export section). Things in a script's export section make it - * into the reST output regardless of whether they have ## comments - * but things outside the export section are only output into the reST - * if they have ## comments. + * the export section). */ virtual void WriteDocFile() const; @@ -93,6 +90,16 @@ public: */ void AddOption(const BroDocObj* o) { options.push_back(o); } + /** + * Schedules documentation of a script constant. An option is + * defined as any variable in the script that is declared 'const' + * and does *not* have the '&redef' attribute. + * @param o A pointer to a BroDocObj which contains the internal + * Bro language representation of the script constant and + * also any associated comments about it. + */ + void AddConstant(const BroDocObj* o) { constants.push_back(o); } + /** * Schedules documentation of a script state variable. A state variable * is defined as any variable in the script that is declared 'global' @@ -167,6 +174,7 @@ protected: std::list port_analysis; std::list options; + std::list constants; std::list state_vars; std::list types; std::list notices; @@ -235,6 +243,19 @@ protected: * within the reST document */ void WriteSectionHeading(const char* heading, char underline) const; + + /** + * Writes out the reST for either the script's public or private interface + * @param heading The title of the interfaces section heading + * @param underline The underline character to use for the interface + * section + * @param subunderline The underline character to use for interface + * sub-sections + * @param isPublic Whether to write out the public or private script + * interface + */ + void WriteInterface(const char* heading, char underline, char subunderline, + bool isPublic) const; private: /** diff --git a/src/ID.cc b/src/ID.cc index c4cf5a58c1..b3fcf8f5c3 100644 --- a/src/ID.cc +++ b/src/ID.cc @@ -651,12 +651,26 @@ void ID::DescribeReST(ODesc* d, bool is_role) const } if ( val && type && - type->InternalType() != TYPE_INTERNAL_OTHER && + type->Tag() != TYPE_FUNC && type->InternalType() != TYPE_INTERNAL_VOID ) { - d->Add(":Init: "); - val->DescribeReST(d); - d->NL(); + if ( type->InternalType() == TYPE_INTERNAL_OTHER ) + { + d->Add(":Init:"); + d->NL(); + d->NL(); + d->Add("::"); + d->NL(); + d->PushIndent(); + val->DescribeReST(d); + d->PopIndent(); + } + else + { + d->Add(":Init: "); + val->DescribeReST(d); + d->NL(); + } } } diff --git a/src/Val.cc b/src/Val.cc index 05ca0adc65..012ca92773 100644 --- a/src/Val.cc +++ b/src/Val.cc @@ -576,9 +576,14 @@ void Val::Describe(ODesc* d) const void Val::DescribeReST(ODesc* d) const { - d->Add("``"); - ValDescribeReST(d); - d->Add("``"); + if ( type->InternalType() == TYPE_INTERNAL_OTHER ) + Describe(d); + else + { + d->Add("``"); + ValDescribeReST(d); + d->Add("``"); + } } void Val::ValDescribe(ODesc* d) const diff --git a/src/parse.y b/src/parse.y index d561fa8e06..1e32bcdbef 100644 --- a/src/parse.y +++ b/src/parse.y @@ -991,6 +991,9 @@ decl: if ( $2->FindAttr(ATTR_REDEF) ) current_reST_doc->AddOption( new BroDocObj($2, reST_doc_comments)); + else + current_reST_doc->AddConstant( + new BroDocObj($2, reST_doc_comments)); } | TOK_REDEF global_id opt_type init_class opt_init opt_attr ';' From a9810d21a93846e13b03727b7768121454b85385 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Tue, 29 Mar 2011 12:10:08 -0500 Subject: [PATCH 26/47] Generated docs now have xref links for attribute values that are enums. --- src/Attr.cc | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/Attr.cc b/src/Attr.cc index 6e6f6c4ca6..5828c5888e 100644 --- a/src/Attr.cc +++ b/src/Attr.cc @@ -61,6 +61,12 @@ void Attr::DescribeReST(ODesc* d) const d->SP(); if ( expr->Type()->Tag() == TYPE_FUNC ) d->Add(":bro:type:`func`"); + else if ( expr->Type()->Tag() == TYPE_ENUM ) + { + d->Add(":bro:enum:`"); + expr->Describe(d); + d->Add("`"); + } else { d->Add("``"); From 94ac3f3c23e9536482739a57aa94f9f626dc1368 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Tue, 29 Mar 2011 14:19:33 -0500 Subject: [PATCH 27/47] Revise the way "port analysis" and "packet filter" documentation is generated. They now appear at the bottom of generated docs. Also, a "more info" link is added which can point to an arbitrary location in any reST document processed by Sphinx. --- src/BroDoc.cc | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/BroDoc.cc b/src/BroDoc.cc index 23ed3ee189..cc209c4524 100644 --- a/src/BroDoc.cc +++ b/src/BroDoc.cc @@ -131,24 +131,23 @@ void BroDoc::WriteDocFile() const if ( ! notices.empty() ) WriteBroDocObjList(notices, "Notices", '-'); + WriteInterface("Public Interface", '-', '~', true); + WriteInterface("Private Interface", '-', '~', false); + if ( ! port_analysis.empty() ) { WriteSectionHeading("Port Analysis", '-'); + WriteToDoc(":ref:`More Information `\n\n"); WriteStringList("%s", port_analysis); } if ( ! packet_filter.empty() ) { WriteSectionHeading("Packet Filter", '-'); - WriteToDoc(".. note:: Filters are only relevant when dynamic protocol " - "detection (DPD) is explicitly turned off (Bro release 1.6 " - "enabled DPD by default).\n\n"); + WriteToDoc(":ref:`More Information `\n\n"); WriteToDoc("Filters added::\n\n"); WriteToDoc("%s\n", packet_filter.c_str()); } - - WriteInterface("Public Interface", '-', '~', true); - WriteInterface("Private Interface", '-', '~', false); } void BroDoc::WriteInterface(const char* heading, char underline, From 090ce2d03c21b5eedf76aff79e86715f552d9a6f Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Tue, 29 Mar 2011 16:54:16 -0500 Subject: [PATCH 28/47] Fix generated script docs displaying functions twice. A function prototype can be declared separately from where it's defined; the doc framework should now recognize them as the same and combine reST documentation associated with either case if both are present. --- src/BroDoc.cc | 48 ++++++++++++++++++++++++++++++++++++++++++------ src/BroDoc.h | 47 +++++++++++++++++++++++++++++++++++------------ src/BroDocObj.cc | 16 +++++++++++++++- src/BroDocObj.h | 12 ++++++++++++ 4 files changed, 104 insertions(+), 19 deletions(-) diff --git a/src/BroDoc.cc b/src/BroDoc.cc index cc209c4524..4b676de8f5 100644 --- a/src/BroDoc.cc +++ b/src/BroDoc.cc @@ -180,7 +180,7 @@ void BroDoc::WriteStringList(const char* format, WriteToDoc(last_format, last->c_str()); } -void BroDoc::WriteBroDocObjList(const std::list& l, +void BroDoc::WriteBroDocObjList(const BroDocObjList& l, bool wantPublic, const char* heading, char underline) const @@ -205,16 +205,37 @@ void BroDoc::WriteBroDocObjList(const std::list& l, } } -void BroDoc::WriteBroDocObjList(const std::list& l, +void BroDoc::WriteBroDocObjList(const BroDocObjMap& m, + bool wantPublic, + const char* heading, + char underline) const + { + BroDocObjMap::const_iterator it; + BroDocObjList l; + for ( it = m.begin(); it != m.end(); ++it ) l.push_back(it->second); + WriteBroDocObjList(l, wantPublic, heading, underline); + } + +void BroDoc::WriteBroDocObjList(const BroDocObjList& l, const char* heading, char underline) const { WriteSectionHeading(heading, underline); - std::list::const_iterator it; + BroDocObjList::const_iterator it; for ( it = l.begin(); it != l.end(); ++it ) (*it)->WriteReST(reST_file); } +void BroDoc::WriteBroDocObjList(const BroDocObjMap& m, + const char* heading, + char underline) const + { + BroDocObjMap::const_iterator it; + BroDocObjList l; + for ( it = m.begin(); it != m.end(); ++it ) l.push_back(it->second); + WriteBroDocObjList(l, heading, underline); + } + void BroDoc::WriteToDoc(const char* format, ...) const { va_list argp; @@ -232,10 +253,25 @@ void BroDoc::WriteSectionHeading(const char* heading, char underline) const WriteToDoc("\n"); } -void BroDoc::FreeBroDocObjPtrList(std::list& l) +void BroDoc::FreeBroDocObjPtrList(BroDocObjList& l) { - std::list::iterator it; - for ( it = l.begin(); it != l.end(); ++it ) + for ( BroDocObjList::const_iterator it = l.begin(); it != l.end(); ++it ) delete *it; l.clear(); } + +void BroDoc::FreeBroDocObjPtrList(BroDocObjMap& l) + { + for ( BroDocObjMap::const_iterator it = l.begin(); it != l.end(); ++it ) + delete it->second; + l.clear(); + } + +void BroDoc::AddFunction(BroDocObj* o) + { + BroDocObjMap::const_iterator it = functions.find(o->Name()); + if ( it == functions.end() ) + functions[o->Name()] = o; + else + functions[o->Name()]->Combine(o); + } diff --git a/src/BroDoc.h b/src/BroDoc.h index b1969d0da9..08477d51c2 100644 --- a/src/BroDoc.h +++ b/src/BroDoc.h @@ -139,7 +139,7 @@ public: * Bro language representation of the script function and * also any associated comments about it. */ - void AddFunction(const BroDocObj* o) { functions.push_back(o); } + void AddFunction(BroDocObj* o); /** * Schedules documentation of a redef done by the script @@ -173,14 +173,17 @@ protected: std::list imports; std::list port_analysis; - std::list options; - std::list constants; - std::list state_vars; - std::list types; - std::list notices; - std::list events; - std::list functions; - std::list redefs; + typedef std::list BroDocObjList; + typedef std::map BroDocObjMap; + + BroDocObjList options; + BroDocObjList constants; + BroDocObjList state_vars; + BroDocObjList types; + BroDocObjList notices; + BroDocObjList events; + BroDocObjMap functions; + BroDocObjList redefs; /** * Writes out a list of strings to the reST document. @@ -213,7 +216,17 @@ protected: * @param underline The character to use to underline the reST * section heading. */ - void WriteBroDocObjList(const std::list& l, + void WriteBroDocObjList(const BroDocObjList& l, + bool wantPublic, + const char* heading, + char underline) const; + + /** + * Wraps the BroDocObjMap into a BroDocObjList and the writes that list + * to the reST document + * @see WriteBroDocObjList(const BroDocObjList&, bool, const char*, char) + */ + void WriteBroDocObjList(const BroDocObjMap& m, bool wantPublic, const char* heading, char underline) const; @@ -225,7 +238,16 @@ protected: * @param underline The character to use to underline the reST * section heading. */ - void WriteBroDocObjList(const std::list& l, + void WriteBroDocObjList(const BroDocObjList& l, + const char* heading, + char underline) const; + + /** + * Wraps the BroDocObjMap into a BroDocObjList and the writes that list + * to the reST document + * @see WriteBroDocObjList(const BroDocObjList&, const char*, char) + */ + void WriteBroDocObjList(const BroDocObjMap& m, const char* heading, char underline) const; @@ -262,7 +284,8 @@ private: * Frees memory allocated to BroDocObj's objects in a given list. * @param a reference to a list of BroDocObj pointers */ - void FreeBroDocObjPtrList(std::list& l); + void FreeBroDocObjPtrList(BroDocObjList& l); + void FreeBroDocObjPtrList(BroDocObjMap& l); static bool IsPublicAPI(const BroDocObj* o) { return o->IsPublicAPI(); } static bool IsPrivateAPI(const BroDocObj* o) { return ! o->IsPublicAPI(); } diff --git a/src/BroDocObj.cc b/src/BroDocObj.cc index c04dba9a89..85ad2b85cd 100644 --- a/src/BroDocObj.cc +++ b/src/BroDocObj.cc @@ -16,7 +16,7 @@ BroDocObj::BroDocObj(const ID* id, std::list*& reST, BroDocObj::~BroDocObj() { - delete reST_doc_strings; + if ( reST_doc_strings ) delete reST_doc_strings; if ( is_fake_id ) delete broID; } @@ -50,3 +50,17 @@ bool BroDocObj::IsPublicAPI() const return (broID->Scope() == SCOPE_GLOBAL) || (broID->Scope() == SCOPE_MODULE && broID->IsExport()); } + +void BroDocObj::Combine(const BroDocObj* o) + { + if ( o->reST_doc_strings ) + { + if ( ! reST_doc_strings ) + reST_doc_strings = new std::list(); + + reST_doc_strings->splice(reST_doc_strings->end(), + *(o->reST_doc_strings)); + } + + delete o; + } diff --git a/src/BroDocObj.h b/src/BroDocObj.h index fc33372043..0660d510f6 100644 --- a/src/BroDocObj.h +++ b/src/BroDocObj.h @@ -66,6 +66,18 @@ public: */ void SetRole(bool b) { use_role = b; } + /** + * Append any reST documentation strings in a given BroDocObj to this + * object's list and then delete the given BroDocObj + * @param o a pointer to a BroDocObj to subsume + */ + void Combine(const BroDocObj* o); + + /** + * @return the name of the wrapped identifier + */ + const char* Name() const { return broID->Name(); } + protected: std::list* reST_doc_strings; const ID* broID; From cf45ae19e168e9af4f18dcbf1b186b7bc50baaa3 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Wed, 30 Mar 2011 10:24:44 -0500 Subject: [PATCH 29/47] Generated script docs can now link to the original source. The original bro source file needs to be copied to the same directory that Sphinx outputs the rendered html. --- src/BroDoc.cc | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/BroDoc.cc b/src/BroDoc.cc index 4b676de8f5..6a91f59153 100644 --- a/src/BroDoc.cc +++ b/src/BroDoc.cc @@ -105,10 +105,9 @@ void BroDoc::WriteDocFile() const { WriteToDoc(".. Automatically generated. Do not edit.\n\n"); - WriteToDoc("%s\n", source_filename.c_str()); - for ( size_t i = 0; i < source_filename.length(); ++i ) - WriteToDoc("="); - WriteToDoc("\n\n"); + WriteSectionHeading(source_filename.c_str(), '='); + + WriteToDoc("\n`Original Source File <%s>`_\n\n", source_filename.c_str()); WriteSectionHeading("Summary", '-'); WriteStringList("%s\n", "%s\n\n", summary); From 2d17ca0942ace2b03acbaddd8ac41d28d84fde29 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Fri, 1 Apr 2011 12:30:22 -0500 Subject: [PATCH 30/47] Generated script docs now have a new summary section. It's a table listing all the identifiers in the script's public interface and an optional, brief (one-sentence) description of each. --- src/BroDoc.cc | 120 ++++++++++++++++++++++++++++++++--------------- src/BroDoc.h | 59 ++++++++++++++++++----- src/BroDocObj.cc | 81 ++++++++++++++++++++++++++++++++ src/BroDocObj.h | 34 ++++++++++++-- src/ID.cc | 27 +++++++++++ src/ID.h | 1 + 6 files changed, 270 insertions(+), 52 deletions(-) diff --git a/src/BroDoc.cc b/src/BroDoc.cc index 6a91f59153..722a8861cd 100644 --- a/src/BroDoc.cc +++ b/src/BroDoc.cc @@ -47,14 +47,7 @@ BroDoc::~BroDoc() if ( reST_file ) if ( fclose( reST_file ) ) fprintf(stderr, "Failed to close %s", reST_filename.c_str()); - FreeBroDocObjPtrList(options); - FreeBroDocObjPtrList(constants); - FreeBroDocObjPtrList(state_vars); - FreeBroDocObjPtrList(types); - FreeBroDocObjPtrList(notices); - FreeBroDocObjPtrList(events); - FreeBroDocObjPtrList(functions); - FreeBroDocObjPtrList(redefs); + FreeBroDocObjPtrList(all); } void BroDoc::AddImport(const std::string& s) @@ -109,7 +102,7 @@ void BroDoc::WriteDocFile() const WriteToDoc("\n`Original Source File <%s>`_\n\n", source_filename.c_str()); - WriteSectionHeading("Summary", '-'); + WriteSectionHeading("Overview", '-'); WriteStringList("%s\n", "%s\n\n", summary); if ( ! imports.empty() ) @@ -120,18 +113,20 @@ void BroDoc::WriteDocFile() const WriteToDoc("\n"); + WriteInterface("Summary", '~', '#', true, true); + if ( ! modules.empty() ) { - WriteSectionHeading("Namespaces", '-'); + WriteSectionHeading("Namespaces", '~'); WriteStringList(".. bro:namespace:: %s\n", modules); WriteToDoc("\n"); } if ( ! notices.empty() ) - WriteBroDocObjList(notices, "Notices", '-'); + WriteBroDocObjList(notices, "Notices", '~'); - WriteInterface("Public Interface", '-', '~', true); - WriteInterface("Private Interface", '-', '~', false); + WriteInterface("Public Interface", '-', '~', true, false); + WriteInterface("Private Interface", '-', '~', false, false); if ( ! port_analysis.empty() ) { @@ -150,16 +145,16 @@ void BroDoc::WriteDocFile() const } void BroDoc::WriteInterface(const char* heading, char underline, - char sub, bool isPublic) const + char sub, bool isPublic, bool isShort) const { WriteSectionHeading(heading, underline); - WriteBroDocObjList(options, isPublic, "Options", sub); - WriteBroDocObjList(constants, isPublic, "Constants", sub); - WriteBroDocObjList(state_vars, isPublic, "State Variables", sub); - WriteBroDocObjList(types, isPublic, "Types", sub); - WriteBroDocObjList(events, isPublic, "Events", sub); - WriteBroDocObjList(functions, isPublic, "Functions", sub); - WriteBroDocObjList(redefs, isPublic, "Redefinitions", sub); + WriteBroDocObjList(options, isPublic, "Options", sub, isShort); + WriteBroDocObjList(constants, isPublic, "Constants", sub, isShort); + WriteBroDocObjList(state_vars, isPublic, "State Variables", sub, isShort); + WriteBroDocObjList(types, isPublic, "Types", sub, isShort); + WriteBroDocObjList(events, isPublic, "Events", sub, isShort); + WriteBroDocObjList(functions, isPublic, "Functions", sub, isShort); + WriteBroDocObjList(redefs, isPublic, "Redefinitions", sub, isShort); } void BroDoc::WriteStringList(const char* format, @@ -179,10 +174,48 @@ void BroDoc::WriteStringList(const char* format, WriteToDoc(last_format, last->c_str()); } +void BroDoc::WriteBroDocObjTable(const BroDocObjList& l) const + { + int max_id_col = 0; + int max_com_col = 0; + BroDocObjList::const_iterator it; + + for ( it = l.begin(); it != l.end(); ++it ) + { + int c = (*it)->ColumnSize(); + if ( c > max_id_col ) max_id_col = c; + c = (*it)->LongestShortDescLen(); + if ( c > max_com_col ) max_com_col = c; + } + + // start table + WriteRepeatedChar('=', max_id_col); + WriteToDoc(" "); + if ( max_com_col == 0 ) WriteToDoc("="); + else WriteRepeatedChar('=', max_com_col); + WriteToDoc("\n"); + + for ( it = l.begin(); it != l.end(); ++it ) + { + if ( it != l.begin() ) + WriteToDoc("\n\n"); + (*it)->WriteReSTCompact(reST_file, max_id_col); + } + + // end table + WriteToDoc("\n"); + WriteRepeatedChar('=', max_id_col); + WriteToDoc(" "); + if ( max_com_col == 0 ) WriteToDoc("="); + else WriteRepeatedChar('=', max_com_col); + WriteToDoc("\n\n"); + } + void BroDoc::WriteBroDocObjList(const BroDocObjList& l, bool wantPublic, const char* heading, - char underline) const + char underline, + bool isShort) const { if ( l.empty() ) return; @@ -196,23 +229,33 @@ void BroDoc::WriteBroDocObjList(const BroDocObjList& l, it = std::find_if(l.begin(), l.end(), f_ptr); if ( it == l.end() ) return; + WriteSectionHeading(heading, underline); + + BroDocObjList filtered_list; + while ( it != l.end() ) { - (*it)->WriteReST(reST_file); + filtered_list.push_back(*it); it = find_if(++it, l.end(), f_ptr); } + + if ( isShort ) + WriteBroDocObjTable(filtered_list); + else + WriteBroDocObjList(filtered_list); } void BroDoc::WriteBroDocObjList(const BroDocObjMap& m, bool wantPublic, const char* heading, - char underline) const + char underline, + bool isShort) const { BroDocObjMap::const_iterator it; BroDocObjList l; for ( it = m.begin(); it != m.end(); ++it ) l.push_back(it->second); - WriteBroDocObjList(l, wantPublic, heading, underline); + WriteBroDocObjList(l, wantPublic, heading, underline, isShort); } void BroDoc::WriteBroDocObjList(const BroDocObjList& l, @@ -220,8 +263,12 @@ void BroDoc::WriteBroDocObjList(const BroDocObjList& l, char underline) const { WriteSectionHeading(heading, underline); - BroDocObjList::const_iterator it; - for ( it = l.begin(); it != l.end(); ++it ) + WriteBroDocObjList(l); + } + +void BroDoc::WriteBroDocObjList(const BroDocObjList& l) const + { + for ( BroDocObjList::const_iterator it = l.begin(); it != l.end(); ++it ) (*it)->WriteReST(reST_file); } @@ -246,12 +293,15 @@ void BroDoc::WriteToDoc(const char* format, ...) const void BroDoc::WriteSectionHeading(const char* heading, char underline) const { WriteToDoc("%s\n", heading); - size_t len = strlen(heading); - for ( size_t i = 0; i < len; ++i ) - WriteToDoc("%c", underline); + WriteRepeatedChar(underline, strlen(heading)); WriteToDoc("\n"); } +void BroDoc::WriteRepeatedChar(char c, size_t n) const + { + for ( size_t i = 0; i < n; ++i ) WriteToDoc("%c", c); + } + void BroDoc::FreeBroDocObjPtrList(BroDocObjList& l) { for ( BroDocObjList::const_iterator it = l.begin(); it != l.end(); ++it ) @@ -259,18 +309,14 @@ void BroDoc::FreeBroDocObjPtrList(BroDocObjList& l) l.clear(); } -void BroDoc::FreeBroDocObjPtrList(BroDocObjMap& l) - { - for ( BroDocObjMap::const_iterator it = l.begin(); it != l.end(); ++it ) - delete it->second; - l.clear(); - } - void BroDoc::AddFunction(BroDocObj* o) { BroDocObjMap::const_iterator it = functions.find(o->Name()); if ( it == functions.end() ) + { functions[o->Name()] = o; + all.push_back(o); + } else functions[o->Name()]->Combine(o); } diff --git a/src/BroDoc.h b/src/BroDoc.h index 08477d51c2..1096a030f3 100644 --- a/src/BroDoc.h +++ b/src/BroDoc.h @@ -88,7 +88,8 @@ public: * Bro language representation of the script option and * also any associated comments about it. */ - void AddOption(const BroDocObj* o) { options.push_back(o); } + void AddOption(const BroDocObj* o) { options.push_back(o); + all.push_back(o); } /** * Schedules documentation of a script constant. An option is @@ -98,7 +99,8 @@ public: * Bro language representation of the script constant and * also any associated comments about it. */ - void AddConstant(const BroDocObj* o) { constants.push_back(o); } + void AddConstant(const BroDocObj* o) { constants.push_back(o); + all.push_back(o); } /** * Schedules documentation of a script state variable. A state variable @@ -107,7 +109,8 @@ public: * Bro language representation of the script state variable * and also any associated comments about it. */ - void AddStateVar(const BroDocObj* o) { state_vars.push_back(o); } + void AddStateVar(const BroDocObj* o) { state_vars.push_back(o); + all.push_back(o); } /** * Schedules documentation of a type declared by the script. @@ -115,7 +118,8 @@ public: * Bro language representation of the script option and * also any associated comments about it. */ - void AddType(const BroDocObj* o) { types.push_back(o); } + void AddType(const BroDocObj* o) { types.push_back(o); + all.push_back(o); } /** * Schedules documentation of a Notice (enum redef) declared by script @@ -123,7 +127,8 @@ public: * Bro language representation of the Notice and also * any associated comments about it. */ - void AddNotice(const BroDocObj* o) { notices.push_back(o); } + void AddNotice(const BroDocObj* o) { notices.push_back(o); + all.push_back(o); } /** * Schedules documentation of an event declared by the script. @@ -131,7 +136,8 @@ public: * Bro language representation of the script event and * also any associated comments about it. */ - void AddEvent(const BroDocObj* o) { events.push_back(o); } + void AddEvent(const BroDocObj* o) { events.push_back(o); + all.push_back(o); } /** * Schedules documentation of a function declared by the script. @@ -147,7 +153,8 @@ public: * Bro language representation of the script identifier * that was redefined and also any associated comments. */ - void AddRedef(const BroDocObj* o) { redefs.push_back(o); } + void AddRedef(const BroDocObj* o) { redefs.push_back(o); + all.push_back(o); } /** * Gets the name of the Bro script source file for which reST @@ -185,6 +192,8 @@ protected: BroDocObjMap functions; BroDocObjList redefs; + BroDocObjList all; + /** * Writes out a list of strings to the reST document. * If the list is empty, prints a newline character. @@ -206,6 +215,13 @@ protected: const std::list& l) const { WriteStringList(format, format, l); } + + /** + * Writes out a table of BroDocObj's to the reST document + * @param l A list of BroDocObj pointers + */ + void WriteBroDocObjTable(const BroDocObjList& l) const; + /** * Writes out a list of BroDocObj objects to the reST document * @param l A list of BroDocObj pointers @@ -215,21 +231,26 @@ protected: * @param heading The title of the section to create in the reST doc. * @param underline The character to use to underline the reST * section heading. + * @param isShort Whether to write the full documentation or a "short" + * version (a single sentence) */ void WriteBroDocObjList(const BroDocObjList& l, bool wantPublic, const char* heading, - char underline) const; + char underline, + bool isShort) const; /** * Wraps the BroDocObjMap into a BroDocObjList and the writes that list * to the reST document - * @see WriteBroDocObjList(const BroDocObjList&, bool, const char*, char) + * @see WriteBroDocObjList(const BroDocObjList&, bool, const char*, char, + bool) */ void WriteBroDocObjList(const BroDocObjMap& m, bool wantPublic, const char* heading, - char underline) const; + char underline, + bool isShort) const; /** * Writes out a list of BroDocObj objects to the reST document @@ -242,6 +263,12 @@ protected: const char* heading, char underline) const; + /** + * Writes out a list of BroDocObj objects to the reST document + * @param l A list of BroDocObj pointers + */ + void WriteBroDocObjList(const BroDocObjList& l) const; + /** * Wraps the BroDocObjMap into a BroDocObjList and the writes that list * to the reST document @@ -266,6 +293,13 @@ protected: */ void WriteSectionHeading(const char* heading, char underline) const; + /** + * Writes out given number of characters to reST document + * @param c the character to write + * @param n the number of characters to write + */ + void WriteRepeatedChar(char c, size_t n) const; + /** * Writes out the reST for either the script's public or private interface * @param heading The title of the interfaces section heading @@ -275,9 +309,11 @@ protected: * sub-sections * @param isPublic Whether to write out the public or private script * interface + * @param isShort Whether to write out the full documentation or a "short" + * description (a single sentence) */ void WriteInterface(const char* heading, char underline, char subunderline, - bool isPublic) const; + bool isPublic, bool isShort) const; private: /** @@ -285,7 +321,6 @@ private: * @param a reference to a list of BroDocObj pointers */ void FreeBroDocObjPtrList(BroDocObjList& l); - void FreeBroDocObjPtrList(BroDocObjMap& l); static bool IsPublicAPI(const BroDocObj* o) { return o->IsPublicAPI(); } static bool IsPrivateAPI(const BroDocObj* o) { return ! o->IsPublicAPI(); } diff --git a/src/BroDocObj.cc b/src/BroDocObj.cc index 85ad2b85cd..1b26334c9e 100644 --- a/src/BroDocObj.cc +++ b/src/BroDocObj.cc @@ -12,6 +12,7 @@ BroDocObj::BroDocObj(const ID* id, std::list*& reST, reST = 0; is_fake_id = is_fake; use_role = 0; + FormulateShortDesc(); } BroDocObj::~BroDocObj() @@ -20,13 +21,84 @@ BroDocObj::~BroDocObj() if ( is_fake_id ) delete broID; } +void BroDocObj::WriteReSTCompact(FILE* file, int max_col) const + { + ODesc desc; + desc.SetQuotes(1); + broID->DescribeReSTShort(&desc); + + fprintf(file, "%s", desc.Description()); + + std::list::const_iterator it; + for ( it = short_desc.begin(); it != short_desc.end(); ++it ) + { + int start_col; + if ( it == short_desc.begin() ) + start_col = max_col - desc.Len() + 1; + else + { + start_col = max_col + 1; + fprintf(file, "\n"); + } + + for ( int i = 0; i < start_col; ++i ) + fprintf(file, " "); + + fprintf(file, "%s", it->c_str()); + } + } + +int BroDocObj::LongestShortDescLen() const + { + size_t max = 0; + std::list::const_iterator it; + for ( it = short_desc.begin(); it != short_desc.end(); ++it ) + if ( it->size() > max ) max = it->size(); + return max; + } + +void BroDocObj::FormulateShortDesc() + { + if ( ! reST_doc_strings ) return; + + short_desc.clear(); + std::list::const_iterator it; + for ( it = reST_doc_strings->begin(); + it != reST_doc_strings->end(); ++it ) + { + // the short description stops at the first sentence + // or the first empty comment + size_t end = it->find_first_of("."); + if ( end == string::npos ) + { + std::string::const_iterator s; + bool empty = true; + for ( s = it->begin(); s != it->end(); ++s ) + if ( *s != ' ' && *s != '\t' && *s != '\n' && *s != '\r' ) + { + empty = false; + short_desc.push_back(*it); + break; + } + if ( empty ) break; + } + else + { + short_desc.push_back(it->substr(0, end + 1)); + break; + } + } + } + void BroDocObj::WriteReST(FILE* file) const { int indent_spaces = 3; ODesc desc; desc.SetIndentSpaces(indent_spaces); desc.SetQuotes(1); + broID->DescribeReST(&desc, use_role); + fprintf(file, "%s", desc.Description()); if ( HasDocumentation() ) @@ -45,6 +117,14 @@ void BroDocObj::WriteReST(FILE* file) const fprintf(file, "\n"); } +int BroDocObj::ColumnSize() const + { + ODesc desc; + desc.SetQuotes(1); + broID->DescribeReSTShort(&desc); + return desc.Len(); + } + bool BroDocObj::IsPublicAPI() const { return (broID->Scope() == SCOPE_GLOBAL) || @@ -63,4 +143,5 @@ void BroDocObj::Combine(const BroDocObj* o) } delete o; + FormulateShortDesc(); } diff --git a/src/BroDocObj.h b/src/BroDocObj.h index 0660d510f6..a6f497db50 100644 --- a/src/BroDocObj.h +++ b/src/BroDocObj.h @@ -28,14 +28,27 @@ public: /** * Writes the reST representation of this object which includes - * 1) Any "##" or "##<" stylized comments. + * 1) a reST friendly description of the ID + * 2) "##" or "##<" stylized comments. * Anything after these style of comments is inserted as-is into * the reST document. - * 2) a reST friendly description of the ID - * @param The (already opened) file to write the reST to. + * @param file The (already opened) file to write the reST to. */ void WriteReST(FILE* file) const; + /** + * Writes a compact version of the ID and associated documentation + * for insertion into a table. + * @param file The (already opened) file to write the reST to. + * @param max_col The maximum length of the first table column + */ + void WriteReSTCompact(FILE* file, int max_col) const; + + /** + * @return the column size required by the reST representation of the ID + */ + int ColumnSize() const; + /** * Check whether this documentation is part of the public API. In * other words, this means that the identifier is declared as part of @@ -78,12 +91,27 @@ public: */ const char* Name() const { return broID->Name(); } + /** + * @return the longest string element of the short description's list of + * strings + */ + int LongestShortDescLen() const; + + protected: std::list* reST_doc_strings; + std::list short_desc; const ID* broID; bool is_fake_id; /**< Whether the ID* is a dummy just for doc purposes */ bool use_role; /**< Whether to use a reST role or directive for the ID */ + /** + * Set the short_desc member to be a subset of reST_doc_strings. + * Specifically, short_desc will be everything in reST_doc_strings + * up until the first period or first empty string list element found. + */ + void FormulateShortDesc(); + private: }; diff --git a/src/ID.cc b/src/ID.cc index b3fcf8f5c3..7678953445 100644 --- a/src/ID.cc +++ b/src/ID.cc @@ -607,6 +607,33 @@ void ID::DescribeExtended(ODesc* d) const } } +void ID::DescribeReSTShort(ODesc* d) const + { + if ( is_type ) + d->Add(":bro:type:`"); + else + d->Add(":bro:id:`"); + d->Add(name); + d->Add("`"); + + if ( type ) + { + d->Add(": "); + d->Add(":bro:type:`"); + if ( ! is_type && type->GetTypeID() ) + d->Add(type->GetTypeID()); + else + d->Add(type_name(type->Tag())); + d->Add("`"); + } + + if ( attrs ) + { + d->SP(); + attrs->DescribeReST(d); + } + } + void ID::DescribeReST(ODesc* d, bool is_role) const { if ( is_role ) diff --git a/src/ID.h b/src/ID.h index 90a0862a18..440f9c8d20 100644 --- a/src/ID.h +++ b/src/ID.h @@ -86,6 +86,7 @@ public: void DescribeExtended(ODesc* d) const; // Produces a description that's reST-ready void DescribeReST(ODesc* d, bool is_role=false) const; + void DescribeReSTShort(ODesc* d) const; bool Serialize(SerialInfo* info) const; static ID* Unserialize(UnserialInfo* info); From 758172120b9498110e86d8968ca8094135e041cb Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Fri, 1 Apr 2011 15:21:15 -0500 Subject: [PATCH 31/47] Revising format of initialization values in generated script docs. Fixed the obvious things: reduced the space taken up by empty tables and allowed record values to span multiple lines (one for each record field). --- src/ID.cc | 28 +++++++++++++++++++--------- src/Val.cc | 47 ++++++++++++++++++++++++++++++++++++++--------- src/Val.h | 3 ++- 3 files changed, 59 insertions(+), 19 deletions(-) diff --git a/src/ID.cc b/src/ID.cc index 7678953445..12b5b7951a 100644 --- a/src/ID.cc +++ b/src/ID.cc @@ -681,20 +681,30 @@ void ID::DescribeReST(ODesc* d, bool is_role) const type->Tag() != TYPE_FUNC && type->InternalType() != TYPE_INTERNAL_VOID ) { + d->Add(":Default:"); if ( type->InternalType() == TYPE_INTERNAL_OTHER ) { - d->Add(":Init:"); - d->NL(); - d->NL(); - d->Add("::"); - d->NL(); - d->PushIndent(); - val->DescribeReST(d); - d->PopIndent(); + switch ( type->Tag() ) { + case TYPE_TABLE: + if ( val->AsTable()->Length() == 0 ) + { + d->Add(" ``{}``"); + d->NL(); + break; + } + default: + d->NL(); + d->NL(); + d->Add("::"); + d->NL(); + d->PushIndent(); + val->DescribeReST(d); + d->PopIndent(); + } } else { - d->Add(":Init: "); + d->SP(); val->DescribeReST(d); d->NL(); } diff --git a/src/Val.cc b/src/Val.cc index 012ca92773..f237ff7bec 100644 --- a/src/Val.cc +++ b/src/Val.cc @@ -576,14 +576,7 @@ void Val::Describe(ODesc* d) const void Val::DescribeReST(ODesc* d) const { - if ( type->InternalType() == TYPE_INTERNAL_OTHER ) - Describe(d); - else - { - d->Add("``"); - ValDescribeReST(d); - d->Add("``"); - } + ValDescribeReST(d); } void Val::ValDescribe(ODesc* d) const @@ -629,7 +622,15 @@ void Val::ValDescribe(ODesc* d) const void Val::ValDescribeReST(ODesc* d) const { - ValDescribe(d); + switch ( type->InternalType() ) { + case TYPE_INTERNAL_OTHER: + Describe(d); + break; + default: + d->Add("``"); + ValDescribe(d); + d->Add("``"); + } } MutableVal::~MutableVal() @@ -2946,6 +2947,34 @@ void RecordVal::Describe(ODesc* d) const d->Add("]"); } +void RecordVal::DescribeReST(ODesc* d) const + { + const val_list* vl = AsRecord(); + int n = vl->length(); + + d->Add("{"); + d->PushIndent(); + + loop_over_list(*vl, i) + { + if ( i > 0 ) + d->NL(); + + d->Add(record_type->FieldName(i)); + + d->Add("="); + + Val* v = (*vl)[i]; + if ( v ) + v->Describe(d); + else + d->Add(""); + } + + d->PopIndent(); + d->Add("}"); + } + IMPLEMENT_SERIAL(RecordVal, SER_RECORD_VAL); bool RecordVal::DoSerialize(SerialInfo* info) const diff --git a/src/Val.h b/src/Val.h index 0d759360a7..0225fb6725 100644 --- a/src/Val.h +++ b/src/Val.h @@ -313,7 +313,7 @@ public: } void Describe(ODesc* d) const; - void DescribeReST(ODesc* d) const; + virtual void DescribeReST(ODesc* d) const; bool Serialize(SerialInfo* info) const; static Val* Unserialize(UnserialInfo* info, TypeTag type = TYPE_ANY) @@ -904,6 +904,7 @@ public: BroObj* GetOrigin() const { return origin; } unsigned int MemoryAllocation() const; + void DescribeReST(ODesc* d) const; protected: friend class Val; From bd523f290520210556e0e8bc97c334bb253073f3 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Mon, 4 Apr 2011 12:29:48 -0500 Subject: [PATCH 32/47] Automatic reST doc markup for event/function params/return comments For a bro script that is documented like: ## ... ## bar: ... ## Returns: ... global foo(bar: string): string; the generated reST documentation will automatically substitute meaningful markup (reST field lists) for the parameter and return type comments --- src/scan.l | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/src/scan.l b/src/scan.l index 5b82a92788..ed599b679b 100644 --- a/src/scan.l +++ b/src/scan.l @@ -159,6 +159,40 @@ ESCSEQ (\\([^\n]|[0-7]+|x[[:xdigit:]]+)) } } +##{OWS}{ID}:.* { + if ( generate_documentation ) + { + // Comment is documenting either a function parameter or return type, + // so appropriate reST markup substitutions are automatically made + // in order to distinguish them from other comments. + const char* id_start = skip_whitespace(yytext + 2); + size_t id_len = strcspn(id_start, ":"); + char* id_name = new char[id_len + 1]; + strncpy(id_name, id_start, id_len); + id_name[id_len] = '\0'; + const char* comment = id_start + id_len + 1; + + std::string doc; + + if ( streq(id_name, "Returns") ) + doc.append(":returns:").append(comment); + else + doc.append(":param ").append(id_name).append(":").append(comment); + + if ( ! reST_doc_comments ) + reST_doc_comments = new std::list(); + + // always insert a blank line so that this param/return markup + // 1) doesn't show up in the summary section in the case that it's + // the first comment for the function/event + // 2) has a blank line between it and non-field-list reST markup, + // which is required for correct HTML rendering by Sphinx + reST_doc_comments->push_back(""); + reST_doc_comments->push_back(doc); + + delete [] id_name; + } +} ##.* { if ( generate_documentation && (yytext[2] != '#') ) From 5183ab409bdd0f7a536a7e81eaeebf549ca6645d Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Wed, 6 Apr 2011 11:59:37 -0500 Subject: [PATCH 33/47] Initial implementation of a `make doc` target to generate script docs. --- CMakeLists.txt | 8 +- doc/CMakeLists.txt | 40 +++++ doc/conf.py.in | 215 +++++++++++++++++++++++++++ doc/example.bro | 178 ++++++++++++++++++++++ doc/scripts/generate_reST_docs.py.in | 121 +++++++++++++++ doc/source/_static/showhide.js | 64 ++++++++ doc/source/_templates/layout.html | 5 + doc/source/builtins.rst | 121 +++++++++++++++ doc/source/common.rst | 19 +++ doc/source/ext/bro.py | 167 +++++++++++++++++++++ doc/source/index.rst | 20 +++ doc/source/policy/index.rst | 8 + 12 files changed, 964 insertions(+), 2 deletions(-) create mode 100644 doc/CMakeLists.txt create mode 100644 doc/conf.py.in create mode 100644 doc/example.bro create mode 100755 doc/scripts/generate_reST_docs.py.in create mode 100644 doc/source/_static/showhide.js create mode 100644 doc/source/_templates/layout.html create mode 100644 doc/source/builtins.rst create mode 100644 doc/source/common.rst create mode 100644 doc/source/ext/bro.py create mode 100644 doc/source/index.rst create mode 100644 doc/source/policy/index.rst diff --git a/CMakeLists.txt b/CMakeLists.txt index 3c3cd05d96..65e1bc5983 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -43,6 +43,11 @@ file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/bro-path-dev.csh "setenv BROPATH `${CMAKE_CURRENT_BINARY_DIR}/bro-path-dev`\n") file(STRINGS "${CMAKE_CURRENT_SOURCE_DIR}/VERSION" VERSION LIMIT_COUNT 1) +string(REPLACE "." " " version_numbers ${VERSION}) +separate_arguments(version_numbers) +list(GET version_numbers 0 VERSION_MAJOR) +list(GET version_numbers 1 VERSION_MINOR) +set(VERSION_MAJ_MIN "${VERSION_MAJOR}.${VERSION_MINOR}") set(EXTRA_COMPILE_FLAGS "-Wall -Wno-unused") @@ -162,8 +167,7 @@ include_directories(${CMAKE_CURRENT_BINARY_DIR}) add_subdirectory(src) add_subdirectory(policy) -#add_subdirectory(scripts) -#add_subdirectory(doc) +add_subdirectory(doc) include(CheckOptionalBuildSources) diff --git a/doc/CMakeLists.txt b/doc/CMakeLists.txt new file mode 100644 index 0000000000..a5a97d5c3b --- /dev/null +++ b/doc/CMakeLists.txt @@ -0,0 +1,40 @@ +set(POLICY_SRC_DIR ${PROJECT_SOURCE_DIR}/policy) +set(DOC_OUTPUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/out) +set(DOC_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/source) +set(DOC_SOURCE_WORKDIR ${CMAKE_CURRENT_BINARY_DIR}/source) + +file(GLOB_RECURSE DOC_SOURCES FOLLOW_SYMLINKS "*") + +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/conf.py.in + ${CMAKE_CURRENT_BINARY_DIR}/conf.py + @ONLY) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/scripts/generate_reST_docs.py.in + ${CMAKE_CURRENT_BINARY_DIR}/generate_reST_docs.py + @ONLY) + +add_custom_target(doc + COMMAND "${CMAKE_COMMAND}" -E copy_directory + ${DOC_SOURCE_DIR} + ${DOC_SOURCE_WORKDIR} + COMMAND python generate_reST_docs.py + COMMAND sphinx-build + -b html + -c ${CMAKE_CURRENT_BINARY_DIR} + -d ${DOC_OUTPUT_DIR}/doctrees + ${DOC_SOURCE_WORKDIR} + ${DOC_OUTPUT_DIR}/html + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + COMMENT "[Sphinx] Generating Script Documentation" + VERBATIM + # SOURCES just adds stuff to IDE projects as a convienience + SOURCES ${DOC_SOURCES}) + +add_dependencies(doc bro) +# TODO: add dependency that's a check for `python`, `sphinx-build`, etc. + +add_custom_target(doc-clean + COMMAND "${CMAKE_COMMAND}" -E remove_directory + ${CMAKE_CURRENT_BINARY_DIR}/source + COMMAND "${CMAKE_COMMAND}" -E remove_directory + ${DOC_OUTPUT_DIR} + VERBATIM) diff --git a/doc/conf.py.in b/doc/conf.py.in new file mode 100644 index 0000000000..67e06abfb3 --- /dev/null +++ b/doc/conf.py.in @@ -0,0 +1,215 @@ +# -*- coding: utf-8 -*- +# +# Bro documentation build configuration file, created by sphinx-quickstart +# +# This file is execfile()d with the current directory set to its containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +import sys, os + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +sys.path.insert(0, os.path.abspath('source/ext')) + +# -- General configuration ----------------------------------------------------- + +# If your documentation needs a minimal Sphinx version, state it here. +#needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be extensions +# coming with Sphinx (named 'sphinx.ext.*') or your custom ones. +extensions = ['bro'] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['source/_templates'] + +# The suffix of source filenames. +source_suffix = '.rst' + +# The encoding of source files. +#source_encoding = 'utf-8-sig' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = u'Bro' +copyright = u'2011, Jon Siwek' + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +version = '@VERSION_MAJ_MIN@' +# The full version, including alpha/beta/rc tags. +release = '@VERSION@' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +#language = None + +# There are two options for replacing |today|: either, you set today to some +# non-false value, then it is used: +#today = '' +# Else, today_fmt is used as the format for a strftime call. +#today_fmt = '%B %d, %Y' + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +exclude_patterns = [] + +# The reST default role (used for this markup: `text`) to use for all documents. +#default_role = None + +# If true, '()' will be appended to :func: etc. cross-reference text. +#add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +#add_module_names = True + +# If true, sectionauthor and moduleauthor directives will be shown in the +# output. They are ignored by default. +show_authors = True + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# A list of ignored prefixes for module index sorting. +#modindex_common_prefix = [] + + +# -- Options for HTML output --------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +html_theme = 'default' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +#html_theme_options = {} + +# Add any paths that contain custom themes here, relative to this directory. +#html_theme_path = [] + +# The name for this set of Sphinx documents. If None, it defaults to +# " v documentation". +#html_title = None + +# A shorter title for the navigation bar. Default is the same as html_title. +#html_short_title = None + +# The name of an image file (relative to this directory) to place at the top +# of the sidebar. +#html_logo = None + +# The name of an image file (within the static path) to use as favicon of the +# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 +# pixels large. +#html_favicon = None + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['source/_static'] + +# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, +# using the given strftime format. +#html_last_updated_fmt = '%b %d, %Y' + +# If true, SmartyPants will be used to convert quotes and dashes to +# typographically correct entities. +#html_use_smartypants = True + +# Custom sidebar templates, maps document names to template names. +#html_sidebars = {} + +# Additional templates that should be rendered to pages, maps page names to +# template names. +#html_additional_pages = {} + +# If false, no module index is generated. +#html_domain_indices = True + +# If false, no index is generated. +#html_use_index = True + +# If true, the index is split into individual pages for each letter. +#html_split_index = False + +# If true, links to the reST sources are added to the pages. +#html_show_sourcelink = True + +# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. +#html_show_sphinx = True + +# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. +#html_show_copyright = True + +# If true, an OpenSearch description file will be output, and all pages will +# contain a tag referring to it. The value of this option must be the +# base URL from which the finished HTML is served. +#html_use_opensearch = '' + +# This is the file name suffix for HTML files (e.g. ".xhtml"). +#html_file_suffix = None + +# Output file base name for HTML help builder. +htmlhelp_basename = 'Brodoc' + + +# -- Options for LaTeX output -------------------------------------------------- + +# The paper size ('letter' or 'a4'). +#latex_paper_size = 'letter' + +# The font size ('10pt', '11pt' or '12pt'). +#latex_font_size = '10pt' + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, author, documentclass [howto/manual]). +latex_documents = [ + ('index', 'Bro.tex', u'Bro Documentation', + u'Jon Siwek', 'manual'), +] + +# The name of an image file (relative to this directory) to place at the top of +# the title page. +#latex_logo = None + +# For "manual" documents, if this is true, then toplevel headings are parts, +# not chapters. +#latex_use_parts = False + +# If true, show page references after internal links. +#latex_show_pagerefs = False + +# If true, show URL addresses after external links. +#latex_show_urls = False + +# Additional stuff for the LaTeX preamble. +#latex_preamble = '' + +# Documents to append as an appendix to all manuals. +#latex_appendices = [] + +# If false, no module index is generated. +#latex_domain_indices = True + + +# -- Options for manual page output -------------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + ('index', 'bro', u'Bro Documentation', + [u'Jon Siwek'], 1) +] diff --git a/doc/example.bro b/doc/example.bro new file mode 100644 index 0000000000..6deab87f45 --- /dev/null +++ b/doc/example.bro @@ -0,0 +1,178 @@ +##! This is an example script that demonstrates how to document. Comments +##! of the form ``##!`` are for the script summary. The contents of +##! these comments are transferred directly into the auto-generated +##! `reStructuredText `_ +##! (reST) document's summary section. +##! +##! .. tip:: You can embed directives and roles within ``##``-stylized comments +##! +##! :Author: Jon Siwek + +# Comments that use a single pound sign (#) are not significant to +# a script's auto-generated documentation, but ones that use a +# double pound sign (##) do matter. In some cases, like record +# field comments, it's necessary to disambiguate the field with +# which a comment associates: e.g. "##<" can be used on the same line +# as a field to signify the comment relates to it and not the +# following field. "##<" is not meant for general use, just +# record/enum fields. +# +# Generally, the auto-doc comments (##) are associated with the +# next declaration/identifier found in the script, but the doc framework +# will track/render identifiers regardless of whether they have any +# of these special comments associated with them. +# +# The first sentence contained within the "##"-stylized comments for +# a given identifier is special in that it will be used as summary +# text in a table containing all such identifiers and short summaries. +# If there are no sentences (text terminated with '.'), then everything +# in the "##"-stylized comments up until the first empty comment +# is taken as the summary text for a given identifier. + +# @load directives are self-documenting +@load notice + +# "module" statements are self-documenting +module Example; + +# redefinitions of "capture_filters" are self-documenting and +# go into the generated documentation's "Packet Filter" section +redef capture_filters += { + ["ssl"] = "tcp port 443", + ["nntps"] = "tcp port 563", +}; + +global example_ports = { + 443/tcp, 563/tcp, +} &redef; + +# redefinitions of "dpd_config" are self-documenting and +# go into the generated doc's "Port Analysis" section +redef dpd_config += { + [ANALYZER_SSL] = [$ports = example_ports] +}; + +# redefinitions of "Notice::Type" are self-documenting, but +# more information can be supplied in two different ways +redef enum Notice += { + ## any number of this type of comment + ## will document "Notice_One" + Notice_One, + Notice_Two, ##< any number of this type of comment + ##< will document "Notice_Two" + Notice_Three, + Notice_Four, +}; + +# Anything declared in the export section will show up in the rendered +# documentation's "public interface" section + +export { + + # these headings don't mean anything special to the + # doc framework right now, I'm just including them + # to make it more clear to the reader how the doc + # framework will actually categorize a script's identifiers + + ############## types ################ + + # Note that I'm just mixing the "##" and "##<" + # types of comments in the following declarations + # as a demonstration. Normally, it would be good style + # to pick one and be consistent. + + ## documentation for "SimpleEnum" + ## goes here. + type SimpleEnum: enum { + ## and more specific info for "ONE" + ## can span multiple lines + ONE, + TWO, ##< or more info like this for "TWO" + ##< can span multiple lines + THREE, + }; + + ## document the "SimpleEnum" redef here + redef enum SimpleEnum += { + FOUR, ##< and some documentation for "FOUR" + ## also "FIVE" for good measure + FIVE + }; + + ## general documentation for a type "SimpleRecord" + ## goes here. + type SimpleRecord: record { + ## counts something + field1: count; + field2: bool; ##< toggles something + }; + + + ## general documentation for a type "ComplexRecord" goes here + type ComplexRecord: record { + field1: count; ##< counts something + field2: bool; ##< toggles something + field3: SimpleRecord; + msg: string &default="blah"; ##< attributes are self-documenting + } &redef; + + ############## options ################ + # right now, I'm just defining an option as + # any const with &redef (something that can + # change at parse time, but not at run time. + + ## add documentation for "an_option" here + const an_option: set[addr, addr, string] &redef; + + # default initialization will be self-documenting + const option_with_init = 0.01 secs &redef; + + ############## state variables ############ + # right now, I'm defining this as any global + # that's not a function/event. doesn't matter + # if &redef attribute is present + + ## put some documentation for "a_var" here + global a_var: bool; + + # attributes are self-documenting + global var_with_attr: count &persistent; + + # it's fine if the type is inferred, that information is self-documenting + global var_without_explicit_type = "this works"; + + ############## functions/events ############ + + ## Summarize purpose of "a_function" here. + ## Give more details about "a_function" here. + ## Separating the documentation of the params/return values with + ## empty comments is optional, but improves readability of script. + ## + ## tag: function arguments can be described + ## like this + ## msg: another param + ## + ## Returns: describe the return type here + global a_function: function(tag: string, msg: string): string; + + ## Summarize "an_event" here. + ## Give more details about "an_event" here. + ## name: describe the argument here + global an_event: event(name: string); +} + +# this function is documented in the "private interface" section +# of generated documentation and any "##"-stylized comments would also +# be rendered there +function function_without_proto(tag: string): string + { + return "blah"; + } + +# this record type is documented in the "private interface" section +# of generated documentation and any "##"-stylized comments would also +# be rendered there +type PrivateRecord: record { + field1: bool; + field2: count; +}; diff --git a/doc/scripts/generate_reST_docs.py.in b/doc/scripts/generate_reST_docs.py.in new file mode 100755 index 0000000000..1d83cc01fb --- /dev/null +++ b/doc/scripts/generate_reST_docs.py.in @@ -0,0 +1,121 @@ +#! /usr/bin/env python + +import os +import subprocess +import shutil +import glob +import string + +BRO = "@CMAKE_BINARY_DIR@/src/bro" +BROPATHDEV = "`@CMAKE_BINARY_DIR@/bro-path-dev`" +BRO_ARGS = "--doc-scripts" +DOC_DST_DIR = "@DOC_SOURCE_WORKDIR@/policy" +BROPATH = subprocess.Popen("@CMAKE_BINARY_DIR@/bro-path-dev", shell=True, + stdout=subprocess.PIPE, stderr=subprocess.STDOUT).stdout.readline() + +class BroToReST: + bro_src_file = "" + doc_src_file = "" + load_via_stdin = False + + def __init__(self, src_file, load_method=False, src_dir="@POLICY_SRC_DIR@"): + self.bro_src_file = os.path.join(src_dir, src_file) + self.load_via_stdin = load_method + + # formulate doc_src_file from src_file + filename = os.path.basename(src_file) + basename, ext = os.path.splitext(filename) + if ext == ".bro": + self.doc_src_file = basename + ".rst" + else: + self.doc_src_file = filename + ".rst" + + def __str__(self): + return "bro_src_file: " + self.bro_src_file \ + + "\ndoc_src_file: " + self.doc_src_file \ + + "\ndoc_dst_file: " + os.path.join(DOC_DST_DIR, self.doc_src_file) \ + + "\nstdin_load: %s" % self.load_via_stdin + + def GenDoc(self): + bro_src_basename = os.path.basename(self.bro_src_file) + + if self.load_via_stdin: + cmd = "echo '@load %s' | %s %s" % (bro_src_basename, BRO, BRO_ARGS) + else: + cmd = "%s %s %s" % (BRO, BRO_ARGS, self.bro_src_file) + + p = subprocess.Popen(cmd, shell=True, env={"BROPATH": BROPATH}) + + if p.wait() == 0: + shutil.copy(self.doc_src_file, DOC_DST_DIR) + shutil.copy(self.bro_src_file, DOC_DST_DIR) + + for leftover in glob.glob("*.rst"): + os.remove(leftover) + +def GenDocs(doc_list, load_method=False): + for f in doc_list: + doc = BroToReST(f, load_method) + print "Generating reST document for " + f + doc.GenDoc() + +# search BROPATH for the script and return the absolute path to it +def FindBroScript(src_file): + for path in string.split(BROPATH, ":"): + abs_path = os.path.join(path, src_file) + if os.path.exists(abs_path): + return abs_path + +# Scripts that can be loaded by bro via command line argument +# TODO: generate docs for more scripts +docs = [ + "alarm.bro", + "arp.bro", + "conn.bro", + "dhcp.bro", + "dns.bro", + "ftp.bro", + "http.bro", + "http-reply.bro", + "http-request.bro", + "irc.bro", + "smtp.bro", + "ssl.bro", + "ssl-ciphers.bro", + "ssl-errors.bro", + "synflood.bro", + "tcp.bro", + "udp.bro", + "weird.bro", +] + +# Scripts that can't be loaded by bro via command line argument (possible +# due to dependency issues), but can be loaded via an @load on stdin +stdin_docs = [ + "notice.bro", +] + +GenDocs(docs) +GenDocs(stdin_docs, True) + +BroToReST("example.bro", False, "@PROJECT_SOURCE_DIR@/doc").GenDoc() + +# Generate documentation for stuff that's always loaded into bro by default +cmd = "echo '' | %s %s" % (BRO, BRO_ARGS) +p = subprocess.Popen(cmd, shell=True, env={"BROPATH": BROPATH}) +if p.wait() == 0: + for doc in glob.glob("*.rst"): + if doc == ".rst": + os.remove(doc) + continue + + basename, ext = os.path.splitext(doc) + basename2, ext = os.path.splitext(basename) + if ext == ".init": + src_file = basename + else: + src_file = basename + ".bro" + src_file = FindBroScript(src_file) + shutil.copy(src_file, DOC_DST_DIR) + shutil.copy(doc, DOC_DST_DIR) + os.remove(doc) diff --git a/doc/source/_static/showhide.js b/doc/source/_static/showhide.js new file mode 100644 index 0000000000..d6a8923143 --- /dev/null +++ b/doc/source/_static/showhide.js @@ -0,0 +1,64 @@ +// make literal blocks corresponding to identifier initial values +// hidden by default +$(document).ready(function() { + + var showText='(Show Value)'; + var hideText='(Hide Value)'; + + var is_visible = false; + + // select field-list tables that come before a literal block + tables = $('.highlight-python').prev('table.docutils.field-list'); + + tables.find('th.field-name').filter(function(index) { + return $(this).html() == "Default :"; + }).next().append(''+showText+''); + + // hide all literal blocks that follow a field-list table + tables.next('.highlight-python').hide(); + + // register handler for clicking a "toggle" link + $('a.toggleLink').click(function() { + is_visible = !is_visible; + + $(this).html( (!is_visible) ? showText : hideText); + + // the link is inside a
and the next + // literal block after the table is the literal block that we want + // to show/hide + $(this).parent().parent().parent().parent().next('.highlight-python').slideToggle('fast'); + + // override default link behavior + return false; + }); +}); + +// make "Private Interface" sections hidden by default +$(document).ready(function() { + + var showText='Show Private Interface (for internal use)'; + var hideText='Hide Private Interface'; + + var is_visible = false; + + // insert show/hide links + $('#private-interface').children(":first-child").after(''+showText+''); + + // wrap all sub-sections in a new div that can be hidden/shown + $('#private-interface').children(".section").wrapAll('
'); + + // hide the given class + $('.private').hide(); + + // register handler for clicking a "toggle" link + $('a.privateToggle').click(function() { + is_visible = !is_visible; + + $(this).html( (!is_visible) ? showText : hideText); + + $('.private').slideToggle('fast'); + + // override default link behavior + return false; + }); +}); diff --git a/doc/source/_templates/layout.html b/doc/source/_templates/layout.html new file mode 100644 index 0000000000..5685e1dc37 --- /dev/null +++ b/doc/source/_templates/layout.html @@ -0,0 +1,5 @@ +{% extends "!layout.html" %} +{% block extrahead %} + + {{ super() }} +{% endblock %} diff --git a/doc/source/builtins.rst b/doc/source/builtins.rst new file mode 100644 index 0000000000..886edab7c7 --- /dev/null +++ b/doc/source/builtins.rst @@ -0,0 +1,121 @@ +Builtins +======== + +Types +----- + +The Bro scripting language supports the following built-in types. + +.. TODO: add documentation + +.. bro:type:: void + +.. bro:type:: bool + +.. bro:type:: int + +.. bro:type:: count + +.. bro:type:: counter + +.. bro:type:: double + +.. bro:type:: time + +.. bro:type:: interval + +.. bro:type:: string + +.. bro:type:: pattern + +.. bro:type:: enum + +.. bro:type:: timer + +.. bro:type:: port + +.. bro:type:: addr + +.. bro:type:: net + +.. bro:type:: subnet + +.. bro:type:: any + +.. bro:type:: table + +.. bro:type:: union + +.. bro:type:: record + +.. bro:type:: types + +.. bro:type:: func + +.. bro:type:: file + +.. bro:type:: vector + +.. TODO: below are kind of "special cases" that bro knows about? + +.. bro:type:: set + +.. bro:type:: function + +.. bro:type:: event + +.. TODO: Notice will get documented as part of notice.bro, which can eventually + be referenced here once that documentation is auto-generated. + +.. bro:type:: Notice + +Attributes +---------- + +The Bro scripting language supports the following built-in attributes. + +.. TODO: add documentation + +.. bro:attr:: &optional + +.. bro:attr:: &default + +.. bro:attr:: &redef + +.. bro:attr:: &rotate_interval + +.. bro:attr:: &rotate_size + +.. bro:attr:: &add_func + +.. bro:attr:: &delete_func + +.. bro:attr:: &expire_func + +.. bro:attr:: &read_expire + +.. bro:attr:: &write_expire + +.. bro:attr:: &create_expire + +.. bro:attr:: &persistent + +.. bro:attr:: &synchronized + +.. bro:attr:: &postprocessor + +.. bro:attr:: &encrypt + +.. bro:attr:: &match + +.. bro:attr:: &disable_print_hook + +.. bro:attr:: &raw_output + +.. bro:attr:: &mergeable + +.. bro:attr:: &priority + +.. bro:attr:: &group + +.. bro:attr:: (&tracked) diff --git a/doc/source/common.rst b/doc/source/common.rst new file mode 100644 index 0000000000..6105585b2c --- /dev/null +++ b/doc/source/common.rst @@ -0,0 +1,19 @@ +Common Documentation +==================== + +.. _common_port_analysis_doc: + +Port Analysis +------------- + +TODO: add some stuff here + +.. _common_packet_filter_doc: + +Packet Filter +------------- + +TODO: add some stuff here + +.. note:: Filters are only relevant when dynamic protocol detection (DPD) + is explicitly turned off (Bro release 1.6 enabled DPD by default). diff --git a/doc/source/ext/bro.py b/doc/source/ext/bro.py new file mode 100644 index 0000000000..7ec02f76a8 --- /dev/null +++ b/doc/source/ext/bro.py @@ -0,0 +1,167 @@ +""" + The Bro domain for Sphinx. +""" + +def setup(Sphinx): + Sphinx.add_domain(BroDomain) + +from sphinx import addnodes +from sphinx.domains import Domain, ObjType, Index +from sphinx.locale import l_, _ +from sphinx.directives import ObjectDescription +from sphinx.roles import XRefRole +from sphinx.util.nodes import make_refnode +import string + +from docutils import nodes +from docutils.parsers.rst import Directive +from docutils.parsers.rst import directives +from docutils.parsers.rst.roles import set_classes + +class BroGeneric(ObjectDescription): + def add_target_and_index(self, name, sig, signode): + targetname = self.objtype + '-' + name + if targetname not in self.state.document.ids: + signode['names'].append(targetname) + signode['ids'].append(targetname) + signode['first'] = (not self.names) + self.state.document.note_explicit_target(signode) + + objects = self.env.domaindata['bro']['objects'] + key = (self.objtype, name) +# this is commented out mostly just to avoid having a special directive +# for events in order to avoid the duplicate warnings in that case + """ + if key in objects: + self.env.warn(self.env.docname, + 'duplicate description of %s %s, ' % + (self.objtype, name) + + 'other instance in ' + + self.env.doc2path(objects[key]), + self.lineno) + """ + objects[key] = self.env.docname + indextext = self.get_index_text(self.objtype, name) + if indextext: + self.indexnode['entries'].append(('single', indextext, + targetname, targetname)) + + def get_index_text(self, objectname, name): + return _('%s (%s)') % (name, self.objtype) + + def handle_signature(self, sig, signode): + signode += addnodes.desc_name("", sig) + return sig + +class BroNamespace(BroGeneric): + def add_target_and_index(self, name, sig, signode): + targetname = self.objtype + '-' + name + if targetname not in self.state.document.ids: + signode['names'].append(targetname) + signode['ids'].append(targetname) + signode['first'] = (not self.names) + self.state.document.note_explicit_target(signode) + + objects = self.env.domaindata['bro']['objects'] + key = (self.objtype, name) + objects[key] = self.env.docname + indextext = self.get_index_text(self.objtype, name) + self.indexnode['entries'].append(('single', indextext, + targetname, targetname)) + self.indexnode['entries'].append(('single', + "namespaces; %s" % (sig), + targetname, targetname)) + + def get_index_text(self, objectname, name): + return _('%s (namespace); %s') % (name, self.env.docname) + + def handle_signature(self, sig, signode): + signode += addnodes.desc_name("", sig) + return sig + +class BroEnum(BroGeneric): + def add_target_and_index(self, name, sig, signode): + targetname = self.objtype + '-' + name + if targetname not in self.state.document.ids: + signode['names'].append(targetname) + signode['ids'].append(targetname) + signode['first'] = (not self.names) + self.state.document.note_explicit_target(signode) + + objects = self.env.domaindata['bro']['objects'] + key = (self.objtype, name) + objects[key] = self.env.docname + indextext = self.get_index_text(self.objtype, name) + #self.indexnode['entries'].append(('single', indextext, + # targetname, targetname)) + m = sig.split() + self.indexnode['entries'].append(('single', + "%s (enum values); %s" % (m[1], m[0]), + targetname, targetname)) + + def handle_signature(self, sig, signode): + m = sig.split() + name = m[0] + signode += addnodes.desc_name("", name) + return name + +class BroIdentifier(BroGeneric): + def get_index_text(self, objectname, name): + return name + +class BroAttribute(BroGeneric): + def get_index_text(self, objectname, name): + return _('%s (attribute)') % (name) + +class BroDomain(Domain): + """Bro domain.""" + name = 'bro' + label = 'Bro' + + object_types = { + 'type': ObjType(l_('type'), 'type'), + 'namespace': ObjType(l_('namespace'), 'namespace'), + 'id': ObjType(l_('id'), 'id'), + 'enum': ObjType(l_('enum'), 'enum'), + 'attr': ObjType(l_('attr'), 'attr'), + } + + directives = { + 'type': BroGeneric, + 'namespace': BroNamespace, + 'id': BroIdentifier, + 'enum': BroEnum, + 'attr': BroAttribute, + } + + roles = { + 'type': XRefRole(), + 'namespace': XRefRole(), + 'id': XRefRole(), + 'enum': XRefRole(), + 'attr': XRefRole(), + } + + initial_data = { + 'objects': {}, # fullname -> docname, objtype + } + + def clear_doc(self, docname): + for (typ, name), doc in self.data['objects'].items(): + if doc == docname: + del self.data['objects'][typ, name] + + def resolve_xref(self, env, fromdocname, builder, typ, target, node, + contnode): + objects = self.data['objects'] + objtypes = self.objtypes_for_role(typ) + for objtype in objtypes: + if (objtype, target) in objects: + return make_refnode(builder, fromdocname, + objects[objtype, target], + objtype + '-' + target, + contnode, target + ' ' + objtype) + + def get_objects(self): + for (typ, name), docname in self.data['objects'].iteritems(): + yield name, name, typ, docname, typ + '-' + name, 1 diff --git a/doc/source/index.rst b/doc/source/index.rst new file mode 100644 index 0000000000..ac74c36d4d --- /dev/null +++ b/doc/source/index.rst @@ -0,0 +1,20 @@ +.. Bro documentation master file + +Welcome to Bro's documentation! +=============================== + +Contents: + +.. toctree:: + :maxdepth: 1 + :glob: + + common + builtins + policy/index + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`search` diff --git a/doc/source/policy/index.rst b/doc/source/policy/index.rst new file mode 100644 index 0000000000..46345a67af --- /dev/null +++ b/doc/source/policy/index.rst @@ -0,0 +1,8 @@ +Policy Files +============ + +.. toctree:: + :maxdepth: 1 + :glob: + + * From 7593d4d3689decb8d1229066e13dc59352e26651 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Wed, 6 Apr 2011 12:08:23 -0500 Subject: [PATCH 34/47] Don't warn about generating reST script documentation for stdin --- src/BroDoc.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/BroDoc.cc b/src/BroDoc.cc index 722a8861cd..ee1f290154 100644 --- a/src/BroDoc.cc +++ b/src/BroDoc.cc @@ -17,7 +17,7 @@ BroDoc::BroDoc(const std::string& sourcename) std::string ext = source_filename.substr(ext_pos + 1); if ( ext_pos == std::string::npos || ext != "bro" ) { - if ( source_filename != "bro.init" ) + if ( source_filename != "bro.init" && source_filename != "" ) { fprintf(stderr, "Warning: documenting file without .bro extension: %s\n", From e4e7a26ba198b8d58cdaa10c2e9679066534781e Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Wed, 6 Apr 2011 12:12:21 -0500 Subject: [PATCH 35/47] Add :download: role to reST docs for linking to original script source files. This role allows Sphinx to copy non-reST files in the source tree into the output tree. --- src/BroDoc.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/BroDoc.cc b/src/BroDoc.cc index ee1f290154..226e3e80a6 100644 --- a/src/BroDoc.cc +++ b/src/BroDoc.cc @@ -100,7 +100,8 @@ void BroDoc::WriteDocFile() const WriteSectionHeading(source_filename.c_str(), '='); - WriteToDoc("\n`Original Source File <%s>`_\n\n", source_filename.c_str()); + WriteToDoc("\n:download:`Original Source File <%s>`\n\n", + source_filename.c_str()); WriteSectionHeading("Overview", '-'); WriteStringList("%s\n", "%s\n\n", summary); From f3b1a6bb9e33267f6758085450a811811a447f53 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Wed, 6 Apr 2011 14:38:35 -0500 Subject: [PATCH 36/47] Implementing capability to logically group generated policy script docs --- doc/scripts/generate_reST_docs.py.in | 108 ++++++++++++++++++--------- doc/source/builtins.rst | 4 +- doc/source/index.rst | 4 + doc/source/policy/bifs.rst | 4 + doc/source/policy/default.rst | 3 + doc/source/policy/index.rst | 6 +- doc/source/policy/internal.rst | 3 + doc/source/policy/user.rst | 3 + 8 files changed, 96 insertions(+), 39 deletions(-) create mode 100644 doc/source/policy/bifs.rst create mode 100644 doc/source/policy/default.rst create mode 100644 doc/source/policy/internal.rst create mode 100644 doc/source/policy/user.rst diff --git a/doc/scripts/generate_reST_docs.py.in b/doc/scripts/generate_reST_docs.py.in index 1d83cc01fb..25a2cd19b6 100755 --- a/doc/scripts/generate_reST_docs.py.in +++ b/doc/scripts/generate_reST_docs.py.in @@ -5,6 +5,7 @@ import subprocess import shutil import glob import string +import sys BRO = "@CMAKE_BINARY_DIR@/src/bro" BROPATHDEV = "`@CMAKE_BINARY_DIR@/bro-path-dev`" @@ -14,13 +15,15 @@ BROPATH = subprocess.Popen("@CMAKE_BINARY_DIR@/bro-path-dev", shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT).stdout.readline() class BroToReST: - bro_src_file = "" - doc_src_file = "" + bro_src_file = None + doc_src_file = None load_via_stdin = False + group = None - def __init__(self, src_file, load_method=False, src_dir="@POLICY_SRC_DIR@"): - self.bro_src_file = os.path.join(src_dir, src_file) + def __init__(self, src_file, load_method=False, search_dir=None, group=None): + self.bro_src_file = FindBroScript(src_file, search_dir) self.load_via_stdin = load_method + self.group = group # formulate doc_src_file from src_file filename = os.path.basename(src_file) @@ -34,7 +37,8 @@ class BroToReST: return "bro_src_file: " + self.bro_src_file \ + "\ndoc_src_file: " + self.doc_src_file \ + "\ndoc_dst_file: " + os.path.join(DOC_DST_DIR, self.doc_src_file) \ - + "\nstdin_load: %s" % self.load_via_stdin + + "\nstdin_load: %s" % self.load_via_stdin \ + + "\ngroup: %s" % self.group def GenDoc(self): bro_src_basename = os.path.basename(self.bro_src_file) @@ -49,56 +53,86 @@ class BroToReST: if p.wait() == 0: shutil.copy(self.doc_src_file, DOC_DST_DIR) shutil.copy(self.bro_src_file, DOC_DST_DIR) + AppendToDocGroup(self.group, self.bro_src_file, self.doc_src_file) for leftover in glob.glob("*.rst"): os.remove(leftover) -def GenDocs(doc_list, load_method=False): - for f in doc_list: - doc = BroToReST(f, load_method) - print "Generating reST document for " + f +def GenDocs(doc_dict, load_method=False): + for k, v in doc_dict.iteritems(): + doc = BroToReST(k, load_method, group=v) + print "Generating reST document for " + k doc.GenDoc() # search BROPATH for the script and return the absolute path to it -def FindBroScript(src_file): - for path in string.split(BROPATH, ":"): +def FindBroScript(src_file, search_dir=None): + if search_dir is None: + search_dir = string.split(BROPATH, ":") + for path in search_dir: abs_path = os.path.join(path, src_file) if os.path.exists(abs_path): return abs_path + print >> sys.stderr, "Couldn't find '%s'" % src_file + return None + +def AppendToDocGroup(group, src_file, doc_file): + if group is None: + return + + group_file = os.path.join(DOC_DST_DIR, group + ".rst") + if not os.path.exists(group_file): + print >> sys.stderr, "Group file doesn't exist: " + group_file + return + + summary_comments = [] + + with open(src_file, 'r') as f: + for line in f: + sum_pos = string.find(line, "##!") + if sum_pos != -1: + summary_comments.append(line[(sum_pos+3):]) + + doc_name, ext = os.path.splitext(doc_file) + + with open(group_file, 'a') as f: + f.write("\n:doc:`%s`\n" % doc_name) + for line in summary_comments: + f.write(line) # Scripts that can be loaded by bro via command line argument # TODO: generate docs for more scripts -docs = [ - "alarm.bro", - "arp.bro", - "conn.bro", - "dhcp.bro", - "dns.bro", - "ftp.bro", - "http.bro", - "http-reply.bro", - "http-request.bro", - "irc.bro", - "smtp.bro", - "ssl.bro", - "ssl-ciphers.bro", - "ssl-errors.bro", - "synflood.bro", - "tcp.bro", - "udp.bro", - "weird.bro", -] +# TODO: the groups are just made up to test the functionality, fix them +docs = { + "alarm.bro": "internal", + "arp.bro": "user", + "conn.bro": "internal", + "dhcp.bro": "user", + "dns.bro": "user", + "ftp.bro": "user", + "http.bro": "user", + "http-reply.bro": None, + "http-request.bro": None, + "irc.bro": "user", + "smtp.bro": "user", + "ssl.bro": "user", + "ssl-ciphers.bro": None, + "ssl-errors.bro": None, + "synflood.bro": "user", + "tcp.bro": "user", + "udp.bro": "user", + "weird.bro": "internal", +} # Scripts that can't be loaded by bro via command line argument (possible # due to dependency issues), but can be loaded via an @load on stdin -stdin_docs = [ - "notice.bro", -] +stdin_docs = { + "notice.bro": "internal", +} GenDocs(docs) GenDocs(stdin_docs, True) -BroToReST("example.bro", False, "@PROJECT_SOURCE_DIR@/doc").GenDoc() +BroToReST("example.bro", False, ["@PROJECT_SOURCE_DIR@/doc"], group="internal").GenDoc() # Generate documentation for stuff that's always loaded into bro by default cmd = "echo '' | %s %s" % (BRO, BRO_ARGS) @@ -118,4 +152,8 @@ if p.wait() == 0: src_file = FindBroScript(src_file) shutil.copy(src_file, DOC_DST_DIR) shutil.copy(doc, DOC_DST_DIR) + if ext == ".bif": + AppendToDocGroup("bifs", src_file, doc) + else: + AppendToDocGroup("default", src_file, doc) os.remove(doc) diff --git a/doc/source/builtins.rst b/doc/source/builtins.rst index 886edab7c7..9c71da61bf 100644 --- a/doc/source/builtins.rst +++ b/doc/source/builtins.rst @@ -1,5 +1,5 @@ -Builtins -======== +Builtin Types and Attributes +============================ Types ----- diff --git a/doc/source/index.rst b/doc/source/index.rst index ac74c36d4d..5f4b8eb6e0 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -11,6 +11,10 @@ Contents: common builtins + policy/default + policy/user + policy/bifs + policy/internal policy/index Indices and tables diff --git a/doc/source/policy/bifs.rst b/doc/source/policy/bifs.rst new file mode 100644 index 0000000000..0c40404058 --- /dev/null +++ b/doc/source/policy/bifs.rst @@ -0,0 +1,4 @@ +Built-In Functions (BIFs) +========================= + +Here's a list of all documentation for BIFs that Bro provides: diff --git a/doc/source/policy/default.rst b/doc/source/policy/default.rst new file mode 100644 index 0000000000..6b79f20c79 --- /dev/null +++ b/doc/source/policy/default.rst @@ -0,0 +1,3 @@ +Bro Scripts Loaded by Default +============================= + diff --git a/doc/source/policy/index.rst b/doc/source/policy/index.rst index 46345a67af..2202eb8cea 100644 --- a/doc/source/policy/index.rst +++ b/doc/source/policy/index.rst @@ -1,5 +1,7 @@ -Policy Files -============ +Index of All Policy Script Documentation +======================================== + +Contents: .. toctree:: :maxdepth: 1 diff --git a/doc/source/policy/internal.rst b/doc/source/policy/internal.rst new file mode 100644 index 0000000000..864ee75f8a --- /dev/null +++ b/doc/source/policy/internal.rst @@ -0,0 +1,3 @@ +Internal Policy Scripts +======================= + diff --git a/doc/source/policy/user.rst b/doc/source/policy/user.rst new file mode 100644 index 0000000000..7a3bba29cb --- /dev/null +++ b/doc/source/policy/user.rst @@ -0,0 +1,3 @@ +User-Facing Policy Scripts +========================== + From b8f6c5bc7db70f9b8b5d3d7d0cc10108814452a4 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Wed, 6 Apr 2011 16:36:14 -0500 Subject: [PATCH 37/47] Improving documention for the Bro script document-generation process Some minor organizational revisions to the python scripting. --- Makefile | 7 ++ doc/CMakeLists.txt | 4 +- doc/README | 45 +++++++- doc/scripts/BroToReST.py.in | 152 +++++++++++++++++++++++++++ doc/scripts/generate_reST_docs.py.in | 102 ++---------------- 5 files changed, 213 insertions(+), 97 deletions(-) create mode 100755 doc/scripts/BroToReST.py.in diff --git a/Makefile b/Makefile index 8bf528dd26..d2620b422c 100644 --- a/Makefile +++ b/Makefile @@ -23,6 +23,13 @@ install: configured clean: configured ( cd $(BUILD) && make clean ) + ( cd $(BUILD) && make doc-clean ) + +doc: configured + ( cd $(BUILD) && make doc ) + +doc-clean: configured + ( cd $(BUILD) && make doc-clean ) dist: cmake_version # Minimum Bro source package diff --git a/doc/CMakeLists.txt b/doc/CMakeLists.txt index a5a97d5c3b..47c03d6cb6 100644 --- a/doc/CMakeLists.txt +++ b/doc/CMakeLists.txt @@ -11,6 +11,9 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/conf.py.in configure_file(${CMAKE_CURRENT_SOURCE_DIR}/scripts/generate_reST_docs.py.in ${CMAKE_CURRENT_BINARY_DIR}/generate_reST_docs.py @ONLY) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/scripts/BroToReST.py.in + ${CMAKE_CURRENT_BINARY_DIR}/BroToReST.py + @ONLY) add_custom_target(doc COMMAND "${CMAKE_COMMAND}" -E copy_directory @@ -30,7 +33,6 @@ add_custom_target(doc SOURCES ${DOC_SOURCES}) add_dependencies(doc bro) -# TODO: add dependency that's a check for `python`, `sphinx-build`, etc. add_custom_target(doc-clean COMMAND "${CMAKE_COMMAND}" -E remove_directory diff --git a/doc/README b/doc/README index 2a34ea6bdf..150260ed09 100644 --- a/doc/README +++ b/doc/README @@ -1 +1,44 @@ -TODO. +This directory contains scripts and templates that can be used to automate +the generation of Bro script documentation. Two build targets are defined +by CMake: + +``make doc`` + + This target depends on a Python interpreter (>=2.5) and + `Sphinx `_ being installed. Sphinx can be + installed like:: + + > sudo easy_install sphinx + + This target will also first build the bro binary if it is not already + since the generation of reStructuredText (reST) documentation from + Bro scripts is integrated within the parsing process. + + After completion, HTML documentation can be located inside the CMake + ``build/`` directory as ``build/doc/out/html``. The generated reST + documentation will be located in ``build/doc/source/policy``. + +``make doc-clean`` + + This target removes Sphinx inputs and outputs from the CMake ``build/`` dir. + +To schedule a script to be documented, edit ``scripts/generate_reST_docs.py.in`` +and try adding the name of the script along with an optional script group to +the ``docs`` dictionary. That python script also shows other, more specialized +methods for generating documentation for some types of corner-cases. + +When adding a new logical grouping for generated scripts, create a new +reST document in ``source/policy/.rst`` and add some default +documentation. References to (and summaries of) documents associated with +the group get appended to this file during the ``make doc`` process. + +The Sphinx source tree template in ``source/`` can be modified to add more +common/general documentation, style sheets, JavaScript, etc. The Sphinx +config file is produced from ``conf.py.in``, so that can be edited to change +various Sphinx options, like setting the default HTML rendering theme. +There is also a custom Sphinx domain implemented in ``source/ext/bro.py`` +which adds some reST directives and roles that aid in generating useful +index entries and cross-references. + +See ``example.bro`` for an example of how to document a Bro script such that +``make doc`` will be able to produce reST/HTML documentation for it. diff --git a/doc/scripts/BroToReST.py.in b/doc/scripts/BroToReST.py.in new file mode 100755 index 0000000000..89a8e1cc57 --- /dev/null +++ b/doc/scripts/BroToReST.py.in @@ -0,0 +1,152 @@ +#! /usr/bin/env python + +import os +import subprocess +import shutil +import glob +import string +import sys + +BRO = "@CMAKE_BINARY_DIR@/src/bro" +BROPATHDEV = "`@CMAKE_BINARY_DIR@/bro-path-dev`" +BRO_ARGS = "--doc-scripts" +DOC_DST_DIR = "@DOC_SOURCE_WORKDIR@/policy" +BROPATH = subprocess.Popen("@CMAKE_BINARY_DIR@/bro-path-dev", shell=True, + stdout=subprocess.PIPE, stderr=subprocess.STDOUT).stdout.readline() + +class BroToReST: + """A class to encapsulate the the generation of reST documentation from + a given Bro script. + """ + + bro_src_file = None + doc_src_file = None + load_via_stdin = False + group = None + + def __init__(self, src_file, load_method=False, search_dir=None, group=None): + """ + :param src_file: the file name of a Bro script (not a path) + :param load_method: T if script must be loaded by Bro via a stdin + redirection of "@load