diff --git a/src/BroBifDoc.cc b/src/BroBifDoc.cc new file mode 100644 index 0000000000..9d358eb04b --- /dev/null +++ b/src/BroBifDoc.cc @@ -0,0 +1,16 @@ +#include +#include +#include + +#include "BroDoc.h" +#include "BroBifDoc.h" + +BroBifDoc::BroBifDoc(const std::string& sourcename) : BroDoc(sourcename) + { + } + +//TODO: this needs to do something different than parent class's version +void BroBifDoc::WriteDocFile() const + { + BroDoc::WriteDocFile(); + } diff --git a/src/BroBifDoc.h b/src/BroBifDoc.h new file mode 100644 index 0000000000..ef2dc5e2d7 --- /dev/null +++ b/src/BroBifDoc.h @@ -0,0 +1,21 @@ +#ifndef brobifdoc_h +#define brobifdoc_h + +#include +#include +#include + +#include "BroDoc.h" + +class BroBifDoc : public BroDoc { +public: + BroBifDoc(const std::string& sourcename); + + void WriteDocFile() const; + +protected: + +private: +}; + +#endif diff --git a/src/BroDoc.cc b/src/BroDoc.cc new file mode 100644 index 0000000000..fb06a32308 --- /dev/null +++ b/src/BroDoc.cc @@ -0,0 +1,163 @@ +#include +#include +#include +#include + +#include "BroDoc.h" +#include "BroDocObj.h" + +BroDoc::BroDoc(const std::string& sourcename) + { + fprintf(stdout, "Documenting source: %s\n", sourcename.c_str()); + + source_filename = sourcename.substr(sourcename.find_last_of('/') + 1); + + size_t ext_pos = source_filename.find_last_of('.'); + std::string ext = source_filename.substr(ext_pos + 1); + if ( ext_pos == std::string::npos || ext != "bro" ) + { + if ( source_filename != "bro.init" ) + { + fprintf(stderr, + "Warning: documenting file without .bro extension: %s\n", + sourcename.c_str()); + } + else + { + // Force the reST documentation file to be "bro.init.rst" + ext_pos = std::string::npos; + } + } + + reST_filename = source_filename.substr(0, ext_pos); + reST_filename += ".rst"; + reST_file = fopen(reST_filename.c_str(), "w"); + + if ( ! reST_file ) + fprintf(stderr, "Failed to open %s", reST_filename.c_str()); + else + fprintf(stdout, "Created reST document: %s\n", reST_filename.c_str()); + } + +BroDoc::~BroDoc() + { + if ( reST_file ) + if ( fclose( reST_file ) ) + fprintf(stderr, "Failed to close %s", reST_filename.c_str()); + FreeBroDocObjPtrList(options); + FreeBroDocObjPtrList(state_vars); + FreeBroDocObjPtrList(types); + FreeBroDocObjPtrList(notices); + FreeBroDocObjPtrList(events); + FreeBroDocObjPtrList(functions); + } + +void BroDoc::SetPacketFilter(const std::string& s) + { + packet_filter = s; + size_t pos1 = s.find("{\n"); + size_t pos2 = s.find("}"); + if ( pos1 != std::string::npos && pos2 != std::string::npos ) + packet_filter = s.substr(pos1 + 2, pos2 - 2); + } + +void BroDoc::WriteDocFile() const + { + WriteToDoc("%s\n", source_filename.c_str()); + for ( size_t i = 0; i < source_filename.length(); ++i ) + WriteToDoc("="); + WriteToDoc("\n\n"); + + WriteSectionHeading("Summary", '-'); + WriteStringList("%s\n", "%s\n\n", summary); + + WriteToDoc(":Author: %s\n", author_name.c_str()); + + WriteToDoc(":Namespaces: "); + WriteStringList("`%s`, ", "`%s`\n", modules); + + WriteToDoc(":Imports:\n"); + WriteStringList(" :bro:script: `%s`\n", + " :bro:script: `%s`\n\n", imports); + + WriteSectionHeading("Public Interface", '-'); + WriteBroDocObjList(options, true, "Options", '~'); + WriteBroDocObjList(state_vars, true, "State Variables", '~'); + WriteBroDocObjList(types, true, "Types", '~'); + WriteBroDocObjList(notices, true, "Notices", '~'); + WriteBroDocObjList(events, true, "Events", '~'); + WriteBroDocObjList(functions, true, "Functions", '~'); + + WriteSectionHeading("Packet Filter", '-'); + WriteToDoc("%s\n", packet_filter.c_str()); + + WriteSectionHeading("Private Interface", '-'); + WriteBroDocObjList(options, false, "Options", '~'); + WriteBroDocObjList(state_vars, false, "State Variables", '~'); + WriteBroDocObjList(types, false, "Types", '~'); + WriteBroDocObjList(notices, false, "Notices", '~'); + WriteBroDocObjList(events, false, "Events", '~'); + WriteBroDocObjList(functions, false, "Functions", '~'); + } + +void BroDoc::WriteStringList(const char* format, + const char* last_format, + const std::list& l) const + { + if ( l.empty() ) return; + std::list::const_iterator it; + std::list::const_iterator last = l.end(); + last--; + for ( it = l.begin(); it != last; ++it ) + WriteToDoc(format, it->c_str()); + WriteToDoc(last_format, last->c_str()); + } + +void BroDoc::WriteBroDocObjList(const std::list& l, + bool exportCond, + const char* heading, + char underline) const + { + WriteSectionHeading(heading, underline); + std::list::const_iterator it; + for ( it = l.begin(); it != l.end(); ++it ) + { + if ( exportCond ) + { + // write out only those in an export section + if ( (*it)->IsPublicAPI() ) + (*it)->WriteReST(reST_file); + } + else + { + // write out only those that have comments and are not exported + if ( !(*it)->IsPublicAPI() && (*it)->HasDocumentation() ) + (*it)->WriteReST(reST_file); + } + } + } + +void BroDoc::WriteToDoc(const char* format, ...) const + { + va_list argp; + va_start(argp, format); + vfprintf(reST_file, format, argp); + va_end(argp); + } + +void BroDoc::WriteSectionHeading(const char* heading, char underline) const + { + WriteToDoc("%s\n", heading); + size_t len = strlen(heading); + for ( size_t i = 0; i < len; ++i ) + WriteToDoc("%c", underline); + WriteToDoc("\n"); + } + +void BroDoc::FreeBroDocObjPtrList(std::list& l) + { + std::list::iterator it; + for ( it = l.begin(); it != l.end(); ++it ) + delete *it; + l.clear(); + } diff --git a/src/BroDoc.h b/src/BroDoc.h new file mode 100644 index 0000000000..683a4cd5a5 --- /dev/null +++ b/src/BroDoc.h @@ -0,0 +1,183 @@ +#ifndef brodoc_h +#define brodoc_h + +#include +#include +#include +#include + +#include "BroDocObj.h" + +class BroDoc { +public: + /** + * BroDoc constructor + * Given a Bro script, opens new file in the current working directory + * that will contain reST documentation generated from the parsing + * of the Bro script. The new reST file will be named similar to + * the filename of the Bro script that generates it, except any + * ".bro" file extension is stripped and ".rst" takes it place. + * If the filename doesn't end in ".bro", then ".rst" is just appended. + * @param sourcename The name of the Bro script for which to generate + * documentation. May contain a path. + */ + BroDoc(const std::string& sourcename); + + /** + * BroDoc destructor + * Closes the file that was opened by the constructor and frees up + * memory taken by BroDocObj objects. + */ + ~BroDoc(); + + /** + * Write out full reST documentation for the Bro script that was parsed. + * BroDoc's default implementation of this function will care + * about whether declarations made in the Bro script are part of + * the public versus private interface (whether things are declared in + * the export section). Things in a script's export section make it + * into the reST output regardless of whether they have ## comments + * but things outside the export section are only output into the reST + * if they have ## comments. + */ + virtual void WriteDocFile() const; + + /** + * Schedules some summarizing text to be output directly into the reST doc. + * This should be called whenever the scanner sees a line in the Bro script + * starting with "##!" + * @param s The summary text to add to the reST doc. + */ + void AddSummary(const std::string& s) { summary.push_back(s); } + + /** + * Schedules an import (@load) to be documented. + * This should be called whenever the scanner sees an @load. + * @param s The name of the imported script. + */ + void AddImport(const std::string& s) { imports.push_back(s); } + + /** + * Schedules a namespace (module) to be documented. + * This should be called whenever the parser sees a TOK_MODULE. + * @param s The namespace (module) identifier's name. + */ + void AddModule(const std::string& s) { modules.push_back(s); } + + /** + * Sets the way the script changes the "capture_filters" table. + * This is determined by the scanner checking for changes to + * the "capture_filters" table after each of Bro's input scripts + * (given as command line arguments to Bro) are finished being parsed. + * @param s The value "capture_filters" as given by TableVal::Describe() + */ + void SetPacketFilter(const std::string& s); + + /** + * Sets the author of the script. + * The scanner should call this when it sees "## Author: ..." + * @param s The name, email, etc. of the script author(s). Must be + * all on one line. + */ + void SetAuthor(const std::string& s) { author_name = s; } + + //TODO: document these better + // the rest of these need to be called from the parser + void AddOption(const BroDocObj* o) { options.push_back(o); } + void AddStateVar(const BroDocObj* o) { state_vars.push_back(o); } + void AddType(const BroDocObj* o) { types.push_back(o); } + void AddNotice(const BroDocObj* o) { notices.push_back(o); } + void AddEvent(const BroDocObj* o) { events.push_back(o); } + void AddFunction(const BroDocObj* o) { functions.push_back(o); } + + /** + * Gets the name of the Bro script source file for which reST + * documentation is being generated. + * @return A char* to the start of the source file's name. + */ + const char* GetSourceFileName() const { return source_filename.c_str(); } + + /** + * Gets the name of the generated reST documentation file. + * @return A char* to the start of the generated reST file's name. + */ + const char* GetOutputFileName() const { return reST_filename.c_str(); } + +protected: + FILE* reST_file; + std::string reST_filename; + std::string source_filename; + std::string author_name; + std::string packet_filter; + + std::list ls; + std::list modules; + std::list summary; + std::list imports; + + std::list options; // identifiers with &redef attr + std::list state_vars; // identifiers without &redef? + std::list types; + std::list notices; + std::list events; + std::list functions; + + /** + * Writes out a list of strings to the reST document. + * @param format A printf style format string for elements of the list + * except for the last one in the list + * @param last_format A printf style format string to use for the last + * element of the list + * @param l A reference to a list of strings + */ + void WriteStringList(const char* format, + const char* last_format, + const std::list& l) const; + + /** + * @see WriteStringList(const char*, const char*, + const std::list&>) + */ + void WriteStringList(const char* format, + const std::list& l) const + { WriteStringList(format, format, l); } + + /** + * Writes out a list of BroDocObj objects to the reST document + * @param l A list of BroDocObj pointers + * @param exportCond If true, filter out objects that are not in an + * export section. If false, filter out those that are in + * an export section. + * @param heading The title of the section to create in the reST doc. + * @param underline The character to use to underline the reST + * section heading. + */ + void WriteBroDocObjList(const std::list& l, + bool exportCond, + const char* heading, + char underline) const; + + /** + * A wrapper to fprintf() that always uses the reST document + * for the FILE* argument. + * @param format A printf style format string. + */ + void WriteToDoc(const char* format, ...) const; + + /** + * Writes out a reST section heading + * @param heading The title of the heading to create + * @param underline The character to use to underline the section title + within the reST document + */ + void WriteSectionHeading(const char* heading, char underline) const; +private: + + /** + * Frees memory allocated to BroDocObj's objects in a given list. + * @param a reference to a list of BroDocObj pointers + */ + void FreeBroDocObjPtrList(std::list& l); +}; + +#endif diff --git a/src/BroDocObj.cc b/src/BroDocObj.cc new file mode 100644 index 0000000000..645c4f1475 --- /dev/null +++ b/src/BroDocObj.cc @@ -0,0 +1,35 @@ +#include +#include +#include +#include "Obj.h" +#include "BroDocObj.h" + +BroDocObj::BroDocObj(const BroObj* obj, + std::list*& reST, + bool exported) + { + broObj = obj; + isExported = exported; + reST_doc_strings = reST; + reST = 0; + } + +BroDocObj::~BroDocObj() + { + delete reST_doc_strings; + } + +void BroDocObj::WriteReST(FILE* file) const + { + if ( reST_doc_strings ) + { + std::list::const_iterator it; + for ( it = reST_doc_strings->begin(); + it != reST_doc_strings->end(); ++it) + fprintf(file, "%s\n", it->c_str()); + } + + ODesc desc; + broObj->Describe(&desc); + fprintf(file, "%s\n", desc.Description()); + } diff --git a/src/BroDocObj.h b/src/BroDocObj.h new file mode 100644 index 0000000000..42bfc8882f --- /dev/null +++ b/src/BroDocObj.h @@ -0,0 +1,72 @@ +#ifndef brodocobj_h +#define brodocobj_h + +#include +#include +#include + +#include "Obj.h" + +class BroDocObj { +public: + /** + * BroDocObj constructor + * @param obj a pointer to a BroObj that is to be documented + * @param reST a reference to a pointer of a list of strings that + represent the reST documentation for the BroObj. The pointer + will be set to 0 after this constructor finishes. + * @param exported whether the BroObj is declared in an export section + */ + BroDocObj(const BroObj* obj, std::list*& reST, bool exported); + + /** + * BroDocObj destructor + * Deallocates the memory associated with the list of reST strings + */ + ~BroDocObj(); + + /** + * writes the reST representation of this object which includes + * 1) any of the "##" comments (stored internally in reST_doc_string) + * To make things easy, I think we should assume that the documenter + * writes their comments such that anything after ## is valid reST + * so that at parse time the ## is just stripped and the remainder + * is scheduled to be inserted as-is into the reST. + * TODO: prepare for some kind of filtering mechanism that transforms + * the reST as written into new reST before being written out. + * This allows for additional custom markup or macros when writing + * pure reST might be inconvenient. + * 2) a reST friendly description of the BroObj + * Could be implemented similar to the virtual BroObj::Describe(ODesc) + * E.g. all subclasses will now need to implement a reSTDescribe(ODesc) + * so that they can describe themselves in terms of the custom reST + * directives/roles that we'll later teach to Sphinx via a "bro domain". + * ID's should be able to implement the reSTDescribe(ODesc) function + * such that their namespace and attributes are output as well. + * @param The (already opened) file to write the reST to. + */ + void WriteReST(FILE* file) const; + + /** + * Check whether this documentation is part of the public API + * (The BroObj declaration was while in an export section). + * @return true if the BroObj was declared in an export section, else false + */ + bool IsPublicAPI() const { return isExported; } + + /** + * Return whether this object has documentation (## comments) + * @return true if the BroObj has comments associated with it + */ + bool HasDocumentation() const { return reST_doc_strings && + reST_doc_strings->size() > 0; } + +protected: + std::list* reST_doc_strings; + const BroObj* broObj; + bool isExported; + +private: +}; + +#endif diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 0f67dc173e..2a4a8e6dee 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -247,6 +247,9 @@ set(bro_SRCS BitTorrent.cc BitTorrentTracker.cc BPF_Program.cc + BroBifDoc.cc + BroDoc.cc + BroDocObj.cc BroString.cc CCL.cc ChunkedIO.cc diff --git a/src/main.cc b/src/main.cc index 82866302fd..38acfaa355 100644 --- a/src/main.cc +++ b/src/main.cc @@ -91,6 +91,7 @@ int optimize = 0; int do_notice_analysis = 0; int rule_bench = 0; int print_loaded_scripts = 0; +int generate_documentation = 0; SecondaryPath* secondary_path = 0; ConnCompressor* conn_compressor = 0; extern char version[]; @@ -145,6 +146,7 @@ void usage() fprintf(stderr, " -h|--help|-? | command line help\n"); fprintf(stderr, " -i|--iface | read from given interface\n"); fprintf(stderr, " -l|--print-scripts | print all loaded scripts\n"); + fprintf(stderr, " -Z|--doc-scripts | generate documentation for all loaded scripts\n"); fprintf(stderr, " -p|--prefix | add given prefix to policy file resolution\n"); fprintf(stderr, " -r|--readfile | read from given tcpdump file\n"); fprintf(stderr, " -y|--flowfile [=] | read from given flow file\n"); @@ -364,6 +366,7 @@ int main(int argc, char** argv) {"help", no_argument, 0, 'h'}, {"iface", required_argument, 0, 'i'}, {"print-scripts", no_argument, 0, 'l'}, + {"doc-scripts", no_argument, 0, 'Z'}, {"prefix", required_argument, 0, 'p'}, {"readfile", required_argument, 0, 'r'}, {"flowfile", required_argument, 0, 'y'}, @@ -440,7 +443,7 @@ int main(int argc, char** argv) opterr = 0; char opts[256]; - safe_strncpy(opts, "A:a:B:D:e:f:I:i:K:n:p:R:r:s:T:t:U:w:x:X:y:Y:z:CFGHLOPSWdghlv", + safe_strncpy(opts, "A:a:B:D:e:f:I:i:K:n:p:R:r:s:T:t:U:w:x:X:y:Y:z:CFGHLOPSWdghlvZ", sizeof(opts)); #ifdef USE_PERFTOOLS @@ -633,6 +636,10 @@ int main(int argc, char** argv) break; #endif + case 'Z': + generate_documentation = 1; + break; + #ifdef USE_IDMEF case 'n': fprintf(stderr, "Using IDMEF XML DTD from %s\n", optarg); diff --git a/src/parse.y b/src/parse.y index 3cf2c07b18..fbbb531050 100644 --- a/src/parse.y +++ b/src/parse.y @@ -73,6 +73,15 @@ #include "DNS.h" #include "RE.h" #include "Scope.h" +#include "BroDoc.h" +#include "BroDocObj.h" + +#include +#include + +extern BroDoc* current_reST_doc; +extern int generate_documentation; +extern std::list* reST_doc_comments; YYLTYPE GetCurrentLocation(); extern int yyerror(const char[]); @@ -785,7 +794,13 @@ formal_args_decl: decl: TOK_MODULE TOK_ID ';' - { current_module = $2; } + { + current_module = $2; + if ( generate_documentation ) + { + current_reST_doc->AddModule(current_module); + } + } | TOK_EXPORT '{' { is_export = true; } decl_list '}' { is_export = false; } diff --git a/src/scan.l b/src/scan.l index 9dc4d828e0..41c9fc08f7 100644 --- a/src/scan.l +++ b/src/scan.l @@ -15,11 +15,16 @@ #include "Debug.h" #include "PolicyFile.h" #include "broparse.h" +#include "BroDoc.h" +#include "BroBifDoc.h" #include +#include +#include extern YYLTYPE yylloc; // holds start line and column of token extern int print_loaded_scripts; +extern int generate_documentation; int nwarn = 0; int nerr = 0; @@ -35,6 +40,8 @@ int_list if_stack; int line_number = 1; int include_level = 0; const char* filename = 0; +BroDoc* current_reST_doc = 0; +static BroDoc* last_reST_doc = 0; char last_tok[128]; @@ -50,6 +57,21 @@ char last_tok[128]; // Files we have already scanned (or are in the process of scanning). static PList(char) files_scanned; +// reST documents that we've created (or have at least opened so far) +static std::list docs_generated; + +// reST comments (those starting with ##) seen so far +std::list* reST_doc_comments = 0; + +// print current contents of reST_doc_comments list to stderr +void print_current_reST_doc_comments(); + +// delete the reST_doc_comments list object +void clear_reST_doc_comments(); + +// adds changes to capture_filter to the current script's reST documentation +static void check_capture_filter_changes(); + class FileInfo { public: FileInfo(string restore_module = ""); @@ -60,6 +82,7 @@ public: const char* name; int line; int level; + BroDoc* doc; }; // A stack of input buffers we're scanning. file_stack[len-1] is the @@ -102,6 +125,30 @@ ESCSEQ (\\([^\n]|[0-7]+|x[[:xdigit:]]+)) %% +##!.* { + // Add this format of comments to the script documentation's "summary" + if ( generate_documentation ) + current_reST_doc->AddSummary(yytext + 3); + } + +##{OWS}Author:.* { + if ( generate_documentation ) + current_reST_doc->SetAuthor( + skip_whitespace( // Skip whitespace after "Author:" + skip_whitespace(yytext + 2) // Skip whitespace after ## + + 7) // Skip "Author:" + ); + } + +##[^#\n].* { + if ( generate_documentation ) + { + if ( ! reST_doc_comments ) + reST_doc_comments = new std::list(); + reST_doc_comments->push_back(yytext + 2); + } +} + #.* /* eat comments */ {WS} /* eat whitespace */ @@ -211,6 +258,17 @@ when return TOK_WHEN; @load{WS}{FILE} { const char* new_file = skip_whitespace(yytext + 5); // Skip "@load". + if ( generate_documentation ) + { + current_reST_doc->AddImport(new_file); + if ( reST_doc_comments ) + { + fprintf(stderr, "Warning: unconsumed reST documentation is being " + "discarded before doing '@load %s' in %s:\n", + new_file, current_reST_doc->GetSourceFileName()); + clear_reST_doc_comments(); + } + } (void) load_files_with_prefix(new_file); } @@ -446,7 +504,6 @@ static int load_files_with_prefix(const char* orig_file) strcpy(new_filename, file); f = search_for_file(new_filename, "bro", &full_filename); - delete [] new_filename; } @@ -496,6 +553,21 @@ static int load_files_with_prefix(const char* orig_file) // Don't delete the old filename - it's pointed to by // every BroObj created when parsing it. yylloc.filename = filename = full_filename; + + if ( generate_documentation ) + { + char* bifExtStart = strstr(full_filename, ".bif.bro"); + BroDoc* reST_doc; + + if ( bifExtStart ) + reST_doc = new BroBifDoc(full_filename); + else + reST_doc = new BroDoc(full_filename); + + docs_generated.push_back(reST_doc); + + current_reST_doc = reST_doc; + } } else @@ -655,6 +727,7 @@ int yywrap() // Stack is now empty. while ( input_files.length() > 0 ) { + check_capture_filter_changes(); if ( load_files_with_prefix(input_files[0]) ) { // Don't delete the filename - it's pointed to by @@ -667,6 +740,7 @@ int yywrap() // if any. (void) input_files.remove_nth(0); } + check_capture_filter_changes(); // Add redef statements for any X=Y command line parameters. if ( params.size() > 0 ) @@ -731,6 +805,18 @@ int yywrap() return 0; } + if ( generate_documentation ) + { + std::list::iterator it; + for ( it = docs_generated.begin(); it != docs_generated.end(); ++it ) + { + (*it)->WriteDocFile(); + delete *it; + } + docs_generated.clear(); + clear_reST_doc_comments(); + } + // Otherwise, we are done. return 1; } @@ -742,6 +828,7 @@ FileInfo::FileInfo(string arg_restore_module) name = ::filename; line = ::line_number; level = ::include_level; + doc = ::current_reST_doc; } FileInfo::~FileInfo() @@ -753,6 +840,8 @@ FileInfo::~FileInfo() yylloc.filename = filename = name; yylloc.first_line = yylloc.last_line = line_number = line; include_level = level; + last_reST_doc = current_reST_doc; + current_reST_doc = doc; if ( restore_module != "" ) current_module = restore_module; @@ -779,3 +868,40 @@ static void report_file() files_reported.append(copy_string(filename)); } +static void check_capture_filter_changes() + { + if ( generate_documentation ) + { + // Lookup the "capture_filters" identifier, if it has any + // defined value, add it to the script's reST documentation, + // and finally clear the table so it doesn't taint the + // documentation for subsequent scripts. + + ID* capture_filters = global_scope()->Lookup("capture_filters"); + if ( capture_filters ) + { + ODesc desc; + capture_filters->ID_Val()->Describe(&desc); + last_reST_doc->SetPacketFilter(desc.Description()); + ( (TableVal*) capture_filters->ID_Val() )->RemoveAll(); + } + } + } + +void print_current_reST_doc_comments() + { + if ( ! reST_doc_comments ) return; + std::list::iterator it; + for (it = reST_doc_comments->begin(); it != reST_doc_comments->end(); ++it) + fprintf(stderr, "##%s\n", it->c_str()); + } + +void clear_reST_doc_comments() + { + if ( ! reST_doc_comments ) return; + fprintf(stderr, "Warning: %lu unconsumed reST comments:\n", + reST_doc_comments->size()); + print_current_reST_doc_comments(); + delete reST_doc_comments; + reST_doc_comments = 0; + }