Bro script documentation framework checkpoint

* New bro runtime options: -Z or --doc-scripts enables documentation mode
* New BroDoc, BroBifDoc, and BroDocObj interfaces to support script
  documentation
* Modifications to the bro scanner (scan.l) to get it to keep track of
  which script is being scanned/parsed and which document is being generated
* Modifications to scan.l and the bro parser (parse.y) to produce/consume
  script comments denoted with "##"
* Documentation is currently generated for the following
** Script author
** Script summary
** @load's
** capture_filters
** modules (namespaces)

Most of the remaining framework/infrastructure work should be in extracting
the interesting BroObj objects as the parser sees them and better formatting
the reST documents.

src/BroBifDoc.cc

@@ -0,0 +1,16 @@
+#include <cstdio>
+#include <string>
+#include <list>
+
+#include "BroDoc.h"
+#include "BroBifDoc.h"
+
+BroBifDoc::BroBifDoc(const std::string& sourcename) : BroDoc(sourcename)
+    {
+    }
+
+//TODO: this needs to do something different than parent class's version
+void BroBifDoc::WriteDocFile() const
+    {
+    BroDoc::WriteDocFile();
+    }

src/BroBifDoc.h

@@ -0,0 +1,21 @@
+#ifndef brobifdoc_h
+#define brobifdoc_h
+
+#include <cstdio>
+#include <string>
+#include <list>
+
+#include "BroDoc.h"
+
+class BroBifDoc : public BroDoc {
+public:
+    BroBifDoc(const std::string& sourcename);
+
+    void WriteDocFile() const;
+
+protected:
+
+private:
+};
+
+#endif

src/BroDoc.cc

@@ -0,0 +1,163 @@
+#include <cstdio>
+#include <cstdarg>
+#include <string>
+#include <list>
+
+#include "BroDoc.h"
+#include "BroDocObj.h"
+
+BroDoc::BroDoc(const std::string& sourcename)
+    {
+    fprintf(stdout, "Documenting source: %s\n", sourcename.c_str());
+
+    source_filename = sourcename.substr(sourcename.find_last_of('/') + 1);
+
+    size_t ext_pos = source_filename.find_last_of('.');
+    std::string ext = source_filename.substr(ext_pos + 1);
+    if ( ext_pos == std::string::npos || ext != "bro" )
+        {
+        if ( source_filename != "bro.init" )
+            {
+            fprintf(stderr,
+                "Warning: documenting file without .bro extension: %s\n",
+                sourcename.c_str());
+            }
+        else
+            {
+            // Force the reST documentation file to be "bro.init.rst"
+            ext_pos = std::string::npos;
+            }
+        }
+
+    reST_filename = source_filename.substr(0, ext_pos);
+    reST_filename += ".rst";
+    reST_file = fopen(reST_filename.c_str(), "w");
+
+    if ( ! reST_file )
+        fprintf(stderr, "Failed to open %s", reST_filename.c_str());
+    else
+        fprintf(stdout, "Created reST document: %s\n", reST_filename.c_str());
+    }
+
+BroDoc::~BroDoc()
+    {
+    if ( reST_file )
+        if ( fclose( reST_file ) )
+            fprintf(stderr, "Failed to close %s", reST_filename.c_str());
+    FreeBroDocObjPtrList(options);
+    FreeBroDocObjPtrList(state_vars);
+    FreeBroDocObjPtrList(types);
+    FreeBroDocObjPtrList(notices);
+    FreeBroDocObjPtrList(events);
+    FreeBroDocObjPtrList(functions);
+    }
+
+void BroDoc::SetPacketFilter(const std::string& s)
+    {
+    packet_filter = s;
+    size_t pos1 = s.find("{\n");
+    size_t pos2 = s.find("}");
+    if ( pos1 != std::string::npos && pos2 != std::string::npos )
+        packet_filter = s.substr(pos1 + 2, pos2 - 2);
+    }
+
+void BroDoc::WriteDocFile() const
+    {
+    WriteToDoc("%s\n", source_filename.c_str());
+    for ( size_t i = 0; i < source_filename.length(); ++i )
+        WriteToDoc("=");
+    WriteToDoc("\n\n");
+
+    WriteSectionHeading("Summary", '-');
+    WriteStringList("%s\n", "%s\n\n", summary);
+
+    WriteToDoc(":Author: %s\n", author_name.c_str());
+
+    WriteToDoc(":Namespaces: ");
+    WriteStringList("`%s`, ", "`%s`\n", modules);
+
+    WriteToDoc(":Imports:\n");
+    WriteStringList("    :bro:script: `%s`\n",
+                    "    :bro:script: `%s`\n\n", imports);
+
+    WriteSectionHeading("Public Interface", '-');
+    WriteBroDocObjList(options, true, "Options", '~');
+    WriteBroDocObjList(state_vars, true, "State Variables", '~');
+    WriteBroDocObjList(types, true, "Types", '~');
+    WriteBroDocObjList(notices, true, "Notices", '~');
+    WriteBroDocObjList(events, true, "Events", '~');
+    WriteBroDocObjList(functions, true, "Functions", '~');
+
+    WriteSectionHeading("Packet Filter", '-');
+    WriteToDoc("%s\n", packet_filter.c_str());
+
+    WriteSectionHeading("Private Interface", '-');
+    WriteBroDocObjList(options, false, "Options", '~');
+    WriteBroDocObjList(state_vars, false, "State Variables", '~');
+    WriteBroDocObjList(types, false, "Types", '~');
+    WriteBroDocObjList(notices, false, "Notices", '~');
+    WriteBroDocObjList(events, false, "Events", '~');
+    WriteBroDocObjList(functions, false, "Functions", '~');
+    }
+
+void BroDoc::WriteStringList(const char* format,
+                             const char* last_format,
+                             const std::list<std::string>& l) const
+    {
+    if ( l.empty() ) return;
+    std::list<std::string>::const_iterator it;
+    std::list<std::string>::const_iterator last = l.end();
+    last--;
+    for ( it = l.begin(); it != last; ++it )
+        WriteToDoc(format, it->c_str());
+    WriteToDoc(last_format, last->c_str());
+    }
+
+void BroDoc::WriteBroDocObjList(const std::list<const BroDocObj*>& l,
+                                bool exportCond,
+                                const char* heading,
+                                char underline) const
+    {
+    WriteSectionHeading(heading, underline);
+    std::list<const BroDocObj*>::const_iterator it;
+    for ( it = l.begin(); it != l.end(); ++it )
+        {
+        if ( exportCond )
+            {
+            // write out only those in an export section
+            if ( (*it)->IsPublicAPI() )
+                (*it)->WriteReST(reST_file);
+            }
+        else
+            {
+            // write out only those that have comments and are not exported
+            if ( !(*it)->IsPublicAPI() && (*it)->HasDocumentation() )
+                (*it)->WriteReST(reST_file);
+            }
+        }
+    }
+
+void BroDoc::WriteToDoc(const char* format, ...) const
+    {
+    va_list argp;
+    va_start(argp, format);
+    vfprintf(reST_file, format, argp);
+    va_end(argp);
+    }
+
+void BroDoc::WriteSectionHeading(const char* heading, char underline) const
+    {
+    WriteToDoc("%s\n", heading);
+    size_t len = strlen(heading);
+    for ( size_t i = 0; i < len; ++i )
+        WriteToDoc("%c", underline);
+    WriteToDoc("\n");
+    }
+
+void BroDoc::FreeBroDocObjPtrList(std::list<const BroDocObj*>& l)
+    {
+    std::list<const BroDocObj*>::iterator it;
+    for ( it = l.begin(); it != l.end(); ++it )
+        delete *it;
+    l.clear();
+    }

src/BroDoc.h

@@ -0,0 +1,183 @@
+#ifndef brodoc_h
+#define brodoc_h
+
+#include <cstdio>
+#include <cstdarg>
+#include <string>
+#include <list>
+
+#include "BroDocObj.h"
+
+class BroDoc {
+public:
+    /**
+     * BroDoc constructor
+     * Given a Bro script, opens new file in the current working directory
+     * that will contain reST documentation generated from the parsing
+     * of the Bro script.  The new reST file will be named similar to
+     * the filename of the Bro script that generates it, except any
+     * ".bro" file extension is stripped and ".rst" takes it place.
+     * If the filename doesn't end in ".bro", then ".rst" is just appended.
+     * @param sourcename The name of the Bro script for which to generate
+     *        documentation.  May contain a path.
+     */
+    BroDoc(const std::string& sourcename);
+
+    /**
+     * BroDoc destructor
+     * Closes the file that was opened by the constructor and frees up
+     * memory taken by BroDocObj objects.
+     */
+    ~BroDoc();
+
+    /**
+     * Write out full reST documentation for the Bro script that was parsed.
+     * BroDoc's default implementation of this function will care
+     * about whether declarations made in the Bro script are part of
+     * the public versus private interface (whether things are declared in
+     * the export section).  Things in a script's export section make it
+     * into the reST output regardless of whether they have ## comments
+     * but things outside the export section are only output into the reST
+     * if they have ## comments.
+     */
+    virtual void WriteDocFile() const;
+
+    /**
+     * Schedules some summarizing text to be output directly into the reST doc.
+     * This should be called whenever the scanner sees a line in the Bro script
+     * starting with "##!"
+     * @param s The summary text to add to the reST doc.
+     */
+    void AddSummary(const std::string& s) { summary.push_back(s); }
+
+    /**
+     * Schedules an import (@load) to be documented.
+     * This should be called whenever the scanner sees an @load.
+     * @param s The name of the imported script.
+     */
+    void AddImport(const std::string& s) { imports.push_back(s); }
+
+    /**
+     * Schedules a namespace (module) to be documented.
+     * This should be called whenever the parser sees a TOK_MODULE.
+     * @param s The namespace (module) identifier's name.
+     */
+    void AddModule(const std::string& s) { modules.push_back(s); }
+
+    /**
+     * Sets the way the script changes the "capture_filters" table.
+     * This is determined by the scanner checking for changes to
+     * the "capture_filters" table after each of Bro's input scripts
+     * (given as command line arguments to Bro) are finished being parsed.
+     * @param s The value "capture_filters" as given by TableVal::Describe()
+     */
+    void SetPacketFilter(const std::string& s);
+
+    /**
+     * Sets the author of the script.
+     * The scanner should call this when it sees "## Author: ..."
+     * @param s The name, email, etc. of the script author(s).  Must be
+     *        all on one line.
+     */
+    void SetAuthor(const std::string& s) { author_name = s; }
+
+    //TODO: document these better
+    // the rest of these need to be called from the parser
+    void AddOption(const BroDocObj* o) { options.push_back(o); }
+    void AddStateVar(const BroDocObj* o) { state_vars.push_back(o); }
+    void AddType(const BroDocObj* o) { types.push_back(o); }
+    void AddNotice(const BroDocObj* o) { notices.push_back(o); }
+    void AddEvent(const BroDocObj* o) { events.push_back(o); }
+    void AddFunction(const BroDocObj* o) { functions.push_back(o); }
+
+    /**
+     * Gets the name of the Bro script source file for which reST
+     * documentation is being generated.
+     * @return A char* to the start of the source file's name.
+     */
+    const char* GetSourceFileName() const { return source_filename.c_str(); }
+
+    /**
+     * Gets the name of the generated reST documentation file.
+     * @return A char* to the start of the generated reST file's name.
+     */
+    const char* GetOutputFileName() const { return reST_filename.c_str(); }
+
+protected:
+    FILE* reST_file;
+    std::string reST_filename;
+    std::string source_filename;
+    std::string author_name;
+    std::string packet_filter;
+
+    std::list<std::string> ls;
+    std::list<std::string> modules;
+    std::list<std::string> summary;
+    std::list<std::string> imports;
+
+    std::list<const BroDocObj*> options;    // identifiers with &redef attr
+    std::list<const BroDocObj*> state_vars; // identifiers without &redef?
+    std::list<const BroDocObj*> types;
+    std::list<const BroDocObj*> notices;
+    std::list<const BroDocObj*> events;
+    std::list<const BroDocObj*> functions;
+
+    /**
+     * Writes out a list of strings to the reST document.
+     * @param format A printf style format string for elements of the list
+     *        except for the last one in the list
+     * @param last_format A printf style format string to use for the last
+     *        element of the list
+     * @param l A reference to a list of strings
+     */
+    void WriteStringList(const char* format,
+                         const char* last_format,
+                         const std::list<std::string>& l) const;
+
+    /**
+     * @see WriteStringList(const char*, const char*,
+                            const std::list<std::string>&>)
+     */
+    void WriteStringList(const char* format,
+                         const std::list<std::string>& l) const
+        { WriteStringList(format, format, l); }
+
+    /**
+     * Writes out a list of BroDocObj objects to the reST document
+     * @param l A list of BroDocObj pointers
+     * @param exportCond If true, filter out objects that are not in an
+     *        export section.  If false, filter out those that are in
+     *        an export section.
+     * @param heading The title of the section to create in the reST doc.
+     * @param underline The character to use to underline the reST
+     *        section heading.
+     */
+    void WriteBroDocObjList(const std::list<const BroDocObj*>& l,
+                            bool exportCond,
+                            const char* heading,
+                            char underline) const;
+
+    /**
+     * A wrapper to fprintf() that always uses the reST document
+     * for the FILE* argument.
+     * @param format A printf style format string.
+     */
+    void WriteToDoc(const char* format, ...) const;
+
+    /**
+     * Writes out a reST section heading
+     * @param heading The title of the heading to create
+     * @param underline The character to use to underline the section title
+              within the reST document
+     */
+    void WriteSectionHeading(const char* heading, char underline) const;
+private:
+
+    /**
+     * Frees memory allocated to BroDocObj's objects in a given list.
+     * @param a reference to a list of BroDocObj pointers
+     */
+    void FreeBroDocObjPtrList(std::list<const BroDocObj*>& l);
+};
+
+#endif

src/BroDocObj.cc

@@ -0,0 +1,35 @@
+#include <cstdio>
+#include <string>
+#include <list>
+#include "Obj.h"
+#include "BroDocObj.h"
+
+BroDocObj::BroDocObj(const BroObj* obj,
+                     std::list<std::string>*& reST,
+                     bool exported)
+    {
+    broObj = obj;
+    isExported = exported;
+    reST_doc_strings = reST;
+    reST = 0;
+    }
+
+BroDocObj::~BroDocObj()
+    {
+    delete reST_doc_strings;
+    }
+
+void BroDocObj::WriteReST(FILE* file) const
+    {
+    if ( reST_doc_strings )
+        {
+        std::list<std::string>::const_iterator it;
+        for ( it = reST_doc_strings->begin();
+              it != reST_doc_strings->end(); ++it)
+            fprintf(file, "%s\n", it->c_str());
+        }
+
+    ODesc desc;
+    broObj->Describe(&desc);
+    fprintf(file, "%s\n", desc.Description());
+    }

src/BroDocObj.h

@@ -0,0 +1,72 @@
+#ifndef brodocobj_h
+#define brodocobj_h
+
+#include <cstdio>
+#include <string>
+#include <list>
+
+#include "Obj.h"
+
+class BroDocObj {
+public:
+    /**
+     * BroDocObj constructor
+     * @param obj a pointer to a BroObj that is to be documented
+     * @param reST a reference to a pointer of a list of strings that
+              represent the reST documentation for the BroObj.  The pointer
+              will be set to 0 after this constructor finishes.
+     * @param exported whether the BroObj is declared in an export section
+     */
+    BroDocObj(const BroObj* obj, std::list<std::string>*& reST, bool exported);
+
+    /**
+     * BroDocObj destructor
+     * Deallocates the memory associated with the list of reST strings
+     */
+    ~BroDocObj();
+
+    /**
+     * writes the reST representation of this object which includes
+     * 1) any of the "##" comments (stored internally in reST_doc_string)
+     *    To make things easy, I think we should assume that the documenter
+     *    writes their comments such that anything after ## is valid reST
+     *    so that at parse time the ## is just stripped and the remainder
+     *    is scheduled to be inserted as-is into the reST.
+     *    TODO: prepare for some kind of filtering mechanism that transforms
+     *    the reST as written into new reST before being written out.
+     *    This allows for additional custom markup or macros when writing
+     *    pure reST might be inconvenient.
+     * 2) a reST friendly description of the BroObj
+     *    Could be implemented similar to the virtual BroObj::Describe(ODesc)
+     *    E.g. all subclasses will now need to implement a reSTDescribe(ODesc)
+     *    so that they can describe themselves in terms of the custom reST
+     *    directives/roles that we'll later teach to Sphinx via a "bro domain".
+     *    ID's should be able to implement the reSTDescribe(ODesc) function
+     *    such that their namespace and attributes are output as well.
+     * @param The (already opened) file to write the reST to.
+     */
+    void WriteReST(FILE* file) const;
+
+    /**
+     * Check whether this documentation is part of the public API
+     * (The BroObj declaration was while in an export section).
+     * @return true if the BroObj was declared in an export section, else false
+     */
+    bool IsPublicAPI() const { return isExported; }
+
+    /**
+     * Return whether this object has documentation (## comments)
+     * @return true if the BroObj has comments associated with it
+     */
+    bool HasDocumentation() const { return reST_doc_strings &&
+                                           reST_doc_strings->size() > 0; }
+
+protected:
+    std::list<std::string>* reST_doc_strings;
+    const BroObj* broObj;
+    bool isExported;
+
+private:
+};
+
+#endif

src/CMakeLists.txt

@@ -247,6 +247,9 @@ set(bro_SRCS
     BitTorrent.cc
     BitTorrentTracker.cc
     BPF_Program.cc
+    BroBifDoc.cc
+    BroDoc.cc
+    BroDocObj.cc
     BroString.cc
     CCL.cc
     ChunkedIO.cc

src/main.cc

@@ -91,6 +91,7 @@ int optimize = 0;
 int do_notice_analysis = 0;
 int rule_bench = 0;
 int print_loaded_scripts = 0;
+int generate_documentation = 0;
 SecondaryPath* secondary_path = 0;
 ConnCompressor* conn_compressor = 0;
 extern char version[];
@@ -145,6 +146,7 @@ void usage()
 	fprintf(stderr, "    -h|--help|-?                   | command line help\n");
 	fprintf(stderr, "    -i|--iface <interface>         | read from given interface\n");
 	fprintf(stderr, "    -l|--print-scripts             | print all loaded scripts\n");
+	fprintf(stderr, "    -Z|--doc-scripts               | generate documentation for all loaded scripts\n");
 	fprintf(stderr, "    -p|--prefix <prefix>           | add given prefix to policy file resolution\n");
 	fprintf(stderr, "    -r|--readfile <readfile>       | read from given tcpdump file\n");
 	fprintf(stderr, "    -y|--flowfile <file>[=<ident>] | read from given flow file\n");
@@ -364,6 +366,7 @@ int main(int argc, char** argv)
 		{"help",		no_argument,		0,	'h'},
 		{"iface",		required_argument,	0,	'i'},
 		{"print-scripts",	no_argument,		0,	'l'},
+		{"doc-scripts",	no_argument,		0,	'Z'},
 		{"prefix",		required_argument,	0,	'p'},
 		{"readfile",		required_argument,	0,	'r'},
 		{"flowfile",		required_argument,	0,	'y'},
@@ -440,7 +443,7 @@ int main(int argc, char** argv)
 	opterr = 0;
 
 	char opts[256];
-	safe_strncpy(opts, "A:a:B:D:e:f:I:i:K:n:p:R:r:s:T:t:U:w:x:X:y:Y:z:CFGHLOPSWdghlv",
+	safe_strncpy(opts, "A:a:B:D:e:f:I:i:K:n:p:R:r:s:T:t:U:w:x:X:y:Y:z:CFGHLOPSWdghlvZ",
 		     sizeof(opts));
 
 #ifdef USE_PERFTOOLS
@@ -633,6 +636,10 @@ int main(int argc, char** argv)
 			break;
 #endif
 
+        case 'Z':
+            generate_documentation = 1;
+            break;
+
 #ifdef USE_IDMEF
 		case 'n':
 			fprintf(stderr, "Using IDMEF XML DTD from %s\n", optarg);

src/parse.y

@@ -73,6 +73,15 @@
 #include "DNS.h"
 #include "RE.h"
 #include "Scope.h"
+#include "BroDoc.h"
+#include "BroDocObj.h"
+
+#include <list>
+#include <string>
+
+extern BroDoc* current_reST_doc;
+extern int generate_documentation;
+extern std::list<std::string>* reST_doc_comments;
 
 YYLTYPE GetCurrentLocation();
 extern int yyerror(const char[]);
@@ -785,7 +794,13 @@ formal_args_decl:
 
 decl:
 		TOK_MODULE TOK_ID ';'
-			{ current_module = $2; }
+			{
+			current_module = $2;
+			if ( generate_documentation )
+				{
+				current_reST_doc->AddModule(current_module);
+				}
+			}
 
 	|	TOK_EXPORT '{' { is_export = true; } decl_list '}'
 			{ is_export = false; }

src/scan.l

@@ -15,11 +15,16 @@
 #include "Debug.h"
 #include "PolicyFile.h"
 #include "broparse.h"
+#include "BroDoc.h"
+#include "BroBifDoc.h"
 
 #include <stack>
+#include <list>
+#include <string>
 
 extern YYLTYPE yylloc;	// holds start line and column of token
 extern int print_loaded_scripts;
+extern int generate_documentation;
 
 int nwarn = 0;
 int nerr = 0;
@@ -35,6 +40,8 @@ int_list if_stack;
 int line_number = 1;
 int include_level = 0;
 const char* filename = 0;
+BroDoc* current_reST_doc = 0;
+static BroDoc* last_reST_doc = 0;
 
 char last_tok[128];
 
@@ -50,6 +57,21 @@ char last_tok[128];
 // Files we have already scanned (or are in the process of scanning).
 static PList(char) files_scanned;
 
+// reST documents that we've created (or have at least opened so far)
+static std::list<BroDoc*> docs_generated;
+
+// reST comments (those starting with ##) seen so far
+std::list<std::string>* reST_doc_comments = 0;
+
+// print current contents of reST_doc_comments list to stderr
+void print_current_reST_doc_comments();
+
+// delete the reST_doc_comments list object
+void clear_reST_doc_comments();
+
+// adds changes to capture_filter to the current script's reST documentation
+static void check_capture_filter_changes();
+
 class FileInfo {
 public:
 	FileInfo(string restore_module = "");
@@ -60,6 +82,7 @@ public:
 	const char* name;
 	int line;
 	int level;
+	BroDoc* doc;
 };
 
 // A stack of input buffers we're scanning.  file_stack[len-1] is the
@@ -102,6 +125,30 @@ ESCSEQ	(\\([^\n]|[0-7]+|x[[:xdigit:]]+))
 
 %%
 
+##!.* {
+	// Add this format of comments to the script documentation's "summary"
+	if ( generate_documentation )
+		current_reST_doc->AddSummary(yytext + 3);
+	}
+
+##{OWS}Author:.* {
+	if ( generate_documentation )
+		current_reST_doc->SetAuthor(
+			skip_whitespace(            // Skip whitespace after "Author:"
+			skip_whitespace(yytext + 2) // Skip whitespace after ##
+			+ 7)                        // Skip "Author:"
+		);
+	}
+
+##[^#\n].* {
+	if ( generate_documentation )
+		{
+		if ( ! reST_doc_comments )
+			reST_doc_comments = new std::list<std::string>();
+		reST_doc_comments->push_back(yytext + 2);
+		}
+}
+
 #.*	/* eat comments */
 
 {WS}	/* eat whitespace */
@@ -211,6 +258,17 @@ when	return TOK_WHEN;
 
 @load{WS}{FILE}	{
 	const char* new_file = skip_whitespace(yytext + 5);	// Skip "@load".
+	if ( generate_documentation )
+		{
+		current_reST_doc->AddImport(new_file);
+		if ( reST_doc_comments )
+			{
+			fprintf(stderr, "Warning: unconsumed reST documentation is being "
+					"discarded before doing '@load %s' in %s:\n",
+					 new_file, current_reST_doc->GetSourceFileName());
+			clear_reST_doc_comments();
+			}
+		}
 	(void) load_files_with_prefix(new_file);
 	}
 
@@ -446,7 +504,6 @@ static int load_files_with_prefix(const char* orig_file)
 				strcpy(new_filename, file);
 
 			f = search_for_file(new_filename, "bro", &full_filename);
-
 			delete [] new_filename;
 			}
 
@@ -496,6 +553,21 @@ static int load_files_with_prefix(const char* orig_file)
 			// Don't delete the old filename - it's pointed to by
 			// every BroObj created when parsing it.
 			yylloc.filename = filename = full_filename;
+
+			if ( generate_documentation )
+				{
+				char* bifExtStart = strstr(full_filename, ".bif.bro");
+				BroDoc* reST_doc;
+
+				if ( bifExtStart )
+					reST_doc = new BroBifDoc(full_filename);
+				else
+					reST_doc = new BroDoc(full_filename);
+
+				docs_generated.push_back(reST_doc);
+
+				current_reST_doc = reST_doc;
+				}
 			}
 
 		else
@@ -655,6 +727,7 @@ int yywrap()
 	// Stack is now empty.
 	while ( input_files.length() > 0 )
 		{
+		check_capture_filter_changes();
 		if ( load_files_with_prefix(input_files[0]) )
 			{
 			// Don't delete the filename - it's pointed to by
@@ -667,6 +740,7 @@ int yywrap()
 		// if any.
 		(void) input_files.remove_nth(0);
 		}
+	check_capture_filter_changes();
 
 	// Add redef statements for any X=Y command line parameters.
 	if ( params.size() > 0 )
@@ -731,6 +805,18 @@ int yywrap()
 		return 0;
 		}
 
+	if ( generate_documentation )
+		{
+		std::list<BroDoc*>::iterator it;
+		for ( it = docs_generated.begin(); it != docs_generated.end(); ++it )
+			{
+			(*it)->WriteDocFile();
+			delete *it;
+			}
+		docs_generated.clear();
+		clear_reST_doc_comments();
+		}
+
 	// Otherwise, we are done.
 	return 1;
 	}
@@ -742,6 +828,7 @@ FileInfo::FileInfo(string arg_restore_module)
 	name = ::filename;
 	line = ::line_number;
 	level = ::include_level;
+	doc = ::current_reST_doc;
 	}
 
 FileInfo::~FileInfo()
@@ -753,6 +840,8 @@ FileInfo::~FileInfo()
 	yylloc.filename = filename = name;
 	yylloc.first_line = yylloc.last_line = line_number = line;
 	include_level = level;
+	last_reST_doc = current_reST_doc;
+	current_reST_doc = doc;
 
 	if ( restore_module != "" )
 		current_module = restore_module;
@@ -779,3 +868,40 @@ static void report_file()
 	files_reported.append(copy_string(filename));
 	}
 
+static void check_capture_filter_changes()
+	{
+	if ( generate_documentation )
+		{
+		// Lookup the "capture_filters" identifier, if it has any
+		// defined value, add it to the script's reST documentation,
+		// and finally clear the table so it doesn't taint the
+		// documentation for subsequent scripts.
+
+		ID* capture_filters = global_scope()->Lookup("capture_filters");
+		if ( capture_filters )
+			{
+				ODesc desc;
+				capture_filters->ID_Val()->Describe(&desc);
+				last_reST_doc->SetPacketFilter(desc.Description());
+				( (TableVal*) capture_filters->ID_Val() )->RemoveAll();
+			}
+		}
+	}
+
+void print_current_reST_doc_comments()
+	{
+	if ( ! reST_doc_comments ) return;
+	std::list<std::string>::iterator it;
+	for (it = reST_doc_comments->begin(); it != reST_doc_comments->end(); ++it)
+		fprintf(stderr, "##%s\n", it->c_str());
+	}
+
+void clear_reST_doc_comments()
+	{
+	if ( ! reST_doc_comments ) return;
+	fprintf(stderr, "Warning: %lu unconsumed reST comments:\n",
+			reST_doc_comments->size());
+	print_current_reST_doc_comments();
+	delete reST_doc_comments;
+	reST_doc_comments = 0;
+	}