Merge remote branch 'origin/topic/jsiwek/doc-framework'

CHANGES

@@ -1,3 +1,25 @@
+1.6-dev.80 Mon Apr 18 14:50:54 PDT 2011
+
+- New framework for generating documentation from Bro scripts. (Jon
+  Siwek)
+
+  This includes:
+
+    - Changes to Bro's scanner/parser to facilitate automatic
+      generation of Bro policy script documentation in
+      reStructuredText format.
+
+    - New command line flags -Z/--doc-scripts to enable the new doc
+      generation mode.
+
+    - Changes to bifcl to pass comments starting with "##" through
+      into the generated .bro script.
+
+    - A "doc" build target for the top-level Makefile to first
+      generate reStructuredText for a defined set of Bro policy
+      scripts, and then run that through Sphinx to create HTML
+      documentation.
+
 1.6-dev.78 Mon Apr 18 12:52:55 PDT 2011
 
 - Adding files to CMake build targets so they show up in generated IDE

CMakeLists.txt

@@ -51,6 +51,11 @@ file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/bro-path-dev.csh
      "setenv BROPATH `${CMAKE_CURRENT_BINARY_DIR}/bro-path-dev`\n")
 
 file(STRINGS "${CMAKE_CURRENT_SOURCE_DIR}/VERSION" VERSION LIMIT_COUNT 1)
+string(REPLACE "." " " version_numbers ${VERSION})
+separate_arguments(version_numbers)
+list(GET version_numbers 0 VERSION_MAJOR)
+list(GET version_numbers 1 VERSION_MINOR)
+set(VERSION_MAJ_MIN "${VERSION_MAJOR}.${VERSION_MINOR}")
 
 set(EXTRA_COMPILE_FLAGS "-Wall -Wno-unused")
 
@@ -170,8 +175,7 @@ include_directories(${CMAKE_CURRENT_BINARY_DIR})
 
 add_subdirectory(src)
 add_subdirectory(policy)
-#add_subdirectory(scripts)
-#add_subdirectory(doc)
+add_subdirectory(doc)
 
 include(CheckOptionalBuildSources)
 

Makefile

@@ -23,6 +23,13 @@ install: configured
 
 clean: configured
 	( cd $(BUILD) && make clean )
+	( cd $(BUILD) && make doc-clean )
+
+doc: configured
+	( cd $(BUILD) && make doc )
+
+doc-clean: configured
+	( cd $(BUILD) && make doc-clean )
 
 dist: cmake_version
 	# Minimum Bro source package

VERSION

@@ -1 +1 @@
-1.6-dev.78
+1.6-dev.80

doc/CMakeLists.txt

@@ -0,0 +1,42 @@
+set(POLICY_SRC_DIR ${PROJECT_SOURCE_DIR}/policy)
+set(DOC_OUTPUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/out)
+set(DOC_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/source)
+set(DOC_SOURCE_WORKDIR ${CMAKE_CURRENT_BINARY_DIR}/source)
+
+file(GLOB_RECURSE DOC_SOURCES FOLLOW_SYMLINKS "*")
+
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/conf.py.in
+               ${CMAKE_CURRENT_BINARY_DIR}/conf.py
+               @ONLY)
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/scripts/generate_reST_docs.py.in
+               ${CMAKE_CURRENT_BINARY_DIR}/generate_reST_docs.py
+               @ONLY)
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/scripts/BroToReST.py.in
+               ${CMAKE_CURRENT_BINARY_DIR}/BroToReST.py
+               @ONLY)
+
+add_custom_target(doc
+                  COMMAND "${CMAKE_COMMAND}" -E copy_directory
+                          ${DOC_SOURCE_DIR}
+                          ${DOC_SOURCE_WORKDIR}
+                  COMMAND python generate_reST_docs.py
+                  COMMAND sphinx-build
+                          -b html
+                          -c ${CMAKE_CURRENT_BINARY_DIR}
+                          -d ${DOC_OUTPUT_DIR}/doctrees
+                          ${DOC_SOURCE_WORKDIR}
+                          ${DOC_OUTPUT_DIR}/html
+                  WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
+                  COMMENT "[Sphinx] Generating Script Documentation"
+                  VERBATIM
+                  # SOURCES just adds stuff to IDE projects as a convienience
+                  SOURCES ${DOC_SOURCES})
+
+add_dependencies(doc bro doc-clean)
+
+add_custom_target(doc-clean
+                  COMMAND "${CMAKE_COMMAND}" -E remove_directory
+                          ${CMAKE_CURRENT_BINARY_DIR}/source
+                  COMMAND "${CMAKE_COMMAND}" -E remove_directory
+                          ${DOC_OUTPUT_DIR}
+                  VERBATIM)

doc/README

@@ -1 +1,44 @@
-TODO.
+This directory contains scripts and templates that can be used to automate
+the generation of Bro script documentation.  Two build targets are defined
+by CMake:
+
+``make doc``
+
+    This target depends on a Python interpreter (>=2.5) and
+    `Sphinx <http://sphinx.pocoo.org/>`_ being installed.  Sphinx can be
+    installed like::
+
+        > sudo easy_install sphinx
+
+    This target will also first build the bro binary if it is not already
+    since the generation of reStructuredText (reST) documentation from
+    Bro scripts is integrated within the parsing process.
+
+    After completion, HTML documentation can be located inside the CMake
+    ``build/`` directory as ``build/doc/out/html``.  The generated reST
+    documentation will be located in ``build/doc/source/policy``.
+
+``make doc-clean``
+
+    This target removes Sphinx inputs and outputs from the CMake ``build/`` dir.
+
+To schedule a script to be documented, edit ``scripts/generate_reST_docs.py.in``
+and try adding the name of the script along with an optional script group to
+the ``docs`` dictionary.  That python script also shows other, more specialized
+methods for generating documentation for some types of corner-cases.
+
+When adding a new logical grouping for generated scripts, create a new
+reST document in ``source/policy/<group_name>.rst`` and add some default
+documentation.  References to (and summaries of) documents associated with
+the group get appended to this file during the ``make doc`` process.
+
+The Sphinx source tree template in ``source/`` can be modified to add more
+common/general documentation, style sheets, JavaScript, etc.  The Sphinx
+config file is produced from ``conf.py.in``, so that can be edited to change
+various Sphinx options, like setting the default HTML rendering theme.
+There is also a custom Sphinx domain implemented in ``source/ext/bro.py``
+which adds some reST directives and roles that aid in generating useful
+index entries and cross-references.
+
+See ``example.bro`` for an example of how to document a Bro script such that
+``make doc`` will be able to produce reST/HTML documentation for it.

doc/conf.py.in

@@ -0,0 +1,215 @@
+# -*- coding: utf-8 -*-
+#
+# Bro documentation build configuration file, created by sphinx-quickstart
+#
+# This file is execfile()d with the current directory set to its containing dir.
+#
+# Note that not all possible configuration values are present in this
+# autogenerated file.
+#
+# All configuration values have a default; values that are commented out
+# serve to show the default.
+
+import sys, os
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+sys.path.insert(0, os.path.abspath('source/ext'))
+
+# -- General configuration -----------------------------------------------------
+
+# If your documentation needs a minimal Sphinx version, state it here.
+#needs_sphinx = '1.0'
+
+# Add any Sphinx extension module names here, as strings. They can be extensions
+# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
+extensions = ['bro']
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['source/_templates']
+
+# The suffix of source filenames.
+source_suffix = '.rst'
+
+# The encoding of source files.
+#source_encoding = 'utf-8-sig'
+
+# The master toctree document.
+master_doc = 'index'
+
+# General information about the project.
+project = u'Bro'
+copyright = u'2011, Jon Siwek'
+
+# The version info for the project you're documenting, acts as replacement for
+# |version| and |release|, also used in various other places throughout the
+# built documents.
+#
+# The short X.Y version.
+version = '@VERSION_MAJ_MIN@'
+# The full version, including alpha/beta/rc tags.
+release = '@VERSION@'
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+#language = None
+
+# There are two options for replacing |today|: either, you set today to some
+# non-false value, then it is used:
+#today = ''
+# Else, today_fmt is used as the format for a strftime call.
+#today_fmt = '%B %d, %Y'
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+exclude_patterns = []
+
+# The reST default role (used for this markup: `text`) to use for all documents.
+#default_role = None
+
+# If true, '()' will be appended to :func: etc. cross-reference text.
+#add_function_parentheses = True
+
+# If true, the current module name will be prepended to all description
+# unit titles (such as .. function::).
+#add_module_names = True
+
+# If true, sectionauthor and moduleauthor directives will be shown in the
+# output. They are ignored by default.
+show_authors = True
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = 'sphinx'
+
+# A list of ignored prefixes for module index sorting.
+#modindex_common_prefix = []
+
+
+# -- Options for HTML output ---------------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+html_theme = 'default'
+
+# Theme options are theme-specific and customize the look and feel of a theme
+# further.  For a list of options available for each theme, see the
+# documentation.
+#html_theme_options = {}
+
+# Add any paths that contain custom themes here, relative to this directory.
+#html_theme_path = []
+
+# The name for this set of Sphinx documents.  If None, it defaults to
+# "<project> v<release> documentation".
+#html_title = None
+
+# A shorter title for the navigation bar.  Default is the same as html_title.
+#html_short_title = None
+
+# The name of an image file (relative to this directory) to place at the top
+# of the sidebar.
+#html_logo = None
+
+# The name of an image file (within the static path) to use as favicon of the
+# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
+# pixels large.
+#html_favicon = None
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['source/_static']
+
+# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
+# using the given strftime format.
+#html_last_updated_fmt = '%b %d, %Y'
+
+# If true, SmartyPants will be used to convert quotes and dashes to
+# typographically correct entities.
+#html_use_smartypants = True
+
+# Custom sidebar templates, maps document names to template names.
+#html_sidebars = {}
+
+# Additional templates that should be rendered to pages, maps page names to
+# template names.
+#html_additional_pages = {}
+
+# If false, no module index is generated.
+#html_domain_indices = True
+
+# If false, no index is generated.
+#html_use_index = True
+
+# If true, the index is split into individual pages for each letter.
+#html_split_index = False
+
+# If true, links to the reST sources are added to the pages.
+#html_show_sourcelink = True
+
+# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
+#html_show_sphinx = True
+
+# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
+#html_show_copyright = True
+
+# If true, an OpenSearch description file will be output, and all pages will
+# contain a <link> tag referring to it.  The value of this option must be the
+# base URL from which the finished HTML is served.
+#html_use_opensearch = ''
+
+# This is the file name suffix for HTML files (e.g. ".xhtml").
+#html_file_suffix = None
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = 'Brodoc'
+
+
+# -- Options for LaTeX output --------------------------------------------------
+
+# The paper size ('letter' or 'a4').
+#latex_paper_size = 'letter'
+
+# The font size ('10pt', '11pt' or '12pt').
+#latex_font_size = '10pt'
+
+# Grouping the document tree into LaTeX files. List of tuples
+# (source start file, target name, title, author, documentclass [howto/manual]).
+latex_documents = [
+  ('index', 'Bro.tex', u'Bro Documentation',
+   u'Jon Siwek', 'manual'),
+]
+
+# The name of an image file (relative to this directory) to place at the top of
+# the title page.
+#latex_logo = None
+
+# For "manual" documents, if this is true, then toplevel headings are parts,
+# not chapters.
+#latex_use_parts = False
+
+# If true, show page references after internal links.
+#latex_show_pagerefs = False
+
+# If true, show URL addresses after external links.
+#latex_show_urls = False
+
+# Additional stuff for the LaTeX preamble.
+#latex_preamble = ''
+
+# Documents to append as an appendix to all manuals.
+#latex_appendices = []
+
+# If false, no module index is generated.
+#latex_domain_indices = True
+
+
+# -- Options for manual page output --------------------------------------------
+
+# One entry per manual page. List of tuples
+# (source start file, name, description, authors, manual section).
+man_pages = [
+    ('index', 'bro', u'Bro Documentation',
+     [u'Jon Siwek'], 1)
+]

doc/example.bro

@@ -0,0 +1,178 @@
+##! This is an example script that demonstrates how to document.  Comments
+##! of the form ``##!`` are for the script summary.  The contents of
+##! these comments are transferred directly into the auto-generated
+##! `reStructuredText <http://docutils.sourceforge.net/rst.html>`_
+##! (reST) document's summary section.
+##!
+##! .. tip:: You can embed directives and roles within ``##``-stylized comments
+##!
+##! :Author: Jon Siwek <jsiwek@ncsa.illinois.edu>
+
+# Comments that use a single pound sign (#) are not significant to
+# a script's auto-generated documentation, but ones that use a
+# double pound sign (##) do matter.  In some cases, like record
+# field comments, it's necessary to disambiguate the field with
+# which a comment associates: e.g. "##<" can be used on the same line
+# as a field to signify the comment relates to it and not the
+# following field.  "##<" is not meant for general use, just
+# record/enum fields.
+#
+# Generally, the auto-doc comments (##) are associated with the
+# next declaration/identifier found in the script, but the doc framework
+# will track/render identifiers regardless of whether they have any
+# of these special comments associated with them.
+#
+# The first sentence contained within the "##"-stylized comments for
+# a given identifier is special in that it will be used as summary
+# text in a table containing all such identifiers and short summaries.
+# If there are no sentences (text terminated with '.'), then everything
+# in the "##"-stylized comments up until the first empty comment
+# is taken as the summary text for a given identifier.
+
+# @load directives are self-documenting
+@load notice
+
+# "module" statements are self-documenting
+module Example;
+
+# redefinitions of "capture_filters" are self-documenting and
+# go into the generated documentation's "Packet Filter" section
+redef capture_filters += {
+    ["ssl"] = "tcp port 443",
+    ["nntps"] = "tcp port 563",
+};
+
+global example_ports = {
+    443/tcp, 563/tcp,
+} &redef;
+
+# redefinitions of "dpd_config" are self-documenting and
+# go into the generated doc's "Port Analysis" section
+redef dpd_config += {
+    [ANALYZER_SSL] = [$ports = example_ports]
+};
+
+# redefinitions of "Notice::Type" are self-documenting, but
+# more information can be supplied in two different ways
+redef enum Notice += {
+    ## any number of this type of comment
+    ## will document "Notice_One"
+    Notice_One,
+    Notice_Two,  ##< any number of this type of comment
+                 ##< will document "Notice_Two"
+    Notice_Three,
+    Notice_Four,
+};
+
+# Anything declared in the export section will show up in the rendered
+# documentation's "public interface" section
+
+export {
+
+    # these headings don't mean anything special to the
+    # doc framework right now, I'm just including them
+    # to make it more clear to the reader how the doc
+    # framework will actually categorize a script's identifiers
+
+    ############## types ################
+
+    # Note that I'm just mixing the "##" and "##<"
+    # types of comments in the following declarations
+    # as a demonstration.  Normally, it would be good style
+    # to pick one and be consistent.
+
+    ## documentation for "SimpleEnum"
+    ## goes here.
+    type SimpleEnum: enum {
+        ## and more specific info for "ONE"
+        ## can span multiple lines
+        ONE,
+        TWO,  ##< or more info like this for "TWO"
+              ##< can span multiple lines
+        THREE,
+    };
+
+    ## document the "SimpleEnum" redef here
+    redef enum SimpleEnum  += {
+        FOUR, ##< and some documentation for "FOUR"
+        ## also "FIVE" for good measure
+        FIVE
+    };
+
+    ## general documentation for a type "SimpleRecord"
+    ## goes here.
+    type SimpleRecord: record {
+        ## counts something
+        field1: count;
+        field2: bool; ##< toggles something
+    };
+
+
+    ## general documentation for a type "ComplexRecord" goes here
+    type ComplexRecord: record {
+        field1: count;               ##< counts something
+        field2: bool;                ##< toggles something
+        field3: SimpleRecord;
+        msg: string &default="blah"; ##< attributes are self-documenting
+    } &redef;
+
+    ############## options ################
+    # right now, I'm just defining an option as
+    # any const with &redef (something that can
+    # change at parse time, but not at run time.
+
+    ## add documentation for "an_option" here
+    const an_option: set[addr, addr, string] &redef;
+
+    # default initialization will be self-documenting
+    const option_with_init = 0.01 secs &redef;
+
+    ############## state variables ############
+    # right now, I'm defining this as any global
+    # that's not a function/event.  doesn't matter
+    # if &redef attribute is present
+
+    ## put some documentation for "a_var" here
+    global a_var: bool;
+
+    # attributes are self-documenting
+    global var_with_attr: count &persistent;
+
+    # it's fine if the type is inferred, that information is self-documenting
+    global var_without_explicit_type = "this works";
+
+    ############## functions/events ############
+
+    ## Summarize purpose of "a_function" here.
+    ## Give more details about "a_function" here.
+    ## Separating the documentation of the params/return values with
+    ## empty comments is optional, but improves readability of script.
+    ##
+    ## tag: function arguments can be described
+    ##      like this
+    ## msg: another param
+    ##
+    ## Returns: describe the return type here
+    global a_function: function(tag: string, msg: string): string;
+
+    ## Summarize "an_event" here.
+    ## Give more details about "an_event" here.
+    ## name: describe the argument here
+    global an_event: event(name: string);
+}
+
+# this function is documented in the "private interface" section
+# of generated documentation and any "##"-stylized comments would also
+# be rendered there
+function function_without_proto(tag: string): string
+    {
+    return "blah";
+    }
+
+# this record type is documented in the "private interface" section
+# of generated documentation and any "##"-stylized comments would also
+# be rendered there
+type PrivateRecord: record {
+    field1: bool;
+    field2: count;
+};

doc/scripts/BroToReST.py.in

@@ -0,0 +1,152 @@
+#! /usr/bin/env python
+
+import os
+import subprocess
+import shutil
+import glob
+import string
+import sys
+
+BRO = "@CMAKE_BINARY_DIR@/src/bro"
+BROPATHDEV = "`@CMAKE_BINARY_DIR@/bro-path-dev`"
+BRO_ARGS = "--doc-scripts"
+DOC_DST_DIR = "@DOC_SOURCE_WORKDIR@/policy"
+BROPATH = subprocess.Popen("@CMAKE_BINARY_DIR@/bro-path-dev", shell=True,
+        stdout=subprocess.PIPE, stderr=subprocess.STDOUT).stdout.readline()
+
+class BroToReST:
+    """A class to encapsulate the the generation of reST documentation from
+    a given Bro script.
+    """
+
+    bro_src_file = None
+    doc_src_file = None
+    load_via_stdin = False
+    group = None
+
+    def __init__(self, src_file, load_method=False, search_dir=None, group=None):
+        """
+        :param src_file: the file name of a Bro script (not a path)
+        :param load_method: T if script must be loaded by Bro via a stdin
+            redirection of "@load <script>", F if script can be loaded as
+            a command line argument to Bro
+        :param search_dir: a list of directories in which to search for
+            src_file.  If None, the default BROPATH is used.
+        :param group: a string representing a logical group that the script's
+            documentation should belong to.  A corresponding <group>.rst
+            document must be pre-existing in the policy/ dir of the source tree
+            used by Sphinx.
+        """
+
+        self.bro_src_file = FindBroScript(src_file, search_dir)
+        self.load_via_stdin = load_method
+        self.group = group
+
+        # formulate doc_src_file from src_file
+        filename = os.path.basename(src_file)
+        basename, ext = os.path.splitext(filename)
+        if ext == ".bro":
+            self.doc_src_file = basename + ".rst"
+        else:
+            self.doc_src_file = filename + ".rst"
+
+    def __str__(self):
+        return "bro_src_file: " + self.bro_src_file \
+           + "\ndoc_src_file: " + self.doc_src_file \
+           + "\ndoc_dst_file: " + os.path.join(DOC_DST_DIR, self.doc_src_file) \
+           + "\nstdin_load: %s" % self.load_via_stdin \
+           + "\ngroup: %s" % self.group
+
+    def GenDoc(self):
+        """Generates the reST documentation for a Bro script and copies
+        both the documentation and original script into Sphinx source tree.
+        If the documentation belongs to a group, the necessary modifications
+        to add it to the group's documentation are done.  Afterwards, any
+        files with a ".rst" suffix are removed for the working directory.
+        """
+
+        bro_src_basename = os.path.basename(self.bro_src_file)
+
+        if self.load_via_stdin:
+            cmd = "echo '@load %s' | %s %s" % (bro_src_basename, BRO, BRO_ARGS)
+        else:
+            cmd = "%s %s %s" % (BRO, BRO_ARGS, self.bro_src_file)
+
+        p = subprocess.Popen(cmd, shell=True, env={"BROPATH": BROPATH})
+
+        if p.wait() == 0:
+            shutil.copy(self.doc_src_file, DOC_DST_DIR)
+            shutil.copy(self.bro_src_file, DOC_DST_DIR)
+            AppendToDocGroup(self.group, self.bro_src_file, self.doc_src_file)
+
+        for leftover in glob.glob("*.rst"):
+            os.remove(leftover)
+
+def GenDocs(doc_dict, load_method=False):
+    """Generates reST documentation for all scripts in the given dictionary.
+
+    :param doc_dict: a dictionary whose keys are file names of Bro scripts
+        (not paths), and whose value is the logical documentation group
+        it belongs to
+    :param load_method: T if script must be loaded by Bro via a stdin
+            redirection of "@load <script>", F if script can be loaded as
+            a command line argument to Bro
+    """
+
+    for k, v in doc_dict.iteritems():
+        doc = BroToReST(k, load_method, group=v)
+        print "Generating reST document for " + k
+        doc.GenDoc()
+
+def FindBroScript(src_file, search_dir=None):
+    """Search a set of paths for a given Bro script and return the absolute
+    path to it.
+
+    :param src_file: the file name of a Bro script (not a path)
+    :param search_dir: a list of directories in which to search for
+        src_file.  If None, the default BROPATH is used.
+    """
+    if search_dir is None:
+        search_dir = string.split(BROPATH, ":")
+    for path in search_dir:
+        abs_path = os.path.join(path, src_file)
+        if os.path.exists(abs_path):
+            return abs_path
+    print >> sys.stderr, "Couldn't find '%s'" % src_file
+    return None
+
+def AppendToDocGroup(group, src_file, doc_file):
+    """Adds a reference to the given documentation for a Bro script
+    to the documentation file for it's associated group.  Also, associated
+    summary text (comments marked up like "##!" in the original Bro script
+    source) are added.
+
+    :param group: a string representing a logical group that the script's
+        documentation should belong to.  A corresponding <group>.rst
+        document must be pre-existing in the policy/ dir of the source tree
+        used by Sphinx.
+    :param src_file: a path to the original Bro script source file
+    :param doc_file: the file name of a script's generated reST document
+    """
+    if group is None:
+        return
+
+    group_file = os.path.join(DOC_DST_DIR, group + ".rst")
+    if not os.path.exists(group_file):
+        print >> sys.stderr, "Group file doesn't exist: " + group_file
+        return
+
+    summary_comments = []
+
+    with open(src_file, 'r') as f:
+        for line in f:
+            sum_pos = string.find(line, "##!")
+            if sum_pos != -1:
+                summary_comments.append(line[(sum_pos+3):])
+
+    doc_name, ext = os.path.splitext(doc_file)
+
+    with open(group_file, 'a') as f:
+        f.write("\n:doc:`%s`\n" % doc_name)
+        for line in summary_comments:
+            f.write(line)

doc/scripts/generate_reST_docs.py.in

@@ -0,0 +1,71 @@
+#! /usr/bin/env python
+
+import os
+import subprocess
+import shutil
+import glob
+import string
+import sys
+from BroToReST import *
+
+# TODO: generate docs for more scripts
+# TODO: the groups are just made up to test the functionality, fix them
+
+# Scripts that can be loaded by bro via command line argument:
+docs = {
+    "alarm.bro":        "internal",
+    "arp.bro":          "user",
+    "conn.bro":         "internal",
+    "dhcp.bro":         "user",
+    "dns.bro":          "user",
+    "ftp.bro":          "user",
+    "http.bro":         "user",
+    "http-reply.bro":   None,
+    "http-request.bro": None,
+    "irc.bro":          "user",
+    "smtp.bro":         "user",
+    "ssl.bro":          "user",
+    "ssl-ciphers.bro":  None,
+    "ssl-errors.bro":   None,
+    "synflood.bro":     "user",
+    "tcp.bro":          "user",
+    "udp.bro":          "user",
+    "weird.bro":        "internal",
+}
+
+# Scripts that can't be loaded by bro via command line argument (possible
+# due to dependency issues), but can be loaded via an @load on stdin:
+stdin_docs = {
+    "notice.bro": "internal",
+}
+
+GenDocs(docs)
+GenDocs(stdin_docs, True)
+
+# The example documentation script doesn't live on the BROPATH, so
+# explicitly generate the docs for it like this:
+BroToReST("example.bro", False, ["@PROJECT_SOURCE_DIR@/doc"], group="internal").GenDoc()
+
+# Generate documentation for stuff that's always loaded into bro by default:
+cmd = "echo '' | %s %s" % (BRO, BRO_ARGS)
+p = subprocess.Popen(cmd, shell=True, env={"BROPATH": BROPATH})
+if p.wait() == 0:
+    for doc in glob.glob("*.rst"):
+        if doc == "<stdin>.rst":
+            os.remove(doc)
+            continue
+
+        basename, ext = os.path.splitext(doc)
+        basename2, ext = os.path.splitext(basename)
+        if ext == ".init":
+            src_file = basename
+        else:
+            src_file = basename + ".bro"
+        src_file = FindBroScript(src_file)
+        shutil.copy(src_file, DOC_DST_DIR)
+        shutil.copy(doc, DOC_DST_DIR)
+        if ext == ".bif":
+            AppendToDocGroup("bifs", src_file, doc)
+        else:
+            AppendToDocGroup("default", src_file, doc)
+        os.remove(doc)

doc/source/_static/showhide.js

@@ -0,0 +1,64 @@
+// make literal blocks corresponding to identifier initial values
+// hidden by default
+$(document).ready(function() {
+
+    var showText='(Show Value)';
+    var hideText='(Hide Value)';
+
+    var is_visible = false;
+
+    // select field-list tables that come before a literal block
+    tables = $('.highlight-python').prev('table.docutils.field-list');
+
+    tables.find('th.field-name').filter(function(index) {
+        return $(this).html() == "Default :";
+    }).next().append('<a href="#" class="toggleLink">'+showText+'</a>');
+
+    // hide all literal blocks that follow a field-list table
+    tables.next('.highlight-python').hide();
+
+    // register handler for clicking a "toggle" link
+    $('a.toggleLink').click(function() {
+        is_visible = !is_visible;
+
+        $(this).html( (!is_visible) ? showText : hideText);
+
+        // the link is inside a <table><tbody><tr><td> and the next
+        // literal block after the table is the literal block that we want
+        // to show/hide
+        $(this).parent().parent().parent().parent().next('.highlight-python').slideToggle('fast');
+
+        // override default link behavior
+        return false;
+    });
+});
+
+// make "Private Interface" sections hidden by default
+$(document).ready(function() {
+
+    var showText='Show Private Interface (for internal use)';
+    var hideText='Hide Private Interface';
+
+    var is_visible = false;
+
+    // insert show/hide links
+    $('#private-interface').children(":first-child").after('<a href="#" class="privateToggle">'+showText+'</a>');
+
+    // wrap all sub-sections in a new div that can be hidden/shown
+    $('#private-interface').children(".section").wrapAll('<div class="private" />');
+
+    // hide the given class
+    $('.private').hide();
+
+    // register handler for clicking a "toggle" link
+    $('a.privateToggle').click(function() {
+        is_visible = !is_visible;
+
+        $(this).html( (!is_visible) ? showText : hideText);
+
+        $('.private').slideToggle('fast');
+
+        // override default link behavior
+        return false;
+    });
+});

doc/source/_templates/layout.html

@@ -0,0 +1,5 @@
+{% extends "!layout.html" %}
+{% block extrahead %}
+    <script type="text/javascript" src="{{ pathto('_static/showhide.js', 1) }}"></script>
+    {{ super() }}
+{% endblock %}

doc/source/builtins.rst

@@ -0,0 +1,121 @@
+Builtin Types and Attributes
+============================
+
+Types
+-----
+
+The Bro scripting language supports the following built-in types.
+
+.. TODO: add documentation
+
+.. bro:type:: void
+
+.. bro:type:: bool
+
+.. bro:type:: int
+
+.. bro:type:: count
+
+.. bro:type:: counter
+
+.. bro:type:: double
+
+.. bro:type:: time
+
+.. bro:type:: interval
+
+.. bro:type:: string
+
+.. bro:type:: pattern
+
+.. bro:type:: enum
+
+.. bro:type:: timer
+
+.. bro:type:: port
+
+.. bro:type:: addr
+
+.. bro:type:: net
+
+.. bro:type:: subnet
+
+.. bro:type:: any
+
+.. bro:type:: table
+
+.. bro:type:: union
+
+.. bro:type:: record
+
+.. bro:type:: types
+
+.. bro:type:: func
+
+.. bro:type:: file
+
+.. bro:type:: vector
+
+.. TODO: below are kind of "special cases" that bro knows about?
+
+.. bro:type:: set
+
+.. bro:type:: function
+
+.. bro:type:: event
+
+.. TODO: Notice will get documented as part of notice.bro, which can eventually
+   be referenced here once that documentation is auto-generated.
+
+.. bro:type:: Notice
+
+Attributes
+----------
+
+The Bro scripting language supports the following built-in attributes.
+
+.. TODO: add documentation
+
+.. bro:attr:: &optional
+
+.. bro:attr:: &default
+
+.. bro:attr:: &redef
+
+.. bro:attr:: &rotate_interval
+
+.. bro:attr:: &rotate_size
+
+.. bro:attr:: &add_func
+
+.. bro:attr:: &delete_func
+
+.. bro:attr:: &expire_func
+
+.. bro:attr:: &read_expire
+
+.. bro:attr:: &write_expire
+
+.. bro:attr:: &create_expire
+
+.. bro:attr:: &persistent
+
+.. bro:attr:: &synchronized
+
+.. bro:attr:: &postprocessor
+
+.. bro:attr:: &encrypt
+
+.. bro:attr:: &match
+
+.. bro:attr:: &disable_print_hook
+
+.. bro:attr:: &raw_output
+
+.. bro:attr:: &mergeable
+
+.. bro:attr:: &priority
+
+.. bro:attr:: &group
+
+.. bro:attr:: (&tracked)

doc/source/common.rst

@@ -0,0 +1,19 @@
+Common Documentation
+====================
+
+.. _common_port_analysis_doc:
+
+Port Analysis
+-------------
+
+TODO: add some stuff here
+
+.. _common_packet_filter_doc:
+
+Packet Filter
+-------------
+
+TODO: add some stuff here
+
+.. note:: Filters are only relevant when dynamic protocol detection (DPD)
+          is explicitly turned off (Bro release 1.6 enabled DPD by default).

doc/source/ext/bro.py

@@ -0,0 +1,167 @@
+"""
+    The Bro domain for Sphinx.
+"""
+
+def setup(Sphinx):
+    Sphinx.add_domain(BroDomain)
+
+from sphinx import addnodes
+from sphinx.domains import Domain, ObjType, Index
+from sphinx.locale import l_, _
+from sphinx.directives import ObjectDescription
+from sphinx.roles import XRefRole
+from sphinx.util.nodes import make_refnode
+import string
+
+from docutils import nodes
+from docutils.parsers.rst import Directive
+from docutils.parsers.rst import directives
+from docutils.parsers.rst.roles import set_classes
+
+class BroGeneric(ObjectDescription):
+    def add_target_and_index(self, name, sig, signode):
+        targetname = self.objtype + '-' + name
+        if targetname not in self.state.document.ids:
+            signode['names'].append(targetname)
+            signode['ids'].append(targetname)
+            signode['first'] = (not self.names)
+            self.state.document.note_explicit_target(signode)
+
+            objects = self.env.domaindata['bro']['objects']
+            key = (self.objtype, name)
+# this is commented out mostly just to avoid having a special directive
+# for events in order to avoid the duplicate warnings in that case
+            """
+            if key in objects:
+                self.env.warn(self.env.docname,
+                              'duplicate description of %s %s, ' %
+                              (self.objtype, name) +
+                              'other instance in ' +
+                              self.env.doc2path(objects[key]),
+                              self.lineno)
+            """
+            objects[key] = self.env.docname
+        indextext = self.get_index_text(self.objtype, name)
+        if indextext:
+            self.indexnode['entries'].append(('single', indextext,
+                                              targetname, targetname))
+
+    def get_index_text(self, objectname, name):
+        return _('%s (%s)') % (name, self.objtype)
+
+    def handle_signature(self, sig, signode):
+        signode += addnodes.desc_name("", sig)
+        return sig
+
+class BroNamespace(BroGeneric):
+    def add_target_and_index(self, name, sig, signode):
+        targetname = self.objtype + '-' + name
+        if targetname not in self.state.document.ids:
+            signode['names'].append(targetname)
+            signode['ids'].append(targetname)
+            signode['first'] = (not self.names)
+            self.state.document.note_explicit_target(signode)
+
+            objects = self.env.domaindata['bro']['objects']
+            key = (self.objtype, name)
+            objects[key] = self.env.docname
+        indextext = self.get_index_text(self.objtype, name)
+        self.indexnode['entries'].append(('single', indextext,
+                                          targetname, targetname))
+        self.indexnode['entries'].append(('single',
+                                          "namespaces; %s" % (sig),
+                                          targetname, targetname))
+
+    def get_index_text(self, objectname, name):
+        return _('%s (namespace); %s') % (name, self.env.docname)
+
+    def handle_signature(self, sig, signode):
+        signode += addnodes.desc_name("", sig)
+        return sig
+
+class BroEnum(BroGeneric):
+    def add_target_and_index(self, name, sig, signode):
+        targetname = self.objtype + '-' + name
+        if targetname not in self.state.document.ids:
+            signode['names'].append(targetname)
+            signode['ids'].append(targetname)
+            signode['first'] = (not self.names)
+            self.state.document.note_explicit_target(signode)
+
+            objects = self.env.domaindata['bro']['objects']
+            key = (self.objtype, name)
+            objects[key] = self.env.docname
+        indextext = self.get_index_text(self.objtype, name)
+        #self.indexnode['entries'].append(('single', indextext,
+        #                                  targetname, targetname))
+        m = sig.split()
+        self.indexnode['entries'].append(('single',
+                                          "%s (enum values); %s" % (m[1], m[0]),
+                                          targetname, targetname))
+
+    def handle_signature(self, sig, signode):
+        m = sig.split()
+        name = m[0]
+        signode += addnodes.desc_name("", name)
+        return name
+
+class BroIdentifier(BroGeneric):
+    def get_index_text(self, objectname, name):
+        return name
+
+class BroAttribute(BroGeneric):
+    def get_index_text(self, objectname, name):
+        return _('%s (attribute)') % (name)
+
+class BroDomain(Domain):
+    """Bro domain."""
+    name = 'bro'
+    label = 'Bro'
+
+    object_types = {
+        'type':             ObjType(l_('type'),             'type'),
+        'namespace':        ObjType(l_('namespace'),        'namespace'),
+        'id':               ObjType(l_('id'),               'id'),
+        'enum':             ObjType(l_('enum'),             'enum'),
+        'attr':             ObjType(l_('attr'),             'attr'),
+    }
+
+    directives = {
+        'type':             BroGeneric,
+        'namespace':        BroNamespace,
+        'id':               BroIdentifier,
+        'enum':             BroEnum,
+        'attr':             BroAttribute,
+    }
+
+    roles = {
+        'type':             XRefRole(),
+        'namespace':        XRefRole(),
+        'id':               XRefRole(),
+        'enum':             XRefRole(),
+        'attr':             XRefRole(),
+    }
+
+    initial_data = {
+        'objects': {},  # fullname -> docname, objtype
+    }
+
+    def clear_doc(self, docname):
+        for (typ, name), doc in self.data['objects'].items():
+            if doc == docname:
+                del self.data['objects'][typ, name]
+
+    def resolve_xref(self, env, fromdocname, builder, typ, target, node,
+                     contnode):
+        objects = self.data['objects']
+        objtypes = self.objtypes_for_role(typ)
+        for objtype in objtypes:
+            if (objtype, target) in objects:
+                return make_refnode(builder, fromdocname,
+                                    objects[objtype, target],
+                                    objtype + '-' + target,
+                                    contnode, target + ' ' + objtype)
+
+    def get_objects(self):
+        for (typ, name), docname in self.data['objects'].iteritems():
+            yield name, name, typ, docname, typ + '-' + name, 1

doc/source/index.rst

@@ -0,0 +1,24 @@
+.. Bro documentation master file
+
+Welcome to Bro's documentation!
+===============================
+
+Contents:
+
+.. toctree::
+   :maxdepth: 1
+   :glob:
+
+   common
+   builtins
+   policy/default
+   policy/user
+   policy/bifs
+   policy/internal
+   policy/index
+
+Indices and tables
+==================
+
+* :ref:`genindex`
+* :ref:`search`

doc/source/policy/bifs.rst

@@ -0,0 +1,4 @@
+Built-In Functions (BIFs)
+=========================
+
+Here's a list of all documentation for BIFs that Bro provides:

doc/source/policy/default.rst

@@ -0,0 +1,3 @@
+Bro Scripts Loaded by Default
+=============================
+

doc/source/policy/index.rst

@@ -0,0 +1,10 @@
+Index of All Policy Script Documentation
+========================================
+
+Contents:
+
+.. toctree::
+   :maxdepth: 1
+   :glob:
+
+   *

doc/source/policy/internal.rst

@@ -0,0 +1,3 @@
+Internal Policy Scripts
+=======================
+

doc/source/policy/user.rst

@@ -0,0 +1,3 @@
+User-Facing Policy Scripts
+==========================
+

src/Attr.cc

@@ -49,6 +49,37 @@ void Attr::Describe(ODesc* d) const
 		}
 	}
 
+void Attr::DescribeReST(ODesc* d) const
+	{
+	d->Add(":bro:attr:`");
+	AddTag(d);
+	d->Add("`");
+
+	if ( expr )
+		{
+		d->SP();
+		d->Add("=");
+		d->SP();
+
+		if ( expr->Type()->Tag() == TYPE_FUNC )
+			d->Add(":bro:type:`func`");
+
+		else if ( expr->Type()->Tag() == TYPE_ENUM )
+			{
+			d->Add(":bro:enum:`");
+			expr->Describe(d);
+			d->Add("`");
+			}
+
+		else
+			{
+			d->Add("``");
+			expr->Describe(d);
+			d-> Add("``");
+			}
+		}
+	}
+
 void Attr::AddTag(ODesc* d) const
 	{
 	if ( d->IsBinary() )
@@ -161,6 +192,17 @@ void Attributes::Describe(ODesc* d) const
 		}
 	}
 
+void Attributes::DescribeReST(ODesc* d) const
+	{
+	loop_over_list(*attrs, i)
+		{
+		if ( i > 0 )
+			d->Add(" ");
+
+		(*attrs)[i]->DescribeReST(d);
+		}
+	}
+
 void Attributes::CheckAttr(Attr* a)
 	{
 	switch ( a->Tag() ) {

src/Attr.h

@@ -51,6 +51,7 @@ public:
 		{ return tag == ATTR_REDEF || tag == ATTR_OPTIONAL; }
 
 	void Describe(ODesc* d) const;
+	void DescribeReST(ODesc* d) const;
 
 protected:
 	void AddTag(ODesc* d) const;
@@ -73,6 +74,7 @@ public:
 	void RemoveAttr(attr_tag t);
 
 	void Describe(ODesc* d) const;
+	void DescribeReST(ODesc* d) const;
 
 	attr_list* Attrs()	{ return attrs; }
 

src/BroBifDoc.cc

@@ -0,0 +1,16 @@
+#include <cstdio>
+#include <string>
+#include <list>
+
+#include "BroDoc.h"
+#include "BroBifDoc.h"
+
+BroBifDoc::BroBifDoc(const std::string& sourcename) : BroDoc(sourcename)
+	{
+	}
+
+// TODO: This needs to do something different than parent class's version.
+void BroBifDoc::WriteDocFile() const
+	{
+	BroDoc::WriteDocFile();
+	}

src/BroBifDoc.h

@@ -0,0 +1,18 @@
+#ifndef brobifdoc_h
+#define brobifdoc_h
+
+#include <cstdio>
+#include <string>
+#include <list>
+
+#include "BroDoc.h"
+
+class BroBifDoc : public BroDoc {
+public:
+	BroBifDoc(const std::string& sourcename);
+	virtual ~BroBifDoc()	{ }
+
+	void WriteDocFile() const;
+};
+
+#endif

src/BroDoc.cc

@@ -0,0 +1,361 @@
+#include <cstdio>
+#include <cstdarg>
+#include <string>
+#include <list>
+
+#include "BroDoc.h"
+#include "BroDocObj.h"
+
+BroDoc::BroDoc(const std::string& sourcename)
+	{
+#ifdef DEBUG
+	fprintf(stdout, "Documenting source: %s\n", sourcename.c_str());
+#endif
+	source_filename = sourcename.substr(sourcename.find_last_of('/') + 1);
+
+	size_t ext_pos = source_filename.find_last_of('.');
+	std::string ext = source_filename.substr(ext_pos + 1);
+
+	if ( ext_pos == std::string::npos || ext != "bro" )
+		{
+		if ( source_filename != "bro.init" && source_filename != "<stdin>" )
+			{
+			fprintf(stderr,
+				"Warning: documenting file without .bro extension: %s\n",
+				sourcename.c_str());
+			}
+		else
+			{
+			// Force the reST documentation file to be "bro.init.rst".
+			ext_pos = std::string::npos;
+			}
+		}
+
+	reST_filename = source_filename.substr(0, ext_pos);
+	reST_filename += ".rst";
+	reST_file = fopen(reST_filename.c_str(), "w");
+
+	if ( ! reST_file )
+		fprintf(stderr, "Failed to open %s", reST_filename.c_str());
+#ifdef DEBUG
+	else
+		fprintf(stdout, "Created reST document: %s\n", reST_filename.c_str());
+#endif
+	}
+
+BroDoc::~BroDoc()
+	{
+	if ( reST_file && fclose( reST_file ) )
+		fprintf(stderr, "Failed to close %s", reST_filename.c_str());
+
+	FreeBroDocObjPtrList(all);
+	}
+
+void BroDoc::AddImport(const std::string& s)
+	{
+	size_t ext_pos = s.find_last_of('.');
+
+	if ( ext_pos == std::string::npos )
+		imports.push_back(s);
+
+	else
+		{
+		if ( s.substr(ext_pos + 1) == "bro" )
+			imports.push_back(s.substr(0, ext_pos));
+		else
+			fprintf(stderr, "Warning: skipped documenting @load of file "
+			                "without .bro extension: %s\n", s.c_str());
+		}
+	}
+
+void BroDoc::SetPacketFilter(const std::string& s)
+	{
+	packet_filter = s;
+	size_t pos1 = s.find("{\n");
+	size_t pos2 = s.find("}");
+
+	if ( pos1 != std::string::npos && pos2 != std::string::npos )
+		packet_filter = s.substr(pos1 + 2, pos2 - 2);
+
+	bool has_non_whitespace = false;
+
+	for ( std::string::const_iterator it = packet_filter.begin();
+		it != packet_filter.end(); ++it )
+		{
+		if ( *it != ' ' && *it != '\t' && *it != '\n' && *it != '\r' )
+			{
+			has_non_whitespace = true;
+			break;
+			}
+		}
+
+	if ( ! has_non_whitespace )
+		packet_filter.clear();
+	}
+
+void BroDoc::AddPortAnalysis(const std::string& analyzer,
+			const std::string& ports)
+	{
+	std::string reST_string = analyzer + "::\n" + ports + "\n\n";
+	port_analysis.push_back(reST_string);
+	}
+
+void BroDoc::WriteDocFile() const
+	{
+	WriteToDoc(".. Automatically generated.  Do not edit.\n\n");
+
+	WriteSectionHeading(source_filename.c_str(), '=');
+
+	WriteToDoc("\n:download:`Original Source File <%s>`\n\n",
+		source_filename.c_str());
+
+	WriteSectionHeading("Overview", '-');
+	WriteStringList("%s\n", "%s\n\n", summary);
+
+	if ( ! imports.empty() )
+		{
+		WriteToDoc(":Imports: ");
+		WriteStringList(":doc:`%s`, ", ":doc:`%s`\n", imports);
+		}
+
+	WriteToDoc("\n");
+
+	WriteInterface("Summary", '~', '#', true, true);
+
+	if ( ! modules.empty() )
+		{
+		WriteSectionHeading("Namespaces", '~');
+		WriteStringList(".. bro:namespace:: %s\n", modules);
+		WriteToDoc("\n");
+		}
+
+	if ( ! notices.empty() )
+		WriteBroDocObjList(notices, "Notices", '~');
+
+	WriteInterface("Public Interface", '-', '~', true, false);
+
+	if ( ! port_analysis.empty() )
+		{
+		WriteSectionHeading("Port Analysis", '-');
+		WriteToDoc(":ref:`More Information <common_port_analysis_doc>`\n\n");
+		WriteStringList("%s", port_analysis);
+		}
+
+	if ( ! packet_filter.empty() )
+		{
+		WriteSectionHeading("Packet Filter", '-');
+		WriteToDoc(":ref:`More Information <common_packet_filter_doc>`\n\n");
+		WriteToDoc("Filters added::\n\n");
+		WriteToDoc("%s\n", packet_filter.c_str());
+		}
+
+	BroDocObjList::const_iterator it;
+	bool hasPrivateIdentifiers = false;
+
+	for ( it = all.begin(); it != all.end(); ++it )
+		{
+		if ( ! IsPublicAPI(*it) )
+			{
+			hasPrivateIdentifiers = true;
+			break;
+			}
+		}
+
+	if ( hasPrivateIdentifiers )
+		WriteInterface("Private Interface", '-', '~', false, false);
+	}
+
+void BroDoc::WriteInterface(const char* heading, char underline,
+			char sub, bool isPublic, bool isShort) const
+	{
+	WriteSectionHeading(heading, underline);
+	WriteBroDocObjList(options, isPublic, "Options", sub, isShort);
+	WriteBroDocObjList(constants, isPublic, "Constants", sub, isShort);
+	WriteBroDocObjList(state_vars, isPublic, "State Variables", sub, isShort);
+	WriteBroDocObjList(types, isPublic, "Types", sub, isShort);
+	WriteBroDocObjList(events, isPublic, "Events", sub, isShort);
+	WriteBroDocObjList(functions, isPublic, "Functions", sub, isShort);
+	WriteBroDocObjList(redefs, isPublic, "Redefinitions", sub, isShort);
+	}
+
+void BroDoc::WriteStringList(const char* format, const char* last_format,
+			const std::list<std::string>& l) const
+	{
+	if ( l.empty() )
+		{
+		WriteToDoc("\n");
+		return;
+		}
+
+	std::list<std::string>::const_iterator it;
+	std::list<std::string>::const_iterator last = l.end();
+	last--;
+
+	for ( it = l.begin(); it != last; ++it )
+		WriteToDoc(format, it->c_str());
+
+	WriteToDoc(last_format, last->c_str());
+	}
+
+void BroDoc::WriteBroDocObjTable(const BroDocObjList& l) const
+	{
+	int max_id_col = 0;
+	int max_com_col = 0;
+	BroDocObjList::const_iterator it;
+
+	for ( it = l.begin(); it != l.end(); ++it )
+		{
+		int c = (*it)->ColumnSize();
+
+		if ( c > max_id_col )
+			max_id_col = c;
+
+		c = (*it)->LongestShortDescLen();
+
+		if ( c > max_com_col )
+			max_com_col = c;
+		}
+
+	// Start table.
+	WriteRepeatedChar('=', max_id_col);
+	WriteToDoc(" ");
+
+	if ( max_com_col == 0 )
+		WriteToDoc("=");
+	else
+		WriteRepeatedChar('=', max_com_col);
+
+	WriteToDoc("\n");
+
+	for ( it = l.begin(); it != l.end(); ++it )
+		{
+		if ( it != l.begin() )
+			WriteToDoc("\n\n");
+		(*it)->WriteReSTCompact(reST_file, max_id_col);
+		}
+
+	// End table.
+	WriteToDoc("\n");
+	WriteRepeatedChar('=', max_id_col);
+	WriteToDoc(" ");
+
+	if ( max_com_col == 0 )
+		WriteToDoc("=");
+	else
+		WriteRepeatedChar('=', max_com_col);
+
+	WriteToDoc("\n\n");
+	}
+
+void BroDoc::WriteBroDocObjList(const BroDocObjList& l, bool wantPublic,
+			const char* heading, char underline, bool isShort) const
+	{
+	if ( l.empty() )
+		return;
+
+	BroDocObjList::const_iterator it;
+	bool (*f_ptr)(const BroDocObj* o) = 0;
+
+	if ( wantPublic )
+		f_ptr = IsPublicAPI;
+	else
+		f_ptr = IsPrivateAPI;
+
+	it = std::find_if(l.begin(), l.end(), f_ptr);
+
+	if ( it == l.end() )
+		return;
+
+	WriteSectionHeading(heading, underline);
+
+	BroDocObjList filtered_list;
+
+	while ( it != l.end() )
+		{
+		filtered_list.push_back(*it);
+		it = find_if(++it, l.end(), f_ptr);
+		}
+
+	if ( isShort )
+		WriteBroDocObjTable(filtered_list);
+	else
+		WriteBroDocObjList(filtered_list);
+	}
+
+void BroDoc::WriteBroDocObjList(const BroDocObjMap& m, bool wantPublic,
+			const char* heading, char underline, bool isShort) const
+	{
+	BroDocObjMap::const_iterator it;
+	BroDocObjList l;
+
+	for ( it = m.begin(); it != m.end(); ++it )
+		l.push_back(it->second);
+
+	WriteBroDocObjList(l, wantPublic, heading, underline, isShort);
+	}
+
+void BroDoc::WriteBroDocObjList(const BroDocObjList& l, const char* heading,
+			char underline) const
+	{
+	WriteSectionHeading(heading, underline);
+	WriteBroDocObjList(l);
+	}
+
+void BroDoc::WriteBroDocObjList(const BroDocObjList& l) const
+	{
+	for ( BroDocObjList::const_iterator it = l.begin(); it != l.end(); ++it )
+		(*it)->WriteReST(reST_file);
+	}
+
+void BroDoc::WriteBroDocObjList(const BroDocObjMap& m, const char* heading,
+			char underline) const
+	{
+	BroDocObjMap::const_iterator it;
+	BroDocObjList l;
+
+	for ( it = m.begin(); it != m.end(); ++it )
+		l.push_back(it->second);
+
+	WriteBroDocObjList(l, heading, underline);
+	}
+
+void BroDoc::WriteToDoc(const char* format, ...) const
+	{
+	va_list argp;
+	va_start(argp, format);
+	vfprintf(reST_file, format, argp);
+	va_end(argp);
+	}
+
+void BroDoc::WriteSectionHeading(const char* heading, char underline) const
+	{
+	WriteToDoc("%s\n", heading);
+	WriteRepeatedChar(underline, strlen(heading));
+	WriteToDoc("\n");
+	}
+
+void BroDoc::WriteRepeatedChar(char c, size_t n) const
+	{
+	for ( size_t i = 0; i < n; ++i )
+		WriteToDoc("%c", c);
+	}
+
+void BroDoc::FreeBroDocObjPtrList(BroDocObjList& l)
+	{
+	for ( BroDocObjList::const_iterator it = l.begin(); it != l.end(); ++it )
+		delete *it;
+
+	l.clear();
+	}
+
+void BroDoc::AddFunction(BroDocObj* o)
+	{
+	BroDocObjMap::const_iterator it = functions.find(o->Name());
+	if ( it == functions.end() )
+		{
+		functions[o->Name()] = o;
+		all.push_back(o);
+		}
+	else
+		functions[o->Name()]->Combine(o);
+	}

src/BroDoc.h

@@ -0,0 +1,362 @@
+#ifndef brodoc_h
+#define brodoc_h
+
+#include <cstdio>
+#include <cstdarg>
+#include <string>
+#include <list>
+
+#include "BroDocObj.h"
+
+/**
+ * This class is used to gather all data relevant to the automatic generation
+ * of a reStructuredText (reST) document from a given Bro script.
+ */
+class BroDoc {
+public:
+	/**
+	 * BroDoc constructor
+	 * Given a Bro script, opens new file in the current working directory
+	 * that will contain reST documentation generated from the parsing
+	 * of the Bro script.  The new reST file will be named similar to
+	 * the filename of the Bro script that generates it, except any
+	 * ".bro" file extension is stripped and ".rst" takes it place.
+	 * If the filename doesn't end in ".bro", then ".rst" is just appended.
+	 * @param sourcename The name of the Bro script for which to generate
+	 *        documentation.  May contain a path.
+	 */
+	BroDoc(const std::string& sourcename);
+
+	/**
+	 * BroDoc destructor
+	 * Closes the file that was opened by the constructor and frees up
+	 * memory taken by BroDocObj objects.
+	 */
+	virtual ~BroDoc();
+
+	/**
+	 * Write out full reST documentation for the Bro script that was parsed.
+	 * BroDoc's default implementation of this function will care
+	 * about whether declarations made in the Bro script are part of
+	 * the public versus private interface (whether things are declared in
+	 * the export section).
+	 */
+	virtual void WriteDocFile() const;
+
+	/**
+	 * Schedules some summarizing text to be output directly into the reST doc.
+	 * This should be called whenever the scanner sees a line in the Bro script
+	 * starting with "##!"
+	 * @param s The summary text to add to the reST doc.
+	 */
+	void AddSummary(const std::string& s)	{ summary.push_back(s); }
+
+	/**
+	 * Schedules an import (@load) to be documented.
+	 * If the script being loaded has a .bro suffix, it is internally stripped.
+	 * This should be called whenever the scanner sees an @load.
+	 * @param s The name of the imported script.
+	 */
+	void AddImport(const std::string& s);
+
+	/**
+	 * Schedules a namespace (module) to be documented.
+	 * This should be called whenever the parser sees a TOK_MODULE.
+	 * @param s The namespace (module) identifier's name.
+	 */
+	void AddModule(const std::string& s)	{ modules.push_back(s); }
+
+	/**
+	 * Sets the way the script changes the "capture_filters" table.
+	 * This is determined by the scanner checking for changes to
+	 * the "capture_filters" table after each of Bro's input scripts
+	 * (given as command line arguments to Bro) are finished being parsed.
+	 * @param s The value "capture_filters" as given by TableVal::Describe()
+	 */
+	void SetPacketFilter(const std::string& s);
+
+	/**
+	 * Schedules documentation of a given set of ports being associated
+	 * with a particular analyzer as a result of the current script
+	 * being loaded -- the way the "dpd_config" table is changed.
+	 * @param analyzer An analyzer that changed the "dpd_config" table.
+	 * @param ports The set of ports assigned to the analyzer in table.
+	 */
+	void AddPortAnalysis(const std::string& analyzer, const std::string& ports);
+
+	/**
+	 * Schedules documentation of a script option.  An option is
+	 * defined as any variable in the script that is declared 'const'
+	 * and has the '&redef' attribute.
+	 * @param o A pointer to a BroDocObj which contains the internal
+	 *        Bro language representation of the script option and
+	 *        also any associated comments about it.
+	 */
+	void AddOption(const BroDocObj* o)
+		{
+		options.push_back(o);
+		all.push_back(o);
+		}
+
+	/**
+	 * Schedules documentation of a script constant.  An option is
+	 * defined as any variable in the script that is declared 'const'
+	 * and does *not* have the '&redef' attribute.
+	 * @param o A pointer to a BroDocObj which contains the internal
+	 *        Bro language representation of the script constant and
+	 *        also any associated comments about it.
+	 */
+	void AddConstant(const BroDocObj* o)
+		{
+		constants.push_back(o);
+		all.push_back(o);
+		}
+
+	/**
+	 * Schedules documentation of a script state variable.  A state variable
+	 * is defined as any variable in the script that is declared 'global'
+	 * @param o A pointer to a BroDocObj which contains the internal
+	 *        Bro language representation of the script state variable
+	 *        and also any associated comments about it.
+	 */
+	void AddStateVar(const BroDocObj* o)
+		{
+		state_vars.push_back(o);
+		all.push_back(o);
+		}
+
+	/**
+	 * Schedules documentation of a type declared by the script.
+	 * @param o A pointer to a BroDocObj which contains the internal
+	 *        Bro language representation of the script option and
+	 *        also any associated comments about it.
+	 */
+	void AddType(const BroDocObj* o)
+		{
+		types.push_back(o);
+		all.push_back(o);
+		}
+
+	/**
+	 * Schedules documentation of a Notice (enum redef) declared by script
+	 * @param o A pointer to a BroDocObj which contains the internal
+	 *        Bro language representation of the Notice and also
+	 *        any associated comments about it.
+	 */
+	void AddNotice(const BroDocObj* o)
+		{
+		notices.push_back(o);
+		all.push_back(o);
+		}
+
+	/**
+	 * Schedules documentation of an event declared by the script.
+	 * @param o A pointer to a BroDocObj which contains the internal
+	 *        Bro language representation of the script event and
+	 *        also any associated comments about it.
+	 */
+	void AddEvent(const BroDocObj* o)
+		{
+		events.push_back(o);
+		all.push_back(o);
+		}
+
+	/**
+	 * Schedules documentation of a function declared by the script.
+	 * @param o A pointer to a BroDocObj which contains the internal
+	 *        Bro language representation of the script function and
+	 *        also any associated comments about it.
+	 */
+	void AddFunction(BroDocObj* o);
+
+	/**
+	 * Schedules documentation of a redef done by the script
+	 * @param o A pointer to a BroDocObj which contains the internal
+	 *        Bro language representation of the script identifier
+	 *        that was redefined and also any associated comments.
+	 */
+	void AddRedef(const BroDocObj* o)
+		{
+		redefs.push_back(o);
+		all.push_back(o);
+		}
+
+	/**
+	 * Gets the name of the Bro script source file for which reST
+	 * documentation is being generated.
+	 * @return A char* to the start of the source file's name.
+	 */
+	const char* GetSourceFileName() const
+		{
+		return source_filename.c_str();
+		}
+
+	/**
+	 * Gets the name of the generated reST documentation file.
+	 * @return A char* to the start of the generated reST file's name.
+	 */
+	const char* GetOutputFileName() const
+		{
+		return reST_filename.c_str();
+		}
+
+protected:
+	FILE* reST_file;
+	std::string reST_filename;
+	std::string source_filename;
+	std::string packet_filter;
+
+	std::list<std::string> modules;
+	std::list<std::string> summary;
+	std::list<std::string> imports;
+	std::list<std::string> port_analysis;
+
+	typedef std::list<const BroDocObj*> BroDocObjList;
+	typedef std::map<std::string, BroDocObj*> BroDocObjMap;
+
+	BroDocObjList options;
+	BroDocObjList constants;
+	BroDocObjList state_vars;
+	BroDocObjList types;
+	BroDocObjList notices;
+	BroDocObjList events;
+	BroDocObjMap functions;
+	BroDocObjList redefs;
+
+	BroDocObjList all;
+
+	/**
+	 * Writes out a list of strings to the reST document.
+	 * If the list is empty, prints a newline character.
+	 * @param format A printf style format string for elements of the list
+	 *        except for the last one in the list
+	 * @param last_format A printf style format string to use for the last
+	 *        element of the list
+	 * @param l A reference to a list of strings
+	 */
+	void WriteStringList(const char* format, const char* last_format,
+			const std::list<std::string>& l) const;
+
+	/**
+	 * @see WriteStringList(const char*, const char*,
+	 *                      const std::list<std::string>&>)
+	 */
+	void WriteStringList(const char* format,
+			const std::list<std::string>& l) const
+		{
+		WriteStringList(format, format, l);
+		}
+
+
+	/**
+	 * Writes out a table of BroDocObj's to the reST document
+	 * @param l A list of BroDocObj pointers
+	 */
+	void WriteBroDocObjTable(const BroDocObjList& l) const;
+
+	/**
+	 * Writes out a list of BroDocObj objects to the reST document
+	 * @param l A list of BroDocObj pointers
+	 * @param wantPublic If true, filter out objects that are not declared
+	 *        in the global scope.  If false, filter out those that are in
+	 *        the global scope.
+	 * @param heading The title of the section to create in the reST doc.
+	 * @param underline The character to use to underline the reST
+	 *        section heading.
+	 * @param isShort Whether to write the full documentation or a "short"
+	 *        version (a single sentence)
+	 */
+	void WriteBroDocObjList(const BroDocObjList& l, bool wantPublic,
+			const char* heading, char underline,
+			bool isShort) const;
+
+	/**
+	 * Wraps the BroDocObjMap into a BroDocObjList and the writes that list
+	 * to the reST document
+	 * @see WriteBroDocObjList(const BroDocObjList&, bool, const char*, char,
+	        bool)
+	 */
+	void WriteBroDocObjList(const BroDocObjMap& m, bool wantPublic,
+			const char* heading, char underline,
+			bool isShort) const;
+
+	/**
+	 * Writes out a list of BroDocObj objects to the reST document
+	 * @param l A list of BroDocObj pointers
+	 * @param heading The title of the section to create in the reST doc.
+	 * @param underline The character to use to underline the reST
+	 *        section heading.
+	 */
+	void WriteBroDocObjList(const BroDocObjList& l, const char* heading,
+			char underline) const;
+
+	/**
+	 * Writes out a list of BroDocObj objects to the reST document
+	 * @param l A list of BroDocObj pointers
+	 */
+	void WriteBroDocObjList(const BroDocObjList& l) const;
+
+	/**
+	 * Wraps the BroDocObjMap into a BroDocObjList and the writes that list
+	 * to the reST document
+	 * @see WriteBroDocObjList(const BroDocObjList&, const char*, char)
+	 */
+	void WriteBroDocObjList(const BroDocObjMap& m, const char* heading,
+			char underline) const;
+
+	/**
+	 * A wrapper to fprintf() that always uses the reST document
+	 * for the FILE* argument.
+	 * @param format A printf style format string.
+	 */
+	void WriteToDoc(const char* format, ...) const;
+
+	/**
+	 * Writes out a reST section heading
+	 * @param heading The title of the heading to create
+	 * @param underline The character to use to underline the section title
+	 *        within the reST document
+	 */
+	void WriteSectionHeading(const char* heading, char underline) const;
+
+	/**
+	 * Writes out given number of characters to reST document
+	 * @param c the character to write
+	 * @param n the number of characters to write
+	 */
+	void WriteRepeatedChar(char c, size_t n) const;
+
+	/**
+	 * Writes out the reST for either the script's public or private interface
+	 * @param heading The title of the interfaces section heading
+	 * @param underline The underline character to use for the interface
+	 *        section
+	 * @param subunderline The underline character to use for interface
+	 *        sub-sections
+	 * @param isPublic Whether to write out the public or private script
+	 *        interface
+	 * @param isShort Whether to write out the full documentation or a "short"
+	 *        description (a single sentence)
+	 */
+	void WriteInterface(const char* heading, char underline, char subunderline,
+			bool isPublic, bool isShort) const;
+private:
+
+	/**
+	 * Frees memory allocated to BroDocObj's objects in a given list.
+	 * @param a reference to a list of BroDocObj pointers
+	 */
+	void FreeBroDocObjPtrList(BroDocObjList& l);
+
+	static bool IsPublicAPI(const BroDocObj* o)
+		{
+		return o->IsPublicAPI();
+		}
+
+	static bool IsPrivateAPI(const BroDocObj* o)
+		{
+		return ! o->IsPublicAPI();
+		}
+};
+
+#endif

src/BroDocObj.cc

@@ -0,0 +1,168 @@
+#include <cstdio>
+#include <string>
+#include <list>
+#include "ID.h"
+#include "BroDocObj.h"
+
+BroDocObj::BroDocObj(const ID* id, std::list<std::string>*& reST,
+			bool is_fake)
+	{
+	broID = id;
+	reST_doc_strings = reST;
+	reST = 0;
+	is_fake_id = is_fake;
+	use_role = 0;
+	FormulateShortDesc();
+	}
+
+BroDocObj::~BroDocObj()
+	{
+	if ( reST_doc_strings )
+		delete reST_doc_strings;
+
+	if ( is_fake_id )
+		delete broID;
+	}
+
+void BroDocObj::WriteReSTCompact(FILE* file, int max_col) const
+	{
+	ODesc desc;
+	desc.SetQuotes(1);
+	broID->DescribeReSTShort(&desc);
+
+	fprintf(file, "%s", desc.Description());
+
+	std::list<std::string>::const_iterator it;
+
+	for ( it = short_desc.begin(); it != short_desc.end(); ++it )
+		{
+		int start_col;
+
+		if ( it == short_desc.begin() )
+			start_col = max_col - desc.Len() + 1;
+		else
+			{
+			start_col = max_col + 1;
+			fprintf(file, "\n");
+			}
+
+		for ( int i = 0; i < start_col; ++i )
+			fprintf(file, " ");
+
+		fprintf(file, "%s", it->c_str());
+		}
+	}
+
+int BroDocObj::LongestShortDescLen() const
+	{
+	size_t max = 0;
+
+	std::list<std::string>::const_iterator it;
+
+	for ( it = short_desc.begin(); it != short_desc.end(); ++it )
+		{
+		if ( it->size() > max )
+			max = it->size();
+		}
+
+	return max;
+	}
+
+void BroDocObj::FormulateShortDesc()
+	{
+	if ( ! reST_doc_strings )
+		return;
+
+	short_desc.clear();
+	std::list<std::string>::const_iterator it;
+
+	for ( it = reST_doc_strings->begin();
+		it != reST_doc_strings->end(); ++it )
+		{
+		// The short description stops at the first sentence or the
+		// first empty comment.
+		size_t end = it->find_first_of(".");
+
+		if ( end == string::npos )
+			{
+			std::string::const_iterator s;
+			bool empty = true;
+
+			for ( s = it->begin(); s != it->end(); ++s )
+				{
+				if ( *s != ' ' && *s != '\t' && *s != '\n' && *s != '\r' )
+					{
+					empty = false;
+					short_desc.push_back(*it);
+					break;
+					}
+				}
+			
+			if ( empty )
+				break;
+			}
+		else
+			{
+			short_desc.push_back(it->substr(0, end + 1));
+			break;
+			}
+		}
+	}
+
+void BroDocObj::WriteReST(FILE* file) const
+	{
+	int indent_spaces = 3;
+	ODesc desc;
+	desc.SetIndentSpaces(indent_spaces);
+	desc.SetQuotes(1);
+
+	broID->DescribeReST(&desc, use_role);
+
+	fprintf(file, "%s", desc.Description());
+
+	if ( HasDocumentation() )
+		{
+		fprintf(file, "\n");
+		std::list<std::string>::const_iterator it;
+
+		for ( it = reST_doc_strings->begin();
+			it != reST_doc_strings->end(); ++it)
+			{
+			for ( int i = 0; i < indent_spaces; ++i )
+				fprintf(file, " ");
+
+			fprintf(file, "%s\n", it->c_str());
+			}
+		}
+
+	fprintf(file, "\n");
+	}
+
+int BroDocObj::ColumnSize() const
+	{
+	ODesc desc;
+	desc.SetQuotes(1);
+	broID->DescribeReSTShort(&desc);
+	return desc.Len();
+	}
+
+bool BroDocObj::IsPublicAPI() const
+	{
+	return (broID->Scope() == SCOPE_GLOBAL) ||
+		(broID->Scope() == SCOPE_MODULE && broID->IsExport());
+	}
+
+void BroDocObj::Combine(const BroDocObj* o)
+	{
+	if ( o->reST_doc_strings )
+		{
+		if ( ! reST_doc_strings )
+			reST_doc_strings = new std::list<std::string>();
+
+		reST_doc_strings->splice(reST_doc_strings->end(),
+			*(o->reST_doc_strings));
+		}
+
+	delete o;
+	FormulateShortDesc();
+	}

src/BroDocObj.h

@@ -0,0 +1,123 @@
+#ifndef brodocobj_h
+#define brodocobj_h
+
+#include <cstdio>
+#include <string>
+#include <list>
+
+#include "ID.h"
+
+/**
+ * This class wraps a Bro script identifier, providing methods relevant
+ * to automatic generation of reStructuredText (reST) documentation for it.
+ */
+class BroDocObj {
+public:
+	/**
+	 * BroDocObj constructor
+	 * @param id a pointer to an identifier that is to be documented
+	 * @param reST a reference to a pointer of a list of strings that
+	 *        represent the reST documentation for the ID.  The pointer
+	 *        will be set to 0 after this constructor finishes.
+	 * @param is_fake whether the ID* is a dummy just for doc purposes
+	 */
+	BroDocObj(const ID* id, std::list<std::string>*& reST,
+			bool is_fake = false);
+
+	/**
+	 * BroDocObj destructor
+	 * Deallocates the memory associated with the list of reST strings
+	 */
+	~BroDocObj();
+
+	/**
+	 * Writes the reST representation of this object which includes
+	 * 1) a reST friendly description of the ID
+	 * 2) "##" or "##<" stylized comments.
+	 *    Anything after these style of comments is inserted as-is into
+	 *    the reST document.
+	 * @param file The (already opened) file to write the reST to.
+	 */
+	void WriteReST(FILE* file) const;
+
+	/**
+	 * Writes a compact version of the ID and associated documentation
+	 * for insertion into a table.
+	 * @param file The (already opened) file to write the reST to.
+	 * @param max_col The maximum length of the first table column
+	 */
+	void WriteReSTCompact(FILE* file, int max_col) const;
+
+	/**
+	 * @return the column size required by the reST representation of the ID
+	 */
+	int ColumnSize() const;
+
+	/**
+	 * Check whether this documentation is part of the public API.  In
+	 * other words, this means that the identifier is declared as part of
+	 * the global scope (has GLOBAL namespace or is exported from another
+	 * namespace).
+	 * @return true if the identifier is part of the script's public API
+	 */
+	bool IsPublicAPI() const;
+
+	/**
+	 * Return whether this object has documentation (## comments)
+	 * @return true if the ID has comments associated with it
+	 */
+	bool HasDocumentation() const
+		{
+		return reST_doc_strings && reST_doc_strings->size() > 0;
+		}
+
+	/**
+	 * @return whether this object will use reST role (T) or directive (F)
+	 * notation for the wrapped identifier.  Roles are usually used
+	 * for cross-referencing.
+	 */
+	bool UseRole() const	{ return use_role; }
+
+	/**
+	 * @param b whether this object will use reST role (T) or directive (F)
+	 * notation for the wrapped identifier.  Roles are usually used
+	 * for cross-referencing.
+	 */
+	void SetRole(bool b)	{ use_role = b; }
+
+	/**
+	 * Append any reST documentation strings in a given BroDocObj to this
+	 * object's list and then delete the given BroDocObj
+	 * @param o a pointer to a BroDocObj to subsume
+	 */
+	void Combine(const BroDocObj* o);
+
+	/**
+	 * @return the name of the wrapped identifier
+	 */
+	const char* Name() const	{ return broID->Name(); }
+
+	/**
+	 * @return the longest string element of the short description's list of
+	 *         strings
+	 */
+	int LongestShortDescLen() const;
+
+protected:
+	std::list<std::string>* reST_doc_strings;
+	std::list<std::string> short_desc;
+	const ID* broID;
+	bool is_fake_id; /**< Whether the ID* is a dummy just for doc purposes */
+	bool use_role; /**< Whether to use a reST role or directive for the ID */
+
+	/**
+	 * Set the short_desc member to be a subset of reST_doc_strings.
+	 * Specifically, short_desc will be everything in reST_doc_strings
+	 * up until the first period or first empty string list element found.
+	 */
+	void FormulateShortDesc();
+
+private:
+};
+
+#endif

src/CMakeLists.txt

@@ -285,6 +285,9 @@ set(bro_SRCS
     BitTorrent.cc
     BitTorrentTracker.cc
     BPF_Program.cc
+    BroBifDoc.cc
+    BroDoc.cc
+    BroDocObj.cc
     BroString.cc
     CCL.cc
     ChunkedIO.cc

src/Desc.cc

@@ -41,6 +41,7 @@ ODesc::ODesc(desc_type t, BroFile* arg_f)
 	want_quotes = 0;
 	do_flush = 1;
 	include_stats = 0;
+	indent_with_spaces = 0;
 	}
 
 ODesc::~ODesc()
@@ -67,6 +68,12 @@ void ODesc::PopIndent()
 	NL();
 	}
 
+void ODesc::PopIndentNoNL()
+	{
+	if ( --indent_level < 0 )
+		internal_error("ODesc::PopIndent underflow");
+	}
+
 void ODesc::Add(const char* s, int do_indent)
 	{
 	unsigned int n = strlen(s);
@@ -178,10 +185,19 @@ void ODesc::AddBytes(const BroString* s)
 	}
 
 void ODesc::Indent()
+	{
+	if ( indent_with_spaces > 0 )
+		{
+		for ( int i = 0; i < indent_level; ++i )
+			for ( int j = 0; j < indent_with_spaces; ++j )
+				Add(" ", 0);
+		}
+	else
 		{
 		for ( int i = 0; i < indent_level; ++i )
 			Add("\t", 0);
 		}
+	}
 
 
 void ODesc::AddBytes(const void* bytes, unsigned int n)

src/Desc.h

@@ -51,8 +51,12 @@ public:
 
 	void PushIndent();
 	void PopIndent();
+	void PopIndentNoNL();
 	int GetIndentLevel() const	{ return indent_level; }
 
+	int IndentSpaces() const	{ return indent_with_spaces; }
+	void SetIndentSpaces(int i)	{ indent_with_spaces = i; }
+
 	void Add(const char* s, int do_indent=1);
 	void AddN(const char* s, int len)	{ AddBytes(s, len); }
 	void Add(int i);
@@ -135,6 +139,7 @@ protected:
 	int want_quotes;
 	int do_flush;
 	int include_stats;
+	int indent_with_spaces;
 };
 
 #endif

src/ID.cc

@@ -607,6 +607,135 @@ void ID::DescribeExtended(ODesc* d) const
 		}
 	}
 
+void ID::DescribeReSTShort(ODesc* d) const
+	{
+	if ( is_type )
+		d->Add(":bro:type:`");
+	else
+		d->Add(":bro:id:`");
+
+	d->Add(name);
+	d->Add("`");
+
+	if ( type )
+		{
+		d->Add(": ");
+		d->Add(":bro:type:`");
+
+		if ( ! is_type && type->GetTypeID() )
+			d->Add(type->GetTypeID());
+		else
+			{
+			TypeTag t = type->Tag();
+
+			switch ( t ) {
+			case TYPE_TABLE:
+				d->Add(type->IsSet() ? "set" : type_name(t));
+				break;
+
+			case TYPE_FUNC:
+				d->Add(type->AsFuncType()->IsEvent() ? "event" : type_name(t));
+				break;
+
+			default:
+				d->Add(type_name(t));
+			}
+			}
+
+		d->Add("`");
+		}
+
+	if ( attrs )
+		{
+		d->SP();
+		attrs->DescribeReST(d);
+		}
+	}
+
+void ID::DescribeReST(ODesc* d, bool is_role) const
+	{
+	if ( is_role )
+		{
+		if ( is_type )
+			d->Add(":bro:type:`");
+		else
+			d->Add(":bro:id:`");
+		d->Add(name);
+		d->Add("`");
+		}
+	else
+		{
+		if ( is_type )
+			d->Add(".. bro:type:: ");
+		else
+			d->Add(".. bro:id:: ");
+		d->Add(name);
+		}
+
+	d->PushIndent();
+	d->NL();
+
+	if ( type )
+		{
+		d->Add(":Type: ");
+
+		if ( ! is_type && type->GetTypeID() )
+			{
+			d->Add(":bro:type:`");
+			d->Add(type->GetTypeID());
+			d->Add("`");
+			}
+		else
+			type->DescribeReST(d);
+
+		d->NL();
+		}
+
+	if ( attrs )
+		{
+		d->Add(":Attributes: ");
+		attrs->DescribeReST(d);
+		d->NL();
+		}
+
+	if ( val && type &&
+		type->Tag() != TYPE_FUNC &&
+		type->InternalType() != TYPE_INTERNAL_VOID )
+		{
+		d->Add(":Default:");
+
+		if ( type->InternalType() == TYPE_INTERNAL_OTHER )
+			{
+			switch ( type->Tag() ) {
+			case TYPE_TABLE:
+				if ( val->AsTable()->Length() == 0 )
+					{
+					d->Add(" ``{}``");
+					d->NL();
+					break;
+					}
+				// Fall-through.
+
+			default:
+				d->NL();
+				d->NL();
+				d->Add("::");
+				d->NL();
+				d->PushIndent();
+				val->DescribeReST(d);
+				d->PopIndent();
+			}
+			}
+
+		else
+			{
+			d->SP();
+			val->DescribeReST(d);
+			d->NL();
+			}
+		}
+	}
+
 #ifdef DEBUG
 void ID::UpdateValID()
 	{

src/ID.h

@@ -84,6 +84,9 @@ public:
 	void Describe(ODesc* d) const;
 	// Adds type and value to description.
 	void DescribeExtended(ODesc* d) const;
+	// Produces a description that's reST-ready.
+	void DescribeReST(ODesc* d, bool is_role=false) const;
+	void DescribeReSTShort(ODesc* d) const;
 
 	bool Serialize(SerialInfo* info) const;
 	static ID* Unserialize(UnserialInfo* info);

src/SerialObj.h

@@ -330,6 +330,29 @@ public:
 		dst = 0;	\
 	}
 
+#define UNSERIALIZE_OPTIONAL_STR_DEL(dst, del)	\
+	{	\
+	bool has_it;	\
+	if ( ! info->s->Read(&has_it, "has_" #dst) )	\
+		{	\
+		delete del;	\
+		return 0;	\
+		}	\
+	\
+	if ( has_it )	\
+		{	\
+		info->s->Read(&dst, 0, "has_" #dst);	\
+		if ( ! dst )	\
+			{	\
+			delete del;	\
+			return 0;	\
+			}	\
+		}	\
+	\
+	else	\
+		dst = 0;	\
+	}
+
 #define UNSERIALIZE_OPTIONAL_STATIC(dst, unserialize, del)	\
 	{	\
 	bool has_it;	\

src/Type.cc

@@ -10,6 +10,12 @@
 #include "Scope.h"
 #include "Serializer.h"
 
+#include <string>
+#include <list>
+#include <map>
+
+extern int generate_documentation;
+
 const char* type_name(TypeTag t)
 	{
 	static char errbuf[512];
@@ -44,6 +50,7 @@ BroType::BroType(TypeTag t, bool arg_base_type)
 	tag = t;
 	is_network_order = 0;
 	base_type = arg_base_type;
+	type_id = 0;
 
 	switch ( tag ) {
 	case TYPE_VOID:
@@ -105,6 +112,12 @@ BroType::BroType(TypeTag t, bool arg_base_type)
 
 	}
 
+BroType::~BroType()
+	{
+	if ( type_id )
+		delete [] type_id;
+	}
+
 int BroType::MatchesIndex(ListExpr*& /* index */) const
 	{
 	return DOES_NOT_MATCH_INDEX;
@@ -139,6 +152,13 @@ void BroType::Describe(ODesc* d) const
 		}
 	}
 
+void BroType::DescribeReST(ODesc* d) const
+	{
+	d->Add(":bro:type:`");
+	d->Add(type_name(Tag()));
+	d->Add("`");
+	}
+
 void BroType::SetError()
 	{
 	tag = TYPE_ERROR;
@@ -195,8 +215,9 @@ BroType* BroType::Unserialize(UnserialInfo* info, TypeTag want)
 	if ( ! t )
 		return 0;
 
-	// For base types, we return our current instance.
-	if ( t->base_type )
+	// For base types, we return our current instance
+	// if not in "documentation mode".
+	if ( t->base_type && ! generate_documentation )
 		{
 		BroType* t2 = ::base_type(TypeTag(t->tag));
 		Unref(t);
@@ -230,6 +251,11 @@ bool BroType::DoSerialize(SerialInfo* info) const
 	void* null = NULL;
 	SERIALIZE(null);
 
+	if ( generate_documentation )
+		{
+		SERIALIZE_OPTIONAL_STR(type_id);
+		}
+
 	info->s->WriteCloseTag("Type");
 
 	return true;
@@ -260,6 +286,11 @@ bool BroType::DoUnserialize(UnserialInfo* info)
 	// attributes_type" for backwards compatibility.
 	UNSERIALIZE_OPTIONAL(not_used_either, BroType::Unserialize(info, TYPE_RECORD));
 
+	if ( generate_documentation )
+		{
+		UNSERIALIZE_OPTIONAL_STR(type_id);
+		}
+
 	return true;
 	}
 
@@ -413,6 +444,52 @@ void IndexType::Describe(ODesc* d) const
 		}
 	}
 
+void IndexType::DescribeReST(ODesc* d) const
+	{
+	d->Add(":bro:type:`");
+
+	if ( IsSet() )
+		d->Add("set");
+	else
+		d->Add(type_name(Tag()));
+
+	d->Add("` ");
+	d->Add("[");
+
+	loop_over_list(*IndexTypes(), i)
+		{
+		if ( i > 0 )
+			d->Add(", ");
+
+		const BroType* t = (*IndexTypes())[i];
+
+		if ( t->GetTypeID() )
+			{
+			d->Add(":bro:type:`");
+			d->Add(t->GetTypeID());
+			d->Add("`");
+			}
+		else
+			t->DescribeReST(d);
+		}
+
+	d->Add("]");
+
+	if ( yield_type )
+		{
+		d->Add(" of ");
+
+		if ( yield_type->GetTypeID() )
+			{
+			d->Add(":bro:type:`");
+			d->Add(yield_type->GetTypeID());
+			d->Add("`");
+			}
+		else
+			yield_type->DescribeReST(d);
+		}
+	}
+
 bool IndexType::IsSubNetIndex() const
 	{
 	const type_list* types = indices->Types();
@@ -647,6 +724,30 @@ void FuncType::Describe(ODesc* d) const
 		}
 	}
 
+void FuncType::DescribeReST(ODesc* d) const
+	{
+	d->Add(":bro:type:`");
+	d->Add(is_event ? "event" : "function");
+	d->Add("`");
+	d->Add(" (");
+	args->DescribeFieldsReST(d, true);
+	d->Add(")");
+
+	if ( yield )
+		{
+		d->AddSP(" :");
+
+		if ( yield->GetTypeID() )
+			{
+			d->Add(":bro:type:`");
+			d->Add(yield->GetTypeID());
+			d->Add("`");
+			}
+		else
+			yield->DescribeReST(d);
+		}
+	}
+
 IMPLEMENT_SERIAL(FuncType, SER_FUNC_TYPE);
 
 bool FuncType::DoSerialize(SerialInfo* info) const
@@ -701,7 +802,10 @@ bool TypeDecl::Serialize(SerialInfo* info) const
 
 	SERIALIZE_OPTIONAL(attrs);
 
-	return type->Serialize(info) && SERIALIZE(id);
+	if ( ! (type->Serialize(info) && SERIALIZE(id)) )
+		return false;
+
+	return true;
 	}
 
 TypeDecl* TypeDecl::Unserialize(UnserialInfo* info)
@@ -720,6 +824,58 @@ TypeDecl* TypeDecl::Unserialize(UnserialInfo* info)
 	return t;
 	}
 
+void TypeDecl::DescribeReST(ODesc* d) const
+	{
+	d->Add(id);
+	d->Add(": ");
+
+	if ( type->GetTypeID() )
+		{
+		d->Add(":bro:type:`");
+		d->Add(type->GetTypeID());
+		d->Add("`");
+		}
+	else
+		type->DescribeReST(d);
+
+	if ( attrs )
+		{
+		d->SP();
+		attrs->DescribeReST(d);
+		}
+	}
+
+CommentedTypeDecl::CommentedTypeDecl(BroType* t, const char* i,
+			attr_list* attrs, std::list<std::string>* cmnt_list)
+	: TypeDecl(t, i, attrs)
+	{
+	comments = cmnt_list;
+	}
+
+CommentedTypeDecl::~CommentedTypeDecl()
+	{
+	if ( comments ) delete comments;
+	}
+
+void CommentedTypeDecl::DescribeReST(ODesc* d) const
+	{
+	TypeDecl::DescribeReST(d);
+
+	if ( comments )
+		{
+		d->PushIndent();
+		std::list<std::string>::const_iterator i;
+
+		for ( i = comments->begin(); i != comments->end(); ++i)
+			{
+			if ( i != comments->begin() ) d->NL();
+			d->Add(i->c_str());
+			}
+
+		d->PopIndentNoNL();
+		}
+	}
+
 RecordField::RecordField(int arg_base, int arg_offset, int arg_total_offset)
 	{
 	base = arg_base;
@@ -755,9 +911,11 @@ void RecordType::Init(TypeList* arg_base)
 	types = 0;
 
 	type_list* t = base->Types();
+
 	loop_over_list(*t, i)
 		{
 		BroType* ti = (*t)[i];
+
 		if ( ti->Tag() != TYPE_RECORD )
 			(*t)[i]->Error("non-record in base type list");
 
@@ -767,6 +925,7 @@ void RecordType::Init(TypeList* arg_base)
 		for ( int j = 0; j < n; ++j )
 			{
 			const TypeDecl* tdij = rti->FieldDecl(j);
+
 			if ( fields->Lookup(tdij->id) )
 				{
 				error("duplicate field", tdij->id);
@@ -774,6 +933,7 @@ void RecordType::Init(TypeList* arg_base)
 				}
 
 			RecordField* rf = new RecordField(i, j, fields->Length());
+
 			if ( fields->Insert(tdij->id, rf) )
 				Internal("duplicate field when constructing record");
 			}
@@ -788,6 +948,7 @@ RecordType::~RecordType()
 		{
 		loop_over_list(*types, i)
 			delete (*types)[i];
+
 		delete types;
 		}
 
@@ -898,6 +1059,13 @@ void RecordType::Describe(ODesc* d) const
 		}
 	}
 
+void RecordType::DescribeReST(ODesc* d) const
+	{
+	d->Add(":bro:type:`record`");
+	d->NL();
+	DescribeFieldsReST(d, false);
+	}
+
 void RecordType::DescribeFields(ODesc* d) const
 	{
 	if ( d->IsReadable() )
@@ -937,6 +1105,29 @@ void RecordType::DescribeFields(ODesc* d) const
 		}
 	}
 
+void RecordType::DescribeFieldsReST(ODesc* d, bool func_args) const
+	{
+	if ( ! func_args )
+		d->PushIndent();
+
+	for ( int i = 0; i < num_fields; ++i )
+		{
+		if ( i > 0 )
+			if ( func_args )
+				d->Add(", ");
+			else
+				{
+				d->NL();
+				d->NL();
+				}
+
+		FieldDecl(i)->DescribeReST(d);
+		}
+
+	if ( ! func_args )
+		d->PopIndentNoNL();
+	}
+
 IMPLEMENT_SERIAL(RecordType, SER_RECORD_TYPE)
 
 bool RecordType::DoSerialize(SerialInfo* info) const
@@ -1094,6 +1285,15 @@ EnumType::~EnumType()
 		delete [] iter->first;
 	}
 
+CommentedEnumType::~CommentedEnumType()
+	{
+	for ( CommentMap::iterator iter = comments.begin(); iter != comments.end(); ++iter )
+		{
+		delete [] iter->first;
+		delete iter->second;
+		}
+	}
+
 // Note, we use error() here (not Error()) to include the current script
 // location in the error message, rather than the one where the type was
 // originally defined.
@@ -1123,6 +1323,26 @@ void EnumType::AddName(const string& module_name, const char* name, bro_int_t va
 	AddNameInternal(module_name, name, val, is_export);
 	}
 
+void CommentedEnumType::AddComment(const string& module_name, const char* name,
+                                   std::list<std::string>* new_comments)
+	{
+	if ( ! new_comments )
+		return;
+
+	string fullname = make_full_var_name(module_name.c_str(), name);
+
+	CommentMap::iterator it = comments.find(fullname.c_str());
+
+	if ( it == comments.end() )
+		comments[copy_string(fullname.c_str())] = new_comments;
+	else
+		{
+		comments[fullname.c_str()]->splice(comments[fullname.c_str()]->end(),
+			*new_comments);
+		delete new_comments;
+		}
+	}
+
 void EnumType::AddNameInternal(const string& module_name, const char* name, bro_int_t val, bool is_export)
 	{
 	ID *id;
@@ -1151,6 +1371,12 @@ void EnumType::AddNameInternal(const string& module_name, const char* name, bro_
 	names[copy_string(fullname.c_str())] = val;
 	}
 
+void CommentedEnumType::AddNameInternal(const string& module_name, const char* name, bro_int_t val, bool is_export)
+	{
+	string fullname = make_full_var_name(module_name.c_str(), name);
+	names[copy_string(fullname.c_str())] = val;
+	}
+
 bro_int_t EnumType::Lookup(const string& module_name, const char* name)
 	{
 	NameMap::iterator pos =
@@ -1172,6 +1398,51 @@ const char* EnumType::Lookup(bro_int_t value)
 	return 0;
 	}
 
+void CommentedEnumType::DescribeReST(ODesc* d) const
+	{
+	// create temporary, reverse name map so that enums can be documented
+	// in ascending order of their actual integral value instead of by name
+	typedef std::map< bro_int_t, const char* > RevNameMap;
+	RevNameMap rev;
+	for ( NameMap::const_iterator it = names.begin(); it != names.end(); ++it )
+		rev[it->second] = it->first;
+
+	d->Add(":bro:type:`");
+	d->Add(type_name(Tag()));
+	d->Add("`");
+	d->PushIndent();
+	d->NL();
+
+	for ( RevNameMap::const_iterator it = rev.begin(); it != rev.end(); ++it )
+		{
+		if ( it != rev.begin() )
+			{
+			d->NL();
+			d->NL();
+			}
+
+		d->Add(".. bro:enum:: ");
+		d->AddSP(it->second);
+		d->Add(GetTypeID());
+
+		CommentMap::const_iterator cmnt_it = comments.find(it->second);
+		if ( cmnt_it != comments.end() )
+			{
+			d->PushIndent();
+			d->NL();
+			std::list<std::string>::const_iterator i;
+			const std::list<std::string>* cmnt_list = cmnt_it->second;
+			for ( i = cmnt_list->begin(); i != cmnt_list->end(); ++i)
+				{
+				if ( i != cmnt_list->begin() ) d->NL();
+				d->Add(i->c_str());
+				}
+			d->PopIndentNoNL();
+			}
+		}
+	d->PopIndentNoNL();
+	}
+
 IMPLEMENT_SERIAL(EnumType, SER_ENUM_TYPE);
 
 bool EnumType::DoSerialize(SerialInfo* info) const

src/Type.h

@@ -6,6 +6,7 @@
 #define type_h
 
 #include <string>
+#include <list>
 #include <map>
 
 #include "Obj.h"
@@ -67,6 +68,7 @@ const int MATCHES_INDEX_VECTOR = 2;
 class BroType : public BroObj {
 public:
 	BroType(TypeTag tag, bool base_type = false);
+	~BroType();
 
 	TypeTag Tag() const		{ return tag; }
 	InternalTypeTag InternalType() const	{ return internal_tag; }
@@ -200,14 +202,18 @@ public:
 	BroType* Ref()		{ ::Ref(this); return this; }
 
 	virtual void Describe(ODesc* d) const;
+	virtual void DescribeReST(ODesc* d) const;
 
 	virtual unsigned MemoryAllocation() const;
 
 	bool Serialize(SerialInfo* info) const;
 	static BroType* Unserialize(UnserialInfo* info, TypeTag want = TYPE_ANY);
 
+	void SetTypeID(const char* id)	{ type_id = id; }
+	const char* GetTypeID() const	{ return type_id; }
+
 protected:
-	BroType()	{ }
+	BroType()	{ type_id = 0; }
 
 	void SetError();
 
@@ -218,6 +224,10 @@ private:
 	InternalTypeTag internal_tag;
 	bool is_network_order;
 	bool base_type;
+
+	// This type_id field is only used by the documentation framework to
+	// track the names of declared types.
+	const char* type_id;
 };
 
 class TypeList : public BroType {
@@ -273,6 +283,7 @@ public:
 	BroType* YieldType();
 
 	void Describe(ODesc* d) const;
+	void DescribeReST(ODesc* d) const;
 
 	// Returns true if this table is solely indexed by subnet.
 	bool IsSubNetIndex() const;
@@ -347,6 +358,7 @@ public:
 	ID* GetReturnValueID() const;
 
 	void Describe(ODesc* d) const;
+	void DescribeReST(ODesc* d) const;
 
 protected:
 	FuncType()	{ args = 0; arg_types = 0; yield = 0; return_value = 0; }
@@ -362,7 +374,7 @@ protected:
 class TypeDecl {
 public:
 	TypeDecl(BroType* t, const char* i, attr_list* attrs = 0);
-	~TypeDecl();
+	virtual ~TypeDecl();
 
 	const Attr* FindAttr(attr_tag a) const
 		{ return attrs ? attrs->FindAttr(a) : 0; }
@@ -370,11 +382,24 @@ public:
 	bool Serialize(SerialInfo* info) const;
 	static TypeDecl* Unserialize(UnserialInfo* info);
 
+	virtual void DescribeReST(ODesc* d) const;
+
 	BroType* type;
 	Attributes* attrs;
 	const char* id;
 };
 
+class CommentedTypeDecl : public TypeDecl {
+public:
+	CommentedTypeDecl(BroType* t, const char* i, attr_list* attrs = 0,
+			std::list<std::string>* cmnt_list = 0);
+	virtual ~CommentedTypeDecl();
+
+	void DescribeReST(ODesc* d) const;
+
+	std::list<std::string>* comments;
+};
+
 class RecordField {
 public:
 	RecordField(int arg_base, int arg_offset, int arg_total_offset);
@@ -410,7 +435,9 @@ public:
 	int NumFields() const			{ return num_fields; }
 
 	void Describe(ODesc* d) const;
+	void DescribeReST(ODesc* d) const;
 	void DescribeFields(ODesc* d) const;
+	void DescribeFieldsReST(ODesc* d, bool func_args) const;
 
 protected:
 	RecordType() { fields = 0; base = 0; types = 0; }
@@ -471,7 +498,8 @@ public:
 protected:
 	DECLARE_SERIAL(EnumType)
 
-	void AddNameInternal(const string& module_name, const char* name, bro_int_t val, bool is_export);
+	virtual void AddNameInternal(const string& module_name,
+			const char* name, bro_int_t val, bool is_export);
 
 	typedef std::map< const char*, bro_int_t, ltstr > NameMap;
 	NameMap names;
@@ -485,6 +513,27 @@ protected:
 	bro_int_t counter;
 };
 
+class CommentedEnumType: public EnumType {
+public:
+	CommentedEnumType() {}
+	~CommentedEnumType();
+
+	void DescribeReST(ODesc* d) const;
+	void AddComment(const string& module_name, const char* name,
+			std::list<std::string>* comments);
+
+protected:
+	// This overriden method does not install the given ID name into a
+	// scope and it also does not do any kind of checking that the
+	// provided name already exists.
+	void AddNameInternal(const string& module_name, const char* name,
+			bro_int_t val, bool is_export);
+
+	// Comments are only filled when in "documentation mode".
+	typedef std::map< const char*, std::list<std::string>*, ltstr > CommentMap;
+	CommentMap comments;
+};
+
 class VectorType : public BroType {
 public:
 	VectorType(BroType* t);

src/Val.cc

@@ -574,6 +574,11 @@ void Val::Describe(ODesc* d) const
 		Val::ValDescribe(d);
 	}
 
+void Val::DescribeReST(ODesc* d) const
+	{
+	ValDescribeReST(d);
+	}
+
 void Val::ValDescribe(ODesc* d) const
 	{
 	if ( d->IsReadable() && type->Tag() == TYPE_BOOL )
@@ -615,6 +620,20 @@ void Val::ValDescribe(ODesc* d) const
 	}
 	}
 
+void Val::ValDescribeReST(ODesc* d) const
+	{
+	switch ( type->InternalType() ) {
+	case TYPE_INTERNAL_OTHER:
+		Describe(d);
+		break;
+
+	default:
+		d->Add("``");
+		ValDescribe(d);
+		d->Add("``");
+	}
+	}
+
 MutableVal::~MutableVal()
 	{
 	for ( list<ID*>::iterator i = aliases.begin(); i != aliases.end(); ++i )
@@ -2929,6 +2948,34 @@ void RecordVal::Describe(ODesc* d) const
 		d->Add("]");
 	}
 
+void RecordVal::DescribeReST(ODesc* d) const
+	{
+	const val_list* vl = AsRecord();
+	int n = vl->length();
+
+	d->Add("{");
+	d->PushIndent();
+
+	loop_over_list(*vl, i)
+		{
+		if ( i > 0 )
+			d->NL();
+
+		d->Add(record_type->FieldName(i));
+		d->Add("=");
+
+		Val* v = (*vl)[i];
+
+		if ( v )
+			v->Describe(d);
+		else
+			d->Add("<uninitialized>");
+		}
+
+	d->PopIndent();
+	d->Add("}");
+	}
+
 IMPLEMENT_SERIAL(RecordVal, SER_RECORD_VAL);
 
 bool RecordVal::DoSerialize(SerialInfo* info) const

src/Val.h

@@ -313,6 +313,7 @@ public:
 		}
 
 	void Describe(ODesc* d) const;
+	virtual void DescribeReST(ODesc* d) const;
 
 	bool Serialize(SerialInfo* info) const;
 	static Val* Unserialize(UnserialInfo* info, TypeTag type = TYPE_ANY)
@@ -347,6 +348,7 @@ protected:
 		}
 
 	virtual void ValDescribe(ODesc* d) const;
+	virtual void ValDescribeReST(ODesc* d) const;
 
 	Val(TypeTag t)
 		{
@@ -902,6 +904,7 @@ public:
 	BroObj* GetOrigin() const	{ return origin; }
 
 	unsigned int MemoryAllocation() const;
+	void DescribeReST(ODesc* d) const;
 
 protected:
 	friend class Val;

src/Var.cc

@@ -12,6 +12,8 @@
 #include "RemoteSerializer.h"
 #include "EventRegistry.h"
 
+extern int generate_documentation;
+
 static Val* init_val(Expr* init, const BroType* t, Val* aggr)
 	{
 	return init->InitVal(t, aggr);
@@ -217,11 +219,44 @@ extern Expr* add_and_assign_local(ID* id, Expr* init, Val* val)
 
 void add_type(ID* id, BroType* t, attr_list* attr, int /* is_event */)
 	{
-	id->SetType(t);
+	BroType* tnew = t;
+
+	// In "documentation mode", we'd like to to be able to associate
+	// an identifier name with a declared type.  Dealing with declared
+	// types that are "aliases" to a builtin type requires that the BroType
+	// is cloned before setting the identifier name that resolves to it.
+	// And still this is not enough to document cases where the declared type
+	// is an alias for another declared type -- but that's not a natural/common
+	// practice.  If documenting that corner case is desired, one way
+	// is to add an ID* to class ID that tracks aliases and set it here if
+	// t->GetTypeID() is true.
+	if ( generate_documentation )
+		{
+		SerializationFormat* form = new BinarySerializationFormat();
+		form->StartWrite();
+		CloneSerializer ss(form);
+		SerialInfo sinfo(&ss);
+		sinfo.cache = false;
+
+		t->Serialize(&sinfo);
+		char* data;
+		uint32 len = form->EndWrite(&data);
+		form->StartRead(data, len);
+
+		UnserialInfo uinfo(&ss);
+		uinfo.cache = false;
+		tnew = t->Unserialize(&uinfo);
+
+		delete [] data;
+
+		tnew->SetTypeID(copy_string(id->Name()));
+		}
+
+	id->SetType(tnew);
 	id->MakeType();
 
 	if ( attr )
-		id->SetAttrs(new Attributes(attr, t));
+		id->SetAttrs(new Attributes(attr, tnew));
 	}
 
 void begin_func(ID* id, const char* module_name, function_flavor flavor,

src/builtin-func.y

@@ -296,7 +296,12 @@ builtin_lang:	definitions
 
 
 definitions:	definitions definition opt_ws
-			{ fprintf(fp_func_def, "%s", $3); }
+			{
+			if ( in_c_code )
+				fprintf(fp_func_def, "%s", $3);
+			else
+				fprintf(fp_bro_init, "%s", $3);
+			}
 	|	opt_ws
 			{
 			int n = 1024 + strlen(input_filename);
@@ -694,6 +699,11 @@ opt_ws:		opt_ws TOK_WS
 			{
 			if ( in_c_code )
 				$$ = concat($1, $2);
+			else
+				if ( $2[1] == '#' )
+					// This is a special type of comment that is used to
+					// generate bro script documentation, so pass it through.
+					$$ = concat($1, $2);
 				else
 					$$ = $1;
 			}

src/input.h

@@ -30,6 +30,8 @@ extern void do_atifdef(const char* id);
 extern void do_atifndef(const char* id);
 extern void do_atelse();
 extern void do_atendif();
+extern void do_doc_token_start();
+extern void do_doc_token_stop();
 
 extern int line_number;
 extern const char* filename;

src/main.cc

@@ -91,6 +91,7 @@ int optimize = 0;
 int do_notice_analysis = 0;
 int rule_bench = 0;
 int print_loaded_scripts = 0;
+int generate_documentation = 0;
 SecondaryPath* secondary_path = 0;
 ConnCompressor* conn_compressor = 0;
 extern char version[];
@@ -142,6 +143,7 @@ void usage()
 	fprintf(stderr, "    -h|--help|-?                   | command line help\n");
 	fprintf(stderr, "    -i|--iface <interface>         | read from given interface\n");
 	fprintf(stderr, "    -l|--print-scripts             | print all loaded scripts\n");
+	fprintf(stderr, "    -Z|--doc-scripts               | generate documentation for all loaded scripts\n");
 	fprintf(stderr, "    -p|--prefix <prefix>           | add given prefix to policy file resolution\n");
 	fprintf(stderr, "    -r|--readfile <readfile>       | read from given tcpdump file\n");
 	fprintf(stderr, "    -y|--flowfile <file>[=<ident>] | read from given flow file\n");
@@ -360,6 +362,7 @@ int main(int argc, char** argv)
 		{"help",		no_argument,		0,	'h'},
 		{"iface",		required_argument,	0,	'i'},
 		{"print-scripts",	no_argument,		0,	'l'},
+		{"doc-scripts",		no_argument,		0,	'Z'},
 		{"prefix",		required_argument,	0,	'p'},
 		{"readfile",		required_argument,	0,	'r'},
 		{"flowfile",		required_argument,	0,	'y'},
@@ -432,7 +435,7 @@ int main(int argc, char** argv)
 	opterr = 0;
 
 	char opts[256];
-	safe_strncpy(opts, "B:D:e:f:I:i:K:n:p:R:r:s:T:t:U:w:x:X:y:Y:z:CFGHLOPSWdghlv",
+	safe_strncpy(opts, "B:D:e:f:I:i:K:n:p:R:r:s:T:t:U:w:x:X:y:Y:z:CFGHLOPSWdghlvZ",
 		     sizeof(opts));
 
 #ifdef USE_PERFTOOLS
@@ -611,6 +614,10 @@ int main(int argc, char** argv)
 			break;
 #endif
 
+		case 'Z':
+			generate_documentation = 1;
+			break;
+
 #ifdef USE_IDMEF
 		case 'n':
 			fprintf(stderr, "Using IDMEF XML DTD from %s\n", optarg);

src/parse.y

@@ -3,7 +3,7 @@
 // See the file "COPYING" in the main distribution directory for copyright.
 %}
 
-%expect 71
+%expect 81
 
 %token TOK_ADD TOK_ADD_TO TOK_ADDR TOK_ALARM TOK_ANY
 %token TOK_ATENDIF TOK_ATELSE TOK_ATIF TOK_ATIFDEF TOK_ATIFNDEF
@@ -28,6 +28,8 @@
 
 %token TOK_DEBUG
 
+%token TOK_DOC TOK_POST_DOC
+
 %left ',' '|'
 %right '=' TOK_ADD_TO TOK_REMOVE_FROM
 %right '?' ':' TOK_USING
@@ -41,7 +43,8 @@
 %right '!'
 %left '$' '[' ']' '(' ')' TOK_HAS_FIELD TOK_HAS_ATTR
 
-%type <str> TOK_ID TOK_PATTERN_TEXT single_pattern
+%type <str> TOK_ID TOK_PATTERN_TEXT single_pattern TOK_DOC TOK_POST_DOC
+%type <str_l> opt_doc_list opt_post_doc_list
 %type <id> local_id global_id event_id global_or_event_id resolve_id begin_func
 %type <id_l> local_id_list
 %type <ic> init_class
@@ -75,6 +78,15 @@
 #include "DNS.h"
 #include "RE.h"
 #include "Scope.h"
+#include "BroDoc.h"
+#include "BroDocObj.h"
+
+#include <list>
+#include <string>
+
+extern BroDoc* current_reST_doc;
+extern int generate_documentation;
+extern std::list<std::string>* reST_doc_comments;
 
 YYLTYPE GetCurrentLocation();
 extern int yyerror(const char[]);
@@ -105,12 +117,22 @@ bool resolving_global_ID = false;
 
 ID* func_id = 0;
 EnumType *cur_enum_type = 0;
+CommentedEnumType *cur_enum_type_doc = 0;
+const char* cur_enum_elem_id = 0;
+
+type_decl_list* fake_type_decl_list = 0;
+TypeDecl* last_fake_type_decl = 0;
 
 static void parser_new_enum (void)
 	{
 	/* Starting a new enum definition. */
 	assert(cur_enum_type == NULL);
 	cur_enum_type = new EnumType();
+
+	// For documentation purposes, a separate type object is created
+	// in order to avoid overlap that can be caused by redefs.
+	if ( generate_documentation )
+		cur_enum_type_doc = new CommentedEnumType();
 	}
 
 static void parser_redef_enum (ID *id)
@@ -126,12 +148,52 @@ static void parser_redef_enum (ID *id)
 		if ( ! cur_enum_type )
 			id->Error("not an enum");
 		}
+
+	if ( generate_documentation )
+		cur_enum_type_doc = new CommentedEnumType();
+	}
+
+static void add_enum_comment (std::list<std::string>* comments)
+	{
+	cur_enum_type_doc->AddComment(current_module, cur_enum_elem_id, comments);
+	}
+
+static ID* create_dummy_id (ID* id, BroType* type)
+	{
+	ID* fake_id = new ID(copy_string(id->Name()), (IDScope) id->Scope(),
+	                     is_export);
+
+	fake_id->SetType(type);
+
+	if ( id->AsType() )
+		{
+		type->SetTypeID(copy_string(id->Name()));
+		fake_id->MakeType();
+		}
+
+	return fake_id;
+	}
+
+static std::list<std::string>* concat_opt_docs (std::list<std::string>* pre,
+                                                std::list<std::string>* post)
+	{
+	if ( ! pre && ! post ) return 0;
+
+	if ( pre && ! post ) return pre;
+
+	if ( ! pre && post ) return post;
+
+	pre->splice(pre->end(), *post);
+	delete post;
+
+	return pre;
 	}
 
 %}
 
 %union {
 	char* str;
+	std::list<std::string>* str_l;
 	ID* id;
 	id_list* id_l;
 	init_class ic;
@@ -546,11 +608,6 @@ opt_expr_list:
 		{ $$ = new ListExpr(); }
 	;
 
-opt_comma:
-		','
-	|
-	;
-
 pattern:
 		pattern '|' single_pattern
 			{
@@ -571,16 +628,45 @@ single_pattern:
 	;
 
 enum_body:
-		enum_body_list opt_comma
+		enum_body_list opt_post_doc_list
 			{
 			$$ = cur_enum_type;
+
+			if ( generate_documentation )
+				{
+				add_enum_comment($2);
+				cur_enum_elem_id = 0;
+				}
+
+			cur_enum_type = NULL;
+			}
+
+	|	enum_body_list ',' opt_post_doc_list
+			{
+			$$ = cur_enum_type;
+
+			if ( generate_documentation )
+				{
+				add_enum_comment($3);
+				cur_enum_elem_id = 0;
+				}
+
 			cur_enum_type = NULL;
 			}
 	;
 
 enum_body_list:
-			enum_body_elem	 /* No action */
-		|	enum_body_list ',' enum_body_elem  /* no action */
+		enum_body_elem opt_post_doc_list
+			{
+			if ( generate_documentation )
+				add_enum_comment($2);
+			}
+
+	|	enum_body_list ',' opt_post_doc_list
+			{
+			if ( generate_documentation )
+				add_enum_comment($3);
+			} enum_body_elem
 ;
 
 enum_body_elem:
@@ -589,17 +675,25 @@ enum_body_elem:
 		   error messages if someboy tries to use constant variables as
 		   enumerator.
 		*/
-		TOK_ID '=' TOK_CONSTANT
+		opt_doc_list TOK_ID '=' TOK_CONSTANT
 			{
-			set_location(@1, @3);
+			set_location(@2, @4);
 			assert(cur_enum_type);
-			if ( $3->Type()->Tag() != TYPE_COUNT )
+
+			if ( $4->Type()->Tag() != TYPE_COUNT )
 				error("enumerator is not a count constant");
 			else
-				cur_enum_type->AddName(current_module, $1, $3->InternalUnsigned(), is_export);
+				cur_enum_type->AddName(current_module, $2, $4->InternalUnsigned(), is_export);
+
+			if ( generate_documentation )
+				{
+				cur_enum_type_doc->AddName(current_module, $2, $4->InternalUnsigned(), is_export);
+				cur_enum_elem_id = $2;
+				add_enum_comment($1);
+				}
 			}
 
-	|	TOK_ID '=' '-' TOK_CONSTANT
+	|	opt_doc_list TOK_ID '=' '-' TOK_CONSTANT
 			{
 			/* We only accept counts as enumerator, but we want to return a nice
 			   error message if users triy to use a negative integer (will also
@@ -608,11 +702,18 @@ enum_body_elem:
 			error("enumerator is not a count constant");
 			}
 
-	|	TOK_ID
+	|	opt_doc_list TOK_ID
 			{
-			set_location(@1);
+			set_location(@2);
 			assert(cur_enum_type);
-			cur_enum_type->AddName(current_module, $1, is_export);
+			cur_enum_type->AddName(current_module, $2, is_export);
+
+			if ( generate_documentation )
+				{
+				cur_enum_type_doc->AddName(current_module, $2, is_export);
+				cur_enum_elem_id = $2;
+				add_enum_comment($1);
+				}
 			}
 	;
 
@@ -704,10 +805,11 @@ type:
 				$$ = new SetType($3, 0);
 				}
 
-	|	TOK_RECORD '{' type_decl_list '}'
+	|	TOK_RECORD '{' { do_doc_token_start(); } type_decl_list '}'
 				{
-				set_location(@1, @4);
-				$$ = new RecordType($3);
+				do_doc_token_stop();
+				set_location(@1, @5);
+				$$ = new RecordType($4);
 				}
 
 	|	TOK_UNION '{' type_list '}'
@@ -717,8 +819,9 @@ type:
 				$$ = 0;
 				}
 
-	|	TOK_ENUM '{' { set_location(@1); parser_new_enum(); } enum_body '}'
+	|	TOK_ENUM '{' { set_location(@1); parser_new_enum(); do_doc_token_start(); } enum_body '}'
 				{
+				do_doc_token_stop();
 				set_location(@1, @5);
 				$4->UpdateLocationEndInfo(@5);
 				$$ = $4;
@@ -796,16 +899,46 @@ type_list:
 
 type_decl_list:
 		type_decl_list type_decl
-			{ $1->append($2); }
+			{
+			$1->append($2);
+
+			if ( generate_documentation && last_fake_type_decl )
+				{
+				fake_type_decl_list->append(last_fake_type_decl);
+				last_fake_type_decl = 0;
+				}
+			}
 	|
-			{ $$ = new type_decl_list(); }
+			{
+			$$ = new type_decl_list();
+
+			if ( generate_documentation )
+				fake_type_decl_list = new type_decl_list();
+			}
 	;
 
 type_decl:
-		TOK_ID ':' type opt_attr ';'
+		opt_doc_list TOK_ID ':' type opt_attr ';' opt_post_doc_list
 			{
-			set_location(@1, @5);
-			$$ = new TypeDecl($3, $1, $4);
+			set_location(@2, @6);
+
+			if ( generate_documentation )
+				{
+				attr_list* a = $5;
+				attr_list* a_copy = 0;
+
+				if ( a )
+					{
+					a_copy = new attr_list;
+					loop_over_list(*a, i)
+						a_copy->append((*a)[i]);
+					}
+
+				last_fake_type_decl = new CommentedTypeDecl(
+					$4, $2, a_copy, concat_opt_docs($1, $7));
+				}
+
+			$$ = new TypeDecl($4, $2, $5);
 			}
 	;
 
@@ -837,31 +970,128 @@ formal_args_decl:
 
 decl:
 		TOK_MODULE TOK_ID ';'
-			{ current_module = $2; }
+			{
+			current_module = $2;
+
+			if ( generate_documentation )
+				current_reST_doc->AddModule(current_module);
+			}
 
 	|	TOK_EXPORT '{' { is_export = true; } decl_list '}'
 			{ is_export = false; }
 
 	|	TOK_GLOBAL global_id opt_type init_class opt_init opt_attr ';'
-			{ add_global($2, $3, $4, $5, $6, VAR_REGULAR); }
+			{
+			add_global($2, $3, $4, $5, $6, VAR_REGULAR);
+
+			if ( generate_documentation )
+				{
+				ID* id = $2;
+				if ( id->Type()->Tag() == TYPE_FUNC )
+					{
+					if ( id->Type()->AsFuncType()->IsEvent() )
+						current_reST_doc->AddEvent(
+							new BroDocObj(id, reST_doc_comments));
+					else
+						current_reST_doc->AddFunction(
+							new BroDocObj(id, reST_doc_comments));
+					}
+
+				else
+					{
+					current_reST_doc->AddStateVar(
+						new BroDocObj(id, reST_doc_comments));
+					}
+				}
+			}
 
 	|	TOK_CONST global_id opt_type init_class opt_init opt_attr ';'
-			{ add_global($2, $3, $4, $5, $6, VAR_CONST); }
+			{
+			add_global($2, $3, $4, $5, $6, VAR_CONST);
+
+			if ( generate_documentation )
+				{
+				if ( $2->FindAttr(ATTR_REDEF) )
+					current_reST_doc->AddOption(
+						new BroDocObj($2, reST_doc_comments));
+				else
+					current_reST_doc->AddConstant(
+						new BroDocObj($2, reST_doc_comments));
+				}
+			}
 
 	|	TOK_REDEF global_id opt_type init_class opt_init opt_attr ';'
-			{ add_global($2, $3, $4, $5, $6, VAR_REDEF); }
+			{
+			add_global($2, $3, $4, $5, $6, VAR_REDEF);
+
+			if ( generate_documentation &&
+				! streq("capture_filters", $2->Name()) &&
+				! streq("dpd_config", $2->Name()) )
+				{
+				ID* fake_id = create_dummy_id($2, $2->Type());
+				BroDocObj* o = new BroDocObj(fake_id, reST_doc_comments, true);
+				o->SetRole(true);
+				current_reST_doc->AddRedef(o);
+				}
+			}
 
 	|	TOK_REDEF TOK_ENUM global_id TOK_ADD_TO
-		'{' { parser_redef_enum($3); } enum_body '}' ';'
-			{ /* no action */ }
+		'{' { parser_redef_enum($3); do_doc_token_start(); } enum_body '}' ';'
+			{
+			do_doc_token_stop();
+
+			if ( generate_documentation )
+				{
+				ID* fake_id = create_dummy_id($3, cur_enum_type_doc);
+				cur_enum_type_doc = 0;
+				BroDocObj* o = new BroDocObj(fake_id, reST_doc_comments, true);
+				o->SetRole(true);
+
+				if ( streq(fake_id->Name(), "Notice" ) )
+					current_reST_doc->AddNotice(o);
+				else
+					current_reST_doc->AddRedef(o);
+				}
+			}
 
 	|	TOK_TYPE global_id ':' refined_type opt_attr ';'
 			{
 			add_type($2, $4, $5, 0);
+
+			if ( generate_documentation )
+				{
+				TypeTag t = $2->AsType()->Tag();
+				if ( t == TYPE_ENUM && cur_enum_type_doc )
+					{
+					ID* fake = create_dummy_id($2, cur_enum_type_doc);
+					cur_enum_type_doc = 0;
+					current_reST_doc->AddType(
+						new BroDocObj(fake, reST_doc_comments, true));
+					}
+
+				else if ( t == TYPE_RECORD && fake_type_decl_list )
+					{
+					BroType* fake_record = new RecordType(fake_type_decl_list);
+					ID* fake = create_dummy_id($2, fake_record);
+					fake_type_decl_list = 0;
+					current_reST_doc->AddType(
+						new BroDocObj(fake, reST_doc_comments, true));
+					}
+
+				else
+					current_reST_doc->AddType(
+						new BroDocObj($2, reST_doc_comments));
+				}
 			}
 
 	|	TOK_EVENT event_id ':' refined_type opt_attr ';'
-			{ add_type($2, $4, $5, 1); }
+			{
+			add_type($2, $4, $5, 1);
+
+			if ( generate_documentation )
+				current_reST_doc->AddEvent(
+					new BroDocObj($2, reST_doc_comments));
+			}
 
 	|	func_hdr func_body
 			{ }
@@ -888,12 +1118,18 @@ func_hdr:
 			begin_func($2, current_module.c_str(),
 				FUNC_FLAVOR_FUNCTION, 0, $3);
 			$$ = $3;
+			if ( generate_documentation )
+				current_reST_doc->AddFunction(
+					new BroDocObj($2, reST_doc_comments));
 			}
 	|	TOK_EVENT event_id func_params
 			{
 			begin_func($2, current_module.c_str(),
 				   FUNC_FLAVOR_EVENT, 0, $3);
 			$$ = $3;
+			if ( generate_documentation )
+				current_reST_doc->AddEvent(
+					new BroDocObj($2, reST_doc_comments));
 			}
 	|	TOK_REDEF TOK_EVENT event_id func_params
 			{
@@ -1302,12 +1538,48 @@ resolve_id:
 			{
 			set_location(@1);
 			$$ = lookup_ID($1, current_module.c_str());
+
 			if ( ! $$ )
 				error("identifier not defined:", $1);
+
 			delete [] $1;
 			}
 	;
 
+opt_post_doc_list:
+		opt_post_doc_list TOK_POST_DOC
+			{
+			$1->push_back($2);
+			$$ = $1;
+			}
+	|
+		TOK_POST_DOC
+			{
+			$$ = new std::list<std::string>();
+			$$->push_back($1);
+			delete [] $1;
+			}
+	|
+			{ $$ = 0; }
+	;
+
+opt_doc_list:
+		opt_doc_list TOK_DOC
+			{
+			$1->push_back($2);
+			$$ = $1;
+			}
+	|
+		TOK_DOC
+			{
+			$$ = new std::list<std::string>();
+			$$->push_back($1);
+			delete [] $1;
+			}
+	|
+			{ $$ = 0; }
+	;
+
 %%
 
 int yyerror(const char msg[])

src/scan.l

@@ -15,11 +15,18 @@
 #include "Debug.h"
 #include "PolicyFile.h"
 #include "broparse.h"
+#include "BroDoc.h"
+#include "BroBifDoc.h"
+#include "Analyzer.h"
+#include "AnalyzerTags.h"
 
 #include <stack>
+#include <list>
+#include <string>
 
 extern YYLTYPE yylloc;	// holds start line and column of token
 extern int print_loaded_scripts;
+extern int generate_documentation;
 
 int nwarn = 0;
 int nerr = 0;
@@ -28,13 +35,13 @@ int nruntime = 0;
 // Track the @if... depth.
 ptr_compat_int current_depth = 0;
 
-declare(List,ptr_compat_int);
-typedef List(ptr_compat_int) int_list;
 int_list if_stack;
 
 int line_number = 1;
 int include_level = 0;
 const char* filename = 0;
+BroDoc* current_reST_doc = 0;
+static BroDoc* last_reST_doc = 0;
 
 char last_tok[128];
 
@@ -50,6 +57,33 @@ char last_tok[128];
 // Files we have already scanned (or are in the process of scanning).
 static PList(char) files_scanned;
 
+// reST documents that we've created (or have at least opened so far).
+static std::list<BroDoc*> docs_generated;
+
+// reST comments (those starting with ##) seen so far.
+std::list<std::string>* reST_doc_comments = 0;
+
+// Print current contents of reST_doc_comments list to stderr.
+void print_current_reST_doc_comments();
+
+// Delete the reST_doc_comments list object.
+void clear_reST_doc_comments();
+
+// Adds changes to capture_filter to the current script's reST documentation.
+static void check_capture_filter_changes();
+
+// Adds changes to dpd_config to the current script's reST documentation.
+static void check_dpd_config_changes();
+
+static const char* canon_doc_comment(const char* comment)
+	{
+	// "##Text" and "## Text" are treated the same in order to be able
+	// to still preserve indentation level, but not unintentionally
+	// signify an indentation level for all the text when using
+	// the "## Text" style.
+	return ( comment[0] == ' ' ) ? comment + 1 : comment;
+	}
+
 class FileInfo {
 public:
 	FileInfo(string restore_module = "");
@@ -60,6 +94,7 @@ public:
 	const char* name;
 	int line;
 	int level;
+	BroDoc* doc;
 };
 
 // A stack of input buffers we're scanning.  file_stack[len-1] is the
@@ -87,6 +122,7 @@ static void report_file();
 
 %x RE
 %x IGNORE
+%s DOC
 
 OWS	[ \t]*
 WS	[ \t]+
@@ -102,11 +138,75 @@ ESCSEQ	(\\([^\n]|[0-7]+|x[[:xdigit:]]+))
 
 %%
 
+##!.* {
+	// Add this format of comments to the script documentation's "summary".
+	if ( generate_documentation )
+		current_reST_doc->AddSummary(canon_doc_comment(yytext + 3));
+	}
+
+<DOC>##<.* {
+	yylval.str = copy_string(canon_doc_comment(yytext + 3));
+	return TOK_POST_DOC;
+}
+
+<DOC>##.* {
+	if ( yytext[2] != '#' )
+		{
+		yylval.str = copy_string(canon_doc_comment(yytext + 2));
+		return TOK_DOC;
+		}
+}
+
+##{OWS}{ID}:.* {
+	if ( generate_documentation )
+		{
+		// Comment is documenting either a function parameter or return type,
+		// so appropriate reST markup substitutions are automatically made
+		// in order to distinguish them from other comments.
+		const char* id_start = skip_whitespace(yytext + 2);
+		size_t id_len = strcspn(id_start, ":");
+		char* id_name = new char[id_len + 1];
+		strncpy(id_name, id_start, id_len);
+		id_name[id_len] = '\0';
+		const char* comment = id_start + id_len + 1;
+
+		std::string doc;
+
+		if ( streq(id_name, "Returns") )
+			doc.append(":returns:").append(comment);
+		else
+			doc.append(":param ").append(id_name).append(":").append(comment);
+
+		if ( ! reST_doc_comments )
+			reST_doc_comments = new std::list<std::string>();
+
+		// always insert a blank line so that this param/return markup
+		// 1) doesn't show up in the summary section in the case that it's
+		//    the first comment for the function/event
+		// 2) has a blank line between it and non-field-list reST markup,
+		//    which is required for correct HTML rendering by Sphinx
+		reST_doc_comments->push_back("");
+		reST_doc_comments->push_back(doc);
+
+		delete [] id_name;
+		}
+}
+
+##.* {
+	if ( generate_documentation && (yytext[2] != '#') )
+		{
+		if ( ! reST_doc_comments )
+			reST_doc_comments = new std::list<std::string>();
+
+		reST_doc_comments->push_back(canon_doc_comment(yytext + 2));
+		}
+}
+
 #.*	/* eat comments */
 
 {WS}	/* eat whitespace */
 
-<INITIAL,IGNORE>\n	{
+<INITIAL,IGNORE,DOC>\n	{
 			report_file();
 			++line_number;
 			++yylloc.first_line;
@@ -211,6 +311,18 @@ when	return TOK_WHEN;
 
 @load{WS}{FILE}	{
 	const char* new_file = skip_whitespace(yytext + 5);	// Skip "@load".
+	if ( generate_documentation )
+		{
+		current_reST_doc->AddImport(new_file);
+
+		if ( reST_doc_comments )
+			{
+			fprintf(stderr, "Warning: unconsumed reST documentation is being "
+					"discarded before doing '@load %s' in %s:\n",
+					 new_file, current_reST_doc->GetSourceFileName());
+			clear_reST_doc_comments();
+			}
+		}
 	(void) load_files_with_prefix(new_file);
 	}
 
@@ -446,7 +558,6 @@ static int load_files_with_prefix(const char* orig_file)
 				strcpy(new_filename, file);
 
 			f = search_for_file(new_filename, "bro", &full_filename);
-
 			delete [] new_filename;
 			}
 
@@ -496,6 +607,21 @@ static int load_files_with_prefix(const char* orig_file)
 			// Don't delete the old filename - it's pointed to by
 			// every BroObj created when parsing it.
 			yylloc.filename = filename = full_filename;
+
+			if ( generate_documentation )
+				{
+				char* bifExtStart = strstr(full_filename, ".bif.bro");
+				BroDoc* reST_doc;
+
+				if ( bifExtStart )
+					reST_doc = new BroBifDoc(full_filename);
+				else
+					reST_doc = new BroDoc(full_filename);
+
+				docs_generated.push_back(reST_doc);
+
+				current_reST_doc = reST_doc;
+				}
 			}
 
 		else
@@ -589,6 +715,18 @@ void do_atendif()
 	--current_depth;
 	}
 
+void do_doc_token_start()
+	{
+	if ( generate_documentation )
+	    BEGIN(DOC);
+	}
+
+void do_doc_token_stop()
+	{
+	if ( generate_documentation )
+	    BEGIN(INITIAL);
+	}
+
 // Be careful to never delete things from this list, as the strings
 // are referred to (in order to save the locations of tokens and statements,
 // for error reporting and debugging).
@@ -655,6 +793,9 @@ int yywrap()
 	// Stack is now empty.
 	while ( input_files.length() > 0 )
 		{
+		check_capture_filter_changes();
+		check_dpd_config_changes();
+
 		if ( load_files_with_prefix(input_files[0]) )
 			{
 			// Don't delete the filename - it's pointed to by
@@ -668,6 +809,9 @@ int yywrap()
 		(void) input_files.remove_nth(0);
 		}
 
+	check_capture_filter_changes();
+	check_dpd_config_changes();
+
 	// Add redef statements for any X=Y command line parameters.
 	if ( params.size() > 0 )
 		{
@@ -731,6 +875,20 @@ int yywrap()
 		return 0;
 		}
 
+	if ( generate_documentation )
+		{
+		std::list<BroDoc*>::iterator it;
+
+		for ( it = docs_generated.begin(); it != docs_generated.end(); ++it )
+			{
+			(*it)->WriteDocFile();
+			delete *it;
+			}
+
+		docs_generated.clear();
+		clear_reST_doc_comments();
+		}
+
 	// Otherwise, we are done.
 	return 1;
 	}
@@ -742,6 +900,7 @@ FileInfo::FileInfo(string arg_restore_module)
 	name = ::filename;
 	line = ::line_number;
 	level = ::include_level;
+	doc = ::current_reST_doc;
 	}
 
 FileInfo::~FileInfo()
@@ -753,6 +912,8 @@ FileInfo::~FileInfo()
 	yylloc.filename = filename = name;
 	yylloc.first_line = yylloc.last_line = line_number = line;
 	include_level = level;
+	last_reST_doc = current_reST_doc;
+	current_reST_doc = doc;
 
 	if ( restore_module != "" )
 		current_module = restore_module;
@@ -779,3 +940,93 @@ static void report_file()
 	files_reported.append(copy_string(filename));
 	}
 
+static void check_capture_filter_changes()
+	{
+	if ( ! generate_documentation )
+	    return;
+
+	// Lookup the "capture_filters" identifier, if it has any defined
+	// value, add it to the script's reST documentation, and finally
+	// clear the table so it doesn't taint the documentation for
+	// subsequent scripts.
+
+	ID* capture_filters = global_scope()->Lookup("capture_filters");
+
+	if ( capture_filters )
+		{
+		ODesc desc;
+		desc.SetIndentSpaces(4);
+		capture_filters->ID_Val()->Describe(&desc);
+		last_reST_doc->SetPacketFilter(desc.Description());
+		capture_filters->ID_Val()->AsTableVal()->RemoveAll();
+		}
+	}
+
+static void check_dpd_config_changes()
+	{
+	if ( ! generate_documentation )
+		return;
+
+	// Lookup the "dpd_config" identifier, if it has any defined value,
+	// add it to the script's documentation, and clear the table so that
+	// it doesn't taint the documentation for subsequent scripts.
+	ID* dpd_config = global_scope()->Lookup("dpd_config");
+	if ( ! dpd_config )
+		return;
+
+	TableVal* dpd_table = dpd_config->ID_Val()->AsTableVal();
+	ListVal* dpd_list = dpd_table->ConvertToList();
+
+	for ( int i = 0; i < dpd_list->Length(); ++i )
+		{
+		Val* key = dpd_list->Index(i);
+		if ( ! key )
+			continue;
+
+		Val* v = dpd_table->Lookup(key);
+		if ( ! v )
+			continue;
+
+		int tag = key->AsListVal()->Index(0)->AsCount();
+		ODesc valdesc;
+		valdesc.SetIndentSpaces(4);
+		valdesc.PushIndent();
+		v->Describe(&valdesc);
+
+		if ( tag < AnalyzerTag::Error || tag > AnalyzerTag::LastAnalyzer )
+			{
+			fprintf(stderr, "Warning: skipped bad analyzer tag: %i\n", tag);
+			continue;
+			}
+
+		last_reST_doc->AddPortAnalysis(
+			Analyzer::GetTagName((AnalyzerTag::Tag)tag),
+			valdesc.Description());
+		}
+
+	dpd_table->RemoveAll();
+	}
+
+void print_current_reST_doc_comments()
+	{
+	if ( ! reST_doc_comments )
+		return;
+
+	std::list<std::string>::iterator it;
+
+	for ( it = reST_doc_comments->begin(); it != reST_doc_comments->end(); ++it )
+		fprintf(stderr, "##%s\n", it->c_str());
+	}
+
+void clear_reST_doc_comments()
+	{
+	if ( ! reST_doc_comments )
+		return;
+
+	fprintf(stderr, "Warning: %lu unconsumed reST comments:\n",
+			reST_doc_comments->size());
+
+	print_current_reST_doc_comments();
+	delete reST_doc_comments;
+	reST_doc_comments = 0;
+	}

testing/btest/Baseline/doc.autogen-reST-enums/autogen-reST-enums.rst

@@ -0,0 +1,99 @@
+.. Automatically generated.  Do not edit.
+
+autogen-reST-enums.bro
+======================
+
+:download:`Original Source File <autogen-reST-enums.bro>`
+
+Overview
+--------
+
+
+Summary
+~~~~~~~
+Types
+#####
+======================================= ======================================
+:bro:type:`TestEnum1`: :bro:type:`enum` There's tons of ways an enum can look.
+
+:bro:type:`TestEnum2`: :bro:type:`enum` The final comma is optional
+======================================= ======================================
+
+Redefinitions
+#############
+======================================= =======================
+:bro:type:`TestEnum1`: :bro:type:`enum` redefs should also work
+
+:bro:type:`TestEnum1`: :bro:type:`enum` now with a comma
+======================================= =======================
+
+Public Interface
+----------------
+Types
+~~~~~
+.. bro:type:: TestEnum1
+
+   :Type: :bro:type:`enum`
+
+      .. bro:enum:: ONE TestEnum1
+
+         like this
+
+      .. bro:enum:: TWO TestEnum1
+
+         or like this
+
+      .. bro:enum:: THREE TestEnum1
+
+         multiple
+         comments
+         and even
+         more comments
+
+   There's tons of ways an enum can look...
+
+.. bro:type:: TestEnum2
+
+   :Type: :bro:type:`enum`
+
+      .. bro:enum:: A TestEnum2
+
+         like this
+
+      .. bro:enum:: B TestEnum2
+
+         or like this
+
+      .. bro:enum:: C TestEnum2
+
+         multiple
+         comments
+         and even
+         more comments
+
+   The final comma is optional
+
+Redefinitions
+~~~~~~~~~~~~~
+:bro:type:`TestEnum1`
+
+   :Type: :bro:type:`enum`
+
+      .. bro:enum:: FOUR TestEnum1
+
+         adding another
+         value
+
+   redefs should also work
+
+:bro:type:`TestEnum1`
+
+   :Type: :bro:type:`enum`
+
+      .. bro:enum:: FIVE TestEnum1
+
+         adding another
+         value
+
+   now with a comma
+

testing/btest/Baseline/doc.autogen-reST-example/example.rst

@@ -0,0 +1,282 @@
+.. Automatically generated.  Do not edit.
+
+example.bro
+===========
+
+:download:`Original Source File <example.bro>`
+
+Overview
+--------
+This is an example script that demonstrates how to document.  Comments
+of the form ``##!`` are for the script summary.  The contents of
+these comments are transferred directly into the auto-generated
+`reStructuredText <http://docutils.sourceforge.net/rst.html>`_
+(reST) document's summary section.
+
+.. tip:: You can embed directives and roles within ``##``-stylized comments
+
+:Author: Jon Siwek <jsiwek@ncsa.illinois.edu>
+
+:Imports: :doc:`notice`
+
+Summary
+~~~~~~~
+Options
+#######
+============================================================================ ======================================
+:bro:id:`Example::an_option`: :bro:type:`set` :bro:attr:`&redef`             add documentation for "an_option" here
+
+:bro:id:`Example::option_with_init`: :bro:type:`interval` :bro:attr:`&redef`
+============================================================================ ======================================
+
+State Variables
+###############
+=========================================================================== =======================================
+:bro:id:`Example::a_var`: :bro:type:`bool`                                  put some documentation for "a_var" here
+
+:bro:id:`Example::var_with_attr`: :bro:type:`count` :bro:attr:`&persistent`
+
+:bro:id:`Example::var_without_explicit_type`: :bro:type:`string`
+=========================================================================== =======================================
+
+Types
+#####
+====================================================== ==========================================================
+:bro:type:`Example::SimpleEnum`: :bro:type:`enum`      documentation for "SimpleEnum"
+                                                       goes here.
+
+:bro:type:`Example::SimpleRecord`: :bro:type:`record`  general documentation for a type "SimpleRecord"
+                                                       goes here.
+
+:bro:type:`Example::ComplexRecord`: :bro:type:`record` general documentation for a type "ComplexRecord" goes here
+====================================================== ==========================================================
+
+Events
+######
+============================================== ==========================
+:bro:id:`Example::an_event`: :bro:type:`event` Summarize "an_event" here.
+============================================== ==========================
+
+Functions
+#########
+=============================================== =======================================
+:bro:id:`Example::a_function`: :bro:type:`func` Summarize purpose of "a_function" here.
+=============================================== =======================================
+
+Redefinitions
+#############
+================================================= ====================================
+:bro:type:`Example::SimpleEnum`: :bro:type:`enum` document the "SimpleEnum" redef here
+================================================= ====================================
+
+Namespaces
+~~~~~~~~~~
+.. bro:namespace:: Example
+
+Notices
+~~~~~~~
+:bro:type:`Notice`
+
+   :Type: :bro:type:`enum`
+
+      .. bro:enum:: Example::Notice_One Notice
+
+         any number of this type of comment
+         will document "Notice_One"
+
+      .. bro:enum:: Example::Notice_Two Notice
+
+         any number of this type of comment
+         will document "Notice_Two"
+
+      .. bro:enum:: Example::Notice_Three Notice
+
+      .. bro:enum:: Example::Notice_Four Notice
+
+Public Interface
+----------------
+Options
+~~~~~~~
+.. bro:id:: Example::an_option
+
+   :Type: :bro:type:`set` [:bro:type:`addr`, :bro:type:`addr`, :bro:type:`string`]
+   :Attributes: :bro:attr:`&redef`
+   :Default: ``{}``
+
+   add documentation for "an_option" here
+
+.. bro:id:: Example::option_with_init
+
+   :Type: :bro:type:`interval`
+   :Attributes: :bro:attr:`&redef`
+   :Default: ``10.0 msecs``
+
+State Variables
+~~~~~~~~~~~~~~~
+.. bro:id:: Example::a_var
+
+   :Type: :bro:type:`bool`
+
+   put some documentation for "a_var" here
+
+.. bro:id:: Example::var_with_attr
+
+   :Type: :bro:type:`count`
+   :Attributes: :bro:attr:`&persistent`
+
+.. bro:id:: Example::var_without_explicit_type
+
+   :Type: :bro:type:`string`
+   :Default: ``"this works"``
+
+Types
+~~~~~
+.. bro:type:: Example::SimpleEnum
+
+   :Type: :bro:type:`enum`
+
+      .. bro:enum:: Example::ONE Example::SimpleEnum
+
+         and more specific info for "ONE"
+         can span multiple lines
+
+      .. bro:enum:: Example::TWO Example::SimpleEnum
+
+         or more info like this for "TWO"
+         can span multiple lines
+
+      .. bro:enum:: Example::THREE Example::SimpleEnum
+
+   documentation for "SimpleEnum"
+   goes here.
+
+.. bro:type:: Example::SimpleRecord
+
+   :Type: :bro:type:`record`
+
+      field1: :bro:type:`count`
+         counts something
+
+      field2: :bro:type:`bool`
+         toggles something
+
+   general documentation for a type "SimpleRecord"
+   goes here.
+
+.. bro:type:: Example::ComplexRecord
+
+   :Type: :bro:type:`record`
+
+      field1: :bro:type:`count`
+         counts something
+
+      field2: :bro:type:`bool`
+         toggles something
+
+      field3: :bro:type:`Example::SimpleRecord`
+
+      msg: :bro:type:`string` :bro:attr:`&default` = ``"blah"`` :bro:attr:`&optional`
+         attributes are self-documenting
+
+   general documentation for a type "ComplexRecord" goes here
+
+Events
+~~~~~~
+.. bro:id:: Example::an_event
+
+   :Type: :bro:type:`event` (name: :bro:type:`string`)
+
+   Summarize "an_event" here.
+   Give more details about "an_event" here.
+   
+   :param name: describe the argument here
+
+Functions
+~~~~~~~~~
+.. bro:id:: Example::a_function
+
+   :Type: :bro:type:`function` (tag: :bro:type:`string`, msg: :bro:type:`string`) : :bro:type:`string`
+
+   Summarize purpose of "a_function" here.
+   Give more details about "a_function" here.
+   Separating the documentation of the params/return values with
+   empty comments is optional, but improves readability of script.
+   
+   
+   :param tag: function arguments can be described
+        like this
+   
+   :param msg: another param
+   
+   
+   :returns: describe the return type here
+
+Redefinitions
+~~~~~~~~~~~~~
+:bro:type:`Example::SimpleEnum`
+
+   :Type: :bro:type:`enum`
+
+      .. bro:enum:: Example::FOUR Example::SimpleEnum
+
+         and some documentation for "FOUR"
+
+      .. bro:enum:: Example::FIVE Example::SimpleEnum
+
+         also "FIVE" for good measure
+
+   document the "SimpleEnum" redef here
+
+Port Analysis
+-------------
+:ref:`More Information <common_port_analysis_doc>`
+
+SSL::
+
+    [ports={
+        563/tcp,
+        443/tcp
+    }]
+
+Packet Filter
+-------------
+:ref:`More Information <common_packet_filter_doc>`
+
+Filters added::
+
+    [nntps] = tcp port 563,
+    [ssl] = tcp port 443
+
+Private Interface
+-----------------
+State Variables
+~~~~~~~~~~~~~~~
+.. bro:id:: Example::example_ports
+
+   :Type: :bro:type:`set` [:bro:type:`port`]
+   :Attributes: :bro:attr:`&redef`
+   :Default:
+
+   ::
+
+      {
+         563/tcp,
+         443/tcp
+      }
+
+Types
+~~~~~
+.. bro:type:: Example::PrivateRecord
+
+   :Type: :bro:type:`record`
+
+      field1: :bro:type:`bool`
+
+      field2: :bro:type:`count`
+
+Functions
+~~~~~~~~~
+.. bro:id:: Example::function_without_proto
+
+   :Type: :bro:type:`function` (tag: :bro:type:`string`) : :bro:type:`string`
+

testing/btest/Baseline/doc.autogen-reST-records/autogen-reST-records.rst

@@ -0,0 +1,52 @@
+.. Automatically generated.  Do not edit.
+
+autogen-reST-records.bro
+========================
+
+:download:`Original Source File <autogen-reST-records.bro>`
+
+Overview
+--------
+
+
+Summary
+~~~~~~~
+Types
+#####
+============================================ ============================================================
+:bro:type:`SimpleRecord`: :bro:type:`record`
+
+:bro:type:`TestRecord`: :bro:type:`record`   Here's the ways records and record fields can be documented.
+============================================ ============================================================
+
+Public Interface
+----------------
+Types
+~~~~~
+.. bro:type:: SimpleRecord
+
+   :Type: :bro:type:`record`
+
+      field1: :bro:type:`bool`
+
+      field2: :bro:type:`count`
+
+.. bro:type:: TestRecord
+
+   :Type: :bro:type:`record`
+
+      A: :bro:type:`count`
+         document ``A``
+
+      B: :bro:type:`bool`
+         document ``B``
+
+      C: :bro:type:`SimpleRecord`
+         and now ``C``
+         is a declared type
+
+      D: :bro:type:`set` [:bro:type:`count`, :bro:type:`bool`]
+         sets/tables should show the index types
+
+   Here's the ways records and record fields can be documented.
+

testing/btest/Baseline/doc.autogen-reST-type-aliases/autogen-reST-type-aliases.rst

@@ -0,0 +1,61 @@
+.. Automatically generated.  Do not edit.
+
+autogen-reST-type-aliases.bro
+=============================
+
+:download:`Original Source File <autogen-reST-type-aliases.bro>`
+
+Overview
+--------
+
+
+Summary
+~~~~~~~
+State Variables
+###############
+======================================= =======================================================
+:bro:id:`a`: :bro:type:`TypeAlias`      But this should reference a type of ``TypeAlias``.
+
+:bro:id:`b`: :bro:type:`OtherTypeAlias` And this should reference a type of ``OtherTypeAlias``.
+======================================= =======================================================
+
+Types
+#####
+============================================ ==========================================================================
+:bro:type:`TypeAlias`: :bro:type:`bool`      This is just an alias for a builtin type ``bool``.
+
+:bro:type:`OtherTypeAlias`: :bro:type:`bool` We decided that creating alias "chains" might now be so useful to document
+                                             so this type just creates a cross reference to ``bool``.
+============================================ ==========================================================================
+
+Public Interface
+----------------
+State Variables
+~~~~~~~~~~~~~~~
+.. bro:id:: a
+
+   :Type: :bro:type:`TypeAlias`
+
+   But this should reference a type of ``TypeAlias``.
+
+.. bro:id:: b
+
+   :Type: :bro:type:`OtherTypeAlias`
+
+   And this should reference a type of ``OtherTypeAlias``.
+
+Types
+~~~~~
+.. bro:type:: TypeAlias
+
+   :Type: :bro:type:`bool`
+
+   This is just an alias for a builtin type ``bool``.
+
+.. bro:type:: OtherTypeAlias
+
+   :Type: :bro:type:`bool`
+
+   We decided that creating alias "chains" might now be so useful to document
+   so this type just creates a cross reference to ``bool``.
+

testing/btest/btest.cfg

@@ -1,6 +1,6 @@
 
 [btest]
-TestDirs    =
+TestDirs    = doc
 TmpDir      = %(testbase)s/.tmp
 BaselineDir = %(testbase)s/Baseline
 IgnoreDirs  = .svn CVS .tmp

testing/btest/doc/autogen-reST-enums.bro

@@ -0,0 +1,36 @@
+# @TEST-EXEC: bro --doc-scripts %INPUT
+# @TEST-EXEC: btest-diff autogen-reST-enums.rst
+
+## There's tons of ways an enum can look...
+type TestEnum1: enum {
+    ## like this
+    ONE,
+    TWO, ##< or like this
+    ## multiple
+    ## comments
+    THREE, ##< and even
+           ##< more comments
+};
+
+## The final comma is optional
+type TestEnum2: enum {
+    ## like this
+    A,
+    B, ##< or like this
+    ## multiple
+    ## comments
+    C  ##< and even
+           ##< more comments
+};
+
+## redefs should also work
+redef enum TestEnum1 += {
+    ## adding another
+    FOUR ##< value
+};
+
+## now with a comma
+redef enum TestEnum1 += {
+    ## adding another
+    FIVE, ##< value
+};

testing/btest/doc/autogen-reST-example

@@ -0,0 +1,2 @@
+@TEST-EXEC: bro --doc-scripts $DIST/doc/example.bro
+@TEST-EXEC: btest-diff example.rst

testing/btest/doc/autogen-reST-records.bro

@@ -0,0 +1,22 @@
+# @TEST-EXEC: bro --doc-scripts %INPUT
+# @TEST-EXEC: btest-diff autogen-reST-records.rst
+
+# undocumented record
+type SimpleRecord: record {
+    field1: bool;
+    field2: count;
+};
+
+## Here's the ways records and record fields can be documented.
+type TestRecord: record {
+    ## document ``A``
+    A: count;
+
+    B: bool;  ##< document ``B``
+
+    ## and now ``C``
+    C: SimpleRecord; ##< is a declared type
+
+    ## sets/tables should show the index types
+    D: set[count, bool];
+};

testing/btest/doc/autogen-reST-type-aliases.bro

@@ -0,0 +1,15 @@
+# @TEST-EXEC: bro --doc-scripts %INPUT
+# @TEST-EXEC: btest-diff autogen-reST-type-aliases.rst
+
+## This is just an alias for a builtin type ``bool``.
+type TypeAlias: bool;
+
+## We decided that creating alias "chains" might now be so useful to document
+## so this type just creates a cross reference to ``bool``.
+type OtherTypeAlias: TypeAlias;
+
+## But this should reference a type of ``TypeAlias``.
+global a: TypeAlias;
+
+## And this should reference a type of ``OtherTypeAlias``.
+global b: OtherTypeAlias;