Overhaul of "doc" build target for generating policy script documentation.

It's now all implemented in CMake scripting.

The generation of reST docs is now a distinct target, "restdoc", while
the target to generate HTML docs, "doc", depends on "restdoc".  reST doc
generation supports incremental builds (documentation for a given policy
script is only regenerated when it is out of date), but HTML doc generation
via ``make doc`` is not incremental (Sphinx always starts with fresh input).

Building the "restdoc" target is now covered by a btest to ensure all
policy scripts are parse-able when Bro is in "doc mode".

Generated reST docs should now support "@load"ing from subdirectories.  e.g.
"@load foo/baz" and "@load bar/baz" will now generate the right xref links.

doc/scripts/CMakeLists.txt

@@ -1,42 +1,250 @@
 set(POLICY_SRC_DIR ${PROJECT_SOURCE_DIR}/policy)
+set(RST_OUTPUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/rest_output)
 set(DOC_OUTPUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/out)
 set(DOC_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/source)
 set(DOC_SOURCE_WORKDIR ${CMAKE_CURRENT_BINARY_DIR}/source)
 
 file(GLOB_RECURSE DOC_SOURCES FOLLOW_SYMLINKS "*")
 
+# configure the Sphinx config file (expand variables CMake might know about)
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/conf.py.in
                ${CMAKE_CURRENT_BINARY_DIR}/conf.py
                @ONLY)
-configure_file(${CMAKE_CURRENT_SOURCE_DIR}/scripts/generate_reST_docs.py.in
-               ${CMAKE_CURRENT_BINARY_DIR}/generate_reST_docs.py
-               @ONLY)
-configure_file(${CMAKE_CURRENT_SOURCE_DIR}/scripts/BroToReST.py.in
-               ${CMAKE_CURRENT_BINARY_DIR}/BroToReST.py
-               @ONLY)
 
+# find out what BROPATH to use when executing bro
+execute_process(COMMAND ${CMAKE_BINARY_DIR}/bro-path-dev
+                OUTPUT_VARIABLE BROPATH
+                RESULT_VARIABLE retval
+                OUTPUT_STRIP_TRAILING_WHITESPACE)
+if (NOT ${retval} EQUAL 0)
+    message(FATAL_ERROR "Problem setting BROPATH")
+endif ()
+
+# This macro is used to add a new makefile target for reST policy script
+# documentation that can be generated using Bro itself to parse policy scripts.
+# It's called like:
+#
+#     rest_target(srcDir broInput [group])
+#
+# srcDir: the directory which contains broInput
+# broInput: the file name of a bro policy script
+# group: optional name of group that the script documentation will belong to
+#
+# In addition to adding the makefile target, several CMake variables are set:
+#
+# MASTER_POLICY_INDEX_TEXT: a running list of policy scripts docs that have
+#   been generated so far, formatted such that it can be appended to a file
+#   that ends in a Sphinx toctree directive
+# ALL_REST_OUTPUTS: a running list (the CMake list type) of all reST docs
+#   that are to be generated
+# MASTER_GROUP_LIST: a running list (the CMake list type) of all script groups
+# ${group}_TEXT: a running list of policy script :doc: references and summary
+#   text for a given group
+#
+macro(REST_TARGET srcDir broInput)
+    get_filename_component(basename ${broInput} NAME_WE)
+    get_filename_component(extension ${broInput} EXT)
+    get_filename_component(relDstDir ${broInput} PATH)
+
+    if (${extension} STREQUAL ".bif.bro")
+        set(basename "${basename}.bif")
+    elseif (${extension} STREQUAL ".init")
+        set(basename "${basename}.init")
+    endif ()
+
+    set (restFile "${basename}.rst")
+
+    if (NOT relDstDir)
+        set(docName "${basename}")
+        set(dstDir "${RST_OUTPUT_DIR}")
+    else ()
+        set(docName "${relDstDir}/${basename}")
+        set(dstDir "${RST_OUTPUT_DIR}/${relDstDir}")
+    endif ()
+
+    set(restOutput "${dstDir}/${restFile}")
+
+    set(indexEntry "   ${docName} <${docName}>")
+    set(MASTER_POLICY_INDEX_TEXT "${MASTER_POLICY_INDEX_TEXT}\n${indexEntry}")
+    list(APPEND ALL_REST_OUTPUTS ${restOutput})
+
+    if (NOT "${ARGN}" STREQUAL "")
+        set(group ${ARGN})
+        # add group to master group list if not already in it
+        list(FIND MASTER_GROUP_LIST ${group} _found)
+        if (_found EQUAL -1)
+            list(APPEND MASTER_GROUP_LIST ${group})
+            if (MASTER_GROUP_LIST_TEXT)
+               set(MASTER_GROUP_LIST_TEXT "${MASTER_GROUP_LIST_TEXT}\n${group}")
+            else ()
+               set(MASTER_GROUP_LIST_TEXT "${group}")
+            endif ()
+        endif ()
+
+        # add script's summary documentation to text associated with the group
+        set(${group}_TEXT "${${group}_TEXT}\n\n:doc:`/policy/${docName}`\n")
+        file(STRINGS ${srcDir}/${broInput} summary_text REGEX "^\#\#!")
+        foreach (line ${summary_text})
+            string(REGEX REPLACE "^\#\#!" "   " line ${line})
+            set(${group}_TEXT "${${group}_TEXT}\n${line}")
+        endforeach ()
+    else ()
+        set(group "")
+    endif ()
+
+    if (${group} STREQUAL "default" OR ${group} STREQUAL "bifs")
+        set(BRO_ARGS --doc-scripts --exec '')
+    else ()
+        set(BRO_ARGS --doc-scripts ${srcDir}/${broInput})
+    endif ()
+
+    add_custom_command(OUTPUT ${restOutput}
+        # delete any leftover state from previous bro runs
+        COMMAND "${CMAKE_COMMAND}"
+        ARGS -E remove_directory .state
+        # generate the reST documentation using bro
+        COMMAND BROPATH=${BROPATH} ${CMAKE_BINARY_DIR}/src/bro
+        ARGS ${BRO_ARGS} || (rm -rf .state *.log *.rst && exit 1)
+        # move generated doc into a new directory tree that
+        # defines the final structure of documents
+        COMMAND "${CMAKE_COMMAND}"
+        ARGS -E make_directory ${dstDir}
+        COMMAND "${CMAKE_COMMAND}"
+        ARGS -E copy ${restFile} ${restOutput}
+        # copy the bro policy script, too
+        COMMAND "${CMAKE_COMMAND}"
+        ARGS -E copy ${srcDir}/${broInput} ${dstDir}
+        # clean up the build directory
+        COMMAND rm
+        ARGS -rf .state *.log *.rst
+        DEPENDS bro
+        DEPENDS ${srcDir}/${broInput}
+        COMMENT "[Bro] Generating reST docs for ${broInput}"
+    )
+
+endmacro(REST_TARGET)
+
+# Schedule Bro scripts for which to generate documentation.
+# Note: the script may be located in a subdirectory off of one of the main
+# directories in BROPATH.  In that case, just list the script as 'foo/bar.bro'
+rest_target(${POLICY_SRC_DIR} alarm.bro                         user)
+rest_target(${POLICY_SRC_DIR} arp.bro                           user)
+rest_target(${POLICY_SRC_DIR} conn.bro                          user)
+rest_target(${POLICY_SRC_DIR} dhcp.bro                          user)
+rest_target(${POLICY_SRC_DIR} dns.bro                           user)
+rest_target(${POLICY_SRC_DIR} ftp.bro                           user)
+rest_target(${POLICY_SRC_DIR} http.bro                          user)
+rest_target(${POLICY_SRC_DIR} http-reply.bro                    user)
+rest_target(${POLICY_SRC_DIR} http-request.bro                  user)
+rest_target(${POLICY_SRC_DIR} irc.bro                           user)
+rest_target(${POLICY_SRC_DIR} smtp.bro                          user)
+rest_target(${POLICY_SRC_DIR} ssl.bro                           user)
+rest_target(${POLICY_SRC_DIR} ssl-ciphers.bro                   user)
+rest_target(${POLICY_SRC_DIR} ssl-errors.bro                    user)
+rest_target(${POLICY_SRC_DIR} synflood.bro                      user)
+rest_target(${POLICY_SRC_DIR} tcp.bro                           user)
+rest_target(${POLICY_SRC_DIR} udp.bro                           user)
+rest_target(${POLICY_SRC_DIR} weird.bro                         user)
+rest_target(${CMAKE_CURRENT_SOURCE_DIR} example.bro             internal)
+
+# Finding out what scripts bro will generate documentation for by default
+# can be done like: `bro --doc-scripts --exec ""`
+rest_target(${POLICY_SRC_DIR} bro.init                          default)
+rest_target(${POLICY_SRC_DIR} logging-ascii.bro                 default)
+rest_target(${POLICY_SRC_DIR} logging.bro                       default)
+rest_target(${POLICY_SRC_DIR} pcap.bro                          default)
+rest_target(${POLICY_SRC_DIR} server-ports.bro                  default)
+rest_target(${CMAKE_BINARY_DIR}/src bro.bif.bro                 bifs)
+rest_target(${CMAKE_BINARY_DIR}/src const.bif.bro               bifs)
+rest_target(${CMAKE_BINARY_DIR}/src event.bif.bro               bifs)
+rest_target(${CMAKE_BINARY_DIR}/src logging.bif.bro             bifs)
+rest_target(${CMAKE_BINARY_DIR}/src strings.bif.bro             bifs)
+rest_target(${CMAKE_BINARY_DIR}/src types.bif.bro               bifs)
+
+# create temporary list of all docs to include in the master policy/index file
+file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/tmp_policy_index
+    "${MASTER_POLICY_INDEX_TEXT}")
+
+# create temporary file containing list of all groups
+file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/group_list
+    "${MASTER_GROUP_LIST_TEXT}")
+
+# create temporary file containing the summary text for all scripts in a group
+foreach (group ${MASTER_GROUP_LIST})
+    file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/${group}_text
+        "${${group}_TEXT}")
+endforeach ()
+
+# remove previously generated docs no longer scheduled for generation
+if (EXISTS ${RST_OUTPUT_DIR})
+    file(GLOB_RECURSE EXISTING_REST_DOCS "${RST_OUTPUT_DIR}/*.rst")
+    foreach (_doc ${EXISTING_REST_DOCS})
+        list(FIND ALL_REST_OUTPUTS ${_doc} _found)
+        if (_found EQUAL -1)
+            file(REMOVE ${_doc})
+            message(STATUS "Removing stale reST doc: ${_doc}")
+        endif ()
+    endforeach ()
+endif ()
+
+# The "restdoc" target uses Bro to parse policy scripts in order to
+# generate reST documentation from them.
+add_custom_target(restdoc
+                  # create symlink to the reST output directory for convenience
+                  COMMAND "${CMAKE_COMMAND}" -E create_symlink
+                          ${RST_OUTPUT_DIR}
+                          ${CMAKE_BINARY_DIR}/reST
+                  DEPENDS ${ALL_REST_OUTPUTS})
+
+# The "restclean" target removes all generated reST documentation from the
+# build directory.
+add_custom_target(restclean
+                  COMMAND "${CMAKE_COMMAND}" -E remove_directory
+                          ${RST_OUTPUT_DIR}
+                  VERBATIM)
+
+# The "doc" target generates reST documentation for any outdated bro scripts
+# and then uses Sphinx to generate HTML documentation from the reST
 add_custom_target(doc
+                  # copy the template documentation to the build directory
+                  # to give as input for sphinx
                   COMMAND "${CMAKE_COMMAND}" -E copy_directory
                           ${DOC_SOURCE_DIR}
                           ${DOC_SOURCE_WORKDIR}
-                  COMMAND python generate_reST_docs.py
+                  # copy generated policy script documentation into the
+                  # working copy of the template documentation
+                  COMMAND "${CMAKE_COMMAND}" -E copy_directory
+                          ${RST_OUTPUT_DIR}
+                          ${DOC_SOURCE_WORKDIR}/policy
+                  # append to the master index of all policy scripts
+                  COMMAND cat ${CMAKE_CURRENT_BINARY_DIR}/tmp_policy_index >>
+                          ${DOC_SOURCE_WORKDIR}/policy/index.rst
+                  # construct the reST file for all groups
+                  COMMAND xargs -I{} sh -c 'cat "$$1_text" >>
+                          "${DOC_SOURCE_WORKDIR}/$$1.rst"' -- {} <
+                          ${CMAKE_CURRENT_BINARY_DIR}/group_list
+                  # tell sphinx to generate html
                   COMMAND sphinx-build
                           -b html
                           -c ${CMAKE_CURRENT_BINARY_DIR}
                           -d ${DOC_OUTPUT_DIR}/doctrees
                           ${DOC_SOURCE_WORKDIR}
                           ${DOC_OUTPUT_DIR}/html
+                  # create symlink to the html output directory for convenience
+                  COMMAND "${CMAKE_COMMAND}" -E create_symlink
+                          ${DOC_OUTPUT_DIR}/html
+                          ${CMAKE_BINARY_DIR}/html
                   WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
-                  COMMENT "[Sphinx] Generating Script Documentation"
+                  COMMENT "[Sphinx] Generating HTML policy script docs"
-                  VERBATIM
                   # SOURCES just adds stuff to IDE projects as a convienience
-                  SOURCES ${DOC_SOURCES})
+                  SOURCES ${DOC_SOURCES}
+                  DEPENDS restdoc docclean)
 
-add_dependencies(doc bro doc-clean)
+# The "docclean" target removes just the Sphinx input/output directories
-
+# from the build directory.
-add_custom_target(doc-clean
+add_custom_target(docclean
                   COMMAND "${CMAKE_COMMAND}" -E remove_directory
-                          ${CMAKE_CURRENT_BINARY_DIR}/source
+                          ${DOC_SOURCE_WORKDIR}
                   COMMAND "${CMAKE_COMMAND}" -E remove_directory
                           ${DOC_OUTPUT_DIR}
                   VERBATIM)

doc/scripts/README

@@ -1,8 +1,21 @@
 This directory contains scripts and templates that can be used to automate
-the generation of Bro script documentation.  Two build targets are defined
+the generation of Bro script documentation.  Several build targets are defined
 by CMake:
 
-``make doc``
+``restdoc``
+
+    This target uses Bro to parse policy scripts in order to generate
+    reStructuredText (reST) documentation from them.  The list of scripts
+    for which to generate reST documentation is defined in the
+    ``CMakeLists.txt`` file in this directory.  Script documentation is
+    rebuild automatically if the policy script from which it is derived
+    or the Bro binary becomes out of date
+
+    The resulting output from this target can be found in the CMake
+    ``build/`` directory inside ``reST`` (a symlink to
+    ``doc/scripts/rest_output``).
+
+``doc``
 
     This target depends on a Python interpreter (>=2.5) and
     `Sphinx <http://sphinx.pocoo.org/>`_ being installed.  Sphinx can be
@@ -10,27 +23,31 @@ by CMake:
 
         > sudo easy_install sphinx
 
-    This target will also first build the bro binary if it is not already
+    This target will first build ``restdoc`` target and then copy the
-    since the generation of reStructuredText (reST) documentation from
+    resulting reST files as an input directory to Sphinx.
-    Bro scripts is integrated within the parsing process.
 
-    After completion, HTML documentation can be located inside the CMake
+    After completion, HTML documentation can be located in the CMake
-    ``build/`` directory as ``doc/scripts/out/html``.  The generated
+    ``build/`` directory inside ``html`` (a symlink to
-    reST documentation will be located in ``doc/scripts/source/policy``.
+    ``doc/scripts/out/html``)
 
-``make doc-clean``
+``restclean``
+
+    This target removes any reST documentation that has been generated so far.
+
+``docclean``
 
     This target removes Sphinx inputs and outputs from the CMake ``build/`` dir.
 
-To schedule a script to be documented, edit ``scripts/generate_reST_docs.py.in``
+To schedule a script to be documented, edit ``CMakeLists.txt`` inside this
-and try adding the name of the script along with an optional script group to
+directory add a call to the ``rest_target()`` macro.  Calling that macro
-the ``docs`` dictionary.  That python script also shows other, more specialized
+with a group name for the script is optional, but if not given, the only
-methods for generating documentation for some types of corner-cases.
+link to the script will be in the master TOC tree for all policy scripts.
 
 When adding a new logical grouping for generated scripts, create a new
-reST document in ``source/policy/<group_name>.rst`` and add some default
+reST document in ``source/<group_name>.rst`` and add some default
-documentation.  References to (and summaries of) documents associated with
+documentation for the group.  References to (and summaries of) documents
-the group get appended to this file during the ``make doc`` process.
+associated with the group get appended to this file during the
+``make doc`` process.
 
 The Sphinx source tree template in ``source/`` can be modified to add more
 common/general documentation, style sheets, JavaScript, etc.  The Sphinx

doc/scripts/scripts/BroToReST.py.in

@@ -1,152 +0,0 @@
-#! /usr/bin/env python
-
-import os
-import subprocess
-import shutil
-import glob
-import string
-import sys
-
-BRO = "@CMAKE_BINARY_DIR@/src/bro"
-BROPATHDEV = "`@CMAKE_BINARY_DIR@/bro-path-dev`"
-BRO_ARGS = "--doc-scripts"
-DOC_DST_DIR = "@DOC_SOURCE_WORKDIR@/policy"
-BROPATH = subprocess.Popen("@CMAKE_BINARY_DIR@/bro-path-dev", shell=True,
-        stdout=subprocess.PIPE, stderr=subprocess.STDOUT).stdout.readline()
-
-class BroToReST:
-    """A class to encapsulate the the generation of reST documentation from
-    a given Bro script.
-    """
-
-    bro_src_file = None
-    doc_src_file = None
-    load_via_stdin = False
-    group = None
-
-    def __init__(self, src_file, load_method=False, search_dir=None, group=None):
-        """
-        :param src_file: the file name of a Bro script (not a path)
-        :param load_method: T if script must be loaded by Bro via a stdin
-            redirection of "@load <script>", F if script can be loaded as
-            a command line argument to Bro
-        :param search_dir: a list of directories in which to search for
-            src_file.  If None, the default BROPATH is used.
-        :param group: a string representing a logical group that the script's
-            documentation should belong to.  A corresponding <group>.rst
-            document must be pre-existing in the policy/ dir of the source tree
-            used by Sphinx.
-        """
-
-        self.bro_src_file = FindBroScript(src_file, search_dir)
-        self.load_via_stdin = load_method
-        self.group = group
-
-        # formulate doc_src_file from src_file
-        filename = os.path.basename(src_file)
-        basename, ext = os.path.splitext(filename)
-        if ext == ".bro":
-            self.doc_src_file = basename + ".rst"
-        else:
-            self.doc_src_file = filename + ".rst"
-
-    def __str__(self):
-        return "bro_src_file: " + self.bro_src_file \
-           + "\ndoc_src_file: " + self.doc_src_file \
-           + "\ndoc_dst_file: " + os.path.join(DOC_DST_DIR, self.doc_src_file) \
-           + "\nstdin_load: %s" % self.load_via_stdin \
-           + "\ngroup: %s" % self.group
-
-    def GenDoc(self):
-        """Generates the reST documentation for a Bro script and copies
-        both the documentation and original script into Sphinx source tree.
-        If the documentation belongs to a group, the necessary modifications
-        to add it to the group's documentation are done.  Afterwards, any
-        files with a ".rst" suffix are removed for the working directory.
-        """
-
-        bro_src_basename = os.path.basename(self.bro_src_file)
-
-        if self.load_via_stdin:
-            cmd = "echo '@load %s' | %s %s" % (bro_src_basename, BRO, BRO_ARGS)
-        else:
-            cmd = "%s %s %s" % (BRO, BRO_ARGS, self.bro_src_file)
-
-        p = subprocess.Popen(cmd, shell=True, env={"BROPATH": BROPATH})
-
-        if p.wait() == 0:
-            shutil.copy(self.doc_src_file, DOC_DST_DIR)
-            shutil.copy(self.bro_src_file, DOC_DST_DIR)
-            AppendToDocGroup(self.group, self.bro_src_file, self.doc_src_file)
-
-        for leftover in glob.glob("*.rst"):
-            os.remove(leftover)
-
-def GenDocs(doc_dict, load_method=False):
-    """Generates reST documentation for all scripts in the given dictionary.
-
-    :param doc_dict: a dictionary whose keys are file names of Bro scripts
-        (not paths), and whose value is the logical documentation group
-        it belongs to
-    :param load_method: T if script must be loaded by Bro via a stdin
-            redirection of "@load <script>", F if script can be loaded as
-            a command line argument to Bro
-    """
-
-    for k, v in doc_dict.iteritems():
-        doc = BroToReST(k, load_method, group=v)
-        print "Generating reST document for " + k
-        doc.GenDoc()
-
-def FindBroScript(src_file, search_dir=None):
-    """Search a set of paths for a given Bro script and return the absolute
-    path to it.
-
-    :param src_file: the file name of a Bro script (not a path)
-    :param search_dir: a list of directories in which to search for
-        src_file.  If None, the default BROPATH is used.
-    """
-    if search_dir is None:
-        search_dir = string.split(BROPATH, ":")
-    for path in search_dir:
-        abs_path = os.path.join(path, src_file)
-        if os.path.exists(abs_path):
-            return abs_path
-    print >> sys.stderr, "Couldn't find '%s'" % src_file
-    return None
-
-def AppendToDocGroup(group, src_file, doc_file):
-    """Adds a reference to the given documentation for a Bro script
-    to the documentation file for it's associated group.  Also, associated
-    summary text (comments marked up like "##!" in the original Bro script
-    source) are added.
-
-    :param group: a string representing a logical group that the script's
-        documentation should belong to.  A corresponding <group>.rst
-        document must be pre-existing in the policy/ dir of the source tree
-        used by Sphinx.
-    :param src_file: a path to the original Bro script source file
-    :param doc_file: the file name of a script's generated reST document
-    """
-    if group is None:
-        return
-
-    group_file = os.path.join(DOC_DST_DIR, group + ".rst")
-    if not os.path.exists(group_file):
-        print >> sys.stderr, "Group file doesn't exist: " + group_file
-        return
-
-    summary_comments = []
-
-    with open(src_file, 'r') as f:
-        for line in f:
-            sum_pos = string.find(line, "##!")
-            if sum_pos != -1:
-                summary_comments.append(line[(sum_pos+3):])
-
-    doc_name, ext = os.path.splitext(doc_file)
-
-    with open(group_file, 'a') as f:
-        f.write("\n:doc:`%s`\n" % doc_name)
-        for line in summary_comments:
-            f.write(line)

doc/scripts/scripts/generate_reST_docs.py.in

@@ -1,71 +0,0 @@
-#! /usr/bin/env python
-
-import os
-import subprocess
-import shutil
-import glob
-import string
-import sys
-from BroToReST import *
-
-# TODO: generate docs for more scripts
-# TODO: the groups are just made up to test the functionality, fix them
-
-# Scripts that can be loaded by bro via command line argument:
-docs = {
-    "alarm.bro":        "internal",
-    "arp.bro":          "user",
-    "conn.bro":         "internal",
-    "dhcp.bro":         "user",
-    "dns.bro":          "user",
-    "ftp.bro":          "user",
-    "http.bro":         "user",
-    "http-reply.bro":   None,
-    "http-request.bro": None,
-    "irc.bro":          "user",
-    "smtp.bro":         "user",
-    "ssl.bro":          "user",
-    "ssl-ciphers.bro":  None,
-    "ssl-errors.bro":   None,
-    "synflood.bro":     "user",
-    "tcp.bro":          "user",
-    "udp.bro":          "user",
-    "weird.bro":        "internal",
-}
-
-# Scripts that can't be loaded by bro via command line argument (possible
-# due to dependency issues), but can be loaded via an @load on stdin:
-stdin_docs = {
-    "notice.bro": "internal",
-}
-
-GenDocs(docs)
-GenDocs(stdin_docs, True)
-
-# The example documentation script doesn't live on the BROPATH, so
-# explicitly generate the docs for it like this:
-BroToReST("example.bro", False, ["@PROJECT_SOURCE_DIR@/doc/scripts"], group="internal").GenDoc()
-
-# Generate documentation for stuff that's always loaded into bro by default:
-cmd = "echo '' | %s %s" % (BRO, BRO_ARGS)
-p = subprocess.Popen(cmd, shell=True, env={"BROPATH": BROPATH})
-if p.wait() == 0:
-    for doc in glob.glob("*.rst"):
-        if doc == "<stdin>.rst":
-            os.remove(doc)
-            continue
-
-        basename, ext = os.path.splitext(doc)
-        basename2, ext = os.path.splitext(basename)
-        if ext == ".init":
-            src_file = basename
-        else:
-            src_file = basename + ".bro"
-        src_file = FindBroScript(src_file)
-        shutil.copy(src_file, DOC_DST_DIR)
-        shutil.copy(doc, DOC_DST_DIR)
-        if ext == ".bif":
-            AppendToDocGroup("bifs", src_file, doc)
-        else:
-            AppendToDocGroup("default", src_file, doc)
-        os.remove(doc)

doc/scripts/source/index.rst

@@ -11,11 +11,11 @@ Contents:
 
    common
    builtins
-   policy/default
-   policy/user
-   policy/bifs
-   policy/internal
    policy/index
+   default
+   bifs
+   user
+   internal
 
 Indices and tables
 ==================

doc/scripts/source/policy/index.rst

@@ -1,10 +1,6 @@
 Index of All Policy Script Documentation
 ========================================
 
-Contents:
-
 .. toctree::
    :maxdepth: 1
-   :glob:
 
-   *

src/BroDoc.cc

@@ -58,15 +58,8 @@ void BroDoc::AddImport(const std::string& s)
 
 	if ( ext_pos == std::string::npos )
 		imports.push_back(s);
-
 	else
-		{
+		imports.push_back(s.substr(0, ext_pos));
-		if ( s.substr(ext_pos + 1) == "bro" )
-			imports.push_back(s.substr(0, ext_pos));
-		else
-			fprintf(stderr, "Warning: skipped documenting @load of file "
-			                "without .bro extension: %s\n", s.c_str());
-		}
 	}
 
 void BroDoc::SetPacketFilter(const std::string& s)
@@ -116,7 +109,13 @@ void BroDoc::WriteDocFile() const
 	if ( ! imports.empty() )
 		{
 		WriteToDoc(":Imports: ");
-		WriteStringList(":doc:`%s`, ", ":doc:`%s`\n", imports);
+		std::list<std::string>::const_iterator it;
+		for ( it = imports.begin(); it != imports.end(); ++it)
+			{
+			if ( it != imports.begin() ) WriteToDoc(", ");
+			WriteToDoc(":doc:`%s </policy/%s>`", it->c_str(), it->c_str());
+			}
+		WriteToDoc("\n");
 		}
 
 	WriteToDoc("\n");

testing/btest/Baseline/doc.autogen-reST-example/example.rst

@@ -17,7 +17,7 @@ these comments are transferred directly into the auto-generated
 
 :Author: Jon Siwek <jsiwek@ncsa.illinois.edu>
 
-:Imports: :doc:`notice`
+:Imports: :doc:`notice </policy/notice>`
 
 Summary
 ~~~~~~~
@@ -234,8 +234,8 @@ Port Analysis
 SSL::
 
     [ports={
-        563/tcp,
+        443/tcp,
-        443/tcp
+        562/tcp
     }]
 
 Packet Filter
@@ -244,8 +244,8 @@ Packet Filter
 
 Filters added::
 
-    [nntps] = tcp port 563,
+    [ssl] = tcp port 443,
-    [ssl] = tcp port 443
+    [nntps] = tcp port 562
 
 Private Interface
 -----------------
@@ -260,8 +260,8 @@ State Variables
    ::
 
       {
-         563/tcp,
+         443/tcp,
-         443/tcp
+         562/tcp
       }
 
 Types

testing/btest/doc/autogen-reST-all

@@ -0,0 +1 @@
+@TEST-EXEC: cd $BUILD && make restdoc

testing/btest/doc/autogen-reST-example

@@ -1,2 +1,2 @@
 @TEST-EXEC: bro --doc-scripts $DIST/doc/scripts/example.bro
-@TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/doc/example-diff-canonifier.py btest-diff example.rst
+@TEST-EXEC: btest-diff example.rst