Integrate new Broxygen functionality into Sphinx.

Add a "broxygen" domain Sphinx extension w/ directives to allow
on-the-fly documentation to be generated w/ Bro and included in files.

This means all autogenerated reST docs are now done by Bro.  The odd
CMake/Python glue scipts which used to generate some portions are now
gone.  Bro and the Sphinx extension handle checking for outdated docs
themselves.

Parallel builds of `make doc` target should now work (mostly because
I don't think there's any tasks that can be done in parallel anymore).

Overall, this seems to simplify things and make the Broxygen-generated
portions of the documentation visible/traceable from the main Sphinx
source tree.  The one odd thing still is that per-script documentation
is rsync'd in to a shadow copy of the Sphinx source tree within the
build dir.  This is less elegant than using the new broxygen extension
to make per-script docs, but rsync is faster and simpler.  Simpler as in
less code because it seems like, in the best case, I'd need to write a
custom Sphinx Builder to be able to get that to even work.

Makefile

@@ -29,18 +29,6 @@ doc: configured
 docclean: configured
 	$(MAKE) -C $(BUILD) $@
 
-restdoc: configured
-	$(MAKE) -C $(BUILD) $@
-
-restclean: configured
-	$(MAKE) -C $(BUILD) $@
-
-broxygen: configured
-	$(MAKE) -C $(BUILD) $@
-
-broxygenclean: configured
-	$(MAKE) -C $(BUILD) $@
-
 dist:
 	@rm -rf $(VERSION_FULL) $(VERSION_FULL).tgz
 	@rm -rf $(VERSION_MIN) $(VERSION_MIN).tgz

doc/CMakeLists.txt

@@ -1,75 +1,86 @@
-set(BIF_SRC_DIR ${PROJECT_SOURCE_DIR}/src)
+set(BROCCOLI_DOCS_SRC ${CMAKE_BINARY_DIR}/aux/broccoli/doc/html)
-set(RST_OUTPUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/rest_output)
+set(BROCCOLI_DOCS_DST ${CMAKE_BINARY_DIR}/html/broccoli-api)
-set(DOC_OUTPUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/out)
+set(SPHINX_INPUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/sphinx_input)
-set(DOC_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR})
+set(SPHINX_OUTPUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/sphinx_output)
-set(DOC_SOURCE_WORKDIR ${CMAKE_CURRENT_BINARY_DIR}/sphinx-sources)
+set(BROXYGEN_SCRIPT_OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/broxygen_script_output)
+set(BROXYGEN_CACHE_DIR ${CMAKE_CURRENT_BINARY_DIR}/broxygen_cache)
 
-set(MASTER_POLICY_INDEX ${CMAKE_CURRENT_BINARY_DIR}/scripts/policy_index)
+# Find out what BROPATH to use when executing bro.
-set(MASTER_PACKAGE_INDEX ${CMAKE_CURRENT_BINARY_DIR}/scripts/pkg_index)
+execute_process(COMMAND ${CMAKE_BINARY_DIR}/bro-path-dev
+                OUTPUT_VARIABLE BROPATH
+                RESULT_VARIABLE retval
+                OUTPUT_STRIP_TRAILING_WHITESPACE)
+if (NOT ${retval} EQUAL 0)
+    message(FATAL_ERROR "Problem setting BROPATH")
+endif ()
 
-file(GLOB_RECURSE DOC_SOURCES FOLLOW_SYMLINKS "*")
+set(BROMAGIC ${BRO_MAGIC_SOURCE_PATH})
 
-# configure the Sphinx config file (expand variables CMake might know about)
+# Configure the Sphinx config file (expand variables CMake might know about).
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/conf.py.in
                ${CMAKE_CURRENT_BINARY_DIR}/conf.py
                @ONLY)
 
-add_subdirectory(scripts)
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/broxygen.conf.in
+               ${CMAKE_CURRENT_BINARY_DIR}/broxygen.conf
+               @ONLY)
 
-# The "broxygen" target generates reST documentation for any outdated bro
+add_custom_target(sphinxdoc
-# scripts and then uses Sphinx to generate HTML documentation from the reST
+    # Copy the template documentation to build directory to use as input tree
-add_custom_target(broxygen
+    # for Sphinx.  This is needed because some parts are dynamically generated
-                  # copy the template documentation to the build directory
+    # in to that tree by Bro/Broxygen.
-                  # to give as input for sphinx
+    COMMAND rsync -q -r --copy-links --times --delete
-                  COMMAND rsync -r --copy-links --times
+            --filter='protect scripts/*'
-                          ${DOC_SOURCE_DIR}/
+            ${CMAKE_CURRENT_SOURCE_DIR}/ ${SPHINX_INPUT_DIR}
-                          ${DOC_SOURCE_WORKDIR}
+    # Use Bro/Broxygen to dynamically generate reST for all Bro scripts.
-                  # copy generated policy script documentation into the
+    COMMAND BROPATH=${BROPATH}
-                  # working copy of the template documentation
+            BROMAGIC=${BROMAGIC}
-                  COMMAND rsync -r --copy-links --times
+            ${CMAKE_BINARY_DIR}/src/bro
-                          ${RST_OUTPUT_DIR}/
+            -X ${CMAKE_CURRENT_BINARY_DIR}/broxygen.conf
-                          ${DOC_SOURCE_WORKDIR}/scripts
+            broxygen >/dev/null
-                  # append to the master index of all policy scripts
+    # Rsync over the generated reST to the Sphinx source tree in the build dir.
-                  COMMAND cat ${MASTER_POLICY_INDEX} >>
+    COMMAND rsync -q -r --copy-links --times --delete --filter='protect *.bro'
-                          ${DOC_SOURCE_WORKDIR}/scripts/scripts.rst
+            ${BROXYGEN_SCRIPT_OUTPUT}/ ${SPHINX_INPUT_DIR}/scripts
-                  # append to the master index of all policy packages
+    # Rsync over Bro scripts to the Sphinx source tree in the build dir.
-                  COMMAND cat ${MASTER_PACKAGE_INDEX} >>
+    # These are used by :download: references in the generated script docs.
-                          ${DOC_SOURCE_WORKDIR}/scripts/packages.rst
+    COMMAND rsync -q -r --copy-links --times --delete
-                  # construct a reST file for each group
+            --filter='protect /base/bif/*' --filter='protect *.rst'
-                  COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/bin/group_index_generator.py
+            --filter='include */' --filter='include *.bro' --filter='exclude *'
-                          ${CMAKE_CURRENT_BINARY_DIR}/scripts/group_list
+            ${CMAKE_SOURCE_DIR}/scripts/ ${SPHINX_INPUT_DIR}/scripts
-                          ${CMAKE_CURRENT_BINARY_DIR}/scripts
+    # Rsync over Bro scripts created by BIF compiler to the Sphinx source tree.
-                          ${DOC_SOURCE_WORKDIR}/scripts
+    COMMAND rsync -q -r --copy-links --times --delete
-                  # tell sphinx to generate html
+            --filter='protect *.rst' --filter='include */'
-                  COMMAND sphinx-build
+            --filter='include *.bro' --filter='exclude *'
-                          -b html
+            ${CMAKE_BINARY_DIR}/scripts/base/bif/
-                          -c ${CMAKE_CURRENT_BINARY_DIR}
+            ${SPHINX_INPUT_DIR}/scripts/base/bif
-                          -d ${DOC_OUTPUT_DIR}/doctrees
+    # Use Sphinx to build HTML.
-                          ${DOC_SOURCE_WORKDIR}
+    COMMAND sphinx-build
-                          ${DOC_OUTPUT_DIR}/html
+            -b html
-                  # create symlink to the html output directory for convenience
+            -c ${CMAKE_CURRENT_BINARY_DIR}
-                  COMMAND "${CMAKE_COMMAND}" -E create_symlink
+            -d ${SPHINX_OUTPUT_DIR}/doctrees
-                          ${DOC_OUTPUT_DIR}/html
+            ${SPHINX_INPUT_DIR}
-                          ${CMAKE_BINARY_DIR}/html
+            ${SPHINX_OUTPUT_DIR}/html
-                  # copy Broccoli API reference into output dir if it exists
+    # Create symlink to the html output directory for convenience.
-                  COMMAND test -d ${CMAKE_BINARY_DIR}/aux/broccoli/doc/html && ( rm -rf ${CMAKE_BINARY_DIR}/html/broccoli-api && cp -r ${CMAKE_BINARY_DIR}/aux/broccoli/doc/html ${CMAKE_BINARY_DIR}/html/broccoli-api ) || true
+    COMMAND "${CMAKE_COMMAND}" -E create_symlink
-                  WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
+            ${SPHINX_OUTPUT_DIR}/html
-                  COMMENT "[Sphinx] Generating HTML policy script docs"
+            ${CMAKE_BINARY_DIR}/html
-                  # SOURCES just adds stuff to IDE projects as a convenience
+    # Copy Broccoli API reference into output dir if it exists.
-                  SOURCES ${DOC_SOURCES})
+    COMMAND test -d ${BROCCOLI_DOCS_SRC} &&
+            ( rm -rf ${BROCCOLI_DOCS_DST} &&
+              cp -r ${BROCCOLI_DOCS_SRC} ${BROCCOLI_DOCS_DST} ) || true
+    WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
+    COMMENT "[Sphinx] Generate HTML documentation in ${CMAKE_BINARY_DIR}/html")
 
-# The "broxygenclean" target removes just the Sphinx input/output directories
+add_dependencies(sphinxdoc bro)
-# from the build directory.
-add_custom_target(broxygenclean
-                  COMMAND "${CMAKE_COMMAND}" -E remove_directory
-                          ${DOC_SOURCE_WORKDIR}
-                  COMMAND "${CMAKE_COMMAND}" -E remove_directory
-                          ${DOC_OUTPUT_DIR}
-                  VERBATIM)
 
-add_dependencies(broxygen restdoc)
+add_custom_target(sphinxdoc_clean
+    COMMAND "${CMAKE_COMMAND}" -E remove_directory ${SPHINX_INPUT_DIR}
+    COMMAND "${CMAKE_COMMAND}" -E remove_directory ${SPHINX_OUTPUT_DIR}
+    COMMAND "${CMAKE_COMMAND}" -E remove_directory ${BROXYGEN_SCRIPT_OUTPUT}
+    COMMAND "${CMAKE_COMMAND}" -E remove_directory ${BROXYGEN_CACHE_DIR}
+    VERBATIM)
 
 add_custom_target(doc)
 add_custom_target(docclean)
-add_dependencies(doc broxygen)
+add_dependencies(doc sphinxdoc)
-add_dependencies(docclean broxygenclean restclean)
+add_dependencies(docclean sphinxdoc_clean)

doc/README

@@ -15,17 +15,16 @@ which adds some reST directives and roles that aid in generating useful
 index entries and cross-references.  Other extensions can be added in
 a similar fashion.
 
-Either the ``make doc`` or ``make broxygen`` targets in the top-level
+The ``make doc`` target in the top-level Makefile can be used to locally
-Makefile can be used to locally render the reST files into HTML.
+render the reST files into HTML.  That target depend on:
-Those targets depend on:
 
 * Python interpreter >= 2.5
 * `Sphinx <http://sphinx.pocoo.org/>`_ >= 1.0.1
 
 After completion, HTML documentation is symlinked in ``build/html``.
 
-There's also ``make docclean`` and ``make broxygenclean`` targets to
+There's also a ``make docclean`` target which deletes any files
-clean the resulting documentation.
+created during the documentation build process.
 
 Notes for Writing Documentation
 -------------------------------

doc/bin/group_index_generator.py

@@ -1,72 +0,0 @@
-#! /usr/bin/env python
-
-# This script automatically generates a reST documents that lists
-# a collection of Bro scripts that are "grouped" together.
-# The summary text (##! comments) of the script is embedded in the list
-#
-# 1st argument is the file containing list of groups
-# 2nd argument is the directory containing ${group}_files lists of
-#   scripts that belong to the group and ${group}_doc_names lists of
-#   document names that can be supplied to a reST :doc: role
-# 3rd argument is a directory in which write a ${group}.rst file that contains
-#   reST style references to script docs along with summary text contained
-#   in original script.  If ${group} ends with "index", then the file
-#   is always clobbered by this script, but for other unique group names,
-#   this script will append to existing files.
-
-import sys
-import os
-import string
-
-group_list = sys.argv[1]
-file_manifest_dir = sys.argv[2]
-output_dir = sys.argv[3]
-
-def make_group_file_index(dir_name, group_name):
-    group_file = os.path.join(dir_name, group_name + ".rst")
-
-    if not os.path.exists(group_file):
-        if not os.path.exists(os.path.dirname(group_file)):
-            os.makedirs(os.path.dirname(group_file))
-
-    if group_name.endswith("index"):
-        with open(group_file, 'w') as f_group_file:
-            f_group_file.write(":orphan:\n\n")
-            title = "Package Index: %s\n" % os.path.dirname(group_name)
-            f_group_file.write(title);
-            for n in range(len(title)):
-                f_group_file.write("=")
-            f_group_file.write("\n");
-
-    return group_file
-
-with open(group_list, 'r') as f_group_list:
-    for group in f_group_list.read().splitlines():
-        #print group
-        group_file = make_group_file_index(output_dir, group)
-        file_manifest = os.path.join(file_manifest_dir, group + "_files")
-        doc_manifest = os.path.join(file_manifest_dir, group + "_doc_names")
-        src_files = []
-        doc_names = []
-
-        with open(file_manifest, 'r') as f_file_manifest:
-            src_files = f_file_manifest.read().splitlines()
-
-        with open(doc_manifest, 'r') as f_doc_manifest:
-            doc_names = f_doc_manifest.read().splitlines()
-
-        for i in range(len(src_files)):
-            src_file = src_files[i]
-            #print "\t" + src_file
-            summary_comments = []
-            with open(src_file, 'r') as f_src_file:
-                for line in f_src_file:
-                    sum_pos = string.find(line, "##!")
-                    if sum_pos != -1:
-                        summary_comments.append(line[(sum_pos+3):])
-            #print summary_comments
-
-            with open(group_file, 'a') as f_group_file:
-                f_group_file.write("\n:doc:`/scripts/%s`\n" % doc_names[i])
-                for line in summary_comments:
-                    f_group_file.write("   " + line)

doc/broxygen.conf.in

@@ -0,0 +1 @@
+script	@BROXYGEN_SCRIPT_OUTPUT@/	*

doc/conf.py.in

@@ -17,7 +17,7 @@ extensions = []
 # If extensions (or modules to document with autodoc) are in another directory,
 # add these directories to sys.path here. If the directory is relative to the
 # documentation root, use os.path.abspath to make it absolute, like shown here.
-sys.path.insert(0, os.path.abspath('sphinx-sources/ext'))
+sys.path.insert(0, os.path.abspath('sphinx_input/ext'))
 
 # ----- Begin of BTest configuration. -----
 btest = os.path.abspath("@CMAKE_SOURCE_DIR@/aux/btest")
@@ -33,6 +33,12 @@ btest_base="@CMAKE_SOURCE_DIR@/testing/btest"
 btest_tests="doc/sphinx"
 # ----- End of BTest configuration. -----
 
+# ----- Begin of Broxygen configuration. -----
+extensions += ["broxygen"]
+broxygen_cache="@BROXYGEN_CACHE_DIR@"
+os.environ["BROPATH"] = "@BROPATH@"
+os.environ["BROMAGIC"] = "@BROMAGIC@"
+# ----- End of Broxygen configuration. -----
 
 # -- General configuration -----------------------------------------------------
 
@@ -47,7 +53,7 @@ os.environ["BRO_SRC_ROOT"] = "@CMAKE_SOURCE_DIR@"
 os.environ["DOC_ROOT"] = "@CMAKE_SOURCE_DIR@/doc"
 
 # Add any paths that contain templates here, relative to this directory.
-templates_path = ['sphinx-sources/_templates', 'sphinx-sources/_static']
+templates_path = ['sphinx_input/_templates', 'sphinx_input/_static']
 
 # The suffix of source filenames.
 source_suffix = '.rst'
@@ -141,7 +147,7 @@ html_theme_options = { }
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,
 # so a file named "default.css" will overwrite the builtin "default.css".
-html_static_path = ['sphinx-sources/_static']
+html_static_path = ['sphinx_input/_static']
 
 # If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
 # using the given strftime format.

doc/ext/bro.py

@@ -191,6 +191,10 @@ class BroNotices(Index):
 
     def generate(self, docnames=None):
         content = {}
+
+        if 'notices' not in self.domain.env.domaindata['bro']:
+            return content, False
+        
         for n in self.domain.env.domaindata['bro']['notices']:
             modname = n[0].split("::")[0]
             entries = content.setdefault(modname, [])

doc/ext/broxygen.py

@@ -0,0 +1,311 @@
+"""
+Broxygen domain for Sphinx.
+
+Adds directives that allow Sphinx to invoke Bro in order to generate script
+reference documentation on the fly.  The directives are:
+
+broxygen:package
+    - Shows links to all scripts contained within matching package(s).
+broxygen:package_index
+    - An index with links to matching package document(s).
+broxygen:script
+    - Reference for matching script(s) (i.e. everything declared by the script).
+broxygen:script_summary
+    - Shows link to matching script(s) with it's summary-section comments.
+broxygen:script_index
+    - An index with links to all matching scrips.
+broxygen:proto_analyzer
+    - All protocol analyzers and their components (events/bifs, etc.)
+broxygen:file_analyzer
+    - All file analyzers and their components (events/bifs, etc.)
+"""
+
+
+from sphinx.domains import Domain, ObjType
+from sphinx.locale import l_
+from docutils.parsers.rst.directives.misc import Include
+
+
+App = None
+
+
+def info(msg):
+    """Use Sphinx builder to output a console message."""
+    global App
+    from sphinx.util.console import blue
+    App.builder.info(blue(msg))
+
+
+def pattern_to_filename_component(pattern):
+    """Replace certain characters in Broxygen config file target pattern.
+
+    Such that it can be used as part of a (sane) filename.
+
+    """
+    return pattern.replace("/", ".").replace("*", "star")
+
+
+def ensure_dir(path):
+    """Should act like ``mkdir -p``."""
+    import os
+    import errno
+
+    try:
+        os.makedirs(path)
+    except OSError as e:
+        if e.errno != errno.EEXIST:
+            raise
+
+
+def generate_config(env, type, pattern):
+    """Create a Broxygen config file for a particular target.
+
+    It can be used by Bro to generate reST docs for that target.
+
+    """
+    import os
+    import tempfile
+    from sphinx.errors import SphinxError
+
+    work_dir = env.config.broxygen_cache
+
+    if not work_dir:
+        raise SphinxError("broxygen_cache not set in sphinx config file")
+
+    ensure_dir(work_dir)
+    prefix = "{0}-{1}-".format(type, pattern_to_filename_component(pattern))
+    (fd, cfg) = tempfile.mkstemp(suffix=".cfg", prefix=prefix, dir=work_dir)
+    generated_file = "{0}.rst".format(cfg)
+    config = "{0}\t{1}\t{2}".format(type, generated_file, pattern)
+    f = os.fdopen(fd, "w")
+    f.write(config)
+    f.close()
+    return (cfg, generated_file)
+
+
+def generate_target(env, type, pattern):
+    """Create a Broxygen target and build it.
+
+    For a target which hasn't been referenced by any other script, this function
+    creates an associated config file then uses Bro w/ it to build the target
+    and stores the target information in the build environment.
+
+    If a script references a target that's already found in the build
+    environment the results of the previous built are re-used.
+
+    """
+    app_data = env.domaindata["broxygen"]
+
+    if (type, pattern) in app_data["targets"]:
+        info("Broxygen has cached doc for target '{0} {1}'".format(
+            type, pattern))
+        return app_data["targets"]
+
+    (cfg, gend_file) = generate_config(env, type, pattern)
+    target = BroxygenTarget(type, pattern, cfg, gend_file)
+    app_data["targets"][(type, pattern)] = target
+    build_target(env, target)
+    info("Broxygen built target '{0} {1}'".format(type, pattern))
+    return target
+
+
+def build_target(env, target):
+    """Invoke a Bro process to build a Broxygen target."""
+    import os
+    import subprocess
+
+    bro_cmd = "bro -X {0} broxygen".format(target.config_file)
+    cwd = os.getcwd()
+    os.chdir(os.path.dirname(target.config_file))
+
+    try:
+        subprocess.check_output(bro_cmd, stderr=subprocess.STDOUT, shell=True)
+    except subprocess.CalledProcessError as e:
+        from sphinx.errors import SphinxError
+        raise SphinxError(
+            "Command '{0}' returned non-zero exit status {1}: {2}".format(
+                e.cmd, e.returncode, e.output))
+    finally:
+        os.chdir(cwd)
+
+
+class BroxygenTarget(object):
+
+    """Some portion of reST documentation that Bro knows how to generate.
+
+    A target is identified by its type and pattern.  E.g. type "script" and
+    pattern "broxygen/example.bro".
+
+    """
+
+    def __init__(self, type, pattern, config_file, generated_file):
+        self.type = type
+        self.pattern = pattern
+        self.config_file = config_file
+        self.generated_file = generated_file
+        self.used_in_docs = set()
+
+
+class BroxygenDirective(Include):
+
+    """Base class for Broxygen directives.
+
+    It can use Bro to generate reST documentation on the fly and embed it in
+    the document at the location of the directive just like the ``.. include::``
+    directive.  The only argument is a pattern to identify to Bro which
+    pieces of documentation it needs to create.
+    """
+
+    required_arguments = 1
+    has_content = False
+
+    target_type = None
+
+    def run(self):
+        env = self.state.document.settings.env
+        info("Broxygen running .. {0}:: {1} in {2}".format(
+            self.name, self.arguments[0], env.docname))
+        target = generate_target(env, self.target_type, self.arguments[0])
+        target.used_in_docs.add(env.docname)
+        self.arguments = [target.generated_file]
+        return super(BroxygenDirective, self).run()
+
+
+class PackageDirective(BroxygenDirective):
+
+    target_type = "package"
+
+
+class PackageIndexDirective(BroxygenDirective):
+
+    target_type = "package_index"
+
+
+class ScriptDirective(BroxygenDirective):
+
+    target_type = "script"
+
+
+class ScriptSummaryDirective(BroxygenDirective):
+
+    target_type = "script_summary"
+
+
+class ScriptIndexDirective(BroxygenDirective):
+
+    target_type = "script_index"
+
+
+class ProtoAnalyzerDirective(BroxygenDirective):
+
+    target_type = "proto_analyzer"
+
+
+class FileAnalyzerDirective(BroxygenDirective):
+
+    target_type = "file_analyzer"
+
+
+class IdentifierDirective(BroxygenDirective):
+
+    target_type = "identifier"
+
+
+class BroxygenDomain(Domain):
+
+    name = "broxygen"
+    label = "Broxygen"
+
+    object_types = {
+        "package":          ObjType(l_("package")),
+        "package_index":    ObjType(l_("package_index")),
+        "script":           ObjType(l_("script")),
+        "script_summary":   ObjType(l_("script_summary")),
+        "script_index":     ObjType(l_("script_index")),
+        "proto_analyzer":   ObjType(l_("proto_analyzer")),
+        "file_analyzer":    ObjType(l_("file_analyzer")),
+        "identifier":       ObjType(l_("identifier")),
+    }
+
+    directives = {
+        "package":          PackageDirective,
+        "package_index":    PackageIndexDirective,
+        "script":           ScriptDirective,
+        "script_summary":   ScriptSummaryDirective,
+        "script_index":     ScriptIndexDirective,
+        "proto_analyzer":   ProtoAnalyzerDirective,
+        "file_analyzer":    FileAnalyzerDirective,
+        "identifier":       IdentifierDirective,
+    }
+
+    roles = {}
+
+    initial_data = {
+        "targets": {}
+    }
+
+    def clear_doc(self, docname):
+        """Update Broxygen targets referenced in docname.
+
+        If it's the last place the target was referenced, remove it from
+        the build environment and delete any generated config/reST files
+        associated with it from the cache.
+
+        """
+        import os
+
+        stale_targets = []
+
+        for (type, pattern), target in self.data["targets"].items():
+            if docname in target.used_in_docs:
+                target.used_in_docs.remove(docname)
+
+                if not target.used_in_docs:
+                    stale_targets.append(target)
+
+        for target in stale_targets:
+            del self.data["targets"][(target.type, target.pattern)]
+            os.remove(target.config_file)
+            os.remove(target.generated_file)
+
+    def get_objects(self):
+        """No Broxygen-generated content is itself linkable/searchable."""
+        return []
+
+
+def env_get_outdated_hook(app, env, added, changed, removed):
+    """Check whether to re-read any documents referencing Broxygen targets.
+
+    To do that we have to ask Bro to rebuild each target and compare the
+    before and after modification times of the generated reST output file.
+    If Bro changed it, then the document containing the Broxygen directive
+    needs to be re-read.
+
+    """
+    import os
+
+    reread = set()
+
+    for target in app.env.domaindata["broxygen"]["targets"].values():
+        before_mtime = os.stat(target.generated_file)
+        build_target(env, target)
+        after_mtime = os.stat(target.generated_file)
+
+        if after_mtime > before_mtime:
+            info("Broxygen target '{0} {1}' outdated".format(
+                target.type, target.pattern))
+
+            for docname in target.used_in_docs:
+                if docname not in removed:
+                    info("  in document: {0}".format(docname))
+                    reread.add(docname)
+
+    return list(reread)
+
+
+def setup(app):
+    global App
+    App = app
+    app.add_domain(BroxygenDomain)
+    app.add_config_value("broxygen_cache", None, "env")
+    app.connect("env-get-outdated", env_get_outdated_hook)

doc/frameworks/logging-dataseries.rst

@@ -104,7 +104,7 @@ code like this to your ``local.bro``:
         }
 
 Bro's DataSeries writer comes with a few tuning options, see
-:doc:`/scripts/base/frameworks/logging/writers/dataseries`.
+:doc:`/scripts/base/frameworks/logging/writers/dataseries.bro`.
 
 Working with DataSeries
 =======================

doc/frameworks/logging.rst

@@ -48,7 +48,7 @@ Basics
 The data fields that a stream records are defined by a record type
 specified when it is created. Let's look at the script generating Bro's
 connection summaries as an example,
-:doc:`/scripts/base/protocols/conn/main`. It defines a record
+:doc:`/scripts/base/protocols/conn/main.bro`. It defines a record
 :bro:type:`Conn::Info` that lists all the fields that go into
 ``conn.log``, each marked with a ``&log`` attribute indicating that it
 is part of the information written out. To write a log record, the
@@ -309,7 +309,7 @@ ASCII Writer Configuration
 --------------------------
 
 The ASCII writer has a number of options for customizing the format of
-its output, see :doc:`/scripts/base/frameworks/logging/writers/ascii`.
+its output, see :doc:`/scripts/base/frameworks/logging/writers/ascii.bro`.
 
 Adding Streams
 ==============
@@ -369,7 +369,7 @@ save the logged ``Foo::Info`` record into the connection record:
         }
 
 See the existing scripts for how to work with such a new connection
-field. A simple example is :doc:`/scripts/base/protocols/syslog/main`.
+field. A simple example is :doc:`/scripts/base/protocols/syslog/main.bro`.
 
 When you are developing scripts that add data to the :bro:type:`connection`
 record, care must be given to when and how long data is stored.

doc/frameworks/notice.rst

@@ -283,7 +283,7 @@ information to suppress duplicates for a configurable period of time.
 The ``$identifier`` field is typically comprised of several pieces of
 data related to the notice that when combined represent a unique
 instance of that notice. Here is an example of the script
-:doc:`/scripts/policy/protocols/ssl/validate-certs` raising a notice
+:doc:`/scripts/policy/protocols/ssl/validate-certs.bro` raising a notice
 for session negotiations where the certificate or certificate chain did
 not validate successfully against the available certificate authority
 certificates.

doc/frameworks/signatures.rst

@@ -46,7 +46,7 @@ signature's event statement (``Found root!``), and data is the last
 piece of payload which triggered the pattern match.
 
 To turn such :bro:id:`signature_match` events into actual alarms, you can
-load Bro's :doc:`/scripts/base/frameworks/signatures/main` script.
+load Bro's :doc:`/scripts/base/frameworks/signatures/main.bro` script.
 This script contains a default event handler that raises
 :bro:enum:`Signatures::Sensitive_Signature` :doc:`Notices <notice>`
 (as well as others; see the beginning of the script).

doc/index.rst

@@ -15,7 +15,7 @@ Bro Documentation
    scripting/index.rst
    frameworks/index.rst
    cluster/index.rst
-   scripts/index.rst
+   script-reference/index.rst
    components/index.rst
 
 * :ref:`General Index <genindex>`

doc/quickstart/index.rst

@@ -197,7 +197,7 @@ the variable's value may not change at run-time, but whose initial value can be
 modified via the ``redef`` operator at parse-time.
 
 So let's continue on our path to modify the behavior for the two SSL
-and SSH notices.  Looking at :doc:`/scripts/base/frameworks/notice/main`,
+and SSH notices.  Looking at :doc:`/scripts/base/frameworks/notice/main.bro`,
 we see that it advertises:
 
 .. code:: bro

doc/script-reference/file-analyzers.rst

@@ -0,0 +1 @@
+.. broxygen:file_analyzer:: *

doc/script-reference/index.rst

@@ -1,5 +1,3 @@
-.. This is a stub doc to which broxygen appends during the build process
-
 ================
 Script Reference
 ================
@@ -7,15 +5,9 @@ Script Reference
 .. toctree::
    :maxdepth: 1
 
-   packages
    proto-analyzers
    file-analyzers
-   notices
    builtins
-   Built-in Functions (BIFs) <base/bif/index>
+   packages
-   internal
    scripts
-
+   Broxygen Example Script </scripts/broxygen/example.bro>
-
-
-   

doc/script-reference/packages.rst

@@ -1,9 +1,7 @@
-.. This is a stub doc to which broxygen appends during the build process
-
 .. _script-packages:
 
-Bro Script Packages
+Bro Package Index
-===================
+=================
 
 Bro has the following script packages (e.g. collections of related scripts in
 a common directory).  If the package directory contains a ``__load__.bro``
@@ -12,3 +10,5 @@ script, it supports being loaded in mass as a whole directory for convenience.
 Packages/scripts in the ``base/`` directory are all loaded by default, while
 ones in ``policy/`` provide functionality and customization options that are
 more appropriate for users to decide whether they'd like to load it or not.
+
+.. broxygen:package_index:: *

doc/script-reference/proto-analyzers.rst

@@ -0,0 +1 @@
+.. broxygen:proto_analyzer:: *

doc/script-reference/scripts.rst

@@ -0,0 +1,5 @@
+================
+Bro Script Index
+================
+
+.. broxygen:script_index:: *

doc/scripting/index.rst

@@ -316,7 +316,7 @@ block that variable is available to any other script through the
 naming convention of ``MODULE::variable_name``.
 
 The declaration below is taken from the
-:doc:`/scripts/policy/protocols/conn/known-hosts` script and
+:doc:`/scripts/policy/protocols/conn/known-hosts.bro` script and
 declares a variable called ``known_hosts`` as a global set of unique
 IP addresses within the ``Known`` namespace and exports it for use
 outside of the ``Known`` namespace.  Were we to want to use the
@@ -348,7 +348,7 @@ constants are used in Bro scripts as containers for configuration
 options.  For example, the configuration option to log password
 decrypted from HTTP streams is stored in
 ``HTTP::default_capture_password`` as shown in the stripped down
-excerpt from :doc:`/scripts/base/protocols/http/main` below.
+excerpt from :doc:`/scripts/base/protocols/http/main.bro` below.
 
 .. btest-include:: ${BRO_SRC_ROOT}/scripts/base/protocols/http/main.bro
    :lines: 8-10,19-21,120
@@ -1182,7 +1182,7 @@ passing in the ``Notice::Info`` record.  The simplest kind of
 action based on the answer.  The hook below adds the
 :bro:enum:`Notice::ACTION_EMAIL` action for the
 ``SSH::Interesting_Hostname_Login`` notice raised in the
-:doc:`/scripts/policy/protocols/ssh/interesting-hostnames` script.
+:doc:`/scripts/policy/protocols/ssh/interesting-hostnames.bro` script.
 
 .. btest-include:: ${DOC_ROOT}/scripting/framework_notice_hook_01.bro
 
@@ -1224,7 +1224,7 @@ Bro.
 .. btest-include:: ${BRO_SRC_ROOT}/scripts/policy/protocols/ssl/expiring-certs.bro
    :lines: 60-63
 
-In the :doc:`/scripts/policy/protocols/ssl/expiring-certs` script
+In the :doc:`/scripts/policy/protocols/ssl/expiring-certs.bro` script
 which identifies when SSL certificates are set to expire and raises
 notices when it crosses a predefined threshold, the call to
 ``NOTICE`` above also sets the ``$identifier`` entry by concatenating

doc/scripts/CMakeLists.txt

@@ -1,226 +0,0 @@
-# find out what BROPATH to use when executing bro
-execute_process(COMMAND ${CMAKE_BINARY_DIR}/bro-path-dev
-                OUTPUT_VARIABLE BROPATH
-                RESULT_VARIABLE retval
-                OUTPUT_STRIP_TRAILING_WHITESPACE)
-if (NOT ${retval} EQUAL 0)
-    message(FATAL_ERROR "Problem setting BROPATH")
-endif ()
-
-# This macro is used to add a new makefile target for reST policy script
-# documentation that can be generated using Bro itself to parse policy scripts.
-# It's called like:
-#
-#     rest_target(srcDir broInput [group])
-#
-# srcDir: the directory which contains broInput
-# broInput: the file name of a bro policy script, any path prefix of this
-#     argument will be used to derive what path under scripts/ the generated
-#     documentation will be placed.
-# group: optional name of group that the script documentation will belong to.
-#     If this is not given, the group is automatically set to any path portion
-#     of the broInput argument.
-#
-# In addition to adding the makefile target, several CMake variables are set:
-#
-# MASTER_POLICY_INDEX_TEXT: a running list of policy scripts docs that have
-#   been generated so far, formatted such that it can be appended to a file
-#   that ends in a Sphinx toctree directive
-# ALL_REST_OUTPUTS: a running list (the CMake list type) of all reST docs
-#   that are to be generated
-# MASTER_GROUP_LIST: a running list (the CMake list type) of all script groups
-# MASTER_PKG_LIST: a running list (the CMake list type) of all script groups
-#   that were defived from the path portion of the broInput argument
-# ${group}_files: a running list of files belonging to a given group, from
-#   which summary text can be extracted at build time
-# ${group}_doc_names: a running list of reST style document names that can be
-#   given to a :doc: role, shared indices with ${group}_files
-
-macro(REST_TARGET srcDir broInput)
-    set(absSrcPath ${srcDir}/${broInput})
-    get_filename_component(basename ${broInput} NAME)
-    string(REPLACE .bro "" basename ${basename})
-    get_filename_component(extension ${broInput} EXT)
-    get_filename_component(relDstDir ${broInput} PATH)
-
-    set(sumTextSrc ${absSrcPath})
-    set(ogSourceFile ${absSrcPath})
-
-    if (NOT relDstDir)
-        set(docName "${basename}")
-        set(dstDir "${RST_OUTPUT_DIR}")
-    else ()
-        set(docName "${relDstDir}/${basename}")
-        set(dstDir "${RST_OUTPUT_DIR}/${relDstDir}")
-    endif ()
-
-    set(restFile "${docName}.rst")
-    string(REPLACE "/" "^" restFile ${restFile})
-    set(restOutput "${dstDir}/${basename}.rst")
-
-    set(MASTER_POLICY_INDEX_TEXT
-        "${MASTER_POLICY_INDEX_TEXT}\n   ${docName} <${docName}>")
-    list(APPEND ALL_REST_OUTPUTS ${restOutput})
-
-    if (NOT "${ARGN}" STREQUAL "")
-        set(group ${ARGN})
-    elseif (relDstDir)
-        set(group ${relDstDir}/index)
-        # add package index to master package list if not already in it
-        # and if a __load__.bro exists in the original script directory
-        list(FIND MASTER_PKG_LIST ${relDstDir} _found)
-        if (_found EQUAL -1)
-            if (EXISTS ${CMAKE_SOURCE_DIR}/scripts/${relDstDir}/__load__.bro)
-                list(APPEND MASTER_PKG_LIST ${relDstDir})
-            endif ()
-        endif ()
-    else ()
-        set(group "")
-    endif ()
-
-    if (NOT "${group}" STREQUAL "")
-        # add group to master group list if not already in it
-        list(FIND MASTER_GROUP_LIST ${group} _found)
-        if (_found EQUAL -1)
-            list(APPEND MASTER_GROUP_LIST ${group})
-            if (MASTER_GROUP_LIST_TEXT)
-               set(MASTER_GROUP_LIST_TEXT "${MASTER_GROUP_LIST_TEXT}\n${group}")
-            else ()
-               set(MASTER_GROUP_LIST_TEXT "${group}")
-            endif ()
-        endif ()
-
-        list(APPEND ${group}_files ${sumTextSrc})
-        list(APPEND ${group}_doc_names ${docName})
-    endif ()
-
-    add_custom_command(OUTPUT ${restOutput}
-        # delete any leftover state from previous bro runs
-        COMMAND "${CMAKE_COMMAND}"
-        ARGS -E remove_directory .state
-        # generate the reST documentation using bro
-        COMMAND BROPATH=${BROPATH}:${srcDir} BROMAGIC=${CMAKE_SOURCE_DIR}/magic/database ${CMAKE_BINARY_DIR}/src/bro
-        ARGS -b -Z ${broInput} || (rm -rf .state *.log *.rst && exit 1)
-        # move generated doc into a new directory tree that
-        # defines the final structure of documents
-        COMMAND "${CMAKE_COMMAND}"
-        ARGS -E make_directory ${dstDir}
-        COMMAND "${CMAKE_COMMAND}"
-        ARGS -E copy ${restFile} ${restOutput}
-        # copy the bro or bif script, too
-        COMMAND "${CMAKE_COMMAND}"
-        ARGS -E copy ${ogSourceFile} ${dstDir}
-        # clean up the build directory
-        COMMAND rm
-        ARGS -rf .state *.log *.rst
-        DEPENDS bro
-        DEPENDS ${absSrcPath}
-        WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
-        COMMENT "[Bro] Generating reST docs for ${broInput}"
-    )
-
-endmacro(REST_TARGET)
-
-# Schedule Bro scripts for which to generate documentation.
-include(DocSourcesList.cmake)
-
-# Macro for generating reST docs that are independent of any particular Bro
-# script.
-macro(INDEPENDENT_REST_TARGET reST_file)
-    add_custom_command(OUTPUT ${reST_file}
-        # delete any leftover state from previous bro runs
-        COMMAND "${CMAKE_COMMAND}"
-        ARGS -E remove_directory .state
-        # generate the reST documentation using bro
-        COMMAND BROPATH=${BROPATH}:${srcDir} BROMAGIC=${CMAKE_SOURCE_DIR}/magic/database ${CMAKE_BINARY_DIR}/src/bro
-        ARGS -b -Z base/init-bare.bro || (rm -rf .state *.log *.rst && exit 1)
-        # move generated doc into a new directory tree that
-        # defines the final structure of documents
-        COMMAND "${CMAKE_COMMAND}"
-        ARGS -E make_directory ${dstDir}
-        COMMAND "${CMAKE_COMMAND}"
-        ARGS -E copy ${reST_file} ${dstDir}
-        # clean up the build directory
-        COMMAND rm
-        ARGS -rf .state *.log *.rst
-        DEPENDS bro
-        WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
-        COMMENT "[Bro] Generating reST docs for ${reST_file}"
-    )
-    list(APPEND ALL_REST_OUTPUTS ${reST_file})
-endmacro(INDEPENDENT_REST_TARGET)
-
-independent_rest_target(proto-analyzers.rst)
-independent_rest_target(file-analyzers.rst)
-
-# create temporary list of all docs to include in the master policy/index file
-file(WRITE ${MASTER_POLICY_INDEX} "${MASTER_POLICY_INDEX_TEXT}")
-
-# create the temporary list of all packages to include in the master
-# policy/packages.rst file
-set(MASTER_PKG_INDEX_TEXT "")
-foreach (pkg ${MASTER_PKG_LIST})
-    set(MASTER_PKG_INDEX_TEXT
-        "${MASTER_PKG_INDEX_TEXT}\n:doc:`${pkg} <${pkg}/index>`\n")
-    if (EXISTS ${CMAKE_SOURCE_DIR}/scripts/${pkg}/README)
-        file(STRINGS ${CMAKE_SOURCE_DIR}/scripts/${pkg}/README pkgreadme)
-        foreach (line ${pkgreadme})
-            set(MASTER_PKG_INDEX_TEXT "${MASTER_PKG_INDEX_TEXT}\n    ${line}")
-        endforeach ()
-        set(MASTER_PKG_INDEX_TEXT "${MASTER_PKG_INDEX_TEXT}\n")
-    endif ()
-endforeach ()
-file(WRITE ${MASTER_PACKAGE_INDEX} "${MASTER_PKG_INDEX_TEXT}")
-
-# create temporary file containing list of all groups
-file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/group_list
-    "${MASTER_GROUP_LIST_TEXT}")
-
-# create temporary files containing list of each source file in a given group
-foreach (group ${MASTER_GROUP_LIST})
-    if (EXISTS ${CMAKE_CURRENT_BINARY_DIR}/${group}_files)
-        file(REMOVE ${CMAKE_CURRENT_BINARY_DIR}/${group}_files)
-    endif ()
-    if (EXISTS ${CMAKE_CURRENT_BINARY_DIR}/${group}_doc_names)
-        file(REMOVE ${CMAKE_CURRENT_BINARY_DIR}/${group}_doc_names)
-    endif ()
-    foreach (src ${${group}_files})
-        file(APPEND ${CMAKE_CURRENT_BINARY_DIR}/${group}_files "${src}\n")
-    endforeach ()
-    foreach (dname ${${group}_doc_names})
-        file(APPEND ${CMAKE_CURRENT_BINARY_DIR}/${group}_doc_names "${dname}\n")
-    endforeach ()
-endforeach ()
-
-# remove previously generated docs no longer scheduled for generation
-if (EXISTS ${RST_OUTPUT_DIR})
-    file(GLOB_RECURSE EXISTING_REST_DOCS "${RST_OUTPUT_DIR}/*.rst")
-    foreach (_doc ${EXISTING_REST_DOCS})
-        list(FIND ALL_REST_OUTPUTS ${_doc} _found)
-        if (_found EQUAL -1)
-            file(REMOVE ${_doc})
-            message(STATUS "Broxygen: remove stale reST doc: ${_doc}")
-            string(REPLACE .rst .bro _brofile ${_doc})
-            if (EXISTS ${_brofile})
-                file(REMOVE ${_brofile})
-                message(STATUS "Broxygen: remove stale bro source: ${_brofile}")
-            endif ()
-        endif ()
-    endforeach ()
-endif ()
-
-# The "restdoc" target uses Bro to parse policy scripts in order to
-# generate reST documentation from them.
-add_custom_target(restdoc
-                  # create symlink to the reST output directory for convenience
-                  COMMAND "${CMAKE_COMMAND}" -E create_symlink
-                          ${RST_OUTPUT_DIR}
-                          ${CMAKE_BINARY_DIR}/reST
-                  DEPENDS ${ALL_REST_OUTPUTS})
-
-# The "restclean" target removes all generated reST documentation from the
-# build directory.
-add_custom_target(restclean
-                  COMMAND "${CMAKE_COMMAND}" -E remove_directory
-                          ${RST_OUTPUT_DIR}
-                  VERBATIM)

doc/scripts/DocSourcesList.cmake

@@ -1,282 +0,0 @@
-# DO NOT EDIT
-# This file is auto-generated from the genDocSourcesList.sh script.
-#
-# This is a list of Bro script sources for which to generate reST documentation.
-# It will be included inline in the CMakeLists.txt found in the same directory
-# in order to create Makefile targets that define how to generate reST from
-# a given Bro script.
-#
-# Note: any path prefix of the script (2nd argument of rest_target macro)
-# will be used to derive what path under scripts/ the generated documentation
-# will be placed.
-
-set(psd ${PROJECT_SOURCE_DIR}/scripts)
-
-rest_target(${CMAKE_CURRENT_SOURCE_DIR} example.bro internal)
-rest_target(${psd} base/init-default.bro internal)
-rest_target(${psd} base/init-bare.bro internal)
-
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/analyzer.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/bloom-filter.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/bro.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/broxygen.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/cardinality-counter.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/const.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/event.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/file_analysis.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/input.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/logging.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_ARP.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_AYIYA.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_BackDoor.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_BitTorrent.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_ConnSize.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_DCE_RPC.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_DHCP.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_DNP3.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_DNS.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_FTP.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_FTP.functions.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_File.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_FileExtract.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_FileExtract.functions.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_FileHash.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_Finger.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_GTPv1.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_Gnutella.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_HTTP.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_HTTP.functions.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_ICMP.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_IRC.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_Ident.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_InterConn.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_Login.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_Login.functions.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_MIME.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_Modbus.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_NCP.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_NTP.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_NetBIOS.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_NetBIOS.functions.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_NetFlow.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_PIA.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_POP3.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_RPC.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_SMB.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_SMTP.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_SMTP.functions.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_SOCKS.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_SSH.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_SSL.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_SSL.functions.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_SteppingStone.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_Syslog.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_TCP.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_TCP.functions.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_Teredo.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_UDP.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_Unified2.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_Unified2.types.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_ZIP.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/reporter.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/strings.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/top-k.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/types.bif.bro)
-rest_target(${psd} base/files/extract/main.bro)
-rest_target(${psd} base/files/hash/main.bro)
-rest_target(${psd} base/files/unified2/main.bro)
-rest_target(${psd} base/frameworks/analyzer/main.bro)
-rest_target(${psd} base/frameworks/cluster/main.bro)
-rest_target(${psd} base/frameworks/cluster/nodes/manager.bro)
-rest_target(${psd} base/frameworks/cluster/nodes/proxy.bro)
-rest_target(${psd} base/frameworks/cluster/nodes/worker.bro)
-rest_target(${psd} base/frameworks/cluster/setup-connections.bro)
-rest_target(${psd} base/frameworks/communication/main.bro)
-rest_target(${psd} base/frameworks/control/main.bro)
-rest_target(${psd} base/frameworks/dpd/main.bro)
-rest_target(${psd} base/frameworks/files/main.bro)
-rest_target(${psd} base/frameworks/input/main.bro)
-rest_target(${psd} base/frameworks/input/readers/ascii.bro)
-rest_target(${psd} base/frameworks/input/readers/benchmark.bro)
-rest_target(${psd} base/frameworks/input/readers/binary.bro)
-rest_target(${psd} base/frameworks/input/readers/raw.bro)
-rest_target(${psd} base/frameworks/input/readers/sqlite.bro)
-rest_target(${psd} base/frameworks/intel/cluster.bro)
-rest_target(${psd} base/frameworks/intel/input.bro)
-rest_target(${psd} base/frameworks/intel/main.bro)
-rest_target(${psd} base/frameworks/logging/main.bro)
-rest_target(${psd} base/frameworks/logging/postprocessors/scp.bro)
-rest_target(${psd} base/frameworks/logging/postprocessors/sftp.bro)
-rest_target(${psd} base/frameworks/logging/writers/ascii.bro)
-rest_target(${psd} base/frameworks/logging/writers/dataseries.bro)
-rest_target(${psd} base/frameworks/logging/writers/elasticsearch.bro)
-rest_target(${psd} base/frameworks/logging/writers/none.bro)
-rest_target(${psd} base/frameworks/logging/writers/sqlite.bro)
-rest_target(${psd} base/frameworks/notice/actions/add-geodata.bro)
-rest_target(${psd} base/frameworks/notice/actions/drop.bro)
-rest_target(${psd} base/frameworks/notice/actions/email_admin.bro)
-rest_target(${psd} base/frameworks/notice/actions/page.bro)
-rest_target(${psd} base/frameworks/notice/actions/pp-alarms.bro)
-rest_target(${psd} base/frameworks/notice/cluster.bro)
-rest_target(${psd} base/frameworks/notice/extend-email/hostnames.bro)
-rest_target(${psd} base/frameworks/notice/main.bro)
-rest_target(${psd} base/frameworks/notice/non-cluster.bro)
-rest_target(${psd} base/frameworks/notice/weird.bro)
-rest_target(${psd} base/frameworks/packet-filter/cluster.bro)
-rest_target(${psd} base/frameworks/packet-filter/main.bro)
-rest_target(${psd} base/frameworks/packet-filter/netstats.bro)
-rest_target(${psd} base/frameworks/packet-filter/utils.bro)
-rest_target(${psd} base/frameworks/reporter/main.bro)
-rest_target(${psd} base/frameworks/signatures/main.bro)
-rest_target(${psd} base/frameworks/software/main.bro)
-rest_target(${psd} base/frameworks/sumstats/cluster.bro)
-rest_target(${psd} base/frameworks/sumstats/main.bro)
-rest_target(${psd} base/frameworks/sumstats/non-cluster.bro)
-rest_target(${psd} base/frameworks/sumstats/plugins/average.bro)
-rest_target(${psd} base/frameworks/sumstats/plugins/hll_unique.bro)
-rest_target(${psd} base/frameworks/sumstats/plugins/last.bro)
-rest_target(${psd} base/frameworks/sumstats/plugins/max.bro)
-rest_target(${psd} base/frameworks/sumstats/plugins/min.bro)
-rest_target(${psd} base/frameworks/sumstats/plugins/sample.bro)
-rest_target(${psd} base/frameworks/sumstats/plugins/std-dev.bro)
-rest_target(${psd} base/frameworks/sumstats/plugins/sum.bro)
-rest_target(${psd} base/frameworks/sumstats/plugins/topk.bro)
-rest_target(${psd} base/frameworks/sumstats/plugins/unique.bro)
-rest_target(${psd} base/frameworks/sumstats/plugins/variance.bro)
-rest_target(${psd} base/frameworks/tunnels/main.bro)
-rest_target(${psd} base/misc/find-checksum-offloading.bro)
-rest_target(${psd} base/protocols/conn/contents.bro)
-rest_target(${psd} base/protocols/conn/inactivity.bro)
-rest_target(${psd} base/protocols/conn/main.bro)
-rest_target(${psd} base/protocols/conn/polling.bro)
-rest_target(${psd} base/protocols/dhcp/consts.bro)
-rest_target(${psd} base/protocols/dhcp/main.bro)
-rest_target(${psd} base/protocols/dhcp/utils.bro)
-rest_target(${psd} base/protocols/dnp3/consts.bro)
-rest_target(${psd} base/protocols/dnp3/main.bro)
-rest_target(${psd} base/protocols/dns/consts.bro)
-rest_target(${psd} base/protocols/dns/main.bro)
-rest_target(${psd} base/protocols/ftp/files.bro)
-rest_target(${psd} base/protocols/ftp/gridftp.bro)
-rest_target(${psd} base/protocols/ftp/info.bro)
-rest_target(${psd} base/protocols/ftp/main.bro)
-rest_target(${psd} base/protocols/ftp/utils-commands.bro)
-rest_target(${psd} base/protocols/ftp/utils.bro)
-rest_target(${psd} base/protocols/http/entities.bro)
-rest_target(${psd} base/protocols/http/files.bro)
-rest_target(${psd} base/protocols/http/main.bro)
-rest_target(${psd} base/protocols/http/utils.bro)
-rest_target(${psd} base/protocols/irc/dcc-send.bro)
-rest_target(${psd} base/protocols/irc/files.bro)
-rest_target(${psd} base/protocols/irc/main.bro)
-rest_target(${psd} base/protocols/modbus/consts.bro)
-rest_target(${psd} base/protocols/modbus/main.bro)
-rest_target(${psd} base/protocols/smtp/entities.bro)
-rest_target(${psd} base/protocols/smtp/files.bro)
-rest_target(${psd} base/protocols/smtp/main.bro)
-rest_target(${psd} base/protocols/socks/consts.bro)
-rest_target(${psd} base/protocols/socks/main.bro)
-rest_target(${psd} base/protocols/ssh/main.bro)
-rest_target(${psd} base/protocols/ssl/consts.bro)
-rest_target(${psd} base/protocols/ssl/main.bro)
-rest_target(${psd} base/protocols/ssl/mozilla-ca-list.bro)
-rest_target(${psd} base/protocols/syslog/consts.bro)
-rest_target(${psd} base/protocols/syslog/main.bro)
-rest_target(${psd} base/utils/active-http.bro)
-rest_target(${psd} base/utils/addrs.bro)
-rest_target(${psd} base/utils/conn-ids.bro)
-rest_target(${psd} base/utils/dir.bro)
-rest_target(${psd} base/utils/directions-and-hosts.bro)
-rest_target(${psd} base/utils/exec.bro)
-rest_target(${psd} base/utils/files.bro)
-rest_target(${psd} base/utils/numbers.bro)
-rest_target(${psd} base/utils/paths.bro)
-rest_target(${psd} base/utils/patterns.bro)
-rest_target(${psd} base/utils/queue.bro)
-rest_target(${psd} base/utils/site.bro)
-rest_target(${psd} base/utils/strings.bro)
-rest_target(${psd} base/utils/thresholds.bro)
-rest_target(${psd} base/utils/time.bro)
-rest_target(${psd} base/utils/urls.bro)
-rest_target(${psd} policy/frameworks/communication/listen.bro)
-rest_target(${psd} policy/frameworks/control/controllee.bro)
-rest_target(${psd} policy/frameworks/control/controller.bro)
-rest_target(${psd} policy/frameworks/dpd/detect-protocols.bro)
-rest_target(${psd} policy/frameworks/dpd/packet-segment-logging.bro)
-rest_target(${psd} policy/frameworks/files/detect-MHR.bro)
-rest_target(${psd} policy/frameworks/files/hash-all-files.bro)
-rest_target(${psd} policy/frameworks/intel/do_notice.bro)
-rest_target(${psd} policy/frameworks/intel/seen/conn-established.bro)
-rest_target(${psd} policy/frameworks/intel/seen/dns.bro)
-rest_target(${psd} policy/frameworks/intel/seen/file-hashes.bro)
-rest_target(${psd} policy/frameworks/intel/seen/file-names.bro)
-rest_target(${psd} policy/frameworks/intel/seen/http-headers.bro)
-rest_target(${psd} policy/frameworks/intel/seen/http-url.bro)
-rest_target(${psd} policy/frameworks/intel/seen/smtp-url-extraction.bro)
-rest_target(${psd} policy/frameworks/intel/seen/smtp.bro)
-rest_target(${psd} policy/frameworks/intel/seen/ssl.bro)
-rest_target(${psd} policy/frameworks/intel/seen/where-locations.bro)
-rest_target(${psd} policy/frameworks/packet-filter/shunt.bro)
-rest_target(${psd} policy/frameworks/software/version-changes.bro)
-rest_target(${psd} policy/frameworks/software/vulnerable.bro)
-rest_target(${psd} policy/integration/barnyard2/main.bro)
-rest_target(${psd} policy/integration/barnyard2/types.bro)
-rest_target(${psd} policy/integration/collective-intel/main.bro)
-rest_target(${psd} policy/misc/app-stats/main.bro)
-rest_target(${psd} policy/misc/app-stats/plugins/facebook.bro)
-rest_target(${psd} policy/misc/app-stats/plugins/gmail.bro)
-rest_target(${psd} policy/misc/app-stats/plugins/google.bro)
-rest_target(${psd} policy/misc/app-stats/plugins/netflix.bro)
-rest_target(${psd} policy/misc/app-stats/plugins/pandora.bro)
-rest_target(${psd} policy/misc/app-stats/plugins/youtube.bro)
-rest_target(${psd} policy/misc/capture-loss.bro)
-rest_target(${psd} policy/misc/detect-traceroute/main.bro)
-rest_target(${psd} policy/misc/known-devices.bro)
-rest_target(${psd} policy/misc/load-balancing.bro)
-rest_target(${psd} policy/misc/loaded-scripts.bro)
-rest_target(${psd} policy/misc/profiling.bro)
-rest_target(${psd} policy/misc/scan.bro)
-rest_target(${psd} policy/misc/stats.bro)
-rest_target(${psd} policy/misc/trim-trace-file.bro)
-rest_target(${psd} policy/protocols/conn/known-hosts.bro)
-rest_target(${psd} policy/protocols/conn/known-services.bro)
-rest_target(${psd} policy/protocols/conn/weirds.bro)
-rest_target(${psd} policy/protocols/dhcp/known-devices-and-hostnames.bro)
-rest_target(${psd} policy/protocols/dns/auth-addl.bro)
-rest_target(${psd} policy/protocols/dns/detect-external-names.bro)
-rest_target(${psd} policy/protocols/ftp/detect-bruteforcing.bro)
-rest_target(${psd} policy/protocols/ftp/detect.bro)
-rest_target(${psd} policy/protocols/ftp/software.bro)
-rest_target(${psd} policy/protocols/http/detect-sqli.bro)
-rest_target(${psd} policy/protocols/http/detect-webapps.bro)
-rest_target(${psd} policy/protocols/http/header-names.bro)
-rest_target(${psd} policy/protocols/http/software-browser-plugins.bro)
-rest_target(${psd} policy/protocols/http/software.bro)
-rest_target(${psd} policy/protocols/http/var-extraction-cookies.bro)
-rest_target(${psd} policy/protocols/http/var-extraction-uri.bro)
-rest_target(${psd} policy/protocols/modbus/known-masters-slaves.bro)
-rest_target(${psd} policy/protocols/modbus/track-memmap.bro)
-rest_target(${psd} policy/protocols/smtp/blocklists.bro)
-rest_target(${psd} policy/protocols/smtp/detect-suspicious-orig.bro)
-rest_target(${psd} policy/protocols/smtp/entities-excerpt.bro)
-rest_target(${psd} policy/protocols/smtp/software.bro)
-rest_target(${psd} policy/protocols/ssh/detect-bruteforcing.bro)
-rest_target(${psd} policy/protocols/ssh/geo-data.bro)
-rest_target(${psd} policy/protocols/ssh/interesting-hostnames.bro)
-rest_target(${psd} policy/protocols/ssh/software.bro)
-rest_target(${psd} policy/protocols/ssl/cert-hash.bro)
-rest_target(${psd} policy/protocols/ssl/expiring-certs.bro)
-rest_target(${psd} policy/protocols/ssl/extract-certs-pem.bro)
-rest_target(${psd} policy/protocols/ssl/known-certs.bro)
-rest_target(${psd} policy/protocols/ssl/notary.bro)
-rest_target(${psd} policy/protocols/ssl/validate-certs.bro)
-rest_target(${psd} policy/tuning/defaults/extracted_file_limits.bro)
-rest_target(${psd} policy/tuning/defaults/packet-fragments.bro)
-rest_target(${psd} policy/tuning/defaults/warnings.bro)
-rest_target(${psd} policy/tuning/logs-to-elasticsearch.bro)
-rest_target(${psd} policy/tuning/track-all-assets.bro)
-rest_target(${psd} site/local-manager.bro)
-rest_target(${psd} site/local-proxy.bro)
-rest_target(${psd} site/local-worker.bro)
-rest_target(${psd} site/local.bro)
-rest_target(${psd} test-all-policy.bro)

doc/scripts/README

@@ -1,44 +0,0 @@
-This directory contains scripts and templates that can be used to automate
-the generation of Bro script documentation.  Several build targets are defined
-by CMake and available in the top-level Makefile:
-
-``restdoc``
-
-    This target uses Bro to parse policy scripts in order to generate
-    reStructuredText (reST) documentation from them.  The list of scripts
-    for which to generate reST documentation is defined in the
-    ``CMakeLists.txt`` file in this directory.  Script documentation is
-    rebuild automatically if the policy script from which it is derived
-    or the Bro binary becomes out of date
-
-    The resulting output from this target can be found in the CMake
-    ``build/`` directory inside ``reST`` (a symlink to
-    ``doc/scripts/rest_output``).
-
-``restclean``
-
-    This target removes any reST documentation that has been generated so far.
-
-The ``genDocSourcesList.sh`` script can be run to automatically generate
-``DocSourcesList.cmake``, which is the file CMake uses to define the list
-of documentation targets.  This script should be run after adding new
-Bro script source files, and the changes commited to git.
-
-If a script shouldn't have documentation generated for it, there's also a
-blacklist manifest that can be maintained in the ``genDocSourcesList.sh``
-script.
-
-The blacklist can also be used if you want to define a certain grouping for
-the script's generated docs to belong to (as opposed to the automatic grouping
-the happens for script packages/directories).  To do that, add the
-script's name to the blacklist, then append a ``rest_target()`` to the
-``statictext`` variable where the first argument is the source directory
-containing the policy script to document, the second argument is the file
-name of the policy script, and the third argument is the path/name of a
-pre-created reST document in the ``../`` source directory to which the
-``make doc`` process can append script documentation references.  This
-pre-created reST document should also then be linked to from the TOC tree
-in ``../index.rst``.
-
-See ``example.bro`` for an example of how to document a Bro script such that
-``make doc`` will be able to produce reST/HTML documentation for it.

doc/scripts/example.rst

@@ -1,291 +0,0 @@
-.. Automatically generated.  Do not edit.
-
-example.bro
-===========
-
-:download:`Original Source File <example.bro>`
-
-Overview
---------
-This is an example script that demonstrates how to document.  Comments
-of the form ``##!`` are for the script summary.  The contents of
-these comments are transferred directly into the auto-generated
-`reStructuredText <http://docutils.sourceforge.net/rst.html>`_
-(reST) document's summary section.
-
-.. tip:: You can embed directives and roles within ``##``-stylized comments.
-
-:Imports: :doc:`policy/frameworks/software/vulnerable </scripts/policy/frameworks/software/vulnerable>`
-
-Summary
-~~~~~~~
-Options
-#######
-============================================================================ ======================================
-:bro:id:`Example::an_option`: :bro:type:`set` :bro:attr:`&redef`             add documentation for "an_option" here
-
-:bro:id:`Example::option_with_init`: :bro:type:`interval` :bro:attr:`&redef`
-============================================================================ ======================================
-
-State Variables
-###############
-=========================================================================== =======================================
-:bro:id:`Example::a_var`: :bro:type:`bool`                                  put some documentation for "a_var" here
-
-:bro:id:`Example::var_with_attr`: :bro:type:`count` :bro:attr:`&persistent`
-
-:bro:id:`Example::var_without_explicit_type`: :bro:type:`string`
-=========================================================================== =======================================
-
-Types
-#####
-====================================================== ==========================================================
-:bro:type:`Example::SimpleEnum`: :bro:type:`enum`      documentation for "SimpleEnum"
-                                                       goes here.
-
-:bro:type:`Example::SimpleRecord`: :bro:type:`record`  general documentation for a type "SimpleRecord"
-                                                       goes here.
-
-:bro:type:`Example::ComplexRecord`: :bro:type:`record` general documentation for a type "ComplexRecord" goes here
-
-:bro:type:`Example::Info`: :bro:type:`record`          An example record to be used with a logging stream.
-====================================================== ==========================================================
-
-Events
-######
-================================================= =============================================================
-:bro:id:`Example::an_event`: :bro:type:`event`    Summarize "an_event" here.
-
-:bro:id:`Example::log_example`: :bro:type:`event` This is a declaration of an example event that can be used in
-                                                  logging streams and is raised once for each log entry.
-================================================= =============================================================
-
-Functions
-#########
-=============================================== =======================================
-:bro:id:`Example::a_function`: :bro:type:`func` Summarize purpose of "a_function" here.
-=============================================== =======================================
-
-Redefinitions
-#############
-===================================================== ========================================
-:bro:type:`Log::ID`: :bro:type:`enum`
-
-:bro:type:`Example::SimpleEnum`: :bro:type:`enum`     document the "SimpleEnum" redef here
-
-:bro:type:`Example::SimpleRecord`: :bro:type:`record` document the record extension redef here
-===================================================== ========================================
-
-Namespaces
-~~~~~~~~~~
-.. bro:namespace:: Example
-
-Notices
-~~~~~~~
-:bro:type:`Notice::Type`
-
-   :Type: :bro:type:`enum`
-
-      .. bro:enum:: Example::Notice_One Notice::Type
-
-         any number of this type of comment
-         will document "Notice_One"
-
-      .. bro:enum:: Example::Notice_Two Notice::Type
-
-         any number of this type of comment
-         will document "Notice_Two"
-
-      .. bro:enum:: Example::Notice_Three Notice::Type
-
-      .. bro:enum:: Example::Notice_Four Notice::Type
-
-Public Interface
-----------------
-Options
-~~~~~~~
-.. bro:id:: Example::an_option
-
-   :Type: :bro:type:`set` [:bro:type:`addr`, :bro:type:`addr`, :bro:type:`string`]
-   :Attributes: :bro:attr:`&redef`
-   :Default: ``{}``
-
-   add documentation for "an_option" here
-
-.. bro:id:: Example::option_with_init
-
-   :Type: :bro:type:`interval`
-   :Attributes: :bro:attr:`&redef`
-   :Default: ``10.0 msecs``
-
-State Variables
-~~~~~~~~~~~~~~~
-.. bro:id:: Example::a_var
-
-   :Type: :bro:type:`bool`
-
-   put some documentation for "a_var" here
-
-.. bro:id:: Example::var_with_attr
-
-   :Type: :bro:type:`count`
-   :Attributes: :bro:attr:`&persistent`
-
-.. bro:id:: Example::var_without_explicit_type
-
-   :Type: :bro:type:`string`
-   :Default: ``"this works"``
-
-Types
-~~~~~
-.. bro:type:: Example::SimpleEnum
-
-   :Type: :bro:type:`enum`
-
-      .. bro:enum:: Example::ONE Example::SimpleEnum
-
-         and more specific info for "ONE"
-         can span multiple lines
-
-      .. bro:enum:: Example::TWO Example::SimpleEnum
-
-         or more info like this for "TWO"
-         can span multiple lines
-
-      .. bro:enum:: Example::THREE Example::SimpleEnum
-
-   documentation for "SimpleEnum"
-   goes here.
-
-.. bro:type:: Example::SimpleRecord
-
-   :Type: :bro:type:`record`
-
-      field1: :bro:type:`count`
-         counts something
-
-      field2: :bro:type:`bool`
-         toggles something
-
-   general documentation for a type "SimpleRecord"
-   goes here.
-
-.. bro:type:: Example::ComplexRecord
-
-   :Type: :bro:type:`record`
-
-      field1: :bro:type:`count`
-         counts something
-
-      field2: :bro:type:`bool`
-         toggles something
-
-      field3: :bro:type:`Example::SimpleRecord`
-
-      msg: :bro:type:`string` :bro:attr:`&default` = ``"blah"`` :bro:attr:`&optional`
-         attributes are self-documenting
-
-   general documentation for a type "ComplexRecord" goes here
-
-.. bro:type:: Example::Info
-
-   :Type: :bro:type:`record`
-
-      ts: :bro:type:`time` :bro:attr:`&log`
-
-      uid: :bro:type:`string` :bro:attr:`&log`
-
-      status: :bro:type:`count` :bro:attr:`&log` :bro:attr:`&optional`
-
-   An example record to be used with a logging stream.
-
-Events
-~~~~~~
-.. bro:id:: Example::an_event
-
-   :Type: :bro:type:`event` (name: :bro:type:`string`)
-
-   Summarize "an_event" here.
-   Give more details about "an_event" here.
-   
-   :param name: describe the argument here
-
-.. bro:id:: Example::log_example
-
-   :Type: :bro:type:`event` (rec: :bro:type:`Example::Info`)
-
-   This is a declaration of an example event that can be used in
-   logging streams and is raised once for each log entry.
-
-Functions
-~~~~~~~~~
-.. bro:id:: Example::a_function
-
-   :Type: :bro:type:`function` (tag: :bro:type:`string`, msg: :bro:type:`string`) : :bro:type:`string`
-
-   Summarize purpose of "a_function" here.
-   Give more details about "a_function" here.
-   Separating the documentation of the params/return values with
-   empty comments is optional, but improves readability of script.
-   
-   
-   :param tag: function arguments can be described
-        like this
-   
-   :param msg: another param
-   
-   
-   :returns: describe the return type here
-
-Redefinitions
-~~~~~~~~~~~~~
-:bro:type:`Log::ID`
-
-   :Type: :bro:type:`enum`
-
-      .. bro:enum:: Example::LOG Log::ID
-
-:bro:type:`Example::SimpleEnum`
-
-   :Type: :bro:type:`enum`
-
-      .. bro:enum:: Example::FOUR Example::SimpleEnum
-
-         and some documentation for "FOUR"
-
-      .. bro:enum:: Example::FIVE Example::SimpleEnum
-
-         also "FIVE" for good measure
-
-   document the "SimpleEnum" redef here
-
-:bro:type:`Example::SimpleRecord`
-
-   :Type: :bro:type:`record`
-
-      field_ext: :bro:type:`string` :bro:attr:`&optional`
-         document the extending field here
-         (or here)
-
-   document the record extension redef here
-
-Port Analysis
--------------
-:ref:`More Information <common_port_analysis_doc>`
-
-SSL::
-
-    [ports={
-        443/tcp,
-        562/tcp
-    }]
-
-Packet Filter
--------------
-:ref:`More Information <common_packet_filter_doc>`
-
-Filters added::
-
-    [ssl] = tcp port 443,
-    [nntps] = tcp port 562
-

doc/scripts/genDocSourcesList.sh

@@ -1,86 +0,0 @@
-#!/usr/bin/env bash
-
-# ./genDocSourcesList.sh [output file]
-#
-# Run this script to a generate file that's used to tell CMake about all the
-# possible scripts for which reST documentation can be created.
-#
-# The optional argument can be used to avoid overwriting the file CMake uses
-# by default.
-#
-# Specific scripts can be blacklisted below when e.g. they currently aren't
-# parseable or they just aren't meant to be documented.
-
-export LC_ALL=C # Make sorting stable.
-
-blacklist ()
-    {
-    if [[ "$blacklist" == "" ]]; then
-        blacklist="$1"
-    else
-        blacklist="$blacklist|$1"
-    fi
-    }
-
-# files passed into this function are meant to be temporary workarounds
-# because they're not finished or otherwise can't be loaded for some reason
-tmp_blacklist ()
-    {
-    echo "Warning: temporarily blacklisted files named '$1'" 1>&2
-    blacklist $1
-    }
-
-blacklist __load__.bro
-blacklist test-all.bro
-blacklist all.bro
-blacklist init-default.bro
-blacklist init-bare.bro
-
-statictext="\
-# DO NOT EDIT
-# This file is auto-generated from the "genDocSourcesList.sh" script.
-#
-# This is a list of Bro script sources for which to generate reST documentation.
-# It will be included inline in the CMakeLists.txt found in the same directory
-# in order to create Makefile targets that define how to generate reST from
-# a given Bro script.
-#
-# Note: any path prefix of the script (2nd argument of rest_target macro)
-# will be used to derive what path under scripts/ the generated documentation
-# will be placed.
-
-set(psd \${PROJECT_SOURCE_DIR}/scripts)
-
-rest_target(\${CMAKE_CURRENT_SOURCE_DIR} example.bro internal)
-rest_target(\${psd} base/init-default.bro internal)
-rest_target(\${psd} base/init-bare.bro internal)
-"
-
-if [[ $# -ge 1 ]]; then
-    outfile=$1
-else
-    outfile=DocSourcesList.cmake
-fi
-
-thisdir="$( cd "$( dirname "$0" )" && pwd )"
-sourcedir=${thisdir}/../..
-
-echo "$statictext" > $outfile
-
-bifs=`( cd ${sourcedir}/build/scripts/base && find . -name \*\.bif.bro | sort )`
-
-for file in $bifs
-do
-    f=${file:2}
-    echo "rest_target(\${CMAKE_BINARY_DIR}/scripts base/$f)" >> $outfile
-done
-
-scriptfiles=`( cd ${sourcedir}/scripts && find . -name \*\.bro | sort )`
-
-for file in $scriptfiles
-do
-    f=${file:2}
-    if [[ ! $f =~ $blacklist ]]; then
-        echo "rest_target(\${psd} $f)" >> $outfile
-    fi
-done

doc/scripts/internal.rst

@@ -1,5 +0,0 @@
-.. This is a stub doc to which broxygen appends during the build process
-
-Internal Scripts
-================
-

doc/scripts/scripts.rst

@@ -1,8 +0,0 @@
-.. This is a stub doc to which broxygen appends during the build process
-
-===============
-All Bro Scripts
-===============
-
-.. toctree::
-   :maxdepth: 1

scripts/base/frameworks/reporter/main.bro

@@ -9,7 +9,7 @@
 ##! Note that this framework deals with the handling of internally generated
 ##! reporter messages, for the interface
 ##! into actually creating reporter messages from the scripting layer, use
-##! the built-in functions in :doc:`/scripts/base/bif/reporter.bif`.
+##! the built-in functions in :doc:`/scripts/base/bif/reporter.bif.bro`.
 
 module Reporter;
 

scripts/base/init-bare.bro

@@ -399,7 +399,7 @@ type NetStats: record {
 	pkts_dropped: count &default=0;	##< Packets reported dropped by the system.
 	## Packets seen on the link. Note that this may differ
 	## from *pkts_recvd* because of a potential capture_filter. See
-	## :doc:`/scripts/base/frameworks/packet-filter/main`. Depending on the
+	## :doc:`/scripts/base/frameworks/packet-filter/main.bro`. Depending on the
 	## packet capture system, this value may not be available and will then
 	## be always set to zero.
 	pkts_link:    count &default=0;
@@ -2768,13 +2768,13 @@ const log_max_size = 0.0 &redef;
 const log_encryption_key = "<undefined>" &redef;
 
 ## Write profiling info into this file in regular intervals. The easiest way to
-## activate profiling is loading :doc:`/scripts/policy/misc/profiling`.
+## activate profiling is loading :doc:`/scripts/policy/misc/profiling.bro`.
 ##
 ## .. bro:see:: profiling_interval expensive_profiling_multiple segment_profiling
 global profiling_file: file &redef;
 
 ## Update interval for profiling (0 disables).  The easiest way to activate
-## profiling is loading  :doc:`/scripts/policy/misc/profiling`.
+## profiling is loading  :doc:`/scripts/policy/misc/profiling.bro`.
 ##
 ## .. bro:see:: profiling_file expensive_profiling_multiple segment_profiling
 const profiling_interval = 0 secs &redef;

scripts/base/protocols/dhcp/main.bro

@@ -4,7 +4,7 @@
 ##!
 ##! If you'd like to track known DHCP devices and to log the hostname
 ##! supplied by the client, see
-##! :doc:`/scripts/policy/protocols/dhcp/known-devices-and-hostnames`.
+##! :doc:`/scripts/policy/protocols/dhcp/known-devices-and-hostnames.bro`.
 
 @load ./utils.bro
 

scripts/base/protocols/http/__load__.bro

@@ -3,4 +3,4 @@
 @load ./utils
 @load ./files
 
-@load-sigs ./dpd.sig
+@load-sigs ./dpd.sig

scripts/broxygen/README

@@ -0,0 +1,4 @@
+This package is loaded during the process which automatically generates
+reference documentation for all Bro scripts (i.e. "Broxygen").  Its only
+purpose is to provide an easy way to load all known Bro scripts plus any
+extra scripts needed or used by the documentation process.

scripts/broxygen/__load__.bro

@@ -0,0 +1,2 @@
+@load test-all-policy.bro
+@load ./example.bro

scripts/policy/misc/known-devices.bro

@@ -7,7 +7,7 @@
 ##!
 ##!     This script will not generate any logs on its own, it needs to be
 ##!     supplied with information from elsewhere, such as
-##!     :doc:`/scripts/policy/protocols/dhcp/known-devices-and-hostnames`.
+##!     :doc:`/scripts/policy/protocols/dhcp/known-devices-and-hostnames.bro`.
 
 module Known;
 

scripts/policy/misc/stats.bro

@@ -1,5 +1,5 @@
 ##! Log memory/packet/lag statistics.  Differs from
-##! :doc:`/scripts/policy/misc/profiling` in that this
+##! :doc:`/scripts/policy/misc/profiling.bro` in that this
 ##! is lighter-weight (much less info, and less load to generate).
 
 @load base/frameworks/notice

src/BroDoc.cc

@@ -611,7 +611,7 @@ void CreateProtoAnalyzerDoc(FILE* f)
 	fprintf(f, "Protocol Analyzers\n");
 	fprintf(f, "==================\n\n");
 	fprintf(f, ".. contents::\n");
-	fprintf(f, "     :depth: 1\n\n");
+	fprintf(f, "     :depth: 2\n\n");
 
 	WriteAnalyzerTagDefn(f, "Analyzer");
 
@@ -641,7 +641,7 @@ void CreateFileAnalyzerDoc(FILE* f)
 	fprintf(f, "File Analyzers\n");
 	fprintf(f, "==============\n\n");
 	fprintf(f, ".. contents::\n");
-	fprintf(f, "     :depth: 1\n\n");
+	fprintf(f, "     :depth: 2\n\n");
 
 	WriteAnalyzerTagDefn(f, "Files");
 

src/Type.cc

@@ -1165,9 +1165,9 @@ void RecordType::DescribeFieldsReST(ODesc* d, bool func_args) const
 		     field_from_script != type_from_script )
 			{
 			d->PushIndent();
-			d->Add(fmt("(from ``redef`` in :doc:`%s`)",
+			d->Add(fmt("(from ``redef`` in :doc:`/scripts/%s`)",
 			           field_from_script.c_str()));
-			d->PopIndentNoNL();
+			d->PopIndent();
 			}
 
 		vector<string> cmnts = doc->GetFieldComments(td->id);
@@ -1513,9 +1513,9 @@ void EnumType::DescribeReST(ODesc* d, bool roles_only) const
 			{
 			d->NL();
 			d->PushIndent();
-			d->Add(fmt("(from ``redef`` in :doc:`%s`)",
+			d->Add(fmt("(from ``redef`` in :doc:`/scripts/%s`)",
 			           enum_from_script.c_str()));
-			d->PopIndentNoNL();
+			d->PopIndent();
 			}
 
 		vector<string> cmnts = doc->GetComments();

src/broxygen/Configuration.cc

@@ -9,7 +9,10 @@
 #include <algorithm>
 #include <map>
 #include <cstdio>
+#include <sys/types.h>
 #include <sys/stat.h>
+#include <fts.h>
+#include <unistd.h>
 
 using namespace broxygen;
 using namespace std;
@@ -160,24 +163,38 @@ void PackageTarget::DoFindDependencies(const vector<Document*>& docs)
 
 void PackageTarget::DoGenerate() const
 	{
-	if ( broxygen_mgr->IsUpToDate(Name(), script_deps) )
+	if ( broxygen_mgr->IsUpToDate(Name(), script_deps) &&
+	     broxygen_mgr->IsUpToDate(Name(), pkg_deps) )
 		return;
 
 	TargetFile file(Name());
 
+	fprintf(file.f, ":orphan:\n\n");
+
 	for ( manifest_t::const_iterator it = pkg_manifest.begin();
 	      it != pkg_manifest.end(); ++it )
 		{
-		fprintf(file.f, "Package: %s\n\n", it->first->Name().c_str());
+		string header = fmt("Package: %s", it->first->Name().c_str());
+		header += "\n" + string(header.size(), '=');
+
+		fprintf(file.f, "%s\n\n", header.c_str());
+
+		vector<string> readme = it->first->GetReadme();
+
+		for ( size_t i = 0; i < readme.size(); ++i )
+			fprintf(file.f, "%s\n", readme[i].c_str());
+
+		fprintf(file.f, "\n");
 
 		for ( size_t i = 0; i < it->second.size(); ++i )
 			{
-			fprintf(file.f, "   :doc:`%s`\n", it->second[i]->Name().c_str());
+			fprintf(file.f, ":doc:`/scripts/%s`\n",
+			        it->second[i]->Name().c_str());
 
 			vector<string> cmnts = it->second[i]->GetComments();
 
 			for ( size_t j = 0; j < cmnts.size(); ++j )
-				fprintf(file.f, "      %s\n", cmnts[j].c_str());
+				fprintf(file.f, "   %s\n", cmnts[j].c_str());
 
 			fprintf(file.f, "\n");
 			}
@@ -211,25 +228,82 @@ void ScriptTarget::DoFindDependencies(const vector<Document*>& docs)
 	if ( script_deps.empty() )
 		reporter->FatalError("No match for Broxygen target '%s' pattern '%s'",
 		                     Name().c_str(), Pattern().c_str());
+
+	if ( ! IsDir() )
+		return;
+
+	for ( size_t i = 0; i < script_deps.size(); ++i )
+		{
+		if ( SafeBasename(script_deps[i]->Name()).result == PACKAGE_LOADER )
+			{
+			string pkg_dir = SafeDirname(script_deps[i]->Name()).result;
+			string target_file = Name() + pkg_dir + "/index.rst";
+			Target* t = PackageTarget::Instantiate(target_file, pkg_dir);
+			t->FindDependencies(docs);
+			pkg_deps.push_back(t);
+			}
+		}
+	}
+
+vector<string> dir_contents_recursive(string dir)
+	{
+	vector<string> rval;
+	struct stat st;
+
+	if ( stat(dir.c_str(), &st) < 0 && errno == ENOENT )
+		return rval;
+
+	while ( dir[dir.size() - 1] == '/' )
+		dir.erase(dir.size() - 1, 1);
+
+	char* dir_copy = copy_string(dir.c_str());
+	char** scan_path = new char*[2];
+	scan_path[0] = dir_copy;
+	scan_path[1] = 0;
+
+	FTS* fts = fts_open(scan_path, FTS_NOCHDIR, 0);
+
+	if ( ! fts )
+		{
+		reporter->Error("fts_open failure: %s", strerror(errno));
+		delete [] scan_path;
+		delete [] dir_copy;
+		return rval;
+		}
+
+	FTSENT* n;
+
+	while ( (n = fts_read(fts)) )
+		{
+		if ( n->fts_info & FTS_F )
+			rval.push_back(n->fts_path);
+		}
+
+	if ( errno )
+		reporter->Error("fts_read failure: %s", strerror(errno));
+
+	if ( fts_close(fts) < 0 )
+		reporter->Error("fts_close failure: %s", strerror(errno));
+
+	delete [] scan_path;
+	delete [] dir_copy;
+	return rval;
 	}
 
 void ScriptTarget::DoGenerate() const
     {
-	if ( Name()[Name().size() - 1] == '/' )
+	if ( IsDir() )
 		{
 		// Target name is a dir, matching scripts are written within that dir
 		// with a dir tree that parallels the script's BROPATH location.
 
+		set<string> targets;
+		vector<string> dir_contents = dir_contents_recursive(Name());
+
 		for ( size_t i = 0; i < script_deps.size(); ++i )
 			{
-			string target_filename = Name() + script_deps[i]->Name();
+			string target_filename = Name() + script_deps[i]->Name() + ".rst";
-			size_t pos = target_filename.rfind(".bro");
+			targets.insert(target_filename);
-
-			if ( pos == target_filename.size() - 4 )
-				target_filename.replace(pos, 4, ".rst");
-			else
-				target_filename += ".rst";
-
 			vector<ScriptDocument*> dep;
 			dep.push_back(script_deps[i]);
 
@@ -241,9 +315,26 @@ void ScriptTarget::DoGenerate() const
 			fprintf(file.f, "%s\n", script_deps[i]->ReStructuredText().c_str());
 			}
 
-		// TODO: could possibly take inventory of files in the dir beforehand,
+		for ( size_t i = 0; i < pkg_deps.size(); ++i )
-		// track all files written, then compare afterwards in order to remove
+			{
-		// stale files.
+			targets.insert(pkg_deps[i]->Name());
+			pkg_deps[i]->Generate();
+			}
+
+		for ( size_t i = 0; i < dir_contents.size(); ++i )
+			{
+			string f = dir_contents[i];
+
+			if ( targets.find(f) != targets.end() )
+				continue;
+
+			if ( unlink(f.c_str()) < 0 )
+				reporter->Warning("Failed to unlink %s: %s", f.c_str(),
+				                  strerror(errno));
+
+			DBG_LOG(DBG_BROXYGEN, "Delete stale script file %s", f.c_str());
+			}
+
 		return;
 		}
 
@@ -300,7 +391,8 @@ void ScriptIndexTarget::DoGenerate() const
 		if ( ! d )
 			continue;
 
-		fprintf(file.f, "   %s <%s>\n", d->Name().c_str(), d->Name().c_str());
+		fprintf(file.f, "   %s </scripts/%s>\n", d->Name().c_str(),
+		        d->Name().c_str());
 		}
 	}
 

src/broxygen/Configuration.h

@@ -146,6 +146,9 @@ public:
 	                           const std::string& pattern)
 		{ return new ScriptTarget(name, pattern); }
 
+	~ScriptTarget()
+		{ for ( size_t i = 0; i < pkg_deps.size(); ++i ) delete pkg_deps[i]; }
+
 protected:
 
 	ScriptTarget(const std::string& name, const std::string& pattern)
@@ -159,6 +162,11 @@ private:
 	void DoFindDependencies(const std::vector<Document*>& docs);
 
 	void DoGenerate() const;
+
+	bool IsDir() const
+		{ return Name()[Name().size() - 1] == '/'; }
+
+	std::vector<Target*> pkg_deps;
 };
 
 class ScriptSummaryTarget : public ScriptTarget {

src/broxygen/Document.cc

@@ -116,7 +116,7 @@ PackageDocument::PackageDocument(const string& arg_name)
 
 string PackageDocument::DoReStructuredText(bool roles_only) const
 	{
-	string rval = fmt(":doc:`%s <%s/index>`\n\n", pkg_name.c_str(),
+	string rval = fmt(":doc:`%s </scripts/%s/index>`\n\n", pkg_name.c_str(),
 	                  pkg_name.c_str());
 
 	for ( size_t i = 0; i < readme.size(); ++i )
@@ -629,15 +629,20 @@ string ScriptDocument::DoReStructuredText(bool roles_only) const
 			if ( it != dependencies.begin() )
 				rval += ", ";
 
-			rval += fmt(":doc:`%s </scripts/%s>`", it->c_str(), it->c_str());
+			string path = find_file(*it, bro_path(), "bro");
-			// TODO linking to packages is a bit different?
+			string doc = *it;
+
+			if ( ! path.empty() && is_dir(path.c_str()) )
+				// Reference the package.
+				doc += "/index";
+
+			rval += fmt(":doc:`%s </scripts/%s>`", it->c_str(), doc.c_str());
 			}
 
 		rval += "\n";
 		}
 
-	// TODO: make this an absolute path?
+	rval += fmt(":Source File: :download:`/scripts/%s`\n", name.c_str());
-	rval += fmt(":Source File: :download:`%s`\n", name.c_str());
 	rval += "\n";
 	rval += make_heading("Summary", '~');
 	rval += make_summary("Options", '#', '=', options);

src/const.bif

@@ -1,6 +1,6 @@
 ##! Declaration of various scripting-layer constants that the Bro core uses
 ##! internally.  Documentation and default values for the scripting-layer
-##! variables themselves are found in :doc:`/scripts/base/init-bare`.
+##! variables themselves are found in :doc:`/scripts/base/init-bare.bro`.
 
 const ignore_keep_alive_rexmit: bool;
 const skip_http_data: bool;

src/event.bif

@@ -360,7 +360,7 @@ event content_gap%(c: connection, is_orig: bool, seq: count, length: count%);
 ##
 ## .. note::
 ##
-##    Bro comes with a script :doc:`/scripts/policy/misc/capture-loss` that uses
+##    Bro comes with a script :doc:`/scripts/policy/misc/capture-loss.bro` that uses
 ##    this event to estimate packet loss and report when a predefined threshold
 ##    is exceeded.
 event gap_report%(dt: interval, info: gap_info%);

src/reporter.bif

@@ -4,7 +4,7 @@
 ##! If event handlers do exist, it's assumed they take care of determining
 ##! how/where to output the messages.
 ##!
-##! See :doc:`/scripts/base/frameworks/reporter/main` for a convenient
+##! See :doc:`/scripts/base/frameworks/reporter/main.bro` for a convenient
 ##! reporter message logging framework.
 
 module Reporter;

testing/btest/Makefile

@@ -21,12 +21,9 @@ cleanup:
 	@rm -f $(DIAG)
 	@rm -f .tmp/script-coverage*
 
-update-doc-sources:
-	../../doc/scripts/genDocSourcesList.sh ../../doc/scripts/DocSourcesList.cmake
-
 # Updates the three coverage tests that usually need tweaking when
 # scripts get added/removed.
-update-coverage-tests: update-doc-sources
+update-coverage-tests:
 	btest -qU coverage.bare-load-baseline
 	btest -qU coverage.default-load-baseline
 	@echo "Use 'git diff' to check updates look right."

testing/btest/coverage/doc.test

@@ -1,8 +0,0 @@
-# This tests that we're generating bro script documentation for all the
-# available bro scripts.  If this fails, then the genDocSources.sh needs
-# to be run to produce a new DocSourcesList.cmake or genDocSources.sh needs
-# to be updated to blacklist undesired scripts. To update, run 
-# "make update-doc-sources"
-#
-# @TEST-EXEC: $DIST/doc/scripts/genDocSourcesList.sh
-# @TEST-EXEC: cmp $DIST/doc/scripts/DocSourcesList.cmake ./DocSourcesList.cmake