Merge remote-tracking branch 'origin/topic/jsiwek/broxygen'

BIT-1098

* origin/topic/jsiwek/broxygen:
  Fix Broxygen-related compile errors.
  Add a Broxygen coverage test.
  Internal Broxygen organization/documentation/polish.
  Add unit tests for Broxygen config file targets.
  Change Broxygen config file format.
  Broxygen doc-related test updates.  Fix two regressions.
  A couple documentation fixes.
  Integrate new Broxygen functionality into Sphinx.
  Implement majority of Broxygen features delegated to Bro.
  Broxygen can now read a config file specifying particular targets.
  Remove unneeded Broxygen comments in scan.bro.
  Replace safe_basename/safe_dirname w/ SafeBasename/SafeDirname.
  Add BIF interface for retrieving comments/docs.
  Quick optimization to Broxygen doc gathering.
  Flesh out Broxygen doc-gathering skeleton.
  Refactor search_for_file() util function.
  Initial skeleton of new Broxygen infrastructure.

CHANGES

@@ -1,4 +1,28 @@
 
+2.2-35 | 2013-12-04 10:10:32 -0800
+
+  * Rework the automated script-reference documentation generation
+    process, broxygen. Addresses BIT-701 and BIT-751. (Jon Siwek)
+
+    Highlights:
+
+        - Remove --doc-scripts and -Z options to toggle documentation
+          mode. The parser is now always instrumented to gather
+          documentation from comments of the form "##", "##!", or
+          "##<".
+
+        - Raw comments are available at runtime through several BIF
+          functions: get_*_comments;
+
+        - Add --broxygen and -X options to toggle generating
+          reST-format documentation output, driven by a config file
+          argument.
+
+        - Add a "broxygen" Sphinx extension domain, allowing certain
+          pieces of documentation to be generated on-the-fly via
+          invoking a Bro process. Re-organized/cleaned up the Sphinx
+          source tree in doc/ to use this in some places.
+
 2.2-11 | 2013-12-03 10:56:28 -0800
 
   * Unit test for broccoli vector support. (Jon Siwek)

CMakeLists.txt

@@ -10,7 +10,6 @@ if (NOT BRO_SCRIPT_INSTALL_PATH)
     # set the default Bro script installation path (user did not specify one)
     set(BRO_SCRIPT_INSTALL_PATH ${BRO_ROOT_DIR}/share/bro)
 endif ()
-set(BRO_SCRIPT_SOURCE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/scripts)
 
 # sanitize the Bro script install directory into an absolute path
 # (CMake is confused by ~ as a representation of home directory)

Makefile

@@ -29,18 +29,6 @@ doc: configured
 docclean: configured
 	$(MAKE) -C $(BUILD) $@
 
-restdoc: configured
-	$(MAKE) -C $(BUILD) $@
-
-restclean: configured
-	$(MAKE) -C $(BUILD) $@
-
-broxygen: configured
-	$(MAKE) -C $(BUILD) $@
-
-broxygenclean: configured
-	$(MAKE) -C $(BUILD) $@
-
 dist:
 	@rm -rf $(VERSION_FULL) $(VERSION_FULL).tgz
 	@rm -rf $(VERSION_MIN) $(VERSION_MIN).tgz

VERSION

@@ -1 +1 @@
-2.2-11
+2.2-35

aux/broccoli

@@ -1 +1 @@
-Subproject commit 756a8a733b1f03b94afcbb93807813a89b3cfb89
+Subproject commit fd28cff404a9f8a8b58972ac31a331c5004e5f0c

bro-path-dev.in

@@ -10,10 +10,4 @@
 #     BROPATH=`./bro-path-dev` ./src/bro
 #
 
-broPolicies=${BRO_SCRIPT_SOURCE_PATH}:${BRO_SCRIPT_SOURCE_PATH}/policy:${BRO_SCRIPT_SOURCE_PATH}/site
+echo .:${CMAKE_SOURCE_DIR}/scripts:${CMAKE_SOURCE_DIR}/scripts/policy:${CMAKE_SOURCE_DIR}/scripts/site:${CMAKE_BINARY_DIR}/scripts
-
-broGenPolicies=${CMAKE_BINARY_DIR}/scripts
-
-installedPolicies=${BRO_SCRIPT_INSTALL_PATH}:${BRO_SCRIPT_INSTALL_PATH}/site
-
-echo .:$broPolicies:$broGenPolicies

doc/CMakeLists.txt

@@ -1,75 +1,86 @@
-set(BIF_SRC_DIR ${PROJECT_SOURCE_DIR}/src)
+set(BROCCOLI_DOCS_SRC ${CMAKE_BINARY_DIR}/aux/broccoli/doc/html)
-set(RST_OUTPUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/rest_output)
+set(BROCCOLI_DOCS_DST ${CMAKE_BINARY_DIR}/html/broccoli-api)
-set(DOC_OUTPUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/out)
+set(SPHINX_INPUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/sphinx_input)
-set(DOC_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR})
+set(SPHINX_OUTPUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/sphinx_output)
-set(DOC_SOURCE_WORKDIR ${CMAKE_CURRENT_BINARY_DIR}/sphinx-sources)
+set(BROXYGEN_SCRIPT_OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/broxygen_script_output)
+set(BROXYGEN_CACHE_DIR ${CMAKE_CURRENT_BINARY_DIR}/broxygen_cache)
 
-set(MASTER_POLICY_INDEX ${CMAKE_CURRENT_BINARY_DIR}/scripts/policy_index)
+# Find out what BROPATH to use when executing bro.
-set(MASTER_PACKAGE_INDEX ${CMAKE_CURRENT_BINARY_DIR}/scripts/pkg_index)
+execute_process(COMMAND ${CMAKE_BINARY_DIR}/bro-path-dev
+                OUTPUT_VARIABLE BROPATH
+                RESULT_VARIABLE retval
+                OUTPUT_STRIP_TRAILING_WHITESPACE)
+if (NOT ${retval} EQUAL 0)
+    message(FATAL_ERROR "Problem setting BROPATH")
+endif ()
 
-file(GLOB_RECURSE DOC_SOURCES FOLLOW_SYMLINKS "*")
+set(BROMAGIC ${BRO_MAGIC_SOURCE_PATH})
 
-# configure the Sphinx config file (expand variables CMake might know about)
+# Configure the Sphinx config file (expand variables CMake might know about).
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/conf.py.in
                ${CMAKE_CURRENT_BINARY_DIR}/conf.py
                @ONLY)
 
-add_subdirectory(scripts)
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/broxygen.conf.in
+               ${CMAKE_CURRENT_BINARY_DIR}/broxygen.conf
+               @ONLY)
 
-# The "broxygen" target generates reST documentation for any outdated bro
+add_custom_target(sphinxdoc
-# scripts and then uses Sphinx to generate HTML documentation from the reST
+    # Copy the template documentation to build directory to use as input tree
-add_custom_target(broxygen
+    # for Sphinx.  This is needed because some parts are dynamically generated
-                  # copy the template documentation to the build directory
+    # in to that tree by Bro/Broxygen.
-                  # to give as input for sphinx
+    COMMAND rsync -q -r --copy-links --times --delete
-                  COMMAND rsync -r --copy-links --times
+            --filter='protect scripts/*'
-                          ${DOC_SOURCE_DIR}/
+            ${CMAKE_CURRENT_SOURCE_DIR}/ ${SPHINX_INPUT_DIR}
-                          ${DOC_SOURCE_WORKDIR}
+    # Use Bro/Broxygen to dynamically generate reST for all Bro scripts.
-                  # copy generated policy script documentation into the
+    COMMAND BROPATH=${BROPATH}
-                  # working copy of the template documentation
+            BROMAGIC=${BROMAGIC}
-                  COMMAND rsync -r --copy-links --times
+            ${CMAKE_BINARY_DIR}/src/bro
-                          ${RST_OUTPUT_DIR}/
+            -X ${CMAKE_CURRENT_BINARY_DIR}/broxygen.conf
-                          ${DOC_SOURCE_WORKDIR}/scripts
+            broxygen >/dev/null
-                  # append to the master index of all policy scripts
+    # Rsync over the generated reST to the Sphinx source tree in the build dir.
-                  COMMAND cat ${MASTER_POLICY_INDEX} >>
+    COMMAND rsync -q -r --copy-links --times --delete --filter='protect *.bro'
-                          ${DOC_SOURCE_WORKDIR}/scripts/scripts.rst
+            ${BROXYGEN_SCRIPT_OUTPUT}/ ${SPHINX_INPUT_DIR}/scripts
-                  # append to the master index of all policy packages
+    # Rsync over Bro scripts to the Sphinx source tree in the build dir.
-                  COMMAND cat ${MASTER_PACKAGE_INDEX} >>
+    # These are used by :download: references in the generated script docs.
-                          ${DOC_SOURCE_WORKDIR}/scripts/packages.rst
+    COMMAND rsync -q -r --copy-links --times --delete
-                  # construct a reST file for each group
+            --filter='protect /base/bif/*' --filter='protect *.rst'
-                  COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/bin/group_index_generator.py
+            --filter='include */' --filter='include *.bro' --filter='exclude *'
-                          ${CMAKE_CURRENT_BINARY_DIR}/scripts/group_list
+            ${CMAKE_SOURCE_DIR}/scripts/ ${SPHINX_INPUT_DIR}/scripts
-                          ${CMAKE_CURRENT_BINARY_DIR}/scripts
+    # Rsync over Bro scripts created by BIF compiler to the Sphinx source tree.
-                          ${DOC_SOURCE_WORKDIR}/scripts
+    COMMAND rsync -q -r --copy-links --times --delete
-                  # tell sphinx to generate html
+            --filter='protect *.rst' --filter='include */'
+            --filter='include *.bro' --filter='exclude *'
+            ${CMAKE_BINARY_DIR}/scripts/base/bif/
+            ${SPHINX_INPUT_DIR}/scripts/base/bif
+    # Use Sphinx to build HTML.
     COMMAND sphinx-build
             -b html
             -c ${CMAKE_CURRENT_BINARY_DIR}
-                          -d ${DOC_OUTPUT_DIR}/doctrees
+            -d ${SPHINX_OUTPUT_DIR}/doctrees
-                          ${DOC_SOURCE_WORKDIR}
+            ${SPHINX_INPUT_DIR}
-                          ${DOC_OUTPUT_DIR}/html
+            ${SPHINX_OUTPUT_DIR}/html
-                  # create symlink to the html output directory for convenience
+    # Create symlink to the html output directory for convenience.
     COMMAND "${CMAKE_COMMAND}" -E create_symlink
-                          ${DOC_OUTPUT_DIR}/html
+            ${SPHINX_OUTPUT_DIR}/html
             ${CMAKE_BINARY_DIR}/html
-                  # copy Broccoli API reference into output dir if it exists
+    # Copy Broccoli API reference into output dir if it exists.
-                  COMMAND test -d ${CMAKE_BINARY_DIR}/aux/broccoli/doc/html && ( rm -rf ${CMAKE_BINARY_DIR}/html/broccoli-api && cp -r ${CMAKE_BINARY_DIR}/aux/broccoli/doc/html ${CMAKE_BINARY_DIR}/html/broccoli-api ) || true
+    COMMAND test -d ${BROCCOLI_DOCS_SRC} &&
+            ( rm -rf ${BROCCOLI_DOCS_DST} &&
+              cp -r ${BROCCOLI_DOCS_SRC} ${BROCCOLI_DOCS_DST} ) || true
     WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
-                  COMMENT "[Sphinx] Generating HTML policy script docs"
+    COMMENT "[Sphinx] Generate HTML documentation in ${CMAKE_BINARY_DIR}/html")
-                  # SOURCES just adds stuff to IDE projects as a convenience
-                  SOURCES ${DOC_SOURCES})
 
-# The "broxygenclean" target removes just the Sphinx input/output directories
+add_dependencies(sphinxdoc bro)
-# from the build directory.
+
-add_custom_target(broxygenclean
+add_custom_target(sphinxdoc_clean
-                  COMMAND "${CMAKE_COMMAND}" -E remove_directory
+    COMMAND "${CMAKE_COMMAND}" -E remove_directory ${SPHINX_INPUT_DIR}
-                          ${DOC_SOURCE_WORKDIR}
+    COMMAND "${CMAKE_COMMAND}" -E remove_directory ${SPHINX_OUTPUT_DIR}
-                  COMMAND "${CMAKE_COMMAND}" -E remove_directory
+    COMMAND "${CMAKE_COMMAND}" -E remove_directory ${BROXYGEN_SCRIPT_OUTPUT}
-                          ${DOC_OUTPUT_DIR}
+    COMMAND "${CMAKE_COMMAND}" -E remove_directory ${BROXYGEN_CACHE_DIR}
     VERBATIM)
 
-add_dependencies(broxygen restdoc)
-
 add_custom_target(doc)
 add_custom_target(docclean)
-add_dependencies(doc broxygen)
+add_dependencies(doc sphinxdoc)
-add_dependencies(docclean broxygenclean restclean)
+add_dependencies(docclean sphinxdoc_clean)

doc/README

@@ -15,17 +15,16 @@ which adds some reST directives and roles that aid in generating useful
 index entries and cross-references.  Other extensions can be added in
 a similar fashion.
 
-Either the ``make doc`` or ``make broxygen`` targets in the top-level
+The ``make doc`` target in the top-level Makefile can be used to locally
-Makefile can be used to locally render the reST files into HTML.
+render the reST files into HTML.  That target depends on:
-Those targets depend on:
 
 * Python interpreter >= 2.5
 * `Sphinx <http://sphinx.pocoo.org/>`_ >= 1.0.1
 
 After completion, HTML documentation is symlinked in ``build/html``.
 
-There's also ``make docclean`` and ``make broxygenclean`` targets to
+There's also a ``make docclean`` target which deletes any files
-clean the resulting documentation.
+created during the documentation build process.
 
 Notes for Writing Documentation
 -------------------------------

doc/bin/group_index_generator.py

@@ -1,72 +0,0 @@
-#! /usr/bin/env python
-
-# This script automatically generates a reST documents that lists
-# a collection of Bro scripts that are "grouped" together.
-# The summary text (##! comments) of the script is embedded in the list
-#
-# 1st argument is the file containing list of groups
-# 2nd argument is the directory containing ${group}_files lists of
-#   scripts that belong to the group and ${group}_doc_names lists of
-#   document names that can be supplied to a reST :doc: role
-# 3rd argument is a directory in which write a ${group}.rst file that contains
-#   reST style references to script docs along with summary text contained
-#   in original script.  If ${group} ends with "index", then the file
-#   is always clobbered by this script, but for other unique group names,
-#   this script will append to existing files.
-
-import sys
-import os
-import string
-
-group_list = sys.argv[1]
-file_manifest_dir = sys.argv[2]
-output_dir = sys.argv[3]
-
-def make_group_file_index(dir_name, group_name):
-    group_file = os.path.join(dir_name, group_name + ".rst")
-
-    if not os.path.exists(group_file):
-        if not os.path.exists(os.path.dirname(group_file)):
-            os.makedirs(os.path.dirname(group_file))
-
-    if group_name.endswith("index"):
-        with open(group_file, 'w') as f_group_file:
-            f_group_file.write(":orphan:\n\n")
-            title = "Package Index: %s\n" % os.path.dirname(group_name)
-            f_group_file.write(title);
-            for n in range(len(title)):
-                f_group_file.write("=")
-            f_group_file.write("\n");
-
-    return group_file
-
-with open(group_list, 'r') as f_group_list:
-    for group in f_group_list.read().splitlines():
-        #print group
-        group_file = make_group_file_index(output_dir, group)
-        file_manifest = os.path.join(file_manifest_dir, group + "_files")
-        doc_manifest = os.path.join(file_manifest_dir, group + "_doc_names")
-        src_files = []
-        doc_names = []
-
-        with open(file_manifest, 'r') as f_file_manifest:
-            src_files = f_file_manifest.read().splitlines()
-
-        with open(doc_manifest, 'r') as f_doc_manifest:
-            doc_names = f_doc_manifest.read().splitlines()
-
-        for i in range(len(src_files)):
-            src_file = src_files[i]
-            #print "\t" + src_file
-            summary_comments = []
-            with open(src_file, 'r') as f_src_file:
-                for line in f_src_file:
-                    sum_pos = string.find(line, "##!")
-                    if sum_pos != -1:
-                        summary_comments.append(line[(sum_pos+3):])
-            #print summary_comments
-
-            with open(group_file, 'a') as f_group_file:
-                f_group_file.write("\n:doc:`/scripts/%s`\n" % doc_names[i])
-                for line in summary_comments:
-                    f_group_file.write("   " + line)

doc/broxygen.conf.in

@@ -0,0 +1 @@
+script	*	@BROXYGEN_SCRIPT_OUTPUT@/

doc/conf.py.in

@@ -17,7 +17,7 @@ extensions = []
 # If extensions (or modules to document with autodoc) are in another directory,
 # add these directories to sys.path here. If the directory is relative to the
 # documentation root, use os.path.abspath to make it absolute, like shown here.
-sys.path.insert(0, os.path.abspath('sphinx-sources/ext'))
+sys.path.insert(0, os.path.abspath('sphinx_input/ext'))
 
 # ----- Begin of BTest configuration. -----
 btest = os.path.abspath("@CMAKE_SOURCE_DIR@/aux/btest")
@@ -33,6 +33,12 @@ btest_base="@CMAKE_SOURCE_DIR@/testing/btest"
 btest_tests="doc/sphinx"
 # ----- End of BTest configuration. -----
 
+# ----- Begin of Broxygen configuration. -----
+extensions += ["broxygen"]
+broxygen_cache="@BROXYGEN_CACHE_DIR@"
+os.environ["BROPATH"] = "@BROPATH@"
+os.environ["BROMAGIC"] = "@BROMAGIC@"
+# ----- End of Broxygen configuration. -----
 
 # -- General configuration -----------------------------------------------------
 
@@ -47,7 +53,7 @@ os.environ["BRO_SRC_ROOT"] = "@CMAKE_SOURCE_DIR@"
 os.environ["DOC_ROOT"] = "@CMAKE_SOURCE_DIR@/doc"
 
 # Add any paths that contain templates here, relative to this directory.
-templates_path = ['sphinx-sources/_templates', 'sphinx-sources/_static']
+templates_path = ['sphinx_input/_templates', 'sphinx_input/_static']
 
 # The suffix of source filenames.
 source_suffix = '.rst'
@@ -141,7 +147,7 @@ html_theme_options = { }
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,
 # so a file named "default.css" will overwrite the builtin "default.css".
-html_static_path = ['sphinx-sources/_static']
+html_static_path = ['sphinx_input/_static']
 
 # If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
 # using the given strftime format.

doc/ext/bro.py

@@ -191,6 +191,10 @@ class BroNotices(Index):
 
     def generate(self, docnames=None):
         content = {}
+
+        if 'notices' not in self.domain.env.domaindata['bro']:
+            return content, False
+
         for n in self.domain.env.domaindata['bro']['notices']:
             modname = n[0].split("::")[0]
             entries = content.setdefault(modname, [])

doc/ext/broxygen.py

@@ -0,0 +1,311 @@
+"""
+Broxygen domain for Sphinx.
+
+Adds directives that allow Sphinx to invoke Bro in order to generate script
+reference documentation on the fly.  The directives are:
+
+broxygen:package
+    - Shows links to all scripts contained within matching package(s).
+broxygen:package_index
+    - An index with links to matching package document(s).
+broxygen:script
+    - Reference for matching script(s) (i.e. everything declared by the script).
+broxygen:script_summary
+    - Shows link to matching script(s) with it's summary-section comments.
+broxygen:script_index
+    - An index with links to all matching scrips.
+broxygen:proto_analyzer
+    - All protocol analyzers and their components (events/bifs, etc.)
+broxygen:file_analyzer
+    - All file analyzers and their components (events/bifs, etc.)
+"""
+
+
+from sphinx.domains import Domain, ObjType
+from sphinx.locale import l_
+from docutils.parsers.rst.directives.misc import Include
+
+
+App = None
+
+
+def info(msg):
+    """Use Sphinx builder to output a console message."""
+    global App
+    from sphinx.util.console import blue
+    App.builder.info(blue(msg))
+
+
+def pattern_to_filename_component(pattern):
+    """Replace certain characters in Broxygen config file target pattern.
+
+    Such that it can be used as part of a (sane) filename.
+
+    """
+    return pattern.replace("/", ".").replace("*", "star")
+
+
+def ensure_dir(path):
+    """Should act like ``mkdir -p``."""
+    import os
+    import errno
+
+    try:
+        os.makedirs(path)
+    except OSError as e:
+        if e.errno != errno.EEXIST:
+            raise
+
+
+def generate_config(env, type, pattern):
+    """Create a Broxygen config file for a particular target.
+
+    It can be used by Bro to generate reST docs for that target.
+
+    """
+    import os
+    import tempfile
+    from sphinx.errors import SphinxError
+
+    work_dir = env.config.broxygen_cache
+
+    if not work_dir:
+        raise SphinxError("broxygen_cache not set in sphinx config file")
+
+    ensure_dir(work_dir)
+    prefix = "{0}-{1}-".format(type, pattern_to_filename_component(pattern))
+    (fd, cfg) = tempfile.mkstemp(suffix=".cfg", prefix=prefix, dir=work_dir)
+    generated_file = "{0}.rst".format(cfg)
+    config = "{0}\t{1}\t{2}".format(type, pattern, generated_file)
+    f = os.fdopen(fd, "w")
+    f.write(config)
+    f.close()
+    return (cfg, generated_file)
+
+
+def generate_target(env, type, pattern):
+    """Create a Broxygen target and build it.
+
+    For a target which hasn't been referenced by any other script, this function
+    creates an associated config file then uses Bro w/ it to build the target
+    and stores the target information in the build environment.
+
+    If a script references a target that's already found in the build
+    environment the results of the previous built are re-used.
+
+    """
+    app_data = env.domaindata["broxygen"]
+
+    if (type, pattern) in app_data["targets"]:
+        info("Broxygen has cached doc for target '{0} {1}'".format(
+            type, pattern))
+        return app_data["targets"]
+
+    (cfg, gend_file) = generate_config(env, type, pattern)
+    target = BroxygenTarget(type, pattern, cfg, gend_file)
+    app_data["targets"][(type, pattern)] = target
+    build_target(env, target)
+    info("Broxygen built target '{0} {1}'".format(type, pattern))
+    return target
+
+
+def build_target(env, target):
+    """Invoke a Bro process to build a Broxygen target."""
+    import os
+    import subprocess
+
+    bro_cmd = "bro -X {0} broxygen".format(target.config_file)
+    cwd = os.getcwd()
+    os.chdir(os.path.dirname(target.config_file))
+
+    try:
+        subprocess.check_output(bro_cmd, stderr=subprocess.STDOUT, shell=True)
+    except subprocess.CalledProcessError as e:
+        from sphinx.errors import SphinxError
+        raise SphinxError(
+            "Command '{0}' returned non-zero exit status {1}: {2}".format(
+                e.cmd, e.returncode, e.output))
+    finally:
+        os.chdir(cwd)
+
+
+class BroxygenTarget(object):
+
+    """Some portion of reST documentation that Bro knows how to generate.
+
+    A target is identified by its type and pattern.  E.g. type "script" and
+    pattern "broxygen/example.bro".
+
+    """
+
+    def __init__(self, type, pattern, config_file, generated_file):
+        self.type = type
+        self.pattern = pattern
+        self.config_file = config_file
+        self.generated_file = generated_file
+        self.used_in_docs = set()
+
+
+class BroxygenDirective(Include):
+
+    """Base class for Broxygen directives.
+
+    It can use Bro to generate reST documentation on the fly and embed it in
+    the document at the location of the directive just like the ``.. include::``
+    directive.  The only argument is a pattern to identify to Bro which
+    pieces of documentation it needs to create.
+    """
+
+    required_arguments = 1
+    has_content = False
+
+    target_type = None
+
+    def run(self):
+        env = self.state.document.settings.env
+        info("Broxygen running .. {0}:: {1} in {2}".format(
+            self.name, self.arguments[0], env.docname))
+        target = generate_target(env, self.target_type, self.arguments[0])
+        target.used_in_docs.add(env.docname)
+        self.arguments = [target.generated_file]
+        return super(BroxygenDirective, self).run()
+
+
+class PackageDirective(BroxygenDirective):
+
+    target_type = "package"
+
+
+class PackageIndexDirective(BroxygenDirective):
+
+    target_type = "package_index"
+
+
+class ScriptDirective(BroxygenDirective):
+
+    target_type = "script"
+
+
+class ScriptSummaryDirective(BroxygenDirective):
+
+    target_type = "script_summary"
+
+
+class ScriptIndexDirective(BroxygenDirective):
+
+    target_type = "script_index"
+
+
+class ProtoAnalyzerDirective(BroxygenDirective):
+
+    target_type = "proto_analyzer"
+
+
+class FileAnalyzerDirective(BroxygenDirective):
+
+    target_type = "file_analyzer"
+
+
+class IdentifierDirective(BroxygenDirective):
+
+    target_type = "identifier"
+
+
+class BroxygenDomain(Domain):
+
+    name = "broxygen"
+    label = "Broxygen"
+
+    object_types = {
+        "package":          ObjType(l_("package")),
+        "package_index":    ObjType(l_("package_index")),
+        "script":           ObjType(l_("script")),
+        "script_summary":   ObjType(l_("script_summary")),
+        "script_index":     ObjType(l_("script_index")),
+        "proto_analyzer":   ObjType(l_("proto_analyzer")),
+        "file_analyzer":    ObjType(l_("file_analyzer")),
+        "identifier":       ObjType(l_("identifier")),
+    }
+
+    directives = {
+        "package":          PackageDirective,
+        "package_index":    PackageIndexDirective,
+        "script":           ScriptDirective,
+        "script_summary":   ScriptSummaryDirective,
+        "script_index":     ScriptIndexDirective,
+        "proto_analyzer":   ProtoAnalyzerDirective,
+        "file_analyzer":    FileAnalyzerDirective,
+        "identifier":       IdentifierDirective,
+    }
+
+    roles = {}
+
+    initial_data = {
+        "targets": {}
+    }
+
+    def clear_doc(self, docname):
+        """Update Broxygen targets referenced in docname.
+
+        If it's the last place the target was referenced, remove it from
+        the build environment and delete any generated config/reST files
+        associated with it from the cache.
+
+        """
+        import os
+
+        stale_targets = []
+
+        for (type, pattern), target in self.data["targets"].items():
+            if docname in target.used_in_docs:
+                target.used_in_docs.remove(docname)
+
+                if not target.used_in_docs:
+                    stale_targets.append(target)
+
+        for target in stale_targets:
+            del self.data["targets"][(target.type, target.pattern)]
+            os.remove(target.config_file)
+            os.remove(target.generated_file)
+
+    def get_objects(self):
+        """No Broxygen-generated content is itself linkable/searchable."""
+        return []
+
+
+def env_get_outdated_hook(app, env, added, changed, removed):
+    """Check whether to re-read any documents referencing Broxygen targets.
+
+    To do that we have to ask Bro to rebuild each target and compare the
+    before and after modification times of the generated reST output file.
+    If Bro changed it, then the document containing the Broxygen directive
+    needs to be re-read.
+
+    """
+    import os
+
+    reread = set()
+
+    for target in app.env.domaindata["broxygen"]["targets"].values():
+        before_mtime = os.stat(target.generated_file)
+        build_target(env, target)
+        after_mtime = os.stat(target.generated_file)
+
+        if after_mtime > before_mtime:
+            info("Broxygen target '{0} {1}' outdated".format(
+                target.type, target.pattern))
+
+            for docname in target.used_in_docs:
+                if docname not in removed:
+                    info("  in document: {0}".format(docname))
+                    reread.add(docname)
+
+    return list(reread)
+
+
+def setup(app):
+    global App
+    App = app
+    app.add_domain(BroxygenDomain)
+    app.add_config_value("broxygen_cache", None, "env")
+    app.connect("env-get-outdated", env_get_outdated_hook)

doc/frameworks/logging-dataseries.rst

@@ -104,7 +104,7 @@ code like this to your ``local.bro``:
         }
 
 Bro's DataSeries writer comes with a few tuning options, see
-:doc:`/scripts/base/frameworks/logging/writers/dataseries`.
+:doc:`/scripts/base/frameworks/logging/writers/dataseries.bro`.
 
 Working with DataSeries
 =======================

doc/frameworks/logging.rst

@@ -48,7 +48,7 @@ Basics
 The data fields that a stream records are defined by a record type
 specified when it is created. Let's look at the script generating Bro's
 connection summaries as an example,
-:doc:`/scripts/base/protocols/conn/main`. It defines a record
+:doc:`/scripts/base/protocols/conn/main.bro`. It defines a record
 :bro:type:`Conn::Info` that lists all the fields that go into
 ``conn.log``, each marked with a ``&log`` attribute indicating that it
 is part of the information written out. To write a log record, the
@@ -309,7 +309,7 @@ ASCII Writer Configuration
 --------------------------
 
 The ASCII writer has a number of options for customizing the format of
-its output, see :doc:`/scripts/base/frameworks/logging/writers/ascii`.
+its output, see :doc:`/scripts/base/frameworks/logging/writers/ascii.bro`.
 
 Adding Streams
 ==============
@@ -369,7 +369,7 @@ save the logged ``Foo::Info`` record into the connection record:
         }
 
 See the existing scripts for how to work with such a new connection
-field. A simple example is :doc:`/scripts/base/protocols/syslog/main`.
+field. A simple example is :doc:`/scripts/base/protocols/syslog/main.bro`.
 
 When you are developing scripts that add data to the :bro:type:`connection`
 record, care must be given to when and how long data is stored.

doc/frameworks/notice.rst

@@ -283,7 +283,7 @@ information to suppress duplicates for a configurable period of time.
 The ``$identifier`` field is typically comprised of several pieces of
 data related to the notice that when combined represent a unique
 instance of that notice. Here is an example of the script
-:doc:`/scripts/policy/protocols/ssl/validate-certs` raising a notice
+:doc:`/scripts/policy/protocols/ssl/validate-certs.bro` raising a notice
 for session negotiations where the certificate or certificate chain did
 not validate successfully against the available certificate authority
 certificates.

doc/frameworks/signatures.rst

@@ -46,7 +46,7 @@ signature's event statement (``Found root!``), and data is the last
 piece of payload which triggered the pattern match.
 
 To turn such :bro:id:`signature_match` events into actual alarms, you can
-load Bro's :doc:`/scripts/base/frameworks/signatures/main` script.
+load Bro's :doc:`/scripts/base/frameworks/signatures/main.bro` script.
 This script contains a default event handler that raises
 :bro:enum:`Signatures::Sensitive_Signature` :doc:`Notices <notice>`
 (as well as others; see the beginning of the script).

doc/frameworks/sumstats.rst

@@ -87,7 +87,7 @@ Taking the previous example even further, we can implement a simple detection
 to demonstrate the thresholding functionality.  This example is a toy to
 demonstrate how thresholding works in Sumstats and is not meant to be a
 real-world functional example, that is left to the
-:doc:`/scripts/policy/misc/scan` script that is included with Bro.
+:doc:`/scripts/policy/misc/scan.bro` script that is included with Bro.
 
 .. btest-include:: ${DOC_ROOT}/frameworks/sumstats-toy-scan.bro
 

doc/index.rst

@@ -15,7 +15,7 @@ Bro Documentation
    scripting/index.rst
    frameworks/index.rst
    cluster/index.rst
-   scripts/index.rst
+   script-reference/index.rst
    components/index.rst
 
 * :ref:`General Index <genindex>`

doc/quickstart/index.rst

@@ -197,7 +197,7 @@ the variable's value may not change at run-time, but whose initial value can be
 modified via the ``redef`` operator at parse-time.
 
 So let's continue on our path to modify the behavior for the two SSL
-and SSH notices.  Looking at :doc:`/scripts/base/frameworks/notice/main`,
+and SSH notices.  Looking at :doc:`/scripts/base/frameworks/notice/main.bro`,
 we see that it advertises:
 
 .. code:: bro

doc/script-reference/file-analyzers.rst

@@ -0,0 +1 @@
+.. broxygen:file_analyzer:: *

doc/script-reference/index.rst

@@ -1,5 +1,3 @@
-.. This is a stub doc to which broxygen appends during the build process
-
 ================
 Script Reference
 ================
@@ -7,15 +5,10 @@ Script Reference
 .. toctree::
    :maxdepth: 1
 
-   packages
+   notices
    proto-analyzers
    file-analyzers
-   notices
    builtins
-   Built-in Functions (BIFs) <base/bif/index>
+   packages
-   internal
    scripts
-
+   Broxygen Example Script </scripts/broxygen/example.bro>
-
-
-   

doc/script-reference/packages.rst

@@ -1,9 +1,7 @@
-.. This is a stub doc to which broxygen appends during the build process
-
 .. _script-packages:
 
-Bro Script Packages
+Bro Package Index
-===================
+=================
 
 Bro has the following script packages (e.g. collections of related scripts in
 a common directory).  If the package directory contains a ``__load__.bro``
@@ -12,3 +10,5 @@ script, it supports being loaded in mass as a whole directory for convenience.
 Packages/scripts in the ``base/`` directory are all loaded by default, while
 ones in ``policy/`` provide functionality and customization options that are
 more appropriate for users to decide whether they'd like to load it or not.
+
+.. broxygen:package_index:: *

doc/script-reference/proto-analyzers.rst

@@ -0,0 +1 @@
+.. broxygen:proto_analyzer:: *

doc/script-reference/scripts.rst

@@ -0,0 +1,5 @@
+================
+Bro Script Index
+================
+
+.. broxygen:script_index:: *

doc/scripting/index.rst

@@ -316,7 +316,7 @@ block that variable is available to any other script through the
 naming convention of ``MODULE::variable_name``.
 
 The declaration below is taken from the
-:doc:`/scripts/policy/protocols/conn/known-hosts` script and
+:doc:`/scripts/policy/protocols/conn/known-hosts.bro` script and
 declares a variable called ``known_hosts`` as a global set of unique
 IP addresses within the ``Known`` namespace and exports it for use
 outside of the ``Known`` namespace.  Were we to want to use the
@@ -348,7 +348,7 @@ constants are used in Bro scripts as containers for configuration
 options.  For example, the configuration option to log password
 decrypted from HTTP streams is stored in
 ``HTTP::default_capture_password`` as shown in the stripped down
-excerpt from :doc:`/scripts/base/protocols/http/main` below.
+excerpt from :doc:`/scripts/base/protocols/http/main.bro` below.
 
 .. btest-include:: ${BRO_SRC_ROOT}/scripts/base/protocols/http/main.bro
    :lines: 8-10,19-21,120
@@ -1182,7 +1182,7 @@ passing in the ``Notice::Info`` record.  The simplest kind of
 action based on the answer.  The hook below adds the
 :bro:enum:`Notice::ACTION_EMAIL` action for the
 ``SSH::Interesting_Hostname_Login`` notice raised in the
-:doc:`/scripts/policy/protocols/ssh/interesting-hostnames` script.
+:doc:`/scripts/policy/protocols/ssh/interesting-hostnames.bro` script.
 
 .. btest-include:: ${DOC_ROOT}/scripting/framework_notice_hook_01.bro
 
@@ -1224,7 +1224,7 @@ Bro.
 .. btest-include:: ${BRO_SRC_ROOT}/scripts/policy/protocols/ssl/expiring-certs.bro
    :lines: 60-63
 
-In the :doc:`/scripts/policy/protocols/ssl/expiring-certs` script
+In the :doc:`/scripts/policy/protocols/ssl/expiring-certs.bro` script
 which identifies when SSL certificates are set to expire and raises
 notices when it crosses a predefined threshold, the call to
 ``NOTICE`` above also sets the ``$identifier`` entry by concatenating

doc/scripts/CMakeLists.txt

@@ -1,226 +0,0 @@
-# find out what BROPATH to use when executing bro
-execute_process(COMMAND ${CMAKE_BINARY_DIR}/bro-path-dev
-                OUTPUT_VARIABLE BROPATH
-                RESULT_VARIABLE retval
-                OUTPUT_STRIP_TRAILING_WHITESPACE)
-if (NOT ${retval} EQUAL 0)
-    message(FATAL_ERROR "Problem setting BROPATH")
-endif ()
-
-# This macro is used to add a new makefile target for reST policy script
-# documentation that can be generated using Bro itself to parse policy scripts.
-# It's called like:
-#
-#     rest_target(srcDir broInput [group])
-#
-# srcDir: the directory which contains broInput
-# broInput: the file name of a bro policy script, any path prefix of this
-#     argument will be used to derive what path under scripts/ the generated
-#     documentation will be placed.
-# group: optional name of group that the script documentation will belong to.
-#     If this is not given, the group is automatically set to any path portion
-#     of the broInput argument.
-#
-# In addition to adding the makefile target, several CMake variables are set:
-#
-# MASTER_POLICY_INDEX_TEXT: a running list of policy scripts docs that have
-#   been generated so far, formatted such that it can be appended to a file
-#   that ends in a Sphinx toctree directive
-# ALL_REST_OUTPUTS: a running list (the CMake list type) of all reST docs
-#   that are to be generated
-# MASTER_GROUP_LIST: a running list (the CMake list type) of all script groups
-# MASTER_PKG_LIST: a running list (the CMake list type) of all script groups
-#   that were defived from the path portion of the broInput argument
-# ${group}_files: a running list of files belonging to a given group, from
-#   which summary text can be extracted at build time
-# ${group}_doc_names: a running list of reST style document names that can be
-#   given to a :doc: role, shared indices with ${group}_files
-
-macro(REST_TARGET srcDir broInput)
-    set(absSrcPath ${srcDir}/${broInput})
-    get_filename_component(basename ${broInput} NAME)
-    string(REPLACE .bro "" basename ${basename})
-    get_filename_component(extension ${broInput} EXT)
-    get_filename_component(relDstDir ${broInput} PATH)
-
-    set(sumTextSrc ${absSrcPath})
-    set(ogSourceFile ${absSrcPath})
-
-    if (NOT relDstDir)
-        set(docName "${basename}")
-        set(dstDir "${RST_OUTPUT_DIR}")
-    else ()
-        set(docName "${relDstDir}/${basename}")
-        set(dstDir "${RST_OUTPUT_DIR}/${relDstDir}")
-    endif ()
-
-    set(restFile "${docName}.rst")
-    string(REPLACE "/" "^" restFile ${restFile})
-    set(restOutput "${dstDir}/${basename}.rst")
-
-    set(MASTER_POLICY_INDEX_TEXT
-        "${MASTER_POLICY_INDEX_TEXT}\n   ${docName} <${docName}>")
-    list(APPEND ALL_REST_OUTPUTS ${restOutput})
-
-    if (NOT "${ARGN}" STREQUAL "")
-        set(group ${ARGN})
-    elseif (relDstDir)
-        set(group ${relDstDir}/index)
-        # add package index to master package list if not already in it
-        # and if a __load__.bro exists in the original script directory
-        list(FIND MASTER_PKG_LIST ${relDstDir} _found)
-        if (_found EQUAL -1)
-            if (EXISTS ${CMAKE_SOURCE_DIR}/scripts/${relDstDir}/__load__.bro)
-                list(APPEND MASTER_PKG_LIST ${relDstDir})
-            endif ()
-        endif ()
-    else ()
-        set(group "")
-    endif ()
-
-    if (NOT "${group}" STREQUAL "")
-        # add group to master group list if not already in it
-        list(FIND MASTER_GROUP_LIST ${group} _found)
-        if (_found EQUAL -1)
-            list(APPEND MASTER_GROUP_LIST ${group})
-            if (MASTER_GROUP_LIST_TEXT)
-               set(MASTER_GROUP_LIST_TEXT "${MASTER_GROUP_LIST_TEXT}\n${group}")
-            else ()
-               set(MASTER_GROUP_LIST_TEXT "${group}")
-            endif ()
-        endif ()
-
-        list(APPEND ${group}_files ${sumTextSrc})
-        list(APPEND ${group}_doc_names ${docName})
-    endif ()
-
-    add_custom_command(OUTPUT ${restOutput}
-        # delete any leftover state from previous bro runs
-        COMMAND "${CMAKE_COMMAND}"
-        ARGS -E remove_directory .state
-        # generate the reST documentation using bro
-        COMMAND BROPATH=${BROPATH}:${srcDir} BROMAGIC=${CMAKE_SOURCE_DIR}/magic/database ${CMAKE_BINARY_DIR}/src/bro
-        ARGS -b -Z ${broInput} || (rm -rf .state *.log *.rst && exit 1)
-        # move generated doc into a new directory tree that
-        # defines the final structure of documents
-        COMMAND "${CMAKE_COMMAND}"
-        ARGS -E make_directory ${dstDir}
-        COMMAND "${CMAKE_COMMAND}"
-        ARGS -E copy ${restFile} ${restOutput}
-        # copy the bro or bif script, too
-        COMMAND "${CMAKE_COMMAND}"
-        ARGS -E copy ${ogSourceFile} ${dstDir}
-        # clean up the build directory
-        COMMAND rm
-        ARGS -rf .state *.log *.rst
-        DEPENDS bro
-        DEPENDS ${absSrcPath}
-        WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
-        COMMENT "[Bro] Generating reST docs for ${broInput}"
-    )
-
-endmacro(REST_TARGET)
-
-# Schedule Bro scripts for which to generate documentation.
-include(DocSourcesList.cmake)
-
-# Macro for generating reST docs that are independent of any particular Bro
-# script.
-macro(INDEPENDENT_REST_TARGET reST_file)
-    add_custom_command(OUTPUT ${reST_file}
-        # delete any leftover state from previous bro runs
-        COMMAND "${CMAKE_COMMAND}"
-        ARGS -E remove_directory .state
-        # generate the reST documentation using bro
-        COMMAND BROPATH=${BROPATH}:${srcDir} BROMAGIC=${CMAKE_SOURCE_DIR}/magic/database ${CMAKE_BINARY_DIR}/src/bro
-        ARGS -b -Z base/init-bare.bro || (rm -rf .state *.log *.rst && exit 1)
-        # move generated doc into a new directory tree that
-        # defines the final structure of documents
-        COMMAND "${CMAKE_COMMAND}"
-        ARGS -E make_directory ${dstDir}
-        COMMAND "${CMAKE_COMMAND}"
-        ARGS -E copy ${reST_file} ${dstDir}
-        # clean up the build directory
-        COMMAND rm
-        ARGS -rf .state *.log *.rst
-        DEPENDS bro
-        WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
-        COMMENT "[Bro] Generating reST docs for ${reST_file}"
-    )
-    list(APPEND ALL_REST_OUTPUTS ${reST_file})
-endmacro(INDEPENDENT_REST_TARGET)
-
-independent_rest_target(proto-analyzers.rst)
-independent_rest_target(file-analyzers.rst)
-
-# create temporary list of all docs to include in the master policy/index file
-file(WRITE ${MASTER_POLICY_INDEX} "${MASTER_POLICY_INDEX_TEXT}")
-
-# create the temporary list of all packages to include in the master
-# policy/packages.rst file
-set(MASTER_PKG_INDEX_TEXT "")
-foreach (pkg ${MASTER_PKG_LIST})
-    set(MASTER_PKG_INDEX_TEXT
-        "${MASTER_PKG_INDEX_TEXT}\n:doc:`${pkg} <${pkg}/index>`\n")
-    if (EXISTS ${CMAKE_SOURCE_DIR}/scripts/${pkg}/README)
-        file(STRINGS ${CMAKE_SOURCE_DIR}/scripts/${pkg}/README pkgreadme)
-        foreach (line ${pkgreadme})
-            set(MASTER_PKG_INDEX_TEXT "${MASTER_PKG_INDEX_TEXT}\n    ${line}")
-        endforeach ()
-        set(MASTER_PKG_INDEX_TEXT "${MASTER_PKG_INDEX_TEXT}\n")
-    endif ()
-endforeach ()
-file(WRITE ${MASTER_PACKAGE_INDEX} "${MASTER_PKG_INDEX_TEXT}")
-
-# create temporary file containing list of all groups
-file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/group_list
-    "${MASTER_GROUP_LIST_TEXT}")
-
-# create temporary files containing list of each source file in a given group
-foreach (group ${MASTER_GROUP_LIST})
-    if (EXISTS ${CMAKE_CURRENT_BINARY_DIR}/${group}_files)
-        file(REMOVE ${CMAKE_CURRENT_BINARY_DIR}/${group}_files)
-    endif ()
-    if (EXISTS ${CMAKE_CURRENT_BINARY_DIR}/${group}_doc_names)
-        file(REMOVE ${CMAKE_CURRENT_BINARY_DIR}/${group}_doc_names)
-    endif ()
-    foreach (src ${${group}_files})
-        file(APPEND ${CMAKE_CURRENT_BINARY_DIR}/${group}_files "${src}\n")
-    endforeach ()
-    foreach (dname ${${group}_doc_names})
-        file(APPEND ${CMAKE_CURRENT_BINARY_DIR}/${group}_doc_names "${dname}\n")
-    endforeach ()
-endforeach ()
-
-# remove previously generated docs no longer scheduled for generation
-if (EXISTS ${RST_OUTPUT_DIR})
-    file(GLOB_RECURSE EXISTING_REST_DOCS "${RST_OUTPUT_DIR}/*.rst")
-    foreach (_doc ${EXISTING_REST_DOCS})
-        list(FIND ALL_REST_OUTPUTS ${_doc} _found)
-        if (_found EQUAL -1)
-            file(REMOVE ${_doc})
-            message(STATUS "Broxygen: remove stale reST doc: ${_doc}")
-            string(REPLACE .rst .bro _brofile ${_doc})
-            if (EXISTS ${_brofile})
-                file(REMOVE ${_brofile})
-                message(STATUS "Broxygen: remove stale bro source: ${_brofile}")
-            endif ()
-        endif ()
-    endforeach ()
-endif ()
-
-# The "restdoc" target uses Bro to parse policy scripts in order to
-# generate reST documentation from them.
-add_custom_target(restdoc
-                  # create symlink to the reST output directory for convenience
-                  COMMAND "${CMAKE_COMMAND}" -E create_symlink
-                          ${RST_OUTPUT_DIR}
-                          ${CMAKE_BINARY_DIR}/reST
-                  DEPENDS ${ALL_REST_OUTPUTS})
-
-# The "restclean" target removes all generated reST documentation from the
-# build directory.
-add_custom_target(restclean
-                  COMMAND "${CMAKE_COMMAND}" -E remove_directory
-                          ${RST_OUTPUT_DIR}
-                  VERBATIM)

doc/scripts/DocSourcesList.cmake

@@ -1,281 +0,0 @@
-# DO NOT EDIT
-# This file is auto-generated from the genDocSourcesList.sh script.
-#
-# This is a list of Bro script sources for which to generate reST documentation.
-# It will be included inline in the CMakeLists.txt found in the same directory
-# in order to create Makefile targets that define how to generate reST from
-# a given Bro script.
-#
-# Note: any path prefix of the script (2nd argument of rest_target macro)
-# will be used to derive what path under scripts/ the generated documentation
-# will be placed.
-
-set(psd ${PROJECT_SOURCE_DIR}/scripts)
-
-rest_target(${CMAKE_CURRENT_SOURCE_DIR} example.bro internal)
-rest_target(${psd} base/init-default.bro internal)
-rest_target(${psd} base/init-bare.bro internal)
-
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/analyzer.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/bloom-filter.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/bro.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/cardinality-counter.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/const.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/event.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/file_analysis.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/input.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/logging.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_ARP.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_AYIYA.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_BackDoor.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_BitTorrent.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_ConnSize.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_DCE_RPC.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_DHCP.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_DNP3.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_DNS.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_FTP.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_FTP.functions.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_File.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_FileExtract.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_FileExtract.functions.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_FileHash.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_Finger.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_GTPv1.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_Gnutella.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_HTTP.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_HTTP.functions.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_ICMP.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_IRC.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_Ident.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_InterConn.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_Login.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_Login.functions.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_MIME.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_Modbus.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_NCP.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_NTP.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_NetBIOS.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_NetBIOS.functions.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_NetFlow.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_PIA.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_POP3.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_RPC.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_SMB.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_SMTP.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_SMTP.functions.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_SOCKS.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_SSH.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_SSL.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_SSL.functions.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_SteppingStone.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_Syslog.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_TCP.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_TCP.functions.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_Teredo.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_UDP.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_Unified2.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_Unified2.types.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_ZIP.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/reporter.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/strings.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/top-k.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/types.bif.bro)
-rest_target(${psd} base/files/extract/main.bro)
-rest_target(${psd} base/files/hash/main.bro)
-rest_target(${psd} base/files/unified2/main.bro)
-rest_target(${psd} base/frameworks/analyzer/main.bro)
-rest_target(${psd} base/frameworks/cluster/main.bro)
-rest_target(${psd} base/frameworks/cluster/nodes/manager.bro)
-rest_target(${psd} base/frameworks/cluster/nodes/proxy.bro)
-rest_target(${psd} base/frameworks/cluster/nodes/worker.bro)
-rest_target(${psd} base/frameworks/cluster/setup-connections.bro)
-rest_target(${psd} base/frameworks/communication/main.bro)
-rest_target(${psd} base/frameworks/control/main.bro)
-rest_target(${psd} base/frameworks/dpd/main.bro)
-rest_target(${psd} base/frameworks/files/main.bro)
-rest_target(${psd} base/frameworks/input/main.bro)
-rest_target(${psd} base/frameworks/input/readers/ascii.bro)
-rest_target(${psd} base/frameworks/input/readers/benchmark.bro)
-rest_target(${psd} base/frameworks/input/readers/binary.bro)
-rest_target(${psd} base/frameworks/input/readers/raw.bro)
-rest_target(${psd} base/frameworks/input/readers/sqlite.bro)
-rest_target(${psd} base/frameworks/intel/cluster.bro)
-rest_target(${psd} base/frameworks/intel/input.bro)
-rest_target(${psd} base/frameworks/intel/main.bro)
-rest_target(${psd} base/frameworks/logging/main.bro)
-rest_target(${psd} base/frameworks/logging/postprocessors/scp.bro)
-rest_target(${psd} base/frameworks/logging/postprocessors/sftp.bro)
-rest_target(${psd} base/frameworks/logging/writers/ascii.bro)
-rest_target(${psd} base/frameworks/logging/writers/dataseries.bro)
-rest_target(${psd} base/frameworks/logging/writers/elasticsearch.bro)
-rest_target(${psd} base/frameworks/logging/writers/none.bro)
-rest_target(${psd} base/frameworks/logging/writers/sqlite.bro)
-rest_target(${psd} base/frameworks/notice/actions/add-geodata.bro)
-rest_target(${psd} base/frameworks/notice/actions/drop.bro)
-rest_target(${psd} base/frameworks/notice/actions/email_admin.bro)
-rest_target(${psd} base/frameworks/notice/actions/page.bro)
-rest_target(${psd} base/frameworks/notice/actions/pp-alarms.bro)
-rest_target(${psd} base/frameworks/notice/cluster.bro)
-rest_target(${psd} base/frameworks/notice/extend-email/hostnames.bro)
-rest_target(${psd} base/frameworks/notice/main.bro)
-rest_target(${psd} base/frameworks/notice/non-cluster.bro)
-rest_target(${psd} base/frameworks/notice/weird.bro)
-rest_target(${psd} base/frameworks/packet-filter/cluster.bro)
-rest_target(${psd} base/frameworks/packet-filter/main.bro)
-rest_target(${psd} base/frameworks/packet-filter/netstats.bro)
-rest_target(${psd} base/frameworks/packet-filter/utils.bro)
-rest_target(${psd} base/frameworks/reporter/main.bro)
-rest_target(${psd} base/frameworks/signatures/main.bro)
-rest_target(${psd} base/frameworks/software/main.bro)
-rest_target(${psd} base/frameworks/sumstats/cluster.bro)
-rest_target(${psd} base/frameworks/sumstats/main.bro)
-rest_target(${psd} base/frameworks/sumstats/non-cluster.bro)
-rest_target(${psd} base/frameworks/sumstats/plugins/average.bro)
-rest_target(${psd} base/frameworks/sumstats/plugins/hll_unique.bro)
-rest_target(${psd} base/frameworks/sumstats/plugins/last.bro)
-rest_target(${psd} base/frameworks/sumstats/plugins/max.bro)
-rest_target(${psd} base/frameworks/sumstats/plugins/min.bro)
-rest_target(${psd} base/frameworks/sumstats/plugins/sample.bro)
-rest_target(${psd} base/frameworks/sumstats/plugins/std-dev.bro)
-rest_target(${psd} base/frameworks/sumstats/plugins/sum.bro)
-rest_target(${psd} base/frameworks/sumstats/plugins/topk.bro)
-rest_target(${psd} base/frameworks/sumstats/plugins/unique.bro)
-rest_target(${psd} base/frameworks/sumstats/plugins/variance.bro)
-rest_target(${psd} base/frameworks/tunnels/main.bro)
-rest_target(${psd} base/misc/find-checksum-offloading.bro)
-rest_target(${psd} base/protocols/conn/contents.bro)
-rest_target(${psd} base/protocols/conn/inactivity.bro)
-rest_target(${psd} base/protocols/conn/main.bro)
-rest_target(${psd} base/protocols/conn/polling.bro)
-rest_target(${psd} base/protocols/dhcp/consts.bro)
-rest_target(${psd} base/protocols/dhcp/main.bro)
-rest_target(${psd} base/protocols/dhcp/utils.bro)
-rest_target(${psd} base/protocols/dnp3/consts.bro)
-rest_target(${psd} base/protocols/dnp3/main.bro)
-rest_target(${psd} base/protocols/dns/consts.bro)
-rest_target(${psd} base/protocols/dns/main.bro)
-rest_target(${psd} base/protocols/ftp/files.bro)
-rest_target(${psd} base/protocols/ftp/gridftp.bro)
-rest_target(${psd} base/protocols/ftp/info.bro)
-rest_target(${psd} base/protocols/ftp/main.bro)
-rest_target(${psd} base/protocols/ftp/utils-commands.bro)
-rest_target(${psd} base/protocols/ftp/utils.bro)
-rest_target(${psd} base/protocols/http/entities.bro)
-rest_target(${psd} base/protocols/http/files.bro)
-rest_target(${psd} base/protocols/http/main.bro)
-rest_target(${psd} base/protocols/http/utils.bro)
-rest_target(${psd} base/protocols/irc/dcc-send.bro)
-rest_target(${psd} base/protocols/irc/files.bro)
-rest_target(${psd} base/protocols/irc/main.bro)
-rest_target(${psd} base/protocols/modbus/consts.bro)
-rest_target(${psd} base/protocols/modbus/main.bro)
-rest_target(${psd} base/protocols/smtp/entities.bro)
-rest_target(${psd} base/protocols/smtp/files.bro)
-rest_target(${psd} base/protocols/smtp/main.bro)
-rest_target(${psd} base/protocols/socks/consts.bro)
-rest_target(${psd} base/protocols/socks/main.bro)
-rest_target(${psd} base/protocols/ssh/main.bro)
-rest_target(${psd} base/protocols/ssl/consts.bro)
-rest_target(${psd} base/protocols/ssl/main.bro)
-rest_target(${psd} base/protocols/ssl/mozilla-ca-list.bro)
-rest_target(${psd} base/protocols/syslog/consts.bro)
-rest_target(${psd} base/protocols/syslog/main.bro)
-rest_target(${psd} base/utils/active-http.bro)
-rest_target(${psd} base/utils/addrs.bro)
-rest_target(${psd} base/utils/conn-ids.bro)
-rest_target(${psd} base/utils/dir.bro)
-rest_target(${psd} base/utils/directions-and-hosts.bro)
-rest_target(${psd} base/utils/exec.bro)
-rest_target(${psd} base/utils/files.bro)
-rest_target(${psd} base/utils/numbers.bro)
-rest_target(${psd} base/utils/paths.bro)
-rest_target(${psd} base/utils/patterns.bro)
-rest_target(${psd} base/utils/queue.bro)
-rest_target(${psd} base/utils/site.bro)
-rest_target(${psd} base/utils/strings.bro)
-rest_target(${psd} base/utils/thresholds.bro)
-rest_target(${psd} base/utils/time.bro)
-rest_target(${psd} base/utils/urls.bro)
-rest_target(${psd} policy/frameworks/communication/listen.bro)
-rest_target(${psd} policy/frameworks/control/controllee.bro)
-rest_target(${psd} policy/frameworks/control/controller.bro)
-rest_target(${psd} policy/frameworks/dpd/detect-protocols.bro)
-rest_target(${psd} policy/frameworks/dpd/packet-segment-logging.bro)
-rest_target(${psd} policy/frameworks/files/detect-MHR.bro)
-rest_target(${psd} policy/frameworks/files/hash-all-files.bro)
-rest_target(${psd} policy/frameworks/intel/do_notice.bro)
-rest_target(${psd} policy/frameworks/intel/seen/conn-established.bro)
-rest_target(${psd} policy/frameworks/intel/seen/dns.bro)
-rest_target(${psd} policy/frameworks/intel/seen/file-hashes.bro)
-rest_target(${psd} policy/frameworks/intel/seen/file-names.bro)
-rest_target(${psd} policy/frameworks/intel/seen/http-headers.bro)
-rest_target(${psd} policy/frameworks/intel/seen/http-url.bro)
-rest_target(${psd} policy/frameworks/intel/seen/smtp-url-extraction.bro)
-rest_target(${psd} policy/frameworks/intel/seen/smtp.bro)
-rest_target(${psd} policy/frameworks/intel/seen/ssl.bro)
-rest_target(${psd} policy/frameworks/intel/seen/where-locations.bro)
-rest_target(${psd} policy/frameworks/packet-filter/shunt.bro)
-rest_target(${psd} policy/frameworks/software/version-changes.bro)
-rest_target(${psd} policy/frameworks/software/vulnerable.bro)
-rest_target(${psd} policy/integration/barnyard2/main.bro)
-rest_target(${psd} policy/integration/barnyard2/types.bro)
-rest_target(${psd} policy/integration/collective-intel/main.bro)
-rest_target(${psd} policy/misc/app-stats/main.bro)
-rest_target(${psd} policy/misc/app-stats/plugins/facebook.bro)
-rest_target(${psd} policy/misc/app-stats/plugins/gmail.bro)
-rest_target(${psd} policy/misc/app-stats/plugins/google.bro)
-rest_target(${psd} policy/misc/app-stats/plugins/netflix.bro)
-rest_target(${psd} policy/misc/app-stats/plugins/pandora.bro)
-rest_target(${psd} policy/misc/app-stats/plugins/youtube.bro)
-rest_target(${psd} policy/misc/capture-loss.bro)
-rest_target(${psd} policy/misc/detect-traceroute/main.bro)
-rest_target(${psd} policy/misc/known-devices.bro)
-rest_target(${psd} policy/misc/load-balancing.bro)
-rest_target(${psd} policy/misc/loaded-scripts.bro)
-rest_target(${psd} policy/misc/profiling.bro)
-rest_target(${psd} policy/misc/scan.bro)
-rest_target(${psd} policy/misc/stats.bro)
-rest_target(${psd} policy/misc/trim-trace-file.bro)
-rest_target(${psd} policy/protocols/conn/known-hosts.bro)
-rest_target(${psd} policy/protocols/conn/known-services.bro)
-rest_target(${psd} policy/protocols/conn/weirds.bro)
-rest_target(${psd} policy/protocols/dhcp/known-devices-and-hostnames.bro)
-rest_target(${psd} policy/protocols/dns/auth-addl.bro)
-rest_target(${psd} policy/protocols/dns/detect-external-names.bro)
-rest_target(${psd} policy/protocols/ftp/detect-bruteforcing.bro)
-rest_target(${psd} policy/protocols/ftp/detect.bro)
-rest_target(${psd} policy/protocols/ftp/software.bro)
-rest_target(${psd} policy/protocols/http/detect-sqli.bro)
-rest_target(${psd} policy/protocols/http/detect-webapps.bro)
-rest_target(${psd} policy/protocols/http/header-names.bro)
-rest_target(${psd} policy/protocols/http/software-browser-plugins.bro)
-rest_target(${psd} policy/protocols/http/software.bro)
-rest_target(${psd} policy/protocols/http/var-extraction-cookies.bro)
-rest_target(${psd} policy/protocols/http/var-extraction-uri.bro)
-rest_target(${psd} policy/protocols/modbus/known-masters-slaves.bro)
-rest_target(${psd} policy/protocols/modbus/track-memmap.bro)
-rest_target(${psd} policy/protocols/smtp/blocklists.bro)
-rest_target(${psd} policy/protocols/smtp/detect-suspicious-orig.bro)
-rest_target(${psd} policy/protocols/smtp/entities-excerpt.bro)
-rest_target(${psd} policy/protocols/smtp/software.bro)
-rest_target(${psd} policy/protocols/ssh/detect-bruteforcing.bro)
-rest_target(${psd} policy/protocols/ssh/geo-data.bro)
-rest_target(${psd} policy/protocols/ssh/interesting-hostnames.bro)
-rest_target(${psd} policy/protocols/ssh/software.bro)
-rest_target(${psd} policy/protocols/ssl/cert-hash.bro)
-rest_target(${psd} policy/protocols/ssl/expiring-certs.bro)
-rest_target(${psd} policy/protocols/ssl/extract-certs-pem.bro)
-rest_target(${psd} policy/protocols/ssl/known-certs.bro)
-rest_target(${psd} policy/protocols/ssl/notary.bro)
-rest_target(${psd} policy/protocols/ssl/validate-certs.bro)
-rest_target(${psd} policy/tuning/defaults/extracted_file_limits.bro)
-rest_target(${psd} policy/tuning/defaults/packet-fragments.bro)
-rest_target(${psd} policy/tuning/defaults/warnings.bro)
-rest_target(${psd} policy/tuning/logs-to-elasticsearch.bro)
-rest_target(${psd} policy/tuning/track-all-assets.bro)
-rest_target(${psd} site/local-manager.bro)
-rest_target(${psd} site/local-proxy.bro)
-rest_target(${psd} site/local-worker.bro)
-rest_target(${psd} site/local.bro)
-rest_target(${psd} test-all-policy.bro)

doc/scripts/README

@@ -1,44 +0,0 @@
-This directory contains scripts and templates that can be used to automate
-the generation of Bro script documentation.  Several build targets are defined
-by CMake and available in the top-level Makefile:
-
-``restdoc``
-
-    This target uses Bro to parse policy scripts in order to generate
-    reStructuredText (reST) documentation from them.  The list of scripts
-    for which to generate reST documentation is defined in the
-    ``CMakeLists.txt`` file in this directory.  Script documentation is
-    rebuild automatically if the policy script from which it is derived
-    or the Bro binary becomes out of date
-
-    The resulting output from this target can be found in the CMake
-    ``build/`` directory inside ``reST`` (a symlink to
-    ``doc/scripts/rest_output``).
-
-``restclean``
-
-    This target removes any reST documentation that has been generated so far.
-
-The ``genDocSourcesList.sh`` script can be run to automatically generate
-``DocSourcesList.cmake``, which is the file CMake uses to define the list
-of documentation targets.  This script should be run after adding new
-Bro script source files, and the changes commited to git.
-
-If a script shouldn't have documentation generated for it, there's also a
-blacklist manifest that can be maintained in the ``genDocSourcesList.sh``
-script.
-
-The blacklist can also be used if you want to define a certain grouping for
-the script's generated docs to belong to (as opposed to the automatic grouping
-the happens for script packages/directories).  To do that, add the
-script's name to the blacklist, then append a ``rest_target()`` to the
-``statictext`` variable where the first argument is the source directory
-containing the policy script to document, the second argument is the file
-name of the policy script, and the third argument is the path/name of a
-pre-created reST document in the ``../`` source directory to which the
-``make doc`` process can append script documentation references.  This
-pre-created reST document should also then be linked to from the TOC tree
-in ``../index.rst``.
-
-See ``example.bro`` for an example of how to document a Bro script such that
-``make doc`` will be able to produce reST/HTML documentation for it.

doc/scripts/example.bro

@@ -1,229 +0,0 @@
-##! This is an example script that demonstrates documentation features.
-##! Comments of the form ``##!`` are for the script summary.  The contents of
-##! these comments are transferred directly into the auto-generated
-##! `reStructuredText <http://docutils.sourceforge.net/rst.html>`_
-##! (reST) document's summary section.
-##!
-##! .. tip:: You can embed directives and roles within ``##``-stylized comments.
-##!
-##! There's also a custom role to reference any identifier node in
-##! the Bro Sphinx domain that's good for "see alsos", e.g.
-##!
-##! See also: :bro:see:`Example::a_var`, :bro:see:`Example::ONE`,
-##! :bro:see:`SSH::Info`
-##!
-##! And a custom directive does the equivalent references:
-##!
-##! .. bro:see:: Example::a_var Example::ONE SSH::Info
-
-# Comments that use a single pound sign (#) are not significant to
-# a script's auto-generated documentation, but ones that use a
-# double pound sign (##) do matter.  In some cases, like record
-# field comments, it's necessary to disambiguate the field with
-# which a comment associates: e.g. "##<" can be used on the same line
-# as a field to signify the comment relates to it and not the
-# following field. "##<" can also be used more generally in any
-# variable declarations to associate with the last-declared identifier.
-#
-# Generally, the auto-doc comments (##) are associated with the
-# next declaration/identifier found in the script, but the doc framework
-# will track/render identifiers regardless of whether they have any
-# of these special comments associated with them.
-#
-# The first sentence contained within the "##"-stylized comments for
-# a given identifier is special in that it will be used as summary
-# text in a table containing all such identifiers and short summaries.
-# If there are no sentences (text terminated with '.'), then everything
-# in the "##"-stylized comments up until the first empty comment
-# is taken as the summary text for a given identifier.
-
-# @load directives are self-documenting
-@load frameworks/software/vulnerable
-
-# "module" statements are self-documenting
-module Example;
-
-# redefinitions of "capture_filters" are self-documenting and
-# go into the generated documentation's "Packet Filter" section
-redef capture_filters += {
-    ["ssl"] = "tcp port 443",
-    ["nntps"] = "tcp port 562",
-};
-
-global example_ports = {
-    443/tcp, 562/tcp,
-} &redef;
-
-
-event bro_init()
-	{
-	Analyzer::register_for_ports(Analyzer::ANALYZER_SSL, example_ports);
-	}
-
-# redefinitions of "Notice::Type" are self-documenting, but
-# more information can be supplied in two different ways
-redef enum Notice::Type += {
-    ## any number of this type of comment
-    ## will document "Notice_One"
-    Notice_One,
-    Notice_Two,  ##< any number of this type of comment
-                 ##< will document "Notice_Two"
-    Notice_Three,
-    Notice_Four,
-};
-
-# Redef'ing the ID enumeration for logging streams is automatically tracked.
-# Comments of the "##" form can be use to further document it, but it's
-# better to do all documentation related to logging in the summary section
-# as is shown above.
-redef enum Log::ID += { LOG };
-
-# Anything declared in the export section will show up in the rendered
-# documentation's "public interface" section
-
-export {
-
-    # these headings don't mean anything special to the
-    # doc framework right now, I'm just including them
-    # to make it more clear to the reader how the doc
-    # framework will actually categorize a script's identifiers
-
-    ############## types ################
-
-    # Note that I'm just mixing the "##" and "##<"
-    # types of comments in the following declarations
-    # as a demonstration.  Normally, it would be good style
-    # to pick one and be consistent.
-
-    ## documentation for "SimpleEnum"
-    ## goes here.
-    type SimpleEnum: enum {
-        ## and more specific info for "ONE"
-        ## can span multiple lines
-        ONE,
-        TWO,  ##< or more info like this for "TWO"
-              ##< can span multiple lines
-        THREE,
-    };
-
-    ## document the "SimpleEnum" redef here
-    redef enum SimpleEnum  += {
-        FOUR, ##< and some documentation for "FOUR"
-        ## also "FIVE" for good measure
-        FIVE
-    };
-
-    ## general documentation for a type "SimpleRecord"
-    ## goes here.
-    type SimpleRecord: record {
-        ## counts something
-        field1: count;
-        field2: bool; ##< toggles something
-    };
-
-    ## document the record extension redef here
-    redef record SimpleRecord += {
-        ## document the extending field here
-        field_ext: string &optional; ##< (or here)
-    };
-
-    ## general documentation for a type "ComplexRecord" goes here
-    type ComplexRecord: record {
-        field1: count;               ##< counts something
-        field2: bool;                ##< toggles something
-        field3: SimpleRecord;
-        msg: string &default="blah"; ##< attributes are self-documenting
-    } &redef;
-
-    ## An example record to be used with a logging stream.
-    type Info: record {
-        ts:       time       &log;
-        uid:      string     &log;
-        status:   count      &log &optional;
-    };
-
-    ############## options ################
-    # right now, I'm just defining an option as
-    # any const with &redef (something that can
-    # change at parse time, but not at run time.
-
-    ## add documentation for "an_option" here
-    const an_option: set[addr, addr, string] &redef;
-
-    # default initialization will be self-documenting
-    const option_with_init = 0.01 secs &redef; ##< More docs can be added here.
-
-    ############## state variables ############
-    # right now, I'm defining this as any global
-    # that's not a function/event.  doesn't matter
-    # if &redef attribute is present
-
-    ## put some documentation for "a_var" here
-    global a_var: bool;
-
-    # attributes are self-documenting
-    global var_with_attr: count &persistent;
-
-    # it's fine if the type is inferred, that information is self-documenting
-    global var_without_explicit_type = "this works";
-
-    ## The first.sentence for the summary text ends here.  And this second
-    ## sentence doesn't show in the short description.
-    global dummy: string;
-
-    ############## functions/events ############
-
-    ## Summarize purpose of "a_function" here.
-    ## Give more details about "a_function" here.
-    ## Separating the documentation of the params/return values with
-    ## empty comments is optional, but improves readability of script.
-    ##
-    ## tag: function arguments can be described
-    ##      like this
-    ## msg: another param
-    ##
-    ## Returns: describe the return type here
-    global a_function: function(tag: string, msg: string): string;
-
-    ## Summarize "an_event" here.
-    ## Give more details about "an_event" here.
-	## Example::an_event should not be confused as a parameter.
-    ## name: describe the argument here
-    global an_event: event(name: string);
-
-    ## This is a declaration of an example event that can be used in
-    ## logging streams and is raised once for each log entry.
-    global log_example: event(rec: Info);
-}
-
-function filter_func(rec: Info): bool
-    {
-    return T;
-    }
-
-# this function is documented in the "private interface" section
-# of generated documentation and any "##"-stylized comments would also
-# be rendered there
-function function_without_proto(tag: string): string
-    {
-    return "blah";
-    }
-
-# this record type is documented in the "private interface" section
-# of generated documentation and any "##"-stylized comments would also
-# be rendered there
-type PrivateRecord: record {
-    field1: bool;
-    field2: count;
-};
-
-event bro_init()
-    {
-    Log::create_stream(Example::LOG, [$columns=Info, $ev=log_example]);
-    Log::add_filter(Example::LOG, [
-        $name="example-filter",
-        $path="example-filter",
-        $pred=filter_func,
-        $exclude=set("ts")
-        ]);
-    }

doc/scripts/example.rst

@@ -1,291 +0,0 @@
-.. Automatically generated.  Do not edit.
-
-example.bro
-===========
-
-:download:`Original Source File <example.bro>`
-
-Overview
---------
-This is an example script that demonstrates how to document.  Comments
-of the form ``##!`` are for the script summary.  The contents of
-these comments are transferred directly into the auto-generated
-`reStructuredText <http://docutils.sourceforge.net/rst.html>`_
-(reST) document's summary section.
-
-.. tip:: You can embed directives and roles within ``##``-stylized comments.
-
-:Imports: :doc:`policy/frameworks/software/vulnerable </scripts/policy/frameworks/software/vulnerable>`
-
-Summary
-~~~~~~~
-Options
-#######
-============================================================================ ======================================
-:bro:id:`Example::an_option`: :bro:type:`set` :bro:attr:`&redef`             add documentation for "an_option" here
-
-:bro:id:`Example::option_with_init`: :bro:type:`interval` :bro:attr:`&redef`
-============================================================================ ======================================
-
-State Variables
-###############
-=========================================================================== =======================================
-:bro:id:`Example::a_var`: :bro:type:`bool`                                  put some documentation for "a_var" here
-
-:bro:id:`Example::var_with_attr`: :bro:type:`count` :bro:attr:`&persistent`
-
-:bro:id:`Example::var_without_explicit_type`: :bro:type:`string`
-=========================================================================== =======================================
-
-Types
-#####
-====================================================== ==========================================================
-:bro:type:`Example::SimpleEnum`: :bro:type:`enum`      documentation for "SimpleEnum"
-                                                       goes here.
-
-:bro:type:`Example::SimpleRecord`: :bro:type:`record`  general documentation for a type "SimpleRecord"
-                                                       goes here.
-
-:bro:type:`Example::ComplexRecord`: :bro:type:`record` general documentation for a type "ComplexRecord" goes here
-
-:bro:type:`Example::Info`: :bro:type:`record`          An example record to be used with a logging stream.
-====================================================== ==========================================================
-
-Events
-######
-================================================= =============================================================
-:bro:id:`Example::an_event`: :bro:type:`event`    Summarize "an_event" here.
-
-:bro:id:`Example::log_example`: :bro:type:`event` This is a declaration of an example event that can be used in
-                                                  logging streams and is raised once for each log entry.
-================================================= =============================================================
-
-Functions
-#########
-=============================================== =======================================
-:bro:id:`Example::a_function`: :bro:type:`func` Summarize purpose of "a_function" here.
-=============================================== =======================================
-
-Redefinitions
-#############
-===================================================== ========================================
-:bro:type:`Log::ID`: :bro:type:`enum`
-
-:bro:type:`Example::SimpleEnum`: :bro:type:`enum`     document the "SimpleEnum" redef here
-
-:bro:type:`Example::SimpleRecord`: :bro:type:`record` document the record extension redef here
-===================================================== ========================================
-
-Namespaces
-~~~~~~~~~~
-.. bro:namespace:: Example
-
-Notices
-~~~~~~~
-:bro:type:`Notice::Type`
-
-   :Type: :bro:type:`enum`
-
-      .. bro:enum:: Example::Notice_One Notice::Type
-
-         any number of this type of comment
-         will document "Notice_One"
-
-      .. bro:enum:: Example::Notice_Two Notice::Type
-
-         any number of this type of comment
-         will document "Notice_Two"
-
-      .. bro:enum:: Example::Notice_Three Notice::Type
-
-      .. bro:enum:: Example::Notice_Four Notice::Type
-
-Public Interface
-----------------
-Options
-~~~~~~~
-.. bro:id:: Example::an_option
-
-   :Type: :bro:type:`set` [:bro:type:`addr`, :bro:type:`addr`, :bro:type:`string`]
-   :Attributes: :bro:attr:`&redef`
-   :Default: ``{}``
-
-   add documentation for "an_option" here
-
-.. bro:id:: Example::option_with_init
-
-   :Type: :bro:type:`interval`
-   :Attributes: :bro:attr:`&redef`
-   :Default: ``10.0 msecs``
-
-State Variables
-~~~~~~~~~~~~~~~
-.. bro:id:: Example::a_var
-
-   :Type: :bro:type:`bool`
-
-   put some documentation for "a_var" here
-
-.. bro:id:: Example::var_with_attr
-
-   :Type: :bro:type:`count`
-   :Attributes: :bro:attr:`&persistent`
-
-.. bro:id:: Example::var_without_explicit_type
-
-   :Type: :bro:type:`string`
-   :Default: ``"this works"``
-
-Types
-~~~~~
-.. bro:type:: Example::SimpleEnum
-
-   :Type: :bro:type:`enum`
-
-      .. bro:enum:: Example::ONE Example::SimpleEnum
-
-         and more specific info for "ONE"
-         can span multiple lines
-
-      .. bro:enum:: Example::TWO Example::SimpleEnum
-
-         or more info like this for "TWO"
-         can span multiple lines
-
-      .. bro:enum:: Example::THREE Example::SimpleEnum
-
-   documentation for "SimpleEnum"
-   goes here.
-
-.. bro:type:: Example::SimpleRecord
-
-   :Type: :bro:type:`record`
-
-      field1: :bro:type:`count`
-         counts something
-
-      field2: :bro:type:`bool`
-         toggles something
-
-   general documentation for a type "SimpleRecord"
-   goes here.
-
-.. bro:type:: Example::ComplexRecord
-
-   :Type: :bro:type:`record`
-
-      field1: :bro:type:`count`
-         counts something
-
-      field2: :bro:type:`bool`
-         toggles something
-
-      field3: :bro:type:`Example::SimpleRecord`
-
-      msg: :bro:type:`string` :bro:attr:`&default` = ``"blah"`` :bro:attr:`&optional`
-         attributes are self-documenting
-
-   general documentation for a type "ComplexRecord" goes here
-
-.. bro:type:: Example::Info
-
-   :Type: :bro:type:`record`
-
-      ts: :bro:type:`time` :bro:attr:`&log`
-
-      uid: :bro:type:`string` :bro:attr:`&log`
-
-      status: :bro:type:`count` :bro:attr:`&log` :bro:attr:`&optional`
-
-   An example record to be used with a logging stream.
-
-Events
-~~~~~~
-.. bro:id:: Example::an_event
-
-   :Type: :bro:type:`event` (name: :bro:type:`string`)
-
-   Summarize "an_event" here.
-   Give more details about "an_event" here.
-   
-   :param name: describe the argument here
-
-.. bro:id:: Example::log_example
-
-   :Type: :bro:type:`event` (rec: :bro:type:`Example::Info`)
-
-   This is a declaration of an example event that can be used in
-   logging streams and is raised once for each log entry.
-
-Functions
-~~~~~~~~~
-.. bro:id:: Example::a_function
-
-   :Type: :bro:type:`function` (tag: :bro:type:`string`, msg: :bro:type:`string`) : :bro:type:`string`
-
-   Summarize purpose of "a_function" here.
-   Give more details about "a_function" here.
-   Separating the documentation of the params/return values with
-   empty comments is optional, but improves readability of script.
-   
-   
-   :param tag: function arguments can be described
-        like this
-   
-   :param msg: another param
-   
-   
-   :returns: describe the return type here
-
-Redefinitions
-~~~~~~~~~~~~~
-:bro:type:`Log::ID`
-
-   :Type: :bro:type:`enum`
-
-      .. bro:enum:: Example::LOG Log::ID
-
-:bro:type:`Example::SimpleEnum`
-
-   :Type: :bro:type:`enum`
-
-      .. bro:enum:: Example::FOUR Example::SimpleEnum
-
-         and some documentation for "FOUR"
-
-      .. bro:enum:: Example::FIVE Example::SimpleEnum
-
-         also "FIVE" for good measure
-
-   document the "SimpleEnum" redef here
-
-:bro:type:`Example::SimpleRecord`
-
-   :Type: :bro:type:`record`
-
-      field_ext: :bro:type:`string` :bro:attr:`&optional`
-         document the extending field here
-         (or here)
-
-   document the record extension redef here
-
-Port Analysis
--------------
-:ref:`More Information <common_port_analysis_doc>`
-
-SSL::
-
-    [ports={
-        443/tcp,
-        562/tcp
-    }]
-
-Packet Filter
--------------
-:ref:`More Information <common_packet_filter_doc>`
-
-Filters added::
-
-    [ssl] = tcp port 443,
-    [nntps] = tcp port 562
-

doc/scripts/genDocSourcesList.sh

@@ -1,86 +0,0 @@
-#!/usr/bin/env bash
-
-# ./genDocSourcesList.sh [output file]
-#
-# Run this script to a generate file that's used to tell CMake about all the
-# possible scripts for which reST documentation can be created.
-#
-# The optional argument can be used to avoid overwriting the file CMake uses
-# by default.
-#
-# Specific scripts can be blacklisted below when e.g. they currently aren't
-# parseable or they just aren't meant to be documented.
-
-export LC_ALL=C # Make sorting stable.
-
-blacklist ()
-    {
-    if [[ "$blacklist" == "" ]]; then
-        blacklist="$1"
-    else
-        blacklist="$blacklist|$1"
-    fi
-    }
-
-# files passed into this function are meant to be temporary workarounds
-# because they're not finished or otherwise can't be loaded for some reason
-tmp_blacklist ()
-    {
-    echo "Warning: temporarily blacklisted files named '$1'" 1>&2
-    blacklist $1
-    }
-
-blacklist __load__.bro
-blacklist test-all.bro
-blacklist all.bro
-blacklist init-default.bro
-blacklist init-bare.bro
-
-statictext="\
-# DO NOT EDIT
-# This file is auto-generated from the "genDocSourcesList.sh" script.
-#
-# This is a list of Bro script sources for which to generate reST documentation.
-# It will be included inline in the CMakeLists.txt found in the same directory
-# in order to create Makefile targets that define how to generate reST from
-# a given Bro script.
-#
-# Note: any path prefix of the script (2nd argument of rest_target macro)
-# will be used to derive what path under scripts/ the generated documentation
-# will be placed.
-
-set(psd \${PROJECT_SOURCE_DIR}/scripts)
-
-rest_target(\${CMAKE_CURRENT_SOURCE_DIR} example.bro internal)
-rest_target(\${psd} base/init-default.bro internal)
-rest_target(\${psd} base/init-bare.bro internal)
-"
-
-if [[ $# -ge 1 ]]; then
-    outfile=$1
-else
-    outfile=DocSourcesList.cmake
-fi
-
-thisdir="$( cd "$( dirname "$0" )" && pwd )"
-sourcedir=${thisdir}/../..
-
-echo "$statictext" > $outfile
-
-bifs=`( cd ${sourcedir}/build/scripts/base && find . -name \*\.bif.bro | sort )`
-
-for file in $bifs
-do
-    f=${file:2}
-    echo "rest_target(\${CMAKE_BINARY_DIR}/scripts base/$f)" >> $outfile
-done
-
-scriptfiles=`( cd ${sourcedir}/scripts && find . -name \*\.bro | sort )`
-
-for file in $scriptfiles
-do
-    f=${file:2}
-    if [[ ! $f =~ $blacklist ]]; then
-        echo "rest_target(\${psd} $f)" >> $outfile
-    fi
-done

doc/scripts/internal.rst

@@ -1,5 +0,0 @@
-.. This is a stub doc to which broxygen appends during the build process
-
-Internal Scripts
-================
-

doc/scripts/scripts.rst

@@ -1,8 +0,0 @@
-.. This is a stub doc to which broxygen appends during the build process
-
-===============
-All Bro Scripts
-===============
-
-.. toctree::
-   :maxdepth: 1

scripts/base/frameworks/reporter/main.bro

@@ -9,7 +9,7 @@
 ##! Note that this framework deals with the handling of internally generated
 ##! reporter messages, for the interface
 ##! into actually creating reporter messages from the scripting layer, use
-##! the built-in functions in :doc:`/scripts/base/bif/reporter.bif`.
+##! the built-in functions in :doc:`/scripts/base/bif/reporter.bif.bro`.
 
 module Reporter;
 

scripts/base/init-bare.bro

@@ -399,7 +399,7 @@ type NetStats: record {
 	pkts_dropped: count &default=0;	##< Packets reported dropped by the system.
 	## Packets seen on the link. Note that this may differ
 	## from *pkts_recvd* because of a potential capture_filter. See
-	## :doc:`/scripts/base/frameworks/packet-filter/main`. Depending on the
+	## :doc:`/scripts/base/frameworks/packet-filter/main.bro`. Depending on the
 	## packet capture system, this value may not be available and will then
 	## be always set to zero.
 	pkts_link:    count &default=0;
@@ -2768,13 +2768,13 @@ const log_max_size = 0.0 &redef;
 const log_encryption_key = "<undefined>" &redef;
 
 ## Write profiling info into this file in regular intervals. The easiest way to
-## activate profiling is loading :doc:`/scripts/policy/misc/profiling`.
+## activate profiling is loading :doc:`/scripts/policy/misc/profiling.bro`.
 ##
 ## .. bro:see:: profiling_interval expensive_profiling_multiple segment_profiling
 global profiling_file: file &redef;
 
 ## Update interval for profiling (0 disables).  The easiest way to activate
-## profiling is loading  :doc:`/scripts/policy/misc/profiling`.
+## profiling is loading  :doc:`/scripts/policy/misc/profiling.bro`.
 ##
 ## .. bro:see:: profiling_file expensive_profiling_multiple segment_profiling
 const profiling_interval = 0 secs &redef;

scripts/base/protocols/dhcp/main.bro

@@ -4,7 +4,7 @@
 ##!
 ##! If you'd like to track known DHCP devices and to log the hostname
 ##! supplied by the client, see
-##! :doc:`/scripts/policy/protocols/dhcp/known-devices-and-hostnames`.
+##! :doc:`/scripts/policy/protocols/dhcp/known-devices-and-hostnames.bro`.
 
 @load ./utils.bro
 

scripts/broxygen/README

@@ -0,0 +1,4 @@
+This package is loaded during the process which automatically generates
+reference documentation for all Bro scripts (i.e. "Broxygen").  Its only
+purpose is to provide an easy way to load all known Bro scripts plus any
+extra scripts needed or used by the documentation process.

scripts/broxygen/__load__.bro

@@ -0,0 +1,14 @@
+@load test-all-policy.bro
+
+# Scripts which are commented out in test-all-policy.bro.
+@load protocols/ssl/notary.bro
+@load frameworks/communication/listen.bro
+@load frameworks/control/controllee.bro
+@load frameworks/control/controller.bro
+
+@load ./example.bro
+
+event bro_init()
+	{
+	terminate();
+	}

scripts/broxygen/example.bro

@@ -0,0 +1,194 @@
+##! This is an example script that demonstrates Broxygen-style
+##! documentation.  It generally will make most sense when viewing
+##! the script's raw source code and comparing to the HTML-rendered
+##! version.
+##!
+##! Comments in the from ``##!`` are meant to summarize the script's
+##! purpose.  They are transferred directly in to the generated
+##! `reStructuredText <http://docutils.sourceforge.net/rst.html>`_
+##! (reST) document associated with the script.
+##!
+##! .. tip:: You can embed directives and roles within ``##``-stylized comments.
+##!
+##! There's also a custom role to reference any identifier node in
+##! the Bro Sphinx domain that's good for "see alsos", e.g.
+##!
+##! See also: :bro:see:`BroxygenExample::a_var`,
+##! :bro:see:`BroxygenExample::ONE`, :bro:see:`SSH::Info`
+##!
+##! And a custom directive does the equivalent references:
+##!
+##! .. bro:see:: BroxygenExample::a_var BroxygenExample::ONE SSH::Info
+
+# Comments that use a single pound sign (#) are not significant to
+# a script's auto-generated documentation, but ones that use a
+# double pound sign (##) do matter.  In some cases, like record
+# field comments, it's necessary to disambiguate the field with
+# which a comment associates: e.g. "##<" can be used on the same line
+# as a field to signify the comment relates to it and not the
+# following field. "##<" can also be used more generally in any
+# variable declarations to associate with the last-declared identifier.
+#
+# Generally, the auto-doc comments (##) are associated with the
+# next declaration/identifier found in the script, but Broxygen
+# will track/render identifiers regardless of whether they have any
+# of these special comments associated with them.
+#
+# The first sentence contained within the "##"-stylized comments for
+# a given identifier is special in that it will be used as summary
+# text in a table containing all such identifiers and short summaries.
+# If there are no sentences (text terminated with '.'), then everything
+# in the "##"-stylized comments up until the first empty comment
+# is taken as the summary text for a given identifier.
+
+# @load directives are self-documenting, don't use any ``##`` style
+# comments with them.
+@load base/frameworks/notice
+@load base/protocols/http
+@load frameworks/software/vulnerable
+
+# "module" statements are self-documenting, don't use any ``##`` style
+# comments with them.
+module BroxygenExample;
+
+# Redefinitions of "Notice::Type" are self-documenting, but
+# more information can be supplied in two different ways.
+redef enum Notice::Type += {
+	## Any number of this type of comment
+	## will document "Broxygen_One".
+	Broxygen_One,
+	Broxygen_Two,  ##< Any number of this type of comment
+	               ##< will document "BROXYGEN_TWO".
+	Broxygen_Three,
+	## Omitting comments is fine, and so is mixing ``##`` and ``##<``, but
+	Broxygen_Four,  ##< it's probably best to use only one style consistently.
+};
+
+# All redefs are automatically tracked.  Comments of the "##" form can be use
+# to further document it, but in some cases, like here, they wouldn't be
+# ading any interesting information that's not implicit.
+redef enum Log::ID += { LOG };
+
+# Only identifiers declared in an export section will show up in generated docs.
+
+export {
+
+	## Documentation for the "SimpleEnum" type goes here.
+	## It can span multiple lines.
+	type SimpleEnum: enum {
+		## Documentation for particular enum values is added like this.
+		## And can also span multiple lines.
+		ONE,
+		TWO,  ##< Or this style is valid to document the preceding enum value.
+		THREE,
+	};
+
+	## Document the "SimpleEnum" redef here with any special info regarding
+	## the *redef* itself.
+	redef enum SimpleEnum  += {
+		FOUR, ##< And some documentation for "FOUR".
+		## Also "FIVE".
+		FIVE
+	};
+
+	## General documentation for a type "SimpleRecord" goes here.
+	## The way fields can be documented is similar to what's already seen
+	## for enums.
+	type SimpleRecord: record {
+		## Counts something.
+		field1: count;
+		field2: bool; ##< Toggles something.
+	};
+
+	## Document the record extension *redef* itself here.
+	redef record SimpleRecord += {
+		## Document the extending field like this.
+		field_ext: string &optional; ##< Or here, like this.
+	};
+
+	## General documentation for a type "ComplexRecord" goes here.
+	type ComplexRecord: record {
+		field1: count;               ##< Counts something.
+		field2: bool;                ##< Toggles something.
+		field3: SimpleRecord;        ##< Broxygen automatically tracks types
+		                             ##< and cross-references are automatically
+		                             ##< inserted in to generated docs.
+		msg: string &default="blah"; ##< Attributes are self-documenting.
+	} &redef;
+
+	## An example record to be used with a logging stream.
+	## Nothing special about it.  If another script redefs this type
+	## to add fields, the generated documentation will show all original
+	## fields plus the extensions and the scripts which contributed to it
+	## (provided they are also @load'ed).
+	type Info: record {
+		ts:       time       &log;
+		uid:      string     &log;
+		status:   count      &log &optional;
+	};
+
+	## Add documentation for "an_option" here.
+	## The type/attribute information is all generated automatically.
+	const an_option: set[addr, addr, string] &redef;
+
+	## Default initialization will be generated automatically.
+	const option_with_init = 0.01 secs &redef; ##< More docs can be added here.
+
+	## Put some documentation for "a_var" here.  Any global/non-const that
+	## isn't a function/event/hook is classified as a "state variable"
+	## in the generated docs.
+	global a_var: bool;
+
+	## Types are inferred, that information is self-documenting.
+	global var_without_explicit_type = "this works";
+
+	## The first sentence for a particular identifier's summary text ends here.
+	## And this second sentence doesn't show in the short description provided
+	## by the table of all identifiers declared by this script.
+	global summary_test: string;
+
+    ## Summarize purpose of "a_function" here.
+    ## Give more details about "a_function" here.
+    ## Separating the documentation of the params/return values with
+    ## empty comments is optional, but improves readability of script.
+    ##
+    ## tag: Function arguments can be described
+    ##      like this.
+	##
+    ## msg: Another param.
+    ##
+    ## Returns: Describe the return type here.
+    global a_function: function(tag: string, msg: string): string;
+
+    ## Summarize "an_event" here.
+    ## Give more details about "an_event" here.
+	##
+	## BroxygenExample::a_function should not be confused as a parameter
+	## in the generated docs, but it also doesn't generate a cross-reference
+	## link.  Use the see role instead: :bro:see:`BroxygenExample::a_function`.
+	##
+    ## name: Describe the argument here.
+    global an_event: event(name: string);
+}
+
+# This function isn't exported, so it won't appear anywhere in the generated
+# documentation.  So using ``##``-style comments is pointless here.
+function function_without_proto(tag: string): string
+    {
+    return "blah";
+    }
+
+# Same thing goes for types -- it's not exported, so it's considered
+# private to this script and comments are only interesting to a person
+# who is already reading the raw source for the script (so don't use
+# ``##`` comments here.
+type PrivateRecord: record {
+    field1: bool;
+    field2: count;
+};
+
+# Event handlers are also an implementation detail of a script, so they
+# don't show up anywhere in the generated documentation.
+event bro_init()
+    {
+    }

scripts/policy/misc/known-devices.bro

@@ -7,7 +7,7 @@
 ##!
 ##!     This script will not generate any logs on its own, it needs to be
 ##!     supplied with information from elsewhere, such as
-##!     :doc:`/scripts/policy/protocols/dhcp/known-devices-and-hostnames`.
+##!     :doc:`/scripts/policy/protocols/dhcp/known-devices-and-hostnames.bro`.
 
 module Known;
 

scripts/policy/misc/scan.bro

@@ -147,11 +147,6 @@ function is_reverse_failed_conn(c: connection): bool
 	return F;
 	}
 
-## Generated for an unsuccessful connection attempt. This
-## event is raised when an originator unsuccessfully attempted
-## to establish a connection. "Unsuccessful" is defined as at least
-## tcp_attempt_delay seconds having elapsed since the originator first sent a
-## connection establishment packet to the destination without seeing a reply.
 event connection_attempt(c: connection)
 	{
 	local is_reverse_scan = F;
@@ -161,9 +156,6 @@ event connection_attempt(c: connection)
 	add_sumstats(c$id, is_reverse_scan);
 	}
 
-## Generated for a rejected TCP connection. This event is raised when an
-## originator attempted to setup a TCP connection but the responder replied with
-## a RST packet denying it.
 event connection_rejected(c: connection)
 	{
 	local is_reverse_scan = F;
@@ -173,9 +165,6 @@ event connection_rejected(c: connection)
 	add_sumstats(c$id, is_reverse_scan);
 	}
 
-## Generated when an endpoint aborted a TCP connection. The event is raised when
-## one endpoint of an *established* TCP connection aborted by sending a RST
-## packet.
 event connection_reset(c: connection)
 	{
 	if ( is_failed_conn(c) )
@@ -184,7 +173,6 @@ event connection_reset(c: connection)
 		add_sumstats(c$id, T);
 	}
 
-## Generated for each still-open connection when Bro terminates.
 event connection_pending(c: connection)
 	{
 	if ( is_failed_conn(c) )

scripts/policy/misc/stats.bro

@@ -1,5 +1,5 @@
 ##! Log memory/packet/lag statistics.  Differs from
-##! :doc:`/scripts/policy/misc/profiling` in that this
+##! :doc:`/scripts/policy/misc/profiling.bro` in that this
 ##! is lighter-weight (much less info, and less load to generate).
 
 @load base/frameworks/notice

src/BroDoc.cc

@@ -1,671 +0,0 @@
-#include <cstdio>
-#include <cstdarg>
-#include <string>
-#include <list>
-#include <algorithm>
-#include <libgen.h>
-
-#include "BroDoc.h"
-#include "BroDocObj.h"
-#include "util.h"
-#include "plugin/Manager.h"
-#include "analyzer/Manager.h"
-#include "analyzer/Component.h"
-#include "file_analysis/Manager.h"
-
-BroDoc::BroDoc(const std::string& rel, const std::string& abs)
-	{
-	size_t f_pos = abs.find_last_of('/');
-
-	if ( std::string::npos == f_pos )
-		source_filename = abs;
-	else
-		source_filename = abs.substr(f_pos + 1);
-
-	if ( rel[0] == '/' || rel[0] == '.' )
-		{
-		// The Bro script isn't being loaded via BROPATH, so just use basename
-		// as the document title.
-		doc_title = source_filename;
-		}
-	else
-		{
-		// Keep the relative directory as part of the document title.
-		if ( rel.size() == 0 || rel[rel.size() - 1] == '/' )
-			doc_title = rel + source_filename;
-		else
-			doc_title = rel + "/" + source_filename;
-		}
-
-	downloadable_filename = source_filename;
-
-#if 0
-	size_t ext_pos = downloadable_filename.find(".bif.bro");
-	if ( std::string::npos != ext_pos )
-		downloadable_filename.erase(ext_pos + 4);
-#endif
-
-	reST_filename = doc_title;
-	size_t ext_pos = reST_filename.find(".bro");
-
-	if ( std::string::npos == ext_pos )
-		reST_filename += ".rst";
-	else
-		reST_filename.replace(ext_pos, 4, ".rst");
-
-	reST_filename = doc_title.substr(0, ext_pos);
-	reST_filename += ".rst";
-
-	// Instead of re-creating the directory hierarchy based on related
-	// loads, just replace the directory separatories such that the reST
-	// output will all be placed in a flat directory (the working dir).
-	std::for_each(reST_filename.begin(), reST_filename.end(), replace_slash());
-
-	reST_file = fopen(reST_filename.c_str(), "w");
-
-	if ( ! reST_file )
-		fprintf(stderr, "Failed to open %s\n", reST_filename.c_str());
-
-#ifdef DOCDEBUG
-	fprintf(stdout, "Documenting absolute source: %s\n", abs.c_str());
-	fprintf(stdout, "\trelative dir: %s\n", rel.c_str());
-	fprintf(stdout, "\tdoc title: %s\n", doc_title.c_str());
-	fprintf(stdout, "\tbro file: %s\n", source_filename.c_str());
-	fprintf(stdout, "\trst file: %s\n", reST_filename.c_str());
-#endif
-	}
-
-BroDoc::~BroDoc()
-	{
-	if ( reST_file && fclose( reST_file ) )
-		fprintf(stderr, "Failed to close %s\n", reST_filename.c_str());
-
-	FreeBroDocObjPtrList(all);
-	}
-
-void BroDoc::AddImport(const std::string& s)
-	{
-	std::string lname(s);
-	// First strip any .bro extension.
-	size_t ext_pos = lname.find(".bro");
-	if ( ext_pos != std::string::npos )
-		lname = lname.substr(0, ext_pos);
-
-	const char* full_filename = NULL;
-	const char* subpath = NULL;
-
-	FILE* f = search_for_file(lname.c_str(), "bro", &full_filename, true,
-	                          &subpath);
-
-	if ( f && full_filename && subpath )
-		{
-		char* tmp = copy_string(full_filename);
-		char* filename = basename(tmp);
-		extern char* PACKAGE_LOADER;
-
-		if ( streq(filename, PACKAGE_LOADER) )
-			{
-			// link to the package's index
-			string pkg(subpath);
-			pkg += "/index";
-			imports.push_back(pkg);
-			}
-		else
-			{
-			if ( subpath[0] == '/' || subpath[0] == '.' )
-				{
-				// it's not a subpath of scripts/, so just add the name of it
-				// as it's given in the @load directive
-				imports.push_back(lname);
-				}
-			else
-				{
-				// combine the base file name of script in the @load directive
-				// with the subpath of BROPATH's scripts/ directory
-				string fname(subpath);
-				char* othertmp = copy_string(lname.c_str());
-				fname.append("/").append(basename(othertmp));
-				imports.push_back(fname);
-				delete [] othertmp;
-				}
-			}
-
-		delete [] tmp;
-		}
-
-	else
-		fprintf(stderr, "Failed to document '@load %s' in file: %s\n",
-		        s.c_str(), reST_filename.c_str());
-
-	if ( f )
-		fclose(f);
-
-	delete [] full_filename;
-	delete [] subpath;
-	}
-
-void BroDoc::SetPacketFilter(const std::string& s)
-	{
-	packet_filter = s;
-	size_t pos1 = s.find("{\n");
-	size_t pos2 = s.find("}");
-
-	if ( pos1 != std::string::npos && pos2 != std::string::npos )
-		packet_filter = s.substr(pos1 + 2, pos2 - 2);
-
-	bool has_non_whitespace = false;
-
-	for ( std::string::const_iterator it = packet_filter.begin();
-		it != packet_filter.end(); ++it )
-		{
-		if ( *it != ' ' && *it != '\t' && *it != '\n' && *it != '\r' )
-			{
-			has_non_whitespace = true;
-			break;
-			}
-		}
-
-	if ( ! has_non_whitespace )
-		packet_filter.clear();
-	}
-
-void BroDoc::WriteDocFile() const
-	{
-	WriteToDoc(reST_file, ".. Automatically generated.  Do not edit.\n\n");
-
-	WriteToDoc(reST_file, ":tocdepth: 3\n\n");
-
-	WriteSectionHeading(reST_file, doc_title.c_str(), '=');
-
-	WriteStringList(reST_file, ".. bro:namespace:: %s\n", modules);
-
-	WriteToDoc(reST_file, "\n");
-
-	// WriteSectionHeading(reST_file, "Overview", '-');
-	WriteStringList(reST_file, "%s\n", summary);
-
-	WriteToDoc(reST_file, "\n");
-
-	if ( ! modules.empty() )
-		{
-		WriteToDoc(reST_file, ":Namespace%s: ", (modules.size() > 1 ? "s" : ""));
-		// WriteStringList(reST_file, ":bro:namespace:`%s`", modules);
-		WriteStringList(reST_file, "``%s``, ", "``%s``", modules);
-		WriteToDoc(reST_file, "\n");
-		}
-
-	if ( ! imports.empty() )
-		{
-		WriteToDoc(reST_file, ":Imports: ");
-		std::list<std::string>::const_iterator it;
-		for ( it = imports.begin(); it != imports.end(); ++it )
-			{
-			if ( it != imports.begin() )
-				WriteToDoc(reST_file, ", ");
-
-			string pretty(*it);
-			size_t pos = pretty.find("/index");
-			if ( pos != std::string::npos && pos + 6 == pretty.size() )
-				pretty = pretty.substr(0, pos);
-			WriteToDoc(reST_file, ":doc:`%s </scripts/%s>`", pretty.c_str(), it->c_str());
-			}
-		WriteToDoc(reST_file, "\n");
-		}
-
-	WriteToDoc(reST_file, ":Source File: :download:`%s`\n",
-		downloadable_filename.c_str());
-
-	WriteToDoc(reST_file, "\n");
-
-	WriteInterface("Summary", '~', '#', true, true);
-
-	if ( ! notices.empty() )
-		WriteBroDocObjList(reST_file, notices, "Notices", '#');
-
-	if ( port_analysis.size() || packet_filter.size() )
-		WriteSectionHeading(reST_file, "Configuration Changes", '#');
-
-	if ( ! port_analysis.empty() )
-		{
-		WriteSectionHeading(reST_file, "Port Analysis", '^');
-		WriteToDoc(reST_file, "Loading this script makes the following changes to "
-		           ":bro:see:`dpd_config`.\n\n");
-		WriteStringList(reST_file, "%s, ", "%s", port_analysis);
-		}
-
-	if ( ! packet_filter.empty() )
-		{
-		WriteSectionHeading(reST_file, "Packet Filter", '^');
-		WriteToDoc(reST_file, "Loading this script makes the following changes to "
-		           ":bro:see:`capture_filters`.\n\n");
-		WriteToDoc(reST_file, "Filters added::\n\n");
-		WriteToDoc(reST_file, "%s\n", packet_filter.c_str());
-		}
-
-	WriteInterface("Detailed Interface", '~', '#', true, false);
-
-#if 0   // Disabled for now.
-	BroDocObjList::const_iterator it;
-	bool hasPrivateIdentifiers = false;
-
-	for ( it = all.begin(); it != all.end(); ++it )
-		{
-		if ( ! IsPublicAPI(*it) )
-			{
-			hasPrivateIdentifiers = true;
-			break;
-			}
-		}
-
-	if ( hasPrivateIdentifiers )
-		WriteInterface("Private Interface", '~', '#', false, false);
-#endif
-	}
-
-void BroDoc::WriteInterface(const char* heading, char underline,
-			char sub, bool isPublic, bool isShort) const
-	{
-	WriteSectionHeading(reST_file, heading, underline);
-	WriteBroDocObjList(reST_file, options, isPublic, "Options", sub, isShort);
-	WriteBroDocObjList(reST_file, constants, isPublic, "Constants", sub, isShort);
-	WriteBroDocObjList(reST_file, state_vars, isPublic, "State Variables", sub, isShort);
-	WriteBroDocObjList(reST_file, types, isPublic, "Types", sub, isShort);
-	WriteBroDocObjList(reST_file, events, isPublic, "Events", sub, isShort);
-	WriteBroDocObjList(reST_file, hooks, isPublic, "Hooks", sub, isShort);
-	WriteBroDocObjList(reST_file, functions, isPublic, "Functions", sub, isShort);
-	WriteBroDocObjList(reST_file, redefs, isPublic, "Redefinitions", sub, isShort);
-	}
-
-void BroDoc::WriteStringList(FILE* f, const char* format, const char* last_format,
-			const std::list<std::string>& l)
-	{
-	if ( l.empty() )
-		{
-		WriteToDoc(f, "\n");
-		return;
-		}
-
-	std::list<std::string>::const_iterator it;
-	std::list<std::string>::const_iterator last = l.end();
-	last--;
-
-	for ( it = l.begin(); it != last; ++it )
-		WriteToDoc(f, format, it->c_str());
-
-	WriteToDoc(f, last_format, last->c_str());
-	}
-
-void BroDoc::WriteBroDocObjTable(FILE* f, const BroDocObjList& l)
-	{
-	int max_id_col = 0;
-	int max_com_col = 0;
-	BroDocObjList::const_iterator it;
-
-	for ( it = l.begin(); it != l.end(); ++it )
-		{
-		int c = (*it)->ColumnSize();
-
-		if ( c > max_id_col )
-			max_id_col = c;
-
-		c = (*it)->LongestShortDescLen();
-
-		if ( c > max_com_col )
-			max_com_col = c;
-		}
-
-	// Start table.
-	WriteRepeatedChar(f, '=', max_id_col);
-	WriteToDoc(f, " ");
-
-	if ( max_com_col == 0 )
-		WriteToDoc(f, "=");
-	else
-		WriteRepeatedChar(f, '=', max_com_col);
-
-	WriteToDoc(f, "\n");
-
-	for ( it = l.begin(); it != l.end(); ++it )
-		{
-		if ( it != l.begin() )
-			WriteToDoc(f, "\n\n");
-		(*it)->WriteReSTCompact(f, max_id_col);
-		}
-
-	// End table.
-	WriteToDoc(f, "\n");
-	WriteRepeatedChar(f, '=', max_id_col);
-	WriteToDoc(f, " ");
-
-	if ( max_com_col == 0 )
-		WriteToDoc(f, "=");
-	else
-		WriteRepeatedChar(f, '=', max_com_col);
-
-	WriteToDoc(f, "\n\n");
-	}
-
-void BroDoc::WriteBroDocObjList(FILE* f, const BroDocObjList& l, bool wantPublic,
-			const char* heading, char underline, bool isShort)
-	{
-	if ( l.empty() )
-		return;
-
-	BroDocObjList::const_iterator it;
-	bool (*f_ptr)(const BroDocObj* o) = 0;
-
-	if ( wantPublic )
-		f_ptr = IsPublicAPI;
-	else
-		f_ptr = IsPrivateAPI;
-
-	it = std::find_if(l.begin(), l.end(), f_ptr);
-
-	if ( it == l.end() )
-		return;
-
-	WriteSectionHeading(f, heading, underline);
-
-	BroDocObjList filtered_list;
-
-	while ( it != l.end() )
-		{
-		filtered_list.push_back(*it);
-		it = find_if(++it, l.end(), f_ptr);
-		}
-
-	if ( isShort )
-		WriteBroDocObjTable(f, filtered_list);
-	else
-		WriteBroDocObjList(f, filtered_list);
-	}
-
-void BroDoc::WriteBroDocObjList(FILE* f, const BroDocObjMap& m, bool wantPublic,
-			const char* heading, char underline, bool isShort)
-	{
-	BroDocObjMap::const_iterator it;
-	BroDocObjList l;
-
-	for ( it = m.begin(); it != m.end(); ++it )
-		l.push_back(it->second);
-
-	WriteBroDocObjList(f, l, wantPublic, heading, underline, isShort);
-	}
-
-void BroDoc::WriteBroDocObjList(FILE* f, const BroDocObjList& l, const char* heading,
-			char underline)
-	{
-	WriteSectionHeading(f, heading, underline);
-	WriteBroDocObjList(f, l);
-	}
-
-void BroDoc::WriteBroDocObjList(FILE* f, const BroDocObjList& l)
-	{
-	for ( BroDocObjList::const_iterator it = l.begin(); it != l.end(); ++it )
-		(*it)->WriteReST(f);
-	}
-
-void BroDoc::WriteBroDocObjList(FILE* f, const BroDocObjMap& m, const char* heading,
-			char underline)
-	{
-	BroDocObjMap::const_iterator it;
-	BroDocObjList l;
-
-	for ( it = m.begin(); it != m.end(); ++it )
-		l.push_back(it->second);
-
-	WriteBroDocObjList(f, l, heading, underline);
-	}
-
-void BroDoc::WriteToDoc(FILE* f, const char* format, ...)
-	{
-	va_list argp;
-	va_start(argp, format);
-	vfprintf(f, format, argp);
-	va_end(argp);
-	}
-
-void BroDoc::WriteSectionHeading(FILE* f, const char* heading, char underline)
-	{
-	WriteToDoc(f, "%s\n", heading);
-	WriteRepeatedChar(f, underline, strlen(heading));
-	WriteToDoc(f, "\n");
-	}
-
-void BroDoc::WriteRepeatedChar(FILE* f, char c, size_t n)
-	{
-	for ( size_t i = 0; i < n; ++i )
-		WriteToDoc(f, "%c", c);
-	}
-
-void BroDoc::FreeBroDocObjPtrList(BroDocObjList& l)
-	{
-	for ( BroDocObjList::const_iterator it = l.begin(); it != l.end(); ++it )
-		delete *it;
-
-	l.clear();
-	}
-
-void BroDoc::AddFunction(BroDocObj* o)
-	{
-	BroDocObjMap::const_iterator it = functions.find(o->Name());
-	if ( it == functions.end() )
-		{
-		functions[o->Name()] = o;
-		all.push_back(o);
-		}
-	else
-		functions[o->Name()]->Combine(o);
-	}
-
-static void WritePluginSectionHeading(FILE* f, const plugin::Plugin* p)
-	{
-	string name = p->Name();
-
-	fprintf(f, "%s\n", name.c_str());
-	for ( size_t i = 0; i < name.size(); ++i )
-		fprintf(f, "-");
-	fprintf(f, "\n\n");
-
-	fprintf(f, "%s\n\n", p->Description());
-	}
-
-static void WriteAnalyzerComponent(FILE* f, const analyzer::Component* c)
-	{
-	EnumType* atag = analyzer_mgr->GetTagEnumType();
-	string tag = fmt("ANALYZER_%s", c->CanonicalName());
-
-	if ( atag->Lookup("Analyzer", tag.c_str()) < 0 )
-		reporter->InternalError("missing analyzer tag for %s", tag.c_str());
-
-	fprintf(f, ":bro:enum:`Analyzer::%s`\n\n", tag.c_str());
-	}
-
-static void WriteAnalyzerComponent(FILE* f, const file_analysis::Component* c)
-	{
-	EnumType* atag = file_mgr->GetTagEnumType();
-	string tag = fmt("ANALYZER_%s", c->CanonicalName());
-
-	if ( atag->Lookup("Files", tag.c_str()) < 0 )
-		reporter->InternalError("missing analyzer tag for %s", tag.c_str());
-
-	fprintf(f, ":bro:enum:`Files::%s`\n\n", tag.c_str());
-	}
-
-static void WritePluginComponents(FILE* f, const plugin::Plugin* p)
-	{
-	plugin::Plugin::component_list components = p->Components();
-	plugin::Plugin::component_list::const_iterator it;
-
-	fprintf(f, "Components\n");
-	fprintf(f, "++++++++++\n\n");
-
-	for ( it = components.begin(); it != components.end(); ++it )
-		{
-		switch ( (*it)->Type() ) {
-		case plugin::component::ANALYZER:
-			{
-			const analyzer::Component* c =
-			        dynamic_cast<const analyzer::Component*>(*it);
-
-			if ( c )
-				WriteAnalyzerComponent(f, c);
-			else
-				reporter->InternalError("component type mismatch");
-			}
-			break;
-
-		case plugin::component::FILE_ANALYZER:
-			{
-			const file_analysis::Component* c =
-			        dynamic_cast<const file_analysis::Component*>(*it);
-
-			if ( c )
-				WriteAnalyzerComponent(f, c);
-			else
-				reporter->InternalError("component type mismatch");
-			}
-			break;
-
-		case plugin::component::READER:
-			reporter->InternalError("docs for READER component unimplemented");
-
-		case plugin::component::WRITER:
-			reporter->InternalError("docs for WRITER component unimplemented");
-
-		default:
-			reporter->InternalError("docs for unknown component unimplemented");
-		}
-		}
-	}
-
-static void WritePluginBifItems(FILE* f, const plugin::Plugin* p,
-                                plugin::BifItem::Type t, const string& heading)
-	{
-	plugin::Plugin::bif_item_list bifitems = p->BifItems();
-	plugin::Plugin::bif_item_list::iterator it = bifitems.begin();
-
-	while ( it != bifitems.end() )
-		{
-		if ( it->GetType() != t )
-			it = bifitems.erase(it);
-		else
-			++it;
-		}
-
-	if ( bifitems.empty() )
-		return;
-
-	fprintf(f, "%s\n", heading.c_str());
-	for ( size_t i = 0; i < heading.size(); ++i )
-		fprintf(f, "+");
-	fprintf(f, "\n\n");
-
-	for ( it = bifitems.begin(); it != bifitems.end(); ++it )
-		{
-		BroDocObj* o = doc_ids[it->GetID()];
-
-		if ( o )
-			o->WriteReST(f);
-		else
-			reporter->Warning("No docs for ID: %s\n", it->GetID());
-		}
-	}
-
-static void WriteAnalyzerTagDefn(FILE* f, EnumType* e, const string& module)
-	{
-	string tag_id= module + "::Tag";
-	e = new CommentedEnumType(e);
-	e->SetTypeID(copy_string(tag_id.c_str()));
-
-	ID* dummy_id = new ID(tag_id.c_str(), SCOPE_GLOBAL, true);
-	dummy_id->SetType(e);
-	dummy_id->MakeType();
-
-	list<string>* r = new list<string>();
-	r->push_back("Unique identifiers for analyzers.");
-
-	BroDocObj bdo(dummy_id, r, true);
-
-	bdo.WriteReST(f);
-	}
-
-static bool ComponentsMatch(const plugin::Plugin* p, plugin::component::Type t,
-                            bool match_empty = false)
-	{
-	plugin::Plugin::component_list components = p->Components();
-	plugin::Plugin::component_list::const_iterator it;
-
-	if ( components.empty() )
-		return match_empty;
-
-	for ( it = components.begin(); it != components.end(); ++it )
-		if ( (*it)->Type() != t )
-			return false;
-
-	return true;
-	}
-
-void CreateProtoAnalyzerDoc(const char* filename)
-	{
-	FILE* f = fopen(filename, "w");
-
-	fprintf(f, "Protocol Analyzers\n");
-	fprintf(f, "==================\n\n\n");
-	fprintf(f, ".. contents::\n");
-	fprintf(f, "     :depth: 1\n\n");
-
-	WriteAnalyzerTagDefn(f, analyzer_mgr->GetTagEnumType(), "Analyzer");
-
-	plugin::Manager::plugin_list plugins = plugin_mgr->Plugins();
-	plugin::Manager::plugin_list::const_iterator it;
-
-	for ( it = plugins.begin(); it != plugins.end(); ++it )
-		{
-		if ( ! ComponentsMatch(*it, plugin::component::ANALYZER, true) )
-			continue;
-
-		WritePluginSectionHeading(f, *it);
-		WritePluginComponents(f, *it);
-		WritePluginBifItems(f, *it, plugin::BifItem::CONSTANT,
-		                    "Options/Constants");
-		WritePluginBifItems(f, *it, plugin::BifItem::GLOBAL, "Globals");
-		WritePluginBifItems(f, *it, plugin::BifItem::TYPE, "Types");
-		WritePluginBifItems(f, *it, plugin::BifItem::EVENT, "Events");
-		WritePluginBifItems(f, *it, plugin::BifItem::FUNCTION, "Functions");
-		}
-
-	fclose(f);
-	}
-
-void CreateFileAnalyzerDoc(const char* filename)
-	{
-	FILE* f = fopen(filename, "w");
-
-	fprintf(f, "File Analyzers\n");
-	fprintf(f, "==============\n\n");
-	fprintf(f, ".. contents::\n");
-	fprintf(f, "     :depth: 1\n\n");
-
-	WriteAnalyzerTagDefn(f, file_mgr->GetTagEnumType(), "Files");
-
-	plugin::Manager::plugin_list plugins = plugin_mgr->Plugins();
-	plugin::Manager::plugin_list::const_iterator it;
-
-	for ( it = plugins.begin(); it != plugins.end(); ++it )
-		{
-		if ( ! ComponentsMatch(*it, plugin::component::FILE_ANALYZER) )
-			continue;
-
-		WritePluginSectionHeading(f, *it);
-		WritePluginComponents(f, *it);
-		WritePluginBifItems(f, *it, plugin::BifItem::CONSTANT,
-		                    "Options/Constants");
-		WritePluginBifItems(f, *it, plugin::BifItem::GLOBAL, "Globals");
-		WritePluginBifItems(f, *it, plugin::BifItem::TYPE, "Types");
-		WritePluginBifItems(f, *it, plugin::BifItem::EVENT, "Events");
-		WritePluginBifItems(f, *it, plugin::BifItem::FUNCTION, "Functions");
-		}
-
-	fclose(f);
-	}

src/BroDoc.h

@@ -1,422 +0,0 @@
-#ifndef brodoc_h
-#define brodoc_h
-
-#include <cstdio>
-#include <cstdarg>
-#include <string>
-#include <list>
-
-#include "BroDocObj.h"
-
-/**
- * This class is used to gather all data relevant to the automatic generation
- * of a reStructuredText (reST) document from a given Bro script.
- */
-class BroDoc {
-public:
-	/**
-	 * BroDoc constructor
-	 * Given a Bro script, opens new file in the current working directory
-	 * that will contain reST documentation generated from the parsing
-	 * of the Bro script.  The new reST file will be named similar to
-	 * the filename of the Bro script that generates it, except any
-	 * ".bro" file extension is stripped and ".rst" takes it place.
-	 * If the filename doesn't end in ".bro", then ".rst" is just appended.
-	 * Any '/' characters in the reST file name that result from choice of
-	 * the 'rel' parameter are replaced with '^'.
-	 * @param rel A string representing a subpath of the root Bro script
-	 *        source/install directory in which the source file is located.
-	 *        It can also be an absolute path, but then the parameter is
-	 *        ignored and the document title is just derived from file name
-	 * @param abs The absolute path to the Bro script for which to generate
-	 *        documentation.
-	 */
-	BroDoc(const std::string& rel, const std::string& abs);
-
-	/**
-	 * BroDoc destructor
-	 * Closes the file that was opened by the constructor and frees up
-	 * memory taken by BroDocObj objects.
-	 */
-	virtual ~BroDoc();
-
-	/**
-	 * Write out full reST documentation for the Bro script that was parsed.
-	 * BroDoc's default implementation of this function will care
-	 * about whether declarations made in the Bro script are part of
-	 * the public versus private interface (whether things are declared in
-	 * the export section).
-	 */
-	virtual void WriteDocFile() const;
-
-	/**
-	 * Schedules some summarizing text to be output directly into the reST doc.
-	 * This should be called whenever the scanner sees a line in the Bro script
-	 * starting with "##!"
-	 * @param s The summary text to add to the reST doc.
-	 */
-	void AddSummary(const std::string& s)	{ summary.push_back(s); }
-
-	/**
-	 * Schedules an import (@load) to be documented.
-	 * If the script being loaded has a .bro suffix, it is internally stripped.
-	 * This should be called whenever the scanner sees an @load.
-	 * @param s The name of the imported script.
-	 */
-	void AddImport(const std::string& s);
-
-	/**
-	 * Schedules a namespace (module) to be documented.
-	 * This should be called whenever the parser sees a TOK_MODULE.
-	 * @param s The namespace (module) identifier's name.
-	 */
-	void AddModule(const std::string& s)	{ modules.push_back(s); }
-
-	/**
-	 * Sets the way the script changes the "capture_filters" table.
-	 * This is determined by the scanner checking for changes to
-	 * the "capture_filters" table after each of Bro's input scripts
-	 * (given as command line arguments to Bro) are finished being parsed.
-	 * @param s The value "capture_filters" as given by TableVal::Describe()
-	 */
-	void SetPacketFilter(const std::string& s);
-
-	/**
-	 * Schedules documentation of a script option.  An option is
-	 * defined as any variable in the script that is declared 'const'
-	 * and has the '&redef' attribute.
-	 * @param o A pointer to a BroDocObj which contains the internal
-	 *        Bro language representation of the script option and
-	 *        also any associated comments about it.
-	 */
-	void AddOption(const BroDocObj* o)
-		{
-		options.push_back(o);
-		all.push_back(o);
-		}
-
-	/**
-	 * Schedules documentation of a script constant.  An option is
-	 * defined as any variable in the script that is declared 'const'
-	 * and does *not* have the '&redef' attribute.
-	 * @param o A pointer to a BroDocObj which contains the internal
-	 *        Bro language representation of the script constant and
-	 *        also any associated comments about it.
-	 */
-	void AddConstant(const BroDocObj* o)
-		{
-		constants.push_back(o);
-		all.push_back(o);
-		}
-
-	/**
-	 * Schedules documentation of a script state variable.  A state variable
-	 * is defined as any variable in the script that is declared 'global'
-	 * @param o A pointer to a BroDocObj which contains the internal
-	 *        Bro language representation of the script state variable
-	 *        and also any associated comments about it.
-	 */
-	void AddStateVar(const BroDocObj* o)
-		{
-		state_vars.push_back(o);
-		all.push_back(o);
-		}
-
-	/**
-	 * Schedules documentation of a type declared by the script.
-	 * @param o A pointer to a BroDocObj which contains the internal
-	 *        Bro language representation of the script option and
-	 *        also any associated comments about it.
-	 */
-	void AddType(const BroDocObj* o)
-		{
-		types.push_back(o);
-		all.push_back(o);
-		}
-
-	/**
-	 * Schedules documentation of a Notice (enum redef) declared by script
-	 * @param o A pointer to a BroDocObj which contains the internal
-	 *        Bro language representation of the Notice and also
-	 *        any associated comments about it.
-	 */
-	void AddNotice(const BroDocObj* o)
-		{
-		notices.push_back(o);
-		all.push_back(o);
-		}
-
-	/**
-	 * Schedules documentation of an event declared by the script.
-	 * @param o A pointer to a BroDocObj which contains the internal
-	 *        Bro language representation of the script event and
-	 *        also any associated comments about it.
-	 */
-	void AddEvent(const BroDocObj* o)
-		{
-		events.push_back(o);
-		all.push_back(o);
-		}
-
-	/**
-	 * Schedules documentation of an event handler declared by the script.
-	 * @param o A pointer to a BroDocObj which contains the internal
-	 *        Bro language representation of the script event handler and
-	 *        also any associated comments about it.
-	 */
-	void AddEventHandler(const BroDocObj* o)
-		{
-		event_handlers.push_back(o);
-		all.push_back(o);
-		}
-
-	/**
-	 * Schedules documentation of a hook declared by the script.
-	 * @param o A pointer to a BroDocObj which contains the internal
-	 *        Bro language representation of the script hook and
-	 *        also any associated comments about it.
-	 */
-	void AddHook(const BroDocObj* o)
-		{
-		hooks.push_back(o);
-		all.push_back(o);
-		}
-
-	/**
-	 * Schedules documentation of a hook handler declared by the script.
-	 * @param o A pointer to a BroDocObj which contains the internal
-	 *        Bro language representation of the script hook handler and
-	 *        also any associated comments about it.
-	 */
-	void AddHookHandler(const BroDocObj* o)
-		{
-		hook_handlers.push_back(o);
-		all.push_back(o);
-		}
-
-	/**
-	 * Schedules documentation of a function declared by the script.
-	 * @param o A pointer to a BroDocObj which contains the internal
-	 *        Bro language representation of the script function and
-	 *        also any associated comments about it.
-	 */
-	void AddFunction(BroDocObj* o);
-
-	/**
-	 * Schedules documentation of a redef done by the script
-	 * @param o A pointer to a BroDocObj which contains the internal
-	 *        Bro language representation of the script identifier
-	 *        that was redefined and also any associated comments.
-	 */
-	void AddRedef(const BroDocObj* o)
-		{
-		redefs.push_back(o);
-		all.push_back(o);
-		}
-
-	/**
-	 * Gets the name of the Bro script source file for which reST
-	 * documentation is being generated.
-	 * @return A char* to the start of the source file's name.
-	 */
-	const char* GetSourceFileName() const
-		{
-		return source_filename.c_str();
-		}
-
-	/**
-	 * Gets the name of the generated reST documentation file.
-	 * @return A char* to the start of the generated reST file's name.
-	 */
-	const char* GetOutputFileName() const
-		{
-		return reST_filename.c_str();
-		}
-
-	typedef std::list<const BroDocObj*> BroDocObjList;
-	typedef std::map<std::string, BroDocObj*> BroDocObjMap;
-
-	/**
-	 * Writes out a table of BroDocObj's to the reST document
-	 * @param f The file to write to.
-	 * @param l A list of BroDocObj pointers
-	 */
-	static void WriteBroDocObjTable(FILE* f, const BroDocObjList& l);
-
-	/**
-	 * Writes out given number of characters to reST document
-	 * @param f The file to write to.
-	 * @param c the character to write
-	 * @param n the number of characters to write
-	 */
-	static void WriteRepeatedChar(FILE* f, char c, size_t n);
-
-	/**
-	 * A wrapper to fprintf() that always uses the reST document
-	 * for the FILE* argument.
-	 * @param f The file to write to.
-	 * @param format A printf style format string.
-	 */
-	static void WriteToDoc(FILE* f, const char* format, ...);
-
-	/**
-	 * Writes out a list of strings to the reST document.
-	 * If the list is empty, prints a newline character.
-	 * @param f The file to write to.
-	 * @param format A printf style format string for elements of the list
-	 *        except for the last one in the list
-	 * @param last_format A printf style format string to use for the last
-	 *        element of the list
-	 * @param l A reference to a list of strings
-	 */
-	static void WriteStringList(FILE* f, const char* format, const char* last_format,
-			const std::list<std::string>& l);
-
-	/**
-	 * @see WriteStringList(FILE* f, const char*, const char*,
-	 *                      const std::list<std::string>&>)
-	 */
-	static void WriteStringList(FILE* f, const char* format,
-			const std::list<std::string>& l){
-		WriteStringList(f, format, format, l);
-		}
-
-	/**
-	 * Writes out a list of BroDocObj objects to the reST document
-	 * @param f The file to write to.
-	 * @param l A list of BroDocObj pointers
-	 * @param wantPublic If true, filter out objects that are not declared
-	 *        in the global scope.  If false, filter out those that are in
-	 *        the global scope.
-	 * @param heading The title of the section to create in the reST doc.
-	 * @param underline The character to use to underline the reST
-	 *        section heading.
-	 * @param isShort Whether to write the full documentation or a "short"
-	 *        version (a single sentence)
-	 */
-	static void WriteBroDocObjList(FILE* f, const BroDocObjList& l, bool wantPublic,
-			const char* heading, char underline,
-			bool isShort);
-
-	/**
-	 * Wraps the BroDocObjMap into a BroDocObjList and the writes that list
-	 * to the reST document
-	 * @see WriteBroDocObjList(FILE* f, const BroDocObjList&, bool, const char*, char,
-	        bool)
-	 */
-	static void WriteBroDocObjList(FILE* f, const BroDocObjMap& m, bool wantPublic,
-			const char* heading, char underline,
-			bool isShort);
-
-	/**
-	 * Writes out a list of BroDocObj objects to the reST document
-	 * @param l A list of BroDocObj pointers
-	 * @param heading The title of the section to create in the reST doc.
-	 * @param underline The character to use to underline the reST
-	 *        section heading.
-	 */
-	static void WriteBroDocObjList(FILE* f, const BroDocObjList& l, const char* heading,
-			char underline);
-
-	/**
-	 * Writes out a list of BroDocObj objects to the reST document
-	 * @param l A list of BroDocObj pointers
-	 */
-	static void WriteBroDocObjList(FILE* f, const BroDocObjList& l);
-
-	/**
-	 * Wraps the BroDocObjMap into a BroDocObjList and the writes that list
-	 * to the reST document
-	 * @see WriteBroDocObjList(FILE* f, const BroDocObjList&, const char*, char)
-	 */
-	static void WriteBroDocObjList(FILE* f, const BroDocObjMap& m, const char* heading,
-			char underline);
-
-	/**
-	 * Writes out a reST section heading
-	 * @param f The file to write to.
-	 * @param heading The title of the heading to create
-	 * @param underline The character to use to underline the section title
-	 *        within the reST document
-	 */
-	static void WriteSectionHeading(FILE* f, const char* heading, char underline);
-
-private:
-	FILE* reST_file;
-	std::string reST_filename;
-	std::string source_filename;	// points to the basename of source file
-	std::string downloadable_filename; // file that will be linked for download
-	std::string doc_title;
-	std::string packet_filter;
-
-	std::list<std::string> modules;
-	std::list<std::string> summary;
-	std::list<std::string> imports;
-	std::list<std::string> port_analysis;
-
-	BroDocObjList options;
-	BroDocObjList constants;
-	BroDocObjList state_vars;
-	BroDocObjList types;
-	BroDocObjList notices;
-	BroDocObjList events;
-	BroDocObjList event_handlers;
-	BroDocObjList hooks;
-	BroDocObjList hook_handlers;
-	BroDocObjMap functions;
-	BroDocObjList redefs;
-
-	BroDocObjList all;
-
-	/**
-	 * Writes out the reST for either the script's public or private interface
-	 * @param heading The title of the interfaces section heading
-	 * @param underline The underline character to use for the interface
-	 *        section
-	 * @param subunderline The underline character to use for interface
-	 *        sub-sections
-	 * @param isPublic Whether to write out the public or private script
-	 *        interface
-	 * @param isShort Whether to write out the full documentation or a "short"
-	 *        description (a single sentence)
-	 */
-	void WriteInterface(const char* heading, char underline, char subunderline,
-			bool isPublic, bool isShort) const;
-
-	/**
-	 * Frees memory allocated to BroDocObj's objects in a given list.
-	 * @param a reference to a list of BroDocObj pointers
-	 */
-	void FreeBroDocObjPtrList(BroDocObjList& l);
-
-	static bool IsPublicAPI(const BroDocObj* o)
-		{
-		return o->IsPublicAPI();
-		}
-
-	static bool IsPrivateAPI(const BroDocObj* o)
-		{
-		return ! o->IsPublicAPI();
-		}
-
-    struct replace_slash {
-        void operator()(char& c)
-            {
-            if ( c == '/' ) c = '^';
-            }
-    };
-};
-
-/**
- * Writes out plugin index documentation for all analyzer plugins.
- * @param filename the name of the file to write.
- */
-void CreateProtoAnalyzerDoc(const char* filename);
-
-/**
- * Writes out plugin index documentation for all file analyzer plugins.
- * @param filename the name of the file to write.
- */
-void CreateFileAnalyzerDoc(const char* filename);
-
-#endif

src/BroDocObj.cc

@@ -1,195 +0,0 @@
-#include <cstdio>
-#include <string>
-#include <list>
-#include "ID.h"
-#include "BroDocObj.h"
-
-map<string, BroDocObj*> doc_ids = map<string, BroDocObj*>();
-
-BroDocObj* BroDocObj::last = 0;
-
-BroDocObj::BroDocObj(const ID* id, std::list<std::string>*& reST,
-			bool is_fake)
-	{
-	last = this;
-	broID = id;
-	reST_doc_strings = reST;
-	reST = 0;
-	is_fake_id = is_fake;
-	use_role = 0;
-	FormulateShortDesc();
-	doc_ids[id->Name()] = this;
-	}
-
-BroDocObj::~BroDocObj()
-	{
-	if ( reST_doc_strings )
-		delete reST_doc_strings;
-
-	if ( is_fake_id )
-		delete broID;
-	}
-
-void BroDocObj::WriteReSTCompact(FILE* file, int max_col) const
-	{
-	ODesc desc;
-	desc.SetQuotes(1);
-	broID->DescribeReSTShort(&desc);
-
-	fprintf(file, "%s", desc.Description());
-
-	std::list<std::string>::const_iterator it;
-
-	for ( it = short_desc.begin(); it != short_desc.end(); ++it )
-		{
-		int start_col;
-
-		if ( it == short_desc.begin() )
-			start_col = max_col - desc.Len() + 1;
-		else
-			{
-			start_col = max_col + 1;
-			fprintf(file, "\n");
-			}
-
-		for ( int i = 0; i < start_col; ++i )
-			fprintf(file, " ");
-
-		fprintf(file, "%s", it->c_str());
-		}
-	}
-
-int BroDocObj::LongestShortDescLen() const
-	{
-	size_t max = 0;
-
-	std::list<std::string>::const_iterator it;
-
-	for ( it = short_desc.begin(); it != short_desc.end(); ++it )
-		{
-		if ( it->size() > max )
-			max = it->size();
-		}
-
-	return max;
-	}
-
-static size_t end_of_first_sentence(string s)
-	{
-	size_t rval = 0;
-
-	while ( (rval = s.find_first_of('.', rval)) != string::npos )
-		{
-		if ( rval == s.size() - 1 )
-			// Period is at end of string.
-			return rval;
-
-		if ( isspace(s[rval + 1]) )
-			// Period has a space after it.
-			return rval;
-
-		// Period has some non-space character after it, keep looking.
-		++rval;
-		}
-
-	return rval;
-	}
-
-void BroDocObj::FormulateShortDesc()
-	{
-	if ( ! reST_doc_strings )
-		return;
-
-	short_desc.clear();
-	std::list<std::string>::const_iterator it;
-
-	for ( it = reST_doc_strings->begin();
-		it != reST_doc_strings->end(); ++it )
-		{
-		// The short description stops at the first sentence or the
-		// first empty comment.
-		size_t end = end_of_first_sentence(*it);
-
-		if ( end == string::npos )
-			{
-			std::string::const_iterator s;
-			bool empty = true;
-
-			for ( s = it->begin(); s != it->end(); ++s )
-				{
-				if ( *s != ' ' && *s != '\t' && *s != '\n' && *s != '\r' )
-					{
-					empty = false;
-					short_desc.push_back(*it);
-					break;
-					}
-				}
-
-			if ( empty )
-				break;
-			}
-		else
-			{
-			short_desc.push_back(it->substr(0, end + 1));
-			break;
-			}
-		}
-	}
-
-void BroDocObj::WriteReST(FILE* file) const
-	{
-	int indent_spaces = 3;
-	ODesc desc;
-	desc.SetIndentSpaces(indent_spaces);
-	desc.SetQuotes(1);
-
-	broID->DescribeReST(&desc, use_role);
-
-	fprintf(file, "%s", desc.Description());
-
-	if ( HasDocumentation() )
-		{
-		fprintf(file, "\n");
-		std::list<std::string>::const_iterator it;
-
-		for ( it = reST_doc_strings->begin();
-			it != reST_doc_strings->end(); ++it)
-			{
-			for ( int i = 0; i < indent_spaces; ++i )
-				fprintf(file, " ");
-
-			fprintf(file, "%s\n", it->c_str());
-			}
-		}
-
-	fprintf(file, "\n");
-	}
-
-int BroDocObj::ColumnSize() const
-	{
-	ODesc desc;
-	desc.SetQuotes(1);
-	broID->DescribeReSTShort(&desc);
-	return desc.Len();
-	}
-
-bool BroDocObj::IsPublicAPI() const
-	{
-	return (broID->Scope() == SCOPE_GLOBAL) ||
-		(broID->Scope() == SCOPE_MODULE && broID->IsExport());
-	}
-
-void BroDocObj::Combine(const BroDocObj* o)
-	{
-	if ( o->reST_doc_strings )
-		{
-		if ( ! reST_doc_strings )
-			reST_doc_strings = new std::list<std::string>();
-
-		reST_doc_strings->splice(reST_doc_strings->end(),
-			*(o->reST_doc_strings));
-		}
-
-	delete o;
-	FormulateShortDesc();
-	}

src/BroDocObj.h

@@ -1,143 +0,0 @@
-#ifndef brodocobj_h
-#define brodocobj_h
-
-#include <cstdio>
-#include <string>
-#include <list>
-#include <map>
-
-#include "ID.h"
-
-/**
- * This class wraps a Bro script identifier, providing methods relevant
- * to automatic generation of reStructuredText (reST) documentation for it.
- */
-class BroDocObj {
-public:
-	/**
-	 * BroDocObj constructor
-	 * @param id a pointer to an identifier that is to be documented
-	 * @param reST a reference to a pointer of a list of strings that
-	 *        represent the reST documentation for the ID.  The pointer
-	 *        will be set to 0 after this constructor finishes.
-	 * @param is_fake whether the ID* is a dummy just for doc purposes
-	 */
-	BroDocObj(const ID* id, std::list<std::string>*& reST,
-			bool is_fake = false);
-
-	/**
-	 * BroDocObj destructor
-	 * Deallocates the memory associated with the list of reST strings
-	 */
-	~BroDocObj();
-
-	/**
-	 * Writes the reST representation of this object which includes
-	 * 1) a reST friendly description of the ID
-	 * 2) "##" or "##<" stylized comments.
-	 *    Anything after these style of comments is inserted as-is into
-	 *    the reST document.
-	 * @param file The (already opened) file to write the reST to.
-	 */
-	void WriteReST(FILE* file) const;
-
-	/**
-	 * Writes a compact version of the ID and associated documentation
-	 * for insertion into a table.
-	 * @param file The (already opened) file to write the reST to.
-	 * @param max_col The maximum length of the first table column
-	 */
-	void WriteReSTCompact(FILE* file, int max_col) const;
-
-	/**
-	 * @return the column size required by the reST representation of the ID
-	 */
-	int ColumnSize() const;
-
-	/**
-	 * Check whether this documentation is part of the public API.  In
-	 * other words, this means that the identifier is declared as part of
-	 * the global scope (has GLOBAL namespace or is exported from another
-	 * namespace).
-	 * @return true if the identifier is part of the script's public API
-	 */
-	bool IsPublicAPI() const;
-
-	/**
-	 * Return whether this object has documentation (## comments)
-	 * @return true if the ID has comments associated with it
-	 */
-	bool HasDocumentation() const
-		{
-		return reST_doc_strings && reST_doc_strings->size() > 0;
-		}
-
-	/**
-	 * @return whether this object will use reST role (T) or directive (F)
-	 * notation for the wrapped identifier.  Roles are usually used
-	 * for cross-referencing.
-	 */
-	bool UseRole() const	{ return use_role; }
-
-	/**
-	 * @param b whether this object will use reST role (T) or directive (F)
-	 * notation for the wrapped identifier.  Roles are usually used
-	 * for cross-referencing.
-	 */
-	void SetRole(bool b)	{ use_role = b; }
-
-	/**
-	 * Append any reST documentation strings in a given BroDocObj to this
-	 * object's list and then delete the given BroDocObj
-	 * @param o a pointer to a BroDocObj to subsume
-	 */
-	void Combine(const BroDocObj* o);
-
-	/**
-	 * @return the name of the wrapped identifier
-	 */
-	const char* Name() const	{ return broID->Name(); }
-
-	/**
-	 * @return the longest string element of the short description's list of
-	 *         strings
-	 */
-	int LongestShortDescLen() const;
-
-	/**
-	 * Adds a reST documentation string to this BroDocObj's list.
-	 * @param s the documentation string to append.
-	 */
-	void AddDocString(const std::string& s)
-		{
-		if ( ! reST_doc_strings )
-			reST_doc_strings = new std::list<std::string>();
-		reST_doc_strings->push_back(s);
-		FormulateShortDesc();
-		}
-
-	static BroDocObj* last;
-
-protected:
-	std::list<std::string>* reST_doc_strings;
-	std::list<std::string> short_desc;
-	const ID* broID;
-	bool is_fake_id; /**< Whether the ID* is a dummy just for doc purposes */
-	bool use_role; /**< Whether to use a reST role or directive for the ID */
-
-	/**
-	 * Set the short_desc member to be a subset of reST_doc_strings.
-	 * Specifically, short_desc will be everything in reST_doc_strings
-	 * up until the first period or first empty string list element found.
-	 */
-	void FormulateShortDesc();
-
-private:
-};
-
-/**
- * Map identifiers to their broxygen documentation objects.
- */
-extern map<string, BroDocObj*> doc_ids;
-
-#endif

src/Brofiler.h

@@ -78,4 +78,6 @@ private:
 	};
 };
 
+extern Brofiler brofiler;
+
 #endif /* BROFILER_H_ */

src/CMakeLists.txt

@@ -154,6 +154,7 @@ set(bro_PLUGIN_LIBS CACHE INTERNAL "plugin libraries" FORCE)
 add_subdirectory(analyzer)
 add_subdirectory(file_analysis)
 add_subdirectory(probabilistic)
+add_subdirectory(broxygen)
 
 set(bro_SUBDIRS
     ${bro_SUBDIR_LIBS}
@@ -249,8 +250,6 @@ set(bro_SRCS
     Attr.cc
     Base64.cc
     BPF_Program.cc
-    BroDoc.cc
-    BroDocObj.cc
     Brofiler.cc
     BroString.cc
     CCL.cc

src/Debug.cc

@@ -342,18 +342,16 @@ vector<ParseLocationRec> parse_location_string(const string& s)
 			if ( ! sscanf(line_string.c_str(), "%d", &plr.line) )
 				plr.type = plrUnknown;
 
-			FILE* throwaway = search_for_file(filename.c_str(), "bro",
+			string path(find_file(filename, bro_path(), "bro"));
-								&full_filename, true, 0);
+
-			if ( ! throwaway )
+			if ( path.empty() )
 				{
 				debug_msg("No such policy file: %s.\n", filename.c_str());
 				plr.type = plrUnknown;
 				return result;
 				}
 
-			fclose(throwaway);
+			loc_filename = copy_string(path.c_str());
-
-			loc_filename = full_filename;
 			plr.type = plrFileAndLine;
 			}
 		}

src/DebugLogger.cc

@@ -17,7 +17,7 @@ DebugLogger::Stream DebugLogger::streams[NUM_DBGS] = {
 	{ "dpd", 0, false }, { "tm", 0, false },
 	{ "logging", 0, false }, {"input", 0, false }, 
 	{ "threading", 0, false }, { "file_analysis", 0, false },
-	{ "plugins", 0, false}
+	{ "plugins", 0, false }, { "broxygen", 0, false }
 };
 
 DebugLogger::DebugLogger(const char* filename)

src/DebugLogger.h

@@ -28,6 +28,7 @@ enum DebugStream {
 	DBG_THREADING,	// Threading system
 	DBG_FILE_ANALYSIS,	// File analysis
 	DBG_PLUGINS,
+	DBG_BROXYGEN,
 
 	NUM_DBGS // Has to be last
 };

src/Desc.h

@@ -69,6 +69,7 @@ public:
 	void PopIndent();
 	void PopIndentNoNL();
 	int GetIndentLevel() const	{ return indent_level; }
+	void ClearIndentLevel() { indent_level = 0; }
 
 	int IndentSpaces() const	{ return indent_with_spaces; }
 	void SetIndentSpaces(int i)	{ indent_with_spaces = i; }

src/Expr.cc

@@ -5172,6 +5172,7 @@ BroType* ListExpr::InitType() const
 			types->append(td);
 			}
 
+
 		return new RecordType(types);
 		}
 

src/ID.cc

@@ -14,6 +14,7 @@
 #include "PersistenceSerializer.h"
 #include "Scope.h"
 #include "Traverse.h"
+#include "broxygen/Manager.h"
 
 ID::ID(const char* arg_name, IDScope arg_scope, bool arg_is_export)
 	{
@@ -631,8 +632,8 @@ void ID::DescribeReSTShort(ODesc* d) const
 		d->Add(": ");
 		d->Add(":bro:type:`");
 
-		if ( ! is_type && type->GetTypeID() )
+		if ( ! is_type && ! type->GetName().empty() )
-			d->Add(type->GetTypeID());
+			d->Add(type->GetName().c_str());
 		else
 			{
 			TypeTag t = type->Tag();
@@ -643,14 +644,14 @@ void ID::DescribeReSTShort(ODesc* d) const
 				break;
 
 			case TYPE_FUNC:
-				d->Add(type->AsFuncType()->FlavorString());
+				d->Add(type->AsFuncType()->FlavorString().c_str());
 				break;
 
 			case TYPE_ENUM:
 				if ( is_type )
 					d->Add(type_name(t));
 				else
-					d->Add(type->AsEnumType()->Name().c_str());
+					d->Add(broxygen_mgr->GetEnumTypeName(Name()).c_str());
 				break;
 
 			default:
@@ -669,9 +670,9 @@ void ID::DescribeReSTShort(ODesc* d) const
 		}
 	}
 
-void ID::DescribeReST(ODesc* d, bool is_role) const
+void ID::DescribeReST(ODesc* d, bool roles_only) const
 	{
-	if ( is_role )
+	if ( roles_only )
 		{
 		if ( is_type )
 			d->Add(":bro:type:`");
@@ -696,14 +697,14 @@ void ID::DescribeReST(ODesc* d, bool is_role) const
 		{
 		d->Add(":Type: ");
 
-		if ( ! is_type && type->GetTypeID() )
+		if ( ! is_type && ! type->GetName().empty() )
 			{
 			d->Add(":bro:type:`");
-			d->Add(type->GetTypeID());
+			d->Add(type->GetName());
 			d->Add("`");
 			}
 		else
-			type->DescribeReST(d);
+			type->DescribeReST(d, roles_only);
 
 		d->NL();
 		}

src/ID.h

@@ -84,7 +84,7 @@ public:
 	// Adds type and value to description.
 	void DescribeExtended(ODesc* d) const;
 	// Produces a description that's reST-ready.
-	void DescribeReST(ODesc* d, bool is_role=false) const;
+	void DescribeReST(ODesc* d, bool roles_only = false) const;
 	void DescribeReSTShort(ODesc* d) const;
 
 	bool Serialize(SerialInfo* info) const;

src/Net.h

@@ -4,6 +4,7 @@
 #define net_h
 
 #include "net_util.h"
+#include "util.h"
 #include "BPF_Program.h"
 #include "List.h"
 #include "PktSrc.h"
@@ -97,15 +98,14 @@ struct ScannedFile {
 	ino_t inode;
 	int include_level;
 	string name;
-	string subpath;		// Path in BROPATH's policy/ containing the file.
 	bool skipped;		// This ScannedFile was @unload'd.
 	bool prefixes_checked;	// If loading prefixes for this file has been tried.
 
-	ScannedFile(ino_t arg_inode, int arg_include_level, string arg_name,
+	ScannedFile(ino_t arg_inode, int arg_include_level, const string& arg_name,
-		    string arg_subpath = "", bool arg_skipped = false,
+		    bool arg_skipped = false,
 		    bool arg_prefixes_checked = false)
 			: inode(arg_inode), include_level(arg_include_level),
-			name(arg_name), subpath(arg_subpath), skipped(arg_skipped),
+			name(arg_name), skipped(arg_skipped),
 			prefixes_checked(arg_prefixes_checked)
 		{ }
 };

src/OSFinger.cc

@@ -294,7 +294,8 @@ void OSFingerprint::load_config(const char* file)
   uint32 ln=0;
   char buf[MAXLINE];
   char* p;
-  FILE* c = search_for_file(file, "osf", 0, false, 0);
+
+  FILE* c = open_file(find_file(file, bro_path(), "osf"));
 
   if (!c)
     {

src/RuleMatcher.cc

@@ -226,7 +226,7 @@ bool RuleMatcher::ReadFiles(const name_list& files)
 
 	for ( int i = 0; i < files.length(); ++i )
 		{
-		rules_in = search_for_file(files[i], "sig", 0, false, 0);
+		rules_in = open_file(find_file(files[i], bro_path(), "sig"));
 		if ( ! rules_in )
 			{
 			reporter->Error("Can't open signature file %s", files[i]);

src/Serializer.h

@@ -125,7 +125,7 @@ protected:
 
 	// This will be increased whenever there is an incompatible change
 	// in the data format.
-	static const uint32 DATA_FORMAT_VERSION = 23;
+	static const uint32 DATA_FORMAT_VERSION = 24;
 
 	ChunkedIO* io;
 

src/Type.cc

@@ -8,13 +8,13 @@
 #include "Scope.h"
 #include "Serializer.h"
 #include "Reporter.h"
+#include "broxygen/Manager.h"
+#include "broxygen/utils.h"
 
 #include <string>
 #include <list>
 #include <map>
 
-extern int generate_documentation;
-
 // Note: This function must be thread-safe.
 const char* type_name(TypeTag t)
 	{
@@ -58,7 +58,6 @@ BroType::BroType(TypeTag t, bool arg_base_type)
 	tag = t;
 	is_network_order = 0;
 	base_type = arg_base_type;
-	type_id = 0;
 
 	switch ( tag ) {
 	case TYPE_VOID:
@@ -121,10 +120,27 @@ BroType::BroType(TypeTag t, bool arg_base_type)
 
 	}
 
-BroType::~BroType()
+BroType* BroType::Clone() const
 	{
-	if ( type_id )
+	SerializationFormat* form = new BinarySerializationFormat();
-		delete [] type_id;
+	form->StartWrite();
+	CloneSerializer ss(form);
+	SerialInfo sinfo(&ss);
+	sinfo.cache = false;
+
+	this->Serialize(&sinfo);
+	char* data = 0;
+	uint32 len = form->EndWrite(&data);
+	form->StartRead(data, len);
+
+	UnserialInfo uinfo(&ss);
+	uinfo.cache = false;
+
+	BroType* rval = this->Unserialize(&uinfo, false);
+	assert(rval != this);
+
+	delete [] data;
+	return rval;
 	}
 
 int BroType::MatchesIndex(ListExpr*& index) const
@@ -170,11 +186,9 @@ void BroType::Describe(ODesc* d) const
 		}
 	}
 
-void BroType::DescribeReST(ODesc* d) const
+void BroType::DescribeReST(ODesc* d, bool roles_only) const
 	{
-	d->Add(":bro:type:`");
+	d->Add(fmt(":bro:type:`%s`", type_name(Tag())));
-	d->Add(type_name(Tag()));
-	d->Add("`");
 	}
 
 void BroType::SetError()
@@ -197,7 +211,7 @@ bool BroType::Serialize(SerialInfo* info) const
 	return ret;
 	}
 
-BroType* BroType::Unserialize(UnserialInfo* info, TypeTag want)
+BroType* BroType::Unserialize(UnserialInfo* info, bool use_existing)
 	{
 	// To avoid external Broccoli clients needing to always send full type
 	// objects, we allow them to give us only the name of a type. To
@@ -231,12 +245,24 @@ BroType* BroType::Unserialize(UnserialInfo* info, TypeTag want)
 
 	BroType* t = (BroType*) SerialObj::Unserialize(info, SER_BRO_TYPE);
 
-	if ( ! t )
+	if ( ! t || ! use_existing )
-		return 0;
+		return t;
 
-	// For base types, we return our current instance
+	if ( ! t->name.empty() )
-	// if not in "documentation mode".
+		{
-	if ( t->base_type && ! generate_documentation )
+		// Avoid creating a new type if it's known by name.
+		// Also avoids loss of base type name alias (from condition below).
+		ID* id = global_scope()->Lookup(t->name.c_str());
+		BroType* t2 = id ? id->AsType() : 0;
+
+		if ( t2 )
+			{
+			Unref(t);
+			return t2->Ref();
+			}
+		}
+
+	if ( t->base_type )
 		{
 		BroType* t2 = ::base_type(TypeTag(t->tag));
 		Unref(t);
@@ -259,21 +285,10 @@ bool BroType::DoSerialize(SerialInfo* info) const
 	if ( ! (SERIALIZE(char(tag)) && SERIALIZE(char(internal_tag))) )
 		return false;
 
-	if ( ! (SERIALIZE(is_network_order) && SERIALIZE(base_type) &&
+	if ( ! (SERIALIZE(is_network_order) && SERIALIZE(base_type)) )
-		// Serialize the former "bool is_global_attributes_type" for
-		// backwards compatibility.
-		SERIALIZE(false)) )
 		return false;
 
-	// Likewise, serialize the former optional "RecordType* attributes_type"
+	SERIALIZE_STR(name.c_str(), name.size());
-	// for backwards compatibility.
-	void* null = NULL;
-	SERIALIZE(null);
-
-	if ( generate_documentation )
-		{
-		SERIALIZE_OPTIONAL_STR(type_id);
-		}
 
 	info->s->WriteCloseTag("Type");
 
@@ -291,24 +306,15 @@ bool BroType::DoUnserialize(UnserialInfo* info)
 	tag = (TypeTag) c1;
 	internal_tag = (InternalTypeTag) c2;
 
-	bool not_used;
+	if ( ! (UNSERIALIZE(&is_network_order) && UNSERIALIZE(&base_type)) )
-
-	if ( ! (UNSERIALIZE(&is_network_order) && UNSERIALIZE(&base_type)
-			// Unerialize the former "bool is_global_attributes_type" for
-			// backwards compatibility.
-			&& UNSERIALIZE(&not_used)) )
 		return 0;
 
-	BroType* not_used_either;
+	const char* n;
+	if ( ! UNSERIALIZE_STR(&n, 0) )
+		return false;
 
-	// Likewise, unserialize the former optional "RecordType*
+	name = n;
-	// attributes_type" for backwards compatibility.
+	delete [] n;
-	UNSERIALIZE_OPTIONAL(not_used_either, BroType::Unserialize(info, TYPE_RECORD));
-
-	if ( generate_documentation )
-		{
-		UNSERIALIZE_OPTIONAL_STR(type_id);
-		}
 
 	return true;
 	}
@@ -463,7 +469,7 @@ void IndexType::Describe(ODesc* d) const
 		}
 	}
 
-void IndexType::DescribeReST(ODesc* d) const
+void IndexType::DescribeReST(ODesc* d, bool roles_only) const
 	{
 	d->Add(":bro:type:`");
 
@@ -482,14 +488,14 @@ void IndexType::DescribeReST(ODesc* d) const
 
 		const BroType* t = (*IndexTypes())[i];
 
-		if ( t->GetTypeID() )
+		if ( ! t->GetName().empty() )
 			{
 			d->Add(":bro:type:`");
-			d->Add(t->GetTypeID());
+			d->Add(t->GetName());
 			d->Add("`");
 			}
 		else
-			t->DescribeReST(d);
+			t->DescribeReST(d, roles_only);
 		}
 
 	d->Add("]");
@@ -498,14 +504,14 @@ void IndexType::DescribeReST(ODesc* d) const
 		{
 		d->Add(" of ");
 
-		if ( yield_type->GetTypeID() )
+		if ( ! yield_type->GetName().empty() )
 			{
 			d->Add(":bro:type:`");
-			d->Add(yield_type->GetTypeID());
+			d->Add(yield_type->GetName());
 			d->Add("`");
 			}
 		else
-			yield_type->DescribeReST(d);
+			yield_type->DescribeReST(d, roles_only);
 		}
 	}
 
@@ -780,7 +786,7 @@ void FuncType::Describe(ODesc* d) const
 		}
 	}
 
-void FuncType::DescribeReST(ODesc* d) const
+void FuncType::DescribeReST(ODesc* d, bool roles_only) const
 	{
 	d->Add(":bro:type:`");
 	d->Add(FlavorString());
@@ -793,14 +799,14 @@ void FuncType::DescribeReST(ODesc* d) const
 		{
 		d->AddSP(" :");
 
-		if ( yield->GetTypeID() )
+		if ( ! yield->GetName().empty() )
 			{
 			d->Add(":bro:type:`");
-			d->Add(yield->GetTypeID());
+			d->Add(yield->GetName());
 			d->Add("`");
 			}
 		else
-			yield->DescribeReST(d);
+			yield->DescribeReST(d, roles_only);
 		}
 	}
 
@@ -885,6 +891,17 @@ TypeDecl::TypeDecl(BroType* t, const char* i, attr_list* arg_attrs, bool in_reco
 	id = i;
 	}
 
+TypeDecl::TypeDecl(const TypeDecl& other)
+	{
+	type = other.type->Ref();
+	attrs = other.attrs;
+
+	if ( attrs )
+		::Ref(attrs);
+
+	id = copy_string(other.id);
+	}
+
 TypeDecl::~TypeDecl()
 	{
 	Unref(type);
@@ -921,19 +938,19 @@ TypeDecl* TypeDecl::Unserialize(UnserialInfo* info)
 	return t;
 	}
 
-void TypeDecl::DescribeReST(ODesc* d) const
+void TypeDecl::DescribeReST(ODesc* d, bool roles_only) const
 	{
 	d->Add(id);
 	d->Add(": ");
 
-	if ( type->GetTypeID() )
+	if ( ! type->GetName().empty() )
 		{
 		d->Add(":bro:type:`");
-		d->Add(type->GetTypeID());
+		d->Add(type->GetName());
 		d->Add("`");
 		}
 	else
-		type->DescribeReST(d);
+		type->DescribeReST(d, roles_only);
 
 	if ( attrs )
 		{
@@ -942,37 +959,6 @@ void TypeDecl::DescribeReST(ODesc* d) const
 		}
 	}
 
-CommentedTypeDecl::CommentedTypeDecl(BroType* t, const char* i,
-			attr_list* attrs, bool in_record, std::list<std::string>* cmnt_list)
-	: TypeDecl(t, i, attrs, in_record)
-	{
-	comments = cmnt_list;
-	}
-
-CommentedTypeDecl::~CommentedTypeDecl()
-	{
-	if ( comments ) delete comments;
-	}
-
-void CommentedTypeDecl::DescribeReST(ODesc* d) const
-	{
-	TypeDecl::DescribeReST(d);
-
-	if ( comments )
-		{
-		d->PushIndent();
-		std::list<std::string>::const_iterator i;
-
-		for ( i = comments->begin(); i != comments->end(); ++i)
-			{
-			if ( i != comments->begin() ) d->NL();
-			d->Add(i->c_str());
-			}
-
-		d->PopIndentNoNL();
-		}
-	}
-
 RecordType::RecordType(type_decl_list* arg_types) : BroType(TYPE_RECORD)
 	{
 	types = arg_types;
@@ -1062,9 +1048,13 @@ void RecordType::Describe(ODesc* d) const
 		}
 	}
 
-void RecordType::DescribeReST(ODesc* d) const
+void RecordType::DescribeReST(ODesc* d, bool roles_only) const
 	{
 	d->Add(":bro:type:`record`");
+
+	if ( num_fields == 0 )
+		return;
+
 	d->NL();
 	DescribeFieldsReST(d, false);
 	}
@@ -1161,7 +1151,63 @@ void RecordType::DescribeFieldsReST(ODesc* d, bool func_args) const
 				}
 			}
 
-		FieldDecl(i)->DescribeReST(d);
+		const TypeDecl* td = FieldDecl(i);
+		td->DescribeReST(d);
+
+		if ( func_args )
+			continue;
+
+		using broxygen::IdentifierInfo;
+		IdentifierInfo* doc = broxygen_mgr->GetIdentifierInfo(GetName());
+
+		if ( ! doc )
+			{
+			reporter->InternalWarning("Failed to lookup record doc: %s",
+			                          GetName().c_str());
+			continue;
+			}
+
+		string field_from_script = doc->GetDeclaringScriptForField(td->id);
+		string type_from_script;
+
+		if ( doc->GetDeclaringScript() )
+			type_from_script = doc->GetDeclaringScript()->Name();
+
+		if ( ! field_from_script.empty() &&
+		     field_from_script != type_from_script )
+			{
+			d->PushIndent();
+			d->Add(fmt("(from ``redef`` in :doc:`/scripts/%s`)",
+			           field_from_script.c_str()));
+			d->PopIndent();
+			}
+
+		vector<string> cmnts = doc->GetFieldComments(td->id);
+
+		if ( cmnts.empty() )
+			continue;
+
+		d->PushIndent();
+
+		for ( size_t i = 0; i < cmnts.size(); ++i )
+			{
+			if ( i > 0 )
+				d->NL();
+
+			if ( IsFunc(td->type->Tag()) )
+				{
+				string s = cmnts[i];
+
+				if ( broxygen::prettify_params(s) )
+					d->NL();
+
+				d->Add(s.c_str());
+				}
+			else
+				d->Add(cmnts[i].c_str());
+			}
+
+		d->PopIndentNoNL();
 		}
 
 	if ( ! func_args )
@@ -1340,38 +1386,12 @@ bool OpaqueType::DoUnserialize(UnserialInfo* info)
 	return true;
 	}
 
-EnumType::EnumType(const string& arg_name)
-: BroType(TYPE_ENUM)
-	{
-	name = arg_name;
-	counter = 0;
-	}
-
-EnumType::EnumType(EnumType* e)
-: BroType(TYPE_ENUM)
-	{
-	name = e->name;
-	counter = e->counter;
-
-	for ( NameMap::iterator it = e->names.begin(); it != e->names.end(); ++it )
-		names[copy_string(it->first)] = it->second;
-	}
-
 EnumType::~EnumType()
 	{
 	for ( NameMap::iterator iter = names.begin(); iter != names.end(); ++iter )
 		delete [] iter->first;
 	}
 
-CommentedEnumType::~CommentedEnumType()
-	{
-	for ( CommentMap::iterator iter = comments.begin(); iter != comments.end(); ++iter )
-		{
-		delete [] iter->first;
-		delete iter->second;
-		}
-	}
-
 // Note, we use reporter->Error() here (not Error()) to include the current script
 // location in the error message, rather than the one where the type was
 // originally defined.
@@ -1384,7 +1404,7 @@ void EnumType::AddName(const string& module_name, const char* name, bool is_expo
 		SetError();
 		return;
 		}
-	AddNameInternal(module_name, name, counter, is_export);
+	CheckAndAddName(module_name, name, counter, is_export);
 	counter++;
 	}
 
@@ -1398,32 +1418,12 @@ void EnumType::AddName(const string& module_name, const char* name, bro_int_t va
 		return;
 		}
 	counter = -1;
-	AddNameInternal(module_name, name, val, is_export);
+	CheckAndAddName(module_name, name, val, is_export);
 	}
 
-void CommentedEnumType::AddComment(const string& module_name, const char* name,
+void EnumType::CheckAndAddName(const string& module_name, const char* name,
-                                   std::list<std::string>* new_comments)
+                               bro_int_t val, bool is_export)
 	{
-	if ( ! new_comments )
-		return;
-
-	string fullname = make_full_var_name(module_name.c_str(), name);
-
-	CommentMap::iterator it = comments.find(fullname.c_str());
-
-	if ( it == comments.end() )
-		comments[copy_string(fullname.c_str())] = new_comments;
-	else
-		{
-		list<string>* prev_comments = comments[fullname.c_str()];
-		prev_comments->splice(prev_comments->end(), *new_comments);
-		delete new_comments;
-		}
-	}
-
-void EnumType::AddNameInternal(const string& module_name, const char* name, bro_int_t val, bool is_export)
-	{
-	ID *id;
 	if ( Lookup(val) )
 		{
 		reporter->Error("enumerator value in enumerated type definition already exists");
@@ -1431,12 +1431,14 @@ void EnumType::AddNameInternal(const string& module_name, const char* name, bro_
 		return;
 		}
 
-	id = lookup_ID(name, module_name.c_str());
+	ID* id = lookup_ID(name, module_name.c_str());
+
 	if ( ! id )
 		{
 		id = install_ID(name, module_name.c_str(), true, is_export);
 		id->SetType(this->Ref());
 		id->SetEnumConst();
+		broxygen_mgr->Identifier(id);
 		}
 	else
 		{
@@ -1445,11 +1447,19 @@ void EnumType::AddNameInternal(const string& module_name, const char* name, bro_
 		return;
 		}
 
-	string fullname = make_full_var_name(module_name.c_str(), name);
+	AddNameInternal(module_name, name, val, is_export);
-	names[copy_string(fullname.c_str())] = val;
+
+	set<BroType*> types = type_aliases[GetName()];
+	set<BroType*>::const_iterator it;
+
+	for ( it = types.begin(); it != types.end(); ++it )
+		if ( *it != this )
+			(*it)->AsEnumType()->AddNameInternal(module_name, name, val,
+							     is_export);
 	}
 
-void CommentedEnumType::AddNameInternal(const string& module_name, const char* name, bro_int_t val, bool is_export)
+void EnumType::AddNameInternal(const string& module_name, const char* name,
+                               bro_int_t val, bool is_export)
 	{
 	string fullname = make_full_var_name(module_name.c_str(), name);
 	names[copy_string(fullname.c_str())] = val;
@@ -1476,57 +1486,83 @@ const char* EnumType::Lookup(bro_int_t value)
 	return 0;
 	}
 
-void EnumType::DescribeReST(ODesc* d) const
+void EnumType::DescribeReST(ODesc* d, bool roles_only) const
 	{
-	d->Add(":bro:type:`");
+	d->Add(":bro:type:`enum`");
-	d->Add(name.c_str());
+
-	d->Add("`");
+	// Create temporary, reverse name map so that enums can be documented
-	}
+	// in ascending order of their actual integral value instead of by name.
+	typedef map< bro_int_t, const char* > RevNameMap;
 
-void CommentedEnumType::DescribeReST(ODesc* d) const
-	{
-	// create temporary, reverse name map so that enums can be documented
-	// in ascending order of their actual integral value instead of by name
-	typedef std::map< bro_int_t, const char* > RevNameMap;
 	RevNameMap rev;
+
 	for ( NameMap::const_iterator it = names.begin(); it != names.end(); ++it )
 		rev[it->second] = it->first;
 
-	d->Add(":bro:type:`");
-	d->Add(type_name(Tag()));
-	d->Add("`");
-	d->PushIndent();
-	d->NL();
-
 	for ( RevNameMap::const_iterator it = rev.begin(); it != rev.end(); ++it )
-		{
-		if ( it != rev.begin() )
 		{
 		d->NL();
-			d->NL();
-			}
-
-		d->Add(".. bro:enum:: ");
-		d->AddSP(it->second);
-		d->Add(GetTypeID());
-
-		CommentMap::const_iterator cmnt_it = comments.find(it->second);
-		if ( cmnt_it != comments.end() )
-			{
 		d->PushIndent();
-			d->NL();
+
-			std::list<std::string>::const_iterator i;
+		if ( roles_only )
-			const std::list<std::string>* cmnt_list = cmnt_it->second;
+			d->Add(fmt(":bro:enum:`%s`", it->second));
-			for ( i = cmnt_list->begin(); i != cmnt_list->end(); ++i)
+		else
+			d->Add(fmt(".. bro:enum:: %s %s", it->second, GetName().c_str()));
+
+		using broxygen::IdentifierInfo;
+		IdentifierInfo* doc = broxygen_mgr->GetIdentifierInfo(it->second);
+
+		if ( ! doc )
 			{
-				if ( i != cmnt_list->begin() ) d->NL();
+			reporter->InternalWarning("Enum %s documentation lookup failure",
-				d->Add(i->c_str());
+			                          it->second);
+			continue;
 			}
+
+		string enum_from_script;
+		string type_from_script;
+
+		if ( doc->GetDeclaringScript() )
+			enum_from_script = doc->GetDeclaringScript()->Name();
+
+		IdentifierInfo* type_doc = broxygen_mgr->GetIdentifierInfo(GetName());
+
+		if ( type_doc && type_doc->GetDeclaringScript() )
+			type_from_script = type_doc->GetDeclaringScript()->Name();
+
+		if ( ! enum_from_script.empty() &&
+		     enum_from_script != type_from_script )
+			{
+			d->NL();
+			d->PushIndent();
+			d->Add(fmt("(from ``redef`` in :doc:`/scripts/%s`)",
+			           enum_from_script.c_str()));
+			d->PopIndent();
+			}
+
+		vector<string> cmnts = doc->GetComments();
+
+		if ( cmnts.empty() )
+			{
+			d->PopIndentNoNL();
+			continue;
+			}
+
+		d->NL();
+		d->PushIndent();
+
+		for ( size_t i = 0; i < cmnts.size(); ++i )
+			{
+			if ( i > 0 )
+				d->NL();
+
+			d->Add(cmnts[i].c_str());
+			}
+
+		d->PopIndentNoNL();
 		d->PopIndentNoNL();
 		}
 	}
-	d->PopIndentNoNL();
-	}
 
 IMPLEMENT_SERIAL(EnumType, SER_ENUM_TYPE);
 

src/Type.h

@@ -4,7 +4,7 @@
 #define type_h
 
 #include <string>
-#include <list>
+#include <set>
 #include <map>
 
 #include "Obj.h"
@@ -81,7 +81,9 @@ const int MATCHES_INDEX_VECTOR = 2;
 class BroType : public BroObj {
 public:
 	BroType(TypeTag tag, bool base_type = false);
-	~BroType();
+	~BroType() { }
+
+	BroType* Clone() const;
 
 	TypeTag Tag() const		{ return tag; }
 	InternalTypeTag InternalType() const	{ return internal_tag; }
@@ -233,18 +235,18 @@ public:
 	BroType* Ref()		{ ::Ref(this); return this; }
 
 	virtual void Describe(ODesc* d) const;
-	virtual void DescribeReST(ODesc* d) const;
+	virtual void DescribeReST(ODesc* d, bool roles_only = false) const;
 
 	virtual unsigned MemoryAllocation() const;
 
 	bool Serialize(SerialInfo* info) const;
-	static BroType* Unserialize(UnserialInfo* info, TypeTag want = TYPE_ANY);
+	static BroType* Unserialize(UnserialInfo* info, bool use_existing = true);
 
-	void SetTypeID(const char* id)	{ type_id = id; }
+	void SetName(const string& arg_name) { name = arg_name; }
-	const char* GetTypeID() const	{ return type_id; }
+	string GetName() const { return name; }
 
 protected:
-	BroType()	{ type_id = 0; }
+	BroType()	{ }
 
 	void SetError();
 
@@ -255,10 +257,7 @@ private:
 	InternalTypeTag internal_tag;
 	bool is_network_order;
 	bool base_type;
-
+	string name;
-	// This type_id field is only used by the documentation framework to
-	// track the names of declared types.
-	const char* type_id;
 };
 
 class TypeList : public BroType {
@@ -314,7 +313,7 @@ public:
 	BroType* YieldType();
 
 	void Describe(ODesc* d) const;
-	void DescribeReST(ODesc* d) const;
+	void DescribeReST(ODesc* d, bool roles_only = false) const;
 
 	// Returns true if this table is solely indexed by subnet.
 	bool IsSubNetIndex() const;
@@ -388,7 +387,7 @@ public:
 	TypeList* ArgTypes() const	{ return arg_types; }
 
 	void Describe(ODesc* d) const;
-	void DescribeReST(ODesc* d) const;
+	void DescribeReST(ODesc* d, bool roles_only = false) const;
 
 protected:
 	FuncType()	{ args = 0; arg_types = 0; yield = 0; flavor = FUNC_FLAVOR_FUNCTION; }
@@ -416,6 +415,7 @@ protected:
 class TypeDecl {
 public:
 	TypeDecl(BroType* t, const char* i, attr_list* attrs = 0, bool in_record = false);
+	TypeDecl(const TypeDecl& other);
 	virtual ~TypeDecl();
 
 	const Attr* FindAttr(attr_tag a) const
@@ -424,24 +424,13 @@ public:
 	bool Serialize(SerialInfo* info) const;
 	static TypeDecl* Unserialize(UnserialInfo* info);
 
-	virtual void DescribeReST(ODesc* d) const;
+	virtual void DescribeReST(ODesc* d, bool roles_only = false) const;
 
 	BroType* type;
 	Attributes* attrs;
 	const char* id;
 };
 
-class CommentedTypeDecl : public TypeDecl {
-public:
-	CommentedTypeDecl(BroType* t, const char* i, attr_list* attrs = 0,
-			bool in_record = false, std::list<std::string>* cmnt_list = 0);
-	virtual ~CommentedTypeDecl();
-
-	void DescribeReST(ODesc* d) const;
-
-	std::list<std::string>* comments;
-};
-
 class RecordType : public BroType {
 public:
 	RecordType(type_decl_list* types);
@@ -473,7 +462,7 @@ public:
 	const char* AddFields(type_decl_list* types, attr_list* attr);
 
 	void Describe(ODesc* d) const;
-	void DescribeReST(ODesc* d) const;
+	void DescribeReST(ODesc* d, bool roles_only = false) const;
 	void DescribeFields(ODesc* d) const;
 	void DescribeFieldsReST(ODesc* d, bool func_args) const;
 
@@ -530,8 +519,7 @@ protected:
 
 class EnumType : public BroType {
 public:
-	EnumType(const string& arg_name);
+	EnumType() : BroType(TYPE_ENUM) { counter = 0; }
-	EnumType(EnumType* e);
 	~EnumType();
 
 	// The value of this name is next internal counter value, starting
@@ -547,15 +535,15 @@ public:
 	bro_int_t Lookup(const string& module_name, const char* name);
 	const char* Lookup(bro_int_t value); // Returns 0 if not found
 
-	string Name() const { return name; }
+	void DescribeReST(ODesc* d, bool roles_only = false) const;
-
-	void DescribeReST(ODesc* d) const;
 
 protected:
-	EnumType() { counter = 0; }
 	DECLARE_SERIAL(EnumType)
 
-	virtual void AddNameInternal(const string& module_name,
+	void AddNameInternal(const string& module_name,
+			const char* name, bro_int_t val, bool is_export);
+
+	void CheckAndAddName(const string& module_name,
 	                     const char* name, bro_int_t val, bool is_export);
 
 	typedef std::map< const char*, bro_int_t, ltstr > NameMap;
@@ -568,31 +556,6 @@ protected:
 	// as a flag to prevent mixing of auto-increment and explicit
 	// enumerator specifications.
 	bro_int_t counter;
-
-	// The name of the enum type is stored for documentation purposes.
-	string name;
-};
-
-class CommentedEnumType: public EnumType {
-public:
-	CommentedEnumType(const string& arg_name) : EnumType(arg_name) {}
-	CommentedEnumType(EnumType* e) : EnumType(e) {}
-	~CommentedEnumType();
-
-	void DescribeReST(ODesc* d) const;
-	void AddComment(const string& module_name, const char* name,
-			std::list<std::string>* comments);
-
-protected:
-	// This overriden method does not install the given ID name into a
-	// scope and it also does not do any kind of checking that the
-	// provided name already exists.
-	void AddNameInternal(const string& module_name, const char* name,
-			bro_int_t val, bool is_export);
-
-	// Comments are only filled when in "documentation mode".
-	typedef std::map< const char*, std::list<std::string>*, ltstr > CommentMap;
-	CommentMap comments;
 };
 
 class VectorType : public BroType {
@@ -617,6 +580,9 @@ protected:
 	BroType* yield_type;
 };
 
+typedef std::map<std::string, std::set<BroType*> > TypeAliasMap;
+extern TypeAliasMap type_aliases;
+
 extern OpaqueType* md5_type;
 extern OpaqueType* sha1_type;
 extern OpaqueType* sha256_type;

src/Var.cc

@@ -10,8 +10,6 @@
 #include "RemoteSerializer.h"
 #include "EventRegistry.h"
 
-extern int generate_documentation;
-
 static Val* init_val(Expr* init, const BroType* t, Val* aggr)
 	{
 	return init->InitVal(t, aggr);
@@ -261,61 +259,26 @@ extern Expr* add_and_assign_local(ID* id, Expr* init, Val* val)
 	return new AssignExpr(new NameExpr(id), init, 0, val);
 	}
 
-void add_type(ID* id, BroType* t, attr_list* attr, int /* is_event */)
+void add_type(ID* id, BroType* t, attr_list* attr)
 	{
-	BroType* tnew = t;
+	string new_type_name = id->Name();
+	string old_type_name = t->GetName();
+	BroType* tnew = 0;
 
-	// In "documentation mode", we'd like to to be able to associate
+	if ( (t->Tag() == TYPE_RECORD || t->Tag() == TYPE_ENUM) &&
-	// an identifier name with a declared type.  Dealing with declared
+	     old_type_name.empty() )
-	// types that are "aliases" to a builtin type requires that the BroType
+		// An extensible type (record/enum) being declared for first time.
-	// is cloned before setting the identifier name that resolves to it.
+		tnew = t;
-	// And still this is not enough to document cases where the declared type
+	else
-	// is an alias for another declared type -- but that's not a natural/common
+		// Clone the type to preserve type name aliasing.
-	// practice.  If documenting that corner case is desired, one way
+		tnew = t->Clone();
-	// is to add an ID* to class ID that tracks aliases and set it here if
-	// t->GetTypeID() is true.
-	if ( generate_documentation )
-		{
-		switch ( t->Tag() ) {
-		// Only "shallow" copy types that may contain records because
-		// we want to be able to see additions to the original record type's
-		// list of fields
-		case TYPE_RECORD:
-			tnew = new RecordType(t->AsRecordType()->Types());
-			break;
-		case TYPE_TABLE:
-			tnew = new TableType(t->AsTableType()->Indices(),
-			                     t->AsTableType()->YieldType());
-			break;
-		case TYPE_VECTOR:
-			tnew = new VectorType(t->AsVectorType()->YieldType());
-			break;
-		case TYPE_FUNC:
-			tnew = new FuncType(t->AsFuncType()->Args(),
-			                    t->AsFuncType()->YieldType(),
-			                    t->AsFuncType()->Flavor());
-			break;
-		default:
-			SerializationFormat* form = new BinarySerializationFormat();
-			form->StartWrite();
-			CloneSerializer ss(form);
-			SerialInfo sinfo(&ss);
-			sinfo.cache = false;
 
-			t->Serialize(&sinfo);
+	type_aliases[new_type_name].insert(tnew);
-			char* data;
-			uint32 len = form->EndWrite(&data);
-			form->StartRead(data, len);
 
-			UnserialInfo uinfo(&ss);
+	if ( new_type_name != old_type_name && ! old_type_name.empty() )
-			uinfo.cache = false;
+		type_aliases[old_type_name].insert(tnew);
-			tnew = t->Unserialize(&uinfo);
 
-			delete [] data;
+	tnew->SetName(id->Name());
-		}
-
-		tnew->SetTypeID(copy_string(id->Name()));
-		}
 
 	id->SetType(tnew);
 	id->MakeType();

src/Var.h

@@ -18,7 +18,7 @@ extern Stmt* add_local(ID* id, BroType* t, init_class c, Expr* init,
 			attr_list* attr, decl_type dt);
 extern Expr* add_and_assign_local(ID* id, Expr* init, Val* val = 0);
 
-extern void add_type(ID* id, BroType* t, attr_list* attr, int is_event);
+extern void add_type(ID* id, BroType* t, attr_list* attr);
 
 extern void begin_func(ID* id, const char* module_name, function_flavor flavor,
 		       int is_redef, FuncType* t);

src/broxygen/CMakeLists.txt

@@ -0,0 +1,25 @@
+# See the file "COPYING" in the main distribution directory for copyright.
+
+include(BroSubdir)
+
+include_directories(BEFORE
+                    ${CMAKE_CURRENT_SOURCE_DIR}
+                    ${CMAKE_CURRENT_BINARY_DIR}
+)
+
+set(broxygen_SRCS
+    Manager.cc
+    Info.h
+    PackageInfo.cc
+    ScriptInfo.cc
+    IdentifierInfo.cc
+    Target.cc
+    Configuration.cc
+    ReStructuredTextTable.cc
+    utils.cc
+)
+
+bif_target(broxygen.bif)
+bro_add_subdir_library(broxygen ${broxygen_SRCS})
+
+add_dependencies(bro_broxygen generate_outputs)

src/broxygen/Configuration.cc

@@ -0,0 +1,103 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#include "Configuration.h"
+#include "utils.h"
+
+#include "util.h"
+#include "Reporter.h"
+
+#include <fstream>
+#include <vector>
+#include <algorithm>
+#include <errno.h>
+
+using namespace broxygen;
+using namespace std;
+
+static TargetFactory create_target_factory()
+	{
+	TargetFactory rval;
+	rval.Register<PackageIndexTarget>("package_index");
+	rval.Register<PackageTarget>("package");
+	rval.Register<ProtoAnalyzerTarget>("proto_analyzer");
+	rval.Register<FileAnalyzerTarget>("file_analyzer");
+	rval.Register<ScriptSummaryTarget>("script_summary");
+	rval.Register<ScriptIndexTarget>("script_index");
+	rval.Register<ScriptTarget>("script");
+	rval.Register<IdentifierTarget>("identifier");
+	return rval;
+	}
+
+Config::Config(const string& arg_file, const string& delim)
+	: file(arg_file), targets(), target_factory(create_target_factory())
+	{
+	if ( file.empty() )
+		return;
+
+	ifstream f(file.c_str());
+
+	if ( ! f.is_open() )
+		reporter->FatalError("failed to open Broxygen config file '%s': %s",
+		                     file.c_str(), strerror(errno));
+
+	string line;
+	unsigned int line_number = 0;
+
+	while ( getline(f, line) )
+		{
+		++line_number;
+		vector<string> tokens;
+		tokenize_string(line, delim, &tokens);
+		tokens.erase(remove(tokens.begin(), tokens.end(), ""), tokens.end());
+
+		if ( tokens.empty() )
+			// Blank line.
+			continue;
+
+		if ( ! tokens[0].empty() && tokens[0][0] == '#' )
+			// Comment
+			continue;
+
+		if ( tokens.size() != 3 )
+			reporter->FatalError("malformed Broxygen target in %s:%u: %s",
+			                     file.c_str(), line_number, line.c_str());
+
+		Target* target = target_factory.Create(tokens[0], tokens[2], tokens[1]);
+
+		if ( ! target )
+			reporter->FatalError("unkown Broxygen target type: %s",
+			                     tokens[0].c_str());
+
+		targets.push_back(target);
+		}
+
+	if ( f.bad() )
+		reporter->InternalError("error reading Broxygen config file '%s': %s",
+		                        file.c_str(), strerror(errno));
+	}
+
+Config::~Config()
+	{
+	for ( size_t i = 0; i < targets.size(); ++i )
+		delete targets[i];
+	}
+
+void Config::FindDependencies(const vector<Info*>& infos)
+	{
+	for ( size_t i = 0; i < targets.size(); ++i )
+		targets[i]->FindDependencies(infos);
+	}
+
+void Config::GenerateDocs() const
+	{
+	for ( size_t i = 0; i < targets.size(); ++i )
+		targets[i]->Generate();
+	}
+
+time_t Config::GetModificationTime() const
+	{
+	if ( file.empty() )
+		return 0;
+
+	return broxygen::get_mtime(file);
+	}

src/broxygen/Configuration.h

@@ -0,0 +1,63 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#ifndef BROXYGEN_CONFIGURATION_H
+#define BROXYGEN_CONFIGURATION_H
+
+#include "Info.h"
+#include "Target.h"
+
+#include <string>
+#include <vector>
+
+namespace broxygen {
+
+/**
+ * Manages the generation of reStructuredText documents corresponding to
+ * particular targets that are specified in a config file.  The config file
+ * is a simple list of one target per line, with the target format being
+ * a tab-delimited list of target-type, target-pattern, and target-output-file.
+ */
+class Config {
+
+public:
+
+	/**
+	 * Read a Broxygen configuration file, parsing all targets in it.
+	 * @param file The file containing a list of Broxygen targets.  If it's
+	 * an empty string most methods are a no-op.
+	 * @param delim The delimiter between target fields.
+	 */
+	Config(const std::string& file, const std::string& delim = "\t");
+
+	/**
+	  * Destructor, cleans up targets created when parsing config file.
+	  */
+	~Config();
+
+	/**
+	 * Resolves dependency information for each target.
+	 * @param infos All known information objects for documentable things.
+	 */
+	void FindDependencies(const std::vector<Info*>& infos);
+
+	/**
+	 * Build each Broxygen target (i.e. write out the reST documents to disk).
+	 */
+	void GenerateDocs() const;
+
+	/**
+	 * @return The modification time of the config file, or 0 if config
+	 * file was specified by an empty string.
+	 */
+	time_t GetModificationTime() const;
+
+private:
+
+	std::string file;
+	std::vector<Target*> targets;
+	TargetFactory target_factory;
+};
+
+} // namespace broxygen
+
+#endif

src/broxygen/IdentifierInfo.cc

@@ -0,0 +1,148 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#include "IdentifierInfo.h"
+#include "utils.h"
+
+#include "Desc.h"
+#include "Val.h"
+
+using namespace std;
+using namespace broxygen;
+
+IdentifierInfo::IdentifierInfo(ID* arg_id, ScriptInfo* script)
+	: Info(),
+	  comments(), id(arg_id), initial_val_desc(), redefs(), fields(),
+	  last_field_seen(), declaring_script(script)
+	{
+	Ref(id);
+
+	if ( id->ID_Val() )
+		{
+		ODesc d;
+		id->ID_Val()->Describe(&d);
+		initial_val_desc = d.Description();
+		}
+	}
+
+IdentifierInfo::~IdentifierInfo()
+	{
+	Unref(id);
+
+	for ( redef_list::const_iterator it = redefs.begin(); it != redefs.end();
+	      ++it )
+		delete *it;
+
+	for ( record_field_map::const_iterator it = fields.begin();
+	      it != fields.end(); ++it )
+		delete it->second;
+	}
+
+void IdentifierInfo::AddRedef(const string& script,
+                                  const vector<string>& comments)
+	{
+	Redefinition* redef = new Redefinition();
+	redef->from_script = script;
+
+	if ( id->ID_Val() )
+		{
+		ODesc d;
+		id->ID_Val()->Describe(&d);
+		redef->new_val_desc = d.Description();
+		}
+
+	redef->comments = comments;
+	redefs.push_back(redef);
+	}
+
+void IdentifierInfo::AddRecordField(const TypeDecl* field,
+				    const string& script,
+				    vector<string>& comments)
+	{
+	RecordField* rf = new RecordField();
+	rf->field = new TypeDecl(*field);
+	rf->from_script = script;
+	rf->comments = comments;
+	fields[rf->field->id] = rf;
+	last_field_seen = rf;
+	}
+
+vector<string> IdentifierInfo::GetComments() const
+	{
+	return comments;
+	}
+
+vector<string> IdentifierInfo::GetFieldComments(const string& field) const
+	{
+	record_field_map::const_iterator it = fields.find(field);
+
+	if ( it == fields.end() )
+		return vector<string>();
+
+	return it->second->comments;
+	}
+
+list<IdentifierInfo::Redefinition>
+IdentifierInfo::GetRedefs(const string& from_script) const
+	{
+	list<Redefinition> rval;
+
+	for ( redef_list::const_iterator it = redefs.begin(); it != redefs.end();
+	      ++it )
+		{
+		if ( from_script == (*it)->from_script )
+			rval.push_back(*(*it));
+		}
+
+	return rval;
+	}
+
+string IdentifierInfo::GetDeclaringScriptForField(const string& field) const
+	{
+	record_field_map::const_iterator it = fields.find(field);
+
+	if ( it == fields.end() )
+		return "";
+
+	return it->second->from_script;
+	}
+
+string IdentifierInfo::DoReStructuredText(bool roles_only) const
+	{
+	ODesc d;
+	d.SetIndentSpaces(3);
+	d.SetQuotes(true);
+	id->DescribeReST(&d, roles_only);
+
+	if ( comments.empty() )
+		return d.Description();
+
+	d.ClearIndentLevel();
+	d.PushIndent();
+
+	for ( size_t i = 0; i < comments.size(); ++i )
+		{
+		if ( i > 0 )
+			d.NL();
+
+		if ( IsFunc(id->Type()->Tag()) )
+			{
+			string s = comments[i];
+
+			if ( broxygen::prettify_params(s) )
+				d.NL();
+
+			d.Add(s.c_str());
+			}
+		else
+			d.Add(comments[i].c_str());
+		}
+
+	return d.Description();
+	}
+
+time_t IdentifierInfo::DoGetModificationTime() const
+	{
+	// Could probably get away with just checking the set of scripts that
+	// contributed to the ID declaration/redefinitions, but this is easier...
+	return declaring_script->GetModificationTime();
+	}

src/broxygen/IdentifierInfo.h

@@ -0,0 +1,164 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#ifndef BROXYGEN_IDENTIFIERINFO_H
+#define BROXYGEN_IDENTIFIERINFO_H
+
+#include "Info.h"
+#include "ScriptInfo.h"
+
+#include "ID.h"
+#include "Type.h"
+
+#include <string>
+#include <vector>
+#include <list>
+#include <map>
+
+namespace broxygen {
+
+class ScriptInfo;
+
+/**
+ * Information regarding a script-level identifier and its documentation.
+ */
+class IdentifierInfo : public Info {
+
+public:
+
+	/**
+	 * Create a new identifier info object.
+	 * @param id The script-level identifier.
+	 * @param script The info object associated with the script in which \a id
+	 * is declared.
+	 */
+	IdentifierInfo(ID* id, ScriptInfo* script);
+
+	/**
+	 * Dtor.  Releases any references to script-level objects.
+	 */
+	~IdentifierInfo();
+
+	/**
+	 * Add a comment associated with the identifier.  If the identifier is a
+	 * record type and it's in the middle of parsing fields, the comment is
+	 * associated with the last field that was parsed.
+	 * @param comment A string extracted from Broxygen-style comment.
+	 */
+	void AddComment(const std::string& comment)
+		{ last_field_seen ? last_field_seen->comments.push_back(comment)
+		                  : comments.push_back(comment); }
+
+	/**
+	 * Associate several comments with the identifier.  They will be appended
+	 * to the end of the list of any current comments.
+	 * @param cmtns A vector of comments to associate.
+	 */
+	void AddComments(const std::vector<std::string>& cmtns)
+		{ comments.insert(comments.end(), cmtns.begin(), cmtns.end()); }
+
+	/**
+	 * Register a redefinition of the identifier.
+	 * @param from_script The script in which the redef occurred.
+	 * @param comments Comments associated with the redef statement.
+	 */
+	void AddRedef(const std::string& from_script,
+	              const std::vector<std::string>& comments);
+
+	/**
+	 * Register a record field associated with the identifier
+	 * (which is implicitly a record type).
+	 * @param field The name/type information of the field.
+	 * @param script The script in which the field was declared.  This may
+	 * differ from the script in which a record type is declared due to redefs.
+	 * @param comments Comments associated with the record field.
+	 */
+	void AddRecordField(const TypeDecl* field, const std::string& script,
+	                    std::vector<std::string>& comments);
+
+	/**
+	 * Signals that a record type has been completely parsed.  This resets
+	 * internal tracking of the last record field seen so that "##<"-style
+	 * comments are correctly associated.
+	 */
+	void CompletedTypeDecl()
+		{ last_field_seen = 0; }
+
+	/**
+	 * @return the script-level ID tracked by this info object.
+	 */
+	ID* GetID() const
+		{ return id; }
+
+	/**
+	 * @return The script which declared the script-level identifier.
+	 */
+	ScriptInfo* GetDeclaringScript() const
+	    { return declaring_script; }
+
+	/**
+	 * @param field A record field name.
+	 * @return The script which declared the record field name.
+	 */
+	std::string GetDeclaringScriptForField(const std::string& field) const;
+
+	/**
+	 * @return All Broxygen comments associated with the identifier.
+	 */
+	std::vector<std::string> GetComments() const;
+
+	/**
+	 * @param field A record field name.
+	 * @return All Broxygen comments associated with the record field.
+	 */
+	std::vector<std::string> GetFieldComments(const std::string& field) const;
+
+	/**
+	 * Tracks useful information related to a redef.
+	 */
+	struct Redefinition {
+		std::string from_script; /**< Name of script doing the redef. */
+		std::string new_val_desc; /**< Description of new value bound to ID. */
+		std::vector<std::string> comments; /**< Broxygen comments on redef. */
+	};
+
+	/**
+	 * Get a list of information about redefinitions of the identifier within
+	 * a particular script.
+	 * @param from_script The name of a script in which to look for redefs.
+	 * @return A list of redefs that occurred in \a from_script.
+	 */
+	std::list<Redefinition> GetRedefs(const std::string& from_script) const;
+
+private:
+
+	time_t DoGetModificationTime() const;
+
+	std::string DoName() const
+		{ return id->Name(); }
+
+	std::string DoReStructuredText(bool roles_only) const;
+
+	struct RecordField {
+		~RecordField()
+			{ delete field; }
+
+		TypeDecl* field;
+		std::string from_script;
+		std::vector<std::string> comments;
+	};
+
+	typedef std::list<Redefinition*> redef_list;
+	typedef std::map<std::string, RecordField*> record_field_map;
+
+	std::vector<std::string> comments;
+	ID* id;
+	std::string initial_val_desc;
+	redef_list redefs;
+	record_field_map fields;
+	RecordField* last_field_seen;
+	ScriptInfo* declaring_script;
+};
+
+} // namespace broxygen
+
+#endif

src/broxygen/Info.h

@@ -0,0 +1,73 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#ifndef BROXYGEN_INFO_H
+#define BROXYGEN_INFO_H
+
+#include <string>
+#include <ctime>
+
+namespace broxygen {
+
+/**
+ * Abstract base class for any thing that Broxygen can document.
+ */
+class Info {
+
+public:
+
+	/**
+	 * Ctor.
+	 */
+	Info()
+		{ }
+
+	/**
+	 * Dtor.
+	 */
+	virtual ~Info()
+		{ }
+
+	/**
+	 * @return The time any information related to the object was last modified.
+	 */
+	time_t GetModificationTime() const
+		{ return DoGetModificationTime(); }
+
+	/**
+	 * @return A unique name for the documentable object.
+	 */
+	std::string Name() const
+		{ return DoName(); }
+
+	/**
+	 * Get a reST representation of the object and any associated documentation.
+	 * @param roles_only True if the reST should only use cross-referencing role
+	 * syntax to refer itself instead of using a directive (which declares this
+	 * reST the authoritative "anchor" for cross-references).
+	 * @return A reST representation of the object and associated documentation.
+	 */
+	std::string ReStructuredText(bool roles_only = false) const
+		{ return DoReStructuredText(roles_only); }
+
+	/**
+	 * Perform any remaining info gathering/initialization that can only be done
+	 * after all script parsing is complete.
+	 */
+	void InitPostScript()
+		{ DoInitPostScript(); }
+
+private:
+
+	virtual time_t DoGetModificationTime() const = 0;
+
+	virtual std::string DoName() const = 0;
+
+	virtual std::string DoReStructuredText(bool roles_only) const = 0;
+
+	virtual void DoInitPostScript()
+		{ }
+};
+
+} // namespace broxygen
+
+#endif

src/broxygen/Manager.cc

@@ -0,0 +1,390 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#include "Manager.h"
+#include "util.h"
+
+#include <utility>
+#include <cstdlib>
+
+using namespace broxygen;
+using namespace std;
+
+static void DbgAndWarn(const char* msg)
+	{
+	reporter->InternalWarning("%s", msg);
+	DBG_LOG(DBG_BROXYGEN, "%s", msg);
+	}
+
+static string RemoveLeadingSpace(const string& s)
+	{
+	if ( s.empty() || s[0] != ' ' )
+		return s;
+
+	// Treat "##Text" and "## Text" the same, so that a single space doesn't
+	// cause reST formatting to think the later is indented a level.
+	string rval = s;
+	rval.erase(0, 1);
+	return rval;
+	}
+
+Manager::Manager(const string& arg_config, const string& bro_command)
+	: disabled(), comment_buffer(), comment_buffer_map(), packages(), scripts(),
+	  identifiers(), all_info(), last_identifier_seen(), incomplete_type(),
+	  enum_mappings(), config(arg_config), bro_mtime()
+	{
+	if ( getenv("BRO_DISABLE_BROXYGEN") )
+		disabled = true;
+
+	string path_to_bro = find_file(bro_command, getenv("PATH"));
+	struct stat s;
+
+	if ( path_to_bro.empty() || stat(path_to_bro.c_str(), &s) < 0 )
+		reporter->InternalError("Broxygen can't get mtime of bro binary %s: %s",
+		                        path_to_bro.c_str(), strerror(errno));
+
+	bro_mtime = s.st_mtime;
+	}
+
+Manager::~Manager()
+	{
+	for ( size_t i = 0; i < all_info.size(); ++i )
+		delete all_info[i];
+	}
+
+void Manager::InitPreScript()
+	{
+	if ( disabled )
+		return;
+	}
+
+void Manager::InitPostScript()
+	{
+	if ( disabled )
+		return;
+
+	for ( size_t i = 0; i < all_info.size(); ++i )
+		all_info[i]->InitPostScript();
+
+	config.FindDependencies(all_info);
+	}
+
+void Manager::GenerateDocs() const
+	{
+	if ( disabled )
+		return;
+
+	config.GenerateDocs();
+	}
+
+void Manager::Script(const string& path)
+	{
+	if ( disabled )
+		return;
+
+	string name = without_bropath_component(path);
+
+	if ( scripts.GetInfo(name) )
+		{
+		DbgAndWarn(fmt("Duplicate script documentation: %s", name.c_str()));
+		return;
+		}
+
+	ScriptInfo* info = new ScriptInfo(name, path);
+	scripts.map[name] = info;
+	all_info.push_back(info);
+	DBG_LOG(DBG_BROXYGEN, "Made ScriptInfo %s", name.c_str());
+
+	if ( ! info->IsPkgLoader() )
+		return;
+
+	name = SafeDirname(name).result;
+
+	if ( packages.GetInfo(name) )
+		{
+		DbgAndWarn(fmt("Duplicate package documentation: %s", name.c_str()));
+		return;
+		}
+
+	PackageInfo* pkginfo = new PackageInfo(name);
+	packages.map[name] = pkginfo;
+	all_info.push_back(pkginfo);
+	DBG_LOG(DBG_BROXYGEN, "Made PackageInfo %s", name.c_str());
+	}
+
+void Manager::ScriptDependency(const string& path, const string& dep)
+	{
+	if ( disabled )
+		return;
+
+	if ( dep.empty() )
+		{
+		DbgAndWarn(fmt("Empty script doc dependency: %s", path.c_str()));
+		return;
+		}
+
+	string name = without_bropath_component(path);
+	string depname = without_bropath_component(dep);
+	ScriptInfo* script_info = scripts.GetInfo(name);
+
+	if ( ! script_info )
+		{
+		DbgAndWarn(fmt("Failed to add script doc dependency %s for %s",
+		               depname.c_str(), name.c_str()));
+		return;
+		}
+
+	script_info->AddDependency(depname);
+	DBG_LOG(DBG_BROXYGEN, "Added script dependency %s for %s",
+	        depname.c_str(), name.c_str());
+
+	for ( size_t i = 0; i < comment_buffer.size(); ++i )
+		DbgAndWarn(fmt("Discarded extraneous Broxygen comment: %s",
+		               comment_buffer[i].c_str()));
+	}
+
+void Manager::ModuleUsage(const string& path, const string& module)
+	{
+	if ( disabled )
+		return;
+
+	string name = without_bropath_component(path);
+	ScriptInfo* script_info = scripts.GetInfo(name);
+
+	if ( ! script_info )
+		{
+		DbgAndWarn(fmt("Failed to add module usage %s in %s",
+		               module.c_str(), name.c_str()));
+		return;
+		}
+
+	script_info->AddModule(module);
+	DBG_LOG(DBG_BROXYGEN, "Added module usage %s in %s",
+	        module.c_str(), name.c_str());
+	}
+
+IdentifierInfo* Manager::CreateIdentifierInfo(ID* id, ScriptInfo* script)
+	{
+	IdentifierInfo* rval = new IdentifierInfo(id, script);
+
+	rval->AddComments(comment_buffer);
+	comment_buffer.clear();
+
+	comment_buffer_map_t::iterator it = comment_buffer_map.find(id->Name());
+
+	if ( it != comment_buffer_map.end() )
+		{
+		rval->AddComments(it->second);
+		comment_buffer_map.erase(it);
+		}
+
+	all_info.push_back(rval);
+	identifiers.map[id->Name()] = rval;
+	last_identifier_seen = rval;
+
+	if ( script )
+		script->AddIdentifierInfo(rval);
+
+	return rval;
+	}
+
+void Manager::StartType(ID* id)
+	{
+	if ( disabled )
+		return;
+
+	if ( id->GetLocationInfo() == &no_location )
+		{
+		DbgAndWarn(fmt("Can't document %s, no location available", id->Name()));
+		return;
+		}
+
+	string script = without_bropath_component(id->GetLocationInfo()->filename);
+	ScriptInfo* script_info = scripts.GetInfo(script);
+
+	if ( ! script_info )
+		{
+		DbgAndWarn(fmt("Can't document identifier %s, lookup of %s failed",
+		               id->Name(), script.c_str()));
+		return;
+		}
+
+	incomplete_type = CreateIdentifierInfo(id, script_info);
+	DBG_LOG(DBG_BROXYGEN, "Made IdentifierInfo (incomplete) %s, in %s",
+	        id->Name(), script.c_str());
+	}
+
+static bool IsEnumType(ID* id)
+	{
+	return id->AsType() ? id->AsType()->Tag() == TYPE_ENUM : false;
+	}
+
+void Manager::Identifier(ID* id)
+	{
+	if ( disabled )
+		return;
+
+	if ( incomplete_type )
+		{
+		if ( incomplete_type->Name() == id->Name() )
+			{
+			DBG_LOG(DBG_BROXYGEN, "Finished document for type %s", id->Name());
+			incomplete_type->CompletedTypeDecl();
+			incomplete_type = 0;
+			return;
+			}
+
+		if ( IsEnumType(incomplete_type->GetID()) )
+			enum_mappings[id->Name()] = incomplete_type->GetID()->Name();
+		}
+
+	IdentifierInfo* id_info = identifiers.GetInfo(id->Name());
+
+	if ( id_info )
+		{
+		if ( IsFunc(id_info->GetID()->Type()->Tag()) )
+			{
+			// Function may already been seen (declaration versus body).
+			id_info->AddComments(comment_buffer);
+			comment_buffer.clear();
+			return;
+			}
+
+		DbgAndWarn(fmt("Duplicate identifier documentation: %s", id->Name()));
+		return;
+		}
+
+	if ( id->GetLocationInfo() == &no_location )
+		{
+		// Internally-created identifier (e.g. file/proto analyzer enum tags).
+		// Handled specially since they don't have a script location.
+		DBG_LOG(DBG_BROXYGEN, "Made internal IdentifierInfo %s",
+		        id->Name());
+		CreateIdentifierInfo(id, 0);
+		return;
+		}
+
+	string script = without_bropath_component(id->GetLocationInfo()->filename);
+	ScriptInfo* script_info = scripts.GetInfo(script);
+
+	if ( ! script_info )
+		{
+		DbgAndWarn(fmt("Can't document identifier %s, lookup of %s failed",
+		               id->Name(), script.c_str()));
+		return;
+		}
+
+	CreateIdentifierInfo(id, script_info);
+	DBG_LOG(DBG_BROXYGEN, "Made IdentifierInfo %s, in script %s",
+	        id->Name(), script.c_str());
+	}
+
+void Manager::RecordField(const ID* id, const TypeDecl* field,
+			  const string& path)
+	{
+	if ( disabled )
+		return;
+
+	IdentifierInfo* idd = identifiers.GetInfo(id->Name());
+
+	if ( ! idd )
+		{
+		DbgAndWarn(fmt("Can't document record field %s, unknown record: %s",
+		               field->id, id->Name()));
+		return;
+		}
+
+	string script = without_bropath_component(path);
+	idd->AddRecordField(field, script, comment_buffer);
+	comment_buffer.clear();
+	DBG_LOG(DBG_BROXYGEN, "Document record field %s, identifier %s, script %s",
+	        field->id, id->Name(), script.c_str());
+	}
+
+void Manager::Redef(const ID* id, const string& path)
+	{
+	if ( disabled )
+		return;
+
+	if ( path == "<params>" )
+		// This is a redef defined on the command line.
+		return;
+
+	IdentifierInfo* id_info = identifiers.GetInfo(id->Name());
+
+	if ( ! id_info )
+		{
+		DbgAndWarn(fmt("Can't document redef of %s, identifier lookup failed",
+		               id->Name()));
+		return;
+		}
+
+	string from_script = without_bropath_component(path);
+	ScriptInfo* script_info = scripts.GetInfo(from_script);
+
+	if ( ! script_info )
+		{
+		DbgAndWarn(fmt("Can't document redef of %s, lookup of %s failed",
+		               id->Name(), from_script.c_str()));
+		return;
+		}
+
+	id_info->AddRedef(from_script, comment_buffer);
+	script_info->AddRedef(id_info);
+	comment_buffer.clear();
+	last_identifier_seen = id_info;
+	DBG_LOG(DBG_BROXYGEN, "Added redef of %s from %s",
+	        id->Name(), from_script.c_str());
+	}
+
+void Manager::SummaryComment(const string& script, const string& comment)
+	{
+	if ( disabled )
+		return;
+
+	string name = without_bropath_component(script);
+	ScriptInfo* info = scripts.GetInfo(name);
+
+	if ( info )
+		info->AddComment(RemoveLeadingSpace(comment));
+	else
+		DbgAndWarn(fmt("Lookup of script %s failed for summary comment %s",
+		               name.c_str(), comment.c_str()));
+	}
+
+void Manager::PreComment(const string& comment)
+	{
+	if ( disabled )
+		return;
+
+	comment_buffer.push_back(RemoveLeadingSpace(comment));
+	}
+
+void Manager::PostComment(const string& comment, const string& id_hint)
+	{
+	if ( disabled )
+		return;
+
+	if ( id_hint.empty() )
+		{
+		if ( last_identifier_seen )
+			last_identifier_seen->AddComment(RemoveLeadingSpace(comment));
+		else
+			DbgAndWarn(fmt("Discarded unassociated Broxygen comment %s",
+			               comment.c_str()));
+
+		return;
+		}
+
+	if ( last_identifier_seen &&
+	     last_identifier_seen->Name() == id_hint )
+		last_identifier_seen->AddComment(RemoveLeadingSpace(comment));
+	else
+		// Assume identifier it's associated w/ is coming later.
+		comment_buffer_map[id_hint].push_back(RemoveLeadingSpace(comment));
+	}
+
+string Manager::GetEnumTypeName(const string& id) const
+	{
+	map<string, string>::const_iterator it = enum_mappings.find(id);
+	return it == enum_mappings.end() ? "" : it->second;
+	}

src/broxygen/Manager.h

@@ -0,0 +1,265 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#ifndef BROXYGEN_MANAGER_H
+#define BROXYGEN_MANAGER_H
+
+#include "Configuration.h"
+#include "Info.h"
+#include "PackageInfo.h"
+#include "ScriptInfo.h"
+#include "IdentifierInfo.h"
+
+#include "Reporter.h"
+#include "ID.h"
+#include "Type.h"
+#include "Val.h"
+
+#include <string>
+#include <vector>
+#include <map>
+#include <ctime>
+#include <sys/stat.h>
+#include <errno.h>
+
+namespace broxygen {
+
+/**
+ * Map of info objects.  Just a wrapper around std::map to improve code
+ * readability (less typedefs for specific map types and not having to use
+ * iterators directly to find a particular info object).
+ */
+template<class T>
+struct InfoMap {
+	typedef std::map<std::string, T*> map_type;
+
+	/**
+	 * @param name Name of an info object to retrieve.
+	 * @return The info object associated with \a name.
+	 */
+	T* GetInfo(const std::string& name) const
+		{
+		typename map_type::const_iterator it = map.find(name);
+		return it == map.end() ? 0 : it->second;
+		}
+
+	map_type map;
+};
+
+/**
+ * Manages all documentation tracking and generation.
+ */
+class Manager {
+
+public:
+
+	/**
+	 * Ctor.
+	 * @param config Path to a Broxygen config file if documentation is to be
+	 * written to disk.
+	 * @param bro_command The command used to invoke the bro process.
+	 * It's used when checking for out-of-date targets.  If the bro binary is
+	 * newer then a target, it needs to be rebuilt.
+	 */
+	Manager(const std::string& config, const std::string& bro_command);
+
+	/**
+	 * Dtor.
+	 */
+	~Manager();
+
+	/**
+	 * Do initialization that needs to happen before scripts are parsed.
+	 * Currently nothing outside of what's done in ctor is needed.
+	 */
+	void InitPreScript();
+
+	/**
+	 * Do initialization that needs to happen after scripts are parsed.
+	 * This is primarly dependency resolution/filtering.
+	 */
+	void InitPostScript();
+
+	/**
+	 * Builds all Broxygen targets specified by config file and write out
+	 * documentation to disk.
+	 */
+	void GenerateDocs() const;
+
+	/**
+	 * Register Bro script for which information/documentation will be gathered.
+	 * @param path Absolute path to Bro script.
+	 */
+	void Script(const std::string& path);
+
+	/**
+	 * Register Bro script dependency ("@load").
+	 * @param path Absolute path to a Bro script.
+	 * @param dep Absolute path to a Bro script being "@load"d from script given
+	 * by \a path.
+	 */
+	void ScriptDependency(const std::string& path, const std::string& dep);
+
+	/**
+	 * Register a module usage (script may export identifiers in to the
+	 * module namespace).
+	 * @param path Absolute path to a Bro script.
+	 * @param module The module which script given by \a path is using.
+	 */
+	void ModuleUsage(const std::string& path, const std::string& module);
+
+	/**
+	 * Signal that a record or enum type is now being parsed.
+	 * @param id The record or enum type identifier.
+	 */
+	void StartType(ID* id);
+
+	/**
+	 * Register a script-level identifier for which information/documentation
+	 * will be gathered.
+	 * @param id The script-level identifier.
+	 */
+	void Identifier(ID* id);
+
+	/**
+	 * Register a record-field for which information/documentation will be
+	 * gathered.
+	 * @param id The identifier of the record type which has the field.
+	 * @param field The field name/type information.
+	 * @param path Absolute path to a Bro script in which this field is
+	 * declared.  This can be different from the place where the record type
+	 * is declared due to redefs.
+	 */
+	void RecordField(const ID* id, const TypeDecl* field,
+	                 const std::string& path);
+
+	/**
+	 * Register a redefinition of a particular identifier.
+	 * @param id The identifier being redef'd.
+	 * @param path Absolute path to a Bro script doing the redef.
+	 */
+	void Redef(const ID* id, const std::string& path);
+
+	/**
+	 * Register Broxygen script summary content.
+	 * @param path Absolute path to a Bro script.
+	 * @param comment Broxygen-style summary comment ("##!") to associate with
+	 * script given by \a path.
+	 */
+	void SummaryComment(const std::string& path, const std::string& comment);
+
+	/**
+	 * Register a Broxygen comment ("##") for an upcoming identifier (i.e.
+	 * this content is buffered and consumed by next identifier/field
+	 * declaration.
+	 * @param comment Content of the Broxygen comment.
+	 */
+	void PreComment(const std::string& comment);
+
+	/**
+	 * Register a Broxygen comment ("##<") for the last identifier seen.
+	 * @param comment Content of the Broxygen comment.
+	 * @param identifier_hint Expected name of identifier with which to
+	 * associate \a comment.
+	 */
+	void PostComment(const std::string& comment,
+	                 const std::string& identifier_hint = "");
+
+	/**
+	 * @param id Name of script-level enum identifier.
+	 * @return The name of the enum's type.
+	 */
+	std::string GetEnumTypeName(const std::string& id) const;
+
+	/**
+	 * @param name Name of a script-level identifier.
+	 * @return an identifier info object associated with \a name or a null
+	 * pointer if it's not a known identifier.
+	 */
+	IdentifierInfo* GetIdentifierInfo(const std::string& name) const
+	    { return identifiers.GetInfo(name); }
+
+	/**
+	 * @param name Name of a Bro script ("normalized" to be a path relative
+	 * to a component within BROPATH).
+	 * @return a script info object associated with \a name or a null pointer
+	 * if it's not a known script name.
+	 */
+	ScriptInfo* GetScriptInfo(const std::string& name) const
+	    { return scripts.GetInfo(name); }
+
+	/**
+	 * @param name Nmae of a Bro script package ("normalized" to be a path
+	 * relative to a component within BROPATH).
+	 * @return a package info object assocated with \a name or a null pointer
+	 * if it's not a known package name.
+	 */
+	PackageInfo* GetPackageInfo(const std::string& name) const
+	    { return packages.GetInfo(name); }
+
+	/**
+	 * Check if a Broxygen target is up-to-date.
+	 * @param target_file output file of a Broxygen target.
+	 * @param dependencies all dependencies of the target.
+	 * @return true if modification time of \a target_file is newer than
+	 * modification time of Bro binary, Broxygen config file, and all
+	 * dependencies, else false.
+	 */
+	template <class T>
+	bool IsUpToDate(const std::string& target_file,
+	                const std::vector<T*>& dependencies) const;
+
+private:
+
+	typedef std::vector<std::string> comment_buffer_t;
+	typedef std::map<std::string, comment_buffer_t> comment_buffer_map_t;
+
+	IdentifierInfo* CreateIdentifierInfo(ID* id, ScriptInfo* script);
+
+	bool disabled;
+	comment_buffer_t comment_buffer; // For whatever next identifier comes in.
+	comment_buffer_map_t comment_buffer_map; // For a particular identifier.
+	InfoMap<PackageInfo> packages;
+	InfoMap<ScriptInfo> scripts;
+	InfoMap<IdentifierInfo> identifiers;
+	std::vector<Info*> all_info;
+	IdentifierInfo* last_identifier_seen;
+	IdentifierInfo* incomplete_type;
+	std::map<std::string, std::string> enum_mappings; // enum id -> enum type id
+	Config config;
+	time_t bro_mtime;
+};
+
+template <class T>
+bool Manager::IsUpToDate(const string& target_file,
+                         const vector<T*>& dependencies) const
+	{
+	struct stat s;
+
+	if ( stat(target_file.c_str(), &s) < 0 )
+		{
+		if ( errno == ENOENT )
+			// Doesn't exist.
+			return false;
+
+		reporter->InternalError("Broxygen failed to stat target file '%s': %s",
+		                        target_file.c_str(), strerror(errno));
+		}
+
+	if ( difftime(bro_mtime, s.st_mtime) > 0 )
+		return false;
+
+	if ( difftime(config.GetModificationTime(), s.st_mtime) > 0 )
+		return false;
+
+	for ( size_t i = 0; i < dependencies.size(); ++i )
+		if ( difftime(dependencies[i]->GetModificationTime(), s.st_mtime) > 0 )
+			return false;
+
+	return true;
+	}
+
+} // namespace broxygen
+
+extern broxygen::Manager* broxygen_mgr;
+
+#endif

src/broxygen/PackageInfo.cc

@@ -0,0 +1,58 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#include "PackageInfo.h"
+#include "utils.h"
+
+#include "Reporter.h"
+
+#include <fstream>
+#include <errno.h>
+
+using namespace std;
+using namespace broxygen;
+
+PackageInfo::PackageInfo(const string& arg_name)
+    : Info(),
+      pkg_name(arg_name), readme()
+	{
+	string readme_file = find_file(pkg_name + "/README", bro_path());
+
+	if ( readme_file.empty() )
+		return;
+
+	ifstream f(readme_file.c_str());
+
+	if ( ! f.is_open() )
+		reporter->InternalWarning("Broxygen failed to open '%s': %s",
+		                          readme_file.c_str(), strerror(errno));
+
+	string line;
+
+	while ( getline(f, line) )
+		readme.push_back(line);
+
+	if ( f.bad() )
+		reporter->InternalWarning("Broxygen error reading '%s': %s",
+		                          readme_file.c_str(), strerror(errno));
+	}
+
+string PackageInfo::DoReStructuredText(bool roles_only) const
+	{
+	string rval = fmt(":doc:`%s </scripts/%s/index>`\n\n", pkg_name.c_str(),
+	                  pkg_name.c_str());
+
+	for ( size_t i = 0; i < readme.size(); ++i )
+		rval += "   " + readme[i] + "\n";
+
+	return rval;
+	}
+
+time_t PackageInfo::DoGetModificationTime() const
+	{
+	string readme_file = find_file(pkg_name + "/README", bro_path());
+
+	if ( readme_file.empty() )
+		return 0;
+
+	return broxygen::get_mtime(readme_file);
+	}

src/broxygen/PackageInfo.h

@@ -0,0 +1,50 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#ifndef BROXYGEN_PACKAGEINFO_H
+#define BROXYGEN_PACKAGEINFO_H
+
+#include "Info.h"
+
+#include <string>
+#include <vector>
+
+namespace broxygen {
+
+/**
+ * Information about a Bro script package.
+ */
+class PackageInfo : public Info {
+
+public:
+
+	/**
+	 * Ctor.
+	 * @param name The name of the Bro script package (relative path from a
+	 * component within BROPATH.
+	 */
+	PackageInfo(const std::string& name);
+
+	/**
+	 * @return The content of the package's README file, each line being
+	 * an element in the returned vector.  If the package has no README, the
+	 * vector is empty.
+	 */
+	std::vector<std::string> GetReadme() const
+		{ return readme; }
+
+private:
+
+	time_t DoGetModificationTime() const;
+
+	std::string DoName() const
+		{ return pkg_name; }
+
+	std::string DoReStructuredText(bool roles_only) const;
+
+	std::string pkg_name;
+	std::vector<std::string> readme;
+};
+
+} // namespace broxygen
+
+#endif

src/broxygen/ReStructuredTextTable.cc

@@ -0,0 +1,68 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#include "ReStructuredTextTable.h"
+
+#include <assert.h>
+
+using namespace std;
+using namespace broxygen;
+
+ReStructuredTextTable::ReStructuredTextTable(size_t arg_num_cols)
+	: num_cols(arg_num_cols), rows(), longest_row_in_column()
+	{
+	for ( size_t i = 0; i < num_cols; ++i )
+		longest_row_in_column.push_back(1);
+	}
+
+void ReStructuredTextTable::AddRow(const vector<string>& new_row)
+	{
+	assert(new_row.size() == num_cols);
+	rows.push_back(new_row);
+
+	for ( size_t i = 0; i < new_row.size(); ++i )
+		if ( new_row[i].size() > longest_row_in_column[i] )
+			longest_row_in_column[i] = new_row[i].size();
+	}
+
+string ReStructuredTextTable::MakeBorder(const vector<size_t> col_sizes,
+                                         char border)
+	{
+	string rval;
+
+	for ( size_t i = 0; i < col_sizes.size(); ++i )
+		{
+		if ( i > 0 )
+			rval += " ";
+
+		rval += string(col_sizes[i], border);
+		}
+
+	rval += "\n";
+	return rval;
+	}
+
+string ReStructuredTextTable::AsString(char border) const
+	{
+	string rval = MakeBorder(longest_row_in_column, border);
+
+	for ( size_t row = 0; row < rows.size(); ++row )
+		{
+		for ( size_t col = 0; col < num_cols; ++col )
+			{
+			if ( col > 0 )
+				{
+				size_t last = rows[row][col - 1].size();
+				size_t longest = longest_row_in_column[col - 1];
+				size_t whitespace = longest - last + 1;
+				rval += string(whitespace, ' ');
+				}
+
+			rval += rows[row][col];
+			}
+
+		rval += "\n";
+		}
+
+	rval += MakeBorder(longest_row_in_column, border);
+	return rval;
+	}

src/broxygen/ReStructuredTextTable.h

@@ -0,0 +1,53 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#ifndef BROXYGEN_RESTTABLE_H
+#define BROXYGEN_RESTTABLE_H
+
+#include <vector>
+#include <string>
+
+namespace broxygen {
+
+/**
+ * A reST table with arbitrary number of columns.
+ */
+class ReStructuredTextTable {
+public:
+
+	/**
+	 * Create the reST table object.
+	 * @param arg_num_cols The number of columns in the table.
+	 */
+	ReStructuredTextTable(size_t arg_num_cols);
+
+	/**
+	 * Add a new content row to the table.
+	 * @param new_row A vector with one element for each column in the table.
+	 */
+	void AddRow(const std::vector<std::string>& new_row);
+
+	/**
+	 * @param col_sizes Vector of column sizes (width in number of characters).
+	 * @param border Character to use for the border.
+	 * @return A border sized appropriated for the table with columns of sizes
+	 * denoted by \a col_sizes.
+	 */
+	static std::string MakeBorder(const std::vector<size_t> col_sizes,
+	                              char border);
+
+	/**
+	 * @param border Character to use for the border.
+	 * @return the reST representation of the table and its content.
+	 */
+	std::string AsString(char border) const;
+
+private:
+
+	size_t num_cols;
+	std::vector<std::vector<std::string> > rows;
+	std::vector<size_t> longest_row_in_column;
+};
+
+} // namespace broxygen
+
+#endif

src/broxygen/ScriptInfo.cc

@@ -0,0 +1,363 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#include "ScriptInfo.h"
+#include "IdentifierInfo.h"
+#include "ReStructuredTextTable.h"
+#include "utils.h"
+#include "Manager.h"
+
+#include "Reporter.h"
+#include "Desc.h"
+
+using namespace std;
+using namespace broxygen;
+
+bool IdInfoComp::operator ()(const IdentifierInfo* lhs,
+                             const IdentifierInfo* rhs) const
+	{
+	return lhs->Name() < rhs->Name();
+	}
+
+static vector<string> summary_comment(const vector<string>& cmnts)
+	{
+	vector<string> rval;
+
+	for ( size_t i = 0; i < cmnts.size(); ++i )
+		{
+		size_t end = broxygen::end_of_first_sentence(cmnts[i]);
+
+		if ( end == string::npos )
+			{
+			if ( broxygen::is_all_whitespace(cmnts[i]) )
+				break;
+
+			rval.push_back(cmnts[i]);
+			}
+		else
+			{
+			rval.push_back(cmnts[i].substr(0, end + 1));
+			break;
+			}
+		}
+
+	return rval;
+	}
+
+static void add_summary_rows(const ODesc& id_desc, const vector<string>& cmnts,
+                             ReStructuredTextTable* table)
+	{
+	vector<string> row;
+	row.push_back(id_desc.Description());
+
+	if ( cmnts.empty() )
+		{
+		row.push_back("");
+		table->AddRow(row);
+		return;
+		}
+
+	row.push_back(cmnts[0]);
+	table->AddRow(row);
+
+	for ( size_t i = 1; i < cmnts.size(); ++i )
+		{
+		row.clear();
+		row.push_back("");
+		row.push_back(cmnts[i]);
+		table->AddRow(row);
+		}
+	}
+
+static string make_summary(const string& heading, char underline, char border,
+                           const id_info_list& id_list)
+	{
+	if ( id_list.empty() )
+		return "";
+
+	ReStructuredTextTable table(2);
+
+	for ( id_info_list::const_iterator it = id_list.begin();
+	      it != id_list.end(); ++it )
+		{
+		ID* id = (*it)->GetID();
+		ODesc d;
+		d.SetQuotes(1);
+		id->DescribeReSTShort(&d);
+		add_summary_rows(d, summary_comment((*it)->GetComments()), &table);
+		}
+
+	return broxygen::make_heading(heading, underline) + table.AsString(border)
+	        + "\n";
+	}
+
+static string make_redef_summary(const string& heading, char underline,
+                                 char border, const string& from_script,
+                                 const id_info_set& id_set)
+	{
+	if ( id_set.empty() )
+		return "";
+
+	ReStructuredTextTable table(2);
+
+	for ( id_info_set::const_iterator it = id_set.begin(); it != id_set.end();
+	      ++it )
+		{
+		ID* id = (*it)->GetID();
+		ODesc d;
+		d.SetQuotes(1);
+		id->DescribeReSTShort(&d);
+
+		typedef list<IdentifierInfo::Redefinition> redef_list;
+		redef_list redefs = (*it)->GetRedefs(from_script);
+
+		for ( redef_list::const_iterator iit = redefs.begin();
+		      iit != redefs.end(); ++iit )
+			add_summary_rows(d, summary_comment(iit->comments), &table);
+		}
+
+	return broxygen::make_heading(heading, underline) + table.AsString(border)
+	        + "\n";
+	}
+
+static string make_details(const string& heading, char underline,
+                           const id_info_list& id_list)
+	{
+	if ( id_list.empty() )
+		return "";
+
+	string rval = broxygen::make_heading(heading, underline);
+
+	for ( id_info_list::const_iterator it = id_list.begin();
+	      it != id_list.end(); ++it )
+		{
+		rval += (*it)->ReStructuredText();
+		rval += "\n\n";
+		}
+
+	return rval;
+	}
+
+static string make_redef_details(const string& heading, char underline,
+                                 const id_info_set& id_set)
+	{
+	if ( id_set.empty() )
+		return "";
+
+	string rval = broxygen::make_heading(heading, underline);
+
+	for ( id_info_set::const_iterator it = id_set.begin();
+	      it != id_set.end(); ++it )
+		{
+		rval += (*it)->ReStructuredText(true);
+		rval += "\n\n";
+		}
+
+	return rval;
+	}
+
+ScriptInfo::ScriptInfo(const string& arg_name, const string& arg_path)
+    : Info(),
+      name(arg_name), path(arg_path),
+      is_pkg_loader(SafeBasename(name).result == PACKAGE_LOADER),
+      dependencies(), module_usages(), comments(), id_info(),
+      options(), constants(), state_vars(), types(), events(), hooks(),
+      functions(), redefs()
+	{
+	}
+
+void ScriptInfo::AddIdentifierInfo(IdentifierInfo* info)
+	{
+	id_info[info->Name()] = info;
+	}
+
+void ScriptInfo::DoInitPostScript()
+	{
+	for ( id_info_map::const_iterator it = id_info.begin();
+	      it != id_info.end(); ++it )
+		{
+		IdentifierInfo* info = it->second;
+		ID* id = info->GetID();
+
+		if ( ! broxygen::is_public_api(id) )
+			continue;
+
+		if ( id->AsType() )
+			{
+			types.push_back(info);
+			DBG_LOG(DBG_BROXYGEN, "Filter id '%s' in '%s' as a type",
+			        id->Name(), name.c_str());
+			continue;
+			}
+
+		if ( IsFunc(id->Type()->Tag()) )
+			{
+			switch ( id->Type()->AsFuncType()->Flavor() ) {
+			case FUNC_FLAVOR_HOOK:
+				DBG_LOG(DBG_BROXYGEN, "Filter id '%s' in '%s' as a hook",
+				        id->Name(), name.c_str());
+				hooks.push_back(info);
+				break;
+			case FUNC_FLAVOR_EVENT:
+				DBG_LOG(DBG_BROXYGEN, "Filter id '%s' in '%s' as a event",
+				        id->Name(), name.c_str());
+				events.push_back(info);
+				break;
+			case FUNC_FLAVOR_FUNCTION:
+				DBG_LOG(DBG_BROXYGEN, "Filter id '%s' in '%s' as a function",
+				        id->Name(), name.c_str());
+				functions.push_back(info);
+				break;
+			default:
+				reporter->InternalError("Invalid function flavor");
+				break;
+			}
+
+			continue;
+			}
+
+		if ( id->IsConst() )
+			{
+			if ( id->FindAttr(ATTR_REDEF) )
+				{
+				DBG_LOG(DBG_BROXYGEN, "Filter id '%s' in '%s' as an option",
+				        id->Name(), name.c_str());
+				options.push_back(info);
+				}
+			else
+				{
+				DBG_LOG(DBG_BROXYGEN, "Filter id '%s' in '%s' as a constant",
+				        id->Name(), name.c_str());
+				constants.push_back(info);
+				}
+
+			continue;
+			}
+
+		if ( id->Type()->Tag() == TYPE_ENUM )
+			// Enums are always referenced/documented from the type's
+			// documentation.
+			continue;
+
+		DBG_LOG(DBG_BROXYGEN, "Filter id '%s' in '%s' as a state variable",
+		        id->Name(), name.c_str());
+		state_vars.push_back(info);
+		}
+	}
+
+vector<string> ScriptInfo::GetComments() const
+	{
+	return comments;
+	}
+
+string ScriptInfo::DoReStructuredText(bool roles_only) const
+	{
+	string rval;
+
+	rval += ":tocdepth: 3\n\n";
+	rval += broxygen::make_heading(name, '=');
+
+	for ( string_set::const_iterator it = module_usages.begin();
+	      it != module_usages.end(); ++it )
+		rval += ".. bro:namespace:: " + *it + "\n";
+
+	rval += "\n";
+
+	for ( size_t i = 0; i < comments.size(); ++i )
+		rval += comments[i] + "\n";
+
+	rval += "\n";
+
+	if ( ! module_usages.empty() )
+		{
+		rval += module_usages.size() > 1 ? ":Namespaces: " : ":Namespace: ";
+
+		for ( string_set::const_iterator it = module_usages.begin();
+		      it != module_usages.end(); ++it )
+			{
+			if ( it != module_usages.begin() )
+				rval += ", ";
+
+			rval += *it;
+			}
+
+		rval += "\n";
+		}
+
+	if ( ! dependencies.empty() )
+		{
+		rval += ":Imports: ";
+
+		for ( string_set::const_iterator it = dependencies.begin();
+		      it != dependencies.end(); ++it )
+			{
+			if ( it != dependencies.begin() )
+				rval += ", ";
+
+			string path = find_file(*it, bro_path(), "bro");
+			string doc = *it;
+
+			if ( ! path.empty() && is_dir(path.c_str()) )
+				// Reference the package.
+				doc += "/index";
+
+			rval += fmt(":doc:`%s </scripts/%s>`", it->c_str(), doc.c_str());
+			}
+
+		rval += "\n";
+		}
+
+	rval += fmt(":Source File: :download:`/scripts/%s`\n", name.c_str());
+	rval += "\n";
+	rval += broxygen::make_heading("Summary", '~');
+	rval += make_summary("Options", '#', '=', options);
+	rval += make_summary("Constants", '#', '=', constants);
+	rval += make_summary("State Variables", '#', '=', state_vars);
+	rval += make_summary("Types", '#', '=', types);
+	rval += make_redef_summary("Redefinitions", '#', '=', name, redefs);
+	rval += make_summary("Events", '#', '=', events);
+	rval += make_summary("Hooks", '#', '=', hooks);
+	rval += make_summary("Functions", '#', '=', functions);
+	rval += "\n";
+	rval += broxygen::make_heading("Detailed Interface", '~');
+	rval += make_details("Options", '#', options);
+	rval += make_details("Constants", '#', constants);
+	rval += make_details("State Variables", '#', state_vars);
+	rval += make_details("Types", '#', types);
+	//rval += make_redef_details("Redefinitions", '#', redefs);
+	rval += make_details("Events", '#', events);
+	rval += make_details("Hooks", '#', hooks);
+	rval += make_details("Functions", '#', functions);
+
+	return rval;
+	}
+
+time_t ScriptInfo::DoGetModificationTime() const
+	{
+	time_t most_recent = broxygen::get_mtime(path);
+
+	for ( string_set::const_iterator it = dependencies.begin();
+	      it != dependencies.end(); ++it )
+		{
+		Info* info = broxygen_mgr->GetScriptInfo(*it);
+
+		if ( ! info )
+			{
+			string pkg_name = *it + "/" + PACKAGE_LOADER;
+			info = broxygen_mgr->GetScriptInfo(pkg_name);
+
+			if ( ! info )
+				reporter->InternalWarning("Broxygen failed to get mtime of %s",
+				                          it->c_str());
+			continue;
+			}
+
+		time_t dep_mtime = info->GetModificationTime();
+
+		if ( dep_mtime > most_recent )
+			most_recent = dep_mtime;
+		}
+
+	return most_recent;
+	}
+
+

src/broxygen/ScriptInfo.h

@@ -0,0 +1,123 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#ifndef BROXYGEN_SCRIPTINFO_H
+#define BROXYGEN_SCRIPTINFO_H
+
+#include "Info.h"
+#include "IdentifierInfo.h"
+
+#include <set>
+#include <list>
+#include <string>
+#include <vector>
+#include <map>
+
+namespace broxygen {
+
+class IdentifierInfo;
+
+struct IdInfoComp {
+	bool operator() (const IdentifierInfo* lhs,
+	                 const IdentifierInfo* rhs) const;
+};
+
+typedef std::set<IdentifierInfo*, IdInfoComp> id_info_set;
+typedef std::list<IdentifierInfo*> id_info_list;
+
+/**
+ * Information about a Bro script.
+ */
+class ScriptInfo : public Info {
+
+public:
+
+	/**
+	 * Ctor.
+	 * @param name Name of script: a path relative to a component in BROPATH.
+	 * @param path Absolute path to the script.
+	 */
+	ScriptInfo(const std::string& name, const std::string& path);
+
+	/**
+	 * Associate a Broxygen summary comment ("##!") with the script.
+	 * @param comment String extracted from the comment.
+	 */
+	void AddComment(const std::string& comment)
+		{ comments.push_back(comment); }
+
+	/**
+	 * Register a dependency on another script.
+	 * @param name Name of a script with this one @loads.  This is the
+	 * "normalized" name (a path relative to a component in BROPATH).
+	 */
+	void AddDependency(const std::string& name)
+		{ dependencies.insert(name); }
+
+	/**
+	 * Register a module usage (signifying the script may export identifiers
+	 * into that modules namespace).
+	 * @param name The name of the module.
+	 */
+	void AddModule(const std::string& name)
+		{ module_usages.insert(name); }
+
+	/**
+	 * Register an identifier declared by this script.
+	 * @param info The identifier info object associated with a script-level
+	 * identifier declared by the script.
+	 */
+	void AddIdentifierInfo(IdentifierInfo* info);
+
+	/**
+	 * Register a redef of an identifier done by this script.
+	 * @param info The identifier info object associated with the script-level
+	 * identifier redef'd by the script.
+	 */
+	void AddRedef(IdentifierInfo* info)
+		{ redefs.insert(info); }
+
+	/**
+	 * @return Whether the script is a package loader (i.e. "__load__.bro").
+	 */
+	bool IsPkgLoader() const
+		{ return is_pkg_loader; }
+
+	/**
+	 * @return All the scripts Broxygen summary comments.
+	 */
+	std::vector<std::string> GetComments() const;
+
+private:
+
+	typedef std::map<std::string, IdentifierInfo*> id_info_map;
+	typedef std::set<std::string> string_set;
+
+	time_t DoGetModificationTime() const;
+
+	std::string DoName() const
+		{ return name; }
+
+	std::string DoReStructuredText(bool roles_only) const;
+
+	void DoInitPostScript() /* override */;
+
+	std::string name;
+	std::string path;
+	bool is_pkg_loader;
+	string_set dependencies;
+	string_set module_usages;
+	std::vector<std::string> comments;
+	id_info_map id_info;
+	id_info_list options;
+	id_info_list constants;
+	id_info_list state_vars;
+	id_info_list types;
+	id_info_list events;
+	id_info_list hooks;
+	id_info_list functions;
+	id_info_set redefs;
+};
+
+} // namespace broxygen
+
+#endif

src/broxygen/Target.cc

@@ -0,0 +1,597 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#include "Target.h"
+#include "Manager.h"
+
+#include "util.h"
+#include "Reporter.h"
+#include "plugin/Manager.h"
+#include "analyzer/Manager.h"
+#include "analyzer/Component.h"
+#include "file_analysis/Manager.h"
+
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fts.h>
+#include <unistd.h>
+
+using namespace std;
+using namespace broxygen;
+
+static void write_plugin_section_heading(FILE* f, const plugin::Plugin* p)
+	{
+	string name = p->Name();
+
+	fprintf(f, "%s\n", name.c_str());
+	for ( size_t i = 0; i < name.size(); ++i )
+		fprintf(f, "-");
+	fprintf(f, "\n\n");
+
+	fprintf(f, "%s\n\n", p->Description());
+	}
+
+static void write_analyzer_component(FILE* f, const analyzer::Component* c)
+	{
+	EnumType* atag = analyzer_mgr->GetTagEnumType();
+	string tag = fmt("ANALYZER_%s", c->CanonicalName());
+
+	if ( atag->Lookup("Analyzer", tag.c_str()) < 0 )
+		reporter->InternalError("missing analyzer tag for %s", tag.c_str());
+
+	fprintf(f, ":bro:enum:`Analyzer::%s`\n\n", tag.c_str());
+	}
+
+static void write_analyzer_component(FILE* f, const file_analysis::Component* c)
+	{
+	EnumType* atag = file_mgr->GetTagEnumType();
+	string tag = fmt("ANALYZER_%s", c->CanonicalName());
+
+	if ( atag->Lookup("Files", tag.c_str()) < 0 )
+		reporter->InternalError("missing analyzer tag for %s", tag.c_str());
+
+	fprintf(f, ":bro:enum:`Files::%s`\n\n", tag.c_str());
+	}
+
+static void write_plugin_components(FILE* f, const plugin::Plugin* p)
+	{
+	plugin::Plugin::component_list components = p->Components();
+	plugin::Plugin::component_list::const_iterator it;
+
+	fprintf(f, "Components\n");
+	fprintf(f, "++++++++++\n\n");
+
+	for ( it = components.begin(); it != components.end(); ++it )
+		{
+		switch ( (*it)->Type() ) {
+		case plugin::component::ANALYZER:
+			{
+			const analyzer::Component* c =
+			        dynamic_cast<const analyzer::Component*>(*it);
+
+			if ( c )
+				write_analyzer_component(f, c);
+			else
+				reporter->InternalError("component type mismatch");
+			}
+			break;
+
+		case plugin::component::FILE_ANALYZER:
+			{
+			const file_analysis::Component* c =
+			        dynamic_cast<const file_analysis::Component*>(*it);
+
+			if ( c )
+				write_analyzer_component(f, c);
+			else
+				reporter->InternalError("component type mismatch");
+			}
+			break;
+
+		case plugin::component::READER:
+			reporter->InternalError("docs for READER component unimplemented");
+
+		case plugin::component::WRITER:
+			reporter->InternalError("docs for WRITER component unimplemented");
+
+		default:
+			reporter->InternalError("docs for unknown component unimplemented");
+		}
+		}
+	}
+
+static void write_plugin_bif_items(FILE* f, const plugin::Plugin* p,
+                                plugin::BifItem::Type t, const string& heading)
+	{
+	plugin::Plugin::bif_item_list bifitems = p->BifItems();
+	plugin::Plugin::bif_item_list::iterator it = bifitems.begin();
+
+	while ( it != bifitems.end() )
+		{
+		if ( it->GetType() != t )
+			it = bifitems.erase(it);
+		else
+			++it;
+		}
+
+	if ( bifitems.empty() )
+		return;
+
+	fprintf(f, "%s\n", heading.c_str());
+	for ( size_t i = 0; i < heading.size(); ++i )
+		fprintf(f, "+");
+	fprintf(f, "\n\n");
+
+	for ( it = bifitems.begin(); it != bifitems.end(); ++it )
+		{
+		broxygen::IdentifierInfo* doc = broxygen_mgr->GetIdentifierInfo(
+		                                        it->GetID());
+
+		if ( doc )
+			fprintf(f, "%s\n\n", doc->ReStructuredText().c_str());
+		else
+			reporter->InternalWarning("Broxygen ID lookup failed: %s\n",
+			                          it->GetID());
+		}
+	}
+
+static void WriteAnalyzerTagDefn(FILE* f, const string& module)
+	{
+	string tag_id = module + "::Tag";
+
+	broxygen::IdentifierInfo* doc = broxygen_mgr->GetIdentifierInfo(tag_id);
+
+	if ( ! doc )
+		reporter->InternalError("Broxygen failed analyzer tag lookup: %s",
+		                        tag_id.c_str());
+
+	fprintf(f, "%s\n", doc->ReStructuredText().c_str());
+	}
+
+static bool ComponentsMatch(const plugin::Plugin* p, plugin::component::Type t,
+                            bool match_empty = false)
+	{
+	plugin::Plugin::component_list components = p->Components();
+	plugin::Plugin::component_list::const_iterator it;
+
+	if ( components.empty() )
+		return match_empty;
+
+	for ( it = components.begin(); it != components.end(); ++it )
+		if ( (*it)->Type() != t )
+			return false;
+
+	return true;
+	}
+
+template<class T>
+static vector<T*> filter_matches(const vector<Info*>& from, Target* t)
+	{
+	vector<T*> rval;
+
+	for ( size_t i = 0; i < from.size(); ++i )
+		{
+		T* d = dynamic_cast<T*>(from[i]);
+
+		if ( ! d )
+			continue;
+
+		if ( t->MatchesPattern(d) )
+			{
+			DBG_LOG(DBG_BROXYGEN, "'%s' matched pattern for target '%s'",
+			        d->Name().c_str(), t->Name().c_str());
+			rval.push_back(d);
+			}
+		}
+
+	return rval;
+	}
+
+TargetFile::TargetFile(const string& arg_name)
+	: name(arg_name), f()
+	{
+	if ( name.find('/') != string::npos )
+		{
+		string dir = SafeDirname(name).result;
+
+		if ( ! ensure_intermediate_dirs(dir.c_str()) )
+			reporter->FatalError("Broxygen failed to make dir %s",
+			                     dir.c_str());
+		}
+
+	f = fopen(name.c_str(), "w");
+
+	if ( ! f )
+		reporter->FatalError("Broxygen failed to open '%s' for writing: %s",
+		                     name.c_str(), strerror(errno));
+	}
+
+TargetFile::~TargetFile()
+	{
+	if ( f )
+		fclose(f);
+
+	DBG_LOG(DBG_BROXYGEN, "Wrote out-of-date target '%s'", name.c_str());
+	}
+
+
+Target::Target(const string& arg_name, const string& arg_pattern)
+    : name(arg_name), pattern(arg_pattern), prefix()
+	{
+	size_t pos = pattern.find('*');
+
+	if ( pos == 0 || pos == string::npos )
+		return;
+
+	prefix = pattern.substr(0, pos);
+	}
+
+bool Target::MatchesPattern(Info* info) const
+	{
+	if ( pattern == "*" )
+		return true;
+
+	if ( prefix.empty() )
+		return info->Name() == pattern;
+
+	return ! strncmp(info->Name().c_str(), prefix.c_str(), prefix.size());
+	}
+
+void AnalyzerTarget::DoFindDependencies(const std::vector<Info *>& infos)
+	{
+	// TODO: really should add to dependency list the tag type's ID and
+	// all bif items for matching analyzer plugins, but that's all dependent
+	// on the bro binary itself, so I'm cheating.
+	}
+
+void AnalyzerTarget::DoGenerate() const
+	{
+	if ( broxygen_mgr->IsUpToDate(Name(), vector<Info*>()) )
+		return;
+
+	if ( Pattern() != "*" )
+		reporter->InternalWarning("Broxygen only implements analyzer target"
+		                          " pattern '*'");
+
+	TargetFile file(Name());
+	CreateAnalyzerDoc(file.f);
+	}
+
+void ProtoAnalyzerTarget::DoCreateAnalyzerDoc(FILE* f) const
+	{
+	fprintf(f, "Protocol Analyzers\n");
+	fprintf(f, "==================\n\n");
+	fprintf(f, ".. contents::\n");
+	fprintf(f, "     :depth: 2\n\n");
+
+	WriteAnalyzerTagDefn(f, "Analyzer");
+
+	plugin::Manager::plugin_list plugins = plugin_mgr->Plugins();
+	plugin::Manager::plugin_list::const_iterator it;
+
+	for ( it = plugins.begin(); it != plugins.end(); ++it )
+		{
+		if ( ! ComponentsMatch(*it, plugin::component::ANALYZER, true) )
+			continue;
+
+		write_plugin_section_heading(f, *it);
+		write_plugin_components(f, *it);
+		write_plugin_bif_items(f, *it, plugin::BifItem::CONSTANT,
+		                       "Options/Constants");
+		write_plugin_bif_items(f, *it, plugin::BifItem::GLOBAL, "Globals");
+		write_plugin_bif_items(f, *it, plugin::BifItem::TYPE, "Types");
+		write_plugin_bif_items(f, *it, plugin::BifItem::EVENT, "Events");
+		write_plugin_bif_items(f, *it, plugin::BifItem::FUNCTION, "Functions");
+		}
+	}
+
+void FileAnalyzerTarget::DoCreateAnalyzerDoc(FILE* f) const
+	{
+	fprintf(f, "File Analyzers\n");
+	fprintf(f, "==============\n\n");
+	fprintf(f, ".. contents::\n");
+	fprintf(f, "     :depth: 2\n\n");
+
+	WriteAnalyzerTagDefn(f, "Files");
+
+	plugin::Manager::plugin_list plugins = plugin_mgr->Plugins();
+	plugin::Manager::plugin_list::const_iterator it;
+
+	for ( it = plugins.begin(); it != plugins.end(); ++it )
+		{
+		if ( ! ComponentsMatch(*it, plugin::component::FILE_ANALYZER) )
+			continue;
+
+		write_plugin_section_heading(f, *it);
+		write_plugin_components(f, *it);
+		write_plugin_bif_items(f, *it, plugin::BifItem::CONSTANT,
+		                       "Options/Constants");
+		write_plugin_bif_items(f, *it, plugin::BifItem::GLOBAL, "Globals");
+		write_plugin_bif_items(f, *it, plugin::BifItem::TYPE, "Types");
+		write_plugin_bif_items(f, *it, plugin::BifItem::EVENT, "Events");
+		write_plugin_bif_items(f, *it, plugin::BifItem::FUNCTION, "Functions");
+		}
+	}
+
+void PackageTarget::DoFindDependencies(const vector<Info*>& infos)
+	{
+	pkg_deps = filter_matches<PackageInfo>(infos, this);
+
+	if ( pkg_deps.empty() )
+		reporter->FatalError("No match for Broxygen target '%s' pattern '%s'",
+		                     Name().c_str(), Pattern().c_str());
+
+	for ( size_t i = 0; i < infos.size(); ++i )
+		{
+		ScriptInfo* script = dynamic_cast<ScriptInfo*>(infos[i]);
+
+		if ( ! script )
+			continue;
+
+		for ( size_t j = 0; j < pkg_deps.size(); ++j )
+			{
+			if ( strncmp(script->Name().c_str(), pkg_deps[j]->Name().c_str(),
+			             pkg_deps[j]->Name().size()))
+				continue;
+
+			DBG_LOG(DBG_BROXYGEN, "Script %s associated with package %s",
+			        script->Name().c_str(), pkg_deps[j]->Name().c_str());
+			pkg_manifest[pkg_deps[j]].push_back(script);
+			script_deps.push_back(script);
+			}
+		}
+	}
+
+void PackageTarget::DoGenerate() const
+	{
+	if ( broxygen_mgr->IsUpToDate(Name(), script_deps) &&
+	     broxygen_mgr->IsUpToDate(Name(), pkg_deps) )
+		return;
+
+	TargetFile file(Name());
+
+	fprintf(file.f, ":orphan:\n\n");
+
+	for ( manifest_t::const_iterator it = pkg_manifest.begin();
+	      it != pkg_manifest.end(); ++it )
+		{
+		string header = fmt("Package: %s", it->first->Name().c_str());
+		header += "\n" + string(header.size(), '=');
+
+		fprintf(file.f, "%s\n\n", header.c_str());
+
+		vector<string> readme = it->first->GetReadme();
+
+		for ( size_t i = 0; i < readme.size(); ++i )
+			fprintf(file.f, "%s\n", readme[i].c_str());
+
+		fprintf(file.f, "\n");
+
+		for ( size_t i = 0; i < it->second.size(); ++i )
+			{
+			fprintf(file.f, ":doc:`/scripts/%s`\n",
+			        it->second[i]->Name().c_str());
+
+			vector<string> cmnts = it->second[i]->GetComments();
+
+			for ( size_t j = 0; j < cmnts.size(); ++j )
+				fprintf(file.f, "   %s\n", cmnts[j].c_str());
+
+			fprintf(file.f, "\n");
+			}
+		}
+	}
+
+void PackageIndexTarget::DoFindDependencies(const vector<Info*>& infos)
+	{
+	pkg_deps = filter_matches<PackageInfo>(infos, this);
+
+	if ( pkg_deps.empty() )
+		reporter->FatalError("No match for Broxygen target '%s' pattern '%s'",
+		                     Name().c_str(), Pattern().c_str());
+	}
+
+void PackageIndexTarget::DoGenerate() const
+	{
+	if ( broxygen_mgr->IsUpToDate(Name(), pkg_deps) )
+		return;
+
+	TargetFile file(Name());
+
+	for ( size_t i = 0; i < pkg_deps.size(); ++i )
+		fprintf(file.f, "%s\n", pkg_deps[i]->ReStructuredText().c_str());
+	}
+
+void ScriptTarget::DoFindDependencies(const vector<Info*>& infos)
+    {
+	script_deps = filter_matches<ScriptInfo>(infos, this);
+
+	if ( script_deps.empty() )
+		reporter->FatalError("No match for Broxygen target '%s' pattern '%s'",
+		                     Name().c_str(), Pattern().c_str());
+
+	if ( ! IsDir() )
+		return;
+
+	for ( size_t i = 0; i < script_deps.size(); ++i )
+		{
+		if ( SafeBasename(script_deps[i]->Name()).result == PACKAGE_LOADER )
+			{
+			string pkg_dir = SafeDirname(script_deps[i]->Name()).result;
+			string target_file = Name() + pkg_dir + "/index.rst";
+			Target* t = new PackageTarget(target_file, pkg_dir);
+			t->FindDependencies(infos);
+			pkg_deps.push_back(t);
+			}
+		}
+	}
+
+vector<string> dir_contents_recursive(string dir)
+	{
+	vector<string> rval;
+	struct stat st;
+
+	if ( stat(dir.c_str(), &st) < 0 && errno == ENOENT )
+		return rval;
+
+	while ( dir[dir.size() - 1] == '/' )
+		dir.erase(dir.size() - 1, 1);
+
+	char* dir_copy = copy_string(dir.c_str());
+	char** scan_path = new char*[2];
+	scan_path[0] = dir_copy;
+	scan_path[1] = 0;
+
+	FTS* fts = fts_open(scan_path, FTS_NOCHDIR, 0);
+
+	if ( ! fts )
+		{
+		reporter->Error("fts_open failure: %s", strerror(errno));
+		delete [] scan_path;
+		delete [] dir_copy;
+		return rval;
+		}
+
+	FTSENT* n;
+
+	while ( (n = fts_read(fts)) )
+		{
+		if ( n->fts_info & FTS_F )
+			rval.push_back(n->fts_path);
+		}
+
+	if ( errno )
+		reporter->Error("fts_read failure: %s", strerror(errno));
+
+	if ( fts_close(fts) < 0 )
+		reporter->Error("fts_close failure: %s", strerror(errno));
+
+	delete [] scan_path;
+	delete [] dir_copy;
+	return rval;
+	}
+
+void ScriptTarget::DoGenerate() const
+    {
+	if ( IsDir() )
+		{
+		// Target name is a dir, matching scripts are written within that dir
+		// with a dir tree that parallels the script's BROPATH location.
+
+		set<string> targets;
+		vector<string> dir_contents = dir_contents_recursive(Name());
+
+		for ( size_t i = 0; i < script_deps.size(); ++i )
+			{
+			string target_filename = Name() + script_deps[i]->Name() + ".rst";
+			targets.insert(target_filename);
+			vector<ScriptInfo*> dep;
+			dep.push_back(script_deps[i]);
+
+			if ( broxygen_mgr->IsUpToDate(target_filename, dep) )
+				continue;
+
+			TargetFile file(target_filename);
+
+			fprintf(file.f, "%s\n", script_deps[i]->ReStructuredText().c_str());
+			}
+
+		for ( size_t i = 0; i < pkg_deps.size(); ++i )
+			{
+			targets.insert(pkg_deps[i]->Name());
+			pkg_deps[i]->Generate();
+			}
+
+		for ( size_t i = 0; i < dir_contents.size(); ++i )
+			{
+			string f = dir_contents[i];
+
+			if ( targets.find(f) != targets.end() )
+				continue;
+
+			if ( unlink(f.c_str()) < 0 )
+				reporter->Warning("Failed to unlink %s: %s", f.c_str(),
+				                  strerror(errno));
+
+			DBG_LOG(DBG_BROXYGEN, "Delete stale script file %s", f.c_str());
+			}
+
+		return;
+		}
+
+	// Target is a single file, all matching scripts get written there.
+
+	if ( broxygen_mgr->IsUpToDate(Name(), script_deps) )
+		return;
+
+	TargetFile file(Name());
+
+	for ( size_t i = 0; i < script_deps.size(); ++i )
+		fprintf(file.f, "%s\n", script_deps[i]->ReStructuredText().c_str());
+	}
+
+void ScriptSummaryTarget::DoGenerate() const
+	{
+	if ( broxygen_mgr->IsUpToDate(Name(), script_deps) )
+		return;
+
+	TargetFile file(Name());
+
+	for ( size_t i = 0; i < script_deps.size(); ++i )
+		{
+		ScriptInfo* d = dynamic_cast<ScriptInfo*>(script_deps[i]);
+
+		if ( ! d )
+			continue;
+
+		fprintf(file.f, ":doc:`/scripts/%s`\n", d->Name().c_str());
+
+		vector<string> cmnts = d->GetComments();
+
+		for ( size_t i = 0; i < cmnts.size(); ++i )
+			fprintf(file.f, "    %s\n", cmnts[i].c_str());
+
+		fprintf(file.f, "\n");
+		}
+	}
+
+void ScriptIndexTarget::DoGenerate() const
+	{
+	if ( broxygen_mgr->IsUpToDate(Name(), script_deps) )
+		return;
+
+	TargetFile file(Name());
+
+	fprintf(file.f, ".. toctree::\n");
+	fprintf(file.f, "   :maxdepth: 1\n\n");
+
+	for ( size_t i = 0; i < script_deps.size(); ++i )
+		{
+		ScriptInfo* d = dynamic_cast<ScriptInfo*>(script_deps[i]);
+
+		if ( ! d )
+			continue;
+
+		fprintf(file.f, "   %s </scripts/%s>\n", d->Name().c_str(),
+		        d->Name().c_str());
+		}
+	}
+
+void IdentifierTarget::DoFindDependencies(const vector<Info*>& infos)
+	{
+	id_deps = filter_matches<IdentifierInfo>(infos, this);
+
+	if ( id_deps.empty() )
+		reporter->FatalError("No match for Broxygen target '%s' pattern '%s'",
+		                     Name().c_str(), Pattern().c_str());
+	}
+
+void IdentifierTarget::DoGenerate() const
+	{
+	if ( broxygen_mgr->IsUpToDate(Name(), id_deps) )
+		return;
+
+	TargetFile file(Name());
+
+	for ( size_t i = 0; i < id_deps.size(); ++i )
+		fprintf(file.f, "%s\n\n", id_deps[i]->ReStructuredText().c_str());
+	}

src/broxygen/Target.h

@@ -0,0 +1,389 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#ifndef BROXYGEN_TARGET_H
+#define BROXYGEN_TARGET_H
+
+#include "Info.h"
+#include "PackageInfo.h"
+#include "ScriptInfo.h"
+#include "IdentifierInfo.h"
+
+#include <map>
+#include <string>
+#include <vector>
+#include <cstdio>
+
+namespace broxygen {
+
+/**
+ * Helper class to create files in arbitrary file paths and automatically
+ * close it on destruction.
+ */
+struct TargetFile {
+	/**
+	 * Open a file.
+	 * @param arg_name Path to a file to create.  It's a fatal error if
+	 * creating it fails.  Creating it will also create any intermediate
+	 * directories that don't already exist.
+	 *
+	 */
+	TargetFile(const std::string& arg_name);
+
+	/**
+	 * Close the file.
+	 */
+	~TargetFile();
+
+	std::string name; /**< File name. */
+	FILE* f; /**< File stream. */
+};
+
+/**
+ * A Broxygen target abstract base class.  A target is generally any portion of
+ * documentation that Bro can build.  It's identified by a type (e.g. script,
+ * identifier, package), a pattern (e.g. "example.bro", "HTTP::Info"), and
+ * a path to an output file.
+ */
+class Target {
+
+public:
+
+	/**
+	 * Ctor.
+	 * @param arg_name output file name of the target.
+	 * @param arg_pattern pattern of info objects the target depends upon.  Only
+	 * exact string and simple prefix matching is currently allowed.
+	 */
+	Target(const std::string& arg_name, const std::string& arg_pattern);
+
+	/**
+	 * Dtor.
+	 */
+	virtual ~Target()
+		{ }
+
+	/**
+	 * Filter out any dependency information from a set of all known info.
+	 * @param infos All known info objects.
+	 */
+	void FindDependencies(const std::vector<Info*>& infos)
+		{ DoFindDependencies(infos); }
+
+	/**
+	 * Build the target by generating its output file.  Implementations may
+	 * not always write to the output file if they determine an existing
+	 * version is already up-to-date.
+	 */
+	void Generate() const
+		{ DoGenerate(); }
+
+	/**
+	 * Check if a particular info object matches the target pattern.
+	 * Currently only exact string and simple prefix matching patterns are
+	 * used.  E.g. for prefix matching "HTTP::*" or "base/protocols/http*".
+	 * @param info An info object for some thing that is documentable.
+	 * @return true if it matches, else false.
+	 */
+	bool MatchesPattern(Info* info) const;
+
+	/**
+	 * @return The output file name of the target.
+	 */
+	std::string Name() const
+		{ return name; }
+
+	/**
+	 * @return The pattern string of the target.
+	 */
+	std::string Pattern() const
+		{ return pattern; }
+
+private:
+
+	virtual void DoFindDependencies(const std::vector<Info*>& infos) = 0;
+
+	virtual void DoGenerate() const = 0;
+
+	std::string name;
+	std::string pattern;
+	std::string prefix;
+};
+
+template<class T>
+static Target* create_target(const std::string& name,
+                             const std::string& pattern)
+	{
+	return new T(name, pattern);
+	}
+
+/**
+ * Factory for creating Target instances.
+ */
+class TargetFactory {
+
+public:
+
+	/**
+	 * Register a new target type.
+	 * @param type_name The target type name as it will appear in Broxygen
+	 * config files.
+	 */
+	template<class T>
+	void Register(const std::string& type_name)
+		{
+		target_creators[type_name] = &create_target<T>;
+		}
+
+	/**
+	 * Instantiate a target.
+	 * @param type_name The target type name as it appears in Broxygen config
+	 * files.
+	 * @param name The output file name of the target.
+	 * @param pattern The dependency pattern of the target.
+	 * @return A new target instance or a pointer if \a type_name is not
+	 * registered.
+	 */
+	Target* Create(const std::string& type_name,
+	               const std::string& name, const std::string& pattern)
+		{
+		target_creator_map::const_iterator it = target_creators.find(type_name);
+
+		if ( it == target_creators.end() )
+			return 0;
+
+		return it->second(name, pattern);
+		}
+
+private:
+
+	typedef Target* (*TargetFactoryFn)(const std::string& name,
+	                                  const std::string& pattern);
+	typedef std::map<std::string, TargetFactoryFn> target_creator_map;
+	target_creator_map target_creators;
+};
+
+/**
+ * Target to build analyzer documentation.
+ */
+class AnalyzerTarget : public Target {
+public:
+
+	/**
+	 * Writes out plugin index documentation for all analyzer plugins.
+	 * @param f an open file stream to write docs into.
+	 */
+	void CreateAnalyzerDoc(FILE* f) const
+		{ return DoCreateAnalyzerDoc(f); }
+
+protected:
+
+	typedef void (*doc_creator_fn)(FILE*);
+
+	AnalyzerTarget(const std::string& name, const std::string& pattern)
+		: Target(name, pattern)
+		{ }
+
+private:
+
+	void DoFindDependencies(const std::vector<Info*>& infos);
+
+	void DoGenerate() const;
+
+	virtual void DoCreateAnalyzerDoc(FILE* f) const = 0;
+};
+
+/**
+ * Target to build protocol analyzer documentation.
+ */
+class ProtoAnalyzerTarget : public AnalyzerTarget {
+public:
+
+	/**
+	 * Ctor.
+	 * @param name Output file name.
+	 * @param pattern Dependency pattern.
+	 */
+	ProtoAnalyzerTarget(const std::string& name, const std::string& pattern)
+		: AnalyzerTarget(name, pattern)
+		{ }
+
+private:
+
+	void DoCreateAnalyzerDoc(FILE* f) const;
+};
+
+/**
+ * Target to build file analyzer documentation.
+ */
+class FileAnalyzerTarget : public AnalyzerTarget {
+public:
+
+	/**
+	 * Ctor.
+	 * @param name Output file name.
+	 * @param pattern Dependency pattern.
+	 */
+	FileAnalyzerTarget(const std::string& name, const std::string& pattern)
+		: AnalyzerTarget(name, pattern)
+		{ }
+
+private:
+
+	void DoCreateAnalyzerDoc(FILE* f) const;
+};
+
+/**
+ * Target to build package documentation.
+ */
+class PackageTarget : public Target {
+public:
+
+	/**
+	 * Ctor.
+	 * @param name Output file name.
+	 * @param pattern Dependency pattern.
+	 */
+	PackageTarget(const std::string& name, const std::string& pattern)
+		: Target(name, pattern), pkg_deps(), script_deps(), pkg_manifest()
+		{ }
+
+private:
+
+	void DoFindDependencies(const std::vector<Info*>& infos);
+
+	void DoGenerate() const;
+
+	std::vector<PackageInfo*> pkg_deps;
+	std::vector<ScriptInfo*> script_deps;
+	typedef std::map<PackageInfo*,std::vector<ScriptInfo*> > manifest_t;
+	manifest_t pkg_manifest;
+};
+
+/**
+ * Target to build package index documentation.
+ */
+class PackageIndexTarget : public Target {
+public:
+
+	/**
+	 * Ctor.
+	 * @param name Output file name.
+	 * @param pattern Dependency pattern.
+	 */
+	PackageIndexTarget(const std::string& name, const std::string& pattern)
+		: Target(name, pattern), pkg_deps()
+		{ }
+
+private:
+
+	void DoFindDependencies(const std::vector<Info*>& infos);
+
+	void DoGenerate() const;
+
+	std::vector<PackageInfo*> pkg_deps;
+};
+
+/**
+ * Target to build script documentation.
+ */
+class ScriptTarget : public Target {
+public:
+
+	/**
+	 * Ctor.
+	 * @param name Output file name or directory.  If it's a directory,
+	 * then one document for each script that matches the pattern is written to
+	 * the directory in a directory structure which mirrors the script's path
+	 * relative to a component in BROPATH.
+	 * @param pattern Dependency pattern.
+	 */
+	ScriptTarget(const std::string& name, const std::string& pattern)
+		: Target(name, pattern), script_deps()
+		{ }
+
+	~ScriptTarget()
+		{ for ( size_t i = 0; i < pkg_deps.size(); ++i ) delete pkg_deps[i]; }
+
+protected:
+
+	std::vector<ScriptInfo*> script_deps;
+
+private:
+
+	void DoFindDependencies(const std::vector<Info*>& infos);
+
+	void DoGenerate() const;
+
+	bool IsDir() const
+		{ return Name()[Name().size() - 1] == '/'; }
+
+	std::vector<Target*> pkg_deps;
+};
+
+/**
+ * Target to build script summary documentation.
+ */
+class ScriptSummaryTarget : public ScriptTarget {
+public:
+
+	/**
+	 * Ctor.
+	 * @param name Output file name.
+	 * @param pattern Dependency pattern.
+	 */
+	ScriptSummaryTarget(const std::string& name, const std::string& pattern)
+		: ScriptTarget(name, pattern)
+		{ }
+
+private:
+
+	void DoGenerate() const /* override */;
+};
+
+/**
+ * Target to build script index documentation.
+ */
+class ScriptIndexTarget : public ScriptTarget {
+public:
+
+	/**
+	 * Ctor.
+	 * @param name Output file name.
+	 * @param pattern Dependency pattern.
+	 */
+	ScriptIndexTarget(const std::string& name, const std::string& pattern)
+		: ScriptTarget(name, pattern)
+		{ }
+
+private:
+
+	void DoGenerate() const /* override */;
+};
+
+/**
+ * Target to build identifier documentation.
+ */
+class IdentifierTarget : public Target {
+public:
+
+	/**
+	 * Ctor.
+	 * @param name Output file name.
+	 * @param pattern Dependency pattern.
+	 */
+	IdentifierTarget(const std::string& name, const std::string& pattern)
+		: Target(name, pattern), id_deps()
+		{ }
+
+private:
+
+	void DoFindDependencies(const std::vector<Info*>& infos);
+
+	void DoGenerate() const;
+
+	std::vector<IdentifierInfo*> id_deps;
+};
+
+} // namespace broxygen
+
+#endif

src/broxygen/broxygen.bif

@@ -0,0 +1,97 @@
+# See the file "COPYING" in the main distribution directory for copyright.
+
+##! Functions for querying script, package, or variable documentation.
+
+%%{
+#include "broxygen/Manager.h"
+#include "util.h"
+
+static StringVal* comments_to_val(const vector<string>& comments)
+	{
+	return new StringVal(implode_string_vector(comments));
+	}
+%%}
+
+## Retrieve the Broxygen-style comments (``##``) associated with an identifier
+## (e.g. a variable or type).
+##
+## name: a script-level identifier for which to retrieve comments.
+##
+## Returns: comments associated with *name*.  If *name* is not a known
+##          identifier, an empty string is returned.
+function get_identifier_comments%(name: string%): string
+	%{
+	using namespace broxygen;
+	IdentifierInfo* d = broxygen_mgr->GetIdentifierInfo(name->CheckString());
+
+	if ( ! d )
+		return new StringVal("");
+
+	return comments_to_val(d->GetComments());
+	%}
+
+## Retrieve the Broxygen-style summary comments (``##!``) associated with
+## a Bro script.
+##
+## name: the name of a Bro script.  It must be a relative path to where
+##       it is located within a particular component of BROPATH and use
+##       the same file name extension/suffix as the actual file (e.g. ".bro").
+##
+## Returns: summary comments associated with script with *name*.  If
+##          *name* is not a known script, an empty string is returned.
+function get_script_comments%(name: string%): string
+	%{
+	using namespace broxygen;
+	ScriptInfo* d = broxygen_mgr->GetScriptInfo(name->CheckString());
+
+	if ( ! d )
+		return new StringVal("");
+
+	return comments_to_val(d->GetComments());
+	%}
+
+## Retrieve the contents of a Bro script package's README file.
+##
+## name: the name of a Bro script package.  It must be a relative path
+##       to where it is located within a particular component of BROPATH.
+##
+## Returns: contents of the package's README file.  If *name* is not a known
+##          package, an empty string is returned.
+function get_package_readme%(name: string%): string
+	%{
+	using namespace broxygen;
+	PackageInfo* d = broxygen_mgr->GetPackageInfo(name->CheckString());
+
+	if ( ! d )
+		return new StringVal("");
+
+	return comments_to_val(d->GetReadme());
+	%}
+
+## Retrieve the Broxygen-style comments (``##``) associated with a record field.
+##
+## name: the name of a record type and a field within it formatted like
+##       a typical record field access: "<record_type>$<field>".
+##
+## Returns: comments associated with the record field.  If *name* does
+##          not point to a known record type or a known field within a record
+##          type, an empty string is returned.
+function get_record_field_comments%(name: string%): string
+	%{
+	using namespace broxygen;
+	string accessor = name->CheckString();
+	size_t i = accessor.find('$');
+
+	if ( i > accessor.size() - 2 )
+		return new StringVal("");
+
+	string id = accessor.substr(0, i);
+
+	IdentifierInfo* d = broxygen_mgr->GetIdentifierInfo(id);
+
+	if ( ! d )
+		return new StringVal("");
+
+	string field = accessor.substr(i + 1);
+	return comments_to_val(d->GetFieldComments(field));
+	%}

src/broxygen/utils.cc

@@ -0,0 +1,129 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#include "utils.h"
+
+#include "Reporter.h"
+
+#include <sys/stat.h>
+#include <errno.h>
+
+using namespace broxygen;
+using namespace std;
+
+bool broxygen::prettify_params(string& s)
+	{
+	size_t identifier_start_pos = 0;
+	bool in_identifier = false;
+	string identifier;
+
+	for ( size_t i = 0; i < s.size(); ++i )
+		{
+		char next = s[i];
+
+		if ( ! in_identifier )
+			{
+			// Pass by leading whitespace.
+			if ( isspace(next) )
+				continue;
+
+			// Only allow alphabetic and '_' as first char of identifier.
+			if ( isalpha(next) || next == '_' )
+				{
+				identifier_start_pos = i;
+				identifier += next;
+				in_identifier = true;
+				continue;
+				}
+
+			// Don't need to change anything.
+			return false;
+			}
+
+		// All other characters of identifier are alphanumeric or '_'.
+		if ( isalnum(next) || next == '_' )
+			{
+			identifier += next;
+			continue;
+			}
+
+		if ( next == ':' )
+			{
+			if ( i + 1 < s.size() && s[i + 1] == ':' )
+				{
+				// It's part of an identifier's namespace scoping.
+				identifier += next;
+				identifier += s[i + 1];
+				++i;
+				continue;
+				}
+
+			// Prettify function param/return value reST markup.
+			string subst;
+
+			if ( identifier == "Returns" )
+				subst = ":returns";
+			else
+				subst = ":param " + identifier;
+
+			s.replace(identifier_start_pos, identifier.size(), subst);
+			return true;
+			}
+
+		// Don't need to change anything.
+		return false;
+		}
+
+	return false;
+	}
+
+bool broxygen::is_public_api(const ID* id)
+	{
+	return (id->Scope() == SCOPE_GLOBAL) ||
+	       (id->Scope() == SCOPE_MODULE && id->IsExport());
+	}
+
+time_t broxygen::get_mtime(const string& filename)
+	{
+	struct stat s;
+
+	if ( stat(filename.c_str(), &s) < 0 )
+		reporter->InternalError("Broxygen failed to stat file '%s': %s",
+		                        filename.c_str(), strerror(errno));
+
+	return s.st_mtime;
+	}
+
+string broxygen::make_heading(const string& heading, char underline)
+	{
+	return heading + "\n" + string(heading.size(), underline) + "\n";
+	}
+
+size_t broxygen::end_of_first_sentence(const string& s)
+	{
+	size_t rval = 0;
+
+	while ( (rval = s.find_first_of('.', rval)) != string::npos )
+		{
+		if ( rval == s.size() - 1 )
+			// Period is at end of string.
+			return rval;
+
+		if ( isspace(s[rval + 1]) )
+			// Period has a space after it.
+			return rval;
+
+		// Period has some non-space character after it, keep looking.
+		++rval;
+		}
+
+	return rval;
+	}
+
+bool broxygen::is_all_whitespace(const string& s)
+	{
+	for ( size_t i = 0; i < s.size(); ++i )
+		if ( ! isspace(s[i]) )
+			return false;
+
+	return true;
+	}

src/broxygen/utils.h

@@ -0,0 +1,61 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#ifndef BROXYGEN_UTILS_H
+#define BROXYGEN_UTILS_H
+
+#include "ID.h"
+
+#include <string>
+
+namespace broxygen {
+
+/**
+ * Transform content of a Broxygen comment which may contain function
+ * parameter or return value documentation to a prettier reST format.
+ * @param s Content from a Broxygen comment to transform. "id: ..." and
+ * "Returns: ..." change to ":param id: ..." and ":returns: ...".
+ * @return Whether any content in \a s was transformed.
+ */
+bool prettify_params(std::string& s);
+
+/**
+ * Check whether an identifier is part of the "public" interface.
+ * @param id A script-level identifier.
+ * @return true if the ID is in the global scope or if it's exported in to
+ * any modules namespace.
+ */
+bool is_public_api(const ID* id);
+
+/**
+ * Get the modification time of a file or abort if there's an error.
+ * @param filename Path to a file.
+ * @return The modification time of \a filename via stat(2).
+ */
+time_t get_mtime(const std::string& filename);
+
+/**
+ * Make a reST-style heading.
+ * @param heading Content of the heading.
+ * @param underline Character in which to underline heading content.
+ * @return underlined heading string.
+ */
+std::string make_heading(const std::string& heading, char underline);
+
+/**
+ * Get the position of the end of the first sentence in a string.
+ * @param s Any string.
+ * @return The position which looks like the end of the first sentence in
+ * \a s or 0 if no such position is found.
+ */
+size_t end_of_first_sentence(const std::string& s);
+
+/**
+ * Check if a string is entirely white space.
+ * @param s Any string.
+ * @return True if \a s is nothing but white space, else false.
+ */
+bool is_all_whitespace(const std::string& s);
+
+} // namespace broxygen
+
+#endif

src/const.bif

@@ -1,6 +1,6 @@
 ##! Declaration of various scripting-layer constants that the Bro core uses
 ##! internally.  Documentation and default values for the scripting-layer
-##! variables themselves are found in :doc:`/scripts/base/init-bare`.
+##! variables themselves are found in :doc:`/scripts/base/init-bare.bro`.
 
 const ignore_keep_alive_rexmit: bool;
 const skip_http_data: bool;

src/event.bif

@@ -360,7 +360,7 @@ event content_gap%(c: connection, is_orig: bool, seq: count, length: count%);
 ##
 ## .. note::
 ##
-##    Bro comes with a script :doc:`/scripts/policy/misc/capture-loss` that uses
+##    Bro comes with a script :doc:`/scripts/policy/misc/capture-loss.bro` that uses
 ##    this event to estimate packet loss and report when a predefined threshold
 ##    is exceeded.
 event gap_report%(dt: interval, info: gap_info%);

src/main.cc

@@ -50,7 +50,6 @@ extern "C" void OPENSSL_add_all_algorithms_conf(void);
 #include "PersistenceSerializer.h"
 #include "EventRegistry.h"
 #include "Stats.h"
-#include "BroDoc.h"
 #include "Brofiler.h"
 
 #include "threading/Manager.h"
@@ -61,8 +60,8 @@ extern "C" void OPENSSL_add_all_algorithms_conf(void);
 #include "analyzer/Manager.h"
 #include "analyzer/Tag.h"
 #include "plugin/Manager.h"
-
 #include "file_analysis/Manager.h"
+#include "broxygen/Manager.h"
 
 #include "binpac_bro.h"
 
@@ -100,6 +99,7 @@ input::Manager* input_mgr = 0;
 plugin::Manager* plugin_mgr = 0;
 analyzer::Manager* analyzer_mgr = 0;
 file_analysis::Manager* file_mgr = 0;
+broxygen::Manager* broxygen_mgr = 0;
 Stmt* stmts;
 EventHandlerPtr net_done = 0;
 RuleMatcher* rule_matcher = 0;
@@ -116,7 +116,6 @@ int signal_val = 0;
 int optimize = 0;
 int do_notice_analysis = 0;
 int rule_bench = 0;
-int generate_documentation = 0;
 SecondaryPath* secondary_path = 0;
 extern char version[];
 char* command_line_policy = 0;
@@ -124,6 +123,8 @@ vector<string> params;
 char* proc_status_file = 0;
 int snaplen = 0;	// this gets set from the scripting-layer's value
 
+TypeAliasMap type_aliases;
+
 OpaqueType* md5_type = 0;
 OpaqueType* sha1_type = 0;
 OpaqueType* sha256_type = 0;
@@ -132,8 +133,6 @@ OpaqueType* cardinality_type = 0;
 OpaqueType* topk_type = 0;
 OpaqueType* bloomfilter_type = 0;
 
-extern std::list<BroDoc*> docs_generated;
-
 // Keep copy of command line
 int bro_argc;
 char** bro_argv;
@@ -203,7 +202,7 @@ void usage()
 	fprintf(stderr, "    -T|--re-level <level>          | set 'RE_level' for rules\n");
 	fprintf(stderr, "    -U|--status-file <file>        | Record process status in file\n");
 	fprintf(stderr, "    -W|--watchdog                  | activate watchdog timer\n");
-	fprintf(stderr, "    -Z|--doc-scripts               | generate documentation for all loaded scripts\n");
+	fprintf(stderr, "    -X|--broxygen                  | generate documentation based on config file\n");
 
 #ifdef USE_PERFTOOLS_DEBUG
 	fprintf(stderr, "    -m|--mem-leaks                 | show leaks  [perftools]\n");
@@ -227,6 +226,7 @@ void usage()
 	fprintf(stderr, "    $BRO_SEED_FILE                 | file to load seeds from (not set)\n");
 	fprintf(stderr, "    $BRO_LOG_SUFFIX                | ASCII log file extension (.%s)\n", logging::writer::Ascii::LogExt().c_str());
 	fprintf(stderr, "    $BRO_PROFILER_FILE             | Output file for script execution statistics (not set)\n");
+	fprintf(stderr, "    $BRO_DISABLE_BROXYGEN          | Disable Broxygen documentation support (%s)\n", getenv("BRO_DISABLE_BROXYGEN") ? "set" : "not set");
 
 	fprintf(stderr, "\n");
 	fprintf(stderr, "    Supported log formats: ");
@@ -373,6 +373,7 @@ void terminate_bro()
 
 	plugin_mgr->FinishPlugins();
 
+	delete broxygen_mgr;
 	delete timer_mgr;
 	delete dns_mgr;
 	delete persistence_serializer;
@@ -473,7 +474,7 @@ int main(int argc, char** argv)
 		{"filter",		required_argument,	0,	'f'},
 		{"help",		no_argument,		0,	'h'},
 		{"iface",		required_argument,	0,	'i'},
-		{"doc-scripts",		no_argument,		0,	'Z'},
+		{"broxygen",		required_argument,		0,	'X'},
 		{"prefix",		required_argument,	0,	'p'},
 		{"readfile",		required_argument,	0,	'r'},
 		{"flowfile",		required_argument,	0,	'y'},
@@ -532,7 +533,7 @@ int main(int argc, char** argv)
 	if ( p )
 		add_to_name_list(p, ':', prefixes);
 
-	string active_file;
+	string broxygen_config;
 
 #ifdef USE_IDMEF
 	string libidmef_dtd_path = "idmef-message.dtd";
@@ -545,7 +546,7 @@ int main(int argc, char** argv)
 	opterr = 0;
 
 	char opts[256];
-	safe_strncpy(opts, "B:D:e:f:I:i:K:l:n:p:R:r:s:T:t:U:w:x:X:y:Y:z:CFGLNOPSWbdghvZ",
+	safe_strncpy(opts, "B:D:e:f:I:i:K:l:n:p:R:r:s:T:t:U:w:x:X:y:Y:z:CFGLNOPSWbdghv",
 		     sizeof(opts));
 
 #ifdef USE_PERFTOOLS_DEBUG
@@ -727,8 +728,8 @@ int main(int argc, char** argv)
 			break;
 #endif
 
-		case 'Z':
+		case 'X':
-			generate_documentation = 1;
+			broxygen_config = optarg;
 			break;
 
 #ifdef USE_IDMEF
@@ -806,6 +807,8 @@ int main(int argc, char** argv)
 	timer_mgr = new PQ_TimerMgr("<GLOBAL>");
 	// timer_mgr = new CQ_TimerMgr();
 
+	broxygen_mgr = new broxygen::Manager(broxygen_config, bro_argv[0]);
+
 	add_input_file("base/init-bare.bro");
 	if ( ! bare_mode )
 		add_input_file("base/init-default.bro");
@@ -848,6 +851,7 @@ int main(int argc, char** argv)
 	plugin_mgr->InitPreScript();
 	analyzer_mgr->InitPreScript();
 	file_mgr->InitPreScript();
+	broxygen_mgr->InitPreScript();
 
 	if ( events_file )
 		event_player = new EventPlayer(events_file);
@@ -880,6 +884,7 @@ int main(int argc, char** argv)
 	plugin_mgr->InitPostScript();
 	analyzer_mgr->InitPostScript();
 	file_mgr->InitPostScript();
+	broxygen_mgr->InitPostScript();
 
 	if ( print_plugins )
 		{
@@ -891,23 +896,6 @@ int main(int argc, char** argv)
 	}
 #endif
 
-	if ( generate_documentation )
-		{
-		CreateProtoAnalyzerDoc("proto-analyzers.rst");
-		CreateFileAnalyzerDoc("file-analyzers.rst");
-
-		std::list<BroDoc*>::iterator it;
-
-		for ( it = docs_generated.begin(); it != docs_generated.end(); ++it )
-			(*it)->WriteDocFile();
-
-		for ( it = docs_generated.begin(); it != docs_generated.end(); ++it )
-			delete *it;
-
-		terminate_bro();
-		return 0;
-		}
-
 	if ( reporter->Errors() > 0 )
 		{
 		delete dns_mgr;
@@ -918,6 +906,8 @@ int main(int argc, char** argv)
 
 	init_general_global_var();
 
+	broxygen_mgr->GenerateDocs();
+
 	if ( user_pcap_filter )
 		{
 		ID* id = global_scope()->Lookup("cmd_line_bpf_filter");

src/parse.y

@@ -2,7 +2,7 @@
 // See the file "COPYING" in the main distribution directory for copyright.
 %}
 
-%expect 85
+%expect 75
 
 %token TOK_ADD TOK_ADD_TO TOK_ADDR TOK_ANY
 %token TOK_ATENDIF TOK_ATELSE TOK_ATIF TOK_ATIFDEF TOK_ATIFNDEF
@@ -28,8 +28,6 @@
 
 %token TOK_DEBUG
 
-%token TOK_DOC TOK_POST_DOC
-
 %token TOK_NO_TEST
 
 %nonassoc TOK_HOOK
@@ -47,8 +45,7 @@
 %left '$' '[' ']' '(' ')' TOK_HAS_FIELD TOK_HAS_ATTR
 
 %type <b> opt_no_test opt_no_test_block
-%type <str> TOK_ID TOK_PATTERN_TEXT single_pattern TOK_DOC TOK_POST_DOC
+%type <str> TOK_ID TOK_PATTERN_TEXT single_pattern
-%type <str_l> opt_doc_list opt_post_doc_list
 %type <id> local_id global_id def_global_id event_id global_or_event_id resolve_id begin_func
 %type <id_l> local_id_list
 %type <ic> init_class
@@ -83,17 +80,13 @@
 #include "RE.h"
 #include "Scope.h"
 #include "Reporter.h"
-#include "BroDoc.h"
-#include "BroDocObj.h"
 #include "Brofiler.h"
+#include "broxygen/Manager.h"
 
-#include <list>
+#include <set>
 #include <string>
 
-extern Brofiler brofiler;
+extern const char* filename;  // Absolute path of file currently being parsed.
-extern BroDoc* current_reST_doc;
-extern int generate_documentation;
-extern std::list<std::string>* reST_doc_comments;
 
 YYLTYPE GetCurrentLocation();
 extern int yyerror(const char[]);
@@ -127,24 +120,13 @@ bool defining_global_ID = false;
 
 ID* func_id = 0;
 EnumType *cur_enum_type = 0;
-CommentedEnumType *cur_enum_type_doc = 0;
-const char* cur_enum_elem_id = 0;
-
-type_decl_list* fake_type_decl_list = 0;
-TypeDecl* last_fake_type_decl = 0;
-
 static ID* cur_decl_type_id = 0;
 
 static void parser_new_enum (void)
 	{
 	/* Starting a new enum definition. */
 	assert(cur_enum_type == NULL);
-	cur_enum_type = new EnumType(cur_decl_type_id->Name());
+	cur_enum_type = new EnumType();
-
-	// For documentation purposes, a separate type object is created
-	// in order to avoid overlap that can be caused by redefs.
-	if ( generate_documentation )
-		cur_enum_type_doc = new CommentedEnumType(cur_decl_type_id->Name());
 	}
 
 static void parser_redef_enum (ID *id)
@@ -160,53 +142,75 @@ static void parser_redef_enum (ID *id)
 		if ( ! cur_enum_type )
 			id->Error("not an enum");
 		}
-
-	if ( generate_documentation )
-		cur_enum_type_doc = new CommentedEnumType(id->Name());
 	}
 
-static void add_enum_comment (std::list<std::string>* comments)
+static type_decl_list* copy_type_decl_list(type_decl_list* tdl)
 	{
-	cur_enum_type_doc->AddComment(current_module, cur_enum_elem_id, comments);
+	if ( ! tdl )
-	}
+		return 0;
 
-static ID* create_dummy_id (ID* id, BroType* type)
+	type_decl_list* rval = new type_decl_list();
+
+	loop_over_list(*tdl, i)
 		{
-	ID* fake_id = new ID(copy_string(id->Name()), (IDScope) id->Scope(),
+		TypeDecl* td = (*tdl)[i];
-	                     is_export);
+		rval->append(new TypeDecl(*td));
+		}
 
-	fake_id->SetType(type->Ref());
+	return rval;
+	}
 
-	if ( id->AsType() )
+static attr_list* copy_attr_list(attr_list* al)
 	{
-		type->SetTypeID(copy_string(id->Name()));
+	if ( ! al )
-		fake_id->MakeType();
+		return 0;
-		}
 
-	return fake_id;
+	attr_list* rval = new attr_list();
-	}
 
-static std::list<std::string>* concat_opt_docs (std::list<std::string>* pre,
+	loop_over_list(*al, i)
-                                                std::list<std::string>* post)
 		{
-	if ( ! pre && ! post ) return 0;
+		Attr* a = (*al)[i];
-
+		::Ref(a);
-	if ( pre && ! post ) return pre;
+		rval->append(a);
-
-	if ( ! pre && post ) return post;
-
-	pre->splice(pre->end(), *post);
-	delete post;
-
-	return pre;
 		}
 
+	return rval;
+	}
+
+static void extend_record(ID* id, type_decl_list* fields, attr_list* attrs)
+	{
+	set<BroType*> types = type_aliases[id->Name()];
+
+	if ( types.empty() )
+		{
+		id->Error("failed to redef record: no types found in alias map");
+		return;
+		}
+
+	for ( set<BroType*>::const_iterator it = types.begin(); it != types.end(); )
+		{
+		RecordType* add_to = (*it)->AsRecordType();
+		const char* error = 0;
+		++it;
+
+		if ( it == types.end() )
+			error = add_to->AddFields(fields, attrs);
+		else
+			error = add_to->AddFields(copy_type_decl_list(fields),
+			                          copy_attr_list(attrs));
+
+		if ( error )
+			{
+			id->Error(error);
+			break;
+			}
+		}
+	}
 %}
 
 %union {
 	bool b;
 	char* str;
-	std::list<std::string>* str_l;
 	ID* id;
 	id_list* id_l;
 	init_class ic;
@@ -699,46 +703,24 @@ single_pattern:
 	;
 
 enum_body:
-		enum_body_list opt_post_doc_list
+		enum_body_list
 			{
 			$$ = cur_enum_type;
-
-			if ( generate_documentation )
-				{
-				add_enum_comment($2);
-				cur_enum_elem_id = 0;
-				}
-
 			cur_enum_type = NULL;
 			}
 
-	|	enum_body_list ',' opt_post_doc_list
+	|	enum_body_list ','
 			{
 			$$ = cur_enum_type;
-
-			if ( generate_documentation )
-				{
-				add_enum_comment($3);
-				cur_enum_elem_id = 0;
-				}
-
 			cur_enum_type = NULL;
 			}
 	;
 
 enum_body_list:
-		enum_body_elem opt_post_doc_list
+		enum_body_elem
-			{
-			if ( generate_documentation )
-				add_enum_comment($2);
-			}
 
-	|	enum_body_list ',' opt_post_doc_list
+	|	enum_body_list ',' enum_body_elem
-			{
+	;
-			if ( generate_documentation )
-				add_enum_comment($3);
-			} enum_body_elem
-;
 
 enum_body_elem:
 		/* TODO: We could also define this as TOK_ID '=' expr, (or
@@ -746,25 +728,19 @@ enum_body_elem:
 		   error messages if someboy tries to use constant variables as
 		   enumerator.
 		*/
-		opt_doc_list TOK_ID '=' TOK_CONSTANT
+		TOK_ID '=' TOK_CONSTANT
 			{
-			set_location(@2, @4);
+			set_location(@1, @3);
 			assert(cur_enum_type);
 
-			if ( $4->Type()->Tag() != TYPE_COUNT )
+			if ( $3->Type()->Tag() != TYPE_COUNT )
 				reporter->Error("enumerator is not a count constant");
 			else
-				cur_enum_type->AddName(current_module, $2, $4->InternalUnsigned(), is_export);
+				cur_enum_type->AddName(current_module, $1,
-
+				                       $3->InternalUnsigned(), is_export);
-			if ( generate_documentation )
-				{
-				cur_enum_type_doc->AddName(current_module, $2, $4->InternalUnsigned(), is_export);
-				cur_enum_elem_id = $2;
-				add_enum_comment($1);
-				}
 			}
 
-	|	opt_doc_list TOK_ID '=' '-' TOK_CONSTANT
+	|	TOK_ID '=' '-' TOK_CONSTANT
 			{
 			/* We only accept counts as enumerator, but we want to return a nice
 			   error message if users triy to use a negative integer (will also
@@ -773,18 +749,11 @@ enum_body_elem:
 			reporter->Error("enumerator is not a count constant");
 			}
 
-	|	opt_doc_list TOK_ID
+	|	TOK_ID
 			{
-			set_location(@2);
+			set_location(@1);
 			assert(cur_enum_type);
-			cur_enum_type->AddName(current_module, $2, is_export);
+			cur_enum_type->AddName(current_module, $1, is_export);
-
-			if ( generate_documentation )
-				{
-				cur_enum_type_doc->AddName(current_module, $2, is_export);
-				cur_enum_elem_id = $2;
-				add_enum_comment($1);
-				}
 			}
 	;
 
@@ -872,12 +841,11 @@ type:
 				}
 
 	|	TOK_RECORD '{'
-			{ ++in_record; do_doc_token_start(); }
+			{ ++in_record; }
 		type_decl_list
 			{ --in_record; }
 		'}'
 				{
-				do_doc_token_stop();
 				set_location(@1, @5);
 				$$ = new RecordType($4);
 				}
@@ -889,9 +857,8 @@ type:
 				$$ = 0;
 				}
 
-	|	TOK_ENUM '{' { set_location(@1); parser_new_enum(); do_doc_token_start(); } enum_body '}'
+	|	TOK_ENUM '{' { set_location(@1); parser_new_enum(); } enum_body '}'
 				{
-				do_doc_token_stop();
 				set_location(@1, @5);
 				$4->UpdateLocationEndInfo(@5);
 				$$ = $4;
@@ -983,45 +950,21 @@ type_decl_list:
 		type_decl_list type_decl
 			{
 			$1->append($2);
-
-			if ( generate_documentation && last_fake_type_decl )
-				{
-				fake_type_decl_list->append(last_fake_type_decl);
-				last_fake_type_decl = 0;
-				}
 			}
 	|
 			{
 			$$ = new type_decl_list();
-
-			if ( generate_documentation )
-				fake_type_decl_list = new type_decl_list();
 			}
 	;
 
 type_decl:
-		opt_doc_list TOK_ID ':' type opt_attr ';' opt_post_doc_list
+		TOK_ID ':' type opt_attr ';'
 			{
-			set_location(@2, @6);
+			set_location(@1, @4);
+			$$ = new TypeDecl($3, $1, $4, (in_record > 0));
 
-			if ( generate_documentation )
+			if ( in_record > 0 && cur_decl_type_id )
-				{
+				broxygen_mgr->RecordField(cur_decl_type_id, $$, ::filename);
-				// TypeDecl ctor deletes the attr list, so make a copy
-				attr_list* a = $5;
-				attr_list* a_copy = 0;
-
-				if ( a )
-					{
-					a_copy = new attr_list;
-					loop_over_list(*a, i)
-						a_copy->append((*a)[i]);
-					}
-
-				last_fake_type_decl = new CommentedTypeDecl(
-					$4, $2, a_copy, (in_record > 0), concat_opt_docs($1, $7));
-				}
-
-			$$ = new TypeDecl($4, $2, $5, (in_record > 0));
 			}
 	;
 
@@ -1055,9 +998,7 @@ decl:
 		TOK_MODULE TOK_ID ';'
 			{
 			current_module = $2;
-
+			broxygen_mgr->ModuleUsage(::filename, current_module);
-			if ( generate_documentation )
-				current_reST_doc->AddModule(current_module);
 			}
 
 	|	TOK_EXPORT '{' { is_export = true; } decl_list '}'
@@ -1066,171 +1007,51 @@ decl:
 	|	TOK_GLOBAL def_global_id opt_type init_class opt_init opt_attr ';'
 			{
 			add_global($2, $3, $4, $5, $6, VAR_REGULAR);
-
+			broxygen_mgr->Identifier($2);
-			if ( generate_documentation )
-				{
-				ID* id = $2;
-				if ( id->Type()->Tag() == TYPE_FUNC )
-					{
-					switch ( id->Type()->AsFuncType()->Flavor() ) {
-
-					case FUNC_FLAVOR_FUNCTION:
-						current_reST_doc->AddFunction(
-							new BroDocObj(id, reST_doc_comments));
-						break;
-
-					case FUNC_FLAVOR_EVENT:
-						current_reST_doc->AddEvent(
-							new BroDocObj(id, reST_doc_comments));
-						break;
-
-					case FUNC_FLAVOR_HOOK:
-						current_reST_doc->AddHook(
-							new BroDocObj(id, reST_doc_comments));
-						break;
-
-					default:
-						reporter->InternalError("invalid function flavor");
-						break;
-					}
-					}
-
-				else
-					{
-					current_reST_doc->AddStateVar(
-						new BroDocObj(id, reST_doc_comments));
-					}
-				}
 			}
 
 	|	TOK_CONST def_global_id opt_type init_class opt_init opt_attr ';'
 			{
 			add_global($2, $3, $4, $5, $6, VAR_CONST);
-
+			broxygen_mgr->Identifier($2);
-			if ( generate_documentation )
-				{
-				if ( $2->FindAttr(ATTR_REDEF) )
-					current_reST_doc->AddOption(
-						new BroDocObj($2, reST_doc_comments));
-				else
-					current_reST_doc->AddConstant(
-						new BroDocObj($2, reST_doc_comments));
-				}
 			}
 
 	|	TOK_REDEF global_id opt_type init_class opt_init opt_attr ';'
 			{
 			add_global($2, $3, $4, $5, $6, VAR_REDEF);
+			broxygen_mgr->Redef($2, ::filename);
+			}
 
-			if ( generate_documentation &&
+	|	TOK_REDEF TOK_ENUM global_id TOK_ADD_TO '{'
-				! streq("capture_filters", $2->Name()) )
+			{ parser_redef_enum($3); broxygen_mgr->Redef($3, ::filename); }
+		enum_body '}' ';'
 			{
-				ID* fake_id = create_dummy_id($2, $2->Type());
+			// Broxygen already grabbed new enum IDs as the type created them.
-				BroDocObj* o = new BroDocObj(fake_id, reST_doc_comments, true);
-				o->SetRole(true);
-				current_reST_doc->AddRedef(o);
-				}
 			}
 
-	|	TOK_REDEF TOK_ENUM global_id TOK_ADD_TO
+	|	TOK_REDEF TOK_RECORD global_id
-		'{' { parser_redef_enum($3); do_doc_token_start(); } enum_body '}' ';'
+			{ cur_decl_type_id = $3; broxygen_mgr->Redef($3, ::filename); }
-			{
+		TOK_ADD_TO '{'
-			do_doc_token_stop();
+			{ ++in_record; }
-
-			if ( generate_documentation )
-				{
-				ID* fake_id = create_dummy_id($3, cur_enum_type_doc);
-				cur_enum_type_doc = 0;
-				BroDocObj* o = new BroDocObj(fake_id, reST_doc_comments, true);
-				o->SetRole(true);
-
-				if ( extract_module_name(fake_id->Name()) == "Notice" &&
-				     extract_var_name(fake_id->Name()) == "Type" )
-					current_reST_doc->AddNotice(o);
-				else
-					current_reST_doc->AddRedef(o);
-				}
-			}
-
-	|	TOK_REDEF TOK_RECORD global_id TOK_ADD_TO
-			'{' { ++in_record; do_doc_token_start(); }
 		type_decl_list
-			{ --in_record; do_doc_token_stop(); } '}' opt_attr ';'
+			{ --in_record; }
+		'}' opt_attr ';'
 			{
+			cur_decl_type_id = 0;
+
 			if ( ! $3->Type() )
 				$3->Error("unknown identifier");
 			else
-				{
+				extend_record($3, $8, $11);
-				RecordType* add_to = $3->Type()->AsRecordType();
-				if ( ! add_to )
-					$3->Error("not a record type");
-				else
-					{
-					const char* error = add_to->AddFields($7, $10);
-					if ( error )
-						$3->Error(error);
-					else if ( generate_documentation )
-						{
-						if ( fake_type_decl_list )
-							{
-							BroType* fake_record =
-								new RecordType(fake_type_decl_list);
-							ID* fake = create_dummy_id($3, fake_record);
-							fake_type_decl_list = 0;
-							BroDocObj* o =
-								new BroDocObj(fake, reST_doc_comments, true);
-							o->SetRole(true);
-							current_reST_doc->AddRedef(o);
-							}
-						else
-							{
-							fprintf(stderr, "Warning: doc mode did not process "
-								"record extension for '%s', CommentedTypeDecl"
-								"list unavailable.\n", $3->Name());
-							}
-						}
-					}
-				}
 			}
 
-	|	TOK_TYPE global_id ':' { cur_decl_type_id = $2; } type opt_attr ';'
+	|	TOK_TYPE global_id ':'
+			{ cur_decl_type_id = $2; broxygen_mgr->StartType($2);  }
+		type opt_attr ';'
 			{
 			cur_decl_type_id = 0;
-			add_type($2, $5, $6, 0);
+			add_type($2, $5, $6);
-
+			broxygen_mgr->Identifier($2);
-			if ( generate_documentation )
-				{
-				TypeTag t = $2->AsType()->Tag();
-				if ( t == TYPE_ENUM && cur_enum_type_doc )
-					{
-					ID* fake = create_dummy_id($2, cur_enum_type_doc);
-					cur_enum_type_doc = 0;
-					current_reST_doc->AddType(
-						new BroDocObj(fake, reST_doc_comments, true));
-					}
-
-				else if ( t == TYPE_RECORD && fake_type_decl_list )
-					{
-					BroType* fake_record = new RecordType(fake_type_decl_list);
-					ID* fake = create_dummy_id($2, fake_record);
-					fake_type_decl_list = 0;
-					current_reST_doc->AddType(
-						new BroDocObj(fake, reST_doc_comments, true));
-					}
-
-				else
-					current_reST_doc->AddType(
-						new BroDocObj($2, reST_doc_comments));
-				}
-			}
-
-	|	TOK_EVENT event_id ':' type_list opt_attr ';'
-			{
-			add_type($2, $4, $5, 1);
-
-			if ( generate_documentation )
-				current_reST_doc->AddEvent(
-					new BroDocObj($2, reST_doc_comments));
 			}
 
 	|	func_hdr func_body
@@ -1258,18 +1079,13 @@ func_hdr:
 			begin_func($2, current_module.c_str(),
 				FUNC_FLAVOR_FUNCTION, 0, $3);
 			$$ = $3;
-			if ( generate_documentation )
+			broxygen_mgr->Identifier($2);
-				current_reST_doc->AddFunction(
-					new BroDocObj($2, reST_doc_comments));
 			}
 	|	TOK_EVENT event_id func_params
 			{
 			begin_func($2, current_module.c_str(),
 				   FUNC_FLAVOR_EVENT, 0, $3);
 			$$ = $3;
-			if ( generate_documentation )
-				current_reST_doc->AddEventHandler(
-					new BroDocObj($2, reST_doc_comments));
 			}
 	|	TOK_HOOK def_global_id func_params
 			{
@@ -1278,9 +1094,6 @@ func_hdr:
 			begin_func($2, current_module.c_str(),
 				   FUNC_FLAVOR_HOOK, 0, $3);
 			$$ = $3;
-			if ( generate_documentation )
-				current_reST_doc->AddHookHandler(
-					new BroDocObj($2, reST_doc_comments));
 			}
 	|	TOK_REDEF TOK_EVENT event_id func_params
 			{
@@ -1729,40 +1542,6 @@ resolve_id:
 			}
 	;
 
-opt_post_doc_list:
-		opt_post_doc_list TOK_POST_DOC
-			{
-			$1->push_back($2);
-			$$ = $1;
-			}
-	|
-		TOK_POST_DOC
-			{
-			$$ = new std::list<std::string>();
-			$$->push_back($1);
-			delete [] $1;
-			}
-	|
-			{ $$ = 0; }
-	;
-
-opt_doc_list:
-		opt_doc_list TOK_DOC
-			{
-			$1->push_back($2);
-			$$ = $1;
-			}
-	|
-		TOK_DOC
-			{
-			$$ = new std::list<std::string>();
-			$$->push_back($1);
-			delete [] $1;
-			}
-	|
-			{ $$ = 0; }
-	;
-
 opt_no_test:
 		TOK_NO_TEST
 			{ $$ = true; }
@@ -1788,10 +1567,6 @@ int yyerror(const char msg[])
 	else
 		sprintf(msgbuf, "%s, at or near \"%s\"", msg, last_tok);
 
-	if ( generate_documentation )
-		strcat(msgbuf, "\nDocumentation mode is enabled: "
-		       "remember to check syntax of ## style comments\n");
-
 	if ( in_debug )
 		g_curr_debug_error = copy_string(msg);
 

src/plugin/ComponentManager.h

@@ -10,6 +10,7 @@
 #include "Var.h"
 #include "Val.h"
 #include "Reporter.h"
+#include "broxygen/Manager.h"
 
 namespace plugin {
 
@@ -130,9 +131,10 @@ template <class T, class C>
 ComponentManager<T, C>::ComponentManager(const string& arg_module)
 	: module(arg_module)
 	{
-	tag_enum_type = new EnumType(module + "::Tag");
+	tag_enum_type = new EnumType();
 	::ID* id = install_ID("Tag", module.c_str(), true, true);
-	add_type(id, tag_enum_type, 0, 0);
+	add_type(id, tag_enum_type, 0);
+	broxygen_mgr->Identifier(id);
 	}
 
 template <class T, class C>

src/reporter.bif

@@ -4,7 +4,7 @@
 ##! If event handlers do exist, it's assumed they take care of determining
 ##! how/where to output the messages.
 ##!
-##! See :doc:`/scripts/base/frameworks/reporter/main` for a convenient
+##! See :doc:`/scripts/base/frameworks/reporter/main.bro` for a convenient
 ##! reporter message logging framework.
 
 module Reporter;

src/scan.l

@@ -23,16 +23,15 @@
 #include "Debug.h"
 #include "PolicyFile.h"
 #include "broparse.h"
-#include "BroDoc.h"
 #include "Reporter.h"
 #include "RE.h"
 #include "Net.h"
 
 #include "analyzer/Analyzer.h"
+#include "broxygen/Manager.h"
 
 extern YYLTYPE yylloc;	// holds start line and column of token
-extern int print_loaded_scripts;
+extern EnumType* cur_enum_type;
-extern int generate_documentation;
 
 // Track the @if... depth.
 ptr_compat_int current_depth = 0;
@@ -40,10 +39,8 @@ ptr_compat_int current_depth = 0;
 int_list if_stack;
 
 int line_number = 1;
-const char* filename = 0;
+const char* filename = 0; // Absolute path of file currently being parsed.
-BroDoc* current_reST_doc = 0;
+static const char* last_id_tok = 0;
-static BroDoc* last_reST_doc = 0;
-string current_scanned_file_path;
 
 char last_tok[128];
 
@@ -56,56 +53,38 @@ char last_tok[128];
 	if ( ((result = fread(buf, 1, max_size, yyin)) == 0) && ferror(yyin) ) \
 		reporter->Error("read failed with \"%s\"", strerror(errno));
 
-// reST documents that we've created (or have at least opened so far).
+static string find_relative_file(const string& filename, const string& ext)
-std::list<BroDoc*> docs_generated;
-
-// reST comments (those starting with ##) seen so far.
-std::list<std::string>* reST_doc_comments = 0;
-
-// Print current contents of reST_doc_comments list to stderr.
-void print_current_reST_doc_comments();
-
-// Delete the reST_doc_comments list object.
-void clear_reST_doc_comments();
-
-// Adds changes to capture_filter to the current script's reST documentation.
-static void check_capture_filter_changes();
-
-static const char* canon_doc_comment(const char* comment)
 	{
-	// "##Text" and "## Text" are treated the same in order to be able
+	if ( filename.empty() )
-	// to still preserve indentation level, but not unintentionally
+		return string();
-	// signify an indentation level for all the text when using
-	// the "## Text" style.
-	return ( comment[0] == ' ' ) ? comment + 1 : comment;
-	}
 
-static std::string canon_doc_func_param(const char* id_start)
+	if ( filename[0] == '.' )
-	{
+		return find_file(filename, SafeDirname(::filename).result, ext);
-	std::string id_name(id_start, strcspn(id_start, ":"));
-	const char* comment = id_start + id_name.size() + 1;
-	std::string doc;
-
-	if ( id_name == "Returns" )
-		doc.append(":returns:").append(comment);
 	else
-		doc.append(":param ").append(id_name).append(":").append(comment);
+		return find_file(filename, bro_path(), ext);
-	return doc;
 	}
 
-static ino_t get_inode_num(FILE* f, const char* filename)
+static ino_t get_inode_num(FILE* f, const string& path)
 	{
 	struct stat b;
 
 	if ( fstat(fileno(f), &b) )
-		{
+		reporter->FatalError("fstat of %s failed: %s\n", path.c_str(),
-		reporter->Error("failed to fstat fd of %s\n", filename);
+		                     strerror(errno));
-		exit(1);
-		}
 
 	return b.st_ino;
 	}
 
+static ino_t get_inode_num(const string& path)
+	{
+	FILE* f = open_file(path);
+
+	if ( ! f )
+		reporter->FatalError("failed to open %s\n", path.c_str());
+
+	return get_inode_num(f, path);
+	}
+
 class FileInfo {
 public:
 	FileInfo(string restore_module = "");
@@ -116,8 +95,6 @@ public:
 	const char* name;
 	int line;
 	int level;
-	BroDoc* doc;
-	string path;
 };
 
 // A stack of input buffers we're scanning.  file_stack[len-1] is the
@@ -141,7 +118,6 @@ static int load_files(const char* file);
 
 %x RE
 %x IGNORE
-%s DOC
 
 OWS	[ \t]*
 WS	[ \t]+
@@ -159,63 +135,19 @@ ESCSEQ	(\\([^\n]|[0-7]+|x[[:xdigit:]]+))
 %%
 
 ##!.* {
-	// Add this format of comments to the script documentation's "summary".
+	broxygen_mgr->SummaryComment(::filename, yytext + 3);
-	if ( generate_documentation )
-		current_reST_doc->AddSummary(canon_doc_comment(yytext + 3));
 	}
 
-<DOC>##<.* {
-	yylval.str = copy_string(canon_doc_comment(yytext + 3));
-	return TOK_POST_DOC;
-}
-
-<DOC>##{OWS}{ID}:{WS}.* {
-	const char* id_start = skip_whitespace(yytext + 2);
-	yylval.str = copy_string(canon_doc_func_param(id_start).c_str());
-	return TOK_DOC;
-}
-
-<DOC>##.* {
-	if ( yytext[2] != '#' )
-		{
-		yylval.str = copy_string(canon_doc_comment(yytext + 2));
-		return TOK_DOC;
-		}
-}
-
-##{OWS}{ID}:{WS}.* {
-	if ( generate_documentation )
-		{
-		// Comment is documenting either a function parameter or return type,
-		// so appropriate reST markup substitutions are automatically made
-		// in order to distinguish them from other comments.
-		if ( ! reST_doc_comments )
-			reST_doc_comments = new std::list<std::string>();
-
-		// always insert a blank line so that this param/return markup
-		// 1) doesn't show up in the summary section in the case that it's
-		//    the first comment for the function/event
-		// 2) has a blank line between it and non-field-list reST markup,
-		//    which is required for correct HTML rendering by Sphinx
-		reST_doc_comments->push_back("");
-		const char* id_start = skip_whitespace(yytext + 2);
-		reST_doc_comments->push_back(canon_doc_func_param(id_start));
-		}
-}
-
 ##<.* {
-	if ( generate_documentation && BroDocObj::last )
+	string hint(cur_enum_type && last_id_tok ?
-		BroDocObj::last->AddDocString(canon_doc_comment(yytext + 3));
+	            make_full_var_name(current_module.c_str(), last_id_tok) : "");
+
+	broxygen_mgr->PostComment(yytext + 3, hint);
 }
 
 ##.* {
-	if ( generate_documentation && (yytext[2] != '#') )
+	if ( yytext[2] != '#' )
-		{
+		broxygen_mgr->PreComment(yytext + 2);
-		if ( ! reST_doc_comments )
-			reST_doc_comments = new std::list<std::string>();
-
-		reST_doc_comments->push_back(canon_doc_comment(yytext + 2));
-		}
 }
 
 #{OWS}@no-test.* return TOK_NO_TEST;
@@ -224,7 +156,7 @@ ESCSEQ	(\\([^\n]|[0-7]+|x[[:xdigit:]]+))
 
 {WS}	/* eat whitespace */
 
-<INITIAL,IGNORE,DOC>\n	{
+<INITIAL,IGNORE>\n	{
 			++line_number;
 			++yylloc.first_line;
 			++yylloc.last_line;
@@ -345,7 +277,7 @@ when	return TOK_WHEN;
 @DEBUG	return TOK_DEBUG;	// marks input for debugger
 
 @DIR	{
-	string rval = current_scanned_file_path;
+	string rval = SafeDirname(::filename).result;
 
 	if ( ! rval.empty() && rval[0] == '.' )
 		{
@@ -361,69 +293,41 @@ when	return TOK_WHEN;
 	}
 
 @FILENAME	{
-	char* filename_copy = copy_string(::filename);
+	RET_CONST(new StringVal(SafeBasename(::filename).result));
-	const char* bname = basename(filename_copy);
-
-	if ( ! bname )
-		reporter->InternalError("basename failed: %s", strerror(errno));
-
-	StringVal* rval = new StringVal(bname);
-	delete [] filename_copy;
-	RET_CONST(rval);
 	}
 
 @load{WS}{FILE}	{
 	const char* new_file = skip_whitespace(yytext + 5);	// Skip "@load".
-	if ( generate_documentation )
+	string loader = ::filename;  // load_files may change ::filename, save copy
-		{
+	string loading = find_relative_file(new_file, "bro");
-		current_reST_doc->AddImport(new_file);
-
-		if ( reST_doc_comments )
-			{
-			fprintf(stderr, "Warning: unconsumed reST documentation is being "
-					"discarded before doing '@load %s' in %s:\n",
-					 new_file, current_reST_doc->GetSourceFileName());
-			clear_reST_doc_comments();
-			}
-		}
 	(void) load_files(new_file);
+	broxygen_mgr->ScriptDependency(loader, loading);
 	}
 
 @load-sigs{WS}{FILE} {
-	const char* new_sig_file = skip_whitespace(yytext + 10);
+	const char* file = skip_whitespace(yytext + 10);
-	const char* full_filename = 0;
+	string path = find_relative_file(file, "sig");
-	FILE* f = search_for_file(new_sig_file, "sig", &full_filename, false, 0);
 
-	if ( f )
+	if ( path.empty() )
-		{
-		sig_files.push_back(full_filename);
-		fclose(f);
-		delete [] full_filename;
-		}
-	else
 		reporter->Error("failed to find file associated with @load-sigs %s",
-		                new_sig_file);
+		                file);
+	else
+		sig_files.push_back(copy_string(path.c_str()));
 	}
 
 @unload{WS}{FILE}	{
 	// Skip "@unload".
-	const char* new_file = skip_whitespace(yytext + 7);
+	const char* file = skip_whitespace(yytext + 7);
-
+	string path = find_relative_file(file, "bro");
-	// All we have to do is pretend we've already scanned it.
-	const char* full_filename;
-	FILE* f = search_for_file(new_file, "bro", &full_filename, true, 0);
-
-	if ( f )
-		{
-		ScannedFile sf(get_inode_num(f, full_filename), file_stack.length(), full_filename, "", true);
-		files_scanned.push_back(sf);
-
-		fclose(f);
-		delete [] full_filename;
-		}
 
+	if ( path.empty() )
+		reporter->Error("failed find file associated with @unload %s", file);
 	else
-		reporter->Error("failed find file associated with @unload %s", new_file);
+		{
+		// All we have to do is pretend we've already scanned it.
+		ScannedFile sf(get_inode_num(path), file_stack.length(), path, true);
+		files_scanned.push_back(sf);
+		}
 	}
 
 @prefixes{WS}("+"?)={WS}{PREFIX}	{
@@ -464,6 +368,7 @@ F	RET_CONST(new Val(false, TYPE_BOOL))
 
 {ID}	{
 	yylval.str = copy_string(yytext);
+	last_id_tok = yylval.str;
 	return TOK_ID;
 	}
 
@@ -577,22 +482,35 @@ YYLTYPE GetCurrentLocation()
 	return currloc;
 	}
 
+
+static bool already_scanned(ino_t i)
+	{
+	list<ScannedFile>::const_iterator it;
+
+	for ( it = files_scanned.begin(); it != files_scanned.end(); ++it )
+		if ( it->inode == i )
+			return true;
+
+	return false;
+	}
+
+static bool already_scanned(const string& path)
+	{
+	return already_scanned(get_inode_num(path));
+	}
+
 static int load_files(const char* orig_file)
 	{
 	// Whether we pushed on a FileInfo that will restore the
 	// current module after the final file has been scanned.
 	bool did_module_restore = false;
-
+	string file_path;
-	const char* full_filename = "<internal error>";
+	FILE* f = 0;
-	const char* bropath_subpath = "<internal error>";
-	const char* bropath_subpath_delete = 0;
-	FILE* f;
 
 	if ( streq(orig_file, "-") )
 		{
 		f = stdin;
-		full_filename = "<stdin>";
+		file_path = "<stdin>";
-		bropath_subpath = "";
 
 		if ( g_policy_debug )
 			{
@@ -603,54 +521,35 @@ static int load_files(const char* orig_file)
 
 	else
 		{
-		f = search_for_file(orig_file, "bro", &full_filename, true, &bropath_subpath);
+		file_path = find_relative_file(orig_file, "bro");
-		bropath_subpath_delete = bropath_subpath; // This will be deleted.
+
+		if ( file_path.empty() )
+			reporter->FatalError("can't find %s", orig_file);
+
+		if ( is_dir(file_path.c_str()) )
+			f = open_package(file_path);
+		else
+			f = open_file(file_path);
+
+		if ( ! f )
+			reporter->FatalError("can't open %s", file_path.c_str());
 		}
 
-	if ( f )
+	ino_t i = get_inode_num(f, file_path);
-		{
-		ino_t i = get_inode_num(f, full_filename);
-		std::list<ScannedFile>::const_iterator it;
 
-		for ( it = files_scanned.begin(); it != files_scanned.end(); ++it )
+	if ( already_scanned(i) )
-			{
-			if ( it->inode == i )
-				{
-				if ( f != stdin )
-					{
-					fclose(f);
-					delete [] full_filename;
-					delete [] bropath_subpath_delete;
-					}
 		return 0;
-				}
-			}
 
-		ScannedFile sf(i, file_stack.length(), full_filename, bropath_subpath);
+	ScannedFile sf(i, file_stack.length(), file_path);
 	files_scanned.push_back(sf);
 
-		if ( g_policy_debug )
+	if ( g_policy_debug && ! file_path.empty() )
 		{
-			// Add the filename to the file mapping
+		// Add the filename to the file mapping table (Debug.h).
-			// table (Debug.h).
 		Filemap* map = new Filemap;
-
+		HashKey* key = new HashKey(file_path.c_str());
-			// Make sure it wasn't already read in.
-			HashKey* key = new HashKey(full_filename);
-			if ( g_dbgfilemaps.Lookup(key) )
-				{
-				// reporter->Warning("Not re-reading policy file; check BRO_PREFIXES:", full_filename);
-				fclose(f);
-				delete key;
-				return 0;
-				}
-			else
-				{
 		g_dbgfilemaps.Insert(key, map);
-				}
+		LoadPolicyFileText(file_path.c_str());
-
-			if ( full_filename )
-				LoadPolicyFileText(full_filename);
 		}
 
 	// Remember where we were.  If this is the first
@@ -666,17 +565,7 @@ static int load_files(const char* orig_file)
 	else
 		file_stack.append(new FileInfo);
 
-		char* tmp = copy_string(full_filename);
+	broxygen_mgr->Script(file_path);
-		current_scanned_file_path = dirname(tmp);
-		delete [] tmp;
-
-		if ( generate_documentation )
-			{
-			current_reST_doc = new BroDoc(bropath_subpath, full_filename);
-			docs_generated.push_back(current_reST_doc);
-			}
-
-		delete [] bropath_subpath_delete;
 
 	// "orig_file", could be an alias for yytext, which is ephemeral
 	//  and will be zapped after the yy_switch_to_buffer() below.
@@ -686,14 +575,7 @@ static int load_files(const char* orig_file)
 
 	// Don't delete the old filename - it's pointed to by
 	// every BroObj created when parsing it.
-		yylloc.filename = filename = full_filename;
+	yylloc.filename = filename = copy_string(file_path.c_str());
-		}
-
-	else
-		{
-		reporter->Error("can't open %s", full_filename);
-		exit(1);
-		}
 
 	return 1;
 	}
@@ -776,28 +658,11 @@ void do_atendif()
 	--current_depth;
 	}
 
-void do_doc_token_start()
-	{
-	if ( generate_documentation )
-	    BEGIN(DOC);
-	}
-
-void do_doc_token_stop()
-	{
-	if ( generate_documentation )
-	    BEGIN(INITIAL);
-	}
-
 // Be careful to never delete things from this list, as the strings
 // are referred to (in order to save the locations of tokens and statements,
 // for error reporting and debugging).
 static name_list input_files;
 
-const char* get_current_input_filename()
-	{
-	return ::filename;
-	}
-
 void add_input_file(const char* file)
 	{
 	if ( ! file )
@@ -852,8 +717,6 @@ int yywrap()
 	// Stack is now empty.
 	while ( input_files.length() > 0 )
 		{
-		check_capture_filter_changes();
-
 		if ( load_files(input_files[0]) )
 			{
 			// Don't delete the filename - it's pointed to by
@@ -867,8 +730,6 @@ int yywrap()
 		(void) input_files.remove_nth(0);
 		}
 
-	check_capture_filter_changes();
-
 	// For each file scanned so far, and for each @prefix, look for a
 	// prefixed and flattened version of the loaded file in BROPATH. The
 	// flattening involves taking the path in BROPATH in which the
@@ -891,23 +752,22 @@ int yywrap()
 			if ( ! prefixes[i][0] )
 				continue;
 
-			string s;
+			string canon = without_bropath_component(it->name);
-			s = dot_canon(it->subpath.c_str(), it->name.c_str(), prefixes[i]);
+			string flat = flatten_script_name(canon, prefixes[i]);
-			FILE* f = search_for_file(s.c_str(), "bro", 0, false, 0);
+			string path = find_relative_file(flat, "bro");
+
+			if ( ! path.empty() )
+				{
+				add_input_file(path.c_str());
+				found_prefixed_files = true;
+				}
 
 			//printf("====== prefix search ======\n");
 			//printf("File  : %s\n", it->name.c_str());
-			//printf("Path  : %s\n", it->subpath.c_str());
+			//printf("Canon : %s\n", canon.c_str());
-			//printf("Dotted: %s\n", s.c_str());
+			//printf("Flat  : %s\n", flat.c_str());
-			//printf("Found : %s\n", f ? "T" : "F");
+			//printf("Found : %s\n", path.empty() ? "F" : "T");
 			//printf("===========================\n");
-
-			if ( f )
-				{
-				add_input_file(s.c_str());
-				found_prefixed_files = true;
-				fclose(f);
-				}
 			}
 		}
 
@@ -977,9 +837,6 @@ int yywrap()
 		return 0;
 		}
 
-	if ( generate_documentation )
-		clear_reST_doc_comments();
-
 	// Otherwise, we are done.
 	return 1;
 	}
@@ -990,8 +847,6 @@ FileInfo::FileInfo(string arg_restore_module)
 	restore_module = arg_restore_module;
 	name = ::filename;
 	line = ::line_number;
-	doc = ::current_reST_doc;
-	path = current_scanned_file_path;
 	}
 
 FileInfo::~FileInfo()
@@ -1002,56 +857,7 @@ FileInfo::~FileInfo()
 	yy_switch_to_buffer(buffer_state);
 	yylloc.filename = filename = name;
 	yylloc.first_line = yylloc.last_line = line_number = line;
-	last_reST_doc = current_reST_doc;
-	current_reST_doc = doc;
-	current_scanned_file_path = path;
 
 	if ( restore_module != "" )
 		current_module = restore_module;
 	}
-
-static void check_capture_filter_changes()
-	{
-	if ( ! generate_documentation )
-	    return;
-
-	// Lookup the "capture_filters" identifier, if it has any defined
-	// value, add it to the script's reST documentation, and finally
-	// clear the table so it doesn't taint the documentation for
-	// subsequent scripts.
-
-	ID* capture_filters = global_scope()->Lookup("capture_filters");
-
-	if ( capture_filters )
-		{
-		ODesc desc;
-		desc.SetIndentSpaces(4);
-		capture_filters->ID_Val()->Describe(&desc);
-		last_reST_doc->SetPacketFilter(desc.Description());
-		capture_filters->ID_Val()->AsTableVal()->RemoveAll();
-		}
-	}
-
-void print_current_reST_doc_comments()
-	{
-	if ( ! reST_doc_comments )
-		return;
-
-	std::list<std::string>::iterator it;
-
-	for ( it = reST_doc_comments->begin(); it != reST_doc_comments->end(); ++it )
-		fprintf(stderr, "##%s\n", it->c_str());
-	}
-
-void clear_reST_doc_comments()
-	{
-	if ( ! reST_doc_comments )
-		return;
-
-	fprintf(stderr, "Warning: %zu unconsumed reST comments:\n",
-			reST_doc_comments->size());
-
-	print_current_reST_doc_comments();
-	delete reST_doc_comments;
-	reST_doc_comments = 0;
-	}

src/util-config.h.in

@@ -1,5 +1,2 @@
 #define BRO_SCRIPT_INSTALL_PATH "@BRO_SCRIPT_INSTALL_PATH@"
-#define BRO_SCRIPT_SOURCE_PATH "@BRO_SCRIPT_SOURCE_PATH@"
-#define BRO_BUILD_SOURCE_PATH "@CMAKE_BINARY_DIR@/src"
-#define BRO_BUILD_SCRIPTS_PATH "@CMAKE_BINARY_DIR@/scripts"
 #define BRO_MAGIC_INSTALL_PATH "@BRO_MAGIC_INSTALL_PATH@"

src/util.cc

@@ -590,6 +590,33 @@ const char* fmt_access_time(double t)
 	return buf;
 	}
 
+bool ensure_intermediate_dirs(const char* dirname)
+	{
+	if ( ! dirname || strlen(dirname) == 0 )
+		return false;
+
+	bool absolute = dirname[0] == '/';
+	string path = normalize_path(dirname);
+
+	vector<string> path_components;
+	tokenize_string(path, "/", &path_components);
+
+	string current_dir;
+
+	for ( size_t i = 0; i < path_components.size(); ++i )
+		{
+		if ( i > 0 || absolute )
+			current_dir += "/";
+
+		current_dir += path_components[i];
+
+		if ( ! ensure_dir(current_dir.c_str()) )
+			return false;
+		}
+
+	return true;
+	}
+
 bool ensure_dir(const char *dirname)
 	{
 	struct stat st;
@@ -909,90 +936,138 @@ string bro_prefixes()
 
 const char* PACKAGE_LOADER = "__load__.bro";
 
-// If filename is pointing to a directory that contains a file called
+FILE* open_file(const string& path, const string& mode)
-// PACKAGE_LOADER, returns the files path. Otherwise returns filename itself.
-// In both cases, the returned string is newly allocated.
-static const char* check_for_dir(const char* filename, bool load_pkgs)
 	{
-	if ( load_pkgs && is_dir(filename) )
+	if ( path.empty() )
-		{
+		return 0;
-		char init_filename_buf[1024];
-		safe_snprintf(init_filename_buf, sizeof(init_filename_buf),
-			      "%s/%s", filename, PACKAGE_LOADER);
 
-		if ( access(init_filename_buf, R_OK) == 0 )
+	FILE* rval = fopen(path.c_str(), mode.c_str());
-			return copy_string(init_filename_buf);
-		}
 
-	return copy_string(filename);
+	if ( ! rval )
-	}
-
-static FILE* open_file(const char* filename, const char** full_filename, bool load_pkgs)
-	{
-	filename = check_for_dir(filename, load_pkgs);
-
-	if ( full_filename )
-		*full_filename = copy_string(filename);
-
-	FILE* f = fopen(filename, "r");
-
-	if ( ! f )
 		{
 		char buf[256];
 		strerror_r(errno, buf, sizeof(buf));
 		reporter->Error("Failed to open file %s: %s", filename, buf);
 		}
 
-	delete [] filename;
+	return rval;
-
-	return f;
 	}
 
-// Canonicalizes a given 'file' that lives in 'path' into a flattened,
+static bool can_read(const string& path)
-// dotted format.  If the optional 'prefix' argument is given, it is
-// prepended to the dotted-format, separated by another dot.
-// If 'file' is __load__.bro, that part is discarded when constructing
-// the final dotted-format.
-string dot_canon(string path, string file, string prefix)
 	{
-	string dottedform(prefix);
+	return access(path.c_str(), R_OK) == 0;
-	if ( prefix != "" )
-		dottedform.append(".");
-	dottedform.append(path);
-	char* tmp = copy_string(file.c_str());
-	char* bname = basename(tmp);
-	if ( ! streq(bname, PACKAGE_LOADER) )
-		{
-		if ( path != "" )
-			dottedform.append(".");
-		dottedform.append(bname);
 	}
+
+FILE* open_package(string& path, const string& mode)
+	{
+	string arg_path = path;
+	path.append("/").append(PACKAGE_LOADER);
+
+	if ( can_read(path) )
+		return open_file(path, mode);
+
+	reporter->Error("Failed to open package '%s': missing '%s' file",
+	                arg_path.c_str(), PACKAGE_LOADER);
+	return 0;
+	}
+
+SafePathOp::SafePathOp(PathOpFn fn, const char* path, bool error_aborts)
+	{
+	DoFunc(fn, path ? path : "", error_aborts);
+	}
+
+SafePathOp::SafePathOp(PathOpFn fn, const string& path, bool error_aborts)
+	{
+	DoFunc(fn, path, error_aborts);
+	}
+
+void SafePathOp::DoFunc(PathOpFn fn, const string& path, bool error_aborts)
+	{
+	char* tmp = copy_string(path.c_str());
+	char* rval = fn(tmp);
+
+	if ( rval )
+		{
+		result = rval;
+		error = false;
+		}
+	else
+		{
+		if ( error_aborts )
+			reporter->InternalError("Path operation failed on %s: %s",
+			                        tmp ? tmp : "<null>", strerror(errno));
+		else
+			error = true;
+		}
+
 	delete [] tmp;
-	size_t n;
-	while ( (n = dottedform.find("/")) != string::npos )
-		dottedform.replace(n, 1, ".");
-	return dottedform;
 	}
 
-// returns a normalized version of a path, removing duplicate slashes,
+string implode_string_vector(const std::vector<std::string>& v,
-// extraneous dots that refer to the current directory, and pops as many
+                             const std::string& delim)
-// parent directories referred to by "../" as possible
+	{
-const char* normalize_path(const char* path)
+	string rval;
+
+	for ( size_t i = 0; i < v.size(); ++i )
+		{
+		if ( i > 0 )
+			rval += delim;
+
+		rval += v[i];
+		}
+
+	return rval;
+	}
+
+string flatten_script_name(const string& name, const string& prefix)
+	{
+	string rval = prefix;
+
+	if ( ! rval.empty() )
+		rval.append(".");
+
+	if ( SafeBasename(name).result == PACKAGE_LOADER )
+		rval.append(SafeDirname(name).result);
+	else
+		rval.append(name);
+
+	size_t i;
+
+	while ( (i = rval.find('/')) != string::npos )
+		rval[i] = '.';
+
+	return rval;
+	}
+
+vector<string>* tokenize_string(string input, const string& delim,
+                                vector<string>* rval)
+	{
+	if ( ! rval )
+		rval = new vector<string>();
+
+	size_t n;
+
+	while ( (n = input.find(delim)) != string::npos )
+		{
+		rval->push_back(input.substr(0, n));
+		input.erase(0, n + 1);
+		}
+
+	rval->push_back(input);
+	return rval;
+	}
+
+
+string normalize_path(const string& path)
 	{
 	size_t n;
-	string p(path);
 	vector<string> components, final_components;
 	string new_path;
 
-	if ( p[0] == '/' )
+	if ( path[0] == '/' )
 		new_path = "/";
 
-	while ( (n = p.find("/")) != string::npos )
+	tokenize_string(path, "/", &components);
-		{
-		components.push_back(p.substr(0, n));
-		p.erase(0, n + 1);
-		}
-	components.push_back(p);
 
 	vector<string>::const_iterator it;
 	for ( it = components.begin(); it != components.end(); ++it )
@@ -1018,127 +1093,82 @@ const char* normalize_path(const char* path)
 	if ( new_path.size() > 1 && new_path[new_path.size() - 1] == '/' )
 		new_path.erase(new_path.size() - 1);
 
-	return copy_string(new_path.c_str());
+	return new_path;
 	}
 
-// Returns the subpath of the root Bro script install/source/build directory in
+string without_bropath_component(const string& path)
-// which the loaded file is located.  If it's not under a subpath of that
-// directory (e.g. cwd or custom path) then the full path is returned.
-void get_script_subpath(const std::string& full_filename, const char** subpath)
 	{
-	size_t p;
+	string rval = normalize_path(path);
-	std::string my_subpath(full_filename);
 
-	// get the parent directory of file (if not already a directory)
+	vector<string> paths;
-	if ( ! is_dir(full_filename.c_str()) )
+	tokenize_string(bro_path(), ":", &paths);
+
+	for ( size_t i = 0; i < paths.size(); ++i )
 		{
-		char* tmp = copy_string(full_filename.c_str());
+		string common = normalize_path(paths[i]);
-		my_subpath = dirname(tmp);
+
-		delete [] tmp;
+		if ( rval.find(common) != 0 )
+			continue;
+
+		// Found the containing directory.
+		rval.erase(0, common.size());
+
+		// Remove leading path separators.
+		while ( rval.size() && rval[0] == '/' )
+			rval.erase(0, 1);
+
+		return rval;
 		}
 
-	// first check if this is some subpath of the installed scripts root path,
+	return rval;
-	// if not check if it's a subpath of the script source root path,
-	// then check if it's a subpath of the build directory (where BIF scripts
-	// will get generated).
-	// If none of those, will just use the given directory.
-	if ( (p = my_subpath.find(BRO_SCRIPT_INSTALL_PATH)) != std::string::npos )
-		my_subpath.erase(0, strlen(BRO_SCRIPT_INSTALL_PATH));
-	else if ( (p = my_subpath.find(BRO_SCRIPT_SOURCE_PATH)) != std::string::npos )
-		my_subpath.erase(0, strlen(BRO_SCRIPT_SOURCE_PATH));
-	else if ( (p = my_subpath.find(BRO_BUILD_SOURCE_PATH)) != std::string::npos )
-		my_subpath.erase(0, strlen(BRO_BUILD_SOURCE_PATH));
-	else if ( (p = my_subpath.find(BRO_BUILD_SCRIPTS_PATH)) != std::string::npos )
-		my_subpath.erase(0, strlen(BRO_BUILD_SCRIPTS_PATH));
-
-	// if root path found, remove path separators until next path component
-	if ( p != std::string::npos )
-		while ( my_subpath.size() && my_subpath[0] == '/' )
-			my_subpath.erase(0, 1);
-
-	*subpath = normalize_path(my_subpath.c_str());
 	}
 
-extern string current_scanned_file_path;
+static string find_file_in_path(const string& filename, const string& path,
-
+                                const string& opt_ext = "")
-FILE* search_for_file(const char* filename, const char* ext,
-			const char** full_filename, bool load_pkgs,
-			const char** bropath_subpath)
 	{
-	// If the file is a literal absolute path we don't have to search,
+	if ( filename.empty() )
-	// just return the result of trying to open it.  If the file is
+		return string();
-	// might be a relative path, check first if it's a real file that
-	// can be referenced from cwd, else we'll try to search for it based
-	// on what path the currently-loading script is in as well as the
-	// standard BROPATH paths.
-	if ( filename[0] == '/' ||
-	    (filename[0] == '.' && access(filename, R_OK) == 0) )
-		{
-		if ( bropath_subpath )
-			{
-			char* tmp = copy_string(filename);
-			*bropath_subpath = copy_string(dirname(tmp));
-			delete [] tmp;
-			}
-		return open_file(filename, full_filename, load_pkgs);
-		}
 
-	char path[1024], full_filename_buf[1024];
+	// If file name is an absolute path, searching within *path* is pointless.
-
+	if ( filename[0] == '/' )
-	// Prepend the currently loading script's path to BROPATH so that
+		{
-	// @loads can be referenced relatively.
+		if ( can_read(filename) )
-	if ( current_scanned_file_path != "" && filename[0] == '.' )
+			return filename;
-		safe_snprintf(path, sizeof(path), "%s:%s",
-		              current_scanned_file_path.c_str(), bro_path());
 		else
-		safe_strncpy(path, bro_path(), sizeof(path));
+			return string();
-
-	char* dir_beginning = path;
-	char* dir_ending = path;
-	int more = *dir_beginning != '\0';
-
-	while ( more )
-		{
-		while ( *dir_ending && *dir_ending != ':' )
-			++dir_ending;
-
-		if ( *dir_ending == ':' )
-			*dir_ending = '\0';
-		else
-			more = 0;
-
-		safe_snprintf(full_filename_buf, sizeof(full_filename_buf),
-				"%s/%s.%s", dir_beginning, filename, ext);
-		if ( access(full_filename_buf, R_OK) == 0 &&
-		     ! is_dir(full_filename_buf) )
-			{
-			if ( bropath_subpath )
-				get_script_subpath(full_filename_buf, bropath_subpath);
-			return open_file(full_filename_buf, full_filename, load_pkgs);
 		}
 
-		safe_snprintf(full_filename_buf, sizeof(full_filename_buf),
+	string abs_path = path + '/' + filename;
-				"%s/%s", dir_beginning, filename);
+
-		if ( access(full_filename_buf, R_OK) == 0 )
+	if ( ! opt_ext.empty() )
 		{
-			if ( bropath_subpath )
+		string with_ext = abs_path + '.' + opt_ext;
-				get_script_subpath(full_filename_buf, bropath_subpath);
+
-			return open_file(full_filename_buf, full_filename, load_pkgs);
+		if ( can_read(with_ext) )
+			return with_ext;
 		}
 
-		dir_beginning = ++dir_ending;
+	if ( can_read(abs_path) )
+		return abs_path;
+
+	return string();
 	}
 
-	if ( full_filename )
+string find_file(const string& filename, const string& path_set,
-		*full_filename = copy_string(filename);
+                 const string& opt_ext)
-	if ( bropath_subpath )
 	{
-			char* tmp = copy_string(filename);
+	vector<string> paths;
-			*bropath_subpath = copy_string(dirname(tmp));
+	tokenize_string(path_set, ":", &paths);
-			delete [] tmp;
+
+	for ( size_t n = 0; n < paths.size(); ++n )
+		{
+		string f = find_file_in_path(filename, paths[n], opt_ext);
+
+		if ( ! f.empty() )
+			return f;
 		}
 
-	return 0;
+	return string();
 	}
 
 FILE* rotate_file(const char* name, RecordVal* rotate_info)

src/util.h

@@ -17,11 +17,13 @@
 #include <stdint.h>
 
 #include <string>
+#include <vector>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 #include <stdarg.h>
 #include <magic.h>
+#include <libgen.h>
 #include "config.h"
 
 #if __STDC__
@@ -104,6 +106,10 @@ std::string extract_ip_and_len(const std::string& i, int* len);
 std::string get_unescaped_string(const std::string& str);
 std::string get_escaped_string(const std::string& str, bool escape_all);
 
+std::vector<std::string>* tokenize_string(std::string input,
+					  const std::string& delim,
+					  std::vector<std::string>* rval = 0);
+
 extern char* copy_string(const char* s);
 extern int streq(const char* s1, const char* s2);
 
@@ -144,6 +150,7 @@ extern const char* fmt(const char* format, ...)
 	myattribute((format (printf, 1, 2)));
 extern const char* fmt_access_time(double time);
 
+extern bool ensure_intermediate_dirs(const char* dirname);
 extern bool ensure_dir(const char *dirname);
 
 // Returns true if path exists and is a directory.
@@ -202,14 +209,100 @@ static const SourceID SOURCE_LOCAL = 0;
 extern void pinpoint();
 extern int int_list_cmp(const void* v1, const void* v2);
 
+// Contains the name of the script file that gets read
+// when a package is loaded (i.e., "__load__.bro).
+extern const char* PACKAGE_LOADER;
+
 extern const char* bro_path();
 extern const char* bro_magic_path();
 extern std::string bro_prefixes();
-std::string dot_canon(std::string path, std::string file, std::string prefix = "");
+
-const char* normalize_path(const char* path);
+/**
-void get_script_subpath(const std::string& full_filename, const char** subpath);
+ * Wrapper class for functions like dirname(3) or basename(3) that won't
-extern FILE* search_for_file(const char* filename, const char* ext,
+ * modify the path argument and may optionally abort execution on error.
-	const char** full_filename, bool load_pkgs, const char** bropath_subpath);
+ */
+class SafePathOp {
+public:
+
+	typedef char*(*PathOpFn)(char*);
+
+	SafePathOp(PathOpFn fn, const char* path, bool error_aborts = true);
+	SafePathOp(PathOpFn fn, const std::string& path, bool error_aborts = true);
+
+	std::string result;
+	bool error;
+
+private:
+
+	void DoFunc(PathOpFn fn, const std::string& path, bool error_aborts = true);
+};
+
+class SafeDirname : public SafePathOp {
+public:
+
+	SafeDirname(const char* path, bool error_aborts = true)
+		: SafePathOp(&dirname, path, error_aborts) { }
+	SafeDirname(const std::string& path, bool error_aborts = true)
+		: SafePathOp(&dirname, path, error_aborts) { }
+};
+
+class SafeBasename : public SafePathOp {
+public:
+	SafeBasename(const char* path, bool error_aborts = true)
+		: SafePathOp(&basename, path, error_aborts) { }
+	SafeBasename(const std::string& path, bool error_aborts = true)
+		: SafePathOp(&basename, path, error_aborts) { }
+};
+
+std::string implode_string_vector(const std::vector<std::string>& v,
+                                  const std::string& delim = "\n");
+
+/**
+ * Flatten a script name by replacing '/' path separators with '.'.
+ * @param file A path to a Bro script.  If it is a __load__.bro, that part
+ *             is discarded when constructing the flattened the name.
+ * @param prefix A string to prepend to the flattened script name.
+ * @return The flattened script name.
+ */
+std::string flatten_script_name(const std::string& name,
+                                const std::string& prefix = "");
+
+/**
+ * Return a canonical/shortened path string by removing superfluous elements
+ * (path delimiters, dots referring to CWD or parent dir).
+ * @param path A filesystem path.
+ * @return A canonical/shortened version of \a path.
+ */
+std::string normalize_path(const std::string& path);
+
+/**
+ * Strip the BROPATH component from a path.
+ * @param path A file/directory path that may be within a BROPATH component.
+ * @return *path* minus the common BROPATH component (if any) removed.
+ */
+std::string without_bropath_component(const std::string& path);
+
+/**
+ * Locate a file within a given search path.
+ * @param filename Name of a file to find.
+ * @param path_set Colon-delimited set of paths to search for the file.
+ * @param opt_ext A filename extension/suffix to allow.
+ * @return Path to the found file, or an empty string if not found.
+ */
+std::string find_file(const std::string& filename, const std::string& path_set,
+                      const std::string& opt_ext = "");
+
+// Wrapper around fopen(3).  Emits an error when failing to open.
+FILE* open_file(const std::string& path, const std::string& mode = "r");
+
+/** Opens a Bro script package.
+ * @param path Location of a Bro script package (a directory).  Will be changed
+ *             to the path of the package's loader script.
+ * @param mode An fopen(3) mode.
+ * @return The return value of fopen(3) on the loader script or null if one
+ *         doesn't exist.
+ */
+FILE* open_package(std::string& path, const std::string& mode = "r");
 
 // Renames the given file to a new temporary name, and opens a new file with
 // the original name. Returns new file or NULL on error. Inits rotate_info if

testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log

@@ -3,7 +3,7 @@
 #empty_field	(empty)
 #unset_field	-
 #path	loaded_scripts
-#open	2013-10-24-21-30-36
+#open	2013-10-30-16-52-11
 #fields	name
 #types	string
 scripts/base/init-bare.bro
@@ -96,8 +96,9 @@ scripts/base/init-bare.bro
         scripts/base/utils/patterns.bro
   build/scripts/base/bif/__load__.bro
     build/scripts/base/bif/bloom-filter.bif.bro
+    build/scripts/base/bif/broxygen.bif.bro
     build/scripts/base/bif/cardinality-counter.bif.bro
     build/scripts/base/bif/top-k.bif.bro
 scripts/policy/misc/loaded-scripts.bro
   scripts/base/utils/paths.bro
-#close	2013-10-24-21-30-36
+#close	2013-10-30-16-52-11

testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log

@@ -3,7 +3,7 @@
 #empty_field	(empty)
 #unset_field	-
 #path	loaded_scripts
-#open	2013-10-24-21-30-37
+#open	2013-10-30-16-52-28
 #fields	name
 #types	string
 scripts/base/init-bare.bro
@@ -96,6 +96,7 @@ scripts/base/init-bare.bro
         scripts/base/utils/patterns.bro
   build/scripts/base/bif/__load__.bro
     build/scripts/base/bif/bloom-filter.bif.bro
+    build/scripts/base/bif/broxygen.bif.bro
     build/scripts/base/bif/cardinality-counter.bif.bro
     build/scripts/base/bif/top-k.bif.bro
 scripts/base/init-default.bro
@@ -220,4 +221,4 @@ scripts/base/init-default.bro
     scripts/base/files/unified2/main.bro
   scripts/base/misc/find-checksum-offloading.bro
 scripts/policy/misc/loaded-scripts.bro
-#close	2013-10-24-21-30-37
+#close	2013-10-30-16-52-28

testing/btest/Baseline/doc.autogen-reST-enums/autogen-reST-enums.rst

@@ -1,116 +0,0 @@
-.. Automatically generated.  Do not edit.
-
-:tocdepth: 3
-
-autogen-reST-enums.bro
-======================
-
-
-
-
-:Source File: :download:`autogen-reST-enums.bro`
-
-Summary
-~~~~~~~
-Options
-#######
-==================================================================== ======================================================================
-:bro:id:`test_enum_option`: :bro:type:`TestEnum1` :bro:attr:`&redef` this should reference the TestEnum1 type and not a generic "enum" type
-==================================================================== ======================================================================
-
-Types
-#####
-======================================= ========================================
-:bro:type:`TestEnum1`: :bro:type:`enum` There's tons of ways an enum can look...
-
-:bro:type:`TestEnum2`: :bro:type:`enum` The final comma is optional
-======================================= ========================================
-
-Redefinitions
-#############
-======================================= =======================
-:bro:type:`TestEnum1`: :bro:type:`enum` redefs should also work
-
-:bro:type:`TestEnum1`: :bro:type:`enum` now with a comma
-======================================= =======================
-
-Detailed Interface
-~~~~~~~~~~~~~~~~~~
-Options
-#######
-.. bro:id:: test_enum_option
-
-   :Type: :bro:type:`TestEnum1`
-   :Attributes: :bro:attr:`&redef`
-   :Default: ``ONE``
-
-   this should reference the TestEnum1 type and not a generic "enum" type
-
-Types
-#####
-.. bro:type:: TestEnum1
-
-   :Type: :bro:type:`enum`
-
-      .. bro:enum:: ONE TestEnum1
-
-         like this
-
-      .. bro:enum:: TWO TestEnum1
-
-         or like this
-
-      .. bro:enum:: THREE TestEnum1
-
-         multiple
-         comments
-         and even
-         more comments
-
-   There's tons of ways an enum can look...
-
-.. bro:type:: TestEnum2
-
-   :Type: :bro:type:`enum`
-
-      .. bro:enum:: A TestEnum2
-
-         like this
-
-      .. bro:enum:: B TestEnum2
-
-         or like this
-
-      .. bro:enum:: C TestEnum2
-
-         multiple
-         comments
-         and even
-         more comments
-
-   The final comma is optional
-
-Redefinitions
-#############
-:bro:type:`TestEnum1`
-
-   :Type: :bro:type:`enum`
-
-      .. bro:enum:: FOUR TestEnum1
-
-         adding another
-         value
-
-   redefs should also work
-
-:bro:type:`TestEnum1`
-
-   :Type: :bro:type:`enum`
-
-      .. bro:enum:: FIVE TestEnum1
-
-         adding another
-         value
-
-   now with a comma
-

testing/btest/Baseline/doc.autogen-reST-example/example.rst

@@ -1,301 +0,0 @@
-.. Automatically generated.  Do not edit.
-
-:tocdepth: 3
-
-example.bro
-===========
-.. bro:namespace:: Example
-
-This is an example script that demonstrates documentation features.
-Comments of the form ``##!`` are for the script summary.  The contents of
-these comments are transferred directly into the auto-generated
-`reStructuredText <http://docutils.sourceforge.net/rst.html>`_
-(reST) document's summary section.
-
-.. tip:: You can embed directives and roles within ``##``-stylized comments.
-
-There's also a custom role to reference any identifier node in
-the Bro Sphinx domain that's good for "see alsos", e.g.
-
-See also: :bro:see:`Example::a_var`, :bro:see:`Example::ONE`,
-:bro:see:`SSH::Info`
-
-And a custom directive does the equivalent references:
-
-.. bro:see:: Example::a_var Example::ONE SSH::Info
-
-:Namespace: ``Example``
-:Imports: :doc:`policy/frameworks/software/vulnerable </scripts/policy/frameworks/software/vulnerable>`
-:Source File: :download:`example.bro`
-
-Summary
-~~~~~~~
-Options
-#######
-============================================================================ ======================================
-:bro:id:`Example::an_option`: :bro:type:`set` :bro:attr:`&redef`             add documentation for "an_option" here
-
-:bro:id:`Example::option_with_init`: :bro:type:`interval` :bro:attr:`&redef` More docs can be added here.
-============================================================================ ======================================
-
-State Variables
-###############
-=========================================================================== ==================================================
-:bro:id:`Example::a_var`: :bro:type:`bool`                                  put some documentation for "a_var" here
-
-:bro:id:`Example::var_with_attr`: :bro:type:`count` :bro:attr:`&persistent`
-
-:bro:id:`Example::var_without_explicit_type`: :bro:type:`string`
-
-:bro:id:`Example::dummy`: :bro:type:`string`                                The first.sentence for the summary text ends here.
-=========================================================================== ==================================================
-
-Types
-#####
-====================================================== ==========================================================
-:bro:type:`Example::SimpleEnum`: :bro:type:`enum`      documentation for "SimpleEnum"
-                                                       goes here.
-
-:bro:type:`Example::SimpleRecord`: :bro:type:`record`  general documentation for a type "SimpleRecord"
-                                                       goes here.
-
-:bro:type:`Example::ComplexRecord`: :bro:type:`record` general documentation for a type "ComplexRecord" goes here
-
-:bro:type:`Example::Info`: :bro:type:`record`          An example record to be used with a logging stream.
-====================================================== ==========================================================
-
-Events
-######
-================================================= =============================================================
-:bro:id:`Example::an_event`: :bro:type:`event`    Summarize "an_event" here.
-
-:bro:id:`Example::log_example`: :bro:type:`event` This is a declaration of an example event that can be used in
-                                                  logging streams and is raised once for each log entry.
-================================================= =============================================================
-
-Functions
-#########
-=================================================== =======================================
-:bro:id:`Example::a_function`: :bro:type:`function` Summarize purpose of "a_function" here.
-=================================================== =======================================
-
-Redefinitions
-#############
-===================================================== ========================================
-:bro:type:`Log::ID`: :bro:type:`enum`
-
-:bro:type:`Example::SimpleEnum`: :bro:type:`enum`     document the "SimpleEnum" redef here
-
-:bro:type:`Example::SimpleRecord`: :bro:type:`record` document the record extension redef here
-===================================================== ========================================
-
-Notices
-#######
-:bro:type:`Notice::Type`
-
-   :Type: :bro:type:`enum`
-
-      .. bro:enum:: Example::Notice_One Notice::Type
-
-         any number of this type of comment
-         will document "Notice_One"
-
-      .. bro:enum:: Example::Notice_Two Notice::Type
-
-         any number of this type of comment
-         will document "Notice_Two"
-
-      .. bro:enum:: Example::Notice_Three Notice::Type
-
-      .. bro:enum:: Example::Notice_Four Notice::Type
-
-Configuration Changes
-#####################
-Packet Filter
-^^^^^^^^^^^^^
-Loading this script makes the following changes to :bro:see:`capture_filters`.
-
-Filters added::
-
-    [ssl] = tcp port 443,
-    [nntps] = tcp port 562
-
-Detailed Interface
-~~~~~~~~~~~~~~~~~~
-Options
-#######
-.. bro:id:: Example::an_option
-
-   :Type: :bro:type:`set` [:bro:type:`addr`, :bro:type:`addr`, :bro:type:`string`]
-   :Attributes: :bro:attr:`&redef`
-   :Default: ``{}``
-
-   add documentation for "an_option" here
-
-.. bro:id:: Example::option_with_init
-
-   :Type: :bro:type:`interval`
-   :Attributes: :bro:attr:`&redef`
-   :Default: ``10.0 msecs``
-
-   More docs can be added here.
-
-State Variables
-###############
-.. bro:id:: Example::a_var
-
-   :Type: :bro:type:`bool`
-
-   put some documentation for "a_var" here
-
-.. bro:id:: Example::var_with_attr
-
-   :Type: :bro:type:`count`
-   :Attributes: :bro:attr:`&persistent`
-
-.. bro:id:: Example::var_without_explicit_type
-
-   :Type: :bro:type:`string`
-   :Default: ``"this works"``
-
-.. bro:id:: Example::dummy
-
-   :Type: :bro:type:`string`
-
-   The first.sentence for the summary text ends here.  And this second
-   sentence doesn't show in the short description.
-
-Types
-#####
-.. bro:type:: Example::SimpleEnum
-
-   :Type: :bro:type:`enum`
-
-      .. bro:enum:: Example::ONE Example::SimpleEnum
-
-         and more specific info for "ONE"
-         can span multiple lines
-
-      .. bro:enum:: Example::TWO Example::SimpleEnum
-
-         or more info like this for "TWO"
-         can span multiple lines
-
-      .. bro:enum:: Example::THREE Example::SimpleEnum
-
-   documentation for "SimpleEnum"
-   goes here.
-
-.. bro:type:: Example::SimpleRecord
-
-   :Type: :bro:type:`record`
-
-      field1: :bro:type:`count`
-         counts something
-
-      field2: :bro:type:`bool`
-         toggles something
-
-   general documentation for a type "SimpleRecord"
-   goes here.
-
-.. bro:type:: Example::ComplexRecord
-
-   :Type: :bro:type:`record`
-
-      field1: :bro:type:`count`
-         counts something
-
-      field2: :bro:type:`bool`
-         toggles something
-
-      field3: :bro:type:`Example::SimpleRecord`
-
-      msg: :bro:type:`string` :bro:attr:`&default` = ``"blah"`` :bro:attr:`&optional`
-         attributes are self-documenting
-
-   general documentation for a type "ComplexRecord" goes here
-
-.. bro:type:: Example::Info
-
-   :Type: :bro:type:`record`
-
-      ts: :bro:type:`time` :bro:attr:`&log`
-
-      uid: :bro:type:`string` :bro:attr:`&log`
-
-      status: :bro:type:`count` :bro:attr:`&log` :bro:attr:`&optional`
-
-   An example record to be used with a logging stream.
-
-Events
-######
-.. bro:id:: Example::an_event
-
-   :Type: :bro:type:`event` (name: :bro:type:`string`)
-
-   Summarize "an_event" here.
-   Give more details about "an_event" here.
-   Example::an_event should not be confused as a parameter.
-   
-   :param name: describe the argument here
-
-.. bro:id:: Example::log_example
-
-   :Type: :bro:type:`event` (rec: :bro:type:`Example::Info`)
-
-   This is a declaration of an example event that can be used in
-   logging streams and is raised once for each log entry.
-
-Functions
-#########
-.. bro:id:: Example::a_function
-
-   :Type: :bro:type:`function` (tag: :bro:type:`string`, msg: :bro:type:`string`) : :bro:type:`string`
-
-   Summarize purpose of "a_function" here.
-   Give more details about "a_function" here.
-   Separating the documentation of the params/return values with
-   empty comments is optional, but improves readability of script.
-   
-   
-   :param tag: function arguments can be described
-        like this
-   
-   :param msg: another param
-   
-   
-   :returns: describe the return type here
-
-Redefinitions
-#############
-:bro:type:`Log::ID`
-
-   :Type: :bro:type:`enum`
-
-      .. bro:enum:: Example::LOG Log::ID
-
-:bro:type:`Example::SimpleEnum`
-
-   :Type: :bro:type:`enum`
-
-      .. bro:enum:: Example::FOUR Example::SimpleEnum
-
-         and some documentation for "FOUR"
-
-      .. bro:enum:: Example::FIVE Example::SimpleEnum
-
-         also "FIVE" for good measure
-
-   document the "SimpleEnum" redef here
-
-:bro:type:`Example::SimpleRecord`
-
-   :Type: :bro:type:`record`
-
-      field_ext: :bro:type:`string` :bro:attr:`&optional`
-         document the extending field here
-         (or here)
-
-   document the record extension redef here
-

testing/btest/Baseline/doc.autogen-reST-func-params/autogen-reST-func-params.rst

@@ -1,58 +0,0 @@
-.. Automatically generated.  Do not edit.
-
-:tocdepth: 3
-
-autogen-reST-func-params.bro
-============================
-
-
-
-
-:Source File: :download:`autogen-reST-func-params.bro`
-
-Summary
-~~~~~~~
-Types
-#####
-======================================== =
-:bro:type:`test_rec`: :bro:type:`record`
-======================================== =
-
-Functions
-#########
-========================================= ======================================
-:bro:id:`test_func`: :bro:type:`function` This is a global function declaration.
-========================================= ======================================
-
-Detailed Interface
-~~~~~~~~~~~~~~~~~~
-Types
-#####
-.. bro:type:: test_rec
-
-   :Type: :bro:type:`record`
-
-      field_func: :bro:type:`function` (i: :bro:type:`int`, j: :bro:type:`int`) : :bro:type:`string`
-         This is a record field function.
-         
-         :param i: First param.
-         :param j: Second param.
-         
-         :returns: A string.
-
-Functions
-#########
-.. bro:id:: test_func
-
-   :Type: :bro:type:`function` (i: :bro:type:`int`, j: :bro:type:`int`) : :bro:type:`string`
-
-   This is a global function declaration.
-   
-   
-   :param i: First param.
-   
-   :param j: Second param.
-   
-   
-   :returns: A string.
-