diff --git a/Makefile b/Makefile index 4984f6db31..f4b2104c73 100644 --- a/Makefile +++ b/Makefile @@ -29,18 +29,6 @@ doc: configured docclean: configured $(MAKE) -C $(BUILD) $@ -restdoc: configured - $(MAKE) -C $(BUILD) $@ - -restclean: configured - $(MAKE) -C $(BUILD) $@ - -broxygen: configured - $(MAKE) -C $(BUILD) $@ - -broxygenclean: configured - $(MAKE) -C $(BUILD) $@ - dist: @rm -rf $(VERSION_FULL) $(VERSION_FULL).tgz @rm -rf $(VERSION_MIN) $(VERSION_MIN).tgz diff --git a/doc/CMakeLists.txt b/doc/CMakeLists.txt index 373b4643ba..9498556edc 100644 --- a/doc/CMakeLists.txt +++ b/doc/CMakeLists.txt @@ -1,75 +1,86 @@ -set(BIF_SRC_DIR ${PROJECT_SOURCE_DIR}/src) -set(RST_OUTPUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/rest_output) -set(DOC_OUTPUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/out) -set(DOC_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}) -set(DOC_SOURCE_WORKDIR ${CMAKE_CURRENT_BINARY_DIR}/sphinx-sources) +set(BROCCOLI_DOCS_SRC ${CMAKE_BINARY_DIR}/aux/broccoli/doc/html) +set(BROCCOLI_DOCS_DST ${CMAKE_BINARY_DIR}/html/broccoli-api) +set(SPHINX_INPUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/sphinx_input) +set(SPHINX_OUTPUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/sphinx_output) +set(BROXYGEN_SCRIPT_OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/broxygen_script_output) +set(BROXYGEN_CACHE_DIR ${CMAKE_CURRENT_BINARY_DIR}/broxygen_cache) -set(MASTER_POLICY_INDEX ${CMAKE_CURRENT_BINARY_DIR}/scripts/policy_index) -set(MASTER_PACKAGE_INDEX ${CMAKE_CURRENT_BINARY_DIR}/scripts/pkg_index) +# Find out what BROPATH to use when executing bro. +execute_process(COMMAND ${CMAKE_BINARY_DIR}/bro-path-dev + OUTPUT_VARIABLE BROPATH + RESULT_VARIABLE retval + OUTPUT_STRIP_TRAILING_WHITESPACE) +if (NOT ${retval} EQUAL 0) + message(FATAL_ERROR "Problem setting BROPATH") +endif () -file(GLOB_RECURSE DOC_SOURCES FOLLOW_SYMLINKS "*") +set(BROMAGIC ${BRO_MAGIC_SOURCE_PATH}) -# configure the Sphinx config file (expand variables CMake might know about) +# Configure the Sphinx config file (expand variables CMake might know about). configure_file(${CMAKE_CURRENT_SOURCE_DIR}/conf.py.in ${CMAKE_CURRENT_BINARY_DIR}/conf.py @ONLY) -add_subdirectory(scripts) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/broxygen.conf.in + ${CMAKE_CURRENT_BINARY_DIR}/broxygen.conf + @ONLY) -# The "broxygen" target generates reST documentation for any outdated bro -# scripts and then uses Sphinx to generate HTML documentation from the reST -add_custom_target(broxygen - # copy the template documentation to the build directory - # to give as input for sphinx - COMMAND rsync -r --copy-links --times - ${DOC_SOURCE_DIR}/ - ${DOC_SOURCE_WORKDIR} - # copy generated policy script documentation into the - # working copy of the template documentation - COMMAND rsync -r --copy-links --times - ${RST_OUTPUT_DIR}/ - ${DOC_SOURCE_WORKDIR}/scripts - # append to the master index of all policy scripts - COMMAND cat ${MASTER_POLICY_INDEX} >> - ${DOC_SOURCE_WORKDIR}/scripts/scripts.rst - # append to the master index of all policy packages - COMMAND cat ${MASTER_PACKAGE_INDEX} >> - ${DOC_SOURCE_WORKDIR}/scripts/packages.rst - # construct a reST file for each group - COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/bin/group_index_generator.py - ${CMAKE_CURRENT_BINARY_DIR}/scripts/group_list - ${CMAKE_CURRENT_BINARY_DIR}/scripts - ${DOC_SOURCE_WORKDIR}/scripts - # tell sphinx to generate html - COMMAND sphinx-build - -b html - -c ${CMAKE_CURRENT_BINARY_DIR} - -d ${DOC_OUTPUT_DIR}/doctrees - ${DOC_SOURCE_WORKDIR} - ${DOC_OUTPUT_DIR}/html - # create symlink to the html output directory for convenience - COMMAND "${CMAKE_COMMAND}" -E create_symlink - ${DOC_OUTPUT_DIR}/html - ${CMAKE_BINARY_DIR}/html - # copy Broccoli API reference into output dir if it exists - COMMAND test -d ${CMAKE_BINARY_DIR}/aux/broccoli/doc/html && ( rm -rf ${CMAKE_BINARY_DIR}/html/broccoli-api && cp -r ${CMAKE_BINARY_DIR}/aux/broccoli/doc/html ${CMAKE_BINARY_DIR}/html/broccoli-api ) || true - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} - COMMENT "[Sphinx] Generating HTML policy script docs" - # SOURCES just adds stuff to IDE projects as a convenience - SOURCES ${DOC_SOURCES}) +add_custom_target(sphinxdoc + # Copy the template documentation to build directory to use as input tree + # for Sphinx. This is needed because some parts are dynamically generated + # in to that tree by Bro/Broxygen. + COMMAND rsync -q -r --copy-links --times --delete + --filter='protect scripts/*' + ${CMAKE_CURRENT_SOURCE_DIR}/ ${SPHINX_INPUT_DIR} + # Use Bro/Broxygen to dynamically generate reST for all Bro scripts. + COMMAND BROPATH=${BROPATH} + BROMAGIC=${BROMAGIC} + ${CMAKE_BINARY_DIR}/src/bro + -X ${CMAKE_CURRENT_BINARY_DIR}/broxygen.conf + broxygen >/dev/null + # Rsync over the generated reST to the Sphinx source tree in the build dir. + COMMAND rsync -q -r --copy-links --times --delete --filter='protect *.bro' + ${BROXYGEN_SCRIPT_OUTPUT}/ ${SPHINX_INPUT_DIR}/scripts + # Rsync over Bro scripts to the Sphinx source tree in the build dir. + # These are used by :download: references in the generated script docs. + COMMAND rsync -q -r --copy-links --times --delete + --filter='protect /base/bif/*' --filter='protect *.rst' + --filter='include */' --filter='include *.bro' --filter='exclude *' + ${CMAKE_SOURCE_DIR}/scripts/ ${SPHINX_INPUT_DIR}/scripts + # Rsync over Bro scripts created by BIF compiler to the Sphinx source tree. + COMMAND rsync -q -r --copy-links --times --delete + --filter='protect *.rst' --filter='include */' + --filter='include *.bro' --filter='exclude *' + ${CMAKE_BINARY_DIR}/scripts/base/bif/ + ${SPHINX_INPUT_DIR}/scripts/base/bif + # Use Sphinx to build HTML. + COMMAND sphinx-build + -b html + -c ${CMAKE_CURRENT_BINARY_DIR} + -d ${SPHINX_OUTPUT_DIR}/doctrees + ${SPHINX_INPUT_DIR} + ${SPHINX_OUTPUT_DIR}/html + # Create symlink to the html output directory for convenience. + COMMAND "${CMAKE_COMMAND}" -E create_symlink + ${SPHINX_OUTPUT_DIR}/html + ${CMAKE_BINARY_DIR}/html + # Copy Broccoli API reference into output dir if it exists. + COMMAND test -d ${BROCCOLI_DOCS_SRC} && + ( rm -rf ${BROCCOLI_DOCS_DST} && + cp -r ${BROCCOLI_DOCS_SRC} ${BROCCOLI_DOCS_DST} ) || true + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + COMMENT "[Sphinx] Generate HTML documentation in ${CMAKE_BINARY_DIR}/html") -# The "broxygenclean" target removes just the Sphinx input/output directories -# from the build directory. -add_custom_target(broxygenclean - COMMAND "${CMAKE_COMMAND}" -E remove_directory - ${DOC_SOURCE_WORKDIR} - COMMAND "${CMAKE_COMMAND}" -E remove_directory - ${DOC_OUTPUT_DIR} - VERBATIM) +add_dependencies(sphinxdoc bro) -add_dependencies(broxygen restdoc) +add_custom_target(sphinxdoc_clean + COMMAND "${CMAKE_COMMAND}" -E remove_directory ${SPHINX_INPUT_DIR} + COMMAND "${CMAKE_COMMAND}" -E remove_directory ${SPHINX_OUTPUT_DIR} + COMMAND "${CMAKE_COMMAND}" -E remove_directory ${BROXYGEN_SCRIPT_OUTPUT} + COMMAND "${CMAKE_COMMAND}" -E remove_directory ${BROXYGEN_CACHE_DIR} + VERBATIM) add_custom_target(doc) add_custom_target(docclean) -add_dependencies(doc broxygen) -add_dependencies(docclean broxygenclean restclean) +add_dependencies(doc sphinxdoc) +add_dependencies(docclean sphinxdoc_clean) diff --git a/doc/README b/doc/README index 0ba0a8587f..c14a3bfbfb 100644 --- a/doc/README +++ b/doc/README @@ -15,17 +15,16 @@ which adds some reST directives and roles that aid in generating useful index entries and cross-references. Other extensions can be added in a similar fashion. -Either the ``make doc`` or ``make broxygen`` targets in the top-level -Makefile can be used to locally render the reST files into HTML. -Those targets depend on: +The ``make doc`` target in the top-level Makefile can be used to locally +render the reST files into HTML. That target depend on: * Python interpreter >= 2.5 * `Sphinx `_ >= 1.0.1 After completion, HTML documentation is symlinked in ``build/html``. -There's also ``make docclean`` and ``make broxygenclean`` targets to -clean the resulting documentation. +There's also a ``make docclean`` target which deletes any files +created during the documentation build process. Notes for Writing Documentation ------------------------------- diff --git a/doc/bin/group_index_generator.py b/doc/bin/group_index_generator.py deleted file mode 100755 index b41dcb379c..0000000000 --- a/doc/bin/group_index_generator.py +++ /dev/null @@ -1,72 +0,0 @@ -#! /usr/bin/env python - -# This script automatically generates a reST documents that lists -# a collection of Bro scripts that are "grouped" together. -# The summary text (##! comments) of the script is embedded in the list -# -# 1st argument is the file containing list of groups -# 2nd argument is the directory containing ${group}_files lists of -# scripts that belong to the group and ${group}_doc_names lists of -# document names that can be supplied to a reST :doc: role -# 3rd argument is a directory in which write a ${group}.rst file that contains -# reST style references to script docs along with summary text contained -# in original script. If ${group} ends with "index", then the file -# is always clobbered by this script, but for other unique group names, -# this script will append to existing files. - -import sys -import os -import string - -group_list = sys.argv[1] -file_manifest_dir = sys.argv[2] -output_dir = sys.argv[3] - -def make_group_file_index(dir_name, group_name): - group_file = os.path.join(dir_name, group_name + ".rst") - - if not os.path.exists(group_file): - if not os.path.exists(os.path.dirname(group_file)): - os.makedirs(os.path.dirname(group_file)) - - if group_name.endswith("index"): - with open(group_file, 'w') as f_group_file: - f_group_file.write(":orphan:\n\n") - title = "Package Index: %s\n" % os.path.dirname(group_name) - f_group_file.write(title); - for n in range(len(title)): - f_group_file.write("=") - f_group_file.write("\n"); - - return group_file - -with open(group_list, 'r') as f_group_list: - for group in f_group_list.read().splitlines(): - #print group - group_file = make_group_file_index(output_dir, group) - file_manifest = os.path.join(file_manifest_dir, group + "_files") - doc_manifest = os.path.join(file_manifest_dir, group + "_doc_names") - src_files = [] - doc_names = [] - - with open(file_manifest, 'r') as f_file_manifest: - src_files = f_file_manifest.read().splitlines() - - with open(doc_manifest, 'r') as f_doc_manifest: - doc_names = f_doc_manifest.read().splitlines() - - for i in range(len(src_files)): - src_file = src_files[i] - #print "\t" + src_file - summary_comments = [] - with open(src_file, 'r') as f_src_file: - for line in f_src_file: - sum_pos = string.find(line, "##!") - if sum_pos != -1: - summary_comments.append(line[(sum_pos+3):]) - #print summary_comments - - with open(group_file, 'a') as f_group_file: - f_group_file.write("\n:doc:`/scripts/%s`\n" % doc_names[i]) - for line in summary_comments: - f_group_file.write(" " + line) diff --git a/doc/broxygen.conf.in b/doc/broxygen.conf.in new file mode 100644 index 0000000000..b27ba6248d --- /dev/null +++ b/doc/broxygen.conf.in @@ -0,0 +1 @@ +script @BROXYGEN_SCRIPT_OUTPUT@/ * diff --git a/doc/conf.py.in b/doc/conf.py.in index 4cf64f10db..f54ca9d8c6 100644 --- a/doc/conf.py.in +++ b/doc/conf.py.in @@ -17,7 +17,7 @@ extensions = [] # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. -sys.path.insert(0, os.path.abspath('sphinx-sources/ext')) +sys.path.insert(0, os.path.abspath('sphinx_input/ext')) # ----- Begin of BTest configuration. ----- btest = os.path.abspath("@CMAKE_SOURCE_DIR@/aux/btest") @@ -33,6 +33,12 @@ btest_base="@CMAKE_SOURCE_DIR@/testing/btest" btest_tests="doc/sphinx" # ----- End of BTest configuration. ----- +# ----- Begin of Broxygen configuration. ----- +extensions += ["broxygen"] +broxygen_cache="@BROXYGEN_CACHE_DIR@" +os.environ["BROPATH"] = "@BROPATH@" +os.environ["BROMAGIC"] = "@BROMAGIC@" +# ----- End of Broxygen configuration. ----- # -- General configuration ----------------------------------------------------- @@ -47,7 +53,7 @@ os.environ["BRO_SRC_ROOT"] = "@CMAKE_SOURCE_DIR@" os.environ["DOC_ROOT"] = "@CMAKE_SOURCE_DIR@/doc" # Add any paths that contain templates here, relative to this directory. -templates_path = ['sphinx-sources/_templates', 'sphinx-sources/_static'] +templates_path = ['sphinx_input/_templates', 'sphinx_input/_static'] # The suffix of source filenames. source_suffix = '.rst' @@ -141,7 +147,7 @@ html_theme_options = { } # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['sphinx-sources/_static'] +html_static_path = ['sphinx_input/_static'] # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. diff --git a/doc/ext/bro.py b/doc/ext/bro.py index 6ef11c37f6..91f42b4e5a 100644 --- a/doc/ext/bro.py +++ b/doc/ext/bro.py @@ -191,6 +191,10 @@ class BroNotices(Index): def generate(self, docnames=None): content = {} + + if 'notices' not in self.domain.env.domaindata['bro']: + return content, False + for n in self.domain.env.domaindata['bro']['notices']: modname = n[0].split("::")[0] entries = content.setdefault(modname, []) diff --git a/doc/ext/broxygen.py b/doc/ext/broxygen.py new file mode 100644 index 0000000000..d7a4e57c21 --- /dev/null +++ b/doc/ext/broxygen.py @@ -0,0 +1,311 @@ +""" +Broxygen domain for Sphinx. + +Adds directives that allow Sphinx to invoke Bro in order to generate script +reference documentation on the fly. The directives are: + +broxygen:package + - Shows links to all scripts contained within matching package(s). +broxygen:package_index + - An index with links to matching package document(s). +broxygen:script + - Reference for matching script(s) (i.e. everything declared by the script). +broxygen:script_summary + - Shows link to matching script(s) with it's summary-section comments. +broxygen:script_index + - An index with links to all matching scrips. +broxygen:proto_analyzer + - All protocol analyzers and their components (events/bifs, etc.) +broxygen:file_analyzer + - All file analyzers and their components (events/bifs, etc.) +""" + + +from sphinx.domains import Domain, ObjType +from sphinx.locale import l_ +from docutils.parsers.rst.directives.misc import Include + + +App = None + + +def info(msg): + """Use Sphinx builder to output a console message.""" + global App + from sphinx.util.console import blue + App.builder.info(blue(msg)) + + +def pattern_to_filename_component(pattern): + """Replace certain characters in Broxygen config file target pattern. + + Such that it can be used as part of a (sane) filename. + + """ + return pattern.replace("/", ".").replace("*", "star") + + +def ensure_dir(path): + """Should act like ``mkdir -p``.""" + import os + import errno + + try: + os.makedirs(path) + except OSError as e: + if e.errno != errno.EEXIST: + raise + + +def generate_config(env, type, pattern): + """Create a Broxygen config file for a particular target. + + It can be used by Bro to generate reST docs for that target. + + """ + import os + import tempfile + from sphinx.errors import SphinxError + + work_dir = env.config.broxygen_cache + + if not work_dir: + raise SphinxError("broxygen_cache not set in sphinx config file") + + ensure_dir(work_dir) + prefix = "{0}-{1}-".format(type, pattern_to_filename_component(pattern)) + (fd, cfg) = tempfile.mkstemp(suffix=".cfg", prefix=prefix, dir=work_dir) + generated_file = "{0}.rst".format(cfg) + config = "{0}\t{1}\t{2}".format(type, generated_file, pattern) + f = os.fdopen(fd, "w") + f.write(config) + f.close() + return (cfg, generated_file) + + +def generate_target(env, type, pattern): + """Create a Broxygen target and build it. + + For a target which hasn't been referenced by any other script, this function + creates an associated config file then uses Bro w/ it to build the target + and stores the target information in the build environment. + + If a script references a target that's already found in the build + environment the results of the previous built are re-used. + + """ + app_data = env.domaindata["broxygen"] + + if (type, pattern) in app_data["targets"]: + info("Broxygen has cached doc for target '{0} {1}'".format( + type, pattern)) + return app_data["targets"] + + (cfg, gend_file) = generate_config(env, type, pattern) + target = BroxygenTarget(type, pattern, cfg, gend_file) + app_data["targets"][(type, pattern)] = target + build_target(env, target) + info("Broxygen built target '{0} {1}'".format(type, pattern)) + return target + + +def build_target(env, target): + """Invoke a Bro process to build a Broxygen target.""" + import os + import subprocess + + bro_cmd = "bro -X {0} broxygen".format(target.config_file) + cwd = os.getcwd() + os.chdir(os.path.dirname(target.config_file)) + + try: + subprocess.check_output(bro_cmd, stderr=subprocess.STDOUT, shell=True) + except subprocess.CalledProcessError as e: + from sphinx.errors import SphinxError + raise SphinxError( + "Command '{0}' returned non-zero exit status {1}: {2}".format( + e.cmd, e.returncode, e.output)) + finally: + os.chdir(cwd) + + +class BroxygenTarget(object): + + """Some portion of reST documentation that Bro knows how to generate. + + A target is identified by its type and pattern. E.g. type "script" and + pattern "broxygen/example.bro". + + """ + + def __init__(self, type, pattern, config_file, generated_file): + self.type = type + self.pattern = pattern + self.config_file = config_file + self.generated_file = generated_file + self.used_in_docs = set() + + +class BroxygenDirective(Include): + + """Base class for Broxygen directives. + + It can use Bro to generate reST documentation on the fly and embed it in + the document at the location of the directive just like the ``.. include::`` + directive. The only argument is a pattern to identify to Bro which + pieces of documentation it needs to create. + """ + + required_arguments = 1 + has_content = False + + target_type = None + + def run(self): + env = self.state.document.settings.env + info("Broxygen running .. {0}:: {1} in {2}".format( + self.name, self.arguments[0], env.docname)) + target = generate_target(env, self.target_type, self.arguments[0]) + target.used_in_docs.add(env.docname) + self.arguments = [target.generated_file] + return super(BroxygenDirective, self).run() + + +class PackageDirective(BroxygenDirective): + + target_type = "package" + + +class PackageIndexDirective(BroxygenDirective): + + target_type = "package_index" + + +class ScriptDirective(BroxygenDirective): + + target_type = "script" + + +class ScriptSummaryDirective(BroxygenDirective): + + target_type = "script_summary" + + +class ScriptIndexDirective(BroxygenDirective): + + target_type = "script_index" + + +class ProtoAnalyzerDirective(BroxygenDirective): + + target_type = "proto_analyzer" + + +class FileAnalyzerDirective(BroxygenDirective): + + target_type = "file_analyzer" + + +class IdentifierDirective(BroxygenDirective): + + target_type = "identifier" + + +class BroxygenDomain(Domain): + + name = "broxygen" + label = "Broxygen" + + object_types = { + "package": ObjType(l_("package")), + "package_index": ObjType(l_("package_index")), + "script": ObjType(l_("script")), + "script_summary": ObjType(l_("script_summary")), + "script_index": ObjType(l_("script_index")), + "proto_analyzer": ObjType(l_("proto_analyzer")), + "file_analyzer": ObjType(l_("file_analyzer")), + "identifier": ObjType(l_("identifier")), + } + + directives = { + "package": PackageDirective, + "package_index": PackageIndexDirective, + "script": ScriptDirective, + "script_summary": ScriptSummaryDirective, + "script_index": ScriptIndexDirective, + "proto_analyzer": ProtoAnalyzerDirective, + "file_analyzer": FileAnalyzerDirective, + "identifier": IdentifierDirective, + } + + roles = {} + + initial_data = { + "targets": {} + } + + def clear_doc(self, docname): + """Update Broxygen targets referenced in docname. + + If it's the last place the target was referenced, remove it from + the build environment and delete any generated config/reST files + associated with it from the cache. + + """ + import os + + stale_targets = [] + + for (type, pattern), target in self.data["targets"].items(): + if docname in target.used_in_docs: + target.used_in_docs.remove(docname) + + if not target.used_in_docs: + stale_targets.append(target) + + for target in stale_targets: + del self.data["targets"][(target.type, target.pattern)] + os.remove(target.config_file) + os.remove(target.generated_file) + + def get_objects(self): + """No Broxygen-generated content is itself linkable/searchable.""" + return [] + + +def env_get_outdated_hook(app, env, added, changed, removed): + """Check whether to re-read any documents referencing Broxygen targets. + + To do that we have to ask Bro to rebuild each target and compare the + before and after modification times of the generated reST output file. + If Bro changed it, then the document containing the Broxygen directive + needs to be re-read. + + """ + import os + + reread = set() + + for target in app.env.domaindata["broxygen"]["targets"].values(): + before_mtime = os.stat(target.generated_file) + build_target(env, target) + after_mtime = os.stat(target.generated_file) + + if after_mtime > before_mtime: + info("Broxygen target '{0} {1}' outdated".format( + target.type, target.pattern)) + + for docname in target.used_in_docs: + if docname not in removed: + info(" in document: {0}".format(docname)) + reread.add(docname) + + return list(reread) + + +def setup(app): + global App + App = app + app.add_domain(BroxygenDomain) + app.add_config_value("broxygen_cache", None, "env") + app.connect("env-get-outdated", env_get_outdated_hook) diff --git a/doc/frameworks/logging-dataseries.rst b/doc/frameworks/logging-dataseries.rst index 667edcbe14..cc479eae76 100644 --- a/doc/frameworks/logging-dataseries.rst +++ b/doc/frameworks/logging-dataseries.rst @@ -104,7 +104,7 @@ code like this to your ``local.bro``: } Bro's DataSeries writer comes with a few tuning options, see -:doc:`/scripts/base/frameworks/logging/writers/dataseries`. +:doc:`/scripts/base/frameworks/logging/writers/dataseries.bro`. Working with DataSeries ======================= diff --git a/doc/frameworks/logging.rst b/doc/frameworks/logging.rst index eb64df4ec3..47d3338e8a 100644 --- a/doc/frameworks/logging.rst +++ b/doc/frameworks/logging.rst @@ -48,7 +48,7 @@ Basics The data fields that a stream records are defined by a record type specified when it is created. Let's look at the script generating Bro's connection summaries as an example, -:doc:`/scripts/base/protocols/conn/main`. It defines a record +:doc:`/scripts/base/protocols/conn/main.bro`. It defines a record :bro:type:`Conn::Info` that lists all the fields that go into ``conn.log``, each marked with a ``&log`` attribute indicating that it is part of the information written out. To write a log record, the @@ -309,7 +309,7 @@ ASCII Writer Configuration -------------------------- The ASCII writer has a number of options for customizing the format of -its output, see :doc:`/scripts/base/frameworks/logging/writers/ascii`. +its output, see :doc:`/scripts/base/frameworks/logging/writers/ascii.bro`. Adding Streams ============== @@ -369,7 +369,7 @@ save the logged ``Foo::Info`` record into the connection record: } See the existing scripts for how to work with such a new connection -field. A simple example is :doc:`/scripts/base/protocols/syslog/main`. +field. A simple example is :doc:`/scripts/base/protocols/syslog/main.bro`. When you are developing scripts that add data to the :bro:type:`connection` record, care must be given to when and how long data is stored. diff --git a/doc/frameworks/notice.rst b/doc/frameworks/notice.rst index 1f0cc9da11..fe94d7f2ca 100644 --- a/doc/frameworks/notice.rst +++ b/doc/frameworks/notice.rst @@ -283,7 +283,7 @@ information to suppress duplicates for a configurable period of time. The ``$identifier`` field is typically comprised of several pieces of data related to the notice that when combined represent a unique instance of that notice. Here is an example of the script -:doc:`/scripts/policy/protocols/ssl/validate-certs` raising a notice +:doc:`/scripts/policy/protocols/ssl/validate-certs.bro` raising a notice for session negotiations where the certificate or certificate chain did not validate successfully against the available certificate authority certificates. diff --git a/doc/frameworks/signatures.rst b/doc/frameworks/signatures.rst index 915133e178..884dcb8a47 100644 --- a/doc/frameworks/signatures.rst +++ b/doc/frameworks/signatures.rst @@ -46,7 +46,7 @@ signature's event statement (``Found root!``), and data is the last piece of payload which triggered the pattern match. To turn such :bro:id:`signature_match` events into actual alarms, you can -load Bro's :doc:`/scripts/base/frameworks/signatures/main` script. +load Bro's :doc:`/scripts/base/frameworks/signatures/main.bro` script. This script contains a default event handler that raises :bro:enum:`Signatures::Sensitive_Signature` :doc:`Notices ` (as well as others; see the beginning of the script). diff --git a/doc/index.rst b/doc/index.rst index b79f1b67a6..34096694b3 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -15,7 +15,7 @@ Bro Documentation scripting/index.rst frameworks/index.rst cluster/index.rst - scripts/index.rst + script-reference/index.rst components/index.rst * :ref:`General Index ` diff --git a/doc/quickstart/index.rst b/doc/quickstart/index.rst index f570ac92db..df66eb36e8 100644 --- a/doc/quickstart/index.rst +++ b/doc/quickstart/index.rst @@ -197,7 +197,7 @@ the variable's value may not change at run-time, but whose initial value can be modified via the ``redef`` operator at parse-time. So let's continue on our path to modify the behavior for the two SSL -and SSH notices. Looking at :doc:`/scripts/base/frameworks/notice/main`, +and SSH notices. Looking at :doc:`/scripts/base/frameworks/notice/main.bro`, we see that it advertises: .. code:: bro diff --git a/doc/scripts/builtins.rst b/doc/script-reference/builtins.rst similarity index 100% rename from doc/scripts/builtins.rst rename to doc/script-reference/builtins.rst diff --git a/doc/script-reference/file-analyzers.rst b/doc/script-reference/file-analyzers.rst new file mode 100644 index 0000000000..81e71969a4 --- /dev/null +++ b/doc/script-reference/file-analyzers.rst @@ -0,0 +1 @@ +.. broxygen:file_analyzer:: * diff --git a/doc/scripts/index.rst b/doc/script-reference/index.rst similarity index 50% rename from doc/scripts/index.rst rename to doc/script-reference/index.rst index 5a19bd23f0..ccb374bd82 100644 --- a/doc/scripts/index.rst +++ b/doc/script-reference/index.rst @@ -1,5 +1,3 @@ -.. This is a stub doc to which broxygen appends during the build process - ================ Script Reference ================ @@ -7,15 +5,9 @@ Script Reference .. toctree:: :maxdepth: 1 - packages proto-analyzers file-analyzers - notices builtins - Built-in Functions (BIFs) - internal + packages scripts - - - - + Broxygen Example Script diff --git a/doc/scripts/packages.rst b/doc/script-reference/packages.rst similarity index 81% rename from doc/scripts/packages.rst rename to doc/script-reference/packages.rst index 9f01a91e91..57fde4834e 100644 --- a/doc/scripts/packages.rst +++ b/doc/script-reference/packages.rst @@ -1,9 +1,7 @@ -.. This is a stub doc to which broxygen appends during the build process - .. _script-packages: -Bro Script Packages -=================== +Bro Package Index +================= Bro has the following script packages (e.g. collections of related scripts in a common directory). If the package directory contains a ``__load__.bro`` @@ -12,3 +10,5 @@ script, it supports being loaded in mass as a whole directory for convenience. Packages/scripts in the ``base/`` directory are all loaded by default, while ones in ``policy/`` provide functionality and customization options that are more appropriate for users to decide whether they'd like to load it or not. + +.. broxygen:package_index:: * diff --git a/doc/script-reference/proto-analyzers.rst b/doc/script-reference/proto-analyzers.rst new file mode 100644 index 0000000000..85c6ec1f12 --- /dev/null +++ b/doc/script-reference/proto-analyzers.rst @@ -0,0 +1 @@ +.. broxygen:proto_analyzer:: * diff --git a/doc/script-reference/scripts.rst b/doc/script-reference/scripts.rst new file mode 100644 index 0000000000..90bbc5543d --- /dev/null +++ b/doc/script-reference/scripts.rst @@ -0,0 +1,5 @@ +================ +Bro Script Index +================ + +.. broxygen:script_index:: * diff --git a/doc/scripting/index.rst b/doc/scripting/index.rst index c6b479a7af..2118e73715 100644 --- a/doc/scripting/index.rst +++ b/doc/scripting/index.rst @@ -316,7 +316,7 @@ block that variable is available to any other script through the naming convention of ``MODULE::variable_name``. The declaration below is taken from the -:doc:`/scripts/policy/protocols/conn/known-hosts` script and +:doc:`/scripts/policy/protocols/conn/known-hosts.bro` script and declares a variable called ``known_hosts`` as a global set of unique IP addresses within the ``Known`` namespace and exports it for use outside of the ``Known`` namespace. Were we to want to use the @@ -348,7 +348,7 @@ constants are used in Bro scripts as containers for configuration options. For example, the configuration option to log password decrypted from HTTP streams is stored in ``HTTP::default_capture_password`` as shown in the stripped down -excerpt from :doc:`/scripts/base/protocols/http/main` below. +excerpt from :doc:`/scripts/base/protocols/http/main.bro` below. .. btest-include:: ${BRO_SRC_ROOT}/scripts/base/protocols/http/main.bro :lines: 8-10,19-21,120 @@ -1182,7 +1182,7 @@ passing in the ``Notice::Info`` record. The simplest kind of action based on the answer. The hook below adds the :bro:enum:`Notice::ACTION_EMAIL` action for the ``SSH::Interesting_Hostname_Login`` notice raised in the -:doc:`/scripts/policy/protocols/ssh/interesting-hostnames` script. +:doc:`/scripts/policy/protocols/ssh/interesting-hostnames.bro` script. .. btest-include:: ${DOC_ROOT}/scripting/framework_notice_hook_01.bro @@ -1224,7 +1224,7 @@ Bro. .. btest-include:: ${BRO_SRC_ROOT}/scripts/policy/protocols/ssl/expiring-certs.bro :lines: 60-63 -In the :doc:`/scripts/policy/protocols/ssl/expiring-certs` script +In the :doc:`/scripts/policy/protocols/ssl/expiring-certs.bro` script which identifies when SSL certificates are set to expire and raises notices when it crosses a predefined threshold, the call to ``NOTICE`` above also sets the ``$identifier`` entry by concatenating diff --git a/doc/scripts/CMakeLists.txt b/doc/scripts/CMakeLists.txt deleted file mode 100644 index fa234e74f2..0000000000 --- a/doc/scripts/CMakeLists.txt +++ /dev/null @@ -1,226 +0,0 @@ -# find out what BROPATH to use when executing bro -execute_process(COMMAND ${CMAKE_BINARY_DIR}/bro-path-dev - OUTPUT_VARIABLE BROPATH - RESULT_VARIABLE retval - OUTPUT_STRIP_TRAILING_WHITESPACE) -if (NOT ${retval} EQUAL 0) - message(FATAL_ERROR "Problem setting BROPATH") -endif () - -# This macro is used to add a new makefile target for reST policy script -# documentation that can be generated using Bro itself to parse policy scripts. -# It's called like: -# -# rest_target(srcDir broInput [group]) -# -# srcDir: the directory which contains broInput -# broInput: the file name of a bro policy script, any path prefix of this -# argument will be used to derive what path under scripts/ the generated -# documentation will be placed. -# group: optional name of group that the script documentation will belong to. -# If this is not given, the group is automatically set to any path portion -# of the broInput argument. -# -# In addition to adding the makefile target, several CMake variables are set: -# -# MASTER_POLICY_INDEX_TEXT: a running list of policy scripts docs that have -# been generated so far, formatted such that it can be appended to a file -# that ends in a Sphinx toctree directive -# ALL_REST_OUTPUTS: a running list (the CMake list type) of all reST docs -# that are to be generated -# MASTER_GROUP_LIST: a running list (the CMake list type) of all script groups -# MASTER_PKG_LIST: a running list (the CMake list type) of all script groups -# that were defived from the path portion of the broInput argument -# ${group}_files: a running list of files belonging to a given group, from -# which summary text can be extracted at build time -# ${group}_doc_names: a running list of reST style document names that can be -# given to a :doc: role, shared indices with ${group}_files - -macro(REST_TARGET srcDir broInput) - set(absSrcPath ${srcDir}/${broInput}) - get_filename_component(basename ${broInput} NAME) - string(REPLACE .bro "" basename ${basename}) - get_filename_component(extension ${broInput} EXT) - get_filename_component(relDstDir ${broInput} PATH) - - set(sumTextSrc ${absSrcPath}) - set(ogSourceFile ${absSrcPath}) - - if (NOT relDstDir) - set(docName "${basename}") - set(dstDir "${RST_OUTPUT_DIR}") - else () - set(docName "${relDstDir}/${basename}") - set(dstDir "${RST_OUTPUT_DIR}/${relDstDir}") - endif () - - set(restFile "${docName}.rst") - string(REPLACE "/" "^" restFile ${restFile}) - set(restOutput "${dstDir}/${basename}.rst") - - set(MASTER_POLICY_INDEX_TEXT - "${MASTER_POLICY_INDEX_TEXT}\n ${docName} <${docName}>") - list(APPEND ALL_REST_OUTPUTS ${restOutput}) - - if (NOT "${ARGN}" STREQUAL "") - set(group ${ARGN}) - elseif (relDstDir) - set(group ${relDstDir}/index) - # add package index to master package list if not already in it - # and if a __load__.bro exists in the original script directory - list(FIND MASTER_PKG_LIST ${relDstDir} _found) - if (_found EQUAL -1) - if (EXISTS ${CMAKE_SOURCE_DIR}/scripts/${relDstDir}/__load__.bro) - list(APPEND MASTER_PKG_LIST ${relDstDir}) - endif () - endif () - else () - set(group "") - endif () - - if (NOT "${group}" STREQUAL "") - # add group to master group list if not already in it - list(FIND MASTER_GROUP_LIST ${group} _found) - if (_found EQUAL -1) - list(APPEND MASTER_GROUP_LIST ${group}) - if (MASTER_GROUP_LIST_TEXT) - set(MASTER_GROUP_LIST_TEXT "${MASTER_GROUP_LIST_TEXT}\n${group}") - else () - set(MASTER_GROUP_LIST_TEXT "${group}") - endif () - endif () - - list(APPEND ${group}_files ${sumTextSrc}) - list(APPEND ${group}_doc_names ${docName}) - endif () - - add_custom_command(OUTPUT ${restOutput} - # delete any leftover state from previous bro runs - COMMAND "${CMAKE_COMMAND}" - ARGS -E remove_directory .state - # generate the reST documentation using bro - COMMAND BROPATH=${BROPATH}:${srcDir} BROMAGIC=${CMAKE_SOURCE_DIR}/magic/database ${CMAKE_BINARY_DIR}/src/bro - ARGS -b -Z ${broInput} || (rm -rf .state *.log *.rst && exit 1) - # move generated doc into a new directory tree that - # defines the final structure of documents - COMMAND "${CMAKE_COMMAND}" - ARGS -E make_directory ${dstDir} - COMMAND "${CMAKE_COMMAND}" - ARGS -E copy ${restFile} ${restOutput} - # copy the bro or bif script, too - COMMAND "${CMAKE_COMMAND}" - ARGS -E copy ${ogSourceFile} ${dstDir} - # clean up the build directory - COMMAND rm - ARGS -rf .state *.log *.rst - DEPENDS bro - DEPENDS ${absSrcPath} - WORKING_DIRECTORY ${CMAKE_BINARY_DIR} - COMMENT "[Bro] Generating reST docs for ${broInput}" - ) - -endmacro(REST_TARGET) - -# Schedule Bro scripts for which to generate documentation. -include(DocSourcesList.cmake) - -# Macro for generating reST docs that are independent of any particular Bro -# script. -macro(INDEPENDENT_REST_TARGET reST_file) - add_custom_command(OUTPUT ${reST_file} - # delete any leftover state from previous bro runs - COMMAND "${CMAKE_COMMAND}" - ARGS -E remove_directory .state - # generate the reST documentation using bro - COMMAND BROPATH=${BROPATH}:${srcDir} BROMAGIC=${CMAKE_SOURCE_DIR}/magic/database ${CMAKE_BINARY_DIR}/src/bro - ARGS -b -Z base/init-bare.bro || (rm -rf .state *.log *.rst && exit 1) - # move generated doc into a new directory tree that - # defines the final structure of documents - COMMAND "${CMAKE_COMMAND}" - ARGS -E make_directory ${dstDir} - COMMAND "${CMAKE_COMMAND}" - ARGS -E copy ${reST_file} ${dstDir} - # clean up the build directory - COMMAND rm - ARGS -rf .state *.log *.rst - DEPENDS bro - WORKING_DIRECTORY ${CMAKE_BINARY_DIR} - COMMENT "[Bro] Generating reST docs for ${reST_file}" - ) - list(APPEND ALL_REST_OUTPUTS ${reST_file}) -endmacro(INDEPENDENT_REST_TARGET) - -independent_rest_target(proto-analyzers.rst) -independent_rest_target(file-analyzers.rst) - -# create temporary list of all docs to include in the master policy/index file -file(WRITE ${MASTER_POLICY_INDEX} "${MASTER_POLICY_INDEX_TEXT}") - -# create the temporary list of all packages to include in the master -# policy/packages.rst file -set(MASTER_PKG_INDEX_TEXT "") -foreach (pkg ${MASTER_PKG_LIST}) - set(MASTER_PKG_INDEX_TEXT - "${MASTER_PKG_INDEX_TEXT}\n:doc:`${pkg} <${pkg}/index>`\n") - if (EXISTS ${CMAKE_SOURCE_DIR}/scripts/${pkg}/README) - file(STRINGS ${CMAKE_SOURCE_DIR}/scripts/${pkg}/README pkgreadme) - foreach (line ${pkgreadme}) - set(MASTER_PKG_INDEX_TEXT "${MASTER_PKG_INDEX_TEXT}\n ${line}") - endforeach () - set(MASTER_PKG_INDEX_TEXT "${MASTER_PKG_INDEX_TEXT}\n") - endif () -endforeach () -file(WRITE ${MASTER_PACKAGE_INDEX} "${MASTER_PKG_INDEX_TEXT}") - -# create temporary file containing list of all groups -file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/group_list - "${MASTER_GROUP_LIST_TEXT}") - -# create temporary files containing list of each source file in a given group -foreach (group ${MASTER_GROUP_LIST}) - if (EXISTS ${CMAKE_CURRENT_BINARY_DIR}/${group}_files) - file(REMOVE ${CMAKE_CURRENT_BINARY_DIR}/${group}_files) - endif () - if (EXISTS ${CMAKE_CURRENT_BINARY_DIR}/${group}_doc_names) - file(REMOVE ${CMAKE_CURRENT_BINARY_DIR}/${group}_doc_names) - endif () - foreach (src ${${group}_files}) - file(APPEND ${CMAKE_CURRENT_BINARY_DIR}/${group}_files "${src}\n") - endforeach () - foreach (dname ${${group}_doc_names}) - file(APPEND ${CMAKE_CURRENT_BINARY_DIR}/${group}_doc_names "${dname}\n") - endforeach () -endforeach () - -# remove previously generated docs no longer scheduled for generation -if (EXISTS ${RST_OUTPUT_DIR}) - file(GLOB_RECURSE EXISTING_REST_DOCS "${RST_OUTPUT_DIR}/*.rst") - foreach (_doc ${EXISTING_REST_DOCS}) - list(FIND ALL_REST_OUTPUTS ${_doc} _found) - if (_found EQUAL -1) - file(REMOVE ${_doc}) - message(STATUS "Broxygen: remove stale reST doc: ${_doc}") - string(REPLACE .rst .bro _brofile ${_doc}) - if (EXISTS ${_brofile}) - file(REMOVE ${_brofile}) - message(STATUS "Broxygen: remove stale bro source: ${_brofile}") - endif () - endif () - endforeach () -endif () - -# The "restdoc" target uses Bro to parse policy scripts in order to -# generate reST documentation from them. -add_custom_target(restdoc - # create symlink to the reST output directory for convenience - COMMAND "${CMAKE_COMMAND}" -E create_symlink - ${RST_OUTPUT_DIR} - ${CMAKE_BINARY_DIR}/reST - DEPENDS ${ALL_REST_OUTPUTS}) - -# The "restclean" target removes all generated reST documentation from the -# build directory. -add_custom_target(restclean - COMMAND "${CMAKE_COMMAND}" -E remove_directory - ${RST_OUTPUT_DIR} - VERBATIM) diff --git a/doc/scripts/DocSourcesList.cmake b/doc/scripts/DocSourcesList.cmake deleted file mode 100644 index 61682d73b3..0000000000 --- a/doc/scripts/DocSourcesList.cmake +++ /dev/null @@ -1,282 +0,0 @@ -# DO NOT EDIT -# This file is auto-generated from the genDocSourcesList.sh script. -# -# This is a list of Bro script sources for which to generate reST documentation. -# It will be included inline in the CMakeLists.txt found in the same directory -# in order to create Makefile targets that define how to generate reST from -# a given Bro script. -# -# Note: any path prefix of the script (2nd argument of rest_target macro) -# will be used to derive what path under scripts/ the generated documentation -# will be placed. - -set(psd ${PROJECT_SOURCE_DIR}/scripts) - -rest_target(${CMAKE_CURRENT_SOURCE_DIR} example.bro internal) -rest_target(${psd} base/init-default.bro internal) -rest_target(${psd} base/init-bare.bro internal) - -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/analyzer.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/bloom-filter.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/bro.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/broxygen.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/cardinality-counter.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/const.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/event.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/file_analysis.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/input.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/logging.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_ARP.events.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_AYIYA.events.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_BackDoor.events.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_BitTorrent.events.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_ConnSize.events.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_DCE_RPC.events.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_DHCP.events.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_DNP3.events.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_DNS.events.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_FTP.events.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_FTP.functions.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_File.events.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_FileExtract.events.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_FileExtract.functions.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_FileHash.events.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_Finger.events.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_GTPv1.events.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_Gnutella.events.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_HTTP.events.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_HTTP.functions.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_ICMP.events.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_IRC.events.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_Ident.events.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_InterConn.events.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_Login.events.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_Login.functions.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_MIME.events.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_Modbus.events.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_NCP.events.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_NTP.events.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_NetBIOS.events.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_NetBIOS.functions.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_NetFlow.events.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_PIA.events.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_POP3.events.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_RPC.events.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_SMB.events.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_SMTP.events.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_SMTP.functions.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_SOCKS.events.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_SSH.events.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_SSL.events.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_SSL.functions.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_SteppingStone.events.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_Syslog.events.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_TCP.events.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_TCP.functions.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_Teredo.events.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_UDP.events.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_Unified2.events.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_Unified2.types.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_ZIP.events.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/reporter.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/strings.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/top-k.bif.bro) -rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/types.bif.bro) -rest_target(${psd} base/files/extract/main.bro) -rest_target(${psd} base/files/hash/main.bro) -rest_target(${psd} base/files/unified2/main.bro) -rest_target(${psd} base/frameworks/analyzer/main.bro) -rest_target(${psd} base/frameworks/cluster/main.bro) -rest_target(${psd} base/frameworks/cluster/nodes/manager.bro) -rest_target(${psd} base/frameworks/cluster/nodes/proxy.bro) -rest_target(${psd} base/frameworks/cluster/nodes/worker.bro) -rest_target(${psd} base/frameworks/cluster/setup-connections.bro) -rest_target(${psd} base/frameworks/communication/main.bro) -rest_target(${psd} base/frameworks/control/main.bro) -rest_target(${psd} base/frameworks/dpd/main.bro) -rest_target(${psd} base/frameworks/files/main.bro) -rest_target(${psd} base/frameworks/input/main.bro) -rest_target(${psd} base/frameworks/input/readers/ascii.bro) -rest_target(${psd} base/frameworks/input/readers/benchmark.bro) -rest_target(${psd} base/frameworks/input/readers/binary.bro) -rest_target(${psd} base/frameworks/input/readers/raw.bro) -rest_target(${psd} base/frameworks/input/readers/sqlite.bro) -rest_target(${psd} base/frameworks/intel/cluster.bro) -rest_target(${psd} base/frameworks/intel/input.bro) -rest_target(${psd} base/frameworks/intel/main.bro) -rest_target(${psd} base/frameworks/logging/main.bro) -rest_target(${psd} base/frameworks/logging/postprocessors/scp.bro) -rest_target(${psd} base/frameworks/logging/postprocessors/sftp.bro) -rest_target(${psd} base/frameworks/logging/writers/ascii.bro) -rest_target(${psd} base/frameworks/logging/writers/dataseries.bro) -rest_target(${psd} base/frameworks/logging/writers/elasticsearch.bro) -rest_target(${psd} base/frameworks/logging/writers/none.bro) -rest_target(${psd} base/frameworks/logging/writers/sqlite.bro) -rest_target(${psd} base/frameworks/notice/actions/add-geodata.bro) -rest_target(${psd} base/frameworks/notice/actions/drop.bro) -rest_target(${psd} base/frameworks/notice/actions/email_admin.bro) -rest_target(${psd} base/frameworks/notice/actions/page.bro) -rest_target(${psd} base/frameworks/notice/actions/pp-alarms.bro) -rest_target(${psd} base/frameworks/notice/cluster.bro) -rest_target(${psd} base/frameworks/notice/extend-email/hostnames.bro) -rest_target(${psd} base/frameworks/notice/main.bro) -rest_target(${psd} base/frameworks/notice/non-cluster.bro) -rest_target(${psd} base/frameworks/notice/weird.bro) -rest_target(${psd} base/frameworks/packet-filter/cluster.bro) -rest_target(${psd} base/frameworks/packet-filter/main.bro) -rest_target(${psd} base/frameworks/packet-filter/netstats.bro) -rest_target(${psd} base/frameworks/packet-filter/utils.bro) -rest_target(${psd} base/frameworks/reporter/main.bro) -rest_target(${psd} base/frameworks/signatures/main.bro) -rest_target(${psd} base/frameworks/software/main.bro) -rest_target(${psd} base/frameworks/sumstats/cluster.bro) -rest_target(${psd} base/frameworks/sumstats/main.bro) -rest_target(${psd} base/frameworks/sumstats/non-cluster.bro) -rest_target(${psd} base/frameworks/sumstats/plugins/average.bro) -rest_target(${psd} base/frameworks/sumstats/plugins/hll_unique.bro) -rest_target(${psd} base/frameworks/sumstats/plugins/last.bro) -rest_target(${psd} base/frameworks/sumstats/plugins/max.bro) -rest_target(${psd} base/frameworks/sumstats/plugins/min.bro) -rest_target(${psd} base/frameworks/sumstats/plugins/sample.bro) -rest_target(${psd} base/frameworks/sumstats/plugins/std-dev.bro) -rest_target(${psd} base/frameworks/sumstats/plugins/sum.bro) -rest_target(${psd} base/frameworks/sumstats/plugins/topk.bro) -rest_target(${psd} base/frameworks/sumstats/plugins/unique.bro) -rest_target(${psd} base/frameworks/sumstats/plugins/variance.bro) -rest_target(${psd} base/frameworks/tunnels/main.bro) -rest_target(${psd} base/misc/find-checksum-offloading.bro) -rest_target(${psd} base/protocols/conn/contents.bro) -rest_target(${psd} base/protocols/conn/inactivity.bro) -rest_target(${psd} base/protocols/conn/main.bro) -rest_target(${psd} base/protocols/conn/polling.bro) -rest_target(${psd} base/protocols/dhcp/consts.bro) -rest_target(${psd} base/protocols/dhcp/main.bro) -rest_target(${psd} base/protocols/dhcp/utils.bro) -rest_target(${psd} base/protocols/dnp3/consts.bro) -rest_target(${psd} base/protocols/dnp3/main.bro) -rest_target(${psd} base/protocols/dns/consts.bro) -rest_target(${psd} base/protocols/dns/main.bro) -rest_target(${psd} base/protocols/ftp/files.bro) -rest_target(${psd} base/protocols/ftp/gridftp.bro) -rest_target(${psd} base/protocols/ftp/info.bro) -rest_target(${psd} base/protocols/ftp/main.bro) -rest_target(${psd} base/protocols/ftp/utils-commands.bro) -rest_target(${psd} base/protocols/ftp/utils.bro) -rest_target(${psd} base/protocols/http/entities.bro) -rest_target(${psd} base/protocols/http/files.bro) -rest_target(${psd} base/protocols/http/main.bro) -rest_target(${psd} base/protocols/http/utils.bro) -rest_target(${psd} base/protocols/irc/dcc-send.bro) -rest_target(${psd} base/protocols/irc/files.bro) -rest_target(${psd} base/protocols/irc/main.bro) -rest_target(${psd} base/protocols/modbus/consts.bro) -rest_target(${psd} base/protocols/modbus/main.bro) -rest_target(${psd} base/protocols/smtp/entities.bro) -rest_target(${psd} base/protocols/smtp/files.bro) -rest_target(${psd} base/protocols/smtp/main.bro) -rest_target(${psd} base/protocols/socks/consts.bro) -rest_target(${psd} base/protocols/socks/main.bro) -rest_target(${psd} base/protocols/ssh/main.bro) -rest_target(${psd} base/protocols/ssl/consts.bro) -rest_target(${psd} base/protocols/ssl/main.bro) -rest_target(${psd} base/protocols/ssl/mozilla-ca-list.bro) -rest_target(${psd} base/protocols/syslog/consts.bro) -rest_target(${psd} base/protocols/syslog/main.bro) -rest_target(${psd} base/utils/active-http.bro) -rest_target(${psd} base/utils/addrs.bro) -rest_target(${psd} base/utils/conn-ids.bro) -rest_target(${psd} base/utils/dir.bro) -rest_target(${psd} base/utils/directions-and-hosts.bro) -rest_target(${psd} base/utils/exec.bro) -rest_target(${psd} base/utils/files.bro) -rest_target(${psd} base/utils/numbers.bro) -rest_target(${psd} base/utils/paths.bro) -rest_target(${psd} base/utils/patterns.bro) -rest_target(${psd} base/utils/queue.bro) -rest_target(${psd} base/utils/site.bro) -rest_target(${psd} base/utils/strings.bro) -rest_target(${psd} base/utils/thresholds.bro) -rest_target(${psd} base/utils/time.bro) -rest_target(${psd} base/utils/urls.bro) -rest_target(${psd} policy/frameworks/communication/listen.bro) -rest_target(${psd} policy/frameworks/control/controllee.bro) -rest_target(${psd} policy/frameworks/control/controller.bro) -rest_target(${psd} policy/frameworks/dpd/detect-protocols.bro) -rest_target(${psd} policy/frameworks/dpd/packet-segment-logging.bro) -rest_target(${psd} policy/frameworks/files/detect-MHR.bro) -rest_target(${psd} policy/frameworks/files/hash-all-files.bro) -rest_target(${psd} policy/frameworks/intel/do_notice.bro) -rest_target(${psd} policy/frameworks/intel/seen/conn-established.bro) -rest_target(${psd} policy/frameworks/intel/seen/dns.bro) -rest_target(${psd} policy/frameworks/intel/seen/file-hashes.bro) -rest_target(${psd} policy/frameworks/intel/seen/file-names.bro) -rest_target(${psd} policy/frameworks/intel/seen/http-headers.bro) -rest_target(${psd} policy/frameworks/intel/seen/http-url.bro) -rest_target(${psd} policy/frameworks/intel/seen/smtp-url-extraction.bro) -rest_target(${psd} policy/frameworks/intel/seen/smtp.bro) -rest_target(${psd} policy/frameworks/intel/seen/ssl.bro) -rest_target(${psd} policy/frameworks/intel/seen/where-locations.bro) -rest_target(${psd} policy/frameworks/packet-filter/shunt.bro) -rest_target(${psd} policy/frameworks/software/version-changes.bro) -rest_target(${psd} policy/frameworks/software/vulnerable.bro) -rest_target(${psd} policy/integration/barnyard2/main.bro) -rest_target(${psd} policy/integration/barnyard2/types.bro) -rest_target(${psd} policy/integration/collective-intel/main.bro) -rest_target(${psd} policy/misc/app-stats/main.bro) -rest_target(${psd} policy/misc/app-stats/plugins/facebook.bro) -rest_target(${psd} policy/misc/app-stats/plugins/gmail.bro) -rest_target(${psd} policy/misc/app-stats/plugins/google.bro) -rest_target(${psd} policy/misc/app-stats/plugins/netflix.bro) -rest_target(${psd} policy/misc/app-stats/plugins/pandora.bro) -rest_target(${psd} policy/misc/app-stats/plugins/youtube.bro) -rest_target(${psd} policy/misc/capture-loss.bro) -rest_target(${psd} policy/misc/detect-traceroute/main.bro) -rest_target(${psd} policy/misc/known-devices.bro) -rest_target(${psd} policy/misc/load-balancing.bro) -rest_target(${psd} policy/misc/loaded-scripts.bro) -rest_target(${psd} policy/misc/profiling.bro) -rest_target(${psd} policy/misc/scan.bro) -rest_target(${psd} policy/misc/stats.bro) -rest_target(${psd} policy/misc/trim-trace-file.bro) -rest_target(${psd} policy/protocols/conn/known-hosts.bro) -rest_target(${psd} policy/protocols/conn/known-services.bro) -rest_target(${psd} policy/protocols/conn/weirds.bro) -rest_target(${psd} policy/protocols/dhcp/known-devices-and-hostnames.bro) -rest_target(${psd} policy/protocols/dns/auth-addl.bro) -rest_target(${psd} policy/protocols/dns/detect-external-names.bro) -rest_target(${psd} policy/protocols/ftp/detect-bruteforcing.bro) -rest_target(${psd} policy/protocols/ftp/detect.bro) -rest_target(${psd} policy/protocols/ftp/software.bro) -rest_target(${psd} policy/protocols/http/detect-sqli.bro) -rest_target(${psd} policy/protocols/http/detect-webapps.bro) -rest_target(${psd} policy/protocols/http/header-names.bro) -rest_target(${psd} policy/protocols/http/software-browser-plugins.bro) -rest_target(${psd} policy/protocols/http/software.bro) -rest_target(${psd} policy/protocols/http/var-extraction-cookies.bro) -rest_target(${psd} policy/protocols/http/var-extraction-uri.bro) -rest_target(${psd} policy/protocols/modbus/known-masters-slaves.bro) -rest_target(${psd} policy/protocols/modbus/track-memmap.bro) -rest_target(${psd} policy/protocols/smtp/blocklists.bro) -rest_target(${psd} policy/protocols/smtp/detect-suspicious-orig.bro) -rest_target(${psd} policy/protocols/smtp/entities-excerpt.bro) -rest_target(${psd} policy/protocols/smtp/software.bro) -rest_target(${psd} policy/protocols/ssh/detect-bruteforcing.bro) -rest_target(${psd} policy/protocols/ssh/geo-data.bro) -rest_target(${psd} policy/protocols/ssh/interesting-hostnames.bro) -rest_target(${psd} policy/protocols/ssh/software.bro) -rest_target(${psd} policy/protocols/ssl/cert-hash.bro) -rest_target(${psd} policy/protocols/ssl/expiring-certs.bro) -rest_target(${psd} policy/protocols/ssl/extract-certs-pem.bro) -rest_target(${psd} policy/protocols/ssl/known-certs.bro) -rest_target(${psd} policy/protocols/ssl/notary.bro) -rest_target(${psd} policy/protocols/ssl/validate-certs.bro) -rest_target(${psd} policy/tuning/defaults/extracted_file_limits.bro) -rest_target(${psd} policy/tuning/defaults/packet-fragments.bro) -rest_target(${psd} policy/tuning/defaults/warnings.bro) -rest_target(${psd} policy/tuning/logs-to-elasticsearch.bro) -rest_target(${psd} policy/tuning/track-all-assets.bro) -rest_target(${psd} site/local-manager.bro) -rest_target(${psd} site/local-proxy.bro) -rest_target(${psd} site/local-worker.bro) -rest_target(${psd} site/local.bro) -rest_target(${psd} test-all-policy.bro) diff --git a/doc/scripts/README b/doc/scripts/README deleted file mode 100644 index a15812609c..0000000000 --- a/doc/scripts/README +++ /dev/null @@ -1,44 +0,0 @@ -This directory contains scripts and templates that can be used to automate -the generation of Bro script documentation. Several build targets are defined -by CMake and available in the top-level Makefile: - -``restdoc`` - - This target uses Bro to parse policy scripts in order to generate - reStructuredText (reST) documentation from them. The list of scripts - for which to generate reST documentation is defined in the - ``CMakeLists.txt`` file in this directory. Script documentation is - rebuild automatically if the policy script from which it is derived - or the Bro binary becomes out of date - - The resulting output from this target can be found in the CMake - ``build/`` directory inside ``reST`` (a symlink to - ``doc/scripts/rest_output``). - -``restclean`` - - This target removes any reST documentation that has been generated so far. - -The ``genDocSourcesList.sh`` script can be run to automatically generate -``DocSourcesList.cmake``, which is the file CMake uses to define the list -of documentation targets. This script should be run after adding new -Bro script source files, and the changes commited to git. - -If a script shouldn't have documentation generated for it, there's also a -blacklist manifest that can be maintained in the ``genDocSourcesList.sh`` -script. - -The blacklist can also be used if you want to define a certain grouping for -the script's generated docs to belong to (as opposed to the automatic grouping -the happens for script packages/directories). To do that, add the -script's name to the blacklist, then append a ``rest_target()`` to the -``statictext`` variable where the first argument is the source directory -containing the policy script to document, the second argument is the file -name of the policy script, and the third argument is the path/name of a -pre-created reST document in the ``../`` source directory to which the -``make doc`` process can append script documentation references. This -pre-created reST document should also then be linked to from the TOC tree -in ``../index.rst``. - -See ``example.bro`` for an example of how to document a Bro script such that -``make doc`` will be able to produce reST/HTML documentation for it. diff --git a/doc/scripts/example.rst b/doc/scripts/example.rst deleted file mode 100644 index b76b9af59b..0000000000 --- a/doc/scripts/example.rst +++ /dev/null @@ -1,291 +0,0 @@ -.. Automatically generated. Do not edit. - -example.bro -=========== - -:download:`Original Source File ` - -Overview --------- -This is an example script that demonstrates how to document. Comments -of the form ``##!`` are for the script summary. The contents of -these comments are transferred directly into the auto-generated -`reStructuredText `_ -(reST) document's summary section. - -.. tip:: You can embed directives and roles within ``##``-stylized comments. - -:Imports: :doc:`policy/frameworks/software/vulnerable ` - -Summary -~~~~~~~ -Options -####### -============================================================================ ====================================== -:bro:id:`Example::an_option`: :bro:type:`set` :bro:attr:`&redef` add documentation for "an_option" here - -:bro:id:`Example::option_with_init`: :bro:type:`interval` :bro:attr:`&redef` -============================================================================ ====================================== - -State Variables -############### -=========================================================================== ======================================= -:bro:id:`Example::a_var`: :bro:type:`bool` put some documentation for "a_var" here - -:bro:id:`Example::var_with_attr`: :bro:type:`count` :bro:attr:`&persistent` - -:bro:id:`Example::var_without_explicit_type`: :bro:type:`string` -=========================================================================== ======================================= - -Types -##### -====================================================== ========================================================== -:bro:type:`Example::SimpleEnum`: :bro:type:`enum` documentation for "SimpleEnum" - goes here. - -:bro:type:`Example::SimpleRecord`: :bro:type:`record` general documentation for a type "SimpleRecord" - goes here. - -:bro:type:`Example::ComplexRecord`: :bro:type:`record` general documentation for a type "ComplexRecord" goes here - -:bro:type:`Example::Info`: :bro:type:`record` An example record to be used with a logging stream. -====================================================== ========================================================== - -Events -###### -================================================= ============================================================= -:bro:id:`Example::an_event`: :bro:type:`event` Summarize "an_event" here. - -:bro:id:`Example::log_example`: :bro:type:`event` This is a declaration of an example event that can be used in - logging streams and is raised once for each log entry. -================================================= ============================================================= - -Functions -######### -=============================================== ======================================= -:bro:id:`Example::a_function`: :bro:type:`func` Summarize purpose of "a_function" here. -=============================================== ======================================= - -Redefinitions -############# -===================================================== ======================================== -:bro:type:`Log::ID`: :bro:type:`enum` - -:bro:type:`Example::SimpleEnum`: :bro:type:`enum` document the "SimpleEnum" redef here - -:bro:type:`Example::SimpleRecord`: :bro:type:`record` document the record extension redef here -===================================================== ======================================== - -Namespaces -~~~~~~~~~~ -.. bro:namespace:: Example - -Notices -~~~~~~~ -:bro:type:`Notice::Type` - - :Type: :bro:type:`enum` - - .. bro:enum:: Example::Notice_One Notice::Type - - any number of this type of comment - will document "Notice_One" - - .. bro:enum:: Example::Notice_Two Notice::Type - - any number of this type of comment - will document "Notice_Two" - - .. bro:enum:: Example::Notice_Three Notice::Type - - .. bro:enum:: Example::Notice_Four Notice::Type - -Public Interface ----------------- -Options -~~~~~~~ -.. bro:id:: Example::an_option - - :Type: :bro:type:`set` [:bro:type:`addr`, :bro:type:`addr`, :bro:type:`string`] - :Attributes: :bro:attr:`&redef` - :Default: ``{}`` - - add documentation for "an_option" here - -.. bro:id:: Example::option_with_init - - :Type: :bro:type:`interval` - :Attributes: :bro:attr:`&redef` - :Default: ``10.0 msecs`` - -State Variables -~~~~~~~~~~~~~~~ -.. bro:id:: Example::a_var - - :Type: :bro:type:`bool` - - put some documentation for "a_var" here - -.. bro:id:: Example::var_with_attr - - :Type: :bro:type:`count` - :Attributes: :bro:attr:`&persistent` - -.. bro:id:: Example::var_without_explicit_type - - :Type: :bro:type:`string` - :Default: ``"this works"`` - -Types -~~~~~ -.. bro:type:: Example::SimpleEnum - - :Type: :bro:type:`enum` - - .. bro:enum:: Example::ONE Example::SimpleEnum - - and more specific info for "ONE" - can span multiple lines - - .. bro:enum:: Example::TWO Example::SimpleEnum - - or more info like this for "TWO" - can span multiple lines - - .. bro:enum:: Example::THREE Example::SimpleEnum - - documentation for "SimpleEnum" - goes here. - -.. bro:type:: Example::SimpleRecord - - :Type: :bro:type:`record` - - field1: :bro:type:`count` - counts something - - field2: :bro:type:`bool` - toggles something - - general documentation for a type "SimpleRecord" - goes here. - -.. bro:type:: Example::ComplexRecord - - :Type: :bro:type:`record` - - field1: :bro:type:`count` - counts something - - field2: :bro:type:`bool` - toggles something - - field3: :bro:type:`Example::SimpleRecord` - - msg: :bro:type:`string` :bro:attr:`&default` = ``"blah"`` :bro:attr:`&optional` - attributes are self-documenting - - general documentation for a type "ComplexRecord" goes here - -.. bro:type:: Example::Info - - :Type: :bro:type:`record` - - ts: :bro:type:`time` :bro:attr:`&log` - - uid: :bro:type:`string` :bro:attr:`&log` - - status: :bro:type:`count` :bro:attr:`&log` :bro:attr:`&optional` - - An example record to be used with a logging stream. - -Events -~~~~~~ -.. bro:id:: Example::an_event - - :Type: :bro:type:`event` (name: :bro:type:`string`) - - Summarize "an_event" here. - Give more details about "an_event" here. - - :param name: describe the argument here - -.. bro:id:: Example::log_example - - :Type: :bro:type:`event` (rec: :bro:type:`Example::Info`) - - This is a declaration of an example event that can be used in - logging streams and is raised once for each log entry. - -Functions -~~~~~~~~~ -.. bro:id:: Example::a_function - - :Type: :bro:type:`function` (tag: :bro:type:`string`, msg: :bro:type:`string`) : :bro:type:`string` - - Summarize purpose of "a_function" here. - Give more details about "a_function" here. - Separating the documentation of the params/return values with - empty comments is optional, but improves readability of script. - - - :param tag: function arguments can be described - like this - - :param msg: another param - - - :returns: describe the return type here - -Redefinitions -~~~~~~~~~~~~~ -:bro:type:`Log::ID` - - :Type: :bro:type:`enum` - - .. bro:enum:: Example::LOG Log::ID - -:bro:type:`Example::SimpleEnum` - - :Type: :bro:type:`enum` - - .. bro:enum:: Example::FOUR Example::SimpleEnum - - and some documentation for "FOUR" - - .. bro:enum:: Example::FIVE Example::SimpleEnum - - also "FIVE" for good measure - - document the "SimpleEnum" redef here - -:bro:type:`Example::SimpleRecord` - - :Type: :bro:type:`record` - - field_ext: :bro:type:`string` :bro:attr:`&optional` - document the extending field here - (or here) - - document the record extension redef here - -Port Analysis -------------- -:ref:`More Information ` - -SSL:: - - [ports={ - 443/tcp, - 562/tcp - }] - -Packet Filter -------------- -:ref:`More Information ` - -Filters added:: - - [ssl] = tcp port 443, - [nntps] = tcp port 562 - diff --git a/doc/scripts/genDocSourcesList.sh b/doc/scripts/genDocSourcesList.sh deleted file mode 100755 index 31905c68db..0000000000 --- a/doc/scripts/genDocSourcesList.sh +++ /dev/null @@ -1,86 +0,0 @@ -#!/usr/bin/env bash - -# ./genDocSourcesList.sh [output file] -# -# Run this script to a generate file that's used to tell CMake about all the -# possible scripts for which reST documentation can be created. -# -# The optional argument can be used to avoid overwriting the file CMake uses -# by default. -# -# Specific scripts can be blacklisted below when e.g. they currently aren't -# parseable or they just aren't meant to be documented. - -export LC_ALL=C # Make sorting stable. - -blacklist () - { - if [[ "$blacklist" == "" ]]; then - blacklist="$1" - else - blacklist="$blacklist|$1" - fi - } - -# files passed into this function are meant to be temporary workarounds -# because they're not finished or otherwise can't be loaded for some reason -tmp_blacklist () - { - echo "Warning: temporarily blacklisted files named '$1'" 1>&2 - blacklist $1 - } - -blacklist __load__.bro -blacklist test-all.bro -blacklist all.bro -blacklist init-default.bro -blacklist init-bare.bro - -statictext="\ -# DO NOT EDIT -# This file is auto-generated from the "genDocSourcesList.sh" script. -# -# This is a list of Bro script sources for which to generate reST documentation. -# It will be included inline in the CMakeLists.txt found in the same directory -# in order to create Makefile targets that define how to generate reST from -# a given Bro script. -# -# Note: any path prefix of the script (2nd argument of rest_target macro) -# will be used to derive what path under scripts/ the generated documentation -# will be placed. - -set(psd \${PROJECT_SOURCE_DIR}/scripts) - -rest_target(\${CMAKE_CURRENT_SOURCE_DIR} example.bro internal) -rest_target(\${psd} base/init-default.bro internal) -rest_target(\${psd} base/init-bare.bro internal) -" - -if [[ $# -ge 1 ]]; then - outfile=$1 -else - outfile=DocSourcesList.cmake -fi - -thisdir="$( cd "$( dirname "$0" )" && pwd )" -sourcedir=${thisdir}/../.. - -echo "$statictext" > $outfile - -bifs=`( cd ${sourcedir}/build/scripts/base && find . -name \*\.bif.bro | sort )` - -for file in $bifs -do - f=${file:2} - echo "rest_target(\${CMAKE_BINARY_DIR}/scripts base/$f)" >> $outfile -done - -scriptfiles=`( cd ${sourcedir}/scripts && find . -name \*\.bro | sort )` - -for file in $scriptfiles -do - f=${file:2} - if [[ ! $f =~ $blacklist ]]; then - echo "rest_target(\${psd} $f)" >> $outfile - fi -done diff --git a/doc/scripts/internal.rst b/doc/scripts/internal.rst deleted file mode 100644 index a6c10f1cfb..0000000000 --- a/doc/scripts/internal.rst +++ /dev/null @@ -1,5 +0,0 @@ -.. This is a stub doc to which broxygen appends during the build process - -Internal Scripts -================ - diff --git a/doc/scripts/scripts.rst b/doc/scripts/scripts.rst deleted file mode 100644 index 8aceacf329..0000000000 --- a/doc/scripts/scripts.rst +++ /dev/null @@ -1,8 +0,0 @@ -.. This is a stub doc to which broxygen appends during the build process - -=============== -All Bro Scripts -=============== - -.. toctree:: - :maxdepth: 1 diff --git a/scripts/base/frameworks/reporter/main.bro b/scripts/base/frameworks/reporter/main.bro index 3668348ef1..873cb15a45 100644 --- a/scripts/base/frameworks/reporter/main.bro +++ b/scripts/base/frameworks/reporter/main.bro @@ -9,7 +9,7 @@ ##! Note that this framework deals with the handling of internally generated ##! reporter messages, for the interface ##! into actually creating reporter messages from the scripting layer, use -##! the built-in functions in :doc:`/scripts/base/bif/reporter.bif`. +##! the built-in functions in :doc:`/scripts/base/bif/reporter.bif.bro`. module Reporter; diff --git a/scripts/base/init-bare.bro b/scripts/base/init-bare.bro index de26e6a41d..8213126b2f 100644 --- a/scripts/base/init-bare.bro +++ b/scripts/base/init-bare.bro @@ -399,7 +399,7 @@ type NetStats: record { pkts_dropped: count &default=0; ##< Packets reported dropped by the system. ## Packets seen on the link. Note that this may differ ## from *pkts_recvd* because of a potential capture_filter. See - ## :doc:`/scripts/base/frameworks/packet-filter/main`. Depending on the + ## :doc:`/scripts/base/frameworks/packet-filter/main.bro`. Depending on the ## packet capture system, this value may not be available and will then ## be always set to zero. pkts_link: count &default=0; @@ -2768,13 +2768,13 @@ const log_max_size = 0.0 &redef; const log_encryption_key = "" &redef; ## Write profiling info into this file in regular intervals. The easiest way to -## activate profiling is loading :doc:`/scripts/policy/misc/profiling`. +## activate profiling is loading :doc:`/scripts/policy/misc/profiling.bro`. ## ## .. bro:see:: profiling_interval expensive_profiling_multiple segment_profiling global profiling_file: file &redef; ## Update interval for profiling (0 disables). The easiest way to activate -## profiling is loading :doc:`/scripts/policy/misc/profiling`. +## profiling is loading :doc:`/scripts/policy/misc/profiling.bro`. ## ## .. bro:see:: profiling_file expensive_profiling_multiple segment_profiling const profiling_interval = 0 secs &redef; diff --git a/scripts/base/protocols/dhcp/main.bro b/scripts/base/protocols/dhcp/main.bro index e5be50b1ab..c94bae0476 100644 --- a/scripts/base/protocols/dhcp/main.bro +++ b/scripts/base/protocols/dhcp/main.bro @@ -4,7 +4,7 @@ ##! ##! If you'd like to track known DHCP devices and to log the hostname ##! supplied by the client, see -##! :doc:`/scripts/policy/protocols/dhcp/known-devices-and-hostnames`. +##! :doc:`/scripts/policy/protocols/dhcp/known-devices-and-hostnames.bro`. @load ./utils.bro diff --git a/scripts/base/protocols/http/__load__.bro b/scripts/base/protocols/http/__load__.bro index 20fcd5f1ec..42a45efb36 100644 --- a/scripts/base/protocols/http/__load__.bro +++ b/scripts/base/protocols/http/__load__.bro @@ -3,4 +3,4 @@ @load ./utils @load ./files -@load-sigs ./dpd.sig \ No newline at end of file +@load-sigs ./dpd.sig diff --git a/scripts/broxygen/README b/scripts/broxygen/README new file mode 100644 index 0000000000..ac7f522285 --- /dev/null +++ b/scripts/broxygen/README @@ -0,0 +1,4 @@ +This package is loaded during the process which automatically generates +reference documentation for all Bro scripts (i.e. "Broxygen"). Its only +purpose is to provide an easy way to load all known Bro scripts plus any +extra scripts needed or used by the documentation process. diff --git a/scripts/broxygen/__load__.bro b/scripts/broxygen/__load__.bro new file mode 100644 index 0000000000..68a0d878b6 --- /dev/null +++ b/scripts/broxygen/__load__.bro @@ -0,0 +1,2 @@ +@load test-all-policy.bro +@load ./example.bro diff --git a/doc/scripts/example.bro b/scripts/broxygen/example.bro similarity index 100% rename from doc/scripts/example.bro rename to scripts/broxygen/example.bro diff --git a/scripts/policy/misc/known-devices.bro b/scripts/policy/misc/known-devices.bro index 16c5250d1c..8378d589f8 100644 --- a/scripts/policy/misc/known-devices.bro +++ b/scripts/policy/misc/known-devices.bro @@ -7,7 +7,7 @@ ##! ##! This script will not generate any logs on its own, it needs to be ##! supplied with information from elsewhere, such as -##! :doc:`/scripts/policy/protocols/dhcp/known-devices-and-hostnames`. +##! :doc:`/scripts/policy/protocols/dhcp/known-devices-and-hostnames.bro`. module Known; diff --git a/scripts/policy/misc/stats.bro b/scripts/policy/misc/stats.bro index 7e1e4b6689..eb1ddb0202 100644 --- a/scripts/policy/misc/stats.bro +++ b/scripts/policy/misc/stats.bro @@ -1,5 +1,5 @@ ##! Log memory/packet/lag statistics. Differs from -##! :doc:`/scripts/policy/misc/profiling` in that this +##! :doc:`/scripts/policy/misc/profiling.bro` in that this ##! is lighter-weight (much less info, and less load to generate). @load base/frameworks/notice diff --git a/src/BroDoc.cc b/src/BroDoc.cc index a7b7580fa6..1239c4efce 100644 --- a/src/BroDoc.cc +++ b/src/BroDoc.cc @@ -611,7 +611,7 @@ void CreateProtoAnalyzerDoc(FILE* f) fprintf(f, "Protocol Analyzers\n"); fprintf(f, "==================\n\n"); fprintf(f, ".. contents::\n"); - fprintf(f, " :depth: 1\n\n"); + fprintf(f, " :depth: 2\n\n"); WriteAnalyzerTagDefn(f, "Analyzer"); @@ -641,7 +641,7 @@ void CreateFileAnalyzerDoc(FILE* f) fprintf(f, "File Analyzers\n"); fprintf(f, "==============\n\n"); fprintf(f, ".. contents::\n"); - fprintf(f, " :depth: 1\n\n"); + fprintf(f, " :depth: 2\n\n"); WriteAnalyzerTagDefn(f, "Files"); diff --git a/src/Type.cc b/src/Type.cc index 83f239bbfe..8ef6471d67 100644 --- a/src/Type.cc +++ b/src/Type.cc @@ -1165,9 +1165,9 @@ void RecordType::DescribeFieldsReST(ODesc* d, bool func_args) const field_from_script != type_from_script ) { d->PushIndent(); - d->Add(fmt("(from ``redef`` in :doc:`%s`)", + d->Add(fmt("(from ``redef`` in :doc:`/scripts/%s`)", field_from_script.c_str())); - d->PopIndentNoNL(); + d->PopIndent(); } vector cmnts = doc->GetFieldComments(td->id); @@ -1513,9 +1513,9 @@ void EnumType::DescribeReST(ODesc* d, bool roles_only) const { d->NL(); d->PushIndent(); - d->Add(fmt("(from ``redef`` in :doc:`%s`)", + d->Add(fmt("(from ``redef`` in :doc:`/scripts/%s`)", enum_from_script.c_str())); - d->PopIndentNoNL(); + d->PopIndent(); } vector cmnts = doc->GetComments(); diff --git a/src/broxygen/Configuration.cc b/src/broxygen/Configuration.cc index d88aa1920c..9d0c0f7015 100644 --- a/src/broxygen/Configuration.cc +++ b/src/broxygen/Configuration.cc @@ -9,7 +9,10 @@ #include #include #include +#include #include +#include +#include using namespace broxygen; using namespace std; @@ -160,24 +163,38 @@ void PackageTarget::DoFindDependencies(const vector& docs) void PackageTarget::DoGenerate() const { - if ( broxygen_mgr->IsUpToDate(Name(), script_deps) ) + if ( broxygen_mgr->IsUpToDate(Name(), script_deps) && + broxygen_mgr->IsUpToDate(Name(), pkg_deps) ) return; TargetFile file(Name()); + fprintf(file.f, ":orphan:\n\n"); + for ( manifest_t::const_iterator it = pkg_manifest.begin(); it != pkg_manifest.end(); ++it ) { - fprintf(file.f, "Package: %s\n\n", it->first->Name().c_str()); + string header = fmt("Package: %s", it->first->Name().c_str()); + header += "\n" + string(header.size(), '='); + + fprintf(file.f, "%s\n\n", header.c_str()); + + vector readme = it->first->GetReadme(); + + for ( size_t i = 0; i < readme.size(); ++i ) + fprintf(file.f, "%s\n", readme[i].c_str()); + + fprintf(file.f, "\n"); for ( size_t i = 0; i < it->second.size(); ++i ) { - fprintf(file.f, " :doc:`%s`\n", it->second[i]->Name().c_str()); + fprintf(file.f, ":doc:`/scripts/%s`\n", + it->second[i]->Name().c_str()); vector cmnts = it->second[i]->GetComments(); for ( size_t j = 0; j < cmnts.size(); ++j ) - fprintf(file.f, " %s\n", cmnts[j].c_str()); + fprintf(file.f, " %s\n", cmnts[j].c_str()); fprintf(file.f, "\n"); } @@ -211,25 +228,82 @@ void ScriptTarget::DoFindDependencies(const vector& docs) if ( script_deps.empty() ) reporter->FatalError("No match for Broxygen target '%s' pattern '%s'", Name().c_str(), Pattern().c_str()); + + if ( ! IsDir() ) + return; + + for ( size_t i = 0; i < script_deps.size(); ++i ) + { + if ( SafeBasename(script_deps[i]->Name()).result == PACKAGE_LOADER ) + { + string pkg_dir = SafeDirname(script_deps[i]->Name()).result; + string target_file = Name() + pkg_dir + "/index.rst"; + Target* t = PackageTarget::Instantiate(target_file, pkg_dir); + t->FindDependencies(docs); + pkg_deps.push_back(t); + } + } + } + +vector dir_contents_recursive(string dir) + { + vector rval; + struct stat st; + + if ( stat(dir.c_str(), &st) < 0 && errno == ENOENT ) + return rval; + + while ( dir[dir.size() - 1] == '/' ) + dir.erase(dir.size() - 1, 1); + + char* dir_copy = copy_string(dir.c_str()); + char** scan_path = new char*[2]; + scan_path[0] = dir_copy; + scan_path[1] = 0; + + FTS* fts = fts_open(scan_path, FTS_NOCHDIR, 0); + + if ( ! fts ) + { + reporter->Error("fts_open failure: %s", strerror(errno)); + delete [] scan_path; + delete [] dir_copy; + return rval; + } + + FTSENT* n; + + while ( (n = fts_read(fts)) ) + { + if ( n->fts_info & FTS_F ) + rval.push_back(n->fts_path); + } + + if ( errno ) + reporter->Error("fts_read failure: %s", strerror(errno)); + + if ( fts_close(fts) < 0 ) + reporter->Error("fts_close failure: %s", strerror(errno)); + + delete [] scan_path; + delete [] dir_copy; + return rval; } void ScriptTarget::DoGenerate() const { - if ( Name()[Name().size() - 1] == '/' ) + if ( IsDir() ) { // Target name is a dir, matching scripts are written within that dir // with a dir tree that parallels the script's BROPATH location. + set targets; + vector dir_contents = dir_contents_recursive(Name()); + for ( size_t i = 0; i < script_deps.size(); ++i ) { - string target_filename = Name() + script_deps[i]->Name(); - size_t pos = target_filename.rfind(".bro"); - - if ( pos == target_filename.size() - 4 ) - target_filename.replace(pos, 4, ".rst"); - else - target_filename += ".rst"; - + string target_filename = Name() + script_deps[i]->Name() + ".rst"; + targets.insert(target_filename); vector dep; dep.push_back(script_deps[i]); @@ -241,9 +315,26 @@ void ScriptTarget::DoGenerate() const fprintf(file.f, "%s\n", script_deps[i]->ReStructuredText().c_str()); } - // TODO: could possibly take inventory of files in the dir beforehand, - // track all files written, then compare afterwards in order to remove - // stale files. + for ( size_t i = 0; i < pkg_deps.size(); ++i ) + { + targets.insert(pkg_deps[i]->Name()); + pkg_deps[i]->Generate(); + } + + for ( size_t i = 0; i < dir_contents.size(); ++i ) + { + string f = dir_contents[i]; + + if ( targets.find(f) != targets.end() ) + continue; + + if ( unlink(f.c_str()) < 0 ) + reporter->Warning("Failed to unlink %s: %s", f.c_str(), + strerror(errno)); + + DBG_LOG(DBG_BROXYGEN, "Delete stale script file %s", f.c_str()); + } + return; } @@ -300,7 +391,8 @@ void ScriptIndexTarget::DoGenerate() const if ( ! d ) continue; - fprintf(file.f, " %s <%s>\n", d->Name().c_str(), d->Name().c_str()); + fprintf(file.f, " %s \n", d->Name().c_str(), + d->Name().c_str()); } } diff --git a/src/broxygen/Configuration.h b/src/broxygen/Configuration.h index 6b5c45ed45..6ea9f640be 100644 --- a/src/broxygen/Configuration.h +++ b/src/broxygen/Configuration.h @@ -146,6 +146,9 @@ public: const std::string& pattern) { return new ScriptTarget(name, pattern); } + ~ScriptTarget() + { for ( size_t i = 0; i < pkg_deps.size(); ++i ) delete pkg_deps[i]; } + protected: ScriptTarget(const std::string& name, const std::string& pattern) @@ -159,6 +162,11 @@ private: void DoFindDependencies(const std::vector& docs); void DoGenerate() const; + + bool IsDir() const + { return Name()[Name().size() - 1] == '/'; } + + std::vector pkg_deps; }; class ScriptSummaryTarget : public ScriptTarget { diff --git a/src/broxygen/Document.cc b/src/broxygen/Document.cc index 2619755b35..a44a8ecbdc 100644 --- a/src/broxygen/Document.cc +++ b/src/broxygen/Document.cc @@ -116,7 +116,7 @@ PackageDocument::PackageDocument(const string& arg_name) string PackageDocument::DoReStructuredText(bool roles_only) const { - string rval = fmt(":doc:`%s <%s/index>`\n\n", pkg_name.c_str(), + string rval = fmt(":doc:`%s `\n\n", pkg_name.c_str(), pkg_name.c_str()); for ( size_t i = 0; i < readme.size(); ++i ) @@ -629,15 +629,20 @@ string ScriptDocument::DoReStructuredText(bool roles_only) const if ( it != dependencies.begin() ) rval += ", "; - rval += fmt(":doc:`%s `", it->c_str(), it->c_str()); - // TODO linking to packages is a bit different? + string path = find_file(*it, bro_path(), "bro"); + string doc = *it; + + if ( ! path.empty() && is_dir(path.c_str()) ) + // Reference the package. + doc += "/index"; + + rval += fmt(":doc:`%s `", it->c_str(), doc.c_str()); } rval += "\n"; } - // TODO: make this an absolute path? - rval += fmt(":Source File: :download:`%s`\n", name.c_str()); + rval += fmt(":Source File: :download:`/scripts/%s`\n", name.c_str()); rval += "\n"; rval += make_heading("Summary", '~'); rval += make_summary("Options", '#', '=', options); diff --git a/src/const.bif b/src/const.bif index ea84b3363d..a45dfb0a7a 100644 --- a/src/const.bif +++ b/src/const.bif @@ -1,6 +1,6 @@ ##! Declaration of various scripting-layer constants that the Bro core uses ##! internally. Documentation and default values for the scripting-layer -##! variables themselves are found in :doc:`/scripts/base/init-bare`. +##! variables themselves are found in :doc:`/scripts/base/init-bare.bro`. const ignore_keep_alive_rexmit: bool; const skip_http_data: bool; diff --git a/src/event.bif b/src/event.bif index ddadb47f8a..79cf46dd8a 100644 --- a/src/event.bif +++ b/src/event.bif @@ -360,7 +360,7 @@ event content_gap%(c: connection, is_orig: bool, seq: count, length: count%); ## ## .. note:: ## -## Bro comes with a script :doc:`/scripts/policy/misc/capture-loss` that uses +## Bro comes with a script :doc:`/scripts/policy/misc/capture-loss.bro` that uses ## this event to estimate packet loss and report when a predefined threshold ## is exceeded. event gap_report%(dt: interval, info: gap_info%); diff --git a/src/reporter.bif b/src/reporter.bif index c0f83205c3..3cdefc4da4 100644 --- a/src/reporter.bif +++ b/src/reporter.bif @@ -4,7 +4,7 @@ ##! If event handlers do exist, it's assumed they take care of determining ##! how/where to output the messages. ##! -##! See :doc:`/scripts/base/frameworks/reporter/main` for a convenient +##! See :doc:`/scripts/base/frameworks/reporter/main.bro` for a convenient ##! reporter message logging framework. module Reporter; diff --git a/testing/btest/Makefile b/testing/btest/Makefile index 47451fbf27..261c7262b0 100644 --- a/testing/btest/Makefile +++ b/testing/btest/Makefile @@ -21,12 +21,9 @@ cleanup: @rm -f $(DIAG) @rm -f .tmp/script-coverage* -update-doc-sources: - ../../doc/scripts/genDocSourcesList.sh ../../doc/scripts/DocSourcesList.cmake - # Updates the three coverage tests that usually need tweaking when # scripts get added/removed. -update-coverage-tests: update-doc-sources +update-coverage-tests: btest -qU coverage.bare-load-baseline btest -qU coverage.default-load-baseline @echo "Use 'git diff' to check updates look right." diff --git a/testing/btest/coverage/doc.test b/testing/btest/coverage/doc.test deleted file mode 100644 index 074e397d88..0000000000 --- a/testing/btest/coverage/doc.test +++ /dev/null @@ -1,8 +0,0 @@ -# This tests that we're generating bro script documentation for all the -# available bro scripts. If this fails, then the genDocSources.sh needs -# to be run to produce a new DocSourcesList.cmake or genDocSources.sh needs -# to be updated to blacklist undesired scripts. To update, run -# "make update-doc-sources" -# -# @TEST-EXEC: $DIST/doc/scripts/genDocSourcesList.sh -# @TEST-EXEC: cmp $DIST/doc/scripts/DocSourcesList.cmake ./DocSourcesList.cmake