From c472931eb9a201141c4aa28ba14628d28233a077 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Wed, 20 Apr 2011 20:09:33 -0500 Subject: [PATCH 01/12] Fixing example.bro's auto-reST generation baseline test. Adds a diff canonifier that skips diffing the places where example.bro may use MutableVal derivatives (e.g. sets/tables), which don't always generate the same ordering in the reST docs across runs. --- .../Baseline/doc.autogen-reST-example/example.rst | 2 +- .../btest/Scripts/doc/example-diff-canonifier.py | 15 +++++++++++++++ testing/btest/btest.cfg | 1 + testing/btest/doc/autogen-reST-example | 2 +- 4 files changed, 18 insertions(+), 2 deletions(-) create mode 100755 testing/btest/Scripts/doc/example-diff-canonifier.py diff --git a/testing/btest/Baseline/doc.autogen-reST-example/example.rst b/testing/btest/Baseline/doc.autogen-reST-example/example.rst index eb125eda23..8e735b787e 100644 --- a/testing/btest/Baseline/doc.autogen-reST-example/example.rst +++ b/testing/btest/Baseline/doc.autogen-reST-example/example.rst @@ -13,7 +13,7 @@ these comments are transferred directly into the auto-generated `reStructuredText `_ (reST) document's summary section. -.. tip:: You can embed directives and roles within ``##``-stylized comments +.. tip:: You can embed directives and roles within ``##``-stylized comments. :Author: Jon Siwek diff --git a/testing/btest/Scripts/doc/example-diff-canonifier.py b/testing/btest/Scripts/doc/example-diff-canonifier.py new file mode 100755 index 0000000000..e0b8c110cc --- /dev/null +++ b/testing/btest/Scripts/doc/example-diff-canonifier.py @@ -0,0 +1,15 @@ +#!/usr/bin/python + +import sys +import re + +# MutableVal derivatives (e.g. sets/tables) don't always generate the same +# ordering in the reST documentation, so just don't bother diffing +# the places where example.bro uses them. + +RE1 = "\d*/tcp" +RE2 = "tcp port \d*" + +for line in sys.stdin.readlines(): + if re.search(RE1, line) is None and re.search(RE2, line) is None: + print line diff --git a/testing/btest/btest.cfg b/testing/btest/btest.cfg index 52f7e6280a..0eee6883ef 100644 --- a/testing/btest/btest.cfg +++ b/testing/btest/btest.cfg @@ -12,5 +12,6 @@ BRO_SEED_FILE=%(testbase)s/random.seed PATH=%(testbase)s/../../build/src:%(testbase)s/../../aux/btest:%(default_path)s TEST_DIFF_CANONIFIER=%(testbase)s/Scripts/diff-canonifier TRACES=%(testbase)s/Traces +SCRIPTS=%(testbase)s/Scripts DIST=%(testbase)s/../.. BUILD=%(testbase)s/../../build diff --git a/testing/btest/doc/autogen-reST-example b/testing/btest/doc/autogen-reST-example index 7870259cad..c47a604373 100644 --- a/testing/btest/doc/autogen-reST-example +++ b/testing/btest/doc/autogen-reST-example @@ -1,2 +1,2 @@ @TEST-EXEC: bro --doc-scripts $DIST/doc/example.bro -@TEST-EXEC: btest-diff example.rst +@TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/doc/example-diff-canonifier.py btest-diff example.rst From 4634d92394ee1e887da151d8dcb35f7c28f56eb8 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Wed, 20 Apr 2011 21:11:32 -0500 Subject: [PATCH 02/12] Move stuff related to policy script documentation from doc/ to doc/scripts/ --- doc/CMakeLists.txt | 43 +----------------- doc/README | 45 +------------------ doc/scripts/CMakeLists.txt | 42 +++++++++++++++++ doc/scripts/README | 44 ++++++++++++++++++ doc/{ => scripts}/conf.py.in | 0 doc/{ => scripts}/example.bro | 0 doc/scripts/{ => scripts}/BroToReST.py.in | 0 .../{ => scripts}/generate_reST_docs.py.in | 2 +- doc/{ => scripts}/source/_static/showhide.js | 0 .../source/_templates/layout.html | 0 doc/{ => scripts}/source/builtins.rst | 0 doc/{ => scripts}/source/common.rst | 0 doc/{ => scripts}/source/ext/bro.py | 0 doc/{ => scripts}/source/index.rst | 0 doc/{ => scripts}/source/policy/bifs.rst | 0 doc/{ => scripts}/source/policy/default.rst | 0 doc/{ => scripts}/source/policy/index.rst | 0 doc/{ => scripts}/source/policy/internal.rst | 0 doc/{ => scripts}/source/policy/user.rst | 0 testing/btest/doc/autogen-reST-example | 2 +- 20 files changed, 90 insertions(+), 88 deletions(-) create mode 100644 doc/scripts/CMakeLists.txt create mode 100644 doc/scripts/README rename doc/{ => scripts}/conf.py.in (100%) rename doc/{ => scripts}/example.bro (100%) rename doc/scripts/{ => scripts}/BroToReST.py.in (100%) rename doc/scripts/{ => scripts}/generate_reST_docs.py.in (95%) rename doc/{ => scripts}/source/_static/showhide.js (100%) rename doc/{ => scripts}/source/_templates/layout.html (100%) rename doc/{ => scripts}/source/builtins.rst (100%) rename doc/{ => scripts}/source/common.rst (100%) rename doc/{ => scripts}/source/ext/bro.py (100%) rename doc/{ => scripts}/source/index.rst (100%) rename doc/{ => scripts}/source/policy/bifs.rst (100%) rename doc/{ => scripts}/source/policy/default.rst (100%) rename doc/{ => scripts}/source/policy/index.rst (100%) rename doc/{ => scripts}/source/policy/internal.rst (100%) rename doc/{ => scripts}/source/policy/user.rst (100%) diff --git a/doc/CMakeLists.txt b/doc/CMakeLists.txt index 00e0974919..e2c3f25f4e 100644 --- a/doc/CMakeLists.txt +++ b/doc/CMakeLists.txt @@ -1,42 +1 @@ -set(POLICY_SRC_DIR ${PROJECT_SOURCE_DIR}/policy) -set(DOC_OUTPUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/out) -set(DOC_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/source) -set(DOC_SOURCE_WORKDIR ${CMAKE_CURRENT_BINARY_DIR}/source) - -file(GLOB_RECURSE DOC_SOURCES FOLLOW_SYMLINKS "*") - -configure_file(${CMAKE_CURRENT_SOURCE_DIR}/conf.py.in - ${CMAKE_CURRENT_BINARY_DIR}/conf.py - @ONLY) -configure_file(${CMAKE_CURRENT_SOURCE_DIR}/scripts/generate_reST_docs.py.in - ${CMAKE_CURRENT_BINARY_DIR}/generate_reST_docs.py - @ONLY) -configure_file(${CMAKE_CURRENT_SOURCE_DIR}/scripts/BroToReST.py.in - ${CMAKE_CURRENT_BINARY_DIR}/BroToReST.py - @ONLY) - -add_custom_target(doc - COMMAND "${CMAKE_COMMAND}" -E copy_directory - ${DOC_SOURCE_DIR} - ${DOC_SOURCE_WORKDIR} - COMMAND python generate_reST_docs.py - COMMAND sphinx-build - -b html - -c ${CMAKE_CURRENT_BINARY_DIR} - -d ${DOC_OUTPUT_DIR}/doctrees - ${DOC_SOURCE_WORKDIR} - ${DOC_OUTPUT_DIR}/html - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} - COMMENT "[Sphinx] Generating Script Documentation" - VERBATIM - # SOURCES just adds stuff to IDE projects as a convienience - SOURCES ${DOC_SOURCES}) - -add_dependencies(doc bro doc-clean) - -add_custom_target(doc-clean - COMMAND "${CMAKE_COMMAND}" -E remove_directory - ${CMAKE_CURRENT_BINARY_DIR}/source - COMMAND "${CMAKE_COMMAND}" -E remove_directory - ${DOC_OUTPUT_DIR} - VERBATIM) +add_subdirectory(scripts) diff --git a/doc/README b/doc/README index 150260ed09..1333ed77b7 100644 --- a/doc/README +++ b/doc/README @@ -1,44 +1 @@ -This directory contains scripts and templates that can be used to automate -the generation of Bro script documentation. Two build targets are defined -by CMake: - -``make doc`` - - This target depends on a Python interpreter (>=2.5) and - `Sphinx `_ being installed. Sphinx can be - installed like:: - - > sudo easy_install sphinx - - This target will also first build the bro binary if it is not already - since the generation of reStructuredText (reST) documentation from - Bro scripts is integrated within the parsing process. - - After completion, HTML documentation can be located inside the CMake - ``build/`` directory as ``build/doc/out/html``. The generated reST - documentation will be located in ``build/doc/source/policy``. - -``make doc-clean`` - - This target removes Sphinx inputs and outputs from the CMake ``build/`` dir. - -To schedule a script to be documented, edit ``scripts/generate_reST_docs.py.in`` -and try adding the name of the script along with an optional script group to -the ``docs`` dictionary. That python script also shows other, more specialized -methods for generating documentation for some types of corner-cases. - -When adding a new logical grouping for generated scripts, create a new -reST document in ``source/policy/.rst`` and add some default -documentation. References to (and summaries of) documents associated with -the group get appended to this file during the ``make doc`` process. - -The Sphinx source tree template in ``source/`` can be modified to add more -common/general documentation, style sheets, JavaScript, etc. The Sphinx -config file is produced from ``conf.py.in``, so that can be edited to change -various Sphinx options, like setting the default HTML rendering theme. -There is also a custom Sphinx domain implemented in ``source/ext/bro.py`` -which adds some reST directives and roles that aid in generating useful -index entries and cross-references. - -See ``example.bro`` for an example of how to document a Bro script such that -``make doc`` will be able to produce reST/HTML documentation for it. +TODO diff --git a/doc/scripts/CMakeLists.txt b/doc/scripts/CMakeLists.txt new file mode 100644 index 0000000000..00e0974919 --- /dev/null +++ b/doc/scripts/CMakeLists.txt @@ -0,0 +1,42 @@ +set(POLICY_SRC_DIR ${PROJECT_SOURCE_DIR}/policy) +set(DOC_OUTPUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/out) +set(DOC_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/source) +set(DOC_SOURCE_WORKDIR ${CMAKE_CURRENT_BINARY_DIR}/source) + +file(GLOB_RECURSE DOC_SOURCES FOLLOW_SYMLINKS "*") + +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/conf.py.in + ${CMAKE_CURRENT_BINARY_DIR}/conf.py + @ONLY) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/scripts/generate_reST_docs.py.in + ${CMAKE_CURRENT_BINARY_DIR}/generate_reST_docs.py + @ONLY) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/scripts/BroToReST.py.in + ${CMAKE_CURRENT_BINARY_DIR}/BroToReST.py + @ONLY) + +add_custom_target(doc + COMMAND "${CMAKE_COMMAND}" -E copy_directory + ${DOC_SOURCE_DIR} + ${DOC_SOURCE_WORKDIR} + COMMAND python generate_reST_docs.py + COMMAND sphinx-build + -b html + -c ${CMAKE_CURRENT_BINARY_DIR} + -d ${DOC_OUTPUT_DIR}/doctrees + ${DOC_SOURCE_WORKDIR} + ${DOC_OUTPUT_DIR}/html + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + COMMENT "[Sphinx] Generating Script Documentation" + VERBATIM + # SOURCES just adds stuff to IDE projects as a convienience + SOURCES ${DOC_SOURCES}) + +add_dependencies(doc bro doc-clean) + +add_custom_target(doc-clean + COMMAND "${CMAKE_COMMAND}" -E remove_directory + ${CMAKE_CURRENT_BINARY_DIR}/source + COMMAND "${CMAKE_COMMAND}" -E remove_directory + ${DOC_OUTPUT_DIR} + VERBATIM) diff --git a/doc/scripts/README b/doc/scripts/README new file mode 100644 index 0000000000..b7114ec262 --- /dev/null +++ b/doc/scripts/README @@ -0,0 +1,44 @@ +This directory contains scripts and templates that can be used to automate +the generation of Bro script documentation. Two build targets are defined +by CMake: + +``make doc`` + + This target depends on a Python interpreter (>=2.5) and + `Sphinx `_ being installed. Sphinx can be + installed like:: + + > sudo easy_install sphinx + + This target will also first build the bro binary if it is not already + since the generation of reStructuredText (reST) documentation from + Bro scripts is integrated within the parsing process. + + After completion, HTML documentation can be located inside the CMake + ``build/`` directory as ``doc/scripts/out/html``. The generated + reST documentation will be located in ``doc/scripts/source/policy``. + +``make doc-clean`` + + This target removes Sphinx inputs and outputs from the CMake ``build/`` dir. + +To schedule a script to be documented, edit ``scripts/generate_reST_docs.py.in`` +and try adding the name of the script along with an optional script group to +the ``docs`` dictionary. That python script also shows other, more specialized +methods for generating documentation for some types of corner-cases. + +When adding a new logical grouping for generated scripts, create a new +reST document in ``source/policy/.rst`` and add some default +documentation. References to (and summaries of) documents associated with +the group get appended to this file during the ``make doc`` process. + +The Sphinx source tree template in ``source/`` can be modified to add more +common/general documentation, style sheets, JavaScript, etc. The Sphinx +config file is produced from ``conf.py.in``, so that can be edited to change +various Sphinx options, like setting the default HTML rendering theme. +There is also a custom Sphinx domain implemented in ``source/ext/bro.py`` +which adds some reST directives and roles that aid in generating useful +index entries and cross-references. + +See ``example.bro`` for an example of how to document a Bro script such that +``make doc`` will be able to produce reST/HTML documentation for it. diff --git a/doc/conf.py.in b/doc/scripts/conf.py.in similarity index 100% rename from doc/conf.py.in rename to doc/scripts/conf.py.in diff --git a/doc/example.bro b/doc/scripts/example.bro similarity index 100% rename from doc/example.bro rename to doc/scripts/example.bro diff --git a/doc/scripts/BroToReST.py.in b/doc/scripts/scripts/BroToReST.py.in similarity index 100% rename from doc/scripts/BroToReST.py.in rename to doc/scripts/scripts/BroToReST.py.in diff --git a/doc/scripts/generate_reST_docs.py.in b/doc/scripts/scripts/generate_reST_docs.py.in similarity index 95% rename from doc/scripts/generate_reST_docs.py.in rename to doc/scripts/scripts/generate_reST_docs.py.in index 3278c42367..c54146314f 100755 --- a/doc/scripts/generate_reST_docs.py.in +++ b/doc/scripts/scripts/generate_reST_docs.py.in @@ -44,7 +44,7 @@ GenDocs(stdin_docs, True) # The example documentation script doesn't live on the BROPATH, so # explicitly generate the docs for it like this: -BroToReST("example.bro", False, ["@PROJECT_SOURCE_DIR@/doc"], group="internal").GenDoc() +BroToReST("example.bro", False, ["@PROJECT_SOURCE_DIR@/doc/scripts"], group="internal").GenDoc() # Generate documentation for stuff that's always loaded into bro by default: cmd = "echo '' | %s %s" % (BRO, BRO_ARGS) diff --git a/doc/source/_static/showhide.js b/doc/scripts/source/_static/showhide.js similarity index 100% rename from doc/source/_static/showhide.js rename to doc/scripts/source/_static/showhide.js diff --git a/doc/source/_templates/layout.html b/doc/scripts/source/_templates/layout.html similarity index 100% rename from doc/source/_templates/layout.html rename to doc/scripts/source/_templates/layout.html diff --git a/doc/source/builtins.rst b/doc/scripts/source/builtins.rst similarity index 100% rename from doc/source/builtins.rst rename to doc/scripts/source/builtins.rst diff --git a/doc/source/common.rst b/doc/scripts/source/common.rst similarity index 100% rename from doc/source/common.rst rename to doc/scripts/source/common.rst diff --git a/doc/source/ext/bro.py b/doc/scripts/source/ext/bro.py similarity index 100% rename from doc/source/ext/bro.py rename to doc/scripts/source/ext/bro.py diff --git a/doc/source/index.rst b/doc/scripts/source/index.rst similarity index 100% rename from doc/source/index.rst rename to doc/scripts/source/index.rst diff --git a/doc/source/policy/bifs.rst b/doc/scripts/source/policy/bifs.rst similarity index 100% rename from doc/source/policy/bifs.rst rename to doc/scripts/source/policy/bifs.rst diff --git a/doc/source/policy/default.rst b/doc/scripts/source/policy/default.rst similarity index 100% rename from doc/source/policy/default.rst rename to doc/scripts/source/policy/default.rst diff --git a/doc/source/policy/index.rst b/doc/scripts/source/policy/index.rst similarity index 100% rename from doc/source/policy/index.rst rename to doc/scripts/source/policy/index.rst diff --git a/doc/source/policy/internal.rst b/doc/scripts/source/policy/internal.rst similarity index 100% rename from doc/source/policy/internal.rst rename to doc/scripts/source/policy/internal.rst diff --git a/doc/source/policy/user.rst b/doc/scripts/source/policy/user.rst similarity index 100% rename from doc/source/policy/user.rst rename to doc/scripts/source/policy/user.rst diff --git a/testing/btest/doc/autogen-reST-example b/testing/btest/doc/autogen-reST-example index c47a604373..7f55384ce6 100644 --- a/testing/btest/doc/autogen-reST-example +++ b/testing/btest/doc/autogen-reST-example @@ -1,2 +1,2 @@ -@TEST-EXEC: bro --doc-scripts $DIST/doc/example.bro +@TEST-EXEC: bro --doc-scripts $DIST/doc/scripts/example.bro @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/doc/example-diff-canonifier.py btest-diff example.rst From 17314fa144c11a0012c2af35e17dc49c9994edb2 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Thu, 21 Apr 2011 13:34:37 -0500 Subject: [PATCH 03/12] Add parser error hint when in doc mode about checking ## comment syntax. --- src/parse.y | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/parse.y b/src/parse.y index 8e8a75cad7..debdf02177 100644 --- a/src/parse.y +++ b/src/parse.y @@ -1622,7 +1622,7 @@ opt_doc_list: int yyerror(const char msg[]) { - char* msgbuf = new char[strlen(msg) + strlen(last_tok) + 64]; + char* msgbuf = new char[strlen(msg) + strlen(last_tok) + 128]; if ( last_tok[0] == '\n' ) sprintf(msgbuf, "%s, on previous line", msg); @@ -1631,6 +1631,10 @@ int yyerror(const char msg[]) else sprintf(msgbuf, "%s, at or near \"%s\"", msg, last_tok); + if ( generate_documentation ) + strcat(msgbuf, "\nDocumentation mode is enabled: " + "remember to check syntax of ## style comments\n"); + error(msgbuf); return 0; From f10d2e10eaee43fc760684980d95f6e8b94ff068 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Tue, 26 Apr 2011 22:13:04 -0500 Subject: [PATCH 04/12] Overhaul of "doc" build target for generating policy script documentation. It's now all implemented in CMake scripting. The generation of reST docs is now a distinct target, "restdoc", while the target to generate HTML docs, "doc", depends on "restdoc". reST doc generation supports incremental builds (documentation for a given policy script is only regenerated when it is out of date), but HTML doc generation via ``make doc`` is not incremental (Sphinx always starts with fresh input). Building the "restdoc" target is now covered by a btest to ensure all policy scripts are parse-able when Bro is in "doc mode". Generated reST docs should now support "@load"ing from subdirectories. e.g. "@load foo/baz" and "@load bar/baz" will now generate the right xref links. --- doc/scripts/CMakeLists.txt | 236 ++++++++++++++++-- doc/scripts/README | 49 ++-- doc/scripts/scripts/BroToReST.py.in | 152 ----------- doc/scripts/scripts/generate_reST_docs.py.in | 71 ------ doc/scripts/source/{policy => }/bifs.rst | 0 doc/scripts/source/{policy => }/default.rst | 0 doc/scripts/source/index.rst | 8 +- doc/scripts/source/{policy => }/internal.rst | 0 doc/scripts/source/policy/index.rst | 4 - doc/scripts/source/{policy => }/user.rst | 0 src/BroDoc.cc | 17 +- .../doc.autogen-reST-example/example.rst | 14 +- testing/btest/doc/autogen-reST-all | 1 + testing/btest/doc/autogen-reST-example | 2 +- 14 files changed, 276 insertions(+), 278 deletions(-) delete mode 100755 doc/scripts/scripts/BroToReST.py.in delete mode 100755 doc/scripts/scripts/generate_reST_docs.py.in rename doc/scripts/source/{policy => }/bifs.rst (100%) rename doc/scripts/source/{policy => }/default.rst (100%) rename doc/scripts/source/{policy => }/internal.rst (100%) rename doc/scripts/source/{policy => }/user.rst (100%) create mode 100644 testing/btest/doc/autogen-reST-all diff --git a/doc/scripts/CMakeLists.txt b/doc/scripts/CMakeLists.txt index 00e0974919..acf9541fae 100644 --- a/doc/scripts/CMakeLists.txt +++ b/doc/scripts/CMakeLists.txt @@ -1,42 +1,250 @@ set(POLICY_SRC_DIR ${PROJECT_SOURCE_DIR}/policy) +set(RST_OUTPUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/rest_output) set(DOC_OUTPUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/out) set(DOC_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/source) set(DOC_SOURCE_WORKDIR ${CMAKE_CURRENT_BINARY_DIR}/source) file(GLOB_RECURSE DOC_SOURCES FOLLOW_SYMLINKS "*") +# configure the Sphinx config file (expand variables CMake might know about) configure_file(${CMAKE_CURRENT_SOURCE_DIR}/conf.py.in ${CMAKE_CURRENT_BINARY_DIR}/conf.py @ONLY) -configure_file(${CMAKE_CURRENT_SOURCE_DIR}/scripts/generate_reST_docs.py.in - ${CMAKE_CURRENT_BINARY_DIR}/generate_reST_docs.py - @ONLY) -configure_file(${CMAKE_CURRENT_SOURCE_DIR}/scripts/BroToReST.py.in - ${CMAKE_CURRENT_BINARY_DIR}/BroToReST.py - @ONLY) +# find out what BROPATH to use when executing bro +execute_process(COMMAND ${CMAKE_BINARY_DIR}/bro-path-dev + OUTPUT_VARIABLE BROPATH + RESULT_VARIABLE retval + OUTPUT_STRIP_TRAILING_WHITESPACE) +if (NOT ${retval} EQUAL 0) + message(FATAL_ERROR "Problem setting BROPATH") +endif () + +# This macro is used to add a new makefile target for reST policy script +# documentation that can be generated using Bro itself to parse policy scripts. +# It's called like: +# +# rest_target(srcDir broInput [group]) +# +# srcDir: the directory which contains broInput +# broInput: the file name of a bro policy script +# group: optional name of group that the script documentation will belong to +# +# In addition to adding the makefile target, several CMake variables are set: +# +# MASTER_POLICY_INDEX_TEXT: a running list of policy scripts docs that have +# been generated so far, formatted such that it can be appended to a file +# that ends in a Sphinx toctree directive +# ALL_REST_OUTPUTS: a running list (the CMake list type) of all reST docs +# that are to be generated +# MASTER_GROUP_LIST: a running list (the CMake list type) of all script groups +# ${group}_TEXT: a running list of policy script :doc: references and summary +# text for a given group +# +macro(REST_TARGET srcDir broInput) + get_filename_component(basename ${broInput} NAME_WE) + get_filename_component(extension ${broInput} EXT) + get_filename_component(relDstDir ${broInput} PATH) + + if (${extension} STREQUAL ".bif.bro") + set(basename "${basename}.bif") + elseif (${extension} STREQUAL ".init") + set(basename "${basename}.init") + endif () + + set (restFile "${basename}.rst") + + if (NOT relDstDir) + set(docName "${basename}") + set(dstDir "${RST_OUTPUT_DIR}") + else () + set(docName "${relDstDir}/${basename}") + set(dstDir "${RST_OUTPUT_DIR}/${relDstDir}") + endif () + + set(restOutput "${dstDir}/${restFile}") + + set(indexEntry " ${docName} <${docName}>") + set(MASTER_POLICY_INDEX_TEXT "${MASTER_POLICY_INDEX_TEXT}\n${indexEntry}") + list(APPEND ALL_REST_OUTPUTS ${restOutput}) + + if (NOT "${ARGN}" STREQUAL "") + set(group ${ARGN}) + # add group to master group list if not already in it + list(FIND MASTER_GROUP_LIST ${group} _found) + if (_found EQUAL -1) + list(APPEND MASTER_GROUP_LIST ${group}) + if (MASTER_GROUP_LIST_TEXT) + set(MASTER_GROUP_LIST_TEXT "${MASTER_GROUP_LIST_TEXT}\n${group}") + else () + set(MASTER_GROUP_LIST_TEXT "${group}") + endif () + endif () + + # add script's summary documentation to text associated with the group + set(${group}_TEXT "${${group}_TEXT}\n\n:doc:`/policy/${docName}`\n") + file(STRINGS ${srcDir}/${broInput} summary_text REGEX "^\#\#!") + foreach (line ${summary_text}) + string(REGEX REPLACE "^\#\#!" " " line ${line}) + set(${group}_TEXT "${${group}_TEXT}\n${line}") + endforeach () + else () + set(group "") + endif () + + if (${group} STREQUAL "default" OR ${group} STREQUAL "bifs") + set(BRO_ARGS --doc-scripts --exec '') + else () + set(BRO_ARGS --doc-scripts ${srcDir}/${broInput}) + endif () + + add_custom_command(OUTPUT ${restOutput} + # delete any leftover state from previous bro runs + COMMAND "${CMAKE_COMMAND}" + ARGS -E remove_directory .state + # generate the reST documentation using bro + COMMAND BROPATH=${BROPATH} ${CMAKE_BINARY_DIR}/src/bro + ARGS ${BRO_ARGS} || (rm -rf .state *.log *.rst && exit 1) + # move generated doc into a new directory tree that + # defines the final structure of documents + COMMAND "${CMAKE_COMMAND}" + ARGS -E make_directory ${dstDir} + COMMAND "${CMAKE_COMMAND}" + ARGS -E copy ${restFile} ${restOutput} + # copy the bro policy script, too + COMMAND "${CMAKE_COMMAND}" + ARGS -E copy ${srcDir}/${broInput} ${dstDir} + # clean up the build directory + COMMAND rm + ARGS -rf .state *.log *.rst + DEPENDS bro + DEPENDS ${srcDir}/${broInput} + COMMENT "[Bro] Generating reST docs for ${broInput}" + ) + +endmacro(REST_TARGET) + +# Schedule Bro scripts for which to generate documentation. +# Note: the script may be located in a subdirectory off of one of the main +# directories in BROPATH. In that case, just list the script as 'foo/bar.bro' +rest_target(${POLICY_SRC_DIR} alarm.bro user) +rest_target(${POLICY_SRC_DIR} arp.bro user) +rest_target(${POLICY_SRC_DIR} conn.bro user) +rest_target(${POLICY_SRC_DIR} dhcp.bro user) +rest_target(${POLICY_SRC_DIR} dns.bro user) +rest_target(${POLICY_SRC_DIR} ftp.bro user) +rest_target(${POLICY_SRC_DIR} http.bro user) +rest_target(${POLICY_SRC_DIR} http-reply.bro user) +rest_target(${POLICY_SRC_DIR} http-request.bro user) +rest_target(${POLICY_SRC_DIR} irc.bro user) +rest_target(${POLICY_SRC_DIR} smtp.bro user) +rest_target(${POLICY_SRC_DIR} ssl.bro user) +rest_target(${POLICY_SRC_DIR} ssl-ciphers.bro user) +rest_target(${POLICY_SRC_DIR} ssl-errors.bro user) +rest_target(${POLICY_SRC_DIR} synflood.bro user) +rest_target(${POLICY_SRC_DIR} tcp.bro user) +rest_target(${POLICY_SRC_DIR} udp.bro user) +rest_target(${POLICY_SRC_DIR} weird.bro user) +rest_target(${CMAKE_CURRENT_SOURCE_DIR} example.bro internal) + +# Finding out what scripts bro will generate documentation for by default +# can be done like: `bro --doc-scripts --exec ""` +rest_target(${POLICY_SRC_DIR} bro.init default) +rest_target(${POLICY_SRC_DIR} logging-ascii.bro default) +rest_target(${POLICY_SRC_DIR} logging.bro default) +rest_target(${POLICY_SRC_DIR} pcap.bro default) +rest_target(${POLICY_SRC_DIR} server-ports.bro default) +rest_target(${CMAKE_BINARY_DIR}/src bro.bif.bro bifs) +rest_target(${CMAKE_BINARY_DIR}/src const.bif.bro bifs) +rest_target(${CMAKE_BINARY_DIR}/src event.bif.bro bifs) +rest_target(${CMAKE_BINARY_DIR}/src logging.bif.bro bifs) +rest_target(${CMAKE_BINARY_DIR}/src strings.bif.bro bifs) +rest_target(${CMAKE_BINARY_DIR}/src types.bif.bro bifs) + +# create temporary list of all docs to include in the master policy/index file +file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/tmp_policy_index + "${MASTER_POLICY_INDEX_TEXT}") + +# create temporary file containing list of all groups +file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/group_list + "${MASTER_GROUP_LIST_TEXT}") + +# create temporary file containing the summary text for all scripts in a group +foreach (group ${MASTER_GROUP_LIST}) + file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/${group}_text + "${${group}_TEXT}") +endforeach () + +# remove previously generated docs no longer scheduled for generation +if (EXISTS ${RST_OUTPUT_DIR}) + file(GLOB_RECURSE EXISTING_REST_DOCS "${RST_OUTPUT_DIR}/*.rst") + foreach (_doc ${EXISTING_REST_DOCS}) + list(FIND ALL_REST_OUTPUTS ${_doc} _found) + if (_found EQUAL -1) + file(REMOVE ${_doc}) + message(STATUS "Removing stale reST doc: ${_doc}") + endif () + endforeach () +endif () + +# The "restdoc" target uses Bro to parse policy scripts in order to +# generate reST documentation from them. +add_custom_target(restdoc + # create symlink to the reST output directory for convenience + COMMAND "${CMAKE_COMMAND}" -E create_symlink + ${RST_OUTPUT_DIR} + ${CMAKE_BINARY_DIR}/reST + DEPENDS ${ALL_REST_OUTPUTS}) + +# The "restclean" target removes all generated reST documentation from the +# build directory. +add_custom_target(restclean + COMMAND "${CMAKE_COMMAND}" -E remove_directory + ${RST_OUTPUT_DIR} + VERBATIM) + +# The "doc" target generates reST documentation for any outdated bro scripts +# and then uses Sphinx to generate HTML documentation from the reST add_custom_target(doc + # copy the template documentation to the build directory + # to give as input for sphinx COMMAND "${CMAKE_COMMAND}" -E copy_directory ${DOC_SOURCE_DIR} ${DOC_SOURCE_WORKDIR} - COMMAND python generate_reST_docs.py + # copy generated policy script documentation into the + # working copy of the template documentation + COMMAND "${CMAKE_COMMAND}" -E copy_directory + ${RST_OUTPUT_DIR} + ${DOC_SOURCE_WORKDIR}/policy + # append to the master index of all policy scripts + COMMAND cat ${CMAKE_CURRENT_BINARY_DIR}/tmp_policy_index >> + ${DOC_SOURCE_WORKDIR}/policy/index.rst + # construct the reST file for all groups + COMMAND xargs -I{} sh -c 'cat "$$1_text" >> + "${DOC_SOURCE_WORKDIR}/$$1.rst"' -- {} < + ${CMAKE_CURRENT_BINARY_DIR}/group_list + # tell sphinx to generate html COMMAND sphinx-build -b html -c ${CMAKE_CURRENT_BINARY_DIR} -d ${DOC_OUTPUT_DIR}/doctrees ${DOC_SOURCE_WORKDIR} ${DOC_OUTPUT_DIR}/html + # create symlink to the html output directory for convenience + COMMAND "${CMAKE_COMMAND}" -E create_symlink + ${DOC_OUTPUT_DIR}/html + ${CMAKE_BINARY_DIR}/html WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} - COMMENT "[Sphinx] Generating Script Documentation" - VERBATIM + COMMENT "[Sphinx] Generating HTML policy script docs" # SOURCES just adds stuff to IDE projects as a convienience - SOURCES ${DOC_SOURCES}) + SOURCES ${DOC_SOURCES} + DEPENDS restdoc docclean) -add_dependencies(doc bro doc-clean) - -add_custom_target(doc-clean +# The "docclean" target removes just the Sphinx input/output directories +# from the build directory. +add_custom_target(docclean COMMAND "${CMAKE_COMMAND}" -E remove_directory - ${CMAKE_CURRENT_BINARY_DIR}/source + ${DOC_SOURCE_WORKDIR} COMMAND "${CMAKE_COMMAND}" -E remove_directory ${DOC_OUTPUT_DIR} VERBATIM) diff --git a/doc/scripts/README b/doc/scripts/README index b7114ec262..85496322d9 100644 --- a/doc/scripts/README +++ b/doc/scripts/README @@ -1,8 +1,21 @@ This directory contains scripts and templates that can be used to automate -the generation of Bro script documentation. Two build targets are defined +the generation of Bro script documentation. Several build targets are defined by CMake: -``make doc`` +``restdoc`` + + This target uses Bro to parse policy scripts in order to generate + reStructuredText (reST) documentation from them. The list of scripts + for which to generate reST documentation is defined in the + ``CMakeLists.txt`` file in this directory. Script documentation is + rebuild automatically if the policy script from which it is derived + or the Bro binary becomes out of date + + The resulting output from this target can be found in the CMake + ``build/`` directory inside ``reST`` (a symlink to + ``doc/scripts/rest_output``). + +``doc`` This target depends on a Python interpreter (>=2.5) and `Sphinx `_ being installed. Sphinx can be @@ -10,27 +23,31 @@ by CMake: > sudo easy_install sphinx - This target will also first build the bro binary if it is not already - since the generation of reStructuredText (reST) documentation from - Bro scripts is integrated within the parsing process. + This target will first build ``restdoc`` target and then copy the + resulting reST files as an input directory to Sphinx. - After completion, HTML documentation can be located inside the CMake - ``build/`` directory as ``doc/scripts/out/html``. The generated - reST documentation will be located in ``doc/scripts/source/policy``. + After completion, HTML documentation can be located in the CMake + ``build/`` directory inside ``html`` (a symlink to + ``doc/scripts/out/html``) -``make doc-clean`` +``restclean`` + + This target removes any reST documentation that has been generated so far. + +``docclean`` This target removes Sphinx inputs and outputs from the CMake ``build/`` dir. -To schedule a script to be documented, edit ``scripts/generate_reST_docs.py.in`` -and try adding the name of the script along with an optional script group to -the ``docs`` dictionary. That python script also shows other, more specialized -methods for generating documentation for some types of corner-cases. +To schedule a script to be documented, edit ``CMakeLists.txt`` inside this +directory add a call to the ``rest_target()`` macro. Calling that macro +with a group name for the script is optional, but if not given, the only +link to the script will be in the master TOC tree for all policy scripts. When adding a new logical grouping for generated scripts, create a new -reST document in ``source/policy/.rst`` and add some default -documentation. References to (and summaries of) documents associated with -the group get appended to this file during the ``make doc`` process. +reST document in ``source/.rst`` and add some default +documentation for the group. References to (and summaries of) documents +associated with the group get appended to this file during the +``make doc`` process. The Sphinx source tree template in ``source/`` can be modified to add more common/general documentation, style sheets, JavaScript, etc. The Sphinx diff --git a/doc/scripts/scripts/BroToReST.py.in b/doc/scripts/scripts/BroToReST.py.in deleted file mode 100755 index 89a8e1cc57..0000000000 --- a/doc/scripts/scripts/BroToReST.py.in +++ /dev/null @@ -1,152 +0,0 @@ -#! /usr/bin/env python - -import os -import subprocess -import shutil -import glob -import string -import sys - -BRO = "@CMAKE_BINARY_DIR@/src/bro" -BROPATHDEV = "`@CMAKE_BINARY_DIR@/bro-path-dev`" -BRO_ARGS = "--doc-scripts" -DOC_DST_DIR = "@DOC_SOURCE_WORKDIR@/policy" -BROPATH = subprocess.Popen("@CMAKE_BINARY_DIR@/bro-path-dev", shell=True, - stdout=subprocess.PIPE, stderr=subprocess.STDOUT).stdout.readline() - -class BroToReST: - """A class to encapsulate the the generation of reST documentation from - a given Bro script. - """ - - bro_src_file = None - doc_src_file = None - load_via_stdin = False - group = None - - def __init__(self, src_file, load_method=False, search_dir=None, group=None): - """ - :param src_file: the file name of a Bro script (not a path) - :param load_method: T if script must be loaded by Bro via a stdin - redirection of "@load