diff --git a/doc/scripts/CMakeLists.txt b/doc/scripts/CMakeLists.txt index 00e0974919..acf9541fae 100644 --- a/doc/scripts/CMakeLists.txt +++ b/doc/scripts/CMakeLists.txt @@ -1,42 +1,250 @@ set(POLICY_SRC_DIR ${PROJECT_SOURCE_DIR}/policy) +set(RST_OUTPUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/rest_output) set(DOC_OUTPUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/out) set(DOC_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/source) set(DOC_SOURCE_WORKDIR ${CMAKE_CURRENT_BINARY_DIR}/source) file(GLOB_RECURSE DOC_SOURCES FOLLOW_SYMLINKS "*") +# configure the Sphinx config file (expand variables CMake might know about) configure_file(${CMAKE_CURRENT_SOURCE_DIR}/conf.py.in ${CMAKE_CURRENT_BINARY_DIR}/conf.py @ONLY) -configure_file(${CMAKE_CURRENT_SOURCE_DIR}/scripts/generate_reST_docs.py.in - ${CMAKE_CURRENT_BINARY_DIR}/generate_reST_docs.py - @ONLY) -configure_file(${CMAKE_CURRENT_SOURCE_DIR}/scripts/BroToReST.py.in - ${CMAKE_CURRENT_BINARY_DIR}/BroToReST.py - @ONLY) +# find out what BROPATH to use when executing bro +execute_process(COMMAND ${CMAKE_BINARY_DIR}/bro-path-dev + OUTPUT_VARIABLE BROPATH + RESULT_VARIABLE retval + OUTPUT_STRIP_TRAILING_WHITESPACE) +if (NOT ${retval} EQUAL 0) + message(FATAL_ERROR "Problem setting BROPATH") +endif () + +# This macro is used to add a new makefile target for reST policy script +# documentation that can be generated using Bro itself to parse policy scripts. +# It's called like: +# +# rest_target(srcDir broInput [group]) +# +# srcDir: the directory which contains broInput +# broInput: the file name of a bro policy script +# group: optional name of group that the script documentation will belong to +# +# In addition to adding the makefile target, several CMake variables are set: +# +# MASTER_POLICY_INDEX_TEXT: a running list of policy scripts docs that have +# been generated so far, formatted such that it can be appended to a file +# that ends in a Sphinx toctree directive +# ALL_REST_OUTPUTS: a running list (the CMake list type) of all reST docs +# that are to be generated +# MASTER_GROUP_LIST: a running list (the CMake list type) of all script groups +# ${group}_TEXT: a running list of policy script :doc: references and summary +# text for a given group +# +macro(REST_TARGET srcDir broInput) + get_filename_component(basename ${broInput} NAME_WE) + get_filename_component(extension ${broInput} EXT) + get_filename_component(relDstDir ${broInput} PATH) + + if (${extension} STREQUAL ".bif.bro") + set(basename "${basename}.bif") + elseif (${extension} STREQUAL ".init") + set(basename "${basename}.init") + endif () + + set (restFile "${basename}.rst") + + if (NOT relDstDir) + set(docName "${basename}") + set(dstDir "${RST_OUTPUT_DIR}") + else () + set(docName "${relDstDir}/${basename}") + set(dstDir "${RST_OUTPUT_DIR}/${relDstDir}") + endif () + + set(restOutput "${dstDir}/${restFile}") + + set(indexEntry " ${docName} <${docName}>") + set(MASTER_POLICY_INDEX_TEXT "${MASTER_POLICY_INDEX_TEXT}\n${indexEntry}") + list(APPEND ALL_REST_OUTPUTS ${restOutput}) + + if (NOT "${ARGN}" STREQUAL "") + set(group ${ARGN}) + # add group to master group list if not already in it + list(FIND MASTER_GROUP_LIST ${group} _found) + if (_found EQUAL -1) + list(APPEND MASTER_GROUP_LIST ${group}) + if (MASTER_GROUP_LIST_TEXT) + set(MASTER_GROUP_LIST_TEXT "${MASTER_GROUP_LIST_TEXT}\n${group}") + else () + set(MASTER_GROUP_LIST_TEXT "${group}") + endif () + endif () + + # add script's summary documentation to text associated with the group + set(${group}_TEXT "${${group}_TEXT}\n\n:doc:`/policy/${docName}`\n") + file(STRINGS ${srcDir}/${broInput} summary_text REGEX "^\#\#!") + foreach (line ${summary_text}) + string(REGEX REPLACE "^\#\#!" " " line ${line}) + set(${group}_TEXT "${${group}_TEXT}\n${line}") + endforeach () + else () + set(group "") + endif () + + if (${group} STREQUAL "default" OR ${group} STREQUAL "bifs") + set(BRO_ARGS --doc-scripts --exec '') + else () + set(BRO_ARGS --doc-scripts ${srcDir}/${broInput}) + endif () + + add_custom_command(OUTPUT ${restOutput} + # delete any leftover state from previous bro runs + COMMAND "${CMAKE_COMMAND}" + ARGS -E remove_directory .state + # generate the reST documentation using bro + COMMAND BROPATH=${BROPATH} ${CMAKE_BINARY_DIR}/src/bro + ARGS ${BRO_ARGS} || (rm -rf .state *.log *.rst && exit 1) + # move generated doc into a new directory tree that + # defines the final structure of documents + COMMAND "${CMAKE_COMMAND}" + ARGS -E make_directory ${dstDir} + COMMAND "${CMAKE_COMMAND}" + ARGS -E copy ${restFile} ${restOutput} + # copy the bro policy script, too + COMMAND "${CMAKE_COMMAND}" + ARGS -E copy ${srcDir}/${broInput} ${dstDir} + # clean up the build directory + COMMAND rm + ARGS -rf .state *.log *.rst + DEPENDS bro + DEPENDS ${srcDir}/${broInput} + COMMENT "[Bro] Generating reST docs for ${broInput}" + ) + +endmacro(REST_TARGET) + +# Schedule Bro scripts for which to generate documentation. +# Note: the script may be located in a subdirectory off of one of the main +# directories in BROPATH. In that case, just list the script as 'foo/bar.bro' +rest_target(${POLICY_SRC_DIR} alarm.bro user) +rest_target(${POLICY_SRC_DIR} arp.bro user) +rest_target(${POLICY_SRC_DIR} conn.bro user) +rest_target(${POLICY_SRC_DIR} dhcp.bro user) +rest_target(${POLICY_SRC_DIR} dns.bro user) +rest_target(${POLICY_SRC_DIR} ftp.bro user) +rest_target(${POLICY_SRC_DIR} http.bro user) +rest_target(${POLICY_SRC_DIR} http-reply.bro user) +rest_target(${POLICY_SRC_DIR} http-request.bro user) +rest_target(${POLICY_SRC_DIR} irc.bro user) +rest_target(${POLICY_SRC_DIR} smtp.bro user) +rest_target(${POLICY_SRC_DIR} ssl.bro user) +rest_target(${POLICY_SRC_DIR} ssl-ciphers.bro user) +rest_target(${POLICY_SRC_DIR} ssl-errors.bro user) +rest_target(${POLICY_SRC_DIR} synflood.bro user) +rest_target(${POLICY_SRC_DIR} tcp.bro user) +rest_target(${POLICY_SRC_DIR} udp.bro user) +rest_target(${POLICY_SRC_DIR} weird.bro user) +rest_target(${CMAKE_CURRENT_SOURCE_DIR} example.bro internal) + +# Finding out what scripts bro will generate documentation for by default +# can be done like: `bro --doc-scripts --exec ""` +rest_target(${POLICY_SRC_DIR} bro.init default) +rest_target(${POLICY_SRC_DIR} logging-ascii.bro default) +rest_target(${POLICY_SRC_DIR} logging.bro default) +rest_target(${POLICY_SRC_DIR} pcap.bro default) +rest_target(${POLICY_SRC_DIR} server-ports.bro default) +rest_target(${CMAKE_BINARY_DIR}/src bro.bif.bro bifs) +rest_target(${CMAKE_BINARY_DIR}/src const.bif.bro bifs) +rest_target(${CMAKE_BINARY_DIR}/src event.bif.bro bifs) +rest_target(${CMAKE_BINARY_DIR}/src logging.bif.bro bifs) +rest_target(${CMAKE_BINARY_DIR}/src strings.bif.bro bifs) +rest_target(${CMAKE_BINARY_DIR}/src types.bif.bro bifs) + +# create temporary list of all docs to include in the master policy/index file +file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/tmp_policy_index + "${MASTER_POLICY_INDEX_TEXT}") + +# create temporary file containing list of all groups +file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/group_list + "${MASTER_GROUP_LIST_TEXT}") + +# create temporary file containing the summary text for all scripts in a group +foreach (group ${MASTER_GROUP_LIST}) + file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/${group}_text + "${${group}_TEXT}") +endforeach () + +# remove previously generated docs no longer scheduled for generation +if (EXISTS ${RST_OUTPUT_DIR}) + file(GLOB_RECURSE EXISTING_REST_DOCS "${RST_OUTPUT_DIR}/*.rst") + foreach (_doc ${EXISTING_REST_DOCS}) + list(FIND ALL_REST_OUTPUTS ${_doc} _found) + if (_found EQUAL -1) + file(REMOVE ${_doc}) + message(STATUS "Removing stale reST doc: ${_doc}") + endif () + endforeach () +endif () + +# The "restdoc" target uses Bro to parse policy scripts in order to +# generate reST documentation from them. +add_custom_target(restdoc + # create symlink to the reST output directory for convenience + COMMAND "${CMAKE_COMMAND}" -E create_symlink + ${RST_OUTPUT_DIR} + ${CMAKE_BINARY_DIR}/reST + DEPENDS ${ALL_REST_OUTPUTS}) + +# The "restclean" target removes all generated reST documentation from the +# build directory. +add_custom_target(restclean + COMMAND "${CMAKE_COMMAND}" -E remove_directory + ${RST_OUTPUT_DIR} + VERBATIM) + +# The "doc" target generates reST documentation for any outdated bro scripts +# and then uses Sphinx to generate HTML documentation from the reST add_custom_target(doc + # copy the template documentation to the build directory + # to give as input for sphinx COMMAND "${CMAKE_COMMAND}" -E copy_directory ${DOC_SOURCE_DIR} ${DOC_SOURCE_WORKDIR} - COMMAND python generate_reST_docs.py + # copy generated policy script documentation into the + # working copy of the template documentation + COMMAND "${CMAKE_COMMAND}" -E copy_directory + ${RST_OUTPUT_DIR} + ${DOC_SOURCE_WORKDIR}/policy + # append to the master index of all policy scripts + COMMAND cat ${CMAKE_CURRENT_BINARY_DIR}/tmp_policy_index >> + ${DOC_SOURCE_WORKDIR}/policy/index.rst + # construct the reST file for all groups + COMMAND xargs -I{} sh -c 'cat "$$1_text" >> + "${DOC_SOURCE_WORKDIR}/$$1.rst"' -- {} < + ${CMAKE_CURRENT_BINARY_DIR}/group_list + # tell sphinx to generate html COMMAND sphinx-build -b html -c ${CMAKE_CURRENT_BINARY_DIR} -d ${DOC_OUTPUT_DIR}/doctrees ${DOC_SOURCE_WORKDIR} ${DOC_OUTPUT_DIR}/html + # create symlink to the html output directory for convenience + COMMAND "${CMAKE_COMMAND}" -E create_symlink + ${DOC_OUTPUT_DIR}/html + ${CMAKE_BINARY_DIR}/html WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} - COMMENT "[Sphinx] Generating Script Documentation" - VERBATIM + COMMENT "[Sphinx] Generating HTML policy script docs" # SOURCES just adds stuff to IDE projects as a convienience - SOURCES ${DOC_SOURCES}) + SOURCES ${DOC_SOURCES} + DEPENDS restdoc docclean) -add_dependencies(doc bro doc-clean) - -add_custom_target(doc-clean +# The "docclean" target removes just the Sphinx input/output directories +# from the build directory. +add_custom_target(docclean COMMAND "${CMAKE_COMMAND}" -E remove_directory - ${CMAKE_CURRENT_BINARY_DIR}/source + ${DOC_SOURCE_WORKDIR} COMMAND "${CMAKE_COMMAND}" -E remove_directory ${DOC_OUTPUT_DIR} VERBATIM) diff --git a/doc/scripts/README b/doc/scripts/README index b7114ec262..85496322d9 100644 --- a/doc/scripts/README +++ b/doc/scripts/README @@ -1,8 +1,21 @@ This directory contains scripts and templates that can be used to automate -the generation of Bro script documentation. Two build targets are defined +the generation of Bro script documentation. Several build targets are defined by CMake: -``make doc`` +``restdoc`` + + This target uses Bro to parse policy scripts in order to generate + reStructuredText (reST) documentation from them. The list of scripts + for which to generate reST documentation is defined in the + ``CMakeLists.txt`` file in this directory. Script documentation is + rebuild automatically if the policy script from which it is derived + or the Bro binary becomes out of date + + The resulting output from this target can be found in the CMake + ``build/`` directory inside ``reST`` (a symlink to + ``doc/scripts/rest_output``). + +``doc`` This target depends on a Python interpreter (>=2.5) and `Sphinx `_ being installed. Sphinx can be @@ -10,27 +23,31 @@ by CMake: > sudo easy_install sphinx - This target will also first build the bro binary if it is not already - since the generation of reStructuredText (reST) documentation from - Bro scripts is integrated within the parsing process. + This target will first build ``restdoc`` target and then copy the + resulting reST files as an input directory to Sphinx. - After completion, HTML documentation can be located inside the CMake - ``build/`` directory as ``doc/scripts/out/html``. The generated - reST documentation will be located in ``doc/scripts/source/policy``. + After completion, HTML documentation can be located in the CMake + ``build/`` directory inside ``html`` (a symlink to + ``doc/scripts/out/html``) -``make doc-clean`` +``restclean`` + + This target removes any reST documentation that has been generated so far. + +``docclean`` This target removes Sphinx inputs and outputs from the CMake ``build/`` dir. -To schedule a script to be documented, edit ``scripts/generate_reST_docs.py.in`` -and try adding the name of the script along with an optional script group to -the ``docs`` dictionary. That python script also shows other, more specialized -methods for generating documentation for some types of corner-cases. +To schedule a script to be documented, edit ``CMakeLists.txt`` inside this +directory add a call to the ``rest_target()`` macro. Calling that macro +with a group name for the script is optional, but if not given, the only +link to the script will be in the master TOC tree for all policy scripts. When adding a new logical grouping for generated scripts, create a new -reST document in ``source/policy/.rst`` and add some default -documentation. References to (and summaries of) documents associated with -the group get appended to this file during the ``make doc`` process. +reST document in ``source/.rst`` and add some default +documentation for the group. References to (and summaries of) documents +associated with the group get appended to this file during the +``make doc`` process. The Sphinx source tree template in ``source/`` can be modified to add more common/general documentation, style sheets, JavaScript, etc. The Sphinx diff --git a/doc/scripts/scripts/BroToReST.py.in b/doc/scripts/scripts/BroToReST.py.in deleted file mode 100755 index 89a8e1cc57..0000000000 --- a/doc/scripts/scripts/BroToReST.py.in +++ /dev/null @@ -1,152 +0,0 @@ -#! /usr/bin/env python - -import os -import subprocess -import shutil -import glob -import string -import sys - -BRO = "@CMAKE_BINARY_DIR@/src/bro" -BROPATHDEV = "`@CMAKE_BINARY_DIR@/bro-path-dev`" -BRO_ARGS = "--doc-scripts" -DOC_DST_DIR = "@DOC_SOURCE_WORKDIR@/policy" -BROPATH = subprocess.Popen("@CMAKE_BINARY_DIR@/bro-path-dev", shell=True, - stdout=subprocess.PIPE, stderr=subprocess.STDOUT).stdout.readline() - -class BroToReST: - """A class to encapsulate the the generation of reST documentation from - a given Bro script. - """ - - bro_src_file = None - doc_src_file = None - load_via_stdin = False - group = None - - def __init__(self, src_file, load_method=False, search_dir=None, group=None): - """ - :param src_file: the file name of a Bro script (not a path) - :param load_method: T if script must be loaded by Bro via a stdin - redirection of "@load