From ceaba8077b34b0c0c406eeae26ba16cf461ea16d Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Mon, 2 May 2011 15:34:34 -0500 Subject: [PATCH] Fixes related to `make doc` handling of script summary text (##! comments) - Summary comments (##!) can now be placed at the beginning of BiF files (but still outside C segments). An issue was fixed where these comments would mistakenly be transferred into the generated .func_def file and cause a compile error. I completely removed writing any opt_ws value into the .func_def file because it was currently not writing anything besides whitespace. - The generation of reST for the collecting of "groups" of policy script documentation now happens at build time of `make doc` through the use of a helper script rather than doing this at configure time so that changes to summary text will always be reflected in the documentation. --- doc/scripts/CMakeLists.txt | 43 +++++++++++++++-------- doc/scripts/group_index_generator.py | 52 ++++++++++++++++++++++++++++ src/builtin-func.y | 2 +- 3 files changed, 81 insertions(+), 16 deletions(-) create mode 100755 doc/scripts/group_index_generator.py diff --git a/doc/scripts/CMakeLists.txt b/doc/scripts/CMakeLists.txt index acf9541fae..88818bb1c4 100644 --- a/doc/scripts/CMakeLists.txt +++ b/doc/scripts/CMakeLists.txt @@ -1,4 +1,5 @@ set(POLICY_SRC_DIR ${PROJECT_SOURCE_DIR}/policy) +set(BIF_SRC_DIR ${PROJECT_SOURCE_DIR}/src) set(RST_OUTPUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/rest_output) set(DOC_OUTPUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/out) set(DOC_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/source) @@ -38,16 +39,22 @@ endif () # ALL_REST_OUTPUTS: a running list (the CMake list type) of all reST docs # that are to be generated # MASTER_GROUP_LIST: a running list (the CMake list type) of all script groups -# ${group}_TEXT: a running list of policy script :doc: references and summary -# text for a given group +# ${group}_files: a running list of files belonging to a given group, from +# which summary text can be extracted at build time +# ${group}_doc_names: a running list of reST style document names that can be +# given to a :doc: role, shared indices with ${group}_files # macro(REST_TARGET srcDir broInput) get_filename_component(basename ${broInput} NAME_WE) get_filename_component(extension ${broInput} EXT) get_filename_component(relDstDir ${broInput} PATH) + set(sumTextSrc ${srcDir}/${broInput}) if (${extension} STREQUAL ".bif.bro") set(basename "${basename}.bif") + # the summary text is taken at configure time, but .bif.bro files + # may not have been generated yet, so read .bif file instead + set(sumTextSrc ${BIF_SRC_DIR}/${basename}) elseif (${extension} STREQUAL ".init") set(basename "${basename}.init") endif () @@ -81,13 +88,8 @@ macro(REST_TARGET srcDir broInput) endif () endif () - # add script's summary documentation to text associated with the group - set(${group}_TEXT "${${group}_TEXT}\n\n:doc:`/policy/${docName}`\n") - file(STRINGS ${srcDir}/${broInput} summary_text REGEX "^\#\#!") - foreach (line ${summary_text}) - string(REGEX REPLACE "^\#\#!" " " line ${line}) - set(${group}_TEXT "${${group}_TEXT}\n${line}") - endforeach () + list(APPEND ${group}_files ${sumTextSrc}) + list(APPEND ${group}_doc_names ${docName}) else () set(group "") endif () @@ -169,10 +171,20 @@ file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/tmp_policy_index file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/group_list "${MASTER_GROUP_LIST_TEXT}") -# create temporary file containing the summary text for all scripts in a group +# create temporary files containing list of each source file in a given group foreach (group ${MASTER_GROUP_LIST}) - file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/${group}_text - "${${group}_TEXT}") + if (EXISTS ${CMAKE_CURRENT_BINARY_DIR}/${group}_files) + file(REMOVE ${CMAKE_CURRENT_BINARY_DIR}/${group}_files) + endif () + if (EXISTS ${CMAKE_CURRENT_BINARY_DIR}/${group}_doc_names) + file(REMOVE ${CMAKE_CURRENT_BINARY_DIR}/${group}_doc_names) + endif () + foreach (src ${${group}_files}) + file(APPEND ${CMAKE_CURRENT_BINARY_DIR}/${group}_files "${src}\n") + endforeach () + foreach (dname ${${group}_doc_names}) + file(APPEND ${CMAKE_CURRENT_BINARY_DIR}/${group}_doc_names "${dname}\n") + endforeach () endforeach () # remove previously generated docs no longer scheduled for generation @@ -219,10 +231,11 @@ add_custom_target(doc # append to the master index of all policy scripts COMMAND cat ${CMAKE_CURRENT_BINARY_DIR}/tmp_policy_index >> ${DOC_SOURCE_WORKDIR}/policy/index.rst - # construct the reST file for all groups - COMMAND xargs -I{} sh -c 'cat "$$1_text" >> - "${DOC_SOURCE_WORKDIR}/$$1.rst"' -- {} < + # construct a reST file for each group + COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/group_index_generator.py ${CMAKE_CURRENT_BINARY_DIR}/group_list + ${CMAKE_CURRENT_BINARY_DIR} + ${DOC_SOURCE_WORKDIR} # tell sphinx to generate html COMMAND sphinx-build -b html diff --git a/doc/scripts/group_index_generator.py b/doc/scripts/group_index_generator.py new file mode 100755 index 0000000000..d143765ef9 --- /dev/null +++ b/doc/scripts/group_index_generator.py @@ -0,0 +1,52 @@ +#! /usr/bin/env python + +# This script automatically generates a reST documents that lists +# a collection of Bro policy scripts that are "grouped" together. +# The summary text (##! comments) of the policy script is embedded +# in the list. +# +# 1st argument is the file containing list of groups +# 2nd argument is the directory containing ${group}_files lists of +# scripts that belong to the group and ${group}_doc_names lists of +# document names that can be supplied to a reST :doc: role +# 3rd argument is a directory in which write a ${group}.rst file (will +# append to existing file) that contains reST style references to +# script docs along with summary text contained in original script + +import sys +import os +import string + +group_list = sys.argv[1] +file_manifest_dir = sys.argv[2] +output_dir = sys.argv[3] + +with open(group_list, 'r') as f_group_list: + for group in f_group_list.read().splitlines(): + #print group + file_manifest = os.path.join(file_manifest_dir, group + "_files") + doc_manifest = os.path.join(file_manifest_dir, group + "_doc_names") + src_files = [] + doc_names = [] + + with open(file_manifest, 'r') as f_file_manifest: + src_files = f_file_manifest.read().splitlines() + + with open(doc_manifest, 'r') as f_doc_manifest: + doc_names = f_doc_manifest.read().splitlines() + + for i in range(len(src_files)): + src_file = src_files[i] + #print "\t" + src_file + summary_comments = [] + with open(src_file, 'r') as f_src_file: + for line in f_src_file: + sum_pos = string.find(line, "##!") + if sum_pos != -1: + summary_comments.append(line[(sum_pos+3):]) + #print summary_comments + group_file = os.path.join(output_dir, group + ".rst") + with open(group_file, 'a') as f_group_file: + f_group_file.write("\n:doc:`/policy/%s`\n" % doc_names[i]) + for line in summary_comments: + f_group_file.write(" " + line) diff --git a/src/builtin-func.y b/src/builtin-func.y index a35d0d9612..4f97135ab5 100644 --- a/src/builtin-func.y +++ b/src/builtin-func.y @@ -319,8 +319,8 @@ definitions: definitions definition opt_ws fprintf(fp_netvar_h, "// %s\n\n", auto_gen_comment); fprintf(fp_netvar_init, "// %s\n\n", auto_gen_comment); + fprintf(fp_bro_init, "%s", $1); fprintf(fp_bro_init, "export {\n"); - fprintf(fp_func_def, "%s", $1); } ;