Fixes related to make doc handling of script summary text (##! comments)

- Summary comments (##!) can now be placed at the beginning of
BiF files (but still outside C segments).  An issue was fixed where
these comments would mistakenly be transferred into the generated
.func_def file and cause a compile error. I completely removed writing
any opt_ws value into the .func_def file because it was currently not
writing anything besides whitespace.

- The generation of reST for the collecting of "groups" of policy
script documentation now happens at build time of `make doc` through the
use of a helper script rather than doing this at configure time so that
changes to summary text will always be reflected in the documentation.

doc/scripts/CMakeLists.txt

@@ -1,4 +1,5 @@
 set(POLICY_SRC_DIR ${PROJECT_SOURCE_DIR}/policy)
+set(BIF_SRC_DIR ${PROJECT_SOURCE_DIR}/src)
 set(RST_OUTPUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/rest_output)
 set(DOC_OUTPUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/out)
 set(DOC_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/source)
@@ -38,16 +39,22 @@ endif ()
 # ALL_REST_OUTPUTS: a running list (the CMake list type) of all reST docs
 #   that are to be generated
 # MASTER_GROUP_LIST: a running list (the CMake list type) of all script groups
-# ${group}_TEXT: a running list of policy script :doc: references and summary
+# ${group}_files: a running list of files belonging to a given group, from
-#   text for a given group
+#   which summary text can be extracted at build time
+# ${group}_doc_names: a running list of reST style document names that can be
+#   given to a :doc: role, shared indices with ${group}_files
 #
 macro(REST_TARGET srcDir broInput)
     get_filename_component(basename ${broInput} NAME_WE)
     get_filename_component(extension ${broInput} EXT)
     get_filename_component(relDstDir ${broInput} PATH)
 
+    set(sumTextSrc ${srcDir}/${broInput})
     if (${extension} STREQUAL ".bif.bro")
         set(basename "${basename}.bif")
+        # the summary text is taken at configure time, but .bif.bro files
+        # may not have been generated yet, so read .bif file instead
+        set(sumTextSrc ${BIF_SRC_DIR}/${basename})
     elseif (${extension} STREQUAL ".init")
         set(basename "${basename}.init")
     endif ()
@@ -81,13 +88,8 @@ macro(REST_TARGET srcDir broInput)
             endif ()
         endif ()
 
-        # add script's summary documentation to text associated with the group
+        list(APPEND ${group}_files ${sumTextSrc})
-        set(${group}_TEXT "${${group}_TEXT}\n\n:doc:`/policy/${docName}`\n")
+        list(APPEND ${group}_doc_names ${docName})
-        file(STRINGS ${srcDir}/${broInput} summary_text REGEX "^\#\#!")
-        foreach (line ${summary_text})
-            string(REGEX REPLACE "^\#\#!" "   " line ${line})
-            set(${group}_TEXT "${${group}_TEXT}\n${line}")
-        endforeach ()
     else ()
         set(group "")
     endif ()
@@ -169,10 +171,20 @@ file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/tmp_policy_index
 file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/group_list
     "${MASTER_GROUP_LIST_TEXT}")
 
-# create temporary file containing the summary text for all scripts in a group
+# create temporary files containing list of each source file in a given group
 foreach (group ${MASTER_GROUP_LIST})
-    file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/${group}_text
+    if (EXISTS ${CMAKE_CURRENT_BINARY_DIR}/${group}_files)
-        "${${group}_TEXT}")
+        file(REMOVE ${CMAKE_CURRENT_BINARY_DIR}/${group}_files)
+    endif ()
+    if (EXISTS ${CMAKE_CURRENT_BINARY_DIR}/${group}_doc_names)
+        file(REMOVE ${CMAKE_CURRENT_BINARY_DIR}/${group}_doc_names)
+    endif ()
+    foreach (src ${${group}_files})
+        file(APPEND ${CMAKE_CURRENT_BINARY_DIR}/${group}_files "${src}\n")
+    endforeach ()
+    foreach (dname ${${group}_doc_names})
+        file(APPEND ${CMAKE_CURRENT_BINARY_DIR}/${group}_doc_names "${dname}\n")
+    endforeach ()
 endforeach ()
 
 # remove previously generated docs no longer scheduled for generation
@@ -219,10 +231,11 @@ add_custom_target(doc
                   # append to the master index of all policy scripts
                   COMMAND cat ${CMAKE_CURRENT_BINARY_DIR}/tmp_policy_index >>
                           ${DOC_SOURCE_WORKDIR}/policy/index.rst
-                  # construct the reST file for all groups
+                  # construct a reST file for each group
-                  COMMAND xargs -I{} sh -c 'cat "$$1_text" >>
+                  COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/group_index_generator.py
-                          "${DOC_SOURCE_WORKDIR}/$$1.rst"' -- {} <
                           ${CMAKE_CURRENT_BINARY_DIR}/group_list
+                          ${CMAKE_CURRENT_BINARY_DIR}
+                          ${DOC_SOURCE_WORKDIR}
                   # tell sphinx to generate html
                   COMMAND sphinx-build
                           -b html

doc/scripts/group_index_generator.py

@@ -0,0 +1,52 @@
+#! /usr/bin/env python
+
+# This script automatically generates a reST documents that lists
+# a collection of Bro policy scripts that are "grouped" together.
+# The summary text (##! comments) of the policy script is embedded
+# in the list.
+#
+# 1st argument is the file containing list of groups
+# 2nd argument is the directory containing ${group}_files lists of
+#   scripts that belong to the group and ${group}_doc_names lists of
+#   document names that can be supplied to a reST :doc: role
+# 3rd argument is a directory in which write a ${group}.rst file (will
+#   append to existing file) that contains reST style references to
+#   script docs along with summary text contained in original script
+
+import sys
+import os
+import string
+
+group_list = sys.argv[1]
+file_manifest_dir = sys.argv[2]
+output_dir = sys.argv[3]
+
+with open(group_list, 'r') as f_group_list:
+    for group in f_group_list.read().splitlines():
+        #print group
+        file_manifest = os.path.join(file_manifest_dir, group + "_files")
+        doc_manifest = os.path.join(file_manifest_dir, group + "_doc_names")
+        src_files = []
+        doc_names = []
+
+        with open(file_manifest, 'r') as f_file_manifest:
+            src_files = f_file_manifest.read().splitlines()
+
+        with open(doc_manifest, 'r') as f_doc_manifest:
+            doc_names = f_doc_manifest.read().splitlines()
+
+        for i in range(len(src_files)):
+            src_file = src_files[i]
+            #print "\t" + src_file
+            summary_comments = []
+            with open(src_file, 'r') as f_src_file:
+                for line in f_src_file:
+                    sum_pos = string.find(line, "##!")
+                    if sum_pos != -1:
+                        summary_comments.append(line[(sum_pos+3):])
+            #print summary_comments
+            group_file = os.path.join(output_dir, group + ".rst")
+            with open(group_file, 'a') as f_group_file:
+                f_group_file.write("\n:doc:`/policy/%s`\n" % doc_names[i])
+                for line in summary_comments:
+                    f_group_file.write("   " + line)

src/builtin-func.y

@@ -319,8 +319,8 @@ definitions:	definitions definition opt_ws
 			fprintf(fp_netvar_h, "// %s\n\n", auto_gen_comment);
 			fprintf(fp_netvar_init, "// %s\n\n", auto_gen_comment);
 
+			fprintf(fp_bro_init, "%s", $1);
 			fprintf(fp_bro_init, "export {\n");
-			fprintf(fp_func_def, "%s", $1);
 			}
 	;