Mirror/zeek
1
0
Fork 0
mirror of https://github.com/zeek/zeek.git synced 2025-10-02 06:38:20 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Improving documention for the Bro script document-generation process

Browse source 
Some minor organizational revisions to the python scripting.
This commit is contained in:
Jon Siwek 2011-04-06 16:36:14 -05:00
parent f3b1a6bb9e
commit b8f6c5bc7d
5 changed files with 213 additions and 97 deletions
Download patch file Download diff file Expand all files Collapse all files

7
Makefile
View file

@ -23,6 +23,13 @@ install: configured
clean: configured
( cd $(BUILD) && make clean )
( cd $(BUILD) && make doc-clean )
doc: configured
( cd $(BUILD) && make doc )
doc-clean: configured
( cd $(BUILD) && make doc-clean )
dist: cmake_version
# Minimum Bro source package

4
doc/CMakeLists.txt
View file

@ -11,6 +11,9 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/conf.py.in
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/scripts/generate_reST_docs.py.in
${CMAKE_CURRENT_BINARY_DIR}/generate_reST_docs.py
@ONLY)
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/scripts/BroToReST.py.in
${CMAKE_CURRENT_BINARY_DIR}/BroToReST.py
@ONLY)
add_custom_target(doc
COMMAND "${CMAKE_COMMAND}" -E copy_directory
@ -30,7 +33,6 @@ add_custom_target(doc
SOURCES ${DOC_SOURCES})
add_dependencies(doc bro)
# TODO: add dependency that's a check for `python`, `sphinx-build`, etc.
add_custom_target(doc-clean
COMMAND "${CMAKE_COMMAND}" -E remove_directory

45
doc/README
View file

@ -1 +1,44 @@
TODO.
This directory contains scripts and templates that can be used to automate
the generation of Bro script documentation. Two build targets are defined
by CMake:
``make doc``
This target depends on a Python interpreter (>=2.5) and
`Sphinx <http://sphinx.pocoo.org/>`_ being installed. Sphinx can be
installed like::
> sudo easy_install sphinx
This target will also first build the bro binary if it is not already
since the generation of reStructuredText (reST) documentation from
Bro scripts is integrated within the parsing process.
After completion, HTML documentation can be located inside the CMake
``build/`` directory as ``build/doc/out/html``. The generated reST
documentation will be located in ``build/doc/source/policy``.
``make doc-clean``
This target removes Sphinx inputs and outputs from the CMake ``build/`` dir.
To schedule a script to be documented, edit ``scripts/generate_reST_docs.py.in``
and try adding the name of the script along with an optional script group to
the ``docs`` dictionary. That python script also shows other, more specialized
methods for generating documentation for some types of corner-cases.
When adding a new logical grouping for generated scripts, create a new
reST document in ``source/policy/<group_name>.rst`` and add some default
documentation. References to (and summaries of) documents associated with
the group get appended to this file during the ``make doc`` process.
The Sphinx source tree template in ``source/`` can be modified to add more
common/general documentation, style sheets, JavaScript, etc. The Sphinx
config file is produced from ``conf.py.in``, so that can be edited to change
various Sphinx options, like setting the default HTML rendering theme.
There is also a custom Sphinx domain implemented in ``source/ext/bro.py``
which adds some reST directives and roles that aid in generating useful
index entries and cross-references.
See ``example.bro`` for an example of how to document a Bro script such that
``make doc`` will be able to produce reST/HTML documentation for it.

152
doc/scripts/BroToReST.py.in Executable file
View file

@ -0,0 +1,152 @@
#! /usr/bin/env python
import os
import subprocess
import shutil
import glob
import string
import sys
BRO = "@CMAKE_BINARY_DIR@/src/bro"
BROPATHDEV = "`@CMAKE_BINARY_DIR@/bro-path-dev`"
BRO_ARGS = "--doc-scripts"
DOC_DST_DIR = "@DOC_SOURCE_WORKDIR@/policy"
BROPATH = subprocess.Popen("@CMAKE_BINARY_DIR@/bro-path-dev", shell=True,
stdout=subprocess.PIPE, stderr=subprocess.STDOUT).stdout.readline()
class BroToReST:
"""A class to encapsulate the the generation of reST documentation from
a given Bro script.
"""
bro_src_file = None
doc_src_file = None
load_via_stdin = False
group = None
def __init__(self, src_file, load_method=False, search_dir=None, group=None):
"""
:param src_file: the file name of a Bro script (not a path)
:param load_method: T if script must be loaded by Bro via a stdin
redirection of "@load <script>", F if script can be loaded as
a command line argument to Bro
:param search_dir: a list of directories in which to search for
src_file. If None, the default BROPATH is used.
:param group: a string representing a logical group that the script's
documentation should belong to. A corresponding <group>.rst
document must be pre-existing in the policy/ dir of the source tree
used by Sphinx.
"""
self.bro_src_file = FindBroScript(src_file, search_dir)
self.load_via_stdin = load_method
self.group = group
# formulate doc_src_file from src_file
filename = os.path.basename(src_file)
basename, ext = os.path.splitext(filename)
if ext == ".bro":
self.doc_src_file = basename + ".rst"
else:
self.doc_src_file = filename + ".rst"
def __str__(self):
return "bro_src_file: " + self.bro_src_file \
+ "\ndoc_src_file: " + self.doc_src_file \
+ "\ndoc_dst_file: " + os.path.join(DOC_DST_DIR, self.doc_src_file) \
+ "\nstdin_load: %s" % self.load_via_stdin \
+ "\ngroup: %s" % self.group
def GenDoc(self):
"""Generates the reST documentation for a Bro script and copies
both the documentation and original script into Sphinx source tree.
If the documentation belongs to a group, the necessary modifications
to add it to the group's documentation are done. Afterwards, any
files with a ".rst" suffix are removed for the working directory.
"""
bro_src_basename = os.path.basename(self.bro_src_file)
if self.load_via_stdin:
cmd = "echo '@load %s' | %s %s" % (bro_src_basename, BRO, BRO_ARGS)
else:
cmd = "%s %s %s" % (BRO, BRO_ARGS, self.bro_src_file)
p = subprocess.Popen(cmd, shell=True, env={"BROPATH": BROPATH})
if p.wait() == 0:
shutil.copy(self.doc_src_file, DOC_DST_DIR)
shutil.copy(self.bro_src_file, DOC_DST_DIR)
AppendToDocGroup(self.group, self.bro_src_file, self.doc_src_file)
for leftover in glob.glob("*.rst"):
os.remove(leftover)
def GenDocs(doc_dict, load_method=False):
"""Generates reST documentation for all scripts in the given dictionary.
:param doc_dict: a dictionary whose keys are file names of Bro scripts
(not paths), and whose value is the logical documentation group
it belongs to
:param load_method: T if script must be loaded by Bro via a stdin
redirection of "@load <script>", F if script can be loaded as
a command line argument to Bro
"""
for k, v in doc_dict.iteritems():
doc = BroToReST(k, load_method, group=v)
print "Generating reST document for " + k
doc.GenDoc()
def FindBroScript(src_file, search_dir=None):
"""Search a set of paths for a given Bro script and return the absolute
path to it.
:param src_file: the file name of a Bro script (not a path)
:param search_dir: a list of directories in which to search for
src_file. If None, the default BROPATH is used.
"""
if search_dir is None:
search_dir = string.split(BROPATH, ":")
for path in search_dir:
abs_path = os.path.join(path, src_file)
if os.path.exists(abs_path):
return abs_path
print >> sys.stderr, "Couldn't find '%s'" % src_file
return None
def AppendToDocGroup(group, src_file, doc_file):
"""Adds a reference to the given documentation for a Bro script
to the documentation file for it's associated group. Also, associated
summary text (comments marked up like "##!" in the original Bro script
source) are added.
:param group: a string representing a logical group that the script's
documentation should belong to. A corresponding <group>.rst
document must be pre-existing in the policy/ dir of the source tree
used by Sphinx.
:param src_file: a path to the original Bro script source file
:param doc_file: the file name of a script's generated reST document
"""
if group is None:
return
group_file = os.path.join(DOC_DST_DIR, group + ".rst")
if not os.path.exists(group_file):
print >> sys.stderr, "Group file doesn't exist: " + group_file
return
summary_comments = []
with open(src_file, 'r') as f:
for line in f:
sum_pos = string.find(line, "##!")
if sum_pos != -1:
summary_comments.append(line[(sum_pos+3):])
doc_name, ext = os.path.splitext(doc_file)
with open(group_file, 'a') as f:
f.write("\n:doc:`%s`\n" % doc_name)
for line in summary_comments:
f.write(line)

102
doc/scripts/generate_reST_docs.py.in
View file

@ -6,102 +6,12 @@ import shutil
import glob
import string
import sys
from BroToReST import *
BRO = "@CMAKE_BINARY_DIR@/src/bro"
BROPATHDEV = "`@CMAKE_BINARY_DIR@/bro-path-dev`"
BRO_ARGS = "--doc-scripts"
DOC_DST_DIR = "@DOC_SOURCE_WORKDIR@/policy"
BROPATH = subprocess.Popen("@CMAKE_BINARY_DIR@/bro-path-dev", shell=True,
stdout=subprocess.PIPE, stderr=subprocess.STDOUT).stdout.readline()
class BroToReST:
bro_src_file = None
doc_src_file = None
load_via_stdin = False
group = None
def __init__(self, src_file, load_method=False, search_dir=None, group=None):
self.bro_src_file = FindBroScript(src_file, search_dir)
self.load_via_stdin = load_method
self.group = group
# formulate doc_src_file from src_file
filename = os.path.basename(src_file)
basename, ext = os.path.splitext(filename)
if ext == ".bro":
self.doc_src_file = basename + ".rst"
else:
self.doc_src_file = filename + ".rst"
def __str__(self):
return "bro_src_file: " + self.bro_src_file \
+ "\ndoc_src_file: " + self.doc_src_file \
+ "\ndoc_dst_file: " + os.path.join(DOC_DST_DIR, self.doc_src_file) \
+ "\nstdin_load: %s" % self.load_via_stdin \
+ "\ngroup: %s" % self.group
def GenDoc(self):
bro_src_basename = os.path.basename(self.bro_src_file)
if self.load_via_stdin:
cmd = "echo '@load %s' | %s %s" % (bro_src_basename, BRO, BRO_ARGS)
else:
cmd = "%s %s %s" % (BRO, BRO_ARGS, self.bro_src_file)
p = subprocess.Popen(cmd, shell=True, env={"BROPATH": BROPATH})
if p.wait() == 0:
shutil.copy(self.doc_src_file, DOC_DST_DIR)
shutil.copy(self.bro_src_file, DOC_DST_DIR)
AppendToDocGroup(self.group, self.bro_src_file, self.doc_src_file)
for leftover in glob.glob("*.rst"):
os.remove(leftover)
def GenDocs(doc_dict, load_method=False):
for k, v in doc_dict.iteritems():
doc = BroToReST(k, load_method, group=v)
print "Generating reST document for " + k
doc.GenDoc()
# search BROPATH for the script and return the absolute path to it
def FindBroScript(src_file, search_dir=None):
if search_dir is None:
search_dir = string.split(BROPATH, ":")
for path in search_dir:
abs_path = os.path.join(path, src_file)
if os.path.exists(abs_path):
return abs_path
print >> sys.stderr, "Couldn't find '%s'" % src_file
return None
def AppendToDocGroup(group, src_file, doc_file):
if group is None:
return
group_file = os.path.join(DOC_DST_DIR, group + ".rst")
if not os.path.exists(group_file):
print >> sys.stderr, "Group file doesn't exist: " + group_file
return
summary_comments = []
with open(src_file, 'r') as f:
for line in f:
sum_pos = string.find(line, "##!")
if sum_pos != -1:
summary_comments.append(line[(sum_pos+3):])
doc_name, ext = os.path.splitext(doc_file)
with open(group_file, 'a') as f:
f.write("\n:doc:`%s`\n" % doc_name)
for line in summary_comments:
f.write(line)
# Scripts that can be loaded by bro via command line argument
# TODO: generate docs for more scripts
# TODO: the groups are just made up to test the functionality, fix them
# Scripts that can be loaded by bro via command line argument:
docs = {
"alarm.bro": "internal",
"arp.bro": "user",
@ -124,7 +34,7 @@ docs = {
}
# Scripts that can't be loaded by bro via command line argument (possible
# due to dependency issues), but can be loaded via an @load on stdin
# due to dependency issues), but can be loaded via an @load on stdin:
stdin_docs = {
"notice.bro": "internal",
}
@ -132,9 +42,11 @@ stdin_docs = {
GenDocs(docs)
GenDocs(stdin_docs, True)
# The example documentation script doesn't live on the BROPATH, so
# explicitly generate the docs for it like this:
BroToReST("example.bro", False, ["@PROJECT_SOURCE_DIR@/doc"], group="internal").GenDoc()
# Generate documentation for stuff that's always loaded into bro by default
# Generate documentation for stuff that's always loaded into bro by default:
cmd = "echo '' | %s %s" % (BRO, BRO_ARGS)
p = subprocess.Popen(cmd, shell=True, env={"BROPATH": BROPATH})
if p.wait() == 0: