Mirror/zeek
1
0
Fork 0
mirror of https://github.com/zeek/zeek.git synced 2025-10-02 06:38:20 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Integrate new Broxygen functionality into Sphinx.

Browse source 
Add a "broxygen" domain Sphinx extension w/ directives to allow
on-the-fly documentation to be generated w/ Bro and included in files.

This means all autogenerated reST docs are now done by Bro.  The odd
CMake/Python glue scipts which used to generate some portions are now
gone.  Bro and the Sphinx extension handle checking for outdated docs
themselves.

Parallel builds of `make doc` target should now work (mostly because
I don't think there's any tasks that can be done in parallel anymore).

Overall, this seems to simplify things and make the Broxygen-generated
portions of the documentation visible/traceable from the main Sphinx
source tree.  The one odd thing still is that per-script documentation
is rsync'd in to a shadow copy of the Sphinx source tree within the
build dir.  This is less elegant than using the new broxygen extension
to make per-script docs, but rsync is faster and simpler.  Simpler as in
less code because it seems like, in the best case, I'd need to write a
custom Sphinx Builder to be able to get that to even work.
This commit is contained in:
Jon Siwek 2013-11-21 14:34:32 -06:00
parent 4f6d01000a
commit 9967aea52c
47 changed files with 578 additions and 1173 deletions
Download patch file Download diff file Expand all files Collapse all files

12
Makefile
View file

@ -29,18 +29,6 @@ doc: configured
docclean: configured
$(MAKE) -C $(BUILD) $@
restdoc: configured
$(MAKE) -C $(BUILD) $@
restclean: configured
$(MAKE) -C $(BUILD) $@
broxygen: configured
$(MAKE) -C $(BUILD) $@
broxygenclean: configured
$(MAKE) -C $(BUILD) $@
dist:
@rm -rf $(VERSION_FULL) $(VERSION_FULL).tgz
@rm -rf $(VERSION_MIN) $(VERSION_MIN).tgz

135
doc/CMakeLists.txt
View file

@ -1,75 +1,86 @@
set(BIF_SRC_DIR ${PROJECT_SOURCE_DIR}/src)
set(RST_OUTPUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/rest_output)
set(DOC_OUTPUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/out)
set(DOC_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR})
set(DOC_SOURCE_WORKDIR ${CMAKE_CURRENT_BINARY_DIR}/sphinx-sources)
set(BROCCOLI_DOCS_SRC ${CMAKE_BINARY_DIR}/aux/broccoli/doc/html)
set(BROCCOLI_DOCS_DST ${CMAKE_BINARY_DIR}/html/broccoli-api)
set(SPHINX_INPUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/sphinx_input)
set(SPHINX_OUTPUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/sphinx_output)
set(BROXYGEN_SCRIPT_OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/broxygen_script_output)
set(BROXYGEN_CACHE_DIR ${CMAKE_CURRENT_BINARY_DIR}/broxygen_cache)
set(MASTER_POLICY_INDEX ${CMAKE_CURRENT_BINARY_DIR}/scripts/policy_index)
set(MASTER_PACKAGE_INDEX ${CMAKE_CURRENT_BINARY_DIR}/scripts/pkg_index)
# Find out what BROPATH to use when executing bro.
execute_process(COMMAND ${CMAKE_BINARY_DIR}/bro-path-dev
OUTPUT_VARIABLE BROPATH
RESULT_VARIABLE retval
OUTPUT_STRIP_TRAILING_WHITESPACE)
if (NOT ${retval} EQUAL 0)
message(FATAL_ERROR "Problem setting BROPATH")
endif ()
file(GLOB_RECURSE DOC_SOURCES FOLLOW_SYMLINKS "*")
set(BROMAGIC ${BRO_MAGIC_SOURCE_PATH})
# configure the Sphinx config file (expand variables CMake might know about)
# Configure the Sphinx config file (expand variables CMake might know about).
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/conf.py.in
${CMAKE_CURRENT_BINARY_DIR}/conf.py
@ONLY)
add_subdirectory(scripts)
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/broxygen.conf.in
${CMAKE_CURRENT_BINARY_DIR}/broxygen.conf
@ONLY)
# The "broxygen" target generates reST documentation for any outdated bro
# scripts and then uses Sphinx to generate HTML documentation from the reST
add_custom_target(broxygen
# copy the template documentation to the build directory
# to give as input for sphinx
COMMAND rsync -r --copy-links --times
${DOC_SOURCE_DIR}/
${DOC_SOURCE_WORKDIR}
# copy generated policy script documentation into the
# working copy of the template documentation
COMMAND rsync -r --copy-links --times
${RST_OUTPUT_DIR}/
${DOC_SOURCE_WORKDIR}/scripts
# append to the master index of all policy scripts
COMMAND cat ${MASTER_POLICY_INDEX} >>
${DOC_SOURCE_WORKDIR}/scripts/scripts.rst
# append to the master index of all policy packages
COMMAND cat ${MASTER_PACKAGE_INDEX} >>
${DOC_SOURCE_WORKDIR}/scripts/packages.rst
# construct a reST file for each group
COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/bin/group_index_generator.py
${CMAKE_CURRENT_BINARY_DIR}/scripts/group_list
${CMAKE_CURRENT_BINARY_DIR}/scripts
${DOC_SOURCE_WORKDIR}/scripts
# tell sphinx to generate html
COMMAND sphinx-build
-b html
-c ${CMAKE_CURRENT_BINARY_DIR}
-d ${DOC_OUTPUT_DIR}/doctrees
${DOC_SOURCE_WORKDIR}
${DOC_OUTPUT_DIR}/html
# create symlink to the html output directory for convenience
COMMAND "${CMAKE_COMMAND}" -E create_symlink
${DOC_OUTPUT_DIR}/html
${CMAKE_BINARY_DIR}/html
# copy Broccoli API reference into output dir if it exists
COMMAND test -d ${CMAKE_BINARY_DIR}/aux/broccoli/doc/html && ( rm -rf ${CMAKE_BINARY_DIR}/html/broccoli-api && cp -r ${CMAKE_BINARY_DIR}/aux/broccoli/doc/html ${CMAKE_BINARY_DIR}/html/broccoli-api ) || true
WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
COMMENT "[Sphinx] Generating HTML policy script docs"
# SOURCES just adds stuff to IDE projects as a convenience
SOURCES ${DOC_SOURCES})
add_custom_target(sphinxdoc
# Copy the template documentation to build directory to use as input tree
# for Sphinx. This is needed because some parts are dynamically generated
# in to that tree by Bro/Broxygen.
COMMAND rsync -q -r --copy-links --times --delete
--filter='protect scripts/*'
${CMAKE_CURRENT_SOURCE_DIR}/ ${SPHINX_INPUT_DIR}
# Use Bro/Broxygen to dynamically generate reST for all Bro scripts.
COMMAND BROPATH=${BROPATH}
BROMAGIC=${BROMAGIC}
${CMAKE_BINARY_DIR}/src/bro
-X ${CMAKE_CURRENT_BINARY_DIR}/broxygen.conf
broxygen >/dev/null
# Rsync over the generated reST to the Sphinx source tree in the build dir.
COMMAND rsync -q -r --copy-links --times --delete --filter='protect *.bro'
${BROXYGEN_SCRIPT_OUTPUT}/ ${SPHINX_INPUT_DIR}/scripts
# Rsync over Bro scripts to the Sphinx source tree in the build dir.
# These are used by :download: references in the generated script docs.
COMMAND rsync -q -r --copy-links --times --delete
--filter='protect /base/bif/*' --filter='protect *.rst'
--filter='include */' --filter='include *.bro' --filter='exclude *'
${CMAKE_SOURCE_DIR}/scripts/ ${SPHINX_INPUT_DIR}/scripts
# Rsync over Bro scripts created by BIF compiler to the Sphinx source tree.
COMMAND rsync -q -r --copy-links --times --delete
--filter='protect *.rst' --filter='include */'
--filter='include *.bro' --filter='exclude *'
${CMAKE_BINARY_DIR}/scripts/base/bif/
${SPHINX_INPUT_DIR}/scripts/base/bif
# Use Sphinx to build HTML.
COMMAND sphinx-build
-b html
-c ${CMAKE_CURRENT_BINARY_DIR}
-d ${SPHINX_OUTPUT_DIR}/doctrees
${SPHINX_INPUT_DIR}
${SPHINX_OUTPUT_DIR}/html
# Create symlink to the html output directory for convenience.
COMMAND "${CMAKE_COMMAND}" -E create_symlink
${SPHINX_OUTPUT_DIR}/html
${CMAKE_BINARY_DIR}/html
# Copy Broccoli API reference into output dir if it exists.
COMMAND test -d ${BROCCOLI_DOCS_SRC} &&
( rm -rf ${BROCCOLI_DOCS_DST} &&
cp -r ${BROCCOLI_DOCS_SRC} ${BROCCOLI_DOCS_DST} ) || true
WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
COMMENT "[Sphinx] Generate HTML documentation in ${CMAKE_BINARY_DIR}/html")
# The "broxygenclean" target removes just the Sphinx input/output directories
# from the build directory.
add_custom_target(broxygenclean
COMMAND "${CMAKE_COMMAND}" -E remove_directory
${DOC_SOURCE_WORKDIR}
COMMAND "${CMAKE_COMMAND}" -E remove_directory
${DOC_OUTPUT_DIR}
VERBATIM)
add_dependencies(sphinxdoc bro)
add_dependencies(broxygen restdoc)
add_custom_target(sphinxdoc_clean
COMMAND "${CMAKE_COMMAND}" -E remove_directory ${SPHINX_INPUT_DIR}
COMMAND "${CMAKE_COMMAND}" -E remove_directory ${SPHINX_OUTPUT_DIR}
COMMAND "${CMAKE_COMMAND}" -E remove_directory ${BROXYGEN_SCRIPT_OUTPUT}
COMMAND "${CMAKE_COMMAND}" -E remove_directory ${BROXYGEN_CACHE_DIR}
VERBATIM)
add_custom_target(doc)
add_custom_target(docclean)
add_dependencies(doc broxygen)
add_dependencies(docclean broxygenclean restclean)
add_dependencies(doc sphinxdoc)
add_dependencies(docclean sphinxdoc_clean)

9
doc/README
View file

@ -15,17 +15,16 @@ which adds some reST directives and roles that aid in generating useful
index entries and cross-references. Other extensions can be added in
a similar fashion.
Either the ``make doc`` or ``make broxygen`` targets in the top-level
Makefile can be used to locally render the reST files into HTML.
Those targets depend on:
The ``make doc`` target in the top-level Makefile can be used to locally
render the reST files into HTML. That target depend on:
* Python interpreter >= 2.5
* `Sphinx <http://sphinx.pocoo.org/>`_ >= 1.0.1
After completion, HTML documentation is symlinked in ``build/html``.
There's also ``make docclean`` and ``make broxygenclean`` targets to
clean the resulting documentation.
There's also a ``make docclean`` target which deletes any files
created during the documentation build process.
Notes for Writing Documentation
-------------------------------

72
doc/bin/group_index_generator.py
View file

@ -1,72 +0,0 @@
#! /usr/bin/env python
# This script automatically generates a reST documents that lists
# a collection of Bro scripts that are "grouped" together.
# The summary text (##! comments) of the script is embedded in the list
#
# 1st argument is the file containing list of groups
# 2nd argument is the directory containing ${group}_files lists of
# scripts that belong to the group and ${group}_doc_names lists of
# document names that can be supplied to a reST :doc: role
# 3rd argument is a directory in which write a ${group}.rst file that contains
# reST style references to script docs along with summary text contained
# in original script. If ${group} ends with "index", then the file
# is always clobbered by this script, but for other unique group names,
# this script will append to existing files.
import sys
import os
import string
group_list = sys.argv[1]
file_manifest_dir = sys.argv[2]
output_dir = sys.argv[3]
def make_group_file_index(dir_name, group_name):
group_file = os.path.join(dir_name, group_name + ".rst")
if not os.path.exists(group_file):
if not os.path.exists(os.path.dirname(group_file)):
os.makedirs(os.path.dirname(group_file))
if group_name.endswith("index"):
with open(group_file, 'w') as f_group_file:
f_group_file.write(":orphan:\n\n")
title = "Package Index: %s\n" % os.path.dirname(group_name)
f_group_file.write(title);
for n in range(len(title)):
f_group_file.write("=")
f_group_file.write("\n");
return group_file
with open(group_list, 'r') as f_group_list:
for group in f_group_list.read().splitlines():
#print group
group_file = make_group_file_index(output_dir, group)
file_manifest = os.path.join(file_manifest_dir, group + "_files")
doc_manifest = os.path.join(file_manifest_dir, group + "_doc_names")
src_files = []
doc_names = []
with open(file_manifest, 'r') as f_file_manifest:
src_files = f_file_manifest.read().splitlines()
with open(doc_manifest, 'r') as f_doc_manifest:
doc_names = f_doc_manifest.read().splitlines()
for i in range(len(src_files)):
src_file = src_files[i]
#print "\t" + src_file
summary_comments = []
with open(src_file, 'r') as f_src_file:
for line in f_src_file:
sum_pos = string.find(line, "##!")
if sum_pos != -1:
summary_comments.append(line[(sum_pos+3):])
#print summary_comments
with open(group_file, 'a') as f_group_file:
f_group_file.write("\n:doc:`/scripts/%s`\n" % doc_names[i])
for line in summary_comments:
f_group_file.write(" " + line)

1
doc/broxygen.conf.in Normal file
View file

@ -0,0 +1 @@
script @BROXYGEN_SCRIPT_OUTPUT@/ *

12
doc/conf.py.in
View file

@ -17,7 +17,7 @@ extensions = []
# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
sys.path.insert(0, os.path.abspath('sphinx-sources/ext'))
sys.path.insert(0, os.path.abspath('sphinx_input/ext'))
# ----- Begin of BTest configuration. -----
btest = os.path.abspath("@CMAKE_SOURCE_DIR@/aux/btest")
@ -33,6 +33,12 @@ btest_base="@CMAKE_SOURCE_DIR@/testing/btest"
btest_tests="doc/sphinx"
# ----- End of BTest configuration. -----
# ----- Begin of Broxygen configuration. -----
extensions += ["broxygen"]
broxygen_cache="@BROXYGEN_CACHE_DIR@"
os.environ["BROPATH"] = "@BROPATH@"
os.environ["BROMAGIC"] = "@BROMAGIC@"
# ----- End of Broxygen configuration. -----
# -- General configuration -----------------------------------------------------
@ -47,7 +53,7 @@ os.environ["BRO_SRC_ROOT"] = "@CMAKE_SOURCE_DIR@"
os.environ["DOC_ROOT"] = "@CMAKE_SOURCE_DIR@/doc"
# Add any paths that contain templates here, relative to this directory.
templates_path = ['sphinx-sources/_templates', 'sphinx-sources/_static']
templates_path = ['sphinx_input/_templates', 'sphinx_input/_static']
# The suffix of source filenames.
source_suffix = '.rst'
@ -141,7 +147,7 @@ html_theme_options = { }
# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['sphinx-sources/_static']
html_static_path = ['sphinx_input/_static']
# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.

4
doc/ext/bro.py
View file

@ -191,6 +191,10 @@ class BroNotices(Index):
def generate(self, docnames=None):
content = {}
if 'notices' not in self.domain.env.domaindata['bro']:
return content, False
for n in self.domain.env.domaindata['bro']['notices']:
modname = n[0].split("::")[0]
entries = content.setdefault(modname, [])

311
doc/ext/broxygen.py Normal file
View file

@ -0,0 +1,311 @@
"""
Broxygen domain for Sphinx.
Adds directives that allow Sphinx to invoke Bro in order to generate script
reference documentation on the fly. The directives are:
broxygen:package
- Shows links to all scripts contained within matching package(s).
broxygen:package_index
- An index with links to matching package document(s).
broxygen:script
- Reference for matching script(s) (i.e. everything declared by the script).
broxygen:script_summary
- Shows link to matching script(s) with it's summary-section comments.
broxygen:script_index
- An index with links to all matching scrips.
broxygen:proto_analyzer
- All protocol analyzers and their components (events/bifs, etc.)
broxygen:file_analyzer
- All file analyzers and their components (events/bifs, etc.)
"""
from sphinx.domains import Domain, ObjType
from sphinx.locale import l_
from docutils.parsers.rst.directives.misc import Include
App = None
def info(msg):
"""Use Sphinx builder to output a console message."""
global App
from sphinx.util.console import blue
App.builder.info(blue(msg))
def pattern_to_filename_component(pattern):
"""Replace certain characters in Broxygen config file target pattern.
Such that it can be used as part of a (sane) filename.
"""
return pattern.replace("/", ".").replace("*", "star")
def ensure_dir(path):
"""Should act like ``mkdir -p``."""
import os
import errno
try:
os.makedirs(path)
except OSError as e:
if e.errno != errno.EEXIST:
raise
def generate_config(env, type, pattern):
"""Create a Broxygen config file for a particular target.
It can be used by Bro to generate reST docs for that target.
"""
import os
import tempfile
from sphinx.errors import SphinxError
work_dir = env.config.broxygen_cache
if not work_dir:
raise SphinxError("broxygen_cache not set in sphinx config file")
ensure_dir(work_dir)
prefix = "{0}-{1}-".format(type, pattern_to_filename_component(pattern))
(fd, cfg) = tempfile.mkstemp(suffix=".cfg", prefix=prefix, dir=work_dir)
generated_file = "{0}.rst".format(cfg)
config = "{0}\t{1}\t{2}".format(type, generated_file, pattern)
f = os.fdopen(fd, "w")
f.write(config)
f.close()
return (cfg, generated_file)
def generate_target(env, type, pattern):
"""Create a Broxygen target and build it.
For a target which hasn't been referenced by any other script, this function
creates an associated config file then uses Bro w/ it to build the target
and stores the target information in the build environment.
If a script references a target that's already found in the build
environment the results of the previous built are re-used.
"""
app_data = env.domaindata["broxygen"]
if (type, pattern) in app_data["targets"]:
info("Broxygen has cached doc for target '{0} {1}'".format(
type, pattern))
return app_data["targets"]
(cfg, gend_file) = generate_config(env, type, pattern)
target = BroxygenTarget(type, pattern, cfg, gend_file)
app_data["targets"][(type, pattern)] = target
build_target(env, target)
info("Broxygen built target '{0} {1}'".format(type, pattern))
return target
def build_target(env, target):
"""Invoke a Bro process to build a Broxygen target."""
import os
import subprocess
bro_cmd = "bro -X {0} broxygen".format(target.config_file)
cwd = os.getcwd()
os.chdir(os.path.dirname(target.config_file))
try:
subprocess.check_output(bro_cmd, stderr=subprocess.STDOUT, shell=True)
except subprocess.CalledProcessError as e:
from sphinx.errors import SphinxError
raise SphinxError(
"Command '{0}' returned non-zero exit status {1}: {2}".format(
e.cmd, e.returncode, e.output))
finally:
os.chdir(cwd)
class BroxygenTarget(object):
"""Some portion of reST documentation that Bro knows how to generate.
A target is identified by its type and pattern. E.g. type "script" and
pattern "broxygen/example.bro".
"""
def __init__(self, type, pattern, config_file, generated_file):
self.type = type
self.pattern = pattern
self.config_file = config_file
self.generated_file = generated_file
self.used_in_docs = set()
class BroxygenDirective(Include):
"""Base class for Broxygen directives.
It can use Bro to generate reST documentation on the fly and embed it in
the document at the location of the directive just like the ``.. include::``
directive. The only argument is a pattern to identify to Bro which
pieces of documentation it needs to create.
"""
required_arguments = 1
has_content = False
target_type = None
def run(self):
env = self.state.document.settings.env
info("Broxygen running .. {0}:: {1} in {2}".format(
self.name, self.arguments[0], env.docname))
target = generate_target(env, self.target_type, self.arguments[0])
target.used_in_docs.add(env.docname)
self.arguments = [target.generated_file]
return super(BroxygenDirective, self).run()
class PackageDirective(BroxygenDirective):
target_type = "package"
class PackageIndexDirective(BroxygenDirective):
target_type = "package_index"
class ScriptDirective(BroxygenDirective):
target_type = "script"
class ScriptSummaryDirective(BroxygenDirective):
target_type = "script_summary"
class ScriptIndexDirective(BroxygenDirective):
target_type = "script_index"
class ProtoAnalyzerDirective(BroxygenDirective):
target_type = "proto_analyzer"
class FileAnalyzerDirective(BroxygenDirective):
target_type = "file_analyzer"
class IdentifierDirective(BroxygenDirective):
target_type = "identifier"
class BroxygenDomain(Domain):
name = "broxygen"
label = "Broxygen"
object_types = {
"package": ObjType(l_("package")),
"package_index": ObjType(l_("package_index")),
"script": ObjType(l_("script")),
"script_summary": ObjType(l_("script_summary")),
"script_index": ObjType(l_("script_index")),
"proto_analyzer": ObjType(l_("proto_analyzer")),
"file_analyzer": ObjType(l_("file_analyzer")),
"identifier": ObjType(l_("identifier")),
}
directives = {
"package": PackageDirective,
"package_index": PackageIndexDirective,
"script": ScriptDirective,
"script_summary": ScriptSummaryDirective,
"script_index": ScriptIndexDirective,
"proto_analyzer": ProtoAnalyzerDirective,
"file_analyzer": FileAnalyzerDirective,
"identifier": IdentifierDirective,
}
roles = {}
initial_data = {
"targets": {}
}
def clear_doc(self, docname):
"""Update Broxygen targets referenced in docname.
If it's the last place the target was referenced, remove it from
the build environment and delete any generated config/reST files
associated with it from the cache.
"""
import os
stale_targets = []
for (type, pattern), target in self.data["targets"].items():
if docname in target.used_in_docs:
target.used_in_docs.remove(docname)
if not target.used_in_docs:
stale_targets.append(target)
for target in stale_targets:
del self.data["targets"][(target.type, target.pattern)]
os.remove(target.config_file)
os.remove(target.generated_file)
def get_objects(self):
"""No Broxygen-generated content is itself linkable/searchable."""
return []
def env_get_outdated_hook(app, env, added, changed, removed):
"""Check whether to re-read any documents referencing Broxygen targets.
To do that we have to ask Bro to rebuild each target and compare the
before and after modification times of the generated reST output file.
If Bro changed it, then the document containing the Broxygen directive
needs to be re-read.
"""
import os
reread = set()
for target in app.env.domaindata["broxygen"]["targets"].values():
before_mtime = os.stat(target.generated_file)
build_target(env, target)
after_mtime = os.stat(target.generated_file)
if after_mtime > before_mtime:
info("Broxygen target '{0} {1}' outdated".format(
target.type, target.pattern))
for docname in target.used_in_docs:
if docname not in removed:
info(" in document: {0}".format(docname))
reread.add(docname)
return list(reread)
def setup(app):
global App
App = app
app.add_domain(BroxygenDomain)
app.add_config_value("broxygen_cache", None, "env")
app.connect("env-get-outdated", env_get_outdated_hook)

2
doc/frameworks/logging-dataseries.rst
View file

@ -104,7 +104,7 @@ code like this to your ``local.bro``:
}
Bro's DataSeries writer comes with a few tuning options, see
:doc:`/scripts/base/frameworks/logging/writers/dataseries`.
:doc:`/scripts/base/frameworks/logging/writers/dataseries.bro`.
Working with DataSeries
=======================

6
doc/frameworks/logging.rst
View file

@ -48,7 +48,7 @@ Basics
The data fields that a stream records are defined by a record type
specified when it is created. Let's look at the script generating Bro's
connection summaries as an example,
:doc:`/scripts/base/protocols/conn/main`. It defines a record
:doc:`/scripts/base/protocols/conn/main.bro`. It defines a record
:bro:type:`Conn::Info` that lists all the fields that go into
``conn.log``, each marked with a ``&log`` attribute indicating that it
is part of the information written out. To write a log record, the
@ -309,7 +309,7 @@ ASCII Writer Configuration
--------------------------
The ASCII writer has a number of options for customizing the format of
its output, see :doc:`/scripts/base/frameworks/logging/writers/ascii`.
its output, see :doc:`/scripts/base/frameworks/logging/writers/ascii.bro`.
Adding Streams
==============
@ -369,7 +369,7 @@ save the logged ``Foo::Info`` record into the connection record:
}
See the existing scripts for how to work with such a new connection
field. A simple example is :doc:`/scripts/base/protocols/syslog/main`.
field. A simple example is :doc:`/scripts/base/protocols/syslog/main.bro`.
When you are developing scripts that add data to the :bro:type:`connection`
record, care must be given to when and how long data is stored.

2
doc/frameworks/notice.rst
View file

@ -283,7 +283,7 @@ information to suppress duplicates for a configurable period of time.
The ``$identifier`` field is typically comprised of several pieces of
data related to the notice that when combined represent a unique
instance of that notice. Here is an example of the script
:doc:`/scripts/policy/protocols/ssl/validate-certs` raising a notice
:doc:`/scripts/policy/protocols/ssl/validate-certs.bro` raising a notice
for session negotiations where the certificate or certificate chain did
not validate successfully against the available certificate authority
certificates.

2
doc/frameworks/signatures.rst
View file

@ -46,7 +46,7 @@ signature's event statement (``Found root!``), and data is the last
piece of payload which triggered the pattern match.
To turn such :bro:id:`signature_match` events into actual alarms, you can
load Bro's :doc:`/scripts/base/frameworks/signatures/main` script.
load Bro's :doc:`/scripts/base/frameworks/signatures/main.bro` script.
This script contains a default event handler that raises
:bro:enum:`Signatures::Sensitive_Signature` :doc:`Notices <notice>`
(as well as others; see the beginning of the script).

2
doc/index.rst
View file

@ -15,7 +15,7 @@ Bro Documentation
scripting/index.rst
frameworks/index.rst
cluster/index.rst
scripts/index.rst
script-reference/index.rst
components/index.rst
* :ref:`General Index <genindex>`

2
doc/quickstart/index.rst
View file

@ -197,7 +197,7 @@ the variable's value may not change at run-time, but whose initial value can be
modified via the ``redef`` operator at parse-time.
So let's continue on our path to modify the behavior for the two SSL
and SSH notices. Looking at :doc:`/scripts/base/frameworks/notice/main`,
and SSH notices. Looking at :doc:`/scripts/base/frameworks/notice/main.bro`,
we see that it advertises:
.. code:: bro

0
doc/scripts/builtins.rst → doc/script-reference/builtins.rst
View file

1
doc/script-reference/file-analyzers.rst Normal file
View file

@ -0,0 +1 @@
.. broxygen:file_analyzer:: *

12
doc/scripts/index.rst → doc/script-reference/index.rst
View file

@ -1,5 +1,3 @@
.. This is a stub doc to which broxygen appends during the build process
================
Script Reference
================
@ -7,15 +5,9 @@ Script Reference
.. toctree::
:maxdepth: 1
packages
proto-analyzers
file-analyzers
notices
builtins
Built-in Functions (BIFs) <base/bif/index>
internal
packages
scripts
Broxygen Example Script </scripts/broxygen/example.bro>

8
doc/scripts/packages.rst → doc/script-reference/packages.rst
View file

@ -1,9 +1,7 @@
.. This is a stub doc to which broxygen appends during the build process
.. _script-packages:
Bro Script Packages
===================
Bro Package Index
=================
Bro has the following script packages (e.g. collections of related scripts in
a common directory). If the package directory contains a ``__load__.bro``
@ -12,3 +10,5 @@ script, it supports being loaded in mass as a whole directory for convenience.
Packages/scripts in the ``base/`` directory are all loaded by default, while
ones in ``policy/`` provide functionality and customization options that are
more appropriate for users to decide whether they'd like to load it or not.
.. broxygen:package_index:: *

1
doc/script-reference/proto-analyzers.rst Normal file
View file

@ -0,0 +1 @@
.. broxygen:proto_analyzer:: *

5
doc/script-reference/scripts.rst Normal file
View file

@ -0,0 +1,5 @@
================
Bro Script Index
================
.. broxygen:script_index:: *

8
doc/scripting/index.rst
View file

@ -316,7 +316,7 @@ block that variable is available to any other script through the
naming convention of ``MODULE::variable_name``.
The declaration below is taken from the
:doc:`/scripts/policy/protocols/conn/known-hosts` script and
:doc:`/scripts/policy/protocols/conn/known-hosts.bro` script and
declares a variable called ``known_hosts`` as a global set of unique
IP addresses within the ``Known`` namespace and exports it for use
outside of the ``Known`` namespace. Were we to want to use the
@ -348,7 +348,7 @@ constants are used in Bro scripts as containers for configuration
options. For example, the configuration option to log password
decrypted from HTTP streams is stored in
``HTTP::default_capture_password`` as shown in the stripped down
excerpt from :doc:`/scripts/base/protocols/http/main` below.
excerpt from :doc:`/scripts/base/protocols/http/main.bro` below.
.. btest-include:: ${BRO_SRC_ROOT}/scripts/base/protocols/http/main.bro
:lines: 8-10,19-21,120
@ -1182,7 +1182,7 @@ passing in the ``Notice::Info`` record. The simplest kind of
action based on the answer. The hook below adds the
:bro:enum:`Notice::ACTION_EMAIL` action for the
``SSH::Interesting_Hostname_Login`` notice raised in the
:doc:`/scripts/policy/protocols/ssh/interesting-hostnames` script.
:doc:`/scripts/policy/protocols/ssh/interesting-hostnames.bro` script.
.. btest-include:: ${DOC_ROOT}/scripting/framework_notice_hook_01.bro
@ -1224,7 +1224,7 @@ Bro.
.. btest-include:: ${BRO_SRC_ROOT}/scripts/policy/protocols/ssl/expiring-certs.bro
:lines: 60-63
In the :doc:`/scripts/policy/protocols/ssl/expiring-certs` script
In the :doc:`/scripts/policy/protocols/ssl/expiring-certs.bro` script
which identifies when SSL certificates are set to expire and raises
notices when it crosses a predefined threshold, the call to
``NOTICE`` above also sets the ``$identifier`` entry by concatenating

226
doc/scripts/CMakeLists.txt
View file

@ -1,226 +0,0 @@
# find out what BROPATH to use when executing bro
execute_process(COMMAND ${CMAKE_BINARY_DIR}/bro-path-dev
OUTPUT_VARIABLE BROPATH
RESULT_VARIABLE retval
OUTPUT_STRIP_TRAILING_WHITESPACE)
if (NOT ${retval} EQUAL 0)
message(FATAL_ERROR "Problem setting BROPATH")
endif ()
# This macro is used to add a new makefile target for reST policy script
# documentation that can be generated using Bro itself to parse policy scripts.
# It's called like:
#
# rest_target(srcDir broInput [group])
#
# srcDir: the directory which contains broInput
# broInput: the file name of a bro policy script, any path prefix of this
# argument will be used to derive what path under scripts/ the generated
# documentation will be placed.
# group: optional name of group that the script documentation will belong to.
# If this is not given, the group is automatically set to any path portion
# of the broInput argument.
#
# In addition to adding the makefile target, several CMake variables are set:
#
# MASTER_POLICY_INDEX_TEXT: a running list of policy scripts docs that have
# been generated so far, formatted such that it can be appended to a file
# that ends in a Sphinx toctree directive
# ALL_REST_OUTPUTS: a running list (the CMake list type) of all reST docs
# that are to be generated
# MASTER_GROUP_LIST: a running list (the CMake list type) of all script groups
# MASTER_PKG_LIST: a running list (the CMake list type) of all script groups
# that were defived from the path portion of the broInput argument
# ${group}_files: a running list of files belonging to a given group, from
# which summary text can be extracted at build time
# ${group}_doc_names: a running list of reST style document names that can be
# given to a :doc: role, shared indices with ${group}_files
macro(REST_TARGET srcDir broInput)
set(absSrcPath ${srcDir}/${broInput})
get_filename_component(basename ${broInput} NAME)
string(REPLACE .bro "" basename ${basename})
get_filename_component(extension ${broInput} EXT)
get_filename_component(relDstDir ${broInput} PATH)
set(sumTextSrc ${absSrcPath})
set(ogSourceFile ${absSrcPath})
if (NOT relDstDir)
set(docName "${basename}")
set(dstDir "${RST_OUTPUT_DIR}")
else ()
set(docName "${relDstDir}/${basename}")
set(dstDir "${RST_OUTPUT_DIR}/${relDstDir}")
endif ()
set(restFile "${docName}.rst")
string(REPLACE "/" "^" restFile ${restFile})
set(restOutput "${dstDir}/${basename}.rst")
set(MASTER_POLICY_INDEX_TEXT
"${MASTER_POLICY_INDEX_TEXT}\n ${docName} <${docName}>")
list(APPEND ALL_REST_OUTPUTS ${restOutput})
if (NOT "${ARGN}" STREQUAL "")
set(group ${ARGN})
elseif (relDstDir)
set(group ${relDstDir}/index)
# add package index to master package list if not already in it
# and if a __load__.bro exists in the original script directory
list(FIND MASTER_PKG_LIST ${relDstDir} _found)
if (_found EQUAL -1)
if (EXISTS ${CMAKE_SOURCE_DIR}/scripts/${relDstDir}/__load__.bro)
list(APPEND MASTER_PKG_LIST ${relDstDir})
endif ()
endif ()
else ()
set(group "")
endif ()
if (NOT "${group}" STREQUAL "")
# add group to master group list if not already in it
list(FIND MASTER_GROUP_LIST ${group} _found)
if (_found EQUAL -1)
list(APPEND MASTER_GROUP_LIST ${group})
if (MASTER_GROUP_LIST_TEXT)
set(MASTER_GROUP_LIST_TEXT "${MASTER_GROUP_LIST_TEXT}\n${group}")
else ()
set(MASTER_GROUP_LIST_TEXT "${group}")
endif ()
endif ()
list(APPEND ${group}_files ${sumTextSrc})
list(APPEND ${group}_doc_names ${docName})
endif ()
add_custom_command(OUTPUT ${restOutput}
# delete any leftover state from previous bro runs
COMMAND "${CMAKE_COMMAND}"
ARGS -E remove_directory .state
# generate the reST documentation using bro
COMMAND BROPATH=${BROPATH}:${srcDir} BROMAGIC=${CMAKE_SOURCE_DIR}/magic/database ${CMAKE_BINARY_DIR}/src/bro
ARGS -b -Z ${broInput} || (rm -rf .state *.log *.rst && exit 1)
# move generated doc into a new directory tree that
# defines the final structure of documents
COMMAND "${CMAKE_COMMAND}"
ARGS -E make_directory ${dstDir}
COMMAND "${CMAKE_COMMAND}"
ARGS -E copy ${restFile} ${restOutput}
# copy the bro or bif script, too
COMMAND "${CMAKE_COMMAND}"
ARGS -E copy ${ogSourceFile} ${dstDir}
# clean up the build directory
COMMAND rm
ARGS -rf .state *.log *.rst
DEPENDS bro
DEPENDS ${absSrcPath}
WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
COMMENT "[Bro] Generating reST docs for ${broInput}"
)
endmacro(REST_TARGET)
# Schedule Bro scripts for which to generate documentation.
include(DocSourcesList.cmake)
# Macro for generating reST docs that are independent of any particular Bro
# script.
macro(INDEPENDENT_REST_TARGET reST_file)
add_custom_command(OUTPUT ${reST_file}
# delete any leftover state from previous bro runs
COMMAND "${CMAKE_COMMAND}"
ARGS -E remove_directory .state
# generate the reST documentation using bro
COMMAND BROPATH=${BROPATH}:${srcDir} BROMAGIC=${CMAKE_SOURCE_DIR}/magic/database ${CMAKE_BINARY_DIR}/src/bro
ARGS -b -Z base/init-bare.bro || (rm -rf .state *.log *.rst && exit 1)
# move generated doc into a new directory tree that
# defines the final structure of documents
COMMAND "${CMAKE_COMMAND}"
ARGS -E make_directory ${dstDir}
COMMAND "${CMAKE_COMMAND}"
ARGS -E copy ${reST_file} ${dstDir}
# clean up the build directory
COMMAND rm
ARGS -rf .state *.log *.rst
DEPENDS bro
WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
COMMENT "[Bro] Generating reST docs for ${reST_file}"
)
list(APPEND ALL_REST_OUTPUTS ${reST_file})
endmacro(INDEPENDENT_REST_TARGET)
independent_rest_target(proto-analyzers.rst)
independent_rest_target(file-analyzers.rst)
# create temporary list of all docs to include in the master policy/index file
file(WRITE ${MASTER_POLICY_INDEX} "${MASTER_POLICY_INDEX_TEXT}")
# create the temporary list of all packages to include in the master
# policy/packages.rst file
set(MASTER_PKG_INDEX_TEXT "")
foreach (pkg ${MASTER_PKG_LIST})
set(MASTER_PKG_INDEX_TEXT
"${MASTER_PKG_INDEX_TEXT}\n:doc:`${pkg} <${pkg}/index>`\n")
if (EXISTS ${CMAKE_SOURCE_DIR}/scripts/${pkg}/README)
file(STRINGS ${CMAKE_SOURCE_DIR}/scripts/${pkg}/README pkgreadme)
foreach (line ${pkgreadme})
set(MASTER_PKG_INDEX_TEXT "${MASTER_PKG_INDEX_TEXT}\n ${line}")
endforeach ()
set(MASTER_PKG_INDEX_TEXT "${MASTER_PKG_INDEX_TEXT}\n")
endif ()
endforeach ()
file(WRITE ${MASTER_PACKAGE_INDEX} "${MASTER_PKG_INDEX_TEXT}")
# create temporary file containing list of all groups
file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/group_list
"${MASTER_GROUP_LIST_TEXT}")
# create temporary files containing list of each source file in a given group
foreach (group ${MASTER_GROUP_LIST})
if (EXISTS ${CMAKE_CURRENT_BINARY_DIR}/${group}_files)
file(REMOVE ${CMAKE_CURRENT_BINARY_DIR}/${group}_files)
endif ()
if (EXISTS ${CMAKE_CURRENT_BINARY_DIR}/${group}_doc_names)
file(REMOVE ${CMAKE_CURRENT_BINARY_DIR}/${group}_doc_names)
endif ()
foreach (src ${${group}_files})
file(APPEND ${CMAKE_CURRENT_BINARY_DIR}/${group}_files "${src}\n")
endforeach ()
foreach (dname ${${group}_doc_names})
file(APPEND ${CMAKE_CURRENT_BINARY_DIR}/${group}_doc_names "${dname}\n")
endforeach ()
endforeach ()
# remove previously generated docs no longer scheduled for generation
if (EXISTS ${RST_OUTPUT_DIR})
file(GLOB_RECURSE EXISTING_REST_DOCS "${RST_OUTPUT_DIR}/*.rst")
foreach (_doc ${EXISTING_REST_DOCS})
list(FIND ALL_REST_OUTPUTS ${_doc} _found)
if (_found EQUAL -1)
file(REMOVE ${_doc})
message(STATUS "Broxygen: remove stale reST doc: ${_doc}")
string(REPLACE .rst .bro _brofile ${_doc})
if (EXISTS ${_brofile})
file(REMOVE ${_brofile})
message(STATUS "Broxygen: remove stale bro source: ${_brofile}")
endif ()
endif ()
endforeach ()
endif ()
# The "restdoc" target uses Bro to parse policy scripts in order to
# generate reST documentation from them.
add_custom_target(restdoc
# create symlink to the reST output directory for convenience
COMMAND "${CMAKE_COMMAND}" -E create_symlink
${RST_OUTPUT_DIR}
${CMAKE_BINARY_DIR}/reST
DEPENDS ${ALL_REST_OUTPUTS})
# The "restclean" target removes all generated reST documentation from the
# build directory.
add_custom_target(restclean
COMMAND "${CMAKE_COMMAND}" -E remove_directory
${RST_OUTPUT_DIR}
VERBATIM)

282
doc/scripts/DocSourcesList.cmake
View file

@ -1,282 +0,0 @@
# DO NOT EDIT
# This file is auto-generated from the genDocSourcesList.sh script.
#
# This is a list of Bro script sources for which to generate reST documentation.
# It will be included inline in the CMakeLists.txt found in the same directory
# in order to create Makefile targets that define how to generate reST from
# a given Bro script.
#
# Note: any path prefix of the script (2nd argument of rest_target macro)
# will be used to derive what path under scripts/ the generated documentation
# will be placed.
set(psd ${PROJECT_SOURCE_DIR}/scripts)
rest_target(${CMAKE_CURRENT_SOURCE_DIR} example.bro internal)
rest_target(${psd} base/init-default.bro internal)
rest_target(${psd} base/init-bare.bro internal)
rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/analyzer.bif.bro)
rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/bloom-filter.bif.bro)
rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/bro.bif.bro)
rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/broxygen.bif.bro)
rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/cardinality-counter.bif.bro)
rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/const.bif.bro)
rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/event.bif.bro)
rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/file_analysis.bif.bro)
rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/input.bif.bro)
rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/logging.bif.bro)
rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_ARP.events.bif.bro)
rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_AYIYA.events.bif.bro)
rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_BackDoor.events.bif.bro)
rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_BitTorrent.events.bif.bro)
rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_ConnSize.events.bif.bro)
rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_DCE_RPC.events.bif.bro)
rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_DHCP.events.bif.bro)
rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_DNP3.events.bif.bro)
rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_DNS.events.bif.bro)
rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_FTP.events.bif.bro)
rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_FTP.functions.bif.bro)
rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_File.events.bif.bro)
rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_FileExtract.events.bif.bro)
rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_FileExtract.functions.bif.bro)
rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_FileHash.events.bif.bro)
rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_Finger.events.bif.bro)
rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_GTPv1.events.bif.bro)
rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_Gnutella.events.bif.bro)
rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_HTTP.events.bif.bro)
rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_HTTP.functions.bif.bro)
rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_ICMP.events.bif.bro)
rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_IRC.events.bif.bro)
rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_Ident.events.bif.bro)
rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_InterConn.events.bif.bro)
rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_Login.events.bif.bro)
rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_Login.functions.bif.bro)
rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_MIME.events.bif.bro)
rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_Modbus.events.bif.bro)
rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_NCP.events.bif.bro)
rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_NTP.events.bif.bro)
rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_NetBIOS.events.bif.bro)
rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_NetBIOS.functions.bif.bro)
rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_NetFlow.events.bif.bro)
rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_PIA.events.bif.bro)
rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_POP3.events.bif.bro)
rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_RPC.events.bif.bro)
rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_SMB.events.bif.bro)
rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_SMTP.events.bif.bro)
rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_SMTP.functions.bif.bro)
rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_SOCKS.events.bif.bro)
rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_SSH.events.bif.bro)
rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_SSL.events.bif.bro)
rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_SSL.functions.bif.bro)
rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_SteppingStone.events.bif.bro)
rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_Syslog.events.bif.bro)
rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_TCP.events.bif.bro)
rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_TCP.functions.bif.bro)
rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_Teredo.events.bif.bro)
rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_UDP.events.bif.bro)
rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_Unified2.events.bif.bro)
rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_Unified2.types.bif.bro)
rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_ZIP.events.bif.bro)
rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/reporter.bif.bro)
rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/strings.bif.bro)
rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/top-k.bif.bro)
rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/types.bif.bro)
rest_target(${psd} base/files/extract/main.bro)
rest_target(${psd} base/files/hash/main.bro)
rest_target(${psd} base/files/unified2/main.bro)
rest_target(${psd} base/frameworks/analyzer/main.bro)
rest_target(${psd} base/frameworks/cluster/main.bro)
rest_target(${psd} base/frameworks/cluster/nodes/manager.bro)
rest_target(${psd} base/frameworks/cluster/nodes/proxy.bro)
rest_target(${psd} base/frameworks/cluster/nodes/worker.bro)
rest_target(${psd} base/frameworks/cluster/setup-connections.bro)
rest_target(${psd} base/frameworks/communication/main.bro)
rest_target(${psd} base/frameworks/control/main.bro)
rest_target(${psd} base/frameworks/dpd/main.bro)
rest_target(${psd} base/frameworks/files/main.bro)
rest_target(${psd} base/frameworks/input/main.bro)
rest_target(${psd} base/frameworks/input/readers/ascii.bro)
rest_target(${psd} base/frameworks/input/readers/benchmark.bro)
rest_target(${psd} base/frameworks/input/readers/binary.bro)
rest_target(${psd} base/frameworks/input/readers/raw.bro)
rest_target(${psd} base/frameworks/input/readers/sqlite.bro)
rest_target(${psd} base/frameworks/intel/cluster.bro)
rest_target(${psd} base/frameworks/intel/input.bro)
rest_target(${psd} base/frameworks/intel/main.bro)
rest_target(${psd} base/frameworks/logging/main.bro)
rest_target(${psd} base/frameworks/logging/postprocessors/scp.bro)
rest_target(${psd} base/frameworks/logging/postprocessors/sftp.bro)
rest_target(${psd} base/frameworks/logging/writers/ascii.bro)
rest_target(${psd} base/frameworks/logging/writers/dataseries.bro)
rest_target(${psd} base/frameworks/logging/writers/elasticsearch.bro)
rest_target(${psd} base/frameworks/logging/writers/none.bro)
rest_target(${psd} base/frameworks/logging/writers/sqlite.bro)
rest_target(${psd} base/frameworks/notice/actions/add-geodata.bro)
rest_target(${psd} base/frameworks/notice/actions/drop.bro)
rest_target(${psd} base/frameworks/notice/actions/email_admin.bro)
rest_target(${psd} base/frameworks/notice/actions/page.bro)
rest_target(${psd} base/frameworks/notice/actions/pp-alarms.bro)
rest_target(${psd} base/frameworks/notice/cluster.bro)
rest_target(${psd} base/frameworks/notice/extend-email/hostnames.bro)
rest_target(${psd} base/frameworks/notice/main.bro)
rest_target(${psd} base/frameworks/notice/non-cluster.bro)
rest_target(${psd} base/frameworks/notice/weird.bro)
rest_target(${psd} base/frameworks/packet-filter/cluster.bro)
rest_target(${psd} base/frameworks/packet-filter/main.bro)
rest_target(${psd} base/frameworks/packet-filter/netstats.bro)
rest_target(${psd} base/frameworks/packet-filter/utils.bro)
rest_target(${psd} base/frameworks/reporter/main.bro)
rest_target(${psd} base/frameworks/signatures/main.bro)
rest_target(${psd} base/frameworks/software/main.bro)
rest_target(${psd} base/frameworks/sumstats/cluster.bro)
rest_target(${psd} base/frameworks/sumstats/main.bro)
rest_target(${psd} base/frameworks/sumstats/non-cluster.bro)
rest_target(${psd} base/frameworks/sumstats/plugins/average.bro)
rest_target(${psd} base/frameworks/sumstats/plugins/hll_unique.bro)
rest_target(${psd} base/frameworks/sumstats/plugins/last.bro)
rest_target(${psd} base/frameworks/sumstats/plugins/max.bro)
rest_target(${psd} base/frameworks/sumstats/plugins/min.bro)
rest_target(${psd} base/frameworks/sumstats/plugins/sample.bro)
rest_target(${psd} base/frameworks/sumstats/plugins/std-dev.bro)
rest_target(${psd} base/frameworks/sumstats/plugins/sum.bro)
rest_target(${psd} base/frameworks/sumstats/plugins/topk.bro)
rest_target(${psd} base/frameworks/sumstats/plugins/unique.bro)
rest_target(${psd} base/frameworks/sumstats/plugins/variance.bro)
rest_target(${psd} base/frameworks/tunnels/main.bro)
rest_target(${psd} base/misc/find-checksum-offloading.bro)
rest_target(${psd} base/protocols/conn/contents.bro)
rest_target(${psd} base/protocols/conn/inactivity.bro)
rest_target(${psd} base/protocols/conn/main.bro)
rest_target(${psd} base/protocols/conn/polling.bro)
rest_target(${psd} base/protocols/dhcp/consts.bro)
rest_target(${psd} base/protocols/dhcp/main.bro)
rest_target(${psd} base/protocols/dhcp/utils.bro)
rest_target(${psd} base/protocols/dnp3/consts.bro)
rest_target(${psd} base/protocols/dnp3/main.bro)
rest_target(${psd} base/protocols/dns/consts.bro)
rest_target(${psd} base/protocols/dns/main.bro)
rest_target(${psd} base/protocols/ftp/files.bro)
rest_target(${psd} base/protocols/ftp/gridftp.bro)
rest_target(${psd} base/protocols/ftp/info.bro)
rest_target(${psd} base/protocols/ftp/main.bro)
rest_target(${psd} base/protocols/ftp/utils-commands.bro)
rest_target(${psd} base/protocols/ftp/utils.bro)
rest_target(${psd} base/protocols/http/entities.bro)
rest_target(${psd} base/protocols/http/files.bro)
rest_target(${psd} base/protocols/http/main.bro)
rest_target(${psd} base/protocols/http/utils.bro)
rest_target(${psd} base/protocols/irc/dcc-send.bro)
rest_target(${psd} base/protocols/irc/files.bro)
rest_target(${psd} base/protocols/irc/main.bro)
rest_target(${psd} base/protocols/modbus/consts.bro)
rest_target(${psd} base/protocols/modbus/main.bro)
rest_target(${psd} base/protocols/smtp/entities.bro)
rest_target(${psd} base/protocols/smtp/files.bro)
rest_target(${psd} base/protocols/smtp/main.bro)
rest_target(${psd} base/protocols/socks/consts.bro)
rest_target(${psd} base/protocols/socks/main.bro)
rest_target(${psd} base/protocols/ssh/main.bro)
rest_target(${psd} base/protocols/ssl/consts.bro)
rest_target(${psd} base/protocols/ssl/main.bro)
rest_target(${psd} base/protocols/ssl/mozilla-ca-list.bro)
rest_target(${psd} base/protocols/syslog/consts.bro)
rest_target(${psd} base/protocols/syslog/main.bro)
rest_target(${psd} base/utils/active-http.bro)
rest_target(${psd} base/utils/addrs.bro)
rest_target(${psd} base/utils/conn-ids.bro)
rest_target(${psd} base/utils/dir.bro)
rest_target(${psd} base/utils/directions-and-hosts.bro)
rest_target(${psd} base/utils/exec.bro)
rest_target(${psd} base/utils/files.bro)
rest_target(${psd} base/utils/numbers.bro)
rest_target(${psd} base/utils/paths.bro)
rest_target(${psd} base/utils/patterns.bro)
rest_target(${psd} base/utils/queue.bro)
rest_target(${psd} base/utils/site.bro)
rest_target(${psd} base/utils/strings.bro)
rest_target(${psd} base/utils/thresholds.bro)
rest_target(${psd} base/utils/time.bro)
rest_target(${psd} base/utils/urls.bro)
rest_target(${psd} policy/frameworks/communication/listen.bro)
rest_target(${psd} policy/frameworks/control/controllee.bro)
rest_target(${psd} policy/frameworks/control/controller.bro)
rest_target(${psd} policy/frameworks/dpd/detect-protocols.bro)
rest_target(${psd} policy/frameworks/dpd/packet-segment-logging.bro)
rest_target(${psd} policy/frameworks/files/detect-MHR.bro)
rest_target(${psd} policy/frameworks/files/hash-all-files.bro)
rest_target(${psd} policy/frameworks/intel/do_notice.bro)
rest_target(${psd} policy/frameworks/intel/seen/conn-established.bro)
rest_target(${psd} policy/frameworks/intel/seen/dns.bro)
rest_target(${psd} policy/frameworks/intel/seen/file-hashes.bro)
rest_target(${psd} policy/frameworks/intel/seen/file-names.bro)
rest_target(${psd} policy/frameworks/intel/seen/http-headers.bro)
rest_target(${psd} policy/frameworks/intel/seen/http-url.bro)
rest_target(${psd} policy/frameworks/intel/seen/smtp-url-extraction.bro)
rest_target(${psd} policy/frameworks/intel/seen/smtp.bro)
rest_target(${psd} policy/frameworks/intel/seen/ssl.bro)
rest_target(${psd} policy/frameworks/intel/seen/where-locations.bro)
rest_target(${psd} policy/frameworks/packet-filter/shunt.bro)
rest_target(${psd} policy/frameworks/software/version-changes.bro)
rest_target(${psd} policy/frameworks/software/vulnerable.bro)
rest_target(${psd} policy/integration/barnyard2/main.bro)
rest_target(${psd} policy/integration/barnyard2/types.bro)
rest_target(${psd} policy/integration/collective-intel/main.bro)
rest_target(${psd} policy/misc/app-stats/main.bro)
rest_target(${psd} policy/misc/app-stats/plugins/facebook.bro)
rest_target(${psd} policy/misc/app-stats/plugins/gmail.bro)
rest_target(${psd} policy/misc/app-stats/plugins/google.bro)
rest_target(${psd} policy/misc/app-stats/plugins/netflix.bro)
rest_target(${psd} policy/misc/app-stats/plugins/pandora.bro)
rest_target(${psd} policy/misc/app-stats/plugins/youtube.bro)
rest_target(${psd} policy/misc/capture-loss.bro)
rest_target(${psd} policy/misc/detect-traceroute/main.bro)
rest_target(${psd} policy/misc/known-devices.bro)
rest_target(${psd} policy/misc/load-balancing.bro)
rest_target(${psd} policy/misc/loaded-scripts.bro)
rest_target(${psd} policy/misc/profiling.bro)
rest_target(${psd} policy/misc/scan.bro)
rest_target(${psd} policy/misc/stats.bro)
rest_target(${psd} policy/misc/trim-trace-file.bro)
rest_target(${psd} policy/protocols/conn/known-hosts.bro)
rest_target(${psd} policy/protocols/conn/known-services.bro)
rest_target(${psd} policy/protocols/conn/weirds.bro)
rest_target(${psd} policy/protocols/dhcp/known-devices-and-hostnames.bro)
rest_target(${psd} policy/protocols/dns/auth-addl.bro)
rest_target(${psd} policy/protocols/dns/detect-external-names.bro)
rest_target(${psd} policy/protocols/ftp/detect-bruteforcing.bro)
rest_target(${psd} policy/protocols/ftp/detect.bro)
rest_target(${psd} policy/protocols/ftp/software.bro)
rest_target(${psd} policy/protocols/http/detect-sqli.bro)
rest_target(${psd} policy/protocols/http/detect-webapps.bro)
rest_target(${psd} policy/protocols/http/header-names.bro)
rest_target(${psd} policy/protocols/http/software-browser-plugins.bro)
rest_target(${psd} policy/protocols/http/software.bro)
rest_target(${psd} policy/protocols/http/var-extraction-cookies.bro)
rest_target(${psd} policy/protocols/http/var-extraction-uri.bro)
rest_target(${psd} policy/protocols/modbus/known-masters-slaves.bro)
rest_target(${psd} policy/protocols/modbus/track-memmap.bro)
rest_target(${psd} policy/protocols/smtp/blocklists.bro)
rest_target(${psd} policy/protocols/smtp/detect-suspicious-orig.bro)
rest_target(${psd} policy/protocols/smtp/entities-excerpt.bro)
rest_target(${psd} policy/protocols/smtp/software.bro)
rest_target(${psd} policy/protocols/ssh/detect-bruteforcing.bro)
rest_target(${psd} policy/protocols/ssh/geo-data.bro)
rest_target(${psd} policy/protocols/ssh/interesting-hostnames.bro)
rest_target(${psd} policy/protocols/ssh/software.bro)
rest_target(${psd} policy/protocols/ssl/cert-hash.bro)
rest_target(${psd} policy/protocols/ssl/expiring-certs.bro)
rest_target(${psd} policy/protocols/ssl/extract-certs-pem.bro)
rest_target(${psd} policy/protocols/ssl/known-certs.bro)
rest_target(${psd} policy/protocols/ssl/notary.bro)
rest_target(${psd} policy/protocols/ssl/validate-certs.bro)
rest_target(${psd} policy/tuning/defaults/extracted_file_limits.bro)
rest_target(${psd} policy/tuning/defaults/packet-fragments.bro)
rest_target(${psd} policy/tuning/defaults/warnings.bro)
rest_target(${psd} policy/tuning/logs-to-elasticsearch.bro)
rest_target(${psd} policy/tuning/track-all-assets.bro)
rest_target(${psd} site/local-manager.bro)
rest_target(${psd} site/local-proxy.bro)
rest_target(${psd} site/local-worker.bro)
rest_target(${psd} site/local.bro)
rest_target(${psd} test-all-policy.bro)

44
doc/scripts/README
View file

@ -1,44 +0,0 @@
This directory contains scripts and templates that can be used to automate
the generation of Bro script documentation. Several build targets are defined
by CMake and available in the top-level Makefile:
``restdoc``
This target uses Bro to parse policy scripts in order to generate
reStructuredText (reST) documentation from them. The list of scripts
for which to generate reST documentation is defined in the
``CMakeLists.txt`` file in this directory. Script documentation is
rebuild automatically if the policy script from which it is derived
or the Bro binary becomes out of date
The resulting output from this target can be found in the CMake
``build/`` directory inside ``reST`` (a symlink to
``doc/scripts/rest_output``).
``restclean``
This target removes any reST documentation that has been generated so far.
The ``genDocSourcesList.sh`` script can be run to automatically generate
``DocSourcesList.cmake``, which is the file CMake uses to define the list
of documentation targets. This script should be run after adding new
Bro script source files, and the changes commited to git.
If a script shouldn't have documentation generated for it, there's also a
blacklist manifest that can be maintained in the ``genDocSourcesList.sh``
script.
The blacklist can also be used if you want to define a certain grouping for
the script's generated docs to belong to (as opposed to the automatic grouping
the happens for script packages/directories). To do that, add the
script's name to the blacklist, then append a ``rest_target()`` to the
``statictext`` variable where the first argument is the source directory
containing the policy script to document, the second argument is the file
name of the policy script, and the third argument is the path/name of a
pre-created reST document in the ``../`` source directory to which the
``make doc`` process can append script documentation references. This
pre-created reST document should also then be linked to from the TOC tree
in ``../index.rst``.
See ``example.bro`` for an example of how to document a Bro script such that
``make doc`` will be able to produce reST/HTML documentation for it.

291
doc/scripts/example.rst
View file

@ -1,291 +0,0 @@
.. Automatically generated. Do not edit.
example.bro
===========
:download:`Original Source File <example.bro>`
Overview
--------
This is an example script that demonstrates how to document. Comments
of the form ``##!`` are for the script summary. The contents of
these comments are transferred directly into the auto-generated
`reStructuredText <http://docutils.sourceforge.net/rst.html>`_
(reST) document's summary section.
.. tip:: You can embed directives and roles within ``##``-stylized comments.
:Imports: :doc:`policy/frameworks/software/vulnerable </scripts/policy/frameworks/software/vulnerable>`
Summary
~~~~~~~
Options
#######
============================================================================ ======================================
:bro:id:`Example::an_option`: :bro:type:`set` :bro:attr:`&redef` add documentation for "an_option" here
:bro:id:`Example::option_with_init`: :bro:type:`interval` :bro:attr:`&redef`
============================================================================ ======================================
State Variables
###############
=========================================================================== =======================================
:bro:id:`Example::a_var`: :bro:type:`bool` put some documentation for "a_var" here
:bro:id:`Example::var_with_attr`: :bro:type:`count` :bro:attr:`&persistent`
:bro:id:`Example::var_without_explicit_type`: :bro:type:`string`
=========================================================================== =======================================
Types
#####
====================================================== ==========================================================
:bro:type:`Example::SimpleEnum`: :bro:type:`enum` documentation for "SimpleEnum"
goes here.
:bro:type:`Example::SimpleRecord`: :bro:type:`record` general documentation for a type "SimpleRecord"
goes here.
:bro:type:`Example::ComplexRecord`: :bro:type:`record` general documentation for a type "ComplexRecord" goes here
:bro:type:`Example::Info`: :bro:type:`record` An example record to be used with a logging stream.
====================================================== ==========================================================
Events
######
================================================= =============================================================
:bro:id:`Example::an_event`: :bro:type:`event` Summarize "an_event" here.
:bro:id:`Example::log_example`: :bro:type:`event` This is a declaration of an example event that can be used in
logging streams and is raised once for each log entry.
================================================= =============================================================
Functions
#########
=============================================== =======================================
:bro:id:`Example::a_function`: :bro:type:`func` Summarize purpose of "a_function" here.
=============================================== =======================================
Redefinitions
#############
===================================================== ========================================
:bro:type:`Log::ID`: :bro:type:`enum`
:bro:type:`Example::SimpleEnum`: :bro:type:`enum` document the "SimpleEnum" redef here
:bro:type:`Example::SimpleRecord`: :bro:type:`record` document the record extension redef here
===================================================== ========================================
Namespaces
~~~~~~~~~~
.. bro:namespace:: Example
Notices
~~~~~~~
:bro:type:`Notice::Type`
:Type: :bro:type:`enum`
.. bro:enum:: Example::Notice_One Notice::Type
any number of this type of comment
will document "Notice_One"
.. bro:enum:: Example::Notice_Two Notice::Type
any number of this type of comment
will document "Notice_Two"
.. bro:enum:: Example::Notice_Three Notice::Type
.. bro:enum:: Example::Notice_Four Notice::Type
Public Interface
----------------
Options
~~~~~~~
.. bro:id:: Example::an_option
:Type: :bro:type:`set` [:bro:type:`addr`, :bro:type:`addr`, :bro:type:`string`]
:Attributes: :bro:attr:`&redef`
:Default: ``{}``
add documentation for "an_option" here
.. bro:id:: Example::option_with_init
:Type: :bro:type:`interval`
:Attributes: :bro:attr:`&redef`
:Default: ``10.0 msecs``
State Variables
~~~~~~~~~~~~~~~
.. bro:id:: Example::a_var
:Type: :bro:type:`bool`
put some documentation for "a_var" here
.. bro:id:: Example::var_with_attr
:Type: :bro:type:`count`
:Attributes: :bro:attr:`&persistent`
.. bro:id:: Example::var_without_explicit_type
:Type: :bro:type:`string`
:Default: ``"this works"``
Types
~~~~~
.. bro:type:: Example::SimpleEnum
:Type: :bro:type:`enum`
.. bro:enum:: Example::ONE Example::SimpleEnum
and more specific info for "ONE"
can span multiple lines
.. bro:enum:: Example::TWO Example::SimpleEnum
or more info like this for "TWO"
can span multiple lines
.. bro:enum:: Example::THREE Example::SimpleEnum
documentation for "SimpleEnum"
goes here.
.. bro:type:: Example::SimpleRecord
:Type: :bro:type:`record`
field1: :bro:type:`count`
counts something
field2: :bro:type:`bool`
toggles something
general documentation for a type "SimpleRecord"
goes here.
.. bro:type:: Example::ComplexRecord
:Type: :bro:type:`record`
field1: :bro:type:`count`
counts something
field2: :bro:type:`bool`
toggles something
field3: :bro:type:`Example::SimpleRecord`
msg: :bro:type:`string` :bro:attr:`&default` = ``"blah"`` :bro:attr:`&optional`
attributes are self-documenting
general documentation for a type "ComplexRecord" goes here
.. bro:type:: Example::Info
:Type: :bro:type:`record`
ts: :bro:type:`time` :bro:attr:`&log`
uid: :bro:type:`string` :bro:attr:`&log`
status: :bro:type:`count` :bro:attr:`&log` :bro:attr:`&optional`
An example record to be used with a logging stream.
Events
~~~~~~
.. bro:id:: Example::an_event
:Type: :bro:type:`event` (name: :bro:type:`string`)
Summarize "an_event" here.
Give more details about "an_event" here.
:param name: describe the argument here
.. bro:id:: Example::log_example
:Type: :bro:type:`event` (rec: :bro:type:`Example::Info`)
This is a declaration of an example event that can be used in
logging streams and is raised once for each log entry.
Functions
~~~~~~~~~
.. bro:id:: Example::a_function
:Type: :bro:type:`function` (tag: :bro:type:`string`, msg: :bro:type:`string`) : :bro:type:`string`
Summarize purpose of "a_function" here.
Give more details about "a_function" here.
Separating the documentation of the params/return values with
empty comments is optional, but improves readability of script.
:param tag: function arguments can be described
like this
:param msg: another param
:returns: describe the return type here
Redefinitions
~~~~~~~~~~~~~
:bro:type:`Log::ID`
:Type: :bro:type:`enum`
.. bro:enum:: Example::LOG Log::ID
:bro:type:`Example::SimpleEnum`
:Type: :bro:type:`enum`
.. bro:enum:: Example::FOUR Example::SimpleEnum
and some documentation for "FOUR"
.. bro:enum:: Example::FIVE Example::SimpleEnum
also "FIVE" for good measure
document the "SimpleEnum" redef here
:bro:type:`Example::SimpleRecord`
:Type: :bro:type:`record`
field_ext: :bro:type:`string` :bro:attr:`&optional`
document the extending field here
(or here)
document the record extension redef here
Port Analysis
-------------
:ref:`More Information <common_port_analysis_doc>`
SSL::
[ports={
443/tcp,
562/tcp
}]
Packet Filter
-------------
:ref:`More Information <common_packet_filter_doc>`
Filters added::
[ssl] = tcp port 443,
[nntps] = tcp port 562

86
doc/scripts/genDocSourcesList.sh
View file

@ -1,86 +0,0 @@
#!/usr/bin/env bash
# ./genDocSourcesList.sh [output file]
#
# Run this script to a generate file that's used to tell CMake about all the
# possible scripts for which reST documentation can be created.
#
# The optional argument can be used to avoid overwriting the file CMake uses
# by default.
#
# Specific scripts can be blacklisted below when e.g. they currently aren't
# parseable or they just aren't meant to be documented.
export LC_ALL=C # Make sorting stable.
blacklist ()
{
if [[ "$blacklist" == "" ]]; then
blacklist="$1"
else
blacklist="$blacklist|$1"
fi
}
# files passed into this function are meant to be temporary workarounds
# because they're not finished or otherwise can't be loaded for some reason
tmp_blacklist ()
{
echo "Warning: temporarily blacklisted files named '$1'" 1>&2
blacklist $1
}
blacklist __load__.bro
blacklist test-all.bro
blacklist all.bro
blacklist init-default.bro
blacklist init-bare.bro
statictext="\
# DO NOT EDIT
# This file is auto-generated from the "genDocSourcesList.sh" script.
#
# This is a list of Bro script sources for which to generate reST documentation.
# It will be included inline in the CMakeLists.txt found in the same directory
# in order to create Makefile targets that define how to generate reST from
# a given Bro script.
#
# Note: any path prefix of the script (2nd argument of rest_target macro)
# will be used to derive what path under scripts/ the generated documentation
# will be placed.
set(psd \${PROJECT_SOURCE_DIR}/scripts)
rest_target(\${CMAKE_CURRENT_SOURCE_DIR} example.bro internal)
rest_target(\${psd} base/init-default.bro internal)
rest_target(\${psd} base/init-bare.bro internal)
"
if [[ $# -ge 1 ]]; then
outfile=$1
else
outfile=DocSourcesList.cmake
fi
thisdir="$( cd "$( dirname "$0" )" && pwd )"
sourcedir=${thisdir}/../..
echo "$statictext" > $outfile
bifs=`( cd ${sourcedir}/build/scripts/base && find . -name \*\.bif.bro | sort )`
for file in $bifs
do
f=${file:2}
echo "rest_target(\${CMAKE_BINARY_DIR}/scripts base/$f)" >> $outfile
done
scriptfiles=`( cd ${sourcedir}/scripts && find . -name \*\.bro | sort )`
for file in $scriptfiles
do
f=${file:2}
if [[ ! $f =~ $blacklist ]]; then
echo "rest_target(\${psd} $f)" >> $outfile
fi
done

5
doc/scripts/internal.rst
View file

@ -1,5 +0,0 @@
.. This is a stub doc to which broxygen appends during the build process
Internal Scripts
================

8
doc/scripts/scripts.rst
View file

@ -1,8 +0,0 @@
.. This is a stub doc to which broxygen appends during the build process
===============
All Bro Scripts
===============
.. toctree::
:maxdepth: 1

2
scripts/base/frameworks/reporter/main.bro
View file

@ -9,7 +9,7 @@
##! Note that this framework deals with the handling of internally generated
##! reporter messages, for the interface
##! into actually creating reporter messages from the scripting layer, use
##! the built-in functions in :doc:`/scripts/base/bif/reporter.bif`.
##! the built-in functions in :doc:`/scripts/base/bif/reporter.bif.bro`.
module Reporter;

6
scripts/base/init-bare.bro
View file

@ -399,7 +399,7 @@ type NetStats: record {
pkts_dropped: count &default=0; ##< Packets reported dropped by the system.
## Packets seen on the link. Note that this may differ
## from *pkts_recvd* because of a potential capture_filter. See
## :doc:`/scripts/base/frameworks/packet-filter/main`. Depending on the
## :doc:`/scripts/base/frameworks/packet-filter/main.bro`. Depending on the
## packet capture system, this value may not be available and will then
## be always set to zero.
pkts_link: count &default=0;
@ -2768,13 +2768,13 @@ const log_max_size = 0.0 &redef;
const log_encryption_key = "<undefined>" &redef;
## Write profiling info into this file in regular intervals. The easiest way to
## activate profiling is loading :doc:`/scripts/policy/misc/profiling`.
## activate profiling is loading :doc:`/scripts/policy/misc/profiling.bro`.
##
## .. bro:see:: profiling_interval expensive_profiling_multiple segment_profiling
global profiling_file: file &redef;
## Update interval for profiling (0 disables). The easiest way to activate
## profiling is loading :doc:`/scripts/policy/misc/profiling`.
## profiling is loading :doc:`/scripts/policy/misc/profiling.bro`.
##
## .. bro:see:: profiling_file expensive_profiling_multiple segment_profiling
const profiling_interval = 0 secs &redef;

2
scripts/base/protocols/dhcp/main.bro
View file

@ -4,7 +4,7 @@
##!
##! If you'd like to track known DHCP devices and to log the hostname
##! supplied by the client, see
##! :doc:`/scripts/policy/protocols/dhcp/known-devices-and-hostnames`.
##! :doc:`/scripts/policy/protocols/dhcp/known-devices-and-hostnames.bro`.
@load ./utils.bro

2
scripts/base/protocols/http/__load__.bro
View file

@ -3,4 +3,4 @@
@load ./utils
@load ./files
@load-sigs ./dpd.sig
@load-sigs ./dpd.sig

4
scripts/broxygen/README Normal file
View file

@ -0,0 +1,4 @@
This package is loaded during the process which automatically generates
reference documentation for all Bro scripts (i.e. "Broxygen"). Its only
purpose is to provide an easy way to load all known Bro scripts plus any
extra scripts needed or used by the documentation process.

2
scripts/broxygen/__load__.bro Normal file
View file

@ -0,0 +1,2 @@
@load test-all-policy.bro
@load ./example.bro

0
doc/scripts/example.bro → scripts/broxygen/example.bro
View file

2
scripts/policy/misc/known-devices.bro
View file

@ -7,7 +7,7 @@
##!
##! This script will not generate any logs on its own, it needs to be
##! supplied with information from elsewhere, such as
##! :doc:`/scripts/policy/protocols/dhcp/known-devices-and-hostnames`.
##! :doc:`/scripts/policy/protocols/dhcp/known-devices-and-hostnames.bro`.
module Known;

2
scripts/policy/misc/stats.bro
View file

@ -1,5 +1,5 @@
##! Log memory/packet/lag statistics. Differs from
##! :doc:`/scripts/policy/misc/profiling` in that this
##! :doc:`/scripts/policy/misc/profiling.bro` in that this
##! is lighter-weight (much less info, and less load to generate).
@load base/frameworks/notice

4
src/BroDoc.cc
View file

@ -611,7 +611,7 @@ void CreateProtoAnalyzerDoc(FILE* f)
fprintf(f, "Protocol Analyzers\n");
fprintf(f, "==================\n\n");
fprintf(f, ".. contents::\n");
fprintf(f, " :depth: 1\n\n");
fprintf(f, " :depth: 2\n\n");
WriteAnalyzerTagDefn(f, "Analyzer");
@ -641,7 +641,7 @@ void CreateFileAnalyzerDoc(FILE* f)
fprintf(f, "File Analyzers\n");
fprintf(f, "==============\n\n");
fprintf(f, ".. contents::\n");
fprintf(f, " :depth: 1\n\n");
fprintf(f, " :depth: 2\n\n");
WriteAnalyzerTagDefn(f, "Files");

8
src/Type.cc
View file

@ -1165,9 +1165,9 @@ void RecordType::DescribeFieldsReST(ODesc* d, bool func_args) const
field_from_script != type_from_script )
{
d->PushIndent();
d->Add(fmt("(from ``redef`` in :doc:`%s`)",
d->Add(fmt("(from ``redef`` in :doc:`/scripts/%s`)",
field_from_script.c_str()));
d->PopIndentNoNL();
d->PopIndent();
}
vector<string> cmnts = doc->GetFieldComments(td->id);
@ -1513,9 +1513,9 @@ void EnumType::DescribeReST(ODesc* d, bool roles_only) const
{
d->NL();
d->PushIndent();
d->Add(fmt("(from ``redef`` in :doc:`%s`)",
d->Add(fmt("(from ``redef`` in :doc:`/scripts/%s`)",
enum_from_script.c_str()));
d->PopIndentNoNL();
d->PopIndent();
}
vector<string> cmnts = doc->GetComments();

126
src/broxygen/Configuration.cc
View file

@ -9,7 +9,10 @@
#include <algorithm>
#include <map>
#include <cstdio>
#include <sys/types.h>
#include <sys/stat.h>
#include <fts.h>
#include <unistd.h>
using namespace broxygen;
using namespace std;
@ -160,24 +163,38 @@ void PackageTarget::DoFindDependencies(const vector<Document*>& docs)
void PackageTarget::DoGenerate() const
{
if ( broxygen_mgr->IsUpToDate(Name(), script_deps) )
if ( broxygen_mgr->IsUpToDate(Name(), script_deps) &&
broxygen_mgr->IsUpToDate(Name(), pkg_deps) )
return;
TargetFile file(Name());
fprintf(file.f, ":orphan:\n\n");
for ( manifest_t::const_iterator it = pkg_manifest.begin();
it != pkg_manifest.end(); ++it )
{
fprintf(file.f, "Package: %s\n\n", it->first->Name().c_str());
string header = fmt("Package: %s", it->first->Name().c_str());
header += "\n" + string(header.size(), '=');
fprintf(file.f, "%s\n\n", header.c_str());
vector<string> readme = it->first->GetReadme();
for ( size_t i = 0; i < readme.size(); ++i )
fprintf(file.f, "%s\n", readme[i].c_str());
fprintf(file.f, "\n");
for ( size_t i = 0; i < it->second.size(); ++i )
{
fprintf(file.f, " :doc:`%s`\n", it->second[i]->Name().c_str());
fprintf(file.f, ":doc:`/scripts/%s`\n",
it->second[i]->Name().c_str());
vector<string> cmnts = it->second[i]->GetComments();
for ( size_t j = 0; j < cmnts.size(); ++j )
fprintf(file.f, " %s\n", cmnts[j].c_str());
fprintf(file.f, " %s\n", cmnts[j].c_str());
fprintf(file.f, "\n");
}
@ -211,25 +228,82 @@ void ScriptTarget::DoFindDependencies(const vector<Document*>& docs)
if ( script_deps.empty() )
reporter->FatalError("No match for Broxygen target '%s' pattern '%s'",
Name().c_str(), Pattern().c_str());
if ( ! IsDir() )
return;
for ( size_t i = 0; i < script_deps.size(); ++i )
{
if ( SafeBasename(script_deps[i]->Name()).result == PACKAGE_LOADER )
{
string pkg_dir = SafeDirname(script_deps[i]->Name()).result;
string target_file = Name() + pkg_dir + "/index.rst";
Target* t = PackageTarget::Instantiate(target_file, pkg_dir);
t->FindDependencies(docs);
pkg_deps.push_back(t);
}
}
}
vector<string> dir_contents_recursive(string dir)
{
vector<string> rval;
struct stat st;
if ( stat(dir.c_str(), &st) < 0 && errno == ENOENT )
return rval;
while ( dir[dir.size() - 1] == '/' )
dir.erase(dir.size() - 1, 1);
char* dir_copy = copy_string(dir.c_str());
char** scan_path = new char*[2];
scan_path[0] = dir_copy;
scan_path[1] = 0;
FTS* fts = fts_open(scan_path, FTS_NOCHDIR, 0);
if ( ! fts )
{
reporter->Error("fts_open failure: %s", strerror(errno));
delete [] scan_path;
delete [] dir_copy;
return rval;
}
FTSENT* n;
while ( (n = fts_read(fts)) )
{
if ( n->fts_info & FTS_F )
rval.push_back(n->fts_path);
}
if ( errno )
reporter->Error("fts_read failure: %s", strerror(errno));
if ( fts_close(fts) < 0 )
reporter->Error("fts_close failure: %s", strerror(errno));
delete [] scan_path;
delete [] dir_copy;
return rval;
}
void ScriptTarget::DoGenerate() const
{
if ( Name()[Name().size() - 1] == '/' )
if ( IsDir() )
{
// Target name is a dir, matching scripts are written within that dir
// with a dir tree that parallels the script's BROPATH location.
set<string> targets;
vector<string> dir_contents = dir_contents_recursive(Name());
for ( size_t i = 0; i < script_deps.size(); ++i )
{
string target_filename = Name() + script_deps[i]->Name();
size_t pos = target_filename.rfind(".bro");
if ( pos == target_filename.size() - 4 )
target_filename.replace(pos, 4, ".rst");
else
target_filename += ".rst";
string target_filename = Name() + script_deps[i]->Name() + ".rst";
targets.insert(target_filename);
vector<ScriptDocument*> dep;
dep.push_back(script_deps[i]);
@ -241,9 +315,26 @@ void ScriptTarget::DoGenerate() const
fprintf(file.f, "%s\n", script_deps[i]->ReStructuredText().c_str());
}
// TODO: could possibly take inventory of files in the dir beforehand,
// track all files written, then compare afterwards in order to remove
// stale files.
for ( size_t i = 0; i < pkg_deps.size(); ++i )
{
targets.insert(pkg_deps[i]->Name());
pkg_deps[i]->Generate();
}
for ( size_t i = 0; i < dir_contents.size(); ++i )
{
string f = dir_contents[i];
if ( targets.find(f) != targets.end() )
continue;
if ( unlink(f.c_str()) < 0 )
reporter->Warning("Failed to unlink %s: %s", f.c_str(),
strerror(errno));
DBG_LOG(DBG_BROXYGEN, "Delete stale script file %s", f.c_str());
}
return;
}
@ -300,7 +391,8 @@ void ScriptIndexTarget::DoGenerate() const
if ( ! d )
continue;
fprintf(file.f, " %s <%s>\n", d->Name().c_str(), d->Name().c_str());
fprintf(file.f, " %s </scripts/%s>\n", d->Name().c_str(),
d->Name().c_str());
}
}

8
src/broxygen/Configuration.h
View file

@ -146,6 +146,9 @@ public:
const std::string& pattern)
{ return new ScriptTarget(name, pattern); }
~ScriptTarget()
{ for ( size_t i = 0; i < pkg_deps.size(); ++i ) delete pkg_deps[i]; }
protected:
ScriptTarget(const std::string& name, const std::string& pattern)
@ -159,6 +162,11 @@ private:
void DoFindDependencies(const std::vector<Document*>& docs);
void DoGenerate() const;
bool IsDir() const
{ return Name()[Name().size() - 1] == '/'; }
std::vector<Target*> pkg_deps;
};
class ScriptSummaryTarget : public ScriptTarget {

15
src/broxygen/Document.cc
View file

@ -116,7 +116,7 @@ PackageDocument::PackageDocument(const string& arg_name)
string PackageDocument::DoReStructuredText(bool roles_only) const
{
string rval = fmt(":doc:`%s <%s/index>`\n\n", pkg_name.c_str(),
string rval = fmt(":doc:`%s </scripts/%s/index>`\n\n", pkg_name.c_str(),
pkg_name.c_str());
for ( size_t i = 0; i < readme.size(); ++i )
@ -629,15 +629,20 @@ string ScriptDocument::DoReStructuredText(bool roles_only) const
if ( it != dependencies.begin() )
rval += ", ";
rval += fmt(":doc:`%s </scripts/%s>`", it->c_str(), it->c_str());
// TODO linking to packages is a bit different?
string path = find_file(*it, bro_path(), "bro");
string doc = *it;
if ( ! path.empty() && is_dir(path.c_str()) )
// Reference the package.
doc += "/index";
rval += fmt(":doc:`%s </scripts/%s>`", it->c_str(), doc.c_str());
}
rval += "\n";
}
// TODO: make this an absolute path?
rval += fmt(":Source File: :download:`%s`\n", name.c_str());
rval += fmt(":Source File: :download:`/scripts/%s`\n", name.c_str());
rval += "\n";
rval += make_heading("Summary", '~');
rval += make_summary("Options", '#', '=', options);

2
src/const.bif
View file

@ -1,6 +1,6 @@
##! Declaration of various scripting-layer constants that the Bro core uses
##! internally. Documentation and default values for the scripting-layer
##! variables themselves are found in :doc:`/scripts/base/init-bare`.
##! variables themselves are found in :doc:`/scripts/base/init-bare.bro`.
const ignore_keep_alive_rexmit: bool;
const skip_http_data: bool;

2
src/event.bif
View file

@ -360,7 +360,7 @@ event content_gap%(c: connection, is_orig: bool, seq: count, length: count%);
##
## .. note::
##
## Bro comes with a script :doc:`/scripts/policy/misc/capture-loss` that uses
## Bro comes with a script :doc:`/scripts/policy/misc/capture-loss.bro` that uses
## this event to estimate packet loss and report when a predefined threshold
## is exceeded.
event gap_report%(dt: interval, info: gap_info%);

2
src/reporter.bif
View file

@ -4,7 +4,7 @@
##! If event handlers do exist, it's assumed they take care of determining
##! how/where to output the messages.
##!
##! See :doc:`/scripts/base/frameworks/reporter/main` for a convenient
##! See :doc:`/scripts/base/frameworks/reporter/main.bro` for a convenient
##! reporter message logging framework.
module Reporter;

5
testing/btest/Makefile
View file

@ -21,12 +21,9 @@ cleanup:
@rm -f $(DIAG)
@rm -f .tmp/script-coverage*
update-doc-sources:
../../doc/scripts/genDocSourcesList.sh ../../doc/scripts/DocSourcesList.cmake
# Updates the three coverage tests that usually need tweaking when
# scripts get added/removed.
update-coverage-tests: update-doc-sources
update-coverage-tests:
btest -qU coverage.bare-load-baseline
btest -qU coverage.default-load-baseline
@echo "Use 'git diff' to check updates look right."

8
testing/btest/coverage/doc.test
View file

@ -1,8 +0,0 @@
# This tests that we're generating bro script documentation for all the
# available bro scripts. If this fails, then the genDocSources.sh needs
# to be run to produce a new DocSourcesList.cmake or genDocSources.sh needs
# to be updated to blacklist undesired scripts. To update, run
# "make update-doc-sources"
#
# @TEST-EXEC: $DIST/doc/scripts/genDocSourcesList.sh
# @TEST-EXEC: cmp $DIST/doc/scripts/DocSourcesList.cmake ./DocSourcesList.cmake