diff --git a/INSTALL b/INSTALL index d1351a502f..a1b1cd99cf 100644 --- a/INSTALL +++ b/INSTALL @@ -39,9 +39,9 @@ Installation To build and install into ``/usr/local/bro``:: - > ./configure - > make - > make install + ./configure + make + make install This will first build Bro into a directory inside the distribution called ``build/``, using default build options. It then installs all @@ -50,7 +50,7 @@ required files into ``/usr/local/bro``, including the Bro binary in You can specify a different installation directory with:: - > ./configure --prefix= + ./configure --prefix= Note that ``/usr`` and ``/opt/bro`` are the standard prefixes for binary Bro packages to be installed, so those are typically not good @@ -71,11 +71,8 @@ Running Bro =========== Bro is a complex program and it takes a bit of time to get familiar -with it. A good place for newcomers to start is the quick start guide -available here: - - http://www.bro-ids.org/documentation/quickstart.html - +with it. A good place for newcomers to start is the +:doc:`quick start guide `. For developers that wish to run Bro directly from the ``build/`` directory (i.e., without performing ``make install``), they will have @@ -83,9 +80,9 @@ to first adjust ``BROPATH`` to look for scripts inside the build directory. Sourcing either ``build/bro-path-dev.sh`` or ``build/bro-path-dev.csh`` as appropriate for the current shell accomplishes this and also augments your ``PATH`` so you can use the -Bro binary directly: +Bro binary directly:: - > ./configure - > make - > source build/bro-path-dev.sh - > bro + ./configure + make + source build/bro-path-dev.sh + bro diff --git a/doc/CHANGES b/doc/CHANGES new file mode 120000 index 0000000000..3e8bc8c0c8 --- /dev/null +++ b/doc/CHANGES @@ -0,0 +1 @@ +../CHANGES \ No newline at end of file diff --git a/doc/CMakeLists.txt b/doc/CMakeLists.txt index acef059ce8..7685230980 100644 --- a/doc/CMakeLists.txt +++ b/doc/CMakeLists.txt @@ -1,4 +1,73 @@ -add_custom_target(doc) -add_custom_target(docclean) +set(BIF_SRC_DIR ${PROJECT_SOURCE_DIR}/src) +set(RST_OUTPUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/rest_output) +set(DOC_OUTPUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/out) +set(DOC_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}) +set(DOC_SOURCE_WORKDIR ${CMAKE_CURRENT_BINARY_DIR}/sphinx-sources) + +set(MASTER_POLICY_INDEX ${CMAKE_CURRENT_BINARY_DIR}/scripts/policy_index) +set(MASTER_PACKAGE_INDEX ${CMAKE_CURRENT_BINARY_DIR}/scripts/pkg_index) + +file(GLOB_RECURSE DOC_SOURCES FOLLOW_SYMLINKS "*") + +# configure the Sphinx config file (expand variables CMake might know about) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/conf.py.in + ${CMAKE_CURRENT_BINARY_DIR}/conf.py + @ONLY) add_subdirectory(scripts) + +# The "broxygen" target generates reST documentation for any outdated bro +# scripts and then uses Sphinx to generate HTML documentation from the reST +add_custom_target(broxygen + # copy the template documentation to the build directory + # to give as input for sphinx + COMMAND "${CMAKE_COMMAND}" -E copy_directory + ${DOC_SOURCE_DIR} + ${DOC_SOURCE_WORKDIR} + # copy generated policy script documentation into the + # working copy of the template documentation + COMMAND "${CMAKE_COMMAND}" -E copy_directory + ${RST_OUTPUT_DIR} + ${DOC_SOURCE_WORKDIR}/scripts + # append to the master index of all policy scripts + COMMAND cat ${MASTER_POLICY_INDEX} >> + ${DOC_SOURCE_WORKDIR}/scripts/index.rst + # append to the master index of all policy packages + COMMAND cat ${MASTER_PACKAGE_INDEX} >> + ${DOC_SOURCE_WORKDIR}/scripts/packages.rst + # construct a reST file for each group + COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/bin/group_index_generator.py + ${CMAKE_CURRENT_BINARY_DIR}/scripts/group_list + ${CMAKE_CURRENT_BINARY_DIR}/scripts + ${DOC_SOURCE_WORKDIR}/scripts + # tell sphinx to generate html + COMMAND sphinx-build + -b html + -c ${CMAKE_CURRENT_BINARY_DIR} + -d ${DOC_OUTPUT_DIR}/doctrees + ${DOC_SOURCE_WORKDIR} + ${DOC_OUTPUT_DIR}/html + # create symlink to the html output directory for convenience + COMMAND "${CMAKE_COMMAND}" -E create_symlink + ${DOC_OUTPUT_DIR}/html + ${CMAKE_BINARY_DIR}/html + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + COMMENT "[Sphinx] Generating HTML policy script docs" + # SOURCES just adds stuff to IDE projects as a convenience + SOURCES ${DOC_SOURCES}) + +# The "sphinxclean" target removes just the Sphinx input/output directories +# from the build directory. +add_custom_target(broxygen-clean + COMMAND "${CMAKE_COMMAND}" -E remove_directory + ${DOC_SOURCE_WORKDIR} + COMMAND "${CMAKE_COMMAND}" -E remove_directory + ${DOC_OUTPUT_DIR} + VERBATIM) + +add_dependencies(broxygen broxygen-clean restdoc) + +add_custom_target(doc) +add_custom_target(docclean) +add_dependencies(doc broxygen) +add_dependencies(docclean broxygen-clean restclean) diff --git a/doc/INSTALL.rst b/doc/INSTALL.rst new file mode 120000 index 0000000000..99d491b4f8 --- /dev/null +++ b/doc/INSTALL.rst @@ -0,0 +1 @@ +../INSTALL \ No newline at end of file diff --git a/doc/README b/doc/README index 1e30d5ccc9..a9f2cdbc4e 100644 --- a/doc/README +++ b/doc/README @@ -5,29 +5,37 @@ Documentation This directory contains Bro documentation in reStructured text format (see http://docutils.sourceforge.net/rst.html). -Please note that for now these files are primarily intended for use on -http://www.bro-ids.org. While the Bro build process will render local -versions into ``build/doc/`` (if docutils is found), the resulting -HTML is very minimalistic and some features are not supported. In -particular, some links will be broken. +It is the root of a Sphinx source tree and can be modified to add more +common/general documentation, style sheets, JavaScript, etc. The Sphinx +config file is produced from ``conf.py.in``, and can be edited to change +various Sphinx options. +There is also a custom Sphinx domain implemented in ``source/ext/bro.py`` +which adds some reST directives and roles that aid in generating useful +index entries and cross-references. Other extensions can be added in +a similar fashion. + +Either the ``make doc`` or ``make broxygen`` can be used to locally +render the reST files into HTML. Those targets depend on: + +* Python interpreter >= 2.5 +* `Sphinx `_ >= 1.0.1 + +After completion, HTML documentation is symlinked in ``build/html``. + +There's also ``make docclean`` and ``make broxygenclean`` targets to +clean the resulting documentation. Notes for Writing Documentation ------------------------------- -* If you want to refer to a Bro script that's part of the - distribution, use {{'`foo.bro - <{{autodoc_bro_scripts}}/path/to/foo.html>`_'}}. For example, - ``{{'{{autodoc_bro_scripts}}/scripts/base/frameworks/notice/main.html}}'}}``. +* If you want to refer to a document that's part of the + distribution, it currently needs to be copied or otherwise symlinked + somewhere in to this Sphinx source tree. Then, it can be referenced + in a toc tree or with the :doc: role. Use the :download: role to + refer to static files that will not undergo sphinx rendering. -* If you want to refer to a page on the Bro web site, use the - ``docroot`` macro (e.g., - ``{{'href="{{docroot}}/download/index.html"'}}). Make sure to - include the ``index.html`` for the main pages, just as in the - example. - -* If you want to refer to page inside this directory, use a relative - path with HTML extension. (e.g., ``href="quickstart.html``). +* If you want to refer to a page on the Bro web site, use an HTTP URL. Guidelines ---------- diff --git a/doc/_static/broxygen.css b/doc/_static/broxygen.css new file mode 100644 index 0000000000..d9c39a8506 --- /dev/null +++ b/doc/_static/broxygen.css @@ -0,0 +1,16 @@ +a:hover +{ + text-decoration:none; + color:#c24444; +} + +div.body h1, div.body h2, div.body h3, div.body h4, div.body h5, div.body h6 +{ +background-color:#ffffff; +border-bottom: 1px solid #aaa; +} + +th.field-name +{ +white-space:nowrap; +} diff --git a/doc/scripts/source/_static/showhide.js b/doc/_static/showhide.js similarity index 100% rename from doc/scripts/source/_static/showhide.js rename to doc/_static/showhide.js diff --git a/doc/_templates/layout.html b/doc/_templates/layout.html new file mode 100644 index 0000000000..90c448037e --- /dev/null +++ b/doc/_templates/layout.html @@ -0,0 +1,10 @@ +{% extends "!layout.html" %} + +{% set css_files = css_files + ["_static/broxygen.css"] %} + +{% block extrahead %} + + +{% endblock %} + +{% block relbar2 %}{% endblock %} diff --git a/doc/scripts/group_index_generator.py b/doc/bin/group_index_generator.py similarity index 100% rename from doc/scripts/group_index_generator.py rename to doc/bin/group_index_generator.py diff --git a/doc/cluster.rst b/doc/cluster.rst index afd5f2fe35..9bffc35fec 100644 --- a/doc/cluster.rst +++ b/doc/cluster.rst @@ -11,9 +11,7 @@ Architecture The figure below illustrates the main components of a Bro cluster. -.. {{git_pull('bro:doc/deployment.png')}} - -.. image:: deployment.bro.png +.. image:: images/deployment.png Tap *** diff --git a/doc/components/broccoli-python/README.rst b/doc/components/broccoli-python/README.rst new file mode 120000 index 0000000000..4187e87202 --- /dev/null +++ b/doc/components/broccoli-python/README.rst @@ -0,0 +1 @@ +../../../aux/broccoli/bindings/broccoli-python/README \ No newline at end of file diff --git a/doc/components/broccoli/README.rst b/doc/components/broccoli/README.rst new file mode 120000 index 0000000000..d32c70ccd9 --- /dev/null +++ b/doc/components/broccoli/README.rst @@ -0,0 +1 @@ +../../../aux/broccoli/README \ No newline at end of file diff --git a/doc/components/broctl/README.rst b/doc/components/broctl/README.rst new file mode 120000 index 0000000000..cba305f48a --- /dev/null +++ b/doc/components/broctl/README.rst @@ -0,0 +1 @@ +../../../aux/broctl/doc/broctl.rst \ No newline at end of file diff --git a/doc/components/btest/README.rst b/doc/components/btest/README.rst new file mode 120000 index 0000000000..0da2935df1 --- /dev/null +++ b/doc/components/btest/README.rst @@ -0,0 +1 @@ +../../../aux/btest/README \ No newline at end of file diff --git a/doc/components/capstats/README.rst b/doc/components/capstats/README.rst new file mode 120000 index 0000000000..cb2380145d --- /dev/null +++ b/doc/components/capstats/README.rst @@ -0,0 +1 @@ +../../../aux/broctl/aux/capstats/README \ No newline at end of file diff --git a/doc/components/pysubnettree/README.rst b/doc/components/pysubnettree/README.rst new file mode 120000 index 0000000000..42ce17d303 --- /dev/null +++ b/doc/components/pysubnettree/README.rst @@ -0,0 +1 @@ +../../../aux/broctl/aux/pysubnettree/README \ No newline at end of file diff --git a/doc/components/trace-summary/README.rst b/doc/components/trace-summary/README.rst new file mode 120000 index 0000000000..78778364bd --- /dev/null +++ b/doc/components/trace-summary/README.rst @@ -0,0 +1 @@ +../../../aux/broctl/aux/trace-summary/README \ No newline at end of file diff --git a/doc/scripts/conf.py.in b/doc/conf.py.in similarity index 87% rename from doc/scripts/conf.py.in rename to doc/conf.py.in index 419209f580..e997796eda 100644 --- a/doc/scripts/conf.py.in +++ b/doc/conf.py.in @@ -15,7 +15,7 @@ import sys, os # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. -sys.path.insert(0, os.path.abspath('source/ext')) +sys.path.insert(0, os.path.abspath('sphinx-sources/ext')) # -- General configuration ----------------------------------------------------- @@ -24,10 +24,10 @@ sys.path.insert(0, os.path.abspath('source/ext')) # Add any Sphinx extension module names here, as strings. They can be extensions # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. -extensions = ['bro'] +extensions = ['bro', 'rst_directive'] # Add any paths that contain templates here, relative to this directory. -templates_path = ['source/_templates'] +templates_path = ['sphinx-sources/_templates', 'sphinx-sources/_static'] # The suffix of source filenames. source_suffix = '.rst' @@ -95,7 +95,31 @@ html_theme = 'default' # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. -#html_theme_options = {} +html_theme_options = { +"rightsidebar": "true", +"stickysidebar": "true", +"externalrefs": "true", +"footerbgcolor": "#333", +"footertextcolor": "#ddd", +"sidebarbgcolor": "#ffffff", +#"sidebarbtncolor": "", +"sidebartextcolor": "#333", +"sidebarlinkcolor": "#2a85a7", +"relbarbgcolor": "#ffffff", +"relbartextcolor": "#333", +"relbarlinkcolor": "#2a85a7", +"bgcolor": "#ffffff", +"textcolor": "#333", +"linkcolor": "#2a85a7", +"visitedlinkcolor": "#2a85a7", +"headbgcolor": "#f0f0f0", +"headtextcolor": "#000", +"headlinkcolor": "#2a85a7", +"codebgcolor": "#FFFAE2", +#"codetextcolor": "", +"bodyfont": "Arial, Helvetica, sans-serif", +"headfont": "Palatino,'Palatino Linotype',Georgia,serif", +} # Add any paths that contain custom themes here, relative to this directory. #html_theme_path = [] @@ -119,7 +143,7 @@ html_theme = 'default' # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['source/_static'] +html_static_path = ['sphinx-sources/_static'] # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. @@ -130,7 +154,9 @@ html_static_path = ['source/_static'] #html_use_smartypants = True # Custom sidebar templates, maps document names to template names. -#html_sidebars = {} +html_sidebars = { +'**': ['localtoc.html', 'sourcelink.html', 'searchbox.html'], +} # Additional templates that should be rendered to pages, maps page names to # template names. @@ -163,7 +189,7 @@ html_static_path = ['source/_static'] #html_file_suffix = None # Output file base name for HTML help builder. -htmlhelp_basename = 'Brodoc' +htmlhelp_basename = 'Broxygen' # -- Options for LaTeX output -------------------------------------------------- diff --git a/doc/scripts/source/ext/bro.py b/doc/ext/bro.py similarity index 100% rename from doc/scripts/source/ext/bro.py rename to doc/ext/bro.py diff --git a/doc/ext/bro_lexer/__init__.py b/doc/ext/bro_lexer/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/doc/ext/bro_lexer/__init__.pyc b/doc/ext/bro_lexer/__init__.pyc new file mode 100644 index 0000000000..1b03eaf1c1 Binary files /dev/null and b/doc/ext/bro_lexer/__init__.pyc differ diff --git a/doc/ext/bro_lexer/bro.py b/doc/ext/bro_lexer/bro.py new file mode 100644 index 0000000000..8cb4475f3b --- /dev/null +++ b/doc/ext/bro_lexer/bro.py @@ -0,0 +1,76 @@ +from pygments.lexer import RegexLexer, bygroups, include +from pygments.token import * + +__all__ = ["BroLexer"] + +class BroLexer(RegexLexer): + name = 'Bro' + aliases = ['bro'] + filenames = ['*.bro'] + + _hex = r'[0-9a-fA-F_]+' + _float = r'((\d*\.?\d+)|(\d+\.?\d*))([eE][-+]?\d+)?' + _h = r'[A-Za-z0-9][-A-Za-z0-9]*' + + tokens = { + 'root': [ + # Whitespace + ('^@.*?\n', Comment.Preproc), + (r'#.*?\n', Comment.Single), + (r'\n', Text), + (r'\s+', Text), + (r'\\\n', Text), + # Keywords + (r'(add|alarm|break|case|const|continue|delete|do|else|enum|event' + r'|export|for|function|if|global|local|module|next' + r'|of|print|redef|return|schedule|when|while)\b', Keyword), + (r'(addr|any|bool|count|counter|double|file|int|interval|net' + r'|pattern|port|record|set|string|subnet|table|time|timer' + r'|vector)\b', Keyword.Type), + (r'(T|F)\b', Keyword.Constant), + (r'(&)((?:add|delete|expire)_func|attr|(create|read|write)_expire' + r'|default|disable_print_hook|raw_output|encrypt|group|log' + r'|mergeable|optional|persistent|priority|redef' + r'|rotate_(?:interval|size)|synchronized)\b', bygroups(Punctuation, + Keyword)), + (r'\s+module\b', Keyword.Namespace), + # Addresses, ports and networks + (r'\d+/(tcp|udp|icmp|unknown)\b', Number), + (r'(\d+\.){3}\d+', Number), + (r'(' + _hex + r'){7}' + _hex, Number), + (r'0x' + _hex + r'(' + _hex + r'|:)*::(' + _hex + r'|:)*', Number), + (r'((\d+|:)(' + _hex + r'|:)*)?::(' + _hex + r'|:)*', Number), + (r'(\d+\.\d+\.|(\d+\.){2}\d+)', Number), + # Hostnames + (_h + r'(\.' + _h + r')+', String), + # Numeric + (_float + r'\s+(day|hr|min|sec|msec|usec)s?\b', Literal.Date), + (r'0[xX]' + _hex, Number.Hex), + (_float, Number.Float), + (r'\d+', Number.Integer), + (r'/', String.Regex, 'regex'), + (r'"', String, 'string'), + # Operators + (r'[!%*/+-:<=>?~|]', Operator), + (r'([-+=&|]{2}|[+-=!><]=)', Operator), + (r'(in|match)\b', Operator.Word), + (r'[{}()\[\]$.,;]', Punctuation), + # Identfier + (r'([_a-zA-Z]\w*)(::)', bygroups(Name, Name.Namespace)), + (r'[a-zA-Z_][a-zA-Z_0-9]*', Name) + ], + 'string': [ + (r'"', String, '#pop'), + (r'\\([\\abfnrtv"\']|x[a-fA-F0-9]{2,4}|[0-7]{1,3})', String.Escape), + (r'[^\\"\n]+', String), + (r'\\\n', String), + (r'\\', String) + ], + 'regex': [ + (r'/', String.Regex, '#pop'), + (r'\\[\\nt/]', String.Regex), # String.Escape is too intense. + (r'[^\\/\n]+', String.Regex), + (r'\\\n', String.Regex), + (r'\\', String.Regex) + ] + } diff --git a/doc/ext/bro_lexer/bro.pyc b/doc/ext/bro_lexer/bro.pyc new file mode 100644 index 0000000000..6471e1528d Binary files /dev/null and b/doc/ext/bro_lexer/bro.pyc differ diff --git a/doc/ext/rst_directive.py b/doc/ext/rst_directive.py new file mode 100644 index 0000000000..434eef2c61 --- /dev/null +++ b/doc/ext/rst_directive.py @@ -0,0 +1,180 @@ +def setup(app): + pass + +# -*- coding: utf-8 -*- +""" + +Modified version of the the Pygments reStructuredText directive. -Robin + +This provides two new directives: + + - .. code:: [] + + Highlights the following code block according to if + given (e.g., "c", "python", etc.). + + - .. console:: + + Highlits the following code block as a shell session. + + For compatibility with the original version, "sourcecode" is + equivalent to "code". + +Original comment: + + The Pygments reStructuredText directive + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + This fragment is a Docutils_ 0.5 directive that renders source code + (to HTML only, currently) via Pygments. + + To use it, adjust the options below and copy the code into a module + that you import on initialization. The code then automatically + registers a ``sourcecode`` directive that you can use instead of + normal code blocks like this:: + + .. sourcecode:: python + + My code goes here. + + If you want to have different code styles, e.g. one with line numbers + and one without, add formatters with their names in the VARIANTS dict + below. You can invoke them instead of the DEFAULT one by using a + directive option:: + + .. sourcecode:: python + :linenos: + + My code goes here. + + Look at the `directive documentation`_ to get all the gory details. + + .. _Docutils: http://docutils.sf.net/ + .. _directive documentation: + http://docutils.sourceforge.net/docs/howto/rst-directives.html + + :copyright: Copyright 2006-2010 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +# Options +# ~~~~~~~ + +# Set to True if you want inline CSS styles instead of classes +INLINESTYLES = False + +from pygments.formatters import HtmlFormatter + +class MyHtmlFormatter(HtmlFormatter): + def format_unencoded(self, tokensource, outfile): + + # A NOP currently. + new_tokens = [] + for (i, piece) in tokensource: + new_tokens += [(i, piece)] + + return super(MyHtmlFormatter, self).format_unencoded(new_tokens, outfile) + +# The default formatter +DEFAULT = MyHtmlFormatter(noclasses=INLINESTYLES, cssclass="pygments") + +# Add name -> formatter pairs for every variant you want to use +VARIANTS = { + # 'linenos': HtmlFormatter(noclasses=INLINESTYLES, linenos=True), +} + + +import textwrap + +from docutils import nodes +from docutils.parsers.rst import directives, Directive + +from pygments import highlight +from pygments.lexers import get_lexer_by_name, guess_lexer, TextLexer +from pygments.token import Text, Keyword, Error, Operator, Name +from pygments.filter import Filter + +# Ugly hack to register the Bro lexer. I'm sure there's a better way to do it, +# but it's not obvious ... +from bro_lexer.bro import BroLexer +from pygments.lexers._mapping import LEXERS +LEXERS['BroLexer'] = ('bro_lexer.bro', BroLexer.name, BroLexer.aliases, BroLexer.filenames, ()) + +class Pygments(Directive): + """ Source code syntax hightlighting. + """ + #max_line_length = 68 + max_line_length = 0 + + required_arguments = 0 + optional_arguments = 1 + final_argument_whitespace = True + option_spec = dict([(key, directives.flag) for key in VARIANTS]) + has_content = True + + def wrapped_content(self): + content = [] + + if Console.max_line_length: + for line in self.content: + content += textwrap.wrap(line, Console.max_line_length, subsequent_indent=" ") + else: + content = self.content + + return u'\n'.join(content) + + def run(self): + self.assert_has_content() + + content = self.wrapped_content() + + if len(self.arguments) > 0: + try: + lexer = get_lexer_by_name(self.arguments[0]) + except (ValueError, IndexError): + # lexer not found, use default. + lexer = TextLexer() + else: + lexer = guess_lexer(content) + + # import sys + # print >>sys.stderr, self.arguments, lexer.__class__ + + # take an arbitrary option if more than one is given + formatter = self.options and VARIANTS[self.options.keys()[0]] or DEFAULT + parsed = highlight(content, lexer, formatter) + return [nodes.raw('', parsed, format='html')] + +class MyFilter(Filter): + def filter(self, lexer, stream): + + bol = True + + for (ttype, value) in stream: + # Color the '>' prompt sign. + if bol and ttype is Text and value == ">": + ttype = Name.Variable.Class # This gives us a nice red. + + # Discolor builtin, that can look funny. + if ttype is Name.Builtin: + ttype = Text + + bol = value.endswith("\n") + + yield (ttype, value) + +class Console(Pygments): + required_arguments = 0 + optional_arguments = 0 + + def run(self): + self.assert_has_content() + content = self.wrapped_content() + lexer = get_lexer_by_name("sh") + lexer.add_filter(MyFilter()) + parsed = highlight(content, lexer, DEFAULT) + return [nodes.raw('', parsed, format='html')] + +directives.register_directive('sourcecode', Pygments) +directives.register_directive('code', Pygments) +directives.register_directive('console', Console) diff --git a/doc/faq.rst b/doc/faq.rst new file mode 100644 index 0000000000..bdb1f50292 --- /dev/null +++ b/doc/faq.rst @@ -0,0 +1,145 @@ + +========================== +Frequently Asked Questions +========================== + +.. raw:: html + +
+ +.. contents:: + +Installation and Configuration +============================== + +How can I tune my operating system for best capture performance? +---------------------------------------------------------------- + +Here are some pointers to more information: + +* Fabian Schneider's research on `high performance packet capture + `_ + +* `NSMWiki `_ has page on + *Collecting Data*. + +* An `IMC 2010 paper + `_ by + Lothar Braun et. al evaluates packet capture performance on + commodity hardware + +What does an error message like ``internal error: NB-DNS error`` mean? +--------------------------------------------------------------------------------------------------------------------------------- + +That often means that DNS is not set up correctly on the system +running Bro. Try verifying from the command line that DNS lookups +work, e.g., ``host www.google.com``. + + +Usage +===== + +How can I identify backscatter? +------------------------------- + +Identifying backscatter via connections labeled as ``OTH`` is not +a reliable means to detect backscatter. Use rather the following +procedure: + +* Enable connection history via ``redef record_state_history=T`` to + track all control/data packet types in connection logs. + +* Backscatter is now visible in terms of connections that never had an + initial ``SYN`` but started instead with a ``SYN-ACK`` or ``RST`` + (though this latter generally is just discarded). + +Is there help for understanding Bro's resource consumption? +----------------------------------------------------------- + +There are two scripts that collect statistics on resource usage: +``stats.bro`` and ``profiling.bro``. The former is quite lightweight, +while the latter should only be used for debugging. Furthermore, +there's also ``print-globals.bro``, which prints the size of all +global script variable at termination. + +How can I capture packets as an unprivileged user? +-------------------------------------------------- + +Normally, unprivileged users cannot capture packets from a network +interface, which means they would not be able to use Bro to read/analyze +live traffic. However, there are ways to enable packet capture +permission for non-root users, which is worth doing in the context of +using Bro to monitor live traffic + +With Linux Capabilities +^^^^^^^^^^^^^^^^^^^^^^^ + +Fully implemented since Linux kernel 2.6.24, capabilities are a way of +parceling superuser privileges into distinct units. Attach capabilities +required to capture packets to the ``bro`` executable file like this: + +.. console:: + + sudo setcap cap_net_raw,cap_net_admin=eip /path/to/bro + +Now any unprivileged user should have the capability to capture packets +using Bro provided that they have the traditional file permissions to +read/execute the ``bro`` binary. + +With BPF Devices +^^^^^^^^^^^^^^^^ + +Systems using Berkeley Packet Filter (BPF) (e.g. FreeBSD & Mac OS X) +can allow users with read access to a BPF device to capture packets from +it using libpcap. + +* Example of manually changing BPF device permissions to allow users in + the ``admin`` group to capture packets: + +.. console:: + + sudo chgrp admin /dev/bpf* + sudo chmod g+r /dev/bpf* + +* Example of configuring devfs to set permissions of BPF devices, adding + entries to ``/etc/devfs.conf`` to grant ``admin`` group permission to + capture packets: + +.. console:: + + sudo sh -c 'echo "own bpf root:admin" >> /etc/devfs.conf' + sudo sh -c 'echo "perm bpf 0640" >> /etc/devfs.conf' + sudo service devfs restart + +.. note:: As of Mac OS X 10.6, the BPF device is on devfs, but the used version + of devfs isn't capable of setting the device permissions. The permissions + can be changed manually, but they will not survive a reboot. + +Why isn't Bro producing the logs I expect? (A Note About Checksums) +------------------------------------------------------------------- + +Normally, Bro's event engine will discard packets which don't have valid +checksums. This can be a problem if one wants to analyze locally +generated/captured traffic on a system that offloads checksumming to the +network adapter. In that case, all transmitted/captured packets will have +bad checksums because they haven't yet been calculated by the NIC, thus +such packets will not undergo analysis defined in Bro policy scripts as they +normally would. Bad checksums in traces may also be a result of some packet +alteration tools. + +Bro has two options to workaround such situations and ignore bad checksums: + +1) The ``-C`` command line option to ``bro``. +2) An option called ``ignore_checksums`` that can be redefined at the policy + policy script layer (e.g. in your ``$PREFIX/share/bro/site/local/bro``): + + .. code:: bro + + redef ignore_checksums = T; + +The other alternative is to disable checksum offloading for your +network adapter, but this is not always possible or desirable. + +.. raw:: html + +
diff --git a/doc/geoip.rst b/doc/geoip.rst index 53413b5bee..bd9ae0c08d 100644 --- a/doc/geoip.rst +++ b/doc/geoip.rst @@ -3,7 +3,7 @@ GeoLocation =========== -.. class:: opening +.. rst-class:: opening During the process of creating policy scripts the need may arise to find the geographic location for an IP address. Bro has support diff --git a/doc/deployment.png b/doc/images/deployment.png similarity index 100% rename from doc/deployment.png rename to doc/images/deployment.png diff --git a/doc/index.rst b/doc/index.rst index c7c85bd21f..022552f3d4 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -1,50 +1,72 @@ - +.. Bro documentation master file Bro Documentation ================= -`Getting Started <{{git('bro:doc/quickstart.rst')}}>`_ - A quick introduction into using Bro 2.x. +.. toctree:: + :maxdepth: 1 -`Bro 1.5 to 2.0 Upgrade Guide <{{git('bro:doc/upgrade.rst')}}>`_ - Guidelines and notes about upgrading from Bro 1.5 to 2.x. Lots of - things have changed, so make sure to read this when upgrading. - -`BroControl <{{git('broctl:doc/broctl.rst')}}>`_ - An interactive console for managing Bro installations. - -`Script Reference <{{autodoc_bro_scripts}}/index.html>`_ - A complete reference of all policy scripts shipped with Bro. - -`FAQ <{{docroot}}/documentation/faq.html>`_ - A list with frequently asked questions. - -`How to Report a Problem <{{docroot}}/documentation/reporting-problems.html>`_ - Some advice for when you see Bro doing something you believe it - shouldn't. + INSTALL + quickstart + upgrade + faq + reporting-problems Frameworks ---------- -Bro comes with a number of frameworks, some of which are described in -more detail here: +.. toctree:: + :maxdepth: 1 -`Notice <{{git('bro:doc/notice.rst')}}>`_ - The notice framework. - -`Logging <{{git('bro:doc/logging.rst')}}>`_ - Customizing and extensing Bro's logging. - -`Cluster <{{git('bro:doc/cluster.rst')}}>`_ - Setting up a Bro Cluster when a single box can't handle the traffic anymore. - -`Signatures <{{git('bro:doc/signatures.rst')}}>`_ - Bro has support for traditional NIDS signatures as well. + notice + logging + cluster + signatures How-Tos ------- -We also collect more specific How-Tos on specific topics: +.. toctree:: + :maxdepth: 1 -`Using GeoIP in Bro scripts <{{git('bro:doc/geoip.rst')}}>`_ - Installation and usage of the the GeoIP library. + geoip + +Script Reference +---------------- + +.. toctree:: + :maxdepth: 1 + + scripts/common + scripts/builtins + scripts/bifs + scripts/packages + scripts/index + +Other Bro Components +-------------------- + +.. toctree:: + :maxdepth: 1 + + components/btest/README + components/broccoli/README + components/broccoli-python/README + components/broctl/README + components/capstats/README + components/pysubnettree/README + components/trace-summary/README + +Indices and tables +------------------ + +* :ref:`genindex` +* :ref:`search` + +Internal References +------------------- + +.. toctree:: + :maxdepth: 1 + + scripts/internal diff --git a/doc/logging.rst b/doc/logging.rst index e7151b7f47..2817cadd45 100644 --- a/doc/logging.rst +++ b/doc/logging.rst @@ -2,7 +2,7 @@ Customizing Bro's Logging ========================== -.. class:: opening +.. rst-class:: opening Bro comes with a flexible key-value based logging interface that allows fine-grained control of what gets logged and how it is diff --git a/doc/notice.rst b/doc/notice.rst index 1322d44585..b1ffdacb75 100644 --- a/doc/notice.rst +++ b/doc/notice.rst @@ -2,7 +2,7 @@ Notice Framework ================ -.. class:: opening +.. rst-class:: opening One of the easiest ways to customize Bro is writing a local notice policy. Bro can detect a large number of potentially interesting diff --git a/doc/quickstart.rst b/doc/quickstart.rst index 22523e1618..852cf7e5e2 100644 --- a/doc/quickstart.rst +++ b/doc/quickstart.rst @@ -3,12 +3,13 @@ .. _MacPorts: http://www.macports.org .. _Fink: http://www.finkproject.org .. _Homebrew: http://mxcl.github.com/homebrew +.. _bro downloads page: http://bro-ids.org/download/index.html ================= Quick Start Guide ================= -.. class:: opening +.. rst-class:: opening The short story for getting Bro up and running in a simple configuration for analysis of either live traffic from a network interface or a packet @@ -26,20 +27,19 @@ source code forms. Pre-Built Binary Release Packages --------------------------------- -See the `downloads page <{{docroot}}/download/index.html>`_ for currently -supported/targeted platforms. +See the `bro downloads page`_ for currently supported/targeted platforms. * RPM .. console:: - > sudo yum localinstall Bro-all*.rpm + sudo yum localinstall Bro-all*.rpm * DEB .. console:: - > sudo gdebi Bro-all-*.deb + sudo gdebi Bro-all-*.deb * MacOS Disk Image with Installer @@ -60,13 +60,13 @@ Required Dependencies .. console:: - > sudo yum install cmake make gcc gcc-c++ flex bison libpcap-devel openssl-devel python-devel swig + sudo yum install cmake make gcc gcc-c++ flex bison libpcap-devel openssl-devel python-devel swig * DEB/Debian-based Linux: .. console:: - > sudo apt-get install cmake make gcc g++ flex bison libpcap-dev libssl-dev python-dev swig + sudo apt-get install cmake make gcc g++ flex bison libpcap-dev libssl-dev python-dev swig * FreeBSD @@ -75,7 +75,7 @@ Required Dependencies .. console:: - > sudo pkg_add -r cmake swig bison python + sudo pkg_add -r cmake swig bison python * Mac OS X @@ -102,21 +102,21 @@ and sendmail for sending emails. .. console:: - > sudo yum install zlib-devel file-devel GeoIP-devel sendmail + sudo yum install zlib-devel file-devel GeoIP-devel sendmail * DEB/Debian-based Linux: .. console:: - > sudo apt-get install zlib1g-dev libmagic-dev libgeoip-dev sendmail + sudo apt-get install zlib1g-dev libmagic-dev libgeoip-dev sendmail * Ports-based FreeBSD .. console:: - > sudo pkg_add -r GeoIP + sudo pkg_add -r GeoIP - libz, libmagic, and sendmail are typically already available. +libz, libmagic, and sendmail are typically already available. * Mac OS X @@ -125,20 +125,20 @@ and sendmail for sending emails. Fink Homebrew), they should be automatically detected and Bro will compile against them. -Additional steps may be needed to `get the right GeoIP database -<{{git('bro:doc/geoip.rst')}}>`_. +Additional steps may be needed to :doc:`get the right GeoIP database ` Compiling Bro Source Code ~~~~~~~~~~~~~~~~~~~~~~~~~ Bro releases are bundled into source packages for convenience and -available from the `downloads page <{{docroot}}/download/index.html>`_. +available from the `bro downloads page`_. -The latest Bro development versions are obtainable through git repositories -hosted at `git.bro-ids.org `_. See our `git development -documentation <{{docroot}}/development/process.html>`_ for comprehensive -information on Bro's use of git revision control, but the short story for -downloading the full source code experience for Bro via git is: +The latest Bro development versions are obtainable through git +repositories hosted at `git.bro-ids.org `_. See +our `git development documentation +`_ for comprehensive +information on Bro's use of git revision control, but the short story +for downloading the full source code experience for Bro via git is: .. console:: @@ -157,9 +157,9 @@ desired root install path): .. console:: - > ./configure --prefix=/desired/install/path - > make - > make install + ./configure --prefix=/desired/install/path + make + make install The default installation prefix is ``/usr/local/bro``, which would typically require root privileges when doing the ``make install``. @@ -174,13 +174,13 @@ Bourne-Shell Syntax: .. console:: - > export PATH=/usr/local/bro/bin:$PATH + export PATH=/usr/local/bro/bin:$PATH C-Shell Syntax: .. console:: - > setenv PATH /usr/local/bro/bin:$PATH + setenv PATH /usr/local/bro/bin:$PATH Or substitute ``/opt/bro/bin`` instead if you installed from a binary package. @@ -211,7 +211,7 @@ Now start the BroControl shell like: .. console:: - > broctl + broctl Since this is the first-time use of the shell, perform an initial installation of the BroControl configuration: @@ -233,10 +233,9 @@ policy and output the results in ``$PREFIX/logs``. .. note:: The user starting BroControl needs permission to capture network traffic. If you are not root, you may need to grant further - privileges to the account you're using; see the `FAQ - <{{docroot}}/documentation/faq.html>`_. Also, if it - looks like Bro is not seeing any traffic, check out the FAQ entry - checksum offloading. + privileges to the account you're using; see the :doc:`FAQ `. + Also, if it looks like Bro is not seeing any traffic, check out + the FAQ entry on checksum offloading. You can leave it running for now, but to stop this Bro instance you would do: @@ -244,9 +243,7 @@ You can leave it running for now, but to stop this Bro instance you would do: [BroControl] > stop -We also recommend to insert the following entry into `crontab`: - -.. console:: +We also recommend to insert the following entry into `crontab`:: 0-59/5 * * * * $PREFIX/bin/broctl cron @@ -373,9 +370,7 @@ the variable's value may not change at run-time, but whose initial value can be modified via the ``redef`` operator at parse-time. So let's continue on our path to modify the behavior for the two SSL -and SSH notices. Looking at -`$PREFIX/share/bro/base/frameworks/notice/main.bro -<{{autodoc_bro_scripts}}/scripts/base/frameworks/notice/main.html>`_, +and SSH notices. Looking at :doc:`scripts/base/frameworks/notice/main`, we see that it advertises: .. code:: bro @@ -477,8 +472,7 @@ tweak the most basic options. Here's some suggestions on what to explore next: * Reading the code of scripts that ship with Bro is also a great way to gain understanding of the language and how you can start writing your own custom analysis. -* Review the `FAQ <{{docroot}}/documentation/faq.html>`_. -* Check out more `documentation <{{docroot}}/documentation/index.html>`_. +* Review the :doc:`FAQ `. * Continue reading below for another mini-tutorial on using Bro as a standalone command-line utility. @@ -496,7 +490,7 @@ Analyzing live traffic from an interface is simple: .. console:: - > bro -i en0 + bro -i en0 ``en0`` can be replaced by the interface of your choice and for the list of scripts, you can just use "all" for now to perform all the default analysis @@ -504,7 +498,7 @@ that's available. Bro will output log files into the working directory. -.. note:: The `FAQ <{{docroot}}/documentation/faq.html>`_ entries about +.. note:: The :doc:`FAQ ` entries about capturing as an unprivileged user and checksum offloading are particularly relevant at this point. @@ -513,7 +507,7 @@ command-line: .. console:: - > bro -i en0 local + bro -i en0 local This will cause Bro to print a warning about lacking the ``Site::local_nets`` variable being configured. You can supply this @@ -522,7 +516,7 @@ in place of the example subnets): .. console:: - > bro -r mypackets.trace local "Site::local_nets += { 1.2.3.0/24, 5.6.7.0/24 }" + bro -r mypackets.trace local "Site::local_nets += { 1.2.3.0/24, 5.6.7.0/24 }" Reading Packet Capture (pcap) Files @@ -533,7 +527,7 @@ like this: .. console:: - > sudo tcpdump -i en0 -s 0 -w mypackets.trace + sudo tcpdump -i en0 -s 0 -w mypackets.trace Where ``en0`` can be replaced by the correct interface for your system as shown by e.g. ``ifconfig``. (The ``-s 0`` argument tells it to capture @@ -544,7 +538,7 @@ and tell Bro to perform all the default analysis on the capture which primarily .. console:: - > bro -r mypackets.trace + bro -r mypackets.trace Bro will output log files into the working directory. @@ -553,7 +547,7 @@ script that we include as a suggested configuration: .. console:: - > bro -r mypackets.trace local + bro -r mypackets.trace local Telling Bro Which Scripts to Load @@ -563,7 +557,7 @@ A command-line invocation of Bro typically looks like: .. console:: - > bro + bro Where the last arguments are the specific policy scripts that this Bro instance will load. These arguments don't have to include the ``.bro`` @@ -578,7 +572,7 @@ logging) and adds SSL certificate validation. .. console:: - > bro -r mypackets.trace protocols/ssl/validate-certs + bro -r mypackets.trace protocols/ssl/validate-certs You might notice that a script you load from the command line uses the ``@load`` directive in the Bro language to declare dependence on other scripts. diff --git a/doc/reporting-problems.rst b/doc/reporting-problems.rst new file mode 100644 index 0000000000..a1105708eb --- /dev/null +++ b/doc/reporting-problems.rst @@ -0,0 +1,194 @@ + +Reporting Problems +================== + +.. rst-class:: opening + + Here we summarizes some steps to follow when you see Bro doing + something it shouldn't. To provide help, it is often crucial for + us to have a way of reliably reproducing the effect you're seeing. + Unfortunately, reproducing problems can be rather tricky with Bro + because more often than not, they occur only in either very rare + situations or only after Bro has been running for some time. In + particular, getting a small trace showing a specific effect can be + a real problem. In the following, we'll summarize some strategies + to this end. + +Reporting Problems +------------------ + +Generally, when you encounter a problem with Bro, the best thing to do +is opening a new ticket in `Bro's issue tracker +`__ and include information on how to +reproduce the issue. Ideallt, your ticket should come with the +following: + +* The Bro version you're using (if working directly from the git + repository, the branch and revision number.) + +* The output you're seeing along with a description what you'd expect + Bro to do instead. + +* A *small* trace in `libpcap format `__ + demonstrating the effect (assuming the problem doesn't happen right + at startup already). + +* The exact command-line you're using to run Bro with that trace. If + you can, please try to run the Bro binary directly from the command + line rather than using BroControl. + +* Any non-standard scripts you're using (but please only those really + necessary; just a small code snippet triggering the problem would + perfect). + +* If you encounter a crash, information from the core dump, such as + the stack backtrace, can be very helpful. See below for more on + this. + + +How Do I Get a Trace File? +-------------------------- + +As Bro is usually running live, coming up with a small trace file that +reproduces a problem can turn out to be quite a challenge. Often it +works to best to start with a large trace that triggers the problem, +and then successively thin it out as much a possible. + +To get to the initial large trace, here are few things you can try: + +* Capture a trace with `tcpdump `__, either + on the same interface Bro is running on, or on another host where + you can generate traffic of the kind likely triggering the problem + (e.g., if you're seeing problems with the HTTP analyzer, record some + of your Web browsing on your desktop.) When using tcpdump, don't + forget to record *complete* packets (``tcpdump -s 0 ...``). You can + reduce the amount of traffic captured by using a suitable BPF filter + (e.g., for HTTP only, try ``port 80``). + +* Bro's command-line option ``-w `` records all packets it + processes into the given the file. You can then later run Bro + offline on this trace and it will process the packets in the same + way as it did live. This is particularly helpful with problems that + only occur after Bro has already been running for some time. For + example, sometimes a crash may be triggered by a particular kind of + traffic only occurring rarely. Running Bro live with ``-w`` and + then, after the crash, offline on the recorded trace might, with a + little bit of luck, reproduce the the problem reliably. However, be + careful with ``-w``: it can result in huge trace files, quickly + filling up your disk. (One way to mitigate the space issues is to + periodically delete the trace file by configuring + ``rotate-logs.bro`` accordingly. BroControl does that for you if you + set its ``SaveTraces`` option.) + +* Finally, you can try running Bro on a publically available trace + file, such as `anonymized FTP traffic `__, `headers-only enterprise traffic + `__, or + `Defcon traffic `__. Some of these + particularly stress certain components of Bro (e.g., the Defcon + traces contain tons of scans). + +Once you have a trace that demonstrates the effect, you will often +notice that it's pretty big, in particular if recorded from the link +you're monitoring. Therefore, the next step is to shrink its size as +much as possible. Here are a few things you can try to this end: + +* Very often, a single connection is able to demonstrate the problem. + If you can identify which one it is (e.g., from one of Bro's + ``*.log`` files) you can extract the connection's packets from the + trace usong tcpdump by filtering for the corresponding 4-tuple of + addresses and ports: + + .. console:: + + > tcpdump -r large.trace -w small.trace host and port and host and port + +* If you can't reduce the problem to a connection, try to identify + either a host pair or a single host triggering it, and filter down + the trace accordingly. + +* You can try to extract a smaller time slice from the trace using + `TCPslice `__. For example, to + extract the first 100 seconds from the trace: + + .. console:: + + # Test comment + > tcpslice +100 out + +Alternatively, tcpdump extracts the first ``n`` packets with its +option ``-c ``. + + +Getting More Information After a Crash +-------------------------------------- + +If Bro crashes, a *core dump* can be very helpful to nail down the +problem. Examining a core is not for the faint of heart but can reveal +extremely useful information. + +First, you should configure Bro with the option ``--enable-debug`` and +recompile; this will disable all compiler optimizations and thus make +the core dump more useful (don't expect great performance with this +version though; compiling Bro without optimization has a noticeable +impact on its CPU usage.). Then enable core dumps if you don't have +already (e.g., ``ulimit -c unlimited`` if you're using a bash). + +Once Bro has crashed, start gdb with the Bro binary and the file +containing the core dump. (Alternatively, you can also run Bro +directly inside gdb instead of working from a core file.) The first +helpful information to include with your tracker ticket is a stack +backtrace, which you get with gdb's ``bt`` command: + +.. console:: + + > gdb bro core + [...] + > bt + + +If the crash occurs inside Bro's script interpreter, the next thing to +do is identifying the line of script code processed just before the +abnormal termination. Look for methods in the stack backtrace which +belong to any of the script interpreter's classes. Roughly speaking, +these are all classes with names ending in ``Expr``, ``Stmt``, or +``Val``. Then climb up the stack with ``up`` until you reach the first +of these methods. The object to which ``this`` is pointing will have a +``Location`` object, which in turn contains the file name and line +number of the corresponding piece of script code. Continuing the +example from above, here's how to get that information: + +.. console:: + + [in gdb] + > up + > ... + > up + > print this->location->filename + > print this->location->first_line + + +If the crash occurs while processing input packets but you cannot +directly tell which connection is responsible (and thus not extract +its packets from the trace as suggested above), try getting the +4-tuple of the connection currently being processed from the core dump +by again examining the stack backtrace, this time looking for methods +belonging to the ``Connection`` class. That class has members +``orig_addr``/``resp_addr`` and ``orig_port``/``resp_port`` storing +(pointers to) the IP addresses and ports respectively: + +.. console:: + + [in gdb] + > up + > ... + > up + > printf "%08x:%04x %08x:%04x\n", *this->orig_addr, this->orig_port, *this->resp_addr, this->resp_port + + +Note that these values are stored in `network byte order +`__ +so you will need flip the bytes around if you are on a low-endian +machine (which is why the above example prints them in hex). For +example, if an IP address prints as ``0100007f`` , that's 127.0.0.1 . + diff --git a/doc/scripts/CMakeLists.txt b/doc/scripts/CMakeLists.txt index 1cf2f768ec..c14eab6d25 100644 --- a/doc/scripts/CMakeLists.txt +++ b/doc/scripts/CMakeLists.txt @@ -1,16 +1,3 @@ -set(BIF_SRC_DIR ${PROJECT_SOURCE_DIR}/src) -set(RST_OUTPUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/rest_output) -set(DOC_OUTPUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/out) -set(DOC_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/source) -set(DOC_SOURCE_WORKDIR ${CMAKE_CURRENT_BINARY_DIR}/source) - -file(GLOB_RECURSE DOC_SOURCES FOLLOW_SYMLINKS "*") - -# configure the Sphinx config file (expand variables CMake might know about) -configure_file(${CMAKE_CURRENT_SOURCE_DIR}/conf.py.in - ${CMAKE_CURRENT_BINARY_DIR}/conf.py - @ONLY) - # find out what BROPATH to use when executing bro execute_process(COMMAND ${CMAKE_BINARY_DIR}/bro-path-dev OUTPUT_VARIABLE BROPATH @@ -48,7 +35,7 @@ endif () # which summary text can be extracted at build time # ${group}_doc_names: a running list of reST style document names that can be # given to a :doc: role, shared indices with ${group}_files -# + macro(REST_TARGET srcDir broInput) set(absSrcPath ${srcDir}/${broInput}) get_filename_component(basename ${broInput} NAME) @@ -86,7 +73,7 @@ macro(REST_TARGET srcDir broInput) elseif (${extension} STREQUAL ".bif.bro") set(group bifs) elseif (relDstDir) - set(pkgIndex scripts/${relDstDir}/index) + set(pkgIndex ${relDstDir}/index) set(group ${pkgIndex}) # add package index to master package list if not already in it list(FIND MASTER_PKG_LIST ${pkgIndex} _found) @@ -134,6 +121,7 @@ macro(REST_TARGET srcDir broInput) ARGS -rf .state *.log *.rst DEPENDS bro DEPENDS ${absSrcPath} + WORKING_DIRECTORY ${CMAKE_BINARY_DIR} COMMENT "[Bro] Generating reST docs for ${broInput}" ) @@ -143,12 +131,10 @@ endmacro(REST_TARGET) include(DocSourcesList.cmake) # create temporary list of all docs to include in the master policy/index file -set(MASTER_POLICY_INDEX ${CMAKE_CURRENT_BINARY_DIR}/policy_index) file(WRITE ${MASTER_POLICY_INDEX} "${MASTER_POLICY_INDEX_TEXT}") # create the temporary list of all packages to include in the master # policy/packages.rst file -set(MASTER_PACKAGE_INDEX ${CMAKE_CURRENT_BINARY_DIR}/pkg_index) set(MASTER_PKG_INDEX_TEXT "") foreach (pkg ${MASTER_PKG_LIST}) # strip of the trailing /index for the link name @@ -186,11 +172,11 @@ if (EXISTS ${RST_OUTPUT_DIR}) list(FIND ALL_REST_OUTPUTS ${_doc} _found) if (_found EQUAL -1) file(REMOVE ${_doc}) - message(STATUS "AutoDoc: remove stale reST doc: ${_doc}") + message(STATUS "Broxygen: remove stale reST doc: ${_doc}") string(REPLACE .rst .bro _brofile ${_doc}) if (EXISTS ${_brofile}) file(REMOVE ${_brofile}) - message(STATUS "AutoDoc: remove stale bro source: ${_brofile}") + message(STATUS "Broxygen: remove stale bro source: ${_brofile}") endif () endif () endforeach () @@ -211,57 +197,3 @@ add_custom_target(restclean COMMAND "${CMAKE_COMMAND}" -E remove_directory ${RST_OUTPUT_DIR} VERBATIM) - -# The "sphinxdoc" target generates reST documentation for any outdated bro -# scripts and then uses Sphinx to generate HTML documentation from the reST -add_custom_target(sphinxdoc - # copy the template documentation to the build directory - # to give as input for sphinx - COMMAND "${CMAKE_COMMAND}" -E copy_directory - ${DOC_SOURCE_DIR} - ${DOC_SOURCE_WORKDIR} - # copy generated policy script documentation into the - # working copy of the template documentation - COMMAND "${CMAKE_COMMAND}" -E copy_directory - ${RST_OUTPUT_DIR} - ${DOC_SOURCE_WORKDIR}/scripts - # append to the master index of all policy scripts - COMMAND cat ${MASTER_POLICY_INDEX} >> - ${DOC_SOURCE_WORKDIR}/scripts/index.rst - # append to the master index of all policy packages - COMMAND cat ${MASTER_PACKAGE_INDEX} >> - ${DOC_SOURCE_WORKDIR}/packages.rst - # construct a reST file for each group - COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/group_index_generator.py - ${CMAKE_CURRENT_BINARY_DIR}/group_list - ${CMAKE_CURRENT_BINARY_DIR} - ${DOC_SOURCE_WORKDIR} - # tell sphinx to generate html - COMMAND sphinx-build - -b html - -c ${CMAKE_CURRENT_BINARY_DIR} - -d ${DOC_OUTPUT_DIR}/doctrees - ${DOC_SOURCE_WORKDIR} - ${DOC_OUTPUT_DIR}/html - # create symlink to the html output directory for convenience - COMMAND "${CMAKE_COMMAND}" -E create_symlink - ${DOC_OUTPUT_DIR}/html - ${CMAKE_BINARY_DIR}/html - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} - COMMENT "[Sphinx] Generating HTML policy script docs" - # SOURCES just adds stuff to IDE projects as a convenience - SOURCES ${DOC_SOURCES}) - -# The "sphinxclean" target removes just the Sphinx input/output directories -# from the build directory. -add_custom_target(sphinxclean - COMMAND "${CMAKE_COMMAND}" -E remove_directory - ${DOC_SOURCE_WORKDIR} - COMMAND "${CMAKE_COMMAND}" -E remove_directory - ${DOC_OUTPUT_DIR} - VERBATIM) - -add_dependencies(sphinxdoc sphinxclean restdoc) - -add_dependencies(doc sphinxdoc) -add_dependencies(docclean sphinxclean restclean) diff --git a/doc/scripts/README b/doc/scripts/README index ca7f28492b..b3e44914f4 100644 --- a/doc/scripts/README +++ b/doc/scripts/README @@ -15,29 +15,10 @@ by CMake: ``build/`` directory inside ``reST`` (a symlink to ``doc/scripts/rest_output``). -``doc`` - - This target depends on a Python interpreter (>=2.5) and - `Sphinx `_ being installed. Sphinx can be - installed like:: - - > sudo easy_install sphinx - - This target will first build ``restdoc`` target and then copy the - resulting reST files as an input directory to Sphinx. - - After completion, HTML documentation can be located in the CMake - ``build/`` directory inside ``html`` (a symlink to - ``doc/scripts/out/html``) - ``restclean`` This target removes any reST documentation that has been generated so far. -``docclean`` - - This target removes Sphinx inputs and outputs from the CMake ``build/`` dir. - The ``genDocSourcesList.sh`` script can be run to automatically generate ``DocSourcesList.cmake``, which is the file CMake uses to define the list of documentation targets. This script should be run after adding new @@ -54,18 +35,10 @@ script's name to the blacklist, then append a ``rest_target()`` to the ``statictext`` variable where the first argument is the source directory containing the policy script to document, the second argument is the file name of the policy script, and the third argument is the path/name of a -pre-created reST document in the ``source/`` directory to which the +pre-created reST document in the ``../`` source directory to which the ``make doc`` process can append script documentation references. This pre-created reST document should also then be linked to from the TOC tree -in ``source/index.rst``. - -The Sphinx source tree template in ``source/`` can be modified to add more -common/general documentation, style sheets, JavaScript, etc. The Sphinx -config file is produced from ``conf.py.in``, so that can be edited to change -various Sphinx options, like setting the default HTML rendering theme. -There is also a custom Sphinx domain implemented in ``source/ext/bro.py`` -which adds some reST directives and roles that aid in generating useful -index entries and cross-references. +in ``../index.rst``. See ``example.bro`` for an example of how to document a Bro script such that ``make doc`` will be able to produce reST/HTML documentation for it. diff --git a/doc/scripts/bifs.rst b/doc/scripts/bifs.rst new file mode 100644 index 0000000000..eaae0e13b8 --- /dev/null +++ b/doc/scripts/bifs.rst @@ -0,0 +1,5 @@ +.. This is a stub doc to which broxygen appends during the build process + +Built-In Functions (BIFs) +========================= + diff --git a/doc/scripts/source/builtins.rst b/doc/scripts/builtins.rst similarity index 100% rename from doc/scripts/source/builtins.rst rename to doc/scripts/builtins.rst diff --git a/doc/scripts/source/common.rst b/doc/scripts/common.rst similarity index 100% rename from doc/scripts/source/common.rst rename to doc/scripts/common.rst diff --git a/doc/scripts/index.rst b/doc/scripts/index.rst new file mode 100644 index 0000000000..5de203afe7 --- /dev/null +++ b/doc/scripts/index.rst @@ -0,0 +1,8 @@ +.. This is a stub doc to which broxygen appends during the build process + +Index of All Bro Scripts +======================== + +.. toctree:: + :maxdepth: 1 + diff --git a/doc/scripts/internal.rst b/doc/scripts/internal.rst new file mode 100644 index 0000000000..a6c10f1cfb --- /dev/null +++ b/doc/scripts/internal.rst @@ -0,0 +1,5 @@ +.. This is a stub doc to which broxygen appends during the build process + +Internal Scripts +================ + diff --git a/doc/scripts/source/packages.rst b/doc/scripts/packages.rst similarity index 78% rename from doc/scripts/source/packages.rst rename to doc/scripts/packages.rst index 35f6633fd4..47e974b0c8 100644 --- a/doc/scripts/source/packages.rst +++ b/doc/scripts/packages.rst @@ -1,7 +1,7 @@ -.. This is a stub doc to which the build process can append. +.. This is a stub doc to which broxygen appends during the build process -Bro Script Packages -=================== +Index of All Bro Script Packages +================================ Bro has the following script packages (e.g. collections of related scripts in a common directory). If the package directory contains a ``__load__.bro`` diff --git a/doc/scripts/source/_templates/layout.html b/doc/scripts/source/_templates/layout.html deleted file mode 100644 index 5685e1dc37..0000000000 --- a/doc/scripts/source/_templates/layout.html +++ /dev/null @@ -1,5 +0,0 @@ -{% extends "!layout.html" %} -{% block extrahead %} - - {{ super() }} -{% endblock %} diff --git a/doc/scripts/source/bifs.rst b/doc/scripts/source/bifs.rst deleted file mode 100644 index 6a42cafafc..0000000000 --- a/doc/scripts/source/bifs.rst +++ /dev/null @@ -1,5 +0,0 @@ -.. This is a stub doc to which the build process can append. - -Built-In Functions (BIFs) -========================= - diff --git a/doc/scripts/source/index.rst b/doc/scripts/source/index.rst deleted file mode 100644 index a144644208..0000000000 --- a/doc/scripts/source/index.rst +++ /dev/null @@ -1,23 +0,0 @@ -.. Bro documentation master file - -Welcome to Bro's documentation! -=============================== - -Contents: - -.. toctree:: - :maxdepth: 1 - :glob: - - common - builtins - internal - bifs - packages - scripts/index - -Indices and tables -================== - -* :ref:`genindex` -* :ref:`search` diff --git a/doc/scripts/source/internal.rst b/doc/scripts/source/internal.rst deleted file mode 100644 index 817c76e725..0000000000 --- a/doc/scripts/source/internal.rst +++ /dev/null @@ -1,5 +0,0 @@ -.. This is a stub doc to which the build process can append. - -Internal Scripts -================ - diff --git a/doc/scripts/source/scripts/index.rst b/doc/scripts/source/scripts/index.rst deleted file mode 100644 index fd11a3924e..0000000000 --- a/doc/scripts/source/scripts/index.rst +++ /dev/null @@ -1,6 +0,0 @@ -Index of All Bro Script Documentation -===================================== - -.. toctree:: - :maxdepth: 1 - diff --git a/doc/signatures.rst b/doc/signatures.rst index 25dfc31f5e..e2f9a8d702 100644 --- a/doc/signatures.rst +++ b/doc/signatures.rst @@ -3,7 +3,7 @@ Signatures ========== -.. class:: opening +.. rst-class:: opening Bro relies primarily on its extensive scripting language for defining and analyzing detection policies. In addition, however, @@ -47,8 +47,8 @@ piece of payload which triggered the pattern match. To turn such ``signature_match`` events into actual alarms, you can load Bro's ``signature.bro`` script. This script contains a default -event handler that raises ``SensitiveSignature`` `Notices -`_ (as well as others; see the beginning of the script). +event handler that raises ``SensitiveSignature`` :doc:`Notices ` +(as well as others; see the beginning of the script). As signatures are independent of Bro's policy scripts, they are put into their own file(s). There are two ways to specify which files diff --git a/doc/upgrade.rst b/doc/upgrade.rst index 696c64ef7f..71cc5e401d 100644 --- a/doc/upgrade.rst +++ b/doc/upgrade.rst @@ -3,7 +3,7 @@ Upgrading From Bro 1.5 to 2.0 ============================= -.. class:: opening +.. rst-class:: opening This guide details differences between Bro versions 1.5 and 2.0 that may be important for users to know as they work on updating @@ -39,7 +39,7 @@ version. The two rules of thumb are: if you need help. Below we summarize changes from 1.x to 2.x in more detail. This list -isn't complete, see the `CHANGES <{{git('bro:CHANGES', 'txt')}}>`_ file in the +isn't complete, see the :download:`CHANGES ` file in the distribution for the full story. Default Scripts @@ -131,8 +131,8 @@ Logging Framework endpoint. - The new logging framework makes it possible to extend, customize, - and filter logs very easily. See `the logging framework - <{{git('bro:doc/logging.rst')}}>`_ more information on usage. + and filter logs very easily. See the :doc:`logging framework ` + more information on usage. - A common pattern found in the new scripts is to store logging stream records for protocols inside the ``connection`` records so that @@ -155,8 +155,7 @@ Notice Framework The way users interact with "notices" has changed significantly in order to make it easier to define a site policy and more extensible -for adding customized actions. See the `the notice framework -<{{git('bro:doc/notice.rst')}}>`_. +for adding customized actions. See the :doc:`notice framework `. New Default Settings @@ -198,7 +197,7 @@ Variable Naming - Identifiers may have been renamed to conform to new `scripting conventions - <{{docroot}}/development/script-conventions.html>`_ + `_ BroControl @@ -240,7 +239,7 @@ Development Infrastructure Bro development has moved from using SVN to Git for revision control. Users that like to use the latest Bro developments by checking it out from the source repositories should see the `development process -<{{docroot}}/development/process.html>`_. Note that all the various +`_. Note that all the various sub-components now reside on their own repositories. However, the top-level Bro repository includes them as git submodules so it's easu to check them all out simultaneously.