Merge remote-tracking branch 'origin/master' into topic/bernhard/input

CHANGES

@@ -1,4 +1,18 @@
 
+2.0-beta-47 | 2011-11-16 08:24:33 -0800
+
+  * Catch if logged sets do not contain only atomic types. (Bernhard
+    Amann)
+
+  * Promote libz and libmagic to required dependencies. (Jon Siwek)
+    
+  * Fix parallel make from top-level to work on more platforms. (Jon
+    Siwek)
+
+  * Add decode_base64_custom(). Addresses #670 (Jon Siwek)
+    
+  * A bunch of Sphinx-doc reorgs and polishing. (Jon Siwek)
+    
 2.0-beta-28 | 2011-11-14 20:09:28 -0800
 
   * Binary packaging script tweaks. We now require CMake 2.8.6. (Jon Siwek)

CMakeLists.txt

@@ -53,6 +53,8 @@ FindRequiredPackage(BISON)
 FindRequiredPackage(PCAP)
 FindRequiredPackage(OpenSSL)
 FindRequiredPackage(BIND)
+FindRequiredPackage(LibMagic)
+FindRequiredPackage(ZLIB)
 
 if (NOT BinPAC_ROOT_DIR AND
     EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/aux/binpac/CMakeLists.txt)
@@ -72,26 +74,12 @@ include_directories(BEFORE
                     ${OpenSSL_INCLUDE_DIR}
                     ${BIND_INCLUDE_DIR}
                     ${BinPAC_INCLUDE_DIR}
+                    ${LibMagic_INCLUDE_DIR}
+                    ${ZLIB_INCLUDE_DIR}
 )
 
 # Optional Dependencies
 
-set(HAVE_LIBMAGIC false)
-find_package(LibMagic)
-if (LIBMAGIC_FOUND)
-    set(HAVE_LIBMAGIC true)
-    include_directories(BEFORE ${LibMagic_INCLUDE_DIR})
-    list(APPEND OPTLIBS ${LibMagic_LIBRARY})
-endif ()
-
-set(HAVE_LIBZ false)
-find_package(ZLIB)
-if (ZLIB_FOUND)
-    set(HAVE_LIBZ true)
-    include_directories(BEFORE ${ZLIB_INCLUDE_DIR})
-    list(APPEND OPTLIBS ${ZLIB_LIBRARY})
-endif ()
-
 set(USE_GEOIP false)
 find_package(LibGeoIP)
 if (LIBGEOIP_FOUND)
@@ -110,6 +98,16 @@ if (ENABLE_PERFTOOLS)
     endif ()
 endif ()
 
+set(brodeps
+    ${BinPAC_LIBRARY}
+    ${PCAP_LIBRARY}
+    ${OpenSSL_LIBRARIES}
+    ${BIND_LIBRARY}
+    ${LibMagic_LIBRARY}
+    ${ZLIB_LIBRARY}
+    ${OPTLIBS}
+)
+
 ########################################################################
 ## System Introspection
 
@@ -184,8 +182,6 @@ message(
     "\nAux. Tools:        ${INSTALL_AUX_TOOLS}"
     "\n"
     "\nGeoIP:             ${USE_GEOIP}"
-    "\nlibz:              ${HAVE_LIBZ}"
-    "\nlibmagic:          ${HAVE_LIBMAGIC}"
     "\nGoogle perftools:  ${USE_PERFTOOLS}"
     "\n"
     "\n================================================================\n"

INSTALL

@@ -14,16 +14,15 @@ before you begin:
 
     * OpenSSL (headers and libraries)   http://www.openssl.org
 
-Bro can make uses of some optional libraries if they are found at
-installation time:
-
     * Libmagic  For identifying file types (e.g., in FTP transfers).
 
-    * LibGeoIP  For geo-locating IP addresses.
-
     * Libz      For decompressing HTTP bodies by the HTTP analyzer, and for
                 compressed Bro-to-Bro communication.
 
+Bro can make uses of some optional libraries if they are found at
+installation time:
+
+    * LibGeoIP  For geo-locating IP addresses.
 
 Bro also needs the following tools, but on most systems they will
 already come preinstalled:
@@ -33,15 +32,14 @@ already come preinstalled:
     * Flex  (Fast Lexical Analyzer)
     * Perl  (Used only during the Bro build process)
 
-
 Installation
 ============
 
 To build and install into ``/usr/local/bro``::
 
-    > ./configure
+    ./configure
-    > make
+    make
-    > make install
+    make install
 
 This will first build Bro into a directory inside the distribution
 called ``build/``, using default build options. It then installs all
@@ -50,7 +48,7 @@ required files into ``/usr/local/bro``, including the Bro binary in
 
 You can specify a different installation directory with::
 
-    > ./configure --prefix=<dir>
+    ./configure --prefix=<dir>
 
 Note that ``/usr`` and ``/opt/bro`` are the standard prefixes for
 binary Bro packages to be installed, so those are typically not good
@@ -71,11 +69,8 @@ Running Bro
 ===========
 
 Bro is a complex program and it takes a bit of time to get familiar
-with it.  A good place for newcomers to start is the quick start guide
+with it.  A good place for newcomers to start is the
-available here:
+:doc:`quick start guide <quickstart>`.
-
-    http://www.bro-ids.org/documentation/quickstart.html
-
 
 For developers that wish to run Bro directly from the ``build/``
 directory (i.e., without performing ``make install``), they will have
@@ -83,9 +78,9 @@ to first adjust ``BROPATH`` to look for scripts inside the build
 directory.  Sourcing either ``build/bro-path-dev.sh`` or
 ``build/bro-path-dev.csh`` as appropriate for the current shell
 accomplishes this and also augments your ``PATH`` so you can use the
-Bro binary directly:
+Bro binary directly::
 
-    > ./configure
+    ./configure
-    > make
+    make
-    > source build/bro-path-dev.sh
+    source build/bro-path-dev.sh
-    > bro <options>
+    bro <options>

Makefile

@@ -11,23 +11,11 @@ VERSION_FULL=$(REPO)-`cat VERSION`
 VERSION_MIN=$(REPO)-`cat VERSION`-minimal
 HAVE_MODULES=git submodule | grep -v cmake >/dev/null
 
-all: configured
+SUBDIRS = $(BUILD)
-	( cd $(BUILD) && make )
+$(SUBDIRS):: configured
+	$(MAKE) -C $@ $(MAKECMDGOALS)
 
-install: configured
+all install install-aux doc docclean clean: $(SUBDIRS)
-	( cd $(BUILD) && make install )
-
-install-aux: configured
-	( cd $(BUILD) && make install-aux )
-
-clean: configured docclean
-	( cd $(BUILD) && make clean )
-
-doc: configured
-	( cd $(BUILD) && make doc )
-
-docclean: configured
-	( cd $(BUILD) && make docclean )
 
 dist:
 	@rm -rf $(VERSION_FULL) $(VERSION_FULL).tgz

VERSION

@@ -1 +1 @@
-2.0-beta-28
+2.0-beta-47

aux/binpac

@@ -1 +1 @@
-Subproject commit 777b8a21c4c74e1f62e8b9896b082e8c059b539f
+Subproject commit 0f99acfbf6205830f0db699a75554262c26427f9

aux/bro-aux

@@ -1 +1 @@
-Subproject commit 906f970df5f708582c7002069b787d5af586b46f
+Subproject commit 1a7a9357fba88a43c90a39d8d72b42fa53b89b75

aux/broccoli

@@ -1 +1 @@
-Subproject commit e02e3cc89a3efb3d7ec376154e24835b4b828be8
+Subproject commit a1a03c686866bd30ee086ff933128055a20ebd56

aux/broctl

@@ -1 +1 @@
-Subproject commit 6fb4e5689d2ae0d1c4ab7af0a8df80e6eaa98fb6
+Subproject commit 03d39aa5d4ab24cd9b8e404a9ceb583d5270444c

cmake

@@ -1 +1 @@
-Subproject commit 704e255d7ef2faf926836c1c64d16c5b8a02b063
+Subproject commit 44f2985475e5ff6cc9061683e21ef4b184bdfc7e

config.h.in

@@ -14,12 +14,6 @@
 /* Define if you have the `getopt_long' function. */
 #cmakedefine HAVE_GETOPT_LONG
 
-/* Define if you have the `magic' library (-lmagic). */
-#cmakedefine HAVE_LIBMAGIC
-
-/* Define if you have the `z' library (-lz). */
-#cmakedefine HAVE_LIBZ
-
 /* We are on a Linux system */
 #cmakedefine HAVE_LINUX
 

doc/CHANGES

@@ -0,0 +1 @@
+../CHANGES

doc/CMakeLists.txt

@@ -1,4 +1,73 @@
-add_custom_target(doc)
+set(BIF_SRC_DIR ${PROJECT_SOURCE_DIR}/src)
-add_custom_target(docclean)
+set(RST_OUTPUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/rest_output)
+set(DOC_OUTPUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/out)
+set(DOC_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR})
+set(DOC_SOURCE_WORKDIR ${CMAKE_CURRENT_BINARY_DIR}/sphinx-sources)
+
+set(MASTER_POLICY_INDEX ${CMAKE_CURRENT_BINARY_DIR}/scripts/policy_index)
+set(MASTER_PACKAGE_INDEX ${CMAKE_CURRENT_BINARY_DIR}/scripts/pkg_index)
+
+file(GLOB_RECURSE DOC_SOURCES FOLLOW_SYMLINKS "*")
+
+# configure the Sphinx config file (expand variables CMake might know about)
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/conf.py.in
+               ${CMAKE_CURRENT_BINARY_DIR}/conf.py
+               @ONLY)
 
 add_subdirectory(scripts)
+
+# The "broxygen" target generates reST documentation for any outdated bro
+# scripts and then uses Sphinx to generate HTML documentation from the reST
+add_custom_target(broxygen
+                  # copy the template documentation to the build directory
+                  # to give as input for sphinx
+                  COMMAND "${CMAKE_COMMAND}" -E copy_directory
+                          ${DOC_SOURCE_DIR}
+                          ${DOC_SOURCE_WORKDIR}
+                  # copy generated policy script documentation into the
+                  # working copy of the template documentation
+                  COMMAND "${CMAKE_COMMAND}" -E copy_directory
+                          ${RST_OUTPUT_DIR}
+                          ${DOC_SOURCE_WORKDIR}/scripts
+                  # append to the master index of all policy scripts
+                  COMMAND cat ${MASTER_POLICY_INDEX} >>
+                          ${DOC_SOURCE_WORKDIR}/scripts/index.rst
+                  # append to the master index of all policy packages
+                  COMMAND cat ${MASTER_PACKAGE_INDEX} >>
+                          ${DOC_SOURCE_WORKDIR}/scripts/packages.rst
+                  # construct a reST file for each group
+                  COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/bin/group_index_generator.py
+                          ${CMAKE_CURRENT_BINARY_DIR}/scripts/group_list
+                          ${CMAKE_CURRENT_BINARY_DIR}/scripts
+                          ${DOC_SOURCE_WORKDIR}/scripts
+                  # tell sphinx to generate html
+                  COMMAND sphinx-build
+                          -b html
+                          -c ${CMAKE_CURRENT_BINARY_DIR}
+                          -d ${DOC_OUTPUT_DIR}/doctrees
+                          ${DOC_SOURCE_WORKDIR}
+                          ${DOC_OUTPUT_DIR}/html
+                  # create symlink to the html output directory for convenience
+                  COMMAND "${CMAKE_COMMAND}" -E create_symlink
+                          ${DOC_OUTPUT_DIR}/html
+                          ${CMAKE_BINARY_DIR}/html
+                  WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
+                  COMMENT "[Sphinx] Generating HTML policy script docs"
+                  # SOURCES just adds stuff to IDE projects as a convenience
+                  SOURCES ${DOC_SOURCES})
+
+# The "sphinxclean" target removes just the Sphinx input/output directories
+# from the build directory.
+add_custom_target(broxygen-clean
+                  COMMAND "${CMAKE_COMMAND}" -E remove_directory
+                          ${DOC_SOURCE_WORKDIR}
+                  COMMAND "${CMAKE_COMMAND}" -E remove_directory
+                          ${DOC_OUTPUT_DIR}
+                  VERBATIM)
+
+add_dependencies(broxygen broxygen-clean restdoc)
+
+add_custom_target(doc)
+add_custom_target(docclean)
+add_dependencies(doc broxygen)
+add_dependencies(docclean broxygen-clean restclean)

doc/INSTALL.rst

@@ -0,0 +1 @@
+../INSTALL

doc/README

@@ -5,29 +5,37 @@ Documentation
 This directory contains Bro documentation in reStructured text format
 (see http://docutils.sourceforge.net/rst.html).
 
-Please note that for now these files are primarily intended for use on
+It is the root of a Sphinx source tree and can be modified to add more
-http://www.bro-ids.org. While the Bro build process will render local
+common/general documentation, style sheets, JavaScript, etc.  The Sphinx
-versions into ``build/doc/`` (if docutils is found), the resulting
+config file is produced from ``conf.py.in``, and can be edited to change
-HTML is very minimalistic and some features are not supported. In
+various Sphinx options.
-particular, some links will be broken.
 
+There is also a custom Sphinx domain implemented in ``source/ext/bro.py``
+which adds some reST directives and roles that aid in generating useful
+index entries and cross-references.  Other extensions can be added in
+a similar fashion.
+
+Either the ``make doc`` or ``make broxygen`` can be used to locally
+render the reST files into HTML.  Those targets depend on:
+
+* Python interpreter >= 2.5
+* `Sphinx <http://sphinx.pocoo.org/>`_ >= 1.0.1
+
+After completion, HTML documentation is symlinked in ``build/html``.
+
+There's also ``make docclean`` and ``make broxygenclean`` targets to
+clean the resulting documentation.
 
 Notes for Writing Documentation
 -------------------------------
 
-* If you want to refer to a Bro script that's part of the
+* If you want to refer to a document that's part of the
-  distribution, use {{'`foo.bro
+  distribution, it currently needs to be copied or otherwise symlinked
-  <{{autodoc_bro_scripts}}/path/to/foo.html>`_'}}. For example,
+  somewhere in to this Sphinx source tree. Then, it can be referenced
-  ``{{'{{autodoc_bro_scripts}}/scripts/base/frameworks/notice/main.html}}'}}``.
+  in a toc tree or with the :doc: role.  Use the :download: role to
+  refer to static files that will not undergo sphinx rendering.
 
-* If you want to refer to a page on the Bro web site, use the
+* If you want to refer to a page on the Bro web site, use an HTTP URL.
-  ``docroot`` macro (e.g.,
-  ``{{'href="{{docroot}}/download/index.html"'}}). Make sure to
-  include the ``index.html`` for the main pages, just as in the
-  example.
-
-* If you want to refer to page inside this directory, use a relative
-  path with HTML extension. (e.g., ``href="quickstart.html``).
 
 Guidelines
 ----------

doc/_static/broxygen.css

@@ -0,0 +1,16 @@
+a:hover
+{
+    text-decoration:none;
+    color:#c24444;
+}
+
+div.body h1, div.body h2, div.body h3, div.body h4, div.body h5, div.body h6
+{
+background-color:#ffffff;
+border-bottom: 1px solid #aaa;
+}
+
+th.field-name
+{
+white-space:nowrap;
+}

doc/_templates/layout.html

@@ -0,0 +1,10 @@
+{% extends "!layout.html" %}
+
+{% set css_files = css_files + ["_static/broxygen.css"] %}
+
+{% block extrahead %}
+<link rel="stylesheet" type="text/css" href="http://www.bro-ids.org/css/bro-base.css" />
+<link rel="stylesheet" type="text/css" href="http://www.bro-ids.org/css/pygments.css" />
+{% endblock %}
+
+{% block relbar2 %}{% endblock %}

doc/cluster.rst

@@ -11,9 +11,7 @@ Architecture
 
 The figure below illustrates the main components of a Bro cluster.
 
-.. {{git_pull('bro:doc/deployment.png')}}
+.. image:: images/deployment.png
-
-.. image:: deployment.bro.png
 
 Tap
 ***

doc/components/broccoli-python/README.rst

@@ -0,0 +1 @@
+../../../aux/broccoli/bindings/broccoli-python/README

doc/components/broccoli/README.rst

@@ -0,0 +1 @@
+../../../aux/broccoli/README

doc/components/broctl/README.rst

@@ -0,0 +1 @@
+../../../aux/broctl/doc/broctl.rst

doc/components/btest/README.rst

@@ -0,0 +1 @@
+../../../aux/btest/README

doc/components/capstats/README.rst

@@ -0,0 +1 @@
+../../../aux/broctl/aux/capstats/README

doc/components/pysubnettree/README.rst

@@ -0,0 +1 @@
+../../../aux/broctl/aux/pysubnettree/README

doc/components/trace-summary/README.rst

@@ -0,0 +1 @@
+../../../aux/broctl/aux/trace-summary/README

doc/conf.py.in

@@ -15,7 +15,7 @@ import sys, os
 # If extensions (or modules to document with autodoc) are in another directory,
 # add these directories to sys.path here. If the directory is relative to the
 # documentation root, use os.path.abspath to make it absolute, like shown here.
-sys.path.insert(0, os.path.abspath('source/ext'))
+sys.path.insert(0, os.path.abspath('sphinx-sources/ext'))
 
 # -- General configuration -----------------------------------------------------
 
@@ -24,10 +24,10 @@ sys.path.insert(0, os.path.abspath('source/ext'))
 
 # Add any Sphinx extension module names here, as strings. They can be extensions
 # coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
-extensions = ['bro']
+extensions = ['bro', 'rst_directive']
 
 # Add any paths that contain templates here, relative to this directory.
-templates_path = ['source/_templates']
+templates_path = ['sphinx-sources/_templates', 'sphinx-sources/_static']
 
 # The suffix of source filenames.
 source_suffix = '.rst'
@@ -95,7 +95,31 @@ html_theme = 'default'
 # Theme options are theme-specific and customize the look and feel of a theme
 # further.  For a list of options available for each theme, see the
 # documentation.
-#html_theme_options = {}
+html_theme_options = {
+"rightsidebar": "true",
+"stickysidebar": "true",
+"externalrefs": "true",
+"footerbgcolor": "#333",
+"footertextcolor": "#ddd",
+"sidebarbgcolor": "#ffffff",
+#"sidebarbtncolor": "",
+"sidebartextcolor": "#333",
+"sidebarlinkcolor": "#2a85a7",
+"relbarbgcolor": "#ffffff",
+"relbartextcolor": "#333",
+"relbarlinkcolor": "#2a85a7",
+"bgcolor": "#ffffff",
+"textcolor": "#333",
+"linkcolor": "#2a85a7",
+"visitedlinkcolor": "#2a85a7",
+"headbgcolor": "#f0f0f0",
+"headtextcolor": "#000",
+"headlinkcolor": "#2a85a7",
+"codebgcolor": "#FFFAE2",
+#"codetextcolor": "",
+"bodyfont": "Arial, Helvetica, sans-serif",
+"headfont": "Palatino,'Palatino Linotype',Georgia,serif",
+}
 
 # Add any paths that contain custom themes here, relative to this directory.
 #html_theme_path = []
@@ -119,7 +143,7 @@ html_theme = 'default'
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,
 # so a file named "default.css" will overwrite the builtin "default.css".
-html_static_path = ['source/_static']
+html_static_path = ['sphinx-sources/_static']
 
 # If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
 # using the given strftime format.
@@ -130,7 +154,9 @@ html_static_path = ['source/_static']
 #html_use_smartypants = True
 
 # Custom sidebar templates, maps document names to template names.
-#html_sidebars = {}
+html_sidebars = {
+'**': ['localtoc.html', 'sourcelink.html', 'searchbox.html'],
+}
 
 # Additional templates that should be rendered to pages, maps page names to
 # template names.
@@ -163,7 +189,7 @@ html_static_path = ['source/_static']
 #html_file_suffix = None
 
 # Output file base name for HTML help builder.
-htmlhelp_basename = 'Brodoc'
+htmlhelp_basename = 'Broxygen'
 
 
 # -- Options for LaTeX output --------------------------------------------------

doc/ext/bro_lexer/bro.py

@@ -0,0 +1,76 @@
+from pygments.lexer import RegexLexer, bygroups, include
+from pygments.token import *
+
+__all__ = ["BroLexer"]
+
+class BroLexer(RegexLexer):
+    name = 'Bro'
+    aliases = ['bro']
+    filenames = ['*.bro']
+
+    _hex = r'[0-9a-fA-F_]+'
+    _float = r'((\d*\.?\d+)|(\d+\.?\d*))([eE][-+]?\d+)?'
+    _h = r'[A-Za-z0-9][-A-Za-z0-9]*'
+
+    tokens = {
+        'root': [
+            # Whitespace
+            ('^@.*?\n', Comment.Preproc),
+            (r'#.*?\n', Comment.Single),
+            (r'\n', Text),
+            (r'\s+', Text),
+            (r'\\\n', Text),
+            # Keywords
+            (r'(add|alarm|break|case|const|continue|delete|do|else|enum|event'
+             r'|export|for|function|if|global|local|module|next'
+             r'|of|print|redef|return|schedule|when|while)\b', Keyword),
+            (r'(addr|any|bool|count|counter|double|file|int|interval|net'
+             r'|pattern|port|record|set|string|subnet|table|time|timer'
+             r'|vector)\b', Keyword.Type),
+            (r'(T|F)\b', Keyword.Constant),
+            (r'(&)((?:add|delete|expire)_func|attr|(create|read|write)_expire'
+             r'|default|disable_print_hook|raw_output|encrypt|group|log'
+             r'|mergeable|optional|persistent|priority|redef'
+             r'|rotate_(?:interval|size)|synchronized)\b', bygroups(Punctuation,
+                 Keyword)),
+            (r'\s+module\b', Keyword.Namespace),
+            # Addresses, ports and networks
+            (r'\d+/(tcp|udp|icmp|unknown)\b', Number),
+            (r'(\d+\.){3}\d+', Number),
+            (r'(' + _hex + r'){7}' + _hex, Number),
+            (r'0x' + _hex + r'(' + _hex + r'|:)*::(' + _hex + r'|:)*', Number),
+            (r'((\d+|:)(' + _hex + r'|:)*)?::(' + _hex + r'|:)*', Number),
+            (r'(\d+\.\d+\.|(\d+\.){2}\d+)', Number),
+            # Hostnames
+            (_h + r'(\.' + _h + r')+', String),
+            # Numeric
+            (_float + r'\s+(day|hr|min|sec|msec|usec)s?\b', Literal.Date),
+            (r'0[xX]' + _hex, Number.Hex),
+            (_float, Number.Float),
+            (r'\d+', Number.Integer),
+            (r'/', String.Regex, 'regex'),
+            (r'"', String, 'string'),
+            # Operators
+            (r'[!%*/+-:<=>?~|]', Operator),
+            (r'([-+=&|]{2}|[+-=!><]=)', Operator),
+            (r'(in|match)\b', Operator.Word),
+            (r'[{}()\[\]$.,;]', Punctuation),
+            # Identfier
+            (r'([_a-zA-Z]\w*)(::)', bygroups(Name, Name.Namespace)),
+            (r'[a-zA-Z_][a-zA-Z_0-9]*', Name)
+        ],
+        'string': [
+            (r'"', String, '#pop'),
+            (r'\\([\\abfnrtv"\']|x[a-fA-F0-9]{2,4}|[0-7]{1,3})', String.Escape),
+            (r'[^\\"\n]+', String),
+            (r'\\\n', String),
+            (r'\\', String)
+        ],
+        'regex': [
+            (r'/', String.Regex, '#pop'),
+            (r'\\[\\nt/]', String.Regex), # String.Escape is too intense.
+            (r'[^\\/\n]+', String.Regex),
+            (r'\\\n', String.Regex),
+            (r'\\', String.Regex)
+        ]
+    }

doc/ext/rst_directive.py

@@ -0,0 +1,180 @@
+def setup(app):
+    pass
+
+# -*- coding: utf-8 -*-
+"""
+
+Modified version of the the Pygments reStructuredText directive. -Robin
+
+This provides two new directives:
+
+    - .. code:: [<format>]
+
+      Highlights the following code block according to <format> if
+      given (e.g., "c", "python", etc.).
+
+    - .. console::
+
+      Highlits the following code block as a shell session.
+
+    For compatibility with the original version, "sourcecode" is
+    equivalent to "code".
+
+Original comment:
+
+    The Pygments reStructuredText directive
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    This fragment is a Docutils_ 0.5 directive that renders source code
+    (to HTML only, currently) via Pygments.
+
+    To use it, adjust the options below and copy the code into a module
+    that you import on initialization.  The code then automatically
+    registers a ``sourcecode`` directive that you can use instead of
+    normal code blocks like this::
+
+        .. sourcecode:: python
+
+            My code goes here.
+
+    If you want to have different code styles, e.g. one with line numbers
+    and one without, add formatters with their names in the VARIANTS dict
+    below.  You can invoke them instead of the DEFAULT one by using a
+    directive option::
+
+        .. sourcecode:: python
+            :linenos:
+
+            My code goes here.
+
+    Look at the `directive documentation`_ to get all the gory details.
+
+    .. _Docutils: http://docutils.sf.net/
+    .. _directive documentation:
+       http://docutils.sourceforge.net/docs/howto/rst-directives.html
+
+    :copyright: Copyright 2006-2010 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+# Options
+# ~~~~~~~
+
+# Set to True if you want inline CSS styles instead of classes
+INLINESTYLES = False
+
+from pygments.formatters import HtmlFormatter
+
+class MyHtmlFormatter(HtmlFormatter):
+    def format_unencoded(self, tokensource, outfile):
+
+        # A NOP currently.
+        new_tokens = []
+        for (i, piece) in tokensource:
+            new_tokens += [(i, piece)]
+
+        return super(MyHtmlFormatter, self).format_unencoded(new_tokens, outfile)
+
+# The default formatter
+DEFAULT = MyHtmlFormatter(noclasses=INLINESTYLES, cssclass="pygments")
+
+# Add name -> formatter pairs for every variant you want to use
+VARIANTS = {
+    # 'linenos': HtmlFormatter(noclasses=INLINESTYLES, linenos=True),
+}
+
+
+import textwrap
+
+from docutils import nodes
+from docutils.parsers.rst import directives, Directive
+
+from pygments import highlight
+from pygments.lexers import get_lexer_by_name, guess_lexer, TextLexer
+from pygments.token import Text, Keyword, Error, Operator, Name
+from pygments.filter import Filter
+
+# Ugly hack to register the Bro lexer. I'm sure there's a better way to do it,
+# but it's not obvious ...
+from bro_lexer.bro import BroLexer
+from pygments.lexers._mapping import LEXERS
+LEXERS['BroLexer'] = ('bro_lexer.bro', BroLexer.name, BroLexer.aliases, BroLexer.filenames, ())
+
+class Pygments(Directive):
+    """ Source code syntax hightlighting.
+    """
+    #max_line_length = 68
+    max_line_length = 0
+
+    required_arguments = 0
+    optional_arguments = 1
+    final_argument_whitespace = True
+    option_spec = dict([(key, directives.flag) for key in VARIANTS])
+    has_content = True
+
+    def wrapped_content(self):
+        content = []
+
+        if Console.max_line_length:
+            for line in self.content:
+                content += textwrap.wrap(line, Console.max_line_length, subsequent_indent="      ")
+        else:
+            content = self.content
+
+        return u'\n'.join(content)
+
+    def run(self):
+        self.assert_has_content()
+
+        content = self.wrapped_content()
+
+        if len(self.arguments) > 0:
+            try:
+                lexer = get_lexer_by_name(self.arguments[0])
+            except (ValueError, IndexError):
+                # lexer not found, use default.
+                lexer = TextLexer()
+        else:
+            lexer = guess_lexer(content)
+
+        # import sys
+        # print >>sys.stderr, self.arguments, lexer.__class__
+
+        # take an arbitrary option if more than one is given
+        formatter = self.options and VARIANTS[self.options.keys()[0]] or DEFAULT
+        parsed = highlight(content, lexer, formatter)
+        return [nodes.raw('', parsed, format='html')]
+
+class MyFilter(Filter):
+    def filter(self, lexer, stream):
+
+        bol = True
+
+        for (ttype, value) in stream:
+            # Color the '>' prompt sign.
+            if bol and ttype is Text and value == ">":
+                ttype = Name.Variable.Class # This gives us a nice red.
+
+            # Discolor builtin, that can look funny.
+            if ttype is Name.Builtin:
+                ttype = Text
+
+            bol = value.endswith("\n")
+
+            yield (ttype, value)
+
+class Console(Pygments):
+    required_arguments = 0
+    optional_arguments = 0
+
+    def run(self):
+        self.assert_has_content()
+        content = self.wrapped_content()
+        lexer = get_lexer_by_name("sh")
+        lexer.add_filter(MyFilter())
+        parsed = highlight(content, lexer, DEFAULT)
+        return [nodes.raw('', parsed, format='html')]
+
+directives.register_directive('sourcecode', Pygments)
+directives.register_directive('code', Pygments)
+directives.register_directive('console', Console)

doc/faq.rst

@@ -0,0 +1,145 @@
+
+==========================
+Frequently Asked Questions
+==========================
+
+.. raw:: html
+
+    <div class="faq">
+
+.. contents::
+
+Installation and Configuration
+==============================
+
+How can I tune my operating system for best capture performance?
+----------------------------------------------------------------
+
+Here are some pointers to more information:
+
+* Fabian Schneider's research on `high performance packet capture
+  <http://www.net.t-labs.tu-berlin.de/research/hppc>`_
+
+* `NSMWiki <http://nsmwiki.org/Main_Page>`_ has page on
+  *Collecting Data*.
+
+* An `IMC 2010 paper
+  <http://conferences.sigcomm.org/imc/2010/papers/p206.pdf>`_ by
+  Lothar Braun et. al evaluates packet capture performance on
+  commodity hardware
+
+What does an error message like ``internal error: NB-DNS error`` mean?
+---------------------------------------------------------------------------------------------------------------------------------
+
+That often means that DNS is not set up correctly on the system
+running Bro. Try verifying from the command line that DNS lookups
+work, e.g., ``host www.google.com``.
+
+
+Usage
+=====
+
+How can I identify backscatter?
+-------------------------------
+
+Identifying backscatter via connections labeled as ``OTH`` is not
+a reliable means to detect backscatter. Use rather the following
+procedure:
+
+* Enable connection history via ``redef record_state_history=T`` to
+  track all control/data packet types in connection logs.
+
+* Backscatter is now visible in terms of connections that never had an
+  initial ``SYN`` but started instead with a ``SYN-ACK`` or ``RST``
+  (though this latter generally is just discarded).
+
+Is there help for understanding Bro's resource consumption?
+-----------------------------------------------------------
+
+There are two scripts that collect statistics on resource usage:
+``stats.bro`` and ``profiling.bro``. The former is quite lightweight,
+while the latter should only be used for debugging. Furthermore,
+there's also ``print-globals.bro``, which prints the size of all
+global script variable at termination.
+
+How can I capture packets as an unprivileged user?
+--------------------------------------------------
+
+Normally, unprivileged users cannot capture packets from a network
+interface, which means they would not be able to use Bro to read/analyze
+live traffic.  However, there are ways to enable packet capture
+permission for non-root users, which is worth doing in the context of
+using Bro to monitor live traffic
+
+With Linux Capabilities
+^^^^^^^^^^^^^^^^^^^^^^^
+
+Fully implemented since Linux kernel 2.6.24, capabilities are a way of
+parceling superuser privileges into distinct units.  Attach capabilities
+required to capture packets to the ``bro`` executable file like this:
+
+.. console::
+
+   sudo setcap cap_net_raw,cap_net_admin=eip /path/to/bro
+
+Now any unprivileged user should have the capability to capture packets
+using Bro provided that they have the traditional file permissions to
+read/execute the ``bro`` binary.
+
+With BPF Devices
+^^^^^^^^^^^^^^^^
+
+Systems using Berkeley Packet Filter (BPF) (e.g. FreeBSD & Mac OS X)
+can allow users with read access to a BPF device to capture packets from
+it using libpcap.
+
+* Example of manually changing BPF device permissions to allow users in
+  the ``admin`` group to capture packets:
+
+.. console::
+
+   sudo chgrp admin /dev/bpf*
+   sudo chmod g+r /dev/bpf*
+
+* Example of configuring devfs to set permissions of BPF devices, adding
+  entries to ``/etc/devfs.conf`` to grant ``admin`` group permission to
+  capture packets:
+
+.. console::
+
+   sudo sh -c 'echo "own    bpf    root:admin" >> /etc/devfs.conf'
+   sudo sh -c 'echo "perm   bpf    0640" >> /etc/devfs.conf'
+   sudo service devfs restart
+
+.. note:: As of Mac OS X 10.6, the BPF device is on devfs, but the used version
+   of devfs isn't capable of setting the device permissions.  The permissions
+   can be changed manually, but they will not survive a reboot.
+
+Why isn't Bro producing the logs I expect? (A Note About Checksums)
+-------------------------------------------------------------------
+
+Normally, Bro's event engine will discard packets which don't have valid
+checksums.  This can be a problem if one wants to analyze locally
+generated/captured traffic on a system that offloads checksumming to the
+network adapter.  In that case, all transmitted/captured packets will have
+bad checksums because they haven't yet been calculated by the NIC, thus
+such packets will not undergo analysis defined in Bro policy scripts as they
+normally would.  Bad checksums in traces may also be a result of some packet
+alteration tools.
+
+Bro has two options to workaround such situations and ignore bad checksums:
+
+1) The ``-C`` command line option to ``bro``.
+2) An option called ``ignore_checksums`` that can be redefined at the policy
+   policy script layer (e.g. in your ``$PREFIX/share/bro/site/local/bro``):
+
+    .. code:: bro
+
+      redef ignore_checksums = T;
+
+The other alternative is to disable checksum offloading for your
+network adapter, but this is not always possible or desirable.
+
+.. raw:: html
+
+    </div>

doc/geoip.rst

@@ -3,7 +3,7 @@
 GeoLocation
 ===========
 
-.. class:: opening
+.. rst-class:: opening
 
     During the process of creating policy scripts the need may arise
     to find the geographic location for an IP address. Bro has support

doc/index.rst

@@ -1,50 +1,72 @@
-
+.. Bro documentation master file
 
 Bro Documentation
 =================
 
-`Getting Started <{{git('bro:doc/quickstart.rst')}}>`_
+.. toctree::
-    A quick introduction into using Bro 2.x.
+   :maxdepth: 1
 
-`Bro 1.5 to 2.0 Upgrade Guide <{{git('bro:doc/upgrade.rst')}}>`_
+   INSTALL
-    Guidelines and notes about upgrading from Bro 1.5 to 2.x. Lots of
+   quickstart
-    things have changed, so make sure to read this when upgrading.
+   upgrade
-
+   faq
-`BroControl <{{git('broctl:doc/broctl.rst')}}>`_
+   reporting-problems
-    An interactive console for managing Bro installations.
-
-`Script Reference <{{autodoc_bro_scripts}}/index.html>`_ 
-    A complete reference of all policy scripts shipped with Bro.
-
-`FAQ <{{docroot}}/documentation/faq.html>`_
-    A list with frequently asked questions.
-
-`How to Report a Problem <{{docroot}}/documentation/reporting-problems.html>`_
-    Some advice for when you see Bro doing something you believe it
-    shouldn't.
 
 Frameworks
 ----------
 
-Bro comes with a number of frameworks, some of which are described in
+.. toctree::
-more detail here:
+   :maxdepth: 1
 
-`Notice <{{git('bro:doc/notice.rst')}}>`_
+   notice
-    The notice framework.
+   logging
-
+   cluster
-`Logging <{{git('bro:doc/logging.rst')}}>`_
+   signatures
-    Customizing and extensing Bro's logging.
-
-`Cluster <{{git('bro:doc/cluster.rst')}}>`_
-    Setting up a Bro Cluster when a single box can't handle the traffic anymore.
-        
-`Signatures <{{git('bro:doc/signatures.rst')}}>`_ 
-    Bro has support for traditional NIDS signatures as well.
 
 How-Tos
 -------
 
-We also collect more specific How-Tos on specific topics:
+.. toctree::
+   :maxdepth: 1
 
-`Using GeoIP in Bro scripts <{{git('bro:doc/geoip.rst')}}>`_
+   geoip
-    Installation and usage of the the GeoIP library.
+
+Script Reference
+----------------
+
+.. toctree::
+   :maxdepth: 1
+
+   scripts/common
+   scripts/builtins
+   scripts/bifs
+   scripts/packages
+   scripts/index
+
+Other Bro Components
+--------------------
+
+.. toctree::
+   :maxdepth: 1
+
+   components/btest/README
+   components/broccoli/README
+   components/broccoli-python/README
+   components/broctl/README
+   components/capstats/README
+   components/pysubnettree/README
+   components/trace-summary/README
+
+Indices and tables
+------------------
+
+* :ref:`genindex`
+* :ref:`search`
+
+Internal References
+-------------------
+
+.. toctree::
+   :maxdepth: 1
+
+   scripts/internal

doc/logging.rst

@@ -2,7 +2,7 @@
 Customizing Bro's Logging
 ==========================
 
-.. class:: opening
+.. rst-class:: opening
 
    Bro comes with a flexible key-value based logging interface that
    allows fine-grained control of what gets logged and how it is

doc/notice.rst

@@ -2,7 +2,7 @@
 Notice Framework
 ================
 
-.. class:: opening
+.. rst-class:: opening
 
     One of the easiest ways to customize Bro is writing a local notice
     policy. Bro can detect a large number of potentially interesting

doc/quickstart.rst

@@ -3,12 +3,13 @@
 .. _MacPorts: http://www.macports.org
 .. _Fink: http://www.finkproject.org
 .. _Homebrew: http://mxcl.github.com/homebrew
+.. _bro downloads page: http://bro-ids.org/download/index.html
 
 =================
 Quick Start Guide
 =================
 
-.. class:: opening
+.. rst-class:: opening
 
    The short story for getting Bro up and running in a simple configuration
    for analysis of either live traffic from a network interface or a packet
@@ -26,20 +27,19 @@ source code forms.
 Pre-Built Binary Release Packages
 ---------------------------------
 
-See the `downloads page <{{docroot}}/download/index.html>`_ for currently
+See the `bro downloads page`_ for currently supported/targeted platforms.
-supported/targeted platforms.
 
 * RPM
 
 .. console::
 
-   > sudo yum localinstall Bro-all*.rpm
+   sudo yum localinstall Bro-all*.rpm
 
 * DEB
 
 .. console::
 
-   > sudo gdebi Bro-all-*.deb
+   sudo gdebi Bro-all-*.deb
 
 * MacOS Disk Image with Installer
 
@@ -60,13 +60,13 @@ Required Dependencies
 
 .. console::
 
-   > sudo yum install cmake make gcc gcc-c++ flex bison libpcap-devel openssl-devel python-devel swig
+   sudo yum install cmake make gcc gcc-c++ flex bison libpcap-devel openssl-devel python-devel swig zlib-devel file-devel
 
 * DEB/Debian-based Linux:
 
 .. console::
 
-   > sudo apt-get install cmake make gcc g++ flex bison libpcap-dev libssl-dev python-dev swig
+   sudo apt-get install cmake make gcc g++ flex bison libpcap-dev libssl-dev python-dev swig zlib1g-dev libmagic-dev
 
 * FreeBSD
 
@@ -75,7 +75,7 @@ Required Dependencies
 
 .. console::
 
-   > sudo pkg_add -r cmake swig bison python
+   sudo pkg_add -r cmake swig bison python
 
 * Mac OS X
 
@@ -94,29 +94,28 @@ Required Dependencies
 Optional Dependencies
 ~~~~~~~~~~~~~~~~~~~~~
 
-Bro can use libmagic for identifying file types, libGeoIP for geo-locating
+Bro can use libGeoIP for geo-locating IP addresses and sendmail for
-IP addresses, libz for (de)compression during analysis and communication,
+sending emails.
-and sendmail for sending emails.
 
 * RPM/RedHat-based Linux:
 
 .. console::
 
-   > sudo yum install zlib-devel file-devel GeoIP-devel sendmail
+   sudo yum install GeoIP-devel sendmail
 
 * DEB/Debian-based Linux:
 
 .. console::
 
-   > sudo apt-get install zlib1g-dev libmagic-dev libgeoip-dev sendmail
+   sudo apt-get install libgeoip-dev sendmail
 
 * Ports-based FreeBSD
 
 .. console::
 
-   > sudo pkg_add -r GeoIP
+   sudo pkg_add -r GeoIP
 
-  libz, libmagic, and sendmail are typically already available.
+  sendmail is typically already available.
 
 * Mac OS X
 
@@ -125,20 +124,20 @@ and sendmail for sending emails.
   Fink Homebrew), they should be automatically detected and Bro will compile
   against them.
 
-Additional steps may be needed to `get the right GeoIP database
+Additional steps may be needed to :doc:`get the right GeoIP database <geoip>`
-<{{git('bro:doc/geoip.rst')}}>`_.
 
 Compiling Bro Source Code
 ~~~~~~~~~~~~~~~~~~~~~~~~~
 
 Bro releases are bundled into source packages for convenience and
-available from the `downloads page <{{docroot}}/download/index.html>`_.
+available from the `bro downloads page`_.
 
-The latest Bro development versions are obtainable through git repositories
+The latest Bro development versions are obtainable through git
-hosted at `git.bro-ids.org <http://git.bro-ids.org>`_.  See our `git development
+repositories hosted at `git.bro-ids.org <http://git.bro-ids.org>`_.  See
-documentation <{{docroot}}/development/process.html>`_ for comprehensive
+our `git development documentation
-information on Bro's use of git revision control, but the short story for
+<http://bro-ids.org/development/process.html>`_ for comprehensive
-downloading the full source code experience for Bro via git is:
+information on Bro's use of git revision control, but the short story
+for downloading the full source code experience for Bro via git is:
 
 .. console::
 
@@ -157,9 +156,9 @@ desired root install path):
 
 .. console::
 
-    > ./configure --prefix=/desired/install/path
+    ./configure --prefix=/desired/install/path
-    > make
+    make
-    > make install
+    make install
 
 The default installation prefix is ``/usr/local/bro``, which would typically
 require root privileges when doing the ``make install``.
@@ -174,13 +173,13 @@ Bourne-Shell Syntax:
 
 .. console::
 
-   > export PATH=/usr/local/bro/bin:$PATH
+   export PATH=/usr/local/bro/bin:$PATH
 
 C-Shell Syntax:
 
 .. console::
 
-   > setenv PATH /usr/local/bro/bin:$PATH
+   setenv PATH /usr/local/bro/bin:$PATH
 
 Or substitute ``/opt/bro/bin`` instead if you installed from a binary package.
 
@@ -211,7 +210,7 @@ Now start the BroControl shell like:
 
 .. console::
 
-   > broctl
+   broctl
 
 Since this is the first-time use of the shell, perform an initial installation
 of the BroControl configuration:
@@ -233,10 +232,9 @@ policy and output the results in ``$PREFIX/logs``.
 
 .. note:: The user starting BroControl needs permission to capture
    network traffic. If you are not root, you may need to grant further
-   privileges to the account you're using; see the `FAQ
+   privileges to the account you're using; see the :doc:`FAQ <faq>`.
-   <{{docroot}}/documentation/faq.html>`_. Also, if it
+   Also, if it looks like Bro is not seeing any traffic, check out
-   looks like Bro is not seeing any traffic, check out the FAQ entry
+   the FAQ entry on checksum offloading.
-   checksum offloading.
 
 You can leave it running for now, but to stop this Bro instance you would do:
 
@@ -244,9 +242,7 @@ You can leave it running for now, but to stop this Bro instance you would do:
 
    [BroControl] > stop
 
-We also recommend to insert the following entry into `crontab`:
+We also recommend to insert the following entry into `crontab`::
-
-.. console::
 
       0-59/5 * * * * $PREFIX/bin/broctl cron
 
@@ -373,9 +369,7 @@ the variable's value may not change at run-time, but whose initial value can be
 modified via the ``redef`` operator at parse-time.
 
 So let's continue on our path to modify the behavior for the two SSL
-and SSH notices.  Looking at
+and SSH notices.  Looking at :doc:`scripts/base/frameworks/notice/main`,
-`$PREFIX/share/bro/base/frameworks/notice/main.bro
-<{{autodoc_bro_scripts}}/scripts/base/frameworks/notice/main.html>`_,
 we see that it advertises:
 
 .. code:: bro
@@ -477,8 +471,7 @@ tweak the most basic options.  Here's some suggestions on what to explore next:
 * Reading the code of scripts that ship with Bro is also a great way to gain
   understanding of the language and how you can start writing your own custom
   analysis.
-* Review the `FAQ <{{docroot}}/documentation/faq.html>`_.
+* Review the :doc:`FAQ <faq>`.
-* Check out more `documentation <{{docroot}}/documentation/index.html>`_.
 * Continue reading below for another mini-tutorial on using Bro as a standalone
   command-line utility.
 
@@ -496,7 +489,7 @@ Analyzing live traffic from an interface is simple:
 
 .. console::
 
-   > bro -i en0 <list of scripts to load>
+   bro -i en0 <list of scripts to load>
 
 ``en0`` can be replaced by the interface of your choice and for the list of
 scripts, you can just use "all" for now to perform all the default analysis
@@ -504,7 +497,7 @@ that's available.
 
 Bro will output log files into the working directory.
 
-.. note:: The `FAQ <{{docroot}}/documentation/faq.html>`_ entries about
+.. note:: The :doc:`FAQ <faq>` entries about
    capturing as an unprivileged user and checksum offloading are particularly
    relevant at this point.
 
@@ -513,7 +506,7 @@ command-line:
 
 .. console::
 
-   > bro -i en0 local
+   bro -i en0 local
 
 This will cause Bro to print a warning about lacking the
 ``Site::local_nets`` variable being configured. You can supply this
@@ -522,7 +515,7 @@ in place of the example subnets):
 
 .. console::
 
-  > bro -r mypackets.trace local "Site::local_nets += { 1.2.3.0/24, 5.6.7.0/24 }"
+   bro -r mypackets.trace local "Site::local_nets += { 1.2.3.0/24, 5.6.7.0/24 }"
 
 
 Reading Packet Capture (pcap) Files
@@ -533,7 +526,7 @@ like this:
 
 .. console::
 
-   > sudo tcpdump -i en0 -s 0 -w mypackets.trace
+   sudo tcpdump -i en0 -s 0 -w mypackets.trace
 
 Where ``en0`` can be replaced by the correct interface for your system as
 shown by e.g. ``ifconfig``. (The ``-s 0`` argument tells it to capture
@@ -544,7 +537,7 @@ and tell Bro to perform all the default analysis on the capture which primarily
 
 .. console::
 
-   > bro -r mypackets.trace
+   bro -r mypackets.trace
 
 Bro will output log files into the working directory.
 
@@ -553,7 +546,7 @@ script that we include as a suggested configuration:
 
 .. console::
 
-  > bro -r mypackets.trace local
+  bro -r mypackets.trace local
 
 
 Telling Bro Which Scripts to Load
@@ -563,7 +556,7 @@ A command-line invocation of Bro typically looks like:
 
 .. console::
 
-   > bro <options> <policies...>
+   bro <options> <policies...>
 
 Where the last arguments are the specific policy scripts that this Bro
 instance will load.  These arguments don't have to include the ``.bro``
@@ -578,7 +571,7 @@ logging) and adds SSL certificate validation.
 
 .. console::
 
-   > bro -r mypackets.trace protocols/ssl/validate-certs
+   bro -r mypackets.trace protocols/ssl/validate-certs
 
 You might notice that a script you load from the command line uses the
 ``@load`` directive in the Bro language to declare dependence on other scripts.

doc/reporting-problems.rst

@@ -0,0 +1,194 @@
+
+Reporting Problems
+==================
+
+.. rst-class:: opening
+
+    Here we summarizes some steps to follow when you see Bro doing
+    something it shouldn't. To provide help, it is often crucial for
+    us to have a way of reliably reproducing the effect you're seeing.
+    Unfortunately, reproducing problems can be rather tricky with Bro
+    because more often than not, they occur only in either very rare
+    situations or only after Bro has been running for some time. In
+    particular, getting a small trace showing a specific effect can be
+    a real problem. In the following, we'll summarize some strategies
+    to this end.
+
+Reporting Problems
+------------------
+
+Generally, when you encounter a problem with Bro, the best thing to do
+is opening a new ticket in `Bro's issue tracker
+<http://tracker.bro-ids.org/>`__ and include information on how to
+reproduce the issue. Ideallt, your ticket should come with the
+following:
+
+* The Bro version you're using (if working directly from the git
+  repository, the branch and revision number.)
+
+* The output you're seeing along with a description what you'd expect
+  Bro to do instead.
+
+* A *small* trace in `libpcap format <http://tcpdump.org>`__
+  demonstrating the effect (assuming the problem doesn't happen right
+  at startup already).
+
+* The exact command-line you're using to run Bro with that trace. If
+  you can, please try to run the Bro binary directly from the command
+  line rather than using BroControl.
+
+* Any non-standard scripts you're using (but please only those really
+  necessary; just a small code snippet triggering the problem would
+  perfect).
+
+* If you encounter a crash, information from the core dump, such as
+  the stack backtrace, can be very helpful. See below for more on
+  this.
+
+
+How Do I Get a Trace File?
+--------------------------
+
+As Bro is usually running live, coming up with a small trace file that
+reproduces a problem can turn out to be quite a challenge. Often it
+works to best to start with a large trace that triggers the problem,
+and then successively thin it out as much a possible.
+
+To get to the initial large trace, here are few things you can try:
+
+* Capture a trace with `tcpdump <http://www.tcpdump.org/>`__, either
+  on the same interface Bro is running on, or on another host where
+  you can generate traffic of the kind likely triggering the problem
+  (e.g., if you're seeing problems with the HTTP analyzer, record some
+  of your Web browsing on your desktop.) When using tcpdump, don't
+  forget to record *complete* packets (``tcpdump -s 0 ...``). You can
+  reduce the amount of traffic captured by using a suitable BPF filter
+  (e.g., for HTTP only, try ``port 80``). 
+
+* Bro's command-line option ``-w <trace>`` records all packets it
+  processes into the given the file. You can then later run Bro
+  offline on this trace and it will process the packets in the same
+  way as it did live. This is particularly helpful with problems that
+  only occur after Bro has already been running for some time. For
+  example, sometimes a crash may be triggered by a particular kind of
+  traffic only occurring rarely. Running Bro live with ``-w`` and
+  then, after the crash, offline on the recorded trace might, with a
+  little bit of luck, reproduce the the problem reliably. However, be
+  careful with ``-w``: it can result in huge trace files, quickly
+  filling up your disk. (One way to mitigate the space issues is to
+  periodically delete the trace file by configuring
+  ``rotate-logs.bro`` accordingly. BroControl does that for you if you
+  set its ``SaveTraces`` option.)
+
+* Finally, you can try running Bro on a publically available trace
+  file, such as `anonymized FTP traffic <http://www-nrg.ee.lbl.gov
+  /anonymized-traces.html>`__, `headers-only enterprise traffic
+  <http://www.icir.org/enterprise-tracing/Overview.html>`__, or
+  `Defcon traffic <http://cctf.shmoo.com/>`__. Some of these
+  particularly stress certain components of Bro (e.g., the Defcon
+  traces contain tons of scans).
+
+Once you have a trace that demonstrates the effect, you will often
+notice that it's pretty big, in particular if recorded from the link
+you're monitoring. Therefore, the next step is to shrink its size as
+much as possible. Here are a few things you can try to this end:
+
+* Very often, a single connection is able to demonstrate the problem.
+  If you can identify which one it is (e.g., from one of Bro's
+  ``*.log`` files) you can extract the connection's packets from the
+  trace usong tcpdump by filtering for the corresponding 4-tuple of
+  addresses and ports:
+
+  .. console::
+    
+    > tcpdump -r large.trace -w small.trace host <ip1> and port <port1> and host <ip2> and port <port2>
+
+* If you can't reduce the problem to a connection, try to identify
+  either a host pair or a single host triggering it, and filter down
+  the trace accordingly.
+
+* You can try to extract a smaller time slice from the trace using 
+  `TCPslice <http://www.tcpdump.org/related.html>`__. For example, to
+  extract the first 100 seconds from the trace:
+
+  .. console::
+
+    # Test comment
+    > tcpslice +100 <in >out
+    
+Alternatively, tcpdump extracts the first ``n`` packets with its
+option ``-c <n>``.
+
+
+Getting More Information After a Crash
+--------------------------------------
+
+If Bro crashes, a *core dump* can be very helpful to nail down the
+problem. Examining a core is not for the faint of heart but can reveal
+extremely useful information.
+
+First, you should configure Bro with the option ``--enable-debug`` and
+recompile; this will disable all compiler optimizations and thus make
+the core dump more useful (don't expect great performance with this
+version though; compiling Bro without optimization has a noticeable
+impact on its CPU usage.). Then enable core dumps if you don't have
+already (e.g., ``ulimit -c unlimited`` if you're using a bash).
+
+Once Bro has crashed, start gdb with the Bro binary and the file
+containing the core dump. (Alternatively, you can also run Bro
+directly inside gdb instead of working from a core file.) The first
+helpful information to include with your tracker ticket is a stack
+backtrace, which you get with gdb's ``bt`` command:
+
+.. console::
+    
+    > gdb bro core
+    [...]
+    > bt
+    
+
+If the crash occurs inside Bro's script interpreter, the next thing to
+do is identifying the line of script code processed just before the
+abnormal termination. Look for methods in the stack backtrace which
+belong to any of the script interpreter's classes. Roughly speaking,
+these are all classes with names ending in ``Expr``, ``Stmt``, or
+``Val``. Then climb up the stack with ``up`` until you reach the first
+of these methods. The object to which ``this`` is pointing will have a
+``Location`` object, which in turn contains the file name and line
+number of the corresponding piece of script code. Continuing the
+example from above, here's how to get that information:
+
+.. console::
+
+    [in gdb]
+    > up
+    > ...
+    > up
+    > print this->location->filename
+    > print this->location->first_line
+    
+
+If the crash occurs while processing input packets but you cannot
+directly tell which connection is responsible (and thus not extract
+its packets from the trace as suggested above), try getting the
+4-tuple of the connection currently being processed from the core dump
+by again examining the stack backtrace, this time looking for methods
+belonging to the ``Connection`` class. That class has members
+``orig_addr``/``resp_addr`` and ``orig_port``/``resp_port`` storing
+(pointers to) the IP addresses and ports respectively:
+
+.. console::
+
+    [in gdb]
+    > up
+    > ...
+    > up
+    > printf "%08x:%04x %08x:%04x\n", *this->orig_addr, this->orig_port, *this->resp_addr, this->resp_port
+
+
+Note that these values are stored in `network byte order
+<http://en.wikipedia.org/wiki/Endianness#Endianness_in_networking>`__
+so you will need flip the bytes around if you are on a low-endian
+machine (which is why the above example prints them in hex). For
+example, if an IP address prints as ``0100007f`` , that's 127.0.0.1 .
+

doc/scripts/CMakeLists.txt

@@ -1,16 +1,3 @@
-set(BIF_SRC_DIR ${PROJECT_SOURCE_DIR}/src)
-set(RST_OUTPUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/rest_output)
-set(DOC_OUTPUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/out)
-set(DOC_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/source)
-set(DOC_SOURCE_WORKDIR ${CMAKE_CURRENT_BINARY_DIR}/source)
-
-file(GLOB_RECURSE DOC_SOURCES FOLLOW_SYMLINKS "*")
-
-# configure the Sphinx config file (expand variables CMake might know about)
-configure_file(${CMAKE_CURRENT_SOURCE_DIR}/conf.py.in
-               ${CMAKE_CURRENT_BINARY_DIR}/conf.py
-               @ONLY)
-
 # find out what BROPATH to use when executing bro
 execute_process(COMMAND ${CMAKE_BINARY_DIR}/bro-path-dev
                 OUTPUT_VARIABLE BROPATH
@@ -48,7 +35,7 @@ endif ()
 #   which summary text can be extracted at build time
 # ${group}_doc_names: a running list of reST style document names that can be
 #   given to a :doc: role, shared indices with ${group}_files
-#
+
 macro(REST_TARGET srcDir broInput)
     set(absSrcPath ${srcDir}/${broInput})
     get_filename_component(basename ${broInput} NAME)
@@ -86,7 +73,7 @@ macro(REST_TARGET srcDir broInput)
     elseif (${extension} STREQUAL ".bif.bro")
         set(group bifs)
     elseif (relDstDir)
-        set(pkgIndex scripts/${relDstDir}/index)
+        set(pkgIndex ${relDstDir}/index)
         set(group ${pkgIndex})
         # add package index to master package list if not already in it
         list(FIND MASTER_PKG_LIST ${pkgIndex} _found)
@@ -134,6 +121,7 @@ macro(REST_TARGET srcDir broInput)
         ARGS -rf .state *.log *.rst
         DEPENDS bro
         DEPENDS ${absSrcPath}
+        WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
         COMMENT "[Bro] Generating reST docs for ${broInput}"
     )
 
@@ -143,12 +131,10 @@ endmacro(REST_TARGET)
 include(DocSourcesList.cmake)
 
 # create temporary list of all docs to include in the master policy/index file
-set(MASTER_POLICY_INDEX ${CMAKE_CURRENT_BINARY_DIR}/policy_index)
 file(WRITE ${MASTER_POLICY_INDEX} "${MASTER_POLICY_INDEX_TEXT}")
 
 # create the temporary list of all packages to include in the master
 # policy/packages.rst file
-set(MASTER_PACKAGE_INDEX ${CMAKE_CURRENT_BINARY_DIR}/pkg_index)
 set(MASTER_PKG_INDEX_TEXT "")
 foreach (pkg ${MASTER_PKG_LIST})
     # strip of the trailing /index for the link name
@@ -186,11 +172,11 @@ if (EXISTS ${RST_OUTPUT_DIR})
         list(FIND ALL_REST_OUTPUTS ${_doc} _found)
         if (_found EQUAL -1)
             file(REMOVE ${_doc})
-            message(STATUS "AutoDoc: remove stale reST doc: ${_doc}")
+            message(STATUS "Broxygen: remove stale reST doc: ${_doc}")
             string(REPLACE .rst .bro _brofile ${_doc})
             if (EXISTS ${_brofile})
                 file(REMOVE ${_brofile})
-                message(STATUS "AutoDoc: remove stale bro source: ${_brofile}")
+                message(STATUS "Broxygen: remove stale bro source: ${_brofile}")
             endif ()
         endif ()
     endforeach ()
@@ -211,57 +197,3 @@ add_custom_target(restclean
                   COMMAND "${CMAKE_COMMAND}" -E remove_directory
                           ${RST_OUTPUT_DIR}
                   VERBATIM)
-
-# The "sphinxdoc" target generates reST documentation for any outdated bro
-# scripts and then uses Sphinx to generate HTML documentation from the reST
-add_custom_target(sphinxdoc
-                  # copy the template documentation to the build directory
-                  # to give as input for sphinx
-                  COMMAND "${CMAKE_COMMAND}" -E copy_directory
-                          ${DOC_SOURCE_DIR}
-                          ${DOC_SOURCE_WORKDIR}
-                  # copy generated policy script documentation into the
-                  # working copy of the template documentation
-                  COMMAND "${CMAKE_COMMAND}" -E copy_directory
-                          ${RST_OUTPUT_DIR}
-                          ${DOC_SOURCE_WORKDIR}/scripts
-                  # append to the master index of all policy scripts
-                  COMMAND cat ${MASTER_POLICY_INDEX} >>
-                          ${DOC_SOURCE_WORKDIR}/scripts/index.rst
-                  # append to the master index of all policy packages
-                  COMMAND cat ${MASTER_PACKAGE_INDEX} >>
-                          ${DOC_SOURCE_WORKDIR}/packages.rst
-                  # construct a reST file for each group
-                  COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/group_index_generator.py
-                          ${CMAKE_CURRENT_BINARY_DIR}/group_list
-                          ${CMAKE_CURRENT_BINARY_DIR}
-                          ${DOC_SOURCE_WORKDIR}
-                  # tell sphinx to generate html
-                  COMMAND sphinx-build
-                          -b html
-                          -c ${CMAKE_CURRENT_BINARY_DIR}
-                          -d ${DOC_OUTPUT_DIR}/doctrees
-                          ${DOC_SOURCE_WORKDIR}
-                          ${DOC_OUTPUT_DIR}/html
-                  # create symlink to the html output directory for convenience
-                  COMMAND "${CMAKE_COMMAND}" -E create_symlink
-                          ${DOC_OUTPUT_DIR}/html
-                          ${CMAKE_BINARY_DIR}/html
-                  WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
-                  COMMENT "[Sphinx] Generating HTML policy script docs"
-                  # SOURCES just adds stuff to IDE projects as a convenience
-                  SOURCES ${DOC_SOURCES})
-
-# The "sphinxclean" target removes just the Sphinx input/output directories
-# from the build directory.
-add_custom_target(sphinxclean
-                  COMMAND "${CMAKE_COMMAND}" -E remove_directory
-                          ${DOC_SOURCE_WORKDIR}
-                  COMMAND "${CMAKE_COMMAND}" -E remove_directory
-                          ${DOC_OUTPUT_DIR}
-                  VERBATIM)
-
-add_dependencies(sphinxdoc sphinxclean restdoc)
-
-add_dependencies(doc sphinxdoc)
-add_dependencies(docclean sphinxclean restclean)

doc/scripts/README

@@ -15,29 +15,10 @@ by CMake:
     ``build/`` directory inside ``reST`` (a symlink to
     ``doc/scripts/rest_output``).
 
-``doc``
-
-    This target depends on a Python interpreter (>=2.5) and
-    `Sphinx <http://sphinx.pocoo.org/>`_ being installed.  Sphinx can be
-    installed like::
-
-        > sudo easy_install sphinx
-
-    This target will first build ``restdoc`` target and then copy the
-    resulting reST files as an input directory to Sphinx.
-
-    After completion, HTML documentation can be located in the CMake
-    ``build/`` directory inside ``html`` (a symlink to
-    ``doc/scripts/out/html``)
-
 ``restclean``
 
     This target removes any reST documentation that has been generated so far.
 
-``docclean``
-
-    This target removes Sphinx inputs and outputs from the CMake ``build/`` dir.
-
 The ``genDocSourcesList.sh`` script can be run to automatically generate
 ``DocSourcesList.cmake``, which is the file CMake uses to define the list
 of documentation targets.  This script should be run after adding new
@@ -54,18 +35,10 @@ script's name to the blacklist, then append a ``rest_target()`` to the
 ``statictext`` variable where the first argument is the source directory
 containing the policy script to document, the second argument is the file
 name of the policy script, and the third argument is the path/name of a
-pre-created reST document in the ``source/`` directory to which the
+pre-created reST document in the ``../`` source directory to which the
 ``make doc`` process can append script documentation references.  This
 pre-created reST document should also then be linked to from the TOC tree
-in ``source/index.rst``.
+in ``../index.rst``.
-
-The Sphinx source tree template in ``source/`` can be modified to add more
-common/general documentation, style sheets, JavaScript, etc.  The Sphinx
-config file is produced from ``conf.py.in``, so that can be edited to change
-various Sphinx options, like setting the default HTML rendering theme.
-There is also a custom Sphinx domain implemented in ``source/ext/bro.py``
-which adds some reST directives and roles that aid in generating useful
-index entries and cross-references.
 
 See ``example.bro`` for an example of how to document a Bro script such that
 ``make doc`` will be able to produce reST/HTML documentation for it.

doc/scripts/bifs.rst

@@ -0,0 +1,5 @@
+.. This is a stub doc to which broxygen appends during the build process
+
+Built-In Functions (BIFs)
+=========================
+

doc/scripts/index.rst

@@ -0,0 +1,8 @@
+.. This is a stub doc to which broxygen appends during the build process
+
+Index of All Bro Scripts
+========================
+
+.. toctree::
+   :maxdepth: 1
+

doc/scripts/internal.rst

@@ -0,0 +1,5 @@
+.. This is a stub doc to which broxygen appends during the build process
+
+Internal Scripts
+================
+

doc/scripts/packages.rst

@@ -1,7 +1,7 @@
-.. This is a stub doc to which the build process can append.
+.. This is a stub doc to which broxygen appends during the build process
 
-Bro Script Packages
+Index of All Bro Script Packages
-===================
+================================
 
 Bro has the following script packages (e.g. collections of related scripts in
 a common directory).  If the package directory contains a ``__load__.bro``

doc/scripts/source/_templates/layout.html

@@ -1,5 +0,0 @@
-{% extends "!layout.html" %}
-{% block extrahead %}
-    <script type="text/javascript" src="{{ pathto('_static/showhide.js', 1) }}"></script>
-    {{ super() }}
-{% endblock %}

doc/scripts/source/bifs.rst

@@ -1,5 +0,0 @@
-.. This is a stub doc to which the build process can append.
-
-Built-In Functions (BIFs)
-=========================
-

doc/scripts/source/index.rst

@@ -1,23 +0,0 @@
-.. Bro documentation master file
-
-Welcome to Bro's documentation!
-===============================
-
-Contents:
-
-.. toctree::
-   :maxdepth: 1
-   :glob:
-
-   common
-   builtins
-   internal
-   bifs
-   packages
-   scripts/index
-
-Indices and tables
-==================
-
-* :ref:`genindex`
-* :ref:`search`

doc/scripts/source/internal.rst

@@ -1,5 +0,0 @@
-.. This is a stub doc to which the build process can append.
-
-Internal Scripts
-================
-

doc/scripts/source/scripts/index.rst

@@ -1,6 +0,0 @@
-Index of All Bro Script Documentation
-=====================================
-
-.. toctree::
-   :maxdepth: 1
-

doc/signatures.rst

@@ -3,7 +3,7 @@
 Signatures
 ==========
 
-.. class:: opening
+.. rst-class:: opening
 
     Bro relies primarily on its extensive scripting language for 
     defining and analyzing detection policies. In addition, however,
@@ -47,8 +47,8 @@ piece of payload which triggered the pattern match.
 
 To turn such ``signature_match`` events into actual alarms, you can
 load Bro's ``signature.bro`` script. This script contains a default
-event handler that raises ``SensitiveSignature`` `Notices
+event handler that raises ``SensitiveSignature`` :doc:`Notices <notice>`
-<notices.html>`_ (as well as others; see the beginning of the script).
+(as well as others; see the beginning of the script).
 
 As signatures are independent of Bro's policy scripts, they are put
 into their own file(s). There are two ways to specify which files

doc/upgrade.rst

@@ -3,7 +3,7 @@
 Upgrading From Bro 1.5 to 2.0
 =============================
 
-.. class:: opening
+.. rst-class:: opening
 
    This guide details differences between Bro versions 1.5 and 2.0
    that may be important for users to know as they work on updating
@@ -39,7 +39,7 @@ version. The two rules of thumb are:
         if you need help.
 
 Below we summarize changes from 1.x to 2.x in more detail. This list
-isn't complete, see the `CHANGES <{{git('bro:CHANGES', 'txt')}}>`_ file in the
+isn't complete, see the :download:`CHANGES <CHANGES>` file in the
 distribution for the full story. 
 
 Default Scripts
@@ -131,8 +131,8 @@ Logging Framework
     endpoint.
 
 - The new logging framework makes it possible to extend, customize,
-  and filter logs very easily. See `the logging framework
+  and filter logs very easily. See the :doc:`logging framework <logging>`
-  <{{git('bro:doc/logging.rst')}}>`_ more information on usage.
+  more information on usage.
 
 - A common pattern found in the new scripts is to store logging stream
   records for protocols inside the ``connection`` records so that
@@ -155,8 +155,7 @@ Notice Framework
 
 The way users interact with "notices" has changed significantly in
 order to make it easier to define a site policy and more extensible
-for adding customized actions. See the `the notice framework
+for adding customized actions. See the :doc:`notice framework <notice>`.
-<{{git('bro:doc/notice.rst')}}>`_.
 
 
 New Default Settings
@@ -198,7 +197,7 @@ Variable Naming
 
 - Identifiers may have been renamed to conform to new `scripting
   conventions
-  <{{docroot}}/development/script-conventions.html>`_
+  <http://www.bro-ids.org/development/script-conventions.html>`_
 
 
 BroControl
@@ -240,7 +239,7 @@ Development Infrastructure
 Bro development has moved from using SVN to Git for revision control.
 Users that like to use the latest Bro developments by checking it out
 from the source repositories should see the `development process
-<{{docroot}}/development/process.html>`_. Note that all the various
+<http://www.bro-ids.org/development/process.html>`_. Note that all the various
 sub-components now reside on their own repositories. However, the
 top-level Bro repository includes them as git submodules so it's easu
 to check them all out simultaneously.

src/Base64.cc

@@ -1,14 +1,27 @@
 #include "config.h"
 #include "Base64.h"
 
-static int base64_table[256];
+int Base64Decoder::default_base64_table[256];
+const string Base64Decoder::default_alphabet = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
 
-static void init_base64_table()
+int* Base64Decoder::InitBase64Table(const string& alphabet)
 	{
-	static int table_initialized = 0;
+	assert(alphabet.size() == 64);
 
-	if ( ++table_initialized > 1 )
+	static bool default_table_initialized = false;
-		return;
+
+	if ( alphabet == default_alphabet && default_table_initialized )
+		return default_base64_table;
+
+	int* base64_table = 0;
+
+	if ( alphabet == default_alphabet )
+		{
+		base64_table = default_base64_table;
+		default_table_initialized = true;
+		}
+	else
+		base64_table = new int[256];
 
 	int i;
 	for ( i = 0; i < 256; ++i )
@@ -16,28 +29,36 @@ static void init_base64_table()
 
 	for ( i = 0; i < 26; ++i )
 		{
-		base64_table['A' + i] = i;
+		base64_table[int(alphabet[0 + i])] = i;
-		base64_table['a' + i] = i + 26;
+		base64_table[int(alphabet[26 + i])] = i + 26;
 		}
 
 	for ( i = 0; i < 10; ++i )
-		base64_table['0' + i] = i + 52;
+		base64_table[int(alphabet[52 + i])] = i + 52;
 
 	// Casts to avoid compiler warnings.
-	base64_table[int('+')] = 62;
+	base64_table[int(alphabet[62])] = 62;
-	base64_table[int('/')] = 63;
+	base64_table[int(alphabet[63])] = 63;
 	base64_table[int('=')] = 0;
+
+	return base64_table;
 	}
 
-Base64Decoder::Base64Decoder(Analyzer* arg_analyzer)
+Base64Decoder::Base64Decoder(Analyzer* arg_analyzer, const string& alphabet)
 	{
-	init_base64_table();
+	base64_table = InitBase64Table(alphabet.size() ? alphabet : default_alphabet);
 	base64_group_next = 0;
 	base64_padding = base64_after_padding = 0;
 	errored = 0;
 	analyzer = arg_analyzer;
 	}
 
+Base64Decoder::~Base64Decoder()
+	{
+	if ( base64_table != default_base64_table )
+		delete base64_table;
+	}
+
 int Base64Decoder::Decode(int len, const char* data, int* pblen, char** pbuf)
 	{
 	int blen;
@@ -142,13 +163,21 @@ int Base64Decoder::Done(int* pblen, char** pbuf)
 	return 0;
 	}
 
-BroString* decode_base64(const BroString* s)
+
+BroString* decode_base64(const BroString* s, const BroString* a)
 	{
+	if ( a && a->Len() != 64 )
+		{
+		reporter->Error("base64 decoding alphabet is not 64 characters: %s",
+		                a->CheckString());
+		return 0;
+		}
+
 	int buf_len = int((s->Len() + 3) / 4) * 3 + 1;
 	int rlen2, rlen = buf_len;
 	char* rbuf2, *rbuf = new char[rlen];
 
-	Base64Decoder dec(0);
+	Base64Decoder dec(0, a ? a->CheckString() : "");
 	if ( dec.Decode(s->Len(), (const char*) s->Bytes(), &rlen, &rbuf) == -1 )
 		goto err;
 

src/Base64.h

@@ -13,11 +13,11 @@
 
 class Base64Decoder {
 public:
-	// <analyzer> is used for error reporting, and it should be zero
+	// <analyzer> is used for error reporting, and it should be zero when
-	// when the decoder is called by the built-in function
+	// the decoder is called by the built-in function decode_base64().
-	// decode_base64().
+	// Empty alphabet indicates the default base64 alphabet.
-	Base64Decoder(Analyzer* analyzer);
+	Base64Decoder(Analyzer* analyzer, const string& alphabet = "");
-	~Base64Decoder()	{ }
+	~Base64Decoder();
 
 	// A note on Decode():
 	//
@@ -57,8 +57,13 @@ protected:
 	int base64_after_padding;
 	int errored;	// if true, we encountered an error - skip further processing
 	Analyzer* analyzer;
+	int* base64_table;
+
+	static int* InitBase64Table(const string& alphabet);
+	static int default_base64_table[256];
+	static const string default_alphabet;
 };
 
-BroString* decode_base64(const BroString* s);
+BroString* decode_base64(const BroString* s, const BroString* a = 0);
 
 #endif /* base64_h */

src/CMakeLists.txt

@@ -423,15 +423,7 @@ add_definitions(-DBRO_BUILD_PATH="${CMAKE_CURRENT_BINARY_DIR}")
 
 add_executable(bro ${bro_SRCS} ${bro_HEADERS})
 
-set(brolibs
+target_link_libraries(bro ${brodeps})
-    ${BinPAC_LIBRARY}
-    ${PCAP_LIBRARY}
-    ${OpenSSL_LIBRARIES}
-    ${BIND_LIBRARY}
-    ${OPTLIBS}
-)
-
-target_link_libraries(bro ${brolibs})
 
 install(TARGETS bro DESTINATION bin)
 install(FILES ${INSTALL_BIF_OUTPUTS} DESTINATION ${BRO_SCRIPT_INSTALL_PATH}/base)

src/ChunkedIO.cc

@@ -1170,8 +1170,6 @@ void ChunkedIOSSL::Stats(char* buffer, int length)
 	ChunkedIO::Stats(buffer + i, length - i);
 	}
 
-#ifdef HAVE_LIBZ
-
 bool CompressedChunkedIO::Init()
 	{
 	zin.zalloc = 0;
@@ -1348,5 +1346,3 @@ void CompressedChunkedIO::Stats(char* buffer, int length)
 	io->Stats(buffer + i, length - i);
 	buffer[length-1] = '\0';
 	}
-
-#endif	/* HAVE_LIBZ */

src/ChunkedIO.h

@@ -287,8 +287,6 @@ private:
 	static SSL_CTX* ctx;
 };
 
-#ifdef HAVE_LIBZ
-
 #include <zlib.h>
 
 // Wrapper class around a another ChunkedIO which the (un-)compresses data.
@@ -335,6 +333,4 @@ protected:
 	unsigned long uncompressed_bytes_written;
 };
 
-#endif	/* HAVE_LIBZ */
-
 #endif

src/FileAnalyzer.cc

@@ -3,23 +3,19 @@
 #include "FileAnalyzer.h"
 #include "Reporter.h"
 
-#ifdef HAVE_LIBMAGIC
 magic_t File_Analyzer::magic = 0;
 magic_t File_Analyzer::magic_mime = 0;
-#endif
 
 File_Analyzer::File_Analyzer(Connection* conn)
 : TCP_ApplicationAnalyzer(AnalyzerTag::File, conn)
 	{
 	buffer_len = 0;
 
-#ifdef HAVE_LIBMAGIC
 	if ( ! magic )
 		{
 		InitMagic(&magic, MAGIC_NONE);
 		InitMagic(&magic_mime, MAGIC_MIME);
 		}
-#endif
 	}
 
 void File_Analyzer::DeliverStream(int len, const u_char* data, bool orig)
@@ -52,13 +48,11 @@ void File_Analyzer::Identify()
 	const char* descr = 0;
 	const char* mime = 0;
 
-#ifdef HAVE_LIBMAGIC
 	if ( magic )
 		descr = magic_buffer(magic, buffer, buffer_len);
 
 	if ( magic_mime )
 		mime = magic_buffer(magic_mime, buffer, buffer_len);
-#endif
 
 	val_list* vl = new val_list;
 	vl->append(BuildConnVal());
@@ -68,7 +62,6 @@ void File_Analyzer::Identify()
 	ConnectionEvent(file_transferred, vl);
 	}
 
-#ifdef HAVE_LIBMAGIC
 void File_Analyzer::InitMagic(magic_t* magic, int flags)
 	{
 	*magic = magic_open(flags);
@@ -83,4 +76,3 @@ void File_Analyzer::InitMagic(magic_t* magic, int flags)
 		*magic = 0;
 		}
 	}
-#endif

src/FileAnalyzer.h

@@ -5,9 +5,7 @@
 
 #include "TCP.h"
 
-#ifdef HAVE_LIBMAGIC
 #include <magic.h>
-#endif
 
 class File_Analyzer : public TCP_ApplicationAnalyzer {
 public:
@@ -31,12 +29,10 @@ protected:
 	char buffer[BUFFER_SIZE];
 	int buffer_len;
 
-#ifdef HAVE_LIBMAGIC
 	static void InitMagic(magic_t* magic, int flags);
 
 	static magic_t magic;
 	static magic_t magic_mime;
-#endif
 };
 
 #endif

src/HTTP.cc

@@ -43,9 +43,7 @@ HTTP_Entity::HTTP_Entity(HTTP_Message *arg_message, MIME_Entity* parent_entity,
 	header_length = 0;
 	deliver_body = (http_entity_data != 0);
 	encoding = IDENTITY;
-#ifdef HAVE_LIBZ
 	zip = 0;
-#endif
 	}
 
 void HTTP_Entity::EndOfData()
@@ -53,7 +51,6 @@ void HTTP_Entity::EndOfData()
 	if ( DEBUG_http )
 		DEBUG_MSG("%.6f: end of data\n", network_time);
 
-#ifdef HAVE_LIBZ
 	if ( zip )
 		{
 		zip->Done();
@@ -61,7 +58,6 @@ void HTTP_Entity::EndOfData()
 		zip = 0;
 		encoding = IDENTITY;
 		}
-#endif
 
 	if ( body_length )
 		http_message->MyHTTP_Analyzer()->
@@ -179,7 +175,6 @@ private:
 
 void HTTP_Entity::DeliverBody(int len, const char* data, int trailing_CRLF)
 	{
-#ifdef HAVE_LIBZ
 	if ( encoding == GZIP || encoding == DEFLATE )
 		{
 		ZIP_Analyzer::Method method =
@@ -198,7 +193,6 @@ void HTTP_Entity::DeliverBody(int len, const char* data, int trailing_CRLF)
 		zip->NextStream(len, (const u_char*) data, false);
 		}
 	else
-#endif
 		DeliverBodyClear(len, data, trailing_CRLF);
 	}
 
@@ -450,9 +444,7 @@ void HTTP_Entity::SubmitAllHeaders()
 	// content-length headers or if connection is to be closed afterwards
 	// anyway.
 	else if ( http_message->MyHTTP_Analyzer()->IsConnectionClose ()
-#ifdef HAVE_LIBZ
 		  || encoding == GZIP || encoding == DEFLATE
-#endif
 		 )
 		{
 		// FIXME: Using INT_MAX is kind of a hack here.  Better

src/HTTP.h

@@ -29,10 +29,8 @@ public:
 			int expect_body);
 	~HTTP_Entity()
 		{
-#ifdef HAVE_LIBZ
 		if ( zip )
 			{ zip->Done(); delete zip; }
-#endif
 		}
 
 	void EndOfData();
@@ -55,9 +53,7 @@ protected:
 	int64_t header_length;
 	int deliver_body;
 	enum { IDENTITY, GZIP, COMPRESS, DEFLATE } encoding;
-#ifdef HAVE_LIBZ
 	ZIP_Analyzer* zip;
-#endif
 
 	MIME_Entity* NewChildEntity() { return new HTTP_Entity(http_message, this, 1); }
 

src/IRC.cc

@@ -1188,15 +1188,10 @@ void IRC_Analyzer::DeliverStream(int length, const u_char* line, bool orig)
 	if ( orig_status == REGISTERED && resp_status == REGISTERED &&
 	     orig_zip_status == ACCEPT_ZIP && resp_zip_status == ACCEPT_ZIP )
 		{
-#ifdef HAVE_LIBZ
 		orig_zip_status = ZIP_LOADED;
 		resp_zip_status = ZIP_LOADED;
 		AddSupportAnalyzer(new ZIP_Analyzer(Conn(), true));
 		AddSupportAnalyzer(new ZIP_Analyzer(Conn(), false));
-#else
-		reporter->Error("IRC analyzer lacking libz support");
-		Remove();
-#endif
 		}
 
 	return;

src/LogMgr.cc

@@ -151,7 +151,7 @@ bool LogVal::IsCompatibleType(BroType* t, bool atomic_only)
 		if ( ! t->IsSet() )
 			return false;
 
-		return IsCompatibleType(t->AsSetType()->Indices()->PureType());
+		return IsCompatibleType(t->AsSetType()->Indices()->PureType(), true);
 		}
 
 	case TYPE_VECTOR:

src/RemoteSerializer.cc

@@ -1222,10 +1222,7 @@ bool RemoteSerializer::SendCapabilities(Peer* peer)
 
 	uint32 caps = 0;
 
-#ifdef HAVE_LIBZ
 	caps |= Peer::COMPRESSION;
-#endif
-
 	caps |= Peer::PID_64BIT;
 	caps |= Peer::NEW_CACHE_STRATEGY;
 
@@ -2106,11 +2103,9 @@ bool RemoteSerializer::ProcessPhaseDone()
 
 bool RemoteSerializer::HandshakeDone(Peer* peer)
 	{
-#ifdef HAVE_LIBZ
 	if ( peer->caps & Peer::COMPRESSION && peer->comp_level > 0 )
 		if ( ! SendToChild(MSG_COMPRESS, peer, 1, peer->comp_level) )
 			return false;
-#endif
 
 	if ( ! (peer->caps & Peer::PID_64BIT) )
 		Log(LogInfo, "peer does not support 64bit PIDs; using compatibility mode", peer);
@@ -3699,11 +3694,6 @@ bool SocketComm::ProcessListen()
 
 bool SocketComm::ProcessParentCompress()
 	{
-#ifndef HAVE_LIBZ
-	InternalError("supposed to enable compression but don't have zlib");
-	return false;
-#else
-
 	assert(parent_args);
 	uint32* args = (uint32*) parent_args->data;
 
@@ -3727,7 +3717,6 @@ bool SocketComm::ProcessParentCompress()
 	Log(fmt("enabling compression (level %d)", level), parent_peer);
 
 	return true;
-#endif
 	}
 
 bool SocketComm::ProcessRemoteMessage(SocketComm::Peer* peer)
@@ -3847,10 +3836,6 @@ bool SocketComm::ProcessPeerCompress(Peer* peer)
 	{
 	peer->state = MSG_NONE;
 
-#ifndef HAVE_LIBZ
-	Error("peer compresses although we do not support it", peer);
-	return false;
-#else
 	if ( ! parent_peer->compressor )
 		{
 		parent_peer->io = new CompressedChunkedIO(parent_peer->io);
@@ -3862,7 +3847,6 @@ bool SocketComm::ProcessPeerCompress(Peer* peer)
 	((CompressedChunkedIO*) peer->io)->EnableDecompression();
 	Log("enabling decompression", peer);
 	return true;
-#endif
 	}
 
 bool SocketComm::Connect(Peer* peer)

src/ZIP.cc

@@ -2,8 +2,6 @@
 
 #include "ZIP.h"
 
-#ifdef HAVE_LIBZ
-
 ZIP_Analyzer::ZIP_Analyzer(Connection* conn, bool orig, Method arg_method)
 : TCP_SupportAnalyzer(AnalyzerTag::Zip, conn, orig)
 	{
@@ -89,4 +87,3 @@ void ZIP_Analyzer::DeliverStream(int len, const u_char* data, bool orig)
 		}
 	while ( zip->avail_out == 0 );
 	}
-#endif

src/ZIP.h

@@ -5,8 +5,6 @@
 
 #include "config.h"
 
-#ifdef HAVE_LIBZ
-
 #include "zlib.h"
 #include "TCP.h"
 
@@ -29,4 +27,3 @@ protected:
 };
 
 #endif
-#endif

src/bro.bif

@@ -1860,6 +1860,18 @@ function decode_base64%(s: string%): string
 		}
 	%}
 
+function decode_base64_custom%(s: string, a: string%): string
+	%{
+	BroString* t = decode_base64(s->AsString(), a->AsString());
+	if ( t )
+		return new StringVal(t);
+	else
+		{
+		reporter->Error("error in decoding string %s", s->CheckString());
+		return new StringVal("");
+		}
+	%}
+
 %%{
 #include "DCE_RPC.h"
 
@@ -3272,18 +3284,15 @@ function enable_raw_output%(f: file%): any
 	%}
 
 %%{
-#ifdef HAVE_LIBMAGIC
 extern "C" {
 #include <magic.h>
 }
-#endif
 %%}
 
 function identify_data%(data: string, return_mime: bool%): string
 	%{
 	const char* descr = "";
 
-#ifdef HAVE_LIBMAGIC
 	static magic_t magic_mime = 0;
 	static magic_t magic_descr = 0;
 
@@ -3309,7 +3318,6 @@ function identify_data%(data: string, return_mime: bool%): string
 		}
 
 	descr = magic_buffer(*magic, data->Bytes(), data->Len());
-#endif
 
 	return new StringVal(descr);
 	%}

testing/btest/Baseline/bifs.decode_base64/out

@@ -0,0 +1,6 @@
+bro
+bro
+bro
+bro
+bro
+bro

testing/btest/bifs/decode_base64.bro

@@ -0,0 +1,14 @@
+# @TEST-EXEC: bro -b %INPUT >out
+# @TEST-EXEC: btest-diff out
+
+global default_alphabet: string = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
+
+global my_alphabet: string = "!#$%&/(),-.:;<>@[]^ `_{|}~abcdefghijklmnopqrstuvwxyz0123456789+?";
+
+print decode_base64("YnJv");
+print decode_base64_custom("YnJv", default_alphabet);
+print decode_base64_custom("}n-v", my_alphabet);
+
+print decode_base64("YnJv");
+print decode_base64_custom("YnJv", default_alphabet);
+print decode_base64_custom("}n-v", my_alphabet);

testing/btest/scripts/base/protocols/http/http-mime-and-md5.bro

@@ -2,7 +2,6 @@
 # will normalize mime types other than the target type to prevent sensitivity
 # to varying versions of libmagic.
 
-# @TEST-REQUIRES: grep -q '#define HAVE_LIBMAGIC' $BUILD/config.h
 # @TEST-EXEC: bro -r $TRACES/http-pipelined-requests.trace %INPUT > output
 # @TEST-EXEC: btest-diff http.log
 

testing/btest/scripts/base/protocols/irc/dcc-extract.test

@@ -2,7 +2,6 @@
 # correctly extracted.  The mime type of the file transferred is normalized
 # to prevent sensitivity to libmagic version being used.
 
-# @TEST-REQUIRES: grep -q '#define HAVE_LIBMAGIC' $BUILD/config.h
 # @TEST-EXEC: bro -r $TRACES/irc-dcc-send.trace %INPUT
 # @TEST-EXEC: btest-diff irc.log
 # @TEST-EXEC: btest-diff irc-dcc-item_192.168.1.77:57655-209.197.168.151:1024_1.dat

testing/btest/scripts/base/protocols/smtp/mime-extract.test

@@ -1,4 +1,3 @@
-# @TEST-REQUIRES: grep -q '#define HAVE_LIBMAGIC' $BUILD/config.h
 # @TEST-EXEC: bro -r $TRACES/smtp.trace %INPUT
 # @TEST-EXEC: btest-diff smtp_entities.log
 # @TEST-EXEC: btest-diff smtp-entity_10.10.1.4:1470-74.53.140.153:25_1.dat

testing/btest/scripts/base/protocols/smtp/mime.test

@@ -1,7 +1,6 @@
 # Checks logging of mime types and md5 calculation.  Mime type in the log
 # is normalized to prevent sensitivity to libmagic version.
 
-# @TEST-REQUIRES: grep -q '#define HAVE_LIBMAGIC' $BUILD/config.h
 # @TEST-EXEC: bro -r $TRACES/smtp.trace %INPUT
 # @TEST-EXEC: btest-diff smtp_entities.log