if
+ given (e.g., "c", "python", etc.).
+
+ - .. console::
+
+ Highlits the following code block as a shell session.
+
+ For compatibility with the original version, "sourcecode" is
+ equivalent to "code".
+
+Original comment:
+
+ The Pygments reStructuredText directive
+ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+ This fragment is a Docutils_ 0.5 directive that renders source code
+ (to HTML only, currently) via Pygments.
+
+ To use it, adjust the options below and copy the code into a module
+ that you import on initialization. The code then automatically
+ registers a ``sourcecode`` directive that you can use instead of
+ normal code blocks like this::
+
+ .. sourcecode:: python
+
+ My code goes here.
+
+ If you want to have different code styles, e.g. one with line numbers
+ and one without, add formatters with their names in the VARIANTS dict
+ below. You can invoke them instead of the DEFAULT one by using a
+ directive option::
+
+ .. sourcecode:: python
+ :linenos:
+
+ My code goes here.
+
+ Look at the `directive documentation`_ to get all the gory details.
+
+ .. _Docutils: http://docutils.sf.net/
+ .. _directive documentation:
+ http://docutils.sourceforge.net/docs/howto/rst-directives.html
+
+ :copyright: Copyright 2006-2010 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+# Options
+# ~~~~~~~
+
+# Set to True if you want inline CSS styles instead of classes
+INLINESTYLES = False
+
+from pygments.formatters import HtmlFormatter
+
+class MyHtmlFormatter(HtmlFormatter):
+ def format_unencoded(self, tokensource, outfile):
+
+ # A NOP currently.
+ new_tokens = []
+ for (i, piece) in tokensource:
+ new_tokens += [(i, piece)]
+
+ return super(MyHtmlFormatter, self).format_unencoded(new_tokens, outfile)
+
+# The default formatter
+DEFAULT = MyHtmlFormatter(noclasses=INLINESTYLES, cssclass="pygments")
+
+# Add name -> formatter pairs for every variant you want to use
+VARIANTS = {
+ # 'linenos': HtmlFormatter(noclasses=INLINESTYLES, linenos=True),
+}
+
+
+import textwrap
+
+from docutils import nodes
+from docutils.parsers.rst import directives, Directive
+
+from pygments import highlight
+from pygments.lexers import get_lexer_by_name, guess_lexer, TextLexer
+from pygments.token import Text, Keyword, Error, Operator, Name
+from pygments.filter import Filter
+
+# Ugly hack to register the Bro lexer. I'm sure there's a better way to do it,
+# but it's not obvious ...
+from bro_lexer.bro import BroLexer
+from pygments.lexers._mapping import LEXERS
+LEXERS['BroLexer'] = ('bro_lexer.bro', BroLexer.name, BroLexer.aliases, BroLexer.filenames, ())
+
+class Pygments(Directive):
+ """ Source code syntax hightlighting.
+ """
+ #max_line_length = 68
+ max_line_length = 0
+
+ required_arguments = 0
+ optional_arguments = 1
+ final_argument_whitespace = True
+ option_spec = dict([(key, directives.flag) for key in VARIANTS])
+ has_content = True
+
+ def wrapped_content(self):
+ content = []
+
+ if Console.max_line_length:
+ for line in self.content:
+ content += textwrap.wrap(line, Console.max_line_length, subsequent_indent=" ")
+ else:
+ content = self.content
+
+ return u'\n'.join(content)
+
+ def run(self):
+ self.assert_has_content()
+
+ content = self.wrapped_content()
+
+ if len(self.arguments) > 0:
+ try:
+ lexer = get_lexer_by_name(self.arguments[0])
+ except (ValueError, IndexError):
+ # lexer not found, use default.
+ lexer = TextLexer()
+ else:
+ lexer = guess_lexer(content)
+
+ # import sys
+ # print >>sys.stderr, self.arguments, lexer.__class__
+
+ # take an arbitrary option if more than one is given
+ formatter = self.options and VARIANTS[self.options.keys()[0]] or DEFAULT
+ parsed = highlight(content, lexer, formatter)
+ return [nodes.raw('', parsed, format='html')]
+
+class MyFilter(Filter):
+ def filter(self, lexer, stream):
+
+ bol = True
+
+ for (ttype, value) in stream:
+ # Color the '>' prompt sign.
+ if bol and ttype is Text and value == ">":
+ ttype = Name.Variable.Class # This gives us a nice red.
+
+ # Discolor builtin, that can look funny.
+ if ttype is Name.Builtin:
+ ttype = Text
+
+ bol = value.endswith("\n")
+
+ yield (ttype, value)
+
+class Console(Pygments):
+ required_arguments = 0
+ optional_arguments = 0
+
+ def run(self):
+ self.assert_has_content()
+ content = self.wrapped_content()
+ lexer = get_lexer_by_name("sh")
+ lexer.add_filter(MyFilter())
+ parsed = highlight(content, lexer, DEFAULT)
+ return [nodes.raw('', parsed, format='html')]
+
+directives.register_directive('sourcecode', Pygments)
+directives.register_directive('code', Pygments)
+directives.register_directive('console', Console)
diff --git a/doc/faq.rst b/doc/faq.rst
new file mode 100644
index 0000000000..286f4b32cd
--- /dev/null
+++ b/doc/faq.rst
@@ -0,0 +1,145 @@
+
+==========================
+Frequently Asked Questions
+==========================
+
+.. raw:: html
+
+
+
+.. contents::
+
+Installation and Configuration
+==============================
+
+How can I tune my operating system for best capture performance?
+----------------------------------------------------------------
+
+Here are some pointers to more information:
+
+* Fabian Schneider's research on `high performance packet capture
+ `_
+
+* `NSMWiki `_ has page on
+ *Collecting Data*.
+
+* An `IMC 2010 paper
+ `_ by
+ Lothar Braun et. al evaluates packet capture performance on
+ commodity hardware
+
+What does an error message like ``internal error: NB-DNS error`` mean?
+---------------------------------------------------------------------------------------------------------------------------------
+
+That often means that DNS is not set up correctly on the system
+running Bro. Try verifying from the command line that DNS lookups
+work, e.g., ``host www.google.com``.
+
+
+Usage
+=====
+
+How can I identify backscatter?
+-------------------------------
+
+Identifying backscatter via connections labeled as ``OTH`` is not
+a reliable means to detect backscatter. Use rather the following
+procedure:
+
+* Enable connection history via ``redef record_state_history=T`` to
+ track all control/data packet types in connection logs.
+
+* Backscatter is now visible in terms of connections that never had an
+ initial ``SYN`` but started instead with a ``SYN-ACK`` or ``RST``
+ (though this latter generally is just discarded).
+
+Is there help for understanding Bro's resource consumption?
+-----------------------------------------------------------
+
+There are two scripts that collect statistics on resource usage:
+``stats.bro`` and ``profiling.bro``. The former is quite lightweight,
+while the latter should only be used for debugging. Furthermore,
+there's also ``print-globals.bro``, which prints the size of all
+global script variable at termination.
+
+How can I capture packets as an unprivileged user?
+--------------------------------------------------
+
+Normally, unprivileged users cannot capture packets from a network
+interface, which means they would not be able to use Bro to read/analyze
+live traffic. However, there are ways to enable packet capture
+permission for non-root users, which is worth doing in the context of
+using Bro to monitor live traffic
+
+With Linux Capabilities
+^^^^^^^^^^^^^^^^^^^^^^^
+
+Fully implemented since Linux kernel 2.6.24, capabilities are a way of
+parceling superuser privileges into distinct units. Attach capabilities
+required to capture packets to the ``bro`` executable file like this:
+
+.. console::
+
+ > sudo setcap cap_net_raw,cap_net_admin=eip /path/to/bro
+
+Now any unprivileged user should have the capability to capture packets
+using Bro provided that they have the traditional file permissions to
+read/execute the ``bro`` binary.
+
+With BPF Devices
+^^^^^^^^^^^^^^^^
+
+Systems using Berkeley Packet Filter (BPF) (e.g. FreeBSD & Mac OS X)
+can allow users with read access to a BPF device to capture packets from
+it using libpcap.
+
+* Example of manually changing BPF device permissions to allow users in
+ the ``admin`` group to capture packets:
+
+.. console::
+
+ > sudo chgrp admin /dev/bpf*
+ > sudo chmod g+r /dev/bpf*
+
+* Example of configuring devfs to set permissions of BPF devices, adding
+ entries to ``/etc/devfs.conf`` to grant ``admin`` group permission to
+ capture packets:
+
+.. console::
+
+ > sudo sh -c 'echo "own bpf root:admin" >> /etc/devfs.conf'
+ > sudo sh -c 'echo "perm bpf 0640" >> /etc/devfs.conf'
+ > sudo service devfs restart
+
+.. note:: As of Mac OS X 10.6, the BPF device is on devfs, but the used version
+ of devfs isn't capable of setting the device permissions. The permissions
+ can be changed manually, but they will not survive a reboot.
+
+Why isn't Bro producing the logs I expect? (A Note About Checksums)
+-------------------------------------------------------------------
+
+Normally, Bro's event engine will discard packets which don't have valid
+checksums. This can be a problem if one wants to analyze locally
+generated/captured traffic on a system that offloads checksumming to the
+network adapter. In that case, all transmitted/captured packets will have
+bad checksums because they haven't yet been calculated by the NIC, thus
+such packets will not undergo analysis defined in Bro policy scripts as they
+normally would. Bad checksums in traces may also be a result of some packet
+alteration tools.
+
+Bro has two options to workaround such situations and ignore bad checksums:
+
+1) The ``-C`` command line option to ``bro``.
+2) An option called ``ignore_checksums`` that can be redefined at the policy
+ policy script layer (e.g. in your ``$PREFIX/share/bro/site/local/bro``):
+
+ .. code:: bro
+
+ redef ignore_checksums = T;
+
+The other alternative is to disable checksum offloading for your
+network adapter, but this is not always possible or desirable.
+
+.. raw:: html
+
+
diff --git a/doc/geoip.rst b/doc/geoip.rst
index 53413b5bee..bd9ae0c08d 100644
--- a/doc/geoip.rst
+++ b/doc/geoip.rst
@@ -3,7 +3,7 @@
GeoLocation
===========
-.. class:: opening
+.. rst-class:: opening
During the process of creating policy scripts the need may arise
to find the geographic location for an IP address. Bro has support
diff --git a/doc/deployment.png b/doc/images/deployment.png
similarity index 100%
rename from doc/deployment.png
rename to doc/images/deployment.png
diff --git a/doc/index.rst b/doc/index.rst
index 56a9e06fe8..022552f3d4 100644
--- a/doc/index.rst
+++ b/doc/index.rst
@@ -6,10 +6,11 @@ Bro Documentation
.. toctree::
:maxdepth: 1
+ INSTALL
quickstart
upgrade
- FAQ
- Reporting Problems
+ faq
+ reporting-problems
Frameworks
----------
@@ -42,6 +43,20 @@ Script Reference
scripts/packages
scripts/index
+Other Bro Components
+--------------------
+
+.. toctree::
+ :maxdepth: 1
+
+ components/btest/README
+ components/broccoli/README
+ components/broccoli-python/README
+ components/broctl/README
+ components/capstats/README
+ components/pysubnettree/README
+ components/trace-summary/README
+
Indices and tables
------------------
diff --git a/doc/logging.rst b/doc/logging.rst
index e7151b7f47..2817cadd45 100644
--- a/doc/logging.rst
+++ b/doc/logging.rst
@@ -2,7 +2,7 @@
Customizing Bro's Logging
==========================
-.. class:: opening
+.. rst-class:: opening
Bro comes with a flexible key-value based logging interface that
allows fine-grained control of what gets logged and how it is
diff --git a/doc/notice.rst b/doc/notice.rst
index 1322d44585..b1ffdacb75 100644
--- a/doc/notice.rst
+++ b/doc/notice.rst
@@ -2,7 +2,7 @@
Notice Framework
================
-.. class:: opening
+.. rst-class:: opening
One of the easiest ways to customize Bro is writing a local notice
policy. Bro can detect a large number of potentially interesting
diff --git a/doc/quickstart.rst b/doc/quickstart.rst
index 22523e1618..c45db71ad9 100644
--- a/doc/quickstart.rst
+++ b/doc/quickstart.rst
@@ -8,7 +8,7 @@
Quick Start Guide
=================
-.. class:: opening
+.. rst-class:: opening
The short story for getting Bro up and running in a simple configuration
for analysis of either live traffic from a network interface or a packet
diff --git a/doc/reporting-problems.rst b/doc/reporting-problems.rst
new file mode 100644
index 0000000000..a1105708eb
--- /dev/null
+++ b/doc/reporting-problems.rst
@@ -0,0 +1,194 @@
+
+Reporting Problems
+==================
+
+.. rst-class:: opening
+
+ Here we summarizes some steps to follow when you see Bro doing
+ something it shouldn't. To provide help, it is often crucial for
+ us to have a way of reliably reproducing the effect you're seeing.
+ Unfortunately, reproducing problems can be rather tricky with Bro
+ because more often than not, they occur only in either very rare
+ situations or only after Bro has been running for some time. In
+ particular, getting a small trace showing a specific effect can be
+ a real problem. In the following, we'll summarize some strategies
+ to this end.
+
+Reporting Problems
+------------------
+
+Generally, when you encounter a problem with Bro, the best thing to do
+is opening a new ticket in `Bro's issue tracker
+`__ and include information on how to
+reproduce the issue. Ideallt, your ticket should come with the
+following:
+
+* The Bro version you're using (if working directly from the git
+ repository, the branch and revision number.)
+
+* The output you're seeing along with a description what you'd expect
+ Bro to do instead.
+
+* A *small* trace in `libpcap format `__
+ demonstrating the effect (assuming the problem doesn't happen right
+ at startup already).
+
+* The exact command-line you're using to run Bro with that trace. If
+ you can, please try to run the Bro binary directly from the command
+ line rather than using BroControl.
+
+* Any non-standard scripts you're using (but please only those really
+ necessary; just a small code snippet triggering the problem would
+ perfect).
+
+* If you encounter a crash, information from the core dump, such as
+ the stack backtrace, can be very helpful. See below for more on
+ this.
+
+
+How Do I Get a Trace File?
+--------------------------
+
+As Bro is usually running live, coming up with a small trace file that
+reproduces a problem can turn out to be quite a challenge. Often it
+works to best to start with a large trace that triggers the problem,
+and then successively thin it out as much a possible.
+
+To get to the initial large trace, here are few things you can try:
+
+* Capture a trace with `tcpdump `__, either
+ on the same interface Bro is running on, or on another host where
+ you can generate traffic of the kind likely triggering the problem
+ (e.g., if you're seeing problems with the HTTP analyzer, record some
+ of your Web browsing on your desktop.) When using tcpdump, don't
+ forget to record *complete* packets (``tcpdump -s 0 ...``). You can
+ reduce the amount of traffic captured by using a suitable BPF filter
+ (e.g., for HTTP only, try ``port 80``).
+
+* Bro's command-line option ``-w `` records all packets it
+ processes into the given the file. You can then later run Bro
+ offline on this trace and it will process the packets in the same
+ way as it did live. This is particularly helpful with problems that
+ only occur after Bro has already been running for some time. For
+ example, sometimes a crash may be triggered by a particular kind of
+ traffic only occurring rarely. Running Bro live with ``-w`` and
+ then, after the crash, offline on the recorded trace might, with a
+ little bit of luck, reproduce the the problem reliably. However, be
+ careful with ``-w``: it can result in huge trace files, quickly
+ filling up your disk. (One way to mitigate the space issues is to
+ periodically delete the trace file by configuring
+ ``rotate-logs.bro`` accordingly. BroControl does that for you if you
+ set its ``SaveTraces`` option.)
+
+* Finally, you can try running Bro on a publically available trace
+ file, such as `anonymized FTP traffic `__, `headers-only enterprise traffic
+ `__, or
+ `Defcon traffic `__. Some of these
+ particularly stress certain components of Bro (e.g., the Defcon
+ traces contain tons of scans).
+
+Once you have a trace that demonstrates the effect, you will often
+notice that it's pretty big, in particular if recorded from the link
+you're monitoring. Therefore, the next step is to shrink its size as
+much as possible. Here are a few things you can try to this end:
+
+* Very often, a single connection is able to demonstrate the problem.
+ If you can identify which one it is (e.g., from one of Bro's
+ ``*.log`` files) you can extract the connection's packets from the
+ trace usong tcpdump by filtering for the corresponding 4-tuple of
+ addresses and ports:
+
+ .. console::
+
+ > tcpdump -r large.trace -w small.trace host and port and host and port
+
+* If you can't reduce the problem to a connection, try to identify
+ either a host pair or a single host triggering it, and filter down
+ the trace accordingly.
+
+* You can try to extract a smaller time slice from the trace using
+ `TCPslice `__. For example, to
+ extract the first 100 seconds from the trace:
+
+ .. console::
+
+ # Test comment
+ > tcpslice +100 out
+
+Alternatively, tcpdump extracts the first ``n`` packets with its
+option ``-c ``.
+
+
+Getting More Information After a Crash
+--------------------------------------
+
+If Bro crashes, a *core dump* can be very helpful to nail down the
+problem. Examining a core is not for the faint of heart but can reveal
+extremely useful information.
+
+First, you should configure Bro with the option ``--enable-debug`` and
+recompile; this will disable all compiler optimizations and thus make
+the core dump more useful (don't expect great performance with this
+version though; compiling Bro without optimization has a noticeable
+impact on its CPU usage.). Then enable core dumps if you don't have
+already (e.g., ``ulimit -c unlimited`` if you're using a bash).
+
+Once Bro has crashed, start gdb with the Bro binary and the file
+containing the core dump. (Alternatively, you can also run Bro
+directly inside gdb instead of working from a core file.) The first
+helpful information to include with your tracker ticket is a stack
+backtrace, which you get with gdb's ``bt`` command:
+
+.. console::
+
+ > gdb bro core
+ [...]
+ > bt
+
+
+If the crash occurs inside Bro's script interpreter, the next thing to
+do is identifying the line of script code processed just before the
+abnormal termination. Look for methods in the stack backtrace which
+belong to any of the script interpreter's classes. Roughly speaking,
+these are all classes with names ending in ``Expr``, ``Stmt``, or
+``Val``. Then climb up the stack with ``up`` until you reach the first
+of these methods. The object to which ``this`` is pointing will have a
+``Location`` object, which in turn contains the file name and line
+number of the corresponding piece of script code. Continuing the
+example from above, here's how to get that information:
+
+.. console::
+
+ [in gdb]
+ > up
+ > ...
+ > up
+ > print this->location->filename
+ > print this->location->first_line
+
+
+If the crash occurs while processing input packets but you cannot
+directly tell which connection is responsible (and thus not extract
+its packets from the trace as suggested above), try getting the
+4-tuple of the connection currently being processed from the core dump
+by again examining the stack backtrace, this time looking for methods
+belonging to the ``Connection`` class. That class has members
+``orig_addr``/``resp_addr`` and ``orig_port``/``resp_port`` storing
+(pointers to) the IP addresses and ports respectively:
+
+.. console::
+
+ [in gdb]
+ > up
+ > ...
+ > up
+ > printf "%08x:%04x %08x:%04x\n", *this->orig_addr, this->orig_port, *this->resp_addr, this->resp_port
+
+
+Note that these values are stored in `network byte order
+`__
+so you will need flip the bytes around if you are on a low-endian
+machine (which is why the above example prints them in hex). For
+example, if an IP address prints as ``0100007f`` , that's 127.0.0.1 .
+
diff --git a/doc/scripts/CMakeLists.txt b/doc/scripts/CMakeLists.txt
index 7b47424c6c..c14eab6d25 100644
--- a/doc/scripts/CMakeLists.txt
+++ b/doc/scripts/CMakeLists.txt
@@ -121,6 +121,7 @@ macro(REST_TARGET srcDir broInput)
ARGS -rf .state *.log *.rst
DEPENDS bro
DEPENDS ${absSrcPath}
+ WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
COMMENT "[Bro] Generating reST docs for ${broInput}"
)
diff --git a/doc/scripts/README b/doc/scripts/README
index ca7f28492b..b3e44914f4 100644
--- a/doc/scripts/README
+++ b/doc/scripts/README
@@ -15,29 +15,10 @@ by CMake:
``build/`` directory inside ``reST`` (a symlink to
``doc/scripts/rest_output``).
-``doc``
-
- This target depends on a Python interpreter (>=2.5) and
- `Sphinx `_ being installed. Sphinx can be
- installed like::
-
- > sudo easy_install sphinx
-
- This target will first build ``restdoc`` target and then copy the
- resulting reST files as an input directory to Sphinx.
-
- After completion, HTML documentation can be located in the CMake
- ``build/`` directory inside ``html`` (a symlink to
- ``doc/scripts/out/html``)
-
``restclean``
This target removes any reST documentation that has been generated so far.
-``docclean``
-
- This target removes Sphinx inputs and outputs from the CMake ``build/`` dir.
-
The ``genDocSourcesList.sh`` script can be run to automatically generate
``DocSourcesList.cmake``, which is the file CMake uses to define the list
of documentation targets. This script should be run after adding new
@@ -54,18 +35,10 @@ script's name to the blacklist, then append a ``rest_target()`` to the
``statictext`` variable where the first argument is the source directory
containing the policy script to document, the second argument is the file
name of the policy script, and the third argument is the path/name of a
-pre-created reST document in the ``source/`` directory to which the
+pre-created reST document in the ``../`` source directory to which the
``make doc`` process can append script documentation references. This
pre-created reST document should also then be linked to from the TOC tree
-in ``source/index.rst``.
-
-The Sphinx source tree template in ``source/`` can be modified to add more
-common/general documentation, style sheets, JavaScript, etc. The Sphinx
-config file is produced from ``conf.py.in``, so that can be edited to change
-various Sphinx options, like setting the default HTML rendering theme.
-There is also a custom Sphinx domain implemented in ``source/ext/bro.py``
-which adds some reST directives and roles that aid in generating useful
-index entries and cross-references.
+in ``../index.rst``.
See ``example.bro`` for an example of how to document a Bro script such that
``make doc`` will be able to produce reST/HTML documentation for it.
diff --git a/doc/signatures.rst b/doc/signatures.rst
index 25dfc31f5e..f69b655624 100644
--- a/doc/signatures.rst
+++ b/doc/signatures.rst
@@ -3,7 +3,7 @@
Signatures
==========
-.. class:: opening
+.. rst-class:: opening
Bro relies primarily on its extensive scripting language for
defining and analyzing detection policies. In addition, however,
diff --git a/doc/upgrade.rst b/doc/upgrade.rst
index 696c64ef7f..ff426b1a77 100644
--- a/doc/upgrade.rst
+++ b/doc/upgrade.rst
@@ -3,7 +3,7 @@
Upgrading From Bro 1.5 to 2.0
=============================
-.. class:: opening
+.. rst-class:: opening
This guide details differences between Bro versions 1.5 and 2.0
that may be important for users to know as they work on updating