Merge branch 'master' into topic/bif_cleanup

CHANGES

@@ -1,4 +1,78 @@
 
+2.0-beta-98 | 2011-12-07 08:12:08 -0800
+
+  * Adapting test-suite's diff-all so that it expands globs in both
+    current and baseline directory. Closes #677. (Robin Sommer)
+
+2.0-beta-97 | 2011-12-06 11:49:29 -0800
+
+  * Omit loading local-<node>.bro scripts from base cluster framework.
+    Addresses #663 (Jon Siwek)
+
+2.0-beta-94 | 2011-12-03 15:57:19 -0800
+
+  * Adapting attribute serialization when talking to Broccoli. (Robin
+    Sommer)
+
+2.0-beta-92 | 2011-12-03 15:56:03 -0800
+
+  * Changes to Broxygen master script package index.  (Jon Siwek)
+
+    - Now only lists packages as those directories in the script hierarchy
+      that contain an __load__.bro file.
+
+    - Script packages (dirs with a __load__.bro file), can now include
+      a README (in reST format) that will automatically be appended
+      under the link to a specific package in the master package
+      index.
+
+2.0-beta-88 | 2011-12-02 17:00:58 -0800
+
+  * Teach LogWriterAscii to use BRO_LOG_SUFFIX environemt variable.
+    Addresses #704. (Jon Siwek)
+
+  * Fix double-free of DNS_Mgr_Request object. Addresses #661.
+    
+  * Add a remote_log_peer event which comes with an event_peer record
+    parameter. Addresses #493. (Jon Siwek)
+
+  * Remove example redef of SMTP::entity_excerpt_len from local.bro.
+    Fixes error emitted when loading local.bro in bare mode. (Jon
+    Siwek)
+
+  * Add missing doc targets to top Makefile; remove old doc/Makefile.
+    Fixes #705. (Jon Siwek)
+
+  * Turn some globals into constants. Addresses #633. (Seth Hall)
+    
+  * Rearrange packet filter and DPD documentation. (Jon Siwek)
+    
+2.0-beta-72 | 2011-11-30 20:16:09 -0800
+
+  * Fine-tuning the Sphinx layout to better match www. (Jon Siwek and
+    Robin Sommer)
+
+2.0-beta-69 | 2011-11-29 16:55:31 -0800
+
+  * Fixing ASCII logger to escape the unset-field place holder if
+    written out literally. (Robin Sommer)
+
+2.0-beta-68 | 2011-11-29 15:23:12 -0800
+
+  * Lots of documentation polishing. (Jon Siwek)
+
+  * Teach Broxygen the ".. bro:see::" directive. (Jon Siwek)
+
+  * Teach Broxygen :bro:see: role for referencing any identifier in
+    the Bro domain. (Jon Siwek)
+
+  * Teach Broxygen to generate an index of Bro notices. (Jon Siwek)
+
+  * Fix order of include directories. (Jon Siwek)
+
+  * Catch if logged vectors do not contain only atomic types.
+    (Bernhard Amann)
+
 2.0-beta-47 | 2011-11-16 08:24:33 -0800
 
   * Catch if logged sets do not contain only atomic types. (Bernhard

Makefile

@@ -29,6 +29,18 @@ doc: configured
 docclean: configured
 	$(MAKE) -C $(BUILD) $@
 
+restdoc: configured
+	$(MAKE) -C $(BUILD) $@
+
+restclean: configured
+	$(MAKE) -C $(BUILD) $@
+
+broxygen: configured
+	$(MAKE) -C $(BUILD) $@
+
+broxygenclean: configured
+	$(MAKE) -C $(BUILD) $@
+
 dist:
 	@rm -rf $(VERSION_FULL) $(VERSION_FULL).tgz
 	@rm -rf $(VERSION_MIN) $(VERSION_MIN).tgz

VERSION

@@ -1 +1 @@
-2.0-beta-47
+2.0-beta-98

aux/binpac

@@ -1 +1 @@
-Subproject commit 34d90437403e4129468f89acce0bd1a99813a2f4
+Subproject commit 82bd9613fb869e44f5f0d7929bdd9a88cde84077

aux/bro-aux

@@ -1 +1 @@
-Subproject commit 7ea5837b4ba8403731ca4a9875616c0ab501342f
+Subproject commit 4d387ce660468b44df99d4c87d6016ae4ed2fdc4

aux/broccoli

@@ -1 +1 @@
-Subproject commit d281350dbcc19c24aa6b6d89a4edc08a5c74a790
+Subproject commit 05d51b55dd5a8430151cff74f7876bd73385d5cd

aux/broctl

@@ -1 +1 @@
-Subproject commit ed4d4ce1add51f0e08e6e8d2f5f247c2cbb422da
+Subproject commit 406c689c88997c4459eaf36bd2c65bce2e847c2d

aux/btest

@@ -1 +1 @@
-Subproject commit 7230a09a8c220d2117e491fdf293bf5c19819b65
+Subproject commit 38890e851416fa9fc827a1d36f06c4cb9f7d4e69

doc/CMakeLists.txt

@@ -51,6 +51,8 @@ add_custom_target(broxygen
                   COMMAND "${CMAKE_COMMAND}" -E create_symlink
                           ${DOC_OUTPUT_DIR}/html
                           ${CMAKE_BINARY_DIR}/html
+                  # copy Broccoli API reference into output dir if it exists
+                  COMMAND test -d ${CMAKE_BINARY_DIR}/aux/broccoli/doc/html && ( rm -rf ${CMAKE_BINARY_DIR}/html/broccoli-api && cp -r ${CMAKE_BINARY_DIR}/aux/broccoli/doc/html ${CMAKE_BINARY_DIR}/html/broccoli-api ) || true
                   WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
                   COMMENT "[Sphinx] Generating HTML policy script docs"
                   # SOURCES just adds stuff to IDE projects as a convenience
@@ -58,16 +60,16 @@ add_custom_target(broxygen
 
 # The "sphinxclean" target removes just the Sphinx input/output directories
 # from the build directory.
-add_custom_target(broxygen-clean
+add_custom_target(broxygenclean
                   COMMAND "${CMAKE_COMMAND}" -E remove_directory
                           ${DOC_SOURCE_WORKDIR}
                   COMMAND "${CMAKE_COMMAND}" -E remove_directory
                           ${DOC_OUTPUT_DIR}
                   VERBATIM)
 
-add_dependencies(broxygen broxygen-clean restdoc)
+add_dependencies(broxygen broxygenclean restdoc)
 
 add_custom_target(doc)
 add_custom_target(docclean)
 add_dependencies(doc broxygen)
-add_dependencies(docclean broxygen-clean restclean)
+add_dependencies(docclean broxygenclean restclean)

doc/Makefile

@@ -1,7 +0,0 @@
-
-all:
-	test -d html || mkdir html
-	for i in *.rst; do echo "$$i ..."; ./bin/rst2html.py $$i >html/`echo $$i | sed 's/rst$$/html/g'`; done
-
-clean:
-	rm -rf html

doc/README

@@ -15,8 +15,9 @@ which adds some reST directives and roles that aid in generating useful
 index entries and cross-references.  Other extensions can be added in
 a similar fashion.
 
-Either the ``make doc`` or ``make broxygen`` can be used to locally
+Either the ``make doc`` or ``make broxygen`` targets in the top-level
-render the reST files into HTML.  Those targets depend on:
+Makefile can be used to locally render the reST files into HTML.
+Those targets depend on:
 
 * Python interpreter >= 2.5
 * `Sphinx <http://sphinx.pocoo.org/>`_ >= 1.0.1

doc/_static/broxygen-extra.css

@@ -1,3 +1,15 @@
 .highlight {
     background-color: #ffffff;
 }
+
+h1 {
+    font-weight: bold;
+    font-size: 32px;
+	line-height:32px;
+    text-align: center;
+    padding-top: 3px;
+    margin-bottom: 30px;
+    font-family: Palatino,'Palatino Linotype',Georgia,serif;;
+    color: #000;
+    border-bottom: 0px;
+}

doc/_templates/layout.html

@@ -2,9 +2,97 @@
 
 {% block extrahead %}
 <link rel="stylesheet" type="text/css" href="http://www.bro-ids.org/css/bro-ids.css" />
+<link rel="stylesheet" type="text/css" href="http://www.bro-ids.org/css/960.css" />
 <link rel="stylesheet" type="text/css" href="http://www.bro-ids.org/css/pygments.css" />
 <link rel="stylesheet" type="text/css" href="{{ pathto('_static/broxygen-extra.css', 1) }}"></script>
 <script type="text/javascript" src="{{ pathto('_static/download.js', 1) }}"></script>
 {% endblock %}
 
+{% block header %}
+<iframe src="http://www.bro-ids.org/frames/header-no-logo.html" width="100%" height="100px" frameborder="0" marginheight="0" scrolling="no" marginwidth="0">
+</iframe>
+{% endblock %}
+
 {% block relbar2 %}{% endblock %}
+{% block relbar1 %}{% endblock %}
+
+{% block content %}
+
+        <div id="bro-main" class="clearfix">
+            <div class="container_12">
+
+                <div class="grid_9">
+
+                    <div>
+                         {{ relbar() }}
+                    </div>
+
+                    <div class="body">
+                        {% block body %}
+                        {% endblock %}
+                    </div>
+                </div>
+
+                <!-- Sidebar -->
+                <div class="grid_3 omega">
+
+                    <div>
+                        <img id="logo" src="{{pathto('_static/logo-bro.png', 1)}}" alt="Logo" />
+                    </div>
+                    <br />
+
+
+                    <div class="widget sidebar-toc">
+                        <h3 class="widgettitle">
+                            Table of Contents
+                        </h3>
+                        <p>
+                            <ul>{{toc}}</ul>
+                        </p>
+                    </div>
+
+                    {% if next %}
+                    <div class="widget">
+                        <h3 class="widgettitle">
+                            Next Page
+                        </h3>
+                        <p>
+                            <a href="{{ next.link|e }}">{{ next.title }}</a>
+                        </p>
+                    </div>
+                    {% endif %}
+
+                    {% if prev %}
+                    <div class="widget">
+                        <h3 class="widgettitle">
+                            Previous Page
+                        </h3>
+                        <p>
+                            <a href="{{ prev.link|e }}">{{ prev.title }}</a>
+                        </p>
+                    </div>
+                    {% endif %}
+
+                </div>
+            </div>
+
+            <div class="container_12">
+                <div class="grid_12 alpha omega">
+                    <div class="center">
+                        <small>
+                            Copyright {{ copyright }}.
+                            Last updated on {{ last_updated }}.
+                            Created using <a href="http://sphinx.pocoo.org/">Sphinx</a> {{ sphinx_version }}.
+                        </small>
+                    </div>
+                </div>
+            </div>
+        </div>
+
+
+{% endblock %}
+
+{% block footer %}
+<iframe src="http://www.bro-ids.org/frames/footer.html" width="100%" height="420px" frameborder="0" marginheight="0" scrolling="no" marginwidth="0">
+</iframe>
+{% endblock %}

doc/bin/group_index_generator.py

@@ -49,6 +49,7 @@ with open(group_list, 'r') as f_group_list:
                 if not os.path.exists(os.path.dirname(group_file)):
                     os.makedirs(os.path.dirname(group_file))
                 with open(group_file, 'w') as f_group_file:
+                    f_group_file.write(":orphan:\n\n")
                     title = "Package Index: %s\n" % os.path.dirname(group)
                     f_group_file.write(title);
                     for n in range(len(title)):

doc/bin/rst2html.py

@@ -1,62 +0,0 @@
-#!/usr/bin/env python
-#
-# Derived from docutils standard rst2html.py.
-#
-# $Id: rst2html.py 4564 2006-05-21 20:44:42Z wiemann $
-# Author: David Goodger <goodger@python.org>
-# Copyright: This module has been placed in the public domain.
-#
-#
-# Extension: we add to dummy directorives "code" and "console" to be
-# compatible with Bro's web site setup.
-
-try:
-    import locale
-    locale.setlocale(locale.LC_ALL, '')
-except:
-    pass
-
-import textwrap
-
-from docutils.core import publish_cmdline, default_description
-
-from docutils import nodes
-from docutils.parsers.rst import directives, Directive
-from docutils.parsers.rst.directives.body import LineBlock
-
-class Literal(Directive):
-    #max_line_length = 68
-    max_line_length = 0
-
-    required_arguments = 0
-    optional_arguments = 1
-    final_argument_whitespace = True
-    has_content = True
-
-    def wrapped_content(self):
-        content = []
-
-        if Literal.max_line_length:
-            for line in self.content:
-                content += textwrap.wrap(line, Literal.max_line_length, subsequent_indent="      ")
-        else:
-            content = self.content
-
-        return u'\n'.join(content)
-
-    def run(self):
-        self.assert_has_content()
-        content = self.wrapped_content()
-        literal = nodes.literal_block(content, content)
-        return [literal]
-
-directives.register_directive('code', Literal)
-directives.register_directive('console', Literal)
-
-description = ('Generates (X)HTML documents from standalone reStructuredText '
-               'sources.  ' + default_description)
-
-publish_cmdline(writer_name='html', description=description)
-
-
-

doc/components/binpac/README.rst

@@ -0,0 +1 @@
+../../../aux/binpac/README

doc/components/bro-aux/README.rst

@@ -0,0 +1 @@
+../../../aux/bro-aux/README

doc/components/broccoli-ruby/README.rst

@@ -0,0 +1 @@
+../../../aux/broccoli/bindings/broccoli-ruby/README

doc/components/broccoli/broccoli-manual.rst

@@ -0,0 +1 @@
+../../../aux/broccoli/doc/broccoli-manual.rst

doc/conf.py.in

@@ -90,44 +90,20 @@ pygments_style = 'sphinx'
 
 # The theme to use for HTML and HTML Help pages.  See the documentation for
 # a list of builtin themes.
-html_theme = 'default'
+html_theme = 'basic'
 
 html_last_updated_fmt = '%B %d, %Y'
 
 # Theme options are theme-specific and customize the look and feel of a theme
 # further.  For a list of options available for each theme, see the
 # documentation.
-html_theme_options = {
+html_theme_options = { }
-"rightsidebar": "true",
-"stickysidebar": "true",
-"externalrefs": "false",
-"footerbgcolor": "#333",
-"footertextcolor": "#ddd",
-"sidebarbgcolor": "#ffffff",
-#"sidebarbtncolor": "",
-"sidebartextcolor": "#333",
-"sidebarlinkcolor": "#2a85a7",
-"relbarbgcolor": "#ffffff",
-"relbartextcolor": "#333",
-"relbarlinkcolor": "#2a85a7",
-"bgcolor": "#ffffff",
-"textcolor": "#333",
-"linkcolor": "#2a85a7",
-"visitedlinkcolor": "#2a85a7",
-"headbgcolor": "#f0f0f0",
-"headtextcolor": "#000",
-"headlinkcolor": "#2a85a7",
-"codebgcolor": "#FFFAE2",
-#"codetextcolor": "",
-"bodyfont": "Arial, Helvetica, sans-serif",
-"headfont": "Palatino,'Palatino Linotype',Georgia,serif",
-}
 
 # Add any paths that contain custom themes here, relative to this directory.
 #html_theme_path = []
 
 # The name for this set of Sphinx documents.  If None, it defaults to
-# "<project> v<release> documentation".
+# "<project> v<release> Documentation".
 #html_title = None
 
 # A shorter title for the navigation bar.  Default is the same as html_title.

doc/index.rst

@@ -1,8 +1,12 @@
 .. Bro documentation master file
 
+=================
 Bro Documentation
 =================
 
+Guides
+------
+
 .. toctree::
    :maxdepth: 1
 
@@ -37,7 +41,6 @@ Script Reference
 .. toctree::
    :maxdepth: 1
 
-   scripts/common
    scripts/builtins
    scripts/bifs
    scripts/packages
@@ -46,16 +49,29 @@ Script Reference
 Other Bro Components
 --------------------
 
+The following are snapshots of documentation for components that come
+with this version of Bro (|version|). Since they can also be used
+independently, see the `download page
+<http://bro-ids.org/download/index.html>`_ for documentation of any
+current, independent component releases.
+
 .. toctree::
    :maxdepth: 1
 
-   components/btest/README
+   BinPAC - A protocol parser generator <components/binpac/README>
-   components/broccoli/README
+   Broccoli - The Bro Client Communication Library (README) <components/broccoli/README>
-   components/broccoli-python/README
+   Broccoli - User Manual <components/broccoli/broccoli-manual>
-   components/broctl/README
+   Broccoli Python Bindings <components/broccoli-python/README>
-   components/capstats/README
+   Broccoli Ruby Bindings <components/broccoli-ruby/README>
-   components/pysubnettree/README
+   BroControl - Interactive Bro management shell <components/broctl/README>
-   components/trace-summary/README
+   Bro-Aux - Small auxiliary tools for Bro <components/bro-aux/README>
+   BTest - A unit testing framework <components/btest/README>
+   Capstats - Command-line packet statistic tool <components/capstats/README>
+   PySubnetTree - Python module for CIDR lookups<components/pysubnettree/README>
+   trace-summary - Script for generating break-downs of network traffic <components/trace-summary/README>
+
+The `Broccoli API Reference <broccoli-api/index.html>`_ may also be of
+interest.
 
 Other Indices and References
 ----------------------------

doc/notice.rst

@@ -29,17 +29,18 @@ definitions of what constitutes an attack or even a compromise differ quite a
 bit between environments, and activity deemed malicious at one site might be
 fully acceptable at another.
 
-Whenever one of Bro's analysis scripts sees something potentially interesting
+Whenever one of Bro's analysis scripts sees something potentially
-it flags the situation by calling the ``NOTICE`` function and giving it a
+interesting it flags the situation by calling the :bro:see:`NOTICE`
-single ``Notice::Info`` record. A Notice has a ``Notice::Type``, which
+function and giving it a single :bro:see:`Notice::Info` record. A Notice
-reflects the kind of activity that has been seen, and it is usually also
+has a :bro:see:`Notice::Type`, which reflects the kind of activity that
-augmented with further context about the situation. 
+has been seen, and it is usually also augmented with further context
+about the situation.
 
 More information about raising notices can be found in the `Raising Notices`_
 section.
 
 Once a notice is raised, it can have any number of actions applied to it by
-the ``Notice::policy`` set which is described in the `Notice Policy`_
+the :bro:see:`Notice::policy` set which is described in the `Notice Policy`_
 section below. Such actions can be to send a mail to the configured
 address(es) or to simply ignore the notice. Currently, the following actions
 are defined:
@@ -52,20 +53,20 @@ are defined:
       - Description
 
     * - Notice::ACTION_LOG
-      - Write the notice to the ``Notice::LOG`` logging stream.
+      - Write the notice to the :bro:see:`Notice::LOG` logging stream.
 
     * - Notice::ACTION_ALARM
-      - Log into the ``Notice::ALARM_LOG`` stream which will rotate
+      - Log into the :bro:see:`Notice::ALARM_LOG` stream which will rotate
         hourly and email the contents to the email address or addresses
-        defined in the ``Notice::mail_dest`` variable.
+        defined in the :bro:see:`Notice::mail_dest` variable.
 
     * - Notice::ACTION_EMAIL
       - Send the notice in an email to the email address or addresses given in
-        the ``Notice::mail_dest`` variable.
+        the :bro:see:`Notice::mail_dest` variable.
 
     * - Notice::ACTION_PAGE
       - Send an email to the email address or addresses given in the
-        ``Notice::mail_page_dest`` variable.
+        :bro:see:`Notice::mail_page_dest` variable.
 
     * - Notice::ACTION_NO_SUPPRESS
       - This action will disable the built in notice suppression for the
@@ -82,15 +83,17 @@ Processing Notices
 Notice Policy
 *************
 
-The predefined set ``Notice::policy`` provides the mechanism for applying
+The predefined set :bro:see:`Notice::policy` provides the mechanism for
-actions and other behavior modifications to notices. Each entry of
+applying actions and other behavior modifications to notices. Each entry
-``Notice::policy`` is a record of the type ``Notice::PolicyItem`` which
+of :bro:see:`Notice::policy` is a record of the type
-defines a condition to be matched against all raised notices and one or more
+:bro:see:`Notice::PolicyItem` which defines a condition to be matched
-of a variety of behavior modifiers. The notice policy is defined by adding any
+against all raised notices and one or more of a variety of behavior
-number of ``Notice::PolicyItem`` records to the ``Notice::policy`` set.
+modifiers. The notice policy is defined by adding any number of
+:bro:see:`Notice::PolicyItem` records to the :bro:see:`Notice::policy`
+set.
 
 Here's a simple example which tells Bro to send an email for all notices of
-type ``SSH::Login`` if the server is 10.0.0.1:
+type :bro:see:`SSH::Login` if the server is 10.0.0.1:
 
 .. code:: bro
 
@@ -113,11 +116,11 @@ flexibility due to having access to Bro's full programming language.
 Predicate Field
 ^^^^^^^^^^^^^^^
 
-The ``Notice::PolicyItem`` record type has a field name ``$pred`` which
+The :bro:see:`Notice::PolicyItem` record type has a field name ``$pred``
-defines the entry's condition in the form of a predicate written as a Bro
+which defines the entry's condition in the form of a predicate written
-function. The function is passed the notice as a ``Notice::Info`` record and
+as a Bro function. The function is passed the notice as a
-it returns a boolean value indicating if the entry is applicable to that
+:bro:see:`Notice::Info` record and it returns a boolean value indicating
-particular notice.
+if the entry is applicable to that particular notice.
 
 .. note::
 
@@ -125,14 +128,14 @@ particular notice.
     (``T``) since an implicit false (``F``) value would never be used.
 
 Bro evaluates the predicates of each entry in the order defined by the
-``$priority`` field in ``Notice::PolicyItem`` records. The valid values are
+``$priority`` field in :bro:see:`Notice::PolicyItem` records. The valid
-0-10 with 10 being earliest evaluated. If ``$priority`` is omitted, the
+values are 0-10 with 10 being earliest evaluated. If ``$priority`` is
-default priority is 5.
+omitted, the default priority is 5.
 
 Behavior Modification Fields
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-There are a set of fields in the ``Notice::PolicyItem`` record type that
+There are a set of fields in the :bro:see:`Notice::PolicyItem` record type that
 indicate ways that either the notice or notice processing should be modified
 if the predicate field (``$pred``) evaluated to true (``T``). Those fields are
 explained in more detail in the following table.
@@ -146,8 +149,8 @@ explained in more detail in the following table.
       - Example
 
     * - ``$action=<Notice::Action>``
-      - Each Notice::PolicyItem can have a single action applied to the notice
+      - Each :bro:see:`Notice::PolicyItem` can have a single action
-        with this field.
+        applied to the notice with this field.
       - ``$action = Notice::ACTION_EMAIL``
 
     * - ``$suppress_for=<interval>`` 
@@ -162,9 +165,9 @@ explained in more detail in the following table.
       - This field can be used for modification of the notice policy
         evaluation. To stop processing of notice policy items before
         evaluating all of them, set this field to ``T`` and make the ``$pred``
-        field return ``T``. ``Notice::PolicyItem`` records defined at a higher
+        field return ``T``. :bro:see:`Notice::PolicyItem` records defined at
-        priority as defined by the ``$priority`` field will still be evaluated
+        a higher priority as defined by the ``$priority`` field will still be
-        but those at a lower priority won't.
+        evaluated but those at a lower priority won't.
       - ``$halt = T``
 
 
@@ -186,11 +189,11 @@ Notice Policy Shortcuts
 Although the notice framework provides a great deal of flexibility and
 configurability there are many times that the full expressiveness isn't needed
 and actually becomes a hindrance to achieving results. The framework provides
-a default ``Notice::policy`` suite as a way of giving users the
+a default :bro:see:`Notice::policy` suite as a way of giving users the
 shortcuts to easily apply many common actions to notices.
 
 These are implemented as sets and tables indexed with a
-``Notice::Type`` enum value. The following table shows and describes
+:bro:see:`Notice::Type` enum value. The following table shows and describes
 all of the variables available for shortcut configuration of the notice
 framework.
 
@@ -201,40 +204,44 @@ framework.
     * - Variable name
       - Description
 
-    * - Notice::ignored_types
+    * - :bro:see:`Notice::ignored_types`
-      - Adding a ``Notice::Type`` to this set results in the notice
+      - Adding a :bro:see:`Notice::Type` to this set results in the notice
         being ignored. It won't have any other action applied to it, not even
-        ``Notice::ACTION_LOG``.
+        :bro:see:`Notice::ACTION_LOG`.
 
-    * - Notice::emailed_types
+    * - :bro:see:`Notice::emailed_types`
-      - Adding a ``Notice::Type`` to this set results in
+      - Adding a :bro:see:`Notice::Type` to this set results in
-        ``Notice::ACTION_EMAIL`` being applied to the notices of that type.
+        :bro:see:`Notice::ACTION_EMAIL` being applied to the notices of
+        that type.
 
-    * - Notice::alarmed_types
+    * - :bro:see:`Notice::alarmed_types`
-      - Adding a Notice::Type to this set results in
+      - Adding a :bro:see:`Notice::Type` to this set results in
-        ``Notice::ACTION_ALARM`` being applied to the notices of that type.
+        :bro:see:`Notice::ACTION_ALARM` being applied to the notices of
+        that type.
 
-    * - Notice::not_suppressed_types
+    * - :bro:see:`Notice::not_suppressed_types`
-      - Adding a ``Notice::Type`` to this set results in that notice no longer
+      - Adding a :bro:see:`Notice::Type` to this set results in that notice
-        undergoing the normal notice suppression that would take place. Be
+        no longer undergoes the normal notice suppression that would
-        careful when using this in production it could result in a dramatic
+        take place. Be careful when using this in production it could
-        increase in the number of notices being processed.
+        result in a dramatic increase in the number of notices being
+        processed.
 
-    * - Notice::type_suppression_intervals
+    * - :bro:see:`Notice::type_suppression_intervals`
-      - This is a table indexed on ``Notice::Type`` and yielding an interval.
+      - This is a table indexed on :bro:see:`Notice::Type` and yielding an
-        It can be used as an easy way to extend the default suppression
+        interval.  It can be used as an easy way to extend the default
-        interval for an entire ``Notice::Type`` without having to create a
+        suppression interval for an entire :bro:see:`Notice::Type`
-        whole ``Notice::policy`` entry and setting the ``$suppress_for``
+        without having to create a whole :bro:see:`Notice::policy` entry
-        field.
+        and setting the ``$suppress_for`` field.
 
 Raising Notices
 ---------------
 
-A script should raise a notice for any occurrence that a user may want to be
+A script should raise a notice for any occurrence that a user may want
-notified about or take action on. For example, whenever the base SSH analysis
+to be notified about or take action on. For example, whenever the base
-scripts sees an SSH session where it is heuristically guessed to be a
+SSH analysis scripts sees an SSH session where it is heuristically
-successful login, it raises a Notice of the type ``SSH::Login``. The code in
+guessed to be a successful login, it raises a Notice of the type
-the base SSH analysis script looks like this:
+:bro:see:`SSH::Login`. The code in the base SSH analysis script looks
+like this:
 
 .. code:: bro
 
@@ -242,10 +249,10 @@ the base SSH analysis script looks like this:
             $msg="Heuristically detected successful SSH login.",
             $conn=c]);
 
-``NOTICE`` is a normal function in the global namespace which wraps a function
+:bro:see:`NOTICE` is a normal function in the global namespace which
-within the ``Notice`` namespace. It takes a single argument of the
+wraps a function within the ``Notice`` namespace. It takes a single
-``Notice::Info`` record type. The most common fields used when raising notices
+argument of the :bro:see:`Notice::Info` record type. The most common
-are described in the following table:
+fields used when raising notices are described in the following table:
 
 .. list-table::
     :widths: 32 40
@@ -295,9 +302,10 @@ are described in the following table:
 
     * - ``$suppress_for``
       - This field can be set if there is a natural suppression interval for
-        the notice that may be different than the default value. The value set
+        the notice that may be different than the default value. The
-        to this field can also be modified by a user's ``Notice::policy`` so
+        value set to this field can also be modified by a user's
-        the value is not set permanently and unchangeably.
+        :bro:see:`Notice::policy` so the value is not set permanently
+        and unchangeably.
 
 When writing Bro scripts which raise notices, some thought should be given to
 what the notice represents and what data should be provided to give a consumer
@@ -325,7 +333,7 @@ The notice framework supports suppression for notices if the author of the
 script that is generating the notice has indicated to the notice framework how
 to identify notices that are intrinsically the same. Identification of these
 "intrinsically duplicate" notices is implemented with an optional field in
-``Notice::Info`` records named ``$identifier`` which is a simple string.
+:bro:see:`Notice::Info` records named ``$identifier`` which is a simple string.
 If the ``$identifier`` and ``$type`` fields are the same for two notices, the
 notice framework actually considers them to be the same thing and can use that
 information to suppress duplicates for a configurable period of time.
@@ -337,12 +345,13 @@ information to suppress duplicates for a configurable period of time.
     could be completely legitimate usage if no notices could ever be
     considered to be duplicates.
 
-The ``$identifier`` field is typically comprised of several pieces of data
+The ``$identifier`` field is typically comprised of several pieces of
-related to the notice that when combined represent a unique instance of that
+data related to the notice that when combined represent a unique
-notice. Here is an example of the script
+instance of that notice. Here is an example of the script
-``policy/protocols/ssl/validate-certs.bro`` raising a notice for session
+:doc:`scripts/policy/protocols/ssl/validate-certs` raising a notice
-negotiations where the certificate or certificate chain did not validate
+for session negotiations where the certificate or certificate chain did
-successfully against the available certificate authority certificates.
+not validate successfully against the available certificate authority
+certificates.
 
 .. code:: bro
 
@@ -369,7 +378,7 @@ it's assumed that the script author who is raising the notice understands the
 full problem set and edge cases of the notice which may not be readily
 apparent to users. If users don't want the suppression to take place or simply
 want a different interval, they can always modify it with the
-``Notice::policy``.
+:bro:see:`Notice::policy`.
 
 
 Extending Notice Framework

doc/scripts/CMakeLists.txt

@@ -73,12 +73,14 @@ macro(REST_TARGET srcDir broInput)
     elseif (${extension} STREQUAL ".bif.bro")
         set(group bifs)
     elseif (relDstDir)
-        set(pkgIndex ${relDstDir}/index)
+        set(group ${relDstDir}/index)
-        set(group ${pkgIndex})
         # add package index to master package list if not already in it
-        list(FIND MASTER_PKG_LIST ${pkgIndex} _found)
+        # and if a __load__.bro exists in the original script directory
+        list(FIND MASTER_PKG_LIST ${relDstDir} _found)
         if (_found EQUAL -1)
-            list(APPEND MASTER_PKG_LIST ${pkgIndex})
+            if (EXISTS ${CMAKE_SOURCE_DIR}/scripts/${relDstDir}/__load__.bro)
+                list(APPEND MASTER_PKG_LIST ${relDstDir})
+            endif ()
         endif ()
     else ()
         set(group "")
@@ -137,11 +139,15 @@ file(WRITE ${MASTER_POLICY_INDEX} "${MASTER_POLICY_INDEX_TEXT}")
 # policy/packages.rst file
 set(MASTER_PKG_INDEX_TEXT "")
 foreach (pkg ${MASTER_PKG_LIST})
-    # strip of the trailing /index for the link name
+    set(MASTER_PKG_INDEX_TEXT
-    get_filename_component(lnktxt ${pkg} PATH)
+        "${MASTER_PKG_INDEX_TEXT}\n:doc:`${pkg} <${pkg}/index>`\n")
-    # pretty-up the link name by removing common scripts/ prefix
+    if (EXISTS ${CMAKE_SOURCE_DIR}/scripts/${pkg}/README)
-    string(REPLACE "scripts/" "" lnktxt "${lnktxt}")
+        file(STRINGS ${CMAKE_SOURCE_DIR}/scripts/${pkg}/README pkgreadme)
-    set(MASTER_PKG_INDEX_TEXT "${MASTER_PKG_INDEX_TEXT}\n   ${lnktxt} <${pkg}>")
+        foreach (line ${pkgreadme})
+            set(MASTER_PKG_INDEX_TEXT "${MASTER_PKG_INDEX_TEXT}\n    ${line}")
+        endforeach ()
+        set(MASTER_PKG_INDEX_TEXT "${MASTER_PKG_INDEX_TEXT}\n")
+    endif ()
 endforeach ()
 file(WRITE ${MASTER_PACKAGE_INDEX} "${MASTER_PKG_INDEX_TEXT}")
 

doc/scripts/README

@@ -1,6 +1,6 @@
 This directory contains scripts and templates that can be used to automate
 the generation of Bro script documentation.  Several build targets are defined
-by CMake:
+by CMake and available in the top-level Makefile:
 
 ``restdoc``
 

doc/scripts/common.rst

@@ -1,19 +0,0 @@
-Common Documentation
-====================
-
-.. _common_port_analysis_doc:
-
-Port Analysis
--------------
-
-TODO: add some stuff here
-
-.. _common_packet_filter_doc:
-
-Packet Filter
--------------
-
-TODO: add some stuff here
-
-.. note:: Filters are only relevant when dynamic protocol detection (DPD)
-          is explicitly turned off (Bro release 1.6 enabled DPD by default).

doc/scripts/index.rst

@@ -1,7 +1,7 @@
 .. This is a stub doc to which broxygen appends during the build process
 
-Index of All Bro Scripts
+Index of All Individual Bro Scripts
-========================
+===================================
 
 .. toctree::
    :maxdepth: 1

doc/scripts/packages.rst

@@ -10,8 +10,3 @@ script, it supports being loaded in mass as a whole directory for convenience.
 Packages/scripts in the ``base/`` directory are all loaded by default, while
 ones in ``policy/`` provide functionality and customization options that are
 more appropriate for users to decide whether they'd like to load it or not.
-
-.. toctree::
-   :maxdepth: 1
-
-

scripts/base/frameworks/cluster/__load__.bro

@@ -28,17 +28,14 @@ redef Communication::listen_port = Cluster::nodes[Cluster::node]$p;
 
 @if ( Cluster::local_node_type() == Cluster::MANAGER )
 @load ./nodes/manager
-@load site/local-manager
 @endif
 
 @if ( Cluster::local_node_type() == Cluster::PROXY )
 @load ./nodes/proxy
-@load site/local-proxy
 @endif
 
 @if ( Cluster::local_node_type() == Cluster::WORKER )
 @load ./nodes/worker
-@load site/local-worker
 @endif
 
 @endif

scripts/base/frameworks/communication/main.bro

@@ -130,6 +130,13 @@ event remote_log(level: count, src: count, msg: string)
 	do_script_log_common(level, src, msg);
 	}
 
+# This is a core generated event.
+event remote_log_peer(p: event_peer, level: count, src: count, msg: string)
+	{
+	local rmsg = fmt("[#%d/%s:%d] %s", p$id, p$host, p$p, msg);
+	do_script_log_common(level, src, rmsg);
+	}
+
 function do_script_log(p: event_peer, msg: string)
 	{
 	do_script_log_common(REMOTE_LOG_INFO, REMOTE_SRC_SCRIPT, msg);

scripts/base/protocols/dns/main.bro

@@ -65,11 +65,11 @@ redef capture_filters += {
 	["netbios-ns"] = "udp port 137", 
 };
 
-global dns_ports = { 53/udp, 53/tcp, 137/udp, 5353/udp, 5355/udp } &redef;
+const dns_ports = { 53/udp, 53/tcp, 137/udp, 5353/udp, 5355/udp };
 redef dpd_config += { [ANALYZER_DNS] = [$ports = dns_ports] };
 
-global dns_udp_ports = { 53/udp, 137/udp, 5353/udp, 5355/udp } &redef;
+const dns_udp_ports = { 53/udp, 137/udp, 5353/udp, 5355/udp };
-global dns_tcp_ports = { 53/tcp } &redef;
+const dns_tcp_ports = { 53/tcp };
 redef dpd_config += { [ANALYZER_DNS_UDP_BINPAC] = [$ports = dns_udp_ports] };
 redef dpd_config += { [ANALYZER_DNS_TCP_BINPAC] = [$ports = dns_tcp_ports] };
 

scripts/base/protocols/irc/main.bro

@@ -41,7 +41,7 @@ redef capture_filters += { ["irc-6668"] = "port 6668" };
 redef capture_filters += { ["irc-6669"] = "port 6669" };
 
 # DPD configuration.
-global irc_ports = { 6666/tcp, 6667/tcp, 6668/tcp, 6669/tcp } &redef;
+const irc_ports = { 6666/tcp, 6667/tcp, 6668/tcp, 6669/tcp };
 redef dpd_config += { [ANALYZER_IRC] = [$ports = irc_ports] };
 
 redef likely_server_ports += { 6666/tcp, 6667/tcp, 6668/tcp, 6669/tcp };

scripts/site/local.bro

@@ -62,11 +62,3 @@ redef signature_files += "frameworks/signatures/detect-windows-shells.sig";
 @load protocols/http/detect-MHR
 # Detect SQL injection attacks
 @load protocols/http/detect-sqli
-
-# Uncomment this redef if you want to extract SMTP MIME entities for 
-# some file types.  The numbers given indicate how many bytes to extract for
-# the various mime types.
-redef SMTP::entity_excerpt_len += {
-#	["text/plain"] = 1024,
-#	["text/html"] = 1024,
-};

src/Attr.cc

@@ -481,7 +481,11 @@ bool Attributes::DoSerialize(SerialInfo* info) const
 	loop_over_list((*attrs), i)
 		{
 		Attr* a = (*attrs)[i];
-		SERIALIZE_OPTIONAL(a->AttrExpr())
+
+		// Broccoli doesn't support expressions.
+		Expr* e = (! info->broccoli_peer) ? a->AttrExpr() : 0;
+		SERIALIZE_OPTIONAL(e);
+
 		if ( ! SERIALIZE(char(a->Tag())) )
 			return false;
 		}

src/BroDoc.cc

@@ -215,14 +215,16 @@ void BroDoc::WriteDocFile() const
 	if ( ! port_analysis.empty() )
 		{
 		WriteSectionHeading("Port Analysis", '-');
-		WriteToDoc(":ref:`More Information <common_port_analysis_doc>`\n\n");
+		WriteToDoc("Loading this script makes the following changes to "
+		           ":bro:see:`dpd_config`.\n\n");
 		WriteStringList("%s", port_analysis);
 		}
 
 	if ( ! packet_filter.empty() )
 		{
 		WriteSectionHeading("Packet Filter", '-');
-		WriteToDoc(":ref:`More Information <common_packet_filter_doc>`\n\n");
+		WriteToDoc("Loading this script makes the following changes to "
+		           ":bro:see:`capture_filters`.\n\n");
 		WriteToDoc("Filters added::\n\n");
 		WriteToDoc("%s\n", packet_filter.c_str());
 		}

src/CMakeLists.txt

@@ -1,4 +1,5 @@
-include_directories(${CMAKE_CURRENT_SOURCE_DIR}
+include_directories(BEFORE
+                    ${CMAKE_CURRENT_SOURCE_DIR}
                     ${CMAKE_CURRENT_BINARY_DIR}
 )
 

src/DNS_Mgr.cc

@@ -595,8 +595,6 @@ void DNS_Mgr::Resolve()
 				}
 			else
 				--num_pending;
-
-			delete dr;
 			}
 		}
 

src/LogMgr.cc

@@ -157,7 +157,7 @@ bool LogVal::IsCompatibleType(BroType* t, bool atomic_only)
 		if ( atomic_only )
 			return false;
 
-		return IsCompatibleType(t->AsVectorType()->YieldType());
+		return IsCompatibleType(t->AsVectorType()->YieldType(), true);
 		}
 
 	default:

src/LogWriterAscii.cc

@@ -88,7 +88,7 @@ bool LogWriterAscii::DoInit(string path, int num_fields,
 	if ( output_to_stdout )
 		path = "/dev/stdout";
 
-	fname = IsSpecial(path) ? path : path + ".log";
+	fname = IsSpecial(path) ? path : path + "." + LogExt();
 
 	if ( ! (file = fopen(fname.c_str(), "w")) )
 		{
@@ -200,10 +200,33 @@ bool LogWriterAscii::DoWriteOne(ODesc* desc, LogVal* val, const LogField* field)
 	case TYPE_FUNC:
 		{
 		int size = val->val.string_val->size();
-		if ( size )
+		const char* data = val->val.string_val->data();
-			desc->AddN(val->val.string_val->data(), val->val.string_val->size());
+
-		else
+		if ( ! size )
+			{
 			desc->AddN(empty_field, empty_field_len);
+			break;
+			}
+
+		if ( size == unset_field_len && memcmp(data, unset_field, size) == 0 )
+			{
+			// The value we'd write out would match exactly the
+			// place-holder we use for unset optional fields. We
+			// escape the first character so that the output
+			// won't be ambigious.
+			static const char hex_chars[] = "0123456789abcdef";
+			char hex[6] = "\\x00";
+			hex[2] = hex_chars[((*data) & 0xf0) >> 4];
+			hex[3] = hex_chars[(*data) & 0x0f];
+			desc->AddRaw(hex, 4);
+
+			++data;
+			--size;
+			}
+
+		if ( size )
+			desc->AddN(data, size);
+
 		break;
 		}
 
@@ -297,7 +320,7 @@ bool LogWriterAscii::DoRotate(string rotated_path, double open,
 	fclose(file);
 	file = 0;
 
-	string nname = rotated_path + ".log";
+	string nname = rotated_path + "." + LogExt();
 	rename(fname.c_str(), nname.c_str());
 
 	if ( ! FinishedRotation(nname, fname, open, close, terminating) )
@@ -315,4 +338,9 @@ bool LogWriterAscii::DoSetBuf(bool enabled)
 	return true;
 	}
 
-
+string LogWriterAscii::LogExt()
+	{
+	const char* ext = getenv("BRO_LOG_SUFFIX");
+	if ( ! ext ) ext = "log";
+	return ext;
+	}

src/LogWriterAscii.h

@@ -13,6 +13,7 @@ public:
 	~LogWriterAscii();
 
 	static LogWriter* Instantiate()	{ return new LogWriterAscii; }
+	static string LogExt();
 
 protected:
 	virtual bool DoInit(string path, int num_fields,

src/RemoteSerializer.cc

@@ -385,6 +385,9 @@ inline void RemoteSerializer::SetupSerialInfo(SerialInfo* info, Peer* peer)
 	     peer->phase == Peer::RUNNING )
 		info->new_cache_strategy = true;
 
+	if ( (peer->caps & Peer::BROCCOLI_PEER) )
+		info->broccoli_peer = true;
+
 	info->include_locations = false;
 	}
 
@@ -1457,7 +1460,7 @@ void RemoteSerializer::Finish()
 		Poll(true);
 	while ( io->CanWrite() );
 
-	loop_over_list(peers, i) 
+	loop_over_list(peers, i)
 		{
 		CloseConnection(peers[i]);
 		}
@@ -2113,6 +2116,9 @@ bool RemoteSerializer::HandshakeDone(Peer* peer)
 	if ( (peer->caps & Peer::NEW_CACHE_STRATEGY) )
 		Log(LogInfo, "peer supports keep-in-cache; using that", peer);
 
+	if ( (peer->caps & Peer::BROCCOLI_PEER) )
+		Log(LogInfo, "peer is a Broccoli", peer);
+
 	if ( peer->logs_requested )
 		log_mgr->SendAllWritersTo(peer->id);
 
@@ -2365,6 +2371,9 @@ bool RemoteSerializer::ProcessSerialization()
 	     current_peer->phase == Peer::RUNNING )
 		info.new_cache_strategy = true;
 
+	if ( current_peer->caps & Peer::BROCCOLI_PEER )
+		info.broccoli_peer = true;
+
 	if ( ! forward_remote_state_changes )
 		ignore_accesses = true;
 
@@ -2923,25 +2932,37 @@ void RemoteSerializer::Log(LogLevel level, const char* msg)
 void RemoteSerializer::Log(LogLevel level, const char* msg, Peer* peer,
 				LogSrc src)
 	{
+	if ( peer )
+		{
+		val_list* vl = new val_list();
+		vl->append(peer->val->Ref());
+		vl->append(new Val(level, TYPE_COUNT));
+		vl->append(new Val(src, TYPE_COUNT));
+		vl->append(new StringVal(msg));
+		mgr.QueueEvent(remote_log_peer, vl);
+		}
+	else
+		{
+		val_list* vl = new val_list();
+		vl->append(new Val(level, TYPE_COUNT));
+		vl->append(new Val(src, TYPE_COUNT));
+		vl->append(new StringVal(msg));
+		mgr.QueueEvent(remote_log, vl);
+		}
+
+#ifdef DEBUG
 	const int BUFSIZE = 1024;
 	char buffer[BUFSIZE];
-
 	int len = 0;
 
 	if ( peer )
-		len += snprintf(buffer + len, sizeof(buffer) - len,
+		len += snprintf(buffer + len, sizeof(buffer) - len, "[#%d/%s:%d] ",
-				"[#%d/%s:%d] ", int(peer->id), ip2a(peer->ip),
+		                int(peer->id), ip2a(peer->ip), peer->port);
-				peer->port);
 
 	len += safe_snprintf(buffer + len, sizeof(buffer) - len, "%s", msg);
 
-	val_list* vl = new val_list();
-	vl->append(new Val(level, TYPE_COUNT));
-	vl->append(new Val(src, TYPE_COUNT));
-	vl->append(new StringVal(buffer));
-	mgr.QueueEvent(remote_log, vl);
-
 	DEBUG_COMM(fmt("parent: %.6f %s", current_time(), buffer));
+#endif
 	}
 
 void RemoteSerializer::RaiseEvent(EventHandlerPtr event, Peer* peer,

src/RemoteSerializer.h

@@ -198,6 +198,7 @@ protected:
 		static const int NO_CACHING = 2;
 		static const int PID_64BIT = 4;
 		static const int NEW_CACHE_STRATEGY = 8;
+		static const int BROCCOLI_PEER = 16;
 
 		// Constants to remember to who did something.
 		static const int NONE = 0;

src/SerialInfo.h

@@ -15,6 +15,7 @@ public:
 		pid_32bit = false;
 		include_locations = true;
 		new_cache_strategy = false;
+		broccoli_peer = false;
 		}
 
 	SerialInfo(const SerialInfo& info)
@@ -28,6 +29,7 @@ public:
 		pid_32bit = info.pid_32bit;
 		include_locations = info.include_locations;
 		new_cache_strategy = info.new_cache_strategy;
+		broccoli_peer = info.broccoli_peer;
 		}
 
 	// Parameters that control serialization.
@@ -46,6 +48,11 @@ public:
 	// If true, we support keeping objs in cache permanently.
 	bool new_cache_strategy;
 
+	// If true, we're connecting to a Broccoli. If so, serialization
+	// specifics may be adapted for functionality Broccoli does not
+	// support.
+	bool broccoli_peer;
+
 	ChunkedIO::Chunk* chunk; // chunk written right before the serialization
 
 	// Attributes set during serialization.
@@ -70,6 +77,7 @@ public:
 		print = 0;
 		pid_32bit = false;
 		new_cache_strategy = false;
+		broccoli_peer = false;
 		}
 
 	UnserialInfo(const UnserialInfo& info)
@@ -86,6 +94,7 @@ public:
 		print = info.print;
 		pid_32bit = info.pid_32bit;
 		new_cache_strategy = info.new_cache_strategy;
+		broccoli_peer = info.broccoli_peer;
 		}
 
 	// Parameters that control unserialization.
@@ -106,6 +115,11 @@ public:
 	// If true, we support keeping objs in cache permanently.
 	bool new_cache_strategy;
 
+	// If true, we're connecting to a Broccoli. If so, serialization
+	// specifics may be adapted for functionality Broccoli does not
+	// support.
+	bool broccoli_peer;
+
 	// If a global ID already exits, of these policies is used.
 	enum {
 		Keep,	// keep the old ID and ignore the new

src/event.bif

@@ -444,6 +444,29 @@ event remote_state_inconsistency%(operation: string, id: string,
 # Generated for communication log message.
 event remote_log%(level: count, src: count, msg: string%);
 
+## Generated for communication log messages. While this event is
+## intended primarily for use by Bro's communication framework, it can also trigger
+## additional code if helpful.  This event is equivalent to
+## :bro:see:`remote_log` except the message is with respect to a certain peer.
+##
+## p: A record describing the remote peer.
+##
+## level: The log level, which is either :bro:enum:`REMOTE_LOG_INFO` or
+##        :bro:enum:`REMOTE_LOG_ERROR`.
+##
+## src: The component of the comminication system that logged the message.
+##      Currently, this will be one of :bro:enum:`REMOTE_SRC_CHILD` (Bro's
+##      child process), :bro:enum:`REMOTE_SRC_PARENT` (Bro's main process), or
+##      :bro:enum:`REMOTE_SRC_SCRIPT` (the script level).
+##
+## msg: The message logged.
+##
+## .. bro:see:: remote_capture_filter remote_connection_closed remote_connection_error
+##    remote_connection_established remote_connection_handshake_done
+##    remote_event_registered  remote_pong remote_state_access_performed
+##    remote_state_inconsistency print_hook remote_log
+event remote_log_peer%(p: event_peer, level: count, src: count, msg: string%);
+
 # Generated when a remote peer has answered to our ping.
 event remote_pong%(p: event_peer, seq: count,
 			d1: interval, d2: interval, d3: interval%);

src/main.cc

@@ -47,6 +47,7 @@ extern "C" void OPENSSL_add_all_algorithms_conf(void);
 #include "ConnCompressor.h"
 #include "DPM.h"
 #include "BroDoc.h"
+#include "LogWriterAscii.h"
 
 #include "binpac_bro.h"
 
@@ -194,6 +195,7 @@ void usage()
 	fprintf(stderr, "    $BRO_PREFIXES                  | prefix list (%s)\n", bro_prefixes());
 	fprintf(stderr, "    $BRO_DNS_FAKE                  | disable DNS lookups (%s)\n", bro_dns_fake());
 	fprintf(stderr, "    $BRO_SEED_FILE                 | file to load seeds from (not set)\n");
+	fprintf(stderr, "    $BRO_LOG_SUFFIX                | ASCII log file extension (.%s)\n", LogWriterAscii::LogExt().c_str());
 
 	exit(1);
 	}

testing/btest/Baseline/doc.autogen-reST-example/example.rst

@@ -281,7 +281,7 @@ Redefinitions
 
 Port Analysis
 -------------
-:ref:`More Information <common_port_analysis_doc>`
+Loading this script makes the following changes to :bro:see:`dpd_config`.
 
 SSL::
 
@@ -292,7 +292,7 @@ SSL::
 
 Packet Filter
 -------------
-:ref:`More Information <common_packet_filter_doc>`
+Loading this script makes the following changes to :bro:see:`capture_filters`.
 
 Filters added::
 

testing/btest/Baseline/scripts.base.frameworks.communication.communication_log_baseline/send.log

@@ -0,0 +1,16 @@
+#separator \x09
+#path	communication
+#fields	ts	peer	src_name	connected_peer_desc	connected_peer_addr	connected_peer_port	level	message
+#types	time	string	string	string	addr	port	string	string
+1322788789.351248	bro	parent	-	-	-	info	[#1/127.0.0.1:47757] added peer
+1322788789.354851	bro	child	-	-	-	info	[#1/127.0.0.1:47757] connected
+1322788789.354956	bro	parent	-	-	-	info	[#1/127.0.0.1:47757] peer connected
+1322788789.354956	bro	parent	-	-	-	info	[#1/127.0.0.1:47757] phase: version
+1322788789.355429	bro	script	-	-	-	info	connection established
+1322788789.355429	bro	script	-	-	-	info	requesting events matching /^?(NOTHING)$?/
+1322788789.355429	bro	script	-	-	-	info	accepting state
+1322788789.355967	bro	parent	-	-	-	info	[#1/127.0.0.1:47757] phase: handshake
+1322788789.355967	bro	parent	-	-	-	info	warning: no events to request
+1322788789.355967	bro	parent	-	-	-	info	terminating...
+1322788789.355967	bro	parent	-	-	-	info	[#1/127.0.0.1:47757] peer_description is bro
+1322788789.355967	bro	parent	-	-	-	info	[#1/127.0.0.1:47757] closing connection

testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape-notset-str/test.log

@@ -0,0 +1,5 @@
+#separator \x09
+#path	test
+#fields	x	y	z
+#types	string	string	string
+\x2d	-	-

testing/btest/core/dns-init.bro

@@ -0,0 +1,9 @@
+# We once had a bug where DNS lookups at init time lead to an immediate crash. 
+#
+# @TEST-EXEC: bro %INPUT >output 2>&1
+# @TEST-EXEC: btest-diff output
+
+const foo: set[addr] = {
+     google.com
+};
+

testing/btest/core/leaks/dns.bro

@@ -4,6 +4,10 @@
 #
 # @TEST-EXEC: HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local bro -m -r $TRACES/wikipedia.trace %INPUT
 
+const foo: set[addr] = {
+     google.com
+};
+
 # Add the state tracking information variable to the connection record
 
 event connection_established(c: connection)

testing/btest/scripts/base/frameworks/communication/communication_log_baseline.bro

@@ -0,0 +1,37 @@
+# 
+# @TEST-EXEC: btest-bg-run receiver bro -b ../receiver.bro
+# @TEST-EXEC: btest-bg-run sender   bro -b ../sender.bro
+# @TEST-EXEC: btest-bg-wait -k 2
+#
+# Don't diff the receiver log just because port is always going to change
+# @TEST-EXEC: egrep -v 'pid|socket buffer size' sender/communication.log >send.log
+# @TEST-EXEC: btest-diff send.log
+
+@TEST-START-FILE sender.bro
+
+@load base/frameworks/communication/main
+
+redef Communication::nodes += {
+    ["foo"] = [$host = 127.0.0.1, $events = /NOTHING/, $connect=T]
+};
+
+event remote_connection_established(p: event_peer)
+	{
+	terminate_communication();
+	terminate();
+	}
+
+@TEST-END-FILE
+
+#############
+
+@TEST-START-FILE receiver.bro
+
+@load frameworks/communication/listen
+
+event remote_connection_closed(p: event_peer)
+	{
+	terminate();
+	}
+
+@TEST-END-FILE

testing/btest/scripts/base/frameworks/logging/ascii-escape-notset-str.bro

@@ -0,0 +1,23 @@
+#
+# @TEST-EXEC: bro -b %INPUT 
+# @TEST-EXEC: btest-diff test.log
+
+module Test;
+
+export {
+	redef enum Log::ID += { LOG };
+
+	type Log: record {
+		x: string &optional;
+		y: string &optional;
+		z: string &optional;
+	} &log;
+}
+
+event bro_init()
+{
+	Log::create_stream(Test::LOG, [$columns=Log]);
+	Log::write(Test::LOG, [$x=LogAscii::unset_field, $z=""]);
+}
+
+

testing/btest/scripts/base/frameworks/logging/env-ext.test

@@ -0,0 +1,2 @@
+# @TEST-EXEC: BRO_LOG_SUFFIX=txt bro -r $TRACES/wikipedia.trace
+# @TEST-EXEC: test -f conn.txt

testing/external/scripts/diff-all

@@ -1,6 +1,10 @@
 #! /usr/bin/env bash
 #
-# Runs btest-diff on $@ and fails if any fails.
+# Runs btest-diff on $@ and fails if any fails. If $@ contains globs, we expand
+# them relative to *both* the current directory and the test's baseline
+# directory so that we spot missing files. Note that you will need to quote
+# the globals in the TEST-EXEC line as otherwise they will have been expanded relative
+# to the current directory already when this scripts runs.
 
 diag=$TEST_DIAGNOSTICS
 
@@ -14,7 +18,10 @@ fi
 
 rc=0;
 
-for i in $@; do
+files_cwd=`ls $@`
+files_baseline=`cd $TEST_BASELINE && ls $@`
+
+for i in `echo $files_cwd $files_baseline | sort | uniq`; do
     if [[ "$i" != "loaded_scripts.log" && "$i" != "prof.log" ]]; then
         if ! btest-diff $i; then
            echo "" >>$diag

testing/external/scripts/perftools-adapt-paths

@@ -0,0 +1,10 @@
+#! /usr/bin/env bash
+#
+# Adapts relative paths in perftools stderr output to work
+# directly from the top-level test directory.
+#
+# Returns an exit code > 0 if there's a leak.
+
+cat $1 | sed "s#bro *\"\./#../../../build/src/bro \".tmp/$TEST_NAME/#g" | sed 's/ *--gv//g' >$1.tmp && mv $1.tmp $1
+
+grep -q "No leaks found" $1

testing/external/scripts/skel/test.skeleton

@@ -1,5 +1,5 @@
 # @TEST-EXEC: zcat $TRACES/trace.gz | bro -r - %INPUT
-# @TEST-EXEC: $SCRIPTS/diff-all *.log
+# @TEST-EXEC: $SCRIPTS/diff-all '*.log'
 
 @load testing-setup
 @load test-all-policy