diff --git a/CHANGES b/CHANGES index 5c1dc5574e..4f34ec88b6 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,51 @@ +2.0-beta-88 | 2011-12-02 17:00:58 -0800 + + * Teach LogWriterAscii to use BRO_LOG_SUFFIX environemt variable. + Addresses #704. (Jon Siwek) + + * Fix double-free of DNS_Mgr_Request object. Addresses #661. + + * Add a remote_log_peer event which comes with an event_peer record + parameter. Addresses #493. (Jon Siwek) + + * Remove example redef of SMTP::entity_excerpt_len from local.bro. + Fixes error emitted when loading local.bro in bare mode. (Jon + Siwek) + + * Add missing doc targets to top Makefile; remove old doc/Makefile. + Fixes #705. (Jon Siwek) + + * Turn some globals into constants. Addresses #633. (Seth Hall) + + * Rearrange packet filter and DPD documentation. (Jon Siwek) + +2.0-beta-72 | 2011-11-30 20:16:09 -0800 + + * Fine-tuning the Sphinx layout to better match www. (Jon Siwek and + Robin Sommer) + +2.0-beta-69 | 2011-11-29 16:55:31 -0800 + + * Fixing ASCII logger to escape the unset-field place holder if + written out literally. (Robin Sommer) + +2.0-beta-68 | 2011-11-29 15:23:12 -0800 + + * Lots of documentation polishing. (Jon Siwek) + + * Teach Broxygen the ".. bro:see::" directive. (Jon Siwek) + + * Teach Broxygen :bro:see: role for referencing any identifier in + the Bro domain. (Jon Siwek) + + * Teach Broxygen to generate an index of Bro notices. (Jon Siwek) + + * Fix order of include directories. (Jon Siwek) + + * Catch if logged vectors do not contain only atomic types. + (Bernhard Amann) + 2.0-beta-47 | 2011-11-16 08:24:33 -0800 * Catch if logged sets do not contain only atomic types. (Bernhard diff --git a/Makefile b/Makefile index e0c2860873..83a149ca58 100644 --- a/Makefile +++ b/Makefile @@ -29,6 +29,18 @@ doc: configured docclean: configured $(MAKE) -C $(BUILD) $@ +restdoc: configured + $(MAKE) -C $(BUILD) $@ + +restclean: configured + $(MAKE) -C $(BUILD) $@ + +broxygen: configured + $(MAKE) -C $(BUILD) $@ + +broxygenclean: configured + $(MAKE) -C $(BUILD) $@ + dist: @rm -rf $(VERSION_FULL) $(VERSION_FULL).tgz @rm -rf $(VERSION_MIN) $(VERSION_MIN).tgz diff --git a/VERSION b/VERSION index deb6c1aac5..b436bdb2cd 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.0-beta-47 +2.0-beta-88 diff --git a/aux/bro-aux b/aux/bro-aux index 7ea5837b4b..4d387ce660 160000 --- a/aux/bro-aux +++ b/aux/bro-aux @@ -1 +1 @@ -Subproject commit 7ea5837b4ba8403731ca4a9875616c0ab501342f +Subproject commit 4d387ce660468b44df99d4c87d6016ae4ed2fdc4 diff --git a/aux/broctl b/aux/broctl index 6771d28af2..be772bbada 160000 --- a/aux/broctl +++ b/aux/broctl @@ -1 +1 @@ -Subproject commit 6771d28af299f025a701e67f51311513af1cbc22 +Subproject commit be772bbada79b106db33fb9de5f56fa71226adc5 diff --git a/doc/CMakeLists.txt b/doc/CMakeLists.txt index 2f5bd93cdf..bdbb0e7b69 100644 --- a/doc/CMakeLists.txt +++ b/doc/CMakeLists.txt @@ -60,16 +60,16 @@ add_custom_target(broxygen # The "sphinxclean" target removes just the Sphinx input/output directories # from the build directory. -add_custom_target(broxygen-clean +add_custom_target(broxygenclean COMMAND "${CMAKE_COMMAND}" -E remove_directory ${DOC_SOURCE_WORKDIR} COMMAND "${CMAKE_COMMAND}" -E remove_directory ${DOC_OUTPUT_DIR} VERBATIM) -add_dependencies(broxygen broxygen-clean restdoc) +add_dependencies(broxygen broxygenclean restdoc) add_custom_target(doc) add_custom_target(docclean) add_dependencies(doc broxygen) -add_dependencies(docclean broxygen-clean restclean) +add_dependencies(docclean broxygenclean restclean) diff --git a/doc/Makefile b/doc/Makefile deleted file mode 100644 index 2756093a27..0000000000 --- a/doc/Makefile +++ /dev/null @@ -1,7 +0,0 @@ - -all: - test -d html || mkdir html - for i in *.rst; do echo "$$i ..."; ./bin/rst2html.py $$i >html/`echo $$i | sed 's/rst$$/html/g'`; done - -clean: - rm -rf html diff --git a/doc/README b/doc/README index a9f2cdbc4e..57d569db84 100644 --- a/doc/README +++ b/doc/README @@ -15,8 +15,9 @@ which adds some reST directives and roles that aid in generating useful index entries and cross-references. Other extensions can be added in a similar fashion. -Either the ``make doc`` or ``make broxygen`` can be used to locally -render the reST files into HTML. Those targets depend on: +Either the ``make doc`` or ``make broxygen`` targets in the top-level +Makefile can be used to locally render the reST files into HTML. +Those targets depend on: * Python interpreter >= 2.5 * `Sphinx `_ >= 1.0.1 diff --git a/doc/_static/broxygen-extra.css b/doc/_static/broxygen-extra.css index dd56416783..d873e286bc 100644 --- a/doc/_static/broxygen-extra.css +++ b/doc/_static/broxygen-extra.css @@ -1,3 +1,15 @@ .highlight { background-color: #ffffff; } + +h1 { + font-weight: bold; + font-size: 32px; + line-height:32px; + text-align: center; + padding-top: 3px; + margin-bottom: 30px; + font-family: Palatino,'Palatino Linotype',Georgia,serif;; + color: #000; + border-bottom: 0px; +} diff --git a/doc/_static/logo-bro.png b/doc/_static/logo-bro.png new file mode 100644 index 0000000000..96cc5d443c Binary files /dev/null and b/doc/_static/logo-bro.png differ diff --git a/doc/_templates/layout.html b/doc/_templates/layout.html index 0c58f34e17..cb6181361d 100644 --- a/doc/_templates/layout.html +++ b/doc/_templates/layout.html @@ -2,21 +2,97 @@ {% block extrahead %} + {% endblock %} {% block header %} - {% endblock %} {% block relbar2 %}{% endblock %} {% block relbar1 %}{% endblock %} +{% block content %} + +
+
+ +
+ +
+ {{ relbar() }} +
+ +
+ {% block body %} + {% endblock %} +
+
+ + +
+ +
+ +
+
+ + +
+

+ Table of Contents +

+

+

    {{toc}}
+

+
+ + {% if next %} +
+

+ Next Page +

+

+ {{ next.title }} +

+
+ {% endif %} + + {% if prev %} +
+

+ Previous Page +

+

+ {{ prev.title }} +

+
+ {% endif %} + +
+
+ +
+
+
+ + Copyright {{ copyright }}. + Last updated on {{ last_updated }}. + Created using Sphinx {{ sphinx_version }}. + +
+
+
+
+ + +{% endblock %} + {% block footer %} -{{ super() }} {% endblock %} diff --git a/doc/bin/rst2html.py b/doc/bin/rst2html.py deleted file mode 100755 index 79c835d6c4..0000000000 --- a/doc/bin/rst2html.py +++ /dev/null @@ -1,62 +0,0 @@ -#!/usr/bin/env python -# -# Derived from docutils standard rst2html.py. -# -# $Id: rst2html.py 4564 2006-05-21 20:44:42Z wiemann $ -# Author: David Goodger -# Copyright: This module has been placed in the public domain. -# -# -# Extension: we add to dummy directorives "code" and "console" to be -# compatible with Bro's web site setup. - -try: - import locale - locale.setlocale(locale.LC_ALL, '') -except: - pass - -import textwrap - -from docutils.core import publish_cmdline, default_description - -from docutils import nodes -from docutils.parsers.rst import directives, Directive -from docutils.parsers.rst.directives.body import LineBlock - -class Literal(Directive): - #max_line_length = 68 - max_line_length = 0 - - required_arguments = 0 - optional_arguments = 1 - final_argument_whitespace = True - has_content = True - - def wrapped_content(self): - content = [] - - if Literal.max_line_length: - for line in self.content: - content += textwrap.wrap(line, Literal.max_line_length, subsequent_indent=" ") - else: - content = self.content - - return u'\n'.join(content) - - def run(self): - self.assert_has_content() - content = self.wrapped_content() - literal = nodes.literal_block(content, content) - return [literal] - -directives.register_directive('code', Literal) -directives.register_directive('console', Literal) - -description = ('Generates (X)HTML documents from standalone reStructuredText ' - 'sources. ' + default_description) - -publish_cmdline(writer_name='html', description=description) - - - diff --git a/doc/conf.py.in b/doc/conf.py.in index 8844370a07..8959c0b2c9 100644 --- a/doc/conf.py.in +++ b/doc/conf.py.in @@ -90,44 +90,20 @@ pygments_style = 'sphinx' # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. -html_theme = 'default' +html_theme = 'basic' html_last_updated_fmt = '%B %d, %Y' # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. -html_theme_options = { -"rightsidebar": "true", -"stickysidebar": "false", -"externalrefs": "false", -"footerbgcolor": "#333", -"footertextcolor": "#ddd", -"sidebarbgcolor": "#ffffff", -#"sidebarbtncolor": "", -"sidebartextcolor": "#333", -"sidebarlinkcolor": "#2a85a7", -"relbarbgcolor": "#ffffff", -"relbartextcolor": "#333", -"relbarlinkcolor": "#2a85a7", -"bgcolor": "#ffffff", -"textcolor": "#333", -"linkcolor": "#2a85a7", -"visitedlinkcolor": "#2a85a7", -"headbgcolor": "#f0f0f0", -"headtextcolor": "#000", -"headlinkcolor": "#2a85a7", -"codebgcolor": "#FFFAE2", -#"codetextcolor": "", -"bodyfont": "Arial, Helvetica, sans-serif", -"headfont": "Palatino,'Palatino Linotype',Georgia,serif", -} +html_theme_options = { } # Add any paths that contain custom themes here, relative to this directory. #html_theme_path = [] # The name for this set of Sphinx documents. If None, it defaults to -# " v documentation". +# " v Documentation". #html_title = None # A shorter title for the navigation bar. Default is the same as html_title. diff --git a/doc/index.rst b/doc/index.rst index 4a72ea9121..16fee55f5c 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -1,9 +1,11 @@ .. Bro documentation master file +================= Bro Documentation ================= -Documentation for version: |version| +Guides +------ .. toctree:: :maxdepth: 1 @@ -40,7 +42,6 @@ Script Reference .. toctree:: :maxdepth: 1 - scripts/common scripts/builtins scripts/bifs scripts/packages diff --git a/doc/notice.rst b/doc/notice.rst index b1ffdacb75..bab44ab9e4 100644 --- a/doc/notice.rst +++ b/doc/notice.rst @@ -29,17 +29,18 @@ definitions of what constitutes an attack or even a compromise differ quite a bit between environments, and activity deemed malicious at one site might be fully acceptable at another. -Whenever one of Bro's analysis scripts sees something potentially interesting -it flags the situation by calling the ``NOTICE`` function and giving it a -single ``Notice::Info`` record. A Notice has a ``Notice::Type``, which -reflects the kind of activity that has been seen, and it is usually also -augmented with further context about the situation. +Whenever one of Bro's analysis scripts sees something potentially +interesting it flags the situation by calling the :bro:see:`NOTICE` +function and giving it a single :bro:see:`Notice::Info` record. A Notice +has a :bro:see:`Notice::Type`, which reflects the kind of activity that +has been seen, and it is usually also augmented with further context +about the situation. More information about raising notices can be found in the `Raising Notices`_ section. Once a notice is raised, it can have any number of actions applied to it by -the ``Notice::policy`` set which is described in the `Notice Policy`_ +the :bro:see:`Notice::policy` set which is described in the `Notice Policy`_ section below. Such actions can be to send a mail to the configured address(es) or to simply ignore the notice. Currently, the following actions are defined: @@ -52,20 +53,20 @@ are defined: - Description * - Notice::ACTION_LOG - - Write the notice to the ``Notice::LOG`` logging stream. + - Write the notice to the :bro:see:`Notice::LOG` logging stream. * - Notice::ACTION_ALARM - - Log into the ``Notice::ALARM_LOG`` stream which will rotate + - Log into the :bro:see:`Notice::ALARM_LOG` stream which will rotate hourly and email the contents to the email address or addresses - defined in the ``Notice::mail_dest`` variable. + defined in the :bro:see:`Notice::mail_dest` variable. * - Notice::ACTION_EMAIL - Send the notice in an email to the email address or addresses given in - the ``Notice::mail_dest`` variable. + the :bro:see:`Notice::mail_dest` variable. * - Notice::ACTION_PAGE - Send an email to the email address or addresses given in the - ``Notice::mail_page_dest`` variable. + :bro:see:`Notice::mail_page_dest` variable. * - Notice::ACTION_NO_SUPPRESS - This action will disable the built in notice suppression for the @@ -82,15 +83,17 @@ Processing Notices Notice Policy ************* -The predefined set ``Notice::policy`` provides the mechanism for applying -actions and other behavior modifications to notices. Each entry of -``Notice::policy`` is a record of the type ``Notice::PolicyItem`` which -defines a condition to be matched against all raised notices and one or more -of a variety of behavior modifiers. The notice policy is defined by adding any -number of ``Notice::PolicyItem`` records to the ``Notice::policy`` set. +The predefined set :bro:see:`Notice::policy` provides the mechanism for +applying actions and other behavior modifications to notices. Each entry +of :bro:see:`Notice::policy` is a record of the type +:bro:see:`Notice::PolicyItem` which defines a condition to be matched +against all raised notices and one or more of a variety of behavior +modifiers. The notice policy is defined by adding any number of +:bro:see:`Notice::PolicyItem` records to the :bro:see:`Notice::policy` +set. Here's a simple example which tells Bro to send an email for all notices of -type ``SSH::Login`` if the server is 10.0.0.1: +type :bro:see:`SSH::Login` if the server is 10.0.0.1: .. code:: bro @@ -113,11 +116,11 @@ flexibility due to having access to Bro's full programming language. Predicate Field ^^^^^^^^^^^^^^^ -The ``Notice::PolicyItem`` record type has a field name ``$pred`` which -defines the entry's condition in the form of a predicate written as a Bro -function. The function is passed the notice as a ``Notice::Info`` record and -it returns a boolean value indicating if the entry is applicable to that -particular notice. +The :bro:see:`Notice::PolicyItem` record type has a field name ``$pred`` +which defines the entry's condition in the form of a predicate written +as a Bro function. The function is passed the notice as a +:bro:see:`Notice::Info` record and it returns a boolean value indicating +if the entry is applicable to that particular notice. .. note:: @@ -125,14 +128,14 @@ particular notice. (``T``) since an implicit false (``F``) value would never be used. Bro evaluates the predicates of each entry in the order defined by the -``$priority`` field in ``Notice::PolicyItem`` records. The valid values are -0-10 with 10 being earliest evaluated. If ``$priority`` is omitted, the -default priority is 5. +``$priority`` field in :bro:see:`Notice::PolicyItem` records. The valid +values are 0-10 with 10 being earliest evaluated. If ``$priority`` is +omitted, the default priority is 5. Behavior Modification Fields ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -There are a set of fields in the ``Notice::PolicyItem`` record type that +There are a set of fields in the :bro:see:`Notice::PolicyItem` record type that indicate ways that either the notice or notice processing should be modified if the predicate field (``$pred``) evaluated to true (``T``). Those fields are explained in more detail in the following table. @@ -146,8 +149,8 @@ explained in more detail in the following table. - Example * - ``$action=`` - - Each Notice::PolicyItem can have a single action applied to the notice - with this field. + - Each :bro:see:`Notice::PolicyItem` can have a single action + applied to the notice with this field. - ``$action = Notice::ACTION_EMAIL`` * - ``$suppress_for=`` @@ -162,9 +165,9 @@ explained in more detail in the following table. - This field can be used for modification of the notice policy evaluation. To stop processing of notice policy items before evaluating all of them, set this field to ``T`` and make the ``$pred`` - field return ``T``. ``Notice::PolicyItem`` records defined at a higher - priority as defined by the ``$priority`` field will still be evaluated - but those at a lower priority won't. + field return ``T``. :bro:see:`Notice::PolicyItem` records defined at + a higher priority as defined by the ``$priority`` field will still be + evaluated but those at a lower priority won't. - ``$halt = T`` @@ -186,11 +189,11 @@ Notice Policy Shortcuts Although the notice framework provides a great deal of flexibility and configurability there are many times that the full expressiveness isn't needed and actually becomes a hindrance to achieving results. The framework provides -a default ``Notice::policy`` suite as a way of giving users the +a default :bro:see:`Notice::policy` suite as a way of giving users the shortcuts to easily apply many common actions to notices. These are implemented as sets and tables indexed with a -``Notice::Type`` enum value. The following table shows and describes +:bro:see:`Notice::Type` enum value. The following table shows and describes all of the variables available for shortcut configuration of the notice framework. @@ -201,40 +204,44 @@ framework. * - Variable name - Description - * - Notice::ignored_types - - Adding a ``Notice::Type`` to this set results in the notice + * - :bro:see:`Notice::ignored_types` + - Adding a :bro:see:`Notice::Type` to this set results in the notice being ignored. It won't have any other action applied to it, not even - ``Notice::ACTION_LOG``. + :bro:see:`Notice::ACTION_LOG`. - * - Notice::emailed_types - - Adding a ``Notice::Type`` to this set results in - ``Notice::ACTION_EMAIL`` being applied to the notices of that type. + * - :bro:see:`Notice::emailed_types` + - Adding a :bro:see:`Notice::Type` to this set results in + :bro:see:`Notice::ACTION_EMAIL` being applied to the notices of + that type. - * - Notice::alarmed_types - - Adding a Notice::Type to this set results in - ``Notice::ACTION_ALARM`` being applied to the notices of that type. + * - :bro:see:`Notice::alarmed_types` + - Adding a :bro:see:`Notice::Type` to this set results in + :bro:see:`Notice::ACTION_ALARM` being applied to the notices of + that type. - * - Notice::not_suppressed_types - - Adding a ``Notice::Type`` to this set results in that notice no longer - undergoing the normal notice suppression that would take place. Be - careful when using this in production it could result in a dramatic - increase in the number of notices being processed. + * - :bro:see:`Notice::not_suppressed_types` + - Adding a :bro:see:`Notice::Type` to this set results in that notice + no longer undergoes the normal notice suppression that would + take place. Be careful when using this in production it could + result in a dramatic increase in the number of notices being + processed. - * - Notice::type_suppression_intervals - - This is a table indexed on ``Notice::Type`` and yielding an interval. - It can be used as an easy way to extend the default suppression - interval for an entire ``Notice::Type`` without having to create a - whole ``Notice::policy`` entry and setting the ``$suppress_for`` - field. + * - :bro:see:`Notice::type_suppression_intervals` + - This is a table indexed on :bro:see:`Notice::Type` and yielding an + interval. It can be used as an easy way to extend the default + suppression interval for an entire :bro:see:`Notice::Type` + without having to create a whole :bro:see:`Notice::policy` entry + and setting the ``$suppress_for`` field. Raising Notices --------------- -A script should raise a notice for any occurrence that a user may want to be -notified about or take action on. For example, whenever the base SSH analysis -scripts sees an SSH session where it is heuristically guessed to be a -successful login, it raises a Notice of the type ``SSH::Login``. The code in -the base SSH analysis script looks like this: +A script should raise a notice for any occurrence that a user may want +to be notified about or take action on. For example, whenever the base +SSH analysis scripts sees an SSH session where it is heuristically +guessed to be a successful login, it raises a Notice of the type +:bro:see:`SSH::Login`. The code in the base SSH analysis script looks +like this: .. code:: bro @@ -242,10 +249,10 @@ the base SSH analysis script looks like this: $msg="Heuristically detected successful SSH login.", $conn=c]); -``NOTICE`` is a normal function in the global namespace which wraps a function -within the ``Notice`` namespace. It takes a single argument of the -``Notice::Info`` record type. The most common fields used when raising notices -are described in the following table: +:bro:see:`NOTICE` is a normal function in the global namespace which +wraps a function within the ``Notice`` namespace. It takes a single +argument of the :bro:see:`Notice::Info` record type. The most common +fields used when raising notices are described in the following table: .. list-table:: :widths: 32 40 @@ -295,9 +302,10 @@ are described in the following table: * - ``$suppress_for`` - This field can be set if there is a natural suppression interval for - the notice that may be different than the default value. The value set - to this field can also be modified by a user's ``Notice::policy`` so - the value is not set permanently and unchangeably. + the notice that may be different than the default value. The + value set to this field can also be modified by a user's + :bro:see:`Notice::policy` so the value is not set permanently + and unchangeably. When writing Bro scripts which raise notices, some thought should be given to what the notice represents and what data should be provided to give a consumer @@ -325,7 +333,7 @@ The notice framework supports suppression for notices if the author of the script that is generating the notice has indicated to the notice framework how to identify notices that are intrinsically the same. Identification of these "intrinsically duplicate" notices is implemented with an optional field in -``Notice::Info`` records named ``$identifier`` which is a simple string. +:bro:see:`Notice::Info` records named ``$identifier`` which is a simple string. If the ``$identifier`` and ``$type`` fields are the same for two notices, the notice framework actually considers them to be the same thing and can use that information to suppress duplicates for a configurable period of time. @@ -337,12 +345,13 @@ information to suppress duplicates for a configurable period of time. could be completely legitimate usage if no notices could ever be considered to be duplicates. -The ``$identifier`` field is typically comprised of several pieces of data -related to the notice that when combined represent a unique instance of that -notice. Here is an example of the script -``policy/protocols/ssl/validate-certs.bro`` raising a notice for session -negotiations where the certificate or certificate chain did not validate -successfully against the available certificate authority certificates. +The ``$identifier`` field is typically comprised of several pieces of +data related to the notice that when combined represent a unique +instance of that notice. Here is an example of the script +:doc:`scripts/policy/protocols/ssl/validate-certs` raising a notice +for session negotiations where the certificate or certificate chain did +not validate successfully against the available certificate authority +certificates. .. code:: bro @@ -369,7 +378,7 @@ it's assumed that the script author who is raising the notice understands the full problem set and edge cases of the notice which may not be readily apparent to users. If users don't want the suppression to take place or simply want a different interval, they can always modify it with the -``Notice::policy``. +:bro:see:`Notice::policy`. Extending Notice Framework diff --git a/doc/scripts/README b/doc/scripts/README index b3e44914f4..a15812609c 100644 --- a/doc/scripts/README +++ b/doc/scripts/README @@ -1,6 +1,6 @@ This directory contains scripts and templates that can be used to automate the generation of Bro script documentation. Several build targets are defined -by CMake: +by CMake and available in the top-level Makefile: ``restdoc`` diff --git a/doc/scripts/common.rst b/doc/scripts/common.rst deleted file mode 100644 index 6105585b2c..0000000000 --- a/doc/scripts/common.rst +++ /dev/null @@ -1,19 +0,0 @@ -Common Documentation -==================== - -.. _common_port_analysis_doc: - -Port Analysis -------------- - -TODO: add some stuff here - -.. _common_packet_filter_doc: - -Packet Filter -------------- - -TODO: add some stuff here - -.. note:: Filters are only relevant when dynamic protocol detection (DPD) - is explicitly turned off (Bro release 1.6 enabled DPD by default). diff --git a/doc/scripts/index.rst b/doc/scripts/index.rst index 5de203afe7..bf0fa25f10 100644 --- a/doc/scripts/index.rst +++ b/doc/scripts/index.rst @@ -1,7 +1,7 @@ .. This is a stub doc to which broxygen appends during the build process -Index of All Bro Scripts -======================== +Index of All Individual Bro Scripts +=================================== .. toctree:: :maxdepth: 1 diff --git a/scripts/base/frameworks/communication/main.bro b/scripts/base/frameworks/communication/main.bro index 569ba140a9..01c608c8db 100644 --- a/scripts/base/frameworks/communication/main.bro +++ b/scripts/base/frameworks/communication/main.bro @@ -130,6 +130,13 @@ event remote_log(level: count, src: count, msg: string) do_script_log_common(level, src, msg); } +# This is a core generated event. +event remote_log_peer(p: event_peer, level: count, src: count, msg: string) + { + local rmsg = fmt("[#%d/%s:%d] %s", p$id, p$host, p$p, msg); + do_script_log_common(level, src, rmsg); + } + function do_script_log(p: event_peer, msg: string) { do_script_log_common(REMOTE_LOG_INFO, REMOTE_SRC_SCRIPT, msg); diff --git a/scripts/base/protocols/dns/main.bro b/scripts/base/protocols/dns/main.bro index 2580b003dd..c8c56bdc00 100644 --- a/scripts/base/protocols/dns/main.bro +++ b/scripts/base/protocols/dns/main.bro @@ -65,11 +65,11 @@ redef capture_filters += { ["netbios-ns"] = "udp port 137", }; -global dns_ports = { 53/udp, 53/tcp, 137/udp, 5353/udp, 5355/udp } &redef; +const dns_ports = { 53/udp, 53/tcp, 137/udp, 5353/udp, 5355/udp }; redef dpd_config += { [ANALYZER_DNS] = [$ports = dns_ports] }; -global dns_udp_ports = { 53/udp, 137/udp, 5353/udp, 5355/udp } &redef; -global dns_tcp_ports = { 53/tcp } &redef; +const dns_udp_ports = { 53/udp, 137/udp, 5353/udp, 5355/udp }; +const dns_tcp_ports = { 53/tcp }; redef dpd_config += { [ANALYZER_DNS_UDP_BINPAC] = [$ports = dns_udp_ports] }; redef dpd_config += { [ANALYZER_DNS_TCP_BINPAC] = [$ports = dns_tcp_ports] }; diff --git a/scripts/base/protocols/irc/main.bro b/scripts/base/protocols/irc/main.bro index 731a943819..e74eea21f1 100644 --- a/scripts/base/protocols/irc/main.bro +++ b/scripts/base/protocols/irc/main.bro @@ -41,7 +41,7 @@ redef capture_filters += { ["irc-6668"] = "port 6668" }; redef capture_filters += { ["irc-6669"] = "port 6669" }; # DPD configuration. -global irc_ports = { 6666/tcp, 6667/tcp, 6668/tcp, 6669/tcp } &redef; +const irc_ports = { 6666/tcp, 6667/tcp, 6668/tcp, 6669/tcp }; redef dpd_config += { [ANALYZER_IRC] = [$ports = irc_ports] }; redef likely_server_ports += { 6666/tcp, 6667/tcp, 6668/tcp, 6669/tcp }; diff --git a/scripts/site/local.bro b/scripts/site/local.bro index 597b92ba3d..8e1f4d802b 100644 --- a/scripts/site/local.bro +++ b/scripts/site/local.bro @@ -62,11 +62,3 @@ redef signature_files += "frameworks/signatures/detect-windows-shells.sig"; @load protocols/http/detect-MHR # Detect SQL injection attacks @load protocols/http/detect-sqli - -# Uncomment this redef if you want to extract SMTP MIME entities for -# some file types. The numbers given indicate how many bytes to extract for -# the various mime types. -redef SMTP::entity_excerpt_len += { -# ["text/plain"] = 1024, -# ["text/html"] = 1024, -}; diff --git a/src/BroDoc.cc b/src/BroDoc.cc index b84b9d023d..91aed5ec62 100644 --- a/src/BroDoc.cc +++ b/src/BroDoc.cc @@ -215,14 +215,16 @@ void BroDoc::WriteDocFile() const if ( ! port_analysis.empty() ) { WriteSectionHeading("Port Analysis", '-'); - WriteToDoc(":ref:`More Information `\n\n"); + WriteToDoc("Loading this script makes the following changes to " + ":bro:see:`dpd_config`.\n\n"); WriteStringList("%s", port_analysis); } if ( ! packet_filter.empty() ) { WriteSectionHeading("Packet Filter", '-'); - WriteToDoc(":ref:`More Information `\n\n"); + WriteToDoc("Loading this script makes the following changes to " + ":bro:see:`capture_filters`.\n\n"); WriteToDoc("Filters added::\n\n"); WriteToDoc("%s\n", packet_filter.c_str()); } diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index f215fe5d9c..55e1ce0e81 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,4 +1,5 @@ -include_directories(${CMAKE_CURRENT_SOURCE_DIR} +include_directories(BEFORE + ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR} ) diff --git a/src/DNS_Mgr.cc b/src/DNS_Mgr.cc index 736c262222..2b9d07a969 100644 --- a/src/DNS_Mgr.cc +++ b/src/DNS_Mgr.cc @@ -595,8 +595,6 @@ void DNS_Mgr::Resolve() } else --num_pending; - - delete dr; } } diff --git a/src/LogMgr.cc b/src/LogMgr.cc index 80f6c26ac0..7fc8d4ef86 100644 --- a/src/LogMgr.cc +++ b/src/LogMgr.cc @@ -159,7 +159,7 @@ bool LogVal::IsCompatibleType(BroType* t, bool atomic_only) if ( atomic_only ) return false; - return IsCompatibleType(t->AsVectorType()->YieldType()); + return IsCompatibleType(t->AsVectorType()->YieldType(), true); } default: diff --git a/src/LogWriterAscii.cc b/src/LogWriterAscii.cc index 9fc71789d8..9b1fda3b62 100644 --- a/src/LogWriterAscii.cc +++ b/src/LogWriterAscii.cc @@ -88,7 +88,7 @@ bool LogWriterAscii::DoInit(string path, int num_fields, if ( output_to_stdout ) path = "/dev/stdout"; - fname = IsSpecial(path) ? path : path + ".log"; + fname = IsSpecial(path) ? path : path + "." + LogExt(); if ( ! (file = fopen(fname.c_str(), "w")) ) { @@ -200,10 +200,33 @@ bool LogWriterAscii::DoWriteOne(ODesc* desc, LogVal* val, const LogField* field) case TYPE_FUNC: { int size = val->val.string_val->size(); - if ( size ) - desc->AddN(val->val.string_val->data(), val->val.string_val->size()); - else + const char* data = val->val.string_val->data(); + + if ( ! size ) + { desc->AddN(empty_field, empty_field_len); + break; + } + + if ( size == unset_field_len && memcmp(data, unset_field, size) == 0 ) + { + // The value we'd write out would match exactly the + // place-holder we use for unset optional fields. We + // escape the first character so that the output + // won't be ambigious. + static const char hex_chars[] = "0123456789abcdef"; + char hex[6] = "\\x00"; + hex[2] = hex_chars[((*data) & 0xf0) >> 4]; + hex[3] = hex_chars[(*data) & 0x0f]; + desc->AddRaw(hex, 4); + + ++data; + --size; + } + + if ( size ) + desc->AddN(data, size); + break; } @@ -297,7 +320,7 @@ bool LogWriterAscii::DoRotate(string rotated_path, double open, fclose(file); file = 0; - string nname = rotated_path + ".log"; + string nname = rotated_path + "." + LogExt(); rename(fname.c_str(), nname.c_str()); if ( ! FinishedRotation(nname, fname, open, close, terminating) ) @@ -315,4 +338,9 @@ bool LogWriterAscii::DoSetBuf(bool enabled) return true; } - +string LogWriterAscii::LogExt() + { + const char* ext = getenv("BRO_LOG_SUFFIX"); + if ( ! ext ) ext = "log"; + return ext; + } diff --git a/src/LogWriterAscii.h b/src/LogWriterAscii.h index 7755f71d06..72127c8b1f 100644 --- a/src/LogWriterAscii.h +++ b/src/LogWriterAscii.h @@ -13,6 +13,7 @@ public: ~LogWriterAscii(); static LogWriter* Instantiate() { return new LogWriterAscii; } + static string LogExt(); protected: virtual bool DoInit(string path, int num_fields, diff --git a/src/RemoteSerializer.cc b/src/RemoteSerializer.cc index a21a7abc60..1ce3b3b087 100644 --- a/src/RemoteSerializer.cc +++ b/src/RemoteSerializer.cc @@ -2923,25 +2923,37 @@ void RemoteSerializer::Log(LogLevel level, const char* msg) void RemoteSerializer::Log(LogLevel level, const char* msg, Peer* peer, LogSrc src) { + if ( peer ) + { + val_list* vl = new val_list(); + vl->append(peer->val->Ref()); + vl->append(new Val(level, TYPE_COUNT)); + vl->append(new Val(src, TYPE_COUNT)); + vl->append(new StringVal(msg)); + mgr.QueueEvent(remote_log_peer, vl); + } + else + { + val_list* vl = new val_list(); + vl->append(new Val(level, TYPE_COUNT)); + vl->append(new Val(src, TYPE_COUNT)); + vl->append(new StringVal(msg)); + mgr.QueueEvent(remote_log, vl); + } + +#ifdef DEBUG const int BUFSIZE = 1024; char buffer[BUFSIZE]; - int len = 0; if ( peer ) - len += snprintf(buffer + len, sizeof(buffer) - len, - "[#%d/%s:%d] ", int(peer->id), ip2a(peer->ip), - peer->port); + len += snprintf(buffer + len, sizeof(buffer) - len, "[#%d/%s:%d] ", + int(peer->id), ip2a(peer->ip), peer->port); len += safe_snprintf(buffer + len, sizeof(buffer) - len, "%s", msg); - val_list* vl = new val_list(); - vl->append(new Val(level, TYPE_COUNT)); - vl->append(new Val(src, TYPE_COUNT)); - vl->append(new StringVal(buffer)); - mgr.QueueEvent(remote_log, vl); - DEBUG_COMM(fmt("parent: %.6f %s", current_time(), buffer)); +#endif } void RemoteSerializer::RaiseEvent(EventHandlerPtr event, Peer* peer, diff --git a/src/event.bif b/src/event.bif index d953ac78fe..0c2f7eb780 100644 --- a/src/event.bif +++ b/src/event.bif @@ -444,6 +444,29 @@ event remote_state_inconsistency%(operation: string, id: string, # Generated for communication log message. event remote_log%(level: count, src: count, msg: string%); +## Generated for communication log messages. While this event is +## intended primarily for use by Bro's communication framework, it can also trigger +## additional code if helpful. This event is equivalent to +## :bro:see:`remote_log` except the message is with respect to a certain peer. +## +## p: A record describing the remote peer. +## +## level: The log level, which is either :bro:enum:`REMOTE_LOG_INFO` or +## :bro:enum:`REMOTE_LOG_ERROR`. +## +## src: The component of the comminication system that logged the message. +## Currently, this will be one of :bro:enum:`REMOTE_SRC_CHILD` (Bro's +## child process), :bro:enum:`REMOTE_SRC_PARENT` (Bro's main process), or +## :bro:enum:`REMOTE_SRC_SCRIPT` (the script level). +## +## msg: The message logged. +## +## .. bro:see:: remote_capture_filter remote_connection_closed remote_connection_error +## remote_connection_established remote_connection_handshake_done +## remote_event_registered remote_pong remote_state_access_performed +## remote_state_inconsistency print_hook remote_log +event remote_log_peer%(p: event_peer, level: count, src: count, msg: string%); + # Generated when a remote peer has answered to our ping. event remote_pong%(p: event_peer, seq: count, d1: interval, d2: interval, d3: interval%); diff --git a/src/main.cc b/src/main.cc index b3f2512b40..20525b876b 100644 --- a/src/main.cc +++ b/src/main.cc @@ -48,6 +48,7 @@ extern "C" void OPENSSL_add_all_algorithms_conf(void); #include "ConnCompressor.h" #include "DPM.h" #include "BroDoc.h" +#include "LogWriterAscii.h" #include "binpac_bro.h" @@ -196,6 +197,7 @@ void usage() fprintf(stderr, " $BRO_PREFIXES | prefix list (%s)\n", bro_prefixes()); fprintf(stderr, " $BRO_DNS_FAKE | disable DNS lookups (%s)\n", bro_dns_fake()); fprintf(stderr, " $BRO_SEED_FILE | file to load seeds from (not set)\n"); + fprintf(stderr, " $BRO_LOG_SUFFIX | ASCII log file extension (.%s)\n", LogWriterAscii::LogExt().c_str()); exit(1); } diff --git a/testing/btest/Baseline/core.dns-init/output b/testing/btest/Baseline/core.dns-init/output new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testing/btest/Baseline/doc.autogen-reST-example/example.rst b/testing/btest/Baseline/doc.autogen-reST-example/example.rst index 880fa0e171..7c61277187 100644 --- a/testing/btest/Baseline/doc.autogen-reST-example/example.rst +++ b/testing/btest/Baseline/doc.autogen-reST-example/example.rst @@ -281,7 +281,7 @@ Redefinitions Port Analysis ------------- -:ref:`More Information ` +Loading this script makes the following changes to :bro:see:`dpd_config`. SSL:: @@ -292,7 +292,7 @@ SSL:: Packet Filter ------------- -:ref:`More Information ` +Loading this script makes the following changes to :bro:see:`capture_filters`. Filters added:: diff --git a/testing/btest/Baseline/scripts.base.frameworks.communication.communication_log_baseline/send.log b/testing/btest/Baseline/scripts.base.frameworks.communication.communication_log_baseline/send.log new file mode 100644 index 0000000000..7f71757ca0 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.communication.communication_log_baseline/send.log @@ -0,0 +1,16 @@ +#separator \x09 +#path communication +#fields ts peer src_name connected_peer_desc connected_peer_addr connected_peer_port level message +#types time string string string addr port string string +1322788789.351248 bro parent - - - info [#1/127.0.0.1:47757] added peer +1322788789.354851 bro child - - - info [#1/127.0.0.1:47757] connected +1322788789.354956 bro parent - - - info [#1/127.0.0.1:47757] peer connected +1322788789.354956 bro parent - - - info [#1/127.0.0.1:47757] phase: version +1322788789.355429 bro script - - - info connection established +1322788789.355429 bro script - - - info requesting events matching /^?(NOTHING)$?/ +1322788789.355429 bro script - - - info accepting state +1322788789.355967 bro parent - - - info [#1/127.0.0.1:47757] phase: handshake +1322788789.355967 bro parent - - - info warning: no events to request +1322788789.355967 bro parent - - - info terminating... +1322788789.355967 bro parent - - - info [#1/127.0.0.1:47757] peer_description is bro +1322788789.355967 bro parent - - - info [#1/127.0.0.1:47757] closing connection diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape-notset-str/test.log b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape-notset-str/test.log new file mode 100644 index 0000000000..683fed60f2 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape-notset-str/test.log @@ -0,0 +1,5 @@ +#separator \x09 +#path test +#fields x y z +#types string string string +\x2d - - diff --git a/testing/btest/core/dns-init.bro b/testing/btest/core/dns-init.bro new file mode 100644 index 0000000000..5a7efff6fb --- /dev/null +++ b/testing/btest/core/dns-init.bro @@ -0,0 +1,9 @@ +# We once had a bug where DNS lookups at init time lead to an immediate crash. +# +# @TEST-EXEC: bro %INPUT >output 2>&1 +# @TEST-EXEC: btest-diff output + +const foo: set[addr] = { + google.com +}; + diff --git a/testing/btest/core/leaks/dns.bro b/testing/btest/core/leaks/dns.bro index 1dce9c2c82..3d3fdc6f09 100644 --- a/testing/btest/core/leaks/dns.bro +++ b/testing/btest/core/leaks/dns.bro @@ -4,6 +4,10 @@ # # @TEST-EXEC: HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local bro -m -r $TRACES/wikipedia.trace %INPUT +const foo: set[addr] = { + google.com +}; + # Add the state tracking information variable to the connection record event connection_established(c: connection) diff --git a/testing/btest/scripts/base/frameworks/communication/communication_log_baseline.bro b/testing/btest/scripts/base/frameworks/communication/communication_log_baseline.bro new file mode 100644 index 0000000000..c3078684af --- /dev/null +++ b/testing/btest/scripts/base/frameworks/communication/communication_log_baseline.bro @@ -0,0 +1,37 @@ +# +# @TEST-EXEC: btest-bg-run receiver bro -b ../receiver.bro +# @TEST-EXEC: btest-bg-run sender bro -b ../sender.bro +# @TEST-EXEC: btest-bg-wait -k 2 +# +# Don't diff the receiver log just because port is always going to change +# @TEST-EXEC: egrep -v 'pid|socket buffer size' sender/communication.log >send.log +# @TEST-EXEC: btest-diff send.log + +@TEST-START-FILE sender.bro + +@load base/frameworks/communication/main + +redef Communication::nodes += { + ["foo"] = [$host = 127.0.0.1, $events = /NOTHING/, $connect=T] +}; + +event remote_connection_established(p: event_peer) + { + terminate_communication(); + terminate(); + } + +@TEST-END-FILE + +############# + +@TEST-START-FILE receiver.bro + +@load frameworks/communication/listen + +event remote_connection_closed(p: event_peer) + { + terminate(); + } + +@TEST-END-FILE diff --git a/testing/btest/scripts/base/frameworks/logging/ascii-escape-notset-str.bro b/testing/btest/scripts/base/frameworks/logging/ascii-escape-notset-str.bro new file mode 100644 index 0000000000..8c1401b179 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/logging/ascii-escape-notset-str.bro @@ -0,0 +1,23 @@ +# +# @TEST-EXEC: bro -b %INPUT +# @TEST-EXEC: btest-diff test.log + +module Test; + +export { + redef enum Log::ID += { LOG }; + + type Log: record { + x: string &optional; + y: string &optional; + z: string &optional; + } &log; +} + +event bro_init() +{ + Log::create_stream(Test::LOG, [$columns=Log]); + Log::write(Test::LOG, [$x=LogAscii::unset_field, $z=""]); +} + + diff --git a/testing/btest/scripts/base/frameworks/logging/env-ext.test b/testing/btest/scripts/base/frameworks/logging/env-ext.test new file mode 100644 index 0000000000..e9f690caa4 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/logging/env-ext.test @@ -0,0 +1,2 @@ +# @TEST-EXEC: BRO_LOG_SUFFIX=txt bro -r $TRACES/wikipedia.trace +# @TEST-EXEC: test -f conn.txt