diff --git a/CHANGES b/CHANGES index 5c1dc5574e..2e2ac75758 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,176 @@ +2.0-beta-155 | 2012-01-03 15:42:32 -0800 + + * Remove dead code related to record type inheritance. (Jon Siwek) + +2.0-beta-152 | 2012-01-03 14:51:34 -0800 + + * Notices now record the transport-layer protocol. (Bernhard Amann) + +2.0-beta-150 | 2012-01-03 14:42:45 -0800 + + * CMake 2.6 top-level 'install' target compat. Fixes #729. (Jon Siwek) + + * Minor fixes to test process. Addresses #298. + + * Increase timeout interval of communication-related btests. (Jon Siwek) + +2.0-beta-145 | 2011-12-19 11:37:15 -0800 + + * Empty fields are now logged as "(empty)" by default. (Robin + Sommer) + + * In log headers, only escape information when necessary. (Robin + Sommer) + +2.0-beta-139 | 2011-12-19 07:06:29 -0800 + + * The hostname notice email extension works now, plus a general + mechanism for adding delayed information to notices. (Seth Hall) + + * Fix &default fields in records not being initialized in coerced + assignments. Addresses #722. (Jon Siwek) + + * Make log headers include the type of data stored inside a set or + vector ("vector[string]"). (Bernhard Amann) + +2.0-beta-126 | 2011-12-18 15:18:05 -0800 + + * DNS updates. (Seth Hall) + + - Fixed some bugs with capturing data in the base DNS script. + + - Answers and TTLs are now vectors. + + - A warning that was being generated (dns_reply_seen_after_done) + from transaction ID reuse is fixed. + + * SSL updates. (Seth Hall) + + - Added is_orig fields to the SSL events and adapted script. + + - Added a field named last_alert to the SSL log. + + - The x509_certificate function has an is_orig field now instead + of is_server and its position in the argument list has moved. + + - A bit of reorganization and cleanup in the core analyzer. (Seth + Hall) + +2.0-beta-121 | 2011-12-18 15:10:15 -0800 + + * Enable warnings for malformed Broxygen xref roles. (Jon Siwek) + + * Fix Broxygen confusing scoped IDs at start of line as function + parameter. (Jon Siwek) + + * Allow Broxygen markup "##<" for more general use. (Jon Siwek) + +2.0-beta-116 | 2011-12-16 02:38:27 -0800 + + * Cleanup some misc Broxygen css/js stuff. (Jon Siwek) + + * Add search box to Broxygen docs. Fixes #726. (Jon Siwek) + + * Fixed major bug with cluster synchronization, which was not + working. (Seth Hall) + + * Fix missing action in notice policy for looking up GeoIP data. + (Jon Siwek) + + * Better persistent state configuration warning messages (fixes + #433). (Jon Siwek) + + * Renaming HTTP::SQL_Injection_Attack_Against to + HTTP::SQL_Injection_Victim. (Seth Hall). + + * Fixed DPD signatures for IRC. Fixes #311. (Seth Hall) + + * Removing Off_Port_Protocol_Found notice. (Seth Hall) + + * Teach Broxygen to more generally reference attribute values by name. (Jon Siwek) + + * SSH::Interesting_Hostname_Login cleanup. Fixes #664. (Seth Hall) + + * Fixed bug that was causing the malware hash registry script to + break. (Seth Hall) + + * Remove remnant of libmagic optionality. (Jon Siwek) + +2.0-beta-98 | 2011-12-07 08:12:08 -0800 + + * Adapting test-suite's diff-all so that it expands globs in both + current and baseline directory. Closes #677. (Robin Sommer) + +2.0-beta-97 | 2011-12-06 11:49:29 -0800 + + * Omit loading local-.bro scripts from base cluster framework. + Addresses #663 (Jon Siwek) + +2.0-beta-94 | 2011-12-03 15:57:19 -0800 + + * Adapting attribute serialization when talking to Broccoli. (Robin + Sommer) + +2.0-beta-92 | 2011-12-03 15:56:03 -0800 + + * Changes to Broxygen master script package index. (Jon Siwek) + + - Now only lists packages as those directories in the script hierarchy + that contain an __load__.bro file. + + - Script packages (dirs with a __load__.bro file), can now include + a README (in reST format) that will automatically be appended + under the link to a specific package in the master package + index. + +2.0-beta-88 | 2011-12-02 17:00:58 -0800 + + * Teach LogWriterAscii to use BRO_LOG_SUFFIX environemt variable. + Addresses #704. (Jon Siwek) + + * Fix double-free of DNS_Mgr_Request object. Addresses #661. + + * Add a remote_log_peer event which comes with an event_peer record + parameter. Addresses #493. (Jon Siwek) + + * Remove example redef of SMTP::entity_excerpt_len from local.bro. + Fixes error emitted when loading local.bro in bare mode. (Jon + Siwek) + + * Add missing doc targets to top Makefile; remove old doc/Makefile. + Fixes #705. (Jon Siwek) + + * Turn some globals into constants. Addresses #633. (Seth Hall) + + * Rearrange packet filter and DPD documentation. (Jon Siwek) + +2.0-beta-72 | 2011-11-30 20:16:09 -0800 + + * Fine-tuning the Sphinx layout to better match www. (Jon Siwek and + Robin Sommer) + +2.0-beta-69 | 2011-11-29 16:55:31 -0800 + + * Fixing ASCII logger to escape the unset-field place holder if + written out literally. (Robin Sommer) + +2.0-beta-68 | 2011-11-29 15:23:12 -0800 + + * Lots of documentation polishing. (Jon Siwek) + + * Teach Broxygen the ".. bro:see::" directive. (Jon Siwek) + + * Teach Broxygen :bro:see: role for referencing any identifier in + the Bro domain. (Jon Siwek) + + * Teach Broxygen to generate an index of Bro notices. (Jon Siwek) + + * Fix order of include directories. (Jon Siwek) + + * Catch if logged vectors do not contain only atomic types. + (Bernhard Amann) + 2.0-beta-47 | 2011-11-16 08:24:33 -0800 * Catch if logged sets do not contain only atomic types. (Bernhard diff --git a/Makefile b/Makefile index e0c2860873..277be4d017 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,7 @@ HAVE_MODULES=git submodule | grep -v cmake >/dev/null all: configured $(MAKE) -C $(BUILD) $@ -install: configured +install: configured all $(MAKE) -C $(BUILD) $@ install-aux: configured @@ -29,6 +29,18 @@ doc: configured docclean: configured $(MAKE) -C $(BUILD) $@ +restdoc: configured + $(MAKE) -C $(BUILD) $@ + +restclean: configured + $(MAKE) -C $(BUILD) $@ + +broxygen: configured + $(MAKE) -C $(BUILD) $@ + +broxygenclean: configured + $(MAKE) -C $(BUILD) $@ + dist: @rm -rf $(VERSION_FULL) $(VERSION_FULL).tgz @rm -rf $(VERSION_MIN) $(VERSION_MIN).tgz diff --git a/VERSION b/VERSION index deb6c1aac5..4fdf9b7419 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.0-beta-47 +2.0-beta-155 diff --git a/aux/binpac b/aux/binpac index 34d9043740..e94d92b01f 160000 --- a/aux/binpac +++ b/aux/binpac @@ -1 +1 @@ -Subproject commit 34d90437403e4129468f89acce0bd1a99813a2f4 +Subproject commit e94d92b01f327655fd2061157942b95ae75b5f0f diff --git a/aux/bro-aux b/aux/bro-aux index 7ea5837b4b..f6b92bf573 160000 --- a/aux/bro-aux +++ b/aux/bro-aux @@ -1 +1 @@ -Subproject commit 7ea5837b4ba8403731ca4a9875616c0ab501342f +Subproject commit f6b92bf5732c26e54eb4387efadc612663980389 diff --git a/aux/broccoli b/aux/broccoli index d281350dbc..c5cee3d574 160000 --- a/aux/broccoli +++ b/aux/broccoli @@ -1 +1 @@ -Subproject commit d281350dbcc19c24aa6b6d89a4edc08a5c74a790 +Subproject commit c5cee3d5746ed3d5c14348c1f264d19404caa761 diff --git a/aux/broctl b/aux/broctl index ed4d4ce1ad..d3d5934310 160000 --- a/aux/broctl +++ b/aux/broctl @@ -1 +1 @@ -Subproject commit ed4d4ce1add51f0e08e6e8d2f5f247c2cbb422da +Subproject commit d3d5934310a94452b1dddabb2e75f6c5c86b4860 diff --git a/aux/btest b/aux/btest index 7230a09a8c..38890e8514 160000 --- a/aux/btest +++ b/aux/btest @@ -1 +1 @@ -Subproject commit 7230a09a8c220d2117e491fdf293bf5c19819b65 +Subproject commit 38890e851416fa9fc827a1d36f06c4cb9f7d4e69 diff --git a/cmake b/cmake index f0f7958639..0c0a469768 160000 --- a/cmake +++ b/cmake @@ -1 +1 @@ -Subproject commit f0f7958639bb921985c1f58f1186da4b49b5d54d +Subproject commit 0c0a4697687df7f17c09391a1d0d95b25297a662 diff --git a/config.h.in b/config.h.in index 62aa34ef43..e3c9984373 100644 --- a/config.h.in +++ b/config.h.in @@ -17,9 +17,6 @@ /* We are on a Linux system */ #cmakedefine HAVE_LINUX -/* Define if you have the header file. */ -#cmakedefine HAVE_MAGIC_H - /* Define if you have the `mallinfo' function. */ #cmakedefine HAVE_MALLINFO diff --git a/doc/CMakeLists.txt b/doc/CMakeLists.txt index 7685230980..bdbb0e7b69 100644 --- a/doc/CMakeLists.txt +++ b/doc/CMakeLists.txt @@ -51,6 +51,8 @@ add_custom_target(broxygen COMMAND "${CMAKE_COMMAND}" -E create_symlink ${DOC_OUTPUT_DIR}/html ${CMAKE_BINARY_DIR}/html + # copy Broccoli API reference into output dir if it exists + COMMAND test -d ${CMAKE_BINARY_DIR}/aux/broccoli/doc/html && ( rm -rf ${CMAKE_BINARY_DIR}/html/broccoli-api && cp -r ${CMAKE_BINARY_DIR}/aux/broccoli/doc/html ${CMAKE_BINARY_DIR}/html/broccoli-api ) || true WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} COMMENT "[Sphinx] Generating HTML policy script docs" # SOURCES just adds stuff to IDE projects as a convenience @@ -58,16 +60,16 @@ add_custom_target(broxygen # The "sphinxclean" target removes just the Sphinx input/output directories # from the build directory. -add_custom_target(broxygen-clean +add_custom_target(broxygenclean COMMAND "${CMAKE_COMMAND}" -E remove_directory ${DOC_SOURCE_WORKDIR} COMMAND "${CMAKE_COMMAND}" -E remove_directory ${DOC_OUTPUT_DIR} VERBATIM) -add_dependencies(broxygen broxygen-clean restdoc) +add_dependencies(broxygen broxygenclean restdoc) add_custom_target(doc) add_custom_target(docclean) add_dependencies(doc broxygen) -add_dependencies(docclean broxygen-clean restclean) +add_dependencies(docclean broxygenclean restclean) diff --git a/doc/Makefile b/doc/Makefile deleted file mode 100644 index 2756093a27..0000000000 --- a/doc/Makefile +++ /dev/null @@ -1,7 +0,0 @@ - -all: - test -d html || mkdir html - for i in *.rst; do echo "$$i ..."; ./bin/rst2html.py $$i >html/`echo $$i | sed 's/rst$$/html/g'`; done - -clean: - rm -rf html diff --git a/doc/README b/doc/README index a9f2cdbc4e..57d569db84 100644 --- a/doc/README +++ b/doc/README @@ -15,8 +15,9 @@ which adds some reST directives and roles that aid in generating useful index entries and cross-references. Other extensions can be added in a similar fashion. -Either the ``make doc`` or ``make broxygen`` can be used to locally -render the reST files into HTML. Those targets depend on: +Either the ``make doc`` or ``make broxygen`` targets in the top-level +Makefile can be used to locally render the reST files into HTML. +Those targets depend on: * Python interpreter >= 2.5 * `Sphinx `_ >= 1.0.1 diff --git a/doc/_static/broxygen-extra.css b/doc/_static/broxygen-extra.css index dd56416783..ec240cec7b 100644 --- a/doc/_static/broxygen-extra.css +++ b/doc/_static/broxygen-extra.css @@ -1,3 +1,16 @@ -.highlight { - background-color: #ffffff; +h1 { + font-weight: bold; + font-size: 32px; + line-height:32px; + text-align: center; + padding-top: 3px; + margin-bottom: 30px; + font-family: Palatino,'Palatino Linotype',Georgia,serif;; + color: #000; + border-bottom: 0px; +} + +th.field-name +{ + white-space:nowrap; } diff --git a/doc/_static/default.css_t b/doc/_static/default.css_t deleted file mode 100644 index 34c2157b25..0000000000 --- a/doc/_static/default.css_t +++ /dev/null @@ -1,309 +0,0 @@ -/* - * default.css_t - * ~~~~~~~~~~~~~ - * - * Sphinx stylesheet -- default theme. - * - * :copyright: Copyright 2007-2011 by the Sphinx team, see AUTHORS. - * :license: BSD, see LICENSE for details. - * - */ - -@import url("basic.css"); - -/* -- page layout ----------------------------------------------------------- */ - -body { - font-family: {{ theme_bodyfont }}; - font-size: 100%; - background-color: {{ theme_footerbgcolor }}; - color: #000; - margin: 0; - padding: 0; -} - -div.document { - background-color: {{ theme_sidebarbgcolor }}; -} - -div.documentwrapper { - float: left; - width: 100%; -} - -div.bodywrapper { - margin: 0 0 0 {{ theme_sidebarwidth|toint }}px; -} - -div.body { - background-color: {{ theme_bgcolor }}; - color: {{ theme_textcolor }}; - padding: 0 20px 30px 20px; -} - -{%- if theme_rightsidebar|tobool %} -div.bodywrapper { - margin: 0 {{ theme_sidebarwidth|toint }}px 0 0; -} -{%- endif %} - -div.footer { - color: {{ theme_footertextcolor }}; - background-color: {{ theme_footerbgcolor }}; - width: 100%; - padding: 9px 0 9px 0; - text-align: center; - font-size: 75%; -} - -div.footer a { - color: {{ theme_footertextcolor }}; - text-decoration: underline; -} - -div.related { - background-color: {{ theme_relbarbgcolor }}; - line-height: 30px; - color: {{ theme_relbartextcolor }}; -} - -div.related a { - color: {{ theme_relbarlinkcolor }}; -} - -div.sphinxsidebar { - {%- if theme_stickysidebar|tobool %} - top: 30px; - bottom: 0; - margin: 0; - position: fixed; - overflow: auto; - height: auto; - {%- endif %} - {%- if theme_rightsidebar|tobool %} - float: right; - {%- if theme_stickysidebar|tobool %} - right: 0; - {%- endif %} - {%- endif %} -} - -{%- if theme_stickysidebar|tobool %} -/* this is nice, but it it leads to hidden headings when jumping - to an anchor */ -/* -div.related { - position: fixed; -} - -div.documentwrapper { - margin-top: 30px; -} -*/ -{%- endif %} - -div.sphinxsidebar h3 { - font-family: {{ theme_bodyfont }}; - color: {{ theme_sidebartextcolor }}; - font-size: 1.4em; - font-weight: normal; - margin: 0; - padding: 0; -} - -div.sphinxsidebar h3 a { - color: {{ theme_sidebartextcolor }}; -} - -div.sphinxsidebar h4 { - font-family: {{ theme_bodyfont }}; - color: {{ theme_sidebartextcolor }}; - font-size: 1.3em; - font-weight: normal; - margin: 5px 0 0 0; - padding: 0; -} - -div.sphinxsidebar p { - color: {{ theme_sidebartextcolor }}; -} - -div.sphinxsidebar p.topless { - margin: 5px 10px 10px 10px; -} - -div.sphinxsidebar ul { - margin: 10px; - padding: 0; - color: {{ theme_sidebartextcolor }}; -} - -div.sphinxsidebar a { - color: {{ theme_sidebarlinkcolor }}; -} - -div.sphinxsidebar input { - border: 1px solid {{ theme_sidebarlinkcolor }}; - font-family: sans-serif; - font-size: 1em; -} - -{% if theme_collapsiblesidebar|tobool %} -/* for collapsible sidebar */ -div#sidebarbutton { - background-color: {{ theme_sidebarbtncolor }}; -} -{% endif %} - -/* -- hyperlink styles ------------------------------------------------------ */ - -a { - color: {{ theme_linkcolor }}; - text-decoration: none; -} - -a:visited { - color: {{ theme_visitedlinkcolor }}; - text-decoration: none; -} - -{% if theme_externalrefs|tobool %} -a.external { - text-decoration: none; - border-bottom: 1px dashed {{ theme_linkcolor }}; -} - -a.external:hover { - text-decoration: none; - border-bottom: none; -} - -a.external:visited { - text-decoration: none; - border-bottom: 1px dashed {{ theme_visitedlinkcolor }}; -} -{% endif %} - -/* -- body styles ----------------------------------------------------------- */ - -div.body h1, -div.body h2, -div.body h3, -div.body h4, -div.body h5, -div.body h6 { - font-family: {{ theme_bodyfont }}; - background-color: #ffffff; - font-weight: normal; - color: {{ theme_headtextcolor }}; - border-bottom: 1px solid #aaa; - margin: 20px -20px 10px -20px; - padding: 3px 0 3px 10px; -} - -div.body h1 { - font-family: {{ theme_headfont }}; - text-align: center; - border-bottom: none; -} - -div.body h1 { margin-top: 0; font-size: 200%; } -div.body h2 { font-size: 160%; } -div.body h3 { font-size: 140%; } -div.body h4 { font-size: 120%; } -div.body h5 { font-size: 110%; } -div.body h6 { font-size: 100%; } - -a.headerlink { - color: {{ theme_headlinkcolor }}; - font-size: 0.8em; - padding: 0 4px 0 4px; - text-decoration: none; -} - -a.headerlink:hover { - background-color: {{ theme_headlinkcolor }}; - color: white; -} - -div.admonition p.admonition-title + p { - display: inline; -} - -div.admonition p { - margin-bottom: 5px; -} - -div.admonition pre { - margin-bottom: 5px; -} - -div.admonition ul, div.admonition ol { - margin-bottom: 5px; -} - -div.note { - background-color: #eee; - border: 1px solid #ccc; -} - -div.seealso { - background-color: #ffc; - border: 1px solid #ff6; -} - -div.warning { - background-color: #ffe4e4; - border: 1px solid #f66; -} - -p.admonition-title { - display: inline; -} - -p.admonition-title:after { - content: ":"; -} - -pre { - padding: 5px; - background-color: {{ theme_codebgcolor }}; - color: {{ theme_codetextcolor }}; - line-height: 120%; - border: 1px solid #ac9; - border-left: none; - border-right: none; -} - -tt { - background-color: #ecf0f3; - padding: 0 1px 0 1px; - font-size: 0.95em; -} - -th { - background-color: #ede; -} - -.warning tt { - background: #efc2c2; -} - -.note tt { - background: #d6d6d6; -} - -.viewcode-back { - font-family: {{ theme_bodyfont }}; -} - -div.viewcode-block:target { - background-color: #f4debf; - border-top: 1px solid #ac9; - border-bottom: 1px solid #ac9; -} - -th.field-name -{ - white-space:nowrap; -} diff --git a/doc/_static/logo-bro.png b/doc/_static/logo-bro.png new file mode 100644 index 0000000000..96cc5d443c Binary files /dev/null and b/doc/_static/logo-bro.png differ diff --git a/doc/_static/showhide.js b/doc/_static/showhide.js deleted file mode 100644 index d6a8923143..0000000000 --- a/doc/_static/showhide.js +++ /dev/null @@ -1,64 +0,0 @@ -// make literal blocks corresponding to identifier initial values -// hidden by default -$(document).ready(function() { - - var showText='(Show Value)'; - var hideText='(Hide Value)'; - - var is_visible = false; - - // select field-list tables that come before a literal block - tables = $('.highlight-python').prev('table.docutils.field-list'); - - tables.find('th.field-name').filter(function(index) { - return $(this).html() == "Default :"; - }).next().append(''+showText+''); - - // hide all literal blocks that follow a field-list table - tables.next('.highlight-python').hide(); - - // register handler for clicking a "toggle" link - $('a.toggleLink').click(function() { - is_visible = !is_visible; - - $(this).html( (!is_visible) ? showText : hideText); - - // the link is inside a
and the next - // literal block after the table is the literal block that we want - // to show/hide - $(this).parent().parent().parent().parent().next('.highlight-python').slideToggle('fast'); - - // override default link behavior - return false; - }); -}); - -// make "Private Interface" sections hidden by default -$(document).ready(function() { - - var showText='Show Private Interface (for internal use)'; - var hideText='Hide Private Interface'; - - var is_visible = false; - - // insert show/hide links - $('#private-interface').children(":first-child").after(''+showText+''); - - // wrap all sub-sections in a new div that can be hidden/shown - $('#private-interface').children(".section").wrapAll('
'); - - // hide the given class - $('.private').hide(); - - // register handler for clicking a "toggle" link - $('a.privateToggle').click(function() { - is_visible = !is_visible; - - $(this).html( (!is_visible) ? showText : hideText); - - $('.private').slideToggle('fast'); - - // override default link behavior - return false; - }); -}); diff --git a/doc/_templates/layout.html b/doc/_templates/layout.html index 518718c55c..a4775f5870 100644 --- a/doc/_templates/layout.html +++ b/doc/_templates/layout.html @@ -2,9 +2,110 @@ {% block extrahead %} + {% endblock %} +{% block header %} + +{% endblock %} + {% block relbar2 %}{% endblock %} +{% block relbar1 %}{% endblock %} + +{% block content %} + +
+
+ +
+ +
+ {{ relbar() }} +
+ +
+ {% block body %} + {% endblock %} +
+
+ + +
+ +
+ +
+
+ + +
+

+ Table of Contents +

+

+

    {{toc}}
+

+
+ + {% if next %} +
+

+ Next Page +

+

+ {{ next.title }} +

+
+ {% endif %} + + {% if prev %} +
+

+ Previous Page +

+

+ {{ prev.title }} +

+
+ {% endif %} + + {%- if pagename != "search" %} + + + {%- endif %} + +
+
+ +
+
+
+ + Copyright {{ copyright }}. + Last updated on {{ last_updated }}. + Created using Sphinx {{ sphinx_version }}. + +
+
+
+
+ + +{% endblock %} + +{% block footer %} + +{% endblock %} diff --git a/doc/bin/group_index_generator.py b/doc/bin/group_index_generator.py index e9e17f3ca9..720081e8b5 100755 --- a/doc/bin/group_index_generator.py +++ b/doc/bin/group_index_generator.py @@ -49,6 +49,7 @@ with open(group_list, 'r') as f_group_list: if not os.path.exists(os.path.dirname(group_file)): os.makedirs(os.path.dirname(group_file)) with open(group_file, 'w') as f_group_file: + f_group_file.write(":orphan:\n\n") title = "Package Index: %s\n" % os.path.dirname(group) f_group_file.write(title); for n in range(len(title)): diff --git a/doc/bin/rst2html.py b/doc/bin/rst2html.py deleted file mode 100755 index 79c835d6c4..0000000000 --- a/doc/bin/rst2html.py +++ /dev/null @@ -1,62 +0,0 @@ -#!/usr/bin/env python -# -# Derived from docutils standard rst2html.py. -# -# $Id: rst2html.py 4564 2006-05-21 20:44:42Z wiemann $ -# Author: David Goodger -# Copyright: This module has been placed in the public domain. -# -# -# Extension: we add to dummy directorives "code" and "console" to be -# compatible with Bro's web site setup. - -try: - import locale - locale.setlocale(locale.LC_ALL, '') -except: - pass - -import textwrap - -from docutils.core import publish_cmdline, default_description - -from docutils import nodes -from docutils.parsers.rst import directives, Directive -from docutils.parsers.rst.directives.body import LineBlock - -class Literal(Directive): - #max_line_length = 68 - max_line_length = 0 - - required_arguments = 0 - optional_arguments = 1 - final_argument_whitespace = True - has_content = True - - def wrapped_content(self): - content = [] - - if Literal.max_line_length: - for line in self.content: - content += textwrap.wrap(line, Literal.max_line_length, subsequent_indent=" ") - else: - content = self.content - - return u'\n'.join(content) - - def run(self): - self.assert_has_content() - content = self.wrapped_content() - literal = nodes.literal_block(content, content) - return [literal] - -directives.register_directive('code', Literal) -directives.register_directive('console', Literal) - -description = ('Generates (X)HTML documents from standalone reStructuredText ' - 'sources. ' + default_description) - -publish_cmdline(writer_name='html', description=description) - - - diff --git a/doc/components/binpac/README.rst b/doc/components/binpac/README.rst new file mode 120000 index 0000000000..4eb90ef658 --- /dev/null +++ b/doc/components/binpac/README.rst @@ -0,0 +1 @@ +../../../aux/binpac/README \ No newline at end of file diff --git a/doc/components/bro-aux/README.rst b/doc/components/bro-aux/README.rst new file mode 120000 index 0000000000..628879525d --- /dev/null +++ b/doc/components/bro-aux/README.rst @@ -0,0 +1 @@ +../../../aux/bro-aux/README \ No newline at end of file diff --git a/doc/components/broccoli-ruby/README.rst b/doc/components/broccoli-ruby/README.rst new file mode 120000 index 0000000000..da71663099 --- /dev/null +++ b/doc/components/broccoli-ruby/README.rst @@ -0,0 +1 @@ +../../../aux/broccoli/bindings/broccoli-ruby/README \ No newline at end of file diff --git a/doc/components/broccoli/broccoli-manual.rst b/doc/components/broccoli/broccoli-manual.rst new file mode 120000 index 0000000000..bd5e8d711f --- /dev/null +++ b/doc/components/broccoli/broccoli-manual.rst @@ -0,0 +1 @@ +../../../aux/broccoli/doc/broccoli-manual.rst \ No newline at end of file diff --git a/doc/conf.py.in b/doc/conf.py.in index 72337b930f..8959c0b2c9 100644 --- a/doc/conf.py.in +++ b/doc/conf.py.in @@ -90,44 +90,20 @@ pygments_style = 'sphinx' # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. -html_theme = 'default' +html_theme = 'basic' html_last_updated_fmt = '%B %d, %Y' # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. -html_theme_options = { -"rightsidebar": "true", -"stickysidebar": "true", -"externalrefs": "false", -"footerbgcolor": "#333", -"footertextcolor": "#ddd", -"sidebarbgcolor": "#ffffff", -#"sidebarbtncolor": "", -"sidebartextcolor": "#333", -"sidebarlinkcolor": "#2a85a7", -"relbarbgcolor": "#ffffff", -"relbartextcolor": "#333", -"relbarlinkcolor": "#2a85a7", -"bgcolor": "#ffffff", -"textcolor": "#333", -"linkcolor": "#2a85a7", -"visitedlinkcolor": "#2a85a7", -"headbgcolor": "#f0f0f0", -"headtextcolor": "#000", -"headlinkcolor": "#2a85a7", -"codebgcolor": "#FFFAE2", -#"codetextcolor": "", -"bodyfont": "Arial, Helvetica, sans-serif", -"headfont": "Palatino,'Palatino Linotype',Georgia,serif", -} +html_theme_options = { } # Add any paths that contain custom themes here, relative to this directory. #html_theme_path = [] # The name for this set of Sphinx documents. If None, it defaults to -# " v documentation". +# " v Documentation". #html_title = None # A shorter title for the navigation bar. Default is the same as html_title. diff --git a/doc/ext/bro.py b/doc/ext/bro.py index a4f0142ce3..9bdd86bd9a 100644 --- a/doc/ext/bro.py +++ b/doc/ext/bro.py @@ -257,6 +257,9 @@ class BroDomain(Domain): objects[objtype, target], objtype + '-' + target, contnode, target + ' ' + objtype) + else: + self.env.warn(fromdocname, + 'unknown target for ":bro:%s:`%s`"' % (typ, target)) def get_objects(self): for (typ, name), docname in self.data['objects'].iteritems(): diff --git a/doc/index.rst b/doc/index.rst index ba3df81e7d..4b362db62e 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -1,8 +1,12 @@ .. Bro documentation master file +================= Bro Documentation ================= +Guides +------ + .. toctree:: :maxdepth: 1 @@ -37,7 +41,6 @@ Script Reference .. toctree:: :maxdepth: 1 - scripts/common scripts/builtins scripts/bifs scripts/packages @@ -46,16 +49,29 @@ Script Reference Other Bro Components -------------------- +The following are snapshots of documentation for components that come +with this version of Bro (|version|). Since they can also be used +independently, see the `download page +`_ for documentation of any +current, independent component releases. + .. toctree:: :maxdepth: 1 - components/btest/README - components/broccoli/README - components/broccoli-python/README - components/broctl/README - components/capstats/README - components/pysubnettree/README - components/trace-summary/README + BinPAC - A protocol parser generator + Broccoli - The Bro Client Communication Library (README) + Broccoli - User Manual + Broccoli Python Bindings + Broccoli Ruby Bindings + BroControl - Interactive Bro management shell + Bro-Aux - Small auxiliary tools for Bro + BTest - A unit testing framework + Capstats - Command-line packet statistic tool + PySubnetTree - Python module for CIDR lookups + trace-summary - Script for generating break-downs of network traffic + +The `Broccoli API Reference `_ may also be of +interest. Other Indices and References ---------------------------- diff --git a/doc/notice.rst b/doc/notice.rst index b1ffdacb75..bab44ab9e4 100644 --- a/doc/notice.rst +++ b/doc/notice.rst @@ -29,17 +29,18 @@ definitions of what constitutes an attack or even a compromise differ quite a bit between environments, and activity deemed malicious at one site might be fully acceptable at another. -Whenever one of Bro's analysis scripts sees something potentially interesting -it flags the situation by calling the ``NOTICE`` function and giving it a -single ``Notice::Info`` record. A Notice has a ``Notice::Type``, which -reflects the kind of activity that has been seen, and it is usually also -augmented with further context about the situation. +Whenever one of Bro's analysis scripts sees something potentially +interesting it flags the situation by calling the :bro:see:`NOTICE` +function and giving it a single :bro:see:`Notice::Info` record. A Notice +has a :bro:see:`Notice::Type`, which reflects the kind of activity that +has been seen, and it is usually also augmented with further context +about the situation. More information about raising notices can be found in the `Raising Notices`_ section. Once a notice is raised, it can have any number of actions applied to it by -the ``Notice::policy`` set which is described in the `Notice Policy`_ +the :bro:see:`Notice::policy` set which is described in the `Notice Policy`_ section below. Such actions can be to send a mail to the configured address(es) or to simply ignore the notice. Currently, the following actions are defined: @@ -52,20 +53,20 @@ are defined: - Description * - Notice::ACTION_LOG - - Write the notice to the ``Notice::LOG`` logging stream. + - Write the notice to the :bro:see:`Notice::LOG` logging stream. * - Notice::ACTION_ALARM - - Log into the ``Notice::ALARM_LOG`` stream which will rotate + - Log into the :bro:see:`Notice::ALARM_LOG` stream which will rotate hourly and email the contents to the email address or addresses - defined in the ``Notice::mail_dest`` variable. + defined in the :bro:see:`Notice::mail_dest` variable. * - Notice::ACTION_EMAIL - Send the notice in an email to the email address or addresses given in - the ``Notice::mail_dest`` variable. + the :bro:see:`Notice::mail_dest` variable. * - Notice::ACTION_PAGE - Send an email to the email address or addresses given in the - ``Notice::mail_page_dest`` variable. + :bro:see:`Notice::mail_page_dest` variable. * - Notice::ACTION_NO_SUPPRESS - This action will disable the built in notice suppression for the @@ -82,15 +83,17 @@ Processing Notices Notice Policy ************* -The predefined set ``Notice::policy`` provides the mechanism for applying -actions and other behavior modifications to notices. Each entry of -``Notice::policy`` is a record of the type ``Notice::PolicyItem`` which -defines a condition to be matched against all raised notices and one or more -of a variety of behavior modifiers. The notice policy is defined by adding any -number of ``Notice::PolicyItem`` records to the ``Notice::policy`` set. +The predefined set :bro:see:`Notice::policy` provides the mechanism for +applying actions and other behavior modifications to notices. Each entry +of :bro:see:`Notice::policy` is a record of the type +:bro:see:`Notice::PolicyItem` which defines a condition to be matched +against all raised notices and one or more of a variety of behavior +modifiers. The notice policy is defined by adding any number of +:bro:see:`Notice::PolicyItem` records to the :bro:see:`Notice::policy` +set. Here's a simple example which tells Bro to send an email for all notices of -type ``SSH::Login`` if the server is 10.0.0.1: +type :bro:see:`SSH::Login` if the server is 10.0.0.1: .. code:: bro @@ -113,11 +116,11 @@ flexibility due to having access to Bro's full programming language. Predicate Field ^^^^^^^^^^^^^^^ -The ``Notice::PolicyItem`` record type has a field name ``$pred`` which -defines the entry's condition in the form of a predicate written as a Bro -function. The function is passed the notice as a ``Notice::Info`` record and -it returns a boolean value indicating if the entry is applicable to that -particular notice. +The :bro:see:`Notice::PolicyItem` record type has a field name ``$pred`` +which defines the entry's condition in the form of a predicate written +as a Bro function. The function is passed the notice as a +:bro:see:`Notice::Info` record and it returns a boolean value indicating +if the entry is applicable to that particular notice. .. note:: @@ -125,14 +128,14 @@ particular notice. (``T``) since an implicit false (``F``) value would never be used. Bro evaluates the predicates of each entry in the order defined by the -``$priority`` field in ``Notice::PolicyItem`` records. The valid values are -0-10 with 10 being earliest evaluated. If ``$priority`` is omitted, the -default priority is 5. +``$priority`` field in :bro:see:`Notice::PolicyItem` records. The valid +values are 0-10 with 10 being earliest evaluated. If ``$priority`` is +omitted, the default priority is 5. Behavior Modification Fields ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -There are a set of fields in the ``Notice::PolicyItem`` record type that +There are a set of fields in the :bro:see:`Notice::PolicyItem` record type that indicate ways that either the notice or notice processing should be modified if the predicate field (``$pred``) evaluated to true (``T``). Those fields are explained in more detail in the following table. @@ -146,8 +149,8 @@ explained in more detail in the following table. - Example * - ``$action=`` - - Each Notice::PolicyItem can have a single action applied to the notice - with this field. + - Each :bro:see:`Notice::PolicyItem` can have a single action + applied to the notice with this field. - ``$action = Notice::ACTION_EMAIL`` * - ``$suppress_for=`` @@ -162,9 +165,9 @@ explained in more detail in the following table. - This field can be used for modification of the notice policy evaluation. To stop processing of notice policy items before evaluating all of them, set this field to ``T`` and make the ``$pred`` - field return ``T``. ``Notice::PolicyItem`` records defined at a higher - priority as defined by the ``$priority`` field will still be evaluated - but those at a lower priority won't. + field return ``T``. :bro:see:`Notice::PolicyItem` records defined at + a higher priority as defined by the ``$priority`` field will still be + evaluated but those at a lower priority won't. - ``$halt = T`` @@ -186,11 +189,11 @@ Notice Policy Shortcuts Although the notice framework provides a great deal of flexibility and configurability there are many times that the full expressiveness isn't needed and actually becomes a hindrance to achieving results. The framework provides -a default ``Notice::policy`` suite as a way of giving users the +a default :bro:see:`Notice::policy` suite as a way of giving users the shortcuts to easily apply many common actions to notices. These are implemented as sets and tables indexed with a -``Notice::Type`` enum value. The following table shows and describes +:bro:see:`Notice::Type` enum value. The following table shows and describes all of the variables available for shortcut configuration of the notice framework. @@ -201,40 +204,44 @@ framework. * - Variable name - Description - * - Notice::ignored_types - - Adding a ``Notice::Type`` to this set results in the notice + * - :bro:see:`Notice::ignored_types` + - Adding a :bro:see:`Notice::Type` to this set results in the notice being ignored. It won't have any other action applied to it, not even - ``Notice::ACTION_LOG``. + :bro:see:`Notice::ACTION_LOG`. - * - Notice::emailed_types - - Adding a ``Notice::Type`` to this set results in - ``Notice::ACTION_EMAIL`` being applied to the notices of that type. + * - :bro:see:`Notice::emailed_types` + - Adding a :bro:see:`Notice::Type` to this set results in + :bro:see:`Notice::ACTION_EMAIL` being applied to the notices of + that type. - * - Notice::alarmed_types - - Adding a Notice::Type to this set results in - ``Notice::ACTION_ALARM`` being applied to the notices of that type. + * - :bro:see:`Notice::alarmed_types` + - Adding a :bro:see:`Notice::Type` to this set results in + :bro:see:`Notice::ACTION_ALARM` being applied to the notices of + that type. - * - Notice::not_suppressed_types - - Adding a ``Notice::Type`` to this set results in that notice no longer - undergoing the normal notice suppression that would take place. Be - careful when using this in production it could result in a dramatic - increase in the number of notices being processed. + * - :bro:see:`Notice::not_suppressed_types` + - Adding a :bro:see:`Notice::Type` to this set results in that notice + no longer undergoes the normal notice suppression that would + take place. Be careful when using this in production it could + result in a dramatic increase in the number of notices being + processed. - * - Notice::type_suppression_intervals - - This is a table indexed on ``Notice::Type`` and yielding an interval. - It can be used as an easy way to extend the default suppression - interval for an entire ``Notice::Type`` without having to create a - whole ``Notice::policy`` entry and setting the ``$suppress_for`` - field. + * - :bro:see:`Notice::type_suppression_intervals` + - This is a table indexed on :bro:see:`Notice::Type` and yielding an + interval. It can be used as an easy way to extend the default + suppression interval for an entire :bro:see:`Notice::Type` + without having to create a whole :bro:see:`Notice::policy` entry + and setting the ``$suppress_for`` field. Raising Notices --------------- -A script should raise a notice for any occurrence that a user may want to be -notified about or take action on. For example, whenever the base SSH analysis -scripts sees an SSH session where it is heuristically guessed to be a -successful login, it raises a Notice of the type ``SSH::Login``. The code in -the base SSH analysis script looks like this: +A script should raise a notice for any occurrence that a user may want +to be notified about or take action on. For example, whenever the base +SSH analysis scripts sees an SSH session where it is heuristically +guessed to be a successful login, it raises a Notice of the type +:bro:see:`SSH::Login`. The code in the base SSH analysis script looks +like this: .. code:: bro @@ -242,10 +249,10 @@ the base SSH analysis script looks like this: $msg="Heuristically detected successful SSH login.", $conn=c]); -``NOTICE`` is a normal function in the global namespace which wraps a function -within the ``Notice`` namespace. It takes a single argument of the -``Notice::Info`` record type. The most common fields used when raising notices -are described in the following table: +:bro:see:`NOTICE` is a normal function in the global namespace which +wraps a function within the ``Notice`` namespace. It takes a single +argument of the :bro:see:`Notice::Info` record type. The most common +fields used when raising notices are described in the following table: .. list-table:: :widths: 32 40 @@ -295,9 +302,10 @@ are described in the following table: * - ``$suppress_for`` - This field can be set if there is a natural suppression interval for - the notice that may be different than the default value. The value set - to this field can also be modified by a user's ``Notice::policy`` so - the value is not set permanently and unchangeably. + the notice that may be different than the default value. The + value set to this field can also be modified by a user's + :bro:see:`Notice::policy` so the value is not set permanently + and unchangeably. When writing Bro scripts which raise notices, some thought should be given to what the notice represents and what data should be provided to give a consumer @@ -325,7 +333,7 @@ The notice framework supports suppression for notices if the author of the script that is generating the notice has indicated to the notice framework how to identify notices that are intrinsically the same. Identification of these "intrinsically duplicate" notices is implemented with an optional field in -``Notice::Info`` records named ``$identifier`` which is a simple string. +:bro:see:`Notice::Info` records named ``$identifier`` which is a simple string. If the ``$identifier`` and ``$type`` fields are the same for two notices, the notice framework actually considers them to be the same thing and can use that information to suppress duplicates for a configurable period of time. @@ -337,12 +345,13 @@ information to suppress duplicates for a configurable period of time. could be completely legitimate usage if no notices could ever be considered to be duplicates. -The ``$identifier`` field is typically comprised of several pieces of data -related to the notice that when combined represent a unique instance of that -notice. Here is an example of the script -``policy/protocols/ssl/validate-certs.bro`` raising a notice for session -negotiations where the certificate or certificate chain did not validate -successfully against the available certificate authority certificates. +The ``$identifier`` field is typically comprised of several pieces of +data related to the notice that when combined represent a unique +instance of that notice. Here is an example of the script +:doc:`scripts/policy/protocols/ssl/validate-certs` raising a notice +for session negotiations where the certificate or certificate chain did +not validate successfully against the available certificate authority +certificates. .. code:: bro @@ -369,7 +378,7 @@ it's assumed that the script author who is raising the notice understands the full problem set and edge cases of the notice which may not be readily apparent to users. If users don't want the suppression to take place or simply want a different interval, they can always modify it with the -``Notice::policy``. +:bro:see:`Notice::policy`. Extending Notice Framework diff --git a/doc/scripts/CMakeLists.txt b/doc/scripts/CMakeLists.txt index c14eab6d25..33d473b005 100644 --- a/doc/scripts/CMakeLists.txt +++ b/doc/scripts/CMakeLists.txt @@ -73,12 +73,14 @@ macro(REST_TARGET srcDir broInput) elseif (${extension} STREQUAL ".bif.bro") set(group bifs) elseif (relDstDir) - set(pkgIndex ${relDstDir}/index) - set(group ${pkgIndex}) + set(group ${relDstDir}/index) # add package index to master package list if not already in it - list(FIND MASTER_PKG_LIST ${pkgIndex} _found) + # and if a __load__.bro exists in the original script directory + list(FIND MASTER_PKG_LIST ${relDstDir} _found) if (_found EQUAL -1) - list(APPEND MASTER_PKG_LIST ${pkgIndex}) + if (EXISTS ${CMAKE_SOURCE_DIR}/scripts/${relDstDir}/__load__.bro) + list(APPEND MASTER_PKG_LIST ${relDstDir}) + endif () endif () else () set(group "") @@ -137,11 +139,15 @@ file(WRITE ${MASTER_POLICY_INDEX} "${MASTER_POLICY_INDEX_TEXT}") # policy/packages.rst file set(MASTER_PKG_INDEX_TEXT "") foreach (pkg ${MASTER_PKG_LIST}) - # strip of the trailing /index for the link name - get_filename_component(lnktxt ${pkg} PATH) - # pretty-up the link name by removing common scripts/ prefix - string(REPLACE "scripts/" "" lnktxt "${lnktxt}") - set(MASTER_PKG_INDEX_TEXT "${MASTER_PKG_INDEX_TEXT}\n ${lnktxt} <${pkg}>") + set(MASTER_PKG_INDEX_TEXT + "${MASTER_PKG_INDEX_TEXT}\n:doc:`${pkg} <${pkg}/index>`\n") + if (EXISTS ${CMAKE_SOURCE_DIR}/scripts/${pkg}/README) + file(STRINGS ${CMAKE_SOURCE_DIR}/scripts/${pkg}/README pkgreadme) + foreach (line ${pkgreadme}) + set(MASTER_PKG_INDEX_TEXT "${MASTER_PKG_INDEX_TEXT}\n ${line}") + endforeach () + set(MASTER_PKG_INDEX_TEXT "${MASTER_PKG_INDEX_TEXT}\n") + endif () endforeach () file(WRITE ${MASTER_PACKAGE_INDEX} "${MASTER_PKG_INDEX_TEXT}") diff --git a/doc/scripts/README b/doc/scripts/README index b3e44914f4..a15812609c 100644 --- a/doc/scripts/README +++ b/doc/scripts/README @@ -1,6 +1,6 @@ This directory contains scripts and templates that can be used to automate the generation of Bro script documentation. Several build targets are defined -by CMake: +by CMake and available in the top-level Makefile: ``restdoc`` diff --git a/doc/scripts/common.rst b/doc/scripts/common.rst deleted file mode 100644 index 6105585b2c..0000000000 --- a/doc/scripts/common.rst +++ /dev/null @@ -1,19 +0,0 @@ -Common Documentation -==================== - -.. _common_port_analysis_doc: - -Port Analysis -------------- - -TODO: add some stuff here - -.. _common_packet_filter_doc: - -Packet Filter -------------- - -TODO: add some stuff here - -.. note:: Filters are only relevant when dynamic protocol detection (DPD) - is explicitly turned off (Bro release 1.6 enabled DPD by default). diff --git a/doc/scripts/example.bro b/doc/scripts/example.bro index d2d0ec6879..9f6f656ee1 100644 --- a/doc/scripts/example.bro +++ b/doc/scripts/example.bro @@ -1,5 +1,5 @@ -##! This is an example script that demonstrates how to document. Comments -##! of the form ``##!`` are for the script summary. The contents of +##! This is an example script that demonstrates documentation features. +##! Comments of the form ``##!`` are for the script summary. The contents of ##! these comments are transferred directly into the auto-generated ##! `reStructuredText `_ ##! (reST) document's summary section. @@ -22,8 +22,8 @@ # field comments, it's necessary to disambiguate the field with # which a comment associates: e.g. "##<" can be used on the same line # as a field to signify the comment relates to it and not the -# following field. "##<" is not meant for general use, just -# record/enum fields. +# following field. "##<" can also be used more generally in any +# variable declarations to associate with the last-declared identifier. # # Generally, the auto-doc comments (##) are associated with the # next declaration/identifier found in the script, but the doc framework @@ -151,7 +151,7 @@ export { const an_option: set[addr, addr, string] &redef; # default initialization will be self-documenting - const option_with_init = 0.01 secs &redef; + const option_with_init = 0.01 secs &redef; ##< More docs can be added here. ############## state variables ############ # right now, I'm defining this as any global @@ -183,6 +183,7 @@ export { ## Summarize "an_event" here. ## Give more details about "an_event" here. + ## Example::an_event should not be confused as a parameter. ## name: describe the argument here global an_event: event(name: string); diff --git a/doc/scripts/index.rst b/doc/scripts/index.rst index 5de203afe7..bf0fa25f10 100644 --- a/doc/scripts/index.rst +++ b/doc/scripts/index.rst @@ -1,7 +1,7 @@ .. This is a stub doc to which broxygen appends during the build process -Index of All Bro Scripts -======================== +Index of All Individual Bro Scripts +=================================== .. toctree:: :maxdepth: 1 diff --git a/doc/scripts/packages.rst b/doc/scripts/packages.rst index 47e974b0c8..56909ffbc6 100644 --- a/doc/scripts/packages.rst +++ b/doc/scripts/packages.rst @@ -10,8 +10,3 @@ script, it supports being loaded in mass as a whole directory for convenience. Packages/scripts in the ``base/`` directory are all loaded by default, while ones in ``policy/`` provide functionality and customization options that are more appropriate for users to decide whether they'd like to load it or not. - -.. toctree:: - :maxdepth: 1 - - diff --git a/scripts/base/frameworks/cluster/__load__.bro b/scripts/base/frameworks/cluster/__load__.bro index 3334164866..bccb35dbb1 100644 --- a/scripts/base/frameworks/cluster/__load__.bro +++ b/scripts/base/frameworks/cluster/__load__.bro @@ -28,17 +28,14 @@ redef Communication::listen_port = Cluster::nodes[Cluster::node]$p; @if ( Cluster::local_node_type() == Cluster::MANAGER ) @load ./nodes/manager -@load site/local-manager @endif @if ( Cluster::local_node_type() == Cluster::PROXY ) @load ./nodes/proxy -@load site/local-proxy @endif @if ( Cluster::local_node_type() == Cluster::WORKER ) @load ./nodes/worker -@load site/local-worker @endif @endif diff --git a/scripts/base/frameworks/cluster/setup-connections.bro b/scripts/base/frameworks/cluster/setup-connections.bro index 059b984d61..4e91036c55 100644 --- a/scripts/base/frameworks/cluster/setup-connections.bro +++ b/scripts/base/frameworks/cluster/setup-connections.bro @@ -41,7 +41,7 @@ event bro_init() &priority=9 { if ( n$node_type == WORKER && n$proxy == node ) Communication::nodes[i] = - [$host=n$ip, $connect=F, $class=i, $events=worker2proxy_events]; + [$host=n$ip, $connect=F, $class=i, $sync=T, $auth=T, $events=worker2proxy_events]; # accepts connections from the previous one. # (This is not ideal for setups with many proxies) diff --git a/scripts/base/frameworks/communication/main.bro b/scripts/base/frameworks/communication/main.bro index 569ba140a9..01c608c8db 100644 --- a/scripts/base/frameworks/communication/main.bro +++ b/scripts/base/frameworks/communication/main.bro @@ -130,6 +130,13 @@ event remote_log(level: count, src: count, msg: string) do_script_log_common(level, src, msg); } +# This is a core generated event. +event remote_log_peer(p: event_peer, level: count, src: count, msg: string) + { + local rmsg = fmt("[#%d/%s:%d] %s", p$id, p$host, p$p, msg); + do_script_log_common(level, src, rmsg); + } + function do_script_log(p: event_peer, msg: string) { do_script_log_common(REMOTE_LOG_INFO, REMOTE_SRC_SCRIPT, msg); diff --git a/scripts/base/frameworks/dpd/dpd.sig b/scripts/base/frameworks/dpd/dpd.sig index 8e07095b41..adda0ce54e 100644 --- a/scripts/base/frameworks/dpd/dpd.sig +++ b/scripts/base/frameworks/dpd/dpd.sig @@ -80,15 +80,15 @@ signature irc_server_reply { tcp-state responder } -signature irc_sig3 { +signature irc_server_to_server1 { ip-proto == tcp - payload /(.*\x0a)*(\x20)*[Ss][Ee][Rr][Vv][Ee][Rr](\x20)+.+\x0a/ + payload /(|.*[\r\n]) *[Ss][Ee][Rr][Vv][Ee][Rr] +[^ ]+ +[0-9]+ +:.+[\r\n]/ } -signature irc_sig4 { +signature irc_server_to_server2 { ip-proto == tcp - payload /(.*\x0a)*(\x20)*[Ss][Ee][Rr][Vv][Ee][Rr](\x20)+.+\x0a/ - requires-reverse-signature irc_sig3 + payload /(|.*[\r\n]) *[Ss][Ee][Rr][Vv][Ee][Rr] +[^ ]+ +[0-9]+ +:.+[\r\n]/ + requires-reverse-signature irc_server_to_server1 enable "irc" } diff --git a/scripts/base/frameworks/logging/writers/ascii.bro b/scripts/base/frameworks/logging/writers/ascii.bro index 5c04fdd3d9..c285512dd5 100644 --- a/scripts/base/frameworks/logging/writers/ascii.bro +++ b/scripts/base/frameworks/logging/writers/ascii.bro @@ -19,8 +19,9 @@ export { ## Separator between set elements. const set_separator = "," &redef; - ## String to use for empty fields. - const empty_field = "-" &redef; + ## String to use for empty fields. This should be different from + ## *unset_field* to make the output non-ambigious. + const empty_field = "(empty)" &redef; ## String to use for an unset &optional field. const unset_field = "-" &redef; diff --git a/scripts/base/frameworks/notice/actions/add-geodata.bro b/scripts/base/frameworks/notice/actions/add-geodata.bro index bc4021abea..9f6909595c 100644 --- a/scripts/base/frameworks/notice/actions/add-geodata.bro +++ b/scripts/base/frameworks/notice/actions/add-geodata.bro @@ -31,6 +31,7 @@ export { ## Add a helper to the notice policy for looking up GeoIP data. redef Notice::policy += { [$pred(n: Notice::Info) = { return (n$note in Notice::lookup_location_types); }, + $action = ACTION_ADD_GEODATA, $priority = 10], }; } diff --git a/scripts/base/frameworks/notice/actions/pp-alarms.bro b/scripts/base/frameworks/notice/actions/pp-alarms.bro index 1284d7885f..609f6a6bf1 100644 --- a/scripts/base/frameworks/notice/actions/pp-alarms.bro +++ b/scripts/base/frameworks/notice/actions/pp-alarms.bro @@ -10,18 +10,22 @@ module Notice; export { ## Activate pretty-printed alarm summaries. const pretty_print_alarms = T &redef; - + ## Address to send the pretty-printed reports to. Default if not set is ## :bro:id:`Notice::mail_dest`. const mail_dest_pretty_printed = "" &redef; - - ## If an address from one of these networks is reported, we mark + + ## If an address from one of these networks is reported, we mark ## the entry with an addition quote symbol (i.e., ">"). Many MUAs ## then highlight such lines differently. global flag_nets: set[subnet] &redef; - + ## Function that renders a single alarm. Can be overidden. global pretty_print_alarm: function(out: file, n: Info) &redef; + + ## Force generating mail file, even if reading from traces or no mail + ## destination is defined. This is mainly for testing. + global force_email_summaries = F &redef; } # We maintain an old-style file recording the pretty-printed alarms. @@ -32,6 +36,9 @@ global pp_alarms_open: bool = F; # Returns True if pretty-printed alarm summaries are activated. function want_pp() : bool { + if ( force_email_summaries ) + return T; + return (pretty_print_alarms && ! reading_traces() && (mail_dest != "" || mail_dest_pretty_printed != "")); } @@ -41,38 +48,49 @@ function pp_open() { if ( pp_alarms_open ) return; - + pp_alarms_open = T; pp_alarms = open(pp_alarms_name); - - local dest = mail_dest_pretty_printed != "" ? mail_dest_pretty_printed - : mail_dest; - - local headers = email_headers("Alarm summary", dest); - write_file(pp_alarms, headers + "\n"); } # Closes and mails out the current output file. -function pp_send() +function pp_send(rinfo: Log::RotationInfo) { if ( ! pp_alarms_open ) return; - + write_file(pp_alarms, "\n\n--\n[Automatically generated]\n\n"); close(pp_alarms); - - system(fmt("/bin/cat %s | %s -t -oi && /bin/rm %s", - pp_alarms_name, sendmail, pp_alarms_name)); - pp_alarms_open = F; + + local from = strftime("%H:%M:%S", rinfo$open); + local to = strftime("%H:%M:%S", rinfo$close); + local subject = fmt("Alarm summary from %s-%s", from, to); + local dest = mail_dest_pretty_printed != "" ? mail_dest_pretty_printed + : mail_dest; + + if ( dest == "" ) + # No mail destination configured, just leave the file alone. This is mainly for + # testing. + return; + + local headers = email_headers(subject, dest); + + local header_name = pp_alarms_name + ".tmp"; + local header = open(header_name); + write_file(header, headers + "\n"); + close(header); + + system(fmt("/bin/cat %s %s | %s -t -oi && /bin/rm -f %s %s", + header_name, pp_alarms_name, sendmail, header_name, pp_alarms_name)); } # Postprocessor function that triggers the email. function pp_postprocessor(info: Log::RotationInfo): bool { if ( want_pp() ) - pp_send(); - + pp_send(info); + return T; } @@ -80,7 +98,7 @@ event bro_init() { if ( ! want_pp() ) return; - + # This replaces the standard non-pretty-printing filter. Log::add_filter(Notice::ALARM_LOG, [$name="alarm-mail", $writer=Log::WRITER_NONE, @@ -92,13 +110,13 @@ event notice(n: Notice::Info) &priority=-5 { if ( ! want_pp() ) return; - - if ( ACTION_LOG !in n$actions ) + + if ( ACTION_ALARM !in n$actions ) return; - + if ( ! pp_alarms_open ) pp_open(); - + pretty_print_alarm(pp_alarms, n); } @@ -108,12 +126,12 @@ function do_msg(out: file, n: Info, line1: string, line2: string, line3: string, @ifdef ( Notice::ACTION_ADD_GEODATA ) # Make tests happy, cyclic dependency. if ( n?$remote_location && n$remote_location?$country_code ) country = fmt(" (remote location %s)", n$remote_location$country_code); -@endif - +@endif + line1 = cat(line1, country); - + local resolved = ""; - + if ( host1 != 0.0.0.0 ) resolved = fmt("%s # %s = %s", resolved, host1, name1); @@ -133,64 +151,64 @@ function do_msg(out: file, n: Info, line1: string, line2: string, line3: string, function pretty_print_alarm(out: file, n: Info) { local pdescr = ""; - + @if ( Cluster::is_enabled() ) pdescr = "local"; - + if ( n?$src_peer ) pdescr = n$src_peer?$descr ? n$src_peer$descr : fmt("%s", n$src_peer$host); pdescr = fmt("<%s> ", pdescr); @endif - + local msg = fmt( "%s%s", pdescr, n$msg); - + local who = ""; local h1 = 0.0.0.0; local h2 = 0.0.0.0; - + local orig_p = ""; local resp_p = ""; - + if ( n?$id ) { - orig_p = fmt(":%s", n$id$orig_p); - resp_p = fmt(":%s", n$id$resp_p); + h1 = n$id$orig_h; + h2 = n$id$resp_h; + who = fmt("%s:%s -> %s:%s", h1, n$id$orig_p, h2, n$id$resp_p); } - - if ( n?$src && n?$dst ) + else if ( n?$src && n?$dst ) { h1 = n$src; h2 = n$dst; - who = fmt("%s%s -> %s%s", h1, orig_p, h2, resp_p); - - if ( n?$uid ) - who = fmt("%s (uid %s)", who, n$uid ); + who = fmt("%s -> %s", h1, h2); } - else if ( n?$src ) { - local p = ""; - - if ( n?$p ) - p = fmt(":%s", n$p); - h1 = n$src; - who = fmt("%s%s", h1, p); + who = fmt("%s%s", h1, (n?$p ? fmt(":%s", n$p) : "")); } - + + if ( n?$uid ) + who = fmt("%s (uid %s)", who, n$uid ); + local flag = (h1 in flag_nets || h2 in flag_nets); - + local line1 = fmt(">%s %D %s %s", (flag ? ">" : " "), network_time(), n$note, who); local line2 = fmt(" %s", msg); local line3 = n?$sub ? fmt(" %s", n$sub) : ""; - + if ( h1 == 0.0.0.0 ) { do_msg(out, n, line1, line2, line3, h1, "", h2, ""); return; } - + + if ( reading_traces() ) + { + do_msg(out, n, line1, line2, line3, h1, "", h2, ""); + return; + } + when ( local h1name = lookup_addr(h1) ) { if ( h2 == 0.0.0.0 ) diff --git a/scripts/base/frameworks/notice/extend-email/hostnames.bro b/scripts/base/frameworks/notice/extend-email/hostnames.bro index a73810c726..17a597678d 100644 --- a/scripts/base/frameworks/notice/extend-email/hostnames.bro +++ b/scripts/base/frameworks/notice/extend-email/hostnames.bro @@ -2,31 +2,46 @@ module Notice; -# This probably doesn't actually work due to the async lookup_addr. +# We have to store references to the notices here because the when statement +# clones the frame which doesn't give us access to modify values outside +# of it's execution scope. (we get a clone of the notice instead of a +# reference to the original notice) +global tmp_notice_storage: table[string] of Notice::Info &create_expire=max_email_delay+10secs; + event Notice::notice(n: Notice::Info) &priority=10 { if ( ! n?$src && ! n?$dst ) return; - + # This should only be done for notices that are being sent to email. if ( ACTION_EMAIL !in n$actions ) return; - + + # I'm not recovering gracefully from the when statements because I want + # the notice framework to detect that something has exceeded the maximum + # allowed email delay and tell the user. + local uid = unique_id(""); + tmp_notice_storage[uid] = n; + local output = ""; if ( n?$src ) { + add n$email_delay_tokens["hostnames-src"]; when ( local src_name = lookup_addr(n$src) ) { - output = string_cat("orig_h/src hostname: ", src_name, "\n"); - n$email_body_sections[|n$email_body_sections|] = output; + output = string_cat("orig/src hostname: ", src_name, "\n"); + tmp_notice_storage[uid]$email_body_sections[|tmp_notice_storage[uid]$email_body_sections|] = output; + delete tmp_notice_storage[uid]$email_delay_tokens["hostnames-src"]; } } if ( n?$dst ) { + add n$email_delay_tokens["hostnames-dst"]; when ( local dst_name = lookup_addr(n$dst) ) { - output = string_cat("resp_h/dst hostname: ", dst_name, "\n"); - n$email_body_sections[|n$email_body_sections|] = output; + output = string_cat("resp/dst hostname: ", dst_name, "\n"); + tmp_notice_storage[uid]$email_body_sections[|tmp_notice_storage[uid]$email_body_sections|] = output; + delete tmp_notice_storage[uid]$email_delay_tokens["hostnames-dst"]; } } } diff --git a/scripts/base/frameworks/notice/main.bro b/scripts/base/frameworks/notice/main.bro index 7d98c6464c..70cf613b72 100644 --- a/scripts/base/frameworks/notice/main.bro +++ b/scripts/base/frameworks/notice/main.bro @@ -8,9 +8,9 @@ module Notice; export { - redef enum Log::ID += { + redef enum Log::ID += { ## This is the primary logging stream for notices. - LOG, + LOG, ## This is the notice policy auditing log. It records what the current ## notice policy is at Bro init time. POLICY_LOG, @@ -18,25 +18,25 @@ export { ALARM_LOG, }; - ## Scripts creating new notices need to redef this enum to add their own + ## Scripts creating new notices need to redef this enum to add their own ## specific notice types which would then get used when they call the ## :bro:id:`NOTICE` function. The convention is to give a general category ## along with the specific notice separating words with underscores and using ## leading capitals on each word except for abbreviations which are kept in - ## all capitals. For example, SSH::Login is for heuristically guessed + ## all capitals. For example, SSH::Login is for heuristically guessed ## successful SSH logins. type Type: enum { ## Notice reporting a count of how often a notice occurred. Tally, }; - + ## These are values representing actions that can be taken with notices. type Action: enum { ## Indicates that there is no action to be taken. ACTION_NONE, ## Indicates that the notice should be sent to the notice logging stream. ACTION_LOG, - ## Indicates that the notice should be sent to the email address(es) + ## Indicates that the notice should be sent to the email address(es) ## configured in the :bro:id:`Notice::mail_dest` variable. ACTION_EMAIL, ## Indicates that the notice should be alarmed. A readable ASCII @@ -47,30 +47,34 @@ export { ## duplicate notice suppression that the notice framework does. ACTION_NO_SUPPRESS, }; - - ## The notice framework is able to do automatic notice supression by + + ## The notice framework is able to do automatic notice supression by ## utilizing the $identifier field in :bro:type:`Info` records. ## Set this to "0secs" to completely disable automated notice suppression. const default_suppression_interval = 1hrs &redef; - + type Info: record { ts: time &log &optional; uid: string &log &optional; id: conn_id &log &optional; - + ## These are shorthand ways of giving the uid and id to a notice. The ## reference to the actual connection will be deleted after applying ## the notice policy. conn: connection &optional; iconn: icmp_conn &optional; - + + ## The transport protocol. Filled automatically when either conn, iconn + ## or p is specified. + proto: transport_proto &log &optional; + ## The :bro:enum:`Notice::Type` of the notice. note: Type &log; ## The human readable message for the notice. msg: string &log &optional; ## The human readable sub-message. sub: string &log &optional; - + ## Source address, if we don't have a :bro:type:`conn_id`. src: addr &log &optional; ## Destination address. @@ -79,33 +83,39 @@ export { p: port &log &optional; ## Associated count, or perhaps a status code. n: count &log &optional; - + ## Peer that raised this notice. src_peer: event_peer &optional; ## Textual description for the peer that raised this notice. peer_descr: string &log &optional; - + ## The actions which have been applied to this notice. actions: set[Notice::Action] &log &optional; - + ## These are policy items that returned T and applied their action ## to the notice. policy_items: set[count] &log &optional; - + ## By adding chunks of text into this element, other scripts can ## expand on notices that are being emailed. The normal way to add text ## is to extend the vector by handling the :bro:id:`Notice::notice` ## event and modifying the notice in place. - email_body_sections: vector of string &default=vector(); - + email_body_sections: vector of string &optional; + + ## Adding a string "token" to this set will cause the notice framework's + ## built-in emailing functionality to delay sending the email until + ## either the token has been removed or the email has been delayed + ## for :bro:id:`max_email_delay`. + email_delay_tokens: set[string] &optional; + ## This field is to be provided when a notice is generated for the ## purpose of deduplicating notices. The identifier string should - ## be unique for a single instance of the notice. This field should be - ## filled out in almost all cases when generating notices to define + ## be unique for a single instance of the notice. This field should be + ## filled out in almost all cases when generating notices to define ## when a notice is conceptually a duplicate of a previous notice. - ## - ## For example, an SSL certificate that is going to expire soon should - ## always have the same identifier no matter the client IP address + ## + ## For example, an SSL certificate that is going to expire soon should + ## always have the same identifier no matter the client IP address ## that connected and resulted in the certificate being exposed. In ## this case, the resp_h, resp_p, and hash of the certificate would be ## used to create this value. The hash of the cert is included @@ -114,19 +124,19 @@ export { ## Another example might be a host downloading a file which triggered ## a notice because the MD5 sum of the file it downloaded was known ## by some set of intelligence. In that case, the orig_h (client) - ## and MD5 sum would be used in this field to dedup because if the + ## and MD5 sum would be used in this field to dedup because if the ## same file is downloaded over and over again you really only want to ## know about it a single time. This makes it possible to send those ## notices to email without worrying so much about sending thousands ## of emails. identifier: string &optional; - + ## This field indicates the length of time that this - ## unique notice should be suppressed. This field is automatically + ## unique notice should be suppressed. This field is automatically ## filled out and should not be written to by any other script. suppress_for: interval &log &optional; }; - + ## Ignored notice types. const ignored_types: set[Notice::Type] = {} &redef; ## Emailed notice types. @@ -135,10 +145,10 @@ export { const alarmed_types: set[Notice::Type] = {} &redef; ## Types that should be suppressed for the default suppression interval. const not_suppressed_types: set[Notice::Type] = {} &redef; - ## This table can be used as a shorthand way to modify suppression + ## This table can be used as a shorthand way to modify suppression ## intervals for entire notice types. const type_suppression_intervals: table[Notice::Type] of interval = {} &redef; - + ## This is the record that defines the items that make up the notice policy. type PolicyItem: record { ## This is the exact positional order in which the :bro:type:`PolicyItem` @@ -149,20 +159,20 @@ export { priority: count &log &default=5; ## An action given to the notice if the predicate return true. action: Notice::Action &log &default=ACTION_NONE; - ## The pred (predicate) field is a function that returns a boolean T - ## or F value. If the predicate function return true, the action in - ## this record is applied to the notice that is given as an argument - ## to the predicate function. If no predicate is supplied, it's + ## The pred (predicate) field is a function that returns a boolean T + ## or F value. If the predicate function return true, the action in + ## this record is applied to the notice that is given as an argument + ## to the predicate function. If no predicate is supplied, it's ## assumed that the PolicyItem always applies. pred: function(n: Notice::Info): bool &log &optional; - ## Indicates this item should terminate policy processing if the + ## Indicates this item should terminate policy processing if the ## predicate returns T. halt: bool &log &default=F; ## This defines the length of time that this particular notice should ## be supressed. suppress_for: interval &log &optional; }; - + ## This is the where the :bro:id:`Notice::policy` is defined. All notice ## processing is done through this variable. const policy: set[PolicyItem] = { @@ -177,66 +187,68 @@ export { [$pred(n: Notice::Info) = { return (n$note in Notice::emailed_types); }, $action = ACTION_EMAIL, $priority = 8], - [$pred(n: Notice::Info) = { - if (n$note in Notice::type_suppression_intervals) + [$pred(n: Notice::Info) = { + if (n$note in Notice::type_suppression_intervals) { n$suppress_for=Notice::type_suppression_intervals[n$note]; return T; } - return F; + return F; }, $action = ACTION_NONE, $priority = 8], [$action = ACTION_LOG, $priority = 0], } &redef; - + ## Local system sendmail program. const sendmail = "/usr/sbin/sendmail" &redef; ## Email address to send notices with the :bro:enum:`ACTION_EMAIL` action ## or to send bulk alarm logs on rotation with :bro:enum:`ACTION_ALARM`. const mail_dest = "" &redef; - + ## Address that emails will be from. const mail_from = "Big Brother " &redef; ## Reply-to address used in outbound email. const reply_to = "" &redef; ## Text string prefixed to the subject of all emails sent out. const mail_subject_prefix = "[Bro]" &redef; + ## The maximum amount of time a plugin can delay email from being sent. + const max_email_delay = 15secs &redef; ## A log postprocessing function that implements emailing the contents ## of a log upon rotation to any configured :bro:id:`Notice::mail_dest`. ## The rotated log is removed upon being sent. global log_mailing_postprocessor: function(info: Log::RotationInfo): bool; - ## This is the event that is called as the entry point to the - ## notice framework by the global :bro:id:`NOTICE` function. By the time + ## This is the event that is called as the entry point to the + ## notice framework by the global :bro:id:`NOTICE` function. By the time ## this event is generated, default values have already been filled out in - ## the :bro:type:`Notice::Info` record and synchronous functions in the + ## the :bro:type:`Notice::Info` record and synchronous functions in the ## :bro:id:`Notice:sync_functions` have already been called. The notice ## policy has also been applied. global notice: event(n: Info); - ## This is a set of functions that provide a synchronous way for scripts + ## This is a set of functions that provide a synchronous way for scripts ## extending the notice framework to run before the normal event based ## notice pathway that most of the notice framework takes. This is helpful ## in cases where an action against a notice needs to happen immediately ## and can't wait the short time for the event to bubble up to the top of - ## the event queue. An example is the IP address dropping script that - ## can block IP addresses that have notices generated because it + ## the event queue. An example is the IP address dropping script that + ## can block IP addresses that have notices generated because it ## needs to operate closer to real time than the event queue allows it to. - ## Normally the event based extension model using the + ## Normally the event based extension model using the ## :bro:id:`Notice::notice` event will work fine if there aren't harder ## real time constraints. const sync_functions: set[function(n: Notice::Info)] = set() &redef; - + ## This event is generated when a notice begins to be suppressed. global begin_suppression: event(n: Notice::Info); ## This event is generated on each occurence of an event being suppressed. global suppressed: event(n: Notice::Info); ## This event is generated when a notice stops being suppressed. global end_suppression: event(n: Notice::Info); - + ## Call this function to send a notice in an email. It is already used ## by default with the built in :bro:enum:`ACTION_EMAIL` and ## :bro:enum:`ACTION_PAGE` actions. @@ -248,12 +260,12 @@ export { ## dest: recipient string to use for the mail ## Returns: a string of mail headers to which an email body can be appended global email_headers: function(subject_desc: string, dest: string): string; - + ## This event can be handled to access the :bro:type:`Info` ## record as it is sent on to the logging framework. global log_notice: event(rec: Info); - - ## This is an internal wrapper for the global NOTICE function. Please + + ## This is an internal wrapper for the global NOTICE function. Please ## disregard. global internal_NOTICE: function(n: Notice::Info); } @@ -264,22 +276,22 @@ function per_notice_suppression_interval(t: table[Notice::Type, string] of Notic local n: Notice::Type; local s: string; [n,s] = idx; - + local suppress_time = t[n,s]$suppress_for - (network_time() - t[n,s]$ts); if ( suppress_time < 0secs ) suppress_time = 0secs; - + # If there is no more suppression time left, the notice needs to be sent # to the end_suppression event. if ( suppress_time == 0secs ) event Notice::end_suppression(t[n,s]); - + return suppress_time; } -# This is the internally maintained notice suppression table. It's +# This is the internally maintained notice suppression table. It's # indexed on the Notice::Type and the $identifier field from the notice. -global suppressing: table[Type, string] of Notice::Info = {} +global suppressing: table[Type, string] of Notice::Info = {} &create_expire=0secs &expire_func=per_notice_suppression_interval; @@ -306,7 +318,7 @@ function log_mailing_postprocessor(info: Log::RotationInfo): bool event bro_init() &priority=5 { Log::create_stream(Notice::LOG, [$columns=Info, $ev=log_notice]); - + Log::create_stream(Notice::ALARM_LOG, [$columns=Notice::Info]); # If Bro is configured for mailing notices, set up mailing for alarms. # Make sure that this alarm log is also output as text so that it can @@ -347,25 +359,49 @@ function email_headers(subject_desc: string, dest: string): string return header_text; } +event delay_sending_email(n: Notice::Info, dest: string, extend: bool) + { + email_notice_to(n, dest, extend); + } + function email_notice_to(n: Notice::Info, dest: string, extend: bool) { if ( reading_traces() || dest == "" ) return; - + + if ( extend ) + { + if ( |n$email_delay_tokens| > 0 ) + { + # If we still are within the max_email_delay, keep delaying. + if ( n$ts + max_email_delay > network_time() ) + { + schedule 1sec { delay_sending_email(n, dest, extend) }; + return; + } + else + { + event reporter_info(network_time(), + fmt("Notice email delay tokens weren't released in time (%s).", n$email_delay_tokens), + ""); + } + } + } + local email_text = email_headers(fmt("%s", n$note), dest); - + # First off, finish the headers and include the human readable messages # then leave a blank line after the message. email_text = string_cat(email_text, "\nMessage: ", n$msg); if ( n?$sub ) email_text = string_cat(email_text, "\nSub-message: ", n$sub); - + email_text = string_cat(email_text, "\n\n"); - + # Next, add information about the connection if it exists. if ( n?$id ) { - email_text = string_cat(email_text, "Connection: ", + email_text = string_cat(email_text, "Connection: ", fmt("%s", n$id$orig_h), ":", fmt("%d", n$id$orig_p), " -> ", fmt("%s", n$id$resp_h), ":", fmt("%d", n$id$resp_p), "\n"); if ( n?$uid ) @@ -373,17 +409,18 @@ function email_notice_to(n: Notice::Info, dest: string, extend: bool) } else if ( n?$src ) email_text = string_cat(email_text, "Address: ", fmt("%s", n$src), "\n"); - + # Add the extended information if it's requested. if ( extend ) { + email_text = string_cat(email_text, "\nEmail Extensions\n"); + email_text = string_cat(email_text, "----------------\n"); for ( i in n$email_body_sections ) { - email_text = string_cat(email_text, "******************\n"); email_text = string_cat(email_text, n$email_body_sections[i], "\n"); } } - + email_text = string_cat(email_text, "\n\n--\n[Automatically generated]\n\n"); piped_exec(fmt("%s -t -oi", sendmail), email_text); } @@ -396,10 +433,10 @@ event notice(n: Notice::Info) &priority=-5 Log::write(Notice::LOG, n); if ( ACTION_ALARM in n$actions ) Log::write(Notice::ALARM_LOG, n); - + # Normally suppress further notices like this one unless directed not to. # n$identifier *must* be specified for suppression to function at all. - if ( n?$identifier && + if ( n?$identifier && ACTION_NO_SUPPRESS !in n$actions && [n$note, n$identifier] !in suppressing && n$suppress_for != 0secs ) @@ -408,8 +445,8 @@ event notice(n: Notice::Info) &priority=-5 event Notice::begin_suppression(n); } } - -## This determines if a notice is being suppressed. It is only used + +## This determines if a notice is being suppressed. It is only used ## internally as part of the mechanics for the global NOTICE function. function is_being_suppressed(n: Notice::Info): bool { @@ -421,7 +458,7 @@ function is_being_suppressed(n: Notice::Info): bool else return F; } - + # Executes a script with all of the notice fields put into the # new process' environment as "BRO_ARG_" variables. function execute_with_notice(cmd: string, n: Notice::Info) @@ -430,9 +467,9 @@ function execute_with_notice(cmd: string, n: Notice::Info) #local tgs = tags(n); #system_env(cmd, tags); } - -# This is run synchronously as a function before all of the other -# notice related functions and events. It also modifies the + +# This is run synchronously as a function before all of the other +# notice related functions and events. It also modifies the # :bro:type:`Notice::Info` record in place. function apply_policy(n: Notice::Info) { @@ -447,7 +484,7 @@ function apply_policy(n: Notice::Info) if ( ! n?$uid ) n$uid = n$conn$uid; } - + if ( n?$id ) { if ( ! n?$src ) @@ -458,8 +495,12 @@ function apply_policy(n: Notice::Info) n$p = n$id$resp_p; } + if ( n?$p ) + n$proto = get_port_transport_proto(n$p); + if ( n?$iconn ) { + n$proto = icmp; if ( ! n?$src ) n$src = n$iconn$orig_h; if ( ! n?$dst ) @@ -469,15 +510,20 @@ function apply_policy(n: Notice::Info) if ( ! n?$src_peer ) n$src_peer = get_event_peer(); if ( ! n?$peer_descr ) - n$peer_descr = n$src_peer?$descr ? + n$peer_descr = n$src_peer?$descr ? n$src_peer$descr : fmt("%s", n$src_peer$host); - + if ( ! n?$actions ) n$actions = set(); - + + if ( ! n?$email_body_sections ) + n$email_body_sections = vector(); + if ( ! n?$email_delay_tokens ) + n$email_delay_tokens = set(); + if ( ! n?$policy_items ) n$policy_items = set(); - + for ( i in ordered_policy ) { # If there's no predicate or the predicate returns F. @@ -485,51 +531,51 @@ function apply_policy(n: Notice::Info) { add n$actions[ordered_policy[i]$action]; add n$policy_items[int_to_count(i)]; - - # If the predicate matched and there was a suppression interval, + + # If the predicate matched and there was a suppression interval, # apply it to the notice now. if ( ordered_policy[i]?$suppress_for ) n$suppress_for = ordered_policy[i]$suppress_for; - + # If the policy item wants to halt policy processing, do it now! if ( ordered_policy[i]$halt ) break; } } - + # Apply the suppression time after applying the policy so that policy - # items can give custom suppression intervals. If there is no + # items can give custom suppression intervals. If there is no # suppression interval given yet, the default is applied. if ( ! n?$suppress_for ) n$suppress_for = default_suppression_interval; - + # Delete the connection record if it's there so we aren't sending that - # to remote machines. It can cause problems due to the size of the + # to remote machines. It can cause problems due to the size of the # connection record. if ( n?$conn ) delete n$conn; if ( n?$iconn ) delete n$iconn; } - -# Create the ordered notice policy automatically which will be used at runtime + +# Create the ordered notice policy automatically which will be used at runtime # for prioritized matching of the notice policy. event bro_init() &priority=10 { # Create the policy log here because it's only written to in this handler. Log::create_stream(Notice::POLICY_LOG, [$columns=PolicyItem]); - + local tmp: table[count] of set[PolicyItem] = table(); for ( pi in policy ) { if ( pi$priority < 0 || pi$priority > 10 ) Reporter::fatal("All Notice::PolicyItem priorities must be within 0 and 10"); - + if ( pi$priority !in tmp ) tmp[pi$priority] = set(); add tmp[pi$priority][pi]; } - + local rev_count = vector(10,9,8,7,6,5,4,3,2,1,0); for ( i in rev_count ) { @@ -545,7 +591,7 @@ event bro_init() &priority=10 } } } - + function internal_NOTICE(n: Notice::Info) { # Suppress this notice if necessary. diff --git a/scripts/base/protocols/dns/main.bro b/scripts/base/protocols/dns/main.bro index 2580b003dd..b8cfc7b44e 100644 --- a/scripts/base/protocols/dns/main.bro +++ b/scripts/base/protocols/dns/main.bro @@ -4,50 +4,50 @@ module DNS; export { redef enum Log::ID += { LOG }; - + type Info: record { - ts: time &log; - uid: string &log; - id: conn_id &log; - proto: transport_proto &log; - trans_id: count &log &optional; - query: string &log &optional; - qclass: count &log &optional; - qclass_name: string &log &optional; - qtype: count &log &optional; - qtype_name: string &log &optional; - rcode: count &log &optional; - rcode_name: string &log &optional; - QR: bool &log &default=F; - AA: bool &log &default=F; - TC: bool &log &default=F; - RD: bool &log &default=F; - RA: bool &log &default=F; - Z: count &log &default=0; - TTL: interval &log &optional; - answers: set[string] &log &optional; - + ts: time &log; + uid: string &log; + id: conn_id &log; + proto: transport_proto &log; + trans_id: count &log &optional; + query: string &log &optional; + qclass: count &log &optional; + qclass_name: string &log &optional; + qtype: count &log &optional; + qtype_name: string &log &optional; + rcode: count &log &optional; + rcode_name: string &log &optional; + QR: bool &log &default=F; + AA: bool &log &default=F; + TC: bool &log &default=F; + RD: bool &log &default=F; + RA: bool &log &default=F; + Z: count &log &default=0; + answers: vector of string &log &optional; + TTLs: vector of interval &log &optional; + ## This value indicates if this request/response pair is ready to be logged. ready: bool &default=F; total_answers: count &optional; total_replies: count &optional; }; - + type State: record { ## Indexed by query id, returns Info record corresponding to ## query/response which haven't completed yet. pending: table[count] of Info &optional; - + ## This is the list of DNS responses that have completed based on the ## number of responses declared and the number received. The contents ## of the set are transaction IDs. finished_answers: set[count] &optional; }; - + global log_dns: event(rec: Info); - + ## This is called by the specific dns_*_reply events with a "reply" which - ## may not represent the full data available from the resource record, but + ## may not represent the full data available from the resource record, but ## it's generally considered a summarization of the response(s). global do_reply: event(c: connection, msg: dns_msg, ans: dns_answer, reply: string); } @@ -58,18 +58,18 @@ redef record connection += { }; # DPD configuration. -redef capture_filters += { +redef capture_filters += { ["dns"] = "port 53", ["mdns"] = "udp and port 5353", ["llmns"] = "udp and port 5355", - ["netbios-ns"] = "udp port 137", + ["netbios-ns"] = "udp port 137", }; -global dns_ports = { 53/udp, 53/tcp, 137/udp, 5353/udp, 5355/udp } &redef; +const dns_ports = { 53/udp, 53/tcp, 137/udp, 5353/udp, 5355/udp }; redef dpd_config += { [ANALYZER_DNS] = [$ports = dns_ports] }; -global dns_udp_ports = { 53/udp, 137/udp, 5353/udp, 5355/udp } &redef; -global dns_tcp_ports = { 53/tcp } &redef; +const dns_udp_ports = { 53/udp, 137/udp, 5353/udp, 5355/udp }; +const dns_tcp_ports = { 53/tcp }; redef dpd_config += { [ANALYZER_DNS_UDP_BINPAC] = [$ports = dns_udp_ports] }; redef dpd_config += { [ANALYZER_DNS_TCP_BINPAC] = [$ports = dns_tcp_ports] }; @@ -89,7 +89,7 @@ function new_session(c: connection, trans_id: count): Info state$finished_answers=set(); c$dns_state = state; } - + local info: Info; info$ts = network_time(); info$id = c$id; @@ -102,23 +102,29 @@ function new_session(c: connection, trans_id: count): Info function set_session(c: connection, msg: dns_msg, is_query: bool) { if ( ! c?$dns_state || msg$id !in c$dns_state$pending ) + { c$dns_state$pending[msg$id] = new_session(c, msg$id); - + # Try deleting this transaction id from the set of finished answers. + # Sometimes hosts will reuse ports and transaction ids and this should + # be considered to be a legit scenario (although bad practice). + delete c$dns_state$finished_answers[msg$id]; + } + c$dns = c$dns_state$pending[msg$id]; c$dns$rcode = msg$rcode; c$dns$rcode_name = base_errors[msg$rcode]; - + if ( ! is_query ) { if ( ! c$dns?$total_answers ) c$dns$total_answers = msg$num_answers; - - if ( c$dns?$total_replies && + + if ( c$dns?$total_replies && c$dns$total_replies != msg$num_answers + msg$num_addl + msg$num_auth ) { - event conn_weird("dns_changed_number_of_responses", c, - fmt("The declared number of responses changed from %d to %d", + event conn_weird("dns_changed_number_of_responses", c, + fmt("The declared number of responses changed from %d to %d", c$dns$total_replies, msg$num_answers + msg$num_addl + msg$num_auth)); } @@ -129,27 +135,30 @@ function set_session(c: connection, msg: dns_msg, is_query: bool) } } } - + event DNS::do_reply(c: connection, msg: dns_msg, ans: dns_answer, reply: string) &priority=5 { set_session(c, msg, F); - c$dns$AA = msg$AA; - c$dns$RA = msg$RA; - c$dns$TTL = ans$TTL; - if ( ans$answer_type == DNS_ANS ) { + c$dns$AA = msg$AA; + c$dns$RA = msg$RA; + if ( msg$id in c$dns_state$finished_answers ) event conn_weird("dns_reply_seen_after_done", c, ""); - + if ( reply != "" ) { if ( ! c$dns?$answers ) - c$dns$answers = set(); - add c$dns$answers[reply]; + c$dns$answers = vector(); + c$dns$answers[|c$dns$answers|] = reply; + + if ( ! c$dns?$TTLs ) + c$dns$TTLs = vector(); + c$dns$TTLs[|c$dns$TTLs|] = ans$TTL; } - + if ( c$dns?$answers && |c$dns$answers| == c$dns$total_answers ) { add c$dns_state$finished_answers[c$dns$trans_id]; @@ -158,13 +167,12 @@ event DNS::do_reply(c: connection, msg: dns_msg, ans: dns_answer, reply: string) } } } - + event DNS::do_reply(c: connection, msg: dns_msg, ans: dns_answer, reply: string) &priority=-5 { if ( c$dns$ready ) { Log::write(DNS::LOG, c$dns); - add c$dns_state$finished_answers[c$dns$trans_id]; # This record is logged and no longer pending. delete c$dns_state$pending[c$dns$trans_id]; } @@ -173,41 +181,41 @@ event DNS::do_reply(c: connection, msg: dns_msg, ans: dns_answer, reply: string) event dns_request(c: connection, msg: dns_msg, query: string, qtype: count, qclass: count) &priority=5 { set_session(c, msg, T); - + c$dns$RD = msg$RD; c$dns$TC = msg$TC; c$dns$qclass = qclass; c$dns$qclass_name = classes[qclass]; c$dns$qtype = qtype; c$dns$qtype_name = query_types[qtype]; - + # Decode netbios name queries - # Note: I'm ignoring the name type for now. Not sure if this should be + # Note: I'm ignoring the name type for now. Not sure if this should be # worked into the query/response in some fashion. if ( c$id$resp_p == 137/udp ) query = decode_netbios_name(query); c$dns$query = query; - + c$dns$Z = msg$Z; } - + event dns_A_reply(c: connection, msg: dns_msg, ans: dns_answer, a: addr) &priority=5 { event DNS::do_reply(c, msg, ans, fmt("%s", a)); } - + event dns_TXT_reply(c: connection, msg: dns_msg, ans: dns_answer, str: string) &priority=5 { event DNS::do_reply(c, msg, ans, str); } - -event dns_AAAA_reply(c: connection, msg: dns_msg, ans: dns_answer, a: addr, + +event dns_AAAA_reply(c: connection, msg: dns_msg, ans: dns_answer, a: addr, astr: string) &priority=5 { # TODO: What should we do with astr? event DNS::do_reply(c, msg, ans, fmt("%s", a)); } - + event dns_NS_reply(c: connection, msg: dns_msg, ans: dns_answer, name: string) &priority=5 { event DNS::do_reply(c, msg, ans, name); @@ -223,12 +231,12 @@ event dns_MX_reply(c: connection, msg: dns_msg, ans: dns_answer, name: string, { event DNS::do_reply(c, msg, ans, name); } - + event dns_PTR_reply(c: connection, msg: dns_msg, ans: dns_answer, name: string) &priority=5 { event DNS::do_reply(c, msg, ans, name); } - + event dns_SOA_reply(c: connection, msg: dns_msg, ans: dns_answer, soa: dns_soa) &priority=5 { event DNS::do_reply(c, msg, ans, soa$mname); @@ -238,7 +246,7 @@ event dns_WKS_reply(c: connection, msg: dns_msg, ans: dns_answer) &priority=5 { event DNS::do_reply(c, msg, ans, ""); } - + event dns_SRV_reply(c: connection, msg: dns_msg, ans: dns_answer) &priority=5 { event DNS::do_reply(c, msg, ans, ""); @@ -247,17 +255,17 @@ event dns_SRV_reply(c: connection, msg: dns_msg, ans: dns_answer) &priority=5 # TODO: figure out how to handle these #event dns_EDNS(c: connection, msg: dns_msg, ans: dns_answer) # { -# +# # } # #event dns_EDNS_addl(c: connection, msg: dns_msg, ans: dns_edns_additional) # { -# +# # } # #event dns_TSIG_addl(c: connection, msg: dns_msg, ans: dns_tsig_additional) # { -# +# # } @@ -271,10 +279,10 @@ event connection_state_remove(c: connection) &priority=-5 { if ( ! c?$dns_state ) return; - - # If Bro is expiring state, we should go ahead and log all unlogged + + # If Bro is expiring state, we should go ahead and log all unlogged # request/response pairs now. for ( trans_id in c$dns_state$pending ) Log::write(DNS::LOG, c$dns_state$pending[trans_id]); } - + diff --git a/scripts/base/protocols/irc/main.bro b/scripts/base/protocols/irc/main.bro index 731a943819..e74eea21f1 100644 --- a/scripts/base/protocols/irc/main.bro +++ b/scripts/base/protocols/irc/main.bro @@ -41,7 +41,7 @@ redef capture_filters += { ["irc-6668"] = "port 6668" }; redef capture_filters += { ["irc-6669"] = "port 6669" }; # DPD configuration. -global irc_ports = { 6666/tcp, 6667/tcp, 6668/tcp, 6669/tcp } &redef; +const irc_ports = { 6666/tcp, 6667/tcp, 6668/tcp, 6669/tcp }; redef dpd_config += { [ANALYZER_IRC] = [$ports = irc_ports] }; redef likely_server_ports += { 6666/tcp, 6667/tcp, 6668/tcp, 6669/tcp }; diff --git a/scripts/base/protocols/ssl/consts.bro b/scripts/base/protocols/ssl/consts.bro index 2026f9bfa2..9d8bc68fd5 100644 --- a/scripts/base/protocols/ssl/consts.bro +++ b/scripts/base/protocols/ssl/consts.bro @@ -13,6 +13,44 @@ export { [TLSv11] = "TLSv11", } &default="UNKNOWN"; + const alert_levels: table[count] of string = { + [1] = "warning", + [2] = "fatal", + } &default=function(i: count):string { return fmt("unknown-%d", i); }; + + const alert_descriptions: table[count] of string = { + [0] = "close_notify", + [10] = "unexpected_message", + [20] = "bad_record_mac", + [21] = "decryption_failed", + [22] = "record_overflow", + [30] = "decompression_failure", + [40] = "handshake_failure", + [41] = "no_certificate", + [42] = "bad_certificate", + [43] = "unsupported_certificate", + [44] = "certificate_revoked", + [45] = "certificate_expired", + [46] = "certificate_unknown", + [47] = "illegal_parameter", + [48] = "unknown_ca", + [49] = "access_denied", + [50] = "decode_error", + [51] = "decrypt_error", + [60] = "export_restriction", + [70] = "protocol_version", + [71] = "insufficient_security", + [80] = "internal_error", + [90] = "user_canceled", + [100] = "no_renegotiation", + [110] = "unsupported_extension", + [111] = "certificate_unobtainable", + [112] = "unrecognized_name", + [113] = "bad_certificate_status_response", + [114] = "bad_certificate_hash_value", + [115] = "unknown_psk_identity", + } &default=function(i: count):string { return fmt("unknown-%d", i); }; + # http://www.iana.org/assignments/tls-extensiontype-values/tls-extensiontype-values.xml const extensions: table[count] of string = { [0] = "server_name", @@ -526,8 +564,7 @@ export { [30] = "akid issuer serial mismatch", [31] = "keyusage no certsign", [32] = "unable to get crl issuer", - [33] = "unhandled critical extension" - + [33] = "unhandled critical extension", }; } diff --git a/scripts/base/protocols/ssl/main.bro b/scripts/base/protocols/ssl/main.bro index c3c04d3c93..4b2fa39696 100644 --- a/scripts/base/protocols/ssl/main.bro +++ b/scripts/base/protocols/ssl/main.bro @@ -16,32 +16,33 @@ export { subject: string &log &optional; not_valid_before: time &log &optional; not_valid_after: time &log &optional; - + last_alert: string &log &optional; + cert: string &optional; cert_chain: vector of string &optional; - + ## This stores the analyzer id used for the analyzer instance attached - ## to each connection. It is not used for logging since it's a + ## to each connection. It is not used for logging since it's a ## meaningless arbitrary number. analyzer_id: count &optional; }; - + ## This is where the default root CA bundle is defined. By loading the ## mozilla-ca-list.bro script it will be set to Mozilla's root CA list. const root_certs: table[string] of string = {} &redef; - - ## If true, detach the SSL analyzer from the connection to prevent + + ## If true, detach the SSL analyzer from the connection to prevent ## continuing to process encrypted traffic. Helps with performance ## (especially with large file transfers). const disable_analyzer_after_detection = T &redef; - + ## The openssl command line utility. If it's in the path the default ## value will work, otherwise a full path string can be supplied for the ## utility. const openssl_util = "openssl" &redef; - + global log_ssl: event(rec: Info); - + const ports = { 443/tcp, 563/tcp, 585/tcp, 614/tcp, 636/tcp, 989/tcp, 990/tcp, 992/tcp, 993/tcp, 995/tcp, 5223/tcp @@ -86,7 +87,7 @@ function set_session(c: connection) if ( ! c?$ssl ) c$ssl = [$ts=network_time(), $uid=c$uid, $id=c$id, $cert_chain=vector()]; } - + function finish(c: connection) { Log::write(SSL::LOG, c$ssl); @@ -98,29 +99,33 @@ function finish(c: connection) event ssl_client_hello(c: connection, version: count, possible_ts: time, session_id: string, ciphers: count_set) &priority=5 { set_session(c); - + # Save the session_id if there is one set. if ( session_id != /^\x00{32}$/ ) c$ssl$session_id = bytestring_to_hexstr(session_id); } - + event ssl_server_hello(c: connection, version: count, possible_ts: time, session_id: string, cipher: count, comp_method: count) &priority=5 { set_session(c); - + c$ssl$version = version_strings[version]; c$ssl$cipher = cipher_desc[cipher]; } -event x509_certificate(c: connection, cert: X509, is_server: bool, chain_idx: count, chain_len: count, der_cert: string) &priority=5 +event x509_certificate(c: connection, is_orig: bool, cert: X509, chain_idx: count, chain_len: count, der_cert: string) &priority=5 { set_session(c); - + + # We aren't doing anything with client certificates yet. + if ( is_orig ) + return; + if ( chain_idx == 0 ) { # Save the primary cert. c$ssl$cert = der_cert; - + # Also save other certificate information about the primary cert. c$ssl$subject = cert$subject; c$ssl$not_valid_before = cert$not_valid_before; @@ -132,20 +137,27 @@ event x509_certificate(c: connection, cert: X509, is_server: bool, chain_idx: co c$ssl$cert_chain[|c$ssl$cert_chain|] = der_cert; } } - -event ssl_extension(c: connection, code: count, val: string) &priority=5 + +event ssl_extension(c: connection, is_orig: bool, code: count, val: string) &priority=5 { set_session(c); - - if ( extensions[code] == "server_name" ) + + if ( is_orig && extensions[code] == "server_name" ) c$ssl$server_name = sub_bytes(val, 6, |val|); } - + +event ssl_alert(c: connection, is_orig: bool, level: count, desc: count) &priority=5 + { + set_session(c); + + c$ssl$last_alert = alert_descriptions[desc]; + } + event ssl_established(c: connection) &priority=5 { set_session(c); } - + event ssl_established(c: connection) &priority=-5 { finish(c); @@ -163,4 +175,4 @@ event protocol_violation(c: connection, atype: count, aid: count, { if ( c?$ssl ) finish(c); - } \ No newline at end of file + } diff --git a/scripts/policy/frameworks/dpd/detect-protocols.bro b/scripts/policy/frameworks/dpd/detect-protocols.bro index 8e1ea1267f..8f4e892ce4 100644 --- a/scripts/policy/frameworks/dpd/detect-protocols.bro +++ b/scripts/policy/frameworks/dpd/detect-protocols.bro @@ -8,7 +8,6 @@ module ProtocolDetector; export { redef enum Notice::Type += { - Off_Port_Protocol_Found, # raised for each connection found Protocol_Found, Server_Found, }; @@ -155,13 +154,10 @@ function report_protocols(c: connection) { if ( [a, c$id$resp_h, c$id$resp_p] in valids ) do_notice(c, a, valids[a, c$id$resp_h, c$id$resp_p]); - else if ( [a, 0.0.0.0, c$id$resp_p] in valids ) do_notice(c, a, valids[a, 0.0.0.0, c$id$resp_p]); else do_notice(c, a, NONE); - - append_addl(c, analyzer_name(a)); } delete conns[c$id]; @@ -218,20 +214,6 @@ event protocol_confirmation(c: connection, atype: count, aid: count) } } -# event connection_analyzer_disabled(c: connection, analyzer: count) -# { -# if ( c$id !in conns ) -# return; -# -# delete conns[c$id][analyzer]; -# } - -function append_proto_addl(c: connection) - { - for ( a in conns[c$id] ) - append_addl(c, fmt_protocol(get_protocol(c, a))); - } - function found_protocol(c: connection, analyzer: count, protocol: string) { # Don't report anything running on a well-known port. diff --git a/scripts/policy/protocols/http/detect-MHR.bro b/scripts/policy/protocols/http/detect-MHR.bro index 3b2e8bf968..fca3bb1ac9 100644 --- a/scripts/policy/protocols/http/detect-MHR.bro +++ b/scripts/policy/protocols/http/detect-MHR.bro @@ -7,9 +7,12 @@ @load base/frameworks/notice @load base/protocols/http +module HTTP; + export { redef enum Notice::Type += { - ## If the MD5 sum of a file transferred over HTTP + ## The MD5 sum of a file transferred over HTTP matched in the + ## malware hash registry. Malware_Hash_Registry_Match }; } diff --git a/scripts/policy/protocols/http/detect-sqli.bro b/scripts/policy/protocols/http/detect-sqli.bro index c4ba7ee74e..3927ce811e 100644 --- a/scripts/policy/protocols/http/detect-sqli.bro +++ b/scripts/policy/protocols/http/detect-sqli.bro @@ -12,12 +12,12 @@ export { SQL_Injection_Attacker, ## Indicates that a host was seen to have SQL injection attacks against ## it. This is tracked by IP address as opposed to hostname. - SQL_Injection_Attack_Against, + SQL_Injection_Victim, }; redef enum Metrics::ID += { - SQL_ATTACKER, - SQL_ATTACKS_AGAINST, + SQLI_ATTACKER, + SQLI_VICTIM, }; redef enum Tags += { @@ -56,14 +56,14 @@ event bro_init() &priority=3 # determine when it looks like an actual attack and how to respond when # thresholds are crossed. - Metrics::add_filter(SQL_ATTACKER, [$log=F, + Metrics::add_filter(SQLI_ATTACKER, [$log=F, $notice_threshold=sqli_requests_threshold, $break_interval=sqli_requests_interval, $note=SQL_Injection_Attacker]); - Metrics::add_filter(SQL_ATTACKS_AGAINST, [$log=F, - $notice_threshold=sqli_requests_threshold, - $break_interval=sqli_requests_interval, - $note=SQL_Injection_Attack_Against]); + Metrics::add_filter(SQLI_VICTIM, [$log=F, + $notice_threshold=sqli_requests_threshold, + $break_interval=sqli_requests_interval, + $note=SQL_Injection_Victim]); } event http_request(c: connection, method: string, original_URI: string, @@ -73,7 +73,7 @@ event http_request(c: connection, method: string, original_URI: string, { add c$http$tags[URI_SQLI]; - Metrics::add_data(SQL_ATTACKER, [$host=c$id$orig_h], 1); - Metrics::add_data(SQL_ATTACKS_AGAINST, [$host=c$id$resp_h], 1); + Metrics::add_data(SQLI_ATTACKER, [$host=c$id$orig_h], 1); + Metrics::add_data(SQLI_VICTIM, [$host=c$id$resp_h], 1); } } diff --git a/scripts/policy/protocols/ssh/interesting-hostnames.bro b/scripts/policy/protocols/ssh/interesting-hostnames.bro index 29886d0eb0..5771770916 100644 --- a/scripts/policy/protocols/ssh/interesting-hostnames.bro +++ b/scripts/policy/protocols/ssh/interesting-hostnames.bro @@ -36,7 +36,9 @@ event SSH::heuristic_successful_login(c: connection) if ( interesting_hostnames in hostname ) { NOTICE([$note=Interesting_Hostname_Login, - $msg=fmt("Interesting login from hostname: %s", hostname), + $msg=fmt("Possible SSH login involving a %s %s with an interesting hostname.", + Site::is_local_addr(host) ? "local" : "remote", + host == c$id$orig_h ? "client" : "server"), $sub=hostname, $conn=c]); } } diff --git a/scripts/policy/protocols/ssl/cert-hash.bro b/scripts/policy/protocols/ssl/cert-hash.bro index 80a937f670..1e47ccac2e 100644 --- a/scripts/policy/protocols/ssl/cert-hash.bro +++ b/scripts/policy/protocols/ssl/cert-hash.bro @@ -10,11 +10,11 @@ export { }; } -event x509_certificate(c: connection, cert: X509, is_server: bool, chain_idx: count, chain_len: count, der_cert: string) &priority=4 +event x509_certificate(c: connection, is_orig: bool, cert: X509, chain_idx: count, chain_len: count, der_cert: string) &priority=4 { # We aren't tracking client certificates yet and we are also only tracking # the primary cert. Watch that this came from an SSL analyzed session too. - if ( ! is_server || chain_idx != 0 || ! c?$ssl ) + if ( is_orig || chain_idx != 0 || ! c?$ssl ) return; c$ssl$cert_hash = md5_hash(der_cert); diff --git a/scripts/policy/protocols/ssl/expiring-certs.bro b/scripts/policy/protocols/ssl/expiring-certs.bro index 50480b3a09..0e4db56bc3 100644 --- a/scripts/policy/protocols/ssl/expiring-certs.bro +++ b/scripts/policy/protocols/ssl/expiring-certs.bro @@ -33,10 +33,11 @@ export { const notify_when_cert_expiring_in = 30days &redef; } -event x509_certificate(c: connection, cert: X509, is_server: bool, chain_idx: count, chain_len: count, der_cert: string) &priority=3 +event x509_certificate(c: connection, is_orig: bool, cert: X509, chain_idx: count, chain_len: count, der_cert: string) &priority=3 { # If this isn't the host cert or we aren't interested in the server, just return. - if ( chain_idx != 0 || + if ( is_orig || + chain_idx != 0 || ! c$ssl?$cert_hash || ! addr_matches_host(c$id$resp_h, notify_certs_expiration) ) return; diff --git a/scripts/policy/protocols/ssl/known-certs.bro b/scripts/policy/protocols/ssl/known-certs.bro index 90f6ee6186..669432e4d9 100644 --- a/scripts/policy/protocols/ssl/known-certs.bro +++ b/scripts/policy/protocols/ssl/known-certs.bro @@ -44,10 +44,10 @@ event bro_init() &priority=5 Log::create_stream(Known::CERTS_LOG, [$columns=CertsInfo, $ev=log_known_certs]); } -event x509_certificate(c: connection, cert: X509, is_server: bool, chain_idx: count, chain_len: count, der_cert: string) &priority=3 +event x509_certificate(c: connection, is_orig: bool, cert: X509, chain_idx: count, chain_len: count, der_cert: string) &priority=3 { # Make sure this is the server cert and we have a hash for it. - if ( chain_idx != 0 || ! c$ssl?$cert_hash ) + if ( is_orig || chain_idx != 0 || ! c$ssl?$cert_hash ) return; local host = c$id$resp_h; diff --git a/scripts/site/local.bro b/scripts/site/local.bro index 597b92ba3d..8e1f4d802b 100644 --- a/scripts/site/local.bro +++ b/scripts/site/local.bro @@ -62,11 +62,3 @@ redef signature_files += "frameworks/signatures/detect-windows-shells.sig"; @load protocols/http/detect-MHR # Detect SQL injection attacks @load protocols/http/detect-sqli - -# Uncomment this redef if you want to extract SMTP MIME entities for -# some file types. The numbers given indicate how many bytes to extract for -# the various mime types. -redef SMTP::entity_excerpt_len += { -# ["text/plain"] = 1024, -# ["text/html"] = 1024, -}; diff --git a/src/Attr.cc b/src/Attr.cc index a5a350f452..aed9165182 100644 --- a/src/Attr.cc +++ b/src/Attr.cc @@ -60,16 +60,19 @@ void Attr::DescribeReST(ODesc* d) const d->Add("="); d->SP(); - if ( expr->Type()->Tag() == TYPE_FUNC ) - d->Add(":bro:type:`func`"); - else if ( expr->Type()->Tag() == TYPE_ENUM ) + if ( expr->Tag() == EXPR_NAME ) { - d->Add(":bro:enum:`"); + d->Add(":bro:see:`"); expr->Describe(d); d->Add("`"); } + else if ( expr->Type()->Tag() == TYPE_FUNC ) + { + d->Add(":bro:type:`func`"); + } + else { d->Add("``"); @@ -481,7 +484,11 @@ bool Attributes::DoSerialize(SerialInfo* info) const loop_over_list((*attrs), i) { Attr* a = (*attrs)[i]; - SERIALIZE_OPTIONAL(a->AttrExpr()) + + // Broccoli doesn't support expressions. + Expr* e = (! info->broccoli_peer) ? a->AttrExpr() : 0; + SERIALIZE_OPTIONAL(e); + if ( ! SERIALIZE(char(a->Tag())) ) return false; } diff --git a/src/BroDoc.cc b/src/BroDoc.cc index b84b9d023d..91aed5ec62 100644 --- a/src/BroDoc.cc +++ b/src/BroDoc.cc @@ -215,14 +215,16 @@ void BroDoc::WriteDocFile() const if ( ! port_analysis.empty() ) { WriteSectionHeading("Port Analysis", '-'); - WriteToDoc(":ref:`More Information `\n\n"); + WriteToDoc("Loading this script makes the following changes to " + ":bro:see:`dpd_config`.\n\n"); WriteStringList("%s", port_analysis); } if ( ! packet_filter.empty() ) { WriteSectionHeading("Packet Filter", '-'); - WriteToDoc(":ref:`More Information `\n\n"); + WriteToDoc("Loading this script makes the following changes to " + ":bro:see:`capture_filters`.\n\n"); WriteToDoc("Filters added::\n\n"); WriteToDoc("%s\n", packet_filter.c_str()); } diff --git a/src/BroDocObj.cc b/src/BroDocObj.cc index d9fe16632b..12753ea15d 100644 --- a/src/BroDocObj.cc +++ b/src/BroDocObj.cc @@ -4,9 +4,12 @@ #include "ID.h" #include "BroDocObj.h" +BroDocObj* BroDocObj::last = 0; + BroDocObj::BroDocObj(const ID* id, std::list*& reST, bool is_fake) { + last = this; broID = id; reST_doc_strings = reST; reST = 0; diff --git a/src/BroDocObj.h b/src/BroDocObj.h index 0ad96afa86..cb512f8cda 100644 --- a/src/BroDocObj.h +++ b/src/BroDocObj.h @@ -103,6 +103,20 @@ public: */ int LongestShortDescLen() const; + /** + * Adds a reST documentation string to this BroDocObj's list. + * @param s the documentation string to append. + */ + void AddDocString(const std::string& s) + { + if ( ! reST_doc_strings ) + reST_doc_strings = new std::list(); + reST_doc_strings->push_back(s); + FormulateShortDesc(); + } + + static BroDocObj* last; + protected: std::list* reST_doc_strings; std::list short_desc; diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 897acc9d37..47314514f6 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,4 +1,5 @@ -include_directories(${CMAKE_CURRENT_SOURCE_DIR} +include_directories(BEFORE + ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR} ) diff --git a/src/DNS_Mgr.cc b/src/DNS_Mgr.cc index 736c262222..2b9d07a969 100644 --- a/src/DNS_Mgr.cc +++ b/src/DNS_Mgr.cc @@ -595,8 +595,6 @@ void DNS_Mgr::Resolve() } else --num_pending; - - delete dr; } } diff --git a/src/Desc.cc b/src/Desc.cc index c70878de34..12b4a524eb 100644 --- a/src/Desc.cc +++ b/src/Desc.cc @@ -41,8 +41,7 @@ ODesc::ODesc(desc_type t, BroFile* arg_f) do_flush = 1; include_stats = 0; indent_with_spaces = 0; - escape = 0; - escape_len = 0; + escape = false; } ODesc::~ODesc() @@ -56,10 +55,9 @@ ODesc::~ODesc() free(base); } -void ODesc::SetEscape(const char* arg_escape, int len) +void ODesc::EnableEscaping() { - escape = arg_escape; - escape_len = len; + escape = true; } void ODesc::PushIndent() @@ -228,6 +226,25 @@ static const char* find_first_unprintable(ODesc* d, const char* bytes, unsigned return 0; } +pair ODesc::FirstEscapeLoc(const char* bytes, size_t n) + { + pair p(find_first_unprintable(this, bytes, n), 1); + + string str(bytes, n); + list::const_iterator it; + for ( it = escape_sequences.begin(); it != escape_sequences.end(); ++it ) + { + size_t pos = str.find(*it); + if ( pos != string::npos && (p.first == 0 || bytes + pos < p.first) ) + { + p.first = bytes + pos; + p.second = it->size(); + } + } + + return p; + } + void ODesc::AddBytes(const void* bytes, unsigned int n) { if ( ! escape ) @@ -241,45 +258,30 @@ void ODesc::AddBytes(const void* bytes, unsigned int n) while ( s < e ) { - const char* t1 = (const char*) memchr(s, escape[0], e - s); - - if ( ! t1 ) - t1 = e; - - const char* t2 = find_first_unprintable(this, s, t1 - s); - - if ( t2 && t2 < t1 ) + pair p = FirstEscapeLoc(s, e - s); + if ( p.first ) { - AddBytesRaw(s, t2 - s); - - char hex[6] = "\\x00"; - hex[2] = hex_chars[((*t2) & 0xf0) >> 4]; - hex[3] = hex_chars[(*t2) & 0x0f]; - AddBytesRaw(hex, 4); - - s = t2 + 1; - continue; + AddBytesRaw(s, p.first - s); + if ( p.second == 1 ) + { + char hex[6] = "\\x00"; + hex[2] = hex_chars[((*p.first) & 0xf0) >> 4]; + hex[3] = hex_chars[(*p.first) & 0x0f]; + AddBytesRaw(hex, 4); + } + else + { + string esc_str = get_escaped_string(string(p.first, p.second), true); + AddBytesRaw(esc_str.c_str(), esc_str.size()); + } + s = p.first + p.second; } - - if ( memcmp(t1, escape, escape_len) != 0 ) - break; - - AddBytesRaw(s, t1 - s); - - for ( int i = 0; i < escape_len; ++i ) + else { - char hex[5] = "\\x00"; - hex[2] = hex_chars[((*t1) & 0xf0) >> 4]; - hex[3] = hex_chars[(*t1) & 0x0f]; - AddBytesRaw(hex, 4); - ++t1; + AddBytesRaw(s, e - s); + break; } - - s = t1; } - - if ( s < e ) - AddBytesRaw(s, e - s); } void ODesc::AddBytesRaw(const void* bytes, unsigned int n) diff --git a/src/Desc.h b/src/Desc.h index 4ed05c1763..27cbd4fa01 100644 --- a/src/Desc.h +++ b/src/Desc.h @@ -4,6 +4,8 @@ #define descriptor_h #include +#include +#include #include "BroString.h" typedef enum { @@ -48,8 +50,13 @@ public: void SetFlush(int arg_do_flush) { do_flush = arg_do_flush; } - // The string passed in must remain valid as long as this object lives. - void SetEscape(const char* escape, int len); + void EnableEscaping(); + void AddEscapeSequence(const char* s) { escape_sequences.push_back(s); } + void AddEscapeSequence(const char* s, size_t n) + { escape_sequences.push_back(string(s, n)); } + void RemoveEscapeSequence(const char* s) { escape_sequences.remove(s); } + void RemoveEscapeSequence(const char* s, size_t n) + { escape_sequences.remove(string(s, n)); } void PushIndent(); void PopIndent(); @@ -133,6 +140,19 @@ protected: void OutOfMemory(); + /** + * Returns the location of the first place in the bytes to be hex-escaped. + * + * @param bytes the starting memory address to start searching for + * escapable character. + * @param n the maximum number of bytes to search. + * @return a pair whose first element represents a starting memory address + * to be escaped up to the number of characters indicated by the + * second element. The first element may be 0 if nothing is + * to be escaped. + */ + pair FirstEscapeLoc(const char* bytes, size_t n); + desc_type type; desc_style style; @@ -140,8 +160,8 @@ protected: unsigned int offset; // where we are in the buffer unsigned int size; // size of buffer in bytes - int escape_len; // number of bytes in to escape sequence - const char* escape; // bytes to escape on output + bool escape; // escape unprintable characters in output? + list escape_sequences; // additional sequences of chars to escape BroFile* f; // or the file we're using. diff --git a/src/Expr.cc b/src/Expr.cc index f6d1fc568e..c34c44a7d1 100644 --- a/src/Expr.cc +++ b/src/Expr.cc @@ -359,7 +359,7 @@ bool NameExpr::DoUnserialize(UnserialInfo* info) if ( id ) ::Ref(id); else - reporter->Warning("unserialized unknown global name"); + reporter->Warning("configuration changed: unserialized unknown global name from persistent state"); delete [] name; } @@ -4053,7 +4053,15 @@ Val* RecordCoerceExpr::Fold(Val* v) const val->Assign(i, rhs); } else - val->Assign(i, 0); + { + const Attr* def = + Type()->AsRecordType()->FieldDecl(i)->FindAttr(ATTR_DEFAULT); + + if ( def ) + val->Assign(i, def->AttrExpr()->Eval(0)); + else + val->Assign(i, 0); + } } return val; diff --git a/src/LogMgr.cc b/src/LogMgr.cc index ab1706ca72..28e9a2ac1f 100644 --- a/src/LogMgr.cc +++ b/src/LogMgr.cc @@ -81,16 +81,18 @@ struct LogMgr::Stream { bool LogField::Read(SerializationFormat* fmt) { int t; + int st; - bool success = (fmt->Read(&name, "name") && fmt->Read(&t, "type")); + bool success = (fmt->Read(&name, "name") && fmt->Read(&t, "type") && fmt->Read(&st, "subtype") ); type = (TypeTag) t; + subtype = (TypeTag) st; return success; } bool LogField::Write(SerializationFormat* fmt) const { - return (fmt->Write(name, "name") && fmt->Write((int)type, "type")); + return (fmt->Write(name, "name") && fmt->Write((int)type, "type") && fmt->Write((int)subtype, "subtype")); } LogVal::~LogVal() @@ -157,7 +159,7 @@ bool LogVal::IsCompatibleType(BroType* t, bool atomic_only) if ( atomic_only ) return false; - return IsCompatibleType(t->AsVectorType()->YieldType()); + return IsCompatibleType(t->AsVectorType()->YieldType(), true); } default: @@ -707,6 +709,14 @@ bool LogMgr::TraverseRecord(Stream* stream, Filter* filter, RecordType* rt, LogField* field = new LogField(); field->name = new_path; field->type = t->Tag(); + if ( field->type == TYPE_TABLE ) + { + field->subtype = t->AsSetType()->Indices()->PureType()->Tag(); + } + else if ( field->type == TYPE_VECTOR ) + { + field->subtype = t->AsVectorType()->YieldType()->Tag(); + } filter->fields[filter->num_fields - 1] = field; } diff --git a/src/LogMgr.h b/src/LogMgr.h index 10530960cb..3eaba360d5 100644 --- a/src/LogMgr.h +++ b/src/LogMgr.h @@ -15,10 +15,12 @@ class SerializationFormat; struct LogField { string name; TypeTag type; + // inner type of sets + TypeTag subtype; - LogField() { } + LogField() { subtype = TYPE_VOID; } LogField(const LogField& other) - : name(other.name), type(other.type) { } + : name(other.name), type(other.type), subtype(other.subtype) { } // (Un-)serialize. bool Read(SerializationFormat* fmt); diff --git a/src/LogWriterAscii.cc b/src/LogWriterAscii.cc index 9fc71789d8..d2c1d91370 100644 --- a/src/LogWriterAscii.cc +++ b/src/LogWriterAscii.cc @@ -6,27 +6,6 @@ #include "LogWriterAscii.h" #include "NetVar.h" -/** - * Takes a string, escapes each character into its equivalent hex code (\x##), and - * returns a string containing all escaped values. - * - * @param str string to escape - * @return A std::string containing a list of escaped hex values of the form \x## - */ -static string get_escaped_string(const std::string& str) -{ - char tbuf[16]; - string esc = ""; - - for ( size_t i = 0; i < str.length(); ++i ) - { - snprintf(tbuf, sizeof(tbuf), "\\x%02x", str[i]); - esc += tbuf; - } - - return esc; -} - LogWriterAscii::LogWriterAscii() { file = 0; @@ -59,7 +38,8 @@ LogWriterAscii::LogWriterAscii() memcpy(header_prefix, BifConst::LogAscii::header_prefix->Bytes(), header_prefix_len); - desc.SetEscape(separator, separator_len); + desc.EnableEscaping(); + desc.AddEscapeSequence(separator, separator_len); } LogWriterAscii::~LogWriterAscii() @@ -88,7 +68,7 @@ bool LogWriterAscii::DoInit(string path, int num_fields, if ( output_to_stdout ) path = "/dev/stdout"; - fname = IsSpecial(path) ? path : path + ".log"; + fname = IsSpecial(path) ? path : path + "." + LogExt(); if ( ! (file = fopen(fname.c_str(), "w")) ) { @@ -102,13 +82,19 @@ bool LogWriterAscii::DoInit(string path, int num_fields, { string str = string(header_prefix, header_prefix_len) + "separator " // Always use space as separator here. - + get_escaped_string(string(separator, separator_len)) + + get_escaped_string(string(separator, separator_len), false) + "\n"; if( fwrite(str.c_str(), str.length(), 1, file) != 1 ) goto write_error; - if ( ! WriteHeaderField("path", path) ) + if ( ! (WriteHeaderField("set_separator", get_escaped_string( + string(set_separator, set_separator_len), false)) && + WriteHeaderField("empty_field", get_escaped_string( + string(empty_field, empty_field_len), false)) && + WriteHeaderField("unset_field", get_escaped_string( + string(unset_field, unset_field_len), false)) && + WriteHeaderField("path", get_escaped_string(path, false))) ) goto write_error; string names; @@ -125,6 +111,12 @@ bool LogWriterAscii::DoInit(string path, int num_fields, const LogField* field = fields[i]; names += field->name; types += type_name(field->type); + if ( (field->type == TYPE_TABLE) || (field->type == TYPE_VECTOR) ) + { + types += "["; + types += type_name(field->subtype); + types += "]"; + } } if ( ! (WriteHeaderField("fields", names) @@ -200,10 +192,33 @@ bool LogWriterAscii::DoWriteOne(ODesc* desc, LogVal* val, const LogField* field) case TYPE_FUNC: { int size = val->val.string_val->size(); - if ( size ) - desc->AddN(val->val.string_val->data(), val->val.string_val->size()); - else + const char* data = val->val.string_val->data(); + + if ( ! size ) + { desc->AddN(empty_field, empty_field_len); + break; + } + + if ( size == unset_field_len && memcmp(data, unset_field, size) == 0 ) + { + // The value we'd write out would match exactly the + // place-holder we use for unset optional fields. We + // escape the first character so that the output + // won't be ambigious. + static const char hex_chars[] = "0123456789abcdef"; + char hex[6] = "\\x00"; + hex[2] = hex_chars[((*data) & 0xf0) >> 4]; + hex[3] = hex_chars[(*data) & 0x0f]; + desc->AddRaw(hex, 4); + + ++data; + --size; + } + + if ( size ) + desc->AddN(data, size); + break; } @@ -215,14 +230,19 @@ bool LogWriterAscii::DoWriteOne(ODesc* desc, LogVal* val, const LogField* field) break; } + desc->AddEscapeSequence(set_separator, set_separator_len); for ( int j = 0; j < val->val.set_val.size; j++ ) { if ( j > 0 ) - desc->AddN(set_separator, set_separator_len); + desc->AddRaw(set_separator, set_separator_len); if ( ! DoWriteOne(desc, val->val.set_val.vals[j], field) ) + { + desc->RemoveEscapeSequence(set_separator, set_separator_len); return false; + } } + desc->RemoveEscapeSequence(set_separator, set_separator_len); break; } @@ -235,14 +255,19 @@ bool LogWriterAscii::DoWriteOne(ODesc* desc, LogVal* val, const LogField* field) break; } + desc->AddEscapeSequence(set_separator, set_separator_len); for ( int j = 0; j < val->val.vector_val.size; j++ ) { if ( j > 0 ) - desc->AddN(set_separator, set_separator_len); + desc->AddRaw(set_separator, set_separator_len); if ( ! DoWriteOne(desc, val->val.vector_val.vals[j], field) ) + { + desc->RemoveEscapeSequence(set_separator, set_separator_len); return false; + } } + desc->RemoveEscapeSequence(set_separator, set_separator_len); break; } @@ -297,7 +322,7 @@ bool LogWriterAscii::DoRotate(string rotated_path, double open, fclose(file); file = 0; - string nname = rotated_path + ".log"; + string nname = rotated_path + "." + LogExt(); rename(fname.c_str(), nname.c_str()); if ( ! FinishedRotation(nname, fname, open, close, terminating) ) @@ -315,4 +340,9 @@ bool LogWriterAscii::DoSetBuf(bool enabled) return true; } - +string LogWriterAscii::LogExt() + { + const char* ext = getenv("BRO_LOG_SUFFIX"); + if ( ! ext ) ext = "log"; + return ext; + } diff --git a/src/LogWriterAscii.h b/src/LogWriterAscii.h index 7755f71d06..72127c8b1f 100644 --- a/src/LogWriterAscii.h +++ b/src/LogWriterAscii.h @@ -13,6 +13,7 @@ public: ~LogWriterAscii(); static LogWriter* Instantiate() { return new LogWriterAscii; } + static string LogExt(); protected: virtual bool DoInit(string path, int num_fields, diff --git a/src/RemoteSerializer.cc b/src/RemoteSerializer.cc index a21a7abc60..b72a6dcc1a 100644 --- a/src/RemoteSerializer.cc +++ b/src/RemoteSerializer.cc @@ -385,6 +385,9 @@ inline void RemoteSerializer::SetupSerialInfo(SerialInfo* info, Peer* peer) peer->phase == Peer::RUNNING ) info->new_cache_strategy = true; + if ( (peer->caps & Peer::BROCCOLI_PEER) ) + info->broccoli_peer = true; + info->include_locations = false; } @@ -1457,7 +1460,7 @@ void RemoteSerializer::Finish() Poll(true); while ( io->CanWrite() ); - loop_over_list(peers, i) + loop_over_list(peers, i) { CloseConnection(peers[i]); } @@ -2113,6 +2116,9 @@ bool RemoteSerializer::HandshakeDone(Peer* peer) if ( (peer->caps & Peer::NEW_CACHE_STRATEGY) ) Log(LogInfo, "peer supports keep-in-cache; using that", peer); + if ( (peer->caps & Peer::BROCCOLI_PEER) ) + Log(LogInfo, "peer is a Broccoli", peer); + if ( peer->logs_requested ) log_mgr->SendAllWritersTo(peer->id); @@ -2365,6 +2371,9 @@ bool RemoteSerializer::ProcessSerialization() current_peer->phase == Peer::RUNNING ) info.new_cache_strategy = true; + if ( current_peer->caps & Peer::BROCCOLI_PEER ) + info.broccoli_peer = true; + if ( ! forward_remote_state_changes ) ignore_accesses = true; @@ -2923,25 +2932,37 @@ void RemoteSerializer::Log(LogLevel level, const char* msg) void RemoteSerializer::Log(LogLevel level, const char* msg, Peer* peer, LogSrc src) { + if ( peer ) + { + val_list* vl = new val_list(); + vl->append(peer->val->Ref()); + vl->append(new Val(level, TYPE_COUNT)); + vl->append(new Val(src, TYPE_COUNT)); + vl->append(new StringVal(msg)); + mgr.QueueEvent(remote_log_peer, vl); + } + else + { + val_list* vl = new val_list(); + vl->append(new Val(level, TYPE_COUNT)); + vl->append(new Val(src, TYPE_COUNT)); + vl->append(new StringVal(msg)); + mgr.QueueEvent(remote_log, vl); + } + +#ifdef DEBUG const int BUFSIZE = 1024; char buffer[BUFSIZE]; - int len = 0; if ( peer ) - len += snprintf(buffer + len, sizeof(buffer) - len, - "[#%d/%s:%d] ", int(peer->id), ip2a(peer->ip), - peer->port); + len += snprintf(buffer + len, sizeof(buffer) - len, "[#%d/%s:%d] ", + int(peer->id), ip2a(peer->ip), peer->port); len += safe_snprintf(buffer + len, sizeof(buffer) - len, "%s", msg); - val_list* vl = new val_list(); - vl->append(new Val(level, TYPE_COUNT)); - vl->append(new Val(src, TYPE_COUNT)); - vl->append(new StringVal(buffer)); - mgr.QueueEvent(remote_log, vl); - DEBUG_COMM(fmt("parent: %.6f %s", current_time(), buffer)); +#endif } void RemoteSerializer::RaiseEvent(EventHandlerPtr event, Peer* peer, diff --git a/src/RemoteSerializer.h b/src/RemoteSerializer.h index f849a6a2b5..b64fdcbe66 100644 --- a/src/RemoteSerializer.h +++ b/src/RemoteSerializer.h @@ -198,6 +198,7 @@ protected: static const int NO_CACHING = 2; static const int PID_64BIT = 4; static const int NEW_CACHE_STRATEGY = 8; + static const int BROCCOLI_PEER = 16; // Constants to remember to who did something. static const int NONE = 0; diff --git a/src/SerialInfo.h b/src/SerialInfo.h index d322aa4b37..aa4c382349 100644 --- a/src/SerialInfo.h +++ b/src/SerialInfo.h @@ -15,6 +15,7 @@ public: pid_32bit = false; include_locations = true; new_cache_strategy = false; + broccoli_peer = false; } SerialInfo(const SerialInfo& info) @@ -28,6 +29,7 @@ public: pid_32bit = info.pid_32bit; include_locations = info.include_locations; new_cache_strategy = info.new_cache_strategy; + broccoli_peer = info.broccoli_peer; } // Parameters that control serialization. @@ -46,6 +48,11 @@ public: // If true, we support keeping objs in cache permanently. bool new_cache_strategy; + // If true, we're connecting to a Broccoli. If so, serialization + // specifics may be adapted for functionality Broccoli does not + // support. + bool broccoli_peer; + ChunkedIO::Chunk* chunk; // chunk written right before the serialization // Attributes set during serialization. @@ -70,6 +77,7 @@ public: print = 0; pid_32bit = false; new_cache_strategy = false; + broccoli_peer = false; } UnserialInfo(const UnserialInfo& info) @@ -86,6 +94,7 @@ public: print = info.print; pid_32bit = info.pid_32bit; new_cache_strategy = info.new_cache_strategy; + broccoli_peer = info.broccoli_peer; } // Parameters that control unserialization. @@ -106,6 +115,11 @@ public: // If true, we support keeping objs in cache permanently. bool new_cache_strategy; + // If true, we're connecting to a Broccoli. If so, serialization + // specifics may be adapted for functionality Broccoli does not + // support. + bool broccoli_peer; + // If a global ID already exits, of these policies is used. enum { Keep, // keep the old ID and ignore the new diff --git a/src/Serializer.h b/src/Serializer.h index db09cc837f..93581d83ce 100644 --- a/src/Serializer.h +++ b/src/Serializer.h @@ -121,7 +121,7 @@ protected: // This will be increased whenever there is an incompatible change // in the data format. - static const uint32 DATA_FORMAT_VERSION = 20; + static const uint32 DATA_FORMAT_VERSION = 21; ChunkedIO* io; diff --git a/src/Type.cc b/src/Type.cc index cd40583aae..4d80eda6f7 100644 --- a/src/Type.cc +++ b/src/Type.cc @@ -876,74 +876,12 @@ void CommentedTypeDecl::DescribeReST(ODesc* d) const } } -RecordField::RecordField(int arg_base, int arg_offset, int arg_total_offset) - { - base = arg_base; - offset = arg_offset; - total_offset = arg_total_offset; - } - RecordType::RecordType(type_decl_list* arg_types) : BroType(TYPE_RECORD) { types = arg_types; - base = 0; - fields = 0; num_fields = types ? types->length() : 0; } -RecordType::RecordType(TypeList* arg_base, type_decl_list* refinements) - : BroType(TYPE_RECORD) - { - if ( refinements ) - arg_base->Append(new RecordType(refinements)); - - Init(arg_base); - } - -void RecordType::Init(TypeList* arg_base) - { - assert(false); // Is this ever used? - - base = arg_base; - - if ( ! base ) - Internal("empty RecordType"); - - fields = new PDict(RecordField)(ORDERED); - types = 0; - - type_list* t = base->Types(); - - loop_over_list(*t, i) - { - BroType* ti = (*t)[i]; - - if ( ti->Tag() != TYPE_RECORD ) - (*t)[i]->Error("non-record in base type list"); - - RecordType* rti = ti->AsRecordType(); - int n = rti->NumFields(); - - for ( int j = 0; j < n; ++j ) - { - const TypeDecl* tdij = rti->FieldDecl(j); - - if ( fields->Lookup(tdij->id) ) - { - reporter->Error("duplicate field %s", tdij->id); - continue; - } - - RecordField* rf = new RecordField(i, j, fields->Length()); - - if ( fields->Insert(tdij->id, rf) ) - Internal("duplicate field when constructing record"); - } - } - - num_fields = fields->Length(); - } - RecordType::~RecordType() { if ( types ) @@ -953,9 +891,6 @@ RecordType::~RecordType() delete types; } - - delete fields; - Unref(base); } int RecordType::HasField(const char* field) const @@ -971,17 +906,7 @@ BroType* RecordType::FieldType(const char* field) const BroType* RecordType::FieldType(int field) const { - if ( types ) - return (*types)[field]->type; - else - { - RecordField* rf = fields->NthEntry(field); - if ( ! rf ) - Internal("missing field in RecordType::FieldType"); - BroType* bt = (*base->Types())[rf->base]; - RecordType* rbt = bt->AsRecordType(); - return rbt->FieldType(rf->offset); - } + return (*types)[field]->type; } Val* RecordType::FieldDefault(int field) const @@ -998,26 +923,14 @@ Val* RecordType::FieldDefault(int field) const int RecordType::FieldOffset(const char* field) const { - if ( types ) + loop_over_list(*types, i) { - loop_over_list(*types, i) - { - TypeDecl* td = (*types)[i]; - if ( streq(td->id, field) ) - return i; - } - - return -1; + TypeDecl* td = (*types)[i]; + if ( streq(td->id, field) ) + return i; } - else - { - RecordField* rf = fields->Lookup(field); - if ( ! rf ) - return -1; - else - return rf->total_offset; - } + return -1; } const char* RecordType::FieldName(int field) const @@ -1027,33 +940,12 @@ const char* RecordType::FieldName(int field) const const TypeDecl* RecordType::FieldDecl(int field) const { - if ( types ) - return (*types)[field]; - else - { - RecordField* rf = fields->NthEntry(field); - if ( ! rf ) - reporter->InternalError("missing field in RecordType::FieldDecl"); - - BroType* bt = (*base->Types())[rf->base]; - RecordType* rbt = bt->AsRecordType(); - return rbt->FieldDecl(rf->offset); - } + return (*types)[field]; } TypeDecl* RecordType::FieldDecl(int field) { - if ( types ) - return (*types)[field]; - else - { - RecordField* rf = fields->NthEntry(field); - if ( ! rf ) - Internal("missing field in RecordType::FieldDecl"); - BroType* bt = (*base->Types())[rf->base]; - RecordType* rbt = bt->AsRecordType(); - return rbt->FieldDecl(rf->offset); - } + return (*types)[field]; } void RecordType::Describe(ODesc* d) const @@ -1151,11 +1043,6 @@ void RecordType::DescribeFields(ODesc* d) const d->SP(); } } - else - { - d->AddCount(1); - base->Describe(d); - } } } @@ -1208,9 +1095,6 @@ bool RecordType::DoSerialize(SerialInfo* info) const else if ( ! SERIALIZE(false) ) return false; - SERIALIZE_OPTIONAL(base); - - // We don't serialize the fields as we can reconstruct them. return true; } @@ -1245,13 +1129,6 @@ bool RecordType::DoUnserialize(UnserialInfo* info) else types = 0; - BroType* type; - UNSERIALIZE_OPTIONAL(type, BroType::Unserialize(info, TYPE_LIST)); - base = (TypeList*) type; - - if ( base ) - Init(base); - return true; } @@ -1594,21 +1471,6 @@ bool VectorType::DoUnserialize(UnserialInfo* info) return yield_type != 0; } -BroType* refine_type(TypeList* base, type_decl_list* refinements) - { - type_list* t = base->Types(); - - if ( t->length() == 1 && ! refinements ) - { // Just a direct reference to a single type. - BroType* rt = (*t)[0]->Ref(); - Unref(base); - return rt; - } - - return new RecordType(base, refinements); - } - - BroType* base_type(TypeTag tag) { static BroType* base_types[NUM_TYPES]; diff --git a/src/Type.h b/src/Type.h index 5ebc5761a3..e935ba2267 100644 --- a/src/Type.h +++ b/src/Type.h @@ -426,20 +426,9 @@ public: std::list* comments; }; -class RecordField { -public: - RecordField(int arg_base, int arg_offset, int arg_total_offset); - - int base; // which base element it belongs to - int offset; // where it is in that base - int total_offset; // where it is in the aggregate record -}; -declare(PDict,RecordField); - class RecordType : public BroType { public: RecordType(type_decl_list* types); - RecordType(TypeList* base, type_decl_list* refinements); ~RecordType(); @@ -473,15 +462,11 @@ public: void DescribeFieldsReST(ODesc* d, bool func_args) const; protected: - RecordType() { fields = 0; base = 0; types = 0; } - - void Init(TypeList* arg_base); + RecordType() { types = 0; } DECLARE_SERIAL(RecordType) int num_fields; - PDict(RecordField)* fields; - TypeList* base; type_decl_list* types; }; @@ -587,10 +572,6 @@ protected: BroType* yield_type; }; - -// Returns the given type refinement, or error_type() if it's illegal. -extern BroType* refine_type(TypeList* base, type_decl_list* refinements); - // Returns the BRO basic (non-parameterized) type with the given type. extern BroType* base_type(TypeTag tag); diff --git a/src/event.bif b/src/event.bif index 289d781819..3f0e2992ed 100644 --- a/src/event.bif +++ b/src/event.bif @@ -279,14 +279,13 @@ event ssh_server_version%(c: connection, version: string%); event ssl_client_hello%(c: connection, version: count, possible_ts: time, session_id: string, ciphers: count_set%); event ssl_server_hello%(c: connection, version: count, possible_ts: time, session_id: string, cipher: count, comp_method: count%); -event ssl_extension%(c: connection, code: count, val: string%); -event ssl_session_ticket_handshake%(c: connection, ticket_lifetime_hint: count, ticket: string%); +event ssl_extension%(c: connection, is_orig: bool, code: count, val: string%); +event ssl_alert%(c: connection, is_orig: bool, level: count, desc: count%); event ssl_established%(c: connection%); -event ssl_alert%(c: connection, level: count, desc: count%); -event x509_certificate%(c: connection, cert: X509, is_server: bool, chain_idx: count, chain_len: count, der_cert: string%); -event x509_extension%(c: connection, data: string%); -event x509_error%(c: connection, err: count%); +event x509_certificate%(c: connection, is_orig: bool, cert: X509, chain_idx: count, chain_len: count, der_cert: string%); +event x509_extension%(c: connection, is_orig: bool, data: string%); +event x509_error%(c: connection, is_orig: bool, err: count%); event stp_create_endp%(c: connection, e: int, is_orig: bool%); event stp_resume_endp%(e: int%); @@ -445,6 +444,29 @@ event remote_state_inconsistency%(operation: string, id: string, # Generated for communication log message. event remote_log%(level: count, src: count, msg: string%); +## Generated for communication log messages. While this event is +## intended primarily for use by Bro's communication framework, it can also trigger +## additional code if helpful. This event is equivalent to +## :bro:see:`remote_log` except the message is with respect to a certain peer. +## +## p: A record describing the remote peer. +## +## level: The log level, which is either :bro:enum:`REMOTE_LOG_INFO` or +## :bro:enum:`REMOTE_LOG_ERROR`. +## +## src: The component of the comminication system that logged the message. +## Currently, this will be one of :bro:enum:`REMOTE_SRC_CHILD` (Bro's +## child process), :bro:enum:`REMOTE_SRC_PARENT` (Bro's main process), or +## :bro:enum:`REMOTE_SRC_SCRIPT` (the script level). +## +## msg: The message logged. +## +## .. bro:see:: remote_capture_filter remote_connection_closed remote_connection_error +## remote_connection_established remote_connection_handshake_done +## remote_event_registered remote_pong remote_state_access_performed +## remote_state_inconsistency print_hook remote_log +event remote_log_peer%(p: event_peer, level: count, src: count, msg: string%); + # Generated when a remote peer has answered to our ping. event remote_pong%(p: event_peer, seq: count, d1: interval, d2: interval, d3: interval%); diff --git a/src/main.cc b/src/main.cc index dfa46c3050..b4a27862c9 100644 --- a/src/main.cc +++ b/src/main.cc @@ -47,6 +47,7 @@ extern "C" void OPENSSL_add_all_algorithms_conf(void); #include "ConnCompressor.h" #include "DPM.h" #include "BroDoc.h" +#include "LogWriterAscii.h" #include "binpac_bro.h" @@ -194,6 +195,7 @@ void usage() fprintf(stderr, " $BRO_PREFIXES | prefix list (%s)\n", bro_prefixes()); fprintf(stderr, " $BRO_DNS_FAKE | disable DNS lookups (%s)\n", bro_dns_fake()); fprintf(stderr, " $BRO_SEED_FILE | file to load seeds from (not set)\n"); + fprintf(stderr, " $BRO_LOG_SUFFIX | ASCII log file extension (.%s)\n", LogWriterAscii::LogExt().c_str()); exit(1); } diff --git a/src/parse.y b/src/parse.y index 495931aae0..7ab6c6bd32 100644 --- a/src/parse.y +++ b/src/parse.y @@ -2,7 +2,7 @@ // See the file "COPYING" in the main distribution directory for copyright. %} -%expect 88 +%expect 87 %token TOK_ADD TOK_ADD_TO TOK_ADDR TOK_ANY %token TOK_ATENDIF TOK_ATELSE TOK_ATIF TOK_ATIFDEF TOK_ATIFNDEF @@ -53,7 +53,7 @@ %type expr init anonymous_function %type event %type stmt stmt_list func_body for_head -%type type opt_type refined_type enum_body +%type type opt_type enum_body %type func_hdr func_params %type type_list %type type_decl formal_args_decl @@ -1104,7 +1104,7 @@ decl: } } - | TOK_TYPE global_id ':' refined_type opt_attr ';' + | TOK_TYPE global_id ':' type opt_attr ';' { add_type($2, $4, $5, 0); @@ -1134,7 +1134,7 @@ decl: } } - | TOK_EVENT event_id ':' refined_type opt_attr ';' + | TOK_EVENT event_id ':' type_list opt_attr ';' { add_type($2, $4, $5, 1); @@ -1220,13 +1220,6 @@ func_params: { $$ = new FuncType($2, base_type(TYPE_VOID), 0); } ; -refined_type: - type_list '{' type_decl_list '}' - { $$ = refine_type($1, $3); } - | type_list - { $$ = refine_type($1, 0); } - ; - opt_type: ':' type { $$ = $2; } diff --git a/src/scan.l b/src/scan.l index 7ebd7894e1..623e0d2ed6 100644 --- a/src/scan.l +++ b/src/scan.l @@ -167,7 +167,7 @@ ESCSEQ (\\([^\n]|[0-7]+|x[[:xdigit:]]+)) return TOK_POST_DOC; } -##{OWS}{ID}:.* { +##{OWS}{ID}:{WS}.* { const char* id_start = skip_whitespace(yytext + 2); yylval.str = copy_string(canon_doc_func_param(id_start).c_str()); return TOK_DOC; @@ -181,7 +181,7 @@ ESCSEQ (\\([^\n]|[0-7]+|x[[:xdigit:]]+)) } } -##{OWS}{ID}:.* { +##{OWS}{ID}:{WS}.* { if ( generate_documentation ) { // Comment is documenting either a function parameter or return type, @@ -201,6 +201,11 @@ ESCSEQ (\\([^\n]|[0-7]+|x[[:xdigit:]]+)) } } +##<.* { + if ( generate_documentation && BroDocObj::last ) + BroDocObj::last->AddDocString(canon_doc_comment(yytext + 3)); +} + ##.* { if ( generate_documentation && (yytext[2] != '#') ) { diff --git a/src/ssl-analyzer.pac b/src/ssl-analyzer.pac index 5440e0f4d5..40b312d9a1 100644 --- a/src/ssl-analyzer.pac +++ b/src/ssl-analyzer.pac @@ -22,11 +22,17 @@ } }; + string orig_label(bool is_orig); void free_X509(void *); X509* d2i_X509_binpac(X509** px, const uint8** in, int len); %} %code{ +string orig_label(bool is_orig) + { + return string(is_orig ? "originator" :"responder"); + } + void free_X509(void* cert) { X509_free((X509*) cert); @@ -117,14 +123,14 @@ refine connection SSL_Conn += { function proc_alert(rec: SSLRecord, level : int, desc : int) : bool %{ BifEvent::generate_ssl_alert(bro_analyzer(), bro_analyzer()->Conn(), - level, desc); + ${rec.is_orig}, level, desc); return true; %} function proc_client_hello(rec: SSLRecord, version : uint16, ts : double, session_id : uint8[], - cipher_suites16 : uint16[], + cipher_suites16 : uint16[], cipher_suites24 : uint24[]) : bool %{ if ( state_ == STATE_TRACK_LOST ) @@ -150,15 +156,15 @@ refine connection SSL_Conn += { cipher_set->Assign(ciph, 0); Unref(ciph); } - + BifEvent::generate_ssl_client_hello(bro_analyzer(), bro_analyzer()->Conn(), version, ts, to_string_val(session_id), cipher_set); - + delete cipher_suites; } - + return true; %} @@ -187,16 +193,16 @@ refine connection SSL_Conn += { std::copy(cipher_suites16->begin(), cipher_suites16->end(), std::back_inserter(*ciphers)); else std::transform(cipher_suites24->begin(), cipher_suites24->end(), std::back_inserter(*ciphers), to_int()); - + BifEvent::generate_ssl_server_hello(bro_analyzer(), bro_analyzer()->Conn(), version, ts, to_string_val(session_id), ciphers->size()==0 ? 0 : ciphers->at(0), comp_method); - + delete ciphers; } - + return true; %} @@ -212,11 +218,11 @@ refine connection SSL_Conn += { return true; %} - function proc_ssl_extension(type: int, data: bytestring) : bool + function proc_ssl_extension(rec: SSLRecord, type: int, data: bytestring) : bool %{ if ( ssl_extension ) BifEvent::generate_ssl_extension(bro_analyzer(), - bro_analyzer()->Conn(), type, + bro_analyzer()->Conn(), ${rec.is_orig}, type, new StringVal(data.length(), (const char*) data.data())); return true; %} @@ -234,7 +240,7 @@ refine connection SSL_Conn += { if ( x509_certificate ) { STACK_OF(X509)* untrusted_certs = 0; - + for ( unsigned int i = 0; i < certificates->size(); ++i ) { const bytestring& cert = (*certificates)[i]; @@ -243,7 +249,7 @@ refine connection SSL_Conn += { if ( ! pTemp ) { BifEvent::generate_x509_error(bro_analyzer(), bro_analyzer()->Conn(), - ERR_get_error()); + ${rec.is_orig}, ERR_get_error()); return false; } @@ -269,8 +275,8 @@ refine connection SSL_Conn += { StringVal* der_cert = new StringVal(cert.length(), (const char*) cert.data()); BifEvent::generate_x509_certificate(bro_analyzer(), bro_analyzer()->Conn(), + ${rec.is_orig}, pX509Cert, - ! ${rec.is_orig}, i, certificates->size(), der_cert); @@ -297,7 +303,7 @@ refine connection SSL_Conn += { StringVal* value = new StringVal(length, (char*)pBuffer); BifEvent::generate_x509_extension(bro_analyzer(), - bro_analyzer()->Conn(), value); + bro_analyzer()->Conn(), ${rec.is_orig}, value); OPENSSL_free(pBuffer); } } @@ -462,5 +468,5 @@ refine typeattr CiphertextRecord += &let { } refine typeattr SSLExtension += &let { - proc : bool = $context.connection.proc_ssl_extension(type, data); + proc : bool = $context.connection.proc_ssl_extension(rec, type, data); }; diff --git a/src/ssl-protocol.pac b/src/ssl-protocol.pac index bd5b692b80..627645e4da 100644 --- a/src/ssl-protocol.pac +++ b/src/ssl-protocol.pac @@ -22,7 +22,6 @@ type uint24 = record { }; string state_label(int state_nr); - string orig_label(bool is_orig); double get_time_from_asn1(const ASN1_TIME * atime); string handshake_type_label(int type); %} @@ -35,7 +34,7 @@ type SSLRecord(is_orig: bool) = record { head2 : uint8; head3 : uint8; head4 : uint8; - rec : RecordText(this, is_orig)[] &length=length, &requires(content_type); + rec : RecordText(this)[] &length=length, &requires(content_type); } &length = length+5, &byteorder=bigendian, &let { version : int = @@ -54,7 +53,7 @@ type SSLRecord(is_orig: bool) = record { }; }; -type RecordText(rec: SSLRecord, is_orig: bool) = case $context.connection.state() of { +type RecordText(rec: SSLRecord) = case $context.connection.state() of { STATE_ABBREV_SERVER_ENCRYPTED, STATE_CLIENT_ENCRYPTED, STATE_COMM_ENCRYPTED, STATE_CONN_ESTABLISHED -> ciphertext : CiphertextRecord(rec); @@ -74,7 +73,7 @@ type PlaintextRecord(rec: SSLRecord) = case rec.content_type of { default -> unknown_record : UnknownRecord(rec); }; -type SSLExtension = record { +type SSLExtension(rec: SSLRecord) = record { type: uint16; data_len: uint16; data: bytestring &length=data_len; @@ -149,10 +148,6 @@ enum AnalyzerState { } } - string orig_label(bool is_orig) - { - return string(is_orig ? "originator" :"responder"); - } double get_time_from_asn1(const ASN1_TIME * atime) { @@ -384,7 +379,7 @@ type ClientHello(rec: SSLRecord) = record { # This weirdness is to deal with the possible existence or absence # of the following fields. ext_len: uint16[] &until($element == 0 || $element != 0); - extensions : SSLExtension[] &until($input.length() == 0); + extensions : SSLExtension(rec)[] &until($input.length() == 0); } &let { state_changed : bool = $context.connection.transition(STATE_INITIAL, diff --git a/src/util.cc b/src/util.cc index f81eff8f22..171756fc1c 100644 --- a/src/util.cc +++ b/src/util.cc @@ -41,6 +41,37 @@ #include "Net.h" #include "Reporter.h" +/** + * Takes a string, escapes characters into equivalent hex codes (\x##), and + * returns a string containing all escaped values. + * + * @param str string to escape + * @param escape_all If true, all characters are escaped. If false, only + * characters are escaped that are either whitespace or not printable in + * ASCII. + * @return A std::string containing a list of escaped hex values of the form + * \x## */ +std::string get_escaped_string(const std::string& str, bool escape_all) +{ + char tbuf[16]; + string esc = ""; + + for ( size_t i = 0; i < str.length(); ++i ) + { + char c = str[i]; + + if ( escape_all || isspace(c) || ! isascii(c) || ! isprint(c) ) + { + snprintf(tbuf, sizeof(tbuf), "\\x%02x", str[i]); + esc += tbuf; + } + else + esc += c; + } + + return esc; +} + char* copy_string(const char* s) { char* c = new char[strlen(s)+1]; diff --git a/src/util.h b/src/util.h index 6e76b0f61f..498bdf00e4 100644 --- a/src/util.h +++ b/src/util.h @@ -89,6 +89,8 @@ void delete_each(T* t) delete *it; } +std::string get_escaped_string(const std::string& str, bool escape_all); + extern char* copy_string(const char* s); extern int streq(const char* s1, const char* s2); diff --git a/testing/btest/Baseline/bifs.records_fields/out b/testing/btest/Baseline/bifs.records_fields/out index b221230fc0..0d52e64255 100644 --- a/testing/btest/Baseline/bifs.records_fields/out +++ b/testing/btest/Baseline/bifs.records_fields/out @@ -1,6 +1,6 @@ -[a=42, b=, c=, d=Bar] +[a=42, b=Foo, c=, d=Bar] { -[b] = [type_name=record, log=F, value=, default_val=Foo], +[b] = [type_name=record, log=F, value=Foo, default_val=Foo], [d] = [type_name=record, log=T, value=Bar, default_val=], [c] = [type_name=record, log=F, value=, default_val=], [a] = [type_name=record, log=F, value=42, default_val=] diff --git a/testing/btest/Baseline/core.dns-init/output b/testing/btest/Baseline/core.dns-init/output new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testing/btest/Baseline/core.expr-exception/reporter.log b/testing/btest/Baseline/core.expr-exception/reporter.log index 2dfe6b7b8e..3767de37d8 100644 --- a/testing/btest/Baseline/core.expr-exception/reporter.log +++ b/testing/btest/Baseline/core.expr-exception/reporter.log @@ -1,13 +1,16 @@ #separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - #path reporter #fields ts level message location #types time enum string string -1300475168.783842 Reporter::ERROR field value missing [c$ftp] /home/robin/bro/master/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 8 -1300475168.915940 Reporter::ERROR field value missing [c$ftp] /home/robin/bro/master/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 8 -1300475168.916118 Reporter::ERROR field value missing [c$ftp] /home/robin/bro/master/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 8 -1300475168.918295 Reporter::ERROR field value missing [c$ftp] /home/robin/bro/master/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 8 -1300475168.952193 Reporter::ERROR field value missing [c$ftp] /home/robin/bro/master/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 8 -1300475168.952228 Reporter::ERROR field value missing [c$ftp] /home/robin/bro/master/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 8 -1300475168.954761 Reporter::ERROR field value missing [c$ftp] /home/robin/bro/master/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 8 -1300475168.962628 Reporter::ERROR field value missing [c$ftp] /home/robin/bro/master/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 8 -1300475169.780331 Reporter::ERROR field value missing [c$ftp] /home/robin/bro/master/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 8 +1300475168.783842 Reporter::ERROR field value missing [c$ftp] /Users/robin/bro/master/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 8 +1300475168.915940 Reporter::ERROR field value missing [c$ftp] /Users/robin/bro/master/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 8 +1300475168.916118 Reporter::ERROR field value missing [c$ftp] /Users/robin/bro/master/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 8 +1300475168.918295 Reporter::ERROR field value missing [c$ftp] /Users/robin/bro/master/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 8 +1300475168.952193 Reporter::ERROR field value missing [c$ftp] /Users/robin/bro/master/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 8 +1300475168.952228 Reporter::ERROR field value missing [c$ftp] /Users/robin/bro/master/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 8 +1300475168.954761 Reporter::ERROR field value missing [c$ftp] /Users/robin/bro/master/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 8 +1300475168.962628 Reporter::ERROR field value missing [c$ftp] /Users/robin/bro/master/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 8 +1300475169.780331 Reporter::ERROR field value missing [c$ftp] /Users/robin/bro/master/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 8 diff --git a/testing/btest/Baseline/core.print-bpf-filters-ipv4/conn.log b/testing/btest/Baseline/core.print-bpf-filters-ipv4/conn.log index 3736748484..5ce968d5e6 100644 --- a/testing/btest/Baseline/core.print-bpf-filters-ipv4/conn.log +++ b/testing/btest/Baseline/core.print-bpf-filters-ipv4/conn.log @@ -1,4 +1,7 @@ #separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - #path conn #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto service duration orig_bytes resp_bytes conn_state local_orig missed_bytes history orig_pkts orig_ip_bytes resp_pkts resp_ip_bytes #types time string addr port addr port enum string interval count count string bool count string count count count count diff --git a/testing/btest/Baseline/core.print-bpf-filters-ipv4/output b/testing/btest/Baseline/core.print-bpf-filters-ipv4/output index 4f6230b768..d7ff523927 100644 --- a/testing/btest/Baseline/core.print-bpf-filters-ipv4/output +++ b/testing/btest/Baseline/core.print-bpf-filters-ipv4/output @@ -1,20 +1,32 @@ #separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - #path packet_filter #fields ts node filter init success #types time string string bool bool -1320367155.152502 - not ip6 T T +1324314285.981347 - not ip6 T T #separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - #path packet_filter #fields ts node filter init success #types time string string bool bool -1320367155.379066 - (((((((((((((((((((((((((port 53) or (tcp port 989)) or (tcp port 443)) or (port 6669)) or (udp and port 5353)) or (port 6668)) or (udp and port 5355)) or (tcp port 22)) or (tcp port 995)) or (port 21)) or (tcp port 25 or tcp port 587)) or (port 6667)) or (tcp port 614)) or (tcp port 990)) or (udp port 137)) or (tcp port 993)) or (tcp port 5223)) or (port 514)) or (tcp port 585)) or (tcp port 992)) or (tcp port 563)) or (tcp port 994)) or (tcp port 636)) or (tcp and port (80 or 81 or 631 or 1080 or 3138 or 8000 or 8080 or 8888))) or (port 6666)) and (not ip6) T T +1324314286.168294 - (((((((((((((((((((((((((port 53) or (tcp port 989)) or (tcp port 443)) or (port 6669)) or (udp and port 5353)) or (port 6668)) or (udp and port 5355)) or (tcp port 22)) or (tcp port 995)) or (port 21)) or (tcp port 25 or tcp port 587)) or (port 6667)) or (tcp port 614)) or (tcp port 990)) or (udp port 137)) or (tcp port 993)) or (tcp port 5223)) or (port 514)) or (tcp port 585)) or (tcp port 992)) or (tcp port 563)) or (tcp port 994)) or (tcp port 636)) or (tcp and port (80 or 81 or 631 or 1080 or 3138 or 8000 or 8080 or 8888))) or (port 6666)) and (not ip6) T T #separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - #path packet_filter #fields ts node filter init success #types time string string bool bool -1320367155.601980 - port 42 T T +1324314286.350780 - port 42 T T #separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - #path packet_filter #fields ts node filter init success #types time string string bool bool -1320367155.826539 - port 56730 T T +1324314286.530768 - port 56730 T T diff --git a/testing/btest/Baseline/core.reporter-error-in-handler/output b/testing/btest/Baseline/core.reporter-error-in-handler/output index bfb2880ed4..3d8aa6ff54 100644 --- a/testing/btest/Baseline/core.reporter-error-in-handler/output +++ b/testing/btest/Baseline/core.reporter-error-in-handler/output @@ -1,2 +1,2 @@ -error in /da/home/robin/bro/seth/testing/btest/.tmp/core.reporter-error-in-handler/reporter-error-in-handler.bro, line 22: no such index (a[2]) +error in /Users/robin/bro/master/testing/btest/.tmp/core.reporter-error-in-handler/reporter-error-in-handler.bro, line 22: no such index (a[2]) 1st error printed on script level diff --git a/testing/btest/Baseline/core.reporter-fmt-strings/output b/testing/btest/Baseline/core.reporter-fmt-strings/output index 10a883cb5d..4842dd9fc5 100644 --- a/testing/btest/Baseline/core.reporter-fmt-strings/output +++ b/testing/btest/Baseline/core.reporter-fmt-strings/output @@ -1 +1 @@ -error in /Users/jsiwek/tmp/bro/testing/btest/.tmp/core.reporter-fmt-strings/reporter-fmt-strings.bro, line 9: not an event (dont_interpret_this(%s)) +error in /Users/robin/bro/master/testing/btest/.tmp/core.reporter-fmt-strings/reporter-fmt-strings.bro, line 9: not an event (dont_interpret_this(%s)) diff --git a/testing/btest/Baseline/core.reporter-parse-error/output b/testing/btest/Baseline/core.reporter-parse-error/output index ca0bc9304b..7606fe5667 100644 --- a/testing/btest/Baseline/core.reporter-parse-error/output +++ b/testing/btest/Baseline/core.reporter-parse-error/output @@ -1 +1 @@ -error in /da/home/robin/bro/seth/testing/btest/.tmp/core.reporter-parse-error/reporter-parse-error.bro, line 7: unknown identifier TESTFAILURE, at or near "TESTFAILURE" +error in /Users/robin/bro/master/testing/btest/.tmp/core.reporter-parse-error/reporter-parse-error.bro, line 7: unknown identifier TESTFAILURE, at or near "TESTFAILURE" diff --git a/testing/btest/Baseline/core.reporter-runtime-error/output b/testing/btest/Baseline/core.reporter-runtime-error/output index 5c0feedf42..3a96954101 100644 --- a/testing/btest/Baseline/core.reporter-runtime-error/output +++ b/testing/btest/Baseline/core.reporter-runtime-error/output @@ -1 +1 @@ -error in /Users/seth/bro.git9/testing/btest/.tmp/core.reporter-runtime-error/reporter-runtime-error.bro, line 12: no such index (a[1]) +error in /Users/robin/bro/master/testing/btest/.tmp/core.reporter-runtime-error/reporter-runtime-error.bro, line 12: no such index (a[1]) diff --git a/testing/btest/Baseline/core.reporter-type-mismatch/output b/testing/btest/Baseline/core.reporter-type-mismatch/output index 6211752225..4c038ea8c5 100644 --- a/testing/btest/Baseline/core.reporter-type-mismatch/output +++ b/testing/btest/Baseline/core.reporter-type-mismatch/output @@ -1,3 +1,3 @@ -error in string and /da/home/robin/bro/seth/testing/btest/.tmp/core.reporter-type-mismatch/reporter-type-mismatch.bro, line 11: arithmetic mixed with non-arithmetic (string and 42) -error in /da/home/robin/bro/seth/testing/btest/.tmp/core.reporter-type-mismatch/reporter-type-mismatch.bro, line 11 and string: type mismatch (42 and string) -error in /da/home/robin/bro/seth/testing/btest/.tmp/core.reporter-type-mismatch/reporter-type-mismatch.bro, line 11: argument type mismatch in event invocation (foo(42)) +error in string and /Users/robin/bro/master/testing/btest/.tmp/core.reporter-type-mismatch/reporter-type-mismatch.bro, line 11: arithmetic mixed with non-arithmetic (string and 42) +error in /Users/robin/bro/master/testing/btest/.tmp/core.reporter-type-mismatch/reporter-type-mismatch.bro, line 11 and string: type mismatch (42 and string) +error in /Users/robin/bro/master/testing/btest/.tmp/core.reporter-type-mismatch/reporter-type-mismatch.bro, line 11: argument type mismatch in event invocation (foo(42)) diff --git a/testing/btest/Baseline/core.reporter/logger-test.log b/testing/btest/Baseline/core.reporter/logger-test.log index 6f7ba1d8c7..bc2abd142a 100644 --- a/testing/btest/Baseline/core.reporter/logger-test.log +++ b/testing/btest/Baseline/core.reporter/logger-test.log @@ -1,6 +1,6 @@ -reporter_info|init test-info|/da/home/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 8|0.000000 -reporter_warning|init test-warning|/da/home/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 9|0.000000 -reporter_error|init test-error|/da/home/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 10|0.000000 -reporter_info|done test-info|/da/home/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 15|0.000000 -reporter_warning|done test-warning|/da/home/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 16|0.000000 -reporter_error|done test-error|/da/home/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 17|0.000000 +reporter_info|init test-info|/Users/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 8|0.000000 +reporter_warning|init test-warning|/Users/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 9|0.000000 +reporter_error|init test-error|/Users/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 10|0.000000 +reporter_info|done test-info|/Users/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 15|0.000000 +reporter_warning|done test-warning|/Users/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 16|0.000000 +reporter_error|done test-error|/Users/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 17|0.000000 diff --git a/testing/btest/Baseline/core.reporter/output b/testing/btest/Baseline/core.reporter/output index 2735adc931..185cabb1eb 100644 --- a/testing/btest/Baseline/core.reporter/output +++ b/testing/btest/Baseline/core.reporter/output @@ -1,3 +1,3 @@ -/da/home/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 52: pre test-info -warning in /da/home/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 53: pre test-warning -error in /da/home/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 54: pre test-error +/Users/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 52: pre test-info +warning in /Users/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 53: pre test-warning +error in /Users/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 54: pre test-error diff --git a/testing/btest/Baseline/core.vlan-mpls/conn.log b/testing/btest/Baseline/core.vlan-mpls/conn.log index 69e23f3875..f3c958ea99 100644 --- a/testing/btest/Baseline/core.vlan-mpls/conn.log +++ b/testing/btest/Baseline/core.vlan-mpls/conn.log @@ -1,4 +1,7 @@ #separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - #path conn #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto service duration orig_bytes resp_bytes conn_state local_orig missed_bytes history orig_pkts orig_ip_bytes resp_pkts resp_ip_bytes #types time string addr port addr port enum string interval count count string bool count string count count count count diff --git a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log index 6819dc0813..8fab67304e 100644 --- a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log @@ -1,4 +1,7 @@ #separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - #path loaded_scripts #fields name #types string diff --git a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log index 7a461a3903..3f77797df8 100644 --- a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log @@ -1,4 +1,7 @@ #separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - #path loaded_scripts #fields name #types string diff --git a/testing/btest/Baseline/doc.autogen-reST-example/example.rst b/testing/btest/Baseline/doc.autogen-reST-example/example.rst index 880fa0e171..46cc4a8227 100644 --- a/testing/btest/Baseline/doc.autogen-reST-example/example.rst +++ b/testing/btest/Baseline/doc.autogen-reST-example/example.rst @@ -7,8 +7,8 @@ example.bro Overview -------- -This is an example script that demonstrates how to document. Comments -of the form ``##!`` are for the script summary. The contents of +This is an example script that demonstrates documentation features. +Comments of the form ``##!`` are for the script summary. The contents of these comments are transferred directly into the auto-generated `reStructuredText `_ (reST) document's summary section. @@ -34,7 +34,7 @@ Options ============================================================================ ====================================== :bro:id:`Example::an_option`: :bro:type:`set` :bro:attr:`&redef` add documentation for "an_option" here -:bro:id:`Example::option_with_init`: :bro:type:`interval` :bro:attr:`&redef` +:bro:id:`Example::option_with_init`: :bro:type:`interval` :bro:attr:`&redef` More docs can be added here. ============================================================================ ====================================== State Variables @@ -128,6 +128,8 @@ Options :Attributes: :bro:attr:`&redef` :Default: ``10.0 msecs`` + More docs can be added here. + State Variables ~~~~~~~~~~~~~~~ .. bro:id:: Example::a_var @@ -217,6 +219,7 @@ Events Summarize "an_event" here. Give more details about "an_event" here. + Example::an_event should not be confused as a parameter. :param name: describe the argument here @@ -281,7 +284,7 @@ Redefinitions Port Analysis ------------- -:ref:`More Information ` +Loading this script makes the following changes to :bro:see:`dpd_config`. SSL:: @@ -292,7 +295,7 @@ SSL:: Packet Filter ------------- -:ref:`More Information ` +Loading this script makes the following changes to :bro:see:`capture_filters`. Filters added:: diff --git a/testing/btest/Baseline/istate.broccoli/bro.log b/testing/btest/Baseline/istate.broccoli/bro.log index eeebe944ef..4fbbfc81ae 100644 --- a/testing/btest/Baseline/istate.broccoli/bro.log +++ b/testing/btest/Baseline/istate.broccoli/bro.log @@ -1,3 +1,3 @@ -ping received, seq 0, 1303093042.542125 at src, 1303093042.583423 at dest, -ping received, seq 1, 1303093043.543167 at src, 1303093043.544026 at dest, -ping received, seq 2, 1303093044.544115 at src, 1303093044.545008 at dest, +ping received, seq 0, 1324314397.698781 at src, 1324314397.699240 at dest, +ping received, seq 1, 1324314398.698905 at src, 1324314398.699094 at dest, +ping received, seq 2, 1324314399.699012 at src, 1324314399.699231 at dest, diff --git a/testing/btest/Baseline/istate.events-ssl/receiver.http.log b/testing/btest/Baseline/istate.events-ssl/receiver.http.log index 06d453c241..1601f8ad3c 100644 --- a/testing/btest/Baseline/istate.events-ssl/receiver.http.log +++ b/testing/btest/Baseline/istate.events-ssl/receiver.http.log @@ -1,5 +1,8 @@ #separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - #path http #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file -#types time string addr port addr port count string string string string string count count count string count string string table string string table string string file -1319568535.914761 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - - - - - text/html - - +#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string file +1324314406.995958 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - (empty) - - - text/html - - diff --git a/testing/btest/Baseline/istate.events-ssl/sender.http.log b/testing/btest/Baseline/istate.events-ssl/sender.http.log index 06d453c241..1601f8ad3c 100644 --- a/testing/btest/Baseline/istate.events-ssl/sender.http.log +++ b/testing/btest/Baseline/istate.events-ssl/sender.http.log @@ -1,5 +1,8 @@ #separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - #path http #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file -#types time string addr port addr port count string string string string string count count count string count string string table string string table string string file -1319568535.914761 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - - - - - text/html - - +#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string file +1324314406.995958 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - (empty) - - - text/html - - diff --git a/testing/btest/Baseline/istate.events/receiver.http.log b/testing/btest/Baseline/istate.events/receiver.http.log index d85d560b6d..25a7f289c0 100644 --- a/testing/btest/Baseline/istate.events/receiver.http.log +++ b/testing/btest/Baseline/istate.events/receiver.http.log @@ -1,5 +1,8 @@ #separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - #path http #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file -#types time string addr port addr port count string string string string string count count count string count string string table string string table string string file -1319568558.542142 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - - - - - text/html - - +#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string file +1324314415.616486 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - (empty) - - - text/html - - diff --git a/testing/btest/Baseline/istate.events/sender.http.log b/testing/btest/Baseline/istate.events/sender.http.log index d85d560b6d..25a7f289c0 100644 --- a/testing/btest/Baseline/istate.events/sender.http.log +++ b/testing/btest/Baseline/istate.events/sender.http.log @@ -1,5 +1,8 @@ #separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - #path http #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file -#types time string addr port addr port count string string string string string count count count string count string string table string string table string string file -1319568558.542142 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - - - - - text/html - - +#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string file +1324314415.616486 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - (empty) - - - text/html - - diff --git a/testing/btest/Baseline/language.record-default-coercion/out b/testing/btest/Baseline/language.record-default-coercion/out new file mode 100644 index 0000000000..2f0e6cd17d --- /dev/null +++ b/testing/btest/Baseline/language.record-default-coercion/out @@ -0,0 +1,4 @@ +[a=13, c=13, v=[]] +0 +[a=13, c=13, v=[test]] +1 diff --git a/testing/btest/Baseline/language.wrong-delete-field/output b/testing/btest/Baseline/language.wrong-delete-field/output index f8271e43c2..c2aae8aae3 100644 --- a/testing/btest/Baseline/language.wrong-delete-field/output +++ b/testing/btest/Baseline/language.wrong-delete-field/output @@ -1 +1 @@ -error in /da/home/robin/bro/seth/testing/btest/.tmp/language.wrong-delete-field/wrong-delete-field.bro, line 10: illegal delete statement (delete x$a) +error in /Users/robin/bro/master/testing/btest/.tmp/language.wrong-delete-field/wrong-delete-field.bro, line 10: illegal delete statement (delete x$a) diff --git a/testing/btest/Baseline/scripts.base.frameworks.communication.communication_log_baseline/send.log b/testing/btest/Baseline/scripts.base.frameworks.communication.communication_log_baseline/send.log new file mode 100644 index 0000000000..e5dfb59592 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.communication.communication_log_baseline/send.log @@ -0,0 +1,19 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path communication +#fields ts peer src_name connected_peer_desc connected_peer_addr connected_peer_port level message +#types time string string string addr port string string +1324314302.411344 bro parent - - - info [#1/127.0.0.1:47757] added peer +1324314302.414978 bro child - - - info [#1/127.0.0.1:47757] connected +1324314302.415099 bro parent - - - info [#1/127.0.0.1:47757] peer connected +1324314302.415099 bro parent - - - info [#1/127.0.0.1:47757] phase: version +1324314302.417446 bro script - - - info connection established +1324314302.417446 bro script - - - info requesting events matching /^?(NOTHING)$?/ +1324314302.417446 bro script - - - info accepting state +1324314302.418003 bro parent - - - info [#1/127.0.0.1:47757] phase: handshake +1324314302.418003 bro parent - - - info warning: no events to request +1324314302.418003 bro parent - - - info terminating... +1324314302.418003 bro parent - - - info [#1/127.0.0.1:47757] peer_description is bro +1324314302.418003 bro parent - - - info [#1/127.0.0.1:47757] closing connection diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.adapt-filter/ssh-new-default.log b/testing/btest/Baseline/scripts.base.frameworks.logging.adapt-filter/ssh-new-default.log index fc2c133dc6..485bfe3eba 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.adapt-filter/ssh-new-default.log +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.adapt-filter/ssh-new-default.log @@ -1,6 +1,9 @@ #separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - #path ssh-new-default #fields t id.orig_h id.orig_p id.resp_h id.resp_p status country #types time addr port addr port string string -1315167052.603186 1.2.3.4 1234 2.3.4.5 80 success unknown -1315167052.603186 1.2.3.4 1234 2.3.4.5 80 failure US +1324314313.140603 1.2.3.4 1234 2.3.4.5 80 success unknown +1324314313.140603 1.2.3.4 1234 2.3.4.5 80 failure US diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-binary/ssh.log b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-binary/ssh.log index b236cb818b..144a7a6426 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-binary/ssh.log +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-binary/ssh.log @@ -1,4 +1,7 @@ -#separator \x7c +#separator | +#set_separator|, +#empty_field|(empty) +#unset_field|- #path|ssh #fields|data|data2 #types|string|string diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-empty/ssh.log b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-empty/ssh.log index e1ba48cf8e..10275205a5 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-empty/ssh.log +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-empty/ssh.log @@ -1,9 +1,12 @@ -PREFIX<>separator \x7c +PREFIX<>separator | +PREFIX<>set_separator|, +PREFIX<>empty_field|EMPTY +PREFIX<>unset_field|NOT-SET PREFIX<>path|ssh PREFIX<>fields|t|id.orig_h|id.orig_p|id.resp_h|id.resp_p|status|country|b PREFIX<>types|time|addr|port|addr|port|string|string|bool -1315167052.828457|1.2.3.4|1234|2.3.4.5|80|success|unknown|NOT-SET -1315167052.828457|1.2.3.4|1234|2.3.4.5|80|NOT-SET|US|NOT-SET -1315167052.828457|1.2.3.4|1234|2.3.4.5|80|failure|UK|NOT-SET -1315167052.828457|1.2.3.4|1234|2.3.4.5|80|NOT-SET|BR|NOT-SET -1315167052.828457|1.2.3.4|1234|2.3.4.5|80|failure|EMPTY|T +1324314313.345323|1.2.3.4|1234|2.3.4.5|80|success|unknown|NOT-SET +1324314313.345323|1.2.3.4|1234|2.3.4.5|80|NOT-SET|US|NOT-SET +1324314313.345323|1.2.3.4|1234|2.3.4.5|80|failure|UK|NOT-SET +1324314313.345323|1.2.3.4|1234|2.3.4.5|80|NOT-SET|BR|NOT-SET +1324314313.345323|1.2.3.4|1234|2.3.4.5|80|failure|EMPTY|T diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape-notset-str/test.log b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape-notset-str/test.log new file mode 100644 index 0000000000..c9e69994fc --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape-notset-str/test.log @@ -0,0 +1,8 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path test +#fields x y z +#types string string string +\x2d - (empty) diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape-odd-url/http.log b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape-odd-url/http.log index db9ce497ed..97744b7df8 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape-odd-url/http.log +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape-odd-url/http.log @@ -1,5 +1,8 @@ #separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - #path http #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file -#types time string addr port addr port count string string string string string count count count string count string string table string string table string string file -1315799856.264750 UWkUyAuUGXf 10.0.1.104 64216 193.40.5.162 80 1 GET lepo.it.da.ut.ee /~cect/teoreetilised seminarid_2010/arheoloogia_uurimisr\xfchma_seminar/Joyce et al - The Languages of Archaeology ~ Dialogue, Narrative and Writing.pdf - Wget/1.12 (darwin10.8.0) 0 346 404 Not Found - - - - - - - text/html - - +#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string file +1315799856.264750 UWkUyAuUGXf 10.0.1.104 64216 193.40.5.162 80 1 GET lepo.it.da.ut.ee /~cect/teoreetilised seminarid_2010/arheoloogia_uurimisr\xfchma_seminar/Joyce et al - The Languages of Archaeology ~ Dialogue, Narrative and Writing.pdf - Wget/1.12 (darwin10.8.0) 0 346 404 Not Found - - - (empty) - - - text/html - - diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape-set-separator/test.log b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape-set-separator/test.log new file mode 100644 index 0000000000..b88627c806 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape-set-separator/test.log @@ -0,0 +1,8 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path test +#fields ss +#types table[string] +CC,AA,\x2c,\x2c\x2c diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape/ssh.log b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape/ssh.log index 3100fa0cb2..0ef81128d3 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape/ssh.log +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape/ssh.log @@ -1,9 +1,12 @@ -#separator \x7c\x7c +#separator || +#set_separator||, +#empty_field||(empty) +#unset_field||- #path||ssh #fields||t||id.orig_h||id.orig_p||id.resp_h||id.resp_p||status||country #types||time||addr||port||addr||port||string||string -1315802040.006123||1.2.3.4||1234||2.3.4.5||80||success||unknown -1315802040.006123||1.2.3.4||1234||2.3.4.5||80||failure||US -1315802040.006123||1.2.3.4||1234||2.3.4.5||80||fa\x7c\x7cure||UK -1315802040.006123||1.2.3.4||1234||2.3.4.5||80||su\x7c\x7cess||BR -1315802040.006123||1.2.3.4||1234||2.3.4.5||80||failure||MX +1324314313.899736||1.2.3.4||1234||2.3.4.5||80||success||unknown +1324314313.899736||1.2.3.4||1234||2.3.4.5||80||failure||US +1324314313.899736||1.2.3.4||1234||2.3.4.5||80||fa\x7c\x7cure||UK +1324314313.899736||1.2.3.4||1234||2.3.4.5||80||su\x7c\x7cess||BR +1324314313.899736||1.2.3.4||1234||2.3.4.5||80||failure||MX diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-options/ssh.log b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-options/ssh.log index 33a922cc2b..f66dec7160 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-options/ssh.log +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-options/ssh.log @@ -1,5 +1,5 @@ -1299718506.38074|1.2.3.4|1234|2.3.4.5|80|success|unknown -1299718506.38074|1.2.3.4|1234|2.3.4.5|80|failure|US -1299718506.38074|1.2.3.4|1234|2.3.4.5|80|failure|UK -1299718506.38074|1.2.3.4|1234|2.3.4.5|80|success|BR -1299718506.38074|1.2.3.4|1234|2.3.4.5|80|failure|MX +1324314313.990741|1.2.3.4|1234|2.3.4.5|80|success|unknown +1324314313.990741|1.2.3.4|1234|2.3.4.5|80|failure|US +1324314313.990741|1.2.3.4|1234|2.3.4.5|80|failure|UK +1324314313.990741|1.2.3.4|1234|2.3.4.5|80|success|BR +1324314313.990741|1.2.3.4|1234|2.3.4.5|80|failure|MX diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-timestamps/test.log b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-timestamps/test.log index 7f512c15d9..00ab6c8ca0 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-timestamps/test.log +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-timestamps/test.log @@ -1,4 +1,7 @@ #separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - #path test #fields data #types time diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.attr-extend/ssh.log b/testing/btest/Baseline/scripts.base.frameworks.logging.attr-extend/ssh.log index c2c32c5c6a..5acaa7b2fc 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.attr-extend/ssh.log +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.attr-extend/ssh.log @@ -1,4 +1,7 @@ #separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - #path ssh #fields status country a1 b1 b2 #types string string count count count diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.attr/ssh.log b/testing/btest/Baseline/scripts.base.frameworks.logging.attr/ssh.log index 18e4d5cbad..086a4836fe 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.attr/ssh.log +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.attr/ssh.log @@ -1,4 +1,7 @@ #separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - #path ssh #fields status country #types string string diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.empty-event/ssh.log b/testing/btest/Baseline/scripts.base.frameworks.logging.empty-event/ssh.log index 49272bfd53..16ba17c62c 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.empty-event/ssh.log +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.empty-event/ssh.log @@ -1,9 +1,12 @@ #separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - #path ssh #fields t id.orig_h id.orig_p id.resp_h id.resp_p status country #types time addr port addr port string string -1315167053.369918 1.2.3.4 1234 2.3.4.5 80 success unknown -1315167053.369918 1.2.3.4 1234 2.3.4.5 80 failure US -1315167053.369918 1.2.3.4 1234 2.3.4.5 80 failure UK -1315167053.369918 1.2.3.4 1234 2.3.4.5 80 success BR -1315167053.369918 1.2.3.4 1234 2.3.4.5 80 failure MX +1324314314.443785 1.2.3.4 1234 2.3.4.5 80 success unknown +1324314314.443785 1.2.3.4 1234 2.3.4.5 80 failure US +1324314314.443785 1.2.3.4 1234 2.3.4.5 80 failure UK +1324314314.443785 1.2.3.4 1234 2.3.4.5 80 success BR +1324314314.443785 1.2.3.4 1234 2.3.4.5 80 failure MX diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.events/output b/testing/btest/Baseline/scripts.base.frameworks.logging.events/output index c3dbf607a6..5da27764a5 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.events/output +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.events/output @@ -1,2 +1,2 @@ -[t=1299718502.96511, id=[orig_h=1.2.3.4, orig_p=1234/tcp, resp_h=2.3.4.5, resp_p=80/tcp], status=success, country=] -[t=1299718502.96511, id=[orig_h=1.2.3.4, orig_p=1234/tcp, resp_h=2.3.4.5, resp_p=80/tcp], status=failure, country=US] +[t=1324314314.738385, id=[orig_h=1.2.3.4, orig_p=1234/tcp, resp_h=2.3.4.5, resp_p=80/tcp], status=success, country=unknown] +[t=1324314314.738385, id=[orig_h=1.2.3.4, orig_p=1234/tcp, resp_h=2.3.4.5, resp_p=80/tcp], status=failure, country=US] diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.exclude/ssh.log b/testing/btest/Baseline/scripts.base.frameworks.logging.exclude/ssh.log index b078b4746a..4ccf4c836a 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.exclude/ssh.log +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.exclude/ssh.log @@ -1,4 +1,7 @@ #separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - #path ssh #fields id.orig_p id.resp_h id.resp_p status country #types port addr port string string diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.file/ssh.log b/testing/btest/Baseline/scripts.base.frameworks.logging.file/ssh.log index 0a988ff9b9..4aa3d8f0a7 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.file/ssh.log +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.file/ssh.log @@ -1,5 +1,8 @@ #separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - #path ssh #fields t f #types time file -1315167053.585834 Foo.log +1324314314.940195 Foo.log diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.include/ssh.log b/testing/btest/Baseline/scripts.base.frameworks.logging.include/ssh.log index 5675ef6632..00242d65c1 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.include/ssh.log +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.include/ssh.log @@ -1,9 +1,12 @@ #separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - #path ssh #fields t id.orig_h #types time addr -1315167053.694473 1.2.3.4 -1315167053.694473 1.2.3.4 -1315167053.694473 1.2.3.4 -1315167053.694473 1.2.3.4 -1315167053.694473 1.2.3.4 +1324314315.040480 1.2.3.4 +1324314315.040480 1.2.3.4 +1324314315.040480 1.2.3.4 +1324314315.040480 1.2.3.4 +1324314315.040480 1.2.3.4 diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.path-func-column-demote/local.log b/testing/btest/Baseline/scripts.base.frameworks.logging.path-func-column-demote/local.log index d8d90cf1fa..e2b3da6efd 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.path-func-column-demote/local.log +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.path-func-column-demote/local.log @@ -1,4 +1,7 @@ #separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - #path local #fields ts id.orig_h #types time addr diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.path-func-column-demote/remote.log b/testing/btest/Baseline/scripts.base.frameworks.logging.path-func-column-demote/remote.log index a17c2821f5..1ac18ff5f7 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.path-func-column-demote/remote.log +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.path-func-column-demote/remote.log @@ -1,4 +1,7 @@ #separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - #path remote #fields ts id.orig_h #types time addr diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.path-func/output b/testing/btest/Baseline/scripts.base.frameworks.logging.path-func/output index 2c196340cc..a6b8a4e090 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.path-func/output +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.path-func/output @@ -6,37 +6,58 @@ static-prefix-1-US.log static-prefix-2-MX2.log static-prefix-2-UK.log #separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - #path static-prefix-0-BR #fields t id.orig_h id.orig_p id.resp_h id.resp_p status country #types time addr port addr port string string -1315167053.803346 1.2.3.4 1234 2.3.4.5 80 success BR +1324314315.385189 1.2.3.4 1234 2.3.4.5 80 success BR #separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - #path static-prefix-0-MX3 #fields t id.orig_h id.orig_p id.resp_h id.resp_p status country #types time addr port addr port string string -1315167053.803346 1.2.3.4 1234 2.3.4.5 80 failure MX3 +1324314315.385189 1.2.3.4 1234 2.3.4.5 80 failure MX3 #separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - #path static-prefix-0-unknown #fields t id.orig_h id.orig_p id.resp_h id.resp_p status country #types time addr port addr port string string -1315167053.803346 1.2.3.4 1234 2.3.4.5 80 success unknown +1324314315.385189 1.2.3.4 1234 2.3.4.5 80 success unknown #separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - #path static-prefix-1-MX #fields t id.orig_h id.orig_p id.resp_h id.resp_p status country #types time addr port addr port string string -1315167053.803346 1.2.3.4 1234 2.3.4.5 80 failure MX +1324314315.385189 1.2.3.4 1234 2.3.4.5 80 failure MX #separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - #path static-prefix-1-US #fields t id.orig_h id.orig_p id.resp_h id.resp_p status country #types time addr port addr port string string -1315167053.803346 1.2.3.4 1234 2.3.4.5 80 failure US +1324314315.385189 1.2.3.4 1234 2.3.4.5 80 failure US #separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - #path static-prefix-2-MX2 #fields t id.orig_h id.orig_p id.resp_h id.resp_p status country #types time addr port addr port string string -1315167053.803346 1.2.3.4 1234 2.3.4.5 80 failure MX2 +1324314315.385189 1.2.3.4 1234 2.3.4.5 80 failure MX2 #separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - #path static-prefix-2-UK #fields t id.orig_h id.orig_p id.resp_h id.resp_p status country #types time addr port addr port string string -1315167053.803346 1.2.3.4 1234 2.3.4.5 80 failure UK +1324314315.385189 1.2.3.4 1234 2.3.4.5 80 failure UK diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.pred/test.failure.log b/testing/btest/Baseline/scripts.base.frameworks.logging.pred/test.failure.log index ba688d7843..733bb02847 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.pred/test.failure.log +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.pred/test.failure.log @@ -1,5 +1,8 @@ #separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - #path test.failure #fields t id.orig_h id.orig_p id.resp_h id.resp_p status country #types time addr port addr port string string -1315167053.923545 1.2.3.4 1234 2.3.4.5 80 failure US +1324314315.498365 1.2.3.4 1234 2.3.4.5 80 failure US diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.pred/test.success.log b/testing/btest/Baseline/scripts.base.frameworks.logging.pred/test.success.log index 7a91b1a2d9..0261caeb06 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.pred/test.success.log +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.pred/test.success.log @@ -1,5 +1,8 @@ #separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - #path test.success #fields t id.orig_h id.orig_p id.resp_h id.resp_p status country #types time addr port addr port string string -1315167053.923545 1.2.3.4 1234 2.3.4.5 80 success - +1324314315.498365 1.2.3.4 1234 2.3.4.5 80 success unknown diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.remote-types/receiver.test.log b/testing/btest/Baseline/scripts.base.frameworks.logging.remote-types/receiver.test.log index c00e7765d5..d9bd34309a 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.remote-types/receiver.test.log +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.remote-types/receiver.test.log @@ -1,5 +1,8 @@ #separator \x09 +#set_separator , +#empty_field EMPTY +#unset_field - #path test #fields b i e c p sn a d t iv s sc ss se vc ve -#types bool int enum count port subnet addr double time interval string table table table vector vector -T -42 Test::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315167054.320958 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY +#types bool int enum count port subnet addr double time interval string table[count] table[string] table[string] vector[count] vector[string] +T -42 Test::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1324314315.880694 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.remote/sender.test.failure.log b/testing/btest/Baseline/scripts.base.frameworks.logging.remote/sender.test.failure.log index aba9fdddd9..6cb58bf4ac 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.remote/sender.test.failure.log +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.remote/sender.test.failure.log @@ -1,7 +1,10 @@ #separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - #path test.failure #fields t id.orig_h id.orig_p id.resp_h id.resp_p status country #types time addr port addr port string string -1315167059.502670 1.2.3.4 1234 2.3.4.5 80 failure US -1315167059.502670 1.2.3.4 1234 2.3.4.5 80 failure UK -1315167059.502670 1.2.3.4 1234 2.3.4.5 80 failure MX +1324314321.061516 1.2.3.4 1234 2.3.4.5 80 failure US +1324314321.061516 1.2.3.4 1234 2.3.4.5 80 failure UK +1324314321.061516 1.2.3.4 1234 2.3.4.5 80 failure MX diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.remote/sender.test.log b/testing/btest/Baseline/scripts.base.frameworks.logging.remote/sender.test.log index b928c37685..f5b79ee2c4 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.remote/sender.test.log +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.remote/sender.test.log @@ -1,9 +1,12 @@ #separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - #path test #fields t id.orig_h id.orig_p id.resp_h id.resp_p status country #types time addr port addr port string string -1315167059.502670 1.2.3.4 1234 2.3.4.5 80 success - -1315167059.502670 1.2.3.4 1234 2.3.4.5 80 failure US -1315167059.502670 1.2.3.4 1234 2.3.4.5 80 failure UK -1315167059.502670 1.2.3.4 1234 2.3.4.5 80 success BR -1315167059.502670 1.2.3.4 1234 2.3.4.5 80 failure MX +1324314321.061516 1.2.3.4 1234 2.3.4.5 80 success unknown +1324314321.061516 1.2.3.4 1234 2.3.4.5 80 failure US +1324314321.061516 1.2.3.4 1234 2.3.4.5 80 failure UK +1324314321.061516 1.2.3.4 1234 2.3.4.5 80 success BR +1324314321.061516 1.2.3.4 1234 2.3.4.5 80 failure MX diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.remote/sender.test.success.log b/testing/btest/Baseline/scripts.base.frameworks.logging.remote/sender.test.success.log index a951c6ed1a..c40e56af93 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.remote/sender.test.success.log +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.remote/sender.test.success.log @@ -1,6 +1,9 @@ #separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - #path test.success #fields t id.orig_h id.orig_p id.resp_h id.resp_p status country #types time addr port addr port string string -1315167059.502670 1.2.3.4 1234 2.3.4.5 80 success - -1315167059.502670 1.2.3.4 1234 2.3.4.5 80 success BR +1324314321.061516 1.2.3.4 1234 2.3.4.5 80 success unknown +1324314321.061516 1.2.3.4 1234 2.3.4.5 80 success BR diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.remove/ssh.failure.log b/testing/btest/Baseline/scripts.base.frameworks.logging.remove/ssh.failure.log index 6185e86028..cb3d4aafb8 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.remove/ssh.failure.log +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.remove/ssh.failure.log @@ -1,6 +1,9 @@ #separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - #path ssh.failure #fields t id.orig_h id.orig_p id.resp_h id.resp_p status country #types time addr port addr port string string -1315167066.575996 1.2.3.4 1234 2.3.4.5 80 failure US -1315167066.575996 1.2.3.4 1234 2.3.4.5 80 failure UK +1324314328.196443 1.2.3.4 1234 2.3.4.5 80 failure US +1324314328.196443 1.2.3.4 1234 2.3.4.5 80 failure UK diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.remove/ssh.log b/testing/btest/Baseline/scripts.base.frameworks.logging.remove/ssh.log index a4ec2dc7de..38a5bb660c 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.remove/ssh.log +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.remove/ssh.log @@ -1,7 +1,10 @@ #separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - #path ssh #fields t id.orig_h id.orig_p id.resp_h id.resp_p status country #types time addr port addr port string string -1315167066.575996 1.2.3.4 1234 2.3.4.5 80 failure US -1315167066.575996 1.2.3.4 1234 2.3.4.5 80 failure UK -1315167066.575996 1.2.3.4 1234 2.3.4.5 80 failure BR +1324314328.196443 1.2.3.4 1234 2.3.4.5 80 failure US +1324314328.196443 1.2.3.4 1234 2.3.4.5 80 failure UK +1324314328.196443 1.2.3.4 1234 2.3.4.5 80 failure BR diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.rotate-custom/out b/testing/btest/Baseline/scripts.base.frameworks.logging.rotate-custom/out index 337ed3ca32..915915f43e 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.rotate-custom/out +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.rotate-custom/out @@ -18,11 +18,14 @@ custom rotate, [writer=Log::WRITER_ASCII, fname=test2-11-03-07_11.00.05.log, pat custom rotate, [writer=Log::WRITER_ASCII, fname=test2-11-03-07_11.59.55.log, path=test2, open=1299499195.0, close=1299499205.0, terminating=F] custom rotate, [writer=Log::WRITER_ASCII, fname=test2-11-03-07_12.00.05.log, path=test2, open=1299499205.0, close=1299502795.0, terminating=F] custom rotate, [writer=Log::WRITER_ASCII, fname=test2-11-03-07_12.59.55.log, path=test2, open=1299502795.0, close=1299502795.0, terminating=T] +#empty_field (empty) #fields t id.orig_h id.orig_p id.resp_h id.resp_p #path test #path test2 #separator \x09 +#set_separator , #types time addr port addr port +#unset_field - 1299466805.000000 10.0.0.1 20 10.0.0.2 1024 1299470395.000000 10.0.0.2 20 10.0.0.3 0 1299470405.000000 10.0.0.1 20 10.0.0.2 1025 diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.rotate/out b/testing/btest/Baseline/scripts.base.frameworks.logging.rotate/out index 74ce45023a..d31783edc4 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.rotate/out +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.rotate/out @@ -10,6 +10,9 @@ test.2011-03-07-11-00-05.log test 11-03-07_11.00.05 11-03-07_12.00.05 0 test.2011-03-07-12-00-05.log test 11-03-07_12.00.05 11-03-07_12.59.55 1 > test.2011-03-07-03-00-05.log #separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - #path test #fields t id.orig_h id.orig_p id.resp_h id.resp_p #types time addr port addr port @@ -17,6 +20,9 @@ test.2011-03-07-12-00-05.log test 11-03-07_12.00.05 11-03-07_12.59.55 1 1299470395.000000 10.0.0.2 20 10.0.0.3 0 > test.2011-03-07-04-00-05.log #separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - #path test #fields t id.orig_h id.orig_p id.resp_h id.resp_p #types time addr port addr port @@ -24,6 +30,9 @@ test.2011-03-07-12-00-05.log test 11-03-07_12.00.05 11-03-07_12.59.55 1 1299473995.000000 10.0.0.2 20 10.0.0.3 1 > test.2011-03-07-05-00-05.log #separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - #path test #fields t id.orig_h id.orig_p id.resp_h id.resp_p #types time addr port addr port @@ -31,6 +40,9 @@ test.2011-03-07-12-00-05.log test 11-03-07_12.00.05 11-03-07_12.59.55 1 1299477595.000000 10.0.0.2 20 10.0.0.3 2 > test.2011-03-07-06-00-05.log #separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - #path test #fields t id.orig_h id.orig_p id.resp_h id.resp_p #types time addr port addr port @@ -38,6 +50,9 @@ test.2011-03-07-12-00-05.log test 11-03-07_12.00.05 11-03-07_12.59.55 1 1299481195.000000 10.0.0.2 20 10.0.0.3 3 > test.2011-03-07-07-00-05.log #separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - #path test #fields t id.orig_h id.orig_p id.resp_h id.resp_p #types time addr port addr port @@ -45,6 +60,9 @@ test.2011-03-07-12-00-05.log test 11-03-07_12.00.05 11-03-07_12.59.55 1 1299484795.000000 10.0.0.2 20 10.0.0.3 4 > test.2011-03-07-08-00-05.log #separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - #path test #fields t id.orig_h id.orig_p id.resp_h id.resp_p #types time addr port addr port @@ -52,6 +70,9 @@ test.2011-03-07-12-00-05.log test 11-03-07_12.00.05 11-03-07_12.59.55 1 1299488395.000000 10.0.0.2 20 10.0.0.3 5 > test.2011-03-07-09-00-05.log #separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - #path test #fields t id.orig_h id.orig_p id.resp_h id.resp_p #types time addr port addr port @@ -59,6 +80,9 @@ test.2011-03-07-12-00-05.log test 11-03-07_12.00.05 11-03-07_12.59.55 1 1299491995.000000 10.0.0.2 20 10.0.0.3 6 > test.2011-03-07-10-00-05.log #separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - #path test #fields t id.orig_h id.orig_p id.resp_h id.resp_p #types time addr port addr port @@ -66,6 +90,9 @@ test.2011-03-07-12-00-05.log test 11-03-07_12.00.05 11-03-07_12.59.55 1 1299495595.000000 10.0.0.2 20 10.0.0.3 7 > test.2011-03-07-11-00-05.log #separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - #path test #fields t id.orig_h id.orig_p id.resp_h id.resp_p #types time addr port addr port @@ -73,6 +100,9 @@ test.2011-03-07-12-00-05.log test 11-03-07_12.00.05 11-03-07_12.59.55 1 1299499195.000000 10.0.0.2 20 10.0.0.3 8 > test.2011-03-07-12-00-05.log #separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - #path test #fields t id.orig_h id.orig_p id.resp_h id.resp_p #types time addr port addr port diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.stdout/output b/testing/btest/Baseline/scripts.base.frameworks.logging.stdout/output index 84521cb645..09afe2031c 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.stdout/output +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.stdout/output @@ -1,9 +1,12 @@ #separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - #path /dev/stdout #fields t id.orig_h id.orig_p id.resp_h id.resp_p status country #types time addr port addr port string string -1315167067.393739 1.2.3.4 1234 2.3.4.5 80 success unknown -1315167067.393739 1.2.3.4 1234 2.3.4.5 80 failure US -1315167067.393739 1.2.3.4 1234 2.3.4.5 80 failure UK -1315167067.393739 1.2.3.4 1234 2.3.4.5 80 success BR -1315167067.393739 1.2.3.4 1234 2.3.4.5 80 failure MX +1324314328.844271 1.2.3.4 1234 2.3.4.5 80 success unknown +1324314328.844271 1.2.3.4 1234 2.3.4.5 80 failure US +1324314328.844271 1.2.3.4 1234 2.3.4.5 80 failure UK +1324314328.844271 1.2.3.4 1234 2.3.4.5 80 success BR +1324314328.844271 1.2.3.4 1234 2.3.4.5 80 failure MX diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.test-logging/ssh.log b/testing/btest/Baseline/scripts.base.frameworks.logging.test-logging/ssh.log index 5b93b6e23b..53292324af 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.test-logging/ssh.log +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.test-logging/ssh.log @@ -1,9 +1,12 @@ #separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - #path ssh #fields t id.orig_h id.orig_p id.resp_h id.resp_p status country #types time addr port addr port string string -1315167067.507542 1.2.3.4 1234 2.3.4.5 80 success unknown -1315167067.507542 1.2.3.4 1234 2.3.4.5 80 failure US -1315167067.507542 1.2.3.4 1234 2.3.4.5 80 failure UK -1315167067.507542 1.2.3.4 1234 2.3.4.5 80 success BR -1315167067.507542 1.2.3.4 1234 2.3.4.5 80 failure MX +1324314328.950525 1.2.3.4 1234 2.3.4.5 80 success unknown +1324314328.950525 1.2.3.4 1234 2.3.4.5 80 failure US +1324314328.950525 1.2.3.4 1234 2.3.4.5 80 failure UK +1324314328.950525 1.2.3.4 1234 2.3.4.5 80 success BR +1324314328.950525 1.2.3.4 1234 2.3.4.5 80 failure MX diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.types/ssh.log b/testing/btest/Baseline/scripts.base.frameworks.logging.types/ssh.log index ffd579c224..74aa0312a1 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.types/ssh.log +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.types/ssh.log @@ -1,5 +1,8 @@ #separator \x09 +#set_separator , +#empty_field EMPTY +#unset_field - #path ssh #fields b i e c p sn a d t iv s sc ss se vc ve f -#types bool int enum count port subnet addr double time interval string table table table vector vector func -T -42 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a} +#types bool int enum count port subnet addr double time interval string table[count] table[string] table[string] vector[count] vector[string] func +T -42 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1324314329.051618 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a} diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.unset-record/testing.log b/testing/btest/Baseline/scripts.base.frameworks.logging.unset-record/testing.log index 12bb1d1704..7956ad11a0 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.unset-record/testing.log +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.unset-record/testing.log @@ -1,4 +1,7 @@ #separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - #path testing #fields a.val1 a.val2 b #types count count count diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.vec/ssh.log b/testing/btest/Baseline/scripts.base.frameworks.logging.vec/ssh.log index b9a54404ed..65ab5592bf 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.vec/ssh.log +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.vec/ssh.log @@ -1,5 +1,8 @@ #separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - #path ssh #fields vec -#types vector +#types vector[string] -,2,-,-,5 diff --git a/testing/btest/Baseline/scripts.base.frameworks.metrics.basic-cluster/manager-1.metrics.log b/testing/btest/Baseline/scripts.base.frameworks.metrics.basic-cluster/manager-1.metrics.log index 1677297ecc..a278bdc56a 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.metrics.basic-cluster/manager-1.metrics.log +++ b/testing/btest/Baseline/scripts.base.frameworks.metrics.basic-cluster/manager-1.metrics.log @@ -1,7 +1,10 @@ #separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - #path metrics #fields ts metric_id filter_name index.host index.str index.network value #types time enum string addr string subnet count -1317950616.401733 TEST_METRIC foo-bar 6.5.4.3 - - 4 -1317950616.401733 TEST_METRIC foo-bar 1.2.3.4 - - 6 -1317950616.401733 TEST_METRIC foo-bar 7.2.1.5 - - 2 +1324314335.570789 TEST_METRIC foo-bar 6.5.4.3 - - 4 +1324314335.570789 TEST_METRIC foo-bar 1.2.3.4 - - 6 +1324314335.570789 TEST_METRIC foo-bar 7.2.1.5 - - 2 diff --git a/testing/btest/Baseline/scripts.base.frameworks.metrics.basic/metrics.log b/testing/btest/Baseline/scripts.base.frameworks.metrics.basic/metrics.log index 45334cf3d7..8ee19c255b 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.metrics.basic/metrics.log +++ b/testing/btest/Baseline/scripts.base.frameworks.metrics.basic/metrics.log @@ -1,7 +1,10 @@ #separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - #path metrics #fields ts metric_id filter_name index.host index.str index.network value #types time enum string addr string subnet count -1315167083.455574 TEST_METRIC foo-bar 6.5.4.3 - - 2 -1315167083.455574 TEST_METRIC foo-bar 1.2.3.4 - - 3 -1315167083.455574 TEST_METRIC foo-bar 7.2.1.5 - - 1 +1324314344.807073 TEST_METRIC foo-bar 6.5.4.3 - - 2 +1324314344.807073 TEST_METRIC foo-bar 1.2.3.4 - - 3 +1324314344.807073 TEST_METRIC foo-bar 7.2.1.5 - - 1 diff --git a/testing/btest/Baseline/scripts.base.frameworks.metrics.cluster-intermediate-update/manager-1.notice.log b/testing/btest/Baseline/scripts.base.frameworks.metrics.cluster-intermediate-update/manager-1.notice.log index f5df2e96f3..59d70896fb 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.metrics.cluster-intermediate-update/manager-1.notice.log +++ b/testing/btest/Baseline/scripts.base.frameworks.metrics.cluster-intermediate-update/manager-1.notice.log @@ -1,5 +1,8 @@ #separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - #path notice -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p note msg sub src dst p n peer_descr actions policy_items suppress_for dropped remote_location.country_code remote_location.region remote_location.city remote_location.latitude remote_location.longitude metric_index.host metric_index.str metric_index.network -#types time string addr port addr port enum string string addr addr port count string table table interval bool string string string double double addr string subnet -1316952194.679491 - - - - - Test_Notice Threshold crossed by metric_index(host=1.2.3.4) 100/100 - 1.2.3.4 - - 100 manager-1 Notice::ACTION_LOG 6 3600.000000 - - - - - - 1.2.3.4 - - +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto note msg sub src dst p n peer_descr actions policy_items suppress_for dropped remote_location.country_code remote_location.region remote_location.city remote_location.latitude remote_location.longitude metric_index.host metric_index.str metric_index.network +#types time string addr port addr port enum enum string string addr addr port count string table[enum] table[count] interval bool string string string double double addr string subnet +1325633225.777902 - - - - - - Test_Notice Threshold crossed by metric_index(host=1.2.3.4) 100/100 - 1.2.3.4 - - 100 manager-1 Notice::ACTION_LOG 6 3600.000000 F - - - - - 1.2.3.4 - - diff --git a/testing/btest/Baseline/scripts.base.frameworks.metrics.notice/notice.log b/testing/btest/Baseline/scripts.base.frameworks.metrics.notice/notice.log index 33745500e0..58346b79e6 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.metrics.notice/notice.log +++ b/testing/btest/Baseline/scripts.base.frameworks.metrics.notice/notice.log @@ -1,6 +1,9 @@ #separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - #path notice -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p note msg sub src dst p n peer_descr actions policy_items suppress_for dropped remote_location.country_code remote_location.region remote_location.city remote_location.latitude remote_location.longitude metric_index.host metric_index.str metric_index.network -#types time string addr port addr port enum string string addr addr port count string table table interval bool string string string double double addr string subnet -1316952223.891502 - - - - - Test_Notice Threshold crossed by metric_index(host=1.2.3.4) 3/2 - 1.2.3.4 - - 3 bro Notice::ACTION_LOG 6 3600.000000 - - - - - - 1.2.3.4 - - -1316952223.891502 - - - - - Test_Notice Threshold crossed by metric_index(host=6.5.4.3) 2/2 - 6.5.4.3 - - 2 bro Notice::ACTION_LOG 6 3600.000000 - - - - - - 6.5.4.3 - - +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto note msg sub src dst p n peer_descr actions policy_items suppress_for dropped remote_location.country_code remote_location.region remote_location.city remote_location.latitude remote_location.longitude metric_index.host metric_index.str metric_index.network +#types time string addr port addr port enum enum string string addr addr port count string table[enum] table[count] interval bool string string string double double addr string subnet +1325633274.875473 - - - - - - Test_Notice Threshold crossed by metric_index(host=1.2.3.4) 3/2 - 1.2.3.4 - - 3 bro Notice::ACTION_LOG 6 3600.000000 F - - - - - 1.2.3.4 - - +1325633274.875473 - - - - - - Test_Notice Threshold crossed by metric_index(host=6.5.4.3) 2/2 - 6.5.4.3 - - 2 bro Notice::ACTION_LOG 6 3600.000000 F - - - - - 6.5.4.3 - - diff --git a/testing/btest/Baseline/scripts.base.frameworks.notice.cluster/manager-1.notice.log b/testing/btest/Baseline/scripts.base.frameworks.notice.cluster/manager-1.notice.log index 0662c13294..10888b21ec 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.notice.cluster/manager-1.notice.log +++ b/testing/btest/Baseline/scripts.base.frameworks.notice.cluster/manager-1.notice.log @@ -1,5 +1,8 @@ #separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - #path notice -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p note msg sub src dst p n peer_descr actions policy_items suppress_for dropped remote_location.country_code remote_location.region remote_location.city remote_location.latitude remote_location.longitude metric_index.host metric_index.str metric_index.network -#types time string addr port addr port enum string string addr addr port count string table table interval bool string string string double double addr string subnet -1316952264.931290 - - - - - Test_Notice test notice! - - - - - worker-1 Notice::ACTION_LOG 6 3600.000000 - - - - - - - - - +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto note msg sub src dst p n peer_descr actions policy_items suppress_for dropped remote_location.country_code remote_location.region remote_location.city remote_location.latitude remote_location.longitude metric_index.host metric_index.str metric_index.network +#types time string addr port addr port enum enum string string addr addr port count string table[enum] table[count] interval bool string string string double double addr string subnet +1325633122.490990 - - - - - - Test_Notice test notice! - - - - - worker-1 Notice::ACTION_LOG 6 3600.000000 F - - - - - - - - diff --git a/testing/btest/Baseline/scripts.base.frameworks.notice.mail-alarms/alarm-mail.txt b/testing/btest/Baseline/scripts.base.frameworks.notice.mail-alarms/alarm-mail.txt new file mode 100644 index 0000000000..e2cd51edd1 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.notice.mail-alarms/alarm-mail.txt @@ -0,0 +1,4 @@ +> 2005-10-07-23:23:55 Test_Notice 141.42.64.125:56730/tcp -> 125.190.109.199:80/tcp (uid arKYeMETxOg) + test + # 141.42.64.125 = 125.190.109.199 = + diff --git a/testing/btest/Baseline/scripts.base.frameworks.notice.suppression-cluster/manager-1.notice.log b/testing/btest/Baseline/scripts.base.frameworks.notice.suppression-cluster/manager-1.notice.log index 6e0214b7d3..5deac88071 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.notice.suppression-cluster/manager-1.notice.log +++ b/testing/btest/Baseline/scripts.base.frameworks.notice.suppression-cluster/manager-1.notice.log @@ -1,5 +1,8 @@ #separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - #path notice -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p note msg sub src dst p n peer_descr actions policy_items suppress_for dropped remote_location.country_code remote_location.region remote_location.city remote_location.latitude remote_location.longitude metric_index.host metric_index.str metric_index.network -#types time string addr port addr port enum string string addr addr port count string table table interval bool string string string double double addr string subnet -1316950574.408256 - - - - - Test_Notice test notice! - - - - - worker-2 Notice::ACTION_LOG 6 3600.000000 - - - - - - - - - +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto note msg sub src dst p n peer_descr actions policy_items suppress_for dropped remote_location.country_code remote_location.region remote_location.city remote_location.latitude remote_location.longitude metric_index.host metric_index.str metric_index.network +#types time string addr port addr port enum enum string string addr addr port count string table[enum] table[count] interval bool string string string double double addr string subnet +1325633150.723248 - - - - - - Test_Notice test notice! - - - - - worker-2 Notice::ACTION_LOG 6 3600.000000 F - - - - - - - - diff --git a/testing/btest/Baseline/scripts.base.frameworks.notice.suppression/notice.log b/testing/btest/Baseline/scripts.base.frameworks.notice.suppression/notice.log index 6b4c925e0f..1d168d7613 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.notice.suppression/notice.log +++ b/testing/btest/Baseline/scripts.base.frameworks.notice.suppression/notice.log @@ -1,5 +1,8 @@ #separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - #path notice -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p note msg sub src dst p n peer_descr actions policy_items suppress_for dropped remote_location.country_code remote_location.region remote_location.city remote_location.latitude remote_location.longitude -#types time string addr port addr port enum string string addr addr port count string table table interval bool string string string double double -1316950497.513136 - - - - - Test_Notice test - - - - - bro Notice::ACTION_LOG 6 3600.000000 - - - - - - +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto note msg sub src dst p n peer_descr actions policy_items suppress_for dropped remote_location.country_code remote_location.region remote_location.city remote_location.latitude remote_location.longitude +#types time string addr port addr port enum enum string string addr addr port count string table[enum] table[count] interval bool string string string double double +1325633207.922993 - - - - - - Test_Notice test - - - - - bro Notice::ACTION_LOG 6 3600.000000 F - - - - - diff --git a/testing/btest/Baseline/scripts.base.protocols.http.100-continue/http.log b/testing/btest/Baseline/scripts.base.protocols.http.100-continue/http.log index 812b4bc151..ddcea2e9c7 100644 --- a/testing/btest/Baseline/scripts.base.protocols.http.100-continue/http.log +++ b/testing/btest/Baseline/scripts.base.protocols.http.100-continue/http.log @@ -1,5 +1,8 @@ #separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - #path http #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file -#types time string addr port addr port count string string string string string count count count string count string string table string string table string string file -1237440095.634312 UWkUyAuUGXf 192.168.3.103 54102 128.146.216.51 80 1 POST www.osu.edu / - curl/7.17.1 (i386-apple-darwin8.11.1) libcurl/7.17.1 zlib/1.2.3 2001 60731 200 OK 100 Continue - - - - - text/html - - +#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string file +1237440095.634312 UWkUyAuUGXf 192.168.3.103 54102 128.146.216.51 80 1 POST www.osu.edu / - curl/7.17.1 (i386-apple-darwin8.11.1) libcurl/7.17.1 zlib/1.2.3 2001 60731 200 OK 100 Continue - (empty) - - - text/html - - diff --git a/testing/btest/Baseline/scripts.base.protocols.http.http-extract-files/http.log b/testing/btest/Baseline/scripts.base.protocols.http.http-extract-files/http.log index 386eaf8901..cec098a50b 100644 --- a/testing/btest/Baseline/scripts.base.protocols.http.http-extract-files/http.log +++ b/testing/btest/Baseline/scripts.base.protocols.http.http-extract-files/http.log @@ -1,5 +1,8 @@ #separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - #path http #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file -#types time string addr port addr port count string string string string string count count count string count string string table string string table string string file -1128727435.634189 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - - - - - text/html - http-item_141.42.64.125:56730-125.190.109.199:80_resp_1.dat +#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string file +1128727435.634189 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - (empty) - - - text/html - http-item_141.42.64.125:56730-125.190.109.199:80_resp_1.dat diff --git a/testing/btest/Baseline/scripts.base.protocols.http.http-mime-and-md5/http.log b/testing/btest/Baseline/scripts.base.protocols.http.http-mime-and-md5/http.log index 9515eb8168..d4e5679da1 100644 --- a/testing/btest/Baseline/scripts.base.protocols.http.http-mime-and-md5/http.log +++ b/testing/btest/Baseline/scripts.base.protocols.http.http-mime-and-md5/http.log @@ -1,9 +1,12 @@ #separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - #path http #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file -#types time string addr port addr port count string string string string string count count count string count string string table string string table string string file -1258577884.844956 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 1 GET www.mozilla.org /style/enhanced.css http://www.mozilla.org/projects/calendar/ Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 2675 200 OK - - - - - - - FAKE_MIME - - -1258577884.960135 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 2 GET www.mozilla.org /script/urchin.js http://www.mozilla.org/projects/calendar/ Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 21421 200 OK - - - - - - - FAKE_MIME - - -1258577885.317160 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 3 GET www.mozilla.org /images/template/screen/bullet_utility.png http://www.mozilla.org/style/screen.css Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 94 200 OK - - - - - - - FAKE_MIME - - -1258577885.349639 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 4 GET www.mozilla.org /images/template/screen/key-point-top.png http://www.mozilla.org/style/screen.css Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 2349 200 OK - - - - - - - image/png e0029eea80812e9a8e57b8d05d52938a - -1258577885.394612 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 5 GET www.mozilla.org /projects/calendar/images/header-sunbird.png http://www.mozilla.org/projects/calendar/calendar.css Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 27579 200 OK - - - - - - - image/png 30aa926344f58019d047e85ba049ca1e - +#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string file +1258577884.844956 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 1 GET www.mozilla.org /style/enhanced.css http://www.mozilla.org/projects/calendar/ Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 2675 200 OK - - - (empty) - - - FAKE_MIME - - +1258577884.960135 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 2 GET www.mozilla.org /script/urchin.js http://www.mozilla.org/projects/calendar/ Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 21421 200 OK - - - (empty) - - - FAKE_MIME - - +1258577885.317160 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 3 GET www.mozilla.org /images/template/screen/bullet_utility.png http://www.mozilla.org/style/screen.css Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 94 200 OK - - - (empty) - - - FAKE_MIME - - +1258577885.349639 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 4 GET www.mozilla.org /images/template/screen/key-point-top.png http://www.mozilla.org/style/screen.css Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 2349 200 OK - - - (empty) - - - image/png e0029eea80812e9a8e57b8d05d52938a - +1258577885.394612 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 5 GET www.mozilla.org /projects/calendar/images/header-sunbird.png http://www.mozilla.org/projects/calendar/calendar.css Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 27579 200 OK - - - (empty) - - - image/png 30aa926344f58019d047e85ba049ca1e - diff --git a/testing/btest/Baseline/scripts.base.protocols.http.http-pipelining/http.log b/testing/btest/Baseline/scripts.base.protocols.http.http-pipelining/http.log index 01d62b3981..dfaf34acbf 100644 --- a/testing/btest/Baseline/scripts.base.protocols.http.http-pipelining/http.log +++ b/testing/btest/Baseline/scripts.base.protocols.http.http-pipelining/http.log @@ -1,9 +1,12 @@ #separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - #path http #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied md5 extraction_file -#types time string addr port addr port count string string string string string count count count string count string string table string string table string file -1258577884.844956 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 1 GET www.mozilla.org /style/enhanced.css http://www.mozilla.org/projects/calendar/ Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 2675 200 OK - - - - - - - - - -1258577884.960135 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 2 GET www.mozilla.org /script/urchin.js http://www.mozilla.org/projects/calendar/ Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 21421 200 OK - - - - - - - - - -1258577885.317160 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 3 GET www.mozilla.org /images/template/screen/bullet_utility.png http://www.mozilla.org/style/screen.css Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 94 200 OK - - - - - - - - - -1258577885.349639 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 4 GET www.mozilla.org /images/template/screen/key-point-top.png http://www.mozilla.org/style/screen.css Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 2349 200 OK - - - - - - - - - -1258577885.394612 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 5 GET www.mozilla.org /projects/calendar/images/header-sunbird.png http://www.mozilla.org/projects/calendar/calendar.css Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 27579 200 OK - - - - - - - - - +#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string file +1258577884.844956 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 1 GET www.mozilla.org /style/enhanced.css http://www.mozilla.org/projects/calendar/ Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 2675 200 OK - - - (empty) - - - - - +1258577884.960135 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 2 GET www.mozilla.org /script/urchin.js http://www.mozilla.org/projects/calendar/ Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 21421 200 OK - - - (empty) - - - - - +1258577885.317160 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 3 GET www.mozilla.org /images/template/screen/bullet_utility.png http://www.mozilla.org/style/screen.css Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 94 200 OK - - - (empty) - - - - - +1258577885.349639 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 4 GET www.mozilla.org /images/template/screen/key-point-top.png http://www.mozilla.org/style/screen.css Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 2349 200 OK - - - (empty) - - - - - +1258577885.394612 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 5 GET www.mozilla.org /projects/calendar/images/header-sunbird.png http://www.mozilla.org/projects/calendar/calendar.css Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 27579 200 OK - - - (empty) - - - - - diff --git a/testing/btest/Baseline/scripts.base.protocols.irc.basic/irc.log b/testing/btest/Baseline/scripts.base.protocols.irc.basic/irc.log index d224556632..39ff897fae 100644 --- a/testing/btest/Baseline/scripts.base.protocols.irc.basic/irc.log +++ b/testing/btest/Baseline/scripts.base.protocols.irc.basic/irc.log @@ -1,8 +1,11 @@ #separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - #path irc #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p nick user channels command value addl tags dcc_file_name dcc_file_size extraction_file -#types time string addr port addr port string string table string string string table string count file -1311189164.119437 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 - - - NICK bloed - - - - - -1311189164.119437 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 bloed - - USER sdkfje sdkfje Montreal.QC.CA.Undernet.org dkdkrwq - - - - -1311189174.474127 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 bloed sdkfje - JOIN #easymovies - - - - - -1311189316.326025 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 bloed sdkfje - DCC #easymovies - - ladyvampress-default(2011-07-07)-OS.zip 42208 - +#types time string addr port addr port string string table[string] string string string table[enum] string count file +1311189164.119437 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 - - - NICK bloed - (empty) - - - +1311189164.119437 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 bloed - - USER sdkfje sdkfje Montreal.QC.CA.Undernet.org dkdkrwq (empty) - - - +1311189174.474127 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 bloed sdkfje - JOIN #easymovies (empty) (empty) - - - +1311189316.326025 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 bloed sdkfje - DCC #easymovies (empty) (empty) ladyvampress-default(2011-07-07)-OS.zip 42208 - diff --git a/testing/btest/Baseline/scripts.base.protocols.irc.dcc-extract/irc.log b/testing/btest/Baseline/scripts.base.protocols.irc.dcc-extract/irc.log index a692d2dd4d..342923ba7b 100644 --- a/testing/btest/Baseline/scripts.base.protocols.irc.dcc-extract/irc.log +++ b/testing/btest/Baseline/scripts.base.protocols.irc.dcc-extract/irc.log @@ -1,8 +1,11 @@ #separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - #path irc #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p nick user channels command value addl tags dcc_file_name dcc_file_size dcc_mime_type extraction_file -#types time string addr port addr port string string table string string string table string count string file -1311189164.119437 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 - - - NICK bloed - - - - - - -1311189164.119437 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 bloed - - USER sdkfje sdkfje Montreal.QC.CA.Undernet.org dkdkrwq - - - - - -1311189174.474127 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 bloed sdkfje - JOIN #easymovies - - - - - - -1311189316.326025 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 bloed sdkfje - DCC #easymovies - IRC::EXTRACTED_FILE ladyvampress-default(2011-07-07)-OS.zip 42208 FAKE_MIME irc-dcc-item_192.168.1.77:57655-209.197.168.151:1024_1.dat +#types time string addr port addr port string string table[string] string string string table[enum] string count string file +1311189164.119437 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 - - - NICK bloed - (empty) - - - - +1311189164.119437 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 bloed - - USER sdkfje sdkfje Montreal.QC.CA.Undernet.org dkdkrwq (empty) - - - - +1311189174.474127 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 bloed sdkfje - JOIN #easymovies (empty) (empty) - - - - +1311189316.326025 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 bloed sdkfje - DCC #easymovies (empty) IRC::EXTRACTED_FILE ladyvampress-default(2011-07-07)-OS.zip 42208 FAKE_MIME irc-dcc-item_192.168.1.77:57655-209.197.168.151:1024_1.dat diff --git a/testing/btest/Baseline/scripts.base.protocols.smtp.basic/smtp.log b/testing/btest/Baseline/scripts.base.protocols.smtp.basic/smtp.log index b93720cfe6..2c1380cb44 100644 --- a/testing/btest/Baseline/scripts.base.protocols.smtp.basic/smtp.log +++ b/testing/btest/Baseline/scripts.base.protocols.smtp.basic/smtp.log @@ -1,5 +1,8 @@ #separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - #path smtp #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth helo mailfrom rcptto date from to reply_to msg_id in_reply_to subject x_originating_ip first_received second_received last_reply path user_agent -#types time string addr port addr port count string string table string string table string string string string addr string string string vector string +#types time string addr port addr port count string string table[string] string string table[string] string string string string addr string string string vector[addr] string 1254722768.219663 arKYeMETxOg 10.10.1.4 1470 74.53.140.153 25 1 GP Mon, 5 Oct 2009 11:36:07 +0530 "Gurpartap Singh" - <000301ca4581$ef9e57f0$cedb07d0$@in> - SMTP - - - 250 OK id=1Mugho-0003Dg-Un 74.53.140.153,10.10.1.4 Microsoft Office Outlook 12.0 diff --git a/testing/btest/Baseline/scripts.base.protocols.smtp.mime-extract/smtp_entities.log b/testing/btest/Baseline/scripts.base.protocols.smtp.mime-extract/smtp_entities.log index 63b287a791..453b55932e 100644 --- a/testing/btest/Baseline/scripts.base.protocols.smtp.mime-extract/smtp_entities.log +++ b/testing/btest/Baseline/scripts.base.protocols.smtp.mime-extract/smtp_entities.log @@ -1,7 +1,10 @@ #separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - #path smtp_entities #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth filename content_len mime_type md5 extraction_file excerpt #types time string addr port addr port count string count string string file string -1254722770.692743 arKYeMETxOg 10.10.1.4 1470 74.53.140.153 25 1 - 79 FAKE_MIME - smtp-entity_10.10.1.4:1470-74.53.140.153:25_1.dat - -1254722770.692743 arKYeMETxOg 10.10.1.4 1470 74.53.140.153 25 1 - 1918 FAKE_MIME - - - -1254722770.692804 arKYeMETxOg 10.10.1.4 1470 74.53.140.153 25 1 NEWS.txt 10823 FAKE_MIME - smtp-entity_10.10.1.4:1470-74.53.140.153:25_2.dat - +1254722770.692743 arKYeMETxOg 10.10.1.4 1470 74.53.140.153 25 1 - 79 FAKE_MIME - smtp-entity_10.10.1.4:1470-74.53.140.153:25_1.dat (empty) +1254722770.692743 arKYeMETxOg 10.10.1.4 1470 74.53.140.153 25 1 - 1918 FAKE_MIME - - (empty) +1254722770.692804 arKYeMETxOg 10.10.1.4 1470 74.53.140.153 25 1 NEWS.txt 10823 FAKE_MIME - smtp-entity_10.10.1.4:1470-74.53.140.153:25_2.dat (empty) diff --git a/testing/btest/Baseline/scripts.base.protocols.smtp.mime/smtp_entities.log b/testing/btest/Baseline/scripts.base.protocols.smtp.mime/smtp_entities.log index e45d8dc757..2b471782d5 100644 --- a/testing/btest/Baseline/scripts.base.protocols.smtp.mime/smtp_entities.log +++ b/testing/btest/Baseline/scripts.base.protocols.smtp.mime/smtp_entities.log @@ -1,7 +1,10 @@ #separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - #path smtp_entities #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth filename content_len mime_type md5 extraction_file excerpt #types time string addr port addr port count string count string string file string -1254722770.692743 arKYeMETxOg 10.10.1.4 1470 74.53.140.153 25 1 - 79 FAKE_MIME 92bca2e6cdcde73647125da7dccbdd07 - - -1254722770.692743 arKYeMETxOg 10.10.1.4 1470 74.53.140.153 25 1 - 1918 FAKE_MIME - - - -1254722770.692804 arKYeMETxOg 10.10.1.4 1470 74.53.140.153 25 1 NEWS.txt 10823 FAKE_MIME a968bb0f9f9d95835b2e74c845877e87 - - +1254722770.692743 arKYeMETxOg 10.10.1.4 1470 74.53.140.153 25 1 - 79 FAKE_MIME 92bca2e6cdcde73647125da7dccbdd07 - (empty) +1254722770.692743 arKYeMETxOg 10.10.1.4 1470 74.53.140.153 25 1 - 1918 FAKE_MIME - - (empty) +1254722770.692804 arKYeMETxOg 10.10.1.4 1470 74.53.140.153 25 1 NEWS.txt 10823 FAKE_MIME a968bb0f9f9d95835b2e74c845877e87 - (empty) diff --git a/testing/btest/Baseline/scripts.policy.protocols.conn.known-hosts/knownhosts-all.log b/testing/btest/Baseline/scripts.policy.protocols.conn.known-hosts/knownhosts-all.log index cde5156594..0799292857 100644 --- a/testing/btest/Baseline/scripts.policy.protocols.conn.known-hosts/knownhosts-all.log +++ b/testing/btest/Baseline/scripts.policy.protocols.conn.known-hosts/knownhosts-all.log @@ -1,4 +1,7 @@ #separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - #path known_hosts #fields ts host #types time addr diff --git a/testing/btest/Baseline/scripts.policy.protocols.conn.known-hosts/knownhosts-local.log b/testing/btest/Baseline/scripts.policy.protocols.conn.known-hosts/knownhosts-local.log index 008eb364ed..6fdba24d39 100644 --- a/testing/btest/Baseline/scripts.policy.protocols.conn.known-hosts/knownhosts-local.log +++ b/testing/btest/Baseline/scripts.policy.protocols.conn.known-hosts/knownhosts-local.log @@ -1,4 +1,7 @@ #separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - #path known_hosts #fields ts host #types time addr diff --git a/testing/btest/Baseline/scripts.policy.protocols.conn.known-hosts/knownhosts-remote.log b/testing/btest/Baseline/scripts.policy.protocols.conn.known-hosts/knownhosts-remote.log index 43b28ded8a..9ef6ee47b7 100644 --- a/testing/btest/Baseline/scripts.policy.protocols.conn.known-hosts/knownhosts-remote.log +++ b/testing/btest/Baseline/scripts.policy.protocols.conn.known-hosts/knownhosts-remote.log @@ -1,4 +1,7 @@ #separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - #path known_hosts #fields ts host #types time addr diff --git a/testing/btest/Baseline/scripts.policy.protocols.conn.known-services/knownservices-all.log b/testing/btest/Baseline/scripts.policy.protocols.conn.known-services/knownservices-all.log index ad9fa52e1c..d53da6f693 100644 --- a/testing/btest/Baseline/scripts.policy.protocols.conn.known-services/knownservices-all.log +++ b/testing/btest/Baseline/scripts.policy.protocols.conn.known-services/knownservices-all.log @@ -1,7 +1,10 @@ #separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - #path known_services #fields ts host port_num port_proto service -#types time addr port enum table +#types time addr port enum table[string] 1308930691.049431 172.16.238.131 22 tcp SSH 1308930694.550308 172.16.238.131 80 tcp HTTP 1308930716.462556 74.125.225.81 80 tcp HTTP diff --git a/testing/btest/Baseline/scripts.policy.protocols.conn.known-services/knownservices-local.log b/testing/btest/Baseline/scripts.policy.protocols.conn.known-services/knownservices-local.log index 1607d69f24..ef1722d6a1 100644 --- a/testing/btest/Baseline/scripts.policy.protocols.conn.known-services/knownservices-local.log +++ b/testing/btest/Baseline/scripts.policy.protocols.conn.known-services/knownservices-local.log @@ -1,7 +1,10 @@ #separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - #path known_services #fields ts host port_num port_proto service -#types time addr port enum table +#types time addr port enum table[string] 1308930691.049431 172.16.238.131 22 tcp SSH 1308930694.550308 172.16.238.131 80 tcp HTTP 1308930718.361665 172.16.238.131 21 tcp FTP diff --git a/testing/btest/Baseline/scripts.policy.protocols.conn.known-services/knownservices-remote.log b/testing/btest/Baseline/scripts.policy.protocols.conn.known-services/knownservices-remote.log index 0d1210c941..3fc68cdb91 100644 --- a/testing/btest/Baseline/scripts.policy.protocols.conn.known-services/knownservices-remote.log +++ b/testing/btest/Baseline/scripts.policy.protocols.conn.known-services/knownservices-remote.log @@ -1,6 +1,9 @@ #separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - #path known_services #fields ts host port_num port_proto service -#types time addr port enum table +#types time addr port enum table[string] 1308930716.462556 74.125.225.81 80 tcp HTTP 1308930726.872485 141.142.192.39 22 tcp SSH diff --git a/testing/btest/Baseline/scripts.policy.protocols.dns.event-priority/dns.log b/testing/btest/Baseline/scripts.policy.protocols.dns.event-priority/dns.log index 945960e03e..9d80898e0f 100644 --- a/testing/btest/Baseline/scripts.policy.protocols.dns.event-priority/dns.log +++ b/testing/btest/Baseline/scripts.policy.protocols.dns.event-priority/dns.log @@ -1,5 +1,8 @@ #separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - #path dns -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto trans_id query qclass qclass_name qtype qtype_name rcode rcode_name QR AA TC RD RA Z TTL answers auth addl -#types time string addr port addr port enum count string count string count string count string bool bool bool bool bool count interval table table table -930613226.529070 UWkUyAuUGXf 212.180.42.100 25000 131.243.64.3 53 tcp 34798 - - - - - 0 NOERROR F F F F T 0 31337.000000 4.3.2.1 - - +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto trans_id query qclass qclass_name qtype qtype_name rcode rcode_name QR AA TC RD RA Z answers TTLs auth addl +#types time string addr port addr port enum count string count string count string count string bool bool bool bool bool count vector[string] vector[interval] table[string] table[string] +930613226.529070 UWkUyAuUGXf 212.180.42.100 25000 131.243.64.3 53 tcp 34798 - - - - - 0 NOERROR F F F F T 0 4.3.2.1 31337.000000 - - diff --git a/testing/btest/core/dns-init.bro b/testing/btest/core/dns-init.bro new file mode 100644 index 0000000000..5a7efff6fb --- /dev/null +++ b/testing/btest/core/dns-init.bro @@ -0,0 +1,9 @@ +# We once had a bug where DNS lookups at init time lead to an immediate crash. +# +# @TEST-EXEC: bro %INPUT >output 2>&1 +# @TEST-EXEC: btest-diff output + +const foo: set[addr] = { + google.com +}; + diff --git a/testing/btest/core/leaks/dns.bro b/testing/btest/core/leaks/dns.bro index 1dce9c2c82..3d3fdc6f09 100644 --- a/testing/btest/core/leaks/dns.bro +++ b/testing/btest/core/leaks/dns.bro @@ -4,6 +4,10 @@ # # @TEST-EXEC: HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local bro -m -r $TRACES/wikipedia.trace %INPUT +const foo: set[addr] = { + google.com +}; + # Add the state tracking information variable to the connection record event connection_established(c: connection) diff --git a/testing/btest/language/record-default-coercion.bro b/testing/btest/language/record-default-coercion.bro new file mode 100644 index 0000000000..7e717c39e2 --- /dev/null +++ b/testing/btest/language/record-default-coercion.bro @@ -0,0 +1,18 @@ +# @TEST-EXEC: bro -b %INPUT >out +# @TEST-EXEC: btest-diff out + +type MyRecord: record { + a: count &default=13; + c: count; + v: vector of string &default=vector(); +}; + +event bro_init() + { + local r: MyRecord = [$c=13]; + print r; + print |r$v|; + r$v[|r$v|] = "test"; + print r; + print |r$v|; + } diff --git a/testing/btest/scripts/base/frameworks/cluster/start-it-up.bro b/testing/btest/scripts/base/frameworks/cluster/start-it-up.bro index d1eb94d5e1..b8ee4c33e8 100644 --- a/testing/btest/scripts/base/frameworks/cluster/start-it-up.bro +++ b/testing/btest/scripts/base/frameworks/cluster/start-it-up.bro @@ -3,7 +3,7 @@ # @TEST-EXEC: btest-bg-run proxy-2 BROPATH=$BROPATH:.. CLUSTER_NODE=proxy-2 bro %INPUT # @TEST-EXEC: btest-bg-run worker-1 BROPATH=$BROPATH:.. CLUSTER_NODE=worker-1 bro %INPUT # @TEST-EXEC: btest-bg-run worker-2 BROPATH=$BROPATH:.. CLUSTER_NODE=worker-2 bro %INPUT -# @TEST-EXEC: btest-bg-wait -k 2 +# @TEST-EXEC: btest-bg-wait -k 10 # @TEST-EXEC: btest-diff manager-1/.stdout # @TEST-EXEC: btest-diff proxy-1/.stdout # @TEST-EXEC: btest-diff proxy-2/.stdout @@ -23,4 +23,4 @@ redef Cluster::nodes = { event remote_connection_handshake_done(p: event_peer) { print "Connected to a peer"; - } \ No newline at end of file + } diff --git a/testing/btest/scripts/base/frameworks/communication/communication_log_baseline.bro b/testing/btest/scripts/base/frameworks/communication/communication_log_baseline.bro new file mode 100644 index 0000000000..3a4c1253eb --- /dev/null +++ b/testing/btest/scripts/base/frameworks/communication/communication_log_baseline.bro @@ -0,0 +1,37 @@ +# +# @TEST-EXEC: btest-bg-run receiver bro -b ../receiver.bro +# @TEST-EXEC: btest-bg-run sender bro -b ../sender.bro +# @TEST-EXEC: btest-bg-wait -k 10 +# +# Don't diff the receiver log just because port is always going to change +# @TEST-EXEC: egrep -v 'pid|socket buffer size' sender/communication.log >send.log +# @TEST-EXEC: btest-diff send.log + +@TEST-START-FILE sender.bro + +@load base/frameworks/communication/main + +redef Communication::nodes += { + ["foo"] = [$host = 127.0.0.1, $events = /NOTHING/, $connect=T] +}; + +event remote_connection_established(p: event_peer) + { + terminate_communication(); + terminate(); + } + +@TEST-END-FILE + +############# + +@TEST-START-FILE receiver.bro + +@load frameworks/communication/listen + +event remote_connection_closed(p: event_peer) + { + terminate(); + } + +@TEST-END-FILE diff --git a/testing/btest/scripts/base/frameworks/control/configuration_update.bro b/testing/btest/scripts/base/frameworks/control/configuration_update.bro index eb86ec58e8..9b16faee69 100644 --- a/testing/btest/scripts/base/frameworks/control/configuration_update.bro +++ b/testing/btest/scripts/base/frameworks/control/configuration_update.bro @@ -1,7 +1,7 @@ # @TEST-EXEC: btest-bg-run controllee BROPATH=$BROPATH:.. bro %INPUT frameworks/control/controllee Communication::listen_port=65531/tcp # @TEST-EXEC: btest-bg-run controller BROPATH=$BROPATH:.. bro %INPUT test-redef frameworks/control/controller Control::host=127.0.0.1 Control::host_port=65531/tcp Control::cmd=configuration_update # @TEST-EXEC: btest-bg-run controller2 BROPATH=$BROPATH:.. bro %INPUT frameworks/control/controller Control::host=127.0.0.1 Control::host_port=65531/tcp Control::cmd=shutdown -# @TEST-EXEC: btest-bg-wait 1 +# @TEST-EXEC: btest-bg-wait 10 # @TEST-EXEC: btest-diff controllee/.stdout redef Communication::nodes = { @@ -23,4 +23,4 @@ event bro_init() event bro_done() { print test_var; - } \ No newline at end of file + } diff --git a/testing/btest/scripts/base/frameworks/control/id_value.bro b/testing/btest/scripts/base/frameworks/control/id_value.bro index 90a5367f76..e06fa46e74 100644 --- a/testing/btest/scripts/base/frameworks/control/id_value.bro +++ b/testing/btest/scripts/base/frameworks/control/id_value.bro @@ -1,6 +1,6 @@ # @TEST-EXEC: btest-bg-run controllee BROPATH=$BROPATH:.. bro %INPUT only-for-controllee frameworks/control/controllee Communication::listen_port=65532/tcp # @TEST-EXEC: btest-bg-run controller BROPATH=$BROPATH:.. bro %INPUT frameworks/control/controller Control::host=127.0.0.1 Control::host_port=65532/tcp Control::cmd=id_value Control::arg=test_var -# @TEST-EXEC: btest-bg-wait -k 1 +# @TEST-EXEC: btest-bg-wait -k 10 # @TEST-EXEC: btest-diff controller/.stdout redef Communication::nodes = { diff --git a/testing/btest/scripts/base/frameworks/control/shutdown.bro b/testing/btest/scripts/base/frameworks/control/shutdown.bro index 73319a7c4a..9953a8382a 100644 --- a/testing/btest/scripts/base/frameworks/control/shutdown.bro +++ b/testing/btest/scripts/base/frameworks/control/shutdown.bro @@ -1,6 +1,6 @@ # @TEST-EXEC: btest-bg-run controllee BROPATH=$BROPATH:.. bro %INPUT frameworks/control/controllee Communication::listen_port=65530/tcp # @TEST-EXEC: btest-bg-run controller BROPATH=$BROPATH:.. bro %INPUT frameworks/control/controller Control::host=127.0.0.1 Control::host_port=65530/tcp Control::cmd=shutdown -# @TEST-EXEC: btest-bg-wait 1 +# @TEST-EXEC: btest-bg-wait 10 redef Communication::nodes = { # We're waiting for connections from this host for control. diff --git a/testing/btest/scripts/base/frameworks/logging/ascii-escape-notset-str.bro b/testing/btest/scripts/base/frameworks/logging/ascii-escape-notset-str.bro new file mode 100644 index 0000000000..8c1401b179 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/logging/ascii-escape-notset-str.bro @@ -0,0 +1,23 @@ +# +# @TEST-EXEC: bro -b %INPUT +# @TEST-EXEC: btest-diff test.log + +module Test; + +export { + redef enum Log::ID += { LOG }; + + type Log: record { + x: string &optional; + y: string &optional; + z: string &optional; + } &log; +} + +event bro_init() +{ + Log::create_stream(Test::LOG, [$columns=Log]); + Log::write(Test::LOG, [$x=LogAscii::unset_field, $z=""]); +} + + diff --git a/testing/btest/scripts/base/frameworks/logging/ascii-escape-set-separator.bro b/testing/btest/scripts/base/frameworks/logging/ascii-escape-set-separator.bro new file mode 100644 index 0000000000..f5fb7a6259 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/logging/ascii-escape-set-separator.bro @@ -0,0 +1,21 @@ +# @TEST-EXEC: bro -b %INPUT +# @TEST-EXEC: btest-diff test.log + +module Test; + +export { + redef enum Log::ID += { LOG }; + + type Log: record { + ss: set[string]; + } &log; +} + +event bro_init() +{ + Log::create_stream(Test::LOG, [$columns=Log]); + + + Log::write(Test::LOG, [$ss=set("AA", ",", ",,", "CC")]); +} + diff --git a/testing/btest/scripts/base/frameworks/logging/env-ext.test b/testing/btest/scripts/base/frameworks/logging/env-ext.test new file mode 100644 index 0000000000..e9f690caa4 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/logging/env-ext.test @@ -0,0 +1,2 @@ +# @TEST-EXEC: BRO_LOG_SUFFIX=txt bro -r $TRACES/wikipedia.trace +# @TEST-EXEC: test -f conn.txt diff --git a/testing/btest/scripts/base/frameworks/logging/remote-types.bro b/testing/btest/scripts/base/frameworks/logging/remote-types.bro index 9af45cf991..4e866cc985 100644 --- a/testing/btest/scripts/base/frameworks/logging/remote-types.bro +++ b/testing/btest/scripts/base/frameworks/logging/remote-types.bro @@ -1,7 +1,7 @@ # # @TEST-EXEC: btest-bg-run sender bro --pseudo-realtime %INPUT ../sender.bro # @TEST-EXEC: btest-bg-run receiver bro --pseudo-realtime %INPUT ../receiver.bro -# @TEST-EXEC: btest-bg-wait -k 1 +# @TEST-EXEC: btest-bg-wait -k 10 # @TEST-EXEC: btest-diff receiver/test.log # @TEST-EXEC: cmp receiver/test.log sender/test.log diff --git a/testing/btest/scripts/base/frameworks/logging/remote.bro b/testing/btest/scripts/base/frameworks/logging/remote.bro index b244c72cdf..8ed3405aed 100644 --- a/testing/btest/scripts/base/frameworks/logging/remote.bro +++ b/testing/btest/scripts/base/frameworks/logging/remote.bro @@ -3,7 +3,7 @@ # @TEST-EXEC: sleep 1 # @TEST-EXEC: btest-bg-run receiver bro --pseudo-realtime %INPUT ../receiver.bro # @TEST-EXEC: sleep 1 -# @TEST-EXEC: btest-bg-wait -k 1 +# @TEST-EXEC: btest-bg-wait -k 10 # @TEST-EXEC: btest-diff sender/test.log # @TEST-EXEC: btest-diff sender/test.failure.log # @TEST-EXEC: btest-diff sender/test.success.log diff --git a/testing/btest/scripts/base/frameworks/metrics/basic-cluster.bro b/testing/btest/scripts/base/frameworks/metrics/basic-cluster.bro index 4b7f177f15..23b87053ab 100644 --- a/testing/btest/scripts/base/frameworks/metrics/basic-cluster.bro +++ b/testing/btest/scripts/base/frameworks/metrics/basic-cluster.bro @@ -3,7 +3,7 @@ # @TEST-EXEC: sleep 1 # @TEST-EXEC: btest-bg-run worker-1 BROPATH=$BROPATH:.. CLUSTER_NODE=worker-1 bro %INPUT # @TEST-EXEC: btest-bg-run worker-2 BROPATH=$BROPATH:.. CLUSTER_NODE=worker-2 bro %INPUT -# @TEST-EXEC: btest-bg-wait -k 6 +# @TEST-EXEC: btest-bg-wait -k 10 # @TEST-EXEC: btest-diff manager-1/metrics.log @TEST-START-FILE cluster-layout.bro diff --git a/testing/btest/scripts/base/frameworks/metrics/cluster-intermediate-update.bro b/testing/btest/scripts/base/frameworks/metrics/cluster-intermediate-update.bro index 89d771e05e..45d44898aa 100644 --- a/testing/btest/scripts/base/frameworks/metrics/cluster-intermediate-update.bro +++ b/testing/btest/scripts/base/frameworks/metrics/cluster-intermediate-update.bro @@ -3,7 +3,7 @@ # @TEST-EXEC: sleep 1 # @TEST-EXEC: btest-bg-run worker-1 BROPATH=$BROPATH:.. CLUSTER_NODE=worker-1 bro %INPUT # @TEST-EXEC: btest-bg-run worker-2 BROPATH=$BROPATH:.. CLUSTER_NODE=worker-2 bro %INPUT -# @TEST-EXEC: btest-bg-wait -k 5 +# @TEST-EXEC: btest-bg-wait -k 10 # @TEST-EXEC: btest-diff manager-1/notice.log @TEST-START-FILE cluster-layout.bro diff --git a/testing/btest/scripts/base/frameworks/notice/cluster.bro b/testing/btest/scripts/base/frameworks/notice/cluster.bro index f44ba72f3a..125d021d82 100644 --- a/testing/btest/scripts/base/frameworks/notice/cluster.bro +++ b/testing/btest/scripts/base/frameworks/notice/cluster.bro @@ -2,7 +2,7 @@ # @TEST-EXEC: btest-bg-run proxy-1 BROPATH=$BROPATH:.. CLUSTER_NODE=proxy-1 bro %INPUT # @TEST-EXEC: sleep 1 # @TEST-EXEC: btest-bg-run worker-1 BROPATH=$BROPATH:.. CLUSTER_NODE=worker-1 bro %INPUT -# @TEST-EXEC: btest-bg-wait -k 6 +# @TEST-EXEC: btest-bg-wait -k 10 # @TEST-EXEC: btest-diff manager-1/notice.log @TEST-START-FILE cluster-layout.bro diff --git a/testing/btest/scripts/base/frameworks/notice/mail-alarms.bro b/testing/btest/scripts/base/frameworks/notice/mail-alarms.bro new file mode 100644 index 0000000000..3116b1025a --- /dev/null +++ b/testing/btest/scripts/base/frameworks/notice/mail-alarms.bro @@ -0,0 +1,17 @@ +# @TEST-EXEC: bro -C -r $TRACES/web.trace %INPUT +# @TEST-EXEC: btest-diff alarm-mail.txt + +redef Notice::policy += { [$action = Notice::ACTION_ALARM, $priority = 1 ] }; +redef Notice::force_email_summaries = T; + +redef enum Notice::Type += { + Test_Notice, +}; + +event connection_established(c: connection) + { + NOTICE([$note=Test_Notice, $conn=c, $msg="test", $identifier="static"]); + } + + + diff --git a/testing/btest/scripts/base/frameworks/notice/suppression-cluster.bro b/testing/btest/scripts/base/frameworks/notice/suppression-cluster.bro index a7e720d5f5..e084fb74e0 100644 --- a/testing/btest/scripts/base/frameworks/notice/suppression-cluster.bro +++ b/testing/btest/scripts/base/frameworks/notice/suppression-cluster.bro @@ -3,7 +3,7 @@ # @TEST-EXEC: sleep 1 # @TEST-EXEC: btest-bg-run worker-1 BROPATH=$BROPATH:.. CLUSTER_NODE=worker-1 bro %INPUT # @TEST-EXEC: btest-bg-run worker-2 BROPATH=$BROPATH:.. CLUSTER_NODE=worker-2 bro %INPUT -# @TEST-EXEC: btest-bg-wait -k 5 +# @TEST-EXEC: btest-bg-wait -k 10 # @TEST-EXEC: btest-diff manager-1/notice.log @TEST-START-FILE cluster-layout.bro diff --git a/testing/btest/scripts/policy/protocols/http/test-sql-injection-regex.bro b/testing/btest/scripts/policy/protocols/http/test-sql-injection-regex.bro index bf8be22210..2e82eb9dfb 100644 --- a/testing/btest/scripts/policy/protocols/http/test-sql-injection-regex.bro +++ b/testing/btest/scripts/policy/protocols/http/test-sql-injection-regex.bro @@ -42,6 +42,8 @@ event bro_init () #add positive_matches["/index.asp?ARF_ID=(1/(1-(asc(mid(now(),18,1))\(2^7) mod 2)))"]; #add positive_matches["/index.php' and 1=convert(int,(select top 1 table_name from information_schema.tables))--sp_password"]; #add positive_matches["/index.php?id=873 and user=0--"]; + #add positive_matches["?id=1;+if+(1=1)+waitfor+delay+'00:00:01'--9"]; + #add positive_matches["?id=1+and+if(1=1,BENCHMARK(728000,MD5(0x41)),0)9"]; # The positive_matches below are from the mod_security evasion challenge. # All supported attacks are uncommented. @@ -95,14 +97,6 @@ event bro_init () #add negative_matches["/index/hmm.gif?utmdt=Record > Create a Graph"]; #add negative_matches["/index.php?test='||\x0aTO_CHAR(foo_bar.Foo_Bar_ID)||"]; - local regex = - /[\?&][^[:blank:]\x00-\x37\|]+?=[\-[:alnum:]%]+([[:blank:]\x00-\x37]|\/\*.*?\*\/)*['"]?([[:blank:]\x00-\x37]|\/\*.*?\*\/|\)?;)+.*?([hH][aA][vV][iI][nN][gG]|[uU][nN][iI][oO][nN]|[eE][xX][eE][cC]|[sS][eE][lL][eE][cC][tT]|[dD][eE][lL][eE][tT][eE]|[dD][rR][oO][pP]|[dD][eE][cC][lL][aA][rR][eE]|[cC][rR][eE][aA][tT][eE]|[iI][nN][sS][eE][rR][tT])([[:blank:]\x00-\x37]|\/\*.*?\*\/)+/ - | /[\?&][^[:blank:]\x00-\x37\|]+?=[\-0-9%]+([[:blank:]\x00-\x37]|\/\*.*?\*\/)*['"]?([[:blank:]\x00-\x37]|\/\*.*?\*\/|\)?;)+([xX]?[oO][rR]|[nN]?[aA][nN][dD])([[:blank:]\x00-\x37]|\/\*.*?\*\/)+['"]?(([^a-zA-Z&]+)?=|[eE][xX][iI][sS][tT][sS])/ - | /[\?&][^[:blank:]\x00-\x37]+?=[\-0-9%]*([[:blank:]\x00-\x37]|\/\*.*?\*\/)*['"]([[:blank:]\x00-\x37]|\/\*.*?\*\/)*(-|=|\+|\|\|)([[:blank:]\x00-\x37]|\/\*.*?\*\/)*([0-9]|\(?[cC][oO][nN][vV][eE][rR][tT]|[cC][aA][sS][tT])/ - | /[\?&][^[:blank:]\x00-\x37\|]+?=([[:blank:]\x00-\x37]|\/\*.*?\*\/)*['"]([[:blank:]\x00-\x37]|\/\*.*?\*\/|;)*([xX]?[oO][rR]|[nN]?[aA][nN][dD]|[hH][aA][vV][iI][nN][gG]|[uU][nN][iI][oO][nN]|[eE][xX][eE][cC]|[sS][eE][lL][eE][cC][tT]|[dD][eE][lL][eE][tT][eE]|[dD][rR][oO][pP]|[dD][eE][cC][lL][aA][rR][eE]|[cC][rR][eE][aA][tT][eE]|[rR][eE][gG][eE][xX][pP]|[iI][nN][sS][eE][rR][tT])([[:blank:]\x00-\x37]|\/\*.*?\*\/|[\[(])+[a-zA-Z&]{2,}/ - | /[\?&][^[:blank:]\x00-\x37]+?=[^\.]*?([cC][hH][aA][rR]|[aA][sS][cC][iI][iI]|[sS][uU][bB][sS][tT][rR][iI][nN][gG]|[tT][rR][uU][nN][cC][aA][tT][eE]|[vV][eE][rR][sS][iI][oO][nN]|[lL][eE][nN][gG][tT][hH])\(/ - | /\/\*![[:digit:]]{5}.*?\*\//; - print "If anything besides this line prints out, there is a problem."; for ( test in positive_matches ) { diff --git a/testing/external/scripts/diff-all b/testing/external/scripts/diff-all index 597c769687..1fd633b1e9 100755 --- a/testing/external/scripts/diff-all +++ b/testing/external/scripts/diff-all @@ -1,6 +1,10 @@ #! /usr/bin/env bash # -# Runs btest-diff on $@ and fails if any fails. +# Runs btest-diff on $@ and fails if any fails. If $@ contains globs, we expand +# them relative to *both* the current directory and the test's baseline +# directory so that we spot missing files. Note that you will need to quote +# the globals in the TEST-EXEC line as otherwise they will have been expanded relative +# to the current directory already when this scripts runs. diag=$TEST_DIAGNOSTICS @@ -14,8 +18,20 @@ fi rc=0; -for i in $@; do - if [[ "$i" != "loaded_scripts.log" && "$i" != "prof.log" ]]; then +files_cwd=`ls $@` +files_baseline=`cd $TEST_BASELINE && ls $@` + +for i in `echo $files_cwd $files_baseline | sort | uniq`; do + if [[ "$i" != "loaded_scripts.log" && "$i" != "prof.log" && "$i" != "debug.log" ]]; then + + if [[ "$i" == "reporter.log" ]]; then + # Do not diff the reporter.log if it only complains about missing + # GeoIP support. + if ! egrep -v "^#|Bro was not configured for GeoIP support" $i; then + continue + fi + fi + if ! btest-diff $i; then echo "" >>$diag echo "#### btest-diff $i" >>$diag diff --git a/testing/external/scripts/perftools-adapt-paths b/testing/external/scripts/perftools-adapt-paths new file mode 100755 index 0000000000..2eda2477c7 --- /dev/null +++ b/testing/external/scripts/perftools-adapt-paths @@ -0,0 +1,10 @@ +#! /usr/bin/env bash +# +# Adapts relative paths in perftools stderr output to work +# directly from the top-level test directory. +# +# Returns an exit code > 0 if there's a leak. + +cat $1 | sed "s#bro *\"\./#../../../build/src/bro \".tmp/$TEST_NAME/#g" | sed 's/ *--gv//g' >$1.tmp && mv $1.tmp $1 + +grep -q "No leaks found" $1 diff --git a/testing/external/scripts/skel/test.skeleton b/testing/external/scripts/skel/test.skeleton index becd970d78..a76f3d4d09 100644 --- a/testing/external/scripts/skel/test.skeleton +++ b/testing/external/scripts/skel/test.skeleton @@ -1,5 +1,5 @@ # @TEST-EXEC: zcat $TRACES/trace.gz | bro -r - %INPUT -# @TEST-EXEC: $SCRIPTS/diff-all *.log +# @TEST-EXEC: $SCRIPTS/diff-all '*.log' @load testing-setup @load test-all-policy diff --git a/testing/scripts/diff-canonifier-external b/testing/scripts/diff-canonifier-external index 76210cc494..1f953183d3 100755 --- a/testing/scripts/diff-canonifier-external +++ b/testing/scripts/diff-canonifier-external @@ -5,3 +5,4 @@ `dirname $0`/diff-remove-timestamps \ | `dirname $0`/diff-remove-uids \ | `dirname $0`/diff-remove-mime-types \ + | `dirname $0`/diff-remove-x509-names \ diff --git a/testing/scripts/diff-remove-x509-names b/testing/scripts/diff-remove-x509-names new file mode 100755 index 0000000000..6209edfc65 --- /dev/null +++ b/testing/scripts/diff-remove-x509-names @@ -0,0 +1,32 @@ +#! /usr/bin/awk -f +# +# A diff canonifier that removes all X.509 Distinguished Name subject fields +# because that output can differ depending on installed OpenSSL version. + +BEGIN { FS="\t"; OFS="\t"; s_col = -1; i_col = -1 } + +/^#fields/ { + for ( i = 2; i < NF; ++i ) + { + if ( $i == "subject" ) + s_col = i-1; + if ( $i == "issuer_subject" ) + i_col = i-1; + } +} + +s_col >= 0 { + if ( $s_col != "-" ) + # Mark that it's set, but ignore content. + $s_col = "+"; +} + +i_col >= 0 { + if ( $i_col != "-" ) + # Mark that it's set, but ignore content. + $i_col = "+"; +} + +{ + print; +}