Merge remote branch 'origin/master' into topic/seth/ssl-binpac

CHANGES

@@ -1,3 +1,130 @@
+1.6-dev.94 Thu Apr 21 19:51:38 PDT 2011
+
+- Fixing generation of config.h. (Jon Siwek)
+    
+- Updates and tests for NetBIOS name BiF. (Seth Hall)
+  
+- Fixing do_split bug(), and adding a test. (Seth Hall)
+    
+- When Bro is given a PRNG seed, it now uses its own internal random
+  number generator that produces consistent results across sytems.
+  Note that this internal generator isn't very good, so it should only
+  be used for testing purpses. (Robin Sommer)
+    
+- The BTest configuration now sets the environemnt variables TZ=UTC
+  and LANG=C to ensure consistent results. (Robin Sommer)
+    
+- Logging fixes. (Robin Sommer)
+    
+1.6-dev.88 Wed Apr 20 20:43:48 PDT 2011
+
+- Implementation of Bro's new logging framework. We will document this
+  separately. (Robin Sommer)
+
+- Already defined record types can now be further extended via the
+  '+=' operator. The added fields must be either &optional or have a
+  &default value. (Robin Sommer)
+    
+  Example:
+    
+        type Foo: record {
+            a: count;
+            b: count &optional;
+        };
+    
+        redef record Foo += {
+            c: count &default=42;
+            d: count &optional;
+        };
+    
+        global f: Foo = [$a=21];
+    
+        print f;
+    
+  Output:
+   
+        [a=21, b=<uninitialized>, c=42, d=<uninitialized>]
+
+- Enabling assignment of empty vectors ("vector()"). (Robin Sommer)
+
+- Fixing attributes to allow &default attributes to be associated with
+  records fields of type tables/sets/vector. (Robin Sommer)
+    
+- '[]' is now a valid record constructor. (Robin Sommer)
+  
+- A instance of a record type A is now coercable into one of type B if
+  the fields of type A are a subset of those of type B. (Robin Sommer)
+
+- A number of bug fixes and enhancements for record/set/table/vector
+  coercion. (Robin Sommer)
+
+- Fixing a problem with records that have optional fields when used as
+  table/set indices. Addresses #367. (Robin Sommer)
+
+- Fixing an off-by-one error in join_string_vec(). (Seth Hall)
+  
+- Updating to_count() to cope with 64bit ints. (Seth Hall)
+  
+- A new BiF count_to_v4_addr() to turn a count into an IPv4 address.
+  (Seth Hall)
+
+1.6-dev.80 Mon Apr 18 14:50:54 PDT 2011
+
+- New framework for generating documentation from Bro scripts. (Jon
+  Siwek)
+
+  This includes:
+
+    - Changes to Bro's scanner/parser to facilitate automatic
+      generation of Bro policy script documentation in
+      reStructuredText format.
+
+    - New command line flags -Z/--doc-scripts to enable the new doc
+      generation mode.
+
+    - Changes to bifcl to pass comments starting with "##" through
+      into the generated .bro script.
+
+    - A "doc" build target for the top-level Makefile to first
+      generate reStructuredText for a defined set of Bro policy
+      scripts, and then run that through Sphinx to create HTML
+      documentation.
+
+1.6-dev.78 Mon Apr 18 12:52:55 PDT 2011
+
+- Adding files to CMake build targets so they show up in generated IDE
+  projects. This addresses #413. (Jon Siwek)
+    
+- Fix unnecessary config.h preprocessor (re)definitions. This
+  addresses #414. (Jon Siwek)
+
+- Updating istate tests. (Robin Sommer)
+  
+- Adding files to CMake build targets so they show up in generated IDE
+  projects.
+    
+- Adding new environment variable BRO_SEED_FILE to set the seed file
+  for the random number generator. (Robin Sommer)
+    
+1.6-dev.71 Fri Apr  1 16:06:33 PDT 2011
+
+- Removing code for the following no longer supported functionality.
+  
+    * Trace rewriting.
+    * DFA state expiration in regexp engine.
+    * Active mapping.
+    * Unused hash functions.
+
+  (Robin Sommer)
+
+- Fixing crashes when SSL is not configured correctly. (Robin Sommer)
+
+1.6-dev.66 Tue Mar 29 21:52:01 PDT 2011
+
+- Initial btest setup (Don Appleman and Robin Sommer)
+  
+- Porting the istate tests to btest (not finished) (Robin Sommer)
+
 1.6-dev.63 Mon Mar 21 16:31:15 PDT 2011
 
 - Changes to the way user-modifiable config files are installed  (Jon Siwek)

CMakeLists.txt

@@ -51,6 +51,11 @@ file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/bro-path-dev.csh
      "setenv BROPATH `${CMAKE_CURRENT_BINARY_DIR}/bro-path-dev`\n")
 
 file(STRINGS "${CMAKE_CURRENT_SOURCE_DIR}/VERSION" VERSION LIMIT_COUNT 1)
+string(REPLACE "." " " version_numbers ${VERSION})
+separate_arguments(version_numbers)
+list(GET version_numbers 0 VERSION_MAJOR)
+list(GET version_numbers 1 VERSION_MINOR)
+set(VERSION_MAJ_MIN "${VERSION_MAJOR}.${VERSION_MINOR}")
 
 set(EXTRA_COMPILE_FLAGS "-Wall -Wno-unused")
 
@@ -170,8 +175,7 @@ include_directories(${CMAKE_CURRENT_BINARY_DIR})
 
 add_subdirectory(src)
 add_subdirectory(policy)
-#add_subdirectory(scripts)
+add_subdirectory(doc)
-#add_subdirectory(doc)
 
 include(CheckOptionalBuildSources)
 

Makefile

@@ -23,6 +23,13 @@ install: configured
 
 clean: configured
 	( cd $(BUILD) && make clean )
+	( cd $(BUILD) && make doc-clean )
+
+doc: configured
+	( cd $(BUILD) && make doc )
+
+doc-clean: configured
+	( cd $(BUILD) && make doc-clean )
 
 dist: cmake_version
 	# Minimum Bro source package

VERSION

@@ -1 +1 @@
-1.6-dev.63
+1.6-dev.94

aux/binpac

@@ -1 +1 @@
-Subproject commit 106cd9782f1f9443743b49adccae26f0ee72621c
+Subproject commit 43994f50b320ff9e1d129b87bbe3a26dbfb3e33d

aux/bro-aux

@@ -1 +1 @@
-Subproject commit 764b6167922662051b62d269ec0fbd14e2ce0c02
+Subproject commit 3ec53f69fd45d8e2c0950dc0e7af117bb3e02c0d

aux/broccoli

@@ -1 +1 @@
-Subproject commit d693ba5f14c33a97e4f149e49196281cc075d289
+Subproject commit 9c5d8c882411dc26de8ef92ff5d77b524b9d48ee

aux/broctl

@@ -1 +1 @@
-Subproject commit b26dc6dc6a9080abe7a062d0882e813acbb63248
+Subproject commit 7f337459574d027559e419219e83c65d8a6e1c23

aux/btest

@@ -1 +1 @@
-Subproject commit acf25f34ab48568f5db919c9e3505ed319daa4e5
+Subproject commit aee56f4632c9a4224727da1b535b673a4bcb5d3b

cmake/CheckFunctions.cmake

@@ -7,9 +7,6 @@ check_function_exists(strerror HAVE_STRERROR)
 check_function_exists(strsep HAVE_STRSEP)
 check_function_exists(sigset HAVE_SIGSET)
 
-if (HAVE_SIGSET)
+if (NOT HAVE_SIGSET)
-    set(SIG_FUNC sigset)
-else ()
-    set(SIG_FUNC signal)
     check_function_exists(sigaction HAVE_SIGACTION)
 endif ()

cmake/CheckTypes.cmake

@@ -4,43 +4,28 @@ check_type_size("long int" SIZEOF_LONG_INT)
 check_type_size("long long" SIZEOF_LONG_LONG)
 check_type_size("void *" SIZEOF_VOID_P)
 
+# checks existence of ${_type}, and if it does not, sets CMake variable ${_var}
+# to alternative type, ${_alt_type}
+macro(CheckType _type _alt_type _var)
+    # don't perform check if we have a result from a previous CMake run
+    if (NOT HAVE_${_var})
+        check_type_size(${_type} ${_var})
+        if (NOT ${_var})
+            set(${_var} ${_alt_type})
+        else ()
+            unset(${_var})
+            unset(${_var} CACHE)
+        endif ()
+    endif ()
+endmacro(CheckType _type _alt_type _var)
+
 set(CMAKE_EXTRA_INCLUDE_FILES sys/types.h)
-
+CheckType(int32_t   int     int32_t)
-check_type_size(int32_t INT32_T)
+CheckType(u_int32_t u_int   u_int32_t)
-if (INT32_T)
+CheckType(u_int16_t u_short u_int16_t)
-    set(INT32_T int32_t)
+CheckType(u_int8_t  u_char  u_int8_t)
-else()
-    set(INT32_T int)
-endif()
-
-check_type_size(u_int32_t U_INT32_T)
-if (U_INT32_T)
-    set(U_INT32_T u_int32_t)
-else ()
-    set(INT32_T u_int)
-endif ()
-
-check_type_size(u_int16_t U_INT16_T)
-if (U_INT16_T)
-    set(U_INT16_T u_int16_t)
-else ()
-    set(INT16_T u_short)
-endif ()
-
-check_type_size(u_int8_t U_INT8_T)
-if (U_INT8_T)
-    set(U_INT8_T u_int8_t)
-else ()
-    set(INT8_T u_char)
-endif ()
-
 set(CMAKE_EXTRA_INCLUDE_FILES)
 
 set(CMAKE_EXTRA_INCLUDE_FILES sys/socket.h)
-check_type_size(socklen_t SOCKLEN_T)
+CheckType(socklen_t int     socklen_t)
-if (SOCKLEN_T)
-    set(SOCKLEN_T socklen_t)
-else ()
-    set(SOCKLEN_T int)
-endif ()
 set(CMAKE_EXTRA_INCLUDE_FILES)

config.h.in

@@ -130,19 +130,21 @@
 #endif
 
 /* Define int32_t */
-#define int32_t @INT32_T@
+#cmakedefine int32_t @int32_t@
 
 /* use sigset() instead of signal() */
-#define signal @SIG_FUNC@
+#ifdef HAVE_SIGSET
+#define signal sigset
+#endif
 
 /* define to int if socklen_t not available */
-#define socklen_t @SOCKLEN_T@
+#cmakedefine socklen_t @socklen_t@
 
 /* Define u_int16_t */
-#define u_int16_t @U_INT16_T@
+#cmakedefine u_int16_t @u_int16_t@
 
 /* Define u_int32_t */
-#define u_int32_t @U_INT32_T@
+#cmakedefine u_int32_t @u_int32_t@
 
 /* Define u_int8_t */
-#define u_int8_t @U_INT8_T@
+#cmakedefine u_int8_t @u_int8_t@

doc/CMakeLists.txt

@@ -0,0 +1,42 @@
+set(POLICY_SRC_DIR ${PROJECT_SOURCE_DIR}/policy)
+set(DOC_OUTPUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/out)
+set(DOC_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/source)
+set(DOC_SOURCE_WORKDIR ${CMAKE_CURRENT_BINARY_DIR}/source)
+
+file(GLOB_RECURSE DOC_SOURCES FOLLOW_SYMLINKS "*")
+
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/conf.py.in
+               ${CMAKE_CURRENT_BINARY_DIR}/conf.py
+               @ONLY)
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/scripts/generate_reST_docs.py.in
+               ${CMAKE_CURRENT_BINARY_DIR}/generate_reST_docs.py
+               @ONLY)
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/scripts/BroToReST.py.in
+               ${CMAKE_CURRENT_BINARY_DIR}/BroToReST.py
+               @ONLY)
+
+add_custom_target(doc
+                  COMMAND "${CMAKE_COMMAND}" -E copy_directory
+                          ${DOC_SOURCE_DIR}
+                          ${DOC_SOURCE_WORKDIR}
+                  COMMAND python generate_reST_docs.py
+                  COMMAND sphinx-build
+                          -b html
+                          -c ${CMAKE_CURRENT_BINARY_DIR}
+                          -d ${DOC_OUTPUT_DIR}/doctrees
+                          ${DOC_SOURCE_WORKDIR}
+                          ${DOC_OUTPUT_DIR}/html
+                  WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
+                  COMMENT "[Sphinx] Generating Script Documentation"
+                  VERBATIM
+                  # SOURCES just adds stuff to IDE projects as a convienience
+                  SOURCES ${DOC_SOURCES})
+
+add_dependencies(doc bro doc-clean)
+
+add_custom_target(doc-clean
+                  COMMAND "${CMAKE_COMMAND}" -E remove_directory
+                          ${CMAKE_CURRENT_BINARY_DIR}/source
+                  COMMAND "${CMAKE_COMMAND}" -E remove_directory
+                          ${DOC_OUTPUT_DIR}
+                  VERBATIM)

doc/README

@@ -1 +1,44 @@
-TODO.
+This directory contains scripts and templates that can be used to automate
+the generation of Bro script documentation.  Two build targets are defined
+by CMake:
+
+``make doc``
+
+    This target depends on a Python interpreter (>=2.5) and
+    `Sphinx <http://sphinx.pocoo.org/>`_ being installed.  Sphinx can be
+    installed like::
+
+        > sudo easy_install sphinx
+
+    This target will also first build the bro binary if it is not already
+    since the generation of reStructuredText (reST) documentation from
+    Bro scripts is integrated within the parsing process.
+
+    After completion, HTML documentation can be located inside the CMake
+    ``build/`` directory as ``build/doc/out/html``.  The generated reST
+    documentation will be located in ``build/doc/source/policy``.
+
+``make doc-clean``
+
+    This target removes Sphinx inputs and outputs from the CMake ``build/`` dir.
+
+To schedule a script to be documented, edit ``scripts/generate_reST_docs.py.in``
+and try adding the name of the script along with an optional script group to
+the ``docs`` dictionary.  That python script also shows other, more specialized
+methods for generating documentation for some types of corner-cases.
+
+When adding a new logical grouping for generated scripts, create a new
+reST document in ``source/policy/<group_name>.rst`` and add some default
+documentation.  References to (and summaries of) documents associated with
+the group get appended to this file during the ``make doc`` process.
+
+The Sphinx source tree template in ``source/`` can be modified to add more
+common/general documentation, style sheets, JavaScript, etc.  The Sphinx
+config file is produced from ``conf.py.in``, so that can be edited to change
+various Sphinx options, like setting the default HTML rendering theme.
+There is also a custom Sphinx domain implemented in ``source/ext/bro.py``
+which adds some reST directives and roles that aid in generating useful
+index entries and cross-references.
+
+See ``example.bro`` for an example of how to document a Bro script such that
+``make doc`` will be able to produce reST/HTML documentation for it.

doc/conf.py.in

@@ -0,0 +1,215 @@
+# -*- coding: utf-8 -*-
+#
+# Bro documentation build configuration file, created by sphinx-quickstart
+#
+# This file is execfile()d with the current directory set to its containing dir.
+#
+# Note that not all possible configuration values are present in this
+# autogenerated file.
+#
+# All configuration values have a default; values that are commented out
+# serve to show the default.
+
+import sys, os
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+sys.path.insert(0, os.path.abspath('source/ext'))
+
+# -- General configuration -----------------------------------------------------
+
+# If your documentation needs a minimal Sphinx version, state it here.
+#needs_sphinx = '1.0'
+
+# Add any Sphinx extension module names here, as strings. They can be extensions
+# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
+extensions = ['bro']
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['source/_templates']
+
+# The suffix of source filenames.
+source_suffix = '.rst'
+
+# The encoding of source files.
+#source_encoding = 'utf-8-sig'
+
+# The master toctree document.
+master_doc = 'index'
+
+# General information about the project.
+project = u'Bro'
+copyright = u'2011, Jon Siwek'
+
+# The version info for the project you're documenting, acts as replacement for
+# |version| and |release|, also used in various other places throughout the
+# built documents.
+#
+# The short X.Y version.
+version = '@VERSION_MAJ_MIN@'
+# The full version, including alpha/beta/rc tags.
+release = '@VERSION@'
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+#language = None
+
+# There are two options for replacing |today|: either, you set today to some
+# non-false value, then it is used:
+#today = ''
+# Else, today_fmt is used as the format for a strftime call.
+#today_fmt = '%B %d, %Y'
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+exclude_patterns = []
+
+# The reST default role (used for this markup: `text`) to use for all documents.
+#default_role = None
+
+# If true, '()' will be appended to :func: etc. cross-reference text.
+#add_function_parentheses = True
+
+# If true, the current module name will be prepended to all description
+# unit titles (such as .. function::).
+#add_module_names = True
+
+# If true, sectionauthor and moduleauthor directives will be shown in the
+# output. They are ignored by default.
+show_authors = True
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = 'sphinx'
+
+# A list of ignored prefixes for module index sorting.
+#modindex_common_prefix = []
+
+
+# -- Options for HTML output ---------------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+html_theme = 'default'
+
+# Theme options are theme-specific and customize the look and feel of a theme
+# further.  For a list of options available for each theme, see the
+# documentation.
+#html_theme_options = {}
+
+# Add any paths that contain custom themes here, relative to this directory.
+#html_theme_path = []
+
+# The name for this set of Sphinx documents.  If None, it defaults to
+# "<project> v<release> documentation".
+#html_title = None
+
+# A shorter title for the navigation bar.  Default is the same as html_title.
+#html_short_title = None
+
+# The name of an image file (relative to this directory) to place at the top
+# of the sidebar.
+#html_logo = None
+
+# The name of an image file (within the static path) to use as favicon of the
+# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
+# pixels large.
+#html_favicon = None
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['source/_static']
+
+# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
+# using the given strftime format.
+#html_last_updated_fmt = '%b %d, %Y'
+
+# If true, SmartyPants will be used to convert quotes and dashes to
+# typographically correct entities.
+#html_use_smartypants = True
+
+# Custom sidebar templates, maps document names to template names.
+#html_sidebars = {}
+
+# Additional templates that should be rendered to pages, maps page names to
+# template names.
+#html_additional_pages = {}
+
+# If false, no module index is generated.
+#html_domain_indices = True
+
+# If false, no index is generated.
+#html_use_index = True
+
+# If true, the index is split into individual pages for each letter.
+#html_split_index = False
+
+# If true, links to the reST sources are added to the pages.
+#html_show_sourcelink = True
+
+# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
+#html_show_sphinx = True
+
+# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
+#html_show_copyright = True
+
+# If true, an OpenSearch description file will be output, and all pages will
+# contain a <link> tag referring to it.  The value of this option must be the
+# base URL from which the finished HTML is served.
+#html_use_opensearch = ''
+
+# This is the file name suffix for HTML files (e.g. ".xhtml").
+#html_file_suffix = None
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = 'Brodoc'
+
+
+# -- Options for LaTeX output --------------------------------------------------
+
+# The paper size ('letter' or 'a4').
+#latex_paper_size = 'letter'
+
+# The font size ('10pt', '11pt' or '12pt').
+#latex_font_size = '10pt'
+
+# Grouping the document tree into LaTeX files. List of tuples
+# (source start file, target name, title, author, documentclass [howto/manual]).
+latex_documents = [
+  ('index', 'Bro.tex', u'Bro Documentation',
+   u'Jon Siwek', 'manual'),
+]
+
+# The name of an image file (relative to this directory) to place at the top of
+# the title page.
+#latex_logo = None
+
+# For "manual" documents, if this is true, then toplevel headings are parts,
+# not chapters.
+#latex_use_parts = False
+
+# If true, show page references after internal links.
+#latex_show_pagerefs = False
+
+# If true, show URL addresses after external links.
+#latex_show_urls = False
+
+# Additional stuff for the LaTeX preamble.
+#latex_preamble = ''
+
+# Documents to append as an appendix to all manuals.
+#latex_appendices = []
+
+# If false, no module index is generated.
+#latex_domain_indices = True
+
+
+# -- Options for manual page output --------------------------------------------
+
+# One entry per manual page. List of tuples
+# (source start file, name, description, authors, manual section).
+man_pages = [
+    ('index', 'bro', u'Bro Documentation',
+     [u'Jon Siwek'], 1)
+]

doc/example.bro

@@ -0,0 +1,178 @@
+##! This is an example script that demonstrates how to document.  Comments
+##! of the form ``##!`` are for the script summary.  The contents of
+##! these comments are transferred directly into the auto-generated
+##! `reStructuredText <http://docutils.sourceforge.net/rst.html>`_
+##! (reST) document's summary section.
+##!
+##! .. tip:: You can embed directives and roles within ``##``-stylized comments.
+##!
+##! :Author: Jon Siwek <jsiwek@ncsa.illinois.edu>
+
+# Comments that use a single pound sign (#) are not significant to
+# a script's auto-generated documentation, but ones that use a
+# double pound sign (##) do matter.  In some cases, like record
+# field comments, it's necessary to disambiguate the field with
+# which a comment associates: e.g. "##<" can be used on the same line
+# as a field to signify the comment relates to it and not the
+# following field.  "##<" is not meant for general use, just
+# record/enum fields.
+#
+# Generally, the auto-doc comments (##) are associated with the
+# next declaration/identifier found in the script, but the doc framework
+# will track/render identifiers regardless of whether they have any
+# of these special comments associated with them.
+#
+# The first sentence contained within the "##"-stylized comments for
+# a given identifier is special in that it will be used as summary
+# text in a table containing all such identifiers and short summaries.
+# If there are no sentences (text terminated with '.'), then everything
+# in the "##"-stylized comments up until the first empty comment
+# is taken as the summary text for a given identifier.
+
+# @load directives are self-documenting
+@load notice
+
+# "module" statements are self-documenting
+module Example;
+
+# redefinitions of "capture_filters" are self-documenting and
+# go into the generated documentation's "Packet Filter" section
+redef capture_filters += {
+    ["ssl"] = "tcp port 443",
+    ["nntps"] = "tcp port 562",
+};
+
+global example_ports = {
+    443/tcp, 562/tcp,
+} &redef;
+
+# redefinitions of "dpd_config" are self-documenting and
+# go into the generated doc's "Port Analysis" section
+redef dpd_config += {
+    [ANALYZER_SSL] = [$ports = example_ports]
+};
+
+# redefinitions of "Notice::Type" are self-documenting, but
+# more information can be supplied in two different ways
+redef enum Notice += {
+    ## any number of this type of comment
+    ## will document "Notice_One"
+    Notice_One,
+    Notice_Two,  ##< any number of this type of comment
+                 ##< will document "Notice_Two"
+    Notice_Three,
+    Notice_Four,
+};
+
+# Anything declared in the export section will show up in the rendered
+# documentation's "public interface" section
+
+export {
+
+    # these headings don't mean anything special to the
+    # doc framework right now, I'm just including them
+    # to make it more clear to the reader how the doc
+    # framework will actually categorize a script's identifiers
+
+    ############## types ################
+
+    # Note that I'm just mixing the "##" and "##<"
+    # types of comments in the following declarations
+    # as a demonstration.  Normally, it would be good style
+    # to pick one and be consistent.
+
+    ## documentation for "SimpleEnum"
+    ## goes here.
+    type SimpleEnum: enum {
+        ## and more specific info for "ONE"
+        ## can span multiple lines
+        ONE,
+        TWO,  ##< or more info like this for "TWO"
+              ##< can span multiple lines
+        THREE,
+    };
+
+    ## document the "SimpleEnum" redef here
+    redef enum SimpleEnum  += {
+        FOUR, ##< and some documentation for "FOUR"
+        ## also "FIVE" for good measure
+        FIVE
+    };
+
+    ## general documentation for a type "SimpleRecord"
+    ## goes here.
+    type SimpleRecord: record {
+        ## counts something
+        field1: count;
+        field2: bool; ##< toggles something
+    };
+
+
+    ## general documentation for a type "ComplexRecord" goes here
+    type ComplexRecord: record {
+        field1: count;               ##< counts something
+        field2: bool;                ##< toggles something
+        field3: SimpleRecord;
+        msg: string &default="blah"; ##< attributes are self-documenting
+    } &redef;
+
+    ############## options ################
+    # right now, I'm just defining an option as
+    # any const with &redef (something that can
+    # change at parse time, but not at run time.
+
+    ## add documentation for "an_option" here
+    const an_option: set[addr, addr, string] &redef;
+
+    # default initialization will be self-documenting
+    const option_with_init = 0.01 secs &redef;
+
+    ############## state variables ############
+    # right now, I'm defining this as any global
+    # that's not a function/event.  doesn't matter
+    # if &redef attribute is present
+
+    ## put some documentation for "a_var" here
+    global a_var: bool;
+
+    # attributes are self-documenting
+    global var_with_attr: count &persistent;
+
+    # it's fine if the type is inferred, that information is self-documenting
+    global var_without_explicit_type = "this works";
+
+    ############## functions/events ############
+
+    ## Summarize purpose of "a_function" here.
+    ## Give more details about "a_function" here.
+    ## Separating the documentation of the params/return values with
+    ## empty comments is optional, but improves readability of script.
+    ##
+    ## tag: function arguments can be described
+    ##      like this
+    ## msg: another param
+    ##
+    ## Returns: describe the return type here
+    global a_function: function(tag: string, msg: string): string;
+
+    ## Summarize "an_event" here.
+    ## Give more details about "an_event" here.
+    ## name: describe the argument here
+    global an_event: event(name: string);
+}
+
+# this function is documented in the "private interface" section
+# of generated documentation and any "##"-stylized comments would also
+# be rendered there
+function function_without_proto(tag: string): string
+    {
+    return "blah";
+    }
+
+# this record type is documented in the "private interface" section
+# of generated documentation and any "##"-stylized comments would also
+# be rendered there
+type PrivateRecord: record {
+    field1: bool;
+    field2: count;
+};

doc/scripts/BroToReST.py.in

@@ -0,0 +1,152 @@
+#! /usr/bin/env python
+
+import os
+import subprocess
+import shutil
+import glob
+import string
+import sys
+
+BRO = "@CMAKE_BINARY_DIR@/src/bro"
+BROPATHDEV = "`@CMAKE_BINARY_DIR@/bro-path-dev`"
+BRO_ARGS = "--doc-scripts"
+DOC_DST_DIR = "@DOC_SOURCE_WORKDIR@/policy"
+BROPATH = subprocess.Popen("@CMAKE_BINARY_DIR@/bro-path-dev", shell=True,
+        stdout=subprocess.PIPE, stderr=subprocess.STDOUT).stdout.readline()
+
+class BroToReST:
+    """A class to encapsulate the the generation of reST documentation from
+    a given Bro script.
+    """
+
+    bro_src_file = None
+    doc_src_file = None
+    load_via_stdin = False
+    group = None
+
+    def __init__(self, src_file, load_method=False, search_dir=None, group=None):
+        """
+        :param src_file: the file name of a Bro script (not a path)
+        :param load_method: T if script must be loaded by Bro via a stdin
+            redirection of "@load <script>", F if script can be loaded as
+            a command line argument to Bro
+        :param search_dir: a list of directories in which to search for
+            src_file.  If None, the default BROPATH is used.
+        :param group: a string representing a logical group that the script's
+            documentation should belong to.  A corresponding <group>.rst
+            document must be pre-existing in the policy/ dir of the source tree
+            used by Sphinx.
+        """
+
+        self.bro_src_file = FindBroScript(src_file, search_dir)
+        self.load_via_stdin = load_method
+        self.group = group
+
+        # formulate doc_src_file from src_file
+        filename = os.path.basename(src_file)
+        basename, ext = os.path.splitext(filename)
+        if ext == ".bro":
+            self.doc_src_file = basename + ".rst"
+        else:
+            self.doc_src_file = filename + ".rst"
+
+    def __str__(self):
+        return "bro_src_file: " + self.bro_src_file \
+           + "\ndoc_src_file: " + self.doc_src_file \
+           + "\ndoc_dst_file: " + os.path.join(DOC_DST_DIR, self.doc_src_file) \
+           + "\nstdin_load: %s" % self.load_via_stdin \
+           + "\ngroup: %s" % self.group
+
+    def GenDoc(self):
+        """Generates the reST documentation for a Bro script and copies
+        both the documentation and original script into Sphinx source tree.
+        If the documentation belongs to a group, the necessary modifications
+        to add it to the group's documentation are done.  Afterwards, any
+        files with a ".rst" suffix are removed for the working directory.
+        """
+
+        bro_src_basename = os.path.basename(self.bro_src_file)
+
+        if self.load_via_stdin:
+            cmd = "echo '@load %s' | %s %s" % (bro_src_basename, BRO, BRO_ARGS)
+        else:
+            cmd = "%s %s %s" % (BRO, BRO_ARGS, self.bro_src_file)
+
+        p = subprocess.Popen(cmd, shell=True, env={"BROPATH": BROPATH})
+
+        if p.wait() == 0:
+            shutil.copy(self.doc_src_file, DOC_DST_DIR)
+            shutil.copy(self.bro_src_file, DOC_DST_DIR)
+            AppendToDocGroup(self.group, self.bro_src_file, self.doc_src_file)
+
+        for leftover in glob.glob("*.rst"):
+            os.remove(leftover)
+
+def GenDocs(doc_dict, load_method=False):
+    """Generates reST documentation for all scripts in the given dictionary.
+
+    :param doc_dict: a dictionary whose keys are file names of Bro scripts
+        (not paths), and whose value is the logical documentation group
+        it belongs to
+    :param load_method: T if script must be loaded by Bro via a stdin
+            redirection of "@load <script>", F if script can be loaded as
+            a command line argument to Bro
+    """
+
+    for k, v in doc_dict.iteritems():
+        doc = BroToReST(k, load_method, group=v)
+        print "Generating reST document for " + k
+        doc.GenDoc()
+
+def FindBroScript(src_file, search_dir=None):
+    """Search a set of paths for a given Bro script and return the absolute
+    path to it.
+
+    :param src_file: the file name of a Bro script (not a path)
+    :param search_dir: a list of directories in which to search for
+        src_file.  If None, the default BROPATH is used.
+    """
+    if search_dir is None:
+        search_dir = string.split(BROPATH, ":")
+    for path in search_dir:
+        abs_path = os.path.join(path, src_file)
+        if os.path.exists(abs_path):
+            return abs_path
+    print >> sys.stderr, "Couldn't find '%s'" % src_file
+    return None
+
+def AppendToDocGroup(group, src_file, doc_file):
+    """Adds a reference to the given documentation for a Bro script
+    to the documentation file for it's associated group.  Also, associated
+    summary text (comments marked up like "##!" in the original Bro script
+    source) are added.
+
+    :param group: a string representing a logical group that the script's
+        documentation should belong to.  A corresponding <group>.rst
+        document must be pre-existing in the policy/ dir of the source tree
+        used by Sphinx.
+    :param src_file: a path to the original Bro script source file
+    :param doc_file: the file name of a script's generated reST document
+    """
+    if group is None:
+        return
+
+    group_file = os.path.join(DOC_DST_DIR, group + ".rst")
+    if not os.path.exists(group_file):
+        print >> sys.stderr, "Group file doesn't exist: " + group_file
+        return
+
+    summary_comments = []
+
+    with open(src_file, 'r') as f:
+        for line in f:
+            sum_pos = string.find(line, "##!")
+            if sum_pos != -1:
+                summary_comments.append(line[(sum_pos+3):])
+
+    doc_name, ext = os.path.splitext(doc_file)
+
+    with open(group_file, 'a') as f:
+        f.write("\n:doc:`%s`\n" % doc_name)
+        for line in summary_comments:
+            f.write(line)

doc/scripts/generate_reST_docs.py.in

@@ -0,0 +1,71 @@
+#! /usr/bin/env python
+
+import os
+import subprocess
+import shutil
+import glob
+import string
+import sys
+from BroToReST import *
+
+# TODO: generate docs for more scripts
+# TODO: the groups are just made up to test the functionality, fix them
+
+# Scripts that can be loaded by bro via command line argument:
+docs = {
+    "alarm.bro":        "internal",
+    "arp.bro":          "user",
+    "conn.bro":         "internal",
+    "dhcp.bro":         "user",
+    "dns.bro":          "user",
+    "ftp.bro":          "user",
+    "http.bro":         "user",
+    "http-reply.bro":   None,
+    "http-request.bro": None,
+    "irc.bro":          "user",
+    "smtp.bro":         "user",
+    "ssl.bro":          "user",
+    "ssl-ciphers.bro":  None,
+    "ssl-errors.bro":   None,
+    "synflood.bro":     "user",
+    "tcp.bro":          "user",
+    "udp.bro":          "user",
+    "weird.bro":        "internal",
+}
+
+# Scripts that can't be loaded by bro via command line argument (possible
+# due to dependency issues), but can be loaded via an @load on stdin:
+stdin_docs = {
+    "notice.bro": "internal",
+}
+
+GenDocs(docs)
+GenDocs(stdin_docs, True)
+
+# The example documentation script doesn't live on the BROPATH, so
+# explicitly generate the docs for it like this:
+BroToReST("example.bro", False, ["@PROJECT_SOURCE_DIR@/doc"], group="internal").GenDoc()
+
+# Generate documentation for stuff that's always loaded into bro by default:
+cmd = "echo '' | %s %s" % (BRO, BRO_ARGS)
+p = subprocess.Popen(cmd, shell=True, env={"BROPATH": BROPATH})
+if p.wait() == 0:
+    for doc in glob.glob("*.rst"):
+        if doc == "<stdin>.rst":
+            os.remove(doc)
+            continue
+
+        basename, ext = os.path.splitext(doc)
+        basename2, ext = os.path.splitext(basename)
+        if ext == ".init":
+            src_file = basename
+        else:
+            src_file = basename + ".bro"
+        src_file = FindBroScript(src_file)
+        shutil.copy(src_file, DOC_DST_DIR)
+        shutil.copy(doc, DOC_DST_DIR)
+        if ext == ".bif":
+            AppendToDocGroup("bifs", src_file, doc)
+        else:
+            AppendToDocGroup("default", src_file, doc)
+        os.remove(doc)

doc/source/_static/showhide.js

@@ -0,0 +1,64 @@
+// make literal blocks corresponding to identifier initial values
+// hidden by default
+$(document).ready(function() {
+
+    var showText='(Show Value)';
+    var hideText='(Hide Value)';
+
+    var is_visible = false;
+
+    // select field-list tables that come before a literal block
+    tables = $('.highlight-python').prev('table.docutils.field-list');
+
+    tables.find('th.field-name').filter(function(index) {
+        return $(this).html() == "Default :";
+    }).next().append('<a href="#" class="toggleLink">'+showText+'</a>');
+
+    // hide all literal blocks that follow a field-list table
+    tables.next('.highlight-python').hide();
+
+    // register handler for clicking a "toggle" link
+    $('a.toggleLink').click(function() {
+        is_visible = !is_visible;
+
+        $(this).html( (!is_visible) ? showText : hideText);
+
+        // the link is inside a <table><tbody><tr><td> and the next
+        // literal block after the table is the literal block that we want
+        // to show/hide
+        $(this).parent().parent().parent().parent().next('.highlight-python').slideToggle('fast');
+
+        // override default link behavior
+        return false;
+    });
+});
+
+// make "Private Interface" sections hidden by default
+$(document).ready(function() {
+
+    var showText='Show Private Interface (for internal use)';
+    var hideText='Hide Private Interface';
+
+    var is_visible = false;
+
+    // insert show/hide links
+    $('#private-interface').children(":first-child").after('<a href="#" class="privateToggle">'+showText+'</a>');
+
+    // wrap all sub-sections in a new div that can be hidden/shown
+    $('#private-interface').children(".section").wrapAll('<div class="private" />');
+
+    // hide the given class
+    $('.private').hide();
+
+    // register handler for clicking a "toggle" link
+    $('a.privateToggle').click(function() {
+        is_visible = !is_visible;
+
+        $(this).html( (!is_visible) ? showText : hideText);
+
+        $('.private').slideToggle('fast');
+
+        // override default link behavior
+        return false;
+    });
+});

doc/source/_templates/layout.html

@@ -0,0 +1,5 @@
+{% extends "!layout.html" %}
+{% block extrahead %}
+    <script type="text/javascript" src="{{ pathto('_static/showhide.js', 1) }}"></script>
+    {{ super() }}
+{% endblock %}

doc/source/builtins.rst

@@ -0,0 +1,121 @@
+Builtin Types and Attributes
+============================
+
+Types
+-----
+
+The Bro scripting language supports the following built-in types.
+
+.. TODO: add documentation
+
+.. bro:type:: void
+
+.. bro:type:: bool
+
+.. bro:type:: int
+
+.. bro:type:: count
+
+.. bro:type:: counter
+
+.. bro:type:: double
+
+.. bro:type:: time
+
+.. bro:type:: interval
+
+.. bro:type:: string
+
+.. bro:type:: pattern
+
+.. bro:type:: enum
+
+.. bro:type:: timer
+
+.. bro:type:: port
+
+.. bro:type:: addr
+
+.. bro:type:: net
+
+.. bro:type:: subnet
+
+.. bro:type:: any
+
+.. bro:type:: table
+
+.. bro:type:: union
+
+.. bro:type:: record
+
+.. bro:type:: types
+
+.. bro:type:: func
+
+.. bro:type:: file
+
+.. bro:type:: vector
+
+.. TODO: below are kind of "special cases" that bro knows about?
+
+.. bro:type:: set
+
+.. bro:type:: function
+
+.. bro:type:: event
+
+.. TODO: Notice will get documented as part of notice.bro, which can eventually
+   be referenced here once that documentation is auto-generated.
+
+.. bro:type:: Notice
+
+Attributes
+----------
+
+The Bro scripting language supports the following built-in attributes.
+
+.. TODO: add documentation
+
+.. bro:attr:: &optional
+
+.. bro:attr:: &default
+
+.. bro:attr:: &redef
+
+.. bro:attr:: &rotate_interval
+
+.. bro:attr:: &rotate_size
+
+.. bro:attr:: &add_func
+
+.. bro:attr:: &delete_func
+
+.. bro:attr:: &expire_func
+
+.. bro:attr:: &read_expire
+
+.. bro:attr:: &write_expire
+
+.. bro:attr:: &create_expire
+
+.. bro:attr:: &persistent
+
+.. bro:attr:: &synchronized
+
+.. bro:attr:: &postprocessor
+
+.. bro:attr:: &encrypt
+
+.. bro:attr:: &match
+
+.. bro:attr:: &disable_print_hook
+
+.. bro:attr:: &raw_output
+
+.. bro:attr:: &mergeable
+
+.. bro:attr:: &priority
+
+.. bro:attr:: &group
+
+.. bro:attr:: (&tracked)

doc/source/common.rst

@@ -0,0 +1,19 @@
+Common Documentation
+====================
+
+.. _common_port_analysis_doc:
+
+Port Analysis
+-------------
+
+TODO: add some stuff here
+
+.. _common_packet_filter_doc:
+
+Packet Filter
+-------------
+
+TODO: add some stuff here
+
+.. note:: Filters are only relevant when dynamic protocol detection (DPD)
+          is explicitly turned off (Bro release 1.6 enabled DPD by default).

doc/source/ext/bro.py

@@ -0,0 +1,167 @@
+"""
+    The Bro domain for Sphinx.
+"""
+
+def setup(Sphinx):
+    Sphinx.add_domain(BroDomain)
+
+from sphinx import addnodes
+from sphinx.domains import Domain, ObjType, Index
+from sphinx.locale import l_, _
+from sphinx.directives import ObjectDescription
+from sphinx.roles import XRefRole
+from sphinx.util.nodes import make_refnode
+import string
+
+from docutils import nodes
+from docutils.parsers.rst import Directive
+from docutils.parsers.rst import directives
+from docutils.parsers.rst.roles import set_classes
+
+class BroGeneric(ObjectDescription):
+    def add_target_and_index(self, name, sig, signode):
+        targetname = self.objtype + '-' + name
+        if targetname not in self.state.document.ids:
+            signode['names'].append(targetname)
+            signode['ids'].append(targetname)
+            signode['first'] = (not self.names)
+            self.state.document.note_explicit_target(signode)
+
+            objects = self.env.domaindata['bro']['objects']
+            key = (self.objtype, name)
+# this is commented out mostly just to avoid having a special directive
+# for events in order to avoid the duplicate warnings in that case
+            """
+            if key in objects:
+                self.env.warn(self.env.docname,
+                              'duplicate description of %s %s, ' %
+                              (self.objtype, name) +
+                              'other instance in ' +
+                              self.env.doc2path(objects[key]),
+                              self.lineno)
+            """
+            objects[key] = self.env.docname
+        indextext = self.get_index_text(self.objtype, name)
+        if indextext:
+            self.indexnode['entries'].append(('single', indextext,
+                                              targetname, targetname))
+
+    def get_index_text(self, objectname, name):
+        return _('%s (%s)') % (name, self.objtype)
+
+    def handle_signature(self, sig, signode):
+        signode += addnodes.desc_name("", sig)
+        return sig
+
+class BroNamespace(BroGeneric):
+    def add_target_and_index(self, name, sig, signode):
+        targetname = self.objtype + '-' + name
+        if targetname not in self.state.document.ids:
+            signode['names'].append(targetname)
+            signode['ids'].append(targetname)
+            signode['first'] = (not self.names)
+            self.state.document.note_explicit_target(signode)
+
+            objects = self.env.domaindata['bro']['objects']
+            key = (self.objtype, name)
+            objects[key] = self.env.docname
+        indextext = self.get_index_text(self.objtype, name)
+        self.indexnode['entries'].append(('single', indextext,
+                                          targetname, targetname))
+        self.indexnode['entries'].append(('single',
+                                          "namespaces; %s" % (sig),
+                                          targetname, targetname))
+
+    def get_index_text(self, objectname, name):
+        return _('%s (namespace); %s') % (name, self.env.docname)
+
+    def handle_signature(self, sig, signode):
+        signode += addnodes.desc_name("", sig)
+        return sig
+
+class BroEnum(BroGeneric):
+    def add_target_and_index(self, name, sig, signode):
+        targetname = self.objtype + '-' + name
+        if targetname not in self.state.document.ids:
+            signode['names'].append(targetname)
+            signode['ids'].append(targetname)
+            signode['first'] = (not self.names)
+            self.state.document.note_explicit_target(signode)
+
+            objects = self.env.domaindata['bro']['objects']
+            key = (self.objtype, name)
+            objects[key] = self.env.docname
+        indextext = self.get_index_text(self.objtype, name)
+        #self.indexnode['entries'].append(('single', indextext,
+        #                                  targetname, targetname))
+        m = sig.split()
+        self.indexnode['entries'].append(('single',
+                                          "%s (enum values); %s" % (m[1], m[0]),
+                                          targetname, targetname))
+
+    def handle_signature(self, sig, signode):
+        m = sig.split()
+        name = m[0]
+        signode += addnodes.desc_name("", name)
+        return name
+
+class BroIdentifier(BroGeneric):
+    def get_index_text(self, objectname, name):
+        return name
+
+class BroAttribute(BroGeneric):
+    def get_index_text(self, objectname, name):
+        return _('%s (attribute)') % (name)
+
+class BroDomain(Domain):
+    """Bro domain."""
+    name = 'bro'
+    label = 'Bro'
+
+    object_types = {
+        'type':             ObjType(l_('type'),             'type'),
+        'namespace':        ObjType(l_('namespace'),        'namespace'),
+        'id':               ObjType(l_('id'),               'id'),
+        'enum':             ObjType(l_('enum'),             'enum'),
+        'attr':             ObjType(l_('attr'),             'attr'),
+    }
+
+    directives = {
+        'type':             BroGeneric,
+        'namespace':        BroNamespace,
+        'id':               BroIdentifier,
+        'enum':             BroEnum,
+        'attr':             BroAttribute,
+    }
+
+    roles = {
+        'type':             XRefRole(),
+        'namespace':        XRefRole(),
+        'id':               XRefRole(),
+        'enum':             XRefRole(),
+        'attr':             XRefRole(),
+    }
+
+    initial_data = {
+        'objects': {},  # fullname -> docname, objtype
+    }
+
+    def clear_doc(self, docname):
+        for (typ, name), doc in self.data['objects'].items():
+            if doc == docname:
+                del self.data['objects'][typ, name]
+
+    def resolve_xref(self, env, fromdocname, builder, typ, target, node,
+                     contnode):
+        objects = self.data['objects']
+        objtypes = self.objtypes_for_role(typ)
+        for objtype in objtypes:
+            if (objtype, target) in objects:
+                return make_refnode(builder, fromdocname,
+                                    objects[objtype, target],
+                                    objtype + '-' + target,
+                                    contnode, target + ' ' + objtype)
+
+    def get_objects(self):
+        for (typ, name), docname in self.data['objects'].iteritems():
+            yield name, name, typ, docname, typ + '-' + name, 1

doc/source/index.rst

@@ -0,0 +1,24 @@
+.. Bro documentation master file
+
+Welcome to Bro's documentation!
+===============================
+
+Contents:
+
+.. toctree::
+   :maxdepth: 1
+   :glob:
+
+   common
+   builtins
+   policy/default
+   policy/user
+   policy/bifs
+   policy/internal
+   policy/index
+
+Indices and tables
+==================
+
+* :ref:`genindex`
+* :ref:`search`

doc/source/policy/bifs.rst

@@ -0,0 +1,4 @@
+Built-In Functions (BIFs)
+=========================
+
+Here's a list of all documentation for BIFs that Bro provides:

doc/source/policy/default.rst

@@ -0,0 +1,3 @@
+Bro Scripts Loaded by Default
+=============================
+

doc/source/policy/index.rst

@@ -0,0 +1,10 @@
+Index of All Policy Script Documentation
+========================================
+
+Contents:
+
+.. toctree::
+   :maxdepth: 1
+   :glob:
+
+   *

doc/source/policy/internal.rst

@@ -0,0 +1,3 @@
+Internal Policy Scripts
+=======================
+

doc/source/policy/user.rst

@@ -0,0 +1,3 @@
+User-Facing Policy Scripts
+==========================
+

policy/all.bro

@@ -53,10 +53,8 @@
 @load http-identified-files.bro
 @load http-reply
 @load http-request
-@load http-rewriter
 @load http
 @load icmp
-@load ident-rewriter
 @load ident
 @load inactivity
 @load interconn
@@ -111,7 +109,6 @@
 @load site
 @load smb
 @load smtp-relay
-@load smtp-rewriter
 @load smtp
 @load snort
 @load software

policy/bro.init

@@ -22,7 +22,7 @@ type conn_id: record {
 	orig_p: port;
 	resp_h: addr;
 	resp_p: port;
-};
+} &log;
 
 type icmp_conn: record {
 	orig_h: addr;
@@ -278,6 +278,9 @@ type entropy_test_result: record {
 @load strings.bif.bro
 @load bro.bif.bro
 
+@load logging  # sic! Not logging.bif.
+@load logging-ascii
+
 global bro_alarm_file: file &redef;
 global alarm_hook: function(msg: string): bool &redef;
 global log_file_name: function(tag: string): string &redef;
@@ -1122,14 +1125,6 @@ type bt_tracker_headers: table[string] of string;
 
 @load event.bif.bro
 
-@load common-rw.bif.bro
-@load finger-rw.bif.bro
-@load ftp-rw.bif.bro
-@load ident-rw.bif.bro
-@load smtp-rw.bif.bro
-@load http-rw.bif.bro
-@load dns-rw.bif.bro
-
 function subst(s: string, from: pattern, to: string): string
 	{
 	local p = split_all(s, from);
@@ -1388,7 +1383,7 @@ const suppress_local_output = F &redef;
 
 # Holds the filename of the trace file given with -w (empty if none).
 const trace_output_file = "";
-  
+
 # If a trace file is given, dump *all* packets seen by Bro into it.
 # By default, Bro applies (very few) heuristics to reduce the volume.
 # A side effect of setting this to true is that we can write the
@@ -1409,34 +1404,3 @@ const skip_http_data = F &redef;
 # Whether the analysis engine parses IP packets encapsulated in
 # UDP tunnels. See also: udp_tunnel_port, policy/udp-tunnel.bro.
 const parse_udp_tunnels = F &redef;
-
-# Whether a commitment is required before writing the transformed
-# trace for a connection into the dump file.
-const requires_trace_commitment = F &redef;
-
-# Whether IP address anonymization is enabled.
-const anonymize_ip_addr = F &redef;
-
-# Whether to omit place holder packets when rewriting.
-const omit_rewrite_place_holder = T &redef;
-
-# Whether trace of various protocols is being rewritten.
-const rewriting_http_trace = F &redef;
-const rewriting_smtp_trace = F &redef;
-const rewriting_ftp_trace = F &redef;
-const rewriting_ident_trace = F &redef;
-const rewriting_finger_trace = F &redef;
-const rewriting_dns_trace = F &redef;
-const rewriting_smb_trace = F &redef;
-
-# Whether we dump selected original packets to the output trace.
-const dump_selected_source_packets = F &redef;
-
-# If true, we dump original packets to the output trace *if and only if*
-# the connection is not rewritten; if false, the policy script can decide
-# whether to dump a particular connection by calling dump_packets_of_connection.
-#
-# NOTE: DO NOT SET THIS TO TRUE WHEN ANONYMIZING A TRACE! 
-# (TODO: this variable should be disabled when using '-A' option)
-const dump_original_packets_if_not_rewriting = F &redef;
-

policy/conn.bro

@@ -14,7 +14,9 @@ const conn_closed = { TCP_CLOSED, TCP_RESET };
 
 global have_FTP = F;	# if true, we've loaded ftp.bro
 global have_SMTP = F;	# if true, we've loaded smtp.bro
-global is_ftp_data_conn: function(c: connection): bool;
+
+# TODO: Do we have a nicer way of defining this prototype?
+export { global FTP::is_ftp_data_conn: function(c: connection): bool; }
 
 # Whether to include connection state history in the logs generated
 # by record_connection.
@@ -186,7 +188,7 @@ function determine_service_non_DPD(c: connection) : string
 			return i;	# return first;
 		}
 
-	else if ( have_FTP && is_ftp_data_conn(c) )
+	else if ( have_FTP && FTP::is_ftp_data_conn(c) )
 		return port_names[20/tcp];
 
 	else if ( [c$id$resp_h, c$id$resp_p] in RPC_server_map )

policy/dns-anonymizer.bro

@@ -1,107 +0,0 @@
-# $Id:$
-
-@load dns
-@load anon
-
-module DNS;
-
-redef rewriting_dns_trace = T;
-
-event dns_message(c: connection, is_orig: bool, msg: dns_msg, len: count)
-	{
-	if ( get_conn_transport_proto(c$id) == udp )
-		rewrite_dns_message(c, is_orig, msg, len);
-	}
-
-event dns_request(c: connection, msg: dns_msg, query: string,
-			qtype: count, qclass: count)
-	{
-	if ( get_conn_transport_proto(c$id) == udp )
-		rewrite_dns_request(c, anonymize_host(query),
-					msg, qtype, qclass);
-	}
-
-event dns_end(c: connection, msg: dns_msg)
-	{
-	if ( get_conn_transport_proto(c$id) == udp )
-		rewrite_dns_end(c, T);
-	}
-
-event dns_query_reply(c: connection, msg: dns_msg, query: string,
-			qtype: count, qclass: count)
-	{
-	rewrite_dns_reply_question(c, msg, anonymize_host(query),
-					qtype, qclass);
-	}
-
-event dns_A_reply(c: connection, msg: dns_msg, ans: dns_answer, a: addr)
-	{
-	ans$query = anonymize_host(ans$query);
-	rewrite_dns_A_reply(c, msg, ans, anonymize_address(a, c$id));
-	}
-
-#### FIXME: ANONYMIZE!
-event dns_AAAA_reply(c: connection, msg: dns_msg, ans: dns_answer,
-			a: addr, astr: string)
-	{
-	ans$query = anonymize_host(ans$query);
-	astr = "::";
-	a = anonymize_address(a, c$id);
-	rewrite_dns_AAAA_reply(c, msg, ans, a, astr);
-	}
-
-event dns_NS_reply(c: connection, msg: dns_msg, ans: dns_answer, name: string)
-	{
-	ans$query = anonymize_host(ans$query);
-	rewrite_dns_NS_reply(c, msg, ans, anonymize_host(name));
-	}
-
-event dns_CNAME_reply(c: connection, msg: dns_msg, ans: dns_answer,
-			name: string)
-	{
-	ans$query = anonymize_host(ans$query);
-	rewrite_dns_CNAME_reply(c, msg, ans, anonymize_host(name));
-	}
-
-event dns_MX_reply(c: connection, msg: dns_msg, ans: dns_answer,
-			name: string, preference: count)
-	{
-	ans$query = anonymize_host(ans$query);
-	rewrite_dns_MX_reply(c, msg, ans, anonymize_host(name), preference);
-	}
-
-event dns_PTR_reply(c: connection, msg: dns_msg, ans: dns_answer, name: string)
-	{
-	ans$query = anonymize_host(ans$query);
-	rewrite_dns_PTR_reply(c, msg, ans, anonymize_host(name));
-	}
-
-event dns_SOA_reply(c: connection, msg: dns_msg, ans: dns_answer, soa: dns_soa)
-	{
-	soa$mname = anonymize_host(soa$mname);
-	soa$rname = anonymize_host(soa$rname);
-	ans$query = anonymize_host(ans$query);
-	rewrite_dns_SOA_reply(c, msg, ans, soa);
-	}
-
-event dns_TXT_reply(c: connection, msg: dns_msg, ans: dns_answer,
-			str: string)
-	{
-	str = anonymize_string(str);
-	ans$query = anonymize_host(ans$query);
-	rewrite_dns_TXT_reply(c, msg, ans, str);
-	}
-
-event dns_EDNS_addl (c: connection, msg: dns_msg, ans: dns_edns_additional)
-	{
-	rewrite_dns_EDNS_addl(c, msg, ans);
-	}
-
-event dns_rejected(c: connection, msg: dns_msg, query: string,
-			qtype: count, qclass: count)
-	{
-	#### Hmmm, this is probably not right - we are going to have to look
-	# at the question type to determine how to anonymize this.
-	rewrite_dns_reply_question(c, msg, anonymize_host(query),
-					qtype, qclass);
-	}

policy/finger.bro

@@ -60,19 +60,6 @@ event finger_request(c: connection, full: bool, username: string, hostname: stri
 		req = fmt("(%s)", req);
 
 	append_addl_marker(c, req, " *");
-
-	if ( rewriting_finger_trace )
-		rewrite_finger_request(c, full,
-				public_user(username) ? username : "private user",
-				hostname);
-	}
-
-event finger_reply(c: connection, reply_line: string)
-	{
-	local id = c$id;
-	if ( rewriting_finger_trace )
-		rewrite_finger_reply(c,
-				authorized_client(id$orig_h) ? "finger reply ..." : reply_line);
 	}
 
 function is_finger_conn(c: connection): bool
@@ -80,10 +67,3 @@ function is_finger_conn(c: connection): bool
 	return c$id$resp_p == finger;
 	}
 
-event connection_state_remove(c: connection)
-	{
-	if ( rewriting_finger_trace && requires_trace_commitment &&
-	     is_finger_conn(c) )
-		# Commit queued packets and all packets in future.
-		rewrite_commit_trace(c, T, T);
-	}

policy/firewall.bro

@@ -151,7 +151,7 @@ function do_match(c: connection, r: rule): bool
 				return F;
 		}
 
-	if ( r$is_ftp && ! is_ftp_data_conn(c) )
+	if ( r$is_ftp && ! FTP::is_ftp_data_conn(c) )
 		return F;
 
 	return T;

policy/ftp-anonymizer.bro

@@ -1,846 +0,0 @@
-# $Id: ftp-anonymizer.bro 47 2004-06-11 07:26:32Z vern $
-
-@load ftp
-@load anon
-
-# Definitions of constants.
-
-# Check if those commands carry any argument; anonymize non-empty
-# argument.
-const ftp_cmds_with_no_arg = {
-	"CDUP", "QUIT", "REIN", "PASV", "STOU",
-	"ABOR", "PWD", "SYST", "NOOP",
-
-	"FEAT",	"XPWD",
-};
-
-const ftp_cmds_with_file_arg = {
-	"APPE", "CWD", "DELE", "LIST", "MKD",
-	"NLST", "RMD", "RNFR", "RNTO", "RETR",
-	"STAT", "STOR", "SMNT",
-	# FTP extensions
-	"SIZE", "MDTM",
-	"MLSD", "MLST",
-	"XCWD",
-};
-
-# For following commands, we check if the argument conforms to the
-# specification -- if so, it is safe to be left in the clear.
-const ftp_cmds_with_safe_arg = {
-	"TYPE", "STRU", "MODE", "ALLO", "REST",
-	"HELP",
-
-	"MACB",	# MacBinary encoding
-};
-
-# ftp_other_cmds can be redefined in site/trace-specific ways.
-const ftp_other_cmds = {
-	"LPRT", "OPTS", "CLNT", "RETP",
-	"EPSV", "XPWD",
-	"SOCK",	# old FTP command (RFC 354)
-} &redef;
-
-# Below defines patterns of arguments of FTP commands
-
-# The following patterns are case-insensitive
-const ftp_safe_cmd_arg_pattern =
-	  /TYPE (([AE]( [NTC])?)|I|(L [0-9]+))/
-	| /STRU [FRP]/
-	| /MODE [SBC]/
-	| /ALLO [0-9]+([ \t]+R[ \t]+[0-9]+)?/
-	| /REST [!-~]+/
-	| /MACB (E|DISABLE|ENABLE)/
-	| /SITE TRUTH ON/
-	&redef;
-
-# The following list includes privacy-safe [cmd, arg] pairs and can be
-# customized for particular traces
-const ftp_safe_arg_list: set[string, string] = {
-} &redef;
-
-# ftp_special_cmd_args offers an even more flexible way of customizing
-# argument anonymization: for each [cmd, arg] pair in the table, the
-# corresponding value will be the anonymized argument.
-const ftp_special_cmd_args: table[string, string] of string = {
-} &redef;
-
-# The following words are safe to be left in the clear as the argument
-# of a HELP command.
-const ftp_help_words = {
-	"USER",	"PORT",	"STOR",	"MSAM",	"RNTO",	"NLST",	"MKD",	"CDUP",
-	"PASS",	"PASV",	"APPE",	"MRSQ",	"ABOR",	"SITE",	"XMKD",	"XCUP",
-	"ACCT",	"TYPE",	"MLFL",	"MRCP",	"DELE",	"SYST",	"RMD",	"STOU",
-	"SMNT",	"STRU",	"MAIL",	"ALLO",	"CWD",	"STAT",	"XRMD",	"SIZE",
-	"REIN",	"MODE",	"MSND",	"REST",	"XCWD",	"HELP",	"PWD",	"MDTM",
-	"QUIT",	"RETR",	"MSOM",	"RNFR",	"LIST",	"NOOP",	"XPWD",
-} &redef;
-
-const ftp_port_pat = /[0-9]+([[:blank:]]*,[[:blank:]]*[0-9]+){5}/;
-
-# Pattern for the argument of EPRT command.
-# TODO: the pattern works fot the common case but is not RFC2428-complete.
-const ftp_eprt_pat = /\|1\|[0-9]{1,3}(\.[0-9]{1,3}){3}\|[0-9]{1,5}\|/;
-
-# IP addresses.
-const ftp_ip_pat = /[0-9]{1,3}(\.[0-9]{1,3}){3}/;
-
-# Domain names (deficiency: domain suffices of countries).
-const ftp_domain_name_pat =
-	/([\-0-9a-zA-Z]+\.)+(com|edu|net|org|gov|mil|uk|fr|nl|es|jp|it)/;
-
-# File names (printable characters).
-const ftp_file_name_pat = /[[:print:]]+/;
-
-# File names that can be left in the clear.
-const ftp_public_files =
-	  /\// | /\.\./		# "/" and ".."
-	| /(\/etc\/|master\.)?(passwd|shadow|s?pwd\.db)/ # ftp_hot_files
-	| /\/(etc|usr\/bin|bin|sbin|kernel)(\/)?/
-	| /\.rhosts/ | /\.forward/			 # ftp_hot_guest_files
-&redef;
-
-const ftp_sensitive_files =
-	  /.*(etc\/|master\.)?(passwd|shadow|s?pwd\.db)/ # ftp_hot_files
-	| /\/(etc|usr\/bin|bin|sbin|kernel)\/.*/
-	| /.*\.rhosts/ | /.*\.forward/			 # ftp_hot_guest_files
-&redef;
-
-# Public servers.
-const ftp_public_servers: set[addr] = {} &redef;
-
-# Whether we keep all file names (valid or invalid) for public servers.
-const ftp_keep_all_files_for_public_servers = F &redef;
-
-# Public files.
-const ftp_known_public_files: set[addr, string] = {} &redef;
-
-# Hidden file/directory.
-const ftp_hidden_file = /.*\/\.[^.\/].*/;
-const ftp_public_hidden_file = /0/ &redef;
-
-# Options for file commands (LIST, NLST) that can be left in the clear.
-const ftp_known_option = /-[[:alpha:]]{1,5}[ ]*/;
-
-const ftp_known_site_cmd = {
-	"UMASK", "GROUP", "INDEX", "GROUPS",
-	"IDLE", "GPASS", "EXEC", "CHECKMETHOD",
-	"CHMOD", "NEWER", "ALIAS", "CHECKSUM",
-	"HELP", "MINFO", "CDPATH",
-
-	"TRUTH", "UTIME",
-} &redef;
-
-const ftp_sensitive_ids: set[string] = {
-	"backdoor", "bomb", "diag", "gdm", "issadmin", "msql", "netfrack",
-	"netphrack", "own", "r00t", "root", "ruut", "smtp", "sundiag", "sync",
-	"sys", "sysadm", "sysdiag", "sysop", "sysoper", "system", "toor", "tour",
-	"y0uar3ownd",
-};
-
-redef anonymize_ip_addr = T;
-redef rewriting_ftp_trace = T;
-
-global ftp_anon_log = open_log_file("ftp-anon") &redef;
-
-# Anonymized arguments, indexed by the anonymization seed.
-global anonymized_args: table[string] of string;
-
-# Arguments left in the clear, indexed by the argument and the context.
-global ftp_arg_left_in_the_clear: set[string, string];
-
-# Valid files on public servers.
-global ftp_valid_public_files: set[addr, string];
-
-type ftp_cmd_arg_anon_result: record {
-	anonymized: bool;
-	cmd: string;
-	arg: string;
-};
-
-
-# Whether anonymize_trace_specific_cmd_arg is defined:
-const trace_specific_cmd_arg_anonymization = F &redef;
-
-# This function is to be defined in a trace-specific script. By
-# default, use ftp-anonymizer-trace.bro.
-
-global anonymize_trace_specific_cmd_arg:
-	function(session: ftp_session_info, cmd: string, arg: string):
-		ftp_cmd_arg_anon_result;
-
-
-# Anonymize FTP replies by message patterns.
-const process_ftp_reply_by_message_pattern = F &redef;
-global anonymize_ftp_reply_by_msg_pattern:
-	function(code: count, act_msg: string,
-		cmd_arg: ftp_cmd_arg, session: ftp_session_info): string;
-
-
-# Anonymize an argument *completely* with a hash value of the string,
-# and log the anonymization.
-function anonymize_arg(typ: string, session: ftp_session_info, cmd: string, arg: string, seed: string): string
-	{
-	if ( arg == "" )
-		return ""; # an empty argument is safe
-
-	local arg_seed = string_cat(typ, seed, arg);
-
-	if ( arg_seed in anonymized_args )
-		return anonymized_args[arg_seed];
-
-	local a = anonymize_string(arg_seed);
-	anonymized_args[arg_seed] = a;
-
-	print ftp_anon_log,
-		fmt("anonymize_arg: (%s) {%s} %s \"%s\" to \"%s\" in [%s]",
-				typ, seed, cmd,
-				to_string_literal(arg), to_string_literal(a),
-				id_string(session$connection_id));
-	return a;
-	}
-
-# This function is called whenever an argument is to be left in the
-# clear. It logs the action if it hasn't occurred before.
-function leave_in_the_clear(msg: string, session: ftp_session_info,
-				arg: string, context: string): string
-	{
-	if ( [arg, context] !in ftp_arg_left_in_the_clear )
-		{
-		add ftp_arg_left_in_the_clear[arg, context];
-		print ftp_anon_log, fmt("leave_in_the_clear: (%s) \"%s\" [%s] in [%s]",
-				msg, to_string_literal(arg), context,
-				id_string(session$connection_id));
-		}
-	return arg;
-	}
-
-
-# Sometimes the argument of a file command contains an option string
-# before the file name, such as in 'LIST -l /xyz/', the following
-# function identifies such option strings and separate the argument
-# accordingly.
-
-type separate_option_str_result: record {
-	opt_str: string;
-	file_name: string;
-};
-
-function separate_option_str(file_name: string): separate_option_str_result
-	{
-	local ret: separate_option_str_result;
-	if ( file_name == /-[[:alpha:]]+( .*)?/ )
-		{
-		local parts = split_all(file_name, /-[[:alpha:]]+[ ]*/);
-		ret$opt_str = string_cat(parts[1], parts[2]);
-		parts[1] = ""; parts[2] = "";
-		ret$file_name = cat_string_array(parts);
-		return ret;
-		}
-	else
-		return [$opt_str = "", $file_name = file_name];
-	}
-
-
-# Anonymize a user id
-type login_status_type: enum {
-	LOGIN_PENDING,
-	LOGIN_SUCCESSFUL,
-	LOGIN_FAILED,
-	LOGIN_UNKNOWN,
-};
-
-function anonymize_user_id(session: ftp_session_info, id: string, login_status: login_status_type, msg: string): string
-	{
-	if ( id in ftp_guest_ids )
-		{
-		leave_in_the_clear("guest_id", session, id, msg);
-		return id;
-		}
-
-	else if ( id in ftp_sensitive_ids && login_status == LOGIN_FAILED )
-		{
-		leave_in_the_clear("sensitive_id", session, id, msg);
-		return id;
-		}
-
-	else
-		return anonymize_arg("user_name", session, "USER", id, cat(session$connection_id$resp_h, login_status));
-	}
-
-# Anonymize a file name argument.
-function anonymize_file_name_arg(session: ftp_session_info, cmd: string, arg: string, valid_file_name: bool): string
-	{
-	local file_name = arg;
-	local opt_str = "";
-	if ( cmd == /LIST|NLST/ )
-		{
-		# Separate the option from file name if there is one
-
-		local ret = separate_option_str(file_name);
-		if ( ret$opt_str != "" )
-			{
-			opt_str = ret$opt_str;
-
-			# Shall we anonymize the option string?
-			if ( opt_str != ftp_known_option )
-				{
-				# Anonymize the option conservatively
-				print ftp_anon_log, fmt("option_anonymized: \"%s\" from (%s %s)",
-					to_string_literal(opt_str), cmd, file_name);
-				opt_str = "-<option>";
-				}
-			else
-				# Leave in the clear
-				print ftp_anon_log, fmt("option_left_in_the_clear: \"%s\" from (%s %s)",
-					to_string_literal(opt_str), cmd, file_name);
-
-			file_name = ret$file_name;
-			}
-		}
-
-	if ( file_name == "" )
-		return opt_str;
-
-	if ( file_name != ftp_file_name_pat )
-		{
-		# Log special file names (e.g. those containing
-		# control characters) for manual inspection -- such
-		# file names are rare and may present problems in
-		# reply anonymization.
-
-		print ftp_anon_log, fmt("unrecognized_file_name: \"%s\" (%s) [%s]",
-			to_string_literal(file_name), cmd, id_string(session$connection_id));
-		}
-
-
-	if ( strstr(file_name, " ") > 0 )
-		{
-		# Log file names that contain spaces (for debugging only)
-
-		print ftp_anon_log, fmt("space_in_file_name: \"%s\" (%s) [%s]",
-			to_string_literal(file_name), cmd, id_string(session$connection_id));
-		}
-
-	# Compute the absolute and clean (without '..' and duplicate
-	# '/') path
-	local abs_path = absolute_path(session, file_name);
-	local resp_h = session$connection_id$resp_h;
-	local known_public_file =
-		[resp_h, abs_path] in ftp_known_public_files ||
-		[resp_h, abs_path] in ftp_valid_public_files;
-
-	if ( file_name == ftp_public_files || abs_path == ftp_public_files )
-		{
-		leave_in_the_clear("public_path_name", session,
-			arg, fmt("(%s %s) %s", cmd, file_name, abs_path));
-		}
-
-	else if ( resp_h in ftp_public_servers &&
-		  (abs_path != ftp_hidden_file ||
-		   abs_path == ftp_public_hidden_file) &&
-		  (ftp_keep_all_files_for_public_servers || valid_file_name ||
-		   known_public_file) )
-		{
-		if ( valid_file_name && ! known_public_file )
-			{
-			add ftp_valid_public_files[resp_h, abs_path];
-			print ftp_anon_log,
-				fmt("valid_public_file: [%s, \"%s\"]",
-					resp_h, to_string_literal(abs_path));
-			}
-
-		leave_in_the_clear("file_on_public_server", session, arg,
-			fmt("%s %s:%s", cmd, session$connection_id$resp_h, abs_path));
-		}
-	else
-		{
-		local anon_type: string;
-
-		if ( file_name == ftp_sensitive_files ||
-		     abs_path == ftp_sensitive_files )
-			anon_type = "sensitive_path_name";
-
-		else if ( abs_path == ftp_hidden_file )
-			anon_type = "hidden_path_name";
-
-		else if ( resp_h in ftp_public_servers )
-			anon_type = "invalid_public_file";
-
-		else
-			anon_type = "path_name";
-
-		file_name = anonymize_arg(anon_type, session, cmd, abs_path, cat("file:", session$connection_id$resp_h));
-		}
-
-	# concatenate the option string with the file_name
-	return string_cat(opt_str, file_name);
-	}
-
-
-# The argument is presumably privacy-safe, but we should not assume
-# that it is the case. Instead, we check if the argument is legal and
-# thus privacy-free.
-function check_safe_arg(session: ftp_session_info, cmd: string,
-			arg: string): string
-	{
-	if ( cmd == "HELP" )
-		return ( arg in ftp_help_words ) ?
-				leave_in_the_clear("known_help_word", session,
-					arg,
-					fmt("%s %s", cmd, arg))
-				: anonymize_arg("unknown_help_string", session, cmd, arg, cmd);
-
-	else
-		# Note that we have already checked the (cmd, arg)
-		# against ftp_safe_cmd_arg_pattern. So the argument
-		# here must have an unrecognized pattern and should be
-		# anonymized.
-		return anonymize_arg("illegal_argument", session, cmd, arg, cmd);
-	}
-
-function check_site_arg(session: ftp_session_info, cmd: string,
-			arg: string): string
-	{
-	local site_cmd_arg = split1(arg, /[ \t]*/);
-
-	local site_cmd = site_cmd_arg[1];
-	local site_cmd_upper = to_upper(site_cmd);
-
-	if ( site_cmd_upper in ftp_known_site_cmd )
-		{
-		if ( length(site_cmd_arg) > 1 )
-			{
-			# If the there is an argument after "SITE <cmd>"
-			local site_arg = site_cmd_arg[2];
-			site_arg =
-				anonymize_arg(fmt("arg_of_site_%s", site_cmd),
-						session, cmd, site_arg, cmd);
-			return string_cat(site_cmd, " ", site_arg);
-			}
-		else
-			{
-			return leave_in_the_clear("known_site_command", session,
-						site_cmd,
-						fmt("%s %s", cmd, arg));
-			}
-		}
-	else
-		return anonymize_arg("site_arg", session, cmd, arg, cmd);
-	}
-
-function anonymize_port_arg(session: ftp_session_info, cmd: string,
-				arg: string): string
-	{
-	local data = parse_ftp_port(arg);
-
-	if ( data$valid )
-		{
-		local a: addr;
-		# Anonymize the address part
-		a = anonymize_address(data$h, session$connection_id);
-		return fmt_ftp_port(a, data$p);
-		}
-	else
-		return anonymize_arg("unrecognized_ftp_port", session, cmd, arg, "");
-	}
-
-# EPRT is an extension to the PORT command
-function anonymize_eprt_arg(session: ftp_session_info, cmd: string,
-				arg: string): string
-	{
-	if ( arg != ftp_eprt_pat )
-		return anonymize_arg("unrecognized_EPRT_arg", session, cmd, arg, "");
-
-	local parts = split(arg, /\|/);
-	# Anonymize the address part
-	local a = parse_dotted_addr(parts[3]);
-	a = anonymize_address(a, session$connection_id);
-	return fmt("|%s|%s|%s|", parts[2], a, parts[4]);
-	}
-
-# Anonymize arguments of commands that we do not understand
-function anonymize_other_arg(session: ftp_session_info, cmd: string, arg: string): string
-	{
-	local anon: string;
-
-	# Try to guess what the arg is
-
-	local data = parse_ftp_port(arg);
-	if ( arg == ftp_port_pat && data$valid )
-		# Here we do not check whether data$h == session$connection_id$orig_h
-		# because sometimes it's not the case, but we will try to anonymize it anyway.
-		{
-		anon = anonymize_port_arg(session, cmd, arg);
-		print ftp_anon_log, fmt("anonymize_arg: (%s) {} %s \"%s\" to \"%s\" in [%s]",
-					"port arg of non-port command",
-					cmd, arg, anon, id_string(session$connection_id));
-		}
-
-	else if ( arg == ftp_ip_pat )
-		{
-		local a = parse_dotted_addr(arg);
-		a = anonymize_address(a, session$connection_id);
-		anon = cat(a);
-		}
-
-	else if ( arg == ftp_domain_name_pat )
-		{
-		anon = "<domain name>";
-		}
-
-	else if ( arg == "." )
-		{
-		anon = arg;
-		leave_in_the_clear(".", session, arg, fmt("%s %s", cmd, arg));
-		}
-
-	else
-		# Anonymize by default.
-		anon = anonymize_arg("cannot_understand_arg",
-					session, cmd, arg, cmd);
-
-	return anon;
-	}
-
-# Anonymize the command and argument, and put the results in
-# cmd_arg$anonymized_{cmd, arg}
-function anonymize_ftp_cmd_arg(session: ftp_session_info,
-				cmd_arg: ftp_cmd_arg)
-	{
-	local cmd = cmd_arg$cmd;
-	local arg = cmd_arg$arg;
-	local anon : string;
-
-	cmd_arg$anonymized_cmd = cmd;
-
-	local ret: ftp_cmd_arg_anon_result;
-
-	if ( trace_specific_cmd_arg_anonymization )
-		ret = anonymize_trace_specific_cmd_arg(session, cmd, arg);
-	else
-		ret$anonymized = F;
-
-	if ( ret$anonymized )
-		{
-		# If the trace-specific anonymization applies to the cmd_arg
-			print ftp_anon_log, fmt("anonymize_arg: (%s) \"%s %s\" to \"%s %s\" in [%s]",
-				"trace-specific",
-				cmd, to_string_literal(arg),
-				ret$cmd, to_string_literal(ret$arg),
-				id_string(session$connection_id));
-		cmd_arg$anonymized_cmd = ret$cmd;
-		anon = ret$arg;
-		}
-
-	else if ( [cmd, arg] in ftp_special_cmd_args )
-		{
-		anon = ftp_special_cmd_args[cmd, arg];
-		print ftp_anon_log, fmt("anonymize_arg: (%s) [%s] \"%s\" to \"%s\" in [%s]",
-					"special_arg_transformation",
-					cmd,
-					to_string_literal(arg),
-					to_string_literal(anon),
-					id_string(session$connection_id));
-		}
-
-	else if ( to_upper(string_cat(cmd, " ", arg)) == ftp_safe_cmd_arg_pattern ||
-		  [cmd, arg] in ftp_safe_arg_list )
-		{
-		leave_in_the_clear("safe_arg", session, arg, fmt("%s %s", cmd, arg));
-		anon = arg;
-		}
-
-	else if ( cmd == "USER" || cmd == "ACCT" )
-		anon = (arg in ftp_guest_ids) ?
-			arg : anonymize_user_id(session, arg, LOGIN_PENDING, "");
-
-	else if ( cmd == "PASS" )
-		anon = "<password>";
-
-	else if ( cmd in ftp_cmds_with_no_arg )
-		anon = (arg == "") ?
-			"" :
-			anonymize_arg("should_have_been_empty",
-					session, cmd, arg, cmd);
-
-	else if ( cmd in ftp_cmds_with_file_arg )
-		{
-		if ( session$user in ftp_guest_ids )
-			anon = ( arg == "" ) ? "" : anonymize_file_name_arg(session, cmd, arg, F);
-		else
-			anon = "<path>";
-		}
-
-	else if ( cmd in ftp_cmds_with_safe_arg )
-		anon = check_safe_arg(session, cmd, arg);
-
-	else if ( cmd == "SITE" )
-		anon = check_site_arg(session, cmd, arg);
-
-	else if ( cmd == "PORT" )
-		anon = anonymize_port_arg(session, cmd, arg);
-
-	else if ( cmd == "EPRT" )
-		anon = anonymize_eprt_arg(session, cmd, arg);
-
-	else if ( cmd == "AUTH" )
-		anon = anonymize_arg("rejected_auth_arg", session, cmd, arg, cmd);
-
-	else
-		{
-		if ( cmd == /<.*>/ )
-			cmd_arg$anonymized_cmd = "";
-
-		else if ( cmd !in ftp_other_cmds )
-			{
-			local a = anonymize_string(string_cat("cmd:", cmd));
-			print ftp_anon_log, fmt("anonymize_cmd: (%s) \"%s\" [%s] to \"%s\" in [%s]",
-				"unrecognized command", to_string_literal(cmd),
-				to_string_literal(arg), a,
-				id_string(session$connection_id));
-			cmd_arg$anonymized_cmd = a;
-			}
-
-		anon = anonymize_other_arg(session, cmd, arg);
-		}
-
-	if ( cmd == "USER" )
-		session$anonymized_user = anon;
-
-	cmd_arg$anonymized_arg = anon;
-	}
-
-
-# We delay anonymization of certain requests till we see the reply:
-# when the argument of a USER command is a sensitive user ID, we
-# anonymize the ID if the login is successful and leave the ID in the
-# clear otherwise.
-#
-# The function returns T if the decision should be delayed.
-
-function delay_rewriting_request(session: ftp_session_info, cmd: string,
-				 arg: string): bool
-	{
-	return (cmd == "USER" && arg !in ftp_guest_ids) ||
-		(cmd == "AUTH") ||
-		(cmd in ftp_cmds_with_file_arg &&
-		 session$connection_id$resp_h in ftp_public_servers);
-	}
-
-function ftp_request_rewrite(c: connection, session: ftp_session_info,
-				cmd_arg: ftp_cmd_arg): bool
-	{
-	local cmd = cmd_arg$cmd;
-	local arg = cmd_arg$arg;
-
-	if ( delay_rewriting_request(session, cmd, arg) )
-		{
-		cmd_arg$rewrite_slot = reserve_rewrite_slot(c);
-		session$delayed_request_rewrite[cmd_arg$seq] = cmd_arg;
-		return F;
-		}
-	else
-		{
-		anonymize_ftp_cmd_arg(session, cmd_arg);
-		rewrite_ftp_request(c, cmd_arg$anonymized_cmd,
-					cmd_arg$anonymized_arg);
-		return T;
-		}
-	}
-
-function do_rewrite_delayed_ftp_request(c: connection, delayed: ftp_cmd_arg)
-	{
-	seek_rewrite_slot(c, delayed$rewrite_slot);
-
-	rewrite_ftp_request(c, delayed$cmd == /<.*>/ ? "" : delayed$cmd,
-				delayed$anonymized_arg);
-	release_rewrite_slot(c, delayed$rewrite_slot);
-
-	delayed$rewrite_slot = 0;
-	}
-
-function delayed_ftp_request_rewrite(c: connection, session: ftp_session_info,
-					delayed: ftp_cmd_arg,
-					current: ftp_cmd_arg,
-					reply_code: count)
-	{
-	local cmd = delayed$cmd;
-	local arg = delayed$arg;
-
-	if ( cmd == "USER" )
-		{
-		delayed$anonymized_cmd = cmd;
-
-		local login_status: login_status_type;
-
-		if ( reply_code == 0 )
-			login_status = LOGIN_UNKNOWN;
-
-		else if ( delayed$seq == current$seq ||
-		     current$cmd == "PASS" ||
-		     current$cmd == "ACCT" )
-				{
-			if ( reply_code >= 330 && reply_code < 340 ) # need PASS/ACCT
-				login_status = LOGIN_PENDING; # wait to see outcome of PASS
-
-			else if ( reply_code >= 400 && reply_code < 600 )
-					login_status = LOGIN_FAILED;
-
-			else if ( reply_code >= 230 && reply_code < 240 )
-				login_status = LOGIN_SUCCESSFUL;
-
-			else
-				login_status = LOGIN_UNKNOWN;
-			}
-
-		else if ( current$cmd == "USER" ) # another login attempt
-			login_status = LOGIN_FAILED;
-
-		else if ( reply_code == 230 )
-			login_status = LOGIN_SUCCESSFUL;
-
-		else if ( reply_code == 530 )
-			login_status = LOGIN_FAILED;
-
-		else
-			login_status = LOGIN_UNKNOWN;
-
-		if ( login_status != LOGIN_PENDING )
-			{
-			delayed$anonymized_arg =
-				anonymize_user_id(session, arg, login_status,
-					fmt("(%s %s) %s %s -> %d", cmd, arg, current$cmd, current$arg, reply_code));
-			do_rewrite_delayed_ftp_request(c, delayed);
-			}
-		}
-
-	else if ( cmd == "AUTH" )
-		{
-		delayed$anonymized_cmd = cmd;
-
-		if ( reply_code >= 500 && reply_code < 600 )
-			# if AUTH fails
-			{
-			anonymize_ftp_cmd_arg(session, delayed);
-			do_rewrite_delayed_ftp_request(c, delayed);
-			}
-
-		else if ( reply_code >= 300 && reply_code < 400 )
-			;
-		else 	# otherwise always anonymize the argument
-			{
-			delayed$anonymized_arg = "<auth_mechanism>";
-			do_rewrite_delayed_ftp_request(c, delayed);
-			}
-		}
-
-	else if ( cmd in ftp_cmds_with_file_arg && session$connection_id$resp_h in ftp_public_servers )
-		{
-		delayed$anonymized_cmd = cmd;
-
-		# The argument represents a valid file name on a
-		# public server only if the operation is successful.
-		delayed$anonymized_arg =
-			anonymize_file_name_arg(session, cmd, arg,
-				(cmd != "LIST" && cmd != "NLST" &&
-				 reply_code >= 100 && reply_code < 300));
-		do_rewrite_delayed_ftp_request(c, delayed);
-		}
-
-	else
-		{
-		print ftp_anon_log, "ERROR! unrecognizable delayed ftp request rewrite";
-
-		anonymize_ftp_cmd_arg(session, delayed);
-		do_rewrite_delayed_ftp_request(c, delayed);
-		}
-	}
-
-function process_delayed_rewrites(c: connection, session: ftp_session_info, reply_code: count, cmd_arg: ftp_cmd_arg)
-	{
-	local written: table[count] of ftp_cmd_arg;
-
-	for ( s in session$delayed_request_rewrite )
-		{
-		local ca = session$delayed_request_rewrite[s];
-		delayed_ftp_request_rewrite(c, session, ca, cmd_arg,
-					reply_code);
-
-		if ( ca$rewrite_slot == 0 )
-			written[ca$seq] = ca;
-		}
-
-	for ( s in written )
-		delete session$delayed_request_rewrite[s];
-	}
-
-function ftp_reply_rewrite(c: connection, session: ftp_session_info,
-				code: count, msg: string, cont_resp: bool,
-				cmd_arg: ftp_cmd_arg)
-	{
-	local actual_code = session$reply_code;
-	local xyz = parse_ftp_reply_code(actual_code);
-
-	process_delayed_rewrites(c, session, actual_code, cmd_arg);
-
-	if ( process_ftp_reply_by_message_pattern )
-		{
-		# See *ftp-reply-pattern.bro* for reply anonymization
-		local anon_msg = anonymize_ftp_reply_by_msg_pattern(actual_code, msg,
-					cmd_arg, session);
-		rewrite_ftp_reply(c, code, anon_msg, cont_resp);
-		}
-	else
-		rewrite_ftp_reply(c, code, "<ftp reply message stripped out>", cont_resp);
-	}
-
-
-const eliminate_scans = F &redef;
-const port_scanners: set[addr] &redef;
-global eliminate_scan_for_host: table[addr] of bool;
-
-function eliminate_scan(id: conn_id): bool
-	{
-	local h = id$resp_h;
-
-	if ( h !in eliminate_scan_for_host )
-		{
-		# if the hash string starts with [0-7], i.e. with probability of 50%
-		eliminate_scan_for_host[h] = (/^[0-7]/ in md5_hmac(h));
-		if ( eliminate_scan_for_host[h] )
-			print ftp_anon_log, fmt("eliminate_scans_for_host %s", h);
-		}
-
-	return eliminate_scan_for_host[h];
-	}
-
-redef call_ftp_connection_remove = T;
-
-function ftp_connection_remove(c: connection)
-	{
-	if ( c$id in ftp_sessions )
-		{
-		local session = ftp_sessions[c$id];
-		process_delayed_rewrites(c, session, 0, find_ftp_pending_cmd(session$pending_requests, 0, ""));
-		}
-
-	if ( eliminate_scans && ! requires_trace_commitment )
-		print ftp_anon_log,
-			fmt("ERROR: requires_trace_commitment must be set to true in order to allow scan elimination");
-
-	if ( requires_trace_commitment )
-		{
-		local id = c$id;
-		local eliminate = F;
-
-		if ( eliminate_scans &&
-		     # To check if the connection is part of a port scan
-		     (id !in ftp_sessions ||
-		      ftp_sessions[id]$num_requests == 0 ||
-		      id$orig_h in port_scanners) &&
-		     eliminate_scan(id) )
-			rewrite_commit_trace(c, F, T);
-		else
-			rewrite_commit_trace(c, T, T);
-		}
-	}

policy/ftp.bro

@@ -348,12 +348,6 @@ event ftp_excessive_filename(session: ftp_session_info,
 	session$log_it = T;
 	}
 
-global ftp_request_rewrite: function(c: connection, session: ftp_session_info,
-					cmd_arg: ftp_cmd_arg);
-global ftp_reply_rewrite: function(c: connection, session: ftp_session_info,
-					code: count, msg: string,
-					cont_resp: bool, cmd_arg: ftp_cmd_arg);
-
 # Returns true if the given string is at least 25% composed of 8-bit
 # characters.
 function is_string_binary(s: string): bool
@@ -518,9 +512,6 @@ event ftp_request(c: connection, command: string, arg: string)
 			}
 		}
 
-	if ( rewriting_ftp_trace )
-		ftp_request_rewrite(c, session, cmd_arg);
-
 	if ( command in ftp_all_cmds )
 		{
 		if ( command in ftp_untested_cmds )
@@ -761,11 +752,6 @@ event ftp_reply(c: connection, code: count, msg: string, cont_resp: bool)
 		do_ftp_reply(c, session, code, msg, cmd_arg$cmd, cmd_arg$arg);
 		}
 
-	if ( rewriting_ftp_trace )
-		{
-		ftp_reply_rewrite(c, session, code, msg, cont_resp, cmd_arg);
-		}
-
 	if ( ! cont_resp )
 		{
 		if ( ftp_cmd_pending(session$pending_requests) )

policy/http-anonymizer.bro

@@ -1,433 +0,0 @@
-# $Id:$
-
-# We can't do HTTP rewriting unless we process everything in the connection.
-@load http-reply
-@load http-entity
-@load http-anon-server
-@load http-anon-useragent
-@load http-anon-utils
-@load http-abstract
-
-@load anon
-
-module HTTP;
-
-redef rewriting_http_trace = T;
-redef http_entity_data_delivery_size = 18874368;
-redef abstract_max_length = 18874368;
-
-const rewrite_header_in_position = F;
-
-const http_response_reasons = {
-	"no content", "ok", "moved permanently", "not modified",
-	"use local copy", "object not found", "forbidden", "okay",
-	"object moved", "found", "http", "redirecting to main server",
-	"internal server error", "not found", "unauthorized", "moved",
-	"redirected", "continue", "access forbidden", "partial content",
-	"redirect", "<empty>", "authorization required",
-	"request time-out", "moved temporarily", "",
-};
-
-const keep_alive_pat = /(([0-9]+|timeout=[0-9]+|max=[0-9]+),?)*/ ;
-
-const content_type =
-	  /video\/(x-flv)(;)?/ 	# video
-	| /audio\/(x-scpls)/
-	| /image\/(gif|bmp|jpeg|pjpeg|tiff|png|x-icon)(;)?(qs\=[0-9](\.[0-9])?)?,?/
-	| /application\/(octet-stream|x-www-form-urlencoded|x-javascript|rss\+xml|x-gzip|x-ns-proxy-autoconfig|pdf|pkix-crl|x-shockwave-flash|postscript|xml|rdf\+xml|excel|msword|x-wais-source)(;)?(charset=(iso-8859-1|iso8859-1|gb2312|windows-1251|windows-1252|utf-8))?/
-	| /text\/(plain|js|html|\*|css|xml|javascript);?(charset=(iso-8859-1|iso8859-1|gb2312|windows-1251|windows-1252|utf-8))?/
-	| /^unknown$/
-	;
-
-const accept_enc_pat =
-	/(((x-)?deflate|(x-)?compress|\*|identity|(x-)?gzip|bzip|bzip2)(\; *q\=[0-9](\.[0-9])?)?,?)*/ ;
-const accept_charset_pat =
-	/((windows-(1252|1251)|big5|iso-8859-(1|15)|\*|utf-(8|16))(\; *q\=[0-9](\.[0-9])?)?,?)*/ ;
-const connection_pat = /((close|keep\-alive|transfer\-encoding|te),?)*/ ;
-
-const http_methods =
-	/get|put|post|head|propfind|connect|options|proppatch|lock|unlock|move|delete|mkcol/ ;
-
-const http_version = /(1\.0|1\.1)/ ;
-
-const last_modified_pat =
-	  /(Sun|Mon|Tue|Wed|Thu|Fri|Sat), [0-9]+ (Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec) [0-9][0-9][0-9][0-9] .*/
-	| /(-)?[0-9]+/
-	;
-
-const vary_pat =
-	/((\*| *|accept|accept\-charset|negotiate|host|user\-agent|accept\-language|accept\-encoding|cookie),?)*/ ;
-
-const accept_lang_pat =
-	/(( *|tw|cs|mx|tr|ru|sk|au|hn|sv|no|bg|en|ko|kr|ca|pl|nz|fr|ch|jo|gb|zh|hk|cn|lv|de|nl|dk|fi|nl|es|pe|it|pt|br|ve|cl|ja|jp|he|ha|ar|us|en-us|da)(\; *q\=[0-9](\.[0-9]+)?)?(,|-|\_)?)*/ ;
-
-const accept_pat =
-	/(( *|audio|application|\*|gif|xml|xhtml\+xml|x-rgb|x-xbm|video|x-gsarcade-launch|mpeg|sgml|tiff|x-rgb|x-xbm|postscript|text|html|x-xbitmap|pjpeg|vnd.ms-powerpoint|vnd.ms-excel|msword|salt\+html|xhtml|plain|jpeg|jpg|x-shockwave-flash|x-|css|image|png|\*)(\; *q\=[0-9]*(\.[0-9]+)?)?(,|\/|\+)?)*/ ;
-
-const tcn_pat = /list|choice|adhoc|re-choose|keep/;
-
-const date_pat =
-	  /(sun|mon|tue|wed|thu|fri|sat)\,*[0-9]+ *(jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec) *[0-9]+ ([0-9]+:)*[0-9]+ gmt/
-	| /(sun|mon|tue|wed|thu|fri|sat)*(jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec) *[0-9]+ *([0-9]+:)*[0-9]+(am|pm)?( *[0-9]+)?( *gmt)?/ ;
-
-const content_encoding_pat = /gzip|deflate|x-compress|x-gzip/;
-
-const hashed_headers =
-	  /COOKIE/
-	| /AUTHOR/
-	| /CACHE-CONTROL/
-	| /ETAG/
-	| /VIA/
-	| /X-VIA/
-	| /IISEXPORT/
-	| /SET-COOKIE/
-	| /X-JUNK/
-	| /PRAGMA/
-	| /AUTHORIZATION/
-	| /X-POWERED-BY/
-	| /X-CACHE/
-	| /X-FORWARDED-FOR/
-	| /X-PAD/
-	| /X-C/
-	| /XSERVER/
-	| /FROM/
-	| /CONTENT-DISPOSITION/
-	| /X-ASPNET-VERSION/
-	| /GUID/
-	| /REGIONDATA/
-	| /CLIENTID/
-	| /X-CACHE-HEADERS-SET-BY/
-	| /X-CACHE-LOOKUP/
-	| /WARNING/
-	| /MICROSOFTOFFICEWEBSERVER/
-	| /IF-NONE-MATCH/
-	| /X-AMZ-ID-[0-9]/
-	| /X-N/
-	| /X-TR/
-	| /X-RSN/
-	#| /X-POOKIE/	# these are weird ... next two are from slashdot
-	#| /X-FRY/
-	#| /X-BENDER/
-	| /RANGE/
-	| /IF-RANGE/
-	| /CONTENT-RANGE/
-	| /AD-REACH/
-	| /HMSERVER/
-	| /STATUS/
-	| /X-SERVED/
-	| /WWW-AUTHENTICATE/
-	| /X-RESPONDING-SERVER/
-	| /MAX-AGE/
-	| /POST-CHECK/
-	| /PRE-CHECK/
-	| /X-CONTENT-ENCODED-BY/
-	| /X-USER-IP/
-	| /X-ICAP-VERSION/
-	| /X-DELPHI/
-	| /AUTHENTICATION-INFO/
-	| /PPSERVER/
-	| /EDGE-CONTROL/
-	| /COMPRESSION-CONTROL/
-	| /CONTENT-MD5/
-	| /X-HOST/
-	| /P3P/
-	;
-
-event http_request(c: connection, method: string,
-		   original_URI: string, unescaped_URI: string, version: string)
-	{
-	if (! rewriting_trace() )
-		return;
-
-	print http_anon_log,
-		fmt(" > %s %s %s ", method, original_URI, version);
-
-	if ( to_lower(method) != http_methods )
-		{
-		print http_anon_log, fmt("*** Unknown method %s", method);
-		method = string_cat(" (anon-unknown) ", anonymize_string(method));
-		}
-
-	original_URI = anonymize_http_URI(original_URI);
-
-	if ( version != http_version )
-		{
-		print http_anon_log, fmt("*** Unknown version %s ", version);
-		version = string_cat(" (anon-unknown) ", anonymize_string(version));
-		}
-
-	print http_anon_log, fmt(" < %s %s %s ", method, original_URI, version);
-
-	rewrite_http_request(c, method, original_URI, version);
-	}
-
-event http_reply(c: connection, version: string, code: count, reason: string)
-	{
-	if ( rewriting_trace() )
-		{
-		reason = to_lower(strip(reason));
-		if ( reason !in http_response_reasons )
-			{
-			print http_anon_log,
-				fmt("*** Unknown reply reason %s ", reason);
-			rewrite_http_reply(c, version, code,
-				anonymize_string(reason));
-			}
-		else
-			rewrite_http_reply(c, version, code, reason);
-		}
-	}
-
-function check_pat(value: string, pat: pattern, name: string): string
-	{
-	if ( value == pat )
-		return value;
-
-	print http_anon_log, fmt("*** invalid %s: %s", name, value);
-	return "(anon-unknown): ";
-	}
-
-function check_pat2(value: string, pat: pattern, name: string): string
-	{
-	if ( value == pat )
-		return value;
-
-	print http_anon_log, fmt("*** invalid %s: %s", name, value);
-	return fmt("(anon-unknown): %s", anonymize_string(value));
-	}
-
-function check_pat3(value: string, pat: pattern): string
-	{
-	if ( value == pat )
-		return value;
-
-	return fmt("(anon-unknown): %s", anonymize_string(value));
-	}
-
-event http_header(c: connection, is_orig: bool, name: string, value: string)
-	{
-	if ( ! rewriting_trace() )
-		return;
-
-	# Only rewrite top-level headers.
-	local s = lookup_http_request_stream(c);
-	local msg = get_http_message(s, is_orig);
-
-	if ( msg$entity_level != 1 )
-		return;
-
-	value = strip(value);
-
-	if ( name == "CONTENT-LENGTH" )
-		{
-		# if ( rewrite_header_in_position )
-		# {
-		#     local p = current_packet(c);
-		#     if ( p$is_orig == is_orig )
-		#     {
-		#         # local s = lookup_http_request_stream(c);
-		#         # local msg = get_http_message(s, is_orig);
-		#         if ( msg$header_slot == 0 )
-		#             msg$header_slot = reserve_rewrite_slot(c);
-		#     }
-		#     else
-		#         print fmt("cannot reserve a slot at %.6f", network_time());
-		# }
-		print http_anon_log,
-			fmt("X-Original-Content-Length: %s --", value);
-		name = "X-Original-Content-Length";
-		}
-
-	else if ( name == "TRANSFER-ENCODING" || name == "TE" )
-		{
-		print http_anon_log, fmt("TRANSFER-ENCOODING: %s --", value);
-		name = "X-Original-Transfer-Encoding";
-		}
-
-	else if ( name == "HOST" )
-		{
-		local anon_host = "";
-
-		if ( value == simple_filename )
-			anon_host = anonymize_path(value);
-		else
-			anon_host = anonymize_host(value);
-
-		print http_anon_log, fmt("HOST: %s > %s", value, anon_host);
-		value = anon_host;
-		}
-
-	else if ( name == "REFERER" )
-		{
-		local anon_ref = anonymize_http_URI(value);
-		print http_anon_log, fmt("REFERER: %s > %s", value, anon_ref);
-		value = anon_ref;
-		}
-
-	else if ( name == "LOCATION" || name == "CONTENT-LOCATION" )
-		value = anonymize_http_URI(value);
-
-	else if ( name == "SERVER" )
-		value = filter_in_http_server(to_lower(value));
-
-	else if ( name == "USER-AGENT" )
-		value = filter_in_http_useragent(to_lower(value));
-
-	else if ( name == "KEEP-ALIVE" )
-		value = check_pat(value, keep_alive_pat, "keep-alive");
-
-	else if ( name == "DATE" || name == "IF-MODIFIED-SINCE" ||
-		  name == "UNLESS-MODIFIED-SINCE" )
-		value = check_pat2(to_lower(value), date_pat, "date");
-
-	else if ( name == "ACCEPT-CHARSET" )
-		value = check_pat(to_lower(value), accept_charset_pat,
-					"accept-charset");
-
-	else if ( name == "CONTENT-TYPE" )
-		{
-		value = check_pat2(to_lower(value), content_type, "content-type");
-		# local stream = lookup_http_request_stream(c);
-		# local the_http_msg = get_http_message(stream, is_orig);
-		# the_http_msg$content_type = value;
-		}
-
-	else if ( name == "ACCEPT-ENCODING" )
-		value = check_pat2(to_lower(value), accept_enc_pat,
-					"accept-encoding");
-
-	else if ( name == "PAGE-COMPLETION-STATUS" )
-		value = check_pat2(to_lower(value), /(ab)?normal/,
-					"page-completion-status");
-
-	else if ( name == "CONNECTION" || name == "PROXY-CONNECTION" )
-		value = check_pat2(to_lower(value), connection_pat,
-					"connection type");
-
-	else if ( name == "LAST-MODIFIED" || name == "EXPIRES" )
-		value = check_pat(value, last_modified_pat, name);
-
-	else if (name == "ACCEPT-LANGUAGE" || name == "LANGUAGE")
-		value = check_pat2(to_lower(value), accept_lang_pat,
-					"accept-language");
-
-	else if ( name == "ACCEPT" )
-		value = check_pat(to_lower(value), accept_pat, "accept");
-
-	else if ( name == "ACCEPT-RANGES" )
-		value = check_pat2(to_lower(value), /(bytes|none) */,
-					"accept-ranges");
-
-	else if ( name == "MIME-VERSION" )
-		value = check_pat3(value, /[0-9]\.[0-9]/);
-
-	else if ( name == "TCN" )
-		value = check_pat3(value, tcn_pat);
-
-	else if ( name == "CONTENT-ENCODING" )
-		value = check_pat2(value, content_encoding_pat,
-					"content-encoding");
-
-	else if ( name == "CONTENT-LANGUAGE" )
-		value = check_pat2(value, accept_lang_pat, "content-language");
-
-	else if ( name == "ALLOW" )
-		value = check_pat3(value, http_methods);
-
-	else if ( name == "AGE" || name == "BANDWIDTH" )
-		value = check_pat3(value, /[0-9]+/);
-
-	else if ( name == "VARY" )
-		value = check_pat2(value, vary_pat, "vary");
-
-	else if ( name == hashed_headers )
-		value = anonymize_string(value);
-
-	else
-		{
-		print http_anon_log, fmt("unknown header: %s : %s", name, value);
-		value = string_cat("(anon-unknown): ", anonymize_string(value));
-		}
-
-	rewrite_http_header(c, is_orig, name, value);
-	}
-
-event http_all_headers(c: connection, is_orig: bool, hlist: mime_header_list)
-	{
-	if ( ! rewriting_trace() )
-		return;
-
-	if ( rewrite_header_in_position )
-		{
-		local p = current_packet(c);
-		if ( p$is_orig == is_orig )
-			{
-			local s = lookup_http_request_stream(c);
-			local msg = get_http_message(s, is_orig);
-			if ( msg$header_slot == 0 )
-				msg$header_slot = reserve_rewrite_slot(c);
-			}
-		else
-			print fmt("cannot reserve a slot at %.6f", network_time());
-
-		# An empty line to mark the end of headers.
-		rewrite_http_data(c, is_orig, "\r\n");
-		}
-	}
-
-event http_message_done(c: connection, is_orig: bool, stat: http_message_stat)
-	{
-	if ( ! rewriting_trace() )
-		return;
-
-	if ( stat$interrupted )
-		{
-		print http_log,
-			fmt("%.6f %s message interrupted at length=%d \"%s\"",
-				network_time(), id_string(c$id),
-				stat$body_length, stat$finish_msg);
-		}
-
-	local s = lookup_http_request_stream(c);
-	local msg = get_http_message(s, is_orig);
-	if ( msg$header_slot > 0 )
-		seek_rewrite_slot(c, msg$header_slot);
-
-	local data_length = 0;
-	local data_hash = "";
-	local sanitized_abstract = "";
-
-	if ( ! is_orig || stat$body_length > 0 )
-		{
-		data_length = byte_len(msg$abstract);
-		data_hash = anonymize_string(msg$abstract);
-		sanitized_abstract = string_fill(data_length, data_hash);
-
-		data_length += stat$content_gap_length;
-
-		rewrite_http_header(c, is_orig, "Content-Length",
-					fmt(" %d", data_length));
-
-		rewrite_http_header(c, is_orig, "X-anon-content-hash",
-					fmt(" %s", data_hash));
-
-		rewrite_http_header(c, is_orig, "X-Actual-Data-Length",
-					fmt(" %d; gap=%d, content-length=%s",
-						stat$body_length,
-						stat$content_gap_length,
-						msg$content_length));
-		}
-
-	if ( msg$header_slot > 0 )
-		{
-		release_rewrite_slot(c, msg$header_slot);
-		msg$header_slot = 0;
-		}
-
-	if ( ! rewrite_header_in_position )
-		# An empty line to mark the end of headers.
-		rewrite_http_data(c, is_orig, "\r\n");
-
-	if ( data_length > 0 )
-		rewrite_http_data(c, is_orig, sanitized_abstract);
-	}

policy/http-rewriter.bro

@@ -1,137 +0,0 @@
-# $Id: http-rewriter.bro 416 2004-09-17 03:52:28Z vern $
-
-# We can't do HTTP rewriting unless we process everything in the connection.
-@load http-reply
-@load http-entity
-
-module HTTP;
-
-redef rewriting_http_trace = T;
-redef http_entity_data_delivery_size = 4096;
-
-const rewrite_header_in_position = F;
-
-event http_request(c: connection, method: string,
-		   original_URI: string, unescaped_URI: string, version: string)
-	{
-	if ( rewriting_trace() )
-		rewrite_http_request(c, method, original_URI, version);
-	}
-
-event http_reply(c: connection, version: string, code: count, reason: string)
-	{
-	if ( rewriting_trace() )
-		rewrite_http_reply(c, version, code, reason);
-	}
-
-event http_header(c: connection, is_orig: bool, name: string, value: string)
-	{
-	if ( ! rewriting_trace() )
-		return;
-
-	# Only rewrite top-level headers.
-	local s = lookup_http_request_stream(c);
-	local msg = get_http_message(s, is_orig);
-
-	if ( msg$entity_level == 1 )
-		{
-		if ( name == "CONTENT-LENGTH" )
-			{
-			if ( rewrite_header_in_position )
-				{
-				local p = current_packet(c);
-				if ( p$is_orig == is_orig )
-					{
-					# local s = lookup_http_request_stream(c);
-					# local msg = get_http_message(s, is_orig);
-					if ( msg$header_slot == 0 )
-						msg$header_slot = reserve_rewrite_slot(c);
-					}
-				else
-					print fmt("cannot reserve a slot at %.6f", network_time());
-				}
-			# rewrite_http_header(c, is_orig,
-			#		"X-Original-Content-Length", value);
-			}
-
-		else if ( name == "TRANSFER-ENCODING" )
-			rewrite_http_header(c, is_orig,
-					"X-Original-Transfer-Encoding", value);
-
-		else
-			rewrite_http_header(c, is_orig, name, value);
-		}
-	}
-
-event http_all_headers(c: connection, is_orig: bool, hlist: mime_header_list)
-	{
-	if ( ! rewriting_trace() )
-		return;
-
-	if ( rewrite_header_in_position )
-		{
-		local p = current_packet(c);
-		if ( p$is_orig == is_orig )
-			{
-			local s = lookup_http_request_stream(c);
-			local msg = get_http_message(s, is_orig);
-			if ( msg$header_slot == 0 )
-				msg$header_slot = reserve_rewrite_slot(c);
-			}
-		else
-			print fmt("cannot reserve a slot at %.6f", network_time());
-
-	      # An empty line to mark the end of headers.
-	      rewrite_http_data(c, is_orig, "\r\n");
-	      }
-	}
-
-event http_message_done(c: connection, is_orig: bool, stat: http_message_stat)
-	{
-	if ( ! rewriting_trace() )
-		return;
-
-	local s = lookup_http_request_stream(c);
-	local msg = get_http_message(s, is_orig);
-	local data_length = 0;
-
-	if ( stat$interrupted )
-		{
-		print http_log,
-			fmt("%.6f %s message interrupted at length=%d \"%s\"",
-				network_time(), id_string(c$id),
-				stat$body_length, stat$finish_msg);
-		}
-
-	if ( msg$header_slot > 0 )
-		seek_rewrite_slot(c, msg$header_slot);
-
-	if ( ! is_orig || stat$body_length > 0 )
-		{
-		if ( include_HTTP_abstract )
-			data_length = byte_len(msg$abstract);
-
-		data_length = data_length + stat$content_gap_length;
-
-		rewrite_http_header(c, is_orig, "Content-Length",
-					fmt(" %d", data_length));
-		}
-
-	rewrite_http_header(c, is_orig, "X-Actual-Data-Length",
-				fmt(" %d; gap=%d, content-length=%s",
-					stat$body_length,
-					stat$content_gap_length,
-					msg$content_length));
-	if ( msg$header_slot > 0 )
-		{
-		release_rewrite_slot(c, msg$header_slot);
-		msg$header_slot = 0;
-		}
-
-	if ( ! rewrite_header_in_position )
-		# An empty line to mark the end of headers.
-		rewrite_http_data(c, is_orig, "\r\n");
-
-	if ( data_length > 0 )
-		rewrite_http_data(c, is_orig, msg$abstract);
-	}

policy/ident-rewriter.bro

@@ -1,115 +0,0 @@
-# $Id: ident-rewriter.bro 47 2004-06-11 07:26:32Z vern $
-
-@load ident
-
-redef rewriting_ident_trace = T;
-
-global public_ident_user_ids = { "root", } &redef;
-global public_ident_systems = { "UNIX", } &redef;
-
-const delay_rewriting_request = T;
-
-function public_ident_user(id: string): bool
-	{
-	return id in public_ident_user_ids;
-	}
-
-function public_system(system: string): bool
-	{
-	return system in public_ident_systems;
-	}
-
-type ident_req: record {
-	lport: port;
-	rport: port;
-	rewrite_slot: count;
-};
-
-# Does not support pipelining ....
-global ident_req_slots: table[addr, port, addr, port] of ident_req;
-
-event ident_request(c: connection, lport: port, rport: port)
-	{
-	if ( ! rewriting_trace() )
-		return;
-
-	local id = c$id;
-	if ( delay_rewriting_request )
-		{
-		local slot = reserve_rewrite_slot(c);
-		ident_req_slots[id$orig_h, id$orig_p, id$resp_h, id$resp_p] =
-			[$lport = lport, $rport = rport, $rewrite_slot = slot];
-		}
-	else
-		rewrite_ident_request(c, lport, rport);
-	}
-
-event ident_reply(c: connection, lport: port, rport: port,
-		user_id: string, system: string)
-	{
-	if ( ! rewriting_trace() )
-		return;
-
-	local id = c$id;
-
-	if ( [id$orig_h, id$orig_p, id$resp_h, id$resp_p] in ident_req_slots )
-		{
-		local req = ident_req_slots[id$orig_h, id$orig_p,
-						id$resp_h, id$resp_p];
-
-		seek_rewrite_slot(c, req$rewrite_slot);
-		rewrite_ident_request(c, req$lport, req$rport);
-		release_rewrite_slot(c, req$rewrite_slot);
-
-		delete ident_req_slots[id$orig_h, id$orig_p,
-					id$resp_h, id$resp_p];
-		}
-
-	rewrite_ident_reply(c, lport, rport,
-			public_system(system) ? system : "OTHER",
-			public_ident_user(user_id) ? user_id : "private user");
-	}
-
-event ident_error(c: connection, lport: port, rport: port, line: string)
-	{
-	if ( ! rewriting_trace() )
-		return;
-
-	local id = c$id;
-
-	if ( [id$orig_h, id$orig_p, id$resp_h, id$resp_p] in ident_req_slots )
-		{
-		local req = ident_req_slots[id$orig_h, id$orig_p,
-						id$resp_h, id$resp_p];
-
-		seek_rewrite_slot(c, req$rewrite_slot);
-		rewrite_ident_request(c, req$lport, req$rport);
-		release_rewrite_slot(c, req$rewrite_slot);
-
-		delete ident_req_slots[id$orig_h, id$orig_p,
-					id$resp_h, id$resp_p];
-		}
-
-	rewrite_ident_error(c, lport, rport, line);
-	}
-
-event connection_state_remove(c: connection)
-	{
-	if ( ! rewriting_trace() )
-		return;
-
-	local id = c$id;
-
-	if ( [id$orig_h, id$orig_p, id$resp_h, id$resp_p] in ident_req_slots )
-		{
-		local req = ident_req_slots[id$orig_h, id$orig_p,
-						id$resp_h, id$resp_p];
-
-		seek_rewrite_slot(c, req$rewrite_slot);
-		rewrite_ident_request(c, req$lport, req$rport);
-		release_rewrite_slot(c, req$rewrite_slot);
-
-		delete ident_req_slots[id$orig_h, id$orig_p,
-					id$resp_h, id$resp_p];
-		}
-	}

policy/logging-ascii.bro

@@ -0,0 +1,29 @@
+##! Interface for the ascii log writer.
+
+module LogAscii;
+
+export {
+	## If true, output everything to stdout rather than
+	## into files. This is primarily for debugging purposes.
+	const output_to_stdout = F &redef;
+
+	## If true, include a header line with column names.
+	const include_header = T &redef;
+
+	# Prefix for the header line if included.
+	const header_prefix = "# " &redef;
+
+	## Separator between fields.
+	const separator = "\t" &redef;
+
+	## Separator between set elements.
+	const set_separator = "," &redef;
+
+	## String to use for empty fields.
+	const empty_field = "" &redef;
+
+	## String to use for an unset &optional field.
+	const unset_field = "-" &redef;
+}
+
+

policy/logging.bro

@@ -0,0 +1,209 @@
+##! The Bro logging interface.
+##!
+##! See XXX for a introduction to Bro's logging framework.
+
+module Log;
+
+# Log::ID and Log::Writer are defined in bro.init due to circular dependencies.
+
+export {
+	## If true, is local logging is by default enabled for all filters.
+	const enable_local_logging = T &redef;
+
+	## If true, is remote logging is by default enabled for all filters.
+	const enable_remote_logging = T &redef;
+
+	## Default writer to use if a filter does not specify
+	## anything else.
+	const default_writer = WRITER_ASCII &redef;
+
+	## Type defining the content of a logging stream.
+	type Stream: record {
+		## A record type defining the log's columns.
+		columns: any;
+
+		## Event that will be raised once for each log entry.
+		## The event receives a single same parameter, an instance of type ``columns``.
+		ev: any &optional;
+	};
+
+	## Filter customizing logging.
+	type Filter: record {
+		## Descriptive name to reference this filter.
+		name: string;
+
+		## The writer to use.
+		writer: Writer &default=default_writer;
+
+		## Predicate indicating whether a log entry should be recorded.
+		## If not given, all entries are recorded.
+		##
+		## rec: An instance of the streams's ``columns`` type with its
+		## fields set to the values to logged.
+		##
+		## Returns: True if the entry is to be recorded.
+		pred: function(rec: any): bool &optional;
+
+		## Output path for recording entries matching this
+		## filter.
+                ##
+		## The specific interpretation of the string is up to
+		## the used writer, and may for example be the destination
+		## file name. Generally, filenames are expected to given
+		## without any extensions; writers will add appropiate 
+		## extensions automatically.
+		path: string &optional;
+
+		## A function returning the output path for recording entries
+		## matching this filter. This is similar to ``path`` yet allows
+		## to compute the string dynamically. It is ok to return
+		## different strings for separate calls, but be careful: it's
+		## easy to flood the disk by returning a new string for each
+		## connection ...
+		path_func: function(id: ID, path: string): string &optional;
+
+		## Subset of column names to record. If not given, all
+		## columns are recorded.
+		include: set[string] &optional;
+
+		## Subset of column names to exclude from recording. If not given,
+		## all columns are recorded.
+		exclude: set[string] &optional;
+
+		## If true, entries are recorded locally.
+		log_local: bool &default=enable_local_logging;
+
+		## If true, entries are passed on to remote peers.
+		log_remote: bool &default=enable_remote_logging;
+	};
+
+	# Log rotation support.
+
+	## Information passed into rotation callback functions.
+	type RotationInfo: record {
+		writer: Writer;	##< Writer.
+		path: string;	##< Original path value.
+		open: time;	##< Time when opened.
+		close: time;	##< Time when closed.
+	};
+
+	## Default rotation interval. Zero disables rotation.
+	const default_rotation_interval = 0secs &redef;
+
+	## Default naming suffix format. Uses a strftime() style.
+	const default_rotation_date_format = "%y-%m-%d_%H.%M.%S" &redef;
+
+	## Default postprocessor for writers outputting into files.
+	const default_rotation_postprocessor = "" &redef;
+
+	## Default function to construct the name of a rotated output file.
+	## The default implementation appends info$date_fmt to the original
+	## file name.
+	##
+	## info: Meta-data about the file to be rotated.
+	global default_rotation_path_func: function(info: RotationInfo) : string &redef;
+
+	## Type for controlling file rotation.
+	type RotationControl: record  {
+		## Rotation interval.
+		interv: interval &default=default_rotation_interval;
+		## Format for timestamps embedded into rotated file names.
+		date_fmt: string &default=default_rotation_date_format;
+		## Postprocessor process to run on rotate file.
+		postprocessor: string &default=default_rotation_postprocessor;
+	};
+
+	## Specifies rotation parameters per ``(id, path)`` tuple.
+	## If a pair is not found in this table, default values defined in
+	## ``RotationControl`` are used.
+	const rotation_control: table[Writer, string] of RotationControl &default=[] &redef;
+
+	## Sentinel value for indicating that a filter was not found when looked up.
+	const no_filter: Filter = [$name="<not found>"]; # Sentinel.
+
+	# TODO: Document.
+	global create_stream: function(id: ID, stream: Stream) : bool;
+	global enable_stream: function(id: ID) : bool;
+	global disable_stream: function(id: ID) : bool;
+	global add_filter: function(id: ID, filter: Filter) : bool;
+	global remove_filter: function(id: ID, name: string) : bool;
+	global get_filter: function(id: ID, name: string) : Filter; # Returns no_filter if not found.
+	global write: function(id: ID, columns: any) : bool;
+	global set_buf: function(id: ID, buffered: bool): bool;
+	global flush: function(id: ID): bool;
+	global add_default_filter: function(id: ID) : bool;
+	global remove_default_filter: function(id: ID) : bool;
+}
+
+# We keep a script-level copy of all filters so that we can manipulate them.
+global filters: table[ID, string] of Filter;
+
+@load logging.bif.bro # Needs Filter and Stream defined.
+
+module Log;
+
+function default_rotation_path_func(info: RotationInfo) : string
+	{
+	local date_fmt = rotation_control[info$writer, info$path]$date_fmt;
+	return fmt("%s-%s", info$path, strftime(date_fmt, info$open));
+	}
+
+function create_stream(id: ID, stream: Stream) : bool
+	{
+	if ( ! __create_stream(id, stream) )
+		return F;
+
+	return add_default_filter(id);
+	}
+
+function disable_stream(id: ID) : bool
+	{
+	if ( ! __disable_stream(id) )
+		return F;
+	}
+						   
+function add_filter(id: ID, filter: Filter) : bool
+	{
+	filters[id, filter$name] = filter;
+	return __add_filter(id, filter);
+	}
+
+function remove_filter(id: ID, name: string) : bool
+	{
+	delete filters[id, name];
+	return __remove_filter(id, name);
+	}
+
+function get_filter(id: ID, name: string) : Filter
+	{
+	if ( [id, name] in filters )
+		return filters[id, name];
+
+	return no_filter;
+	}
+
+function write(id: ID, columns: any) : bool
+	{
+	return __write(id, columns);
+	}
+
+function set_buf(id: ID, buffered: bool): bool
+	{
+	return __set_buf(id, buffered);
+	}
+
+function flush(id: ID): bool
+	{
+	return __flush(id);
+	}
+
+function add_default_filter(id: ID) : bool
+	{
+	return add_filter(id, [$name="default"]);
+	}
+
+function remove_default_filter(id: ID) : bool
+	{
+	return remove_filter(id, "default");
+	}
+

policy/mime.bro

@@ -74,13 +74,14 @@ function mime_header_subject(session: mime_session_info,
 ### This is a bit clunky.  These are functions we call out to, defined
 # elsewhere.  The way we really ought to do this is to have them passed
 # in during initialization.  But for now, we presume knowledge of their
-# names in global scope.
+# names.
-module GLOBAL;
+export
-global check_relay_3:
+	{
-	function(session: MIME::mime_session_info, msg_id: string);
+	global SMTP::check_relay_3:
-global check_relay_4:
+		function(session: MIME::mime_session_info, msg_id: string);
-	function(session: MIME::mime_session_info, content_hash: string);
+	global SMTP::check_relay_4:
-module MIME;
+		function(session: MIME::mime_session_info, content_hash: string);
+	}
 
 function mime_header_message_id(session: mime_session_info, name: string, arg: string)
 	{
@@ -107,7 +108,7 @@ function mime_header_message_id(session: mime_session_info, name: string, arg: s
 	s = t[1];
 
 	if ( session$level == 1 && SMTP::process_smtp_relay )
-		check_relay_3(session, s);
+		SMTP::check_relay_3(session, s);
 	}
 
 redef mime_header_handler = {

policy/remote.bro

@@ -43,6 +43,9 @@ export {
 		# Whether to perform state synchronization with peer.
 		sync: bool &default = T;
 
+		# Whether to request logs from the peer.
+		request_logs: bool &default = F;
+
 		# When performing state synchronization, whether we consider
 		# our state to be authoritative.  If so, we will send the peer
 		# our current set when the connection is set up.
@@ -176,6 +179,12 @@ function setup_peer(p: event_peer, dst: Destination)
 		request_remote_sync(p, dst$auth);
 		}
 
+	if ( dst$request_logs )
+		{
+		do_script_log(p, "requesting logs");
+		request_remote_logs(p);
+		}
+
 	dst$peer = p;
 	dst$connected = T;
 	connected_peers[p$id] = dst;

policy/smb-anonymizer.bro

@@ -1,80 +0,0 @@
-# $Id:$
-
-redef rewriting_smb_trace = T;
-
-
-event smb_message(c: connection, hdr: smb_hdr, is_orig: bool, cmd: string, body_length: count, body: string)
-	{
-	}
-
-event smb_com_tree_connect_andx(c: connection, hdr: smb_hdr, path: string, service: string)
-	{
-	}
-
-event smb_com_tree_disconnect(c: connection, hdr: smb_hdr)
-	{
-	}
-
-event smb_com_nt_create_andx(c: connection, hdr: smb_hdr, name: string)
-	{
-	}
-
-event smb_com_transaction(c: connection, hdr: smb_hdr, trans: smb_trans, data: smb_trans_data, is_orig: bool)
-	{
-	}
-
-event smb_com_transaction2(c: connection, hdr: smb_hdr, trans: smb_trans, data: smb_trans_data, is_orig: bool)
-	{
-	}
-
-event smb_com_trans_mailslot(c: connection, hdr: smb_hdr, trans: smb_trans, data: smb_trans_data, is_orig: bool)
-	{
-	}
-
-event smb_com_trans_rap(c: connection, hdr: smb_hdr, trans: smb_trans, data: smb_trans_data, is_orig: bool)
-	{
-	}
-
-event smb_com_trans_pipe(c: connection, hdr: smb_hdr, trans: smb_trans, data: smb_trans_data, is_orig: bool)
-	{
-	}
-
-event smb_com_read_andx(c: connection, hdr: smb_hdr, data: string)
-	{
-	}
-
-event smb_com_write_andx(c: connection, hdr: smb_hdr, data: string)
-	{
-	}
-
-event smb_get_dfs_referral(c: connection, hdr: smb_hdr, max_referral_level: count, file_name: string)
-	{
-	}
-
-event smb_com_negotiate(c: connection, hdr: smb_hdr)
-	{
-	}
-
-event smb_com_negotiate_response(c: connection, hdr: smb_hdr, dialect_index: count)
-	{
-	}
-
-event smb_com_setup_andx(c: connection, hdr: smb_hdr)
-	{
-	}
-
-event smb_com_generic_andx(c: connection, hdr: smb_hdr)
-	{
-	}
-
-event smb_com_close(c: connection, hdr: smb_hdr)
-	{
-	}
-
-event smb_com_logoff_andx(c: connection, hdr: smb_hdr)
-	{
-	}
-
-event smb_error(c: connection, hdr: smb_hdr, cmd: count, cmd_str: string, data: string)
-	{
-	}

policy/smtp-rewriter.bro

@@ -1,94 +0,0 @@
-# $Id: smtp-rewriter.bro 4758 2007-08-10 06:49:23Z vern $
-
-@load smtp
-@load mime 	# need mime for content hash
-
-module SMTP;
-
-redef rewriting_smtp_trace = T;
-
-# We want this event handler to execute *after* the one in smtp.bro.
-event smtp_request(c: connection, is_orig: bool, command: string, arg: string)
-	{
-	if ( ! rewriting_trace() )
-		return;
-
-	local session = smtp_sessions[c$id];
-
-	if ( command != ">" )
-		{
-		if ( command == "." )
-			{
-			# A hack before we have MIME rewriter.
-			# rewrite_smtp_data(c, is_orig, fmt("X-number-of-lines: %d",
-			# 			session$num_lines_in_body));
-			rewrite_smtp_data(c, is_orig, fmt("X-number-of-bytes: %d",
-						session$num_bytes_in_body));
-
-			# Write empty line to avoid MIME analyzer complaints.
-			rewrite_smtp_data(c, is_orig, "");
-			rewrite_smtp_data(c, is_orig, fmt("%s", session$content_hash));
-			}
-
-		if ( command in smtp_legal_cmds )
-			{
-			# Avoid the situation in which we mistake
-			# mail contents for SMTP commands.
-			rewrite_smtp_request(c, is_orig, command, arg);
-			rewrite_push_packet(c, is_orig);
-			}
-		}
-	}
-
-event smtp_reply(c: connection, is_orig: bool, code: count, cmd: string,
-			msg: string, cont_resp: bool)
-	{
-	if ( ! rewriting_trace() )
-		return;
-
-	rewrite_smtp_reply(c, is_orig, code, msg, cont_resp);
-	}
-
-function starts_with_leading_whitespace(s: string): bool
-	{
-	return /^[ \t]/ in s;
-	}
-
-function rewrite_smtp_header_line(c: connection, is_orig: bool,
-				session: smtp_session_info, line: string)
-	{
-	if ( starts_with_leading_whitespace(line) )
-		{ # a continuing header
-		if ( session$keep_current_header )
-			rewrite_smtp_data(c, is_orig, line);
-		}
-	else
-		{
-		session$keep_current_header = F;
-
-		local pair = split1(line, /:/);
-		if ( length(pair) < 2 )
-			{
-			session$keep_current_header = T;
-			rewrite_smtp_data(c, is_orig, line);
-			}
-		else
-			{
-			local field_name = to_upper(pair[1]);
-
-			# Currently, the MIME analyzer is sensitive to
-			# CONTENT-TYPE and CONTENT_TRANSFER_ENCODING,
-			# so we want to remove these when anonymizing,
-			# because we can't ensure their integrity when
-			# rewriting message bodies.
-			#
-			# To be conservative, however, we strip out *all*
-			# CONTENT-* headers.
-			if ( /^CONTENT-/ !in field_name )
-				{
-				session$keep_current_header = T;
-				rewrite_smtp_data(c, is_orig, line);
-				}
-			}
-		}
-	}

policy/smtp.bro

@@ -505,15 +505,9 @@ event connection_state_remove(c: connection)
 		}
 	}
 
-global rewrite_smtp_header_line:
-	function(c: connection, is_orig: bool,
-			session: smtp_session_info, line: string);
-
 function smtp_header_line(c: connection, is_orig: bool,
 				session: smtp_session_info, line: string)
 	{
-	if ( rewriting_smtp_trace )
-		rewrite_smtp_header_line(c, is_orig, session, line);
 	}
 
 function smtp_body_line(c: connection, is_orig: bool,

src/Active.cc

@@ -1,218 +0,0 @@
-// $Id: Active.cc 1282 2005-09-07 17:02:02Z vern $
-
-#include "config.h"
-
-#include <sys/types.h>
-#include <sys/socket.h>
-
-#include <netinet/in.h>
-
-#include <arpa/inet.h>
-
-#include <map>
-
-#include "Active.h"
-#include "util.h"
-#include "Dict.h"
-
-declare(PDict,string);
-typedef PDict(string) MachineMap;
-declare(PDict,MachineMap);
-typedef PDict(MachineMap) ActiveMap;
-declare(PDict,NumericData);
-typedef PDict(NumericData) ActiveMapNumeric;
-
-static ActiveMap active_map;
-static ActiveMapNumeric active_map_numeric;
-
-static MachineMap default_values;
-static NumericData default_values_numeric;
-
-static bool map_was_loaded = false;
-
-bool get_map_result(uint32 ip_addr, const char* key, string& result)
-	{
-	const MachineMap* machine_map;
-#ifndef ACTIVE_MAPPING
-	machine_map = &default_values;
-#else
-	HashKey machinekey(ip_addr);
-	machine_map = active_map.Lookup(&machinekey);
-
-	if ( ! machine_map )
-		machine_map = &default_values;
-#endif
-	HashKey mapkey(key);
-	string* entry = machine_map->Lookup(&mapkey);
-	if ( ! entry )
-		{
-		entry = default_values.Lookup(&mapkey);
-		}
-
-	if ( ! entry )
-		{
-		internal_error("Unknown active mapping entry requested: %s", key);
-		return false;
-		}
-
-	result = *entry;
-
-	return true;
-	}
-
-bool get_map_result(uint32 ip_addr, const NumericData*& result)
-	{
-#ifndef ACTIVE_MAPPING
-	result = &default_values_numeric;
-	return true;
-#endif
-	HashKey machinekey(&ip_addr, 1);
-	NumericData* entry = active_map_numeric.Lookup(&machinekey);
-
-	if ( ! entry )
-		result = &default_values_numeric;
-	else
-		result = entry;
-
-	return true;
-	}
-
-
-char* chop (char* s)
-	{
-	s[strlen(s) - 1] = 0;
-	return s;
-	}
-
-map < const char *, ReassemblyPolicy, ltstr > reassem_names;
-
-// Remember to index the table with IP address in network order!
-bool load_mapping_table(const char* map_file)
-	{
-	reassem_names["BSD"] = RP_BSD;
-	reassem_names["linux"] = RP_LINUX;
-	reassem_names["last"] = RP_LAST;
-	reassem_names["first"] = RP_FIRST;
-
-	// Default values are read in from AM file under IP address 0.0.0.0
-	default_values.Insert(new HashKey("accepts_rst_outside_window"),
-			      new string("no"));
-	default_values.Insert(new HashKey("accepts_rst_in_window"),
-			      new string("yes"));
-	default_values.Insert(new HashKey("accepts_rst_in_sequence"),
-			      new string("yes"));
-	default_values.Insert(new HashKey("mtu"),
-			      new string("0"));
-
-	default_values_numeric.path_MTU = 0;   // 0 = unknown
-	default_values_numeric.hops = 0;       // 0 = unknown;
-	default_values_numeric.ip_reassem = RP_UNKNOWN;
-	default_values_numeric.tcp_reassem = RP_UNKNOWN;
-	default_values_numeric.accepts_rst_in_window = true;
-	default_values_numeric.accepts_rst_outside_window = false;
-
-
-	if ( map_file && strlen(map_file) )
-		{
-		FILE* f = fopen(map_file, "r");
-		if ( ! f )
-			return false;
-
-		char buf[512];
-		if ( ! fgets(buf, sizeof(buf), f) )
-			error("Error reading mapping file.\n");
-
-		int num_fields = atoi(buf);
-
-		string* field_names = new string[num_fields];
-		for ( int i = 0; i < num_fields; ++i )
-			{
-			if ( ! fgets(buf, sizeof(buf), f) )
-				error("Error reading mapping file.\n");
-			field_names[i] = chop(buf);
-			}
-
-		if ( ! fgets(buf, sizeof(buf), f) )
-			error("Error reading mapping file.\n");
-
-		int num_machines = atoi(buf);
-
-		for ( int j = 0; j < num_machines; ++j )
-			{ // read ip address, parse it
-			if ( ! fgets(buf, sizeof(buf), f) )
-				error("Error reading mapping file.\n");
-
-			uint32 ip;
-			in_addr in;
-			if ( ! inet_aton(chop(buf), &in) )
-				error("Error reading mapping file.\n");
-
-			ip = in.s_addr;
-
-			MachineMap* newmap;
-			NumericData* newnumeric;
-
-			if ( ip ) // ip = 0.0.0.0 = default values
-				{
-				newmap = new MachineMap;
-				newnumeric = new NumericData;
-				}
-			else
-				{
-				newmap = &default_values;
-				newnumeric = &default_values_numeric;
-				}
-
-			for ( int i = 0; i < num_fields; ++i )
-				{
-				if ( ! fgets(buf, sizeof(buf), f) )
-					error("Error reading mapping file.\n");
-
-				string fname = field_names[i];
-
-				chop(buf);
-
-				// Don't try to parse an unknown value ("").
-				if ( streq(buf, "") )
-					continue;
-
-				HashKey mapkey(fname.c_str());
-				newmap->Insert(&mapkey, new string(buf));
-
-				if ( fname == "mtu" )
-					newnumeric->path_MTU = atoi(buf);
-
-				else if ( fname == "hops" )
-					newnumeric->hops = atoi(buf);
-
-				else if ( fname == "tcp_segment_overlap" )
-					newnumeric->tcp_reassem = reassem_names[buf];
-
-				else if ( fname == "overlap_policy" )
-					newnumeric->ip_reassem = reassem_names[buf];
-
-				else if ( fname == "accepts_rst_in_window" )
-					newnumeric->accepts_rst_in_window = streq(buf, "yes");
-
-				else if ( fname == "accepts_rst_outside_window" )
-					newnumeric->accepts_rst_outside_window = streq(buf, "yes");
-
-				else if ( fname == "accepts_rst_in_sequence" )
-					newnumeric->accepts_rst_in_sequence = streq(buf, "yes");
-
-				else
-					warn("unrecognized mapping file tag:", fname.c_str());
-				}
-
-			HashKey machinekey(&ip, 1);
-			active_map.Insert(&machinekey, newmap);
-			active_map_numeric.Insert(&machinekey, newnumeric);
-			}
-
-		delete [] field_names;
-		}
-
-	map_was_loaded = true;
-
-	return true;
-	}

src/Active.h

@@ -1,44 +0,0 @@
-// $Id: Active.h 80 2004-07-14 20:15:50Z jason $
-
-#ifndef active_h
-#define active_h
-
-#include <string>
-using namespace std;
-
-#include "util.h"
-
-enum ReassemblyPolicy {
-	RP_UNKNOWN,
-	RP_BSD,	// Left-trim to equal or lower offset frags
-	RP_LINUX,	// Left-trim to strictly lower offset frags
-	RP_FIRST,	// Accept only new (no previous value) octets
-	RP_LAST	// Accept all
-};
-
-struct NumericData {
-	ReassemblyPolicy ip_reassem;
-	ReassemblyPolicy tcp_reassem;
-	unsigned short path_MTU;	// 0 = unknown
-	unsigned char hops;	// 0 = unknown
-	bool accepts_rst_in_window;
-	bool accepts_rst_outside_window;
-	bool accepts_rst_in_sequence;
-};
-
-// Return value is whether or not there was a known result for that
-// machine (actually, IP address in network order); if not, a default
-// is returned. Note that the map data is a string in all cases: the
-// numeric form is more efficient and is to be preferred ordinarily.
-bool get_map_result(uint32 ip_addr, const char* key, string& result);
-
-// Basically a special case of get_map_result(), but returns numbers
-// directly for efficiency reasons, since these are frequently
-// looked up. ### Perhaps a better generic mechanism is in order.
-bool get_map_result(uint32 ip_addr, const NumericData*& result);
-
-// Reads in AM data from the specified file (basically [IP, policy] tuples)
-// Should be called at initialization time.
-bool load_mapping_table(const char* map_file);
-
-#endif

src/Analyzer.cc

@@ -747,15 +747,6 @@ void Analyzer::FlipRoles()
 	resp_supporters = tmp;
 	}
 
-int Analyzer::RewritingTrace()
-	{
-	LOOP_OVER_CHILDREN(i)
-		if ( (*i)->RewritingTrace() )
-			return 1;
-
-	return 0;
-	}
-
 void Analyzer::ProtocolConfirmation()
 	{
 	if ( protocol_confirmed )
@@ -900,17 +891,10 @@ void SupportAnalyzer::ForwardUndelivered(int seq, int len, bool is_orig)
 		Parent()->Undelivered(seq, len, is_orig);
 	}
 
-TransportLayerAnalyzer::~TransportLayerAnalyzer()
-	{
-	delete rewriter;
-	}
 
 void TransportLayerAnalyzer::Done()
 	{
 	Analyzer::Done();
-
-	if ( rewriter )
-		rewriter->Done();
 	}
 
 void TransportLayerAnalyzer::SetContentsFile(unsigned int /* direction */,
@@ -925,14 +909,6 @@ BroFile* TransportLayerAnalyzer::GetContentsFile(unsigned int /* direction */) c
 	return 0;
 	}
 
-void TransportLayerAnalyzer::SetTraceRewriter(Rewriter* r)
-	{
-	if ( rewriter )
-		rewriter->Done();
-	delete rewriter;
-	rewriter = r;
-	}
-
 void TransportLayerAnalyzer::PacketContents(const u_char* data, int len)
 	{
 	if ( packet_contents && len > 0 )

src/Analyzer.h

@@ -225,10 +225,6 @@ public:
 	virtual void ProtocolViolation(const char* reason,
 					const char* data = 0, int len = 0);
 
-	// Returns true if the analyzer or one of its children is rewriting
-	// the trace.
-	virtual int RewritingTrace();
-
 	virtual unsigned int MemoryAllocation() const;
 
 	// The following methods are proxies: calls are directly forwarded
@@ -367,9 +363,7 @@ private:
 class TransportLayerAnalyzer : public Analyzer {
 public:
 	TransportLayerAnalyzer(AnalyzerTag::Tag tag, Connection* conn)
-		: Analyzer(tag, conn)	{ pia = 0; rewriter = 0; }
+		: Analyzer(tag, conn)	{ pia = 0; }
-
-	virtual ~TransportLayerAnalyzer();
 
 	virtual void Done();
 	virtual void UpdateEndpointVal(RecordVal* endp, int is_orig) = 0;
@@ -381,11 +375,6 @@ public:
 	void SetPIA(PIA* arg_PIA)	{ pia = arg_PIA; }
 	PIA* GetPIA() const		{ return pia; }
 
-	Rewriter* TraceRewriter()	{ return rewriter; }
-
-	// Takes ownership.
-	void SetTraceRewriter(Rewriter* r);
-
 	// Raises packet_contents event.
 	void PacketContents(const u_char* data, int len);
 
@@ -394,7 +383,6 @@ protected:
 
 private:
 	PIA* pia;
-	Rewriter* rewriter;
 };
 
 #endif

src/Anon.cc

@@ -17,7 +17,7 @@ AnonymizeIPAddr* ip_anonymizer[NUM_ADDR_ANONYMIZATION_METHODS] = {0};
 
 static uint32 rand32()
 	{
-	return ((random() & 0xffff) << 16) | (random() & 0xffff);
+	return ((bro_random() & 0xffff) << 16) | (bro_random() & 0xffff);
 	}
 
 // From tcpdpriv.

src/Attr.cc

@@ -7,6 +7,7 @@
 #include "Attr.h"
 #include "Expr.h"
 #include "Serializer.h"
+#include "LogMgr.h"
 
 const char* attr_name(attr_tag t)
 	{
@@ -49,6 +50,37 @@ void Attr::Describe(ODesc* d) const
 		}
 	}
 
+void Attr::DescribeReST(ODesc* d) const
+	{
+	d->Add(":bro:attr:`");
+	AddTag(d);
+	d->Add("`");
+
+	if ( expr )
+		{
+		d->SP();
+		d->Add("=");
+		d->SP();
+
+		if ( expr->Type()->Tag() == TYPE_FUNC )
+			d->Add(":bro:type:`func`");
+
+		else if ( expr->Type()->Tag() == TYPE_ENUM )
+			{
+			d->Add(":bro:enum:`");
+			expr->Describe(d);
+			d->Add("`");
+			}
+
+		else
+			{
+			d->Add("``");
+			expr->Describe(d);
+			d-> Add("``");
+			}
+		}
+	}
+
 void Attr::AddTag(ODesc* d) const
 	{
 	if ( d->IsBinary() )
@@ -57,10 +89,11 @@ void Attr::AddTag(ODesc* d) const
 		d->Add(attr_name(Tag()));
 	}
 
-Attributes::Attributes(attr_list* a, BroType* t)
+Attributes::Attributes(attr_list* a, BroType* t, bool arg_in_record)
 	{
 	attrs = new attr_list(a->length());
 	type = t->Ref();
+	in_record = arg_in_record;
 
 	SetLocationInfo(&start_location, &end_location);
 
@@ -161,6 +194,17 @@ void Attributes::Describe(ODesc* d) const
 		}
 	}
 
+void Attributes::DescribeReST(ODesc* d) const
+	{
+	loop_over_list(*attrs, i)
+		{
+		if ( i > 0 )
+			d->Add(" ");
+
+		(*attrs)[i]->DescribeReST(d);
+		}
+	}
+
 void Attributes::CheckAttr(Attr* a)
 	{
 	switch ( a->Tag() ) {
@@ -199,27 +243,72 @@ void Attributes::CheckAttr(Attr* a)
 		{
 		BroType* atype = a->AttrExpr()->Type();
 
-		if ( type->Tag() != TYPE_TABLE || type->IsSet() )
+		if ( type->Tag() != TYPE_TABLE || (type->IsSet() && ! in_record) )
 			{
-			if ( ! same_type(atype, type) )
+			if ( same_type(atype, type) )
-				a->AttrExpr()->Error("&default value has inconsistent type", type);
+				// Ok.
-			break;
+				break;
+
+			// Record defaults may be promotable.
+			if ( (type->Tag() == TYPE_RECORD && atype->Tag() == TYPE_RECORD &&
+			      record_promotion_compatible(atype->AsRecordType(),
+							  type->AsRecordType())) )
+				// Ok.
+				break;
+
+			a->AttrExpr()->Error("&default value has inconsistent type", type);
 			}
 
 		TableType* tt = type->AsTableType();
+		BroType* ytype = tt->YieldType();
 
-		if ( ! same_type(atype, tt->YieldType()) )
+		if ( ! in_record )
 			{
-			// It can still be a default function.
+			// &default applies to the type itself.
-			if ( atype->Tag() == TYPE_FUNC )
+			if ( ! same_type(atype, ytype) )
 				{
-				FuncType* f = atype->AsFuncType();
+				// It can still be a default function.
-				if ( ! f->CheckArgs(tt->IndexTypes()) ||
+				if ( atype->Tag() == TYPE_FUNC )
-				     ! same_type(f->YieldType(), tt->YieldType()) )
+					{
-					Error("&default function type clash");
+					FuncType* f = atype->AsFuncType();
+					if ( ! f->CheckArgs(tt->IndexTypes()) ||
+					     ! same_type(f->YieldType(), ytype) )
+						Error("&default function type clash");
+
+					// Ok.
+					break;
+					}
+
+				// Table defaults may be promotable.
+				if ( (ytype->Tag() == TYPE_RECORD && atype->Tag() == TYPE_RECORD &&
+				      record_promotion_compatible(atype->AsRecordType(),
+								  ytype->AsRecordType())) )
+					// Ok.
+					break;
+
+				Error("&default value has inconsistent type 2");
 				}
-			else
+
-				Error("&default value has inconsistent type");
+			// Ok.
+			break;
+			}
+
+		else
+			{
+			// &default applies to record field.
+
+			if ( same_type(atype, type) ||
+			     (atype->Tag() == TYPE_TABLE && atype->AsTableType()->IsUnspecifiedTable()) )
+				// Ok.
+				break;
+
+			// Table defaults may be promotable.
+			if ( (ytype->Tag() == TYPE_RECORD && atype->Tag() == TYPE_RECORD &&
+			      record_promotion_compatible(atype->AsRecordType(), ytype->AsRecordType())) )
+				// Ok.
+				break;
+
+			Error("&default value has inconsistent type");
 			}
 		}
 		break;
@@ -316,10 +405,12 @@ void Attributes::CheckAttr(Attr* a)
 	case ATTR_GROUP:
 		if ( type->Tag() != TYPE_FUNC ||
 		     ! type->AsFuncType()->IsEvent() )
-			{
 			Error("&group only applicable to events");
-			break;
+		break;
-			}
+
+	case ATTR_LOG:
+		if ( ! LogVal::IsCompatibleType(type) )
+			Error("&log applied to a type that cannot be logged");
 		break;
 
 	default:
@@ -327,6 +418,41 @@ void Attributes::CheckAttr(Attr* a)
 	}
 	}
 
+bool Attributes::operator==(const Attributes& other) const
+	{
+	if ( ! attrs )
+		return other.attrs;
+
+	if ( ! other.attrs )
+		return false;
+
+	loop_over_list(*attrs, i)
+		{
+		Attr* a = (*attrs)[i];
+		Attr* o = other.FindAttr(a->Tag());
+
+		if ( ! o )
+			return false;
+
+		if ( ! (*a == *o) )
+			return false;
+		}
+
+	loop_over_list(*other.attrs, j)
+		{
+		Attr* o = (*other.attrs)[j];
+		Attr* a = FindAttr(o->Tag());
+
+		if ( ! a )
+			return false;
+
+		if ( ! (*a == *o) )
+			return false;
+		}
+
+	return true;
+	}
+
 bool Attributes::Serialize(SerialInfo* info) const
 	{
 	return SerialObj::Serialize(info);

src/Attr.h

@@ -35,6 +35,7 @@ typedef enum {
 	ATTR_MERGEABLE,
 	ATTR_PRIORITY,
 	ATTR_GROUP,
+	ATTR_LOG,
 	ATTR_TRACKED,	// hidden attribute, tracked by NotifierRegistry
 #define NUM_ATTRS (int(ATTR_TRACKED) + 1)
 } attr_tag;
@@ -51,6 +52,21 @@ public:
 		{ return tag == ATTR_REDEF || tag == ATTR_OPTIONAL; }
 
 	void Describe(ODesc* d) const;
+	void DescribeReST(ODesc* d) const;
+
+	bool operator==(const Attr& other) const
+		{
+		if ( tag != other.tag )
+			return false;
+
+		if ( expr || other.expr )
+			// If any has an expression and they aren't the same object, we
+			// declare them unequal, as we can't really find out if the two
+			// expressions are equivalent.
+			return (expr == other.expr);
+
+		return true;
+		}
 
 protected:
 	void AddTag(ODesc* d) const;
@@ -62,7 +78,7 @@ protected:
 // Manages a collection of attributes.
 class Attributes : public BroObj {
 public:
-	Attributes(attr_list* a, BroType* t);
+	Attributes(attr_list* a, BroType* t, bool in_record);
 	~Attributes();
 
 	void AddAttr(Attr* a);
@@ -73,12 +89,15 @@ public:
 	void RemoveAttr(attr_tag t);
 
 	void Describe(ODesc* d) const;
+	void DescribeReST(ODesc* d) const;
 
 	attr_list* Attrs()	{ return attrs; }
 
 	bool Serialize(SerialInfo* info) const;
 	static Attributes* Unserialize(UnserialInfo* info);
 
+	bool operator==(const Attributes& other) const;
+
 protected:
 	Attributes()	{ type = 0; attrs = 0; }
 	void CheckAttr(Attr* attr);
@@ -87,6 +106,7 @@ protected:
 
 	BroType* type;
 	attr_list* attrs;
+	bool in_record;
 };
 
 #endif

src/BroBifDoc.cc

@@ -0,0 +1,16 @@
+#include <cstdio>
+#include <string>
+#include <list>
+
+#include "BroDoc.h"
+#include "BroBifDoc.h"
+
+BroBifDoc::BroBifDoc(const std::string& sourcename) : BroDoc(sourcename)
+	{
+	}
+
+// TODO: This needs to do something different than parent class's version.
+void BroBifDoc::WriteDocFile() const
+	{
+	BroDoc::WriteDocFile();
+	}

src/BroBifDoc.h

@@ -0,0 +1,18 @@
+#ifndef brobifdoc_h
+#define brobifdoc_h
+
+#include <cstdio>
+#include <string>
+#include <list>
+
+#include "BroDoc.h"
+
+class BroBifDoc : public BroDoc {
+public:
+	BroBifDoc(const std::string& sourcename);
+	virtual ~BroBifDoc()	{ }
+
+	void WriteDocFile() const;
+};
+
+#endif

src/BroDoc.cc

@@ -0,0 +1,362 @@
+#include <cstdio>
+#include <cstdarg>
+#include <string>
+#include <list>
+#include <algorithm>
+
+#include "BroDoc.h"
+#include "BroDocObj.h"
+
+BroDoc::BroDoc(const std::string& sourcename)
+	{
+#ifdef DEBUG
+	fprintf(stdout, "Documenting source: %s\n", sourcename.c_str());
+#endif
+	source_filename = sourcename.substr(sourcename.find_last_of('/') + 1);
+
+	size_t ext_pos = source_filename.find_last_of('.');
+	std::string ext = source_filename.substr(ext_pos + 1);
+
+	if ( ext_pos == std::string::npos || ext != "bro" )
+		{
+		if ( source_filename != "bro.init" && source_filename != "<stdin>" )
+			{
+			fprintf(stderr,
+				"Warning: documenting file without .bro extension: %s\n",
+				sourcename.c_str());
+			}
+		else
+			{
+			// Force the reST documentation file to be "bro.init.rst".
+			ext_pos = std::string::npos;
+			}
+		}
+
+	reST_filename = source_filename.substr(0, ext_pos);
+	reST_filename += ".rst";
+	reST_file = fopen(reST_filename.c_str(), "w");
+
+	if ( ! reST_file )
+		fprintf(stderr, "Failed to open %s", reST_filename.c_str());
+#ifdef DEBUG
+	else
+		fprintf(stdout, "Created reST document: %s\n", reST_filename.c_str());
+#endif
+	}
+
+BroDoc::~BroDoc()
+	{
+	if ( reST_file && fclose( reST_file ) )
+		fprintf(stderr, "Failed to close %s", reST_filename.c_str());
+
+	FreeBroDocObjPtrList(all);
+	}
+
+void BroDoc::AddImport(const std::string& s)
+	{
+	size_t ext_pos = s.find_last_of('.');
+
+	if ( ext_pos == std::string::npos )
+		imports.push_back(s);
+
+	else
+		{
+		if ( s.substr(ext_pos + 1) == "bro" )
+			imports.push_back(s.substr(0, ext_pos));
+		else
+			fprintf(stderr, "Warning: skipped documenting @load of file "
+			                "without .bro extension: %s\n", s.c_str());
+		}
+	}
+
+void BroDoc::SetPacketFilter(const std::string& s)
+	{
+	packet_filter = s;
+	size_t pos1 = s.find("{\n");
+	size_t pos2 = s.find("}");
+
+	if ( pos1 != std::string::npos && pos2 != std::string::npos )
+		packet_filter = s.substr(pos1 + 2, pos2 - 2);
+
+	bool has_non_whitespace = false;
+
+	for ( std::string::const_iterator it = packet_filter.begin();
+		it != packet_filter.end(); ++it )
+		{
+		if ( *it != ' ' && *it != '\t' && *it != '\n' && *it != '\r' )
+			{
+			has_non_whitespace = true;
+			break;
+			}
+		}
+
+	if ( ! has_non_whitespace )
+		packet_filter.clear();
+	}
+
+void BroDoc::AddPortAnalysis(const std::string& analyzer,
+			const std::string& ports)
+	{
+	std::string reST_string = analyzer + "::\n" + ports + "\n\n";
+	port_analysis.push_back(reST_string);
+	}
+
+void BroDoc::WriteDocFile() const
+	{
+	WriteToDoc(".. Automatically generated.  Do not edit.\n\n");
+
+	WriteSectionHeading(source_filename.c_str(), '=');
+
+	WriteToDoc("\n:download:`Original Source File <%s>`\n\n",
+		source_filename.c_str());
+
+	WriteSectionHeading("Overview", '-');
+	WriteStringList("%s\n", "%s\n\n", summary);
+
+	if ( ! imports.empty() )
+		{
+		WriteToDoc(":Imports: ");
+		WriteStringList(":doc:`%s`, ", ":doc:`%s`\n", imports);
+		}
+
+	WriteToDoc("\n");
+
+	WriteInterface("Summary", '~', '#', true, true);
+
+	if ( ! modules.empty() )
+		{
+		WriteSectionHeading("Namespaces", '~');
+		WriteStringList(".. bro:namespace:: %s\n", modules);
+		WriteToDoc("\n");
+		}
+
+	if ( ! notices.empty() )
+		WriteBroDocObjList(notices, "Notices", '~');
+
+	WriteInterface("Public Interface", '-', '~', true, false);
+
+	if ( ! port_analysis.empty() )
+		{
+		WriteSectionHeading("Port Analysis", '-');
+		WriteToDoc(":ref:`More Information <common_port_analysis_doc>`\n\n");
+		WriteStringList("%s", port_analysis);
+		}
+
+	if ( ! packet_filter.empty() )
+		{
+		WriteSectionHeading("Packet Filter", '-');
+		WriteToDoc(":ref:`More Information <common_packet_filter_doc>`\n\n");
+		WriteToDoc("Filters added::\n\n");
+		WriteToDoc("%s\n", packet_filter.c_str());
+		}
+
+	BroDocObjList::const_iterator it;
+	bool hasPrivateIdentifiers = false;
+
+	for ( it = all.begin(); it != all.end(); ++it )
+		{
+		if ( ! IsPublicAPI(*it) )
+			{
+			hasPrivateIdentifiers = true;
+			break;
+			}
+		}
+
+	if ( hasPrivateIdentifiers )
+		WriteInterface("Private Interface", '-', '~', false, false);
+	}
+
+void BroDoc::WriteInterface(const char* heading, char underline,
+			char sub, bool isPublic, bool isShort) const
+	{
+	WriteSectionHeading(heading, underline);
+	WriteBroDocObjList(options, isPublic, "Options", sub, isShort);
+	WriteBroDocObjList(constants, isPublic, "Constants", sub, isShort);
+	WriteBroDocObjList(state_vars, isPublic, "State Variables", sub, isShort);
+	WriteBroDocObjList(types, isPublic, "Types", sub, isShort);
+	WriteBroDocObjList(events, isPublic, "Events", sub, isShort);
+	WriteBroDocObjList(functions, isPublic, "Functions", sub, isShort);
+	WriteBroDocObjList(redefs, isPublic, "Redefinitions", sub, isShort);
+	}
+
+void BroDoc::WriteStringList(const char* format, const char* last_format,
+			const std::list<std::string>& l) const
+	{
+	if ( l.empty() )
+		{
+		WriteToDoc("\n");
+		return;
+		}
+
+	std::list<std::string>::const_iterator it;
+	std::list<std::string>::const_iterator last = l.end();
+	last--;
+
+	for ( it = l.begin(); it != last; ++it )
+		WriteToDoc(format, it->c_str());
+
+	WriteToDoc(last_format, last->c_str());
+	}
+
+void BroDoc::WriteBroDocObjTable(const BroDocObjList& l) const
+	{
+	int max_id_col = 0;
+	int max_com_col = 0;
+	BroDocObjList::const_iterator it;
+
+	for ( it = l.begin(); it != l.end(); ++it )
+		{
+		int c = (*it)->ColumnSize();
+
+		if ( c > max_id_col )
+			max_id_col = c;
+
+		c = (*it)->LongestShortDescLen();
+
+		if ( c > max_com_col )
+			max_com_col = c;
+		}
+
+	// Start table.
+	WriteRepeatedChar('=', max_id_col);
+	WriteToDoc(" ");
+
+	if ( max_com_col == 0 )
+		WriteToDoc("=");
+	else
+		WriteRepeatedChar('=', max_com_col);
+
+	WriteToDoc("\n");
+
+	for ( it = l.begin(); it != l.end(); ++it )
+		{
+		if ( it != l.begin() )
+			WriteToDoc("\n\n");
+		(*it)->WriteReSTCompact(reST_file, max_id_col);
+		}
+
+	// End table.
+	WriteToDoc("\n");
+	WriteRepeatedChar('=', max_id_col);
+	WriteToDoc(" ");
+
+	if ( max_com_col == 0 )
+		WriteToDoc("=");
+	else
+		WriteRepeatedChar('=', max_com_col);
+
+	WriteToDoc("\n\n");
+	}
+
+void BroDoc::WriteBroDocObjList(const BroDocObjList& l, bool wantPublic,
+			const char* heading, char underline, bool isShort) const
+	{
+	if ( l.empty() )
+		return;
+
+	BroDocObjList::const_iterator it;
+	bool (*f_ptr)(const BroDocObj* o) = 0;
+
+	if ( wantPublic )
+		f_ptr = IsPublicAPI;
+	else
+		f_ptr = IsPrivateAPI;
+
+	it = std::find_if(l.begin(), l.end(), f_ptr);
+
+	if ( it == l.end() )
+		return;
+
+	WriteSectionHeading(heading, underline);
+
+	BroDocObjList filtered_list;
+
+	while ( it != l.end() )
+		{
+		filtered_list.push_back(*it);
+		it = find_if(++it, l.end(), f_ptr);
+		}
+
+	if ( isShort )
+		WriteBroDocObjTable(filtered_list);
+	else
+		WriteBroDocObjList(filtered_list);
+	}
+
+void BroDoc::WriteBroDocObjList(const BroDocObjMap& m, bool wantPublic,
+			const char* heading, char underline, bool isShort) const
+	{
+	BroDocObjMap::const_iterator it;
+	BroDocObjList l;
+
+	for ( it = m.begin(); it != m.end(); ++it )
+		l.push_back(it->second);
+
+	WriteBroDocObjList(l, wantPublic, heading, underline, isShort);
+	}
+
+void BroDoc::WriteBroDocObjList(const BroDocObjList& l, const char* heading,
+			char underline) const
+	{
+	WriteSectionHeading(heading, underline);
+	WriteBroDocObjList(l);
+	}
+
+void BroDoc::WriteBroDocObjList(const BroDocObjList& l) const
+	{
+	for ( BroDocObjList::const_iterator it = l.begin(); it != l.end(); ++it )
+		(*it)->WriteReST(reST_file);
+	}
+
+void BroDoc::WriteBroDocObjList(const BroDocObjMap& m, const char* heading,
+			char underline) const
+	{
+	BroDocObjMap::const_iterator it;
+	BroDocObjList l;
+
+	for ( it = m.begin(); it != m.end(); ++it )
+		l.push_back(it->second);
+
+	WriteBroDocObjList(l, heading, underline);
+	}
+
+void BroDoc::WriteToDoc(const char* format, ...) const
+	{
+	va_list argp;
+	va_start(argp, format);
+	vfprintf(reST_file, format, argp);
+	va_end(argp);
+	}
+
+void BroDoc::WriteSectionHeading(const char* heading, char underline) const
+	{
+	WriteToDoc("%s\n", heading);
+	WriteRepeatedChar(underline, strlen(heading));
+	WriteToDoc("\n");
+	}
+
+void BroDoc::WriteRepeatedChar(char c, size_t n) const
+	{
+	for ( size_t i = 0; i < n; ++i )
+		WriteToDoc("%c", c);
+	}
+
+void BroDoc::FreeBroDocObjPtrList(BroDocObjList& l)
+	{
+	for ( BroDocObjList::const_iterator it = l.begin(); it != l.end(); ++it )
+		delete *it;
+
+	l.clear();
+	}
+
+void BroDoc::AddFunction(BroDocObj* o)
+	{
+	BroDocObjMap::const_iterator it = functions.find(o->Name());
+	if ( it == functions.end() )
+		{
+		functions[o->Name()] = o;
+		all.push_back(o);
+		}
+	else
+		functions[o->Name()]->Combine(o);
+	}

src/BroDoc.h

@@ -0,0 +1,362 @@
+#ifndef brodoc_h
+#define brodoc_h
+
+#include <cstdio>
+#include <cstdarg>
+#include <string>
+#include <list>
+
+#include "BroDocObj.h"
+
+/**
+ * This class is used to gather all data relevant to the automatic generation
+ * of a reStructuredText (reST) document from a given Bro script.
+ */
+class BroDoc {
+public:
+	/**
+	 * BroDoc constructor
+	 * Given a Bro script, opens new file in the current working directory
+	 * that will contain reST documentation generated from the parsing
+	 * of the Bro script.  The new reST file will be named similar to
+	 * the filename of the Bro script that generates it, except any
+	 * ".bro" file extension is stripped and ".rst" takes it place.
+	 * If the filename doesn't end in ".bro", then ".rst" is just appended.
+	 * @param sourcename The name of the Bro script for which to generate
+	 *        documentation.  May contain a path.
+	 */
+	BroDoc(const std::string& sourcename);
+
+	/**
+	 * BroDoc destructor
+	 * Closes the file that was opened by the constructor and frees up
+	 * memory taken by BroDocObj objects.
+	 */
+	virtual ~BroDoc();
+
+	/**
+	 * Write out full reST documentation for the Bro script that was parsed.
+	 * BroDoc's default implementation of this function will care
+	 * about whether declarations made in the Bro script are part of
+	 * the public versus private interface (whether things are declared in
+	 * the export section).
+	 */
+	virtual void WriteDocFile() const;
+
+	/**
+	 * Schedules some summarizing text to be output directly into the reST doc.
+	 * This should be called whenever the scanner sees a line in the Bro script
+	 * starting with "##!"
+	 * @param s The summary text to add to the reST doc.
+	 */
+	void AddSummary(const std::string& s)	{ summary.push_back(s); }
+
+	/**
+	 * Schedules an import (@load) to be documented.
+	 * If the script being loaded has a .bro suffix, it is internally stripped.
+	 * This should be called whenever the scanner sees an @load.
+	 * @param s The name of the imported script.
+	 */
+	void AddImport(const std::string& s);
+
+	/**
+	 * Schedules a namespace (module) to be documented.
+	 * This should be called whenever the parser sees a TOK_MODULE.
+	 * @param s The namespace (module) identifier's name.
+	 */
+	void AddModule(const std::string& s)	{ modules.push_back(s); }
+
+	/**
+	 * Sets the way the script changes the "capture_filters" table.
+	 * This is determined by the scanner checking for changes to
+	 * the "capture_filters" table after each of Bro's input scripts
+	 * (given as command line arguments to Bro) are finished being parsed.
+	 * @param s The value "capture_filters" as given by TableVal::Describe()
+	 */
+	void SetPacketFilter(const std::string& s);
+
+	/**
+	 * Schedules documentation of a given set of ports being associated
+	 * with a particular analyzer as a result of the current script
+	 * being loaded -- the way the "dpd_config" table is changed.
+	 * @param analyzer An analyzer that changed the "dpd_config" table.
+	 * @param ports The set of ports assigned to the analyzer in table.
+	 */
+	void AddPortAnalysis(const std::string& analyzer, const std::string& ports);
+
+	/**
+	 * Schedules documentation of a script option.  An option is
+	 * defined as any variable in the script that is declared 'const'
+	 * and has the '&redef' attribute.
+	 * @param o A pointer to a BroDocObj which contains the internal
+	 *        Bro language representation of the script option and
+	 *        also any associated comments about it.
+	 */
+	void AddOption(const BroDocObj* o)
+		{
+		options.push_back(o);
+		all.push_back(o);
+		}
+
+	/**
+	 * Schedules documentation of a script constant.  An option is
+	 * defined as any variable in the script that is declared 'const'
+	 * and does *not* have the '&redef' attribute.
+	 * @param o A pointer to a BroDocObj which contains the internal
+	 *        Bro language representation of the script constant and
+	 *        also any associated comments about it.
+	 */
+	void AddConstant(const BroDocObj* o)
+		{
+		constants.push_back(o);
+		all.push_back(o);
+		}
+
+	/**
+	 * Schedules documentation of a script state variable.  A state variable
+	 * is defined as any variable in the script that is declared 'global'
+	 * @param o A pointer to a BroDocObj which contains the internal
+	 *        Bro language representation of the script state variable
+	 *        and also any associated comments about it.
+	 */
+	void AddStateVar(const BroDocObj* o)
+		{
+		state_vars.push_back(o);
+		all.push_back(o);
+		}
+
+	/**
+	 * Schedules documentation of a type declared by the script.
+	 * @param o A pointer to a BroDocObj which contains the internal
+	 *        Bro language representation of the script option and
+	 *        also any associated comments about it.
+	 */
+	void AddType(const BroDocObj* o)
+		{
+		types.push_back(o);
+		all.push_back(o);
+		}
+
+	/**
+	 * Schedules documentation of a Notice (enum redef) declared by script
+	 * @param o A pointer to a BroDocObj which contains the internal
+	 *        Bro language representation of the Notice and also
+	 *        any associated comments about it.
+	 */
+	void AddNotice(const BroDocObj* o)
+		{
+		notices.push_back(o);
+		all.push_back(o);
+		}
+
+	/**
+	 * Schedules documentation of an event declared by the script.
+	 * @param o A pointer to a BroDocObj which contains the internal
+	 *        Bro language representation of the script event and
+	 *        also any associated comments about it.
+	 */
+	void AddEvent(const BroDocObj* o)
+		{
+		events.push_back(o);
+		all.push_back(o);
+		}
+
+	/**
+	 * Schedules documentation of a function declared by the script.
+	 * @param o A pointer to a BroDocObj which contains the internal
+	 *        Bro language representation of the script function and
+	 *        also any associated comments about it.
+	 */
+	void AddFunction(BroDocObj* o);
+
+	/**
+	 * Schedules documentation of a redef done by the script
+	 * @param o A pointer to a BroDocObj which contains the internal
+	 *        Bro language representation of the script identifier
+	 *        that was redefined and also any associated comments.
+	 */
+	void AddRedef(const BroDocObj* o)
+		{
+		redefs.push_back(o);
+		all.push_back(o);
+		}
+
+	/**
+	 * Gets the name of the Bro script source file for which reST
+	 * documentation is being generated.
+	 * @return A char* to the start of the source file's name.
+	 */
+	const char* GetSourceFileName() const
+		{
+		return source_filename.c_str();
+		}
+
+	/**
+	 * Gets the name of the generated reST documentation file.
+	 * @return A char* to the start of the generated reST file's name.
+	 */
+	const char* GetOutputFileName() const
+		{
+		return reST_filename.c_str();
+		}
+
+protected:
+	FILE* reST_file;
+	std::string reST_filename;
+	std::string source_filename;
+	std::string packet_filter;
+
+	std::list<std::string> modules;
+	std::list<std::string> summary;
+	std::list<std::string> imports;
+	std::list<std::string> port_analysis;
+
+	typedef std::list<const BroDocObj*> BroDocObjList;
+	typedef std::map<std::string, BroDocObj*> BroDocObjMap;
+
+	BroDocObjList options;
+	BroDocObjList constants;
+	BroDocObjList state_vars;
+	BroDocObjList types;
+	BroDocObjList notices;
+	BroDocObjList events;
+	BroDocObjMap functions;
+	BroDocObjList redefs;
+
+	BroDocObjList all;
+
+	/**
+	 * Writes out a list of strings to the reST document.
+	 * If the list is empty, prints a newline character.
+	 * @param format A printf style format string for elements of the list
+	 *        except for the last one in the list
+	 * @param last_format A printf style format string to use for the last
+	 *        element of the list
+	 * @param l A reference to a list of strings
+	 */
+	void WriteStringList(const char* format, const char* last_format,
+			const std::list<std::string>& l) const;
+
+	/**
+	 * @see WriteStringList(const char*, const char*,
+	 *                      const std::list<std::string>&>)
+	 */
+	void WriteStringList(const char* format,
+			const std::list<std::string>& l) const
+		{
+		WriteStringList(format, format, l);
+		}
+
+
+	/**
+	 * Writes out a table of BroDocObj's to the reST document
+	 * @param l A list of BroDocObj pointers
+	 */
+	void WriteBroDocObjTable(const BroDocObjList& l) const;
+
+	/**
+	 * Writes out a list of BroDocObj objects to the reST document
+	 * @param l A list of BroDocObj pointers
+	 * @param wantPublic If true, filter out objects that are not declared
+	 *        in the global scope.  If false, filter out those that are in
+	 *        the global scope.
+	 * @param heading The title of the section to create in the reST doc.
+	 * @param underline The character to use to underline the reST
+	 *        section heading.
+	 * @param isShort Whether to write the full documentation or a "short"
+	 *        version (a single sentence)
+	 */
+	void WriteBroDocObjList(const BroDocObjList& l, bool wantPublic,
+			const char* heading, char underline,
+			bool isShort) const;
+
+	/**
+	 * Wraps the BroDocObjMap into a BroDocObjList and the writes that list
+	 * to the reST document
+	 * @see WriteBroDocObjList(const BroDocObjList&, bool, const char*, char,
+	        bool)
+	 */
+	void WriteBroDocObjList(const BroDocObjMap& m, bool wantPublic,
+			const char* heading, char underline,
+			bool isShort) const;
+
+	/**
+	 * Writes out a list of BroDocObj objects to the reST document
+	 * @param l A list of BroDocObj pointers
+	 * @param heading The title of the section to create in the reST doc.
+	 * @param underline The character to use to underline the reST
+	 *        section heading.
+	 */
+	void WriteBroDocObjList(const BroDocObjList& l, const char* heading,
+			char underline) const;
+
+	/**
+	 * Writes out a list of BroDocObj objects to the reST document
+	 * @param l A list of BroDocObj pointers
+	 */
+	void WriteBroDocObjList(const BroDocObjList& l) const;
+
+	/**
+	 * Wraps the BroDocObjMap into a BroDocObjList and the writes that list
+	 * to the reST document
+	 * @see WriteBroDocObjList(const BroDocObjList&, const char*, char)
+	 */
+	void WriteBroDocObjList(const BroDocObjMap& m, const char* heading,
+			char underline) const;
+
+	/**
+	 * A wrapper to fprintf() that always uses the reST document
+	 * for the FILE* argument.
+	 * @param format A printf style format string.
+	 */
+	void WriteToDoc(const char* format, ...) const;
+
+	/**
+	 * Writes out a reST section heading
+	 * @param heading The title of the heading to create
+	 * @param underline The character to use to underline the section title
+	 *        within the reST document
+	 */
+	void WriteSectionHeading(const char* heading, char underline) const;
+
+	/**
+	 * Writes out given number of characters to reST document
+	 * @param c the character to write
+	 * @param n the number of characters to write
+	 */
+	void WriteRepeatedChar(char c, size_t n) const;
+
+	/**
+	 * Writes out the reST for either the script's public or private interface
+	 * @param heading The title of the interfaces section heading
+	 * @param underline The underline character to use for the interface
+	 *        section
+	 * @param subunderline The underline character to use for interface
+	 *        sub-sections
+	 * @param isPublic Whether to write out the public or private script
+	 *        interface
+	 * @param isShort Whether to write out the full documentation or a "short"
+	 *        description (a single sentence)
+	 */
+	void WriteInterface(const char* heading, char underline, char subunderline,
+			bool isPublic, bool isShort) const;
+private:
+
+	/**
+	 * Frees memory allocated to BroDocObj's objects in a given list.
+	 * @param a reference to a list of BroDocObj pointers
+	 */
+	void FreeBroDocObjPtrList(BroDocObjList& l);
+
+	static bool IsPublicAPI(const BroDocObj* o)
+		{
+		return o->IsPublicAPI();
+		}
+
+	static bool IsPrivateAPI(const BroDocObj* o)
+		{
+		return ! o->IsPublicAPI();
+		}
+};
+
+#endif

src/BroDocObj.cc

@@ -0,0 +1,168 @@
+#include <cstdio>
+#include <string>
+#include <list>
+#include "ID.h"
+#include "BroDocObj.h"
+
+BroDocObj::BroDocObj(const ID* id, std::list<std::string>*& reST,
+			bool is_fake)
+	{
+	broID = id;
+	reST_doc_strings = reST;
+	reST = 0;
+	is_fake_id = is_fake;
+	use_role = 0;
+	FormulateShortDesc();
+	}
+
+BroDocObj::~BroDocObj()
+	{
+	if ( reST_doc_strings )
+		delete reST_doc_strings;
+
+	if ( is_fake_id )
+		delete broID;
+	}
+
+void BroDocObj::WriteReSTCompact(FILE* file, int max_col) const
+	{
+	ODesc desc;
+	desc.SetQuotes(1);
+	broID->DescribeReSTShort(&desc);
+
+	fprintf(file, "%s", desc.Description());
+
+	std::list<std::string>::const_iterator it;
+
+	for ( it = short_desc.begin(); it != short_desc.end(); ++it )
+		{
+		int start_col;
+
+		if ( it == short_desc.begin() )
+			start_col = max_col - desc.Len() + 1;
+		else
+			{
+			start_col = max_col + 1;
+			fprintf(file, "\n");
+			}
+
+		for ( int i = 0; i < start_col; ++i )
+			fprintf(file, " ");
+
+		fprintf(file, "%s", it->c_str());
+		}
+	}
+
+int BroDocObj::LongestShortDescLen() const
+	{
+	size_t max = 0;
+
+	std::list<std::string>::const_iterator it;
+
+	for ( it = short_desc.begin(); it != short_desc.end(); ++it )
+		{
+		if ( it->size() > max )
+			max = it->size();
+		}
+
+	return max;
+	}
+
+void BroDocObj::FormulateShortDesc()
+	{
+	if ( ! reST_doc_strings )
+		return;
+
+	short_desc.clear();
+	std::list<std::string>::const_iterator it;
+
+	for ( it = reST_doc_strings->begin();
+		it != reST_doc_strings->end(); ++it )
+		{
+		// The short description stops at the first sentence or the
+		// first empty comment.
+		size_t end = it->find_first_of(".");
+
+		if ( end == string::npos )
+			{
+			std::string::const_iterator s;
+			bool empty = true;
+
+			for ( s = it->begin(); s != it->end(); ++s )
+				{
+				if ( *s != ' ' && *s != '\t' && *s != '\n' && *s != '\r' )
+					{
+					empty = false;
+					short_desc.push_back(*it);
+					break;
+					}
+				}
+			
+			if ( empty )
+				break;
+			}
+		else
+			{
+			short_desc.push_back(it->substr(0, end + 1));
+			break;
+			}
+		}
+	}
+
+void BroDocObj::WriteReST(FILE* file) const
+	{
+	int indent_spaces = 3;
+	ODesc desc;
+	desc.SetIndentSpaces(indent_spaces);
+	desc.SetQuotes(1);
+
+	broID->DescribeReST(&desc, use_role);
+
+	fprintf(file, "%s", desc.Description());
+
+	if ( HasDocumentation() )
+		{
+		fprintf(file, "\n");
+		std::list<std::string>::const_iterator it;
+
+		for ( it = reST_doc_strings->begin();
+			it != reST_doc_strings->end(); ++it)
+			{
+			for ( int i = 0; i < indent_spaces; ++i )
+				fprintf(file, " ");
+
+			fprintf(file, "%s\n", it->c_str());
+			}
+		}
+
+	fprintf(file, "\n");
+	}
+
+int BroDocObj::ColumnSize() const
+	{
+	ODesc desc;
+	desc.SetQuotes(1);
+	broID->DescribeReSTShort(&desc);
+	return desc.Len();
+	}
+
+bool BroDocObj::IsPublicAPI() const
+	{
+	return (broID->Scope() == SCOPE_GLOBAL) ||
+		(broID->Scope() == SCOPE_MODULE && broID->IsExport());
+	}
+
+void BroDocObj::Combine(const BroDocObj* o)
+	{
+	if ( o->reST_doc_strings )
+		{
+		if ( ! reST_doc_strings )
+			reST_doc_strings = new std::list<std::string>();
+
+		reST_doc_strings->splice(reST_doc_strings->end(),
+			*(o->reST_doc_strings));
+		}
+
+	delete o;
+	FormulateShortDesc();
+	}

src/BroDocObj.h

@@ -0,0 +1,123 @@
+#ifndef brodocobj_h
+#define brodocobj_h
+
+#include <cstdio>
+#include <string>
+#include <list>
+
+#include "ID.h"
+
+/**
+ * This class wraps a Bro script identifier, providing methods relevant
+ * to automatic generation of reStructuredText (reST) documentation for it.
+ */
+class BroDocObj {
+public:
+	/**
+	 * BroDocObj constructor
+	 * @param id a pointer to an identifier that is to be documented
+	 * @param reST a reference to a pointer of a list of strings that
+	 *        represent the reST documentation for the ID.  The pointer
+	 *        will be set to 0 after this constructor finishes.
+	 * @param is_fake whether the ID* is a dummy just for doc purposes
+	 */
+	BroDocObj(const ID* id, std::list<std::string>*& reST,
+			bool is_fake = false);
+
+	/**
+	 * BroDocObj destructor
+	 * Deallocates the memory associated with the list of reST strings
+	 */
+	~BroDocObj();
+
+	/**
+	 * Writes the reST representation of this object which includes
+	 * 1) a reST friendly description of the ID
+	 * 2) "##" or "##<" stylized comments.
+	 *    Anything after these style of comments is inserted as-is into
+	 *    the reST document.
+	 * @param file The (already opened) file to write the reST to.
+	 */
+	void WriteReST(FILE* file) const;
+
+	/**
+	 * Writes a compact version of the ID and associated documentation
+	 * for insertion into a table.
+	 * @param file The (already opened) file to write the reST to.
+	 * @param max_col The maximum length of the first table column
+	 */
+	void WriteReSTCompact(FILE* file, int max_col) const;
+
+	/**
+	 * @return the column size required by the reST representation of the ID
+	 */
+	int ColumnSize() const;
+
+	/**
+	 * Check whether this documentation is part of the public API.  In
+	 * other words, this means that the identifier is declared as part of
+	 * the global scope (has GLOBAL namespace or is exported from another
+	 * namespace).
+	 * @return true if the identifier is part of the script's public API
+	 */
+	bool IsPublicAPI() const;
+
+	/**
+	 * Return whether this object has documentation (## comments)
+	 * @return true if the ID has comments associated with it
+	 */
+	bool HasDocumentation() const
+		{
+		return reST_doc_strings && reST_doc_strings->size() > 0;
+		}
+
+	/**
+	 * @return whether this object will use reST role (T) or directive (F)
+	 * notation for the wrapped identifier.  Roles are usually used
+	 * for cross-referencing.
+	 */
+	bool UseRole() const	{ return use_role; }
+
+	/**
+	 * @param b whether this object will use reST role (T) or directive (F)
+	 * notation for the wrapped identifier.  Roles are usually used
+	 * for cross-referencing.
+	 */
+	void SetRole(bool b)	{ use_role = b; }
+
+	/**
+	 * Append any reST documentation strings in a given BroDocObj to this
+	 * object's list and then delete the given BroDocObj
+	 * @param o a pointer to a BroDocObj to subsume
+	 */
+	void Combine(const BroDocObj* o);
+
+	/**
+	 * @return the name of the wrapped identifier
+	 */
+	const char* Name() const	{ return broID->Name(); }
+
+	/**
+	 * @return the longest string element of the short description's list of
+	 *         strings
+	 */
+	int LongestShortDescLen() const;
+
+protected:
+	std::list<std::string>* reST_doc_strings;
+	std::list<std::string> short_desc;
+	const ID* broID;
+	bool is_fake_id; /**< Whether the ID* is a dummy just for doc purposes */
+	bool use_role; /**< Whether to use a reST role or directive for the ID */
+
+	/**
+	 * Set the short_desc member to be a subset of reST_doc_strings.
+	 * Specifically, short_desc will be everything in reST_doc_strings
+	 * up until the first period or first empty string list element found.
+	 */
+	void FormulateShortDesc();
+
+private:
+};
+
+#endif

src/CMakeLists.txt

@@ -81,9 +81,14 @@ flex_target(Scanner scan.l ${CMAKE_CURRENT_BINARY_DIR}/scan.cc
 ## bifcl (BIF compiler) target
 
 set(bifcl_SRCS
+   ${BISON_BIFParser_INPUT}
+   ${FLEX_BIFScanner_INPUT}
    ${BISON_BIFParser_OUTPUTS}
    ${FLEX_BIFScanner_OUTPUTS}
-   bif_arg.cc module_util.cc
+   bif_arg.cc
+   module_util.cc
+   bif_arg.h
+   module_util.h
 )
 
 add_executable(bifcl ${bifcl_SRCS})
@@ -127,18 +132,11 @@ endmacro(GET_BIF_OUTPUT_FILES)
 
 set(BIF_SRCS
     bro.bif
+    logging.bif
     event.bif
     const.bif
     types.bif
-    common-rw.bif
-    finger-rw.bif
-    ident-rw.bif
-    dns-rw.bif
-    ftp-rw.bif
-    smtp-rw.bif
-    http-rw.bif
     strings.bif
-    smb-rw.bif
 )
 
 foreach (bift ${BIF_SRCS})
@@ -157,6 +155,7 @@ set(BINPAC_AUXSRC
 # A macro to define a command that uses the BinPac compiler to
 # produce C++ code that implements a protocol parser/analyzer
 # The outputs of the command are appended to list ALL_BINPAC_OUTPUTS
+# All arguments to this macro are appended to list ALL_BINPAC_INPUTS
 macro(BINPAC_TARGET pacFile)
     get_filename_component(basename ${pacFile} NAME_WE)
     add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${basename}_pac.h
@@ -169,6 +168,7 @@ macro(BINPAC_TARGET pacFile)
                                ${BINPAC_AUXSRC} ${ARGN}
                        COMMENT "[BINPAC] Processing ${pacFile}"
     )
+    list(APPEND ALL_BINPAC_INPUTS ${ARGV})
     list(APPEND ALL_BINPAC_OUTPUTS
          ${CMAKE_CURRENT_BINARY_DIR}/${basename}_pac.h
          ${CMAKE_CURRENT_BINARY_DIR}/${basename}_pac.cc) 
@@ -203,6 +203,25 @@ binpac_target(ssl.pac
 ########################################################################
 ## bro target
 
+# This macro stores associated headers for any C/C++ source files given
+# as arguments (past _var) as a list in the CMake variable named "_var".
+macro(COLLECT_HEADERS _var)
+    foreach (src ${ARGN})
+        get_filename_component(ext ${src} EXT)
+        if (${ext} STREQUAL ".cc" OR ${ext} STREQUAL ".c")
+            get_filename_component(base ${src} NAME_WE)
+            get_filename_component(dir ${src} PATH)
+            if (NOT "${dir}")
+                set(dir ${CMAKE_CURRENT_SOURCE_DIR})
+            endif ()
+            set(header "${dir}/${base}.h")
+            if (EXISTS ${header})
+                list(APPEND ${_var} ${header})
+            endif ()
+        endif ()
+    endforeach ()
+endmacro(COLLECT_HEADERS _var)
+
 # define a command that's used to run the make_dbg_constants.pl script
 # building the bro binary depends on the outputs of this script
 add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/DebugCmdConstants.h
@@ -216,33 +235,47 @@ add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/DebugCmdConstants.h
                    WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
 )
 
-set(dns_SRCS nb_dns.c nb_dns.h)
+set(dns_SRCS nb_dns.c)
 set_source_files_properties(nb_dns.c PROPERTIES COMPILE_FLAGS
                             -fno-strict-aliasing)
 
-#set(openssl_SRCS X509.cc SSLCiphers.cc SSLInterpreter.cc SSLProxy.cc
+#set(openssl_SRCS
-#                 SSLv2.cc SSLv3.cc SSLv3Automaton.cc)
+#    X509.cc
+#    SSLCiphers.cc
+#    SSLInterpreter.cc
+#    SSLProxy.cc
+#    SSLv2.cc
+#    SSLv3.cc
+#    SSLv3Automaton.cc
+#)
 set(openssl_SRCS)
 
-
 if (USE_NMALLOC)
     set(malloc_SRCS malloc.c)
 endif ()
 
 set(bro_SRCS
     ${CMAKE_CURRENT_BINARY_DIR}/version.c
+    ${BIF_SRCS}
     ${ALL_BIF_OUTPUTS}
+    ${BINPAC_AUXSRC}
+    ${ALL_BINPAC_INPUTS}
     ${ALL_BINPAC_OUTPUTS}
     ${TRANSFORMED_BISON_OUTPUTS}
     ${FLEX_RuleScanner_OUTPUTS}
+    ${FLEX_RuleScanner_INPUT}
+    ${BISON_RuleParser_INPUT}
     ${FLEX_REScanner_OUTPUTS}
+    ${FLEX_REScanner_INPUT}
+    ${BISON_REParser_INPUT}
     ${FLEX_Scanner_OUTPUTS}
+    ${FLEX_Scanner_INPUT}
+    ${BISON_Parser_INPUT}
     ${CMAKE_CURRENT_BINARY_DIR}/DebugCmdConstants.h
     main.cc
     net_util.cc
     util.cc
     module_util.cc
-    Active.cc
     Analyzer.cc
     Anon.cc
     ARP.cc
@@ -252,6 +285,9 @@ set(bro_SRCS
     BitTorrent.cc
     BitTorrentTracker.cc
     BPF_Program.cc
+    BroBifDoc.cc
+    BroDoc.cc
+    BroDocObj.cc
     BroString.cc
     CCL.cc
     ChunkedIO.cc
@@ -302,6 +338,9 @@ set(bro_SRCS
     IRC.cc
     List.cc
     Logger.cc
+    LogMgr.cc
+    LogWriter.cc
+    LogWriterAscii.cc
     Login.cc
     MIME.cc
     NCP.cc
@@ -354,12 +393,10 @@ set(bro_SRCS
     TCP.cc
     TCP_Endpoint.cc
     TCP_Reassembler.cc
-    TCP_Rewriter.cc
     Telnet.cc
     Timer.cc
     Traverse.cc
     Trigger.cc
-    TwoWise.cc
     Type.cc
     UDP.cc
     Val.cc
@@ -371,19 +408,18 @@ set(bro_SRCS
     md5.c
     patricia.c
     setsignal.c
-    UDP_Rewriter.cc
-    DNS_Rewriter.cc
     PacketDumper.cc
-    Rewriter.cc
     strsep.c
     ${dns_SRCS}
     ${malloc_SRCS}
     ${openssl_SRCS}
 )
 
+collect_headers(bro_HEADERS ${bro_SRCS})
+
 add_definitions(-DPOLICYDEST="${POLICYDIR}")
 
-add_executable(bro ${bro_SRCS})
+add_executable(bro ${bro_SRCS} ${bro_HEADERS})
 
 set(brolibs
     ${BinPAC_LIBRARY}

src/CompHash.cc

@@ -65,11 +65,22 @@ CompositeHash::~CompositeHash()
 
 // Computes the piece of the hash for Val*, returning the new kp.
 char* CompositeHash::SingleValHash(int type_check, char* kp0,
-					BroType* bt, Val* v) const
+				   BroType* bt, Val* v, bool optional) const
 	{
 	char* kp1 = 0;
 	InternalTypeTag t = bt->InternalType();
 
+	if ( optional )
+		{
+		// Add a marker saying whether the optional field is set.
+		char* kp = AlignAndPadType<char>(kp0);
+		*kp = ( v ? 1 : 0);
+		kp0 = reinterpret_cast<char*>(kp+1);
+
+		if ( ! v ) 
+			return kp0;
+		}
+
 	if ( type_check )
 		{
 		InternalTypeTag vt = v->Type()->InternalType();
@@ -163,12 +174,16 @@ char* CompositeHash::SingleValHash(int type_check, char* kp0,
 			for ( int i = 0; i < num_fields; ++i )
 				{
 				Val* rv_i = rv->Lookup(i);
-				if ( ! rv_i )
+
+				Attributes* a = rt->FieldDecl(i)->attrs;
+				bool optional = (a && a->FindAttr(ATTR_OPTIONAL));
+
+				if ( ! (rv_i || optional) )
 					return 0;
 
 				if ( ! (kp = SingleValHash(type_check, kp,
 							   rt->FieldType(i),
-							   rv_i)) )
+							   rv_i, optional)) )
 					return 0;
 				}
 
@@ -248,7 +263,7 @@ HashKey* CompositeHash::ComputeHash(const Val* v, int type_check) const
 	char* kp = k;
 	loop_over_list(*tl, i)
 		{
-		kp = SingleValHash(type_check, kp, (*tl)[i], (*vl)[i]);
+		kp = SingleValHash(type_check, kp, (*tl)[i], (*vl)[i], false);
 		if ( ! kp )
 			return 0;
 		}
@@ -315,10 +330,13 @@ HashKey* CompositeHash::ComputeSingletonHash(const Val* v, int type_check) const
 	}
 
 int CompositeHash::SingleTypeKeySize(BroType* bt, const Val* v,
-					int type_check, int sz) const
+				     int type_check, int sz, bool optional) const
 	{
 	InternalTypeTag t = bt->InternalType();
 
+	if ( optional )
+		sz = SizeAlign(sz, sizeof(char));
+
 	if ( type_check && v )
 		{
 		InternalTypeTag vt = v->Type()->InternalType();
@@ -369,9 +387,12 @@ int CompositeHash::SingleTypeKeySize(BroType* bt, const Val* v,
 
 			for ( int i = 0; i < num_fields; ++i )
 				{
+				Attributes* a = rt->FieldDecl(i)->attrs;
+				bool optional = (a && a->FindAttr(ATTR_OPTIONAL));
+
 				sz = SingleTypeKeySize(rt->FieldType(i),
-							rv ? rv->Lookup(i) : 0,
+						       rv ? rv->Lookup(i) : 0,
-							type_check, sz);
+						       type_check, sz, optional);
 				if ( ! sz )
 					return 0;
 				}
@@ -386,7 +407,7 @@ int CompositeHash::SingleTypeKeySize(BroType* bt, const Val* v,
 
 	case TYPE_INTERNAL_STRING:
 		if ( ! v )
-			return 0;
+			return optional ? sz : 0;
 
 		// Factor in length field.
 		sz = SizeAlign(sz, sizeof(int));
@@ -418,7 +439,7 @@ int CompositeHash::ComputeKeySize(const Val* v, int type_check) const
 	loop_over_list(*tl, i)
 		{
 		sz = SingleTypeKeySize((*tl)[i], v ? v->AsListVal()->Index(i) : 0,
-				       type_check, sz);
+				       type_check, sz, false);
 		if ( ! sz )
 			return 0;
 		}
@@ -495,20 +516,20 @@ ListVal* CompositeHash::RecoverVals(const HashKey* k) const
 	loop_over_list(*tl, i)
 		{
 		Val* v;
-		kp = RecoverOneVal(k, kp, k_end, (*tl)[i], v);
+		kp = RecoverOneVal(k, kp, k_end, (*tl)[i], v, false);
 		ASSERT(v);
 		l->Append(v);
 		}
 
 	if ( kp != k_end )
-		internal_error("under-ran key in CompositeHash::DescribeKey");
+		internal_error("under-ran key in CompositeHash::DescribeKey %ld", k_end - kp);
 
 	return l;
 	}
 
 const char* CompositeHash::RecoverOneVal(const HashKey* k, const char* kp0,
 					 const char* const k_end, BroType* t,
-					 Val*& pval) const
+					 Val*& pval, bool optional) const
 	{
 	// k->Size() == 0 for a single empty string.
 	if ( kp0 >= k_end && k->Size() > 0 )
@@ -516,9 +537,20 @@ const char* CompositeHash::RecoverOneVal(const HashKey* k, const char* kp0,
 
 	TypeTag tag = t->Tag();
 	InternalTypeTag it = t->InternalType();
-
 	const char* kp1 = 0;
 
+	if ( optional )
+		{
+		const char* kp = AlignType<char>(kp0);
+		kp0 = kp1 = reinterpret_cast<const char*>(kp+1);
+
+		if ( ! *kp )
+			{
+			pval = 0;
+			return kp0;
+			}
+		}
+
 	switch ( it ) {
 	case TYPE_INTERNAL_INT:
 		{
@@ -647,9 +679,13 @@ const char* CompositeHash::RecoverOneVal(const HashKey* k, const char* kp0,
 			for ( i = 0; i < num_fields; ++i )
 				{
 				Val* v;
+
+				Attributes* a = rt->FieldDecl(i)->attrs;
+				bool optional = (a && a->FindAttr(ATTR_OPTIONAL));
+
 				kp = RecoverOneVal(k, kp, k_end,
-				                   rt->FieldType(i), v);
+				                   rt->FieldType(i), v, optional);
-				if ( ! v )
+				if ( ! (v || optional) )
 					{
 					internal_error("didn't recover expected number of fields from HashKey");
 					pval = 0;

src/CompHash.h

@@ -29,8 +29,8 @@ protected:
 
 	// Computes the piece of the hash for Val*, returning the new kp.
 	// Used as a helper for ComputeHash in the non-singleton case.
-	char* SingleValHash(int type_check, char* kp,
+	char* SingleValHash(int type_check, char* kp, BroType* bt, Val* v,
-				BroType* bt, Val* v) const;
+			    bool optional) const;
 
 	// Recovers just one Val of possibly many; called from RecoverVals.
 	// Upon return, pval will point to the recovered Val of type t.
@@ -38,7 +38,7 @@ protected:
 	// upon errors, so there is no return value for invalid input.
 	const char* RecoverOneVal(const HashKey* k,
 				  const char* kp, const char* const k_end,
-				  BroType* t, Val*& pval) const;
+				  BroType* t, Val*& pval, bool optional) const;
 
 	// Rounds the given pointer up to the nearest multiple of the
 	// given size, if not already a multiple.
@@ -77,7 +77,7 @@ protected:
 	int ComputeKeySize(const Val* v = 0, int type_check = 1) const;
 
 	int SingleTypeKeySize(BroType*, const Val*,
-				int type_check, int sz) const;
+			      int type_check, int sz, bool optional) const;
 
 	TypeList* type;
 	char* key;	// space for composite key

src/Conn.cc

@@ -755,16 +755,6 @@ void Connection::FlipRoles()
 		root_analyzer->FlipRoles();
 	}
 
-int Connection::RewritingTrace()
-	{
-	return root_analyzer ? root_analyzer->RewritingTrace() : 0;
-	}
-
-Rewriter* Connection::TraceRewriter() const
-	{
-	return root_analyzer ? root_analyzer->TraceRewriter() : 0;
-	}
-
 unsigned int Connection::MemoryAllocation() const
 	{
 	return padded_sizeof(*this)

src/Conn.h

@@ -23,7 +23,6 @@ class RuleHdrTest;
 class Specific_RE_Matcher;
 class TransportLayerAnalyzer;
 class RuleEndpointState;
-class Rewriter;
 
 typedef enum {
 	NUL_IN_LINE,
@@ -135,17 +134,6 @@ public:
 
 	TransportProto ConnTransport() const { return proto; }
 
-	// If we are rewriting the trace of the connection, then we do
-	// not record original packets. We are rewriting if at least one,
-	// then the analyzer is rewriting.
-	int RewritingTrace();
-
-	// If we are rewriting trace, we need a handle to the rewriter.
-	// Returns 0 if not rewriting.  (Note that if multiple analyzers
-	// want to rewrite, only one of them is returned.  It's undefined
-	// which one.)
-	Rewriter* TraceRewriter() const;
-
 	// True if we should record subsequent packets (either headers or
 	// in their entirety, depending on record_contents).  We still
 	// record subsequent SYN/FIN/RST, regardless of how this is set.

src/ConnCompressor.cc

@@ -45,9 +45,6 @@
 // - We don't match signatures on connections which are completely handled
 //   by the compressor. Matching would require significant additional state
 //   w/o being very helpful.
-//
-// - Trace rewriting doesn't work if the compressor is turned on (this is
-//   not a conceptual problem, but simply not implemented).
 
 
 #ifdef DEBUG

src/DFA.cc

@@ -21,11 +21,10 @@ DFA_State::DFA_State(int arg_state_num, const EquivClass* ec,
 	nfa_states = arg_nfa_states;
 	accept = arg_accept;
 	mark = 0;
-	lock = 0;
 
 	SymPartition(ec);
 
-	xtions = new DFA_State_Handle*[num_sym];
+	xtions = new DFA_State*[num_sym];
 
 	for ( int i = 0; i < num_sym; ++i )
 		xtions[i] = DFA_UNCOMPUTED_STATE_PTR;
@@ -33,31 +32,14 @@ DFA_State::DFA_State(int arg_state_num, const EquivClass* ec,
 
 DFA_State::~DFA_State()
 	{
-	for ( int i = 0; i < num_sym; ++i )
-		{
-		DFA_State_Handle* s = xtions[i];
-		if ( s && s != DFA_UNCOMPUTED_STATE_PTR )
-			StateUnref(s);
-		}
-
 	delete [] xtions;
 	delete nfa_states;
 	delete accept;
 	delete meta_ec;
 	}
 
-void DFA_State::AddXtion(int sym, DFA_State_Handle* next_state)
+void DFA_State::AddXtion(int sym, DFA_State* next_state)
 	{
-	// The order is important here: first StateRef() the new,
-	// then StateUnref() the old.  Otherwise, we may get a problem
-	// if both are equal.
-
-	if ( next_state )
-		StateRef(next_state);
-
-	if ( xtions[sym] && xtions[sym] != DFA_UNCOMPUTED_STATE_PTR )
-		StateUnref(xtions[sym]);
-
 	xtions[sym] = next_state;
 	}
 
@@ -94,14 +76,10 @@ void DFA_State::SymPartition(const EquivClass* ec)
 	meta_ec->BuildECs();
 	}
 
-DFA_State_Handle* DFA_State::ComputeXtion(int sym, DFA_Machine* machine)
+DFA_State* DFA_State::ComputeXtion(int sym, DFA_Machine* machine)
 	{
-	// Make sure we will not expire...
-	assert(IsLocked());
-
 	int equiv_sym = meta_ec->EquivRep(sym);
-	if ( xtions[equiv_sym] != DFA_UNCOMPUTED_STATE_PTR &&
+	if ( xtions[equiv_sym] != DFA_UNCOMPUTED_STATE_PTR )
-	     StateIsValid(xtions[equiv_sym]) )
 		{
 		AddXtion(sym, xtions[equiv_sym]);
 		return xtions[sym];
@@ -109,7 +87,7 @@ DFA_State_Handle* DFA_State::ComputeXtion(int sym, DFA_Machine* machine)
 
 	const EquivClass* ec = machine->EC();
 
-	DFA_State_Handle* next_d;
+	DFA_State* next_d;
 
 	NFA_state_list* ns = SymFollowSet(equiv_sym, ec);
 	if ( ns->length() > 0 )
@@ -211,10 +189,10 @@ void DFA_State::ClearMarks()
 
 		for ( int i = 0; i < num_sym; ++i )
 			{
-			DFA_State_Handle* s = xtions[i];
+			DFA_State* s = xtions[i];
 
 			if ( s && s != DFA_UNCOMPUTED_STATE_PTR )
-				(*xtions[i])->ClearMarks();
+				xtions[i]->ClearMarks();
 			}
 		}
 	}
@@ -243,7 +221,7 @@ void DFA_State::Dump(FILE* f, DFA_Machine* m)
 	int num_trans = 0;
 	for ( int sym = 0; sym < num_sym; ++sym )
 		{
-		DFA_State_Handle* s = xtions[sym];
+		DFA_State* s = xtions[sym];
 
 		if ( ! s )
 			continue;
@@ -271,7 +249,7 @@ void DFA_State::Dump(FILE* f, DFA_Machine* m)
 		else
 			fprintf(f, "%stransition on %s to state %d",
 				++num_trans == 1 ? "\t" : "\n\t", xbuf,
-				(*s)->StateNum());
+				s->StateNum());
 
 		sym = i - 1;
 		}
@@ -283,10 +261,10 @@ void DFA_State::Dump(FILE* f, DFA_Machine* m)
 
 	for ( int sym = 0; sym < num_sym; ++sym )
 		{
-		DFA_State_Handle* s = xtions[sym];
+		DFA_State* s = xtions[sym];
 
 		if ( s && s != DFA_UNCOMPUTED_STATE_PTR )
-			(*s)->Dump(f, m);
+			s->Dump(f, m);
 		}
 	}
 
@@ -294,7 +272,7 @@ void DFA_State::Stats(unsigned int* computed, unsigned int* uncomputed)
 	{
 	for ( int sym = 0; sym < num_sym; ++sym )
 		{
-		DFA_State_Handle* s = xtions[sym];
+		DFA_State* s = xtions[sym];
 
 		if ( s == DFA_UNCOMPUTED_STATE_PTR )
 			(*uncomputed)++;
@@ -313,11 +291,9 @@ unsigned int DFA_State::Size()
 		+ (centry ? padded_sizeof(CacheEntry) : 0);
 	}
 
-
 DFA_State_Cache::DFA_State_Cache(int arg_maxsize)
 	{
 	maxsize = arg_maxsize;
-	head = tail = 0;
 	hits = misses = 0;
 	}
 
@@ -328,13 +304,12 @@ DFA_State_Cache::~DFA_State_Cache()
 	while ( (e = (CacheEntry*) states.NextEntry(i)) )
 		{
 		assert(e->state);
-		StateInvalidate(e->state);
 		delete e->hash;
 		delete e;
 		}
 	}
 
-DFA_State_Handle* DFA_State_Cache::Lookup(const NFA_state_list& nfas,
+DFA_State* DFA_State_Cache::Lookup(const NFA_state_list& nfas,
 						HashKey** hash)
 	{
 	// We assume that state ID's don't exceed 10 digits, plus
@@ -380,100 +355,24 @@ DFA_State_Handle* DFA_State_Cache::Lookup(const NFA_state_list& nfas,
 	delete *hash;
 	*hash = 0;
 
-	MoveToFront(e);
-
 	return e->state;
 	}
 
-DFA_State_Handle* DFA_State_Cache::Insert(DFA_State* state, HashKey* hash)
+DFA_State* DFA_State_Cache::Insert(DFA_State* state, HashKey* hash)
 	{
 	CacheEntry* e;
 
-#ifdef EXPIRE_DFA_STATES
-	if ( states.Length() == maxsize )
-		{
-		// Remove oldest unlocked entry.
-		for ( e = tail; e; e = e->prev )
-			if ( ! (*e->state)->lock )
-				break;
-		if ( e )
-			Remove(e);
-		}
-#endif
-
 	e = new CacheEntry;
 
-#ifdef EXPIRE_DFA_STATES
-	// Insert as head.
-	e->state = new DFA_State_Handle(state);
-	e->state->state->centry = e;
-#else
 	e->state = state;
 	e->state->centry = e;
-#endif
 	e->hash = hash;
-	e->prev = 0;
-	e->next = head;
-	if ( head )
-		head->prev = e;
-	head = e;
-	if ( ! tail )
-		tail = e;
 
 	states.Insert(hash, e);
 
 	return e->state;
 	}
 
-void DFA_State_Cache::Remove(CacheEntry* e)
-	{
-	if ( e == head )
-		{
-		head = e->next;
-		if ( head )
-			head->prev = 0;
-		}
-	else
-		e->prev->next = e->next;
-
-	if ( e == tail )
-		{
-		tail = e->prev;
-		if ( tail )
-			tail->next = 0;
-		}
-	else
-		e->next->prev = e->prev;
-
-	states.Remove(e->hash);
-
-	assert(e->state);
-	StateInvalidate(e->state);
-	delete e->hash;
-	delete e;
-	}
-
-void DFA_State_Cache::MoveToFront(CacheEntry* e)
-	{
-	++hits;
-
-	if ( e->prev )
-		{
-		e->prev->next = e->next;
-
-		if ( e->next )
-			e->next->prev = e->prev;
-		else
-			tail = e->prev;
-
-		e->prev = 0;
-		e->next = head;
-
-		head->prev = e;
-		head = e;
-		}
-	}
-
 void DFA_State_Cache::GetStats(Stats* s)
 	{
 	s->dfa_states = 0;
@@ -490,9 +389,9 @@ void DFA_State_Cache::GetStats(Stats* s)
 	while ( (e = (CacheEntry*) states.NextEntry(i)) )
 		{
 		++s->dfa_states;
-		s->nfa_states += (*e->state)->NFAStateNum();
+		s->nfa_states += e->state->NFAStateNum();
-		(*e->state)->Stats(&s->computed, &s->uncomputed);
+		e->state->Stats(&s->computed, &s->uncomputed);
-		s->mem += pad_size((*e->state)->Size()) + padded_sizeof(*e->state);
+		s->mem += pad_size(e->state->Size()) + padded_sizeof(*e->state);
 		}
 	}
 
@@ -514,9 +413,6 @@ DFA_Machine::DFA_Machine(NFA_Machine* n, EquivClass* arg_ec)
 		{
 		NFA_state_list* state_set = epsilon_closure(ns);
 		(void) StateSetToDFA_State(state_set, start_state, ec);
-
-		StateRef(start_state);
-		StateLock(start_state);
 		}
 	else
 		start_state = 0; // Jam
@@ -524,12 +420,6 @@ DFA_Machine::DFA_Machine(NFA_Machine* n, EquivClass* arg_ec)
 
 DFA_Machine::~DFA_Machine()
 	{
-	if ( start_state )
-		{
-		StateUnlock(start_state);
-		StateUnref(start_state);
-		}
-
 	delete dfa_state_cache;
 	Unref(nfa);
 	}
@@ -541,8 +431,8 @@ void DFA_Machine::Describe(ODesc* d) const
 
 void DFA_Machine::Dump(FILE* f)
 	{
-	(*start_state)->Dump(f, this);
+	start_state->Dump(f, this);
-	(*start_state)->ClearMarks();
+	start_state->ClearMarks();
 	}
 
 void DFA_Machine::DumpStats(FILE* f)
@@ -571,12 +461,11 @@ unsigned int DFA_Machine::MemoryAllocation() const
 	}
 
 int DFA_Machine::StateSetToDFA_State(NFA_state_list* state_set,
-				DFA_State_Handle*& d, const EquivClass* ec)
+				DFA_State*& d, const EquivClass* ec)
 	{
 	HashKey* hash;
 	d = dfa_state_cache->Lookup(*state_set, &hash);
 
-	assert((! d) || StateIsValid(d));
 	if ( d )
 		return 0;
 

src/DFA.h

@@ -8,57 +8,12 @@
 
 #include <assert.h>
 
-// It's possible to use a fixed size cache of computed states for each DFA.
-// If the number of DFA states reaches the given limit, old states are expired
-// on a least-recently-used basis. This may impact the performance significantly
-// if expired states have to be recalculated regularly, but it limits the
-// amount of memory taken by a DFA.
-//
-// Enable by configuring with --with-expire-dfa-states.
-
 class DFA_State;
 
-// The cache marks expired states as invalid.
-#define DFA_INVALID_STATE_PTR ((DFA_State*) -1)
-
 // Transitions to the uncomputed state indicate that we haven't yet
 // computed the state to go to.
 #define DFA_UNCOMPUTED_STATE -2
-#define DFA_UNCOMPUTED_STATE_PTR ((DFA_State_Handle*) DFA_UNCOMPUTED_STATE)
+#define DFA_UNCOMPUTED_STATE_PTR ((DFA_State*) DFA_UNCOMPUTED_STATE)
-
-#ifdef EXPIRE_DFA_STATES
-
-class DFA_State_Handle {
-public:
-	// The reference counting keeps track of this *handle* (not the state).
-	void Ref()	{ assert(state); ++refcount; }
-	void Unref()
-		{
-		if ( --refcount == 0 )
-			delete this;
-		}
-
-	inline void Invalidate();
-	bool IsValid() const		{ return state != DFA_INVALID_STATE_PTR; }
-
-	DFA_State* State() const	{ return state; }
-	DFA_State* operator->() const	{ return state; }
-
-protected:
-	friend class DFA_State_Cache;
-
-	DFA_State_Handle(DFA_State* arg_state)
-		{ state = arg_state; refcount = 1; }
-
-	inline ~DFA_State_Handle();
-
-	DFA_State* state;
-	int refcount;
-};
-
-#else
-typedef DFA_State DFA_State_Handle;
-#endif
 
 #include "NFA.h"
 
@@ -76,9 +31,9 @@ public:
 
 	int StateNum() const		{ return state_num; }
 	int NFAStateNum() const		{ return nfa_states->length(); }
-	void AddXtion(int sym, DFA_State_Handle* next_state);
+	void AddXtion(int sym, DFA_State* next_state);
 
-	inline DFA_State_Handle* Xtion(int sym, DFA_Machine* machine);
+	inline DFA_State* Xtion(int sym, DFA_Machine* machine);
 
 	const AcceptingSet* Accept() const	{ return accept; }
 	void SymPartition(const EquivClass* ec);
@@ -98,43 +53,29 @@ public:
 	void Stats(unsigned int* computed, unsigned int* uncomputed);
 	unsigned int Size();
 
-	// Locking a state will keep it from expiring from a cache.
-	void Lock()	{ ++lock; }
-	void Unlock()	{ --lock; }
-
-#ifdef EXPIRE_DFA_STATES
-	bool IsLocked()	{ return lock != 0; }
-#else
-	bool IsLocked()	{ return true; }
-	DFA_State* operator->(){ return this; }
-#endif
-
 protected:
 	friend class DFA_State_Cache;
 
-	DFA_State_Handle* ComputeXtion(int sym, DFA_Machine* machine);
+	DFA_State* ComputeXtion(int sym, DFA_Machine* machine);
 	void AppendIfNew(int sym, int_list* sym_list);
 
 	int state_num;
 	int num_sym;
 
-	DFA_State_Handle** xtions;
+	DFA_State** xtions;
 
 	AcceptingSet* accept;
 	NFA_state_list* nfa_states;
 	EquivClass* meta_ec;	// which ec's make same transition
 	DFA_State* mark;
-	int lock;
 	CacheEntry* centry;
 
 	static unsigned int transition_counter;	// see Xtion()
 };
 
 struct CacheEntry {
-	DFA_State_Handle* state;
+	DFA_State* state;
 	HashKey* hash;
-	CacheEntry* next;
-	CacheEntry* prev;
 };
 
 class DFA_State_Cache {
@@ -143,13 +84,11 @@ public:
 	~DFA_State_Cache();
 
 	// If the caller stores the handle, it has to call Ref() on it.
-	DFA_State_Handle* Lookup(const NFA_state_list& nfa_states,
+	DFA_State* Lookup(const NFA_state_list& nfa_states,
 					HashKey** hash);
 
 	// Takes ownership of both; hash is the one returned by Lookup().
-	DFA_State_Handle* Insert(DFA_State* state, HashKey* hash);
+	DFA_State* Insert(DFA_State* state, HashKey* hash);
-
-	void MoveToFront(DFA_State* state)	{ MoveToFront(state->centry); }
 
 	int NumEntries() const	{ return states.Length(); }
 
@@ -168,9 +107,6 @@ public:
 	void GetStats(Stats* s);
 
 private:
-	void Remove(CacheEntry* e);
-	void MoveToFront(CacheEntry* e);
-
 	int maxsize;
 
 	int hits;	// Statistics
@@ -180,10 +116,6 @@ private:
 
 	// Hash indexed by NFA states (MD5s of them, actually).
 	PDict(CacheEntry) states;
-
-	// List in LRU order.
-	CacheEntry* head;
-	CacheEntry* tail;
 };
 
 declare(PList,DFA_State);
@@ -196,7 +128,7 @@ public:
 			int* acc_array);
 	~DFA_Machine();
 
-	DFA_State_Handle* StartState() const	{ return start_state; }
+	DFA_State* StartState() const	{ return start_state; }
 
 	int NumStates() const	{ return dfa_state_cache->NumEntries(); }
 
@@ -217,74 +149,18 @@ protected:
 	int state_count;
 
 	// The state list has to be sorted according to IDs.
-	int StateSetToDFA_State(NFA_state_list* state_set, DFA_State_Handle*& d,
+	int StateSetToDFA_State(NFA_state_list* state_set, DFA_State*& d,
 				const EquivClass* ec);
 	const EquivClass* EC() const	{ return ec; }
 
 	EquivClass* ec;	// equivalence classes corresponding to NFAs
-	DFA_State_Handle* start_state;
+	DFA_State* start_state;
 	DFA_State_Cache* dfa_state_cache;
 
 	NFA_Machine* nfa;
 };
 
-#ifdef EXPIRE_DFA_STATES
+inline DFA_State* DFA_State::Xtion(int sym, DFA_Machine* machine)
-
-inline DFA_State_Handle* DFA_State::Xtion(int sym, DFA_Machine* machine)
-	{
-	Lock();
-
-	// This is just a clumsy form of sampling... Instead of moving
-	// the state to the front of our LRU cache on each transition (which
-	// would be quite often) we just do it on every nth transition
-	// (counted across all DFA states). This is based on the observation
-	// that a very few of all states are used most of time.
-	// (currently n=10000; should it be configurable?)
-	if ( transition_counter++ % 10000 == 0 )
-		machine->Cache()->MoveToFront(this);
-
-	DFA_State_Handle* h;
-
-	if ( xtions[sym] == DFA_UNCOMPUTED_STATE_PTR ||
-	     (xtions[sym] && ! xtions[sym]->IsValid()) )
-		h = ComputeXtion(sym, machine);
-	else
-		h = xtions[sym];
-
-	Unlock();
-
-	return h;
-	}
-
-inline DFA_State_Handle::~DFA_State_Handle()
-	{
-	if ( state != DFA_INVALID_STATE_PTR )
-		delete state;
-	}
-
-inline void DFA_State_Handle::Invalidate()
-	{
-	assert(state!=DFA_INVALID_STATE_PTR);
-	delete state;
-	state = DFA_INVALID_STATE_PTR;
-	Unref();
-	}
-
-// Not nice but helps avoiding some overhead in the non-expiration case.
-static inline void StateLock(DFA_State_Handle* s)	{ s->State()->Lock(); }
-static inline void StateUnlock(DFA_State_Handle* s)	{ s->State()->Unlock(); }
-static inline void StateRef(DFA_State_Handle* s)	{ s->Ref(); }
-static inline void StateUnref(DFA_State_Handle* s)	{ s->Unref(); }
-static inline void StateInvalidate(DFA_State_Handle* s)	{ s->Invalidate(); }
-
-static inline bool StateIsValid(DFA_State_Handle* s)
-	{
-	return ! s || s->IsValid();
-	}
-
-#else
-
-inline DFA_State_Handle* DFA_State::Xtion(int sym, DFA_Machine* machine)
 	{
 	if ( xtions[sym] == DFA_UNCOMPUTED_STATE_PTR )
 		return ComputeXtion(sym, machine);
@@ -292,13 +168,4 @@ inline DFA_State_Handle* DFA_State::Xtion(int sym, DFA_Machine* machine)
 		return xtions[sym];
 	}
 
-static inline void StateLock(DFA_State_Handle* s)	{ }
-static inline void StateUnlock(DFA_State_Handle* s)	{ }
-static inline void StateRef(DFA_State_Handle* s)	{ }
-static inline void StateUnref(DFA_State_Handle* s)	{ }
-static inline void StateInvalidate(DFA_State_Handle* s)	{ }
-static inline bool StateIsValid(DFA_State_Handle* s)	{ return true; }
-
-#endif
-
 #endif

src/DNS.cc

@@ -13,8 +13,6 @@
 #include "DNS.h"
 #include "Sessions.h"
 #include "Event.h"
-#include "DNS_Rewriter.h"
-#include "TCP_Rewriter.h"
 
 DNS_Interpreter::DNS_Interpreter(Analyzer* arg_analyzer)
 	{
@@ -1134,11 +1132,6 @@ DNS_Analyzer::~DNS_Analyzer()
 
 void DNS_Analyzer::Init()
 	{
-	if ( transformed_pkt_dump && RewritingTrace() &&
-	     Conn()->ConnTransport() == TRANSPORT_UDP )
-		Conn()->GetRootAnalyzer()->SetTraceRewriter(
-			new DNS_Rewriter(this, transformed_pkt_dump_MTU,
-			transformed_pkt_dump));
 	}
 
 void DNS_Analyzer::Done()
@@ -1196,5 +1189,3 @@ void DNS_Analyzer::ExpireTimer(double t)
 		ADD_ANALYZER_TIMER(&DNS_Analyzer::ExpireTimer,
 				t + dns_session_timeout, 1, TIMER_DNS_EXPIRE);
 	}
-
-#include "dns-rw.bif.func_def"

src/DNS.h

@@ -265,11 +265,6 @@ public:
 	virtual void Done();
 	virtual void ConnectionClosed(TCP_Endpoint* endpoint,
 					TCP_Endpoint* peer, int gen_event);
-	virtual int RewritingTrace()
-		{
-		return BifConst::rewriting_dns_trace ||
-			TCP_ApplicationAnalyzer::RewritingTrace();
-		}
 
 	void ExpireTimer(double t);
 

src/DNS_Rewriter.cc

@@ -1,587 +0,0 @@
-// $Id:$
-//
-// See the file "COPYING" in the main distribution directory for copyright.
-
-#include <sys/types.h>
-#include <sys/socket.h>
-#include <netinet/in.h>
-#include <arpa/nameser.h>
-#include <arpa/inet.h>
-#include <resolv.h>
-
-#include "NetVar.h"
-#include "DNS.h"
-#include "Val.h"
-#include "TCP.h"
-#include "Anon.h"
-#include "DNS_Rewriter.h"
-
-DNS_Rewriter::DNS_Rewriter(Analyzer* analyzer, int arg_MTU,
-				PacketDumper* dumper)
-: UDP_Rewriter(analyzer, arg_MTU, dumper)
-	{
-	pkt_size = 0;
-	current_pkt_id = 0;
-
-	pkt = new u_char[DNS_PKT_SIZE + DNS_HDR_SIZE];
-	}
-
-void DNS_Rewriter::DnsCopyHeader(Val* msg)
-	{
-	// New header - reset packet size.
-	pkt_size = 0;
-
-	// Move msg->AsRecordVal() to a RecordVal* to optimize.
-	const RecordVal* msg_rec = msg->AsRecordVal();
-	int id = msg_rec->Lookup(0)->AsCount();
-	int opcode = msg_rec->Lookup(1)->AsCount();
-	int rcode = msg_rec->Lookup(2)->AsCount();
-	int QR = msg_rec->Lookup(3)->AsBool();
-	int AA = msg_rec->Lookup(4)->AsBool();
-	int TC = msg_rec->Lookup(5)->AsBool();
-	int RD = msg_rec->Lookup(6)->AsBool();
-	int RA = msg_rec->Lookup(7)->AsBool();
-	int Z = msg_rec->Lookup(8)->AsCount();
-	int qdcount = msg_rec->Lookup(9)->AsCount();
-	int ancount = msg_rec->Lookup(10)->AsCount();
-	int nscount = msg_rec->Lookup(11)->AsCount();
-	int arcount = msg_rec->Lookup(12)->AsCount();
-
-	current_pkt_id = id;
-
-	// Set the DNS flags.
-	uint16 flags = (QR << 15) | (AA << 10) | (TC << 9) |
-			(RD << 8) | (RA << 7) | (Z << 4);
-
-	flags |= rcode | (opcode << 11);
-
-	(void) WriteShortVal(id);
-	(void) WriteShortVal(flags);
-	(void) WriteShortVal(qdcount);
-	(void) WriteShortVal(ancount);
-	(void) WriteShortVal(nscount);
-	(void) WriteShortVal(arcount);
-
-	// We've finished the header.
-	pkt_size = DNS_HDR_SIZE;
-
-	// Assign all the pointers for dn_comp().
-	dpp = dn_ptrs;
-	*dpp++ = pkt;
-	*dpp++ = 0;
-
-	last_dn_ptr = dn_ptrs + sizeof dn_ptrs / sizeof dn_ptrs[0];
-	}
-
-int DNS_Rewriter::DnsCopyQuery(Val* val)
-	{
-	const RecordVal* val_rec = val->AsRecordVal();
-
-	// int type = val_rec->Lookup(0)->AsCount();
-
-	const BroString* query = val_rec->Lookup(1)->AsString();
-	int atype = val_rec->Lookup(2)->AsCount();
-	int aclass = val_rec->Lookup(3)->AsCount();
-
-	return DnsCopyQuery(query, atype, aclass);
-	}
-
-// Copy the question part of the query into memory.
-// Return the number of bytes that the query string compressed to.
-int DNS_Rewriter::DnsCopyQuery(const BroString* query, uint32 qtype,
-				uint32 qclass)
-	{
-	int len = query->Len();
-	int psize = pkt_size;
-
-	// Encode the query string.
-	const char* dname = (char*) query->Bytes();
-	len = dn_comp(dname, pkt + pkt_size, DNS_PKT_SIZE - pkt_size,
-			dn_ptrs, last_dn_ptr);
-
-	// Can't encode in less than 2 bytes, or about to overwrite.
-	if ( len < 1 || pkt_size + len + 4 > DNS_PKT_SIZE )
-		{
-		warn("dn_comp couldn't encode name into packet");
-		return 0;
-		}
-
-	pkt_size += len;
-
-	// Set type.
-	if ( ! WriteShortVal(qtype) )
-		{
-		pkt_size = psize;
-		return 0;
-		}
-
-	// Set class.
-	if ( ! WriteShortVal(qclass) )
-		{
-		pkt_size = psize;
-		return 0;
-		}
-
-	return len;
-	}
-
-
-// PTR, NS and CNAME are all the same.
-void DNS_Rewriter::DnsCopyPTR(Val* ans, const BroString* name)
-	{
-	DnsCopyCNAME(ans, name);
-	}
-
-// Copy an NS RR into the packet.
-void DNS_Rewriter::DnsCopyNS( Val* ans, const BroString* name)
-	{
-	DnsCopyCNAME(ans, name);
-	}
-
-// Copy an A RR into the packet.
-void DNS_Rewriter::DnsCopyA(Val* ans, uint32 addr)
-	{
-	int psize = pkt_size;
-
-	// Put query part into packet.
-	int len = DnsCopyQuery(ans);
-
-	if ( ! len )
-		return;
-
-	double TTL = ans->AsRecordVal()->Lookup(4)->AsInterval();
-	if ( ! WriteDoubleAsInt(TTL) )
-		{
-		pkt_size = psize;
-		return;
-		}
-
-	// Now we put in how long the resource data is (A rec is always 4).
-	if ( ! WriteShortVal(4) )
-		{
-		pkt_size = psize;
-		return;
-		}
-
-	// Stick in the address (already in network byte order).
-	if ( ! WriteVal(uint32(ntohl(addr))) )
-		{
-		pkt_size = psize;
-		return;
-		}
-	}
-
-// Copy an AAAA RR into the packet.
-void DNS_Rewriter::DnsCopyAAAA(Val* ans, addr_type addr, const BroString* addrstr)
-	{
-	int psize = pkt_size;
-
-
-	// Put query part into packet.
-	int len = DnsCopyQuery(ans);
-	if ( ! len || pkt_size + 6 > DNS_PKT_SIZE )
-		return;
-
-	double TTL = ans->AsRecordVal()->Lookup(4)->AsInterval();
-	if ( ! WriteDoubleAsInt(TTL))
-		{
-		pkt_size = psize;
-		return;
-		}
-
-	// Now we put in how long the resource data is (AAAA rec is always 16).
-	if ( ! WriteShortVal(16) )
-		{
-		pkt_size = psize;
-		return;
-		}
-#ifdef BROv6
-	if ( ! WriteVal(addr) )
-		{
-		pkt_size = psize;
-		return;
-		}
-#else
-	uint32 addr_copy[4];
-	char* addr_tmp = addrstr->Render(BroString::ESC_NONE);
-	inet_pton(AF_INET6, addr_tmp, addr_copy);
-
-	if ( ! WriteVal(addr_copy) )
-		{
-		pkt_size = psize;
-		return;
-		}
-
-	delete addr_tmp;
-#endif
-
-	}
-
-// Copy a CNAME RR into the packet.
-void DNS_Rewriter::DnsCopyCNAME(Val* ans, const BroString* name)
-	{
-	int psize = pkt_size;
-
-	// Put query part into packet.
-	int len = DnsCopyQuery(ans);
-	if ( ! len || pkt_size + 6 > DNS_PKT_SIZE )
-		return;
-
-	double TTL = ans->AsRecordVal()->Lookup(4)->AsInterval();
-	if ( ! WriteDoubleAsInt(TTL))
-		{
-		pkt_size = psize;
-		return;
-		}
-
-	// Resource length (domain name length in packet).
-	// Have to skip till it's encoded, remember this spot.
-	u_char* resource_len = pkt + pkt_size;
-	pkt_size += 2;
-
-	// Encode the domain name.
-	const char* dname = (char*) name->CheckString();
-	len = dn_comp(dname, pkt + pkt_size, DNS_PKT_SIZE - pkt_size,
-			dn_ptrs, last_dn_ptr);
-
-	if ( len < 1 )
-		{
-		pkt_size = psize;
-		return;
-		}
-
-	pkt_size += len;
-
-	// Now we put in how long the name was to encode.
-	uint16 net_rdlen = htons(short(len));
-	memcpy(resource_len, &net_rdlen, sizeof(uint16));
-	}
-
-// Copy a CNAME RR into the packet.
-void DNS_Rewriter::DnsCopyTXT(Val* ans, const BroString* name)
-	{
-	int psize = pkt_size;
-
-	// Put query part into packet.
-	int len = DnsCopyQuery(ans);
-	if ( ! len || pkt_size + 6 > DNS_PKT_SIZE )
-		return;
-
-	double TTL = ans->AsRecordVal()->Lookup(4)->AsInterval();
-	if ( ! WriteDoubleAsInt(TTL))
-		{
-		pkt_size = psize;
-		return;
-		}
-
-	if ( ! WriteShortVal(name->Len()+1))
-		{
-		pkt_size = psize;
-		return;
-		}
-
-	if ( ! WriteVal(uint8(name->Len())))
-		{
-		pkt_size = psize;
-		return;
-		}
-
-	if ( ! WriteVal(name))
-		{
-		pkt_size = psize;
-		return;
-		}
-
-	}
-
-// Copy an MX RR into the packet.
-void DNS_Rewriter::DnsCopyMX(Val* ans, const BroString* name, uint32 preference)
-	{
-	int psize = pkt_size;
-
-	// Put query part into packet.
-	int len = DnsCopyQuery(ans);
-
-	if ( ! len || pkt_size + len + 6 > DNS_PKT_SIZE )
-		{
-		warn("DnsCopyMX: packet too large");
-		return;
-		}
-
-	double TTL = ans->AsRecordVal()->Lookup(4)->AsInterval();
-	if ( ! WriteDoubleAsInt(TTL) )
-		{
-		pkt_size = psize;
-		warn("DnsCopyMX: packet too large");
-		return;
-		}
-
-	// Resource length (domain name length in packet).
-	// Have to skip till it's, remember this spot.
-	u_char* resource_len = pkt + pkt_size;
-	pkt_size += 2;
-
-	if ( ! WriteShortVal(preference))
-		{
-		pkt_size = psize;
-		warn("DnsCopyMX: packet too large");
-		return;
-		}
-
-	// Encode the domain name.
-	const char* dname = (char*) name->CheckString();
-	len += dn_comp(dname, pkt + pkt_size, DNS_PKT_SIZE - pkt_size,
-			dn_ptrs, last_dn_ptr);
-
-	if ( len < 1 )
-		{
-		pkt_size = psize;
-		warn("DnsCopyMX: packet too large");
-		return;
-		}
-
-	pkt_size += len;
-
-	// 2 bytes for the preference size above.
-	len += 2;
-
-	// Now we put in how long the name was to encode.
-	uint16 net_rdlen = htons(short(len));
-	memcpy(resource_len, &net_rdlen, sizeof(uint16));
-	}
-
-// Copy an SOA RR into the packet.
-void DNS_Rewriter::DnsCopySOA(Val* ans, Val* soa)
-	{
-	u_char* resource_len;
-	int resource_offset = 0;
-	int psize = pkt_size;
-
-	const RecordVal* soa_rec = soa->AsRecordVal();
-
-	const BroString* mname = soa_rec->Lookup(0)->AsString();
-	const BroString* rname = soa_rec->Lookup(1)->AsString();
-	uint32 serial = soa_rec->Lookup(2)->AsCount();
-	double refresh = soa_rec->Lookup(3)->AsInterval();
-	double retry = soa_rec->Lookup(4)->AsInterval();
-	double expire = soa_rec->Lookup(5)->AsInterval();
-	double minimum = soa_rec->Lookup(6)->AsInterval();
-
-	// Put query part into packet.
-	int len = DnsCopyQuery(ans);
-
-	if ( ! len || len + 6 > DNS_PKT_SIZE )
-		return;
-
-	double TTL = ans->AsRecordVal()->Lookup(4)->AsInterval();
-	if ( ! WriteDoubleAsInt(TTL) )
-		{
-		pkt_size = psize;
-		return;
-		}
-
-	// Resource length: have to skip till it's encoded.
-	// Remember this spot and offset.
-	resource_len = pkt + pkt_size;
-	pkt_size += 2;
-
-	// Start counting from here (after rdlength).
-	resource_offset = pkt_size;
-
-	// Encode the domain name.
-	const char* dname = (char*) mname->CheckString();
-	len = dn_comp(dname, pkt + pkt_size, DNS_PKT_SIZE - pkt_size,
-			dn_ptrs, last_dn_ptr);
-
-	if ( len < 1 )
-		{
-		pkt_size = psize;
-		return;
-		}
-
-	pkt_size += len;
-
-	// Encode the domain name.
-	dname = (char*) rname->CheckString();
-	len = dn_comp(dname, pkt + pkt_size, DNS_PKT_SIZE - pkt_size,
-			dn_ptrs, last_dn_ptr);
-	if ( len < 1 )
-		{
-		pkt_size = psize;
-		return;
-		}
-
-	pkt_size += len;
-
-	if ( ! WriteVal(serial) || ! WriteDoubleAsInt(refresh) ||
-	     ! WriteDoubleAsInt(retry) || ! WriteDoubleAsInt(expire) ||
-	     ! WriteDoubleAsInt(minimum) )
-		{
-		pkt_size = psize;
-		return;
-		}
-
-	// Now we put in how long this packet was.
-	uint16 net_rdlen = htons(short(pkt_size - resource_offset));
-	memcpy(resource_len, &net_rdlen, sizeof(uint16));
-	}
-
-void DNS_Rewriter::DnsCopyEDNSaddl(Val* ans)
-	{
-	const RecordVal* ans_rec = ans->AsRecordVal();
-
-	int ans_type = ans_rec->Lookup(0)->AsCount();
-	// BroString* query_name = ans_rec->Lookup(1)->AsString();
-	int atype = ans_rec->Lookup(2)->AsCount();
-	int aclass = ans_rec->Lookup(3)->AsCount();
-	int return_error = ans_rec->Lookup(4)->AsCount();
-	int version = ans_rec->Lookup(5)->AsCount();
-	int z = ans_rec->Lookup(6)->AsCount();
-	double ttl = ans_rec->Lookup(7)->AsInterval();
-	int is_query = ans_rec->Lookup(8)->AsCount();
-
-	int rcode = return_error;
-	int ecode = 0;
-
-	int psize = pkt_size;
-
-	if ( return_error > 0xff )
-		{
-		rcode &= 0xff;
-		ecode = return_error >> 8;
-		}
-
-	// Stick the version onto the ecode.
-	ecode = (ecode << 8) | version;
-
-	// Write fixed part of OPT RR
-	// Name '0'.
-	memset(pkt + pkt_size, 0, 1);
-	++pkt_size;
-
-	// Type (either 29 or 41).
-	if ( ! WriteShortVal(atype) )
-		{
-		pkt_size = psize;
-		return;
-		}
-
-	// UDP playload size
-	if ( ! WriteShortVal(aclass) )
-		{
-		pkt_size = psize;
-		return;
-		}
-
-	// Extended rcode + version.
-	if ( ! WriteShortVal(ecode) )
-		{
-		pkt_size = psize;
-		return;
-		}
-
-	// Z field.
-	if ( ! WriteShortVal(z) )
-		{
-		pkt_size = psize;
-		return;
-		}
-
-	// Data length (XXX:for now its zero!).
-	if ( ! WriteShortVal(0) )
-		{
-		pkt_size = psize;
-		return;
-		}
-
-	// Don't write data (XXX:we don't have it!).
-	}
-
-// Does this packet match the current packet being worked on?
-int DNS_Rewriter::DnsPktMatch(Val* msg)
-	{
-	return msg->AsRecordVal()->Lookup(0)->AsInt() == current_pkt_id;
-	}
-
-// Supports copying of TXT values.
-int DNS_Rewriter::WriteVal(const BroString* val)
-	{
-	int n = val->Len();
-
-        if ( pkt_size + n > DNS_PKT_SIZE )
-		{
-		warn("WriteVal: couldn't write data into packet");
-		return 0;
-		}
-
-        char* new_val = val->Render(BroString::ESC_NONE);
-        memcpy(pkt + pkt_size, new_val, n);
-        pkt_size += n;
-
-        delete[] new_val;
-
-        return n;
-	}
-
-int DNS_Rewriter::WriteVal(const uint32* val)
-	{
-	if ( pkt_size + 16 > DNS_PKT_SIZE )
-		{
-		warn("WriteVal: couldn't write data into packet");
-		return 0;
-		}
-
-	memcpy(pkt + pkt_size, &val[0], sizeof(uint32)); pkt_size += 4;
-	memcpy(pkt + pkt_size, &val[1], sizeof(uint32)); pkt_size += 4;
-	memcpy(pkt + pkt_size, &val[2], sizeof(uint32)); pkt_size += 4;
-	memcpy(pkt + pkt_size, &val[3], sizeof(uint32)); pkt_size += 4;
-
-	return sizeof(uint32) * 4;
-	}
-
-// Write a 32 bit value given in host order to the packet.
-int DNS_Rewriter::WriteVal(uint32 val)
-	{
-	if ( pkt_size + 4 > DNS_PKT_SIZE )
-		{
-		warn("WriteVal: couldn't write data into packet");
-		return 0;
-		}
-
-	uint32 net_val = htonl(val);
-	memcpy(pkt + pkt_size, &net_val, sizeof(uint32));
-	pkt_size += 4;
-
-	return sizeof(uint32);
-	}
-
-// Write a 16 bit value given in host order to the packet.
-int DNS_Rewriter::WriteVal(uint16 val)
-	{
-	if ( pkt_size + 2 > DNS_PKT_SIZE )
-		{
-		warn("WriteShortVal: couldn't write data into packet");
-		return 0;
-		}
-
-	uint16 net_val = htons(val);
-	memcpy(pkt + pkt_size, &net_val, sizeof(uint16));
-	pkt_size += 2;
-
-	return sizeof(uint16);
-	}
-
-// Write a 8 bit value given in host order to the packet.
-int DNS_Rewriter::WriteVal(uint8 val)
-	{
-	if ( pkt_size + 1 > DNS_PKT_SIZE )
-		{
-		warn("WriteVal: couldn't write data into packet");
-		return 0;
-		}
-
-	memcpy(pkt + pkt_size, &val, sizeof(uint8));
-	pkt_size += sizeof(uint8);
-
-	return sizeof(uint8);
-	}

src/DNS_Rewriter.h

@@ -1,70 +0,0 @@
-// $Id:$
-//
-// See the file "COPYING" in the main distribution directory for copyright.
-
-#ifndef dns_rewriter_h
-#define dns_rewriter_h
-
-#include "UDP.h"
-#include "UDP_Rewriter.h"
-#include "Rewriter.h"
-
-#define DNS_HDR_SIZE 12
-
-// DNS packets size. 512 is the *normal* size, but some packets are bigger
-// than this, and the anonymization process can expand packets, so we
-// pad this way out.
-#define DNS_PKT_SIZE (512*4)
-
-class DNS_Rewriter: public UDP_Rewriter {
-public:
-	DNS_Rewriter(Analyzer* analyzer, int arg_MTU, PacketDumper* dumper);
-	virtual ~DNS_Rewriter()	{ delete pkt;}
-
-	void DnsCopyHeader(Val* val);
-
-	int DnsCopyQuery(const BroString* query, uint32 qtype, uint32 qclass);
-	int DnsCopyQuery(Val* val);
-
-	void DnsCopyNS(Val* ans, const BroString* name);
-	void DnsCopyPTR(Val* ans, const BroString* name);
-	void DnsCopyCNAME(Val* ans, const BroString* name);
-	void DnsCopyTXT(Val* ans, const BroString* name);
-	void DnsCopyA(Val* ans, uint32 addr);
-
-	// AAAA is weird, because the address is an IPv4 type.
-	// If we don't have IPv6, and if it's IPv6, it's a pointer
-	// to valid data.
-	void DnsCopyAAAA(Val* ans, addr_type addr, const BroString* addrstr);
-
-	void DnsCopyMX(Val* ans, const BroString* name, uint32 preference);
-	void DnsCopySOA(Val* ans, Val* soa);
-	void DnsCopyEDNSaddl(Val* ans);
-
-	int DnsPktMatch(Val* val);
-	const u_char* Packet() const	{ return pkt; }
-	int PacketSize() const	{ return pkt_size; }
-	void SetOrig( int orig )	{ is_orig = orig; }
-	int IsOrig()			{ return is_orig; }
-
-	int WriteDoubleAsInt(double d)	{ return WriteVal(uint32(d)); }
-	int WriteShortVal(uint16 val)	{ return WriteVal(uint16(val)); }
-	int WriteVal(uint32 val);
-	int WriteVal(uint16 val);
-	int WriteVal(uint8 val);
-	int WriteVal(const uint32* val);
-	int WriteVal(const BroString* val);
-
-private:
-	u_char* pkt;		// the DNS packet
-	int pkt_size;		// size of the packet
-	int current_pkt_id;	// current ID (sanity checking)
-
-	int is_orig;
-
-	u_char* dn_ptrs[30];	// pointer to names in DNS packet
-	u_char** dpp;		// points to current position in DNS packet
-	u_char** last_dn_ptr;	// points to last entry in dn_ptrs
-};
-
-#endif

src/DebugLogger.cc

@@ -17,6 +17,7 @@ DebugLogger::Stream DebugLogger::streams[NUM_DBGS] = {
 	{ "compressor", 0, false }, {"string", 0, false },
 	{ "notifiers", 0, false },  { "main-loop", 0, false },
 	{ "dpd", 0, false }, { "tm", 0, false },
+	{ "logging", 0, false }
 };
 
 DebugLogger::DebugLogger(const char* filename)

src/DebugLogger.h

@@ -20,11 +20,12 @@ enum DebugStream {
 	DBG_STATE,	// StateAccess logging
 	DBG_CHUNKEDIO,	// ChunkedIO logging
 	DBG_COMPRESSOR,	// Connection compressor
-	DBG_STRING,     // String code
+	DBG_STRING,	// String code
-	DBG_NOTIFIERS,   // Notifiers (see StateAccess.h)
+	DBG_NOTIFIERS,	// Notifiers (see StateAccess.h)
 	DBG_MAINLOOP,	// Main IOSource loop
 	DBG_DPD,	// Dynamic application detection framework
-	DBG_TM,	// Time-machine packet input via Brocolli
+	DBG_TM,		// Time-machine packet input via Brocolli
+	DBG_LOGGING,	// Logging streams
 
 	NUM_DBGS // Has to be last
 };

src/Desc.cc

@@ -41,6 +41,9 @@ ODesc::ODesc(desc_type t, BroFile* arg_f)
 	want_quotes = 0;
 	do_flush = 1;
 	include_stats = 0;
+	indent_with_spaces = 0;
+	escape = 0;
+	escape_len = 0;
 	}
 
 ODesc::~ODesc()
@@ -54,6 +57,12 @@ ODesc::~ODesc()
 		free(base);
 	}
 
+void ODesc::SetEscape(const char* arg_escape, int len)
+	{
+	escape = arg_escape;
+	escape_len = len;
+	}
+
 void ODesc::PushIndent()
 	{
 	++indent_level;
@@ -67,6 +76,12 @@ void ODesc::PopIndent()
 	NL();
 	}
 
+void ODesc::PopIndentNoNL()
+	{
+	if ( --indent_level < 0 )
+		internal_error("ODesc::PopIndent underflow");
+	}
+
 void ODesc::Add(const char* s, int do_indent)
 	{
 	unsigned int n = strlen(s);
@@ -179,12 +194,57 @@ void ODesc::AddBytes(const BroString* s)
 
 void ODesc::Indent()
 	{
-	for ( int i = 0; i < indent_level; ++i )
+	if ( indent_with_spaces > 0 )
-		Add("\t", 0);
+		{
+		for ( int i = 0; i < indent_level; ++i )
+			for ( int j = 0; j < indent_with_spaces; ++j )
+				Add(" ", 0);
+		}
+	else
+		{
+		for ( int i = 0; i < indent_level; ++i )
+			Add("\t", 0);
+		}
 	}
 
+static const char hex_chars[] = "0123456789ABCDEF";
 
 void ODesc::AddBytes(const void* bytes, unsigned int n)
+	{
+	if ( ! escape )
+		return AddBytesRaw(bytes, n);
+
+	const char* s = (const char*) bytes;
+	const char* e = (const char*) bytes + n;
+
+	while ( s < e )
+		{
+		const char* t = (const char*) memchr(s, escape[0], e - s);
+
+		if ( ! t )
+			break;
+
+		if ( memcmp(t, escape, escape_len) != 0 )
+			break;
+
+		AddBytesRaw(s, t - s);
+
+		for ( int i = 0; i < escape_len; ++i )
+			{
+			char hex[5] = "\\x00";
+			hex[2] = hex_chars[(*t) >> 4];
+			hex[3] = hex_chars[(*t) & 0x0f];
+			AddBytesRaw(hex, sizeof(hex));
+			++t;
+			}
+
+		s = t;
+		}
+
+	AddBytesRaw(s, e - s);
+	}
+
+void ODesc::AddBytesRaw(const void* bytes, unsigned int n)
 	{
 	if ( n == 0 )
 		return;

src/Desc.h

@@ -49,10 +49,17 @@ public:
 
 	void SetFlush(int arg_do_flush)	{ do_flush = arg_do_flush; }
 
+	// The string passed in must remain valid as long as this object lives.
+	void SetEscape(const char* escape, int len);
+
 	void PushIndent();
 	void PopIndent();
+	void PopIndentNoNL();
 	int GetIndentLevel() const	{ return indent_level; }
 
+	int IndentSpaces() const	{ return indent_with_spaces; }
+	void SetIndentSpaces(int i)	{ indent_with_spaces = i; }
+
 	void Add(const char* s, int do_indent=1);
 	void AddN(const char* s, int len)	{ AddBytes(s, len); }
 	void Add(int i);
@@ -93,6 +100,9 @@ public:
 				Add("\n", 0);
 			}
 
+	// Bypasses the escaping enabled via SetEscape().
+	void AddRaw(const char* s, int len)	{ AddBytesRaw(s, len); }
+
 	// Returns the description as a string.
 	const char* Description() const		{ return (const char*) base; }
 
@@ -115,6 +125,7 @@ protected:
 	void Indent();
 
 	void AddBytes(const void* bytes, unsigned int n);
+	void AddBytesRaw(const void* bytes, unsigned int n);
 
 	// Make buffer big enough for n bytes beyond bufp.
 	void Grow(unsigned int n);
@@ -128,6 +139,9 @@ protected:
 	unsigned int offset;	// where we are in the buffer
 	unsigned int size;	// size of buffer in bytes
 
+	int escape_len;	// number of bytes in to escape sequence
+	const char* escape;	// bytes to escape on output
+
 	BroFile* f;	// or the file we're using.
 
 	int indent_level;
@@ -135,6 +149,7 @@ protected:
 	int want_quotes;
 	int do_flush;
 	int include_stats;
+	int indent_with_spaces;
 };
 
 #endif

src/Event.cc

@@ -120,9 +120,6 @@ void EventMgr::Drain()
 
 	// Note: we might eventually need a general way to specify things to
 	// do after draining events.
-	extern void flush_rewriter_packet();
-	flush_rewriter_packet();
-
 	draining = false;
 
 	// We evaluate Triggers here. While this is somewhat unrelated to event

src/EventHandler.cc

@@ -23,6 +23,11 @@ EventHandler::~EventHandler()
 	delete [] group;
 	}
 
+EventHandler::operator bool() const
+	{
+	return enabled && ((local && local->HasBodies()) || receivers.length());
+	}
+
 FuncType* EventHandler::FType()
 	{
 	if ( type )

src/EventHandler.h

@@ -34,8 +34,7 @@ public:
 	void Call(val_list* vl, bool no_remote = false);
 
 	// Returns true if there is at least one local or remote handler.
-	operator  bool() const
+	operator  bool() const;
-		{ return enabled && (local || receivers.length()); }
 
 	void SetUsed()          { used = true; }
 	bool Used()             { return used; }

src/Expr.cc

@@ -284,7 +284,7 @@ Val* NameExpr::Eval(Frame* f) const
 	Val* v;
 
 	if ( id->AsType() )
-		RunTime("cannot evaluate type name");
+		return new Val(id->AsType(), true);
 
 	if ( id->IsGlobal() )
 		v = id->ID_Val();
@@ -2497,12 +2497,7 @@ AssignExpr::AssignExpr(Expr* arg_op1, Expr* arg_op2, int arg_is_init,
 bool AssignExpr::TypeCheck()
 	{
 	TypeTag bt1 = op1->Type()->Tag();
-	if ( IsVector(bt1) )
-		bt1 = op1->Type()->AsVectorType()->YieldType()->Tag();
-
 	TypeTag bt2 = op2->Type()->Tag();
-	if ( IsVector(bt2) )
-		bt2 = op2->Type()->AsVectorType()->YieldType()->Tag();
 
 	if ( bt1 == TYPE_LIST && bt2 == TYPE_ANY )
 		// This is ok because we cannot explicitly declare lists on
@@ -2531,16 +2526,42 @@ bool AssignExpr::TypeCheck()
 		return true;
 		}
 
+	if ( bt1 == TYPE_VECTOR && bt2 == bt1 &&
+	     op2->Type()->AsVectorType()->IsUnspecifiedVector() )
+		{
+		op2 = new VectorCoerceExpr(op2, op1->Type()->AsVectorType());
+		return true;
+		}
+
+	if ( op1->Type()->Tag() == TYPE_RECORD &&
+	     op2->Type()->Tag() == TYPE_RECORD )
+		{
+		if ( same_type(op1->Type(), op2->Type()) )
+			{
+			RecordType* rt1 = op1->Type()->AsRecordType();
+			RecordType* rt2 = op2->Type()->AsRecordType();
+
+			// Make sure the attributes match as well.
+			for ( int i = 0; i < rt1->NumFields(); ++i )
+				{
+				const TypeDecl* td1 = rt1->FieldDecl(i);
+				const TypeDecl* td2 = rt2->FieldDecl(i);
+
+				if ( same_attrs(td1->attrs, td2->attrs) )
+					// Everything matches.
+					return true;
+				}
+			}
+
+		// Need to coerce.
+		op2 = new RecordCoerceExpr(op2, op1->Type()->AsRecordType());
+		return true;
+		}
+
 	if ( ! same_type(op1->Type(), op2->Type()) )
 		{
-		if ( op1->Type()->Tag() == TYPE_RECORD &&
+		ExprError("type clash in assignment");
-		     op2->Type()->Tag() == TYPE_RECORD )
+		return false;
-			op2 = new RecordCoerceExpr(op2, op1->Type()->AsRecordType());
-		else
-			{
-			ExprError("type clash in assignment");
-			return false;
-			}
 		}
 
 	return true;
@@ -3290,48 +3311,12 @@ RecordConstructorExpr::RecordConstructorExpr(ListExpr* constructor_list)
 
 Val* RecordConstructorExpr::InitVal(const BroType* t, Val* aggr) const
 	{
-	if ( ! aggr )
+	RecordVal* rv = Eval(0)->AsRecordVal();
-		aggr = new RecordVal(const_cast<RecordType*>(t->AsRecordType()));
+	RecordVal* ar = rv->CoerceTo(t->AsRecordType(), aggr);
 
-	if ( record_promotion_compatible(t->AsRecordType(), Type()->AsRecordType()) )
+	if ( ar )
 		{
-		RecordVal* ar = aggr->AsRecordVal();
-		RecordType* ar_t = aggr->Type()->AsRecordType();
-
-		RecordVal* rv = Eval(0)->AsRecordVal();
-		RecordType* rv_t = rv->Type()->AsRecordType();
-
-		int i;
-		for ( i = 0; i < rv_t->NumFields(); ++i )
-			{
-			int t_i = ar_t->FieldOffset(rv_t->FieldName(i));
-
-			if ( t_i < 0 )
-				{
-				char buf[512];
-				safe_snprintf(buf, sizeof(buf),
-					      "orphan field \"%s\" in initialization",
-					      rv_t->FieldName(i));
-				Error(buf);
-				break;
-				}
-
-			else
-				ar->Assign(t_i, rv->Lookup(i)->Ref());
-			}
-
-		for ( i = 0; i < ar_t->NumFields(); ++i )
-			if ( ! ar->Lookup(i) &&
-			     ! ar_t->FieldDecl(i)->FindAttr(ATTR_OPTIONAL) )
-				{
-				char buf[512];
-				safe_snprintf(buf, sizeof(buf),
-					      "non-optional field \"%s\" missing in initialization", ar_t->FieldName(i));
-				Error(buf);
-				}
-
 		Unref(rv);
-
 		return ar;
 		}
 
@@ -3400,7 +3385,7 @@ TableConstructorExpr::TableConstructorExpr(ListExpr* constructor_list,
 			SetError("values in table(...) constructor do not specify a table");
 		}
 
-	attrs = arg_attrs ? new Attributes(arg_attrs, type) : 0;
+	attrs = arg_attrs ? new Attributes(arg_attrs, type, false) : 0;
 	}
 
 Val* TableConstructorExpr::Eval(Frame* f) const
@@ -3466,7 +3451,7 @@ SetConstructorExpr::SetConstructorExpr(ListExpr* constructor_list,
 	else if ( type->Tag() != TYPE_TABLE || ! type->AsTableType()->IsSet() )
 		SetError("values in set(...) constructor do not specify a set");
 
-	attrs = arg_attrs ? new Attributes(arg_attrs, type) : 0;
+	attrs = arg_attrs ? new Attributes(arg_attrs, type, false) : 0;
 	}
 
 Val* SetConstructorExpr::Eval(Frame* f) const
@@ -3523,6 +3508,13 @@ VectorConstructorExpr::VectorConstructorExpr(ListExpr* constructor_list)
 	if ( IsError() )
 		return;
 
+	if ( constructor_list->Exprs().length() == 0 )
+		{
+		// vector().
+		SetType(new ::VectorType(base_type(TYPE_ANY)));
+		return;
+		}
+
 	BroType* t = merge_type_list(constructor_list);
 	if ( t )
 		{
@@ -3994,15 +3986,8 @@ RecordCoerceExpr::RecordCoerceExpr(Expr* op, RecordType* r)
 			{
 			int t_i = t_r->FieldOffset(sub_r->FieldName(i));
 			if ( t_i < 0 )
-				{
+				// Orphane field in rhs, that's ok.
-				// Same as in RecordConstructorExpr::InitVal.
-				char buf[512];
-				safe_snprintf(buf, sizeof(buf),
-					      "orphan record field \"%s\"",
-					      sub_r->FieldName(i));
-				Error(buf);
 				continue;
-				}
 
 			BroType* sub_t_i = sub_r->FieldType(i);
 			BroType* sup_t_i = t_r->FieldType(t_i);
@@ -4047,7 +4032,23 @@ Val* RecordCoerceExpr::Fold(Val* v) const
 	for ( int i = 0; i < map_size; ++i )
 		{
 		if ( map[i] >= 0 )
-			val->Assign(i, rv->Lookup(map[i])->Ref());
+			{
+			Val* rhs = rv->Lookup(map[i]);
+			if ( ! rhs )
+				{
+				const Attr* def = rv->Type()->AsRecordType()->FieldDecl(
+					map[i])->FindAttr(ATTR_DEFAULT);
+
+				if ( def )
+					rhs = def->AttrExpr()->Eval(0);
+				}
+
+			if ( rhs )
+				rhs = rhs->Ref();
+
+			assert(rhs || Type()->AsRecordType()->FieldDecl(i)->FindAttr(ATTR_OPTIONAL));
+			val->Assign(i, rhs);
+			}
 		else
 			val->Assign(i, 0);
 		}
@@ -4132,6 +4133,50 @@ bool TableCoerceExpr::DoUnserialize(UnserialInfo* info)
 	return true;
 	}
 
+VectorCoerceExpr::VectorCoerceExpr(Expr* op, VectorType* v)
+: UnaryExpr(EXPR_VECTOR_COERCE, op)
+	{
+	if ( IsError() )
+		return;
+
+	SetType(v->Ref());
+
+	if ( Type()->Tag() != TYPE_VECTOR )
+		ExprError("coercion to non-vector");
+
+	else if ( op->Type()->Tag() != TYPE_VECTOR )
+		ExprError("coercion of non-vector to vector");
+	}
+
+
+VectorCoerceExpr::~VectorCoerceExpr()
+	{
+	}
+
+Val* VectorCoerceExpr::Fold(Val* v) const
+	{
+	VectorVal* vv = v->AsVectorVal();
+
+	if ( vv->Size() > 0 )
+		Internal("coercion of non-empty vector");
+
+	return new VectorVal(Type()->Ref()->AsVectorType());
+	}
+
+IMPLEMENT_SERIAL(VectorCoerceExpr, SER_VECTOR_COERCE_EXPR);
+
+bool VectorCoerceExpr::DoSerialize(SerialInfo* info) const
+	{
+	DO_SERIALIZE(SER_VECTOR_COERCE_EXPR, UnaryExpr);
+	return true;
+	}
+
+bool VectorCoerceExpr::DoUnserialize(UnserialInfo* info)
+	{
+	DO_UNSERIALIZE(UnaryExpr);
+	return true;
+	}
+
 FlattenExpr::FlattenExpr(Expr* arg_op)
 : UnaryExpr(EXPR_FLATTEN, arg_op)
 	{
@@ -5308,27 +5353,52 @@ int check_and_promote_expr(Expr*& e, BroType* t)
 		return 1;
 		}
 
-	else if ( ! same_type(t, et) )
+	if ( t->Tag() == TYPE_RECORD && et->Tag() == TYPE_RECORD )
 		{
-		if ( t->Tag() == TYPE_RECORD && et->Tag() == TYPE_RECORD )
+		RecordType* t_r = t->AsRecordType();
-			{
+		RecordType* et_r = et->AsRecordType();
-			RecordType* t_r = t->AsRecordType();
-			RecordType* et_r = et->AsRecordType();
 
-			if ( record_promotion_compatible(t_r, et_r) )
+		if ( same_type(t, et) )
+			{
+			// Make sure the attributes match as well.
+			for ( int i = 0; i < t_r->NumFields(); ++i )
 				{
-				e = new RecordCoerceExpr(e, t_r);
+				const TypeDecl* td1 = t_r->FieldDecl(i);
-				return 1;
+				const TypeDecl* td2 = et_r->FieldDecl(i);
+
+				if ( same_attrs(td1->attrs, td2->attrs) )
+					// Everything matches perfectly.
+					return 1;
 				}
 			}
 
-		else if ( t->Tag() == TYPE_TABLE && et->Tag() == TYPE_TABLE &&
+		if ( record_promotion_compatible(t_r, et_r) )
+			{
+			e = new RecordCoerceExpr(e, t_r);
+			return 1;
+			}
+
+		t->Error("incompatible record types", e);
+		return 0;
+		}
+
+
+	if ( ! same_type(t, et) )
+		{
+		if ( t->Tag() == TYPE_TABLE && et->Tag() == TYPE_TABLE &&
 			  et->AsTableType()->IsUnspecifiedTable() )
 			{
 			e = new TableCoerceExpr(e, t->AsTableType());
 			return 1;
 			}
 
+		if ( t->Tag() == TYPE_VECTOR && et->Tag() == TYPE_VECTOR &&
+		     et->AsVectorType()->IsUnspecifiedVector() )
+			{
+			e = new VectorCoerceExpr(e, t->AsVectorType());
+			return 1;
+			}
+
 		t->Error("type clash", e);
 		return 0;
 		}

src/Expr.h

@@ -40,7 +40,10 @@ typedef enum {
 	EXPR_CALL,
 	EXPR_EVENT,
 	EXPR_SCHEDULE,
-	EXPR_ARITH_COERCE, EXPR_RECORD_COERCE, EXPR_TABLE_COERCE,
+	EXPR_ARITH_COERCE,
+	EXPR_RECORD_COERCE,
+	EXPR_TABLE_COERCE,
+	EXPR_VECTOR_COERCE,
 	EXPR_SIZE,
 	EXPR_FLATTEN,
 #define NUM_EXPRS (int(EXPR_FLATTEN) + 1)
@@ -895,6 +898,20 @@ protected:
 	DECLARE_SERIAL(TableCoerceExpr);
 };
 
+class VectorCoerceExpr : public UnaryExpr {
+public:
+	VectorCoerceExpr(Expr* op, VectorType* v);
+	~VectorCoerceExpr();
+
+protected:
+	friend class Expr;
+	VectorCoerceExpr()	{ }
+
+	Val* Fold(Val* v) const;
+
+	DECLARE_SERIAL(VectorCoerceExpr);
+};
+
 // An internal operator for flattening array indices that are records
 // into a list of individual values.
 class FlattenExpr : public UnaryExpr {

src/FTP.cc

@@ -10,7 +10,6 @@
 #include "FTP.h"
 #include "NVT.h"
 #include "Event.h"
-#include "TCP_Rewriter.h"
 
 FTP_Analyzer::FTP_Analyzer(Connection* conn)
 : TCP_ApplicationAnalyzer(AnalyzerTag::FTP, conn)
@@ -169,5 +168,3 @@ void FTP_Analyzer::DeliverStream(int length, const u_char* data, bool orig)
 	ConnectionEvent(f, vl);
 	}
 
-
-#include "ftp-rw.bif.func_def"

src/FTP.h

@@ -14,11 +14,6 @@ public:
 
 	virtual void Done();
 	virtual void DeliverStream(int len, const u_char* data, bool orig);
-	virtual int RewritingTrace()
-		{
-		return BifConst::rewriting_ftp_trace ||
-			TCP_ApplicationAnalyzer::RewritingTrace();
-		}
 
 	static Analyzer* InstantiateAnalyzer(Connection* conn)
 		{

src/Finger.cc

@@ -9,7 +9,6 @@
 #include "NetVar.h"
 #include "Finger.h"
 #include "Event.h"
-#include "TCP_Rewriter.h"
 #include "ContentLine.h"
 
 Finger_Analyzer::Finger_Analyzer(Connection* conn)
@@ -42,7 +41,7 @@ void Finger_Analyzer::DeliverStream(int length, const u_char* data, bool is_orig
 
 	if ( is_orig )
 		{
-		
+
 		if ( ! finger_request )
 			return;
 
@@ -87,5 +86,3 @@ void Finger_Analyzer::DeliverStream(int length, const u_char* data, bool is_orig
 		ConnectionEvent(finger_reply, vl);
 		}
 	}
-
-#include "finger-rw.bif.func_def"

src/Finger.h

@@ -17,8 +17,6 @@ public:
 	virtual void Done();
 	// Line-based input.
 	virtual void DeliverStream(int len, const u_char* data, bool orig);
-	virtual int RewritingTrace()
-		{ return BifConst::rewriting_finger_trace || TCP_ApplicationAnalyzer::RewritingTrace(); }
 
 	static Analyzer* InstantiateAnalyzer(Connection* conn)
 		{ return new Finger_Analyzer(conn); }

src/Func.cc

@@ -239,11 +239,15 @@ BroFunc::BroFunc(ID* arg_id, Stmt* arg_body, id_list* aggr_inits,
 : Func(BRO_FUNC)
 	{
 	id = arg_id;
-	Body b;
-	b.stmts = AddInits(arg_body, aggr_inits);
-	b.priority = 0;
-	bodies.push_back(b);
 	frame_size = arg_frame_size;
+
+	if ( arg_body )
+		{
+		Body b;
+		b.stmts = AddInits(arg_body, aggr_inits);
+		b.priority = 0;
+		bodies.push_back(b);
+		}
 	}
 
 BroFunc::~BroFunc()
@@ -267,6 +271,13 @@ Val* BroFunc::Call(val_list* args, Frame* parent) const
 #ifdef PROFILE_BRO_FUNCTIONS
 	DEBUG_MSG("Function: %s\n", id->Name());
 #endif
+	if ( ! bodies.size() ) 
+		{
+		// Can only happen for events.
+		assert(IsEvent());
+		return 0 ;
+		}
+
 	SegmentProfiler(segment_logger, location);
 	Frame* f = new Frame(frame_size, this, args);
 
@@ -497,17 +508,11 @@ void builtin_run_time(const char* msg, BroObj* arg)
 	}
 
 #include "bro.bif.func_h"
-
+#include "logging.bif.func_h"
-#include "common-rw.bif.func_h"
-#include "finger-rw.bif.func_h"
-#include "ftp-rw.bif.func_h"
-#include "http-rw.bif.func_h"
-#include "ident-rw.bif.func_h"
-#include "smtp-rw.bif.func_h"
 #include "strings.bif.func_h"
-#include "dns-rw.bif.func_h"
 
 #include "bro.bif.func_def"
+#include "logging.bif.func_def"
 #include "strings.bif.func_def"
 
 void init_builtin_funcs()
@@ -519,15 +524,8 @@ void init_builtin_funcs()
 	gap_info = internal_type("gap_info")->AsRecordType();
 
 #include "bro.bif.func_init"
-
+#include "logging.bif.func_init"
-#include "common-rw.bif.func_init"
-#include "finger-rw.bif.func_init"
-#include "ftp-rw.bif.func_init"
-#include "http-rw.bif.func_init"
-#include "ident-rw.bif.func_init"
-#include "smtp-rw.bif.func_init"
 #include "strings.bif.func_init"
-#include "dns-rw.bif.func_init"
 
 	did_builtin_init = true;
 	}

src/Func.h

@@ -15,6 +15,7 @@ class FuncType;
 class Stmt;
 class Frame;
 class ID;
+class CallExpr;
 
 class Func : public BroObj {
 public:
@@ -36,7 +37,8 @@ public:
 			{ return priority > other.priority; } // reverse sort
 	};
 
-	virtual const vector<Body>& GetBodies() const	{ return bodies; }
+	const vector<Body>& GetBodies() const	{ return bodies; }
+	bool HasBodies() const	{ return bodies.size(); }
 
 	// virtual Val* Call(ListExpr* args) const = 0;
 	virtual Val* Call(val_list* args, Frame* parent = 0) const = 0;

src/H3.h

@@ -52,8 +52,8 @@
 //     together to get the same result as hashing the full string.
 // Any number of hash functions can be created by creating new instances of H3,
 //     with the same or different template parameters.  The hash function is
-//     randomly generated using random(); you must call srandom() before the
+//     randomly generated using bro_random(); you must call init_random_seed()
-//     H3 constructor if you wish to seed it.
+//     before the H3 constructor if you wish to seed it.
 
 
 #ifndef H3_H
@@ -96,7 +96,7 @@ H3<T,N>::H3()
 	bit_lookup[bit] = 0;
 	for (size_t i = 0; i < sizeof(T)/2; i++) {
 	    // assume random() returns at least 16 random bits
-	    bit_lookup[bit] = (bit_lookup[bit] << 16) | (random() & 0xFFFF);
+	    bit_lookup[bit] = (bit_lookup[bit] << 16) | (bro_random() & 0xFFFF);
 	}
     }
 

src/HTTP.cc

@@ -12,7 +12,6 @@
 #include "HTTP.h"
 #include "Event.h"
 #include "MIME.h"
-#include "TCP_Rewriter.h"
 
 const bool DEBUG_http = false;
 
@@ -1756,5 +1755,3 @@ BroString* unescape_URI(const u_char* line, const u_char* line_end,
 
 	return new BroString(1, decoded_URI, URI_p - decoded_URI);
 	}
-
-#include "http-rw.bif.func_def"

src/HTTP.h

@@ -169,8 +169,6 @@ public:
 	virtual void Done();
 	virtual void DeliverStream(int len, const u_char* data, bool orig);
 	virtual void Undelivered(int seq, int len, bool orig);
-	virtual int RewritingTrace()
-		{ return BifConst::rewriting_http_trace || TCP_ApplicationAnalyzer::RewritingTrace(); }
 
 	// Overriden from TCP_ApplicationAnalyzer
 	virtual void EndpointEOF(bool is_orig);

src/Hash.cc

@@ -21,202 +21,14 @@
 
 #include "Hash.h"
 
-// Define *one* of the following as the universal hash function family to use.
-
-// #define USE_DIETZFELBINGER	// TwoWise
-#define USE_H3
-// #define USE_UHASH_CW
-// #define USE_UMAC_NH
-
-int hash_cnt_all = 0, hash_cnt_uhash = 0;
-
-#if defined(USE_DIETZFELBINGER)
-
-#include "TwoWise.h"
-const TwoWise* two_wise = 0;
-
-#elif defined(USE_H3)
-
 #include "H3.h"
 const H3<hash_t, UHASH_KEY_SIZE>* h3;
 
-#elif defined(USE_UHASH_CW)
-
-// The Carter-Wegman family of universal hash functions.
-// f(x) = (sum(a_i * x_i) mod p) mod N
-// where p is a prime number between N and 2N.
-// Here N = 2^32.
-
-class UHashCW {
-	typedef uint32 word_t;
-public:
-	UHashCW(int arg_max_key_size)
-		{
-		max_num_words = (arg_max_key_size + sizeof(word_t) - 1) /
-				sizeof(word_t);
-
-		a = new word_t[max_num_words + 1];
-		x = new word_t[max_num_words + 1];
-
-		for ( int i = 0; i < max_num_words + 1; ++i )
-			a[i] = rand32bit();
-
-		b = rand64bit();
-		}
-
-	~UHashCW()
-		{
-		delete [] a;
-		delete [] x;
-		}
-
-	uint32 hash(int len, const u_char* data) const
-		{
-		int xlen = (len + sizeof(word_t) - 1) / sizeof(word_t);
-		ASSERT(xlen <= max_num_words);
-
-		x[xlen] = 0;
-		x[xlen-1] = 0;	// pad with 0
-
-		memcpy(static_cast<void *>(x), data, len);
-
-		uint64 h = b;
-		for ( int i = 0; i < xlen; ++i )
-			h += (static_cast<uint64>(x[i]) * a[i]);
-
-		h += static_cast<uint64>(len) * a[xlen];
-
-		// h = h % kPrime
-		//
-		// Here we use a trick given that h is a Mersenne prime:
-		//
-		// Let K = 2^61. Let h = a * K + b.
-		// Thus, h = a * (K-1) + (a + b).
-
-		h = (h & kPrime) + (h >> 61);
-		if ( h >= kPrime )
-			h -= kPrime;
-
-		// h = h % 2^32
-		return static_cast<uint32>(0xffffffffUL & h);
-		}
-
-protected:
-	static const uint64 kPrime = (static_cast<uint64>(1) << 61) - 1;
-
-	int max_num_words;
-	word_t* a;
-	uint64 b;
-	word_t* x;
-};
-
-const UHashCW* uhash_cw = 0;
-
-#elif defined(USE_UMAC_NH)
-
-// Use the NH hash function proposed in UMAC.
-// (See http://www.cs.ucdavis.edu/~rogaway/umac/)
-//
-// Essentially, it is computed as:
-//
-//	H = (x_0 +_16 k_0) * (x_1 +_16 k_1) +
-//		(x_2 +_16 k_2) * (x_3 +_16 k_3) + ...
-//
-// where {k_i} are keys for universal hashing,
-// {x_i} are data words, and +_16 means plus mod 2^16.
-//
-// This is faster than UHASH_CW because no modulo operation
-// is needed.  But note that it is 2^-16 universal, while the
-// other universal functions are (almost) 2^-32 universal.
-//
-// Note: UMAC now has a code release under a BSD-like license, and we may want
-// to consider using it instead of our home-grown code.
-
-#ifndef DEBUG
-#error "UMAC/NH is experimental code."
-#endif
-
-class UMacNH {
-	// NH uses 16-bit words
-	typedef uint16 word_t;
-public:
-	UMacNH(int arg_max_key_size)
-		{
-		max_num_words = (arg_max_key_size + sizeof(word_t) - 1) /
-				sizeof(word_t);
-
-		// Make max_num_words 2n+1
-		if ( max_num_words % 2 == 0 )
-			++max_num_words;
-
-		a = new word_t[max_num_words + 1];
-		x = new word_t[max_num_words + 1];
-
-		for ( int i = 0; i < max_num_words + 1; ++i )
-			a[i] = rand16bit();
-		}
-
-	~UMacNH()
-		{
-		delete [] a;
-		delete [] x;
-		}
-
-	uint32 hash(int len, const u_char* data) const
-		{
-		int xlen = (len + sizeof(word_t) - 1) / sizeof(word_t);
-		if ( xlen % 2 == 0 )
-			++xlen;
-
-		ASSERT(xlen <= max_num_words);
-
-		x[xlen] = len;
-		x[xlen-1] = 0;	// pad with 0
-		if ( xlen >= 2 )
-			x[xlen-2] = 0;
-
-		memcpy(static_cast<void *>(x), data, len);
-
-		uint32 h = 0;
-		for ( int i = 0; i <= xlen; i += 2 )
-			h += (static_cast<uint32>(x[i] + a[i]) *
-			      static_cast<uint32>(x[i+1] + a[i+1]));
-
-		return h;
-		}
-
-protected:
-	int max_num_words;
-	word_t* a;
-	word_t* x;
-};
-
-const UMacNH* umac_nh = 0;
-
-#else
-
-#ifdef DEBUG
-#error "No universal hash function is used."
-#endif
-
-#endif
-
 void init_hash_function()
 	{
 	// Make sure we have already called init_random_seed().
 	ASSERT(hmac_key_set);
-
-	// Both Dietzfelbinger and H3 use random() to generate keys
-	// -- is it strong enough?
-#if defined(USE_DIETZFELBINGER)
-	two_wise = new TwoWise((UHASH_KEY_SIZE + 3) >> 2);
-#elif defined(USE_H3)
 	h3 = new H3<hash_t, UHASH_KEY_SIZE>();
-#elif defined(USE_UHASH_CW)
-	uhash_cw = new UHashCW(UHASH_KEY_SIZE);
-#elif defined(USE_UMAC_NH)
-	umac_nh = new UMacNH(UHASH_KEY_SIZE);
-#endif
 	}
 
 HashKey::HashKey(bro_int_t i)
@@ -354,24 +166,11 @@ void* HashKey::CopyKey(const void* k, int s) const
 
 hash_t HashKey::HashBytes(const void* bytes, int size)
 	{
-	++hash_cnt_all;
-
 	if ( size <= UHASH_KEY_SIZE )
 		{
 		const uint8* b = reinterpret_cast<const uint8*>(bytes);
-		++hash_cnt_uhash;
-#if defined(USE_DIETZFELBINGER)
-		return two_wise->Hash(size, b);
-#elif defined(USE_H3)
 		// H3 doesn't check if size is zero
 		return ( size == 0 ) ? 0 : (*h3)(bytes, size);
-#elif defined(USE_UHASH_CW)
-		return uhash_cw->hash(size, b);
-#elif defined(USE_UMAC_NH)
-		return umac_nh->hash(size, b);
-#else
-		--hash_cnt_uhash;
-#endif
 		}
 
 	// Fall back to HMAC/MD5 for longer data (which is usually rare).

src/Hash.h

@@ -86,7 +86,6 @@ protected:
 	int size, hash;
 };
 
-extern int hash_cnt_all, hash_cnt_uhash;
 extern void init_hash_function();
 
 #endif

src/ID.cc

@@ -235,6 +235,25 @@ void ID::UpdateValAttrs()
 				}
 			}
 		}
+
+	if ( Type()->Tag() == TYPE_RECORD )
+		{
+		Attr* attr = attrs->FindAttr(ATTR_LOG);
+		if ( attr )
+			{
+			// Apply &log to all record fields.
+			RecordType* rt = Type()->AsRecordType();
+			for ( int i = 0; i < rt->NumFields(); ++i )
+				{
+				TypeDecl* fd = rt->FieldDecl(i);
+
+				if ( ! fd->attrs )
+					fd->attrs = new Attributes(new attr_list, rt->FieldType(i), true);
+
+				fd->attrs->AddAttr(new Attr(ATTR_LOG));
+				}
+			}
+		}
 	}
 
 void ID::AddAttrs(Attributes* a)
@@ -607,6 +626,135 @@ void ID::DescribeExtended(ODesc* d) const
 		}
 	}
 
+void ID::DescribeReSTShort(ODesc* d) const
+	{
+	if ( is_type )
+		d->Add(":bro:type:`");
+	else
+		d->Add(":bro:id:`");
+
+	d->Add(name);
+	d->Add("`");
+
+	if ( type )
+		{
+		d->Add(": ");
+		d->Add(":bro:type:`");
+
+		if ( ! is_type && type->GetTypeID() )
+			d->Add(type->GetTypeID());
+		else
+			{
+			TypeTag t = type->Tag();
+
+			switch ( t ) {
+			case TYPE_TABLE:
+				d->Add(type->IsSet() ? "set" : type_name(t));
+				break;
+
+			case TYPE_FUNC:
+				d->Add(type->AsFuncType()->IsEvent() ? "event" : type_name(t));
+				break;
+
+			default:
+				d->Add(type_name(t));
+			}
+			}
+
+		d->Add("`");
+		}
+
+	if ( attrs )
+		{
+		d->SP();
+		attrs->DescribeReST(d);
+		}
+	}
+
+void ID::DescribeReST(ODesc* d, bool is_role) const
+	{
+	if ( is_role )
+		{
+		if ( is_type )
+			d->Add(":bro:type:`");
+		else
+			d->Add(":bro:id:`");
+		d->Add(name);
+		d->Add("`");
+		}
+	else
+		{
+		if ( is_type )
+			d->Add(".. bro:type:: ");
+		else
+			d->Add(".. bro:id:: ");
+		d->Add(name);
+		}
+
+	d->PushIndent();
+	d->NL();
+
+	if ( type )
+		{
+		d->Add(":Type: ");
+
+		if ( ! is_type && type->GetTypeID() )
+			{
+			d->Add(":bro:type:`");
+			d->Add(type->GetTypeID());
+			d->Add("`");
+			}
+		else
+			type->DescribeReST(d);
+
+		d->NL();
+		}
+
+	if ( attrs )
+		{
+		d->Add(":Attributes: ");
+		attrs->DescribeReST(d);
+		d->NL();
+		}
+
+	if ( val && type &&
+		type->Tag() != TYPE_FUNC &&
+		type->InternalType() != TYPE_INTERNAL_VOID )
+		{
+		d->Add(":Default:");
+
+		if ( type->InternalType() == TYPE_INTERNAL_OTHER )
+			{
+			switch ( type->Tag() ) {
+			case TYPE_TABLE:
+				if ( val->AsTable()->Length() == 0 )
+					{
+					d->Add(" ``{}``");
+					d->NL();
+					break;
+					}
+				// Fall-through.
+
+			default:
+				d->NL();
+				d->NL();
+				d->Add("::");
+				d->NL();
+				d->PushIndent();
+				val->DescribeReST(d);
+				d->PopIndent();
+			}
+			}
+
+		else
+			{
+			d->SP();
+			val->DescribeReST(d);
+			d->NL();
+			}
+		}
+	}
+
 #ifdef DEBUG
 void ID::UpdateValID()
 	{

src/ID.h

@@ -84,6 +84,9 @@ public:
 	void Describe(ODesc* d) const;
 	// Adds type and value to description.
 	void DescribeExtended(ODesc* d) const;
+	// Produces a description that's reST-ready.
+	void DescribeReST(ODesc* d, bool is_role=false) const;
+	void DescribeReSTShort(ODesc* d) const;
 
 	bool Serialize(SerialInfo* info) const;
 	static ID* Unserialize(UnserialInfo* info);

src/Ident.cc

@@ -9,7 +9,6 @@
 #include "NetVar.h"
 #include "Ident.h"
 #include "Event.h"
-#include "TCP_Rewriter.h"
 
 Ident_Analyzer::Ident_Analyzer(Connection* conn)
 : TCP_ApplicationAnalyzer(AnalyzerTag::Ident, conn)
@@ -245,6 +244,3 @@ void Ident_Analyzer::BadReply(int length, const char* line)
 		did_bad_reply = 1;
 		}
 	}
-
-#include "ident-rw.bif.func_def"
-

src/Ident.h

@@ -14,11 +14,6 @@ public:
 	virtual void Done();
 
 	virtual void DeliverStream(int length, const u_char* data, bool is_orig);
-	virtual int RewritingTrace()
-		{
-		return BifConst::rewriting_ident_trace ||
-			TCP_ApplicationAnalyzer::RewritingTrace();
-		}
 
 	static Analyzer* InstantiateAnalyzer(Connection* conn)
 		{ return new Ident_Analyzer(conn); }

src/LogMgr.h

@@ -0,0 +1,133 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+//
+// A class managing log writers and filters.
+
+#ifndef LOGMGR_H
+#define LOGMGR_H
+
+#include "Val.h"
+#include "EventHandler.h"
+#include "RemoteSerializer.h"
+
+class SerializationFormat;
+
+// Description of a log field.
+struct LogField {
+	string name;
+	TypeTag type;
+
+	LogField()	{ }
+	LogField(const LogField& other)
+		: name(other.name), type(other.type)	{ }
+
+	// (Un-)serialize.
+	bool Read(SerializationFormat* fmt);
+	bool Write(SerializationFormat* fmt) const;
+};
+
+// Values as logged by a writer.
+struct LogVal {
+	TypeTag type;
+	bool present; // False for unset fields.
+
+	// The following union is a subset of BroValUnion, including only the
+	// types we can log directly.
+	struct set_t { bro_int_t size; LogVal** vals; };
+	typedef set_t vec_t;
+
+	union _val {
+		bro_int_t int_val;
+		bro_uint_t uint_val;
+		uint32 addr_val[NUM_ADDR_WORDS];
+		subnet_type subnet_val;
+		double double_val;
+		string* string_val;
+		set_t set_val;
+		vec_t vector_val;
+	} val;
+
+	LogVal(TypeTag arg_type = TYPE_ERROR, bool arg_present = true)
+		: type(arg_type), present(arg_present)	{}
+	~LogVal();
+
+	// (Un-)serialize.
+	bool Read(SerializationFormat* fmt);
+	bool Write(SerializationFormat* fmt) const;
+
+	// Returns true if the type can be logged the framework. If
+	// `atomic_only` is true, will not permit composite types.
+	static bool IsCompatibleType(BroType* t, bool atomic_only=false);
+
+private:
+	LogVal(const LogVal& other)	{ }
+};
+
+class LogWriter;
+class RemoteSerializer;
+class RotationTimer;
+
+class LogMgr {
+public:
+	LogMgr();
+	~LogMgr();
+
+	// These correspond to the BiFs visible on the scripting layer. The
+	// actual BiFs just forward here.
+	bool CreateStream(EnumVal* id, RecordVal* stream);
+	bool EnableStream(EnumVal* id);
+	bool DisableStream(EnumVal* id);
+	bool AddFilter(EnumVal* id, RecordVal* filter);
+	bool RemoveFilter(EnumVal* id, StringVal* name);
+	bool RemoveFilter(EnumVal* id, string name);
+	bool Write(EnumVal* id, RecordVal* columns);
+	bool SetBuf(EnumVal* id, bool enabled);	// Adjusts all writers.
+	bool Flush(EnumVal* id);		// Flushes all writers..
+
+protected:
+	friend class LogWriter;
+	friend class RemoteSerializer;
+	friend class RotationTimer;
+
+	//// Function also used by the RemoteSerializer.
+
+	// Takes ownership of fields.
+	LogWriter* CreateWriter(EnumVal* id, EnumVal* writer, string path,
+				int num_fields, LogField** fields);
+
+	// Takes ownership of values..
+	bool Write(EnumVal* id, EnumVal* writer, string path,
+		   int num_fields, LogVal** vals);
+
+	// Announces all instantiated writers to peer.
+	void SendAllWritersTo(RemoteSerializer::PeerID peer);
+
+	//// Functions safe to use by writers.
+
+	// Reports an error for the given writer.
+	void Error(LogWriter* writer, const char* msg);
+
+private:
+	struct Filter;
+	struct Stream;
+	struct WriterInfo;
+
+	bool TraverseRecord(Stream* stream, Filter* filter, RecordType* rt,
+			    TableVal* include, TableVal* exclude, string path, list<int> indices);
+
+	LogVal** RecordToFilterVals(Stream* stream, Filter* filter,
+				    RecordVal* columns);
+
+	LogVal* ValToLogVal(Val* val, BroType* ty = 0);
+	Stream* FindStream(EnumVal* id);
+	void RemoveDisabledWriters(Stream* stream);
+	void InstallRotationTimer(WriterInfo* winfo);
+	void Rotate(WriterInfo* info);
+	RecordVal* LookupRotationControl(EnumVal* writer, string path);
+	Filter* FindFilter(EnumVal* id, StringVal* filter);
+
+	vector<Stream *> streams;	// Indexed by stream enum.
+};
+
+extern LogMgr* log_mgr;
+
+#endif

src/LogWriter.cc

@@ -0,0 +1,188 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#include "util.h"
+#include "LogWriter.h"
+
+LogWriter::LogWriter()
+	{
+	buf = 0;
+	buf_len = 1024;
+	buffering = true;
+	disabled = false;
+	}
+
+LogWriter::~LogWriter()
+	{
+	if ( buf )
+		free(buf);
+
+	delete [] fields;
+	}
+
+bool LogWriter::Init(string arg_path, int arg_num_fields,
+		     const LogField* const * arg_fields)
+	{
+	path = arg_path;
+	num_fields = arg_num_fields;
+	fields = arg_fields;
+
+	if ( ! DoInit(arg_path, arg_num_fields, arg_fields) )
+		{
+		disabled = true;
+		return false;
+		}
+
+	return true;
+	}
+
+bool LogWriter::Write(int arg_num_fields, LogVal** vals)
+	{
+	// Double-check that the arguments match. If we get this from remote,
+	// something might be mixed up.
+	if ( num_fields != arg_num_fields )
+		{
+		DBG_LOG(DBG_LOGGING, "Number of fields don't match in LogWriter::Write() (%d vs. %d)",
+			arg_num_fields, num_fields);
+
+		return false;
+		}
+
+	for ( int i = 0; i < num_fields; ++i )
+		{
+		if ( vals[i]->type != fields[i]->type )
+			{
+			DBG_LOG(DBG_LOGGING, "Field type doesn't match in LogWriter::Write() (%d vs. %d)",
+				vals[i]->type, fields[i]->type);
+			return false;
+			}
+		}
+
+	bool result = DoWrite(num_fields, fields, vals);
+
+	DeleteVals(vals);
+
+	if ( ! result )
+		disabled = true;
+
+	return result;
+	}
+
+bool LogWriter::SetBuf(bool enabled)
+	{
+	if ( enabled == buffering )
+		// No change.
+		return true;
+
+	buffering = enabled;
+
+	if ( ! DoSetBuf(enabled) )
+		{
+		disabled = true;
+		return false;
+		}
+
+	return true;
+	}
+
+bool LogWriter::Rotate(string rotated_path, string postprocessor, double open,
+		       double close, bool terminating)
+	{
+	if ( ! DoRotate(rotated_path, postprocessor, open, close, terminating) )
+		{
+		disabled = true;
+		return false;
+		}
+
+	return true;
+	}
+
+bool LogWriter::Flush()
+	{
+	if ( ! DoFlush() )
+		{
+		disabled = true;
+		return false;
+		}
+
+	return true;
+	}
+
+void LogWriter::Finish()
+	{
+	DoFinish();
+	}
+
+const char* LogWriter::Fmt(const char* format, ...)
+	{
+	if ( ! buf )
+		buf = (char*) malloc(buf_len);
+
+	va_list al;
+	va_start(al, format);
+	int n = safe_vsnprintf(buf, buf_len, format, al);
+	va_end(al);
+
+	if ( (unsigned int) n >= buf_len )
+		{ // Not enough room, grow the buffer.
+		buf_len = n + 32;
+		buf = (char*) realloc(buf, buf_len);
+
+		// Is it portable to restart?
+		va_start(al, format);
+		n = safe_vsnprintf(buf, buf_len, format, al);
+		va_end(al);
+		}
+
+	return buf;
+	}
+
+void LogWriter::Error(const char *msg)
+	{
+	log_mgr->Error(this, msg);
+	}
+
+void LogWriter::DeleteVals(LogVal** vals)
+	{
+	for ( int i = 0; i < num_fields; i++ )
+		delete vals[i];
+	}
+
+bool LogWriter::RunPostProcessor(string fname, string postprocessor,
+				 string old_name, double open, double close,
+				 bool terminating)
+	{
+	// This function operates in a way that is backwards-compatible with
+	// the old Bro log rotation scheme.
+
+	if ( ! postprocessor.size() )
+		return true;
+
+	const char* const fmt = "%y-%m-%d_%H.%M.%S";
+
+	struct tm tm1;
+	struct tm tm2;
+
+	time_t tt1 = (time_t)open;
+	time_t tt2 = (time_t)close;
+
+	localtime_r(&tt1, &tm1);
+	localtime_r(&tt2, &tm2);
+
+	char buf1[128];
+	char buf2[128];
+
+	strftime(buf1, sizeof(buf1), fmt, &tm1);
+	strftime(buf2, sizeof(buf2), fmt, &tm2);
+
+	string cmd = postprocessor;
+	cmd += " " + fname;
+	cmd += " " + old_name;
+	cmd += " " + string(buf1);
+	cmd += " " + string(buf2);
+	cmd += " " + string(terminating ? "1" : "0");
+	cmd += " &";
+
+	system(cmd.c_str());
+
+	return true;
+	}

src/LogWriter.h

@@ -0,0 +1,188 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+//
+// Interface API for a log writer backend. The LogMgr creates a separate
+// writer instance of pair of (writer type, output path).
+//
+// Note thay classes derived from LogWriter must be fully thread-safe and not
+// use any non-thread-safe Bro functionality (which includes almost
+// everything ...). In particular, do not use fmt() but LogWriter::Fmt()!.
+//
+// The one exception to this rule is the constructor: it is guaranteed to be
+// executed inside the main thread and can thus in particular access global
+// script variables.
+
+#ifndef LOGWRITER_H
+#define LOGWRITER_H
+
+#include "LogMgr.h"
+#include "BroString.h"
+
+class LogWriter {
+public:
+	LogWriter();
+	virtual ~LogWriter();
+
+	//// Interface methods to interact with the writer. Note that these
+	//// methods are not necessarily thread-safe and must be called only
+	//// from the main thread (which will typically mean only from the
+	//// LogMgr). In particular, they must not be called from the
+	//// writer's derived implementation.
+
+	// One-time initialization of the writer to define the logged fields.
+	// Interpretation of "path" is left to the writer, and will be
+	// corresponding the value configured on the script-level.
+	// 
+	// Returns false if an error occured, in which case the writer must
+	// not be used further.
+	//
+	// The new instance takes ownership of "fields", and will delete them
+	// when done.
+	bool Init(string path, int num_fields, const LogField* const * fields);
+
+	// Writes one log entry. The method takes ownership of "vals" and
+	// will return immediately after queueing the write request, which is
+	// potentially before output has actually been written out.
+	//
+	// num_fields and the types of the LogVals must match what was passed
+	// to Init().
+	//
+	// Returns false if an error occured, in which case the writer must
+	// not be used any further.
+	bool Write(int num_fields, LogVal** vals);
+
+	// Sets the buffering status for the writer, if the writer supports
+	// that. (If not, it will be ignored).
+	bool SetBuf(bool enabled);
+
+	// Flushes any currently buffered output, if the writer supports 
+	// that. (If not, it will be ignored).
+	bool Flush();
+
+	// Triggers rotation, if the writer supports that. (If not, it will
+	// be ignored).
+	bool Rotate(string rotated_path, string postprocessor, double open,
+		    double close, bool terminating);
+
+	// Finishes writing to this logger regularly. Must not be called if
+	// an error has been indicated earlier. After calling this, no
+	// further writing must be performed.
+	void Finish();
+
+	//// Thread-safe methods that may be called from the writer
+	//// implementation.
+
+	// The following methods return the information as passed to Init().
+	const string Path() const	{ return path; }
+	int NumFields() const	{ return num_fields; }
+	const LogField* const * Fields() const	{ return fields; }
+
+protected:
+
+	// Methods for writers to override. If any of these returs false, it
+	// will be assumed that a fatal error has occured that prevents the
+	// writer from further operation. It will then be disabled and
+	// deleted. When return false, the writer should also report the
+	// error via Error(). Note that even if a writer does not support the
+	// functionality for one these methods (like rotation), it must still
+	// return true if that is not to be considered a fatal error.
+	//
+	// Called once for initialization of the writer.
+	virtual bool DoInit(string path, int num_fields,
+			    const LogField* const * fields) = 0;
+
+	// Called once per log entry to record.
+	virtual bool DoWrite(int num_fields, const LogField* const * fields,
+			     LogVal** vals) = 0;
+
+	// Called when the buffering status for this writer is changed. If
+	// buffering is disabled, the writer should attempt to write out
+	// information as quickly as possible even if doing so may have a
+	// performance impact. If enabled (which is the default), it may
+	// buffer data as helpful and write it out later in a way optimized
+	// for performance. The current buffering state can be queried via
+	// IsBuf().
+	//
+	// A writer may ignore buffering changes if it doesn't fit with its
+	// semantics (but must still return true in that case).
+	virtual bool DoSetBuf(bool enabled) = 0;
+
+	// Called to flush any currently buffered output.
+	//
+	// A writer may ignore flush requests if it doesn't fit with its
+	// semantics (but must still return true in that case).
+	virtual bool DoFlush() = 0;
+
+	// Called when a log output is to be rotated. Most directly this only
+	// applies to writers writing into files, which should then close the
+	// current file and open a new one.  However, a writer may also
+	// trigger other apppropiate actions if semantics are similar.
+	//
+	// "rotate_path" reflects the path to where the rotated output is to
+	// be moved, with specifics depending on the writer. It should
+	// generally be interpreted in a way consistent with that of "path"
+	// as passed into DoInit(). As an example, for file-based output, 
+	// "rotate_path" could be the original filename extended with a
+	// timestamp indicating the time of the rotation.
+
+	// "postprocessor" is the name of a command to execute on the rotated
+	// file. If empty, no postprocessing should take place; if given but
+	// the writer doesn't support postprocessing, it can be ignored (but
+	// the method must still return true in that case).
+
+	// "open" and "close" are the network time's when the *current* file
+	// was opened and closed, respectively.
+	//
+	// "terminating" indicated whether the rotation request occurs due
+	// the main Bro prcoess terminating (and not because we've reach a
+	// regularly scheduled time for rotation).
+	//
+	// A writer may ignore rotation requests if it doesn't fit with its
+	// semantics (but must still return true in that case).
+	virtual bool DoRotate(string rotated_path, string postprocessor,
+			      double open, double close, bool terminating) = 0;
+
+	// Called once on termination. Not called when any of the other
+	// methods has previously signaled an error, i.e., executing this
+	// method signals a regular shutdown of the writer.
+	virtual void DoFinish() = 0;
+
+	//// Methods for writers to use. These are thread-safe.
+
+	// A thread-safe version of fmt().
+	const char* Fmt(const char* format, ...);
+
+	// Returns the current buffering state.
+	bool IsBuf()	{ return buffering; }
+
+	// Reports an error to the user.
+	void Error(const char *msg);
+
+	// Runs a post-processor on the given file. Parameters correspond to
+	// those of DoRotate(). 
+	bool RunPostProcessor(string fname, string postprocessor,
+			      string old_name, double open, double close,
+			      bool terminating);
+
+private:
+	friend class LogMgr;
+
+	// When an error occurs, we call this method to set a flag marking
+	// the writer as disabled. The LogMgr will check the flag later and
+	// remove the writer.
+	bool Disabled()	{ return disabled; }
+
+	// Deletes the values as passed into Write().
+	void DeleteVals(LogVal** vals);
+
+	string path;
+	int num_fields;
+	const LogField* const * fields;
+	bool buffering;
+	bool disabled;
+
+	// For implementing Fmt().
+	char* buf;
+	unsigned int buf_len;
+};
+
+#endif