Merge branch 'master' into topic/gilbert/rand-pool

CHANGES

@@ -1,4 +1,66 @@
 
+1.6-dev-1095 | 2011-08-13 11:59:07 -0700
+
+  * A larger number of script documentation updates. Closes #543. (Jon
+    Siwek)
+
+  * Workaround for FreeBSD CMake port missing debug flags. (Jon Siwek)
+    
+  * piped_exec() can now deal with null bytes. (Seth Hall)
+    
+  * Fix vector initialization for lists of records with optional
+    types. Closes #485. (Jon Siwek)
+
+  * Fix redef'ing records with &default empty set fields. Closes #460.
+    (Jon Siwek)
+
+  * Fix ConnSize_Analyzer when used in conjunction with the connection
+    compressor. (Gregor Maier)
+
+  * Fix reporter using part of the actual message as a format string.
+    (Jon Siwek)
+
+  * Fixing reporter's location tracking. Closes #492. (Robin Sommer)
+    
+  * Turning DNS errors into warnings. Closes #255. (Robin Sommer)
+    
+  * Logging's path_func now receives the log record as argument.
+    Closes #555. (Robin Sommer)
+
+  * Functions can now be logged; their full body gets recorded. 
+    Closes #506. (Robin Sommer)
+
+  * Bugfix for hostname notice email extension. (Seth Hall)
+    
+  * Updates for notice framework. (Seth Hall)
+    
+    - New ACTION_ADD_GEODATA to add geodata to notices in an extension
+      field named remote_location.
+
+    - Loading extend-email/hostnames by default now that it only does
+      anything when the ACTION_EMAIL action is applied (finally).
+
+  * Updates to local.bro (Seth Hall)
+
+  * Added the profiling script. (Seth Hall)
+    
+  * Updates for SSH scripts. (Seth Hall)
+
+  * ConnSize analyzer is turned on by default now. (Seth Hall)
+
+  * Updates for the build system and site local scripts for cluster.
+    (Seth Hall)
+
+  * HTTP now uses the extract_filename_from_content_disposition function. (Seth Hall)
+
+  * Major SMTP script refactor. Closes #509. (Jon Siwek and Seth Hall)
+    
+  * New variable Site::local_nets_table in utils/site for mapping
+    address to defined local subnet.
+
+  * Metrics framework updates, more to come. (Seth Hall)
+    
+
 1.6-dev-1061 | 2011-08-08 18:25:27 -0700
 
   * A set of new/changed tests regarding the new policy script

CMakeLists.txt

@@ -35,14 +35,16 @@ endif ()
 ## Project/Build Configuration
 
 set(BRO_ROOT_DIR ${CMAKE_INSTALL_PREFIX})
-if (NOT POLICYDIR)
+if (NOT BRO_SCRIPT_INSTALL_PATH)
-    # set the default policy installation path (user did not specify one)
+    # set the default Bro script installation path (user did not specify one)
-    set(POLICYDIR ${BRO_ROOT_DIR}/share/bro)
+    set(BRO_SCRIPT_INSTALL_PATH ${BRO_ROOT_DIR}/share/bro)
 endif ()
+set(BRO_SCRIPT_SOURCE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/scripts)
 
-# sanitize the policy install directory into an absolute path
+# sanitize the Bro script install directory into an absolute path
 # (CMake is confused by ~ as a representation of home directory)
-get_filename_component(POLICYDIR ${POLICYDIR} ABSOLUTE)
+get_filename_component(BRO_SCRIPT_INSTALL_PATH ${BRO_SCRIPT_INSTALL_PATH}
+    ABSOLUTE)
 
 configure_file(bro-path-dev.in ${CMAKE_CURRENT_BINARY_DIR}/bro-path-dev)
 file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/bro-path-dev.sh
@@ -63,7 +65,8 @@ set(EXTRA_COMPILE_FLAGS "-Wall -Wno-unused")
 
 if (ENABLE_DEBUG)
     set(CMAKE_BUILD_TYPE Debug)
-    set(EXTRA_COMPILE_FLAGS "${EXTRA_COMPILE_FLAGS} -DDEBUG")
+    # manual add of -g works around its omission in FreeBSD's CMake port
+    set(EXTRA_COMPILE_FLAGS "${EXTRA_COMPILE_FLAGS} -g -DDEBUG")
 else ()
     set(CMAKE_BUILD_TYPE RelWithDebInfo)
 endif ()
@@ -212,7 +215,7 @@ message(
     "\n====================|  Bro Build Summary  |====================="
     "\n"
     "\nInstall prefix:    ${CMAKE_INSTALL_PREFIX}"
-    "\nPolicy dir:        ${POLICYDIR}"
+    "\nBro Script Path:   ${BRO_SCRIPT_INSTALL_PATH}"
     "\nDebug mode:        ${ENABLE_DEBUG}"
     "\n"
     "\nCC:                ${CMAKE_C_COMPILER}"

VERSION

@@ -1 +1 @@
-1.6-dev-1061
+1.6-dev-1095

aux/binpac

@@ -1 +1 @@
-Subproject commit 7cdd9c39d97c2984293fbe4a6dbe9ac0b33ecbfa
+Subproject commit a3a9410dedc842f6bb9859642f334ed354633b57

aux/bro-aux

@@ -1 +1 @@
-Subproject commit 86990f1640d986e39d5bb1287dbeb03b59a464f0
+Subproject commit d68b98bb995a105b257f805ec4ff22c4929c7476

aux/broccoli

@@ -1 +1 @@
-Subproject commit 45f577240089d63dd0dc58be564280725a97acec
+Subproject commit 03e6d398edf422140ba9f50e6fabbec33ee2f3cb

aux/broctl

@@ -1 +1 @@
-Subproject commit dbbe6c81ef40666338c950d8f69dc8597f2adc70
+Subproject commit c39622855e3c3a5cc94c7376f86184ed1db1939a

aux/btest

@@ -1 +1 @@
-Subproject commit ab78a66dd782f165ddf921faaf1f065b2f987481
+Subproject commit d1c620d98ce9d9c0b203314108b413784965d2ed

bro-path-dev.in

@@ -1,7 +1,7 @@
 #!/bin/sh
-# After configured by CMake, this file prints the absolute path to policy
+# After configured by CMake, this file prints the absolute path to Bro scripts
-# files that come with the source distributions of Bro as well as policy 
+# that come with the source distributions of Bro as well as scripts that are
-# files that are generated by the BIF compiler at compile time.
+# generated by the BIF compiler at compile time.
 #
 # The intended use of this script is to make it easier to run Bro from
 # the build directory, avoiding the need to install it.  This could be
@@ -10,10 +10,10 @@
 #     BROPATH=`./bro-path-dev` ./src/bro
 #
 
-broPolicies=${PROJECT_SOURCE_DIR}/scripts:${PROJECT_SOURCE_DIR}/scripts/policy:${PROJECT_SOURCE_DIR}/scripts/site
+broPolicies=${BRO_SCRIPT_SOURCE_PATH}:${BRO_SCRIPT_SOURCE_PATH}/policy:${BRO_SCRIPT_SOURCE_PATH}/site
 
 broGenPolicies=${CMAKE_BINARY_DIR}/src
 
-installedPolicies=${POLICYDIR}:${POLICYDIR}/site
+installedPolicies=${BRO_SCRIPT_INSTALL_PATH}:${BRO_SCRIPT_INSTALL_PATH}/site
 
 echo .:$broPolicies:$broGenPolicies

configure

@@ -22,7 +22,7 @@ Usage: $0 [OPTION]... [VAR=VALUE]...
 
   Installation Directories:
     --prefix=PREFIX        installation directory [/usr/local/bro]
-    --policydir=PATH       policy file installation directory
+    --scriptdir=PATH       root installation directory for Bro scripts
                            [PREFIX/share/bro]
 
   Optional Features:
@@ -85,7 +85,7 @@ CMakeCacheEntries=""
 append_cache_entry CMAKE_INSTALL_PREFIX PATH   /usr/local/bro
 append_cache_entry BRO_ROOT_DIR         PATH   /usr/local/bro
 append_cache_entry PY_MOD_INSTALL_DIR   PATH   /usr/local/bro/lib/broctl
-append_cache_entry POLICYDIR            STRING /usr/local/bro/share/bro
+append_cache_entry BRO_SCRIPT_INSTALL_PATH STRING /usr/local/bro/share/bro
 append_cache_entry ENABLE_DEBUG         BOOL   false
 append_cache_entry BROv6                BOOL   false
 append_cache_entry ENABLE_PERFTOOLS     BOOL   false
@@ -118,13 +118,13 @@ while [ $# -ne 0 ]; do
             append_cache_entry CMAKE_INSTALL_PREFIX PATH   $optarg
             append_cache_entry BRO_ROOT_DIR         PATH   $optarg
             append_cache_entry PY_MOD_INSTALL_DIR   PATH   $optarg/lib/broctl
-            if [ "$user_set_policydir" != "true" ]; then
+            if [ "$user_set_scriptdir" != "true" ]; then
-                append_cache_entry POLICYDIR        STRING $optarg/share/bro
+                append_cache_entry BRO_SCRIPT_INSTALL_PATH STRING $optarg/share/bro
             fi
             ;;
-        --policydir=*)
+        --scriptdir=*)
-            append_cache_entry POLICYDIR            STRING $optarg
+            append_cache_entry BRO_SCRIPT_INSTALL_PATH STRING $optarg
-            user_set_policydir="true"
+            user_set_scriptdir="true"
             ;;
         --enable-debug)
             append_cache_entry ENABLE_DEBUG         BOOL   true

doc/scripts/CMakeLists.txt

@@ -57,10 +57,12 @@ macro(REST_TARGET srcDir broInput)
     get_filename_component(relDstDir ${broInput} PATH)
 
     set(sumTextSrc ${absSrcPath})
+    set(ogSourceFile ${absSrcPath})
     if (${extension} STREQUAL ".bif.bro")
+        set(ogSourceFile ${BIF_SRC_DIR}/${basename})
         # the summary text is taken at configure time, but .bif.bro files
         # may not have been generated yet, so read .bif file instead
-        set(sumTextSrc ${BIF_SRC_DIR}/${basename})
+        set(sumTextSrc ${ogSourceFile})
     endif ()
 
     if (NOT relDstDir)
@@ -124,9 +126,9 @@ macro(REST_TARGET srcDir broInput)
         ARGS -E make_directory ${dstDir}
         COMMAND "${CMAKE_COMMAND}"
         ARGS -E copy ${restFile} ${restOutput}
-        # copy the bro policy script, too
+        # copy the bro or bif script, too
         COMMAND "${CMAKE_COMMAND}"
-        ARGS -E copy ${absSrcPath} ${dstDir}
+        ARGS -E copy ${ogSourceFile} ${dstDir}
         # clean up the build directory
         COMMAND rm
         ARGS -rf .state *.log *.rst
@@ -151,6 +153,8 @@ set(MASTER_PKG_INDEX_TEXT "")
 foreach (pkg ${MASTER_PKG_LIST})
     # strip of the trailing /index for the link name
     get_filename_component(lnktxt ${pkg} PATH)
+    # pretty-up the link name by removing common scripts/ prefix
+    string(REPLACE "scripts/" "" lnktxt "${lnktxt}")
     set(MASTER_PKG_INDEX_TEXT "${MASTER_PKG_INDEX_TEXT}\n   ${lnktxt} <${pkg}>")
 endforeach ()
 file(WRITE ${MASTER_PACKAGE_INDEX} "${MASTER_PKG_INDEX_TEXT}")

doc/scripts/DocSourcesList.cmake

@@ -13,22 +13,14 @@
 set(psd ${PROJECT_SOURCE_DIR}/scripts)
 
 rest_target(${CMAKE_CURRENT_SOURCE_DIR} example.bro internal)
-rest_target(${psd} base/bro.init internal)
+rest_target(${psd} base/init-default.bro internal)
-rest_target(${psd} base/all.bro internal)
+rest_target(${psd} base/init-bare.bro internal)
 
 rest_target(${CMAKE_BINARY_DIR}/src bro.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/src common-rw.bif.bro)
 rest_target(${CMAKE_BINARY_DIR}/src const.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/src dns-rw.bif.bro)
 rest_target(${CMAKE_BINARY_DIR}/src event.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/src finger-rw.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/src ftp-rw.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/src http-rw.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/src ident-rw.bif.bro)
 rest_target(${CMAKE_BINARY_DIR}/src logging.bif.bro)
 rest_target(${CMAKE_BINARY_DIR}/src reporter.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/src smb-rw.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/src smtp-rw.bif.bro)
 rest_target(${CMAKE_BINARY_DIR}/src strings.bif.bro)
 rest_target(${CMAKE_BINARY_DIR}/src types.bif.bro)
 rest_target(${psd} base/frameworks/cluster/main.bro)
@@ -42,9 +34,8 @@ rest_target(${psd} base/frameworks/dpd/main.bro)
 rest_target(${psd} base/frameworks/intel/main.bro)
 rest_target(${psd} base/frameworks/logging/main.bro)
 rest_target(${psd} base/frameworks/logging/writers/ascii.bro)
-rest_target(${psd} base/frameworks/metrics/conn-example.bro)
-rest_target(${psd} base/frameworks/metrics/http-example.bro)
 rest_target(${psd} base/frameworks/metrics/main.bro)
+rest_target(${psd} base/frameworks/notice/actions/add-geodata.bro)
 rest_target(${psd} base/frameworks/notice/actions/drop.bro)
 rest_target(${psd} base/frameworks/notice/actions/email_admin.bro)
 rest_target(${psd} base/frameworks/notice/actions/page.bro)
@@ -56,6 +47,7 @@ rest_target(${psd} base/frameworks/packet-filter/netstats.bro)
 rest_target(${psd} base/frameworks/reporter/main.bro)
 rest_target(${psd} base/frameworks/signatures/main.bro)
 rest_target(${psd} base/frameworks/software/main.bro)
+rest_target(${psd} base/frameworks/time-machine/notice.bro)
 rest_target(${psd} base/protocols/conn/contents.bro)
 rest_target(${psd} base/protocols/conn/inactivity.bro)
 rest_target(${psd} base/protocols/conn/main.bro)
@@ -72,6 +64,13 @@ rest_target(${psd} base/protocols/http/partial-content.bro)
 rest_target(${psd} base/protocols/http/utils.bro)
 rest_target(${psd} base/protocols/irc/dcc-send.bro)
 rest_target(${psd} base/protocols/irc/main.bro)
+rest_target(${psd} base/protocols/mime/base.bro)
+rest_target(${psd} base/protocols/mime/file-extract.bro)
+rest_target(${psd} base/protocols/mime/file-hash.bro)
+rest_target(${psd} base/protocols/mime/file-ident.bro)
+rest_target(${psd} base/protocols/rpc/base.bro)
+rest_target(${psd} base/protocols/smtp/entities-excerpt.bro)
+rest_target(${psd} base/protocols/smtp/entities.bro)
 rest_target(${psd} base/protocols/smtp/main.bro)
 rest_target(${psd} base/protocols/ssh/main.bro)
 rest_target(${psd} base/protocols/ssl/consts.bro)
@@ -95,6 +94,9 @@ rest_target(${psd} policy/frameworks/control/controllee.bro)
 rest_target(${psd} policy/frameworks/control/controller.bro)
 rest_target(${psd} policy/frameworks/dpd/detect-protocols.bro)
 rest_target(${psd} policy/frameworks/dpd/packet-segment-logging.bro)
+rest_target(${psd} policy/frameworks/metrics/conn-example.bro)
+rest_target(${psd} policy/frameworks/metrics/http-example.bro)
+rest_target(${psd} policy/frameworks/metrics/ssl-example.bro)
 rest_target(${psd} policy/frameworks/software/version-changes.bro)
 rest_target(${psd} policy/frameworks/software/vulnerable.bro)
 rest_target(${psd} policy/integration/barnyard2/base.bro)
@@ -102,10 +104,10 @@ rest_target(${psd} policy/integration/barnyard2/event.bro)
 rest_target(${psd} policy/integration/barnyard2/types.bro)
 rest_target(${psd} policy/misc/analysis-groups.bro)
 rest_target(${psd} policy/misc/loaded-scripts.bro)
+rest_target(${psd} policy/misc/profiling.bro)
 rest_target(${psd} policy/misc/trim-trace-file.bro)
 rest_target(${psd} policy/protocols/conn/known-hosts.bro)
 rest_target(${psd} policy/protocols/conn/known-services.bro)
-rest_target(${psd} policy/protocols/conn/scan.bro)
 rest_target(${psd} policy/protocols/dns/auth-addl.bro)
 rest_target(${psd} policy/protocols/dns/detect-external-names.bro)
 rest_target(${psd} policy/protocols/ftp/detect.bro)
@@ -118,8 +120,12 @@ rest_target(${psd} policy/protocols/http/headers.bro)
 rest_target(${psd} policy/protocols/http/software.bro)
 rest_target(${psd} policy/protocols/http/var-extraction-cookies.bro)
 rest_target(${psd} policy/protocols/http/var-extraction-uri.bro)
+rest_target(${psd} policy/protocols/smtp/blocklists.bro)
 rest_target(${psd} policy/protocols/smtp/detect-suspicious-orig.bro)
 rest_target(${psd} policy/protocols/smtp/software.bro)
+rest_target(${psd} policy/protocols/ssh/detect-bruteforcing.bro)
+rest_target(${psd} policy/protocols/ssh/geo-data.bro)
+rest_target(${psd} policy/protocols/ssh/interesting-hostnames.bro)
 rest_target(${psd} policy/protocols/ssh/software.bro)
 rest_target(${psd} policy/protocols/ssl/known-certs.bro)
 rest_target(${psd} policy/protocols/ssl/validate-certs.bro)
@@ -127,4 +133,8 @@ rest_target(${psd} policy/tuning/defaults/packet-fragments.bro)
 rest_target(${psd} policy/tuning/defaults/remove-high-volume-notices.bro)
 rest_target(${psd} policy/tuning/defaults/warnings.bro)
 rest_target(${psd} policy/tuning/track-all-assets.bro)
+rest_target(${psd} site/local-manager.bro)
+rest_target(${psd} site/local-proxy.bro)
+rest_target(${psd} site/local-worker.bro)
 rest_target(${psd} site/local.bro)
+rest_target(${psd} test-all-policy.bro)

doc/scripts/README

@@ -44,7 +44,7 @@ of documentation targets.  This script should be run after adding new
 Bro script source files, and the changes commited to git.
 
 If a script shouldn't have documentation generated for it, there's also a
-blacklist variable that can be maintained in the ``genDocSourcesList.sh``
+blacklist manifest that can be maintained in the ``genDocSourcesList.sh``
 script.
 
 The blacklist can also be used if you want to define a certain grouping for

doc/scripts/genDocSourcesList.sh

@@ -11,8 +11,31 @@
 # Specific scripts can be blacklisted below when e.g. they currently aren't
 # parseable or they just aren't meant to be documented.
 
-blacklist="__load__.bro|test-all.bro|all.bro"
+blacklist ()
-blacklist_addl="hot.conn.bro"
+    {
+    if [[ "$blacklist" == "" ]]; then
+        blacklist="$1"
+    else
+        blacklist="$blacklist|$1"
+    fi
+    }
+
+# files passed into this function are meant to be temporary workarounds
+# because they're not finished or otherwise can't be loaded for some reason
+tmp_blacklist ()
+    {
+    echo "Warning: temporarily blacklisted files named '$1'" 1>&2
+    blacklist $1
+    }
+
+blacklist __load__.bro
+blacklist test-all.bro
+blacklist all.bro
+blacklist init-default.bro
+blacklist init-bare.bro
+
+tmp_blacklist hot.conn.bro
+tmp_blacklist scan.bro
 
 statictext="\
 # DO NOT EDIT
@@ -30,8 +53,8 @@ statictext="\
 set(psd \${PROJECT_SOURCE_DIR}/scripts)
 
 rest_target(\${CMAKE_CURRENT_SOURCE_DIR} example.bro internal)
-rest_target(\${psd} base/bro.init internal)
+rest_target(\${psd} base/init-default.bro internal)
-rest_target(\${psd} base/all.bro internal)
+rest_target(\${psd} base/init-bare.bro internal)
 "
 
 if [[ $# -ge 1 ]]; then
@@ -58,7 +81,7 @@ scriptfiles=`( cd ${sourcedir}/scripts && find . -name \*\.bro | sort )`
 for file in $scriptfiles
 do
     f=${file:2}
-    if [[ (! $f =~ $blacklist) && (! $f =~ $blacklist_addl)  ]]; then
+    if [[ ! $f =~ $blacklist ]]; then
         echo "rest_target(\${psd} $f)" >> $outfile
     fi
 done

doc/scripts/group_index_generator.py

@@ -1,9 +1,8 @@
 #! /usr/bin/env python
 
 # This script automatically generates a reST documents that lists
-# a collection of Bro policy scripts that are "grouped" together.
+# a collection of Bro scripts that are "grouped" together.
-# The summary text (##! comments) of the policy script is embedded
+# The summary text (##! comments) of the script is embedded in the list
-# in the list.
 #
 # 1st argument is the file containing list of groups
 # 2nd argument is the directory containing ${group}_files lists of
@@ -57,6 +56,6 @@ with open(group_list, 'r') as f_group_list:
                     f_group_file.write("\n");
 
             with open(group_file, 'a') as f_group_file:
-                f_group_file.write("\n:doc:`/policy/%s`\n" % doc_names[i])
+                f_group_file.write("\n:doc:`/scripts/%s`\n" % doc_names[i])
                 for line in summary_comments:
                     f_group_file.write("   " + line)

doc/scripts/source/index.rst

@@ -14,7 +14,7 @@ Contents:
    internal
    bifs
    packages
-   policy/index
+   scripts/index
 
 Indices and tables
 ==================

doc/scripts/source/internal.rst

@@ -1,5 +1,5 @@
 .. This is a stub doc to which the build process can append.
 
-Internal Policy Scripts
+Internal Scripts
-=======================
+================
 

doc/scripts/source/packages.rst

@@ -1,11 +1,15 @@
 .. This is a stub doc to which the build process can append.
 
-Policy Script Packages
+Bro Script Packages
-======================
+===================
 
-Bro has the following policy script packages (e.g. collections of related
+Bro has the following script packages (e.g. collections of related scripts in
-policy scripts).  If the package contains a ``__load__.bro`` script, it
+a common directory).  If the package directory contains a ``__load__.bro``
-supports being loaded in mass as a whole directory for convenience.
+script, it supports being loaded in mass as a whole directory for convenience.
+
+Packages/scripts in the ``base/`` directory are all loaded by default, while
+ones in ``policy/`` provide functionality and customization options that are
+more appropriate for users to decide whether they'd like to load it or not.
 
 .. toctree::
    :maxdepth: 1

doc/scripts/source/policy/index.rst

@@ -1,6 +0,0 @@
-Index of All Policy Script Documentation
-========================================
-
-.. toctree::
-   :maxdepth: 1
-

doc/scripts/source/scripts/index.rst

@@ -0,0 +1,6 @@
+Index of All Bro Script Documentation
+=====================================
+
+.. toctree::
+   :maxdepth: 1
+

scripts/CMakeLists.txt

@@ -1,18 +1,27 @@
 include(InstallPackageConfigFile)
 
-install(DIRECTORY ./ DESTINATION ${POLICYDIR} FILES_MATCHING
+install(DIRECTORY ./ DESTINATION ${BRO_SCRIPT_INSTALL_PATH} FILES_MATCHING
-        PATTERN "all.bro" EXCLUDE
+        PATTERN "site/local*" EXCLUDE
-        PATTERN "site/local.bro" EXCLUDE
-        PATTERN "bro.init"
         PATTERN "*.bro"
         PATTERN "*.sig"
-        PATTERN "*.osf"
+        PATTERN "*.fp"
 )
 
-# Install as a config file since the local.bro script is meant to be
+# Install all local* scripts as config files since they are meant to be
 # user modify-able.
 InstallPackageConfigFile(
     ${CMAKE_CURRENT_SOURCE_DIR}/site/local.bro
-    ${POLICYDIR}/site
+    ${BRO_SCRIPT_INSTALL_PATH}/site
     local.bro)
-
+InstallPackageConfigFile(
+    ${CMAKE_CURRENT_SOURCE_DIR}/site/local-manager.bro
+    ${BRO_SCRIPT_INSTALL_PATH}/site
+    local-manager.bro)
+InstallPackageConfigFile(
+    ${CMAKE_CURRENT_SOURCE_DIR}/site/local-proxy.bro
+    ${BRO_SCRIPT_INSTALL_PATH}/site
+    local-proxy.bro)
+InstallPackageConfigFile(
+    ${CMAKE_CURRENT_SOURCE_DIR}/site/local-worker.bro
+    ${BRO_SCRIPT_INSTALL_PATH}/site
+    local-worker.bro)

scripts/base/frameworks/cluster/nodes/manager.bro

@@ -10,6 +10,9 @@
 
 @prefixes += cluster-manager
 
+# Load the script for local site configuration for the manager node.
+@load site/local-manager
+
 ## Turn off remote logging since this is the manager and should only log here.
 redef Log::enable_remote_logging = F;
 

scripts/base/frameworks/cluster/nodes/proxy.bro

@@ -1,6 +1,9 @@
 
 @prefixes += cluster-proxy
 
+# Load the script for local site configuration for proxy nodes.
+@load site/local-proxy
+
 ## The proxy only syncs state; does not forward events.
 redef forward_remote_events = F;
 redef forward_remote_state_changes = T;

scripts/base/frameworks/cluster/nodes/worker.bro

@@ -1,6 +1,9 @@
 
 @prefixes += cluster-worker
 
+# Load the script for local site configuration for the worker nodes.
+@load site/local-worker
+
 ## Don't do any local logging.
 redef Log::enable_local_logging = F;
 

scripts/base/frameworks/cluster/setup-connections.bro

@@ -1,3 +1,6 @@
+@load ./main
+@load base/frameworks/communication/main
+
 module Cluster;
 
 event bro_init() &priority=9

scripts/base/frameworks/logging/main.bro

@@ -33,10 +33,12 @@ export {
 	##
 	## id: The log stream.
 	## path: A suggested path value, which may be either the filter's ``path``
-	## if defined or a fall-back generated internally.
+	##       if defined or a fall-back generated internally.
+	## rec: An instance of the streams's ``columns`` type with its
+	##      fields set to the values to logged.
 	##
 	## Returns: The path to be used for the filter.
-	global default_path_func: function(id: ID, path: string) : string &redef;
+	global default_path_func: function(id: ID, path: string, rec: any) : string &redef;
 
 	## Filter customizing logging.
 	type Filter: record {
@@ -71,7 +73,15 @@ export {
 		## different strings for separate calls, but be careful: it's
 		## easy to flood the disk by returning a new string for each
 		## connection ...
-		path_func: function(id: ID, path: string): string &optional;
+		##
+		## id: The log stream.
+		## path: A suggested path value, which may be either the filter's ``path``
+		##       if defined or a fall-back generated internally.
+		## rec: An instance of the streams's ``columns`` type with its
+		##      fields set to the values to logged.
+		##
+		## Returns: The path to be used for the filter.
+		path_func: function(id: ID, path: string, rec: any): string &optional;
 
 		## Subset of column names to record. If not given, all
 		## columns are recorded.
@@ -160,7 +170,7 @@ function __default_rotation_postprocessor(info: RotationInfo) : bool
 		return default_rotation_postprocessors[info$writer](info);
 	}
 
-function default_path_func(id: ID, path: string) : string
+function default_path_func(id: ID, path: string, rec: any) : string
 	{
 	# TODO for Seth: Do what you want. :)
 	return path;

scripts/base/frameworks/notice/__load__.bro

@@ -6,7 +6,8 @@
 @load ./actions/drop
 @load ./actions/email_admin
 @load ./actions/page
+@load ./actions/add-geodata
 
-# Load the script to add hostnames to emails by default.
+# There shouldn't be any defaul toverhead from loading these since they 
-# NOTE: this exposes a memleak in async DNS lookups.
+# *should* only do anything when notices have the ACTION_EMAIL action applied.
-#@load ./extend-email/hostnames
+@load ./extend-email/hostnames

scripts/base/frameworks/notice/actions/add-geodata.bro

@@ -0,0 +1,47 @@
+##! This script adds geographic location data to notices for the "remote"
+##! host in a connection.  It does make the assumption that one of the 
+##! addresses in a connection is "local" and one is "remote" which is 
+##! probably a safe assumption to make in most cases.  If both addresses
+##! are remote, it will use the $src address.
+
+module Notice;
+
+export {
+	redef enum Action += {
+		## Indicates that the notice should have geodata added for the
+		## "remote" host.  :bro:id:`Site::local_nets` must be defined
+		## in order for this to work.
+		ACTION_ADD_GEODATA
+	};
+	
+	redef record Info += {
+		## If libGeoIP support is built in, notices can have geographic
+		## information attached to them.
+		remote_location: geo_location  &log &optional;
+	};
+	
+	## Notice types which should have the "remote" location looked up.
+	## If GeoIP support is not built in, this does nothing.
+	const lookup_location_types: set[Notice::Type] = {} &redef;
+	
+	## Add a helper to the notice policy for looking up GeoIP data.
+	redef Notice::policy += {
+		[$pred(n: Notice::Info) = { return (n$note in Notice::lookup_location_types); },
+		 $priority = 10],
+	};
+}
+
+# This is handled at a high priority in case other notice handlers 
+# want to use the data.
+event notice(n: Notice::Info) &priority=10
+	{
+	if ( ACTION_ADD_GEODATA in n$actions &&
+	     |Site::local_nets| > 0 &&
+	     ! n?$remote_location )
+		{
+		if ( n?$src && ! Site::is_local_addr(n$src) )
+			n$remote_location = lookup_location(n$src);
+		else if ( n?$dst && ! Site::is_local_addr(n$dst) )
+			n$remote_location = lookup_location(n$dst);
+		}
+	}

scripts/base/frameworks/notice/actions/email_admin.bro

@@ -1,4 +1,3 @@
-
 module Notice;
 
 export {

scripts/base/frameworks/notice/extend-email/hostnames.bro

@@ -8,7 +8,7 @@ event Notice::notice(n: Notice::Info) &priority=10
 		return;
 	
 	# This should only be done for notices that are being sent to email.
-	if ( ACTION_EMAIL !in n$action )
+	if ( ACTION_EMAIL !in n$actions )
 		return;
 		
 	local output = "";

scripts/base/frameworks/notice/weird.bro

@@ -1,4 +1,3 @@
-
 module Weird;
 
 export {

scripts/base/init-bare.bro

@@ -509,7 +509,7 @@ const tunnel_port = 0/udp &redef;
 # packets and IP-level bytes transfered by each endpoint. If
 # true, these values are returned in the connection's endpoint
 # record val.
-const use_conn_size_analyzer = F &redef;
+const use_conn_size_analyzer = T &redef;
 
 const UDP_INACTIVE = 0;
 const UDP_ACTIVE = 1;	# means we've seen something from this endpoint

scripts/base/protocols/http/main.bro

@@ -214,7 +214,7 @@ event http_header(c: connection, is_orig: bool, name: string, value: string) &pr
 			c$http$response_content_length = extract_count(value);
 		else if ( name == "CONTENT-DISPOSITION" &&
 		          /[fF][iI][lL][eE][nN][aA][mM][eE]/ in value )
-			c$http$filename = sub(value, /^.*[fF][iI][lL][eE][nN][aA][mM][eE]=/, "");
+			c$http$filename = extract_filename_from_content_disposition(value);
 		}
 	}
 	

scripts/base/protocols/mime/base.bro

@@ -1,7 +1,7 @@
 ##! The mime script does analysis of MIME encoded messages seen in certain
 ##! protocols (only SMTP and POP3 at the moment).
 
-@load utils/strings
+@load base/utils/strings
 
 module MIME;
 

scripts/base/protocols/mime/file-extract.bro

@@ -1,5 +1,5 @@
-@load protocols/mime/file-ident
+@load ./file-ident
-@load utils/files
+@load base/utils/files
 
 module MIME;
 
@@ -16,11 +16,13 @@ export {
 		extract_file:         bool    &default=F;
 	
 		## Store the file handle here for the file currently being extracted.
-		extraction_file:      file    &optional;
+		extraction_file:      file    &log &optional;
+	};
 
+	redef record State += {
 		## Store a count of the number of files that have been transferred in
 		## this conversation to create unique file names on disk.
-		num_extracted_files:  count   &optional;
+		num_extracted_files:  count   &default=0;
 	};
 }
 
@@ -34,7 +36,7 @@ event mime_segment_data(c: connection, length: count, data: string) &priority=3
 	{
 	if ( c$mime$extract_file && c$mime$content_len == 0 )
 		{
-		local suffix = fmt("%d.dat", ++c$mime$num_extracted_files);
+		local suffix = fmt("%d.dat", ++c$mime_state$num_extracted_files);
 		local fname = generate_extraction_filename(extraction_prefix, c, suffix);
 		c$mime$extraction_file = open(fname);
 		enable_raw_output(c$mime$extraction_file);

scripts/base/protocols/mime/file-hash.bro

@@ -1,4 +1,4 @@
-@load protocols/mime/file-ident
+@load ./file-ident
 
 module MIME;
 

scripts/base/protocols/mime/file-ident.bro

@@ -1,4 +1,4 @@
-@load protocols/mime/base
+@load ./base
 
 module MIME;
 

scripts/base/protocols/smtp/__load__.bro

@@ -1 +1,3 @@
 @load ./main
+@load ./entities
+@load ./entities-excerpt

scripts/base/protocols/smtp/entities-excerpt.bro

@@ -0,0 +1,52 @@
+##! This script is for optionally adding a body excerpt to the SMTP
+##! entities log.
+
+@load ./entities
+
+module SMTP;
+
+export {
+	redef record SMTP::EntityInfo += {
+		## The entity body excerpt.
+		excerpt:    string &log &default="";
+		
+		## Internal tracking to know how much of the body should be included
+		## in the excerpt.
+		excerpt_len: count &optional;
+	};
+	
+	## This is the default value for how much of the entity body should be
+	## included for all MIME entities.
+	const default_entity_excerpt_len = 0 &redef;
+	
+	## This table defines how much of various entity bodies should be 
+	## included in excerpts.
+	const entity_excerpt_len: table[string] of count = {} 
+		&redef
+		&default = default_entity_excerpt_len;
+}
+
+event mime_segment_data(c: connection, length: count, data: string) &priority=-1
+	{
+	if ( ! c?$smtp ) return;
+	
+	if ( c$smtp$current_entity$content_len == 0 )
+		c$smtp$current_entity$excerpt_len = entity_excerpt_len[c$smtp$current_entity$mime_type];
+	}
+
+event mime_segment_data(c: connection, length: count, data: string) &priority=-2
+	{
+	if ( ! c?$smtp ) return;
+	
+	local ent = c$smtp$current_entity;
+	if ( ent$content_len < ent$excerpt_len )
+		{
+		if ( ent$content_len + length < ent$excerpt_len )
+			ent$excerpt = cat(ent$excerpt, data);
+		else
+			{
+			local x_bytes = ent$excerpt_len - ent$content_len;
+			ent$excerpt = cat(ent$excerpt, sub_bytes(data, 1, x_bytes));
+			}
+		}
+	}

scripts/base/protocols/smtp/entities.bro

@@ -0,0 +1,234 @@
+##! Analysis and logging for MIME entities found in SMTP sessions.
+
+@load base/utils/strings
+@load base/utils/files
+@load ./main
+
+module SMTP;
+
+export {
+	redef enum Notice::Type += {
+		## Indicates that an MD5 sum was calculated for a MIME message.
+		MD5,
+	};
+
+	redef enum Log::ID += { SMTP_ENTITIES };
+
+	type EntityInfo: record {
+		## This is the timestamp of when the MIME content transfer began.
+		ts:               time    &log;
+		uid:              string  &log;
+		id:               conn_id &log;
+		## Internally generated "message id" that ties back to the particular
+		## message in the SMTP log where this entity was seen.
+		mid:              string  &log;
+		## The filename seen in the Content-Disposition header.
+		filename:         string  &log &optional;
+		## Track how many bytes of the MIME encoded file have been seen.
+		content_len:      count   &log &default=0;
+		## The mime type of the entity discovered through magic bytes identification.
+		mime_type:        string  &log &optional;
+		
+		## The calculated MD5 sum for the MIME entity.
+		md5:              string  &log &optional;
+		## Optionally calculate the file's MD5 sum.  Must be set prior to the 
+		## first data chunk being see in an event.
+		calc_md5:         bool    &default=F;
+		## This boolean value indicates if an MD5 sum is being calculated 
+		## for the current file transfer.
+		calculating_md5:  bool    &default=F;
+		
+		## Optionally write the file to disk.  Must be set prior to first 
+		## data chunk being seen in an event.
+		extract_file:     bool    &default=F;
+		## Store the file handle here for the file currently being extracted.
+		extraction_file:  file    &log &optional;
+	};
+
+	redef record Info += {
+		## The in-progress entity information.
+		current_entity:   EntityInfo &optional;
+	};
+
+	redef record State += {
+		## Store a count of the number of files that have been transferred in
+		## a conversation to create unique file names on disk.
+		num_extracted_files:  count   &default=0;
+		## Track the number of MIME encoded files transferred during a session.
+		mime_level:           count   &default=0;
+	};
+
+	## Generate MD5 sums for these filetypes.
+	const generate_md5 = /application\/x-dosexec/    # Windows and DOS executables
+	                   | /application\/x-executable/ # *NIX executable binary
+	                   &redef;
+
+	## Pattern of file mime types to extract from MIME bodies.
+	const extract_file_types = /NO_DEFAULT/ &redef;
+
+	## The on-disk prefix for files to be extracted from MIME entity bodies.
+	const extraction_prefix = "smtp-entity" &redef;
+
+	global log_mime: event(rec: EntityInfo);
+}
+
+event bro_init() &priority=5
+	{
+	Log::create_stream(SMTP_ENTITIES, [$columns=EntityInfo, $ev=log_mime]);
+	}
+
+function set_session(c: connection, new_entity: bool)
+	{
+	if ( ! c$smtp?$current_entity || new_entity )
+		{
+		local info: EntityInfo;
+		info$ts=network_time();
+		info$uid=c$uid;
+		info$id=c$id;
+		info$mid=c$smtp$mid;
+		
+		c$smtp$current_entity = info;
+		++c$smtp_state$mime_level;
+		}
+	}
+
+event mime_begin_entity(c: connection) &priority=10
+	{
+	if ( ! c?$smtp ) return;
+	
+	set_session(c, T);
+	}
+
+# This has priority -10 because other handlers need to know the current
+# content_len before it's updated by this handler.
+event mime_segment_data(c: connection, length: count, data: string) &priority=-10
+	{
+	if ( ! c?$smtp ) return;
+	
+	c$smtp$current_entity$content_len = c$smtp$current_entity$content_len + length;
+	}
+
+event mime_segment_data(c: connection, length: count, data: string) &priority=7
+    {
+	if ( ! c?$smtp ) return;
+	if ( c$smtp$current_entity$content_len == 0 )
+		c$smtp$current_entity$mime_type = split1(identify_data(data, T), /;/)[1];
+	}
+
+event mime_segment_data(c: connection, length: count, data: string) &priority=-5
+	{
+	if ( ! c?$smtp ) return;
+
+	if ( c$smtp$current_entity$content_len == 0 )
+		{
+		if ( generate_md5 in c$smtp$current_entity$mime_type )
+			c$smtp$current_entity$calc_md5 = T;
+
+		if ( c$smtp$current_entity$calc_md5 )
+			{
+			c$smtp$current_entity$calculating_md5 = T;
+			md5_hash_init(c$id);
+			}
+		}
+
+	if ( c$smtp$current_entity$calculating_md5 )
+		md5_hash_update(c$id, data);
+}
+
+## In the event of a content gap during the MIME transfer, detect the state for
+## the MD5 sum calculation and stop calculating the MD5 since it would be
+## incorrect anyway.
+event content_gap(c: connection, is_orig: bool, seq: count, length: count) &priority=5
+	{
+	if ( is_orig || ! c?$smtp || ! c$smtp?$current_entity ) return;
+
+	if ( c$smtp$current_entity$calculating_md5 )
+		{
+		c$smtp$current_entity$calculating_md5 = F;
+		md5_hash_finish(c$id);
+		}
+	}
+
+event mime_end_entity(c: connection) &priority=-3
+    {
+	# TODO: this check is only due to a bug in mime_end_entity that
+	#       causes the event to be generated twice for the same real event.
+	if ( ! c?$smtp || ! c$smtp?$current_entity )
+		return;
+
+	if ( c$smtp$current_entity$calculating_md5 )
+		{
+		c$smtp$current_entity$md5 = md5_hash_finish(c$id);
+
+		NOTICE([$note=MD5, $msg=fmt("Calculated a hash for a MIME entity from %s", c$id$orig_h),
+				$sub=c$smtp$current_entity$md5, $conn=c]);
+		}
+	}
+
+event mime_one_header(c: connection, h: mime_header_rec)
+	{
+	if ( ! c?$smtp ) return;
+	
+	if ( h$name == "CONTENT-DISPOSITION" &&
+	     /[fF][iI][lL][eE][nN][aA][mM][eE]/ in h$value )
+		c$smtp$current_entity$filename = extract_filename_from_content_disposition(h$value);
+	}
+
+event mime_end_entity(c: connection) &priority=-5
+	{
+	if ( ! c?$smtp ) return;
+
+	# This check and the delete below are just to cope with a bug where
+	# mime_end_entity can be generated multiple times for the same event.
+	if ( ! c$smtp?$current_entity )
+		return;
+
+	# Only log is there was some content.
+	if ( c$smtp$current_entity$content_len > 0 )
+		Log::write(SMTP_ENTITIES, c$smtp$current_entity);
+
+	delete c$smtp$current_entity;
+	}
+
+event mime_segment_data(c: connection, length: count, data: string) &priority=5
+	{
+	if ( ! c?$smtp ) return;
+	
+	if ( extract_file_types in c$smtp$current_entity$mime_type )
+		c$smtp$current_entity$extract_file = T;
+	}
+
+event mime_segment_data(c: connection, length: count, data: string) &priority=3
+	{
+	if ( ! c?$smtp ) return;
+	
+	if ( c$smtp$current_entity$extract_file && 
+	     c$smtp$current_entity$content_len == 0 )
+		{
+		local suffix = fmt("%d.dat", ++c$smtp_state$num_extracted_files);
+		local fname = generate_extraction_filename(extraction_prefix, c, suffix);
+		c$smtp$current_entity$extraction_file = open(fname);
+		enable_raw_output(c$smtp$current_entity$extraction_file);
+		}
+	}
+
+event mime_segment_data(c: connection, length: count, data: string) &priority=-5
+	{
+	if ( ! c?$smtp ) return;
+	
+	if ( c$smtp$current_entity$extract_file && c$smtp$current_entity?$extraction_file )
+		print c$smtp$current_entity$extraction_file, data;
+	}
+
+event mime_end_entity(c: connection) &priority=-3
+	{
+	if ( ! c?$smtp ) return;
+	
+	# TODO: this check is only due to a bug in mime_end_entity that
+	#       causes the event to be generated twice for the same real event.
+	if ( ! c$smtp?$current_entity )
+		return;
+
+	if ( c$smtp$current_entity?$extraction_file )
+		close(c$smtp$current_entity$extraction_file);
+	}

scripts/base/protocols/smtp/main.bro

@@ -4,17 +4,14 @@ module SMTP;
 export {
 	redef enum Log::ID += { SMTP };
 
-	redef enum Notice::Type += { 
-		## Indicates that the server sent a reply mentioning an SMTP block list.
-		BL_Error_Message, 
-		## Indicates the client's address is seen in the block list error message.
-		BL_Blocked_Host, 
-	};
-
 	type Info: record {
 		ts:                time            &log;
 		uid:               string          &log;
 		id:                conn_id         &log;
+		## This is an internally generated "message id" that can be used to
+		## map between SMTP messages and MIME entities in the SMTP entities
+		## log.
+		mid:               string          &log;
 		helo:              string          &log &optional;
 		mailfrom:          string          &log &optional;
 		rcptto:            set[string]     &log &optional;
@@ -30,19 +27,13 @@ export {
 		second_received:   string          &log &optional;
 		## The last message the server sent to the client.
 		last_reply:        string          &log &optional;
-		files:             set[string]     &log &optional;
 		path:              vector of addr  &log &optional;
 		user_agent:        string          &log &optional;
 		
-		## Indicate if this session is currently transmitting SMTP message 
-		## envelope headers.
-		in_headers:        bool            &default=F;
 		## Indicate if the "Received: from" headers should still be processed.
 		process_received_from: bool        &default=T;
-		## Maintain the current header for cases where there is header wrapping.
+		## Indicates if client activity has been seen, but not yet logged
-		current_header:    string          &default="";
+		has_client_activity:  bool            &default=F;
-		## Indicate when the message is logged and no longer applicable.
-		done:              bool            &default=F;
 	};
 	
 	type State: record {
@@ -62,25 +53,6 @@ export {
 	##    NO_HOSTS - never capture the path.
 	const mail_path_capture = ALL_HOSTS &redef;
 		
-	# This matches content in SMTP error messages that indicate some
-	# block list doesn't like the connection/mail.
-	const bl_error_messages = 
-	    /spamhaus\.org\//
-	  | /sophos\.com\/security\//
-	  | /spamcop\.net\/bl/
-	  | /cbl\.abuseat\.org\// 
-	  | /sorbs\.net\// 
-	  | /bsn\.borderware\.com\//
-	  | /mail-abuse\.com\//
-	  | /b\.barracudacentral\.com\//
-	  | /psbl\.surriel\.com\// 
-	  | /antispam\.imp\.ch\// 
-	  | /dyndns\.com\/.*spam/
-	  | /rbl\.knology\.net\//
-	  | /intercept\.datapacket\.net\//
-	  | /uceprotect\.net\//
-	  | /hostkarma\.junkemailfilter\.com\// &redef;
-	
 	global log_smtp: event(rec: Info);
 	
 	## Configure the default ports for SMTP analysis.
@@ -121,6 +93,7 @@ function new_smtp_log(c: connection): Info
 	l$ts=network_time();
 	l$uid=c$uid;
 	l$id=c$id;
+	l$mid=unique_id("@");
 	if ( c?$smtp_state && c$smtp_state?$helo )
 		l$helo = c$smtp_state$helo;
 	
@@ -136,20 +109,14 @@ function set_smtp_session(c: connection)
 	if ( ! c?$smtp_state )
 		c$smtp_state = [];
 	
-	if ( ! c?$smtp || c$smtp$done )
+	if ( ! c?$smtp )
-		{
 		c$smtp = new_smtp_log(c);
-		}
 	}
 
-
 function smtp_message(c: connection)
 	{
-	Log::write(SMTP, c$smtp);
+	if ( c$smtp$has_client_activity )
-
+		Log::write(SMTP, c$smtp);
-	c$smtp$done = T;
-	# Track the number of messages seen in this session.
-	++c$smtp_state$messages_transferred;
 	}
 	
 event smtp_request(c: connection, is_orig: bool, command: string, arg: string) &priority=5
@@ -157,6 +124,9 @@ event smtp_request(c: connection, is_orig: bool, command: string, arg: string) &
 	set_smtp_session(c);
 	local upper_command = to_upper(command);
 
+	if ( upper_command != "QUIT" )
+		c$smtp$has_client_activity = T;
+	
 	if ( upper_command == "HELO" || upper_command == "EHLO" )
 		{
 		c$smtp_state$helo = arg;
@@ -172,26 +142,11 @@ event smtp_request(c: connection, is_orig: bool, command: string, arg: string) &
 
 	else if ( upper_command == "MAIL" && /^[fF][rR][oO][mM]:/ in arg )
 		{
-		# In case this is not the first message in a session we want to 
-		# essentially write out a log, clear the session tracking, and begin
-		# new session tracking.
-		if ( c$smtp_state$messages_transferred > 0 )
-			{
-			smtp_message(c);
-			set_smtp_session(c);
-			}
-		
 		local partially_done = split1(arg, /:[[:blank:]]*/)[2];
 		c$smtp$mailfrom = split1(partially_done, /[[:blank:]]?/)[1];
 		}
-		
-	else if ( upper_command == "DATA" )
-		{
-		c$smtp$in_headers = T;
-		}
 	}
 	
-
 event smtp_reply(c: connection, is_orig: bool, code: count, cmd: string,
                  msg: string, cont_resp: bool) &priority=5
 	{
@@ -199,152 +154,83 @@ event smtp_reply(c: connection, is_orig: bool, code: count, cmd: string,
 	
 	# This continually overwrites, but we want the last reply,
 	# so this actually works fine.
-	if ( code != 421 && code >= 400 )
+	c$smtp$last_reply = fmt("%d %s", code, msg);
+	}
+
+event smtp_reply(c: connection, is_orig: bool, code: count, cmd: string,
+                 msg: string, cont_resp: bool) &priority=-5
+	{
+	set_smtp_session(c);
+	if ( cmd == "." )
 		{
-		c$smtp$last_reply = fmt("%d %s", code, msg);
+		# Track the number of messages seen in this session.
-
+		++c$smtp_state$messages_transferred;
-		# Raise a notice when an SMTP error about a block list is discovered.
+		smtp_message(c);
-		if ( bl_error_messages in msg )
+		c$smtp = new_smtp_log(c);
-			{
-			local note = BL_Error_Message;
-			local message = fmt("%s received an error message mentioning an SMTP block list", c$id$orig_h);
-
-			# Determine if the originator's IP address is in the message.
-			local ips = find_ip_addresses(msg);
-			local text_ip = "";
-			if ( |ips| > 0 && to_addr(ips[0]) == c$id$orig_h )
-				{
-				note = BL_Blocked_Host;
-				message = fmt("%s is on an SMTP block list", c$id$orig_h);
-				}
-			
-			NOTICE([$note=note, $conn=c, $msg=message, $sub=msg]);
-			}
 		}
 	}
 
-event smtp_data(c: connection, is_orig: bool, data: string) &priority=5
+event mime_one_header(c: connection, h: mime_header_rec) &priority=5
 	{
-	# Is there something we should be handling from the server?
+	if ( ! c?$smtp ) return;
-	if ( ! is_orig ) return;
+	c$smtp$has_client_activity = T;
 
-	set_smtp_session(c);
+	if ( h$name == "MESSAGE-ID" )
+		c$smtp$msg_id = h$value;
 
-	if ( ! c$smtp$in_headers )
+	else if ( h$name == "RECEIVED" )
-		{
-		if ( /^[cC][oO][nN][tT][eE][nN][tT]-[dD][iI][sS].*[fF][iI][lL][eE][nN][aA][mM][eE]/ in data )
-			{
-			if ( ! c$smtp?$files )
-				c$smtp$files = set();
-			data = sub(data, /^.*[fF][iI][lL][eE][nN][aA][mM][eE]=/, "");
-			add c$smtp$files[data];
-			}
-		return;
-		}
-
-	if ( /^[[:blank:]]*$/ in data )
-		c$smtp$in_headers = F;
-
-	# This is to reconstruct headers that tend to wrap around.
-	if ( /^[[:blank:]]/ in data )
-		{
-		# Remove all but a single space at the beginning (this seems to follow
-		# the most common behavior).
-		data = sub(data, /^[[:blank:]]*/, " ");
-		if ( c$smtp$current_header == "MESSAGE-ID" )
-			c$smtp$msg_id += data;
-		else if ( c$smtp$current_header == "RECEIVED" )
-			c$smtp$first_received += data;
-		else if ( c$smtp$current_header == "IN-REPLY-TO" )
-			c$smtp$in_reply_to += data;
-		else if ( c$smtp$current_header == "SUBJECCT" )
-			c$smtp$subject += data;
-		else if ( c$smtp$current_header == "FROM" )
-			c$smtp$from += data;
-		else if ( c$smtp$current_header == "REPLY-TO" )
-			c$smtp$reply_to += data;
-		else if ( c$smtp$current_header == "USER-AGENT" )
-			c$smtp$user_agent += data;
-		return;
-		}
-	# Once there isn't a line starting with a blank, we're not continuing a 
-	# header anymore.
-	c$smtp$current_header = "";
-	
-	local header_parts = split1(data, /:[[:blank:]]*/);
-	# TODO: do something in this case?  This would definitely be odd.
-	# Header wrapping needs to be handled more elegantly.  This will happen
-	# if the header value is wrapped immediately after the header key.
-	if ( |header_parts| != 2 )
-		return;
-	
-	local header_key = to_upper(header_parts[1]);
-	c$smtp$current_header = header_key;
-	
-	local header_val = header_parts[2];
-	
-	if ( header_key == "MESSAGE-ID" )
-		c$smtp$msg_id = header_val;
-	
-	else if ( header_key == "RECEIVED" )
 		{
 		if ( c$smtp?$first_received )
 			c$smtp$second_received = c$smtp$first_received;
-		c$smtp$first_received = header_val;
+		c$smtp$first_received = h$value;
 		}
 
-	else if ( header_key == "IN-REPLY-TO" )
+	else if ( h$name == "IN-REPLY-TO" )
-		c$smtp$in_reply_to = header_val;
+		c$smtp$in_reply_to = h$value;
 
-	else if ( header_key == "DATE" )
+	else if ( h$name == "SUBJECT" )
-		c$smtp$date = header_val;
+		c$smtp$subject = h$value;
 
-	else if ( header_key == "FROM" )
+	else if ( h$name == "FROM" )
-		c$smtp$from = header_val;
+		c$smtp$from = h$value;
 
-	else if ( header_key == "TO" )
+	else if ( h$name == "REPLY-TO" )
+		c$smtp$reply_to = h$value;
+
+	else if ( h$name == "DATE" )
+		c$smtp$date = h$value;
+
+	else if ( h$name == "TO" )
 		{
 		if ( ! c$smtp?$to )
 			c$smtp$to = set();
-		add c$smtp$to[header_val];
+		add c$smtp$to[h$value];
 		}
 
-	else if ( header_key == "REPLY-TO" )
+	else if ( h$name == "X-ORIGINATING-IP" )
-		c$smtp$reply_to = header_val;
-	
-	else if ( header_key == "SUBJECT" )
-		c$smtp$subject = header_val;
-
-	else if ( header_key == "X-ORIGINATING-IP" )
 		{
-		local addresses = find_ip_addresses(header_val);
+		local addresses = find_ip_addresses(h$value);
 		if ( 1 in addresses )
 			c$smtp$x_originating_ip = to_addr(addresses[1]);
 		}
 	
-	else if ( header_key == "X-MAILER" || 
+	else if ( h$name == "X-MAILER" ||
-	          header_key == "USER-AGENT" ||
+	          h$name == "USER-AGENT" ||
-	          header_key == "X-USER-AGENT" )
+	          h$name == "X-USER-AGENT" )
-		{
+		c$smtp$user_agent = h$value;
-		c$smtp$user_agent = header_val;
-		# Explicitly set the current header here because there are several 
-		# headers bulked under this same key.
-		c$smtp$current_header = "USER-AGENT";
-		}
 	}
 	
 # This event handler builds the "Received From" path by reading the 
 # headers in the mail
-event smtp_data(c: connection, is_orig: bool, data: string) &priority=3
+event mime_one_header(c: connection, h: mime_header_rec) &priority=3
 	{
 	# If we've decided that we're done watching the received headers for
 	# whatever reason, we're done.  Could be due to only watching until 
 	# local addresses are seen in the received from headers.
-	if ( c$smtp$current_header != "RECEIVED" ||
+	if ( ! c?$smtp || h$name != "RECEIVED" || ! c$smtp$process_received_from )
-	     ! c$smtp$process_received_from )
 		return;
 
-	local text_ip = find_address_in_smtp_header(data);
+	local text_ip = find_address_in_smtp_header(h$value);
 	if ( text_ip == "" )
 		return;
 	local ip = to_addr(text_ip);
@@ -354,14 +240,12 @@ event smtp_data(c: connection, is_orig: bool, data: string) &priority=3
 		{
 		c$smtp$process_received_from = F;
 		}
-	
 	if ( c$smtp$path[|c$smtp$path|-1] != ip )
 		c$smtp$path[|c$smtp$path|] = ip;
 	}
 
-
 event connection_state_remove(c: connection) &priority=-5
 	{
-	if ( c?$smtp && ! c$smtp$done )
+	if ( c?$smtp )
 		smtp_message(c);
 	}

scripts/base/protocols/ssh/main.bro

@@ -1,74 +1,58 @@
+##! Base SSH analysis script.  The heuristic to blindly determine success or 
+##! failure for SSH connections is implemented here.  At this time, it only
+##! uses the size of the data being returned from the server to make the
+##! heuristic determination about success of the connection.  
+##! Requires that :bro:id:`use_conn_size_analyzer` is set to T!  The heuristic
+##! is not attempted if the connection size analyzer isn't enabled.
 
 module SSH;
 
 export {
 	redef enum Log::ID += { SSH };
 
-	redef enum Notice::Type += {
-		Login,
-		Password_Guessing,
-		Login_By_Password_Guesser,
-		Login_From_Interesting_Hostname,
-		Bytecount_Inconsistency,
-	};
-
 	type Info: record {
 		ts:              time         &log;
 		uid:             string       &log;
 		id:              conn_id      &log;
+		## Indicates if the login was heuristically guessed to be "success"
+		## or "failure".
 		status:          string       &log &optional;
-		direction:       string       &log &optional;
+		## Direction of the connection.  If the client was a local host 
-		remote_location: geo_location &log &optional;
+		## logging into an external host, this would be OUTBOUD.  INBOUND
+		## would be set for the opposite situation.
+		# TODO: handle local-local and remote-remote better.
+		direction:       Direction    &log &optional;
+		## The software string given by the client.
 		client:          string       &log &optional;
+		## The software string given by the server.
 		server:          string       &log &optional;
+		## The amount of data returned from the server.  This is currently
+		## the only measure of the success heuristic and it is logged to 
+		## assist analysts looking at the logs to make their own determination
+		## about the success on a case-by-case basis.
 		resp_size:       count        &log &default=0;
 		
 		## Indicate if the SSH session is done being watched.
 		done:            bool         &default=F;
 	};
 	
-	const password_guesses_limit = 30 &redef;
+	## The size in bytes at which the SSH connection is presumed to be
-	
+	## successful.
-	# The size in bytes at which the SSH connection is presumed to be
-	# successful.
 	const authentication_data_size = 5500 &redef;
 	
-	# The amount of time to remember presumed non-successful logins to build
+	## If true, we tell the event engine to not look at further data
-	# model of a password guesser.
+	## packets after the initial SSH handshake. Helps with performance
-	const guessing_timeout = 30 mins &redef;
+	## (especially with large file transfers) but precludes some
-
+	## kinds of analyses (e.g., tracking connection size).
-	# The set of countries for which you'd like to throw notices upon successful login
-	#   requires Bro compiled with libGeoIP support
-	const watched_countries: set[string] = {"RO"} &redef;
-
-	# Strange/bad host names to originate successful SSH logins
-	const interesting_hostnames =
-			/^d?ns[0-9]*\./ |
-			/^smtp[0-9]*\./ |
-			/^mail[0-9]*\./ |
-			/^pop[0-9]*\./  |
-			/^imap[0-9]*\./ |
-			/^www[0-9]*\./  |
-			/^ftp[0-9]*\./  &redef;
-
-	# This is a table with orig subnet as the key, and subnet as the value.
-	const ignore_guessers: table[subnet] of subnet &redef;
-	
-	# If true, we tell the event engine to not look at further data
-	# packets after the initial SSH handshake. Helps with performance
-	# (especially with large file transfers) but precludes some
-	# kinds of analyses (e.g., tracking connection size).
 	const skip_processing_after_detection = F &redef;
 	
-	# Keeps count of how many rejections a host has had
+	## This event is generated when the heuristic thinks that a login
-	global password_rejections: table[addr] of TrackCount 
+	## was successful.
-		&write_expire=guessing_timeout
+	global heuristic_successful_login: event(c: connection);
-		&synchronized;
 	
-	# Keeps track of hosts identified as guessing passwords
+	## This event is generated when the heuristic thinks that a login
-	# TODO: guessing_timeout doesn't work correctly here.  If a user redefs
+	## failed.
-	#       the variable, it won't take effect.
+	global heuristic_failed_login: event(c: connection);
-	global password_guessers: set[addr] &read_expire=guessing_timeout+1hr &synchronized;
 	
 	global log_ssh: event(rec: Info);
 }
@@ -106,116 +90,51 @@ function check_ssh_connection(c: connection, done: bool)
 	
 	# If this is still a live connection and the byte count has not
 	# crossed the threshold, just return and let the resheduled check happen later.
-	if ( !done && c$resp$size < authentication_data_size )
+	if ( !done && c$resp$num_bytes_ip < authentication_data_size )
 		return;
 
 	# Make sure the server has sent back more than 50 bytes to filter out
 	# hosts that are just port scanning.  Nothing is ever logged if the server
 	# doesn't send back at least 50 bytes.
-	if ( c$resp$size < 50 )
+	if ( c$resp$num_bytes_ip < 50 )
 		return;
 
-	local status = "failure";
+	c$ssh$direction = Site::is_local_addr(c$id$orig_h) ? OUTBOUND : INBOUND;
-	local direction = Site::is_local_addr(c$id$orig_h) ? "to" : "from";
+	c$ssh$resp_size = c$resp$num_bytes_ip;
-	local location: geo_location;
-	location = (direction == "to") ? lookup_location(c$id$resp_h) : lookup_location(c$id$orig_h);
 	
-	if ( done && c$resp$size < authentication_data_size )
+	if ( c$resp$num_bytes_ip < authentication_data_size )
 		{
-		# presumed failure
+		c$ssh$status  = "failure";
-		if ( c$id$orig_h !in password_rejections )
+		event SSH::heuristic_failed_login(c);
-			password_rejections[c$id$orig_h] = new_track_count();
-			
-		# Track the number of rejections
-		if ( !(c$id$orig_h in ignore_guessers &&
-		       c$id$resp_h in ignore_guessers[c$id$orig_h]) )
-			++password_rejections[c$id$orig_h]$n;
-			
-		if ( default_check_threshold(password_rejections[c$id$orig_h]) )
-			{
-			add password_guessers[c$id$orig_h];
-			NOTICE([$note=Password_Guessing,
-			        $conn=c,
-			        $msg=fmt("SSH password guessing by %s", c$id$orig_h),
-			        $sub=fmt("%d failed logins", password_rejections[c$id$orig_h]$n),
-			        $n=password_rejections[c$id$orig_h]$n]);
-			}
 		}
-	# TODO: This is to work around a quasi-bug in Bro which occasionally 
+	else
-	#       causes the byte count to be oversized.
-	#   Watch for Gregors work that adds an actual counter of bytes transferred.
-	else if ( c$resp$size < 20000000 ) 
 		{ 
 		# presumed successful login
-		status = "success";
+		c$ssh$status = "success";
-		c$ssh$done = T;
+		event SSH::heuristic_successful_login(c);
-
-		if ( c$id$orig_h in password_rejections &&
-		     password_rejections[c$id$orig_h]$n > password_guesses_limit &&
-		     c$id$orig_h !in password_guessers )
-			{
-			add password_guessers[c$id$orig_h];
-			NOTICE([$note=Login_By_Password_Guesser,
-			        $conn=c,
-			        $n=password_rejections[c$id$orig_h]$n,
-			        $msg=fmt("Successful SSH login by password guesser %s", c$id$orig_h),
-			        $sub=fmt("%d failed logins", password_rejections[c$id$orig_h]$n)]);
-			}
-		
-		local message = fmt("SSH login %s %s \"%s\" \"%s\" %f %f %s (triggered with %d bytes)",
-		              direction, location$country_code, location$region, location$city,
-		              location$latitude, location$longitude,
-		              id_string(c$id), c$resp$size);
-		NOTICE([$note=Login,
-		        $conn=c,
-		        $msg=message,
-		        $sub=location$country_code]);
-		
-		# Check to see if this login came from an interesting hostname
-		when ( local hostname = lookup_addr(c$id$orig_h) )
-			{
-			if ( interesting_hostnames in hostname )
-				{
-				NOTICE([$note=Login_From_Interesting_Hostname,
-				        $conn=c,
-				        $msg=fmt("Strange login from %s", hostname),
-				        $sub=hostname]);
-				}
-			}
-			
-		if ( location$country_code in watched_countries )
-			{
-			
-			}
-			
 		}
-	else if ( c$resp$size >= 200000000 ) 
-		{
-		NOTICE([$note=Bytecount_Inconsistency,
-		        $conn=c,
-		        $msg="During byte counting in SSH analysis, an overly large value was seen.",
-		        $sub=fmt("%d",c$resp$size)]);
-		}
-
-	c$ssh$remote_location = location;
-	c$ssh$status = status;
-	c$ssh$direction = direction;
-	c$ssh$resp_size = c$resp$size;
-	
-	Log::write(SSH, c$ssh);
 	
 	# Set the "done" flag to prevent the watching event from rescheduling
 	# after detection is done.
-	c$ssh$done;
+	c$ssh$done=T;
 	
-	# Stop watching this connection, we don't care about it anymore.
 	if ( skip_processing_after_detection )
 		{
+		# Stop watching this connection, we don't care about it anymore.
 		skip_further_processing(c$id);
 		set_record_packets(c$id, F);
 		}
 	}
 
+event SSH::heuristic_successful_login(c: connection) &priority=-5
+	{
+	Log::write(SSH, c$ssh);
+	}
+event SSH::heuristic_failed_login(c: connection) &priority=-5
+	{
+	Log::write(SSH, c$ssh);
+	}
+
 event connection_state_remove(c: connection) &priority=-5
 	{
 	if ( c?$ssh )
@@ -226,7 +145,7 @@ event ssh_watcher(c: connection)
 	{
 	local id = c$id;
 	# don't go any further if this connection is gone already!
-	if ( !connection_exists(id) )
+	if ( ! connection_exists(id) )
 		return;
 
 	check_ssh_connection(c, F);
@@ -244,5 +163,9 @@ event ssh_client_version(c: connection, version: string) &priority=5
 	{
 	set_session(c);
 	c$ssh$client = version;
-	schedule +15secs { ssh_watcher(c) };
+	
+	# The heuristic detection for SSH relies on the ConnSize analyzer.
+	# Don't do the heuristics if it's disabled.
+	if ( use_conn_size_analyzer )
+		schedule +15secs { ssh_watcher(c) };
 	}

scripts/base/utils/files.bro

@@ -13,3 +13,14 @@ function generate_extraction_filename(prefix: string, c: connection, suffix: str
 		
 	return conn_info;
 	}
+	
+## For CONTENT-DISPOSITION headers, this function can be used to extract 
+## the filename.
+function extract_filename_from_content_disposition(data: string): string
+	{
+	local filename = sub(data, /^.*[fF][iI][lL][eE][nN][aA][mM][eE]=/, "");
+	# Remove quotes around the filename if they are there.
+	if ( /^\"/ in filename )
+		filename =  split_n(filename, /\"/, F, 2)[2];
+	return filename;
+	}

scripts/policy/frameworks/communication/listen-clear.bro

@@ -1,5 +1,7 @@
 ##! Listen for other Bro instances to make unencrypted connections.
 
+@load base/frameworks/communication
+
 module Communication;
 
 export {

scripts/policy/frameworks/communication/listen-ssl.bro

@@ -1,5 +1,7 @@
 ##! Listen for other Bro instances and encrypt the connection with SSL.
 
+@load base/frameworks/communication
+
 module Communication;
 
 export {

scripts/policy/frameworks/control/controller.bro

@@ -1,3 +1,4 @@
+@load base/frameworks/communication
 
 module Control;
 

scripts/policy/misc/profiling.bro

@@ -0,0 +1,19 @@
+##! Turns on profiling of Bro resource consumption.
+
+module Profiling;
+
+redef profiling_file = open_log_file("prof");
+
+export {
+	## Cheap profiling every 15 seconds.
+	redef profiling_interval = 15 secs &redef;
+}
+
+# Expensive profiling every 5 minutes.
+redef expensive_profiling_multiple = 20;
+
+event bro_init()
+	{
+	set_buf(profiling_file, F);
+	}
+

scripts/policy/protocols/conn/scan.bro

@@ -1,4 +1,4 @@
-@load frameworks/notice
+@load base/frameworks/notice
 @load port-name
 
 module Scan;

scripts/policy/protocols/http/detect-MHR.bro

@@ -1,7 +1,7 @@
 ##! This script takes MD5 sums of files transferred over HTTP and checks them with
 ##! Team Cymru's Malware Hash Registry (http://www.team-cymru.org/Services/MHR/).
 ##! By default, not all file transfers will have MD5 sums calculated.  Read the
-##! documentation for the protocols/http/file-hash.bro script to see how to 
+##! documentation for the base/protocols/http/file-hash.bro script to see how to 
 ##! configure which transfers will have hashes calculated.
 
 export {

scripts/policy/protocols/smtp/blocklists.bro

@@ -0,0 +1,58 @@
+
+@load base/protocols/smtp
+
+module SMTP;
+
+export {
+	redef enum Notice::Type += { 
+		## Indicates that the server sent a reply mentioning an SMTP block list.
+		Blocklist_Error_Message,
+		## Indicates the client's address is seen in the block list error message.
+		Blocklist_Blocked_Host,
+	};
+
+	# This matches content in SMTP error messages that indicate some
+	# block list doesn't like the connection/mail.
+	const blocklist_error_messages = 
+		  /spamhaus\.org\//
+		| /sophos\.com\/security\//
+		| /spamcop\.net\/bl/
+		| /cbl\.abuseat\.org\// 
+		| /sorbs\.net\// 
+		| /bsn\.borderware\.com\//
+		| /mail-abuse\.com\//
+		| /b\.barracudacentral\.com\//
+		| /psbl\.surriel\.com\// 
+		| /antispam\.imp\.ch\// 
+		| /dyndns\.com\/.*spam/
+		| /rbl\.knology\.net\//
+		| /intercept\.datapacket\.net\//
+		| /uceprotect\.net\//
+		| /hostkarma\.junkemailfilter\.com\// &redef;
+	
+}
+
+event smtp_reply(c: connection, is_orig: bool, code: count, cmd: string,
+                 msg: string, cont_resp: bool) &priority=3
+	{
+	if ( code >= 400 && code != 421 )
+		{
+		# Raise a notice when an SMTP error about a block list is discovered.
+		if ( blocklist_error_messages in msg )
+			{
+			local note = Blocklist_Error_Message;
+			local message = fmt("%s received an error message mentioning an SMTP block list", c$id$orig_h);
+
+			# Determine if the originator's IP address is in the message.
+			local ips = find_ip_addresses(msg);
+			local text_ip = "";
+			if ( |ips| > 0 && to_addr(ips[0]) == c$id$orig_h )
+				{
+				note = Blocklist_Blocked_Host;
+				message = fmt("%s is on an SMTP block list", c$id$orig_h);
+				}
+			
+			NOTICE([$note=note, $conn=c, $msg=message, $sub=msg]);
+			}
+		}
+	}

scripts/policy/protocols/smtp/software.bro

@@ -43,10 +43,10 @@ export {
 	                   | /ZimbraWebClient/ &redef;
 }
 
-event smtp_data(c: connection, is_orig: bool, data: string) &priority=4
+event mime_one_header(c: connection, h: mime_header_rec) &priority=4
 	{
-	if ( c$smtp$current_header == "USER-AGENT" &&
+	if ( ! c?$smtp ) return;
-	     webmail_user_agents in c$smtp$user_agent )
+	if ( h$name == "USER-AGENT" && webmail_user_agents in c$smtp$user_agent )
 		c$smtp$is_webmail = T;
 	}
 

scripts/policy/protocols/ssh/detect-bruteforcing.bro

@@ -0,0 +1,79 @@
+
+module SSH;
+
+export {
+	redef enum Notice::Type += {
+		## Indicates that a host has been identified as crossing the 
+		## :bro:id:`password_guesses_limit` threshold with heuristically
+		## determined failed logins.
+		Password_Guessing,
+		## Indicates that a host previously identified as a "password guesser"
+		## has now had a heuristically successful login attempt.
+		Login_By_Password_Guesser,
+	};
+
+	## The number of failed SSH connections before a host is designated as
+	## guessing passwords.
+	const password_guesses_limit = 30 &redef;
+
+	## The amount of time to remember presumed non-successful logins to build
+	## model of a password guesser.
+	const guessing_timeout = 30 mins &redef;
+
+	## This value can be used to exclude hosts or entire networks from being 
+	## tracked as potential "guessers".  There are cases where the success
+	## heuristic fails and this acts as the whitelist.  The index represents 
+	## client subnets and the yield value represents server subnets.
+	const ignore_guessers: table[subnet] of subnet &redef;
+
+	## Keeps count of how many rejections a host has had.
+	global password_rejections: table[addr] of TrackCount 
+		&write_expire=guessing_timeout
+		&synchronized;
+
+	## Keeps track of hosts identified as guessing passwords.
+	global password_guessers: set[addr] &read_expire=guessing_timeout+1hr &synchronized;
+}
+
+event SSH::heuristic_successful_login(c: connection)
+	{
+	local id = c$id;
+	
+	# TODO: this should be migrated to the metrics framework.
+	if ( id$orig_h in password_rejections &&
+	     password_rejections[id$orig_h]$n > password_guesses_limit &&
+	     id$orig_h !in password_guessers )
+		{
+		add password_guessers[id$orig_h];
+		NOTICE([$note=Login_By_Password_Guesser,
+		        $conn=c,
+		        $n=password_rejections[id$orig_h]$n,
+		        $msg=fmt("Successful SSH login by password guesser %s", id$orig_h),
+		        $sub=fmt("%d failed logins", password_rejections[id$orig_h]$n)]);
+		}
+	}
+
+event SSH::heuristic_failed_login(c: connection)
+	{
+	local id = c$id;
+	
+	# presumed failure
+	if ( id$orig_h !in password_rejections )
+		password_rejections[id$orig_h] = new_track_count();
+	
+	# Track the number of rejections
+	# TODO: this should be migrated to the metrics framework.
+	if ( ! (id$orig_h in ignore_guessers &&
+	        id$resp_h in ignore_guessers[id$orig_h]) )
+		++password_rejections[id$orig_h]$n;
+	
+	if ( default_check_threshold(password_rejections[id$orig_h]) )
+		{
+		add password_guessers[id$orig_h];
+		NOTICE([$note=Password_Guessing,
+		        $conn=c,
+		        $msg=fmt("SSH password guessing by %s", id$orig_h),
+		        $sub=fmt("%d apparently failed logins", password_rejections[id$orig_h]$n),
+		        $n=password_rejections[id$orig_h]$n]);
+		}
+	}

scripts/policy/protocols/ssh/geo-data.bro

@@ -0,0 +1,39 @@
+##! This implements all of the additional information and geodata detections 
+##! for SSH analysis.
+
+module SSH;
+
+export {
+	redef enum Notice::Type += {
+		## If an SSH login is seen to or from a "watched" country based on the
+		## :bro:id:`SSH::watched_countries` variable then this notice will
+		## be generated.
+		Login_From_Watched_Country,
+	};
+	
+	## The set of countries for which you'd like to throw notices upon 
+	## successful login
+	const watched_countries: set[string] = {"RO"} &redef;
+
+	redef record Info += {
+		## Add geographic data related to the "remote" host of the connection.
+		remote_location: geo_location &log &optional;
+	};
+}
+
+event SSH::heuristic_successful_login(c: connection) &priority=5
+	{
+	local location: geo_location;
+	location = (c$ssh$direction == OUTBOUND) ? 
+		lookup_location(c$id$resp_h) : lookup_location(c$id$orig_h);
+	
+	# Add the location data to the SSH record.
+	c$ssh$remote_location = location;
+	
+	if ( location$country_code in watched_countries )
+		{
+		NOTICE([$note=Login_From_Watched_Country,
+		        $conn=c,
+		        $msg=fmt("SSH login from watched country: %s", location$country_code)]);
+		}
+	}

scripts/policy/protocols/ssh/interesting-hostnames.bro

@@ -0,0 +1,50 @@
+
+module SSH;
+
+export {
+	redef enum Notice::Type += {
+		## Generated if a login originates from a host matched by the 
+		## :bro:id:`interesting_hostnames` regular expression.
+		Login_From_Interesting_Hostname,
+		## Generated if a login goes to a host matched by the 
+		## :bro:id:`interesting_hostnames` regular expression.
+		Login_To_Interesting_Hostname,
+	};
+	
+	## Strange/bad host names to see successful SSH logins from or to.
+	const interesting_hostnames =
+			/^d?ns[0-9]*\./ |
+			/^smtp[0-9]*\./ |
+			/^mail[0-9]*\./ |
+			/^pop[0-9]*\./  |
+			/^imap[0-9]*\./ |
+			/^www[0-9]*\./  |
+			/^ftp[0-9]*\./  &redef;
+}
+
+event SSH::heuristic_successful_login(c: connection)
+	{
+	# Check to see if this login came from an interesting hostname.
+	when ( local orig_hostname = lookup_addr(c$id$orig_h) )
+		{
+		if ( interesting_hostnames in orig_hostname )
+			{
+			NOTICE([$note=Login_From_Interesting_Hostname,
+			        $conn=c,
+			        $msg=fmt("Interesting login from hostname: %s", orig_hostname),
+			        $sub=orig_hostname]);
+			}
+		}
+	# Check to see if this login went to an interesting hostname.
+	when ( local resp_hostname = lookup_addr(c$id$orig_h) )
+		{
+		if ( interesting_hostnames in resp_hostname )
+			{
+			NOTICE([$note=Login_To_Interesting_Hostname,
+			        $conn=c,
+			        $msg=fmt("Interesting login to hostname: %s", resp_hostname),
+			        $sub=resp_hostname]);
+			}
+		}
+	}
+

scripts/policy/protocols/ssh/software.bro

@@ -3,8 +3,8 @@ module SSH;
 
 export {
 	redef enum Software::Type += {
-		SSH_SERVER,
+		SERVER,
-		SSH_CLIENT,
+		CLIENT,
 	};
 }
 
@@ -12,7 +12,7 @@ event ssh_client_version(c: connection, version: string) &priority=4
 	{
 	# Get rid of the protocol information when passing to the software framework.
 	local cleaned_version = sub(version, /^SSH[0-9\.\-]+/, "");
-	local si = Software::parse(cleaned_version, c$id$orig_h, SSH_CLIENT);
+	local si = Software::parse(cleaned_version, c$id$orig_h, CLIENT);
 	Software::found(c$id, si);
 	}
 
@@ -20,6 +20,6 @@ event ssh_server_version(c: connection, version: string) &priority=4
 	{
 	# Get rid of the protocol information when passing to the software framework.
 	local cleaned_version = sub(version, /SSH[0-9\.\-]{2,}/, "");
-	local si = Software::parse(cleaned_version, c$id$resp_h, SSH_SERVER);
+	local si = Software::parse(cleaned_version, c$id$resp_h, SERVER);
 	Software::found(c$id, si);
 	}

scripts/site/local-manager.bro

@@ -0,0 +1,7 @@
+##! Local site policy loaded only by the manager in a cluster.
+
+# If you are running a cluster you should define your Notice::policy here 
+# so that notice processing occurs on the manager.
+redef Notice::policy += {
+
+};

scripts/site/local-proxy.bro

@@ -0,0 +1,2 @@
+##! Local site policy loaded only by the proxies if Bro is running as a cluster.
+

scripts/site/local-worker.bro

@@ -0,0 +1 @@
+##! Local site policy loaded only by the workers if Bro is running as a cluster.

scripts/site/local.bro

@@ -22,6 +22,7 @@ redef signature_files += "frameworks/signatures/detect-windows-shells.sig";
 
 # Load all of the scripts that detect software in various protocols.
 @load protocols/http/software
+#@load protocols/http/detect-webapps
 @load protocols/ftp/software
 @load protocols/smtp/software
 @load protocols/ssh/software
@@ -44,3 +45,24 @@ redef signature_files += "frameworks/signatures/detect-windows-shells.sig";
 
 # Load the script to enable SSL/TLS certificate validation.
 @load protocols/ssl/validate-certs
+
+# If you have libGeoIP support built in, do some geographic detections and 
+# logging for SSH traffic.
+@load protocols/ssh/geo-data
+# Detect hosts doing SSH bruteforce attacks.
+@load protocols/ssh/detect-bruteforcing
+# Detect logins using "interesting" hostnames.
+@load protocols/ssh/interesting-hostnames
+
+# Detect MD5 sums in Team Cymru's Malware Hash Registry.
+@load protocols/http/detect-MHR
+# Detect SQL injection attacks
+@load protocols/http/detect-sqli
+
+# Uncomment this redef if you want to extract SMTP MIME entities for 
+# some file types.  The numbers given indicate how many bytes to extract for
+# the various mime types.
+redef SMTP::entity_excerpt_len += {
+#	["text/plain"] = 1024,
+#	["text/html"] = 1024,
+};

src/BroDoc.cc

@@ -20,8 +20,8 @@ BroDoc::BroDoc(const std::string& rel, const std::string& abs)
 
 	if ( rel[0] == '/' || rel[0] == '.' )
 		{
-		// The Bro script must not be on a subpath of the policy/ dir of
+		// The Bro script isn't being loaded via BROPATH, so just use basename
-		// BROPATH, so just use the basename as the document title.
+		// as the document title.
 		doc_title = source_filename;
 		}
 	else
@@ -33,8 +33,14 @@ BroDoc::BroDoc(const std::string& rel, const std::string& abs)
 			doc_title = rel + "/" + source_filename;
 		}
 
+	downloadable_filename = source_filename;
+
+	size_t ext_pos = downloadable_filename.find(".bif.bro");
+	if ( std::string::npos != ext_pos )
+		downloadable_filename.erase(ext_pos + 4);
+
 	reST_filename = doc_title;
-	size_t ext_pos = reST_filename.find(".bro");
+	ext_pos = reST_filename.find(".bro");
 
 	if ( std::string::npos == ext_pos )
 		reST_filename += ".rst";
@@ -103,14 +109,14 @@ void BroDoc::AddImport(const std::string& s)
 			{
 			if ( subpath[0] == '/' || subpath[0] == '.' )
 				{
-				// it's not a subpath of policy/, so just add the name of it
+				// it's not a subpath of scripts/, so just add the name of it
 				// as it's given in the @load directive
 				imports.push_back(lname);
 				}
 			else
 				{
 				// combine the base file name of script in the @load directive
-				// with the subpath of BROPATH's policy/ directory
+				// with the subpath of BROPATH's scripts/ directory
 				string fname(subpath);
 				char* othertmp = copy_string(lname.c_str());
 				fname.append("/").append(basename(othertmp));
@@ -167,7 +173,7 @@ void BroDoc::WriteDocFile() const
 	WriteSectionHeading(doc_title.c_str(), '=');
 
 	WriteToDoc("\n:download:`Original Source File <%s>`\n\n",
-		source_filename.c_str());
+		downloadable_filename.c_str());
 
 	WriteSectionHeading("Overview", '-');
 	WriteStringList("%s\n", "%s\n\n", summary);
@@ -185,7 +191,7 @@ void BroDoc::WriteDocFile() const
 			size_t pos = pretty.find("/index");
 			if ( pos != std::string::npos && pos + 6 == pretty.size() )
 				pretty = pretty.substr(0, pos);
-			WriteToDoc(":doc:`%s </policy/%s>`", pretty.c_str(), it->c_str());
+			WriteToDoc(":doc:`%s </scripts/%s>`", pretty.c_str(), it->c_str());
 			}
 		WriteToDoc("\n");
 		}

src/BroDoc.h

@@ -24,10 +24,9 @@ public:
 	 * If the filename doesn't end in ".bro", then ".rst" is just appended.
 	 * Any '/' characters in the reST file name that result from choice of
 	 * the 'rel' parameter are replaced with '^'.
-	 * @param subpath A string representing a subpath of BROPATH's policy/
+	 * @param rel A string representing a subpath of the root Bro script
-	 *        directory in which the source file is located.  It can
+	 *        source/install directory in which the source file is located.
-	 *        also be full path to the file or a full path that's in BROPATH,
+	 *        It can also be an absolute path, but then the parameter is
-	 *        but in either of those cases, the parameter is essentially
 	 *        ignored and the document title is just derived from file name
 	 * @param abs The absolute path to the Bro script for which to generate
 	 *        documentation.
@@ -211,6 +210,7 @@ protected:
 	FILE* reST_file;
 	std::string reST_filename;
 	std::string source_filename;	// points to the basename of source file
+	std::string downloadable_filename; // file that will be linked for download
 	std::string doc_title;
 	std::string packet_filter;
 

src/CMakeLists.txt

@@ -408,7 +408,8 @@ set(bro_SRCS
 
 collect_headers(bro_HEADERS ${bro_SRCS})
 
-add_definitions(-DPOLICYDEST="${POLICYDIR}")
+add_definitions(-DBRO_SCRIPT_INSTALL_PATH="${BRO_SCRIPT_INSTALL_PATH}")
+add_definitions(-DBRO_SCRIPT_SOURCE_PATH="${BRO_SCRIPT_SOURCE_PATH}")
 
 add_executable(bro ${bro_SRCS} ${bro_HEADERS})
 
@@ -423,7 +424,7 @@ set(brolibs
 target_link_libraries(bro ${brolibs})
 
 install(TARGETS bro DESTINATION bin)
-install(FILES ${INSTALL_BIF_OUTPUTS} DESTINATION ${POLICYDIR})
+install(FILES ${INSTALL_BIF_OUTPUTS} DESTINATION ${BRO_SCRIPT_INSTALL_PATH})
 
 set(BRO_EXE bro
     CACHE STRING "Bro executable binary" FORCE)

src/ConnCompressor.cc

@@ -866,15 +866,10 @@ void ConnCompressor::Event(const PendingConn* pending, double t,
 
 			if ( ConnSize_Analyzer::Available() )
 				{
+				// Fill in optional fields if ConnSize_Analyzer is on.
 				orig_endp->Assign(2, new Val(pending->num_pkts, TYPE_COUNT));
 				orig_endp->Assign(3, new Val(pending->num_bytes_ip, TYPE_COUNT));
 				}
-			else
-				{
-				orig_endp->Assign(2, new Val(0, TYPE_COUNT));
-				orig_endp->Assign(3, new Val(0, TYPE_COUNT));
-				}
-
 
 			resp_endp->Assign(0, new Val(0, TYPE_COUNT));
 			resp_endp->Assign(1, new Val(resp_state, TYPE_COUNT));
@@ -900,14 +895,10 @@ void ConnCompressor::Event(const PendingConn* pending, double t,
 
 			if ( ConnSize_Analyzer::Available() )
 				{
+				// Fill in optional fields if ConnSize_Analyzer is on
 				resp_endp->Assign(2, new Val(pending->num_pkts, TYPE_COUNT));
 				resp_endp->Assign(3, new Val(pending->num_bytes_ip, TYPE_COUNT));
 				}
-			else
-				{
-				resp_endp->Assign(2, new Val(0, TYPE_COUNT));
-				resp_endp->Assign(3, new Val(0, TYPE_COUNT));
-				}
 
 			DBG_LOG(DBG_COMPRESSOR, "%s swapped direction", fmt_conn_id(pending));
 			}

src/DNS_Mgr.cc

@@ -1071,7 +1071,7 @@ void  DNS_Mgr::Process()
 	int status = nb_dns_activity(nb_dns, &r, err);
 
 	if ( status < 0 )
-		reporter->InternalError("NB-DNS error in DNS_Mgr::Process (%s)", err);
+		reporter->Warning("NB-DNS error in DNS_Mgr::Process (%s)", err);
 
 	else if ( status > 0 )
 		{

src/Expr.cc

@@ -5020,8 +5020,9 @@ Val* ListExpr::InitVal(const BroType* t, Val* aggr) const
 		loop_over_list(exprs, i)
 			{
 			Expr* e = exprs[i];
+			check_and_promote_expr(e, vec->Type()->AsVectorType()->YieldType());
 			Val* v = e->Eval(0);
-			if ( ! vec->Assign(i, v, e) )
+			if ( ! vec->Assign(i, v->RefCnt() == 1 ? v->Ref() : v, e) )
 				{
 				e->Error(fmt("type mismatch at index %d", i));
 				return 0;

src/LogMgr.cc

@@ -89,7 +89,7 @@ bool LogField::Write(SerializationFormat* fmt) const
 
 LogVal::~LogVal()
 	{
-	if ( (type == TYPE_ENUM || type == TYPE_STRING || type == TYPE_FILE)
+	if ( (type == TYPE_ENUM || type == TYPE_STRING || type == TYPE_FILE || type == TYPE_FUNC)
 	     && present )
 		delete val.string_val;
 
@@ -130,6 +130,7 @@ bool LogVal::IsCompatibleType(BroType* t, bool atomic_only)
 	case TYPE_ENUM:
 	case TYPE_STRING:
 	case TYPE_FILE:
+	case TYPE_FUNC:
 		return true;
 
 	case TYPE_RECORD:
@@ -231,6 +232,7 @@ bool LogVal::Read(SerializationFormat* fmt)
 	case TYPE_ENUM:
 	case TYPE_STRING:
 	case TYPE_FILE:
+	case TYPE_FUNC:
 		{
 		val.string_val = new string;
 		return fmt->Read(val.string_val, "string");
@@ -343,6 +345,7 @@ bool LogVal::Write(SerializationFormat* fmt) const
 	case TYPE_ENUM:
 	case TYPE_STRING:
 	case TYPE_FILE:
+	case TYPE_FUNC:
 		return fmt->Write(*val.string_val, "string");
 
 	case TYPE_TABLE:
@@ -648,6 +651,11 @@ bool LogMgr::TraverseRecord(Stream* stream, Filter* filter, RecordType* rt,
 				// That's ok, we handle it below.
 				}
 
+			else if ( t->Tag() == TYPE_FUNC )
+				{
+				// That's ok, we handle it below.
+				}
+
 			else
 				{
 				reporter->Error("unsupported field type for log column");
@@ -894,9 +902,10 @@ bool LogMgr::Write(EnumVal* id, RecordVal* columns)
 
 		if ( filter->path_func )
 			{
-			val_list vl(2);
+			val_list vl(3);
 			vl.append(id->Ref());
 			vl.append(filter->path_val->Ref());
+			vl.append(columns->Ref());
 			Val* v = filter->path_func->Call(&vl);
 
 			if ( ! v->Type()->Tag() == TYPE_STRING )
@@ -907,6 +916,7 @@ bool LogMgr::Write(EnumVal* id, RecordVal* columns)
 				}
 
 			path = v->AsString()->CheckString();
+			Unref(v);
 
 #ifdef DEBUG
 			DBG_LOG(DBG_LOGGING, "Path function for filter '%s' on stream '%s' return '%s'",
@@ -1074,6 +1084,15 @@ LogVal* LogMgr::ValToLogVal(Val* val, BroType* ty)
 		break;
 		}
 
+	case TYPE_FUNC:
+		{
+		ODesc d;
+		const Func* f = val->AsFunc();
+		f->Describe(&d);
+		lval->val.string_val = new string(d.Description());
+		break;
+		}
+
 	case TYPE_TABLE:
 		{
 		ListVal* set = val->AsTableVal()->ConvertToPureList();

src/LogWriterAscii.cc

@@ -155,6 +155,7 @@ bool LogWriterAscii::DoWriteOne(ODesc* desc, LogVal* val, const LogField* field)
 	case TYPE_ENUM:
 	case TYPE_STRING:
 	case TYPE_FILE:
+	case TYPE_FUNC:
 		{
 		int size = val->val.string_val->size();
 		if ( size )

src/Obj.cc

@@ -127,6 +127,7 @@ void BroObj::BadTag(const char* msg, const char* t1, const char* t2) const
 	ODesc d;
 	DoMsg(&d, out);
 	reporter->FatalError("%s", d.Description());
+	reporter->PopLocation();
 	}
 
 void BroObj::Internal(const char* msg) const
@@ -134,6 +135,7 @@ void BroObj::Internal(const char* msg) const
 	ODesc d;
 	DoMsg(&d, msg);
 	reporter->InternalError("%s", d.Description());
+	reporter->PopLocation();
 	}
 
 void BroObj::InternalWarning(const char* msg) const
@@ -141,6 +143,7 @@ void BroObj::InternalWarning(const char* msg) const
 	ODesc d;
 	DoMsg(&d, msg);
 	reporter->InternalWarning("%s", d.Description());
+	reporter->PopLocation();
 	}
 
 void BroObj::AddLocation(ODesc* d) const

src/Reporter.cc

@@ -302,7 +302,7 @@ void Reporter::DoLog(const char* prefix, EventHandlerPtr event, FILE* out, Conne
 		s += buffer;
 		s += "\n";
 
-		fprintf(out, s.c_str());
+		fprintf(out, "%s", s.c_str());
 		}
 
 	if ( alloced )

src/bro.bif

@@ -3630,17 +3630,28 @@ function NFS3::mode2string%(mode: count%): string
 function piped_exec%(program: string, to_write: string%): bool
 	%{
 	const char* prog = program->CheckString();
+
 	FILE* f = popen(prog, "w");
 	if ( ! f )
 		{
 		reporter->Error("Failed to popen %s", prog);
-		return new Val(false, TYPE_BOOL);
+		return new Val(0, TYPE_BOOL);
 		}
 
-	fprintf(f, "%s", to_write->CheckString());
+	const u_char* input_data = to_write->Bytes();
+	int input_data_len = to_write->Len();
+
+	int bytes_written = fwrite(input_data, 1, input_data_len, f);
+
 	pclose(f);
 
-	return new Val(true, TYPE_BOOL);
+	if ( bytes_written != input_data_len )
+		{
+		reporter->Error("Failed to write all given data to %s", prog);
+		return new Val(0, TYPE_BOOL);
+		}
+
+	return new Val(1, TYPE_BOOL);
 	%}
 
 ## Enables the communication system. Note that by default,

src/parse.y

@@ -1070,10 +1070,10 @@ decl:
 			}
 
 	|	TOK_REDEF TOK_RECORD global_id TOK_ADD_TO
-			'{' { do_doc_token_start(); } type_decl_list '}' opt_attr ';'
+			'{' { ++in_record; do_doc_token_start(); }
+			type_decl_list
+			{ --in_record; do_doc_token_stop(); } '}' opt_attr ';'
 			{
-			do_doc_token_stop();
-
 			if ( ! $3->Type() )
 				$3->Error("unknown identifier");
 			else
@@ -1083,7 +1083,7 @@ decl:
 					$3->Error("not a record type");
 				else
 					{
-					const char* error = add_to->AddFields($7, $9);
+					const char* error = add_to->AddFields($7, $10);
 					if ( error )
 						$3->Error(error);
 					else if ( generate_documentation )

src/util.cc

@@ -757,9 +757,9 @@ const char* bro_path()
 	const char* path = getenv("BROPATH");
 	if ( ! path )
 		path = ".:"
-			POLICYDEST ":"
+			BRO_SCRIPT_INSTALL_PATH ":"
-			POLICYDEST "/policy" ":"
+			BRO_SCRIPT_INSTALL_PATH "/policy" ":"
-			POLICYDEST "/site";
+			BRO_SCRIPT_INSTALL_PATH "/site";
 
 	return path;
 	}
@@ -891,60 +891,36 @@ const char* normalize_path(const char* path)
 	return copy_string(new_path.c_str());
 	}
 
-// Returns the subpath of BROPATH's policy/ directory in which the loaded
+// Returns the subpath of the root Bro script install/source directory in
-// file in located.  If it's not under a subpath of policy/ then the full
+// which the loaded file is located.  If it's not under a subpath of that
-// path is returned, else the subpath of policy/ concatentated with any
+// directory (e.g. cwd or custom path) then the full path is returned.
-// directory prefix of the file is returned.
+void get_script_subpath(const std::string& full_filename, const char** subpath)
-void get_policy_subpath(const char* dir, const char* file, const char** subpath)
 	{
-	// first figure out if this is a subpath of policy/
+	size_t p;
-	const char* ploc = strstr(dir, "policy");
+	std::string my_subpath(full_filename);
-	if ( ploc )
-		if ( ploc[6] == '\0' )
-			*subpath = copy_string(ploc + 6);
-		else if ( ploc[6] == '/' )
-			*subpath = copy_string(ploc + 7);
-		else
-			*subpath = copy_string(dir);
-	else
-		*subpath = copy_string(dir);
 
-	// and now add any directory parts of the filename
+	// get the parent directory of file (if not already a directory)
-	char full_filename_buf[1024];
+	if ( ! is_dir(full_filename.c_str()) )
-	safe_snprintf(full_filename_buf, sizeof(full_filename_buf),
-			"%s/%s", dir, file);
-	char* tmp = copy_string(file);
-	const char* fdir = 0;
-
-	if ( is_dir(full_filename_buf) )
-		fdir = file;
-
-	if ( ! fdir )
-		fdir = dirname(tmp);
-
-	if ( ! streq(fdir, ".") )
 		{
-		size_t full_subpath_len = strlen(*subpath) + strlen(fdir) + 1;
+		char* tmp = copy_string(full_filename.c_str());
-		bool needslash = false;
+		my_subpath = dirname(tmp);
-		if ( strlen(*subpath) != 0 && (*subpath)[strlen(*subpath) - 1] != '/' )
+		delete [] tmp;
-			{
-			++full_subpath_len;
-			needslash = true;
-			}
-
-		char* full_subpath = new char[full_subpath_len];
-		strcpy(full_subpath, *subpath);
-		if ( needslash )
-			strcat(full_subpath, "/");
-		strcat(full_subpath, fdir);
-		delete [] *subpath;
-		*subpath = full_subpath;
 		}
 
-	const char* normalized_subpath = normalize_path(*subpath);
+	// first check if this is some subpath of the installed scripts root path,
-	delete [] tmp;
+	// if not check if it's a subpath of the script source root path,
-	delete [] *subpath;
+	// if neither, will just use the given directory
-	*subpath = normalized_subpath;
+	if ( (p=my_subpath.find(BRO_SCRIPT_INSTALL_PATH)) != std::string::npos )
+		my_subpath.erase(0, strlen(BRO_SCRIPT_INSTALL_PATH));
+	else if ( (p=my_subpath.find(BRO_SCRIPT_SOURCE_PATH)) != std::string::npos )
+		my_subpath.erase(0, strlen(BRO_SCRIPT_SOURCE_PATH));
+
+	// if root path found, remove path separators until next path component
+	if ( p != std::string::npos )
+		while ( my_subpath.size() && my_subpath[0] == '/' )
+			my_subpath.erase(0, 1);
+
+	*subpath = normalize_path(my_subpath.c_str());
 	}
 
 extern string current_scanned_file_path;
@@ -1001,7 +977,7 @@ FILE* search_for_file(const char* filename, const char* ext,
 		     ! is_dir(full_filename_buf) )
 			{
 			if ( bropath_subpath )
-				get_policy_subpath(dir_beginning, filename, bropath_subpath);
+				get_script_subpath(full_filename_buf, bropath_subpath);
 			return open_file(full_filename_buf, full_filename, load_pkgs);
 			}
 
@@ -1010,7 +986,7 @@ FILE* search_for_file(const char* filename, const char* ext,
 		if ( access(full_filename_buf, R_OK) == 0 )
 			{
 			if ( bropath_subpath )
-				get_policy_subpath(dir_beginning, filename, bropath_subpath);
+				get_script_subpath(full_filename_buf, bropath_subpath);
 			return open_file(full_filename_buf, full_filename, load_pkgs);
 			}
 

src/util.h

@@ -179,7 +179,7 @@ extern const char* bro_path();
 extern const char* bro_prefixes();
 std::string dot_canon(std::string path, std::string file, std::string prefix = "");
 const char* normalize_path(const char* path);
-void get_policy_subpath(const char* dir, const char* file, const char** subpath);
+void get_script_subpath(const std::string& full_filename, const char** subpath);
 extern FILE* search_for_file(const char* filename, const char* ext,
 	const char** full_filename, bool load_pkgs, const char** bropath_subpath);
 

testing/btest/Baseline/core.load-prefixes/output

@@ -1,4 +1,4 @@
-loaded lcl2.site.bro
+loaded lcl2.base.utils.site.bro
-loaded lcl.site.bro
+loaded lcl.base.utils.site.bro
-loaded lcl2.protocols.http.bro
+loaded lcl2.base.protocols.http.bro
-loaded lcl.protocols.http.bro
+loaded lcl.base.protocols.http.bro

testing/btest/Baseline/core.reporter-fmt-strings/output

@@ -0,0 +1 @@
+error in /Users/jsiwek/tmp/bro/testing/btest/.tmp/core.reporter-fmt-strings/reporter-fmt-strings.bro, line 9: not an event (dont_interpret_this(%s))

testing/btest/Baseline/doc.autogen-reST-example/example.rst

@@ -29,7 +29,7 @@ each of "columns", "event", "filter" depending on exactly what it's doing.
 
 :Author: Jon Siwek <jsiwek@ncsa.illinois.edu>
 
-:Imports: :doc:`frameworks/notice </policy/frameworks/notice/index>`
+:Imports: :doc:`policy/frameworks/software/vulnerable </scripts/policy/frameworks/software/vulnerable>`
 
 Summary
 ~~~~~~~

testing/btest/Baseline/language.record-extension/output

@@ -1,2 +1,10 @@
-[a=21, b=<uninitialized>, c=42, d=<uninitialized>]
+[a=21, b=<uninitialized>, myset={
-[a=21, b=<uninitialized>, c=42, d=XXX]
+
+}, c=42, d=<uninitialized>, anotherset={
+
+}]
+[a=21, b=<uninitialized>, myset={
+
+}, c=42, d=XXX, anotherset={
+
+}]

testing/btest/Baseline/language.vector-list-init-records/output

@@ -0,0 +1,3 @@
+element 0 = [s=bar, o=check]
+element 1 = [s=baz, o=<uninitialized>]
+[[s=bar, o=check], [s=baz, o=<uninitialized>]]

testing/btest/Baseline/policy.frameworks.logging.adapt-filter/ssh-new-default.log

@@ -1,3 +1,3 @@
 # t	id.orig_h	id.orig_p	id.resp_h	id.resp_p	status	country
-1299718503.40319	1.2.3.4	1234	2.3.4.5	80	success	unknown
+1313212563.234939	1.2.3.4	1234	2.3.4.5	80	success	unknown
-1299718503.40319	1.2.3.4	1234	2.3.4.5	80	failure	US
+1313212563.234939	1.2.3.4	1234	2.3.4.5	80	failure	US

testing/btest/Baseline/policy.frameworks.logging.path-func/output

@@ -1,13 +1,21 @@
-static-prefix-0.log
+static-prefix-0-BR.log
-static-prefix-1.log
+static-prefix-0-MX3.log
-static-prefix-2.log
+static-prefix-0-unknown.log
+static-prefix-1-MX.log
+static-prefix-1-US.log
+static-prefix-2-MX2.log
+static-prefix-2-UK.log
 # t	id.orig_h	id.orig_p	id.resp_h	id.resp_p	status	country
-1299718503.05867	1.2.3.4	1234	2.3.4.5	80	success	unknown
+1313212701.542245	1.2.3.4	1234	2.3.4.5	80	success	BR
-1299718503.05867	1.2.3.4	1234	2.3.4.5	80	success	BR
-1299718503.05867	1.2.3.4	1234	2.3.4.5	80	failure	MX3
 # t	id.orig_h	id.orig_p	id.resp_h	id.resp_p	status	country
-1299718503.05867	1.2.3.4	1234	2.3.4.5	80	failure	US
+1313212701.542245	1.2.3.4	1234	2.3.4.5	80	failure	MX3
-1299718503.05867	1.2.3.4	1234	2.3.4.5	80	failure	MX
 # t	id.orig_h	id.orig_p	id.resp_h	id.resp_p	status	country
-1299718503.05867	1.2.3.4	1234	2.3.4.5	80	failure	UK
+1313212701.542245	1.2.3.4	1234	2.3.4.5	80	success	unknown
-1299718503.05867	1.2.3.4	1234	2.3.4.5	80	failure	MX2
+# t	id.orig_h	id.orig_p	id.resp_h	id.resp_p	status	country
+1313212701.542245	1.2.3.4	1234	2.3.4.5	80	failure	MX
+# t	id.orig_h	id.orig_p	id.resp_h	id.resp_p	status	country
+1313212701.542245	1.2.3.4	1234	2.3.4.5	80	failure	US
+# t	id.orig_h	id.orig_p	id.resp_h	id.resp_p	status	country
+1313212701.542245	1.2.3.4	1234	2.3.4.5	80	failure	MX2
+# t	id.orig_h	id.orig_p	id.resp_h	id.resp_p	status	country
+1313212701.542245	1.2.3.4	1234	2.3.4.5	80	failure	UK

testing/btest/Baseline/policy.misc.default-loaded-scripts/canonified_loaded_scripts.log

@@ -27,6 +27,8 @@
 2	scripts/base/frameworks/notice/./actions/drop.bro
 2	scripts/base/frameworks/notice/./actions/email_admin.bro
 2	scripts/base/frameworks/notice/./actions/page.bro
+2	scripts/base/frameworks/notice/./actions/add-geodata.bro
+2	scripts/base/frameworks/notice/./extend-email/hostnames.bro
 1	scripts/base/frameworks/dpd/__load__.bro
 2	scripts/base/frameworks/dpd/./main.bro
 1	scripts/base/frameworks/signatures/__load__.bro
@@ -41,6 +43,7 @@
 1	scripts/base/frameworks/metrics/__load__.bro
 2	scripts/base/frameworks/metrics/./main.bro
 1	scripts/base/frameworks/communication/__load__.bro
+2	scripts/base/frameworks/communication/./main.bro
 1	scripts/base/frameworks/control/__load__.bro
 2	scripts/base/frameworks/control/./main.bro
 1	scripts/base/frameworks/cluster/__load__.bro
@@ -69,6 +72,8 @@
 2	scripts/base/protocols/irc/./dcc-send.bro
 1	scripts/base/protocols/smtp/__load__.bro
 2	scripts/base/protocols/smtp/./main.bro
+2	scripts/base/protocols/smtp/./entities.bro
+2	scripts/base/protocols/smtp/./entities-excerpt.bro
 1	scripts/base/protocols/ssh/__load__.bro
 2	scripts/base/protocols/ssh/./main.bro
 1	scripts/base/protocols/ssl/__load__.bro

testing/btest/Baseline/policy.protocols.smtp.basic/smtp.log

@@ -0,0 +1,2 @@
+# ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	mid	helo	mailfrom	rcptto	date	from	to	reply_to	msg_id	in_reply_to	subject	x_originating_ip	first_received	second_received	last_reply	path	user_agent
+1254722768.219663	56gKBmhBBB6	10.10.1.4	1470	74.53.140.153	25	@50da4BEzauh	GP	<gurpartap@patriots.in>	<raj_deol2002in@yahoo.co.in>	Mon, 5 Oct 2009 11:36:07 +0530	"Gurpartap Singh" <gurpartap@patriots.in>	<raj_deol2002in@yahoo.co.in>	-	<000301ca4581$ef9e57f0$cedb07d0$@in>	-	SMTP	-	-	-	250 OK id=1Mugho-0003Dg-Un	74.53.140.153,10.10.1.4	Microsoft Office Outlook 12.0

testing/btest/Baseline/policy.protocols.smtp.mime-extract/smtp-entity_10.10.1.4:1470-74.53.140.153:25_1.dat

@@ -0,0 +1,13 @@
+Hello
+
+ 
+
+I send u smtp pcap file 
+
+Find the attachment
+
+ 
+
+GPS
+
+

testing/btest/Baseline/policy.protocols.smtp.mime-extract/smtp-entity_10.10.1.4:1470-74.53.140.153:25_2.dat

@@ -0,0 +1,264 @@
+Version 4.9.9.1
+* Many bug fixes
+* Improved editor
+
+Version 4.9.9.0
+* Support for latest Mingw compiler system builds
+* Bug fixes
+
+Version 4.9.8.9
+* New code tooltip display
+* Improved Indent/Unindent and Remove Comment
+* Improved automatic indent
+* Added support for the "interface" keyword
+* WebUpdate should now report installation problems from PackMan
+* New splash screen and association icons
+* Improved installer
+* Many bug fixes
+
+Version 4.9.8.7
+* Added support for GCC > 3.2
+* Debug variables are now resent during next debug session
+* Watched Variables not in correct context are now kept and updated when it is needed
+* Added new compiler/linker options: 20
+  - Strip executable
+  - Generate instructions for a specific machine (i386, i486, i586, i686, pentium, pentium-mmx, pentiumpro, pentium2, pentium3, pentium4, 20
+    k6, k6-2, k6-3, athlon, athlon-tbird, athlon-4, athlon-xp, athlon-mp, winchip-c6, winchip2, k8, c3 and c3-2)
+  - Enable use of processor specific built-in functions (mmmx, sse, sse2, pni, 3dnow)
+* "Default" button in Compiler Options is back
+* Error messages parsing improved
+* Bug fixes
+
+Version 4.9.8.5
+* Added the possibility to modify the value of a variable during debugging (right click on a watch variable and select "Modify value")
+* During Dev-C++ First Time COnfiguration window, users can now choose between using or not class browser and code completion features.
+* Many bug fixes
+
+Version 4.9.8.4
+* Added the possibility to specify an include directory for the code completion cache to be created at Dev-C++ first startup
+* Improved code completion cache
+* WebUpdate will now backup downloaded DevPaks in Dev-C++\Packages directory, and Dev-C++ executable in devcpp.exe.BACKUP
+* Big speed up in function parameters listing while editing
+* Bug fixes
+
+Version 4.9.8.3
+* On Dev-C++ first time configuration dialog, a code completion cache of all the standard 20
+  include files can now be generated.
+* Improved WebUpdate module
+* Many bug fixes
+
+Version 4.9.8.2
+* New debug feature for DLLs: attach to a running process
+* New project option: Use custom Makefile. 20
+* New WebUpdater module.
+* Allow user to specify an alternate configuration file in Environment Options 20
+  (still can be overriden by using "-c" command line parameter).
+* Lots of bug fixes.
+
+Version 4.9.8.1
+* When creating a DLL, the created static lib respects now the project-defined output directory
+
+Version 4.9.8.0
+* Changed position of compiler/linker parameters in Project Options.
+* Improved help file
+* Bug fixes
+
+Version 4.9.7.9
+* Resource errors are now reported in the Resource sheet
+* Many bug fixes
+
+Version 4.9.7.8
+* Made whole bottom report control floating instead of only debug output.
+* Many bug fixes
+
+Version 4.9.7.7
+* Printing settings are now saved
+* New environment options : "watch variable under mouse" and "Report watch errors"
+* Bug fixes
+
+Version 4.9.7.6
+* Debug variable browser
+* Added possibility to include in a Template the Project's directories (include, libs and ressources)
+* Changed tint of Class browser pictures colors to match the New Look style
+* Bug fixes
+
+Version 4.9.7.5
+* Bug fixes
+
+Version 4.9.7.4
+* When compiling with debugging symbols, an extra definition is passed to the
+  compiler: -D__DEBUG__
+* Each project creates a <project_name>_private.h file containing version
+  information definitions
+* When compiling the current file only, no dependency checks are performed
+* ~300% Speed-up in class parser
+* Added "External programs" in Tools/Environment Options (for units "Open with")
+* Added "Open with" in project units context menu
+* Added "Classes" toolbar
+* Fixed pre-compilation dependency checks to work correctly
+* Added new file menu entry: Save Project As
+* Bug-fix for double quotes in devcpp.cfg file read by vUpdate
+* Other bug fixes
+
+Version 4.9.7.3
+* When adding debugging symbols on request, remove "-s" option from linker
+* Compiling progress window
+* Environment options : "Show progress window" and "Auto-close progress window"
+* Bug fixes
+
+Version 4.9.7.2
+* Bug fixes
+
+Version 4.9.7.1
+* "Build priority" per-unit
+* "Include file in linking process" per-unit
+* New feature: compile current file only
+* Separated C++ compiler options from C compiler options in Makefile (see bug report #654744)
+* Separated C++ include dirs from C include dirs in Makefile (see bug report #654744)
+* Necessary UI changes in Project Options
+* Added display of project filename, project output and a summary of the project files in Project Options General tab.
+* Fixed the "compiler-dirs-with-spaces" bug that crept-in in 4.9.7.0
+* Multi-select files in project-view (when "double-click to open" is configured in Environment Settings)
+* Resource files are treated as ordinary files now
+* Updates in "Project Options/Files" code
+* MSVC import now creates the folders structure of the original VC project
+* Bug fixes
+
+Version 4.9.7.0
+* Allow customizing of per-unit compile command in projects
+* Added two new macros: <DATE> and <DATETIME>
+* Added support for macros in the "default source code" (Tools/Editor Options/Code)
+* Separated layout info from project file. It is now kept in a different file
+  (the same filename as the project's but with extension ".layout"). If you
+  have your project under CVS control, you ''ll know why this had to happen...
+* Compiler settings per-project
+* Compiler set per-project
+* Implemented new compiler settings framework
+* "Compile as C++" per-unit
+* "Include file in compilation process" per-unit
+* Project version info (creates the relevant VERSIONINFO struct in the private
+  resource)
+* Support XP Themes (creates the CommonControls 6.0 manifest file and includes
+  it in the private resource)
+* Added CVS "login" and "logout" commands
+* Project manager and debugging window (in Debug tab) can now be trasnformed into floating windows.
+* Added "Add Library" button in Project Options
+* Bug fixes
+
+Version 4.9.6.9
+* Implemented search in help files for the word at cursor (context sensitive help)
+* Implemented "compiler sets" infrastructure to switch between different compilers easily (e.g. gcc-2.95 and gcc-3.2)
+* Added "Files" tab in CVS form to allow selection of more than one file for
+  the requested CVS action
+  20
+Version 4.9.6.8
+* support for DLL application hosting, for debugging and executing DLLs under Dev-C++.
+* New class browser option: "Show inherited members"
+* Added support for the '::' member access operator in code-completion
+* Added *working* function arguments hint
+* Added bracket highlighting. When the caret is on a bracket, that bracket and
+  its counterpart are highlighted
+* Nested folders in project view
+
+Version 4.9.6.7
+* XP Theme support
+* Added CVS commands "Add" and "Remove"
+* Added configuration option for "Templates Directory" in "Environment Options"
+* Code-completion updates
+* Bug fixes
+
+Version 4.9.6.6
+* Editor colors are initialized properly on Dev-C++ first-run
+* Added doxygen-style comments in NewClass, NewMemberFunction and NewMemberVariable wizards
+* Added file's date/time stamp in File/Properties window
+* Current windows listing in Window menu
+* Bug fixes
+
+Version 4.9.6.5
+* CVS support
+* Window list (in Window menu)
+* bug fixes
+
+version 4.9.6.4
+* added ENTER key for opening file in project browser, DEL to delete from the project.
+* bug fixes
+
+version 4.9.6.3
+* Bug fixes
+
+version 4.9.6.2
+* Bug fixes
+
+version 4.9.6.1
+* New "Abort compilation" button
+* Bug fixes
+* Now checks for vRoach existance when sending a crash report
+
+Version 4.9.5.5
+* New option in Editor Options: Show editor hints. User can disable the hints
+  displayed in the editor when the mouse moves over a word. Since this was the
+  cause of many errors (although it should be fixed by now), we are giving the
+  user the option to disable this feature.
+* New option in Editor Options (code-completion): Use code-completion cache.
+  Well, it adds caching to code-completion. Depending on the cache size,
+  the program may take a bit longer to start-up, but provides very fast
+  code-completion and the user has all the commands (belonging to the files
+  he added in the cache) at his fingertips. If, for example, the user adds
+  "windows.h", he gets all the WinAPI! If he adds "wx/wx.h", he gets all of
+  wxWindows! You get the picture...
+* Removed "Only show classes from current file" option in class browser settings.
+  It used to be a checkbox, allowing only two states (on or off), but there is
+  a third relevant option now: "Project classes" so it didn't fit the purpose...
+  The user can define this in the class browser's context menu under "View mode".
+* Fixed the dreaded "Clock skew detected" compiler warning!
+* Fixed many class browser bugs, including some that had to do with class folders.
+
+Version 4.9.5.4
+* Under NT, 2000 and XP, user application data directory will be used to store config files (i.e : C:\Documents and Settings\Username\Local Settings\Application Data)
+
+Version 4.9.5.3
+* Added ExceptionsAnalyzer. If the devcpp.map file is in the devcpp.exe directory
+  then we even get a stack trace in the bug report!
+* Added new WebUpdate module (inactive temporarily).
+* Added new code for code-completion caching of files (disabled - work in progress).
+
+Version 4.9.5.2
+* Added new option in class-browser: Use colors
+  (available when right-clicking the class-browser
+  and selecting "View mode").
+* Dev-C++ now traps access violation of your programs (and of itself too ;)
+
+Version 4.9.5.1
+* Implemented the "File/Export/Project to HTML" function.
+* Added "Tip of the day" system.
+* When running a source file in explorer, don't spawn new instance.
+  Instead open the file in an already launched Dev-C++.
+* Class-parser speed-up (50% to 85% improvement timed!!!)
+* Many code-completion updates. Now takes into account context,
+  class inheritance and visibility (shows items only from files
+  #included directly or indirectly)!
+* Caching of result set of code-completion for speed-up.
+* New option "Execution/Parameters" (and "Debug/Parameters").
+
+Version 4.9.5.0 (5.0 beta 5):
+* CPU Window (still in development)
+* ToDo  list
+* Backtrace in debugging
+* Run to cursor
+* Folders in Project and Class Browser
+* Send custom commands to GDB
+* Makefile can now be customized.
+* Modified the behaviour of the -c param : 20
+  -c <config file directory>
+* Saving of custom syntax parameter group
+* Possibility of changing compilers and tools filename.
+* Many bug fixes
+
+
+Version 4.9.4.1 (5.0 beta 4.1):
+
+* back to gcc 2.95.3
+* Profiling support
+* new update/packages checker (vUpdate)
+* Lots of bugfixes
+

testing/btest/Baseline/policy.protocols.smtp.mime-extract/smtp_entities.log

@@ -0,0 +1,4 @@
+# ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	mid	filename	content_len	mime_type	md5	extraction_file	excerpt
+1254722770.692743	56gKBmhBBB6	10.10.1.4	1470	74.53.140.153	25	@50da4BEzauh	-	79	FAKE_MIME	-	smtp-entity_10.10.1.4:1470-74.53.140.153:25_1.dat	-
+1254722770.692743	56gKBmhBBB6	10.10.1.4	1470	74.53.140.153	25	@50da4BEzauh	-	1918	FAKE_MIME	-	-	-
+1254722770.692804	56gKBmhBBB6	10.10.1.4	1470	74.53.140.153	25	@50da4BEzauh	NEWS.txt	10823	FAKE_MIME	-	smtp-entity_10.10.1.4:1470-74.53.140.153:25_2.dat	-

testing/btest/Baseline/policy.protocols.smtp.mime/smtp_entities.log

@@ -0,0 +1,4 @@
+# ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	mid	filename	content_len	mime_type	md5	extraction_file	excerpt
+1254722770.692743	56gKBmhBBB6	10.10.1.4	1470	74.53.140.153	25	@50da4BEzauh	-	79	FAKE_MIME	92bca2e6cdcde73647125da7dccbdd07	-	-
+1254722770.692743	56gKBmhBBB6	10.10.1.4	1470	74.53.140.153	25	@50da4BEzauh	-	1918	FAKE_MIME	-	-	-
+1254722770.692804	56gKBmhBBB6	10.10.1.4	1470	74.53.140.153	25	@50da4BEzauh	NEWS.txt	10823	FAKE_MIME	a968bb0f9f9d95835b2e74c845877e87	-	-

testing/btest/bifs/piped_exec.bro

@@ -1,6 +1,12 @@
 # @TEST-EXEC: bro %INPUT >output
 # @TEST-EXEC: btest-diff output
+# @TEST-EXEC: btest-diff test.txt
+
 
 global cmds = "print \"hello world\";";
 cmds = string_cat(cmds, "\nprint \"foobar\";");
 piped_exec("bro", cmds);
+
+# Test null output.
+piped_exec("cat > test.txt", "\x00\x00hello\x00\x00");
+

testing/btest/core/load-prefixes.bro

@@ -9,11 +9,11 @@
 @TEST-END-FILE
 
 @TEST-START-FILE lcl.base.utils.site.bro
-print "loaded lcl.base.site.bro";
+print "loaded lcl.base.utils.site.bro";
 @TEST-END-FILE
 
 @TEST-START-FILE lcl2.base.utils.site.bro
-print "loaded lcl2.base.site.bro";
+print "loaded lcl2.base.utils.site.bro";
 @TEST-END-FILE
 
 @TEST-START-FILE lcl.base.protocols.http.bro

testing/btest/core/reporter-fmt-strings.bro

@@ -0,0 +1,10 @@
+# The format string below should end up as a literal part of the reporter's
+# error message to stderr and shouldn't be replaced internally.
+#
+# @TEST-EXEC-FAIL: bro %INPUT >output 2>&1
+# @TEST-EXEC:      TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff output
+
+event bro_init()
+{
+    event dont_interpret_this("%s");
+}

testing/btest/doc/blacklist-reminder.test

@@ -0,0 +1,8 @@
+# This test will fail if there are Bro scripts that have been temporarily
+# blacklisted from the documentation generation process for some reason
+# (e.g. they're a work-in-progress or otherwise fail to parse).  It's meant
+# to serve as a reminder that some future action may be needed to generate
+# documentation for the blacklisted scripts.
+#
+# @TEST-EXEC: $DIST/doc/scripts/genDocSourcesList.sh
+# @TEST-EXEC: btest-diff .stderr

testing/btest/language/record-extension.bro

@@ -4,11 +4,13 @@
 type Foo: record {
         a: count;
         b: count &optional;
+        myset: set[count] &default=set();
 };
 
 redef record Foo += {
         c: count &default=42;
         d: count &optional;
+        anotherset: set[count] &default=set();
 };
 
 global f1: Foo = [$a=21];

testing/btest/language/vector-list-init-records.bro

@@ -0,0 +1,20 @@
+# Initializing a vector with a list of records should promote elements as
+# necessary to match the vector's yield type.
+
+# @TEST-EXEC: bro %INPUT >output
+# @TEST-EXEC: btest-diff output
+
+type Foo: record {
+	s: string;
+	o: string &optional;
+};
+
+const v: vector of Foo = {
+	[$s="bar", $o="check"],
+	[$s="baz"]
+};
+
+for ( i in v )
+	print fmt("element %d = %s", i, v[i]);
+
+print v;

testing/btest/policy/frameworks/logging/path-func.bro

@@ -21,11 +21,11 @@ export {
 
 global c = -1;
 
-function path_func(id: Log::ID, path: string) : string
+function path_func(id: Log::ID, path: string, rec: Log) : string
 	{
 	c = (c + 1) % 3;
 
-	return fmt("%s-%d", path, c);
+	return fmt("%s-%d-%s", path, c, rec$country);
 	}
 
 event bro_init()

testing/btest/policy/frameworks/logging/types.bro

@@ -29,9 +29,18 @@ export {
 		se: set[string];
 		vc: vector of count;
 		ve: vector of string;
+		f: function(i: count) : string;
 	} &log;
 }
 
+function foo(i : count) : string
+	{
+	if ( i > 0 )
+		return "Foo";
+	else
+		return "Bar";
+	}
+
 event bro_init()
 {
 	Log::create_stream(SSH, [$columns=Log]);
@@ -56,7 +65,8 @@ event bro_init()
 		$ss=set("AA", "BB", "CC"),
 		$se=empty_set,
 		$vc=vector(10, 20, 30),
-		$ve=empty_vector
+		$ve=empty_vector,
+		$f=foo
 		]);
 }
 

testing/btest/policy/protocols/smtp/basic.test

@@ -0,0 +1,4 @@
+# @TEST-EXEC: bro -r $TRACES/smtp.trace %INPUT
+# @TEST-EXEC: btest-diff smtp.log
+
+@load base/protocols/smtp

testing/btest/policy/protocols/smtp/mime-extract.test

@@ -0,0 +1,25 @@
+# @TEST-REQUIRES: grep -q '#define HAVE_LIBMAGIC' $BUILD/config.h
+# @TEST-EXEC: bro -r $TRACES/smtp.trace %INPUT
+# @TEST-EXEC: btest-diff smtp_entities.log
+# @TEST-EXEC: btest-diff smtp-entity_10.10.1.4:1470-74.53.140.153:25_1.dat
+# @TEST-EXEC: btest-diff smtp-entity_10.10.1.4:1470-74.53.140.153:25_2.dat
+# @TEST-EXEC: bro -r $TRACES/smtp.trace %INPUT SMTP::extraction_prefix="test"
+# @TEST-EXEC: test -e test_10.10.1.4:1470-74.53.140.153:25_1.dat
+# @TEST-EXEC: test -e test_10.10.1.4:1470-74.53.140.153:25_2.dat
+
+@load base/protocols/smtp
+
+redef SMTP::extract_file_types=/text\/plain/;
+
+event bro_init()
+	{
+	Log::remove_default_filter(SMTP::SMTP_ENTITIES);
+	Log::add_filter(SMTP::SMTP_ENTITIES, [$name="normalized-mime-types",
+									  	$pred=function(rec: SMTP::EntityInfo): bool
+		{
+		if ( rec?$mime_type )
+			rec$mime_type = "FAKE_MIME";
+		return T;
+		}
+	]);
+	}