Merge remote-tracking branch 'origin/topic/jsiwek/autodoc-fixes'

* origin/topic/jsiwek/autodoc-fixes:
  Update doc sources and touch up a few script comments.
  Fixes for script auto-documentation.

Conflicts:
	scripts/base/frameworks/logging/main.bro

CHANGES

@@ -1,4 +1,66 @@
 
+1.6-dev-1095 | 2011-08-13 11:59:07 -0700
+
+  * A larger number of script documentation updates. Closes #543. (Jon
+    Siwek)
+
+  * Workaround for FreeBSD CMake port missing debug flags. (Jon Siwek)
+    
+  * piped_exec() can now deal with null bytes. (Seth Hall)
+    
+  * Fix vector initialization for lists of records with optional
+    types. Closes #485. (Jon Siwek)
+
+  * Fix redef'ing records with &default empty set fields. Closes #460.
+    (Jon Siwek)
+
+  * Fix ConnSize_Analyzer when used in conjunction with the connection
+    compressor. (Gregor Maier)
+
+  * Fix reporter using part of the actual message as a format string.
+    (Jon Siwek)
+
+  * Fixing reporter's location tracking. Closes #492. (Robin Sommer)
+    
+  * Turning DNS errors into warnings. Closes #255. (Robin Sommer)
+    
+  * Logging's path_func now receives the log record as argument.
+    Closes #555. (Robin Sommer)
+
+  * Functions can now be logged; their full body gets recorded. 
+    Closes #506. (Robin Sommer)
+
+  * Bugfix for hostname notice email extension. (Seth Hall)
+    
+  * Updates for notice framework. (Seth Hall)
+    
+    - New ACTION_ADD_GEODATA to add geodata to notices in an extension
+      field named remote_location.
+
+    - Loading extend-email/hostnames by default now that it only does
+      anything when the ACTION_EMAIL action is applied (finally).
+
+  * Updates to local.bro (Seth Hall)
+
+  * Added the profiling script. (Seth Hall)
+    
+  * Updates for SSH scripts. (Seth Hall)
+
+  * ConnSize analyzer is turned on by default now. (Seth Hall)
+
+  * Updates for the build system and site local scripts for cluster.
+    (Seth Hall)
+
+  * HTTP now uses the extract_filename_from_content_disposition function. (Seth Hall)
+
+  * Major SMTP script refactor. Closes #509. (Jon Siwek and Seth Hall)
+    
+  * New variable Site::local_nets_table in utils/site for mapping
+    address to defined local subnet.
+
+  * Metrics framework updates, more to come. (Seth Hall)
+    
+
 1.6-dev-1061 | 2011-08-08 18:25:27 -0700
 
   * A set of new/changed tests regarding the new policy script

CMakeLists.txt

@@ -35,14 +35,16 @@ endif ()
 ## Project/Build Configuration
 
 set(BRO_ROOT_DIR ${CMAKE_INSTALL_PREFIX})
-if (NOT POLICYDIR)
+if (NOT BRO_SCRIPT_INSTALL_PATH)
-    # set the default policy installation path (user did not specify one)
+    # set the default Bro script installation path (user did not specify one)
-    set(POLICYDIR ${BRO_ROOT_DIR}/share/bro)
+    set(BRO_SCRIPT_INSTALL_PATH ${BRO_ROOT_DIR}/share/bro)
 endif ()
+set(BRO_SCRIPT_SOURCE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/scripts)
 
-# sanitize the policy install directory into an absolute path
+# sanitize the Bro script install directory into an absolute path
 # (CMake is confused by ~ as a representation of home directory)
-get_filename_component(POLICYDIR ${POLICYDIR} ABSOLUTE)
+get_filename_component(BRO_SCRIPT_INSTALL_PATH ${BRO_SCRIPT_INSTALL_PATH}
+    ABSOLUTE)
 
 configure_file(bro-path-dev.in ${CMAKE_CURRENT_BINARY_DIR}/bro-path-dev)
 file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/bro-path-dev.sh
@@ -213,7 +215,7 @@ message(
     "\n====================|  Bro Build Summary  |====================="
     "\n"
     "\nInstall prefix:    ${CMAKE_INSTALL_PREFIX}"
-    "\nPolicy dir:        ${POLICYDIR}"
+    "\nBro Script Path:   ${BRO_SCRIPT_INSTALL_PATH}"
     "\nDebug mode:        ${ENABLE_DEBUG}"
     "\n"
     "\nCC:                ${CMAKE_C_COMPILER}"

VERSION

@@ -1 +1 @@
-1.6-dev-1061
+1.6-dev-1095

bro-path-dev.in

@@ -1,7 +1,7 @@
 #!/bin/sh
-# After configured by CMake, this file prints the absolute path to policy
+# After configured by CMake, this file prints the absolute path to Bro scripts
-# files that come with the source distributions of Bro as well as policy 
+# that come with the source distributions of Bro as well as scripts that are
-# files that are generated by the BIF compiler at compile time.
+# generated by the BIF compiler at compile time.
 #
 # The intended use of this script is to make it easier to run Bro from
 # the build directory, avoiding the need to install it.  This could be
@@ -10,10 +10,10 @@
 #     BROPATH=`./bro-path-dev` ./src/bro
 #
 
-broPolicies=${PROJECT_SOURCE_DIR}/scripts:${PROJECT_SOURCE_DIR}/scripts/policy:${PROJECT_SOURCE_DIR}/scripts/site
+broPolicies=${BRO_SCRIPT_SOURCE_PATH}:${BRO_SCRIPT_SOURCE_PATH}/policy:${BRO_SCRIPT_SOURCE_PATH}/site
 
 broGenPolicies=${CMAKE_BINARY_DIR}/src
 
-installedPolicies=${POLICYDIR}:${POLICYDIR}/site
+installedPolicies=${BRO_SCRIPT_INSTALL_PATH}:${BRO_SCRIPT_INSTALL_PATH}/site
 
 echo .:$broPolicies:$broGenPolicies

configure

@@ -22,7 +22,7 @@ Usage: $0 [OPTION]... [VAR=VALUE]...
 
   Installation Directories:
     --prefix=PREFIX        installation directory [/usr/local/bro]
-    --policydir=PATH       policy file installation directory
+    --scriptdir=PATH       root installation directory for Bro scripts
                            [PREFIX/share/bro]
 
   Optional Features:
@@ -85,7 +85,7 @@ CMakeCacheEntries=""
 append_cache_entry CMAKE_INSTALL_PREFIX PATH   /usr/local/bro
 append_cache_entry BRO_ROOT_DIR         PATH   /usr/local/bro
 append_cache_entry PY_MOD_INSTALL_DIR   PATH   /usr/local/bro/lib/broctl
-append_cache_entry POLICYDIR            STRING /usr/local/bro/share/bro
+append_cache_entry BRO_SCRIPT_INSTALL_PATH STRING /usr/local/bro/share/bro
 append_cache_entry ENABLE_DEBUG         BOOL   false
 append_cache_entry BROv6                BOOL   false
 append_cache_entry ENABLE_PERFTOOLS     BOOL   false
@@ -118,13 +118,13 @@ while [ $# -ne 0 ]; do
             append_cache_entry CMAKE_INSTALL_PREFIX PATH   $optarg
             append_cache_entry BRO_ROOT_DIR         PATH   $optarg
             append_cache_entry PY_MOD_INSTALL_DIR   PATH   $optarg/lib/broctl
-            if [ "$user_set_policydir" != "true" ]; then
+            if [ "$user_set_scriptdir" != "true" ]; then
-                append_cache_entry POLICYDIR        STRING $optarg/share/bro
+                append_cache_entry BRO_SCRIPT_INSTALL_PATH STRING $optarg/share/bro
             fi
             ;;
-        --policydir=*)
+        --scriptdir=*)
-            append_cache_entry POLICYDIR            STRING $optarg
+            append_cache_entry BRO_SCRIPT_INSTALL_PATH STRING $optarg
-            user_set_policydir="true"
+            user_set_scriptdir="true"
             ;;
         --enable-debug)
             append_cache_entry ENABLE_DEBUG         BOOL   true

doc/scripts/CMakeLists.txt

@@ -57,10 +57,12 @@ macro(REST_TARGET srcDir broInput)
     get_filename_component(relDstDir ${broInput} PATH)
 
     set(sumTextSrc ${absSrcPath})
+    set(ogSourceFile ${absSrcPath})
     if (${extension} STREQUAL ".bif.bro")
+        set(ogSourceFile ${BIF_SRC_DIR}/${basename})
         # the summary text is taken at configure time, but .bif.bro files
         # may not have been generated yet, so read .bif file instead
-        set(sumTextSrc ${BIF_SRC_DIR}/${basename})
+        set(sumTextSrc ${ogSourceFile})
     endif ()
 
     if (NOT relDstDir)
@@ -124,9 +126,9 @@ macro(REST_TARGET srcDir broInput)
         ARGS -E make_directory ${dstDir}
         COMMAND "${CMAKE_COMMAND}"
         ARGS -E copy ${restFile} ${restOutput}
-        # copy the bro policy script, too
+        # copy the bro or bif script, too
         COMMAND "${CMAKE_COMMAND}"
-        ARGS -E copy ${absSrcPath} ${dstDir}
+        ARGS -E copy ${ogSourceFile} ${dstDir}
         # clean up the build directory
         COMMAND rm
         ARGS -rf .state *.log *.rst
@@ -151,6 +153,8 @@ set(MASTER_PKG_INDEX_TEXT "")
 foreach (pkg ${MASTER_PKG_LIST})
     # strip of the trailing /index for the link name
     get_filename_component(lnktxt ${pkg} PATH)
+    # pretty-up the link name by removing common scripts/ prefix
+    string(REPLACE "scripts/" "" lnktxt "${lnktxt}")
     set(MASTER_PKG_INDEX_TEXT "${MASTER_PKG_INDEX_TEXT}\n   ${lnktxt} <${pkg}>")
 endforeach ()
 file(WRITE ${MASTER_PACKAGE_INDEX} "${MASTER_PKG_INDEX_TEXT}")

doc/scripts/DocSourcesList.cmake

@@ -13,22 +13,14 @@
 set(psd ${PROJECT_SOURCE_DIR}/scripts)
 
 rest_target(${CMAKE_CURRENT_SOURCE_DIR} example.bro internal)
-rest_target(${psd} base/bro.init internal)
+rest_target(${psd} base/init-default.bro internal)
-rest_target(${psd} base/all.bro internal)
+rest_target(${psd} base/init-bare.bro internal)
 
 rest_target(${CMAKE_BINARY_DIR}/src bro.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/src common-rw.bif.bro)
 rest_target(${CMAKE_BINARY_DIR}/src const.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/src dns-rw.bif.bro)
 rest_target(${CMAKE_BINARY_DIR}/src event.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/src finger-rw.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/src ftp-rw.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/src http-rw.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/src ident-rw.bif.bro)
 rest_target(${CMAKE_BINARY_DIR}/src logging.bif.bro)
 rest_target(${CMAKE_BINARY_DIR}/src reporter.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/src smb-rw.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/src smtp-rw.bif.bro)
 rest_target(${CMAKE_BINARY_DIR}/src strings.bif.bro)
 rest_target(${CMAKE_BINARY_DIR}/src types.bif.bro)
 rest_target(${psd} base/frameworks/cluster/main.bro)
@@ -42,9 +34,8 @@ rest_target(${psd} base/frameworks/dpd/main.bro)
 rest_target(${psd} base/frameworks/intel/main.bro)
 rest_target(${psd} base/frameworks/logging/main.bro)
 rest_target(${psd} base/frameworks/logging/writers/ascii.bro)
-rest_target(${psd} base/frameworks/metrics/conn-example.bro)
-rest_target(${psd} base/frameworks/metrics/http-example.bro)
 rest_target(${psd} base/frameworks/metrics/main.bro)
+rest_target(${psd} base/frameworks/notice/actions/add-geodata.bro)
 rest_target(${psd} base/frameworks/notice/actions/drop.bro)
 rest_target(${psd} base/frameworks/notice/actions/email_admin.bro)
 rest_target(${psd} base/frameworks/notice/actions/page.bro)
@@ -56,6 +47,7 @@ rest_target(${psd} base/frameworks/packet-filter/netstats.bro)
 rest_target(${psd} base/frameworks/reporter/main.bro)
 rest_target(${psd} base/frameworks/signatures/main.bro)
 rest_target(${psd} base/frameworks/software/main.bro)
+rest_target(${psd} base/frameworks/time-machine/notice.bro)
 rest_target(${psd} base/protocols/conn/contents.bro)
 rest_target(${psd} base/protocols/conn/inactivity.bro)
 rest_target(${psd} base/protocols/conn/main.bro)
@@ -72,6 +64,13 @@ rest_target(${psd} base/protocols/http/partial-content.bro)
 rest_target(${psd} base/protocols/http/utils.bro)
 rest_target(${psd} base/protocols/irc/dcc-send.bro)
 rest_target(${psd} base/protocols/irc/main.bro)
+rest_target(${psd} base/protocols/mime/base.bro)
+rest_target(${psd} base/protocols/mime/file-extract.bro)
+rest_target(${psd} base/protocols/mime/file-hash.bro)
+rest_target(${psd} base/protocols/mime/file-ident.bro)
+rest_target(${psd} base/protocols/rpc/base.bro)
+rest_target(${psd} base/protocols/smtp/entities-excerpt.bro)
+rest_target(${psd} base/protocols/smtp/entities.bro)
 rest_target(${psd} base/protocols/smtp/main.bro)
 rest_target(${psd} base/protocols/ssh/main.bro)
 rest_target(${psd} base/protocols/ssl/consts.bro)
@@ -95,6 +94,9 @@ rest_target(${psd} policy/frameworks/control/controllee.bro)
 rest_target(${psd} policy/frameworks/control/controller.bro)
 rest_target(${psd} policy/frameworks/dpd/detect-protocols.bro)
 rest_target(${psd} policy/frameworks/dpd/packet-segment-logging.bro)
+rest_target(${psd} policy/frameworks/metrics/conn-example.bro)
+rest_target(${psd} policy/frameworks/metrics/http-example.bro)
+rest_target(${psd} policy/frameworks/metrics/ssl-example.bro)
 rest_target(${psd} policy/frameworks/software/version-changes.bro)
 rest_target(${psd} policy/frameworks/software/vulnerable.bro)
 rest_target(${psd} policy/integration/barnyard2/base.bro)
@@ -102,10 +104,10 @@ rest_target(${psd} policy/integration/barnyard2/event.bro)
 rest_target(${psd} policy/integration/barnyard2/types.bro)
 rest_target(${psd} policy/misc/analysis-groups.bro)
 rest_target(${psd} policy/misc/loaded-scripts.bro)
+rest_target(${psd} policy/misc/profiling.bro)
 rest_target(${psd} policy/misc/trim-trace-file.bro)
 rest_target(${psd} policy/protocols/conn/known-hosts.bro)
 rest_target(${psd} policy/protocols/conn/known-services.bro)
-rest_target(${psd} policy/protocols/conn/scan.bro)
 rest_target(${psd} policy/protocols/dns/auth-addl.bro)
 rest_target(${psd} policy/protocols/dns/detect-external-names.bro)
 rest_target(${psd} policy/protocols/ftp/detect.bro)
@@ -118,8 +120,12 @@ rest_target(${psd} policy/protocols/http/headers.bro)
 rest_target(${psd} policy/protocols/http/software.bro)
 rest_target(${psd} policy/protocols/http/var-extraction-cookies.bro)
 rest_target(${psd} policy/protocols/http/var-extraction-uri.bro)
+rest_target(${psd} policy/protocols/smtp/blocklists.bro)
 rest_target(${psd} policy/protocols/smtp/detect-suspicious-orig.bro)
 rest_target(${psd} policy/protocols/smtp/software.bro)
+rest_target(${psd} policy/protocols/ssh/detect-bruteforcing.bro)
+rest_target(${psd} policy/protocols/ssh/geo-data.bro)
+rest_target(${psd} policy/protocols/ssh/interesting-hostnames.bro)
 rest_target(${psd} policy/protocols/ssh/software.bro)
 rest_target(${psd} policy/protocols/ssl/known-certs.bro)
 rest_target(${psd} policy/protocols/ssl/validate-certs.bro)
@@ -127,4 +133,8 @@ rest_target(${psd} policy/tuning/defaults/packet-fragments.bro)
 rest_target(${psd} policy/tuning/defaults/remove-high-volume-notices.bro)
 rest_target(${psd} policy/tuning/defaults/warnings.bro)
 rest_target(${psd} policy/tuning/track-all-assets.bro)
+rest_target(${psd} site/local-manager.bro)
+rest_target(${psd} site/local-proxy.bro)
+rest_target(${psd} site/local-worker.bro)
 rest_target(${psd} site/local.bro)
+rest_target(${psd} test-all-policy.bro)

doc/scripts/README

@@ -44,7 +44,7 @@ of documentation targets.  This script should be run after adding new
 Bro script source files, and the changes commited to git.
 
 If a script shouldn't have documentation generated for it, there's also a
-blacklist variable that can be maintained in the ``genDocSourcesList.sh``
+blacklist manifest that can be maintained in the ``genDocSourcesList.sh``
 script.
 
 The blacklist can also be used if you want to define a certain grouping for

doc/scripts/genDocSourcesList.sh

@@ -11,8 +11,31 @@
 # Specific scripts can be blacklisted below when e.g. they currently aren't
 # parseable or they just aren't meant to be documented.
 
-blacklist="__load__.bro|test-all.bro|all.bro"
+blacklist ()
-blacklist_addl="hot.conn.bro"
+    {
+    if [[ "$blacklist" == "" ]]; then
+        blacklist="$1"
+    else
+        blacklist="$blacklist|$1"
+    fi
+    }
+
+# files passed into this function are meant to be temporary workarounds
+# because they're not finished or otherwise can't be loaded for some reason
+tmp_blacklist ()
+    {
+    echo "Warning: temporarily blacklisted files named '$1'" 1>&2
+    blacklist $1
+    }
+
+blacklist __load__.bro
+blacklist test-all.bro
+blacklist all.bro
+blacklist init-default.bro
+blacklist init-bare.bro
+
+tmp_blacklist hot.conn.bro
+tmp_blacklist scan.bro
 
 statictext="\
 # DO NOT EDIT
@@ -30,8 +53,8 @@ statictext="\
 set(psd \${PROJECT_SOURCE_DIR}/scripts)
 
 rest_target(\${CMAKE_CURRENT_SOURCE_DIR} example.bro internal)
-rest_target(\${psd} base/bro.init internal)
+rest_target(\${psd} base/init-default.bro internal)
-rest_target(\${psd} base/all.bro internal)
+rest_target(\${psd} base/init-bare.bro internal)
 "
 
 if [[ $# -ge 1 ]]; then
@@ -58,7 +81,7 @@ scriptfiles=`( cd ${sourcedir}/scripts && find . -name \*\.bro | sort )`
 for file in $scriptfiles
 do
     f=${file:2}
-    if [[ (! $f =~ $blacklist) && (! $f =~ $blacklist_addl)  ]]; then
+    if [[ ! $f =~ $blacklist ]]; then
         echo "rest_target(\${psd} $f)" >> $outfile
     fi
 done

doc/scripts/group_index_generator.py

@@ -1,9 +1,8 @@
 #! /usr/bin/env python
 
 # This script automatically generates a reST documents that lists
-# a collection of Bro policy scripts that are "grouped" together.
+# a collection of Bro scripts that are "grouped" together.
-# The summary text (##! comments) of the policy script is embedded
+# The summary text (##! comments) of the script is embedded in the list
-# in the list.
 #
 # 1st argument is the file containing list of groups
 # 2nd argument is the directory containing ${group}_files lists of
@@ -57,6 +56,6 @@ with open(group_list, 'r') as f_group_list:
                     f_group_file.write("\n");
 
             with open(group_file, 'a') as f_group_file:
-                f_group_file.write("\n:doc:`/policy/%s`\n" % doc_names[i])
+                f_group_file.write("\n:doc:`/scripts/%s`\n" % doc_names[i])
                 for line in summary_comments:
                     f_group_file.write("   " + line)

doc/scripts/source/index.rst

@@ -14,7 +14,7 @@ Contents:
    internal
    bifs
    packages
-   policy/index
+   scripts/index
 
 Indices and tables
 ==================

doc/scripts/source/internal.rst

@@ -1,5 +1,5 @@
 .. This is a stub doc to which the build process can append.
 
-Internal Policy Scripts
+Internal Scripts
-=======================
+================
 

doc/scripts/source/packages.rst

@@ -1,11 +1,15 @@
 .. This is a stub doc to which the build process can append.
 
-Policy Script Packages
+Bro Script Packages
-======================
+===================
 
-Bro has the following policy script packages (e.g. collections of related
+Bro has the following script packages (e.g. collections of related scripts in
-policy scripts).  If the package contains a ``__load__.bro`` script, it
+a common directory).  If the package directory contains a ``__load__.bro``
-supports being loaded in mass as a whole directory for convenience.
+script, it supports being loaded in mass as a whole directory for convenience.
+
+Packages/scripts in the ``base/`` directory are all loaded by default, while
+ones in ``policy/`` provide functionality and customization options that are
+more appropriate for users to decide whether they'd like to load it or not.
 
 .. toctree::
    :maxdepth: 1

doc/scripts/source/policy/index.rst

@@ -1,6 +0,0 @@
-Index of All Policy Script Documentation
-========================================
-
-.. toctree::
-   :maxdepth: 1
-

doc/scripts/source/scripts/index.rst

@@ -0,0 +1,6 @@
+Index of All Bro Script Documentation
+=====================================
+
+.. toctree::
+   :maxdepth: 1
+

scripts/CMakeLists.txt

@@ -1,6 +1,6 @@
 include(InstallPackageConfigFile)
 
-install(DIRECTORY ./ DESTINATION ${POLICYDIR} FILES_MATCHING
+install(DIRECTORY ./ DESTINATION ${BRO_SCRIPT_INSTALL_PATH} FILES_MATCHING
         PATTERN "site/local*" EXCLUDE
         PATTERN "*.bro"
         PATTERN "*.sig"
@@ -11,17 +11,17 @@ install(DIRECTORY ./ DESTINATION ${POLICYDIR} FILES_MATCHING
 # user modify-able.
 InstallPackageConfigFile(
     ${CMAKE_CURRENT_SOURCE_DIR}/site/local.bro
-    ${POLICYDIR}/site
+    ${BRO_SCRIPT_INSTALL_PATH}/site
     local.bro)
 InstallPackageConfigFile(
     ${CMAKE_CURRENT_SOURCE_DIR}/site/local-manager.bro
-    ${POLICYDIR}/site
+    ${BRO_SCRIPT_INSTALL_PATH}/site
     local-manager.bro)
 InstallPackageConfigFile(
     ${CMAKE_CURRENT_SOURCE_DIR}/site/local-proxy.bro
-    ${POLICYDIR}/site
+    ${BRO_SCRIPT_INSTALL_PATH}/site
     local-proxy.bro)
 InstallPackageConfigFile(
     ${CMAKE_CURRENT_SOURCE_DIR}/site/local-worker.bro
-    ${POLICYDIR}/site
+    ${BRO_SCRIPT_INSTALL_PATH}/site
     local-worker.bro)

scripts/base/frameworks/cluster/nodes/manager.bro

@@ -10,7 +10,7 @@
 
 @prefixes += cluster-manager
 
-## Load the script for local site configuration for the manager node.
+# Load the script for local site configuration for the manager node.
 @load site/local-manager
 
 ## Turn off remote logging since this is the manager and should only log here.

scripts/base/frameworks/cluster/nodes/proxy.bro

@@ -1,7 +1,7 @@
 
 @prefixes += cluster-proxy
 
-## Load the script for local site configuration for proxy nodes.
+# Load the script for local site configuration for proxy nodes.
 @load site/local-proxy
 
 ## The proxy only syncs state; does not forward events.

scripts/base/frameworks/cluster/nodes/worker.bro

@@ -1,7 +1,7 @@
 
 @prefixes += cluster-worker
 
-## Load the script for local site configuration for the worker nodes.
+# Load the script for local site configuration for the worker nodes.
 @load site/local-worker
 
 ## Don't do any local logging.

scripts/base/frameworks/cluster/setup-connections.bro

@@ -1,3 +1,5 @@
+@load ./main
+@load base/frameworks/communication/main
 
 module Cluster;
 

scripts/base/frameworks/notice/actions/email_admin.bro

@@ -1,4 +1,3 @@
-
 module Notice;
 
 export {

scripts/base/frameworks/notice/weird.bro

@@ -1,4 +1,3 @@
-
 module Weird;
 
 export {

scripts/base/protocols/mime/base.bro

@@ -1,7 +1,7 @@
 ##! The mime script does analysis of MIME encoded messages seen in certain
 ##! protocols (only SMTP and POP3 at the moment).
 
-@load utils/strings
+@load base/utils/strings
 
 module MIME;
 

scripts/base/protocols/mime/file-extract.bro

@@ -1,5 +1,5 @@
-@load protocols/mime/file-ident
+@load ./file-ident
-@load utils/files
+@load base/utils/files
 
 module MIME;
 

scripts/base/protocols/mime/file-hash.bro

@@ -1,4 +1,4 @@
-@load protocols/mime/file-ident
+@load ./file-ident
 
 module MIME;
 

scripts/base/protocols/mime/file-ident.bro

@@ -1,4 +1,4 @@
-@load protocols/mime/base
+@load ./base
 
 module MIME;
 

scripts/policy/frameworks/communication/listen-clear.bro

@@ -1,5 +1,7 @@
 ##! Listen for other Bro instances to make unencrypted connections.
 
+@load base/frameworks/communication
+
 module Communication;
 
 export {

scripts/policy/frameworks/communication/listen-ssl.bro

@@ -1,5 +1,7 @@
 ##! Listen for other Bro instances and encrypt the connection with SSL.
 
+@load base/frameworks/communication
+
 module Communication;
 
 export {

scripts/policy/frameworks/control/controller.bro

@@ -1,3 +1,4 @@
+@load base/frameworks/communication
 
 module Control;
 

scripts/policy/protocols/conn/scan.bro

@@ -1,4 +1,4 @@
-@load frameworks/notice
+@load base/frameworks/notice
 @load port-name
 
 module Scan;

src/BroDoc.cc

@@ -20,8 +20,8 @@ BroDoc::BroDoc(const std::string& rel, const std::string& abs)
 
 	if ( rel[0] == '/' || rel[0] == '.' )
 		{
-		// The Bro script must not be on a subpath of the policy/ dir of
+		// The Bro script isn't being loaded via BROPATH, so just use basename
-		// BROPATH, so just use the basename as the document title.
+		// as the document title.
 		doc_title = source_filename;
 		}
 	else
@@ -33,8 +33,14 @@ BroDoc::BroDoc(const std::string& rel, const std::string& abs)
 			doc_title = rel + "/" + source_filename;
 		}
 
+	downloadable_filename = source_filename;
+
+	size_t ext_pos = downloadable_filename.find(".bif.bro");
+	if ( std::string::npos != ext_pos )
+		downloadable_filename.erase(ext_pos + 4);
+
 	reST_filename = doc_title;
-	size_t ext_pos = reST_filename.find(".bro");
+	ext_pos = reST_filename.find(".bro");
 
 	if ( std::string::npos == ext_pos )
 		reST_filename += ".rst";
@@ -103,14 +109,14 @@ void BroDoc::AddImport(const std::string& s)
 			{
 			if ( subpath[0] == '/' || subpath[0] == '.' )
 				{
-				// it's not a subpath of policy/, so just add the name of it
+				// it's not a subpath of scripts/, so just add the name of it
 				// as it's given in the @load directive
 				imports.push_back(lname);
 				}
 			else
 				{
 				// combine the base file name of script in the @load directive
-				// with the subpath of BROPATH's policy/ directory
+				// with the subpath of BROPATH's scripts/ directory
 				string fname(subpath);
 				char* othertmp = copy_string(lname.c_str());
 				fname.append("/").append(basename(othertmp));
@@ -167,7 +173,7 @@ void BroDoc::WriteDocFile() const
 	WriteSectionHeading(doc_title.c_str(), '=');
 
 	WriteToDoc("\n:download:`Original Source File <%s>`\n\n",
-		source_filename.c_str());
+		downloadable_filename.c_str());
 
 	WriteSectionHeading("Overview", '-');
 	WriteStringList("%s\n", "%s\n\n", summary);
@@ -185,7 +191,7 @@ void BroDoc::WriteDocFile() const
 			size_t pos = pretty.find("/index");
 			if ( pos != std::string::npos && pos + 6 == pretty.size() )
 				pretty = pretty.substr(0, pos);
-			WriteToDoc(":doc:`%s </policy/%s>`", pretty.c_str(), it->c_str());
+			WriteToDoc(":doc:`%s </scripts/%s>`", pretty.c_str(), it->c_str());
 			}
 		WriteToDoc("\n");
 		}

src/BroDoc.h

@@ -24,10 +24,9 @@ public:
 	 * If the filename doesn't end in ".bro", then ".rst" is just appended.
 	 * Any '/' characters in the reST file name that result from choice of
 	 * the 'rel' parameter are replaced with '^'.
-	 * @param subpath A string representing a subpath of BROPATH's policy/
+	 * @param rel A string representing a subpath of the root Bro script
-	 *        directory in which the source file is located.  It can
+	 *        source/install directory in which the source file is located.
-	 *        also be full path to the file or a full path that's in BROPATH,
+	 *        It can also be an absolute path, but then the parameter is
-	 *        but in either of those cases, the parameter is essentially
 	 *        ignored and the document title is just derived from file name
 	 * @param abs The absolute path to the Bro script for which to generate
 	 *        documentation.
@@ -211,6 +210,7 @@ protected:
 	FILE* reST_file;
 	std::string reST_filename;
 	std::string source_filename;	// points to the basename of source file
+	std::string downloadable_filename; // file that will be linked for download
 	std::string doc_title;
 	std::string packet_filter;
 

src/CMakeLists.txt

@@ -408,7 +408,8 @@ set(bro_SRCS
 
 collect_headers(bro_HEADERS ${bro_SRCS})
 
-add_definitions(-DPOLICYDEST="${POLICYDIR}")
+add_definitions(-DBRO_SCRIPT_INSTALL_PATH="${BRO_SCRIPT_INSTALL_PATH}")
+add_definitions(-DBRO_SCRIPT_SOURCE_PATH="${BRO_SCRIPT_SOURCE_PATH}")
 
 add_executable(bro ${bro_SRCS} ${bro_HEADERS})
 
@@ -423,7 +424,7 @@ set(brolibs
 target_link_libraries(bro ${brolibs})
 
 install(TARGETS bro DESTINATION bin)
-install(FILES ${INSTALL_BIF_OUTPUTS} DESTINATION ${POLICYDIR})
+install(FILES ${INSTALL_BIF_OUTPUTS} DESTINATION ${BRO_SCRIPT_INSTALL_PATH})
 
 set(BRO_EXE bro
     CACHE STRING "Bro executable binary" FORCE)

src/util.cc

@@ -757,9 +757,9 @@ const char* bro_path()
 	const char* path = getenv("BROPATH");
 	if ( ! path )
 		path = ".:"
-			POLICYDEST ":"
+			BRO_SCRIPT_INSTALL_PATH ":"
-			POLICYDEST "/policy" ":"
+			BRO_SCRIPT_INSTALL_PATH "/policy" ":"
-			POLICYDEST "/site";
+			BRO_SCRIPT_INSTALL_PATH "/site";
 
 	return path;
 	}
@@ -891,60 +891,36 @@ const char* normalize_path(const char* path)
 	return copy_string(new_path.c_str());
 	}
 
-// Returns the subpath of BROPATH's policy/ directory in which the loaded
+// Returns the subpath of the root Bro script install/source directory in
-// file in located.  If it's not under a subpath of policy/ then the full
+// which the loaded file is located.  If it's not under a subpath of that
-// path is returned, else the subpath of policy/ concatentated with any
+// directory (e.g. cwd or custom path) then the full path is returned.
-// directory prefix of the file is returned.
+void get_script_subpath(const std::string& full_filename, const char** subpath)
-void get_policy_subpath(const char* dir, const char* file, const char** subpath)
 	{
-	// first figure out if this is a subpath of policy/
+	size_t p;
-	const char* ploc = strstr(dir, "policy");
+	std::string my_subpath(full_filename);
-	if ( ploc )
-		if ( ploc[6] == '\0' )
-			*subpath = copy_string(ploc + 6);
-		else if ( ploc[6] == '/' )
-			*subpath = copy_string(ploc + 7);
-		else
-			*subpath = copy_string(dir);
-	else
-		*subpath = copy_string(dir);
 
-	// and now add any directory parts of the filename
+	// get the parent directory of file (if not already a directory)
-	char full_filename_buf[1024];
+	if ( ! is_dir(full_filename.c_str()) )
-	safe_snprintf(full_filename_buf, sizeof(full_filename_buf),
-			"%s/%s", dir, file);
-	char* tmp = copy_string(file);
-	const char* fdir = 0;
-
-	if ( is_dir(full_filename_buf) )
-		fdir = file;
-
-	if ( ! fdir )
-		fdir = dirname(tmp);
-
-	if ( ! streq(fdir, ".") )
 		{
-		size_t full_subpath_len = strlen(*subpath) + strlen(fdir) + 1;
+		char* tmp = copy_string(full_filename.c_str());
-		bool needslash = false;
+		my_subpath = dirname(tmp);
-		if ( strlen(*subpath) != 0 && (*subpath)[strlen(*subpath) - 1] != '/' )
-			{
-			++full_subpath_len;
-			needslash = true;
-			}
-
-		char* full_subpath = new char[full_subpath_len];
-		strcpy(full_subpath, *subpath);
-		if ( needslash )
-			strcat(full_subpath, "/");
-		strcat(full_subpath, fdir);
-		delete [] *subpath;
-		*subpath = full_subpath;
-		}
-
-	const char* normalized_subpath = normalize_path(*subpath);
 		delete [] tmp;
-	delete [] *subpath;
+		}
-	*subpath = normalized_subpath;
+
+	// first check if this is some subpath of the installed scripts root path,
+	// if not check if it's a subpath of the script source root path,
+	// if neither, will just use the given directory
+	if ( (p=my_subpath.find(BRO_SCRIPT_INSTALL_PATH)) != std::string::npos )
+		my_subpath.erase(0, strlen(BRO_SCRIPT_INSTALL_PATH));
+	else if ( (p=my_subpath.find(BRO_SCRIPT_SOURCE_PATH)) != std::string::npos )
+		my_subpath.erase(0, strlen(BRO_SCRIPT_SOURCE_PATH));
+
+	// if root path found, remove path separators until next path component
+	if ( p != std::string::npos )
+		while ( my_subpath.size() && my_subpath[0] == '/' )
+			my_subpath.erase(0, 1);
+
+	*subpath = normalize_path(my_subpath.c_str());
 	}
 
 extern string current_scanned_file_path;
@@ -1001,7 +977,7 @@ FILE* search_for_file(const char* filename, const char* ext,
 		     ! is_dir(full_filename_buf) )
 			{
 			if ( bropath_subpath )
-				get_policy_subpath(dir_beginning, filename, bropath_subpath);
+				get_script_subpath(full_filename_buf, bropath_subpath);
 			return open_file(full_filename_buf, full_filename, load_pkgs);
 			}
 
@@ -1010,7 +986,7 @@ FILE* search_for_file(const char* filename, const char* ext,
 		if ( access(full_filename_buf, R_OK) == 0 )
 			{
 			if ( bropath_subpath )
-				get_policy_subpath(dir_beginning, filename, bropath_subpath);
+				get_script_subpath(full_filename_buf, bropath_subpath);
 			return open_file(full_filename_buf, full_filename, load_pkgs);
 			}
 

src/util.h

@@ -179,7 +179,7 @@ extern const char* bro_path();
 extern const char* bro_prefixes();
 std::string dot_canon(std::string path, std::string file, std::string prefix = "");
 const char* normalize_path(const char* path);
-void get_policy_subpath(const char* dir, const char* file, const char** subpath);
+void get_script_subpath(const std::string& full_filename, const char** subpath);
 extern FILE* search_for_file(const char* filename, const char* ext,
 	const char** full_filename, bool load_pkgs, const char** bropath_subpath);
 

testing/btest/Baseline/core.load-prefixes/output

@@ -1,4 +1,4 @@
-loaded lcl2.site.bro
+loaded lcl2.base.utils.site.bro
-loaded lcl.site.bro
+loaded lcl.base.utils.site.bro
-loaded lcl2.protocols.http.bro
+loaded lcl2.base.protocols.http.bro
-loaded lcl.protocols.http.bro
+loaded lcl.base.protocols.http.bro

testing/btest/Baseline/doc.autogen-reST-example/example.rst

@@ -29,7 +29,7 @@ each of "columns", "event", "filter" depending on exactly what it's doing.
 
 :Author: Jon Siwek <jsiwek@ncsa.illinois.edu>
 
-:Imports: :doc:`frameworks/notice </policy/frameworks/notice/index>`
+:Imports: :doc:`policy/frameworks/software/vulnerable </scripts/policy/frameworks/software/vulnerable>`
 
 Summary
 ~~~~~~~

testing/btest/Baseline/policy.misc.default-loaded-scripts/canonified_loaded_scripts.log

@@ -27,6 +27,8 @@
 2	scripts/base/frameworks/notice/./actions/drop.bro
 2	scripts/base/frameworks/notice/./actions/email_admin.bro
 2	scripts/base/frameworks/notice/./actions/page.bro
+2	scripts/base/frameworks/notice/./actions/add-geodata.bro
+2	scripts/base/frameworks/notice/./extend-email/hostnames.bro
 1	scripts/base/frameworks/dpd/__load__.bro
 2	scripts/base/frameworks/dpd/./main.bro
 1	scripts/base/frameworks/signatures/__load__.bro
@@ -41,6 +43,7 @@
 1	scripts/base/frameworks/metrics/__load__.bro
 2	scripts/base/frameworks/metrics/./main.bro
 1	scripts/base/frameworks/communication/__load__.bro
+2	scripts/base/frameworks/communication/./main.bro
 1	scripts/base/frameworks/control/__load__.bro
 2	scripts/base/frameworks/control/./main.bro
 1	scripts/base/frameworks/cluster/__load__.bro
@@ -69,6 +72,8 @@
 2	scripts/base/protocols/irc/./dcc-send.bro
 1	scripts/base/protocols/smtp/__load__.bro
 2	scripts/base/protocols/smtp/./main.bro
+2	scripts/base/protocols/smtp/./entities.bro
+2	scripts/base/protocols/smtp/./entities-excerpt.bro
 1	scripts/base/protocols/ssh/__load__.bro
 2	scripts/base/protocols/ssh/./main.bro
 1	scripts/base/protocols/ssl/__load__.bro

testing/btest/core/load-prefixes.bro

@@ -9,11 +9,11 @@
 @TEST-END-FILE
 
 @TEST-START-FILE lcl.base.utils.site.bro
-print "loaded lcl.base.site.bro";
+print "loaded lcl.base.utils.site.bro";
 @TEST-END-FILE
 
 @TEST-START-FILE lcl2.base.utils.site.bro
-print "loaded lcl2.base.site.bro";
+print "loaded lcl2.base.utils.site.bro";
 @TEST-END-FILE
 
 @TEST-START-FILE lcl.base.protocols.http.bro

testing/btest/doc/blacklist-reminder.test

@@ -0,0 +1,8 @@
+# This test will fail if there are Bro scripts that have been temporarily
+# blacklisted from the documentation generation process for some reason
+# (e.g. they're a work-in-progress or otherwise fail to parse).  It's meant
+# to serve as a reminder that some future action may be needed to generate
+# documentation for the blacklisted scripts.
+#
+# @TEST-EXEC: $DIST/doc/scripts/genDocSourcesList.sh
+# @TEST-EXEC: btest-diff .stderr