Merge remote-tracking branch 'origin/topic/jsiwek/autodoc-fixes'

* origin/topic/jsiwek/autodoc-fixes: Update doc sources and touch up a few script comments. Fixes for script auto-documentation. Conflicts: scripts/base/frameworks/logging/main.bro
2025-10-02 06:38:20 +00:00 · 2011-08-13 10:31:38 -07:00 · 2011-08-13 10:31:38 -07:00 · 1ca9def9e6
commit 1ca9def9e6
parent 1aff02ea96 a3147033e2
43 changed files with 260 additions and 155 deletions
--- a/62
+++ b/62
@ -1,4 +1,66 @@

+1.6-dev-1095 | 2011-08-13 11:59:07 -0700
+
+  * A larger number of script documentation updates. Closes #543. (Jon
+    Siwek)
+
+  * Workaround for FreeBSD CMake port missing debug flags. (Jon Siwek)
+    
+  * piped_exec() can now deal with null bytes. (Seth Hall)
+    
+  * Fix vector initialization for lists of records with optional
+    types. Closes #485. (Jon Siwek)
+
+  * Fix redef'ing records with &default empty set fields. Closes #460.
+    (Jon Siwek)
+
+  * Fix ConnSize_Analyzer when used in conjunction with the connection
+    compressor. (Gregor Maier)
+
+  * Fix reporter using part of the actual message as a format string.
+    (Jon Siwek)
+
+  * Fixing reporter's location tracking. Closes #492. (Robin Sommer)
+    
+  * Turning DNS errors into warnings. Closes #255. (Robin Sommer)
+    
+  * Logging's path_func now receives the log record as argument.
+    Closes #555. (Robin Sommer)
+
+  * Functions can now be logged; their full body gets recorded. 
+    Closes #506. (Robin Sommer)
+
+  * Bugfix for hostname notice email extension. (Seth Hall)
+    
+  * Updates for notice framework. (Seth Hall)
+    
+    - New ACTION_ADD_GEODATA to add geodata to notices in an extension
+      field named remote_location.
+
+    - Loading extend-email/hostnames by default now that it only does
+      anything when the ACTION_EMAIL action is applied (finally).
+
+  * Updates to local.bro (Seth Hall)
+
+  * Added the profiling script. (Seth Hall)
+    
+  * Updates for SSH scripts. (Seth Hall)
+
+  * ConnSize analyzer is turned on by default now. (Seth Hall)
+
+  * Updates for the build system and site local scripts for cluster.
+    (Seth Hall)
+
+  * HTTP now uses the extract_filename_from_content_disposition function. (Seth Hall)
+
+  * Major SMTP script refactor. Closes #509. (Jon Siwek and Seth Hall)
+    
+  * New variable Site::local_nets_table in utils/site for mapping
+    address to defined local subnet.
+
+  * Metrics framework updates, more to come. (Seth Hall)
+    
+
 1.6-dev-1061 | 2011-08-08 18:25:27 -0700

  * A set of new/changed tests regarding the new policy script
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@ -35,14 +35,16 @@ endif ()
 ## Project/Build Configuration

 set(BRO_ROOT_DIR ${CMAKE_INSTALL_PREFIX})
-if (NOT POLICYDIR)
-    # set the default policy installation path (user did not specify one)
-    set(POLICYDIR ${BRO_ROOT_DIR}/share/bro)
+if (NOT BRO_SCRIPT_INSTALL_PATH)
+    # set the default Bro script installation path (user did not specify one)
+    set(BRO_SCRIPT_INSTALL_PATH ${BRO_ROOT_DIR}/share/bro)
 endif ()
+set(BRO_SCRIPT_SOURCE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/scripts)

-# sanitize the policy install directory into an absolute path
+# sanitize the Bro script install directory into an absolute path
 # (CMake is confused by ~ as a representation of home directory)
-get_filename_component(POLICYDIR ${POLICYDIR} ABSOLUTE)
+get_filename_component(BRO_SCRIPT_INSTALL_PATH ${BRO_SCRIPT_INSTALL_PATH}
+    ABSOLUTE)

 configure_file(bro-path-dev.in ${CMAKE_CURRENT_BINARY_DIR}/bro-path-dev)
 file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/bro-path-dev.sh
@ -213,7 +215,7 @@ message(
    "\n====================|  Bro Build Summary  |====================="
    "\n"
    "\nInstall prefix:    ${CMAKE_INSTALL_PREFIX}"
-    "\nPolicy dir:        ${POLICYDIR}"
+    "\nBro Script Path:   ${BRO_SCRIPT_INSTALL_PATH}"
    "\nDebug mode:        ${ENABLE_DEBUG}"
    "\n"
    "\nCC:                ${CMAKE_C_COMPILER}"
--- a/2
+++ b/2
@ -1 +1 @@
-1.6-dev-1061
+1.6-dev-1095
--- a/bro-path-dev.in
+++ b/bro-path-dev.in
@ -1,7 +1,7 @@
 #!/bin/sh
-# After configured by CMake, this file prints the absolute path to policy
-# files that come with the source distributions of Bro as well as policy 
-# files that are generated by the BIF compiler at compile time.
+# After configured by CMake, this file prints the absolute path to Bro scripts
+# that come with the source distributions of Bro as well as scripts that are
+# generated by the BIF compiler at compile time.
 #
 # The intended use of this script is to make it easier to run Bro from
 # the build directory, avoiding the need to install it.  This could be
@ -10,10 +10,10 @@
 #     BROPATH=`./bro-path-dev` ./src/bro
 #

-broPolicies=${PROJECT_SOURCE_DIR}/scripts:${PROJECT_SOURCE_DIR}/scripts/policy:${PROJECT_SOURCE_DIR}/scripts/site
+broPolicies=${BRO_SCRIPT_SOURCE_PATH}:${BRO_SCRIPT_SOURCE_PATH}/policy:${BRO_SCRIPT_SOURCE_PATH}/site

 broGenPolicies=${CMAKE_BINARY_DIR}/src

-installedPolicies=${POLICYDIR}:${POLICYDIR}/site
+installedPolicies=${BRO_SCRIPT_INSTALL_PATH}:${BRO_SCRIPT_INSTALL_PATH}/site

 echo .:$broPolicies:$broGenPolicies
--- a/14
+++ b/14
@ -22,7 +22,7 @@ Usage: $0 [OPTION]... [VAR=VALUE]...

  Installation Directories:
    --prefix=PREFIX        installation directory [/usr/local/bro]
-    --policydir=PATH       policy file installation directory
+    --scriptdir=PATH       root installation directory for Bro scripts
                           [PREFIX/share/bro]

  Optional Features:
@ -85,7 +85,7 @@ CMakeCacheEntries=""
 append_cache_entry CMAKE_INSTALL_PREFIX PATH   /usr/local/bro
 append_cache_entry BRO_ROOT_DIR         PATH   /usr/local/bro
 append_cache_entry PY_MOD_INSTALL_DIR   PATH   /usr/local/bro/lib/broctl
-append_cache_entry POLICYDIR            STRING /usr/local/bro/share/bro
+append_cache_entry BRO_SCRIPT_INSTALL_PATH STRING /usr/local/bro/share/bro
 append_cache_entry ENABLE_DEBUG         BOOL   false
 append_cache_entry BROv6                BOOL   false
 append_cache_entry ENABLE_PERFTOOLS     BOOL   false
@ -118,13 +118,13 @@ while [ $# -ne 0 ]; do
            append_cache_entry CMAKE_INSTALL_PREFIX PATH   $optarg
            append_cache_entry BRO_ROOT_DIR         PATH   $optarg
            append_cache_entry PY_MOD_INSTALL_DIR   PATH   $optarg/lib/broctl
-            if [ "$user_set_policydir" != "true" ]; then
-                append_cache_entry POLICYDIR        STRING $optarg/share/bro
+            if [ "$user_set_scriptdir" != "true" ]; then
+                append_cache_entry BRO_SCRIPT_INSTALL_PATH STRING $optarg/share/bro
            fi
            ;;
-        --policydir=*)
-            append_cache_entry POLICYDIR            STRING $optarg
-            user_set_policydir="true"
+        --scriptdir=*)
+            append_cache_entry BRO_SCRIPT_INSTALL_PATH STRING $optarg
+            user_set_scriptdir="true"
            ;;
        --enable-debug)
            append_cache_entry ENABLE_DEBUG         BOOL   true
--- a/doc/scripts/CMakeLists.txt
+++ b/doc/scripts/CMakeLists.txt
@ -57,10 +57,12 @@ macro(REST_TARGET srcDir broInput)
    get_filename_component(relDstDir ${broInput} PATH)

    set(sumTextSrc ${absSrcPath})
+    set(ogSourceFile ${absSrcPath})
    if (${extension} STREQUAL ".bif.bro")
+        set(ogSourceFile ${BIF_SRC_DIR}/${basename})
        # the summary text is taken at configure time, but .bif.bro files
        # may not have been generated yet, so read .bif file instead
-        set(sumTextSrc ${BIF_SRC_DIR}/${basename})
+        set(sumTextSrc ${ogSourceFile})
    endif ()

    if (NOT relDstDir)
@ -124,9 +126,9 @@ macro(REST_TARGET srcDir broInput)
        ARGS -E make_directory ${dstDir}
        COMMAND "${CMAKE_COMMAND}"
        ARGS -E copy ${restFile} ${restOutput}
-        # copy the bro policy script, too
+        # copy the bro or bif script, too
        COMMAND "${CMAKE_COMMAND}"
-        ARGS -E copy ${absSrcPath} ${dstDir}
+        ARGS -E copy ${ogSourceFile} ${dstDir}
        # clean up the build directory
        COMMAND rm
        ARGS -rf .state *.log *.rst
@ -151,6 +153,8 @@ set(MASTER_PKG_INDEX_TEXT "")
 foreach (pkg ${MASTER_PKG_LIST})
    # strip of the trailing /index for the link name
    get_filename_component(lnktxt ${pkg} PATH)
+    # pretty-up the link name by removing common scripts/ prefix
+    string(REPLACE "scripts/" "" lnktxt "${lnktxt}")
    set(MASTER_PKG_INDEX_TEXT "${MASTER_PKG_INDEX_TEXT}\n   ${lnktxt} <${pkg}>")
 endforeach ()
 file(WRITE ${MASTER_PACKAGE_INDEX} "${MASTER_PKG_INDEX_TEXT}")
--- a/doc/scripts/DocSourcesList.cmake
+++ b/doc/scripts/DocSourcesList.cmake
@ -13,22 +13,14 @@
 set(psd ${PROJECT_SOURCE_DIR}/scripts)

 rest_target(${CMAKE_CURRENT_SOURCE_DIR} example.bro internal)
-rest_target(${psd} base/bro.init internal)
-rest_target(${psd} base/all.bro internal)
+rest_target(${psd} base/init-default.bro internal)
+rest_target(${psd} base/init-bare.bro internal)

 rest_target(${CMAKE_BINARY_DIR}/src bro.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/src common-rw.bif.bro)
 rest_target(${CMAKE_BINARY_DIR}/src const.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/src dns-rw.bif.bro)
 rest_target(${CMAKE_BINARY_DIR}/src event.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/src finger-rw.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/src ftp-rw.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/src http-rw.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/src ident-rw.bif.bro)
 rest_target(${CMAKE_BINARY_DIR}/src logging.bif.bro)
 rest_target(${CMAKE_BINARY_DIR}/src reporter.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/src smb-rw.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/src smtp-rw.bif.bro)
 rest_target(${CMAKE_BINARY_DIR}/src strings.bif.bro)
 rest_target(${CMAKE_BINARY_DIR}/src types.bif.bro)
 rest_target(${psd} base/frameworks/cluster/main.bro)
@ -42,9 +34,8 @@ rest_target(${psd} base/frameworks/dpd/main.bro)
 rest_target(${psd} base/frameworks/intel/main.bro)
 rest_target(${psd} base/frameworks/logging/main.bro)
 rest_target(${psd} base/frameworks/logging/writers/ascii.bro)
-rest_target(${psd} base/frameworks/metrics/conn-example.bro)
-rest_target(${psd} base/frameworks/metrics/http-example.bro)
 rest_target(${psd} base/frameworks/metrics/main.bro)
+rest_target(${psd} base/frameworks/notice/actions/add-geodata.bro)
 rest_target(${psd} base/frameworks/notice/actions/drop.bro)
 rest_target(${psd} base/frameworks/notice/actions/email_admin.bro)
 rest_target(${psd} base/frameworks/notice/actions/page.bro)
@ -56,6 +47,7 @@ rest_target(${psd} base/frameworks/packet-filter/netstats.bro)
 rest_target(${psd} base/frameworks/reporter/main.bro)
 rest_target(${psd} base/frameworks/signatures/main.bro)
 rest_target(${psd} base/frameworks/software/main.bro)
+rest_target(${psd} base/frameworks/time-machine/notice.bro)
 rest_target(${psd} base/protocols/conn/contents.bro)
 rest_target(${psd} base/protocols/conn/inactivity.bro)
 rest_target(${psd} base/protocols/conn/main.bro)
@ -72,6 +64,13 @@ rest_target(${psd} base/protocols/http/partial-content.bro)
 rest_target(${psd} base/protocols/http/utils.bro)
 rest_target(${psd} base/protocols/irc/dcc-send.bro)
 rest_target(${psd} base/protocols/irc/main.bro)
+rest_target(${psd} base/protocols/mime/base.bro)
+rest_target(${psd} base/protocols/mime/file-extract.bro)
+rest_target(${psd} base/protocols/mime/file-hash.bro)
+rest_target(${psd} base/protocols/mime/file-ident.bro)
+rest_target(${psd} base/protocols/rpc/base.bro)
+rest_target(${psd} base/protocols/smtp/entities-excerpt.bro)
+rest_target(${psd} base/protocols/smtp/entities.bro)
 rest_target(${psd} base/protocols/smtp/main.bro)
 rest_target(${psd} base/protocols/ssh/main.bro)
 rest_target(${psd} base/protocols/ssl/consts.bro)
@ -95,6 +94,9 @@ rest_target(${psd} policy/frameworks/control/controllee.bro)
 rest_target(${psd} policy/frameworks/control/controller.bro)
 rest_target(${psd} policy/frameworks/dpd/detect-protocols.bro)
 rest_target(${psd} policy/frameworks/dpd/packet-segment-logging.bro)
+rest_target(${psd} policy/frameworks/metrics/conn-example.bro)
+rest_target(${psd} policy/frameworks/metrics/http-example.bro)
+rest_target(${psd} policy/frameworks/metrics/ssl-example.bro)
 rest_target(${psd} policy/frameworks/software/version-changes.bro)
 rest_target(${psd} policy/frameworks/software/vulnerable.bro)
 rest_target(${psd} policy/integration/barnyard2/base.bro)
@ -102,10 +104,10 @@ rest_target(${psd} policy/integration/barnyard2/event.bro)
 rest_target(${psd} policy/integration/barnyard2/types.bro)
 rest_target(${psd} policy/misc/analysis-groups.bro)
 rest_target(${psd} policy/misc/loaded-scripts.bro)
+rest_target(${psd} policy/misc/profiling.bro)
 rest_target(${psd} policy/misc/trim-trace-file.bro)
 rest_target(${psd} policy/protocols/conn/known-hosts.bro)
 rest_target(${psd} policy/protocols/conn/known-services.bro)
-rest_target(${psd} policy/protocols/conn/scan.bro)
 rest_target(${psd} policy/protocols/dns/auth-addl.bro)
 rest_target(${psd} policy/protocols/dns/detect-external-names.bro)
 rest_target(${psd} policy/protocols/ftp/detect.bro)
@ -118,8 +120,12 @@ rest_target(${psd} policy/protocols/http/headers.bro)
 rest_target(${psd} policy/protocols/http/software.bro)
 rest_target(${psd} policy/protocols/http/var-extraction-cookies.bro)
 rest_target(${psd} policy/protocols/http/var-extraction-uri.bro)
+rest_target(${psd} policy/protocols/smtp/blocklists.bro)
 rest_target(${psd} policy/protocols/smtp/detect-suspicious-orig.bro)
 rest_target(${psd} policy/protocols/smtp/software.bro)
+rest_target(${psd} policy/protocols/ssh/detect-bruteforcing.bro)
+rest_target(${psd} policy/protocols/ssh/geo-data.bro)
+rest_target(${psd} policy/protocols/ssh/interesting-hostnames.bro)
 rest_target(${psd} policy/protocols/ssh/software.bro)
 rest_target(${psd} policy/protocols/ssl/known-certs.bro)
 rest_target(${psd} policy/protocols/ssl/validate-certs.bro)
@ -127,4 +133,8 @@ rest_target(${psd} policy/tuning/defaults/packet-fragments.bro)
 rest_target(${psd} policy/tuning/defaults/remove-high-volume-notices.bro)
 rest_target(${psd} policy/tuning/defaults/warnings.bro)
 rest_target(${psd} policy/tuning/track-all-assets.bro)
+rest_target(${psd} site/local-manager.bro)
+rest_target(${psd} site/local-proxy.bro)
+rest_target(${psd} site/local-worker.bro)
 rest_target(${psd} site/local.bro)
+rest_target(${psd} test-all-policy.bro)
--- a/doc/scripts/README
+++ b/doc/scripts/README
@ -44,7 +44,7 @@ of documentation targets.  This script should be run after adding new
 Bro script source files, and the changes commited to git.

 If a script shouldn't have documentation generated for it, there's also a
-blacklist variable that can be maintained in the ``genDocSourcesList.sh``
+blacklist manifest that can be maintained in the ``genDocSourcesList.sh``
 script.

 The blacklist can also be used if you want to define a certain grouping for
--- a/doc/scripts/genDocSourcesList.sh
+++ b/doc/scripts/genDocSourcesList.sh
@ -11,8 +11,31 @@
 # Specific scripts can be blacklisted below when e.g. they currently aren't
 # parseable or they just aren't meant to be documented.

-blacklist="__load__.bro|test-all.bro|all.bro"
-blacklist_addl="hot.conn.bro"
+blacklist ()
+    {
+    if [[ "$blacklist" == "" ]]; then
+        blacklist="$1"
+    else
+        blacklist="$blacklist|$1"
+    fi
+    }
+
+# files passed into this function are meant to be temporary workarounds
+# because they're not finished or otherwise can't be loaded for some reason
+tmp_blacklist ()
+    {
+    echo "Warning: temporarily blacklisted files named '$1'" 1>&2
+    blacklist $1
+    }
+
+blacklist __load__.bro
+blacklist test-all.bro
+blacklist all.bro
+blacklist init-default.bro
+blacklist init-bare.bro
+
+tmp_blacklist hot.conn.bro
+tmp_blacklist scan.bro

 statictext="\
 # DO NOT EDIT
@ -30,8 +53,8 @@ statictext="\
 set(psd \${PROJECT_SOURCE_DIR}/scripts)

 rest_target(\${CMAKE_CURRENT_SOURCE_DIR} example.bro internal)
-rest_target(\${psd} base/bro.init internal)
-rest_target(\${psd} base/all.bro internal)
+rest_target(\${psd} base/init-default.bro internal)
+rest_target(\${psd} base/init-bare.bro internal)
 "

 if [[ $# -ge 1 ]]; then
@ -58,7 +81,7 @@ scriptfiles=`( cd ${sourcedir}/scripts && find . -name \*\.bro | sort )`
 for file in $scriptfiles
 do
    f=${file:2}
-    if [[ (! $f =~ $blacklist) && (! $f =~ $blacklist_addl)  ]]; then
+    if [[ ! $f =~ $blacklist ]]; then
        echo "rest_target(\${psd} $f)" >> $outfile
    fi
 done
--- a/doc/scripts/group_index_generator.py
+++ b/doc/scripts/group_index_generator.py
@ -1,9 +1,8 @@
 #! /usr/bin/env python

 # This script automatically generates a reST documents that lists
-# a collection of Bro policy scripts that are "grouped" together.
-# The summary text (##! comments) of the policy script is embedded
-# in the list.
+# a collection of Bro scripts that are "grouped" together.
+# The summary text (##! comments) of the script is embedded in the list
 #
 # 1st argument is the file containing list of groups
 # 2nd argument is the directory containing ${group}_files lists of
@ -57,6 +56,6 @@ with open(group_list, 'r') as f_group_list:
                    f_group_file.write("\n");

            with open(group_file, 'a') as f_group_file:
-                f_group_file.write("\n:doc:`/policy/%s`\n" % doc_names[i])
+                f_group_file.write("\n:doc:`/scripts/%s`\n" % doc_names[i])
                for line in summary_comments:
                    f_group_file.write("   " + line)
--- a/doc/scripts/source/index.rst
+++ b/doc/scripts/source/index.rst
@ -14,7 +14,7 @@ Contents:
   internal
   bifs
   packages
-   policy/index
+   scripts/index

 Indices and tables
 ==================
--- a/doc/scripts/source/internal.rst
+++ b/doc/scripts/source/internal.rst
@ -1,5 +1,5 @@
 .. This is a stub doc to which the build process can append.

-Internal Policy Scripts
-=======================
+Internal Scripts
+================

--- a/doc/scripts/source/packages.rst
+++ b/doc/scripts/source/packages.rst
@ -1,11 +1,15 @@
 .. This is a stub doc to which the build process can append.

-Policy Script Packages
-======================
+Bro Script Packages
+===================

-Bro has the following policy script packages (e.g. collections of related
-policy scripts).  If the package contains a ``__load__.bro`` script, it
-supports being loaded in mass as a whole directory for convenience.
+Bro has the following script packages (e.g. collections of related scripts in
+a common directory).  If the package directory contains a ``__load__.bro``
+script, it supports being loaded in mass as a whole directory for convenience.
+
+Packages/scripts in the ``base/`` directory are all loaded by default, while
+ones in ``policy/`` provide functionality and customization options that are
+more appropriate for users to decide whether they'd like to load it or not.

 .. toctree::
   :maxdepth: 1
--- a/doc/scripts/source/policy/index.rst
+++ b/doc/scripts/source/policy/index.rst
@ -1,6 +0,0 @@
-Index of All Policy Script Documentation
-========================================
-
-.. toctree::
-   :maxdepth: 1
-
--- a/doc/scripts/source/scripts/index.rst
+++ b/doc/scripts/source/scripts/index.rst
@ -0,0 +1,6 @@
+Index of All Bro Script Documentation
+=====================================
+
+.. toctree::
+   :maxdepth: 1
+
--- a/scripts/CMakeLists.txt
+++ b/scripts/CMakeLists.txt
@ -1,6 +1,6 @@
 include(InstallPackageConfigFile)

-install(DIRECTORY ./ DESTINATION ${POLICYDIR} FILES_MATCHING
+install(DIRECTORY ./ DESTINATION ${BRO_SCRIPT_INSTALL_PATH} FILES_MATCHING
        PATTERN "site/local*" EXCLUDE
        PATTERN "*.bro"
        PATTERN "*.sig"
@ -11,17 +11,17 @@ install(DIRECTORY ./ DESTINATION ${POLICYDIR} FILES_MATCHING
 # user modify-able.
 InstallPackageConfigFile(
    ${CMAKE_CURRENT_SOURCE_DIR}/site/local.bro
-    ${POLICYDIR}/site
+    ${BRO_SCRIPT_INSTALL_PATH}/site
    local.bro)
 InstallPackageConfigFile(
    ${CMAKE_CURRENT_SOURCE_DIR}/site/local-manager.bro
-    ${POLICYDIR}/site
+    ${BRO_SCRIPT_INSTALL_PATH}/site
    local-manager.bro)
 InstallPackageConfigFile(
    ${CMAKE_CURRENT_SOURCE_DIR}/site/local-proxy.bro
-    ${POLICYDIR}/site
+    ${BRO_SCRIPT_INSTALL_PATH}/site
    local-proxy.bro)
 InstallPackageConfigFile(
    ${CMAKE_CURRENT_SOURCE_DIR}/site/local-worker.bro
-    ${POLICYDIR}/site
+    ${BRO_SCRIPT_INSTALL_PATH}/site
    local-worker.bro)
--- a/scripts/base/frameworks/cluster/nodes/manager.bro
+++ b/scripts/base/frameworks/cluster/nodes/manager.bro
@ -10,7 +10,7 @@

@prefixes += cluster-manager

-## Load the script for local site configuration for the manager node.
+# Load the script for local site configuration for the manager node.
@load site/local-manager

 ## Turn off remote logging since this is the manager and should only log here.
--- a/scripts/base/frameworks/cluster/nodes/proxy.bro
+++ b/scripts/base/frameworks/cluster/nodes/proxy.bro
@ -1,7 +1,7 @@

@prefixes += cluster-proxy

-## Load the script for local site configuration for proxy nodes.
+# Load the script for local site configuration for proxy nodes.
@load site/local-proxy

 ## The proxy only syncs state; does not forward events.
--- a/scripts/base/frameworks/cluster/nodes/worker.bro
+++ b/scripts/base/frameworks/cluster/nodes/worker.bro
@ -1,7 +1,7 @@

@prefixes += cluster-worker

-## Load the script for local site configuration for the worker nodes.
+# Load the script for local site configuration for the worker nodes.
@load site/local-worker

 ## Don't do any local logging.
--- a/scripts/base/frameworks/cluster/setup-connections.bro
+++ b/scripts/base/frameworks/cluster/setup-connections.bro
@ -1,3 +1,5 @@
+@load ./main
+@load base/frameworks/communication/main

 module Cluster;

--- a/scripts/base/frameworks/communication/load.bro
+++ b/scripts/base/frameworks/communication/load.bro
@ -1 +1 @@
-@load ./main
+@load ./main
--- a/scripts/base/frameworks/notice/actions/email_admin.bro
+++ b/scripts/base/frameworks/notice/actions/email_admin.bro
@ -1,4 +1,3 @@
-
 module Notice;

 export {
@ -25,4 +24,4 @@ event notice(n: Notice::Info) &priority=-5
 		if ( email != "" )
 			email_notice_to(n, email, T);
 		}
-	}
+	}
--- a/scripts/base/frameworks/notice/extend-email/hostnames.bro
+++ b/scripts/base/frameworks/notice/extend-email/hostnames.bro
@ -37,4 +37,4 @@ event Notice::notice(n: Notice::Info) &priority=10
 	
 	if ( output != "" )
 		n$email_body_sections[|n$email_body_sections|] = output;
-	}
+	}
--- a/scripts/base/frameworks/notice/weird.bro
+++ b/scripts/base/frameworks/notice/weird.bro
@ -1,4 +1,3 @@
-
 module Weird;

 export {
--- a/scripts/base/protocols/mime/base.bro
+++ b/scripts/base/protocols/mime/base.bro
@ -1,7 +1,7 @@
 ##! The mime script does analysis of MIME encoded messages seen in certain
 ##! protocols (only SMTP and POP3 at the moment).

-@load utils/strings
+@load base/utils/strings

 module MIME;

--- a/scripts/base/protocols/mime/file-extract.bro
+++ b/scripts/base/protocols/mime/file-extract.bro
@ -1,5 +1,5 @@
-@load protocols/mime/file-ident
-@load utils/files
+@load ./file-ident
+@load base/utils/files

 module MIME;

--- a/scripts/base/protocols/mime/file-hash.bro
+++ b/scripts/base/protocols/mime/file-hash.bro
@ -1,4 +1,4 @@
-@load protocols/mime/file-ident
+@load ./file-ident

 module MIME;

@ -75,4 +75,4 @@ event mime_end_entity(c: connection) &priority=-3
 		NOTICE([$note=MD5, $msg=fmt("Calculated a hash for a MIME entity from %s", c$id$orig_h),
 		        $sub=c$mime$md5, $conn=c]);
 		}
-	}
+	}
--- a/scripts/base/protocols/mime/file-ident.bro
+++ b/scripts/base/protocols/mime/file-ident.bro
@ -1,4 +1,4 @@
-@load protocols/mime/base
+@load ./base

 module MIME;

--- a/scripts/policy/frameworks/communication/listen-clear.bro
+++ b/scripts/policy/frameworks/communication/listen-clear.bro
@ -1,5 +1,7 @@
 ##! Listen for other Bro instances to make unencrypted connections.

+@load base/frameworks/communication
+
 module Communication;

 export {
--- a/scripts/policy/frameworks/communication/listen-ssl.bro
+++ b/scripts/policy/frameworks/communication/listen-ssl.bro
@ -1,5 +1,7 @@
 ##! Listen for other Bro instances and encrypt the connection with SSL.

+@load base/frameworks/communication
+
 module Communication;

 export {
--- a/scripts/policy/frameworks/control/controller.bro
+++ b/scripts/policy/frameworks/control/controller.bro
@ -1,3 +1,4 @@
+@load base/frameworks/communication

 module Control;

@ -99,4 +100,4 @@ event remote_connection_handshake_done(p: event_peer) &priority=-10
 		# Signal configuration update to peer.
 		event Control::configuration_update_request();
 		}
-	}
+	}
--- a/scripts/policy/protocols/conn/scan.bro
+++ b/scripts/policy/protocols/conn/scan.bro
@ -1,4 +1,4 @@
-@load frameworks/notice
+@load base/frameworks/notice
@load port-name

 module Scan;
--- a/src/BroDoc.cc
+++ b/src/BroDoc.cc
@ -20,8 +20,8 @@ BroDoc::BroDoc(const std::string& rel, const std::string& abs)

 	if ( rel[0] == '/' || rel[0] == '.' )
 		{
-		// The Bro script must not be on a subpath of the policy/ dir of
-		// BROPATH, so just use the basename as the document title.
+		// The Bro script isn't being loaded via BROPATH, so just use basename
+		// as the document title.
 		doc_title = source_filename;
 		}
 	else
@ -33,8 +33,14 @@ BroDoc::BroDoc(const std::string& rel, const std::string& abs)
 			doc_title = rel + "/" + source_filename;
 		}

+	downloadable_filename = source_filename;
+
+	size_t ext_pos = downloadable_filename.find(".bif.bro");
+	if ( std::string::npos != ext_pos )
+		downloadable_filename.erase(ext_pos + 4);
+
 	reST_filename = doc_title;
-	size_t ext_pos = reST_filename.find(".bro");
+	ext_pos = reST_filename.find(".bro");

 	if ( std::string::npos == ext_pos )
 		reST_filename += ".rst";
@ -103,14 +109,14 @@ void BroDoc::AddImport(const std::string& s)
 			{
 			if ( subpath[0] == '/' || subpath[0] == '.' )
 				{
-				// it's not a subpath of policy/, so just add the name of it
+				// it's not a subpath of scripts/, so just add the name of it
 				// as it's given in the @load directive
 				imports.push_back(lname);
 				}
 			else
 				{
 				// combine the base file name of script in the @load directive
-				// with the subpath of BROPATH's policy/ directory
+				// with the subpath of BROPATH's scripts/ directory
 				string fname(subpath);
 				char* othertmp = copy_string(lname.c_str());
 				fname.append("/").append(basename(othertmp));
@ -167,7 +173,7 @@ void BroDoc::WriteDocFile() const
 	WriteSectionHeading(doc_title.c_str(), '=');

 	WriteToDoc("\n:download:`Original Source File <%s>`\n\n",
-		source_filename.c_str());
+		downloadable_filename.c_str());

 	WriteSectionHeading("Overview", '-');
 	WriteStringList("%s\n", "%s\n\n", summary);
@ -185,7 +191,7 @@ void BroDoc::WriteDocFile() const
 			size_t pos = pretty.find("/index");
 			if ( pos != std::string::npos && pos + 6 == pretty.size() )
 				pretty = pretty.substr(0, pos);
-			WriteToDoc(":doc:`%s </policy/%s>`", pretty.c_str(), it->c_str());
+			WriteToDoc(":doc:`%s </scripts/%s>`", pretty.c_str(), it->c_str());
 			}
 		WriteToDoc("\n");
 		}
--- a/src/BroDoc.h
+++ b/src/BroDoc.h
@ -24,10 +24,9 @@ public:
 	 * If the filename doesn't end in ".bro", then ".rst" is just appended.
 	 * Any '/' characters in the reST file name that result from choice of
 	 * the 'rel' parameter are replaced with '^'.
-	 * @param subpath A string representing a subpath of BROPATH's policy/
-	 *        directory in which the source file is located.  It can
-	 *        also be full path to the file or a full path that's in BROPATH,
-	 *        but in either of those cases, the parameter is essentially
+	 * @param rel A string representing a subpath of the root Bro script
+	 *        source/install directory in which the source file is located.
+	 *        It can also be an absolute path, but then the parameter is
 	 *        ignored and the document title is just derived from file name
 	 * @param abs The absolute path to the Bro script for which to generate
 	 *        documentation.
@ -211,6 +210,7 @@ protected:
 	FILE* reST_file;
 	std::string reST_filename;
 	std::string source_filename;	// points to the basename of source file
+	std::string downloadable_filename; // file that will be linked for download
 	std::string doc_title;
 	std::string packet_filter;

--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@ -408,7 +408,8 @@ set(bro_SRCS

 collect_headers(bro_HEADERS ${bro_SRCS})

-add_definitions(-DPOLICYDEST="${POLICYDIR}")
+add_definitions(-DBRO_SCRIPT_INSTALL_PATH="${BRO_SCRIPT_INSTALL_PATH}")
+add_definitions(-DBRO_SCRIPT_SOURCE_PATH="${BRO_SCRIPT_SOURCE_PATH}")

 add_executable(bro ${bro_SRCS} ${bro_HEADERS})

@ -423,7 +424,7 @@ set(brolibs
 target_link_libraries(bro ${brolibs})

 install(TARGETS bro DESTINATION bin)
-install(FILES ${INSTALL_BIF_OUTPUTS} DESTINATION ${POLICYDIR})
+install(FILES ${INSTALL_BIF_OUTPUTS} DESTINATION ${BRO_SCRIPT_INSTALL_PATH})

 set(BRO_EXE bro
    CACHE STRING "Bro executable binary" FORCE)
--- a/src/util.cc
+++ b/src/util.cc
@ -757,9 +757,9 @@ const char* bro_path()
 	const char* path = getenv("BROPATH");
 	if ( ! path )
 		path = ".:"
-			POLICYDEST ":"
-			POLICYDEST "/policy" ":"
-			POLICYDEST "/site";
+			BRO_SCRIPT_INSTALL_PATH ":"
+			BRO_SCRIPT_INSTALL_PATH "/policy" ":"
+			BRO_SCRIPT_INSTALL_PATH "/site";

 	return path;
 	}
@ -891,60 +891,36 @@ const char* normalize_path(const char* path)
 	return copy_string(new_path.c_str());
 	}

-// Returns the subpath of BROPATH's policy/ directory in which the loaded
-// file in located.  If it's not under a subpath of policy/ then the full
-// path is returned, else the subpath of policy/ concatentated with any
-// directory prefix of the file is returned.
-void get_policy_subpath(const char* dir, const char* file, const char** subpath)
+// Returns the subpath of the root Bro script install/source directory in
+// which the loaded file is located.  If it's not under a subpath of that
+// directory (e.g. cwd or custom path) then the full path is returned.
+void get_script_subpath(const std::string& full_filename, const char** subpath)
 	{
-	// first figure out if this is a subpath of policy/
-	const char* ploc = strstr(dir, "policy");
-	if ( ploc )
-		if ( ploc[6] == '\0' )
-			*subpath = copy_string(ploc + 6);
-		else if ( ploc[6] == '/' )
-			*subpath = copy_string(ploc + 7);
-		else
-			*subpath = copy_string(dir);
-	else
-		*subpath = copy_string(dir);
+	size_t p;
+	std::string my_subpath(full_filename);

-	// and now add any directory parts of the filename
-	char full_filename_buf[1024];
-	safe_snprintf(full_filename_buf, sizeof(full_filename_buf),
-			"%s/%s", dir, file);
-	char* tmp = copy_string(file);
-	const char* fdir = 0;
-
-	if ( is_dir(full_filename_buf) )
-		fdir = file;
-
-	if ( ! fdir )
-		fdir = dirname(tmp);
-
-	if ( ! streq(fdir, ".") )
+	// get the parent directory of file (if not already a directory)
+	if ( ! is_dir(full_filename.c_str()) )
 		{
-		size_t full_subpath_len = strlen(*subpath) + strlen(fdir) + 1;
-		bool needslash = false;
-		if ( strlen(*subpath) != 0 && (*subpath)[strlen(*subpath) - 1] != '/' )
-			{
-			++full_subpath_len;
-			needslash = true;
-			}
-
-		char* full_subpath = new char[full_subpath_len];
-		strcpy(full_subpath, *subpath);
-		if ( needslash )
-			strcat(full_subpath, "/");
-		strcat(full_subpath, fdir);
-		delete [] *subpath;
-		*subpath = full_subpath;
+		char* tmp = copy_string(full_filename.c_str());
+		my_subpath = dirname(tmp);
+		delete [] tmp;
 		}

-	const char* normalized_subpath = normalize_path(*subpath);
-	delete [] tmp;
-	delete [] *subpath;
-	*subpath = normalized_subpath;
+	// first check if this is some subpath of the installed scripts root path,
+	// if not check if it's a subpath of the script source root path,
+	// if neither, will just use the given directory
+	if ( (p=my_subpath.find(BRO_SCRIPT_INSTALL_PATH)) != std::string::npos )
+		my_subpath.erase(0, strlen(BRO_SCRIPT_INSTALL_PATH));
+	else if ( (p=my_subpath.find(BRO_SCRIPT_SOURCE_PATH)) != std::string::npos )
+		my_subpath.erase(0, strlen(BRO_SCRIPT_SOURCE_PATH));
+
+	// if root path found, remove path separators until next path component
+	if ( p != std::string::npos )
+		while ( my_subpath.size() && my_subpath[0] == '/' )
+			my_subpath.erase(0, 1);
+
+	*subpath = normalize_path(my_subpath.c_str());
 	}

 extern string current_scanned_file_path;
@ -1001,7 +977,7 @@ FILE* search_for_file(const char* filename, const char* ext,
 		     ! is_dir(full_filename_buf) )
 			{
 			if ( bropath_subpath )
-				get_policy_subpath(dir_beginning, filename, bropath_subpath);
+				get_script_subpath(full_filename_buf, bropath_subpath);
 			return open_file(full_filename_buf, full_filename, load_pkgs);
 			}

@ -1010,7 +986,7 @@ FILE* search_for_file(const char* filename, const char* ext,
 		if ( access(full_filename_buf, R_OK) == 0 )
 			{
 			if ( bropath_subpath )
-				get_policy_subpath(dir_beginning, filename, bropath_subpath);
+				get_script_subpath(full_filename_buf, bropath_subpath);
 			return open_file(full_filename_buf, full_filename, load_pkgs);
 			}

--- a/src/util.h
+++ b/src/util.h
@ -179,7 +179,7 @@ extern const char* bro_path();
 extern const char* bro_prefixes();
 std::string dot_canon(std::string path, std::string file, std::string prefix = "");
 const char* normalize_path(const char* path);
-void get_policy_subpath(const char* dir, const char* file, const char** subpath);
+void get_script_subpath(const std::string& full_filename, const char** subpath);
 extern FILE* search_for_file(const char* filename, const char* ext,
 	const char** full_filename, bool load_pkgs, const char** bropath_subpath);

--- a/testing/btest/Baseline/core.load-prefixes/output
+++ b/testing/btest/Baseline/core.load-prefixes/output
@ -1,4 +1,4 @@
-loaded lcl2.site.bro
-loaded lcl.site.bro
-loaded lcl2.protocols.http.bro
-loaded lcl.protocols.http.bro
+loaded lcl2.base.utils.site.bro
+loaded lcl.base.utils.site.bro
+loaded lcl2.base.protocols.http.bro
+loaded lcl.base.protocols.http.bro
--- a/testing/btest/Baseline/doc.autogen-reST-example/example.rst
+++ b/testing/btest/Baseline/doc.autogen-reST-example/example.rst
@ -29,7 +29,7 @@ each of "columns", "event", "filter" depending on exactly what it's doing.

 :Author: Jon Siwek <jsiwek@ncsa.illinois.edu>

-:Imports: :doc:`frameworks/notice </policy/frameworks/notice/index>`
+:Imports: :doc:`policy/frameworks/software/vulnerable </scripts/policy/frameworks/software/vulnerable>`

 Summary
 ~~~~~~~
--- a/testing/btest/Baseline/doc.blacklist-reminder/.stderr
+++ b/testing/btest/Baseline/doc.blacklist-reminder/.stderr
--- a/testing/btest/Baseline/policy.misc.default-loaded-scripts/canonified_loaded_scripts.log
+++ b/testing/btest/Baseline/policy.misc.default-loaded-scripts/canonified_loaded_scripts.log
@ -27,6 +27,8 @@
 2	scripts/base/frameworks/notice/./actions/drop.bro
 2	scripts/base/frameworks/notice/./actions/email_admin.bro
 2	scripts/base/frameworks/notice/./actions/page.bro
+2	scripts/base/frameworks/notice/./actions/add-geodata.bro
+2	scripts/base/frameworks/notice/./extend-email/hostnames.bro
 1	scripts/base/frameworks/dpd/__load__.bro
 2	scripts/base/frameworks/dpd/./main.bro
 1	scripts/base/frameworks/signatures/__load__.bro
@ -41,6 +43,7 @@
 1	scripts/base/frameworks/metrics/__load__.bro
 2	scripts/base/frameworks/metrics/./main.bro
 1	scripts/base/frameworks/communication/__load__.bro
+2	scripts/base/frameworks/communication/./main.bro
 1	scripts/base/frameworks/control/__load__.bro
 2	scripts/base/frameworks/control/./main.bro
 1	scripts/base/frameworks/cluster/__load__.bro
@ -69,6 +72,8 @@
 2	scripts/base/protocols/irc/./dcc-send.bro
 1	scripts/base/protocols/smtp/__load__.bro
 2	scripts/base/protocols/smtp/./main.bro
+2	scripts/base/protocols/smtp/./entities.bro
+2	scripts/base/protocols/smtp/./entities-excerpt.bro
 1	scripts/base/protocols/ssh/__load__.bro
 2	scripts/base/protocols/ssh/./main.bro
 1	scripts/base/protocols/ssl/__load__.bro
--- a/testing/btest/core/load-prefixes.bro
+++ b/testing/btest/core/load-prefixes.bro
@ -9,11 +9,11 @@
@TEST-END-FILE

@TEST-START-FILE lcl.base.utils.site.bro
-print "loaded lcl.base.site.bro";
+print "loaded lcl.base.utils.site.bro";
@TEST-END-FILE

@TEST-START-FILE lcl2.base.utils.site.bro
-print "loaded lcl2.base.site.bro";
+print "loaded lcl2.base.utils.site.bro";
@TEST-END-FILE

@TEST-START-FILE lcl.base.protocols.http.bro
--- a/testing/btest/doc/blacklist-reminder.test
+++ b/testing/btest/doc/blacklist-reminder.test
@ -0,0 +1,8 @@
+# This test will fail if there are Bro scripts that have been temporarily
+# blacklisted from the documentation generation process for some reason
+# (e.g. they're a work-in-progress or otherwise fail to parse).  It's meant
+# to serve as a reminder that some future action may be needed to generate
+# documentation for the blacklisted scripts.
+#
+# @TEST-EXEC: $DIST/doc/scripts/genDocSourcesList.sh
+# @TEST-EXEC: btest-diff .stderr