diff --git a/CHANGES b/CHANGES
index 31fddce833..2cbebd0255 100644
--- a/CHANGES
+++ b/CHANGES
@@ -1,4 +1,226 @@
+2.2-117 | 2014-01-23 14:18:19 -0800
+
+ * Fixing initialization context in anonymous functions. (Robin
+ Sommer)
+
+2.2-115 | 2014-01-22 12:11:18 -0800
+
+ * Add unit tests for new Bro Manual docs. (Jon Siwek)
+
+ * New content for the "Using Bro" section of the manual. (Rafael
+ Bonilla/Jon Siwek)
+
+2.2-105 | 2014-01-20 12:16:48 -0800
+
+ * Support GRE tunnel decapsulation, including enhanced GRE headers.
+ GRE tunnels are treated just like IP-in-IP tunnels by parsing past
+ the GRE header in between the delivery and payload IP packets.
+ Addresses BIT-867. (Jon Siwek)
+
+ * Simplify FragReassembler memory management. (Jon Siwek)
+
+2.2-102 | 2014-01-20 12:00:29 -0800
+
+ * Include file information (MIME type and description) into notice
+ emails if available. (Justin Azoff)
+
+2.2-100 | 2014-01-20 11:54:58 -0800
+
+ * Fix caching of recently validated SSL certifcates. (Justin Azoff)
+
+2.2-98 | 2014-01-20 11:50:32 -0800
+
+ * For notice suppresion, instead of storing the entire notice in
+ Notice::suppressing, just store the time the notice should be
+ suppressed until. This saves significant memory but can no longer
+ raise end_suppression, which has been removed. (Justin Azoff)
+
+2.2-96 | 2014-01-20 11:41:07 -0800
+
+ * Integrate libmagic 5.16. Bro now now always relies on
+ builtin/shipped magic library/database. (Jon Siwek)
+
+ * Bro now requires a CMake 2.8.x, but no longer a pre-installed
+ libmagic. (Jon Siwek)
+
+2.2-93 | 2014-01-13 09:16:51 -0800
+
+ * Fixing compile problems with some versions of libc++. Reported by
+ Craig Leres. (Robin Sommer)
+
+2.2-91 | 2014-01-13 01:33:28 -0800
+
+ * Improve GeoIP City database support. When trying to open a city
+ database, it now considers both the "REV0" and "REV1" versions of
+ the city database instead of just the former. (Jon Siwek)
+
+ * Broxygen init fixes. Addresses BIT-1110. (Jon Siwek)
+
+ - Don't check mtime of bro binary if BRO_DISABLE_BROXYGEN env var set.
+
+ - Fix failure to locate bro binary if invoking from a relative
+ path and '.' isn't in PATH.
+
+ * Fix for packet writing to make it use the global snap length.
+ (Seth Hall)
+
+ * Fix for traffic with TCP segmentation offloading with IP header
+ len field being set to zero. (Seth Hall)
+
+ * Canonify output of a unit test. (Jon Siwek)
+
+ * A set of documentation updates. (Daniel Thayer)
+
+ - Fix typo in Bro 2.2 NEWS on string indexing.
+ - Fix typo in the Quick Start Guide, and clarified the
+ instructions about modifying crontab.
+ - Add/fix documentation for missing/misnamed event parameters.
+ - Fix typos in BIF documentation of hexstr_to_bytestring.
+ - Update the documentation of types and attributes.
+ - Documented the new substring extraction functionality.
+ - Clarified the description of "&priority" and "void".
+
+2.2-75 | 2013-12-18 08:36:50 -0800
+
+ * Fixing segfault with mismatching set &default in record fields.
+ (Robin Sommer)
+
+2.2-74 | 2013-12-16 08:49:55 -0800
+
+ * Improve warnings emitted from raw/execute input reader. (Jon
+ Siwek)
+
+ * Further improve core.when-interpreter-exceptions unit test. (Jon
+ Siwek)
+
+2.2-72 | 2013-12-12 07:12:47 -0800
+
+ * Improve the core.when-interpreter-exceptions unit test to prevent
+ it from occasionally timing out. (Jon Siwek)
+
+2.2-70 | 2013-12-10 15:02:50 -0800
+
+ * Fix (harmless) uninitialized field in basename/dirname util
+ wrapper. (Jon Siwek)
+
+2.2-68 | 2013-12-09 15:19:37 -0800
+
+ * Several improvements to input framework error handling for more
+ robustness and more helpful error messages. Includes tests for
+ many cases. (Bernhard Amann)
+
+2.2-66 | 2013-12-09 13:54:16 -0800
+
+ * Fix table &default reference counting for record ctor expressions.
+ (Jon Siwek)
+
+ * Close signature files after done parsing. (Jon Siwek)
+
+ * Fix unlikely null ptr deref in broxygen::Manager. (Jon Siwek)
+
+ * FreeBSD build fix addendum: unintended variable shadowing. (Jon
+ Siwek)
+
+ * Fix build on FreeBSD. basename(3)/dirname(3) const-ness may vary
+ w/ platform. (Jon Siwek)
+
+ * Updated software framework to support parsing IE11 user-agent
+ strings. (Seth Hall)
+
+ * Fix the irc_reply event for several server message types. (Seth
+ Hall)
+
+ * Fix memory leak in input framework. If the input framework was
+ used to read event streams and those streams contained records
+ with more than one field, not all elements of the threading Values
+ were cleaned up. Addresses BIT-1103. (Bernhard Amann)
+
+ * Minor Broxygen improvements. Addresses BIT-1098. (Jon Siwek)
+
+2.2-51 | 2013-12-05 07:53:37 -0800
+
+ * Improve a unit test involving 'when' conditionals. (Jon Siwek)
+
+2.2-48 | 2013-12-04 13:45:47 -0800
+
+ * Support omission of string slice low/high indices, BIT-1097.
+
+ Omission of the low index defaults to 0:
+
+ s = "12345"; s[:3] == "123"
+
+ Omission of the high index defaults to length of the string:
+
+ s = "12345"; s[3:] == "45" (Jon Siwek)
+
+ * Tweak to SMTP script to adjust for new string slicing behaviour.
+ (Robin Sommer)
+
+ * Test updates. (Robin Sommer)
+
+2.2-44 | 2013-12-04 12:41:51 -0800
+
+ * Fix string slice notation. Addresses BIT-1097. (Jon Siwek)
+
+ Slice ranges were not correctly determined for negative indices
+ and also off by one in general (included one more element at the
+ end of the substring than what actually matched the index range).
+ It's now equivalent to Python slice notation. Accessing a string
+ at a single index is also the same as Python except that an
+ out-of-range index returns an empty string instead of throwing an
+ expection.
+
+2.2-41 | 2013-12-04 12:40:51 -0800
+
+ * Updating tests. (Robin Sommer)
+
+2.2-40 | 2013-12-04 12:16:38 -0800
+
+ * ssl_client_hello() now receives a vector of ciphers, instead of a
+ set, to preserve their order. (Bernhard Amann)
+
+2.2-38 | 2013-12-04 12:10:54 -0800
+
+ * New script misc/dump-events.bro, along with core support, that
+ dumps events Bro is raising in an easily readable form for
+ debugging. (Robin Sommer)
+
+ * Prettyfing Describe() for record types. If a record type has a
+ name and ODesc is set to short, we now print the name instead of
+ the full field list. (Robin Sommer)
+
+2.2-35 | 2013-12-04 10:10:32 -0800
+
+ * Rework the automated script-reference documentation generation
+ process, broxygen. Addresses BIT-701 and BIT-751. (Jon Siwek)
+
+ Highlights:
+
+ - Remove --doc-scripts and -Z options to toggle documentation
+ mode. The parser is now always instrumented to gather
+ documentation from comments of the form "##", "##!", or
+ "##<".
+
+ - Raw comments are available at runtime through several BIF
+ functions: get_*_comments;
+
+ - Add --broxygen and -X options to toggle generating
+ reST-format documentation output, driven by a config file
+ argument.
+
+ - Add a "broxygen" Sphinx extension domain, allowing certain
+ pieces of documentation to be generated on-the-fly via
+ invoking a Bro process. Re-organized/cleaned up the Sphinx
+ source tree in doc/ to use this in some places.
+
+2.2-11 | 2013-12-03 10:56:28 -0800
+
+ * Unit test for broccoli vector support. (Jon Siwek)
+
+ * Changed ordering of Bro type tag enum, which was out of sync. (Jon
+ Siwek)
+
2.2-9 | 2013-11-18 14:03:21 -0800
* Update local.bro for Bro >= 2.2. The commented out Notice::policy
diff --git a/CMakeLists.txt b/CMakeLists.txt
index e8ed7975b7..483f88babf 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -3,7 +3,7 @@ project(Bro C CXX)
# When changing the minimum version here, also adapt
# cmake/BroPluginDynamic and
# aux/bro-aux/plugin-support/skeleton/CMakeLists.txt
-cmake_minimum_required(VERSION 2.6.3 FATAL_ERROR)
+cmake_minimum_required(VERSION 2.8.0 FATAL_ERROR)
include(cmake/CommonCMakeConfig.cmake)
@@ -15,7 +15,6 @@ if (NOT BRO_SCRIPT_INSTALL_PATH)
# set the default Bro script installation path (user did not specify one)
set(BRO_SCRIPT_INSTALL_PATH ${BRO_ROOT_DIR}/share/bro)
endif ()
-set(BRO_SCRIPT_SOURCE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/scripts)
# sanitize the Bro script install directory into an absolute path
# (CMake is confused by ~ as a representation of home directory)
@@ -49,6 +48,32 @@ set(VERSION_MAJ_MIN "${VERSION_MAJOR}.${VERSION_MINOR}")
########################################################################
## Dependency Configuration
+include(ExternalProject)
+
+# LOG_* options to ExternalProject_Add appear in CMake 2.8.3. If
+# available, using them hides external project configure/build output.
+if("${CMAKE_VERSION}" VERSION_GREATER 2.8.2)
+ set(EXTERNAL_PROJECT_LOG_OPTIONS
+ LOG_DOWNLOAD 1 LOG_UPDATE 1 LOG_CONFIGURE 1 LOG_BUILD 1 LOG_INSTALL 1)
+else()
+ set(EXTERNAL_PROJECT_LOG_OPTIONS)
+endif()
+
+set(LIBMAGIC_PREFIX ${CMAKE_CURRENT_BINARY_DIR}/libmagic-prefix)
+set(LIBMAGIC_INCLUDE_DIR ${LIBMAGIC_PREFIX}/include)
+set(LIBMAGIC_LIB_DIR ${LIBMAGIC_PREFIX}/lib)
+set(LIBMAGIC_LIBRARY ${LIBMAGIC_LIB_DIR}/libmagic.a)
+ExternalProject_Add(libmagic
+ PREFIX ${LIBMAGIC_PREFIX}
+ URL ${CMAKE_CURRENT_SOURCE_DIR}/src/3rdparty/file-5.16.tar.gz
+ CONFIGURE_COMMAND ./configure --enable-static --disable-shared
+ --prefix=${LIBMAGIC_PREFIX}
+ --includedir=${LIBMAGIC_INCLUDE_DIR}
+ --libdir=${LIBMAGIC_LIB_DIR}
+ BUILD_IN_SOURCE 1
+ ${EXTERNAL_PROJECT_LOG_OPTIONS}
+)
+
include(FindRequiredPackage)
# Check cache value first to avoid displaying "Found sed" messages everytime
@@ -67,7 +92,6 @@ FindRequiredPackage(BISON)
FindRequiredPackage(PCAP)
FindRequiredPackage(OpenSSL)
FindRequiredPackage(BIND)
-FindRequiredPackage(LibMagic)
FindRequiredPackage(ZLIB)
if (NOT BinPAC_ROOT_DIR AND
@@ -83,18 +107,12 @@ if (MISSING_PREREQS)
message(FATAL_ERROR "Configuration aborted due to missing prerequisites")
endif ()
-set(libmagic_req 5.04)
-if ( LibMagic_VERSION VERSION_LESS ${libmagic_req} )
- message(FATAL_ERROR "libmagic of at least version ${libmagic_req} required "
- "(found ${LibMagic_VERSION})")
-endif ()
-
include_directories(BEFORE
${PCAP_INCLUDE_DIR}
${OpenSSL_INCLUDE_DIR}
${BIND_INCLUDE_DIR}
${BinPAC_INCLUDE_DIR}
- ${LibMagic_INCLUDE_DIR}
+ ${LIBMAGIC_INCLUDE_DIR}
${ZLIB_INCLUDE_DIR}
)
@@ -173,7 +191,7 @@ set(brodeps
${PCAP_LIBRARY}
${OpenSSL_LIBRARIES}
${BIND_LIBRARY}
- ${LibMagic_LIBRARY}
+ ${LIBMAGIC_LIBRARY}
${ZLIB_LIBRARY}
${OPTLIBS}
)
diff --git a/Makefile b/Makefile
index 4984f6db31..f4b2104c73 100644
--- a/Makefile
+++ b/Makefile
@@ -29,18 +29,6 @@ doc: configured
docclean: configured
$(MAKE) -C $(BUILD) $@
-restdoc: configured
- $(MAKE) -C $(BUILD) $@
-
-restclean: configured
- $(MAKE) -C $(BUILD) $@
-
-broxygen: configured
- $(MAKE) -C $(BUILD) $@
-
-broxygenclean: configured
- $(MAKE) -C $(BUILD) $@
-
dist:
@rm -rf $(VERSION_FULL) $(VERSION_FULL).tgz
@rm -rf $(VERSION_MIN) $(VERSION_MIN).tgz
diff --git a/NEWS b/NEWS
index 4ae5669466..df0bd07f4c 100644
--- a/NEWS
+++ b/NEWS
@@ -4,6 +4,38 @@ release. For an exhaustive list of changes, see the ``CHANGES`` file
(note that submodules, such as BroControl and Broccoli, come with
their own ``CHANGES``.)
+Bro 2.3
+=======
+
+[In progress]
+
+Dependencies
+------------
+
+- Bro no longer requires a pre-installed libmagic (because it now
+ ships its own).
+
+- Compiling from source now needs a CMake version >= 2.8.0.
+
+New Functionality
+-----------------
+
+- Support for GRE tunnel decapsulation, including enhanced GRE
+ headers. GRE tunnels are treated just like IP-in-IP tunnels by
+ parsing past the GRE header in between the delivery and payload IP
+ packets.
+
+Changed Functionality
+---------------------
+
+- string slices now exclude the end index (e.g., "123"[1:2] returns
+ "2"). Generally, Bro's string slices now behave similar to Python.
+
+- ssl_client_hello() now receives a vector of ciphers, instead of a
+ set, to preserve their order.
+
+- Notice::end_suppression() has been removed.
+
Bro 2.2
=======
@@ -180,9 +212,9 @@ New Functionality
global s = MySet([$c=1], [$c=2]);
- Strings now support the subscript operator to extract individual
- characters and substrings (e.g., ``s[4]``, ``s[1,5]``). The index
+ characters and substrings (e.g., ``s[4]``, ``s[1:5]``). The index
expression can take up to two indices for the start and end index of
- the substring to return (e.g. ``mystring[1,3]``).
+ the substring to return (e.g. ``mystring[1:3]``).
- Functions now support default parameters, e.g.::
diff --git a/VERSION b/VERSION
index ac3e73a062..33ad1f2a99 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-2.2-9
+2.2-117
diff --git a/aux/broccoli b/aux/broccoli
index 889f9c6594..17ec437752 160000
--- a/aux/broccoli
+++ b/aux/broccoli
@@ -1 +1 @@
-Subproject commit 889f9c65944ceac20ad9230efc39d33e6e1221c3
+Subproject commit 17ec437752837fb4214abfb0a2da49df74668d5d
diff --git a/aux/broctl b/aux/broctl
index 0cd102805e..6e01d6972f 160000
--- a/aux/broctl
+++ b/aux/broctl
@@ -1 +1 @@
-Subproject commit 0cd102805e73343cab3f9fd4a76552e13940dad9
+Subproject commit 6e01d6972f02d68ee82d05f392d1a00725595b7f
diff --git a/aux/btest b/aux/btest
index ce366206e3..26c3136d56 160000
--- a/aux/btest
+++ b/aux/btest
@@ -1 +1 @@
-Subproject commit ce366206e3407e534a786ad572c342e9f9fef26b
+Subproject commit 26c3136d56493017bc33c5a2f22ae393d585c2d9
diff --git a/bro-path-dev.in b/bro-path-dev.in
index 2c17d057c9..de8b0274b9 100755
--- a/bro-path-dev.in
+++ b/bro-path-dev.in
@@ -10,10 +10,4 @@
# BROPATH=`./bro-path-dev` ./src/bro
#
-broPolicies=${BRO_SCRIPT_SOURCE_PATH}:${BRO_SCRIPT_SOURCE_PATH}/policy:${BRO_SCRIPT_SOURCE_PATH}/site
-
-broGenPolicies=${CMAKE_BINARY_DIR}/scripts
-
-installedPolicies=${BRO_SCRIPT_INSTALL_PATH}:${BRO_SCRIPT_INSTALL_PATH}/site
-
-echo .:$broPolicies:$broGenPolicies
+echo .:${CMAKE_SOURCE_DIR}/scripts:${CMAKE_SOURCE_DIR}/scripts/policy:${CMAKE_SOURCE_DIR}/scripts/site:${CMAKE_BINARY_DIR}/scripts
diff --git a/cmake b/cmake
index 856b311ae1..67da63d43e 160000
--- a/cmake
+++ b/cmake
@@ -1 +1 @@
-Subproject commit 856b311ae19c49b9602885eb950361342ed76df7
+Subproject commit 67da63d43e734111b324d8ed045e188e0a28ebf2
diff --git a/doc/CMakeLists.txt b/doc/CMakeLists.txt
index 373b4643ba..9498556edc 100644
--- a/doc/CMakeLists.txt
+++ b/doc/CMakeLists.txt
@@ -1,75 +1,86 @@
-set(BIF_SRC_DIR ${PROJECT_SOURCE_DIR}/src)
-set(RST_OUTPUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/rest_output)
-set(DOC_OUTPUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/out)
-set(DOC_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR})
-set(DOC_SOURCE_WORKDIR ${CMAKE_CURRENT_BINARY_DIR}/sphinx-sources)
+set(BROCCOLI_DOCS_SRC ${CMAKE_BINARY_DIR}/aux/broccoli/doc/html)
+set(BROCCOLI_DOCS_DST ${CMAKE_BINARY_DIR}/html/broccoli-api)
+set(SPHINX_INPUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/sphinx_input)
+set(SPHINX_OUTPUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/sphinx_output)
+set(BROXYGEN_SCRIPT_OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/broxygen_script_output)
+set(BROXYGEN_CACHE_DIR ${CMAKE_CURRENT_BINARY_DIR}/broxygen_cache)
-set(MASTER_POLICY_INDEX ${CMAKE_CURRENT_BINARY_DIR}/scripts/policy_index)
-set(MASTER_PACKAGE_INDEX ${CMAKE_CURRENT_BINARY_DIR}/scripts/pkg_index)
+# Find out what BROPATH to use when executing bro.
+execute_process(COMMAND ${CMAKE_BINARY_DIR}/bro-path-dev
+ OUTPUT_VARIABLE BROPATH
+ RESULT_VARIABLE retval
+ OUTPUT_STRIP_TRAILING_WHITESPACE)
+if (NOT ${retval} EQUAL 0)
+ message(FATAL_ERROR "Problem setting BROPATH")
+endif ()
-file(GLOB_RECURSE DOC_SOURCES FOLLOW_SYMLINKS "*")
+set(BROMAGIC ${BRO_MAGIC_SOURCE_PATH})
-# configure the Sphinx config file (expand variables CMake might know about)
+# Configure the Sphinx config file (expand variables CMake might know about).
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/conf.py.in
${CMAKE_CURRENT_BINARY_DIR}/conf.py
@ONLY)
-add_subdirectory(scripts)
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/broxygen.conf.in
+ ${CMAKE_CURRENT_BINARY_DIR}/broxygen.conf
+ @ONLY)
-# The "broxygen" target generates reST documentation for any outdated bro
-# scripts and then uses Sphinx to generate HTML documentation from the reST
-add_custom_target(broxygen
- # copy the template documentation to the build directory
- # to give as input for sphinx
- COMMAND rsync -r --copy-links --times
- ${DOC_SOURCE_DIR}/
- ${DOC_SOURCE_WORKDIR}
- # copy generated policy script documentation into the
- # working copy of the template documentation
- COMMAND rsync -r --copy-links --times
- ${RST_OUTPUT_DIR}/
- ${DOC_SOURCE_WORKDIR}/scripts
- # append to the master index of all policy scripts
- COMMAND cat ${MASTER_POLICY_INDEX} >>
- ${DOC_SOURCE_WORKDIR}/scripts/scripts.rst
- # append to the master index of all policy packages
- COMMAND cat ${MASTER_PACKAGE_INDEX} >>
- ${DOC_SOURCE_WORKDIR}/scripts/packages.rst
- # construct a reST file for each group
- COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/bin/group_index_generator.py
- ${CMAKE_CURRENT_BINARY_DIR}/scripts/group_list
- ${CMAKE_CURRENT_BINARY_DIR}/scripts
- ${DOC_SOURCE_WORKDIR}/scripts
- # tell sphinx to generate html
- COMMAND sphinx-build
- -b html
- -c ${CMAKE_CURRENT_BINARY_DIR}
- -d ${DOC_OUTPUT_DIR}/doctrees
- ${DOC_SOURCE_WORKDIR}
- ${DOC_OUTPUT_DIR}/html
- # create symlink to the html output directory for convenience
- COMMAND "${CMAKE_COMMAND}" -E create_symlink
- ${DOC_OUTPUT_DIR}/html
- ${CMAKE_BINARY_DIR}/html
- # copy Broccoli API reference into output dir if it exists
- COMMAND test -d ${CMAKE_BINARY_DIR}/aux/broccoli/doc/html && ( rm -rf ${CMAKE_BINARY_DIR}/html/broccoli-api && cp -r ${CMAKE_BINARY_DIR}/aux/broccoli/doc/html ${CMAKE_BINARY_DIR}/html/broccoli-api ) || true
- WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
- COMMENT "[Sphinx] Generating HTML policy script docs"
- # SOURCES just adds stuff to IDE projects as a convenience
- SOURCES ${DOC_SOURCES})
+add_custom_target(sphinxdoc
+ # Copy the template documentation to build directory to use as input tree
+ # for Sphinx. This is needed because some parts are dynamically generated
+ # in to that tree by Bro/Broxygen.
+ COMMAND rsync -q -r --copy-links --times --delete
+ --filter='protect scripts/*'
+ ${CMAKE_CURRENT_SOURCE_DIR}/ ${SPHINX_INPUT_DIR}
+ # Use Bro/Broxygen to dynamically generate reST for all Bro scripts.
+ COMMAND BROPATH=${BROPATH}
+ BROMAGIC=${BROMAGIC}
+ ${CMAKE_BINARY_DIR}/src/bro
+ -X ${CMAKE_CURRENT_BINARY_DIR}/broxygen.conf
+ broxygen >/dev/null
+ # Rsync over the generated reST to the Sphinx source tree in the build dir.
+ COMMAND rsync -q -r --copy-links --times --delete --filter='protect *.bro'
+ ${BROXYGEN_SCRIPT_OUTPUT}/ ${SPHINX_INPUT_DIR}/scripts
+ # Rsync over Bro scripts to the Sphinx source tree in the build dir.
+ # These are used by :download: references in the generated script docs.
+ COMMAND rsync -q -r --copy-links --times --delete
+ --filter='protect /base/bif/*' --filter='protect *.rst'
+ --filter='include */' --filter='include *.bro' --filter='exclude *'
+ ${CMAKE_SOURCE_DIR}/scripts/ ${SPHINX_INPUT_DIR}/scripts
+ # Rsync over Bro scripts created by BIF compiler to the Sphinx source tree.
+ COMMAND rsync -q -r --copy-links --times --delete
+ --filter='protect *.rst' --filter='include */'
+ --filter='include *.bro' --filter='exclude *'
+ ${CMAKE_BINARY_DIR}/scripts/base/bif/
+ ${SPHINX_INPUT_DIR}/scripts/base/bif
+ # Use Sphinx to build HTML.
+ COMMAND sphinx-build
+ -b html
+ -c ${CMAKE_CURRENT_BINARY_DIR}
+ -d ${SPHINX_OUTPUT_DIR}/doctrees
+ ${SPHINX_INPUT_DIR}
+ ${SPHINX_OUTPUT_DIR}/html
+ # Create symlink to the html output directory for convenience.
+ COMMAND "${CMAKE_COMMAND}" -E create_symlink
+ ${SPHINX_OUTPUT_DIR}/html
+ ${CMAKE_BINARY_DIR}/html
+ # Copy Broccoli API reference into output dir if it exists.
+ COMMAND test -d ${BROCCOLI_DOCS_SRC} &&
+ ( rm -rf ${BROCCOLI_DOCS_DST} &&
+ cp -r ${BROCCOLI_DOCS_SRC} ${BROCCOLI_DOCS_DST} ) || true
+ WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
+ COMMENT "[Sphinx] Generate HTML documentation in ${CMAKE_BINARY_DIR}/html")
-# The "broxygenclean" target removes just the Sphinx input/output directories
-# from the build directory.
-add_custom_target(broxygenclean
- COMMAND "${CMAKE_COMMAND}" -E remove_directory
- ${DOC_SOURCE_WORKDIR}
- COMMAND "${CMAKE_COMMAND}" -E remove_directory
- ${DOC_OUTPUT_DIR}
- VERBATIM)
+add_dependencies(sphinxdoc bro)
-add_dependencies(broxygen restdoc)
+add_custom_target(sphinxdoc_clean
+ COMMAND "${CMAKE_COMMAND}" -E remove_directory ${SPHINX_INPUT_DIR}
+ COMMAND "${CMAKE_COMMAND}" -E remove_directory ${SPHINX_OUTPUT_DIR}
+ COMMAND "${CMAKE_COMMAND}" -E remove_directory ${BROXYGEN_SCRIPT_OUTPUT}
+ COMMAND "${CMAKE_COMMAND}" -E remove_directory ${BROXYGEN_CACHE_DIR}
+ VERBATIM)
add_custom_target(doc)
add_custom_target(docclean)
-add_dependencies(doc broxygen)
-add_dependencies(docclean broxygenclean restclean)
+add_dependencies(doc sphinxdoc)
+add_dependencies(docclean sphinxdoc_clean)
diff --git a/doc/README b/doc/README
index 0ba0a8587f..5104f79801 100644
--- a/doc/README
+++ b/doc/README
@@ -15,17 +15,16 @@ which adds some reST directives and roles that aid in generating useful
index entries and cross-references. Other extensions can be added in
a similar fashion.
-Either the ``make doc`` or ``make broxygen`` targets in the top-level
-Makefile can be used to locally render the reST files into HTML.
-Those targets depend on:
+The ``make doc`` target in the top-level Makefile can be used to locally
+render the reST files into HTML. That target depends on:
* Python interpreter >= 2.5
* `Sphinx `_ >= 1.0.1
After completion, HTML documentation is symlinked in ``build/html``.
-There's also ``make docclean`` and ``make broxygenclean`` targets to
-clean the resulting documentation.
+There's also a ``make docclean`` target which deletes any files
+created during the documentation build process.
Notes for Writing Documentation
-------------------------------
diff --git a/doc/_static/basic.css b/doc/_static/basic.css
index 1332c7b048..26e3450b65 100644
--- a/doc/_static/basic.css
+++ b/doc/_static/basic.css
@@ -439,8 +439,17 @@ td.linenos pre {
color: #aaa;
}
+.highlight-guess {
+ overflow:auto;
+}
+
+.highlight-none {
+ overflow:auto;
+}
+
table.highlighttable {
margin-left: 0.5em;
+ overflow:scroll;
}
table.highlighttable td {
diff --git a/doc/bin/group_index_generator.py b/doc/bin/group_index_generator.py
deleted file mode 100755
index b41dcb379c..0000000000
--- a/doc/bin/group_index_generator.py
+++ /dev/null
@@ -1,72 +0,0 @@
-#! /usr/bin/env python
-
-# This script automatically generates a reST documents that lists
-# a collection of Bro scripts that are "grouped" together.
-# The summary text (##! comments) of the script is embedded in the list
-#
-# 1st argument is the file containing list of groups
-# 2nd argument is the directory containing ${group}_files lists of
-# scripts that belong to the group and ${group}_doc_names lists of
-# document names that can be supplied to a reST :doc: role
-# 3rd argument is a directory in which write a ${group}.rst file that contains
-# reST style references to script docs along with summary text contained
-# in original script. If ${group} ends with "index", then the file
-# is always clobbered by this script, but for other unique group names,
-# this script will append to existing files.
-
-import sys
-import os
-import string
-
-group_list = sys.argv[1]
-file_manifest_dir = sys.argv[2]
-output_dir = sys.argv[3]
-
-def make_group_file_index(dir_name, group_name):
- group_file = os.path.join(dir_name, group_name + ".rst")
-
- if not os.path.exists(group_file):
- if not os.path.exists(os.path.dirname(group_file)):
- os.makedirs(os.path.dirname(group_file))
-
- if group_name.endswith("index"):
- with open(group_file, 'w') as f_group_file:
- f_group_file.write(":orphan:\n\n")
- title = "Package Index: %s\n" % os.path.dirname(group_name)
- f_group_file.write(title);
- for n in range(len(title)):
- f_group_file.write("=")
- f_group_file.write("\n");
-
- return group_file
-
-with open(group_list, 'r') as f_group_list:
- for group in f_group_list.read().splitlines():
- #print group
- group_file = make_group_file_index(output_dir, group)
- file_manifest = os.path.join(file_manifest_dir, group + "_files")
- doc_manifest = os.path.join(file_manifest_dir, group + "_doc_names")
- src_files = []
- doc_names = []
-
- with open(file_manifest, 'r') as f_file_manifest:
- src_files = f_file_manifest.read().splitlines()
-
- with open(doc_manifest, 'r') as f_doc_manifest:
- doc_names = f_doc_manifest.read().splitlines()
-
- for i in range(len(src_files)):
- src_file = src_files[i]
- #print "\t" + src_file
- summary_comments = []
- with open(src_file, 'r') as f_src_file:
- for line in f_src_file:
- sum_pos = string.find(line, "##!")
- if sum_pos != -1:
- summary_comments.append(line[(sum_pos+3):])
- #print summary_comments
-
- with open(group_file, 'a') as f_group_file:
- f_group_file.write("\n:doc:`/scripts/%s`\n" % doc_names[i])
- for line in summary_comments:
- f_group_file.write(" " + line)
diff --git a/doc/broids/index.rst b/doc/broids/index.rst
new file mode 100644
index 0000000000..46e0d6ded6
--- /dev/null
+++ b/doc/broids/index.rst
@@ -0,0 +1,80 @@
+
+.. _bro-ids:
+
+=======
+Bro IDS
+=======
+
+An Intrusion Detection System (IDS) allows you to detect suspicious
+activities happening on your network as a result of a past or active
+attack. Because of its programming capabilities, Bro can easily be
+configured to behave like traditional IDSs and detect common attacks
+with well known patterns, or you can create your own scripts to detect
+conditions specific to your particular case.
+
+In the following sections, we present a few examples of common uses of
+Bro as an IDS.
+
+------------------------------------------------
+Detecting an FTP Bruteforce attack and notifying
+------------------------------------------------
+
+For the purpose of this exercise, we define FTP bruteforcing as too many
+rejected usernames and passwords occurring from a single address. We
+start by defining a threshold for the number of attempts and a
+monitoring interval in minutes as well as a new notice type.
+
+.. btest-include:: ${BRO_SRC_ROOT}/scripts/policy/protocols/ftp/detect-bruteforcing.bro
+ :lines: 9-25
+
+Now, using the ftp_reply event, we check for error codes from the `500
+series `_
+for the "USER" and "PASS" commands, representing rejected usernames or
+passwords. For this, we can use the :bro:see:`FTP::parse_ftp_reply_code`
+function to break down the reply code and check if the first digit is a
+"5" or not. If true, we then use the :ref:`Summary Statistics Framework
+` to keep track of the number of failed attempts.
+
+.. btest-include:: ${BRO_SRC_ROOT}/scripts/policy/protocols/ftp/detect-bruteforcing.bro
+ :lines: 52-60
+
+Next, we use the SumStats framework to raise a notice of the attack of
+the attack when the number of failed attempts exceeds the specified
+threshold during the measuring interval.
+
+.. btest-include:: ${BRO_SRC_ROOT}/scripts/policy/protocols/ftp/detect-bruteforcing.bro
+ :lines: 28-50
+
+Below is the final code for our script.
+
+.. btest-include:: ${BRO_SRC_ROOT}/scripts/policy/protocols/ftp/detect-bruteforcing.bro
+
+.. btest:: ftp-bruteforce
+
+ @TEST-EXEC: btest-rst-cmd bro -r ${TRACES}/ftp/bruteforce.pcap protocols/ftp/detect-bruteforcing.bro
+ @TEST-EXEC: btest-rst-include notice.log
+
+As a final note, the :doc:`detect-bruteforcing.bro
+` script above is
+include with Bro out of the box, so you only need to load it at startup
+to instruct Bro to detect and notify of FTP bruteforce attacks.
+
+-------------
+Other Attacks
+-------------
+
+Detecting SQL Injection attacks
+-------------------------------
+
+Checking files against known malware hashes
+-------------------------------------------
+
+Files transmitted on your network could either be completely harmless or
+contain viruses and other threats. One possible action against this
+threat is to compute the hashes of the files and compare them against a
+list of known malware hashes. Bro simplifies this task by offering a
+:doc:`detect-MHR.bro `
+script that creates and compares hashes against the `Malware Hash
+Registry `_ maintained by Team
+Cymru. You only need to load this script along with your other scripts
+at startup time.
diff --git a/doc/broxygen.conf.in b/doc/broxygen.conf.in
new file mode 100644
index 0000000000..1e70445f58
--- /dev/null
+++ b/doc/broxygen.conf.in
@@ -0,0 +1 @@
+script * @BROXYGEN_SCRIPT_OUTPUT@/
diff --git a/doc/conf.py.in b/doc/conf.py.in
index 4cf64f10db..91e16452f3 100644
--- a/doc/conf.py.in
+++ b/doc/conf.py.in
@@ -17,7 +17,7 @@ extensions = []
# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
-sys.path.insert(0, os.path.abspath('sphinx-sources/ext'))
+sys.path.insert(0, os.path.abspath('sphinx_input/ext'))
# ----- Begin of BTest configuration. -----
btest = os.path.abspath("@CMAKE_SOURCE_DIR@/aux/btest")
@@ -33,6 +33,13 @@ btest_base="@CMAKE_SOURCE_DIR@/testing/btest"
btest_tests="doc/sphinx"
# ----- End of BTest configuration. -----
+# ----- Begin of Broxygen configuration. -----
+extensions += ["broxygen"]
+bro_binary = os.path.abspath("@CMAKE_SOURCE_DIR@/build/src/bro")
+broxygen_cache="@BROXYGEN_CACHE_DIR@"
+os.environ["BROPATH"] = "@BROPATH@"
+os.environ["BROMAGIC"] = "@BROMAGIC@"
+# ----- End of Broxygen configuration. -----
# -- General configuration -----------------------------------------------------
@@ -47,7 +54,7 @@ os.environ["BRO_SRC_ROOT"] = "@CMAKE_SOURCE_DIR@"
os.environ["DOC_ROOT"] = "@CMAKE_SOURCE_DIR@/doc"
# Add any paths that contain templates here, relative to this directory.
-templates_path = ['sphinx-sources/_templates', 'sphinx-sources/_static']
+templates_path = ['sphinx_input/_templates', 'sphinx_input/_static']
# The suffix of source filenames.
source_suffix = '.rst'
@@ -141,7 +148,7 @@ html_theme_options = { }
# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
-html_static_path = ['sphinx-sources/_static']
+html_static_path = ['sphinx_input/_static']
# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
diff --git a/doc/ext/bro.py b/doc/ext/bro.py
index 6ef11c37f6..9295c63312 100644
--- a/doc/ext/bro.py
+++ b/doc/ext/bro.py
@@ -191,6 +191,10 @@ class BroNotices(Index):
def generate(self, docnames=None):
content = {}
+
+ if 'notices' not in self.domain.env.domaindata['bro']:
+ return content, False
+
for n in self.domain.env.domaindata['bro']['notices']:
modname = n[0].split("::")[0]
entries = content.setdefault(modname, [])
diff --git a/doc/ext/broxygen.py b/doc/ext/broxygen.py
new file mode 100644
index 0000000000..b6b47bb82b
--- /dev/null
+++ b/doc/ext/broxygen.py
@@ -0,0 +1,317 @@
+"""
+Broxygen domain for Sphinx.
+
+Adds directives that allow Sphinx to invoke Bro in order to generate script
+reference documentation on the fly. The directives are:
+
+broxygen:package
+ - Shows links to all scripts contained within matching package(s).
+broxygen:package_index
+ - An index with links to matching package document(s).
+broxygen:script
+ - Reference for matching script(s) (i.e. everything declared by the script).
+broxygen:script_summary
+ - Shows link to matching script(s) with it's summary-section comments.
+broxygen:script_index
+ - An index with links to all matching scrips.
+broxygen:proto_analyzer
+ - All protocol analyzers and their components (events/bifs, etc.)
+broxygen:file_analyzer
+ - All file analyzers and their components (events/bifs, etc.)
+"""
+
+
+from sphinx.domains import Domain, ObjType
+from sphinx.locale import l_
+from docutils.parsers.rst.directives.misc import Include
+
+
+App = None
+
+
+def info(msg):
+ """Use Sphinx builder to output a console message."""
+ global App
+ from sphinx.util.console import blue
+ App.builder.info(blue(msg))
+
+
+def pattern_to_filename_component(pattern):
+ """Replace certain characters in Broxygen config file target pattern.
+
+ Such that it can be used as part of a (sane) filename.
+
+ """
+ return pattern.replace("/", ".").replace("*", "star")
+
+
+def ensure_dir(path):
+ """Should act like ``mkdir -p``."""
+ import os
+ import errno
+
+ try:
+ os.makedirs(path)
+ except OSError as e:
+ if e.errno != errno.EEXIST:
+ raise
+
+
+def generate_config(env, type, pattern):
+ """Create a Broxygen config file for a particular target.
+
+ It can be used by Bro to generate reST docs for that target.
+
+ """
+ import os
+ import tempfile
+ from sphinx.errors import SphinxError
+
+ work_dir = env.config.broxygen_cache
+
+ if not work_dir:
+ raise SphinxError("broxygen_cache not set in sphinx config file")
+
+ ensure_dir(work_dir)
+ prefix = "{0}-{1}-".format(type, pattern_to_filename_component(pattern))
+ (fd, cfg) = tempfile.mkstemp(suffix=".cfg", prefix=prefix, dir=work_dir)
+ generated_file = "{0}.rst".format(cfg)
+ config = "{0}\t{1}\t{2}".format(type, pattern, generated_file)
+ f = os.fdopen(fd, "w")
+ f.write(config)
+ f.close()
+ return (cfg, generated_file)
+
+
+def generate_target(env, type, pattern):
+ """Create a Broxygen target and build it.
+
+ For a target which hasn't been referenced by any other script, this function
+ creates an associated config file then uses Bro w/ it to build the target
+ and stores the target information in the build environment.
+
+ If a script references a target that's already found in the build
+ environment the results of the previous built are re-used.
+
+ """
+ app_data = env.domaindata["broxygen"]
+
+ if (type, pattern) in app_data["targets"]:
+ info("Broxygen has cached doc for target '{0} {1}'".format(
+ type, pattern))
+ return app_data["targets"]
+
+ (cfg, gend_file) = generate_config(env, type, pattern)
+ target = BroxygenTarget(type, pattern, cfg, gend_file)
+ app_data["targets"][(type, pattern)] = target
+ build_target(env, target)
+ info("Broxygen built target '{0} {1}'".format(type, pattern))
+ return target
+
+
+def build_target(env, target):
+ """Invoke a Bro process to build a Broxygen target."""
+ import os
+ import subprocess
+
+ path_to_bro = env.config.bro_binary
+
+ if not path_to_bro:
+ raise SphinxError("'bro' not set in sphinx config file (path to bro)")
+
+ bro_cmd = "{0} -X {1} broxygen".format(path_to_bro, target.config_file)
+ cwd = os.getcwd()
+ os.chdir(os.path.dirname(target.config_file))
+
+ try:
+ subprocess.check_output(bro_cmd, stderr=subprocess.STDOUT, shell=True)
+ except subprocess.CalledProcessError as e:
+ from sphinx.errors import SphinxError
+ raise SphinxError(
+ "Command '{0}' returned non-zero exit status {1}: {2}".format(
+ e.cmd, e.returncode, e.output))
+ finally:
+ os.chdir(cwd)
+
+
+class BroxygenTarget(object):
+
+ """Some portion of reST documentation that Bro knows how to generate.
+
+ A target is identified by its type and pattern. E.g. type "script" and
+ pattern "broxygen/example.bro".
+
+ """
+
+ def __init__(self, type, pattern, config_file, generated_file):
+ self.type = type
+ self.pattern = pattern
+ self.config_file = config_file
+ self.generated_file = generated_file
+ self.used_in_docs = set()
+
+
+class BroxygenDirective(Include):
+
+ """Base class for Broxygen directives.
+
+ It can use Bro to generate reST documentation on the fly and embed it in
+ the document at the location of the directive just like the ``.. include::``
+ directive. The only argument is a pattern to identify to Bro which
+ pieces of documentation it needs to create.
+ """
+
+ required_arguments = 1
+ has_content = False
+
+ target_type = None
+
+ def run(self):
+ env = self.state.document.settings.env
+ info("Broxygen running .. {0}:: {1} in {2}".format(
+ self.name, self.arguments[0], env.docname))
+ target = generate_target(env, self.target_type, self.arguments[0])
+ target.used_in_docs.add(env.docname)
+ self.arguments = [target.generated_file]
+ return super(BroxygenDirective, self).run()
+
+
+class PackageDirective(BroxygenDirective):
+
+ target_type = "package"
+
+
+class PackageIndexDirective(BroxygenDirective):
+
+ target_type = "package_index"
+
+
+class ScriptDirective(BroxygenDirective):
+
+ target_type = "script"
+
+
+class ScriptSummaryDirective(BroxygenDirective):
+
+ target_type = "script_summary"
+
+
+class ScriptIndexDirective(BroxygenDirective):
+
+ target_type = "script_index"
+
+
+class ProtoAnalyzerDirective(BroxygenDirective):
+
+ target_type = "proto_analyzer"
+
+
+class FileAnalyzerDirective(BroxygenDirective):
+
+ target_type = "file_analyzer"
+
+
+class IdentifierDirective(BroxygenDirective):
+
+ target_type = "identifier"
+
+
+class BroxygenDomain(Domain):
+
+ name = "broxygen"
+ label = "Broxygen"
+
+ object_types = {
+ "package": ObjType(l_("package")),
+ "package_index": ObjType(l_("package_index")),
+ "script": ObjType(l_("script")),
+ "script_summary": ObjType(l_("script_summary")),
+ "script_index": ObjType(l_("script_index")),
+ "proto_analyzer": ObjType(l_("proto_analyzer")),
+ "file_analyzer": ObjType(l_("file_analyzer")),
+ "identifier": ObjType(l_("identifier")),
+ }
+
+ directives = {
+ "package": PackageDirective,
+ "package_index": PackageIndexDirective,
+ "script": ScriptDirective,
+ "script_summary": ScriptSummaryDirective,
+ "script_index": ScriptIndexDirective,
+ "proto_analyzer": ProtoAnalyzerDirective,
+ "file_analyzer": FileAnalyzerDirective,
+ "identifier": IdentifierDirective,
+ }
+
+ roles = {}
+
+ initial_data = {
+ "targets": {}
+ }
+
+ def clear_doc(self, docname):
+ """Update Broxygen targets referenced in docname.
+
+ If it's the last place the target was referenced, remove it from
+ the build environment and delete any generated config/reST files
+ associated with it from the cache.
+
+ """
+ import os
+
+ stale_targets = []
+
+ for (type, pattern), target in self.data["targets"].items():
+ if docname in target.used_in_docs:
+ target.used_in_docs.remove(docname)
+
+ if not target.used_in_docs:
+ stale_targets.append(target)
+
+ for target in stale_targets:
+ del self.data["targets"][(target.type, target.pattern)]
+ os.remove(target.config_file)
+ os.remove(target.generated_file)
+
+ def get_objects(self):
+ """No Broxygen-generated content is itself linkable/searchable."""
+ return []
+
+
+def env_get_outdated_hook(app, env, added, changed, removed):
+ """Check whether to re-read any documents referencing Broxygen targets.
+
+ To do that we have to ask Bro to rebuild each target and compare the
+ before and after modification times of the generated reST output file.
+ If Bro changed it, then the document containing the Broxygen directive
+ needs to be re-read.
+
+ """
+ import os
+
+ reread = set()
+
+ for target in app.env.domaindata["broxygen"]["targets"].values():
+ before_mtime = os.stat(target.generated_file)
+ build_target(env, target)
+ after_mtime = os.stat(target.generated_file)
+
+ if after_mtime > before_mtime:
+ info("Broxygen target '{0} {1}' outdated".format(
+ target.type, target.pattern))
+
+ for docname in target.used_in_docs:
+ if docname not in removed:
+ info(" in document: {0}".format(docname))
+ reread.add(docname)
+
+ return list(reread)
+
+
+def setup(app):
+ global App
+ App = app
+ app.add_domain(BroxygenDomain)
+ app.add_config_value("bro_binary", None, "env")
+ app.add_config_value("broxygen_cache", None, "env")
+ app.connect("env-get-outdated", env_get_outdated_hook)
diff --git a/doc/frameworks/file-analysis.rst b/doc/frameworks/file-analysis.rst
index 50fb89317c..e70b124af7 100644
--- a/doc/frameworks/file-analysis.rst
+++ b/doc/frameworks/file-analysis.rst
@@ -1,3 +1,6 @@
+
+.. _file-analysis-framework:
+
=============
File Analysis
=============
diff --git a/doc/frameworks/logging-dataseries.rst b/doc/frameworks/logging-dataseries.rst
index 667edcbe14..cc479eae76 100644
--- a/doc/frameworks/logging-dataseries.rst
+++ b/doc/frameworks/logging-dataseries.rst
@@ -104,7 +104,7 @@ code like this to your ``local.bro``:
}
Bro's DataSeries writer comes with a few tuning options, see
-:doc:`/scripts/base/frameworks/logging/writers/dataseries`.
+:doc:`/scripts/base/frameworks/logging/writers/dataseries.bro`.
Working with DataSeries
=======================
diff --git a/doc/frameworks/logging.rst b/doc/frameworks/logging.rst
index eb64df4ec3..47d3338e8a 100644
--- a/doc/frameworks/logging.rst
+++ b/doc/frameworks/logging.rst
@@ -48,7 +48,7 @@ Basics
The data fields that a stream records are defined by a record type
specified when it is created. Let's look at the script generating Bro's
connection summaries as an example,
-:doc:`/scripts/base/protocols/conn/main`. It defines a record
+:doc:`/scripts/base/protocols/conn/main.bro`. It defines a record
:bro:type:`Conn::Info` that lists all the fields that go into
``conn.log``, each marked with a ``&log`` attribute indicating that it
is part of the information written out. To write a log record, the
@@ -309,7 +309,7 @@ ASCII Writer Configuration
--------------------------
The ASCII writer has a number of options for customizing the format of
-its output, see :doc:`/scripts/base/frameworks/logging/writers/ascii`.
+its output, see :doc:`/scripts/base/frameworks/logging/writers/ascii.bro`.
Adding Streams
==============
@@ -369,7 +369,7 @@ save the logged ``Foo::Info`` record into the connection record:
}
See the existing scripts for how to work with such a new connection
-field. A simple example is :doc:`/scripts/base/protocols/syslog/main`.
+field. A simple example is :doc:`/scripts/base/protocols/syslog/main.bro`.
When you are developing scripts that add data to the :bro:type:`connection`
record, care must be given to when and how long data is stored.
diff --git a/doc/frameworks/notice.rst b/doc/frameworks/notice.rst
index 1f0cc9da11..2c20149ce5 100644
--- a/doc/frameworks/notice.rst
+++ b/doc/frameworks/notice.rst
@@ -1,4 +1,6 @@
+.. _notice-framework:
+
Notice Framework
================
@@ -283,7 +285,7 @@ information to suppress duplicates for a configurable period of time.
The ``$identifier`` field is typically comprised of several pieces of
data related to the notice that when combined represent a unique
instance of that notice. Here is an example of the script
-:doc:`/scripts/policy/protocols/ssl/validate-certs` raising a notice
+:doc:`/scripts/policy/protocols/ssl/validate-certs.bro` raising a notice
for session negotiations where the certificate or certificate chain did
not validate successfully against the available certificate authority
certificates.
diff --git a/doc/frameworks/signatures.rst b/doc/frameworks/signatures.rst
index 915133e178..884dcb8a47 100644
--- a/doc/frameworks/signatures.rst
+++ b/doc/frameworks/signatures.rst
@@ -46,7 +46,7 @@ signature's event statement (``Found root!``), and data is the last
piece of payload which triggered the pattern match.
To turn such :bro:id:`signature_match` events into actual alarms, you can
-load Bro's :doc:`/scripts/base/frameworks/signatures/main` script.
+load Bro's :doc:`/scripts/base/frameworks/signatures/main.bro` script.
This script contains a default event handler that raises
:bro:enum:`Signatures::Sensitive_Signature` :doc:`Notices `
(as well as others; see the beginning of the script).
diff --git a/doc/frameworks/sumstats.rst b/doc/frameworks/sumstats.rst
index 057cc87fc1..aaed35be29 100644
--- a/doc/frameworks/sumstats.rst
+++ b/doc/frameworks/sumstats.rst
@@ -1,3 +1,6 @@
+
+.. _sumstats-framework:
+
==================
Summary Statistics
==================
@@ -87,7 +90,7 @@ Taking the previous example even further, we can implement a simple detection
to demonstrate the thresholding functionality. This example is a toy to
demonstrate how thresholding works in Sumstats and is not meant to be a
real-world functional example, that is left to the
-:doc:`/scripts/policy/misc/scan` script that is included with Bro.
+:doc:`/scripts/policy/misc/scan.bro` script that is included with Bro.
.. btest-include:: ${DOC_ROOT}/frameworks/sumstats-toy-scan.bro
diff --git a/doc/httpmonitor/file_extraction.bro b/doc/httpmonitor/file_extraction.bro
new file mode 100644
index 0000000000..b2318c595e
--- /dev/null
+++ b/doc/httpmonitor/file_extraction.bro
@@ -0,0 +1,24 @@
+
+global mime_to_ext: table[string] of string = {
+ ["application/x-dosexec"] = "exe",
+ ["text/plain"] = "txt",
+ ["image/jpeg"] = "jpg",
+ ["image/png"] = "png",
+ ["text/html"] = "html",
+};
+
+event file_new(f: fa_file)
+ {
+ if ( f$source != "HTTP" )
+ return;
+
+ if ( ! f?$mime_type )
+ return;
+
+ if ( f$mime_type !in mime_to_ext )
+ return;
+
+ local fname = fmt("%s-%s.%s", f$source, f$id, mime_to_ext[f$mime_type]);
+ print fmt("Extracting file %s", fname);
+ Files::add_analyzer(f, Files::ANALYZER_EXTRACT, [$extract_filename=fname]);
+ }
diff --git a/doc/httpmonitor/http_proxy_01.bro b/doc/httpmonitor/http_proxy_01.bro
new file mode 100644
index 0000000000..76555b6646
--- /dev/null
+++ b/doc/httpmonitor/http_proxy_01.bro
@@ -0,0 +1,5 @@
+event http_reply(c: connection, version: string, code: count, reason: string)
+ {
+ if ( /^[hH][tT][tT][pP]:/ in c$http$uri && c$http$status_code == 200 )
+ print fmt("A local server is acting as an open proxy: %s", c$id$resp_h);
+ }
diff --git a/doc/httpmonitor/http_proxy_02.bro b/doc/httpmonitor/http_proxy_02.bro
new file mode 100644
index 0000000000..cdbd722619
--- /dev/null
+++ b/doc/httpmonitor/http_proxy_02.bro
@@ -0,0 +1,26 @@
+
+module HTTP;
+
+export {
+
+ global success_status_codes: set[count] = {
+ 200,
+ 201,
+ 202,
+ 203,
+ 204,
+ 205,
+ 206,
+ 207,
+ 208,
+ 226,
+ 304
+ };
+}
+
+event http_reply(c: connection, version: string, code: count, reason: string)
+ {
+ if ( /^[hH][tT][tT][pP]:/ in c$http$uri &&
+ c$http$status_code in HTTP::success_status_codes )
+ print fmt("A local server is acting as an open proxy: %s", c$id$resp_h);
+ }
diff --git a/doc/httpmonitor/http_proxy_03.bro b/doc/httpmonitor/http_proxy_03.bro
new file mode 100644
index 0000000000..17bfdcb95b
--- /dev/null
+++ b/doc/httpmonitor/http_proxy_03.bro
@@ -0,0 +1,31 @@
+
+@load base/utils/site
+
+redef Site::local_nets += { 192.168.0.0/16 };
+
+module HTTP;
+
+export {
+
+ global success_status_codes: set[count] = {
+ 200,
+ 201,
+ 202,
+ 203,
+ 204,
+ 205,
+ 206,
+ 207,
+ 208,
+ 226,
+ 304
+ };
+}
+
+event http_reply(c: connection, version: string, code: count, reason: string)
+ {
+ if ( Site::is_local_addr(c$id$resp_h) &&
+ /^[hH][tT][tT][pP]:/ in c$http$uri &&
+ c$http$status_code in HTTP::success_status_codes )
+ print fmt("A local server is acting as an open proxy: %s", c$id$resp_h);
+ }
diff --git a/doc/httpmonitor/http_proxy_04.bro b/doc/httpmonitor/http_proxy_04.bro
new file mode 100644
index 0000000000..1f11be3670
--- /dev/null
+++ b/doc/httpmonitor/http_proxy_04.bro
@@ -0,0 +1,40 @@
+@load base/utils/site
+@load base/frameworks/notice
+
+redef Site::local_nets += { 192.168.0.0/16 };
+
+module HTTP;
+
+export {
+
+ redef enum Notice::Type += {
+ Open_Proxy
+ };
+
+ global success_status_codes: set[count] = {
+ 200,
+ 201,
+ 202,
+ 203,
+ 204,
+ 205,
+ 206,
+ 207,
+ 208,
+ 226,
+ 304
+ };
+}
+
+event http_reply(c: connection, version: string, code: count, reason: string)
+ {
+ if ( Site::is_local_addr(c$id$resp_h) &&
+ /^[hH][tT][tT][pP]:/ in c$http$uri &&
+ c$http$status_code in HTTP::success_status_codes )
+ NOTICE([$note=HTTP::Open_Proxy,
+ $msg=fmt("A local server is acting as an open proxy: %s",
+ c$id$resp_h),
+ $conn=c,
+ $identifier=cat(c$id$resp_h),
+ $suppress_for=1day]);
+ }
diff --git a/doc/httpmonitor/index.rst b/doc/httpmonitor/index.rst
new file mode 100644
index 0000000000..f6b2f5e122
--- /dev/null
+++ b/doc/httpmonitor/index.rst
@@ -0,0 +1,163 @@
+
+.. _http-monitor:
+
+================================
+Monitoring HTTP Traffic with Bro
+================================
+
+Bro can be used to log the entire HTTP traffic from your network to the
+http.log file. This file can then be used for analysis and auditing
+purposes.
+
+In the sections below we briefly explain the structure of the http.log
+file. Then, we show you how to perform basic HTTP traffic monitoring and
+analysis tasks with Bro. Some of these ideas and techniques can later be
+applied to monitor different protocols in a similar way.
+
+----------------------------
+Introduction to the HTTP log
+----------------------------
+
+The http.log file contains a summary of all HTTP requests and responses
+sent over a Bro-monitored network. Here are the first few columns of
+``http.log``::
+
+ # ts uid orig_h orig_p resp_h resp_p
+ 1311627961.8 HSH4uV8KVJg 192.168.1.100 52303 192.150.187.43 80
+
+Every single line in this log starts with a timestamp, a unique
+connection identifier (UID), and a connection 4-tuple (originator
+host/port and responder host/port). The UID can be used to identify all
+logged activity (possibly across multiple log files) associated with a
+given connection 4-tuple over its lifetime.
+
+The remaining columns detail the activity that's occurring. For
+example, the columns on the line below (shortened for brevity) show a
+request to the root of Bro website::
+
+ # method host uri referrer user_agent
+ GET bro.org / - <...>Chrome/12.0.742.122<...>
+
+Network administrators and security engineers, for instance, can use the
+information in this log to understand the HTTP activity on the network
+and troubleshoot network problems or search for anomalous activities. At
+this point, we would like to stress out the fact that there is no just
+one right way to perform analysis; it will depend on the expertise of
+the person doing the analysis and the specific details of the task to
+accomplish.
+
+For more information about how to handle the HTTP protocol in Bro,
+including a complete list of the fields available in http.log, go to
+Bro's :doc:`HTTP script reference
+`.
+
+------------------------
+Detecting a Proxy Server
+------------------------
+
+A proxy server is a device on your network configured to request a
+service on behalf of a third system; one of the most common examples is
+a Web proxy server. A client without Internet access connects to the
+proxy and requests a Web page; the proxy then sends the request to the
+actual Web server, receives the response and passes it to the original
+client.
+
+Proxies were conceived to help manage a network and provide better
+encapsulation. By themselves, proxies are not a security threat, but a
+misconfigured or unauthorized proxy can allow others, either inside or
+outside the network, to access any Web site and even conduct malicious
+activities anonymously using the network resources.
+
+What Proxy Server traffic looks like
+-------------------------------------
+
+In general, when a client starts talking with a proxy server, the
+traffic consists of two parts: (i) a GET request, and (ii) an HTTP/
+reply::
+
+ Request: GET http://www.bro.org/ HTTP/1.1
+ Reply: HTTP/1.0 200 OK
+
+This will differ from traffic between a client and a normal Web server
+because GET requests should not include "http" on the string. So we can
+use this to identify a proxy server.
+
+We can write a basic script in Bro to handle the http_reply event and
+detect a reply for a ``GET http://`` request.
+
+.. btest-include:: ${DOC_ROOT}/httpmonitor/http_proxy_01.bro
+
+.. btest:: http_proxy_01
+
+ @TEST-EXEC: btest-rst-cmd bro -r ${TRACES}/http/proxy.pcap ${DOC_ROOT}/httpmonitor/http_proxy_01.bro
+
+Basically, the script is checking for a "200 OK" status code on a reply
+for a request that includes "http:" (case insensitive). In reality, the
+HTTP protocol defines several success status codes other than 200, so we
+will extend our basic script to also consider the additional codes.
+
+.. btest-include:: ${DOC_ROOT}/httpmonitor/http_proxy_02.bro
+
+.. btest:: http_proxy_02
+
+ @TEST-EXEC: btest-rst-cmd bro -r ${TRACES}/http/proxy.pcap ${DOC_ROOT}/httpmonitor/http_proxy_02.bro
+
+Next, we will make sure that the responding proxy is part of our local
+network.
+
+.. btest-include:: ${DOC_ROOT}/httpmonitor/http_proxy_03.bro
+
+.. btest:: http_proxy_03
+
+ @TEST-EXEC: btest-rst-cmd bro -r ${TRACES}/http/proxy.pcap ${DOC_ROOT}/httpmonitor/http_proxy_03.bro
+
+.. note::
+
+ The redefinition of :bro:see:`Site::local_nets` is only done inside
+ this script to make it a self-contained example. It's typically
+ redefined somewhere else.
+
+Finally, our goal should be to generate an alert when a proxy has been
+detected instead of printing a message on the console output. For that,
+we will tag the traffic accordingly and define a new ``Open_Proxy``
+``Notice`` type to alert of all tagged communications. Once a
+notification has been fired, we will further suppress it for one day.
+Below is the complete script.
+
+.. btest-include:: ${DOC_ROOT}/httpmonitor/http_proxy_04.bro
+
+.. btest:: http_proxy_04
+
+ @TEST-EXEC: btest-rst-cmd bro -r ${TRACES}/http/proxy.pcap ${DOC_ROOT}/httpmonitor/http_proxy_04.bro
+ @TEST-EXEC: btest-rst-include notice.log
+
+Note that this script only logs the presence of the proxy to
+``notice.log``, but if an additional email is desired (and email
+functionality is enabled), then that's done simply by redefining
+:bro:see:`Notice::emailed_types` to add the ``Open_proxy`` notice type
+to it.
+
+----------------
+Inspecting Files
+----------------
+
+Files are often transmitted on regular HTTP conversations between a
+client and a server. Most of the time these files are harmless, just
+images and some other multimedia content, but there are also types of
+files, specially executable files, that can damage your system. We can
+instruct Bro to create a copy of all files of certain types that it sees
+using the :ref:`File Analysis Framework `
+(introduced with Bro 2.2):
+
+.. btest-include:: ${DOC_ROOT}/httpmonitor/file_extraction.bro
+
+.. btest:: file_extraction
+
+ @TEST-EXEC: btest-rst-cmd -n 5 bro -r ${TRACES}/http/bro.org.pcap ${DOC_ROOT}/httpmonitor/file_extraction.bro
+
+Here, the ``mime_to_ext`` table serves two purposes. It defines which
+mime types to extract and also the file suffix of the extracted files.
+Extracted files are written to a new ``extract_files`` subdirectory.
+Also note that the first conditional in the :bro:see:`file_new` event
+handler can be removed to make this behavior generic to other protocols
+besides HTTP.
diff --git a/doc/index.rst b/doc/index.rst
index b79f1b67a6..3da35e7b7a 100644
--- a/doc/index.rst
+++ b/doc/index.rst
@@ -1,9 +1,12 @@
.. Bro documentation master file
-=================
-Bro Documentation
-=================
+==========
+Bro Manual
+==========
+
+Introduction Section
+====================
.. toctree::
:maxdepth: 2
@@ -11,13 +14,36 @@ Bro Documentation
intro/index.rst
install/index.rst
quickstart/index.rst
- using/index.rst
+
+..
+
+Using Bro Section
+=================
+
+.. toctree::
+ :maxdepth: 2
+
+ logs/index.rst
+ httpmonitor/index.rst
+ broids/index.rst
+ mimestats/index.rst
+ cluster/index.rst
+
+..
+
+Reference Section
+=================
+
+.. toctree::
+ :maxdepth: 2
+
scripting/index.rst
frameworks/index.rst
- cluster/index.rst
- scripts/index.rst
+ script-reference/index.rst
components/index.rst
+..
+
* :ref:`General Index `
* :ref:`search`
diff --git a/doc/install/install.rst b/doc/install/install.rst
index 7532a82cea..3678d948c2 100644
--- a/doc/install/install.rst
+++ b/doc/install/install.rst
@@ -29,14 +29,13 @@ before you begin:
* Libpcap (http://www.tcpdump.org)
* OpenSSL libraries (http://www.openssl.org)
* BIND8 library
- * Libmagic 5.04 or greater
* Libz
* Bash (for BroControl)
* Python (for BroControl)
To build Bro from source, the following additional dependencies are required:
- * CMake 2.6.3 or greater (http://www.cmake.org)
+ * CMake 2.8.0 or greater (http://www.cmake.org)
* Make
* C/C++ compiler
* SWIG (http://www.swig.org)
@@ -44,7 +43,6 @@ To build Bro from source, the following additional dependencies are required:
* Flex (Fast Lexical Analyzer)
* Libpcap headers (http://www.tcpdump.org)
* OpenSSL headers (http://www.openssl.org)
- * libmagic headers
* zlib headers
* Perl
@@ -55,13 +53,13 @@ that ``bash`` and ``python`` are in your ``PATH``):
.. console::
- sudo yum install cmake make gcc gcc-c++ flex bison libpcap-devel openssl-devel python-devel swig zlib-devel file-devel
+ sudo yum install cmake make gcc gcc-c++ flex bison libpcap-devel openssl-devel python-devel swig zlib-devel
* DEB/Debian-based Linux:
.. console::
- sudo apt-get install cmake make gcc g++ flex bison libpcap-dev libssl-dev python-dev swig zlib1g-dev libmagic-dev
+ sudo apt-get install cmake make gcc g++ flex bison libpcap-dev libssl-dev python-dev swig zlib1g-dev
* FreeBSD:
@@ -78,15 +76,11 @@ that ``bash`` and ``python`` are in your ``PATH``):
then going through its "Preferences..." -> "Downloads" menus to
install the "Command Line Tools" component.
- Lion (10.7) and Mountain Lion (10.8) come with all required
- dependencies except for CMake_, SWIG_, and ``libmagic``.
-
+ OS X comes with all required dependencies except for CMake_ and SWIG_.
Distributions of these dependencies can likely be obtained from your
preferred Mac OS X package management system (e.g. MacPorts_, Fink_,
- or Homebrew_).
-
- Specifically for MacPorts, the ``cmake``, ``swig``,
- ``swig-python`` and ``file`` packages provide the required dependencies.
+ or Homebrew_). Specifically for MacPorts, the ``cmake``, ``swig``,
+ ``swig-python`` and packages provide the required dependencies.
Optional Dependencies
diff --git a/doc/using/index.rst b/doc/logs/index.rst
similarity index 78%
rename from doc/using/index.rst
rename to doc/logs/index.rst
index 1ad05d74f8..b71546db72 100644
--- a/doc/using/index.rst
+++ b/doc/logs/index.rst
@@ -1,9 +1,9 @@
-.. _using-bro:
+.. _bro-logging:
-=========
-Using Bro
-=========
+===========
+Bro Logging
+===========
.. contents::
@@ -251,3 +251,42 @@ stream and Bro is able to extract and track that information for you,
giving you an in-depth and structured view into HTTP traffic on your
network.
+-----------------------
+Common Log Files
+-----------------------
+As a monitoring tool, Bro records a detailed view of the traffic inspected and the events generated in
+a series of relevant log files. These files can later be reviewed for monitoring, auditing and troubleshooting
+purposes.
+
+In this section we present a brief explanation of the most commonly used log files generated by Bro including links
+to descriptions of some of the fields for each log type.
+
++-----------------+---------------------------------------+------------------------------+
+| Log File | Description | Field Descriptions |
++=================+=======================================+==============================+
+| http.log | Shows all HTTP requests and replies | :bro:type:`HTTP::Info` |
++-----------------+---------------------------------------+------------------------------+
+| ftp.log | Records FTP activity | :bro:type:`FTP::Info` |
++-----------------+---------------------------------------+------------------------------+
+| ssl.log | Records SSL sessions including | :bro:type:`SSL::Info` |
+| | certificates used | |
++-----------------+---------------------------------------+------------------------------+
+| known_certs.log | Includes SSL certificates used | :bro:type:`Known::CertsInfo` |
++-----------------+---------------------------------------+------------------------------+
+| smtp.log | Summarizes SMTP traffic on a network | :bro:type:`SMTP::Info` |
++-----------------+---------------------------------------+------------------------------+
+| dns.log | Shows all DNS activity on a network | :bro:type:`DNS::Info` |
++-----------------+---------------------------------------+------------------------------+
+| conn.log | Records all connections seen by Bro | :bro:type:`Conn::Info` |
++-----------------+---------------------------------------+------------------------------+
+| dpd.log | Shows network activity on | :bro:type:`DPD::Info` |
+| | non-standard ports | |
++-----------------+---------------------------------------+------------------------------+
+| files.log | Records information about all files | :bro:type:`Files::Info` |
+| | transmitted over the network | |
++-----------------+---------------------------------------+------------------------------+
+| weird.log | Records unexpected protocol-level | :bro:type:`Weird::Info` |
+| | activity | |
++-----------------+---------------------------------------+------------------------------+
+
+
diff --git a/doc/mimestats/index.rst b/doc/mimestats/index.rst
new file mode 100644
index 0000000000..df17f4872f
--- /dev/null
+++ b/doc/mimestats/index.rst
@@ -0,0 +1,71 @@
+
+.. _mime-stats:
+
+====================
+MIME Type Statistics
+====================
+
+Files are constantly transmitted over HTTP on regular networks. These
+files belong to a specific category (i.e., executable, text, image,
+etc.) identified by a `Multipurpose Internet Mail Extension (MIME)
+`_. Although MIME was originally
+developed to identify the type of non-text attachments on email, it is
+also used by Web browser to identify the type of files transmitted and
+present them accordingly.
+
+In this tutorial, we will show how to use the Sumstats Framework to
+collect some statistics information based on MIME types, specifically
+the total number of occurrences, size in bytes, and number of unique
+hosts transmitting files over HTTP per each type. For instructions about
+extracting and creating a local copy of these files, visit :ref:`this
+` tutorial instead.
+
+------------------------------------------------
+MIME Statistics with Sumstats
+------------------------------------------------
+
+When working with the :ref:`Summary Statistics Framework
+`, you need to define three different pieces: (i)
+Observations, where the event is observed and fed into the framework.
+(ii) Reducers, where observations are collected and measured. (iii)
+Sumstats, where the main functionality is implemented.
+
+So, we start by defining our observation along with a record to store
+all statistics values and an observation interval. We are conducting our
+observation on the :bro:see:`HTTP::log_http` event and we are interested
+in the MIME type, size of the file ("response_body_len") and the
+originator host ("orig_h"). We use the MIME type as our key and create
+observers for the other two values.
+
+.. btest-include:: ${DOC_ROOT}/mimestats/mimestats.bro
+ :lines: 6-29, 54-64
+
+Next, we create the reducers. The first one will accumulate file sizes
+and the second one will make sure we only store a host ID once. Below is
+the partial code from a :bro:see:`bro_init` handler.
+
+.. btest-include:: ${DOC_ROOT}/mimestats/mimestats.bro
+ :lines: 34-37
+
+In our final step, we create the SumStats where we check for the
+observation interval and once it expires, we populate the record
+(defined above) with all the relevant data and write it to a log.
+
+.. btest-include:: ${DOC_ROOT}/mimestats/mimestats.bro
+ :lines: 38-51
+
+Putting everything together we end up with the following final code for
+our script.
+
+.. btest-include:: ${DOC_ROOT}/mimestats/mimestats.bro
+
+.. btest:: mimestats
+
+ @TEST-EXEC: btest-rst-cmd bro -r ${TRACES}/http/bro.org.pcap ${DOC_ROOT}/mimestats/mimestats.bro
+ @TEST-EXEC: btest-rst-include mime_metrics.log
+
+.. note::
+
+ The redefinition of :bro:see:`Site::local_nets` is only done inside
+ this script to make it a self-contained example. It's typically
+ redefined somewhere else.
diff --git a/doc/mimestats/mimestats.bro b/doc/mimestats/mimestats.bro
new file mode 100644
index 0000000000..b854b26c2d
--- /dev/null
+++ b/doc/mimestats/mimestats.bro
@@ -0,0 +1,64 @@
+@load base/utils/site
+@load base/frameworks/sumstats
+
+redef Site::local_nets += { 10.0.0.0/8 };
+
+module MimeMetrics;
+
+export {
+
+ redef enum Log::ID += { LOG };
+
+ type Info: record {
+ ## Timestamp when the log line was finished and written.
+ ts: time &log;
+ ## Time interval that the log line covers.
+ ts_delta: interval &log;
+ ## The mime type
+ mtype: string &log;
+ ## The number of unique local hosts that fetched this mime type
+ uniq_hosts: count &log;
+ ## The number of hits to the mime type
+ hits: count &log;
+ ## The total number of bytes received by this mime type
+ bytes: count &log;
+ };
+
+ ## The frequency of logging the stats collected by this script.
+ const break_interval = 5mins &redef;
+}
+
+event bro_init() &priority=3
+ {
+ Log::create_stream(MimeMetrics::LOG, [$columns=Info]);
+ local r1: SumStats::Reducer = [$stream="mime.bytes",
+ $apply=set(SumStats::SUM)];
+ local r2: SumStats::Reducer = [$stream="mime.hits",
+ $apply=set(SumStats::UNIQUE)];
+ SumStats::create([$name="mime-metrics",
+ $epoch=break_interval,
+ $reducers=set(r1, r2),
+ $epoch_result(ts: time, key: SumStats::Key, result: SumStats::Result) =
+ {
+ local l: Info;
+ l$ts = network_time();
+ l$ts_delta = break_interval;
+ l$mtype = key$str;
+ l$bytes = double_to_count(floor(result["mime.bytes"]$sum));
+ l$hits = result["mime.hits"]$num;
+ l$uniq_hosts = result["mime.hits"]$unique;
+ Log::write(MimeMetrics::LOG, l);
+ }]);
+ }
+
+event HTTP::log_http(rec: HTTP::Info)
+ {
+ if ( Site::is_local_addr(rec$id$orig_h) && rec?$resp_mime_types )
+ {
+ local mime_type = rec$resp_mime_types[0];
+ SumStats::observe("mime.bytes", [$str=mime_type],
+ [$num=rec$response_body_len]);
+ SumStats::observe("mime.hits", [$str=mime_type],
+ [$str=cat(rec$id$orig_h)]);
+ }
+ }
diff --git a/doc/quickstart/index.rst b/doc/quickstart/index.rst
index f570ac92db..49a909a37f 100644
--- a/doc/quickstart/index.rst
+++ b/doc/quickstart/index.rst
@@ -13,7 +13,7 @@ Bro works on most modern, Unix-based systems and requires no custom
hardware. It can be downloaded in either pre-built binary package or
source code forms. See :ref:`installing-bro` for instructions on how to
install Bro. Below, ``$PREFIX`` is used to reference the Bro
-installation root directory, which by default is ``/usr/local/`` if
+installation root directory, which by default is ``/usr/local/bro/`` if
you install from source.
Managing Bro with BroControl
@@ -26,8 +26,8 @@ traffic-monitoring cluster.
A Minimal Starting Configuration
--------------------------------
-These are the basic configuration changes to make for a minimal BroControl installation
-that will manage a single Bro instance on the ``localhost``:
+These are the basic configuration changes to make for a minimal BroControl
+installation that will manage a single Bro instance on the ``localhost``:
1) In ``$PREFIX/etc/node.cfg``, set the right interface to monitor.
2) In ``$PREFIX/etc/networks.cfg``, comment out the default settings and add
@@ -72,7 +72,8 @@ You can leave it running for now, but to stop this Bro instance you would do:
[BroControl] > stop
-We also recommend to insert the following entry into `crontab`::
+We also recommend to insert the following entry into the crontab of the user
+running BroControl::
0-59/5 * * * * $PREFIX/bin/broctl cron
@@ -197,7 +198,7 @@ the variable's value may not change at run-time, but whose initial value can be
modified via the ``redef`` operator at parse-time.
So let's continue on our path to modify the behavior for the two SSL
-and SSH notices. Looking at :doc:`/scripts/base/frameworks/notice/main`,
+and SSH notices. Looking at :doc:`/scripts/base/frameworks/notice/main.bro`,
we see that it advertises:
.. code:: bro
diff --git a/doc/scripts/builtins.rst b/doc/script-reference/builtins.rst
similarity index 95%
rename from doc/scripts/builtins.rst
rename to doc/script-reference/builtins.rst
index aa1a097683..85e9cd14c8 100644
--- a/doc/scripts/builtins.rst
+++ b/doc/script-reference/builtins.rst
@@ -23,7 +23,8 @@ The Bro scripting language supports the following built-in types.
.. bro:type:: void
- An internal Bro type representing the absence of a return type for a
+ An internal Bro type (i.e., "void" is not a reserved keyword in the Bro
+ scripting language) representing the absence of a return type for a
function.
.. bro:type:: bool
@@ -132,10 +133,23 @@ The Bro scripting language supports the following built-in types.
Strings support concatenation (``+``), and assignment (``=``, ``+=``).
Strings also support the comparison operators (``==``, ``!=``, ``<``,
- ``<=``, ``>``, ``>=``). Substring searching can be performed using
- the "in" or "!in" operators (e.g., "bar" in "foobar" yields true).
- The number of characters in a string can be found by enclosing the
- string within pipe characters (e.g., ``|"abc"|`` is 3).
+ ``<=``, ``>``, ``>=``). The number of characters in a string can be
+ found by enclosing the string within pipe characters (e.g., ``|"abc"|``
+ is 3).
+
+ The subscript operator can extract an individual character or a substring
+ of a string (string indexing is zero-based, but an index of
+ -1 refers to the last character in the string, and -2 refers to the
+ second-to-last character, etc.). When extracting a substring, the
+ starting and ending index values are separated by a colon. For example::
+
+ local orig = "0123456789";
+ local third_char = orig[2];
+ local last_char = orig[-1];
+ local first_three_chars = orig[0:2];
+
+ Substring searching can be performed using the "in" or "!in"
+ operators (e.g., "bar" in "foobar" yields true).
Note that Bro represents strings internally as a count and vector of
bytes rather than a NUL-terminated byte string (although string
@@ -767,7 +781,7 @@ The Bro scripting language supports the following built-in types.
.. bro:type:: hook
A hook is another flavor of function that shares characteristics of
- both a :bro:type:`function` and a :bro:type:`event`. They are like
+ both a :bro:type:`function` and an :bro:type:`event`. They are like
events in that many handler bodies can be defined for the same hook
identifier and the order of execution can be enforced with
:bro:attr:`&priority`. They are more like functions in the way they
@@ -856,14 +870,14 @@ scripting language supports the following built-in attributes.
.. bro:attr:: &optional
Allows a record field to be missing. For example the type ``record {
- a: int, b: port &optional }`` could be instantiated both as
+ a: addr; b: port &optional; }`` could be instantiated both as
singleton ``[$a=127.0.0.1]`` or pair ``[$a=127.0.0.1, $b=80/tcp]``.
.. bro:attr:: &default
Uses a default value for a record field, a function/hook/event
parameter, or container elements. For example, ``table[int] of
- string &default="foo" }`` would create a table that returns the
+ string &default="foo"`` would create a table that returns the
:bro:type:`string` ``"foo"`` for any non-existing index.
.. bro:attr:: &redef
@@ -901,7 +915,7 @@ scripting language supports the following built-in attributes.
Called right before a container element expires. The function's
first parameter is of the same type of the container and the second
parameter the same type of the container's index. The return
- value is a :bro:type:`interval` indicating the amount of additional
+ value is an :bro:type:`interval` indicating the amount of additional
time to wait before expiring the container element at the given
index (which will trigger another execution of this function).
@@ -925,7 +939,7 @@ scripting language supports the following built-in attributes.
.. bro:attr:: &persistent
- Makes a variable persistent, i.e., its value is writen to disk (per
+ Makes a variable persistent, i.e., its value is written to disk (per
default at shutdown time).
.. bro:attr:: &synchronized
@@ -957,8 +971,9 @@ scripting language supports the following built-in attributes.
.. bro:attr:: &priority
- Specifies the execution priority of an event handler. Higher values
- are executed before lower ones. The default value is 0.
+ Specifies the execution priority (as a signed integer) of a hook or
+ event handler. Higher values are executed before lower ones. The
+ default value is 0.
.. bro:attr:: &group
diff --git a/doc/script-reference/file-analyzers.rst b/doc/script-reference/file-analyzers.rst
new file mode 100644
index 0000000000..81e71969a4
--- /dev/null
+++ b/doc/script-reference/file-analyzers.rst
@@ -0,0 +1 @@
+.. broxygen:file_analyzer:: *
diff --git a/doc/scripts/index.rst b/doc/script-reference/index.rst
similarity index 54%
rename from doc/scripts/index.rst
rename to doc/script-reference/index.rst
index 5a19bd23f0..bd600e4a97 100644
--- a/doc/scripts/index.rst
+++ b/doc/script-reference/index.rst
@@ -1,5 +1,3 @@
-.. This is a stub doc to which broxygen appends during the build process
-
================
Script Reference
================
@@ -7,15 +5,10 @@ Script Reference
.. toctree::
:maxdepth: 1
- packages
+ notices
proto-analyzers
file-analyzers
- notices
builtins
- Built-in Functions (BIFs)
- internal
+ packages
scripts
-
-
-
-
+ Broxygen Example Script
diff --git a/doc/scripts/notices.rst b/doc/script-reference/notices.rst
similarity index 100%
rename from doc/scripts/notices.rst
rename to doc/script-reference/notices.rst
diff --git a/doc/scripts/packages.rst b/doc/script-reference/packages.rst
similarity index 81%
rename from doc/scripts/packages.rst
rename to doc/script-reference/packages.rst
index 9f01a91e91..57fde4834e 100644
--- a/doc/scripts/packages.rst
+++ b/doc/script-reference/packages.rst
@@ -1,9 +1,7 @@
-.. This is a stub doc to which broxygen appends during the build process
-
.. _script-packages:
-Bro Script Packages
-===================
+Bro Package Index
+=================
Bro has the following script packages (e.g. collections of related scripts in
a common directory). If the package directory contains a ``__load__.bro``
@@ -12,3 +10,5 @@ script, it supports being loaded in mass as a whole directory for convenience.
Packages/scripts in the ``base/`` directory are all loaded by default, while
ones in ``policy/`` provide functionality and customization options that are
more appropriate for users to decide whether they'd like to load it or not.
+
+.. broxygen:package_index:: *
diff --git a/doc/script-reference/proto-analyzers.rst b/doc/script-reference/proto-analyzers.rst
new file mode 100644
index 0000000000..85c6ec1f12
--- /dev/null
+++ b/doc/script-reference/proto-analyzers.rst
@@ -0,0 +1 @@
+.. broxygen:proto_analyzer:: *
diff --git a/doc/script-reference/scripts.rst b/doc/script-reference/scripts.rst
new file mode 100644
index 0000000000..90bbc5543d
--- /dev/null
+++ b/doc/script-reference/scripts.rst
@@ -0,0 +1,5 @@
+================
+Bro Script Index
+================
+
+.. broxygen:script_index:: *
diff --git a/doc/scripting/index.rst b/doc/scripting/index.rst
index 16ca6b6d58..e42aa55e2c 100644
--- a/doc/scripting/index.rst
+++ b/doc/scripting/index.rst
@@ -316,7 +316,7 @@ block that variable is available to any other script through the
naming convention of ``MODULE::variable_name``.
The declaration below is taken from the
-:doc:`/scripts/policy/protocols/conn/known-hosts` script and
+:doc:`/scripts/policy/protocols/conn/known-hosts.bro` script and
declares a variable called ``known_hosts`` as a global set of unique
IP addresses within the ``Known`` namespace and exports it for use
outside of the ``Known`` namespace. Were we to want to use the
@@ -348,7 +348,7 @@ constants are used in Bro scripts as containers for configuration
options. For example, the configuration option to log password
decrypted from HTTP streams is stored in
``HTTP::default_capture_password`` as shown in the stripped down
-excerpt from :doc:`/scripts/base/protocols/http/main` below.
+excerpt from :doc:`/scripts/base/protocols/http/main.bro` below.
.. btest-include:: ${BRO_SRC_ROOT}/scripts/base/protocols/http/main.bro
:lines: 8-10,19-21,120
@@ -1182,7 +1182,7 @@ passing in the ``Notice::Info`` record. The simplest kind of
action based on the answer. The hook below adds the
:bro:enum:`Notice::ACTION_EMAIL` action for the
``SSH::Interesting_Hostname_Login`` notice raised in the
-:doc:`/scripts/policy/protocols/ssh/interesting-hostnames` script.
+:doc:`/scripts/policy/protocols/ssh/interesting-hostnames.bro` script.
.. btest-include:: ${DOC_ROOT}/scripting/framework_notice_hook_01.bro
@@ -1224,7 +1224,7 @@ Bro.
.. btest-include:: ${BRO_SRC_ROOT}/scripts/policy/protocols/ssl/expiring-certs.bro
:lines: 60-63
-In the :doc:`/scripts/policy/protocols/ssl/expiring-certs` script
+In the :doc:`/scripts/policy/protocols/ssl/expiring-certs.bro` script
which identifies when SSL certificates are set to expire and raises
notices when it crosses a predefined threshold, the call to
``NOTICE`` above also sets the ``$identifier`` entry by concatenating
diff --git a/doc/scripts/CMakeLists.txt b/doc/scripts/CMakeLists.txt
deleted file mode 100644
index fa234e74f2..0000000000
--- a/doc/scripts/CMakeLists.txt
+++ /dev/null
@@ -1,226 +0,0 @@
-# find out what BROPATH to use when executing bro
-execute_process(COMMAND ${CMAKE_BINARY_DIR}/bro-path-dev
- OUTPUT_VARIABLE BROPATH
- RESULT_VARIABLE retval
- OUTPUT_STRIP_TRAILING_WHITESPACE)
-if (NOT ${retval} EQUAL 0)
- message(FATAL_ERROR "Problem setting BROPATH")
-endif ()
-
-# This macro is used to add a new makefile target for reST policy script
-# documentation that can be generated using Bro itself to parse policy scripts.
-# It's called like:
-#
-# rest_target(srcDir broInput [group])
-#
-# srcDir: the directory which contains broInput
-# broInput: the file name of a bro policy script, any path prefix of this
-# argument will be used to derive what path under scripts/ the generated
-# documentation will be placed.
-# group: optional name of group that the script documentation will belong to.
-# If this is not given, the group is automatically set to any path portion
-# of the broInput argument.
-#
-# In addition to adding the makefile target, several CMake variables are set:
-#
-# MASTER_POLICY_INDEX_TEXT: a running list of policy scripts docs that have
-# been generated so far, formatted such that it can be appended to a file
-# that ends in a Sphinx toctree directive
-# ALL_REST_OUTPUTS: a running list (the CMake list type) of all reST docs
-# that are to be generated
-# MASTER_GROUP_LIST: a running list (the CMake list type) of all script groups
-# MASTER_PKG_LIST: a running list (the CMake list type) of all script groups
-# that were defived from the path portion of the broInput argument
-# ${group}_files: a running list of files belonging to a given group, from
-# which summary text can be extracted at build time
-# ${group}_doc_names: a running list of reST style document names that can be
-# given to a :doc: role, shared indices with ${group}_files
-
-macro(REST_TARGET srcDir broInput)
- set(absSrcPath ${srcDir}/${broInput})
- get_filename_component(basename ${broInput} NAME)
- string(REPLACE .bro "" basename ${basename})
- get_filename_component(extension ${broInput} EXT)
- get_filename_component(relDstDir ${broInput} PATH)
-
- set(sumTextSrc ${absSrcPath})
- set(ogSourceFile ${absSrcPath})
-
- if (NOT relDstDir)
- set(docName "${basename}")
- set(dstDir "${RST_OUTPUT_DIR}")
- else ()
- set(docName "${relDstDir}/${basename}")
- set(dstDir "${RST_OUTPUT_DIR}/${relDstDir}")
- endif ()
-
- set(restFile "${docName}.rst")
- string(REPLACE "/" "^" restFile ${restFile})
- set(restOutput "${dstDir}/${basename}.rst")
-
- set(MASTER_POLICY_INDEX_TEXT
- "${MASTER_POLICY_INDEX_TEXT}\n ${docName} <${docName}>")
- list(APPEND ALL_REST_OUTPUTS ${restOutput})
-
- if (NOT "${ARGN}" STREQUAL "")
- set(group ${ARGN})
- elseif (relDstDir)
- set(group ${relDstDir}/index)
- # add package index to master package list if not already in it
- # and if a __load__.bro exists in the original script directory
- list(FIND MASTER_PKG_LIST ${relDstDir} _found)
- if (_found EQUAL -1)
- if (EXISTS ${CMAKE_SOURCE_DIR}/scripts/${relDstDir}/__load__.bro)
- list(APPEND MASTER_PKG_LIST ${relDstDir})
- endif ()
- endif ()
- else ()
- set(group "")
- endif ()
-
- if (NOT "${group}" STREQUAL "")
- # add group to master group list if not already in it
- list(FIND MASTER_GROUP_LIST ${group} _found)
- if (_found EQUAL -1)
- list(APPEND MASTER_GROUP_LIST ${group})
- if (MASTER_GROUP_LIST_TEXT)
- set(MASTER_GROUP_LIST_TEXT "${MASTER_GROUP_LIST_TEXT}\n${group}")
- else ()
- set(MASTER_GROUP_LIST_TEXT "${group}")
- endif ()
- endif ()
-
- list(APPEND ${group}_files ${sumTextSrc})
- list(APPEND ${group}_doc_names ${docName})
- endif ()
-
- add_custom_command(OUTPUT ${restOutput}
- # delete any leftover state from previous bro runs
- COMMAND "${CMAKE_COMMAND}"
- ARGS -E remove_directory .state
- # generate the reST documentation using bro
- COMMAND BROPATH=${BROPATH}:${srcDir} BROMAGIC=${CMAKE_SOURCE_DIR}/magic/database ${CMAKE_BINARY_DIR}/src/bro
- ARGS -b -Z ${broInput} || (rm -rf .state *.log *.rst && exit 1)
- # move generated doc into a new directory tree that
- # defines the final structure of documents
- COMMAND "${CMAKE_COMMAND}"
- ARGS -E make_directory ${dstDir}
- COMMAND "${CMAKE_COMMAND}"
- ARGS -E copy ${restFile} ${restOutput}
- # copy the bro or bif script, too
- COMMAND "${CMAKE_COMMAND}"
- ARGS -E copy ${ogSourceFile} ${dstDir}
- # clean up the build directory
- COMMAND rm
- ARGS -rf .state *.log *.rst
- DEPENDS bro
- DEPENDS ${absSrcPath}
- WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
- COMMENT "[Bro] Generating reST docs for ${broInput}"
- )
-
-endmacro(REST_TARGET)
-
-# Schedule Bro scripts for which to generate documentation.
-include(DocSourcesList.cmake)
-
-# Macro for generating reST docs that are independent of any particular Bro
-# script.
-macro(INDEPENDENT_REST_TARGET reST_file)
- add_custom_command(OUTPUT ${reST_file}
- # delete any leftover state from previous bro runs
- COMMAND "${CMAKE_COMMAND}"
- ARGS -E remove_directory .state
- # generate the reST documentation using bro
- COMMAND BROPATH=${BROPATH}:${srcDir} BROMAGIC=${CMAKE_SOURCE_DIR}/magic/database ${CMAKE_BINARY_DIR}/src/bro
- ARGS -b -Z base/init-bare.bro || (rm -rf .state *.log *.rst && exit 1)
- # move generated doc into a new directory tree that
- # defines the final structure of documents
- COMMAND "${CMAKE_COMMAND}"
- ARGS -E make_directory ${dstDir}
- COMMAND "${CMAKE_COMMAND}"
- ARGS -E copy ${reST_file} ${dstDir}
- # clean up the build directory
- COMMAND rm
- ARGS -rf .state *.log *.rst
- DEPENDS bro
- WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
- COMMENT "[Bro] Generating reST docs for ${reST_file}"
- )
- list(APPEND ALL_REST_OUTPUTS ${reST_file})
-endmacro(INDEPENDENT_REST_TARGET)
-
-independent_rest_target(proto-analyzers.rst)
-independent_rest_target(file-analyzers.rst)
-
-# create temporary list of all docs to include in the master policy/index file
-file(WRITE ${MASTER_POLICY_INDEX} "${MASTER_POLICY_INDEX_TEXT}")
-
-# create the temporary list of all packages to include in the master
-# policy/packages.rst file
-set(MASTER_PKG_INDEX_TEXT "")
-foreach (pkg ${MASTER_PKG_LIST})
- set(MASTER_PKG_INDEX_TEXT
- "${MASTER_PKG_INDEX_TEXT}\n:doc:`${pkg} <${pkg}/index>`\n")
- if (EXISTS ${CMAKE_SOURCE_DIR}/scripts/${pkg}/README)
- file(STRINGS ${CMAKE_SOURCE_DIR}/scripts/${pkg}/README pkgreadme)
- foreach (line ${pkgreadme})
- set(MASTER_PKG_INDEX_TEXT "${MASTER_PKG_INDEX_TEXT}\n ${line}")
- endforeach ()
- set(MASTER_PKG_INDEX_TEXT "${MASTER_PKG_INDEX_TEXT}\n")
- endif ()
-endforeach ()
-file(WRITE ${MASTER_PACKAGE_INDEX} "${MASTER_PKG_INDEX_TEXT}")
-
-# create temporary file containing list of all groups
-file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/group_list
- "${MASTER_GROUP_LIST_TEXT}")
-
-# create temporary files containing list of each source file in a given group
-foreach (group ${MASTER_GROUP_LIST})
- if (EXISTS ${CMAKE_CURRENT_BINARY_DIR}/${group}_files)
- file(REMOVE ${CMAKE_CURRENT_BINARY_DIR}/${group}_files)
- endif ()
- if (EXISTS ${CMAKE_CURRENT_BINARY_DIR}/${group}_doc_names)
- file(REMOVE ${CMAKE_CURRENT_BINARY_DIR}/${group}_doc_names)
- endif ()
- foreach (src ${${group}_files})
- file(APPEND ${CMAKE_CURRENT_BINARY_DIR}/${group}_files "${src}\n")
- endforeach ()
- foreach (dname ${${group}_doc_names})
- file(APPEND ${CMAKE_CURRENT_BINARY_DIR}/${group}_doc_names "${dname}\n")
- endforeach ()
-endforeach ()
-
-# remove previously generated docs no longer scheduled for generation
-if (EXISTS ${RST_OUTPUT_DIR})
- file(GLOB_RECURSE EXISTING_REST_DOCS "${RST_OUTPUT_DIR}/*.rst")
- foreach (_doc ${EXISTING_REST_DOCS})
- list(FIND ALL_REST_OUTPUTS ${_doc} _found)
- if (_found EQUAL -1)
- file(REMOVE ${_doc})
- message(STATUS "Broxygen: remove stale reST doc: ${_doc}")
- string(REPLACE .rst .bro _brofile ${_doc})
- if (EXISTS ${_brofile})
- file(REMOVE ${_brofile})
- message(STATUS "Broxygen: remove stale bro source: ${_brofile}")
- endif ()
- endif ()
- endforeach ()
-endif ()
-
-# The "restdoc" target uses Bro to parse policy scripts in order to
-# generate reST documentation from them.
-add_custom_target(restdoc
- # create symlink to the reST output directory for convenience
- COMMAND "${CMAKE_COMMAND}" -E create_symlink
- ${RST_OUTPUT_DIR}
- ${CMAKE_BINARY_DIR}/reST
- DEPENDS ${ALL_REST_OUTPUTS})
-
-# The "restclean" target removes all generated reST documentation from the
-# build directory.
-add_custom_target(restclean
- COMMAND "${CMAKE_COMMAND}" -E remove_directory
- ${RST_OUTPUT_DIR}
- VERBATIM)
diff --git a/doc/scripts/DocSourcesList.cmake b/doc/scripts/DocSourcesList.cmake
deleted file mode 100644
index fdf5990659..0000000000
--- a/doc/scripts/DocSourcesList.cmake
+++ /dev/null
@@ -1,281 +0,0 @@
-# DO NOT EDIT
-# This file is auto-generated from the genDocSourcesList.sh script.
-#
-# This is a list of Bro script sources for which to generate reST documentation.
-# It will be included inline in the CMakeLists.txt found in the same directory
-# in order to create Makefile targets that define how to generate reST from
-# a given Bro script.
-#
-# Note: any path prefix of the script (2nd argument of rest_target macro)
-# will be used to derive what path under scripts/ the generated documentation
-# will be placed.
-
-set(psd ${PROJECT_SOURCE_DIR}/scripts)
-
-rest_target(${CMAKE_CURRENT_SOURCE_DIR} example.bro internal)
-rest_target(${psd} base/init-default.bro internal)
-rest_target(${psd} base/init-bare.bro internal)
-
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/analyzer.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/bloom-filter.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/bro.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/cardinality-counter.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/const.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/event.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/file_analysis.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/input.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/logging.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_ARP.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_AYIYA.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_BackDoor.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_BitTorrent.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_ConnSize.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_DCE_RPC.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_DHCP.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_DNP3.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_DNS.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_FTP.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_FTP.functions.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_File.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_FileExtract.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_FileExtract.functions.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_FileHash.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_Finger.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_GTPv1.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_Gnutella.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_HTTP.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_HTTP.functions.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_ICMP.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_IRC.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_Ident.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_InterConn.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_Login.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_Login.functions.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_MIME.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_Modbus.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_NCP.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_NTP.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_NetBIOS.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_NetBIOS.functions.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_NetFlow.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_PIA.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_POP3.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_RPC.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_SMB.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_SMTP.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_SMTP.functions.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_SOCKS.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_SSH.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_SSL.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_SSL.functions.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_SteppingStone.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_Syslog.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_TCP.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_TCP.functions.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_Teredo.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_UDP.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_Unified2.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_Unified2.types.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_ZIP.events.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/reporter.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/strings.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/top-k.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/types.bif.bro)
-rest_target(${psd} base/files/extract/main.bro)
-rest_target(${psd} base/files/hash/main.bro)
-rest_target(${psd} base/files/unified2/main.bro)
-rest_target(${psd} base/frameworks/analyzer/main.bro)
-rest_target(${psd} base/frameworks/cluster/main.bro)
-rest_target(${psd} base/frameworks/cluster/nodes/manager.bro)
-rest_target(${psd} base/frameworks/cluster/nodes/proxy.bro)
-rest_target(${psd} base/frameworks/cluster/nodes/worker.bro)
-rest_target(${psd} base/frameworks/cluster/setup-connections.bro)
-rest_target(${psd} base/frameworks/communication/main.bro)
-rest_target(${psd} base/frameworks/control/main.bro)
-rest_target(${psd} base/frameworks/dpd/main.bro)
-rest_target(${psd} base/frameworks/files/main.bro)
-rest_target(${psd} base/frameworks/input/main.bro)
-rest_target(${psd} base/frameworks/input/readers/ascii.bro)
-rest_target(${psd} base/frameworks/input/readers/benchmark.bro)
-rest_target(${psd} base/frameworks/input/readers/binary.bro)
-rest_target(${psd} base/frameworks/input/readers/raw.bro)
-rest_target(${psd} base/frameworks/input/readers/sqlite.bro)
-rest_target(${psd} base/frameworks/intel/cluster.bro)
-rest_target(${psd} base/frameworks/intel/input.bro)
-rest_target(${psd} base/frameworks/intel/main.bro)
-rest_target(${psd} base/frameworks/logging/main.bro)
-rest_target(${psd} base/frameworks/logging/postprocessors/scp.bro)
-rest_target(${psd} base/frameworks/logging/postprocessors/sftp.bro)
-rest_target(${psd} base/frameworks/logging/writers/ascii.bro)
-rest_target(${psd} base/frameworks/logging/writers/dataseries.bro)
-rest_target(${psd} base/frameworks/logging/writers/elasticsearch.bro)
-rest_target(${psd} base/frameworks/logging/writers/none.bro)
-rest_target(${psd} base/frameworks/logging/writers/sqlite.bro)
-rest_target(${psd} base/frameworks/notice/actions/add-geodata.bro)
-rest_target(${psd} base/frameworks/notice/actions/drop.bro)
-rest_target(${psd} base/frameworks/notice/actions/email_admin.bro)
-rest_target(${psd} base/frameworks/notice/actions/page.bro)
-rest_target(${psd} base/frameworks/notice/actions/pp-alarms.bro)
-rest_target(${psd} base/frameworks/notice/cluster.bro)
-rest_target(${psd} base/frameworks/notice/extend-email/hostnames.bro)
-rest_target(${psd} base/frameworks/notice/main.bro)
-rest_target(${psd} base/frameworks/notice/non-cluster.bro)
-rest_target(${psd} base/frameworks/notice/weird.bro)
-rest_target(${psd} base/frameworks/packet-filter/cluster.bro)
-rest_target(${psd} base/frameworks/packet-filter/main.bro)
-rest_target(${psd} base/frameworks/packet-filter/netstats.bro)
-rest_target(${psd} base/frameworks/packet-filter/utils.bro)
-rest_target(${psd} base/frameworks/reporter/main.bro)
-rest_target(${psd} base/frameworks/signatures/main.bro)
-rest_target(${psd} base/frameworks/software/main.bro)
-rest_target(${psd} base/frameworks/sumstats/cluster.bro)
-rest_target(${psd} base/frameworks/sumstats/main.bro)
-rest_target(${psd} base/frameworks/sumstats/non-cluster.bro)
-rest_target(${psd} base/frameworks/sumstats/plugins/average.bro)
-rest_target(${psd} base/frameworks/sumstats/plugins/hll_unique.bro)
-rest_target(${psd} base/frameworks/sumstats/plugins/last.bro)
-rest_target(${psd} base/frameworks/sumstats/plugins/max.bro)
-rest_target(${psd} base/frameworks/sumstats/plugins/min.bro)
-rest_target(${psd} base/frameworks/sumstats/plugins/sample.bro)
-rest_target(${psd} base/frameworks/sumstats/plugins/std-dev.bro)
-rest_target(${psd} base/frameworks/sumstats/plugins/sum.bro)
-rest_target(${psd} base/frameworks/sumstats/plugins/topk.bro)
-rest_target(${psd} base/frameworks/sumstats/plugins/unique.bro)
-rest_target(${psd} base/frameworks/sumstats/plugins/variance.bro)
-rest_target(${psd} base/frameworks/tunnels/main.bro)
-rest_target(${psd} base/misc/find-checksum-offloading.bro)
-rest_target(${psd} base/protocols/conn/contents.bro)
-rest_target(${psd} base/protocols/conn/inactivity.bro)
-rest_target(${psd} base/protocols/conn/main.bro)
-rest_target(${psd} base/protocols/conn/polling.bro)
-rest_target(${psd} base/protocols/dhcp/consts.bro)
-rest_target(${psd} base/protocols/dhcp/main.bro)
-rest_target(${psd} base/protocols/dhcp/utils.bro)
-rest_target(${psd} base/protocols/dnp3/consts.bro)
-rest_target(${psd} base/protocols/dnp3/main.bro)
-rest_target(${psd} base/protocols/dns/consts.bro)
-rest_target(${psd} base/protocols/dns/main.bro)
-rest_target(${psd} base/protocols/ftp/files.bro)
-rest_target(${psd} base/protocols/ftp/gridftp.bro)
-rest_target(${psd} base/protocols/ftp/info.bro)
-rest_target(${psd} base/protocols/ftp/main.bro)
-rest_target(${psd} base/protocols/ftp/utils-commands.bro)
-rest_target(${psd} base/protocols/ftp/utils.bro)
-rest_target(${psd} base/protocols/http/entities.bro)
-rest_target(${psd} base/protocols/http/files.bro)
-rest_target(${psd} base/protocols/http/main.bro)
-rest_target(${psd} base/protocols/http/utils.bro)
-rest_target(${psd} base/protocols/irc/dcc-send.bro)
-rest_target(${psd} base/protocols/irc/files.bro)
-rest_target(${psd} base/protocols/irc/main.bro)
-rest_target(${psd} base/protocols/modbus/consts.bro)
-rest_target(${psd} base/protocols/modbus/main.bro)
-rest_target(${psd} base/protocols/smtp/entities.bro)
-rest_target(${psd} base/protocols/smtp/files.bro)
-rest_target(${psd} base/protocols/smtp/main.bro)
-rest_target(${psd} base/protocols/socks/consts.bro)
-rest_target(${psd} base/protocols/socks/main.bro)
-rest_target(${psd} base/protocols/ssh/main.bro)
-rest_target(${psd} base/protocols/ssl/consts.bro)
-rest_target(${psd} base/protocols/ssl/main.bro)
-rest_target(${psd} base/protocols/ssl/mozilla-ca-list.bro)
-rest_target(${psd} base/protocols/syslog/consts.bro)
-rest_target(${psd} base/protocols/syslog/main.bro)
-rest_target(${psd} base/utils/active-http.bro)
-rest_target(${psd} base/utils/addrs.bro)
-rest_target(${psd} base/utils/conn-ids.bro)
-rest_target(${psd} base/utils/dir.bro)
-rest_target(${psd} base/utils/directions-and-hosts.bro)
-rest_target(${psd} base/utils/exec.bro)
-rest_target(${psd} base/utils/files.bro)
-rest_target(${psd} base/utils/numbers.bro)
-rest_target(${psd} base/utils/paths.bro)
-rest_target(${psd} base/utils/patterns.bro)
-rest_target(${psd} base/utils/queue.bro)
-rest_target(${psd} base/utils/site.bro)
-rest_target(${psd} base/utils/strings.bro)
-rest_target(${psd} base/utils/thresholds.bro)
-rest_target(${psd} base/utils/time.bro)
-rest_target(${psd} base/utils/urls.bro)
-rest_target(${psd} policy/frameworks/communication/listen.bro)
-rest_target(${psd} policy/frameworks/control/controllee.bro)
-rest_target(${psd} policy/frameworks/control/controller.bro)
-rest_target(${psd} policy/frameworks/dpd/detect-protocols.bro)
-rest_target(${psd} policy/frameworks/dpd/packet-segment-logging.bro)
-rest_target(${psd} policy/frameworks/files/detect-MHR.bro)
-rest_target(${psd} policy/frameworks/files/hash-all-files.bro)
-rest_target(${psd} policy/frameworks/intel/do_notice.bro)
-rest_target(${psd} policy/frameworks/intel/seen/conn-established.bro)
-rest_target(${psd} policy/frameworks/intel/seen/dns.bro)
-rest_target(${psd} policy/frameworks/intel/seen/file-hashes.bro)
-rest_target(${psd} policy/frameworks/intel/seen/file-names.bro)
-rest_target(${psd} policy/frameworks/intel/seen/http-headers.bro)
-rest_target(${psd} policy/frameworks/intel/seen/http-url.bro)
-rest_target(${psd} policy/frameworks/intel/seen/smtp-url-extraction.bro)
-rest_target(${psd} policy/frameworks/intel/seen/smtp.bro)
-rest_target(${psd} policy/frameworks/intel/seen/ssl.bro)
-rest_target(${psd} policy/frameworks/intel/seen/where-locations.bro)
-rest_target(${psd} policy/frameworks/packet-filter/shunt.bro)
-rest_target(${psd} policy/frameworks/software/version-changes.bro)
-rest_target(${psd} policy/frameworks/software/vulnerable.bro)
-rest_target(${psd} policy/integration/barnyard2/main.bro)
-rest_target(${psd} policy/integration/barnyard2/types.bro)
-rest_target(${psd} policy/integration/collective-intel/main.bro)
-rest_target(${psd} policy/misc/app-stats/main.bro)
-rest_target(${psd} policy/misc/app-stats/plugins/facebook.bro)
-rest_target(${psd} policy/misc/app-stats/plugins/gmail.bro)
-rest_target(${psd} policy/misc/app-stats/plugins/google.bro)
-rest_target(${psd} policy/misc/app-stats/plugins/netflix.bro)
-rest_target(${psd} policy/misc/app-stats/plugins/pandora.bro)
-rest_target(${psd} policy/misc/app-stats/plugins/youtube.bro)
-rest_target(${psd} policy/misc/capture-loss.bro)
-rest_target(${psd} policy/misc/detect-traceroute/main.bro)
-rest_target(${psd} policy/misc/known-devices.bro)
-rest_target(${psd} policy/misc/load-balancing.bro)
-rest_target(${psd} policy/misc/loaded-scripts.bro)
-rest_target(${psd} policy/misc/profiling.bro)
-rest_target(${psd} policy/misc/scan.bro)
-rest_target(${psd} policy/misc/stats.bro)
-rest_target(${psd} policy/misc/trim-trace-file.bro)
-rest_target(${psd} policy/protocols/conn/known-hosts.bro)
-rest_target(${psd} policy/protocols/conn/known-services.bro)
-rest_target(${psd} policy/protocols/conn/weirds.bro)
-rest_target(${psd} policy/protocols/dhcp/known-devices-and-hostnames.bro)
-rest_target(${psd} policy/protocols/dns/auth-addl.bro)
-rest_target(${psd} policy/protocols/dns/detect-external-names.bro)
-rest_target(${psd} policy/protocols/ftp/detect-bruteforcing.bro)
-rest_target(${psd} policy/protocols/ftp/detect.bro)
-rest_target(${psd} policy/protocols/ftp/software.bro)
-rest_target(${psd} policy/protocols/http/detect-sqli.bro)
-rest_target(${psd} policy/protocols/http/detect-webapps.bro)
-rest_target(${psd} policy/protocols/http/header-names.bro)
-rest_target(${psd} policy/protocols/http/software-browser-plugins.bro)
-rest_target(${psd} policy/protocols/http/software.bro)
-rest_target(${psd} policy/protocols/http/var-extraction-cookies.bro)
-rest_target(${psd} policy/protocols/http/var-extraction-uri.bro)
-rest_target(${psd} policy/protocols/modbus/known-masters-slaves.bro)
-rest_target(${psd} policy/protocols/modbus/track-memmap.bro)
-rest_target(${psd} policy/protocols/smtp/blocklists.bro)
-rest_target(${psd} policy/protocols/smtp/detect-suspicious-orig.bro)
-rest_target(${psd} policy/protocols/smtp/entities-excerpt.bro)
-rest_target(${psd} policy/protocols/smtp/software.bro)
-rest_target(${psd} policy/protocols/ssh/detect-bruteforcing.bro)
-rest_target(${psd} policy/protocols/ssh/geo-data.bro)
-rest_target(${psd} policy/protocols/ssh/interesting-hostnames.bro)
-rest_target(${psd} policy/protocols/ssh/software.bro)
-rest_target(${psd} policy/protocols/ssl/cert-hash.bro)
-rest_target(${psd} policy/protocols/ssl/expiring-certs.bro)
-rest_target(${psd} policy/protocols/ssl/extract-certs-pem.bro)
-rest_target(${psd} policy/protocols/ssl/known-certs.bro)
-rest_target(${psd} policy/protocols/ssl/notary.bro)
-rest_target(${psd} policy/protocols/ssl/validate-certs.bro)
-rest_target(${psd} policy/tuning/defaults/extracted_file_limits.bro)
-rest_target(${psd} policy/tuning/defaults/packet-fragments.bro)
-rest_target(${psd} policy/tuning/defaults/warnings.bro)
-rest_target(${psd} policy/tuning/logs-to-elasticsearch.bro)
-rest_target(${psd} policy/tuning/track-all-assets.bro)
-rest_target(${psd} site/local-manager.bro)
-rest_target(${psd} site/local-proxy.bro)
-rest_target(${psd} site/local-worker.bro)
-rest_target(${psd} site/local.bro)
-rest_target(${psd} test-all-policy.bro)
diff --git a/doc/scripts/README b/doc/scripts/README
deleted file mode 100644
index a15812609c..0000000000
--- a/doc/scripts/README
+++ /dev/null
@@ -1,44 +0,0 @@
-This directory contains scripts and templates that can be used to automate
-the generation of Bro script documentation. Several build targets are defined
-by CMake and available in the top-level Makefile:
-
-``restdoc``
-
- This target uses Bro to parse policy scripts in order to generate
- reStructuredText (reST) documentation from them. The list of scripts
- for which to generate reST documentation is defined in the
- ``CMakeLists.txt`` file in this directory. Script documentation is
- rebuild automatically if the policy script from which it is derived
- or the Bro binary becomes out of date
-
- The resulting output from this target can be found in the CMake
- ``build/`` directory inside ``reST`` (a symlink to
- ``doc/scripts/rest_output``).
-
-``restclean``
-
- This target removes any reST documentation that has been generated so far.
-
-The ``genDocSourcesList.sh`` script can be run to automatically generate
-``DocSourcesList.cmake``, which is the file CMake uses to define the list
-of documentation targets. This script should be run after adding new
-Bro script source files, and the changes commited to git.
-
-If a script shouldn't have documentation generated for it, there's also a
-blacklist manifest that can be maintained in the ``genDocSourcesList.sh``
-script.
-
-The blacklist can also be used if you want to define a certain grouping for
-the script's generated docs to belong to (as opposed to the automatic grouping
-the happens for script packages/directories). To do that, add the
-script's name to the blacklist, then append a ``rest_target()`` to the
-``statictext`` variable where the first argument is the source directory
-containing the policy script to document, the second argument is the file
-name of the policy script, and the third argument is the path/name of a
-pre-created reST document in the ``../`` source directory to which the
-``make doc`` process can append script documentation references. This
-pre-created reST document should also then be linked to from the TOC tree
-in ``../index.rst``.
-
-See ``example.bro`` for an example of how to document a Bro script such that
-``make doc`` will be able to produce reST/HTML documentation for it.
diff --git a/doc/scripts/example.bro b/doc/scripts/example.bro
deleted file mode 100644
index ff6e4e34d2..0000000000
--- a/doc/scripts/example.bro
+++ /dev/null
@@ -1,229 +0,0 @@
-##! This is an example script that demonstrates documentation features.
-##! Comments of the form ``##!`` are for the script summary. The contents of
-##! these comments are transferred directly into the auto-generated
-##! `reStructuredText `_
-##! (reST) document's summary section.
-##!
-##! .. tip:: You can embed directives and roles within ``##``-stylized comments.
-##!
-##! There's also a custom role to reference any identifier node in
-##! the Bro Sphinx domain that's good for "see alsos", e.g.
-##!
-##! See also: :bro:see:`Example::a_var`, :bro:see:`Example::ONE`,
-##! :bro:see:`SSH::Info`
-##!
-##! And a custom directive does the equivalent references:
-##!
-##! .. bro:see:: Example::a_var Example::ONE SSH::Info
-
-# Comments that use a single pound sign (#) are not significant to
-# a script's auto-generated documentation, but ones that use a
-# double pound sign (##) do matter. In some cases, like record
-# field comments, it's necessary to disambiguate the field with
-# which a comment associates: e.g. "##<" can be used on the same line
-# as a field to signify the comment relates to it and not the
-# following field. "##<" can also be used more generally in any
-# variable declarations to associate with the last-declared identifier.
-#
-# Generally, the auto-doc comments (##) are associated with the
-# next declaration/identifier found in the script, but the doc framework
-# will track/render identifiers regardless of whether they have any
-# of these special comments associated with them.
-#
-# The first sentence contained within the "##"-stylized comments for
-# a given identifier is special in that it will be used as summary
-# text in a table containing all such identifiers and short summaries.
-# If there are no sentences (text terminated with '.'), then everything
-# in the "##"-stylized comments up until the first empty comment
-# is taken as the summary text for a given identifier.
-
-# @load directives are self-documenting
-@load frameworks/software/vulnerable
-
-# "module" statements are self-documenting
-module Example;
-
-# redefinitions of "capture_filters" are self-documenting and
-# go into the generated documentation's "Packet Filter" section
-redef capture_filters += {
- ["ssl"] = "tcp port 443",
- ["nntps"] = "tcp port 562",
-};
-
-global example_ports = {
- 443/tcp, 562/tcp,
-} &redef;
-
-
-event bro_init()
- {
- Analyzer::register_for_ports(Analyzer::ANALYZER_SSL, example_ports);
- }
-
-# redefinitions of "Notice::Type" are self-documenting, but
-# more information can be supplied in two different ways
-redef enum Notice::Type += {
- ## any number of this type of comment
- ## will document "Notice_One"
- Notice_One,
- Notice_Two, ##< any number of this type of comment
- ##< will document "Notice_Two"
- Notice_Three,
- Notice_Four,
-};
-
-# Redef'ing the ID enumeration for logging streams is automatically tracked.
-# Comments of the "##" form can be use to further document it, but it's
-# better to do all documentation related to logging in the summary section
-# as is shown above.
-redef enum Log::ID += { LOG };
-
-# Anything declared in the export section will show up in the rendered
-# documentation's "public interface" section
-
-export {
-
- # these headings don't mean anything special to the
- # doc framework right now, I'm just including them
- # to make it more clear to the reader how the doc
- # framework will actually categorize a script's identifiers
-
- ############## types ################
-
- # Note that I'm just mixing the "##" and "##<"
- # types of comments in the following declarations
- # as a demonstration. Normally, it would be good style
- # to pick one and be consistent.
-
- ## documentation for "SimpleEnum"
- ## goes here.
- type SimpleEnum: enum {
- ## and more specific info for "ONE"
- ## can span multiple lines
- ONE,
- TWO, ##< or more info like this for "TWO"
- ##< can span multiple lines
- THREE,
- };
-
- ## document the "SimpleEnum" redef here
- redef enum SimpleEnum += {
- FOUR, ##< and some documentation for "FOUR"
- ## also "FIVE" for good measure
- FIVE
- };
-
- ## general documentation for a type "SimpleRecord"
- ## goes here.
- type SimpleRecord: record {
- ## counts something
- field1: count;
- field2: bool; ##< toggles something
- };
-
- ## document the record extension redef here
- redef record SimpleRecord += {
- ## document the extending field here
- field_ext: string &optional; ##< (or here)
- };
-
- ## general documentation for a type "ComplexRecord" goes here
- type ComplexRecord: record {
- field1: count; ##< counts something
- field2: bool; ##< toggles something
- field3: SimpleRecord;
- msg: string &default="blah"; ##< attributes are self-documenting
- } &redef;
-
- ## An example record to be used with a logging stream.
- type Info: record {
- ts: time &log;
- uid: string &log;
- status: count &log &optional;
- };
-
- ############## options ################
- # right now, I'm just defining an option as
- # any const with &redef (something that can
- # change at parse time, but not at run time.
-
- ## add documentation for "an_option" here
- const an_option: set[addr, addr, string] &redef;
-
- # default initialization will be self-documenting
- const option_with_init = 0.01 secs &redef; ##< More docs can be added here.
-
- ############## state variables ############
- # right now, I'm defining this as any global
- # that's not a function/event. doesn't matter
- # if &redef attribute is present
-
- ## put some documentation for "a_var" here
- global a_var: bool;
-
- # attributes are self-documenting
- global var_with_attr: count &persistent;
-
- # it's fine if the type is inferred, that information is self-documenting
- global var_without_explicit_type = "this works";
-
- ## The first.sentence for the summary text ends here. And this second
- ## sentence doesn't show in the short description.
- global dummy: string;
-
- ############## functions/events ############
-
- ## Summarize purpose of "a_function" here.
- ## Give more details about "a_function" here.
- ## Separating the documentation of the params/return values with
- ## empty comments is optional, but improves readability of script.
- ##
- ## tag: function arguments can be described
- ## like this
- ## msg: another param
- ##
- ## Returns: describe the return type here
- global a_function: function(tag: string, msg: string): string;
-
- ## Summarize "an_event" here.
- ## Give more details about "an_event" here.
- ## Example::an_event should not be confused as a parameter.
- ## name: describe the argument here
- global an_event: event(name: string);
-
- ## This is a declaration of an example event that can be used in
- ## logging streams and is raised once for each log entry.
- global log_example: event(rec: Info);
-}
-
-function filter_func(rec: Info): bool
- {
- return T;
- }
-
-# this function is documented in the "private interface" section
-# of generated documentation and any "##"-stylized comments would also
-# be rendered there
-function function_without_proto(tag: string): string
- {
- return "blah";
- }
-
-# this record type is documented in the "private interface" section
-# of generated documentation and any "##"-stylized comments would also
-# be rendered there
-type PrivateRecord: record {
- field1: bool;
- field2: count;
-};
-
-event bro_init()
- {
- Log::create_stream(Example::LOG, [$columns=Info, $ev=log_example]);
- Log::add_filter(Example::LOG, [
- $name="example-filter",
- $path="example-filter",
- $pred=filter_func,
- $exclude=set("ts")
- ]);
- }
diff --git a/doc/scripts/example.rst b/doc/scripts/example.rst
deleted file mode 100644
index b76b9af59b..0000000000
--- a/doc/scripts/example.rst
+++ /dev/null
@@ -1,291 +0,0 @@
-.. Automatically generated. Do not edit.
-
-example.bro
-===========
-
-:download:`Original Source File `
-
-Overview
---------
-This is an example script that demonstrates how to document. Comments
-of the form ``##!`` are for the script summary. The contents of
-these comments are transferred directly into the auto-generated
-`reStructuredText `_
-(reST) document's summary section.
-
-.. tip:: You can embed directives and roles within ``##``-stylized comments.
-
-:Imports: :doc:`policy/frameworks/software/vulnerable `
-
-Summary
-~~~~~~~
-Options
-#######
-============================================================================ ======================================
-:bro:id:`Example::an_option`: :bro:type:`set` :bro:attr:`&redef` add documentation for "an_option" here
-
-:bro:id:`Example::option_with_init`: :bro:type:`interval` :bro:attr:`&redef`
-============================================================================ ======================================
-
-State Variables
-###############
-=========================================================================== =======================================
-:bro:id:`Example::a_var`: :bro:type:`bool` put some documentation for "a_var" here
-
-:bro:id:`Example::var_with_attr`: :bro:type:`count` :bro:attr:`&persistent`
-
-:bro:id:`Example::var_without_explicit_type`: :bro:type:`string`
-=========================================================================== =======================================
-
-Types
-#####
-====================================================== ==========================================================
-:bro:type:`Example::SimpleEnum`: :bro:type:`enum` documentation for "SimpleEnum"
- goes here.
-
-:bro:type:`Example::SimpleRecord`: :bro:type:`record` general documentation for a type "SimpleRecord"
- goes here.
-
-:bro:type:`Example::ComplexRecord`: :bro:type:`record` general documentation for a type "ComplexRecord" goes here
-
-:bro:type:`Example::Info`: :bro:type:`record` An example record to be used with a logging stream.
-====================================================== ==========================================================
-
-Events
-######
-================================================= =============================================================
-:bro:id:`Example::an_event`: :bro:type:`event` Summarize "an_event" here.
-
-:bro:id:`Example::log_example`: :bro:type:`event` This is a declaration of an example event that can be used in
- logging streams and is raised once for each log entry.
-================================================= =============================================================
-
-Functions
-#########
-=============================================== =======================================
-:bro:id:`Example::a_function`: :bro:type:`func` Summarize purpose of "a_function" here.
-=============================================== =======================================
-
-Redefinitions
-#############
-===================================================== ========================================
-:bro:type:`Log::ID`: :bro:type:`enum`
-
-:bro:type:`Example::SimpleEnum`: :bro:type:`enum` document the "SimpleEnum" redef here
-
-:bro:type:`Example::SimpleRecord`: :bro:type:`record` document the record extension redef here
-===================================================== ========================================
-
-Namespaces
-~~~~~~~~~~
-.. bro:namespace:: Example
-
-Notices
-~~~~~~~
-:bro:type:`Notice::Type`
-
- :Type: :bro:type:`enum`
-
- .. bro:enum:: Example::Notice_One Notice::Type
-
- any number of this type of comment
- will document "Notice_One"
-
- .. bro:enum:: Example::Notice_Two Notice::Type
-
- any number of this type of comment
- will document "Notice_Two"
-
- .. bro:enum:: Example::Notice_Three Notice::Type
-
- .. bro:enum:: Example::Notice_Four Notice::Type
-
-Public Interface
-----------------
-Options
-~~~~~~~
-.. bro:id:: Example::an_option
-
- :Type: :bro:type:`set` [:bro:type:`addr`, :bro:type:`addr`, :bro:type:`string`]
- :Attributes: :bro:attr:`&redef`
- :Default: ``{}``
-
- add documentation for "an_option" here
-
-.. bro:id:: Example::option_with_init
-
- :Type: :bro:type:`interval`
- :Attributes: :bro:attr:`&redef`
- :Default: ``10.0 msecs``
-
-State Variables
-~~~~~~~~~~~~~~~
-.. bro:id:: Example::a_var
-
- :Type: :bro:type:`bool`
-
- put some documentation for "a_var" here
-
-.. bro:id:: Example::var_with_attr
-
- :Type: :bro:type:`count`
- :Attributes: :bro:attr:`&persistent`
-
-.. bro:id:: Example::var_without_explicit_type
-
- :Type: :bro:type:`string`
- :Default: ``"this works"``
-
-Types
-~~~~~
-.. bro:type:: Example::SimpleEnum
-
- :Type: :bro:type:`enum`
-
- .. bro:enum:: Example::ONE Example::SimpleEnum
-
- and more specific info for "ONE"
- can span multiple lines
-
- .. bro:enum:: Example::TWO Example::SimpleEnum
-
- or more info like this for "TWO"
- can span multiple lines
-
- .. bro:enum:: Example::THREE Example::SimpleEnum
-
- documentation for "SimpleEnum"
- goes here.
-
-.. bro:type:: Example::SimpleRecord
-
- :Type: :bro:type:`record`
-
- field1: :bro:type:`count`
- counts something
-
- field2: :bro:type:`bool`
- toggles something
-
- general documentation for a type "SimpleRecord"
- goes here.
-
-.. bro:type:: Example::ComplexRecord
-
- :Type: :bro:type:`record`
-
- field1: :bro:type:`count`
- counts something
-
- field2: :bro:type:`bool`
- toggles something
-
- field3: :bro:type:`Example::SimpleRecord`
-
- msg: :bro:type:`string` :bro:attr:`&default` = ``"blah"`` :bro:attr:`&optional`
- attributes are self-documenting
-
- general documentation for a type "ComplexRecord" goes here
-
-.. bro:type:: Example::Info
-
- :Type: :bro:type:`record`
-
- ts: :bro:type:`time` :bro:attr:`&log`
-
- uid: :bro:type:`string` :bro:attr:`&log`
-
- status: :bro:type:`count` :bro:attr:`&log` :bro:attr:`&optional`
-
- An example record to be used with a logging stream.
-
-Events
-~~~~~~
-.. bro:id:: Example::an_event
-
- :Type: :bro:type:`event` (name: :bro:type:`string`)
-
- Summarize "an_event" here.
- Give more details about "an_event" here.
-
- :param name: describe the argument here
-
-.. bro:id:: Example::log_example
-
- :Type: :bro:type:`event` (rec: :bro:type:`Example::Info`)
-
- This is a declaration of an example event that can be used in
- logging streams and is raised once for each log entry.
-
-Functions
-~~~~~~~~~
-.. bro:id:: Example::a_function
-
- :Type: :bro:type:`function` (tag: :bro:type:`string`, msg: :bro:type:`string`) : :bro:type:`string`
-
- Summarize purpose of "a_function" here.
- Give more details about "a_function" here.
- Separating the documentation of the params/return values with
- empty comments is optional, but improves readability of script.
-
-
- :param tag: function arguments can be described
- like this
-
- :param msg: another param
-
-
- :returns: describe the return type here
-
-Redefinitions
-~~~~~~~~~~~~~
-:bro:type:`Log::ID`
-
- :Type: :bro:type:`enum`
-
- .. bro:enum:: Example::LOG Log::ID
-
-:bro:type:`Example::SimpleEnum`
-
- :Type: :bro:type:`enum`
-
- .. bro:enum:: Example::FOUR Example::SimpleEnum
-
- and some documentation for "FOUR"
-
- .. bro:enum:: Example::FIVE Example::SimpleEnum
-
- also "FIVE" for good measure
-
- document the "SimpleEnum" redef here
-
-:bro:type:`Example::SimpleRecord`
-
- :Type: :bro:type:`record`
-
- field_ext: :bro:type:`string` :bro:attr:`&optional`
- document the extending field here
- (or here)
-
- document the record extension redef here
-
-Port Analysis
--------------
-:ref:`More Information `
-
-SSL::
-
- [ports={
- 443/tcp,
- 562/tcp
- }]
-
-Packet Filter
--------------
-:ref:`More Information `
-
-Filters added::
-
- [ssl] = tcp port 443,
- [nntps] = tcp port 562
-
diff --git a/doc/scripts/genDocSourcesList.sh b/doc/scripts/genDocSourcesList.sh
deleted file mode 100755
index 31905c68db..0000000000
--- a/doc/scripts/genDocSourcesList.sh
+++ /dev/null
@@ -1,86 +0,0 @@
-#!/usr/bin/env bash
-
-# ./genDocSourcesList.sh [output file]
-#
-# Run this script to a generate file that's used to tell CMake about all the
-# possible scripts for which reST documentation can be created.
-#
-# The optional argument can be used to avoid overwriting the file CMake uses
-# by default.
-#
-# Specific scripts can be blacklisted below when e.g. they currently aren't
-# parseable or they just aren't meant to be documented.
-
-export LC_ALL=C # Make sorting stable.
-
-blacklist ()
- {
- if [[ "$blacklist" == "" ]]; then
- blacklist="$1"
- else
- blacklist="$blacklist|$1"
- fi
- }
-
-# files passed into this function are meant to be temporary workarounds
-# because they're not finished or otherwise can't be loaded for some reason
-tmp_blacklist ()
- {
- echo "Warning: temporarily blacklisted files named '$1'" 1>&2
- blacklist $1
- }
-
-blacklist __load__.bro
-blacklist test-all.bro
-blacklist all.bro
-blacklist init-default.bro
-blacklist init-bare.bro
-
-statictext="\
-# DO NOT EDIT
-# This file is auto-generated from the "genDocSourcesList.sh" script.
-#
-# This is a list of Bro script sources for which to generate reST documentation.
-# It will be included inline in the CMakeLists.txt found in the same directory
-# in order to create Makefile targets that define how to generate reST from
-# a given Bro script.
-#
-# Note: any path prefix of the script (2nd argument of rest_target macro)
-# will be used to derive what path under scripts/ the generated documentation
-# will be placed.
-
-set(psd \${PROJECT_SOURCE_DIR}/scripts)
-
-rest_target(\${CMAKE_CURRENT_SOURCE_DIR} example.bro internal)
-rest_target(\${psd} base/init-default.bro internal)
-rest_target(\${psd} base/init-bare.bro internal)
-"
-
-if [[ $# -ge 1 ]]; then
- outfile=$1
-else
- outfile=DocSourcesList.cmake
-fi
-
-thisdir="$( cd "$( dirname "$0" )" && pwd )"
-sourcedir=${thisdir}/../..
-
-echo "$statictext" > $outfile
-
-bifs=`( cd ${sourcedir}/build/scripts/base && find . -name \*\.bif.bro | sort )`
-
-for file in $bifs
-do
- f=${file:2}
- echo "rest_target(\${CMAKE_BINARY_DIR}/scripts base/$f)" >> $outfile
-done
-
-scriptfiles=`( cd ${sourcedir}/scripts && find . -name \*\.bro | sort )`
-
-for file in $scriptfiles
-do
- f=${file:2}
- if [[ ! $f =~ $blacklist ]]; then
- echo "rest_target(\${psd} $f)" >> $outfile
- fi
-done
diff --git a/doc/scripts/internal.rst b/doc/scripts/internal.rst
deleted file mode 100644
index a6c10f1cfb..0000000000
--- a/doc/scripts/internal.rst
+++ /dev/null
@@ -1,5 +0,0 @@
-.. This is a stub doc to which broxygen appends during the build process
-
-Internal Scripts
-================
-
diff --git a/doc/scripts/scripts.rst b/doc/scripts/scripts.rst
deleted file mode 100644
index 8aceacf329..0000000000
--- a/doc/scripts/scripts.rst
+++ /dev/null
@@ -1,8 +0,0 @@
-.. This is a stub doc to which broxygen appends during the build process
-
-===============
-All Bro Scripts
-===============
-
-.. toctree::
- :maxdepth: 1
diff --git a/magic b/magic
index e87fe13a7b..99c6b89230 160000
--- a/magic
+++ b/magic
@@ -1 +1 @@
-Subproject commit e87fe13a7b776182ffc8c75076d42702f5c28fed
+Subproject commit 99c6b89230e2b9b0e781c42b0b9412d2ab4e14b2
diff --git a/scripts/base/frameworks/notice/cluster.bro b/scripts/base/frameworks/notice/cluster.bro
index f197761acf..3c3fbc6d36 100644
--- a/scripts/base/frameworks/notice/cluster.bro
+++ b/scripts/base/frameworks/notice/cluster.bro
@@ -23,7 +23,8 @@ redef Cluster::worker2manager_events += /Notice::cluster_notice/;
@if ( Cluster::local_node_type() != Cluster::MANAGER )
event Notice::begin_suppression(n: Notice::Info)
{
- suppressing[n$note, n$identifier] = n;
+ local suppress_until = n$ts + n$suppress_for;
+ suppressing[n$note, n$identifier] = suppress_until;
}
@endif
diff --git a/scripts/base/frameworks/notice/main.bro b/scripts/base/frameworks/notice/main.bro
index a5f17a4979..bbd4643b3d 100644
--- a/scripts/base/frameworks/notice/main.bro
+++ b/scripts/base/frameworks/notice/main.bro
@@ -242,12 +242,6 @@ export {
## being suppressed.
global suppressed: event(n: Notice::Info);
- ## This event is generated when a notice stops being suppressed.
- ##
- ## n: The record containing notice data regarding the notice type
- ## that was being suppressed.
- global end_suppression: event(n: Notice::Info);
-
## Call this function to send a notice in an email. It is already used
## by default with the built in :bro:enum:`Notice::ACTION_EMAIL` and
## :bro:enum:`Notice::ACTION_PAGE` actions.
@@ -285,27 +279,22 @@ export {
}
# This is used as a hack to implement per-item expiration intervals.
-function per_notice_suppression_interval(t: table[Notice::Type, string] of Notice::Info, idx: any): interval
+function per_notice_suppression_interval(t: table[Notice::Type, string] of time, idx: any): interval
{
local n: Notice::Type;
local s: string;
[n,s] = idx;
- local suppress_time = t[n,s]$suppress_for - (network_time() - t[n,s]$ts);
+ local suppress_time = t[n,s] - network_time();
if ( suppress_time < 0secs )
suppress_time = 0secs;
- # If there is no more suppression time left, the notice needs to be sent
- # to the end_suppression event.
- if ( suppress_time == 0secs )
- event Notice::end_suppression(t[n,s]);
-
return suppress_time;
}
# This is the internally maintained notice suppression table. It's
# indexed on the Notice::Type and the $identifier field from the notice.
-global suppressing: table[Type, string] of Notice::Info = {}
+global suppressing: table[Type, string] of time = {}
&create_expire=0secs
&expire_func=per_notice_suppression_interval;
@@ -400,11 +389,22 @@ function email_notice_to(n: Notice::Info, dest: string, extend: bool)
# First off, finish the headers and include the human readable messages
# then leave a blank line after the message.
- email_text = string_cat(email_text, "\nMessage: ", n$msg);
- if ( n?$sub )
- email_text = string_cat(email_text, "\nSub-message: ", n$sub);
+ email_text = string_cat(email_text, "\nMessage: ", n$msg, "\n");
- email_text = string_cat(email_text, "\n\n");
+ if ( n?$sub )
+ email_text = string_cat(email_text, "Sub-message: ", n$sub, "\n");
+
+ email_text = string_cat(email_text, "\n");
+
+ # Add information about the file if it exists.
+ if ( n?$file_desc )
+ email_text = string_cat(email_text, "File Description: ", n$file_desc, "\n");
+
+ if ( n?$file_mime_type )
+ email_text = string_cat(email_text, "File MIME Type: ", n$file_mime_type, "\n");
+
+ if ( n?$file_desc || n?$file_mime_type )
+ email_text = string_cat(email_text, "\n");
# Next, add information about the connection if it exists.
if ( n?$id )
@@ -467,7 +467,8 @@ hook Notice::notice(n: Notice::Info) &priority=-5
[n$note, n$identifier] !in suppressing &&
n$suppress_for != 0secs )
{
- suppressing[n$note, n$identifier] = n;
+ local suppress_until = n$ts + n$suppress_for;
+ suppressing[n$note, n$identifier] = suppress_until;
event Notice::begin_suppression(n);
}
}
diff --git a/scripts/base/frameworks/reporter/main.bro b/scripts/base/frameworks/reporter/main.bro
index 3668348ef1..873cb15a45 100644
--- a/scripts/base/frameworks/reporter/main.bro
+++ b/scripts/base/frameworks/reporter/main.bro
@@ -9,7 +9,7 @@
##! Note that this framework deals with the handling of internally generated
##! reporter messages, for the interface
##! into actually creating reporter messages from the scripting layer, use
-##! the built-in functions in :doc:`/scripts/base/bif/reporter.bif`.
+##! the built-in functions in :doc:`/scripts/base/bif/reporter.bif.bro`.
module Reporter;
diff --git a/scripts/base/frameworks/software/main.bro b/scripts/base/frameworks/software/main.bro
index 5080c9f888..c8f413a8f2 100644
--- a/scripts/base/frameworks/software/main.bro
+++ b/scripts/base/frameworks/software/main.bro
@@ -209,7 +209,7 @@ function parse_mozilla(unparsed_version: string): Description
if ( 2 in parts )
v = parse(parts[2])$version;
}
- else if ( / MSIE / in unparsed_version )
+ else if ( / MSIE |Trident\// in unparsed_version )
{
software_name = "MSIE";
if ( /Trident\/4\.0/ in unparsed_version )
@@ -218,6 +218,8 @@ function parse_mozilla(unparsed_version: string): Description
v = [$major=9,$minor=0];
else if ( /Trident\/6\.0/ in unparsed_version )
v = [$major=10,$minor=0];
+ else if ( /Trident\/7\.0/ in unparsed_version )
+ v = [$major=11,$minor=0];
else
{
parts = split_all(unparsed_version, /MSIE [0-9]{1,2}\.*[0-9]*b?[0-9]*/);
diff --git a/scripts/base/init-bare.bro b/scripts/base/init-bare.bro
index 928697edb0..d551fe3148 100644
--- a/scripts/base/init-bare.bro
+++ b/scripts/base/init-bare.bro
@@ -406,7 +406,7 @@ type NetStats: record {
pkts_dropped: count &default=0; ##< Packets reported dropped by the system.
## Packets seen on the link. Note that this may differ
## from *pkts_recvd* because of a potential capture_filter. See
- ## :doc:`/scripts/base/frameworks/packet-filter/main`. Depending on the
+ ## :doc:`/scripts/base/frameworks/packet-filter/main.bro`. Depending on the
## packet capture system, this value may not be available and will then
## be always set to zero.
pkts_link: count &default=0;
@@ -514,7 +514,7 @@ type script_id: record {
## directly and then remove this alias.
type id_table: table[string] of script_id;
-## Meta-information about a record-field.
+## Meta-information about a record field.
##
## .. bro:see:: record_fields record_field_table
type record_field: record {
@@ -536,6 +536,25 @@ type record_field: record {
## directly and then remove this alias.
type record_field_table: table[string] of record_field;
+## Meta-information about a parameter to a function/event.
+##
+## .. bro:see:: call_argument_vector new_event
+type call_argument: record {
+ name: string; ##< The name of the parameter.
+ type_name: string; ##< The name of the parameters's type.
+ default_val: any &optional; ##< The value of the :bro:attr:`&default` attribute if defined.
+
+ ## The value of the parameter as passed into a given call instance.
+ ## Might be unset in the case a :bro:attr:`&default` attribute is
+ ## defined.
+ value: any &optional;
+};
+
+## Vector type used to capture parameters of a function/event call.
+##
+## .. bro:see:: call_argument new_event
+type call_argument_vector: vector of call_argument;
+
# todo:: Do we still need these here? Can they move into the packet filter
# framework?
#
@@ -2775,13 +2794,13 @@ const log_max_size = 0.0 &redef;
const log_encryption_key = "" &redef;
## Write profiling info into this file in regular intervals. The easiest way to
-## activate profiling is loading :doc:`/scripts/policy/misc/profiling`.
+## activate profiling is loading :doc:`/scripts/policy/misc/profiling.bro`.
##
## .. bro:see:: profiling_interval expensive_profiling_multiple segment_profiling
global profiling_file: file &redef;
## Update interval for profiling (0 disables). The easiest way to activate
-## profiling is loading :doc:`/scripts/policy/misc/profiling`.
+## profiling is loading :doc:`/scripts/policy/misc/profiling.bro`.
##
## .. bro:see:: profiling_file expensive_profiling_multiple segment_profiling
const profiling_interval = 0 secs &redef;
@@ -3045,6 +3064,9 @@ export {
## Toggle whether to do GTPv1 decapsulation.
const enable_gtpv1 = T &redef;
+ ## Toggle whether to do GRE decapsulation.
+ const enable_gre = T &redef;
+
## With this option set, the Teredo analysis will first check to see if
## other protocol analyzers have confirmed that they think they're
## parsing the right protocol and only continue with Teredo tunnel
@@ -3070,7 +3092,8 @@ export {
## may work better.
const delay_gtp_confirmation = F &redef;
- ## How often to cleanup internal state for inactive IP tunnels.
+ ## How often to cleanup internal state for inactive IP tunnels
+ ## (includes GRE tunnels).
const ip_tunnel_timeout = 24hrs &redef;
} # end export
module GLOBAL;
diff --git a/scripts/base/protocols/dhcp/main.bro b/scripts/base/protocols/dhcp/main.bro
index e5be50b1ab..c94bae0476 100644
--- a/scripts/base/protocols/dhcp/main.bro
+++ b/scripts/base/protocols/dhcp/main.bro
@@ -4,7 +4,7 @@
##!
##! If you'd like to track known DHCP devices and to log the hostname
##! supplied by the client, see
-##! :doc:`/scripts/policy/protocols/dhcp/known-devices-and-hostnames`.
+##! :doc:`/scripts/policy/protocols/dhcp/known-devices-and-hostnames.bro`.
@load ./utils.bro
diff --git a/scripts/base/protocols/http/__load__.bro b/scripts/base/protocols/http/__load__.bro
index 20fcd5f1ec..42a45efb36 100644
--- a/scripts/base/protocols/http/__load__.bro
+++ b/scripts/base/protocols/http/__load__.bro
@@ -3,4 +3,4 @@
@load ./utils
@load ./files
-@load-sigs ./dpd.sig
\ No newline at end of file
+@load-sigs ./dpd.sig
diff --git a/scripts/base/protocols/smtp/main.bro b/scripts/base/protocols/smtp/main.bro
index fb3095c121..d3bd8a97b4 100644
--- a/scripts/base/protocols/smtp/main.bro
+++ b/scripts/base/protocols/smtp/main.bro
@@ -291,7 +291,7 @@ function describe(rec: Info): string
{
if ( |rec$subject| > 20 )
{
- abbrev_subject = rec$subject[0:20] + "...";
+ abbrev_subject = rec$subject[0:21] + "...";
}
}
diff --git a/scripts/base/protocols/ssl/main.bro b/scripts/base/protocols/ssl/main.bro
index 9eab0ca970..3958a90fa2 100644
--- a/scripts/base/protocols/ssl/main.bro
+++ b/scripts/base/protocols/ssl/main.bro
@@ -153,7 +153,7 @@ function finish(c: connection)
disable_analyzer(c$id, c$ssl$analyzer_id);
}
-event ssl_client_hello(c: connection, version: count, possible_ts: time, client_random: string, session_id: string, ciphers: count_set) &priority=5
+event ssl_client_hello(c: connection, version: count, possible_ts: time, client_random: string, session_id: string, ciphers: index_vec) &priority=5
{
set_session(c);
diff --git a/scripts/broxygen/README b/scripts/broxygen/README
new file mode 100644
index 0000000000..ac7f522285
--- /dev/null
+++ b/scripts/broxygen/README
@@ -0,0 +1,4 @@
+This package is loaded during the process which automatically generates
+reference documentation for all Bro scripts (i.e. "Broxygen"). Its only
+purpose is to provide an easy way to load all known Bro scripts plus any
+extra scripts needed or used by the documentation process.
diff --git a/scripts/broxygen/__load__.bro b/scripts/broxygen/__load__.bro
new file mode 100644
index 0000000000..8db4a7c1b8
--- /dev/null
+++ b/scripts/broxygen/__load__.bro
@@ -0,0 +1,15 @@
+@load test-all-policy.bro
+
+# Scripts which are commented out in test-all-policy.bro.
+@load protocols/ssl/notary.bro
+@load frameworks/communication/listen.bro
+@load frameworks/control/controllee.bro
+@load frameworks/control/controller.bro
+@load policy/misc/dump-events.bro
+
+@load ./example.bro
+
+event bro_init()
+ {
+ terminate();
+ }
diff --git a/scripts/broxygen/example.bro b/scripts/broxygen/example.bro
new file mode 100644
index 0000000000..65cc5ff1c7
--- /dev/null
+++ b/scripts/broxygen/example.bro
@@ -0,0 +1,194 @@
+##! This is an example script that demonstrates Broxygen-style
+##! documentation. It generally will make most sense when viewing
+##! the script's raw source code and comparing to the HTML-rendered
+##! version.
+##!
+##! Comments in the from ``##!`` are meant to summarize the script's
+##! purpose. They are transferred directly in to the generated
+##! `reStructuredText `_
+##! (reST) document associated with the script.
+##!
+##! .. tip:: You can embed directives and roles within ``##``-stylized comments.
+##!
+##! There's also a custom role to reference any identifier node in
+##! the Bro Sphinx domain that's good for "see alsos", e.g.
+##!
+##! See also: :bro:see:`BroxygenExample::a_var`,
+##! :bro:see:`BroxygenExample::ONE`, :bro:see:`SSH::Info`
+##!
+##! And a custom directive does the equivalent references:
+##!
+##! .. bro:see:: BroxygenExample::a_var BroxygenExample::ONE SSH::Info
+
+# Comments that use a single pound sign (#) are not significant to
+# a script's auto-generated documentation, but ones that use a
+# double pound sign (##) do matter. In some cases, like record
+# field comments, it's necessary to disambiguate the field with
+# which a comment associates: e.g. "##<" can be used on the same line
+# as a field to signify the comment relates to it and not the
+# following field. "##<" can also be used more generally in any
+# variable declarations to associate with the last-declared identifier.
+#
+# Generally, the auto-doc comments (##) are associated with the
+# next declaration/identifier found in the script, but Broxygen
+# will track/render identifiers regardless of whether they have any
+# of these special comments associated with them.
+#
+# The first sentence contained within the "##"-stylized comments for
+# a given identifier is special in that it will be used as summary
+# text in a table containing all such identifiers and short summaries.
+# If there are no sentences (text terminated with '.'), then everything
+# in the "##"-stylized comments up until the first empty comment
+# is taken as the summary text for a given identifier.
+
+# @load directives are self-documenting, don't use any ``##`` style
+# comments with them.
+@load base/frameworks/notice
+@load base/protocols/http
+@load frameworks/software/vulnerable
+
+# "module" statements are self-documenting, don't use any ``##`` style
+# comments with them.
+module BroxygenExample;
+
+# Redefinitions of "Notice::Type" are self-documenting, but
+# more information can be supplied in two different ways.
+redef enum Notice::Type += {
+ ## Any number of this type of comment
+ ## will document "Broxygen_One".
+ Broxygen_One,
+ Broxygen_Two, ##< Any number of this type of comment
+ ##< will document "BROXYGEN_TWO".
+ Broxygen_Three,
+ ## Omitting comments is fine, and so is mixing ``##`` and ``##<``, but
+ Broxygen_Four, ##< it's probably best to use only one style consistently.
+};
+
+# All redefs are automatically tracked. Comments of the "##" form can be use
+# to further document it, but in some cases, like here, they wouldn't be
+# ading any interesting information that's not implicit.
+redef enum Log::ID += { LOG };
+
+# Only identifiers declared in an export section will show up in generated docs.
+
+export {
+
+ ## Documentation for the "SimpleEnum" type goes here.
+ ## It can span multiple lines.
+ type SimpleEnum: enum {
+ ## Documentation for particular enum values is added like this.
+ ## And can also span multiple lines.
+ ONE,
+ TWO, ##< Or this style is valid to document the preceding enum value.
+ THREE,
+ };
+
+ ## Document the "SimpleEnum" redef here with any special info regarding
+ ## the *redef* itself.
+ redef enum SimpleEnum += {
+ FOUR, ##< And some documentation for "FOUR".
+ ## Also "FIVE".
+ FIVE
+ };
+
+ ## General documentation for a type "SimpleRecord" goes here.
+ ## The way fields can be documented is similar to what's already seen
+ ## for enums.
+ type SimpleRecord: record {
+ ## Counts something.
+ field1: count;
+ field2: bool; ##< Toggles something.
+ };
+
+ ## Document the record extension *redef* itself here.
+ redef record SimpleRecord += {
+ ## Document the extending field like this.
+ field_ext: string &optional; ##< Or here, like this.
+ };
+
+ ## General documentation for a type "ComplexRecord" goes here.
+ type ComplexRecord: record {
+ field1: count; ##< Counts something.
+ field2: bool; ##< Toggles something.
+ field3: SimpleRecord; ##< Broxygen automatically tracks types
+ ##< and cross-references are automatically
+ ##< inserted in to generated docs.
+ msg: string &default="blah"; ##< Attributes are self-documenting.
+ } &redef;
+
+ ## An example record to be used with a logging stream.
+ ## Nothing special about it. If another script redefs this type
+ ## to add fields, the generated documentation will show all original
+ ## fields plus the extensions and the scripts which contributed to it
+ ## (provided they are also @load'ed).
+ type Info: record {
+ ts: time &log;
+ uid: string &log;
+ status: count &log &optional;
+ };
+
+ ## Add documentation for "an_option" here.
+ ## The type/attribute information is all generated automatically.
+ const an_option: set[addr, addr, string] &redef;
+
+ ## Default initialization will be generated automatically.
+ const option_with_init = 0.01 secs &redef; ##< More docs can be added here.
+
+ ## Put some documentation for "a_var" here. Any global/non-const that
+ ## isn't a function/event/hook is classified as a "state variable"
+ ## in the generated docs.
+ global a_var: bool;
+
+ ## Types are inferred, that information is self-documenting.
+ global var_without_explicit_type = "this works";
+
+ ## The first sentence for a particular identifier's summary text ends here.
+ ## And this second sentence doesn't show in the short description provided
+ ## by the table of all identifiers declared by this script.
+ global summary_test: string;
+
+ ## Summarize purpose of "a_function" here.
+ ## Give more details about "a_function" here.
+ ## Separating the documentation of the params/return values with
+ ## empty comments is optional, but improves readability of script.
+ ##
+ ## tag: Function arguments can be described
+ ## like this.
+ ##
+ ## msg: Another param.
+ ##
+ ## Returns: Describe the return type here.
+ global a_function: function(tag: string, msg: string): string;
+
+ ## Summarize "an_event" here.
+ ## Give more details about "an_event" here.
+ ##
+ ## BroxygenExample::a_function should not be confused as a parameter
+ ## in the generated docs, but it also doesn't generate a cross-reference
+ ## link. Use the see role instead: :bro:see:`BroxygenExample::a_function`.
+ ##
+ ## name: Describe the argument here.
+ global an_event: event(name: string);
+}
+
+# This function isn't exported, so it won't appear anywhere in the generated
+# documentation. So using ``##``-style comments is pointless here.
+function function_without_proto(tag: string): string
+ {
+ return "blah";
+ }
+
+# Same thing goes for types -- it's not exported, so it's considered
+# private to this script and comments are only interesting to a person
+# who is already reading the raw source for the script (so don't use
+# ``##`` comments here.
+type PrivateRecord: record {
+ field1: bool;
+ field2: count;
+};
+
+# Event handlers are also an implementation detail of a script, so they
+# don't show up anywhere in the generated documentation.
+event bro_init()
+ {
+ }
diff --git a/scripts/policy/misc/dump-events.bro b/scripts/policy/misc/dump-events.bro
new file mode 100644
index 0000000000..a0c36753cb
--- /dev/null
+++ b/scripts/policy/misc/dump-events.bro
@@ -0,0 +1,40 @@
+##! This script dumps the events that Bro raises out to standard output in a
+##! readable form. This is for debugging only and allows to understand events and
+##! their parameters as Bro processes input. Note that it will show only events
+##! for which a handler is defined.
+
+module DumpEvents;
+
+export {
+ ## If true, include event arguments in output.
+ const include_args = T &redef;
+
+ ## Only include events matching the given pattern into output. By default, the
+ ## pattern matches all events.
+ const include = /.*/ &redef;
+}
+
+event new_event(name: string, args: call_argument_vector)
+ {
+ if ( include !in name )
+ return;
+
+ print fmt("%17.6f %s", network_time(), name);
+
+ if ( ! include_args || |args| == 0 )
+ return;
+
+ for ( i in args )
+ {
+ local a = args[i];
+
+ local proto = fmt("%s: %s", a$name, a$type_name);
+
+ if ( a?$value )
+ print fmt(" [%d] %-18s = %s", i, proto, a$value);
+ else
+ print fmt(" | %-18s = %s [default]", proto, a$value);
+ }
+
+ print "";
+ }
diff --git a/scripts/policy/misc/known-devices.bro b/scripts/policy/misc/known-devices.bro
index 16c5250d1c..8378d589f8 100644
--- a/scripts/policy/misc/known-devices.bro
+++ b/scripts/policy/misc/known-devices.bro
@@ -7,7 +7,7 @@
##!
##! This script will not generate any logs on its own, it needs to be
##! supplied with information from elsewhere, such as
-##! :doc:`/scripts/policy/protocols/dhcp/known-devices-and-hostnames`.
+##! :doc:`/scripts/policy/protocols/dhcp/known-devices-and-hostnames.bro`.
module Known;
diff --git a/scripts/policy/misc/scan.bro b/scripts/policy/misc/scan.bro
index e458f6c450..d70f8f9e79 100644
--- a/scripts/policy/misc/scan.bro
+++ b/scripts/policy/misc/scan.bro
@@ -147,11 +147,6 @@ function is_reverse_failed_conn(c: connection): bool
return F;
}
-## Generated for an unsuccessful connection attempt. This
-## event is raised when an originator unsuccessfully attempted
-## to establish a connection. "Unsuccessful" is defined as at least
-## tcp_attempt_delay seconds having elapsed since the originator first sent a
-## connection establishment packet to the destination without seeing a reply.
event connection_attempt(c: connection)
{
local is_reverse_scan = F;
@@ -161,9 +156,6 @@ event connection_attempt(c: connection)
add_sumstats(c$id, is_reverse_scan);
}
-## Generated for a rejected TCP connection. This event is raised when an
-## originator attempted to setup a TCP connection but the responder replied with
-## a RST packet denying it.
event connection_rejected(c: connection)
{
local is_reverse_scan = F;
@@ -173,9 +165,6 @@ event connection_rejected(c: connection)
add_sumstats(c$id, is_reverse_scan);
}
-## Generated when an endpoint aborted a TCP connection. The event is raised when
-## one endpoint of an *established* TCP connection aborted by sending a RST
-## packet.
event connection_reset(c: connection)
{
if ( is_failed_conn(c) )
@@ -184,7 +173,6 @@ event connection_reset(c: connection)
add_sumstats(c$id, T);
}
-## Generated for each still-open connection when Bro terminates.
event connection_pending(c: connection)
{
if ( is_failed_conn(c) )
diff --git a/scripts/policy/misc/stats.bro b/scripts/policy/misc/stats.bro
index 7e1e4b6689..eb1ddb0202 100644
--- a/scripts/policy/misc/stats.bro
+++ b/scripts/policy/misc/stats.bro
@@ -1,5 +1,5 @@
##! Log memory/packet/lag statistics. Differs from
-##! :doc:`/scripts/policy/misc/profiling` in that this
+##! :doc:`/scripts/policy/misc/profiling.bro` in that this
##! is lighter-weight (much less info, and less load to generate).
@load base/frameworks/notice
diff --git a/scripts/policy/protocols/ssl/validate-certs.bro b/scripts/policy/protocols/ssl/validate-certs.bro
index b34ec5a09a..886c28b6ac 100644
--- a/scripts/policy/protocols/ssl/validate-certs.bro
+++ b/scripts/policy/protocols/ssl/validate-certs.bro
@@ -40,6 +40,7 @@ event ssl_established(c: connection) &priority=3
{
local result = x509_verify(c$ssl$cert, c$ssl$cert_chain, root_certs);
c$ssl$validation_status = x509_err2str(result);
+ recently_validated_certs[c$ssl$cert_hash] = c$ssl$validation_status;
}
if ( c$ssl$validation_status != "ok" )
diff --git a/scripts/test-all-policy.bro b/scripts/test-all-policy.bro
index d6383af38b..3a0bd17614 100644
--- a/scripts/test-all-policy.bro
+++ b/scripts/test-all-policy.bro
@@ -48,6 +48,7 @@
@load misc/capture-loss.bro
@load misc/detect-traceroute/__load__.bro
@load misc/detect-traceroute/main.bro
+# @load misc/dump-events.bro
@load misc/known-devices.bro
@load misc/load-balancing.bro
@load misc/loaded-scripts.bro
diff --git a/src/3rdparty b/src/3rdparty
index 12b5cb446c..42a4c9694a 160000
--- a/src/3rdparty
+++ b/src/3rdparty
@@ -1 +1 @@
-Subproject commit 12b5cb446c8128bb22e5cbd7baa7d53669539487
+Subproject commit 42a4c9694a2b2677b050fbb7cbae26bc5ec4605a
diff --git a/src/Attr.cc b/src/Attr.cc
index 244d1e1687..d6d0f6e68d 100644
--- a/src/Attr.cc
+++ b/src/Attr.cc
@@ -317,8 +317,9 @@ void Attributes::CheckAttr(Attr* a)
break;
// Table defaults may be promotable.
- if ( (ytype->Tag() == TYPE_RECORD && atype->Tag() == TYPE_RECORD &&
- record_promotion_compatible(atype->AsRecordType(), ytype->AsRecordType())) )
+ if ( ytype && ytype->Tag() == TYPE_RECORD &&
+ atype->Tag() == TYPE_RECORD &&
+ record_promotion_compatible(atype->AsRecordType(), ytype->AsRecordType()) )
// Ok.
break;
diff --git a/src/BroDoc.h b/src/BroDoc.h
deleted file mode 100644
index 081df698d9..0000000000
--- a/src/BroDoc.h
+++ /dev/null
@@ -1,422 +0,0 @@
-#ifndef brodoc_h
-#define brodoc_h
-
-#include
-#include
-#include
-#include
-
-#include "BroDocObj.h"
-
-/**
- * This class is used to gather all data relevant to the automatic generation
- * of a reStructuredText (reST) document from a given Bro script.
- */
-class BroDoc {
-public:
- /**
- * BroDoc constructor
- * Given a Bro script, opens new file in the current working directory
- * that will contain reST documentation generated from the parsing
- * of the Bro script. The new reST file will be named similar to
- * the filename of the Bro script that generates it, except any
- * ".bro" file extension is stripped and ".rst" takes it place.
- * If the filename doesn't end in ".bro", then ".rst" is just appended.
- * Any '/' characters in the reST file name that result from choice of
- * the 'rel' parameter are replaced with '^'.
- * @param rel A string representing a subpath of the root Bro script
- * source/install directory in which the source file is located.
- * It can also be an absolute path, but then the parameter is
- * ignored and the document title is just derived from file name
- * @param abs The absolute path to the Bro script for which to generate
- * documentation.
- */
- BroDoc(const std::string& rel, const std::string& abs);
-
- /**
- * BroDoc destructor
- * Closes the file that was opened by the constructor and frees up
- * memory taken by BroDocObj objects.
- */
- virtual ~BroDoc();
-
- /**
- * Write out full reST documentation for the Bro script that was parsed.
- * BroDoc's default implementation of this function will care
- * about whether declarations made in the Bro script are part of
- * the public versus private interface (whether things are declared in
- * the export section).
- */
- virtual void WriteDocFile() const;
-
- /**
- * Schedules some summarizing text to be output directly into the reST doc.
- * This should be called whenever the scanner sees a line in the Bro script
- * starting with "##!"
- * @param s The summary text to add to the reST doc.
- */
- void AddSummary(const std::string& s) { summary.push_back(s); }
-
- /**
- * Schedules an import (@load) to be documented.
- * If the script being loaded has a .bro suffix, it is internally stripped.
- * This should be called whenever the scanner sees an @load.
- * @param s The name of the imported script.
- */
- void AddImport(const std::string& s);
-
- /**
- * Schedules a namespace (module) to be documented.
- * This should be called whenever the parser sees a TOK_MODULE.
- * @param s The namespace (module) identifier's name.
- */
- void AddModule(const std::string& s) { modules.push_back(s); }
-
- /**
- * Sets the way the script changes the "capture_filters" table.
- * This is determined by the scanner checking for changes to
- * the "capture_filters" table after each of Bro's input scripts
- * (given as command line arguments to Bro) are finished being parsed.
- * @param s The value "capture_filters" as given by TableVal::Describe()
- */
- void SetPacketFilter(const std::string& s);
-
- /**
- * Schedules documentation of a script option. An option is
- * defined as any variable in the script that is declared 'const'
- * and has the '&redef' attribute.
- * @param o A pointer to a BroDocObj which contains the internal
- * Bro language representation of the script option and
- * also any associated comments about it.
- */
- void AddOption(const BroDocObj* o)
- {
- options.push_back(o);
- all.push_back(o);
- }
-
- /**
- * Schedules documentation of a script constant. An option is
- * defined as any variable in the script that is declared 'const'
- * and does *not* have the '&redef' attribute.
- * @param o A pointer to a BroDocObj which contains the internal
- * Bro language representation of the script constant and
- * also any associated comments about it.
- */
- void AddConstant(const BroDocObj* o)
- {
- constants.push_back(o);
- all.push_back(o);
- }
-
- /**
- * Schedules documentation of a script state variable. A state variable
- * is defined as any variable in the script that is declared 'global'
- * @param o A pointer to a BroDocObj which contains the internal
- * Bro language representation of the script state variable
- * and also any associated comments about it.
- */
- void AddStateVar(const BroDocObj* o)
- {
- state_vars.push_back(o);
- all.push_back(o);
- }
-
- /**
- * Schedules documentation of a type declared by the script.
- * @param o A pointer to a BroDocObj which contains the internal
- * Bro language representation of the script option and
- * also any associated comments about it.
- */
- void AddType(const BroDocObj* o)
- {
- types.push_back(o);
- all.push_back(o);
- }
-
- /**
- * Schedules documentation of a Notice (enum redef) declared by script
- * @param o A pointer to a BroDocObj which contains the internal
- * Bro language representation of the Notice and also
- * any associated comments about it.
- */
- void AddNotice(const BroDocObj* o)
- {
- notices.push_back(o);
- all.push_back(o);
- }
-
- /**
- * Schedules documentation of an event declared by the script.
- * @param o A pointer to a BroDocObj which contains the internal
- * Bro language representation of the script event and
- * also any associated comments about it.
- */
- void AddEvent(const BroDocObj* o)
- {
- events.push_back(o);
- all.push_back(o);
- }
-
- /**
- * Schedules documentation of an event handler declared by the script.
- * @param o A pointer to a BroDocObj which contains the internal
- * Bro language representation of the script event handler and
- * also any associated comments about it.
- */
- void AddEventHandler(const BroDocObj* o)
- {
- event_handlers.push_back(o);
- all.push_back(o);
- }
-
- /**
- * Schedules documentation of a hook declared by the script.
- * @param o A pointer to a BroDocObj which contains the internal
- * Bro language representation of the script hook and
- * also any associated comments about it.
- */
- void AddHook(const BroDocObj* o)
- {
- hooks.push_back(o);
- all.push_back(o);
- }
-
- /**
- * Schedules documentation of a hook handler declared by the script.
- * @param o A pointer to a BroDocObj which contains the internal
- * Bro language representation of the script hook handler and
- * also any associated comments about it.
- */
- void AddHookHandler(const BroDocObj* o)
- {
- hook_handlers.push_back(o);
- all.push_back(o);
- }
-
- /**
- * Schedules documentation of a function declared by the script.
- * @param o A pointer to a BroDocObj which contains the internal
- * Bro language representation of the script function and
- * also any associated comments about it.
- */
- void AddFunction(BroDocObj* o);
-
- /**
- * Schedules documentation of a redef done by the script
- * @param o A pointer to a BroDocObj which contains the internal
- * Bro language representation of the script identifier
- * that was redefined and also any associated comments.
- */
- void AddRedef(const BroDocObj* o)
- {
- redefs.push_back(o);
- all.push_back(o);
- }
-
- /**
- * Gets the name of the Bro script source file for which reST
- * documentation is being generated.
- * @return A char* to the start of the source file's name.
- */
- const char* GetSourceFileName() const
- {
- return source_filename.c_str();
- }
-
- /**
- * Gets the name of the generated reST documentation file.
- * @return A char* to the start of the generated reST file's name.
- */
- const char* GetOutputFileName() const
- {
- return reST_filename.c_str();
- }
-
- typedef std::list BroDocObjList;
- typedef std::map BroDocObjMap;
-
- /**
- * Writes out a table of BroDocObj's to the reST document
- * @param f The file to write to.
- * @param l A list of BroDocObj pointers
- */
- static void WriteBroDocObjTable(FILE* f, const BroDocObjList& l);
-
- /**
- * Writes out given number of characters to reST document
- * @param f The file to write to.
- * @param c the character to write
- * @param n the number of characters to write
- */
- static void WriteRepeatedChar(FILE* f, char c, size_t n);
-
- /**
- * A wrapper to fprintf() that always uses the reST document
- * for the FILE* argument.
- * @param f The file to write to.
- * @param format A printf style format string.
- */
- static void WriteToDoc(FILE* f, const char* format, ...);
-
- /**
- * Writes out a list of strings to the reST document.
- * If the list is empty, prints a newline character.
- * @param f The file to write to.
- * @param format A printf style format string for elements of the list
- * except for the last one in the list
- * @param last_format A printf style format string to use for the last
- * element of the list
- * @param l A reference to a list of strings
- */
- static void WriteStringList(FILE* f, const char* format, const char* last_format,
- const std::list& l);
-
- /**
- * @see WriteStringList(FILE* f, const char*, const char*,
- * const std::list&>)
- */
- static void WriteStringList(FILE* f, const char* format,
- const std::list& l){
- WriteStringList(f, format, format, l);
- }
-
- /**
- * Writes out a list of BroDocObj objects to the reST document
- * @param f The file to write to.
- * @param l A list of BroDocObj pointers
- * @param wantPublic If true, filter out objects that are not declared
- * in the global scope. If false, filter out those that are in
- * the global scope.
- * @param heading The title of the section to create in the reST doc.
- * @param underline The character to use to underline the reST
- * section heading.
- * @param isShort Whether to write the full documentation or a "short"
- * version (a single sentence)
- */
- static void WriteBroDocObjList(FILE* f, const BroDocObjList& l, bool wantPublic,
- const char* heading, char underline,
- bool isShort);
-
- /**
- * Wraps the BroDocObjMap into a BroDocObjList and the writes that list
- * to the reST document
- * @see WriteBroDocObjList(FILE* f, const BroDocObjList&, bool, const char*, char,
- bool)
- */
- static void WriteBroDocObjList(FILE* f, const BroDocObjMap& m, bool wantPublic,
- const char* heading, char underline,
- bool isShort);
-
- /**
- * Writes out a list of BroDocObj objects to the reST document
- * @param l A list of BroDocObj pointers
- * @param heading The title of the section to create in the reST doc.
- * @param underline The character to use to underline the reST
- * section heading.
- */
- static void WriteBroDocObjList(FILE* f, const BroDocObjList& l, const char* heading,
- char underline);
-
- /**
- * Writes out a list of BroDocObj objects to the reST document
- * @param l A list of BroDocObj pointers
- */
- static void WriteBroDocObjList(FILE* f, const BroDocObjList& l);
-
- /**
- * Wraps the BroDocObjMap into a BroDocObjList and the writes that list
- * to the reST document
- * @see WriteBroDocObjList(FILE* f, const BroDocObjList&, const char*, char)
- */
- static void WriteBroDocObjList(FILE* f, const BroDocObjMap& m, const char* heading,
- char underline);
-
- /**
- * Writes out a reST section heading
- * @param f The file to write to.
- * @param heading The title of the heading to create
- * @param underline The character to use to underline the section title
- * within the reST document
- */
- static void WriteSectionHeading(FILE* f, const char* heading, char underline);
-
-private:
- FILE* reST_file;
- std::string reST_filename;
- std::string source_filename; // points to the basename of source file
- std::string downloadable_filename; // file that will be linked for download
- std::string doc_title;
- std::string packet_filter;
-
- std::list modules;
- std::list summary;
- std::list imports;
- std::list port_analysis;
-
- BroDocObjList options;
- BroDocObjList constants;
- BroDocObjList state_vars;
- BroDocObjList types;
- BroDocObjList notices;
- BroDocObjList events;
- BroDocObjList event_handlers;
- BroDocObjList hooks;
- BroDocObjList hook_handlers;
- BroDocObjMap functions;
- BroDocObjList redefs;
-
- BroDocObjList all;
-
- /**
- * Writes out the reST for either the script's public or private interface
- * @param heading The title of the interfaces section heading
- * @param underline The underline character to use for the interface
- * section
- * @param subunderline The underline character to use for interface
- * sub-sections
- * @param isPublic Whether to write out the public or private script
- * interface
- * @param isShort Whether to write out the full documentation or a "short"
- * description (a single sentence)
- */
- void WriteInterface(const char* heading, char underline, char subunderline,
- bool isPublic, bool isShort) const;
-
- /**
- * Frees memory allocated to BroDocObj's objects in a given list.
- * @param a reference to a list of BroDocObj pointers
- */
- void FreeBroDocObjPtrList(BroDocObjList& l);
-
- static bool IsPublicAPI(const BroDocObj* o)
- {
- return o->IsPublicAPI();
- }
-
- static bool IsPrivateAPI(const BroDocObj* o)
- {
- return ! o->IsPublicAPI();
- }
-
- struct replace_slash {
- void operator()(char& c)
- {
- if ( c == '/' ) c = '^';
- }
- };
-};
-
-/**
- * Writes out plugin index documentation for all analyzer plugins.
- * @param filename the name of the file to write.
- */
-void CreateProtoAnalyzerDoc(const char* filename);
-
-/**
- * Writes out plugin index documentation for all file analyzer plugins.
- * @param filename the name of the file to write.
- */
-void CreateFileAnalyzerDoc(const char* filename);
-
-#endif
diff --git a/src/BroDocObj.cc b/src/BroDocObj.cc
deleted file mode 100644
index 063bdb1941..0000000000
--- a/src/BroDocObj.cc
+++ /dev/null
@@ -1,195 +0,0 @@
-#include
-#include
-#include
-#include "ID.h"
-#include "BroDocObj.h"
-
-map doc_ids = map();
-
-BroDocObj* BroDocObj::last = 0;
-
-BroDocObj::BroDocObj(const ID* id, std::list*& reST,
- bool is_fake)
- {
- last = this;
- broID = id;
- reST_doc_strings = reST;
- reST = 0;
- is_fake_id = is_fake;
- use_role = 0;
- FormulateShortDesc();
- doc_ids[id->Name()] = this;
- }
-
-BroDocObj::~BroDocObj()
- {
- if ( reST_doc_strings )
- delete reST_doc_strings;
-
- if ( is_fake_id )
- delete broID;
- }
-
-void BroDocObj::WriteReSTCompact(FILE* file, int max_col) const
- {
- ODesc desc;
- desc.SetQuotes(1);
- broID->DescribeReSTShort(&desc);
-
- fprintf(file, "%s", desc.Description());
-
- std::list::const_iterator it;
-
- for ( it = short_desc.begin(); it != short_desc.end(); ++it )
- {
- int start_col;
-
- if ( it == short_desc.begin() )
- start_col = max_col - desc.Len() + 1;
- else
- {
- start_col = max_col + 1;
- fprintf(file, "\n");
- }
-
- for ( int i = 0; i < start_col; ++i )
- fprintf(file, " ");
-
- fprintf(file, "%s", it->c_str());
- }
- }
-
-int BroDocObj::LongestShortDescLen() const
- {
- size_t max = 0;
-
- std::list::const_iterator it;
-
- for ( it = short_desc.begin(); it != short_desc.end(); ++it )
- {
- if ( it->size() > max )
- max = it->size();
- }
-
- return max;
- }
-
-static size_t end_of_first_sentence(string s)
- {
- size_t rval = 0;
-
- while ( (rval = s.find_first_of('.', rval)) != string::npos )
- {
- if ( rval == s.size() - 1 )
- // Period is at end of string.
- return rval;
-
- if ( isspace(s[rval + 1]) )
- // Period has a space after it.
- return rval;
-
- // Period has some non-space character after it, keep looking.
- ++rval;
- }
-
- return rval;
- }
-
-void BroDocObj::FormulateShortDesc()
- {
- if ( ! reST_doc_strings )
- return;
-
- short_desc.clear();
- std::list::const_iterator it;
-
- for ( it = reST_doc_strings->begin();
- it != reST_doc_strings->end(); ++it )
- {
- // The short description stops at the first sentence or the
- // first empty comment.
- size_t end = end_of_first_sentence(*it);
-
- if ( end == string::npos )
- {
- std::string::const_iterator s;
- bool empty = true;
-
- for ( s = it->begin(); s != it->end(); ++s )
- {
- if ( *s != ' ' && *s != '\t' && *s != '\n' && *s != '\r' )
- {
- empty = false;
- short_desc.push_back(*it);
- break;
- }
- }
-
- if ( empty )
- break;
- }
- else
- {
- short_desc.push_back(it->substr(0, end + 1));
- break;
- }
- }
- }
-
-void BroDocObj::WriteReST(FILE* file) const
- {
- int indent_spaces = 3;
- ODesc desc;
- desc.SetIndentSpaces(indent_spaces);
- desc.SetQuotes(1);
-
- broID->DescribeReST(&desc, use_role);
-
- fprintf(file, "%s", desc.Description());
-
- if ( HasDocumentation() )
- {
- fprintf(file, "\n");
- std::list::const_iterator it;
-
- for ( it = reST_doc_strings->begin();
- it != reST_doc_strings->end(); ++it)
- {
- for ( int i = 0; i < indent_spaces; ++i )
- fprintf(file, " ");
-
- fprintf(file, "%s\n", it->c_str());
- }
- }
-
- fprintf(file, "\n");
- }
-
-int BroDocObj::ColumnSize() const
- {
- ODesc desc;
- desc.SetQuotes(1);
- broID->DescribeReSTShort(&desc);
- return desc.Len();
- }
-
-bool BroDocObj::IsPublicAPI() const
- {
- return (broID->Scope() == SCOPE_GLOBAL) ||
- (broID->Scope() == SCOPE_MODULE && broID->IsExport());
- }
-
-void BroDocObj::Combine(const BroDocObj* o)
- {
- if ( o->reST_doc_strings )
- {
- if ( ! reST_doc_strings )
- reST_doc_strings = new std::list();
-
- reST_doc_strings->splice(reST_doc_strings->end(),
- *(o->reST_doc_strings));
- }
-
- delete o;
- FormulateShortDesc();
- }
diff --git a/src/BroDocObj.h b/src/BroDocObj.h
deleted file mode 100644
index ab42dc3c94..0000000000
--- a/src/BroDocObj.h
+++ /dev/null
@@ -1,143 +0,0 @@
-#ifndef brodocobj_h
-#define brodocobj_h
-
-#include
-#include
-#include
-#include