Merge remote-tracking branch 'origin/master' into topic/seth/smb

# Conflicts:
#	scripts/base/init-default.bro

.gitmodules

@@ -22,3 +22,6 @@
 [submodule "aux/plugins"]
 	path = aux/plugins
 	url = git://git.bro.org/bro-plugins
+[submodule "aux/broker"]
+	path = aux/broker
+	url = git://git.bro.org/broker

CMakeLists.txt

@@ -2,7 +2,7 @@ project(Bro C CXX)
 
 # When changing the minimum version here, also adapt
 # aux/bro-aux/plugin-support/skeleton/CMakeLists.txt
-cmake_minimum_required(VERSION 2.6.3 FATAL_ERROR)
+cmake_minimum_required(VERSION 2.8 FATAL_ERROR)
 
 include(cmake/CommonCMakeConfig.cmake)
 
@@ -15,6 +15,11 @@ if (NOT BRO_SCRIPT_INSTALL_PATH)
     set(BRO_SCRIPT_INSTALL_PATH ${BRO_ROOT_DIR}/share/bro)
 endif ()
 
+if (NOT BRO_MAN_INSTALL_PATH)
+    # set the default Bro man page installation path (user did not specify one)
+    set(BRO_MAN_INSTALL_PATH ${BRO_ROOT_DIR}/share/man)
+endif ()
+
 # sanitize the Bro script install directory into an absolute path
 # (CMake is confused by ~ as a representation of home directory)
 get_filename_component(BRO_SCRIPT_INSTALL_PATH ${BRO_SCRIPT_INSTALL_PATH}
@@ -26,12 +31,12 @@ configure_file(bro-path-dev.in ${CMAKE_CURRENT_BINARY_DIR}/bro-path-dev)
 
 file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/bro-path-dev.sh
      "export BROPATH=`${CMAKE_CURRENT_BINARY_DIR}/bro-path-dev`\n"
-     "export BRO_PLUGIN_PATH=\"${CMAKE_CURRENT_BINARY_DIR}/src:${BRO_PLUGIN_INSTALL_PATH}\"\n"
+     "export BRO_PLUGIN_PATH=\"${CMAKE_CURRENT_BINARY_DIR}/src\":${BRO_PLUGIN_PATH}\n"
      "export PATH=\"${CMAKE_CURRENT_BINARY_DIR}/src\":$PATH\n")
 
 file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/bro-path-dev.csh
      "setenv BROPATH `${CMAKE_CURRENT_BINARY_DIR}/bro-path-dev`\n"
-     "setenv BRO_PLUGIN_PATH \"${CMAKE_CURRENT_BINARY_DIR}/src:${BRO_PLUGIN_INSTALL_PATH}\"\n"
+     "setenv BRO_PLUGIN_PATH \"${CMAKE_CURRENT_BINARY_DIR}/src\":${BRO_PLUGIN_PATH}\n"
      "setenv PATH \"${CMAKE_CURRENT_BINARY_DIR}/src\":$PATH\n")
 
 file(STRINGS "${CMAKE_CURRENT_SOURCE_DIR}/VERSION" VERSION LIMIT_COUNT 1)
@@ -56,7 +61,7 @@ if (NOT SED_EXE)
     endif ()
 endif ()
 
-FindRequiredPackage(Perl)
+FindRequiredPackage(PythonInterp)
 FindRequiredPackage(FLEX)
 FindRequiredPackage(BISON)
 FindRequiredPackage(PCAP)
@@ -108,7 +113,7 @@ if (NOT DISABLE_PERFTOOLS)
    find_package(GooglePerftools)
 endif ()
 
-if (GOOGLEPERFTOOLS_FOUND)
+if (GOOGLEPERFTOOLS_FOUND OR TCMALLOC_FOUND)
     set(HAVE_PERFTOOLS true)
     # Non-Linux systems may not be well-supported by gperftools, so
     # require explicit request from user to enable it in that case.
@@ -160,21 +165,30 @@ include(PCAPTests)
 include(OpenSSLTests)
 include(CheckNameserCompat)
 include(GetArchitecture)
+include(RequireCXX11)
 
 # Tell the plugin code that we're building as part of the main tree.
 set(BRO_PLUGIN_INTERNAL_BUILD true CACHE INTERNAL "" FORCE)
 
-configure_file(${CMAKE_CURRENT_SOURCE_DIR}/config.h.in
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/bro-config.h.in
-               ${CMAKE_CURRENT_BINARY_DIR}/config.h)
+               ${CMAKE_CURRENT_BINARY_DIR}/bro-config.h)
 
 include_directories(${CMAKE_CURRENT_BINARY_DIR})
 
 ########################################################################
 ## Recurse on sub-directories
 
+if ( ENABLE_BROKER )
+    add_subdirectory(aux/broker)
+    set(brodeps ${brodeps} broker)
+    add_definitions(-DENABLE_BROKER)
+    include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR}/aux/broker)
+endif ()
+
 add_subdirectory(src)
 add_subdirectory(scripts)
 add_subdirectory(doc)
+add_subdirectory(man)
 
 include(CheckOptionalBuildSources)
 
@@ -218,6 +232,8 @@ message(
     "\nCXXFLAGS:          ${CMAKE_CXX_FLAGS} ${CMAKE_CXX_FLAGS_${BuildType}}"
     "\nCPP:               ${CMAKE_CXX_COMPILER}"
     "\n"
+    "\nBroker:            ${ENABLE_BROKER}"
+    "\nBroker Python:     ${BROKER_PYTHON_BINDINGS}"
     "\nBroccoli:          ${INSTALL_BROCCOLI}"
     "\nBroctl:            ${INSTALL_BROCTL}"
     "\nAux. Tools:        ${INSTALL_AUX_TOOLS}"

COPYING

@@ -1,4 +1,4 @@
-Copyright (c) 1995-2013, The Regents of the University of California
+Copyright (c) 1995-2015, The Regents of the University of California
 through the Lawrence Berkeley National Laboratory and the
 International Computer Science Institute. All rights reserved.
 

INSTALL

@@ -1,3 +0,0 @@
-
-See doc/install/install.rst for installation instructions.
-

INSTALL

@@ -0,0 +1 @@
+doc/install/install.rst

Makefile

@@ -48,15 +48,18 @@ bindist:
 
 distclean:
 	rm -rf $(BUILD)
+	$(MAKE) -C testing $@
 
 test:
-	@( cd testing && make )
+	-@( cd testing && make )
 
-test-all: test
+test-aux:
-	test -d aux/broctl && ( cd aux/broctl && make test )
+	-test -d aux/broctl && ( cd aux/broctl && make test-all )
-	test -d aux/btest  && ( cd aux/btest && make test )
+	-test -d aux/btest  && ( cd aux/btest && make test )
-	test -d aux/bro-aux && ( cd aux/bro-aux && make test )
+	-test -d aux/bro-aux && ( cd aux/bro-aux && make test )
-	test -d aux/plugins && ( cd aux/plugins && make test-all )
+	-test -d aux/plugins && ( cd aux/plugins && make test-all )
+
+test-all: test test-aux
 
 configured:
 	@test -d $(BUILD) || ( echo "Error: No build/ directory found. Did you run configure?" && exit 1 )

NEWS

@@ -4,11 +4,70 @@ release. For an exhaustive list of changes, see the ``CHANGES`` file
 (note that submodules, such as BroControl and Broccoli, come with
 their own ``CHANGES``.)
 
-Bro 2.4 (in progress)
+Bro 2.5 (in progress)
 =====================
 
-Dependencies
+New Dependencies
-------------
+----------------
+
+- Bro now requires a compiler with C++11 support for building the
+  source code.
+
+- Bro now requires the C++ Actor Framework, CAF, which must be
+  installed first. See http://actor-framework.org.
+
+- Bro now requires Python instead of Perl to compile the source code.
+
+- The pcap buffer size can set through the new option Pcap::bufsize.
+
+New Functionality
+-----------------
+
+- Bro now tracks VLAN IDs. To record them inside the connection log,
+  load protocols/conn/vlan-logging.bro.
+
+- A new per-packet event raw_packet() provides access to layer 2
+  information. Use with care, generating events per packet is
+  expensive.
+
+- A new built-in function, decode_base64_conn() for Base64 decoding.
+  It works like decode_base64() but receives an additional connection
+  argument that will be used for decoding errors into weird.log
+  (instead of reporter.log).
+
+- The IRC analyzer now recognizes StartTLS sessions and enable the SSL
+  analyzer for them.
+
+- New Bro plugins in aux/plugins:
+
+    - af_packet: Native AF_PACKET support.
+    - myricom: Native Myricom SNF v3 support.
+    - pf_ring: Native PF_RING support.
+    - redis: An experimental log writer for Redis.
+    - tcprs: An TCP-level analyzer detecting retransmissions, reordering, and more.
+
+Changed Functionality
+---------------------
+
+- Some script-level identifier have changed their names:
+
+      snaplen                  -> Pcap::snaplen
+      precompile_pcap_filter() -> Pcap::precompile_pcap_filter()
+      install_pcap_filter()    -> Pcap::install_pcap_filter()
+      pcap_error()             -> Pcap::pcap_error()
+
+
+Deprecated Functionality
+------------------------
+
+    - The built-in functions decode_base64_custom() and
+      encode_base64_custom() are no longer needed and will be removed
+      in the future. Their functionality is now provided directly by
+      decode_base64() and encode_base64(), which take an optional
+      parameter to change the Base64 alphabet.
+
+Bro 2.4
+=======
 
 New Functionality
 -----------------
@@ -16,20 +75,257 @@ New Functionality
 - Bro now has support for external plugins that can extend its core
   functionality, like protocol/file analysis, via shared libraries.
   Plugins can be developed and distributed externally, and will be
-  pulled in dynamically at startup. Currently, a plugin can provide
+  pulled in dynamically at startup (the environment variables
-  custom protocol analyzers, file analyzers, log writers[TODO], input
+  BRO_PLUGIN_PATH and BRO_PLUGIN_ACTIVATE can be used to specify the
-  readers[TODO], packet sources[TODO], and new built-in functions. A
+  locations and names of plugins to activate). Currently, a plugin
-  plugin can furthermore hook into Bro's processing a number of places
+  can provide custom protocol analyzers, file analyzers, log writers,
-  to add custom logic.
+  input readers, packet sources and dumpers, and new built-in functions.
+  A plugin can furthermore hook into Bro's processing at a number of
+  places to add custom logic.
 
   See https://www.bro.org/sphinx-git/devel/plugins.html for more
   information on writing plugins.
 
+- Bro now has support for the MySQL wire protocol. Activity gets
+  logged into mysql.log.
+
+- Bro now parses DTLS traffic. Activity gets logged into ssl.log.
+
+- Bro now has support for the Kerberos KRB5 protocol over TCP and
+  UDP. Activity gets logged into kerberos.log.
+
+- Bro now has an RDP analyzer. Activity gets logged into rdp.log.
+
+- Bro now has a file analyzer for Portable Executables. Activity gets
+  logged into pe.log.
+
+- Bro now has support for the SIP protocol over UDP. Activity gets
+  logged into sip.log.
+
+- Bro now features a completely rewritten, enhanced SSH analyzer.  The
+  new analyzer is able to determine if logins failed or succeeded in
+  most circumstances, logs a lot more more information about SSH
+  sessions, supports v1, and introduces the intelligence type
+  ``Intel::PUBKEY_HASH`` and location ``SSH::IN_SERVER_HOST_KEY``. The
+  analayzer also generates a set of additional events
+  (``ssh_auth_successful``, ``ssh_auth_failed``, ``ssh_capabilities``,
+  ``ssh2_server_host_key``, ``ssh1_server_host_key``,
+  ``ssh_encrypted_packet``, ``ssh2_dh_server_params``,
+  ``ssh2_gss_error``, ``ssh2_ecc_key``). See next section for
+  incompatible SSH changes.
+
+- Bro's file analysis now supports reassembly of files that are not
+  transferred/seen sequentially.  The default file reassembly buffer
+  size is set with the ``Files::reassembly_buffer_size`` variable.
+
+- Bro's file type identification has been greatly improved (new file types,
+  bug fixes, and performance improvements).
+
+- Bro's scripting language now has a ``while`` statement::
+
+        while ( i < 5 )
+            print ++i;
+
+  ``next`` and ``break`` can be used inside the loop's body just like
+  with ``for`` loops.
+
+- Bro now integrates Broker, a new communication library. See
+  aux/broker/README for more information on Broker, and
+  doc/frameworks/broker.rst for the corresponding Bro script API.
+
+  With Broker, Bro has the similar capabilities of exchanging events and
+  logs with remote peers (either another Bro process or some other
+  application that uses Broker).  It also includes a key-value store
+  API that can be used to share state between peers and optionally
+  allow data to persist on disk for longer-term storage.
+
+  Broker support is by default off for now; it can be enabled at
+  configure time with --enable-broker. It requires CAF version 0.13+
+  (https://github.com/actor-framework/actor-framework) as well as a
+  C++11 compiler (e.g. GCC 4.8+ or Clang 3.3+).
+
+  Broker will become a mandatory dependency in future Bro versions and
+  replace the current communication and serialization system.
+
+- Add --enable-c++11 configure flag to compile Bro's source code in
+  C++11 mode with a corresponding compiler. Note that 2.4 will be the
+  last version of Bro that compiles without C++11 support.
+
+- The SSL analysis now alerts when encountering SSL connections with
+  old protocol versions or unsafe cipher suites. It also gained
+  extended reporting of weak keys, caching of already validated
+  certificates, and full support for TLS record defragmentation. SSL generally
+  became much more robust and added several fields to ssl.log (while
+  removing some others).
+
+- A new icmp_sent_payload event provides access to ICMP payload.
+
+- The input framework's raw reader now supports seeking by adding an
+  option "offset" to the config map. Positive offsets are interpreted
+  to be from the beginning of the file, negative from the end of the
+  file (-1 is end of file).
+
+- One can now raise events when a connection crosses a given size
+  threshold in terms of packets or bytes. The primary API for that
+  functionality is in base/protocols/conn/thresholds.bro.
+
+- There is a new command-line option -Q/--time that prints Bro's execution
+  time and memory usage to stderr.
+
+- BroControl now has a new command "deploy" which is equivalent to running
+  the "check", "install", "stop", and "start" commands (in that order).
+
+- BroControl now has a new option "StatusCmdShowAll" that controls whether
+  or not the broctl "status" command gathers all of the status information.
+  This option can be used to make the "status" command run significantly
+  faster (in this case, the "Peers" column will not be shown in the output).
+
+- BroControl now has a new option "StatsLogEnable" that controls whether
+  or not broctl will record information to the "stats.log" file.  This option
+  can be used to make the "broctl cron" command run slightly faster (in this
+  case, "broctl cron" will also no longer send email about not seeing any
+  packets on the monitoring interfaces).
+
+- BroControl now has a new option "MailHostUpDown" which controls whether or
+  not the "broctl cron" command will send email when it notices that a host
+  in the cluster is up or down.
+
+- BroControl now has a new option "CommandTimeout" which specifies the number
+  of seconds to wait for a command that broctl ran to return results.
+
 Changed Functionality
 ---------------------
 
 - bro-cut has been rewritten in C, and is hence much faster.
 
+- File analysis
+
+    * Removed ``fa_file`` record's ``mime_type`` and ``mime_types``
+      fields.  The event ``file_sniff`` has been added which provides
+      the same information.  The ``mime_type`` field of ``Files::Info``
+      also still has this info.
+
+    * The earliest point that new mime type information is available is
+      in the ``file_sniff`` event which comes after the ``file_new`` and
+      ``file_over_new_connection`` events.  Scripts which inspected mime
+      type info within those events will need to be adapted.  (Note: for
+      users that worked w/ versions of Bro from git, for a while there was
+      also an event called ``file_mime_type`` which is now replaced with
+      the ``file_sniff`` event).
+
+    * Removed ``Files::add_analyzers_for_mime_type`` function.
+
+    * Removed ``offset`` parameter of the ``file_extraction_limit``
+      event.  Since file extraction now internally depends on file
+      reassembly for non-sequential files, "offset" can be obtained
+      with other information already available -- adding together
+      ``seen_bytes`` and ``missed_bytes`` fields of the ``fa_file``
+      record gives how many bytes have been written so far (i.e.
+      the "offset").
+
+- The SSH changes come with a few incompatibilities. The following
+  events have been renamed:
+
+    * ``SSH::heuristic_failed_login`` to ``ssh_auth_failed``
+    * ``SSH::heuristic_successful_login`` to ``ssh_auth_successful``
+
+  The ``SSH::Info`` status field has been removed and replaced with
+  the ``auth_success`` field.  This field has been changed from a
+  string that was previously ``success``, ``failure`` or
+  ``undetermined`` to a boolean. a boolean that is ``T``, ``F``, or
+  unset.
+
+- The has_valid_octets function now uses a string_vec parameter instead of
+  string_array.
+
+- conn.log gained a new field local_resp that works like local_orig,
+  just for the responder address of the connection.
+
+- GRE tunnels are now identified as ``Tunnel::GRE`` instead of
+  ``Tunnel::IP``.
+
+- The default name for extracted files changed from extract-protocol-id
+  to extract-timestamp-protocol-id.
+
+- The weird named "unmatched_HTTP_reply" has been removed since it can
+  be detected at the script-layer and is handled correctly by the
+  default HTTP scripts.
+
+- When adding a logging filter to a stream, the filter can now inherit
+  a default ``path`` field from the associated ``Log::Stream`` record.
+
+- When adding a logging filter to a stream, the
+  ``Log::default_path_func`` is now only automatically added to the
+  filter if it has neither a ``path`` nor a ``path_func`` already
+  explicitly set.  Before, the default path function would always be set
+  for all filters which didn't specify their own ``path_func``.
+
+- BroControl now establishes only one ssh connection from the manager to
+  each remote host in a cluster configuration (previously, there would be
+  one ssh connection per remote Bro process).
+
+- BroControl now uses SQLite to record state information instead of a
+  plain text file (the file "spool/broctl.dat" is no longer used).
+  On FreeBSD, this means that there is a new dependency on the package
+  "py27-sqlite3".
+
+- BroControl now records the expected running state of each Bro node right
+  before each start or stop.  The "broctl cron" command uses this info to
+  either start or stop Bro nodes as needed so that the actual state matches
+  the expected state (previously, "broctl cron" could only start nodes in
+  the "crashed" state, and could never stop a node).
+
+- BroControl now sends all normal command output (i.e., not error messages)
+  to stdout.  Error messages are still sent to stderr, however.
+
+- The capability of processing NetFlow input has been removed for the
+  time being.  Therefore, the -y/--flowfile and -Y/--netflow command-line
+  options have been removed, and the netflow_v5_header and netflow_v5_record
+  events have been removed.
+
+- The -D/--dfa-size command-line option has been removed.
+
+- The -L/--rule-benchmark command-line option has been removed.
+
+- The -O/--optimize command-line option has been removed.
+
+- The deprecated fields "hot" and "addl" have been removed from the
+  connection record. Likewise, the functions append_addl() and
+  append_addl_marker() have been removed.
+
+- Log files now escape non-printable characters consistently as "\xXX'.
+  Furthermore, backslashes are escaped as "\\", making the
+  representation fully reversible.
+
+Deprecated Functionality
+------------------------
+
+- The split* family of functions are to be replaced with alternate
+  versions that return a vector of strings rather than a table of
+  strings. This also allows deprecation for some related string
+  concatenation/extraction functions. Note that the new functions use
+  0-based indexing, rather than 1-based.
+
+  The full list of now deprecated functions is:
+
+    * split: use split_string instead.
+
+    * split1: use split_string1 instead.
+
+    * split_all: use split_string_all instead.
+
+    * split_n: use split_string_n instead.
+
+    * cat_string_array: see join_string_vec instead.
+
+    * cat_string_array_n: see join_string_vec instead.
+
+    * join_string_array: see join_string_vec instead.
+
+    * sort_string_array: use sort instead.
+
+    * find_ip_addresses: use extract_ip_addresses instead.
+
 Bro 2.3
 =======
 

README.rst

@@ -0,0 +1 @@
+README

VERSION

@@ -1 +1 @@
-2.3-183
+2.4-228

aux/binpac

@@ -1 +1 @@
-Subproject commit 3a4684801aafa0558383199e9abd711650b53af9
+Subproject commit 214294c502d377bb7bf511eac8c43608e54c875a

aux/bro-aux

@@ -1 +1 @@
-Subproject commit 9ea20c3905bd3fd5109849c474a2f2b4ed008357
+Subproject commit 4e0d2bff4b2c287f66186c3654ef784bb0748d11

aux/broccoli

@@ -1 +1 @@
-Subproject commit 33d0ed4a54a6ecf08a0b5fe18831aa413b437066
+Subproject commit 959cc0a8181e7f4b07559a6aecca2a0d7d3d445c

aux/broctl

@@ -1 +1 @@
-Subproject commit 2f808bc8541378b1a4953cca02c58c43945d154f
+Subproject commit 1d0ca4753471cf822f612dc0d0e9bf9a439a994b

aux/broker

@@ -0,0 +1 @@
+Subproject commit 9a2e8ec7b365bde282edc7301c7936eed6b4fbbb

aux/btest

@@ -1 +1 @@
-Subproject commit 1efa4d10f943351efea96def68e598b053fd217a
+Subproject commit 71a1e3efc437aa9f981be71affa1c4615e8d98a5

aux/plugins

@@ -1 +1 @@
-Subproject commit 23055b473c689a79da12b2825d8388f71f28c709
+Subproject commit 35007df0974b566f75d7c82af5b4d5a022333d87

cmake

@@ -1 +1 @@
-Subproject commit 03de0cc467d2334dcb851eddd843d59fef217909
+Subproject commit 843cdf6a91f06e5407bffbc79a343bff3cf4c81f

configure

@@ -24,6 +24,13 @@ Usage: $0 [OPTION]... [VAR=VALUE]...
     --prefix=PREFIX        installation directory [/usr/local/bro]
     --scriptdir=PATH       root installation directory for Bro scripts
                            [PREFIX/share/bro]
+    --localstatedir=PATH   when using BroControl, path to store log files
+                           and run-time data (within log/ and spool/ subdirs)
+                           [PREFIX]
+    --spooldir=PATH        when using BroControl, path to store run-time data
+                           [PREFIX/spool]
+    --logdir=PATH          when using BroControl, path to store log file
+                           [PREFIX/logs]
     --conf-files-dir=PATH  config files installation directory [PREFIX/etc]
 
   Optional Features:
@@ -34,11 +41,13 @@ Usage: $0 [OPTION]... [VAR=VALUE]...
     --enable-perftools-debug use Google's perftools for debugging
     --enable-jemalloc      link against jemalloc
     --enable-ruby          build ruby bindings for broccoli (deprecated)
+    --disable-broker       disable use of the Broker communication library
     --disable-broccoli     don't build or install the Broccoli library
     --disable-broctl       don't install Broctl
     --disable-auxtools     don't build or install auxiliary tools
     --disable-perftools    don't try to build with Google Perftools
     --disable-python       don't try to build python bindings for broccoli
+    --disable-pybroker     don't try to build python bindings for broker
 
   Required Packages in Non-Standard Locations:
     --with-openssl=PATH    path to OpenSSL install root
@@ -47,19 +56,22 @@ Usage: $0 [OPTION]... [VAR=VALUE]...
     --with-binpac=PATH     path to BinPAC install root
     --with-flex=PATH       path to flex executable
     --with-bison=PATH      path to bison executable
-    --with-perl=PATH       path to perl executable
+    --with-python=PATH     path to Python executable
+    --with-libcaf=PATH     path to C++ Actor Framework installation
+                           (a required Broker dependency)
 
   Optional Packages in Non-Standard Locations:
     --with-geoip=PATH      path to the libGeoIP install root
     --with-perftools=PATH  path to Google Perftools install root
     --with-jemalloc=PATH   path to jemalloc install root
-    --with-python=PATH     path to Python interpreter
     --with-python-lib=PATH path to libpython
     --with-python-inc=PATH path to Python headers
     --with-ruby=PATH       path to ruby interpreter
     --with-ruby-lib=PATH   path to ruby library
     --with-ruby-inc=PATH   path to ruby headers
     --with-swig=PATH       path to SWIG executable
+    --with-rocksdb=PATH    path to RocksDB installation
+                           (an optional Broker dependency)
 
   Packaging Options (for developers):
     --binary-package       toggle special logic for binary packaging
@@ -81,7 +93,7 @@ Usage: $0 [OPTION]... [VAR=VALUE]...
 sourcedir="$( cd "$( dirname "$0" )" && pwd )"
 
 # Function to append a CMake cache entry definition to the
-# CMakeCacheEntries variable
+# CMakeCacheEntries variable.
 #   $1 is the cache entry variable name
 #   $2 is the cache entry variable type
 #   $3 is the cache entry variable value
@@ -89,6 +101,17 @@ append_cache_entry () {
     CMakeCacheEntries="$CMakeCacheEntries -D $1:$2=$3"
 }
 
+# Function to remove a CMake cache entry definition from the
+# CMakeCacheEntries variable
+#   $1 is the cache entry variable name
+remove_cache_entry () {
+    CMakeCacheEntries="$CMakeCacheEntries -U $1"
+
+    # Even with -U, cmake still warns by default if
+    # added previously with -D.
+    CMakeCacheEntries="$CMakeCacheEntries --no-warn-unused-cli"
+}
+
 # set defaults
 builddir=build
 prefix=/usr/local/bro
@@ -98,10 +121,13 @@ append_cache_entry BRO_ROOT_DIR         PATH   $prefix
 append_cache_entry PY_MOD_INSTALL_DIR   PATH   $prefix/lib/broctl
 append_cache_entry BRO_SCRIPT_INSTALL_PATH STRING $prefix/share/bro
 append_cache_entry BRO_ETC_INSTALL_DIR  PATH   $prefix/etc
+append_cache_entry BROKER_PYTHON_HOME   PATH   $prefix
+append_cache_entry BROKER_PYTHON_BINDINGS BOOL false
 append_cache_entry ENABLE_DEBUG         BOOL   false
 append_cache_entry ENABLE_PERFTOOLS     BOOL   false
 append_cache_entry ENABLE_PERFTOOLS_DEBUG BOOL false
 append_cache_entry ENABLE_JEMALLOC      BOOL   false
+append_cache_entry ENABLE_BROKER        BOOL   true
 append_cache_entry BinPAC_SKIP_INSTALL  BOOL   true
 append_cache_entry BUILD_SHARED_LIBS    BOOL   true
 append_cache_entry INSTALL_AUX_TOOLS    BOOL   true
@@ -135,6 +161,10 @@ while [ $# -ne 0 ]; do
             append_cache_entry CMAKE_INSTALL_PREFIX PATH   $optarg
             append_cache_entry BRO_ROOT_DIR         PATH   $optarg
             append_cache_entry PY_MOD_INSTALL_DIR   PATH   $optarg/lib/broctl
+
+            if [ -z "$user_disabled_broker" ]; then
+                append_cache_entry BROKER_PYTHON_HOME   PATH   $optarg
+            fi
             ;;
         --scriptdir=*)
             append_cache_entry BRO_SCRIPT_INSTALL_PATH STRING $optarg
@@ -144,6 +174,15 @@ while [ $# -ne 0 ]; do
             append_cache_entry BRO_ETC_INSTALL_DIR PATH $optarg
             user_set_conffilesdir="true"
             ;;
+        --localstatedir=*)
+            append_cache_entry BRO_LOCAL_STATE_DIR  PATH   $optarg
+            ;;
+        --spooldir=*)
+            append_cache_entry BRO_SPOOL_DIR  PATH   $optarg
+            ;;
+        --logdir=*)
+            append_cache_entry BRO_LOG_DIR  PATH   $optarg
+            ;;
         --enable-debug)
             append_cache_entry ENABLE_DEBUG         BOOL   true
             ;;
@@ -160,6 +199,11 @@ while [ $# -ne 0 ]; do
         --enable-jemalloc)
             append_cache_entry ENABLE_JEMALLOC     BOOL   true
             ;;
+        --disable-broker)
+            append_cache_entry ENABLE_BROKER        BOOL   false
+            remove_cache_entry BROKER_PYTHON_HOME
+            user_disabled_broker="true"
+            ;;
         --disable-broccoli)
             append_cache_entry INSTALL_BROCCOLI     BOOL   false
             ;;
@@ -175,6 +219,9 @@ while [ $# -ne 0 ]; do
         --disable-python)
             append_cache_entry DISABLE_PYTHON_BINDINGS     BOOL   true
             ;;
+        --disable-pybroker)
+            append_cache_entry DISABLE_PYBROKER     BOOL   true
+            ;;
         --enable-ruby)
             append_cache_entry DISABLE_RUBY_BINDINGS     BOOL   false
             ;;
@@ -196,9 +243,6 @@ while [ $# -ne 0 ]; do
         --with-bison=*)
             append_cache_entry BISON_EXECUTABLE PATH $optarg
             ;;
-        --with-perl=*)
-            append_cache_entry PERL_EXECUTABLE PATH $optarg
-            ;;
         --with-geoip=*)
             append_cache_entry LibGeoIP_ROOT_DIR PATH $optarg
             ;;
@@ -232,6 +276,12 @@ while [ $# -ne 0 ]; do
         --with-swig=*)
             append_cache_entry SWIG_EXECUTABLE      PATH    $optarg
             ;;
+        --with-libcaf=*)
+            append_cache_entry LIBCAF_ROOT_DIR      PATH   $optarg
+            ;;
+        --with-rocksdb=*)
+            append_cache_entry ROCKSDB_ROOT_DIR     PATH   $optarg
+            ;;
         --binary-package)
             append_cache_entry BINARY_PACKAGING_MODE BOOL true
             ;;

doc/components/bro-plugins/README.rst

@@ -0,0 +1 @@
+../../../aux/plugins/README

doc/components/bro-plugins/dataseries/README.rst

@@ -0,0 +1 @@
+../../../../aux/plugins/dataseries/README

doc/components/bro-plugins/elasticsearch/README.rst

@@ -0,0 +1 @@
+../../../../aux/plugins/elasticsearch/README

doc/components/bro-plugins/netmap/README.rst

@@ -0,0 +1 @@
+../../../../aux/plugins/netmap/README

doc/components/bro-plugins/pf_ring/README.rst

@@ -0,0 +1 @@
+../../../../aux/plugins/pf_ring/README

doc/components/bro-plugins/redis/README.rst

@@ -0,0 +1 @@
+../../../../aux/plugins/redis/README

doc/components/broker/README.rst

@@ -0,0 +1 @@
+../../../aux/broker/README

doc/components/broker/broker-manual.rst

@@ -0,0 +1 @@
+../../../aux/broker/broker-manual.rst

doc/components/index.rst

@@ -17,8 +17,11 @@ current, independent component releases.
    Broccoli - User Manual <broccoli/broccoli-manual>
    Broccoli Python Bindings <broccoli-python/README>
    Broccoli Ruby Bindings <broccoli-ruby/README>
+   Broker - Bro's (New) Messaging Library (README) <broker/README>
+   Broker - User Manual <broker/broker-manual.rst>
    BroControl - Interactive Bro management shell <broctl/README>
    Bro-Aux - Small auxiliary tools for Bro <bro-aux/README>
+   Bro-Plugins - A collection of plugins for Bro <bro-plugins/README>
    BTest - A unit testing framework <btest/README>
    Capstats - Command-line packet statistic tool <capstats/README>
    PySubnetTree - Python module for CIDR lookups<pysubnettree/README>

doc/devel/plugins.rst

@@ -3,7 +3,7 @@
 Writing Bro Plugins
 ===================
 
-Bro is internally moving to a plugin structure that enables extending
+Bro internally provides a plugin API that enables extending
 the system dynamically, without modifying the core code base. That way
 custom code remains self-contained and can be maintained, compiled,
 and installed independently. Currently, plugins can add the following
@@ -17,11 +17,11 @@ functionality to Bro:
 
     - File analyzers.
 
-    - Packet sources and packet dumpers. TODO: Not yet.
+    - Packet sources and packet dumpers.
 
-    - Logging framework backends. TODO: Not yet.
+    - Logging framework backends.
 
-    - Input framework readers. TODO: Not yet.
+    - Input framework readers.
 
 A plugin's functionality is available to the user just as if Bro had
 the corresponding code built-in. Indeed, internally many of Bro's
@@ -32,7 +32,7 @@ Quick Start
 ===========
 
 Writing a basic plugin is quite straight-forward as long as one
-follows a few conventions. In the following we walk a simple example
+follows a few conventions. In the following we create a simple example
 plugin that adds a new built-in function (bif) to Bro: we'll add 
 ``rot13(s: string) : string``, a function that rotates every character
 in a string by 13 places.
@@ -42,18 +42,17 @@ certain structure. To get started, Bro's distribution provides a
 helper script ``aux/bro-aux/plugin-support/init-plugin`` that creates
 a skeleton plugin that can then be customized. Let's use that::
 
-    # mkdir rot13-plugin
+    # init-plugin ./rot13-plugin Demo Rot13
-    # cd rot13-plugin
-    # init-plugin Demo Rot13
 
-As you can see the script takes two arguments. The first is a
+As you can see, the script takes three arguments. The first is a
-namespace the plugin will live in, and the second a descriptive name
+directory inside which the plugin skeleton will be created.  The second
-for the plugin itself. Bro uses the combination of the two to identify
+is the namespace the plugin will live in, and the third is a descriptive
-a plugin. The namespace serves to avoid naming conflicts between
+name for the plugin itself relative to the namespace. Bro uses the
-plugins written by independent developers; pick, e.g., the name of
+combination of namespace and name to identify a plugin. The namespace
-your organisation. The namespace ``Bro`` is reserved for functionality
+serves to avoid naming conflicts between plugins written by independent
-distributed by the Bro Project. In our example, the plugin will be
+developers; pick, e.g., the name of your organisation. The namespace
-called ``Demo::Rot13``.
+``Bro`` is reserved for functionality distributed by the Bro Project. In
+our example, the plugin will be called ``Demo::Rot13``.
 
 The ``init-plugin`` script puts a number of files in place. The full
 layout is described later. For now, all we need is
@@ -61,7 +60,7 @@ layout is described later. For now, all we need is
 there as follows::
 
     # cat src/rot13.bif
-    module CaesarCipher;
+    module Demo;
 
     function rot13%(s: string%) : string
         %{
@@ -82,21 +81,25 @@ The syntax of this file is just like any other ``*.bif`` file; we
 won't go into it here.
 
 Now we can already compile our plugin, we just need to tell the
-configure script put in place by ``init-plugin`` where the Bro source
+configure script (that ``init-plugin`` created) where the Bro
-tree is located (Bro needs to have been built there first)::
+source tree is located (Bro needs to have been built there first)::
 
+    # cd rot13-plugin
     # ./configure --bro-dist=/path/to/bro/dist && make
     [... cmake output ...]
 
-Now our ``rot13-plugin`` directory has everything that it needs
+This builds the plugin in a subdirectory ``build/``. In fact, that
-for Bro to recognize it as a dynamic plugin. Once we point Bro to it,
+subdirectory *becomes* the plugin: when ``make`` finishes, ``build/``
-it will pull it in automatically, as we can check with the ``-N``
+has everything it needs for Bro to recognize it as a dynamic plugin.
+
+Let's try that. Once we point Bro to the ``build/`` directory, it will
+pull in our new plugin automatically, as we can check with the ``-N``
 option::
 
-    # export BRO_PLUGIN_PATH=/path/to/rot13-plugin
+    # export BRO_PLUGIN_PATH=/path/to/rot13-plugin/build
     # bro -N
     [...]
-    Plugin: Demo::Rot13 - <Insert brief description of plugin> (dynamic, version 1)
+    Demo::Rot13 - <Insert description> (dynamic, version 0.1)
     [...]
 
 That looks quite good, except for the dummy description that we should
@@ -105,34 +108,36 @@ is about.  We do this by editing the ``config.description`` line in
 ``src/Plugin.cc``, like this::
 
     [...]
-    plugin::Configuration Configure()
+    plugin::Configuration Plugin::Configure()
         {
         plugin::Configuration config;
         config.name = "Demo::Rot13";
         config.description = "Caesar cipher rotating a string's characters by 13 places.";
-        config.version.major = 1;
+        config.version.major = 0;
-        config.version.minor = 0;
+        config.version.minor = 1;
         return config;
         }
     [...]
 
+Now rebuild and verify that the description is visible::
+
     # make
     [...]
     # bro -N | grep Rot13
-    Plugin: Demo::Rot13 - Caesar cipher rotating a string's characters by 13 places. (dynamic, version 1)
+    Demo::Rot13 - Caesar cipher rotating a string's characters by 13 places. (dynamic, version 0.1)
 
-Better. Bro can also show us what exactly the plugin provides with the
+Bro can also show us what exactly the plugin provides with the
 more verbose option ``-NN``::
 
     # bro -NN
     [...]
-    Plugin: Demo::Rot13 - Caesar cipher rotating a string's characters by 13 places. (dynamic, version 1)
+    Demo::Rot13 - Caesar cipher rotating a string's characters by 13 places. (dynamic, version 0.1)
-        [Function] CaesarCipher::rot13
+        [Function] Demo::rot13
     [...]
 
 There's our function. Now let's use it::
 
-    # bro -e 'print CaesarCipher::rot13("Hello")'
+    # bro -e 'print Demo::rot13("Hello")'
     Uryyb
 
 It works. We next install the plugin along with Bro itself, so that it
@@ -141,36 +146,42 @@ environment variable. If we first unset the variable, the function
 will no longer be available::
 
     # unset BRO_PLUGIN_PATH
-    # bro -e 'print CaesarCipher::rot13("Hello")'
+    # bro -e 'print Demo::rot13("Hello")'
-    error in <command line>, line 1: unknown identifier CaesarCipher::rot13, at or near "CaesarCipher::rot13"
+    error in <command line>, line 1: unknown identifier Demo::rot13, at or near "Demo::rot13"
 
 Once we install it, it works again::
 
     # make install
-    # bro -e 'print CaesarCipher::rot13("Hello")'
+    # bro -e 'print Demo::rot13("Hello")'
     Uryyb
 
 The installed version went into
 ``<bro-install-prefix>/lib/bro/plugins/Demo_Rot13``.
 
-We can distribute the plugin in either source or binary form by using
+One can distribute the plugin independently of Bro for others to use.
-the Makefile's ``sdist`` and ``bdist`` target, respectively. Both
+To distribute in source form, just remove the ``build/`` directory
-create corrsponding tarballs::
+(``make distclean`` does that) and then tar up the whole ``rot13-plugin/``
+directory. Others then follow the same process as above after
+unpacking.
 
-    # make sdist
+To distribute the plugin in binary form, the build process
-    [...]
+conveniently creates a corresponding tarball in ``build/dist/``. In
-    Source distribution in build/sdist/Demo_Rot13.tar.gz
+this case, it's called ``Demo_Rot13-0.1.tar.gz``, with the version
+number coming out of the ``VERSION`` file that ``init-plugin`` put
+into place. The binary tarball has everything needed to run the
+plugin, but no further source files. Optionally, one can include
+further files by specifying them in the plugin's ``CMakeLists.txt``
+through the ``bro_plugin_dist_files`` macro; the skeleton does that
+for ``README``, ``VERSION``, ``CHANGES``, and ``COPYING``. To use the
+plugin through the binary tarball, just unpack it into
+``<bro-install-prefix>/lib/bro/plugins/``.  Alternatively, if you unpack
+it in another location, then you need to point ``BRO_PLUGIN_PATH`` there.
 
-    # make bdist
+Before distributing your plugin, you should edit some of the meta
-    [...]
+files that ``init-plugin`` puts in place. Edit ``README`` and
-    Binary distribution in build/Demo_Rot13-darwin-x86_64.tar.gz
+``VERSION``, and update ``CHANGES`` when you make changes. Also put a
-
+license file in place as ``COPYING``; if BSD is fine, you will find a
-The source archive will contain everything in the plugin directory
+template in ``COPYING.edit-me``.
-except any generated files. The binary archive will contain anything
-needed to install and run the plugin, i.e., just what ``make install``
-puts into place as well. As the binary distribution is
-platform-dependent, its name includes the OS and architecture the
-plugin was built on.
 
 Plugin Directory Layout
 =======================
@@ -179,14 +190,14 @@ A plugin's directory needs to follow a set of conventions so that Bro
 (1) recognizes it as a plugin, and (2) knows what to load.  While
 ``init-plugin`` takes care of most of this, the following is the full
 story. We'll use ``<base>`` to represent a plugin's top-level
-directory.
+directory. With the skeleton, ``<base>`` corresponds to ``build/``.
 
 ``<base>/__bro_plugin__``
     A file that marks a directory as containing a Bro plugin. The file
     must exist, and its content must consist of a single line with the
     qualified name of the plugin (e.g., "Demo::Rot13").
 
-``<base>/lib/<plugin-name>-<os>-<arch>.so``
+``<base>/lib/<plugin-name>.<os>-<arch>.so``
     The shared library containing the plugin's compiled code. Bro will
     load this in dynamically at run-time if OS and architecture match
     the current platform.
@@ -198,16 +209,25 @@ directory.
     "@load"ed.
 
 ``scripts``/__load__.bro
-    A Bro script that will be loaded immediately when the plugin gets
+    A Bro script that will be loaded when the plugin gets activated.
-    activated. See below for more information on activating plugins.
+    When this script executes, any BiF elements that the plugin
+    defines will already be available. See below for more information
+    on activating plugins.
+
+``scripts``/__preload__.bro
+    A Bro script that will be loaded when the plugin gets activated,
+    but before any BiF elements become available. See below for more
+    information on activating plugins.
 
 ``lib/bif/``
     Directory with auto-generated Bro scripts that declare the plugin's
     bif elements. The files here are produced by ``bifcl``.
 
+Any other files in ``<base>`` are ignored by Bro.
+
 By convention, a plugin should put its custom scripts into sub folders
-of ``scripts/``, i.e., ``scripts/<script-namespace>/<script>.bro`` to
+of ``scripts/``, i.e., ``scripts/<plugin-namespace>/<plugin-name>/<script>.bro``
-avoid conflicts. As usual, you can then put a ``__load__.bro`` in
+to avoid conflicts. As usual, you can then put a ``__load__.bro`` in
 there as well so that, e.g., ``@load Demo/Rot13`` could load a whole
 module in the form of multiple individual scripts.
 
@@ -229,15 +249,33 @@ their source directory (after ``make`` and setting Bro's
 install``).
 
 ``make install`` copies over the ``lib`` and ``scripts`` directories,
-as well as the ``__bro_plugin__`` magic file and the ``README`` (which
+as well as the ``__bro_plugin__`` magic file and any further
-you should customize). One can add further CMake ``install`` rules to
+distribution files specified in ``CMakeLists.txt`` (e.g., README,
-install additional files if needed.
+VERSION). You can find a full list of files installed in
+``build/MANIFEST``. Behind the scenes, ``make install`` really just
+unpacks the binary tarball from ``build/dist`` into the destination
+directory.
 
-``init-plugin`` will never overwrite existing files, so it's safe to
+``init-plugin`` will never overwrite existing files. If its target
-rerun in an existing plugin directory; it only put files in place that
+directory already exists, it will by default decline to do anything.
-don't exist yet. That also provides a convenient way to revert a file
+You can run it with ``-u`` instead to update an existing plugin,
-back to what ``init-plugin`` created originally: just delete it and
+however it will never overwrite any existing files; it will only put
-rerun.
+in place files it doesn't find yet. To revert a file back to what
+``init-plugin`` created originally, delete it first and then rerun
+with ``-u``.
+
+``init-plugin`` puts a ``configure`` script in place that wraps
+``cmake`` with a more familiar configure-style configuration. By
+default, the script provides two options for specifying paths to the
+Bro source (``--bro-dist``) and to the plugin's installation directory
+(``--install-root``). To extend ``configure`` with plugin-specific
+options (such as search paths for its dependencies) don't edit the
+script directly but instead extend ``configure.plugin``, which
+``configure`` includes. That way you will be able to more easily
+update ``configure`` in the future when the distribution version
+changes. In ``configure.plugin`` you can use the predefined shell
+function ``append_cache_entry`` to seed values into the CMake cache;
+see the installed skeleton version and existing plugins for examples.
 
 Activating a Plugin
 ===================
@@ -248,7 +286,9 @@ Activating a plugin will:
     1. Load the dynamic module
     2. Make any bif items available
     3. Add the ``scripts/`` directory to ``BROPATH``
-    4. Load ``scripts/__load__.bro``
+    4. Load ``scripts/__preload__.bro``
+    5. Make BiF elements available to scripts.
+    6. Load ``scripts/__load__.bro``
 
 By default, Bro will automatically activate all dynamic plugins found
 in its search path ``BRO_PLUGIN_PATH``. However, in bare mode (``bro
@@ -265,23 +305,25 @@ plugins to unconditionally activate, even in bare mode.
 activated plugins. Note that plugins compiled statically into Bro are
 always activated, and hence show up as such even in bare mode.
 
-Plugin Component
+Plugin Components
-================
+=================
 
-The following gives additional information about providing individual
+The following subsections detail providing individual types of
-types of functionality via plugins. Note that a single plugin can
+functionality via plugins. Note that a single plugin can provide more
-provide more than one type. For example, a plugin could provide
+than one component type. For example, a plugin could provide multiple
-multiple protocol analyzers at once; or both a logging backend and
+protocol analyzers at once; or both a logging backend and input reader
-input reader at the same time.
+at the same time.
 
-We now walk briefly through the specifics of providing a specific type
+.. todo::
-of functionality (a *component*) through a plugin. We'll focus on
+
-their interfaces to the plugin system, rather than specifics on
+    These subsections are mostly missing right now, as much of their
-writing the corresponding logic (usually the best way to get going on
+    content isn't actually plugin-specific, but concerns generally
-that is to start with an existing plugin providing a corresponding
+    writing such functionality for Bro. The best way to get started
-component and adapt that). We'll also point out how the CMake
+    right now is to look at existing code implementing similar
-infrastructure put in place by the ``init-plugin`` helper script ties
+    functionality, either as a plugin or inside Bro proper. Also, for
-the various pieces together.
+    each component type there's a unit test in
+    ``testing/btest/plugins`` creating a basic plugin skeleton with a
+    corresponding component.
 
 Bro Scripts
 -----------
@@ -315,22 +357,22 @@ TODO.
 Logging Writer
 --------------
 
-Not yet available as plugins.
+TODO.
 
 Input Reader
 ------------
 
-Not yet available as plugins.
+TODO.
 
 Packet Sources
 --------------
 
-Not yet available as plugins.
+TODO.
 
 Packet Dumpers
 --------------
 
-Not yet available as plugins.
+TODO.
 
 Hooks
 =====
@@ -341,19 +383,20 @@ Testing Plugins
 ===============
 
 A plugin should come with a test suite to exercise its functionality.
-The ``init-plugin`` script puts in place a basic </btest/README> setup
+The ``init-plugin`` script puts in place a basic
+:doc:`BTest <../../components/btest/README>` setup
 to start with. Initially, it comes with a single test that just checks
 that Bro loads the plugin correctly. It won't have a baseline yet, so
 let's get that in place::
 
     # cd tests
     # btest -d
-    [  0%] plugin.loading ... failed
+    [  0%] rot13.show-plugin ... failed
     % 'btest-diff output' failed unexpectedly (exit code 100)
     % cat .diag
     == File ===============================
-    Demo::Rot13 - Caesar cipher rotating a string's characters by 13 places. (dynamic, version 1.0)
+    Demo::Rot13 - Caesar cipher rotating a string's characters by 13 places. (dynamic, version 0.1)
-        [Function] CaesarCipher::rot13
+        [Function] Demo::rot13
 
     == Error ===============================
     test-diff: no baseline found.
@@ -373,20 +416,20 @@ Now let's add a custom test that ensures that our bif works
 correctly::
 
     # cd tests
-    # cat >plugin/rot13.bro
+    # cat >rot13/bif-rot13.bro
 
     # @TEST-EXEC: bro %INPUT >output
     # @TEST-EXEC: btest-diff output
 
     event bro_init()
         {
-        print CaesarCipher::rot13("Hello");
+        print Demo::rot13("Hello");
         }
 
 Check the output::
 
-    # btest -d plugin/rot13.bro
+    # btest -d rot13/bif-rot13.bro
-    [  0%] plugin.rot13 ... failed
+    [  0%] rot13.bif-rot13 ... failed
     % 'btest-diff output' failed unexpectedly (exit code 100)
     % cat .diag
     == File ===============================
@@ -401,7 +444,7 @@ Check the output::
 
 Install the baseline::
 
-    # btest -U plugin/rot13.bro
+    # btest -U rot13/bif-rot13.bro
     all 1 tests successful
 
 Run the test-suite::
@@ -412,25 +455,31 @@ Run the test-suite::
 Debugging Plugins
 =================
 
-Plugins can use Bro's standard debug logger by using the
+If your plugin isn't loading as expected, Bro's debugging facilities
-``PLUGIN_DBG_LOG(<plugin>, <args>)`` macro (defined in
+can help illuminate what's going on. To enable, recompile Bro
-``DebugLogger.h``), where ``<plugin>`` is the ``Plugin`` instance and
+with debugging support (``./configure --enable-debug``), and
-``<args>`` are printf-style arguments, just as with Bro's standard
+afterwards rebuild your plugin as well. If you then run Bro with ``-B
-debuggging macros.
+plugins``, it will produce a file ``debug.log`` that records details
-
+about the process for searching, loading, and activating plugins. 
-At runtime, one then activates a plugin's debugging output with ``-B
-plugin-<name>``, where ``<name>`` is the name of the plugin as
-returned by its ``Configure()`` method, yet with the
-namespace-separator ``::`` replaced with a simple dash. Example: If
-the plugin is called ``Bro::Demo``, use ``-B plugin-Bro-Demo``. As
-usual, the debugging output will be recorded to ``debug.log`` if Bro's
-compiled in debug mode.
 
+To generate your own debugging output from inside your plugin, you can
+add a custom debug stream by using the ``PLUGIN_DBG_LOG(<plugin>,
+<args>)`` macro (defined in ``DebugLogger.h``), where ``<plugin>`` is
+the ``Plugin`` instance and ``<args>`` are printf-style arguments,
+just as with Bro's standard debugging macros (grep for ``DBG_LOG`` in
+Bro's ``src/`` to see examples). At runtime, you can then activate
+your plugin's debugging output with ``-B plugin-<name>``, where
+``<name>`` is the name of the plugin as returned by its
+``Configure()`` method, yet with the namespace-separator ``::``
+replaced with a simple dash. Example: If the plugin is called
+``Demo::Rot13``, use ``-B plugin-Demo-Rot13``. As usual, the debugging
+output will be recorded to ``debug.log`` if Bro's compiled in debug
+mode.
 
 Documenting Plugins
 ===================
 
-..todo::
+.. todo::
 
     Integrate all this with Broxygen.
 

doc/ext/bro.py

@@ -176,6 +176,10 @@ class BroIdentifier(BroGeneric):
     def get_index_text(self, objectname, name):
         return name
 
+class BroKeyword(BroGeneric):
+    def get_index_text(self, objectname, name):
+        return name
+
 class BroAttribute(BroGeneric):
     def get_index_text(self, objectname, name):
         return _('%s (attribute)') % (name)
@@ -213,6 +217,7 @@ class BroDomain(Domain):
         'type':             ObjType(l_('type'),             'type'),
         'namespace':        ObjType(l_('namespace'),        'namespace'),
         'id':               ObjType(l_('id'),               'id'),
+        'keyword':          ObjType(l_('keyword'),          'keyword'),
         'enum':             ObjType(l_('enum'),             'enum'),
         'attr':             ObjType(l_('attr'),             'attr'),
     }
@@ -221,6 +226,7 @@ class BroDomain(Domain):
         'type':             BroGeneric,
         'namespace':        BroNamespace,
         'id':               BroIdentifier,
+        'keyword':          BroKeyword,
         'enum':             BroEnum,
         'attr':             BroAttribute,
     }
@@ -229,6 +235,7 @@ class BroDomain(Domain):
         'type':             XRefRole(),
         'namespace':        XRefRole(),
         'id':               XRefRole(),
+        'keyword':          XRefRole(),
         'enum':             XRefRole(),
         'attr':             XRefRole(),
         'see':              XRefRole(),

doc/ext/rst_directive.py

@@ -135,7 +135,10 @@ class Pygments(Directive):
                 # lexer not found, use default.
                 lexer = TextLexer()
         else:
+            try:
                 lexer = guess_lexer(content)
+            except:
+                lexer = TextLexer()
 
         # import sys
         # print >>sys.stderr, self.arguments, lexer.__class__

doc/frameworks/broker.rst

@@ -0,0 +1,200 @@
+
+.. _brokercomm-framework:
+
+======================================
+Broker-Enabled Communication Framework
+======================================
+
+.. rst-class:: opening
+
+    Bro can now use the `Broker Library
+    <../components/broker/README.html>`_ to exchange information with
+    other Bro processes.
+
+.. contents::
+
+Connecting to Peers
+===================
+
+Communication via Broker must first be turned on via
+:bro:see:`BrokerComm::enable`.
+
+Bro can accept incoming connections by calling :bro:see:`BrokerComm::listen`
+and then monitor connection status updates via the
+:bro:see:`BrokerComm::incoming_connection_established` and
+:bro:see:`BrokerComm::incoming_connection_broken` events.
+
+.. btest-include:: ${DOC_ROOT}/frameworks/broker/connecting-listener.bro
+
+Bro can initiate outgoing connections by calling :bro:see:`BrokerComm::connect`
+and then monitor connection status updates via the
+:bro:see:`BrokerComm::outgoing_connection_established`,
+:bro:see:`BrokerComm::outgoing_connection_broken`, and
+:bro:see:`BrokerComm::outgoing_connection_incompatible` events.
+
+.. btest-include:: ${DOC_ROOT}/frameworks/broker/connecting-connector.bro
+
+Remote Printing
+===============
+
+To receive remote print messages, first use the
+:bro:see:`BrokerComm::subscribe_to_prints` function to advertise to peers a
+topic prefix of interest and then create an event handler for
+:bro:see:`BrokerComm::print_handler` to handle any print messages that are
+received.
+
+.. btest-include:: ${DOC_ROOT}/frameworks/broker/printing-listener.bro
+
+To send remote print messages, just call :bro:see:`BrokerComm::print`.
+
+.. btest-include:: ${DOC_ROOT}/frameworks/broker/printing-connector.bro
+
+Notice that the subscriber only used the prefix "bro/print/", but is
+able to receive messages with full topics of "bro/print/hi",
+"bro/print/stuff", and "bro/print/bye".  The model here is that the
+publisher of a message checks for all subscribers who advertised
+interest in a prefix of that message's topic and sends it to them.
+
+Message Format
+--------------
+
+For other applications that want to exchange print messages with Bro,
+the Broker message format is simply:
+
+.. code:: c++
+
+    broker::message{std::string{}};
+
+Remote Events
+=============
+
+Receiving remote events is similar to remote prints.  Just use the
+:bro:see:`BrokerComm::subscribe_to_events` function and possibly define any
+new events along with handlers that peers may want to send.
+
+.. btest-include:: ${DOC_ROOT}/frameworks/broker/events-listener.bro
+
+There are two different ways to send events.  The first is to call the
+:bro:see:`BrokerComm::event` function directly.  The second option is to call
+the :bro:see:`BrokerComm::auto_event` function where you specify a
+particular event that will be automatically sent to peers whenever the
+event is called locally via the normal event invocation syntax.
+
+.. btest-include:: ${DOC_ROOT}/frameworks/broker/events-connector.bro
+
+Again, the subscription model is prefix-based.
+
+Message Format
+--------------
+
+For other applications that want to exchange event messages with Bro,
+the Broker message format is:
+
+.. code:: c++
+
+    broker::message{std::string{}, ...};
+
+The first parameter is the name of the event and the remaining ``...``
+are its arguments, which are any of the supported Broker data types as
+they correspond to the Bro types for the event named in the first
+parameter of the message.
+
+Remote Logging
+==============
+
+.. btest-include:: ${DOC_ROOT}/frameworks/broker/testlog.bro
+
+Use the :bro:see:`BrokerComm::subscribe_to_logs` function to advertise interest
+in logs written by peers.  The topic names that Bro uses are implicitly of the
+form "bro/log/<stream-name>".
+
+.. btest-include:: ${DOC_ROOT}/frameworks/broker/logs-listener.bro
+
+To send remote logs either redef :bro:see:`Log::enable_remote_logging` or
+use the :bro:see:`BrokerComm::enable_remote_logs` function.  The former
+allows any log stream to be sent to peers while the latter enables remote
+logging for particular streams.
+
+.. btest-include:: ${DOC_ROOT}/frameworks/broker/logs-connector.bro
+
+Message Format
+--------------
+
+For other applications that want to exchange log messages with Bro,
+the Broker message format is:
+
+.. code:: c++
+
+    broker::message{broker::enum_value{}, broker::record{}};
+
+The enum value corresponds to the stream's :bro:see:`Log::ID` value, and
+the record corresponds to a single entry of that log's columns record,
+in this case a ``Test::Info`` value.
+
+Tuning Access Control
+=====================
+
+By default, endpoints do not restrict the message topics that it sends
+to peers and do not restrict what message topics and data store
+identifiers get advertised to peers.  These are the default
+:bro:see:`BrokerComm::EndpointFlags` supplied to :bro:see:`BrokerComm::enable`.
+
+If not using the ``auto_publish`` flag, one can use the
+:bro:see:`BrokerComm::publish_topic` and :bro:see:`BrokerComm::unpublish_topic`
+functions to manipulate the set of message topics (must match exactly)
+that are allowed to be sent to peer endpoints.  These settings take
+precedence over the per-message ``peers`` flag supplied to functions
+that take a :bro:see:`BrokerComm::SendFlags` such as :bro:see:`BrokerComm::print`,
+:bro:see:`BrokerComm::event`, :bro:see:`BrokerComm::auto_event` or
+:bro:see:`BrokerComm::enable_remote_logs`.
+
+If not using the ``auto_advertise`` flag, one can use the
+:bro:see:`BrokerComm::advertise_topic` and
+:bro:see:`BrokerComm::unadvertise_topic` functions
+to manipulate the set of topic prefixes that are allowed to be
+advertised to peers.  If an endpoint does not advertise a topic prefix, then
+the only way peers can send messages to it is via the ``unsolicited``
+flag of :bro:see:`BrokerComm::SendFlags` and choosing a topic with a matching
+prefix (i.e. full topic may be longer than receivers prefix, just the
+prefix needs to match).
+
+Distributed Data Stores
+=======================
+
+There are three flavors of key-value data store interfaces: master,
+clone, and frontend.
+
+A frontend is the common interface to query and modify data stores.
+That is, a clone is a specific type of frontend and a master is also a
+specific type of frontend, but a standalone frontend can also exist to
+e.g. query and modify the contents of a remote master store without
+actually "owning" any of the contents itself.
+
+A master data store can be cloned from remote peers which may then
+perform lightweight, local queries against the clone, which
+automatically stays synchronized with the master store.  Clones cannot
+modify their content directly, instead they send modifications to the
+centralized master store which applies them and then broadcasts them to
+all clones.
+
+Master and clone stores get to choose what type of storage backend to
+use.  E.g. In-memory versus SQLite for persistence.  Note that if clones
+are used, then data store sizes must be able to fit within memory
+regardless of the storage backend as a single snapshot of the master
+store is sent in a single chunk to initialize the clone.
+
+Data stores also support expiration on a per-key basis either using an
+absolute point in time or a relative amount of time since the entry's
+last modification time.
+
+.. btest-include:: ${DOC_ROOT}/frameworks/broker/stores-listener.bro
+
+.. btest-include:: ${DOC_ROOT}/frameworks/broker/stores-connector.bro
+
+In the above example, if a local copy of the store contents isn't
+needed, just replace the :bro:see:`BrokerStore::create_clone` call with
+:bro:see:`BrokerStore::create_frontend`.  Queries will then be made against
+the remote master store instead of the local clone.
+
+Note that all data store queries must be made within Bro's asynchronous
+``when`` statements and must specify a timeout block.

doc/frameworks/broker/connecting-connector.bro

@@ -0,0 +1,18 @@
+const broker_port: port = 9999/tcp &redef;
+redef exit_only_after_terminate = T;
+redef BrokerComm::endpoint_name = "connector";
+
+event bro_init()
+	{
+	BrokerComm::enable();
+	BrokerComm::connect("127.0.0.1", broker_port, 1sec);
+	}
+
+event BrokerComm::outgoing_connection_established(peer_address: string,
+                                            peer_port: port,
+                                            peer_name: string)
+	{
+	print "BrokerComm::outgoing_connection_established",
+		  peer_address, peer_port, peer_name;
+	terminate();
+	}

doc/frameworks/broker/connecting-listener.bro

@@ -0,0 +1,20 @@
+const broker_port: port = 9999/tcp &redef;
+redef exit_only_after_terminate = T;
+redef BrokerComm::endpoint_name = "listener";
+
+event bro_init()
+	{
+	BrokerComm::enable();
+	BrokerComm::listen(broker_port, "127.0.0.1");
+	}
+
+event BrokerComm::incoming_connection_established(peer_name: string)
+	{
+	print "BrokerComm::incoming_connection_established", peer_name;
+	}
+
+event BrokerComm::incoming_connection_broken(peer_name: string)
+	{
+	print "BrokerComm::incoming_connection_broken", peer_name;
+	terminate();
+	}

doc/frameworks/broker/events-connector.bro

@@ -0,0 +1,31 @@
+const broker_port: port = 9999/tcp &redef;
+redef exit_only_after_terminate = T;
+redef BrokerComm::endpoint_name = "connector";
+global my_event: event(msg: string, c: count);
+global my_auto_event: event(msg: string, c: count);
+
+event bro_init()
+	{
+	BrokerComm::enable();
+	BrokerComm::connect("127.0.0.1", broker_port, 1sec);
+	BrokerComm::auto_event("bro/event/my_auto_event", my_auto_event);
+	}
+
+event BrokerComm::outgoing_connection_established(peer_address: string,
+                                            peer_port: port,
+                                            peer_name: string)
+	{
+	print "BrokerComm::outgoing_connection_established",
+	      peer_address, peer_port, peer_name;
+	BrokerComm::event("bro/event/my_event", BrokerComm::event_args(my_event, "hi", 0));
+	event my_auto_event("stuff", 88);
+	BrokerComm::event("bro/event/my_event", BrokerComm::event_args(my_event, "...", 1));
+	event my_auto_event("more stuff", 51);
+	BrokerComm::event("bro/event/my_event", BrokerComm::event_args(my_event, "bye", 2));
+	}
+
+event BrokerComm::outgoing_connection_broken(peer_address: string,
+                                       peer_port: port)
+	{
+	terminate();
+	}

doc/frameworks/broker/events-listener.bro

@@ -0,0 +1,36 @@
+const broker_port: port = 9999/tcp &redef;
+redef exit_only_after_terminate = T;
+redef BrokerComm::endpoint_name = "listener";
+global msg_count = 0;
+global my_event: event(msg: string, c: count);
+global my_auto_event: event(msg: string, c: count);
+
+event bro_init()
+	{
+	BrokerComm::enable();
+	BrokerComm::subscribe_to_events("bro/event/");
+	BrokerComm::listen(broker_port, "127.0.0.1");
+	}
+
+event BrokerComm::incoming_connection_established(peer_name: string)
+	{
+	print "BrokerComm::incoming_connection_established", peer_name;
+	}
+
+event my_event(msg: string, c: count)
+	{
+	++msg_count;
+	print "got my_event", msg, c;
+
+	if ( msg_count == 5 )
+		terminate();
+	}
+
+event my_auto_event(msg: string, c: count)
+	{
+	++msg_count;
+	print "got my_auto_event", msg, c;
+
+	if ( msg_count == 5 )
+		terminate();
+	}

doc/frameworks/broker/logs-connector.bro

@@ -0,0 +1,40 @@
+@load ./testlog
+
+const broker_port: port = 9999/tcp &redef;
+redef exit_only_after_terminate = T;
+redef BrokerComm::endpoint_name = "connector";
+redef Log::enable_local_logging = F;
+redef Log::enable_remote_logging = F;
+global n = 0;
+
+event bro_init()
+	{
+	BrokerComm::enable();
+	BrokerComm::enable_remote_logs(Test::LOG);
+	BrokerComm::connect("127.0.0.1", broker_port, 1sec);
+	}
+
+event do_write()
+	{
+	if ( n == 6 )
+		return;
+
+	Log::write(Test::LOG, [$msg = "ping", $num = n]);
+	++n;
+	event do_write();
+	}
+
+event BrokerComm::outgoing_connection_established(peer_address: string,
+                                            peer_port: port,
+                                            peer_name: string)
+	{
+	print "BrokerComm::outgoing_connection_established",
+	      peer_address, peer_port, peer_name;
+	event do_write();
+	}
+
+event BrokerComm::outgoing_connection_broken(peer_address: string,
+                                       peer_port: port)
+	{
+	terminate();
+	}

doc/frameworks/broker/logs-listener.bro

@@ -0,0 +1,25 @@
+@load ./testlog
+
+const broker_port: port = 9999/tcp &redef;
+redef exit_only_after_terminate = T;
+redef BrokerComm::endpoint_name = "listener";
+
+event bro_init()
+	{
+	BrokerComm::enable();
+	BrokerComm::subscribe_to_logs("bro/log/Test::LOG");
+	BrokerComm::listen(broker_port, "127.0.0.1");
+	}
+
+event BrokerComm::incoming_connection_established(peer_name: string)
+	{
+	print "BrokerComm::incoming_connection_established", peer_name;
+	}
+
+event Test::log_test(rec: Test::Info)
+	{
+	print "wrote log", rec;
+
+	if ( rec$num == 5 )
+		terminate();
+	}

doc/frameworks/broker/printing-connector.bro

@@ -0,0 +1,26 @@
+const broker_port: port = 9999/tcp &redef;
+redef exit_only_after_terminate = T;
+redef BrokerComm::endpoint_name = "connector";
+
+event bro_init()
+	{
+	BrokerComm::enable();
+	BrokerComm::connect("127.0.0.1", broker_port, 1sec);
+	}
+
+event BrokerComm::outgoing_connection_established(peer_address: string,
+                                            peer_port: port,
+                                            peer_name: string)
+	{
+	print "BrokerComm::outgoing_connection_established",
+	      peer_address, peer_port, peer_name;
+	BrokerComm::print("bro/print/hi", "hello");
+	BrokerComm::print("bro/print/stuff", "...");
+	BrokerComm::print("bro/print/bye", "goodbye");
+	}
+
+event BrokerComm::outgoing_connection_broken(peer_address: string,
+                                       peer_port: port)
+	{
+	terminate();
+	}

doc/frameworks/broker/printing-listener.bro

@@ -0,0 +1,25 @@
+const broker_port: port = 9999/tcp &redef;
+redef exit_only_after_terminate = T;
+redef BrokerComm::endpoint_name = "listener";
+global msg_count = 0;
+
+event bro_init()
+	{
+	BrokerComm::enable();
+	BrokerComm::subscribe_to_prints("bro/print/");
+	BrokerComm::listen(broker_port, "127.0.0.1");
+	}
+
+event BrokerComm::incoming_connection_established(peer_name: string)
+	{
+	print "BrokerComm::incoming_connection_established", peer_name;
+	}
+
+event BrokerComm::print_handler(msg: string)
+	{
+	++msg_count;
+	print "got print message", msg;
+
+	if ( msg_count == 3 )
+		terminate();
+	}

doc/frameworks/broker/stores-connector.bro

@@ -0,0 +1,53 @@
+const broker_port: port = 9999/tcp &redef;
+redef exit_only_after_terminate = T;
+
+global h: opaque of BrokerStore::Handle;
+
+function dv(d: BrokerComm::Data): BrokerComm::DataVector
+	{
+	local rval: BrokerComm::DataVector;
+	rval[0] = d;
+	return rval;
+	}
+
+global ready: event();
+
+event BrokerComm::outgoing_connection_broken(peer_address: string,
+                                       peer_port: port)
+	{
+	terminate();
+	}
+
+event BrokerComm::outgoing_connection_established(peer_address: string,
+                                            peer_port: port,
+                                            peer_name: string)
+	{
+	local myset: set[string] = {"a", "b", "c"};
+	local myvec: vector of string = {"alpha", "beta", "gamma"};
+	h = BrokerStore::create_master("mystore");
+	BrokerStore::insert(h, BrokerComm::data("one"), BrokerComm::data(110));
+	BrokerStore::insert(h, BrokerComm::data("two"), BrokerComm::data(223));
+	BrokerStore::insert(h, BrokerComm::data("myset"), BrokerComm::data(myset));
+	BrokerStore::insert(h, BrokerComm::data("myvec"), BrokerComm::data(myvec));
+	BrokerStore::increment(h, BrokerComm::data("one"));
+	BrokerStore::decrement(h, BrokerComm::data("two"));
+	BrokerStore::add_to_set(h, BrokerComm::data("myset"), BrokerComm::data("d"));
+	BrokerStore::remove_from_set(h, BrokerComm::data("myset"), BrokerComm::data("b"));
+	BrokerStore::push_left(h, BrokerComm::data("myvec"), dv(BrokerComm::data("delta")));
+	BrokerStore::push_right(h, BrokerComm::data("myvec"), dv(BrokerComm::data("omega")));
+
+	when ( local res = BrokerStore::size(h) )
+		{
+		print "master size", res;
+		event ready();
+		}
+	timeout 10sec
+		{ print "timeout"; }
+	}
+
+event bro_init()
+	{
+	BrokerComm::enable();
+	BrokerComm::connect("127.0.0.1", broker_port, 1secs);
+	BrokerComm::auto_event("bro/event/ready", ready);
+	}

doc/frameworks/broker/stores-listener.bro

@@ -0,0 +1,43 @@
+const broker_port: port = 9999/tcp &redef;
+redef exit_only_after_terminate = T;
+
+global h: opaque of BrokerStore::Handle;
+global expected_key_count = 4;
+global key_count = 0;
+
+function do_lookup(key: string)
+	{
+	when ( local res = BrokerStore::lookup(h, BrokerComm::data(key)) )
+		{
+		++key_count;
+		print "lookup", key, res;
+
+		if ( key_count == expected_key_count )
+			terminate();
+		}
+	timeout 10sec
+		{ print "timeout", key; }
+	}
+
+event ready()
+	{
+	h = BrokerStore::create_clone("mystore");
+
+	when ( local res = BrokerStore::keys(h) )
+		{
+		print "clone keys", res;
+		do_lookup(BrokerComm::refine_to_string(BrokerComm::vector_lookup(res$result, 0)));
+		do_lookup(BrokerComm::refine_to_string(BrokerComm::vector_lookup(res$result, 1)));
+		do_lookup(BrokerComm::refine_to_string(BrokerComm::vector_lookup(res$result, 2)));
+		do_lookup(BrokerComm::refine_to_string(BrokerComm::vector_lookup(res$result, 3)));
+		}
+	timeout 10sec
+		{ print "timeout"; }
+	}
+
+event bro_init()
+	{
+	BrokerComm::enable();
+	BrokerComm::subscribe_to_events("bro/event/ready");
+	BrokerComm::listen(broker_port, "127.0.0.1");
+	}

doc/frameworks/broker/testlog.bro

@@ -0,0 +1,18 @@
+module Test;
+
+export {
+	redef enum Log::ID += { LOG };
+
+	type Info: record {
+		msg: string &log;
+		num: count &log;
+	};
+
+	global log_test: event(rec: Test::Info);
+}
+
+event bro_init() &priority=5
+	{
+	BrokerComm::enable();
+	Log::create_stream(Test::LOG, [$columns=Test::Info, $ev=log_test, $path="test"]);
+	}

doc/frameworks/file_analysis_02.bro

@@ -1,7 +1,8 @@
-event file_new(f: fa_file)
+event file_sniff(f: fa_file, meta: fa_metadata)
     {
+	if ( ! meta?$mime_type ) return;
     print "new file", f$id;
-    if ( f?$mime_type && f$mime_type == "text/plain" )
+    if ( meta$mime_type == "text/plain" )
         Files::add_analyzer(f, Files::ANALYZER_MD5);
     }
 

doc/frameworks/geoip.rst

@@ -20,11 +20,13 @@ GeoLocation
 Install libGeoIP
 ----------------
 
+Before building Bro, you need to install libGeoIP.
+
 * FreeBSD:
 
   .. console::
 
-      sudo pkg_add -r GeoIP
+      sudo pkg install GeoIP
 
 * RPM/RedHat-based Linux:
 
@@ -40,80 +42,99 @@ Install libGeoIP
 
 * Mac OS X:
 
-  Vanilla OS X installations don't ship with libGeoIP, but if
+  You need to install from your preferred package management system
-  installed from your preferred package management system (e.g.
+  (e.g. MacPorts, Fink, or Homebrew).  The name of the package that you need
-  MacPorts, Fink, or Homebrew), they should be automatically detected
+  may be libgeoip, geoip, or geoip-dev, depending on which package management
-  and Bro will compile against them.
+  system you are using.
 
 
 GeoIPLite Database Installation
-------------------------------------
+-------------------------------
 
 A country database for GeoIPLite is included when you do the C API
 install, but for Bro, we are using the city database which includes
 cities and regions in addition to countries.
 
 `Download <http://www.maxmind.com/app/geolitecity>`__ the GeoLite city
-binary database.
+binary database:
 
-  .. console::
+.. console::
 
     wget http://geolite.maxmind.com/download/geoip/database/GeoLiteCity.dat.gz
     gunzip GeoLiteCity.dat.gz
 
-Next, the file needs to be put in the database directory.  This directory
+Next, the file needs to be renamed and put in the GeoIP database directory.
-should already exist and will vary depending on which platform and package
+This directory should already exist and will vary depending on which platform
-you are using.  For FreeBSD, use ``/usr/local/share/GeoIP``.  For Linux,
+and package you are using.  For FreeBSD, use ``/usr/local/share/GeoIP``.  For
-use ``/usr/share/GeoIP`` or ``/var/lib/GeoIP`` (choose whichever one
+Linux, use ``/usr/share/GeoIP`` or ``/var/lib/GeoIP`` (choose whichever one
 already exists).
     
-  .. console::
+.. console::
 
     mv GeoLiteCity.dat <path_to_database_dir>/GeoIPCity.dat
 
+Note that there is a separate database for IPv6 addresses, which can also
+be installed if you want GeoIP functionality for IPv6.
+
+Testing
+-------
+
+Before using the GeoIP functionality, it is a good idea to verify that
+everything is setup correctly.  After installing libGeoIP and the GeoIP city
+database, and building Bro, you can quickly check if the GeoIP functionality
+works by running a command like this:
+
+.. console::
+
+    bro -e "print lookup_location(8.8.8.8);"
+
+If you see an error message similar to "Failed to open GeoIP City database",
+then you may need to either rename or move your GeoIP city database file (the
+error message should give you the full pathname of the database file that
+Bro is looking for).
+
+If you see an error message similar to "Bro was not configured for GeoIP
+support", then you need to rebuild Bro and make sure it is linked against
+libGeoIP.  Normally, if libGeoIP is installed correctly then it should
+automatically be found when building Bro.  If this doesn't happen, then
+you may need to specify the path to the libGeoIP installation
+(e.g. ``./configure --with-geoip=<path>``).
 
 Usage
 -----
 
-There is a single built in function that provides the GeoIP
+There is a built-in function that provides the GeoIP functionality:
-functionality:
 
 .. code:: bro
 
     function lookup_location(a:addr): geo_location
 
-There is also the :bro:see:`geo_location` data structure that is returned
+The return value of the :bro:see:`lookup_location` function is a record
-from the :bro:see:`lookup_location` function:
+type called :bro:see:`geo_location`, and it consists of several fields
-
+containing the country, region, city, latitude, and longitude of the specified
-.. code:: bro
+IP address.  Since one or more fields in this record will be uninitialized
-
+for some IP addresses (for example, the country and region of an IP address
-    type geo_location: record {
+might be known, but the city could be unknown), a field should be checked
-      country_code: string;
+if it has a value before trying to access the value.
-      region: string;
-      city: string;
-      latitude: double;
-      longitude: double;
-    };
-
 
 Example
 -------
 
-To write a line in a log file for every ftp connection from hosts in
+To show every ftp connection from hosts in Ohio, this is now very easy:
-Ohio, this is now very easy:
 
 .. code:: bro
 
-    global ftp_location_log: file = open_log_file("ftp-location");
-    
     event ftp_reply(c: connection, code: count, msg: string, cont_resp: bool)
     {
       local client = c$id$orig_h;
       local loc = lookup_location(client);
-      if (loc$region == "OH" && loc$country_code == "US")
+
+      if (loc?$region && loc$region == "OH" && loc$country_code == "US")
       {
-        print ftp_location_log, fmt("FTP Connection from:%s (%s,%s,%s)", client, loc$city, loc$region, loc$country_code); 
+        local city = loc?$city ? loc$city : "<unknown>";
+
+        print fmt("FTP Connection from:%s (%s,%s,%s)", client, city,
+          loc$region, loc$country_code);
       }
     }
 
-

doc/frameworks/index.rst

@@ -14,4 +14,4 @@ Frameworks
    notice
    signatures
    sumstats
-
+   broker

doc/frameworks/input.rst

@@ -32,7 +32,8 @@ For this example we assume that we want to import data from a blacklist
 that contains server IP addresses as well as the timestamp and the reason
 for the block.
 
-An example input file could look like this:
+An example input file could look like this (note that all fields must be
+tab-separated):
 
 ::
 
@@ -63,19 +64,23 @@ The two records are defined as:
                 reason: string;
         };
 
-Note that the names of the fields in the record definitions have to correspond
+Note that the names of the fields in the record definitions must correspond
 to the column names listed in the '#fields' line of the log file, in this
-case 'ip', 'timestamp', and 'reason'.
+case 'ip', 'timestamp', and 'reason'.  Also note that the ordering of the
+columns does not matter, because each column is identified by name.
 
-The log file is read into the table with a simple call of the ``add_table``
+The log file is read into the table with a simple call of the
-function:
+:bro:id:`Input::add_table` function:
 
 .. code:: bro
 
         global blacklist: table[addr] of Val = table();
 
-        Input::add_table([$source="blacklist.file", $name="blacklist", $idx=Idx, $val=Val, $destination=blacklist]);
+        event bro_init() {
+            Input::add_table([$source="blacklist.file", $name="blacklist",
+                              $idx=Idx, $val=Val, $destination=blacklist]);
             Input::remove("blacklist");
+        }
 
 With these three lines we first create an empty table that should contain the
 blacklist data and then instruct the input framework to open an input stream
@@ -92,7 +97,7 @@ Because of this, the data is not immediately accessible. Depending on the
 size of the data source it might take from a few milliseconds up to a few
 seconds until all data is present in the table. Please note that this means
 that when Bro is running without an input source or on very short captured
-files, it might terminate before the data is present in the system (because
+files, it might terminate before the data is present in the table (because
 Bro already handled all packets before the import thread finished).
 
 Subsequent calls to an input source are queued until the previous action has
@@ -101,8 +106,8 @@ been completed. Because of this, it is, for example, possible to call
 will remain queued until the first read has been completed.
 
 Once the input framework finishes reading from a data source, it fires
-the ``end_of_data`` event. Once this event has been received all data
+the :bro:id:`Input::end_of_data` event. Once this event has been received all
-from the input file is available in the table.
+data from the input file is available in the table.
 
 .. code:: bro
 
@@ -111,9 +116,9 @@ from the input file is available in the table.
                 print blacklist;
         }
 
-The table can also already be used while the data is still being read - it
+The table can be used while the data is still being read - it
-just might not contain all lines in the input file when the event has not
+just might not contain all lines from the input file before the event has
-yet fired. After it has been populated it can be used like any other Bro
+fired. After the table has been populated it can be used like any other Bro
 table and blacklist entries can easily be tested:
 
 .. code:: bro
@@ -130,10 +135,11 @@ changing. For these cases, the Bro input framework supports several ways to
 deal with changing data files.
 
 The first, very basic method is an explicit refresh of an input stream. When
-an input stream is open, the function ``force_update`` can be called. This
+an input stream is open (this means it has not yet been removed by a call to
-will trigger a complete refresh of the table; any changed elements from the
+:bro:id:`Input::remove`), the function :bro:id:`Input::force_update` can be
-file will be updated.  After the update is finished the ``end_of_data``
+called.  This will trigger a complete refresh of the table; any changed
-event will be raised.
+elements from the file will be updated.  After the update is finished the
+:bro:id:`Input::end_of_data` event will be raised.
 
 In our example the call would look like:
 
@@ -141,30 +147,35 @@ In our example the call would look like:
 
         Input::force_update("blacklist");
 
-The input framework also supports two automatic refresh modes. The first mode
+Alternatively, the input framework can automatically refresh the table
-continually checks if a file has been changed. If the file has been changed, it
+contents when it detects a change to the input file.  To use this feature,
+you need to specify a non-default read mode by setting the ``mode`` option
+of the :bro:id:`Input::add_table` call.  Valid values are ``Input::MANUAL``
+(the default), ``Input::REREAD`` and ``Input::STREAM``.  For example,
+setting the value of the ``mode`` option in the previous example
+would look like this:
+
+.. code:: bro
+
+        Input::add_table([$source="blacklist.file", $name="blacklist",
+                          $idx=Idx, $val=Val, $destination=blacklist,
+                          $mode=Input::REREAD]);
+
+When using the reread mode (i.e., ``$mode=Input::REREAD``), Bro continually
+checks if the input file has been changed. If the file has been changed, it
 is re-read and the data in the Bro table is updated to reflect the current
 state.  Each time a change has been detected and all the new data has been
 read into the table, the ``end_of_data`` event is raised.
 
-The second mode is a streaming mode. This mode assumes that the source data
+When using the streaming mode (i.e., ``$mode=Input::STREAM``), Bro assumes
-file is an append-only file to which new data is continually appended. Bro
+that the source data file is an append-only file to which new data is
-continually checks for new data at the end of the file and will add the new
+continually appended. Bro continually checks for new data at the end of
-data to the table.  If newer lines in the file have the same index as previous
+the file and will add the new data to the table.  If newer lines in the
-lines, they will overwrite the values in the output table.  Because of the
+file have the same index as previous lines, they will overwrite the
-nature of streaming reads (data is continually added to the table),
+values in the output table.  Because of the nature of streaming reads
-the ``end_of_data`` event is never raised when using streaming reads.
+(data is continually added to the table), the ``end_of_data`` event
+is never raised when using streaming reads.
 
-The reading mode can be selected by setting the ``mode`` option of the
-add_table call.  Valid values are ``MANUAL`` (the default), ``REREAD``
-and ``STREAM``.
-
-Hence, when adding ``$mode=Input::REREAD`` to the previous example, the
-blacklist table will always reflect the state of the blacklist input file.
-
-.. code:: bro
-
-        Input::add_table([$source="blacklist.file", $name="blacklist", $idx=Idx, $val=Val, $destination=blacklist, $mode=Input::REREAD]);
 
 Receiving change events
 -----------------------
@@ -173,34 +184,40 @@ When re-reading files, it might be interesting to know exactly which lines in
 the source files have changed.
 
 For this reason, the input framework can raise an event each time when a data
-item is added to, removed from or changed in a table.
+item is added to, removed from, or changed in a table.
 
-The event definition looks like this:
+The event definition looks like this (note that you can change the name of
+this event in your own Bro script):
 
 .. code:: bro
 
-        event entry(description: Input::TableDescription, tpe: Input::Event, left: Idx, right: Val) {
+        event entry(description: Input::TableDescription, tpe: Input::Event,
-                # act on values
+                    left: Idx, right: Val) {
+                # do something here...
+                print fmt("%s = %s", left, right);
         }
 
-The event has to be specified in ``$ev`` in the ``add_table`` call:
+The event must be specified in ``$ev`` in the ``add_table`` call:
 
 .. code:: bro
 
-        Input::add_table([$source="blacklist.file", $name="blacklist", $idx=Idx, $val=Val, $destination=blacklist, $mode=Input::REREAD, $ev=entry]);
+        Input::add_table([$source="blacklist.file", $name="blacklist",
+                          $idx=Idx, $val=Val, $destination=blacklist,
+                          $mode=Input::REREAD, $ev=entry]);
 
-The ``description`` field of the event contains the arguments that were
+The ``description`` argument of the event contains the arguments that were
 originally supplied to the add_table call.  Hence, the name of the stream can,
-for example, be accessed with ``description$name``. ``tpe`` is an enum
+for example, be accessed with ``description$name``. The ``tpe`` argument of the
-containing the type of the change that occurred.
+event is an enum containing the type of the change that occurred.
 
 If a line that was not previously present in the table has been added,
-then ``tpe`` will contain ``Input::EVENT_NEW``. In this case ``left`` contains
+then the value of ``tpe`` will be ``Input::EVENT_NEW``. In this case ``left``
-the index of the added table entry and ``right`` contains the values of the
+contains the index of the added table entry and ``right`` contains the
-added entry.
+values of the added entry.
 
 If a table entry that already was present is altered during the re-reading or
-streaming read of a file, ``tpe`` will contain ``Input::EVENT_CHANGED``. In
+streaming read of a file, then the value of ``tpe`` will be
+``Input::EVENT_CHANGED``. In
 this case ``left`` contains the index of the changed table entry and ``right``
 contains the values of the entry before the change. The reason for this is
 that the table already has been updated when the event is raised. The current
@@ -208,8 +225,9 @@ value in the table can be ascertained by looking up the current table value.
 Hence it is possible to compare the new and the old values of the table.
 
 If a table element is removed because it was no longer present during a
-re-read, then ``tpe`` will contain ``Input::REMOVED``.  In this case ``left``
+re-read, then the value of ``tpe`` will be ``Input::EVENT_REMOVED``.  In this
-contains the index and ``right`` the values of the removed element.
+case ``left`` contains the index and ``right`` the values of the removed
+element.
 
 
 Filtering data during import
@@ -222,24 +240,26 @@ can either accept or veto the change by returning true for an accepted
 change and false for a rejected change. Furthermore, it can alter the data
 before it is written to the table.
 
-The following example filter will reject to add entries to the table when
+The following example filter will reject adding entries to the table when
 they were generated over a month ago. It will accept all changes and all
 removals of values that are already present in the table.
 
 .. code:: bro
 
-        Input::add_table([$source="blacklist.file", $name="blacklist", $idx=Idx, $val=Val, $destination=blacklist, $mode=Input::REREAD,
+        Input::add_table([$source="blacklist.file", $name="blacklist",
+                          $idx=Idx, $val=Val, $destination=blacklist,
+                          $mode=Input::REREAD,
                           $pred(typ: Input::Event, left: Idx, right: Val) = {
                             if ( typ != Input::EVENT_NEW ) {
                                 return T;
                             }
-                                return ( ( current_time() - right$timestamp ) < (30 day) );
+                            return (current_time() - right$timestamp) < 30day;
                           }]);
 
 To change elements while they are being imported, the predicate function can
 manipulate ``left`` and ``right``. Note that predicate functions are called
 before the change is committed to the table. Hence, when a table element is
-changed (``tpe`` is ``INPUT::EVENT_CHANGED``), ``left`` and ``right``
+changed (``typ`` is ``Input::EVENT_CHANGED``), ``left`` and ``right``
 contain the new values, but the destination (``blacklist`` in our example)
 still contains the old values. This allows predicate functions to examine
 the changes between the old and the new version before deciding if they
@@ -250,14 +270,19 @@ Different readers
 
 The input framework supports different kinds of readers for different kinds
 of source data files. At the moment, the default reader reads ASCII files
-formatted in the Bro log file format (tab-separated values). At the moment,
+formatted in the Bro log file format (tab-separated values with a "#fields"
-Bro comes with two other readers. The ``RAW`` reader reads a file that is
+header line).  Several other readers are included in Bro.
-split by a specified record separator (usually newline). The contents are
+
+The raw reader reads a file that is
+split by a specified record separator (newline by default). The contents are
 returned line-by-line as strings; it can, for example, be used to read
 configuration files and the like and is probably
 only useful in the event mode and not for reading data to tables.
 
-Another included reader is the ``BENCHMARK`` reader, which is being used
+The binary reader is intended to be used with file analysis input streams (and
+is the default type of reader for those streams).
+
+The benchmark reader is being used
 to optimize the speed of the input framework. It can generate arbitrary
 amounts of semi-random data in all Bro data types supported by the input
 framework.
@@ -270,75 +295,17 @@ aforementioned ones:
 
    logging-input-sqlite
 
-Add_table options
------------------
-
-This section lists all possible options that can be used for the add_table
-function and gives a short explanation of their use. Most of the options
-already have been discussed in the previous sections.
-
-The possible fields that can be set for a table stream are:
-
-        ``source``
-                A mandatory string identifying the source of the data.
-                For the ASCII reader this is the filename.
-
-        ``name``
-                A mandatory name for the filter that can later be used
-                to manipulate it further.
-
-        ``idx``
-                Record type that defines the index of the table.
-
-        ``val``
-                Record type that defines the values of the table.
-
-        ``reader``
-                The reader used for this stream. Default is ``READER_ASCII``.
-
-        ``mode``
-                The mode in which the stream is opened. Possible values are
-                ``MANUAL``, ``REREAD`` and ``STREAM``.  Default is ``MANUAL``.
-                ``MANUAL`` means that the file is not updated after it has
-                been read. Changes to the file will not be reflected in the
-                data Bro knows.  ``REREAD`` means that the whole file is read
-                again each time a change is found. This should be used for
-                files that are mapped to a table where individual lines can
-                change.  ``STREAM`` means that the data from the file is
-                streamed. Events / table entries will be generated as new
-                data is appended to the file.
-
-        ``destination``
-                The destination table.
-
-        ``ev``
-                Optional event that is raised, when values are added to,
-                changed in, or deleted from the table.  Events are passed an
-                Input::Event description as the first argument, the index
-                record as the second argument and the values as the third
-                argument.
-
-        ``pred``
-                Optional predicate, that can prevent entries from being added
-                to the table and events from being sent.
-
-        ``want_record``
-                Boolean value, that defines if the event wants to receive the
-                fields inside of a single record value, or individually
-                (default).  This can be used if ``val`` is a record
-                containing only one type. In this case, if ``want_record`` is
-                set to false, the table will contain elements of the type
-                contained in ``val``.
 
 Reading Data to Events
 ======================
 
 The second supported mode of the input framework is reading data to Bro
-events instead of reading them to a table using event streams.
+events instead of reading them to a table.
 
 Event streams work very similarly to table streams that were already
 discussed in much detail. To read the blacklist of the previous example
-into an event stream, the following Bro code could be used:
+into an event stream, the :bro:id:`Input::add_event` function is used.
+For example:
 
 .. code:: bro
 
@@ -348,12 +315,15 @@ into an event stream, the following Bro code could be used:
                 reason: string;
         };
 
-        event blacklistentry(description: Input::EventDescription, tpe: Input::Event, ip: addr, timestamp: time, reason: string) {
+        event blacklistentry(description: Input::EventDescription,
-                # work with event data
+                             t: Input::Event, data: Val) {
+                # do something here...
+                print "data:", data;
         }
 
         event bro_init() {
-                Input::add_event([$source="blacklist.file", $name="blacklist", $fields=Val, $ev=blacklistentry]);
+                Input::add_event([$source="blacklist.file", $name="blacklist",
+                                  $fields=Val, $ev=blacklistentry]);
         }
 
 
@@ -364,52 +334,3 @@ data types are provided in a single record definition.
 Apart from this, event streams work exactly the same as table streams and
 support most of the options that are also supported for table streams.
 
-The options that can be set when creating an event stream with
-``add_event`` are:
-
-        ``source``
-                A mandatory string identifying the source of the data.
-                For the ASCII reader this is the filename.
-
-        ``name``
-                A mandatory name for the stream that can later be used
-                to remove it.
-
-        ``fields``
-                Name of a record type containing the fields, which should be
-                retrieved from the input stream.
-
-        ``ev``
-                The event which is fired, after a line has been read from the
-                input source.  The first argument that is passed to the event
-                is an Input::Event structure, followed by the data, either
-                inside of a record (if ``want_record is set``) or as
-                individual fields.  The Input::Event structure can contain
-                information, if the received line is ``NEW``, has been
-                ``CHANGED`` or ``DELETED``. Since the ASCII reader cannot
-                track this information for event filters, the value is
-                always ``NEW`` at the moment.
-
-        ``mode``
-                The mode in which the stream is opened. Possible values are
-                ``MANUAL``, ``REREAD`` and ``STREAM``.  Default is ``MANUAL``.
-                ``MANUAL`` means that the file is not updated after it has
-                been read. Changes to the file will not be reflected in the
-                data Bro knows.  ``REREAD`` means that the whole file is read
-                again each time a change is found. This should be used for
-                files that are mapped to a table where individual lines can
-                change.  ``STREAM`` means that the data from the file is
-                streamed. Events / table entries will be generated as new
-                data is appended to the file.
-
-        ``reader``
-                The reader used for this stream. Default is ``READER_ASCII``.
-
-        ``want_record``
-                Boolean value, that defines if the event wants to receive the
-                fields inside of a single record value, or individually
-                (default). If this is set to true, the event will receive a
-                single record of the type provided in ``fields``.
-
-
-

doc/frameworks/intel.rst

@@ -14,32 +14,35 @@ consume that data, make it available for matching, and provide
 infrastructure around improving performance, memory utilization, and
 generally making all of this easier.
 
-Data in the Intelligence Framework is the atomic piece of intelligence
+Data in the Intelligence Framework is an atomic piece of intelligence
 such as an IP address or an e-mail address along with a suite of
 metadata about it such as a freeform source field, a freeform
 descriptive field and a URL which might lead to more information about
 the specific item.  The metadata in the default scripts has been
 deliberately kept minimal so that the community can find the
-appropriate fields that need added by writing scripts which extend the
+appropriate fields that need to be added by writing scripts which extend the
 base record using the normal record extension mechanism.
 
 Quick Start
 -----------
 
-Load the package of scripts that sends data into the Intelligence
-Framework to be checked by loading this script in local.bro::
-
-	@load policy/frameworks/intel/seen
-
 Refer to the "Loading Intelligence" section below to see the format
 for Intelligence Framework text files, then load those text files with
 this line in local.bro::
 
 	redef Intel::read_files += { "/somewhere/yourdata.txt" };
 
-The data itself only needs to reside on the manager if running in a
+The text files need to reside only on the manager if running in a
 cluster.
 
+Add the following line to local.bro in order to load the scripts
+that send "seen" data into the Intelligence Framework to be checked against
+the loaded intelligence data::
+
+	@load policy/frameworks/intel/seen
+
+Intelligence data matches will be logged to the intel.log file.
+
 Architecture
 ------------
 
@@ -58,8 +61,10 @@ manager is the only node that needs the intelligence data.  The
 intelligence framework has distribution mechanisms which will push
 data out to all of the nodes that need it.
 
-Here is an example of the intelligence data format.  Note that all
+Here is an example of the intelligence data format (note that there will be
-whitespace field separators are literal tabs and fields containing only a
+additional fields if you are using CIF intelligence data or if you are
+using the policy/frameworks/intel/do_notice script).  Note that all fields
+must be separated by a single tab character and fields containing only a
 hyphen are considered to be null values. ::
 
 	#fields	indicator	indicator_type	meta.source	meta.desc	meta.url
@@ -69,8 +74,21 @@ hyphen are considered to be null values. ::
 For a list of all built-in `indicator_type` values, please refer to the
 documentation of :bro:see:`Intel::Type`.
 
-To load the data once files are created, use the following example
+Note that if you are using data from the Collective Intelligence Framework,
-code to define files to load with your own file names of course::
+then you will need to add the following line to your local.bro in order
+to support additional metadata fields used by CIF::
+
+	@load policy/integration/collective-intel
+
+There is a simple mechanism to raise a Bro notice (of type Intel::Notice)
+for user-specified intelligence matches.  To use this feature, add the
+following line to local.bro in order to support additional metadata fields
+(documented in the :bro:see:`Intel::MetaData` record)::
+
+	@load policy/frameworks/intel/do_notice
+
+To load the data once the files are created, use the following example
+to specify which files to load (with your own file names of course)::
 
 	redef Intel::read_files += {
 		"/somewhere/feed1.txt",
@@ -85,24 +103,23 @@ Seen Data
 
 When some bit of data is extracted (such as an email address in the
 "From" header in a message over SMTP), the Intelligence Framework
-needs to be informed that this data was discovered and it's presence
+needs to be informed that this data was discovered so that its presence
-should be checked within the intelligence data set.  This is
+will be checked within the loaded intelligence data.  This is
-accomplished through the :bro:see:`Intel::seen` function.
+accomplished through the :bro:see:`Intel::seen` function, however
+typically users won't need to work with this function due to the
+scripts included with Bro that will call this function.
 
-Typically users won't need to work with this function due to built in
+To load all of the scripts included with Bro for sending "seen" data to
-hook scripts that Bro ships with that will "see" data and send it into
+the intelligence framework, just add this line to local.bro::
-the intelligence framework.  A user may only need to load the entire
+
-package of hook scripts as a module or pick and choose specific
+	@load policy/frameworks/intel/seen
-scripts to load.  Keep in mind that as more data is sent into the
+
+Alternatively, specific scripts in that directory can be loaded.
+Keep in mind that as more data is sent into the
 intelligence framework, the CPU load consumed by Bro will increase
 depending on how many times the :bro:see:`Intel::seen` function is
 being called which is heavily traffic dependent.
 
-The full package of hook scripts that Bro ships with for sending this
-"seen" data into the intelligence framework can be loading by adding
-this line to local.bro::
-
-	@load policy/frameworks/intel/seen
 
 Intelligence Matches
 ********************
@@ -111,6 +128,7 @@ Against all hopes, most networks will eventually have a hit on
 intelligence data which could indicate a possible compromise or other
 unwanted activity.  The Intelligence Framework provides an event that
 is generated whenever a match is discovered named :bro:see:`Intel::match`.
+
 Due to design restrictions placed upon
 the intelligence framework, there is no assurance as to where this
 event will be generated.  It could be generated on the worker where
@@ -119,3 +137,7 @@ handled, only the data given as event arguments to the event can be
 assured since the host where the data was seen may not be where
 ``Intel::match`` is handled.
 
+Intelligence matches are logged to the intel.log file.  For a description of
+each field in that file, see the documentation for the :bro:see:`Intel::Info`
+record.
+

doc/frameworks/logging-input-sqlite.rst

@@ -23,17 +23,18 @@ In contrast to the ASCII reader and writer, the SQLite plugins have not yet
 seen extensive use in production environments. While we are not aware
 of any issues with them, we urge to caution when using them
 in production environments. There could be lingering issues which only occur
-when the plugins are used with high amounts of data or in high-load environments.
+when the plugins are used with high amounts of data or in high-load
+environments.
 
 Logging Data into SQLite Databases
 ==================================
 
 Logging support for SQLite is available in all Bro installations starting with
-version 2.2. There is no need to load any additional scripts or for any compile-time
+version 2.2. There is no need to load any additional scripts or for any
-configurations.
+compile-time configurations.
 
-Sending data from existing logging streams to SQLite is rather straightforward. You
+Sending data from existing logging streams to SQLite is rather straightforward.
-have to define a filter which specifies SQLite as the writer.
+You have to define a filter which specifies SQLite as the writer.
 
 The following example code adds SQLite as a filter for the connection log:
 
@@ -44,15 +45,15 @@ The following example code adds SQLite as a filter for the connection log:
     # Make sure this parses correctly at least.
     @TEST-EXEC: bro ${DOC_ROOT}/frameworks/sqlite-conn-filter.bro
 
-Bro will create the database file ``/var/db/conn.sqlite``, if it does not already exist.
+Bro will create the database file ``/var/db/conn.sqlite``, if it does not
-It will also create a table with the name ``conn`` (if it does not exist) and start
+already exist.  It will also create a table with the name ``conn`` (if it
-appending connection information to the table.
+does not exist) and start appending connection information to the table.
 
-At the moment, SQLite databases are not rotated the same way ASCII log-files are. You
+At the moment, SQLite databases are not rotated the same way ASCII log-files
-have to take care to create them in an adequate location.
+are. You have to take care to create them in an adequate location.
 
-If you examine the resulting SQLite database, the schema will contain the same fields
+If you examine the resulting SQLite database, the schema will contain the
-that are present in the ASCII log files::
+same fields that are present in the ASCII log files::
 
     # sqlite3 /var/db/conn.sqlite
 
@@ -67,35 +68,39 @@ that are present in the ASCII log files::
     'id.orig_p' integer,
     ...
 
-Note that the ASCII ``conn.log`` will still be created. To disable the ASCII writer for a
+Note that the ASCII ``conn.log`` will still be created. To prevent this file
-log stream, you can remove the default filter:
+from being created, you can remove the default filter:
 
 .. code:: bro
 
     Log::remove_filter(Conn::LOG, "default");
 
 
-To create a custom SQLite log file, you have to create a new log stream that contains
+To create a custom SQLite log file, you have to create a new log stream
-just the information you want to commit to the database. Please refer to the
+that contains just the information you want to commit to the database.
-:ref:`framework-logging` documentation on how to create custom log streams.
+Please refer to the :ref:`framework-logging` documentation on how to
+create custom log streams.
 
 Reading Data from SQLite Databases
 ==================================
 
-Like logging support, support for reading data from SQLite databases is built into Bro starting
+Like logging support, support for reading data from SQLite databases is
-with version 2.2.
+built into Bro starting with version 2.2.
 
-Just as with the text-based input readers (please refer to the :ref:`framework-input`
+Just as with the text-based input readers (please refer to the
-documentation for them and for basic information on how to use the input-framework), the SQLite reader
+:ref:`framework-input` documentation for them and for basic information
-can be used to read data - in this case the result of SQL queries - into tables or into events.
+on how to use the input framework), the SQLite reader can be used to
+read data - in this case the result of SQL queries - into tables or into
+events.
 
 Reading Data into Tables
 ------------------------
 
-To read data from a SQLite database, we first have to provide Bro with the information, how
+To read data from a SQLite database, we first have to provide Bro with
-the resulting data will be structured. For this example, we expect that we have a SQLite database,
+the information, how the resulting data will be structured. For this
-which contains host IP addresses and the user accounts that are allowed to log into a specific
+example, we expect that we have a SQLite database, which contains
-machine.
+host IP addresses and the user accounts that are allowed to log into
+a specific machine.
 
 The SQLite commands to create the schema are as follows::
 
@@ -107,8 +112,8 @@ The SQLite commands to create the schema are as follows::
     insert into machines_to_users values ('192.168.17.2', 'bernhard');
     insert into machines_to_users values ('192.168.17.3', 'seth,matthias');
 
-After creating a file called ``hosts.sqlite`` with this content, we can read the resulting table
+After creating a file called ``hosts.sqlite`` with this content, we can
-into Bro:
+read the resulting table into Bro:
 
 .. btest-include:: ${DOC_ROOT}/frameworks/sqlite-read-table.bro
 
@@ -117,22 +122,25 @@ into Bro:
     # Make sure this parses correctly at least.
     @TEST-EXEC: bro ${DOC_ROOT}/frameworks/sqlite-read-table.bro
 
-Afterwards, that table can be used to check logins into hosts against the available
+Afterwards, that table can be used to check logins into hosts against
-userlist.
+the available userlist.
 
 Turning Data into Events
 ------------------------
 
-The second mode is to use the SQLite reader to output the input data as events. Typically there
+The second mode is to use the SQLite reader to output the input data as events.
-are two reasons to do this. First, when the structure of the input data is too complicated
+Typically there are two reasons to do this. First, when the structure of
-for a direct table import. In this case, the data can be read into an event which can then
+the input data is too complicated for a direct table import. In this case,
-create the necessary data structures in Bro in scriptland.
+the data can be read into an event which can then create the necessary
+data structures in Bro in scriptland.
 
-The second reason is, that the dataset is too big to hold it in memory. In this case, the checks
+The second reason is, that the dataset is too big to hold it in memory. In
-can be performed on-demand, when Bro encounters a situation where it needs additional information.
+this case, the checks can be performed on-demand, when Bro encounters a
+situation where it needs additional information.
 
-An example for this would be an internal huge database with malware hashes. Live database queries
+An example for this would be an internal huge database with malware
-could be used to check the sporadically happening downloads against the database.
+hashes. Live database queries could be used to check the sporadically
+happening downloads against the database.
 
 The SQLite commands to create the schema are as follows::
 
@@ -151,9 +159,10 @@ The SQLite commands to create the schema are as follows::
     insert into malware_hashes values ('73f45106968ff8dc51fba105fa91306af1ff6666', 'ftp-trace');
 
 
-The following code uses the file-analysis framework to get the sha1 hashes of files that are
+The following code uses the file-analysis framework to get the sha1 hashes
-transmitted over the network. For each hash, a SQL-query is run against SQLite. If the query
+of files that are transmitted over the network. For each hash, a SQL-query
-returns with a result, we had a hit against our malware-database and output the matching hash.
+is run against SQLite. If the query returns with a result, we had a hit
+against our malware-database and output the matching hash.
 
 .. btest-include:: ${DOC_ROOT}/frameworks/sqlite-read-events.bro
 
@@ -162,5 +171,5 @@ returns with a result, we had a hit against our malware-database and output the
     # Make sure this parses correctly at least.
     @TEST-EXEC: bro ${DOC_ROOT}/frameworks/sqlite-read-events.bro
 
-If you run this script against the trace in ``testing/btest/Traces/ftp/ipv4.trace``, you
+If you run this script against the trace in
-will get one hit.
+``testing/btest/Traces/ftp/ipv4.trace``, you will get one hit.

doc/frameworks/logging.rst

@@ -19,195 +19,144 @@ Terminology
 
 Bro's logging interface is built around three main abstractions:
 
-    Log streams
+    Streams
-        A stream corresponds to a single log. It defines the set of
+        A log stream corresponds to a single log. It defines the set of
-        fields that a log consists of with their names and fields.
+        fields that a log consists of with their names and types.
-        Examples are the ``conn`` for recording connection summaries,
+        Examples are the ``conn`` stream for recording connection summaries,
         and the ``http`` stream for recording HTTP activity.
 
     Filters
         Each stream has a set of filters attached to it that determine
         what information gets written out. By default, each stream has
-        one default filter that just logs everything directly to disk
+        one default filter that just logs everything directly to disk.
-        with an automatically generated file name. However, further
+        However, additional filters can be added to record only a subset
-        filters can be added to record only a subset, split a stream
+        of the log records, write to different outputs, or set a custom
-        into different outputs, or to even duplicate the log to
+        rotation interval.  If all filters are removed from a stream,
-        multiple outputs. If all filters are removed from a stream,
+        then output is disabled for that stream.
-        all output is disabled.
 
     Writers
-        A writer defines the actual output format for the information
+        Each filter has a writer.  A writer defines the actual output
-        being logged. At the moment, Bro comes with only one type of
+        format for the information being logged. The default writer is
-        writer, which produces tab separated ASCII files. In the
+        the ASCII writer, which produces tab-separated ASCII files. Other
-        future we will add further writers, like for binary output and
+        writers are available, like for binary output or direct logging
-        direct logging into a database.
+        into a database.
 
-Basics
+There are several different ways to customize Bro's logging: you can create
-======
+a new log stream, you can extend an existing log with new fields, you
+can apply filters to an existing log stream, or you can customize the output
+format by setting log writer options.  All of these approaches are
+described in this document.
 
-The data fields that a stream records are defined by a record type
+Streams
-specified when it is created. Let's look at the script generating Bro's
+=======
-connection summaries as an example,
-:doc:`/scripts/base/protocols/conn/main.bro`. It defines a record
-:bro:type:`Conn::Info` that lists all the fields that go into
-``conn.log``, each marked with a ``&log`` attribute indicating that it
-is part of the information written out. To write a log record, the
-script then passes an instance of :bro:type:`Conn::Info` to the logging
-framework's :bro:id:`Log::write` function.
 
-By default, each stream automatically gets a filter named ``default``
+In order to log data to a new log stream, all of the following needs to be
-that generates the normal output by recording all record fields into a
+done:
-single output file.
 
-In the following, we summarize ways in which the logging can be
+- A :bro:type:`record` type must be defined which consists of all the
-customized. We continue using the connection summaries as our example
+  fields that will be logged (by convention, the name of this record type is
-to work with.
+  usually "Info").
+- A log stream ID (an :bro:type:`enum` with type name "Log::ID") must be
+  defined that uniquely identifies the new log stream.
+- A log stream must be created using the :bro:id:`Log::create_stream` function.
+- When the data to be logged becomes available, the :bro:id:`Log::write`
+  function must be called.
 
-Filtering
+In the following example, we create a new module "Foo" which creates
----------
+a new log stream.
-
-To create a new output file for an existing stream, you can add a
-new filter. A filter can, e.g., restrict the set of fields being
-logged:
 
 .. code:: bro
 
-    event bro_init()
+    module Foo;
-        {
+
-        # Add a new filter to the Conn::LOG stream that logs only
+    export {
-        # timestamp and originator address.
+        # Create an ID for our new stream. By convention, this is
-        local filter: Log::Filter = [$name="orig-only", $path="origs", $include=set("ts", "id.orig_h")];
+        # called "LOG".
-        Log::add_filter(Conn::LOG, filter);
+        redef enum Log::ID += { LOG };
+
+        # Define the record type that will contain the data to log.
+        type Info: record {
+            ts: time        &log;
+            id: conn_id     &log;
+            service: string &log &optional;
+            missed_bytes: count &log &default=0;
+        };
     }
 
-Note the fields that are set for the filter:
+    # Optionally, we can add a new field to the connection record so that
+    # the data we are logging (our "Info" record) will be easily
+    # accessible in a variety of event handlers.
+    redef record connection += {
+        # By convention, the name of this new field is the lowercase name
+        # of the module.
+        foo: Info &optional;
+    };
 
-    ``name``
+    # This event is handled at a priority higher than zero so that if
-        A mandatory name for the filter that can later be used
+    # users modify this stream in another script, they can do so at the
-        to manipulate it further.
+    # default priority of zero.
+    event bro_init() &priority=5
+        {
+        # Create the stream. This adds a default filter automatically.
+        Log::create_stream(Foo::LOG, [$columns=Info, $path="foo"]);
+        }
 
-    ``path``
+In the definition of the "Info" record above, notice that each field has the
-        The filename for the output file, without any extension (which
+:bro:attr:`&log` attribute.  Without this attribute, a field will not appear in
-        may be automatically added by the writer). Default path values
+the log output. Also notice one field has the :bro:attr:`&optional` attribute.
-        are generated by taking the stream's ID and munging it slightly.
+This indicates that the field might not be assigned any value before the
-        :bro:enum:`Conn::LOG` is converted into ``conn``,
+log record is written.  Finally, a field with the :bro:attr:`&default`
-        :bro:enum:`PacketFilter::LOG` is converted into
+attribute has a default value assigned to it automatically.
-        ``packet_filter``, and :bro:enum:`Known::CERTS_LOG` is
-        converted into ``known_certs``.
 
-    ``include``
+At this point, the only thing missing is a call to the :bro:id:`Log::write`
-        A set limiting the fields to the ones given. The names
+function to send data to the logging framework.  The actual event handler
-        correspond to those in the :bro:type:`Conn::Info` record, with
+where this should take place will depend on where your data becomes available.
-        sub-records unrolled by concatenating fields (separated with
+In this example, the :bro:id:`connection_established` event provides our data,
-        dots).
+and we also store a copy of the data being logged into the
-
+:bro:type:`connection` record:
-Using the code above, you will now get a new log file ``origs.log``
-that looks like this::
-
-    #separator \x09
-    #path   origs
-    #fields ts      id.orig_h
-    #types  time    addr
-    1128727430.350788       141.42.64.125
-    1128727435.450898       141.42.64.125
-
-If you want to make this the only log file for the stream, you can
-remove the default filter (which, conveniently, has the name
-``default``):
 
 .. code:: bro
 
-    event bro_init()
+    event connection_established(c: connection)
         {
-        # Remove the filter called "default".
+        local rec: Foo::Info = [$ts=network_time(), $id=c$id];
-        Log::remove_filter(Conn::LOG, "default");
+
+        # Store a copy of the data in the connection record so other
+        # event handlers can access it.
+        c$foo = rec;
+
+        Log::write(Foo::LOG, rec);
         }
 
-An alternate approach to "turning off" a log is to completely disable
+If you run Bro with this script, a new log file ``foo.log`` will be created.
-the stream:
+Although we only specified four fields in the "Info" record above, the
+log output will actually contain seven fields because one of the fields
+(the one named "id") is itself a record type.  Since a :bro:type:`conn_id`
+record has four fields, then each of these fields is a separate column in
+the log output.  Note that the way that such fields are named in the log
+output differs slightly from the way we would refer to the same field
+in a Bro script (each dollar sign is replaced with a period).  For example,
+to access the first field of a ``conn_id`` in a Bro script we would use
+the notation ``id$orig_h``, but that field is named ``id.orig_h``
+in the log output.
 
-.. code:: bro
+When you are developing scripts that add data to the :bro:type:`connection`
+record, care must be given to when and how long data is stored.
+Normally data saved to the connection record will remain there for the
+duration of the connection and from a practical perspective it's not
+uncommon to need to delete that data before the end of the connection.
 
-    event bro_init()
-        {
-        Log::disable_stream(Conn::LOG);
-        }
 
-If you want to skip only some fields but keep the rest, there is a
+Add Fields to a Log
-corresponding ``exclude`` filter attribute that you can use instead of
+-------------------
-``include`` to list only the ones you are not interested in.
 
-A filter can also determine output paths *dynamically* based on the
+You can add additional fields to a log by extending the record
-record being logged. That allows, e.g., to record local and remote
+type that defines its content, and setting a value for the new fields
-connections into separate files. To do this, you define a function
+before each log record is written.
-that returns the desired path:
 
-.. code:: bro
+Let's say we want to add a boolean field ``is_private`` to
-
+:bro:type:`Conn::Info` that indicates whether the originator IP address
-    function split_log(id: Log::ID, path: string, rec: Conn::Info) : string
+is part of the :rfc:`1918` space:
-        {
-        # Return "conn-local" if originator is a local IP, otherwise "conn-remote".
-        local lr = Site::is_local_addr(rec$id$orig_h) ? "local" : "remote";
-        return fmt("%s-%s", path, lr);
-        }
-
-    event bro_init()
-        {
-        local filter: Log::Filter = [$name="conn-split", $path_func=split_log, $include=set("ts", "id.orig_h")];
-        Log::add_filter(Conn::LOG, filter);
-        }
-
-Running this will now produce two files, ``local.log`` and
-``remote.log``, with the corresponding entries. One could extend this
-further for example to log information by subnets or even by IP
-address. Be careful, however, as it is easy to create many files very
-quickly ...
-
-.. sidebar:: A More Generic Path Function
-
-    The ``split_log`` method has one draw-back: it can be used
-    only with the :bro:enum:`Conn::LOG` stream as the record type is hardcoded
-    into its argument list. However, Bro allows to do a more generic
-    variant:
-
-    .. code:: bro
-
-        function split_log(id: Log::ID, path: string, rec: record { id: conn_id; } ) : string
-            {
-            return Site::is_local_addr(rec$id$orig_h) ? "local" : "remote";
-            }
-
-    This function can be used with all log streams that have records
-    containing an ``id: conn_id`` field.
-
-While so far we have seen how to customize the columns being logged,
-you can also control which records are written out by providing a
-predicate that will be called for each log record:
-
-.. code:: bro
-
-    function http_only(rec: Conn::Info) : bool
-        {
-        # Record only connections with successfully analyzed HTTP traffic
-        return rec$service == "http";
-        }
-
-    event bro_init()
-        {
-        local filter: Log::Filter = [$name="http-only", $path="conn-http", $pred=http_only];
-        Log::add_filter(Conn::LOG, filter);
-        }
-
-This will result in a log file ``conn-http.log`` that contains only
-traffic detected and analyzed as HTTP traffic.
-
-Extending
----------
-
-You can add further fields to a log stream by extending the record
-type that defines its content. Let's say we want to add a boolean
-field ``is_private`` to :bro:type:`Conn::Info` that indicates whether the
-originator IP address is part of the :rfc:`1918` space:
 
 .. code:: bro
 
@@ -218,9 +167,21 @@ originator IP address is part of the :rfc:`1918` space:
         is_private: bool &default=F &log;
     };
 
+As this example shows, when extending a log stream's "Info" record, each
+new field must always be declared either with a ``&default`` value or
+as ``&optional``.  Furthermore, you need to add the ``&log`` attribute
+or otherwise the field won't appear in the log file.
 
-Now we need to set the field. A connection's summary is generated at
+Now we need to set the field.  Although the details vary depending on which
-the time its state is removed from memory. We can add another handler
+log is being extended, in general it is important to choose a suitable event
+in which to set the additional fields because we need to make sure that
+the fields are set before the log record is written.  Sometimes the right
+choice is the same event which writes the log record, but at a higher
+priority (in order to ensure that the event handler that sets the additional
+fields is executed before the event handler that writes the log record).
+
+In this example, since a connection's summary is generated at
+the time its state is removed from memory, we can add another handler
 at that time that sets our field correctly:
 
 .. code:: bro
@@ -232,31 +193,58 @@ at that time that sets our field correctly:
         }
 
 Now ``conn.log`` will show a new field ``is_private`` of type
-``bool``.
+``bool``.  If you look at the Bro script which defines the connection
+log stream :doc:`/scripts/base/protocols/conn/main.bro`, you will see
+that ``Log::write`` gets called in an event handler for the
+same event as used in this example to set the additional fields, but at a
+lower priority than the one used in this example (i.e., the log record gets
+written after we assign the ``is_private`` field).
 
-Notes:
+For extending logs this way, one needs a bit of knowledge about how
+the script that creates the log stream is organizing its state
+keeping. Most of the standard Bro scripts attach their log state to
+the :bro:type:`connection` record where it can then be accessed, just
+like ``c$conn`` above. For example, the HTTP analysis adds a field
+``http`` of type :bro:type:`HTTP::Info` to the :bro:type:`connection`
+record.
 
-- For extending logs this way, one needs a bit of knowledge about how
-  the script that creates the log stream is organizing its state
-  keeping. Most of the standard Bro scripts attach their log state to
-  the :bro:type:`connection` record where it can then be accessed, just
-  as the ``c$conn`` above. For example, the HTTP analysis adds a field
-  ``http`` of type :bro:type:`HTTP::Info` to the :bro:type:`connection`
-  record. See the script reference for more information.
 
-- When extending records as shown above, the new fields must always be
+Define a Logging Event
-  declared either with a ``&default`` value or as ``&optional``.
+----------------------
-  Furthermore, you need to add the ``&log`` attribute or otherwise the
-  field won't appear in the output.
-
-Hooking into the Logging
-------------------------
 
 Sometimes it is helpful to do additional analysis of the information
 being logged. For these cases, a stream can specify an event that will
-be generated every time a log record is written to it. All of Bro's
+be generated every time a log record is written to it.  To do this, we
-default log streams define such an event. For example, the connection
+need to modify the example module shown above to look something like this:
-log stream raises the event :bro:id:`Conn::log_conn`. You
+
+.. code:: bro
+
+    module Foo;
+
+    export {
+        redef enum Log::ID += { LOG };
+
+        type Info: record {
+            ts: time     &log;
+            id: conn_id  &log;
+            service: string &log &optional;
+            missed_bytes: count &log &default=0;
+        };
+
+        # Define a logging event. By convention, this is called
+        # "log_<stream>".
+        global log_foo: event(rec: Info);
+    }
+
+    event bro_init() &priority=5
+        {
+        # Specify the "log_foo" event here in order for Bro to raise it.
+        Log::create_stream(Foo::LOG, [$columns=Info, $ev=log_foo,
+                           $path="foo"]);
+        }
+
+All of Bro's default log streams define such an event. For example, the
+connection log stream raises the event :bro:id:`Conn::log_conn`. You
 could use that for example for flagging when a connection to a
 specific destination exceeds a certain duration:
 
@@ -270,7 +258,7 @@ specific destination exceeds a certain duration:
 
     event Conn::log_conn(rec: Conn::Info)
         {
-        if ( rec$duration > 5mins )
+        if ( rec?$duration && rec$duration > 5mins )
             NOTICE([$note=Long_Conn_Found,
                     $msg=fmt("unusually long conn to %s", rec$id$resp_h),
                     $id=rec$id]);
@@ -281,15 +269,196 @@ externally with Perl scripts. Much of what such an external script
 would do later offline, one may instead do directly inside of Bro in
 real-time.
 
-Rotation
+Disable a Stream
---------
+----------------
 
-By default, no log rotation occurs, but it's globally controllable for all
+One way to "turn off" a log is to completely disable the stream.  For
-filters by redefining the :bro:id:`Log::default_rotation_interval` option:
+example, the following example will prevent the conn.log from being written:
 
 .. code:: bro
 
-    redef Log::default_rotation_interval = 1 hr;
+    event bro_init()
+        {
+        Log::disable_stream(Conn::LOG);
+        }
+
+Note that this must run after the stream is created, so the priority
+of this event handler must be lower than the priority of the event handler
+where the stream was created.
+
+
+Filters
+=======
+
+A stream has one or more filters attached to it (a stream without any filters
+will not produce any log output).  When a stream is created, it automatically
+gets a default filter attached to it.  This default filter can be removed
+or replaced, or other filters can be added to the stream.  This is accomplished
+by using either the :bro:id:`Log::add_filter` or :bro:id:`Log::remove_filter`
+function.  This section shows how to use filters to do such tasks as
+rename a log file, split the output into multiple files, control which
+records are written, and set a custom rotation interval.
+
+Rename Log File
+---------------
+
+Normally, the log filename for a given log stream is determined when the
+stream is created, unless you explicitly specify a different one by adding
+a filter.
+
+The easiest way to change a log filename is to simply replace the
+default log filter with a new filter that specifies a value for the "path"
+field.  In this example, "conn.log" will be changed to "myconn.log":
+
+.. code:: bro
+
+    event bro_init()
+        {
+        # Replace default filter for the Conn::LOG stream in order to
+        # change the log filename.
+
+        local f = Log::get_filter(Conn::LOG, "default");
+        f$path = "myconn";
+        Log::add_filter(Conn::LOG, f);
+        }
+
+Keep in mind that the "path" field of a log filter never contains the
+filename extension.  The extension will be determined later by the log writer.
+
+Add a New Log File
+------------------
+
+Normally, a log stream writes to only one log file.  However, you can
+add filters so that the stream writes to multiple files.  This is useful
+if you want to restrict the set of fields being logged to the new file.
+
+In this example, a new filter is added to the Conn::LOG stream that writes
+two fields to a new log file:
+
+.. code:: bro
+
+    event bro_init()
+        {
+        # Add a new filter to the Conn::LOG stream that logs only
+        # timestamp and originator address.
+
+        local filter: Log::Filter = [$name="orig-only", $path="origs",
+                                     $include=set("ts", "id.orig_h")];
+        Log::add_filter(Conn::LOG, filter);
+        }
+
+
+Notice how the "include" filter attribute specifies a set that limits the
+fields to the ones given. The names correspond to those in the
+:bro:type:`Conn::Info` record (however, because the "id" field is itself a
+record, we can specify an individual field of "id" by the dot notation
+shown in the example).
+
+Using the code above, in addition to the regular ``conn.log``, you will
+now also get a new log file ``origs.log`` that looks like the regular
+``conn.log``, but will have only the fields specified in the "include"
+filter attribute.
+
+If you want to skip only some fields but keep the rest, there is a
+corresponding ``exclude`` filter attribute that you can use instead of
+``include`` to list only the ones you are not interested in.
+
+If you want to make this the only log file for the stream, you can
+remove the default filter:
+
+.. code:: bro
+
+    event bro_init()
+        {
+        # Remove the filter called "default".
+        Log::remove_filter(Conn::LOG, "default");
+        }
+
+Determine Log Path Dynamically
+------------------------------
+
+Instead of using the "path" filter attribute, a filter can determine
+output paths *dynamically* based on the record being logged. That
+allows, e.g., to record local and remote connections into separate
+files. To do this, you define a function that returns the desired path,
+and use the "path_func" filter attribute:
+
+.. code:: bro
+
+    # Note: if using BroControl then you don't need to redef local_nets.
+    redef Site::local_nets = { 192.168.0.0/16 };
+
+    function myfunc(id: Log::ID, path: string, rec: Conn::Info) : string
+        {
+        # Return "conn-local" if originator is a local IP, otherwise
+        # return "conn-remote".
+        local r = Site::is_local_addr(rec$id$orig_h) ? "local" : "remote";
+        return fmt("%s-%s", path, r);
+        }
+
+    event bro_init()
+        {
+        local filter: Log::Filter = [$name="conn-split",
+                 $path_func=myfunc, $include=set("ts", "id.orig_h")];
+        Log::add_filter(Conn::LOG, filter);
+        }
+
+Running this will now produce two new files, ``conn-local.log`` and
+``conn-remote.log``, with the corresponding entries (for this example to work,
+the ``Site::local_nets`` must specify your local network). One could extend
+this further for example to log information by subnets or even by IP
+address. Be careful, however, as it is easy to create many files very
+quickly.
+
+The ``myfunc`` function has one drawback: it can be used
+only with the :bro:enum:`Conn::LOG` stream as the record type is hardcoded
+into its argument list. However, Bro allows to do a more generic
+variant:
+
+.. code:: bro
+
+    function myfunc(id: Log::ID, path: string,
+                    rec: record { id: conn_id; } ) : string
+        {
+        local r = Site::is_local_addr(rec$id$orig_h) ? "local" : "remote";
+        return fmt("%s-%s", path, r);
+        }
+
+This function can be used with all log streams that have records
+containing an ``id: conn_id`` field.
+
+Filter Log Records
+------------------
+
+We have seen how to customize the columns being logged, but
+you can also control which records are written out by providing a
+predicate that will be called for each log record:
+
+.. code:: bro
+
+    function http_only(rec: Conn::Info) : bool
+        {
+        # Record only connections with successfully analyzed HTTP traffic
+        return rec?$service && rec$service == "http";
+        }
+
+    event bro_init()
+        {
+        local filter: Log::Filter = [$name="http-only", $path="conn-http",
+                                     $pred=http_only];
+        Log::add_filter(Conn::LOG, filter);
+        }
+
+This will result in a new log file ``conn-http.log`` that contains only
+the log records from ``conn.log`` that are analyzed as HTTP traffic.
+
+Rotation
+--------
+
+The log rotation interval is globally controllable for all
+filters by redefining the :bro:id:`Log::default_rotation_interval` option
+(note that when using BroControl, this option is set automatically via
+the BroControl configuration).
 
 Or specifically for certain :bro:type:`Log::Filter` instances by setting
 their ``interv`` field.  Here's an example of changing just the
@@ -301,90 +470,73 @@ their ``interv`` field.  Here's an example of changing just the
         {
         local f = Log::get_filter(Conn::LOG, "default");
         f$interv = 1 min;
-        Log::remove_filter(Conn::LOG, "default");
         Log::add_filter(Conn::LOG, f);
         }
 
-ASCII Writer Configuration
+Writers
---------------------------
+=======
 
-The ASCII writer has a number of options for customizing the format of
+Each filter has a writer.  If you do not specify a writer when adding a
-its output, see :doc:`/scripts/base/frameworks/logging/writers/ascii.bro`.
+filter to a stream, then the ASCII writer is the default.
 
-Adding Streams
+There are two ways to specify a non-default writer.  To change the default
-==============
+writer for all log filters, just redefine the :bro:id:`Log::default_writer`
+option.  Alternatively, you can specify the writer to use on a per-filter
+basis by setting a value for the filter's "writer" field.  Consult the
+documentation of the writer to use to see if there are other options that are
+needed.
 
-It's easy to create a new log stream for custom scripts. Here's an
+ASCII Writer
-example for the ``Foo`` module:
+------------
+
+By default, the ASCII writer outputs log files that begin with several
+lines of metadata, followed by the actual log output.  The metadata
+describes the format of the log file, the "path" of the log (i.e., the log
+filename without file extension), and also specifies the time that the log
+was created and the time when Bro finished writing to it.
+The ASCII writer has a number of options for customizing the format of its
+output, see :doc:`/scripts/base/frameworks/logging/writers/ascii.bro`.
+If you change the output format options, then be careful to check whether
+your postprocessing scripts can still recognize your log files.
+
+Some writer options are global (i.e., they affect all log filters using
+that log writer).  For example, to change the output format of all ASCII
+logs to JSON format:
 
 .. code:: bro
 
-    module Foo;
+    redef LogAscii::use_json = T;
 
-    export {
+Some writer options are filter-specific (i.e., they affect only the filters
-        # Create an ID for our new stream. By convention, this is
+that explicitly specify the option).  For example, to change the output
-        # called "LOG".
+format of the ``conn.log`` only:
-        redef enum Log::ID += { LOG };
 
-        # Define the fields. By convention, the type is called "Info".
+.. code:: bro
-        type Info: record {
-            ts: time     &log;
-            id: conn_id  &log;
-        };
 
-        # Define a hook event. By convention, this is called
+    event bro_init()
-        # "log_<stream>".
-        global log_foo: event(rec: Info);
-
-    }
-
-    # This event should be handled at a higher priority so that when
-    # users modify your stream later and they do it at priority 0,
-    # their code runs after this.
-    event bro_init() &priority=5
         {
-        # Create the stream. This also adds a default filter automatically.
+        local f = Log::get_filter(Conn::LOG, "default");
-        Log::create_stream(Foo::LOG, [$columns=Info, $ev=log_foo]);
+        # Use tab-separated-value mode
+        f$config = table(["tsv"] = "T");
+        Log::add_filter(Conn::LOG, f);
         }
 
-You can also add the state to the :bro:type:`connection` record to make
-it easily accessible across event handlers:
-
-.. code:: bro
-
-    redef record connection += {
-        foo: Info &optional;
-        }
-
-Now you can use the :bro:id:`Log::write` method to output log records and
-save the logged ``Foo::Info`` record into the connection record:
-
-.. code:: bro
-
-    event connection_established(c: connection)
-        {
-        local rec: Foo::Info = [$ts=network_time(), $id=c$id];
-        c$foo = rec;
-        Log::write(Foo::LOG, rec);
-        }
-
-See the existing scripts for how to work with such a new connection
-field. A simple example is :doc:`/scripts/base/protocols/syslog/main.bro`.
-
-When you are developing scripts that add data to the :bro:type:`connection`
-record, care must be given to when and how long data is stored.
-Normally data saved to the connection record will remain there for the
-duration of the connection and from a practical perspective it's not
-uncommon to need to delete that data before the end of the connection.
 
 Other Writers
 -------------
 
-Bro supports the following built-in output formats other than ASCII:
+Bro supports the following additional built-in output formats:
 
 .. toctree::
    :maxdepth: 1
 
    logging-input-sqlite
 
-Further formats are available as external plugins.
+Additional writers are available as external plugins:
+
+.. toctree::
+   :maxdepth: 1
+
+   ../components/bro-plugins/dataseries/README
+   ../components/bro-plugins/elasticsearch/README
+

doc/frameworks/notice.rst

@@ -88,15 +88,15 @@ directly make modifications to the :bro:see:`Notice::Info` record
 given as the argument to the hook.
 
 Here's a simple example which tells Bro to send an email for all notices of
-type :bro:see:`SSH::Password_Guessing` if the server is 10.0.0.1:
+type :bro:see:`SSH::Password_Guessing` if the guesser attempted to log in to
+the server at 192.168.56.103:
 
-.. code:: bro
+.. btest-include:: ${DOC_ROOT}/frameworks/notice_ssh_guesser.bro
 
-    hook Notice::policy(n: Notice::Info)
+.. btest:: notice_ssh_guesser.bro
-      {
+
-      if ( n$note == SSH::Password_Guessing && n$id$resp_h == 10.0.0.1 )
+    @TEST-EXEC: btest-rst-cmd bro -C -r ${TRACES}/ssh/sshguess.pcap ${DOC_ROOT}/frameworks/notice_ssh_guesser.bro
-        add n$actions[Notice::ACTION_EMAIL];
+    @TEST-EXEC: btest-rst-cmd cat notice.log
-      }
 
 .. note::
 
@@ -112,8 +112,7 @@ a hook body to run before default hook bodies might look like this:
 
     hook Notice::policy(n: Notice::Info) &priority=5
         {
-      if ( n$note == SSH::Password_Guessing && n$id$resp_h == 10.0.0.1 )
+        # Insert your code here.
-        add n$actions[Notice::ACTION_EMAIL];
         }
 
 Hooks can also abort later hook bodies with the ``break`` keyword. This
@@ -271,7 +270,7 @@ script that is generating the notice has indicated to the notice framework how
 to identify notices that are intrinsically the same. Identification of these
 "intrinsically duplicate" notices is implemented with an optional field in
 :bro:see:`Notice::Info` records named ``$identifier`` which is a simple string.
-If the ``$identifier`` and ``$type`` fields are the same for two notices, the
+If the ``$identifier`` and ``$note`` fields are the same for two notices, the
 notice framework actually considers them to be the same thing and can use that
 information to suppress duplicates for a configurable period of time.
 

doc/frameworks/notice_ssh_guesser.bro

@@ -0,0 +1,10 @@
+
+@load protocols/ssh/detect-bruteforcing
+
+redef SSH::password_guesses_limit=10;
+
+hook Notice::policy(n: Notice::Info)
+	{
+	if ( n$note == SSH::Password_Guessing && /192\.168\.56\.103/ in n$sub )
+		add n$actions[Notice::ACTION_EMAIL];
+	}

doc/httpmonitor/file_extraction.bro

@@ -7,18 +7,18 @@ global mime_to_ext: table[string] of string = {
 	["text/html"] = "html",
 };
 
-event file_new(f: fa_file)
+event file_sniff(f: fa_file, meta: fa_metadata)
 	{
 	if ( f$source != "HTTP" )
 		return;
 
-	if ( ! f?$mime_type )
+	if ( ! meta?$mime_type )
 		return;
 
-	if ( f$mime_type !in mime_to_ext )
+	if ( meta$mime_type !in mime_to_ext )
 		return;
 
-	local fname = fmt("%s-%s.%s", f$source, f$id, mime_to_ext[f$mime_type]);
+	local fname = fmt("%s-%s.%s", f$source, f$id, mime_to_ext[meta$mime_type]);
 	print fmt("Extracting file %s", fname);
 	Files::add_analyzer(f, Files::ANALYZER_EXTRACT, [$extract_filename=fname]);
 	}

doc/index.rst

@@ -45,7 +45,13 @@ Reference Section
    script-reference/index.rst
    components/index.rst
 
-..
+Development
+===========
+
+.. toctree::
+   :maxdepth: 2
+
+   devel/plugins.rst
 
 * :ref:`General Index <genindex>`
 * :ref:`search`

doc/install/guidelines.rst

@@ -8,10 +8,12 @@ How to Upgrade
 If you're doing an upgrade install (rather than a fresh install),
 there's two suggested approaches: either install Bro using the same
 installation prefix directory as before, or pick a new prefix and copy
-local customizations over.  Regardless of which approach you choose,
+local customizations over.
-if you are using BroControl, then after upgrading Bro you will need to
+
-run "broctl check" (to verify that your new configuration is OK)
+Regardless of which approach you choose, if you are using BroControl, then
-and "broctl install" to complete the upgrade process.
+before doing the upgrade you should stop all running Bro processes with the
+"broctl stop" command.  After the upgrade is complete then you will need
+to run "broctl deploy".
 
 In the following we summarize general guidelines for upgrading, see
 the :ref:`release-notes` for version-specific information.
@@ -44,4 +46,4 @@ where Bro was originally installed).  Review the files for differences
 before copying and make adjustments as necessary (use the new version for
 differences that aren't a result of a local change).  Of particular note,
 the copied version of ``$prefix/etc/broctl.cfg`` is likely to need changes
-to the ``SpoolDir`` and ``LogDir`` settings.
+to any settings that specify a pathname.

doc/install/install.rst

@@ -4,7 +4,7 @@
 .. _MacPorts: http://www.macports.org
 .. _Fink: http://www.finkproject.org
 .. _Homebrew: http://brew.sh
-.. _bro downloads page: http://bro.org/download/index.html
+.. _bro downloads page: https://www.bro.org/download/index.html
 
 .. _installing-bro:
 
@@ -32,22 +32,24 @@ before you begin:
     * Libz
     * Bash (for BroControl)
     * Python (for BroControl)
+    * C++ Actor Framework (CAF) version 0.14 (http://actor-framework.org)
 
 To build Bro from source, the following additional dependencies are required:
 
-    * CMake 2.6.3 or greater            (http://www.cmake.org)
+    * CMake 2.8 or greater              (http://www.cmake.org)
     * Make
-    * C/C++ compiler
+    * C/C++ compiler with C++11 support (GCC 4.8+ or Clang 3.3+)
     * SWIG                              (http://www.swig.org)
     * Bison (GNU Parser Generator)
     * Flex  (Fast Lexical Analyzer)
     * Libpcap headers                   (http://www.tcpdump.org)
     * OpenSSL headers                   (http://www.openssl.org)
     * zlib headers
-    * Perl
+    * Python
 
-To install the required dependencies, you can use (when done, make sure
+To install CAF, first download the source code of the required version from: https://github.com/actor-framework/actor-framework/releases
-that ``bash`` and ``python`` are in your ``PATH``):
+
+To install the required dependencies, you can use:
 
 * RPM/RedHat-based Linux:
 
@@ -68,19 +70,26 @@ that ``bash`` and ``python`` are in your ``PATH``):
 
   .. console::
 
-      sudo pkg_add -r bash cmake swig bison python perl
+      sudo pkg install bash cmake swig bison python py27-sqlite3
+
+  Note that in older versions of FreeBSD, you might have to use the
+  "pkg_add -r" command instead of "pkg install".
 
 * Mac OS X:
 
-  Compiling source code on Macs requires first downloading Xcode_,
+  Compiling source code on Macs requires first installing Xcode_ (in older
-  then going through its "Preferences..." -> "Downloads" menus to
+  versions of Xcode, you would then need to go through its
-  install the "Command Line Tools" component.
+  "Preferences..." -> "Downloads" menus to install the "Command Line Tools"
+  component).
 
-  OS X comes with all required dependencies except for CMake_ and SWIG_.
+  OS X comes with all required dependencies except for CMake_, SWIG_,
-  Distributions of these dependencies can likely be obtained from your
+  OpenSSL, and CAF. (OpenSSL used to be part of OS X versions 10.10
-  preferred Mac OS X package management system (e.g. MacPorts_, Fink_,
+  and older, for which it does not need to be installed manually. It
-  or Homebrew_).  Specifically for MacPorts, the ``cmake``, ``swig``,
+  was removed in OS X 10.11). Distributions of these dependencies can
-  and ``swig-python`` packages provide the required dependencies.
+  likely be obtained from your preferred Mac OS X package management
+  system (e.g. Homebrew_, MacPorts_, or Fink_). Specifically for
+  Homebrew, the ``cmake``, ``swig``, ``openssl`` and ``caf`` packages
+  provide the required dependencies.
 
 
 Optional Dependencies
@@ -93,8 +102,9 @@ build time:
     * sendmail (enables Bro and BroControl to send mail)
     * curl (used by a Bro script that implements active HTTP)
     * gperftools (tcmalloc is used to improve memory and CPU usage)
+    * jemalloc (http://www.canonware.com/jemalloc/)
+    * PF_RING (Linux only, see :doc:`Cluster Configuration <../configuration/index>`)
     * ipsumdump (for trace-summary; http://www.cs.ucla.edu/~kohler/ipsumdump)
-    * Ruby executable, library, and headers (for Broccoli Ruby bindings)
 
 LibGeoIP is probably the most interesting and can be installed
 on most platforms by following the instructions for :ref:`installing
@@ -110,40 +120,30 @@ code forms.
 
 
 Using Pre-Built Binary Release Packages
-=======================================
+---------------------------------------
 
 See the `bro downloads page`_ for currently supported/targeted
-platforms for binary releases.
+platforms for binary releases and for installation instructions.
 
-* RPM
+* Linux Packages
 
-  .. console::
+  Linux based binary installations are usually performed by adding
-
+  information about the Bro packages to the respective system packaging
-      sudo yum localinstall Bro-*.rpm
+  tool. Then the usual system utilities such as ``apt``, ``dnf``, ``yum``,
-
+  or ``zypper`` are used to perform the installation.
-* DEB
-
-  .. console::
-
-      sudo gdebi Bro-*.deb
-
-* MacOS Disk Image with Installer
-
-  Just open the ``Bro-*.dmg`` and then run the ``.pkg`` installer.
-  Everything installed by the package will go into ``/opt/bro``.
 
 The primary install prefix for binary packages is ``/opt/bro``.
-Non-MacOS packages that include BroControl also put variable/runtime
-data (e.g. Bro logs) in ``/var/opt/bro``.
 
 Installing from Source
-==========================
+----------------------
 
 Bro releases are bundled into source packages for convenience and are
-available on the `bro downloads page`_. Alternatively, the latest
+available on the `bro downloads page`_.
-Bro development version can be obtained through git repositories
+
+Alternatively, the latest Bro development version
+can be obtained through git repositories
 hosted at ``git.bro.org``.  See our `git development documentation
-<http://bro.org/development/howtos/process.html>`_ for comprehensive
+<https://www.bro.org/development/howtos/process.html>`_ for comprehensive
 information on Bro's use of git revision control, but the short story
 for downloading the full source code experience for Bro via git is:
 
@@ -164,13 +164,23 @@ run ``./configure --help``):
     make
     make install
 
+If the ``configure`` script fails, then it is most likely because it either
+couldn't find a required dependency or it couldn't find a sufficiently new
+version of a dependency.  Assuming that you already installed all required
+dependencies, then you may need to use one of the ``--with-*`` options
+that can be given to the ``configure`` script to help it locate a dependency.
+
 The default installation path is ``/usr/local/bro``, which would typically
 require root privileges when doing the ``make install``.  A different
-installation path can be chosen by specifying the ``--prefix`` option.
+installation path can be chosen by specifying the ``configure`` script
-Note that ``/usr`` and ``/opt/bro`` are the
+``--prefix`` option.  Note that ``/usr`` and ``/opt/bro`` are the
 standard prefixes for binary Bro packages to be installed, so those are
 typically not good choices unless you are creating such a package.
 
+OpenBSD users, please see our `FAQ
+<https://www.bro.org/documentation/faq.html>`_ if you are having
+problems installing Bro.
+
 Depending on the Bro package you downloaded, there may be auxiliary
 tools and libraries available in the ``aux/`` directory. Some of them
 will be automatically built and installed along with Bro. There are
@@ -179,10 +189,6 @@ turn off unwanted auxiliary projects that would otherwise be installed
 automatically.  Finally, use ``make install-aux`` to install some of
 the other programs that are in the ``aux/bro-aux`` directory.
 
-OpenBSD users, please see our `FAQ
-<//www.bro.org/documentation/faq.html>`_ if you are having
-problems installing Bro.
-
 Finally, if you want to build the Bro documentation (not required, because
 all of the documentation for the latest Bro release is available on the
 Bro web site), there are instructions in ``doc/README`` in the source
@@ -191,7 +197,7 @@ distribution.
 Configure the Run-Time Environment
 ==================================
 
-Just remember that you may need to adjust your ``PATH`` environment variable
+You may want to adjust your ``PATH`` environment variable
 according to the platform/shell/package you're using.  For example:
 
 Bourne-Shell Syntax:

doc/logs/index.rst

@@ -112,6 +112,8 @@ default, including:
 
 As you can see, some log files are specific to a particular protocol,
 while others aggregate information across different types of activity. 
+For a complete list of log files and a description of its purpose, 
+see :doc:`Log Files <../script-reference/log-files>`.
 
 .. _bro-cut:
 
@@ -250,44 +252,3 @@ protocol, it can have multiple ``GET``/``POST``/etc requests in a
 stream and Bro is able to extract and track that information for you,
 giving you an in-depth and structured view into HTTP traffic on your
 network.
-
------------------------
-Common Log Files
------------------------
-As a monitoring tool, Bro records a detailed view of the traffic inspected 
-and the events generated in a series of relevant log files. These files can 
-later be reviewed for monitoring, auditing and troubleshooting purposes.
-
-In this section we present a brief explanation of the most commonly used log 
-files generated by Bro including links to descriptions of some of the fields 
-for each log type.
-
-+-----------------+---------------------------------------+------------------------------+
-| Log File        | Description                           | Field Descriptions           |
-+=================+=======================================+==============================+
-| http.log        | Shows all HTTP requests and replies   | :bro:type:`HTTP::Info`       |
-+-----------------+---------------------------------------+------------------------------+
-| ftp.log         | Records FTP activity                  | :bro:type:`FTP::Info`        |
-+-----------------+---------------------------------------+------------------------------+
-| ssl.log         | Records SSL sessions including        | :bro:type:`SSL::Info`        |
-|                 | certificates used                     |                              |
-+-----------------+---------------------------------------+------------------------------+
-| known_certs.log | Includes SSL certificates used        | :bro:type:`Known::CertsInfo` |
-+-----------------+---------------------------------------+------------------------------+
-| smtp.log        | Summarizes SMTP traffic on a network  | :bro:type:`SMTP::Info`       |
-+-----------------+---------------------------------------+------------------------------+
-| dns.log         | Shows all DNS activity on a network   | :bro:type:`DNS::Info`        |
-+-----------------+---------------------------------------+------------------------------+
-| conn.log        | Records all connections seen by Bro   | :bro:type:`Conn::Info`       |
-+-----------------+---------------------------------------+------------------------------+
-| dpd.log         | Shows network activity on             | :bro:type:`DPD::Info`        |
-|                 | non-standard ports                    |                              |
-+-----------------+---------------------------------------+------------------------------+
-| files.log       | Records information about all files   | :bro:type:`Files::Info`      |
-|                 | transmitted over the network          |                              |
-+-----------------+---------------------------------------+------------------------------+
-| weird.log       | Records unexpected protocol-level     | :bro:type:`Weird::Info`      |
-|                 | activity                              |                              |
-+-----------------+---------------------------------------+------------------------------+
-
-

doc/mimestats/mimestats.bro

@@ -30,7 +30,7 @@ export {
 
 event bro_init() &priority=3
 	{
-	Log::create_stream(MimeMetrics::LOG, [$columns=Info]);
+	Log::create_stream(MimeMetrics::LOG, [$columns=Info, $path="mime_metrics"]);
 	local r1: SumStats::Reducer = [$stream="mime.bytes",
 	                               $apply=set(SumStats::SUM)];
 	local r2: SumStats::Reducer = [$stream="mime.hits", 

doc/quickstart/conditional-notice.bro

@@ -0,0 +1,24 @@
+@load protocols/ssl/expiring-certs
+
+const watched_servers: set[addr] = {
+	87.98.220.10,
+} &redef;
+
+# Site::local_nets usually isn't something you need to modify if
+# BroControl automatically sets it up from networks.cfg.  It's
+# shown here for completeness.
+redef Site::local_nets += {
+	87.98.0.0/16,
+};
+
+hook Notice::policy(n: Notice::Info)
+	{
+	if ( n$note != SSL::Certificate_Expired )
+		return;
+
+	if ( n$id$resp_h !in watched_servers )
+		return;
+
+	add n$actions[Notice::ACTION_EMAIL];
+	}
+

doc/quickstart/index.rst

@@ -24,9 +24,10 @@ Managing Bro with BroControl
 BroControl is an interactive shell for easily operating/managing Bro
 installations on a single system or even across multiple systems in a
 traffic-monitoring cluster.  This section explains how to use BroControl
-to manage a stand-alone Bro installation.  For instructions on how to
+to manage a stand-alone Bro installation.  For a complete reference on
-configure a Bro cluster, see the :doc:`Cluster Configuration
+BroControl, see the :doc:`BroControl <../components/broctl/README>`
-<../configuration/index>` documentation.
+documentation.  For instructions on how to configure a Bro cluster,
+see the :doc:`Cluster Configuration <../configuration/index>` documentation.
 
 A Minimal Starting Configuration
 --------------------------------
@@ -156,9 +157,11 @@ changes we want to make:
    notice that means an SSL connection was established and the server's
    certificate couldn't be validated using Bro's default trust roots, but
    we want to ignore it.
-2) ``SSH::Login`` is a notice type that is triggered when an SSH connection
+2) ``SSL::Certificate_Expired`` is a notice type that is triggered when
-   attempt looks like it may have been successful, and we want email when
+   an SSL connection was established using an expired certificate.  We
-   that happens, but only for certain servers.
+   want email when that happens, but only for certain servers on the
+   local network (Bro can also proactively monitor for certs that will
+   soon expire, but this is just for demonstration purposes).
 
 We've defined *what* we want to do, but need to know *where* to do it.
 The answer is to use a script written in the Bro programming language, so
@@ -203,7 +206,7 @@ the variable's value may not change at run-time, but whose initial value can be
 modified via the ``redef`` operator at parse-time.
 
 Let's continue on our path to modify the behavior for the two SSL
-and SSH notices.  Looking at :doc:`/scripts/base/frameworks/notice/main.bro`,
+notices.  Looking at :doc:`/scripts/base/frameworks/notice/main.bro`,
 we see that it advertises:
 
 .. code:: bro
@@ -216,7 +219,7 @@ we see that it advertises:
         const ignored_types: set[Notice::Type] = {} &redef;
     }
 
-That's exactly what we want to do for the SSL notice.  Add to ``local.bro``:
+That's exactly what we want to do for the first notice.  Add to ``local.bro``:
 
 .. code:: bro
 
@@ -248,38 +251,30 @@ is valid before installing it and then restarting the Bro instance:
    stopping bro ...
    starting bro ...
 
-Now that the SSL notice is ignored, let's look at how to send an email on
+Now that the SSL notice is ignored, let's look at how to send an email
-the SSH notice.  The notice framework has a similar option called
+on the other notice.  The notice framework has a similar option called
-``emailed_types``, but using that would generate email for all SSH servers and
+``emailed_types``, but using that would generate email for all SSL
-we only want email for logins to certain ones.  There is a ``policy`` hook
+servers with expired certificates and we only want email for connections
-that is actually what is used to implement the simple functionality of
+to certain ones.  There is a ``policy`` hook that is actually what is
-``ignored_types`` and
+used to implement the simple functionality of ``ignored_types`` and
-``emailed_types``, but it's extensible such that the condition and action taken
+``emailed_types``, but it's extensible such that the condition and
-on notices can be user-defined.
+action taken on notices can be user-defined.
 
-In ``local.bro``, let's define a new ``policy`` hook handler body
+In ``local.bro``, let's define a new ``policy`` hook handler body:
-that takes the email action for SSH logins only for a defined set of servers:
 
-.. code:: bro
+.. btest-include:: ${DOC_ROOT}/quickstart/conditional-notice.bro
 
-    const watched_servers: set[addr] = {
+.. btest:: conditional-notice
-        192.168.1.100,
-        192.168.1.101,
-        192.168.1.102,
-    } &redef;
 
-   hook Notice::policy(n: Notice::Info)
+    @TEST-EXEC: btest-rst-cmd bro -r ${TRACES}/tls/tls-expired-cert.trace ${DOC_ROOT}/quickstart/conditional-notice.bro
-       {
+    @TEST-EXEC: btest-rst-cmd cat notice.log
-       if ( n$note == SSH::SUCCESSFUL_LOGIN && n$id$resp_h in watched_servers )
-            add n$actions[Notice::ACTION_EMAIL];
-       }
 
 You'll just have to trust the syntax for now, but what we've done is
 first declare our own variable to hold a set of watched addresses,
-``watched_servers``; then added a hook handler body to the policy that will
+``watched_servers``; then added a hook handler body to the policy that
-generate an email whenever the notice type is an SSH login and the responding
+will generate an email whenever the notice type is an SSL expired
-host stored
+certificate and the responding host stored inside the ``Info`` record's
-inside the ``Info`` record's connection field is in the set of watched servers.
+connection field is in the set of watched servers.
 
 .. note:: Record field member access is done with the '$' character
    instead of a '.' as might be expected from other languages, in

doc/script-reference/attributes.rst

@@ -0,0 +1,241 @@
+Attributes
+==========
+
+The Bro scripting language supports the following attributes.
+
++-----------------------------+-----------------------------------------------+
+| Name                        | Description                                   |
++=============================+===============================================+
+| :bro:attr:`&redef`          |Redefine a global constant or extend a type.   |
++-----------------------------+-----------------------------------------------+
+| :bro:attr:`&priority`       |Specify priority for event handler or hook.    |
++-----------------------------+-----------------------------------------------+
+| :bro:attr:`&log`            |Mark a record field as to be written to a log. |
++-----------------------------+-----------------------------------------------+
+| :bro:attr:`&optional`       |Allow a record field value to be missing.      |
++-----------------------------+-----------------------------------------------+
+| :bro:attr:`&default`        |Specify a default value.                       |
++-----------------------------+-----------------------------------------------+
+| :bro:attr:`&add_func`       |Specify a function to call for each "redef +=".|
++-----------------------------+-----------------------------------------------+
+| :bro:attr:`&delete_func`    |Same as "&add_func", except for "redef -=".    |
++-----------------------------+-----------------------------------------------+
+| :bro:attr:`&expire_func`    |Specify a function to call when container      |
+|                             |element expires.                               |
++-----------------------------+-----------------------------------------------+
+| :bro:attr:`&read_expire`    |Specify a read timeout interval.               |
++-----------------------------+-----------------------------------------------+
+| :bro:attr:`&write_expire`   |Specify a write timeout interval.              |
++-----------------------------+-----------------------------------------------+
+| :bro:attr:`&create_expire`  |Specify a creation timeout interval.           |
++-----------------------------+-----------------------------------------------+
+| :bro:attr:`&synchronized`   |Synchronize a variable across nodes.           |
++-----------------------------+-----------------------------------------------+
+| :bro:attr:`&persistent`     |Make a variable persistent (written to disk).  |
++-----------------------------+-----------------------------------------------+
+| :bro:attr:`&rotate_interval`|Rotate a file after specified interval.        |
++-----------------------------+-----------------------------------------------+
+| :bro:attr:`&rotate_size`    |Rotate a file after specified file size.       |
++-----------------------------+-----------------------------------------------+
+| :bro:attr:`&encrypt`        |Encrypt a file when writing to disk.           |
++-----------------------------+-----------------------------------------------+
+| :bro:attr:`&raw_output`     |Open file in raw mode (chars. are not escaped).|
++-----------------------------+-----------------------------------------------+
+| :bro:attr:`&mergeable`      |Prefer set union for synchronized state.       |
++-----------------------------+-----------------------------------------------+
+| :bro:attr:`&error_handler`  |Used internally for reporter framework events. |
++-----------------------------+-----------------------------------------------+
+| :bro:attr:`&type_column`    |Used by input framework for "port" type.       |
++-----------------------------+-----------------------------------------------+
+| :bro:attr:`&deprecated`     |Marks an identifier as deprecated.             |
++-----------------------------+-----------------------------------------------+
+
+Here is a more detailed explanation of each attribute:
+
+.. bro:attr:: &redef
+
+    Allows use of a :bro:keyword:`redef` to redefine initial values of
+    global variables (i.e., variables declared either :bro:keyword:`global`
+    or :bro:keyword:`const`).  Example::
+
+        const clever = T &redef;
+        global cache_size = 256 &redef;
+
+    Note that a variable declared "global" can also have its value changed
+    with assignment statements (doesn't matter if it has the "&redef"
+    attribute or not).
+
+.. bro:attr:: &priority
+
+    Specifies the execution priority (as a signed integer) of a hook or
+    event handler. Higher values are executed before lower ones. The
+    default value is 0.  Example::
+
+        event bro_init() &priority=10
+        {
+            print "high priority";
+        }
+
+.. bro:attr:: &log
+
+    Writes a :bro:type:`record` field to the associated log stream.
+
+.. bro:attr:: &optional
+
+    Allows a record field value to be missing (i.e., neither initialized nor
+    ever assigned a value).
+
+    In this example, the record could be instantiated with either
+    "myrec($a=127.0.0.1)" or "myrec($a=127.0.0.1, $b=80/tcp)"::
+
+        type myrec: record { a: addr; b: port &optional; };
+
+    The ``?$`` operator can be used to check if a record field has a value or
+    not (it returns a ``bool`` value of ``T`` if the field has a value,
+    and ``F`` if not).
+
+.. bro:attr:: &default
+
+    Specifies a default value for a record field, container element, or a
+    function/hook/event parameter.
+
+    In this example, the record could be instantiated with either
+    "myrec($a=5, $c=3.14)" or "myrec($a=5, $b=53/udp, $c=3.14)"::
+
+        type myrec: record { a: count; b: port &default=80/tcp; c: double; };
+
+    In this example, the table will return the string ``"foo"`` for any
+    attempted access to a non-existing index::
+
+        global mytable: table[count] of string &default="foo";
+
+    When used with function/hook/event parameters, all of the parameters
+    with the "&default" attribute must come after all other parameters.
+    For example, the following function could be called either as "myfunc(5)"
+    or as "myfunc(5, 53/udp)"::
+
+        function myfunc(a: count, b: port &default=80/tcp)
+        {
+            print a, b;
+        }
+
+.. bro:attr:: &add_func
+
+    Can be applied to an identifier with &redef to specify a function to
+    be called any time a "redef <id> += ..." declaration is parsed.  The
+    function takes two arguments of the same type as the identifier, the first
+    being the old value of the variable and the second being the new
+    value given after the "+=" operator in the "redef" declaration.  The
+    return value of the function will be the actual new value of the
+    variable after the "redef" declaration is parsed.
+
+.. bro:attr:: &delete_func
+
+    Same as :bro:attr:`&add_func`, except for :bro:keyword:`redef` declarations
+    that use the "-=" operator.
+
+.. bro:attr:: &expire_func
+
+    Called right before a container element expires.  The function's
+    first parameter is of the same type of the container and the second
+    parameter the same type of the container's index.  The return
+    value is an :bro:type:`interval` indicating the amount of additional
+    time to wait before expiring the container element at the given
+    index (which will trigger another execution of this function).
+
+.. bro:attr:: &read_expire
+
+    Specifies a read expiration timeout for container elements. That is,
+    the element expires after the given amount of time since the last
+    time it has been read. Note that a write also counts as a read.
+
+.. bro:attr:: &write_expire
+
+    Specifies a write expiration timeout for container elements. That
+    is, the element expires after the given amount of time since the
+    last time it has been written.
+
+.. bro:attr:: &create_expire
+
+    Specifies a creation expiration timeout for container elements. That
+    is, the element expires after the given amount of time since it has
+    been inserted into the container, regardless of any reads or writes.
+
+.. bro:attr:: &synchronized
+
+    Synchronizes variable accesses across nodes. The value of a
+    ``&synchronized`` variable is automatically propagated to all peers
+    when it changes.
+
+.. bro:attr:: &persistent
+
+    Makes a variable persistent, i.e., its value is written to disk (per
+    default at shutdown time).
+
+.. bro:attr:: &rotate_interval
+
+    Rotates a file after a specified interval.
+
+    Note: This attribute is deprecated and will be removed in a future release.
+
+.. bro:attr:: &rotate_size
+
+    Rotates a file after it has reached a given size in bytes.
+
+    Note: This attribute is deprecated and will be removed in a future release.
+
+.. bro:attr:: &encrypt
+
+    Encrypts files right before writing them to disk.
+
+    Note: This attribute is deprecated and will be removed in a future release.
+
+.. bro:attr:: &raw_output
+
+    Opens a file in raw mode, i.e., non-ASCII characters are not
+    escaped.
+
+.. bro:attr:: &mergeable
+
+    Prefers merging sets on assignment for synchronized state. This
+    attribute is used in conjunction with :bro:attr:`&synchronized`
+    container types: when the same container is updated at two peers
+    with different values, the propagation of the state causes a race
+    condition, where the last update succeeds. This can cause
+    inconsistencies and can be avoided by unifying the two sets, rather
+    than merely overwriting the old value.
+
+.. bro:attr:: &error_handler
+
+    Internally set on the events that are associated with the reporter
+    framework: :bro:id:`reporter_info`, :bro:id:`reporter_warning`, and
+    :bro:id:`reporter_error`.  It prevents any handlers of those events
+    from being able to generate reporter messages that go through any of
+    those events (i.e., it prevents an infinite event recursion).  Instead,
+    such nested reporter messages are output to stderr.
+
+.. bro:attr:: &type_column
+
+    Used by the input framework. It can be used on columns of type
+    :bro:type:`port` (such a column only contains the port number) and
+    specifies the name of an additional column in
+    the input file which specifies the protocol of the port (tcp/udp/icmp).
+
+    In the following example, the input file would contain four columns
+    named "ip", "srcp", "proto", and "msg"::
+
+        type Idx: record {
+            ip: addr;
+        };
+
+
+        type Val: record {
+            srcp: port &type_column = "proto";
+            msg: string;
+        };
+
+.. bro:attr:: &deprecated
+
+    The associated identifier is marked as deprecated and will be
+    removed in a future version of Bro.  Look in the NEWS file for more
+    instructions to migrate code that uses deprecated functionality.

doc/script-reference/directives.rst

@@ -0,0 +1,190 @@
+Directives
+==========
+
+The Bro scripting language supports a number of directives that can
+affect which scripts will be loaded or which lines in a script will be
+executed.  Directives are evaluated before script execution begins.
+
+.. bro:keyword:: @DEBUG
+
+    TODO
+
+
+.. bro:keyword:: @DIR
+
+    Expands to the directory pathname where the current script is located.
+
+    Example::
+
+        print "Directory:", @DIR;
+
+
+.. bro:keyword:: @FILENAME
+
+    Expands to the filename of the current script.
+
+    Example::
+
+        print "File:", @FILENAME;
+
+.. bro:keyword:: @load
+
+    Loads the specified Bro script, specified as the relative pathname
+    of the file (relative to one of the directories in Bro's file search path).
+    If the Bro script filename ends with ".bro", then you don't need to
+    specify the file extension.  The filename cannot contain any whitespace.
+
+    In this example, Bro will try to load a script
+    "policy/misc/capture-loss.bro" by looking in each directory in the file
+    search path (the file search path can be changed by setting the BROPATH
+    environment variable)::
+
+        @load policy/misc/capture-loss
+
+    If you specify the name of a directory instead of a filename, then
+    Bro will try to load a file in that directory called "__load__.bro"
+    (presumably that file will contain additional "@load" directives).
+
+    In this example, Bro will try to load a file "tuning/defaults/__load__.bro"
+    by looking in each directory in the file search path::
+
+        @load tuning/defaults
+
+    The purpose of this directive is to ensure that all script dependencies
+    are satisfied, and to avoid having to list every needed Bro script
+    on the command-line.  Bro keeps track of which scripts have been
+    loaded, so it is not an error to load a script more than once (once
+    a script has been loaded, any subsequent "@load" directives
+    for that script are ignored).
+
+
+.. bro:keyword:: @load-plugin
+
+    Activate a dynamic plugin with the specified plugin name.  The specified
+    plugin must be located in Bro's plugin search path.  Example::
+
+        @load-plugin Demo::Rot13
+
+    By default, Bro will automatically activate all dynamic plugins found
+    in the plugin search path (the search path can be changed by setting
+    the environment variable BRO_PLUGIN_PATH to a colon-separated list of
+    directories). However, in bare mode ("bro -b"), dynamic plugins can be
+    activated only by using "@load-plugin", or by specifying the full
+    plugin name on the Bro command-line (e.g., "bro Demo::Rot13"), or by
+    setting the environment variable BRO_PLUGIN_ACTIVATE to a
+    comma-separated list of plugin names.
+
+
+.. bro:keyword:: @load-sigs
+
+    This works similarly to "@load", except that in this case the filename
+    represents a signature file (not a Bro script).  If the signature filename
+    ends with ".sig", then you don't need to specify the file extension
+    in the "@load-sigs" directive.  The filename cannot contain any
+    whitespace.
+
+    In this example, Bro will try to load a signature file
+    "base/protocols/ssl/dpd.sig"::
+
+        @load-sigs base/protocols/ssl/dpd
+
+    The format for a signature file is explained in the documentation for the
+    `Signature Framework <../frameworks/signatures.html>`_.
+
+
+.. bro:keyword:: @unload
+
+    This specifies a Bro script that we don't want to load (so a subsequent
+    attempt to load the specified script will be skipped).  However,
+    if the specified script has already been loaded, then this directive
+    has no affect.
+
+    In the following example, if the "policy/misc/capture-loss.bro" script
+    has not been loaded yet, then Bro will not load it::
+
+        @unload policy/misc/capture-loss
+
+
+.. bro:keyword:: @prefixes
+
+    Specifies a filename prefix to use when looking for script files
+    to load automatically.  The prefix cannot contain any whitespace.
+
+    In the following example, the prefix "cluster" is used and all prefixes
+    that were previously specified are not used::
+
+        @prefixes = cluster
+
+    In the following example, the prefix "cluster-manager" is used in
+    addition to any previously-specified prefixes::
+
+        @prefixes += cluster-manager
+
+    The way this works is that after Bro parses all script files, then for each
+    loaded script Bro will take the absolute path of the script and then
+    it removes the portion of the directory path that is in Bro's file
+    search path.  Then it replaces each "/" character with a period "."
+    and then prepends the prefix (specified in the "@prefixes" directive)
+    followed by a period.  The resulting filename is searched for in each
+    directory in Bro's file search path.  If a matching file is found, then
+    the file is automatically loaded.
+
+    For example, if a script called "local.bro" has been loaded, and a prefix
+    of "test" was specified, then Bro will look for a file named
+    "test.local.bro" in each directory of Bro's file search path.
+
+    An alternative way to specify prefixes is to use the "-p" Bro
+    command-line option.
+
+.. bro:keyword:: @if
+
+    The specified expression must evaluate to type :bro:type:`bool`.  If the
+    value is true, then the following script lines (up to the next "@else"
+    or "@endif") are available to be executed.
+
+    Example::
+
+        @if ( ver == 2 )
+            print "version 2 detected";
+        @endif
+
+.. bro:keyword:: @ifdef
+
+    This works like "@if", except that the result is true if the specified
+    identifier is defined.
+
+    Example::
+
+        @ifdef ( pi )
+            print "pi is defined";
+        @endif
+
+.. bro:keyword:: @ifndef
+
+    This works exactly like "@ifdef", except that the result is true if the
+    specified identifier is not defined.
+
+    Example::
+
+        @ifndef ( pi )
+            print "pi is not defined";
+        @endif
+
+.. bro:keyword:: @else
+
+    This directive is optional after an "@if", "@ifdef", or
+    "@ifndef".  If present, it provides an else clause.
+
+    Example::
+
+        @ifdef ( pi )
+            print "pi is defined";
+        @else
+            print "pi is not defined";
+        @endif
+
+.. bro:keyword:: @endif
+
+    This directive is required to terminate each "@if", "@ifdef", or
+    "@ifndef".
+

doc/script-reference/index.rst

@@ -5,10 +5,17 @@ Script Reference
 .. toctree::
    :maxdepth: 1
 
+   operators
+   types
+   attributes
+   statements
+   directives
+   log-files
    notices
    proto-analyzers
    file-analyzers
-   builtins
    packages
    scripts
    Broxygen Example Script </scripts/broxygen/example.bro>
+
+

doc/script-reference/log-files.rst

@@ -0,0 +1,158 @@
+=========
+Log Files
+=========
+
+Listed below are the log files generated by Bro, including a brief description
+of the log file and links to descriptions of the fields for each log
+type.
+
+Network Protocols
+-----------------
+
++----------------------------+---------------------------------------+---------------------------------+
+| Log File                   | Description                           | Field Descriptions              |
++============================+=======================================+=================================+
+| conn.log                   | TCP/UDP/ICMP connections              | :bro:type:`Conn::Info`          |
++----------------------------+---------------------------------------+---------------------------------+
+| dhcp.log                   | DHCP leases                           | :bro:type:`DHCP::Info`          |
++----------------------------+---------------------------------------+---------------------------------+
+| dnp3.log                   | DNP3 requests and replies             | :bro:type:`DNP3::Info`          |
++----------------------------+---------------------------------------+---------------------------------+
+| dns.log                    | DNS activity                          | :bro:type:`DNS::Info`           |
++----------------------------+---------------------------------------+---------------------------------+
+| ftp.log                    | FTP activity                          | :bro:type:`FTP::Info`           |
++----------------------------+---------------------------------------+---------------------------------+
+| http.log                   | HTTP requests and replies             | :bro:type:`HTTP::Info`          |
++----------------------------+---------------------------------------+---------------------------------+
+| irc.log                    | IRC commands and responses            | :bro:type:`IRC::Info`           |
++----------------------------+---------------------------------------+---------------------------------+
+| kerberos.log               | Kerberos                              | :bro:type:`KRB::Info`           |
++----------------------------+---------------------------------------+---------------------------------+
+| modbus.log                 | Modbus commands and responses         | :bro:type:`Modbus::Info`        |
++----------------------------+---------------------------------------+---------------------------------+
+| modbus_register_change.log | Tracks changes to Modbus holding      | :bro:type:`Modbus::MemmapInfo`  |
+|                            | registers                             |                                 |
++----------------------------+---------------------------------------+---------------------------------+
+| mysql.log                  | MySQL                                 | :bro:type:`MySQL::Info`         |
++----------------------------+---------------------------------------+---------------------------------+
+| radius.log                 | RADIUS authentication attempts        | :bro:type:`RADIUS::Info`        |
++----------------------------+---------------------------------------+---------------------------------+
+| rdp.log                    | RDP                                   | :bro:type:`RDP::Info`           |
++----------------------------+---------------------------------------+---------------------------------+
+| sip.log                    | SIP                                   | :bro:type:`SIP::Info`           |
++----------------------------+---------------------------------------+---------------------------------+
+| smtp.log                   | SMTP transactions                     | :bro:type:`SMTP::Info`          |
++----------------------------+---------------------------------------+---------------------------------+
+| snmp.log                   | SNMP messages                         | :bro:type:`SNMP::Info`          |
++----------------------------+---------------------------------------+---------------------------------+
+| socks.log                  | SOCKS proxy requests                  | :bro:type:`SOCKS::Info`         |
++----------------------------+---------------------------------------+---------------------------------+
+| ssh.log                    | SSH connections                       | :bro:type:`SSH::Info`           |
++----------------------------+---------------------------------------+---------------------------------+
+| ssl.log                    | SSL/TLS handshake info                | :bro:type:`SSL::Info`           |
++----------------------------+---------------------------------------+---------------------------------+
+| syslog.log                 | Syslog messages                       | :bro:type:`Syslog::Info`        |
++----------------------------+---------------------------------------+---------------------------------+
+| tunnel.log                 | Tunneling protocol events             | :bro:type:`Tunnel::Info`        |
++----------------------------+---------------------------------------+---------------------------------+
+
+Files
+-----
+
++----------------------------+---------------------------------------+---------------------------------+
+| Log File                   | Description                           | Field Descriptions              |
++============================+=======================================+=================================+
+| files.log                  | File analysis results                 | :bro:type:`Files::Info`         |
++----------------------------+---------------------------------------+---------------------------------+
+| pe.log                     | Portable Executable (PE)              | :bro:type:`PE::Info`            |
++----------------------------+---------------------------------------+---------------------------------+
+| x509.log                   | X.509 certificate info                | :bro:type:`X509::Info`          |
++----------------------------+---------------------------------------+---------------------------------+
+
+Detection
+---------
+
++----------------------------+---------------------------------------+---------------------------------+
+| Log File                   | Description                           | Field Descriptions              |
++============================+=======================================+=================================+
+| intel.log                  | Intelligence data matches             | :bro:type:`Intel::Info`         |
++----------------------------+---------------------------------------+---------------------------------+
+| notice.log                 | Bro notices                           | :bro:type:`Notice::Info`        |
++----------------------------+---------------------------------------+---------------------------------+
+| notice_alarm.log           | The alarm stream                      | :bro:enum:`Notice::ACTION_ALARM`|
++----------------------------+---------------------------------------+---------------------------------+
+| signatures.log             | Signature matches                     | :bro:type:`Signatures::Info`    |
++----------------------------+---------------------------------------+---------------------------------+
+| traceroute.log             | Traceroute detection                  | :bro:type:`Traceroute::Info`    |
++----------------------------+---------------------------------------+---------------------------------+
+
+
+Network Observations
+--------------------
+
++----------------------------+---------------------------------------+---------------------------------+
+| Log File                   | Description                           | Field Descriptions              |
++============================+=======================================+=================================+
+| app_stats.log              | Web app usage statistics              | :bro:type:`AppStats::Info`      |
++----------------------------+---------------------------------------+---------------------------------+
+| known_certs.log            | SSL certificates                      | :bro:type:`Known::CertsInfo`    |
++----------------------------+---------------------------------------+---------------------------------+
+| known_devices.log          | MAC addresses of devices on the       | :bro:type:`Known::DevicesInfo`  |
+|                            | network                               |                                 |
++----------------------------+---------------------------------------+---------------------------------+
+| known_hosts.log            | Hosts that have completed TCP         | :bro:type:`Known::HostsInfo`    |
+|                            | handshakes                            |                                 |
++----------------------------+---------------------------------------+---------------------------------+
+| known_modbus.log           | Modbus masters and slaves             | :bro:type:`Known::ModbusInfo`   |
++----------------------------+---------------------------------------+---------------------------------+
+| known_services.log         | Services running on hosts             | :bro:type:`Known::ServicesInfo` |
++----------------------------+---------------------------------------+---------------------------------+
+| software.log               | Software being used on the network    | :bro:type:`Software::Info`      |
++----------------------------+---------------------------------------+---------------------------------+
+
+Miscellaneous
+-------------
+
++----------------------------+---------------------------------------+---------------------------------+
+| Log File                   | Description                           | Field Descriptions              |
++============================+=======================================+=================================+
+| barnyard2.log              | Alerts received from Barnyard2        | :bro:type:`Barnyard2::Info`     |
++----------------------------+---------------------------------------+---------------------------------+
+| dpd.log                    | Dynamic protocol detection failures   | :bro:type:`DPD::Info`           |
++----------------------------+---------------------------------------+---------------------------------+
+| unified2.log               | Interprets Snort's unified output     | :bro:type:`Unified2::Info`      |
++----------------------------+---------------------------------------+---------------------------------+
+| weird.log                  | Unexpected network-level activity     | :bro:type:`Weird::Info`         |
++----------------------------+---------------------------------------+---------------------------------+
+
+Bro Diagnostics
+---------------
+
++----------------------------+---------------------------------------+---------------------------------+
+| Log File                   | Description                           | Field Descriptions              |
++============================+=======================================+=================================+
+| capture_loss.log           | Packet loss rate                      | :bro:type:`CaptureLoss::Info`   |
++----------------------------+---------------------------------------+---------------------------------+
+| cluster.log                | Bro cluster messages                  | :bro:type:`Cluster::Info`       |
++----------------------------+---------------------------------------+---------------------------------+
+| communication.log          | Communication events between Bro or   | :bro:type:`Communication::Info` |
+|                            | Broccoli instances                    |                                 |
++----------------------------+---------------------------------------+---------------------------------+
+| loaded_scripts.log         | Shows all scripts loaded by Bro       | :bro:type:`LoadedScripts::Info` |
++----------------------------+---------------------------------------+---------------------------------+
+| packet_filter.log          | List packet filters that were applied | :bro:type:`PacketFilter::Info`  |
++----------------------------+---------------------------------------+---------------------------------+
+| prof.log                   | Profiling statistics (to create this  | N/A                             |
+|                            | log, load policy/misc/profiling.bro)  |                                 |
++----------------------------+---------------------------------------+---------------------------------+
+| reporter.log               | Internal error/warning/info messages  | :bro:type:`Reporter::Info`      |
++----------------------------+---------------------------------------+---------------------------------+
+| stats.log                  | Memory/event/packet/lag statistics    | :bro:type:`Stats::Info`         |
++----------------------------+---------------------------------------+---------------------------------+
+| stderr.log                 | Captures standard error when Bro is   | N/A                             |
+|                            | started from BroControl               |                                 |
++----------------------------+---------------------------------------+---------------------------------+
+| stdout.log                 | Captures standard output when Bro is  | N/A                             |
+|                            | started from BroControl               |                                 |
++----------------------------+---------------------------------------+---------------------------------+
+

doc/script-reference/operators.rst

@@ -0,0 +1,191 @@
+Operators
+=========
+
+The Bro scripting language supports the following operators.  Note that
+each data type only supports a subset of these operators.  For more
+details, see the documentation about the `data types <types.html>`_.
+
+Relational operators
+--------------------
+
+The relational operators evaluate to type :bro:type:`bool`.
+
++------------------------------+--------------+
+| Name                         | Syntax       |
++==============================+==============+
+| Equality                     | *a* == *b*   |
++------------------------------+--------------+
+| Inequality                   | *a* != *b*   |
++------------------------------+--------------+
+| Less than                    | *a* < *b*    |
++------------------------------+--------------+
+| Less than or equal           | *a* <= *b*   |
++------------------------------+--------------+
+| Greater than                 | *a* > *b*    |
++------------------------------+--------------+
+| Greater than or equal        | *a* >= *b*   |
++------------------------------+--------------+
+
+
+Logical operators
+-----------------
+
+The logical operators require operands of type :bro:type:`bool`, and
+evaluate to type :bro:type:`bool`.
+
++------------------------------+--------------+
+| Name                         | Syntax       |
++==============================+==============+
+| Logical AND                  | *a* && *b*   |
++------------------------------+--------------+
+| Logical OR                   | *a* \|\| *b* |
++------------------------------+--------------+
+| Logical NOT                  | ! *a*        |
++------------------------------+--------------+
+
+
+Arithmetic operators
+--------------------
+
++------------------------------+-------------+-------------------------------+
+| Name                         | Syntax      | Notes                         |
++==============================+=============+===============================+
+| Addition                     | *a* + *b*   | For :bro:type:`string`        |
+|                              |             | operands, this performs       |
+|                              |             | string concatenation.         |
++------------------------------+-------------+-------------------------------+
+| Subtraction                  | *a* - *b*   |                               |
++------------------------------+-------------+-------------------------------+
+| Multiplication               | *a* \* *b*  |                               |
++------------------------------+-------------+-------------------------------+
+| Division                     | *a* / *b*   | For :bro:type:`int` or        |
+|                              |             | :bro:type:`count` operands,   |
+|                              |             | the fractional part of the    |
+|                              |             | result is dropped.            |
++------------------------------+-------------+-------------------------------+
+| Modulo                       | *a* % *b*   | Operand types cannot be       |
+|                              |             | "double".                     |
++------------------------------+-------------+-------------------------------+
+| Unary plus                   | \+ *a*      |                               |
++------------------------------+-------------+-------------------------------+
+| Unary minus                  | \- *a*      |                               |
++------------------------------+-------------+-------------------------------+
+| Pre-increment                | ++ *a*      | Operand type cannot be        |
+|                              |             | "double".                     |
++------------------------------+-------------+-------------------------------+
+| Pre-decrement                | ``--`` *a*  | Operand type cannot be        |
+|                              |             | "double".                     |
++------------------------------+-------------+-------------------------------+
+| Absolute value               | \| *a* \|   | If operand is                 |
+|                              |             | :bro:type:`string`,           |
+|                              |             | :bro:type:`set`,              |
+|                              |             | :bro:type:`table`, or         |
+|                              |             | :bro:type:`vector`, this      |
+|                              |             | evaluates to number           |
+|                              |             | of elements.                  |
++------------------------------+-------------+-------------------------------+
+
+
+Assignment operators
+--------------------
+
+The assignment operators evaluate to the result of the assignment.
+
++------------------------------+-------------+
+| Name                         | Syntax      |
++==============================+=============+
+| Assignment                   | *a* = *b*   |
++------------------------------+-------------+
+| Addition assignment          | *a* += *b*  |
++------------------------------+-------------+
+| Subtraction assignment       | *a* -= *b*  |
++------------------------------+-------------+
+
+
+Record field operators
+----------------------
+
+The record field operators take a :bro:type:`record` as the first operand,
+and a field name as the second operand.  For both operators, the specified
+field name must be in the declaration of the record type.
+
++------------------------------+-------------+-------------------------------+
+| Name                         | Syntax      | Notes                         |
++==============================+=============+===============================+
+| Field access                 | *a* $ *b*   |                               |
++------------------------------+-------------+-------------------------------+
+| Field value existence test   | *a* ?$ *b*  | Evaluates to type             |
+|                              |             | :bro:type:`bool`.             |
+|                              |             | True if the specified field   |
+|                              |             | has been assigned a value, or |
+|                              |             | false if not.                 |
++------------------------------+-------------+-------------------------------+
+
+
+Other operators
+---------------
+
++--------------------------------+-------------------+------------------------+
+| Name                           | Syntax            | Notes                  |
++================================+===================+========================+
+| Membership test                | *a* in *b*        |Evaluates to type       |
+|                                |                   |:bro:type:`bool`. Do not|
+|                                |                   |confuse this use of "in"|
+|                                |                   |with that used in a     |
+|                                |                   |:bro:keyword:`for`      |
+|                                |                   |statement.              |
++--------------------------------+-------------------+------------------------+
+| Non-membership test            | *a* !in *b*       |This is the logical NOT |
+|                                |                   |of the "in" operator.   |
+|                                |                   |For example: "a !in b"  |
+|                                |                   |is equivalent to        |
+|                                |                   |"!(a in b)".            |
++--------------------------------+-------------------+------------------------+
+| Table or vector element access | *a* [ *b* ]       |This operator can also  |
+|                                |                   |be used with a          |
+|                                |                   |:bro:type:`set`, but    |
+|                                |                   |only with the           |
+|                                |                   |:bro:keyword:`add` or   |
+|                                |                   |:bro:keyword:`delete`   |
+|                                |                   |statement.              |
++--------------------------------+-------------------+------------------------+
+| Substring extraction           | *a* [ *b* : *c* ] |See the                 |
+|                                |                   |:bro:type:`string` type |
+|                                |                   |for more details.       |
++--------------------------------+-------------------+------------------------+
+| Create a deep copy             | copy ( *a* )      |This is relevant only   |
+|                                |                   |for data types that are |
+|                                |                   |assigned by reference,  |
+|                                |                   |such as                 |
+|                                |                   |:bro:type:`vector`,     |
+|                                |                   |:bro:type:`set`,        |
+|                                |                   |:bro:type:`table`,      |
+|                                |                   |and :bro:type:`record`. |
++--------------------------------+-------------------+------------------------+
+| Module namespace access        | *a* \:\: *b*      |The first operand is the|
+|                                |                   |module name, and the    |
+|                                |                   |second operand is an    |
+|                                |                   |identifier that refers  |
+|                                |                   |to a global variable,   |
+|                                |                   |enumeration constant, or|
+|                                |                   |user-defined type that  |
+|                                |                   |was exported from the   |
+|                                |                   |module.                 |
++--------------------------------+-------------------+------------------------+
+| Conditional                    | *a* ? *b* : *c*   |The first operand must  |
+|                                |                   |evaluate to type        |
+|                                |                   |:bro:type:`bool`.       |
+|                                |                   |If true, then the       |
+|                                |                   |second expression is    |
+|                                |                   |evaluated and is the    |
+|                                |                   |result of the entire    |
+|                                |                   |expression.  Otherwise, |
+|                                |                   |the third expression is |
+|                                |                   |evaluated and is the    |
+|                                |                   |result of the entire    |
+|                                |                   |expression. The types of|
+|                                |                   |the second and third    |
+|                                |                   |operands must be        |
+|                                |                   |compatible.             |
++--------------------------------+-------------------+------------------------+
+

doc/script-reference/statements.rst

@@ -0,0 +1,650 @@
+Declarations and Statements
+===========================
+
+The Bro scripting language supports the following declarations and
+statements.
+
+
+Declarations
+~~~~~~~~~~~~
+
++----------------------------+-----------------------------+
+| Name                       | Description                 |
++============================+=============================+
+| :bro:keyword:`module`      | Change the current module   |
++----------------------------+-----------------------------+
+| :bro:keyword:`export`      | Export identifiers from the |
+|                            | current module              |
++----------------------------+-----------------------------+
+| :bro:keyword:`global`      | Declare a global variable   |
++----------------------------+-----------------------------+
+| :bro:keyword:`const`       | Declare a constant          |
++----------------------------+-----------------------------+
+| :bro:keyword:`type`        | Declare a user-defined type |
++----------------------------+-----------------------------+
+| :bro:keyword:`redef`       | Redefine a global value or  |
+|                            | extend a user-defined type  |
++----------------------------+-----------------------------+
+| `function/event/hook`_     | Declare a function, event   |
+|                            | handler, or hook            |
++----------------------------+-----------------------------+
+
+Statements
+~~~~~~~~~~
+
++----------------------------+------------------------+
+| Name                       | Description            |
++============================+========================+
+| :bro:keyword:`local`       | Declare a local        |
+|                            | variable               |
++----------------------------+------------------------+
+| :bro:keyword:`add`,        | Add or delete          |
+| :bro:keyword:`delete`      | elements               |
++----------------------------+------------------------+
+| :bro:keyword:`print`       | Print to stdout or a   |
+|                            | file                   |
++----------------------------+------------------------+
+| :bro:keyword:`for`,        | Loop over each         |
+| :bro:keyword:`while`,      | element in a container |
+| :bro:keyword:`next`,       | object (``for``), or   |
+| :bro:keyword:`break`       | as long as a condition |
+|                            | evaluates to true      |
+|                            | (``while``).           |
++----------------------------+------------------------+
+| :bro:keyword:`if`          | Evaluate boolean       |
+|                            | expression and if true,|
+|                            | execute a statement    |
++----------------------------+------------------------+
+| :bro:keyword:`switch`,     | Evaluate expression    |
+| :bro:keyword:`break`,      | and execute statement  |
+| :bro:keyword:`fallthrough` | with a matching value  |
++----------------------------+------------------------+
+| :bro:keyword:`when`        | Asynchronous execution |
++----------------------------+------------------------+
+| :bro:keyword:`event`,      | Invoke or schedule     |
+| :bro:keyword:`schedule`    | an event handler       |
++----------------------------+------------------------+
+| :bro:keyword:`return`      | Return from function,  |
+|                            | hook, or event handler |
++----------------------------+------------------------+
+
+Declarations
+------------
+
+Declarations cannot occur within a function, hook, or event handler.
+
+Declarations must appear before any statements (except those statements
+that are in a function, hook, or event handler) in the concatenation of
+all loaded Bro scripts.
+
+.. bro:keyword:: module
+
+    The "module" keyword is used to change the current module.  This
+    affects the scope of any subsequently declared global identifiers.
+
+    Example::
+
+        module mymodule;
+
+    If a global identifier is declared after a "module" declaration,
+    then its scope ends at the end of the current Bro script or at the
+    next "module" declaration, whichever comes first.  However, if a
+    global identifier is declared after a "module" declaration, but inside
+    an :bro:keyword:`export` block, then its scope ends at the end of the
+    last loaded Bro script, but it must be referenced using the namespace
+    operator (``::``) in other modules.
+
+    There can be any number of "module" declarations in a Bro script.
+    The same "module" declaration can appear in any number of different
+    Bro scripts.
+
+
+.. bro:keyword:: export
+
+    An "export" block contains one or more declarations
+    (no statements are allowed in an "export" block) that the current
+    module is exporting.  This enables these global identifiers to be visible
+    in other modules (but not prior to their declaration) via the namespace
+    operator (``::``).  See the :bro:keyword:`module` keyword for a more
+    detailed explanation.
+
+    Example::
+
+        export {
+            redef enum Log::ID += { LOG };
+
+            type Info: record {
+                ts: time &log;
+                uid: string &log;
+            };
+
+            const conntime = 30sec &redef;
+        }
+
+    Note that the braces in an "export" block are always required
+    (they do not indicate a compound statement).  Also, no semicolon is
+    needed to terminate an "export" block.
+
+.. bro:keyword:: global
+
+    Variables declared with the "global" keyword will be global.
+
+    If a type is not specified, then an initializer is required so that
+    the type can be inferred.  Likewise, if an initializer is not supplied,
+    then the type must be specified.  In some cases, when the type cannot
+    be correctly inferred, the type must be specified even when an
+    initializer is present.  Example::
+
+        global pi = 3.14;
+        global hosts: set[addr];
+        global ciphers: table[string] of string = table();
+
+    Variable declarations outside of any function, hook, or event handler are
+    required to use this keyword (unless they are declared with the
+    :bro:keyword:`const` keyword instead).
+
+    Definitions of functions, hooks, and event handlers are not allowed
+    to use the "global" keyword.  However, function declarations (i.e., no
+    function body is provided) can use the "global" keyword.
+
+    The scope of a global variable begins where the declaration is located,
+    and extends through all remaining Bro scripts that are loaded (however,
+    see the :bro:keyword:`module` keyword for an explanation of how modules
+    change the visibility of global identifiers).
+
+
+.. bro:keyword:: const
+
+    A variable declared with the "const" keyword will be constant.
+
+    Variables declared as constant are required to be initialized at the
+    time of declaration.  Normally, the type is inferred from the initializer,
+    but the type can be explicitly specified.  Example::
+
+        const pi = 3.14;
+        const ssh_port: port = 22/tcp;
+
+    The value of a constant cannot be changed.  The only exception is if the
+    variable is a global constant and has the :bro:attr:`&redef`
+    attribute, but even then its value can be changed only with a
+    :bro:keyword:`redef`.
+
+    The scope of a constant is local if the declaration is in a
+    function, hook, or event handler, and global otherwise.
+
+    Note that the "const" keyword cannot be used with either the "local"
+    or "global" keywords (i.e., "const" replaces "local" and "global").
+
+
+.. bro:keyword:: type
+
+   The "type" keyword is used to declare a user-defined type.  The name
+   of this new type has global scope and can be used anywhere a built-in
+   type name can occur.
+
+   The "type" keyword is most commonly used when defining a
+   :bro:type:`record` or an :bro:type:`enum`, but is also useful when
+   dealing with more complex types.
+
+   Example::
+
+       type mytype: table[count] of table[addr, port] of string;
+       global myvar: mytype;
+
+.. bro:keyword:: redef
+
+    There are three ways that "redef" can be used:  to change the value of
+    a global variable (but only if it has the :bro:attr:`&redef` attribute),
+    to extend a record type or enum type, or to specify
+    a new event handler body that replaces all those that were previously
+    defined.
+
+    If you're using "redef" to change a global variable (defined using either
+    :bro:keyword:`const` or :bro:keyword:`global`), then the variable that you
+    want to change must have the :bro:attr:`&redef` attribute.  If the variable
+    you're changing is a table, set, or pattern, you can use ``+=`` to add
+    new elements, or you can use ``=`` to specify a new value (all previous
+    contents of the object are removed).  If the variable you're changing is a
+    set or table, then you can use the ``-=`` operator to remove the
+    specified elements (nothing happens for specified elements that don't
+    exist).  If the variable you are changing is not a table, set, or pattern,
+    then you must use the ``=`` operator.
+
+    Examples::
+
+        redef pi = 3.14;
+
+    If you're using "redef" to extend a record or enum, then you must
+    use the ``+=`` assignment operator.
+    For an enum, you can add more enumeration constants, and for a record
+    you can add more record fields (however, each record field in the "redef"
+    must have either the :bro:attr:`&optional` or :bro:attr:`&default`
+    attribute).
+
+    Examples::
+
+        redef enum color += { Blue, Red };
+        redef record MyRecord += { n2:int &optional; s2:string &optional; };
+
+    If you're using "redef" to specify a new event handler body that
+    replaces all those that were previously defined (i.e., any subsequently
+    defined event handler body will not be affected by this "redef"), then
+    the syntax is the same as a regular event handler definition except for
+    the presence of the "redef" keyword.
+
+    Example::
+
+        redef event myevent(s:string) { print "Redefined", s; }
+
+
+.. _function/event/hook:
+
+**function/event/hook**
+    For details on how to declare a :bro:type:`function`,
+    :bro:type:`event` handler, or :bro:type:`hook`,
+    see the documentation for those types.
+
+
+Statements
+----------
+
+Statements (except those contained within a function, hook, or event
+handler) can appear only after all global declarations in the concatenation
+of all loaded Bro scripts.
+
+Each statement in a Bro script must be terminated with a semicolon (with a
+few exceptions noted below).  An individual statement can span multiple
+lines.
+
+Here are the statements that the Bro scripting language supports.
+
+.. bro:keyword:: add
+
+    The "add" statement is used to add an element to a :bro:type:`set`.
+    Nothing happens if the specified element already exists in the set.
+
+    Example::
+
+        local myset: set[string];
+        add myset["test"];
+
+.. bro:keyword:: break
+
+    The "break" statement is used to break out of a :bro:keyword:`switch`,
+    :bro:keyword:`for`, or :bro:keyword:`while` statement.
+
+
+.. bro:keyword:: delete
+
+    The "delete" statement is used to remove an element from a
+    :bro:type:`set` or :bro:type:`table`.  Nothing happens if the
+    specified element does not exist in the set or table.
+
+    Example::
+
+        local myset = set("this", "test");
+        local mytable = table(["key1"] = 80/tcp, ["key2"] = 53/udp);
+        delete myset["test"];
+        delete mytable["key1"];
+
+.. bro:keyword:: event
+
+    The "event" statement immediately queues invocation of an event handler.
+
+    Example::
+
+        event myevent("test", 5);
+
+.. bro:keyword:: fallthrough
+
+    The "fallthrough" statement can be used as the last statement in a
+    "case" block to indicate that execution should continue into the
+    next "case" or "default" label.
+
+    For an example, see the :bro:keyword:`switch` statement.
+
+.. bro:keyword:: for
+
+    A "for" loop iterates over each element in a string, set, vector, or
+    table and executes a statement for each iteration.  Currently,
+    modifying a container's membership while iterating over it may
+    result in undefined behavior, so avoid adding or removing elements
+    inside the loop.
+
+    For each iteration of the loop, a loop variable will be assigned to an
+    element if the expression evaluates to a string or set, or an index if
+    the expression evaluates to a vector or table.  Then the statement
+    is executed.  However, the statement will not be executed if the expression
+    evaluates to an object with no elements.
+
+    If the expression is a table or a set with more than one index, then the
+    loop variable must be specified as a comma-separated list of different
+    loop variables (one for each index), enclosed in brackets.
+
+    A :bro:keyword:`break` statement can be used at any time to immediately
+    terminate the "for" loop, and a :bro:keyword:`next` statement can be
+    used to skip to the next loop iteration.
+
+    Note that the loop variable in a "for" statement is not allowed to be
+    a global variable, and it does not need to be declared prior to the "for"
+    statement.  The type will be inferred from the elements of the
+    expression.
+
+    Example::
+
+        local myset = set(80/tcp, 81/tcp);
+        local mytable = table([10.0.0.1, 80/tcp]="s1", [10.0.0.2, 81/tcp]="s2");
+
+        for (p in myset)
+            print p;
+
+        for ([i,j] in mytable) {
+            if (mytable[i,j] == "done")
+                break;
+            if (mytable[i,j] == "skip")
+                next;
+            print i,j;
+        }
+
+
+.. bro:keyword:: if
+
+    Evaluates a given expression, which must yield a :bro:type:`bool` value.
+    If true, then a specified statement is executed.  If false, then
+    the statement is not executed.  Example::
+
+        if ( x == 2 ) print "x is 2";
+
+
+    However, if the expression evaluates to false and if an "else" is
+    provided, then the statement following the "else" is executed.  Example::
+
+        if ( x == 2 )
+            print "x is 2";
+        else
+            print "x is not 2";
+
+.. bro:keyword:: local
+
+    A variable declared with the "local" keyword will be local.  If a type
+    is not specified, then an initializer is required so that the type can
+    be inferred.  Likewise, if an initializer is not supplied, then the
+    type must be specified.
+
+    Examples::
+
+        local x1 = 5.7;
+        local x2: double;
+        local x3: double = 5.7;
+
+    Variable declarations inside a function, hook, or event handler are
+    required to use this keyword (the only two exceptions are variables
+    declared with :bro:keyword:`const`, and variables implicitly declared in a
+    :bro:keyword:`for` statement).
+
+    The scope of a local variable starts at the location where it is declared
+    and persists to the end of the function, hook,
+    or event handler in which it is declared (this is true even if the
+    local variable was declared within a `compound statement`_ or is the loop
+    variable in a "for" statement).
+
+
+.. bro:keyword:: next
+
+    The "next" statement can only appear within a :bro:keyword:`for` or
+    :bro:keyword:`while` loop.  It causes execution to skip to the next
+    iteration.
+
+
+.. bro:keyword:: print
+
+    The "print" statement takes a comma-separated list of one or more
+    expressions.  Each expression in the list is evaluated and then converted
+    to a string.  Then each string is printed, with each string separated by
+    a comma in the output.
+
+    Examples::
+
+        print 3.14;
+        print "Results", x, y;
+
+    By default, the "print" statement writes to the standard
+    output (stdout).  However, if the first expression is of type
+    :bro:type:`file`, then "print" writes to that file.
+
+    If a string contains non-printable characters (i.e., byte values that are
+    not in the range 32 - 126), then the "print" statement converts each
+    non-printable character to an escape sequence before it is printed.
+
+    For more control over how the strings are formatted, see the :bro:id:`fmt`
+    function.
+
+.. bro:keyword:: return
+
+    The "return" statement immediately exits the current function, hook, or
+    event handler.  For a function, the specified expression (if any) is
+    evaluated and returned.  A "return" statement in a hook or event handler
+    cannot return a value because event handlers and hooks do not have
+    return types.
+
+    Examples::
+
+        function my_func(): string
+        {
+            return "done";
+        }
+
+        event my_event(n: count)
+        {
+            if ( n == 0 ) return;
+
+            print n;
+        }
+
+    There is a special form of the "return" statement that is only allowed
+    in functions.  Syntactically, it looks like a :bro:keyword:`when` statement
+    immediately preceded by the "return" keyword.  This form of the "return"
+    statement is used to specify a function that delays its result (such a
+    function can only be called in the expression of a :bro:keyword:`when`
+    statement).  The function returns at the time the "when"
+    statement's condition becomes true, and the function returns the value
+    that the "when" statement's body returns (or if the condition does
+    not become true within the specified timeout interval, then the function
+    returns the value that the "timeout" block returns).
+
+    Example::
+
+      global X: table[string] of count;
+
+      function a() : count
+            {
+            # This delays until condition becomes true.
+            return when ( "a" in X )
+                  {
+                  return X["a"];
+                  }
+            timeout 30 sec
+                  {
+                  return 0;
+                  }
+            }
+
+      event bro_init()
+            {
+            # Installs a trigger which fires if a() returns 42.
+            when ( a() == 42 )
+                print "expected result";
+
+            print "Waiting for a() to return...";
+            X["a"] = 42;
+            }
+
+
+.. bro:keyword:: schedule
+
+    The "schedule" statement is used to raise a specified event with
+    specified parameters at a later time specified as an :bro:type:`interval`.
+
+    Example::
+
+        schedule 30sec { myevent(x, y, z) };
+
+    Note that the braces are always required (they do not indicate a
+    `compound statement`_).
+
+    Note that "schedule" is actually an expression that returns a value
+    of type "timer", but in practice the return value is not used.
+
+.. bro:keyword:: switch
+
+    A "switch" statement evaluates a given expression and jumps to
+    the first "case" label which contains a matching value (the result of the
+    expression must be type-compatible with all of the values in all of the
+    "case" labels).  If there is no matching value, then execution jumps to
+    the "default" label instead, and if there is no "default" label then
+    execution jumps out of the "switch" block.
+
+    Here is an example (assuming that "get_day_of_week" is a
+    function that returns a string)::
+
+        switch get_day_of_week()
+            {
+            case "Sa", "Su":
+                print "weekend";
+                fallthrough;
+            case "Mo", "Tu", "We", "Th", "Fr":
+                print "valid result";
+                break;
+            default:
+                print "invalid result";
+                break;
+            }
+
+    A "switch" block can have any number of "case" labels, and one
+    optional "default" label.
+
+    A "case" label can have a comma-separated list of
+    more than one value.  A value in a "case" label can be an expression,
+    but it must be a constant expression (i.e., the expression can consist
+    only of constants).
+
+    Each "case" and the "default" block must
+    end with either a :bro:keyword:`break`, :bro:keyword:`fallthrough`, or
+    :bro:keyword:`return` statement (although "return" is allowed only
+    if the "switch" statement is inside a function, hook, or event handler).
+    If a "case" (or "default") block contain more than one statement, then
+    there is no need to wrap them in braces.
+
+    Note that the braces in a "switch" statement are always required (these
+    do not indicate the presence of a `compound statement`_), and that no
+    semicolon is needed at the end of a "switch" statement.
+
+
+.. bro:keyword:: when
+
+    Evaluates a given expression, which must result in a value of type
+    :bro:type:`bool`.  When the value of the expression becomes available
+    and if the result is true, then a specified statement is executed.
+
+    In the following example, if the expression evaluates to true, then
+    the "print" statement is executed::
+
+        when ( (local x = foo()) && x == 42 )
+            print x;
+
+    However, if a timeout is specified, and if the expression does not
+    evaluate to true within the specified timeout interval, then the
+    statement following the "timeout" keyword is executed::
+
+        when ( (local x = foo()) && x == 42 )
+            print x;
+        timeout 5sec {
+            print "timeout";
+        }
+
+    Note that when a timeout is specified the braces are
+    always required (these do not indicate a `compound statement`_).
+
+    The expression in a "when" statement can contain a declaration of a local
+    variable but only if the declaration is written in the form
+    "local *var* = *init*" (example: "local x = myfunction()").  This form
+    of a local declaration is actually an expression, the result of which
+    is always a boolean true value.
+
+    The expression in a "when" statement can contain an asynchronous function
+    call such as :bro:id:`lookup_hostname` (in fact, this is the only place
+    such a function can be called), but it can also contain an ordinary
+    function call.  When an asynchronous function call is in the expression,
+    then Bro will continue processing statements in the script following
+    the "when" statement, and when the result of the function call is available
+    Bro will finish evaluating the expression in the "when" statement.
+    See the :bro:keyword:`return` statement for an explanation of how to
+    create an asynchronous function in a Bro script.
+
+.. bro:keyword:: while
+
+    A "while" loop iterates over a body statement as long as a given
+    condition remains true.
+
+    A :bro:keyword:`break` statement can be used at any time to immediately
+    terminate the "while" loop, and a :bro:keyword:`next` statement can be
+    used to skip to the next loop iteration.
+
+    Example::
+
+        local i = 0;
+
+        while ( i < 5 )
+            print ++i;
+
+        while ( some_cond() )
+            {
+            local finish_up = F;
+
+            if ( skip_ahead() )
+                next;
+
+            [...]
+
+            if ( finish_up )
+                break;
+
+            [...]
+            }
+
+.. _compound statement:
+
+**compound statement**
+    A compound statement is created by wrapping zero or more statements in
+    braces ``{ }``.  Individual statements inside the braces need to be
+    terminated by a semicolon, but a semicolon is not needed at the end
+    (outside of the braces) of a compound statement.
+
+    A compound statement is required in order to execute more than one
+    statement in the body of a :bro:keyword:`for`, :bro:keyword:`while`,
+    :bro:keyword:`if`, or :bro:keyword:`when` statement.
+
+    Example::
+
+        if ( x == 2 ) {
+            print "x is 2";
+            ++x;
+        }
+
+    Note that there are other places in the Bro scripting language that use
+    braces, but that do not indicate the presence of a compound
+    statement (these are noted in the documentation).
+
+.. _null:
+
+**null statement**
+    The null statement (executing it has no effect) consists of just a
+    semicolon.  This might be useful during testing or debugging a Bro script
+    in places where a statement is required, but it is probably not useful
+    otherwise.
+
+    Example::
+
+        if ( x == 2 )
+            ;
+

doc/script-reference/types.rst

@@ -1,92 +1,114 @@
-Types and Attributes
-====================
-
 Types
------
+=====
 
-Every value in a Bro script has a type (see below for a list of all built-in
+The Bro scripting language supports the following built-in types:
-types).  Although Bro variables have static types (meaning that their type
-is fixed), their type is inferred from the value to which they are
-initially assigned when the variable is declared without an explicit type
-name.
 
-Automatic conversions happen when a binary operator has operands of 
++-----------------------+--------------------+
-different types.  Automatic conversions are limited to converting between
+| Name                  | Description        |
-numeric types.  The numeric types are ``int``, ``count``, and ``double``
++=======================+====================+
-(``bool`` is not a numeric type).
+| :bro:type:`bool`      | Boolean            |
-When an automatic conversion occurs, values are promoted to the "highest"
++-----------------------+--------------------+
-type in the expression. In general, this promotion follows a simple
+| :bro:type:`count`,    | Numeric types      |
-hierarchy: ``double`` is highest, ``int`` comes next, and ``count`` is
+| :bro:type:`int`,      |                    |
-lowest.
+| :bro:type:`double`    |                    |
++-----------------------+--------------------+
+| :bro:type:`time`,     | Time types         |
+| :bro:type:`interval`  |                    |
++-----------------------+--------------------+
+| :bro:type:`string`    | String             |
++-----------------------+--------------------+
+| :bro:type:`pattern`   | Regular expression |
++-----------------------+--------------------+
+| :bro:type:`port`,     | Network types      |
+| :bro:type:`addr`,     |                    |
+| :bro:type:`subnet`    |                    |
++-----------------------+--------------------+
+| :bro:type:`enum`      | Enumeration        |
+|                       | (user-defined type)|
++-----------------------+--------------------+
+| :bro:type:`table`,    | Container types    |
+| :bro:type:`set`,      |                    |
+| :bro:type:`vector`,   |                    |
+| :bro:type:`record`    |                    |
++-----------------------+--------------------+
+| :bro:type:`function`, | Executable types   |
+| :bro:type:`event`,    |                    |
+| :bro:type:`hook`      |                    |
++-----------------------+--------------------+
+| :bro:type:`file`      | File type (only    |
+|                       | for writing)       |
++-----------------------+--------------------+
+| :bro:type:`opaque`    | Opaque type (for   |
+|                       | some built-in      |
+|                       | functions)         |
++-----------------------+--------------------+
+| :bro:type:`any`       | Any type (for      |
+|                       | functions or       |
+|                       | containers)        |
++-----------------------+--------------------+
 
-The Bro scripting language supports the following built-in types.
+Here is a more detailed description of each type:
-
-.. bro:type:: void
-
-    An internal Bro type (i.e., "void" is not a reserved keyword in the Bro
-    scripting language) representing the absence of a return type for a
-    function.
 
 .. bro:type:: bool
 
     Reflects a value with one of two meanings: true or false.  The two
-    ``bool`` constants are ``T`` and ``F``.
+    "bool" constants are ``T`` and ``F``.
 
-    The ``bool`` type supports the following operators: equality/inequality
+    The "bool" type supports the following operators: equality/inequality
     (``==``, ``!=``), logical and/or (``&&``, ``||``), logical
-    negation (``!``), and absolute value (where ``|T|`` is 1, and ``|F|`` is 0).
+    negation (``!``), and absolute value (where ``|T|`` is 1, and ``|F|`` is 0,
+    and in both cases the result type is :bro:type:`count`).
 
 .. bro:type:: int
 
-    A numeric type representing a 64-bit signed integer.  An ``int`` constant
+    A numeric type representing a 64-bit signed integer.  An "int" constant
-    is a string of digits preceded by a ``+`` or ``-`` sign, e.g.
+    is a string of digits preceded by a "+" or "-" sign, e.g.
     ``-42`` or ``+5`` (the "+" sign is optional but see note about type
-    inferencing below).  An ``int`` constant can also be written in
+    inferencing below).  An "int" constant can also be written in
     hexadecimal notation (in which case "0x" must be between the sign and
     the hex digits), e.g. ``-0xFF`` or ``+0xabc123``.
 
-    The ``int`` type supports the following operators:  arithmetic
+    The "int" type supports the following operators:  arithmetic
     operators (``+``, ``-``, ``*``, ``/``, ``%``), comparison operators
     (``==``, ``!=``, ``<``, ``<=``, ``>``, ``>=``), assignment operators
     (``=``, ``+=``, ``-=``), pre-increment (``++``), pre-decrement
-    (``--``), and absolute value (e.g., ``|-3|`` is 3).
+    (``--``), unary plus and minus (``+``, ``-``), and absolute value
+    (e.g., ``|-3|`` is 3, but the result type is :bro:type:`count`).
 
     When using type inferencing use care so that the
-    intended type is inferred, e.g. ``local size_difference = 0`` will
+    intended type is inferred, e.g. "local size_difference = 0" will
-    infer :bro:type:`count`, while ``local size_difference = +0``
+    infer ":bro:type:`count`", while "local size_difference = +0"
-    will infer :bro:type:`int`.
+    will infer "int".
 
 .. bro:type:: count
 
-    A numeric type representing a 64-bit unsigned integer.  A ``count``
+    A numeric type representing a 64-bit unsigned integer.  A "count"
-    constant is a string of digits, e.g. ``1234`` or ``0``.  A ``count``
+    constant is a string of digits, e.g. ``1234`` or ``0``.  A "count"
     can also be written in hexadecimal notation (in which case "0x" must
     precede the hex digits), e.g. ``0xff`` or ``0xABC123``.
 
-    The ``count`` type supports the same operators as the :bro:type:`int`
+    The "count" type supports the same operators as the ":bro:type:`int`"
-    type.  A unary plus or minus applied to a ``count`` results in an ``int``.
+    type, but a unary plus or minus applied to a "count" results in an
-
+    "int".
-.. bro:type:: counter
-
-    An alias to :bro:type:`count`.
 
 .. bro:type:: double
 
     A numeric type representing a double-precision floating-point
     number.  Floating-point constants are written as a string of digits
     with an optional decimal point, optional scale-factor in scientific
-    notation, and optional ``+`` or ``-`` sign.  Examples are ``-1234``,
+    notation, and optional "+" or "-" sign.  Examples are ``-1234``,
     ``-1234e0``, ``3.14159``, and ``.003E-23``.
 
-    The ``double`` type supports the following operators:  arithmetic
+    The "double" type supports the following operators:  arithmetic
     operators (``+``, ``-``, ``*``, ``/``), comparison operators
     (``==``, ``!=``, ``<``, ``<=``, ``>``, ``>=``), assignment operators
-    (``=``, ``+=``, ``-=``), and absolute value (e.g., ``|-3.14|`` is 3.14).
+    (``=``, ``+=``, ``-=``), unary plus and minus (``+``, ``-``), and
+    absolute value (e.g., ``|-3.14|`` is 3.14).
 
     When using type inferencing use care so that the
-    intended type is inferred, e.g. ``local size_difference = 5`` will
+    intended type is inferred, e.g. "local size_difference = 5" will
-    infer :bro:type:`count`, while ``local size_difference = 5.0``
+    infer ":bro:type:`count`", while "local size_difference = 5.0"
-    will infer :bro:type:`double`.
+    will infer "double".
 
 .. bro:type:: time
 
@@ -97,10 +119,10 @@ The Bro scripting language supports the following built-in types.
 
     Time values support the comparison operators (``==``, ``!=``, ``<``,
     ``<=``, ``>``, ``>=``).  A ``time`` value can be subtracted from
-    another ``time`` value to produce an ``interval`` value.  An ``interval``
+    another ``time`` value to produce an :bro:type:`interval` value.  An
-    value can be added to, or subtracted from, a ``time`` value to produce a
+    ``interval`` value can be added to, or subtracted from, a ``time`` value
-    ``time`` value.  The absolute value of a ``time`` value is a ``double``
+    to produce a ``time`` value.  The absolute value of a ``time`` value is
-    with the same numeric value.
+    a :bro:type:`double` with the same numeric value.
 
 .. bro:type:: interval
 
@@ -115,52 +137,58 @@ The Bro scripting language supports the following built-in types.
     ``3.5mins``.  An ``interval`` can also be negated, for example
     ``-12 hr`` represents "twelve hours in the past".
 
-    Intervals support addition and subtraction.  Intervals also support
+    Intervals support addition and subtraction, the comparison operators
-    division (in which case the result is a ``double`` value), the
+    (``==``, ``!=``, ``<``, ``<=``, ``>``, ``>=``), the assignment
-    comparison operators (``==``, ``!=``, ``<``, ``<=``, ``>``, ``>=``),
+    operators (``=``, ``+=``, ``-=``), and unary plus and minus (``+``, ``-``).
-    and the assignment operators (``=``, ``+=``, ``-=``).  Also, an
+
-    ``interval`` can be multiplied or divided by an arithmetic type
+    Intervals also support division (in which case the result is a
-    (``count``, ``int``, or ``double``) to produce an ``interval`` value.
+    :bro:type:`double` value).  An ``interval`` can be multiplied or divided
-    The absolute value of an ``interval`` is a ``double`` value equal to the
+    by an arithmetic type (``count``, ``int``, or ``double``) to produce
-    number of seconds in the ``interval`` (e.g., ``|-1 min|`` is 60).
+    an ``interval`` value.  The absolute value of an ``interval`` is a
+    ``double`` value equal to the number of seconds in the ``interval``
+    (e.g., ``|-1 min|`` is 60.0).
 
 .. bro:type:: string
 
-    A type used to hold character-string values which represent text.
+    A type used to hold character-string values which represent text, although
-    String constants are created by enclosing text in double quotes (")
+    strings in a Bro script can actually contain any arbitrary binary data.
-    and the backslash character (\\) introduces escape sequences (all of
+
-    the C-style escape sequences are supported).
+    String constants are created by enclosing text within a pair of double
+    quotes (").  A string constant cannot span multiple lines in a Bro script.
+    The backslash character (\\) introduces escape sequences.  The
+    following escape sequences are recognized: ``\n``, ``\t``, ``\v``, ``\b``,
+    ``\r``, ``\f``, ``\a``, ``\ooo`` (where each 'o' is an octal digit),
+    ``\xhh`` (where each 'h' is a hexadecimal digit).  For escape sequences
+    that don't match any of these, Bro will just remove the backslash (so
+    to represent a literal backslash in a string constant, you just use
+    two consecutive backslashes).
 
     Strings support concatenation (``+``), and assignment (``=``, ``+=``).
     Strings also support the comparison operators (``==``, ``!=``, ``<``,
     ``<=``, ``>``, ``>=``).  The number of characters in a string can be
     found by enclosing the string within pipe characters (e.g., ``|"abc"|``
-    is 3).
+    is 3).  Substring searching can be performed using the "in" or "!in"
-
-    The subscript operator can extract an individual character or a substring
-    of a string (string indexing is zero-based, but an index of
-    -1 refers to the last character in the string, and -2 refers to the
-    second-to-last character, etc.).  When extracting a substring, the
-    starting and ending index values are separated by a colon.  For example::
-
-        local orig = "0123456789";
-        local third_char = orig[2];
-        local last_char = orig[-1];
-        local first_three_chars = orig[0:2];
-
-    Substring searching can be performed using the "in" or "!in"
     operators (e.g., "bar" in "foobar" yields true).
 
-    Note that Bro represents strings internally as a count and vector of
+    The subscript operator can extract a substring of a string.  To do this,
-    bytes rather than a NUL-terminated byte string (although string
+    specify the starting index to extract (if the starting index is omitted,
-    constants are also automatically NUL-terminated).  This is because
+    then zero is assumed), followed by a colon and index
-    network traffic can easily introduce NULs into strings either by
+    one past the last character to extract (if the last index is omitted,
-    nature of an application, inadvertently, or maliciously.  And while
+    then the extracted substring will go to the end of the original string).
-    NULs are allowed in Bro strings, when present in strings passed as
+    However, if both the colon and last index are omitted, then a string of
-    arguments to many functions, a run-time error can occur as their
+    length one is extracted.  String indexing is zero-based, but an index
-    presence likely indicates a sort of problem.  In that case, the
+    of -1 refers to the last character in the string, and -2 refers to the
-    string will also only be represented to the user as the literal
+    second-to-last character, etc.  Here are a few examples::
-    "<string-with-NUL>" string.
+
+        local orig = "0123456789";
+        local second_char = orig[1];
+        local last_char = orig[-1];
+        local first_two_chars = orig[:2];
+        local last_two_chars = orig[8:];
+        local no_first_and_last = orig[1:9];
+
+    Note that the subscript operator cannot be used to modify a string (i.e.,
+    it cannot be on the left side of an assignment operator).
 
 .. bro:type:: pattern
 
@@ -174,7 +202,7 @@ The Bro scripting language supports the following built-in types.
     and embedded.
 
     In exact matching the ``==`` equality relational operator is used
-    with one :bro:type:`pattern` operand and one :bro:type:`string`
+    with one "pattern" operand and one ":bro:type:`string`"
     operand (order of operands does not matter) to check whether the full
     string exactly matches the pattern.  In exact matching, the ``^``
     beginning-of-line and ``$`` end-of-line anchors are redundant since
@@ -190,8 +218,8 @@ The Bro scripting language supports the following built-in types.
     yields false.  The ``!=`` operator would yield the negation of ``==``.
 
     In embedded matching the ``in`` operator is used with one
-    :bro:type:`pattern` operand (which must be on the left-hand side) and
+    "pattern" operand (which must be on the left-hand side) and
-    one :bro:type:`string` operand, but tests whether the pattern
+    one ":bro:type:`string`" operand, but tests whether the pattern
     appears anywhere within the given string.  For example::
 
         /foo|bar/ in "foobar"
@@ -203,27 +231,12 @@ The Bro scripting language supports the following built-in types.
     is false since "oob" does not appear at the start of "foobar".  The
     ``!in`` operator would yield the negation of ``in``.
 
-.. bro:type:: enum
-
-    A type allowing the specification of a set of related values that
-    have no further structure.  An example declaration:
-
-    .. code:: bro
-
-        type color: enum { Red, White, Blue, };
-
-    The last comma after ``Blue`` is optional.
-
-    The only operations allowed on enumerations are equality comparisons
-    (``==``, ``!=``) and assignment (``=``).
-    Enumerations do not have associated values or ordering.
-
 .. bro:type:: port
 
-    A type representing transport-level port numbers.  Besides TCP and
+    A type representing transport-level port numbers (besides TCP and
     UDP ports, there is a concept of an ICMP "port" where the source
     port is the ICMP message type and the destination port the ICMP
-    message code.  A ``port`` constant is written as an unsigned integer
+    message code).  A ``port`` constant is written as an unsigned integer
     followed by one of ``/tcp``, ``/udp``, ``/icmp``, or ``/unknown``.
 
     Ports support the comparison operators (``==``, ``!=``, ``<``, ``<=``,
@@ -255,14 +268,6 @@ The Bro scripting language supports the following built-in types.
     address) are treated internally as IPv4 addresses (for example,
     ``[::ffff:192.168.1.100]`` is equal to ``192.168.1.100``).
 
-    Hostname constants can also be used, but since a hostname can
-    correspond to multiple IP addresses, the type of such a variable is a
-    :bro:type:`set` of :bro:type:`addr` elements. For example:
-
-    .. code:: bro
-
-        local a = www.google.com;
-
     Addresses can be compared for equality (``==``, ``!=``),
     and also for ordering (``<``, ``<=``, ``>``, ``>=``).  The absolute value
     of an address gives the size in bits (32 for IPv4, and 128 for IPv6).
@@ -285,9 +290,17 @@ The Bro scripting language supports the following built-in types.
         if ( a in s )
             print "true";
 
-    Note that you can check if a given ``addr`` is IPv4 or IPv6 using
+    You can check if a given ``addr`` is IPv4 or IPv6 using
     the :bro:id:`is_v4_addr` and :bro:id:`is_v6_addr` built-in functions.
 
+    Note that hostname constants can also be used, but since a hostname can
+    correspond to multiple IP addresses, the type of such a variable is
+    "set[addr]". For example:
+
+    .. code:: bro
+
+        local a = www.google.com;
+
 .. bro:type:: subnet
 
     A type representing a block of IP addresses in CIDR notation.  A
@@ -296,13 +309,24 @@ The Bro scripting language supports the following built-in types.
     number.  For example, ``192.168.0.0/16`` or ``[fe80::]/64``.
 
     Subnets can be compared for equality (``==``, ``!=``).  An
-    :bro:type:`addr` can be checked for inclusion in a subnet using
+    "addr" can be checked for inclusion in a subnet using
-    the "in" or "!in" operators.
+    the ``in`` or ``!in`` operators.
 
-.. bro:type:: any
+.. bro:type:: enum
 
-    Used to bypass strong typing.  For example, a function can take an
+    A type allowing the specification of a set of related values that
-    argument of type ``any`` when it may be of different types.
+    have no further structure.  An example declaration:
+
+    .. code:: bro
+
+        type color: enum { Red, White, Blue, };
+
+    The last comma after ``Blue`` is optional.  Both the type name ``color``
+    and the individual values (``Red``, etc.) have global scope.
+
+    Enumerations do not have associated values or ordering.
+    The only operations allowed on enumerations are equality comparisons
+    (``==``, ``!=``) and assignment (``=``).
 
 .. bro:type:: table
 
@@ -316,24 +340,28 @@ The Bro scripting language supports the following built-in types.
 
         table [ type^+ ] of type
 
-    where *type^+* is one or more types, separated by commas.  For example:
+    where *type^+* is one or more types, separated by commas.  The
+    index type cannot be any of the following types:  pattern, table, set,
+    vector, file, opaque, any.
+
+    Here is an example of declaring a table indexed by "count" values
+    and yielding "string" values:
 
     .. code:: bro
 
         global a: table[count] of string;
 
-    declares a table indexed by :bro:type:`count` values and yielding
+    The yield type can also be more complex:
-    :bro:type:`string` values.  The yield type can also be more complex:
 
     .. code:: bro
 
         global a: table[count] of table[addr, port] of string;
 
-    which declares a table indexed by :bro:type:`count` and yielding
+    which declares a table indexed by "count" and yielding
-    another :bro:type:`table` which is indexed by an :bro:type:`addr`
+    another "table" which is indexed by an "addr"
-    and :bro:type:`port` to yield a :bro:type:`string`.
+    and "port" to yield a "string".
 
-    Initialization of tables occurs by enclosing a set of initializers within
+    One way to initialize a table is by enclosing a set of initializers within
     braces, for example:
 
     .. code:: bro
@@ -343,18 +371,17 @@ The Bro scripting language supports the following built-in types.
             [5] = "five",
         };
 
-    A table constructor (equivalent to above example) can also be used
+    A table constructor can also be used to create a table:
-    to create a table:
 
     .. code:: bro
 
-        global t2: table[count] of string = table(
+        global t2 = table(
-            [11] = "eleven",
+            [192.168.0.2, 22/tcp] = "ssh",
-            [5] = "five"
+            [192.168.0.3, 80/tcp] = "http"
         );
 
     Table constructors can also be explicitly named by a type, which is
-    useful for when a more complex index type could otherwise be
+    useful when a more complex index type could otherwise be
     ambiguous:
 
     .. code:: bro
@@ -381,17 +408,7 @@ The Bro scripting language supports the following built-in types.
 
         if ( 13 in t )
             ...
-
+        if ( [192.168.0.2, 22/tcp] in t2 )
-    Iterate over tables with a ``for`` loop:
-
-    .. code:: bro
-
-        local t: table[count] of string;
-        for ( n in t )
-            ...
-
-        local services: table[addr, port] of string;
-        for ( [a, p] in services )
             ...
 
     Add or overwrite individual table elements by assignment:
@@ -400,7 +417,7 @@ The Bro scripting language supports the following built-in types.
 
         t[13] = "thirteen";
 
-    Remove individual table elements with ``delete``:
+    Remove individual table elements with :bro:keyword:`delete`:
 
     .. code:: bro
 
@@ -416,6 +433,9 @@ The Bro scripting language supports the following built-in types.
 
         |t|
 
+    See the :bro:keyword:`for` statement for info on how to iterate over
+    the elements in a table.
+
 .. bro:type:: set
 
     A set is like a :bro:type:`table`, but it is a collection of indices
@@ -424,27 +444,26 @@ The Bro scripting language supports the following built-in types.
 
         set [ type^+ ]
 
-    where *type^+* is one or more types separated by commas.
+    where *type^+* is one or more types separated by commas.  The
+    index type cannot be any of the following types:  pattern, table, set,
+    vector, file, opaque, any.
 
-    Sets are initialized by listing elements enclosed by curly braces:
+    Sets can be initialized by listing elements enclosed by curly braces:
 
     .. code:: bro
 
         global s: set[port] = { 21/tcp, 23/tcp, 80/tcp, 443/tcp };
         global s2: set[port, string] = { [21/tcp, "ftp"], [23/tcp, "telnet"] };
 
-    The types are explicitly shown in the example above, but they could
-    have been left to type inference.
-
     A set constructor (equivalent to above example) can also be used to
     create a set:
 
     .. code:: bro
 
-        global s3: set[port] = set(21/tcp, 23/tcp, 80/tcp, 443/tcp);
+        global s3 = set(21/tcp, 23/tcp, 80/tcp, 443/tcp);
 
     Set constructors can also be explicitly named by a type, which is
-    useful for when a more complex index type could otherwise be
+    useful when a more complex index type could otherwise be
     ambiguous:
 
     .. code:: bro
@@ -465,18 +484,10 @@ The Bro scripting language supports the following built-in types.
         if ( 21/tcp in s )
             ...
 
-        if ( 21/tcp !in s )
+        if ( [21/tcp, "ftp"] !in s2 )
             ...
 
-    Iterate over a set with a ``for`` loop:
+    Elements are added with :bro:keyword:`add`:
-
-    .. code:: bro
-
-        local s: set[port];
-        for ( p in s )
-            ...
-
-    Elements are added with ``add``:
 
     .. code:: bro
 
@@ -485,7 +496,7 @@ The Bro scripting language supports the following built-in types.
     Nothing happens if the element with value ``22/tcp`` was already present in
     the set.
 
-    And removed with ``delete``:
+    And removed with :bro:keyword:`delete`:
 
     .. code:: bro
 
@@ -501,6 +512,9 @@ The Bro scripting language supports the following built-in types.
 
         |s|
 
+    See the :bro:keyword:`for` statement for info on how to iterate over
+    the elements in a set.
+
 .. bro:type:: vector
 
     A vector is like a :bro:type:`table`, except it's always indexed by a
@@ -515,7 +529,7 @@ The Bro scripting language supports the following built-in types.
 
     .. code:: bro
 
-        global v: vector of string = vector("one", "two", "three");
+        local v = vector("one", "two", "three");
 
     Vector constructors can also be explicitly named by a type, which
     is useful for when a more complex yield type could otherwise be
@@ -539,14 +553,6 @@ The Bro scripting language supports the following built-in types.
 
         print v[2];
 
-    Iterate over a vector with a ``for`` loop:
-
-    .. code:: bro
-
-        local v: vector of string;
-        for ( n in v )
-            ...
-
     An element can be added to a vector by assigning the value (a value
     that already exists at that index will be overwritten):
 
@@ -577,11 +583,17 @@ The Bro scripting language supports the following built-in types.
     The resulting vector of bool is the logical "and" (or logical "or") of
     each element of the operand vectors.
 
+    See the :bro:keyword:`for` statement for info on how to iterate over
+    the elements in a vector.
+
 .. bro:type:: record
 
-    A ``record`` is a collection of values.  Each value has a field name
+    A "record" is a collection of values.  Each value has a field name
     and a type.  Values do not need to have the same type and the types
-    have no restrictions.  An example record type definition:
+    have no restrictions.  Field names must follow the same syntax as
+    regular variable names (except that field names are allowed to be the
+    same as local or global variables).  An example record type
+    definition:
 
     .. code:: bro
 
@@ -590,85 +602,44 @@ The Bro scripting language supports the following built-in types.
             s: string &optional;
         };
 
-    Access to a record field uses the dollar sign (``$``) operator:
+    Records can be initialized or assigned as a whole in three different ways.
-
-    .. code:: bro
-
-        global r: MyRecordType;
-        r$c = 13;
-
-    Record assignment can be done field by field or as a whole like:
-
-    .. code:: bro
-
-        r = [$c = 13, $s = "thirteen"];
-
     When assigning a whole record value, all fields that are not
     :bro:attr:`&optional` or have a :bro:attr:`&default` attribute must
-    be specified.
+    be specified.  First, there's a constructor syntax:
-
-    To test for existence of a field that is :bro:attr:`&optional`, use the
-    ``?$`` operator:
 
     .. code:: bro
 
-        if ( r?$s )
+        local r: MyRecordType = record($c = 7);
-            ...
-
-    Records can also be created using a constructor syntax:
-
-    .. code:: bro
-
-        global r2: MyRecordType = record($c = 7);
 
     And the constructor can be explicitly named by type, too, which
-    is arguably more readable code:
+    is arguably more readable:
 
     .. code:: bro
 
-        global r3 = MyRecordType($c = 42);
+        local r = MyRecordType($c = 42);
 
-.. bro:type:: opaque
+    And the third way is like this:
-
-    A data type whose actual representation/implementation is
-    intentionally hidden, but whose values may be passed to certain
-    functions that can actually access the internal/hidden resources.
-    Opaque types are differentiated from each other by qualifying them
-    like ``opaque of md5`` or ``opaque of sha1``.  Any valid identifier
-    can be used as the type qualifier.
-
-    An example use of this type is the set of built-in functions which
-    perform hashing:
 
     .. code:: bro
 
-        local handle: opaque of md5 = md5_hash_init();
+        local r: MyRecordType = [$c = 13, $s = "thirteen"];
-        md5_hash_update(handle, "test");
-        md5_hash_update(handle, "testing");
-        print md5_hash_finish(handle);
 
-    Here the opaque type is used to provide a handle to a particular
+    Access to a record field uses the dollar sign (``$``) operator, and
-    resource which is calculating an MD5 checksum incrementally over
+    record fields can be assigned with this:
-    time, but the details of that resource aren't relevant, it's only
-    necessary to have a handle as a way of identifying it and
-    distinguishing it from other such resources.
-
-.. bro:type:: file
-
-    Bro supports writing to files, but not reading from them.  Files
-    can be opened using either the :bro:id:`open` or :bro:id:`open_for_append`
-    built-in functions, and closed using the :bro:id:`close` built-in
-    function. For example, declare, open, and write to a file
-    and finally close it like:
 
     .. code:: bro
 
-        global f: file = open("myfile");
+        local r: MyRecordType;
-        print f, "hello, world";
+        r$c = 13;
-        close(f);
 
-    Writing to files like this for logging usually isn't recommended, for better
+    To test if a field that is :bro:attr:`&optional` has been assigned a
-    logging support see :doc:`/frameworks/logging`.
+    value, use the ``?$`` operator (it returns a :bro:type:`bool` value of
+    ``T`` if the field has been assigned a value, or ``F`` if not):
+
+    .. code:: bro
+
+        if ( r ?$ s )
+            ...
 
 .. bro:type:: function
 
@@ -700,6 +671,16 @@ The Bro scripting language supports the following built-in types.
     type, but when it is, the return type and argument list (including the
     name of each argument) must match exactly.
 
+    Here is an example function that takes no parameters and does not
+    return a value:
+
+    .. code:: bro
+
+        function my_func()
+            {
+            print "my_func";
+            }
+
     Function types don't need to have a name and can be assigned anonymously:
 
     .. code:: bro
@@ -742,9 +723,20 @@ The Bro scripting language supports the following built-in types.
     Event handlers are nearly identical in both syntax and semantics to
     a :bro:type:`function`, with the two differences being that event
     handlers have no return type since they never return a value, and
-    you cannot call an event handler.  Instead of directly calling an
+    you cannot call an event handler.
-    event handler from a script, event handler bodies are executed when
+
-    they are invoked by one of three different methods:
+    Example:
+
+    .. code:: bro
+
+        event my_event(r: bool, s: string)
+        {
+            print "my_event", r, s;
+        }
+
+    Instead of directly calling an event handler from a script, event
+    handler bodies are executed when they are invoked by one of three
+    different methods:
 
     - From the event engine
 
@@ -765,7 +757,7 @@ The Bro scripting language supports the following built-in types.
         This assumes that ``password_exposed`` was previously declared
         as an event handler type with compatible arguments.
 
-    - Via the ``schedule`` expression in a script
+    - Via the :bro:keyword:`schedule` expression in a script
 
         This delays the invocation of event handlers until some time in
         the future.  For example:
@@ -789,8 +781,8 @@ The Bro scripting language supports the following built-in types.
     immediate and they do not get scheduled through an event queue.
     Also, a unique feature of a hook is that a given hook handler body
     can short-circuit the execution of remaining hook handlers simply by
-    exiting from the body as a result of a ``break`` statement (as
+    exiting from the body as a result of a :bro:keyword:`break` statement (as
-    opposed to a ``return`` or just reaching the end of the body).
+    opposed to a :bro:keyword:`return` or just reaching the end of the body).
 
     A hook type is declared like::
 
@@ -859,142 +851,60 @@ The Bro scripting language supports the following built-in types.
     executed due to one handler body exiting as a result of a ``break``
     statement.
 
-Attributes
+.. bro:type:: file
-----------
 
-Attributes occur at the end of type/event declarations and change their
+    Bro supports writing to files, but not reading from them (to read from
-behavior. The syntax is ``&key`` or ``&key=val``, e.g., ``type T:
+    files see the :doc:`/frameworks/input`).  Files
-set[count] &read_expire=5min`` or ``event foo() &priority=-3``.  The Bro
+    can be opened using either the :bro:id:`open` or :bro:id:`open_for_append`
-scripting language supports the following built-in attributes.
+    built-in functions, and closed using the :bro:id:`close` built-in
+    function.  For example, declare, open, and write to a file and finally
+    close it like:
 
-.. bro:attr:: &optional
+    .. code:: bro
 
-    Allows a record field to be missing. For example the type ``record {
+        local f = open("myfile");
-    a: addr; b: port &optional; }`` could be instantiated both as
+        print f, "hello, world";
-    singleton ``[$a=127.0.0.1]`` or pair ``[$a=127.0.0.1, $b=80/tcp]``.
+        close(f);
 
-.. bro:attr:: &default
+    Writing to files like this for logging usually isn't recommended, for better
+    logging support see :doc:`/frameworks/logging`.
 
-    Uses a default value for a record field, a function/hook/event
+.. bro:type:: opaque
-    parameter, or container elements.  For example, ``table[int] of
-    string &default="foo"`` would create a table that returns the
-    :bro:type:`string` ``"foo"`` for any non-existing index.
 
-.. bro:attr:: &redef
+    A data type whose actual representation/implementation is
+    intentionally hidden, but whose values may be passed to certain
+    built-in functions that can actually access the internal/hidden resources.
+    Opaque types are differentiated from each other by qualifying them
+    like "opaque of md5" or "opaque of sha1".
 
-    Allows for redefinition of initial object values. This is typically
+    An example use of this type is the set of built-in functions which
-    used with constants, for example, ``const clever = T &redef;`` would
+    perform hashing:
-    allow the constant to be redefined at some later point during script
-    execution.
 
-.. bro:attr:: &rotate_interval
+    .. code:: bro
 
-    Rotates a file after a specified interval.
+        local handle = md5_hash_init();
+        md5_hash_update(handle, "test");
+        md5_hash_update(handle, "testing");
+        print md5_hash_finish(handle);
 
-.. bro:attr:: &rotate_size
+    Here the opaque type is used to provide a handle to a particular
+    resource which is calculating an MD5 hash incrementally over
+    time, but the details of that resource aren't relevant, it's only
+    necessary to have a handle as a way of identifying it and
+    distinguishing it from other such resources.
 
-    Rotates a file after it has reached a given size in bytes.
+.. bro:type:: any
 
-.. bro:attr:: &add_func
+    Used to bypass strong typing.  For example, a function can take an
+    argument of type ``any`` when it may be of different types.
+    The only operation allowed on a variable of type ``any`` is assignment.
 
-    Can be applied to an identifier with &redef to specify a function to
+    Note that users aren't expected to use this type.  It's provided mainly
-    be called any time a "redef <id> += ..." declaration is parsed.  The
+    for use by some built-in functions and scripts included with Bro.
-    function takes two arguments of the same type as the identifier, the first
-    being the old value of the variable and the second being the new
-    value given after the "+=" operator in the "redef" declaration.  The
-    return value of the function will be the actual new value of the
-    variable after the "redef" declaration is parsed.
 
-.. bro:attr:: &delete_func
+.. bro:type:: void
 
-    Same as &add_func, except for "redef" declarations that use the "-="
+    An internal Bro type (i.e., "void" is not a reserved keyword in the Bro
-    operator.
+    scripting language) representing the absence of a return type for a
+    function.
 
-.. bro:attr:: &expire_func
-
-    Called right before a container element expires.  The function's
-    first parameter is of the same type of the container and the second
-    parameter the same type of the container's index.  The return
-    value is an :bro:type:`interval` indicating the amount of additional
-    time to wait before expiring the container element at the given
-    index (which will trigger another execution of this function).
-
-.. bro:attr:: &read_expire
-
-    Specifies a read expiration timeout for container elements. That is,
-    the element expires after the given amount of time since the last
-    time it has been read. Note that a write also counts as a read.
-
-.. bro:attr:: &write_expire
-
-    Specifies a write expiration timeout for container elements. That
-    is, the element expires after the given amount of time since the
-    last time it has been written.
-
-.. bro:attr:: &create_expire
-
-    Specifies a creation expiration timeout for container elements. That
-    is, the element expires after the given amount of time since it has
-    been inserted into the container, regardless of any reads or writes.
-
-.. bro:attr:: &persistent
-
-    Makes a variable persistent, i.e., its value is written to disk (per
-    default at shutdown time).
-
-.. bro:attr:: &synchronized
-
-    Synchronizes variable accesses across nodes. The value of a
-    ``&synchronized`` variable is automatically propagated to all peers
-    when it changes.
-
-.. bro:attr:: &encrypt
-
-    Encrypts files right before writing them to disk.
-
-.. TODO: needs to be documented in more detail.
-
-.. bro:attr:: &raw_output
-
-    Opens a file in raw mode, i.e., non-ASCII characters are not
-    escaped.
-
-.. bro:attr:: &mergeable
-
-    Prefers set union to assignment for synchronized state. This
-    attribute is used in conjunction with :bro:attr:`&synchronized`
-    container types: when the same container is updated at two peers
-    with different value, the propagation of the state causes a race
-    condition, where the last update succeeds. This can cause
-    inconsistencies and can be avoided by unifying the two sets, rather
-    than merely overwriting the old value.
-
-.. bro:attr:: &priority
-
-    Specifies the execution priority (as a signed integer) of a hook or
-    event handler. Higher values are executed before lower ones. The
-    default value is 0.
-
-.. bro:attr:: &group
-
-    Groups event handlers such that those in the same group can be
-    jointly activated or deactivated.
-
-.. bro:attr:: &log
-
-    Writes a record field to the associated log stream.
-
-.. bro:attr:: &error_handler
-
-    Internally set on the events that are associated with the reporter
-    framework: :bro:id:`reporter_info`, :bro:id:`reporter_warning`, and
-    :bro:id:`reporter_error`.  It prevents any handlers of those events
-    from being able to generate reporter messages that go through any of
-    those events (i.e., it prevents an infinite event recursion).  Instead,
-    such nested reporter messages are output to stderr.
-
-.. bro:attr:: &type_column
-
-    Used by the input framework. It can be used on columns of type
-    :bro:type:`port` and specifies the name of an additional column in
-    the input file which specifies the protocol of the port (tcp/udp/icmp).

doc/scripting/data_struct_record_01.bro

@@ -4,7 +4,7 @@ type Service: record {
     rfc: count;
 };
 
-function print_service(serv: Service): string
+function print_service(serv: Service)
     {
     print fmt("Service: %s(RFC%d)",serv$name, serv$rfc);
     

doc/scripting/data_struct_record_02.bro

@@ -9,7 +9,7 @@ type System: record {
     services: set[Service];
     };
 
-function print_service(serv: Service): string
+function print_service(serv: Service)
     {
     print fmt("  Service: %s(RFC%d)",serv$name, serv$rfc);
     
@@ -17,7 +17,7 @@ function print_service(serv: Service): string
         print fmt("    port: %s", p);
     }
 
-function print_system(sys: System): string
+function print_system(sys: System)
     {
     print fmt("System: %s", sys$name);
     

doc/scripting/data_struct_vector_declaration.bro

@@ -10,6 +10,6 @@ event bro_init()
     
     print fmt("contents of v1: %s", v1);
     print fmt("length of v1: %d", |v1|);
-    print fmt("contents of v1: %s", v2);
+    print fmt("contents of v2: %s", v2);
     print fmt("length of v2: %d", |v2|);
     }

doc/scripting/data_type_pattern_01.bro

@@ -1,6 +1,6 @@
 event bro_init()
     {
-    local test_string = "The quick brown fox jumped over the lazy dog.";
+    local test_string = "The quick brown fox jumps over the lazy dog.";
     local test_pattern = /quick|lazy/;
     
     if ( test_pattern in test_string )

doc/scripting/framework_logging_factorial_02.bro

@@ -23,7 +23,7 @@ function factorial(n: count): count
 event bro_init()
     {
     # Create the logging stream.
-    Log::create_stream(LOG, [$columns=Info]);
+    Log::create_stream(LOG, [$columns=Info, $path="factor"]);
     }
 
 event bro_done()

doc/scripting/framework_logging_factorial_03.bro

@@ -37,7 +37,7 @@ function mod5(id: Log::ID, path: string, rec: Factor::Info) : string
 
 event bro_init()
     {
-    Log::create_stream(LOG, [$columns=Info]);
+    Log::create_stream(LOG, [$columns=Info, $path="factor"]);
     
     local filter: Log::Filter = [$name="split-mod5s", $path_func=mod5];
     Log::add_filter(Factor::LOG, filter);

doc/scripting/framework_logging_factorial_04.bro

@@ -22,7 +22,7 @@ function factorial(n: count): count
 
 event bro_init()
     {
-    Log::create_stream(LOG, [$columns=Info, $ev=log_factor]);
+    Log::create_stream(LOG, [$columns=Info, $ev=log_factor, $path="factor"]);
     }
 
 event bro_done()

doc/scripting/index.rst

@@ -103,9 +103,9 @@ In the ``file_hash`` event handler, there is an ``if`` statement that is used
 to check for the correct type of hash, in this case
 a SHA1 hash.  It also checks for a mime type we've defined as
 being of interest as defined in the constant ``match_file_types``.
-The comparison is made against the expression ``f$mime_type``, which uses
+The comparison is made against the expression ``f$info$mime_type``, which uses
 the ``$`` dereference operator to check the value ``mime_type``
-inside the variable ``f``.  If the entire expression evaluates to true,
+inside the variable ``f$info``.  If the entire expression evaluates to true,
 then a helper function is called to do the rest of the work.  In that
 function, a local variable is defined to hold a string comprised of
 the SHA1 hash concatenated with ``.malware.hash.cymru.com``; this
@@ -260,7 +260,7 @@ originating host is referenced by ``c$id$orig_h`` which if given a
 narrative relates to ``orig_h`` which is a member of ``id`` which is
 a member of the data structure referred to as ``c`` that was passed
 into the event handler. Given that the responder port
-``c$id$resp_p`` is ``53/tcp``, it's likely that Bro's base HTTP scripts
+``c$id$resp_p`` is ``80/tcp``, it's likely that Bro's base HTTP scripts
 can further populate the connection record.  Let's load the
 ``base/protocols/http`` scripts and check the output of our script. 
 
@@ -363,7 +363,7 @@ decrypted from HTTP streams is stored in
 excerpt from :doc:`/scripts/base/protocols/http/main.bro` below.
 
 .. btest-include:: ${BRO_SRC_ROOT}/scripts/base/protocols/http/main.bro
-   :lines: 9-11,20-22,121
+   :lines: 9-11,20-22,125
 
 Because the constant was declared with the ``&redef`` attribute, if we
 needed to turn this option on globally, we could do so by adding the
@@ -826,7 +826,7 @@ example of the ``record`` data type in the earlier sections, the
 ``conn.log``, is shown by the excerpt below.
 
 .. btest-include:: ${BRO_SRC_ROOT}/scripts/base/protocols/conn/main.bro
-   :lines: 10-12,16-17,19,21,23,25,28,31,35,38,57,63,69,92,95,99,102,106,110-111,116
+   :lines: 10-12,16-17,19,21,23,25,28,31,35,38,57,63,69,75,98,101,105,108,112,116-117,122
 
 Looking at the structure of the definition, a new collection of data
 types is being defined as a type called ``Info``.  Since this type

man/CMakeLists.txt

@@ -0,0 +1,5 @@
+
+install(DIRECTORY . DESTINATION ${BRO_MAN_INSTALL_PATH}/man8 FILES_MATCHING
+        PATTERN "*.8"
+)
+

man/bro.8

@@ -0,0 +1,165 @@
+.TH BRO "8" "November 2014" "bro" "System Administration Utilities"
+.SH NAME
+bro \- passive network traffic analyzer
+.SH SYNOPSIS
+.B bro
+\/\fP [\fIoptions\fR] [\fIfile\fR ...]
+.SH DESCRIPTION
+Bro is primarily a security monitor that inspects all traffic on a link in
+depth for signs of suspicious activity. More generally, however, Bro
+supports a wide range of traffic analysis tasks even outside of the
+security domain, including performance measurements and helping with
+trouble-shooting.
+
+Bro comes with built-in functionality for a range of analysis and detection
+tasks, including detecting malware by interfacing to external registries,
+reporting vulnerable versions of software seen on the network, identifying
+popular web applications, detecting SSH brute-forcing, validating SSL
+certificate chains, among others.
+.SH OPTIONS
+.TP
+.B <file>
+policy file, or read stdin
+.TP
+\fB\-a\fR,\ \-\-parse\-only
+exit immediately after parsing scripts
+.TP
+\fB\-b\fR,\ \-\-bare\-mode
+don't load scripts from the base/ directory
+.TP
+\fB\-d\fR,\ \-\-debug\-policy
+activate policy file debugging
+.TP
+\fB\-e\fR,\ \-\-exec <bro code>
+augment loaded policies by given code
+.TP
+\fB\-f\fR,\ \-\-filter <filter>
+tcpdump filter
+.TP
+\fB\-g\fR,\ \-\-dump\-config
+dump current config into .state dir
+.TP
+\fB\-h\fR,\ \-\-help|\-?
+command line help
+.TP
+\fB\-i\fR,\ \-\-iface <interface>
+read from given interface
+.TP
+\fB\-p\fR,\ \-\-prefix <prefix>
+add given prefix to policy file resolution
+.TP
+\fB\-r\fR,\ \-\-readfile <readfile>
+read from given tcpdump file
+.TP
+\fB\-s\fR,\ \-\-rulefile <rulefile>
+read rules from given file
+.TP
+\fB\-t\fR,\ \-\-tracefile <tracefile>
+activate execution tracing
+.TP
+\fB\-w\fR,\ \-\-writefile <writefile>
+write to given tcpdump file
+.TP
+\fB\-v\fR,\ \-\-version
+print version and exit
+.TP
+\fB\-x\fR,\ \-\-print\-state <file.bst>
+print contents of state file
+.TP
+\fB\-z\fR,\ \-\-analyze <analysis>
+run the specified policy file analysis
+.TP
+\fB\-C\fR,\ \-\-no\-checksums
+ignore checksums
+.TP
+\fB\-F\fR,\ \-\-force\-dns
+force DNS
+.TP
+\fB\-I\fR,\ \-\-print\-id <ID name>
+print out given ID
+.TP
+\fB\-J\fR,\ \-\-set\-seed <seed>
+set the random number seed
+.TP
+\fB\-K\fR,\ \-\-md5\-hashkey <hashkey>
+set key for MD5\-keyed hashing
+.TP
+\fB\-N\fR,\ \-\-print\-plugins
+print available plugins and exit (\fB\-NN\fR for verbose)
+.TP
+\fB\-P\fR,\ \-\-prime\-dns
+prime DNS
+.TP
+\fB\-Q\fR,\ \-\-time
+print execution time summary to stderr
+.TP
+\fB\-R\fR,\ \-\-replay <events.bst>
+replay events
+.TP
+\fB\-S\fR,\ \-\-debug\-rules
+enable rule debugging
+.TP
+\fB\-T\fR,\ \-\-re\-level <level>
+set 'RE_level' for rules
+.TP
+\fB\-U\fR,\ \-\-status\-file <file>
+Record process status in file
+.TP
+\fB\-W\fR,\ \-\-watchdog
+activate watchdog timer
+.TP
+\fB\-X\fR,\ \-\-broxygen <cfgfile>
+generate documentation based on config file
+.TP
+\fB\-\-pseudo\-realtime[=\fR<speedup>]
+enable pseudo\-realtime for performance evaluation (default 1)
+.TP
+\fB\-\-load\-seeds\fR <file>
+load seeds from given file
+.TP
+\fB\-\-save\-seeds\fR <file>
+save seeds to given file
+.TP
+The following option is available only when Bro is built with the \-\-enable\-debug configure option:
+.TP
+\fB\-B\fR,\ \-\-debug <dbgstreams>
+Enable debugging output for selected streams ('-B help' for help)
+.TP
+The following options are available only when Bro is built with gperftools support (use the \-\-enable\-perftools and \-\-enable\-perftools\-debug configure options):
+.TP
+\fB\-m\fR,\ \-\-mem-leaks
+show leaks
+.TP
+\fB\-M\fR,\ \-\-mem-profile
+record heap
+.SH ENVIRONMENT
+.TP
+.B BROPATH
+file search path
+.TP
+.B BRO_PLUGIN_PATH
+plugin search path
+.TP
+.B BRO_PLUGIN_ACTIVATE
+plugins to always activate
+.TP
+.B BRO_PREFIXES
+prefix list
+.TP
+.B BRO_DNS_FAKE
+disable DNS lookups
+.TP
+.B BRO_SEED_FILE
+file to load seeds from
+.TP
+.B BRO_LOG_SUFFIX
+ASCII log file extension
+.TP
+.B BRO_PROFILER_FILE
+Output file for script execution statistics
+.TP
+.B BRO_DISABLE_BROXYGEN
+Disable Broxygen documentation support
+.SH AUTHOR
+.B bro
+was written by The Bro Project <info@bro.org>.

pkg/check-cmake

@@ -1,14 +0,0 @@
-#!/bin/sh
-
-# CMake/CPack versions before 2.8.3 have bugs that can create bad packages
-# Since packages will be built on several different systems, a single
-# version of CMake is required to obtain consistency, but can be increased
-# as new versions of CMake come out that also produce working packages.
-
-CMAKE_PACK_REQ="cmake version 2.8.6"
-CMAKE_VER=`cmake -version`
-
-if [ "${CMAKE_VER}" != "${CMAKE_PACK_REQ}" ]; then
-    echo "Package creation requires ${CMAKE_PACK_REQ}" >&2
-    exit 1
-fi

pkg/make-deb-packages

@@ -3,8 +3,6 @@
 # This script generates binary DEB packages.
 # They can be found in ../build/ after running.
 
-./check-cmake || { exit 1; }
-
 # The DEB CPack generator depends on `dpkg-shlibdeps` to automatically
 # determine what dependencies to set for the packages
 type dpkg-shlibdeps > /dev/null 2>&1 || {
@@ -16,6 +14,7 @@ the 'dpkg-dev' package, please install it first.
 }
 
 prefix=/opt/bro
+localstatedir=/var/opt/bro
 
 # During the packaging process, `dpkg-shlibs` will fail if used on a library
 # that links to other internal/project libraries unless an RPATH is used or
@@ -31,7 +30,7 @@ cd ..
 ( cd build && make package )
 
 # Full Bro package
-./configure --prefix=${prefix} --pkg-name-prefix=Bro --binary-package
+./configure --prefix=${prefix} --localstatedir=${localstatedir} --pkg-name-prefix=Bro --binary-package
 ( cd build && make package )
 
 # Broccoli
@@ -42,6 +41,6 @@ cd ../..
 
 # Broctl
 cd aux/broctl
-./configure --prefix=${prefix} --binary-package
+./configure --prefix=${prefix} --localstatedir=${localstatedir} --binary-package
 ( cd build && make package && mv *.deb ../../../build/ )
 cd ../..

pkg/make-mac-packages

@@ -3,14 +3,6 @@
 # This script creates binary packages for Mac OS X.
 # They can be found in ../build/ after running.
 
-cmake -P /dev/stdin << "EOF"
-if ( ${CMAKE_VERSION} VERSION_LESS 2.8.9 )
-    message(FATAL_ERROR "CMake >= 2.8.9 required to build package")
-endif ()
-EOF
-
-[ $? -ne 0 ] && exit 1;
-
 type sw_vers > /dev/null 2>&1 || {
     echo "Unable to get Mac OS X version" >&2;
     exit 1;

pkg/make-rpm-packages

@@ -3,8 +3,6 @@
 # This script generates binary RPM packages.
 # They can be found in ../build/ after running.
 
-./check-cmake || { exit 1; }
-
 # The RPM CPack generator depends on `rpmbuild` to create packages
 type rpmbuild > /dev/null 2>&1 || {
     echo "\
@@ -15,6 +13,7 @@ the 'rpm-build' package, please install it first.
 }
 
 prefix=/opt/bro
+localstatedir=/var/opt/bro
 
 cd ..
 
@@ -24,7 +23,7 @@ cd ..
 ( cd build && make package )
 
 # Full Bro package
-./configure --prefix=${prefix} --pkg-name-prefix=Bro --binary-package
+./configure --prefix=${prefix} --localstatedir=${localstatedir} --pkg-name-prefix=Bro --binary-package
 ( cd build && make package )
 
 # Broccoli
@@ -35,6 +34,6 @@ cd ../..
 
 # Broctl
 cd aux/broctl
-./configure --prefix=${prefix} --binary-package
+./configure --prefix=${prefix} --localstatedir=${localstatedir} --binary-package
 ( cd build && make package && mv *.rpm ../../../build/ )
 cd ../..

scripts/base/files/extract/main.bro

@@ -53,7 +53,8 @@ function set_limit(f: fa_file, args: Files::AnalyzerArgs, n: count): bool
 function on_add(f: fa_file, args: Files::AnalyzerArgs)
 	{
 	if ( ! args?$extract_filename )
-		args$extract_filename = cat("extract-", f$source, "-", f$id);
+		args$extract_filename = cat("extract-", f$last_active, "-", f$source,
+		                            "-", f$id);
 
 	f$info$extracted = args$extract_filename;
 	args$extract_filename = build_path_compressed(prefix, args$extract_filename);

scripts/base/files/pe/README

@@ -0,0 +1 @@
+Support for Portable Executable (PE) file analysis.

scripts/base/files/pe/__load__.bro

@@ -0,0 +1,2 @@
+@load ./consts
+@load ./main

scripts/base/files/pe/consts.bro

@@ -0,0 +1,184 @@
+
+module PE;
+
+export {
+	const machine_types: table[count] of string = {
+		[0x00]   = "UNKNOWN",
+		[0x1d3]  = "AM33",
+		[0x8664] = "AMD64",
+		[0x1c0]  = "ARM",
+		[0x1c4]  = "ARMNT",
+		[0xaa64] = "ARM64",
+		[0xebc]  = "EBC",
+		[0x14c]  = "I386",
+		[0x200]  = "IA64",
+		[0x9041] = "M32R",
+		[0x266]  = "MIPS16",
+		[0x366]  = "MIPSFPU",
+		[0x466]  = "MIPSFPU16",
+		[0x1f0]  = "POWERPC",
+		[0x1f1]  = "POWERPCFP",
+		[0x166]  = "R4000",
+		[0x1a2]  = "SH3",
+		[0x1a3]  = "SH3DSP",
+		[0x1a6]  = "SH4",
+		[0x1a8]  = "SH5",
+		[0x1c2]  = "THUMB",
+		[0x169]  = "WCEMIPSV2"
+	} &default=function(i: count):string { return fmt("unknown-%d", i); };
+
+	const file_characteristics: table[count] of string = {
+		[0x1]    = "RELOCS_STRIPPED",
+		[0x2]    = "EXECUTABLE_IMAGE",
+		[0x4]    = "LINE_NUMS_STRIPPED",
+		[0x8]    = "LOCAL_SYMS_STRIPPED",
+		[0x10]   = "AGGRESSIVE_WS_TRIM",
+		[0x20]   = "LARGE_ADDRESS_AWARE",
+		[0x80]   = "BYTES_REVERSED_LO",
+		[0x100]  = "32BIT_MACHINE",
+		[0x200]  = "DEBUG_STRIPPED",
+		[0x400]  = "REMOVABLE_RUN_FROM_SWAP",
+		[0x800]  = "NET_RUN_FROM_SWAP",
+		[0x1000] = "SYSTEM",
+		[0x2000] = "DLL",
+		[0x4000] = "UP_SYSTEM_ONLY",
+		[0x8000] = "BYTES_REVERSED_HI"
+	} &default=function(i: count):string { return fmt("unknown-%d", i); };
+
+	const dll_characteristics: table[count] of string = {
+		[0x40]   = "DYNAMIC_BASE",
+		[0x80]   = "FORCE_INTEGRITY",
+		[0x100]  = "NX_COMPAT",
+		[0x200]  = "NO_ISOLATION",
+		[0x400]  = "NO_SEH",
+		[0x800]  = "NO_BIND",
+		[0x2000] = "WDM_DRIVER",
+		[0x8000] = "TERMINAL_SERVER_AWARE"
+	} &default=function(i: count):string { return fmt("unknown-%d", i); };
+
+	const windows_subsystems: table[count] of string = {
+		[0]  = "UNKNOWN",
+		[1]  = "NATIVE",
+		[2]  = "WINDOWS_GUI",
+		[3]  = "WINDOWS_CUI",
+		[7]  = "POSIX_CUI",
+		[9]  = "WINDOWS_CE_GUI",
+		[10] = "EFI_APPLICATION",
+		[11] = "EFI_BOOT_SERVICE_DRIVER",
+		[12] = "EFI_RUNTIME_
DRIVER",
+		[13] = "EFI_ROM",
+		[14] = "XBOX"
+	} &default=function(i: count):string { return fmt("unknown-%d", i); };
+
+	const directories: table[count] of string = {
+		[0]  = "Export Table",
+		[1]  = "Import Table",
+		[2]  = "Resource Table",
+		[3]  = "Exception Table",
+		[4]  = "Certificate Table",
+		[5]  = "Base Relocation Table",
+		[6]  = "Debug",
+		[7]  = "Architecture",
+		[8]  = "Global Ptr",
+		[9]  = "TLS Table",
+		[10] = "Load Config Table",
+		[11] = "Bound Import",
+		[12] = "IAT",
+		[13] = "Delay Import Descriptor",
+		[14] = "CLR Runtime Header",
+		[15] = "Reserved"
+	} &default=function(i: count):string { return fmt("unknown-%d", i); };
+
+	const section_characteristics: table[count] of string = {
+		[0x8]        = "TYPE_NO_PAD",
+		[0x20]       = "CNT_CODE",
+		[0x40]       = "CNT_INITIALIZED_DATA",
+		[0x80]       = "CNT_UNINITIALIZED_DATA",
+		[0x100]      = "LNK_OTHER",
+		[0x200]      = "LNK_INFO",
+		[0x800]      = "LNK_REMOVE",
+		[0x1000]     = "LNK_COMDAT",
+		[0x8000]     = "GPREL",
+		[0x20000]    = "MEM_16BIT",
+		[0x40000]    = "MEM_LOCKED",
+		[0x80000]    = "MEM_PRELOAD",
+		[0x100000]   = "ALIGN_1BYTES",
+		[0x200000]   = "ALIGN_2BYTES",
+		[0x300000]   = "ALIGN_4BYTES",
+		[0x400000]   = "ALIGN_8BYTES",
+		[0x500000]   = "ALIGN_16BYTES",
+		[0x600000]   = "ALIGN_32BYTES",
+		[0x700000]   = "ALIGN_64BYTES",
+		[0x800000]   = "ALIGN_128BYTES",
+		[0x900000]   = "ALIGN_256BYTES",
+		[0xa00000]   = "ALIGN_512BYTES",
+		[0xb00000]   = "ALIGN_1024BYTES",
+		[0xc00000]   = "ALIGN_2048BYTES",
+		[0xd00000]   = "ALIGN_4096BYTES",
+		[0xe00000]   = "ALIGN_8192BYTES",
+		[0x1000000]  = "LNK_NRELOC_OVFL",
+		[0x2000000]  = "MEM_DISCARDABLE",
+		[0x4000000]  = "MEM_NOT_CACHED",
+		[0x8000000]  = "MEM_NOT_PAGED",
+		[0x10000000] = "MEM_SHARED",
+		[0x20000000] = "MEM_EXECUTE",
+		[0x40000000] = "MEM_READ",
+		[0x80000000] = "MEM_WRITE"
+	} &default=function(i: count):string { return fmt("unknown-%d", i); };
+
+	const os_versions: table[count, count] of string = {
+		[10,0] = "Windows 10",
+		[6,4]  = "Windows 10 Technical Preview",
+		[6,3]  = "Windows 8.1 or Server 2012 R2",
+		[6,2]  = "Windows 8 or Server 2012",
+		[6,1]  = "Windows 7 or Server 2008 R2",
+		[6,0]  = "Windows Vista or Server 2008",
+		[5,2]  = "Windows XP x64 or Server 2003",
+		[5,1]  = "Windows XP",
+		[5,0]  = "Windows 2000",
+		[4,90] = "Windows Me",
+		[4,10] = "Windows 98",
+		[4,0]  = "Windows 95 or NT 4.0",
+		[3,51] = "Windows NT 3.51",
+		[3,50] = "Windows NT 3.5",
+		[3,2]  = "Windows 3.2",
+		[3,11] = "Windows for Workgroups 3.11",
+		[3,10] = "Windows 3.1 or NT 3.1",
+		[3,0]  = "Windows 3.0",
+		[2,11] = "Windows 2.11",
+		[2,10] = "Windows 2.10",
+		[2,0] = "Windows 2.0",
+		[1,4] = "Windows 1.04",
+		[1,3] = "Windows 1.03",
+		[1,1] = "Windows 1.01",
+		[1,0] = "Windows 1.0",
+	} &default=function(i: count, j: count):string { return fmt("unknown-%d.%d", i, j); };
+
+	const section_descs: table[string] of string = {
+		[".bss"]      = "Uninitialized data",
+		[".cormeta"]  = "CLR metadata that indicates that the object file contains managed code",
+		[".data"]     = "Initialized data",
+		[".debug$F"]  = "Generated FPO debug information",
+		[".debug$P"]  = "Precompiled debug types",
+		[".debug$S"]  = "Debug symbols",
+		[".debug$T"]  = "Debug types",
+		[".drective"] = "Linker options",
+		[".edata"]    = "Export tables",
+		[".idata"]    = "Import tables",
+		[".idlsym"]   = "Includes registered SEH to support IDL attributes",
+		[".pdata"]    = "Exception information",
+		[".rdata"]    = "Read-only initialized data",
+		[".reloc"]    = "Image relocations",
+		[".rsrc"]     = "Resource directory",
+		[".sbss"]     = "GP-relative uninitialized data",
+		[".sdata"]    = "GP-relative initialized data",
+		[".srdata"]   = "GP-relative read-only data",
+		[".sxdata"]   = "Registered exception handler data",
+		[".text"]     = "Executable code",
+		[".tls"]      = "Thread-local storage",
+		[".tls$"]     = "Thread-local storage",
+		[".vsdata"]   = "GP-relative initialized data",
+		[".xdata"]    = "Exception information",
+	} &default=function(i: string):string { return fmt("unknown-%s", i); };
+
+}

scripts/base/files/pe/main.bro

@@ -0,0 +1,137 @@
+module PE;
+
+@load ./consts.bro
+
+export {
+	redef enum Log::ID += { LOG };
+
+	type Info: record {
+		## Current timestamp.
+		ts:                  time              &log;
+		## File id of this portable executable file.
+		id:                  string            &log;
+		## The target machine that the file was compiled for.
+		machine:             string            &log &optional;
+		## The time that the file was created at.
+		compile_ts:          time              &log &optional;
+		## The required operating system.
+		os:                  string            &log &optional;
+		## The subsystem that is required to run this file.
+		subsystem:           string            &log &optional;
+		## Is the file an executable, or just an object file?
+		is_exe:              bool              &log &default=T;
+		## Is the file a 64-bit executable?
+		is_64bit:            bool              &log &default=T;
+		## Does the file support Address Space Layout Randomization?
+		uses_aslr:           bool              &log &default=F;
+		## Does the file support Data Execution Prevention?
+		uses_dep:            bool              &log &default=F;
+		## Does the file enforce code integrity checks?
+		uses_code_integrity: bool              &log &default=F;
+		## Does the file use structured exception handing?
+		uses_seh:            bool              &log &default=T;
+		## Does the file have an import table?
+		has_import_table:    bool              &log &optional;
+		## Does the file have an export table?
+		has_export_table:    bool              &log &optional;
+		## Does the file have an attribute certificate table?
+		has_cert_table:      bool              &log &optional;
+		## Does the file have a debug table?
+		has_debug_data:      bool              &log &optional;
+		## The names of the sections, in order.
+		section_names:       vector of string  &log &optional;
+	};
+
+	## Event for accessing logged records.
+	global log_pe: event(rec: Info);
+
+	## A hook that gets called when we first see a PE file.
+	global set_file: hook(f: fa_file);
+}
+
+redef record fa_file += {
+	pe: Info &optional;
+};
+
+const pe_mime_types = { "application/x-dosexec" };
+
+event bro_init() &priority=5
+	{
+	Files::register_for_mime_types(Files::ANALYZER_PE, pe_mime_types);
+	Log::create_stream(LOG, [$columns=Info, $ev=log_pe, $path="pe"]);
+	}
+
+hook set_file(f: fa_file) &priority=5
+	{
+	if ( ! f?$pe )
+		f$pe = [$ts=network_time(), $id=f$id];
+	}
+
+event pe_dos_header(f: fa_file, h: PE::DOSHeader) &priority=5
+	{
+	hook set_file(f);
+	}
+
+event pe_file_header(f: fa_file, h: PE::FileHeader) &priority=5
+	{
+	hook set_file(f);
+
+	f$pe$machine    = machine_types[h$machine];
+	f$pe$compile_ts = h$ts;
+	f$pe$is_exe     = ( h$optional_header_size > 0 );
+
+	for ( c in h$characteristics )
+		{
+		if ( file_characteristics[c] == "32BIT_MACHINE" )
+			f$pe$is_64bit = F;
+		}
+	}
+
+event pe_optional_header(f: fa_file, h: PE::OptionalHeader) &priority=5
+	{
+	hook set_file(f);
+
+	# Only EXEs have optional headers
+	if ( ! f$pe$is_exe )
+		return;
+
+	f$pe$os        = os_versions[h$os_version_major, h$os_version_minor];
+	f$pe$subsystem = windows_subsystems[h$subsystem];
+
+	for ( c in h$dll_characteristics )
+		{
+		if ( dll_characteristics[c] == "DYNAMIC_BASE" )
+			f$pe$uses_aslr = T;
+		if ( dll_characteristics[c] == "FORCE_INTEGRITY" )
+			f$pe$uses_code_integrity = T;
+		if ( dll_characteristics[c] == "NX_COMPAT" )
+			f$pe$uses_dep = T;
+		if ( dll_characteristics[c] == "NO_SEH" )
+			f$pe$uses_seh = F;
+		}
+
+	f$pe$has_export_table = (|h$table_sizes| > 0 && h$table_sizes[0] > 0);
+	f$pe$has_import_table = (|h$table_sizes| > 1 && h$table_sizes[1] > 0);
+	f$pe$has_cert_table   = (|h$table_sizes| > 4 && h$table_sizes[4] > 0);
+	f$pe$has_debug_data   = (|h$table_sizes| > 6 && h$table_sizes[6] > 0);
+	}
+
+event pe_section_header(f: fa_file, h: PE::SectionHeader) &priority=5
+	{
+	hook set_file(f);
+
+	# Only EXEs have section headers
+	if ( ! f$pe$is_exe )
+		return;
+
+	if ( ! f$pe?$section_names )
+		f$pe$section_names = vector();
+	f$pe$section_names[|f$pe$section_names|] = h$name;
+	}
+
+event file_state_remove(f: fa_file) &priority=-5
+	{
+	if ( f?$pe && f$pe?$machine )
+		Log::write(LOG, f$pe);
+	}
+

scripts/base/files/unified2/main.bro

@@ -71,11 +71,50 @@ global classification_map: table[count] of string;
 global sid_map: table[count] of string;
 global gen_map: table[count] of string;
 
+global num_classification_map_reads = 0;
+global num_sid_map_reads = 0;
+global num_gen_map_reads = 0;
+global watching = F;
+
 # For reading in config files.
 type OneLine: record {
 	line: string;
 };
 
+function mappings_initialized(): bool
+	{
+	return num_classification_map_reads > 0 &&
+	       num_sid_map_reads > 0 &&
+	       num_gen_map_reads > 0;
+	}
+
+function start_watching()
+	{
+	if ( watching )
+		return;
+
+	watching = T;
+
+	if ( watch_dir != "" )
+		{
+		Dir::monitor(watch_dir, function(fname: string)
+			{
+			Input::add_analysis([$source=fname,
+			                     $reader=Input::READER_BINARY,
+			                     $mode=Input::STREAM,
+			                     $name=fname]);
+			}, 10secs);
+		}
+
+	if ( watch_file != "" )
+		{
+		Input::add_analysis([$source=watch_file,
+		                     $reader=Input::READER_BINARY,
+		                     $mode=Input::STREAM,
+		                     $name=watch_file]);
+		}
+	}
+
 function create_info(ev: IDSEvent): Info
 	{
 	local info = Info($ts=ev$ts,
@@ -113,34 +152,56 @@ redef record fa_file += {
 
 event Unified2::read_sid_msg_line(desc: Input::EventDescription, tpe: Input::Event, line: string)
 	{
-	local parts = split_n(line, / \|\| /, F, 100);
+	local parts = split_string_n(line, / \|\| /, F, 100);
-	if ( |parts| >= 2 && /^[0-9]+$/ in parts[1] )
+	if ( |parts| >= 2 && /^[0-9]+$/ in parts[0] )
-		sid_map[to_count(parts[1])] = parts[2];
+		sid_map[to_count(parts[0])] = parts[1];
 	}
 
 event Unified2::read_gen_msg_line(desc: Input::EventDescription, tpe: Input::Event, line: string)
 	{
-	local parts = split_n(line, / \|\| /, F, 3);
+	local parts = split_string_n(line, / \|\| /, F, 3);
-	if ( |parts| >= 2 && /^[0-9]+$/ in parts[1] )
+	if ( |parts| >= 2 && /^[0-9]+$/ in parts[0] )
-		gen_map[to_count(parts[1])] = parts[3];
+		gen_map[to_count(parts[0])] = parts[2];
 	}
 
 event Unified2::read_classification_line(desc: Input::EventDescription, tpe: Input::Event, line: string)
 	{
-	local parts = split_n(line, /: /, F, 2);
+	local parts = split_string_n(line, /: /, F, 2);
 	if ( |parts| == 2 )
 		{
-		local parts2 = split_n(parts[2], /,/, F, 4);
+		local parts2 = split_string_n(parts[1], /,/, F, 4);
 		if ( |parts2| > 1 )
-			classification_map[|classification_map|+1] = parts2[1];
+			classification_map[|classification_map|+1] = parts2[0];
 		}
 	}
 
+event Input::end_of_data(name: string, source: string)
+	{
+	if ( name == classification_config )
+		++num_classification_map_reads;
+	else if ( name == sid_msg )
+		++num_sid_map_reads;
+	else if ( name == gen_msg )
+		++num_gen_map_reads;
+	else
+		return;
+
+	if ( watching )
+		return;
+
+	if ( mappings_initialized() )
+		start_watching();
+	}
+
 event bro_init() &priority=5
 	{
-	Log::create_stream(Unified2::LOG, [$columns=Info, $ev=log_unified2]);
+	Log::create_stream(Unified2::LOG, [$columns=Info, $ev=log_unified2, $path="unified2"]);
 
-	if ( sid_msg != "" )
+	if ( sid_msg == "" )
+		{
+		num_sid_map_reads = 1;
+		}
+	else
 		{
 		Input::add_event([$source=sid_msg,
 		                  $reader=Input::READER_RAW,
@@ -151,7 +212,11 @@ event bro_init() &priority=5
 		                  $ev=Unified2::read_sid_msg_line]);
 		}
 
-	if ( gen_msg != "" )
+	if ( gen_msg == "" )
+		{
+		num_gen_map_reads = 1;
+		}
+	else
 		{
 		Input::add_event([$source=gen_msg,
 		                  $name=gen_msg,
@@ -162,7 +227,11 @@ event bro_init() &priority=5
 		                  $ev=Unified2::read_gen_msg_line]);
 		}
 
-	if ( classification_config != "" )
+	if ( classification_config == "" )
+		{
+		num_classification_map_reads = 1;
+		}
+	else
 		{
 		Input::add_event([$source=classification_config,
 		                  $name=classification_config,
@@ -173,32 +242,16 @@ event bro_init() &priority=5
 		                  $ev=Unified2::read_classification_line]);
 		}
 
-	if ( watch_dir != "" )
+	if ( mappings_initialized() )
-		{
+		start_watching();
-		Dir::monitor(watch_dir, function(fname: string)
-			{
-			Input::add_analysis([$source=fname,
-			                     $reader=Input::READER_BINARY,
-			                     $mode=Input::STREAM,
-			                     $name=fname]);
-			}, 10secs);
-		}
-
-	if ( watch_file != "" )
-		{
-		Input::add_analysis([$source=watch_file,
-		                     $reader=Input::READER_BINARY,
-		                     $mode=Input::STREAM,
-		                     $name=watch_file]);
-		}
 	}
 
 event file_new(f: fa_file)
 	{
 	local file_dir = "";
-	local parts = split_all(f$source, /\/[^\/]*$/);
+	local parts = split_string_all(f$source, /\/[^\/]*$/);
 	if ( |parts| == 3 )
-		file_dir = parts[1];
+		file_dir = parts[0];
 
 	if ( (watch_file != "" && f$source == watch_file) || 
 	     (watch_dir != "" && compress_path(watch_dir) == file_dir) )

scripts/base/files/x509/main.bro

@@ -36,7 +36,7 @@ export {
 
 event bro_init() &priority=5
 	{
-	Log::create_stream(X509::LOG, [$columns=Info, $ev=log_x509]);
+	Log::create_stream(X509::LOG, [$columns=Info, $ev=log_x509, $path="x509"]);
 	}
 
 redef record Files::Info += {
@@ -47,6 +47,9 @@ redef record Files::Info += {
 
 event x509_certificate(f: fa_file, cert_ref: opaque of x509, cert: X509::Certificate) &priority=5
 	{
+	if ( ! f$info?$mime_type )
+		f$info$mime_type = "application/pkix-cert";
+
 	f$info$x509 = [$ts=f$info$ts, $id=f$id, $certificate=cert, $handle=cert_ref];
 	}
 

scripts/base/frameworks/broker/README

@@ -0,0 +1,2 @@
+The Broker communication framework facilitates connecting to remote Bro
+instances to share state and transfer events.

scripts/base/frameworks/broker/__load__.bro

@@ -0,0 +1 @@
+@load ./main

scripts/base/frameworks/broker/main.bro

@@ -0,0 +1,103 @@
+##! Various data structure definitions for use with Bro's communication system.
+
+module BrokerComm;
+
+export {
+
+	## A name used to identify this endpoint to peers.
+	## .. bro:see:: BrokerComm::connect BrokerComm::listen
+	const endpoint_name = "" &redef;
+
+	## Change communication behavior.
+	type EndpointFlags: record {
+		## Whether to restrict message topics that can be published to peers.
+		auto_publish: bool &default = T;
+		## Whether to restrict what message topics or data store identifiers
+		## the local endpoint advertises to peers (e.g. subscribing to
+		## events or making a master data store available).
+		auto_advertise: bool &default = T;
+	};
+
+	## Fine-grained tuning of communication behavior for a particular message.
+	type SendFlags: record {
+		## Send the message to the local endpoint.
+		self: bool &default = F;
+		## Send the message to peer endpoints that advertise interest in
+		## the topic associated with the message.
+		peers: bool &default = T;
+		## Send the message to peer endpoints even if they don't advertise
+		## interest in the topic associated with the message.
+		unsolicited: bool &default = F;
+	};
+
+	## Opaque communication data.
+	type Data: record {
+		d: opaque of BrokerComm::Data &optional;
+	};
+
+	## Opaque communication data.
+	type DataVector: vector of BrokerComm::Data;
+
+	## Opaque event communication data.
+	type EventArgs: record {
+		## The name of the event.  Not set if invalid event or arguments.
+		name: string &optional;
+		## The arguments to the event.
+		args: DataVector;
+	};
+
+	## Opaque communication data used as a convenient way to wrap key-value
+	## pairs that comprise table entries.
+	type TableItem : record {
+		key: BrokerComm::Data;
+		val: BrokerComm::Data;
+	};
+}
+
+module BrokerStore;
+
+export {
+
+	## Whether a data store query could be completed or not.
+	type QueryStatus: enum {
+		SUCCESS,
+		FAILURE,
+	};
+
+	## An expiry time for a key-value pair inserted in to a data store.
+	type ExpiryTime: record {
+		## Absolute point in time at which to expire the entry.
+		absolute: time &optional;
+		## A point in time relative to the last modification time at which
+		## to expire the entry.  New modifications will delay the expiration.
+		since_last_modification: interval &optional;
+	};
+
+	## The result of a data store query.
+	type QueryResult: record {
+		## Whether the query completed or not.
+		status: BrokerStore::QueryStatus;
+		## The result of the query.  Certain queries may use a particular
+		## data type (e.g. querying store size always returns a count, but
+		## a lookup may return various data types).
+		result: BrokerComm::Data;
+	};
+
+	## Options to tune the SQLite storage backend.
+	type SQLiteOptions: record {
+		## File system path of the database.
+		path: string &default = "store.sqlite";
+	};
+
+	## Options to tune the RocksDB storage backend.
+	type RocksDBOptions: record {
+		## File system path of the database.
+		path: string &default = "store.rocksdb";
+	};
+
+	## Options to tune the particular storage backends.
+	type BackendOptions: record {
+		sqlite: SQLiteOptions &default = SQLiteOptions();
+		rocksdb: RocksDBOptions &default = RocksDBOptions();
+	};
+}

scripts/base/frameworks/cluster/main.bro

@@ -126,6 +126,9 @@ export {
 	## This is usually supplied on the command line for each instance
 	## of the cluster that is started up.
 	const node = getenv("CLUSTER_NODE") &redef;
+
+	## Interval for retrying failed connections between cluster nodes.
+	const retry_interval = 1min &redef;
 }
 
 function is_enabled(): bool
@@ -159,5 +162,5 @@ event bro_init() &priority=5
 		terminate();
 		}
 
-	Log::create_stream(Cluster::LOG, [$columns=Info]);
+	Log::create_stream(Cluster::LOG, [$columns=Info, $path="cluster"]);
 	}

scripts/base/frameworks/cluster/setup-connections.bro

@@ -39,7 +39,7 @@ event bro_init() &priority=9
 				Communication::nodes["time-machine"] = [$host=nodes[i]$ip,
 				                                        $zone_id=nodes[i]$zone_id,
 				                                        $p=nodes[i]$p,
-				                                        $connect=T, $retry=1min,
+				                                        $connect=T, $retry=retry_interval,
 				                                        $events=tm2manager_events];
 			}
 
@@ -58,7 +58,7 @@ event bro_init() &priority=9
 				if ( n?$proxy )
 					Communication::nodes[i]
 					     = [$host=n$ip, $zone_id=n$zone_id, $p=n$p,
-					        $connect=T, $auth=F, $sync=T, $retry=1mins];
+					        $connect=T, $auth=F, $sync=T, $retry=retry_interval];
 				else if ( me?$proxy && me$proxy == i )
 					Communication::nodes[me$proxy]
 					     = [$host=nodes[i]$ip, $zone_id=nodes[i]$zone_id,
@@ -70,7 +70,7 @@ event bro_init() &priority=9
 				Communication::nodes["manager"] = [$host=nodes[i]$ip,
 				                                   $zone_id=nodes[i]$zone_id,
 				                                   $p=nodes[i]$p,
-				                                   $connect=T, $retry=1mins, 
+				                                   $connect=T, $retry=retry_interval,
 				                                   $class=node,
 				                                   $events=manager2proxy_events];
 			}
@@ -80,7 +80,7 @@ event bro_init() &priority=9
 				Communication::nodes["manager"] = [$host=nodes[i]$ip,
 				                                   $zone_id=nodes[i]$zone_id,
 				                                   $p=nodes[i]$p,
-				                                   $connect=T, $retry=1mins, 
+				                                   $connect=T, $retry=retry_interval,
 				                                   $class=node,
 				                                   $events=manager2worker_events];
 
@@ -88,7 +88,7 @@ event bro_init() &priority=9
 				Communication::nodes["proxy"] = [$host=nodes[i]$ip,
 				                                 $zone_id=nodes[i]$zone_id,
 				                                 $p=nodes[i]$p,
-				                                 $connect=T, $retry=1mins, 
+				                                 $connect=T, $retry=retry_interval,
 				                                 $sync=T, $class=node,
 				                                 $events=proxy2worker_events];
 
@@ -98,7 +98,7 @@ event bro_init() &priority=9
 				                                        $zone_id=nodes[i]$zone_id,
 				                                        $p=nodes[i]$p,
 				                                        $connect=T,
-				                                        $retry=1min,
+				                                        $retry=retry_interval,
 				                                        $events=tm2worker_events];
 
 			}

scripts/base/frameworks/communication/main.bro

@@ -164,7 +164,7 @@ const src_names = {
 
 event bro_init() &priority=5
 	{
-	Log::create_stream(Communication::LOG, [$columns=Info]);
+	Log::create_stream(Communication::LOG, [$columns=Info, $path="communication"]);
 	}
 
 function do_script_log_common(level: count, src: count, msg: string)

scripts/base/frameworks/dpd/main.bro

@@ -38,7 +38,7 @@ redef record connection += {
 
 event bro_init() &priority=5
 	{
-	Log::create_stream(DPD::LOG, [$columns=Info]);
+	Log::create_stream(DPD::LOG, [$columns=Info, $path="dpd"]);
 	}
 
 event protocol_confirmation(c: connection, atype: Analyzer::Tag, aid: count) &priority=10

scripts/base/frameworks/files/magic/__load__.bro

@@ -1,2 +1,9 @@
+@load-sigs ./archive
+@load-sigs ./audio
+@load-sigs ./font
 @load-sigs ./general
+@load-sigs ./image
+@load-sigs ./msoffice
+@load-sigs ./video
+
 @load-sigs ./libmagic

scripts/base/frameworks/files/magic/archive.sig

@@ -0,0 +1,176 @@
+
+signature file-tar {
+    file-magic /^[[:print:]\x00]{100}([[:digit:]\x20]{7}\x00){3}([[:digit:]\x20]{11}\x00){2}([[:digit:]\x00\x20]{7}[\x20\x00])[0-7\x00]/
+    file-mime "application/x-tar", 100
+}
+
+# This is low priority so that files using zip as a
+# container will be identified correctly.
+signature file-zip {
+	file-mime "application/zip", 10
+	file-magic /^PK\x03\x04.{2}/
+}
+
+# Multivolume Zip archive
+signature file-multi-zip {
+	file-mime "application/zip", 10
+	file-magic /^PK\x07\x08PK\x03\x04/
+}
+
+# RAR
+signature file-rar {
+	file-mime "application/x-rar", 70
+	file-magic /^Rar!/
+}
+
+# GZIP
+signature file-gzip {
+	file-mime "application/x-gzip", 100
+	file-magic /\x1f\x8b/
+}
+
+# Microsoft Cabinet
+signature file-ms-cab {
+	file-mime "application/vnd.ms-cab-compressed", 110
+	file-magic /^MSCF\x00\x00\x00\x00/
+}
+
+# Mac OS X DMG files
+signature file-dmg {
+	file-magic /^(\x78\x01\x73\x0D\x62\x62\x60|\x78\xDA\x63\x60\x18\x05|\x78\x01\x63\x60\x18\x05|\x78\xDA\x73\x0D|\x78[\x01\xDA]\xED[\xD0-\xD9])/
+	file-mime "application/x-dmg", 100
+}
+
+# XAR (eXtensible ARchive) format.
+# Mac OS X uses this for the .pkg format.
+signature file-xar {
+	file-magic /^xar\!/
+	file-mime "application/x-xar", 100
+}
+
+# RPM
+signature file-magic-auto352 {
+	file-mime "application/x-rpm", 70
+	file-magic /^(drpm|\xed\xab\xee\xdb)/
+}
+
+# StuffIt
+signature file-stuffit {
+	file-mime "application/x-stuffit", 70
+	file-magic /^(SIT\x21|StuffIt)/
+}
+
+# Archived data
+signature file-x-archive {
+	file-mime "application/x-archive", 70
+	file-magic /^!?<ar(ch)?>/
+}
+
+# ARC archive data
+signature file-arc {
+	file-mime "application/x-arc", 70
+	file-magic /^[\x00-\x7f]{2}[\x02-\x0a\x14\x48]\x1a/
+}
+
+# EET archive
+signature file-eet {
+	file-mime "application/x-eet", 70
+	file-magic /^\x1e\xe7\xff\x00/
+}
+
+# Zoo archive
+signature file-zoo {
+	file-mime "application/x-zoo", 70
+	file-magic /^.{20}\xdc\xa7\xc4\xfd/
+}
+
+# LZ4 compressed data (legacy format)
+signature file-lz4-legacy {
+	file-mime "application/x-lz4", 70
+	file-magic /(\x02\x21\x4c\x18)/
+}
+
+# LZ4 compressed data
+signature file-lz4 {
+	file-mime "application/x-lz4", 70
+	file-magic /^\x04\x22\x4d\x18/
+}
+
+# LRZIP compressed data
+signature file-lrzip {
+	file-mime "application/x-lrzip", 1
+	file-magic /^LRZI/
+}
+
+# LZIP compressed data
+signature file-lzip {
+	file-mime "application/x-lzip", 70
+	file-magic /^LZIP/
+}
+
+# Self-extracting PKZIP archive
+signature file-magic-auto434 {
+	file-mime "application/zip", 340
+	file-magic /^MZ.{28}(Copyright 1989\x2d1990 PKWARE Inc|PKLITE Copr)\x2e/
+}
+
+# LHA archive (LZH)
+signature file-lzh {
+	file-mime "application/x-lzh", 80
+	file-magic /^.{2}-(lh[ abcdex0-9]|lz[s2-8]|lz[s2-8]|pm[s012]|pc1)-/
+}
+
+# WARC Archive
+signature file-warc {
+	file-mime "application/warc", 50
+	file-magic /^WARC\x2f/
+}
+
+# 7-zip archive data
+signature file-7zip {
+	file-mime "application/x-7z-compressed", 50
+	file-magic /^7z\xbc\xaf\x27\x1c/
+}
+
+# XZ compressed data
+signature file-xz {
+	file-mime "application/x-xz", 90
+	file-magic /^\xfd7zXZ\x00/
+}
+
+# LHa self-extracting archive
+signature file-magic-auto436 {
+	file-mime "application/x-lha", 120
+	file-magic /^MZ.{34}LH[aA]\x27s SFX/
+}
+
+# ARJ archive data
+signature file-arj {
+	file-mime "application/x-arj", 50
+	file-magic /^\x60\xea/
+}
+
+# Byte-swapped cpio archive
+signature file-bs-cpio {
+	file-mime "application/x-cpio", 50
+	file-magic /(\x71\xc7|\xc7\x71)/
+}
+
+# CPIO archive
+signature file-cpio {
+	file-mime "application/x-cpio", 50
+	file-magic /^(\xc7\x71|\x71\xc7)/
+}
+
+# Compress'd data
+signature file-compress {
+	file-mime "application/x-compress", 50
+	file-magic /^\x1f\x9d/
+}
+
+# LZMA compressed data
+signature file-lzma {
+	file-mime "application/x-lzma", 71
+	file-magic /^\x5d\x00\x00/
+}
+