Merge branch 'master' into topic/johanna/openflow

.gitmodules

@@ -22,3 +22,6 @@
 [submodule "aux/plugins"]
 	path = aux/plugins
 	url = git://git.bro.org/bro-plugins
+[submodule "aux/broker"]
+	path = aux/broker
+	url = git://git.bro.org/broker

CHANGES

@@ -1,4 +1,557 @@
 
+2.3-680 | 2015-04-06 16:02:43 -0500
+
+  * BIT-1371: remove CMake version check from binary package scripts.
+    (Jon Siwek)
+
+2.3-679 | 2015-04-06 10:16:36 -0500
+
+  * Increase some unit test timeouts. (Jon Siwek)
+
+  * Fix Coverity warning in RDP analyzer. (Jon Siwek)
+
+2.3-676 | 2015-04-02 10:10:39 -0500
+
+  * BIT-1366: improve checksum offloading warning.
+    (Frank Meier, Jon Siwek)
+
+2.3-675 | 2015-03-30 17:05:05 -0500
+
+  * Add an RDP analyzer. (Josh Liburdi, Seth Hall, Johanna Amann)
+
+2.3-640 | 2015-03-30 13:51:51 -0500
+
+  * BIT-1359: Limit maximum number of DTLS fragments to 30. (Johanna Amann)
+
+2.3-637 | 2015-03-30 12:02:07 -0500
+
+  * Increase timeout duration in some broker tests. (Jon Siwek)
+
+2.3-636 | 2015-03-30 11:26:32 -0500
+
+  * Updates related to SSH analysis. (Jon Siwek)
+
+    - Some scripts used wrong SSH module/namespace scoping on events.
+    - Fix outdated notice documentation related to SSH password guessing.
+    - Add a unit test for SSH pasword guessing notice.
+
+2.3-635 | 2015-03-30 11:02:45 -0500
+
+  * Fix outdated documentation unit tests. (Jon Siwek)
+
+2.3-634 | 2015-03-30 10:22:45 -0500
+
+  * Add a canonifier to a unit test's output. (Jon Siwek)
+
+2.3-633 | 2015-03-25 18:32:59 -0700
+
+  * Log::write in signature framework was missing timestamp.
+    (Andrew Benson/Michel Laterman)
+
+2.3-631 | 2015-03-25 11:03:12 -0700
+
+  * New SSH analyzer. (Vlad Grigorescu)
+
+2.3-600 | 2015-03-25 10:23:46 -0700
+
+  * Add defensive checks in code to calculate log rotation intervals.
+    (Pete Nelson).
+
+2.3-597 | 2015-03-23 12:50:04 -0700
+
+  * DTLS analyzer. (Johanna Amann)
+
+  * Implement correct parsing of TLS record fragmentation. (Johanna
+    Amann)
+
+2.3-582 | 2015-03-23 11:34:25 -0700
+
+  * BIT-1313: In debug builds, "bro -B <x>" now supports "all" and
+    "help" for "<x>". "all" enables all debug streams. "help" prints a
+    list of available debug streams. (John Donnelly/Robin Sommer).
+
+  * BIT-1324: Allow logging filters to inherit default path from
+    stream. This allows the path for the default filter to be
+    specified explicitly through $path="..." when creating a stream.
+    Adapted the existing Log::create_stream calls to explicitly
+    specify a path value. (Jon Siwek)
+
+  * BIT-1199: Change the way the input framework deals with values it
+    cannot convert into BroVals, raising error messages instead of
+    aborting execution. (Johanna Amann)
+
+  * BIT-788: Use DNS QR field to better identify flow direction. (Jon
+    Siwek)
+
+2.3-572 | 2015-03-23 13:04:53 -0500
+
+  * BIT-1226: Fix an example in quickstart docs. (Jon siwek)
+
+2.3-570 | 2015-03-23 09:51:20 -0500
+
+  * Correct a spelling error (Daniel Thayer)
+
+  * Improvement to SSL analyzer failure mode. (Johanna Amann)
+
+2.3-565 | 2015-03-20 16:27:41 -0500
+
+  * BIT-978: Improve documentation of 'for' loop iterator invalidation.
+    (Jon Siwek)
+
+2.3-564 | 2015-03-20 11:12:02 -0500
+
+  * BIT-725: Remove "unmatched_HTTP_reply" weird. (Jon Siwek)
+
+2.3-562 | 2015-03-20 10:31:02 -0500
+
+  * BIT-1207: Add unit test to catch breaking changes to local.bro
+    (Jon Siwek)
+
+  * Fix failing sqlite leak test (Johanna Amann)
+
+2.3-560 | 2015-03-19 13:17:39 -0500
+
+  * BIT-1255: Increase default values of
+    "tcp_max_above_hole_without_any_acks" and "tcp_max_initial_window"
+    from 4096 to 16384 bytes. (Jon Siwek)
+
+2.3-559 | 2015-03-19 12:14:33 -0500
+
+  * BIT-849: turn SMTP reporter warnings into weirds,
+    "smtp_nested_mail_transaction" and "smtp_unmatched_end_of_data".
+    (Jon Siwek)
+
+2.3-558 | 2015-03-18 22:50:55 -0400
+
+  * DNS: Log the type number for the DNS_RR_unknown_type weird. (Vlad Grigorescu)
+
+2.3-555 | 2015-03-17 15:57:13 -0700
+
+  * Splitting test-all Makefile target into Bro tests and test-aux.
+    (Robin Sommer)
+
+2.3-554 | 2015-03-17 15:40:39 -0700
+
+  * Deprecate &rotate_interval, &rotate_size, &encrypt. Addresses
+    BIT-1305. (Jon Siwek)
+
+2.3-549 | 2015-03-17 09:12:18 -0700
+
+  * BIT-1077: Fix HTTP::log_server_header_names. Before, it just
+    re-logged fields from the client side. (Jon Siwek)
+
+2.3-547 | 2015-03-17 09:07:51 -0700
+
+  * Update certificate validation script to cache valid intermediate
+    chains that it encounters on the wire and use those to try to
+    validate chains that might be missing intermediate certificates.
+    (Johanna Amann)
+
+2.3-541 | 2015-03-13 15:44:08 -0500
+
+  * Make INSTALL a symlink to doc/install/install.rst (Jon siwek)
+
+  * Fix Broxygen coverage. (Jon Siwek)
+
+2.3-539 | 2015-03-13 14:19:27 -0500
+
+  * BIT-1335: Include timestamp in default extracted file names.
+    And add a policy script to extract all files. (Jon Siwek)
+
+  * BIT-1311: Identify GRE tunnels as Tunnel::GRE, not Tunnel::IP.
+    (Jon Siwek)
+
+  * BIT-1309: Add Connection class getter methods for flow labels.
+    (Jon Siwek)
+
+2.3-536 | 2015-03-12 16:16:24 -0500
+
+  * Fix Broker leak tests. (Jon Siwek)
+
+2.3-534 | 2015-03-12 10:59:49 -0500
+
+  * Update NEWS file. (Jon Siwek)
+
+2.3-533 | 2015-03-12 10:18:53 -0500
+
+  * Give broker python bindings default install path within --prefix.
+    (Jon Siwek)
+
+2.3-530 | 2015-03-10 13:22:39 -0500
+
+  * Fix broker data stores in absence of --enable-debug. (Jon Siwek)
+
+2.3-529 | 2015-03-09 13:14:27 -0500
+
+  * Fix format specifier in SSL protocol violation. (Jon Siwek)
+
+2.3-526 | 2015-03-06 12:48:49 -0600
+
+  * Fix build warnings, clarify broker requirements, update submodule.
+    (Jon Siwek)
+
+  * Rename comm/ directories to broker/ (Jon Siwek)
+
+  * Rename broker-related namespaces. (Jon Siwek)
+
+  * Improve remote logging via broker by only sending fields w/ &log.
+    (Jon Siwek)
+
+  * Disable a stream's remote logging via broker if it fails. (Jon Siwek)
+
+  * Improve some broker communication unit tests. (Jon Siwek)
+
+2.3-518 | 2015-03-04 13:13:50 -0800
+
+  * Add bytes_recvd to stats.log recording the number of bytes
+    received, according to packet headers. (Mike Smiley)
+
+2.3-516 | 2015-03-04 12:30:06 -0800
+
+  * Extract most specific Common Name from SSL certificates (Johanna
+    Amann)
+
+  * Send CN and SAN fields of SSL certificates to the Intel framework.
+    (Johanna Amann)
+
+2.3-511 | 2015-03-02 18:07:17 -0800
+
+  * Changes to plugin meta hooks for function calls. (Gilbert Clark)
+
+        - Add frame argument.
+
+        - Change return value to tuple unambigiously whether hook
+          returned a result.
+
+2.3-493 | 2015-03-02 17:17:32 -0800
+
+  * Extend the SSL weak-keys policy file to also alert when
+    encountering SSL connections with old versions as well as unsafe
+    cipher suites. (Johanna Amann)
+
+  * Make the notice suppression handling of other SSL policy files a
+    tad more robust. (Johanna Amann)
+
+2.3-491 | 2015-03-02 17:12:56 -0800
+
+  * Updating docs for recent addition of local_resp. (Robin Sommer)
+
+2.3-489 | 2015-03-02 15:29:30 -0800
+
+  * Integrate Broker, Bro's new communication library. (Jon Siwek)
+
+    See aux/broker/README for more information on Broker, and
+    doc/frameworks/comm.rst for the corresponding Bro script API.
+
+    Broker support is by default off for now; it can be enabled at
+    configure time with --enable-broker. It requires CAF
+    (https://github.com/actor-framework/actor-framework); for now iot
+    needs CAF's "develop" branch. Broker also requires a C++11
+    compiler.
+
+    Broker will become a mandatory dependency in future Bro versions.
+
+  * Add --enable-c++11 configure flag to compile Bro's source code in
+    C++11 mode with a corresponding compiler. (Jon Siwek)
+
+2.3-451 | 2015-02-24 16:37:08 -0800
+
+  * Updating submodule(s).
+
+2.3-448 | 2015-02-23 16:58:10 -0800
+
+  * Updating NEWS. (Robin Sommer)
+
+2.3-447 | 2015-02-23 16:28:30 -0800
+
+  * Fix potential crash in logging framework when deserializing
+    WriterInfo from remote. where config is present. Testcase crashes
+    on unpatched versions of Bro. (Aaron Eppert)
+
+  * Fix wrong value test in WriterBackend. (Aaron Eppert)
+
+2.3-442 | 2015-02-23 13:29:30 -0800
+
+  * Add a "local_resp" field to conn.log, along the lines of the
+    existing "local_orig". (Mike Smiley)
+
+2.3-440 | 2015-02-23 11:39:17 -0600
+
+  * Updating plugin docs to recent changes. (Robin Sommer)
+
+  * Updating plugin tests to recent changes. (Robin Sommer)
+
+  * Making plugin names case-insensitive for some internal comparisions.
+    Makes plugin system more tolerant against spelling inconsistencies
+    are hard to catch otherwise. (Robin Sommer)
+
+  * Explicitly removing some old scripts on install that have moved
+    into plugins to prevent them causing confusion. (Robin Sommer)
+
+  * BIT-1312: Removing setting installation plugin path from
+    bro-path-dev.sh.  Also, adding to existing BRO_PLUGIN_PATH rather
+    than replacing. (Robin Sommer)
+
+  * Creating the installation directory for plugins at install time.
+    (Robin Sommer)
+
+2.3-427 | 2015-02-20 13:49:33 -0800
+
+  * Removing dependency on PCAP_NETMASK_UNKNOWN to compile with
+    libpcap < 1.1.1. (Robin Sommer)
+
+2.3-426 | 2015-02-20 12:45:51 -0800
+
+  * Add 'while' statement to Bro language. Really. (Jon Siwek)
+
+2.3-424 | 2015-02-20 12:39:10 -0800
+
+  * Add the ability to remove surrounding braces from the JSON
+    formatter. (Seth Hall)
+
+2.3-419 | 2015-02-13 09:10:44 -0600
+
+  * BIT-1011: Update the SOCKS analyzer to support user/pass login.
+    (Nicolas Retrain, Seth Hall, Jon Siwek)
+
+    - Add a new field to socks.log: "password".
+    - Two new events: "socks_login_userpass_request" and
+      "socks_login_userpass_reply".
+    - Two new weirds for unsupported SOCKS authentication method or
+      version.
+    - A new test for authenticated socks traffic.
+
+2.3-416 | 2015-02-12 12:18:42 -0600
+
+  * Submodule update - newest sqlite version (Johanna Amann)
+
+  * Fix use of deprecated gperftools headers. (Jon Siwek)
+
+2.3-413 | 2015-02-08 18:23:05 -0800
+
+  * Fixing analyzer tag types for some Files::* functions. (Robin Sommer)
+
+  * Changing load order for plugin scripts. (Robin Sommer)
+
+2.3-411 | 2015-02-05 10:05:48 -0600
+
+  * Fix file analysis of files with total size below the bof_buffer size
+    never delivering content to stream analyzers. (Seth Hall)
+
+  * Add/fix log fields in x509 diff canonifier. (Jon Siwek)
+
+  * "id" not defined for debug code when using -DPROFILE_BRO_FUNCTIONS
+    (Mike Smiley)
+
+2.3-406 | 2015-02-03 17:02:45 -0600
+
+  * Add x509 canonifier to a unit test. (Jon Siwek)
+
+2.3-405 | 2015-02-02 11:14:24 -0600
+
+  * Fix memory leak in new split_string* functions. (Jon Siwek)
+
+2.3-404 | 2015-01-30 14:23:27 -0800
+
+  * Update documentation (broken links, outdated tests). (Jon Siwek)
+
+  * Deprecate split* family of BIFs. (Jon Siwek)
+
+    These functions are now deprecated in favor of alternative versions that
+    return a vector of strings rather than a table of strings.
+
+    Deprecated functions:
+
+    - split: use split_string instead.
+    - split1: use split_string1 instead.
+    - split_all: use split_string_all instead.
+    - split_n: use split_string_n instead.
+    - cat_string_array: see join_string_vec instead.
+    - cat_string_array_n: see join_string_vec instead.
+    - join_string_array: see join_string_vec instead.
+    - sort_string_array: use sort instead instead.
+    - find_ip_addresses: use extract_ip_addresses instead.
+
+    Changed functions:
+
+    - has_valid_octets: uses a string_vec parameter instead of string_array.
+
+    Addresses BIT-924.
+
+  * Add a new attribute: &deprecated. While scripts are parsed, a
+    warning is raised for each usage of an identifier marked as
+    &deprecated.  This also works for BIFs. Addresses BIT-924,
+    BIT-757. (Jon Siwek)
+
+2.3-397 | 2015-01-27 10:13:10 -0600
+
+  * Handle guess_lexer exceptions in pygments reST directive (Jon Siwek)
+
+2.3-396 | 2015-01-23 10:49:15 -0600
+
+  * DNP3: fix reachable assertion and buffer over-read/overflow.
+    CVE number pending. (Travis Emmert, Jon Siwek)
+
+  * Update binpac: Fix potential out-of-bounds memory reads in generated
+    code. CVE-2014-9586. (John Villamil and Chris Rohlf - Yahoo
+    Paranoids, Jon Siwek)
+
+  * Fixing (harmless) Coverity warning. (Robin Sommer)
+
+2.3-392 | 2015-01-15 09:44:15 -0800
+
+  * Small changes to EC curve names in a newer draft. (Johanna Amann)
+
+2.3-390 | 2015-01-14 13:27:34 -0800
+
+  * Updating MySQL analyses. (Vlad Grigorescu)
+     - Use a boolean success instead of a result string.
+     - Change the affected_rows response detail string to a "rows" count.
+     - Fix the state tracking to log incomplete command.
+
+  * Extend DNP3 to support communication over UDP. (Hui Lin)
+
+  * Fix a bug in DNP3 determining the length of an object in some
+    cases. (Hui Lin)
+
+2.3-376 | 2015-01-12 09:38:10 -0600
+
+  * Improve documentation for connection_established event. (Jon Siwek)
+
+2.3-375 | 2015-01-08 13:10:09 -0600
+
+  * Increase minimum required CMake version to 2.8. (Jon Siwek)
+
+2.3-374 | 2015-01-07 10:03:17 -0600
+
+  * Improve documentation of the Intelligence Framework. (Daniel Thayer)
+
+2.3-371 | 2015-01-06 09:58:09 -0600
+
+  * Update/improve file mime type identification. (Seth Hall)
+
+     - Change to the default BOF buffer size to 3000 (was 1024).
+
+     - Reorganized MS signatures into a separate file.
+
+     - Remove all of the x-c detections.  Nearly all false positives.
+
+     - Improve TAR detections, removing old, back up TAR detections.
+
+     - Remove one of the x-elc detections that was too loose
+       and caused many false positives.
+
+     - Improved lots of the signatures and added new ones. (Seth Hall)
+
+  * Add support for file reassembly in the file analysis framework
+    (Seth Hall, Jon Siwek).
+
+     - The reassembly behavior can be modified per-file by enabling or
+       disabling the reassembler and/or modifying the size of the
+       reassembly buffer.
+
+     - Changed the file extraction analyzer to use stream-wise input to
+       avoid issues with the chunk-wise approach not immediately
+       triggering the file_new event due to mime-type detection delay.
+       Before, early chunks frequently ended up lost.  Extraction also
+       will now explicitly NUL-fill gaps in the file instead of
+       implicitly relying on pwrite to do it.
+
+2.3-349 | 2015-01-05 15:21:13 -0600
+
+  * Fix race condition in unified2 file analyzer startup. (Jon siwek)
+
+2.3-348 | 2014-12-31 09:19:34 -0800
+
+  * Changing Makefile's test-all to run test-all for broctl, which now
+    executes trace-summary tests as well. (Robin Sommer)
+
+2.3-345 | 2014-12-31 09:06:15 -0800
+
+  * Correct a typo in the Notice framework doc. (Daniel Thayer)
+
+2.3-343 | 2014-12-12 12:43:46 -0800
+
+  * Fix PIA packet replay to deliver copy of IP header. This prevented
+    one from writing a packet-wise analyzer that needs access to IP
+    headers and can be attached to a connection via signature match.
+    Addresses BIT-1298 (Jon Siwek)
+
+2.3-338 | 2014-12-08 13:56:19 -0800
+
+  * Add man page for Bro. (Raúl Benencia)
+
+  * Updating doc baselines. (Robin Sommer)
+
+2.3-334 | 2014-12-03 14:22:07 -0800
+
+  * Fix compound assignment to require proper L-value. Addresses
+    BIT-1295. (Jon Siwek)
+
+2.3-332 | 2014-12-03 14:14:11 -0800
+
+  * Make using local IDs in @if directives an error. Addresses
+    BIT-1296. (Jon Siwek)
+
+2.3-330 | 2014-12-03 14:10:39 -0800
+
+  * Fix some "make doc" warnings and update some doc tests. (Daniel
+    Thayer)
+
+2.3-328 | 2014-12-02 08:13:10 -0500
+
+  * Update windows-version-detection.bro to add support for
+    Windows 10. (Michal Purzynski)
+
+2.3-326 | 2014-12-01 12:10:27 -0600
+
+  * BIFScanner: fix invalid characters in generated preprocessor macros.
+    (Hilko Bengen)
+
+  * BIT-1294: fix exec.bro from mutating Input::end_of_data event
+    parameters. (Johanna Amann)
+
+  * Add/invoke "distclean" for testing directories. (Raúl Benencia)
+
+  * Delete prebuilt python bytecode files from git. (Jon Siwek)
+
+  * Add Windows detection based on CryptoAPI HTTP traffic as a software
+    framework policy script. (Vlad Grigorescu)
+
+2.3-316 | 2014-11-25 17:35:06 -0800
+
+  * Make the SSL analyzer skip further processing once encountering
+    situations which are very probably non-recoverable. (Johanna
+    Amann)
+
+2.3-313 | 2014-11-25 14:27:07 -0800
+
+  * Make SSL v2 protocol tests more strict. In its former state they
+    triggered on http traffic over port 443 sometimes. Found by Michał
+    Purzyński. (Johanna Amann)
+
+  * Fix X509 analyzer to correctly return ECDSA as the key_type for
+    ECDSA certs. Bug found by Michał Purzyński. (Johanna Amann)
+
+2.3-310 | 2014-11-19 10:56:59 -0600
+
+  * Disable verbose bison output. (Jon Siwek)
+
+2.3-309 | 2014-11-18 12:17:53 -0800
+
+  * New decompose_uri() function in base/utils/urls that splits a URI
+    into its pieces. (Anthony Kasza).
+
+2.3-305 | 2014-11-18 11:09:04 -0800
+
+  * Improve coercion of &default expressions. Addresses BIT-1288. (Jon
+    Siwek)
+
+2.3-303 | 2014-11-18 10:53:04 -0800
+
+  * For DH key exchanges, use p as the parameter for weak key
+    exchanges. (Johanna Amann)
+
 2.3-301 | 2014-11-11 13:47:27 -0800
 
   * Add builtin function enum_to_int() that converts an enum into a
@@ -15,6 +568,11 @@
     logged into mysql.log. Supports protocol versions 9 and 10. (Vlad
     Grigorescu)
 
+2.3-280 | 2014-11-05 09:46:33 -0500
+
+  * Add Windows detection based on CryptoAPI HTTP traffic as a
+    software framework policy script. (Vlad Grigorescu)
+
 2.3-278 | 2014-11-03 18:55:18 -0800
 
   * Add new curves from draft-ietf-tls-negotiated-ff-dhe to SSL

CMakeLists.txt

@@ -2,7 +2,7 @@ project(Bro C CXX)
 
 # When changing the minimum version here, also adapt
 # aux/bro-aux/plugin-support/skeleton/CMakeLists.txt
-cmake_minimum_required(VERSION 2.6.3 FATAL_ERROR)
+cmake_minimum_required(VERSION 2.8 FATAL_ERROR)
 
 include(cmake/CommonCMakeConfig.cmake)
 
@@ -15,6 +15,11 @@ if (NOT BRO_SCRIPT_INSTALL_PATH)
     set(BRO_SCRIPT_INSTALL_PATH ${BRO_ROOT_DIR}/share/bro)
 endif ()
 
+if (NOT BRO_MAN_INSTALL_PATH)
+    # set the default Bro man page installation path (user did not specify one)
+    set(BRO_MAN_INSTALL_PATH ${BRO_ROOT_DIR}/share/man)
+endif ()
+
 # sanitize the Bro script install directory into an absolute path
 # (CMake is confused by ~ as a representation of home directory)
 get_filename_component(BRO_SCRIPT_INSTALL_PATH ${BRO_SCRIPT_INSTALL_PATH}
@@ -26,12 +31,12 @@ configure_file(bro-path-dev.in ${CMAKE_CURRENT_BINARY_DIR}/bro-path-dev)
 
 file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/bro-path-dev.sh
      "export BROPATH=`${CMAKE_CURRENT_BINARY_DIR}/bro-path-dev`\n"
-     "export BRO_PLUGIN_PATH=\"${CMAKE_CURRENT_BINARY_DIR}/src:${BRO_PLUGIN_INSTALL_PATH}\"\n"
+     "export BRO_PLUGIN_PATH=\"${CMAKE_CURRENT_BINARY_DIR}/src\":${BRO_PLUGIN_PATH}\n"
      "export PATH=\"${CMAKE_CURRENT_BINARY_DIR}/src\":$PATH\n")
 
 file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/bro-path-dev.csh
      "setenv BROPATH `${CMAKE_CURRENT_BINARY_DIR}/bro-path-dev`\n"
-     "setenv BRO_PLUGIN_PATH \"${CMAKE_CURRENT_BINARY_DIR}/src:${BRO_PLUGIN_INSTALL_PATH}\"\n"
+     "setenv BRO_PLUGIN_PATH \"${CMAKE_CURRENT_BINARY_DIR}/src\":${BRO_PLUGIN_PATH}\n"
      "setenv PATH \"${CMAKE_CURRENT_BINARY_DIR}/src\":$PATH\n")
 
 file(STRINGS "${CMAKE_CURRENT_SOURCE_DIR}/VERSION" VERSION LIMIT_COUNT 1)
@@ -172,9 +177,21 @@ include_directories(${CMAKE_CURRENT_BINARY_DIR})
 ########################################################################
 ## Recurse on sub-directories
 
+if ( ENABLE_CXX11 )
+    include(RequireCXX11)
+endif ()
+
+if ( ENABLE_BROKER )
+    add_subdirectory(aux/broker)
+    set(brodeps ${brodeps} broker)
+    add_definitions(-DENABLE_BROKER)
+    include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR}/aux/broker)
+endif ()
+
 add_subdirectory(src)
 add_subdirectory(scripts)
 add_subdirectory(doc)
+add_subdirectory(man)
 
 include(CheckOptionalBuildSources)
 
@@ -218,6 +235,7 @@ message(
     "\nCXXFLAGS:          ${CMAKE_CXX_FLAGS} ${CMAKE_CXX_FLAGS_${BuildType}}"
     "\nCPP:               ${CMAKE_CXX_COMPILER}"
     "\n"
+    "\nBroker:            ${ENABLE_BROKER}"
     "\nBroccoli:          ${INSTALL_BROCCOLI}"
     "\nBroctl:            ${INSTALL_BROCTL}"
     "\nAux. Tools:        ${INSTALL_AUX_TOOLS}"

INSTALL

@@ -1,3 +0,0 @@
-
-See doc/install/install.rst for installation instructions.
-

INSTALL

@@ -0,0 +1 @@
+doc/install/install.rst

Makefile

@@ -48,15 +48,18 @@ bindist:
 
 distclean:
 	rm -rf $(BUILD)
+	$(MAKE) -C testing $@
 
 test:
-	@( cd testing && make )
+	-@( cd testing && make )
 
-test-all: test
+test-aux:
-	test -d aux/broctl && ( cd aux/broctl && make test )
+	-test -d aux/broctl && ( cd aux/broctl && make test-all )
-	test -d aux/btest  && ( cd aux/btest && make test )
+	-test -d aux/btest  && ( cd aux/btest && make test )
-	test -d aux/bro-aux && ( cd aux/bro-aux && make test )
+	-test -d aux/bro-aux && ( cd aux/bro-aux && make test )
-	test -d aux/plugins && ( cd aux/plugins && make test-all )
+	-test -d aux/plugins && ( cd aux/plugins && make test-all )
+
+test-all: test test-aux
 
 configured:
 	@test -d $(BUILD) || ( echo "Error: No build/ directory found. Did you run configure?" && exit 1 )

NEWS

@@ -28,11 +28,134 @@ New Functionality
 - Bro now has supoprt for the MySQL wire protocol. Activity gets
   logged into mysql.log.
 
+- Bro now features a completely rewritten, enhanced SSH analyzer. A lot
+  more information about SSH sessions is logged. The analyzer is able to
+  determine if logins failed or succeeded in most circumstances.
+
+- Bro's file analysis now supports reassembly of files that are not
+  transferred/seen sequentially.
+
+- Bro's scripting language now has a ``while`` statement::
+
+        while ( i < 5 )
+            print ++i;
+
+  ``next`` and ``break`` can be used inside the loop's body just like
+  with ``for`` loops.
+
+- Bro now integrates Broker, it's new communication library. See
+  aux/broker/README for more information on Broker, and
+  doc/frameworks/comm.rst for the corresponding Bro script API.
+
+  TODO: Extend with some more information on  Broker.
+
+  Broker support is by default off for now; it can be enabled at
+  configure time with --enable-broker. It requires CAF version 0.13+
+  (https://github.com/actor-framework/actor-framework) as well as a
+  C++11 compiler (e.g. GCC 4.8+ or Clang 3.3+).
+
+  Broker will become a mandatory dependency in future Bro versions.
+
+- Add --enable-c++11 configure flag to compile Bro's source code in
+  C++11 mode with a corresponding compiler. Note that 2.4 will be the
+  last version of Bro that compiles without C++11 support.
+
+- The SSL analysis now alert when encountering SSL connections with
+  old protocol versions or unsafe cipher suites.
+
+- [TODO] Add new BroControl features.
+
+- A new icmp_sent_payload event provides access to ICMP payload.
+
+- Bro now parses DTLS traffic.
+
+- Bro now has an RDP analyzer.
+
 Changed Functionality
 ---------------------
 
 - bro-cut has been rewritten in C, and is hence much faster.
 
+- File analysis
+
+    * Removed ``fa_file`` record's ``mime_type`` and ``mime_types``
+      fields.  The events ``file_mime_type`` and ``file_mime_types``
+      have been added which contain the same information.  The
+      ``mime_type`` field of ``Files::Info`` also still has this info.
+
+    * The earliest point that new mime type information is available is
+      in the ``file_mime_type`` event which comes after the ``file_new``
+      and ``file_over_new_connection`` events.  Scripts which inspected
+      mime type info within those events will need to be adapted.
+
+    * Removed ``Files::add_analyzers_for_mime_type`` function.
+
+    * Removed ``offset`` parameter of the ``file_extraction_limit``
+      event.  Since file extraction now internally depends on file
+      reassembly for non-sequential files, "offset" can be obtained
+      with other information already available -- adding together
+      ``seen_bytes`` and ``missed_bytes`` fields of the ``fa_file``
+      record gives the how many bytes have been written so far (i.e.
+      the "offset").
+
+- has_valid_octets: now uses a string_vec parameter instead of
+  string_array.
+
+- conn.log gained a new field local_resp that works like local_orig,
+  just for the responder address of the connection.
+
+- GRE tunnels are now identified as ``Tunnel::GRE`` instead of
+  ``Tunnel::IP``.
+
+- The default name for extracted files changed from extract-protocol-id
+  to extract-timestamp-protocol-id.
+
+- [TODO] Add changed BroControl features.
+
+- The weird named "unmatched_HTTP_reply" has been removed since it can
+  be detected at the script-layer and is handled correctly by the
+  default HTTP scripts.
+
+- When adding a logging filter to a stream, the filter can now inherit
+  a default ``path`` field from the associated ``Log::Stream`` record.
+
+- When adding a logging filter to a stream, the
+  ``Log::default_path_func`` is now only automatically added to the
+  filter if it has neither a ``path`` nor a ``path_func`` already
+  explicitly set.  Before, the default path function would always be set
+  for all filters which didn't specify their own ``path_func``.
+
+- TODO: what SSH events got changed or removed?
+
+Deprecated Functionality
+------------------------
+
+- The split* family of functions are to be replaced with alternate
+  versions that return a vector of strings rather than a table of
+  strings. This also allows deprecation for some related string
+  concatenation/extraction functions. Note that the new functions use
+  0-based indexing, rather than 1-based.
+
+  The full list of now deprecation functions is:
+
+    * split: use split_string instead.
+
+    * split1: use split_string1 instead.
+
+    * split_all: use split_string_all instead.
+
+    * split_n: use split_string_n instead.
+
+    * cat_string_array: see join_string_vec instead.
+
+    * cat_string_array_n: see join_string_vec instead.
+
+    * join_string_array: see join_string_vec instead.
+
+    * sort_string_array: use sort instead.
+
+    * find_ip_addresses: use extract_ip_addresses instead.
+
 Bro 2.3
 =======
 

VERSION

@@ -1 +1 @@
-2.3-301
+2.3-680

aux/binpac

@@ -1 +1 @@
-Subproject commit 77a86591dcf89d7252d3676d3f1199d6c927d073
+Subproject commit ab50e5115bc0d217552a63f15382e45ed608f5fc

aux/bro-aux

@@ -1 +1 @@
-Subproject commit 977654dc51ab08a2afde32241f108cdb4a581d8f
+Subproject commit 462e300bf9c37dcc39b70a4c2d89d19f7351c804

aux/broccoli

@@ -1 +1 @@
-Subproject commit acb8fbe8e7bc6ace5135fb73dca8e29432cdc1ca
+Subproject commit 45276b39a946d70095c983753cd321ad07dcf285

aux/broctl

@@ -1 +1 @@
-Subproject commit 39e865dec9611b9b53b609cbc8df519cebae0a1e
+Subproject commit e864a0949e52a797f4000194b5c2980cf3618deb

aux/broker

@@ -0,0 +1 @@
+Subproject commit 1a2ab9ee7c80ca905e86a2a11283e7c0477341a9

aux/btest

@@ -1 +1 @@
-Subproject commit d67d89aaee32ad5edb9068db55d1310c2f36970a
+Subproject commit d69df586c91531db0c3abe838b10a429dda4fa87

aux/plugins

@@ -1 +1 @@
-Subproject commit ad600b5bdcd56a2723e323c0f2c8e1708956ca4f
+Subproject commit 7a14085394e54a950e477eb4fafb3827ff8dbdc3

cmake

@@ -1 +1 @@
-Subproject commit 1316c07f7059647b6c4a496ea36e4b83bb5d8f0f
+Subproject commit 2fd35ab6a6245a005828c32f0aa87eb21698c054

configure

@@ -41,6 +41,9 @@ Usage: $0 [OPTION]... [VAR=VALUE]...
     --enable-perftools-debug use Google's perftools for debugging
     --enable-jemalloc      link against jemalloc
     --enable-ruby          build ruby bindings for broccoli (deprecated)
+    --enable-c++11         build using the C++11 standard
+    --enable-broker        enable use of the Broker communication library
+                           (requires C++ Actor Framework and C++11)
     --disable-broccoli     don't build or install the Broccoli library
     --disable-broctl       don't install Broctl
     --disable-auxtools     don't build or install auxiliary tools
@@ -55,6 +58,8 @@ Usage: $0 [OPTION]... [VAR=VALUE]...
     --with-flex=PATH       path to flex executable
     --with-bison=PATH      path to bison executable
     --with-perl=PATH       path to perl executable
+    --with-libcaf=PATH     path to C++ Actor Framework installation
+                           (a required Broker dependency)
 
   Optional Packages in Non-Standard Locations:
     --with-geoip=PATH      path to the libGeoIP install root
@@ -67,6 +72,8 @@ Usage: $0 [OPTION]... [VAR=VALUE]...
     --with-ruby-lib=PATH   path to ruby library
     --with-ruby-inc=PATH   path to ruby headers
     --with-swig=PATH       path to SWIG executable
+    --with-rocksdb=PATH    path to RocksDB installation
+                           (an optional Broker dependency)
 
   Packaging Options (for developers):
     --binary-package       toggle special logic for binary packaging
@@ -142,6 +149,10 @@ while [ $# -ne 0 ]; do
             append_cache_entry CMAKE_INSTALL_PREFIX PATH   $optarg
             append_cache_entry BRO_ROOT_DIR         PATH   $optarg
             append_cache_entry PY_MOD_INSTALL_DIR   PATH   $optarg/lib/broctl
+
+            if [ -n "$user_enabled_broker" ]; then
+                append_cache_entry BROKER_PYTHON_HOME   PATH   $prefix
+            fi
             ;;
         --scriptdir=*)
             append_cache_entry BRO_SCRIPT_INSTALL_PATH STRING $optarg
@@ -176,6 +187,15 @@ while [ $# -ne 0 ]; do
         --enable-jemalloc)
             append_cache_entry ENABLE_JEMALLOC     BOOL   true
             ;;
+        --enable-c++11)
+            append_cache_entry ENABLE_CXX11         BOOL   true
+            ;;
+        --enable-broker)
+            append_cache_entry ENABLE_CXX11         BOOL   true
+            append_cache_entry ENABLE_BROKER        BOOL   true
+            append_cache_entry BROKER_PYTHON_HOME   PATH   $prefix
+            user_enabled_broker="true"
+            ;;
         --disable-broccoli)
             append_cache_entry INSTALL_BROCCOLI     BOOL   false
             ;;
@@ -248,6 +268,12 @@ while [ $# -ne 0 ]; do
         --with-swig=*)
             append_cache_entry SWIG_EXECUTABLE      PATH    $optarg
             ;;
+        --with-libcaf=*)
+            append_cache_entry LIBCAF_ROOT_DIR      PATH   $optarg
+            ;;
+        --with-rocksdb=*)
+            append_cache_entry ROCKSDB_ROOT_DIR     PATH   $optarg
+            ;;
         --binary-package)
             append_cache_entry BINARY_PACKAGING_MODE BOOL true
             ;;

doc/components/broker/README.rst

@@ -0,0 +1 @@
+../../../aux/broker/README

doc/components/broker/broker-manual.rst

@@ -0,0 +1 @@
+../../../aux/broker/broker-manual.rst

doc/components/index.rst

@@ -17,6 +17,8 @@ current, independent component releases.
    Broccoli - User Manual <broccoli/broccoli-manual>
    Broccoli Python Bindings <broccoli-python/README>
    Broccoli Ruby Bindings <broccoli-ruby/README>
+   Broker - Bro's (New) Messaging Library (README) <broker/README>
+   Broker - User Manual <broker/broker-manual.rst>
    BroControl - Interactive Bro management shell <broctl/README>
    Bro-Aux - Small auxiliary tools for Bro <bro-aux/README>
    BTest - A unit testing framework <btest/README>

doc/devel/plugins.rst

@@ -3,7 +3,7 @@
 Writing Bro Plugins
 ===================
 
-Bro is internally moving to a plugin structure that enables extending
+Bro internally provides plugin API that enables extending
 the system dynamically, without modifying the core code base. That way
 custom code remains self-contained and can be maintained, compiled,
 and installed independently. Currently, plugins can add the following
@@ -42,18 +42,17 @@ certain structure. To get started, Bro's distribution provides a
 helper script ``aux/bro-aux/plugin-support/init-plugin`` that creates
 a skeleton plugin that can then be customized. Let's use that::
 
-    # mkdir rot13-plugin
+    # init-plugin ./rot13-plugin Demo Rot13
-    # cd rot13-plugin
-    # init-plugin Demo Rot13
 
-As you can see the script takes two arguments. The first is a
+As you can see, the script takes three arguments. The first is a
-namespace the plugin will live in, and the second a descriptive name
+directory inside which the plugin skeleton will be created.  The second
-for the plugin itself. Bro uses the combination of the two to identify
+is the namespace the plugin will live in, and the third is a descriptive
-a plugin. The namespace serves to avoid naming conflicts between
+name for the plugin itself relative to the namespace. Bro uses the
-plugins written by independent developers; pick, e.g., the name of
+combination of namespace and name to identify a plugin. The namespace
-your organisation. The namespace ``Bro`` is reserved for functionality
+serves to avoid naming conflicts between plugins written by independent
-distributed by the Bro Project. In our example, the plugin will be
+developers; pick, e.g., the name of your organisation. The namespace
-called ``Demo::Rot13``.
+``Bro`` is reserved for functionality distributed by the Bro Project. In
+our example, the plugin will be called ``Demo::Rot13``.
 
 The ``init-plugin`` script puts a number of files in place. The full
 layout is described later. For now, all we need is
@@ -61,7 +60,7 @@ layout is described later. For now, all we need is
 there as follows::
 
     # cat src/rot13.bif
-    module CaesarCipher;
+    module Demo;
 
     function rot13%(s: string%) : string
         %{
@@ -82,18 +81,22 @@ The syntax of this file is just like any other ``*.bif`` file; we
 won't go into it here.
 
 Now we can already compile our plugin, we just need to tell the
-configure script put in place by ``init-plugin`` where the Bro source
+configure script that ``init-plugin`` put in place where the Bro
-tree is located (Bro needs to have been built there first)::
+source tree is located (Bro needs to have been built there first)::
 
+    # cd rot13-plugin
     # ./configure --bro-dist=/path/to/bro/dist && make
     [... cmake output ...]
 
-Now our ``rot13-plugin`` directory has everything that it needs
+This builds the plugin in a subdirectory ``build/``. In fact, that
-for Bro to recognize it as a dynamic plugin. Once we point Bro to it,
+subdirectory *becomes* the plugin: when ``make`` finishes, ``build/``
-it will pull it in automatically, as we can check with the ``-N``
+has everything it needs for Bro to recognize it as a dynamic plugin.
+
+Let's try that. Once we point Bro to the ``build/`` directory, it will
+pull in our new plugin automatically, as we can check with the ``-N``
 option::
 
-    # export BRO_PLUGIN_PATH=/path/to/rot13-plugin
+    # export BRO_PLUGIN_PATH=/path/to/rot13-plugin/build
     # bro -N
     [...]
     Plugin: Demo::Rot13 - <Insert brief description of plugin> (dynamic, version 1)
@@ -127,12 +130,12 @@ more verbose option ``-NN``::
     # bro -NN
     [...]
     Plugin: Demo::Rot13 - Caesar cipher rotating a string's characters by 13 places. (dynamic, version 1)
-        [Function] CaesarCipher::rot13
+        [Function] Demo::rot13
     [...]
 
 There's our function. Now let's use it::
 
-    # bro -e 'print CaesarCipher::rot13("Hello")'
+    # bro -e 'print Demo::rot13("Hello")'
     Uryyb
 
 It works. We next install the plugin along with Bro itself, so that it
@@ -141,36 +144,40 @@ environment variable. If we first unset the variable, the function
 will no longer be available::
 
     # unset BRO_PLUGIN_PATH
-    # bro -e 'print CaesarCipher::rot13("Hello")'
+    # bro -e 'print Demo::rot13("Hello")'
-    error in <command line>, line 1: unknown identifier CaesarCipher::rot13, at or near "CaesarCipher::rot13"
+    error in <command line>, line 1: unknown identifier Demo::rot13, at or near "Demo::rot13"
 
 Once we install it, it works again::
 
     # make install
-    # bro -e 'print CaesarCipher::rot13("Hello")'
+    # bro -e 'print Demo::rot13("Hello")'
     Uryyb
 
 The installed version went into
 ``<bro-install-prefix>/lib/bro/plugins/Demo_Rot13``.
 
-We can distribute the plugin in either source or binary form by using
+One can distribute the plugin independently of Bro for others to use.
-the Makefile's ``sdist`` and ``bdist`` target, respectively. Both
+To distribute in source form, just remove the ``build/`` (``make
-create corrsponding tarballs::
+distclean`` does that) and then tar up the whole ``rot13-plugin/``
+directory. Others then follow the same process as above after
+unpacking. To distribute the plugin in binary form, the build process
+conveniently creates a corresponding tarball in ``build/dist/``. In
+this case, it's called ``Demo_Rot13-0.1.tar.gz``, with the version
+number coming out of the ``VERSION`` file that ``init-plugin`` put
+into place. The binary tarball has everything needed to run the
+plugin, but no further source files. Optionally, one can include
+further files by specifying them in the plugin's ``CMakeLists.txt``
+through the ``bro_plugin_dist_files`` macro; the skeleton does that
+for ``README``, ``VERSION``, ``CHANGES``, and ``COPYING``. To use the
+plugin through the binary tarball, just unpack it and point
+``BRO_PLUGIN_PATH`` there; or copy it into
+``<bro-install-prefix>/lib/bro/plugins/`` directly.
 
-    # make sdist
+Before distributing your plugin, you should edit some of the meta
-    [...]
+files that ``init-plugin`` puts in place. Edit ``README`` and
-    Source distribution in build/sdist/Demo_Rot13.tar.gz
+``VERSION``, and update ``CHANGES`` when you make changes. Also put a
-
+license file in place as ``COPYING``; if BSD is fine, you find a
-    # make bdist
+template in ``COPYING.edit-me``.
-    [...]
-    Binary distribution in build/Demo_Rot13-darwin-x86_64.tar.gz
-
-The source archive will contain everything in the plugin directory
-except any generated files. The binary archive will contain anything
-needed to install and run the plugin, i.e., just what ``make install``
-puts into place as well. As the binary distribution is
-platform-dependent, its name includes the OS and architecture the
-plugin was built on.
 
 Plugin Directory Layout
 =======================
@@ -179,7 +186,7 @@ A plugin's directory needs to follow a set of conventions so that Bro
 (1) recognizes it as a plugin, and (2) knows what to load.  While
 ``init-plugin`` takes care of most of this, the following is the full
 story. We'll use ``<base>`` to represent a plugin's top-level
-directory.
+directory. With the skeleton, ``<base>`` corresponds to ``build/``.
 
 ``<base>/__bro_plugin__``
     A file that marks a directory as containing a Bro plugin. The file
@@ -205,6 +212,8 @@ directory.
     Directory with auto-generated Bro scripts that declare the plugin's
     bif elements. The files here are produced by ``bifcl``.
 
+Any other files in ``<base>`` are ignored by Bro.
+
 By convention, a plugin should put its custom scripts into sub folders
 of ``scripts/``, i.e., ``scripts/<script-namespace>/<script>.bro`` to
 avoid conflicts. As usual, you can then put a ``__load__.bro`` in
@@ -229,15 +238,19 @@ their source directory (after ``make`` and setting Bro's
 install``).
 
 ``make install`` copies over the ``lib`` and ``scripts`` directories,
-as well as the ``__bro_plugin__`` magic file and the ``README`` (which
+as well as the ``__bro_plugin__`` magic file and any further
-you should customize). One can add further CMake ``install`` rules to
+distribution files specified in ``CMakeLists.txt`` (e.g., README,
-install additional files if needed.
+VERSION). You can find a full list of files installed in
+``build/MANIFEST``. Behind the scenes, ``make install`` really just
+copies over the binary tarball in ``build/dist``. 
 
-``init-plugin`` will never overwrite existing files, so it's safe to
+``init-plugin`` will never overwrite existing files. If its target
-rerun in an existing plugin directory; it only put files in place that
+directory already exists, it will be default decline to do anything.
-don't exist yet. That also provides a convenient way to revert a file
+You can run it with ``-u`` instead to update an existing plugin,
-back to what ``init-plugin`` created originally: just delete it and
+however it will never overwrite any existing files; it will only put
-rerun.
+in place files it doesn't find yet. To revert a file back to what
+``init-plugin`` created originally, delete it first and then rerun
+with ``-u``.
 
 Activating a Plugin
 ===================
@@ -355,7 +368,7 @@ let's get that in place::
     % cat .diag
     == File ===============================
     Demo::Rot13 - Caesar cipher rotating a string's characters by 13 places. (dynamic, version 1.0)
-        [Function] CaesarCipher::rot13
+        [Function] Demo::rot13
 
     == Error ===============================
     test-diff: no baseline found.
@@ -375,14 +388,14 @@ Now let's add a custom test that ensures that our bif works
 correctly::
 
     # cd tests
-    # cat >plugin/rot13.bro
+    # cat >rot13/bif-rot13.bro
 
     # @TEST-EXEC: bro %INPUT >output
     # @TEST-EXEC: btest-diff output
 
     event bro_init()
         {
-        print CaesarCipher::rot13("Hello");
+        print Demo::rot13("Hello");
         }
 
 Check the output::
@@ -415,7 +428,7 @@ Debugging Plugins
 =================
 
 If your plugin isn't loading as expected, Bro's debugging facilities
-can help to illuminate what's going on. To enable, recompile Bro
+can help illuminate what's going on. To enable, recompile Bro
 with debugging support (``./configure --enable-debug``), and
 afterwards rebuild your plugin as well. If you then run Bro with ``-B
 plugins``, it will produce a file ``debug.log`` that records details
@@ -435,7 +448,6 @@ replaced with a simple dash. Example: If the plugin is called
 output will be recorded to ``debug.log`` if Bro's compiled in debug
 mode.
 
-
 Documenting Plugins
 ===================
 

doc/ext/rst_directive.py

@@ -135,7 +135,10 @@ class Pygments(Directive):
                 # lexer not found, use default.
                 lexer = TextLexer()
         else:
-            lexer = guess_lexer(content)
+            try:
+                lexer = guess_lexer(content)
+            except:
+                lexer = TextLexer()
 
         # import sys
         # print >>sys.stderr, self.arguments, lexer.__class__

doc/frameworks/broker.rst

@@ -0,0 +1,202 @@
+
+.. _brokercomm-framework:
+
+======================================
+Broker-Enabled Communication Framework
+======================================
+
+.. rst-class:: opening
+
+    Bro can now use the `Broker Library
+    <../components/broker/README.html>`_ to exchange information with
+    other Bro processes.  To enable it run Bro's ``configure`` script
+    with the ``--enable-broker`` option.  Note that a C++11 compatible
+    compiler (e.g. GCC 4.8+ or Clang 3.3+) is required as well as the
+    `C++ Actor Framework <http://actor-framework.org/>`_.
+
+.. contents::
+
+Connecting to Peers
+===================
+
+Communication via Broker must first be turned on via
+:bro:see:`BrokerComm::enable`.
+
+Bro can accept incoming connections by calling :bro:see:`BrokerComm::listen`
+and then monitor connection status updates via
+:bro:see:`BrokerComm::incoming_connection_established` and
+:bro:see:`BrokerComm::incoming_connection_broken`.
+
+.. btest-include:: ${DOC_ROOT}/frameworks/broker/connecting-listener.bro
+
+Bro can initiate outgoing connections by calling :bro:see:`BrokerComm::connect`
+and then monitor connection status updates via
+:bro:see:`BrokerComm::outgoing_connection_established`,
+:bro:see:`BrokerComm::outgoing_connection_broken`, and
+:bro:see:`BrokerComm::outgoing_connection_incompatible`.
+
+.. btest-include:: ${DOC_ROOT}/frameworks/broker/connecting-connector.bro
+
+Remote Printing
+===============
+
+To receive remote print messages, first use
+:bro:see:`BrokerComm::subscribe_to_prints` to advertise to peers a topic
+prefix of interest and then create an event handler for
+:bro:see:`BrokerComm::print_handler` to handle any print messages that are
+received.
+
+.. btest-include:: ${DOC_ROOT}/frameworks/broker/printing-listener.bro
+
+To send remote print messages, just call :bro:see:`BrokerComm::print`.
+
+.. btest-include:: ${DOC_ROOT}/frameworks/broker/printing-connector.bro
+
+Notice that the subscriber only used the prefix "bro/print/", but is
+able to receive messages with full topics of "bro/print/hi",
+"bro/print/stuff", and "bro/print/bye".  The model here is that the
+publisher of a message checks for all subscribers who advertised
+interest in a prefix of that message's topic and sends it to them.
+
+Message Format
+--------------
+
+For other applications that want to exchange print messages with Bro,
+the Broker message format is simply:
+
+.. code:: c++
+
+    broker::message{std::string{}};
+
+Remote Events
+=============
+
+Receiving remote events is similar to remote prints.  Just use
+:bro:see:`BrokerComm::subscribe_to_events` and possibly define any new events
+along with handlers that peers may want to send.
+
+.. btest-include:: ${DOC_ROOT}/frameworks/broker/events-listener.bro
+
+To send events, there are two choices.  The first is to use call
+:bro:see:`BrokerComm::event` directly.  The second option is to use
+:bro:see:`BrokerComm::auto_event` to make it so a particular event is
+automatically sent to peers whenever it is called locally via the normal
+event invocation syntax.
+
+.. btest-include:: ${DOC_ROOT}/frameworks/broker/events-connector.bro
+
+Again, the subscription model is prefix-based.
+
+Message Format
+--------------
+
+For other applications that want to exchange event messages with Bro,
+the Broker message format is:
+
+.. code:: c++
+
+    broker::message{std::string{}, ...};
+
+The first parameter is the name of the event and the remaining ``...``
+are its arguments, which are any of the support Broker data types as
+they correspond to the Bro types for the event named in the first
+parameter of the message.
+
+Remote Logging
+==============
+
+.. btest-include:: ${DOC_ROOT}/frameworks/broker/testlog.bro
+
+Use :bro:see:`BrokerComm::subscribe_to_logs` to advertise interest in logs
+written by peers.  The topic names that Bro uses are implicitly of the
+form "bro/log/<stream-name>".
+
+.. btest-include:: ${DOC_ROOT}/frameworks/broker/logs-listener.bro
+
+To send remote logs either use :bro:see:`Log::enable_remote_logging` or
+:bro:see:`BrokerComm::enable_remote_logs`.  The former allows any log stream
+to be sent to peers while the later toggles remote logging for
+particular streams.
+
+.. btest-include:: ${DOC_ROOT}/frameworks/broker/logs-connector.bro
+
+Message Format
+--------------
+
+For other applications that want to exchange logs messages with Bro,
+the Broker message format is:
+
+.. code:: c++
+
+    broker::message{broker::enum_value{}, broker::record{}};
+
+The enum value corresponds to the stream's :bro:see:`Log::ID` value, and
+the record corresponds to a single entry of that log's columns record,
+in this case a ``Test::INFO`` value.
+
+Tuning Access Control
+=====================
+
+By default, endpoints do not restrict the message topics that it sends
+to peers and do not restrict what message topics and data store
+identifiers get advertised to peers.  These are the default
+:bro:see:`BrokerComm::EndpointFlags` supplied to :bro:see:`BrokerComm::enable`.
+
+If not using the ``auto_publish`` flag, one can use the
+:bro:see:`BrokerComm::publish_topic` and :bro:see:`BrokerComm::unpublish_topic`
+functions to manipulate the set of message topics (must match exactly)
+that are allowed to be sent to peer endpoints.  These settings take
+precedence over the per-message ``peers`` flag supplied to functions
+that take a :bro:see:`BrokerComm::SendFlags` such as :bro:see:`BrokerComm::print`,
+:bro:see:`BrokerComm::event`, :bro:see:`BrokerComm::auto_event` or
+:bro:see:`BrokerComm::enable_remote_logs`.
+
+If not using the ``auto_advertise`` flag, one can use the
+:bro:see:`BrokerComm::advertise_topic` and :bro:see:`BrokerComm::unadvertise_topic`
+to manupulate the set of topic prefixes that are allowed to be
+advertised to peers.  If an endpoint does not advertise a topic prefix,
+the only way a peers can send messages to it is via the ``unsolicited``
+flag of :bro:see:`BrokerComm::SendFlags`  and choosing a topic with a matching
+prefix (i.e. full topic may be longer than receivers prefix, just the
+prefix needs to match).
+
+Distributed Data Stores
+=======================
+
+There are three flavors of key-value data store interfaces: master,
+clone, and frontend.
+
+A frontend is the common interface to query and modify data stores.
+That is, a clone is a specific type of frontend and a master is also a
+specific type of frontend, but a standalone frontend can also exist to
+e.g. query and modify the contents of a remote master store without
+actually "owning" any of the contents itself.
+
+A master data store can be be cloned from remote peers which may then
+perform lightweight, local queries against the clone, which
+automatically stays synchronized with the master store.  Clones cannot
+modify their content directly, instead they send modifications to the
+centralized master store which applies them and then broadcasts them to
+all clones.
+
+Master and clone stores get to choose what type of storage backend to
+use.  E.g. In-memory versus SQLite for persistence.  Note that if clones
+are used, data store sizes should still be able to fit within memory
+regardless of the storage backend as a single snapshot of the master
+store is sent in a single chunk to initialize the clone.
+
+Data stores also support expiration on a per-key basis either using an
+absolute point in time or a relative amount of time since the entry's
+last modification time.
+
+.. btest-include:: ${DOC_ROOT}/frameworks/broker/stores-listener.bro
+
+.. btest-include:: ${DOC_ROOT}/frameworks/broker/stores-connector.bro
+
+In the above example, if a local copy of the store contents isn't
+needed, just replace the :bro:see:`BrokerStore::create_clone` call with
+:bro:see:`BrokerStore::create_frontend`.  Queries will then be made against
+the remote master store instead of the local clone.
+
+Note that all queries are made within Bro's asynchrounous ``when``
+statements and must specify a timeout block.

doc/frameworks/broker/connecting-connector.bro

@@ -0,0 +1,19 @@
+
+const broker_port: port &redef;
+redef exit_only_after_terminate = T;
+redef BrokerComm::endpoint_name = "connector";
+
+event bro_init()
+	{
+	BrokerComm::enable();
+	BrokerComm::connect("127.0.0.1", broker_port, 1sec);
+	}
+
+event BrokerComm::outgoing_connection_established(peer_address: string,
+                                            peer_port: port,
+                                            peer_name: string)
+	{
+	print "BrokerComm::outgoing_connection_established",
+		  peer_address, peer_port, peer_name;
+	terminate();
+	}

doc/frameworks/broker/connecting-listener.bro

@@ -0,0 +1,21 @@
+
+const broker_port: port &redef;
+redef exit_only_after_terminate = T;
+redef BrokerComm::endpoint_name = "listener";
+
+event bro_init()
+	{
+	BrokerComm::enable();
+	BrokerComm::listen(broker_port, "127.0.0.1");
+	}
+
+event BrokerComm::incoming_connection_established(peer_name: string)
+	{
+	print "BrokerComm::incoming_connection_established", peer_name;
+	}
+
+event BrokerComm::incoming_connection_broken(peer_name: string)
+	{
+	print "BrokerComm::incoming_connection_broken", peer_name;
+	terminate();
+	}

doc/frameworks/broker/events-connector.bro

@@ -0,0 +1,31 @@
+const broker_port: port &redef;
+redef exit_only_after_terminate = T;
+redef BrokerComm::endpoint_name = "connector";
+global my_event: event(msg: string, c: count);
+global my_auto_event: event(msg: string, c: count);
+
+event bro_init()
+	{
+	BrokerComm::enable();
+	BrokerComm::connect("127.0.0.1", broker_port, 1sec);
+	BrokerComm::auto_event("bro/event/my_auto_event", my_auto_event);
+	}
+
+event BrokerComm::outgoing_connection_established(peer_address: string,
+                                            peer_port: port,
+                                            peer_name: string)
+	{
+	print "BrokerComm::outgoing_connection_established",
+	      peer_address, peer_port, peer_name;
+	BrokerComm::event("bro/event/my_event", BrokerComm::event_args(my_event, "hi", 0));
+	event my_auto_event("stuff", 88);
+	BrokerComm::event("bro/event/my_event", BrokerComm::event_args(my_event, "...", 1));
+	event my_auto_event("more stuff", 51);
+	BrokerComm::event("bro/event/my_event", BrokerComm::event_args(my_event, "bye", 2));
+	}
+
+event BrokerComm::outgoing_connection_broken(peer_address: string,
+                                       peer_port: port)
+	{
+	terminate();
+	}

doc/frameworks/broker/events-listener.bro

@@ -0,0 +1,37 @@
+
+const broker_port: port &redef;
+redef exit_only_after_terminate = T;
+redef BrokerComm::endpoint_name = "listener";
+global msg_count = 0;
+global my_event: event(msg: string, c: count);
+global my_auto_event: event(msg: string, c: count);
+
+event bro_init()
+	{
+	BrokerComm::enable();
+	BrokerComm::subscribe_to_events("bro/event/");
+	BrokerComm::listen(broker_port, "127.0.0.1");
+	}
+
+event BrokerComm::incoming_connection_established(peer_name: string)
+	{
+	print "BrokerComm::incoming_connection_established", peer_name;
+	}
+
+event my_event(msg: string, c: count)
+	{
+	++msg_count;
+	print "got my_event", msg, c;
+
+	if ( msg_count == 5 )
+		terminate();
+	}
+
+event my_auto_event(msg: string, c: count)
+	{
+	++msg_count;
+	print "got my_auto_event", msg, c;
+
+	if ( msg_count == 5 )
+		terminate();
+	}

doc/frameworks/broker/logs-connector.bro

@@ -0,0 +1,40 @@
+@load ./testlog
+
+const broker_port: port &redef;
+redef exit_only_after_terminate = T;
+redef BrokerComm::endpoint_name = "connector";
+redef Log::enable_local_logging = F;
+redef Log::enable_remote_logging = F;
+global n = 0;
+
+event bro_init()
+	{
+	BrokerComm::enable();
+	BrokerComm::enable_remote_logs(Test::LOG);
+	BrokerComm::connect("127.0.0.1", broker_port, 1sec);
+	}
+
+event do_write()
+	{
+	if ( n == 6 )
+		return;
+
+	Log::write(Test::LOG, [$msg = "ping", $num = n]);
+	++n;
+	event do_write();
+	}
+
+event BrokerComm::outgoing_connection_established(peer_address: string,
+                                            peer_port: port,
+                                            peer_name: string)
+	{
+	print "BrokerComm::outgoing_connection_established",
+	      peer_address, peer_port, peer_name;
+	event do_write();
+	}
+
+event BrokerComm::outgoing_connection_broken(peer_address: string,
+                                       peer_port: port)
+	{
+	terminate();
+	}

doc/frameworks/broker/logs-listener.bro

@@ -0,0 +1,25 @@
+@load ./testlog
+
+const broker_port: port &redef;
+redef exit_only_after_terminate = T;
+redef BrokerComm::endpoint_name = "listener";
+
+event bro_init()
+	{
+	BrokerComm::enable();
+	BrokerComm::subscribe_to_logs("bro/log/Test::LOG");
+	BrokerComm::listen(broker_port, "127.0.0.1");
+	}
+
+event BrokerComm::incoming_connection_established(peer_name: string)
+	{
+	print "BrokerComm::incoming_connection_established", peer_name;
+	}
+
+event Test::log_test(rec: Test::Info)
+	{
+	print "wrote log", rec;
+
+	if ( rec$num == 5 )
+		terminate();
+	}

doc/frameworks/broker/printing-connector.bro

@@ -0,0 +1,26 @@
+const broker_port: port &redef;
+redef exit_only_after_terminate = T;
+redef BrokerComm::endpoint_name = "connector";
+
+event bro_init()
+	{
+	BrokerComm::enable();
+	BrokerComm::connect("127.0.0.1", broker_port, 1sec);
+	}
+
+event BrokerComm::outgoing_connection_established(peer_address: string,
+                                            peer_port: port,
+                                            peer_name: string)
+	{
+	print "BrokerComm::outgoing_connection_established",
+	      peer_address, peer_port, peer_name;
+	BrokerComm::print("bro/print/hi", "hello");
+	BrokerComm::print("bro/print/stuff", "...");
+	BrokerComm::print("bro/print/bye", "goodbye");
+	}
+
+event BrokerComm::outgoing_connection_broken(peer_address: string,
+                                       peer_port: port)
+	{
+	terminate();
+	}

doc/frameworks/broker/printing-listener.bro

@@ -0,0 +1,26 @@
+
+const broker_port: port &redef;
+redef exit_only_after_terminate = T;
+redef BrokerComm::endpoint_name = "listener";
+global msg_count = 0;
+
+event bro_init()
+	{
+	BrokerComm::enable();
+	BrokerComm::subscribe_to_prints("bro/print/");
+	BrokerComm::listen(broker_port, "127.0.0.1");
+	}
+
+event BrokerComm::incoming_connection_established(peer_name: string)
+	{
+	print "BrokerComm::incoming_connection_established", peer_name;
+	}
+
+event BrokerComm::print_handler(msg: string)
+	{
+	++msg_count;
+	print "got print message", msg;
+
+	if ( msg_count == 3 )
+		terminate();
+	}

doc/frameworks/broker/stores-connector.bro

@@ -0,0 +1,53 @@
+const broker_port: port &redef;
+redef exit_only_after_terminate = T;
+
+global h: opaque of BrokerStore::Handle;
+
+function dv(d: BrokerComm::Data): BrokerComm::DataVector
+	{
+	local rval: BrokerComm::DataVector;
+	rval[0] = d;
+	return rval;
+	}
+
+global ready: event();
+
+event BrokerComm::outgoing_connection_broken(peer_address: string,
+                                       peer_port: port)
+	{
+	terminate();
+	}
+
+event BrokerComm::outgoing_connection_established(peer_address: string,
+                                            peer_port: port,
+                                            peer_name: string)
+	{
+	local myset: set[string] = {"a", "b", "c"};
+	local myvec: vector of string = {"alpha", "beta", "gamma"};
+	h = BrokerStore::create_master("mystore");
+	BrokerStore::insert(h, BrokerComm::data("one"), BrokerComm::data(110));
+	BrokerStore::insert(h, BrokerComm::data("two"), BrokerComm::data(223));
+	BrokerStore::insert(h, BrokerComm::data("myset"), BrokerComm::data(myset));
+	BrokerStore::insert(h, BrokerComm::data("myvec"), BrokerComm::data(myvec));
+	BrokerStore::increment(h, BrokerComm::data("one"));
+	BrokerStore::decrement(h, BrokerComm::data("two"));
+	BrokerStore::add_to_set(h, BrokerComm::data("myset"), BrokerComm::data("d"));
+	BrokerStore::remove_from_set(h, BrokerComm::data("myset"), BrokerComm::data("b"));
+	BrokerStore::push_left(h, BrokerComm::data("myvec"), dv(BrokerComm::data("delta")));
+	BrokerStore::push_right(h, BrokerComm::data("myvec"), dv(BrokerComm::data("omega")));
+
+	when ( local res = BrokerStore::size(h) )
+		{
+		print "master size", res;
+		event ready();
+		}
+	timeout 10sec
+		{ print "timeout"; }
+	}
+
+event bro_init()
+	{
+	BrokerComm::enable();
+	BrokerComm::connect("127.0.0.1", broker_port, 1secs);
+	BrokerComm::auto_event("bro/event/ready", ready);
+	}

doc/frameworks/broker/stores-listener.bro

@@ -0,0 +1,43 @@
+const broker_port: port &redef;
+redef exit_only_after_terminate = T;
+
+global h: opaque of BrokerStore::Handle;
+global expected_key_count = 4;
+global key_count = 0;
+
+function do_lookup(key: string)
+	{
+	when ( local res = BrokerStore::lookup(h, BrokerComm::data(key)) )
+		{
+		++key_count;
+		print "lookup", key, res;
+
+		if ( key_count == expected_key_count )
+			terminate();
+		}
+	timeout 10sec
+		{ print "timeout", key; }
+	}
+
+event ready()
+	{
+	h = BrokerStore::create_clone("mystore");
+
+	when ( local res = BrokerStore::keys(h) )
+		{
+		print "clone keys", res;
+		do_lookup(BrokerComm::refine_to_string(BrokerComm::vector_lookup(res$result, 0)));
+		do_lookup(BrokerComm::refine_to_string(BrokerComm::vector_lookup(res$result, 1)));
+		do_lookup(BrokerComm::refine_to_string(BrokerComm::vector_lookup(res$result, 2)));
+		do_lookup(BrokerComm::refine_to_string(BrokerComm::vector_lookup(res$result, 3)));
+		}
+	timeout 10sec
+		{ print "timeout"; }
+	}
+
+event bro_init()
+	{
+	BrokerComm::enable();
+	BrokerComm::subscribe_to_events("bro/event/ready");
+	BrokerComm::listen(broker_port, "127.0.0.1");
+	}

doc/frameworks/broker/testlog.bro

@@ -0,0 +1,19 @@
+
+module Test;
+
+export {
+	redef enum Log::ID += { LOG };
+
+	type Info: record {
+		msg: string &log;
+		num: count &log;
+	};
+
+	global log_test: event(rec: Test::Info);
+}
+
+event bro_init() &priority=5
+	{
+	BrokerComm::enable();
+	Log::create_stream(Test::LOG, [$columns=Test::Info, $ev=log_test, $path="test"]);
+	}

doc/frameworks/file_analysis_02.bro

@@ -1,7 +1,7 @@
-event file_new(f: fa_file)
+event file_mime_type(f: fa_file, mime_type: string)
     {
     print "new file", f$id;
-    if ( f?$mime_type && f$mime_type == "text/plain" )
+    if ( mime_type == "text/plain" )
         Files::add_analyzer(f, Files::ANALYZER_MD5);
     }
 

doc/frameworks/index.rst

@@ -14,4 +14,4 @@ Frameworks
    notice
    signatures
    sumstats
-
+   broker

doc/frameworks/intel.rst

@@ -14,32 +14,35 @@ consume that data, make it available for matching, and provide
 infrastructure around improving performance, memory utilization, and
 generally making all of this easier.
 
-Data in the Intelligence Framework is the atomic piece of intelligence
+Data in the Intelligence Framework is an atomic piece of intelligence
 such as an IP address or an e-mail address along with a suite of
 metadata about it such as a freeform source field, a freeform
 descriptive field and a URL which might lead to more information about
 the specific item.  The metadata in the default scripts has been
 deliberately kept minimal so that the community can find the
-appropriate fields that need added by writing scripts which extend the
+appropriate fields that need to be added by writing scripts which extend the
 base record using the normal record extension mechanism.
 
 Quick Start
 -----------
 
-Load the package of scripts that sends data into the Intelligence
-Framework to be checked by loading this script in local.bro::
-
-	@load policy/frameworks/intel/seen
-
 Refer to the "Loading Intelligence" section below to see the format
 for Intelligence Framework text files, then load those text files with
 this line in local.bro::
 
 	redef Intel::read_files += { "/somewhere/yourdata.txt" };
 
-The data itself only needs to reside on the manager if running in a
+The text files need to reside only on the manager if running in a
 cluster.
 
+Add the following line to local.bro in order to load the scripts
+that send "seen" data into the Intelligence Framework to be checked against
+the loaded intelligence data::
+
+	@load policy/frameworks/intel/seen
+
+Intelligence data matches will be logged to the intel.log file.
+
 Architecture
 ------------
 
@@ -58,8 +61,10 @@ manager is the only node that needs the intelligence data.  The
 intelligence framework has distribution mechanisms which will push
 data out to all of the nodes that need it.
 
-Here is an example of the intelligence data format.  Note that all
+Here is an example of the intelligence data format (note that there will be
-whitespace field separators are literal tabs and fields containing only a
+additional fields if you are using CIF intelligence data or if you are
+using the policy/frameworks/intel/do_notice script).  Note that all fields
+must be separated by a single tab character and fields containing only a
 hyphen are considered to be null values. ::
 
 	#fields	indicator	indicator_type	meta.source	meta.desc	meta.url
@@ -69,8 +74,21 @@ hyphen are considered to be null values. ::
 For a list of all built-in `indicator_type` values, please refer to the
 documentation of :bro:see:`Intel::Type`.
 
-To load the data once files are created, use the following example
+Note that if you are using data from the Collective Intelligence Framework,
-code to define files to load with your own file names of course::
+then you will need to add the following line to your local.bro in order
+to support additional metadata fields used by CIF::
+
+	@load policy/integration/collective-intel
+
+There is a simple mechanism to raise a Bro notice (of type Intel::Notice)
+for user-specified intelligence matches.  To use this feature, add the
+following line to local.bro in order to support additional metadata fields
+(documented in the :bro:see:`Intel::MetaData` record)::
+
+	@load policy/frameworks/intel/do_notice
+
+To load the data once the files are created, use the following example
+to specify which files to load (with your own file names of course)::
 
 	redef Intel::read_files += {
 		"/somewhere/feed1.txt",
@@ -85,24 +103,23 @@ Seen Data
 
 When some bit of data is extracted (such as an email address in the
 "From" header in a message over SMTP), the Intelligence Framework
-needs to be informed that this data was discovered and it's presence
+needs to be informed that this data was discovered so that its presence
-should be checked within the intelligence data set.  This is
+will be checked within the loaded intelligence data.  This is
-accomplished through the :bro:see:`Intel::seen` function.
+accomplished through the :bro:see:`Intel::seen` function, however
+typically users won't need to work with this function due to the
+scripts included with Bro that will call this function.
 
-Typically users won't need to work with this function due to built in
+To load all of the scripts included with Bro for sending "seen" data to
-hook scripts that Bro ships with that will "see" data and send it into
+the intelligence framework, just add this line to local.bro::
-the intelligence framework.  A user may only need to load the entire
+
-package of hook scripts as a module or pick and choose specific
+	@load policy/frameworks/intel/seen
-scripts to load.  Keep in mind that as more data is sent into the
+
+Alternatively, specific scripts in that directory can be loaded.
+Keep in mind that as more data is sent into the
 intelligence framework, the CPU load consumed by Bro will increase
 depending on how many times the :bro:see:`Intel::seen` function is
 being called which is heavily traffic dependent.
 
-The full package of hook scripts that Bro ships with for sending this
-"seen" data into the intelligence framework can be loading by adding
-this line to local.bro::
-
-	@load policy/frameworks/intel/seen
 
 Intelligence Matches
 ********************
@@ -111,6 +128,7 @@ Against all hopes, most networks will eventually have a hit on
 intelligence data which could indicate a possible compromise or other
 unwanted activity.  The Intelligence Framework provides an event that
 is generated whenever a match is discovered named :bro:see:`Intel::match`.
+
 Due to design restrictions placed upon
 the intelligence framework, there is no assurance as to where this
 event will be generated.  It could be generated on the worker where
@@ -119,3 +137,7 @@ handled, only the data given as event arguments to the event can be
 assured since the host where the data was seen may not be where
 ``Intel::match`` is handled.
 
+Intelligence matches are logged to the intel.log file.  For a description of
+each field in that file, see the documentation for the :bro:see:`Intel::Info`
+record.
+

doc/frameworks/logging.rst

@@ -344,7 +344,7 @@ example for the ``Foo`` module:
     event bro_init() &priority=5
         {
         # Create the stream. This also adds a default filter automatically.
-        Log::create_stream(Foo::LOG, [$columns=Info, $ev=log_foo]);
+        Log::create_stream(Foo::LOG, [$columns=Info, $ev=log_foo, $path="foo"]);
         }
 
 You can also add the state to the :bro:type:`connection` record to make

doc/frameworks/notice.rst

@@ -88,15 +88,15 @@ directly make modifications to the :bro:see:`Notice::Info` record
 given as the argument to the hook.
 
 Here's a simple example which tells Bro to send an email for all notices of
-type :bro:see:`SSH::Password_Guessing` if the server is 10.0.0.1:
+type :bro:see:`SSH::Password_Guessing` if the guesser attempted to log in to
+the server at 192.168.56.103:
 
-.. code:: bro
+.. btest-include:: ${DOC_ROOT}/frameworks/notice_ssh_guesser.bro
 
-    hook Notice::policy(n: Notice::Info)
+.. btest:: notice_ssh_guesser.bro
-      {
+
-      if ( n$note == SSH::Password_Guessing && n$id$resp_h == 10.0.0.1 )
+    @TEST-EXEC: btest-rst-cmd bro -C -r ${TRACES}/ssh/sshguess.pcap ${DOC_ROOT}/frameworks/notice_ssh_guesser.bro
-        add n$actions[Notice::ACTION_EMAIL];
+    @TEST-EXEC: btest-rst-cmd cat notice.log
-      }
 
 .. note::
 
@@ -111,10 +111,9 @@ a hook body to run before default hook bodies might look like this:
 .. code:: bro
 
     hook Notice::policy(n: Notice::Info) &priority=5
-      {
+        {
-      if ( n$note == SSH::Password_Guessing && n$id$resp_h == 10.0.0.1 )
+        # Insert your code here.
-        add n$actions[Notice::ACTION_EMAIL];
+        }
-      }
 
 Hooks can also abort later hook bodies with the ``break`` keyword. This
 is primarily useful if one wants to completely preempt processing by
@@ -271,7 +270,7 @@ script that is generating the notice has indicated to the notice framework how
 to identify notices that are intrinsically the same. Identification of these
 "intrinsically duplicate" notices is implemented with an optional field in
 :bro:see:`Notice::Info` records named ``$identifier`` which is a simple string.
-If the ``$identifier`` and ``$type`` fields are the same for two notices, the
+If the ``$identifier`` and ``$note`` fields are the same for two notices, the
 notice framework actually considers them to be the same thing and can use that
 information to suppress duplicates for a configurable period of time.
 

doc/frameworks/notice_ssh_guesser.bro

@@ -0,0 +1,10 @@
+
+@load protocols/ssh/detect-bruteforcing
+
+redef SSH::password_guesses_limit=10;
+
+hook Notice::policy(n: Notice::Info)
+	{
+	if ( n$note == SSH::Password_Guessing && /192\.168\.56\.103/ in n$sub )
+		add n$actions[Notice::ACTION_EMAIL];
+	}

doc/httpmonitor/file_extraction.bro

@@ -7,18 +7,15 @@ global mime_to_ext: table[string] of string = {
 	["text/html"] = "html",
 };
 
-event file_new(f: fa_file)
+event file_mime_type(f: fa_file, mime_type: string)
 	{
 	if ( f$source != "HTTP" )
 		return;
 
-	if ( ! f?$mime_type )
+	if ( mime_type !in mime_to_ext )
 		return;
 
-	if ( f$mime_type !in mime_to_ext )
+	local fname = fmt("%s-%s.%s", f$source, f$id, mime_to_ext[mime_type]);
-		return;
-
-	local fname = fmt("%s-%s.%s", f$source, f$id, mime_to_ext[f$mime_type]);
 	print fmt("Extracting file %s", fname);
 	Files::add_analyzer(f, Files::ANALYZER_EXTRACT, [$extract_filename=fname]);
 	}

doc/install/install.rst

@@ -35,7 +35,7 @@ before you begin:
 
 To build Bro from source, the following additional dependencies are required:
 
-    * CMake 2.6.3 or greater            (http://www.cmake.org)
+    * CMake 2.8 or greater              (http://www.cmake.org)
     * Make
     * C/C++ compiler
     * SWIG                              (http://www.swig.org)

doc/logs/index.rst

@@ -113,7 +113,7 @@ default, including:
 As you can see, some log files are specific to a particular protocol,
 while others aggregate information across different types of activity. 
 For a complete list of log files and a description of its purpose, 
-see :doc:`List of Log Files <../script-reference/list-of-log-files>`.
+see :doc:`Log Files <../script-reference/log-files>`.
 
 .. _bro-cut:
 

doc/mimestats/mimestats.bro

@@ -30,7 +30,7 @@ export {
 
 event bro_init() &priority=3
 	{
-	Log::create_stream(MimeMetrics::LOG, [$columns=Info]);
+	Log::create_stream(MimeMetrics::LOG, [$columns=Info, $path="mime_metrics"]);
 	local r1: SumStats::Reducer = [$stream="mime.bytes",
 	                               $apply=set(SumStats::SUM)];
 	local r2: SumStats::Reducer = [$stream="mime.hits", 

doc/quickstart/conditional-notice.bro

@@ -0,0 +1,24 @@
+@load protocols/ssl/expiring-certs
+
+const watched_servers: set[addr] = {
+	87.98.220.10,
+} &redef;
+
+# Site::local_nets usually isn't something you need to modify if
+# BroControl automatically sets it up from networks.cfg.  It's
+# shown here for completeness.
+redef Site::local_nets += {
+	87.98.0.0/16,
+};
+
+hook Notice::policy(n: Notice::Info)
+	{
+	if ( n$note != SSL::Certificate_Expired )
+		return;
+
+	if ( n$id$resp_h !in watched_servers )
+		return;
+
+	add n$actions[Notice::ACTION_EMAIL];
+	}
+

doc/quickstart/index.rst

@@ -156,9 +156,11 @@ changes we want to make:
    notice that means an SSL connection was established and the server's
    certificate couldn't be validated using Bro's default trust roots, but
    we want to ignore it.
-2) ``SSH::Login`` is a notice type that is triggered when an SSH connection
+2) ``SSL::Certificate_Expired`` is a notice type that is triggered when
-   attempt looks like it may have been successful, and we want email when
+   an SSL connection was established using an expired certificate.  We
-   that happens, but only for certain servers.
+   want email when that happens, but only for certain servers on the
+   local network (Bro can also proactively monitor for certs that will
+   soon expire, but this is just for demonstration purposes).
 
 We've defined *what* we want to do, but need to know *where* to do it.
 The answer is to use a script written in the Bro programming language, so
@@ -203,7 +205,7 @@ the variable's value may not change at run-time, but whose initial value can be
 modified via the ``redef`` operator at parse-time.
 
 Let's continue on our path to modify the behavior for the two SSL
-and SSH notices.  Looking at :doc:`/scripts/base/frameworks/notice/main.bro`,
+notices.  Looking at :doc:`/scripts/base/frameworks/notice/main.bro`,
 we see that it advertises:
 
 .. code:: bro
@@ -216,7 +218,7 @@ we see that it advertises:
         const ignored_types: set[Notice::Type] = {} &redef;
     }
 
-That's exactly what we want to do for the SSL notice.  Add to ``local.bro``:
+That's exactly what we want to do for the first notice.  Add to ``local.bro``:
 
 .. code:: bro
 
@@ -248,38 +250,30 @@ is valid before installing it and then restarting the Bro instance:
    stopping bro ...
    starting bro ...
 
-Now that the SSL notice is ignored, let's look at how to send an email on
+Now that the SSL notice is ignored, let's look at how to send an email
-the SSH notice.  The notice framework has a similar option called
+on the other notice.  The notice framework has a similar option called
-``emailed_types``, but using that would generate email for all SSH servers and
+``emailed_types``, but using that would generate email for all SSL
-we only want email for logins to certain ones.  There is a ``policy`` hook
+servers with expired certificates and we only want email for connections
-that is actually what is used to implement the simple functionality of
+to certain ones.  There is a ``policy`` hook that is actually what is
-``ignored_types`` and
+used to implement the simple functionality of ``ignored_types`` and
-``emailed_types``, but it's extensible such that the condition and action taken
+``emailed_types``, but it's extensible such that the condition and
-on notices can be user-defined.
+action taken on notices can be user-defined.
 
-In ``local.bro``, let's define a new ``policy`` hook handler body
+In ``local.bro``, let's define a new ``policy`` hook handler body:
-that takes the email action for SSH logins only for a defined set of servers:
 
-.. code:: bro
+.. btest-include:: ${DOC_ROOT}/quickstart/conditional-notice.bro
 
-    const watched_servers: set[addr] = {
+.. btest:: conditional-notice
-        192.168.1.100,
-        192.168.1.101,
-        192.168.1.102,
-    } &redef;
 
-   hook Notice::policy(n: Notice::Info)
+    @TEST-EXEC: btest-rst-cmd bro -r ${TRACES}/tls/tls-expired-cert.trace ${DOC_ROOT}/quickstart/conditional-notice.bro
-       {
+    @TEST-EXEC: btest-rst-cmd cat notice.log
-       if ( n$note == SSH::SUCCESSFUL_LOGIN && n$id$resp_h in watched_servers )
-            add n$actions[Notice::ACTION_EMAIL];
-       }
 
 You'll just have to trust the syntax for now, but what we've done is
 first declare our own variable to hold a set of watched addresses,
-``watched_servers``; then added a hook handler body to the policy that will
+``watched_servers``; then added a hook handler body to the policy that
-generate an email whenever the notice type is an SSH login and the responding
+will generate an email whenever the notice type is an SSL expired
-host stored
+certificate and the responding host stored inside the ``Info`` record's
-inside the ``Info`` record's connection field is in the set of watched servers.
+connection field is in the set of watched servers.
 
 .. note:: Record field member access is done with the '$' character
    instead of a '.' as might be expected from other languages, in

doc/script-reference/attributes.rst

@@ -43,12 +43,12 @@ The Bro scripting language supports the following attributes.
 +-----------------------------+-----------------------------------------------+
 | :bro:attr:`&mergeable`      |Prefer set union for synchronized state.       |
 +-----------------------------+-----------------------------------------------+
-| :bro:attr:`&group`          |Group event handlers to activate/deactivate.   |
-+-----------------------------+-----------------------------------------------+
 | :bro:attr:`&error_handler`  |Used internally for reporter framework events. |
 +-----------------------------+-----------------------------------------------+
 | :bro:attr:`&type_column`    |Used by input framework for "port" type.       |
 +-----------------------------+-----------------------------------------------+
+| :bro:attr:`&deprecated`     |Marks an identifier as deprecated.             |
++-----------------------------+-----------------------------------------------+
 
 Here is a more detailed explanation of each attribute:
 
@@ -196,11 +196,6 @@ Here is a more detailed explanation of each attribute:
     inconsistencies and can be avoided by unifying the two sets, rather
     than merely overwriting the old value.
 
-.. bro:attr:: &group
-
-    Groups event handlers such that those in the same group can be
-    jointly activated or deactivated.
-
 .. bro:attr:: &error_handler
 
     Internally set on the events that are associated with the reporter
@@ -230,3 +225,9 @@ Here is a more detailed explanation of each attribute:
             msg: string;
         };
 
+.. bro:attr:: &deprecated
+
+    The associated identifier is marked as deprecated and will be
+    removed in a future version of Bro.  Look in the NEWS file for more
+    explanation and/or instructions to migrate code that uses deprecated
+    functionality.

doc/script-reference/statements.rst

@@ -45,8 +45,11 @@ Statements
 |                            | file                   |
 +----------------------------+------------------------+
 | :bro:keyword:`for`,        | Loop over each         |
-| :bro:keyword:`next`,       | element in a container |
+| :bro:keyword:`while`,      | element in a container |
-| :bro:keyword:`break`       | object                 |
+| :bro:keyword:`next`,       | object (``for``), or   |
+| :bro:keyword:`break`       | as long as a condition |
+|                            | evaluates to true      |
+|                            | (``while``).           |
 +----------------------------+------------------------+
 | :bro:keyword:`if`          | Evaluate boolean       |
 |                            | expression and if true,|
@@ -291,7 +294,10 @@ Here are the statements that the Bro scripting language supports.
 .. bro:keyword:: for
 
     A "for" loop iterates over each element in a string, set, vector, or
-    table and executes a statement for each iteration.
+    table and executes a statement for each iteration.  Currently,
+    modifying a container's membership while iterating over it may
+    result in undefined behavior, so avoid adding or removing elements
+    inside the loop.
 
     For each iteration of the loop, a loop variable will be assigned to an
     element if the expression evaluates to a string or set, or an index if
@@ -563,6 +569,36 @@ Here are the statements that the Bro scripting language supports.
     See the :bro:keyword:`return` statement for an explanation of how to
     create an asynchronous function in a Bro script.
 
+.. bro:keyword:: while
+
+    A "while" loop iterates over a body statement as long a given
+    condition remains true.
+
+    A :bro:keyword:`break` statement can be used at any time to immediately
+    terminate the "while" loop, and a :bro:keyword:`next` statement can be
+    used to skip to the next loop iteration.
+
+    Example::
+
+        local i = 0;
+
+        while ( i < 5 )
+            print ++i;
+
+        while ( some_cond() )
+            {
+            local finish_up = F;
+
+            if ( skip_ahead() )
+                next;
+
+            [...]
+
+            if ( finish_up )
+                break;
+
+            [...]
+            }
 
 .. _compound statement:
 

doc/scripting/framework_logging_factorial_02.bro

@@ -23,7 +23,7 @@ function factorial(n: count): count
 event bro_init()
     {
     # Create the logging stream.
-    Log::create_stream(LOG, [$columns=Info]);
+    Log::create_stream(LOG, [$columns=Info, $path="factor"]);
     }
 
 event bro_done()

doc/scripting/framework_logging_factorial_03.bro

@@ -37,7 +37,7 @@ function mod5(id: Log::ID, path: string, rec: Factor::Info) : string
 
 event bro_init()
     {
-    Log::create_stream(LOG, [$columns=Info]);
+    Log::create_stream(LOG, [$columns=Info, $path="factor"]);
     
     local filter: Log::Filter = [$name="split-mod5s", $path_func=mod5];
     Log::add_filter(Factor::LOG, filter);

doc/scripting/framework_logging_factorial_04.bro

@@ -22,7 +22,7 @@ function factorial(n: count): count
 
 event bro_init()
     {
-    Log::create_stream(LOG, [$columns=Info, $ev=log_factor]);
+    Log::create_stream(LOG, [$columns=Info, $ev=log_factor, $path="factor"]);
     }
 
 event bro_done()

doc/scripting/index.rst

@@ -103,9 +103,9 @@ In the ``file_hash`` event handler, there is an ``if`` statement that is used
 to check for the correct type of hash, in this case
 a SHA1 hash.  It also checks for a mime type we've defined as
 being of interest as defined in the constant ``match_file_types``.
-The comparison is made against the expression ``f$mime_type``, which uses
+The comparison is made against the expression ``f$info$mime_type``, which uses
 the ``$`` dereference operator to check the value ``mime_type``
-inside the variable ``f``.  If the entire expression evaluates to true,
+inside the variable ``f$info``.  If the entire expression evaluates to true,
 then a helper function is called to do the rest of the work.  In that
 function, a local variable is defined to hold a string comprised of
 the SHA1 hash concatenated with ``.malware.hash.cymru.com``; this
@@ -826,7 +826,7 @@ example of the ``record`` data type in the earlier sections, the
 ``conn.log``, is shown by the excerpt below.
 
 .. btest-include:: ${BRO_SRC_ROOT}/scripts/base/protocols/conn/main.bro
-   :lines: 10-12,16-17,19,21,23,25,28,31,35,38,57,63,69,92,95,99,102,106,110-111,116
+   :lines: 10-12,16-17,19,21,23,25,28,31,35,38,57,63,69,75,98,101,105,108,112,116-117,122
 
 Looking at the structure of the definition, a new collection of data
 types is being defined as a type called ``Info``.  Since this type

man/CMakeLists.txt

@@ -0,0 +1,5 @@
+
+install(DIRECTORY . DESTINATION ${BRO_MAN_INSTALL_PATH}/man8 FILES_MATCHING
+        PATTERN "*.8"
+)
+

man/bro.8

@@ -0,0 +1,164 @@
+.TH BRO "8" "November 2014" "bro" "System Administration Utilities"
+.SH NAME
+bro \- passive network traffic analyzer
+.SH SYNOPSIS
+.B bro
+\/\fP [\fIoptions\fR] [\fIfile\fR ...]
+.SH DESCRIPTION
+Bro is primarily a security monitor that inspects all traffic on a link in
+depth for signs of suspicious activity. More generally, however, Bro
+supports a wide range of traffic analysis tasks even outside of the
+security domain, including performance measurements and helping with
+trouble-shooting.
+
+Bro comes with built-in functionality for a range of analysis and detection
+tasks, including detecting malware by interfacing to external registries,
+reporting vulnerable versions of software seen on the network, identifying
+popular web applications, detecting SSH brute-forcing, validating SSL
+certificate chains, among others.
+.SH OPTIONS
+.TP
+.B <file>
+policy file, or read stdin
+.TP
+\fB\-a\fR,\ \-\-parse\-only
+exit immediately after parsing scripts
+.TP
+\fB\-b\fR,\ \-\-bare\-mode
+don't load scripts from the base/ directory
+.TP
+\fB\-d\fR,\ \-\-debug\-policy
+activate policy file debugging
+.TP
+\fB\-e\fR,\ \-\-exec <bro code>
+augment loaded policies by given code
+.TP
+\fB\-f\fR,\ \-\-filter <filter>
+tcpdump filter
+.TP
+\fB\-g\fR,\ \-\-dump\-config
+dump current config into .state dir
+.TP
+\fB\-h\fR,\ \-\-help|\-?
+command line help
+.TP
+\fB\-i\fR,\ \-\-iface <interface>
+read from given interface
+.TP
+\fB\-p\fR,\ \-\-prefix <prefix>
+add given prefix to policy file resolution
+.TP
+\fB\-r\fR,\ \-\-readfile <readfile>
+read from given tcpdump file
+.TP
+\fB\-y\fR,\ \-\-flowfile <file>[=<ident>]
+read from given flow file
+.TP
+\fB\-Y\fR,\ \-\-netflow <ip>:<prt>[=<id>]
+read flow from socket
+.TP
+\fB\-s\fR,\ \-\-rulefile <rulefile>
+read rules from given file
+.TP
+\fB\-t\fR,\ \-\-tracefile <tracefile>
+activate execution tracing
+.TP
+\fB\-w\fR,\ \-\-writefile <writefile>
+write to given tcpdump file
+.TP
+\fB\-v\fR,\ \-\-version
+print version and exit
+.TP
+\fB\-x\fR,\ \-\-print\-state <file.bst>
+print contents of state file
+.TP
+\fB\-z\fR,\ \-\-analyze <analysis>
+run the specified policy file analysis
+.TP
+\fB\-C\fR,\ \-\-no\-checksums
+ignore checksums
+.TP
+\fB\-D\fR,\ \-\-dfa\-size <size>
+DFA state cache size
+.TP
+\fB\-F\fR,\ \-\-force\-dns
+force DNS
+.TP
+\fB\-I\fR,\ \-\-print\-id <ID name>
+print out given ID
+.TP
+\fB\-K\fR,\ \-\-md5\-hashkey <hashkey>
+set key for MD5\-keyed hashing
+.TP
+\fB\-L\fR,\ \-\-rule\-benchmark
+benchmark for rules
+.TP
+\fB\-N\fR,\ \-\-print\-plugins
+print available plugins and exit (\fB\-NN\fR for verbose)
+.TP
+\fB\-O\fR,\ \-\-optimize
+optimize policy script
+.TP
+\fB\-P\fR,\ \-\-prime\-dns
+prime DNS
+.TP
+\fB\-Q\fR,\ \-\-time
+print execution time summary to stderr
+.TP
+\fB\-R\fR,\ \-\-replay <events.bst>
+replay events
+.TP
+\fB\-S\fR,\ \-\-debug\-rules
+enable rule debugging
+.TP
+\fB\-T\fR,\ \-\-re\-level <level>
+set 'RE_level' for rules
+.TP
+\fB\-U\fR,\ \-\-status\-file <file>
+Record process status in file
+.TP
+\fB\-W\fR,\ \-\-watchdog
+activate watchdog timer
+.TP
+\fB\-X\fR,\ \-\-broxygen
+generate documentation based on config file
+.TP
+\fB\-\-pseudo\-realtime[=\fR<speedup>]
+enable pseudo\-realtime for performance evaluation (default 1)
+.TP
+\fB\-\-load\-seeds\fR <file>
+load seeds from given file
+.TP
+\fB\-\-save\-seeds\fR <file>
+save seeds to given file
+.SH ENVIRONMENT
+.TP
+.B BROPATH
+file search path
+.TP
+.B BRO_PLUGIN_PATH
+plugin search path
+.TP
+.B BRO_PLUGIN_ACTIVATE
+plugins to always activate
+.TP
+.B BRO_PREFIXES
+prefix list
+.TP
+.B BRO_DNS_FAKE
+disable DNS lookups
+.TP
+.B BRO_SEED_FILE
+file to load seeds from
+.TP
+.B BRO_LOG_SUFFIX
+ASCII log file extension
+.TP
+.B BRO_PROFILER_FILE
+Output file for script execution statistics
+.TP
+.B BRO_DISABLE_BROXYGEN
+Disable Broxygen documentation support
+.SH AUTHOR
+.B bro
+was written by The Bro Project <info@bro.org>.

pkg/check-cmake

@@ -1,14 +0,0 @@
-#!/bin/sh
-
-# CMake/CPack versions before 2.8.3 have bugs that can create bad packages
-# Since packages will be built on several different systems, a single
-# version of CMake is required to obtain consistency, but can be increased
-# as new versions of CMake come out that also produce working packages.
-
-CMAKE_PACK_REQ="cmake version 2.8.6"
-CMAKE_VER=`cmake -version`
-
-if [ "${CMAKE_VER}" != "${CMAKE_PACK_REQ}" ]; then
-    echo "Package creation requires ${CMAKE_PACK_REQ}" >&2
-    exit 1
-fi

pkg/make-deb-packages

@@ -3,8 +3,6 @@
 # This script generates binary DEB packages.
 # They can be found in ../build/ after running.
 
-./check-cmake || { exit 1; }
-
 # The DEB CPack generator depends on `dpkg-shlibdeps` to automatically
 # determine what dependencies to set for the packages
 type dpkg-shlibdeps > /dev/null 2>&1 || {

pkg/make-mac-packages

@@ -3,14 +3,6 @@
 # This script creates binary packages for Mac OS X.
 # They can be found in ../build/ after running.
 
-cmake -P /dev/stdin << "EOF"
-if ( ${CMAKE_VERSION} VERSION_LESS 2.8.9 )
-    message(FATAL_ERROR "CMake >= 2.8.9 required to build package")
-endif ()
-EOF
-
-[ $? -ne 0 ] && exit 1;
-
 type sw_vers > /dev/null 2>&1 || {
     echo "Unable to get Mac OS X version" >&2;
     exit 1;

pkg/make-rpm-packages

@@ -3,8 +3,6 @@
 # This script generates binary RPM packages.
 # They can be found in ../build/ after running.
 
-./check-cmake || { exit 1; }
-
 # The RPM CPack generator depends on `rpmbuild` to create packages
 type rpmbuild > /dev/null 2>&1 || {
     echo "\

scripts/base/files/extract/main.bro

@@ -53,7 +53,8 @@ function set_limit(f: fa_file, args: Files::AnalyzerArgs, n: count): bool
 function on_add(f: fa_file, args: Files::AnalyzerArgs)
 	{
 	if ( ! args?$extract_filename )
-		args$extract_filename = cat("extract-", f$source, "-", f$id);
+		args$extract_filename = cat("extract-", f$last_active, "-", f$source,
+		                            "-", f$id);
 
 	f$info$extracted = args$extract_filename;
 	args$extract_filename = build_path_compressed(prefix, args$extract_filename);

scripts/base/files/unified2/main.bro

@@ -71,11 +71,50 @@ global classification_map: table[count] of string;
 global sid_map: table[count] of string;
 global gen_map: table[count] of string;
 
+global num_classification_map_reads = 0;
+global num_sid_map_reads = 0;
+global num_gen_map_reads = 0;
+global watching = F;
+
 # For reading in config files.
 type OneLine: record {
 	line: string;
 };
 
+function mappings_initialized(): bool
+	{
+	return num_classification_map_reads > 0 &&
+	       num_sid_map_reads > 0 &&
+	       num_gen_map_reads > 0;
+	}
+
+function start_watching()
+	{
+	if ( watching )
+		return;
+
+	watching = T;
+
+	if ( watch_dir != "" )
+		{
+		Dir::monitor(watch_dir, function(fname: string)
+			{
+			Input::add_analysis([$source=fname,
+			                     $reader=Input::READER_BINARY,
+			                     $mode=Input::STREAM,
+			                     $name=fname]);
+			}, 10secs);
+		}
+
+	if ( watch_file != "" )
+		{
+		Input::add_analysis([$source=watch_file,
+		                     $reader=Input::READER_BINARY,
+		                     $mode=Input::STREAM,
+		                     $name=watch_file]);
+		}
+	}
+
 function create_info(ev: IDSEvent): Info
 	{
 	local info = Info($ts=ev$ts,
@@ -113,34 +152,56 @@ redef record fa_file += {
 
 event Unified2::read_sid_msg_line(desc: Input::EventDescription, tpe: Input::Event, line: string)
 	{
-	local parts = split_n(line, / \|\| /, F, 100);
+	local parts = split_string_n(line, / \|\| /, F, 100);
-	if ( |parts| >= 2 && /^[0-9]+$/ in parts[1] )
+	if ( |parts| >= 2 && /^[0-9]+$/ in parts[0] )
-		sid_map[to_count(parts[1])] = parts[2];
+		sid_map[to_count(parts[0])] = parts[1];
 	}
 
 event Unified2::read_gen_msg_line(desc: Input::EventDescription, tpe: Input::Event, line: string)
 	{
-	local parts = split_n(line, / \|\| /, F, 3);
+	local parts = split_string_n(line, / \|\| /, F, 3);
-	if ( |parts| >= 2 && /^[0-9]+$/ in parts[1] )
+	if ( |parts| >= 2 && /^[0-9]+$/ in parts[0] )
-		gen_map[to_count(parts[1])] = parts[3];
+		gen_map[to_count(parts[0])] = parts[2];
 	}
 
 event Unified2::read_classification_line(desc: Input::EventDescription, tpe: Input::Event, line: string)
 	{
-	local parts = split_n(line, /: /, F, 2);
+	local parts = split_string_n(line, /: /, F, 2);
 	if ( |parts| == 2 )
 		{
-		local parts2 = split_n(parts[2], /,/, F, 4);
+		local parts2 = split_string_n(parts[1], /,/, F, 4);
 		if ( |parts2| > 1 )
-			classification_map[|classification_map|+1] = parts2[1];
+			classification_map[|classification_map|+1] = parts2[0];
 		}
 	}
 
+event Input::end_of_data(name: string, source: string)
+	{
+	if ( name == classification_config )
+		++num_classification_map_reads;
+	else if ( name == sid_msg )
+		++num_sid_map_reads;
+	else if ( name == gen_msg )
+		++num_gen_map_reads;
+	else
+		return;
+
+	if ( watching )
+		return;
+
+	if ( mappings_initialized() )
+		start_watching();
+	}
+
 event bro_init() &priority=5
 	{
-	Log::create_stream(Unified2::LOG, [$columns=Info, $ev=log_unified2]);
+	Log::create_stream(Unified2::LOG, [$columns=Info, $ev=log_unified2, $path="unified2"]);
 
-	if ( sid_msg != "" )
+	if ( sid_msg == "" )
+		{
+		num_sid_map_reads = 1;
+		}
+	else
 		{
 		Input::add_event([$source=sid_msg,
 		                  $reader=Input::READER_RAW,
@@ -151,7 +212,11 @@ event bro_init() &priority=5
 		                  $ev=Unified2::read_sid_msg_line]);
 		}
 
-	if ( gen_msg != "" )
+	if ( gen_msg == "" )
+		{
+		num_gen_map_reads = 1;
+		}
+	else
 		{
 		Input::add_event([$source=gen_msg,
 		                  $name=gen_msg,
@@ -162,7 +227,11 @@ event bro_init() &priority=5
 		                  $ev=Unified2::read_gen_msg_line]);
 		}
 
-	if ( classification_config != "" )
+	if ( classification_config == "" )
+		{
+		num_classification_map_reads = 1;
+		}
+	else
 		{
 		Input::add_event([$source=classification_config,
 		                  $name=classification_config,
@@ -173,32 +242,16 @@ event bro_init() &priority=5
 		                  $ev=Unified2::read_classification_line]);
 		}
 
-	if ( watch_dir != "" )
+	if ( mappings_initialized() )
-		{
+		start_watching();
-		Dir::monitor(watch_dir, function(fname: string)
-			{
-			Input::add_analysis([$source=fname,
-			                     $reader=Input::READER_BINARY,
-			                     $mode=Input::STREAM,
-			                     $name=fname]);
-			}, 10secs);
-		}
-
-	if ( watch_file != "" )
-		{
-		Input::add_analysis([$source=watch_file,
-		                     $reader=Input::READER_BINARY,
-		                     $mode=Input::STREAM,
-		                     $name=watch_file]);
-		}
 	}
 
 event file_new(f: fa_file)
 	{
 	local file_dir = "";
-	local parts = split_all(f$source, /\/[^\/]*$/);
+	local parts = split_string_all(f$source, /\/[^\/]*$/);
 	if ( |parts| == 3 )
-		file_dir = parts[1];
+		file_dir = parts[0];
 
 	if ( (watch_file != "" && f$source == watch_file) || 
 	     (watch_dir != "" && compress_path(watch_dir) == file_dir) )

scripts/base/files/x509/main.bro

@@ -36,7 +36,7 @@ export {
 
 event bro_init() &priority=5
 	{
-	Log::create_stream(X509::LOG, [$columns=Info, $ev=log_x509]);
+	Log::create_stream(X509::LOG, [$columns=Info, $ev=log_x509, $path="x509"]);
 	}
 
 redef record Files::Info += {

scripts/base/frameworks/broker/__load__.bro

@@ -0,0 +1 @@
+@load ./main

scripts/base/frameworks/broker/main.bro

@@ -0,0 +1,103 @@
+##! Various data structure definitions for use with Bro's communication system.
+
+module BrokerComm;
+
+export {
+
+	## A name used to identify this endpoint to peers.
+	## .. bro:see:: BrokerComm::connect BrokerComm::listen
+	const endpoint_name = "" &redef;
+
+	## Change communication behavior.
+	type EndpointFlags: record {
+		## Whether to restrict message topics that can be published to peers.
+		auto_publish: bool &default = T;
+		## Whether to restrict what message topics or data store identifiers
+		## the local endpoint advertises to peers (e.g. subscribing to
+		## events or making a master data store available).
+		auto_advertise: bool &default = T;
+	};
+
+	## Fine-grained tuning of communication behavior for a particular message.
+	type SendFlags: record {
+		## Send the message to the local endpoint.
+		self: bool &default = F;
+		## Send the message to peer endpoints that advertise interest in
+		## the topic associated with the message.
+		peers: bool &default = T;
+		## Send the message to peer endpoints even if they don't advertise
+		## interest in the topic associated with the message.
+		unsolicited: bool &default = F;
+	};
+
+	## Opaque communication data.
+	type Data: record {
+		d: opaque of BrokerComm::Data &optional;
+	};
+
+	## Opaque communication data.
+	type DataVector: vector of BrokerComm::Data;
+
+	## Opaque event communication data.
+	type EventArgs: record {
+		## The name of the event.  Not set if invalid event or arguments.
+		name: string &optional;
+		## The arguments to the event.
+		args: DataVector;
+	};
+
+	## Opaque communication data used as a convenient way to wrap key-value
+	## pairs that comprise table entries.
+	type TableItem : record {
+		key: BrokerComm::Data;
+		val: BrokerComm::Data;
+	};
+}
+
+module BrokerStore;
+
+export {
+
+	## Whether a data store query could be completed or not.
+	type QueryStatus: enum {
+		SUCCESS,
+		FAILURE,
+	};
+
+	## An expiry time for a key-value pair inserted in to a data store.
+	type ExpiryTime: record {
+		## Absolute point in time at which to expire the entry.
+		absolute: time &optional;
+		## A point in time relative to the last modification time at which
+		## to expire the entry.  New modifications will delay the expiration.
+		since_last_modification: interval &optional;
+	};
+
+	## The result of a data store query.
+	type QueryResult: record {
+		## Whether the query completed or not.
+		status: BrokerStore::QueryStatus;
+		## The result of the query.  Certain queries may use a particular
+		## data type (e.g. querying store size always returns a count, but
+		## a lookup may return various data types).
+		result: BrokerComm::Data;
+	};
+
+	## Options to tune the SQLite storage backend.
+	type SQLiteOptions: record {
+		## File system path of the database.
+		path: string &default = "store.sqlite";
+	};
+
+	## Options to tune the RocksDB storage backend.
+	type RocksDBOptions: record {
+		## File system path of the database.
+		path: string &default = "store.rocksdb";
+	};
+
+	## Options to tune the particular storage backends.
+	type BackendOptions: record {
+		sqlite: SQLiteOptions &default = SQLiteOptions();
+		rocksdb: RocksDBOptions &default = RocksDBOptions();
+	};
+}

scripts/base/frameworks/cluster/main.bro

@@ -159,5 +159,5 @@ event bro_init() &priority=5
 		terminate();
 		}
 	
-	Log::create_stream(Cluster::LOG, [$columns=Info]);
+	Log::create_stream(Cluster::LOG, [$columns=Info, $path="cluster"]);
 	}

scripts/base/frameworks/communication/main.bro

@@ -164,7 +164,7 @@ const src_names = {
 
 event bro_init() &priority=5
 	{
-	Log::create_stream(Communication::LOG, [$columns=Info]);
+	Log::create_stream(Communication::LOG, [$columns=Info, $path="communication"]);
 	}
 
 function do_script_log_common(level: count, src: count, msg: string)

scripts/base/frameworks/dpd/main.bro

@@ -38,7 +38,7 @@ redef record connection += {
 
 event bro_init() &priority=5
 	{
-	Log::create_stream(DPD::LOG, [$columns=Info]);
+	Log::create_stream(DPD::LOG, [$columns=Info, $path="dpd"]);
 	}
 
 event protocol_confirmation(c: connection, atype: Analyzer::Tag, aid: count) &priority=10

scripts/base/frameworks/files/magic/__load__.bro

@@ -1,2 +1,3 @@
 @load-sigs ./general
+@load-sigs ./msoffice
 @load-sigs ./libmagic

scripts/base/frameworks/files/magic/general.sig

@@ -1,16 +1,137 @@
 # General purpose file magic signatures.
 
 signature file-plaintext {
-    file-magic /([[:print:][:space:]]{10})/
+    file-magic /^([[:print:][:space:]]{10})/
     file-mime "text/plain", -20
 }
 
 signature file-tar {
-    file-magic /([[:print:]\x00]){100}(([[:digit:]\x00\x20]){8}){3}/
+    file-magic /^[[:print:]\x00]{100}([[:digit:]\x20]{7}\x00){3}([[:digit:]\x20]{11}\x00){2}([[:digit:]\x00\x20]{7}[\x20\x00])[0-7\x00]/
-    file-mime "application/x-tar", 150
+    file-mime "application/x-tar", 100
 }
 
+signature file-zip {
+	file-mime "application/zip", 10
+	file-magic /^PK\x03\x04.{2}/
+}
+
+signature file-jar {
+	file-mime "application/java-archive", 100
+	file-magic /^PK\x03\x04.{1,200}\x14\x00..META-INF\/MANIFEST\.MF/
+}
+
+signature file-java-applet {
+	file-magic /^\xca\xfe\xba\xbe...[\x2e-\x34]/
+	file-mime "application/x-java-applet", 71
+}
+
+# Shockwave flash
 signature file-swf {
-	file-magic /(F|C|Z)WS/
+	file-magic /^(F|C|Z)WS/
 	file-mime "application/x-shockwave-flash", 60
 }
+
+# Microsoft Outlook's Transport Neutral Encapsulation Format
+signature file-tnef {
+	file-magic /^\x78\x9f\x3e\x22/
+	file-mime "application/vnd.ms-tnef", 100
+}
+
+# Mac OS X DMG files
+signature file-dmg {
+	file-magic /^(\x78\x01\x73\x0D\x62\x62\x60|\x78\xDA\x63\x60\x18\x05|\x78\x01\x63\x60\x18\x05|\x78\xDA\x73\x0D|\x78[\x01\xDA]\xED[\xD0-\xD9])/
+	file-mime "application/x-dmg", 100
+}
+
+# Mac OS X Mach-O executable
+signature file-mach-o {
+	file-magic /^[\xce\xcf]\xfa\xed\xfe/
+	file-mime "application/x-mach-o-executable", 100
+}
+
+# Mac OS X Universal Mach-O executable
+signature file-mach-o-universal {
+	file-magic /^\xca\xfe\xba\xbe..\x00[\x01-\x14]/
+	file-mime "application/x-mach-o-executable", 100
+}
+
+# XAR (eXtensible ARchive) format. 
+# Mac OS X uses this for the .pkg format.
+signature file-xar {
+	file-magic /^xar\!/
+	file-mime "application/x-xar", 100
+}
+
+signature file-pkcs7 {
+	file-magic /^MIME-Version:.*protocol=\"application\/pkcs7-signature\"/
+	file-mime "application/pkcs7-signature", 100
+}
+
+# Concatenated X.509 certificates in textual format.
+signature file-pem {
+	file-magic /^-----BEGIN CERTIFICATE-----/
+	file-mime "application/x-pem"
+}
+
+# Java Web Start file.
+signature file-jnlp {
+	file-magic /^\<jnlp\x20/
+	file-mime "application/x-java-jnlp-file", 100
+}
+
+signature file-ico {
+	file-magic /^\x00\x00\x01\x00/
+	file-mime "image/x-icon", 70
+}
+
+signature file-cur {
+	file-magic /^\x00\x00\x02\x00/
+	file-mime "image/x-cursor", 70
+}
+
+signature file-pcap {
+	file-magic /^(\xa1\xb2\xc3\xd4|\xd4\xc3\xb2\xa1)/
+	file-mime "application/vnd.tcpdump.pcap", 70
+}
+
+signature file-pcap-ng {
+	file-magic /^\x0a\x0d\x0d\x0a.{4}(\x1a\x2b\x3c\x4d|\x4d\x3c\x2b\x1a)/
+	file-mime "application/vnd.tcpdump.pcap", 100
+}
+
+signature file-shellscript {
+	file-mime "text/x-shellscript", 250
+	file-magic /^\x23\x21[^\n]{1,15}bin\/(env[[:space:]]+)?(ba|tc|c|z|fa|ae|k)?sh/
+}
+
+signature file-perl {
+	file-magic /^\x23\x21[^\n]{1,15}bin\/(env[[:space:]]+)?perl/
+	file-mime "text/x-perl", 60
+}
+
+signature file-ruby {
+	file-magic /^\x23\x21[^\n]{1,15}bin\/(env[[:space:]]+)?ruby/
+	file-mime "text/x-ruby", 60
+}
+
+signature file-python {
+	file-magic /^\x23\x21[^\n]{1,15}bin\/(env[[:space:]]+)?python/
+	file-mime "text/x-python", 60
+}
+
+signature file-php {
+	file-magic /^.*<\?php/
+	file-mime "text/x-php", 40
+}
+
+# Stereolithography ASCII format
+signature file-stl-ascii {
+	file-magic /^solid\x20/
+	file-mime "application/sla", 10
+}
+
+# Sketchup model file
+signature file-skp {
+	file-magic /^\xFF\xFE\xFF\x0E\x53\x00\x6B\x00\x65\x00\x74\x00\x63\x00\x68\x00\x55\x00\x70\x00\x20\x00\x4D\x00\x6F\x00\x64\x00\x65\x00\x6C\x00/
+	file-mime "application/skp", 100
+}

scripts/base/frameworks/files/magic/libmagic.sig

@@ -7,42 +7,18 @@
 # The instrumented version of the `file` command used to generate these
 # is located at: https://github.com/jsiwek/file/tree/bro-signatures.
 
-# >2080  string,=Foglio di lavoro Microsoft Exce (len=31), ["%s"], swap_endian=0
-signature file-magic-auto0 {
-	file-mime "application/vnd.ms-excel", 340
-	file-magic /(.{2080})(Foglio di lavoro Microsoft Exce)/
-}
-
 # >2  string,=---BEGIN PGP PUBLIC KEY BLOCK- (len=30), ["PGP public key block"], swap_endian=0
 signature file-magic-auto1 {
 	file-mime "application/pgp-keys", 330
 	file-magic /(.{2})(\x2d\x2d\x2dBEGIN PGP PUBLIC KEY BLOCK\x2d)/
 }
 
-# >2080  string,=Microsoft Excel 5.0 Worksheet (len=29), ["%s"], swap_endian=0
-signature file-magic-auto2 {
-	file-mime "application/vnd.ms-excel", 320
-	file-magic /(.{2080})(Microsoft Excel 5\x2e0 Worksheet)/
-}
-
 # >11  string,=must be converted with BinHex (len=29), ["BinHex binary text"], swap_endian=0
 signature file-magic-auto3 {
 	file-mime "application/mac-binhex40", 320
 	file-magic /(.{11})(must be converted with BinHex)/
 }
 
-# >2080  string,=Microsoft Word 6.0 Document (len=27), ["%s"], swap_endian=0
-signature file-magic-auto4 {
-	file-mime "application/msword", 300
-	file-magic /(.{2080})(Microsoft Word 6\x2e0 Document)/
-}
-
-# >2080  string,=Documento Microsoft Word 6 (len=26), ["Spanish Microsoft Word 6 document data"], swap_endian=0
-signature file-magic-auto5 {
-	file-mime "application/msword", 290
-	file-magic /(.{2080})(Documento Microsoft Word 6)/
-}
-
 # >0  string,=-----BEGIN PGP SIGNATURE- (len=25), ["PGP signature"], swap_endian=0
 signature file-magic-auto6 {
 	file-mime "application/pgp-signature", 280
@@ -92,36 +68,6 @@ signature file-magic-auto13 {
 	file-magic /(\x23\x21 ?\x2fusr\x2flocal\x2fbin\x2fgawk)/
 }
 
-# >0  string/wt,=#! /usr/local/bin/bash (len=22), ["Bourne-Again shell script text executable"], swap_endian=0
-signature file-magic-auto14 {
-	file-mime "text/x-shellscript", 250
-	file-magic /(\x23\x21 ?\x2fusr\x2flocal\x2fbin\x2fbash)/
-}
-
-# >0  string/wt,=#! /usr/local/bin/tcsh (len=22), ["Tenex C shell script text executable"], swap_endian=0
-signature file-magic-auto15 {
-	file-mime "text/x-shellscript", 250
-	file-magic /(\x23\x21 ?\x2fusr\x2flocal\x2fbin\x2ftcsh)/
-}
-
-# >0  string/wt,=#! /usr/local/bin/zsh (len=21), ["Paul Falstad's zsh script text executable"], swap_endian=0
-signature file-magic-auto16 {
-	file-mime "text/x-shellscript", 240
-	file-magic /(\x23\x21 ?\x2fusr\x2flocal\x2fbin\x2fzsh)/
-}
-
-# >0  string/wt,=#! /usr/local/bin/ash (len=21), ["Neil Brown's ash script text executable"], swap_endian=0
-signature file-magic-auto17 {
-	file-mime "text/x-shellscript", 240
-	file-magic /(\x23\x21 ?\x2fusr\x2flocal\x2fbin\x2fash)/
-}
-
-# >0  string/wt,=#! /usr/local/bin/ae (len=20), ["Neil Brown's ae script text executable"], swap_endian=0
-signature file-magic-auto18 {
-	file-mime "text/x-shellscript", 230
-	file-magic /(\x23\x21 ?\x2fusr\x2flocal\x2fbin\x2fae)/
-}
-
 # >0  string,=# PaCkAgE DaTaStReAm (len=20), ["pkg Datastream (SVR4)"], swap_endian=0
 signature file-magic-auto19 {
 	file-mime "application/x-svr4-package", 230
@@ -140,30 +86,12 @@ signature file-magic-auto21 {
 	file-magic /(\x5bKDE Desktop Entry\x5d)/
 }
 
-# >512  string,=R\000o\000o\000t\000 \000E\000n\000t\000r\000y (len=19), ["Microsoft Word Document"], swap_endian=0
-signature file-magic-auto22 {
-	file-mime "application/msword", 220
-	file-magic /(.{512})(R\x00o\x00o\x00t\x00 \x00E\x00n\x00t\x00r\x00y)/
-}
-
 # >0  string,=!<arch>\n__________E (len=19), ["MIPS archive"], swap_endian=0
 signature file-magic-auto23 {
 	file-mime "application/x-archive", 220
 	file-magic /(\x21\x3carch\x3e\x0a\x5f\x5f\x5f\x5f\x5f\x5f\x5f\x5f\x5f\x5fE)/
 }
 
-# >0  string/wt,=#! /usr/local/tcsh (len=18), ["Tenex C shell script text executable"], swap_endian=0
-signature file-magic-auto24 {
-	file-mime "text/x-shellscript", 210
-	file-magic /(\x23\x21 ?\x2fusr\x2flocal\x2ftcsh)/
-}
-
-# >0  string/wt,=#! /usr/local/bash (len=18), ["Bourne-Again shell script text executable"], swap_endian=0
-signature file-magic-auto25 {
-	file-mime "text/x-shellscript", 210
-	file-magic /(\x23\x21 ?\x2fusr\x2flocal\x2fbash)/
-}
-
 # >0  string/t,=# KDE Config File (len=17), ["KDE config file"], swap_endian=0
 signature file-magic-auto26 {
 	file-mime "application/x-kdelnk", 200
@@ -189,12 +117,6 @@ signature file-magic-auto29 {
 	file-magic /(\x23\x21 ?\x2fusr\x2fbin\x2fnawk)/
 }
 
-# >0  string/wt,=#! /usr/bin/tcsh (len=16), ["Tenex C shell script text executable"], swap_endian=0
-signature file-magic-auto30 {
-	file-mime "text/x-shellscript", 190
-	file-magic /(\x23\x21 ?\x2fusr\x2fbin\x2ftcsh)/
-}
-
 # >0  string/wt,=#! /usr/bin/gawk (len=16), ["GNU awk script text executable"], swap_endian=0
 signature file-magic-auto31 {
 	file-mime "text/x-gawk", 190
@@ -207,12 +129,6 @@ signature file-magic-auto32 {
 	file-magic /(.{369})(MICROSOFT PIFEX\x00)/
 }
 
-# >0  string/wt,=#! /usr/bin/bash (len=16), ["Bourne-Again shell script text executable"], swap_endian=0
-signature file-magic-auto33 {
-	file-mime "text/x-shellscript", 190
-	file-magic /(\x23\x21 ?\x2fusr\x2fbin\x2fbash)/
-}
-
 # >0  string/w,=#VRML V1.0 ascii (len=16), ["VRML 1 file"], swap_endian=0
 signature file-magic-auto34 {
 	file-mime "model/vrml", 190
@@ -334,12 +250,6 @@ signature file-magic-auto51 {
 	file-magic /(\x23\x21 ?\x2fusr\x2fbin\x2fawk)/
 }
 
-# >0  string/wt,=#! /usr/bin/zsh (len=15), ["Paul Falstad's zsh script text executable"], swap_endian=0
-signature file-magic-auto52 {
-	file-mime "text/x-shellscript", 180
-	file-magic /(\x23\x21 ?\x2fusr\x2fbin\x2fzsh)/
-}
-
 # >0  string,=MAS_UTrack_V00 (len=14), [""], swap_endian=0
 # >>14  string,>/0 (len=2), ["ultratracker V1.%.1s module sound data"], swap_endian=0
 signature file-magic-auto53 {
@@ -457,12 +367,6 @@ signature file-magic-auto70 {
 	file-magic /(\x3cmap ?version)/
 }
 
-# >0  string/wt,=#! /bin/tcsh (len=12), ["Tenex C shell script text executable"], swap_endian=0
-signature file-magic-auto71 {
-	file-mime "text/x-shellscript", 150
-	file-magic /(\x23\x21 ?\x2fbin\x2ftcsh)/
-}
-
 # >0  string/wt,=#! /bin/nawk (len=12), ["new awk script text executable"], swap_endian=0
 signature file-magic-auto72 {
 	file-mime "text/x-nawk", 150
@@ -475,12 +379,6 @@ signature file-magic-auto73 {
 	file-magic /(\x23\x21 ?\x2fbin\x2fgawk)/
 }
 
-# >0  string/wt,=#! /bin/bash (len=12), ["Bourne-Again shell script text executable"], swap_endian=0
-signature file-magic-auto74 {
-	file-mime "text/x-shellscript", 150
-	file-magic /(\x23\x21 ?\x2fbin\x2fbash)/
-}
-
 # >0  string/wt,=#! /bin/awk (len=11), ["awk script text executable"], swap_endian=0
 signature file-magic-auto75 {
 	file-mime "text/x-awk", 140
@@ -505,24 +403,6 @@ signature file-magic-auto78 {
 	file-magic /(d8\x3aannounce)/
 }
 
-# >0  string/wt,=#! /bin/csh (len=11), ["C shell script text executable"], swap_endian=0
-signature file-magic-auto79 {
-	file-mime "text/x-shellscript", 140
-	file-magic /(\x23\x21 ?\x2fbin\x2fcsh)/
-}
-
-# >0  string/wt,=#! /bin/ksh (len=11), ["Korn shell script text executable"], swap_endian=0
-signature file-magic-auto80 {
-	file-mime "text/x-shellscript", 140
-	file-magic /(\x23\x21 ?\x2fbin\x2fksh)/
-}
-
-# >0  string/wt,=#! /bin/zsh (len=11), ["Paul Falstad's zsh script text executable"], swap_endian=0
-signature file-magic-auto81 {
-	file-mime "text/x-shellscript", 140
-	file-magic /(\x23\x21 ?\x2fbin\x2fzsh)/
-}
-
 # >0  string/c,=BEGIN:VCARD (len=11), ["vCard visiting card"], swap_endian=0
 signature file-magic-auto82 {
 	file-mime "text/x-vcard", 140
@@ -545,12 +425,6 @@ signature file-magic-auto84 {
 	file-magic /(Forward to)/
 }
 
-# >0  string/wt,=#! /bin/sh (len=10), ["POSIX shell script text executable"], swap_endian=0
-signature file-magic-auto85 {
-	file-mime "text/x-shellscript", 130
-	file-magic /(\x23\x21 ?\x2fbin\x2fsh)/
-}
-
 # >0  string,=II*\000\020\000\000\000CR (len=10), ["Canon CR2 raw image data"], swap_endian=0
 signature file-magic-auto86 {
 	file-mime "image/x-canon-cr2", 130
@@ -585,12 +459,6 @@ signature file-magic-auto90 {
 	file-magic /(\x3cBookFile)/
 }
 
-# >2112  string,=MSWordDoc (len=9), ["Microsoft Word document data"], swap_endian=0
-signature file-magic-auto91 {
-	file-mime "application/msword", 120
-	file-magic /(.{2112})(MSWordDoc)/
-}
-
 # >0  string/t,=N#! rnews (len=9), ["mailed, batched news text"], swap_endian=0
 signature file-magic-auto92 {
 	file-mime "message/rfc822", 120
@@ -656,12 +524,6 @@ signature file-magic-auto100 {
 	file-magic /(MSCF\x00\x00\x00\x00)/
 }
 
-# >0  string/b,=\320\317\021\340\241\261\032\341 (len=8), ["Microsoft Office Document"], swap_endian=0
-signature file-magic-auto101 {
-	file-mime "application/msword", 110
-	file-magic /(\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1)/
-}
-
 # >21  string/c,=!SCREAM! (len=8), ["Screamtracker 2 module sound data"], swap_endian=0
 signature file-magic-auto102 {
 	file-mime "audio/x-mod", 110
@@ -754,10 +616,10 @@ signature file-magic-auto116 {
 }
 
 # >257  string,=ustar  \000 (len=8), ["GNU tar archive"], swap_endian=0
-signature file-magic-auto117 {
+#signature file-magic-auto117 {
-	file-mime "application/x-tar", 110
+#	file-mime "application/x-tar", 110
-	file-magic /(.{257})(ustar  \x00)/
+#	file-magic /(.{257})(ustar  \x00)/
-}
+#}
 
 # >0  string,=<MIFFile (len=8), ["FrameMaker MIF (ASCII) file"], swap_endian=0
 signature file-magic-auto118 {
@@ -771,12 +633,6 @@ signature file-magic-auto119 {
 	file-magic /(PK\x07\x08PK\x03\x04)/
 }
 
-# >0  string/b,=\t\004\006\000\000\000\020\000 (len=8), ["Microsoft Excel Worksheet"], swap_endian=0
-signature file-magic-auto120 {
-	file-mime "application/vnd.ms-excel", 110
-	file-magic /(\x09\x04\x06\x00\x00\x00\x10\x00)/
-}
-
 # >0  string/b,=WordPro\000 (len=8), ["Lotus WordPro"], swap_endian=0
 signature file-magic-auto121 {
 	file-mime "application/vnd.lotus-wordpro", 110
@@ -844,10 +700,10 @@ signature file-magic-auto130 {
 }
 
 # >257  string,=ustar\000 (len=6), ["POSIX tar archive"], swap_endian=0
-signature file-magic-auto131 {
+#signature file-magic-auto131 {
-	file-mime "application/x-tar", 90
+#	file-mime "application/x-tar", 90
-	file-magic /(.{257})(ustar\x00)/
+#	file-magic /(.{257})(ustar\x00)/
-}
+#}
 
 # >0  string,=AC1.40 (len=6), ["DWG AutoDesk AutoCAD Release 1.40"], swap_endian=0
 signature file-magic-auto132 {
@@ -994,12 +850,6 @@ signature file-magic-auto155 {
 	file-magic /(\x23 xmcd)/
 }
 
-# >0  string/b,=\333\245-\000\000\000 (len=6), ["Microsoft Office Document"], swap_endian=0
-signature file-magic-auto156 {
-	file-mime "application/msword", 90
-	file-magic /(\xdb\xa5\x2d\x00\x00\x00)/
-}
-
 # >2  string,=MMXPR3 (len=6), ["Motorola Quark Express Document (English)"], swap_endian=0
 signature file-magic-auto157 {
 	file-mime "application/x-quark-xpress-3", 90
@@ -1046,36 +896,6 @@ signature file-magic-auto162 {
 	file-magic /(\x3c\x3fxml)(.{15})(.*)( xmlns\x3d)(['"]http:\x2f\x2fwww.opengis.net\x2fkml)/
 }
 
-# >0  string,=PK\003\004 (len=4), [""], swap_endian=0
-# >>30  regex,=[Content_Types].xml|_rels/.rels (len=31), [""], swap_endian=0
-# >>>18 (lelong,+49), search/2000,=PK\003\004 (len=4), [""], swap_endian=0
-# >>>>&26  search/1000,=PK\003\004 (len=4), [""], swap_endian=0
-# >>>>>&26  string,=word/ (len=5), ["Microsoft Word 2007+"], swap_endian=0
-signature file-magic-auto163 {
-	file-mime "application/vnd.openxmlformats-officedocument.wordprocessingml.document", 80
-	file-magic /(PK\x03\x04)(.{26})(\[Content_Types\].xml|_rels\x2f.rels)(.*)(PK\x03\x04)(.{26})(.*)(PK\x03\x04)(.{26})(word\x2f)/
-}
-
-# >0  string,=PK\003\004 (len=4), [""], swap_endian=0
-# >>30  regex,=[Content_Types].xml|_rels/.rels (len=31), [""], swap_endian=0
-# >>>18 (lelong,+49), search/2000,=PK\003\004 (len=4), [""], swap_endian=0
-# >>>>&26  search/1000,=PK\003\004 (len=4), [""], swap_endian=0
-# >>>>>&26  string,=ppt/ (len=4), ["Microsoft PowerPoint 2007+"], swap_endian=0
-signature file-magic-auto164 {
-	file-mime "application/vnd.openxmlformats-officedocument.presentationml.presentation", 70
-	file-magic /(PK\x03\x04)(.{26})(\[Content_Types\].xml|_rels\x2f.rels)(.*)(PK\x03\x04)(.{26})(.*)(PK\x03\x04)(.{26})(ppt\x2f)/
-}
-
-# >0  string,=PK\003\004 (len=4), [""], swap_endian=0
-# >>30  regex,=[Content_Types].xml|_rels/.rels (len=31), [""], swap_endian=0
-# >>>18 (lelong,+49), search/2000,=PK\003\004 (len=4), [""], swap_endian=0
-# >>>>&26  search/1000,=PK\003\004 (len=4), [""], swap_endian=0
-# >>>>>&26  string,=xl/ (len=3), ["Microsoft Excel 2007+"], swap_endian=0
-signature file-magic-auto165 {
-	file-mime "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", 60
-	file-magic /(PK\x03\x04)(.{26})(\[Content_Types\].xml|_rels\x2f.rels)(.*)(PK\x03\x04)(.{26})(.*)(PK\x03\x04)(.{26})(xl\x2f)/
-}
-
 # >60  string,=RINEX (len=5), [""], swap_endian=0
 # >>80  search/256,=XXRINEXB (len=8), ["RINEX Data, GEO SBAS Broadcast"], swap_endian=0
 # >>>5  string,x, [", version %6.6s"], swap_endian=0
@@ -1229,30 +1049,12 @@ signature file-magic-auto187 {
 	file-magic /(\x00\x01\x00\x00\x00)/
 }
 
-# >0  string/b,=PO^Q` (len=5), ["Microsoft Word 6.0 Document"], swap_endian=0
-signature file-magic-auto188 {
-	file-mime "application/msword", 80
-	file-magic /(PO\x5eQ\x60)/
-}
-
 # >0  string,=%PDF- (len=5), ["PDF document"], swap_endian=0
 signature file-magic-auto189 {
 	file-mime "application/pdf", 80
 	file-magic /(\x25PDF\x2d)/
 }
 
-# >2114  string,=Biff5 (len=5), ["Microsoft Excel 5.0 Worksheet"], swap_endian=0
-signature file-magic-auto190 {
-	file-mime "application/vnd.ms-excel", 80
-	file-magic /(.{2114})(Biff5)/
-}
-
-# >2121  string,=Biff5 (len=5), ["Microsoft Excel 5.0 Worksheet"], swap_endian=0
-signature file-magic-auto191 {
-	file-mime "application/vnd.ms-excel", 80
-	file-magic /(.{2121})(Biff5)/
-}
-
 # >0  string/t,=Path: (len=5), ["news text"], swap_endian=0
 signature file-magic-auto192 {
 	file-mime "message/news", 80
@@ -1383,12 +1185,6 @@ signature file-magic-auto211 {
 	file-magic /(\x00\x00\x00\x01)([\x07\x27\x47\x67\x87\xa7\xc7\xe7])/
 }
 
-# >0  belong&,=-889275714 (0xcafebabe), [""], swap_endian=0
-signature file-magic-auto212 {
-	file-mime "application/x-java-applet", 71
-	file-magic /(\xca\xfe\xba\xbe)/
-}
-
 # >0  belong&ffffffffffffff00,=256 (0x00000100), [""], swap_endian=0
 # >>3  byte&,=0xba, ["MPEG sequence"], swap_endian=0
 signature file-magic-auto213 {
@@ -1706,46 +1502,6 @@ signature file-magic-auto245 {
 	file-magic /(PK\x03\x04)(.{22})(\x08\x00\x00\x00mimetypeapplication\x2f)(epub\x2bzip)/
 }
 
-# Seems redundant with other zip signature below.
-# >0  string,=PK\003\004 (len=4), [""], swap_endian=0
-# >>26  string,=\b\000\000\000mimetypeapplication/ (len=24), [""], swap_endian=0
-# >>>50  string,!epub+zip (len=8), [""], swap_endian=0
-# >>>>50  string,!vnd.oasis.opendocument. (len=23), [""], swap_endian=0
-# >>>>>50  string,!vnd.sun.xml. (len=12), [""], swap_endian=0
-# >>>>>>50  string,!vnd.kde. (len=8), [""], swap_endian=0
-# >>>>>>>38  regex,=[!-OQ-~]+ (len=9), ["Zip data (MIME type "%s"?)"], swap_endian=0
-#signature file-magic-auto246 {
-#	file-mime "application/zip", 39
-#	file-magic /(PK\x03\x04)(.{22})(\x08\x00\x00\x00mimetypeapplication\x2f)/
-#}
-
-# >0  string,=PK\003\004 (len=4), [""], swap_endian=0
-# >>26  string,=\b\000\000\000mimetype (len=12), [""], swap_endian=0
-# >>>38  string,!application/ (len=12), [""], swap_endian=0
-# >>>>38  regex,=[!-OQ-~]+ (len=9), ["Zip data (MIME type "%s"?)"], swap_endian=0
-signature file-magic-auto247 {
-	file-mime "application/zip", 39
-	file-magic /(PK\x03\x04)(.{22})(\x08\x00\x00\x00mimetype)/
-}
-
-# The indirect offset makes this difficult to convert.
-# The (.*) may be too generous.
-# >0  string,=PK\003\004 (len=4), [""], swap_endian=0
-# >>26 (leshort,+30), leshort&,=-13570 (0xcafe), ["Java archive data (JAR)"], swap_endian=0
-signature file-magic-auto248 {
-	file-mime "application/java-archive", 50
-	file-magic /(PK\x03\x04)(.*)(\xfe\xca)/
-}
-
-# The indeirect offset and string inequality make this difficult to convert.
-# >0  string,=PK\003\004 (len=4), [""], swap_endian=0
-# >>26 (leshort,+30), leshort&,!-13570 (0xcafe), [""], swap_endian=0
-# >>>26  string,!\b\000\000\000mimetype (len=12), ["Zip archive data"], swap_endian=0
-signature file-magic-auto249 {
-	file-mime "application/zip", 10
-	file-magic /(PK\x03\x04)(.{2})/
-}
-
 # >0  belong&,=442 (0x000001ba), [""], swap_endian=0
 # >>4  byte&,&0x40, [""], swap_endian=0
 signature file-magic-auto250 {
@@ -2065,18 +1821,6 @@ signature file-magic-auto299 {
 	file-magic /(PDN3)/
 }
 
-# >0  ulelong&,=2712847316 (0xa1b2c3d4), ["tcpdump capture file (little-endian)"], swap_endian=0
-signature file-magic-auto300 {
-	file-mime "application/vnd.tcpdump.pcap", 70
-	file-magic /(\xd4\xc3\xb2\xa1)/
-}
-
-# >0  ubelong&,=2712847316 (0xa1b2c3d4), ["tcpdump capture file (big-endian)"], swap_endian=0
-signature file-magic-auto301 {
-	file-mime "application/vnd.tcpdump.pcap", 70
-	file-magic /(\xa1\xb2\xc3\xd4)/
-}
-
 # >0  belong&,=-17957139 (0xfeedfeed), ["Java KeyStore"], swap_endian=0
 signature file-magic-auto302 {
 	file-mime "application/x-java-keystore", 70
@@ -2297,12 +2041,6 @@ signature file-magic-auto335 {
 	file-magic /(SIT\x21)/
 }
 
-# >0  lelong&,=574529400 (0x223e9f78), ["Transport Neutral Encapsulation Format"], swap_endian=0
-signature file-magic-auto336 {
-	file-mime "application/vnd.ms-tnef", 70
-	file-magic /(\x78\x9f\x3e\x22)/
-}
-
 # >0  string,=<ar> (len=4), ["System V Release 1 ar archive"], swap_endian=0
 signature file-magic-auto337 {
 	file-mime "application/x-archive", 70
@@ -2433,48 +2171,6 @@ signature file-magic-auto357 {
 	file-magic /(RIFF)(.{4})(AVI )/
 }
 
-# >0  belong&,=834535424 (0x31be0000), ["Microsoft Word Document"], swap_endian=0
-signature file-magic-auto358 {
-	file-mime "application/msword", 70
-	file-magic /(\x31\xbe\x00\x00)/
-}
-
-# >0  string/b,=\3767\000# (len=4), ["Microsoft Office Document"], swap_endian=0
-signature file-magic-auto359 {
-	file-mime "application/msword", 70
-	file-magic /(\xfe7\x00\x23)/
-}
-
-# >0  string/b,=\333\245-\000 (len=4), ["Microsoft WinWord 2.0 Document"], swap_endian=0
-signature file-magic-auto360 {
-	file-mime "application/msword", 70
-	file-magic /(\xdb\xa5\x2d\x00)/
-}
-
-# >0  string/b,=\333\245-\000 (len=4), ["Microsoft WinWord 2.0 Document"], swap_endian=0
-signature file-magic-auto361 {
-	file-mime "application/msword", 70
-	file-magic /(\xdb\xa5\x2d\x00)/
-}
-
-# >0  belong&,=6656 (0x00001a00), ["Lotus 1-2-3"], swap_endian=0
-signature file-magic-auto362 {
-	file-mime "application/x-123", 70
-	file-magic /(\x00\x00\x1a\x00)/
-}
-
-# >0  belong&,=512 (0x00000200), ["Lotus 1-2-3"], swap_endian=0
-signature file-magic-auto363 {
-	file-mime "application/x-123", 70
-	file-magic /(\x00\x00\x02\x00)/
-}
-
-# >0  string/b,=\000\000\001\000 (len=4), ["MS Windows icon resource"], swap_endian=0
-signature file-magic-auto364 {
-	file-mime "image/x-icon", 70
-	file-magic /(\x00\x00\x01\x00)/
-}
-
 # >0  lelong&,=268435536 (0x10000050), ["Psion Series 5"], swap_endian=0
 # >>4  lelong&,=268435565 (0x1000006d), ["database"], swap_endian=0
 # >>>8  lelong&,=268435588 (0x10000084), ["Agenda file"], swap_endian=0
@@ -2737,12 +2433,6 @@ signature file-magic-auto403 {
 	file-magic /(SBI)/
 }
 
-# >0  string/b,=\224\246. (len=3), ["Microsoft Word Document"], swap_endian=0
-signature file-magic-auto404 {
-	file-mime "application/msword", 60
-	file-magic /(\x94\xa6\x2e)/
-}
-
 # >0  string,=\004%! (len=3), ["PostScript document text"], swap_endian=0
 signature file-magic-auto405 {
 	file-mime "application/postscript", 60
@@ -2763,17 +2453,11 @@ signature file-magic-auto407 {
 	file-magic /(.*)([ \x09]*(class|module)[ \x09][A-Z])((modul|includ)e [A-Z]|def [a-z])(^[ \x09]*end([ \x09]*[;#].*)?$)/
 }
 
-# >512  string/b,=\354\245\301 (len=3), ["Microsoft Word Document"], swap_endian=0
-signature file-magic-auto408 {
-	file-mime "application/msword", 60
-	file-magic /(.{512})(\xec\xa5\xc1)/
-}
-
 # >0  regex/20,=^\.[A-Za-z0-9][A-Za-z0-9][ \t] (len=29), ["troff or preprocessor input text"], swap_endian=0
-signature file-magic-auto411 {
+#signature file-magic-auto411 {
-	file-mime "text/troff", 59
+#	file-mime "text/troff", 59
-	file-magic /(^\.[A-Za-z0-9][A-Za-z0-9][ \x09])/
+#	file-magic /(^\.[A-Za-z0-9][A-Za-z0-9][ \x09])/
-}
+#}
 
 # >0  search/4096,=\documentclass (len=14), ["LaTeX 2e document text"], swap_endian=0
 signature file-magic-auto412 {
@@ -2806,10 +2490,10 @@ signature file-magic-auto416 {
 }
 
 # >0  regex/20,=^\.[A-Za-z0-9][A-Za-z0-9]$ (len=26), ["troff or preprocessor input text"], swap_endian=0
-signature file-magic-auto417 {
+#signature file-magic-auto417 {
-	file-mime "text/troff", 56
+#	file-mime "text/troff", 56
-	file-magic /(^\.[A-Za-z0-9][A-Za-z0-9]$)/
+#	file-magic /(^\.[A-Za-z0-9][A-Za-z0-9]$)/
-}
+#}
 
 # >0  search/w/1,=#! /usr/bin/php (len=15), ["PHP script text executable"], swap_endian=0
 signature file-magic-auto418 {
@@ -2829,30 +2513,12 @@ signature file-magic-auto420 {
 	file-magic /(.*)(eval \x22exec \x2fusr\x2fbin\x2fperl)/
 }
 
-# >0  search/w/1,=#! /usr/local/bin/python (len=24), ["Python script text executable"], swap_endian=0
-signature file-magic-auto421 {
-	file-mime "text/x-python", 54
-	file-magic /(.*)(\x23\x21 ?\x2fusr\x2flocal\x2fbin\x2fpython)/
-}
-
 # >0  search/1,=Common subdirectories:  (len=23), ["diff output text"], swap_endian=0
 signature file-magic-auto422 {
 	file-mime "text/x-diff", 53
 	file-magic /(.*)(Common subdirectories\x3a )/
 }
 
-# >0  search/1,=#! /usr/bin/env python (len=22), ["Python script text executable"], swap_endian=0
-signature file-magic-auto423 {
-	file-mime "text/x-python", 52
-	file-magic /(.*)(\x23\x21 \x2fusr\x2fbin\x2fenv python)/
-}
-
-# >0  search/w/1,=#! /usr/local/bin/ruby (len=22), ["Ruby script text executable"], swap_endian=0
-signature file-magic-auto424 {
-	file-mime "text/x-ruby", 52
-	file-magic /(.*)(\x23\x21 ?\x2fusr\x2flocal\x2fbin\x2fruby)/
-}
-
 # >0  search/w/1,=#! /usr/local/bin/wish (len=22), ["Tcl/Tk script text executable"], swap_endian=0
 signature file-magic-auto425 {
 	file-mime "text/x-tcl", 52
@@ -2871,12 +2537,6 @@ signature file-magic-auto427 {
 	file-magic /(\xff\xd8)/
 }
 
-# >0  search/1,=#!/usr/bin/env python (len=21), ["Python script text executable"], swap_endian=0
-signature file-magic-auto428 {
-	file-mime "text/x-python", 51
-	file-magic /(.*)(\x23\x21\x2fusr\x2fbin\x2fenv python)/
-}
-
 # >0  search/1,=#!/usr/bin/env nodejs (len=21), ["Node.js script text executable"], swap_endian=0
 signature file-magic-auto429 {
 	file-mime "application/javascript", 51
@@ -3189,12 +2849,6 @@ signature file-magic-auto474 {
 	file-magic /(\x25\x21)/
 }
 
-# >0  search/1,=#! /usr/bin/env ruby (len=20), ["Ruby script text executable"], swap_endian=0
-signature file-magic-auto475 {
-	file-mime "text/x-ruby", 50
-	file-magic /(.*)(\x23\x21 \x2fusr\x2fbin\x2fenv ruby)/
-}
-
 # >0  regex/1,=(^[0-9]{5})[acdn][w] (len=20), ["MARC21 Classification"], swap_endian=0
 signature file-magic-auto476 {
 	file-mime "application/marc", 50
@@ -3228,10 +2882,10 @@ signature file-magic-auto480 {
 }
 
 # >0  string,=\n( (len=2), ["Emacs v18 byte-compiled Lisp data"], swap_endian=0
-signature file-magic-auto481 {
+#signature file-magic-auto481 {
-	file-mime "application/x-elc", 50
+#	file-mime "application/x-elc", 50
-	file-magic /(\x0a\x28)/
+#	file-magic /(\x0a\x28)/
-}
+#}
 
 # >0  string,=\021\t (len=2), ["Award BIOS Logo, 136 x 126"], swap_endian=0
 signature file-magic-auto482 {
@@ -3305,17 +2959,17 @@ signature file-magic-auto493 {
 	file-magic /(\xf7\x02)/
 }
 
-# >2  string,=\000\021 (len=2), ["TeX font metric data"], swap_endian=0
+## >2  string,=\000\021 (len=2), ["TeX font metric data"], swap_endian=0
-signature file-magic-auto494 {
+#signature file-magic-auto494 {
-	file-mime "application/x-tex-tfm", 50
+#	file-mime "application/x-tex-tfm", 50
-	file-magic /(.{2})(\x00\x11)/
+#	file-magic /(.{2})(\x00\x11)/
-}
+#}
-
+#
-# >2  string,=\000\022 (len=2), ["TeX font metric data"], swap_endian=0
+## >2  string,=\000\022 (len=2), ["TeX font metric data"], swap_endian=0
-signature file-magic-auto495 {
+#signature file-magic-auto495 {
-	file-mime "application/x-tex-tfm", 50
+#	file-mime "application/x-tex-tfm", 50
-	file-magic /(.{2})(\x00\x12)/
+#	file-magic /(.{2})(\x00\x12)/
-}
+#}
 
 # >0  beshort&,=-31486 (0x8502), ["GPG encrypted data"], swap_endian=0
 signature file-magic-auto496 {
@@ -3470,12 +3124,6 @@ signature file-magic-auto514 {
 	file-magic /(.*)(\x23\x21 \x2fusr\x2fbin\x2fenv lua)/
 }
 
-# >0  search/1,=#!/usr/bin/env ruby (len=19), ["Ruby script text executable"], swap_endian=0
-signature file-magic-auto515 {
-	file-mime "text/x-ruby", 49
-	file-magic /(.*)(\x23\x21\x2fusr\x2fbin\x2fenv ruby)/
-}
-
 # >0  search/1,=#! /usr/bin/env tcl (len=19), ["Tcl script text executable"], swap_endian=0
 signature file-magic-auto516 {
 	file-mime "text/x-tcl", 49
@@ -3493,12 +3141,6 @@ signature file-magic-auto519 {
 	file-magic /(.*)(\x23\x21\x2fusr\x2fbin\x2fenv lua)/
 }
 
-# >0  search/w/1,=#! /usr/bin/python (len=18), ["Python script text executable"], swap_endian=0
-signature file-magic-auto520 {
-	file-mime "text/x-python", 48
-	file-magic /(.*)(\x23\x21 ?\x2fusr\x2fbin\x2fpython)/
-}
-
 # >0  search/w/1,=#!/usr/bin/nodejs (len=17), ["Node.js script text executable"], swap_endian=0
 signature file-magic-auto521 {
 	file-mime "application/javascript", 47
@@ -3506,10 +3148,10 @@ signature file-magic-auto521 {
 }
 
 # >0  regex,=^class[ \t\n]+ (len=12), ["C++ source text"], swap_endian=0
-signature file-magic-auto522 {
+#signature file-magic-auto522 {
-	file-mime "text/x-c++", 47
+#	file-mime "text/x-c++", 47
-	file-magic /(.*)(class[ \x09\x0a]+[[:alnum:]_]+)(.*)(\x7b)(.*)(public:)/
+#	file-magic /(.*)(class[ \x09\x0a]+[[:alnum:]_]+)(.*)(\x7b)(.*)(public:)/
-}
+#}
 
 # >0  search/1,=This is Info file (len=17), ["GNU Info text"], swap_endian=0
 signature file-magic-auto528 {
@@ -3658,12 +3300,6 @@ signature file-magic-auto545 {
 	file-magic /(.*)(\x23\x21 ?\x2fusr\x2fbin\x2fwish)/
 }
 
-# >0  search/w/1,=#! /usr/bin/ruby (len=16), ["Ruby script text executable"], swap_endian=0
-signature file-magic-auto546 {
-	file-mime "text/x-ruby", 46
-	file-magic /(.*)(\x23\x21 ?\x2fusr\x2fbin\x2fruby)/
-}
-
 # >0  search/w/1,=#! /usr/bin/lua (len=15), ["Lua script text executable"], swap_endian=0
 signature file-magic-auto547 {
 	file-mime "text/x-lua", 45
@@ -3727,10 +3363,10 @@ signature file-magic-auto556 {
 }
 
 # >0  regex,=^extern[ \t\n]+ (len=13), ["C source text"], swap_endian=0
-signature file-magic-auto557 {
+#signature file-magic-auto557 {
-	file-mime "text/x-c", 43
+#	file-mime "text/x-c", 43
-	file-magic /(.*)(extern[ \x09\x0a]+)/
+#	file-magic /(.*)(extern[ \x09\x0a]+)/
-}
+#}
 
 # >0  search/4096,=% -*-latex-*- (len=13), ["LaTeX document text"], swap_endian=0
 signature file-magic-auto558 {
@@ -3746,10 +3382,10 @@ signature file-magic-auto558 {
 #}
 
 # >0  regex,=^struct[ \t\n]+ (len=13), ["C source text"], swap_endian=0
-signature file-magic-auto560 {
+#signature file-magic-auto560 {
-	file-mime "text/x-c", 43
+#	file-mime "text/x-c", 43
-	file-magic /(.*)(struct[ \x09\x0a]+)/
+#	file-magic /(.*)(struct[ \x09\x0a]+)/
-}
+#}
 
 # >0  search/w/1,=#!/bin/nodejs (len=13), ["Node.js script text executable"], swap_endian=0
 signature file-magic-auto561 {
@@ -3802,10 +3438,10 @@ signature file-magic-auto567 {
 }
 
 # >0  regex,=^char[ \t\n]+ (len=11), ["C source text"], swap_endian=0
-signature file-magic-auto568 {
+#signature file-magic-auto568 {
-	file-mime "text/x-c", 41
+#	file-mime "text/x-c", 41
-	file-magic /(.*)(char[ \x09\x0a]+)/
+#	file-magic /(.*)(char[ \x09\x0a]+)/
-}
+#}
 
 # >0  search/1,=#! (len=2), [""], swap_endian=0
 # >>0  regex,=^#!.*/bin/perl$ (len=15), ["Perl script text executable"], swap_endian=0
@@ -3887,23 +3523,11 @@ signature file-magic-auto578 {
 	file-magic /(^dnl )/
 }
 
-# >0  regex,=^all: (len=5), ["makefile script text"], swap_endian=0
-signature file-magic-auto579 {
-	file-mime "text/x-makefile", 40
-	file-magic /(^all:)/
-}
-
-# >0  regex,=^.PRECIOUS (len=10), ["makefile script text"], swap_endian=0
-signature file-magic-auto580 {
-	file-mime "text/x-makefile", 40
-	file-magic /(^.PRECIOUS)/
-}
-
 # >0  search/8192,=main( (len=5), ["C source text"], swap_endian=0
-signature file-magic-auto581 {
+#signature file-magic-auto581 {
-	file-mime "text/x-c", 40
+#	file-mime "text/x-c", 40
-	file-magic /(.*)(main\x28)/
+#	file-magic /(.*)(main\x28)/
-}
+#}
 
 # Not specific enough.
 # >0  search/1,=\" (len=2), ["troff or preprocessor input text"], swap_endian=0
@@ -3932,22 +3556,22 @@ signature file-magic-auto584 {
 #}
 
 # >0  regex,=^#include (len=9), ["C source text"], swap_endian=0
-signature file-magic-auto586 {
+#signature file-magic-auto586 {
-	file-mime "text/x-c", 39
+#	file-mime "text/x-c", 39
-	file-magic /(.*)(#include)/
+#	file-magic /(.*)(#include)/
-}
+#}
 
 # >0  search/1,=.\" (len=3), ["troff or preprocessor input text"], swap_endian=0
-signature file-magic-auto587 {
+#signature file-magic-auto587 {
-	file-mime "text/troff", 39
+#	file-mime "text/troff", 39
-	file-magic /(.*)(\x2e\x5c\x22)/
+#	file-magic /(.*)(\x2e\x5c\x22)/
-}
+#}
 
 # >0  search/1,='\" (len=3), ["troff or preprocessor input text"], swap_endian=0
-signature file-magic-auto588 {
+#signature file-magic-auto588 {
-	file-mime "text/troff", 39
+#	file-mime "text/troff", 39
-	file-magic /(.*)(\x27\x5c\x22)/
+#	file-magic /(.*)(\x27\x5c\x22)/
-}
+#}
 
 # >0  search/1,=<TeXmacs| (len=9), ["TeXmacs document text"], swap_endian=0
 signature file-magic-auto589 {
@@ -3974,10 +3598,10 @@ signature file-magic-auto592 {
 }
 
 # >0  search/1,=''' (len=3), ["troff or preprocessor input text"], swap_endian=0
-signature file-magic-auto593 {
+#signature file-magic-auto593 {
-	file-mime "text/troff", 39
+#	file-mime "text/troff", 39
-	file-magic /(.*)(\x27\x27\x27)/
+#	file-magic /(.*)(\x27\x27\x27)/
-}
+#}
 
 # >0  search/4096,=try: (len=4), [""], swap_endian=0
 # >>&0  regex,=^\s*except.*: (len=13), ["Python script text executable"], swap_endian=0
@@ -3999,12 +3623,6 @@ signature file-magic-auto596 {
 	file-magic /(.*)(\x22LIBHDR\x22)/
 }
 
-# >0  regex,=^SUBDIRS (len=8), ["automake makefile script text"], swap_endian=0
-signature file-magic-auto597 {
-	file-mime "text/x-makefile", 38
-	file-magic /(.*)(SUBDIRS)/
-}
-
 # >0  search/4096,=(defvar  (len=8), ["Lisp/Scheme program text"], swap_endian=0
 signature file-magic-auto598 {
 	file-mime "text/x-lisp", 38
@@ -4031,19 +3649,6 @@ signature file-magic-auto600 {
 #	file-magic /(.*)(\x2a\x2a\x2a )/
 #}
 
-# >0  search/1,='.\" (len=4), ["troff or preprocessor input text"], swap_endian=0
-signature file-magic-auto602 {
-	file-mime "text/troff", 38
-	file-magic /(.*)(\x27\x2e\x5c\x22)/
-}
-
-# LDFLAGS appears in other contexts, e.g. shell script.
-# >0  regex,=^LDFLAGS (len=8), ["makefile script text"], swap_endian=0
-#signature file-magic-auto603 {
-#	file-mime "text/x-makefile", 38
-#	file-magic /(.*)(LDFLAGS)/
-#}
-
 # >0  search/8192,="libhdr" (len=8), ["BCPL source text"], swap_endian=0
 signature file-magic-auto604 {
 	file-mime "text/x-bcpl", 38
@@ -4057,12 +3662,6 @@ signature file-magic-auto604 {
 #	file-magic /(^record)/
 #}
 
-# >0  regex,=^CFLAGS (len=7), ["makefile script text"], swap_endian=0
-signature file-magic-auto606 {
-	file-mime "text/x-makefile", 37
-	file-magic /(.*)(CFLAGS)/
-}
-
 # >0  search/4096,=(defun  (len=7), ["Lisp/Scheme program text"], swap_endian=0
 signature file-magic-auto607 {
 	file-mime "text/x-lisp", 37

scripts/base/frameworks/files/magic/msoffice.sig

@@ -0,0 +1,28 @@
+
+# This signature is non-specific and terrible but after
+# searching for a long time there doesn't seem to be a 
+# better option.  
+signature file-msword {
+	file-magic /^\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1/
+	file-mime "application/msword", 50
+}
+
+signature file-ooxml {
+	file-magic /^PK\x03\x04\x14\x00\x06\x00/
+	file-mime "application/vnd.openxmlformats-officedocument", 50
+}
+
+signature file-docx {
+	file-magic /^PK\x03\x04.{26}(\[Content_Types\]\.xml|_rels\x2f\.rels|word\x2f).*PK\x03\x04.{26}word\x2f/
+	file-mime "application/vnd.openxmlformats-officedocument.wordprocessingml.document", 80
+}
+
+signature file-xlsx {
+	file-magic /^PK\x03\x04.{26}(\[Content_Types\]\.xml|_rels\x2f\.rels|xl\2f).*PK\x03\x04.{26}xl\x2f/
+	file-mime "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", 80
+}
+
+signature file-pptx {
+	file-magic /^PK\x03\x04.{26}(\[Content_Types\]\.xml|_rels\x2f\.rels|ppt\x2f).*PK\x03\x04.{26}ppt\x2f/
+	file-mime "application/vnd.openxmlformats-officedocument.presentationml.presentation", 80
+}

scripts/base/frameworks/files/main.bro

@@ -100,8 +100,9 @@ export {
 		## during the process of analysis e.g. due to dropped packets.
 		missing_bytes: count &log &default=0;
 
-		## The number of not all-in-sequence bytes in the file stream that
+		## The number of bytes in the file stream that were not delivered to
-		## were delivered to file analyzers due to reassembly buffer overflow.
+		## stream file analyzers.  This could be overlapping bytes or 
+		## bytes that couldn't be reassembled.
 		overflow_bytes: count &log &default=0;
 
 		## Whether the file analysis timed out at least once for the file.
@@ -124,6 +125,37 @@ export {
 	## generate two handles that would hash to the same file id.
 	const salt = "I recommend changing this." &redef;
 
+	## Decide if you want to automatically attached analyzers to 
+	## files based on the detected mime type of the file.
+	const analyze_by_mime_type_automatically = T &redef;
+
+	## The default setting for if the file reassembler is enabled for 
+	## each file.
+	const enable_reassembler = T &redef;
+
+	## The default per-file reassembly buffer size.
+	const reassembly_buffer_size = 1048576 &redef;
+
+	## Allows the file reassembler to be used if it's necessary because the
+	## file is transferred out of order.
+	##
+	## f: the file.
+	global enable_reassembly: function(f: fa_file);
+
+	## Disables the file reassembler on this file.  If the file is not 
+	## transferred out of order this will have no effect.
+	##
+	## f: the file.
+	global disable_reassembly: function(f: fa_file);
+
+	## Set the maximum size the reassembly buffer is allowed to grow
+	## for the given file.
+	##
+	## f: the file.
+	##
+	## max: Maximum allowed size of the reassembly buffer.
+	global set_reassembly_buffer_size: function(f: fa_file, max: count);
+
 	## Sets the *timeout_interval* field of :bro:see:`fa_file`, which is
 	## used to determine the length of inactivity that is allowed for a file
 	## before internal state related to it is cleaned up.  When used within
@@ -153,15 +185,6 @@ export {
 	                              tag: Files::Tag,
 	                              args: AnalyzerArgs &default=AnalyzerArgs()): bool;
 
-	## Adds all analyzers associated with a give MIME type to the analysis of
-	## a file.  Note that analyzers added via MIME types cannot take further
-	## arguments.
-	##
-	## f: the file.
-	##
-	## mtype: the MIME type; it will be compared case-insensitive.
-	global add_analyzers_for_mime_type: function(f: fa_file, mtype: string);
-
 	## Removes an analyzer from the analysis of a given file.
 	##
 	## f: the file.
@@ -244,7 +267,7 @@ export {
 	## mts: The set of MIME types, each in the form "foo/bar" (case-insensitive).
 	##
 	## Returns: True if the MIME types were successfully registered.
-	global register_for_mime_types: function(tag: Analyzer::Tag, mts: set[string]) : bool;
+	global register_for_mime_types: function(tag: Files::Tag, mts: set[string]) : bool;
 
 	## Registers a MIME type for an analyzer. If a future file with this type is seen,
 	## the analyzer will be automatically assigned to parsing it. The function *adds*
@@ -255,20 +278,20 @@ export {
 	## mt: The MIME type in the form "foo/bar" (case-insensitive).
 	##
 	## Returns: True if the MIME type was successfully registered.
-	global register_for_mime_type: function(tag: Analyzer::Tag, mt: string) : bool;
+	global register_for_mime_type: function(tag: Files::Tag, mt: string) : bool;
 
 	## Returns a set of all MIME types currently registered for a specific analyzer.
 	##
 	## tag: The tag of the analyzer.
 	##
 	## Returns: The set of MIME types.
-	global registered_mime_types: function(tag: Analyzer::Tag) : set[string];
+	global registered_mime_types: function(tag: Files::Tag) : set[string];
 
 	## Returns a table of all MIME-type-to-analyzer mappings currently registered.
 	##
 	## Returns: A table mapping each analyzer to the set of MIME types
 	##          registered for it.
-	global all_registered_mime_types: function() : table[Analyzer::Tag] of set[string];
+	global all_registered_mime_types: function() : table[Files::Tag] of set[string];
 
 	## Event that can be handled to access the Info record as it is sent on
 	## to the logging framework.
@@ -283,13 +306,14 @@ redef record fa_file += {
 global registered_protocols: table[Analyzer::Tag] of ProtoRegistration = table();
 
 # Store the MIME type to analyzer mappings.
-global mime_types: table[Analyzer::Tag] of set[string];
+global mime_types: table[Files::Tag] of set[string];
+global mime_type_to_analyzers: table[string] of set[Files::Tag];
 
 global analyzer_add_callbacks: table[Files::Tag] of function(f: fa_file, args: AnalyzerArgs) = table();
 
 event bro_init() &priority=5
 	{
-	Log::create_stream(Files::LOG, [$columns=Info, $ev=log_files]);
+	Log::create_stream(Files::LOG, [$columns=Info, $ev=log_files, $path="files"]);
 	}
 
 function set_info(f: fa_file)
@@ -313,8 +337,6 @@ function set_info(f: fa_file)
 	f$info$overflow_bytes = f$overflow_bytes;
 	if ( f?$is_orig )
 		f$info$is_orig = f$is_orig;
-	if ( f?$mime_type )
-		f$info$mime_type = f$mime_type;
 	}
 
 function set_timeout_interval(f: fa_file, t: interval): bool
@@ -322,6 +344,21 @@ function set_timeout_interval(f: fa_file, t: interval): bool
 	return __set_timeout_interval(f$id, t);
 	}
 
+function enable_reassembly(f: fa_file)
+	{
+	__enable_reassembly(f$id);
+	}
+
+function disable_reassembly(f: fa_file)
+	{
+	__disable_reassembly(f$id);
+	}
+
+function set_reassembly_buffer_size(f: fa_file, max: count)
+	{
+	__set_reassembly_buffer(f$id, max);
+	}
+
 function add_analyzer(f: fa_file, tag: Files::Tag, args: AnalyzerArgs): bool
 	{
 	add f$info$analyzers[Files::analyzer_name(tag)];
@@ -337,15 +374,6 @@ function add_analyzer(f: fa_file, tag: Files::Tag, args: AnalyzerArgs): bool
 	return T;
 	}
 
-function add_analyzers_for_mime_type(f: fa_file, mtype: string)
-	{
-	local dummy_args: AnalyzerArgs;
-	local analyzers = __add_analyzers_for_mime_type(f$id, mtype, dummy_args);
-
-	for ( tag in analyzers )
-		add f$info$analyzers[Files::analyzer_name(tag)];
-	}
-
 function register_analyzer_add_callback(tag: Files::Tag, callback: function(f: fa_file, args: AnalyzerArgs))
 	{
 	analyzer_add_callbacks[tag] = callback;
@@ -366,17 +394,87 @@ function analyzer_name(tag: Files::Tag): string
 	return __analyzer_name(tag);
 	}
 
+function register_protocol(tag: Analyzer::Tag, reg: ProtoRegistration): bool
+	{
+	local result = (tag !in registered_protocols);
+	registered_protocols[tag] = reg;
+	return result;
+	}
+
+function register_for_mime_types(tag: Files::Tag, mime_types: set[string]) : bool
+	{
+	local rc = T;
+
+	for ( mt in mime_types )
+		{
+		if ( ! register_for_mime_type(tag, mt) )
+			rc = F;
+		}
+
+	return rc;
+	}
+
+function register_for_mime_type(tag: Files::Tag, mt: string) : bool
+	{
+	if ( tag !in mime_types )
+		{
+		mime_types[tag] = set();
+		}
+	add mime_types[tag][mt];
+
+	if ( mt !in mime_type_to_analyzers )
+		{
+		mime_type_to_analyzers[mt] = set();
+		}
+	add mime_type_to_analyzers[mt][tag];
+
+	return T;
+	}
+
+function registered_mime_types(tag: Files::Tag) : set[string]
+	{
+	return tag in mime_types ? mime_types[tag] : set();
+	}
+
+function all_registered_mime_types(): table[Files::Tag] of set[string]
+	{
+	return mime_types;
+	}
+
+function describe(f: fa_file): string
+	{
+	local tag = Analyzer::get_tag(f$source);
+	if ( tag !in registered_protocols )
+		return "";
+
+	local handler = registered_protocols[tag];
+	return handler$describe(f);
+	}
+
+event get_file_handle(tag: Files::Tag, c: connection, is_orig: bool) &priority=5
+	{
+	if ( tag !in registered_protocols )
+		return;
+
+	local handler = registered_protocols[tag];
+	set_file_handle(handler$get_file_handle(c, is_orig));
+	}
+
 event file_new(f: fa_file) &priority=10
 	{
 	set_info(f);
 
-	if ( f?$mime_type )
+	if ( enable_reassembler )
-		add_analyzers_for_mime_type(f, f$mime_type);
+		{
+		Files::enable_reassembly(f);
+		Files::set_reassembly_buffer_size(f, reassembly_buffer_size);
+		}
 	}
 
 event file_over_new_connection(f: fa_file, c: connection, is_orig: bool) &priority=10
 	{
 	set_info(f);
+
 	add f$info$conn_uids[c$uid];
 	local cid = c$id;
 	add f$info$tx_hosts[f$is_orig ? cid$orig_h : cid$resp_h];
@@ -386,6 +484,24 @@ event file_over_new_connection(f: fa_file, c: connection, is_orig: bool) &priori
 	add f$info$rx_hosts[f$is_orig ? cid$resp_h : cid$orig_h];
 	}
 
+event file_mime_type(f: fa_file, mime_type: string) &priority=10
+	{
+	set_info(f);
+
+	f$info$mime_type = mime_type;
+
+	if ( analyze_by_mime_type_automatically &&
+	     mime_type in mime_type_to_analyzers )
+		{
+		local analyzers = mime_type_to_analyzers[mime_type];
+		for ( a in analyzers )
+			{
+			add f$info$analyzers[Files::analyzer_name(a)];
+			Files::add_analyzer(f, a);
+			}
+		}
+	}
+
 event file_timeout(f: fa_file) &priority=10
 	{
 	set_info(f);
@@ -401,64 +517,3 @@ event file_state_remove(f: fa_file) &priority=-10
 	{
 	Log::write(Files::LOG, f$info);
 	}
-
-function register_protocol(tag: Analyzer::Tag, reg: ProtoRegistration): bool
-	{
-	local result = (tag !in registered_protocols);
-	registered_protocols[tag] = reg;
-	return result;
-	}
-
-function register_for_mime_types(tag: Analyzer::Tag, mime_types: set[string]) : bool
-	{
-	local rc = T;
-
-	for ( mt in mime_types )
-		{
-		if ( ! register_for_mime_type(tag, mt) )
-			rc = F;
-		}
-
-	return rc;
-	}
-
-function register_for_mime_type(tag: Analyzer::Tag, mt: string) : bool
-	{
-	if ( ! __register_for_mime_type(tag, mt) )
-		return F;
-
-	if ( tag !in mime_types )
-		mime_types[tag] = set();
-
-	add mime_types[tag][mt];
-	return T;
-	}
-
-function registered_mime_types(tag: Analyzer::Tag) : set[string]
-	{
-	return tag in mime_types ? mime_types[tag] : set();
-	}
-
-function all_registered_mime_types(): table[Analyzer::Tag] of set[string]
-	{
-	return mime_types;
-	}
-
-function describe(f: fa_file): string
-	{
-	local tag = Analyzer::get_tag(f$source);
-	if ( tag !in registered_protocols )
-		return "";
-
-	local handler = registered_protocols[tag];
-	return handler$describe(f);
-	}
-
-event get_file_handle(tag: Analyzer::Tag, c: connection, is_orig: bool) &priority=5
-	{
-	if ( tag !in registered_protocols )
-		return;
-
-	local handler = registered_protocols[tag];
-	set_file_handle(handler$get_file_handle(c, is_orig));
-	}

scripts/base/frameworks/intel/main.bro

@@ -32,6 +32,8 @@ export {
 		FILE_NAME,
 		## Certificate SHA-1 hash.
 		CERT_HASH,
+		## Public key MD5 hash. (SSH server host keys are a good example.)
+		PUBKEY_HASH,
 	};
 	
 	## Data about an :bro:type:`Intel::Item`.
@@ -67,6 +69,7 @@ export {
 		IN_ANYWHERE,
 	};
 
+	## Information about a piece of "seen" data.
 	type Seen: record {
 		## The string if the data is about a string.
 		indicator:       string        &log &optional;
@@ -124,7 +127,7 @@ export {
 		sources:  set[string]    &log &default=string_set();
 	};
 
-	## Intelligence data manipulation functions.
+	## Intelligence data manipulation function.
 	global insert: function(item: Item);
 
 	## Function to declare discovery of a piece of data in order to check
@@ -173,7 +176,7 @@ global min_data_store: MinDataStore &redef;
 
 event bro_init() &priority=5
 	{
-	Log::create_stream(LOG, [$columns=Info, $ev=log_intel]);
+	Log::create_stream(LOG, [$columns=Info, $ev=log_intel, $path="intel"]);
 	}
 
 function find(s: Seen): bool
@@ -289,8 +292,8 @@ event Intel::match(s: Seen, items: set[Item]) &priority=5
 		if ( ! info?$fuid )
 			info$fuid = s$f$id;
 
-		if ( ! info?$file_mime_type && s$f?$mime_type )
+		if ( ! info?$file_mime_type && s$f?$info && s$f$info?$mime_type )
-			info$file_mime_type = s$f$mime_type;
+			info$file_mime_type = s$f$info$mime_type;
 
 		if ( ! info?$file_desc )
 			info$file_desc = Files::describe(s$f);

scripts/base/frameworks/logging/main.bro

@@ -50,11 +50,17 @@ export {
 		## The event receives a single same parameter, an instance of
 		## type ``columns``.
 		ev: any &optional;
+
+		## A path that will be inherited by any filters added to the
+		## stream which do not already specify their own path.
+		path: string &optional;
 	};
 
 	## Builds the default path values for log filters if not otherwise
 	## specified by a filter. The default implementation uses *id*
-	## to derive a name.
+	## to derive a name.  Upon adding a filter to a stream, if neither
+	## ``path`` nor ``path_func`` is explicitly set by them, then
+	## this function is used as the ``path_func``.
 	##
 	## id: The ID associated with the log stream.
 	##
@@ -143,7 +149,9 @@ export {
 		## to compute the string dynamically. It is ok to return
 		## different strings for separate calls, but be careful: it's
 		## easy to flood the disk by returning a new string for each
-		## connection.
+		## connection.  Upon adding a filter to a stream, if neither
+		## ``path`` nor ``path_func`` is explicitly set by them, then
+		## :bro:see:`default_path_func` is used.
 		##
 		## id: The ID associated with the log stream.
 		##
@@ -379,6 +387,8 @@ export {
 	global active_streams: table[ID] of Stream = table();
 }
 
+global all_streams: table[ID] of Stream = table();
+
 # We keep a script-level copy of all filters so that we can manipulate them.
 global filters: table[ID, string] of Filter;
 
@@ -405,30 +415,30 @@ function default_path_func(id: ID, path: string, rec: any) : string
 
 	local id_str = fmt("%s", id);
 
-	local parts = split1(id_str, /::/);
+	local parts = split_string1(id_str, /::/);
 	if ( |parts| == 2 )
 		{
 		# Example: Notice::LOG -> "notice"
-		if ( parts[2] == "LOG" )
+		if ( parts[1] == "LOG" )
 			{
-			local module_parts = split_n(parts[1], /[^A-Z][A-Z][a-z]*/, T, 4);
+			local module_parts = split_string_n(parts[0], /[^A-Z][A-Z][a-z]*/, T, 4);
 			local output = "";
-			if ( 1 in module_parts )
+			if ( 0 in module_parts )
-				output = module_parts[1];
+				output = module_parts[0];
+			if ( 1 in module_parts && module_parts[1] != "" )
+				output = cat(output, sub_bytes(module_parts[1],1,1), "_", sub_bytes(module_parts[1], 2, |module_parts[1]|));
 			if ( 2 in module_parts && module_parts[2] != "" )
-				output = cat(output, sub_bytes(module_parts[2],1,1), "_", sub_bytes(module_parts[2], 2, |module_parts[2]|));
+				output = cat(output, "_", module_parts[2]);
 			if ( 3 in module_parts && module_parts[3] != "" )
-				output = cat(output, "_", module_parts[3]);
+				output = cat(output, sub_bytes(module_parts[3],1,1), "_", sub_bytes(module_parts[3], 2, |module_parts[3]|));
-			if ( 4 in module_parts && module_parts[4] != "" )
-				output = cat(output, sub_bytes(module_parts[4],1,1), "_", sub_bytes(module_parts[4], 2, |module_parts[4]|));
 			return to_lower(output);
 			}
 
 		# Example: Notice::POLICY_LOG -> "notice_policy"
-		if ( /_LOG$/ in parts[2] )
+		if ( /_LOG$/ in parts[1] )
-			parts[2] = sub(parts[2], /_LOG$/, "");
+			parts[1] = sub(parts[1], /_LOG$/, "");
 
-		return cat(to_lower(parts[1]),"_",to_lower(parts[2]));
+		return cat(to_lower(parts[0]),"_",to_lower(parts[1]));
 		}
 	else
 		return to_lower(id_str);
@@ -463,6 +473,7 @@ function create_stream(id: ID, stream: Stream) : bool
 		return F;
 
 	active_streams[id] = stream;
+	all_streams[id] = stream;
 
 	return add_default_filter(id);
 	}
@@ -470,6 +481,7 @@ function create_stream(id: ID, stream: Stream) : bool
 function remove_stream(id: ID) : bool
 	{
 	delete active_streams[id];
+	delete all_streams[id];
 	return __remove_stream(id);
 	}
 
@@ -482,10 +494,12 @@ function disable_stream(id: ID) : bool
 
 function add_filter(id: ID, filter: Filter) : bool
 	{
-	# This is a work-around for the fact that we can't forward-declare
+	local stream = all_streams[id];
-	# the default_path_func and then use it as &default in the record
+
-	# definition.
+	if ( stream?$path && ! filter?$path )
-	if ( ! filter?$path_func )
+		filter$path = stream$path;
+
+	if ( ! filter?$path && ! filter?$path_func )
 		filter$path_func = default_path_func;
 
 	filters[id, filter$name] = filter;

scripts/base/frameworks/notice/main.bro

@@ -19,9 +19,9 @@ export {
 	## the :bro:id:`NOTICE` function.  The convention is to give a general
 	## category along with the specific notice separating words with
 	## underscores and using leading capitals on each word except for
-	## abbreviations which are kept in all capitals.  For example,
+	## abbreviations which are kept in all capitals. For example,
 	## SSH::Password_Guessing is for hosts that have crossed a threshold of
-	## heuristically determined failed SSH logins.
+	## failed SSH logins.
 	type Type: enum {
 		## Notice reporting a count of how often a notice occurred.
 		Tally,
@@ -349,9 +349,9 @@ function log_mailing_postprocessor(info: Log::RotationInfo): bool
 
 event bro_init() &priority=5
 	{
-	Log::create_stream(Notice::LOG, [$columns=Info, $ev=log_notice]);
+	Log::create_stream(Notice::LOG, [$columns=Info, $ev=log_notice, $path="notice"]);
 
-	Log::create_stream(Notice::ALARM_LOG, [$columns=Notice::Info]);
+	Log::create_stream(Notice::ALARM_LOG, [$columns=Notice::Info, $path="notice_alarm"]);
 	# If Bro is configured for mailing notices, set up mailing for alarms.
 	# Make sure that this alarm log is also output as text so that it can
 	# be packaged up and emailed later.
@@ -531,8 +531,8 @@ function create_file_info(f: fa_file): Notice::FileInfo
 	local fi: Notice::FileInfo = Notice::FileInfo($fuid = f$id,
 	                                              $desc = Files::describe(f));
 
-	if ( f?$mime_type )
+	if ( f?$info && f$info?$mime_type )
-		fi$mime = f$mime_type;
+		fi$mime = f$info$mime_type;
 
 	if ( f?$conns && |f$conns| == 1 )
 		for ( id in f$conns )

scripts/base/frameworks/notice/weird.bro

@@ -294,7 +294,7 @@ global current_conn: connection;
 
 event bro_init() &priority=5
 	{
-	Log::create_stream(Weird::LOG, [$columns=Info, $ev=log_weird]);
+	Log::create_stream(Weird::LOG, [$columns=Info, $ev=log_weird, $path="weird"]);
 	}
 
 function flow_id_string(src: addr, dst: addr): string

scripts/base/frameworks/packet-filter/main.bro

@@ -159,7 +159,7 @@ event filter_change_tracking()
 
 event bro_init() &priority=5
 	{
-	Log::create_stream(PacketFilter::LOG, [$columns=Info]);
+	Log::create_stream(PacketFilter::LOG, [$columns=Info, $path="packet_filter"]);
 
 	# Preverify the capture and restrict filters to give more granular failure messages.
 	for ( id in capture_filters )

scripts/base/frameworks/reporter/main.bro

@@ -45,7 +45,7 @@ export {
 
 event bro_init() &priority=5
 	{
-	Log::create_stream(Reporter::LOG, [$columns=Info]);
+	Log::create_stream(Reporter::LOG, [$columns=Info, $path="reporter"]);
 	}
 
 event reporter_info(t: time, msg: string, location: string) &priority=-5

scripts/base/frameworks/signatures/main.bro

@@ -142,7 +142,7 @@ global did_sig_log: set[string] &read_expire = 1 hr;
 
 event bro_init()
 	{
-	Log::create_stream(Signatures::LOG, [$columns=Info, $ev=log_signature]);
+	Log::create_stream(Signatures::LOG, [$columns=Info, $ev=log_signature, $path="signatures"]);
 	}
 		
 # Returns true if the given signature has already been triggered for the given
@@ -277,7 +277,7 @@ event signature_match(state: signature_state, msg: string, data: string)
 				orig, sig_id, hcount);
 
 		Log::write(Signatures::LOG, 
-			[$note=Multiple_Sig_Responders,
+			[$ts=network_time(), $note=Multiple_Sig_Responders,
 		     $src_addr=orig, $sig_id=sig_id, $event_msg=msg,
 		     $host_count=hcount, $sub_msg=horz_scan_msg]);
 

scripts/base/frameworks/software/main.bro

@@ -105,7 +105,7 @@ export {
 
 event bro_init() &priority=5
 	{
-	Log::create_stream(Software::LOG, [$columns=Info, $ev=log_software]);
+	Log::create_stream(Software::LOG, [$columns=Info, $ev=log_software, $path="software"]);
 	}
 	
 type Description: record {
@@ -133,62 +133,62 @@ function parse(unparsed_version: string): Description
 		{
 		# The regular expression should match the complete version number
 		# and software name.
-		local version_parts = split_n(unparsed_version, /\/?( [\(])?v?[0-9\-\._, ]{2,}/, T, 1);
+		local version_parts = split_string_n(unparsed_version, /\/?( [\(])?v?[0-9\-\._, ]{2,}/, T, 1);
-		if ( 1 in version_parts )
+		if ( 0 in version_parts )
 			{
-			if ( /^\(/ in version_parts[1] )
+			if ( /^\(/ in version_parts[0] )
-				software_name = strip(sub(version_parts[1], /[\(]/, ""));
+				software_name = strip(sub(version_parts[0], /[\(]/, ""));
 			else
-				software_name = strip(version_parts[1]);
+				software_name = strip(version_parts[0]);
 			}
 		if ( |version_parts| >= 2 )
 			{
 			# Remove the name/version separator if it's left at the beginning
 			# of the version number from the previous split_all.
-			local sv = strip(version_parts[2]);
+			local sv = strip(version_parts[1]);
 			if ( /^[\/\-\._v\(]/ in sv )
-				sv = strip(sub(version_parts[2], /^\(?[\/\-\._v\(]/, ""));
+				sv = strip(sub(version_parts[1], /^\(?[\/\-\._v\(]/, ""));
-			local version_numbers = split_n(sv, /[\-\._,\[\(\{ ]/, F, 3);
+			local version_numbers = split_string_n(sv, /[\-\._,\[\(\{ ]/, F, 3);
-			if ( 5 in version_numbers && version_numbers[5] != "" )
+			if ( 4 in version_numbers && version_numbers[4] != "" )
-				v$addl = strip(version_numbers[5]);
+				v$addl = strip(version_numbers[4]);
-			else if ( 3 in version_parts && version_parts[3] != "" &&
+			else if ( 2 in version_parts && version_parts[2] != "" &&
-			          version_parts[3] != ")" )
+			          version_parts[2] != ")" )
 				{
-				if ( /^[[:blank:]]*\([a-zA-Z0-9\-\._[:blank:]]*\)/ in version_parts[3] )
+				if ( /^[[:blank:]]*\([a-zA-Z0-9\-\._[:blank:]]*\)/ in version_parts[2] )
 					{
-					v$addl = split_n(version_parts[3], /[\(\)]/, F, 2)[2];
+					v$addl = split_string_n(version_parts[2], /[\(\)]/, F, 2)[1];
 					}
 				else
 					{
-					local vp = split_n(version_parts[3], /[\-\._,;\[\]\(\)\{\} ]/, F, 3);
+					local vp = split_string_n(version_parts[2], /[\-\._,;\[\]\(\)\{\} ]/, F, 3);
-					if ( |vp| >= 1 && vp[1] != "" )
+					if ( |vp| >= 1 && vp[0] != "" )
+						{
+						v$addl = strip(vp[0]);
+						}
+					else if ( |vp| >= 2 && vp[1] != "" )
 						{
 						v$addl = strip(vp[1]);
 						}
-					else if ( |vp| >= 2 && vp[2] != "" )
+					else if ( |vp| >= 3 && vp[2] != "" )
 						{
 						v$addl = strip(vp[2]);
 						}
-					else if ( |vp| >= 3 && vp[3] != "" )
-						{
-						v$addl = strip(vp[3]);
-						}
 					else
 						{
-						v$addl = strip(version_parts[3]);
+						v$addl = strip(version_parts[2]);
 						}
 						
 					}
 				}
 			
-			if ( 4 in version_numbers && version_numbers[4] != "" )
-				v$minor3 = extract_count(version_numbers[4]);
 			if ( 3 in version_numbers && version_numbers[3] != "" )
-				v$minor2 = extract_count(version_numbers[3]);
+				v$minor3 = extract_count(version_numbers[3]);
 			if ( 2 in version_numbers && version_numbers[2] != "" )
-				v$minor = extract_count(version_numbers[2]);
+				v$minor2 = extract_count(version_numbers[2]);
 			if ( 1 in version_numbers && version_numbers[1] != "" )
-				v$major = extract_count(version_numbers[1]);
+				v$minor = extract_count(version_numbers[1]);
+			if ( 0 in version_numbers && version_numbers[0] != "" )
+				v$major = extract_count(version_numbers[0]);
 			}
 		}
 	
@@ -200,14 +200,14 @@ function parse_mozilla(unparsed_version: string): Description
 	{
 	local software_name = "<unknown browser>";
 	local v: Version;
-	local parts: table[count] of string;
+	local parts: string_vec;
 	
 	if ( /Opera [0-9\.]*$/ in unparsed_version )
 		{
 		software_name = "Opera";
-		parts = split_all(unparsed_version, /Opera [0-9\.]*$/);
+		parts = split_string_all(unparsed_version, /Opera [0-9\.]*$/);
-		if ( 2 in parts )
+		if ( 1 in parts )
-			v = parse(parts[2])$version;
+			v = parse(parts[1])$version;
 		}
 	else if ( / MSIE |Trident\// in unparsed_version )
 		{
@@ -222,28 +222,28 @@ function parse_mozilla(unparsed_version: string): Description
 			v = [$major=11,$minor=0];
 		else
 			{
-			parts = split_all(unparsed_version, /MSIE [0-9]{1,2}\.*[0-9]*b?[0-9]*/);
+			parts = split_string_all(unparsed_version, /MSIE [0-9]{1,2}\.*[0-9]*b?[0-9]*/);
-			if ( 2 in parts )
+			if ( 1 in parts )
-				v = parse(parts[2])$version;
+				v = parse(parts[1])$version;
 			}
 		}
 	else if ( /Version\/.*Safari\// in unparsed_version )
 		{
 		software_name = "Safari";
-		parts = split_all(unparsed_version, /Version\/[0-9\.]*/);
+		parts = split_string_all(unparsed_version, /Version\/[0-9\.]*/);
-		if ( 2 in parts )
+		if ( 1 in parts )
 			{
-			v = parse(parts[2])$version;
+			v = parse(parts[1])$version;
 			if ( / Mobile\/?.* Safari/ in unparsed_version )
 				v$addl = "Mobile";
 			}
 		}
 	else if ( /(Firefox|Netscape|Thunderbird)\/[0-9\.]*/ in unparsed_version )
 		{
-		parts = split_all(unparsed_version, /(Firefox|Netscape|Thunderbird)\/[0-9\.]*/);
+		parts = split_string_all(unparsed_version, /(Firefox|Netscape|Thunderbird)\/[0-9\.]*/);
-		if ( 2 in parts )
+		if ( 1 in parts )
 			{
-			local tmp_s = parse(parts[2]);
+			local tmp_s = parse(parts[1]);
 			software_name = tmp_s$name;
 			v = tmp_s$version;
 			}
@@ -251,48 +251,48 @@ function parse_mozilla(unparsed_version: string): Description
 	else if ( /Chrome\/.*Safari\// in unparsed_version )
 		{
 		software_name = "Chrome";
-		parts = split_all(unparsed_version, /Chrome\/[0-9\.]*/);
+		parts = split_string_all(unparsed_version, /Chrome\/[0-9\.]*/);
-		if ( 2 in parts )
+		if ( 1 in parts )
-			v = parse(parts[2])$version;
+			v = parse(parts[1])$version;
 		}
 	else if ( /^Opera\// in unparsed_version )
 		{
 		if ( /Opera M(ini|obi)\// in unparsed_version )
 			{
-			parts = split_all(unparsed_version, /Opera M(ini|obi)/);
+			parts = split_string_all(unparsed_version, /Opera M(ini|obi)/);
-			if ( 2 in parts )
+			if ( 1 in parts )
-				software_name = parts[2];
+				software_name = parts[1];
-			parts = split_all(unparsed_version, /Version\/[0-9\.]*/);
+			parts = split_string_all(unparsed_version, /Version\/[0-9\.]*/);
-			if ( 2 in parts )
+			if ( 1 in parts )
-				v = parse(parts[2])$version;
+				v = parse(parts[1])$version;
 			else
 				{
-				parts = split_all(unparsed_version, /Opera Mini\/[0-9\.]*/);
+				parts = split_string_all(unparsed_version, /Opera Mini\/[0-9\.]*/);
-				if ( 2 in parts )
+				if ( 1 in parts )
-					v = parse(parts[2])$version;
+					v = parse(parts[1])$version;
 				}
 			}
 		else
 			{
 			software_name = "Opera";
-			parts = split_all(unparsed_version, /Version\/[0-9\.]*/);
+			parts = split_string_all(unparsed_version, /Version\/[0-9\.]*/);
-			if ( 2 in parts )
+			if ( 1 in parts )
-				v = parse(parts[2])$version;
+				v = parse(parts[1])$version;
 			}
 		}
 	else if ( /AppleWebKit\/[0-9\.]*/ in unparsed_version )
 		{
 		software_name = "Unspecified WebKit";
-		parts = split_all(unparsed_version, /AppleWebKit\/[0-9\.]*/);
+		parts = split_string_all(unparsed_version, /AppleWebKit\/[0-9\.]*/);
-		if ( 2 in parts )
+		if ( 1 in parts )
-			v = parse(parts[2])$version;
+			v = parse(parts[1])$version;
 		}
 	else if ( / Java\/[0-9]\./ in unparsed_version )
 		{
 		software_name = "Java";
-		parts = split_all(unparsed_version, /Java\/[0-9\._]*/);
+		parts = split_string_all(unparsed_version, /Java\/[0-9\._]*/);
-		if ( 2 in parts )
+		if ( 1 in parts )
-			v = parse(parts[2])$version;
+			v = parse(parts[1])$version;
 		}
 
 	return [$version=v, $unparsed_version=unparsed_version, $name=software_name];

scripts/base/frameworks/tunnels/main.bro

@@ -89,7 +89,7 @@ redef likely_server_ports += { ayiya_ports, teredo_ports, gtpv1_ports };
 
 event bro_init() &priority=5
 	{
-	Log::create_stream(Tunnel::LOG, [$columns=Info]);
+	Log::create_stream(Tunnel::LOG, [$columns=Info, $path="tunnel"]);
 
 	Analyzer::register_for_ports(Analyzer::ANALYZER_AYIYA, ayiya_ports);
 	Analyzer::register_for_ports(Analyzer::ANALYZER_TEREDO, teredo_ports);

scripts/base/init-bare.bro

@@ -353,9 +353,10 @@ type connection: record {
 ## gives up and discards any internal state related to the file.
 const default_file_timeout_interval: interval = 2 mins &redef;
 
-## Default amount of bytes that file analysis will buffer before raising
+## Default amount of bytes that file analysis will buffer in order to use
-## :bro:see:`file_new`.
+## for mime type matching.  File analyzers attached at the time of mime type
-const default_file_bof_buffer_size: count = 1024 &redef;
+## matching or later, will receive a copy of this buffer.
+const default_file_bof_buffer_size: count = 4096 &redef;
 
 ## A file that Bro is analyzing.  This is Bro's type for describing the basic
 ## internal metadata collected about a "file", which is essentially just a
@@ -394,8 +395,10 @@ type fa_file: record {
 	## during the process of analysis e.g. due to dropped packets.
 	missing_bytes: count &default=0;
 
-	## The number of not all-in-sequence bytes in the file stream that
+	## The number of bytes in the file stream that were not delivered to
-	## were delivered to file analyzers due to reassembly buffer overflow.
+	## stream file analyzers.  Generally, this consists of bytes that
+	## couldn't be reassembled, either because reassembly simply isn't
+	## enabled, or due to size limitations of the reassembly buffer.
 	overflow_bytes: count &default=0;
 
 	## The amount of time between receiving new data for this file that
@@ -409,16 +412,6 @@ type fa_file: record {
 	## The content of the beginning of a file up to *bof_buffer_size* bytes.
 	## This is also the buffer that's used for file/mime type detection.
 	bof_buffer: string &optional;
-
-	## The mime type of the strongest file magic signature matches against
-	## the data chunk in *bof_buffer*, or in the cases where no buffering
-	## of the beginning of file occurs, an initial guess of the mime type
-	## based on the first data seen.
-	mime_type: string &optional;
-
-	## All mime types that matched file magic signatures against the data
-	## chunk in *bof_buffer*, in order of their strength value.
-	mime_types: mime_matches &optional;
 } &redef;
 
 ## Fields of a SYN packet.
@@ -447,6 +440,7 @@ type NetStats: record {
 	## packet capture system, this value may not be available and will then
 	## be always set to zero.
 	pkts_link:    count &default=0;
+	bytes_recvd:  count &default=0;	##< Bytes received by Bro.
 };
 
 ## Statistics about Bro's resource consumption.
@@ -935,7 +929,7 @@ const tcp_storm_interarrival_thresh = 1 sec &redef;
 ## seeing our peer's ACKs.  Set to zero to turn off this determination.
 ##
 ## .. bro:see:: tcp_max_above_hole_without_any_acks tcp_excessive_data_without_further_acks
-const tcp_max_initial_window = 4096 &redef;
+const tcp_max_initial_window = 16384 &redef;
 
 ## If we're not seeing our peer's ACKs, the maximum volume of data above a
 ## sequence hole that we'll tolerate before assuming that there's been a packet
@@ -943,7 +937,7 @@ const tcp_max_initial_window = 4096 &redef;
 ## don't ever give up.
 ##
 ## .. bro:see:: tcp_max_initial_window tcp_excessive_data_without_further_acks
-const tcp_max_above_hole_without_any_acks = 4096 &redef;
+const tcp_max_above_hole_without_any_acks = 16384 &redef;
 
 ## If we've seen this much data without any of it being acked, we give up
 ## on that connection to avoid memory exhaustion due to buffering all that
@@ -2222,6 +2216,41 @@ export {
 	const heartbeat_interval = 1.0 secs &redef;
 }
 
+module SSH;
+
+export {
+	## The client and server each have some preferences for the algorithms used
+	## in each direction.
+	type Algorithm_Prefs: record {
+		## The algorithm preferences for client to server communication
+		client_to_server: vector of string &optional;
+		## The algorithm preferences for server to client communication
+		server_to_client: vector of string &optional;
+	};
+
+	## This record lists the preferences of an SSH endpoint for
+	## algorithm selection. During the initial :abbr:`SSH (Secure Shell)`
+	## key exchange, each endpoint lists the algorithms
+	## that it supports, in order of preference. See
+	## :rfc:`4253#section-7.1` for details.
+	type Capabilities: record {
+		## Key exchange algorithms
+		kex_algorithms:             string_vec;
+		## The algorithms supported for the server host key
+		server_host_key_algorithms: string_vec;
+		## Symmetric encryption algorithm preferences
+		encryption_algorithms:      Algorithm_Prefs;
+		## Symmetric MAC algorithm preferences
+		mac_algorithms:             Algorithm_Prefs;
+		## Compression algorithm preferences
+		compression_algorithms:     Algorithm_Prefs;
+		## Language preferences
+		languages:                  Algorithm_Prefs &optional;
+		## Are these the capabilities of the server?
+		is_server:                  bool;
+	};
+}
+
 module GLOBAL;
 
 ## An NTP message.
@@ -2781,19 +2810,20 @@ export {
 module X509;
 export {
 	type Certificate: record {
-		version: count;	##< Version number.
+		version: count &log;	##< Version number.
-		serial: string;	##< Serial number.
+		serial: string &log;	##< Serial number.
-		subject: string;	##< Subject.
+		subject: string &log;	##< Subject.
-		issuer: string;	##< Issuer.
+		issuer: string &log;	##< Issuer.
-		not_valid_before: time;	##< Timestamp before when certificate is not valid.
+		cn: string &optional; ##< Last (most specific) common name.
-		not_valid_after: time;	##< Timestamp after when certificate is not valid.
+		not_valid_before: time &log;	##< Timestamp before when certificate is not valid.
-		key_alg: string;	##< Name of the key algorithm
+		not_valid_after: time &log;	##< Timestamp after when certificate is not valid.
-		sig_alg: string;	##< Name of the signature algorithm
+		key_alg: string &log;	##< Name of the key algorithm
-		key_type: string &optional;	##< Key type, if key parseable by openssl (either rsa, dsa or ec)
+		sig_alg: string &log;	##< Name of the signature algorithm
-		key_length: count &optional;	##< Key length in bits
+		key_type: string &optional &log;	##< Key type, if key parseable by openssl (either rsa, dsa or ec)
-		exponent: string &optional;	##< Exponent, if RSA-certificate
+		key_length: count &optional &log;	##< Key length in bits
-		curve: string &optional;	##< Curve, if EC-certificate
+		exponent: string &optional &log;	##< Exponent, if RSA-certificate
-	} &log;
+		curve: string &optional &log;	##< Curve, if EC-certificate
+	};
 
 	type Extension: record {
 		name: string;	##< Long name of extension. oid if name not known
@@ -2854,7 +2884,44 @@ export {
 		attributes    : RADIUS::Attributes &optional;
 	};
 }
-module GLOBAL;
+
+module RDP;
+export {
+	type RDP::EarlyCapabilityFlags: record {
+		support_err_info_pdu:       bool;
+		want_32bpp_session:         bool;
+		support_statusinfo_pdu:     bool;
+		strong_asymmetric_keys:     bool;
+		support_monitor_layout_pdu: bool;
+		support_netchar_autodetect: bool;
+		support_dynvc_gfx_protocol: bool;
+		support_dynamic_time_zone:  bool;
+		support_heartbeat_pdu:      bool;
+	};
+
+	type RDP::ClientCoreData: record {
+		version_major:          count;
+		version_minor:          count;
+		desktop_width:          count;
+		desktop_height:         count;
+		color_depth:            count;
+		sas_sequence:           count;
+		keyboard_layout:        count;
+		client_build:           count;
+		client_name:            string;
+		keyboard_type:          count;
+		keyboard_sub:           count;
+		keyboard_function_key:  count;
+		ime_file_name:          string;
+		post_beta2_color_depth: count  &optional;
+		client_product_id:      string &optional;
+		serial_number:          count  &optional;
+		high_color_depth:       count  &optional;
+		supported_color_depths: count  &optional;
+		ec_flags:               RDP::EarlyCapabilityFlags &optional;
+		dig_product_id:         string &optional;
+	};
+}
 
 @load base/bif/plugins/Bro_SNMP.types.bif
 
@@ -3365,6 +3432,7 @@ const bits_per_uid: count = 96 &redef;
 
 # Load these frameworks here because they use fairly deep integration with
 # BiFs and script-land defined types.
+@load base/frameworks/broker
 @load base/frameworks/logging
 @load base/frameworks/input
 @load base/frameworks/analyzer

scripts/base/init-default.bro

@@ -49,6 +49,7 @@
 @load base/protocols/mysql
 @load base/protocols/pop3
 @load base/protocols/radius
+@load base/protocols/rdp
 @load base/protocols/snmp
 @load base/protocols/smtp
 @load base/protocols/socks

scripts/base/misc/find-checksum-offloading.bro

@@ -50,7 +50,7 @@ event ChecksumOffloading::check()
 			bad_checksum_msg += "UDP";
 			}
 
-		local message = fmt("Your %s invalid %s checksums, most likely from NIC checksum offloading.", packet_src, bad_checksum_msg);
+		local message = fmt("Your %s invalid %s checksums, most likely from NIC checksum offloading.  By default, packets with invalid checksums are discarded by Bro unless using the -C command-line option or toggling the 'ignore_checksums' variable.  Alternatively, disable checksum offloading by the network adapter to ensure Bro analyzes the actual checksums that are transmitted.", packet_src, bad_checksum_msg);
 		Reporter::warning(message);
 		done = T;
 		}

scripts/base/protocols/conn/main.bro

@@ -62,6 +62,12 @@ export {
 		## field will be left empty at all times.
 		local_orig:   bool            &log &optional;
 
+		## If the connection is responded to locally, this value will be T.
+		## If it was responded to remotely it will be F.  In the case that
+		## the :bro:id:`Site::local_nets` variable is undefined, this
+		## field will be left empty at all times.
+		local_resp:   bool            &log &optional;
+
 		## Indicates the number of bytes missed in content gaps, which
 		## is representative of packet loss.  A value other than zero
 		## will normally cause protocol analysis to fail but some
@@ -121,7 +127,7 @@ redef record connection += {
 
 event bro_init() &priority=5
 	{
-	Log::create_stream(Conn::LOG, [$columns=Info, $ev=log_conn]);
+	Log::create_stream(Conn::LOG, [$columns=Info, $ev=log_conn, $path="conn"]);
 	}
 
 function conn_state(c: connection, trans: transport_proto): string
@@ -201,7 +207,10 @@ function set_conn(c: connection, eoc: bool)
 		add c$conn$tunnel_parents[c$tunnel[|c$tunnel|-1]$uid];
 	c$conn$proto=get_port_transport_proto(c$id$resp_p);
 	if( |Site::local_nets| > 0 )
+		{
 		c$conn$local_orig=Site::is_local_addr(c$id$orig_h);
+		c$conn$local_resp=Site::is_local_addr(c$id$resp_h);
+		}
 
 	if ( eoc )
 		{

scripts/base/protocols/dhcp/main.bro

@@ -49,7 +49,7 @@ redef likely_server_ports += { 67/udp };
 
 event bro_init() &priority=5
 	{
-	Log::create_stream(DHCP::LOG, [$columns=Info, $ev=log_dhcp]);
+	Log::create_stream(DHCP::LOG, [$columns=Info, $ev=log_dhcp, $path="dhcp"]);
 	Analyzer::register_for_ports(Analyzer::ANALYZER_DHCP, ports);
 	}
 

scripts/base/protocols/dhcp/utils.bro

@@ -13,7 +13,7 @@ export {
 
 function reverse_ip(ip: addr): addr
 	{
-	local octets = split(cat(ip), /\./);
+	local octets = split_string(cat(ip), /\./);
-	return to_addr(cat(octets[4], ".", octets[3], ".", octets[2], ".", octets[1]));
+	return to_addr(cat(octets[3], ".", octets[2], ".", octets[1], ".", octets[0]));
 	}
 

scripts/base/protocols/dnp3/dpd.sig

@@ -5,5 +5,11 @@ signature dpd_dnp3_server {
 	ip-proto == tcp
 	payload /\x05\x64/
 	tcp-state responder
- 	enable "dnp3"
+ 	enable "dnp3_tcp"
+}
+
+signature dpd_dnp3_server_udp {
+	ip-proto == udp
+	payload /\x05\x64/
+	enable "dnp3_udp"
 }

scripts/base/protocols/dnp3/main.bro

@@ -31,16 +31,16 @@ redef record connection += {
 	dnp3: Info &optional;
 };
 
-const ports = { 20000/tcp };
+const ports = { 20000/tcp , 20000/udp };
 redef likely_server_ports += { ports };
 
 event bro_init() &priority=5
 	{
-	Log::create_stream(DNP3::LOG, [$columns=Info, $ev=log_dnp3]);
+	Log::create_stream(DNP3::LOG, [$columns=Info, $ev=log_dnp3, $path="dnp3"]);
-	Analyzer::register_for_ports(Analyzer::ANALYZER_DNP3, ports);
+	Analyzer::register_for_ports(Analyzer::ANALYZER_DNP3_TCP, ports);
 	}
 
-event dnp3_application_request_header(c: connection, is_orig: bool, fc: count)
+event dnp3_application_request_header(c: connection, is_orig: bool, application_control: count, fc: count)
 	{
 	if ( ! c?$dnp3 )
 		c$dnp3 = [$ts=network_time(), $uid=c$uid, $id=c$id];
@@ -49,7 +49,7 @@ event dnp3_application_request_header(c: connection, is_orig: bool, fc: count)
 	c$dnp3$fc_request = function_codes[fc];
 	}
 
-event dnp3_application_response_header(c: connection, is_orig: bool, fc: count, iin: count)
+event dnp3_application_response_header(c: connection, is_orig: bool, application_control: count, fc: count, iin: count)
 	{
 	if ( ! c?$dnp3 )
 		c$dnp3 = [$ts=network_time(), $uid=c$uid, $id=c$id];

scripts/base/protocols/dns/main.bro

@@ -150,7 +150,7 @@ redef likely_server_ports += { ports };
 
 event bro_init() &priority=5
 	{
-	Log::create_stream(DNS::LOG, [$columns=Info, $ev=log_dns]);
+	Log::create_stream(DNS::LOG, [$columns=Info, $ev=log_dns, $path="dns"]);
 	Analyzer::register_for_ports(Analyzer::ANALYZER_DNS, ports);
 	}
 
@@ -305,6 +305,9 @@ hook DNS::do_reply(c: connection, msg: dns_msg, ans: dns_answer, reply: string)
 
 	if ( ans$answer_type == DNS_ANS )
 		{
+		if ( ! c$dns?$query )
+			c$dns$query = ans$query;
+
 		c$dns$AA    = msg$AA;
 		c$dns$RA    = msg$RA;
 

scripts/base/protocols/ftp/files.bro

@@ -17,6 +17,10 @@ export {
 
 	## Describe the file being transferred.
 	global describe_file: function(f: fa_file): string;
+
+	redef record fa_file += { 
+		ftp: FTP::Info &optional;
+	};
 }
 
 function get_file_handle(c: connection, is_orig: bool): string
@@ -48,7 +52,6 @@ event bro_init() &priority=5
 	                          $describe        = FTP::describe_file]);
 	}
 
-
 event file_over_new_connection(f: fa_file, c: connection, is_orig: bool) &priority=5
 	{
 	if ( [c$id$resp_h, c$id$resp_p] !in ftp_data_expected ) 
@@ -56,6 +59,14 @@ event file_over_new_connection(f: fa_file, c: connection, is_orig: bool) &priori
 
 	local ftp = ftp_data_expected[c$id$resp_h, c$id$resp_p];
 	ftp$fuid = f$id;
-	if ( f?$mime_type )
+
-		ftp$mime_type = f$mime_type;
+	f$ftp = ftp;
+	}
+
+event file_mime_type(f: fa_file, mime_type: string) &priority=5
+	{
+	if ( ! f?$ftp )
+		return;
+
+	f$ftp$mime_type = mime_type;
 	}

scripts/base/protocols/ftp/main.bro

@@ -52,7 +52,7 @@ redef likely_server_ports += { ports };
 
 event bro_init() &priority=5
 	{
-	Log::create_stream(FTP::LOG, [$columns=Info, $ev=log_ftp]);
+	Log::create_stream(FTP::LOG, [$columns=Info, $ev=log_ftp, $path="ftp"]);
 	Analyzer::register_for_ports(Analyzer::ANALYZER_FTP, ports);
 	}
 
@@ -274,7 +274,7 @@ event file_transferred(c: connection, prefix: string, descr: string,
 	if ( [id$resp_h, id$resp_p] in ftp_data_expected )
 		{
 		local s = ftp_data_expected[id$resp_h, id$resp_p];
-		s$mime_type = split1(mime_type, /;/)[1];
+		s$mime_type = split_string1(mime_type, /;/)[0];
 		}
 	}
 

scripts/base/protocols/http/entities.bro

@@ -35,6 +35,10 @@ export {
 		## body.
 		resp_mime_depth: count            &default=0;
 	};
+
+	redef record fa_file += {
+		http: HTTP::Info &optional;
+	};
 }
 
 event http_begin_entity(c: connection, is_orig: bool) &priority=10
@@ -67,6 +71,8 @@ event file_over_new_connection(f: fa_file, c: connection, is_orig: bool) &priori
 	{
 	if ( f$source == "HTTP" && c?$http ) 
 		{
+		f$http = c$http;
+
 		if ( c$http?$current_entity && c$http$current_entity?$filename )
 			f$info$filename = c$http$current_entity$filename;
 
@@ -76,14 +82,6 @@ event file_over_new_connection(f: fa_file, c: connection, is_orig: bool) &priori
 				c$http$orig_fuids = string_vec(f$id);
 			else
 				c$http$orig_fuids[|c$http$orig_fuids|] = f$id;
-
-			if ( f?$mime_type )
-				{
-				if ( ! c$http?$orig_mime_types )
-					c$http$orig_mime_types = string_vec(f$mime_type);
-				else
-					c$http$orig_mime_types[|c$http$orig_mime_types|] = f$mime_type;
-				}
 			}
 		else
 			{
@@ -91,17 +89,29 @@ event file_over_new_connection(f: fa_file, c: connection, is_orig: bool) &priori
 				c$http$resp_fuids = string_vec(f$id);
 			else
 				c$http$resp_fuids[|c$http$resp_fuids|] = f$id;
-
-			if ( f?$mime_type )
-				{
-				if ( ! c$http?$resp_mime_types )
-					c$http$resp_mime_types = string_vec(f$mime_type);
-				else
-					c$http$resp_mime_types[|c$http$resp_mime_types|] = f$mime_type;
-				}
 			}
 		}
+	}
 
+event file_mime_type(f: fa_file, mime_type: string) &priority=5
+	{
+	if ( ! f?$http || ! f?$is_orig )
+		return;
+
+	if ( f$is_orig )
+		{
+		if ( ! f$http?$orig_mime_types )
+			f$http$orig_mime_types = string_vec(mime_type);
+		else
+			f$http$orig_mime_types[|f$http$orig_mime_types|] = mime_type;
+		}
+	else
+		{
+		if ( ! f$http?$resp_mime_types )
+			f$http$resp_mime_types = string_vec(mime_type);
+		else
+			f$http$resp_mime_types[|f$http$resp_mime_types|] = mime_type;
+		}
 	}
 
 event http_end_entity(c: connection, is_orig: bool) &priority=5

scripts/base/protocols/http/main.bro

@@ -135,7 +135,7 @@ redef likely_server_ports += { ports };
 # Initialize the HTTP logging stream and ports.
 event bro_init() &priority=5
 	{
-	Log::create_stream(HTTP::LOG, [$columns=Info, $ev=log_http]);
+	Log::create_stream(HTTP::LOG, [$columns=Info, $ev=log_http, $path="http"]);
 	Analyzer::register_for_ports(Analyzer::ANALYZER_HTTP, ports);
 	}
 
@@ -242,7 +242,7 @@ event http_header(c: connection, is_orig: bool, name: string, value: string) &pr
 
 		else if ( name == "HOST" )
 			# The split is done to remove the occasional port value that shows up here.
-			c$http$host = split1(value, /:/)[1];
+			c$http$host = split_string1(value, /:/)[0];
 
 		else if ( name == "RANGE" )
 			c$http$range_request = T;
@@ -262,12 +262,12 @@ event http_header(c: connection, is_orig: bool, name: string, value: string) &pr
 			if ( /^[bB][aA][sS][iI][cC] / in value )
 				{
 				local userpass = decode_base64(sub(value, /[bB][aA][sS][iI][cC][[:blank:]]/, ""));
-				local up = split(userpass, /:/);
+				local up = split_string(userpass, /:/);
 				if ( |up| >= 2 )
 					{
-					c$http$username = up[1];
+					c$http$username = up[0];
 					if ( c$http$capture_password )
-						c$http$password = up[2];
+						c$http$password = up[1];
 					}
 				else
 					{

scripts/base/protocols/http/utils.bro

@@ -42,12 +42,12 @@ function extract_keys(data: string, kv_splitter: pattern): string_vec
 	{
 	local key_vec: vector of string = vector();
 	
-	local parts = split(data, kv_splitter);
+	local parts = split_string(data, kv_splitter);
 	for ( part_index in parts )
 		{
-		local key_val = split1(parts[part_index], /=/);
+		local key_val = split_string1(parts[part_index], /=/);
-		if ( 1 in key_val )
+		if ( 0 in key_val )
-			key_vec[|key_vec|] = key_val[1];
+			key_vec[|key_vec|] = key_val[0];
 		}
 	return key_vec;
 	}

scripts/base/protocols/irc/files.bro

@@ -12,6 +12,10 @@ export {
 
 	## Default file handle provider for IRC.
 	global get_file_handle: function(c: connection, is_orig: bool): string;
+
+	redef record fa_file += {
+		irc: IRC::Info &optional;
+	};
 }
 
 function get_file_handle(c: connection, is_orig: bool): string
@@ -34,6 +38,12 @@ event file_over_new_connection(f: fa_file, c: connection, is_orig: bool) &priori
 	irc$fuid = f$id;
 	if ( irc?$dcc_file_name )
 		f$info$filename = irc$dcc_file_name;
-	if ( f?$mime_type )
+
-		irc$dcc_mime_type = f$mime_type;
+	f$irc = irc;
+	}
+
+event file_mime_type(f: fa_file, mime_type: string) &priority=5
+	{
+	if ( f?$irc )
+		f$irc$dcc_mime_type = mime_type;
 	}

scripts/base/protocols/irc/main.bro

@@ -43,7 +43,7 @@ redef likely_server_ports += { ports };
 
 event bro_init() &priority=5
 	{
-	Log::create_stream(IRC::LOG, [$columns=Info, $ev=irc_log]);
+	Log::create_stream(IRC::LOG, [$columns=Info, $ev=irc_log, $path="irc"]);
 	Analyzer::register_for_ports(Analyzer::ANALYZER_IRC, ports);
 	}
 	

scripts/base/protocols/modbus/main.bro

@@ -34,7 +34,7 @@ redef likely_server_ports += { ports };
 
 event bro_init() &priority=5
 	{
-	Log::create_stream(Modbus::LOG, [$columns=Info, $ev=log_modbus]);
+	Log::create_stream(Modbus::LOG, [$columns=Info, $ev=log_modbus, $path="modbus"]);
 	Analyzer::register_for_ports(Analyzer::ANALYZER_MODBUS, ports);
 	}
 

scripts/base/protocols/mysql/main.bro

@@ -18,8 +18,10 @@ export {
 		cmd:	string	&log;
 		## The argument issued to the command
 		arg:	string	&log;
-		## The result (error, OK, etc.) from the server
+		## Did the server tell us that the command succeeded?
-		result: string &log &optional;
+		success: bool &log &optional;
+		## The number of affected rows, if any
+		rows: count &log &optional;
 		## Server message, if any
 		response: string &log &optional;
 	};
@@ -37,7 +39,7 @@ const ports = { 1434/tcp, 3306/tcp };
 
 event bro_init() &priority=5
 	{
-	Log::create_stream(mysql::LOG, [$columns=Info, $ev=log_mysql]);
+	Log::create_stream(mysql::LOG, [$columns=Info, $ev=log_mysql, $path="mysql"]);
 	Analyzer::register_for_ports(Analyzer::ANALYZER_MYSQL, ports);
 	}
 
@@ -57,16 +59,21 @@ event mysql_handshake(c: connection, username: string)
 
 event mysql_command_request(c: connection, command: count, arg: string) &priority=5
 	{
-	if ( ! c?$mysql )
+	if ( c?$mysql )
 		{
-		local info: Info;
+		# We got a request, but we haven't logged our
-		info$ts = network_time();
+		# previous request yet, so let's do that now.
-		info$uid = c$uid;
+		Log::write(mysql::LOG, c$mysql);
-		info$id = c$id;
+		delete c$mysql;
-		info$cmd = commands[command];
-		info$arg = sub(arg, /\0$/, "");
-		c$mysql = info;
 		}
+
+	local info: Info;
+	info$ts = network_time();
+	info$uid = c$uid;
+	info$id = c$id;
+	info$cmd = commands[command];
+	info$arg = sub(arg, /\0$/, "");
+	c$mysql = info;
 	}
 
 event mysql_command_request(c: connection, command: count, arg: string) &priority=-5
@@ -83,7 +90,7 @@ event mysql_error(c: connection, code: count, msg: string) &priority=5
 	{
 	if ( c?$mysql )
 		{
-		c$mysql$result = "error";
+		c$mysql$success = F;
 		c$mysql$response = msg;
 		}
 	}
@@ -101,8 +108,8 @@ event mysql_ok(c: connection, affected_rows: count) &priority=5
 	{
 	if ( c?$mysql )
 		{
-		c$mysql$result = "ok";
+		c$mysql$success = T;
-		c$mysql$response = fmt("Affected rows: %d", affected_rows);
+		c$mysql$rows = affected_rows;
 		}
 	}
 
@@ -114,3 +121,12 @@ event mysql_ok(c: connection, affected_rows: count) &priority=-5
 		delete c$mysql;
 		}
 	}
+
+event connection_state_remove(c: connection) &priority=-5
+	{
+	if ( c?$mysql )
+		{
+		Log::write(mysql::LOG, c$mysql);
+		delete c$mysql;
+		}
+	}