Merge remote-tracking branch 'origin/master' into topic/johanna/tls-more-data

CHANGES

@@ -1,4 +1,69 @@
 
+2.5-884 | 2018-08-20 15:39:21 -0500
+
+  * Fix outdated documentation test baselines (Jon Siwek, Corelight)
+
+  * Add 'smtp_excessive_pending_cmds' weird (Jon Siwek, Corelight)
+
+  * Fix SMTP command string comparisons (Jon Siwek, Corelight)
+
+  * Improve handling of empty lines in several text protocol analyzers
+    (Jon Siwek, Corelight)
+
+  * Add rate-limiting sampling mechanism for weird events
+
+    The generation of weird events, by default, are now rate-limited
+    according to these tunable options:
+
+      - Weird::sampling_whitelist
+      - Weird::sampling_threshold
+      - Weird::sampling_rate
+      - Weird::sampling_duration
+
+    The new get_reporter_stats() BIF also allows one to query the
+    total number of weirds generated (pre-sampling) which the new
+    policy/misc/weird-stats.bro script uses periodically to populate
+    a weird_stats.log.
+
+    There's also new reporter BIFs to allow generating weirds from the
+    script-layer such that they go through the same, internal
+    rate-limiting/sampling mechanisms:
+
+      - Reporter::conn_weird
+      - Reporter::flow_weird
+      - Reporter::net_weird
+
+    Some of the code was adapted from previous work by Johanna Amann.
+    (Jon Siwek, Corelight)
+
+  * Teach timestamp canonifier about timestamps before ~2001
+    (Jon Siwek, Corelight)
+
+2.5-877 | 2018-08-20 14:58:58 -0500
+
+  * Remove the node-specific local-*.bro scripts (Daniel Thayer)
+
+2.5-875 | 2018-08-20 12:45:32 -0500
+
+  * Improve diff-remove-abspath canonifier: collapse '/' sequences
+    (Jon Siwek, Corelight)
+
+  * Remove unused redef-able constants (Daniel Thayer)
+
+  * Convert some redef-able constants to runtime options (Daniel Thayer)
+
+2.5-870 | 2018-08-17 17:07:57 -0500
+
+  * Documentation improvements (Daniel Thayer)
+
+2.5-855 | 2018-08-17 16:34:51 -0500
+
+  * Add script to support the old DHCP events (Vlad Grigorescu)
+
+2.5-852 | 2018-08-17 15:15:55 -0500
+
+  * BIT-466: add redef += support to vectors (Jon Siwek, Corelight)
+
 2.5-850 | 2018-08-17 11:12:53 -0500
 
   * BIT-1815: move SMB::write_cmd_log functionality into policy/ script

NEWS

@@ -114,7 +114,11 @@ New Functionality
   - Option variables: The new "option" keyword allows variables to be
     declared as runtime options. Such variables cannot be changed
     using normal assignments. Instead, they can be changed using the
-    new function Option::set.
+    new function Config::set_value.  This function will automatically
+    apply the change to all nodes in a cluster.  Note that options can also
+    be changed using the new function Option::set, but this function will
+    not send the change to any other nodes, so Config::set_value should
+    typically be used instead of Option::set.
 
     It is possible to "subscribe" to an option through
     Option::set_change_handler, which will trigger a handler callback
@@ -162,7 +166,7 @@ New Functionality
         }
 
     The specified file will now be monitored continuously for changes, so
-    that writing "testbool T" into /path/to/config.dat will
+    that writing "TestConfig::testbool T" into /path/to/config.dat will
     automatically update the option's value accordingly.
 
     The configuration framework creates a config.log that shows all
@@ -220,7 +224,6 @@ New Functionality
   ssl_server_curve event, ssl_server_curve is now marked as deprecated.
 
 - Functions for retrieving files by their ID have been added:
-
   Files::file_exists, Files::lookup_File
 
 - New functions in the logging API: Log::get_filter_names, Log::enable_stream
@@ -237,13 +240,13 @@ New Functionality
     mount_proc_umnt, mount_proc_umnt_all, mount_proc_not_implemented,
     mount_reply_status.
 
-- Added new NFS events: nfs_proc_symlink, nfs_proc_link, nfs_proc_sattr
+- Added new NFS events: nfs_proc_symlink, nfs_proc_link, nfs_proc_sattr.
 
 - The SMB scripts in policy/protocols/smb are now moved into base/protocols/smb
   and loaded/enabled by default.
 
 - Added new SMB events: smb1_transaction_secondary_request,
-  smb1_transaction2_secondary_request, smb1_transaction_response
+  smb1_transaction2_secondary_request, smb1_transaction_response.
 
 - Bro can now decrypt Kerberos tickets, and retrieve the authentication from
   them, given a suitable keytab file.
@@ -260,20 +263,18 @@ New Functionality
 
 - You can now specify that a pattern matches in a case-insensitive
   fashion by adding 'i' to the end of its specification.  So for example
-  /fOO/i == "Foo" yields T, as does /fOO/i in "xFoObar".  Characters
+  /fOO/i == "Foo" yields T, as does /fOO/i in "xFoObar".
-  enclosed in quotes however keep their casing, so /"fOO"/i in "xFoObar"
-  yields F, though it yields T for "xfOObar".
 
   You can achieve the same functionality for a subpattern enclosed in
   parentheses by adding "?i:" to the open parenthesis.  So for example
-  "/foo|(?i:bar)/" will match "BaR", but not "FoO".
+  /foo|(?i:bar)/ will match "BaR", but not "FoO".
 
   For both ways of specifying case-insensitivity, characters enclosed in
-  double quotes maintain their case-sensitivity.  So for example /"foo"/i
+  double quotes remain case-sensitive.  So for example /"foo"/i will not
-  will not match "Foo", but it will match "foo".
+  match "Foo", but it will match "foo".
 
 - "make install" now installs Bro's include headers (and more) into
-  --prefix so that compiling plugins does no longer need access to a
+  --prefix so that compiling plugins no longer needs access to a
   source/build tree. For OS distributions, this also facilitates
   creating "bro-devel" packages providing all files necessary to build
   plugins.
@@ -313,7 +314,10 @@ New Functionality
   of non-equality, proper superset, and superset-or-equal.
 
 - An expression of the form "v += e" will append the value of the expression
-  "e" to the end of the vector "v" (of course assuming type-compatbility).
+  "e" to the end of the vector "v" (of course assuming type-compatibility).
+  "redef v += { a, b, c }" will similarly extend a vector previously declared
+  with &redef by appending the result of expressions "a", "b", and "c" to
+  the vector at initialization-time.
 
 Changed Functionality
 ---------------------
@@ -333,9 +337,9 @@ Changed Functionality
     script since it's generally less relevant now with the updated log.
 
   - Removed the base/protocols/dhcp/utils.bro script and thus the
-    'reverse_ip' function.
+    "reverse_ip" function.
 
-  - Replaced all DHCP events with the single 'dhcp_message' event.
+  - Replaced all DHCP events with the single "dhcp_message" event.
     The list of removed events includes:
 
     - dhcp_discover
@@ -347,6 +351,13 @@ Changed Functionality
     - dhcp_release
     - dhcp_inform
 
+  - A new script, policy/protocols/dhcp/deprecated_events.bro, may be loaded
+    to aid those transitioning away from the list of "removed" events above.
+    The script provides definitions for the old events and automatically
+    generates them from a dhcp_message handler, thus providing equivalent
+    functionality to the previous Bro release.  Such usage emits deprecation
+    warnings.
+
 - Removed policy/misc/known-devices.bro script and thus
   known_devices.log will no longer be created.
 
@@ -371,9 +382,9 @@ Changed Functionality
   redef SOCKS::default_capture_password = T;
 
 - The DNS base scripts no longer generate some noisy and annoying
-  weirds (dns_unmatched_msg, dns_unmatched_msg_quantity, dns_unmatched_reply)
+  weirds (dns_unmatched_msg, dns_unmatched_msg_quantity, dns_unmatched_reply).
 
-- The 'tunnel_parents' field of conn.log is now marked &optional, so, for
+- The "tunnel_parents" field of conn.log is now marked &optional, so, for
   the default configuration of logs, this field will show "-" instead of
   "(empty)" for connections that lack any tunneling.
 
@@ -393,11 +404,12 @@ Changed Functionality
   - event ssl_server_signature now has an additional argument
     "signature_and_hashalgorithm".
 
-- The "dnp3_header_block" event no longer has the "start" parameter
+- The "dnp3_header_block" event no longer has the "start" parameter.
 
 - The string_to_pattern() built-in (and the now-deprecated merge_pattern()
   built-in) is no longer restricted to only be called at initialization time.
 
+
 - GeoIP Legacy Database support has been replaced with GeoIP2 MaxMind DB
   format support.
 
@@ -407,6 +419,21 @@ Changed Functionality
     after January 2, 2019.  It's also noted that all GeoIP Legacy databases
     may be discontinued as they are superseded by GeoIP2.
 
+- "Weird" events are now generally suppressed/sampled by default according to
+  some tunable parameters:
+
+  - Weird::sampling_whitelist
+  - Weird::sampling_threshold
+  - Weird::sampling_rate
+  - Weird::sampling_duration
+
+  Those options can be changed if one needs the previous behavior of
+  a "net_weird", "flow_weird", or "conn_weird" event being raised for
+  every single event.  Otherwise, there is a new weird_stats.log which
+  contains concise summaries of weird counts per type per time period
+  and the original weird.log may not differ much either, except in
+  the cases where a particular weird type exceeds the sampling threshold.
+
 Removed Functionality
 ---------------------
 
@@ -416,10 +443,12 @@ Removed Functionality
   https://github.com/bro/packages for a list of Bro packages currently
   available.
 
-- BroControl: The option 'IPv6Comm' and 'ZoneID' options are no longer
+- The "ocsp_request" event no longer has "requestorName" parameter.
+
+- BroControl: The "IPv6Comm" and "ZoneID" options are no longer
   available (though Broker should be able to handle IPv6 automatically).
 
-- The "ocsp_request" event no longer has "requestorName" parameter.
+- The node-specific ``site/local-*.bro`` scripts have been removed.
 
 Deprecated Functionality
 ------------------------
@@ -435,10 +464,6 @@ Deprecated Functionality
   as BiFs like send_id(). Use Broker data stores and the new
   configuration framework instead.
 
-- BroControl: The 'update' command is deprecated and scheduled for
-  removal with the next Bro release. Bro's new configuration framework
-  is taking its place.
-
 - Mixing of scalars and vectors, such as "v + e" yielding a vector
   corresponding to the vector v with the scalar e added to each of
   its elements, has been deprecated.
@@ -449,6 +474,10 @@ Deprecated Functionality
 - The undocumented feature of using "&&" and "||" operators for patterns
   has been deprecated.
 
+- BroControl: The "update" command is deprecated and scheduled for
+  removal with the next Bro release. Bro's new configuration framework
+  is taking its place.
+
 Bro 2.5.1
 =========
 

VERSION

@@ -1 +1 @@
-2.5-850
+2.5-884

aux/broctl

@@ -1 +1 @@
-Subproject commit 70a8b2e15105f4c238765a882151718162e46208
+Subproject commit 336e719c33d64efebe65f11322e2fbc6d11c946b

aux/broker

@@ -1 +1 @@
-Subproject commit e0f9f6504db9285a48e0be490abddf959999a404
+Subproject commit 2727afff785201168b33b0c7448c886643f10e08

doc/frameworks/configuration.rst

@@ -11,7 +11,7 @@ Bro includes a "configuration framework" that allows
 updating script options dynamically at runtime. This functionality
 consists of several components: an "option" declaration, the
 ability to specify input files to enable changing the value of options at
-runtime, a couple of built-in functions, and a log file "config.log"
+runtime, a couple of functions, and a log file "config.log"
 which contains information about every change to option values.
 
 
@@ -50,14 +50,28 @@ The "option" keyword allows variables to be declared as configuration options.
         option my_networks: set[subnet] = {};
         option enable_feature = F;
         option hostname = "testsystem";
+        option timeout = 1min;
+        option my_ports: vector of port = {};
     }
 
 The rules regarding options can be thought of as being in between global
 variables and constants.  Like global variables, options cannot be declared
 inside a function, hook, or event handler.  Like constants, options must be
-initialized when declared.  The value of an option can change at runtime,
+initialized when declared (the type can often be inferred from the initializer
+but may need to be specified).  The value of an option can change at runtime,
 but options cannot be assigned a new value using normal assignments.
 
+The initial value of an option can be redefined with a :bro:keyword:`redef`
+declaration just like for global variables and constants.  The only difference
+being that there is no need to specify the :bro:attr:`&redef` attribute in
+the declaration of an option.  For example, given the above option
+declarations, here are some possible redefs:
+
+.. code:: bro
+
+    redef TestModule::enable_feature = T;
+    redef TestModule::my_networks += { 10.1.0.0/16, 10.2.0.0/16 };
+
 
 Changing options
 ----------------
@@ -70,21 +84,32 @@ The format for these files looks like this:
 
     [option name][tab/spaces][new value]
 
-Configuration files can be specified by adding them to Config::config_files.
+Configuration files can be specified by adding them
+to :bro:id:`Config::config_files`.  Note that in a cluster configuration,
+only the manager node attempts to read the specified configuration files.
+
 For example, simply add something like this to local.bro:
 
 .. code:: bro
 
     redef Config::config_files += { "/path/to/config.dat" };
 
-The specified configuration file will then be monitored continuously for changes,
+The specified configuration file will then be monitored continuously for
-so that writing ``TestModule::enable_feature T`` into that file will
+changes, so that writing ``TestModule::enable_feature T`` into that file will
-automatically update the option's value accordingly.  Here is an example
+automatically update the option's value accordingly (in a cluster
-configuration file::
+configuration, the change will be sent from the manager to all other nodes in
+the cluster).  Here is an example configuration file::
 
     TestModule::my_networks 10.0.12.0/24,192.168.17.0/24
     TestModule::enable_feature  T
     TestModule::hostname  host-1
+    TestModule::timeout 50.5
+    TestModule::my_ports 80/tcp,53/udp
+
+Note that as seen in the above example, for options of
+type :bro:type:`interval`, the numeric value in the config file
+is interpreted as seconds and there cannot be any time units
+(such as sec, min, etc.).
 
 Internally, the configuration framework uses the Bro input framework
 with a type of input reader specifically for reading config files. Users
@@ -94,13 +119,34 @@ for configuration files: the files need no header lines and either
 tabs or spaces are accepted as separators.
 
 If you inspect the configuration framework scripts, you will notice that the
-scripts simply catch events from the input framework and then a built-in
+scripts simply catch events from the input framework and then a
-function :bro:see:`Option::set` is called to set an option to the new value.
+function :bro:see:`Config::set_value` is called to set an option to the new
-If you want to change an option yourself during runtime, you can
+value.  If you want to change an option yourself during runtime, you can
-call Option::set directly from a script.
+call Config::set_value directly from a script (in a cluster configuration,
+this only needs to happen on the manager, as the change will be automatically
+sent to all other nodes in the cluster).
 
-The log file "config.log" contains information about each configuration
+Note that some data types (such as pattern, table, and record) are not
-change that occurs during runtime.
+supported by the config input reader.  In that case you would need to use
+the Config::set_value function to change the value of such an option as
+shown in the following example.
+
+.. code:: bro
+
+    module TestModule;
+
+    export {
+        option host_port: table[addr] of port = {};
+    }
+
+    event bro_init() {
+        local t: table[addr] of port = { [10.0.0.2] = 123/tcp };
+        Config::set_value("TestModule::host_port", t);
+    }
+
+Regardless of whether an option change is triggered by a config file or by
+the Config::set_value function, the change is always logged to the
+log file "config.log".
 
 
 Change handlers
@@ -114,7 +160,11 @@ accordingly):
 
 .. code:: bro
 
+    module TestModule;
+
+    export {
         option testaddr = 127.0.0.1;
+    }
 
     # Note: the data type of 2nd parameter and return type must match
     function change_addr(ID: string, new_value: addr): addr
@@ -125,24 +175,26 @@ accordingly):
 
     event bro_init()
         {
-        Option::set_change_handler("testaddr", change_addr);
+        Option::set_change_handler("TestModule::testaddr", change_addr);
         }
 
-Each time the specified option value is changed, the change handler
+Immediately before the specified option value is changed, the change handler
-function will be called before the change is performed.  The value returned
+function will be called.  The value returned by the change handler is the
-by the change handler is the value finally assigned to the option. This
+value finally assigned to the option. This allows, for example, checking of
-allows, for example, checking of values to reject invalid input (the original
+values to reject invalid input (the original value can be returned to reject
-value can be returned to reject the change).
-
-A change handler can optionally have a third argument, which is the location
-string (this is normally the pathname of the configuration file that triggered
 the change).
 
-It is also possible to chain together multiple change handlers.  In this
+It is possible define multiple change handlers for a single option.  In
-case, the value returned by the first change handler is the "new value" seen
+this case, the change handlers are chained together: the value returned by the
-by the next change handler, and so on.  The built-in function
+first change handler is the "new value" seen by the next change handler, and
-:bro:see:`Option::set_change_handler` takes an optional third argument
+so on.  The built-in function :bro:see:`Option::set_change_handler` takes an
-that can specify a priority for the handlers.
+optional third argument that can specify a priority for the handlers.
+
+A change handler function can optionally have a third argument of type
+string.  When a config file triggers a change, then the third argument is
+the pathname of the config file.  When the Config::set_value function triggers
+a change, then the third argument of the change handler is the value passed
+to the optional third argument of the Config::set_value function.
 
 Note that change handlers are also used internally by the
 configuration framework. If you look at the script level source code of

doc/frameworks/geoip.rst

@@ -10,8 +10,8 @@ GeoLocation
     During the process of creating policy scripts the need may arise
     to find the geographic location for an IP address. Bro had support
     for the `GeoIP library <http://www.maxmind.com/app/c>`__ at the
-    policy script level from release 1.3 to 2.5.X to account for this
+    policy script level from release 1.3 to 2.5.x to account for this
-    need.  Starting with release 2.6 GeoIP support requires `libmaxminddb
+    need.  Starting with release 2.6, GeoIP support requires `libmaxminddb
     <https://github.com/maxmind/libmaxminddb/releases>`__.
     To use this functionality, you need to first install the libmaxminddb
     software, and then install the GeoLite2 city database before building
@@ -19,17 +19,11 @@ GeoLocation
 
 .. contents::
 
-Install libGeoIP
+Install libmaxminddb
-----------------
+--------------------
 
 Before building Bro, you need to install libmaxminddb.
 
-* FreeBSD:
-
-  .. console::
-
-      sudo pkg install libmaxminddb
-
 * RPM/RedHat-based Linux:
 
   .. console::
@@ -42,12 +36,17 @@ Before building Bro, you need to install libmaxminddb.
 
       sudo apt-get install libmaxminddb-dev
 
+* FreeBSD:
+
+  .. console::
+
+      sudo pkg install libmaxminddb
+
 * Mac OS X:
 
   You need to install from your preferred package management system
-  (e.g. MacPorts, Fink, or Homebrew).  The name of the package that you need
+  (e.g. Homebrew, MacPorts, or Fink).  For Homebrew, the name of the package
-  may be libmaxminddb, maxminddb, or libmaxminddb-dev, depending on which
+  that you need is libmaxminddb.
-  package management system you are using.
 
 
 GeoLite2-City Database Installation
@@ -64,8 +63,8 @@ the GeoLite2 city binary database:
     wget http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.tar.gz
     tar zxf GeoLite2-City.tar.gz
 
-Next, the file "GeoLite2-City_YYYYMMDD/GeoLite2-City.mmdb" needs to be renamed
+Next, the file "GeoLite2-City_YYYYMMDD/GeoLite2-City.mmdb" needs to be moved
-and put in the GeoIP database directory.  This directory should already exist
+to the GeoIP database directory.  This directory might already exist
 and will vary depending on which platform and package you are using.  For
 FreeBSD, use ``/usr/local/share/GeoIP``.  For Linux, use ``/usr/share/GeoIP``
 or ``/var/lib/GeoIP`` (choose whichever one already exists).
@@ -88,22 +87,23 @@ functionality works by running a command like this:
 
 If you see an error message similar to "Failed to open GeoIP location
 database", then you may need to either rename or move your GeoIP
-location database file.  Bro looks for location database files in the
+location database file.  If the :bro:see:`mmdb_dir` value is set to a
-following order by default:
+directory pathname (it is not set by default), then Bro looks for location
+database files in that directory.  If none are found or if mmdb_dir is not set,
+then Bro looks for location database files in the following order:
 
-    /usr/share/GeoIP/GeoLite2-City.mmdb
+* /usr/share/GeoIP/GeoLite2-City.mmdb
-    /var/lib/GeoIP/GeoLite2-City.mmdb
+* /var/lib/GeoIP/GeoLite2-City.mmdb
-    /usr/local/share/GeoIP/GeoLite2-City.mmdb
+* /usr/local/share/GeoIP/GeoLite2-City.mmdb
-    /usr/local/var/GeoIP/GeoLite2-City.mmdb
+* /usr/local/var/GeoIP/GeoLite2-City.mmdb
-    /usr/share/GeoIP/GeoLite2-Country.mmdb
+* /usr/share/GeoIP/GeoLite2-Country.mmdb
-    /var/lib/GeoIP/GeoLite2-Country.mmdb
+* /var/lib/GeoIP/GeoLite2-Country.mmdb
-    /usr/local/share/GeoIP/GeoLite2-Country.mmdb
+* /usr/local/share/GeoIP/GeoLite2-Country.mmdb
-    /usr/local/var/GeoIP/GeoLite2-Country.mmdb
+* /usr/local/var/GeoIP/GeoLite2-Country.mmdb
 
 If you see an error message similar to "Bro was not configured for GeoIP
-support", then you either need to rebuild Bro and make sure it is linked
+support", then you need to rebuild Bro and make sure it is linked
-against libmaxminddb or else set the :bro:see:`mmdb_dir` value
+against libmaxminddb.  Normally, if libmaxminddb is installed correctly then it
-correctly.  Normally, if libmaxminddb is installed correctly then it
 should automatically be found when building Bro.  If this doesn't
 happen, then you may need to specify the path to the libmaxminddb
 installation (e.g. ``./configure --with-geoip=<path>``).

doc/install/install.rst

@@ -60,6 +60,9 @@ To install the required dependencies, you can use:
 
      sudo apt-get install cmake make gcc g++ flex bison libpcap-dev libssl-dev python-dev swig zlib1g-dev
 
+  If your system uses Python 2.7, then you will also need to install the
+  "python-ipaddress" package.
+
 * FreeBSD:
 
   Most required dependencies should come with a minimal FreeBSD install
@@ -114,7 +117,7 @@ Optional Dependencies
 Bro can make use of some optional libraries and tools if they are found at
 build time:
 
-    * LibGeoIP (for geolocating IP addresses)
+    * libmaxminddb (for geolocating IP addresses)
     * sendmail (enables Bro and BroControl to send mail)
     * curl (used by a Bro script that implements active HTTP)
     * gperftools (tcmalloc is used to improve memory and CPU usage)
@@ -122,9 +125,9 @@ build time:
     * PF_RING (Linux only, see :doc:`Cluster Configuration <../configuration/index>`)
     * ipsumdump (for trace-summary; http://www.cs.ucla.edu/~kohler/ipsumdump)
 
-LibGeoIP is probably the most interesting and can be installed
+Geolocation is probably the most interesting and can be installed
 on most platforms by following the instructions for :ref:`installing
-libGeoIP and the GeoIP database
+the GeoIP library and database
 <geolocation>`.
 
 

doc/script-reference/log-files.rst

@@ -14,8 +14,6 @@ Network Protocols
 +============================+=======================================+=================================+
 | conn.log                   | TCP/UDP/ICMP connections              | :bro:type:`Conn::Info`          |
 +----------------------------+---------------------------------------+---------------------------------+
-| config.log                 | Configuration option changes          | :bro:type:`Config::Info`        |
-+----------------------------+---------------------------------------+---------------------------------+
 | dce_rpc.log                | Distributed Computing Environment/RPC | :bro:type:`DCE_RPC::Info`       |
 +----------------------------+---------------------------------------+---------------------------------+
 | dhcp.log                   | DHCP leases                           | :bro:type:`DHCP::Info`          |
@@ -154,6 +152,8 @@ Miscellaneous
 +----------------------------+---------------------------------------+---------------------------------+
 | weird.log                  | Unexpected network-level activity     | :bro:type:`Weird::Info`         |
 +----------------------------+---------------------------------------+---------------------------------+
+| weird-stats.log            | Statistics about unexpected activity  | :bro:type:`WeirdStats::Info`    |
++----------------------------+---------------------------------------+---------------------------------+
 
 Bro Diagnostics
 ---------------
@@ -161,12 +161,14 @@ Bro Diagnostics
 +----------------------------+---------------------------------------+---------------------------------+
 | Log File                   | Description                           | Field Descriptions              |
 +============================+=======================================+=================================+
+| broker.log                 | Peering status events between Bro or  | :bro:type:`Broker::Info`        |
+|                            | Broker-enabled processes              |                                 |
++----------------------------+---------------------------------------+---------------------------------+
 | capture_loss.log           | Packet loss rate                      | :bro:type:`CaptureLoss::Info`   |
 +----------------------------+---------------------------------------+---------------------------------+
 | cluster.log                | Bro cluster messages                  | :bro:type:`Cluster::Info`       |
 +----------------------------+---------------------------------------+---------------------------------+
-| broker.log                 | Peering status events between Bro or  | :bro:type:`Broker::Info`        |
+| config.log                 | Configuration option changes          | :bro:type:`Config::Info`        |
-|                            | Broker-enabled processes              |                                 |
 +----------------------------+---------------------------------------+---------------------------------+
 | loaded_scripts.log         | Shows all scripts loaded by Bro       | :bro:type:`LoadedScripts::Info` |
 +----------------------------+---------------------------------------+---------------------------------+

doc/script-reference/operators.rst

@@ -10,6 +10,11 @@ Relational operators
 
 The relational operators evaluate to type :bro:type:`bool`.
 
+In addition to numeric operands, the relational operators also work with
+operands of type :bro:type:`interval`, :bro:type:`time`, :bro:type:`string`,
+:bro:type:`port`, :bro:type:`addr`, and :bro:type:`set`.
+
+
 +------------------------------+--------------+
 | Name                         | Syntax       |
 +==============================+==============+
@@ -104,12 +109,27 @@ only.
 | Bitwise complement           | ~ *a*       |
 +------------------------------+-------------+
 
+Set operators
+-------------
+
++------------------------------+-------------+
+| Name                         | Syntax      |
++==============================+=============+
+| Set intersection             | *s1* & *s2* |
++------------------------------+-------------+
+| Set union                    | *s1* | *s2* |
++------------------------------+-------------+
+| Set difference               | *s1* - *s2* |
++------------------------------+-------------+
 
 Assignment operators
 --------------------
 
 The assignment operators evaluate to the result of the assignment.
 
+The "+=" operator can also be used to append an element to the end of a
+vector.  For example, ``v += e`` is equivalent to ``v[|v|] = e``.
+
 +------------------------------+-------------+
 | Name                         | Syntax      |
 +==============================+=============+
@@ -215,7 +235,14 @@ Other operators
 | Name                           | Syntax            | Notes                  |
 +================================+===================+========================+
 | Membership test                | *a* in *b*        |Evaluates to type       |
-|                                |                   |:bro:type:`bool`. Do not|
+|                                |                   |:bro:type:`bool`. Works |
+|                                |                   |with :bro:type:`string`,|
+|                                |                   |:bro:type:`pattern`,    |
+|                                |                   |:bro:type:`subnet`,     |
+|                                |                   |:bro:type:`set`,        |
+|                                |                   |:bro:type:`table`, or   |
+|                                |                   |:bro:type:`vector`      |
+|                                |                   |operands.  Do not       |
 |                                |                   |confuse this use of "in"|
 |                                |                   |with that used in a     |
 |                                |                   |:bro:keyword:`for`      |

doc/script-reference/statements.rst

@@ -190,7 +190,11 @@ all loaded Bro scripts.
         option hostname = "host-1";
         option peers: set[addr] = {};
 
-    The value of an option cannot be changed by an assignment statement.
+    The initial value can be redefined with a :bro:keyword:`redef`.
+
+    The value of an option cannot be changed by an assignment statement, but
+    it can be changed by either the :bro:id:`Config::set_value` function or
+    by changing a config file specified in :bro:id:`Config::config_files`.
 
     The scope of an option is global.
 
@@ -215,26 +219,30 @@ all loaded Bro scripts.
 
 .. bro:keyword:: redef
 
-    There are three ways that "redef" can be used:  to change the value of
+    There are several ways that "redef" can be used:  to redefine the initial
-    a global variable (but only if it has the :bro:attr:`&redef` attribute),
+    value of a global variable or runtime option, to extend a record type or
-    to extend a record type or enum type, or to specify
+    enum type, or to specify a new event handler body that replaces all those
-    a new event handler body that replaces all those that were previously
+    that were previously defined.
-    defined.
 
-    If you're using "redef" to change a global variable (defined using either
+    If you're using "redef" to redefine the initial value of a global variable
-    :bro:keyword:`const` or :bro:keyword:`global`), then the variable that you
+    (defined using either :bro:keyword:`const` or :bro:keyword:`global`), then
-    want to change must have the :bro:attr:`&redef` attribute.  If the variable
+    the variable that you want to change must have the :bro:attr:`&redef`
-    you're changing is a table, set, or pattern, you can use ``+=`` to add
+    attribute.  You can use "redef" to redefine the initial value of a
-    new elements, or you can use ``=`` to specify a new value (all previous
+    runtime option (defined using :bro:keyword:`option`) even if it doesn't
-    contents of the object are removed).  If the variable you're changing is a
+    have the :bro:attr:`&redef` attribute.
-    set or table, then you can use the ``-=`` operator to remove the
+
-    specified elements (nothing happens for specified elements that don't
+    If the variable you're changing is a table, set, vector, or pattern, you can
+    use ``+=`` to add new elements, or you can use ``=`` to specify a new value
+    (all previous contents of the object are removed).  If the variable you're
+    changing is a set or table, then you can use the ``-=`` operator to remove
+    the specified elements (nothing happens for specified elements that don't
     exist).  If the variable you are changing is not a table, set, or pattern,
     then you must use the ``=`` operator.
 
     Examples::
 
         redef pi = 3.14;
+        redef set_of_ports += { 22/tcp, 53/udp };
 
     If you're using "redef" to extend a record or enum, then you must
     use the ``+=`` assignment operator.

doc/script-reference/types.rst

@@ -253,10 +253,10 @@ Here is a more detailed description of each type:
 
     When specifying a pattern, you can add a final ``i`` specifier to
     mark it as case-insensitive.  For example, ``/foo|bar/i`` will match
-    a "foo", "Foo", "BaR", etc.
+    "foo", "Foo", "BaR", etc.
 
     You can also introduce a case-insensitive sub-pattern by enclosing it
-    in ``(?i:``<pattern>``)``.  So, for example, ``/foo|(?i:bar)/`` will
+    in ``(?i:<pattern>)``.  So, for example, ``/foo|(?i:bar)/`` will
     match "foo" and "BaR", but *not* "Foo".
 
     For both ways of specifying case-insensitivity, characters enclosed
@@ -545,13 +545,14 @@ Here is a more detailed description of each type:
         |s|
 
     You can compute the union, intersection, or difference of two sets
-    using the ``|``, ``&``, and ``-`` operators.  You can compare
+    using the ``|``, ``&``, and ``-`` operators.
-    sets for equality (they have exactly the same elements) using ``==``.
+
-    The ``<`` operator returns ``T`` if the lefthand operand is a proper
+    You can compare sets for equality (they have exactly the same elements)
-    subset of the righthand operand.  Similarly, ``<=`` returns ``T``
+    using ``==``.  The ``<`` operator returns ``T`` if the lefthand operand
-    if the lefthand operator is a subset (not necessarily proper, i.e.,
+    is a proper subset of the righthand operand.  Similarly, ``<=``
-    it may be equal to the righthand operand).  The operators ``!=``, ``>``
+    returns ``T`` if the lefthand operator is a subset (not necessarily proper,
-    and ``>=`` provide the expected complementary operations.
+    i.e., it may be equal to the righthand operand).  The operators ``!=``,
+    ``>`` and ``>=`` provide the expected complementary operations.
 
     See the :bro:keyword:`for` statement for info on how to iterate over
     the elements in a set.
@@ -601,8 +602,9 @@ Here is a more detailed description of each type:
 
         v[3] = "four";
 
-    The number of elements in a vector can be obtained by placing the vector
+    The size of a vector (this is one greater than the highest index value, and
-    identifier between vertical pipe characters:
+    is normally equal to the number of elements in the vector) can be obtained
+    by placing the vector identifier between vertical pipe characters:
 
     .. code:: bro
 
@@ -622,6 +624,11 @@ Here is a more detailed description of each type:
 
         v[|v|] = e;
 
+    The "in" operator can be used to check if a value has been assigned at a
+    specified index value in the vector.  For example, if a vector has size 4,
+    then the expression ``3 in v`` would yield true and ``4 in v`` would yield
+    false.
+
     Vectors of integral types (``int`` or ``count``) support the pre-increment
     (``++``) and pre-decrement operators (``--``), which will increment or
     decrement each element in the vector.

scripts/CMakeLists.txt

@@ -14,19 +14,3 @@ InstallPackageConfigFile(
     ${CMAKE_CURRENT_SOURCE_DIR}/site/local.bro
     ${BRO_SCRIPT_INSTALL_PATH}/site
     local.bro)
-InstallPackageConfigFile(
-    ${CMAKE_CURRENT_SOURCE_DIR}/site/local-manager.bro
-    ${BRO_SCRIPT_INSTALL_PATH}/site
-    local-manager.bro)
-InstallPackageConfigFile(
-    ${CMAKE_CURRENT_SOURCE_DIR}/site/local-logger.bro
-    ${BRO_SCRIPT_INSTALL_PATH}/site
-    local-logger.bro)
-InstallPackageConfigFile(
-    ${CMAKE_CURRENT_SOURCE_DIR}/site/local-proxy.bro
-    ${BRO_SCRIPT_INSTALL_PATH}/site
-    local-proxy.bro)
-InstallPackageConfigFile(
-    ${CMAKE_CURRENT_SOURCE_DIR}/site/local-worker.bro
-    ${BRO_SCRIPT_INSTALL_PATH}/site
-    local-worker.bro)

scripts/base/frameworks/config/input.bro

@@ -60,8 +60,8 @@ event InputConfig::new_value(name: string, source: string, id: string, value: an
 
 function read_config(filename: string)
 	{
-	# Only read the configuration on the manager. The other nodes are being fed from
+	# Only read the configuration on the manager. The other nodes are being fed
-	# the manager.
+	# from the manager.
 	if ( Cluster::is_enabled() && Cluster::local_node_type() != Cluster::MANAGER )
 		return;
 

scripts/base/frameworks/config/main.bro

@@ -28,15 +28,15 @@ export {
 	## record as it is sent on to the logging framework.
 	global log_config: event(rec: Info);
 
-	## Broker topic for announcing new configuration value. Sending new_value,
+	## Broker topic for announcing new configuration values. Sending new_value,
-	## peers can send configuration changes that will be distributed accross
+	## peers can send configuration changes that will be distributed across
 	## the entire cluster.
 	const change_topic = "bro/config/change";
 
 	## This function is the config framework layer around the lower-level
 	## :bro:see:`Option::set` call. Config::set_value will set the configuration
 	## value for all nodes in the cluster, no matter where it was called. Note
-	## that `bro:see:`Option::set` does not distribute configuration changes
+	## that :bro:see:`Option::set` does not distribute configuration changes
 	## to other nodes.
 	##
 	## ID: The ID of the option to update.
@@ -45,7 +45,7 @@ export {
 	##
 	## location: Optional parameter detailing where this change originated from.
 	##
-	## Returns: true on success, false when an error ocured.
+	## Returns: true on success, false when an error occurs.
 	global set_value: function(ID: string, val: any, location: string &default = "" &optional): bool;
 }
 
@@ -73,7 +73,7 @@ event Config::cluster_set_option(ID: string, val: any, location: string)
 function set_value(ID: string, val: any, location: string &default = "" &optional): bool
 	{
 	local cache_val: any;
-	# first cache value in case setting it succeeds and we have to store it.
+	# First cache value in case setting it succeeds and we have to store it.
 	if ( Cluster::local_node_type() == Cluster::MANAGER )
 		cache_val = copy(val);
 	# First try setting it locally - abort if not possible.
@@ -151,8 +151,8 @@ event bro_init() &priority=10
 
 	# Limit logging to the manager - everyone else just feeds off it.
 @if ( !Cluster::is_enabled() || Cluster::local_node_type() == Cluster::MANAGER )
-	# Iterate over all existing options and add ourselves as change handlers with
+	# Iterate over all existing options and add ourselves as change handlers
-	# a low priority so that we can log the changes.
+	# with a low priority so that we can log the changes.
 	local gids = global_ids();
 	for ( i in gids )
 		{

scripts/base/frameworks/netcontrol/catch-and-release.bro

@@ -142,7 +142,7 @@ export {
 
 	## If true, catch and release warns if packets of an IP address are still seen after it
 	## should have been blocked.
-	const catch_release_warn_blocked_ip_encountered = F &redef;
+	option catch_release_warn_blocked_ip_encountered = F;
 
 	## Time intervals for which subsequent drops of the same IP take
 	## effect.

scripts/base/frameworks/notice/actions/add-geodata.bro

@@ -19,7 +19,7 @@ export {
 	};
 	
 	redef record Info += {
-		## If libGeoIP support is built in, notices can have geographic
+		## If GeoIP support is built in, notices can have geographic
 		## information attached to them.
 		remote_location: geo_location  &log &optional;
 	};

scripts/base/frameworks/notice/main.bro

@@ -201,14 +201,14 @@ export {
 	## Address that emails will be from.
 	##
 	## Note that this is overridden by the BroControl MailFrom option.
-	const mail_from           = "Big Brother <bro@localhost>" &redef;
+	option mail_from           = "Big Brother <bro@localhost>";
 	## Reply-to address used in outbound email.
-	const reply_to            = "" &redef;
+	option reply_to            = "";
 	## Text string prefixed to the subject of all emails sent out.
 	##
 	## Note that this is overridden by the BroControl MailSubjectPrefix
 	## option.
-	const mail_subject_prefix = "[Bro]" &redef;
+	option mail_subject_prefix = "[Bro]";
 	## The maximum amount of time a plugin can delay email from being sent.
 	const max_email_delay     = 15secs &redef;
 

scripts/base/init-bare.bro

@@ -82,6 +82,13 @@ type addr_vec: vector of addr;
 ##    directly and then remove this alias.
 type table_string_of_string: table[string] of string;
 
+## A table of counts indexed by strings.
+##
+## .. todo:: We need this type definition only for declaring builtin functions
+##    via ``bifcl``. We should extend ``bifcl`` to understand composite types
+##    directly and then remove this alias.
+type table_string_of_count: table[string] of count;
+
 ## A set of file analyzer tags.
 ##
 ## .. todo:: We need this type definition only for declaring builtin functions
@@ -626,6 +633,17 @@ type BrokerStats: record {
 	num_ids_outgoing: count;
 };
 
+## Statistics about reporter messages and weirds.
+##
+## .. bro:see:: get_reporter_stats
+type ReporterStats: record {
+	## Number of total weirds encountered, before any rate-limiting.
+	weirds: count;
+	## Number of times each individual weird is encountered, before any
+	## rate-limiting is applied.
+	weirds_by_type:	table[string] of count;
+};
+
 ## Deprecated.
 ##
 ## .. todo:: Remove. It's still declared internally but doesn't seem  used anywhere
@@ -4826,6 +4844,35 @@ export {
 	type Cluster::Pool: record {};
 }
 
+module Weird;
+export {
+	## Prevents rate-limiting sampling of any weirds named in the table.
+	const sampling_whitelist: set[string] &redef;
+
+	## How many weirds of a given type to tolerate before sampling begins.
+	## i.e. this many consecutive weirds of a given type will be allowed to
+	## raise events for script-layer handling before being rate-limited.
+	const sampling_threshold = 25 &redef;
+
+	## The rate-limiting sampling rate.  One out of every of this number of
+	## rate-limited weirds of a given type will be allowed to raise events
+	## for further script-layer handling.
+	const sampling_rate = 1000 &redef;
+
+	## How long a weird of a given type is allowed to keep state/counters in
+	## memory.  For "net" weirds an expiration timer starts per weird name when
+	## first initializing its counter.  For "flow" weirds an expiration timer
+	## starts once per src/dst IP pair for the first weird of any name.  For
+	## "conn" weirds, counters and expiration timers are kept for the duration
+	## of the connection for each named weird and reset when necessary.  e.g.
+	## if a "conn" weird by the name of "foo" is seen more than
+	## :bro:see:`Weird::sampling_threshold` times, then an expiration timer
+	## begins for "foo" and upon triggering will reset the counter for "foo"
+	## and unthrottle its rate-limiting until it once again exceeds the
+	## threshold.
+	const sampling_duration = 10min &redef;
+}
+
 module GLOBAL;
 
 ## Seed for hashes computed internally for probabilistic data structures. Using

scripts/base/protocols/dhcp/main.bro

@@ -84,7 +84,7 @@ export {
 	## The maximum amount of time that a transation ID will be watched
 	## for to try and tie messages together into a single DHCP
 	## transaction narrative.
-	const DHCP::max_txid_watch_time = 30secs &redef;
+	option DHCP::max_txid_watch_time = 30secs;
 
 	## This event is used internally to distribute data around clusters
 	## since DHCP doesn't follow the normal "connection" model used by

scripts/base/protocols/dns/main.bro

@@ -113,11 +113,6 @@ export {
 	## DNS message query/transaction ID.
 	type PendingMessages: table[count] of Queue::Queue;
 
-	## The amount of time that DNS queries or replies for a given
-	## query/transaction ID are allowed to be queued while waiting for
-	## a matching reply or query.
-	const pending_msg_expiry_interval = 2min &redef;
-
 	## Give up trying to match pending DNS queries or replies for a given
 	## query/transaction ID once this number of unmatched queries or replies
 	## is reached (this shouldn't happen unless either the DNS server/resolver

scripts/base/protocols/ftp/main.bro

@@ -17,13 +17,13 @@ export {
 	redef enum Log::ID += { LOG };
 
 	## List of commands that should have their command/response pairs logged.
-	const logged_commands = {
+	option logged_commands = {
 		"APPE", "DELE", "RETR", "STOR", "STOU", "ACCT", "PORT", "PASV", "EPRT",
 		"EPSV"
-	} &redef;
+	};
 
 	## User IDs that can be considered "anonymous".
-	const guest_ids = { "anonymous", "ftp", "ftpuser", "guest" } &redef;
+	option guest_ids = { "anonymous", "ftp", "ftpuser", "guest" };
 
 	## This record is to hold a parsed FTP reply code.  For example, for the
 	## 201 status code, the digits would be parsed as: x->2, y->0, z->1.

scripts/base/protocols/http/main.bro

@@ -109,7 +109,7 @@ export {
 	## A list of HTTP methods. Other methods will generate a weird. Note
 	## that the HTTP analyzer will only accept methods consisting solely
 	## of letters ``[A-Za-z]``.
-	const http_methods: set[string] = {
+	option http_methods: set[string] = {
 		"GET", "POST", "HEAD", "OPTIONS",
 		"PUT", "DELETE", "TRACE", "CONNECT",
 		# HTTP methods for distributed authoring:
@@ -117,7 +117,7 @@ export {
 		"COPY", "MOVE", "LOCK", "UNLOCK",
 		"POLL", "REPORT", "SUBSCRIBE", "BMOVE",
 		"SEARCH"
-	} &redef;
+	};
 
 	## Event that can be handled to access the HTTP record as it is sent on
 	## to the logging framework.
@@ -199,7 +199,7 @@ event http_request(c: connection, method: string, original_URI: string,
 	c$http$uri = unescaped_URI;
 
 	if ( method !in http_methods )
-		event conn_weird("unknown_HTTP_method", c, method);
+		Reporter::conn_weird("unknown_HTTP_method", c, method);
 	}
 
 event http_reply(c: connection, version: string, code: count, reason: string) &priority=5

scripts/base/protocols/krb/main.bro

@@ -48,7 +48,7 @@ export {
 	};
 
 	## The server response error texts which are *not* logged.
-	const ignored_errors: set[string] = {
+	option ignored_errors: set[string] = {
 		# This will significantly increase the noisiness of the log.
 		# However, one attack is to iterate over principals, looking
 		# for ones that don't require preauth, and then performn
@@ -58,7 +58,7 @@ export {
 		# This is a more specific version of NEEDED_PREAUTH that's used
 		# by Windows AD Kerberos.
 		"Need to use PA-ENC-TIMESTAMP/PA-PK-AS-REQ",
-	} &redef;
+	};
 
 	## Event that can be handled to access the KRB record as it is sent on
 	## to the logging framework.

scripts/base/protocols/sip/main.bro

@@ -80,9 +80,9 @@ export {
 	## A list of SIP methods. Other methods will generate a weird. Note
 	## that the SIP analyzer will only accept methods consisting solely
 	## of letters ``[A-Za-z]``.
-	const sip_methods: set[string] = {
+	option sip_methods: set[string] = {
 		"REGISTER", "INVITE", "ACK", "CANCEL", "BYE", "OPTIONS", "NOTIFY", "SUBSCRIBE"
-	} &redef;
+	};
 
 	## Event that can be handled to access the SIP record as it is sent on
 	## to the logging framework.
@@ -168,7 +168,7 @@ event sip_request(c: connection, method: string, original_URI: string, version:
 	c$sip$uri = original_URI;
 
 	if ( method !in sip_methods )
-		event conn_weird("unknown_SIP_method", c, method);
+		Reporter::conn_weird("unknown_SIP_method", c, method);
 	}
 
 event sip_reply(c: connection, version: string, code: count, reason: string) &priority=5

scripts/base/protocols/smb/main.bro

@@ -33,14 +33,14 @@ export {
 	};
 
 	## The file actions which are logged.
-	const logged_file_actions: set[Action] = {
+	option logged_file_actions: set[Action] = {
 		FILE_OPEN,
 		FILE_RENAME,
 		FILE_DELETE,
 
 		PRINT_OPEN,
 		PRINT_CLOSE,
-	} &redef;
+	};
 
 	## This record is for the smb_files.log
 	type FileInfo: record {

scripts/base/protocols/smb/smb1-main.bro

@@ -309,7 +309,7 @@ event smb_pipe_request(c: connection, hdr: SMB1::Header, op_num: count)
 	if ( ! f?$uuid )
 		{
 		# TODO: figure out why this is happening.
-		event conn_weird("smb_pipe_request_missing_uuid", c, "");
+		Reporter::conn_weird("smb_pipe_request_missing_uuid", c, "");
 		return;
 		}
 	local arg = fmt("%s: %s",

scripts/base/protocols/ssl/main.bro

@@ -263,7 +263,7 @@ event ssl_extension_server_name(c: connection, is_orig: bool, names: string_vec)
 		{
 		c$ssl$server_name = names[0];
 		if ( |names| > 1 )
-			event conn_weird("SSL_many_server_names", c, cat(names));
+			Reporter::conn_weird("SSL_many_server_names", c, cat(names));
 		}
 	}
 

scripts/base/utils/exec.bro

@@ -42,9 +42,6 @@ export {
 	## Returns: A record representing the full results from the
 	##          external program execution.
 	global run: function(cmd: Command): Result;
-
-	## The system directory for temporary files.
-	const tmp_dir = "/tmp" &redef;
 }
 
 # Indexed by command uid.

scripts/base/utils/geoip-distance.bro

@@ -1,7 +1,7 @@
 ##! Functions to calculate distance between two locations, based on GeoIP data.
 
 ## Returns the distance between two IP addresses using the haversine formula,
-## based on GeoIP database locations.  Requires Bro to be built with libgeoip.
+## based on GeoIP database locations.  Requires Bro to be built with GeoIP.
 ##
 ## a1: First IP address.
 ##

scripts/broxygen/__load__.bro

@@ -6,6 +6,7 @@
 @load frameworks/control/controller.bro
 @load frameworks/files/extract-all-files.bro
 @load policy/misc/dump-events.bro
+@load policy/protocols/dhcp/deprecated_events.bro
 
 @load ./example.bro
 

scripts/policy/frameworks/dpd/packet-segment-logging.bro

@@ -16,7 +16,7 @@ export {
 	};
 
 	## Size of the packet segment to display in the DPD log.
-	const packet_segment_size: int = 255 &redef;
+	option packet_segment_size: int = 255;
 }
 
 

scripts/policy/frameworks/files/detect-MHR.bro

@@ -32,7 +32,7 @@ export {
 	## A/V engines.  Team Cymru returns a percentage to indicate how
 	## many A/V engines flagged the sample as malicious. This threshold
 	## allows you to require a minimum detection rate.
-	const notice_threshold = 10 &redef;
+	option notice_threshold = 10;
 }
 
 function do_mhr_lookup(hash: string, fi: Notice::FileInfo)

scripts/policy/frameworks/software/version-changes.bro

@@ -19,7 +19,7 @@ export {
 	## Some software is more interesting when the version changes and this
 	## is a set of all software that should raise a notice when a different
 	## version is seen on a host.
-	const interesting_version_changes: set[string] = { } &redef;
+	option interesting_version_changes: set[string] = {};
 }
 
 event Software::version_change(old: Software::Info, new: Software::Info)

scripts/policy/misc/capture-loss.bro

@@ -44,7 +44,7 @@ export {
 	## when the :bro:enum:`CaptureLoss::Too_Much_Loss` notice should be
 	## generated. The value is expressed as a double between 0 and 1 with 1
 	## being 100%.
-	const too_much_loss: double = 0.1 &redef;
+	option too_much_loss: double = 0.1;
 }
 
 event CaptureLoss::take_measurement(last_ts: time, last_acks: count, last_gaps: count)

scripts/policy/misc/dump-events.bro

@@ -7,7 +7,7 @@ module DumpEvents;
 
 export {
 	## If true, include event arguments in output.
-	const include_args = T &redef;
+	option include_args = T;
 
 	## Only include events matching the given pattern into output. By default, the
 	## pattern matches all events.

scripts/policy/misc/stats.bro

@@ -8,7 +8,7 @@ export {
 	redef enum Log::ID += { LOG };
 
 	## How often stats are reported.
-	const report_interval = 5min &redef;
+	option report_interval = 5min;
 
 	type Info: record {
 		## Timestamp for the measurement.

scripts/policy/misc/weird-stats.bro

@@ -0,0 +1,100 @@
+##! Log weird statistics.
+
+@load base/frameworks/sumstats
+@load base/frameworks/cluster
+
+module WeirdStats;
+
+export {
+	redef enum Log::ID += { LOG };
+
+	## How often stats are reported.
+	const weird_stat_interval = 15min &redef;
+
+	type Info: record {
+		## Timestamp for the measurement.
+		ts: time &log;
+		## Name of the weird.
+		name: string &log;
+		## Number of times weird was seen since the last stats interval.
+		num_seen: count &log;
+	};
+
+	global log_weird_stats: event(rec: Info);
+}
+
+global this_epoch_weirds: table[string] of double;
+global last_epoch_weirds: table[string] of double;
+
+function weird_epoch_results(ts: time, key: SumStats::Key, result: SumStats::Result)
+	{
+	this_epoch_weirds[key$str]=result["weirds.encountered"]$sum;
+	}
+
+function weird_epoch_finished(ts: time)
+	{
+	for ( n in this_epoch_weirds )
+		{
+		local last_count: double = 0.0;
+
+		if ( n in last_epoch_weirds )
+			last_count = last_epoch_weirds[n];
+
+		local num_seen: double = this_epoch_weirds[n] - last_count;
+
+		if ( num_seen > 0.0 )
+			Log::write(LOG, Info($ts = ts, $name = n,
+			                     $num_seen = double_to_count(num_seen)));
+		}
+
+	last_epoch_weirds = this_epoch_weirds;
+	this_epoch_weirds = table();
+	}
+
+event bro_init() &priority=5
+	{
+	Log::create_stream(WeirdStats::LOG,
+	                   [$columns = Info, $ev = log_weird_stats,
+	                    $path="weird_stats"]);
+	local r1 = SumStats::Reducer($stream = "weirds.encountered",
+	                             $apply = set(SumStats::SUM));
+	SumStats::create([$name = "weirds.statistics",
+	                  $epoch = weird_stat_interval, $reducers = set(r1),
+	                  $epoch_result = weird_epoch_results,
+	                  $epoch_finished = weird_epoch_finished]);
+	}
+
+module SumStats;
+
+function observe_weird_stats()
+	{
+	local rs = get_reporter_stats();
+
+	for ( n in rs$weirds_by_type )
+		SumStats::observe("weirds.encountered", SumStats::Key($str = n),
+		                  SumStats::Observation($dbl=rs$weirds_by_type[n]+0.0));
+	}
+
+@if ( Cluster::is_enabled() )
+
+# I'm not sure if this is a hack or not: the manager will generate this
+# event at the end of its epoch so workers can handle it just in time to
+# generate the necessary stats.  Alternative may be workers generating the
+# stats individually/proactively in their own finish_epoch, but that may be
+# less synchronized?
+event SumStats::cluster_ss_request(uid: string, ss_name: string, cleanup: bool) &priority=10
+	{
+	observe_weird_stats();
+	}
+
+@else
+
+event SumStats::finish_epoch(ss: SumStat) &priority=10
+	{
+	if ( ss$name != "weirds.statistics" )
+		return;
+
+	observe_weird_stats();
+	}
+
+@endif

scripts/policy/protocols/dhcp/deprecated_events.bro

@@ -0,0 +1,272 @@
+##! Bro 2.6 removed certain DHCP events, but scripts in the Bro
+##! ecosystem are still relying on those events. As a transition, this
+##! script will handle the new event, and generate the old events,
+##! which are marked as deprecated.  Note: This script should be
+##! removed in the next Bro version after 2.6.
+
+@load base/protocols/dhcp
+
+## A DHCP message.
+##
+## .. note:: This type is included to support the deprecated events dhcp_ack,
+##           dhcp_decline, dhcp_discover, dhcp_inform, dhcp_nak, dhcp_offer,
+##           dhcp_release and dhcp_request and is thus similarly deprecated
+##           itself. Use :bro:see:`dhcp_message` instead.
+##
+## .. bro:see:: dhcp_message dhcp_ack dhcp_decline dhcp_discover
+##              dhcp_inform dhcp_nak dhcp_offer dhcp_release dhcp_request
+type dhcp_msg: record {
+	op: count;      ##< Message OP code. 1 = BOOTREQUEST, 2 = BOOTREPLY
+	m_type: count;  ##< The type of DHCP message.
+	xid: count;     ##< Transaction ID of a DHCP session.
+	h_addr: string; ##< Hardware address of the client.
+	ciaddr: addr;   ##< Original IP address of the client.
+	yiaddr: addr;   ##< IP address assigned to the client.
+};
+
+## A list of router addresses offered by a DHCP server.
+##
+## .. note:: This type is included to support the deprecated events dhcp_ack
+##           and dhcp_offer and is thus similarly deprecated
+##           itself. Use :bro:see:`dhcp_message` instead.
+##
+## .. bro:see:: dhcp_message dhcp_ack dhcp_offer
+type dhcp_router_list: table[count] of addr;
+
+## Generated for DHCP messages of type *DHCPDISCOVER* (client broadcast to locate
+## available servers).
+##
+## c: The connection record describing the underlying UDP flow.
+##
+## msg: The parsed type-independent part of the DHCP message.
+##
+## req_addr: The specific address requested by the client.
+##
+## host_name: The value of the host name option, if specified by the client.
+##
+## .. bro:see:: dhcp_message dhcp_discover dhcp_offer dhcp_request
+##              dhcp_decline dhcp_ack dhcp_nak dhcp_release dhcp_inform
+##
+## .. note:: This event has been deprecated, and will be removed in the next version.
+##    Use dhcp_message instead.
+##
+## .. note:: Bro does not support broadcast packets (as used by the DHCP
+##    protocol). It treats broadcast addresses just like any other and
+##    associates packets into transport-level flows in the same way as usual.
+##
+global dhcp_discover: event(c: connection, msg: dhcp_msg, req_addr: addr, host_name: string) &deprecated;
+
+## Generated for DHCP messages of type *DHCPOFFER* (server to client in response
+## to DHCPDISCOVER with offer of configuration parameters).
+##
+## c: The connection record describing the underlying UDP flow.
+##
+## msg: The parsed type-independent part of the DHCP message.
+##
+## mask: The subnet mask specified by the message.
+##
+## router: The list of routers specified by the message.
+##
+## lease: The least interval specified by the message.
+##
+## serv_addr: The server address specified by the message.
+##
+## host_name: Optional host name value. May differ from the host name requested
+##            from the client.
+##
+## .. bro:see:: dhcp_message dhcp_discover dhcp_request dhcp_decline
+##              dhcp_ack dhcp_nak dhcp_release dhcp_inform
+##
+## .. note:: This event has been deprecated, and will be removed in the next version.
+##    Use dhcp_message instead.
+##
+## .. note:: Bro does not support broadcast packets (as used by the DHCP
+##    protocol). It treats broadcast addresses just like any other and
+##    associates packets into transport-level flows in the same way as usual.
+##
+global dhcp_offer: event(c: connection, msg: dhcp_msg, mask: addr, router: dhcp_router_list, lease: interval, serv_addr: addr, host_name: string) &deprecated;
+
+## Generated for DHCP messages of type *DHCPREQUEST* (Client message to servers either
+## (a) requesting offered parameters from one server and implicitly declining offers
+## from all others, (b) confirming correctness of previously allocated address after,
+## e.g., system reboot, or (c) extending the lease on a particular network address.)
+##
+## c: The connection record describing the underlying UDP flow.
+##
+## msg: The parsed type-independent part of the DHCP message.
+##
+## req_addr: The client address specified by the message.
+##
+## serv_addr: The server address specified by the message.
+##
+## host_name: The value of the host name option, if specified by the client.
+##
+## .. bro:see:: dhcp_message dhcp_discover dhcp_offer dhcp_decline
+##    	       dhcp_ack dhcp_nak dhcp_release dhcp_inform
+##
+## .. note:: This event has been deprecated, and will be removed in the next version.
+##    Use dhcp_message instead.
+##
+## .. note:: Bro does not support broadcast packets (as used by the DHCP
+##    protocol). It treats broadcast addresses just like any other and
+##    associates packets into transport-level flows in the same way as usual.
+##
+global dhcp_request: event(c: connection, msg: dhcp_msg, req_addr: addr, serv_addr: addr, host_name: string) &deprecated;
+
+## Generated for DHCP messages of type *DHCPDECLINE* (Client to server indicating
+## network address is already in use).
+##
+## c: The connection record describing the underlying UDP flow.
+##
+## msg: The parsed type-independent part of the DHCP message.
+##
+## host_name: Optional host name value.
+##
+## .. bro:see:: dhcp_message dhcp_discover dhcp_offer dhcp_request
+##              dhcp_ack dhcp_nak dhcp_release dhcp_inform
+##
+## .. note:: This event has been deprecated, and will be removed in the next version.
+##    Use dhcp_message instead.
+##
+## .. note:: Bro does not support broadcast packets (as used by the DHCP
+##    protocol). It treats broadcast addresses just like any other and
+##    associates packets into transport-level flows in the same way as usual.
+##
+global dhcp_decline: event(c: connection, msg: dhcp_msg, host_name: string) &deprecated;
+
+## Generated for DHCP messages of type *DHCPACK* (Server to client with configuration
+## parameters, including committed network address).
+##
+## c: The connection record describing the underlying UDP flow.
+##
+## msg: The parsed type-independent part of the DHCP message.
+##
+## mask: The subnet mask specified by the message.
+##
+## router: The list of routers specified by the message.
+##
+## lease: The least interval specified by the message.
+##
+## serv_addr: The server address specified by the message.
+##
+## host_name: Optional host name value. May differ from the host name requested
+##            from the client.
+##
+## .. bro:see:: dhcp_message dhcp_discover dhcp_offer dhcp_request
+##              dhcp_decline dhcp_nak dhcp_release dhcp_inform
+##
+## .. note:: This event has been deprecated, and will be removed in the next version.
+##    Use dhcp_message instead.
+##
+global dhcp_ack: event(c: connection, msg: dhcp_msg, mask: addr, router: dhcp_router_list, lease: interval, serv_addr: addr, host_name: string) &deprecated;
+
+## Generated for DHCP messages of type *DHCPNAK* (Server to client indicating client's
+## notion of network address is incorrect (e.g., client has moved to new subnet) or
+## client's lease has expired).
+##
+## c: The connection record describing the underlying UDP flow.
+##
+## msg: The parsed type-independent part of the DHCP message.
+##
+## host_name: Optional host name value.
+##
+## .. bro:see:: dhcp_message dhcp_discover dhcp_offer dhcp_request
+##              dhcp_decline dhcp_ack dhcp_release dhcp_inform
+##
+## .. note:: This event has been deprecated, and will be removed in the next version.
+##    Use dhcp_message instead.
+##
+## .. note:: Bro does not support broadcast packets (as used by the DHCP
+##    protocol). It treats broadcast addresses just like any other and
+##    associates packets into transport-level flows in the same way as usual.
+##
+global dhcp_nak: event(c: connection, msg: dhcp_msg, host_name: string) &deprecated;
+
+## Generated for DHCP messages of type *DHCPRELEASE* (Client to server relinquishing
+## network address and cancelling remaining lease).
+##
+## c: The connection record describing the underlying UDP flow.
+##
+## msg: The parsed type-independent part of the DHCP message.
+##
+## host_name: The value of the host name option, if specified by the client.
+##
+## .. bro:see:: dhcp_message dhcp_discover dhcp_offer dhcp_request
+##              dhcp_decline dhcp_ack dhcp_nak dhcp_inform
+##
+## .. note:: This event has been deprecated, and will be removed in the next version.
+##    Use dhcp_message instead.
+##
+global dhcp_release: event(c: connection, msg: dhcp_msg, host_name: string) &deprecated;
+
+## Generated for DHCP messages of type *DHCPINFORM* (Client to server, asking only for
+## local configuration parameters; client already has externally configured network
+## address).
+##
+## c: The connection record describing the underlying UDP flow.
+##
+## msg: The parsed type-independent part of the DHCP message.
+##
+## host_name: The value of the host name option, if specified by the client.
+##
+## .. bro:see:: dhcp_message dhcp_discover dhcp_offer dhcp_request
+##              dhcp_decline dhcp_ack dhcp_nak dhcp_release
+##
+## .. note:: This event has been deprecated, and will be removed in the next version.
+##    Use dhcp_message instead.
+##
+## .. note:: Bro does not support broadcast packets (as used by the DHCP
+##    protocol). It treats broadcast addresses just like any other and
+##    associates packets into transport-level flows in the same way as usual.
+##
+global dhcp_inform: event(c: connection, msg: dhcp_msg, host_name: string) &deprecated;
+
+event dhcp_message(c: connection, is_orig: bool, msg: DHCP::Msg, options: DHCP::Options)
+	{
+	local old_msg: dhcp_msg = [$op=msg$op, $m_type=msg$m_type, $xid=msg$xid,
+	                           $h_addr=msg$chaddr, $ciaddr=msg$ciaddr, $yiaddr=msg$yiaddr];
+
+	local routers = dhcp_router_list();
+
+	if ( options?$routers )
+		for ( i in options$routers )
+			routers[|routers|] = options$routers[i];
+
+	# These fields are technically optional, but aren't listed as such in the event.
+	# We give it some defaults in order to suppress errors.
+	local ar = ( options?$addr_request ) ? options$addr_request : 0.0.0.0;
+	local hn = ( options?$host_name ) ? options$host_name : "";
+	local le = ( options?$lease ) ? options$lease : 0 secs;
+	local sm = ( options?$subnet_mask ) ? options$subnet_mask : 255.255.255.255;
+	local sa = ( options?$serv_addr ) ? options$serv_addr : 0.0.0.0;
+
+	switch ( DHCP::message_types[msg$m_type] ) {
+	case "DISCOVER":
+		event dhcp_discover(c, old_msg, ar, hn);
+		break;
+	case "OFFER":
+		event dhcp_offer(c, old_msg, sm, routers, le, sa, hn);
+		break;
+	case "REQUEST":
+		event dhcp_request(c, old_msg, ar, sa, hn);
+		break;
+	case "DECLINE":
+		event dhcp_decline(c, old_msg, hn);
+		break;
+	case "ACK":
+		event dhcp_ack(c, old_msg, sm, routers, le, sa, hn);
+		break;
+	case "NAK":
+		event dhcp_nak(c, old_msg, hn);
+		break;
+	case "RELEASE":
+		event dhcp_release(c, old_msg, hn);
+		break;
+	case "INFORM":
+		event dhcp_inform(c, old_msg, hn);
+		break;
+	default:
+		# This isn't a weird, it's just a DHCP message type the old scripts don't handle
+		break;
+		}
+	}

scripts/policy/protocols/http/header-names.bro

@@ -18,10 +18,10 @@ export {
 	};
 	
 	## A boolean value to determine if client header names are to be logged.
-	const log_client_header_names = T &redef;
+	option log_client_header_names = T;
 	
 	## A boolean value to determine if server header names are to be logged.
-	const log_server_header_names = F &redef;
+	option log_server_header_names = F;
 }
 
 event http_header(c: connection, is_orig: bool, name: string, value: string) &priority=3

scripts/policy/protocols/smb/log-cmds.bro

@@ -11,9 +11,9 @@ export {
 	};
 
 	## The server response statuses which are *not* logged.
-	const ignored_command_statuses: set[string] = {
+	option ignored_command_statuses: set[string] = {
 		"MORE_PROCESSING_REQUIRED",
-	} &redef;
+	};
 }
 
 ## Internal use only.

scripts/policy/protocols/smtp/detect-suspicious-orig.bro

@@ -10,9 +10,9 @@ export {
 
 	## Places where it's suspicious for mail to originate from represented
 	## as all-capital, two character country codes (e.g., US).  It requires
-	## libGeoIP support built in.
+	## Bro to be built with GeoIP support.
-	const suspicious_origination_countries: set[string] = {} &redef;
+	option suspicious_origination_countries: set[string] = {};
-	const suspicious_origination_networks: set[subnet] = {} &redef;
+	option suspicious_origination_networks: set[subnet] = {};
 
 }
 

scripts/policy/protocols/smtp/entities-excerpt.bro

@@ -14,7 +14,7 @@ export {
 	## This is the default value for how much of the entity body should be
 	## included for all MIME entities.  The lesser of this value and
 	## :bro:see:`default_file_bof_buffer_size` will be used.
-	const default_entity_excerpt_len = 0 &redef;
+	option default_entity_excerpt_len = 0;
 }
 
 event file_new(f: fa_file) &priority=5

scripts/policy/protocols/ssh/geo-data.bro

@@ -21,7 +21,7 @@ export {
 
 	## The set of countries for which you'd like to generate notices upon
 	## successful login.
-	const watched_countries: set[string] = {"RO"} &redef;
+	option watched_countries: set[string] = {"RO"};
 }
 
 function get_location(c: connection): geo_location

scripts/policy/protocols/ssl/expiring-certs.bro

@@ -27,11 +27,11 @@ export {
 	## notices will be suppressed by the notice framework for 1 day after 
 	## a particular certificate has had a notice generated.
 	## Choices are: LOCAL_HOSTS, REMOTE_HOSTS, ALL_HOSTS, NO_HOSTS
-	const notify_certs_expiration = LOCAL_HOSTS &redef;
+	option notify_certs_expiration = LOCAL_HOSTS;
 	
 	## The time before a certificate is going to expire that you would like
 	## to start receiving :bro:enum:`SSL::Certificate_Expires_Soon` notices.
-	const notify_when_cert_expiring_in = 30days &redef;
+	option notify_when_cert_expiring_in = 30days;
 }
 
 event ssl_established(c: connection) &priority=3

scripts/policy/protocols/ssl/weak-keys.bro

@@ -21,24 +21,24 @@ export {
 	## keys/ciphers/protocol_versions.  By default, these notices will be suppressed
 	## by the notice framework for 1 day after a particular host has had a notice
 	## generated. Choices are: LOCAL_HOSTS, REMOTE_HOSTS, ALL_HOSTS, NO_HOSTS
-	const notify_weak_keys = LOCAL_HOSTS &redef;
+	option notify_weak_keys = LOCAL_HOSTS;
 
 	## The minimal key length in bits that is considered to be safe. Any shorter
 	## (non-EC) key lengths will trigger a notice.
-	const notify_minimal_key_length = 2048 &redef;
+	option notify_minimal_key_length = 2048;
 
 	## Warn if the DH key length is smaller than the certificate key length. This is
 	## potentially unsafe because it gives a wrong impression of safety due to the
 	## certificate key length. However, it is very common and cannot be avoided in some
 	## settings (e.g. with old jave clients).
-	const notify_dh_length_shorter_cert_length = T &redef;
+	option notify_dh_length_shorter_cert_length = T;
 
 	## Warn if a server negotiates a SSL session with a protocol version smaller than
 	## the specified version. By default, the minimal version is TLSv10 because SSLv2
 	## and v3 have serious security issued.
 	## See https://tools.ietf.org/html/draft-thomson-sslv3-diediedie-00
 	## To disable, set to SSLv20
-	const tls_minimum_version = TLSv10 &redef;
+	option tls_minimum_version = TLSv10;
 
 	## Warn if a server negotiates an unsafe cipher suite. By default, we only warn when
 	## encountering old export cipher suites, or RC4 (see RFC7465).

scripts/site/local-logger.bro

@@ -1 +0,0 @@
-##! Local site policy loaded only by the logger if Bro is running as a cluster.

scripts/site/local-manager.bro

@@ -1 +0,0 @@
-##! Local site policy loaded only by the manager if Bro is running as a cluster.

scripts/site/local-proxy.bro

@@ -1 +0,0 @@
-##! Local site policy loaded only by the proxies if Bro is running as a cluster.

scripts/site/local-worker.bro

@@ -1 +0,0 @@
-##! Local site policy loaded only by the workers if Bro is running as a cluster.

scripts/site/local.bro

@@ -66,7 +66,7 @@
 # certificate notary service; see http://notary.icsi.berkeley.edu .
 # @load protocols/ssl/notary
 
-# If you have libGeoIP support built in, do some geographic detections and
+# If you have GeoIP support built in, do some geographic detections and
 # logging for SSH traffic.
 @load protocols/ssh/geo-data
 # Detect hosts doing SSH bruteforce attacks.

scripts/test-all-policy.bro

@@ -54,12 +54,14 @@
 @load misc/profiling.bro
 @load misc/scan.bro
 @load misc/stats.bro
+@load misc/weird-stats.bro
 @load misc/trim-trace-file.bro
 @load protocols/conn/known-hosts.bro
 @load protocols/conn/known-services.bro
 @load protocols/conn/mac-logging.bro
 @load protocols/conn/vlan-logging.bro
 @load protocols/conn/weirds.bro
+#@load protocols/dhcp/deprecated_events.bro
 @load protocols/dhcp/msg-orig.bro
 @load protocols/dhcp/software.bro
 @load protocols/dhcp/sub-opts.bro

src/Conn.cc

@@ -1071,3 +1071,28 @@ void Connection::CheckFlowLabel(bool is_orig, uint32 flow_label)
 	else
 		saw_first_resp_packet = 1;
 	}
+
+bool Connection::PermitWeird(const char* name, uint64 threshold, uint64 rate,
+                             double duration)
+	{
+	auto& state = weird_state[name];
+	++state.count;
+
+	if ( state.count < threshold )
+		return true;
+
+	if ( state.count == threshold )
+		state.sampling_start_time = network_time;
+	else
+		{
+		if ( network_time > state.sampling_start_time + duration )
+			{
+			state.sampling_start_time = 0;
+			state.count = 1;
+			return true;
+			}
+		}
+
+	auto num_above_threshold = state.count - threshold;
+	return num_above_threshold % rate == 0;
+	}

src/Conn.h

@@ -5,6 +5,9 @@
 
 #include <sys/types.h>
 
+#include <unordered_map>
+#include <string>
+
 #include "Dict.h"
 #include "Val.h"
 #include "Timer.h"
@@ -275,6 +278,9 @@ public:
 	uint32 GetOrigFlowLabel() { return orig_flow_label; }
 	uint32 GetRespFlowLabel() { return resp_flow_label; }
 
+	bool PermitWeird(const char* name, uint64 threshold, uint64 rate,
+	                 double duration);
+
 protected:
 
 	Connection()	{ persistent = 0; }
@@ -339,6 +345,14 @@ protected:
 	analyzer::pia::PIA* primary_PIA;
 
 	Bro::UID uid;	// Globally unique connection ID.
+
+	struct WeirdState {
+		WeirdState() { count = 0; sampling_start_time = 0; }
+		uint64 count = 0;
+		double sampling_start_time = 0;
+	};
+
+	std::unordered_map<std::string, WeirdState> weird_state;
 };
 
 class ConnectionTimer : public Timer {

src/Func.cc

@@ -722,6 +722,7 @@ void init_builtin_funcs()
 	FileAnalysisStats = internal_type("FileAnalysisStats")->AsRecordType();
 	ThreadStats = internal_type("ThreadStats")->AsRecordType();
 	BrokerStats = internal_type("BrokerStats")->AsRecordType();
+	ReporterStats = internal_type("ReporterStats")->AsRecordType();
 
 	var_sizes = internal_type("var_sizes")->AsTableType();
 

src/ID.cc

@@ -140,10 +140,11 @@ void ID::SetVal(Val* v, init_class c)
 		}
 
 	if ( type->Tag() != TYPE_TABLE &&
-	     (type->Tag() != TYPE_PATTERN || c == INIT_REMOVE) )
+	     (type->Tag() != TYPE_PATTERN || c == INIT_REMOVE) &&
+	     (type->Tag() != TYPE_VECTOR  || c == INIT_REMOVE) )
 		{
 		if ( c == INIT_EXTRA )
-			Error("+= initializer only applies to tables, sets and patterns", v);
+			Error("+= initializer only applies to tables, sets, vectors and patterns", v);
 		else
 			Error("-= initializer only applies to tables and sets", v);
 		}

src/Reporter.cc

@@ -10,6 +10,7 @@
 #include "NetVar.h"
 #include "Net.h"
 #include "Conn.h"
+#include "Timer.h"
 #include "plugin/Plugin.h"
 #include "plugin/Manager.h"
 
@@ -36,6 +37,11 @@ Reporter::Reporter()
 	warnings_to_stderr = true;
 	errors_to_stderr = true;
 
+	weird_count = 0;
+	weird_sampling_rate = 0;
+	weird_sampling_duration = 0;
+	weird_sampling_threshold = 0;
+
 	openlog("bro", 0, LOG_LOCAL5);
 	}
 
@@ -49,6 +55,24 @@ void Reporter::InitOptions()
 	info_to_stderr = internal_const_val("Reporter::info_to_stderr")->AsBool();
 	warnings_to_stderr = internal_const_val("Reporter::warnings_to_stderr")->AsBool();
 	errors_to_stderr = internal_const_val("Reporter::errors_to_stderr")->AsBool();
+	weird_sampling_rate = internal_const_val("Weird::sampling_rate")->AsCount();
+	weird_sampling_threshold = internal_const_val("Weird::sampling_threshold")->AsCount();
+	weird_sampling_duration = internal_const_val("Weird::sampling_duration")->AsInterval();
+	auto wl_val = internal_const_val("Weird::sampling_whitelist")->AsTableVal();
+	auto wl_table = wl_val->AsTable();
+
+	HashKey* k;
+	IterCookie* c = wl_table->InitForIteration();
+	TableEntryVal* v;
+
+	while ( (v = wl_table->NextEntry(k, c)) )
+		{
+		auto index = wl_val->RecoverIndex(k);
+		string key = index->Index(0)->AsString()->CheckString();
+		weird_sampling_whitelist.emplace(move(key));
+		Unref(index);
+		delete k;
+		}
 	}
 
 void Reporter::Info(const char* fmt, ...)
@@ -221,23 +245,121 @@ void Reporter::WeirdFlowHelper(const IPAddr& orig, const IPAddr& resp, const cha
 	delete vl;
 	}
 
+void Reporter::UpdateWeirdStats(const char* name)
+	{
+	++weird_count;
+	++weird_count_by_type[name];
+	}
+
+class NetWeirdTimer : public Timer {
+public:
+	NetWeirdTimer(double t, const char* name, double timeout)
+	: Timer(t + timeout, TIMER_NET_WEIRD_EXPIRE), weird_name(name)
+		{}
+
+	void Dispatch(double t, int is_expire) override
+		{ reporter->ResetNetWeird(weird_name); }
+
+	std::string weird_name;
+};
+
+class FlowWeirdTimer : public Timer {
+public:
+	using IPPair = std::pair<IPAddr, IPAddr>;
+
+	FlowWeirdTimer(double t, IPPair p, double timeout)
+	: Timer(t + timeout, TIMER_FLOW_WEIRD_EXPIRE), endpoints(p)
+		{}
+
+	void Dispatch(double t, int is_expire) override
+		{ reporter->ResetFlowWeird(endpoints.first, endpoints.second); }
+
+	IPPair endpoints;
+};
+
+void Reporter::ResetNetWeird(const std::string& name)
+	{
+	net_weird_state.erase(name);
+	}
+
+void Reporter::ResetFlowWeird(const IPAddr& orig, const IPAddr& resp)
+	{
+	flow_weird_state.erase(std::make_pair(orig, resp));
+	}
+
+bool Reporter::PermitNetWeird(const char* name)
+	{
+	auto& count = net_weird_state[name];
+	++count;
+
+	if ( count == 1 )
+		timer_mgr->Add(new NetWeirdTimer(network_time, name,
+		                                 weird_sampling_duration));
+
+	if ( count < weird_sampling_threshold )
+		return true;
+
+	auto num_above_threshold = count - weird_sampling_threshold;
+	return num_above_threshold % weird_sampling_rate == 0;
+	}
+
+bool Reporter::PermitFlowWeird(const char* name,
+                               const IPAddr& orig, const IPAddr& resp)
+	{
+	auto endpoints = std::make_pair(orig, resp);
+	auto& map = flow_weird_state[endpoints];
+
+	if ( map.empty() )
+		timer_mgr->Add(new FlowWeirdTimer(network_time, endpoints,
+		                                  weird_sampling_duration));
+
+	auto& count = map[name];
+	++count;
+
+	if ( count < weird_sampling_threshold )
+		return true;
+
+	auto num_above_threshold = count - weird_sampling_threshold;
+	return num_above_threshold % weird_sampling_rate == 0;
+	}
+
 void Reporter::Weird(const char* name)
 	{
+	UpdateWeirdStats(name);
+
+	if ( ! WeirdOnSamplingWhiteList(name) )
+		{
+		if ( ! PermitNetWeird(name) )
+			return;
+		}
+
 	WeirdHelper(net_weird, 0, 0, "%s", name);
 	}
 
 void Reporter::Weird(Connection* conn, const char* name, const char* addl)
 	{
-	WeirdHelper(conn_weird, conn->BuildConnVal(), addl, "%s", name);
+	UpdateWeirdStats(name);
+
+	if ( ! WeirdOnSamplingWhiteList(name) )
+		{
+		if ( ! conn->PermitWeird(name, weird_sampling_threshold,
+		                         weird_sampling_rate, weird_sampling_duration) )
+			return;
 		}
 
-void Reporter::Weird(Val* conn_val, const char* name, const char* addl)
+	WeirdHelper(conn_weird, conn->BuildConnVal(), addl, "%s", name);
-	{
-	WeirdHelper(conn_weird, conn_val, addl, "%s", name);
 	}
 
 void Reporter::Weird(const IPAddr& orig, const IPAddr& resp, const char* name)
 	{
+	UpdateWeirdStats(name);
+
+	if ( ! WeirdOnSamplingWhiteList(name) )
+		{
+		if ( ! PermitFlowWeird(name, orig, resp) )
+			 return;
+		}
+
 	WeirdFlowHelper(orig, resp, "%s", name);
 	}
 

src/Reporter.h

@@ -7,6 +7,10 @@
 
 #include <list>
 #include <utility>
+#include <string>
+#include <map>
+#include <unordered_set>
+#include <unordered_map>
 
 #include "util.h"
 #include "EventHandler.h"
@@ -36,6 +40,11 @@ protected:
 
 class Reporter {
 public:
+	using IPPair = std::pair<IPAddr, IPAddr>;
+	using WeirdCountMap = std::unordered_map<std::string, uint64>;
+	using WeirdFlowMap = std::map<IPPair, WeirdCountMap>;
+	using WeirdSet = std::unordered_set<std::string>;
+
 	Reporter();
 	~Reporter();
 
@@ -76,7 +85,6 @@ public:
 	// that may lead to incorrectly processing a connnection.
 	void Weird(const char* name);	// Raises net_weird().
 	void Weird(Connection* conn, const char* name, const char* addl = "");	// Raises conn_weird().
-	void Weird(Val* conn_val, const char* name, const char* addl = "");	// Raises conn_weird().
 	void Weird(const IPAddr& orig, const IPAddr& resp, const char* name);	// Raises flow_weird().
 
 	// Syslog a message. This methods does nothing if we're running
@@ -120,6 +128,30 @@ public:
 	// Signals that we're done processing an error handler event.
 	void EndErrorHandler()	{ --in_error_handler; }
 
+	/**
+	 * Reset/cleanup state tracking for a "net" weird.
+	 */
+	void ResetNetWeird(const std::string& name);
+
+	/**
+	 * Reset/cleanup state tracking for a "flow" weird.
+	 */
+	void ResetFlowWeird(const IPAddr& orig, const IPAddr& resp);
+
+	/**
+	 * Return the total number of weirds generated (counts weirds before
+	 * any rate-limiting occurs).
+	 */
+	uint64 GetWeirdCount() const
+		{ return weird_count; }
+
+	/**
+	 * Return number of weirds generated per weird type/name (counts weirds
+	 * before any rate-limiting occurs).
+	 */
+	const WeirdCountMap& GetWeirdsByType() const
+		{ return weird_count_by_type; }
+
 private:
 	void DoLog(const char* prefix, EventHandlerPtr event, FILE* out,
 		   Connection* conn, val_list* addl, bool location, bool time,
@@ -129,6 +161,11 @@ private:
 	// contain format specifiers
 	void WeirdHelper(EventHandlerPtr event, Val* conn_val, const char* addl, const char* fmt_name, ...) __attribute__((format(printf, 5, 6)));;
 	void WeirdFlowHelper(const IPAddr& orig, const IPAddr& resp, const char* fmt_name, ...) __attribute__((format(printf, 4, 5)));;
+	void UpdateWeirdStats(const char* name);
+	inline bool WeirdOnSamplingWhiteList(const char* name)
+		{ return weird_sampling_whitelist.find(name) != weird_sampling_whitelist.end(); }
+	bool PermitNetWeird(const char* name);
+	bool PermitFlowWeird(const char* name, const IPAddr& o, const IPAddr& r);
 
 	int errors;
 	bool via_events;
@@ -138,6 +175,18 @@ private:
 	bool errors_to_stderr;
 
 	std::list<std::pair<const Location*, const Location*> > locations;
+
+	uint64 weird_count;
+	WeirdCountMap weird_count_by_type;
+
+	WeirdCountMap net_weird_state;
+	WeirdFlowMap flow_weird_state;
+
+	WeirdSet weird_sampling_whitelist;
+	uint64 weird_sampling_threshold;
+	uint64 weird_sampling_rate;
+	double weird_sampling_duration;
+
 };
 
 extern Reporter* reporter;

src/Timer.cc

@@ -18,12 +18,14 @@ const char* TimerNames[] = {
 	"ConnectionStatusUpdateTimer",
 	"DNSExpireTimer",
 	"FileAnalysisInactivityTimer",
+	"FlowWeirdTimer",
 	"FragTimer",
 	"IncrementalSendTimer",
 	"IncrementalWriteTimer",
 	"InterconnTimer",
 	"IPTunnelInactivityTimer",
 	"NetbiosExpireTimer",
+	"NetWeirdTimer",
 	"NetworkTimer",
 	"NTPExpireTimer",
 	"ProfileTimer",

src/Timer.h

@@ -23,12 +23,14 @@ enum TimerType {
 	TIMER_CONN_STATUS_UPDATE,
 	TIMER_DNS_EXPIRE,
 	TIMER_FILE_ANALYSIS_INACTIVITY,
+	TIMER_FLOW_WEIRD_EXPIRE,
 	TIMER_FRAG,
 	TIMER_INCREMENTAL_SEND,
 	TIMER_INCREMENTAL_WRITE,
 	TIMER_INTERCONN,
 	TIMER_IP_TUNNEL_INACTIVITY,
 	TIMER_NB_EXPIRE,
+	TIMER_NET_WEIRD_EXPIRE,
 	TIMER_NETWORK,
 	TIMER_NTP_EXPIRE,
 	TIMER_PROFILE,

src/Val.cc

@@ -3321,6 +3321,29 @@ bool VectorVal::AssignRepeat(unsigned int index, unsigned int how_many,
 	return true;
 	}
 
+int VectorVal::AddTo(Val* val, int /* is_first_init */) const
+	{
+	if ( val->Type()->Tag() != TYPE_VECTOR )
+		{
+		val->Error("not a vector");
+		return 0;
+		}
+
+	VectorVal* v = val->AsVectorVal();
+
+	if ( ! same_type(type, v->Type()) )
+		{
+		type->Error("vector type clash", v->Type());
+		return 0;
+		}
+
+	auto last_idx = v->Size();
+
+	for ( auto i = 0u; i < Size(); ++i )
+		v->Assign(last_idx++, Lookup(i)->Ref());
+
+	return 1;
+	}
 
 Val* VectorVal::Lookup(unsigned int index) const
 	{

src/Val.h

@@ -1046,6 +1046,10 @@ public:
 	bool AssignRepeat(unsigned int index, unsigned int how_many,
 			  Val* element);
 
+	// Add this value to the given value (if appropriate).
+	// Returns true if succcessful.
+	int AddTo(Val* v, int is_first_init) const override;
+
 	// Returns nil if no element was at that value.
 	// Lookup does NOT grow the vector to this size.
 	// The Val* variant assumes that the index Val* has been type-checked.

src/analyzer/protocol/finger/Finger.cc

@@ -41,6 +41,9 @@ void Finger_Analyzer::DeliverStream(int length, const u_char* data, bool is_orig
 	const char* line = (const char*) data;
 	const char* end_of_line = line + length;
 
+	if ( length == 0 )
+		return;
+
 	if ( is_orig )
 		{
 

src/analyzer/protocol/ftp/FTP.cc

@@ -69,6 +69,10 @@ void FTP_Analyzer::DeliverStream(int length, const u_char* data, bool orig)
 	const char* line = (const char*) data;
 	const char* end_of_line = line + length;
 
+	if ( length == 0 )
+		// Could emit "ftp empty request/reply" weird, but maybe not worth it.
+		return;
+
 	val_list* vl = new val_list;
 	vl->append(BuildConnVal());
 

src/analyzer/protocol/ident/Ident.cc

@@ -56,6 +56,9 @@ void Ident_Analyzer::DeliverStream(int length, const u_char* data, bool is_orig)
 	if ( TCP() )
 		s = is_orig ? TCP()->Orig() : TCP()->Resp();
 
+	if ( length == 0 )
+		return;
+
 	if ( is_orig )
 		{
 		if ( ! ident_request )

src/analyzer/protocol/pop3/POP3.cc

@@ -132,6 +132,9 @@ static string trim_whitespace(const char* in)
 
 void POP3_Analyzer::ProcessRequest(int length, const char* line)
 	{
+	if ( length == 0 )
+		return;
+
 	if ( waitingForAuthentication )
 		{
 		++authLines;

src/analyzer/protocol/smtp/SMTP.cc

@@ -269,6 +269,8 @@ void SMTP_Analyzer::ProcessLine(int length, const char* line, bool orig)
 			if ( smtp_request )
 				{
 				int data_len = end_of_line - line;
+
+				if ( cmd_len > 0 || data_len > 0 )
 					RequestEvent(cmd_len, cmd, data_len, line);
 				}
 
@@ -379,8 +381,18 @@ void SMTP_Analyzer::NewCmd(const int cmd_code)
 		if ( first_cmd < 0 )
 			first_cmd = cmd_code;
 		else
+			{
+			auto constexpr max_pending_cmd_q_size = 1000;
+
+			if ( pending_cmd_q.size() == max_pending_cmd_q_size )
+				{
+				Weird("smtp_excessive_pending_cmds");
+				pending_cmd_q.clear();
+				}
+
 			pending_cmd_q.push_back(cmd_code);
 			}
+		}
 	else
 		first_cmd = cmd_code;
 	}
@@ -805,12 +817,22 @@ void SMTP_Analyzer::UpdateState(const int cmd_code, const int reply_code, bool o
 #endif
 	}
 
+static bool istrequal(const char* s, const char* cmd, int s_len)
+	{
+	auto cmd_len = strlen(cmd);
+
+	if ( cmd_len != s_len )
+		return false;
+
+	return strncasecmp(s, cmd, s_len) == 0;
+	}
+
 void SMTP_Analyzer::ProcessExtension(int ext_len, const char* ext)
 	{
 	if ( ! ext )
 		return;
 
-	if ( ! strncasecmp(ext, "PIPELINING", ext_len) )
+	if ( istrequal(ext, "PIPELINING", ext_len) )
 		pipelining = 1;
 	}
 
@@ -820,11 +842,11 @@ int SMTP_Analyzer::ParseCmd(int cmd_len, const char* cmd)
 		return -1;
 
 	// special case because we cannot define our usual macros with "-"
-	if ( strncmp(cmd, "X-ANONYMOUSTLS", cmd_len) == 0 )
+	if ( istrequal(cmd, "X-ANONYMOUSTLS", cmd_len) )
 		return SMTP_CMD_X_ANONYMOUSTLS;
 
 	for ( int code = SMTP_CMD_EHLO; code < SMTP_CMD_LAST; ++code )
-		if ( ! strncasecmp(cmd, smtp_cmd_word[code - SMTP_CMD_EHLO], cmd_len) )
+		if ( istrequal(cmd, smtp_cmd_word[code - SMTP_CMD_EHLO], cmd_len) )
 			return code;
 
 	return -1;

src/reporter.bif

@@ -71,3 +71,44 @@ function Reporter::fatal%(msg: string%): bool
 	reporter->PopLocation();
 	return new Val(1, TYPE_BOOL);
 	%}
+
+## Generates a "net" weird.
+##
+## name: the name of the weird.
+##
+## Returns: Always true.
+function Reporter::net_weird%(name: string%): bool
+	%{
+	reporter->Weird(name->CheckString());
+	return new Val(1, TYPE_BOOL);
+	%}
+
+## Generates a "flow" weird.
+##
+## name: the name of the weird.
+##
+## orig: the originator host associated with the weird.
+##
+## resp: the responder host associated with the weird.
+##
+## Returns: Always true.
+function Reporter::flow_weird%(name: string, orig: addr, resp: addr%): bool
+	%{
+	reporter->Weird(orig->AsAddr(), resp->AsAddr(), name->CheckString());
+	return new Val(1, TYPE_BOOL);
+	%}
+
+## Generates a "conn" weird.
+##
+## name: the name of the weird.
+##
+## c: the connection associated with the weird.
+##
+## addl: additional information to accompany the weird.
+##
+## Returns: Always true.
+function Reporter::conn_weird%(name: string, c: connection, addl: string &default=""%): bool
+	%{
+	reporter->Weird(c, name->CheckString(), addl->CheckString());
+	return new Val(1, TYPE_BOOL);
+	%}

src/stats.bif

@@ -16,6 +16,7 @@ RecordType* ThreadStats;
 RecordType* TimerStats;
 RecordType* FileAnalysisStats;
 RecordType* BrokerStats;
+RecordType* ReporterStats;
 %%}
 
 ## Returns packet capture statistics. Statistics include the number of
@@ -35,6 +36,7 @@ RecordType* BrokerStats;
 ##              get_thread_stats
 ##              get_timer_stats
 ##              get_broker_stats
+##              get_reporter_stats
 function get_net_stats%(%): NetStats
 	%{
 	uint64 recv = 0;
@@ -83,6 +85,7 @@ function get_net_stats%(%): NetStats
 ##              get_thread_stats
 ##              get_timer_stats
 ##              get_broker_stats
+##              get_reporter_stats
 function get_conn_stats%(%): ConnStats
 	%{
 	RecordVal* r = new RecordVal(ConnStats);
@@ -133,6 +136,7 @@ function get_conn_stats%(%): ConnStats
 ##              get_thread_stats
 ##              get_timer_stats
 ##              get_broker_stats
+##              get_reporter_stats
 function get_proc_stats%(%): ProcStats
 	%{
 	struct rusage ru;
@@ -189,6 +193,7 @@ function get_proc_stats%(%): ProcStats
 ##              get_thread_stats
 ##              get_timer_stats
 ##              get_broker_stats
+##              get_reporter_stats
 function get_event_stats%(%): EventStats
 	%{
 	RecordVal* r = new RecordVal(EventStats);
@@ -215,6 +220,7 @@ function get_event_stats%(%): EventStats
 ##              get_thread_stats
 ##              get_timer_stats
 ##              get_broker_stats
+##              get_reporter_stats
 function get_reassembler_stats%(%): ReassemblerStats
 	%{
 	RecordVal* r = new RecordVal(ReassemblerStats);
@@ -243,6 +249,7 @@ function get_reassembler_stats%(%): ReassemblerStats
 ##              get_thread_stats
 ##              get_timer_stats
 ##              get_broker_stats
+##              get_reporter_stats
 function get_dns_stats%(%): DNSStats
 	%{
 	RecordVal* r = new RecordVal(DNSStats);
@@ -276,6 +283,7 @@ function get_dns_stats%(%): DNSStats
 ##              get_reassembler_stats
 ##              get_thread_stats
 ##              get_broker_stats
+##              get_reporter_stats
 function get_timer_stats%(%): TimerStats
 	%{
 	RecordVal* r = new RecordVal(TimerStats);
@@ -303,6 +311,7 @@ function get_timer_stats%(%): TimerStats
 ##              get_thread_stats
 ##              get_timer_stats
 ##              get_broker_stats
+##              get_reporter_stats
 function get_file_analysis_stats%(%): FileAnalysisStats
 	%{
 	RecordVal* r = new RecordVal(FileAnalysisStats);
@@ -330,6 +339,7 @@ function get_file_analysis_stats%(%): FileAnalysisStats
 ##              get_reassembler_stats
 ##              get_timer_stats
 ##              get_broker_stats
+##              get_reporter_stats
 function get_thread_stats%(%): ThreadStats
 	%{
 	RecordVal* r = new RecordVal(ThreadStats);
@@ -355,6 +365,7 @@ function get_thread_stats%(%): ThreadStats
 ##              get_thread_stats
 ##              get_timer_stats
 ##              get_broker_stats
+##              get_reporter_stats
 function get_gap_stats%(%): GapStats
 	%{
 	RecordVal* r = new RecordVal(GapStats);
@@ -386,6 +397,7 @@ function get_gap_stats%(%): GapStats
 ##              get_thread_stats
 ##              get_timer_stats
 ##              get_broker_stats
+##              get_reporter_stats
 function get_matcher_stats%(%): MatcherStats
 	%{
 	RecordVal* r = new RecordVal(MatcherStats);
@@ -423,6 +435,7 @@ function get_matcher_stats%(%): MatcherStats
 ##              get_thread_stats
 ##              get_timer_stats
 ##              get_broker_stats
+##              get_reporter_stats
 function get_broker_stats%(%): BrokerStats
 	%{
 	RecordVal* r = new RecordVal(BrokerStats);
@@ -441,3 +454,39 @@ function get_broker_stats%(%): BrokerStats
 
 	return r;
 	%}
+
+## Returns statistics about reporter messages and weirds.
+##
+## Returns: A record with reporter statistics.
+##
+## .. bro:see:: get_conn_stats
+##              get_dns_stats
+##              get_event_stats
+##              get_file_analysis_stats
+##              get_gap_stats
+##              get_matcher_stats
+##              get_net_stats
+##              get_proc_stats
+##              get_reassembler_stats
+##              get_thread_stats
+##              get_timer_stats
+##              get_broker_stats
+function get_reporter_stats%(%): ReporterStats
+	%{
+	RecordVal* r = new RecordVal(ReporterStats);
+	int n = 0;
+
+	TableVal* weirds_by_type = new TableVal(internal_type("table_string_of_count")->AsTableType());
+
+	for ( auto& kv : reporter->GetWeirdsByType() )
+		{
+		Val* weird = new StringVal(kv.first);
+		weirds_by_type->Assign(weird, new Val(kv.second, TYPE_COUNT));
+		Unref(weird);
+		}
+
+	r->Assign(n++, new Val(reporter->GetWeirdCount(), TYPE_COUNT));
+	r->Assign(n++, weirds_by_type);
+
+	return r;
+	%}

testing/btest/Baseline/core.reporter-weird-sampling/output

@@ -0,0 +1,351 @@
+net_weird, my_net_weird
+flow_weird, my_flow_weird
+conn_weird, my_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, my_net_weird
+flow_weird, my_flow_weird
+conn_weird, my_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, my_net_weird
+flow_weird, my_flow_weird
+conn_weird, my_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, my_net_weird
+flow_weird, my_flow_weird
+conn_weird, my_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, my_net_weird
+flow_weird, my_flow_weird
+conn_weird, my_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, my_net_weird
+flow_weird, my_flow_weird
+conn_weird, my_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, my_net_weird
+flow_weird, my_flow_weird
+conn_weird, my_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, my_net_weird
+flow_weird, my_flow_weird
+conn_weird, my_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, my_net_weird
+flow_weird, my_flow_weird
+conn_weird, my_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, my_net_weird
+flow_weird, my_flow_weird
+conn_weird, my_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, my_net_weird
+flow_weird, my_flow_weird
+conn_weird, my_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, my_net_weird
+flow_weird, my_flow_weird
+conn_weird, my_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, my_net_weird
+flow_weird, my_flow_weird
+conn_weird, my_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, my_net_weird
+flow_weird, my_flow_weird
+conn_weird, my_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, my_net_weird
+flow_weird, my_flow_weird
+conn_weird, my_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, my_net_weird
+flow_weird, my_flow_weird
+conn_weird, my_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, my_net_weird
+flow_weird, my_flow_weird
+conn_weird, my_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, my_net_weird
+flow_weird, my_flow_weird
+conn_weird, my_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, my_net_weird
+flow_weird, my_flow_weird
+conn_weird, my_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, my_net_weird
+flow_weird, my_flow_weird
+conn_weird, my_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, my_net_weird
+flow_weird, my_flow_weird
+conn_weird, my_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, my_net_weird
+flow_weird, my_flow_weird
+conn_weird, my_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, my_net_weird
+flow_weird, my_flow_weird
+conn_weird, my_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, my_net_weird
+flow_weird, my_flow_weird
+conn_weird, my_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, my_net_weird
+flow_weird, my_flow_weird
+conn_weird, my_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, my_net_weird
+flow_weird, my_flow_weird
+conn_weird, my_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird
+net_weird, my_net_weird
+flow_weird, my_flow_weird
+conn_weird, my_conn_weird
+net_weird, whitelisted_net_weird
+flow_weird, whitelisted_flow_weird
+conn_weird, whitelisted_conn_weird

testing/btest/Baseline/coverage.bare-mode-errors/errors

@@ -0,0 +1,16 @@
+warning in /home/jon/projects/bro/bro/testing/btest/../../scripts/policy/protocols/dhcp/deprecated_events.bro, line 245: deprecated (dhcp_discover)
+warning in /home/jon/projects/bro/bro/testing/btest/../../scripts/policy/protocols/dhcp/deprecated_events.bro, line 248: deprecated (dhcp_offer)
+warning in /home/jon/projects/bro/bro/testing/btest/../../scripts/policy/protocols/dhcp/deprecated_events.bro, line 251: deprecated (dhcp_request)
+warning in /home/jon/projects/bro/bro/testing/btest/../../scripts/policy/protocols/dhcp/deprecated_events.bro, line 254: deprecated (dhcp_decline)
+warning in /home/jon/projects/bro/bro/testing/btest/../../scripts/policy/protocols/dhcp/deprecated_events.bro, line 257: deprecated (dhcp_ack)
+warning in /home/jon/projects/bro/bro/testing/btest/../../scripts/policy/protocols/dhcp/deprecated_events.bro, line 260: deprecated (dhcp_nak)
+warning in /home/jon/projects/bro/bro/testing/btest/../../scripts/policy/protocols/dhcp/deprecated_events.bro, line 263: deprecated (dhcp_release)
+warning in /home/jon/projects/bro/bro/testing/btest/../../scripts/policy/protocols/dhcp/deprecated_events.bro, line 266: deprecated (dhcp_inform)
+warning in /home/jon/projects/bro/bro/scripts/policy/protocols/dhcp/deprecated_events.bro, line 245: deprecated (dhcp_discover)
+warning in /home/jon/projects/bro/bro/scripts/policy/protocols/dhcp/deprecated_events.bro, line 248: deprecated (dhcp_offer)
+warning in /home/jon/projects/bro/bro/scripts/policy/protocols/dhcp/deprecated_events.bro, line 251: deprecated (dhcp_request)
+warning in /home/jon/projects/bro/bro/scripts/policy/protocols/dhcp/deprecated_events.bro, line 254: deprecated (dhcp_decline)
+warning in /home/jon/projects/bro/bro/scripts/policy/protocols/dhcp/deprecated_events.bro, line 257: deprecated (dhcp_ack)
+warning in /home/jon/projects/bro/bro/scripts/policy/protocols/dhcp/deprecated_events.bro, line 260: deprecated (dhcp_nak)
+warning in /home/jon/projects/bro/bro/scripts/policy/protocols/dhcp/deprecated_events.bro, line 263: deprecated (dhcp_release)
+warning in /home/jon/projects/bro/bro/scripts/policy/protocols/dhcp/deprecated_events.bro, line 266: deprecated (dhcp_inform)

testing/btest/Baseline/coverage.find-bro-logs/out

@@ -55,4 +55,5 @@ traceroute
 tunnel
 unified2
 weird
+weird_stats
 x509

testing/btest/Baseline/doc.broxygen.all_scripts/.stderr

@@ -1,2 +1,10 @@
+warning in /home/jon/projects/bro/bro/scripts/policy/protocols/dhcp/deprecated_events.bro, line 245: deprecated (dhcp_discover)
+warning in /home/jon/projects/bro/bro/scripts/policy/protocols/dhcp/deprecated_events.bro, line 248: deprecated (dhcp_offer)
+warning in /home/jon/projects/bro/bro/scripts/policy/protocols/dhcp/deprecated_events.bro, line 251: deprecated (dhcp_request)
+warning in /home/jon/projects/bro/bro/scripts/policy/protocols/dhcp/deprecated_events.bro, line 254: deprecated (dhcp_decline)
+warning in /home/jon/projects/bro/bro/scripts/policy/protocols/dhcp/deprecated_events.bro, line 257: deprecated (dhcp_ack)
+warning in /home/jon/projects/bro/bro/scripts/policy/protocols/dhcp/deprecated_events.bro, line 260: deprecated (dhcp_nak)
+warning in /home/jon/projects/bro/bro/scripts/policy/protocols/dhcp/deprecated_events.bro, line 263: deprecated (dhcp_release)
+warning in /home/jon/projects/bro/bro/scripts/policy/protocols/dhcp/deprecated_events.bro, line 266: deprecated (dhcp_inform)
 error in /home/jon/projects/bro/bro/scripts/policy/frameworks/control/controller.bro, line 22: The '' control command is unknown.
 <params>, line 1: received termination signal

testing/btest/Baseline/doc.sphinx.include-scripts_policy_frameworks_files_detect-MHR_bro/output

@@ -36,7 +36,7 @@ export {
 	## A/V engines.  Team Cymru returns a percentage to indicate how
 	## many A/V engines flagged the sample as malicious. This threshold
 	## allows you to require a minimum detection rate.
-	const notice_threshold = 10 &redef;
+	option notice_threshold = 10;
 }
 
 function do_mhr_lookup(hash: string, fi: Notice::FileInfo)

testing/btest/Baseline/doc.sphinx.include-scripts_policy_frameworks_files_detect-MHR_bro@3/output

@@ -27,5 +27,5 @@ export {
 	## A/V engines.  Team Cymru returns a percentage to indicate how
 	## many A/V engines flagged the sample as malicious. This threshold
 	## allows you to require a minimum detection rate.
-	const notice_threshold = 10 &redef;
+	option notice_threshold = 10;
 }

testing/btest/Baseline/language.redef-vector/out

@@ -0,0 +1,3 @@
+[testing, blah, foo, foo, testing]
+[one, two, three]
+[a, b, c, one, two, three, a, b, c, abc, d]

testing/btest/Baseline/plugins.hooks/output

@@ -274,7 +274,7 @@
 0.000000   MetaHookPost  CallFunction(Log::__create_stream, <frame>, (Weird::LOG, [columns=<no value description>, ev=Weird::log_weird, path=weird])) -> <no result>
 0.000000   MetaHookPost  CallFunction(Log::__create_stream, <frame>, (X509::LOG, [columns=<no value description>, ev=X509::log_x509, path=x509])) -> <no result>
 0.000000   MetaHookPost  CallFunction(Log::__create_stream, <frame>, (mysql::LOG, [columns=<no value description>, ev=MySQL::log_mysql, path=mysql])) -> <no result>
-0.000000   MetaHookPost  CallFunction(Log::__write, <frame>, (PacketFilter::LOG, [ts=1534522064.090237, node=bro, filter=ip or not ip, init=T, success=T])) -> <no result>
+0.000000   MetaHookPost  CallFunction(Log::__write, <frame>, (PacketFilter::LOG, [ts=1534783830.174975, node=bro, filter=ip or not ip, init=T, success=T])) -> <no result>
 0.000000   MetaHookPost  CallFunction(Log::add_default_filter, <frame>, (Broker::LOG)) -> <no result>
 0.000000   MetaHookPost  CallFunction(Log::add_default_filter, <frame>, (Cluster::LOG)) -> <no result>
 0.000000   MetaHookPost  CallFunction(Log::add_default_filter, <frame>, (Config::LOG)) -> <no result>
@@ -459,10 +459,21 @@
 0.000000   MetaHookPost  CallFunction(Log::create_stream, <frame>, (Weird::LOG, [columns=<no value description>, ev=Weird::log_weird, path=weird])) -> <no result>
 0.000000   MetaHookPost  CallFunction(Log::create_stream, <frame>, (X509::LOG, [columns=<no value description>, ev=X509::log_x509, path=x509])) -> <no result>
 0.000000   MetaHookPost  CallFunction(Log::create_stream, <frame>, (mysql::LOG, [columns=<no value description>, ev=MySQL::log_mysql, path=mysql])) -> <no result>
-0.000000   MetaHookPost  CallFunction(Log::write, <frame>, (PacketFilter::LOG, [ts=1534522064.090237, node=bro, filter=ip or not ip, init=T, success=T])) -> <no result>
+0.000000   MetaHookPost  CallFunction(Log::write, <frame>, (PacketFilter::LOG, [ts=1534783830.174975, node=bro, filter=ip or not ip, init=T, success=T])) -> <no result>
 0.000000   MetaHookPost  CallFunction(NetControl::check_plugins, <frame>, ()) -> <no result>
 0.000000   MetaHookPost  CallFunction(NetControl::init, <null>, ()) -> <no result>
 0.000000   MetaHookPost  CallFunction(Notice::want_pp, <frame>, ()) -> <no result>
+0.000000   MetaHookPost  CallFunction(Option::set_change_handler, <frame>, (DHCP::max_txid_watch_time, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100)) -> <no result>
+0.000000   MetaHookPost  CallFunction(Option::set_change_handler, <frame>, (FTP::guest_ids, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100)) -> <no result>
+0.000000   MetaHookPost  CallFunction(Option::set_change_handler, <frame>, (FTP::logged_commands, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100)) -> <no result>
+0.000000   MetaHookPost  CallFunction(Option::set_change_handler, <frame>, (HTTP::http_methods, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100)) -> <no result>
+0.000000   MetaHookPost  CallFunction(Option::set_change_handler, <frame>, (KRB::ignored_errors, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100)) -> <no result>
+0.000000   MetaHookPost  CallFunction(Option::set_change_handler, <frame>, (NetControl::catch_release_warn_blocked_ip_encountered, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100)) -> <no result>
+0.000000   MetaHookPost  CallFunction(Option::set_change_handler, <frame>, (Notice::mail_from, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100)) -> <no result>
+0.000000   MetaHookPost  CallFunction(Option::set_change_handler, <frame>, (Notice::mail_subject_prefix, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100)) -> <no result>
+0.000000   MetaHookPost  CallFunction(Option::set_change_handler, <frame>, (Notice::reply_to, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100)) -> <no result>
+0.000000   MetaHookPost  CallFunction(Option::set_change_handler, <frame>, (SIP::sip_methods, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100)) -> <no result>
+0.000000   MetaHookPost  CallFunction(Option::set_change_handler, <frame>, (SMB::logged_file_actions, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100)) -> <no result>
 0.000000   MetaHookPost  CallFunction(PacketFilter::build, <frame>, ()) -> <no result>
 0.000000   MetaHookPost  CallFunction(PacketFilter::combine_filters, <frame>, (ip or not ip, and, )) -> <no result>
 0.000000   MetaHookPost  CallFunction(PacketFilter::install, <frame>, ()) -> <no result>
@@ -1097,7 +1108,7 @@
 0.000000   MetaHookPre   CallFunction(Log::__create_stream, <frame>, (Weird::LOG, [columns=<no value description>, ev=Weird::log_weird, path=weird]))
 0.000000   MetaHookPre   CallFunction(Log::__create_stream, <frame>, (X509::LOG, [columns=<no value description>, ev=X509::log_x509, path=x509]))
 0.000000   MetaHookPre   CallFunction(Log::__create_stream, <frame>, (mysql::LOG, [columns=<no value description>, ev=MySQL::log_mysql, path=mysql]))
-0.000000   MetaHookPre   CallFunction(Log::__write, <frame>, (PacketFilter::LOG, [ts=1534522064.090237, node=bro, filter=ip or not ip, init=T, success=T]))
+0.000000   MetaHookPre   CallFunction(Log::__write, <frame>, (PacketFilter::LOG, [ts=1534783830.174975, node=bro, filter=ip or not ip, init=T, success=T]))
 0.000000   MetaHookPre   CallFunction(Log::add_default_filter, <frame>, (Broker::LOG))
 0.000000   MetaHookPre   CallFunction(Log::add_default_filter, <frame>, (Cluster::LOG))
 0.000000   MetaHookPre   CallFunction(Log::add_default_filter, <frame>, (Config::LOG))
@@ -1282,10 +1293,21 @@
 0.000000   MetaHookPre   CallFunction(Log::create_stream, <frame>, (Weird::LOG, [columns=<no value description>, ev=Weird::log_weird, path=weird]))
 0.000000   MetaHookPre   CallFunction(Log::create_stream, <frame>, (X509::LOG, [columns=<no value description>, ev=X509::log_x509, path=x509]))
 0.000000   MetaHookPre   CallFunction(Log::create_stream, <frame>, (mysql::LOG, [columns=<no value description>, ev=MySQL::log_mysql, path=mysql]))
-0.000000   MetaHookPre   CallFunction(Log::write, <frame>, (PacketFilter::LOG, [ts=1534522064.090237, node=bro, filter=ip or not ip, init=T, success=T]))
+0.000000   MetaHookPre   CallFunction(Log::write, <frame>, (PacketFilter::LOG, [ts=1534783830.174975, node=bro, filter=ip or not ip, init=T, success=T]))
 0.000000   MetaHookPre   CallFunction(NetControl::check_plugins, <frame>, ())
 0.000000   MetaHookPre   CallFunction(NetControl::init, <null>, ())
 0.000000   MetaHookPre   CallFunction(Notice::want_pp, <frame>, ())
+0.000000   MetaHookPre   CallFunction(Option::set_change_handler, <frame>, (DHCP::max_txid_watch_time, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100))
+0.000000   MetaHookPre   CallFunction(Option::set_change_handler, <frame>, (FTP::guest_ids, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100))
+0.000000   MetaHookPre   CallFunction(Option::set_change_handler, <frame>, (FTP::logged_commands, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100))
+0.000000   MetaHookPre   CallFunction(Option::set_change_handler, <frame>, (HTTP::http_methods, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100))
+0.000000   MetaHookPre   CallFunction(Option::set_change_handler, <frame>, (KRB::ignored_errors, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100))
+0.000000   MetaHookPre   CallFunction(Option::set_change_handler, <frame>, (NetControl::catch_release_warn_blocked_ip_encountered, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100))
+0.000000   MetaHookPre   CallFunction(Option::set_change_handler, <frame>, (Notice::mail_from, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100))
+0.000000   MetaHookPre   CallFunction(Option::set_change_handler, <frame>, (Notice::mail_subject_prefix, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100))
+0.000000   MetaHookPre   CallFunction(Option::set_change_handler, <frame>, (Notice::reply_to, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100))
+0.000000   MetaHookPre   CallFunction(Option::set_change_handler, <frame>, (SIP::sip_methods, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100))
+0.000000   MetaHookPre   CallFunction(Option::set_change_handler, <frame>, (SMB::logged_file_actions, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100))
 0.000000   MetaHookPre   CallFunction(PacketFilter::build, <frame>, ())
 0.000000   MetaHookPre   CallFunction(PacketFilter::combine_filters, <frame>, (ip or not ip, and, ))
 0.000000   MetaHookPre   CallFunction(PacketFilter::install, <frame>, ())
@@ -1919,7 +1941,7 @@
 0.000000 | HookCallFunction Log::__create_stream(Weird::LOG, [columns=<no value description>, ev=Weird::log_weird, path=weird])
 0.000000 | HookCallFunction Log::__create_stream(X509::LOG, [columns=<no value description>, ev=X509::log_x509, path=x509])
 0.000000 | HookCallFunction Log::__create_stream(mysql::LOG, [columns=<no value description>, ev=MySQL::log_mysql, path=mysql])
-0.000000 | HookCallFunction Log::__write(PacketFilter::LOG, [ts=1534522064.090237, node=bro, filter=ip or not ip, init=T, success=T])
+0.000000 | HookCallFunction Log::__write(PacketFilter::LOG, [ts=1534783830.174975, node=bro, filter=ip or not ip, init=T, success=T])
 0.000000 | HookCallFunction Log::add_default_filter(Broker::LOG)
 0.000000 | HookCallFunction Log::add_default_filter(Cluster::LOG)
 0.000000 | HookCallFunction Log::add_default_filter(Config::LOG)
@@ -2104,10 +2126,21 @@
 0.000000 | HookCallFunction Log::create_stream(Weird::LOG, [columns=<no value description>, ev=Weird::log_weird, path=weird])
 0.000000 | HookCallFunction Log::create_stream(X509::LOG, [columns=<no value description>, ev=X509::log_x509, path=x509])
 0.000000 | HookCallFunction Log::create_stream(mysql::LOG, [columns=<no value description>, ev=MySQL::log_mysql, path=mysql])
-0.000000 | HookCallFunction Log::write(PacketFilter::LOG, [ts=1534522064.090237, node=bro, filter=ip or not ip, init=T, success=T])
+0.000000 | HookCallFunction Log::write(PacketFilter::LOG, [ts=1534783830.174975, node=bro, filter=ip or not ip, init=T, success=T])
 0.000000 | HookCallFunction NetControl::check_plugins()
 0.000000 | HookCallFunction NetControl::init()
 0.000000 | HookCallFunction Notice::want_pp()
+0.000000 | HookCallFunction Option::set_change_handler(DHCP::max_txid_watch_time, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100)
+0.000000 | HookCallFunction Option::set_change_handler(FTP::guest_ids, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100)
+0.000000 | HookCallFunction Option::set_change_handler(FTP::logged_commands, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100)
+0.000000 | HookCallFunction Option::set_change_handler(HTTP::http_methods, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100)
+0.000000 | HookCallFunction Option::set_change_handler(KRB::ignored_errors, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100)
+0.000000 | HookCallFunction Option::set_change_handler(NetControl::catch_release_warn_blocked_ip_encountered, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100)
+0.000000 | HookCallFunction Option::set_change_handler(Notice::mail_from, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100)
+0.000000 | HookCallFunction Option::set_change_handler(Notice::mail_subject_prefix, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100)
+0.000000 | HookCallFunction Option::set_change_handler(Notice::reply_to, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100)
+0.000000 | HookCallFunction Option::set_change_handler(SIP::sip_methods, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100)
+0.000000 | HookCallFunction Option::set_change_handler(SMB::logged_file_actions, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100)
 0.000000 | HookCallFunction PacketFilter::build()
 0.000000 | HookCallFunction PacketFilter::combine_filters(ip or not ip, and, )
 0.000000 | HookCallFunction PacketFilter::install()
@@ -2462,7 +2495,7 @@
 0.000000 | HookLoadFile  base<...>/x509
 0.000000 | HookLoadFile  base<...>/xmpp
 0.000000 | HookLogInit   packet_filter 1/1 {ts (time), node (string), filter (string), init (bool), success (bool)}
-0.000000 | HookLogWrite  packet_filter [ts=1534522064.090237, node=bro, filter=ip or not ip, init=T, success=T]
+0.000000 | HookLogWrite  packet_filter [ts=1534783830.174975, node=bro, filter=ip or not ip, init=T, success=T]
 0.000000 | HookQueueEvent NetControl::init()
 0.000000 | HookQueueEvent bro_init()
 0.000000 | HookQueueEvent filter_change_tracking()

testing/btest/Baseline/scripts.policy.misc.weird-stats-cluster/manager-1.weird_stats.log

@@ -0,0 +1,13 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	weird_stats
+#open	2018-07-26-23-11-27
+#fields	ts	name	num_seen
+#types	time	string	count
+1532646687.827249	weird3	1
+1532646687.827249	weird2	1000
+1532646687.827249	weird1	2000
+1532646692.877464	weird1	2
+#close	2018-07-26-23-11-34

testing/btest/Baseline/scripts.policy.misc.weird-stats/bro.weird_stats.log

@@ -0,0 +1,12 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	weird_stats
+#open	2018-07-27-00-20-35
+#fields	ts	name	num_seen
+#types	time	string	count
+1532650834.978616	my_weird	1000
+1532650840.011592	my_weird	2000
+1532650845.043367	my_weird	10
+#close	2018-07-27-00-20-47

testing/btest/core/reporter-weird-sampling.bro

@@ -0,0 +1,55 @@
+# @TEST-EXEC: bro -b -r $TRACES/http/bro.org.pcap %INPUT >output
+# @TEST-EXEC: btest-diff output
+
+redef Weird::sampling_duration = 5sec;
+redef Weird::sampling_threshold = 10;
+redef Weird::sampling_rate = 10;
+redef Weird::sampling_whitelist = set("whitelisted_net_weird",
+                                      "whitelisted_flow_weird",
+                                      "whitelisted_conn_weird");
+
+event conn_weird(name: string, c: connection, addl: string)
+	{
+	print "conn_weird", name;
+	}
+
+event flow_weird(name: string, src: addr, dst: addr)
+	{
+	print "flow_weird", name;
+	}
+
+event net_weird(name: string)
+	{
+	print "net_weird", name;
+	}
+
+event gen_weirds(c: connection)
+	{
+	local num = 30;
+
+	while ( num != 0 )
+		{
+		Reporter::net_weird("my_net_weird");
+		Reporter::flow_weird("my_flow_weird", c$id$orig_h, c$id$resp_h);
+		Reporter::conn_weird("my_conn_weird", c);
+
+		Reporter::net_weird("whitelisted_net_weird");
+		Reporter::flow_weird("whitelisted_flow_weird", c$id$orig_h, c$id$resp_h);
+		Reporter::conn_weird("whitelisted_conn_weird", c);
+		--num;
+		}
+	}
+
+global did_one_connection = F;
+
+event new_connection(c: connection)
+	{
+	if ( did_one_connection )
+		return;
+
+	did_one_connection = T;
+	event gen_weirds(c);             # should permit 10 + 2 of each "my" weird
+	schedule 2sec { gen_weirds(c) }; # should permit 3 of each "my" weird
+	schedule 7sec { gen_weirds(c) }; # should permit 10 + 2 of each "my" weird
+	# Total of 27 "my" weirds of each type and 90 of each "whitelisted" type
+	}

testing/btest/coverage/bare-mode-errors.test

@@ -6,4 +6,4 @@
 #
 # @TEST-EXEC: test -d $DIST/scripts
 # @TEST-EXEC: for script in `find $DIST/scripts/ -name \*\.bro`; do bro -b --parse-only $script >>errors 2>&1; done
-# @TEST-EXEC: btest-diff errors
+# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff errors

testing/btest/doc/sphinx/include-scripts_policy_frameworks_files_detect-MHR_bro.btest

@@ -36,7 +36,7 @@ export {
 	## A/V engines.  Team Cymru returns a percentage to indicate how
 	## many A/V engines flagged the sample as malicious. This threshold
 	## allows you to require a minimum detection rate.
-	const notice_threshold = 10 &redef;
+	option notice_threshold = 10;
 }
 
 function do_mhr_lookup(hash: string, fi: Notice::FileInfo)

testing/btest/doc/sphinx/include-scripts_policy_frameworks_files_detect-MHR_bro@3.btest

@@ -27,5 +27,5 @@ export {
 	## A/V engines.  Team Cymru returns a percentage to indicate how
 	## many A/V engines flagged the sample as malicious. This threshold
 	## allows you to require a minimum detection rate.
-	const notice_threshold = 10 &redef;
+	option notice_threshold = 10;
 }

testing/btest/language/redef-vector.bro

@@ -0,0 +1,18 @@
+# @TEST-EXEC: bro -b %INPUT >out
+# @TEST-EXEC: btest-diff out
+
+const foo: vector of string &redef;
+redef foo += { "testing", "blah", "foo", "foo", "testing" };
+
+const bar: vector of string = vector() &redef;
+redef bar += { "one", "two", "three" };
+
+const baz: vector of string = vector("a", "b", "c") &redef;
+redef baz += { "one", "two", "three" };
+redef baz += { "a", "b", "c" };
+const d = "d";
+redef baz += { "a" + "b" + "c", d };
+
+print foo;
+print bar;
+print baz;

testing/btest/scripts/policy/misc/weird-stats-cluster.bro

@@ -0,0 +1,93 @@
+# @TEST-SERIALIZE: comm
+#
+# @TEST-EXEC: btest-bg-run manager-1 BROPATH=$BROPATH:.. CLUSTER_NODE=manager-1 bro %INPUT
+# @TEST-EXEC: btest-bg-run worker-1  BROPATH=$BROPATH:.. CLUSTER_NODE=worker-1 bro %INPUT
+# @TEST-EXEC: btest-bg-run worker-2  BROPATH=$BROPATH:.. CLUSTER_NODE=worker-2 bro %INPUT
+# @TEST-EXEC: btest-bg-wait 20
+
+# @TEST-EXEC: btest-diff manager-1/weird_stats.log
+
+@TEST-START-FILE cluster-layout.bro
+redef Cluster::nodes = {
+	["manager-1"] = [$node_type=Cluster::MANAGER, $ip=127.0.0.1, $p=37757/tcp],
+	["worker-1"]  = [$node_type=Cluster::WORKER,  $ip=127.0.0.1, $p=37760/tcp, $manager="manager-1", $interface="eth0"],
+	["worker-2"]  = [$node_type=Cluster::WORKER,  $ip=127.0.0.1, $p=37761/tcp, $manager="manager-1", $interface="eth1"],
+};
+@TEST-END-FILE
+
+@load misc/weird-stats
+
+redef Cluster::retry_interval = 1sec;
+redef Broker::default_listen_retry = 1sec;
+redef Broker::default_connect_retry = 1sec;
+
+redef Log::enable_local_logging = T;
+redef Log::default_rotation_interval = 0secs;
+redef WeirdStats::weird_stat_interval = 5secs;
+
+event terminate_me()
+	{
+	terminate();
+	}
+
+event Broker::peer_lost(endpoint: Broker::EndpointInfo, msg: string)
+	{
+	terminate();
+	}
+
+event ready_again()
+	{
+	Reporter::net_weird("weird1");
+
+	if ( Cluster::node == "worker-2" )
+		{
+		schedule 5secs { terminate_me() };
+		}
+	}
+
+event ready_for_data()
+	{
+	local n = 0;
+
+	if ( Cluster::node == "worker-1" )
+		{
+		while ( n < 1000 )
+			{
+			Reporter::net_weird("weird1");
+			++n;
+			}
+
+		Reporter::net_weird("weird3");
+		}
+	else if ( Cluster::node == "worker-2" )
+		{
+		while ( n < 1000 )
+			{
+			Reporter::net_weird("weird1");
+			Reporter::net_weird("weird2");
+			++n;
+			}
+		}
+
+	schedule 5secs { ready_again() };
+	}
+
+
+@if ( Cluster::local_node_type() == Cluster::MANAGER )
+
+event bro_init()
+	{
+	Broker::auto_publish(Cluster::worker_topic, ready_for_data);
+	}
+
+global peer_count = 0;
+
+event Broker::peer_added(endpoint: Broker::EndpointInfo, msg: string)
+	{
+	++peer_count;
+
+	if ( peer_count == 2 )
+		event ready_for_data();
+	}
+
+@endif

testing/btest/scripts/policy/misc/weird-stats.bro

@@ -0,0 +1,32 @@
+# @TEST-EXEC: btest-bg-run bro bro %INPUT
+# @TEST-EXEC: btest-bg-wait 20
+# @TEST-EXEC: btest-diff bro/weird_stats.log
+
+@load misc/weird-stats.bro
+
+redef exit_only_after_terminate = T;
+redef WeirdStats::weird_stat_interval = 5sec;
+
+event die()
+	{
+	terminate();
+	}
+
+event gen_weirds(n: count, done: bool &default = F)
+	{
+	while ( n != 0 )
+		{
+		Reporter::net_weird("my_weird");
+		--n;
+		}
+
+	if ( done )
+		schedule 5sec { die() };
+	}
+
+event bro_init()
+	{
+	event gen_weirds(1000);
+	schedule 7.5sec { gen_weirds(2000) } ;
+	schedule 12.5sec { gen_weirds(10, T) } ;
+	}

testing/scripts/diff-remove-abspath

@@ -2,4 +2,11 @@
 #
 # Replace absolute paths with the basename.
 
+if [ `uname` == "Linux" ]; then
+   sed="sed -r"
+else
+   sed="sed -E"
+fi
+
+$sed 's#/+#/#g' | \
 sed 's#/\([^/]\{1,\}/\)\{1,\}\([^/]\{1,\}\)#<...>/\2#g'

testing/scripts/diff-remove-timestamps

@@ -9,5 +9,5 @@ else
    sed="sed -E"
 fi
 
-$sed 's/(0\.000000)|([0-9]{10}\.[0-9]{2,8})/XXXXXXXXXX.XXXXXX/g' | \
+$sed 's/(0\.000000)|([0-9]{9,10}\.[0-9]{2,8})/XXXXXXXXXX.XXXXXX/g' | \
 $sed 's/^ *#(open|close).(19|20)..-..-..-..-..-..$/#\1 XXXX-XX-XX-XX-XX-XX/g'