diff --git a/CHANGES b/CHANGES index c51cd00bbf..b1c90ff1ab 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,69 @@ +2.5-884 | 2018-08-20 15:39:21 -0500 + + * Fix outdated documentation test baselines (Jon Siwek, Corelight) + + * Add 'smtp_excessive_pending_cmds' weird (Jon Siwek, Corelight) + + * Fix SMTP command string comparisons (Jon Siwek, Corelight) + + * Improve handling of empty lines in several text protocol analyzers + (Jon Siwek, Corelight) + + * Add rate-limiting sampling mechanism for weird events + + The generation of weird events, by default, are now rate-limited + according to these tunable options: + + - Weird::sampling_whitelist + - Weird::sampling_threshold + - Weird::sampling_rate + - Weird::sampling_duration + + The new get_reporter_stats() BIF also allows one to query the + total number of weirds generated (pre-sampling) which the new + policy/misc/weird-stats.bro script uses periodically to populate + a weird_stats.log. + + There's also new reporter BIFs to allow generating weirds from the + script-layer such that they go through the same, internal + rate-limiting/sampling mechanisms: + + - Reporter::conn_weird + - Reporter::flow_weird + - Reporter::net_weird + + Some of the code was adapted from previous work by Johanna Amann. + (Jon Siwek, Corelight) + + * Teach timestamp canonifier about timestamps before ~2001 + (Jon Siwek, Corelight) + +2.5-877 | 2018-08-20 14:58:58 -0500 + + * Remove the node-specific local-*.bro scripts (Daniel Thayer) + +2.5-875 | 2018-08-20 12:45:32 -0500 + + * Improve diff-remove-abspath canonifier: collapse '/' sequences + (Jon Siwek, Corelight) + + * Remove unused redef-able constants (Daniel Thayer) + + * Convert some redef-able constants to runtime options (Daniel Thayer) + +2.5-870 | 2018-08-17 17:07:57 -0500 + + * Documentation improvements (Daniel Thayer) + +2.5-855 | 2018-08-17 16:34:51 -0500 + + * Add script to support the old DHCP events (Vlad Grigorescu) + +2.5-852 | 2018-08-17 15:15:55 -0500 + + * BIT-466: add redef += support to vectors (Jon Siwek, Corelight) + 2.5-850 | 2018-08-17 11:12:53 -0500 * BIT-1815: move SMB::write_cmd_log functionality into policy/ script diff --git a/NEWS b/NEWS index e5f5b28054..d119befbb0 100644 --- a/NEWS +++ b/NEWS @@ -114,7 +114,11 @@ New Functionality - Option variables: The new "option" keyword allows variables to be declared as runtime options. Such variables cannot be changed using normal assignments. Instead, they can be changed using the - new function Option::set. + new function Config::set_value. This function will automatically + apply the change to all nodes in a cluster. Note that options can also + be changed using the new function Option::set, but this function will + not send the change to any other nodes, so Config::set_value should + typically be used instead of Option::set. It is possible to "subscribe" to an option through Option::set_change_handler, which will trigger a handler callback @@ -162,7 +166,7 @@ New Functionality } The specified file will now be monitored continuously for changes, so - that writing "testbool T" into /path/to/config.dat will + that writing "TestConfig::testbool T" into /path/to/config.dat will automatically update the option's value accordingly. The configuration framework creates a config.log that shows all @@ -220,7 +224,6 @@ New Functionality ssl_server_curve event, ssl_server_curve is now marked as deprecated. - Functions for retrieving files by their ID have been added: - Files::file_exists, Files::lookup_File - New functions in the logging API: Log::get_filter_names, Log::enable_stream @@ -237,13 +240,13 @@ New Functionality mount_proc_umnt, mount_proc_umnt_all, mount_proc_not_implemented, mount_reply_status. -- Added new NFS events: nfs_proc_symlink, nfs_proc_link, nfs_proc_sattr +- Added new NFS events: nfs_proc_symlink, nfs_proc_link, nfs_proc_sattr. - The SMB scripts in policy/protocols/smb are now moved into base/protocols/smb and loaded/enabled by default. - Added new SMB events: smb1_transaction_secondary_request, - smb1_transaction2_secondary_request, smb1_transaction_response + smb1_transaction2_secondary_request, smb1_transaction_response. - Bro can now decrypt Kerberos tickets, and retrieve the authentication from them, given a suitable keytab file. @@ -260,20 +263,18 @@ New Functionality - You can now specify that a pattern matches in a case-insensitive fashion by adding 'i' to the end of its specification. So for example - /fOO/i == "Foo" yields T, as does /fOO/i in "xFoObar". Characters - enclosed in quotes however keep their casing, so /"fOO"/i in "xFoObar" - yields F, though it yields T for "xfOObar". + /fOO/i == "Foo" yields T, as does /fOO/i in "xFoObar". You can achieve the same functionality for a subpattern enclosed in parentheses by adding "?i:" to the open parenthesis. So for example - "/foo|(?i:bar)/" will match "BaR", but not "FoO". + /foo|(?i:bar)/ will match "BaR", but not "FoO". For both ways of specifying case-insensitivity, characters enclosed in - double quotes maintain their case-sensitivity. So for example /"foo"/i - will not match "Foo", but it will match "foo". + double quotes remain case-sensitive. So for example /"foo"/i will not + match "Foo", but it will match "foo". - "make install" now installs Bro's include headers (and more) into - --prefix so that compiling plugins does no longer need access to a + --prefix so that compiling plugins no longer needs access to a source/build tree. For OS distributions, this also facilitates creating "bro-devel" packages providing all files necessary to build plugins. @@ -313,7 +314,10 @@ New Functionality of non-equality, proper superset, and superset-or-equal. - An expression of the form "v += e" will append the value of the expression - "e" to the end of the vector "v" (of course assuming type-compatbility). + "e" to the end of the vector "v" (of course assuming type-compatibility). + "redef v += { a, b, c }" will similarly extend a vector previously declared + with &redef by appending the result of expressions "a", "b", and "c" to + the vector at initialization-time. Changed Functionality --------------------- @@ -333,9 +337,9 @@ Changed Functionality script since it's generally less relevant now with the updated log. - Removed the base/protocols/dhcp/utils.bro script and thus the - 'reverse_ip' function. + "reverse_ip" function. - - Replaced all DHCP events with the single 'dhcp_message' event. + - Replaced all DHCP events with the single "dhcp_message" event. The list of removed events includes: - dhcp_discover @@ -347,6 +351,13 @@ Changed Functionality - dhcp_release - dhcp_inform + - A new script, policy/protocols/dhcp/deprecated_events.bro, may be loaded + to aid those transitioning away from the list of "removed" events above. + The script provides definitions for the old events and automatically + generates them from a dhcp_message handler, thus providing equivalent + functionality to the previous Bro release. Such usage emits deprecation + warnings. + - Removed policy/misc/known-devices.bro script and thus known_devices.log will no longer be created. @@ -371,9 +382,9 @@ Changed Functionality redef SOCKS::default_capture_password = T; - The DNS base scripts no longer generate some noisy and annoying - weirds (dns_unmatched_msg, dns_unmatched_msg_quantity, dns_unmatched_reply) + weirds (dns_unmatched_msg, dns_unmatched_msg_quantity, dns_unmatched_reply). -- The 'tunnel_parents' field of conn.log is now marked &optional, so, for +- The "tunnel_parents" field of conn.log is now marked &optional, so, for the default configuration of logs, this field will show "-" instead of "(empty)" for connections that lack any tunneling. @@ -393,11 +404,12 @@ Changed Functionality - event ssl_server_signature now has an additional argument "signature_and_hashalgorithm". -- The "dnp3_header_block" event no longer has the "start" parameter +- The "dnp3_header_block" event no longer has the "start" parameter. - The string_to_pattern() built-in (and the now-deprecated merge_pattern() built-in) is no longer restricted to only be called at initialization time. + - GeoIP Legacy Database support has been replaced with GeoIP2 MaxMind DB format support. @@ -407,6 +419,21 @@ Changed Functionality after January 2, 2019. It's also noted that all GeoIP Legacy databases may be discontinued as they are superseded by GeoIP2. +- "Weird" events are now generally suppressed/sampled by default according to + some tunable parameters: + + - Weird::sampling_whitelist + - Weird::sampling_threshold + - Weird::sampling_rate + - Weird::sampling_duration + + Those options can be changed if one needs the previous behavior of + a "net_weird", "flow_weird", or "conn_weird" event being raised for + every single event. Otherwise, there is a new weird_stats.log which + contains concise summaries of weird counts per type per time period + and the original weird.log may not differ much either, except in + the cases where a particular weird type exceeds the sampling threshold. + Removed Functionality --------------------- @@ -416,10 +443,12 @@ Removed Functionality https://github.com/bro/packages for a list of Bro packages currently available. -- BroControl: The option 'IPv6Comm' and 'ZoneID' options are no longer +- The "ocsp_request" event no longer has "requestorName" parameter. + +- BroControl: The "IPv6Comm" and "ZoneID" options are no longer available (though Broker should be able to handle IPv6 automatically). -- The "ocsp_request" event no longer has "requestorName" parameter. +- The node-specific ``site/local-*.bro`` scripts have been removed. Deprecated Functionality ------------------------ @@ -435,10 +464,6 @@ Deprecated Functionality as BiFs like send_id(). Use Broker data stores and the new configuration framework instead. -- BroControl: The 'update' command is deprecated and scheduled for - removal with the next Bro release. Bro's new configuration framework - is taking its place. - - Mixing of scalars and vectors, such as "v + e" yielding a vector corresponding to the vector v with the scalar e added to each of its elements, has been deprecated. @@ -449,6 +474,10 @@ Deprecated Functionality - The undocumented feature of using "&&" and "||" operators for patterns has been deprecated. +- BroControl: The "update" command is deprecated and scheduled for + removal with the next Bro release. Bro's new configuration framework + is taking its place. + Bro 2.5.1 ========= diff --git a/VERSION b/VERSION index 3e949de53a..4285fbbeff 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.5-850 +2.5-884 diff --git a/aux/broctl b/aux/broctl index 70a8b2e151..336e719c33 160000 --- a/aux/broctl +++ b/aux/broctl @@ -1 +1 @@ -Subproject commit 70a8b2e15105f4c238765a882151718162e46208 +Subproject commit 336e719c33d64efebe65f11322e2fbc6d11c946b diff --git a/aux/broker b/aux/broker index e0f9f6504d..2727afff78 160000 --- a/aux/broker +++ b/aux/broker @@ -1 +1 @@ -Subproject commit e0f9f6504db9285a48e0be490abddf959999a404 +Subproject commit 2727afff785201168b33b0c7448c886643f10e08 diff --git a/doc/frameworks/configuration.rst b/doc/frameworks/configuration.rst index efb5182301..23a384a280 100644 --- a/doc/frameworks/configuration.rst +++ b/doc/frameworks/configuration.rst @@ -11,7 +11,7 @@ Bro includes a "configuration framework" that allows updating script options dynamically at runtime. This functionality consists of several components: an "option" declaration, the ability to specify input files to enable changing the value of options at -runtime, a couple of built-in functions, and a log file "config.log" +runtime, a couple of functions, and a log file "config.log" which contains information about every change to option values. @@ -50,14 +50,28 @@ The "option" keyword allows variables to be declared as configuration options. option my_networks: set[subnet] = {}; option enable_feature = F; option hostname = "testsystem"; + option timeout = 1min; + option my_ports: vector of port = {}; } The rules regarding options can be thought of as being in between global variables and constants. Like global variables, options cannot be declared inside a function, hook, or event handler. Like constants, options must be -initialized when declared. The value of an option can change at runtime, +initialized when declared (the type can often be inferred from the initializer +but may need to be specified). The value of an option can change at runtime, but options cannot be assigned a new value using normal assignments. +The initial value of an option can be redefined with a :bro:keyword:`redef` +declaration just like for global variables and constants. The only difference +being that there is no need to specify the :bro:attr:`&redef` attribute in +the declaration of an option. For example, given the above option +declarations, here are some possible redefs: + +.. code:: bro + + redef TestModule::enable_feature = T; + redef TestModule::my_networks += { 10.1.0.0/16, 10.2.0.0/16 }; + Changing options ---------------- @@ -70,21 +84,32 @@ The format for these files looks like this: [option name][tab/spaces][new value] -Configuration files can be specified by adding them to Config::config_files. +Configuration files can be specified by adding them +to :bro:id:`Config::config_files`. Note that in a cluster configuration, +only the manager node attempts to read the specified configuration files. + For example, simply add something like this to local.bro: .. code:: bro redef Config::config_files += { "/path/to/config.dat" }; -The specified configuration file will then be monitored continuously for changes, -so that writing ``TestModule::enable_feature T`` into that file will -automatically update the option's value accordingly. Here is an example -configuration file:: +The specified configuration file will then be monitored continuously for +changes, so that writing ``TestModule::enable_feature T`` into that file will +automatically update the option's value accordingly (in a cluster +configuration, the change will be sent from the manager to all other nodes in +the cluster). Here is an example configuration file:: TestModule::my_networks 10.0.12.0/24,192.168.17.0/24 TestModule::enable_feature T TestModule::hostname host-1 + TestModule::timeout 50.5 + TestModule::my_ports 80/tcp,53/udp + +Note that as seen in the above example, for options of +type :bro:type:`interval`, the numeric value in the config file +is interpreted as seconds and there cannot be any time units +(such as sec, min, etc.). Internally, the configuration framework uses the Bro input framework with a type of input reader specifically for reading config files. Users @@ -94,13 +119,34 @@ for configuration files: the files need no header lines and either tabs or spaces are accepted as separators. If you inspect the configuration framework scripts, you will notice that the -scripts simply catch events from the input framework and then a built-in -function :bro:see:`Option::set` is called to set an option to the new value. -If you want to change an option yourself during runtime, you can -call Option::set directly from a script. +scripts simply catch events from the input framework and then a +function :bro:see:`Config::set_value` is called to set an option to the new +value. If you want to change an option yourself during runtime, you can +call Config::set_value directly from a script (in a cluster configuration, +this only needs to happen on the manager, as the change will be automatically +sent to all other nodes in the cluster). -The log file "config.log" contains information about each configuration -change that occurs during runtime. +Note that some data types (such as pattern, table, and record) are not +supported by the config input reader. In that case you would need to use +the Config::set_value function to change the value of such an option as +shown in the following example. + +.. code:: bro + + module TestModule; + + export { + option host_port: table[addr] of port = {}; + } + + event bro_init() { + local t: table[addr] of port = { [10.0.0.2] = 123/tcp }; + Config::set_value("TestModule::host_port", t); + } + +Regardless of whether an option change is triggered by a config file or by +the Config::set_value function, the change is always logged to the +log file "config.log". Change handlers @@ -114,7 +160,11 @@ accordingly): .. code:: bro - option testaddr = 127.0.0.1; + module TestModule; + + export { + option testaddr = 127.0.0.1; + } # Note: the data type of 2nd parameter and return type must match function change_addr(ID: string, new_value: addr): addr @@ -125,24 +175,26 @@ accordingly): event bro_init() { - Option::set_change_handler("testaddr", change_addr); + Option::set_change_handler("TestModule::testaddr", change_addr); } -Each time the specified option value is changed, the change handler -function will be called before the change is performed. The value returned -by the change handler is the value finally assigned to the option. This -allows, for example, checking of values to reject invalid input (the original -value can be returned to reject the change). - -A change handler can optionally have a third argument, which is the location -string (this is normally the pathname of the configuration file that triggered +Immediately before the specified option value is changed, the change handler +function will be called. The value returned by the change handler is the +value finally assigned to the option. This allows, for example, checking of +values to reject invalid input (the original value can be returned to reject the change). -It is also possible to chain together multiple change handlers. In this -case, the value returned by the first change handler is the "new value" seen -by the next change handler, and so on. The built-in function -:bro:see:`Option::set_change_handler` takes an optional third argument -that can specify a priority for the handlers. +It is possible define multiple change handlers for a single option. In +this case, the change handlers are chained together: the value returned by the +first change handler is the "new value" seen by the next change handler, and +so on. The built-in function :bro:see:`Option::set_change_handler` takes an +optional third argument that can specify a priority for the handlers. + +A change handler function can optionally have a third argument of type +string. When a config file triggers a change, then the third argument is +the pathname of the config file. When the Config::set_value function triggers +a change, then the third argument of the change handler is the value passed +to the optional third argument of the Config::set_value function. Note that change handlers are also used internally by the configuration framework. If you look at the script level source code of diff --git a/doc/frameworks/geoip.rst b/doc/frameworks/geoip.rst index cd41c6f54c..06829bfcd5 100644 --- a/doc/frameworks/geoip.rst +++ b/doc/frameworks/geoip.rst @@ -10,8 +10,8 @@ GeoLocation During the process of creating policy scripts the need may arise to find the geographic location for an IP address. Bro had support for the `GeoIP library `__ at the - policy script level from release 1.3 to 2.5.X to account for this - need. Starting with release 2.6 GeoIP support requires `libmaxminddb + policy script level from release 1.3 to 2.5.x to account for this + need. Starting with release 2.6, GeoIP support requires `libmaxminddb `__. To use this functionality, you need to first install the libmaxminddb software, and then install the GeoLite2 city database before building @@ -19,17 +19,11 @@ GeoLocation .. contents:: -Install libGeoIP ----------------- +Install libmaxminddb +-------------------- Before building Bro, you need to install libmaxminddb. -* FreeBSD: - - .. console:: - - sudo pkg install libmaxminddb - * RPM/RedHat-based Linux: .. console:: @@ -42,12 +36,17 @@ Before building Bro, you need to install libmaxminddb. sudo apt-get install libmaxminddb-dev +* FreeBSD: + + .. console:: + + sudo pkg install libmaxminddb + * Mac OS X: You need to install from your preferred package management system - (e.g. MacPorts, Fink, or Homebrew). The name of the package that you need - may be libmaxminddb, maxminddb, or libmaxminddb-dev, depending on which - package management system you are using. + (e.g. Homebrew, MacPorts, or Fink). For Homebrew, the name of the package + that you need is libmaxminddb. GeoLite2-City Database Installation @@ -64,8 +63,8 @@ the GeoLite2 city binary database: wget http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.tar.gz tar zxf GeoLite2-City.tar.gz -Next, the file "GeoLite2-City_YYYYMMDD/GeoLite2-City.mmdb" needs to be renamed -and put in the GeoIP database directory. This directory should already exist +Next, the file "GeoLite2-City_YYYYMMDD/GeoLite2-City.mmdb" needs to be moved +to the GeoIP database directory. This directory might already exist and will vary depending on which platform and package you are using. For FreeBSD, use ``/usr/local/share/GeoIP``. For Linux, use ``/usr/share/GeoIP`` or ``/var/lib/GeoIP`` (choose whichever one already exists). @@ -88,22 +87,23 @@ functionality works by running a command like this: If you see an error message similar to "Failed to open GeoIP location database", then you may need to either rename or move your GeoIP -location database file. Bro looks for location database files in the -following order by default: +location database file. If the :bro:see:`mmdb_dir` value is set to a +directory pathname (it is not set by default), then Bro looks for location +database files in that directory. If none are found or if mmdb_dir is not set, +then Bro looks for location database files in the following order: - /usr/share/GeoIP/GeoLite2-City.mmdb - /var/lib/GeoIP/GeoLite2-City.mmdb - /usr/local/share/GeoIP/GeoLite2-City.mmdb - /usr/local/var/GeoIP/GeoLite2-City.mmdb - /usr/share/GeoIP/GeoLite2-Country.mmdb - /var/lib/GeoIP/GeoLite2-Country.mmdb - /usr/local/share/GeoIP/GeoLite2-Country.mmdb - /usr/local/var/GeoIP/GeoLite2-Country.mmdb +* /usr/share/GeoIP/GeoLite2-City.mmdb +* /var/lib/GeoIP/GeoLite2-City.mmdb +* /usr/local/share/GeoIP/GeoLite2-City.mmdb +* /usr/local/var/GeoIP/GeoLite2-City.mmdb +* /usr/share/GeoIP/GeoLite2-Country.mmdb +* /var/lib/GeoIP/GeoLite2-Country.mmdb +* /usr/local/share/GeoIP/GeoLite2-Country.mmdb +* /usr/local/var/GeoIP/GeoLite2-Country.mmdb If you see an error message similar to "Bro was not configured for GeoIP -support", then you either need to rebuild Bro and make sure it is linked -against libmaxminddb or else set the :bro:see:`mmdb_dir` value -correctly. Normally, if libmaxminddb is installed correctly then it +support", then you need to rebuild Bro and make sure it is linked +against libmaxminddb. Normally, if libmaxminddb is installed correctly then it should automatically be found when building Bro. If this doesn't happen, then you may need to specify the path to the libmaxminddb installation (e.g. ``./configure --with-geoip=``). diff --git a/doc/install/install.rst b/doc/install/install.rst index cc8d81b14f..dee3ba0d0c 100644 --- a/doc/install/install.rst +++ b/doc/install/install.rst @@ -60,6 +60,9 @@ To install the required dependencies, you can use: sudo apt-get install cmake make gcc g++ flex bison libpcap-dev libssl-dev python-dev swig zlib1g-dev + If your system uses Python 2.7, then you will also need to install the + "python-ipaddress" package. + * FreeBSD: Most required dependencies should come with a minimal FreeBSD install @@ -114,7 +117,7 @@ Optional Dependencies Bro can make use of some optional libraries and tools if they are found at build time: - * LibGeoIP (for geolocating IP addresses) + * libmaxminddb (for geolocating IP addresses) * sendmail (enables Bro and BroControl to send mail) * curl (used by a Bro script that implements active HTTP) * gperftools (tcmalloc is used to improve memory and CPU usage) @@ -122,9 +125,9 @@ build time: * PF_RING (Linux only, see :doc:`Cluster Configuration <../configuration/index>`) * ipsumdump (for trace-summary; http://www.cs.ucla.edu/~kohler/ipsumdump) -LibGeoIP is probably the most interesting and can be installed +Geolocation is probably the most interesting and can be installed on most platforms by following the instructions for :ref:`installing -libGeoIP and the GeoIP database +the GeoIP library and database `. diff --git a/doc/script-reference/log-files.rst b/doc/script-reference/log-files.rst index 8f7d87d89a..992845cbba 100644 --- a/doc/script-reference/log-files.rst +++ b/doc/script-reference/log-files.rst @@ -14,8 +14,6 @@ Network Protocols +============================+=======================================+=================================+ | conn.log | TCP/UDP/ICMP connections | :bro:type:`Conn::Info` | +----------------------------+---------------------------------------+---------------------------------+ -| config.log | Configuration option changes | :bro:type:`Config::Info` | -+----------------------------+---------------------------------------+---------------------------------+ | dce_rpc.log | Distributed Computing Environment/RPC | :bro:type:`DCE_RPC::Info` | +----------------------------+---------------------------------------+---------------------------------+ | dhcp.log | DHCP leases | :bro:type:`DHCP::Info` | @@ -154,6 +152,8 @@ Miscellaneous +----------------------------+---------------------------------------+---------------------------------+ | weird.log | Unexpected network-level activity | :bro:type:`Weird::Info` | +----------------------------+---------------------------------------+---------------------------------+ +| weird-stats.log | Statistics about unexpected activity | :bro:type:`WeirdStats::Info` | ++----------------------------+---------------------------------------+---------------------------------+ Bro Diagnostics --------------- @@ -161,12 +161,14 @@ Bro Diagnostics +----------------------------+---------------------------------------+---------------------------------+ | Log File | Description | Field Descriptions | +============================+=======================================+=================================+ +| broker.log | Peering status events between Bro or | :bro:type:`Broker::Info` | +| | Broker-enabled processes | | ++----------------------------+---------------------------------------+---------------------------------+ | capture_loss.log | Packet loss rate | :bro:type:`CaptureLoss::Info` | +----------------------------+---------------------------------------+---------------------------------+ | cluster.log | Bro cluster messages | :bro:type:`Cluster::Info` | +----------------------------+---------------------------------------+---------------------------------+ -| broker.log | Peering status events between Bro or | :bro:type:`Broker::Info` | -| | Broker-enabled processes | | +| config.log | Configuration option changes | :bro:type:`Config::Info` | +----------------------------+---------------------------------------+---------------------------------+ | loaded_scripts.log | Shows all scripts loaded by Bro | :bro:type:`LoadedScripts::Info` | +----------------------------+---------------------------------------+---------------------------------+ diff --git a/doc/script-reference/operators.rst b/doc/script-reference/operators.rst index 8f64617494..b8a34050aa 100644 --- a/doc/script-reference/operators.rst +++ b/doc/script-reference/operators.rst @@ -10,6 +10,11 @@ Relational operators The relational operators evaluate to type :bro:type:`bool`. +In addition to numeric operands, the relational operators also work with +operands of type :bro:type:`interval`, :bro:type:`time`, :bro:type:`string`, +:bro:type:`port`, :bro:type:`addr`, and :bro:type:`set`. + + +------------------------------+--------------+ | Name | Syntax | +==============================+==============+ @@ -104,12 +109,27 @@ only. | Bitwise complement | ~ *a* | +------------------------------+-------------+ +Set operators +------------- + ++------------------------------+-------------+ +| Name | Syntax | ++==============================+=============+ +| Set intersection | *s1* & *s2* | ++------------------------------+-------------+ +| Set union | *s1* | *s2* | ++------------------------------+-------------+ +| Set difference | *s1* - *s2* | ++------------------------------+-------------+ Assignment operators -------------------- The assignment operators evaluate to the result of the assignment. +The "+=" operator can also be used to append an element to the end of a +vector. For example, ``v += e`` is equivalent to ``v[|v|] = e``. + +------------------------------+-------------+ | Name | Syntax | +==============================+=============+ @@ -215,7 +235,14 @@ Other operators | Name | Syntax | Notes | +================================+===================+========================+ | Membership test | *a* in *b* |Evaluates to type | -| | |:bro:type:`bool`. Do not| +| | |:bro:type:`bool`. Works | +| | |with :bro:type:`string`,| +| | |:bro:type:`pattern`, | +| | |:bro:type:`subnet`, | +| | |:bro:type:`set`, | +| | |:bro:type:`table`, or | +| | |:bro:type:`vector` | +| | |operands. Do not | | | |confuse this use of "in"| | | |with that used in a | | | |:bro:keyword:`for` | diff --git a/doc/script-reference/statements.rst b/doc/script-reference/statements.rst index 9e061d4df7..30e3128529 100644 --- a/doc/script-reference/statements.rst +++ b/doc/script-reference/statements.rst @@ -190,7 +190,11 @@ all loaded Bro scripts. option hostname = "host-1"; option peers: set[addr] = {}; - The value of an option cannot be changed by an assignment statement. + The initial value can be redefined with a :bro:keyword:`redef`. + + The value of an option cannot be changed by an assignment statement, but + it can be changed by either the :bro:id:`Config::set_value` function or + by changing a config file specified in :bro:id:`Config::config_files`. The scope of an option is global. @@ -215,26 +219,30 @@ all loaded Bro scripts. .. bro:keyword:: redef - There are three ways that "redef" can be used: to change the value of - a global variable (but only if it has the :bro:attr:`&redef` attribute), - to extend a record type or enum type, or to specify - a new event handler body that replaces all those that were previously - defined. + There are several ways that "redef" can be used: to redefine the initial + value of a global variable or runtime option, to extend a record type or + enum type, or to specify a new event handler body that replaces all those + that were previously defined. - If you're using "redef" to change a global variable (defined using either - :bro:keyword:`const` or :bro:keyword:`global`), then the variable that you - want to change must have the :bro:attr:`&redef` attribute. If the variable - you're changing is a table, set, or pattern, you can use ``+=`` to add - new elements, or you can use ``=`` to specify a new value (all previous - contents of the object are removed). If the variable you're changing is a - set or table, then you can use the ``-=`` operator to remove the - specified elements (nothing happens for specified elements that don't + If you're using "redef" to redefine the initial value of a global variable + (defined using either :bro:keyword:`const` or :bro:keyword:`global`), then + the variable that you want to change must have the :bro:attr:`&redef` + attribute. You can use "redef" to redefine the initial value of a + runtime option (defined using :bro:keyword:`option`) even if it doesn't + have the :bro:attr:`&redef` attribute. + + If the variable you're changing is a table, set, vector, or pattern, you can + use ``+=`` to add new elements, or you can use ``=`` to specify a new value + (all previous contents of the object are removed). If the variable you're + changing is a set or table, then you can use the ``-=`` operator to remove + the specified elements (nothing happens for specified elements that don't exist). If the variable you are changing is not a table, set, or pattern, then you must use the ``=`` operator. Examples:: redef pi = 3.14; + redef set_of_ports += { 22/tcp, 53/udp }; If you're using "redef" to extend a record or enum, then you must use the ``+=`` assignment operator. diff --git a/doc/script-reference/types.rst b/doc/script-reference/types.rst index cfb47270ff..1c7a177ebe 100644 --- a/doc/script-reference/types.rst +++ b/doc/script-reference/types.rst @@ -253,10 +253,10 @@ Here is a more detailed description of each type: When specifying a pattern, you can add a final ``i`` specifier to mark it as case-insensitive. For example, ``/foo|bar/i`` will match - a "foo", "Foo", "BaR", etc. + "foo", "Foo", "BaR", etc. You can also introduce a case-insensitive sub-pattern by enclosing it - in ``(?i:````)``. So, for example, ``/foo|(?i:bar)/`` will + in ``(?i:)``. So, for example, ``/foo|(?i:bar)/`` will match "foo" and "BaR", but *not* "Foo". For both ways of specifying case-insensitivity, characters enclosed @@ -545,13 +545,14 @@ Here is a more detailed description of each type: |s| You can compute the union, intersection, or difference of two sets - using the ``|``, ``&``, and ``-`` operators. You can compare - sets for equality (they have exactly the same elements) using ``==``. - The ``<`` operator returns ``T`` if the lefthand operand is a proper - subset of the righthand operand. Similarly, ``<=`` returns ``T`` - if the lefthand operator is a subset (not necessarily proper, i.e., - it may be equal to the righthand operand). The operators ``!=``, ``>`` - and ``>=`` provide the expected complementary operations. + using the ``|``, ``&``, and ``-`` operators. + + You can compare sets for equality (they have exactly the same elements) + using ``==``. The ``<`` operator returns ``T`` if the lefthand operand + is a proper subset of the righthand operand. Similarly, ``<=`` + returns ``T`` if the lefthand operator is a subset (not necessarily proper, + i.e., it may be equal to the righthand operand). The operators ``!=``, + ``>`` and ``>=`` provide the expected complementary operations. See the :bro:keyword:`for` statement for info on how to iterate over the elements in a set. @@ -601,8 +602,9 @@ Here is a more detailed description of each type: v[3] = "four"; - The number of elements in a vector can be obtained by placing the vector - identifier between vertical pipe characters: + The size of a vector (this is one greater than the highest index value, and + is normally equal to the number of elements in the vector) can be obtained + by placing the vector identifier between vertical pipe characters: .. code:: bro @@ -622,6 +624,11 @@ Here is a more detailed description of each type: v[|v|] = e; + The "in" operator can be used to check if a value has been assigned at a + specified index value in the vector. For example, if a vector has size 4, + then the expression ``3 in v`` would yield true and ``4 in v`` would yield + false. + Vectors of integral types (``int`` or ``count``) support the pre-increment (``++``) and pre-decrement operators (``--``), which will increment or decrement each element in the vector. diff --git a/scripts/CMakeLists.txt b/scripts/CMakeLists.txt index fb20ba6fdb..96c682871a 100644 --- a/scripts/CMakeLists.txt +++ b/scripts/CMakeLists.txt @@ -14,19 +14,3 @@ InstallPackageConfigFile( ${CMAKE_CURRENT_SOURCE_DIR}/site/local.bro ${BRO_SCRIPT_INSTALL_PATH}/site local.bro) -InstallPackageConfigFile( - ${CMAKE_CURRENT_SOURCE_DIR}/site/local-manager.bro - ${BRO_SCRIPT_INSTALL_PATH}/site - local-manager.bro) -InstallPackageConfigFile( - ${CMAKE_CURRENT_SOURCE_DIR}/site/local-logger.bro - ${BRO_SCRIPT_INSTALL_PATH}/site - local-logger.bro) -InstallPackageConfigFile( - ${CMAKE_CURRENT_SOURCE_DIR}/site/local-proxy.bro - ${BRO_SCRIPT_INSTALL_PATH}/site - local-proxy.bro) -InstallPackageConfigFile( - ${CMAKE_CURRENT_SOURCE_DIR}/site/local-worker.bro - ${BRO_SCRIPT_INSTALL_PATH}/site - local-worker.bro) diff --git a/scripts/base/frameworks/config/input.bro b/scripts/base/frameworks/config/input.bro index aaef2e59b1..7c1f37567b 100644 --- a/scripts/base/frameworks/config/input.bro +++ b/scripts/base/frameworks/config/input.bro @@ -60,8 +60,8 @@ event InputConfig::new_value(name: string, source: string, id: string, value: an function read_config(filename: string) { - # Only read the configuration on the manager. The other nodes are being fed from - # the manager. + # Only read the configuration on the manager. The other nodes are being fed + # from the manager. if ( Cluster::is_enabled() && Cluster::local_node_type() != Cluster::MANAGER ) return; diff --git a/scripts/base/frameworks/config/main.bro b/scripts/base/frameworks/config/main.bro index 6268ee4a8d..30ddeaf3b9 100644 --- a/scripts/base/frameworks/config/main.bro +++ b/scripts/base/frameworks/config/main.bro @@ -28,15 +28,15 @@ export { ## record as it is sent on to the logging framework. global log_config: event(rec: Info); - ## Broker topic for announcing new configuration value. Sending new_value, - ## peers can send configuration changes that will be distributed accross + ## Broker topic for announcing new configuration values. Sending new_value, + ## peers can send configuration changes that will be distributed across ## the entire cluster. const change_topic = "bro/config/change"; ## This function is the config framework layer around the lower-level ## :bro:see:`Option::set` call. Config::set_value will set the configuration ## value for all nodes in the cluster, no matter where it was called. Note - ## that `bro:see:`Option::set` does not distribute configuration changes + ## that :bro:see:`Option::set` does not distribute configuration changes ## to other nodes. ## ## ID: The ID of the option to update. @@ -45,7 +45,7 @@ export { ## ## location: Optional parameter detailing where this change originated from. ## - ## Returns: true on success, false when an error ocured. + ## Returns: true on success, false when an error occurs. global set_value: function(ID: string, val: any, location: string &default = "" &optional): bool; } @@ -73,7 +73,7 @@ event Config::cluster_set_option(ID: string, val: any, location: string) function set_value(ID: string, val: any, location: string &default = "" &optional): bool { local cache_val: any; - # first cache value in case setting it succeeds and we have to store it. + # First cache value in case setting it succeeds and we have to store it. if ( Cluster::local_node_type() == Cluster::MANAGER ) cache_val = copy(val); # First try setting it locally - abort if not possible. @@ -151,8 +151,8 @@ event bro_init() &priority=10 # Limit logging to the manager - everyone else just feeds off it. @if ( !Cluster::is_enabled() || Cluster::local_node_type() == Cluster::MANAGER ) - # Iterate over all existing options and add ourselves as change handlers with - # a low priority so that we can log the changes. + # Iterate over all existing options and add ourselves as change handlers + # with a low priority so that we can log the changes. local gids = global_ids(); for ( i in gids ) { diff --git a/scripts/base/frameworks/netcontrol/catch-and-release.bro b/scripts/base/frameworks/netcontrol/catch-and-release.bro index 57f135de10..0b87d0bf85 100644 --- a/scripts/base/frameworks/netcontrol/catch-and-release.bro +++ b/scripts/base/frameworks/netcontrol/catch-and-release.bro @@ -142,7 +142,7 @@ export { ## If true, catch and release warns if packets of an IP address are still seen after it ## should have been blocked. - const catch_release_warn_blocked_ip_encountered = F &redef; + option catch_release_warn_blocked_ip_encountered = F; ## Time intervals for which subsequent drops of the same IP take ## effect. diff --git a/scripts/base/frameworks/notice/actions/add-geodata.bro b/scripts/base/frameworks/notice/actions/add-geodata.bro index c762be3b86..ab9957327a 100644 --- a/scripts/base/frameworks/notice/actions/add-geodata.bro +++ b/scripts/base/frameworks/notice/actions/add-geodata.bro @@ -19,7 +19,7 @@ export { }; redef record Info += { - ## If libGeoIP support is built in, notices can have geographic + ## If GeoIP support is built in, notices can have geographic ## information attached to them. remote_location: geo_location &log &optional; }; diff --git a/scripts/base/frameworks/notice/main.bro b/scripts/base/frameworks/notice/main.bro index 3d4e354394..21e3d2079c 100644 --- a/scripts/base/frameworks/notice/main.bro +++ b/scripts/base/frameworks/notice/main.bro @@ -201,14 +201,14 @@ export { ## Address that emails will be from. ## ## Note that this is overridden by the BroControl MailFrom option. - const mail_from = "Big Brother " &redef; + option mail_from = "Big Brother "; ## Reply-to address used in outbound email. - const reply_to = "" &redef; + option reply_to = ""; ## Text string prefixed to the subject of all emails sent out. ## ## Note that this is overridden by the BroControl MailSubjectPrefix ## option. - const mail_subject_prefix = "[Bro]" &redef; + option mail_subject_prefix = "[Bro]"; ## The maximum amount of time a plugin can delay email from being sent. const max_email_delay = 15secs &redef; diff --git a/scripts/base/init-bare.bro b/scripts/base/init-bare.bro index 8febc9dae3..5ac2527bae 100644 --- a/scripts/base/init-bare.bro +++ b/scripts/base/init-bare.bro @@ -82,6 +82,13 @@ type addr_vec: vector of addr; ## directly and then remove this alias. type table_string_of_string: table[string] of string; +## A table of counts indexed by strings. +## +## .. todo:: We need this type definition only for declaring builtin functions +## via ``bifcl``. We should extend ``bifcl`` to understand composite types +## directly and then remove this alias. +type table_string_of_count: table[string] of count; + ## A set of file analyzer tags. ## ## .. todo:: We need this type definition only for declaring builtin functions @@ -626,6 +633,17 @@ type BrokerStats: record { num_ids_outgoing: count; }; +## Statistics about reporter messages and weirds. +## +## .. bro:see:: get_reporter_stats +type ReporterStats: record { + ## Number of total weirds encountered, before any rate-limiting. + weirds: count; + ## Number of times each individual weird is encountered, before any + ## rate-limiting is applied. + weirds_by_type: table[string] of count; +}; + ## Deprecated. ## ## .. todo:: Remove. It's still declared internally but doesn't seem used anywhere @@ -4826,6 +4844,35 @@ export { type Cluster::Pool: record {}; } +module Weird; +export { + ## Prevents rate-limiting sampling of any weirds named in the table. + const sampling_whitelist: set[string] &redef; + + ## How many weirds of a given type to tolerate before sampling begins. + ## i.e. this many consecutive weirds of a given type will be allowed to + ## raise events for script-layer handling before being rate-limited. + const sampling_threshold = 25 &redef; + + ## The rate-limiting sampling rate. One out of every of this number of + ## rate-limited weirds of a given type will be allowed to raise events + ## for further script-layer handling. + const sampling_rate = 1000 &redef; + + ## How long a weird of a given type is allowed to keep state/counters in + ## memory. For "net" weirds an expiration timer starts per weird name when + ## first initializing its counter. For "flow" weirds an expiration timer + ## starts once per src/dst IP pair for the first weird of any name. For + ## "conn" weirds, counters and expiration timers are kept for the duration + ## of the connection for each named weird and reset when necessary. e.g. + ## if a "conn" weird by the name of "foo" is seen more than + ## :bro:see:`Weird::sampling_threshold` times, then an expiration timer + ## begins for "foo" and upon triggering will reset the counter for "foo" + ## and unthrottle its rate-limiting until it once again exceeds the + ## threshold. + const sampling_duration = 10min &redef; +} + module GLOBAL; ## Seed for hashes computed internally for probabilistic data structures. Using diff --git a/scripts/base/protocols/dhcp/main.bro b/scripts/base/protocols/dhcp/main.bro index 2f0bb6c933..b31c623afa 100644 --- a/scripts/base/protocols/dhcp/main.bro +++ b/scripts/base/protocols/dhcp/main.bro @@ -84,7 +84,7 @@ export { ## The maximum amount of time that a transation ID will be watched ## for to try and tie messages together into a single DHCP ## transaction narrative. - const DHCP::max_txid_watch_time = 30secs &redef; + option DHCP::max_txid_watch_time = 30secs; ## This event is used internally to distribute data around clusters ## since DHCP doesn't follow the normal "connection" model used by diff --git a/scripts/base/protocols/dns/main.bro b/scripts/base/protocols/dns/main.bro index 127a06b5a0..c98f0b2201 100644 --- a/scripts/base/protocols/dns/main.bro +++ b/scripts/base/protocols/dns/main.bro @@ -113,11 +113,6 @@ export { ## DNS message query/transaction ID. type PendingMessages: table[count] of Queue::Queue; - ## The amount of time that DNS queries or replies for a given - ## query/transaction ID are allowed to be queued while waiting for - ## a matching reply or query. - const pending_msg_expiry_interval = 2min &redef; - ## Give up trying to match pending DNS queries or replies for a given ## query/transaction ID once this number of unmatched queries or replies ## is reached (this shouldn't happen unless either the DNS server/resolver diff --git a/scripts/base/protocols/ftp/main.bro b/scripts/base/protocols/ftp/main.bro index 8a24a9d92f..b3eaf79370 100644 --- a/scripts/base/protocols/ftp/main.bro +++ b/scripts/base/protocols/ftp/main.bro @@ -17,13 +17,13 @@ export { redef enum Log::ID += { LOG }; ## List of commands that should have their command/response pairs logged. - const logged_commands = { + option logged_commands = { "APPE", "DELE", "RETR", "STOR", "STOU", "ACCT", "PORT", "PASV", "EPRT", "EPSV" - } &redef; + }; ## User IDs that can be considered "anonymous". - const guest_ids = { "anonymous", "ftp", "ftpuser", "guest" } &redef; + option guest_ids = { "anonymous", "ftp", "ftpuser", "guest" }; ## This record is to hold a parsed FTP reply code. For example, for the ## 201 status code, the digits would be parsed as: x->2, y->0, z->1. diff --git a/scripts/base/protocols/http/main.bro b/scripts/base/protocols/http/main.bro index 51a89a33b9..c4487e0b51 100644 --- a/scripts/base/protocols/http/main.bro +++ b/scripts/base/protocols/http/main.bro @@ -109,7 +109,7 @@ export { ## A list of HTTP methods. Other methods will generate a weird. Note ## that the HTTP analyzer will only accept methods consisting solely ## of letters ``[A-Za-z]``. - const http_methods: set[string] = { + option http_methods: set[string] = { "GET", "POST", "HEAD", "OPTIONS", "PUT", "DELETE", "TRACE", "CONNECT", # HTTP methods for distributed authoring: @@ -117,7 +117,7 @@ export { "COPY", "MOVE", "LOCK", "UNLOCK", "POLL", "REPORT", "SUBSCRIBE", "BMOVE", "SEARCH" - } &redef; + }; ## Event that can be handled to access the HTTP record as it is sent on ## to the logging framework. @@ -199,7 +199,7 @@ event http_request(c: connection, method: string, original_URI: string, c$http$uri = unescaped_URI; if ( method !in http_methods ) - event conn_weird("unknown_HTTP_method", c, method); + Reporter::conn_weird("unknown_HTTP_method", c, method); } event http_reply(c: connection, version: string, code: count, reason: string) &priority=5 diff --git a/scripts/base/protocols/krb/main.bro b/scripts/base/protocols/krb/main.bro index 02abced683..e45ce92cd5 100644 --- a/scripts/base/protocols/krb/main.bro +++ b/scripts/base/protocols/krb/main.bro @@ -48,7 +48,7 @@ export { }; ## The server response error texts which are *not* logged. - const ignored_errors: set[string] = { + option ignored_errors: set[string] = { # This will significantly increase the noisiness of the log. # However, one attack is to iterate over principals, looking # for ones that don't require preauth, and then performn @@ -58,7 +58,7 @@ export { # This is a more specific version of NEEDED_PREAUTH that's used # by Windows AD Kerberos. "Need to use PA-ENC-TIMESTAMP/PA-PK-AS-REQ", - } &redef; + }; ## Event that can be handled to access the KRB record as it is sent on ## to the logging framework. diff --git a/scripts/base/protocols/sip/main.bro b/scripts/base/protocols/sip/main.bro index f4dba22876..1089f0c1a4 100644 --- a/scripts/base/protocols/sip/main.bro +++ b/scripts/base/protocols/sip/main.bro @@ -80,9 +80,9 @@ export { ## A list of SIP methods. Other methods will generate a weird. Note ## that the SIP analyzer will only accept methods consisting solely ## of letters ``[A-Za-z]``. - const sip_methods: set[string] = { + option sip_methods: set[string] = { "REGISTER", "INVITE", "ACK", "CANCEL", "BYE", "OPTIONS", "NOTIFY", "SUBSCRIBE" - } &redef; + }; ## Event that can be handled to access the SIP record as it is sent on ## to the logging framework. @@ -168,7 +168,7 @@ event sip_request(c: connection, method: string, original_URI: string, version: c$sip$uri = original_URI; if ( method !in sip_methods ) - event conn_weird("unknown_SIP_method", c, method); + Reporter::conn_weird("unknown_SIP_method", c, method); } event sip_reply(c: connection, version: string, code: count, reason: string) &priority=5 diff --git a/scripts/base/protocols/smb/main.bro b/scripts/base/protocols/smb/main.bro index 7e8969594d..14ecbecf46 100644 --- a/scripts/base/protocols/smb/main.bro +++ b/scripts/base/protocols/smb/main.bro @@ -33,14 +33,14 @@ export { }; ## The file actions which are logged. - const logged_file_actions: set[Action] = { + option logged_file_actions: set[Action] = { FILE_OPEN, FILE_RENAME, FILE_DELETE, PRINT_OPEN, PRINT_CLOSE, - } &redef; + }; ## This record is for the smb_files.log type FileInfo: record { diff --git a/scripts/base/protocols/smb/smb1-main.bro b/scripts/base/protocols/smb/smb1-main.bro index 44210e88f0..54add50b1f 100644 --- a/scripts/base/protocols/smb/smb1-main.bro +++ b/scripts/base/protocols/smb/smb1-main.bro @@ -309,7 +309,7 @@ event smb_pipe_request(c: connection, hdr: SMB1::Header, op_num: count) if ( ! f?$uuid ) { # TODO: figure out why this is happening. - event conn_weird("smb_pipe_request_missing_uuid", c, ""); + Reporter::conn_weird("smb_pipe_request_missing_uuid", c, ""); return; } local arg = fmt("%s: %s", diff --git a/scripts/base/protocols/ssl/main.bro b/scripts/base/protocols/ssl/main.bro index 463ecbb9ea..0418a5cebe 100644 --- a/scripts/base/protocols/ssl/main.bro +++ b/scripts/base/protocols/ssl/main.bro @@ -263,7 +263,7 @@ event ssl_extension_server_name(c: connection, is_orig: bool, names: string_vec) { c$ssl$server_name = names[0]; if ( |names| > 1 ) - event conn_weird("SSL_many_server_names", c, cat(names)); + Reporter::conn_weird("SSL_many_server_names", c, cat(names)); } } diff --git a/scripts/base/utils/exec.bro b/scripts/base/utils/exec.bro index 61488a1249..5079396bbc 100644 --- a/scripts/base/utils/exec.bro +++ b/scripts/base/utils/exec.bro @@ -42,9 +42,6 @@ export { ## Returns: A record representing the full results from the ## external program execution. global run: function(cmd: Command): Result; - - ## The system directory for temporary files. - const tmp_dir = "/tmp" &redef; } # Indexed by command uid. diff --git a/scripts/base/utils/geoip-distance.bro b/scripts/base/utils/geoip-distance.bro index 3bcf93f828..8d3149cb03 100644 --- a/scripts/base/utils/geoip-distance.bro +++ b/scripts/base/utils/geoip-distance.bro @@ -1,7 +1,7 @@ ##! Functions to calculate distance between two locations, based on GeoIP data. ## Returns the distance between two IP addresses using the haversine formula, -## based on GeoIP database locations. Requires Bro to be built with libgeoip. +## based on GeoIP database locations. Requires Bro to be built with GeoIP. ## ## a1: First IP address. ## diff --git a/scripts/broxygen/__load__.bro b/scripts/broxygen/__load__.bro index dbc175c2a8..2c89631e76 100644 --- a/scripts/broxygen/__load__.bro +++ b/scripts/broxygen/__load__.bro @@ -6,6 +6,7 @@ @load frameworks/control/controller.bro @load frameworks/files/extract-all-files.bro @load policy/misc/dump-events.bro +@load policy/protocols/dhcp/deprecated_events.bro @load ./example.bro diff --git a/scripts/policy/frameworks/dpd/packet-segment-logging.bro b/scripts/policy/frameworks/dpd/packet-segment-logging.bro index c1d6699352..35a52c3870 100644 --- a/scripts/policy/frameworks/dpd/packet-segment-logging.bro +++ b/scripts/policy/frameworks/dpd/packet-segment-logging.bro @@ -16,7 +16,7 @@ export { }; ## Size of the packet segment to display in the DPD log. - const packet_segment_size: int = 255 &redef; + option packet_segment_size: int = 255; } diff --git a/scripts/policy/frameworks/files/detect-MHR.bro b/scripts/policy/frameworks/files/detect-MHR.bro index 6917212356..760c5b3693 100644 --- a/scripts/policy/frameworks/files/detect-MHR.bro +++ b/scripts/policy/frameworks/files/detect-MHR.bro @@ -32,7 +32,7 @@ export { ## A/V engines. Team Cymru returns a percentage to indicate how ## many A/V engines flagged the sample as malicious. This threshold ## allows you to require a minimum detection rate. - const notice_threshold = 10 &redef; + option notice_threshold = 10; } function do_mhr_lookup(hash: string, fi: Notice::FileInfo) diff --git a/scripts/policy/frameworks/software/version-changes.bro b/scripts/policy/frameworks/software/version-changes.bro index 5494f14bca..215a64d6b7 100644 --- a/scripts/policy/frameworks/software/version-changes.bro +++ b/scripts/policy/frameworks/software/version-changes.bro @@ -19,7 +19,7 @@ export { ## Some software is more interesting when the version changes and this ## is a set of all software that should raise a notice when a different ## version is seen on a host. - const interesting_version_changes: set[string] = { } &redef; + option interesting_version_changes: set[string] = {}; } event Software::version_change(old: Software::Info, new: Software::Info) diff --git a/scripts/policy/misc/capture-loss.bro b/scripts/policy/misc/capture-loss.bro index 648e3d6717..8bd0a61a05 100644 --- a/scripts/policy/misc/capture-loss.bro +++ b/scripts/policy/misc/capture-loss.bro @@ -44,7 +44,7 @@ export { ## when the :bro:enum:`CaptureLoss::Too_Much_Loss` notice should be ## generated. The value is expressed as a double between 0 and 1 with 1 ## being 100%. - const too_much_loss: double = 0.1 &redef; + option too_much_loss: double = 0.1; } event CaptureLoss::take_measurement(last_ts: time, last_acks: count, last_gaps: count) diff --git a/scripts/policy/misc/dump-events.bro b/scripts/policy/misc/dump-events.bro index a0c36753cb..78772b265b 100644 --- a/scripts/policy/misc/dump-events.bro +++ b/scripts/policy/misc/dump-events.bro @@ -7,7 +7,7 @@ module DumpEvents; export { ## If true, include event arguments in output. - const include_args = T &redef; + option include_args = T; ## Only include events matching the given pattern into output. By default, the ## pattern matches all events. diff --git a/scripts/policy/misc/stats.bro b/scripts/policy/misc/stats.bro index c307ece849..0bbf5c8aac 100644 --- a/scripts/policy/misc/stats.bro +++ b/scripts/policy/misc/stats.bro @@ -8,7 +8,7 @@ export { redef enum Log::ID += { LOG }; ## How often stats are reported. - const report_interval = 5min &redef; + option report_interval = 5min; type Info: record { ## Timestamp for the measurement. diff --git a/scripts/policy/misc/weird-stats.bro b/scripts/policy/misc/weird-stats.bro new file mode 100644 index 0000000000..d08ce9381d --- /dev/null +++ b/scripts/policy/misc/weird-stats.bro @@ -0,0 +1,100 @@ +##! Log weird statistics. + +@load base/frameworks/sumstats +@load base/frameworks/cluster + +module WeirdStats; + +export { + redef enum Log::ID += { LOG }; + + ## How often stats are reported. + const weird_stat_interval = 15min &redef; + + type Info: record { + ## Timestamp for the measurement. + ts: time &log; + ## Name of the weird. + name: string &log; + ## Number of times weird was seen since the last stats interval. + num_seen: count &log; + }; + + global log_weird_stats: event(rec: Info); +} + +global this_epoch_weirds: table[string] of double; +global last_epoch_weirds: table[string] of double; + +function weird_epoch_results(ts: time, key: SumStats::Key, result: SumStats::Result) + { + this_epoch_weirds[key$str]=result["weirds.encountered"]$sum; + } + +function weird_epoch_finished(ts: time) + { + for ( n in this_epoch_weirds ) + { + local last_count: double = 0.0; + + if ( n in last_epoch_weirds ) + last_count = last_epoch_weirds[n]; + + local num_seen: double = this_epoch_weirds[n] - last_count; + + if ( num_seen > 0.0 ) + Log::write(LOG, Info($ts = ts, $name = n, + $num_seen = double_to_count(num_seen))); + } + + last_epoch_weirds = this_epoch_weirds; + this_epoch_weirds = table(); + } + +event bro_init() &priority=5 + { + Log::create_stream(WeirdStats::LOG, + [$columns = Info, $ev = log_weird_stats, + $path="weird_stats"]); + local r1 = SumStats::Reducer($stream = "weirds.encountered", + $apply = set(SumStats::SUM)); + SumStats::create([$name = "weirds.statistics", + $epoch = weird_stat_interval, $reducers = set(r1), + $epoch_result = weird_epoch_results, + $epoch_finished = weird_epoch_finished]); + } + +module SumStats; + +function observe_weird_stats() + { + local rs = get_reporter_stats(); + + for ( n in rs$weirds_by_type ) + SumStats::observe("weirds.encountered", SumStats::Key($str = n), + SumStats::Observation($dbl=rs$weirds_by_type[n]+0.0)); + } + +@if ( Cluster::is_enabled() ) + +# I'm not sure if this is a hack or not: the manager will generate this +# event at the end of its epoch so workers can handle it just in time to +# generate the necessary stats. Alternative may be workers generating the +# stats individually/proactively in their own finish_epoch, but that may be +# less synchronized? +event SumStats::cluster_ss_request(uid: string, ss_name: string, cleanup: bool) &priority=10 + { + observe_weird_stats(); + } + +@else + +event SumStats::finish_epoch(ss: SumStat) &priority=10 + { + if ( ss$name != "weirds.statistics" ) + return; + + observe_weird_stats(); + } + +@endif diff --git a/scripts/policy/protocols/dhcp/deprecated_events.bro b/scripts/policy/protocols/dhcp/deprecated_events.bro new file mode 100644 index 0000000000..941e5c72c3 --- /dev/null +++ b/scripts/policy/protocols/dhcp/deprecated_events.bro @@ -0,0 +1,272 @@ +##! Bro 2.6 removed certain DHCP events, but scripts in the Bro +##! ecosystem are still relying on those events. As a transition, this +##! script will handle the new event, and generate the old events, +##! which are marked as deprecated. Note: This script should be +##! removed in the next Bro version after 2.6. + +@load base/protocols/dhcp + +## A DHCP message. +## +## .. note:: This type is included to support the deprecated events dhcp_ack, +## dhcp_decline, dhcp_discover, dhcp_inform, dhcp_nak, dhcp_offer, +## dhcp_release and dhcp_request and is thus similarly deprecated +## itself. Use :bro:see:`dhcp_message` instead. +## +## .. bro:see:: dhcp_message dhcp_ack dhcp_decline dhcp_discover +## dhcp_inform dhcp_nak dhcp_offer dhcp_release dhcp_request +type dhcp_msg: record { + op: count; ##< Message OP code. 1 = BOOTREQUEST, 2 = BOOTREPLY + m_type: count; ##< The type of DHCP message. + xid: count; ##< Transaction ID of a DHCP session. + h_addr: string; ##< Hardware address of the client. + ciaddr: addr; ##< Original IP address of the client. + yiaddr: addr; ##< IP address assigned to the client. +}; + +## A list of router addresses offered by a DHCP server. +## +## .. note:: This type is included to support the deprecated events dhcp_ack +## and dhcp_offer and is thus similarly deprecated +## itself. Use :bro:see:`dhcp_message` instead. +## +## .. bro:see:: dhcp_message dhcp_ack dhcp_offer +type dhcp_router_list: table[count] of addr; + +## Generated for DHCP messages of type *DHCPDISCOVER* (client broadcast to locate +## available servers). +## +## c: The connection record describing the underlying UDP flow. +## +## msg: The parsed type-independent part of the DHCP message. +## +## req_addr: The specific address requested by the client. +## +## host_name: The value of the host name option, if specified by the client. +## +## .. bro:see:: dhcp_message dhcp_discover dhcp_offer dhcp_request +## dhcp_decline dhcp_ack dhcp_nak dhcp_release dhcp_inform +## +## .. note:: This event has been deprecated, and will be removed in the next version. +## Use dhcp_message instead. +## +## .. note:: Bro does not support broadcast packets (as used by the DHCP +## protocol). It treats broadcast addresses just like any other and +## associates packets into transport-level flows in the same way as usual. +## +global dhcp_discover: event(c: connection, msg: dhcp_msg, req_addr: addr, host_name: string) &deprecated; + +## Generated for DHCP messages of type *DHCPOFFER* (server to client in response +## to DHCPDISCOVER with offer of configuration parameters). +## +## c: The connection record describing the underlying UDP flow. +## +## msg: The parsed type-independent part of the DHCP message. +## +## mask: The subnet mask specified by the message. +## +## router: The list of routers specified by the message. +## +## lease: The least interval specified by the message. +## +## serv_addr: The server address specified by the message. +## +## host_name: Optional host name value. May differ from the host name requested +## from the client. +## +## .. bro:see:: dhcp_message dhcp_discover dhcp_request dhcp_decline +## dhcp_ack dhcp_nak dhcp_release dhcp_inform +## +## .. note:: This event has been deprecated, and will be removed in the next version. +## Use dhcp_message instead. +## +## .. note:: Bro does not support broadcast packets (as used by the DHCP +## protocol). It treats broadcast addresses just like any other and +## associates packets into transport-level flows in the same way as usual. +## +global dhcp_offer: event(c: connection, msg: dhcp_msg, mask: addr, router: dhcp_router_list, lease: interval, serv_addr: addr, host_name: string) &deprecated; + +## Generated for DHCP messages of type *DHCPREQUEST* (Client message to servers either +## (a) requesting offered parameters from one server and implicitly declining offers +## from all others, (b) confirming correctness of previously allocated address after, +## e.g., system reboot, or (c) extending the lease on a particular network address.) +## +## c: The connection record describing the underlying UDP flow. +## +## msg: The parsed type-independent part of the DHCP message. +## +## req_addr: The client address specified by the message. +## +## serv_addr: The server address specified by the message. +## +## host_name: The value of the host name option, if specified by the client. +## +## .. bro:see:: dhcp_message dhcp_discover dhcp_offer dhcp_decline +## dhcp_ack dhcp_nak dhcp_release dhcp_inform +## +## .. note:: This event has been deprecated, and will be removed in the next version. +## Use dhcp_message instead. +## +## .. note:: Bro does not support broadcast packets (as used by the DHCP +## protocol). It treats broadcast addresses just like any other and +## associates packets into transport-level flows in the same way as usual. +## +global dhcp_request: event(c: connection, msg: dhcp_msg, req_addr: addr, serv_addr: addr, host_name: string) &deprecated; + +## Generated for DHCP messages of type *DHCPDECLINE* (Client to server indicating +## network address is already in use). +## +## c: The connection record describing the underlying UDP flow. +## +## msg: The parsed type-independent part of the DHCP message. +## +## host_name: Optional host name value. +## +## .. bro:see:: dhcp_message dhcp_discover dhcp_offer dhcp_request +## dhcp_ack dhcp_nak dhcp_release dhcp_inform +## +## .. note:: This event has been deprecated, and will be removed in the next version. +## Use dhcp_message instead. +## +## .. note:: Bro does not support broadcast packets (as used by the DHCP +## protocol). It treats broadcast addresses just like any other and +## associates packets into transport-level flows in the same way as usual. +## +global dhcp_decline: event(c: connection, msg: dhcp_msg, host_name: string) &deprecated; + +## Generated for DHCP messages of type *DHCPACK* (Server to client with configuration +## parameters, including committed network address). +## +## c: The connection record describing the underlying UDP flow. +## +## msg: The parsed type-independent part of the DHCP message. +## +## mask: The subnet mask specified by the message. +## +## router: The list of routers specified by the message. +## +## lease: The least interval specified by the message. +## +## serv_addr: The server address specified by the message. +## +## host_name: Optional host name value. May differ from the host name requested +## from the client. +## +## .. bro:see:: dhcp_message dhcp_discover dhcp_offer dhcp_request +## dhcp_decline dhcp_nak dhcp_release dhcp_inform +## +## .. note:: This event has been deprecated, and will be removed in the next version. +## Use dhcp_message instead. +## +global dhcp_ack: event(c: connection, msg: dhcp_msg, mask: addr, router: dhcp_router_list, lease: interval, serv_addr: addr, host_name: string) &deprecated; + +## Generated for DHCP messages of type *DHCPNAK* (Server to client indicating client's +## notion of network address is incorrect (e.g., client has moved to new subnet) or +## client's lease has expired). +## +## c: The connection record describing the underlying UDP flow. +## +## msg: The parsed type-independent part of the DHCP message. +## +## host_name: Optional host name value. +## +## .. bro:see:: dhcp_message dhcp_discover dhcp_offer dhcp_request +## dhcp_decline dhcp_ack dhcp_release dhcp_inform +## +## .. note:: This event has been deprecated, and will be removed in the next version. +## Use dhcp_message instead. +## +## .. note:: Bro does not support broadcast packets (as used by the DHCP +## protocol). It treats broadcast addresses just like any other and +## associates packets into transport-level flows in the same way as usual. +## +global dhcp_nak: event(c: connection, msg: dhcp_msg, host_name: string) &deprecated; + +## Generated for DHCP messages of type *DHCPRELEASE* (Client to server relinquishing +## network address and cancelling remaining lease). +## +## c: The connection record describing the underlying UDP flow. +## +## msg: The parsed type-independent part of the DHCP message. +## +## host_name: The value of the host name option, if specified by the client. +## +## .. bro:see:: dhcp_message dhcp_discover dhcp_offer dhcp_request +## dhcp_decline dhcp_ack dhcp_nak dhcp_inform +## +## .. note:: This event has been deprecated, and will be removed in the next version. +## Use dhcp_message instead. +## +global dhcp_release: event(c: connection, msg: dhcp_msg, host_name: string) &deprecated; + +## Generated for DHCP messages of type *DHCPINFORM* (Client to server, asking only for +## local configuration parameters; client already has externally configured network +## address). +## +## c: The connection record describing the underlying UDP flow. +## +## msg: The parsed type-independent part of the DHCP message. +## +## host_name: The value of the host name option, if specified by the client. +## +## .. bro:see:: dhcp_message dhcp_discover dhcp_offer dhcp_request +## dhcp_decline dhcp_ack dhcp_nak dhcp_release +## +## .. note:: This event has been deprecated, and will be removed in the next version. +## Use dhcp_message instead. +## +## .. note:: Bro does not support broadcast packets (as used by the DHCP +## protocol). It treats broadcast addresses just like any other and +## associates packets into transport-level flows in the same way as usual. +## +global dhcp_inform: event(c: connection, msg: dhcp_msg, host_name: string) &deprecated; + +event dhcp_message(c: connection, is_orig: bool, msg: DHCP::Msg, options: DHCP::Options) + { + local old_msg: dhcp_msg = [$op=msg$op, $m_type=msg$m_type, $xid=msg$xid, + $h_addr=msg$chaddr, $ciaddr=msg$ciaddr, $yiaddr=msg$yiaddr]; + + local routers = dhcp_router_list(); + + if ( options?$routers ) + for ( i in options$routers ) + routers[|routers|] = options$routers[i]; + + # These fields are technically optional, but aren't listed as such in the event. + # We give it some defaults in order to suppress errors. + local ar = ( options?$addr_request ) ? options$addr_request : 0.0.0.0; + local hn = ( options?$host_name ) ? options$host_name : ""; + local le = ( options?$lease ) ? options$lease : 0 secs; + local sm = ( options?$subnet_mask ) ? options$subnet_mask : 255.255.255.255; + local sa = ( options?$serv_addr ) ? options$serv_addr : 0.0.0.0; + + switch ( DHCP::message_types[msg$m_type] ) { + case "DISCOVER": + event dhcp_discover(c, old_msg, ar, hn); + break; + case "OFFER": + event dhcp_offer(c, old_msg, sm, routers, le, sa, hn); + break; + case "REQUEST": + event dhcp_request(c, old_msg, ar, sa, hn); + break; + case "DECLINE": + event dhcp_decline(c, old_msg, hn); + break; + case "ACK": + event dhcp_ack(c, old_msg, sm, routers, le, sa, hn); + break; + case "NAK": + event dhcp_nak(c, old_msg, hn); + break; + case "RELEASE": + event dhcp_release(c, old_msg, hn); + break; + case "INFORM": + event dhcp_inform(c, old_msg, hn); + break; + default: + # This isn't a weird, it's just a DHCP message type the old scripts don't handle + break; + } + } diff --git a/scripts/policy/protocols/http/header-names.bro b/scripts/policy/protocols/http/header-names.bro index 1b256226dd..9f4e83638c 100644 --- a/scripts/policy/protocols/http/header-names.bro +++ b/scripts/policy/protocols/http/header-names.bro @@ -18,10 +18,10 @@ export { }; ## A boolean value to determine if client header names are to be logged. - const log_client_header_names = T &redef; + option log_client_header_names = T; ## A boolean value to determine if server header names are to be logged. - const log_server_header_names = F &redef; + option log_server_header_names = F; } event http_header(c: connection, is_orig: bool, name: string, value: string) &priority=3 diff --git a/scripts/policy/protocols/smb/log-cmds.bro b/scripts/policy/protocols/smb/log-cmds.bro index 6890535c3b..53e309c5ea 100644 --- a/scripts/policy/protocols/smb/log-cmds.bro +++ b/scripts/policy/protocols/smb/log-cmds.bro @@ -11,9 +11,9 @@ export { }; ## The server response statuses which are *not* logged. - const ignored_command_statuses: set[string] = { + option ignored_command_statuses: set[string] = { "MORE_PROCESSING_REQUIRED", - } &redef; + }; } ## Internal use only. diff --git a/scripts/policy/protocols/smtp/detect-suspicious-orig.bro b/scripts/policy/protocols/smtp/detect-suspicious-orig.bro index 6fe37e02a8..58eecceb11 100644 --- a/scripts/policy/protocols/smtp/detect-suspicious-orig.bro +++ b/scripts/policy/protocols/smtp/detect-suspicious-orig.bro @@ -9,10 +9,10 @@ export { }; ## Places where it's suspicious for mail to originate from represented - ## as all-capital, two character country codes (e.g., US). It requires - ## libGeoIP support built in. - const suspicious_origination_countries: set[string] = {} &redef; - const suspicious_origination_networks: set[subnet] = {} &redef; + ## as all-capital, two character country codes (e.g., US). It requires + ## Bro to be built with GeoIP support. + option suspicious_origination_countries: set[string] = {}; + option suspicious_origination_networks: set[subnet] = {}; } diff --git a/scripts/policy/protocols/smtp/entities-excerpt.bro b/scripts/policy/protocols/smtp/entities-excerpt.bro index 0e596ebdd1..a3c35507ca 100644 --- a/scripts/policy/protocols/smtp/entities-excerpt.bro +++ b/scripts/policy/protocols/smtp/entities-excerpt.bro @@ -14,7 +14,7 @@ export { ## This is the default value for how much of the entity body should be ## included for all MIME entities. The lesser of this value and ## :bro:see:`default_file_bof_buffer_size` will be used. - const default_entity_excerpt_len = 0 &redef; + option default_entity_excerpt_len = 0; } event file_new(f: fa_file) &priority=5 diff --git a/scripts/policy/protocols/ssh/geo-data.bro b/scripts/policy/protocols/ssh/geo-data.bro index 7235f24199..c8f73b6ac7 100644 --- a/scripts/policy/protocols/ssh/geo-data.bro +++ b/scripts/policy/protocols/ssh/geo-data.bro @@ -21,7 +21,7 @@ export { ## The set of countries for which you'd like to generate notices upon ## successful login. - const watched_countries: set[string] = {"RO"} &redef; + option watched_countries: set[string] = {"RO"}; } function get_location(c: connection): geo_location diff --git a/scripts/policy/protocols/ssl/expiring-certs.bro b/scripts/policy/protocols/ssl/expiring-certs.bro index 04ebeb3c5a..1e806942d7 100644 --- a/scripts/policy/protocols/ssl/expiring-certs.bro +++ b/scripts/policy/protocols/ssl/expiring-certs.bro @@ -27,11 +27,11 @@ export { ## notices will be suppressed by the notice framework for 1 day after ## a particular certificate has had a notice generated. ## Choices are: LOCAL_HOSTS, REMOTE_HOSTS, ALL_HOSTS, NO_HOSTS - const notify_certs_expiration = LOCAL_HOSTS &redef; + option notify_certs_expiration = LOCAL_HOSTS; ## The time before a certificate is going to expire that you would like ## to start receiving :bro:enum:`SSL::Certificate_Expires_Soon` notices. - const notify_when_cert_expiring_in = 30days &redef; + option notify_when_cert_expiring_in = 30days; } event ssl_established(c: connection) &priority=3 diff --git a/scripts/policy/protocols/ssl/weak-keys.bro b/scripts/policy/protocols/ssl/weak-keys.bro index a8859b2e59..afebb035a3 100644 --- a/scripts/policy/protocols/ssl/weak-keys.bro +++ b/scripts/policy/protocols/ssl/weak-keys.bro @@ -21,24 +21,24 @@ export { ## keys/ciphers/protocol_versions. By default, these notices will be suppressed ## by the notice framework for 1 day after a particular host has had a notice ## generated. Choices are: LOCAL_HOSTS, REMOTE_HOSTS, ALL_HOSTS, NO_HOSTS - const notify_weak_keys = LOCAL_HOSTS &redef; + option notify_weak_keys = LOCAL_HOSTS; ## The minimal key length in bits that is considered to be safe. Any shorter ## (non-EC) key lengths will trigger a notice. - const notify_minimal_key_length = 2048 &redef; + option notify_minimal_key_length = 2048; ## Warn if the DH key length is smaller than the certificate key length. This is ## potentially unsafe because it gives a wrong impression of safety due to the ## certificate key length. However, it is very common and cannot be avoided in some ## settings (e.g. with old jave clients). - const notify_dh_length_shorter_cert_length = T &redef; + option notify_dh_length_shorter_cert_length = T; ## Warn if a server negotiates a SSL session with a protocol version smaller than ## the specified version. By default, the minimal version is TLSv10 because SSLv2 ## and v3 have serious security issued. ## See https://tools.ietf.org/html/draft-thomson-sslv3-diediedie-00 ## To disable, set to SSLv20 - const tls_minimum_version = TLSv10 &redef; + option tls_minimum_version = TLSv10; ## Warn if a server negotiates an unsafe cipher suite. By default, we only warn when ## encountering old export cipher suites, or RC4 (see RFC7465). diff --git a/scripts/site/local-logger.bro b/scripts/site/local-logger.bro deleted file mode 100644 index 0642e86ce3..0000000000 --- a/scripts/site/local-logger.bro +++ /dev/null @@ -1 +0,0 @@ -##! Local site policy loaded only by the logger if Bro is running as a cluster. diff --git a/scripts/site/local-manager.bro b/scripts/site/local-manager.bro deleted file mode 100644 index 5e6005f21e..0000000000 --- a/scripts/site/local-manager.bro +++ /dev/null @@ -1 +0,0 @@ -##! Local site policy loaded only by the manager if Bro is running as a cluster. diff --git a/scripts/site/local-proxy.bro b/scripts/site/local-proxy.bro deleted file mode 100644 index 478ba6d048..0000000000 --- a/scripts/site/local-proxy.bro +++ /dev/null @@ -1 +0,0 @@ -##! Local site policy loaded only by the proxies if Bro is running as a cluster. diff --git a/scripts/site/local-worker.bro b/scripts/site/local-worker.bro deleted file mode 100644 index b2a100e135..0000000000 --- a/scripts/site/local-worker.bro +++ /dev/null @@ -1 +0,0 @@ -##! Local site policy loaded only by the workers if Bro is running as a cluster. \ No newline at end of file diff --git a/scripts/site/local.bro b/scripts/site/local.bro index 605b5ab75d..97a8f078de 100644 --- a/scripts/site/local.bro +++ b/scripts/site/local.bro @@ -66,7 +66,7 @@ # certificate notary service; see http://notary.icsi.berkeley.edu . # @load protocols/ssl/notary -# If you have libGeoIP support built in, do some geographic detections and +# If you have GeoIP support built in, do some geographic detections and # logging for SSH traffic. @load protocols/ssh/geo-data # Detect hosts doing SSH bruteforce attacks. diff --git a/scripts/test-all-policy.bro b/scripts/test-all-policy.bro index 7bdd2d4997..11824c2c6a 100644 --- a/scripts/test-all-policy.bro +++ b/scripts/test-all-policy.bro @@ -54,12 +54,14 @@ @load misc/profiling.bro @load misc/scan.bro @load misc/stats.bro +@load misc/weird-stats.bro @load misc/trim-trace-file.bro @load protocols/conn/known-hosts.bro @load protocols/conn/known-services.bro @load protocols/conn/mac-logging.bro @load protocols/conn/vlan-logging.bro @load protocols/conn/weirds.bro +#@load protocols/dhcp/deprecated_events.bro @load protocols/dhcp/msg-orig.bro @load protocols/dhcp/software.bro @load protocols/dhcp/sub-opts.bro diff --git a/src/Conn.cc b/src/Conn.cc index 447f730418..c69f155731 100644 --- a/src/Conn.cc +++ b/src/Conn.cc @@ -1071,3 +1071,28 @@ void Connection::CheckFlowLabel(bool is_orig, uint32 flow_label) else saw_first_resp_packet = 1; } + +bool Connection::PermitWeird(const char* name, uint64 threshold, uint64 rate, + double duration) + { + auto& state = weird_state[name]; + ++state.count; + + if ( state.count < threshold ) + return true; + + if ( state.count == threshold ) + state.sampling_start_time = network_time; + else + { + if ( network_time > state.sampling_start_time + duration ) + { + state.sampling_start_time = 0; + state.count = 1; + return true; + } + } + + auto num_above_threshold = state.count - threshold; + return num_above_threshold % rate == 0; + } diff --git a/src/Conn.h b/src/Conn.h index 07765ee474..ae639d6341 100644 --- a/src/Conn.h +++ b/src/Conn.h @@ -5,6 +5,9 @@ #include +#include +#include + #include "Dict.h" #include "Val.h" #include "Timer.h" @@ -275,6 +278,9 @@ public: uint32 GetOrigFlowLabel() { return orig_flow_label; } uint32 GetRespFlowLabel() { return resp_flow_label; } + bool PermitWeird(const char* name, uint64 threshold, uint64 rate, + double duration); + protected: Connection() { persistent = 0; } @@ -339,6 +345,14 @@ protected: analyzer::pia::PIA* primary_PIA; Bro::UID uid; // Globally unique connection ID. + + struct WeirdState { + WeirdState() { count = 0; sampling_start_time = 0; } + uint64 count = 0; + double sampling_start_time = 0; + }; + + std::unordered_map weird_state; }; class ConnectionTimer : public Timer { diff --git a/src/Func.cc b/src/Func.cc index 703cc04d29..015829e2a1 100644 --- a/src/Func.cc +++ b/src/Func.cc @@ -722,6 +722,7 @@ void init_builtin_funcs() FileAnalysisStats = internal_type("FileAnalysisStats")->AsRecordType(); ThreadStats = internal_type("ThreadStats")->AsRecordType(); BrokerStats = internal_type("BrokerStats")->AsRecordType(); + ReporterStats = internal_type("ReporterStats")->AsRecordType(); var_sizes = internal_type("var_sizes")->AsTableType(); diff --git a/src/ID.cc b/src/ID.cc index a68abb6264..9e4eb7d245 100644 --- a/src/ID.cc +++ b/src/ID.cc @@ -140,10 +140,11 @@ void ID::SetVal(Val* v, init_class c) } if ( type->Tag() != TYPE_TABLE && - (type->Tag() != TYPE_PATTERN || c == INIT_REMOVE) ) + (type->Tag() != TYPE_PATTERN || c == INIT_REMOVE) && + (type->Tag() != TYPE_VECTOR || c == INIT_REMOVE) ) { if ( c == INIT_EXTRA ) - Error("+= initializer only applies to tables, sets and patterns", v); + Error("+= initializer only applies to tables, sets, vectors and patterns", v); else Error("-= initializer only applies to tables and sets", v); } diff --git a/src/Reporter.cc b/src/Reporter.cc index eb89a29d30..59e877bb7c 100644 --- a/src/Reporter.cc +++ b/src/Reporter.cc @@ -10,6 +10,7 @@ #include "NetVar.h" #include "Net.h" #include "Conn.h" +#include "Timer.h" #include "plugin/Plugin.h" #include "plugin/Manager.h" @@ -36,6 +37,11 @@ Reporter::Reporter() warnings_to_stderr = true; errors_to_stderr = true; + weird_count = 0; + weird_sampling_rate = 0; + weird_sampling_duration = 0; + weird_sampling_threshold = 0; + openlog("bro", 0, LOG_LOCAL5); } @@ -49,6 +55,24 @@ void Reporter::InitOptions() info_to_stderr = internal_const_val("Reporter::info_to_stderr")->AsBool(); warnings_to_stderr = internal_const_val("Reporter::warnings_to_stderr")->AsBool(); errors_to_stderr = internal_const_val("Reporter::errors_to_stderr")->AsBool(); + weird_sampling_rate = internal_const_val("Weird::sampling_rate")->AsCount(); + weird_sampling_threshold = internal_const_val("Weird::sampling_threshold")->AsCount(); + weird_sampling_duration = internal_const_val("Weird::sampling_duration")->AsInterval(); + auto wl_val = internal_const_val("Weird::sampling_whitelist")->AsTableVal(); + auto wl_table = wl_val->AsTable(); + + HashKey* k; + IterCookie* c = wl_table->InitForIteration(); + TableEntryVal* v; + + while ( (v = wl_table->NextEntry(k, c)) ) + { + auto index = wl_val->RecoverIndex(k); + string key = index->Index(0)->AsString()->CheckString(); + weird_sampling_whitelist.emplace(move(key)); + Unref(index); + delete k; + } } void Reporter::Info(const char* fmt, ...) @@ -221,23 +245,121 @@ void Reporter::WeirdFlowHelper(const IPAddr& orig, const IPAddr& resp, const cha delete vl; } +void Reporter::UpdateWeirdStats(const char* name) + { + ++weird_count; + ++weird_count_by_type[name]; + } + +class NetWeirdTimer : public Timer { +public: + NetWeirdTimer(double t, const char* name, double timeout) + : Timer(t + timeout, TIMER_NET_WEIRD_EXPIRE), weird_name(name) + {} + + void Dispatch(double t, int is_expire) override + { reporter->ResetNetWeird(weird_name); } + + std::string weird_name; +}; + +class FlowWeirdTimer : public Timer { +public: + using IPPair = std::pair; + + FlowWeirdTimer(double t, IPPair p, double timeout) + : Timer(t + timeout, TIMER_FLOW_WEIRD_EXPIRE), endpoints(p) + {} + + void Dispatch(double t, int is_expire) override + { reporter->ResetFlowWeird(endpoints.first, endpoints.second); } + + IPPair endpoints; +}; + +void Reporter::ResetNetWeird(const std::string& name) + { + net_weird_state.erase(name); + } + +void Reporter::ResetFlowWeird(const IPAddr& orig, const IPAddr& resp) + { + flow_weird_state.erase(std::make_pair(orig, resp)); + } + +bool Reporter::PermitNetWeird(const char* name) + { + auto& count = net_weird_state[name]; + ++count; + + if ( count == 1 ) + timer_mgr->Add(new NetWeirdTimer(network_time, name, + weird_sampling_duration)); + + if ( count < weird_sampling_threshold ) + return true; + + auto num_above_threshold = count - weird_sampling_threshold; + return num_above_threshold % weird_sampling_rate == 0; + } + +bool Reporter::PermitFlowWeird(const char* name, + const IPAddr& orig, const IPAddr& resp) + { + auto endpoints = std::make_pair(orig, resp); + auto& map = flow_weird_state[endpoints]; + + if ( map.empty() ) + timer_mgr->Add(new FlowWeirdTimer(network_time, endpoints, + weird_sampling_duration)); + + auto& count = map[name]; + ++count; + + if ( count < weird_sampling_threshold ) + return true; + + auto num_above_threshold = count - weird_sampling_threshold; + return num_above_threshold % weird_sampling_rate == 0; + } + void Reporter::Weird(const char* name) { + UpdateWeirdStats(name); + + if ( ! WeirdOnSamplingWhiteList(name) ) + { + if ( ! PermitNetWeird(name) ) + return; + } + WeirdHelper(net_weird, 0, 0, "%s", name); } void Reporter::Weird(Connection* conn, const char* name, const char* addl) { - WeirdHelper(conn_weird, conn->BuildConnVal(), addl, "%s", name); - } + UpdateWeirdStats(name); -void Reporter::Weird(Val* conn_val, const char* name, const char* addl) - { - WeirdHelper(conn_weird, conn_val, addl, "%s", name); + if ( ! WeirdOnSamplingWhiteList(name) ) + { + if ( ! conn->PermitWeird(name, weird_sampling_threshold, + weird_sampling_rate, weird_sampling_duration) ) + return; + } + + WeirdHelper(conn_weird, conn->BuildConnVal(), addl, "%s", name); } void Reporter::Weird(const IPAddr& orig, const IPAddr& resp, const char* name) { + UpdateWeirdStats(name); + + if ( ! WeirdOnSamplingWhiteList(name) ) + { + if ( ! PermitFlowWeird(name, orig, resp) ) + return; + } + WeirdFlowHelper(orig, resp, "%s", name); } diff --git a/src/Reporter.h b/src/Reporter.h index 85b9a483a2..f5090ee91e 100644 --- a/src/Reporter.h +++ b/src/Reporter.h @@ -7,6 +7,10 @@ #include #include +#include +#include +#include +#include #include "util.h" #include "EventHandler.h" @@ -36,6 +40,11 @@ protected: class Reporter { public: + using IPPair = std::pair; + using WeirdCountMap = std::unordered_map; + using WeirdFlowMap = std::map; + using WeirdSet = std::unordered_set; + Reporter(); ~Reporter(); @@ -76,7 +85,6 @@ public: // that may lead to incorrectly processing a connnection. void Weird(const char* name); // Raises net_weird(). void Weird(Connection* conn, const char* name, const char* addl = ""); // Raises conn_weird(). - void Weird(Val* conn_val, const char* name, const char* addl = ""); // Raises conn_weird(). void Weird(const IPAddr& orig, const IPAddr& resp, const char* name); // Raises flow_weird(). // Syslog a message. This methods does nothing if we're running @@ -120,6 +128,30 @@ public: // Signals that we're done processing an error handler event. void EndErrorHandler() { --in_error_handler; } + /** + * Reset/cleanup state tracking for a "net" weird. + */ + void ResetNetWeird(const std::string& name); + + /** + * Reset/cleanup state tracking for a "flow" weird. + */ + void ResetFlowWeird(const IPAddr& orig, const IPAddr& resp); + + /** + * Return the total number of weirds generated (counts weirds before + * any rate-limiting occurs). + */ + uint64 GetWeirdCount() const + { return weird_count; } + + /** + * Return number of weirds generated per weird type/name (counts weirds + * before any rate-limiting occurs). + */ + const WeirdCountMap& GetWeirdsByType() const + { return weird_count_by_type; } + private: void DoLog(const char* prefix, EventHandlerPtr event, FILE* out, Connection* conn, val_list* addl, bool location, bool time, @@ -129,6 +161,11 @@ private: // contain format specifiers void WeirdHelper(EventHandlerPtr event, Val* conn_val, const char* addl, const char* fmt_name, ...) __attribute__((format(printf, 5, 6)));; void WeirdFlowHelper(const IPAddr& orig, const IPAddr& resp, const char* fmt_name, ...) __attribute__((format(printf, 4, 5)));; + void UpdateWeirdStats(const char* name); + inline bool WeirdOnSamplingWhiteList(const char* name) + { return weird_sampling_whitelist.find(name) != weird_sampling_whitelist.end(); } + bool PermitNetWeird(const char* name); + bool PermitFlowWeird(const char* name, const IPAddr& o, const IPAddr& r); int errors; bool via_events; @@ -138,6 +175,18 @@ private: bool errors_to_stderr; std::list > locations; + + uint64 weird_count; + WeirdCountMap weird_count_by_type; + + WeirdCountMap net_weird_state; + WeirdFlowMap flow_weird_state; + + WeirdSet weird_sampling_whitelist; + uint64 weird_sampling_threshold; + uint64 weird_sampling_rate; + double weird_sampling_duration; + }; extern Reporter* reporter; diff --git a/src/Timer.cc b/src/Timer.cc index 4fe998606b..101733028c 100644 --- a/src/Timer.cc +++ b/src/Timer.cc @@ -18,12 +18,14 @@ const char* TimerNames[] = { "ConnectionStatusUpdateTimer", "DNSExpireTimer", "FileAnalysisInactivityTimer", + "FlowWeirdTimer", "FragTimer", "IncrementalSendTimer", "IncrementalWriteTimer", "InterconnTimer", "IPTunnelInactivityTimer", "NetbiosExpireTimer", + "NetWeirdTimer", "NetworkTimer", "NTPExpireTimer", "ProfileTimer", diff --git a/src/Timer.h b/src/Timer.h index ea410e5c7b..8d6de857a0 100644 --- a/src/Timer.h +++ b/src/Timer.h @@ -23,12 +23,14 @@ enum TimerType { TIMER_CONN_STATUS_UPDATE, TIMER_DNS_EXPIRE, TIMER_FILE_ANALYSIS_INACTIVITY, + TIMER_FLOW_WEIRD_EXPIRE, TIMER_FRAG, TIMER_INCREMENTAL_SEND, TIMER_INCREMENTAL_WRITE, TIMER_INTERCONN, TIMER_IP_TUNNEL_INACTIVITY, TIMER_NB_EXPIRE, + TIMER_NET_WEIRD_EXPIRE, TIMER_NETWORK, TIMER_NTP_EXPIRE, TIMER_PROFILE, diff --git a/src/Val.cc b/src/Val.cc index 7879d282b2..71d38e3c63 100644 --- a/src/Val.cc +++ b/src/Val.cc @@ -3321,6 +3321,29 @@ bool VectorVal::AssignRepeat(unsigned int index, unsigned int how_many, return true; } +int VectorVal::AddTo(Val* val, int /* is_first_init */) const + { + if ( val->Type()->Tag() != TYPE_VECTOR ) + { + val->Error("not a vector"); + return 0; + } + + VectorVal* v = val->AsVectorVal(); + + if ( ! same_type(type, v->Type()) ) + { + type->Error("vector type clash", v->Type()); + return 0; + } + + auto last_idx = v->Size(); + + for ( auto i = 0u; i < Size(); ++i ) + v->Assign(last_idx++, Lookup(i)->Ref()); + + return 1; + } Val* VectorVal::Lookup(unsigned int index) const { diff --git a/src/Val.h b/src/Val.h index bb18dceb4f..f189668b28 100644 --- a/src/Val.h +++ b/src/Val.h @@ -1046,6 +1046,10 @@ public: bool AssignRepeat(unsigned int index, unsigned int how_many, Val* element); + // Add this value to the given value (if appropriate). + // Returns true if succcessful. + int AddTo(Val* v, int is_first_init) const override; + // Returns nil if no element was at that value. // Lookup does NOT grow the vector to this size. // The Val* variant assumes that the index Val* has been type-checked. diff --git a/src/analyzer/protocol/finger/Finger.cc b/src/analyzer/protocol/finger/Finger.cc index e1be27e795..0aeb544a8a 100644 --- a/src/analyzer/protocol/finger/Finger.cc +++ b/src/analyzer/protocol/finger/Finger.cc @@ -41,6 +41,9 @@ void Finger_Analyzer::DeliverStream(int length, const u_char* data, bool is_orig const char* line = (const char*) data; const char* end_of_line = line + length; + if ( length == 0 ) + return; + if ( is_orig ) { diff --git a/src/analyzer/protocol/ftp/FTP.cc b/src/analyzer/protocol/ftp/FTP.cc index 70d1be5777..b062f5aa7e 100644 --- a/src/analyzer/protocol/ftp/FTP.cc +++ b/src/analyzer/protocol/ftp/FTP.cc @@ -69,6 +69,10 @@ void FTP_Analyzer::DeliverStream(int length, const u_char* data, bool orig) const char* line = (const char*) data; const char* end_of_line = line + length; + if ( length == 0 ) + // Could emit "ftp empty request/reply" weird, but maybe not worth it. + return; + val_list* vl = new val_list; vl->append(BuildConnVal()); diff --git a/src/analyzer/protocol/ident/Ident.cc b/src/analyzer/protocol/ident/Ident.cc index 27eafb5426..423028968c 100644 --- a/src/analyzer/protocol/ident/Ident.cc +++ b/src/analyzer/protocol/ident/Ident.cc @@ -56,6 +56,9 @@ void Ident_Analyzer::DeliverStream(int length, const u_char* data, bool is_orig) if ( TCP() ) s = is_orig ? TCP()->Orig() : TCP()->Resp(); + if ( length == 0 ) + return; + if ( is_orig ) { if ( ! ident_request ) diff --git a/src/analyzer/protocol/pop3/POP3.cc b/src/analyzer/protocol/pop3/POP3.cc index b7d6aa0dcb..4f5e5819a6 100644 --- a/src/analyzer/protocol/pop3/POP3.cc +++ b/src/analyzer/protocol/pop3/POP3.cc @@ -132,6 +132,9 @@ static string trim_whitespace(const char* in) void POP3_Analyzer::ProcessRequest(int length, const char* line) { + if ( length == 0 ) + return; + if ( waitingForAuthentication ) { ++authLines; diff --git a/src/analyzer/protocol/smtp/SMTP.cc b/src/analyzer/protocol/smtp/SMTP.cc index 8296f83cb3..7077ee8bfb 100644 --- a/src/analyzer/protocol/smtp/SMTP.cc +++ b/src/analyzer/protocol/smtp/SMTP.cc @@ -269,7 +269,9 @@ void SMTP_Analyzer::ProcessLine(int length, const char* line, bool orig) if ( smtp_request ) { int data_len = end_of_line - line; - RequestEvent(cmd_len, cmd, data_len, line); + + if ( cmd_len > 0 || data_len > 0 ) + RequestEvent(cmd_len, cmd, data_len, line); } if ( cmd_code != SMTP_CMD_END_OF_DATA ) @@ -379,7 +381,17 @@ void SMTP_Analyzer::NewCmd(const int cmd_code) if ( first_cmd < 0 ) first_cmd = cmd_code; else + { + auto constexpr max_pending_cmd_q_size = 1000; + + if ( pending_cmd_q.size() == max_pending_cmd_q_size ) + { + Weird("smtp_excessive_pending_cmds"); + pending_cmd_q.clear(); + } + pending_cmd_q.push_back(cmd_code); + } } else first_cmd = cmd_code; @@ -805,12 +817,22 @@ void SMTP_Analyzer::UpdateState(const int cmd_code, const int reply_code, bool o #endif } +static bool istrequal(const char* s, const char* cmd, int s_len) + { + auto cmd_len = strlen(cmd); + + if ( cmd_len != s_len ) + return false; + + return strncasecmp(s, cmd, s_len) == 0; + } + void SMTP_Analyzer::ProcessExtension(int ext_len, const char* ext) { if ( ! ext ) return; - if ( ! strncasecmp(ext, "PIPELINING", ext_len) ) + if ( istrequal(ext, "PIPELINING", ext_len) ) pipelining = 1; } @@ -820,11 +842,11 @@ int SMTP_Analyzer::ParseCmd(int cmd_len, const char* cmd) return -1; // special case because we cannot define our usual macros with "-" - if ( strncmp(cmd, "X-ANONYMOUSTLS", cmd_len) == 0 ) + if ( istrequal(cmd, "X-ANONYMOUSTLS", cmd_len) ) return SMTP_CMD_X_ANONYMOUSTLS; for ( int code = SMTP_CMD_EHLO; code < SMTP_CMD_LAST; ++code ) - if ( ! strncasecmp(cmd, smtp_cmd_word[code - SMTP_CMD_EHLO], cmd_len) ) + if ( istrequal(cmd, smtp_cmd_word[code - SMTP_CMD_EHLO], cmd_len) ) return code; return -1; diff --git a/src/reporter.bif b/src/reporter.bif index 3cdefc4da4..a6463f6226 100644 --- a/src/reporter.bif +++ b/src/reporter.bif @@ -71,3 +71,44 @@ function Reporter::fatal%(msg: string%): bool reporter->PopLocation(); return new Val(1, TYPE_BOOL); %} + +## Generates a "net" weird. +## +## name: the name of the weird. +## +## Returns: Always true. +function Reporter::net_weird%(name: string%): bool + %{ + reporter->Weird(name->CheckString()); + return new Val(1, TYPE_BOOL); + %} + +## Generates a "flow" weird. +## +## name: the name of the weird. +## +## orig: the originator host associated with the weird. +## +## resp: the responder host associated with the weird. +## +## Returns: Always true. +function Reporter::flow_weird%(name: string, orig: addr, resp: addr%): bool + %{ + reporter->Weird(orig->AsAddr(), resp->AsAddr(), name->CheckString()); + return new Val(1, TYPE_BOOL); + %} + +## Generates a "conn" weird. +## +## name: the name of the weird. +## +## c: the connection associated with the weird. +## +## addl: additional information to accompany the weird. +## +## Returns: Always true. +function Reporter::conn_weird%(name: string, c: connection, addl: string &default=""%): bool + %{ + reporter->Weird(c, name->CheckString(), addl->CheckString()); + return new Val(1, TYPE_BOOL); + %} diff --git a/src/stats.bif b/src/stats.bif index 2a525a1790..e3ce6f0245 100644 --- a/src/stats.bif +++ b/src/stats.bif @@ -16,6 +16,7 @@ RecordType* ThreadStats; RecordType* TimerStats; RecordType* FileAnalysisStats; RecordType* BrokerStats; +RecordType* ReporterStats; %%} ## Returns packet capture statistics. Statistics include the number of @@ -35,6 +36,7 @@ RecordType* BrokerStats; ## get_thread_stats ## get_timer_stats ## get_broker_stats +## get_reporter_stats function get_net_stats%(%): NetStats %{ uint64 recv = 0; @@ -83,6 +85,7 @@ function get_net_stats%(%): NetStats ## get_thread_stats ## get_timer_stats ## get_broker_stats +## get_reporter_stats function get_conn_stats%(%): ConnStats %{ RecordVal* r = new RecordVal(ConnStats); @@ -133,6 +136,7 @@ function get_conn_stats%(%): ConnStats ## get_thread_stats ## get_timer_stats ## get_broker_stats +## get_reporter_stats function get_proc_stats%(%): ProcStats %{ struct rusage ru; @@ -189,6 +193,7 @@ function get_proc_stats%(%): ProcStats ## get_thread_stats ## get_timer_stats ## get_broker_stats +## get_reporter_stats function get_event_stats%(%): EventStats %{ RecordVal* r = new RecordVal(EventStats); @@ -215,6 +220,7 @@ function get_event_stats%(%): EventStats ## get_thread_stats ## get_timer_stats ## get_broker_stats +## get_reporter_stats function get_reassembler_stats%(%): ReassemblerStats %{ RecordVal* r = new RecordVal(ReassemblerStats); @@ -243,6 +249,7 @@ function get_reassembler_stats%(%): ReassemblerStats ## get_thread_stats ## get_timer_stats ## get_broker_stats +## get_reporter_stats function get_dns_stats%(%): DNSStats %{ RecordVal* r = new RecordVal(DNSStats); @@ -276,6 +283,7 @@ function get_dns_stats%(%): DNSStats ## get_reassembler_stats ## get_thread_stats ## get_broker_stats +## get_reporter_stats function get_timer_stats%(%): TimerStats %{ RecordVal* r = new RecordVal(TimerStats); @@ -303,6 +311,7 @@ function get_timer_stats%(%): TimerStats ## get_thread_stats ## get_timer_stats ## get_broker_stats +## get_reporter_stats function get_file_analysis_stats%(%): FileAnalysisStats %{ RecordVal* r = new RecordVal(FileAnalysisStats); @@ -330,6 +339,7 @@ function get_file_analysis_stats%(%): FileAnalysisStats ## get_reassembler_stats ## get_timer_stats ## get_broker_stats +## get_reporter_stats function get_thread_stats%(%): ThreadStats %{ RecordVal* r = new RecordVal(ThreadStats); @@ -355,6 +365,7 @@ function get_thread_stats%(%): ThreadStats ## get_thread_stats ## get_timer_stats ## get_broker_stats +## get_reporter_stats function get_gap_stats%(%): GapStats %{ RecordVal* r = new RecordVal(GapStats); @@ -386,6 +397,7 @@ function get_gap_stats%(%): GapStats ## get_thread_stats ## get_timer_stats ## get_broker_stats +## get_reporter_stats function get_matcher_stats%(%): MatcherStats %{ RecordVal* r = new RecordVal(MatcherStats); @@ -423,6 +435,7 @@ function get_matcher_stats%(%): MatcherStats ## get_thread_stats ## get_timer_stats ## get_broker_stats +## get_reporter_stats function get_broker_stats%(%): BrokerStats %{ RecordVal* r = new RecordVal(BrokerStats); @@ -441,3 +454,39 @@ function get_broker_stats%(%): BrokerStats return r; %} + +## Returns statistics about reporter messages and weirds. +## +## Returns: A record with reporter statistics. +## +## .. bro:see:: get_conn_stats +## get_dns_stats +## get_event_stats +## get_file_analysis_stats +## get_gap_stats +## get_matcher_stats +## get_net_stats +## get_proc_stats +## get_reassembler_stats +## get_thread_stats +## get_timer_stats +## get_broker_stats +function get_reporter_stats%(%): ReporterStats + %{ + RecordVal* r = new RecordVal(ReporterStats); + int n = 0; + + TableVal* weirds_by_type = new TableVal(internal_type("table_string_of_count")->AsTableType()); + + for ( auto& kv : reporter->GetWeirdsByType() ) + { + Val* weird = new StringVal(kv.first); + weirds_by_type->Assign(weird, new Val(kv.second, TYPE_COUNT)); + Unref(weird); + } + + r->Assign(n++, new Val(reporter->GetWeirdCount(), TYPE_COUNT)); + r->Assign(n++, weirds_by_type); + + return r; + %} diff --git a/testing/btest/Baseline/core.reporter-weird-sampling/output b/testing/btest/Baseline/core.reporter-weird-sampling/output new file mode 100644 index 0000000000..b9be6227dd --- /dev/null +++ b/testing/btest/Baseline/core.reporter-weird-sampling/output @@ -0,0 +1,351 @@ +net_weird, my_net_weird +flow_weird, my_flow_weird +conn_weird, my_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, my_net_weird +flow_weird, my_flow_weird +conn_weird, my_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, my_net_weird +flow_weird, my_flow_weird +conn_weird, my_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, my_net_weird +flow_weird, my_flow_weird +conn_weird, my_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, my_net_weird +flow_weird, my_flow_weird +conn_weird, my_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, my_net_weird +flow_weird, my_flow_weird +conn_weird, my_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, my_net_weird +flow_weird, my_flow_weird +conn_weird, my_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, my_net_weird +flow_weird, my_flow_weird +conn_weird, my_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, my_net_weird +flow_weird, my_flow_weird +conn_weird, my_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, my_net_weird +flow_weird, my_flow_weird +conn_weird, my_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, my_net_weird +flow_weird, my_flow_weird +conn_weird, my_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, my_net_weird +flow_weird, my_flow_weird +conn_weird, my_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, my_net_weird +flow_weird, my_flow_weird +conn_weird, my_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, my_net_weird +flow_weird, my_flow_weird +conn_weird, my_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, my_net_weird +flow_weird, my_flow_weird +conn_weird, my_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, my_net_weird +flow_weird, my_flow_weird +conn_weird, my_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, my_net_weird +flow_weird, my_flow_weird +conn_weird, my_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, my_net_weird +flow_weird, my_flow_weird +conn_weird, my_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, my_net_weird +flow_weird, my_flow_weird +conn_weird, my_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, my_net_weird +flow_weird, my_flow_weird +conn_weird, my_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, my_net_weird +flow_weird, my_flow_weird +conn_weird, my_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, my_net_weird +flow_weird, my_flow_weird +conn_weird, my_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, my_net_weird +flow_weird, my_flow_weird +conn_weird, my_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, my_net_weird +flow_weird, my_flow_weird +conn_weird, my_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, my_net_weird +flow_weird, my_flow_weird +conn_weird, my_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, my_net_weird +flow_weird, my_flow_weird +conn_weird, my_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, my_net_weird +flow_weird, my_flow_weird +conn_weird, my_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird diff --git a/testing/btest/Baseline/coverage.bare-mode-errors/errors b/testing/btest/Baseline/coverage.bare-mode-errors/errors index e69de29bb2..8664c8a4d1 100644 --- a/testing/btest/Baseline/coverage.bare-mode-errors/errors +++ b/testing/btest/Baseline/coverage.bare-mode-errors/errors @@ -0,0 +1,16 @@ +warning in /home/jon/projects/bro/bro/testing/btest/../../scripts/policy/protocols/dhcp/deprecated_events.bro, line 245: deprecated (dhcp_discover) +warning in /home/jon/projects/bro/bro/testing/btest/../../scripts/policy/protocols/dhcp/deprecated_events.bro, line 248: deprecated (dhcp_offer) +warning in /home/jon/projects/bro/bro/testing/btest/../../scripts/policy/protocols/dhcp/deprecated_events.bro, line 251: deprecated (dhcp_request) +warning in /home/jon/projects/bro/bro/testing/btest/../../scripts/policy/protocols/dhcp/deprecated_events.bro, line 254: deprecated (dhcp_decline) +warning in /home/jon/projects/bro/bro/testing/btest/../../scripts/policy/protocols/dhcp/deprecated_events.bro, line 257: deprecated (dhcp_ack) +warning in /home/jon/projects/bro/bro/testing/btest/../../scripts/policy/protocols/dhcp/deprecated_events.bro, line 260: deprecated (dhcp_nak) +warning in /home/jon/projects/bro/bro/testing/btest/../../scripts/policy/protocols/dhcp/deprecated_events.bro, line 263: deprecated (dhcp_release) +warning in /home/jon/projects/bro/bro/testing/btest/../../scripts/policy/protocols/dhcp/deprecated_events.bro, line 266: deprecated (dhcp_inform) +warning in /home/jon/projects/bro/bro/scripts/policy/protocols/dhcp/deprecated_events.bro, line 245: deprecated (dhcp_discover) +warning in /home/jon/projects/bro/bro/scripts/policy/protocols/dhcp/deprecated_events.bro, line 248: deprecated (dhcp_offer) +warning in /home/jon/projects/bro/bro/scripts/policy/protocols/dhcp/deprecated_events.bro, line 251: deprecated (dhcp_request) +warning in /home/jon/projects/bro/bro/scripts/policy/protocols/dhcp/deprecated_events.bro, line 254: deprecated (dhcp_decline) +warning in /home/jon/projects/bro/bro/scripts/policy/protocols/dhcp/deprecated_events.bro, line 257: deprecated (dhcp_ack) +warning in /home/jon/projects/bro/bro/scripts/policy/protocols/dhcp/deprecated_events.bro, line 260: deprecated (dhcp_nak) +warning in /home/jon/projects/bro/bro/scripts/policy/protocols/dhcp/deprecated_events.bro, line 263: deprecated (dhcp_release) +warning in /home/jon/projects/bro/bro/scripts/policy/protocols/dhcp/deprecated_events.bro, line 266: deprecated (dhcp_inform) diff --git a/testing/btest/Baseline/coverage.find-bro-logs/out b/testing/btest/Baseline/coverage.find-bro-logs/out index 70a635b24f..cac0156707 100644 --- a/testing/btest/Baseline/coverage.find-bro-logs/out +++ b/testing/btest/Baseline/coverage.find-bro-logs/out @@ -55,4 +55,5 @@ traceroute tunnel unified2 weird +weird_stats x509 diff --git a/testing/btest/Baseline/doc.broxygen.all_scripts/.stderr b/testing/btest/Baseline/doc.broxygen.all_scripts/.stderr index 2bc5465045..34c0d82da2 100644 --- a/testing/btest/Baseline/doc.broxygen.all_scripts/.stderr +++ b/testing/btest/Baseline/doc.broxygen.all_scripts/.stderr @@ -1,2 +1,10 @@ +warning in /home/jon/projects/bro/bro/scripts/policy/protocols/dhcp/deprecated_events.bro, line 245: deprecated (dhcp_discover) +warning in /home/jon/projects/bro/bro/scripts/policy/protocols/dhcp/deprecated_events.bro, line 248: deprecated (dhcp_offer) +warning in /home/jon/projects/bro/bro/scripts/policy/protocols/dhcp/deprecated_events.bro, line 251: deprecated (dhcp_request) +warning in /home/jon/projects/bro/bro/scripts/policy/protocols/dhcp/deprecated_events.bro, line 254: deprecated (dhcp_decline) +warning in /home/jon/projects/bro/bro/scripts/policy/protocols/dhcp/deprecated_events.bro, line 257: deprecated (dhcp_ack) +warning in /home/jon/projects/bro/bro/scripts/policy/protocols/dhcp/deprecated_events.bro, line 260: deprecated (dhcp_nak) +warning in /home/jon/projects/bro/bro/scripts/policy/protocols/dhcp/deprecated_events.bro, line 263: deprecated (dhcp_release) +warning in /home/jon/projects/bro/bro/scripts/policy/protocols/dhcp/deprecated_events.bro, line 266: deprecated (dhcp_inform) error in /home/jon/projects/bro/bro/scripts/policy/frameworks/control/controller.bro, line 22: The '' control command is unknown. , line 1: received termination signal diff --git a/testing/btest/Baseline/doc.sphinx.include-scripts_policy_frameworks_files_detect-MHR_bro/output b/testing/btest/Baseline/doc.sphinx.include-scripts_policy_frameworks_files_detect-MHR_bro/output index 03ba9cb3cd..8efd4099ad 100644 --- a/testing/btest/Baseline/doc.sphinx.include-scripts_policy_frameworks_files_detect-MHR_bro/output +++ b/testing/btest/Baseline/doc.sphinx.include-scripts_policy_frameworks_files_detect-MHR_bro/output @@ -36,7 +36,7 @@ export { ## A/V engines. Team Cymru returns a percentage to indicate how ## many A/V engines flagged the sample as malicious. This threshold ## allows you to require a minimum detection rate. - const notice_threshold = 10 &redef; + option notice_threshold = 10; } function do_mhr_lookup(hash: string, fi: Notice::FileInfo) diff --git a/testing/btest/Baseline/doc.sphinx.include-scripts_policy_frameworks_files_detect-MHR_bro@3/output b/testing/btest/Baseline/doc.sphinx.include-scripts_policy_frameworks_files_detect-MHR_bro@3/output index 30c6b1040e..f1d91b8392 100644 --- a/testing/btest/Baseline/doc.sphinx.include-scripts_policy_frameworks_files_detect-MHR_bro@3/output +++ b/testing/btest/Baseline/doc.sphinx.include-scripts_policy_frameworks_files_detect-MHR_bro@3/output @@ -27,5 +27,5 @@ export { ## A/V engines. Team Cymru returns a percentage to indicate how ## many A/V engines flagged the sample as malicious. This threshold ## allows you to require a minimum detection rate. - const notice_threshold = 10 &redef; + option notice_threshold = 10; } diff --git a/testing/btest/Baseline/language.redef-vector/out b/testing/btest/Baseline/language.redef-vector/out new file mode 100644 index 0000000000..73a5ef460d --- /dev/null +++ b/testing/btest/Baseline/language.redef-vector/out @@ -0,0 +1,3 @@ +[testing, blah, foo, foo, testing] +[one, two, three] +[a, b, c, one, two, three, a, b, c, abc, d] diff --git a/testing/btest/Baseline/plugins.hooks/output b/testing/btest/Baseline/plugins.hooks/output index 20e01df16f..4a456b3f8a 100644 --- a/testing/btest/Baseline/plugins.hooks/output +++ b/testing/btest/Baseline/plugins.hooks/output @@ -274,7 +274,7 @@ 0.000000 MetaHookPost CallFunction(Log::__create_stream, , (Weird::LOG, [columns=, ev=Weird::log_weird, path=weird])) -> 0.000000 MetaHookPost CallFunction(Log::__create_stream, , (X509::LOG, [columns=, ev=X509::log_x509, path=x509])) -> 0.000000 MetaHookPost CallFunction(Log::__create_stream, , (mysql::LOG, [columns=, ev=MySQL::log_mysql, path=mysql])) -> -0.000000 MetaHookPost CallFunction(Log::__write, , (PacketFilter::LOG, [ts=1534522064.090237, node=bro, filter=ip or not ip, init=T, success=T])) -> +0.000000 MetaHookPost CallFunction(Log::__write, , (PacketFilter::LOG, [ts=1534783830.174975, node=bro, filter=ip or not ip, init=T, success=T])) -> 0.000000 MetaHookPost CallFunction(Log::add_default_filter, , (Broker::LOG)) -> 0.000000 MetaHookPost CallFunction(Log::add_default_filter, , (Cluster::LOG)) -> 0.000000 MetaHookPost CallFunction(Log::add_default_filter, , (Config::LOG)) -> @@ -459,10 +459,21 @@ 0.000000 MetaHookPost CallFunction(Log::create_stream, , (Weird::LOG, [columns=, ev=Weird::log_weird, path=weird])) -> 0.000000 MetaHookPost CallFunction(Log::create_stream, , (X509::LOG, [columns=, ev=X509::log_x509, path=x509])) -> 0.000000 MetaHookPost CallFunction(Log::create_stream, , (mysql::LOG, [columns=, ev=MySQL::log_mysql, path=mysql])) -> -0.000000 MetaHookPost CallFunction(Log::write, , (PacketFilter::LOG, [ts=1534522064.090237, node=bro, filter=ip or not ip, init=T, success=T])) -> +0.000000 MetaHookPost CallFunction(Log::write, , (PacketFilter::LOG, [ts=1534783830.174975, node=bro, filter=ip or not ip, init=T, success=T])) -> 0.000000 MetaHookPost CallFunction(NetControl::check_plugins, , ()) -> 0.000000 MetaHookPost CallFunction(NetControl::init, , ()) -> 0.000000 MetaHookPost CallFunction(Notice::want_pp, , ()) -> +0.000000 MetaHookPost CallFunction(Option::set_change_handler, , (DHCP::max_txid_watch_time, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100)) -> +0.000000 MetaHookPost CallFunction(Option::set_change_handler, , (FTP::guest_ids, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100)) -> +0.000000 MetaHookPost CallFunction(Option::set_change_handler, , (FTP::logged_commands, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100)) -> +0.000000 MetaHookPost CallFunction(Option::set_change_handler, , (HTTP::http_methods, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100)) -> +0.000000 MetaHookPost CallFunction(Option::set_change_handler, , (KRB::ignored_errors, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100)) -> +0.000000 MetaHookPost CallFunction(Option::set_change_handler, , (NetControl::catch_release_warn_blocked_ip_encountered, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100)) -> +0.000000 MetaHookPost CallFunction(Option::set_change_handler, , (Notice::mail_from, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100)) -> +0.000000 MetaHookPost CallFunction(Option::set_change_handler, , (Notice::mail_subject_prefix, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100)) -> +0.000000 MetaHookPost CallFunction(Option::set_change_handler, , (Notice::reply_to, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100)) -> +0.000000 MetaHookPost CallFunction(Option::set_change_handler, , (SIP::sip_methods, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100)) -> +0.000000 MetaHookPost CallFunction(Option::set_change_handler, , (SMB::logged_file_actions, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100)) -> 0.000000 MetaHookPost CallFunction(PacketFilter::build, , ()) -> 0.000000 MetaHookPost CallFunction(PacketFilter::combine_filters, , (ip or not ip, and, )) -> 0.000000 MetaHookPost CallFunction(PacketFilter::install, , ()) -> @@ -1097,7 +1108,7 @@ 0.000000 MetaHookPre CallFunction(Log::__create_stream, , (Weird::LOG, [columns=, ev=Weird::log_weird, path=weird])) 0.000000 MetaHookPre CallFunction(Log::__create_stream, , (X509::LOG, [columns=, ev=X509::log_x509, path=x509])) 0.000000 MetaHookPre CallFunction(Log::__create_stream, , (mysql::LOG, [columns=, ev=MySQL::log_mysql, path=mysql])) -0.000000 MetaHookPre CallFunction(Log::__write, , (PacketFilter::LOG, [ts=1534522064.090237, node=bro, filter=ip or not ip, init=T, success=T])) +0.000000 MetaHookPre CallFunction(Log::__write, , (PacketFilter::LOG, [ts=1534783830.174975, node=bro, filter=ip or not ip, init=T, success=T])) 0.000000 MetaHookPre CallFunction(Log::add_default_filter, , (Broker::LOG)) 0.000000 MetaHookPre CallFunction(Log::add_default_filter, , (Cluster::LOG)) 0.000000 MetaHookPre CallFunction(Log::add_default_filter, , (Config::LOG)) @@ -1282,10 +1293,21 @@ 0.000000 MetaHookPre CallFunction(Log::create_stream, , (Weird::LOG, [columns=, ev=Weird::log_weird, path=weird])) 0.000000 MetaHookPre CallFunction(Log::create_stream, , (X509::LOG, [columns=, ev=X509::log_x509, path=x509])) 0.000000 MetaHookPre CallFunction(Log::create_stream, , (mysql::LOG, [columns=, ev=MySQL::log_mysql, path=mysql])) -0.000000 MetaHookPre CallFunction(Log::write, , (PacketFilter::LOG, [ts=1534522064.090237, node=bro, filter=ip or not ip, init=T, success=T])) +0.000000 MetaHookPre CallFunction(Log::write, , (PacketFilter::LOG, [ts=1534783830.174975, node=bro, filter=ip or not ip, init=T, success=T])) 0.000000 MetaHookPre CallFunction(NetControl::check_plugins, , ()) 0.000000 MetaHookPre CallFunction(NetControl::init, , ()) 0.000000 MetaHookPre CallFunction(Notice::want_pp, , ()) +0.000000 MetaHookPre CallFunction(Option::set_change_handler, , (DHCP::max_txid_watch_time, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100)) +0.000000 MetaHookPre CallFunction(Option::set_change_handler, , (FTP::guest_ids, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100)) +0.000000 MetaHookPre CallFunction(Option::set_change_handler, , (FTP::logged_commands, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100)) +0.000000 MetaHookPre CallFunction(Option::set_change_handler, , (HTTP::http_methods, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100)) +0.000000 MetaHookPre CallFunction(Option::set_change_handler, , (KRB::ignored_errors, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100)) +0.000000 MetaHookPre CallFunction(Option::set_change_handler, , (NetControl::catch_release_warn_blocked_ip_encountered, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100)) +0.000000 MetaHookPre CallFunction(Option::set_change_handler, , (Notice::mail_from, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100)) +0.000000 MetaHookPre CallFunction(Option::set_change_handler, , (Notice::mail_subject_prefix, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100)) +0.000000 MetaHookPre CallFunction(Option::set_change_handler, , (Notice::reply_to, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100)) +0.000000 MetaHookPre CallFunction(Option::set_change_handler, , (SIP::sip_methods, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100)) +0.000000 MetaHookPre CallFunction(Option::set_change_handler, , (SMB::logged_file_actions, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100)) 0.000000 MetaHookPre CallFunction(PacketFilter::build, , ()) 0.000000 MetaHookPre CallFunction(PacketFilter::combine_filters, , (ip or not ip, and, )) 0.000000 MetaHookPre CallFunction(PacketFilter::install, , ()) @@ -1919,7 +1941,7 @@ 0.000000 | HookCallFunction Log::__create_stream(Weird::LOG, [columns=, ev=Weird::log_weird, path=weird]) 0.000000 | HookCallFunction Log::__create_stream(X509::LOG, [columns=, ev=X509::log_x509, path=x509]) 0.000000 | HookCallFunction Log::__create_stream(mysql::LOG, [columns=, ev=MySQL::log_mysql, path=mysql]) -0.000000 | HookCallFunction Log::__write(PacketFilter::LOG, [ts=1534522064.090237, node=bro, filter=ip or not ip, init=T, success=T]) +0.000000 | HookCallFunction Log::__write(PacketFilter::LOG, [ts=1534783830.174975, node=bro, filter=ip or not ip, init=T, success=T]) 0.000000 | HookCallFunction Log::add_default_filter(Broker::LOG) 0.000000 | HookCallFunction Log::add_default_filter(Cluster::LOG) 0.000000 | HookCallFunction Log::add_default_filter(Config::LOG) @@ -2104,10 +2126,21 @@ 0.000000 | HookCallFunction Log::create_stream(Weird::LOG, [columns=, ev=Weird::log_weird, path=weird]) 0.000000 | HookCallFunction Log::create_stream(X509::LOG, [columns=, ev=X509::log_x509, path=x509]) 0.000000 | HookCallFunction Log::create_stream(mysql::LOG, [columns=, ev=MySQL::log_mysql, path=mysql]) -0.000000 | HookCallFunction Log::write(PacketFilter::LOG, [ts=1534522064.090237, node=bro, filter=ip or not ip, init=T, success=T]) +0.000000 | HookCallFunction Log::write(PacketFilter::LOG, [ts=1534783830.174975, node=bro, filter=ip or not ip, init=T, success=T]) 0.000000 | HookCallFunction NetControl::check_plugins() 0.000000 | HookCallFunction NetControl::init() 0.000000 | HookCallFunction Notice::want_pp() +0.000000 | HookCallFunction Option::set_change_handler(DHCP::max_txid_watch_time, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100) +0.000000 | HookCallFunction Option::set_change_handler(FTP::guest_ids, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100) +0.000000 | HookCallFunction Option::set_change_handler(FTP::logged_commands, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100) +0.000000 | HookCallFunction Option::set_change_handler(HTTP::http_methods, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100) +0.000000 | HookCallFunction Option::set_change_handler(KRB::ignored_errors, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100) +0.000000 | HookCallFunction Option::set_change_handler(NetControl::catch_release_warn_blocked_ip_encountered, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100) +0.000000 | HookCallFunction Option::set_change_handler(Notice::mail_from, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100) +0.000000 | HookCallFunction Option::set_change_handler(Notice::mail_subject_prefix, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100) +0.000000 | HookCallFunction Option::set_change_handler(Notice::reply_to, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100) +0.000000 | HookCallFunction Option::set_change_handler(SIP::sip_methods, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100) +0.000000 | HookCallFunction Option::set_change_handler(SMB::logged_file_actions, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100) 0.000000 | HookCallFunction PacketFilter::build() 0.000000 | HookCallFunction PacketFilter::combine_filters(ip or not ip, and, ) 0.000000 | HookCallFunction PacketFilter::install() @@ -2462,7 +2495,7 @@ 0.000000 | HookLoadFile base<...>/x509 0.000000 | HookLoadFile base<...>/xmpp 0.000000 | HookLogInit packet_filter 1/1 {ts (time), node (string), filter (string), init (bool), success (bool)} -0.000000 | HookLogWrite packet_filter [ts=1534522064.090237, node=bro, filter=ip or not ip, init=T, success=T] +0.000000 | HookLogWrite packet_filter [ts=1534783830.174975, node=bro, filter=ip or not ip, init=T, success=T] 0.000000 | HookQueueEvent NetControl::init() 0.000000 | HookQueueEvent bro_init() 0.000000 | HookQueueEvent filter_change_tracking() diff --git a/testing/btest/Baseline/scripts.policy.misc.weird-stats-cluster/manager-1.weird_stats.log b/testing/btest/Baseline/scripts.policy.misc.weird-stats-cluster/manager-1.weird_stats.log new file mode 100644 index 0000000000..001da38e49 --- /dev/null +++ b/testing/btest/Baseline/scripts.policy.misc.weird-stats-cluster/manager-1.weird_stats.log @@ -0,0 +1,13 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path weird_stats +#open 2018-07-26-23-11-27 +#fields ts name num_seen +#types time string count +1532646687.827249 weird3 1 +1532646687.827249 weird2 1000 +1532646687.827249 weird1 2000 +1532646692.877464 weird1 2 +#close 2018-07-26-23-11-34 diff --git a/testing/btest/Baseline/scripts.policy.misc.weird-stats/bro.weird_stats.log b/testing/btest/Baseline/scripts.policy.misc.weird-stats/bro.weird_stats.log new file mode 100644 index 0000000000..e44145ed86 --- /dev/null +++ b/testing/btest/Baseline/scripts.policy.misc.weird-stats/bro.weird_stats.log @@ -0,0 +1,12 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path weird_stats +#open 2018-07-27-00-20-35 +#fields ts name num_seen +#types time string count +1532650834.978616 my_weird 1000 +1532650840.011592 my_weird 2000 +1532650845.043367 my_weird 10 +#close 2018-07-27-00-20-47 diff --git a/testing/btest/core/reporter-weird-sampling.bro b/testing/btest/core/reporter-weird-sampling.bro new file mode 100644 index 0000000000..d9d99681c4 --- /dev/null +++ b/testing/btest/core/reporter-weird-sampling.bro @@ -0,0 +1,55 @@ +# @TEST-EXEC: bro -b -r $TRACES/http/bro.org.pcap %INPUT >output +# @TEST-EXEC: btest-diff output + +redef Weird::sampling_duration = 5sec; +redef Weird::sampling_threshold = 10; +redef Weird::sampling_rate = 10; +redef Weird::sampling_whitelist = set("whitelisted_net_weird", + "whitelisted_flow_weird", + "whitelisted_conn_weird"); + +event conn_weird(name: string, c: connection, addl: string) + { + print "conn_weird", name; + } + +event flow_weird(name: string, src: addr, dst: addr) + { + print "flow_weird", name; + } + +event net_weird(name: string) + { + print "net_weird", name; + } + +event gen_weirds(c: connection) + { + local num = 30; + + while ( num != 0 ) + { + Reporter::net_weird("my_net_weird"); + Reporter::flow_weird("my_flow_weird", c$id$orig_h, c$id$resp_h); + Reporter::conn_weird("my_conn_weird", c); + + Reporter::net_weird("whitelisted_net_weird"); + Reporter::flow_weird("whitelisted_flow_weird", c$id$orig_h, c$id$resp_h); + Reporter::conn_weird("whitelisted_conn_weird", c); + --num; + } + } + +global did_one_connection = F; + +event new_connection(c: connection) + { + if ( did_one_connection ) + return; + + did_one_connection = T; + event gen_weirds(c); # should permit 10 + 2 of each "my" weird + schedule 2sec { gen_weirds(c) }; # should permit 3 of each "my" weird + schedule 7sec { gen_weirds(c) }; # should permit 10 + 2 of each "my" weird + # Total of 27 "my" weirds of each type and 90 of each "whitelisted" type + } diff --git a/testing/btest/coverage/bare-mode-errors.test b/testing/btest/coverage/bare-mode-errors.test index 27e10373ea..24f3aced9d 100644 --- a/testing/btest/coverage/bare-mode-errors.test +++ b/testing/btest/coverage/bare-mode-errors.test @@ -6,4 +6,4 @@ # # @TEST-EXEC: test -d $DIST/scripts # @TEST-EXEC: for script in `find $DIST/scripts/ -name \*\.bro`; do bro -b --parse-only $script >>errors 2>&1; done -# @TEST-EXEC: btest-diff errors +# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff errors diff --git a/testing/btest/doc/sphinx/include-scripts_policy_frameworks_files_detect-MHR_bro.btest b/testing/btest/doc/sphinx/include-scripts_policy_frameworks_files_detect-MHR_bro.btest index 03ba9cb3cd..8efd4099ad 100644 --- a/testing/btest/doc/sphinx/include-scripts_policy_frameworks_files_detect-MHR_bro.btest +++ b/testing/btest/doc/sphinx/include-scripts_policy_frameworks_files_detect-MHR_bro.btest @@ -36,7 +36,7 @@ export { ## A/V engines. Team Cymru returns a percentage to indicate how ## many A/V engines flagged the sample as malicious. This threshold ## allows you to require a minimum detection rate. - const notice_threshold = 10 &redef; + option notice_threshold = 10; } function do_mhr_lookup(hash: string, fi: Notice::FileInfo) diff --git a/testing/btest/doc/sphinx/include-scripts_policy_frameworks_files_detect-MHR_bro@3.btest b/testing/btest/doc/sphinx/include-scripts_policy_frameworks_files_detect-MHR_bro@3.btest index 30c6b1040e..f1d91b8392 100644 --- a/testing/btest/doc/sphinx/include-scripts_policy_frameworks_files_detect-MHR_bro@3.btest +++ b/testing/btest/doc/sphinx/include-scripts_policy_frameworks_files_detect-MHR_bro@3.btest @@ -27,5 +27,5 @@ export { ## A/V engines. Team Cymru returns a percentage to indicate how ## many A/V engines flagged the sample as malicious. This threshold ## allows you to require a minimum detection rate. - const notice_threshold = 10 &redef; + option notice_threshold = 10; } diff --git a/testing/btest/language/redef-vector.bro b/testing/btest/language/redef-vector.bro new file mode 100644 index 0000000000..26dc2109ba --- /dev/null +++ b/testing/btest/language/redef-vector.bro @@ -0,0 +1,18 @@ +# @TEST-EXEC: bro -b %INPUT >out +# @TEST-EXEC: btest-diff out + +const foo: vector of string &redef; +redef foo += { "testing", "blah", "foo", "foo", "testing" }; + +const bar: vector of string = vector() &redef; +redef bar += { "one", "two", "three" }; + +const baz: vector of string = vector("a", "b", "c") &redef; +redef baz += { "one", "two", "three" }; +redef baz += { "a", "b", "c" }; +const d = "d"; +redef baz += { "a" + "b" + "c", d }; + +print foo; +print bar; +print baz; diff --git a/testing/btest/scripts/policy/misc/weird-stats-cluster.bro b/testing/btest/scripts/policy/misc/weird-stats-cluster.bro new file mode 100644 index 0000000000..492b682c4d --- /dev/null +++ b/testing/btest/scripts/policy/misc/weird-stats-cluster.bro @@ -0,0 +1,93 @@ +# @TEST-SERIALIZE: comm +# +# @TEST-EXEC: btest-bg-run manager-1 BROPATH=$BROPATH:.. CLUSTER_NODE=manager-1 bro %INPUT +# @TEST-EXEC: btest-bg-run worker-1 BROPATH=$BROPATH:.. CLUSTER_NODE=worker-1 bro %INPUT +# @TEST-EXEC: btest-bg-run worker-2 BROPATH=$BROPATH:.. CLUSTER_NODE=worker-2 bro %INPUT +# @TEST-EXEC: btest-bg-wait 20 + +# @TEST-EXEC: btest-diff manager-1/weird_stats.log + +@TEST-START-FILE cluster-layout.bro +redef Cluster::nodes = { + ["manager-1"] = [$node_type=Cluster::MANAGER, $ip=127.0.0.1, $p=37757/tcp], + ["worker-1"] = [$node_type=Cluster::WORKER, $ip=127.0.0.1, $p=37760/tcp, $manager="manager-1", $interface="eth0"], + ["worker-2"] = [$node_type=Cluster::WORKER, $ip=127.0.0.1, $p=37761/tcp, $manager="manager-1", $interface="eth1"], +}; +@TEST-END-FILE + +@load misc/weird-stats + +redef Cluster::retry_interval = 1sec; +redef Broker::default_listen_retry = 1sec; +redef Broker::default_connect_retry = 1sec; + +redef Log::enable_local_logging = T; +redef Log::default_rotation_interval = 0secs; +redef WeirdStats::weird_stat_interval = 5secs; + +event terminate_me() + { + terminate(); + } + +event Broker::peer_lost(endpoint: Broker::EndpointInfo, msg: string) + { + terminate(); + } + +event ready_again() + { + Reporter::net_weird("weird1"); + + if ( Cluster::node == "worker-2" ) + { + schedule 5secs { terminate_me() }; + } + } + +event ready_for_data() + { + local n = 0; + + if ( Cluster::node == "worker-1" ) + { + while ( n < 1000 ) + { + Reporter::net_weird("weird1"); + ++n; + } + + Reporter::net_weird("weird3"); + } + else if ( Cluster::node == "worker-2" ) + { + while ( n < 1000 ) + { + Reporter::net_weird("weird1"); + Reporter::net_weird("weird2"); + ++n; + } + } + + schedule 5secs { ready_again() }; + } + + +@if ( Cluster::local_node_type() == Cluster::MANAGER ) + +event bro_init() + { + Broker::auto_publish(Cluster::worker_topic, ready_for_data); + } + +global peer_count = 0; + +event Broker::peer_added(endpoint: Broker::EndpointInfo, msg: string) + { + ++peer_count; + + if ( peer_count == 2 ) + event ready_for_data(); + } + +@endif diff --git a/testing/btest/scripts/policy/misc/weird-stats.bro b/testing/btest/scripts/policy/misc/weird-stats.bro new file mode 100644 index 0000000000..b26fce8e47 --- /dev/null +++ b/testing/btest/scripts/policy/misc/weird-stats.bro @@ -0,0 +1,32 @@ +# @TEST-EXEC: btest-bg-run bro bro %INPUT +# @TEST-EXEC: btest-bg-wait 20 +# @TEST-EXEC: btest-diff bro/weird_stats.log + +@load misc/weird-stats.bro + +redef exit_only_after_terminate = T; +redef WeirdStats::weird_stat_interval = 5sec; + +event die() + { + terminate(); + } + +event gen_weirds(n: count, done: bool &default = F) + { + while ( n != 0 ) + { + Reporter::net_weird("my_weird"); + --n; + } + + if ( done ) + schedule 5sec { die() }; + } + +event bro_init() + { + event gen_weirds(1000); + schedule 7.5sec { gen_weirds(2000) } ; + schedule 12.5sec { gen_weirds(10, T) } ; + } diff --git a/testing/scripts/diff-remove-abspath b/testing/scripts/diff-remove-abspath index 361ad3fa6d..a981206e83 100755 --- a/testing/scripts/diff-remove-abspath +++ b/testing/scripts/diff-remove-abspath @@ -2,4 +2,11 @@ # # Replace absolute paths with the basename. +if [ `uname` == "Linux" ]; then + sed="sed -r" +else + sed="sed -E" +fi + +$sed 's#/+#/#g' | \ sed 's#/\([^/]\{1,\}/\)\{1,\}\([^/]\{1,\}\)#<...>/\2#g' diff --git a/testing/scripts/diff-remove-timestamps b/testing/scripts/diff-remove-timestamps index 770a181c59..325268d832 100755 --- a/testing/scripts/diff-remove-timestamps +++ b/testing/scripts/diff-remove-timestamps @@ -9,5 +9,5 @@ else sed="sed -E" fi -$sed 's/(0\.000000)|([0-9]{10}\.[0-9]{2,8})/XXXXXXXXXX.XXXXXX/g' | \ +$sed 's/(0\.000000)|([0-9]{9,10}\.[0-9]{2,8})/XXXXXXXXXX.XXXXXX/g' | \ $sed 's/^ *#(open|close).(19|20)..-..-..-..-..-..$/#\1 XXXX-XX-XX-XX-XX-XX/g'