diff --git a/CHANGES b/CHANGES index 42ac97e455..77b404540f 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,47 @@ +2.4-169 | 2015-10-01 17:21:21 -0700 + + * Fixed parsing of V_ASN1_GENERALIZEDTIME timestamps in x509 + certificates. (Yun Zheng Hu) + + * Improve X509 end-of-string-check code. (Johanna Amann) + + * Refactor X509 generalizedtime support and test. (Johanna Amann) + + * Fix case of offset=-1 (EOF) for RAW reader. Addresses BIT-1479. + (Johanna Amann) + + * Improve a number of test canonifiers. (Daniel Thayer) + + * Remove unnecessary use of TEST_DIFF_CANONIFIER. (Daniel Thayer) + + * Fixed some test canonifiers to read only from stdin + + * Remove unused test canonifier scripts. (Daniel Thayer) + + * A potpourri of updates and improvements across the documentation. + (Daniel Thayer) + + * Add configure option to disable Broker Python bindings. Also + improve the configure summary output to more clearly show whether + or not Broker Python bindings will be built. (Daniel Thayer) + +2.4-131 | 2015-09-11 12:16:39 -0700 + + * Add README.rst symlink. Addresses BIT-1413 (Vlad Grigorescu) + +2.4-129 | 2015-09-11 11:56:04 -0700 + + * hash-all-files.bro depends on base/files/hash (Richard van den Berg) + + * Make dns_max_queries redef-able, and bump default to 25. Addresses + BIT-1460 (Vlad Grigorescu) + +2.4-125 | 2015-09-03 20:10:36 -0700 + + * Move SIP analyzer to flowunit instead of datagram Addresses + BIT-1458 (Vlad Grigorescu) + 2.4-122 | 2015-08-31 14:39:41 -0700 * Add a number of out-of-bound checks to layer 2 code. Addresses diff --git a/CMakeLists.txt b/CMakeLists.txt index bf55696eb6..846f2b484a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -234,6 +234,7 @@ message( "\nCPP: ${CMAKE_CXX_COMPILER}" "\n" "\nBroker: ${ENABLE_BROKER}" + "\nBroker Python: ${BROKER_PYTHON_BINDINGS}" "\nBroccoli: ${INSTALL_BROCCOLI}" "\nBroctl: ${INSTALL_BROCTL}" "\nAux. Tools: ${INSTALL_AUX_TOOLS}" diff --git a/README.rst b/README.rst new file mode 120000 index 0000000000..100b93820a --- /dev/null +++ b/README.rst @@ -0,0 +1 @@ +README \ No newline at end of file diff --git a/VERSION b/VERSION index 4365e48ae5..622ec2383c 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.4-122 +2.4-169 diff --git a/aux/binpac b/aux/binpac index ff16caf3d8..214294c502 160000 --- a/aux/binpac +++ b/aux/binpac @@ -1 +1 @@ -Subproject commit ff16caf3d8c5b12febd465a8ddd1524af60eae1a +Subproject commit 214294c502d377bb7bf511eac8c43608e54c875a diff --git a/aux/bro-aux b/aux/bro-aux index 2ec49971f1..4e0d2bff4b 160000 --- a/aux/bro-aux +++ b/aux/bro-aux @@ -1 +1 @@ -Subproject commit 2ec49971f12176e1fabe9db21445435b77bad68e +Subproject commit 4e0d2bff4b2c287f66186c3654ef784bb0748d11 diff --git a/aux/broccoli b/aux/broccoli index 0c051fb343..8046800085 160000 --- a/aux/broccoli +++ b/aux/broccoli @@ -1 +1 @@ -Subproject commit 0c051fb3439abe7b4c915dbdaa751e91140dcf1e +Subproject commit 80468000859bcb7c3784c69280888fcfe89d8922 diff --git a/aux/broctl b/aux/broctl index 992a79e1e3..921b0abcb9 160000 --- a/aux/broctl +++ b/aux/broctl @@ -1 +1 @@ -Subproject commit 992a79e1e36cef032373bf42cff456bb3598597d +Subproject commit 921b0abcb967666d8349c0c6c2bb8e41e1300579 diff --git a/aux/broker b/aux/broker index 9e640c393a..e7da54a3f4 160000 --- a/aux/broker +++ b/aux/broker @@ -1 +1 @@ -Subproject commit 9e640c393a2144a48a464bdcbe685743131299b8 +Subproject commit e7da54a3f40e71ca9020f9846256f60c0b885963 diff --git a/aux/btest b/aux/btest index 25658b96d2..ce1d474859 160000 --- a/aux/btest +++ b/aux/btest @@ -1 +1 @@ -Subproject commit 25658b96d252786a4428418cc837486b4d07bbcf +Subproject commit ce1d474859cc8a0f39d5eaf69fb1bb56eb1a5161 diff --git a/aux/plugins b/aux/plugins index 082676f548..9b7943e1a6 160000 --- a/aux/plugins +++ b/aux/plugins @@ -1 +1 @@ -Subproject commit 082676f54874de968bc95bb8fede13a6c2521b5e +Subproject commit 9b7943e1a61062005f01b48eaad11bbb3b7ae757 diff --git a/cmake b/cmake index 0fab31c3b3..843cdf6a91 160000 --- a/cmake +++ b/cmake @@ -1 +1 @@ -Subproject commit 0fab31c3b3b6606831364a9c4266128bb7e53465 +Subproject commit 843cdf6a91f06e5407bffbc79a343bff3cf4c81f diff --git a/configure b/configure index 3e844735a5..f94085f9d3 100755 --- a/configure +++ b/configure @@ -47,6 +47,7 @@ Usage: $0 [OPTION]... [VAR=VALUE]... --disable-auxtools don't build or install auxiliary tools --disable-perftools don't try to build with Google Perftools --disable-python don't try to build python bindings for broccoli + --disable-pybroker don't try to build python bindings for broker Required Packages in Non-Standard Locations: --with-openssl=PATH path to OpenSSL install root @@ -121,6 +122,7 @@ append_cache_entry PY_MOD_INSTALL_DIR PATH $prefix/lib/broctl append_cache_entry BRO_SCRIPT_INSTALL_PATH STRING $prefix/share/bro append_cache_entry BRO_ETC_INSTALL_DIR PATH $prefix/etc append_cache_entry BROKER_PYTHON_HOME PATH $prefix +append_cache_entry BROKER_PYTHON_BINDINGS BOOL false append_cache_entry ENABLE_DEBUG BOOL false append_cache_entry ENABLE_PERFTOOLS BOOL false append_cache_entry ENABLE_PERFTOOLS_DEBUG BOOL false @@ -217,6 +219,9 @@ while [ $# -ne 0 ]; do --disable-python) append_cache_entry DISABLE_PYTHON_BINDINGS BOOL true ;; + --disable-pybroker) + append_cache_entry DISABLE_PYBROKER BOOL true + ;; --enable-ruby) append_cache_entry DISABLE_RUBY_BINDINGS BOOL false ;; diff --git a/doc/components/bro-plugins/pf_ring/README.rst b/doc/components/bro-plugins/pf_ring/README.rst new file mode 120000 index 0000000000..5ea666e8c9 --- /dev/null +++ b/doc/components/bro-plugins/pf_ring/README.rst @@ -0,0 +1 @@ +../../../../aux/plugins/pf_ring/README \ No newline at end of file diff --git a/doc/components/bro-plugins/redis/README.rst b/doc/components/bro-plugins/redis/README.rst new file mode 120000 index 0000000000..c42051828e --- /dev/null +++ b/doc/components/bro-plugins/redis/README.rst @@ -0,0 +1 @@ +../../../../aux/plugins/redis/README \ No newline at end of file diff --git a/doc/devel/plugins.rst b/doc/devel/plugins.rst index 0ed22a0cb9..dc1c9a3cd4 100644 --- a/doc/devel/plugins.rst +++ b/doc/devel/plugins.rst @@ -286,9 +286,9 @@ Activating a plugin will: 1. Load the dynamic module 2. Make any bif items available 3. Add the ``scripts/`` directory to ``BROPATH`` - 5. Load ``scripts/__preload__.bro`` - 6. Make BiF elements available to scripts. - 7. Load ``scripts/__load__.bro`` + 4. Load ``scripts/__preload__.bro`` + 5. Make BiF elements available to scripts. + 6. Load ``scripts/__load__.bro`` By default, Bro will automatically activate all dynamic plugins found in its search path ``BRO_PLUGIN_PATH``. However, in bare mode (``bro diff --git a/doc/frameworks/broker.rst b/doc/frameworks/broker.rst index 3cd8dab6e3..8c5ed24e25 100644 --- a/doc/frameworks/broker.rst +++ b/doc/frameworks/broker.rst @@ -9,10 +9,7 @@ Broker-Enabled Communication Framework Bro can now use the `Broker Library <../components/broker/README.html>`_ to exchange information with - other Bro processes. To enable it run Bro's ``configure`` script - with the ``--enable-broker`` option. Note that a C++11 compatible - compiler (e.g. GCC 4.8+ or Clang 3.3+) is required as well as the - `C++ Actor Framework `_. + other Bro processes. .. contents:: @@ -23,26 +20,26 @@ Communication via Broker must first be turned on via :bro:see:`BrokerComm::enable`. Bro can accept incoming connections by calling :bro:see:`BrokerComm::listen` -and then monitor connection status updates via +and then monitor connection status updates via the :bro:see:`BrokerComm::incoming_connection_established` and -:bro:see:`BrokerComm::incoming_connection_broken`. +:bro:see:`BrokerComm::incoming_connection_broken` events. .. btest-include:: ${DOC_ROOT}/frameworks/broker/connecting-listener.bro Bro can initiate outgoing connections by calling :bro:see:`BrokerComm::connect` -and then monitor connection status updates via +and then monitor connection status updates via the :bro:see:`BrokerComm::outgoing_connection_established`, :bro:see:`BrokerComm::outgoing_connection_broken`, and -:bro:see:`BrokerComm::outgoing_connection_incompatible`. +:bro:see:`BrokerComm::outgoing_connection_incompatible` events. .. btest-include:: ${DOC_ROOT}/frameworks/broker/connecting-connector.bro Remote Printing =============== -To receive remote print messages, first use -:bro:see:`BrokerComm::subscribe_to_prints` to advertise to peers a topic -prefix of interest and then create an event handler for +To receive remote print messages, first use the +:bro:see:`BrokerComm::subscribe_to_prints` function to advertise to peers a +topic prefix of interest and then create an event handler for :bro:see:`BrokerComm::print_handler` to handle any print messages that are received. @@ -71,17 +68,17 @@ the Broker message format is simply: Remote Events ============= -Receiving remote events is similar to remote prints. Just use -:bro:see:`BrokerComm::subscribe_to_events` and possibly define any new events -along with handlers that peers may want to send. +Receiving remote events is similar to remote prints. Just use the +:bro:see:`BrokerComm::subscribe_to_events` function and possibly define any +new events along with handlers that peers may want to send. .. btest-include:: ${DOC_ROOT}/frameworks/broker/events-listener.bro -To send events, there are two choices. The first is to use call -:bro:see:`BrokerComm::event` directly. The second option is to use -:bro:see:`BrokerComm::auto_event` to make it so a particular event is -automatically sent to peers whenever it is called locally via the normal -event invocation syntax. +There are two different ways to send events. The first is to call the +:bro:see:`BrokerComm::event` function directly. The second option is to call +the :bro:see:`BrokerComm::auto_event` function where you specify a +particular event that will be automatically sent to peers whenever the +event is called locally via the normal event invocation syntax. .. btest-include:: ${DOC_ROOT}/frameworks/broker/events-connector.bro @@ -98,7 +95,7 @@ the Broker message format is: broker::message{std::string{}, ...}; The first parameter is the name of the event and the remaining ``...`` -are its arguments, which are any of the support Broker data types as +are its arguments, which are any of the supported Broker data types as they correspond to the Bro types for the event named in the first parameter of the message. @@ -107,23 +104,23 @@ Remote Logging .. btest-include:: ${DOC_ROOT}/frameworks/broker/testlog.bro -Use :bro:see:`BrokerComm::subscribe_to_logs` to advertise interest in logs -written by peers. The topic names that Bro uses are implicitly of the +Use the :bro:see:`BrokerComm::subscribe_to_logs` function to advertise interest +in logs written by peers. The topic names that Bro uses are implicitly of the form "bro/log/". .. btest-include:: ${DOC_ROOT}/frameworks/broker/logs-listener.bro -To send remote logs either use :bro:see:`Log::enable_remote_logging` or -:bro:see:`BrokerComm::enable_remote_logs`. The former allows any log stream -to be sent to peers while the later toggles remote logging for -particular streams. +To send remote logs either redef :bro:see:`Log::enable_remote_logging` or +use the :bro:see:`BrokerComm::enable_remote_logs` function. The former +allows any log stream to be sent to peers while the latter enables remote +logging for particular streams. .. btest-include:: ${DOC_ROOT}/frameworks/broker/logs-connector.bro Message Format -------------- -For other applications that want to exchange logs messages with Bro, +For other applications that want to exchange log messages with Bro, the Broker message format is: .. code:: c++ @@ -132,7 +129,7 @@ the Broker message format is: The enum value corresponds to the stream's :bro:see:`Log::ID` value, and the record corresponds to a single entry of that log's columns record, -in this case a ``Test::INFO`` value. +in this case a ``Test::Info`` value. Tuning Access Control ===================== @@ -152,11 +149,12 @@ that take a :bro:see:`BrokerComm::SendFlags` such as :bro:see:`BrokerComm::print :bro:see:`BrokerComm::enable_remote_logs`. If not using the ``auto_advertise`` flag, one can use the -:bro:see:`BrokerComm::advertise_topic` and :bro:see:`BrokerComm::unadvertise_topic` -to manupulate the set of topic prefixes that are allowed to be -advertised to peers. If an endpoint does not advertise a topic prefix, -the only way a peers can send messages to it is via the ``unsolicited`` -flag of :bro:see:`BrokerComm::SendFlags` and choosing a topic with a matching +:bro:see:`BrokerComm::advertise_topic` and +:bro:see:`BrokerComm::unadvertise_topic` functions +to manipulate the set of topic prefixes that are allowed to be +advertised to peers. If an endpoint does not advertise a topic prefix, then +the only way peers can send messages to it is via the ``unsolicited`` +flag of :bro:see:`BrokerComm::SendFlags` and choosing a topic with a matching prefix (i.e. full topic may be longer than receivers prefix, just the prefix needs to match). @@ -172,7 +170,7 @@ specific type of frontend, but a standalone frontend can also exist to e.g. query and modify the contents of a remote master store without actually "owning" any of the contents itself. -A master data store can be be cloned from remote peers which may then +A master data store can be cloned from remote peers which may then perform lightweight, local queries against the clone, which automatically stays synchronized with the master store. Clones cannot modify their content directly, instead they send modifications to the @@ -181,7 +179,7 @@ all clones. Master and clone stores get to choose what type of storage backend to use. E.g. In-memory versus SQLite for persistence. Note that if clones -are used, data store sizes should still be able to fit within memory +are used, then data store sizes must be able to fit within memory regardless of the storage backend as a single snapshot of the master store is sent in a single chunk to initialize the clone. @@ -198,5 +196,5 @@ needed, just replace the :bro:see:`BrokerStore::create_clone` call with :bro:see:`BrokerStore::create_frontend`. Queries will then be made against the remote master store instead of the local clone. -Note that all queries are made within Bro's asynchrounous ``when`` -statements and must specify a timeout block. +Note that all data store queries must be made within Bro's asynchronous +``when`` statements and must specify a timeout block. diff --git a/doc/frameworks/broker/connecting-connector.bro b/doc/frameworks/broker/connecting-connector.bro index a7e621e4a6..cd5c74add8 100644 --- a/doc/frameworks/broker/connecting-connector.bro +++ b/doc/frameworks/broker/connecting-connector.bro @@ -1,4 +1,3 @@ - const broker_port: port = 9999/tcp &redef; redef exit_only_after_terminate = T; redef BrokerComm::endpoint_name = "connector"; diff --git a/doc/frameworks/broker/connecting-listener.bro b/doc/frameworks/broker/connecting-listener.bro index c37af3ae4d..21c67f9696 100644 --- a/doc/frameworks/broker/connecting-listener.bro +++ b/doc/frameworks/broker/connecting-listener.bro @@ -1,4 +1,3 @@ - const broker_port: port = 9999/tcp &redef; redef exit_only_after_terminate = T; redef BrokerComm::endpoint_name = "listener"; diff --git a/doc/frameworks/broker/events-listener.bro b/doc/frameworks/broker/events-listener.bro index aa6ea9ee4e..dc18795903 100644 --- a/doc/frameworks/broker/events-listener.bro +++ b/doc/frameworks/broker/events-listener.bro @@ -1,4 +1,3 @@ - const broker_port: port = 9999/tcp &redef; redef exit_only_after_terminate = T; redef BrokerComm::endpoint_name = "listener"; diff --git a/doc/frameworks/broker/printing-listener.bro b/doc/frameworks/broker/printing-listener.bro index 080d09e8f5..f55c5b9bad 100644 --- a/doc/frameworks/broker/printing-listener.bro +++ b/doc/frameworks/broker/printing-listener.bro @@ -1,4 +1,3 @@ - const broker_port: port = 9999/tcp &redef; redef exit_only_after_terminate = T; redef BrokerComm::endpoint_name = "listener"; diff --git a/doc/frameworks/broker/testlog.bro b/doc/frameworks/broker/testlog.bro index f63c19ac48..506d359bb7 100644 --- a/doc/frameworks/broker/testlog.bro +++ b/doc/frameworks/broker/testlog.bro @@ -1,4 +1,3 @@ - module Test; export { diff --git a/doc/frameworks/geoip.rst b/doc/frameworks/geoip.rst index 98252d7184..d756f97589 100644 --- a/doc/frameworks/geoip.rst +++ b/doc/frameworks/geoip.rst @@ -20,11 +20,13 @@ GeoLocation Install libGeoIP ---------------- +Before building Bro, you need to install libGeoIP. + * FreeBSD: .. console:: - sudo pkg_add -r GeoIP + sudo pkg install GeoIP * RPM/RedHat-based Linux: @@ -40,80 +42,99 @@ Install libGeoIP * Mac OS X: - Vanilla OS X installations don't ship with libGeoIP, but if - installed from your preferred package management system (e.g. - MacPorts, Fink, or Homebrew), they should be automatically detected - and Bro will compile against them. + You need to install from your preferred package management system + (e.g. MacPorts, Fink, or Homebrew). The name of the package that you need + may be libgeoip, geoip, or geoip-dev, depending on which package management + system you are using. GeoIPLite Database Installation ------------------------------------- +------------------------------- A country database for GeoIPLite is included when you do the C API install, but for Bro, we are using the city database which includes cities and regions in addition to countries. `Download `__ the GeoLite city -binary database. +binary database: - .. console:: +.. console:: wget http://geolite.maxmind.com/download/geoip/database/GeoLiteCity.dat.gz gunzip GeoLiteCity.dat.gz -Next, the file needs to be put in the database directory. This directory -should already exist and will vary depending on which platform and package -you are using. For FreeBSD, use ``/usr/local/share/GeoIP``. For Linux, -use ``/usr/share/GeoIP`` or ``/var/lib/GeoIP`` (choose whichever one +Next, the file needs to be renamed and put in the GeoIP database directory. +This directory should already exist and will vary depending on which platform +and package you are using. For FreeBSD, use ``/usr/local/share/GeoIP``. For +Linux, use ``/usr/share/GeoIP`` or ``/var/lib/GeoIP`` (choose whichever one already exists). - .. console:: +.. console:: mv GeoLiteCity.dat /GeoIPCity.dat +Note that there is a separate database for IPv6 addresses, which can also +be installed if you want GeoIP functionality for IPv6. + +Testing +------- + +Before using the GeoIP functionality, it is a good idea to verify that +everything is setup correctly. After installing libGeoIP and the GeoIP city +database, and building Bro, you can quickly check if the GeoIP functionality +works by running a command like this: + +.. console:: + + bro -e "print lookup_location(8.8.8.8);" + +If you see an error message similar to "Failed to open GeoIP City database", +then you may need to either rename or move your GeoIP city database file (the +error message should give you the full pathname of the database file that +Bro is looking for). + +If you see an error message similar to "Bro was not configured for GeoIP +support", then you need to rebuild Bro and make sure it is linked against +libGeoIP. Normally, if libGeoIP is installed correctly then it should +automatically be found when building Bro. If this doesn't happen, then +you may need to specify the path to the libGeoIP installation +(e.g. ``./configure --with-geoip=``). Usage ----- -There is a single built in function that provides the GeoIP -functionality: +There is a built-in function that provides the GeoIP functionality: .. code:: bro function lookup_location(a:addr): geo_location -There is also the :bro:see:`geo_location` data structure that is returned -from the :bro:see:`lookup_location` function: - -.. code:: bro - - type geo_location: record { - country_code: string; - region: string; - city: string; - latitude: double; - longitude: double; - }; - +The return value of the :bro:see:`lookup_location` function is a record +type called :bro:see:`geo_location`, and it consists of several fields +containing the country, region, city, latitude, and longitude of the specified +IP address. Since one or more fields in this record will be uninitialized +for some IP addresses (for example, the country and region of an IP address +might be known, but the city could be unknown), a field should be checked +if it has a value before trying to access the value. Example ------- -To write a line in a log file for every ftp connection from hosts in -Ohio, this is now very easy: +To show every ftp connection from hosts in Ohio, this is now very easy: .. code:: bro - global ftp_location_log: file = open_log_file("ftp-location"); - event ftp_reply(c: connection, code: count, msg: string, cont_resp: bool) { local client = c$id$orig_h; local loc = lookup_location(client); - if (loc$region == "OH" && loc$country_code == "US") + + if (loc?$region && loc$region == "OH" && loc$country_code == "US") { - print ftp_location_log, fmt("FTP Connection from:%s (%s,%s,%s)", client, loc$city, loc$region, loc$country_code); + local city = loc?$city ? loc$city : ""; + + print fmt("FTP Connection from:%s (%s,%s,%s)", client, city, + loc$region, loc$country_code); } } - diff --git a/doc/frameworks/input.rst b/doc/frameworks/input.rst index ef40756a26..aa2dce6417 100644 --- a/doc/frameworks/input.rst +++ b/doc/frameworks/input.rst @@ -32,7 +32,8 @@ For this example we assume that we want to import data from a blacklist that contains server IP addresses as well as the timestamp and the reason for the block. -An example input file could look like this: +An example input file could look like this (note that all fields must be +tab-separated): :: @@ -63,19 +64,23 @@ The two records are defined as: reason: string; }; -Note that the names of the fields in the record definitions have to correspond +Note that the names of the fields in the record definitions must correspond to the column names listed in the '#fields' line of the log file, in this -case 'ip', 'timestamp', and 'reason'. +case 'ip', 'timestamp', and 'reason'. Also note that the ordering of the +columns does not matter, because each column is identified by name. -The log file is read into the table with a simple call of the ``add_table`` -function: +The log file is read into the table with a simple call of the +:bro:id:`Input::add_table` function: .. code:: bro global blacklist: table[addr] of Val = table(); - Input::add_table([$source="blacklist.file", $name="blacklist", $idx=Idx, $val=Val, $destination=blacklist]); - Input::remove("blacklist"); + event bro_init() { + Input::add_table([$source="blacklist.file", $name="blacklist", + $idx=Idx, $val=Val, $destination=blacklist]); + Input::remove("blacklist"); + } With these three lines we first create an empty table that should contain the blacklist data and then instruct the input framework to open an input stream @@ -92,7 +97,7 @@ Because of this, the data is not immediately accessible. Depending on the size of the data source it might take from a few milliseconds up to a few seconds until all data is present in the table. Please note that this means that when Bro is running without an input source or on very short captured -files, it might terminate before the data is present in the system (because +files, it might terminate before the data is present in the table (because Bro already handled all packets before the import thread finished). Subsequent calls to an input source are queued until the previous action has @@ -101,8 +106,8 @@ been completed. Because of this, it is, for example, possible to call will remain queued until the first read has been completed. Once the input framework finishes reading from a data source, it fires -the ``end_of_data`` event. Once this event has been received all data -from the input file is available in the table. +the :bro:id:`Input::end_of_data` event. Once this event has been received all +data from the input file is available in the table. .. code:: bro @@ -111,9 +116,9 @@ from the input file is available in the table. print blacklist; } -The table can also already be used while the data is still being read - it -just might not contain all lines in the input file when the event has not -yet fired. After it has been populated it can be used like any other Bro +The table can be used while the data is still being read - it +just might not contain all lines from the input file before the event has +fired. After the table has been populated it can be used like any other Bro table and blacklist entries can easily be tested: .. code:: bro @@ -130,10 +135,11 @@ changing. For these cases, the Bro input framework supports several ways to deal with changing data files. The first, very basic method is an explicit refresh of an input stream. When -an input stream is open, the function ``force_update`` can be called. This -will trigger a complete refresh of the table; any changed elements from the -file will be updated. After the update is finished the ``end_of_data`` -event will be raised. +an input stream is open (this means it has not yet been removed by a call to +:bro:id:`Input::remove`), the function :bro:id:`Input::force_update` can be +called. This will trigger a complete refresh of the table; any changed +elements from the file will be updated. After the update is finished the +:bro:id:`Input::end_of_data` event will be raised. In our example the call would look like: @@ -141,30 +147,35 @@ In our example the call would look like: Input::force_update("blacklist"); -The input framework also supports two automatic refresh modes. The first mode -continually checks if a file has been changed. If the file has been changed, it +Alternatively, the input framework can automatically refresh the table +contents when it detects a change to the input file. To use this feature, +you need to specify a non-default read mode by setting the ``mode`` option +of the :bro:id:`Input::add_table` call. Valid values are ``Input::MANUAL`` +(the default), ``Input::REREAD`` and ``Input::STREAM``. For example, +setting the value of the ``mode`` option in the previous example +would look like this: + +.. code:: bro + + Input::add_table([$source="blacklist.file", $name="blacklist", + $idx=Idx, $val=Val, $destination=blacklist, + $mode=Input::REREAD]); + +When using the reread mode (i.e., ``$mode=Input::REREAD``), Bro continually +checks if the input file has been changed. If the file has been changed, it is re-read and the data in the Bro table is updated to reflect the current state. Each time a change has been detected and all the new data has been read into the table, the ``end_of_data`` event is raised. -The second mode is a streaming mode. This mode assumes that the source data -file is an append-only file to which new data is continually appended. Bro -continually checks for new data at the end of the file and will add the new -data to the table. If newer lines in the file have the same index as previous -lines, they will overwrite the values in the output table. Because of the -nature of streaming reads (data is continually added to the table), -the ``end_of_data`` event is never raised when using streaming reads. +When using the streaming mode (i.e., ``$mode=Input::STREAM``), Bro assumes +that the source data file is an append-only file to which new data is +continually appended. Bro continually checks for new data at the end of +the file and will add the new data to the table. If newer lines in the +file have the same index as previous lines, they will overwrite the +values in the output table. Because of the nature of streaming reads +(data is continually added to the table), the ``end_of_data`` event +is never raised when using streaming reads. -The reading mode can be selected by setting the ``mode`` option of the -add_table call. Valid values are ``MANUAL`` (the default), ``REREAD`` -and ``STREAM``. - -Hence, when adding ``$mode=Input::REREAD`` to the previous example, the -blacklist table will always reflect the state of the blacklist input file. - -.. code:: bro - - Input::add_table([$source="blacklist.file", $name="blacklist", $idx=Idx, $val=Val, $destination=blacklist, $mode=Input::REREAD]); Receiving change events ----------------------- @@ -173,34 +184,40 @@ When re-reading files, it might be interesting to know exactly which lines in the source files have changed. For this reason, the input framework can raise an event each time when a data -item is added to, removed from or changed in a table. +item is added to, removed from, or changed in a table. -The event definition looks like this: +The event definition looks like this (note that you can change the name of +this event in your own Bro script): .. code:: bro - event entry(description: Input::TableDescription, tpe: Input::Event, left: Idx, right: Val) { - # act on values + event entry(description: Input::TableDescription, tpe: Input::Event, + left: Idx, right: Val) { + # do something here... + print fmt("%s = %s", left, right); } -The event has to be specified in ``$ev`` in the ``add_table`` call: +The event must be specified in ``$ev`` in the ``add_table`` call: .. code:: bro - Input::add_table([$source="blacklist.file", $name="blacklist", $idx=Idx, $val=Val, $destination=blacklist, $mode=Input::REREAD, $ev=entry]); + Input::add_table([$source="blacklist.file", $name="blacklist", + $idx=Idx, $val=Val, $destination=blacklist, + $mode=Input::REREAD, $ev=entry]); -The ``description`` field of the event contains the arguments that were +The ``description`` argument of the event contains the arguments that were originally supplied to the add_table call. Hence, the name of the stream can, -for example, be accessed with ``description$name``. ``tpe`` is an enum -containing the type of the change that occurred. +for example, be accessed with ``description$name``. The ``tpe`` argument of the +event is an enum containing the type of the change that occurred. If a line that was not previously present in the table has been added, -then ``tpe`` will contain ``Input::EVENT_NEW``. In this case ``left`` contains -the index of the added table entry and ``right`` contains the values of the -added entry. +then the value of ``tpe`` will be ``Input::EVENT_NEW``. In this case ``left`` +contains the index of the added table entry and ``right`` contains the +values of the added entry. If a table entry that already was present is altered during the re-reading or -streaming read of a file, ``tpe`` will contain ``Input::EVENT_CHANGED``. In +streaming read of a file, then the value of ``tpe`` will be +``Input::EVENT_CHANGED``. In this case ``left`` contains the index of the changed table entry and ``right`` contains the values of the entry before the change. The reason for this is that the table already has been updated when the event is raised. The current @@ -208,8 +225,9 @@ value in the table can be ascertained by looking up the current table value. Hence it is possible to compare the new and the old values of the table. If a table element is removed because it was no longer present during a -re-read, then ``tpe`` will contain ``Input::REMOVED``. In this case ``left`` -contains the index and ``right`` the values of the removed element. +re-read, then the value of ``tpe`` will be ``Input::EVENT_REMOVED``. In this +case ``left`` contains the index and ``right`` the values of the removed +element. Filtering data during import @@ -222,24 +240,26 @@ can either accept or veto the change by returning true for an accepted change and false for a rejected change. Furthermore, it can alter the data before it is written to the table. -The following example filter will reject to add entries to the table when +The following example filter will reject adding entries to the table when they were generated over a month ago. It will accept all changes and all removals of values that are already present in the table. .. code:: bro - Input::add_table([$source="blacklist.file", $name="blacklist", $idx=Idx, $val=Val, $destination=blacklist, $mode=Input::REREAD, - $pred(typ: Input::Event, left: Idx, right: Val) = { - if ( typ != Input::EVENT_NEW ) { - return T; - } - return ( ( current_time() - right$timestamp ) < (30 day) ); - }]); + Input::add_table([$source="blacklist.file", $name="blacklist", + $idx=Idx, $val=Val, $destination=blacklist, + $mode=Input::REREAD, + $pred(typ: Input::Event, left: Idx, right: Val) = { + if ( typ != Input::EVENT_NEW ) { + return T; + } + return (current_time() - right$timestamp) < 30day; + }]); To change elements while they are being imported, the predicate function can manipulate ``left`` and ``right``. Note that predicate functions are called before the change is committed to the table. Hence, when a table element is -changed (``tpe`` is ``INPUT::EVENT_CHANGED``), ``left`` and ``right`` +changed (``typ`` is ``Input::EVENT_CHANGED``), ``left`` and ``right`` contain the new values, but the destination (``blacklist`` in our example) still contains the old values. This allows predicate functions to examine the changes between the old and the new version before deciding if they @@ -250,14 +270,19 @@ Different readers The input framework supports different kinds of readers for different kinds of source data files. At the moment, the default reader reads ASCII files -formatted in the Bro log file format (tab-separated values). At the moment, -Bro comes with two other readers. The ``RAW`` reader reads a file that is -split by a specified record separator (usually newline). The contents are +formatted in the Bro log file format (tab-separated values with a "#fields" +header line). Several other readers are included in Bro. + +The raw reader reads a file that is +split by a specified record separator (newline by default). The contents are returned line-by-line as strings; it can, for example, be used to read configuration files and the like and is probably only useful in the event mode and not for reading data to tables. -Another included reader is the ``BENCHMARK`` reader, which is being used +The binary reader is intended to be used with file analysis input streams (and +is the default type of reader for those streams). + +The benchmark reader is being used to optimize the speed of the input framework. It can generate arbitrary amounts of semi-random data in all Bro data types supported by the input framework. @@ -270,75 +295,17 @@ aforementioned ones: logging-input-sqlite -Add_table options ------------------ - -This section lists all possible options that can be used for the add_table -function and gives a short explanation of their use. Most of the options -already have been discussed in the previous sections. - -The possible fields that can be set for a table stream are: - - ``source`` - A mandatory string identifying the source of the data. - For the ASCII reader this is the filename. - - ``name`` - A mandatory name for the filter that can later be used - to manipulate it further. - - ``idx`` - Record type that defines the index of the table. - - ``val`` - Record type that defines the values of the table. - - ``reader`` - The reader used for this stream. Default is ``READER_ASCII``. - - ``mode`` - The mode in which the stream is opened. Possible values are - ``MANUAL``, ``REREAD`` and ``STREAM``. Default is ``MANUAL``. - ``MANUAL`` means that the file is not updated after it has - been read. Changes to the file will not be reflected in the - data Bro knows. ``REREAD`` means that the whole file is read - again each time a change is found. This should be used for - files that are mapped to a table where individual lines can - change. ``STREAM`` means that the data from the file is - streamed. Events / table entries will be generated as new - data is appended to the file. - - ``destination`` - The destination table. - - ``ev`` - Optional event that is raised, when values are added to, - changed in, or deleted from the table. Events are passed an - Input::Event description as the first argument, the index - record as the second argument and the values as the third - argument. - - ``pred`` - Optional predicate, that can prevent entries from being added - to the table and events from being sent. - - ``want_record`` - Boolean value, that defines if the event wants to receive the - fields inside of a single record value, or individually - (default). This can be used if ``val`` is a record - containing only one type. In this case, if ``want_record`` is - set to false, the table will contain elements of the type - contained in ``val``. Reading Data to Events ====================== The second supported mode of the input framework is reading data to Bro -events instead of reading them to a table using event streams. +events instead of reading them to a table. Event streams work very similarly to table streams that were already discussed in much detail. To read the blacklist of the previous example -into an event stream, the following Bro code could be used: +into an event stream, the :bro:id:`Input::add_event` function is used. +For example: .. code:: bro @@ -348,12 +315,15 @@ into an event stream, the following Bro code could be used: reason: string; }; - event blacklistentry(description: Input::EventDescription, tpe: Input::Event, ip: addr, timestamp: time, reason: string) { - # work with event data + event blacklistentry(description: Input::EventDescription, + t: Input::Event, data: Val) { + # do something here... + print "data:", data; } event bro_init() { - Input::add_event([$source="blacklist.file", $name="blacklist", $fields=Val, $ev=blacklistentry]); + Input::add_event([$source="blacklist.file", $name="blacklist", + $fields=Val, $ev=blacklistentry]); } @@ -364,52 +334,3 @@ data types are provided in a single record definition. Apart from this, event streams work exactly the same as table streams and support most of the options that are also supported for table streams. -The options that can be set when creating an event stream with -``add_event`` are: - - ``source`` - A mandatory string identifying the source of the data. - For the ASCII reader this is the filename. - - ``name`` - A mandatory name for the stream that can later be used - to remove it. - - ``fields`` - Name of a record type containing the fields, which should be - retrieved from the input stream. - - ``ev`` - The event which is fired, after a line has been read from the - input source. The first argument that is passed to the event - is an Input::Event structure, followed by the data, either - inside of a record (if ``want_record is set``) or as - individual fields. The Input::Event structure can contain - information, if the received line is ``NEW``, has been - ``CHANGED`` or ``DELETED``. Since the ASCII reader cannot - track this information for event filters, the value is - always ``NEW`` at the moment. - - ``mode`` - The mode in which the stream is opened. Possible values are - ``MANUAL``, ``REREAD`` and ``STREAM``. Default is ``MANUAL``. - ``MANUAL`` means that the file is not updated after it has - been read. Changes to the file will not be reflected in the - data Bro knows. ``REREAD`` means that the whole file is read - again each time a change is found. This should be used for - files that are mapped to a table where individual lines can - change. ``STREAM`` means that the data from the file is - streamed. Events / table entries will be generated as new - data is appended to the file. - - ``reader`` - The reader used for this stream. Default is ``READER_ASCII``. - - ``want_record`` - Boolean value, that defines if the event wants to receive the - fields inside of a single record value, or individually - (default). If this is set to true, the event will receive a - single record of the type provided in ``fields``. - - - diff --git a/doc/frameworks/logging-input-sqlite.rst b/doc/frameworks/logging-input-sqlite.rst index 6f5e867686..e0f10308ae 100644 --- a/doc/frameworks/logging-input-sqlite.rst +++ b/doc/frameworks/logging-input-sqlite.rst @@ -23,17 +23,18 @@ In contrast to the ASCII reader and writer, the SQLite plugins have not yet seen extensive use in production environments. While we are not aware of any issues with them, we urge to caution when using them in production environments. There could be lingering issues which only occur -when the plugins are used with high amounts of data or in high-load environments. +when the plugins are used with high amounts of data or in high-load +environments. Logging Data into SQLite Databases ================================== Logging support for SQLite is available in all Bro installations starting with -version 2.2. There is no need to load any additional scripts or for any compile-time -configurations. +version 2.2. There is no need to load any additional scripts or for any +compile-time configurations. -Sending data from existing logging streams to SQLite is rather straightforward. You -have to define a filter which specifies SQLite as the writer. +Sending data from existing logging streams to SQLite is rather straightforward. +You have to define a filter which specifies SQLite as the writer. The following example code adds SQLite as a filter for the connection log: @@ -44,15 +45,15 @@ The following example code adds SQLite as a filter for the connection log: # Make sure this parses correctly at least. @TEST-EXEC: bro ${DOC_ROOT}/frameworks/sqlite-conn-filter.bro -Bro will create the database file ``/var/db/conn.sqlite``, if it does not already exist. -It will also create a table with the name ``conn`` (if it does not exist) and start -appending connection information to the table. +Bro will create the database file ``/var/db/conn.sqlite``, if it does not +already exist. It will also create a table with the name ``conn`` (if it +does not exist) and start appending connection information to the table. -At the moment, SQLite databases are not rotated the same way ASCII log-files are. You -have to take care to create them in an adequate location. +At the moment, SQLite databases are not rotated the same way ASCII log-files +are. You have to take care to create them in an adequate location. -If you examine the resulting SQLite database, the schema will contain the same fields -that are present in the ASCII log files:: +If you examine the resulting SQLite database, the schema will contain the +same fields that are present in the ASCII log files:: # sqlite3 /var/db/conn.sqlite @@ -75,27 +76,31 @@ from being created, you can remove the default filter: Log::remove_filter(Conn::LOG, "default"); -To create a custom SQLite log file, you have to create a new log stream that contains -just the information you want to commit to the database. Please refer to the -:ref:`framework-logging` documentation on how to create custom log streams. +To create a custom SQLite log file, you have to create a new log stream +that contains just the information you want to commit to the database. +Please refer to the :ref:`framework-logging` documentation on how to +create custom log streams. Reading Data from SQLite Databases ================================== -Like logging support, support for reading data from SQLite databases is built into Bro starting -with version 2.2. +Like logging support, support for reading data from SQLite databases is +built into Bro starting with version 2.2. -Just as with the text-based input readers (please refer to the :ref:`framework-input` -documentation for them and for basic information on how to use the input-framework), the SQLite reader -can be used to read data - in this case the result of SQL queries - into tables or into events. +Just as with the text-based input readers (please refer to the +:ref:`framework-input` documentation for them and for basic information +on how to use the input framework), the SQLite reader can be used to +read data - in this case the result of SQL queries - into tables or into +events. Reading Data into Tables ------------------------ -To read data from a SQLite database, we first have to provide Bro with the information, how -the resulting data will be structured. For this example, we expect that we have a SQLite database, -which contains host IP addresses and the user accounts that are allowed to log into a specific -machine. +To read data from a SQLite database, we first have to provide Bro with +the information, how the resulting data will be structured. For this +example, we expect that we have a SQLite database, which contains +host IP addresses and the user accounts that are allowed to log into +a specific machine. The SQLite commands to create the schema are as follows:: @@ -107,8 +112,8 @@ The SQLite commands to create the schema are as follows:: insert into machines_to_users values ('192.168.17.2', 'bernhard'); insert into machines_to_users values ('192.168.17.3', 'seth,matthias'); -After creating a file called ``hosts.sqlite`` with this content, we can read the resulting table -into Bro: +After creating a file called ``hosts.sqlite`` with this content, we can +read the resulting table into Bro: .. btest-include:: ${DOC_ROOT}/frameworks/sqlite-read-table.bro @@ -117,22 +122,25 @@ into Bro: # Make sure this parses correctly at least. @TEST-EXEC: bro ${DOC_ROOT}/frameworks/sqlite-read-table.bro -Afterwards, that table can be used to check logins into hosts against the available -userlist. +Afterwards, that table can be used to check logins into hosts against +the available userlist. Turning Data into Events ------------------------ -The second mode is to use the SQLite reader to output the input data as events. Typically there -are two reasons to do this. First, when the structure of the input data is too complicated -for a direct table import. In this case, the data can be read into an event which can then -create the necessary data structures in Bro in scriptland. +The second mode is to use the SQLite reader to output the input data as events. +Typically there are two reasons to do this. First, when the structure of +the input data is too complicated for a direct table import. In this case, +the data can be read into an event which can then create the necessary +data structures in Bro in scriptland. -The second reason is, that the dataset is too big to hold it in memory. In this case, the checks -can be performed on-demand, when Bro encounters a situation where it needs additional information. +The second reason is, that the dataset is too big to hold it in memory. In +this case, the checks can be performed on-demand, when Bro encounters a +situation where it needs additional information. -An example for this would be an internal huge database with malware hashes. Live database queries -could be used to check the sporadically happening downloads against the database. +An example for this would be an internal huge database with malware +hashes. Live database queries could be used to check the sporadically +happening downloads against the database. The SQLite commands to create the schema are as follows:: @@ -151,9 +159,10 @@ The SQLite commands to create the schema are as follows:: insert into malware_hashes values ('73f45106968ff8dc51fba105fa91306af1ff6666', 'ftp-trace'); -The following code uses the file-analysis framework to get the sha1 hashes of files that are -transmitted over the network. For each hash, a SQL-query is run against SQLite. If the query -returns with a result, we had a hit against our malware-database and output the matching hash. +The following code uses the file-analysis framework to get the sha1 hashes +of files that are transmitted over the network. For each hash, a SQL-query +is run against SQLite. If the query returns with a result, we had a hit +against our malware-database and output the matching hash. .. btest-include:: ${DOC_ROOT}/frameworks/sqlite-read-events.bro @@ -162,5 +171,5 @@ returns with a result, we had a hit against our malware-database and output the # Make sure this parses correctly at least. @TEST-EXEC: bro ${DOC_ROOT}/frameworks/sqlite-read-events.bro -If you run this script against the trace in ``testing/btest/Traces/ftp/ipv4.trace``, you -will get one hit. +If you run this script against the trace in +``testing/btest/Traces/ftp/ipv4.trace``, you will get one hit. diff --git a/doc/install/guidelines.rst b/doc/install/guidelines.rst index d1e1777165..a56110f865 100644 --- a/doc/install/guidelines.rst +++ b/doc/install/guidelines.rst @@ -46,4 +46,4 @@ where Bro was originally installed). Review the files for differences before copying and make adjustments as necessary (use the new version for differences that aren't a result of a local change). Of particular note, the copied version of ``$prefix/etc/broctl.cfg`` is likely to need changes -to the ``SpoolDir`` and ``LogDir`` settings. +to any settings that specify a pathname. diff --git a/doc/install/install.rst b/doc/install/install.rst index ff8d83ad97..ca1ea7f26a 100644 --- a/doc/install/install.rst +++ b/doc/install/install.rst @@ -4,7 +4,7 @@ .. _MacPorts: http://www.macports.org .. _Fink: http://www.finkproject.org .. _Homebrew: http://brew.sh -.. _bro downloads page: http://bro.org/download/index.html +.. _bro downloads page: https://www.bro.org/download/index.html .. _installing-bro: @@ -32,13 +32,13 @@ before you begin: * Libz * Bash (for BroControl) * Python (for BroControl) - * C++ Actor Framework (CAF) (http://actor-framework.org) + * C++ Actor Framework (CAF) version 0.14 (http://actor-framework.org) To build Bro from source, the following additional dependencies are required: * CMake 2.8 or greater (http://www.cmake.org) * Make - * C/C++ compiler with C++11 support + * C/C++ compiler with C++11 support (GCC 4.8+ or Clang 3.3+) * SWIG (http://www.swig.org) * Bison (GNU Parser Generator) * Flex (Fast Lexical Analyzer) @@ -47,9 +47,7 @@ To build Bro from source, the following additional dependencies are required: * zlib headers * Python -.. todo:: - - Update with instructions for installing CAF. +To install CAF, first download the source code of the required version from: https://github.com/actor-framework/actor-framework/releases To install the required dependencies, you can use: @@ -84,11 +82,11 @@ To install the required dependencies, you can use: "Preferences..." -> "Downloads" menus to install the "Command Line Tools" component). - OS X comes with all required dependencies except for CMake_ and SWIG_. + OS X comes with all required dependencies except for CMake_, SWIG_, and CAF. Distributions of these dependencies can likely be obtained from your - preferred Mac OS X package management system (e.g. MacPorts_, Fink_, - or Homebrew_). Specifically for MacPorts, the ``cmake``, ``swig``, - and ``swig-python`` packages provide the required dependencies. + preferred Mac OS X package management system (e.g. Homebrew_, MacPorts_, + or Fink_). Specifically for Homebrew, the ``cmake``, ``swig``, + and ``caf`` packages provide the required dependencies. Optional Dependencies @@ -101,6 +99,8 @@ build time: * sendmail (enables Bro and BroControl to send mail) * curl (used by a Bro script that implements active HTTP) * gperftools (tcmalloc is used to improve memory and CPU usage) + * jemalloc (http://www.canonware.com/jemalloc/) + * PF_RING (Linux only, see :doc:`Cluster Configuration <../configuration/index>`) * ipsumdump (for trace-summary; http://www.cs.ucla.edu/~kohler/ipsumdump) LibGeoIP is probably the most interesting and can be installed @@ -117,7 +117,7 @@ code forms. Using Pre-Built Binary Release Packages -======================================= +--------------------------------------- See the `bro downloads page`_ for currently supported/targeted platforms for binary releases and for installation instructions. @@ -138,13 +138,15 @@ platforms for binary releases and for installation instructions. The primary install prefix for binary packages is ``/opt/bro``. Installing from Source -====================== +---------------------- Bro releases are bundled into source packages for convenience and are -available on the `bro downloads page`_. Alternatively, the latest -Bro development version can be obtained through git repositories +available on the `bro downloads page`_. + +Alternatively, the latest Bro development version +can be obtained through git repositories hosted at ``git.bro.org``. See our `git development documentation -`_ for comprehensive +`_ for comprehensive information on Bro's use of git revision control, but the short story for downloading the full source code experience for Bro via git is: @@ -165,13 +167,23 @@ run ``./configure --help``): make make install +If the ``configure`` script fails, then it is most likely because it either +couldn't find a required dependency or it couldn't find a sufficiently new +version of a dependency. Assuming that you already installed all required +dependencies, then you may need to use one of the ``--with-*`` options +that can be given to the ``configure`` script to help it locate a dependency. + The default installation path is ``/usr/local/bro``, which would typically require root privileges when doing the ``make install``. A different -installation path can be chosen by specifying the ``--prefix`` option. -Note that ``/usr`` and ``/opt/bro`` are the +installation path can be chosen by specifying the ``configure`` script +``--prefix`` option. Note that ``/usr`` and ``/opt/bro`` are the standard prefixes for binary Bro packages to be installed, so those are typically not good choices unless you are creating such a package. +OpenBSD users, please see our `FAQ +`_ if you are having +problems installing Bro. + Depending on the Bro package you downloaded, there may be auxiliary tools and libraries available in the ``aux/`` directory. Some of them will be automatically built and installed along with Bro. There are @@ -180,10 +192,6 @@ turn off unwanted auxiliary projects that would otherwise be installed automatically. Finally, use ``make install-aux`` to install some of the other programs that are in the ``aux/bro-aux`` directory. -OpenBSD users, please see our `FAQ -`_ if you are having -problems installing Bro. - Finally, if you want to build the Bro documentation (not required, because all of the documentation for the latest Bro release is available on the Bro web site), there are instructions in ``doc/README`` in the source @@ -192,7 +200,7 @@ distribution. Configure the Run-Time Environment ================================== -Just remember that you may need to adjust your ``PATH`` environment variable +You may want to adjust your ``PATH`` environment variable according to the platform/shell/package you're using. For example: Bourne-Shell Syntax: diff --git a/doc/script-reference/attributes.rst b/doc/script-reference/attributes.rst index d37cc2a98a..fec72570d2 100644 --- a/doc/script-reference/attributes.rst +++ b/doc/script-reference/attributes.rst @@ -54,13 +54,16 @@ Here is a more detailed explanation of each attribute: .. bro:attr:: &redef - Allows for redefinition of initial values of global objects declared as - constant. - - In this example, the constant (assuming it is global) can be redefined - with a :bro:keyword:`redef` at some later point:: + Allows use of a :bro:keyword:`redef` to redefine initial values of + global variables (i.e., variables declared either :bro:keyword:`global` + or :bro:keyword:`const`). Example:: const clever = T &redef; + global cache_size = 256 &redef; + + Note that a variable declared "global" can also have its value changed + with assignment statements (doesn't matter if it has the "&redef" + attribute or not). .. bro:attr:: &priority diff --git a/doc/script-reference/statements.rst b/doc/script-reference/statements.rst index 1f5b388e7f..e2f93a5627 100644 --- a/doc/script-reference/statements.rst +++ b/doc/script-reference/statements.rst @@ -71,9 +71,11 @@ Statements Declarations ------------ -The following global declarations cannot occur within a function, hook, or -event handler. Also, these declarations cannot appear after any statements -that are outside of a function, hook, or event handler. +Declarations cannot occur within a function, hook, or event handler. + +Declarations must appear before any statements (except those statements +that are in a function, hook, or event handler) in the concatenation of +all loaded Bro scripts. .. bro:keyword:: module @@ -126,9 +128,12 @@ that are outside of a function, hook, or event handler. .. bro:keyword:: global Variables declared with the "global" keyword will be global. + If a type is not specified, then an initializer is required so that the type can be inferred. Likewise, if an initializer is not supplied, - then the type must be specified. Example:: + then the type must be specified. In some cases, when the type cannot + be correctly inferred, the type must be specified even when an + initializer is present. Example:: global pi = 3.14; global hosts: set[addr]; @@ -136,10 +141,11 @@ that are outside of a function, hook, or event handler. Variable declarations outside of any function, hook, or event handler are required to use this keyword (unless they are declared with the - :bro:keyword:`const` keyword). Definitions of functions, hooks, and - event handlers are not allowed to use the "global" - keyword (they already have global scope), except function declarations - where no function body is supplied use the "global" keyword. + :bro:keyword:`const` keyword instead). + + Definitions of functions, hooks, and event handlers are not allowed + to use the "global" keyword. However, function declarations (i.e., no + function body is provided) can use the "global" keyword. The scope of a global variable begins where the declaration is located, and extends through all remaining Bro scripts that are loaded (however, @@ -150,18 +156,22 @@ that are outside of a function, hook, or event handler. .. bro:keyword:: const A variable declared with the "const" keyword will be constant. + Variables declared as constant are required to be initialized at the - time of declaration. Example:: + time of declaration. Normally, the type is inferred from the initializer, + but the type can be explicitly specified. Example:: const pi = 3.14; const ssh_port: port = 22/tcp; - The value of a constant cannot be changed later (the only - exception is if the variable is global and has the :bro:attr:`&redef` - attribute, then its value can be changed only with a :bro:keyword:`redef`). + The value of a constant cannot be changed. The only exception is if the + variable is a global constant and has the :bro:attr:`&redef` + attribute, but even then its value can be changed only with a + :bro:keyword:`redef`. The scope of a constant is local if the declaration is in a function, hook, or event handler, and global otherwise. + Note that the "const" keyword cannot be used with either the "local" or "global" keywords (i.e., "const" replaces "local" and "global"). @@ -184,7 +194,8 @@ that are outside of a function, hook, or event handler. .. bro:keyword:: redef There are three ways that "redef" can be used: to change the value of - a global variable, to extend a record type or enum type, or to specify + a global variable (but only if it has the :bro:attr:`&redef` attribute), + to extend a record type or enum type, or to specify a new event handler body that replaces all those that were previously defined. @@ -237,13 +248,14 @@ that are outside of a function, hook, or event handler. Statements ---------- +Statements (except those contained within a function, hook, or event +handler) can appear only after all global declarations in the concatenation +of all loaded Bro scripts. + Each statement in a Bro script must be terminated with a semicolon (with a few exceptions noted below). An individual statement can span multiple lines. -All statements (except those contained within a function, hook, or event -handler) must appear after all global declarations. - Here are the statements that the Bro scripting language supports. .. bro:keyword:: add diff --git a/doc/script-reference/types.rst b/doc/script-reference/types.rst index cc601db75f..847e0f8fab 100644 --- a/doc/script-reference/types.rst +++ b/doc/script-reference/types.rst @@ -340,15 +340,18 @@ Here is a more detailed description of each type: table [ type^+ ] of type - where *type^+* is one or more types, separated by commas. - For example: + where *type^+* is one or more types, separated by commas. The + index type cannot be any of the following types: pattern, table, set, + vector, file, opaque, any. + + Here is an example of declaring a table indexed by "count" values + and yielding "string" values: .. code:: bro global a: table[count] of string; - declares a table indexed by "count" values and yielding - "string" values. The yield type can also be more complex: + The yield type can also be more complex: .. code:: bro @@ -441,7 +444,9 @@ Here is a more detailed description of each type: set [ type^+ ] - where *type^+* is one or more types separated by commas. + where *type^+* is one or more types separated by commas. The + index type cannot be any of the following types: pattern, table, set, + vector, file, opaque, any. Sets can be initialized by listing elements enclosed by curly braces: diff --git a/doc/scripting/data_struct_record_01.bro b/doc/scripting/data_struct_record_01.bro index a80d30faae..ab28501f96 100644 --- a/doc/scripting/data_struct_record_01.bro +++ b/doc/scripting/data_struct_record_01.bro @@ -4,7 +4,7 @@ type Service: record { rfc: count; }; -function print_service(serv: Service): string +function print_service(serv: Service) { print fmt("Service: %s(RFC%d)",serv$name, serv$rfc); diff --git a/doc/scripting/data_struct_record_02.bro b/doc/scripting/data_struct_record_02.bro index b10b3feac0..515c8a716c 100644 --- a/doc/scripting/data_struct_record_02.bro +++ b/doc/scripting/data_struct_record_02.bro @@ -9,7 +9,7 @@ type System: record { services: set[Service]; }; -function print_service(serv: Service): string +function print_service(serv: Service) { print fmt(" Service: %s(RFC%d)",serv$name, serv$rfc); @@ -17,7 +17,7 @@ function print_service(serv: Service): string print fmt(" port: %s", p); } -function print_system(sys: System): string +function print_system(sys: System) { print fmt("System: %s", sys$name); diff --git a/scripts/base/frameworks/input/main.bro b/scripts/base/frameworks/input/main.bro index fa766ba27b..3df418315f 100644 --- a/scripts/base/frameworks/input/main.bro +++ b/scripts/base/frameworks/input/main.bro @@ -1,18 +1,25 @@ ##! The input framework provides a way to read previously stored data either -##! as an event stream or into a bro table. +##! as an event stream or into a Bro table. module Input; export { type Event: enum { + ## New data has been imported. EVENT_NEW = 0, + ## Existing data has been changed. EVENT_CHANGED = 1, + ## Previously existing data has been removed. EVENT_REMOVED = 2, }; + ## Type that defines the input stream read mode. type Mode: enum { + ## Do not automatically reread the file after it has been read. MANUAL = 0, + ## Reread the entire file each time a change is found. REREAD = 1, + ## Read data from end of file each time new data is appended. STREAM = 2 }; @@ -24,20 +31,20 @@ export { ## Separator between fields. ## Please note that the separator has to be exactly one character long. - ## Can be overwritten by individual writers. + ## Individual readers can use a different value. const separator = "\t" &redef; ## Separator between set elements. ## Please note that the separator has to be exactly one character long. - ## Can be overwritten by individual writers. + ## Individual readers can use a different value. const set_separator = "," &redef; ## String to use for empty fields. - ## Can be overwritten by individual writers. + ## Individual readers can use a different value. const empty_field = "(empty)" &redef; ## String to use for an unset &optional field. - ## Can be overwritten by individual writers. + ## Individual readers can use a different value. const unset_field = "-" &redef; ## Flag that controls if the input framework accepts records @@ -47,11 +54,11 @@ export { ## abort. Defaults to false (abort). const accept_unsupported_types = F &redef; - ## TableFilter description type used for the `table` method. + ## A table input stream type used to send data to a Bro table. type TableDescription: record { # Common definitions for tables and events - ## String that allows the reader to find the source. + ## String that allows the reader to find the source of the data. ## For `READER_ASCII`, this is the filename. source: string; @@ -61,7 +68,8 @@ export { ## Read mode to use for this stream. mode: Mode &default=default_mode; - ## Descriptive name. Used to remove a stream at a later time. + ## Name of the input stream. This is used by some functions to + ## manipulate the stream. name: string; # Special definitions for tables @@ -73,31 +81,35 @@ export { idx: any; ## Record that defines the values used as the elements of the table. - ## If this is undefined, then *destination* has to be a set. + ## If this is undefined, then *destination* must be a set. val: any &optional; - ## Defines if the value of the table is a record (default), or a single value. - ## When this is set to false, then *val* can only contain one element. + ## Defines if the value of the table is a record (default), or a single + ## value. When this is set to false, then *val* can only contain one + ## element. want_record: bool &default=T; - ## The event that is raised each time a value is added to, changed in or removed - ## from the table. The event will receive an Input::Event enum as the first - ## argument, the *idx* record as the second argument and the value (record) as the - ## third argument. - ev: any &optional; # event containing idx, val as values. + ## The event that is raised each time a value is added to, changed in, + ## or removed from the table. The event will receive an + ## Input::TableDescription as the first argument, an Input::Event + ## enum as the second argument, the *idx* record as the third argument + ## and the value (record) as the fourth argument. + ev: any &optional; - ## Predicate function that can decide if an insertion, update or removal should - ## really be executed. Parameters are the same as for the event. If true is - ## returned, the update is performed. If false is returned, it is skipped. + ## Predicate function that can decide if an insertion, update or removal + ## should really be executed. Parameters have same meaning as for the + ## event. + ## If true is returned, the update is performed. If false is returned, + ## it is skipped. pred: function(typ: Input::Event, left: any, right: any): bool &optional; - ## A key/value table that will be passed on the reader. - ## Interpretation of the values is left to the writer, but + ## A key/value table that will be passed to the reader. + ## Interpretation of the values is left to the reader, but ## usually they will be used for configuration purposes. - config: table[string] of string &default=table(); + config: table[string] of string &default=table(); }; - ## EventFilter description type used for the `event` method. + ## An event input stream type used to send input data to a Bro event. type EventDescription: record { # Common definitions for tables and events @@ -116,19 +128,26 @@ export { # Special definitions for events - ## Record describing the fields to be retrieved from the source input. + ## Record type describing the fields to be retrieved from the input + ## source. fields: any; - ## If this is false, the event receives each value in fields as a separate argument. - ## If this is set to true (default), the event receives all fields in a single record value. + ## If this is false, the event receives each value in *fields* as a + ## separate argument. + ## If this is set to true (default), the event receives all fields in + ## a single record value. want_record: bool &default=T; - ## The event that is raised each time a new line is received from the reader. - ## The event will receive an Input::Event enum as the first element, and the fields as the following arguments. + ## The event that is raised each time a new line is received from the + ## reader. The event will receive an Input::EventDescription record + ## as the first argument, an Input::Event enum as the second + ## argument, and the fields (as specified in *fields*) as the following + ## arguments (this will either be a single record value containing + ## all fields, or each field value as a separate argument). ev: any; - ## A key/value table that will be passed on the reader. - ## Interpretation of the values is left to the writer, but + ## A key/value table that will be passed to the reader. + ## Interpretation of the values is left to the reader, but ## usually they will be used for configuration purposes. config: table[string] of string &default=table(); }; @@ -155,28 +174,29 @@ export { ## field will be the same value as the *source* field. name: string; - ## A key/value table that will be passed on the reader. - ## Interpretation of the values is left to the writer, but + ## A key/value table that will be passed to the reader. + ## Interpretation of the values is left to the reader, but ## usually they will be used for configuration purposes. config: table[string] of string &default=table(); }; - ## Create a new table input from a given source. + ## Create a new table input stream from a given source. ## ## description: `TableDescription` record describing the source. ## ## Returns: true on success. global add_table: function(description: Input::TableDescription) : bool; - ## Create a new event input from a given source. + ## Create a new event input stream from a given source. ## ## description: `EventDescription` record describing the source. ## ## Returns: true on success. global add_event: function(description: Input::EventDescription) : bool; - ## Create a new file analysis input from a given source. Data read from - ## the source is automatically forwarded to the file analysis framework. + ## Create a new file analysis input stream from a given source. Data read + ## from the source is automatically forwarded to the file analysis + ## framework. ## ## description: A record describing the source. ## @@ -199,7 +219,11 @@ export { ## Event that is called when the end of a data source has been reached, ## including after an update. - global end_of_data: event(name: string, source:string); + ## + ## name: Name of the input stream. + ## + ## source: String that identifies the data source (such as the filename). + global end_of_data: event(name: string, source: string); } @load base/bif/input.bif diff --git a/scripts/base/frameworks/input/readers/raw.bro b/scripts/base/frameworks/input/readers/raw.bro index b1e0fb6831..a1e95b71a1 100644 --- a/scripts/base/frameworks/input/readers/raw.bro +++ b/scripts/base/frameworks/input/readers/raw.bro @@ -11,7 +11,9 @@ export { ## ## name: name of the input stream. ## source: source of the input stream. - ## exit_code: exit code of the program, or number of the signal that forced the program to exit. - ## signal_exit: false when program exited normally, true when program was forced to exit by a signal. + ## exit_code: exit code of the program, or number of the signal that forced + ## the program to exit. + ## signal_exit: false when program exited normally, true when program was + ## forced to exit by a signal. global process_finished: event(name: string, source:string, exit_code:count, signal_exit:bool); } diff --git a/scripts/base/init-bare.bro b/scripts/base/init-bare.bro index 8b133f568c..2d8a20e7a3 100644 --- a/scripts/base/init-bare.bro +++ b/scripts/base/init-bare.bro @@ -349,7 +349,7 @@ type connection: record { ## The outer VLAN, if applicable, for this connection. vlan: int &optional; - ## The VLAN vlan, if applicable, for this connection. + ## The inner VLAN, if applicable, for this connection. inner_vlan: int &optional; }; @@ -2509,7 +2509,7 @@ global dns_skip_all_addl = T &redef; ## If a DNS request includes more than this many queries, assume it's non-DNS ## traffic and do not process it. Set to 0 to turn off this functionality. -global dns_max_queries = 5; +global dns_max_queries = 25 &redef; ## HTTP session statistics. ## diff --git a/scripts/base/protocols/conn/main.bro b/scripts/base/protocols/conn/main.bro index 12014d84a0..015c5520db 100644 --- a/scripts/base/protocols/conn/main.bro +++ b/scripts/base/protocols/conn/main.bro @@ -87,8 +87,8 @@ export { ## f packet with FIN bit set ## r packet with RST bit set ## c packet with a bad checksum - ## i inconsistent packet (FIN+RST bits both set) - ## q multi-flag packet (SYN+FIN or SYN+RST bits both set) + ## i inconsistent packet (e.g. FIN+RST bits set) + ## q multi-flag packet (SYN+FIN or SYN+RST bits set) ## ====== ==================================================== ## ## If the event comes from the originator, the letter is in diff --git a/scripts/policy/frameworks/files/hash-all-files.bro b/scripts/policy/frameworks/files/hash-all-files.bro index 74bea47bb9..f076abdd91 100644 --- a/scripts/policy/frameworks/files/hash-all-files.bro +++ b/scripts/policy/frameworks/files/hash-all-files.bro @@ -1,5 +1,7 @@ ##! Perform MD5 and SHA1 hashing on all files. +@load base/files/hash + event file_new(f: fa_file) { Files::add_analyzer(f, Files::ANALYZER_MD5); diff --git a/src/analyzer/protocol/sip/sip-protocol.pac b/src/analyzer/protocol/sip/sip-protocol.pac index ce26b8be95..15f07df44a 100644 --- a/src/analyzer/protocol/sip/sip-protocol.pac +++ b/src/analyzer/protocol/sip/sip-protocol.pac @@ -1,16 +1,6 @@ -enum ExpectBody { - BODY_EXPECTED, - BODY_NOT_EXPECTED, - BODY_MAYBE, -}; - type SIP_TOKEN = RE/[^()<>@,;:\\"\/\[\]?={} \t]+/; type SIP_WS = RE/[ \t]*/; -type SIP_COLON = RE/:/; -type SIP_TO_EOL = RE/[^\r\n]*/; -type SIP_EOL = RE/(\r\n){1,2}/; type SIP_URI = RE/[[:alnum:]@[:punct:]]+/; -type SIP_NL = RE/(\r\n)/; type SIP_PDU(is_orig: bool) = case is_orig of { true -> request: SIP_Request; @@ -18,14 +8,12 @@ type SIP_PDU(is_orig: bool) = case is_orig of { }; type SIP_Request = record { - request: SIP_RequestLine; - newline: SIP_NL; + request: SIP_RequestLine &oneline; msg: SIP_Message; }; type SIP_Reply = record { - reply: SIP_ReplyLine; - newline: SIP_NL; + reply: SIP_ReplyLine &oneline; msg: SIP_Message; }; @@ -34,7 +22,7 @@ type SIP_RequestLine = record { : SIP_WS; uri: SIP_URI; : SIP_WS; - version: SIP_Version; + version: SIP_Version &restofdata; } &oneline; type SIP_ReplyLine = record { @@ -42,7 +30,7 @@ type SIP_ReplyLine = record { : SIP_WS; status: SIP_Status; : SIP_WS; - reason: SIP_TO_EOL; + reason: bytestring &restofdata; } &oneline; type SIP_Status = record { @@ -52,7 +40,7 @@ type SIP_Status = record { }; type SIP_Version = record { - : "SIP/"; + : "SIP/"; vers_str: RE/[0-9]+\.[0-9]+/; } &let { vers_num: double = bytestring_to_double(vers_str); @@ -69,11 +57,10 @@ type SIP_HEADER_NAME = RE/[^: \t]+/; type SIP_Header = record { name: SIP_HEADER_NAME; : SIP_WS; - : SIP_COLON; + : ":"; : SIP_WS; - value: SIP_TO_EOL; - : SIP_EOL; -} &oneline &byteorder=bigendian; + value: bytestring &restofdata; +} &oneline; type SIP_Body = record { body: bytestring &length = $context.flow.get_content_length(); diff --git a/src/analyzer/protocol/sip/sip.pac b/src/analyzer/protocol/sip/sip.pac index f527a90117..15addb8c1e 100644 --- a/src/analyzer/protocol/sip/sip.pac +++ b/src/analyzer/protocol/sip/sip.pac @@ -21,7 +21,7 @@ connection SIP_Conn(bro_analyzer: BroAnalyzer) { %include sip-protocol.pac flow SIP_Flow(is_orig: bool) { - datagram = SIP_PDU(is_orig) withcontext(connection, this); + flowunit = SIP_PDU(is_orig) withcontext(connection, this); }; %include sip-analyzer.pac diff --git a/src/analyzer/protocol/sip/sip_TCP.pac b/src/analyzer/protocol/sip/sip_TCP.pac index 5546d28ece..2e51675dea 100644 --- a/src/analyzer/protocol/sip/sip_TCP.pac +++ b/src/analyzer/protocol/sip/sip_TCP.pac @@ -24,7 +24,7 @@ connection SIP_Conn(bro_analyzer: BroAnalyzer) { %include sip-protocol.pac flow SIP_Flow(is_orig: bool) { - datagram = SIP_PDU(is_orig) withcontext(connection, this); + flowunit = SIP_PDU(is_orig) withcontext(connection, this); }; %include sip-analyzer.pac diff --git a/src/bro.bif b/src/bro.bif index 04394434b3..b0465b9609 100644 --- a/src/bro.bif +++ b/src/bro.bif @@ -2725,13 +2725,12 @@ function hexstr_to_bytestring%(hexstr: string%): string ## ## s: The string to encode. ## -## a: An optional custom alphabet. The empty string indicates the default alphabet. -## If given, the length of *a* must be 64. For example, a custom alphabet could be -## ``"!#$%&/(),-.:;<>@[]^ `_{|}~abcdefghijklmnopqrstuvwxyz0123456789+?"``. +## a: An optional custom alphabet. The empty string indicates the default +## alphabet. If given, the string must consist of 64 unique characters. ## ## Returns: The encoded version of *s*. ## -## .. bro:see:: decode_base64 decode_base64_conn +## .. bro:see:: decode_base64 function encode_base64%(s: string, a: string &default=""%): string %{ BroString* t = encode_base64(s->AsString(), a->AsString()); @@ -2749,13 +2748,12 @@ function encode_base64%(s: string, a: string &default=""%): string ## ## s: The string to encode. ## -## a: An optional custom alphabet. The empty string indicates the default alphabet. -## If given, the length of *a* must be 64. For example, a custom alphabet could be -## ``"!#$%&/(),-.:;<>@[]^ `_{|}~abcdefghijklmnopqrstuvwxyz0123456789+?"``. +## a: The custom alphabet. The string must consist of 64 unique +## characters. The empty string indicates the default alphabet. ## ## Returns: The encoded version of *s*. ## -## .. bro:see:: encode_base64 decode_base64 decode_base64_conn +## .. bro:see:: encode_base64 function encode_base64_custom%(s: string, a: string%): string &deprecated %{ BroString* t = encode_base64(s->AsString(), a->AsString()); @@ -2772,13 +2770,12 @@ function encode_base64_custom%(s: string, a: string%): string &deprecated ## ## s: The Base64-encoded string. ## -## a: An optional custom alphabet. The empty string indicates the default alphabet. -## If given, the length of *a* must be 64. For example, a custom alphabet could be -## ``"!#$%&/(),-.:;<>@[]^ `_{|}~abcdefghijklmnopqrstuvwxyz0123456789+?"``. +## a: An optional custom alphabet. The empty string indicates the default +## alphabet. If given, the string must consist of 64 unique characters. ## ## Returns: The decoded version of *s*. ## -## .. bro:see:: decode_base64_intern encode_base64 +## .. bro:see:: decode_base64_conn encode_base64 function decode_base64%(s: string, a: string &default=""%): string %{ BroString* t = decode_base64(s->AsString(), a->AsString()); @@ -2793,19 +2790,18 @@ function decode_base64%(s: string, a: string &default=""%): string ## Decodes a Base64-encoded string that was derived from processing a connection. ## If an error is encountered decoding the string, that will be logged to -## ``weird.log`` with the associated connection, +## ``weird.log`` with the associated connection. ## ## cid: The identifier of the connection that the encoding originates from. ## ## s: The Base64-encoded string. ## -## a: An optional custom alphabet. The empty string indicates the default alphabet. -## If given, the length of *a* must be 64. For example, a custom alphabet could be -## ``"!#$%&/(),-.:;<>@[]^ `_{|}~abcdefghijklmnopqrstuvwxyz0123456789+?"``. +## a: An optional custom alphabet. The empty string indicates the default +## alphabet. If given, the string must consist of 64 unique characters. ## ## Returns: The decoded version of *s*. ## -## .. bro:see:: decode_base64 encode_base64_intern +## .. bro:see:: decode_base64 function decode_base64_conn%(cid: conn_id, s: string, a: string &default=""%): string %{ Connection* conn = sessions->FindConnection(cid); @@ -2829,13 +2825,12 @@ function decode_base64_conn%(cid: conn_id, s: string, a: string &default=""%): s ## ## s: The Base64-encoded string. ## -## a: The custom alphabet. The empty string indicates the default alphabet. The -## length of *a* must be 64. For example, a custom alphabet could be -## ``"!#$%&/(),-.:;<>@[]^ `_{|}~abcdefghijklmnopqrstuvwxyz0123456789+?"``. +## a: The custom alphabet. The string must consist of 64 unique characters. +## The empty string indicates the default alphabet. ## ## Returns: The decoded version of *s*. ## -## .. bro:see:: decode_base64 decode_base64_conn encode_base64 +## .. bro:see:: decode_base64 decode_base64_conn function decode_base64_custom%(s: string, a: string%): string &deprecated %{ BroString* t = decode_base64(s->AsString(), a->AsString()); diff --git a/src/file_analysis/analyzer/x509/X509.cc b/src/file_analysis/analyzer/x509/X509.cc index d9604740a7..e8ea5cb7b4 100644 --- a/src/file_analysis/analyzer/x509/X509.cc +++ b/src/file_analysis/analyzer/x509/X509.cc @@ -521,7 +521,7 @@ double file_analysis::X509::GetTimeFromAsn1(const ASN1_TIME* atime, const char* const char *fid = arg_fid ? arg_fid : ""; time_t lResult = 0; - char lBuffer[24]; + char lBuffer[26]; char* pBuffer = lBuffer; const char *pString = (const char *) atime->data; @@ -535,16 +535,35 @@ double file_analysis::X509::GetTimeFromAsn1(const ASN1_TIME* atime, const char* return 0; } + if ( pString[remaining-1] != 'Z' ) + { + // not valid according to RFC 2459 4.1.2.5.1 + reporter->Weird(fmt("Could not parse UTC time in non-YY-format in X509 certificate (x509 %s)", fid)); + return 0; + } + + // year is first two digits in YY format. Buffer expects YYYY format. + if ( pString[0] - '0' < 50 ) // RFC 2459 4.1.2.5.1 + { + *(pBuffer++) = '2'; + *(pBuffer++) = '0'; + } + else + { + *(pBuffer++) = '1'; + *(pBuffer++) = '9'; + } + memcpy(pBuffer, pString, 10); pBuffer += 10; pString += 10; remaining -= 10; } - - else + else if ( atime->type == V_ASN1_GENERALIZEDTIME ) { // generalized time. We apparently ignore the YYYYMMDDHH case // for now and assume we always have minutes and seconds. + // This should be ok because it is specified as a requirement in RFC 2459 4.1.2.5.2 if ( remaining < 12 || remaining > 23 ) { @@ -557,6 +576,11 @@ double file_analysis::X509::GetTimeFromAsn1(const ASN1_TIME* atime, const char* pString += 12; remaining -= 12; } + else + { + reporter->Weird(fmt("Invalid time type in X509 certificate (fuid %s)", fid)); + return 0; + } if ( (remaining == 0) || (*pString == 'Z') || (*pString == '-') || (*pString == '+') ) { @@ -620,15 +644,15 @@ double file_analysis::X509::GetTimeFromAsn1(const ASN1_TIME* atime, const char* } tm lTime; - lTime.tm_sec = ((lBuffer[10] - '0') * 10) + (lBuffer[11] - '0'); - lTime.tm_min = ((lBuffer[8] - '0') * 10) + (lBuffer[9] - '0'); - lTime.tm_hour = ((lBuffer[6] - '0') * 10) + (lBuffer[7] - '0'); - lTime.tm_mday = ((lBuffer[4] - '0') * 10) + (lBuffer[5] - '0'); - lTime.tm_mon = (((lBuffer[2] - '0') * 10) + (lBuffer[3] - '0')) - 1; - lTime.tm_year = ((lBuffer[0] - '0') * 10) + (lBuffer[1] - '0'); + lTime.tm_sec = ((lBuffer[12] - '0') * 10) + (lBuffer[13] - '0'); + lTime.tm_min = ((lBuffer[10] - '0') * 10) + (lBuffer[11] - '0'); + lTime.tm_hour = ((lBuffer[8] - '0') * 10) + (lBuffer[9] - '0'); + lTime.tm_mday = ((lBuffer[6] - '0') * 10) + (lBuffer[7] - '0'); + lTime.tm_mon = (((lBuffer[4] - '0') * 10) + (lBuffer[5] - '0')) - 1; + lTime.tm_year = (lBuffer[0] - '0') * 1000 + (lBuffer[1] - '0') * 100 + ((lBuffer[2] - '0') * 10) + (lBuffer[3] - '0'); - if ( lTime.tm_year < 50 ) - lTime.tm_year += 100; // RFC 2459 + if ( lTime.tm_year > 1900) + lTime.tm_year -= 1900; lTime.tm_wday = 0; lTime.tm_yday = 0; diff --git a/src/input/readers/raw/Raw.cc b/src/input/readers/raw/Raw.cc index 2aae96abf7..76d8958fea 100644 --- a/src/input/readers/raw/Raw.cc +++ b/src/input/readers/raw/Raw.cc @@ -302,8 +302,10 @@ bool Raw::OpenInput() if ( offset ) { - int whence = (offset > 0) ? SEEK_SET : SEEK_END; - if ( fseek(file, offset, whence) < 0 ) + int whence = (offset >= 0) ? SEEK_SET : SEEK_END; + int64_t pos = (offset >= 0) ? offset : offset + 1; // we want -1 to be the end of the file + + if ( fseek(file, pos, whence) < 0 ) { char buf[256]; strerror_r(errno, buf, sizeof(buf)); @@ -395,8 +397,6 @@ bool Raw::DoInit(const ReaderInfo& info, int num_fields, const Field* const* fie { string offset_s = it->second; offset = strtoll(offset_s.c_str(), 0, 10); - if ( offset < 0 ) - offset++; // we want -1 to be the end of the file } else if ( it != info.config.end() ) { diff --git a/testing/btest/Baseline/core.x509-generalizedtime/output b/testing/btest/Baseline/core.x509-generalizedtime/output new file mode 100644 index 0000000000..349551efe5 --- /dev/null +++ b/testing/btest/Baseline/core.x509-generalizedtime/output @@ -0,0 +1,16 @@ +----- x509_certificate ---- +serial: 03E8 +not_valid_before: 2015-09-01-13:33:37.000000000 (epoch: 1441114417.0) +not_valid_after : 2025-09-01-13:33:37.000000000 (epoch: 1756733617.0) +----- x509_certificate ---- +serial: 99FAA8037A4EB2FAEF84EB5E55D5B8C8 +not_valid_before: 2011-05-04-00:00:00.000000000 (epoch: 1304467200.0) +not_valid_after : 2016-07-04-23:59:59.000000000 (epoch: 1467676799.0) +----- x509_certificate ---- +serial: 1690C329B6780607511F05B0344846CB +not_valid_before: 2010-04-16-00:00:00.000000000 (epoch: 1271376000.0) +not_valid_after : 2020-05-30-10:48:38.000000000 (epoch: 1590835718.0) +----- x509_certificate ---- +serial: 01 +not_valid_before: 2000-05-30-10:48:38.000000000 (epoch: 959683718.0) +not_valid_after : 2020-05-30-10:48:38.000000000 (epoch: 1590835718.0) diff --git a/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_broker_connecting-connector_bro/output b/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_broker_connecting-connector_bro/output index 0953d88a3e..042b8999f3 100644 --- a/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_broker_connecting-connector_bro/output +++ b/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_broker_connecting-connector_bro/output @@ -2,7 +2,6 @@ connecting-connector.bro - const broker_port: port = 9999/tcp &redef; redef exit_only_after_terminate = T; redef BrokerComm::endpoint_name = "connector"; diff --git a/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_broker_connecting-listener_bro/output b/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_broker_connecting-listener_bro/output index 2879beb396..33e3df2330 100644 --- a/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_broker_connecting-listener_bro/output +++ b/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_broker_connecting-listener_bro/output @@ -2,7 +2,6 @@ connecting-listener.bro - const broker_port: port = 9999/tcp &redef; redef exit_only_after_terminate = T; redef BrokerComm::endpoint_name = "listener"; diff --git a/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_broker_events-listener_bro/output b/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_broker_events-listener_bro/output index 59e697601b..9f004692cb 100644 --- a/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_broker_events-listener_bro/output +++ b/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_broker_events-listener_bro/output @@ -2,7 +2,6 @@ events-listener.bro - const broker_port: port = 9999/tcp &redef; redef exit_only_after_terminate = T; redef BrokerComm::endpoint_name = "listener"; diff --git a/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_broker_printing-listener_bro/output b/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_broker_printing-listener_bro/output index 9cb48a0528..fb416612ab 100644 --- a/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_broker_printing-listener_bro/output +++ b/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_broker_printing-listener_bro/output @@ -2,7 +2,6 @@ printing-listener.bro - const broker_port: port = 9999/tcp &redef; redef exit_only_after_terminate = T; redef BrokerComm::endpoint_name = "listener"; diff --git a/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_broker_testlog_bro/output b/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_broker_testlog_bro/output index da2261ebc4..c87fc3cd6f 100644 --- a/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_broker_testlog_bro/output +++ b/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_broker_testlog_bro/output @@ -2,7 +2,6 @@ testlog.bro - module Test; export { diff --git a/testing/btest/Baseline/doc.sphinx.include-doc_scripting_data_struct_record_01_bro/output b/testing/btest/Baseline/doc.sphinx.include-doc_scripting_data_struct_record_01_bro/output index ea390412f6..e67783fdeb 100644 --- a/testing/btest/Baseline/doc.sphinx.include-doc_scripting_data_struct_record_01_bro/output +++ b/testing/btest/Baseline/doc.sphinx.include-doc_scripting_data_struct_record_01_bro/output @@ -8,7 +8,7 @@ type Service: record { rfc: count; }; -function print_service(serv: Service): string +function print_service(serv: Service) { print fmt("Service: %s(RFC%d)",serv$name, serv$rfc); diff --git a/testing/btest/Baseline/doc.sphinx.include-doc_scripting_data_struct_record_02_bro/output b/testing/btest/Baseline/doc.sphinx.include-doc_scripting_data_struct_record_02_bro/output index 143e6c5672..04da3522f2 100644 --- a/testing/btest/Baseline/doc.sphinx.include-doc_scripting_data_struct_record_02_bro/output +++ b/testing/btest/Baseline/doc.sphinx.include-doc_scripting_data_struct_record_02_bro/output @@ -13,7 +13,7 @@ type System: record { services: set[Service]; }; -function print_service(serv: Service): string +function print_service(serv: Service) { print fmt(" Service: %s(RFC%d)",serv$name, serv$rfc); @@ -21,7 +21,7 @@ function print_service(serv: Service): string print fmt(" port: %s", p); } -function print_system(sys: System): string +function print_system(sys: System) { print fmt("System: %s", sys$name); diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.raw.offset/out b/testing/btest/Baseline/scripts.base.frameworks.input.raw.offset/out index 3af2451db9..c8dd06805b 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.input.raw.offset/out +++ b/testing/btest/Baseline/scripts.base.frameworks.input.raw.offset/out @@ -1,2 +1,3 @@ fkh:KH;fdkncv;ISEUp34:Fkdj;YVpIODhfDF F +hi diff --git a/testing/btest/Baseline/scripts.base.protocols.sip.wireshark/sip.log b/testing/btest/Baseline/scripts.base.protocols.sip.wireshark/sip.log index 19f05ec1b9..047fa4e2d1 100644 --- a/testing/btest/Baseline/scripts.base.protocols.sip.wireshark/sip.log +++ b/testing/btest/Baseline/scripts.base.protocols.sip.wireshark/sip.log @@ -3,7 +3,7 @@ #empty_field (empty) #unset_field - #path sip -#open 2015-04-30-03-33-33 +#open 2015-09-03-21-02-33 #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method uri date request_from request_to response_from response_to reply_to call_id seq subject request_path response_path user_agent status_code status_msg warning request_body_len response_body_len content_type #types time string addr port addr port count string string string string string string string string string string string vector[string] vector[string] string count string string string string string 1120469572.844249 CXWv6p3arKYeMETxOg 192.168.1.2 5060 212.242.33.35 5060 0 REGISTER sip:sip.cybercity.dk - ;tag=00-04092-1701af62-120c67172 - 578222729-4665d775@578222732-4665d772 68 REGISTER - SIP/2.0/UDP 192.168.1.2 SIP/2.0/UDP 192.168.1.2;received=80.230.219.70;rport=5060 Nero SIPPS IP Phone Version 2.0.51.16 401 Unauthorized - 0 0 - @@ -37,8 +37,9 @@ 1120470900.060556 CIPOse170MGiRM1Qf4 192.168.1.2 5060 212.242.33.35 5060 0 ACK sip:0097239287044@sip.cybercity.dk - "arik" ;tag=00-04083-1701ba17-57d493ef5 - - - 24487391-449bf2a0@192.168.1.2 2 ACK - SIP/2.0/UDP 192.168.1.2 (empty) - - - - 0 - - 1120470966.443914 C7XEbhP654jzLoe3a 192.168.1.2 5060 212.242.33.35 5060 0 INVITE sip:35104724@sip.cybercity.dk - "arik" "arik" ;tag=00-04079-1701ba6f-3e08e2f66 - 11894297-4432a9f8@192.168.1.2 1 INVITE - SIP/2.0/UDP 192.168.1.2:5060 SIP/2.0/UDP 192.168.1.2:5060;received=80.230.219.70;rport=5060 Nero SIPPS IP Phone Version 2.0.51.16 407 authentication required - 270 0 - 1120470966.606422 C7XEbhP654jzLoe3a 192.168.1.2 5060 212.242.33.35 5060 0 INVITE sip:35104724@sip.cybercity.dk Mon, 04 Jul 2005 09:56:06 GMT "arik" "arik" - 11894297-4432a9f8@192.168.1.2 2 INVITE - SIP/2.0/UDP 192.168.1.2:5060,SIP/2.0/UDP 192.168.1.2 SIP/2.0/UDP 192.168.1.2;received=80.230.219.70;rport=5060 Nero SIPPS IP Phone Version 2.0.51.16 100 Trying - 270 0 - +1120470966.606422 C7XEbhP654jzLoe3a 192.168.1.2 5060 212.242.33.35 5060 0 INVITE sip:35104724@sip.cybercity.dk Mon, 04 Jul 2005 09:56:06 GMT "arik" "arik" ;tag=00-04075-1701baa2-2dfdf7c21 - 11894297-4432a9f8@192.168.1.2 2 INVITE - SIP/2.0/UDP 192.168.1.2:5060,SIP/2.0/UDP 192.168.1.2 SIP/2.0/UDP 192.168.1.2;received=80.230.219.70;rport=5060,SIP/2.0/UDP 192.168.1.2;received=80.230.219.70;rport=5060 Nero SIPPS IP Phone Version 2.0.51.16 183 In band info available - 270 199 application/sdp 1120470966.606422 C7XEbhP654jzLoe3a 192.168.1.2 5060 212.242.33.35 5060 0 INVITE sip:35104724@sip.cybercity.dk Mon, 04 Jul 2005 09:56:06 GMT "arik" "arik" ;tag=00-04075-1701baa2-2dfdf7c21 - 11894297-4432a9f8@192.168.1.2 2 INVITE - SIP/2.0/UDP 192.168.1.2:5060,SIP/2.0/UDP 192.168.1.2 SIP/2.0/UDP 192.168.1.2;received=80.230.219.70;rport=5060,SIP/2.0/UDP 192.168.1.2;received=80.230.219.70;rport=5060,SIP/2.0/UDP 192.168.1.2;received=80.230.219.70;rport=5060 Nero SIPPS IP Phone Version 2.0.51.16 480 Error - 270 0 application/sdp 1120470984.353086 C7XEbhP654jzLoe3a 192.168.1.2 5060 212.242.33.35 5060 0 REGISTER sip:sip.cybercity.dk - ;tag=00-04074-1701bac9-1daa0b4c5 - 29858147-465b0752@29858051-465b07b2 5 REGISTER - SIP/2.0/UDP 192.168.1.2,SIP/2.0/UDP 192.168.1.2 SIP/2.0/UDP 192.168.1.2;received=80.230.219.70;rport=5060 Nero SIPPS IP Phone Version 2.0.51.16 401 Unauthorized - 0 0 - 1120471018.723316 C7XEbhP654jzLoe3a 192.168.1.2 5060 212.242.33.35 5060 0 REGISTER sip:sip.cybercity.dk - - 29858147-465b0752@29858051-465b07b2 6 REGISTER - SIP/2.0/UDP 192.168.1.2 SIP/2.0/UDP 192.168.1.2;received=80.230.219.70;rport=5060 Nero SIPPS IP Phone Version 2.0.51.16 100 Trying - 0 0 - 1120471018.723316 C7XEbhP654jzLoe3a 192.168.1.2 5060 212.242.33.35 5060 0 REGISTER sip:sip.cybercity.dk - ;tag=00-04087-1701bae7-76fb74995 - 29858147-465b0752@29858051-465b07b2 6 REGISTER - SIP/2.0/UDP 192.168.1.2 SIP/2.0/UDP 192.168.1.2;received=80.230.219.70;rport=5060,SIP/2.0/UDP 192.168.1.2;received=80.230.219.70;rport=5060 Nero SIPPS IP Phone Version 2.0.51.16 200 OK - 0 0 - -#close 2015-04-30-03-33-33 +#close 2015-09-03-21-02-33 diff --git a/testing/btest/Traces/tls/x509-generalizedtime.pcap b/testing/btest/Traces/tls/x509-generalizedtime.pcap new file mode 100644 index 0000000000..6f026034df Binary files /dev/null and b/testing/btest/Traces/tls/x509-generalizedtime.pcap differ diff --git a/testing/btest/core/x509-generalizedtime.bro b/testing/btest/core/x509-generalizedtime.bro new file mode 100644 index 0000000000..b69ab31743 --- /dev/null +++ b/testing/btest/core/x509-generalizedtime.bro @@ -0,0 +1,10 @@ +# @TEST-EXEC: bro -C -r $TRACES/tls/x509-generalizedtime.pcap %INPUT >>output 2>&1 +# @TEST-EXEC: bro -C -r $TRACES/tls/tls1.2.trace %INPUT >>output 2>&1 +# @TEST-EXEC: btest-diff output +event x509_certificate(f: fa_file, cert_ref: opaque of x509, cert: X509::Certificate) + { + print "----- x509_certificate ----"; + print fmt("serial: %s", cert$serial); + print fmt("not_valid_before: %T (epoch: %s)", cert$not_valid_before, cert$not_valid_before); + print fmt("not_valid_after : %T (epoch: %s)", cert$not_valid_after, cert$not_valid_after); + } diff --git a/testing/btest/doc/sphinx/include-doc_frameworks_broker_connecting-connector_bro.btest b/testing/btest/doc/sphinx/include-doc_frameworks_broker_connecting-connector_bro.btest index 0953d88a3e..042b8999f3 100644 --- a/testing/btest/doc/sphinx/include-doc_frameworks_broker_connecting-connector_bro.btest +++ b/testing/btest/doc/sphinx/include-doc_frameworks_broker_connecting-connector_bro.btest @@ -2,7 +2,6 @@ connecting-connector.bro - const broker_port: port = 9999/tcp &redef; redef exit_only_after_terminate = T; redef BrokerComm::endpoint_name = "connector"; diff --git a/testing/btest/doc/sphinx/include-doc_frameworks_broker_connecting-listener_bro.btest b/testing/btest/doc/sphinx/include-doc_frameworks_broker_connecting-listener_bro.btest index 2879beb396..33e3df2330 100644 --- a/testing/btest/doc/sphinx/include-doc_frameworks_broker_connecting-listener_bro.btest +++ b/testing/btest/doc/sphinx/include-doc_frameworks_broker_connecting-listener_bro.btest @@ -2,7 +2,6 @@ connecting-listener.bro - const broker_port: port = 9999/tcp &redef; redef exit_only_after_terminate = T; redef BrokerComm::endpoint_name = "listener"; diff --git a/testing/btest/doc/sphinx/include-doc_frameworks_broker_events-listener_bro.btest b/testing/btest/doc/sphinx/include-doc_frameworks_broker_events-listener_bro.btest index 59e697601b..9f004692cb 100644 --- a/testing/btest/doc/sphinx/include-doc_frameworks_broker_events-listener_bro.btest +++ b/testing/btest/doc/sphinx/include-doc_frameworks_broker_events-listener_bro.btest @@ -2,7 +2,6 @@ events-listener.bro - const broker_port: port = 9999/tcp &redef; redef exit_only_after_terminate = T; redef BrokerComm::endpoint_name = "listener"; diff --git a/testing/btest/doc/sphinx/include-doc_frameworks_broker_printing-listener_bro.btest b/testing/btest/doc/sphinx/include-doc_frameworks_broker_printing-listener_bro.btest index 9cb48a0528..fb416612ab 100644 --- a/testing/btest/doc/sphinx/include-doc_frameworks_broker_printing-listener_bro.btest +++ b/testing/btest/doc/sphinx/include-doc_frameworks_broker_printing-listener_bro.btest @@ -2,7 +2,6 @@ printing-listener.bro - const broker_port: port = 9999/tcp &redef; redef exit_only_after_terminate = T; redef BrokerComm::endpoint_name = "listener"; diff --git a/testing/btest/doc/sphinx/include-doc_frameworks_broker_testlog_bro.btest b/testing/btest/doc/sphinx/include-doc_frameworks_broker_testlog_bro.btest index da2261ebc4..c87fc3cd6f 100644 --- a/testing/btest/doc/sphinx/include-doc_frameworks_broker_testlog_bro.btest +++ b/testing/btest/doc/sphinx/include-doc_frameworks_broker_testlog_bro.btest @@ -2,7 +2,6 @@ testlog.bro - module Test; export { diff --git a/testing/btest/doc/sphinx/include-doc_scripting_data_struct_record_01_bro.btest b/testing/btest/doc/sphinx/include-doc_scripting_data_struct_record_01_bro.btest index ea390412f6..e67783fdeb 100644 --- a/testing/btest/doc/sphinx/include-doc_scripting_data_struct_record_01_bro.btest +++ b/testing/btest/doc/sphinx/include-doc_scripting_data_struct_record_01_bro.btest @@ -8,7 +8,7 @@ type Service: record { rfc: count; }; -function print_service(serv: Service): string +function print_service(serv: Service) { print fmt("Service: %s(RFC%d)",serv$name, serv$rfc); diff --git a/testing/btest/doc/sphinx/include-doc_scripting_data_struct_record_02_bro.btest b/testing/btest/doc/sphinx/include-doc_scripting_data_struct_record_02_bro.btest index 143e6c5672..04da3522f2 100644 --- a/testing/btest/doc/sphinx/include-doc_scripting_data_struct_record_02_bro.btest +++ b/testing/btest/doc/sphinx/include-doc_scripting_data_struct_record_02_bro.btest @@ -13,7 +13,7 @@ type System: record { services: set[Service]; }; -function print_service(serv: Service): string +function print_service(serv: Service) { print fmt(" Service: %s(RFC%d)",serv$name, serv$rfc); @@ -21,7 +21,7 @@ function print_service(serv: Service): string print fmt(" port: %s", p); } -function print_system(sys: System): string +function print_system(sys: System) { print fmt("System: %s", sys$name); diff --git a/testing/btest/plugins/writer.bro b/testing/btest/plugins/writer.bro index 8cecff6843..732d726fd7 100644 --- a/testing/btest/plugins/writer.bro +++ b/testing/btest/plugins/writer.bro @@ -4,5 +4,5 @@ # @TEST-EXEC: BRO_PLUGIN_PATH=`pwd` bro -NN Demo::Foo >>output # @TEST-EXEC: echo === >>output # @TEST-EXEC: BRO_PLUGIN_PATH=`pwd` bro -r $TRACES/socks.trace Log::default_writer=Log::WRITER_FOO %INPUT | sort >>output -# @TEST-EXEC: TEST_DIFF_CANONIFIER=diff-remove-timestamps btest-diff output +# @TEST-EXEC: btest-diff output diff --git a/testing/btest/scripts/base/frameworks/input/raw/offset.bro b/testing/btest/scripts/base/frameworks/input/raw/offset.bro index 8161785fdd..5ab2d84655 100644 --- a/testing/btest/scripts/base/frameworks/input/raw/offset.bro +++ b/testing/btest/scripts/base/frameworks/input/raw/offset.bro @@ -1,5 +1,8 @@ +# @TEST-EXEC: cp input.log input2.log # @TEST-EXEC: btest-bg-run bro bro -b %INPUT -# @TEST-EXEC: btest-bg-wait 5 +# @TEST-EXEC: sleep 2 +# @TEST-EXEC: echo "hi" >> input2.log +# @TEST-EXEC: btest-bg-wait 10 # @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-sort btest-diff out @TEST-START-FILE input.log @@ -7,6 +10,7 @@ sdfkh:KH;fdkncv;ISEUp34:Fkdj;YVpIODhfDF @TEST-END-FILE redef exit_only_after_terminate = T; +@load base/frameworks/communication # keep network time running global outfile: file; global try: count; @@ -21,9 +25,8 @@ event line(description: Input::EventDescription, tpe: Input::Event, s: string) { print outfile, s; try = try + 1; - if ( try == 2 ) + if ( try == 3 ) { - Input::remove("input"); close(outfile); terminate(); } @@ -39,7 +42,11 @@ event bro_init() local config_strings_two: table[string] of string = { ["offset"] = "-3", # 2 characters before end, last char is newline. }; + local config_strings_three: table[string] of string = { + ["offset"] = "-1", # End of file + }; Input::add_event([$source="../input.log", $config=config_strings, $reader=Input::READER_RAW, $mode=Input::STREAM, $name="input", $fields=Val, $ev=line, $want_record=F]); Input::add_event([$source="../input.log", $config=config_strings_two, $reader=Input::READER_RAW, $mode=Input::STREAM, $name="input2", $fields=Val, $ev=line, $want_record=F]); + Input::add_event([$source="../input2.log", $config=config_strings_three, $reader=Input::READER_RAW, $mode=Input::STREAM, $name="input3", $fields=Val, $ev=line, $want_record=F]); } diff --git a/testing/scripts/diff-canon-notice-policy b/testing/scripts/diff-canon-notice-policy deleted file mode 100755 index f05abaa103..0000000000 --- a/testing/scripts/diff-canon-notice-policy +++ /dev/null @@ -1,10 +0,0 @@ -#! /usr/bin/awk -f -# -# A diff canonifier that removes the priorities in notice_policy.log. - -/^#/ && $2 == "notice_policy" { filter = 1; } - -filter == 1 && /^[^#]/ { sub("^[0-9]*", "X"); } - -{ print; } - diff --git a/testing/scripts/diff-canonifier-external b/testing/scripts/diff-canonifier-external index ee6405b3a8..611d7c7baf 100755 --- a/testing/scripts/diff-canonifier-external +++ b/testing/scripts/diff-canonifier-external @@ -18,7 +18,6 @@ fi | `dirname $0`/diff-remove-uids \ | `dirname $0`/diff-remove-file-ids \ | `dirname $0`/diff-remove-x509-names \ - | `dirname $0`/diff-canon-notice-policy \ | `dirname $0`/diff-sort \ | eval $addl diff --git a/testing/scripts/diff-remove-fields b/testing/scripts/diff-remove-fields index 7f18748a5f..3b20425f72 100755 --- a/testing/scripts/diff-remove-fields +++ b/testing/scripts/diff-remove-fields @@ -4,8 +4,8 @@ # prefix. if [ $# != 1 ]; then - echo "usage: `basename $0` " - exit 1 + echo "usage: `basename $0` " + exit 1 fi awk -v "PREFIX=$1" ' @@ -18,17 +18,15 @@ BEGIN { FS="\t"; OFS="\t"; } if ( index($i, PREFIX) == 1 ) rem[i-1] = 1; } - print; - next; +} + +/^[^#]/ { + for ( i in rem ) + # Mark that it is set, but ignore content. + $i = "+"; } { - for ( i in rem ) - # Mark that it iss set, but ignore content. - $i = "+"; - print; } - ' - diff --git a/testing/scripts/diff-remove-file-ids b/testing/scripts/diff-remove-file-ids index 965a74442e..d6c3e7c813 100755 --- a/testing/scripts/diff-remove-file-ids +++ b/testing/scripts/diff-remove-file-ids @@ -1,7 +1,8 @@ -#! /usr/bin/awk -f +#! /usr/bin/env bash # # A diff canonifier that removes all file IDs from files.log +awk ' BEGIN { FS="\t"; OFS="\t"; @@ -12,13 +13,15 @@ $1 == "#path" && $2 == "files" { process = 1; } -process && column1 > 0 && column2 > 0 { - $column1 = "XXXXXXXXXXX"; - $column2 = "XXXXXXXXXXX"; +/^[^#]/ { + if ( process && column1 > 0 && column2 > 0 ) { + $column1 = "XXXXXXXXXXX"; + $column2 = "XXXXXXXXXXX"; + } } -/^#/ { - for ( i = 0; i < NF; ++i ) { +/^#fields/ { + for ( i = 2; i <= NF; ++i ) { if ( $i == "fuid" ) column1 = i - 1; @@ -28,6 +31,4 @@ process && column1 > 0 && column2 > 0 { } { print } - - - +' diff --git a/testing/scripts/diff-remove-mime-types b/testing/scripts/diff-remove-mime-types deleted file mode 100755 index b8cc3d1e6d..0000000000 --- a/testing/scripts/diff-remove-mime-types +++ /dev/null @@ -1,29 +0,0 @@ -#! /usr/bin/awk -f -# -# A diff canonifier that removes all MIME types because libmagic output -# can differ between installations. - -BEGIN { FS="\t"; OFS="\t"; type_col = -1; desc_col = -1 } - -/^#fields/ { - for ( i = 2; i < NF; ++i ) - { - if ( $i == "mime_type" ) - type_col = i-1; - if ( $i == "mime_desc" ) - desc_col = i-1; - } -} - -function remove_mime (n) { - if ( n >= 0 && $n != "-" ) - # Mark that it's set, but ignore content. - $n = "+" -} - -remove_mime(type_col) -remove_mime(desc_col) - -{ - print; -} diff --git a/testing/scripts/diff-remove-uids b/testing/scripts/diff-remove-uids index 8e12b7abe5..4d4d041b12 100755 --- a/testing/scripts/diff-remove-uids +++ b/testing/scripts/diff-remove-uids @@ -1,21 +1,22 @@ -#! /usr/bin/awk -f +#! /usr/bin/env bash # # A diff canonifier that removes all connection UIDs. +awk ' BEGIN { FS="\t"; OFS="\t"; } -column > 0 { - $column = "XXXXXXXXXXX"; +/^[^#]/ { + if ( column > 0 ) { + $column = "XXXXXXXXXXX"; } +} -/^#/ { - for ( i = 0; i < NF; ++i ) { +/^#fields/ { + for ( i = 2; i <= NF; ++i ) { if ( $i == "uid" ) column = i - 1; } } { print } - - - +' diff --git a/testing/scripts/diff-remove-x509-names b/testing/scripts/diff-remove-x509-names index 2bd7787611..3214c892bd 100755 --- a/testing/scripts/diff-remove-x509-names +++ b/testing/scripts/diff-remove-x509-names @@ -1,8 +1,9 @@ -#! /usr/bin/awk -f +#! /usr/bin/env bash # # A diff canonifier that removes all X.509 Distinguished Name subject fields # because that output can differ depending on installed OpenSSL version. +awk ' BEGIN { FS="\t"; OFS="\t"; s_col = -1; i_col = -1; is_col = -1; cs_col = -1; ci_col = -1; cert_subj_col = -1; cert_issuer_col = -1 } /^#/ { @@ -33,46 +34,47 @@ BEGIN { FS="\t"; OFS="\t"; s_col = -1; i_col = -1; is_col = -1; cs_col = -1; ci_ s_col > 0 { if ( $s_col != "-" ) - # Mark that it's set, but ignore content. + # Mark that it is set, but ignore content. $s_col = "+"; } i_col > 0 { if ( $i_col != "-" ) - # Mark that it's set, but ignore content. + # Mark that it is set, but ignore content. $i_col = "+"; } is_col > 0 { if ( $is_col != "-" ) - # Mark that it's set, but ignore content. + # Mark that it is set, but ignore content. $is_col = "+"; } cs_col > 0 { if ( $cs_col != "-" ) - # Mark that it's set, but ignore content. + # Mark that it is set, but ignore content. $cs_col = "+"; } ci_col > 0 { if ( $ci_col != "-" ) - # Mark that it's set, but ignore content. + # Mark that it is set, but ignore content. $ci_col = "+"; } cert_subj_col > 0 { if ( $cert_subj_col != "-" ) - # Mark that it's set, but ignore content. + # Mark that it is set, but ignore content. $cert_subj_col = "+"; } cert_issuer_col > 0 { if ( $cert_issuer_col != "-" ) - # Mark that it's set, but ignore content. + # Mark that it is set, but ignore content. $cert_issuer_col = "+"; } { print; } +'