diff --git a/CHANGES b/CHANGES index f59a401151..6e4b211418 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,127 @@ +2.1-386 | 2013-03-22 12:41:50 -0700 + + * Added reverse() function to strings.bif. (Yun Zheng Hu) + +2.1-384 | 2013-03-22 12:10:14 -0700 + + * Fix record constructors in table initializer indices. Addresses + #660. (Jon Siwek) + +2.1-382 | 2013-03-22 12:01:34 -0700 + + * Add support for 802.1ah (Q-in-Q). Addresses #641. (Seth Hall) + +2.1-380 | 2013-03-18 12:18:10 -0700 + + * Fix gcc compile warnings in base64 encoder and benchmark reader. + (Bernhard Amann) + +2.1-377 | 2013-03-17 17:36:09 -0700 + + * Fixing potential leak in DNS error case. (Vlad Grigorescu) + +2.1-375 | 2013-03-17 13:14:26 -0700 + + * Add base64 encoding functionality, including new BiFs + encode_base64() and encode_base64_custom(). (Bernhard Amann) + + * Replace call to external "openssl" in extract-certs-pem.bro with + that encode_base64(). (Bernhard Amann) + + * Adding a test for extract-certs-pem.pem. (Robin Sommer) + + * Renaming Base64Decoder to Base64Converter. (Robin Sommer) + +2.1-366 | 2013-03-17 12:35:59 -0700 + + * Correctly handle DNS lookups for software version ranges. (Seth + Hall) + + * Improvements to vulnerable software detection. (Seth Hall) + + - Add a DNS based updating method. This needs to be tested + still. + + - Vulnerable version ranges are used now instead of only single + versions. This can deal with software with multiple stable + major versions. + + * Update software version parsing and comparison to account for a + third numeric subversion. Also, $addl is now compared numerically + if the value is actually numeric. (Seth Hall) + +2.1-361 | 2013-03-13 07:18:22 -0700 + + * Add check for truncated link frames. Addresses #962. (Jacob + Baines) + + * Fix large memory allocation in IP fragment reassembly. Addresses + #961. (Jacob Baines) + +2.1-357 | 2013-03-08 09:18:35 -0800 + + * Fix race-condition in table-event test. (Bernhard Amann) + + * s/bro-ids.org/bro.org/g. (Robin Sommer) + +2.1-353 | 2013-03-07 13:31:37 -0800 + + * Fix function type-equivalence requiring same parameter names. + Addresses #957. (Jon Siwek) + +2.1-351 | 2013-03-07 13:27:29 -0800 + + * Fix new/delete mismatch. Addresses #958. (Jacob Baines) + + * Fix compiler warnings. (Jon Siwek) + +2.1-347 | 2013-03-06 16:48:44 -0800 + + * Remove unused parameter from vector assignment method. (Bernhard Amann) + + * Remove the byte_len() and length() bifs. (Bernhard Amann) + +2.1-342 | 2013-03-06 15:42:52 -0800 + + * Moved the Notice::notice event and Notice::policy table to both be + hooks. See documentation and NEWS for information. (Seth Hall). + +2.1-338 | 2013-03-06 15:10:43 -0800 + + * Fix init of local sets/vectors via curly brace initializer lists. + (Jon Siwek) + +2.1-336 | 2013-03-06 15:08:06 -0800 + + * Fix memory leaks resulting from 'when' and 'return when' + statements. Addresses #946. (Jon Siwek) + + * Fix three bugs with 'when' and 'return when' statements. Addresses + #946. (Jon Siwek) + +2.1-333 | 2013-03-06 14:59:47 -0800 + + * Add parsing for GTPv1 extension headers and control messages. (Jon Siwek) + + This includes: + + - A new generic gtpv1_message() event generated for any GTP + message type. + + - Specific events for the create/update/delete PDP context + request/response messages. + + Addresses #934. + +2.1-331 | 2013-03-06 14:54:33 -0800 + + * Fix possible null pointer dereference in identify_data BIF. Also + centralized libmagic calls for consistent error handling/output. + (Jon Siwek) + + * Fix build on OpenBSD 5.2. (Jon Siwek) + 2.1-328 | 2013-02-05 01:34:29 -0500 * New script to query the ICSI Certificate Notary diff --git a/INSTALL b/INSTALL index 084ca9dc2e..4604a26bf5 100644 --- a/INSTALL +++ b/INSTALL @@ -4,7 +4,7 @@ .. _MacPorts: http://www.macports.org .. _Fink: http://www.finkproject.org .. _Homebrew: http://mxcl.github.com/homebrew -.. _bro downloads page: http://bro-ids.org/download/index.html +.. _bro downloads page: http://bro.org/download/index.html ============== Installing Bro @@ -189,15 +189,15 @@ Bro releases are bundled into source packages for convenience and available from the `bro downloads page`_. Alternatively, the latest Bro development version can be obtained through git -repositories hosted at `git.bro-ids.org `_. See +repositories hosted at `git.bro.org `_. See our `git development documentation -`_ for comprehensive +`_ for comprehensive information on Bro's use of git revision control, but the short story for downloading the full source code experience for Bro via git is: .. console:: - git clone --recursive git://git.bro-ids.org/bro + git clone --recursive git://git.bro.org/bro .. note:: If you choose to clone the ``bro`` repository non-recursively for a "minimal Bro experience", be aware that compiling it depends on @@ -230,7 +230,7 @@ automatically. Finally, use ``make install-aux`` to install some of the other programs that are in the ``aux/bro-aux`` directory. OpenBSD users, please see our FAQ at -http://www.bro-ids.org/documentation/faq.html if you are having +http://www.bro.org/documentation/faq.html if you are having problems installing Bro. @@ -298,7 +298,7 @@ Running Bro Bro is a complex program and it takes a bit of time to get familiar with it. A good place for newcomers to start is the Quick Start Guide -at http://www.bro-ids.org/documentation/quickstart.html. +at http://www.bro.org/documentation/quickstart.html. For developers that wish to run Bro directly from the ``build/`` directory (i.e., without performing ``make install``), they will have diff --git a/NEWS b/NEWS index e3f20d0455..8605dcdbd4 100644 --- a/NEWS +++ b/NEWS @@ -67,6 +67,7 @@ Changed Functionality - md5_*, sha1_*, sha256_*, and entropy_* have all changed their signatures to work with opaque types (see above). + - Removed a now unused argument from "do_split" helper function. - "this" is no longer a reserved keyword. @@ -81,6 +82,50 @@ Changed Functionality value can now be set with the new broctl.cfg option "MailAlarmsInterval". +- We have completely reworded the "notice_policy" mechanism. It now no + linger uses a record of policy items but a "hook", a new language + element that's roughly equivalent to a function with multiple + bodies. The documentation [TODO: insert link] describes how to use + the new notice policy. For existing code, the two main changes are: + + - What used to be a "redef" of "Notice::policy" now becomes a hook + implementation. Example: + + Old: + + redef Notice::policy += { + [$pred(n: Notice::Info) = { + return n$note == SSH::Login && n$id$resp_h == 10.0.0.1; + }, + $action = Notice::ACTION_EMAIL] + }; + + New: + + hook Notice::policy(n: Notice::Info) + { + if ( n$note == SSH::Login && n$id$resp_h == 10.0.0.1 ) + add n$actions[Notice::ACTION_EMAIL]; + } + + - notice() is now likewise a hook, no longer an event. If you have + handlers for that event, you'll likely just need to change the + type accordingly. Example: + + Old: + + event notice(n: Notice::Info) { ... } + + New: + + hook notice(n: Notice::Info) { ... } + +- The notice_policy.log is gone. That's a result of the new notice + policy setup. + +- Removed the byte_len() and length() bif functions. Use the "|...|" + operator instead. + Bro 2.1 ------- @@ -247,7 +292,7 @@ Bro 2.0 As the version number jump suggests, Bro 2.0 is a major upgrade and lots of things have changed. We have assembled a separate upgrade guide with the most important changes compared to Bro 1.5 at -http://www.bro-ids.org/documentation/upgrade.html. You can find +http://www.bro.org/documentation/upgrade.html. You can find the offline version of that document in ``doc/upgrade.rst.``. Compared to the earlier 2.0 Beta version, the major changes in the @@ -255,7 +300,7 @@ final release are: * The default scripts now come with complete reference documentation. See - http://www.bro-ids.org/documentation/index.html. + http://www.bro.org/documentation/index.html. * libz and libmagic are now required dependencies. diff --git a/README b/README index c837afaf92..734246c914 100644 --- a/README +++ b/README @@ -11,7 +11,7 @@ Please see COPYING for licensing information. For more documentation, research publications, and community contact information, please see Bro's home page: - http://www.bro-ids.org + http://www.bro.org On behalf of the Bro Development Team, diff --git a/VERSION b/VERSION index 70056e0c53..95253817be 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.1-328 +2.1-386 diff --git a/aux/binpac b/aux/binpac index 2fd9086c9d..72d121ade5 160000 --- a/aux/binpac +++ b/aux/binpac @@ -1 +1 @@ -Subproject commit 2fd9086c9dc0e76f6ff1ae04a60cbbce60507aab +Subproject commit 72d121ade5a37df83d3252646de51cb77ce69a89 diff --git a/aux/bro-aux b/aux/bro-aux index bea556198b..7068100754 160000 --- a/aux/bro-aux +++ b/aux/bro-aux @@ -1 +1 @@ -Subproject commit bea556198b69d30d64c0cf1b594e6de71176df6f +Subproject commit 70681007546aad6e5648494e882b71adb9165105 diff --git a/aux/broccoli b/aux/broccoli index c1ba9b44c4..e64204fec5 160000 --- a/aux/broccoli +++ b/aux/broccoli @@ -1 +1 @@ -Subproject commit c1ba9b44c4815c61c54c968f462ec5b0865e5990 +Subproject commit e64204fec55759c614a276c1933bbff2069a63db diff --git a/aux/broctl b/aux/broctl index 2bf6b37177..2b35d03313 160000 --- a/aux/broctl +++ b/aux/broctl @@ -1 +1 @@ -Subproject commit 2bf6b37177b895329173acac2bb98f38a8783bc1 +Subproject commit 2b35d0331366865fbf0119919cc9692d55c4538c diff --git a/aux/btest b/aux/btest index ba0700fe44..d5b8df42cb 160000 --- a/aux/btest +++ b/aux/btest @@ -1 +1 @@ -Subproject commit ba0700fe448895b654b90d50f389f6f1341234cb +Subproject commit d5b8df42cb9c398142e02d4bf8ede835fd0227f4 diff --git a/cmake b/cmake index 14537f56d6..94e72a3075 160000 --- a/cmake +++ b/cmake @@ -1 +1 @@ -Subproject commit 14537f56d66b18ab9d5024f798caf4d1f356fc67 +Subproject commit 94e72a3075bb0b9550ad05758963afda394bfb2c diff --git a/doc/_templates/layout.html b/doc/_templates/layout.html index 77d9d1de1c..2f8ea02aff 100644 --- a/doc/_templates/layout.html +++ b/doc/_templates/layout.html @@ -10,7 +10,7 @@ {% endblock %} {% block header %} - {% endblock %} @@ -108,6 +108,6 @@ {% endblock %} {% block footer %} - {% endblock %} diff --git a/doc/index.rst b/doc/index.rst index cea72d1759..29b29541b4 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -53,7 +53,7 @@ Other Bro Components The following are snapshots of documentation for components that come with this version of Bro (|version|). Since they can also be used independently, see the `download page -`_ for documentation of any +`_ for documentation of any current, independent component releases. .. toctree:: diff --git a/doc/notice.rst b/doc/notice.rst index 5849e605a9..e6d4326db1 100644 --- a/doc/notice.rst +++ b/doc/notice.rst @@ -6,7 +6,7 @@ Notice Framework One of the easiest ways to customize Bro is writing a local notice policy. Bro can detect a large number of potentially interesting - situations, and the notice policy tells which of them the user wants to be + situations, and the notice policy hook which of them the user wants to be acted upon in some manner. In particular, the notice policy can specify actions to be taken, such as sending an email or compiling regular alarm emails. This page gives an introduction into writing such a notice @@ -24,8 +24,8 @@ of interest for the user. However, none of these scripts determines the importance of what it finds itself. Instead, the scripts only flag situations as *potentially* interesting, leaving it to the local configuration to define which of them are in fact actionable. This decoupling of detection and -reporting allows Bro to address the different needs that sites have: -definitions of what constitutes an attack or even a compromise differ quite a +reporting allows Bro to address the different needs that sites have. +Definitions of what constitutes an attack or even a compromise differ quite a bit between environments, and activity deemed malicious at one site might be fully acceptable at another. @@ -40,7 +40,7 @@ More information about raising notices can be found in the `Raising Notices`_ section. Once a notice is raised, it can have any number of actions applied to it by -the :bro:see:`Notice::policy` set which is described in the `Notice Policy`_ +writing :bro:see:`Notice::policy` hooks which is described in the `Notice Policy`_ section below. Such actions can be to send a mail to the configured address(es) or to simply ignore the notice. Currently, the following actions are defined: @@ -68,13 +68,7 @@ are defined: - Send an email to the email address or addresses given in the :bro:see:`Notice::mail_page_dest` variable. - * - Notice::ACTION_NO_SUPPRESS - - This action will disable the built in notice suppression for the - notice. Keep in mind that this action will need to be applied to - every notice that shouldn't be suppressed including each of the future - notices that would have normally been suppressed. - -How these notice actions are applied to notices is discussed in the +How these notice actions are applied to notices is discussed in the `Notice Policy`_ and `Notice Policy Shortcuts`_ sections. Processing Notices @@ -83,26 +77,24 @@ Processing Notices Notice Policy ************* -The predefined set :bro:see:`Notice::policy` provides the mechanism for -applying actions and other behavior modifications to notices. Each entry -of :bro:see:`Notice::policy` is a record of the type -:bro:see:`Notice::PolicyItem` which defines a condition to be matched -against all raised notices and one or more of a variety of behavior -modifiers. The notice policy is defined by adding any number of -:bro:see:`Notice::PolicyItem` records to the :bro:see:`Notice::policy` -set. +The hook :bro:see:`Notice::policy` provides the mechanism for applying +actions and generally modifying the notice before it's sent onward to +the action plugins. Hooks can be thought of as multi-bodied functions +and using them looks very similar to handling events. The difference +is that they don't go through the event queue like events. Users should +directly make modifications to the :bro:see:`Notice::Info` record +given as the argument to the hook. Here's a simple example which tells Bro to send an email for all notices of type :bro:see:`SSH::Login` if the server is 10.0.0.1: .. code:: bro - redef Notice::policy += { - [$pred(n: Notice::Info) = { - return n$note == SSH::Login && n$id$resp_h == 10.0.0.1; - }, - $action = Notice::ACTION_EMAIL] - }; + hook Notice::policy(n: Notice::Info) + { + if ( n$note == SSH::Login && n$id$resp_h == 10.0.0.1 ) + add n$actions[Notice::ACTION_EMAIL]; + } .. note:: @@ -110,78 +102,21 @@ type :bro:see:`SSH::Login` if the server is 10.0.0.1: such that it is only raised when Bro heuristically detects a successful login. No apparently failed logins will raise this notice. -While the syntax might look a bit convoluted at first, it provides a lot of -flexibility due to having access to Bro's full programming language. - -Predicate Field -^^^^^^^^^^^^^^^ - -The :bro:see:`Notice::PolicyItem` record type has a field name ``$pred`` -which defines the entry's condition in the form of a predicate written -as a Bro function. The function is passed the notice as a -:bro:see:`Notice::Info` record and it returns a boolean value indicating -if the entry is applicable to that particular notice. - -.. note:: - - The lack of a predicate in a ``Notice::PolicyItem`` is implicitly true - (``T``) since an implicit false (``F``) value would never be used. - -Bro evaluates the predicates of each entry in the order defined by the -``$priority`` field in :bro:see:`Notice::PolicyItem` records. The valid -values are 0-10 with 10 being earliest evaluated. If ``$priority`` is -omitted, the default priority is 5. - -Behavior Modification Fields -^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -There are a set of fields in the :bro:see:`Notice::PolicyItem` record type that -indicate ways that either the notice or notice processing should be modified -if the predicate field (``$pred``) evaluated to true (``T``). Those fields are -explained in more detail in the following table. - -.. list-table:: - :widths: 20 30 20 - :header-rows: 1 - - * - Field - - Description - - Example - - * - ``$action=`` - - Each :bro:see:`Notice::PolicyItem` can have a single action - applied to the notice with this field. - - ``$action = Notice::ACTION_EMAIL`` - - * - ``$suppress_for=`` - - This field makes it possible for a user to modify the behavior of the - notice framework's automated suppression of intrinsically similar - notices. More information about the notice framework's automated - suppression can be found in the `Automated Suppression`_ section of - this document. - - ``$suppress_for = 10mins`` - - * - ``$halt=`` - - This field can be used for modification of the notice policy - evaluation. To stop processing of notice policy items before - evaluating all of them, set this field to ``T`` and make the ``$pred`` - field return ``T``. :bro:see:`Notice::PolicyItem` records defined at - a higher priority as defined by the ``$priority`` field will still be - evaluated but those at a lower priority won't. - - ``$halt = T`` - - +Hooks can also have priorities applied to order their execution like events +with a default priority of 0. Greater values are executed first. Setting +a hook body to run before default hook bodies might look like this: .. code:: bro - redef Notice::policy += { - [$pred(n: Notice::Info) = { - return n$note == SSH::Login && n$id$resp_h == 10.0.0.1; - }, - $action = Notice::ACTION_EMAIL, - $priority=5] - }; + hook Notice::policy(n: Notice::Info) &priority=5 + { + if ( n$note == SSH::Login && n$id$resp_h == 10.0.0.1 ) + add n$actions[Notice::ACTION_EMAIL]; + } +Hooks can also abort later hook bodies with the ``break`` keyword. This +is primarily useful if one wants to completely preempt processing by +lower priority :bro:see:`Notice::policy` hooks. Notice Policy Shortcuts *********************** @@ -189,7 +124,7 @@ Notice Policy Shortcuts Although the notice framework provides a great deal of flexibility and configurability there are many times that the full expressiveness isn't needed and actually becomes a hindrance to achieving results. The framework provides -a default :bro:see:`Notice::policy` suite as a way of giving users the +a default :bro:see:`Notice::policy` hook body as a way of giving users the shortcuts to easily apply many common actions to notices. These are implemented as sets and tables indexed with a @@ -245,7 +180,7 @@ like this: .. code:: bro - NOTICE([$note=SSH::Login, + NOTICE([$note=SSH::Login, $msg="Heuristically detected successful SSH login.", $conn=c]); @@ -263,7 +198,7 @@ fields used when raising notices are described in the following table: * - ``$note`` - This field is required and is an enum value which represents the - notice type. + notice type. * - ``$msg`` - This is a human readable message which is meant to provide more @@ -355,9 +290,9 @@ certificates. .. code:: bro - NOTICE([$note=SSL::Invalid_Server_Cert, + NOTICE([$note=SSL::Invalid_Server_Cert, $msg=fmt("SSL certificate validation failed with (%s)", c$ssl$validation_status), - $sub=c$ssl$subject, + $sub=c$ssl$subject, $conn=c, $identifier=cat(c$id$resp_h,c$id$resp_p,c$ssl$validation_status,c$ssl$cert_hash)]); @@ -377,19 +312,45 @@ Setting the ``$identifier`` field is left to those raising notices because it's assumed that the script author who is raising the notice understands the full problem set and edge cases of the notice which may not be readily apparent to users. If users don't want the suppression to take place or simply -want a different interval, they can always modify it with the -:bro:see:`Notice::policy`. +want a different interval, they can set a notice's suppression +interval to ``0secs`` or delete the value from the ``$identifier`` field in +a :bro:see:`Notice::policy` hook. Extending Notice Framework -------------------------- -Adding Custom Notice Actions -**************************** +There are a couple of mechanism currently for extending the notice framework +and adding new capability. Extending Notice Emails *********************** +If there is extra information that you would like to add to emails, that is +possible to add by writing :bro:see:`Notice::policy` hooks. + +There is a field in the :bro:see:`Notice::Info` record named +``$email_body_sections`` which will be included verbatim when email is being +sent. An example of including some information from an HTTP request is +included below. + +.. code:: bro + + hook Notice::policy(n: Notice::Info) + { + if ( n?$conn && n$conn?$http && n$conn$http?$host ) + n$email_body_sections[|email_body_sections|] = fmt("HTTP host header: %s", n$conn$http$host); + } + + Cluster Considerations ---------------------- +As a user/developer of Bro, the main cluster concern with the notice framework +is understanding what runs where. When a notice is generated on a worker, the +worker checks to see if the notice shoudl be suppressed based on information +locally maintained in the worker process. If it's not being +suppressed, the worker forwards the notice directly to the manager and does no more +local processing. The manager then runs the :bro:see:`Notice::policy` hook and +executes all of the actions determined to be run. + diff --git a/doc/quickstart.rst b/doc/quickstart.rst index 89c9be5a87..9f64e36d89 100644 --- a/doc/quickstart.rst +++ b/doc/quickstart.rst @@ -111,7 +111,7 @@ protocol-dependent activity that's occurring. E.g. ``http.log``'s next few columns (shortened for brevity) show a request to the root of Bro website:: # method host uri referrer user_agent - GET bro-ids.org / - <...>Chrome/12.0.742.122<...> + GET bro.org / - <...>Chrome/12.0.742.122<...> Some logs are worth explicit mention: diff --git a/doc/reporting-problems.rst b/doc/reporting-problems.rst index 5e55b2ac90..903df76257 100644 --- a/doc/reporting-problems.rst +++ b/doc/reporting-problems.rst @@ -19,7 +19,7 @@ Reporting Problems Generally, when you encounter a problem with Bro, the best thing to do is opening a new ticket in `Bro's issue tracker -`__ and include information on how to +`__ and include information on how to reproduce the issue. Ideally, your ticket should come with the following: diff --git a/doc/scripts/DocSourcesList.cmake b/doc/scripts/DocSourcesList.cmake index 6cdbf2b621..8afe2b5545 100644 --- a/doc/scripts/DocSourcesList.cmake +++ b/doc/scripts/DocSourcesList.cmake @@ -37,6 +37,7 @@ rest_target(${psd} base/frameworks/file-analysis/main.bro) rest_target(${psd} base/frameworks/input/main.bro) rest_target(${psd} base/frameworks/input/readers/ascii.bro) rest_target(${psd} base/frameworks/input/readers/benchmark.bro) +rest_target(${psd} base/frameworks/input/readers/binary.bro) rest_target(${psd} base/frameworks/input/readers/raw.bro) rest_target(${psd} base/frameworks/intel/cluster.bro) rest_target(${psd} base/frameworks/intel/input.bro) @@ -59,6 +60,7 @@ rest_target(${psd} base/frameworks/notice/actions/pp-alarms.bro) rest_target(${psd} base/frameworks/notice/cluster.bro) rest_target(${psd} base/frameworks/notice/extend-email/hostnames.bro) rest_target(${psd} base/frameworks/notice/main.bro) +rest_target(${psd} base/frameworks/notice/non-cluster.bro) rest_target(${psd} base/frameworks/notice/weird.bro) rest_target(${psd} base/frameworks/packet-filter/main.bro) rest_target(${psd} base/frameworks/packet-filter/netstats.bro) @@ -73,21 +75,25 @@ rest_target(${psd} base/protocols/conn/main.bro) rest_target(${psd} base/protocols/conn/polling.bro) rest_target(${psd} base/protocols/dns/consts.bro) rest_target(${psd} base/protocols/dns/main.bro) +rest_target(${psd} base/protocols/ftp/file-analysis.bro) rest_target(${psd} base/protocols/ftp/file-extract.bro) rest_target(${psd} base/protocols/ftp/gridftp.bro) rest_target(${psd} base/protocols/ftp/main.bro) rest_target(${psd} base/protocols/ftp/utils-commands.bro) +rest_target(${psd} base/protocols/http/file-analysis.bro) rest_target(${psd} base/protocols/http/file-extract.bro) rest_target(${psd} base/protocols/http/file-hash.bro) rest_target(${psd} base/protocols/http/file-ident.bro) rest_target(${psd} base/protocols/http/main.bro) rest_target(${psd} base/protocols/http/utils.bro) rest_target(${psd} base/protocols/irc/dcc-send.bro) +rest_target(${psd} base/protocols/irc/file-analysis.bro) rest_target(${psd} base/protocols/irc/main.bro) rest_target(${psd} base/protocols/modbus/consts.bro) rest_target(${psd} base/protocols/modbus/main.bro) rest_target(${psd} base/protocols/smtp/entities-excerpt.bro) rest_target(${psd} base/protocols/smtp/entities.bro) +rest_target(${psd} base/protocols/smtp/file-analysis.bro) rest_target(${psd} base/protocols/smtp/main.bro) rest_target(${psd} base/protocols/socks/consts.bro) rest_target(${psd} base/protocols/socks/main.bro) diff --git a/doc/upgrade.rst b/doc/upgrade.rst index 9ea006f488..539757537d 100644 --- a/doc/upgrade.rst +++ b/doc/upgrade.rst @@ -254,7 +254,7 @@ Variable Naming - Identifiers may have been renamed to conform to new `scripting conventions - `_ + `_ BroControl @@ -296,7 +296,7 @@ Development Infrastructure Bro development has moved from using SVN to Git for revision control. Users that want to use the latest Bro development snapshot by checking it out from the source repositories should see the `development process -`_. Note that all the various +`_. Note that all the various sub-components now reside in their own repositories. However, the top-level Bro repository includes them as git submodules so it's easy to check them all out simultaneously. diff --git a/scripts/base/frameworks/cluster/main.bro b/scripts/base/frameworks/cluster/main.bro index 766dea912f..4184ad6ded 100644 --- a/scripts/base/frameworks/cluster/main.bro +++ b/scripts/base/frameworks/cluster/main.bro @@ -39,7 +39,7 @@ export { ## The node type doing all the actual traffic analysis. WORKER, ## A node acting as a traffic recorder using the - ## `Time Machine `_ software. + ## `Time Machine `_ software. TIME_MACHINE, }; diff --git a/scripts/base/frameworks/file-analysis/main.bro b/scripts/base/frameworks/file-analysis/main.bro index 43c0c7a3ac..d4ddde1d23 100644 --- a/scripts/base/frameworks/file-analysis/main.bro +++ b/scripts/base/frameworks/file-analysis/main.bro @@ -18,23 +18,20 @@ export { const default_reassembly_buffer_size: count = 1024*1024 &redef; ## The default buffer size used for storing the beginning of files. - # TODO: what's a reasonable default? - const default_bof_buffer_size: count = 256 &redef; + const default_bof_buffer_size: count = 1024 &redef; ## The default amount of time file analysis will wait for new file data ## before giving up. - ## TODO: what's a reasonable default? - #const default_timeout_interval: interval = 2 mins &redef; - const default_timeout_interval: interval = 10 sec &redef; + const default_timeout_interval: interval = 2 mins &redef; - ## The default amount of data that a user is allowed to extract - ## from a file to an event with the - ## :bro:see:`FileAnalysis::ACTION_DATA_EVENT` action. - ## TODO: what's a reasonable default? - const default_data_event_len: count = 1024*1024 &redef; + # Needed a forward declaration for event parameters... + type Info: record {}; type ActionArgs: record { + act: Action; extract_filename: string &optional; + chunk_event: event(info: Info, data: string, off: count) &optional; + stream_event: event(info: Info, data: string) &optional; }; type ActionResults: record { @@ -52,15 +49,16 @@ export { ## from a container file as part of the analysis. parent_file_id: string &log &optional; - ## The network protocol over which the file was transferred. - protocol: string &log &optional; + ## An identification of the source of the file data. E.g. it may be + ## a network protocol over which it was transferred, or a local file + ## path which was read, or some other input source. + source: string &log &optional; - ## The set of connections over which the file was transferred, - ## indicated by UID strings. - conn_uids: set[string] &log &optional; - ## The set of connections over which the file was transferred, - ## indicated by 5-tuples. - conn_ids: set[conn_id] &optional; + ## The set of connections over which the file was transferred. + conns: table[conn_id] of connection &optional; + + ## The time at which the last activity for the file was seen. + last_active: time &log; ## Number of bytes provided to the file analysis engine for the file. seen_bytes: count &log &default=0; @@ -82,18 +80,100 @@ export { ## the analysis engine will wait before giving up on it. timeout_interval: interval &log &default=default_timeout_interval; + ## The number of bytes at the beginning of a file to save for later + ## inspection in *bof_buffer* field of + ## :bro:see:`FileAnalysis::ActionResults`. + bof_buffer_size: count &log &default=default_bof_buffer_size; + + ## The content of the beginning of a file up to *bof_buffer_size* bytes. + ## This is also the buffer that's used for file/mime type detection. + bof_buffer: string &optional; + + ## An initial guess at file type. + file_type: string &log &optional; + ## An initial guess at mime type. + mime_type: string &log &optional; + ## Actions that have been added to the analysis of this file. - actions: vector of Action &default=vector(); - - ## The corresponding arguments supplied to each element of *actions*. - action_args: vector of ActionArgs &default=vector(); - - ## Some actions may directly yield results in this record. - action_results: ActionResults; + ## Not meant to be modified directly by scripts. + actions: table[ActionArgs] of ActionResults; } &redef; ## TODO: document global policy: hook(trig: Trigger, info: Info); + const disable: table[AnalyzerTag] of bool = table() &redef; + # TODO: wrapper functions for BiFs ? + + ## Event that can be handled to access the Info record as it is sent on + ## to the logging framework. + global log_file_analysis: event(rec: Info); + + ## The salt concatenated to unique file handle strings generated by + ## :bro:see:`FileAnalysis::handle_callbacks` before hashing them + ## in to a file id (the *file_id* field of :bro:see:`FileAnalysis::Info`). + ## Provided to help mitigate the possiblility of manipulating parts of + ## network connections that factor in to the file handle in order to + ## generate two handles that would hash to the same file id. + const salt = "I recommend changing this." &redef; } + +event bro_init() &priority=5 + { + Log::create_stream(FileAnalysis::LOG, + [$columns=Info, $ev=log_file_analysis]); + } + +redef record FileAnalysis::Info += { + conn_uids: set[string] &log &optional; + actions_taken: set[Action] &log &optional; + extracted_files: set[string] &log &optional; + md5: string &log &optional; + sha1: string &log &optional; + sha256: string &log &optional; +}; + +hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info) + &priority=-10 + { + if ( trig != FileAnalysis::TRIGGER_EOF && + trig != FileAnalysis::TRIGGER_DONE ) return; + + info$conn_uids = set(); + if ( info?$conns ) + for ( cid in info$conns ) + add info$conn_uids[info$conns[cid]$uid]; + + info$actions_taken = set(); + info$extracted_files = set(); + + for ( act in info$actions ) + { + add info$actions_taken[act$act]; + local result: FileAnalysis::ActionResults = info$actions[act]; + + switch ( act$act ) { + case FileAnalysis::ACTION_EXTRACT: + add info$extracted_files[act$extract_filename]; + break; + case FileAnalysis::ACTION_MD5: + if ( result?$md5 ) + info$md5 = result$md5; + break; + case FileAnalysis::ACTION_SHA1: + if ( result?$sha1 ) + info$sha1 = result$sha1; + break; + case FileAnalysis::ACTION_SHA256: + if ( result?$sha256 ) + info$sha256 = result$sha256; + break; + case FileAnalysis::ACTION_DATA_EVENT: + # no direct result + break; + } + } + + Log::write(FileAnalysis::LOG, info); + } diff --git a/scripts/base/frameworks/input/__load__.bro b/scripts/base/frameworks/input/__load__.bro index 0e7d8ffb73..53b2be8364 100644 --- a/scripts/base/frameworks/input/__load__.bro +++ b/scripts/base/frameworks/input/__load__.bro @@ -2,4 +2,5 @@ @load ./readers/ascii @load ./readers/raw @load ./readers/benchmark +@load ./readers/binary diff --git a/scripts/base/frameworks/input/readers/binary.bro b/scripts/base/frameworks/input/readers/binary.bro new file mode 100644 index 0000000000..abd7944ff2 --- /dev/null +++ b/scripts/base/frameworks/input/readers/binary.bro @@ -0,0 +1,8 @@ +##! Interface for the binary input reader. + +module InputBinary; + +export { + ## Size of data chunks to read from the input file at a time. + const chunk_size = 1024 &redef; +} diff --git a/scripts/base/frameworks/notice/__load__.bro b/scripts/base/frameworks/notice/__load__.bro index 4548e98dc2..bcf3cd6898 100644 --- a/scripts/base/frameworks/notice/__load__.bro +++ b/scripts/base/frameworks/notice/__load__.bro @@ -17,7 +17,9 @@ @if ( Cluster::is_enabled() ) @load ./cluster +@else +@load ./non-cluster @endif # Load here so that it can check whether clustering is enabled. -@load ./actions/pp-alarms +@load ./actions/pp-alarms \ No newline at end of file diff --git a/scripts/base/frameworks/notice/actions/add-geodata.bro b/scripts/base/frameworks/notice/actions/add-geodata.bro index 9f6909595c..c762be3b86 100644 --- a/scripts/base/frameworks/notice/actions/add-geodata.bro +++ b/scripts/base/frameworks/notice/actions/add-geodata.bro @@ -27,18 +27,17 @@ export { ## Notice types which should have the "remote" location looked up. ## If GeoIP support is not built in, this does nothing. const lookup_location_types: set[Notice::Type] = {} &redef; - - ## Add a helper to the notice policy for looking up GeoIP data. - redef Notice::policy += { - [$pred(n: Notice::Info) = { return (n$note in Notice::lookup_location_types); }, - $action = ACTION_ADD_GEODATA, - $priority = 10], - }; } +hook policy(n: Notice::Info) &priority=10 + { + if ( n$note in Notice::lookup_location_types ) + add n$actions[ACTION_ADD_GEODATA]; + } + # This is handled at a high priority in case other notice handlers # want to use the data. -event notice(n: Notice::Info) &priority=10 +hook notice(n: Notice::Info) &priority=10 { if ( ACTION_ADD_GEODATA in n$actions && |Site::local_nets| > 0 && diff --git a/scripts/base/frameworks/notice/actions/drop.bro b/scripts/base/frameworks/notice/actions/drop.bro index 0116dd4ed4..1befd8644f 100644 --- a/scripts/base/frameworks/notice/actions/drop.bro +++ b/scripts/base/frameworks/notice/actions/drop.bro @@ -17,20 +17,13 @@ export { }; } -# This is a little awkward because we want to inject drop along with the -# synchronous functions. -event bro_init() +hook notice(n: Notice::Info) { - local drop_func = function(n: Notice::Info) + if ( ACTION_DROP in n$actions ) { - if ( ACTION_DROP in n$actions ) - { - #local drop = React::drop_address(n$src, ""); - #local addl = drop?$sub ? fmt(" %s", drop$sub) : ""; - #n$dropped = drop$note != Drop::AddressDropIgnored; - #n$msg += fmt(" [%s%s]", drop$note, addl); - } - }; - - add Notice::sync_functions[drop_func]; + #local drop = React::drop_address(n$src, ""); + #local addl = drop?$sub ? fmt(" %s", drop$sub) : ""; + #n$dropped = drop$note != Drop::AddressDropIgnored; + #n$msg += fmt(" [%s%s]", drop$note, addl); + } } diff --git a/scripts/base/frameworks/notice/actions/email_admin.bro b/scripts/base/frameworks/notice/actions/email_admin.bro index 7484a1c606..fb82f2b960 100644 --- a/scripts/base/frameworks/notice/actions/email_admin.bro +++ b/scripts/base/frameworks/notice/actions/email_admin.bro @@ -18,7 +18,7 @@ export { }; } -event notice(n: Notice::Info) &priority=-5 +hook notice(n: Notice::Info) &priority=-5 { if ( |Site::local_admins| > 0 && ACTION_EMAIL_ADMIN in n$actions ) diff --git a/scripts/base/frameworks/notice/actions/page.bro b/scripts/base/frameworks/notice/actions/page.bro index 16a3463126..e29b2bf0ee 100644 --- a/scripts/base/frameworks/notice/actions/page.bro +++ b/scripts/base/frameworks/notice/actions/page.bro @@ -15,7 +15,7 @@ export { const mail_page_dest = "" &redef; } -event notice(n: Notice::Info) &priority=-5 +hook notice(n: Notice::Info) &priority=-5 { if ( ACTION_PAGE in n$actions ) email_notice_to(n, mail_page_dest, F); diff --git a/scripts/base/frameworks/notice/actions/pp-alarms.bro b/scripts/base/frameworks/notice/actions/pp-alarms.bro index 0f19d3da8f..52312c2624 100644 --- a/scripts/base/frameworks/notice/actions/pp-alarms.bro +++ b/scripts/base/frameworks/notice/actions/pp-alarms.bro @@ -105,7 +105,7 @@ event bro_init() $postprocessor=pp_postprocessor]); } -event notice(n: Notice::Info) &priority=-5 +hook notice(n: Notice::Info) &priority=-5 { if ( ! want_pp() ) return; diff --git a/scripts/base/frameworks/notice/cluster.bro b/scripts/base/frameworks/notice/cluster.bro index e812c3fdca..c722f37bab 100644 --- a/scripts/base/frameworks/notice/cluster.bro +++ b/scripts/base/frameworks/notice/cluster.bro @@ -21,30 +21,10 @@ redef Cluster::manager2worker_events += /Notice::begin_suppression/; redef Cluster::worker2manager_events += /Notice::cluster_notice/; @if ( Cluster::local_node_type() != Cluster::MANAGER ) - -# The notice policy is completely handled by the manager and shouldn't be -# done by workers or proxies to save time for packet processing. -redef Notice::policy = table(); - event Notice::begin_suppression(n: Notice::Info) { suppressing[n$note, n$identifier] = n; } - -event Notice::notice(n: Notice::Info) - { - # Send the locally generated notice on to the manager. - event Notice::cluster_notice(n); - } - -event bro_init() &priority=-3 - { - # Workers and proxies need to disable the notice streams because notice - # events are forwarded directly instead of being logged remotely. - Log::disable_stream(Notice::LOG); - Log::disable_stream(Notice::POLICY_LOG); - Log::disable_stream(Notice::ALARM_LOG); - } @endif @if ( Cluster::local_node_type() == Cluster::MANAGER ) @@ -54,3 +34,19 @@ event Notice::cluster_notice(n: Notice::Info) NOTICE(n); } @endif + +module GLOBAL; + +## This is the entry point in the global namespace for the notice framework. +function NOTICE(n: Notice::Info) + { + # Suppress this notice if necessary. + if ( Notice::is_being_suppressed(n) ) + return; + + if ( Cluster::local_node_type() == Cluster::MANAGER ) + Notice::internal_NOTICE(n); + else + # For non-managers, send the notice on to the manager. + event Notice::cluster_notice(n); + } diff --git a/scripts/base/frameworks/notice/extend-email/hostnames.bro b/scripts/base/frameworks/notice/extend-email/hostnames.bro index 2ec6dbb23f..18442f453d 100644 --- a/scripts/base/frameworks/notice/extend-email/hostnames.bro +++ b/scripts/base/frameworks/notice/extend-email/hostnames.bro @@ -13,7 +13,7 @@ module Notice; # reference to the original notice) global tmp_notice_storage: table[string] of Notice::Info &create_expire=max_email_delay+10secs; -event Notice::notice(n: Notice::Info) &priority=10 +hook notice(n: Notice::Info) &priority=10 { if ( ! n?$src && ! n?$dst ) return; diff --git a/scripts/base/frameworks/notice/main.bro b/scripts/base/frameworks/notice/main.bro index e9b29e7392..71071df9ab 100644 --- a/scripts/base/frameworks/notice/main.bro +++ b/scripts/base/frameworks/notice/main.bro @@ -10,9 +10,6 @@ export { redef enum Log::ID += { ## This is the primary logging stream for notices. LOG, - ## This is the notice policy auditing log. It records what the current - ## notice policy is at Bro init time. - POLICY_LOG, ## This is the alarm stream. ALARM_LOG, }; @@ -42,9 +39,6 @@ export { ## version of the alarm log is emailed in bulk to the address(es) ## configured in :bro:id:`Notice::mail_dest`. ACTION_ALARM, - ## Indicates that the notice should not be supressed by the normal - ## duplicate notice suppression that the notice framework does. - ACTION_NO_SUPPRESS, }; ## The notice framework is able to do automatic notice supression by @@ -64,7 +58,7 @@ export { ## A connection 4-tuple identifying the endpoints concerned with the ## notice. id: conn_id &log &optional; - + ## A shorthand way of giving the uid and id to a notice. The ## reference to the actual connection will be deleted after applying ## the notice policy. @@ -102,10 +96,6 @@ export { ## The actions which have been applied to this notice. actions: set[Notice::Action] &log &optional; - ## These are policy items that returned T and applied their action - ## to the notice. - policy_items: set[count] &log &optional; - ## By adding chunks of text into this element, other scripts can ## expand on notices that are being emailed. The normal way to add text ## is to extend the vector by handling the :bro:id:`Notice::notice` @@ -142,9 +132,8 @@ export { identifier: string &optional; ## This field indicates the length of time that this - ## unique notice should be suppressed. This field is automatically - ## filled out and should not be written to by any other script. - suppress_for: interval &log &optional; + ## unique notice should be suppressed. + suppress_for: interval &log &default=default_suppression_interval; }; ## Ignored notice types. @@ -159,58 +148,8 @@ export { ## intervals for entire notice types. const type_suppression_intervals: table[Notice::Type] of interval = {} &redef; - ## This is the record that defines the items that make up the notice policy. - type PolicyItem: record { - ## This is the exact positional order in which the - ## :bro:type:`Notice::PolicyItem` records are checked. - ## This is set internally by the notice framework. - position: count &log &optional; - ## Define the priority for this check. Items are checked in ordered - ## from highest value (10) to lowest value (0). - priority: count &log &default=5; - ## An action given to the notice if the predicate return true. - action: Notice::Action &log &default=ACTION_NONE; - ## The pred (predicate) field is a function that returns a boolean T - ## or F value. If the predicate function return true, the action in - ## this record is applied to the notice that is given as an argument - ## to the predicate function. If no predicate is supplied, it's - ## assumed that the PolicyItem always applies. - pred: function(n: Notice::Info): bool &log &optional; - ## Indicates this item should terminate policy processing if the - ## predicate returns T. - halt: bool &log &default=F; - ## This defines the length of time that this particular notice should - ## be supressed. - suppress_for: interval &log &optional; - }; - - ## Defines a notice policy that is extensible on a per-site basis. - ## All notice processing is done through this variable. - const policy: set[PolicyItem] = { - [$pred(n: Notice::Info) = { return (n$note in Notice::ignored_types); }, - $halt=T, $priority = 9], - [$pred(n: Notice::Info) = { return (n$note in Notice::not_suppressed_types); }, - $action = ACTION_NO_SUPPRESS, - $priority = 9], - [$pred(n: Notice::Info) = { return (n$note in Notice::alarmed_types); }, - $action = ACTION_ALARM, - $priority = 8], - [$pred(n: Notice::Info) = { return (n$note in Notice::emailed_types); }, - $action = ACTION_EMAIL, - $priority = 8], - [$pred(n: Notice::Info) = { - if (n$note in Notice::type_suppression_intervals) - { - n$suppress_for=Notice::type_suppression_intervals[n$note]; - return T; - } - return F; - }, - $action = ACTION_NONE, - $priority = 8], - [$action = ACTION_LOG, - $priority = 0], - } &redef; + ## The hook to modify notice handling. + global policy: hook(n: Notice::Info); ## Local system sendmail program. const sendmail = "/usr/sbin/sendmail" &redef; @@ -240,25 +179,11 @@ export { ## This is the event that is called as the entry point to the ## notice framework by the global :bro:id:`NOTICE` function. By the time ## this event is generated, default values have already been filled out in - ## the :bro:type:`Notice::Info` record and synchronous functions in the - ## :bro:id:`Notice::sync_functions` have already been called. The notice + ## the :bro:type:`Notice::Info` record and the notice ## policy has also been applied. ## ## n: The record containing notice data. - global notice: event(n: Info); - - ## This is a set of functions that provide a synchronous way for scripts - ## extending the notice framework to run before the normal event based - ## notice pathway that most of the notice framework takes. This is helpful - ## in cases where an action against a notice needs to happen immediately - ## and can't wait the short time for the event to bubble up to the top of - ## the event queue. An example is the IP address dropping script that - ## can block IP addresses that have notices generated because it - ## needs to operate closer to real time than the event queue allows it to. - ## Normally the event based extension model using the - ## :bro:id:`Notice::notice` event will work fine if there aren't harder - ## real time constraints. - const sync_functions: set[function(n: Notice::Info)] = set() &redef; + global notice: hook(n: Info); ## This event is generated when a notice begins to be suppressed. ## @@ -266,6 +191,11 @@ export { ## about to be suppressed. global begin_suppression: event(n: Notice::Info); + ## A function to determine if an event is supposed to be suppressed. + ## + ## n: The record containing the notice in question. + global is_being_suppressed: function(n: Notice::Info): bool; + ## This event is generated on each occurence of an event being suppressed. ## ## n: The record containing notice data regarding the notice type @@ -299,13 +229,13 @@ export { ## ## Returns: a string of mail headers to which an email body can be appended global email_headers: function(subject_desc: string, dest: string): string; - + ## This event can be handled to access the :bro:type:`Notice::Info` ## record as it is sent on to the logging framework. ## ## rec: The record containing notice data before it is logged. global log_notice: event(rec: Info); - + ## This is an internal wrapper for the global :bro:id:`NOTICE` function; ## disregard. ## @@ -338,10 +268,6 @@ global suppressing: table[Type, string] of Notice::Info = {} &create_expire=0secs &expire_func=per_notice_suppression_interval; -# This is an internal variable used to store the notice policy ordered by -# priority. -global ordered_policy: vector of PolicyItem = vector(); - function log_mailing_postprocessor(info: Log::RotationInfo): bool { if ( ! reading_traces() && mail_dest != "" ) @@ -424,9 +350,7 @@ function email_notice_to(n: Notice::Info, dest: string, extend: bool) } else { - event reporter_info(network_time(), - fmt("Notice email delay tokens weren't released in time (%s).", n$email_delay_tokens), - ""); + Reporter::info(fmt("Notice email delay tokens weren't released in time (%s).", n$email_delay_tokens)); } } } @@ -468,7 +392,26 @@ function email_notice_to(n: Notice::Info, dest: string, extend: bool) piped_exec(fmt("%s -t -oi", sendmail), email_text); } -event notice(n: Notice::Info) &priority=-5 +hook Notice::policy(n: Notice::Info) &priority=10 + { + if ( n$note in Notice::ignored_types ) + break; + + if ( n$note in Notice::not_suppressed_types ) + n$suppress_for=0secs; + if ( n$note in Notice::alarmed_types ) + add n$actions[ACTION_ALARM]; + if ( n$note in Notice::emailed_types ) + add n$actions[ACTION_EMAIL]; + + if ( n$note in Notice::type_suppression_intervals ) + n$suppress_for=Notice::type_suppression_intervals[n$note]; + + # Logging is a default action. It can be removed in a later hook if desired. + add n$actions[ACTION_LOG]; + } + +hook Notice::notice(n: Notice::Info) &priority=-5 { if ( ACTION_EMAIL in n$actions ) email_notice_to(n, mail_dest, T); @@ -480,7 +423,6 @@ event notice(n: Notice::Info) &priority=-5 # Normally suppress further notices like this one unless directed not to. # n$identifier *must* be specified for suppression to function at all. if ( n?$identifier && - ACTION_NO_SUPPRESS !in n$actions && [n$note, n$identifier] !in suppressing && n$suppress_for != 0secs ) { @@ -488,8 +430,8 @@ event notice(n: Notice::Info) &priority=-5 event Notice::begin_suppression(n); } } - -## This determines if a notice is being suppressed. It is only used + +## This determines if a notice is being suppressed. It is only used ## internally as part of the mechanics for the global :bro:id:`NOTICE` ## function. function is_being_suppressed(n: Notice::Info): bool @@ -539,7 +481,7 @@ function apply_policy(n: Notice::Info) n$p = n$id$resp_p; } - if ( n?$p ) + if ( n?$p ) n$proto = get_port_transport_proto(n$p); if ( n?$iconn ) @@ -565,27 +507,8 @@ function apply_policy(n: Notice::Info) if ( ! n?$email_delay_tokens ) n$email_delay_tokens = set(); - if ( ! n?$policy_items ) - n$policy_items = set(); - - for ( i in ordered_policy ) - { - # If there's no predicate or the predicate returns F. - if ( ! ordered_policy[i]?$pred || ordered_policy[i]$pred(n) ) - { - add n$actions[ordered_policy[i]$action]; - add n$policy_items[int_to_count(i)]; - - # If the predicate matched and there was a suppression interval, - # apply it to the notice now. - if ( ordered_policy[i]?$suppress_for ) - n$suppress_for = ordered_policy[i]$suppress_for; - - # If the policy item wants to halt policy processing, do it now! - if ( ordered_policy[i]$halt ) - break; - } - } + # Apply the hook based policy. + hook Notice::policy(n); # Apply the suppression time after applying the policy so that policy # items can give custom suppression intervals. If there is no @@ -602,61 +525,15 @@ function apply_policy(n: Notice::Info) delete n$iconn; } -# Create the ordered notice policy automatically which will be used at runtime -# for prioritized matching of the notice policy. -event bro_init() &priority=10 - { - # Create the policy log here because it's only written to in this handler. - Log::create_stream(Notice::POLICY_LOG, [$columns=PolicyItem]); - - local tmp: table[count] of set[PolicyItem] = table(); - for ( pi in policy ) - { - if ( pi$priority < 0 || pi$priority > 10 ) - Reporter::fatal("All Notice::PolicyItem priorities must be within 0 and 10"); - - if ( pi$priority !in tmp ) - tmp[pi$priority] = set(); - add tmp[pi$priority][pi]; - } - - local rev_count = vector(10,9,8,7,6,5,4,3,2,1,0); - for ( i in rev_count ) - { - local j = rev_count[i]; - if ( j in tmp ) - { - for ( pi in tmp[j] ) - { - pi$position = |ordered_policy|; - ordered_policy[|ordered_policy|] = pi; - Log::write(Notice::POLICY_LOG, pi); - } - } - } - } - function internal_NOTICE(n: Notice::Info) { - # Suppress this notice if necessary. - if ( is_being_suppressed(n) ) - return; - # Fill out fields that might be empty and do the policy processing. apply_policy(n); - # Run the synchronous functions with the notice. - for ( func in sync_functions ) - func(n); - # Generate the notice event with the notice. - event Notice::notice(n); + hook Notice::notice(n); } module GLOBAL; -## This is the entry point in the global namespace for notice framework. -function NOTICE(n: Notice::Info) - { - Notice::internal_NOTICE(n); - } +global NOTICE: function(n: Notice::Info); diff --git a/scripts/base/frameworks/notice/non-cluster.bro b/scripts/base/frameworks/notice/non-cluster.bro new file mode 100644 index 0000000000..002ec0da34 --- /dev/null +++ b/scripts/base/frameworks/notice/non-cluster.bro @@ -0,0 +1,14 @@ + +@load ./main + +module GLOBAL; + +## This is the entry point in the global namespace for notice framework. +function NOTICE(n: Notice::Info) + { + # Suppress this notice if necessary. + if ( Notice::is_being_suppressed(n) ) + return; + + Notice::internal_NOTICE(n); + } diff --git a/scripts/base/frameworks/signatures/main.bro b/scripts/base/frameworks/signatures/main.bro index 4102214075..7e2f71ea8c 100644 --- a/scripts/base/frameworks/signatures/main.bro +++ b/scripts/base/frameworks/signatures/main.bro @@ -161,7 +161,7 @@ event signature_match(state: signature_state, msg: string, data: string) return; # Trim the matched data down to something reasonable - if ( byte_len(data) > 140 ) + if ( |data| > 140 ) data = fmt("%s...", sub_bytes(data, 0, 140)); local src_addr: addr; @@ -259,8 +259,8 @@ event signature_match(state: signature_state, msg: string, data: string) add vert_table[orig, resp][sig_id]; - local hcount = length(horiz_table[orig, sig_id]); - local vcount = length(vert_table[orig, resp]); + local hcount = |horiz_table[orig, sig_id]|; + local vcount = |vert_table[orig, resp]|; if ( hcount in horiz_scan_thresholds && hcount != last_hthresh[orig] ) { diff --git a/scripts/base/frameworks/software/main.bro b/scripts/base/frameworks/software/main.bro index 7471076335..fefc465e1a 100644 --- a/scripts/base/frameworks/software/main.bro +++ b/scripts/base/frameworks/software/main.bro @@ -29,6 +29,8 @@ export { minor: count &optional; ## Minor subversion number minor2: count &optional; + ## Minor updates number + minor3: count &optional; ## Additional version string (e.g. "beta42") addl: string &optional; } &log; @@ -146,10 +148,10 @@ function parse(unparsed_version: string): Description if ( /^[\/\-\._v\(]/ in sv ) sv = strip(sub(version_parts[2], /^\(?[\/\-\._v\(]/, "")); local version_numbers = split_n(sv, /[\-\._,\[\(\{ ]/, F, 3); - if ( 4 in version_numbers && version_numbers[4] != "" ) - v$addl = strip(version_numbers[4]); + if ( 5 in version_numbers && version_numbers[5] != "" ) + v$addl = strip(version_numbers[5]); else if ( 3 in version_parts && version_parts[3] != "" && - version_parts[3] != ")" ) + version_parts[3] != ")" ) { if ( /^[[:blank:]]*\([a-zA-Z0-9\-\._[:blank:]]*\)/ in version_parts[3] ) { @@ -177,7 +179,9 @@ function parse(unparsed_version: string): Description } } - + + if ( 4 in version_numbers && version_numbers[4] != "" ) + v$minor3 = extract_count(version_numbers[4]); if ( 3 in version_numbers && version_numbers[3] != "" ) v$minor2 = extract_count(version_numbers[3]); if ( 2 in version_numbers && version_numbers[2] != "" ) @@ -332,8 +336,25 @@ function cmp_versions(v1: Version, v2: Version): int return v1?$minor2 ? 1 : -1; } + if ( v1?$minor3 && v2?$minor3 ) + { + if ( v1$minor3 < v2$minor3 ) + return -1; + if ( v1$minor3 > v2$minor3 ) + return 1; + } + else + { + if ( !v1?$minor3 && !v2?$minor3 ) + { } + else + return v1?$minor3 ? 1 : -1; + } + if ( v1?$addl && v2?$addl ) + { return strcmp(v1$addl, v2$addl); + } else { if ( !v1?$addl && !v2?$addl ) @@ -341,6 +362,9 @@ function cmp_versions(v1: Version, v2: Version): int else return v1?$addl ? 1 : -1; } + + # A catcher return that should never be reached...hopefully + return 0; } function software_endpoint_name(id: conn_id, host: addr): string @@ -351,10 +375,11 @@ function software_endpoint_name(id: conn_id, host: addr): string # Convert a version into a string "a.b.c-x". function software_fmt_version(v: Version): string { - return fmt("%d.%d.%d%s", - v?$major ? v$major : 0, - v?$minor ? v$minor : 0, - v?$minor2 ? v$minor2 : 0, + return fmt("%s%s%s%s%s", + v?$major ? fmt("%d", v$major) : "0", + v?$minor ? fmt(".%d", v$minor) : "", + v?$minor2 ? fmt(".%d", v$minor2) : "", + v?$minor3 ? fmt(".%d", v$minor3) : "", v?$addl ? fmt("-%s", v$addl) : ""); } diff --git a/scripts/base/frameworks/tunnels/main.bro b/scripts/base/frameworks/tunnels/main.bro index a3db7061d3..a8fc6c8236 100644 --- a/scripts/base/frameworks/tunnels/main.bro +++ b/scripts/base/frameworks/tunnels/main.bro @@ -88,10 +88,10 @@ redef dpd_config += { [ANALYZER_AYIYA] = [$ports = ayiya_ports] }; const teredo_ports = { 3544/udp }; redef dpd_config += { [ANALYZER_TEREDO] = [$ports = teredo_ports] }; -const gtpv1u_ports = { 2152/udp }; -redef dpd_config += { [ANALYZER_GTPV1] = [$ports = gtpv1u_ports] }; +const gtpv1_ports = { 2152/udp, 2123/udp }; +redef dpd_config += { [ANALYZER_GTPV1] = [$ports = gtpv1_ports] }; -redef likely_server_ports += { ayiya_ports, teredo_ports, gtpv1u_ports }; +redef likely_server_ports += { ayiya_ports, teredo_ports, gtpv1_ports }; event bro_init() &priority=5 { diff --git a/scripts/base/init-bare.bro b/scripts/base/init-bare.bro index 0920857660..798eb387b3 100644 --- a/scripts/base/init-bare.bro +++ b/scripts/base/init-bare.bro @@ -300,7 +300,7 @@ type connection: record { ## one protocol analyzer is able to parse the same data. If so, all will ## be recorded. Also note that the recorced services are independent of any ## transport-level protocols. - service: set[string]; + service: set[string]; addl: string; ##< Deprecated. hot: count; ##< Deprecated. history: string; ##< State history of connections. See *history* in :bro:see:`Conn::Info`. @@ -1488,6 +1488,146 @@ type gtpv1_hdr: record { next_type: count &optional; }; +type gtp_cause: count; +type gtp_imsi: count; +type gtp_teardown_ind: bool; +type gtp_nsapi: count; +type gtp_recovery: count; +type gtp_teid1: count; +type gtp_teid_control_plane: count; +type gtp_charging_id: count; +type gtp_charging_gateway_addr: addr; +type gtp_trace_reference: count; +type gtp_trace_type: count; +type gtp_tft: string; +type gtp_trigger_id: string; +type gtp_omc_id: string; +type gtp_reordering_required: bool; +type gtp_proto_config_options: string; +type gtp_charging_characteristics: count; +type gtp_selection_mode: count; +type gtp_access_point_name: string; +type gtp_msisdn: string; + +type gtp_gsn_addr: record { + ## If the GSN Address information element has length 4 or 16, then this + ## field is set to be the informational element's value interpreted as + ## an IPv4 or IPv6 address, respectively. + ip: addr &optional; + ## This field is set if it's not an IPv4 or IPv6 address. + other: string &optional; +}; + +type gtp_end_user_addr: record { + pdp_type_org: count; + pdp_type_num: count; + ## Set if the End User Address information element is IPv4/IPv6. + pdp_ip: addr &optional; + ## Set if the End User Address information element isn't IPv4/IPv6. + pdp_other_addr: string &optional; +}; + +type gtp_rai: record { + mcc: count; + mnc: count; + lac: count; + rac: count; +}; + +type gtp_qos_profile: record { + priority: count; + data: string; +}; + +type gtp_private_extension: record { + id: count; + value: string; +}; + +type gtp_create_pdp_ctx_request_elements: record { + imsi: gtp_imsi &optional; + rai: gtp_rai &optional; + recovery: gtp_recovery &optional; + select_mode: gtp_selection_mode &optional; + data1: gtp_teid1; + cp: gtp_teid_control_plane &optional; + nsapi: gtp_nsapi; + linked_nsapi: gtp_nsapi &optional; + charge_character: gtp_charging_characteristics &optional; + trace_ref: gtp_trace_reference &optional; + trace_type: gtp_trace_type &optional; + end_user_addr: gtp_end_user_addr &optional; + ap_name: gtp_access_point_name &optional; + opts: gtp_proto_config_options &optional; + signal_addr: gtp_gsn_addr; + user_addr: gtp_gsn_addr; + msisdn: gtp_msisdn &optional; + qos_prof: gtp_qos_profile; + tft: gtp_tft &optional; + trigger_id: gtp_trigger_id &optional; + omc_id: gtp_omc_id &optional; + ext: gtp_private_extension &optional; +}; + +type gtp_create_pdp_ctx_response_elements: record { + cause: gtp_cause; + reorder_req: gtp_reordering_required &optional; + recovery: gtp_recovery &optional; + data1: gtp_teid1 &optional; + cp: gtp_teid_control_plane &optional; + charging_id: gtp_charging_id &optional; + end_user_addr: gtp_end_user_addr &optional; + opts: gtp_proto_config_options &optional; + cp_addr: gtp_gsn_addr &optional; + user_addr: gtp_gsn_addr &optional; + qos_prof: gtp_qos_profile &optional; + charge_gateway: gtp_charging_gateway_addr &optional; + ext: gtp_private_extension &optional; +}; + +type gtp_update_pdp_ctx_request_elements: record { + imsi: gtp_imsi &optional; + rai: gtp_rai &optional; + recovery: gtp_recovery &optional; + data1: gtp_teid1; + cp: gtp_teid_control_plane &optional; + nsapi: gtp_nsapi; + trace_ref: gtp_trace_reference &optional; + trace_type: gtp_trace_type &optional; + cp_addr: gtp_gsn_addr; + user_addr: gtp_gsn_addr; + qos_prof: gtp_qos_profile; + tft: gtp_tft &optional; + trigger_id: gtp_trigger_id &optional; + omc_id: gtp_omc_id &optional; + ext: gtp_private_extension &optional; + end_user_addr: gtp_end_user_addr &optional; +}; + +type gtp_update_pdp_ctx_response_elements: record { + cause: gtp_cause; + recovery: gtp_recovery &optional; + data1: gtp_teid1 &optional; + cp: gtp_teid_control_plane &optional; + charging_id: gtp_charging_id &optional; + cp_addr: gtp_gsn_addr &optional; + user_addr: gtp_gsn_addr &optional; + qos_prof: gtp_qos_profile &optional; + charge_gateway: gtp_charging_gateway_addr &optional; + ext: gtp_private_extension &optional; +}; + +type gtp_delete_pdp_ctx_request_elements: record { + teardown_ind: gtp_teardown_ind &optional; + nsapi: gtp_nsapi; + ext: gtp_private_extension &optional; +}; + +type gtp_delete_pdp_ctx_response_elements: record { + cause: gtp_cause; + ext: gtp_private_extension &optional; +}; + ## Definition of "secondary filters". A secondary filter is a BPF filter given as ## index in this table. For each such filter, the corresponding event is raised for ## all matching packets. diff --git a/scripts/base/protocols/ftp/__load__.bro b/scripts/base/protocols/ftp/__load__.bro index 15c61be614..464571dc7d 100644 --- a/scripts/base/protocols/ftp/__load__.bro +++ b/scripts/base/protocols/ftp/__load__.bro @@ -1,4 +1,5 @@ @load ./utils-commands @load ./main +@load ./file-analysis @load ./file-extract @load ./gridftp diff --git a/scripts/base/protocols/ftp/file-analysis.bro b/scripts/base/protocols/ftp/file-analysis.bro new file mode 100644 index 0000000000..9131be6ab2 --- /dev/null +++ b/scripts/base/protocols/ftp/file-analysis.bro @@ -0,0 +1,50 @@ +@load ./main +@load base/utils/conn-ids +@load base/frameworks/file-analysis/main + +module FTP; + +export { + ## Determines whether the default :bro:see:`get_file_handle` handler + ## is used to return file handles to the file analysis framework. + ## Redefine to true in order to provide a custom handler which overrides + ## the default for FTP. + const disable_default_file_handle_provider: bool = F &redef; + + ## Default file handle provider for FTP. + function get_file_handle(c: connection, is_orig: bool): string + { + if ( [c$id$resp_h, c$id$resp_p] !in ftp_data_expected ) return ""; + + local info: FTP::Info = ftp_data_expected[c$id$resp_h, c$id$resp_p]; + + local rval = fmt("%s %s %s", ANALYZER_FTP_DATA, c$start_time, + id_string(c$id)); + + if ( info$passive ) + # FTP client initiates data channel. + if ( is_orig ) + # Don't care about FTP client data. + return ""; + else + # Do care about FTP server data. + return rval; + else + # FTP server initiates dta channel. + if ( is_orig ) + # Do care about FTP server data. + return rval; + else + # Don't care about FTP client data. + return ""; + } +} + +module GLOBAL; + +event get_file_handle(tag: AnalyzerTag, c: connection, is_orig: bool) + { + if ( tag != ANALYZER_FTP_DATA ) return; + if ( FTP::disable_default_file_handle_provider ) return; + return_file_handle(FTP::get_file_handle(c, is_orig)); + } diff --git a/scripts/base/protocols/ftp/file-extract.bro b/scripts/base/protocols/ftp/file-extract.bro index 7cee4995ba..b659e779a0 100644 --- a/scripts/base/protocols/ftp/file-extract.bro +++ b/scripts/base/protocols/ftp/file-extract.bro @@ -13,54 +13,96 @@ export { const extraction_prefix = "ftp-item" &redef; } +global extract_count: count = 0; + redef record Info += { ## On disk file where it was extracted to. - extraction_file: file &log &optional; + extraction_file: string &log &optional; ## Indicates if the current command/response pair should attempt to ## extract the file if a file was transferred. extract_file: bool &default=F; - - ## Internal tracking of the total number of files extracted during this - ## session. - num_extracted_files: count &default=0; }; -event file_transferred(c: connection, prefix: string, descr: string, - mime_type: string) &priority=3 +hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info) + &priority=5 { - local id = c$id; - if ( [id$resp_h, id$resp_p] !in ftp_data_expected ) - return; - - local s = ftp_data_expected[id$resp_h, id$resp_p]; + if ( trig != FileAnalysis::TRIGGER_NEW ) return; + if ( ! info?$source ) return; + if ( info$source != "FTP_DATA" ) return; + if ( ! info?$conns ) return; - if ( extract_file_types in s$mime_type ) + local fname: string = fmt("%s-%s-%d.dat", extraction_prefix, info$file_id, + extract_count); + local extracting: bool = F; + + for ( cid in info$conns ) { - s$extract_file = T; - ++s$num_extracted_files; + local c: connection = info$conns[cid]; + + if ( [cid$resp_h, cid$resp_p] !in ftp_data_expected ) next; + + local s = ftp_data_expected[cid$resp_h, cid$resp_p]; + + if ( ! s$extract_file ) next; + + if ( ! extracting ) + { + FileAnalysis::add_action(info$file_id, + [$act=FileAnalysis::ACTION_EXTRACT, + $extract_filename=fname]); + extracting = T; + ++extract_count; + } } } -event file_transferred(c: connection, prefix: string, descr: string, - mime_type: string) &priority=-4 +hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info) + &priority=5 { - local id = c$id; - if ( [id$resp_h, id$resp_p] !in ftp_data_expected ) - return; - - local s = ftp_data_expected[id$resp_h, id$resp_p]; - - if ( s$extract_file ) - { - local suffix = fmt("%d.dat", s$num_extracted_files); - local fname = generate_extraction_filename(extraction_prefix, c, suffix); - s$extraction_file = open(fname); - if ( s$passive ) - set_contents_file(id, CONTENTS_RESP, s$extraction_file); - else - set_contents_file(id, CONTENTS_ORIG, s$extraction_file); - } + if ( trig != FileAnalysis::TRIGGER_TYPE ) return; + if ( ! info?$mime_type ) return; + if ( ! info?$source ) return; + if ( info$source != "FTP_DATA" ) return; + if ( extract_file_types !in info$mime_type ) return; + + for ( act in info$actions ) + if ( act$act == FileAnalysis::ACTION_EXTRACT ) return; + + local fname: string = fmt("%s-%s-%d.dat", extraction_prefix, info$file_id, + extract_count); + ++extract_count; + FileAnalysis::add_action(info$file_id, [$act=FileAnalysis::ACTION_EXTRACT, + $extract_filename=fname]); + } + +hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info) + &priority=-5 + { + if ( trig != FileAnalysis::TRIGGER_EOF && + trig != FileAnalysis::TRIGGER_DONE ) return; + if ( ! info?$source ) return; + if ( info$source != "FTP_DATA" ) return; + + for ( act in info$actions ) + if ( act$act == FileAnalysis::ACTION_EXTRACT ) + { + local s: FTP::Info; + s$ts = network_time(); + s$tags = set(); + s$user = ""; + s$extraction_file = act$extract_filename; + + if ( info?$conns ) + for ( cid in info$conns ) + { + s$uid = info$conns[cid]$uid; + s$id = cid; + break; + } + + Log::write(FTP::LOG, s); + } } event log_ftp(rec: Info) &priority=-10 diff --git a/scripts/base/protocols/ftp/main.bro b/scripts/base/protocols/ftp/main.bro index 3d7b1fe61a..817e00b188 100644 --- a/scripts/base/protocols/ftp/main.bro +++ b/scripts/base/protocols/ftp/main.bro @@ -16,7 +16,8 @@ export { ## List of commands that should have their command/response pairs logged. const logged_commands = { - "APPE", "DELE", "RETR", "STOR", "STOU", "ACCT" + "APPE", "DELE", "RETR", "STOR", "STOU", "ACCT", "PORT", "PASV", "EPRT", + "EPSV" } &redef; ## This setting changes if passwords used in FTP sessions are captured or not. @@ -24,6 +25,18 @@ export { ## User IDs that can be considered "anonymous". const guest_ids = { "anonymous", "ftp", "ftpuser", "guest" } &redef; + + ## The expected endpoints of an FTP data channel. + type ExpectedDataChannel: record { + ## Whether PASV mode is toggled for control channel. + passive: bool &log; + ## The host that will be initiating the data connection. + orig_h: addr &log; + ## The host that will be accepting the data connection. + resp_h: addr &log; + ## The port at which the acceptor is listening for the data connection. + resp_p: port &log; + }; type Info: record { ## Time when the command was sent. @@ -54,7 +67,10 @@ export { reply_msg: string &log &optional; ## Arbitrary tags that may indicate a particular attribute of this command. tags: set[string] &log &default=set(); - + + ## Expected FTP data channel. + data_channel: ExpectedDataChannel &log &optional; + ## Current working directory that this session is in. By making ## the default value '/.', we can indicate that unless something ## more concrete is discovered that the existing but unknown @@ -103,7 +119,7 @@ redef dpd_config += { [ANALYZER_FTP] = [$ports = ports] }; redef likely_server_ports += { 21/tcp, 2811/tcp }; # Establish the variable for tracking expected connections. -global ftp_data_expected: table[addr, port] of Info &create_expire=5mins; +global ftp_data_expected: table[addr, port] of Info &read_expire=5mins; event bro_init() &priority=5 { @@ -180,7 +196,7 @@ function ftp_message(s: Info) delete s$arg; else s$arg=arg; - + Log::write(FTP::LOG, s); } @@ -190,8 +206,19 @@ function ftp_message(s: Info) delete s$mime_type; delete s$mime_desc; delete s$file_size; + # Same with data channel. + delete s$data_channel; # Tags are cleared everytime too. - delete s$tags; + s$tags = set(); + } + +function add_expected_data_channel(s: Info, chan: ExpectedDataChannel) + { + s$passive = chan$passive; + s$data_channel = chan; + ftp_data_expected[chan$resp_h, chan$resp_p] = s; + expect_connection(chan$orig_h, chan$resp_h, chan$resp_p, ANALYZER_FTP_DATA, + 5mins); } event ftp_request(c: connection, command: string, arg: string) &priority=5 @@ -226,9 +253,8 @@ event ftp_request(c: connection, command: string, arg: string) &priority=5 if ( data$valid ) { - c$ftp$passive=F; - ftp_data_expected[data$h, data$p] = c$ftp; - expect_connection(id$resp_h, data$h, data$p, ANALYZER_FILE, 5mins); + add_expected_data_channel(c$ftp, [$passive=F, $orig_h=id$resp_h, + $resp_h=data$h, $resp_p=data$p]); } else { @@ -279,9 +305,9 @@ event ftp_reply(c: connection, code: count, msg: string, cont_resp: bool) &prior if ( code == 229 && data$h == [::] ) data$h = id$resp_h; - - ftp_data_expected[data$h, data$p] = c$ftp; - expect_connection(id$orig_h, data$h, data$p, ANALYZER_FILE, 5mins); + + add_expected_data_channel(c$ftp, [$passive=T, $orig_h=id$orig_h, + $resp_h=data$h, $resp_p=data$p]); } else { @@ -331,14 +357,11 @@ event file_transferred(c: connection, prefix: string, descr: string, } } -event file_transferred(c: connection, prefix: string, descr: string, - mime_type: string) &priority=-5 +event connection_state_remove(c: connection) &priority=-5 { - local id = c$id; - if ( [id$resp_h, id$resp_p] in ftp_data_expected ) - delete ftp_data_expected[id$resp_h, id$resp_p]; + delete ftp_data_expected[c$id$resp_h, c$id$resp_p]; } - + # Use state remove event to cover connections terminated by RST. event connection_state_remove(c: connection) &priority=-5 { diff --git a/scripts/base/protocols/http/__load__.bro b/scripts/base/protocols/http/__load__.bro index 314f04b872..58618dedc7 100644 --- a/scripts/base/protocols/http/__load__.bro +++ b/scripts/base/protocols/http/__load__.bro @@ -1,5 +1,6 @@ @load ./main @load ./utils +@load ./file-analysis @load ./file-ident @load ./file-hash @load ./file-extract diff --git a/scripts/base/protocols/http/file-analysis.bro b/scripts/base/protocols/http/file-analysis.bro new file mode 100644 index 0000000000..b20e561dfc --- /dev/null +++ b/scripts/base/protocols/http/file-analysis.bro @@ -0,0 +1,36 @@ +@load ./main +@load ./utils +@load base/utils/conn-ids +@load base/frameworks/file-analysis/main + +module HTTP; + +export { + ## Determines whether the default :bro:see:`get_file_handle` handler + ## is used to return file handles to the file analysis framework. + ## Redefine to true in order to provide a custom handler which overrides + ## the default HTTP. + const disable_default_file_handle_provider: bool = F &redef; + + ## Default file handle provider for HTTP. + function get_file_handle(c: connection, is_orig: bool): string + { + if ( ! c?$http ) return ""; + + if ( c$http$range_request ) + return fmt("%s %s %s %s", ANALYZER_HTTP, is_orig, c$id$orig_h, + build_url(c$http)); + + return fmt("%s %s %s %s %s", ANALYZER_HTTP, c$start_time, is_orig, + c$http$trans_depth, id_string(c$id)); + } +} + +module GLOBAL; + +event get_file_handle(tag: AnalyzerTag, c: connection, is_orig: bool) + { + if ( tag != ANALYZER_HTTP ) return; + if ( HTTP::disable_default_file_handle_provider ) return; + return_file_handle(HTTP::get_file_handle(c, is_orig)); + } diff --git a/scripts/base/protocols/http/file-extract.bro b/scripts/base/protocols/http/file-extract.bro index 466d18c3b4..079636fb6a 100644 --- a/scripts/base/protocols/http/file-extract.bro +++ b/scripts/base/protocols/http/file-extract.bro @@ -2,8 +2,7 @@ ##! the message body from the server can be extracted with this script. @load ./main -@load ./file-ident -@load base/utils/files +@load ./file-analysis module HTTP; @@ -16,45 +15,77 @@ export { redef record Info += { ## On-disk file where the response body was extracted to. - extraction_file: file &log &optional; + extraction_file: string &log &optional; ## Indicates if the response body is to be extracted or not. Must be - ## set before or by the first :bro:id:`http_entity_data` event for the - ## content. + ## set before or by the first :bro:enum:`FileAnalysis::TRIGGER_NEW` + ## for the file content. extract_file: bool &default=F; }; } -event http_entity_data(c: connection, is_orig: bool, length: count, data: string) &priority=-5 +global extract_count: count = 0; + +hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info) + &priority=5 { - # Client body extraction is not currently supported in this script. - if ( is_orig ) - return; - - if ( c$http$first_chunk ) + if ( trig != FileAnalysis::TRIGGER_TYPE ) return; + if ( ! info?$mime_type ) return; + if ( ! info?$source ) return; + if ( info$source != "HTTP" ) return; + if ( extract_file_types !in info$mime_type ) return; + + for ( act in info$actions ) + if ( act$act == FileAnalysis::ACTION_EXTRACT ) return; + + local fname: string = fmt("%s-%s-%d.dat", extraction_prefix, info$file_id, + extract_count); + ++extract_count; + FileAnalysis::add_action(info$file_id, [$act=FileAnalysis::ACTION_EXTRACT, + $extract_filename=fname]); + + if ( ! info?$conns ) return; + + for ( cid in info$conns ) { - if ( c$http?$mime_type && - extract_file_types in c$http$mime_type ) - { - c$http$extract_file = T; - } - + local c: connection = info$conns[cid]; + + if ( ! c?$http ) next; + + c$http$extraction_file = fname; + } + } + +hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info) + &priority=5 + { + if ( trig != FileAnalysis::TRIGGER_NEW ) return; + if ( ! info?$source ) return; + if ( info$source != "HTTP" ) return; + if ( ! info?$conns ) return; + + local fname: string = fmt("%s-%s-%d.dat", extraction_prefix, info$file_id, + extract_count); + local extracting: bool = F; + + for ( cid in info$conns ) + { + local c: connection = info$conns[cid]; + + if ( ! c?$http ) next; + if ( c$http$extract_file ) { - local suffix = fmt("%s_%d.dat", is_orig ? "orig" : "resp", c$http_state$current_response); - local fname = generate_extraction_filename(extraction_prefix, c, suffix); - - c$http$extraction_file = open(fname); - enable_raw_output(c$http$extraction_file); + if ( ! extracting ) + { + FileAnalysis::add_action(info$file_id, + [$act=FileAnalysis::ACTION_EXTRACT, + $extract_filename=fname]); + extracting = T; + ++extract_count; + } + + c$http$extraction_file = fname; } } - - if ( c$http?$extraction_file ) - print c$http$extraction_file, data; - } - -event http_end_entity(c: connection, is_orig: bool) - { - if ( c$http?$extraction_file ) - close(c$http$extraction_file); } diff --git a/scripts/base/protocols/http/file-hash.bro b/scripts/base/protocols/http/file-hash.bro index 2545cbf817..e24cda0b72 100644 --- a/scripts/base/protocols/http/file-hash.bro +++ b/scripts/base/protocols/http/file-hash.bro @@ -1,15 +1,11 @@ ##! Calculate hashes for HTTP body transfers. -@load ./file-ident +@load ./main +@load ./file-analysis module HTTP; export { - redef enum Notice::Type += { - ## Indicates that an MD5 sum was calculated for an HTTP response body. - MD5, - }; - redef record Info += { ## MD5 sum for a file transferred over HTTP calculated from the ## response body. @@ -19,10 +15,6 @@ export { ## if a file should have an MD5 sum generated. It must be ## set to T at the time of or before the first chunk of body data. calc_md5: bool &default=F; - - ## Indicates if an MD5 sum is being calculated for the current - ## request/response pair. - md5_handle: opaque of md5 &optional; }; ## Generate MD5 sums for these filetypes. @@ -31,62 +23,67 @@ export { &redef; } -## Initialize and calculate the hash. -event http_entity_data(c: connection, is_orig: bool, length: count, data: string) &priority=5 +hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info) + &priority=5 { - if ( is_orig || ! c?$http ) return; - - if ( c$http$first_chunk ) + if ( trig != FileAnalysis::TRIGGER_TYPE ) return; + if ( ! info?$mime_type ) return; + if ( ! info?$source ) return; + if ( info$source != "HTTP" ) return; + + if ( generate_md5 in info$mime_type ) + FileAnalysis::add_action(info$file_id, [$act=FileAnalysis::ACTION_MD5]); + else if ( info?$conns ) { - if ( c$http$calc_md5 || - (c$http?$mime_type && generate_md5 in c$http$mime_type) ) + for ( cid in info$conns ) { - c$http$md5_handle = md5_hash_init(); + local c: connection = info$conns[cid]; + + if ( ! c?$http ) next; + + if ( c$http$calc_md5 ) + { + FileAnalysis::add_action(info$file_id, + [$act=FileAnalysis::ACTION_MD5]); + return; + } } } - - if ( c$http?$md5_handle ) - md5_hash_update(c$http$md5_handle, data); - } - -## In the event of a content gap during a file transfer, detect the state for -## the MD5 sum calculation and stop calculating the MD5 since it would be -## incorrect anyway. -event content_gap(c: connection, is_orig: bool, seq: count, length: count) &priority=5 - { - if ( is_orig || ! c?$http || ! c$http?$md5_handle ) return; - - set_state(c, F, is_orig); - md5_hash_finish(c$http$md5_handle); # Ignore return value. - delete c$http$md5_handle; } -## When the file finishes downloading, finish the hash and generate a notice. -event http_message_done(c: connection, is_orig: bool, stat: http_message_stat) &priority=-3 +hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info) + &priority=5 { - if ( is_orig || ! c?$http ) return; - - if ( c$http?$md5_handle ) + if ( trig != FileAnalysis::TRIGGER_DONE && + trig != FileAnalysis::TRIGGER_EOF ) return; + if ( ! info?$source ) return; + if ( info$source != "HTTP" ) return; + if ( ! info?$conns ) return; + + local act: FileAnalysis::ActionArgs = [$act=FileAnalysis::ACTION_MD5]; + + if ( act !in info$actions ) return; + + local result = info$actions[act]; + + if ( ! result?$md5 ) return; + + for ( cid in info$conns ) { - local url = build_url_http(c$http); - c$http$md5 = md5_hash_finish(c$http$md5_handle); - delete c$http$md5_handle; - - NOTICE([$note=MD5, $msg=fmt("%s %s %s", c$id$orig_h, c$http$md5, url), - $sub=c$http$md5, $conn=c]); + local c: connection = info$conns[cid]; + + if ( ! c?$http ) next; + + c$http$md5 = result$md5; } } -event connection_state_remove(c: connection) &priority=-5 +hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info) + &priority=5 { - if ( c?$http_state && - c$http_state$current_response in c$http_state$pending && - c$http_state$pending[c$http_state$current_response]?$md5_handle ) - { - # The MD5 sum isn't going to be saved anywhere since the entire - # body wouldn't have been seen anyway and we'd just be giving an - # incorrect MD5 sum. - md5_hash_finish(c$http$md5_handle); - delete c$http$md5_handle; - } + if ( trig != FileAnalysis::TRIGGER_GAP ) return; + if ( ! info?$source ) return; + if ( info$source != "HTTP" ) return; + + FileAnalysis::remove_action(info$file_id, [$act=FileAnalysis::ACTION_MD5]); } diff --git a/scripts/base/protocols/http/file-ident.bro b/scripts/base/protocols/http/file-ident.bro index 706ea58558..71802c3bde 100644 --- a/scripts/base/protocols/http/file-ident.bro +++ b/scripts/base/protocols/http/file-ident.bro @@ -1,15 +1,9 @@ ##! Identification of file types in HTTP response bodies with file content sniffing. -@load base/frameworks/signatures @load base/frameworks/notice @load ./main @load ./utils - -# Add the magic number signatures to the core signature set. -@load-sigs ./file-ident.sig - -# Ignore the signatures used to match files -redef Signatures::ignored_ids += /^matchfile-/; +@load ./file-analysis module HTTP; @@ -22,11 +16,6 @@ export { redef record Info += { ## Mime type of response body identified by content sniffing. mime_type: string &log &optional; - - ## Indicates that no data of the current file transfer has been - ## seen yet. After the first :bro:id:`http_entity_data` event, it - ## will be set to F. - first_chunk: bool &default=T; }; ## Mapping between mime types and regular expressions for URLs @@ -43,43 +32,34 @@ export { const ignored_incorrect_file_type_urls = /^$/ &redef; } -event signature_match(state: signature_state, msg: string, data: string) &priority=5 +hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info) + &priority=5 { - # Only signatures matching file types are dealt with here. - if ( /^matchfile-/ !in state$sig_id ) return; + if ( trig != FileAnalysis::TRIGGER_TYPE ) return; + if ( ! info?$mime_type ) return; + if ( ! info?$source ) return; + if ( info$source != "HTTP" ) return; + if ( ! info?$conns ) return; - local c = state$conn; - set_state(c, F, F); - - # Not much point in any of this if we don't know about the HTTP session. - if ( ! c?$http ) return; - - # Set the mime type that was detected. - c$http$mime_type = msg; - - if ( msg in mime_types_extensions && - c$http?$uri && mime_types_extensions[msg] !in c$http$uri ) + for ( cid in info$conns ) { + local c: connection = info$conns[cid]; + + if ( ! c?$http ) next; + + c$http$mime_type = info$mime_type; + + if ( info$mime_type !in mime_types_extensions ) next; + if ( ! c$http?$uri ) next; + if ( mime_types_extensions[info$mime_type] in c$http$uri ) next; + local url = build_url_http(c$http); - - if ( url == ignored_incorrect_file_type_urls ) - return; - - local message = fmt("%s %s %s", msg, c$http$method, url); + + if ( url == ignored_incorrect_file_type_urls ) next; + + local message = fmt("%s %s %s", info$mime_type, c$http$method, url); NOTICE([$note=Incorrect_File_Type, $msg=message, $conn=c]); } } - -event http_entity_data(c: connection, is_orig: bool, length: count, data: string) &priority=5 - { - if ( c$http$first_chunk && ! c$http?$mime_type ) - c$http$mime_type = split1(identify_data(data, T), /;/)[1]; - } - -event http_entity_data(c: connection, is_orig: bool, length: count, data: string) &priority=-10 - { - if ( c$http$first_chunk ) - c$http$first_chunk=F; - } diff --git a/scripts/base/protocols/http/file-ident.sig b/scripts/base/protocols/http/file-ident.sig deleted file mode 100644 index 971a32bbfc..0000000000 --- a/scripts/base/protocols/http/file-ident.sig +++ /dev/null @@ -1,144 +0,0 @@ -# These signatures are used as a replacement for libmagic. The signature -# name needs to start with "matchfile" and the "event" directive takes -# the mime type of the file matched by the http-reply-body pattern. -# -# Signatures from: http://www.garykessler.net/library/file_sigs.html - -signature matchfile-exe { - http-reply-body /\x4D\x5A/ - event "application/x-dosexec" -} - -signature matchfile-elf { - http-reply-body /\x7F\x45\x4C\x46/ - event "application/x-executable" -} - -signature matchfile-script { - # This is meant to match the interpreter declaration at the top of many - # interpreted scripts. - http-reply-body /\#\![[:blank:]]?\// - event "application/x-script" -} - -signature matchfile-wmv { - http-reply-body /\x30\x26\xB2\x75\x8E\x66\xCF\x11\xA6\xD9\x00\xAA\x00\x62\xCE\x6C/ - event "video/x-ms-wmv" -} - -signature matchfile-flv { - http-reply-body /\x46\x4C\x56\x01/ - event "video/x-flv" -} - -signature matchfile-swf { - http-reply-body /[\x46\x43]\x57\x53/ - event "application/x-shockwave-flash" -} - -signature matchfile-jar { - http-reply-body /\x5F\x27\xA8\x89/ - event "application/java-archive" -} - -signature matchfile-class { - http-reply-body /\xCA\xFE\xBA\xBE/ - event "application/java-byte-code" -} - -signature matchfile-msoffice-2007 { - # MS Office 2007 XML documents - http-reply-body /\x50\x4B\x03\x04\x14\x00\x06\x00/ - event "application/msoffice" -} - -signature matchfile-msoffice { - # Older MS Office files - http-reply-body /\xD0\xCF\x11\xE0\xA1\xB1\x1A\xE1/ - event "application/msoffice" -} - -signature matchfile-rtf { - http-reply-body /\x7B\x5C\x72\x74\x66\x31/ - event "application/rtf" -} - -signature matchfile-lnk { - http-reply-body /\x4C\x00\x00\x00\x01\x14\x02\x00\x00\x00\x00\x00\xC0\x00\x00\x00\x00\x00\x00\x46/ - event "application/x-ms-shortcut" -} - -signature matchfile-torrent { - http-reply-body /\x64\x38\x3A\x61\x6E\x6E\x6F\x75\x6E\x63\x65/ - event "application/x-bittorrent" -} - -signature matchfile-pdf { - http-reply-body /\x25\x50\x44\x46/ - event "application/pdf" -} - -signature matchfile-html { - http-reply-body /<[hH][tT][mM][lL]/ - event "text/html" -} - -signature matchfile-html2 { - http-reply-body / info$bof_buffer_size ) + info$bof_buffer_size = default_entity_excerpt_len; } -event mime_segment_data(c: connection, length: count, data: string) &priority=-2 +hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info) + &priority=5 { - if ( ! c?$smtp ) return; - - local ent = c$smtp$current_entity; - if ( ent$content_len < ent$excerpt_len ) + if ( trig != FileAnalysis::TRIGGER_BOF_BUFFER ) return; + if ( ! info?$bof_buffer ) return; + if ( ! info?$source ) return; + if ( info$source != "SMTP" ) return; + if ( ! info?$conns ) return; + + for ( cid in info$conns ) { - if ( ent$content_len + length < ent$excerpt_len ) - ent$excerpt = cat(ent$excerpt, data); - else - { - local x_bytes = ent$excerpt_len - ent$content_len; - ent$excerpt = cat(ent$excerpt, sub_bytes(data, 1, x_bytes)); - } + local c: connection = info$conns[cid]; + + if ( ! c?$smtp ) next; + + if ( default_entity_excerpt_len > 0 ) + c$smtp$current_entity$excerpt = + info$bof_buffer[0:default_entity_excerpt_len]; } } diff --git a/scripts/base/protocols/smtp/entities.bro b/scripts/base/protocols/smtp/entities.bro index 80d6089ce7..d009068ad0 100644 --- a/scripts/base/protocols/smtp/entities.bro +++ b/scripts/base/protocols/smtp/entities.bro @@ -7,11 +7,6 @@ module SMTP; export { - redef enum Notice::Type += { - ## Indicates that an MD5 sum was calculated for a MIME message. - MD5, - }; - redef enum Log::ID += { ENTITIES_LOG }; type EntityInfo: record { @@ -34,15 +29,12 @@ export { ## Optionally calculate the file's MD5 sum. Must be set prior to the ## first data chunk being see in an event. calc_md5: bool &default=F; - ## This boolean value indicates if an MD5 sum is being calculated - ## for the current file transfer. - md5_handle: opaque of md5 &optional; ## Optionally write the file to disk. Must be set prior to first ## data chunk being seen in an event. extract_file: bool &default=F; ## Store the file handle here for the file currently being extracted. - extraction_file: file &log &optional; + extraction_file: string &log &optional; }; redef record Info += { @@ -51,9 +43,6 @@ export { }; redef record State += { - ## Store a count of the number of files that have been transferred in - ## a conversation to create unique file names on disk. - num_extracted_files: count &default=0; ## Track the number of MIME encoded files transferred during a session. mime_level: count &default=0; }; @@ -77,6 +66,8 @@ export { global log_mime: event(rec: EntityInfo); } +global extract_count: count = 0; + event bro_init() &priority=5 { Log::create_stream(SMTP::ENTITIES_LOG, [$columns=EntityInfo, $ev=log_mime]); @@ -100,74 +91,155 @@ function set_session(c: connection, new_entity: bool) event mime_begin_entity(c: connection) &priority=10 { if ( ! c?$smtp ) return; - + set_session(c, T); } -# This has priority -10 because other handlers need to know the current -# content_len before it's updated by this handler. -event mime_segment_data(c: connection, length: count, data: string) &priority=-10 +hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info) + &priority=5 { - if ( ! c?$smtp ) return; - - c$smtp$current_entity$content_len = c$smtp$current_entity$content_len + length; - } + if ( trig != FileAnalysis::TRIGGER_NEW ) return; + if ( ! info?$source ) return; + if ( info$source != "SMTP" ) return; + if ( ! info?$conns ) return; -event mime_segment_data(c: connection, length: count, data: string) &priority=7 - { - if ( ! c?$smtp ) return; - if ( c$smtp$current_entity$content_len == 0 ) - c$smtp$current_entity$mime_type = split1(identify_data(data, T), /;/)[1]; - } + local fname: string = fmt("%s-%s-%d.dat", extraction_prefix, info$file_id, + extract_count); + local extracting: bool = F; -event mime_segment_data(c: connection, length: count, data: string) &priority=-5 - { - if ( ! c?$smtp ) return; - - if ( c$smtp$current_entity$content_len == 0 ) + for ( cid in info$conns ) { - local entity = c$smtp$current_entity; - if ( generate_md5 in entity$mime_type && ! never_calc_md5 ) - entity$calc_md5 = T; + local c: connection = info$conns[cid]; - if ( entity$calc_md5 ) - entity$md5_handle = md5_hash_init(); - } + if ( ! c?$smtp ) next; - if ( c$smtp$current_entity?$md5_handle ) - md5_hash_update(entity$md5_handle, data); -} + if ( c$smtp$current_entity$extract_file ) + { + if ( ! extracting ) + { + FileAnalysis::add_action(info$file_id, + [$act=FileAnalysis::ACTION_EXTRACT, + $extract_filename=fname]); + extracting = T; + ++extract_count; + } -## In the event of a content gap during the MIME transfer, detect the state for -## the MD5 sum calculation and stop calculating the MD5 since it would be -## incorrect anyway. -event content_gap(c: connection, is_orig: bool, seq: count, length: count) &priority=5 - { - if ( is_orig || ! c?$smtp || ! c$smtp?$current_entity ) return; + c$smtp$current_entity$extraction_file = fname; + } - local entity = c$smtp$current_entity; - if ( entity?$md5_handle ) - { - md5_hash_finish(entity$md5_handle); - delete entity$md5_handle; + if ( c$smtp$current_entity$calc_md5 ) + FileAnalysis::add_action(info$file_id, + [$act=FileAnalysis::ACTION_MD5]); } } -event mime_end_entity(c: connection) &priority=-3 - { - # TODO: this check is only due to a bug in mime_end_entity that - # causes the event to be generated twice for the same real event. - if ( ! c?$smtp || ! c$smtp?$current_entity ) +function check_extract_by_type(info: FileAnalysis::Info) + { + if ( extract_file_types !in info$mime_type ) return; + + for ( act in info$actions ) + if ( act$act == FileAnalysis::ACTION_EXTRACT ) return; + + local fname: string = fmt("%s-%s-%d.dat", extraction_prefix, info$file_id, + extract_count); + ++extract_count; + FileAnalysis::add_action(info$file_id, [$act=FileAnalysis::ACTION_EXTRACT, + $extract_filename=fname]); + + if ( ! info?$conns ) return; + + for ( cid in info$conns ) + { + local c: connection = info$conns[cid]; + + if ( ! c?$smtp ) next; + + c$smtp$current_entity$extraction_file = fname; + } + } + +function check_md5_by_type(info: FileAnalysis::Info) + { + if ( never_calc_md5 ) return; + if ( generate_md5 !in info$mime_type ) return; + + FileAnalysis::add_action(info$file_id, [$act=FileAnalysis::ACTION_MD5]); + } + +hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info) + &priority=5 + { + if ( trig != FileAnalysis::TRIGGER_TYPE ) return; + if ( ! info?$mime_type ) return; + if ( ! info?$source ) return; + if ( info$source != "SMTP" ) return; + + if ( info?$conns ) + for ( cid in info$conns ) + { + local c: connection = info$conns[cid]; + + if ( ! c?$smtp ) next; + + c$smtp$current_entity$mime_type = info$mime_type; + } + + check_extract_by_type(info); + check_md5_by_type(info); + } + +hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info) + &priority=5 + { + if ( trig != FileAnalysis::TRIGGER_GAP ) return; + if ( ! info?$source ) return; + if ( info$source != "SMTP" ) return; + if ( ! info?$conns ) return; + + for ( cid in info$conns ) + { + local c: connection = info$conns[cid]; + + if ( ! c?$smtp ) next; + if ( ! c$smtp?$current_entity ) next; + + FileAnalysis::remove_action(info$file_id, + [$act=FileAnalysis::ACTION_MD5]); + } + } + +hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info) + &priority=5 + { + if ( trig != FileAnalysis::TRIGGER_EOF && + trig != FileAnalysis::TRIGGER_DONE ) return; + if ( ! info?$source ) return; + if ( info$source != "SMTP" ) return; + if ( ! info?$conns ) return; + + for ( cid in info$conns ) + { + local c: connection = info$conns[cid]; + + if ( ! c?$smtp ) next; + if ( ! c$smtp?$current_entity ) next; + # Only log is there was some content. + if ( info$seen_bytes == 0 ) next; + + local act: FileAnalysis::ActionArgs = [$act=FileAnalysis::ACTION_MD5]; + + if ( act in info$actions ) + { + local result = info$actions[act]; + if ( result?$md5 ) + c$smtp$current_entity$md5 = result$md5; + } + + c$smtp$current_entity$content_len = info$seen_bytes; + + Log::write(SMTP::ENTITIES_LOG, c$smtp$current_entity); + delete c$smtp$current_entity; return; - - local entity = c$smtp$current_entity; - if ( entity?$md5_handle ) - { - entity$md5 = md5_hash_finish(entity$md5_handle); - delete entity$md5_handle; - - NOTICE([$note=MD5, $msg=fmt("Calculated a hash for a MIME entity from %s", c$id$orig_h), - $sub=entity$md5, $conn=c]); } } @@ -179,66 +251,7 @@ event mime_one_header(c: connection, h: mime_header_rec) /[fF][iI][lL][eE][nN][aA][mM][eE]/ in h$value ) c$smtp$current_entity$filename = extract_filename_from_content_disposition(h$value); - if ( h$name == "CONTENT-TYPE" && + if ( h$name == "CONTENT-TYPE" && /[nN][aA][mM][eE][:blank:]*=/ in h$value ) c$smtp$current_entity$filename = extract_filename_from_content_disposition(h$value); } - -event mime_end_entity(c: connection) &priority=-5 - { - if ( ! c?$smtp ) return; - - # This check and the delete below are just to cope with a bug where - # mime_end_entity can be generated multiple times for the same event. - if ( ! c$smtp?$current_entity ) - return; - - # Only log is there was some content. - if ( c$smtp$current_entity$content_len > 0 ) - Log::write(SMTP::ENTITIES_LOG, c$smtp$current_entity); - - delete c$smtp$current_entity; - } - -event mime_segment_data(c: connection, length: count, data: string) &priority=5 - { - if ( ! c?$smtp ) return; - - if ( extract_file_types in c$smtp$current_entity$mime_type ) - c$smtp$current_entity$extract_file = T; - } - -event mime_segment_data(c: connection, length: count, data: string) &priority=3 - { - if ( ! c?$smtp ) return; - - if ( c$smtp$current_entity$extract_file && - c$smtp$current_entity$content_len == 0 ) - { - local suffix = fmt("%d.dat", ++c$smtp_state$num_extracted_files); - local fname = generate_extraction_filename(extraction_prefix, c, suffix); - c$smtp$current_entity$extraction_file = open(fname); - enable_raw_output(c$smtp$current_entity$extraction_file); - } - } - -event mime_segment_data(c: connection, length: count, data: string) &priority=-5 - { - if ( ! c?$smtp ) return; - - if ( c$smtp$current_entity$extract_file && c$smtp$current_entity?$extraction_file ) - print c$smtp$current_entity$extraction_file, data; - } - -event mime_end_entity(c: connection) &priority=-3 - { - if ( ! c?$smtp ) return; - - # TODO: this check is only due to a bug in mime_end_entity that - # causes the event to be generated twice for the same real event. - if ( ! c$smtp?$current_entity ) - return; - - if ( c$smtp$current_entity?$extraction_file ) - close(c$smtp$current_entity$extraction_file); - } diff --git a/scripts/base/protocols/smtp/file-analysis.bro b/scripts/base/protocols/smtp/file-analysis.bro new file mode 100644 index 0000000000..cbc33aa375 --- /dev/null +++ b/scripts/base/protocols/smtp/file-analysis.bro @@ -0,0 +1,32 @@ +@load ./main +@load ./entities +@load base/utils/conn-ids +@load base/frameworks/file-analysis/main + +module SMTP; + +export { + ## Determines whether the default :bro:see:`get_file_handle` handler + ## is used to return file handles to the file analysis framework. + ## Redefine to true in order to provide a custom handler which overrides + ## the default for SMTP. + const disable_default_file_handle_provider: bool = F &redef; + + ## Default file handle provider for SMTP. + function get_file_handle(c: connection, is_orig: bool): string + { + if ( ! c?$smtp ) return ""; + + return fmt("%s %s %s %s", ANALYZER_SMTP, c$start_time, + c$smtp$trans_depth, c$smtp_state$mime_level); + } +} + +module GLOBAL; + +event get_file_handle(tag: AnalyzerTag, c: connection, is_orig: bool) + { + if ( tag != ANALYZER_SMTP ) return; + if ( SMTP::disable_default_file_handle_provider ) return; + return_file_handle(SMTP::get_file_handle(c, is_orig)); + } diff --git a/scripts/base/protocols/ssl/main.bro b/scripts/base/protocols/ssl/main.bro index 2dcbfee5ef..c6c4091a87 100644 --- a/scripts/base/protocols/ssl/main.bro +++ b/scripts/base/protocols/ssl/main.bro @@ -67,11 +67,6 @@ export { ## (especially with large file transfers). const disable_analyzer_after_detection = T &redef; - ## The openssl command line utility. If it's in the path the default - ## value will work, otherwise a full path string can be supplied for the - ## utility. - const openssl_util = "openssl" &redef; - ## The maximum amount of time a script can delay records from being logged. const max_log_delay = 15secs &redef; diff --git a/scripts/base/utils/paths.bro b/scripts/base/utils/paths.bro index 92f0745be4..aa083ddf5b 100644 --- a/scripts/base/utils/paths.bro +++ b/scripts/base/utils/paths.bro @@ -27,7 +27,7 @@ function compress_path(dir: string): string const cdup_sep = /((\/)*([^\/]|\\\/)+)?((\/)+\.\.(\/)*)/; local parts = split_n(dir, cdup_sep, T, 1); - if ( length(parts) > 1 ) + if ( |parts| > 1 ) { # reaching a point with two parent dir references back-to-back means # we don't know about anything higher in the tree to pop off diff --git a/scripts/base/utils/strings.bro b/scripts/base/utils/strings.bro index 2836f368b4..560ba6d160 100644 --- a/scripts/base/utils/strings.bro +++ b/scripts/base/utils/strings.bro @@ -6,7 +6,7 @@ ## characters. function is_string_binary(s: string): bool { - return byte_len(gsub(s, /[\x00-\x7f]/, "")) * 100 / |s| >= 25; + return |gsub(s, /[\x00-\x7f]/, "")| * 100 / |s| >= 25; } ## Joins a set of string together, with elements delimited by a constant string. diff --git a/scripts/policy/frameworks/intel/smtp-url-extraction.bro b/scripts/policy/frameworks/intel/smtp-url-extraction.bro index 78c8f74bf2..c5b5085a10 100644 --- a/scripts/policy/frameworks/intel/smtp-url-extraction.bro +++ b/scripts/policy/frameworks/intel/smtp-url-extraction.bro @@ -1,15 +1,34 @@ @load base/frameworks/intel +@load base/protocols/smtp/file-analysis @load base/utils/urls @load ./where-locations -event mime_segment_data(c: connection, length: count, data: string) &priority=3 +event intel_mime_data(info: FileAnalysis::Info, data: string) { - local urls = find_all_urls_without_scheme(data); - for ( url in urls ) + if ( ! info?$conns ) return; + + for ( cid in info$conns ) { - Intel::seen([$str=url, - $str_type=Intel::URL, - $conn=c, - $where=SMTP::IN_MESSAGE]); + local c: connection = info$conns[cid]; + local urls = find_all_urls_without_scheme(data); + for ( url in urls ) + { + Intel::seen([$str=url, + $str_type=Intel::URL, + $conn=c, + $where=SMTP::IN_MESSAGE]); + } } - } \ No newline at end of file + } + +hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info) + &priority=5 + { + if ( trig != FileAnalysis::TRIGGER_NEW ) return; + if ( ! info?$source ) return; + if ( info$source != "SMTP" ) return; + + FileAnalysis::add_action(info$file_id, + [$act=FileAnalysis::ACTION_DATA_EVENT, + $stream_event=intel_mime_data]); + } diff --git a/scripts/policy/frameworks/software/vulnerable.bro b/scripts/policy/frameworks/software/vulnerable.bro index c2c2ba5b32..aedb309dba 100644 --- a/scripts/policy/frameworks/software/vulnerable.bro +++ b/scripts/policy/frameworks/software/vulnerable.bro @@ -1,7 +1,8 @@ -##! Provides a variable to define vulnerable versions of software and if a -##! a version of that software as old or older than the defined version a +##! Provides a variable to define vulnerable versions of software and if a +##! a version of that software as old or older than the defined version a ##! notice will be generated. +@load base/frameworks/control @load base/frameworks/notice @load base/frameworks/software @@ -13,17 +14,126 @@ export { Vulnerable_Version, }; - ## This is a table of software versions indexed by the name of the - ## software and yielding the latest version that is vulnerable. - const vulnerable_versions: table[string] of Version &redef; + type VulnerableVersionRange: record { + ## The minimal version of a vulnerable version range. This + ## field can be undefined if all previous versions of a piece + ## of software are vulnerable. + min: Software::Version &optional; + ## The maximum vulnerable version. This field is deliberately + ## not optional because a maximum vulnerable version must + ## always be defined. This assumption may become incorrent + ## if all future versions of some software are to be considered + ## vulnerable. :) + max: Software::Version; + }; + + ## The DNS zone where runtime vulnerable software updates will + ## be loaded from. + const vulnerable_versions_update_endpoint = "" &redef; + + ## The interval at which vulnerable versions should grab updates + ## over DNS. + const vulnerable_versions_update_interval = 1hr &redef; + + ## This is a table of software versions indexed by the name of the + ## software and a set of version ranges that are declared to be + ## vulnerable for that software. + const vulnerable_versions: table[string] of set[VulnerableVersionRange] = table() &redef; } +global internal_vulnerable_versions: table[string] of set[VulnerableVersionRange] = table(); + +event Control::configuration_update() + { + internal_vulnerable_versions = table(); + + # Copy the const vulnerable versions into the global modifiable one. + for ( sw in vulnerable_versions ) + internal_vulnerable_versions[sw] = vulnerable_versions[sw]; + } + +function decode_vulnerable_version_range(vuln_sw: string): VulnerableVersionRange + { + # Create a max value with a dunce value only because the $max field + # is not optional. + local vvr: Software::VulnerableVersionRange = [$max=[$major=0]]; + + if ( /max=/ !in vuln_sw ) + { + Reporter::warning(fmt("The vulnerable software detection script encountered a version with no max value (which is required). %s", vuln_sw)); + return vvr; + } + + local versions = split1(vuln_sw, /\x09/); + + for ( i in versions ) + { + local field_and_ver = split1(versions[i], /=/); + if ( |field_and_ver| != 2 ) + return vvr; #failure! + + local ver = Software::parse(field_and_ver[2])$version; + if ( field_and_ver[1] == "min" ) + vvr$min = ver; + else if ( field_and_ver[1] == "max" ) + vvr$max = ver; + } + + return vvr; + } + +event grab_vulnerable_versions(i: count) + { + if ( vulnerable_versions_update_endpoint == "" ) + { + # Reschedule this event in case the user updates the setting at runtime. + schedule vulnerable_versions_update_interval { grab_vulnerable_versions(1) }; + return; + } + + when ( local result = lookup_hostname_txt(cat(i,".",vulnerable_versions_update_endpoint)) ) + { + local parts = split1(result, /\x09/); + if ( |parts| != 2 ) #failure or end of list! + { + schedule vulnerable_versions_update_interval { grab_vulnerable_versions(1) }; + return; + } + + local sw = parts[1]; + local vvr = decode_vulnerable_version_range(parts[2]); + if ( sw !in internal_vulnerable_versions ) + internal_vulnerable_versions[sw] = set(); + add internal_vulnerable_versions[sw][vvr]; + + event grab_vulnerable_versions(i+1); + } + timeout 5secs + { + # In case a lookup fails, try starting over in one minute. + schedule 1min { grab_vulnerable_versions(1) }; + } + } + +event bro_init() + { + event grab_vulnerable_versions(1); + } + event log_software(rec: Info) { - if ( rec$name in vulnerable_versions && - cmp_versions(rec$version, vulnerable_versions[rec$name]) <= 0 ) + if ( rec$name !in internal_vulnerable_versions ) + return; + + for ( version_range in internal_vulnerable_versions[rec$name] ) { - NOTICE([$note=Vulnerable_Version, $src=rec$host, - $msg=fmt("A vulnerable version of software was detected: %s", software_fmt(rec))]); + if ( cmp_versions(rec$version, version_range$max) <= 0 && + (!version_range?$min || cmp_versions(rec$version, version_range$min) >= 0) ) + { + # The software is inside a vulnerable version range. + NOTICE([$note=Vulnerable_Version, $src=rec$host, + $msg=fmt("%s is running %s which is vulnerable.", rec$host, software_fmt(rec)), + $sub=software_fmt(rec)]); + } } } diff --git a/scripts/policy/protocols/http/detect-MHR.bro b/scripts/policy/protocols/http/detect-MHR.bro index 1898022978..0594276c93 100644 --- a/scripts/policy/protocols/http/detect-MHR.bro +++ b/scripts/policy/protocols/http/detect-MHR.bro @@ -32,7 +32,7 @@ event log_http(rec: HTTP::Info) { # Data is returned as " " local MHR_answer = split1(MHR_result, / /); - if ( length(MHR_answer) == 2 && to_count(MHR_answer[2]) >= MHR_threshold ) + if ( |MHR_answer| == 2 && to_count(MHR_answer[2]) >= MHR_threshold ) { local url = HTTP::build_url_http(rec); local message = fmt("%s %s %s", rec$id$orig_h, rec$md5, url); diff --git a/scripts/policy/protocols/ssl/extract-certs-pem.bro b/scripts/policy/protocols/ssl/extract-certs-pem.bro index 420c60a4fd..0a412a119d 100644 --- a/scripts/policy/protocols/ssl/extract-certs-pem.bro +++ b/scripts/policy/protocols/ssl/extract-certs-pem.bro @@ -4,14 +4,10 @@ ##! ##! ..note:: ##! -##! - It doesn't work well on a cluster because each worker will write its +##! - It doesn't work well on a cluster because each worker will write its ##! own certificate files and no duplicate checking is done across ##! clusters so each node would log each certificate. ##! -##! - If there is a certificate input based vulnerability found in the -##! openssl command line utility, you could be in trouble because this -##! script uses that utility to convert from DER to PEM certificates. -##! @load base/protocols/ssl @load base/utils/directions-and-hosts @@ -20,7 +16,7 @@ module SSL; export { - ## Control if host certificates offered by the defined hosts + ## Control if host certificates offered by the defined hosts ## will be written to the PEM certificates file. ## Choices are: LOCAL_HOSTS, REMOTE_HOSTS, ALL_HOSTS, NO_HOSTS const extract_certs_pem = LOCAL_HOSTS &redef; @@ -35,15 +31,33 @@ event ssl_established(c: connection) &priority=5 { if ( ! c$ssl?$cert ) return; + if ( ! addr_matches_host(c$id$resp_h, extract_certs_pem) ) return; - + if ( c$ssl$cert_hash in extracted_certs ) # If we already extracted this cert, don't do it again. return; - + add extracted_certs[c$ssl$cert_hash]; - local side = Site::is_local_addr(c$id$resp_h) ? "local" : "remote"; - local cmd = fmt("%s x509 -inform DER -outform PEM >> certs-%s.pem", openssl_util, side); - piped_exec(cmd, c$ssl$cert); + local filename = Site::is_local_addr(c$id$resp_h) ? "certs-local.pem" : "certs-remote.pem"; + local outfile = open_for_append(filename); + + print outfile, "-----BEGIN CERTIFICATE-----"; + + # Encode to base64 and format to fit 50 lines. Otherwise openssl won't like it later. + local lines = split_all(encode_base64(c$ssl$cert), /.{50}/); + local i = 1; + for ( line in lines ) + { + if ( |lines[i]| > 0 ) + { + print outfile, lines[i]; + } + i+=1; + } + + print outfile, "-----END CERTIFICATE-----"; + print outfile, ""; + close(outfile); } diff --git a/scripts/site/local.bro b/scripts/site/local.bro index a080300185..6dff7881f5 100644 --- a/scripts/site/local.bro +++ b/scripts/site/local.bro @@ -14,13 +14,6 @@ # information. @load frameworks/software/vulnerable -# Example vulnerable software. This needs to be updated and maintained over -# time as new vulnerabilities are discovered. -redef Software::vulnerable_versions += { - ["Flash"] = [$major=10,$minor=2,$minor2=153,$addl="1"], - ["Java"] = [$major=1,$minor=6,$minor2=0,$addl="22"], -}; - # Detect software changing (e.g. attacker installing hacked SSHD). @load frameworks/software/version-changes diff --git a/src/Analyzer.cc b/src/Analyzer.cc index 15926dfa2a..8bb05429c1 100644 --- a/src/Analyzer.cc +++ b/src/Analyzer.cc @@ -150,6 +150,10 @@ const Analyzer::Config Analyzer::analyzer_configs[] = { { AnalyzerTag::File, "FILE", File_Analyzer::InstantiateAnalyzer, File_Analyzer::Available, 0, false }, + { AnalyzerTag::IRC_Data, "IRC_DATA", IRC_Data::InstantiateAnalyzer, + IRC_Data::Available, 0, false }, + { AnalyzerTag::FTP_Data, "FTP_DATA", FTP_Data::InstantiateAnalyzer, + FTP_Data::Available, 0, false }, { AnalyzerTag::Backdoor, "BACKDOOR", BackDoor_Analyzer::InstantiateAnalyzer, BackDoor_Analyzer::Available, 0, false }, diff --git a/src/AnalyzerTags.h b/src/AnalyzerTags.h index 38e47cf8fc..8429dec335 100644 --- a/src/AnalyzerTags.h +++ b/src/AnalyzerTags.h @@ -41,7 +41,7 @@ namespace AnalyzerTag { GTPv1, // Other - File, Backdoor, InterConn, SteppingStone, TCPStats, + File, IRC_Data, FTP_Data, Backdoor, InterConn, SteppingStone, TCPStats, ConnSize, // Support-analyzers diff --git a/src/Base64.cc b/src/Base64.cc index b0da8ea74c..d3abd9b563 100644 --- a/src/Base64.cc +++ b/src/Base64.cc @@ -1,10 +1,48 @@ #include "config.h" #include "Base64.h" +#include -int Base64Decoder::default_base64_table[256]; -const string Base64Decoder::default_alphabet = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; +int Base64Converter::default_base64_table[256]; +const string Base64Converter::default_alphabet = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; -int* Base64Decoder::InitBase64Table(const string& alphabet) +void Base64Converter::Encode(int len, const unsigned char* data, int* pblen, char** pbuf) + { + int blen; + char *buf; + + if ( ! pbuf ) + reporter->InternalError("nil pointer to encoding result buffer"); + + if ( *pbuf && (*pblen % 4 != 0) ) + reporter->InternalError("Base64 encode buffer not a multiple of 4"); + + if ( *pbuf ) + { + buf = *pbuf; + blen = *pblen; + } + else + { + blen = (int)(4 * ceil((double)len / 3)); + *pbuf = buf = new char[blen]; + *pblen = blen; + } + + for ( int i = 0, j = 0; (i < len) && ( j < blen ); ) + { + uint32_t bit32 = data[i++] << 16; + bit32 += (i++ < len ? data[i-1] : 0) << 8; + bit32 += i++ < len ? data[i-1] : 0; + + buf[j++] = alphabet[(bit32 >> 18) & 0x3f]; + buf[j++] = alphabet[(bit32 >> 12) & 0x3f]; + buf[j++] = (i == (len+2)) ? '=' : alphabet[(bit32 >> 6) & 0x3f]; + buf[j++] = (i >= (len+1)) ? '=' : alphabet[bit32 & 0x3f]; + } + } + + +int* Base64Converter::InitBase64Table(const string& alphabet) { assert(alphabet.size() == 64); @@ -44,26 +82,42 @@ int* Base64Decoder::InitBase64Table(const string& alphabet) return base64_table; } -Base64Decoder::Base64Decoder(Analyzer* arg_analyzer, const string& alphabet) + + +Base64Converter::Base64Converter(Analyzer* arg_analyzer, const string& arg_alphabet) { - base64_table = InitBase64Table(alphabet.size() ? alphabet : default_alphabet); + if ( arg_alphabet.size() > 0 ) + { + assert(arg_alphabet.size() == 64); + alphabet = arg_alphabet; + } + else + { + alphabet = default_alphabet; + } + + base64_table = 0; base64_group_next = 0; base64_padding = base64_after_padding = 0; errored = 0; analyzer = arg_analyzer; } -Base64Decoder::~Base64Decoder() +Base64Converter::~Base64Converter() { if ( base64_table != default_base64_table ) delete base64_table; } -int Base64Decoder::Decode(int len, const char* data, int* pblen, char** pbuf) +int Base64Converter::Decode(int len, const char* data, int* pblen, char** pbuf) { int blen; char* buf; + // Initialization of table on first_time call of Decode. + if ( ! base64_table ) + base64_table = InitBase64Table(alphabet); + if ( ! pbuf ) reporter->InternalError("nil pointer to decoding result buffer"); @@ -145,7 +199,7 @@ int Base64Decoder::Decode(int len, const char* data, int* pblen, char** pbuf) return dlen; } -int Base64Decoder::Done(int* pblen, char** pbuf) +int Base64Converter::Done(int* pblen, char** pbuf) { const char* padding = "==="; @@ -177,7 +231,7 @@ BroString* decode_base64(const BroString* s, const BroString* a) int rlen2, rlen = buf_len; char* rbuf2, *rbuf = new char[rlen]; - Base64Decoder dec(0, a ? a->CheckString() : ""); + Base64Converter dec(0, a ? a->CheckString() : ""); if ( dec.Decode(s->Len(), (const char*) s->Bytes(), &rlen, &rbuf) == -1 ) goto err; @@ -195,3 +249,21 @@ err: delete [] rbuf; return 0; } + +BroString* encode_base64(const BroString* s, const BroString* a) + { + if ( a && a->Len() != 64 ) + { + reporter->Error("base64 alphabet is not 64 characters: %s", + a->CheckString()); + return 0; + } + + char* outbuf = 0; + int outlen = 0; + Base64Converter enc(0, a ? a->CheckString() : ""); + enc.Encode(s->Len(), (const unsigned char*) s->Bytes(), &outlen, &outbuf); + + return new BroString(1, (u_char*)outbuf, outlen); + } + diff --git a/src/Base64.h b/src/Base64.h index 0e02c94cf0..8cfeffde74 100644 --- a/src/Base64.h +++ b/src/Base64.h @@ -10,14 +10,13 @@ #include "Analyzer.h" // Maybe we should have a base class for generic decoders? - -class Base64Decoder { +class Base64Converter { public: // is used for error reporting, and it should be zero when - // the decoder is called by the built-in function decode_base64(). + // the decoder is called by the built-in function decode_base64() or encode_base64(). // Empty alphabet indicates the default base64 alphabet. - Base64Decoder(Analyzer* analyzer, const string& alphabet = ""); - ~Base64Decoder(); + Base64Converter(Analyzer* analyzer, const string& alphabet = ""); + ~Base64Converter(); // A note on Decode(): // @@ -30,6 +29,7 @@ public: // is not enough output buffer space. int Decode(int len, const char* data, int* blen, char** buf); + void Encode(int len, const unsigned char* data, int* blen, char** buf); int Done(int* pblen, char** pbuf); int HasData() const { return base64_group_next != 0; } @@ -39,7 +39,7 @@ public: const char* ErrorMsg() const { return error_msg; } void IllegalEncoding(const char* msg) - { + { // strncpy(error_msg, msg, sizeof(error_msg)); if ( analyzer ) analyzer->Weird("base64_illegal_encoding", msg); @@ -51,19 +51,22 @@ protected: char error_msg[256]; protected: + static const string default_alphabet; + string alphabet; + + static int* InitBase64Table(const string& alphabet); + static int default_base64_table[256]; char base64_group[4]; int base64_group_next; int base64_padding; int base64_after_padding; + int* base64_table; int errored; // if true, we encountered an error - skip further processing Analyzer* analyzer; - int* base64_table; - static int* InitBase64Table(const string& alphabet); - static int default_base64_table[256]; - static const string default_alphabet; }; BroString* decode_base64(const BroString* s, const BroString* a = 0); +BroString* encode_base64(const BroString* s, const BroString* a = 0); #endif /* base64_h */ diff --git a/src/BroString.cc b/src/BroString.cc index e05995b156..19f7a16566 100644 --- a/src/BroString.cc +++ b/src/BroString.cc @@ -369,7 +369,7 @@ VectorVal* BroString:: VecToPolicy(Vec* vec) BroString* string = (*vec)[i]; StringVal* val = new StringVal(string->Len(), (const char*) string->Bytes()); - result->Assign(i+1, val, 0); + result->Assign(i+1, val); } return result; diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 8652990461..415e418dbc 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -451,14 +451,18 @@ set(bro_SRCS input/readers/Ascii.cc input/readers/Raw.cc input/readers/Benchmark.cc + input/readers/Binary.cc file_analysis/Manager.cc file_analysis/Info.cc file_analysis/InfoTimer.cc + file_analysis/PendingFile.cc file_analysis/FileID.h file_analysis/Action.h + file_analysis/ActionSet.cc file_analysis/Extract.cc file_analysis/Hash.cc + file_analysis/DataEvent.cc file_analysis/analyzers/PE.cc nb_dns.c diff --git a/src/CompHash.cc b/src/CompHash.cc index 306b3ea83e..05d3e515d2 100644 --- a/src/CompHash.cc +++ b/src/CompHash.cc @@ -856,7 +856,7 @@ const char* CompositeHash::RecoverOneVal(const HashKey* k, const char* kp0, if ( have_val ) kp1 = RecoverOneVal(k, kp1, k_end, vt->YieldType(), value, false); - vv->Assign(index, value, 0); + vv->Assign(index, value); } pval = vv; diff --git a/src/Debug.cc b/src/Debug.cc index 535e193685..8cf2e38596 100644 --- a/src/Debug.cc +++ b/src/Debug.cc @@ -763,7 +763,7 @@ int dbg_handle_debug_input() Frame* curr_frame = g_frame_stack.back(); const BroFunc* func = curr_frame->GetFunction(); if ( func ) - current_module = func->GetID()->ModuleName(); + current_module = extract_module_name(func->Name()); else current_module = GLOBAL_MODULE_NAME; diff --git a/src/Event.cc b/src/Event.cc index 97f29000d6..9250b61e76 100644 --- a/src/Event.cc +++ b/src/Event.cc @@ -6,6 +6,7 @@ #include "Func.h" #include "NetVar.h" #include "Trigger.h" +#include "file_analysis/Manager.h" EventMgr mgr; @@ -124,6 +125,8 @@ void EventMgr::Drain() // processing, we ensure that it's done at a regular basis by checking // them here. Trigger::EvaluatePending(); + + file_mgr->EventDrainDone(); } void EventMgr::Describe(ODesc* d) const diff --git a/src/Expr.cc b/src/Expr.cc index 3df4d781a0..dd514df74b 100644 --- a/src/Expr.cc +++ b/src/Expr.cc @@ -485,7 +485,7 @@ Val* UnaryExpr::Eval(Frame* f) const for ( unsigned int i = 0; i < v_op->Size(); ++i ) { Val* v_i = v_op->Lookup(i); - result->Assign(i, v_i ? Fold(v_i) : 0, this); + result->Assign(i, v_i ? Fold(v_i) : 0); } Unref(v); @@ -625,10 +625,9 @@ Val* BinaryExpr::Eval(Frame* f) const if ( v_op1->Lookup(i) && v_op2->Lookup(i) ) v_result->Assign(i, Fold(v_op1->Lookup(i), - v_op2->Lookup(i)), - this); + v_op2->Lookup(i))); else - v_result->Assign(i, 0, this); + v_result->Assign(i, 0); // SetError("undefined element in vector operation"); } @@ -648,10 +647,9 @@ Val* BinaryExpr::Eval(Frame* f) const if ( vv_i ) v_result->Assign(i, is_vec1 ? - Fold(vv_i, v2) : Fold(v1, vv_i), - this); + Fold(vv_i, v2) : Fold(v1, vv_i)); else - v_result->Assign(i, 0, this); + v_result->Assign(i, 0); // SetError("Undefined element in vector operation"); } @@ -1049,10 +1047,10 @@ Val* IncrExpr::Eval(Frame* f) const if ( elt ) { Val* new_elt = DoSingleEval(f, elt); - v_vec->Assign(i, new_elt, this, OP_INCR); + v_vec->Assign(i, new_elt, OP_INCR); } else - v_vec->Assign(i, 0, this, OP_INCR); + v_vec->Assign(i, 0, OP_INCR); } op->Assign(f, v_vec, OP_INCR); } @@ -1919,7 +1917,7 @@ Val* BoolExpr::Eval(Frame* f) const result = new VectorVal(Type()->AsVectorType()); result->Resize(vector_v->Size()); result->AssignRepeat(0, result->Size(), - scalar_v, this); + scalar_v); } else result = vector_v->Ref()->AsVectorVal(); @@ -1957,10 +1955,10 @@ Val* BoolExpr::Eval(Frame* f) const (! op1->IsZero() && ! op2->IsZero()) : (! op1->IsZero() || ! op2->IsZero()); - result->Assign(i, new Val(local_result, TYPE_BOOL), this); + result->Assign(i, new Val(local_result, TYPE_BOOL)); } else - result->Assign(i, 0, this); + result->Assign(i, 0); } Unref(v1); @@ -2334,10 +2332,9 @@ Val* CondExpr::Eval(Frame* f) const if ( local_cond ) result->Assign(i, local_cond->IsZero() ? - b->Lookup(i) : a->Lookup(i), - this); + b->Lookup(i) : a->Lookup(i)); else - result->Assign(i, 0, this); + result->Assign(i, 0); } return result; @@ -2507,15 +2504,27 @@ bool AssignExpr::TypeCheck(attr_list* attrs) attr_copy->append((*attrs)[i]); } - op2 = new TableConstructorExpr(op2->AsListExpr(), attr_copy); + if ( op1->Type()->IsSet() ) + op2 = new SetConstructorExpr(op2->AsListExpr(), attr_copy); + else + op2 = new TableConstructorExpr(op2->AsListExpr(), attr_copy); + return true; } - if ( bt1 == TYPE_VECTOR && bt2 == bt1 && - op2->Type()->AsVectorType()->IsUnspecifiedVector() ) + if ( bt1 == TYPE_VECTOR ) { - op2 = new VectorCoerceExpr(op2, op1->Type()->AsVectorType()); - return true; + if ( bt2 == bt1 && op2->Type()->AsVectorType()->IsUnspecifiedVector() ) + { + op2 = new VectorCoerceExpr(op2, op1->Type()->AsVectorType()); + return true; + } + + if ( op2->Tag() == EXPR_LIST ) + { + op2 = new VectorConstructorExpr(op2->AsListExpr()); + return true; + } } if ( op1->Type()->Tag() == TYPE_RECORD && @@ -2961,7 +2970,7 @@ Val* IndexExpr::Eval(Frame* f) const for ( unsigned int i = 0; i < v_v2->Size(); ++i ) { if ( v_v2->Lookup(i)->AsBool() ) - v_result->Assign(v_result->Size() + 1, v_v1->Lookup(i), this); + v_result->Assign(v_result->Size() + 1, v_v1->Lookup(i)); } } else @@ -2971,7 +2980,7 @@ Val* IndexExpr::Eval(Frame* f) const // Probably only do this if *all* are negative. v_result->Resize(v_v2->Size()); for ( unsigned int i = 0; i < v_v2->Size(); ++i ) - v_result->Assign(i, v_v1->Lookup(v_v2->Lookup(i)->CoerceToInt()), this); + v_result->Assign(i, v_v1->Lookup(v_v2->Lookup(i)->CoerceToInt())); } } else @@ -3048,7 +3057,7 @@ void IndexExpr::Assign(Frame* f, Val* v, Opcode op) switch ( v1->Type()->Tag() ) { case TYPE_VECTOR: - if ( ! v1->AsVectorVal()->Assign(v2, v, this, op) ) + if ( ! v1->AsVectorVal()->Assign(v2, v, op) ) Internal("assignment failed"); break; @@ -3620,7 +3629,7 @@ Val* VectorConstructorExpr::Eval(Frame* f) const { Expr* e = exprs[i]; Val* v = e->Eval(f); - if ( ! vec->Assign(i, v, e) ) + if ( ! vec->Assign(i, v) ) { Error(fmt("type mismatch at index %d", i), e); return 0; @@ -3644,7 +3653,7 @@ Val* VectorConstructorExpr::InitVal(const BroType* t, Val* aggr) const Expr* e = exprs[i]; Val* v = check_and_promote(e->Eval(0), t->YieldType(), 1); - if ( ! v || ! vec->Assign(i, v, e) ) + if ( ! v || ! vec->Assign(i, v) ) { Error(fmt("initialization type mismatch at index %d", i), e); return 0; @@ -3865,9 +3874,9 @@ Val* ArithCoerceExpr::Fold(Val* v) const { Val* elt = vv->Lookup(i); if ( elt ) - result->Assign(i, FoldSingleVal(elt, t), this); + result->Assign(i, FoldSingleVal(elt, t)); else - result->Assign(i, 0, this); + result->Assign(i, 0); } return result; @@ -4639,12 +4648,16 @@ Val* CallExpr::Eval(Frame* f) const { const ::Func* func = func_val->AsFunc(); calling_expr = this; + const CallExpr* current_call = f ? f->GetCall() : 0; if ( f ) f->SetCall(this); + ret = func->Call(v, f); // No try/catch here; we pass exceptions upstream. + if ( f ) - f->ClearCall(); + f->SetCall(current_call); + // Don't Unref() the arguments, as Func::Call already did that. delete v; @@ -4971,14 +4984,22 @@ Val* ListExpr::InitVal(const BroType* t, Val* aggr) const { ListVal* v = new ListVal(TYPE_ANY); + const type_list* tl = type->AsTypeList()->Types(); + if ( exprs.length() != tl->length() ) + { + Error("index mismatch", t); + return 0; + } + loop_over_list(exprs, i) { - Val* vi = exprs[i]->InitVal(t, 0); + Val* vi = exprs[i]->InitVal((*tl)[i], 0); if ( ! vi ) { Unref(v); return 0; } + v->Append(vi); } return v; @@ -5042,7 +5063,7 @@ Val* ListExpr::InitVal(const BroType* t, Val* aggr) const Expr* e = exprs[i]; check_and_promote_expr(e, vec->Type()->AsVectorType()->YieldType()); Val* v = e->Eval(0); - if ( ! vec->Assign(i, v, e) ) + if ( ! vec->Assign(i, v) ) { e->Error(fmt("type mismatch at index %d", i)); return 0; diff --git a/src/FileAnalyzer.cc b/src/FileAnalyzer.cc index 27592ea34c..508ae23335 100644 --- a/src/FileAnalyzer.cc +++ b/src/FileAnalyzer.cc @@ -3,34 +3,24 @@ #include "file_analysis/Manager.h" #include "FileAnalyzer.h" #include "Reporter.h" +#include "util.h" magic_t File_Analyzer::magic = 0; magic_t File_Analyzer::magic_mime = 0; -File_Analyzer::File_Analyzer(Connection* conn) -: TCP_ApplicationAnalyzer(AnalyzerTag::File, conn) +File_Analyzer::File_Analyzer(AnalyzerTag::Tag tag, Connection* conn) +: TCP_ApplicationAnalyzer(tag, conn) { buffer_len = 0; - if ( ! magic ) - { - InitMagic(&magic, MAGIC_NONE); - InitMagic(&magic_mime, MAGIC_MIME); - } - - char op[256], rp[256]; - modp_ulitoa10(ntohs(conn->OrigPort()), op); - modp_ulitoa10(ntohs(conn->RespPort()), rp); - unique_file = "TCPFile " + conn->OrigAddr().AsString() + ":" + op + "->" + - conn->RespAddr().AsString() + ":" + rp; + bro_init_magic(&magic, MAGIC_NONE); + bro_init_magic(&magic_mime, MAGIC_MIME); } void File_Analyzer::DeliverStream(int len, const u_char* data, bool orig) { TCP_ApplicationAnalyzer::DeliverStream(len, data, orig); - file_mgr->DataIn(unique_file, data, len, Conn()); - int n = min(len, BUFFER_SIZE - buffer_len); if ( n ) @@ -47,16 +37,12 @@ void File_Analyzer::DeliverStream(int len, const u_char* data, bool orig) void File_Analyzer::Undelivered(int seq, int len, bool orig) { TCP_ApplicationAnalyzer::Undelivered(seq, len, orig); - - file_mgr->Gap(unique_file, seq, len); } void File_Analyzer::Done() { TCP_ApplicationAnalyzer::Done(); - file_mgr->EndOfFile(unique_file, Conn()); - if ( buffer_len && buffer_len != BUFFER_SIZE ) Identify(); } @@ -67,10 +53,10 @@ void File_Analyzer::Identify() const char* mime = 0; if ( magic ) - descr = magic_buffer(magic, buffer, buffer_len); + descr = bro_magic_buffer(magic, buffer, buffer_len); if ( magic_mime ) - mime = magic_buffer(magic_mime, buffer, buffer_len); + mime = bro_magic_buffer(magic_mime, buffer, buffer_len); val_list* vl = new val_list; vl->append(BuildConnVal()); @@ -80,17 +66,48 @@ void File_Analyzer::Identify() ConnectionEvent(file_transferred, vl); } -void File_Analyzer::InitMagic(magic_t* magic, int flags) +IRC_Data::IRC_Data(Connection* conn) + : File_Analyzer(AnalyzerTag::IRC_Data, conn) { - *magic = magic_open(flags); - - if ( ! *magic ) - reporter->Error("can't init libmagic: %s", magic_error(*magic)); - - else if ( magic_load(*magic, 0) < 0 ) - { - reporter->Error("can't load magic file: %s", magic_error(*magic)); - magic_close(*magic); - *magic = 0; - } + } + +void IRC_Data::Done() + { + File_Analyzer::Done(); + file_mgr->EndOfFile(GetTag(), Conn()); + } + +void IRC_Data::DeliverStream(int len, const u_char* data, bool orig) + { + File_Analyzer::DeliverStream(len, data, orig); + file_mgr->DataIn(data, len, GetTag(), Conn(), orig); + } + +void IRC_Data::Undelivered(int seq, int len, bool orig) + { + File_Analyzer::Undelivered(seq, len, orig); + file_mgr->Gap(seq, len, GetTag(), Conn(), orig); + } + +FTP_Data::FTP_Data(Connection* conn) + : File_Analyzer(AnalyzerTag::FTP_Data, conn) + { + } + +void FTP_Data::Done() + { + File_Analyzer::Done(); + file_mgr->EndOfFile(GetTag(), Conn()); + } + +void FTP_Data::DeliverStream(int len, const u_char* data, bool orig) + { + File_Analyzer::DeliverStream(len, data, orig); + file_mgr->DataIn(data, len, GetTag(), Conn(), orig); + } + +void FTP_Data::Undelivered(int seq, int len, bool orig) + { + File_Analyzer::Undelivered(seq, len, orig); + file_mgr->Gap(seq, len, GetTag(), Conn(), orig); } diff --git a/src/FileAnalyzer.h b/src/FileAnalyzer.h index 9b737856fc..c4bd084cdc 100644 --- a/src/FileAnalyzer.h +++ b/src/FileAnalyzer.h @@ -10,7 +10,7 @@ class File_Analyzer : public TCP_ApplicationAnalyzer { public: - File_Analyzer(Connection* conn); + File_Analyzer(AnalyzerTag::Tag tag, Connection* conn); virtual void Done(); @@ -19,7 +19,7 @@ public: void Undelivered(int seq, int len, bool orig); static Analyzer* InstantiateAnalyzer(Connection* conn) - { return new File_Analyzer(conn); } + { return new File_Analyzer(AnalyzerTag::File, conn); } static bool Available() { return file_transferred; } @@ -32,12 +32,42 @@ protected: char buffer[BUFFER_SIZE]; int buffer_len; - static void InitMagic(magic_t* magic, int flags); - static magic_t magic; static magic_t magic_mime; +}; - string unique_file; +class IRC_Data : public File_Analyzer { +public: + + IRC_Data(Connection* conn); + + virtual void Done(); + + virtual void DeliverStream(int len, const u_char* data, bool orig); + + void Undelivered(int seq, int len, bool orig); + + static Analyzer* InstantiateAnalyzer(Connection* conn) + { return new IRC_Data(conn); } + + static bool Available() { return true; } +}; + +class FTP_Data : public File_Analyzer { +public: + + FTP_Data(Connection* conn); + + virtual void Done(); + + virtual void DeliverStream(int len, const u_char* data, bool orig); + + void Undelivered(int seq, int len, bool orig); + + static Analyzer* InstantiateAnalyzer(Connection* conn) + { return new FTP_Data(conn); } + + static bool Available() { return true; } }; #endif diff --git a/src/Frag.cc b/src/Frag.cc index d873f5bc0c..4b9047d072 100644 --- a/src/Frag.cc +++ b/src/Frag.cc @@ -100,6 +100,13 @@ void FragReassembler::AddFragment(double t, const IP_Hdr* ip, const u_char* pkt) int offset = ip->FragOffset(); int len = ip->TotalLen(); int hdr_len = ip->HdrLen(); + + if ( len < hdr_len ) + { + s->Weird("fragment_protocol_inconsistency", ip); + return; + } + int upper_seq = offset + len - hdr_len; if ( ! offset ) diff --git a/src/Frame.cc b/src/Frame.cc index f86fa32805..8754c02a9f 100644 --- a/src/Frame.cc +++ b/src/Frame.cc @@ -87,8 +87,11 @@ Frame* Frame::Clone() void Frame::SetTrigger(Trigger* arg_trigger) { + ClearTrigger(); + if ( arg_trigger ) Ref(arg_trigger); + trigger = arg_trigger; } diff --git a/src/Func.cc b/src/Func.cc index a0d303dcf6..30db27fc71 100644 --- a/src/Func.cc +++ b/src/Func.cc @@ -54,13 +54,13 @@ bool did_builtin_init = false; vector Func::unique_ids; -Func::Func() : scope(0), id(0), return_value(0) +Func::Func() : scope(0), type(0) { unique_id = unique_ids.size(); unique_ids.push_back(this); } -Func::Func(Kind arg_kind) : scope(0), kind(arg_kind), id(0), return_value(0) +Func::Func(Kind arg_kind) : scope(0), kind(arg_kind), type(0) { unique_id = unique_ids.size(); unique_ids.push_back(this); @@ -68,6 +68,7 @@ Func::Func(Kind arg_kind) : scope(0), kind(arg_kind), id(0), return_value(0) Func::~Func() { + Unref(type); } void Func::AddBody(Stmt* /* new_body */, id_list* /* new_inits */, @@ -129,6 +130,12 @@ bool Func::DoSerialize(SerialInfo* info) const if ( ! SERIALIZE(char(kind) ) ) return false; + if ( ! type->Serialize(info) ) + return false; + + if ( ! SERIALIZE(Name()) ) + return false; + // We don't serialize scope as only global functions are considered here // anyway. return true; @@ -160,12 +167,25 @@ bool Func::DoUnserialize(UnserialInfo* info) return false; kind = (Kind) c; + + type = BroType::Unserialize(info); + if ( ! type ) + return false; + + const char* n; + if ( ! UNSERIALIZE_STR(&n, 0) ) + return false; + + name = n; + delete [] n; + return true; } void Func::DescribeDebug(ODesc* d, const val_list* args) const { - id->Describe(d); + d->Add(Name()); + RecordType* func_args = FType()->Args(); if ( args ) @@ -196,21 +216,6 @@ void Func::DescribeDebug(ODesc* d, const val_list* args) const } } -void Func::SetID(ID *arg_id) - { - id = arg_id; - - return_value = - new ID(string(string(id->Name()) + "_returnvalue").c_str(), - SCOPE_FUNCTION, false); - return_value->SetType(FType()->YieldType()->Ref()); - } - -ID* Func::GetReturnValueID() const - { - return return_value; - } - TraversalCode Func::Traverse(TraversalCallback* cb) const { // FIXME: Make a fake scope for builtins? @@ -226,12 +231,6 @@ TraversalCode Func::Traverse(TraversalCallback* cb) const tc = scope->Traverse(cb); HANDLE_TC_STMT_PRE(tc); - if ( GetReturnValueID() ) - { - tc = GetReturnValueID()->Traverse(cb); - HANDLE_TC_STMT_PRE(tc); - } - for ( unsigned int i = 0; i < bodies.size(); ++i ) { tc = bodies[i].stmts->Traverse(cb); @@ -249,7 +248,8 @@ BroFunc::BroFunc(ID* arg_id, Stmt* arg_body, id_list* aggr_inits, int arg_frame_size, int priority) : Func(BRO_FUNC) { - id = arg_id; + name = arg_id->Name(); + type = arg_id->Type()->Ref(); frame_size = arg_frame_size; if ( arg_body ) @@ -263,7 +263,6 @@ BroFunc::BroFunc(ID* arg_id, Stmt* arg_body, id_list* aggr_inits, BroFunc::~BroFunc() { - Unref(id); for ( unsigned int i = 0; i < bodies.size(); ++i ) Unref(bodies[i].stmts); } @@ -378,7 +377,8 @@ Val* BroFunc::Call(val_list* args, Frame* parent) const (flow != FLOW_RETURN /* we fell off the end */ || ! result /* explicit return with no result */) && ! f->HasDelayed() ) - reporter->Warning("non-void function returns without a value: %s", id->Name()); + reporter->Warning("non-void function returns without a value: %s", + Name()); if ( result && g_trace_state.DoTrace() ) { @@ -421,8 +421,7 @@ void BroFunc::AddBody(Stmt* new_body, id_list* new_inits, int new_frame_size, void BroFunc::Describe(ODesc* d) const { - if ( id ) - id->Describe(d); + d->Add(Name()); d->NL(); d->AddCount(frame_size); @@ -450,14 +449,14 @@ IMPLEMENT_SERIAL(BroFunc, SER_BRO_FUNC); bool BroFunc::DoSerialize(SerialInfo* info) const { DO_SERIALIZE(SER_BRO_FUNC, Func); - return id->Serialize(info) && SERIALIZE(frame_size); + return SERIALIZE(frame_size); } bool BroFunc::DoUnserialize(UnserialInfo* info) { DO_UNSERIALIZE(Func); - id = ID::Unserialize(info); - return id && UNSERIALIZE(&frame_size); + + return UNSERIALIZE(&frame_size); } BuiltinFunc::BuiltinFunc(built_in_func arg_func, const char* arg_name, @@ -465,15 +464,16 @@ BuiltinFunc::BuiltinFunc(built_in_func arg_func, const char* arg_name, : Func(BUILTIN_FUNC) { func = arg_func; - name = copy_string(make_full_var_name(GLOBAL_MODULE_NAME, arg_name).c_str()); + name = make_full_var_name(GLOBAL_MODULE_NAME, arg_name); is_pure = arg_is_pure; - id = lookup_ID(name, GLOBAL_MODULE_NAME, false); + ID* id = lookup_ID(Name(), GLOBAL_MODULE_NAME, false); if ( ! id ) - reporter->InternalError("built-in function %s missing", name); + reporter->InternalError("built-in function %s missing", Name()); if ( id->HasVal() ) - reporter->InternalError("built-in function %s multiply defined", name); + reporter->InternalError("built-in function %s multiply defined", Name()); + type = id->Type()->Ref(); id->SetVal(new Val(this)); } @@ -491,7 +491,7 @@ Val* BuiltinFunc::Call(val_list* args, Frame* parent) const #ifdef PROFILE_BRO_FUNCTIONS DEBUG_MSG("Function: %s\n", Name()); #endif - SegmentProfiler(segment_logger, name); + SegmentProfiler(segment_logger, Name()); if ( sample_logger ) sample_logger->FunctionSeen(this); @@ -522,8 +522,7 @@ Val* BuiltinFunc::Call(val_list* args, Frame* parent) const void BuiltinFunc::Describe(ODesc* d) const { - if ( id ) - id->Describe(d); + d->Add(Name()); d->AddCount(is_pure); } @@ -532,16 +531,13 @@ IMPLEMENT_SERIAL(BuiltinFunc, SER_BUILTIN_FUNC); bool BuiltinFunc::DoSerialize(SerialInfo* info) const { DO_SERIALIZE(SER_BUILTIN_FUNC, Func); - - // We ignore the ID. Func::Serialize() will rebind us anyway. - return SERIALIZE(name); + return true; } bool BuiltinFunc::DoUnserialize(UnserialInfo* info) { DO_UNSERIALIZE(Func); - id = 0; - return UNSERIALIZE_STR(&name, 0); + return true; } void builtin_error(const char* msg, BroObj* arg) diff --git a/src/Func.h b/src/Func.h index 7f9627b66d..3413e0bef1 100644 --- a/src/Func.h +++ b/src/Func.h @@ -47,15 +47,11 @@ public: virtual void SetScope(Scope* newscope) { scope = newscope; } virtual Scope* GetScope() const { return scope; } - virtual FuncType* FType() const - { - return (FuncType*) id->Type()->AsFuncType(); - } + virtual FuncType* FType() const { return type->AsFuncType(); } Kind GetKind() const { return kind; } - const ID* GetID() const { return id; } - void SetID(ID *arg_id); + const char* Name() const { return name.c_str(); } virtual void Describe(ODesc* d) const = 0; virtual void DescribeDebug(ODesc* d, const val_list* args) const; @@ -64,7 +60,6 @@ public: bool Serialize(SerialInfo* info) const; static Func* Unserialize(UnserialInfo* info); - ID* GetReturnValueID() const; virtual TraversalCode Traverse(TraversalCallback* cb) const; uint32 GetUniqueFuncID() const { return unique_id; } @@ -79,8 +74,8 @@ protected: vector bodies; Scope* scope; Kind kind; - ID* id; - ID* return_value; + BroType* type; + string name; uint32 unique_id; static vector unique_ids; }; @@ -119,18 +114,16 @@ public: int IsPure() const; Val* Call(val_list* args, Frame* parent) const; - const char* Name() const { return name; } built_in_func TheFunc() const { return func; } void Describe(ODesc* d) const; protected: - BuiltinFunc() { func = 0; name = 0; is_pure = 0; } + BuiltinFunc() { func = 0; is_pure = 0; } DECLARE_SERIAL(BuiltinFunc); built_in_func func; - const char* name; int is_pure; }; diff --git a/src/HTTP.cc b/src/HTTP.cc index 7e4079b853..3ae17714d6 100644 --- a/src/HTTP.cc +++ b/src/HTTP.cc @@ -12,6 +12,7 @@ #include "HTTP.h" #include "Event.h" #include "MIME.h" +#include "file_analysis/Manager.h" const bool DEBUG_http = false; @@ -41,9 +42,13 @@ HTTP_Entity::HTTP_Entity(HTTP_Message *arg_message, MIME_Entity* parent_entity, expect_data_length = 0; body_length = 0; header_length = 0; - deliver_body = (http_entity_data != 0); + deliver_body = true; encoding = IDENTITY; zip = 0; + is_partial_content = false; + offset = 0; + instance_length = -1; // unspecified + send_size = true; } void HTTP_Entity::EndOfData() @@ -233,6 +238,11 @@ int HTTP_Entity::Undelivered(int64_t len) if ( end_of_data && in_header ) return 0; + file_mgr->Gap(body_length, len, + http_message->MyHTTP_Analyzer()->GetTag(), + http_message->MyHTTP_Analyzer()->Conn(), + http_message->IsOrig()); + if ( chunked_transfer_state != NON_CHUNKED_TRANSFER ) { if ( chunked_transfer_state == EXPECT_CHUNK_DATA && @@ -277,6 +287,38 @@ void HTTP_Entity::SubmitData(int len, const char* buf) { if ( deliver_body ) MIME_Entity::SubmitData(len, buf); + + if ( send_size && ( encoding == GZIP || encoding == DEFLATE ) ) + // Auto-decompress in DeliverBody invalidates sizes derived from headers + send_size = false; + + if ( is_partial_content ) + { + if ( send_size && instance_length > 0 ) + file_mgr->SetSize(instance_length, + http_message->MyHTTP_Analyzer()->GetTag(), + http_message->MyHTTP_Analyzer()->Conn(), + http_message->IsOrig()); + file_mgr->DataIn(reinterpret_cast(buf), len, offset, + http_message->MyHTTP_Analyzer()->GetTag(), + http_message->MyHTTP_Analyzer()->Conn(), + http_message->IsOrig()); + offset += len; + } + else + { + if ( send_size && content_length > 0 ) + file_mgr->SetSize(content_length, + http_message->MyHTTP_Analyzer()->GetTag(), + http_message->MyHTTP_Analyzer()->Conn(), + http_message->IsOrig()); + file_mgr->DataIn(reinterpret_cast(buf), len, + http_message->MyHTTP_Analyzer()->GetTag(), + http_message->MyHTTP_Analyzer()->Conn(), + http_message->IsOrig()); + } + + send_size = false; } void HTTP_Entity::SetPlainDelivery(int64_t length) @@ -307,9 +349,7 @@ void HTTP_Entity::SubmitHeader(MIME_Header* h) } // Figure out content-length for HTTP 206 Partial Content response - // that uses multipart/byteranges content-type. - else if ( strcasecmp_n(h->get_name(), "content-range") == 0 && Parent() && - Parent()->MIMEContentType() == CONTENT_TYPE_MULTIPART && + else if ( strcasecmp_n(h->get_name(), "content-range") == 0 && http_message->MyHTTP_Analyzer()->HTTP_ReplyCode() == 206 ) { data_chunk_t vt = h->get_value_token(); @@ -333,7 +373,7 @@ void HTTP_Entity::SubmitHeader(MIME_Header* h) } string byte_range_resp_spec = byte_range.substr(0, p); - string instance_length = byte_range.substr(p + 1); + string instance_length_str = byte_range.substr(p + 1); p = byte_range_resp_spec.find("-"); if ( p == string::npos ) @@ -348,7 +388,7 @@ void HTTP_Entity::SubmitHeader(MIME_Header* h) if ( DEBUG_http ) DEBUG_MSG("Parsed Content-Range: %s %s-%s/%s\n", byte_unit.c_str(), first_byte_pos.c_str(), last_byte_pos.c_str(), - instance_length.c_str()); + instance_length_str.c_str()); int64_t f, l; atoi_n(first_byte_pos.size(), first_byte_pos.c_str(), 0, 10, f); @@ -359,7 +399,18 @@ void HTTP_Entity::SubmitHeader(MIME_Header* h) DEBUG_MSG("Content-Range length = %"PRId64"\n", len); if ( len > 0 ) + { + if ( instance_length_str != "*" ) + { + if ( ! atoi_n(instance_length_str.size(), + instance_length_str.c_str(), 0, 10, + instance_length) ) + instance_length = 0; + } + is_partial_content = true; + offset = f; content_length = len; + } else { http_message->Weird("HTTP_non_positive_content_range"); @@ -512,6 +563,11 @@ void HTTP_Message::Done(const int interrupted, const char* detail) // DEBUG_MSG("%.6f HTTP message done.\n", network_time); top_level->EndOfData(); + if ( is_orig || MyHTTP_Analyzer()->HTTP_ReplyCode() != 206 ) + // multipart/byteranges may span multiple connections + file_mgr->EndOfFile(MyHTTP_Analyzer()->GetTag(), + MyHTTP_Analyzer()->Conn(), is_orig); + if ( http_message_done ) { val_list* vl = new val_list; @@ -586,6 +642,9 @@ void HTTP_Message::EndEntity(MIME_Entity* entity) // SubmitAllHeaders (through EndOfData). if ( entity == top_level ) Done(); + else if ( is_orig || MyHTTP_Analyzer()->HTTP_ReplyCode() != 206 ) + file_mgr->EndOfFile(MyHTTP_Analyzer()->GetTag(), + MyHTTP_Analyzer()->Conn(), is_orig); } void HTTP_Message::SubmitHeader(MIME_Header* h) @@ -641,9 +700,6 @@ void HTTP_Message::SubmitData(int len, const char* buf) int HTTP_Message::RequestBuffer(int* plen, char** pbuf) { - if ( ! http_entity_data ) - return 0; - if ( ! data_buffer ) if ( ! InitBuffer(mime_segment_length) ) return 0; @@ -846,6 +902,13 @@ void HTTP_Analyzer::Done() Unref(unanswered_requests.front()); unanswered_requests.pop(); } + + file_mgr->EndOfFile(GetTag(), Conn(), true); + /* TODO: this might be nice to have, but reply code is cleared by now. + if ( HTTP_ReplyCode() != 206 ) + // multipart/byteranges may span multiple connections + file_mgr->EndOfFile(GetTag(), Conn(), false); + */ } void HTTP_Analyzer::DeliverStream(int len, const u_char* data, bool is_orig) diff --git a/src/HTTP.h b/src/HTTP.h index c9d8ae55d1..81c7385825 100644 --- a/src/HTTP.h +++ b/src/HTTP.h @@ -55,6 +55,10 @@ protected: int deliver_body; enum { IDENTITY, GZIP, COMPRESS, DEFLATE } encoding; ZIP_Analyzer* zip; + bool is_partial_content; + uint64_t offset; + int64_t instance_length; // total length indicated by content-range + bool send_size; // whether to send size indication to FAF MIME_Entity* NewChildEntity() { return new HTTP_Entity(http_message, this, 1); } diff --git a/src/ICMP.cc b/src/ICMP.cc index b9b4e89404..5634e3885f 100644 --- a/src/ICMP.cc +++ b/src/ICMP.cc @@ -829,7 +829,7 @@ VectorVal* ICMP_Analyzer::BuildNDOptionsVal(int caplen, const u_char* data) data += length; caplen -= length; - vv->Assign(vv->Size(), rv, 0); + vv->Assign(vv->Size(), rv); } return vv; diff --git a/src/IP.cc b/src/IP.cc index 16424e26f2..78311cc2d2 100644 --- a/src/IP.cc +++ b/src/IP.cc @@ -63,7 +63,7 @@ static VectorVal* BuildOptionsVal(const u_char* data, int len) len -= opt->ip6o_len + off; } - vv->Assign(vv->Size(), rv, 0); + vv->Assign(vv->Size(), rv); } return vv; @@ -626,7 +626,7 @@ VectorVal* IPv6_Hdr_Chain::BuildVal() const reporter->InternalError("IPv6_Hdr_Chain bad header %d", type); break; } - rval->Assign(rval->Size(), ext_hdr, 0); + rval->Assign(rval->Size(), ext_hdr); } return rval; diff --git a/src/MIME.cc b/src/MIME.cc index 4a7c0268b0..12729f1a07 100644 --- a/src/MIME.cc +++ b/src/MIME.cc @@ -5,6 +5,7 @@ #include "Event.h" #include "Reporter.h" #include "digest.h" +#include "file_analysis/Manager.h" // Here are a few things to do: // @@ -810,7 +811,7 @@ void MIME_Entity::StartDecodeBase64() if ( base64_decoder ) reporter->InternalError("previous Base64 decoder not released!"); - base64_decoder = new Base64Decoder(message->GetAnalyzer()); + base64_decoder = new Base64Converter(message->GetAnalyzer()); } void MIME_Entity::FinishDecodeBase64() @@ -1019,6 +1020,8 @@ void MIME_Mail::Done() } MIME_Message::Done(); + + file_mgr->EndOfFile(analyzer->GetTag(), analyzer->Conn()); } MIME_Mail::~MIME_Mail() @@ -1030,6 +1033,7 @@ MIME_Mail::~MIME_Mail() void MIME_Mail::BeginEntity(MIME_Entity* /* entity */) { + cur_entity_len = 0; if ( mime_begin_entity ) { val_list* vl = new val_list; @@ -1065,6 +1069,8 @@ void MIME_Mail::EndEntity(MIME_Entity* /* entity */) vl->append(analyzer->BuildConnVal()); analyzer->ConnectionEvent(mime_end_entity, vl); } + + file_mgr->EndOfFile(analyzer->GetTag(), analyzer->Conn()); } void MIME_Mail::SubmitHeader(MIME_Header* h) @@ -1122,6 +1128,11 @@ void MIME_Mail::SubmitData(int len, const char* buf) analyzer->ConnectionEvent(mime_segment_data, vl); } + // is_orig param not available, doesn't matter as long as it's consistent + file_mgr->DataIn(reinterpret_cast(buf), len, + analyzer->GetTag(), analyzer->Conn(), false); + cur_entity_len += len; + buffer_start = (buf + len) - (char*)data_buffer->Bytes(); } @@ -1193,6 +1204,12 @@ void MIME_Mail::SubmitEvent(int event_type, const char* detail) } } +void MIME_Mail::Undelivered(int len) + { + // is_orig param not available, doesn't matter as long as it's consistent + file_mgr->Gap(cur_entity_len, len, analyzer->GetTag(), analyzer->Conn(), + false); + } int strcasecmp_n(data_chunk_t s, const char* t) { diff --git a/src/MIME.h b/src/MIME.h index ffff30e387..eb30a43bd3 100644 --- a/src/MIME.h +++ b/src/MIME.h @@ -131,7 +131,7 @@ protected: int GetDataBuffer(); void DataOctet(char ch); void DataOctets(int len, const char* data); - void SubmitData(int len, const char* buf); + virtual void SubmitData(int len, const char* buf); virtual void SubmitHeader(MIME_Header* h); // Submit all headers in member "headers". @@ -163,7 +163,7 @@ protected: MIME_Entity* parent; MIME_Entity* current_child_entity; - Base64Decoder* base64_decoder; + Base64Converter* base64_decoder; int data_buf_length; char* data_buf_data; @@ -238,6 +238,7 @@ public: int RequestBuffer(int* plen, char** pbuf); void SubmitAllData(); void SubmitEvent(int event_type, const char* detail); + void Undelivered(int len); protected: int min_overlap_length; @@ -252,6 +253,8 @@ protected: vector all_content; BroString* data_buffer; + + uint64 cur_entity_len; }; diff --git a/src/NFS.cc b/src/NFS.cc index 2911ee7f59..d64f7755a3 100644 --- a/src/NFS.cc +++ b/src/NFS.cc @@ -599,7 +599,7 @@ RecordVal* NFS_Interp::nfs3_readdir_reply(bool isplus, const u_char*& buf, entry->Assign(4, nfs3_post_op_fh(buf,n)); } - entries->Assign(pos, entry, 0); + entries->Assign(pos, entry); pos++; } diff --git a/src/NetVar.cc b/src/NetVar.cc index 99a4023062..d433c5d781 100644 --- a/src/NetVar.cc +++ b/src/NetVar.cc @@ -5,7 +5,6 @@ #include "Var.h" #include "NetVar.h" -RecordType* gtpv1_hdr_type; RecordType* conn_id; RecordType* endpoint; RecordType* endpoint_stats; @@ -311,7 +310,6 @@ void init_net_var() #include "reporter.bif.netvar_init" #include "file_analysis.bif.netvar_init" - gtpv1_hdr_type = internal_type("gtpv1_hdr")->AsRecordType(); conn_id = internal_type("conn_id")->AsRecordType(); endpoint = internal_type("endpoint")->AsRecordType(); endpoint_stats = internal_type("endpoint_stats")->AsRecordType(); diff --git a/src/NetVar.h b/src/NetVar.h index 7537b0793f..b2da89df6b 100644 --- a/src/NetVar.h +++ b/src/NetVar.h @@ -8,7 +8,6 @@ #include "EventRegistry.h" #include "Stats.h" -extern RecordType* gtpv1_hdr_type; extern RecordType* conn_id; extern RecordType* endpoint; extern RecordType* endpoint_stats; diff --git a/src/OpaqueVal.h b/src/OpaqueVal.h index 0428e50bdb..a4a1741334 100644 --- a/src/OpaqueVal.h +++ b/src/OpaqueVal.h @@ -36,7 +36,7 @@ public: u_char key[MD5_DIGEST_LENGTH], u_char result[MD5_DIGEST_LENGTH]); - MD5Val() : HashVal(new OpaqueType("md5")) { } + MD5Val() : HashVal(new OpaqueType("md5")) { Unref(Type()); } protected: friend class Val; @@ -55,7 +55,7 @@ class SHA1Val : public HashVal { public: static void digest(val_list& vlist, u_char result[SHA_DIGEST_LENGTH]); - SHA1Val() : HashVal(new OpaqueType("sha1")) { } + SHA1Val() : HashVal(new OpaqueType("sha1")) { Unref(Type()); } protected: friend class Val; @@ -74,7 +74,7 @@ class SHA256Val : public HashVal { public: static void digest(val_list& vlist, u_char result[SHA256_DIGEST_LENGTH]); - SHA256Val() : HashVal(new OpaqueType("sha256")) { } + SHA256Val() : HashVal(new OpaqueType("sha256")) { Unref(Type()); } protected: friend class Val; diff --git a/src/PktSrc.cc b/src/PktSrc.cc index 2e6953f7f8..105dc90d30 100644 --- a/src/PktSrc.cc +++ b/src/PktSrc.cc @@ -231,6 +231,15 @@ void PktSrc::Process() data += get_link_header_size(datalink); data += 4; // Skip the vlan header pkt_hdr_size = 0; + + // Check for 802.1ah (Q-in-Q) containing IP. + // Only do a second layer of vlan tag + // stripping because there is no + // specification that allows for deeper + // nesting. + if ( ((data[2] << 8) + data[3]) == 0x0800 ) + data += 4; + break; // PPPoE carried over the ethernet frame. diff --git a/src/RE.cc b/src/RE.cc index b6f1a1361f..87117c1c3a 100644 --- a/src/RE.cc +++ b/src/RE.cc @@ -496,7 +496,7 @@ static RE_Matcher* matcher_merge(const RE_Matcher* re1, const RE_Matcher* re2, safe_snprintf(merge_text, n, "(%s)%s(%s)", text1, merge_op, text2); RE_Matcher* merge = new RE_Matcher(merge_text); - delete merge_text; + delete [] merge_text; merge->Compile(); diff --git a/src/SMTP.cc b/src/SMTP.cc index 85a3bc79dc..0f6dafedc4 100644 --- a/src/SMTP.cc +++ b/src/SMTP.cc @@ -85,9 +85,13 @@ void SMTP_Analyzer::Undelivered(int seq, int len, bool is_orig) Unexpected(is_orig, "content gap", buf_len, buf); if ( state == SMTP_IN_DATA ) + { // Record the SMTP data gap and terminate the // ongoing mail transaction. + if ( mail ) + mail->Undelivered(len); EndData(); + } if ( line_after_gap ) { diff --git a/src/SerialObj.cc b/src/SerialObj.cc index 73cab275c2..ab7f63e823 100644 --- a/src/SerialObj.cc +++ b/src/SerialObj.cc @@ -155,7 +155,7 @@ SerialObj* SerialObj::Unserialize(UnserialInfo* info, SerialType type) else { // Broccoli compatibility mode with 32bit pids. - uint32 tmp; + uint32 tmp = 0; result = UNSERIALIZE(&full_obj) && UNSERIALIZE(&tmp); pid = tmp; } diff --git a/src/Sessions.cc b/src/Sessions.cc index 6f42e5726b..2e5a6ded30 100644 --- a/src/Sessions.cc +++ b/src/Sessions.cc @@ -223,6 +223,12 @@ void NetSessions::NextPacket(double t, const struct pcap_pkthdr* hdr, // we look to see if what we have is consistent with an // IPv4 packet. If not, it's either ARP or IPv6 or weird. + if ( hdr_size > static_cast(hdr->caplen) ) + { + Weird("truncated_link_frame", hdr, pkt); + return; + } + uint32 caplen = hdr->caplen - hdr_size; if ( caplen < sizeof(struct ip) ) { diff --git a/src/SmithWaterman.cc b/src/SmithWaterman.cc index ef329e49a5..e051032756 100644 --- a/src/SmithWaterman.cc +++ b/src/SmithWaterman.cc @@ -96,12 +96,12 @@ VectorVal* BroSubstring::VecToPolicy(Vec* vec) align_val->Assign(0, new StringVal(new BroString(*align.string))); align_val->Assign(1, new Val(align.index, TYPE_COUNT)); - aligns->Assign(j+1, align_val, 0); + aligns->Assign(j+1, align_val); } st_val->Assign(1, aligns); st_val->Assign(2, new Val(bst->IsNewAlignment(), TYPE_BOOL)); - result->Assign(i+1, st_val, 0); + result->Assign(i+1, st_val); } } diff --git a/src/StateAccess.cc b/src/StateAccess.cc index 2d0a8dfc5a..aa4a1f36d2 100644 --- a/src/StateAccess.cc +++ b/src/StateAccess.cc @@ -371,7 +371,7 @@ void StateAccess::Replay() CheckOld("index assign", target.id, op1.val, op3, v->AsVectorVal()->Lookup(index)); - v->AsVectorVal()->Assign(index, op2 ? op2->Ref() : 0, 0); + v->AsVectorVal()->Assign(index, op2 ? op2->Ref() : 0); } else @@ -421,7 +421,7 @@ void StateAccess::Replay() Val* lookup_op1 = v->AsVectorVal()->Lookup(index); int delta = lookup_op1->CoerceToInt() + amount; Val* new_val = new Val(delta, t); - v->AsVectorVal()->Assign(index, new_val, 0); + v->AsVectorVal()->Assign(index, new_val); } else @@ -926,17 +926,22 @@ void NotifierRegistry::Register(ID* id, NotifierRegistry::Notifier* notifier) DBG_LOG(DBG_NOTIFIERS, "registering ID %s for notifier %s", id->Name(), notifier->Name()); + Attr* attr = new Attr(ATTR_TRACKED); + if ( id->Attrs() ) - id->Attrs()->AddAttr(new Attr(ATTR_TRACKED)); + { + if ( ! id->Attrs()->FindAttr(ATTR_TRACKED) ) + id->Attrs()->AddAttr(attr); + } else { attr_list* a = new attr_list; - Attr* attr = new Attr(ATTR_TRACKED); a->append(attr); id->SetAttrs(new Attributes(a, id->Type(), false)); - Unref(attr); } + Unref(attr); + NotifierMap::iterator i = ids.find(id->Name()); if ( i != ids.end() ) @@ -967,7 +972,9 @@ void NotifierRegistry::Unregister(ID* id, NotifierRegistry::Notifier* notifier) if ( i == ids.end() ) return; + Attr* attr = id->Attrs()->FindAttr(ATTR_TRACKED); id->Attrs()->RemoveAttr(ATTR_TRACKED); + Unref(attr); NotifierSet* s = i->second; s->erase(notifier); diff --git a/src/Stats.cc b/src/Stats.cc index 8d48c47a25..1bccb8f9be 100644 --- a/src/Stats.cc +++ b/src/Stats.cc @@ -338,7 +338,7 @@ SampleLogger::~SampleLogger() void SampleLogger::FunctionSeen(const Func* func) { - load_samples->Assign(new StringVal(func->GetID()->Name()), 0); + load_samples->Assign(new StringVal(func->Name()), 0); } void SampleLogger::LocationSeen(const Location* loc) diff --git a/src/Timer.h b/src/Timer.h index 615c8bf69a..f3192cbd79 100644 --- a/src/Timer.h +++ b/src/Timer.h @@ -23,6 +23,7 @@ enum TimerType { TIMER_CONN_STATUS_UPDATE, TIMER_DNS_EXPIRE, TIMER_FILE_ANALYSIS_INACTIVITY, + TIMER_FILE_ANALYSIS_DRAIN, TIMER_FRAG, TIMER_INCREMENTAL_SEND, TIMER_INCREMENTAL_WRITE, diff --git a/src/Trigger.cc b/src/Trigger.cc index b7e08b557e..a567528232 100644 --- a/src/Trigger.cc +++ b/src/Trigger.cc @@ -242,6 +242,7 @@ bool Trigger::Eval() trigger->Cache(frame->GetCall(), v); trigger->Release(); + frame->ClearTrigger(); } Unref(v); @@ -330,6 +331,7 @@ void Trigger::Timeout() #endif trigger->Cache(frame->GetCall(), v); trigger->Release(); + frame->ClearTrigger(); } Unref(v); @@ -424,6 +426,12 @@ Val* Trigger::Lookup(const CallExpr* expr) return (i != cache.end()) ? i->second : 0; } +void Trigger::Disable() + { + UnregisterAll(); + disabled = true; + } + const char* Trigger::Name() const { assert(location); diff --git a/src/Trigger.h b/src/Trigger.h index 8e04fb9189..b752ea8ada 100644 --- a/src/Trigger.h +++ b/src/Trigger.h @@ -49,7 +49,7 @@ public: // Disable this trigger completely. Needed because Unref'ing the trigger // may not immediately delete it as other references may still exist. - void Disable() { disabled = true; } + void Disable(); virtual void Describe(ODesc* d) const { d->Add(""); } @@ -79,7 +79,6 @@ private: friend class TriggerTimer; void Init(); - void DeleteTrigger(); void Register(ID* id); void Register(Val* val); void UnregisterAll(); diff --git a/src/TunnelEncapsulation.h b/src/TunnelEncapsulation.h index e8ca7a48b6..a43f7c0b57 100644 --- a/src/TunnelEncapsulation.h +++ b/src/TunnelEncapsulation.h @@ -186,7 +186,7 @@ public: if ( conns ) { for ( size_t i = 0; i < conns->size(); ++i ) - vv->Assign(i, (*conns)[i].GetRecordVal(), 0); + vv->Assign(i, (*conns)[i].GetRecordVal()); } return vv; diff --git a/src/Type.cc b/src/Type.cc index 1fb813efa1..db6e940e87 100644 --- a/src/Type.cc +++ b/src/Type.cc @@ -696,7 +696,9 @@ string FuncType::FlavorString() const FuncType::~FuncType() { + Unref(args); Unref(arg_types); + Unref(yield); } BroType* FuncType::YieldType() @@ -710,7 +712,7 @@ int FuncType::MatchesIndex(ListExpr*& index) const MATCHES_INDEX_SCALAR : DOES_NOT_MATCH_INDEX; } -int FuncType::CheckArgs(const type_list* args) const +int FuncType::CheckArgs(const type_list* args, bool is_init) const { const type_list* my_args = arg_types->Types(); @@ -718,7 +720,7 @@ int FuncType::CheckArgs(const type_list* args) const return 0; for ( int i = 0; i < my_args->length(); ++i ) - if ( ! same_type((*args)[i], (*my_args)[i]) ) + if ( ! same_type((*args)[i], (*my_args)[i], is_init) ) return 0; return 1; @@ -1720,7 +1722,7 @@ int same_type(const BroType* t1, const BroType* t2, int is_init) return 0; } - return same_type(ft1->Args(), ft2->Args(), is_init); + return ft1->CheckArgs(ft2->ArgTypes()->Types(), is_init); } case TYPE_RECORD: diff --git a/src/Type.h b/src/Type.h index 249d8709c5..0b7620cd68 100644 --- a/src/Type.h +++ b/src/Type.h @@ -370,11 +370,9 @@ public: { Unref(yield); yield = 0; flavor = arg_flav; } int MatchesIndex(ListExpr*& index) const; - int CheckArgs(const type_list* args) const; + int CheckArgs(const type_list* args, bool is_init = false) const; - TypeList* ArgTypes() { return arg_types; } - - ID* GetReturnValueID() const; + TypeList* ArgTypes() const { return arg_types; } void Describe(ODesc* d) const; void DescribeReST(ODesc* d) const; diff --git a/src/Val.cc b/src/Val.cc index 20051aff5f..5133550236 100644 --- a/src/Val.cc +++ b/src/Val.cc @@ -2921,8 +2921,7 @@ VectorVal::~VectorVal() delete val.vector_val; } -bool VectorVal::Assign(unsigned int index, Val* element, const Expr* assigner, - Opcode op) +bool VectorVal::Assign(unsigned int index, Val* element, Opcode op) { if ( element && ! same_type(element->Type(), vector_type->YieldType(), 0) ) @@ -2983,12 +2982,12 @@ bool VectorVal::Assign(unsigned int index, Val* element, const Expr* assigner, } bool VectorVal::AssignRepeat(unsigned int index, unsigned int how_many, - Val* element, const Expr* assigner) + Val* element) { ResizeAtLeast(index + how_many); for ( unsigned int i = index; i < index + how_many; ++i ) - if ( ! Assign(i, element, assigner) ) + if ( ! Assign(i, element ) ) return false; return true; @@ -3089,7 +3088,7 @@ bool VectorVal::DoUnserialize(UnserialInfo* info) { Val* v; UNSERIALIZE_OPTIONAL(v, Val::Unserialize(info, TYPE_ANY)); - Assign(i, v, 0); + Assign(i, v); } return true; diff --git a/src/Val.h b/src/Val.h index 8ebfb2de0e..4b2705c5b4 100644 --- a/src/Val.h +++ b/src/Val.h @@ -968,18 +968,16 @@ public: // Note: does NOT Ref() the element! Remember to do so unless // the element was just created and thus has refcount 1. // - bool Assign(unsigned int index, Val* element, const Expr* assigner, - Opcode op = OP_ASSIGN); - bool Assign(Val* index, Val* element, const Expr* assigner, - Opcode op = OP_ASSIGN) + bool Assign(unsigned int index, Val* element, Opcode op = OP_ASSIGN); + bool Assign(Val* index, Val* element, Opcode op = OP_ASSIGN) { return Assign(index->AsListVal()->Index(0)->CoerceToUnsigned(), - element, assigner, op); + element, op); } // Assigns the value to how_many locations starting at index. bool AssignRepeat(unsigned int index, unsigned int how_many, - Val* element, const Expr* assigner); + Val* element); // Returns nil if no element was at that value. // Lookup does NOT grow the vector to this size. diff --git a/src/bro.bif b/src/bro.bif index 8cea9d9123..34ea9642a3 100644 --- a/src/bro.bif +++ b/src/bro.bif @@ -16,6 +16,8 @@ #include "digest.h" #include "Reporter.h" #include "IPAddr.h" +#include "util.h" +#include "file_analysis/Manager.h" using namespace std; @@ -844,38 +846,21 @@ extern "C" { ## return_mime: If true, the function returns a short MIME type string (e.g., ## ``text/plain`` instead of a more elaborate textual description). ## -## Returns: The MIME type of *data*. +## Returns: The MIME type of *data*, or "" if there was an error. function identify_data%(data: string, return_mime: bool%): string %{ - const char* descr = ""; - static magic_t magic_mime = 0; static magic_t magic_descr = 0; magic_t* magic = return_mime ? &magic_mime : &magic_descr; + bro_init_magic(magic, return_mime ? MAGIC_MIME : MAGIC_NONE); if( ! *magic ) - { - *magic = magic_open(return_mime ? MAGIC_MIME : MAGIC_NONE); + return new StringVal(""); - if ( ! *magic ) - { - reporter->Error("can't init libmagic: %s", magic_error(*magic)); - return new StringVal(""); - } + const char* desc = bro_magic_buffer(*magic, data->Bytes(), data->Len()); - if ( magic_load(*magic, 0) < 0 ) - { - reporter->Error("can't load magic file: %s", magic_error(*magic)); - magic_close(*magic); - *magic = 0; - return new StringVal(""); - } - } - - descr = magic_buffer(*magic, data->Bytes(), data->Len()); - - return new StringVal(descr); + return new StringVal(desc ? desc : ""); %} ## Performs an entropy test on the given data. @@ -1033,29 +1018,6 @@ function clear_table%(v: any%): any return 0; %} -## Returns the number of elements in a container. This function works with all -## container types, i.e., sets, tables, and vectors. -## -## v: The container whose elements are counted. -## -## Returns: The number of elements in *v*. -function length%(v: any%): count - %{ - TableVal* tv = v->Type()->Tag() == TYPE_TABLE ? v->AsTableVal() : 0; - - if ( tv ) - return new Val(tv->Size(), TYPE_COUNT); - - else if ( v->Type()->Tag() == TYPE_VECTOR ) - return new Val(v->AsVectorVal()->Size(), TYPE_COUNT); - - else - { - builtin_error("length() requires a table/set/vector argument"); - return new Val(0, TYPE_COUNT); - } - %} - ## Checks whether two objects reference the same internal object. This function ## uses equality comparison of C++ raw pointer values to determine if the two ## objects are the same. @@ -1349,7 +1311,7 @@ function order%(v: any, ...%) : index_vec for ( i = 0; i < n; ++i ) { int ind = ind_vv[i]; - result_v->Assign(i, new Val(ind, TYPE_COUNT), 0); + result_v->Assign(i, new Val(ind, TYPE_COUNT)); } return result_v; @@ -1633,7 +1595,7 @@ function record_type_to_vector%(rt: string%): string_vec for ( int i = 0; i < type->NumFields(); ++i ) { StringVal* val = new StringVal(type->FieldName(i)); - result->Assign(i+1, val, 0); + result->Assign(i+1, val); } } @@ -2139,7 +2101,7 @@ function routing0_data_to_addrs%(s: string%): addr_vec while ( len > 0 ) { IPAddr a(IPv6, (const uint32*) bytes, IPAddr::Network); - rval->Assign(rval->Size(), new AddrVal(a), 0); + rval->Assign(rval->Size(), new AddrVal(a)); bytes += 16; len -= 16; } @@ -2162,7 +2124,7 @@ function addr_to_counts%(a: addr%): index_vec int len = a->AsAddr().GetBytes(&bytes); for ( int i = 0; i < len; ++i ) - rval->Assign(i, new Val(ntohl(bytes[i]), TYPE_COUNT), 0); + rval->Assign(i, new Val(ntohl(bytes[i]), TYPE_COUNT)); return rval; %} @@ -2868,13 +2830,55 @@ function bytestring_to_hexstr%(bytestring: string%): string return new StringVal(hexstr); %} +## Encodes a Base64-encoded string. +## +## s: The string to encode +## +## Returns: The encoded version of *s*. +## +## .. bro:see:: encode_base64_custom, decode_base64 +function encode_base64%(s: string%): string + %{ + BroString* t = encode_base64(s->AsString()); + if ( t ) + return new StringVal(t); + else + { + reporter->Error("error in encoding string %s", s->CheckString()); + return new StringVal(""); + } + %} + +## Encodes a Base64-encoded string with a custom alphabet. +## +## s: The string to encode +## +## a: The custom alphabet. The empty string indicates the default alphabet. The +## length of *a* must be 64. For example, a custom alphabet could be +## ``"!#$%&/(),-.:;<>@[]^ `_{|}~abcdefghijklmnopqrstuvwxyz0123456789+?"``. +## +## Returns: The encoded version of *s*. +## +## .. bro:see:: encode_base64, decode_base64_custom +function encode_base64_custom%(s: string, a: string%): string + %{ + BroString* t = encode_base64(s->AsString(), a->AsString()); + if ( t ) + return new StringVal(t); + else + { + reporter->Error("error in encoding string %s", s->CheckString()); + return new StringVal(""); + } + %} + ## Decodes a Base64-encoded string. ## ## s: The Base64-encoded string. ## ## Returns: The decoded version of *s*. ## -## .. bro:see:: decode_base64_custom +## .. bro:see:: decode_base64_custom, encode_base64 function decode_base64%(s: string%): string %{ BroString* t = decode_base64(s->AsString()); @@ -2897,7 +2901,7 @@ function decode_base64%(s: string%): string ## ## Returns: The decoded version of *s*. ## -## .. bro:see:: decode_base64 +## .. bro:see:: decode_base64, encode_base64_custom function decode_base64_custom%(s: string, a: string%): string %{ BroString* t = decode_base64(s->AsString(), a->AsString()); @@ -3574,7 +3578,7 @@ function lookup_addr%(host: addr%) : string ## Issues an asynchronous TEXT DNS lookup and delays the function result. ## This function can therefore only be called inside a ``when`` condition, -## e.g., ``when ( local h = lookup_hostname_txt("www.bro-ids.org") ) { f(h); }``. +## e.g., ``when ( local h = lookup_hostname_txt("www.bro.org") ) { f(h); }``. ## ## host: The hostname to lookup. ## @@ -3603,7 +3607,7 @@ function lookup_hostname_txt%(host: string%) : string ## Issues an asynchronous DNS lookup and delays the function result. ## This function can therefore only be called inside a ``when`` condition, -## e.g., ``when ( local h = lookup_hostname("www.bro-ids.org") ) { f(h); }``. +## e.g., ``when ( local h = lookup_hostname("www.bro.org") ) { f(h); }``. ## ## host: The hostname to lookup. ## @@ -5534,6 +5538,22 @@ function match_signatures%(c: connection, pattern_type: int, s: string, return new Val(1, TYPE_BOOL); %} +## For use within a :bro:see:`get_file_handle` handler to return a unique +## identifier to associate with some buffered input to the file analysis +## framework. The buffered data will then immediately be allowed to pass +## pass through the file analysis framework and execute any policy hooks +## that are available. If an empty string is returned, that signifies that +## the buffered data will be discarded with no further action taken on it. +## +## handle: A string that uniquely identifies a file. +## +## .. bro:see:: get_file_handle FileAnalysis::policy +function return_file_handle%(handle: string%): any + %{ + file_mgr->ReceiveHandle(handle->CheckString()); + return 0; + %} + # =========================================================================== # # Deprecated Functions diff --git a/src/bro_inet_ntop.h b/src/bro_inet_ntop.h index 00326b092e..c018403893 100644 --- a/src/bro_inet_ntop.h +++ b/src/bro_inet_ntop.h @@ -5,6 +5,7 @@ extern "C" { #endif +#include #include const char * diff --git a/src/const.bif b/src/const.bif index ea84b3363d..31e6ccee1a 100644 --- a/src/const.bif +++ b/src/const.bif @@ -23,3 +23,5 @@ const Tunnel::delay_gtp_confirmation: bool; const Tunnel::ip_tunnel_timeout: interval; const Threading::heartbeat_interval: interval; + +const FileAnalysis::salt: string; diff --git a/src/event.bif b/src/event.bif index 8dd940f38b..aadaf31454 100644 --- a/src/event.bif +++ b/src/event.bif @@ -577,6 +577,13 @@ event teredo_origin_indication%(outer: connection, inner: teredo_hdr%); ## it may become particularly expensive for real-time analysis. event teredo_bubble%(outer: connection, inner: teredo_hdr%); +## Generated for any GTP message with a GTPv1 header. +## +## c: The connection over which the message is sent. +## +## hdr: The GTPv1 header. +event gtpv1_message%(c: connection, hdr: gtpv1_hdr%); + ## Generated for GTPv1 G-PDU packets. That is, packets with a UDP payload ## that includes a GTP header followed by an IPv4 or IPv6 packet. ## @@ -590,6 +597,60 @@ event teredo_bubble%(outer: connection, inner: teredo_hdr%); ## it may become particularly expensive for real-time analysis. event gtpv1_g_pdu_packet%(outer: connection, inner_gtp: gtpv1_hdr, inner_ip: pkt_hdr%); +## Generated for GTPv1-C Create PDP Context Request messages. +## +## c: The connection over which the message is sent. +## +## hdr: The GTPv1 header. +## +## elements: The set of Information Elements comprising the message. +event gtpv1_create_pdp_ctx_request%(c: connection, hdr: gtpv1_hdr, elements: gtp_create_pdp_ctx_request_elements%); + +## Generated for GTPv1-C Create PDP Context Response messages. +## +## c: The connection over which the message is sent. +## +## hdr: The GTPv1 header. +## +## elements: The set of Information Elements comprising the message. +event gtpv1_create_pdp_ctx_response%(c: connection, hdr: gtpv1_hdr, elements: gtp_create_pdp_ctx_response_elements%); + +## Generated for GTPv1-C Update PDP Context Request messages. +## +## c: The connection over which the message is sent. +## +## hdr: The GTPv1 header. +## +## elements: The set of Information Elements comprising the message. +event gtpv1_update_pdp_ctx_request%(c: connection, hdr: gtpv1_hdr, elements: gtp_update_pdp_ctx_request_elements%); + +## Generated for GTPv1-C Update PDP Context Response messages. +## +## c: The connection over which the message is sent. +## +## hdr: The GTPv1 header. +## +## elements: The set of Information Elements comprising the message. +event gtpv1_update_pdp_ctx_response%(c: connection, hdr: gtpv1_hdr, elements: gtp_update_pdp_ctx_response_elements%); + +## Generated for GTPv1-C Delete PDP Context Request messages. +## +## c: The connection over which the message is sent. +## +## hdr: The GTPv1 header. +## +## elements: The set of Information Elements comprising the message. +event gtpv1_delete_pdp_ctx_request%(c: connection, hdr: gtpv1_hdr, elements: gtp_delete_pdp_ctx_request_elements%); + +## Generated for GTPv1-C Delete PDP Context Response messages. +## +## c: The connection over which the message is sent. +## +## hdr: The GTPv1 header. +## +## elements: The set of Information Elements comprising the message. +event gtpv1_delete_pdp_ctx_response%(c: connection, hdr: gtpv1_hdr, elements: gtp_delete_pdp_ctx_response_elements%); + ## Generated for every packet that has a non-empty transport-layer payload. ## This is a very low-level and expensive event that should be avoided when ## at all possible. It's usually infeasible to handle when processing even @@ -6920,6 +6981,22 @@ event reporter_error%(t: time, msg: string, location: string%) &error_handler; ## recursively for each ``@load``. event bro_script_loaded%(path: string, level: count%); +## This event is handled to provide feedback to the file analysis framework +## about how to identify the logical "file" to which some data/input +## belongs. All incoming data to the framework is buffered, and depends +## on a handler for this event to return a string value that uniquely +## identifies a file. Among all handlers of this event, exactly one must +## call :bro:see:`return_file_handle`. +## +## tag: The analyzer which is carrying the file data. +## +## c: The connection which is carrying the file data. +## +## is_orig: The direction the file data is flowing over the connection. +## +## .. bro:see:: return_file_handle +event get_file_handle%(tag: count, c: connection, is_orig: bool%); + ## Deprecated. Will be removed. event stp_create_endp%(c: connection, e: int, is_orig: bool%); diff --git a/src/file_analysis.bif b/src/file_analysis.bif index 20f8710086..ba62e58855 100644 --- a/src/file_analysis.bif +++ b/src/file_analysis.bif @@ -35,16 +35,12 @@ enum Trigger %{ ## Raised when the beginning of a file is available and that beginning ## is at least the number of bytes indicated by the *bof_buffer_size* ## field of :bro:see:`FileAnalysis::Info`. - TRIGGER_BOF_BUFFER_AVAIL, - ## Raised when the mime type of a file is matched based on magic - ## numbers. TODO: re-purposing protocols/http/file-ident.sig for - ## doing this is tricky since the signature engine doesn't expect - ## to be decoupled from connections, so figure out what work needs - ## done there. - TRIGGER_MIME_TYPE, - ## Raised when the end of a file is detected. If the file is not - ## being transferred linearly, then this doesn't have to mean the full - ## file has been transferred. + TRIGGER_BOF_BUFFER, + ## Raised when an initial guess at the file/mime type of a file is matched + ## based on magic numbers. + TRIGGER_TYPE, + ## Raised to signal that no more file data is incoming and it couldn't be + ## determined whether the full file was actually seen. TRIGGER_EOF, ## The reassembly buffer for the file filled and had to be discarded. ## The *undelivered* field of :bro:see:`FileAnalysis::Info` will @@ -61,41 +57,71 @@ enum Action %{ ACTION_MD5, ACTION_SHA1, ACTION_SHA256, + ACTION_DATA_EVENT, ACTION_PE_ANALYZER, %} function FileAnalysis::postpone_timeout%(file_id: string%): bool %{ - using namespace file_analysis; + using file_analysis::FileID; bool result = file_mgr->PostponeTimeout(FileID(file_id->CheckString())); return new Val(result, TYPE_BOOL); %} -function FileAnalysis::add_action%(file_id: string, - action: FileAnalysis::Action, - args: any%): bool +function FileAnalysis::add_action%(file_id: string, args: any%): bool %{ - using namespace file_analysis; - RecordVal* rv = args->AsRecordVal()->CoerceTo( - BifType::Record::FileAnalysis::ActionArgs); - bool result = file_mgr->AddAction(FileID(file_id->CheckString()), - action->AsEnumVal(), rv); + using file_analysis::FileID; + using BifType::Record::FileAnalysis::ActionArgs; + RecordVal* rv = args->AsRecordVal()->CoerceTo(ActionArgs); + bool result = file_mgr->AddAction(FileID(file_id->CheckString()), rv); Unref(rv); return new Val(result, TYPE_BOOL); %} -function FileAnalysis::remove_action%(file_id: string, - action: FileAnalysis::Action%): bool +function FileAnalysis::remove_action%(file_id: string, args: any%): bool %{ - using namespace file_analysis; - bool result = file_mgr->RemoveAction(FileID(file_id->CheckString()), - action->AsEnumVal()); + using file_analysis::FileID; + using BifType::Record::FileAnalysis::ActionArgs; + RecordVal* rv = args->AsRecordVal()->CoerceTo(ActionArgs); + bool result = file_mgr->RemoveAction(FileID(file_id->CheckString()), rv); + Unref(rv); return new Val(result, TYPE_BOOL); %} function FileAnalysis::stop%(file_id: string%): bool %{ - using namespace file_analysis; - bool result = file_mgr->RemoveFile(FileID(file_id->CheckString())); + using file_analysis::FileID; + bool result = file_mgr->IgnoreFile(FileID(file_id->CheckString())); return new Val(result, TYPE_BOOL); %} + +function FileAnalysis::data_stream%(source: string, data: string%): any + %{ + file_mgr->DataIn(data->Bytes(), data->Len(), source->CheckString()); + return 0; + %} + +function FileAnalysis::data_chunk%(source: string, data: string, + offset: count%): any + %{ + file_mgr->DataIn(data->Bytes(), data->Len(), offset, source->CheckString()); + return 0; + %} + +function FileAnalysis::gap%(source: string, offset: count, len: count%): any + %{ + file_mgr->Gap(offset, len, source->CheckString()); + return 0; + %} + +function FileAnalysis::set_size%(source: string, size: count%): any + %{ + file_mgr->SetSize(size, source->CheckString()); + return 0; + %} + +function FileAnalysis::eof%(source: string%): any + %{ + file_mgr->EndOfFile(source->CheckString()); + return 0; + %} diff --git a/src/file_analysis/Action.h b/src/file_analysis/Action.h index 78611b4ef1..6ddb67367b 100644 --- a/src/file_analysis/Action.h +++ b/src/file_analysis/Action.h @@ -16,7 +16,11 @@ class Info; class Action { public: - virtual ~Action() {} + virtual ~Action() + { + DBG_LOG(DBG_FILE_ANALYSIS, "Destroy action %d", tag); + Unref(args); + } /** * Subclasses may override this to receive file data non-sequentially. @@ -35,7 +39,9 @@ public: { return true; } /** - * Subclasses may override this to specifically handle the end of a file. + * Subclasses may override this to specifically handle an EOF signal, + * which means no more data is going to be incoming and the action/analyzer + * may be deleted/cleaned up soon. * @return true if the action is still in a valid state to continue * receiving data/events or false if it's essentially "done". */ @@ -50,17 +56,45 @@ public: virtual bool Undelivered(uint64 offset, uint64 len) { return true; } + /** + * @return the action type enum value. + */ ActionTag Tag() const { return tag; } + /** + * @return the ActionArgs associated with the aciton. + */ + RecordVal* Args() const { return args; } + + /** + * @return the file_analysis::Info object to which the action is attached. + */ + Info* GetInfo() const { return info; } + + /** + * @return the action tag equivalent of the 'act' field from the ActionArgs + * value \a args. + */ + static ActionTag ArgsTag(const RecordVal* args) + { + using BifType::Record::FileAnalysis::ActionArgs; + return static_cast( + args->Lookup(ActionArgs->FieldOffset("act"))->AsEnum()); + } + protected: - Action(Info* arg_info, ActionTag arg_tag) : info(arg_info), tag(arg_tag) {} + Action(RecordVal* arg_args, Info* arg_info) + : tag(Action::ArgsTag(arg_args)), args(arg_args->Ref()->AsRecordVal()), + info(arg_info) + {} - Info* info; ActionTag tag; + RecordVal* args; + Info* info; }; -typedef Action* (*ActionInstantiator)(const RecordVal* args, Info* info); +typedef Action* (*ActionInstantiator)(RecordVal* args, Info* info); } // namespace file_analysis diff --git a/src/file_analysis/ActionSet.cc b/src/file_analysis/ActionSet.cc new file mode 100644 index 0000000000..51cab26478 --- /dev/null +++ b/src/file_analysis/ActionSet.cc @@ -0,0 +1,187 @@ +#include "ActionSet.h" +#include "Info.h" +#include "Action.h" +#include "Extract.h" +#include "DataEvent.h" +#include "Hash.h" + +using namespace file_analysis; + +// keep in order w/ declared enum values in file_analysis.bif +static ActionInstantiator action_factory[] = { + Extract::Instantiate, + MD5::Instantiate, + SHA1::Instantiate, + SHA256::Instantiate, + DataEvent::Instantiate, +}; + +static void action_del_func(void* v) + { + delete (Action*) v; + } + +ActionSet::ActionSet(Info* arg_info) : info(arg_info) + { + TypeList* t = new TypeList(); + t->Append(BifType::Record::FileAnalysis::ActionArgs->Ref()); + action_hash = new CompositeHash(t); + Unref(t); + action_map.SetDeleteFunc(action_del_func); + } + +ActionSet::~ActionSet() + { + while ( ! mod_queue.empty() ) + { + Modification* mod = mod_queue.front(); + mod->Abort(); + delete mod; + mod_queue.pop(); + } + delete action_hash; + } + +bool ActionSet::AddAction(RecordVal* args) + { + HashKey* key = GetKey(args); + + if ( action_map.Lookup(key) ) + { + DBG_LOG(DBG_FILE_ANALYSIS, "Instantiate action %d skipped for file id" + " %s: already exists", Action::ArgsTag(args), + info->GetFileID().c_str()); + delete key; + return true; + } + + Action* act = InstantiateAction(args); + + if ( ! act ) + { + delete key; + return false; + } + + InsertAction(act, key); + + return true; + } + +bool ActionSet::QueueAddAction(RecordVal* args) + { + HashKey* key = GetKey(args); + Action* act = InstantiateAction(args); + + if ( ! act ) + { + delete key; + return false; + } + + mod_queue.push(new Add(act, key)); + + return true; + } + +bool ActionSet::Add::Perform(ActionSet* set) + { + if ( set->action_map.Lookup(key) ) + { + DBG_LOG(DBG_FILE_ANALYSIS, "Add action %d skipped for file id" + " %s: already exists", act->Tag(), + act->GetInfo()->GetFileID().c_str()); + Abort(); + return true; + } + + set->InsertAction(act, key); + return true; + } + +bool ActionSet::RemoveAction(const RecordVal* args) + { + return RemoveAction(Action::ArgsTag(args), GetKey(args)); + } + +bool ActionSet::RemoveAction(ActionTag tag, HashKey* key) + { + Action* act = (Action*) action_map.Remove(key); + delete key; + + if ( ! act ) + { + DBG_LOG(DBG_FILE_ANALYSIS, "Skip remove action %d for file id %s", + tag, info->GetFileID().c_str()); + return false; + } + + DBG_LOG(DBG_FILE_ANALYSIS, "Remove action %d for file id %s", act->Tag(), + info->GetFileID().c_str()); + delete act; + return true; + } + +bool ActionSet::QueueRemoveAction(const RecordVal* args) + { + HashKey* key = GetKey(args); + ActionTag tag = Action::ArgsTag(args); + + mod_queue.push(new Remove(tag, key)); + + return action_map.Lookup(key); + } + +bool ActionSet::Remove::Perform(ActionSet* set) + { + return set->RemoveAction(tag, key); + } + +HashKey* ActionSet::GetKey(const RecordVal* args) const + { + HashKey* key = action_hash->ComputeHash(args, 1); + if ( ! key ) + reporter->InternalError("ActionArgs type mismatch"); + return key; + } + +Action* ActionSet::InstantiateAction(RecordVal* args) const + { + Action* act = action_factory[Action::ArgsTag(args)](args, info); + + if ( ! act ) + { + DBG_LOG(DBG_FILE_ANALYSIS, "Instantiate action %d failed for file id", + " %s", Action::ArgsTag(args), info->GetFileID().c_str()); + return 0; + } + + return act; + } + +void ActionSet::InsertAction(Action* act, HashKey* key) + { + DBG_LOG(DBG_FILE_ANALYSIS, "Add action %d for file id %s", act->Tag(), + info->GetFileID().c_str()); + action_map.Insert(key, act); + delete key; + info->GetVal()->Lookup(Info::actions_idx)->AsTableVal()->Assign(act->Args(), + new RecordVal(BifType::Record::FileAnalysis::ActionResults)); + } + +void ActionSet::DrainModifications() + { + if ( mod_queue.empty() ) return; + + DBG_LOG(DBG_FILE_ANALYSIS, "Start flushing action mod queue of file id %s", + info->GetFileID().c_str()); + do + { + Modification* mod = mod_queue.front(); + mod->Perform(this); + delete mod; + mod_queue.pop(); + } while ( ! mod_queue.empty() ); + DBG_LOG(DBG_FILE_ANALYSIS, "End flushing action mod queue of file id %s", + info->GetFileID().c_str()); + } diff --git a/src/file_analysis/ActionSet.h b/src/file_analysis/ActionSet.h new file mode 100644 index 0000000000..e1f1355aa9 --- /dev/null +++ b/src/file_analysis/ActionSet.h @@ -0,0 +1,104 @@ +#ifndef FILE_ANALYSIS_ACTIONSET_H +#define FILE_ANALYSIS_ACTIONSET_H + +#include + +#include "Action.h" +#include "Dict.h" +#include "CompHash.h" +#include "Val.h" + +namespace file_analysis { + +class Info; +declare(PDict,Action); + +class ActionSet { +public: + + ActionSet(Info* arg_info); + + ~ActionSet(); + + /** + * @return true if action was instantiated/attached, else false. + */ + bool AddAction(RecordVal* args); + + /** + * @return true if action was able to be instantiated, else false. + */ + bool QueueAddAction(RecordVal* args); + + /** + * @return false if action didn't exist and so wasn't removed, else true. + */ + bool RemoveAction(const RecordVal* args); + + /** + * @return true if action exists at time of call, else false; + */ + bool QueueRemoveAction(const RecordVal* args); + + /** + * Perform all queued modifications to the currently active actions. + */ + void DrainModifications(); + + IterCookie* InitForIteration() const + { return action_map.InitForIteration(); } + + Action* NextEntry(IterCookie* c) + { return action_map.NextEntry(c); } + +protected: + + HashKey* GetKey(const RecordVal* args) const; + Action* InstantiateAction(RecordVal* args) const; + void InsertAction(Action* act, HashKey* key); + bool RemoveAction(ActionTag tag, HashKey* key); + + Info* info; + CompositeHash* action_hash; /**< ActionArgs hashes Action map lookup. */ + PDict(Action) action_map; /**< Actions indexed by ActionArgs. */ + + class Modification { + public: + virtual ~Modification() {} + virtual bool Perform(ActionSet* set) = 0; + virtual void Abort() = 0; + }; + + class Add : public Modification { + public: + Add(Action* arg_act, HashKey* arg_key) + : Modification(), act(arg_act), key(arg_key) {} + virtual ~Add() {} + virtual bool Perform(ActionSet* set); + virtual void Abort() { delete act; delete key; } + + protected: + Action* act; + HashKey* key; + }; + + class Remove : public Modification { + public: + Remove(ActionTag arg_tag, HashKey* arg_key) + : Modification(), tag(arg_tag), key(arg_key) {} + virtual ~Remove() {} + virtual bool Perform(ActionSet* set); + virtual void Abort() { delete key; } + + protected: + ActionTag tag; + HashKey* key; + }; + + typedef queue ModQueue; + ModQueue mod_queue; +}; + +} // namespace file_analysiss + +#endif diff --git a/src/file_analysis/DataEvent.cc b/src/file_analysis/DataEvent.cc new file mode 100644 index 0000000000..4be18589b4 --- /dev/null +++ b/src/file_analysis/DataEvent.cc @@ -0,0 +1,65 @@ +#include + +#include "DataEvent.h" +#include "EventRegistry.h" +#include "Event.h" +#include "util.h" + +using namespace file_analysis; + +DataEvent::DataEvent(RecordVal* args, Info* info, + EventHandlerPtr ce, EventHandlerPtr se) + : Action(args, info), chunk_event(ce), stream_event(se) + { + } + +Action* DataEvent::Instantiate(RecordVal* args, Info* info) + { + using BifType::Record::FileAnalysis::ActionArgs; + + const char* chunk_field = "chunk_event"; + const char* stream_field = "stream_event"; + int chunk_off = ActionArgs->FieldOffset(chunk_field); + int stream_off = ActionArgs->FieldOffset(stream_field); + + Val* chunk_val = args->Lookup(chunk_off); + Val* stream_val = args->Lookup(stream_off); + + if ( ! chunk_val && ! stream_val ) return 0; + + EventHandlerPtr chunk; + EventHandlerPtr stream; + + if ( chunk_val ) + chunk = event_registry->Lookup(chunk_val->AsFunc()->Name()); + + if ( stream_val ) + stream = event_registry->Lookup(stream_val->AsFunc()->Name()); + + return new DataEvent(args, info, chunk, stream); + } + +bool DataEvent::DeliverChunk(const u_char* data, uint64 len, uint64 offset) + { + if ( ! chunk_event ) return true; + + val_list* args = new val_list; + args->append(info->GetVal()->Ref()); + args->append(new StringVal(new BroString(data, len, 0))); + args->append(new Val(offset, TYPE_COUNT)); + mgr.QueueEvent(chunk_event, args); + + return true; + } + +bool DataEvent::DeliverStream(const u_char* data, uint64 len) + { + if ( ! stream_event ) return true; + + val_list* args = new val_list; + args->append(info->GetVal()->Ref()); + args->append(new StringVal(new BroString(data, len, 0))); + mgr.QueueEvent(stream_event, args); + + return true; + } diff --git a/src/file_analysis/DataEvent.h b/src/file_analysis/DataEvent.h new file mode 100644 index 0000000000..083b40bbdc --- /dev/null +++ b/src/file_analysis/DataEvent.h @@ -0,0 +1,35 @@ +#ifndef FILE_ANALYSIS_DATAEVENT_H +#define FILE_ANALYSIS_DATAEVENT_H + +#include + +#include "Val.h" +#include "Info.h" +#include "Action.h" + +namespace file_analysis { + +/** + * An action to send file data to script-layer events. + */ +class DataEvent : public Action { +public: + + static Action* Instantiate(RecordVal* args, Info* info); + + virtual bool DeliverChunk(const u_char* data, uint64 len, uint64 offset); + + virtual bool DeliverStream(const u_char* data, uint64 len); + +protected: + + DataEvent(RecordVal* args, Info* info, + EventHandlerPtr ce, EventHandlerPtr se); + + EventHandlerPtr chunk_event; + EventHandlerPtr stream_event; +}; + +} // namespace file_analysis + +#endif diff --git a/src/file_analysis/Extract.cc b/src/file_analysis/Extract.cc index 3a4897e5cb..c05d37e7ce 100644 --- a/src/file_analysis/Extract.cc +++ b/src/file_analysis/Extract.cc @@ -5,9 +5,8 @@ using namespace file_analysis; -Extract::Extract(Info* arg_info, const string& arg_filename) - : Action(arg_info, BifEnum::FileAnalysis::ACTION_EXTRACT), - filename(arg_filename) +Extract::Extract(RecordVal* args, Info* info, const string& arg_filename) + : Action(args, info), filename(arg_filename) { fd = open(filename.c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0666); @@ -26,21 +25,19 @@ Extract::~Extract() safe_close(fd); } -Action* Extract::Instantiate(const RecordVal* args, Info* info) +Action* Extract::Instantiate(RecordVal* args, Info* info) { + using BifType::Record::FileAnalysis::ActionArgs; const char* field = "extract_filename"; - int off = BifType::Record::FileAnalysis::ActionArgs->FieldOffset(field); - Val* v = args->Lookup(off); + Val* v = args->Lookup(ActionArgs->FieldOffset(field)); if ( ! v ) return 0; - return new Extract(info, v->AsString()->CheckString()); + return new Extract(args, info, v->AsString()->CheckString()); } bool Extract::DeliverChunk(const u_char* data, uint64 len, uint64 offset) { - Action::DeliverChunk(data, len, offset); - if ( ! fd ) return false; safe_pwrite(fd, data, len, offset); diff --git a/src/file_analysis/Extract.h b/src/file_analysis/Extract.h index df03d40967..47bcf202cb 100644 --- a/src/file_analysis/Extract.h +++ b/src/file_analysis/Extract.h @@ -15,7 +15,7 @@ namespace file_analysis { class Extract : public Action { public: - static Action* Instantiate(const RecordVal* args, Info* info); + static Action* Instantiate(RecordVal* args, Info* info); virtual ~Extract(); @@ -23,7 +23,7 @@ public: protected: - Extract(Info* arg_info, const string& arg_filename); + Extract(RecordVal* args, Info* info, const string& arg_filename); string filename; int fd; diff --git a/src/file_analysis/FileID.h b/src/file_analysis/FileID.h index c339445ea8..6d594d6b37 100644 --- a/src/file_analysis/FileID.h +++ b/src/file_analysis/FileID.h @@ -9,8 +9,8 @@ namespace file_analysis { * identify files, and others which use a pretty hash (the FileID) to identify * files. A FileID is primarily used in methods which interface with the * script-layer, while the unique strings are used for methods which interface - * with protocol analyzers (to better accomodate the possibility that a file - * can be distributed over different connections and thus analyzer instances). + * with protocol analyzers or anything that sends data to the file analysis + * framework. */ struct FileID { string id; diff --git a/src/file_analysis/Hash.cc b/src/file_analysis/Hash.cc index 1bb8e48e5d..145974e8cc 100644 --- a/src/file_analysis/Hash.cc +++ b/src/file_analysis/Hash.cc @@ -5,32 +5,33 @@ using namespace file_analysis; -Hash::Hash(Info* arg_info, ActionTag tag, HashVal* hv) - : Action(arg_info, tag), hash(hv) +Hash::Hash(RecordVal* args, Info* info, HashVal* hv, const char* field) + : Action(args, info), hash(hv), fed(false) { + using BifType::Record::FileAnalysis::ActionResults; + if ( (result_field_idx = ActionResults->FieldOffset(field)) < 0 ) + reporter->InternalError("Missing ActionResults field: %s", field); hash->Init(); } Hash::~Hash() { - // maybe it's all there... - Finalize(); - delete hash; + Unref(hash); } bool Hash::DeliverStream(const u_char* data, uint64 len) { - Action::DeliverStream(data, len); - if ( ! hash->IsValid() ) return false; + if ( ! fed ) + fed = len > 0; + hash->Feed(data, len); return true; } bool Hash::EndOfFile() { - Action::EndOfFile(); Finalize(); return false; } @@ -42,13 +43,8 @@ bool Hash::Undelivered(uint64 offset, uint64 len) void Hash::Finalize() { - if ( ! hash->IsValid() ) return; + if ( ! hash->IsValid() || ! fed ) return; StringVal* sv = hash->Get(); - int i = GetResultFieldOffset(); - - if ( i < 0 ) - reporter->InternalError("Hash Action result field not found"); - - info->Results()->Assign(i, sv); + info->GetResults(args)->Assign(result_field_idx, sv); } diff --git a/src/file_analysis/Hash.h b/src/file_analysis/Hash.h index 8b101ac7b6..90345771ba 100644 --- a/src/file_analysis/Hash.h +++ b/src/file_analysis/Hash.h @@ -26,61 +26,52 @@ public: protected: - Hash(Info* arg_info, ActionTag arg_tag, HashVal* hv); + Hash(RecordVal* args, Info* info, HashVal* hv, const char* field); void Finalize(); - virtual int GetResultFieldOffset() const = 0; - HashVal* hash; + bool fed; + int result_field_idx; }; class MD5 : public Hash { public: - static Action* Instantiate(const RecordVal* args, Info* info) - { return new MD5(info); } + static Action* Instantiate(RecordVal* args, Info* info) + { return new MD5(args, info); } protected: - MD5(Info* arg_info) - : Hash(arg_info, BifEnum::FileAnalysis::ACTION_MD5, new MD5Val()) {} - - virtual int GetResultFieldOffset() const - { return BifType::Record::FileAnalysis::ActionResults-> - FieldOffset("md5"); } + MD5(RecordVal* args, Info* info) + : Hash(args, info, new MD5Val(), "md5") + {} }; class SHA1 : public Hash { public: - static Action* Instantiate(const RecordVal* args, Info* info) - { return new SHA1(info); } + static Action* Instantiate(RecordVal* args, Info* info) + { return new SHA1(args, info); } protected: - SHA1(Info* arg_info) - : Hash(arg_info, BifEnum::FileAnalysis::ACTION_SHA1, new SHA1Val()) {} - - virtual int GetResultFieldOffset() const - { return BifType::Record::FileAnalysis::ActionResults-> - FieldOffset("sha1"); } + SHA1(RecordVal* args, Info* info) + : Hash(args, info, new SHA1Val(), "sha1") + {} }; class SHA256 : public Hash { public: - static Action* Instantiate(const RecordVal* args, Info* info) - { return new SHA256(info); } + static Action* Instantiate(RecordVal* args, Info* info) + { return new SHA256(args, info); } protected: - SHA256(Info* arg_info) - : Hash(arg_info, BifEnum::FileAnalysis::ACTION_SHA256, new SHA256Val()) {} - - virtual int GetResultFieldOffset() const - { return BifType::Record::FileAnalysis::ActionResults-> - FieldOffset("sha256"); } + SHA256(RecordVal* args, Info* info) + : Hash(args, info, new SHA256Val(), "sha256") + {} }; } // namespace file_analysis diff --git a/src/file_analysis/Info.cc b/src/file_analysis/Info.cc index 367da820d5..51c7a53241 100644 --- a/src/file_analysis/Info.cc +++ b/src/file_analysis/Info.cc @@ -1,40 +1,25 @@ #include +#include #include "Info.h" #include "InfoTimer.h" #include "FileID.h" +#include "Manager.h" #include "Reporter.h" #include "Val.h" - -#include "Action.h" -#include "Extract.h" -#include "Hash.h" -#include "analyzers/PE.h" +#include "Type.h" +#include "Analyzer.h" using namespace file_analysis; -// keep in order w/ declared enum values in file_analysis.bif -static ActionInstantiator action_factory[] = { - Extract::Instantiate, - MD5::Instantiate, - SHA1::Instantiate, - SHA256::Instantiate, - PE_Analyzer::Instantiate, -}; - -static TableVal* empty_conn_id_set() +static Val* empty_connection_table() { - TypeList* set_index = new TypeList(conn_id); - set_index->Append(conn_id->Ref()); - return new TableVal(new SetType(set_index, 0)); - } - -static StringVal* get_conn_uid_val(Connection* conn) - { - char tmp[20]; - if ( ! conn->GetUID() ) - conn->SetUID(calculate_unique_id()); - return new StringVal(uitoa_n(conn->GetUID(), tmp, sizeof(tmp), 62)); + TypeList* tbl_index = new TypeList(conn_id); + tbl_index->Append(conn_id->Ref()); + TableType* tbl_type = new TableType(tbl_index, connection_type->Ref()); + Val* rval = new TableVal(tbl_type); + Unref(tbl_type); + return rval; } static RecordVal* get_conn_id_val(const Connection* conn) @@ -49,82 +34,112 @@ static RecordVal* get_conn_id_val(const Connection* conn) int Info::file_id_idx = -1; int Info::parent_file_id_idx = -1; -int Info::protocol_idx = -1; -int Info::conn_uids_idx = -1; -int Info::conn_ids_idx = -1; +int Info::source_idx = -1; +int Info::conns_idx = -1; +int Info::last_active_idx = -1; int Info::seen_bytes_idx = -1; int Info::total_bytes_idx = -1; int Info::missing_bytes_idx = -1; int Info::overflow_bytes_idx = -1; int Info::timeout_interval_idx = -1; +int Info::bof_buffer_size_idx = -1; +int Info::bof_buffer_idx = -1; +int Info::file_type_idx = -1; +int Info::mime_type_idx = -1; int Info::actions_idx = -1; -int Info::action_args_idx = -1; -int Info::action_results_idx = -1; -void Info::InitFieldIndices() +magic_t Info::magic = 0; +magic_t Info::magic_mime = 0; + +string Info::salt; + +void Info::StaticInit() { if ( file_id_idx != -1 ) return; + file_id_idx = Idx("file_id"); parent_file_id_idx = Idx("parent_file_id"); - protocol_idx = Idx("protocol"); - conn_uids_idx = Idx("conn_uids"); - conn_ids_idx = Idx("conn_ids"); + source_idx = Idx("source"); + conns_idx = Idx("conns"); + last_active_idx = Idx("last_active"); seen_bytes_idx = Idx("seen_bytes"); total_bytes_idx = Idx("total_bytes"); missing_bytes_idx = Idx("missing_bytes"); overflow_bytes_idx = Idx("overflow_bytes"); timeout_interval_idx = Idx("timeout_interval"); + bof_buffer_size_idx = Idx("bof_buffer_size"); + bof_buffer_idx = Idx("bof_buffer"); + file_type_idx = Idx("file_type"); + mime_type_idx = Idx("mime_type"); actions_idx = Idx("actions"); - action_args_idx = Idx("action_args"); - action_results_idx = Idx("action_results"); + + bro_init_magic(&magic, MAGIC_NONE); + bro_init_magic(&magic_mime, MAGIC_MIME); + + salt = BifConst::FileAnalysis::salt->CheckString(); } -Info::Info(const string& unique, Connection* conn, const string& protocol) - : file_id(unique), unique(unique), val(0), last_activity_time(network_time), - postpone_timeout(false), need_reassembly(false) +Info::Info(const string& unique, Connection* conn, AnalyzerTag::Tag tag) + : file_id(""), unique(unique), val(0), postpone_timeout(false), + need_reassembly(false), done(false), actions(this) { - InitFieldIndices(); + StaticInit(); char id[20]; - uitoa_n(calculate_unique_id(), id, sizeof(id), 62); + uint64 hash[2]; + string msg(unique + salt); + MD5(reinterpret_cast(msg.data()), msg.size(), + reinterpret_cast(hash)); + uitoa_n(hash[0], id, sizeof(id), 62); - DBG_LOG(DBG_FILE_ANALYSIS, "Creating new Info object %s", id); + DBG_LOG(DBG_FILE_ANALYSIS, "Creating new Info object %s (%s)", id, + unique.c_str()); val = new RecordVal(BifType::Record::FileAnalysis::Info); val->Assign(file_id_idx, new StringVal(id)); file_id = FileID(id); - UpdateConnectionFields(conn); + if ( conn ) + { + // add source and connection fields + val->Assign(source_idx, new StringVal(Analyzer::GetTagName(tag))); + UpdateConnectionFields(conn); + } + else + // use the unique file handle as source + val->Assign(source_idx, new StringVal(unique.c_str())); - if ( protocol != "" ) - val->Assign(protocol_idx, new StringVal(protocol.c_str())); - - ScheduleInactivityTimer(); + UpdateLastActivityTime(); } Info::~Info() { - ActionMap::const_iterator it; - for ( it = actions.begin(); it != actions.end(); ++it ) - delete it->second; - - DBG_LOG(DBG_FILE_ANALYSIS, "Destroying Info object %s",file_id.c_str()); + DBG_LOG(DBG_FILE_ANALYSIS, "Destroying Info object %s", file_id.c_str()); Unref(val); } +void Info::UpdateLastActivityTime() + { + val->Assign(last_active_idx, new Val(network_time, TYPE_TIME)); + } + +double Info::GetLastActivityTime() const + { + return val->Lookup(last_active_idx)->AsTime(); + } + void Info::UpdateConnectionFields(Connection* conn) { if ( ! conn ) return; - Val* conn_uids = val->Lookup(conn_uids_idx); - Val* conn_ids = val->Lookup(conn_ids_idx); - if ( ! conn_uids ) - val->Assign(conn_uids_idx, conn_uids = new TableVal(string_set)); - if ( ! conn_ids ) - val->Assign(conn_ids_idx, conn_ids = empty_conn_id_set()); + Val* conns = val->Lookup(conns_idx); - conn_uids->AsTableVal()->Assign(get_conn_uid_val(conn), 0); - conn_ids->AsTableVal()->Assign(get_conn_id_val(conn), 0); + if ( ! conns ) + val->Assign(conns_idx, conns = empty_connection_table()); + + Val* idx = get_conn_id_val(conn); + conns->AsTableVal()->Assign(idx, conn->BuildConnVal()); + Unref(idx); } uint64 Info::LookupFieldDefaultCount(int idx) const @@ -152,14 +167,23 @@ int Info::Idx(const string& field) return rval; } -double Info::TimeoutInterval() const +double Info::GetTimeoutInterval() const { return LookupFieldDefaultInterval(timeout_interval_idx); } -RecordVal* Info::Results() const +RecordVal* Info::GetResults(RecordVal* args) const { - return val->Lookup(action_results_idx)->AsRecordVal(); + TableVal* actions_table = val->Lookup(actions_idx)->AsTableVal(); + RecordVal* rval = actions_table->Lookup(args)->AsRecordVal(); + + if ( ! rval ) + { + rval = new RecordVal(BifType::Record::FileAnalysis::ActionResults); + actions_table->Assign(args, rval); + } + + return rval; } void Info::IncrementByteCount(uint64 size, int field_idx) @@ -184,62 +208,96 @@ bool Info::IsComplete() const void Info::ScheduleInactivityTimer() const { - timer_mgr->Add(new InfoTimer(network_time, file_id, TimeoutInterval())); + timer_mgr->Add(new InfoTimer(network_time, file_id, GetTimeoutInterval())); } -bool Info::AddAction(ActionTag act, RecordVal* args) +bool Info::AddAction(RecordVal* args) { - if ( actions.find(act) != actions.end() ) return false; + return done ? false : actions.QueueAddAction(args); + } - ActionTag tag = static_cast(act); +bool Info::RemoveAction(const RecordVal* args) + { + return done ? false : actions.QueueRemoveAction(args); + } - Action* a = action_factory[act](args, this); +bool Info::BufferBOF(const u_char* data, uint64 len) + { + if ( bof_buffer.full || bof_buffer.replayed ) return false; - if ( ! a ) return false; + if ( bof_buffer.chunks.size() == 0 ) + file_mgr->EvaluatePolicy(BifEnum::FileAnalysis::TRIGGER_BOF, this); - DBG_LOG(DBG_FILE_ANALYSIS, "Add action %d for file id %s", act, - file_id.c_str()); - actions[act] = a; + uint64 desired_size = LookupFieldDefaultCount(bof_buffer_size_idx); - VectorVal* av = val->LookupWithDefault(actions_idx)->AsVectorVal(); - VectorVal* aav = val->LookupWithDefault(action_args_idx)->AsVectorVal(); + /* Leaving out this optimization (I think) for now to keep things simpler. + // If first chunk satisfies desired size, do everything now without copying. + if ( bof_buffer.chunks.empty() && len >= desired_size ) + { + bof_buffer.full = bof_buffer.replayed = true; + val->Assign(bof_buffer_idx, new StringVal(new BroString(data, len, 0))); + file_mgr->EvaluatePolicy(TRIGGER_BOF_BUFFER, this); + // TODO: libmagic stuff + return false; + } + */ - EnumVal* ev = new EnumVal(act, BifType::Enum::FileAnalysis::Action); - av->Assign(av->Size(), ev, 0); - aav->Assign(aav->Size(), args->Ref(), 0); + bof_buffer.chunks.push_back(new BroString(data, len, 0)); + bof_buffer.size += len; - Unref(av); - Unref(aav); + if ( bof_buffer.size >= desired_size ) + { + bof_buffer.full = true; + ReplayBOF(); + } return true; } -bool Info::RemoveAction(ActionTag act) +void Info::ReplayBOF() { - ActionMap::iterator it = actions.find(act); + if ( bof_buffer.replayed ) return; + bof_buffer.replayed = true; - if ( it == actions.end() ) return false; + if ( bof_buffer.chunks.empty() ) return; - return RemoveAction(it); - } + BroString* bs = concatenate(bof_buffer.chunks); + const char* desc = bro_magic_buffer(magic, bs->Bytes(), bs->Len()); + const char* mime = bro_magic_buffer(magic_mime, bs->Bytes(), bs->Len()); -bool Info::RemoveAction(const ActionMap::iterator& it) - { - DBG_LOG(DBG_FILE_ANALYSIS, "Remove action %d for file id %s", it->first, - file_id.c_str()); - delete it->second; - actions.erase(it); - return true; + val->Assign(bof_buffer_idx, new StringVal(bs)); + + if ( desc ) + val->Assign(file_type_idx, new StringVal(desc)); + + if ( mime ) + val->Assign(mime_type_idx, new StringVal(mime)); + + using BifEnum::FileAnalysis::TRIGGER_BOF_BUFFER; + file_mgr->EvaluatePolicy(TRIGGER_BOF_BUFFER, this); + + if ( desc || mime ) + file_mgr->EvaluatePolicy(BifEnum::FileAnalysis::TRIGGER_TYPE, this); + + for ( size_t i = 0; i < bof_buffer.chunks.size(); ++i ) + DataIn(bof_buffer.chunks[i]->Bytes(), bof_buffer.chunks[i]->Len()); } void Info::DataIn(const u_char* data, uint64 len, uint64 offset) { - ActionMap::iterator it = actions.begin(); - while ( it != actions.end() ) - if ( ! it->second->DeliverChunk(data, len, offset) ) - RemoveAction(it++); - else - ++it; + actions.DrainModifications(); + // TODO: attempt libmagic stuff here before doing reassembly? + + Action* act = 0; + IterCookie* c = actions.InitForIteration(); + + while ( (act = actions.NextEntry(c)) ) + { + if ( ! act->DeliverChunk(data, len, offset) ) + actions.QueueRemoveAction(act->Args()); + } + + actions.DrainModifications(); // TODO: check reassembly requirement based on buffer size in record if ( need_reassembly ) @@ -247,53 +305,86 @@ void Info::DataIn(const u_char* data, uint64 len, uint64 offset) // TODO } - // TODO: reassembly stuff, possibly having to deliver chunks if buffer full - // and incrememt overflow bytes + // TODO: reassembly overflow stuff, increment overflow count, eval trigger IncrementByteCount(len, seen_bytes_idx); } void Info::DataIn(const u_char* data, uint64 len) { - ActionMap::iterator it = actions.begin(); - while ( it != actions.end() ) + actions.DrainModifications(); + + if ( BufferBOF(data, len) ) return; + + Action* act = 0; + IterCookie* c = actions.InitForIteration(); + + while ( (act = actions.NextEntry(c)) ) { - if ( ! it->second->DeliverStream(data, len) ) + if ( ! act->DeliverStream(data, len) ) { - RemoveAction(it++); + actions.QueueRemoveAction(act->Args()); continue; } uint64 offset = LookupFieldDefaultCount(seen_bytes_idx) + LookupFieldDefaultCount(missing_bytes_idx); - if ( ! it->second->DeliverChunk(data, len, offset) ) - RemoveAction(it++); - else - ++it; + if ( ! act->DeliverChunk(data, len, offset) ) + actions.QueueRemoveAction(act->Args()); } + actions.DrainModifications(); IncrementByteCount(len, seen_bytes_idx); } void Info::EndOfFile() { - ActionMap::iterator it = actions.begin(); - while ( it != actions.end() ) - if ( ! it->second->EndOfFile() ) - RemoveAction(it++); - else - ++it; + if ( done ) return; + + actions.DrainModifications(); + + // Send along anything that's been buffered, but never flushed. + ReplayBOF(); + + done = true; + + Action* act = 0; + IterCookie* c = actions.InitForIteration(); + + while ( (act = actions.NextEntry(c)) ) + { + if ( ! act->EndOfFile() ) + actions.QueueRemoveAction(act->Args()); + } + + if ( IsComplete() ) + file_mgr->EvaluatePolicy(BifEnum::FileAnalysis::TRIGGER_DONE, this); + else + file_mgr->EvaluatePolicy(BifEnum::FileAnalysis::TRIGGER_EOF, this); + + actions.DrainModifications(); } void Info::Gap(uint64 offset, uint64 len) { - ActionMap::iterator it = actions.begin(); - while ( it != actions.end() ) - if ( ! it->second->Undelivered(offset, len) ) - RemoveAction(it++); - else - ++it; + actions.DrainModifications(); + // If we were buffering the beginning of the file, a gap means we've got + // as much contiguous stuff at the beginning as possible, so work with that. + ReplayBOF(); + + Action* act = 0; + IterCookie* c = actions.InitForIteration(); + + while ( (act = actions.NextEntry(c)) ) + { + if ( ! act->Undelivered(offset, len) ) + actions.QueueRemoveAction(act->Args()); + } + + file_mgr->EvaluatePolicy(BifEnum::FileAnalysis::TRIGGER_GAP, this); + + actions.DrainModifications(); IncrementByteCount(len, missing_bytes_idx); } diff --git a/src/file_analysis/Info.h b/src/file_analysis/Info.h index 7a89e7d898..21589a54c5 100644 --- a/src/file_analysis/Info.h +++ b/src/file_analysis/Info.h @@ -2,12 +2,15 @@ #define FILE_ANALYSIS_INFO_H #include -#include +#include +#include +#include "AnalyzerTags.h" #include "Conn.h" #include "Val.h" -#include "Action.h" +#include "ActionSet.h" #include "FileID.h" +#include "BroString.h" namespace file_analysis { @@ -19,10 +22,15 @@ public: ~Info(); + /** + * @return the #val record. + */ + RecordVal* GetVal() const { return val; } + /** * @return value (seconds) of the "timeout_interval" field from #val record. */ - double TimeoutInterval() const; + double GetTimeoutInterval() const; /** * @return value of the "file_id" field from #val record. @@ -30,24 +38,26 @@ public: FileID GetFileID() const { return file_id; } /** - * @return record val of the "action_results" field from #val record. + * @return looks up the value of the "actions" field in the #val record at + * the index corresponding to \a args. If there was no value at + * the index, it is created. */ - RecordVal* Results() const; + RecordVal* GetResults(RecordVal* args) const; /** * @return the string which uniquely identifies the file. */ - string Unique() const { return unique; } + string GetUnique() const { return unique; } /** - * @return #last_activity_time + * @return value of "last_active" field in #val record; */ - double LastActivityTime() const { return last_activity_time; } + double GetLastActivityTime() const; /** - * Refreshes #last_activity_time with current network time. + * Refreshes "last_active" field of #val record with current network time. */ - void UpdateLastActivityTime() { last_activity_time = network_time; } + void UpdateLastActivityTime(); /** * Set "total_bytes" field of #val record to \a size. @@ -64,21 +74,22 @@ public: /** * Create a timer to be dispatched after the amount of time indicated by * the "timeout_interval" field of the #val record in order to check if - * #last_activity_time is old enough to timeout analysis of the file. + * "last_active" field is old enough to timeout analysis of the file. */ void ScheduleInactivityTimer() const; /** - * Attaches an action. Only one action per type can be attached at a time. - * @return true if the action was attached, else false. + * Queues attaching an action. Only one action per type can be attached at + * a time unless the arguments differ. + * @return false if action can't be instantiated, else true. */ - bool AddAction(ActionTag act, RecordVal* args); + bool AddAction(RecordVal* args); /** - * Removes an action. - * @return true if the action was removed, else false. + * Queues removal of an action. + * @return true if action was active at time of call, else false. */ - bool RemoveAction(ActionTag act); + bool RemoveAction(const RecordVal* args); /** * Pass in non-sequential data and deliver to attached actions/analyzers. @@ -104,13 +115,11 @@ protected: friend class Manager; - typedef map ActionMap; - /** * Constructor; only file_analysis::Manager should be creating these. */ Info(const string& unique, Connection* conn = 0, - const string& protocol = ""); + AnalyzerTag::Tag tag = AnalyzerTag::Error); /** * Updates the "conn_ids" and "conn_uids" fields in #val record with the @@ -136,18 +145,34 @@ protected: double LookupFieldDefaultInterval(int idx) const; /** - * Removes an action. - * @return true if the action was removed, else false. + * Buffers incoming data at the beginning of a file. + * @return true if buffering is still required, else false */ - bool RemoveAction(const ActionMap::iterator& it); + bool BufferBOF(const u_char* data, uint64 len); + + /** + * Forward any beginning-of-file buffered data on to DataIn stream. + */ + void ReplayBOF(); FileID file_id; /**< A pretty hash that likely identifies file*/ string unique; /**< A string that uniquely identifies file */ RecordVal* val; /**< \c FileAnalysis::Info from script layer. */ - double last_activity_time; /**< Time of last activity. */ bool postpone_timeout; /**< Whether postponing timeout is requested. */ bool need_reassembly; /**< Whether file stream reassembly is needed. */ - ActionMap actions; /**< Actions/analysis to perform on file. */ + bool done; /**< If this object is about to be deleted. */ + ActionSet actions; + + struct BOF_Buffer { + BOF_Buffer() : full(false), replayed(false), size(0) {} + ~BOF_Buffer() + { for ( size_t i = 0; i < chunks.size(); ++i ) delete chunks[i]; } + + bool full; + bool replayed; + uint64 size; + BroString::CVec chunks; + } bof_buffer; /**< Beginning of file buffer. */ /** * @return the field offset in #val record corresponding to \a field_name. @@ -155,23 +180,31 @@ protected: static int Idx(const string& field_name); /** - * Initializes the index offsets for fields in \c FileAnalysis::info record. + * Initializes static member. */ - static void InitFieldIndices(); + static void StaticInit(); + static magic_t magic; + static magic_t magic_mime; + + static string salt; + +public: static int file_id_idx; static int parent_file_id_idx; - static int protocol_idx; - static int conn_uids_idx; - static int conn_ids_idx; + static int source_idx; + static int conns_idx; + static int last_active_idx; static int seen_bytes_idx; static int total_bytes_idx; static int missing_bytes_idx; static int overflow_bytes_idx; static int timeout_interval_idx; + static int bof_buffer_size_idx; + static int bof_buffer_idx; + static int file_type_idx; + static int mime_type_idx; static int actions_idx; - static int action_args_idx; - static int action_results_idx; }; } // namespace file_analysis diff --git a/src/file_analysis/InfoTimer.cc b/src/file_analysis/InfoTimer.cc index f1bb524d40..9859839c5a 100644 --- a/src/file_analysis/InfoTimer.cc +++ b/src/file_analysis/InfoTimer.cc @@ -3,13 +3,21 @@ using namespace file_analysis; + +InfoTimer::InfoTimer(double t, const FileID& id, double interval) + : Timer(t + interval, TIMER_FILE_ANALYSIS_INACTIVITY), file_id(id) + { + DBG_LOG(DBG_FILE_ANALYSIS, "New %f second timeout timer for %s", + file_id.c_str(), interval); + } + void InfoTimer::Dispatch(double t, int is_expire) { Info* info = file_mgr->Lookup(file_id); if ( ! info ) return; - double last_active = info->LastActivityTime(); + double last_active = info->GetLastActivityTime(); double inactive_time = t > last_active ? t - last_active : 0.0; DBG_LOG(DBG_FILE_ANALYSIS, "Checking inactivity for %s, last active at %f, " @@ -23,7 +31,7 @@ void InfoTimer::Dispatch(double t, int is_expire) return; } - if ( inactive_time >= info->TimeoutInterval() ) + if ( inactive_time >= info->GetTimeoutInterval() ) file_mgr->Timeout(file_id); else if ( ! is_expire ) info->ScheduleInactivityTimer(); diff --git a/src/file_analysis/InfoTimer.h b/src/file_analysis/InfoTimer.h index ac0d8b6b00..08aefb5d6a 100644 --- a/src/file_analysis/InfoTimer.h +++ b/src/file_analysis/InfoTimer.h @@ -13,10 +13,7 @@ namespace file_analysis { class InfoTimer : public Timer { public: - InfoTimer(double t, const FileID& id, double interval) - : Timer(t + interval, TIMER_FILE_ANALYSIS_INACTIVITY), file_id(id) {} - - ~InfoTimer() {} + InfoTimer(double t, const FileID& id, double interval); /** * Check inactivity of file_analysis::Info corresponding to #file_id, diff --git a/src/file_analysis/Manager.cc b/src/file_analysis/Manager.cc index dbb8366ded..21d9c6bd07 100644 --- a/src/file_analysis/Manager.cc +++ b/src/file_analysis/Manager.cc @@ -4,9 +4,13 @@ #include "Manager.h" #include "Info.h" #include "Action.h" +#include "Var.h" +#include "Event.h" using namespace file_analysis; +TableVal* Manager::disabled = 0; + Manager::Manager() { } @@ -25,57 +29,145 @@ void Manager::Terminate() Timeout(keys[i], true); } -static void check_file_done(Info* info) +void Manager::ReceiveHandle(const string& handle) { - if ( info->IsComplete() ) + if ( pending.empty() ) + reporter->InternalError("File analysis underflow"); + + PendingFile* pf = pending.front(); + if ( ! handle.empty() ) + pf->Finish(handle); + delete pf; + pending.pop(); + } + +void Manager::EventDrainDone() + { + if ( pending.empty() ) return; + + reporter->Error("Too few return_file_handle() calls, discarding pending" + " file analysis input."); + + while ( ! pending.empty() ) { - Manager::EvaluatePolicy(BifEnum::FileAnalysis::TRIGGER_DONE, info); - file_mgr->RemoveFile(info->GetFileID()); + delete pending.front(); + pending.pop(); } } -void Manager::DataIn(const string& unique, const u_char* data, uint64 len, - uint64 offset, Connection* conn, const string& protocol) +void Manager::DataIn(const u_char* data, uint64 len, uint64 offset, + AnalyzerTag::Tag tag, Connection* conn, bool is_orig) { - Info* info = GetInfo(unique, conn, protocol); + if ( IsDisabled(tag) ) return; + if ( ! QueueHandleEvent(tag, conn, is_orig) ) return; + pending.push(new PendingDataInChunk(data, len, offset, tag, conn)); + } + +void Manager::DataIn(const u_char* data, uint64 len, uint64 offset, + const string& unique) + { + DataIn(data, len, offset, GetInfo(unique)); + } + +void Manager::DataIn(const u_char* data, uint64 len, uint64 offset, + Info* info) + { + if ( ! info ) return; + info->DataIn(data, len, offset); - check_file_done(info); + + if ( info->IsComplete() ) + RemoveFile(info->GetUnique()); } -void Manager::DataIn(const string& unique, const u_char* data, uint64 len, - Connection* conn, const string& protocol) +void Manager::DataIn(const u_char* data, uint64 len, AnalyzerTag::Tag tag, + Connection* conn, bool is_orig) { - Info* info = GetInfo(unique, conn, protocol); + if ( IsDisabled(tag) ) return; + if ( ! QueueHandleEvent(tag, conn, is_orig) ) return; + pending.push(new PendingDataInStream(data, len, tag, conn)); + } + +void Manager::DataIn(const u_char* data, uint64 len, const string& unique) + { + DataIn(data, len, GetInfo(unique)); + } + +void Manager::DataIn(const u_char* data, uint64 len, Info* info) + { + if ( ! info ) return; + info->DataIn(data, len); - check_file_done(info); + + if ( info->IsComplete() ) + RemoveFile(info->GetUnique()); } -void Manager::EndOfFile(const string& unique, Connection* conn, - const string& protocol) +void Manager::EndOfFile(AnalyzerTag::Tag tag, Connection* conn) { - Info* info = GetInfo(unique, conn, protocol); - info->EndOfFile(); - Manager::EvaluatePolicy(BifEnum::FileAnalysis::TRIGGER_EOF, info); + EndOfFile(tag, conn, true); + EndOfFile(tag, conn, false); } -void Manager::Gap(const string& unique, uint64 offset, uint64 len, - Connection* conn, const string& protocol) +void Manager::EndOfFile(AnalyzerTag::Tag tag, Connection* conn, bool is_orig) { - Info* info = GetInfo(unique, conn, protocol); + if ( IsDisabled(tag) ) return; + if ( ! QueueHandleEvent(tag, conn, is_orig) ) return; + pending.push(new PendingEOF(tag, conn)); + } + +void Manager::EndOfFile(const string& unique) + { + RemoveFile(unique); + } + +void Manager::Gap(uint64 offset, uint64 len, AnalyzerTag::Tag tag, + Connection* conn, bool is_orig) + { + if ( IsDisabled(tag) ) return; + if ( ! QueueHandleEvent(tag, conn, is_orig) ) return; + pending.push(new PendingGap(offset, len, tag, conn)); + } + +void Manager::Gap(uint64 offset, uint64 len, const string& unique) + { + Gap(offset, len, GetInfo(unique)); + } + +void Manager::Gap(uint64 offset, uint64 len, Info* info) + { + if ( ! info ) return; + info->Gap(offset, len); - Manager::EvaluatePolicy(BifEnum::FileAnalysis::TRIGGER_GAP, info); } -void Manager::SetSize(const string& unique, uint64 size, - Connection* conn, const string& protocol) +void Manager::SetSize(uint64 size, AnalyzerTag::Tag tag, Connection* conn, + bool is_orig) { - Info* info = GetInfo(unique, conn, protocol); + if ( IsDisabled(tag) ) return; + if ( ! QueueHandleEvent(tag, conn, is_orig) ) return; + pending.push(new PendingSize(size, tag, conn)); + } + +void Manager::SetSize(uint64 size, const string& unique) + { + SetSize(size, GetInfo(unique)); + } + +void Manager::SetSize(uint64 size, Info* info) + { + if ( ! info ) return; + info->SetTotalBytes(size); - check_file_done(info); + + if ( info->IsComplete() ) + RemoveFile(info->GetUnique()); } void Manager::EvaluatePolicy(BifEnum::FileAnalysis::Trigger t, Info* info) { + if ( IsIgnored(info->GetUnique()) ) return; + const ID* id = global_scope()->Lookup("FileAnalysis::policy"); assert(id); const Func* hook = id->ID_Val()->AsFunc(); @@ -100,43 +192,46 @@ bool Manager::PostponeTimeout(const FileID& file_id) const return true; } -bool Manager::AddAction(const FileID& file_id, EnumVal* act, - RecordVal* args) const +bool Manager::AddAction(const FileID& file_id, RecordVal* args) const { Info* info = Lookup(file_id); if ( ! info ) return false; - return info->AddAction(static_cast(act->AsEnum()), args); + return info->AddAction(args); } -bool Manager::RemoveAction(const FileID& file_id, EnumVal* act) const +bool Manager::RemoveAction(const FileID& file_id, const RecordVal* args) const { Info* info = Lookup(file_id); if ( ! info ) return false; - return info->RemoveAction(static_cast(act->AsEnum())); + return info->RemoveAction(args); } Info* Manager::GetInfo(const string& unique, Connection* conn, - const string& protocol) + AnalyzerTag::Tag tag) { + if ( IsIgnored(unique) ) return 0; + Info* rval = str_map[unique]; if ( ! rval ) { - rval = str_map[unique] = new Info(unique, conn, protocol); + rval = str_map[unique] = new Info(unique, conn, tag); FileID id = rval->GetFileID(); if ( id_map[id] ) { reporter->Error("Evicted duplicate file ID: %s", id.c_str()); - RemoveFile(id); + RemoveFile(unique); } id_map[id] = rval; - Manager::EvaluatePolicy(BifEnum::FileAnalysis::TRIGGER_NEW, rval); + file_mgr->EvaluatePolicy(BifEnum::FileAnalysis::TRIGGER_NEW, rval); + rval->ScheduleInactivityTimer(); + if ( IsIgnored(unique) ) return 0; } else { @@ -162,7 +257,7 @@ void Manager::Timeout(const FileID& file_id, bool is_terminating) if ( ! info ) return; - Manager::EvaluatePolicy(BifEnum::FileAnalysis::TRIGGER_TIMEOUT, info); + file_mgr->EvaluatePolicy(BifEnum::FileAnalysis::TRIGGER_TIMEOUT, info); if ( info->postpone_timeout && ! is_terminating ) { @@ -176,18 +271,75 @@ void Manager::Timeout(const FileID& file_id, bool is_terminating) DBG_LOG(DBG_FILE_ANALYSIS, "File analysis timeout for %s", info->GetFileID().c_str()); - RemoveFile(file_id); + RemoveFile(info->GetUnique()); } -bool Manager::RemoveFile(const FileID& file_id) +bool Manager::IgnoreFile(const FileID& file_id) { IDMap::iterator it = id_map.find(file_id); if ( it == id_map.end() ) return false; - if ( ! str_map.erase(it->second->Unique()) ) - reporter->Error("No string mapping for file ID %s", file_id.c_str()); - delete it->second; - id_map.erase(it); + DBG_LOG(DBG_FILE_ANALYSIS, "Ignore FileID %s", file_id.c_str()); + + ignored.insert(it->second->GetUnique()); + + return true; + } + +bool Manager::RemoveFile(const string& unique) + { + StrMap::iterator it = str_map.find(unique); + + if ( it == str_map.end() ) return false; + + it->second->EndOfFile(); + + FileID id = it->second->GetFileID(); + + DBG_LOG(DBG_FILE_ANALYSIS, "Remove FileID %s", id.c_str()); + + if ( ! id_map.erase(id) ) + reporter->Error("No mapping for fileID %s", id.c_str()); + + ignored.erase(unique); + delete it->second; + str_map.erase(unique); + return true; + } + +bool Manager::IsIgnored(const string& unique) + { + return ignored.find(unique) != ignored.end(); + } + +bool Manager::IsDisabled(AnalyzerTag::Tag tag) + { + if ( ! disabled ) + disabled = internal_const_val("FileAnalysis::disable")->AsTableVal(); + + Val* index = new Val(tag, TYPE_COUNT); + Val* yield = disabled->Lookup(index); + Unref(index); + + if ( ! yield ) return false; + + bool rval = yield->AsBool(); + Unref(yield); + + return rval; + } + +bool Manager::QueueHandleEvent(AnalyzerTag::Tag tag, Connection* conn, + bool is_orig) + { + if ( ! get_file_handle ) return false; + + val_list* vl = new val_list(); + vl->append(new Val(tag, TYPE_COUNT)); + vl->append(conn->BuildConnVal()); + vl->append(new Val(is_orig, TYPE_BOOL)); + + mgr.QueueEvent(get_file_handle, vl); return true; } diff --git a/src/file_analysis/Manager.h b/src/file_analysis/Manager.h index 7dfaf5a665..930297e3d2 100644 --- a/src/file_analysis/Manager.h +++ b/src/file_analysis/Manager.h @@ -3,14 +3,20 @@ #include #include +#include +#include #include "Net.h" +#include "AnalyzerTags.h" #include "Conn.h" #include "Val.h" +#include "Analyzer.h" +#include "Timer.h" #include "Info.h" #include "InfoTimer.h" #include "FileID.h" +#include "PendingFile.h" namespace file_analysis { @@ -29,42 +35,66 @@ public: */ void Terminate(); + /** + * Associates a handle with the next element in the #pending queue, which + * will immediately push that element all the way through the file analysis + * framework, possibly evaluating any policy hooks. + */ + void ReceiveHandle(const string& handle); + + /** + * Called when all events have been drained from the event queue. + * There should be no pending file input/data at this point. + */ + void EventDrainDone(); + /** * Pass in non-sequential file data. */ - void DataIn(const string& unique, const u_char* data, uint64 len, - uint64 offset, Connection* conn = 0, - const string& protocol = ""); + void DataIn(const u_char* data, uint64 len, uint64 offset, + AnalyzerTag::Tag tag, Connection* conn, bool is_orig); + void DataIn(const u_char* data, uint64 len, uint64 offset, + const string& unique); + void DataIn(const u_char* data, uint64 len, uint64 offset, + Info* info); /** * Pass in sequential file data. */ - void DataIn(const string& unique, const u_char* data, uint64 len, - Connection* conn = 0, const string& protocol = ""); + void DataIn(const u_char* data, uint64 len, AnalyzerTag::Tag tag, + Connection* conn, bool is_orig); + void DataIn(const u_char* data, uint64 len, const string& unique); + void DataIn(const u_char* data, uint64 len, Info* info); /** * Signal the end of file data. */ - void EndOfFile(const string& unique, Connection* conn = 0, - const string& protocol = ""); + void EndOfFile(AnalyzerTag::Tag tag, Connection* conn); + void EndOfFile(AnalyzerTag::Tag tag, Connection* conn, bool is_orig); + void EndOfFile(const string& unique); /** * Signal a gap in the file data stream. */ - void Gap(const string& unique, uint64 offset, uint64 len, - Connection* conn = 0, const string& protocol = ""); + void Gap(uint64 offset, uint64 len, AnalyzerTag::Tag tag, Connection* conn, + bool is_orig); + void Gap(uint64 offset, uint64 len, const string& unique); + void Gap(uint64 offset, uint64 len, Info* info); /** * Provide the expected number of bytes that comprise a file. */ - void SetSize(const string& unique, uint64 size, Connection* conn = 0, - const string& protocol = ""); + void SetSize(uint64 size, AnalyzerTag::Tag tag, Connection* conn, + bool is_orig); + void SetSize(uint64 size, const string& unique); + void SetSize(uint64 size, Info* info); /** - * Discard the file_analysis::Info object associated with \a file_id. + * Starts ignoring a file, which will finally be removed from internal + * mappings on EOF or TIMEOUT. * @return false if file identifier did not map to anything, else true. */ - bool RemoveFile(const FileID& file_id); + bool IgnoreFile(const FileID& file_id); /** * If called during \c FileAnalysis::policy evaluation for a @@ -73,37 +103,43 @@ public: bool PostponeTimeout(const FileID& file_id) const; /** - * Attaches an action to the file identifier. Only one action of a given - * type can be attached per file identifier at a time. - * @return true if the action was attached, else false. + * Queue attachment of an action to the file identifier. Multiple actions + * of a given type can be attached per file identifier at a time as long as + * the arguments differ. + * @return false if the action failed to be instantiated, else true. */ - bool AddAction(const FileID& file_id, EnumVal* act, RecordVal* args) const; + bool AddAction(const FileID& file_id, RecordVal* args) const; /** - * Removes an action for a given file identifier. - * @return true if the action was removed, else false. + * Queue removal of an action for a given file identifier. + * @return true if the action is active at the time of call, else false. */ - bool RemoveAction(const FileID& file_id, EnumVal* act) const; + bool RemoveAction(const FileID& file_id, const RecordVal* args) const; /** * Calls the \c FileAnalysis::policy hook. */ - static void EvaluatePolicy(BifEnum::FileAnalysis::Trigger t, Info* info); + void EvaluatePolicy(BifEnum::FileAnalysis::Trigger t, Info* info); protected: friend class InfoTimer; + friend class PendingFile; typedef map StrMap; + typedef set StrSet; typedef map IDMap; + typedef queue PendingQueue; /** - * @return the Info object mapped to \a unique. One is created if mapping - * doesn't exist. If it did exist, the activity time is refreshed - * and connection-related fields of the record value may be updated. + * @return the Info object mapped to \a unique or a null pointer if analysis + * is being ignored for the associated file. An Info object may be + * created if a mapping doesn't exist, and if it did exist, the + * activity time is refreshed along with any connection-related + * fields. */ Info* GetInfo(const string& unique, Connection* conn = 0, - const string& protocol = ""); + AnalyzerTag::Tag tag = AnalyzerTag::Error); /** * @return the Info object mapped to \a file_id, or a null pointer if no @@ -117,8 +153,35 @@ protected: */ void Timeout(const FileID& file_id, bool is_terminating = ::terminating); + /** + * Immediately remove file_analysis::Info object associated with \a unique. + * @return false if file string did not map to anything, else true. + */ + bool RemoveFile(const string& unique); + + /** + * @return whether the file mapped to \a unique is being ignored. + */ + bool IsIgnored(const string& unique); + + /** + * @return whether file analysis is disabled for the given analyzer. + */ + static bool IsDisabled(AnalyzerTag::Tag tag); + + /** + * Queues \c get_file_handle event in order to retrieve unique file handle. + * @return true if there is a handler for the event, else false. + */ + static bool QueueHandleEvent(AnalyzerTag::Tag tag, Connection* conn, + bool is_orig); + StrMap str_map; /**< Map unique strings to \c FileAnalysis::Info records. */ IDMap id_map; /**< Map file IDs to \c FileAnalysis::Info records. */ + StrSet ignored; /**< Ignored files. Will be finally removed on EOF. */ + PendingQueue pending; /**< Files awaiting a unique handle. */ + + static TableVal* disabled; /**< Table of disabled analyzers. */ }; } // namespace file_analysis diff --git a/src/file_analysis/PendingFile.cc b/src/file_analysis/PendingFile.cc new file mode 100644 index 0000000000..6d8f2d6e45 --- /dev/null +++ b/src/file_analysis/PendingFile.cc @@ -0,0 +1,111 @@ +#include "PendingFile.h" +#include "Manager.h" + +using namespace file_analysis; + +static void copy_data(const u_char** dst, const u_char* src, uint64 len) + { + u_char* tmp = new u_char[len]; + memcpy(tmp, src, len); + *dst = tmp; + } + +static string conn_str(Connection* c) + { + char op[256], rp[256]; + modp_ulitoa10(ntohs(c->OrigPort()), op); + modp_ulitoa10(ntohs(c->RespPort()), rp); + string rval = c->OrigAddr().AsString() + ":" + op + "->" + + c->RespAddr().AsString() + ":" + rp; + return rval; + } + +PendingFile::PendingFile(Connection* arg_conn, AnalyzerTag::Tag arg_tag) + : conn(arg_conn), tag(arg_tag) + { + Ref(conn); + DBG_LOG(DBG_FILE_ANALYSIS, "New pending file: %s", conn_str(conn).c_str()); + } + +PendingFile::~PendingFile() + { + Unref(conn); + DBG_LOG(DBG_FILE_ANALYSIS, "Delete pending file: %s", + conn_str(conn).c_str()); + } + +Info* PendingFile::GetInfo(const string& handle) const + { + return file_mgr->GetInfo(handle, conn, tag); + } + +PendingDataInChunk::PendingDataInChunk(const u_char* arg_data, uint64 arg_len, + uint64 arg_offset, + AnalyzerTag::Tag arg_tag, + Connection* arg_conn) + : PendingFile(arg_conn, arg_tag), len(arg_len), + offset(arg_offset) + { + copy_data(&data, arg_data, len); + } + +void PendingDataInChunk::Finish(const string& handle) const + { + file_mgr->DataIn(data, len, offset, GetInfo(handle)); + } + +PendingDataInChunk::~PendingDataInChunk() + { + delete [] data; + } + +PendingDataInStream::PendingDataInStream(const u_char* arg_data, uint64 arg_len, + AnalyzerTag::Tag arg_tag, + Connection* arg_conn) + : PendingFile(arg_conn, arg_tag), len(arg_len) + { + copy_data(&data, arg_data, len); + } + +void PendingDataInStream::Finish(const string& handle) const + { + file_mgr->DataIn(data, len, GetInfo(handle)); + } + +PendingDataInStream::~PendingDataInStream() + { + delete [] data; + } + +PendingGap::PendingGap(uint64 arg_offset, uint64 arg_len, + AnalyzerTag::Tag arg_tag, Connection* arg_conn) + : PendingFile(arg_conn, arg_tag), offset(arg_offset), + len(arg_len) + { + } + +void PendingGap::Finish(const string& handle) const + { + file_mgr->Gap(offset, len, GetInfo(handle)); + } + +PendingEOF::PendingEOF(AnalyzerTag::Tag arg_tag, Connection* arg_conn) + : PendingFile(arg_conn, arg_tag) + { + } + +void PendingEOF::Finish(const string& handle) const + { + file_mgr->EndOfFile(handle); + } + +PendingSize::PendingSize(uint64 arg_size, AnalyzerTag::Tag arg_tag, + Connection* arg_conn) + : PendingFile(arg_conn, arg_tag), size(arg_size) + { + } + +void PendingSize::Finish(const string& handle) const + { + file_mgr->SetSize(size, GetInfo(handle)); + } diff --git a/src/file_analysis/PendingFile.h b/src/file_analysis/PendingFile.h new file mode 100644 index 0000000000..58b842d969 --- /dev/null +++ b/src/file_analysis/PendingFile.h @@ -0,0 +1,99 @@ +#ifndef FILE_ANALYSIS_PENDINGFILE_H +#define FILE_ANALYSIS_PENDINGFILE_H + +#include "AnalyzerTags.h" +#include "Conn.h" +#include "Info.h" + +namespace file_analysis { + +class PendingFile { +public: + + virtual ~PendingFile(); + + virtual void Finish(const string& handle) const = 0; + +protected: + + PendingFile(Connection* arg_conn, + AnalyzerTag::Tag arg_tag = AnalyzerTag::Error); + + Info* GetInfo(const string& handle) const; + + Connection* conn; + AnalyzerTag::Tag tag; +}; + +class PendingDataInChunk : public PendingFile { +public: + + PendingDataInChunk(const u_char* arg_data, uint64 arg_len, + uint64 arg_offset, AnalyzerTag::Tag arg_tag, + Connection* arg_conn); + + virtual ~PendingDataInChunk(); + + virtual void Finish(const string& handle) const; + +protected: + + const u_char* data; + uint64 len; + uint64 offset; +}; + +class PendingDataInStream : public PendingFile { +public: + + PendingDataInStream(const u_char* arg_data, uint64 arg_len, + AnalyzerTag::Tag arg_tag, Connection* arg_conn); + + virtual ~PendingDataInStream(); + + virtual void Finish(const string& handle) const; + +protected: + + const u_char* data; + uint64 len; +}; + +class PendingGap : public PendingFile { +public: + + PendingGap(uint64 arg_offset, uint64 arg_len, AnalyzerTag::Tag arg_tag, + Connection* arg_conn); + + virtual void Finish(const string& handle) const; + +protected: + + uint64 offset; + uint64 len; +}; + +class PendingEOF : public PendingFile { +public: + + PendingEOF(AnalyzerTag::Tag arg_tag, Connection* arg_conn); + + virtual void Finish(const string& handle) const; +}; + +class PendingSize : public PendingFile { +public: + + PendingSize(uint64 arg_size, AnalyzerTag::Tag arg_tag, + Connection* arg_conn); + + virtual void Finish(const string& handle) const; + +protected: + + uint64 size; +}; + +} // namespace file_analysis + +#endif diff --git a/src/gtpv1-analyzer.pac b/src/gtpv1-analyzer.pac index 08fbdef74a..23281c1bb8 100644 --- a/src/gtpv1-analyzer.pac +++ b/src/gtpv1-analyzer.pac @@ -1,4 +1,607 @@ +%code{ +RecordVal* BuildGTPv1Hdr(const GTPv1_Header* pdu) + { + RecordVal* rv = new RecordVal(BifType::Record::gtpv1_hdr); + + rv->Assign(0, new Val(pdu->version(), TYPE_COUNT)); + rv->Assign(1, new Val(pdu->pt_flag(), TYPE_BOOL)); + rv->Assign(2, new Val(pdu->rsv(), TYPE_BOOL)); + rv->Assign(3, new Val(pdu->e_flag(), TYPE_BOOL)); + rv->Assign(4, new Val(pdu->s_flag(), TYPE_BOOL)); + rv->Assign(5, new Val(pdu->pn_flag(), TYPE_BOOL)); + rv->Assign(6, new Val(pdu->msg_type(), TYPE_COUNT)); + rv->Assign(7, new Val(pdu->length(), TYPE_COUNT)); + rv->Assign(8, new Val(pdu->teid(), TYPE_COUNT)); + + if ( pdu->has_opt() ) + { + rv->Assign(9, new Val(pdu->opt_hdr()->seq(), TYPE_COUNT)); + rv->Assign(10, new Val(pdu->opt_hdr()->n_pdu(), TYPE_COUNT)); + rv->Assign(11, new Val(pdu->opt_hdr()->next_type(), TYPE_COUNT)); + } + + return rv; + } + +Val* BuildIMSI(const InformationElement* ie) + { + return new Val(ie->imsi()->value(), TYPE_COUNT); + } + +Val* BuildRAI(const InformationElement* ie) + { + RecordVal* ev = new RecordVal(BifType::Record::gtp_rai); + ev->Assign(0, new Val(ie->rai()->mcc(), TYPE_COUNT)); + ev->Assign(1, new Val(ie->rai()->mnc(), TYPE_COUNT)); + ev->Assign(2, new Val(ie->rai()->lac(), TYPE_COUNT)); + ev->Assign(3, new Val(ie->rai()->rac(), TYPE_COUNT)); + return ev; + } + +Val* BuildRecovery(const InformationElement* ie) + { + return new Val(ie->recovery()->restart_counter(), TYPE_COUNT); + } + +Val* BuildSelectionMode(const InformationElement* ie) + { + return new Val(ie->selection_mode()->mode(), TYPE_COUNT); + } + +Val* BuildTEID1(const InformationElement* ie) + { + return new Val(ie->teid1()->value(), TYPE_COUNT); + } + +Val* BuildTEID_ControlPlane(const InformationElement* ie) + { + return new Val(ie->teidcp()->value(), TYPE_COUNT); + } + +Val* BuildNSAPI(const InformationElement* ie) + { + return new Val(ie->nsapi()->nsapi(), TYPE_COUNT); + } + +Val* BuildChargingCharacteristics(const InformationElement* ie) + { + return new Val(ie->charging_characteristics()->value(), TYPE_COUNT); + } + +Val* BuildTraceReference(const InformationElement* ie) + { + return new Val(ie->trace_reference()->value(), TYPE_COUNT); + } + +Val* BuildTraceType(const InformationElement* ie) + { + return new Val(ie->trace_type()->value(), TYPE_COUNT); + } + +Val* BuildEndUserAddr(const InformationElement* ie) + { + RecordVal* ev = new RecordVal(BifType::Record::gtp_end_user_addr); + ev->Assign(0, new Val(ie->end_user_addr()->pdp_type_org(), TYPE_COUNT)); + ev->Assign(1, new Val(ie->end_user_addr()->pdp_type_num(), TYPE_COUNT)); + + int len = ie->end_user_addr()->pdp_addr().length(); + + if ( len > 0 ) + { + const uint8* d = ie->end_user_addr()->pdp_addr().data(); + + switch ( ie->end_user_addr()->pdp_type_num() ) { + case 0x21: + ev->Assign(2, new AddrVal( + IPAddr(IPv4, (const uint32*) d, IPAddr::Network))); + break; + case 0x57: + ev->Assign(2, new AddrVal( + IPAddr(IPv6, (const uint32*) d, IPAddr::Network))); + break; + default: + ev->Assign(3, new StringVal( + new BroString((const u_char*) d, len, 0))); + break; + } + } + + return ev; + } + +Val* BuildAccessPointName(const InformationElement* ie) + { + BroString* bs = new BroString((const u_char*) ie->ap_name()->value().data(), + ie->ap_name()->value().length(), 0); + return new StringVal(bs); + } + +Val* BuildProtoConfigOptions(const InformationElement* ie) + { + const u_char* d = (const u_char*) ie->proto_config_opts()->value().data(); + int len = ie->proto_config_opts()->value().length(); + return new StringVal(new BroString(d, len, 0)); + } + +Val* BuildGSN_Addr(const InformationElement* ie) + { + RecordVal* ev = new RecordVal(BifType::Record::gtp_gsn_addr); + + int len = ie->gsn_addr()->value().length(); + const uint8* d = ie->gsn_addr()->value().data(); + + if ( len == 4 ) + ev->Assign(0, new AddrVal( + IPAddr(IPv4, (const uint32*) d, IPAddr::Network))); + else if ( len == 16 ) + ev->Assign(0, new AddrVal( + IPAddr(IPv6, (const uint32*) d, IPAddr::Network))); + else + ev->Assign(1, new StringVal(new BroString((const u_char*) d, len, 0))); + + return ev; + } + +Val* BuildMSISDN(const InformationElement* ie) + { + const u_char* d = (const u_char*) ie->msisdn()->value().data(); + int len = ie->msisdn()->value().length(); + return new StringVal(new BroString(d, len, 0)); + } + +Val* BuildQoS_Profile(const InformationElement* ie) + { + RecordVal* ev = new RecordVal(BifType::Record::gtp_qos_profile); + + const u_char* d = (const u_char*) ie->qos_profile()->data().data(); + int len = ie->qos_profile()->data().length(); + + ev->Assign(0, new Val(ie->qos_profile()->alloc_retention_priority(), + TYPE_COUNT)); + ev->Assign(1, new StringVal(new BroString(d, len, 0))); + + return ev; + } + +Val* BuildTrafficFlowTemplate(const InformationElement* ie) + { + const uint8* d = ie->traffic_flow_template()->value().data(); + int len = ie->traffic_flow_template()->value().length(); + return new StringVal(new BroString((const u_char*) d, len, 0)); + } + +Val* BuildTriggerID(const InformationElement* ie) + { + const uint8* d = ie->trigger_id()->value().data(); + int len = ie->trigger_id()->value().length(); + return new StringVal(new BroString((const u_char*) d, len, 0)); + } + +Val* BuildOMC_ID(const InformationElement* ie) + { + const uint8* d = ie->omc_id()->value().data(); + int len = ie->omc_id()->value().length(); + return new StringVal(new BroString((const u_char*) d, len, 0)); + } + +Val* BuildPrivateExt(const InformationElement* ie) + { + RecordVal* ev = new RecordVal(BifType::Record::gtp_private_extension); + + const uint8* d = ie->private_ext()->value().data(); + int len = ie->private_ext()->value().length(); + + ev->Assign(0, new Val(ie->private_ext()->id(), TYPE_COUNT)); + ev->Assign(1, new StringVal(new BroString((const u_char*) d, len, 0))); + + return ev; + } + +Val* BuildCause(const InformationElement* ie) + { + return new Val(ie->cause()->value(), TYPE_COUNT); + } + +Val* BuildReorderReq(const InformationElement* ie) + { + return new Val(ie->reorder_req()->req(), TYPE_BOOL); + } + +Val* BuildChargingID(const InformationElement* ie) + { + return new Val(ie->charging_id()->value(), TYPE_COUNT);; + } + +Val* BuildChargingGatewayAddr(const InformationElement* ie) + { + const uint8* d = ie->charging_gateway_addr()->value().data(); + int len = ie->charging_gateway_addr()->value().length(); + if ( len == 4 ) + return new AddrVal(IPAddr(IPv4, (const uint32*) d, IPAddr::Network)); + else if ( len == 16 ) + return new AddrVal(IPAddr(IPv6, (const uint32*) d, IPAddr::Network)); + else + return 0; + } + +Val* BuildTeardownInd(const InformationElement* ie) + { + return new Val(ie->teardown_ind()->ind(), TYPE_BOOL); + } + +void CreatePDP_Request(const BroAnalyzer& a, const GTPv1_Header* pdu) + { + if ( ! ::gtpv1_create_pdp_ctx_request ) return; + + RecordVal* rv = new RecordVal( + BifType::Record::gtp_create_pdp_ctx_request_elements); + + const vector * v = pdu->create_pdp_ctx_request(); + + bool second_nsapi = false; + bool second_gsn_addr = false; + + for ( size_t i = 0; i < v->size(); ++i ) + { + const InformationElement* ie = (*v)[i]; + + switch ( ie->type() ) { + case GTPv1::TYPE_IMSI: + rv->Assign(0, BuildIMSI(ie)); + break; + case GTPv1::TYPE_RAI: + rv->Assign(1, BuildRAI(ie)); + break; + case GTPv1::TYPE_RECOVERY: + rv->Assign(2, BuildRecovery(ie)); + break; + case GTPv1::TYPE_SELECTION_MODE: + rv->Assign(3, BuildSelectionMode(ie)); + break; + case GTPv1::TYPE_TEID1: + rv->Assign(4, BuildTEID1(ie)); + break; + case GTPv1::TYPE_TEID_CONTROL_PLANE: + rv->Assign(5, BuildTEID_ControlPlane(ie)); + break; + case GTPv1::TYPE_NSAPI: + if ( second_nsapi ) + rv->Assign(7, BuildNSAPI(ie)); + else + { + second_nsapi = true; + rv->Assign(6, BuildNSAPI(ie)); + } + break; + case GTPv1::TYPE_CHARGING_CHARACTERISTICS: + rv->Assign(8, BuildChargingCharacteristics(ie)); + break; + case GTPv1::TYPE_TRACE_REFERENCE: + rv->Assign(9, BuildTraceReference(ie)); + break; + case GTPv1::TYPE_TRACE_TYPE: + rv->Assign(10, BuildTraceType(ie)); + break; + case GTPv1::TYPE_END_USER_ADDR: + rv->Assign(11, BuildEndUserAddr(ie)); + break; + case GTPv1::TYPE_ACCESS_POINT_NAME: + rv->Assign(12, BuildAccessPointName(ie)); + break; + case GTPv1::TYPE_PROTO_CONFIG_OPTIONS: + rv->Assign(13, BuildProtoConfigOptions(ie)); + break; + case GTPv1::TYPE_GSN_ADDR: + if ( second_gsn_addr ) + rv->Assign(15, BuildGSN_Addr(ie)); + else + { + second_gsn_addr = true; + rv->Assign(14, BuildGSN_Addr(ie)); + } + break; + case GTPv1::TYPE_MSISDN: + rv->Assign(16, BuildMSISDN(ie)); + break; + case GTPv1::TYPE_QOS_PROFILE: + rv->Assign(17, BuildQoS_Profile(ie)); + break; + case GTPv1::TYPE_TRAFFIC_FLOW_TEMPLATE: + rv->Assign(18, BuildTrafficFlowTemplate(ie)); + break; + case GTPv1::TYPE_TRIGGER_ID: + rv->Assign(19, BuildTriggerID(ie)); + break; + case GTPv1::TYPE_OMC_ID: + rv->Assign(20, BuildOMC_ID(ie)); + break; + case GTPv1::TYPE_PRIVATE_EXT: + rv->Assign(21, BuildPrivateExt(ie)); + break; + default: + a->Weird(fmt("gtp_invalid_info_element_%d", (*v)[i]->type())); + break; + } + } + + BifEvent::generate_gtpv1_create_pdp_ctx_request(a, a->Conn(), + BuildGTPv1Hdr(pdu), rv); + } + +void CreatePDP_Response(const BroAnalyzer& a, const GTPv1_Header* pdu) + { + if ( ! ::gtpv1_create_pdp_ctx_response ) + return; + + RecordVal* rv = new RecordVal( + BifType::Record::gtp_create_pdp_ctx_response_elements); + + const vector * v = pdu->create_pdp_ctx_response(); + + bool second_gsn_addr = false; + + for ( size_t i = 0; i < v->size(); ++i ) + { + const InformationElement* ie = (*v)[i]; + + switch ( ie->type() ) { + case GTPv1::TYPE_CAUSE: + rv->Assign(0, BuildCause(ie)); + break; + case GTPv1::TYPE_REORDER_REQ: + rv->Assign(1, BuildReorderReq(ie)); + break; + case GTPv1::TYPE_RECOVERY: + rv->Assign(2, BuildRecovery(ie)); + break; + case GTPv1::TYPE_TEID1: + rv->Assign(3, BuildTEID1(ie)); + break; + case GTPv1::TYPE_TEID_CONTROL_PLANE: + rv->Assign(4, BuildTEID_ControlPlane(ie)); + break; + case GTPv1::TYPE_CHARGING_ID: + rv->Assign(5, BuildChargingID(ie)); + break; + case GTPv1::TYPE_END_USER_ADDR: + rv->Assign(6, BuildEndUserAddr(ie)); + break; + case GTPv1::TYPE_PROTO_CONFIG_OPTIONS: + rv->Assign(7, BuildProtoConfigOptions(ie)); + break; + case GTPv1::TYPE_GSN_ADDR: + if ( second_gsn_addr ) + rv->Assign(9, BuildGSN_Addr(ie)); + else + { + second_gsn_addr = true; + rv->Assign(8, BuildGSN_Addr(ie)); + } + break; + case GTPv1::TYPE_QOS_PROFILE: + rv->Assign(10, BuildQoS_Profile(ie)); + break; + case GTPv1::TYPE_CHARGING_GATEWAY_ADDR: + rv->Assign(11, BuildChargingGatewayAddr(ie)); + break; + case GTPv1::TYPE_PRIVATE_EXT: + rv->Assign(12, BuildPrivateExt(ie)); + break; + default: + a->Weird(fmt("gtp_invalid_info_element_%d", (*v)[i]->type())); + break; + } + } + + BifEvent::generate_gtpv1_create_pdp_ctx_response(a, a->Conn(), + BuildGTPv1Hdr(pdu), rv); + } + +void UpdatePDP_Request(const BroAnalyzer& a, const GTPv1_Header* pdu) + { + if ( ! ::gtpv1_update_pdp_ctx_request ) + return; + + RecordVal* rv = new RecordVal( + BifType::Record::gtp_update_pdp_ctx_request_elements); + + const vector * v = pdu->update_pdp_ctx_request(); + + bool second_gsn_addr = false; + + for ( size_t i = 0; i < v->size(); ++i ) + { + const InformationElement* ie = (*v)[i]; + + switch ( ie->type() ) { + case GTPv1::TYPE_IMSI: + rv->Assign(0, BuildIMSI(ie)); + break; + case GTPv1::TYPE_RAI: + rv->Assign(1, BuildRAI(ie)); + break; + case GTPv1::TYPE_RECOVERY: + rv->Assign(2, BuildRecovery(ie)); + break; + case GTPv1::TYPE_TEID1: + rv->Assign(3, BuildTEID1(ie)); + break; + case GTPv1::TYPE_TEID_CONTROL_PLANE: + rv->Assign(4, BuildTEID_ControlPlane(ie)); + break; + case GTPv1::TYPE_NSAPI: + rv->Assign(5, BuildNSAPI(ie)); + break; + case GTPv1::TYPE_TRACE_REFERENCE: + rv->Assign(6, BuildTraceReference(ie)); + break; + case GTPv1::TYPE_TRACE_TYPE: + rv->Assign(7, BuildTraceType(ie)); + break; + case GTPv1::TYPE_GSN_ADDR: + if ( second_gsn_addr ) + rv->Assign(9, BuildGSN_Addr(ie)); + else + { + second_gsn_addr = true; + rv->Assign(8, BuildGSN_Addr(ie)); + } + break; + case GTPv1::TYPE_QOS_PROFILE: + rv->Assign(10, BuildQoS_Profile(ie)); + break; + case GTPv1::TYPE_TRAFFIC_FLOW_TEMPLATE: + rv->Assign(11, BuildTrafficFlowTemplate(ie)); + break; + case GTPv1::TYPE_TRIGGER_ID: + rv->Assign(12, BuildTriggerID(ie)); + break; + case GTPv1::TYPE_OMC_ID: + rv->Assign(13, BuildOMC_ID(ie)); + break; + case GTPv1::TYPE_PRIVATE_EXT: + rv->Assign(14, BuildPrivateExt(ie)); + break; + case GTPv1::TYPE_END_USER_ADDR: + rv->Assign(15, BuildEndUserAddr(ie)); + break; + default: + a->Weird(fmt("gtp_invalid_info_element_%d", (*v)[i]->type())); + break; + } + } + + BifEvent::generate_gtpv1_update_pdp_ctx_request(a, a->Conn(), + BuildGTPv1Hdr(pdu), rv); + } + +void UpdatePDP_Response(const BroAnalyzer& a, const GTPv1_Header* pdu) + { + if ( ! ::gtpv1_update_pdp_ctx_response ) + return; + + RecordVal* rv = new RecordVal( + BifType::Record::gtp_update_pdp_ctx_response_elements); + + const vector * v = pdu->update_pdp_ctx_response(); + + bool second_gsn_addr = false; + + for ( size_t i = 0; i < v->size(); ++i ) + { + const InformationElement* ie = (*v)[i]; + + switch ( ie->type() ) { + case GTPv1::TYPE_CAUSE: + rv->Assign(0, BuildCause(ie)); + break; + case GTPv1::TYPE_RECOVERY: + rv->Assign(1, BuildRecovery(ie)); + break; + case GTPv1::TYPE_TEID1: + rv->Assign(2, BuildTEID1(ie)); + break; + case GTPv1::TYPE_TEID_CONTROL_PLANE: + rv->Assign(3, BuildTEID_ControlPlane(ie)); + break; + case GTPv1::TYPE_CHARGING_ID: + rv->Assign(4, BuildChargingID(ie)); + break; + case GTPv1::TYPE_GSN_ADDR: + if ( second_gsn_addr ) + rv->Assign(6, BuildGSN_Addr(ie)); + else + { + second_gsn_addr = true; + rv->Assign(5, BuildGSN_Addr(ie)); + } + break; + case GTPv1::TYPE_QOS_PROFILE: + rv->Assign(7, BuildQoS_Profile(ie)); + break; + case GTPv1::TYPE_CHARGING_GATEWAY_ADDR: + rv->Assign(8, BuildChargingGatewayAddr(ie)); + break; + case GTPv1::TYPE_PRIVATE_EXT: + rv->Assign(9, BuildPrivateExt(ie)); + break; + default: + a->Weird(fmt("gtp_invalid_info_element_%d", (*v)[i]->type())); + break; + } + } + + BifEvent::generate_gtpv1_update_pdp_ctx_response(a, a->Conn(), + BuildGTPv1Hdr(pdu), rv); + } + +void DeletePDP_Request(const BroAnalyzer& a, const GTPv1_Header* pdu) + { + if ( ! ::gtpv1_delete_pdp_ctx_request ) + return; + + RecordVal* rv = new RecordVal( + BifType::Record::gtp_delete_pdp_ctx_request_elements); + + const vector * v = pdu->delete_pdp_ctx_request(); + + for ( size_t i = 0; i < v->size(); ++i ) + { + const InformationElement* ie = (*v)[i]; + + switch ( ie->type() ) { + case GTPv1::TYPE_TEARDOWN_IND: + rv->Assign(0, BuildTeardownInd(ie)); + break; + case GTPv1::TYPE_NSAPI: + rv->Assign(1, BuildNSAPI(ie)); + break; + case GTPv1::TYPE_PRIVATE_EXT: + rv->Assign(2, BuildPrivateExt(ie)); + break; + default: + a->Weird(fmt("gtp_invalid_info_element_%d", (*v)[i]->type())); + break; + } + } + + BifEvent::generate_gtpv1_delete_pdp_ctx_request(a, a->Conn(), + BuildGTPv1Hdr(pdu), rv); + } + +void DeletePDP_Response(const BroAnalyzer& a, const GTPv1_Header* pdu) + { + if ( ! ::gtpv1_delete_pdp_ctx_response ) + return; + + RecordVal* rv = new RecordVal( + BifType::Record::gtp_delete_pdp_ctx_response_elements); + + const vector * v = pdu->delete_pdp_ctx_response(); + + for ( size_t i = 0; i < v->size(); ++i ) + { + const InformationElement* ie = (*v)[i]; + + switch ( ie->type() ) { + case GTPv1::TYPE_CAUSE: + rv->Assign(0, BuildCause(ie)); + break; + case GTPv1::TYPE_PRIVATE_EXT: + rv->Assign(1, BuildPrivateExt(ie)); + break; + default: + a->Weird(fmt("gtp_invalid_info_element_%d", (*v)[i]->type())); + break; + } + } + + BifEvent::generate_gtpv1_delete_pdp_ctx_response(a, a->Conn(), + BuildGTPv1Hdr(pdu), rv); + } +%} + connection GTPv1_Conn(bro_analyzer: BroAnalyzer) { upflow = GTPv1_Flow(true); @@ -27,17 +630,17 @@ connection GTPv1_Conn(bro_analyzer: BroAnalyzer) %} } -%code{ -inline void violate(const char* r, const BroAnalyzer& a, const bytestring& p) - { - a->ProtocolViolation(r, (const char*) p.data(), p.length()); - } -%} - flow GTPv1_Flow(is_orig: bool) { datagram = GTPv1_Header withcontext(connection, this); + function violate(r: string, pdu: GTPv1_Header): void + %{ + BroAnalyzer a = connection()->bro_analyzer(); + const_bytestring b = ${pdu.sourcedata}; + a->ProtocolViolation(r.c_str(), (const char*) b.begin(), b.length()); + %} + function process_gtpv1(pdu: GTPv1_Header): bool %{ BroAnalyzer a = connection()->bro_analyzer(); @@ -55,14 +658,14 @@ flow GTPv1_Flow(is_orig: bool) if ( e && e->LastType() == BifEnum::Tunnel::GTPv1 ) { // GTP is never tunneled in GTP so, this must be a regular packet - violate("GTP-in-GTP", a, ${pdu.packet}); + violate("GTP-in-GTP", pdu); return false; } if ( ${pdu.version} != 1 ) { // Only know of GTPv1 with Version == 1 - violate("GTPv1 bad Version", a, ${pdu.packet}); + violate("GTPv1 bad Version", pdu); return false; } @@ -72,21 +675,46 @@ flow GTPv1_Flow(is_orig: bool) return false; } - if ( ${pdu.e_flag} ) - { - // TODO: can't currently parse past extension headers - return false; - } + if ( ::gtpv1_message ) + BifEvent::generate_gtpv1_message(a, c, BuildGTPv1Hdr(pdu)); - if ( ${pdu.msg_type} != 0xff ) - { - // Only interested in decapsulating user plane data beyond here. + switch ( ${pdu.msg_type} ) { + case 16: + CreatePDP_Request(a, pdu); + return true; + case 17: + CreatePDP_Response(a, pdu); + return true; + case 18: + UpdatePDP_Request(a, pdu); + return true; + case 19: + UpdatePDP_Response(a, pdu); + return true; + case 20: + DeletePDP_Request(a, pdu); + return true; + case 21: + DeletePDP_Response(a, pdu); + return true; + case 255: + return process_g_pdu(pdu); + default: return false; - } + } + + return false; + %} + + function process_g_pdu(pdu: GTPv1_Header): bool + %{ + BroAnalyzer a = connection()->bro_analyzer(); + Connection *c = a->Conn(); + const EncapsulationStack* e = c->GetEncapsulation(); if ( ${pdu.packet}.length() < (int)sizeof(struct ip) ) { - violate("Truncated GTPv1", a, ${pdu.packet}); + violate("Truncated GTPv1", pdu); return false; } @@ -94,7 +722,7 @@ flow GTPv1_Flow(is_orig: bool) if ( ip->ip_v != 4 && ip->ip_v != 6 ) { - violate("non-IP packet in GTPv1", a, ${pdu.packet}); + violate("non-IP packet in GTPv1", pdu); return false; } @@ -113,10 +741,10 @@ flow GTPv1_Flow(is_orig: bool) } else if ( result < 0 ) - violate("Truncated GTPv1", a, ${pdu.packet}); + violate("Truncated GTPv1", pdu); else - violate("GTPv1 payload length", a, ${pdu.packet}); + violate("GTPv1 payload length", pdu); if ( result != 0 ) { @@ -125,37 +753,16 @@ flow GTPv1_Flow(is_orig: bool) } if ( ::gtpv1_g_pdu_packet ) - { - RecordVal* rv = new RecordVal(gtpv1_hdr_type); - - rv->Assign(0, new Val(${pdu.version}, TYPE_COUNT)); - rv->Assign(1, new Val(${pdu.pt_flag}, TYPE_BOOL)); - rv->Assign(2, new Val(${pdu.rsv}, TYPE_BOOL)); - rv->Assign(3, new Val(${pdu.e_flag}, TYPE_BOOL)); - rv->Assign(4, new Val(${pdu.s_flag}, TYPE_BOOL)); - rv->Assign(5, new Val(${pdu.pn_flag}, TYPE_BOOL)); - rv->Assign(6, new Val(${pdu.msg_type}, TYPE_COUNT)); - rv->Assign(7, new Val(ntohs(${pdu.length}), TYPE_COUNT)); - rv->Assign(8, new Val(ntohl(${pdu.teid}), TYPE_COUNT)); - - if ( ${pdu.has_opt} ) - { - rv->Assign(9, new Val(ntohs(${pdu.opt_hdr.seq}), TYPE_COUNT)); - rv->Assign(10, new Val(${pdu.opt_hdr.n_pdu}, TYPE_COUNT)); - rv->Assign(11, new Val(${pdu.opt_hdr.next_type}, TYPE_COUNT)); - } - - BifEvent::generate_gtpv1_g_pdu_packet(a, c, rv, + BifEvent::generate_gtpv1_g_pdu_packet(a, c, BuildGTPv1Hdr(pdu), inner->BuildPktHdrVal()); - } EncapsulatingConn ec(c, BifEnum::Tunnel::GTPv1); sessions->DoNextInnerPacket(network_time(), 0, inner, e, ec); - return (result == 0) ? true : false; + return true; %} - }; refine typeattr GTPv1_Header += &let { proc_gtpv1 = $context.flow.process_gtpv1(this); }; + diff --git a/src/gtpv1-protocol.pac b/src/gtpv1-protocol.pac index 5bf31a48ee..0a87665fd2 100644 --- a/src/gtpv1-protocol.pac +++ b/src/gtpv1-protocol.pac @@ -4,11 +4,27 @@ type GTPv1_Header = record { msg_type: uint8; length: uint16; teid: uint32; + opt: case has_opt of { true -> opt_hdr: GTPv1_Opt_Header; false -> no_opt: empty; - } &requires(has_opt); - packet: bytestring &restofdata; + }; + + ext: case e_flag of { + true -> ext_hdrs: GTPv1_Ext_Header[] &until($element.next_type == 0); + false -> no_ext: empty; + }; + + msg: case msg_type of { + 16 -> create_pdp_ctx_request: InformationElement[]; + 17 -> create_pdp_ctx_response: InformationElement[]; + 18 -> update_pdp_ctx_request: InformationElement[]; + 19 -> update_pdp_ctx_response: InformationElement[]; + 20 -> delete_pdp_ctx_request: InformationElement[]; + 21 -> delete_pdp_ctx_response: InformationElement[]; + 255 -> packet: bytestring &restofdata; + default -> unknown: bytestring &restofdata; + }; } &let { version: uint8 = (flags & 0xE0) >> 5; @@ -18,10 +34,463 @@ type GTPv1_Header = record { s_flag: bool = flags & 0x02; pn_flag: bool = flags & 0x01; has_opt: bool = flags & 0x07; -} &byteorder = littleendian; +} &byteorder = bigendian, &exportsourcedata; type GTPv1_Opt_Header = record { seq: uint16; n_pdu: uint8; next_type: uint8; -} +}; + +type GTPv1_Ext_Header = record { + length: uint8; + contents: bytestring &length=(length * 4 - 2); + next_type: uint8; +}; + +enum InfoElementType { + TYPE_CAUSE = 1, + TYPE_IMSI = 2, + TYPE_RAI = 3, + TYPE_TLLI = 4, + TYPE_P_TMSI = 5, + TYPE_REORDER_REQ = 8, + TYPE_AUTHN_TRIPLET = 9, + TYPE_MAP_CAUSE = 11, + TYPE_P_TMSI_SIG = 12, + TYPE_MS_VALID = 13, + TYPE_RECOVERY = 14, + TYPE_SELECTION_MODE = 15, + TYPE_TEID1 = 16, + TYPE_TEID_CONTROL_PLANE = 17, + TYPE_TEID2 = 18, + TYPE_TEARDOWN_IND = 19, + TYPE_NSAPI = 20, + TYPE_RANAP_CAUSE = 21, + TYPE_RAB_CTX = 22, + TYPE_RADIO_PRIORITY_SMS = 23, + TYPE_RADIO_PRIORITY = 24, + TYPE_PACKET_FLOW_ID = 25, + TYPE_CHARGING_CHARACTERISTICS = 26, + TYPE_TRACE_REFERENCE = 27, + TYPE_TRACE_TYPE = 28, + TYPE_MS_NOT_REACHABLE_REASON = 29, + TYPE_CHARGING_ID = 127, + TYPE_END_USER_ADDR = 128, + TYPE_MM_CTX = 129, + TYPE_PDP_CTX = 130, + TYPE_ACCESS_POINT_NAME = 131, + TYPE_PROTO_CONFIG_OPTIONS = 132, + TYPE_GSN_ADDR = 133, + TYPE_MSISDN = 134, + TYPE_QOS_PROFILE = 135, + TYPE_AUTHN_QUINTUPLET = 136, + TYPE_TRAFFIC_FLOW_TEMPLATE = 137, + TYPE_TARGET_ID = 138, + TYPE_UTRAN_TRANSPARENT_CONTAINER = 139, + TYPE_RAB_SETUP_INFO = 140, + TYPE_EXT_HEADER_TYPE_LIST = 141, + TYPE_TRIGGER_ID = 142, + TYPE_OMC_ID = 143, + TYPE_CHARGING_GATEWAY_ADDR = 251, + TYPE_PRIVATE_EXT = 255, +}; + +type InformationElement = record { + type: uint8; + + len: case is_tlv of { + true -> tlv_len: uint16; + false -> no_len: empty; + }; + + value: case type of { + TYPE_CAUSE -> cause: Cause; + TYPE_IMSI -> imsi: IMSI; + TYPE_RAI -> rai: RAI; + TYPE_TLLI -> tlli: TLLI; + TYPE_P_TMSI -> p_tmsi: P_TMSI; + TYPE_REORDER_REQ -> reorder_req: ReorderReq; + TYPE_AUTHN_TRIPLET -> authn_triplet: AuthN_Triplet; + TYPE_MAP_CAUSE -> map_cause: MAP_Cause; + TYPE_P_TMSI_SIG -> p_tmsi_sig: P_TMSI_Sig; + TYPE_MS_VALID -> ms_valid: MS_Valid; + TYPE_RECOVERY -> recovery: Recovery; + TYPE_SELECTION_MODE -> selection_mode: SelectionMode; + TYPE_TEID1 -> teid1: TEID1; + TYPE_TEID_CONTROL_PLANE -> teidcp: TEID_ControlPlane; + TYPE_TEID2 -> teid2: TEID2; + TYPE_TEARDOWN_IND -> teardown_ind: TeardownInd; + TYPE_NSAPI -> nsapi: NSAPI; + TYPE_RANAP_CAUSE -> ranap_cause: RANAP_Cause; + TYPE_RAB_CTX -> rab_ctx: RAB_Ctx; + TYPE_RADIO_PRIORITY_SMS -> radio_priority_sms: RadioPrioritySMS; + TYPE_RADIO_PRIORITY -> radio_priority: RadioPriority; + TYPE_PACKET_FLOW_ID -> packet_flow_id: PacketFlowID; + TYPE_CHARGING_CHARACTERISTICS -> charging_characteristics: ChargingCharacteristics; + TYPE_TRACE_REFERENCE -> trace_reference: TraceReference; + TYPE_TRACE_TYPE -> trace_type: TraceType; + TYPE_MS_NOT_REACHABLE_REASON -> ms_not_reachable_reason: MS_Not_Reachable_Reason; + TYPE_CHARGING_ID -> charging_id: ChargingID; + TYPE_END_USER_ADDR -> end_user_addr: EndUserAddr(length); + TYPE_MM_CTX -> mm_ctx: MM_Ctx(length); + TYPE_PDP_CTX -> pdp_ctx: PDP_Ctx(length); + TYPE_ACCESS_POINT_NAME -> ap_name: AP_Name(length); + TYPE_PROTO_CONFIG_OPTIONS -> proto_config_opts: ProtoConfigOpts(length); + TYPE_GSN_ADDR -> gsn_addr: GSN_Addr(length); + TYPE_MSISDN -> msisdn: MSISDN(length); + TYPE_QOS_PROFILE -> qos_profile: QoS_Profile(length); + TYPE_AUTHN_QUINTUPLET -> authn_quintuplet: AuthN_Quintuplet(length); + TYPE_TRAFFIC_FLOW_TEMPLATE -> traffic_flow_template: TrafficFlowTemplate(length); + TYPE_TARGET_ID -> target_id: TargetID(length); + TYPE_UTRAN_TRANSPARENT_CONTAINER -> utran_transparent_container: UTRAN_TransparentContainer(length); + TYPE_RAB_SETUP_INFO -> rab_setup_info: RAB_SetupInfo(length); + TYPE_EXT_HEADER_TYPE_LIST -> ext_hdr_type_list: ExtHdrTypeList(length); + TYPE_TRIGGER_ID -> trigger_id: TriggerID(length); + TYPE_OMC_ID -> omc_id: OMC_ID(length); + TYPE_CHARGING_GATEWAY_ADDR -> charging_gateway_addr: ChargingGatewayAddr(length); + TYPE_PRIVATE_EXT -> private_ext: PrivateExt(length); + default -> unknown: bytestring &length=length; + } &requires(length); + +} &let { + is_tlv: bool = (type & 0x80); + length: uint16 = is_tlv ? tlv_len : Get_IE_Len(type); +}; + +type Cause = record { + value: uint8; +}; + +function decode_imsi(v: uint8[8]): uint64 + %{ + uint64 rval = 0; + uint8 digits[16]; + for ( size_t i = 0; i < v->size(); ++i ) + { + digits[2 * i + 1] = ((*v)[i] & 0xf0) >> 4; + digits[2 * i] = (*v)[i] & 0x0f; + } + int power = 0; + for ( int i = 15; i >= 0; --i ) + { + if ( digits[i] == 0x0f ) continue; + rval += digits[i] * pow(10, power); + ++power; + } + return rval; + %} + +type IMSI = record { + tbcd_encoded_value: uint8[8]; +} &let { + value: uint64 = decode_imsi(tbcd_encoded_value); +}; + +type RAI = record { + mcc2_mcc1: uint8; + mnc3_mcc3: uint8; + mnc2_mnc1: uint8; + lac: uint16; + rac: uint8; +} &let { + mcc1: uint8 = (mcc2_mcc1 & 0x0f); + mcc2: uint8 = ((mcc2_mcc1 & 0xf0)>>4); + mcc3: uint8 = (mnc3_mcc3 & 0x0f); + mcc: uint16 = mcc1 * 100 + mcc2 * 10 + mcc3; + mnc1: uint8 = (mnc2_mnc1 & 0x0f); + mnc2: uint8 = ((mnc2_mnc1 & 0xf0)>>4); + mnc3: uint8 = (mnc3_mcc3 & 0xf0)>>4; + mnc: uint16 = (mnc3 & 0x0f) ? mnc1 * 10 + mnc2 : mnc1 * 100 + mnc2 * 10 + mnc3; +}; + +type TLLI = record { + value: uint32; +}; + +type P_TMSI = record { + value: uint32; +}; + +type ReorderReq = record { + value: uint8; +} &let { + req: bool = value & 0x01; +}; + +type AuthN_Triplet = record { + rand: bytestring &length=16; + sres: uint32; + kc: uint64; +}; + +type MAP_Cause = record { + value: uint8; +}; + +type P_TMSI_Sig = record { + value: bytestring &length=3; +}; + +type MS_Valid = record { + value: uint8; +}; + +type Recovery = record { + restart_counter: uint8; +}; + +type SelectionMode = record { + value: uint8; +} &let { + mode: uint8 = value & 0x01; +}; + +type TEID1 = record { + value: uint32; +}; + +type TEID_ControlPlane = record { + value: uint32; +}; + +type TEID2 = record { + spare_nsapi: uint8; + teid2: uint32; +}; + +type TeardownInd = record { + value: uint8; +} &let { + ind: bool = value & 0x01; +}; + +type NSAPI = record { + xxxx_nsapi: uint8; +} &let { + nsapi: uint8 = xxxx_nsapi & 0x0f; +}; + +type RANAP_Cause = record { + value: uint8; +}; + +type RAB_Ctx = record { + spare_nsapi: uint8; + dl_gtpu_seq_num: uint16; + ul_gtpu_seq_num: uint16; + dl_pdcp_seq_num: uint16; + ul_pdcp_seq_num: uint16; +}; + +type RadioPrioritySMS = record { + value: uint8; +}; + +type RadioPriority = record { + nsapi_radio_priority: uint8; +}; + +type PacketFlowID = record { + rsv_nsapi: uint8; + packet_flow_id: uint8; +}; + +type ChargingCharacteristics = record { + value: uint16; +}; + +type TraceReference = record { + value: uint16; +}; + +type TraceType = record { + value: uint16; +}; + +type MS_Not_Reachable_Reason = record { + value: uint8; +}; + +type ChargingID = record { + value: uint32; +}; + +type EndUserAddr(n: uint16) = record { + spare_pdp_type_org: uint8; + pdp_type_num: uint8; + pdp_addr: bytestring &length=(n-2); +} &let { + pdp_type_org: uint8 = spare_pdp_type_org & 0x0f; +}; + +type MM_Ctx(n: uint16) = record { + spare_cksn_ksi: uint8; + security_params: uint8; + + keys: case gsm_keys of { + true -> kc: uint64; + false -> ck_ik: bytestring &length=32; + }; + + vector_len: case have_triplets of { + true -> no_quint_len: empty; + false -> quint_len: uint16; + }; + + vectors: case have_triplets of { + true -> triplets: AuthN_Triplet[num_vectors]; + false -> quintuplets: AuthN_Quintuplet(quint_len)[num_vectors]; + } &requires(num_vectors); + + drx_param: uint16; + ms_net_capability_len: uint8; + ms_net_capability: bytestring &length=ms_net_capability_len; + container_len: uint16; + container: bytestring &length=container_len; + +} &let { + security_mode: uint8 = security_params >> 6; + gsm_keys: bool = security_mode & 0x01; + have_triplets: bool = (security_mode == 1); + num_vectors: uint8 = (security_params & 0x38) >> 3; +}; + +type PDP_Ctx(n: uint16) = record { + rsv_nsapi: uint8; + xxxx_sapi: uint8; + qos_sub_len: uint8; + qos_sub: QoS_Profile(qos_sub_len); + qos_req_len: uint8; + qos_req: QoS_Profile(qos_req_len); + qos_neg_len: uint8; + qos_neg: QoS_Profile(qos_neg_len); + snd: uint16; + snu: uint16; + send_npdu_num: uint8; + recv_npdu_num: uint8; + ul_teid_cp: TEID_ControlPlane; + ul_teid_data1: TEID1; + pdp_ctx_id: uint8; + spare_pdp_type_org: uint8; + pdp_type_num: uint8; + pdp_addr_len: uint8; + pdp_addr: bytestring &length=pdp_addr_len; + ggsn_addr_control_plane_len: uint8; + ggsn_addr_control_plane: bytestring &length=ggsn_addr_control_plane_len; + ggsn_addr_user_traffic_len: uint8; + ggsn_addr_user_traffic: bytestring &length=ggsn_addr_user_traffic_len; + apn_len: uint8; + apn: AP_Name(apn_len); + spare_transaction_id: uint8; + transaction_id: uint8; +}; + +type AP_Name(n: uint16) = record { + value: bytestring &length=n; +}; + +type ProtoConfigOpts(n: uint16) = record { + value: bytestring &length=n; +}; + +type GSN_Addr(n: uint16) = record { + value: bytestring &length=n; +}; + +type MSISDN(n: uint16) = record { + value: bytestring &length=n; +}; + +type QoS_Profile(n: uint16) = record { + alloc_retention_priority: uint8; + data: bytestring &length=n-1; +}; + +type AuthN_Quintuplet(n: uint16) = record { + rand: bytestring &length=16; + xres_len: uint8; + xres: bytestring &length=xres_len; + ck: bytestring &length=16; + ik: bytestring &length=16; + autn_len: uint8; + autn: bytestring &length=autn_len; +}; + +type TrafficFlowTemplate(n: uint16) = record { + value: bytestring &length=n; +}; + +type TargetID(n: uint16) = record { + value: bytestring &length=n; +}; + +type UTRAN_TransparentContainer(n: uint16) = record { + value: bytestring &length=n; +}; + +type RAB_SetupInfo(n: uint16) = record { + xxxx_nsapi: uint8; + + have_teid: case n of { + 1 -> no_teid: empty; + default -> teid: TEID1; + }; + + have_addr: case n of { + 1 -> no_addr: empty; + default -> rnc_addr: bytestring &length=n-5; + }; +}; + +type ExtHdrTypeList(n: uint16) = record { + value: uint8[n]; +}; + +type TriggerID(n: uint16) = record { + value: bytestring &length=n; +}; + +type OMC_ID(n: uint16) = record { + value: bytestring &length=n; +}; + +type ChargingGatewayAddr(n: uint16) = record { + value: bytestring &length=n; +}; + +type PrivateExt(n: uint16) = record { + id: uint16; + value: bytestring &length=n-2; +}; + +function Get_IE_Len(t: uint8): uint16 = + case t of { + TYPE_CAUSE -> 1; + TYPE_IMSI -> 8; + TYPE_RAI -> 6; + TYPE_TLLI -> 4; + TYPE_P_TMSI -> 4; + TYPE_REORDER_REQ -> 1; + TYPE_AUTHN_TRIPLET -> 28; + TYPE_MAP_CAUSE -> 1; + TYPE_P_TMSI_SIG -> 3; + TYPE_MS_VALID -> 1; + TYPE_RECOVERY -> 1; + TYPE_SELECTION_MODE -> 1; + TYPE_TEID1 -> 4; + TYPE_TEID_CONTROL_PLANE -> 4; + TYPE_TEID2 -> 5; + TYPE_TEARDOWN_IND -> 1; + TYPE_NSAPI -> 1; + TYPE_RANAP_CAUSE -> 1; + TYPE_RAB_CTX -> 9; + TYPE_RADIO_PRIORITY_SMS -> 1; + TYPE_RADIO_PRIORITY -> 1; + TYPE_PACKET_FLOW_ID -> 2; + TYPE_CHARGING_CHARACTERISTICS -> 2; + TYPE_TRACE_REFERENCE -> 2; + TYPE_TRACE_TYPE -> 2; + TYPE_MS_NOT_REACHABLE_REASON -> 1; + TYPE_CHARGING_ID -> 4; + }; diff --git a/src/input.bif b/src/input.bif index 199b665fa6..d7e4de5463 100644 --- a/src/input.bif +++ b/src/input.bif @@ -57,3 +57,6 @@ const autospread: double; const addfactor: count; const stopspreadat: count; const timedspread: double; + +module InputBinary; +const chunk_size: count; diff --git a/src/input/Manager.cc b/src/input/Manager.cc index d9006d66a2..1b4f9daf04 100644 --- a/src/input/Manager.cc +++ b/src/input/Manager.cc @@ -8,6 +8,7 @@ #include "readers/Ascii.h" #include "readers/Raw.h" #include "readers/Benchmark.h" +#include "readers/Binary.h" #include "Event.h" #include "EventHandler.h" @@ -34,6 +35,7 @@ ReaderDefinition input_readers[] = { { BifEnum::Input::READER_ASCII, "Ascii", 0, reader::Ascii::Instantiate }, { BifEnum::Input::READER_RAW, "Raw", 0, reader::Raw::Instantiate }, { BifEnum::Input::READER_BENCHMARK, "Benchmark", 0, reader::Benchmark::Instantiate }, + { BifEnum::Input::READER_BINARY, "Binary", 0, reader::Binary::Instantiate }, // End marker { BifEnum::Input::READER_DEFAULT, "None", 0, (ReaderBackend* (*)(ReaderFrontend* frontend))0 } @@ -483,7 +485,7 @@ bool Manager::CreateEventStream(RecordVal* fval) Unref(fields); // ref'd by lookupwithdefault stream->num_fields = fieldsV.size(); stream->fields = fields->Ref()->AsRecordType(); - stream->event = event_registry->Lookup(event->GetID()->Name()); + stream->event = event_registry->Lookup(event->Name()); stream->want_record = ( want_record->InternalInt() == 1 ); Unref(want_record); // ref'd by lookupwithdefault @@ -644,7 +646,7 @@ bool Manager::CreateTableStream(RecordVal* fval) stream->tab = dst->AsTableVal(); stream->rtype = val ? val->AsRecordType() : 0; stream->itype = idx->AsRecordType(); - stream->event = event ? event_registry->Lookup(event->GetID()->Name()) : 0; + stream->event = event ? event_registry->Lookup(event->Name()) : 0; stream->currDict = new PDict(InputHash); stream->currDict->SetDeleteFunc(input_hash_delete_func); stream->lastDict = new PDict(InputHash); @@ -2107,7 +2109,7 @@ Val* Manager::ValueToVal(const Value* val, BroType* request_type) VectorType* vt = new VectorType(type->Ref()); VectorVal* v = new VectorVal(vt); for ( int i = 0; i < val->val.vector_val.size; i++ ) - v->Assign(i, ValueToVal( val->val.set_val.vals[i], type ), 0); + v->Assign(i, ValueToVal( val->val.set_val.vals[i], type )); Unref(vt); return v; diff --git a/src/input/readers/Benchmark.cc b/src/input/readers/Benchmark.cc index 0c25092e08..0584037e05 100644 --- a/src/input/readers/Benchmark.cc +++ b/src/input/readers/Benchmark.cc @@ -8,6 +8,7 @@ #include #include #include +#include #include "../../threading/Manager.h" @@ -71,7 +72,9 @@ string Benchmark::RandomString(const int len) double Benchmark::CurrTime() { struct timeval tv; - assert ( gettimeofday(&tv, 0) >= 0 ); + if ( gettimeofday(&tv, 0) != 0 ) { + FatalError(Fmt("Could not get time: %d", errno)); + } return double(tv.tv_sec) + double(tv.tv_usec) / 1e6; } diff --git a/src/input/readers/Binary.cc b/src/input/readers/Binary.cc new file mode 100644 index 0000000000..da86753303 --- /dev/null +++ b/src/input/readers/Binary.cc @@ -0,0 +1,264 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#include + +#include "Binary.h" +#include "NetVar.h" + +#include "../../threading/SerialTypes.h" + +using namespace input::reader; +using threading::Value; +using threading::Field; + +streamsize Binary::chunk_size = 0; + +Binary::Binary(ReaderFrontend *frontend) + : ReaderBackend(frontend), in(0), mtime(0), firstrun(true) + { + if ( ! chunk_size ) + { + chunk_size = BifConst::InputBinary::chunk_size; + if ( ! chunk_size ) + chunk_size = 1024; + } + } + +Binary::~Binary() + { + DoClose(); + } + +void Binary::DoClose() + { + if ( in ) + CloseInput(); + } + +bool Binary::OpenInput() + { + in = new ifstream(fname.c_str(), ios_base::in | ios_base::binary); + + if ( in->fail() ) + { + Error(Fmt("Init: cannot open %s", fname.c_str())); + return false; + } + + return true; + } + +bool Binary::CloseInput() + { + if ( ! in || ! in->is_open() ) + { + InternalError(Fmt("Trying to close closed file for stream %s", + fname.c_str())); + return false; + } + +#ifdef DEBUG + Debug(DBG_INPUT, "Binary reader starting close"); +#endif + + in->close(); + delete in; + in = 0; + +#ifdef DEBUG + Debug(DBG_INPUT, "Binary reader finished close"); +#endif + + return true; + } + +bool Binary::DoInit(const ReaderInfo& info, int num_fields, + const Field* const* fields) + { + in = 0; + mtime = 0; + firstrun = true; + + if ( ! info.source || strlen(info.source) == 0 ) + { + Error("No source path provided"); + return false; + } + + if ( num_fields != 1 ) + { + Error("Filter for binary reader contains more than one field. Filters " + "for binary reader must contain exactly one string field. " + "Filter ignored."); + return false; + } + + if ( fields[0]->type != TYPE_STRING ) + { + Error("Filter for binary reader contains a non-string field."); + return false; + } + + // do Initialization + fname = info.source; + + if ( ! OpenInput() ) return false; + + if ( UpdateModificationTime() == -1 ) return false; + +#ifdef DEBUG + Debug(DBG_INPUT, "Binary reader created, will perform first update"); +#endif + + // after initialization - do update + DoUpdate(); + +#ifdef DEBUG + Debug(DBG_INPUT, "Binary reader did first update"); +#endif + + return true; + } + +streamsize Binary::GetChunk(char** chunk) + { + if ( in->peek() == std::iostream::traits_type::eof() ) + return 0; + + if ( in->eof() == true || in->fail() == true ) + return 0; + + *chunk = new char[chunk_size]; + + in->read(*chunk, chunk_size); + + streamsize bytes_read = in->gcount(); + + if ( ! bytes_read ) + { + delete *chunk; + *chunk = 0; + return 0; + } + + // probably faster to just not resize if bytes_read < chunk_size, since + // length of valid data is known + + return bytes_read; + } + +int Binary::UpdateModificationTime() + { + struct stat sb; + + if ( stat(fname.c_str(), &sb) == -1 ) + { + Error(Fmt("Could not get stat for %s", fname.c_str())); + return -1; + } + + if ( sb.st_mtime <= mtime ) + // no change + return 0; + + mtime = sb.st_mtime; + return 1; + } + +// read the entire file and send appropriate thingies back to InputMgr +bool Binary::DoUpdate() + { + if ( firstrun ) + firstrun = false; + + else + { + switch ( Info().mode ) { + case MODE_REREAD: + { + switch ( UpdateModificationTime() ) { + case -1: + return false; // error + case 0: + return true; // no change + case 1: + break; // file changed. reread. + default: + assert(false); + } + // fallthrough + } + + case MODE_MANUAL: + case MODE_STREAM: + if ( Info().mode == MODE_STREAM && in ) + { + in->clear(); // remove end of file evil bits + break; + } + + CloseInput(); + if ( ! OpenInput() ) + return false; + + break; + + default: + assert(false); + } + } + + char* chunk = 0; + streamsize size = 0; + while ( (size = GetChunk(&chunk)) ) + { + assert (NumFields() == 1); + + Value** fields = new Value*[1]; + + // filter has exactly one text field. convert to it. + Value* val = new Value(TYPE_STRING, true); + val->val.string_val.data = chunk; + val->val.string_val.length = size; + fields[0] = val; + + if ( Info().mode == MODE_STREAM ) + Put(fields); + else + SendEntry(fields); + } + + if ( Info().mode != MODE_STREAM ) + EndCurrentSend(); + +#ifdef DEBUG + Debug(DBG_INPUT, "DoUpdate finished successfully"); +#endif + + return true; + } + +bool Binary::DoHeartbeat(double network_time, double current_time) + { + switch ( Info().mode ) { + case MODE_MANUAL: + // yay, we do nothing :) + break; + + case MODE_REREAD: + case MODE_STREAM: +#ifdef DEBUG + Debug(DBG_INPUT, "Starting Heartbeat update"); +#endif + Update(); // call update and not DoUpdate, because update + // checks disabled. +#ifdef DEBUG + Debug(DBG_INPUT, "Finished with heartbeat update"); +#endif + break; + default: + assert(false); + } + + return true; + } diff --git a/src/input/readers/Binary.h b/src/input/readers/Binary.h new file mode 100644 index 0000000000..2705800ab8 --- /dev/null +++ b/src/input/readers/Binary.h @@ -0,0 +1,48 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#ifndef INPUT_READERS_BINARY_H +#define INPUT_READERS_BINARY_H + +#include "../ReaderBackend.h" +#include + +namespace input { namespace reader { + +/** + * Binary mode file reader. + */ +class Binary : public ReaderBackend { +public: + Binary(ReaderFrontend* frontend); + + ~Binary(); + + static ReaderBackend* Instantiate(ReaderFrontend* frontend) + { return new Binary(frontend); } + +protected: + virtual bool DoInit(const ReaderInfo& info, int arg_num_fields, + const threading::Field* const* fields); + virtual void DoClose(); + virtual bool DoUpdate(); + virtual bool DoHeartbeat(double network_time, double current_time); + +private: + bool OpenInput(); + bool CloseInput(); + streamsize GetChunk(char** chunk); + int UpdateModificationTime(); + + string fname; + ifstream* in; + time_t mtime; + bool firstrun; + + // options set from the script-level. + static streamsize chunk_size; +}; + +} +} + +#endif /* INPUT_READERS_BINARY_H */ diff --git a/src/logging/Manager.cc b/src/logging/Manager.cc index f62cd1685d..1ab83d84ba 100644 --- a/src/logging/Manager.cc +++ b/src/logging/Manager.cc @@ -365,7 +365,7 @@ bool Manager::CreateStream(EnumVal* id, RecordVal* sval) streams[idx]->id = id->Ref()->AsEnumVal(); streams[idx]->enabled = true; streams[idx]->name = id->Type()->AsEnumType()->Lookup(idx); - streams[idx]->event = event ? event_registry->Lookup(event->GetID()->Name()) : 0; + streams[idx]->event = event ? event_registry->Lookup(event->Name()) : 0; streams[idx]->columns = columns->Ref()->AsRecordType(); DBG_LOG(DBG_LOGGING, "Created new logging stream '%s', raising event %s", diff --git a/src/logging/writers/DataSeries.cc b/src/logging/writers/DataSeries.cc index 27e021ba2c..bc15c6f5b9 100644 --- a/src/logging/writers/DataSeries.cc +++ b/src/logging/writers/DataSeries.cc @@ -167,7 +167,7 @@ string DataSeries::BuildDSSchemaFromFieldTypes(const vector& vals, string xmlschema = "\n"; + + "\" version=\"1.0\" namespace=\"bro.org\">\n"; for( size_t i = 0; i < vals.size(); ++i ) { diff --git a/src/modbus-analyzer.pac b/src/modbus-analyzer.pac index b03df9d188..a9c773b9e9 100644 --- a/src/modbus-analyzer.pac +++ b/src/modbus-analyzer.pac @@ -149,7 +149,7 @@ refine flow ModbusTCP_Flow += { for ( unsigned int i=0; i < ${message.registers}->size(); ++i ) { Val* r = new Val(${message.registers[i]}, TYPE_COUNT); - t->Assign(i, r, 0, OP_ASSIGN); + t->Assign(i, r); } BifEvent::generate_modbus_read_holding_registers_response(connection()->bro_analyzer(), @@ -192,7 +192,7 @@ refine flow ModbusTCP_Flow += { for ( unsigned int i=0; i < (${message.registers})->size(); ++i ) { Val* r = new Val(${message.registers[i]}, TYPE_COUNT); - t->Assign(i, r, 0, OP_ASSIGN); + t->Assign(i, r); } BifEvent::generate_modbus_read_input_registers_response(connection()->bro_analyzer(), @@ -335,7 +335,7 @@ refine flow ModbusTCP_Flow += { for ( unsigned int i = 0; i < (${message.registers}->size()); ++i ) { Val* r = new Val(${message.registers[i]}, TYPE_COUNT); - t->Assign(i, r, 0, OP_ASSIGN); + t->Assign(i, r); } BifEvent::generate_modbus_write_multiple_registers_request(connection()->bro_analyzer(), @@ -371,13 +371,13 @@ refine flow ModbusTCP_Flow += { //for ( unsigned int i = 0; i < (${message.references}->size()); ++i ) // { // Val* r = new Val((${message.references[i].ref_type}), TYPE_COUNT); - // t->Assign(i, r, 0, OP_ASSIGN); + // t->Assign(i, r); // // Val* k = new Val((${message.references[i].file_num}), TYPE_COUNT); - // t->Assign(i, k, 0, OP_ASSIGN); + // t->Assign(i, k); // // Val* l = new Val((${message.references[i].record_num}), TYPE_COUNT); - // t->Assign(i, l, 0, OP_ASSIGN); + // t->Assign(i, l); // } BifEvent::generate_modbus_read_file_record_request(connection()->bro_analyzer(), @@ -398,7 +398,7 @@ refine flow ModbusTCP_Flow += { // { // //TODO: work the reference type in here somewhere // Val* r = new Val(${message.references[i].record_data}), TYPE_COUNT); - // t->Assign(i, r, 0, OP_ASSIGN); + // t->Assign(i, r); // } BifEvent::generate_modbus_read_file_record_response(connection()->bro_analyzer(), @@ -418,18 +418,18 @@ refine flow ModbusTCP_Flow += { //for ( unsigned int i = 0; i < (${message.references}->size()); ++i ) // { // Val* r = new Val((${message.references[i].ref_type}), TYPE_COUNT); - // t->Assign(i, r, 0, OP_ASSIGN); + // t->Assign(i, r); // // Val* k = new Val((${message.references[i].file_num}), TYPE_COUNT); - // t->Assign(i, k, 0, OP_ASSIGN); + // t->Assign(i, k); // // Val* n = new Val((${message.references[i].record_num}), TYPE_COUNT); - // t->Assign(i, n, 0, OP_ASSIGN); + // t->Assign(i, n); // // for ( unsigned int j = 0; j < (${message.references[i].register_value}->size()); ++j ) // { // k = new Val((${message.references[i].register_value[j]}), TYPE_COUNT); - // t->Assign(i, k, 0, OP_ASSIGN); + // t->Assign(i, k); // } // } @@ -451,18 +451,18 @@ refine flow ModbusTCP_Flow += { //for ( unsigned int i = 0; i < (${messages.references}->size()); ++i ) // { // Val* r = new Val((${message.references[i].ref_type}), TYPE_COUNT); - // t->Assign(i, r, 0, OP_ASSIGN); + // t->Assign(i, r); // // Val* f = new Val((${message.references[i].file_num}), TYPE_COUNT); - // t->Assign(i, f, 0, OP_ASSIGN); + // t->Assign(i, f); // // Val* rn = new Val((${message.references[i].record_num}), TYPE_COUNT); - // t->Assign(i, rn, 0, OP_ASSIGN); + // t->Assign(i, rn); // // for ( unsigned int j = 0; j<(${message.references[i].register_value}->size()); ++j ) // { // Val* k = new Val((${message.references[i].register_value[j]}), TYPE_COUNT); - // t->Assign(i, k, 0, OP_ASSIGN); + // t->Assign(i, k); // } BifEvent::generate_modbus_write_file_record_response(connection()->bro_analyzer(), @@ -519,7 +519,7 @@ refine flow ModbusTCP_Flow += { for ( unsigned int i = 0; i < ${message.write_register_values}->size(); ++i ) { Val* r = new Val(${message.write_register_values[i]}, TYPE_COUNT); - t->Assign(i, r, 0, OP_ASSIGN); + t->Assign(i, r); } BifEvent::generate_modbus_read_write_multiple_registers_request(connection()->bro_analyzer(), @@ -550,7 +550,7 @@ refine flow ModbusTCP_Flow += { for ( unsigned int i = 0; i < ${message.registers}->size(); ++i ) { Val* r = new Val(${message.registers[i]}, TYPE_COUNT); - t->Assign(i, r, 0, OP_ASSIGN); + t->Assign(i, r); } BifEvent::generate_modbus_read_write_multiple_registers_response(connection()->bro_analyzer(), @@ -593,7 +593,7 @@ refine flow ModbusTCP_Flow += { for ( unsigned int i = 0; i < (${message.register_data})->size(); ++i ) { Val* r = new Val(${message.register_data[i]}, TYPE_COUNT); - t->Assign(i, r, 0, OP_ASSIGN); + t->Assign(i, r); } BifEvent::generate_modbus_read_fifo_queue_response(connection()->bro_analyzer(), diff --git a/src/nb_dns.c b/src/nb_dns.c index e8595e6837..33a00837e4 100644 --- a/src/nb_dns.c +++ b/src/nb_dns.c @@ -265,6 +265,7 @@ _nb_dns_mkquery(register struct nb_dns_info *nd, register const char *name, default: snprintf(errstr, NB_DNS_ERRSIZE, "_nb_dns_mkquery: bad family %d", atype); + free(ne); return (-1); } diff --git a/src/netflow-analyzer.pac b/src/netflow-analyzer.pac index e89a0181a2..666de076c8 100644 --- a/src/netflow-analyzer.pac +++ b/src/netflow-analyzer.pac @@ -38,7 +38,7 @@ flow NetFlow_Flow { # %cleanup does not only put the cleanup code into the destructor, # but also at the end of the catch clause in NewData(). This is # different from the documentation at - # http://www.bro-ids.org/wiki/index.php/BinPAC_Userguide#.25cleanup.7B....25.7D + # http://www.bro.org/wiki/index.php/BinPAC_Userguide#.25cleanup.7B....25.7D # # Unfortunately this means that we cannot clean up the identifier # string. Note that IOSource destructors seemingly are never diff --git a/src/patricia.c b/src/patricia.c index 6998576fdb..c4815b40ec 100644 --- a/src/patricia.c +++ b/src/patricia.c @@ -336,7 +336,7 @@ ascii2prefix (int family, char *string) memcpy (save, string, cp - string); save[cp - string] = '\0'; string = save; - if (bitlen < 0 || bitlen > maxbitlen) + if (bitlen > maxbitlen) bitlen = maxbitlen; } else { diff --git a/src/strings.bif b/src/strings.bif index dc5e064dc6..e19e970aa0 100644 --- a/src/strings.bif +++ b/src/strings.bif @@ -265,19 +265,6 @@ function edit%(arg_s: string, arg_edit_char: string%): string return new StringVal(new BroString(1, byte_vec(new_s), ind)); %} -## Returns the number of characters (bytes) in the given string. The -## length computation includes any embedded NULs, and also a trailing NUL, -## if any (which is why the function isn't called ``strlen``; to remind -## the user that Bro strings can include NULs). -## -## s: The string to compute the length for. -## -## Returns: The number of characters in *s*. -function byte_len%(s: string%): count - %{ - return new Val(s->Len(), TYPE_COUNT); - %} - ## Get a substring from a string, given a starting position and length. ## ## s: The string to obtain a substring from. @@ -866,7 +853,7 @@ function str_split%(s: string, idx: index_vec%): string_vec for ( BroString::VecIt it = result->begin(); it != result->end(); ++it, ++i ) - result_v->Assign(i, new StringVal(*it), 0); + result_v->Assign(i, new StringVal(*it)); // StringVal now possesses string. delete result; @@ -1135,3 +1122,16 @@ function hexdump%(data_str: string%) : string return result; %} + +## Returns a reversed copy of the string +## +## str: The string to reverse. +## +## Returns: A reversed copy of *str* +## +function reverse%(str: string%) : string + %{ + string s = string((const char*)str->Bytes(), str->Len()); + reverse(s.begin(), s.end()); + return new StringVal(s.length(), (const char*)s.c_str()); + %} diff --git a/src/threading/SerialTypes.h b/src/threading/SerialTypes.h index 60aee2411e..f4f0bc0957 100644 --- a/src/threading/SerialTypes.h +++ b/src/threading/SerialTypes.h @@ -2,6 +2,7 @@ #ifndef THREADING_SERIALIZATIONTYPES_H #define THREADING_SERIALIZATIONTYPES_H +#include #include #include #include diff --git a/src/types.bif b/src/types.bif index 888310419c..420908b76b 100644 --- a/src/types.bif +++ b/src/types.bif @@ -196,6 +196,7 @@ enum Reader %{ READER_ASCII, READER_RAW, READER_BENCHMARK, + READER_BINARY, %} enum Event %{ @@ -211,3 +212,17 @@ enum Mode %{ %} module GLOBAL; + +type gtpv1_hdr: record; +type gtp_create_pdp_ctx_request_elements: record; +type gtp_create_pdp_ctx_response_elements: record; +type gtp_update_pdp_ctx_request_elements: record; +type gtp_update_pdp_ctx_response_elements: record; +type gtp_delete_pdp_ctx_request_elements: record; +type gtp_delete_pdp_ctx_response_elements: record; + +type gtp_end_user_addr: record; +type gtp_rai: record; +type gtp_qos_profile: record; +type gtp_private_extension: record; +type gtp_gsn_addr: record; diff --git a/src/util.cc b/src/util.cc index 31cc18c4e0..717c7f216c 100644 --- a/src/util.cc +++ b/src/util.cc @@ -1552,3 +1552,37 @@ void operator delete[](void* v) } #endif + +void bro_init_magic(magic_t* cookie_ptr, int flags) + { + if ( ! cookie_ptr || *cookie_ptr ) + return; + + *cookie_ptr = magic_open(flags); + + if ( ! *cookie_ptr ) + { + const char* err = magic_error(*cookie_ptr); + reporter->Error("can't init libmagic: %s", err ? err : "unknown"); + } + + else if ( magic_load(*cookie_ptr, 0) < 0 ) + { + const char* err = magic_error(*cookie_ptr); + reporter->Error("can't load magic file: %s", err ? err : "unknown"); + magic_close(*cookie_ptr); + *cookie_ptr = 0; + } + } + +const char* bro_magic_buffer(magic_t cookie, const void* buffer, size_t length) + { + const char* rval = magic_buffer(cookie, buffer, length); + if ( ! rval ) + { + const char* err = magic_error(cookie); + reporter->Error("magic_buffer error: %s", err ? err : "unknown"); + } + + return rval; + } diff --git a/src/util.h b/src/util.h index 6190f013e8..716fead054 100644 --- a/src/util.h +++ b/src/util.h @@ -15,6 +15,7 @@ #include #include #include +#include #include "config.h" #if __STDC__ @@ -75,7 +76,7 @@ typedef int32 ptr_compat_int; #define PRI_PTR_COMPAT_INT PRId32 #define PRI_PTR_COMPAT_UINT PRIu32 #else -# error "Unusual pointer size. Please report to bro@bro-ids.org." +# error "Unusual pointer size. Please report to bro@bro.org." #endif extern "C" @@ -368,4 +369,7 @@ struct CompareString } }; +void bro_init_magic(magic_t* cookie_ptr, int flags); +const char* bro_magic_buffer(magic_t cookie, const void* buffer, size_t length); + #endif diff --git a/testing/btest/Baseline/bifs.byte_len/out b/testing/btest/Baseline/bifs.byte_len/out deleted file mode 100644 index b4de394767..0000000000 --- a/testing/btest/Baseline/bifs.byte_len/out +++ /dev/null @@ -1 +0,0 @@ -11 diff --git a/testing/btest/Baseline/bifs.encode_base64/out b/testing/btest/Baseline/bifs.encode_base64/out new file mode 100644 index 0000000000..84c2c98264 --- /dev/null +++ b/testing/btest/Baseline/bifs.encode_base64/out @@ -0,0 +1,6 @@ +YnJv +YnJv +}n-v +cGFkZGluZw== +cGFkZGluZzE= +cGFkZGluZzEy diff --git a/testing/btest/Baseline/bifs.length/out b/testing/btest/Baseline/bifs.length/out deleted file mode 100644 index ad43182650..0000000000 --- a/testing/btest/Baseline/bifs.length/out +++ /dev/null @@ -1,6 +0,0 @@ -1 -4 -2 -0 -0 -0 diff --git a/testing/btest/Baseline/bifs.reverse/out b/testing/btest/Baseline/bifs.reverse/out new file mode 100644 index 0000000000..97b53df561 --- /dev/null +++ b/testing/btest/Baseline/bifs.reverse/out @@ -0,0 +1,7 @@ +!dlrow olleh +hello world! +risetovotesir +ff00 +00ff +3039 +A diff --git a/testing/btest/Baseline/core.q-in-q/conn.log b/testing/btest/Baseline/core.q-in-q/conn.log new file mode 100644 index 0000000000..00b1b9894f --- /dev/null +++ b/testing/btest/Baseline/core.q-in-q/conn.log @@ -0,0 +1,11 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path conn +#open 2013-03-22-16-36-54 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto service duration orig_bytes resp_bytes conn_state local_orig missed_bytes history orig_pkts orig_ip_bytes resp_pkts resp_ip_bytes tunnel_parents +#types time string addr port addr port enum string interval count count string bool count string count count count count table[string] +1363900699.548138 UWkUyAuUGXf 172.19.51.37 47808 172.19.51.63 47808 udp - 0.000100 36 0 S0 - 0 D 2 92 0 0 (empty) +1363900699.549647 arKYeMETxOg 193.1.186.60 9875 224.2.127.254 9875 udp - 0.000139 552 0 S0 - 0 D 2 608 0 0 (empty) +#close 2013-03-22-16-36-54 diff --git a/testing/btest/Baseline/core.tunnels.ayiya/http.log b/testing/btest/Baseline/core.tunnels.ayiya/http.log index 2a97fd9b69..902d0e0fb9 100644 --- a/testing/btest/Baseline/core.tunnels.ayiya/http.log +++ b/testing/btest/Baseline/core.tunnels.ayiya/http.log @@ -3,10 +3,10 @@ #empty_field (empty) #unset_field - #path http -#open 2009-11-08-04-41-41 +#open 2013-03-22-14-38-11 #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file -#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string file -1257655301.652206 5OKnoww6xl4 2001:4978:f:4c::2 53382 2001:4860:b002::68 80 1 GET ipv6.google.com / - Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.5; en; rv:1.9.0.15pre) Gecko/2009091516 Camino/2.0b4 (like Firefox/3.0.15pre) 0 10102 200 OK - - - (empty) - - - text/html - - +#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string string +1257655301.652206 5OKnoww6xl4 2001:4978:f:4c::2 53382 2001:4860:b002::68 80 1 GET ipv6.google.com / - Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.5; en; rv:1.9.0.15pre) Gecko/2009091516 Camino/2.0b4 (like Firefox/3.0.15pre) 0 10102 200 OK - - - (empty) - - - text/html; charset=us-ascii - - 1257655302.514424 5OKnoww6xl4 2001:4978:f:4c::2 53382 2001:4860:b002::68 80 2 GET ipv6.google.com /csi?v=3&s=webhp&action=&tran=undefined&e=17259,19771,21517,21766,21887,22212&ei=BUz2Su7PMJTglQfz3NzCAw&rt=prt.77,xjs.565,ol.645 http://ipv6.google.com/ Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.5; en; rv:1.9.0.15pre) Gecko/2009091516 Camino/2.0b4 (like Firefox/3.0.15pre) 0 0 204 No Content - - - (empty) - - - - - - 1257655303.603569 5OKnoww6xl4 2001:4978:f:4c::2 53382 2001:4860:b002::68 80 3 GET ipv6.google.com /gen_204?atyp=i&ct=fade&cad=1254&ei=BUz2Su7PMJTglQfz3NzCAw&zx=1257655303600 http://ipv6.google.com/ Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.5; en; rv:1.9.0.15pre) Gecko/2009091516 Camino/2.0b4 (like Firefox/3.0.15pre) 0 0 204 No Content - - - (empty) - - - - - - -#close 2009-11-08-04-41-57 +#close 2013-03-22-14-38-11 diff --git a/testing/btest/Baseline/core.tunnels.gtp.different_dl_and_ul/http.log b/testing/btest/Baseline/core.tunnels.gtp.different_dl_and_ul/http.log index 66b17e1200..02c5cf6e63 100644 --- a/testing/btest/Baseline/core.tunnels.gtp.different_dl_and_ul/http.log +++ b/testing/btest/Baseline/core.tunnels.gtp.different_dl_and_ul/http.log @@ -3,9 +3,9 @@ #empty_field (empty) #unset_field - #path http -#open 2012-10-19-17-03-55 +#open 2013-03-22-14-37-45 #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file -#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string file -1333458850.340368 arKYeMETxOg 10.131.17.170 51803 173.199.115.168 80 1 GET cdn.epicgameads.com /ads/flash/728x90_nx8com.swf?clickTAG=http://www.epicgameads.com/ads/bannerclickPage.php?id=e3ubwU6IF&pd=1&adid=0&icpc=1&axid=0&uctt=1&channel=4&cac=1&t=728x90&cb=1333458879 http://www.epicgameads.com/ads/banneriframe.php?id=e3ubwU6IF&t=728x90&channel=4&cb=1333458905296 Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0) 0 31461 200 OK - - - (empty) - - - application/x-shockwave-flash - - -1333458850.399501 arKYeMETxOg 10.131.17.170 51803 173.199.115.168 80 2 GET cdn.epicgameads.com /ads/flash/728x90_nx8com.swf?clickTAG=http://www.epicgameads.com/ads/bannerclickPage.php?id=e3ubwU6IF&pd=1&adid=0&icpc=1&axid=0&uctt=1&channel=0&cac=1&t=728x90&cb=1333458881 http://www.epicgameads.com/ads/banneriframe.php?id=e3ubwU6IF&t=728x90&cb=1333458920207 Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0) 0 31461 200 OK - - - (empty) - - - application/x-shockwave-flash - - -#close 2012-10-19-17-03-55 +#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string string +1333458850.340368 arKYeMETxOg 10.131.17.170 51803 173.199.115.168 80 1 GET cdn.epicgameads.com /ads/flash/728x90_nx8com.swf?clickTAG=http://www.epicgameads.com/ads/bannerclickPage.php?id=e3ubwU6IF&pd=1&adid=0&icpc=1&axid=0&uctt=1&channel=4&cac=1&t=728x90&cb=1333458879 http://www.epicgameads.com/ads/banneriframe.php?id=e3ubwU6IF&t=728x90&channel=4&cb=1333458905296 Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0) 0 31461 200 OK - - - (empty) - - - application/x-shockwave-flash; charset=binary - - +1333458850.399501 arKYeMETxOg 10.131.17.170 51803 173.199.115.168 80 2 GET cdn.epicgameads.com /ads/flash/728x90_nx8com.swf?clickTAG=http://www.epicgameads.com/ads/bannerclickPage.php?id=e3ubwU6IF&pd=1&adid=0&icpc=1&axid=0&uctt=1&channel=0&cac=1&t=728x90&cb=1333458881 http://www.epicgameads.com/ads/banneriframe.php?id=e3ubwU6IF&t=728x90&cb=1333458920207 Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0) 0 31461 200 OK - - - (empty) - - - application/x-shockwave-flash; charset=binary - - +#close 2013-03-22-14-37-45 diff --git a/testing/btest/Baseline/core.tunnels.gtp.ext_header/out b/testing/btest/Baseline/core.tunnels.gtp.ext_header/out new file mode 100644 index 0000000000..e76a7f35de --- /dev/null +++ b/testing/btest/Baseline/core.tunnels.gtp.ext_header/out @@ -0,0 +1,2 @@ +gtpv1_message, [orig_h=10.155.148.149, orig_p=9000/udp, resp_h=10.155.148.157, resp_p=2152/udp] +[version=1, pt_flag=T, rsv=F, e_flag=T, s_flag=T, pn_flag=F, msg_type=255, length=1508, teid=1050199, seq=5, n_pdu=0, next_type=192] diff --git a/testing/btest/Baseline/core.tunnels.gtp.non_recursive/out b/testing/btest/Baseline/core.tunnels.gtp.non_recursive/out index a299c4d592..49bb5f7399 100644 --- a/testing/btest/Baseline/core.tunnels.gtp.non_recursive/out +++ b/testing/btest/Baseline/core.tunnels.gtp.non_recursive/out @@ -1 +1 @@ -protocol_violation, [orig_h=74.125.216.149, orig_p=2152/udp, resp_h=10.131.138.69, resp_p=2152/udp], GTP-in-GTP [n\xd9'|\x00\x00\x01\xb6[\xf6\xdc0\xb7d\xe5\xe6\xa76\x91\xfbk\x0e\x02\xc8A\x05\xa8\xe6\xf3Gi\x80(]\xcew\x84\xae}\xd2...] +protocol_violation, [orig_h=74.125.216.149, orig_p=2152/udp, resp_h=10.131.138.69, resp_p=2152/udp], GTP-in-GTP [\x80\xe1Bc.\xe20\xebn\xd9'|\x00\x00\x01\xb6[\xf6\xdc0\xb7d\xe5\xe6\xa76\x91\xfbk\x0e\x02\xc8A\x05\xa8\xe6\xf3Gi\x80...] diff --git a/testing/btest/Baseline/core.tunnels.gtp.outer_ip_frag/http.log b/testing/btest/Baseline/core.tunnels.gtp.outer_ip_frag/http.log index 8a994d56af..cbe52f5252 100644 --- a/testing/btest/Baseline/core.tunnels.gtp.outer_ip_frag/http.log +++ b/testing/btest/Baseline/core.tunnels.gtp.outer_ip_frag/http.log @@ -3,8 +3,8 @@ #empty_field (empty) #unset_field - #path http -#open 2012-10-19-16-44-02 +#open 2013-03-22-14-37-46 #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file -#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string file -1333458850.375568 arKYeMETxOg 10.131.47.185 1923 79.101.110.141 80 1 GET o-o.preferred.telekomrs-beg1.v2.lscache8.c.youtube.com /videoplayback?upn=MTU2MDY5NzQ5OTM0NTI3NDY4NDc&sparams=algorithm,burst,cp,factor,id,ip,ipbits,itag,source,upn,expire&fexp=912300,907210&algorithm=throttle-factor&itag=34&ip=212.0.0.0&burst=40&sver=3&signature=832FB1042E20780CFCA77A4DB5EA64AC593E8627.D1166C7E8365732E52DAFD68076DAE0146E0AE01&source=youtube&expire=1333484980&key=yt1&ipbits=8&factor=1.25&cp=U0hSSFRTUl9NSkNOMl9MTVZKOjh5eEN2SG8tZF84&id=ebf1e932d4bd1286&cm2=1 http://s.ytimg.com/yt/swfbin/watch_as3-vflqrJwOA.swf Mozilla/5.0 (Windows NT 5.1) AppleWebKit/535.11 (KHTML, like Gecko; X-SBLSP) Chrome/17.0.963.83 Safari/535.11 0 56320 206 Partial Content - - - (empty) - - - application/octet-stream - - -#close 2012-10-19-16-44-02 +#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string string +1333458850.375568 arKYeMETxOg 10.131.47.185 1923 79.101.110.141 80 1 GET o-o.preferred.telekomrs-beg1.v2.lscache8.c.youtube.com /videoplayback?upn=MTU2MDY5NzQ5OTM0NTI3NDY4NDc&sparams=algorithm,burst,cp,factor,id,ip,ipbits,itag,source,upn,expire&fexp=912300,907210&algorithm=throttle-factor&itag=34&ip=212.0.0.0&burst=40&sver=3&signature=832FB1042E20780CFCA77A4DB5EA64AC593E8627.D1166C7E8365732E52DAFD68076DAE0146E0AE01&source=youtube&expire=1333484980&key=yt1&ipbits=8&factor=1.25&cp=U0hSSFRTUl9NSkNOMl9MTVZKOjh5eEN2SG8tZF84&id=ebf1e932d4bd1286&cm2=1 http://s.ytimg.com/yt/swfbin/watch_as3-vflqrJwOA.swf Mozilla/5.0 (Windows NT 5.1) AppleWebKit/535.11 (KHTML, like Gecko; X-SBLSP) Chrome/17.0.963.83 Safari/535.11 0 56320 206 Partial Content - - - (empty) - - - - - - +#close 2013-03-22-14-37-46 diff --git a/testing/btest/Baseline/core.tunnels.gtp.pdp_ctx_messages/out b/testing/btest/Baseline/core.tunnels.gtp.pdp_ctx_messages/out new file mode 100644 index 0000000000..fcdfe94824 --- /dev/null +++ b/testing/btest/Baseline/core.tunnels.gtp.pdp_ctx_messages/out @@ -0,0 +1,24 @@ +gtpv1_message, [orig_h=192.169.100.1, orig_p=34273/udp, resp_h=10.100.200.33, resp_p=2123/udp] +[version=1, pt_flag=T, rsv=F, e_flag=F, s_flag=T, pn_flag=F, msg_type=16, length=137, teid=0, seq=4875, n_pdu=0, next_type=0] +gtp create request, [orig_h=192.169.100.1, orig_p=34273/udp, resp_h=10.100.200.33, resp_p=2123/udp] +[version=1, pt_flag=T, rsv=F, e_flag=F, s_flag=T, pn_flag=F, msg_type=16, length=137, teid=0, seq=4875, n_pdu=0, next_type=0] +[imsi=460004100000101, rai=[mcc=460, mnc=6, lac=65534, rac=255], recovery=176, select_mode=1, data1=854600697, cp=854600697, nsapi=5, linked_nsapi=, charge_character=, trace_ref=, trace_type=, end_user_addr=[pdp_type_org=1, pdp_type_num=33, pdp_ip=, pdp_other_addr=], ap_name=^Feetest, opts=\x80\x80!^V^A^A\0^V^C^F\0\0\0\0\x81^F\0\0\0\0\x83^F\0\0\0\0, signal_addr=[ip=192.169.100.1, other=], user_addr=[ip=192.169.100.1, other=], msisdn=\x91hQ"^A\0^A\xf1, qos_prof=[priority=2, data=\x1bB\x1fs\x8c@@tK@@], tft=, trigger_id=, omc_id=, ext=[id=10923, value=^B^A^C]] +gtpv1_message, [orig_h=192.169.100.1, orig_p=34273/udp, resp_h=10.100.200.33, resp_p=2123/udp] +[version=1, pt_flag=T, rsv=F, e_flag=F, s_flag=T, pn_flag=F, msg_type=17, length=101, teid=854600697, seq=4875, n_pdu=0, next_type=0] +gtp create response, [orig_h=192.169.100.1, orig_p=34273/udp, resp_h=10.100.200.33, resp_p=2123/udp] +[version=1, pt_flag=T, rsv=F, e_flag=F, s_flag=T, pn_flag=F, msg_type=17, length=101, teid=854600697, seq=4875, n_pdu=0, next_type=0] +[cause=128, reorder_req=F, recovery=24, data1=268435589, cp=268435584, charging_id=103000009, end_user_addr=[pdp_type_org=1, pdp_type_num=33, pdp_ip=192.168.252.130, pdp_other_addr=], opts=\x80\x80!^P^D^A\0^P\x81^F\0\0\0\0\x83^F\0\0\0\0\x80!^J^C^A\0^J^C^F\xc0\xa8\xfc\x82, cp_addr=[ip=10.100.200.34, other=], user_addr=[ip=10.100.200.49, other=], qos_prof=[priority=2, data=\x1bB\x1fs\x8c@@tK@@], charge_gateway=, ext=] +gtpv1_message, [orig_h=127.0.0.2, orig_p=2123/udp, resp_h=127.0.0.1, resp_p=2123/udp] +[version=1, pt_flag=T, rsv=F, e_flag=F, s_flag=T, pn_flag=F, msg_type=1, length=4, teid=0, seq=3072, n_pdu=0, next_type=0] +gtpv1_message, [orig_h=127.0.0.2, orig_p=2123/udp, resp_h=127.0.0.1, resp_p=2123/udp] +[version=1, pt_flag=T, rsv=F, e_flag=F, s_flag=T, pn_flag=F, msg_type=2, length=6, teid=0, seq=3072, n_pdu=0, next_type=0] +gtpv1_message, [orig_h=127.0.0.2, orig_p=2123/udp, resp_h=127.0.0.1, resp_p=2123/udp] +[version=1, pt_flag=T, rsv=F, e_flag=F, s_flag=T, pn_flag=F, msg_type=16, length=104, teid=0, seq=3073, n_pdu=0, next_type=0] +gtp create request, [orig_h=127.0.0.2, orig_p=2123/udp, resp_h=127.0.0.1, resp_p=2123/udp] +[version=1, pt_flag=T, rsv=F, e_flag=F, s_flag=T, pn_flag=F, msg_type=16, length=104, teid=0, seq=3073, n_pdu=0, next_type=0] +[imsi=240010123456789, rai=, recovery=3, select_mode=1, data1=1, cp=1, nsapi=0, linked_nsapi=, charge_character=2048, trace_ref=, trace_type=, end_user_addr=[pdp_type_org=1, pdp_type_num=33, pdp_ip=, pdp_other_addr=], ap_name=^Hinternet, opts=\x80\xc0#^Q^A^A\0^Q^Cmig^Hhemmelig, signal_addr=[ip=127.0.0.2, other=], user_addr=[ip=127.0.0.2, other=], msisdn=\x91d^G^R2T\xf6, qos_prof=[priority=0, data=^K\x92\x1f], tft=, trigger_id=, omc_id=, ext=] +gtpv1_message, [orig_h=127.0.0.2, orig_p=2123/udp, resp_h=127.0.0.1, resp_p=2123/udp] +[version=1, pt_flag=T, rsv=F, e_flag=F, s_flag=T, pn_flag=F, msg_type=17, length=78, teid=1, seq=3073, n_pdu=0, next_type=0] +gtp create response, [orig_h=127.0.0.2, orig_p=2123/udp, resp_h=127.0.0.1, resp_p=2123/udp] +[version=1, pt_flag=T, rsv=F, e_flag=F, s_flag=T, pn_flag=F, msg_type=17, length=78, teid=1, seq=3073, n_pdu=0, next_type=0] +[cause=128, reorder_req=F, recovery=1, data1=1, cp=1, charging_id=1, end_user_addr=[pdp_type_org=1, pdp_type_num=33, pdp_ip=192.168.0.2, pdp_other_addr=], opts=\x80\x80!^P^B\0\0^P\x81^F\0\0\0\0\x83^F\0\0\0\0, cp_addr=[ip=127.0.0.1, other=], user_addr=[ip=127.0.0.1, other=], qos_prof=[priority=0, data=^K\x92\x1f], charge_gateway=, ext=] diff --git a/testing/btest/Baseline/core.tunnels.gtp.unknown_or_too_short/dpd.log b/testing/btest/Baseline/core.tunnels.gtp.unknown_or_too_short/dpd.log index 221fa16f4f..fcd110f8ab 100644 --- a/testing/btest/Baseline/core.tunnels.gtp.unknown_or_too_short/dpd.log +++ b/testing/btest/Baseline/core.tunnels.gtp.unknown_or_too_short/dpd.log @@ -3,8 +3,8 @@ #empty_field (empty) #unset_field - #path dpd -#open 2012-10-19-17-38-54 +#open 2013-01-25-21-49-19 #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto analyzer failure_reason #types time string addr port addr port enum string string -1333458853.075889 UWkUyAuUGXf 173.86.159.28 2152 213.72.147.186 2152 udp GTPV1 Truncated GTPv1 [E\x00\x05\xc8G\xea@\x00\x80\x06\xb6\x83\x0a\x83w&\xd9\x14\x9c\x04\xd9\xc2\x00P\xddh\xb4\x8f41eVP\x10\x10\xe0u\xcf\x00\x00...] -#close 2012-10-19-17-38-54 +1333458853.075889 UWkUyAuUGXf 173.86.159.28 2152 213.72.147.186 2152 udp GTPV1 Truncated GTPv1 [0\xff\x00\xac\x98\x13\x01LE\x00\x05\xc8G\xea@\x00\x80\x06\xb6\x83\x0a\x83w&\xd9\x14\x9c\x04\xd9\xc2\x00P\xddh\xb4\x8f41eV...] +#close 2013-01-25-21-49-19 diff --git a/testing/btest/Baseline/core.tunnels.gtp.unknown_or_too_short/tunnel.log b/testing/btest/Baseline/core.tunnels.gtp.unknown_or_too_short/tunnel.log index 659090a581..070a754702 100644 --- a/testing/btest/Baseline/core.tunnels.gtp.unknown_or_too_short/tunnel.log +++ b/testing/btest/Baseline/core.tunnels.gtp.unknown_or_too_short/tunnel.log @@ -3,9 +3,9 @@ #empty_field (empty) #unset_field - #path tunnel -#open 2012-10-19-17-38-54 +#open 2013-01-25-21-49-19 #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p tunnel_type action #types time string addr port addr port enum enum 1333458853.034734 UWkUyAuUGXf 173.86.159.28 2152 213.72.147.186 2152 Tunnel::GTPv1 Tunnel::DISCOVER 1333458853.108391 UWkUyAuUGXf 173.86.159.28 2152 213.72.147.186 2152 Tunnel::GTPv1 Tunnel::CLOSE -#close 2012-10-19-17-38-54 +#close 2013-01-25-21-49-19 diff --git a/testing/btest/Baseline/core.tunnels.teredo/http.log b/testing/btest/Baseline/core.tunnels.teredo/http.log index c77297c58d..fe015a130d 100644 --- a/testing/btest/Baseline/core.tunnels.teredo/http.log +++ b/testing/btest/Baseline/core.tunnels.teredo/http.log @@ -3,11 +3,11 @@ #empty_field (empty) #unset_field - #path http -#open 2008-05-16-15-50-58 +#open 2013-03-22-14-37-44 #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file -#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string file -1210953057.917183 3PKsZ2Uye21 192.168.2.16 1578 75.126.203.78 80 1 POST download913.avast.com /cgi-bin/iavs4stats.cgi - Syncer/4.80 (av_pro-1169;f) 589 0 204 - - - (empty) - - - text/plain - - -1210953061.585996 70MGiRM1Qf4 2001:0:4137:9e50:8000:f12a:b9c8:2815 1286 2001:4860:0:2001::68 80 1 GET ipv6.google.com / - Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9b5) Gecko/2008032620 Firefox/3.0b5 0 6640 200 OK - - - (empty) - - - text/html - - -1210953073.381474 70MGiRM1Qf4 2001:0:4137:9e50:8000:f12a:b9c8:2815 1286 2001:4860:0:2001::68 80 2 GET ipv6.google.com /search?hl=en&q=Wireshark+!&btnG=Google+Search http://ipv6.google.com/ Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9b5) Gecko/2008032620 Firefox/3.0b5 0 25119 200 OK - - - (empty) - - - text/html - - -1210953074.674817 c4Zw9TmAE05 192.168.2.16 1580 67.228.110.120 80 1 GET www.wireshark.org / http://ipv6.google.com/search?hl=en&q=Wireshark+%21&btnG=Google+Search Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9b5) Gecko/2008032620 Firefox/3.0b5 0 11845 200 OK - - - (empty) - - - text/xml - - -#close 2008-05-16-15-51-16 +#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string string +1210953057.917183 3PKsZ2Uye21 192.168.2.16 1578 75.126.203.78 80 1 POST download913.avast.com /cgi-bin/iavs4stats.cgi - Syncer/4.80 (av_pro-1169;f) 589 0 204 - - - (empty) - - - text/plain; charset=us-ascii - - +1210953061.585996 70MGiRM1Qf4 2001:0:4137:9e50:8000:f12a:b9c8:2815 1286 2001:4860:0:2001::68 80 1 GET ipv6.google.com / - Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9b5) Gecko/2008032620 Firefox/3.0b5 0 6640 200 OK - - - (empty) - - - text/html; charset=us-ascii - - +1210953073.381474 70MGiRM1Qf4 2001:0:4137:9e50:8000:f12a:b9c8:2815 1286 2001:4860:0:2001::68 80 2 GET ipv6.google.com /search?hl=en&q=Wireshark+!&btnG=Google+Search http://ipv6.google.com/ Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9b5) Gecko/2008032620 Firefox/3.0b5 0 25119 200 OK - - - (empty) - - - text/html; charset=us-ascii - - +1210953074.674817 c4Zw9TmAE05 192.168.2.16 1580 67.228.110.120 80 1 GET www.wireshark.org / http://ipv6.google.com/search?hl=en&q=Wireshark+%21&btnG=Google+Search Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9b5) Gecko/2008032620 Firefox/3.0b5 0 11845 200 OK - - - (empty) - - - application/xml; charset=us-ascii - - +#close 2013-03-22-14-37-44 diff --git a/testing/btest/Baseline/core.tunnels.teredo_bubble_with_payload/http.log b/testing/btest/Baseline/core.tunnels.teredo_bubble_with_payload/http.log index e0b223d114..6d6f00a151 100644 --- a/testing/btest/Baseline/core.tunnels.teredo_bubble_with_payload/http.log +++ b/testing/btest/Baseline/core.tunnels.teredo_bubble_with_payload/http.log @@ -3,9 +3,9 @@ #empty_field (empty) #unset_field - #path http -#open 2012-06-19-17-39-37 +#open 2013-03-22-14-37-44 #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file -#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string file -1340127577.361683 FrJExwHcSal 2001:0:4137:9e50:8000:f12a:b9c8:2815 1286 2001:4860:0:2001::68 80 1 GET ipv6.google.com / - Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9b5) Gecko/2008032620 Firefox/3.0b5 0 6640 200 OK - - - (empty) - - - text/html - - -1340127577.379360 FrJExwHcSal 2001:0:4137:9e50:8000:f12a:b9c8:2815 1286 2001:4860:0:2001::68 80 2 GET ipv6.google.com /search?hl=en&q=Wireshark+!&btnG=Google+Search http://ipv6.google.com/ Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9b5) Gecko/2008032620 Firefox/3.0b5 0 25119 200 OK - - - (empty) - - - text/html - - -#close 2012-06-19-17-39-37 +#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string string +1340127577.361683 FrJExwHcSal 2001:0:4137:9e50:8000:f12a:b9c8:2815 1286 2001:4860:0:2001::68 80 1 GET ipv6.google.com / - Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9b5) Gecko/2008032620 Firefox/3.0b5 0 6640 200 OK - - - (empty) - - - text/html; charset=us-ascii - - +1340127577.379360 FrJExwHcSal 2001:0:4137:9e50:8000:f12a:b9c8:2815 1286 2001:4860:0:2001::68 80 2 GET ipv6.google.com /search?hl=en&q=Wireshark+!&btnG=Google+Search http://ipv6.google.com/ Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9b5) Gecko/2008032620 Firefox/3.0b5 0 25119 200 OK - - - (empty) - - - text/html; charset=us-ascii - - +#close 2013-03-22-14-37-44 diff --git a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log index 018480850c..0c9f74db81 100644 --- a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log @@ -29,6 +29,7 @@ scripts/base/init-bare.bro scripts/base/frameworks/input/./readers/ascii.bro scripts/base/frameworks/input/./readers/raw.bro scripts/base/frameworks/input/./readers/benchmark.bro + scripts/base/frameworks/input/./readers/binary.bro scripts/base/frameworks/file-analysis/__load__.bro scripts/base/frameworks/file-analysis/./main.bro build/src/base/file_analysis.bif.bro diff --git a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log index e33af537bb..a34ae7faa3 100644 --- a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log @@ -3,7 +3,7 @@ #empty_field (empty) #unset_field - #path loaded_scripts -#open 2013-01-16-18-05-21 +#open 2013-02-11-18-44-43 #fields name #types string scripts/base/init-bare.bro @@ -29,6 +29,7 @@ scripts/base/init-bare.bro scripts/base/frameworks/input/./readers/ascii.bro scripts/base/frameworks/input/./readers/raw.bro scripts/base/frameworks/input/./readers/benchmark.bro + scripts/base/frameworks/input/./readers/binary.bro scripts/base/frameworks/file-analysis/__load__.bro scripts/base/frameworks/file-analysis/./main.bro build/src/base/file_analysis.bif.bro @@ -56,6 +57,7 @@ scripts/base/init-default.bro scripts/base/frameworks/cluster/./main.bro scripts/base/frameworks/control/__load__.bro scripts/base/frameworks/control/./main.bro + scripts/base/frameworks/notice/./non-cluster.bro scripts/base/frameworks/notice/./actions/pp-alarms.bro scripts/base/frameworks/dpd/__load__.bro scripts/base/frameworks/dpd/./main.bro @@ -89,6 +91,7 @@ scripts/base/init-default.bro scripts/base/protocols/ftp/__load__.bro scripts/base/protocols/ftp/./utils-commands.bro scripts/base/protocols/ftp/./main.bro + scripts/base/protocols/ftp/./file-analysis.bro scripts/base/protocols/ftp/./file-extract.bro scripts/base/protocols/ftp/./gridftp.bro scripts/base/protocols/ssl/__load__.bro @@ -98,12 +101,14 @@ scripts/base/init-default.bro scripts/base/protocols/http/__load__.bro scripts/base/protocols/http/./main.bro scripts/base/protocols/http/./utils.bro + scripts/base/protocols/http/./file-analysis.bro scripts/base/protocols/http/./file-ident.bro scripts/base/protocols/http/./file-hash.bro scripts/base/protocols/http/./file-extract.bro scripts/base/protocols/irc/__load__.bro scripts/base/protocols/irc/./main.bro scripts/base/protocols/irc/./dcc-send.bro + scripts/base/protocols/irc/./file-analysis.bro scripts/base/protocols/modbus/__load__.bro scripts/base/protocols/modbus/./consts.bro scripts/base/protocols/modbus/./main.bro @@ -111,6 +116,7 @@ scripts/base/init-default.bro scripts/base/protocols/smtp/./main.bro scripts/base/protocols/smtp/./entities.bro scripts/base/protocols/smtp/./entities-excerpt.bro + scripts/base/protocols/smtp/./file-analysis.bro scripts/base/protocols/socks/__load__.bro scripts/base/protocols/socks/./consts.bro scripts/base/protocols/socks/./main.bro @@ -121,4 +127,4 @@ scripts/base/init-default.bro scripts/base/protocols/syslog/./main.bro scripts/base/misc/find-checksum-offloading.bro scripts/policy/misc/loaded-scripts.bro -#close 2013-01-16-18-05-21 +#close 2013-02-11-18-44-43 diff --git a/testing/btest/Baseline/istate.events-ssl/receiver.http.log b/testing/btest/Baseline/istate.events-ssl/receiver.http.log index 3fc7f1b66f..aa69373171 100644 --- a/testing/btest/Baseline/istate.events-ssl/receiver.http.log +++ b/testing/btest/Baseline/istate.events-ssl/receiver.http.log @@ -3,8 +3,8 @@ #empty_field (empty) #unset_field - #path http -#open 2012-07-20-01-53-03 +#open 2013-03-22-21-05-55 #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file -#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string file -1342749182.906082 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - (empty) - - - text/html - - -#close 2012-07-20-01-53-04 +#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string string +1363986354.505533 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - (empty) - - - - - - +#close 2013-03-22-21-05-56 diff --git a/testing/btest/Baseline/istate.events-ssl/sender.http.log b/testing/btest/Baseline/istate.events-ssl/sender.http.log index 3fc7f1b66f..aa69373171 100644 --- a/testing/btest/Baseline/istate.events-ssl/sender.http.log +++ b/testing/btest/Baseline/istate.events-ssl/sender.http.log @@ -3,8 +3,8 @@ #empty_field (empty) #unset_field - #path http -#open 2012-07-20-01-53-03 +#open 2013-03-22-21-05-55 #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file -#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string file -1342749182.906082 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - (empty) - - - text/html - - -#close 2012-07-20-01-53-04 +#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string string +1363986354.505533 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - (empty) - - - - - - +#close 2013-03-22-21-05-56 diff --git a/testing/btest/Baseline/istate.events/receiver.http.log b/testing/btest/Baseline/istate.events/receiver.http.log index 6862c08b98..2531eb4bc0 100644 --- a/testing/btest/Baseline/istate.events/receiver.http.log +++ b/testing/btest/Baseline/istate.events/receiver.http.log @@ -3,8 +3,8 @@ #empty_field (empty) #unset_field - #path http -#open 2012-07-20-01-53-12 +#open 2013-03-22-21-03-17 #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file -#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string file -1342749191.765740 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - (empty) - - - text/html - - -#close 2012-07-20-01-53-13 +#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string string +1363986197.076696 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - (empty) - - - - - - +#close 2013-03-22-21-03-18 diff --git a/testing/btest/Baseline/istate.events/sender.http.log b/testing/btest/Baseline/istate.events/sender.http.log index 6862c08b98..2531eb4bc0 100644 --- a/testing/btest/Baseline/istate.events/sender.http.log +++ b/testing/btest/Baseline/istate.events/sender.http.log @@ -3,8 +3,8 @@ #empty_field (empty) #unset_field - #path http -#open 2012-07-20-01-53-12 +#open 2013-03-22-21-03-17 #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file -#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string file -1342749191.765740 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - (empty) - - - text/html - - -#close 2012-07-20-01-53-13 +#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string string +1363986197.076696 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - (empty) - - - - - - +#close 2013-03-22-21-03-18 diff --git a/testing/btest/Baseline/language.container-ctor-scope/out b/testing/btest/Baseline/language.container-ctor-scope/out new file mode 100644 index 0000000000..4f9acfce86 --- /dev/null +++ b/testing/btest/Baseline/language.container-ctor-scope/out @@ -0,0 +1,44 @@ +{ +[2/tcp] = 2, +[1/tcp] = 1, +[3/tcp] = 3 +} +{ +[2/tcp] = 2, +[1/tcp] = 1, +[3/tcp] = 3 +} +{ +2/tcp, +1/tcp, +3/tcp +} +{ +2/tcp, +1/tcp, +3/tcp +} +[1/tcp, 2/tcp, 3/tcp, 1/tcp] +[1/tcp, 2/tcp, 3/tcp, 1/tcp] +{ +[2/tcp] = 2, +[1/tcp] = 1, +[3/tcp] = 3 +} +{ +[2/tcp] = 2, +[1/tcp] = 1, +[3/tcp] = 3 +} +{ +2/tcp, +1/tcp, +3/tcp +} +{ +2/tcp, +1/tcp, +3/tcp +} +[1/tcp, 2/tcp, 3/tcp, 1/tcp] +[1/tcp, 2/tcp, 3/tcp, 1/tcp] diff --git a/testing/btest/Baseline/language.func-assignment/out b/testing/btest/Baseline/language.func-assignment/out new file mode 100644 index 0000000000..b569f7ee55 --- /dev/null +++ b/testing/btest/Baseline/language.func-assignment/out @@ -0,0 +1,4 @@ +Brogrammers, like bowties, are cool. Brogrammers, like bowties, are cool. Brogrammers, like bowties, are cool. +Brogrammers, like bowties, are cool. Brogrammers, like bowties, are cool. +BROGRAMMERS, LIKE BOWTIES, ARE COOL. +BROGRAMMERS, LIKE BOWTIES, ARE COOL. diff --git a/testing/btest/Baseline/language.returnwhen/bro..stdout b/testing/btest/Baseline/language.returnwhen/bro..stdout new file mode 100644 index 0000000000..d213d7bd02 --- /dev/null +++ b/testing/btest/Baseline/language.returnwhen/bro..stdout @@ -0,0 +1,12 @@ +dummy from async_func() from bro_init() +async_func() return result in bro_init(), flag in my_set +dummy from bro_init() when block +hi! +dummy from async_func() from do_another() +async_func() return result in do_another(), flag in my_set +dummy from do_another() when block +hi! +dummy from async_func() from do_another() +async_func() return result in do_another(), timeout +dummy from do_another() when block +hi! diff --git a/testing/btest/Baseline/language.table-init-record-idx-2/output b/testing/btest/Baseline/language.table-init-record-idx-2/output new file mode 100644 index 0000000000..1496863177 --- /dev/null +++ b/testing/btest/Baseline/language.table-init-record-idx-2/output @@ -0,0 +1,25 @@ +following should all be true... +T +T +T +T +T +T +T +T +1 +1 +1 +following should all be false... +F +F +F +now here's the foo table... +{ +[[a=foo, b=2], 2] = 2, +[[a=baz, b=6], 6] = 6, +[[a=bar, b=4], 4] = 4, +[[a=baz, b=5], 5] = 5, +[[a=bar, b=3], 3] = 3, +[[a=foo, b=1], 1] = 1 +} diff --git a/testing/btest/Baseline/language.table-init-record-idx-3/output b/testing/btest/Baseline/language.table-init-record-idx-3/output new file mode 100644 index 0000000000..b8d1540d0a --- /dev/null +++ b/testing/btest/Baseline/language.table-init-record-idx-3/output @@ -0,0 +1,25 @@ +following should all be true... +T +T +T +T +T +T +T +T +1 +1 +1 +following should all be false... +F +F +F +now here's the foo table... +{ +[[a=baz, b=5]] = 5, +[[a=foo, b=2]] = 2, +[[a=baz, b=6]] = 6, +[[a=foo, b=1]] = 1, +[[a=bar, b=4]] = 4, +[[a=bar, b=3]] = 3 +} diff --git a/testing/btest/Baseline/language.table-init-record-idx-4/language.table-init-record-idx-2/output b/testing/btest/Baseline/language.table-init-record-idx-4/language.table-init-record-idx-2/output new file mode 100644 index 0000000000..1496863177 --- /dev/null +++ b/testing/btest/Baseline/language.table-init-record-idx-4/language.table-init-record-idx-2/output @@ -0,0 +1,25 @@ +following should all be true... +T +T +T +T +T +T +T +T +1 +1 +1 +following should all be false... +F +F +F +now here's the foo table... +{ +[[a=foo, b=2], 2] = 2, +[[a=baz, b=6], 6] = 6, +[[a=bar, b=4], 4] = 4, +[[a=baz, b=5], 5] = 5, +[[a=bar, b=3], 3] = 3, +[[a=foo, b=1], 1] = 1 +} diff --git a/testing/btest/Baseline/language.table-init-record-idx-4/output b/testing/btest/Baseline/language.table-init-record-idx-4/output new file mode 100644 index 0000000000..1496863177 --- /dev/null +++ b/testing/btest/Baseline/language.table-init-record-idx-4/output @@ -0,0 +1,25 @@ +following should all be true... +T +T +T +T +T +T +T +T +1 +1 +1 +following should all be false... +F +F +F +now here's the foo table... +{ +[[a=foo, b=2], 2] = 2, +[[a=baz, b=6], 6] = 6, +[[a=bar, b=4], 4] = 4, +[[a=baz, b=5], 5] = 5, +[[a=bar, b=3], 3] = 3, +[[a=foo, b=1], 1] = 1 +} diff --git a/testing/btest/Baseline/language.table-init-record-idx/output b/testing/btest/Baseline/language.table-init-record-idx/output new file mode 100644 index 0000000000..b8d1540d0a --- /dev/null +++ b/testing/btest/Baseline/language.table-init-record-idx/output @@ -0,0 +1,25 @@ +following should all be true... +T +T +T +T +T +T +T +T +1 +1 +1 +following should all be false... +F +F +F +now here's the foo table... +{ +[[a=baz, b=5]] = 5, +[[a=foo, b=2]] = 2, +[[a=baz, b=6]] = 6, +[[a=foo, b=1]] = 1, +[[a=bar, b=4]] = 4, +[[a=bar, b=3]] = 3 +} diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.actions.data_event/out b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.actions.data_event/out new file mode 100644 index 0000000000..2f26373b88 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.actions.data_event/out @@ -0,0 +1,28 @@ +FileAnalysis::TRIGGER_NEW +Cx92a0ym5R8, 0, 0 +FileAnalysis::TRIGGER_BOF +FileAnalysis::TRIGGER_BOF_BUFFER +^J0.26 | 201 +FileAnalysis::TRIGGER_TYPE +file type is set +mime type is set +file_stream, Cx92a0ym5R8, 1500, ^J0.26 | 2012-08-24 15:10:04 -0700^J^J * Fixing update-changes, which could pick the wrong control file. (Robin Sommer)^J^J * Fixing GPG signing script. (Robin Sommer)^J^J0.25 | 2012-08-01 13:55:46 -0500^J^J * Fix configure script to exit with non-zero status on error (Jon Siwek)^J^J0.24 | 2012-07-05 12:50:43 -0700^J^J * Raise minimum required CMake version to 2.6.3 (Jon Siwek)^J^J * Adding script to delete old fully-merged branches. (Robin Sommer)^J^J0.23-2 | 2012-01-25 13:24:01 -0800^J^J * Fix a bro-cut error message. (Daniel Thayer)^J^J0.23 | 2012-01-11 12:16:11 -0800^J^J * Tweaks to release scripts, plus a new one for signing files.^J (Robin Sommer)^J^J0.22 | 2012-01-10 16:45:19 -0800^J^J * Tweaks for OpenBSD support. (Jon Siwek)^J^J * bro-cut extensions and fixes. (Robin Sommer)^J ^J - If no field names are given on the command line, we now pass through^J all fields. Adresses #657.^J^J - Removing some GNUism from awk script. Addresses #653.^J^J - Added option for time output in UTC. Addresses #668.^J^J - Added output field separator option -F. Addresses #649.^J^J - Fixing option -c: only some header lines were passed through^J rather than all. (Robin Sommer)^J^J * Fix parallel make portability. (Jon Siwek)^J^J0.21-9 | 2011-11-07 05:44:14 -0800^J^J * Fixing compiler warnings. Addresses #388. (Jon Siwek)^J^J0.21-2 | 2011-11-02 18:12:13 -0700^J^J * Fix for misnaming temp file in update-changes script. (Robin Sommer)^J^J0.21-1 | 2011-11-02 18:10:39 -0700^J^J * Little fix for make-relea +file_chunk, Cx92a0ym5R8, 1500, 0, ^J0.26 | 2012-08-24 15:10:04 -0700^J^J * Fixing update-changes, which could pick the wrong control file. (Robin Sommer)^J^J * Fixing GPG signing script. (Robin Sommer)^J^J0.25 | 2012-08-01 13:55:46 -0500^J^J * Fix configure script to exit with non-zero status on error (Jon Siwek)^J^J0.24 | 2012-07-05 12:50:43 -0700^J^J * Raise minimum required CMake version to 2.6.3 (Jon Siwek)^J^J * Adding script to delete old fully-merged branches. (Robin Sommer)^J^J0.23-2 | 2012-01-25 13:24:01 -0800^J^J * Fix a bro-cut error message. (Daniel Thayer)^J^J0.23 | 2012-01-11 12:16:11 -0800^J^J * Tweaks to release scripts, plus a new one for signing files.^J (Robin Sommer)^J^J0.22 | 2012-01-10 16:45:19 -0800^J^J * Tweaks for OpenBSD support. (Jon Siwek)^J^J * bro-cut extensions and fixes. (Robin Sommer)^J ^J - If no field names are given on the command line, we now pass through^J all fields. Adresses #657.^J^J - Removing some GNUism from awk script. Addresses #653.^J^J - Added option for time output in UTC. Addresses #668.^J^J - Added output field separator option -F. Addresses #649.^J^J - Fixing option -c: only some header lines were passed through^J rather than all. (Robin Sommer)^J^J * Fix parallel make portability. (Jon Siwek)^J^J0.21-9 | 2011-11-07 05:44:14 -0800^J^J * Fixing compiler warnings. Addresses #388. (Jon Siwek)^J^J0.21-2 | 2011-11-02 18:12:13 -0700^J^J * Fix for misnaming temp file in update-changes script. (Robin Sommer)^J^J0.21-1 | 2011-11-02 18:10:39 -0700^J^J * Little fix for make-relea +file_stream, Cx92a0ym5R8, 1024, se script, which could pick out the wrong^J tag. (Robin Sommer)^J^J0.21 | 2011-10-27 17:40:45 -0700^J^J * Fixing bro-cut's usage message and argument error handling. (Robin Sommer)^J^J * Bugfix in update-changes script. (Robin Sommer)^J^J * update-changes now ignores commits it did itself. (Robin Sommer)^J^J * Fix a bug in the update-changes script. (Robin Sommer)^J^J * bro-cut now always installs to $prefix/bin by `make install`. (Jon Siwek)^J^J * Options to adjust time format for bro-cut. (Robin Sommer)^J^J The default with -d is now ISO format. The new option "-D "^J specifies a custom strftime()-style format string. Alternatively,^J the environment variable BRO_CUT_TIMEFMT can set the format as^J well.^J^J * bro-cut now understands the field separator header. (Robin Sommer)^J^J * Renaming options -h/-H -> -c/-C, and doing some general cleanup.^J^J0.2 | 2011-10-25 19:53:57 -0700^J^J * Adding support for replacing version string in a setup.py. (Robin^J Sommer)^J^J * Change generated root cert DN indices f +file_chunk, Cx92a0ym5R8, 1024, 1500, se script, which could pick out the wrong^J tag. (Robin Sommer)^J^J0.21 | 2011-10-27 17:40:45 -0700^J^J * Fixing bro-cut's usage message and argument error handling. (Robin Sommer)^J^J * Bugfix in update-changes script. (Robin Sommer)^J^J * update-changes now ignores commits it did itself. (Robin Sommer)^J^J * Fix a bug in the update-changes script. (Robin Sommer)^J^J * bro-cut now always installs to $prefix/bin by `make install`. (Jon Siwek)^J^J * Options to adjust time format for bro-cut. (Robin Sommer)^J^J The default with -d is now ISO format. The new option "-D "^J specifies a custom strftime()-style format string. Alternatively,^J the environment variable BRO_CUT_TIMEFMT can set the format as^J well.^J^J * bro-cut now understands the field separator header. (Robin Sommer)^J^J * Renaming options -h/-H -> -c/-C, and doing some general cleanup.^J^J0.2 | 2011-10-25 19:53:57 -0700^J^J * Adding support for replacing version string in a setup.py. (Robin^J Sommer)^J^J * Change generated root cert DN indices f +file_stream, Cx92a0ym5R8, 476, ormat for RFC2253^J compliance. (Jon Siwek)^J^J * New tool devel-tools/check-release to run before making releases.^J (Robin Sommer)^J^J * devel-tools/update-changes gets a new option -a to amend to^J previous commit if possible. Default is now not to (used to be the^J opposite). (Robin Sommer)^J^J * Change Mozilla trust root generation to index certs by subject DN. (Jon Siwek)^J^J * Change distclean to only remove build dir. (Jon Siwek)^J^J * Make dist now cleans the +file_chunk, Cx92a0ym5R8, 476, 2524, ormat for RFC2253^J compliance. (Jon Siwek)^J^J * New tool devel-tools/check-release to run before making releases.^J (Robin Sommer)^J^J * devel-tools/update-changes gets a new option -a to amend to^J previous commit if possible. Default is now not to (used to be the^J opposite). (Robin Sommer)^J^J * Change Mozilla trust root generation to index certs by subject DN. (Jon Siwek)^J^J * Change distclean to only remove build dir. (Jon Siwek)^J^J * Make dist now cleans the +file_stream, Cx92a0ym5R8, 1024, copied source (Jon Siwek)^J^J * Small tweak to make-release for forced git-clean. (Jon Siwek)^J^J * Fix to not let updates scripts loose their executable permissions.^J (Robin Sommer)^J^J * devel-tools/update-changes now looks for a 'release' tag to^J idenfify the stable version, and 'beta' for the beta versions.^J (Robin Sommer).^J^J * Distribution cleanup. (Robin Sommer)^J^J * New script devel-tools/make-release to create source tar balls.^J (Robin Sommer)^J^J * Removing bdcat. With the new log format, this isn't very useful^J anymore. (Robin Sommer)^J^J * Adding script that shows all pending git fastpath commits. (Robin^J Sommer)^J^J * Script to measure CPU time by loading an increasing set of^J scripts. (Robin Sommer)^J^J * extract-conn script now deals wit *.gz files. (Robin Sommer)^J^J * Tiny update to output a valid CA list file for SSL cert^J validation. (Seth Hall)^J^J * Adding "install-aux" target. Addresses #622. (Jon Siwek)^J^J * Distribution cleanup. (Jon Siwek and Robin Sommer)^J^J * FindPCAP +file_chunk, Cx92a0ym5R8, 1024, 3000, copied source (Jon Siwek)^J^J * Small tweak to make-release for forced git-clean. (Jon Siwek)^J^J * Fix to not let updates scripts loose their executable permissions.^J (Robin Sommer)^J^J * devel-tools/update-changes now looks for a 'release' tag to^J idenfify the stable version, and 'beta' for the beta versions.^J (Robin Sommer).^J^J * Distribution cleanup. (Robin Sommer)^J^J * New script devel-tools/make-release to create source tar balls.^J (Robin Sommer)^J^J * Removing bdcat. With the new log format, this isn't very useful^J anymore. (Robin Sommer)^J^J * Adding script that shows all pending git fastpath commits. (Robin^J Sommer)^J^J * Script to measure CPU time by loading an increasing set of^J scripts. (Robin Sommer)^J^J * extract-conn script now deals wit *.gz files. (Robin Sommer)^J^J * Tiny update to output a valid CA list file for SSL cert^J validation. (Seth Hall)^J^J * Adding "install-aux" target. Addresses #622. (Jon Siwek)^J^J * Distribution cleanup. (Jon Siwek and Robin Sommer)^J^J * FindPCAP +FileAnalysis::TRIGGER_DONE +Cx92a0ym5R8, 4705, 0 +[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp] +total bytes: 4705 +source: HTTP +SHA1: 1dd7ac0398df6cbc0696445a91ec681facf4dc47 +MD5: 397168fd09991a0e712254df7bc639ac +SHA256: 4e7c7ef0984119447e743e3ec77e1de52713e345cde03fe7df753a35849bed18 +file_stream, Cx92a0ym5R8, 476, now links against thread library when necessary (e.g.^J PF_RING's libpcap) (Jon Siwek)^J^J * Install binaries with an RPATH (Jon Siwek)^J^J * Workaround for FreeBSD CMake port missing debug flags (Jon Siwek)^J^J * Rewrite of the update-changes script. (Robin Sommer)^J^J0.1-1 | 2011-06-14 21:12:41 -0700^J^J * Add a script for generating Mozilla's CA list for the SSL analyzer.^J (Seth Hall)^J^J0.1 | 2011-04-01 16:28:22 -0700^J^J * Converting build process to CMake. (Jon Siwek)^J +file_chunk, Cx92a0ym5R8, 476, 4024, now links against thread library when necessary (e.g.^J PF_RING's libpcap) (Jon Siwek)^J^J * Install binaries with an RPATH (Jon Siwek)^J^J * Workaround for FreeBSD CMake port missing debug flags (Jon Siwek)^J^J * Rewrite of the update-changes script. (Robin Sommer)^J^J0.1-1 | 2011-06-14 21:12:41 -0700^J^J * Add a script for generating Mozilla's CA list for the SSL analyzer.^J (Seth Hall)^J^J0.1 | 2011-04-01 16:28:22 -0700^J^J * Converting build process to CMake. (Jon Siwek)^J +file_stream, Cx92a0ym5R8, 205, ^J * Removing cf/hf/ca-* from distribution. The README has a note where^J to find them now. (Robin Sommer)^J^J * General cleanup. (Robin Sommer)^J^J * Initial import of bro/aux from SVN r7088. (Jon Siwek)^J +file_chunk, Cx92a0ym5R8, 205, 4500, ^J * Removing cf/hf/ca-* from distribution. The README has a note where^J to find them now. (Robin Sommer)^J^J * General cleanup. (Robin Sommer)^J^J * Initial import of bro/aux from SVN r7088. (Jon Siwek)^J diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.bifs.postpone_timeout/bro..stdout b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.bifs.postpone_timeout/bro..stdout new file mode 100644 index 0000000000..52f774b8c1 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.bifs.postpone_timeout/bro..stdout @@ -0,0 +1,16 @@ +FileAnalysis::TRIGGER_NEW +oDwT1BbzjM1, 0, 0 +FileAnalysis::TRIGGER_DONE +oDwT1BbzjM1, 1022920, 0 +[orig_h=192.168.72.14, orig_p=3254/tcp, resp_h=65.54.95.206, resp_p=80/tcp] +total bytes: 1022920 +source: HTTP +FileAnalysis::TRIGGER_NEW +oDwT1BbzjM1, 0, 0 +FileAnalysis::TRIGGER_TIMEOUT +FileAnalysis::TRIGGER_TIMEOUT +FileAnalysis::TRIGGER_EOF +oDwT1BbzjM1, 206024, 0 +[orig_h=192.168.72.14, orig_p=3257/tcp, resp_h=65.54.95.14, resp_p=80/tcp] +total bytes: 1022920 +source: HTTP diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.bifs.remove_action/get.out b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.bifs.remove_action/get.out new file mode 100644 index 0000000000..d0ad118ac1 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.bifs.remove_action/get.out @@ -0,0 +1,13 @@ +FileAnalysis::TRIGGER_NEW +Cx92a0ym5R8, 0, 0 +FileAnalysis::TRIGGER_BOF +FileAnalysis::TRIGGER_BOF_BUFFER +^J0.26 | 201 +FileAnalysis::TRIGGER_TYPE +file type is set +mime type is set +FileAnalysis::TRIGGER_DONE +Cx92a0ym5R8, 4705, 0 +[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp] +total bytes: 4705 +source: HTTP diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.bifs.stop/get.out b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.bifs.stop/get.out new file mode 100644 index 0000000000..c9a704444d --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.bifs.stop/get.out @@ -0,0 +1,2 @@ +FileAnalysis::TRIGGER_NEW +Cx92a0ym5R8, 0, 0 diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.ftp/out b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.ftp/out new file mode 100644 index 0000000000..c0fb36c86e --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.ftp/out @@ -0,0 +1,15 @@ +FileAnalysis::TRIGGER_NEW +sidhzrR4IT8, 0, 0 +FileAnalysis::TRIGGER_BOF +FileAnalysis::TRIGGER_BOF_BUFFER +The Nationa +FileAnalysis::TRIGGER_TYPE +file type is set +mime type is set +FileAnalysis::TRIGGER_EOF +sidhzrR4IT8, 16557, 0 +[orig_h=141.142.228.5, orig_p=50737/tcp, resp_h=141.142.192.162, resp_p=38141/tcp] +source: FTP_DATA +SHA1: 44586aed07cfe19cad25076af98f535585cd5797 +MD5: 7192a8075196267203adb3dfaa5c908d +SHA256: 202674eba48e832690a4475113acf8b16a3f6c82c04c94b36bb2c7ce457ac8d2 diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.ftp/thefile b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.ftp/thefile new file mode 100644 index 0000000000..ffa6b5b161 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.ftp/thefile @@ -0,0 +1,425 @@ +The National Center for Supercomputing Applications 1/28/92 +Anonymous FTP Server General Information + +This file contains information about the general structure, as well as +information on how to obtain files and documentation from the FTP server. +NCSA software and documentation can also be obtained through the the U.S. +Mail. Instructions are included for using this method as well. + +Information about the Software Development Group and NCSA software can be +found in the /ncsapubs directory in a file called TechResCatalog. + + +THE UNIVERSITY OF ILLINOIS GIVES NO WARRANTY, EXPRESSED OR IMPLIED, FOR THE +SOFTWARE AND/OR DOCUMENTATION PROVIDED, INCLUDING, WITHOUT LIMITATION, +WARRANTY OF MERCHANTABILITY AND WARRANTY OF FITNESS FOR A PARTICULAR PURPOSE. + + +_____________________________________________________________ + +FTP INSTRUCTIONS + +Most NCSA Software is released into the public domain. That is, for these +programs, the public domain has all rights for future licensing, resale, +and publication of available packages. If you are connected to Internet +(NSFNET, ARPANET, MILNET, etc) you may download NCSA software and documentation and source code if it is available, at no charge from the anonymous file +transfer protocol (FTP) server at NCSA where you got this file. The procedure +you should follow to do so is presented below. If you have any questions +regarding this procedure or whether you are connected to Internet, consult your local system administration or network expert. + +1. Log on to a host at your site that is connected to the Internet and is + running software supporting the FTP command. + +2. Invoke FTP on most systems by entering the Internet address of the server. + Type the following at the shell (usually "%") prompt: + + % ftp ftp.ncsa.uiuc.edu + +3. Log in by entering anonymous for the name. + +4. Enter your local email address (login@host) for the password. + +5. Enter the following at the "ftp>" prompt to copy a text file from our + server to your local host: + + ftp> get filename + + where "filename" is the name of the file you want a copy of. For example, + to get a copy of this file from the server enter: + + ftp> get README.FIRST + + To get a copy of our software brochure, enter: + + ftp> cd ncsapubs + get TechResCatalog + + NOTE: Some of the filenames on the server are rather long to aid in + identification. Some operating systems may have problems with names + this long. To change the name the file will have on your local + machine type the following at the "ftp>" prompt ("remoteName" is the + name of the file on the server and "localName" is the name you want + the file to have on your local machine): + + ftp> get remoteName localName + + Example: + + ftp> get TechResCatalog catalog.txt + + +6. For files that are not text files (almost everything else) you will need to + specify that you want to transfer binary files. Do this by typing the + following at the "ftp>" prompt: + + ftp> type binary + + You can now use the "get" command to download binary files. To switch back + to ASCII text transfers type: + + ftp> type ascii + +7. The "ls" and "cd" commands can be used at the "ftp>" prompt to list and + change directories as in the shell. + +8. Enter "quit" or "bye" to exit FTP and return to your local host. + + +_____________________________________________________________ + +FTP SOFTWARE BY MAIL + +To obtain an order form, send your request to the following address: + +FTP Archive Tapes +c/o Debbie Shirley +152 Computing Applications Building +605 East Springfield Avenue +Champaign, IL 61820 + +or call: +Debbie at (217) 244-4130 + + +_____________________________________________________________ + +VIRUS INFORMATION + +The Software Development Group at NCSA is very virus-conscious. We routinely +check our machines for viruses and recommend that you do so also. For the +Macintoshes we use Disinfectant. You can obtain a copy of Disinfectant from +the /Mac/Utilities directory. + +If you use Microsoft DOS or Windows you can find the latest virus scan from +the anonymous site oak.oakland.edu in the /SimTel/msdos/virus directory. + +_____________________________________________________________ + +GENERAL INFORMATION + + +DIRECTORY STRUCTURE + +The FTP server is organized as specified below: + + /Mac Macintosh software + /PC IBM PC software + /Unix Software for machines running UNIX or equivalent OS + /Unix/SGI Software that primarily runs on Silicon Graphics + machines only + /Visualization Software tools for data visualization. + /Web World Wide Web tools, including Mosaic, httpd, + and html editors. + /HDF Hierarchical Data Format applications and tools + /Samples Samples that can be used with most of NCSA software + tools + /Documentation Currently being constructed, check each application's + directory for documentation + /ncsapubs Information produced by the Publications group, + including Metacenter announcements, data link & access, + a software listing, start-up guides, and other + reference documents. + /misc Miscellaneous documentation and software + /incoming directory for contributions + /outgoing swap directory + +Information for a particular application can be found in the README file, +located in the same directory as the application. The README files contain +information on new features, known bugs, compile information, and other +important notes. + +All directories on the FTP server contain an INDEX file. These files outline +the hierarchical structure of the directory and (recursively) all files and +directories contained within it. The INDEX at the root level contains the +structure of the enire server listing all files and directories on it. The +INDEX file in each software directory contains additional information about +each file. The letter in parenthesis after the file name indicates how the +file should be downloaded: ascii (a), binary (b), or mac binary (m). + +The "misc" directories found in some software tool directories contain +supplementary code or other information. Refer to the README file in that +directory for a description of what is contained within the "misc" directory. + +The "contrib" directories contain contributed software. This directory usually +contains NCSA source that has been modified by people outside of NCSA as well +as binaries compiled on different platforms not available to the Software +Development Group. If you have modified NCSA software or would like to share +some code please contact the developer of the source so arrangemnts can be +made to upload it to the "incoming" directory. If you are downloading +software from the "contrib" directory please note that this software is not +supported by NCSA and has not been checked for viruses (see statement on +viruses above). NCSA may not be held responsible for anything resulting from +use of the contributed software. *** RUN AT YOUR OWN RISK *** + + +FILE NAMES + +All file names consist of the name of the tool, the version number, and one or +more extensions. The extensions identify what type of information is contained +in the file, and what format it is in. For example, here is a list of files in +the /Mac/DataScope directory: + + DataScope2.0.1.asc.tar.Z + DataScope2.0.1.src.sit.hqx + DataScope2.0.1.smp.sit.hqx + DataScope2.0.1.mac.sit.hqx + DataScope2.0.1.msw.sit.hqx + +The first three character extension indicates what type of data can be found in +that file (ASCII documentation, source, samples, etc.). The other extensions +indicate what format the files are in. The extensions ".tar" and ".sit" +indicate types of archives, and the ".Z" and ".hqx" indicate compression and +encoding schemes. (See below for instructions on extracting files that have +been archived and/or compressed.) Following are a list of extensions and their +meanings: + + .sn3 Sun 3 executables + .sn4 Sun 4 executables + .386 Sun 386i executables + .sgi Silicon Graphics Iris executables + .dgl Silicon Graphics Iris using DGL executables + .rs6 IBM RS6000 executables + .cv2 Convex 2 executables + .cv3 Convex 3 executables + .cr2 Cray 2 executables + .crY CrayYMP executables + .d31 DEC 3100 executables + .m88 Motorola 88k executables + .m68 Motorola 68k executables + .exe IBM PC executables + .mac Macintosh executables + .src source code + .smp sample files + .asc ASCII text documentation + .msw Microsoft Word documentation + .ps postscript documentation + .man formatted man page + .shar Bourne shell archive + .sit archive created by Macintosh application, StuffIt + .hqx encoded with Macintosh application, BinHex + .sea Self extracting Macintosh archive + .tar archive created with UNIX tar command + .Z compressed with UNIX compress command + +The files in the PC directory are the only exception to this naming convention. +In order to conform with the DOS convention of eight character file names and +one, three character extension, the names for PC files are slightly different. +Whenever possible the scheme outlined above is used, but the names are usually +abbreviated and all but one of the dots "." have been omitted. + + +_______________________________________________________________________________ +EXTRACTING ARCHIVED FILES + + +INSTRUCTIONS FOR MACINTOSH FILES + +If a file ends with the extension ".sit" it must be unstuffed with either the +shareware program StuffIt or the Public Domain program UnStuffIt. Files ending +with the ".hqx" must be decoded with BinHex. These programs can be found on +the FTP server in the /Mac/Utilities directory. Note that the BinHex program +must be downloaded with MacBinary enabled, and the StuffIt program must be +decoded before it can be used. Files downloaded from the server may be both +Stuffed (".sit" extension) and BinHexed (".hqx" extension). These files must +be first decoded and then unstuffed. + +To decode a file with the ".hqx" extension (a BinHexed file): + + 1. Download the file to your Macintosh. + 2. Start the application BinHex by double-clicking on it. + 3. From the "File" menu in BinHex, choose "UpLoad -> Application". + 4. Choose the ".hqx" file to be decoded and select "Open". + 5. The suggested file name will appear in a dialog box. + 6. Select "Save" to decode the file. + +To uncompress a file with the ".sit" extension (a Stuffed file): + + 1. Download the file to your Macintosh. + 2. Start the application Stuffit by double-clicking on it. + 3. From the "File" menu in Stuffit, choose "Open Archive...". + 4. Choose the ".sit" file to be unstuffed and select "Open". A window with + all the files contained in the stuffed file will appear. + 5. Choose "Select All" in the "Edit" menu to select all of the files. + 6. Click on the "Extract" box at the bottom of the window. + 7. Select "Save All" in the dialog box to save all the selected files in + the current directory. + + +INSTRUCTIONS FOR PC FILES + +Most IBM PC files are archived and compressed using the pkzip utility. +(If you do not have the pkzip utility on your PC, you may obtain it from the +FTP server by anonymous ftp. The file you need is called pkz110.exe and it +is located in /PC/Telnet/contributions. Set the ftp mode to binary and "get" +the file pkz110.exe. Then, on your PC, run PKZ110.EXE with no arguments and +several files will be self-extracted, including one called PKUNZIP.EXE. It +may then be convenient to copy PKUNZIP.EXE to the directory where you have +placed, or are going to place, your Telnet files.) +To extract these files, first download the file with the ".zip" extension to +your PC and then type the following at the DOS prompt: + + > pkunzip -d filename.zip + +where "filename" is the name of the file you want to unarchive. + + +INSTRUCTIONS FOR UNIX FILES + +Most files on the FTP server will be both tarred and compressed. For more +information on the "tar" and "compress" commands you can type "man tar" and +"man compress" at your shell prompt to see the online manual page for these +commands, or ask your system administrator for help. You should first +uncompress and then unarchive files ending in ".tar.Z" with the following +procedure. + +Files with the ".Z" extension have been compressed with the UNIX "compress" +command. To uncompress these files type the following at the shell prompt: + + % uncompress filename.Z + +where "filename.Z" is the name of the file ending with the ".Z" extension that +you wish to uncompress. + +Files with the ".tar" extension have been archived with the UNIX "tar" command. +To extract the files type the following at the shell prompt: + + % tar xf filename.tar + +Some files are archived using a shell archive utility and are indicated as such +with the ".shar" extension. To extract the files type the following at the +shell prompt: + + % sh filename.shar + + +_______________________________________________________________________________ +DOCUMENTATION + +NCSA offers users several documentation formats for its programs including +ASCII text, Microsoft Word, and postscript. If one of these formats does not +fit your needs, documentaion can be obtained through the mail at the following +address: + +Documentation Orders +c/o Debbie Shirley +152 Computing Applications Building +605 East Springfield Avenue +Champaign, IL 61820 + +or call: + +(217) 244-4130 + +Members of the Software Development Group within NCSA are currently working +on videotapes that demonstrate and also offer tutorials for NCSA programs. A +note will be posted here when these tapes are available for distribution. + + +ASCII FORMAT + +ASCII text files are provided for all software and are indicated with the +".asc" extension. Helpful figures and diagrams obviously cannot be included +in this form of documentation. We suggest you use the other forms of +documentation if possible. + + +MICROSOFT WORD FORMAT + +If you are a Macintosh user, please download documents with the ".msw" +extension. These files should also be stuffed and BinHexed (information on +extracting these files from the archive is contained earlier in this file). +The documents can be previewed and printed using the Microsoft Word +application. Word documents contain text, images, and formatting. + + +POSTSCRIPT FORMAT + +If you are a UNIX user and/or have access to a postscript printer, please +download files with the ".pos" extension. The documents can be previewed using +a poscript previewer or can be printed directly to a poscript printer using a +command like "lpr". + + +_______________________________________________________________________________ +BUG REPORTS AND SUPPORT + +The Software Development Group at NCSA is very interested in how the software +tools developed here are being used. Please send any comments or suggestions +you may have to the appropriate address. + +NOTE: This is a new kind of shareware. You share your science and +successes with us, and we can get more resources to share more +NCSA software with you. + +If you want to see more NCSA software, please send us a letter, + email or US Mail, telling us what you are doing with our software. +We need to know: + + (1) What science you are working on - an abstract of your + work would be fine. + + (2) How NCSA software has helped you, for example, by increasing + your productivity or allowing you to do things you could + not do before. + +We encourage you to cite the use of any NCSA software you have used in +your publications. A bibliography of your work would be extremely +helpful. + + +NCSA Telnet for the Macintosh: Please allow ***time*** for a response. + +Bug reports, questions, suggestions may be sent to the addresses below. + + mactelnet@ncsa.uiuc.edu (Internet) + +NCSA Telnet for PCs: Please allow ***time*** for a response. + +Bug reports, questions, suggestions may be sent to: + pctelnet@ncsa.uiuc.edu (Internet) + +All other NCSA software: + +Bug reports should be emailed to the adresses below. Be sure to check the +BUGS NOTES section of the README file before sending email. +Please allow ***time*** for a response. + + bugs@ncsa.uiuc.edu (Internet) + + +Questions regarding NCSA developed software tools may be sent to the address +below. Please allow ***time*** for a response. + + softdev@ncsa.uiuc.edu (Internet) +_______________________________________________________________________________ +COPYRIGHTS AND TRADEMARKS + +Apple +Motorola +Digital Equipment Corp. +Silicon Graphics Inc. +International Business Machines +Sun Microsystems +UNIX +StuffIt +Microsoft diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.get/Cx92a0ym5R8-file b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.get/Cx92a0ym5R8-file new file mode 100644 index 0000000000..e68dee74e6 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.get/Cx92a0ym5R8-file @@ -0,0 +1,159 @@ + +0.26 | 2012-08-24 15:10:04 -0700 + + * Fixing update-changes, which could pick the wrong control file. (Robin Sommer) + + * Fixing GPG signing script. (Robin Sommer) + +0.25 | 2012-08-01 13:55:46 -0500 + + * Fix configure script to exit with non-zero status on error (Jon Siwek) + +0.24 | 2012-07-05 12:50:43 -0700 + + * Raise minimum required CMake version to 2.6.3 (Jon Siwek) + + * Adding script to delete old fully-merged branches. (Robin Sommer) + +0.23-2 | 2012-01-25 13:24:01 -0800 + + * Fix a bro-cut error message. (Daniel Thayer) + +0.23 | 2012-01-11 12:16:11 -0800 + + * Tweaks to release scripts, plus a new one for signing files. + (Robin Sommer) + +0.22 | 2012-01-10 16:45:19 -0800 + + * Tweaks for OpenBSD support. (Jon Siwek) + + * bro-cut extensions and fixes. (Robin Sommer) + + - If no field names are given on the command line, we now pass through + all fields. Adresses #657. + + - Removing some GNUism from awk script. Addresses #653. + + - Added option for time output in UTC. Addresses #668. + + - Added output field separator option -F. Addresses #649. + + - Fixing option -c: only some header lines were passed through + rather than all. (Robin Sommer) + + * Fix parallel make portability. (Jon Siwek) + +0.21-9 | 2011-11-07 05:44:14 -0800 + + * Fixing compiler warnings. Addresses #388. (Jon Siwek) + +0.21-2 | 2011-11-02 18:12:13 -0700 + + * Fix for misnaming temp file in update-changes script. (Robin Sommer) + +0.21-1 | 2011-11-02 18:10:39 -0700 + + * Little fix for make-release script, which could pick out the wrong + tag. (Robin Sommer) + +0.21 | 2011-10-27 17:40:45 -0700 + + * Fixing bro-cut's usage message and argument error handling. (Robin Sommer) + + * Bugfix in update-changes script. (Robin Sommer) + + * update-changes now ignores commits it did itself. (Robin Sommer) + + * Fix a bug in the update-changes script. (Robin Sommer) + + * bro-cut now always installs to $prefix/bin by `make install`. (Jon Siwek) + + * Options to adjust time format for bro-cut. (Robin Sommer) + + The default with -d is now ISO format. The new option "-D " + specifies a custom strftime()-style format string. Alternatively, + the environment variable BRO_CUT_TIMEFMT can set the format as + well. + + * bro-cut now understands the field separator header. (Robin Sommer) + + * Renaming options -h/-H -> -c/-C, and doing some general cleanup. + +0.2 | 2011-10-25 19:53:57 -0700 + + * Adding support for replacing version string in a setup.py. (Robin + Sommer) + + * Change generated root cert DN indices format for RFC2253 + compliance. (Jon Siwek) + + * New tool devel-tools/check-release to run before making releases. + (Robin Sommer) + + * devel-tools/update-changes gets a new option -a to amend to + previous commit if possible. Default is now not to (used to be the + opposite). (Robin Sommer) + + * Change Mozilla trust root generation to index certs by subject DN. (Jon Siwek) + + * Change distclean to only remove build dir. (Jon Siwek) + + * Make dist now cleans the copied source (Jon Siwek) + + * Small tweak to make-release for forced git-clean. (Jon Siwek) + + * Fix to not let updates scripts loose their executable permissions. + (Robin Sommer) + + * devel-tools/update-changes now looks for a 'release' tag to + idenfify the stable version, and 'beta' for the beta versions. + (Robin Sommer). + + * Distribution cleanup. (Robin Sommer) + + * New script devel-tools/make-release to create source tar balls. + (Robin Sommer) + + * Removing bdcat. With the new log format, this isn't very useful + anymore. (Robin Sommer) + + * Adding script that shows all pending git fastpath commits. (Robin + Sommer) + + * Script to measure CPU time by loading an increasing set of + scripts. (Robin Sommer) + + * extract-conn script now deals wit *.gz files. (Robin Sommer) + + * Tiny update to output a valid CA list file for SSL cert + validation. (Seth Hall) + + * Adding "install-aux" target. Addresses #622. (Jon Siwek) + + * Distribution cleanup. (Jon Siwek and Robin Sommer) + + * FindPCAP now links against thread library when necessary (e.g. + PF_RING's libpcap) (Jon Siwek) + + * Install binaries with an RPATH (Jon Siwek) + + * Workaround for FreeBSD CMake port missing debug flags (Jon Siwek) + + * Rewrite of the update-changes script. (Robin Sommer) + +0.1-1 | 2011-06-14 21:12:41 -0700 + + * Add a script for generating Mozilla's CA list for the SSL analyzer. + (Seth Hall) + +0.1 | 2011-04-01 16:28:22 -0700 + + * Converting build process to CMake. (Jon Siwek) + + * Removing cf/hf/ca-* from distribution. The README has a note where + to find them now. (Robin Sommer) + + * General cleanup. (Robin Sommer) + + * Initial import of bro/aux from SVN r7088. (Jon Siwek) diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.get/get-gzip.out b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.get/get-gzip.out new file mode 100644 index 0000000000..bc0eee737c --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.get/get-gzip.out @@ -0,0 +1,15 @@ +FileAnalysis::TRIGGER_NEW +kg59rqyYxN, 0, 0 +FileAnalysis::TRIGGER_BOF +FileAnalysis::TRIGGER_BOF_BUFFER +{^J "origin +FileAnalysis::TRIGGER_TYPE +file type is set +mime type is set +FileAnalysis::TRIGGER_EOF +kg59rqyYxN, 197, 0 +[orig_h=141.142.228.5, orig_p=50153/tcp, resp_h=54.243.118.187, resp_p=80/tcp] +source: HTTP +SHA1: e351b8c693c3353716787c02e2923f4d12ebbb31 +MD5: 5baba7eea57bc8a42a92c817ed566d72 +SHA256: 202b775be087f5af98e95120e42769a9b3488f84c5aa79c4f4c1093d348f849c diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.get/get.out b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.get/get.out new file mode 100644 index 0000000000..34d7e942a2 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.get/get.out @@ -0,0 +1,16 @@ +FileAnalysis::TRIGGER_NEW +Cx92a0ym5R8, 0, 0 +FileAnalysis::TRIGGER_BOF +FileAnalysis::TRIGGER_BOF_BUFFER +^J0.26 | 201 +FileAnalysis::TRIGGER_TYPE +file type is set +mime type is set +FileAnalysis::TRIGGER_DONE +Cx92a0ym5R8, 4705, 0 +[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp] +total bytes: 4705 +source: HTTP +SHA1: 1dd7ac0398df6cbc0696445a91ec681facf4dc47 +MD5: 397168fd09991a0e712254df7bc639ac +SHA256: 4e7c7ef0984119447e743e3ec77e1de52713e345cde03fe7df753a35849bed18 diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.get/kg59rqyYxN-file b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.get/kg59rqyYxN-file new file mode 100644 index 0000000000..c8435ed950 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.get/kg59rqyYxN-file @@ -0,0 +1,11 @@ +{ + "origin": "10.224.189.238", + "headers": { + "Host": "httpbin.org", + "Connection": "close", + "Accept": "*/*", + "User-Agent": "curl/7.29.0" + }, + "gzipped": true, + "method": "GET" +} \ No newline at end of file diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.partial-content/a.out b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.partial-content/a.out new file mode 100644 index 0000000000..addb0c0b4a --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.partial-content/a.out @@ -0,0 +1,8 @@ +FileAnalysis::TRIGGER_NEW +7gZBKVUgy4l, 0, 0 +FileAnalysis::TRIGGER_DONE +7gZBKVUgy4l, 555523, 0 +[orig_h=10.101.84.70, orig_p=10978/tcp, resp_h=129.174.93.161, resp_p=80/tcp] +[orig_h=10.101.84.70, orig_p=10977/tcp, resp_h=129.174.93.161, resp_p=80/tcp] +total bytes: 555523 +source: HTTP diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.partial-content/a.size b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.partial-content/a.size new file mode 100644 index 0000000000..13d0c3c958 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.partial-content/a.size @@ -0,0 +1 @@ +555523 7gZBKVUgy4l-file0 diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.partial-content/b.out b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.partial-content/b.out new file mode 100644 index 0000000000..c9f234c5e8 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.partial-content/b.out @@ -0,0 +1,15 @@ +FileAnalysis::TRIGGER_NEW +oDwT1BbzjM1, 0, 0 +FileAnalysis::TRIGGER_DONE +oDwT1BbzjM1, 1022920, 0 +[orig_h=192.168.72.14, orig_p=3254/tcp, resp_h=65.54.95.206, resp_p=80/tcp] +total bytes: 1022920 +source: HTTP +FileAnalysis::TRIGGER_NEW +oDwT1BbzjM1, 0, 0 +FileAnalysis::TRIGGER_TIMEOUT +FileAnalysis::TRIGGER_EOF +oDwT1BbzjM1, 206024, 0 +[orig_h=192.168.72.14, orig_p=3257/tcp, resp_h=65.54.95.14, resp_p=80/tcp] +total bytes: 1022920 +source: HTTP diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.partial-content/b.size b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.partial-content/b.size new file mode 100644 index 0000000000..c1c1d71db7 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.partial-content/b.size @@ -0,0 +1 @@ +1022920 oDwT1BbzjM1-file0 diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.partial-content/c.out b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.partial-content/c.out new file mode 100644 index 0000000000..547aa0b568 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.partial-content/c.out @@ -0,0 +1,8 @@ +FileAnalysis::TRIGGER_NEW +uHS14uhRKGe, 0, 0 +FileAnalysis::TRIGGER_DONE +uHS14uhRKGe, 498702, 0 +[orig_h=10.45.179.94, orig_p=19950/tcp, resp_h=129.174.93.170, resp_p=80/tcp] +[orig_h=10.45.179.94, orig_p=19953/tcp, resp_h=129.174.93.170, resp_p=80/tcp] +total bytes: 498668 +source: HTTP diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.partial-content/c.size b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.partial-content/c.size new file mode 100644 index 0000000000..da0f4d480c --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.partial-content/c.size @@ -0,0 +1 @@ +498668 uHS14uhRKGe-file0 diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.pipeline/CCU3vUEr06l-file b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.pipeline/CCU3vUEr06l-file new file mode 100644 index 0000000000..41e217d6c9 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.pipeline/CCU3vUEr06l-file @@ -0,0 +1,636 @@ +//-- Google Analytics Urchin Module +//-- Copyright 2007 Google, All Rights Reserved. + +//-- Urchin On Demand Settings ONLY +var _uacct=""; // set up the Urchin Account +var _userv=0; // service mode (0=local,1=remote,2=both) + +//-- UTM User Settings +var _ufsc=1; // set client info flag (1=on|0=off) +var _udn="auto"; // (auto|none|domain) set the domain name for cookies +var _uhash="on"; // (on|off) unique domain hash for cookies +var _utimeout="1800"; // set the inactive session timeout in seconds +var _ugifpath="/images/__utm.gif"; // set the web path to the __utm.gif file +var _utsp="|"; // transaction field separator +var _uflash=1; // set flash version detect option (1=on|0=off) +var _utitle=1; // set the document title detect option (1=on|0=off) +var _ulink=0; // enable linker functionality (1=on|0=off) +var _uanchor=0; // enable use of anchors for campaign (1=on|0=off) +var _utcp="/"; // the cookie path for tracking +var _usample=100; // The sampling % of visitors to track (1-100). + +//-- UTM Campaign Tracking Settings +var _uctm=1; // set campaign tracking module (1=on|0=off) +var _ucto="15768000"; // set timeout in seconds (6 month default) +var _uccn="utm_campaign"; // name +var _ucmd="utm_medium"; // medium (cpc|cpm|link|email|organic) +var _ucsr="utm_source"; // source +var _uctr="utm_term"; // term/keyword +var _ucct="utm_content"; // content +var _ucid="utm_id"; // id number +var _ucno="utm_nooverride"; // don't override + +//-- Auto/Organic Sources and Keywords +var _uOsr=new Array(); +var _uOkw=new Array(); +_uOsr[0]="google"; _uOkw[0]="q"; +_uOsr[1]="yahoo"; _uOkw[1]="p"; +_uOsr[2]="msn"; _uOkw[2]="q"; +_uOsr[3]="aol"; _uOkw[3]="query"; +_uOsr[4]="aol"; _uOkw[4]="encquery"; +_uOsr[5]="lycos"; _uOkw[5]="query"; +_uOsr[6]="ask"; _uOkw[6]="q"; +_uOsr[7]="altavista"; _uOkw[7]="q"; +_uOsr[8]="netscape"; _uOkw[8]="query"; +_uOsr[9]="cnn"; _uOkw[9]="query"; +_uOsr[10]="looksmart"; _uOkw[10]="qt"; +_uOsr[11]="about"; _uOkw[11]="terms"; +_uOsr[12]="mamma"; _uOkw[12]="query"; +_uOsr[13]="alltheweb"; _uOkw[13]="q"; +_uOsr[14]="gigablast"; _uOkw[14]="q"; +_uOsr[15]="voila"; _uOkw[15]="rdata"; +_uOsr[16]="virgilio"; _uOkw[16]="qs"; +_uOsr[17]="live"; _uOkw[17]="q"; +_uOsr[18]="baidu"; _uOkw[18]="wd"; +_uOsr[19]="alice"; _uOkw[19]="qs"; +_uOsr[20]="yandex"; _uOkw[20]="text"; +_uOsr[21]="najdi"; _uOkw[21]="q"; +_uOsr[22]="aol"; _uOkw[22]="q"; +_uOsr[23]="club-internet"; _uOkw[23]="q"; +_uOsr[24]="mama"; _uOkw[24]="query"; +_uOsr[25]="seznam"; _uOkw[25]="q"; +_uOsr[26]="search"; _uOkw[26]="q"; +_uOsr[27]="szukaj"; _uOkw[27]="szukaj"; +_uOsr[28]="szukaj"; _uOkw[28]="qt"; +_uOsr[29]="netsprint"; _uOkw[29]="q"; +_uOsr[30]="google.interia"; _uOkw[30]="q"; +_uOsr[31]="szukacz"; _uOkw[31]="q"; +_uOsr[32]="yam"; _uOkw[32]="k"; +_uOsr[33]="pchome"; _uOkw[33]="q"; + + +//-- Auto/Organic Keywords to Ignore +var _uOno=new Array(); +//_uOno[0]="urchin"; +//_uOno[1]="urchin.com"; +//_uOno[2]="www.urchin.com"; + +//-- Referral domains to Ignore +var _uRno=new Array(); +//_uRno[0]=".urchin.com"; + +//-- **** Don't modify below this point *** +var _uff,_udh,_udt,_ubl=0,_udo="",_uu,_ufns=0,_uns=0,_ur="-",_ufno=0,_ust=0,_ubd=document,_udl=_ubd.location,_udlh="",_uwv="1"; +var _ugifpath2="http://www.google-analytics.com/__utm.gif"; +if (_udl.hash) _udlh=_udl.href.substring(_udl.href.indexOf('#')); +if (_udl.protocol=="https:") _ugifpath2="https://ssl.google-analytics.com/__utm.gif"; +if (!_utcp || _utcp=="") _utcp="/"; +function urchinTracker(page) { + if (_udl.protocol=="file:") return; + if (_uff && (!page || page=="")) return; + var a,b,c,xx,v,z,k,x="",s="",f=0; + var nx=" expires="+_uNx()+";"; + var dc=_ubd.cookie; + _udh=_uDomain(); + if (!_uVG()) return; + _uu=Math.round(Math.random()*2147483647); + _udt=new Date(); + _ust=Math.round(_udt.getTime()/1000); + a=dc.indexOf("__utma="+_udh); + b=dc.indexOf("__utmb="+_udh); + c=dc.indexOf("__utmc="+_udh); + if (_udn && _udn!="") { _udo=" domain="+_udn+";"; } + if (_utimeout && _utimeout!="") { + x=new Date(_udt.getTime()+(_utimeout*1000)); + x=" expires="+x.toGMTString()+";"; + } + if (_ulink) { + if (_uanchor && _udlh && _udlh!="") s=_udlh+"&"; + s+=_udl.search; + if(s && s!="" && s.indexOf("__utma=")>=0) { + if (!(_uIN(a=_uGC(s,"__utma=","&")))) a="-"; + if (!(_uIN(b=_uGC(s,"__utmb=","&")))) b="-"; + if (!(_uIN(c=_uGC(s,"__utmc=","&")))) c="-"; + v=_uGC(s,"__utmv=","&"); + z=_uGC(s,"__utmz=","&"); + k=_uGC(s,"__utmk=","&"); + xx=_uGC(s,"__utmx=","&"); + if ((k*1) != ((_uHash(a+b+c+xx+z+v)*1)+(_udh*1))) {_ubl=1;a="-";b="-";c="-";xx="-";z="-";v="-";} + if (a!="-" && b!="-" && c!="-") f=1; + else if(a!="-") f=2; + } + } + if(f==1) { + _ubd.cookie="__utma="+a+"; path="+_utcp+";"+nx+_udo; + _ubd.cookie="__utmb="+b+"; path="+_utcp+";"+x+_udo; + _ubd.cookie="__utmc="+c+"; path="+_utcp+";"+_udo; + } else if (f==2) { + a=_uFixA(s,"&",_ust); + _ubd.cookie="__utma="+a+"; path="+_utcp+";"+nx+_udo; + _ubd.cookie="__utmb="+_udh+"; path="+_utcp+";"+x+_udo; + _ubd.cookie="__utmc="+_udh+"; path="+_utcp+";"+_udo; + _ufns=1; + } else if (a>=0 && b>=0 && c>=0) { + _ubd.cookie="__utmb="+_udh+"; path="+_utcp+";"+x+_udo; + } else { + if (a>=0) a=_uFixA(_ubd.cookie,";",_ust); + else a=_udh+"."+_uu+"."+_ust+"."+_ust+"."+_ust+".1"; + _ubd.cookie="__utma="+a+"; path="+_utcp+";"+nx+_udo; + _ubd.cookie="__utmb="+_udh+"; path="+_utcp+";"+x+_udo; + _ubd.cookie="__utmc="+_udh+"; path="+_utcp+";"+_udo; + _ufns=1; + } + if (_ulink && xx && xx!="" && xx!="-") { + xx=_uUES(xx); + if (xx.indexOf(";")==-1) _ubd.cookie="__utmx="+xx+"; path="+_utcp+";"+nx+_udo; + } + if (_ulink && v && v!="" && v!="-") { + v=_uUES(v); + if (v.indexOf(";")==-1) _ubd.cookie="__utmv="+v+"; path="+_utcp+";"+nx+_udo; + } + _uInfo(page); + _ufns=0; + _ufno=0; + if (!page || page=="") _uff=1; +} +function _uInfo(page) { + var p,s="",dm="",pg=_udl.pathname+_udl.search; + if (page && page!="") pg=_uES(page,1); + _ur=_ubd.referrer; + if (!_ur || _ur=="") { _ur="-"; } + else { + dm=_ubd.domain; + if(_utcp && _utcp!="/") dm+=_utcp; + p=_ur.indexOf(dm); + if ((p>=0) && (p<=8)) { _ur="0"; } + if (_ur.indexOf("[")==0 && _ur.lastIndexOf("]")==(_ur.length-1)) { _ur="-"; } + } + s+="&utmn="+_uu; + if (_ufsc) s+=_uBInfo(); + if (_uctm) s+=_uCInfo(); + if (_utitle && _ubd.title && _ubd.title!="") s+="&utmdt="+_uES(_ubd.title); + if (_udl.hostname && _udl.hostname!="") s+="&utmhn="+_uES(_udl.hostname); + s+="&utmr="+_ur; + s+="&utmp="+pg; + if ((_userv==0 || _userv==2) && _uSP()) { + var i=new Image(1,1); + i.src=_ugifpath+"?"+"utmwv="+_uwv+s; + i.onload=function() {_uVoid();} + } + if ((_userv==1 || _userv==2) && _uSP()) { + var i2=new Image(1,1); + i2.src=_ugifpath2+"?"+"utmwv="+_uwv+s+"&utmac="+_uacct+"&utmcc="+_uGCS(); + i2.onload=function() { _uVoid(); } + } + return; +} +function _uVoid() { return; } +function _uCInfo() { + if (!_ucto || _ucto=="") { _ucto="15768000"; } + if (!_uVG()) return; + var c="",t="-",t2="-",t3="-",o=0,cs=0,cn=0,i=0,z="-",s=""; + if (_uanchor && _udlh && _udlh!="") s=_udlh+"&"; + s+=_udl.search; + var x=new Date(_udt.getTime()+(_ucto*1000)); + var dc=_ubd.cookie; + x=" expires="+x.toGMTString()+";"; + if (_ulink && !_ubl) { + z=_uUES(_uGC(s,"__utmz=","&")); + if (z!="-" && z.indexOf(";")==-1) { _ubd.cookie="__utmz="+z+"; path="+_utcp+";"+x+_udo; return ""; } + } + z=dc.indexOf("__utmz="+_udh); + if (z>-1) { z=_uGC(dc,"__utmz="+_udh,";"); } + else { z="-"; } + t=_uGC(s,_ucid+"=","&"); + t2=_uGC(s,_ucsr+"=","&"); + t3=_uGC(s,"gclid=","&"); + if ((t!="-" && t!="") || (t2!="-" && t2!="") || (t3!="-" && t3!="")) { + if (t!="-" && t!="") c+="utmcid="+_uEC(t); + if (t2!="-" && t2!="") { if (c != "") c+="|"; c+="utmcsr="+_uEC(t2); } + if (t3!="-" && t3!="") { if (c != "") c+="|"; c+="utmgclid="+_uEC(t3); } + t=_uGC(s,_uccn+"=","&"); + if (t!="-" && t!="") c+="|utmccn="+_uEC(t); + else c+="|utmccn=(not+set)"; + t=_uGC(s,_ucmd+"=","&"); + if (t!="-" && t!="") c+="|utmcmd="+_uEC(t); + else c+="|utmcmd=(not+set)"; + t=_uGC(s,_uctr+"=","&"); + if (t!="-" && t!="") c+="|utmctr="+_uEC(t); + else { t=_uOrg(1); if (t!="-" && t!="") c+="|utmctr="+_uEC(t); } + t=_uGC(s,_ucct+"=","&"); + if (t!="-" && t!="") c+="|utmcct="+_uEC(t); + t=_uGC(s,_ucno+"=","&"); + if (t=="1") o=1; + if (z!="-" && o==1) return ""; + } + if (c=="-" || c=="") { c=_uOrg(); if (z!="-" && _ufno==1) return ""; } + if (c=="-" || c=="") { if (_ufns==1) c=_uRef(); if (z!="-" && _ufno==1) return ""; } + if (c=="-" || c=="") { + if (z=="-" && _ufns==1) { c="utmccn=(direct)|utmcsr=(direct)|utmcmd=(none)"; } + if (c=="-" || c=="") return ""; + } + if (z!="-") { + i=z.indexOf("."); + if (i>-1) i=z.indexOf(".",i+1); + if (i>-1) i=z.indexOf(".",i+1); + if (i>-1) i=z.indexOf(".",i+1); + t=z.substring(i+1,z.length); + if (t.toLowerCase()==c.toLowerCase()) cs=1; + t=z.substring(0,i); + if ((i=t.lastIndexOf(".")) > -1) { + t=t.substring(i+1,t.length); + cn=(t*1); + } + } + if (cs==0 || _ufns==1) { + t=_uGC(dc,"__utma="+_udh,";"); + if ((i=t.lastIndexOf(".")) > 9) { + _uns=t.substring(i+1,t.length); + _uns=(_uns*1); + } + cn++; + if (_uns==0) _uns=1; + _ubd.cookie="__utmz="+_udh+"."+_ust+"."+_uns+"."+cn+"."+c+"; path="+_utcp+"; "+x+_udo; + } + if (cs==0 || _ufns==1) return "&utmcn=1"; + else return "&utmcr=1"; +} +function _uRef() { + if (_ur=="0" || _ur=="" || _ur=="-") return ""; + var i=0,h,k,n; + if ((i=_ur.indexOf("://"))<0) return ""; + h=_ur.substring(i+3,_ur.length); + if (h.indexOf("/") > -1) { + k=h.substring(h.indexOf("/"),h.length); + if (k.indexOf("?") > -1) k=k.substring(0,k.indexOf("?")); + h=h.substring(0,h.indexOf("/")); + } + h=h.toLowerCase(); + n=h; + if ((i=n.indexOf(":")) > -1) n=n.substring(0,i); + for (var ii=0;ii<_uRno.length;ii++) { + if ((i=n.indexOf(_uRno[ii].toLowerCase())) > -1 && n.length==(i+_uRno[ii].length)) { _ufno=1; break; } + } + if (h.indexOf("www.")==0) h=h.substring(4,h.length); + return "utmccn=(referral)|utmcsr="+_uEC(h)+"|"+"utmcct="+_uEC(k)+"|utmcmd=referral"; +} +function _uOrg(t) { + if (_ur=="0" || _ur=="" || _ur=="-") return ""; + var i=0,h,k; + if ((i=_ur.indexOf("://")) < 0) return ""; + h=_ur.substring(i+3,_ur.length); + if (h.indexOf("/") > -1) { + h=h.substring(0,h.indexOf("/")); + } + for (var ii=0;ii<_uOsr.length;ii++) { + if (h.toLowerCase().indexOf(_uOsr[ii].toLowerCase()) > -1) { + if ((i=_ur.indexOf("?"+_uOkw[ii]+"=")) > -1 || (i=_ur.indexOf("&"+_uOkw[ii]+"=")) > -1) { + k=_ur.substring(i+_uOkw[ii].length+2,_ur.length); + if ((i=k.indexOf("&")) > -1) k=k.substring(0,i); + for (var yy=0;yy<_uOno.length;yy++) { + if (_uOno[yy].toLowerCase()==k.toLowerCase()) { _ufno=1; break; } + } + if (t) return _uEC(k); + else return "utmccn=(organic)|utmcsr="+_uEC(_uOsr[ii])+"|"+"utmctr="+_uEC(k)+"|utmcmd=organic"; + } + } + } + return ""; +} +function _uBInfo() { + var sr="-",sc="-",ul="-",fl="-",cs="-",je=1; + var n=navigator; + if (self.screen) { + sr=screen.width+"x"+screen.height; + sc=screen.colorDepth+"-bit"; + } else if (self.java) { + var j=java.awt.Toolkit.getDefaultToolkit(); + var s=j.getScreenSize(); + sr=s.width+"x"+s.height; + } + if (n.language) { ul=n.language.toLowerCase(); } + else if (n.browserLanguage) { ul=n.browserLanguage.toLowerCase(); } + je=n.javaEnabled()?1:0; + if (_uflash) fl=_uFlash(); + if (_ubd.characterSet) cs=_uES(_ubd.characterSet); + else if (_ubd.charset) cs=_uES(_ubd.charset); + return "&utmcs="+cs+"&utmsr="+sr+"&utmsc="+sc+"&utmul="+ul+"&utmje="+je+"&utmfl="+fl; +} +function __utmSetTrans() { + var e; + if (_ubd.getElementById) e=_ubd.getElementById("utmtrans"); + else if (_ubd.utmform && _ubd.utmform.utmtrans) e=_ubd.utmform.utmtrans; + if (!e) return; + var l=e.value.split("UTM:"); + var i,i2,c; + if (_userv==0 || _userv==2) i=new Array(); + if (_userv==1 || _userv==2) { i2=new Array(); c=_uGCS(); } + + for (var ii=0;ii=2;ii--) { + try { + var fl=eval("new ActiveXObject('ShockwaveFlash.ShockwaveFlash."+ii+"');"); + if (fl) { f=ii + '.0'; break; } + } + catch(e) {} + } + } + return f; +} +function __utmLinker(l,h) { + if (!_ulink) return; + var p,k,a="-",b="-",c="-",x="-",z="-",v="-"; + var dc=_ubd.cookie; + if (!l || l=="") return; + var iq = l.indexOf("?"); + var ih = l.indexOf("#"); + if (dc) { + a=_uES(_uGC(dc,"__utma="+_udh,";")); + b=_uES(_uGC(dc,"__utmb="+_udh,";")); + c=_uES(_uGC(dc,"__utmc="+_udh,";")); + x=_uES(_uGC(dc,"__utmx="+_udh,";")); + z=_uES(_uGC(dc,"__utmz="+_udh,";")); + v=_uES(_uGC(dc,"__utmv="+_udh,";")); + k=(_uHash(a+b+c+x+z+v)*1)+(_udh*1); + p="__utma="+a+"&__utmb="+b+"&__utmc="+c+"&__utmx="+x+"&__utmz="+z+"&__utmv="+v+"&__utmk="+k; + } + if (p) { + if (h && ih>-1) return; + if (h) { _udl.href=l+"#"+p; } + else { + if (iq==-1 && ih==-1) _udl.href=l+"?"+p; + else if (ih==-1) _udl.href=l+"&"+p; + else if (iq==-1) _udl.href=l.substring(0,ih-1)+"?"+p+l.substring(ih); + else _udl.href=l.substring(0,ih-1)+"&"+p+l.substring(ih); + } + } else { _udl.href=l; } +} +function __utmLinkPost(f,h) { + if (!_ulink) return; + var p,k,a="-",b="-",c="-",x="-",z="-",v="-"; + var dc=_ubd.cookie; + if (!f || !f.action) return; + var iq = f.action.indexOf("?"); + var ih = f.action.indexOf("#"); + if (dc) { + a=_uES(_uGC(dc,"__utma="+_udh,";")); + b=_uES(_uGC(dc,"__utmb="+_udh,";")); + c=_uES(_uGC(dc,"__utmc="+_udh,";")); + x=_uES(_uGC(dc,"__utmx="+_udh,";")); + z=_uES(_uGC(dc,"__utmz="+_udh,";")); + v=_uES(_uGC(dc,"__utmv="+_udh,";")); + k=(_uHash(a+b+c+x+z+v)*1)+(_udh*1); + p="__utma="+a+"&__utmb="+b+"&__utmc="+c+"&__utmx="+x+"&__utmz="+z+"&__utmv="+v+"&__utmk="+k; + } + if (p) { + if (h && ih>-1) return; + if (h) { f.action+="#"+p; } + else { + if (iq==-1 && ih==-1) f.action+="?"+p; + else if (ih==-1) f.action+="&"+p; + else if (iq==-1) f.action=f.action.substring(0,ih-1)+"?"+p+f.action.substring(ih); + else f.action=f.action.substring(0,ih-1)+"&"+p+f.action.substring(ih); + } + } + return; +} +function __utmSetVar(v) { + if (!v || v=="") return; + if (!_udo || _udo == "") { + _udh=_uDomain(); + if (_udn && _udn!="") { _udo=" domain="+_udn+";"; } + } + if (!_uVG()) return; + var r=Math.round(Math.random() * 2147483647); + _ubd.cookie="__utmv="+_udh+"."+_uES(v)+"; path="+_utcp+"; expires="+_uNx()+";"+_udo; + var s="&utmt=var&utmn="+r; + if ((_userv==0 || _userv==2) && _uSP()) { + var i=new Image(1,1); + i.src=_ugifpath+"?"+"utmwv="+_uwv+s; + i.onload=function() { _uVoid(); } + } + if ((_userv==1 || _userv==2) && _uSP()) { + var i2=new Image(1,1); + i2.src=_ugifpath2+"?"+"utmwv="+_uwv+s+"&utmac="+_uacct+"&utmcc="+_uGCS(); + i2.onload=function() { _uVoid(); } + } +} +function _uGCS() { + var t,c="",dc=_ubd.cookie; + if ((t=_uGC(dc,"__utma="+_udh,";"))!="-") c+=_uES("__utma="+t+";+"); + if ((t=_uGC(dc,"__utmb="+_udh,";"))!="-") c+=_uES("__utmb="+t+";+"); + if ((t=_uGC(dc,"__utmc="+_udh,";"))!="-") c+=_uES("__utmc="+t+";+"); + if ((t=_uGC(dc,"__utmx="+_udh,";"))!="-") c+=_uES("__utmx="+t+";+"); + if ((t=_uGC(dc,"__utmz="+_udh,";"))!="-") c+=_uES("__utmz="+t+";+"); + if ((t=_uGC(dc,"__utmv="+_udh,";"))!="-") c+=_uES("__utmv="+t+";"); + if (c.charAt(c.length-1)=="+") c=c.substring(0,c.length-1); + return c; +} +function _uGC(l,n,s) { + if (!l || l=="" || !n || n=="" || !s || s=="") return "-"; + var i,i2,i3,c="-"; + i=l.indexOf(n); + i3=n.indexOf("=")+1; + if (i > -1) { + i2=l.indexOf(s,i); if (i2 < 0) { i2=l.length; } + c=l.substring((i+i3),i2); + } + return c; +} +function _uDomain() { + if (!_udn || _udn=="" || _udn=="none") { _udn=""; return 1; } + if (_udn=="auto") { + var d=_ubd.domain; + if (d.substring(0,4)=="www.") { + d=d.substring(4,d.length); + } + _udn=d; + } + _udn = _udn.toLowerCase(); + if (_uhash=="off") return 1; + return _uHash(_udn); +} +function _uHash(d) { + if (!d || d=="") return 1; + var h=0,g=0; + for (var i=d.length-1;i>=0;i--) { + var c=parseInt(d.charCodeAt(i)); + h=((h << 6) & 0xfffffff) + c + (c << 14); + if ((g=h & 0xfe00000)!=0) h=(h ^ (g >> 21)); + } + return h; +} +function _uFixA(c,s,t) { + if (!c || c=="" || !s || s=="" || !t || t=="") return "-"; + var a=_uGC(c,"__utma="+_udh,s); + var lt=0,i=0; + if ((i=a.lastIndexOf(".")) > 9) { + _uns=a.substring(i+1,a.length); + _uns=(_uns*1)+1; + a=a.substring(0,i); + if ((i=a.lastIndexOf(".")) > 7) { + lt=a.substring(i+1,a.length); + a=a.substring(0,i); + } + if ((i=a.lastIndexOf(".")) > 5) { + a=a.substring(0,i); + } + a+="."+lt+"."+t+"."+_uns; + } + return a; +} +function _uTrim(s) { + if (!s || s=="") return ""; + while ((s.charAt(0)==' ') || (s.charAt(0)=='\n') || (s.charAt(0,1)=='\r')) s=s.substring(1,s.length); + while ((s.charAt(s.length-1)==' ') || (s.charAt(s.length-1)=='\n') || (s.charAt(s.length-1)=='\r')) s=s.substring(0,s.length-1); + return s; +} +function _uEC(s) { + var n=""; + if (!s || s=="") return ""; + for (var i=0;i0) r=a.substring(i+1,i2); else return ""; + if ((i=a.indexOf(".",i2+1))>0) t=a.substring(i2+1,i); else return ""; + if (f) { + return r; + } else { + var c=new Array('A','B','C','D','E','F','G','H','J','K','L','M','N','P','R','S','T','U','V','W','X','Y','Z','1','2','3','4','5','6','7','8','9'); + return c[r>>28&m]+c[r>>23&m]+c[r>>18&m]+c[r>>13&m]+"-"+c[r>>8&m]+c[r>>3&m]+c[((r&7)<<2)+(t>>30&3)]+c[t>>25&m]+c[t>>20&m]+"-"+c[t>>15&m]+c[t>>10&m]+c[t>>5&m]+c[t&m]; + } +} +function _uIN(n) { + if (!n) return false; + for (var i=0;i"9") && (c!=".")) return false; + } + return true; +} +function _uES(s,u) { + if (typeof(encodeURIComponent) == 'function') { + if (u) return encodeURI(s); + else return encodeURIComponent(s); + } else { + return escape(s); + } +} +function _uUES(s) { + if (typeof(decodeURIComponent) == 'function') { + return decodeURIComponent(s); + } else { + return unescape(s); + } +} +function _uVG() { + if((_udn.indexOf("www.google.") == 0 || _udn.indexOf(".google.") == 0 || _udn.indexOf("google.") == 0) && _utcp=='/' && _udn.indexOf("google.org")==-1) { + return false; + } + return true; +} +function _uSP() { + var s=100; + if (_usample) s=_usample; + if(s>=100 || s<=0) return true; + return ((__utmVisitorCode(1)%10000)<(s*100)); +} +function urchinPathCopy(p){ + var d=document,nx,tx,sx,i,c,cs,t,h,o; + cs=new Array("a","b","c","v","x","z"); + h=_uDomain(); if (_udn && _udn!="") o=" domain="+_udn+";"; + nx=_uNx()+";"; + tx=new Date(); tx.setTime(tx.getTime()+(_utimeout*1000)); + tx=tx.toGMTString()+";"; + sx=new Date(); sx.setTime(sx.getTime()+(_ucto*1000)); + sx=sx.toGMTString()+";"; + for (i=0;i<6;i++){ + t=" expires="; + if (i==1) t+=tx; else if (i==2) t=""; else if (i==5) t+=sx; else t+=nx; + c=_uGC(d.cookie,"__utm"+cs[i]+"="+h,";"); + if (c!="-") d.cookie="__utm"+cs[i]+"="+c+"; path="+p+";"+t+o; + } +} +function _uCO() { + if (!_utk || _utk=="" || _utk.length<10) return; + var d='www.google.com'; + if (_utk.charAt(0)=='!') d='analytics.corp.google.com'; + _ubd.cookie="GASO="+_utk+"; path="+_utcp+";"+_udo; + var sc=document.createElement('script'); + sc.type='text/javascript'; + sc.id="_gasojs"; + sc.src='https://'+d+'/analytics/reporting/overlay_js?gaso='+_utk+'&'+Math.random(); + document.getElementsByTagName('head')[0].appendChild(sc); +} +function _uGT() { + var h=location.hash, a; + if (h && h!="" && h.indexOf("#gaso=")==0) { + a=_uGC(h,"gaso=","&"); + } else { + a=_uGC(_ubd.cookie,"GASO=",";"); + } + return a; +} +var _utk=_uGT(); +if (_utk && _utk!="" && _utk.length>10) { + if (window.addEventListener) { + window.addEventListener('load', _uCO, false); + } else if (window.attachEvent) { + window.attachEvent('onload', _uCO); + } +} + +function _uNx() { + return (new Date((new Date()).getTime()+63072000000)).toGMTString(); +} diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.pipeline/HCzA0dVwDPj-file b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.pipeline/HCzA0dVwDPj-file new file mode 100644 index 0000000000..cdaf76ed70 Binary files /dev/null and b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.pipeline/HCzA0dVwDPj-file differ diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.pipeline/a1Zu1fteVEf-file b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.pipeline/a1Zu1fteVEf-file new file mode 100644 index 0000000000..1383a33f48 Binary files /dev/null and b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.pipeline/a1Zu1fteVEf-file differ diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.pipeline/aFQKI8SPOL2-file b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.pipeline/aFQKI8SPOL2-file new file mode 100644 index 0000000000..4c2be686a9 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.pipeline/aFQKI8SPOL2-file @@ -0,0 +1,63 @@ +/* +************************************************* + +MOZILLA.ORG +Enhanced Screen Styles +(Only applicable if JavaScript is enabled) + +Created by the friendly folks at Happy Cog +http://www.happycog.com/ + +************************************************* +*/ + + +#quick-search label { position:absolute; left: 48px; top:14px; text-transform:uppercase; font-weight:bold; color:#dddfe0; font-size: 85%; /* 14px x 0.85 = 12px */} + + +/* community feed */ +#community-sub .rows li.over { background-color: #f5f6f6; } +#community-sub { position:relative; z-index:200; } + + +/* project carousel */ +.project-carousel { background:#CBCCCC url(/images/template/screen/bg_project_carousel.png) repeat-y right top; margin-right:-461px; padding:6px 0 6px 6px; position:relative; } +.project-carousel .pagination { position:absolute; right:471px; top:1em; padding-right:47px; } +.project-carousel .prev, .project-carousel .next { width:18px; height:21px; float:left; overflow:hidden; background:transparent url(/images/template/widgets/carousel-arrows.png) no-repeat 0 -21px; text-indent:-999px; position:absolute; top:0; right:0; } +.project-carousel .prev:hover, .project-carousel .next:hover { border-bottom:none; } + .project-carousel .prev { width:19px; right:18px; } + .project-carousel .next { background-position:-18px -21px; width:19px; } + .project-carousel .next.disabled { background-position:-18px top; } + .project-carousel .prev.disabled { background-position:left top; } +.projects-wrap { background: #dee0e1 url(/images/template/screen/bg_projects.png) repeat-y right center; height:14em; } + .projects-wrap .projects-fade-left { background:transparent url(/images/template/widgets/carousel-fade.png) repeat-y 0 0; position:absolute; padding:0 0 0 20px; height:10em; z-index:100; } + .projects-wrap .projects-fade-right { background:transparent url(/images/template/widgets/carousel-fade.png) repeat-y 0 100%; position:absolute; padding:0 0 0 20px; height:10em; z-index:100; } +.projects { background:none; height:10em; margin:0 451px 0 0; overflow:hidden; padding:2em 0; position:relative; } +.project { overflow:visible; height:auto; margin:0; } +.project-first { padding-left:20px; } + +/* project carousel */ +#home #sub { position:relative; } + +/* browser specific css */ +.projects { + border-radius:0; + -moz-border-radius:0; + -webkit-border-radius:0; +} +.projects-wrap { + border-radius:10px; + -moz-border-radius:10px; + -webkit-border-radius:10px; + + border-top-right-radius:0; + -moz-border-top-right-radius:0; + -webkit-border-top-right-radius:0; + + border-bottom-right-radius:0; + -moz-border-bottom-right-radius:0; + -webkit-border-bottom-right-radius:0; +} + +/* debug */ +/*#sub { display:none; }*/ diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.pipeline/out b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.pipeline/out new file mode 100644 index 0000000000..77ca415398 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.pipeline/out @@ -0,0 +1,78 @@ +FileAnalysis::TRIGGER_NEW +aFQKI8SPOL2, 0, 0 +FileAnalysis::TRIGGER_BOF +FileAnalysis::TRIGGER_BOF_BUFFER +/*^J******** +FileAnalysis::TRIGGER_TYPE +file type is set +mime type is set +FileAnalysis::TRIGGER_EOF +aFQKI8SPOL2, 2675, 0 +[orig_h=192.168.1.104, orig_p=1673/tcp, resp_h=63.245.209.11, resp_p=80/tcp] +source: HTTP +SHA1: 0e42ae17eea9b074981bd3a34535ad3a22d02706 +MD5: b932c3310ce47e158d1a5a42e0b01279 +SHA256: 5b037a2c5e36f56e63a3012c73e46a04b27741d8ff8f8b62c832fb681fc60f42 +FileAnalysis::TRIGGER_NEW +CCU3vUEr06l, 0, 0 +FileAnalysis::TRIGGER_BOF +FileAnalysis::TRIGGER_BOF_BUFFER +//-- Google +FileAnalysis::TRIGGER_TYPE +file type is set +mime type is set +FileAnalysis::TRIGGER_EOF +CCU3vUEr06l, 21421, 0 +[orig_h=192.168.1.104, orig_p=1673/tcp, resp_h=63.245.209.11, resp_p=80/tcp] +source: HTTP +SHA1: 8f241117afaa8ca5f41dc059e66d75c283dcc983 +MD5: e732f7bf1d7cb4eedcb1661697d7bc8c +SHA256: 6a509fd05aa7c8fa05080198894bb19e638554ffcee0e0b3d7bc8ff54afee1da +FileAnalysis::TRIGGER_NEW +HCzA0dVwDPj, 0, 0 +FileAnalysis::TRIGGER_BOF +FileAnalysis::TRIGGER_BOF_BUFFER +GIF89a^D\0^D\0\xb3 +FileAnalysis::TRIGGER_TYPE +file type is set +mime type is set +FileAnalysis::TRIGGER_DONE +HCzA0dVwDPj, 94, 0 +[orig_h=192.168.1.104, orig_p=1673/tcp, resp_h=63.245.209.11, resp_p=80/tcp] +total bytes: 94 +source: HTTP +SHA1: 81f5f056ce5e97d940854bb0c48017b45dd9f15e +MD5: d903de7e30db1691d3130ba5eae6b9a7 +SHA256: 6fb22aa9d780ea63bd7a2e12b92b16fcbf1c4874f1d3e11309a5ba984433c315 +FileAnalysis::TRIGGER_NEW +a1Zu1fteVEf, 0, 0 +FileAnalysis::TRIGGER_BOF +FileAnalysis::TRIGGER_BOF_BUFFER +\x89PNG^M^J^Z^J\0\0\0 +FileAnalysis::TRIGGER_TYPE +file type is set +mime type is set +FileAnalysis::TRIGGER_DONE +a1Zu1fteVEf, 2349, 0 +[orig_h=192.168.1.104, orig_p=1673/tcp, resp_h=63.245.209.11, resp_p=80/tcp] +total bytes: 2349 +source: HTTP +SHA1: 560eab5a0177246827a94042dd103916d8765ac7 +MD5: e0029eea80812e9a8e57b8d05d52938a +SHA256: e0b4500c1fd1d675da4137461cbe64d3c8489f4180d194e47683b20e7fb876f4 +FileAnalysis::TRIGGER_NEW +xXlF7wFdsR, 0, 0 +FileAnalysis::TRIGGER_BOF +FileAnalysis::TRIGGER_BOF_BUFFER +\x89PNG^M^J^Z^J\0\0\0 +FileAnalysis::TRIGGER_TYPE +file type is set +mime type is set +FileAnalysis::TRIGGER_DONE +xXlF7wFdsR, 27579, 0 +[orig_h=192.168.1.104, orig_p=1673/tcp, resp_h=63.245.209.11, resp_p=80/tcp] +total bytes: 27579 +source: HTTP +SHA1: ee2b41bdef85de14ef332da14fc392f110b84249 +MD5: 30aa926344f58019d047e85ba049ca1e +SHA256: eb482bda230a215b90aedbfe1eee72b8193608df76a319aaf11fb85511579a1e diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.pipeline/xXlF7wFdsR-file b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.pipeline/xXlF7wFdsR-file new file mode 100644 index 0000000000..f98abd969e Binary files /dev/null and b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.pipeline/xXlF7wFdsR-file differ diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.post/PZS1XGHkIf1-file b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.post/PZS1XGHkIf1-file new file mode 100644 index 0000000000..c62a7ee379 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.post/PZS1XGHkIf1-file @@ -0,0 +1,17 @@ +{ + "origin": "10.142.133.148", + "files": {}, + "form": null, + "url": "http://httpbin.org/post", + "args": {}, + "headers": { + "Content-Length": "11", + "Connection": "close", + "Accept": "*/*", + "User-Agent": "curl/7.29.0", + "Host": "httpbin.org", + "Content-Type": "application/x-www-form-urlencoded" + }, + "json": null, + "data": "hello world" +} \ No newline at end of file diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.post/out b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.post/out new file mode 100644 index 0000000000..e6d417be47 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.post/out @@ -0,0 +1,32 @@ +FileAnalysis::TRIGGER_NEW +v5HLI7MxPQh, 0, 0 +FileAnalysis::TRIGGER_BOF +FileAnalysis::TRIGGER_BOF_BUFFER +hello world +FileAnalysis::TRIGGER_TYPE +file type is set +mime type is set +FileAnalysis::TRIGGER_DONE +v5HLI7MxPQh, 11, 0 +[orig_h=141.142.228.5, orig_p=53595/tcp, resp_h=54.243.55.129, resp_p=80/tcp] +total bytes: 11 +source: HTTP +SHA1: 2aae6c35c94fcfb415dbe95f408b9ce91ee846ed +MD5: 5eb63bbbe01eeed093cb22bb8f5acdc3 +SHA256: b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9 +FileAnalysis::TRIGGER_NEW +PZS1XGHkIf1, 0, 0 +FileAnalysis::TRIGGER_BOF +FileAnalysis::TRIGGER_BOF_BUFFER +{^J "origin +FileAnalysis::TRIGGER_TYPE +file type is set +mime type is set +FileAnalysis::TRIGGER_DONE +PZS1XGHkIf1, 366, 0 +[orig_h=141.142.228.5, orig_p=53595/tcp, resp_h=54.243.55.129, resp_p=80/tcp] +total bytes: 366 +source: HTTP +SHA1: 6a1582672c203210c6d18d700322060b676365e7 +MD5: c9337794df612aeaa901dcf9fa446bca +SHA256: 8eb24c16df7cb45cb6a1790b0d26ad2571f754228d0ac111b3ac59adbfecbeb8 diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.post/v5HLI7MxPQh-file b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.post/v5HLI7MxPQh-file new file mode 100644 index 0000000000..95d09f2b10 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.post/v5HLI7MxPQh-file @@ -0,0 +1 @@ +hello world \ No newline at end of file diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.input.basic/bro..stdout b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.input.basic/bro..stdout new file mode 100644 index 0000000000..cc19da8e34 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.input.basic/bro..stdout @@ -0,0 +1,14 @@ +FileAnalysis::TRIGGER_NEW +nYgPNGLrZf9, 0, 0 +FileAnalysis::TRIGGER_BOF +FileAnalysis::TRIGGER_BOF_BUFFER +#separator +FileAnalysis::TRIGGER_TYPE +file type is set +mime type is set +FileAnalysis::TRIGGER_EOF +nYgPNGLrZf9, 311, 0 +source: ../input.log +SHA1: 0a0f20de89c86d7bce1301af6548d6e9ae87b0f1 +MD5: bf4dfa6169b74146da5236e918743599 +SHA256: 4e573192c5ea75da72494812fe24dae53a577837b2079df012fd464903d68a6f diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.irc/out b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.irc/out new file mode 100644 index 0000000000..739ed1fbcc --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.irc/out @@ -0,0 +1,15 @@ +FileAnalysis::TRIGGER_NEW +wqKMAamJVSb, 0, 0 +FileAnalysis::TRIGGER_BOF +FileAnalysis::TRIGGER_BOF_BUFFER +PK^C^D^T\0\0\0^H\0\xae +FileAnalysis::TRIGGER_TYPE +file type is set +mime type is set +FileAnalysis::TRIGGER_EOF +wqKMAamJVSb, 42208, 0 +[orig_h=192.168.1.77, orig_p=57655/tcp, resp_h=209.197.168.151, resp_p=1024/tcp] +source: IRC_DATA +SHA1: 8abe0239263fd7326eb803d4465cf494f8bea218 +MD5: 8c0803242f549c2780cb88b9a9215c65 +SHA256: e4f0b0b9d7580e7a22dc1093c8db4df7d0115a4f3b03cc2875cc69705f0d0204 diff --git a/testing/btest/Baseline/scripts.base.protocols.irc.dcc-extract/irc-dcc-item_192.168.1.77:57655-209.197.168.151:1024_1.dat b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.irc/thefile similarity index 100% rename from testing/btest/Baseline/scripts.base.protocols.irc.dcc-extract/irc-dcc-item_192.168.1.77:57655-209.197.168.151:1024_1.dat rename to testing/btest/Baseline/scripts.base.frameworks.file-analysis.irc/thefile diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.logging/file_analysis.log b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.logging/file_analysis.log new file mode 100644 index 0000000000..4cd85b8277 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.logging/file_analysis.log @@ -0,0 +1,10 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path file_analysis +#open 2013-03-26-20-26-26 +#fields file_id parent_file_id source last_active seen_bytes total_bytes missing_bytes overflow_bytes timeout_interval bof_buffer_size file_type mime_type conn_uids actions_taken extracted_files md5 sha1 sha256 +#types string string string time count count count count interval count string string table[string] table[enum] table[string] string string string +Cx92a0ym5R8 - HTTP 1362692527.009775 4705 4705 0 0 120.000000 1024 set set UWkUyAuUGXf FileAnalysis::ACTION_SHA1,FileAnalysis::ACTION_EXTRACT,FileAnalysis::ACTION_DATA_EVENT,FileAnalysis::ACTION_MD5,FileAnalysis::ACTION_SHA256 Cx92a0ym5R8-file 397168fd09991a0e712254df7bc639ac 1dd7ac0398df6cbc0696445a91ec681facf4dc47 4e7c7ef0984119447e743e3ec77e1de52713e345cde03fe7df753a35849bed18 +#close 2013-03-26-20-26-26 diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.smtp/out b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.smtp/out new file mode 100644 index 0000000000..70dc926ff5 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.smtp/out @@ -0,0 +1,45 @@ +FileAnalysis::TRIGGER_NEW +cwR7l6Zctxb, 0, 0 +FileAnalysis::TRIGGER_BOF +FileAnalysis::TRIGGER_BOF_BUFFER +Hello^M^J^M^J ^M +FileAnalysis::TRIGGER_TYPE +file type is set +mime type is set +FileAnalysis::TRIGGER_EOF +cwR7l6Zctxb, 79, 0 +[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp] +source: SMTP +SHA1: b7e497be8a9f5e2c4b6980fceb015360f98f4a13 +MD5: 92bca2e6cdcde73647125da7dccbdd07 +SHA256: 785a8a044d1454ec88837108f443bbb30cc4f529393ffd57118261036bfe59f5 +FileAnalysis::TRIGGER_NEW +ZAOEQmRyxv1, 0, 0 +FileAnalysis::TRIGGER_BOF +FileAnalysis::TRIGGER_BOF_BUFFER + + + + + + + + + + + +
+ +

Hello

+ +

 

+ +

I send u smtp pcap file

+ +

Find the attachment

+ +

 

+ +

GPS

+ +
+ + + + + diff --git a/testing/btest/Baseline/scripts.base.protocols.smtp.mime-extract/smtp-entity_10.10.1.4:1470-74.53.140.153:25_2.dat b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.smtp/thefile2 similarity index 100% rename from testing/btest/Baseline/scripts.base.protocols.smtp.mime-extract/smtp-entity_10.10.1.4:1470-74.53.140.153:25_2.dat rename to testing/btest/Baseline/scripts.base.frameworks.file-analysis.smtp/thefile2 diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.tableevent/out b/testing/btest/Baseline/scripts.base.frameworks.input.tableevent/out index d76c63ef31..54048a86b8 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.input.tableevent/out +++ b/testing/btest/Baseline/scripts.base.frameworks.input.tableevent/out @@ -1,189 +1,21 @@ -[source=../input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, name=input, destination={ -[2] = T, -[4] = F, -[6] = F, -[7] = T, -[1] = T, -[5] = F, -[3] = F -}, idx=, val=, want_record=F, ev=line -{ -print outfile, description; -print outfile, tpe; -print outfile, left; -print outfile, right; -try = try + 1; -if (7 == try) -{ -close(outfile); -terminate(); -} - -}, pred=, config={ - -}] Input::EVENT_NEW [i=1] T -[source=../input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, name=input, destination={ -[2] = T, -[4] = F, -[6] = F, -[7] = T, -[1] = T, -[5] = F, -[3] = F -}, idx=, val=, want_record=F, ev=line -{ -print outfile, description; -print outfile, tpe; -print outfile, left; -print outfile, right; -try = try + 1; -if (7 == try) -{ -close(outfile); -terminate(); -} - -}, pred=, config={ - -}] Input::EVENT_NEW [i=2] T -[source=../input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, name=input, destination={ -[2] = T, -[4] = F, -[6] = F, -[7] = T, -[1] = T, -[5] = F, -[3] = F -}, idx=, val=, want_record=F, ev=line -{ -print outfile, description; -print outfile, tpe; -print outfile, left; -print outfile, right; -try = try + 1; -if (7 == try) -{ -close(outfile); -terminate(); -} - -}, pred=, config={ - -}] Input::EVENT_NEW [i=3] F -[source=../input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, name=input, destination={ -[2] = T, -[4] = F, -[6] = F, -[7] = T, -[1] = T, -[5] = F, -[3] = F -}, idx=, val=, want_record=F, ev=line -{ -print outfile, description; -print outfile, tpe; -print outfile, left; -print outfile, right; -try = try + 1; -if (7 == try) -{ -close(outfile); -terminate(); -} - -}, pred=, config={ - -}] Input::EVENT_NEW [i=4] F -[source=../input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, name=input, destination={ -[2] = T, -[4] = F, -[6] = F, -[7] = T, -[1] = T, -[5] = F, -[3] = F -}, idx=, val=, want_record=F, ev=line -{ -print outfile, description; -print outfile, tpe; -print outfile, left; -print outfile, right; -try = try + 1; -if (7 == try) -{ -close(outfile); -terminate(); -} - -}, pred=, config={ - -}] Input::EVENT_NEW [i=5] F -[source=../input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, name=input, destination={ -[2] = T, -[4] = F, -[6] = F, -[7] = T, -[1] = T, -[5] = F, -[3] = F -}, idx=, val=, want_record=F, ev=line -{ -print outfile, description; -print outfile, tpe; -print outfile, left; -print outfile, right; -try = try + 1; -if (7 == try) -{ -close(outfile); -terminate(); -} - -}, pred=, config={ - -}] Input::EVENT_NEW [i=6] F -[source=../input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, name=input, destination={ -[2] = T, -[4] = F, -[6] = F, -[7] = T, -[1] = T, -[5] = F, -[3] = F -}, idx=, val=, want_record=F, ev=line -{ -print outfile, description; -print outfile, tpe; -print outfile, left; -print outfile, right; -try = try + 1; -if (7 == try) -{ -close(outfile); -terminate(); -} - -}, pred=, config={ - -}] Input::EVENT_NEW [i=7] T diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape-odd-url/http.log b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape-odd-url/http.log index f1ff4db3b8..6a58e612c8 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape-odd-url/http.log +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape-odd-url/http.log @@ -3,8 +3,8 @@ #empty_field (empty) #unset_field - #path http -#open 2011-09-12-03-57-36 +#open 2013-03-22-14-38-21 #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file -#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string file -1315799856.264750 UWkUyAuUGXf 10.0.1.104 64216 193.40.5.162 80 1 GET lepo.it.da.ut.ee /~cect/teoreetilised seminarid_2010/arheoloogia_uurimisr\xfchma_seminar/Joyce et al - The Languages of Archaeology ~ Dialogue, Narrative and Writing.pdf - Wget/1.12 (darwin10.8.0) 0 346 404 Not Found - - - (empty) - - - text/html - - -#close 2011-09-12-03-57-37 +#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string string +1315799856.264750 UWkUyAuUGXf 10.0.1.104 64216 193.40.5.162 80 1 GET lepo.it.da.ut.ee /~cect/teoreetilised seminarid_2010/arheoloogia_uurimisr\xfchma_seminar/Joyce et al - The Languages of Archaeology ~ Dialogue, Narrative and Writing.pdf - Wget/1.12 (darwin10.8.0) 0 346 404 Not Found - - - (empty) - - - text/html; charset=iso-8859-1 - - +#close 2013-03-22-14-38-21 diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.options/ssh.ds.xml b/testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.options/ssh.ds.xml index cacc3b0ea4..a1e65c254e 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.options/ssh.ds.xml +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.options/ssh.ds.xml @@ -1,4 +1,4 @@ - + diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.rotate/out b/testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.rotate/out index 1e5e1b05c6..94f25c37f4 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.rotate/out +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.rotate/out @@ -19,7 +19,7 @@ test.2011-03-07-12-00-05.ds test 11-03-07_12.00.05 11-03-07_12.59.55 1 dataserie - + @@ -47,7 +47,7 @@ t id.orig_h id.orig_p id.resp_h id.resp_p - + @@ -75,7 +75,7 @@ t id.orig_h id.orig_p id.resp_h id.resp_p - + @@ -103,7 +103,7 @@ t id.orig_h id.orig_p id.resp_h id.resp_p - + @@ -131,7 +131,7 @@ t id.orig_h id.orig_p id.resp_h id.resp_p - + @@ -159,7 +159,7 @@ t id.orig_h id.orig_p id.resp_h id.resp_p - + @@ -187,7 +187,7 @@ t id.orig_h id.orig_p id.resp_h id.resp_p - + @@ -215,7 +215,7 @@ t id.orig_h id.orig_p id.resp_h id.resp_p - + @@ -243,7 +243,7 @@ t id.orig_h id.orig_p id.resp_h id.resp_p - + @@ -271,7 +271,7 @@ t id.orig_h id.orig_p id.resp_h id.resp_p - + diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.test-logging/ssh.ds.txt b/testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.test-logging/ssh.ds.txt index e6abc3f1f6..225217faea 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.test-logging/ssh.ds.txt +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.test-logging/ssh.ds.txt @@ -8,7 +8,7 @@ - + diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.time-as-int/conn.ds.txt b/testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.time-as-int/conn.ds.txt index 5d95fdc0c0..bd3d367632 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.time-as-int/conn.ds.txt +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.time-as-int/conn.ds.txt @@ -8,7 +8,7 @@ - + diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.wikipedia/conn.ds.txt b/testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.wikipedia/conn.ds.txt index 5af6f702b8..a85ae29346 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.wikipedia/conn.ds.txt +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.wikipedia/conn.ds.txt @@ -8,7 +8,7 @@ - + diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.wikipedia/http.ds.txt b/testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.wikipedia/http.ds.txt index 756a49a427..bcfad8fd80 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.wikipedia/http.ds.txt +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.wikipedia/http.ds.txt @@ -8,7 +8,7 @@ - + @@ -61,7 +61,7 @@ - + # Extent, type='http' ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.writer-path-conflict/http.log b/testing/btest/Baseline/scripts.base.frameworks.logging.writer-path-conflict/http.log index 97273995bc..5d707d5cb8 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.writer-path-conflict/http.log +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.writer-path-conflict/http.log @@ -3,9 +3,9 @@ #empty_field (empty) #unset_field - #path http -#open 2011-03-18-19-06-08 +#open 2013-03-22-14-38-24 #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file -#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string file +#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string string 1300475168.784020 j4u32Pc5bif 141.142.220.118 48649 208.80.152.118 80 1 GET bits.wikimedia.org /skins-1.5/monobook/main.css http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified - - - (empty) - - - - - - 1300475168.916018 VW0XPVINV8a 141.142.220.118 49997 208.80.152.3 80 1 GET upload.wikimedia.org /wikipedia/commons/6/63/Wikipedia-logo.png http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified - - - (empty) - - - - - - 1300475168.916183 3PKsZ2Uye21 141.142.220.118 49996 208.80.152.3 80 1 GET upload.wikimedia.org /wikipedia/commons/thumb/b/bb/Wikipedia_wordmark.svg/174px-Wikipedia_wordmark.svg.png http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified - - - (empty) - - - - - - @@ -20,4 +20,4 @@ 1300475169.014619 Tw8jXtpTGu6 141.142.220.118 50000 208.80.152.3 80 2 GET upload.wikimedia.org /wikipedia/commons/thumb/4/4a/Commons-logo.svg/35px-Commons-logo.svg.png http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified - - - (empty) - - - - - - 1300475169.014593 P654jzLoe3a 141.142.220.118 49999 208.80.152.3 80 2 GET upload.wikimedia.org /wikipedia/commons/thumb/9/91/Wikiversity-logo.svg/35px-Wikiversity-logo.svg.png http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified - - - (empty) - - - - - - 1300475169.014927 0Q4FH8sESw5 141.142.220.118 50001 208.80.152.3 80 2 GET upload.wikimedia.org /wikipedia/commons/thumb/7/75/Wikimedia_Community_Logo.svg/35px-Wikimedia_Community_Logo.svg.png http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified - - - (empty) - - - - - - -#close 2011-03-18-19-06-13 +#close 2013-03-22-14-38-24 diff --git a/testing/btest/Baseline/scripts.base.frameworks.metrics.cluster-intermediate-update/manager-1.notice.log b/testing/btest/Baseline/scripts.base.frameworks.metrics.cluster-intermediate-update/manager-1.notice.log index 217b3ed49b..c87853e2b4 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.metrics.cluster-intermediate-update/manager-1.notice.log +++ b/testing/btest/Baseline/scripts.base.frameworks.metrics.cluster-intermediate-update/manager-1.notice.log @@ -3,8 +3,8 @@ #empty_field (empty) #unset_field - #path notice -#open 2012-07-20-01-50-59 -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto note msg sub src dst p n peer_descr actions policy_items suppress_for dropped remote_location.country_code remote_location.region remote_location.city remote_location.latitude remote_location.longitude metric_index.host metric_index.str metric_index.network -#types time string addr port addr port enum enum string string addr addr port count string table[enum] table[count] interval bool string string string double double addr string subnet -1342749059.978651 - - - - - - Test_Notice Threshold crossed by metric_index(host=1.2.3.4) 100/100 - 1.2.3.4 - - 100 manager-1 Notice::ACTION_LOG 6 3600.000000 F - - - - - 1.2.3.4 - - -#close 2012-07-20-01-51-08 +#open 2013-02-11-18-41-03 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto note msg sub src dst p n peer_descr actions suppress_for dropped remote_location.country_code remote_location.region remote_location.city remote_location.latitude remote_location.longitude metric_index.host metric_index.str metric_index.network +#types time string addr port addr port enum enum string string addr addr port count string table[enum] interval bool string string string double double addr string subnet +1360608063.517719 - - - - - - Test_Notice Threshold crossed by metric_index(host=1.2.3.4) 100/100 - 1.2.3.4 - - 100 manager-1 Notice::ACTION_LOG 3600.000000 F - - - - - 1.2.3.4 - - +#close 2013-02-11-18-41-03 diff --git a/testing/btest/Baseline/scripts.base.frameworks.notice.cluster/manager-1.notice.log b/testing/btest/Baseline/scripts.base.frameworks.notice.cluster/manager-1.notice.log index 6c93cb875e..ddbb59c565 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.notice.cluster/manager-1.notice.log +++ b/testing/btest/Baseline/scripts.base.frameworks.notice.cluster/manager-1.notice.log @@ -3,8 +3,8 @@ #empty_field (empty) #unset_field - #path notice -#open 2012-07-20-01-51-18 -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto note msg sub src dst p n peer_descr actions policy_items suppress_for dropped remote_location.country_code remote_location.region remote_location.city remote_location.latitude remote_location.longitude metric_index.host metric_index.str metric_index.network -#types time string addr port addr port enum enum string string addr addr port count string table[enum] table[count] interval bool string string string double double addr string subnet -1342749078.270791 - - - - - - Test_Notice test notice! - - - - - worker-1 Notice::ACTION_LOG 6 3600.000000 F - - - - - - - - -#close 2012-07-20-01-51-27 +#open 2013-02-11-18-45-43 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto note msg sub src dst p n peer_descr actions suppress_for dropped remote_location.country_code remote_location.region remote_location.city remote_location.latitude remote_location.longitude metric_index.host metric_index.str metric_index.network +#types time string addr port addr port enum enum string string addr addr port count string table[enum] interval bool string string string double double addr string subnet +1360608343.088948 - - - - - - Test_Notice test notice! - - - - - worker-1 Notice::ACTION_LOG 3600.000000 F - - - - - - - - +#close 2013-02-11-18-45-43 diff --git a/testing/btest/Baseline/scripts.base.frameworks.notice.suppression-cluster/manager-1.notice.log b/testing/btest/Baseline/scripts.base.frameworks.notice.suppression-cluster/manager-1.notice.log index 88f25b066f..2f163a5491 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.notice.suppression-cluster/manager-1.notice.log +++ b/testing/btest/Baseline/scripts.base.frameworks.notice.suppression-cluster/manager-1.notice.log @@ -3,8 +3,8 @@ #empty_field (empty) #unset_field - #path notice -#open 2012-07-20-01-51-36 -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto note msg sub src dst p n peer_descr actions policy_items suppress_for dropped remote_location.country_code remote_location.region remote_location.city remote_location.latitude remote_location.longitude metric_index.host metric_index.str metric_index.network -#types time string addr port addr port enum enum string string addr addr port count string table[enum] table[count] interval bool string string string double double addr string subnet -1342749096.545663 - - - - - - Test_Notice test notice! - - - - - worker-2 Notice::ACTION_LOG 6 3600.000000 F - - - - - - - - -#close 2012-07-20-01-51-45 +#open 2013-02-11-18-45-14 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto note msg sub src dst p n peer_descr actions suppress_for dropped remote_location.country_code remote_location.region remote_location.city remote_location.latitude remote_location.longitude metric_index.host metric_index.str metric_index.network +#types time string addr port addr port enum enum string string addr addr port count string table[enum] interval bool string string string double double addr string subnet +1360608314.794257 - - - - - - Test_Notice test notice! - - - - - worker-2 Notice::ACTION_LOG 3600.000000 F - - - - - - - - +#close 2013-02-11-18-45-17 diff --git a/testing/btest/Baseline/scripts.base.frameworks.notice.suppression/notice.log b/testing/btest/Baseline/scripts.base.frameworks.notice.suppression/notice.log index 7c7254f87e..f7f4148548 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.notice.suppression/notice.log +++ b/testing/btest/Baseline/scripts.base.frameworks.notice.suppression/notice.log @@ -3,8 +3,8 @@ #empty_field (empty) #unset_field - #path notice -#open 2012-07-20-01-49-23 -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto note msg sub src dst p n peer_descr actions policy_items suppress_for dropped remote_location.country_code remote_location.region remote_location.city remote_location.latitude remote_location.longitude -#types time string addr port addr port enum enum string string addr addr port count string table[enum] table[count] interval bool string string string double double -1342748963.685754 - - - - - - Test_Notice test - - - - - bro Notice::ACTION_LOG 6 3600.000000 F - - - - - -#close 2012-07-20-01-49-23 +#open 2013-02-11-18-32-39 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto note msg sub src dst p n peer_descr actions suppress_for dropped remote_location.country_code remote_location.region remote_location.city remote_location.latitude remote_location.longitude +#types time string addr port addr port enum enum string string addr addr port count string table[enum] interval bool string string string double double +1360607559.193954 - - - - - - Test_Notice test - - - - - bro Notice::ACTION_LOG 3600.000000 F - - - - - +#close 2013-02-11-18-32-39 diff --git a/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-extract/conn.log b/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-extract/conn.log new file mode 100644 index 0000000000..52f7d90401 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-extract/conn.log @@ -0,0 +1,14 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path conn +#open 2013-03-27-17-47-03 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto service duration orig_bytes resp_bytes conn_state local_orig missed_bytes history orig_pkts orig_ip_bytes resp_pkts resp_ip_bytes tunnel_parents +#types time string addr port addr port enum string interval count count string bool count string count count count count table[string] +1329843175.736107 arKYeMETxOg 141.142.220.235 37604 199.233.217.249 56666 tcp ftp-data 0.112432 0 342 SF - 0 ShAdfFa 4 216 4 562 (empty) +1329843179.871641 k6kgXLOoSKl 141.142.220.235 59378 199.233.217.249 56667 tcp ftp-data 0.111218 0 77 SF - 0 ShAdfFa 4 216 4 297 (empty) +1329843194.151526 nQcgTWjvg4c 199.233.217.249 61920 141.142.220.235 33582 tcp ftp-data 0.056211 342 0 SF - 0 ShADaFf 5 614 3 164 (empty) +1329843197.783443 j4u32Pc5bif 199.233.217.249 61918 141.142.220.235 37835 tcp ftp-data 0.056005 77 0 SF - 0 ShADaFf 5 349 3 164 (empty) +1329843161.968492 UWkUyAuUGXf 141.142.220.235 50003 199.233.217.249 21 tcp ftp 38.055625 180 3146 SF - 0 ShAdDfFa 38 2164 25 4458 (empty) +#close 2013-03-27-17-47-03 diff --git a/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-extract/ftp-item-BTsa70Ua9x7-1.dat b/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-extract/ftp-item-BTsa70Ua9x7-1.dat new file mode 100644 index 0000000000..a59965e6f6 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-extract/ftp-item-BTsa70Ua9x7-1.dat @@ -0,0 +1,5 @@ +User-agent: * +Disallow: *.tgz +Disallow: *.gz +Disallow: *.tbz +Disallow: *.bz2 diff --git a/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-extract/ftp-item-Rqjkzoroau4-0.dat b/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-extract/ftp-item-Rqjkzoroau4-0.dat new file mode 100644 index 0000000000..8bd2e31300 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-extract/ftp-item-Rqjkzoroau4-0.dat @@ -0,0 +1,6 @@ +total 98028 +lrwxrwxr-x 1 root wheel 32 Aug 16 2009 .message -> pub/NetBSD/README.export-control +drwxr-x--x 3 root wheel 512 Aug 16 2009 etc +-rw-rw-r-- 1 600 netbsd 50158695 Feb 21 03:10 ls-lRA.gz +drwxr-xr-x 7 root wheel 512 Aug 20 2009 pub +-rw-rw-r-- 1 root wheel 77 Aug 16 2009 robots.txt diff --git a/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-extract/ftp-item-VLQvJybrm38-2.dat b/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-extract/ftp-item-VLQvJybrm38-2.dat new file mode 100644 index 0000000000..8bd2e31300 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-extract/ftp-item-VLQvJybrm38-2.dat @@ -0,0 +1,6 @@ +total 98028 +lrwxrwxr-x 1 root wheel 32 Aug 16 2009 .message -> pub/NetBSD/README.export-control +drwxr-x--x 3 root wheel 512 Aug 16 2009 etc +-rw-rw-r-- 1 600 netbsd 50158695 Feb 21 03:10 ls-lRA.gz +drwxr-xr-x 7 root wheel 512 Aug 20 2009 pub +-rw-rw-r-- 1 root wheel 77 Aug 16 2009 robots.txt diff --git a/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-extract/ftp-item-zrfwSs9K1yk-3.dat b/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-extract/ftp-item-zrfwSs9K1yk-3.dat new file mode 100644 index 0000000000..a59965e6f6 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-extract/ftp-item-zrfwSs9K1yk-3.dat @@ -0,0 +1,5 @@ +User-agent: * +Disallow: *.tgz +Disallow: *.gz +Disallow: *.tbz +Disallow: *.bz2 diff --git a/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-extract/ftp.log b/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-extract/ftp.log new file mode 100644 index 0000000000..89526602c2 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-extract/ftp.log @@ -0,0 +1,21 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path ftp +#open 2013-03-27-17-47-03 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p user password command arg mime_type mime_desc file_size reply_code reply_msg tags data_channel.passive data_channel.orig_h data_channel.resp_h data_channel.resp_p extraction_file +#types time string addr port addr port string string string string string string count count string table[string] bool addr addr port string +1329843175.680248 UWkUyAuUGXf 141.142.220.235 50003 199.233.217.249 21 anonymous test PASV - - - - 227 Entering Passive Mode (199,233,217,249,221,90) (empty) T 141.142.220.235 199.233.217.249 56666 - +1329843175.791528 UWkUyAuUGXf 141.142.220.235 50003 199.233.217.249 21 anonymous test LIST - - - - 226 Transfer complete. (empty) - - - - - +1329843179.815947 UWkUyAuUGXf 141.142.220.235 50003 199.233.217.249 21 anonymous test PASV - - - - 227 Entering Passive Mode (199,233,217,249,221,91) (empty) T 141.142.220.235 199.233.217.249 56667 - +1329843193.984222 arKYeMETxOg 141.142.220.235 37604 199.233.217.249 56666 - - - - - - - - (empty) - - - - ftp-item-Rqjkzoroau4-0.dat +1329843193.984222 k6kgXLOoSKl 141.142.220.235 59378 199.233.217.249 56667 - - - - - - - - (empty) - - - - ftp-item-BTsa70Ua9x7-1.dat +1329843179.926563 UWkUyAuUGXf 141.142.220.235 50003 199.233.217.249 21 anonymous test RETR ftp://199.233.217.249/./robots.txt text/plain ASCII text 77 226 Transfer complete. (empty) - - - - - +1329843194.040188 UWkUyAuUGXf 141.142.220.235 50003 199.233.217.249 21 anonymous test PORT 141,142,220,235,131,46 - - - 200 PORT command successful. (empty) F 199.233.217.249 141.142.220.235 33582 - +1329843194.095782 UWkUyAuUGXf 141.142.220.235 50003 199.233.217.249 21 anonymous test LIST - - - - 226 Transfer complete. (empty) - - - - - +1329843197.672179 UWkUyAuUGXf 141.142.220.235 50003 199.233.217.249 21 anonymous test PORT 141,142,220,235,147,203 - - - 200 PORT command successful. (empty) F 199.233.217.249 141.142.220.235 37835 - +1329843199.968212 nQcgTWjvg4c 199.233.217.249 61920 141.142.220.235 33582 - - - - - - - - (empty) - - - - ftp-item-VLQvJybrm38-2.dat +1329843197.727769 UWkUyAuUGXf 141.142.220.235 50003 199.233.217.249 21 anonymous test RETR ftp://199.233.217.249/./robots.txt text/plain ASCII text, with CRLF line terminators 77 226 Transfer complete. (empty) - - - - - +1329843200.079930 j4u32Pc5bif 199.233.217.249 61918 141.142.220.235 37835 - - - - - - - - (empty) - - - - ftp-item-zrfwSs9K1yk-3.dat +#close 2013-03-27-17-47-03 diff --git a/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-ipv4/ftp.log b/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-ipv4/ftp.log index 0d0a8f57f1..6b05d924d3 100644 --- a/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-ipv4/ftp.log +++ b/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-ipv4/ftp.log @@ -3,9 +3,13 @@ #empty_field (empty) #unset_field - #path ftp -#open 2012-02-21-16-53-13 -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p user password command arg mime_type mime_desc file_size reply_code reply_msg tags extraction_file -#types time string addr port addr port string string string string string string count count string table[string] file -1329843179.926563 UWkUyAuUGXf 141.142.220.235 50003 199.233.217.249 21 anonymous test RETR ftp://199.233.217.249/./robots.txt text/plain ASCII text 77 226 Transfer complete. - - -1329843197.727769 UWkUyAuUGXf 141.142.220.235 50003 199.233.217.249 21 anonymous test RETR ftp://199.233.217.249/./robots.txt text/plain ASCII text, with CRLF line terminators 77 226 Transfer complete. - - -#close 2012-02-21-16-53-20 +#open 2013-03-27-17-47-22 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p user password command arg mime_type mime_desc file_size reply_code reply_msg tags data_channel.passive data_channel.orig_h data_channel.resp_h data_channel.resp_p extraction_file +#types time string addr port addr port string string string string string string count count string table[string] bool addr addr port string +1329843175.680248 UWkUyAuUGXf 141.142.220.235 50003 199.233.217.249 21 anonymous test PASV - - - - 227 Entering Passive Mode (199,233,217,249,221,90) (empty) T 141.142.220.235 199.233.217.249 56666 - +1329843179.815947 UWkUyAuUGXf 141.142.220.235 50003 199.233.217.249 21 anonymous test PASV - - - - 227 Entering Passive Mode (199,233,217,249,221,91) (empty) T 141.142.220.235 199.233.217.249 56667 - +1329843179.926563 UWkUyAuUGXf 141.142.220.235 50003 199.233.217.249 21 anonymous test RETR ftp://199.233.217.249/./robots.txt text/plain ASCII text 77 226 Transfer complete. (empty) - - - - - +1329843194.040188 UWkUyAuUGXf 141.142.220.235 50003 199.233.217.249 21 anonymous test PORT 141,142,220,235,131,46 - - - 200 PORT command successful. (empty) F 199.233.217.249 141.142.220.235 33582 - +1329843197.672179 UWkUyAuUGXf 141.142.220.235 50003 199.233.217.249 21 anonymous test PORT 141,142,220,235,147,203 - - - 200 PORT command successful. (empty) F 199.233.217.249 141.142.220.235 37835 - +1329843197.727769 UWkUyAuUGXf 141.142.220.235 50003 199.233.217.249 21 anonymous test RETR ftp://199.233.217.249/./robots.txt text/plain ASCII text, with CRLF line terminators 77 226 Transfer complete. (empty) - - - - - +#close 2013-03-27-17-47-22 diff --git a/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-ipv6/ftp.log b/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-ipv6/ftp.log index 62ea4df18d..ea0c07a0c9 100644 --- a/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-ipv6/ftp.log +++ b/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-ipv6/ftp.log @@ -3,9 +3,14 @@ #empty_field (empty) #unset_field - #path ftp -#open 2012-02-15-17-43-07 -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p user password command arg mime_type mime_desc file_size reply_code reply_msg tags extraction_file -#types time string addr port addr port string string string string string string count count string table[string] file -1329327787.396984 UWkUyAuUGXf 2001:470:1f11:81f:c999:d94:aa7c:2e3e 49185 2001:470:4867:99::21 21 anonymous test RETR ftp://[2001:470:4867:99::21]/robots.txt - - 77 226 Transfer complete. - - -1329327795.463946 UWkUyAuUGXf 2001:470:1f11:81f:c999:d94:aa7c:2e3e 49185 2001:470:4867:99::21 21 anonymous test RETR ftp://[2001:470:4867:99::21]/robots.txt - - 77 226 Transfer complete. - - -#close 2012-02-15-17-43-24 +#open 2013-03-27-17-50-35 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p user password command arg mime_type mime_desc file_size reply_code reply_msg tags data_channel.passive data_channel.orig_h data_channel.resp_h data_channel.resp_p extraction_file +#types time string addr port addr port string string string string string string count count string table[string] bool addr addr port string +1329327783.207785 UWkUyAuUGXf 2001:470:1f11:81f:c999:d94:aa7c:2e3e 49185 2001:470:4867:99::21 21 anonymous test EPSV - - - - 229 Entering Extended Passive Mode (|||57086|) (empty) T 2001:470:1f11:81f:c999:d94:aa7c:2e3e 2001:470:4867:99::21 57086 - +1329327786.415755 UWkUyAuUGXf 2001:470:1f11:81f:c999:d94:aa7c:2e3e 49185 2001:470:4867:99::21 21 anonymous test EPSV - - - - 229 Entering Extended Passive Mode (|||57087|) (empty) T 2001:470:1f11:81f:c999:d94:aa7c:2e3e 2001:470:4867:99::21 57087 - +1329327787.180814 UWkUyAuUGXf 2001:470:1f11:81f:c999:d94:aa7c:2e3e 49185 2001:470:4867:99::21 21 anonymous test EPSV - - - - 229 Entering Extended Passive Mode (|||57088|) (empty) T 2001:470:1f11:81f:c999:d94:aa7c:2e3e 2001:470:4867:99::21 57088 - +1329327787.396984 UWkUyAuUGXf 2001:470:1f11:81f:c999:d94:aa7c:2e3e 49185 2001:470:4867:99::21 21 anonymous test RETR ftp://[2001:470:4867:99::21]/robots.txt - - 77 226 Transfer complete. (empty) - - - - - +1329327795.355248 UWkUyAuUGXf 2001:470:1f11:81f:c999:d94:aa7c:2e3e 49185 2001:470:4867:99::21 21 anonymous test EPRT |2|2001:470:1f11:81f:c999:d94:aa7c:2e3e|49189| - - - 200 EPRT command successful. (empty) F 2001:470:4867:99::21 2001:470:1f11:81f:c999:d94:aa7c:2e3e 49189 - +1329327795.463946 UWkUyAuUGXf 2001:470:1f11:81f:c999:d94:aa7c:2e3e 49185 2001:470:4867:99::21 21 anonymous test RETR ftp://[2001:470:4867:99::21]/robots.txt - - 77 226 Transfer complete. (empty) - - - - - +1329327799.799327 UWkUyAuUGXf 2001:470:1f11:81f:c999:d94:aa7c:2e3e 49185 2001:470:4867:99::21 21 anonymous test EPRT |2|2001:470:1f11:81f:c999:d94:aa7c:2e3e|49190| - - - 200 EPRT command successful. (empty) F 2001:470:4867:99::21 2001:470:1f11:81f:c999:d94:aa7c:2e3e 49190 - +#close 2013-03-27-17-50-35 diff --git a/testing/btest/Baseline/scripts.base.protocols.ftp.gridftp/notice.log b/testing/btest/Baseline/scripts.base.protocols.ftp.gridftp/notice.log index f9292344a8..da5489e0b7 100644 --- a/testing/btest/Baseline/scripts.base.protocols.ftp.gridftp/notice.log +++ b/testing/btest/Baseline/scripts.base.protocols.ftp.gridftp/notice.log @@ -3,8 +3,8 @@ #empty_field (empty) #unset_field - #path notice -#open 2012-10-05-21-45-15 -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto note msg sub src dst p n peer_descr actions policy_items suppress_for dropped remote_location.country_code remote_location.region remote_location.city remote_location.latitude remote_location.longitude metric_index.host metric_index.str metric_index.network -#types time string addr port addr port enum enum string string addr addr port count string table[enum] table[count] interval bool string string string double double addr string subnet -1348168976.558309 arKYeMETxOg 192.168.57.103 35391 192.168.57.101 55968 tcp GridFTP::Data_Channel GridFTP data channel over threshold 2 bytes - 192.168.57.103 192.168.57.101 55968 - bro Notice::ACTION_LOG 6 3600.000000 F - - - - - - - - -#close 2012-10-05-21-45-15 +#open 2013-02-11-18-33-41 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto note msg sub src dst p n peer_descr actions suppress_for dropped remote_location.country_code remote_location.region remote_location.city remote_location.latitude remote_location.longitude metric_index.host metric_index.str metric_index.network +#types time string addr port addr port enum enum string string addr addr port count string table[enum] interval bool string string string double double addr string subnet +1348168976.558309 arKYeMETxOg 192.168.57.103 35391 192.168.57.101 55968 tcp GridFTP::Data_Channel GridFTP data channel over threshold 2 bytes - 192.168.57.103 192.168.57.101 55968 - bro Notice::ACTION_LOG 3600.000000 F - - - - - - - - +#close 2013-02-11-18-33-41 diff --git a/testing/btest/Baseline/scripts.base.protocols.http.100-continue/http.log b/testing/btest/Baseline/scripts.base.protocols.http.100-continue/http.log index 13c8b12502..f6920ac6b3 100644 --- a/testing/btest/Baseline/scripts.base.protocols.http.100-continue/http.log +++ b/testing/btest/Baseline/scripts.base.protocols.http.100-continue/http.log @@ -3,8 +3,8 @@ #empty_field (empty) #unset_field - #path http -#open 2009-03-19-05-21-36 +#open 2013-03-22-14-38-28 #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file -#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string file -1237440095.634312 UWkUyAuUGXf 192.168.3.103 54102 128.146.216.51 80 1 POST www.osu.edu / - curl/7.17.1 (i386-apple-darwin8.11.1) libcurl/7.17.1 zlib/1.2.3 2001 60731 200 OK 100 Continue - (empty) - - - text/html - - -#close 2009-03-19-05-21-36 +#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string string +1237440095.634312 UWkUyAuUGXf 192.168.3.103 54102 128.146.216.51 80 1 POST www.osu.edu / - curl/7.17.1 (i386-apple-darwin8.11.1) libcurl/7.17.1 zlib/1.2.3 2001 60731 200 OK 100 Continue - (empty) - - - text/html; charset=us-ascii - - +#close 2013-03-22-14-38-28 diff --git a/testing/btest/Baseline/scripts.base.protocols.http.http-extract-files/http-item_141.42.64.125:56730-125.190.109.199:80_resp_1.dat b/testing/btest/Baseline/scripts.base.protocols.http.http-extract-files/http-item-BFymS6bFgT3-0.dat similarity index 100% rename from testing/btest/Baseline/scripts.base.protocols.http.http-extract-files/http-item_141.42.64.125:56730-125.190.109.199:80_resp_1.dat rename to testing/btest/Baseline/scripts.base.protocols.http.http-extract-files/http-item-BFymS6bFgT3-0.dat diff --git a/testing/btest/Baseline/scripts.base.protocols.http.http-extract-files/http.log b/testing/btest/Baseline/scripts.base.protocols.http.http-extract-files/http.log index 0d61a6c8b3..f42a66f796 100644 --- a/testing/btest/Baseline/scripts.base.protocols.http.http-extract-files/http.log +++ b/testing/btest/Baseline/scripts.base.protocols.http.http-extract-files/http.log @@ -3,8 +3,8 @@ #empty_field (empty) #unset_field - #path http -#open 2005-10-07-23-23-56 +#open 2013-03-22-14-38-28 #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file -#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string file -1128727435.634189 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - (empty) - - - text/html - http-item_141.42.64.125:56730-125.190.109.199:80_resp_1.dat -#close 2005-10-07-23-23-57 +#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string string +1128727435.634189 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - (empty) - - - text/html; charset=us-ascii - http-item-BFymS6bFgT3-0.dat +#close 2013-03-22-14-38-28 diff --git a/testing/btest/Baseline/scripts.base.protocols.http.http-methods/http.log b/testing/btest/Baseline/scripts.base.protocols.http.http-methods/http.log index ac84e5ae7a..bdc29bd6b9 100644 --- a/testing/btest/Baseline/scripts.base.protocols.http.http-methods/http.log +++ b/testing/btest/Baseline/scripts.base.protocols.http.http-methods/http.log @@ -3,56 +3,56 @@ #empty_field (empty) #unset_field - #path http -#open 2012-12-07-04-43-19 +#open 2013-03-25-20-20-22 #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file -#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string file -1354328870.191989 UWkUyAuUGXf 128.2.6.136 46562 173.194.75.103 80 1 OPTIONS www.google.com * - - 0 962 405 Method Not Allowed - - - (empty) - - - text/html - - -1354328874.237327 arKYeMETxOg 128.2.6.136 46563 173.194.75.103 80 1 OPTIONS www.google.com HTTP/1.1 - - 0 925 400 Bad Request - - - (empty) - - - text/html - - -1354328874.299063 k6kgXLOoSKl 128.2.6.136 46564 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html - - -1354328874.342591 nQcgTWjvg4c 128.2.6.136 46565 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html - - -1354328874.364020 j4u32Pc5bif 128.2.6.136 46566 173.194.75.103 80 1 GET www.google.com / - - 0 43911 200 OK - - - (empty) - - - text/html - - -1354328878.470424 TEfuqmmG4bh 128.2.6.136 46567 173.194.75.103 80 1 GET www.google.com / - - 0 43983 200 OK - - - (empty) - - - text/html - - -1354328882.575456 FrJExwHcSal 128.2.6.136 46568 173.194.75.103 80 1 GET www.google.com /HTTP/1.1 - - 0 1207 403 Forbidden - - - (empty) - - - text/html - - -1354328882.928027 5OKnoww6xl4 128.2.6.136 46569 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html - - -1354328882.968948 3PKsZ2Uye21 128.2.6.136 46570 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html - - -1354328882.990373 VW0XPVINV8a 128.2.6.136 46571 173.194.75.103 80 1 GET www.google.com / - - 0 43913 200 OK - - - (empty) - - - text/html - - -1354328887.114613 fRFu0wcOle6 128.2.6.136 46572 173.194.75.103 80 0 - - - - - 0 961 405 Method Not Allowed - - - (empty) - - - text/html - - -1354328891.161077 qSsw6ESzHV4 128.2.6.136 46573 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html - - -1354328891.204740 iE6yhOq3SF 128.2.6.136 46574 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html - - -1354328891.245592 GSxOnSLghOa 128.2.6.136 46575 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html - - -1354328891.287655 qCaWGmzFtM5 128.2.6.136 46576 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html - - -1354328891.309065 70MGiRM1Qf4 128.2.6.136 46577 173.194.75.103 80 1 CCM_POST www.google.com / - - 0 963 405 Method Not Allowed - - - (empty) - - - text/html - - -1354328895.355012 h5DsfNtYzi1 128.2.6.136 46578 173.194.75.103 80 1 CCM_POST www.google.com /HTTP/1.1 - - 0 925 400 Bad Request - - - (empty) - - - text/html - - -1354328895.416133 P654jzLoe3a 128.2.6.136 46579 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html - - -1354328895.459490 Tw8jXtpTGu6 128.2.6.136 46580 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html - - -1354328895.480865 c4Zw9TmAE05 128.2.6.136 46581 173.194.75.103 80 1 CCM_POST www.google.com / - - 0 963 405 Method Not Allowed - - - (empty) - - - text/html - - -1354328899.526682 EAr0uf4mhq 128.2.6.136 46582 173.194.75.103 80 1 CONNECT www.google.com / - - 0 925 400 Bad Request - - - (empty) - - - text/html - - -1354328903.572533 GvmoxJFXdTa 128.2.6.136 46583 173.194.75.103 80 1 CONNECT www.google.com /HTTP/1.1 - - 0 925 400 Bad Request - - - (empty) - - - text/html - - -1354328903.634196 0Q4FH8sESw5 128.2.6.136 46584 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html - - -1354328903.676395 slFea8xwSmb 128.2.6.136 46585 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html - - -1354328903.697693 UfGkYA2HI2g 128.2.6.136 46586 173.194.75.103 80 1 CONNECT www.google.com / - - 0 925 400 Bad Request - - - (empty) - - - text/html - - -1354328907.743696 i2rO3KD1Syg 128.2.6.136 46587 173.194.75.103 80 1 TRACE www.google.com / - - 0 960 405 Method Not Allowed - - - (empty) - - - text/html - - -1354328911.790590 2cx26uAvUPl 128.2.6.136 46588 173.194.75.103 80 1 TRACE www.google.com /HTTP/1.1 - - 0 925 400 Bad Request - - - (empty) - - - text/html - - -1354328911.853464 BWaU4aSuwkc 128.2.6.136 46589 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html - - -1354328911.897044 10XodEwRycf 128.2.6.136 46590 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html - - -1354328911.918511 zno26fFZkrh 128.2.6.136 46591 173.194.75.103 80 1 TRACE www.google.com / - - 0 960 405 Method Not Allowed - - - (empty) - - - text/html - - -1354328915.964678 v5rgkJBig5l 128.2.6.136 46592 173.194.75.103 80 1 DELETE www.google.com / - - 0 961 405 Method Not Allowed - - - (empty) - - - text/html - - -1354328920.010458 eWZCH7OONC1 128.2.6.136 46593 173.194.75.103 80 1 DELETE www.google.com /HTTP/1.1 - - 0 925 400 Bad Request - - - (empty) - - - text/html - - -1354328920.072101 0Pwk3ntf8O3 128.2.6.136 46594 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html - - -1354328920.114526 0HKorjr8Zp7 128.2.6.136 46595 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html - - -1354328920.136714 yC2d6kVg709 128.2.6.136 46596 173.194.75.103 80 1 DELETE www.google.com / - - 0 961 405 Method Not Allowed - - - (empty) - - - text/html - - -1354328924.183211 VcgagLjnO92 128.2.6.136 46597 173.194.75.103 80 1 PUT www.google.com / - - 0 934 411 Length Required - - - (empty) - - - text/html - - -1354328924.224567 bdRoHfaPBo3 128.2.6.136 46598 173.194.75.103 80 1 PUT www.google.com /HTTP/1.1 - - 0 934 411 Length Required - - - (empty) - - - text/html - - -1354328924.287402 zHqb7t7kv28 128.2.6.136 46599 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html - - -1354328924.328257 rrZWoMUQpv8 128.2.6.136 46600 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html - - -1354328924.350343 xNYSS2hJkle 128.2.6.136 46601 173.194.75.103 80 1 PUT www.google.com / - - 0 934 411 Length Required - - - (empty) - - - text/html - - -1354328924.391728 vMVjlplKKbd 128.2.6.136 46602 173.194.75.103 80 1 POST www.google.com / - - 0 934 411 Length Required - - - (empty) - - - text/html - - -1354328924.433150 3omNawSNrxj 128.2.6.136 46603 173.194.75.103 80 1 POST www.google.com /HTTP/1.1 - - 0 934 411 Length Required - - - (empty) - - - text/html - - -1354328924.496732 Rv8AJVfi9Zi 128.2.6.136 46604 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html - - -1354328924.537671 wEyF3OvvcQe 128.2.6.136 46605 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html - - +#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string string +1354328870.191989 UWkUyAuUGXf 128.2.6.136 46562 173.194.75.103 80 1 OPTIONS www.google.com * - - 0 962 405 Method Not Allowed - - - (empty) - - - text/html; charset=utf-8 - - +1354328874.237327 arKYeMETxOg 128.2.6.136 46563 173.194.75.103 80 1 OPTIONS www.google.com HTTP/1.1 - - 0 925 400 Bad Request - - - (empty) - - - text/html; charset=utf-8 - - +1354328874.299063 k6kgXLOoSKl 128.2.6.136 46564 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html; charset=utf-8 - - +1354328874.342591 nQcgTWjvg4c 128.2.6.136 46565 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html; charset=utf-8 - - +1354328874.364020 j4u32Pc5bif 128.2.6.136 46566 173.194.75.103 80 1 GET www.google.com / - - 0 43911 200 OK - - - (empty) - - - text/html; charset=us-ascii - - +1354328878.470424 TEfuqmmG4bh 128.2.6.136 46567 173.194.75.103 80 1 GET www.google.com / - - 0 43983 200 OK - - - (empty) - - - text/html; charset=us-ascii - - +1354328882.575456 FrJExwHcSal 128.2.6.136 46568 173.194.75.103 80 1 GET www.google.com /HTTP/1.1 - - 0 1207 403 Forbidden - - - (empty) - - - text/html; charset=us-ascii - - +1354328882.928027 5OKnoww6xl4 128.2.6.136 46569 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html; charset=utf-8 - - +1354328882.968948 3PKsZ2Uye21 128.2.6.136 46570 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html; charset=utf-8 - - +1354328882.990373 VW0XPVINV8a 128.2.6.136 46571 173.194.75.103 80 1 GET www.google.com / - - 0 43913 200 OK - - - (empty) - - - text/html; charset=us-ascii - - +1354328887.114613 fRFu0wcOle6 128.2.6.136 46572 173.194.75.103 80 0 - - - - - 0 961 405 Method Not Allowed - - - (empty) - - - text/html; charset=utf-8 - - +1354328891.161077 qSsw6ESzHV4 128.2.6.136 46573 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html; charset=utf-8 - - +1354328891.204740 iE6yhOq3SF 128.2.6.136 46574 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html; charset=utf-8 - - +1354328891.245592 GSxOnSLghOa 128.2.6.136 46575 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html; charset=utf-8 - - +1354328891.287655 qCaWGmzFtM5 128.2.6.136 46576 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html; charset=utf-8 - - +1354328891.309065 70MGiRM1Qf4 128.2.6.136 46577 173.194.75.103 80 1 CCM_POST www.google.com / - - 0 963 405 Method Not Allowed - - - (empty) - - - text/html; charset=utf-8 - - +1354328895.355012 h5DsfNtYzi1 128.2.6.136 46578 173.194.75.103 80 1 CCM_POST www.google.com /HTTP/1.1 - - 0 925 400 Bad Request - - - (empty) - - - text/html; charset=utf-8 - - +1354328895.416133 P654jzLoe3a 128.2.6.136 46579 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html; charset=utf-8 - - +1354328895.459490 Tw8jXtpTGu6 128.2.6.136 46580 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html; charset=utf-8 - - +1354328895.480865 c4Zw9TmAE05 128.2.6.136 46581 173.194.75.103 80 1 CCM_POST www.google.com / - - 0 963 405 Method Not Allowed - - - (empty) - - - text/html; charset=utf-8 - - +1354328899.526682 EAr0uf4mhq 128.2.6.136 46582 173.194.75.103 80 1 CONNECT www.google.com / - - 0 925 400 Bad Request - - - (empty) - - - text/html; charset=utf-8 - - +1354328903.572533 GvmoxJFXdTa 128.2.6.136 46583 173.194.75.103 80 1 CONNECT www.google.com /HTTP/1.1 - - 0 925 400 Bad Request - - - (empty) - - - text/html; charset=utf-8 - - +1354328903.634196 0Q4FH8sESw5 128.2.6.136 46584 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html; charset=utf-8 - - +1354328903.676395 slFea8xwSmb 128.2.6.136 46585 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html; charset=utf-8 - - +1354328903.697693 UfGkYA2HI2g 128.2.6.136 46586 173.194.75.103 80 1 CONNECT www.google.com / - - 0 925 400 Bad Request - - - (empty) - - - text/html; charset=utf-8 - - +1354328907.743696 i2rO3KD1Syg 128.2.6.136 46587 173.194.75.103 80 1 TRACE www.google.com / - - 0 960 405 Method Not Allowed - - - (empty) - - - text/html; charset=utf-8 - - +1354328911.790590 2cx26uAvUPl 128.2.6.136 46588 173.194.75.103 80 1 TRACE www.google.com /HTTP/1.1 - - 0 925 400 Bad Request - - - (empty) - - - text/html; charset=utf-8 - - +1354328911.853464 BWaU4aSuwkc 128.2.6.136 46589 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html; charset=utf-8 - - +1354328911.897044 10XodEwRycf 128.2.6.136 46590 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html; charset=utf-8 - - +1354328911.918511 zno26fFZkrh 128.2.6.136 46591 173.194.75.103 80 1 TRACE www.google.com / - - 0 960 405 Method Not Allowed - - - (empty) - - - text/html; charset=utf-8 - - +1354328915.964678 v5rgkJBig5l 128.2.6.136 46592 173.194.75.103 80 1 DELETE www.google.com / - - 0 961 405 Method Not Allowed - - - (empty) - - - text/html; charset=utf-8 - - +1354328920.010458 eWZCH7OONC1 128.2.6.136 46593 173.194.75.103 80 1 DELETE www.google.com /HTTP/1.1 - - 0 925 400 Bad Request - - - (empty) - - - text/html; charset=utf-8 - - +1354328920.072101 0Pwk3ntf8O3 128.2.6.136 46594 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html; charset=utf-8 - - +1354328920.114526 0HKorjr8Zp7 128.2.6.136 46595 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html; charset=utf-8 - - +1354328920.136714 yC2d6kVg709 128.2.6.136 46596 173.194.75.103 80 1 DELETE www.google.com / - - 0 961 405 Method Not Allowed - - - (empty) - - - text/html; charset=utf-8 - - +1354328924.183211 VcgagLjnO92 128.2.6.136 46597 173.194.75.103 80 1 PUT www.google.com / - - 0 934 411 Length Required - - - (empty) - - - text/html; charset=utf-8 - - +1354328924.224567 bdRoHfaPBo3 128.2.6.136 46598 173.194.75.103 80 1 PUT www.google.com /HTTP/1.1 - - 0 934 411 Length Required - - - (empty) - - - text/html; charset=utf-8 - - +1354328924.287402 zHqb7t7kv28 128.2.6.136 46599 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html; charset=utf-8 - - +1354328924.328257 rrZWoMUQpv8 128.2.6.136 46600 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html; charset=utf-8 - - +1354328924.350343 xNYSS2hJkle 128.2.6.136 46601 173.194.75.103 80 1 PUT www.google.com / - - 0 934 411 Length Required - - - (empty) - - - text/html; charset=utf-8 - - +1354328924.391728 vMVjlplKKbd 128.2.6.136 46602 173.194.75.103 80 1 POST www.google.com / - - 0 934 411 Length Required - - - (empty) - - - text/html; charset=utf-8 - - +1354328924.433150 3omNawSNrxj 128.2.6.136 46603 173.194.75.103 80 1 POST www.google.com /HTTP/1.1 - - 0 934 411 Length Required - - - (empty) - - - text/html; charset=utf-8 - - +1354328924.496732 Rv8AJVfi9Zi 128.2.6.136 46604 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html; charset=utf-8 - - +1354328924.537671 wEyF3OvvcQe 128.2.6.136 46605 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html; charset=utf-8 - - 1354328924.559704 E490YZTUozc 128.2.6.136 46606 173.194.75.103 80 1 HEAD www.google.com / - - 0 0 200 OK - - - (empty) - - - - - - 1354328928.625437 YIeWJmXWNWj 128.2.6.136 46607 173.194.75.103 80 1 HEAD www.google.com / - - 0 0 200 OK - - - (empty) - - - - - - 1354328932.692706 ydiZblvsYri 128.2.6.136 46608 173.194.75.103 80 1 HEAD www.google.com /HTTP/1.1 - - 0 0 400 Bad Request - - - (empty) - - - - - - -1354328932.754657 HFYOnBqSE5e 128.2.6.136 46609 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html - - -1354328932.796568 JcUvhfWUMgd 128.2.6.136 46610 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html - - -#close 2012-12-07-04-43-19 +1354328932.754657 HFYOnBqSE5e 128.2.6.136 46609 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html; charset=utf-8 - - +1354328932.796568 JcUvhfWUMgd 128.2.6.136 46610 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html; charset=utf-8 - - +#close 2013-03-25-20-20-22 diff --git a/testing/btest/Baseline/scripts.base.protocols.http.http-mime-and-md5/http.log b/testing/btest/Baseline/scripts.base.protocols.http.http-mime-and-md5/http.log index 409d8fc812..61b1e16a2f 100644 --- a/testing/btest/Baseline/scripts.base.protocols.http.http-mime-and-md5/http.log +++ b/testing/btest/Baseline/scripts.base.protocols.http.http-mime-and-md5/http.log @@ -3,12 +3,12 @@ #empty_field (empty) #unset_field - #path http -#open 2009-11-18-20-58-04 +#open 2013-03-22-16-25-59 #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file -#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string file -1258577884.844956 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 1 GET www.mozilla.org /style/enhanced.css http://www.mozilla.org/projects/calendar/ Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 2675 200 OK - - - (empty) - - - FAKE_MIME - - -1258577884.960135 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 2 GET www.mozilla.org /script/urchin.js http://www.mozilla.org/projects/calendar/ Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 21421 200 OK - - - (empty) - - - FAKE_MIME - - -1258577885.317160 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 3 GET www.mozilla.org /images/template/screen/bullet_utility.png http://www.mozilla.org/style/screen.css Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 94 200 OK - - - (empty) - - - FAKE_MIME - - -1258577885.349639 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 4 GET www.mozilla.org /images/template/screen/key-point-top.png http://www.mozilla.org/style/screen.css Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 2349 200 OK - - - (empty) - - - image/png e0029eea80812e9a8e57b8d05d52938a - -1258577885.394612 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 5 GET www.mozilla.org /projects/calendar/images/header-sunbird.png http://www.mozilla.org/projects/calendar/calendar.css Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 27579 200 OK - - - (empty) - - - image/png 30aa926344f58019d047e85ba049ca1e - -#close 2009-11-18-20-58-32 +#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string string +1258577884.844956 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 1 GET www.mozilla.org /style/enhanced.css http://www.mozilla.org/projects/calendar/ Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 2675 200 OK - - - (empty) - - - text/plain; charset=us-ascii - - +1258577884.960135 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 2 GET www.mozilla.org /script/urchin.js http://www.mozilla.org/projects/calendar/ Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 21421 200 OK - - - (empty) - - - text/plain; charset=us-ascii - - +1258577885.317160 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 3 GET www.mozilla.org /images/template/screen/bullet_utility.png http://www.mozilla.org/style/screen.css Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 94 200 OK - - - (empty) - - - image/gif; charset=binary - - +1258577885.349639 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 4 GET www.mozilla.org /images/template/screen/key-point-top.png http://www.mozilla.org/style/screen.css Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 2349 200 OK - - - (empty) - - - image/png; charset=binary e0029eea80812e9a8e57b8d05d52938a - +1258577885.394612 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 5 GET www.mozilla.org /projects/calendar/images/header-sunbird.png http://www.mozilla.org/projects/calendar/calendar.css Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 27579 200 OK - - - (empty) - - - image/png; charset=binary 30aa926344f58019d047e85ba049ca1e - +#close 2013-03-22-16-25-59 diff --git a/testing/btest/Baseline/scripts.base.protocols.http.http-pipelining/http.log b/testing/btest/Baseline/scripts.base.protocols.http.http-pipelining/http.log index 6b5e395902..d7791097a9 100644 --- a/testing/btest/Baseline/scripts.base.protocols.http.http-pipelining/http.log +++ b/testing/btest/Baseline/scripts.base.protocols.http.http-pipelining/http.log @@ -3,12 +3,12 @@ #empty_field (empty) #unset_field - #path http -#open 2009-11-18-20-58-04 +#open 2013-03-22-14-38-28 #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied md5 extraction_file -#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string file +#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string 1258577884.844956 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 1 GET www.mozilla.org /style/enhanced.css http://www.mozilla.org/projects/calendar/ Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 2675 200 OK - - - (empty) - - - - - 1258577884.960135 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 2 GET www.mozilla.org /script/urchin.js http://www.mozilla.org/projects/calendar/ Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 21421 200 OK - - - (empty) - - - - - 1258577885.317160 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 3 GET www.mozilla.org /images/template/screen/bullet_utility.png http://www.mozilla.org/style/screen.css Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 94 200 OK - - - (empty) - - - - - 1258577885.349639 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 4 GET www.mozilla.org /images/template/screen/key-point-top.png http://www.mozilla.org/style/screen.css Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 2349 200 OK - - - (empty) - - - - - 1258577885.394612 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 5 GET www.mozilla.org /projects/calendar/images/header-sunbird.png http://www.mozilla.org/projects/calendar/calendar.css Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 27579 200 OK - - - (empty) - - - - - -#close 2009-11-18-20-58-32 +#close 2013-03-22-14-38-28 diff --git a/testing/btest/Baseline/scripts.base.protocols.irc.basic/irc.log b/testing/btest/Baseline/scripts.base.protocols.irc.basic/irc.log index 46adaa4c3e..64bdb41861 100644 --- a/testing/btest/Baseline/scripts.base.protocols.irc.basic/irc.log +++ b/testing/btest/Baseline/scripts.base.protocols.irc.basic/irc.log @@ -3,11 +3,11 @@ #empty_field (empty) #unset_field - #path irc -#open 2011-07-20-19-12-44 +#open 2013-03-27-18-51-40 #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p nick user command value addl dcc_file_name dcc_file_size extraction_file -#types time string addr port addr port string string string string string string count file +#types time string addr port addr port string string string string string string count string 1311189164.119437 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 - - NICK bloed - - - - 1311189164.119437 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 bloed - USER sdkfje sdkfje Montreal.QC.CA.Undernet.org dkdkrwq - - - 1311189174.474127 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 bloed sdkfje JOIN #easymovies (empty) - - - 1311189316.326025 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 bloed sdkfje DCC #easymovies (empty) ladyvampress-default(2011-07-07)-OS.zip 42208 - -#close 2011-07-20-19-15-42 +#close 2013-03-27-18-51-40 diff --git a/testing/btest/Baseline/scripts.base.protocols.irc.dcc-extract/irc-dcc-item-wqKMAamJVSb-0.dat b/testing/btest/Baseline/scripts.base.protocols.irc.dcc-extract/irc-dcc-item-wqKMAamJVSb-0.dat new file mode 100644 index 0000000000..d4ec9e374b Binary files /dev/null and b/testing/btest/Baseline/scripts.base.protocols.irc.dcc-extract/irc-dcc-item-wqKMAamJVSb-0.dat differ diff --git a/testing/btest/Baseline/scripts.base.protocols.irc.dcc-extract/irc.log b/testing/btest/Baseline/scripts.base.protocols.irc.dcc-extract/irc.log index e204a627b1..4e70587ff0 100644 --- a/testing/btest/Baseline/scripts.base.protocols.irc.dcc-extract/irc.log +++ b/testing/btest/Baseline/scripts.base.protocols.irc.dcc-extract/irc.log @@ -3,11 +3,11 @@ #empty_field (empty) #unset_field - #path irc -#open 2011-07-20-19-12-44 +#open 2013-03-27-18-49-16 #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p nick user command value addl dcc_file_name dcc_file_size dcc_mime_type extraction_file -#types time string addr port addr port string string string string string string count string file +#types time string addr port addr port string string string string string string count string string 1311189164.119437 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 - - NICK bloed - - - - - 1311189164.119437 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 bloed - USER sdkfje sdkfje Montreal.QC.CA.Undernet.org dkdkrwq - - - - 1311189174.474127 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 bloed sdkfje JOIN #easymovies (empty) - - - - -1311189316.326025 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 bloed sdkfje DCC #easymovies (empty) ladyvampress-default(2011-07-07)-OS.zip 42208 FAKE_MIME irc-dcc-item_192.168.1.77:57655-209.197.168.151:1024_1.dat -#close 2011-07-20-19-15-42 +1311189316.326025 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 bloed sdkfje DCC #easymovies (empty) ladyvampress-default(2011-07-07)-OS.zip 42208 FAKE_MIME irc-dcc-item-wqKMAamJVSb-0.dat +#close 2013-03-27-18-49-16 diff --git a/testing/btest/Baseline/scripts.base.protocols.smtp.mime-extract/smtp-entity-Ltd7QO7jEv3-1.dat b/testing/btest/Baseline/scripts.base.protocols.smtp.mime-extract/smtp-entity-Ltd7QO7jEv3-1.dat new file mode 100644 index 0000000000..9eb3055735 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.protocols.smtp.mime-extract/smtp-entity-Ltd7QO7jEv3-1.dat @@ -0,0 +1,264 @@ +Version 4.9.9.1 +* Many bug fixes +* Improved editor + +Version 4.9.9.0 +* Support for latest Mingw compiler system builds +* Bug fixes + +Version 4.9.8.9 +* New code tooltip display +* Improved Indent/Unindent and Remove Comment +* Improved automatic indent +* Added support for the "interface" keyword +* WebUpdate should now report installation problems from PackMan +* New splash screen and association icons +* Improved installer +* Many bug fixes + +Version 4.9.8.7 +* Added support for GCC > 3.2 +* Debug variables are now resent during next debug session +* Watched Variables not in correct context are now kept and updated when it is needed +* Added new compiler/linker options: 20 + - Strip executable + - Generate instructions for a specific machine (i386, i486, i586, i686, pentium, pentium-mmx, pentiumpro, pentium2, pentium3, pentium4, 20 + k6, k6-2, k6-3, athlon, athlon-tbird, athlon-4, athlon-xp, athlon-mp, winchip-c6, winchip2, k8, c3 and c3-2) + - Enable use of processor specific built-in functions (mmmx, sse, sse2, pni, 3dnow) +* "Default" button in Compiler Options is back +* Error messages parsing improved +* Bug fixes + +Version 4.9.8.5 +* Added the possibility to modify the value of a variable during debugging (right click on a watch variable and select "Modify value") +* During Dev-C++ First Time COnfiguration window, users can now choose between using or not class browser and code completion features. +* Many bug fixes + +Version 4.9.8.4 +* Added the possibility to specify an include directory for the code completion cache to be created at Dev-C++ first startup +* Improved code completion cache +* WebUpdate will now backup downloaded DevPaks in Dev-C++\Packages directory, and Dev-C++ executable in devcpp.exe.BACKUP +* Big speed up in function parameters listing while editing +* Bug fixes + +Version 4.9.8.3 +* On Dev-C++ first time configuration dialog, a code completion cache of all the standard 20 + include files can now be generated. +* Improved WebUpdate module +* Many bug fixes + +Version 4.9.8.2 +* New debug feature for DLLs: attach to a running process +* New project option: Use custom Makefile. 20 +* New WebUpdater module. +* Allow user to specify an alternate configuration file in Environment Options 20 + (still can be overriden by using "-c" command line parameter). +* Lots of bug fixes. + +Version 4.9.8.1 +* When creating a DLL, the created static lib respects now the project-defined output directory + +Version 4.9.8.0 +* Changed position of compiler/linker parameters in Project Options. +* Improved help file +* Bug fixes + +Version 4.9.7.9 +* Resource errors are now reported in the Resource sheet +* Many bug fixes + +Version 4.9.7.8 +* Made whole bottom report control floating instead of only debug output. +* Many bug fixes + +Version 4.9.7.7 +* Printing settings are now saved +* New environment options : "watch variable under mouse" and "Report watch errors" +* Bug fixes + +Version 4.9.7.6 +* Debug variable browser +* Added possibility to include in a Template the Project's directories (include, libs and ressources) +* Changed tint of Class browser pictures colors to match the New Look style +* Bug fixes + +Version 4.9.7.5 +* Bug fixes + +Version 4.9.7.4 +* When compiling with debugging symbols, an extra definition is passed to the + compiler: -D__DEBUG__ +* Each project creates a _private.h file containing version + information definitions +* When compiling the current file only, no dependency checks are performed +* ~300% Speed-up in class parser +* Added "External programs" in Tools/Environment Options (for units "Open with") +* Added "Open with" in project units context menu +* Added "Classes" toolbar +* Fixed pre-compilation dependency checks to work correctly +* Added new file menu entry: Save Project As +* Bug-fix for double quotes in devcpp.cfg file read by vUpdate +* Other bug fixes + +Version 4.9.7.3 +* When adding debugging symbols on request, remove "-s" option from linker +* Compiling progress window +* Environment options : "Show progress window" and "Auto-close progress window" +* Bug fixes + +Version 4.9.7.2 +* Bug fixes + +Version 4.9.7.1 +* "Build priority" per-unit +* "Include file in linking process" per-unit +* New feature: compile current file only +* Separated C++ compiler options from C compiler options in Makefile (see bug report #654744) +* Separated C++ include dirs from C include dirs in Makefile (see bug report #654744) +* Necessary UI changes in Project Options +* Added display of project filename, project output and a summary of the project files in Project Options General tab. +* Fixed the "compiler-dirs-with-spaces" bug that crept-in in 4.9.7.0 +* Multi-select files in project-view (when "double-click to open" is configured in Environment Settings) +* Resource files are treated as ordinary files now +* Updates in "Project Options/Files" code +* MSVC import now creates the folders structure of the original VC project +* Bug fixes + +Version 4.9.7.0 +* Allow customizing of per-unit compile command in projects +* Added two new macros: and +* Added support for macros in the "default source code" (Tools/Editor Options/Code) +* Separated layout info from project file. It is now kept in a different file + (the same filename as the project's but with extension ".layout"). If you + have your project under CVS control, you ''ll know why this had to happen... +* Compiler settings per-project +* Compiler set per-project +* Implemented new compiler settings framework +* "Compile as C++" per-unit +* "Include file in compilation process" per-unit +* Project version info (creates the relevant VERSIONINFO struct in the private + resource) +* Support XP Themes (creates the CommonControls 6.0 manifest file and includes + it in the private resource) +* Added CVS "login" and "logout" commands +* Project manager and debugging window (in Debug tab) can now be trasnformed into floating windows. +* Added "Add Library" button in Project Options +* Bug fixes + +Version 4.9.6.9 +* Implemented search in help files for the word at cursor (context sensitive help) +* Implemented "compiler sets" infrastructure to switch between different compilers easily (e.g. gcc-2.95 and gcc-3.2) +* Added "Files" tab in CVS form to allow selection of more than one file for + the requested CVS action + 20 +Version 4.9.6.8 +* support for DLL application hosting, for debugging and executing DLLs under Dev-C++. +* New class browser option: "Show inherited members" +* Added support for the '::' member access operator in code-completion +* Added *working* function arguments hint +* Added bracket highlighting. When the caret is on a bracket, that bracket and + its counterpart are highlighted +* Nested folders in project view + +Version 4.9.6.7 +* XP Theme support +* Added CVS commands "Add" and "Remove" +* Added configuration option for "Templates Directory" in "Environment Options" +* Code-completion updates +* Bug fixes + +Version 4.9.6.6 +* Editor colors are initialized properly on Dev-C++ first-run +* Added doxygen-style comments in NewClass, NewMemberFunction and NewMemberVariable wizards +* Added file's date/time stamp in File/Properties window +* Current windows listing in Window menu +* Bug fixes + +Version 4.9.6.5 +* CVS support +* Window list (in Window menu) +* bug fixes + +version 4.9.6.4 +* added ENTER key for opening file in project browser, DEL to delete from the project. +* bug fixes + +version 4.9.6.3 +* Bug fixes + +version 4.9.6.2 +* Bug fixes + +version 4.9.6.1 +* New "Abort compilation" button +* Bug fixes +* Now checks for vRoach existance when sending a crash report + +Version 4.9.5.5 +* New option in Editor Options: Show editor hints. User can disable the hints + displayed in the editor when the mouse moves over a word. Since this was the + cause of many errors (although it should be fixed by now), we are giving the + user the option to disable this feature. +* New option in Editor Options (code-completion): Use code-completion cache. + Well, it adds caching to code-completion. Depending on the cache size, + the program may take a bit longer to start-up, but provides very fast + code-completion and the user has all the commands (belonging to the files + he added in the cache) at his fingertips. If, for example, the user adds + "windows.h", he gets all the WinAPI! If he adds "wx/wx.h", he gets all of + wxWindows! You get the picture... +* Removed "Only show classes from current file" option in class browser settings. + It used to be a checkbox, allowing only two states (on or off), but there is + a third relevant option now: "Project classes" so it didn't fit the purpose... + The user can define this in the class browser's context menu under "View mode". +* Fixed the dreaded "Clock skew detected" compiler warning! +* Fixed many class browser bugs, including some that had to do with class folders. + +Version 4.9.5.4 +* Under NT, 2000 and XP, user application data directory will be used to store config files (i.e : C:\Documents and Settings\Username\Local Settings\Application Data) + +Version 4.9.5.3 +* Added ExceptionsAnalyzer. If the devcpp.map file is in the devcpp.exe directory + then we even get a stack trace in the bug report! +* Added new WebUpdate module (inactive temporarily). +* Added new code for code-completion caching of files (disabled - work in progress). + +Version 4.9.5.2 +* Added new option in class-browser: Use colors + (available when right-clicking the class-browser + and selecting "View mode"). +* Dev-C++ now traps access violation of your programs (and of itself too ;) + +Version 4.9.5.1 +* Implemented the "File/Export/Project to HTML" function. +* Added "Tip of the day" system. +* When running a source file in explorer, don't spawn new instance. + Instead open the file in an already launched Dev-C++. +* Class-parser speed-up (50% to 85% improvement timed!!!) +* Many code-completion updates. Now takes into account context, + class inheritance and visibility (shows items only from files + #included directly or indirectly)! +* Caching of result set of code-completion for speed-up. +* New option "Execution/Parameters" (and "Debug/Parameters"). + +Version 4.9.5.0 (5.0 beta 5): +* CPU Window (still in development) +* ToDo list +* Backtrace in debugging +* Run to cursor +* Folders in Project and Class Browser +* Send custom commands to GDB +* Makefile can now be customized. +* Modified the behaviour of the -c param : 20 + -c +* Saving of custom syntax parameter group +* Possibility of changing compilers and tools filename. +* Many bug fixes + + +Version 4.9.4.1 (5.0 beta 4.1): + +* back to gcc 2.95.3 +* Profiling support +* new update/packages checker (vUpdate) +* Lots of bugfixes + diff --git a/testing/btest/Baseline/scripts.base.protocols.smtp.mime-extract/smtp-entity-cwR7l6Zctxb-0.dat b/testing/btest/Baseline/scripts.base.protocols.smtp.mime-extract/smtp-entity-cwR7l6Zctxb-0.dat new file mode 100644 index 0000000000..f4dd7d22f4 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.protocols.smtp.mime-extract/smtp-entity-cwR7l6Zctxb-0.dat @@ -0,0 +1,13 @@ +Hello + + + +I send u smtp pcap file + +Find the attachment + + + +GPS + + diff --git a/testing/btest/Baseline/scripts.base.protocols.smtp.mime-extract/smtp_entities.log b/testing/btest/Baseline/scripts.base.protocols.smtp.mime-extract/smtp_entities.log index 396a2e058d..0ee8782362 100644 --- a/testing/btest/Baseline/scripts.base.protocols.smtp.mime-extract/smtp_entities.log +++ b/testing/btest/Baseline/scripts.base.protocols.smtp.mime-extract/smtp_entities.log @@ -3,10 +3,10 @@ #empty_field (empty) #unset_field - #path smtp_entities -#open 2009-10-05-06-06-10 +#open 2013-03-26-20-43-14 #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth filename content_len mime_type md5 extraction_file excerpt -#types time string addr port addr port count string count string string file string -1254722770.692743 arKYeMETxOg 10.10.1.4 1470 74.53.140.153 25 1 - 79 FAKE_MIME - smtp-entity_10.10.1.4:1470-74.53.140.153:25_1.dat (empty) -1254722770.692743 arKYeMETxOg 10.10.1.4 1470 74.53.140.153 25 1 - 1918 FAKE_MIME - - (empty) -1254722770.692804 arKYeMETxOg 10.10.1.4 1470 74.53.140.153 25 1 NEWS.txt 10823 FAKE_MIME - smtp-entity_10.10.1.4:1470-74.53.140.153:25_2.dat (empty) -#close 2009-10-05-06-06-16 +#types time string addr port addr port count string count string string string string +1254722770.692743 arKYeMETxOg 10.10.1.4 1470 74.53.140.153 25 1 - 79 text/plain; charset=us-ascii - smtp-entity-cwR7l6Zctxb-0.dat (empty) +1254722770.692743 arKYeMETxOg 10.10.1.4 1470 74.53.140.153 25 1 - 1918 text/html; charset=us-ascii - - (empty) +1254722770.692804 arKYeMETxOg 10.10.1.4 1470 74.53.140.153 25 1 NEWS.txt 10823 text/plain; charset=us-ascii - smtp-entity-Ltd7QO7jEv3-1.dat (empty) +#close 2013-03-26-20-43-14 diff --git a/testing/btest/Baseline/scripts.base.protocols.smtp.mime/smtp_entities.log b/testing/btest/Baseline/scripts.base.protocols.smtp.mime/smtp_entities.log index 1abe35e90f..0a342c48dc 100644 --- a/testing/btest/Baseline/scripts.base.protocols.smtp.mime/smtp_entities.log +++ b/testing/btest/Baseline/scripts.base.protocols.smtp.mime/smtp_entities.log @@ -3,10 +3,10 @@ #empty_field (empty) #unset_field - #path smtp_entities -#open 2009-10-05-06-06-10 +#open 2013-03-26-20-39-07 #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth filename content_len mime_type md5 extraction_file excerpt -#types time string addr port addr port count string count string string file string -1254722770.692743 arKYeMETxOg 10.10.1.4 1470 74.53.140.153 25 1 - 79 FAKE_MIME 92bca2e6cdcde73647125da7dccbdd07 - (empty) -1254722770.692743 arKYeMETxOg 10.10.1.4 1470 74.53.140.153 25 1 - 1918 FAKE_MIME - - (empty) -1254722770.692804 arKYeMETxOg 10.10.1.4 1470 74.53.140.153 25 1 NEWS.txt 10823 FAKE_MIME a968bb0f9f9d95835b2e74c845877e87 - (empty) -#close 2009-10-05-06-06-16 +#types time string addr port addr port count string count string string string string +1254722770.692743 arKYeMETxOg 10.10.1.4 1470 74.53.140.153 25 1 - 79 text/plain; charset=us-ascii 92bca2e6cdcde73647125da7dccbdd07 - (empty) +1254722770.692743 arKYeMETxOg 10.10.1.4 1470 74.53.140.153 25 1 - 1918 text/html; charset=us-ascii - - (empty) +1254722770.692804 arKYeMETxOg 10.10.1.4 1470 74.53.140.153 25 1 NEWS.txt 10823 text/plain; charset=us-ascii a968bb0f9f9d95835b2e74c845877e87 - (empty) +#close 2013-03-26-20-39-07 diff --git a/testing/btest/Baseline/scripts.policy.protocols.ssl.extract-certs-pem/certs-remote.pem b/testing/btest/Baseline/scripts.policy.protocols.ssl.extract-certs-pem/certs-remote.pem new file mode 100644 index 0000000000..1767a32095 --- /dev/null +++ b/testing/btest/Baseline/scripts.policy.protocols.ssl.extract-certs-pem/certs-remote.pem @@ -0,0 +1,34 @@ +-----BEGIN CERTIFICATE----- +MIIEfDCCA+WgAwIBAgIQBKeBFvADKDvaK4RiBJ+eyzANBgkqhk +iG9w0BAQUFADCBujEfMB0GA1UEChMWVmVyaVNpZ24gVHJ1c3Qg +TmV0d29yazEXMBUGA1UECxMOVmVyaVNpZ24sIEluYy4xMzAxBg +NVBAsTKlZlcmlTaWduIEludGVybmF0aW9uYWwgU2VydmVyIENB +IC0gQ2xhc3MgMzFJMEcGA1UECxNAd3d3LnZlcmlzaWduLmNvbS +9DUFMgSW5jb3JwLmJ5IFJlZi4gTElBQklMSVRZIExURC4oYyk5 +NyBWZXJpU2lnbjAeFw0wNjExMTQwMDAwMDBaFw0wNzExMTQyMz +U5NTlaMIHAMQswCQYDVQQGEwJERTEPMA0GA1UECBMGQmF5ZXJu +MREwDwYDVQQHFAhNdWVuY2hlbjE3MDUGA1UEChQuQUdJUyBBbG +xpYW56IERyZXNkbmVyIEluZm9ybWF0aW9uc3N5c3RlbWUgR21i +SDEzMDEGA1UECxQqVGVybXMgb2YgdXNlIGF0IHd3dy52ZXJpc2 +lnbi5jb20vcnBhIChjKTAwMR8wHQYDVQQDFBZ3d3cuZHJlc2Ru +ZXItcHJpdmF0LmRlMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQ +KBgQDrqHR+++O06r6LHD3t6oYEYlHgKlqehm+Yy7zF7cXIylad +TJJY4WsTb7y35S6YQPeP1qPACqtGUhs4/AUg54Duxl3VuwP8xY +O6mmcI/Sy6owiU8LMfFij2BWZbv3+oWfq+mWs2YrhuxoNHU2MP +WrRRwYioVbnUMW09KkqVCtF7hwIDAQABo4IBeTCCAXUwCQYDVR +0TBAIwADALBgNVHQ8EBAMCBaAwRgYDVR0fBD8wPTA7oDmgN4Y1 +aHR0cDovL2NybC52ZXJpc2lnbi5jb20vQ2xhc3MzSW50ZXJuYX +Rpb25hbFNlcnZlci5jcmwwRAYDVR0gBD0wOzA5BgtghkgBhvhF +AQcXAzAqMCgGCCsGAQUFBwIBFhxodHRwczovL3d3dy52ZXJpc2 +lnbi5jb20vcnBhMCgGA1UdJQQhMB8GCWCGSAGG+EIEAQYIKwYB +BQUHAwEGCCsGAQUFBwMCMDQGCCsGAQUFBwEBBCgwJjAkBggrBg +EFBQcwAYYYaHR0cDovL29jc3AudmVyaXNpZ24uY29tMG0GCCsG +AQUFBwEMBGEwX6FdoFswWTBXMFUWCWltYWdlL2dpZjAhMB8wBw +YFKw4DAhoEFI/l0xqGrI2Oa8PPgGrUSBgsexkuMCUWI2h0dHA6 +Ly9sb2dvLnZlcmlzaWduLmNvbS92c2xvZ28uZ2lmMA0GCSqGSI +b3DQEBBQUAA4GBAC9z4m/BniN+WVCJlXhv6QS9mFRTYOwIUtIK +KZKabarVsWfBYt7JGE5XPWmcsgNmkgO76E3FmNQvQtm20uCXEF +h2z+fWp8y72yXuQl3L8HSr0lTl6LpRD6TDPjT6UvKg5nr0j9x2 +Qr09/HjAt+teLR/FoF7foBGH+MNYEMh5KPjk +-----END CERTIFICATE----- + diff --git a/testing/btest/README b/testing/btest/README index 6e8abd24ed..a635251939 100644 --- a/testing/btest/README +++ b/testing/btest/README @@ -1,7 +1,7 @@ This a test suite of small "unit tests" that verify individual pieces of Bro functionality. They all utilize BTest, a simple framework/driver for writing unit tests. More information about BTest can be found at -http://www.bro-ids.org/development/btest.html +http://www.bro.org/development/btest.html The test suite's BTest configuration is handled through the ``btest.cfg`` file. Of particular interest is the "TestDirs" settings, @@ -61,7 +61,7 @@ Adding Tests ============= See either the `BTest documentation -`_ or the existing unit +`_ or the existing unit tests for examples of what they actually look like. The essential components of a new test include: diff --git a/testing/btest/Traces/ftp-ipv4.trace b/testing/btest/Traces/ftp/ipv4.trace similarity index 100% rename from testing/btest/Traces/ftp-ipv4.trace rename to testing/btest/Traces/ftp/ipv4.trace diff --git a/testing/btest/Traces/ipv6-ftp.trace b/testing/btest/Traces/ftp/ipv6.trace similarity index 100% rename from testing/btest/Traces/ipv6-ftp.trace rename to testing/btest/Traces/ftp/ipv6.trace diff --git a/testing/btest/Traces/ftp/retr.trace b/testing/btest/Traces/ftp/retr.trace new file mode 100644 index 0000000000..a63ec5e153 Binary files /dev/null and b/testing/btest/Traces/ftp/retr.trace differ diff --git a/testing/btest/Traces/http-100-continue.trace b/testing/btest/Traces/http/100-continue.trace similarity index 100% rename from testing/btest/Traces/http-100-continue.trace rename to testing/btest/Traces/http/100-continue.trace diff --git a/testing/btest/Traces/http/206_example_a.pcap b/testing/btest/Traces/http/206_example_a.pcap new file mode 100644 index 0000000000..ca134bf701 Binary files /dev/null and b/testing/btest/Traces/http/206_example_a.pcap differ diff --git a/testing/btest/Traces/http/206_example_b.pcap b/testing/btest/Traces/http/206_example_b.pcap new file mode 100644 index 0000000000..2752452587 Binary files /dev/null and b/testing/btest/Traces/http/206_example_b.pcap differ diff --git a/testing/btest/Traces/http/206_example_c.pcap b/testing/btest/Traces/http/206_example_c.pcap new file mode 100644 index 0000000000..990de43a79 Binary files /dev/null and b/testing/btest/Traces/http/206_example_c.pcap differ diff --git a/testing/btest/Traces/http-byteranges.trace b/testing/btest/Traces/http/byteranges.trace similarity index 100% rename from testing/btest/Traces/http-byteranges.trace rename to testing/btest/Traces/http/byteranges.trace diff --git a/testing/btest/Traces/http/get-gzip.trace b/testing/btest/Traces/http/get-gzip.trace new file mode 100644 index 0000000000..8e228e4125 Binary files /dev/null and b/testing/btest/Traces/http/get-gzip.trace differ diff --git a/testing/btest/Traces/http/get.trace b/testing/btest/Traces/http/get.trace new file mode 100644 index 0000000000..f098e04ae6 Binary files /dev/null and b/testing/btest/Traces/http/get.trace differ diff --git a/testing/btest/Traces/http-methods.trace b/testing/btest/Traces/http/methods.trace similarity index 100% rename from testing/btest/Traces/http-methods.trace rename to testing/btest/Traces/http/methods.trace diff --git a/testing/btest/Traces/http-pipelined-requests.trace b/testing/btest/Traces/http/pipelined-requests.trace similarity index 100% rename from testing/btest/Traces/http-pipelined-requests.trace rename to testing/btest/Traces/http/pipelined-requests.trace diff --git a/testing/btest/Traces/http/post.trace b/testing/btest/Traces/http/post.trace new file mode 100644 index 0000000000..91a8ffce17 Binary files /dev/null and b/testing/btest/Traces/http/post.trace differ diff --git a/testing/btest/Traces/q-in-q.trace b/testing/btest/Traces/q-in-q.trace new file mode 100644 index 0000000000..39969c8063 Binary files /dev/null and b/testing/btest/Traces/q-in-q.trace differ diff --git a/testing/btest/Traces/ssl.v3.trace b/testing/btest/Traces/ssl.v3.trace new file mode 100644 index 0000000000..3bcb9729a2 Binary files /dev/null and b/testing/btest/Traces/ssl.v3.trace differ diff --git a/testing/btest/Traces/tunnels/gtp/gtp_control_prime.pcap b/testing/btest/Traces/tunnels/gtp/gtp_control_prime.pcap new file mode 100644 index 0000000000..ebed358a26 Binary files /dev/null and b/testing/btest/Traces/tunnels/gtp/gtp_control_prime.pcap differ diff --git a/testing/btest/Traces/tunnels/gtp/gtp_create_pdp_ctx.pcap b/testing/btest/Traces/tunnels/gtp/gtp_create_pdp_ctx.pcap new file mode 100644 index 0000000000..78283156d5 Binary files /dev/null and b/testing/btest/Traces/tunnels/gtp/gtp_create_pdp_ctx.pcap differ diff --git a/testing/btest/Traces/tunnels/gtp/gtp_ext_header.pcap b/testing/btest/Traces/tunnels/gtp/gtp_ext_header.pcap new file mode 100644 index 0000000000..453ebf934a Binary files /dev/null and b/testing/btest/Traces/tunnels/gtp/gtp_ext_header.pcap differ diff --git a/testing/btest/bifs/byte_len.bro b/testing/btest/bifs/byte_len.bro deleted file mode 100644 index bd15b0c390..0000000000 --- a/testing/btest/bifs/byte_len.bro +++ /dev/null @@ -1,10 +0,0 @@ -# -# @TEST-EXEC: bro -b %INPUT >out -# @TEST-EXEC: btest-diff out - -event bro_init() - { - local a = "hello\0there"; - - print byte_len(a); - } diff --git a/testing/btest/bifs/encode_base64.bro b/testing/btest/bifs/encode_base64.bro new file mode 100644 index 0000000000..a351392bb5 --- /dev/null +++ b/testing/btest/bifs/encode_base64.bro @@ -0,0 +1,14 @@ +# @TEST-EXEC: bro -b %INPUT >out +# @TEST-EXEC: btest-diff out + +global default_alphabet: string = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + +global my_alphabet: string = "!#$%&/(),-.:;<>@[]^ `_{|}~abcdefghijklmnopqrstuvwxyz0123456789+?"; + +print encode_base64("bro"); +print encode_base64_custom("bro", default_alphabet); +print encode_base64_custom("bro", my_alphabet); + +print encode_base64("padding"); +print encode_base64("padding1"); +print encode_base64("padding12"); diff --git a/testing/btest/bifs/length.bro b/testing/btest/bifs/length.bro deleted file mode 100644 index ca82d7eab7..0000000000 --- a/testing/btest/bifs/length.bro +++ /dev/null @@ -1,22 +0,0 @@ -# -# @TEST-EXEC: bro -b %INPUT > out -# @TEST-EXEC: btest-diff out - -event bro_init() - { - local mytable: table[string] of string = { ["key1"] = "val1" }; - local myset: set[count] = set( 3, 6, 2, 7 ); - local myvec: vector of string = vector( "value1", "value2" ); - - print length(mytable); - print length(myset); - print length(myvec); - - mytable = table(); - myset = set(); - myvec = vector(); - - print length(mytable); - print length(myset); - print length(myvec); - } diff --git a/testing/btest/bifs/reverse.bro b/testing/btest/bifs/reverse.bro new file mode 100644 index 0000000000..bbb386bb80 --- /dev/null +++ b/testing/btest/bifs/reverse.bro @@ -0,0 +1,19 @@ +# +# @TEST-EXEC: bro -b %INPUT >out +# @TEST-EXEC: btest-diff out + +event bro_init() + { + local s1 = "hello world!"; + local s2 = "rise to vote sir"; + local s3 = "\xff\x00"; + local s4 = "\xff\x39\x30\xff"; + + print reverse(s1); + print reverse(reverse(s1)); + print subst_string(reverse(s2), " ", ""); + print bytestring_to_hexstr(s3); + print bytestring_to_hexstr(reverse(s3)); + print bytestring_to_hexstr(reverse(sub_bytes(s4, 2, 2))); + print reverse("A"); + } diff --git a/testing/btest/core/ipv6-flow-labels.test b/testing/btest/core/ipv6-flow-labels.test index b4e60cb0a4..2265cd55d4 100644 --- a/testing/btest/core/ipv6-flow-labels.test +++ b/testing/btest/core/ipv6-flow-labels.test @@ -1,4 +1,4 @@ -# @TEST-EXEC: bro -b -r $TRACES/ipv6-ftp.trace %INPUT >output +# @TEST-EXEC: bro -b -r $TRACES/ftp/ipv6.trace %INPUT >output # @TEST-EXEC: btest-diff output function print_connection(c: connection, event_name: string) diff --git a/testing/btest/core/leaks/file-analysis-http-get.bro b/testing/btest/core/leaks/file-analysis-http-get.bro new file mode 100644 index 0000000000..ff54917e15 --- /dev/null +++ b/testing/btest/core/leaks/file-analysis-http-get.bro @@ -0,0 +1,14 @@ +# Needs perftools support. +# +# @TEST-REQUIRES: bro --help 2>&1 | grep -q mem-leaks +# +# @TEST-GROUP: leaks +# +# @TEST-EXEC: HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local bro -m -r $TRACES/http/get.trace $SCRIPTS/file-analysis-test.bro %INPUT + +redef test_file_analysis_source = "HTTP"; + +redef test_get_file_name = function(info: FileAnalysis::Info): string + { + return fmt("%s-file", info$file_id); + }; diff --git a/testing/btest/core/leaks/returnwhen.bro b/testing/btest/core/leaks/returnwhen.bro new file mode 100644 index 0000000000..9fd9a794cd --- /dev/null +++ b/testing/btest/core/leaks/returnwhen.bro @@ -0,0 +1,84 @@ +# Needs perftools support. +# +# @TEST-GROUP: leaks +# +# @TEST-REQUIRES: bro --help 2>&1 | grep -q mem-leaks +# +# @TEST-EXEC: btest-bg-run bro HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local bro -m -b %INPUT +# @TEST-EXEC: btest-bg-wait 15 + +redef exit_only_after_terminate = T; + +global my_set: set[string] = set(); +global flag: string = "flag"; +global done: bool = F; + +function dummyfunc(s: string): string + { + return "dummy " + s; + } + +function async_func(s: string): string + { + print dummyfunc("from async_func() " + s); + + return when ( flag in my_set ) + { + return flag + " in my_set"; + } + timeout 3sec + { + return "timeout"; + } + } + +event set_flag() + { + add my_set[flag]; + } + +event do_another() + { + delete my_set[flag]; + + local local_dummy = dummyfunc; + + local anon = function(s: string): string { return s + "!"; }; + + if ( ! done ) + schedule 1sec { set_flag() }; + + when ( local result = async_func("from do_another()") ) + { + print "async_func() return result in do_another()", result; + print local_dummy("from do_another() when block"); + print anon("hi"); + if ( result == "timeout" ) + terminate(); + else + { + done = T; + schedule 10msec { do_another() }; + } + } + } + +event bro_init() + { + local local_dummy = dummyfunc; + + local anon = function(s: string): string { return s + "!"; }; + + schedule 1sec { set_flag() }; + + when ( local result = async_func("from bro_init()") ) + { + print "async_func() return result in bro_init()", result; + print local_dummy("from bro_init() when block"); + print anon("hi"); + if ( result == "timeout" ) terminate(); + schedule 10msec { do_another() }; + } + } + + diff --git a/testing/btest/core/leaks/vector-val-bifs.test b/testing/btest/core/leaks/vector-val-bifs.test index d42e273bc5..775f06e987 100644 --- a/testing/btest/core/leaks/vector-val-bifs.test +++ b/testing/btest/core/leaks/vector-val-bifs.test @@ -8,7 +8,7 @@ # assuming that it didn't automatically Ref the VectorType argument and thus # leaked that memeory. # -# @TEST-EXEC: HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local bro -m -b -r $TRACES/ftp-ipv4.trace %INPUT >output +# @TEST-EXEC: HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local bro -m -b -r $TRACES/ftp/ipv4.trace %INPUT >output # @TEST-EXEC: btest-diff output function myfunc(aa: interval, bb: interval): int diff --git a/testing/btest/core/q-in-q.bro b/testing/btest/core/q-in-q.bro new file mode 100644 index 0000000000..7444e7b458 --- /dev/null +++ b/testing/btest/core/q-in-q.bro @@ -0,0 +1,2 @@ +# @TEST-EXEC: bro -r $TRACES/q-in-q.trace +# @TEST-EXEC: btest-diff conn.log diff --git a/testing/btest/core/tunnels/gtp/ext_header.test b/testing/btest/core/tunnels/gtp/ext_header.test new file mode 100644 index 0000000000..6316acb184 --- /dev/null +++ b/testing/btest/core/tunnels/gtp/ext_header.test @@ -0,0 +1,8 @@ +# @TEST-EXEC: bro -r $TRACES/tunnels/gtp/gtp_ext_header.pcap %INPUT >out +# @TEST-EXEC: btest-diff out + +event gtpv1_message(c: connection, hdr: gtpv1_hdr) + { + print "gtpv1_message", c$id; + print hdr; + } diff --git a/testing/btest/core/tunnels/gtp/pdp_ctx_messages.test b/testing/btest/core/tunnels/gtp/pdp_ctx_messages.test new file mode 100644 index 0000000000..7405c8d019 --- /dev/null +++ b/testing/btest/core/tunnels/gtp/pdp_ctx_messages.test @@ -0,0 +1,56 @@ +# @TEST-EXEC: bro -r $TRACES/tunnels/gtp/gtp_control_prime.pcap -r $TRACES/tunnels/gtp/gtp_create_pdp_ctx.pcap %INPUT >out +# @TEST-EXEC: btest-diff out + +event gtpv1_message(c: connection, hdr: gtpv1_hdr) + { + print "gtpv1_message", c$id; + print hdr; + } + +event gtpv1_create_pdp_ctx_request(c: connection, hdr: gtpv1_hdr, + elements: gtp_create_pdp_ctx_request_elements) + { + print "gtp create request", c$id; + print hdr; + print elements; + } + +event gtpv1_create_pdp_ctx_response(c: connection, hdr: gtpv1_hdr, + elements: gtp_create_pdp_ctx_response_elements) + { + print "gtp create response", c$id; + print hdr; + print elements; + } + +event gtpv1_update_pdp_ctx_request(c: connection, hdr: gtpv1_hdr, + elements: gtp_update_pdp_ctx_request_elements) + { + print "gtp update request", c$id; + print hdr; + print elements; + } + +event gtpv1_update_pdp_ctx_response(c: connection, hdr: gtpv1_hdr, + elements: gtp_update_pdp_ctx_response_elements) + { + print "gtp update response", c$id; + print hdr; + print elements; + } + +event gtpv1_delete_pdp_ctx_request(c: connection, hdr: gtpv1_hdr, + elements: gtp_delete_pdp_ctx_request_elements) + { + print "gtp delete request", c$id; + print hdr; + print elements; + } + +event gtpv1_delete_pdp_ctx_response(c: connection, hdr: gtpv1_hdr, + elements: gtp_delete_pdp_ctx_response_elements) + { + print "gtp delete response", c$id; + print hdr; + print elements; + } diff --git a/testing/btest/core/when-interpreter-exceptions.bro b/testing/btest/core/when-interpreter-exceptions.bro index b1880c5139..9b6f689dcf 100644 --- a/testing/btest/core/when-interpreter-exceptions.bro +++ b/testing/btest/core/when-interpreter-exceptions.bro @@ -1,6 +1,6 @@ # @TEST-EXEC: btest-bg-run bro bro -b --pseudo-realtime -r $TRACES/rotation.trace %INPUT # @TEST-EXEC: btest-bg-wait -k 3 -# @TEST-EXEC: TEST_DIFF_CANONIFIER="$SCRIPTS/diff-remove-abspath | $SCRIPTS/diff-remove-timestamps" btest-diff bro/.stderr +# @TEST-EXEC: TEST_DIFF_CANONIFIER="$SCRIPTS/diff-remove-abspath | $SCRIPTS/diff-remove-timestamps | $SCRIPTS/diff-sort" btest-diff bro/.stderr # @TEST-EXEC: btest-diff bro/.stdout # interpreter exceptions in "when" blocks shouldn't cause termination diff --git a/testing/btest/istate/events-ssl.bro b/testing/btest/istate/events-ssl.bro index e4440834a7..8eca37c581 100644 --- a/testing/btest/istate/events-ssl.bro +++ b/testing/btest/istate/events-ssl.bro @@ -40,7 +40,15 @@ redef tcp_close_delay = 0secs; redef ssl_ca_certificate = "../ca_cert.pem"; redef ssl_private_key = "../bro.pem"; redef ssl_passphrase = "my-password"; - + +# File analysis that populates fields in the http.log would make the sender's +# log differ from the receiver's since hooks don't get sent to peers. +hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info) + &priority=10 + { + FileAnalysis::stop(info$file_id); + } + @TEST-END-FILE ############# diff --git a/testing/btest/istate/events.bro b/testing/btest/istate/events.bro index c292f77113..2e81883a1e 100644 --- a/testing/btest/istate/events.bro +++ b/testing/btest/istate/events.bro @@ -36,6 +36,14 @@ redef peer_description = "events-send"; # it gets propagated but that's ok.) redef tcp_close_delay = 0secs; +# File analysis that populates fields in the http.log would make the sender's +# log differ from the receiver's since hooks don't get sent to peers. +hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info) + &priority=10 + { + FileAnalysis::stop(info$file_id); + } + @TEST-END-FILE ############# diff --git a/testing/btest/language/container-ctor-scope.bro b/testing/btest/language/container-ctor-scope.bro new file mode 100644 index 0000000000..fd1939a459 --- /dev/null +++ b/testing/btest/language/container-ctor-scope.bro @@ -0,0 +1,38 @@ +# @TEST-EXEC: bro -b %INPUT >out +# @TEST-EXEC: btest-diff out + +# All various container contructors should work at both global and local scope. + +global gt1: table[port] of count = table( [1/tcp] = 1, [2/tcp] = 2, [3/tcp] = 3 ); +global gs1: set[port] = set( 1/tcp, 2/tcp, 3/tcp ); +global gv1: vector of port = vector( 1/tcp, 2/tcp, 3/tcp, 1/tcp ); + +global gt2: table[port] of count = { [1/tcp] = 1, [2/tcp] = 2, [3/tcp] = 3 }; +global gs2: set[port] = { 1/tcp, 2/tcp, 3/tcp }; +global gv2: vector of port = { 1/tcp, 2/tcp, 3/tcp, 1/tcp }; + +local t1: table[port] of count = table( [1/tcp] = 1, [2/tcp] = 2, [3/tcp] = 3 ); +local s1: set[port] = set( 1/tcp, 2/tcp, 3/tcp ); +local v1: vector of port = vector( 1/tcp, 2/tcp, 3/tcp, 1/tcp ); + +local t2: table[port] of count = { [1/tcp] = 1, [2/tcp] = 2, [3/tcp] = 3 }; +local s2: set[port] = { 1/tcp, 2/tcp, 3/tcp }; +local v2: vector of port = { 1/tcp, 2/tcp, 3/tcp, 1/tcp }; + +print gt1; +print gt2; + +print gs1; +print gs2; + +print gv1; +print gv2; + +print t1; +print t2; + +print s1; +print s2; + +print v1; +print v2; diff --git a/testing/btest/language/func-assignment.bro b/testing/btest/language/func-assignment.bro new file mode 100644 index 0000000000..576d7f3270 --- /dev/null +++ b/testing/btest/language/func-assignment.bro @@ -0,0 +1,39 @@ +# @TEST-EXEC: bro -b %INPUT >out +# @TEST-EXEC: btest-diff out + +function double_string(s: string): string + { + return string_cat(s, " ", s); + } + +function triple_string(str: string): string + { + return string_cat(str, " ", str, " ", str); + } + +type sample_function: record { + s: string; + f: function(str: string): string; +}; + +event bro_init() + { + local test_sf: sample_function; + test_sf$s = "Brogrammers, like bowties, are cool."; + + test_sf$f = triple_string; + print test_sf$f(test_sf$s); + + test_sf$f = double_string; + print test_sf$f(test_sf$s); + + # Works as expected + test_sf$f = function(str: string): string + { return to_upper(str); }; + print test_sf$f(test_sf$s); + + # Func arg names shouldn't factor in to the type check. + test_sf$f = function(s: string): string + { return to_upper(s); }; + print test_sf$f(test_sf$s); + } diff --git a/testing/btest/language/returnwhen.bro b/testing/btest/language/returnwhen.bro new file mode 100644 index 0000000000..593841eb7e --- /dev/null +++ b/testing/btest/language/returnwhen.bro @@ -0,0 +1,79 @@ +# @TEST-EXEC: btest-bg-run bro bro -b %INPUT +# @TEST-EXEC: btest-bg-wait 15 +# @TEST-EXEC: btest-diff bro/.stdout + +redef exit_only_after_terminate = T; + +global my_set: set[string] = set(); +global flag: string = "flag"; +global done: bool = F; + +function dummyfunc(s: string): string + { + return "dummy " + s; + } + +function async_func(s: string): string + { + print dummyfunc("from async_func() " + s); + + return when ( flag in my_set ) + { + return flag + " in my_set"; + } + timeout 3sec + { + return "timeout"; + } + } + +event set_flag() + { + add my_set[flag]; + } + +event do_another() + { + delete my_set[flag]; + + local local_dummy = dummyfunc; + + local anon = function(s: string): string { return s + "!"; }; + + if ( ! done ) + schedule 1sec { set_flag() }; + + when ( local result = async_func("from do_another()") ) + { + print "async_func() return result in do_another()", result; + print local_dummy("from do_another() when block"); + print anon("hi"); + if ( result == "timeout" ) + terminate(); + else + { + done = T; + schedule 10msec { do_another() }; + } + } + } + +event bro_init() + { + local local_dummy = dummyfunc; + + local anon = function(s: string): string { return s + "!"; }; + + schedule 1sec { set_flag() }; + + when ( local result = async_func("from bro_init()") ) + { + print "async_func() return result in bro_init()", result; + print local_dummy("from bro_init() when block"); + print anon("hi"); + if ( result == "timeout" ) terminate(); + schedule 10msec { do_another() }; + } + } + + diff --git a/testing/btest/language/table-init-container-ctors.bro b/testing/btest/language/table-init-container-ctors.bro index 4829f41688..1f9e18d848 100644 --- a/testing/btest/language/table-init-container-ctors.bro +++ b/testing/btest/language/table-init-container-ctors.bro @@ -2,7 +2,7 @@ # @TEST-EXEC: btest-diff output # The various container constructor expressions should work in table -# initialization lists. +# initialization lists (as yields). type set_yield: set[string, count]; type vector_yield: vector of count; diff --git a/testing/btest/language/table-init-record-idx.bro b/testing/btest/language/table-init-record-idx.bro new file mode 100644 index 0000000000..db9716dc42 --- /dev/null +++ b/testing/btest/language/table-init-record-idx.bro @@ -0,0 +1,216 @@ +# @TEST-EXEC: bro -b %INPUT >output +# @TEST-EXEC: btest-diff output + +# Record constructors should work in table initializers + +type r: record { + a: string; + b: count; +}; + +global a: r = [$a="foo", $b=1]; +global b: r = [$a="foo", $b=2]; +global c: r = [$a="bar", $b=3]; +global d: r = [$a="bar", $b=4]; +global e: r = [$a="baz", $b=5]; +global f: r = [$a="baz", $b=6]; + +global foo: table[r] of count = { + [a] = 1, + [record($a="foo", $b=2)] = 2, + [[$a="bar", $b=3]] = 3, +}; + +foo[d] = 4; +foo[[$a="baz", $b=5]] = 5; +foo[record($a="baz", $b=6)] = 6; + +print "following should all be true..."; + +print a in foo; +print b in foo; +print c in foo; +print d in foo; +print e in foo; +print f in foo; + +print [$a="foo", $b=1] in foo; +print record($a="foo", $b=1) in foo; + +print foo[a]; +print foo[[$a="foo", $b=1]]; +print foo[record($a="foo", $b=1)]; + +print "following should all be false..."; + +local bah: r = [$a="bah", $b=0]; + +print bah in foo; +print [$a="bah", $b=0] in foo; +print record($a="bah", $b=0) in foo; + +print "now here's the foo table..."; + +print foo; + +# @TEST-START-NEXT + +# They can be part of a compound index type, too... + +type r: record { + a: string; + b: count; +}; + +global a: r = [$a="foo", $b=1]; +global b: r = [$a="foo", $b=2]; +global c: r = [$a="bar", $b=3]; +global d: r = [$a="bar", $b=4]; +global e: r = [$a="baz", $b=5]; +global f: r = [$a="baz", $b=6]; + +global foo: table[r, count] of count = { + [a, 1] = 1, + [record($a="foo", $b=2), 2] = 2, + [[$a="bar", $b=3], 3] = 3, +}; + +foo[d, 4] = 4; +foo[[$a="baz", $b=5], 5] = 5; +foo[record($a="baz", $b=6), 6] = 6; + +print "following should all be true..."; + +print [a, 1] in foo; +print [b, 2] in foo; +print [c, 3] in foo; +print [d, 4] in foo; +print [e, 5] in foo; +print [f, 6] in foo; + +print [[$a="foo", $b=1], 1] in foo; +print [record($a="foo", $b=1), 1] in foo; + +print foo[a, 1]; +print foo[[$a="foo", $b=1], 1]; +print foo[record($a="foo", $b=1), 1]; + +print "following should all be false..."; + +local bah: r = [$a="bah", $b=0]; + +print [bah, 0] in foo; +print [[$a="bah", $b=0], 0] in foo; +print [record($a="bah", $b=0), 0] in foo; + +print "now here's the foo table..."; + +print foo; + +# @TEST-START-NEXT + +# Now checking table() ctor versus { } initializer + +type r: record { + a: string; + b: count; +}; + +global a: r = [$a="foo", $b=1]; +global b: r = [$a="foo", $b=2]; +global c: r = [$a="bar", $b=3]; +global d: r = [$a="bar", $b=4]; +global e: r = [$a="baz", $b=5]; +global f: r = [$a="baz", $b=6]; + +global foo: table[r] of count = table( + [a] = 1, + [record($a="foo", $b=2)] = 2, + [[$a="bar", $b=3]] = 3 +); + +foo[d] = 4; +foo[[$a="baz", $b=5]] = 5; +foo[record($a="baz", $b=6)] = 6; + +print "following should all be true..."; + +print a in foo; +print b in foo; +print c in foo; +print d in foo; +print e in foo; +print f in foo; + +print [$a="foo", $b=1] in foo; +print record($a="foo", $b=1) in foo; + +print foo[a]; +print foo[[$a="foo", $b=1]]; +print foo[record($a="foo", $b=1)]; + +print "following should all be false..."; + +local bah: r = [$a="bah", $b=0]; + +print bah in foo; +print [$a="bah", $b=0] in foo; +print record($a="bah", $b=0) in foo; + +print "now here's the foo table..."; + +print foo; + +# @TEST-START-NEXT + +# Now checking table() ctor versus { } initializer for compound index + +type r: record { + a: string; + b: count; +}; + +global a: r = [$a="foo", $b=1]; +global b: r = [$a="foo", $b=2]; +global c: r = [$a="bar", $b=3]; +global d: r = [$a="bar", $b=4]; +global e: r = [$a="baz", $b=5]; +global f: r = [$a="baz", $b=6]; + +global foo: table[r, count] of count = table( + [a, 1] = 1, + [record($a="foo", $b=2), 2] = 2, + [[$a="bar", $b=3], 3] = 3 +); + +foo[d, 4] = 4; +foo[[$a="baz", $b=5], 5] = 5; +foo[record($a="baz", $b=6), 6] = 6; + +print "following should all be true..."; + +print [a, 1] in foo; +print [b, 2] in foo; +print [c, 3] in foo; +print [d, 4] in foo; +print [e, 5] in foo; +print [f, 6] in foo; + +print [[$a="foo", $b=1], 1] in foo; +print [record($a="foo", $b=1), 1] in foo; + +print foo[a, 1]; +print foo[[$a="foo", $b=1], 1]; +print foo[record($a="foo", $b=1), 1]; + +print "following should all be false..."; + +local bah: r = [$a="bah", $b=0]; + +print [bah, 0] in foo; +print [[$a="bah", $b=0], 0] in foo; +print [record($a="bah", $b=0), 0] in foo; + +print "now here's the foo table..."; + +print foo; diff --git a/testing/btest/scripts/base/frameworks/file-analysis/actions/data_event.bro b/testing/btest/scripts/base/frameworks/file-analysis/actions/data_event.bro new file mode 100644 index 0000000000..bcecbd8aa3 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/file-analysis/actions/data_event.bro @@ -0,0 +1,4 @@ +# @TEST-EXEC: bro -r $TRACES/http/get.trace $SCRIPTS/file-analysis-test.bro %INPUT >out +# @TEST-EXEC: btest-diff out + +redef test_print_file_data_events = T; diff --git a/testing/btest/scripts/base/frameworks/file-analysis/bifs/postpone_timeout.bro b/testing/btest/scripts/base/frameworks/file-analysis/bifs/postpone_timeout.bro new file mode 100644 index 0000000000..a19fd621a4 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/file-analysis/bifs/postpone_timeout.bro @@ -0,0 +1,30 @@ +# @TEST-EXEC: btest-bg-run bro bro -r $TRACES/http/206_example_b.pcap $SCRIPTS/file-analysis-test.bro %INPUT +# @TEST-EXEC: btest-bg-wait 8 +# @TEST-EXEC: btest-diff bro/.stdout + +global cnt: count = 0; +global timeout_cnt: count = 0; + +redef FileAnalysis::default_timeout_interval=2sec; + +redef test_file_analysis_source = "HTTP"; + +redef test_get_file_name = function(info: FileAnalysis::Info): string + { + local rval: string = fmt("%s-file%d", info$file_id, cnt); + ++cnt; + return rval; + }; + +redef exit_only_after_terminate = T; + +hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info) + { + if ( trig != FileAnalysis::TRIGGER_TIMEOUT ) return; + + if ( timeout_cnt < 1 ) + FileAnalysis::postpone_timeout(info$file_id); + else + terminate(); + ++timeout_cnt; + } diff --git a/testing/btest/scripts/base/frameworks/file-analysis/bifs/remove_action.bro b/testing/btest/scripts/base/frameworks/file-analysis/bifs/remove_action.bro new file mode 100644 index 0000000000..a2e97a31a4 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/file-analysis/bifs/remove_action.bro @@ -0,0 +1,20 @@ +# @TEST-EXEC: bro -r $TRACES/http/get.trace $SCRIPTS/file-analysis-test.bro %INPUT >get.out +# @TEST-EXEC: btest-diff get.out + +redef test_file_analysis_source = "HTTP"; + +redef test_get_file_name = function(info: FileAnalysis::Info): string + { + return fmt("%s-file", info$file_id); + }; + +hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info) + { + if ( trig != FileAnalysis::TRIGGER_TYPE ) return; + for ( act in test_file_actions ) + FileAnalysis::remove_action(info$file_id, act); + local filename = test_get_file_name(info); + FileAnalysis::remove_action(info$file_id, + [$act=FileAnalysis::ACTION_EXTRACT, + $extract_filename=filename]); + } diff --git a/testing/btest/scripts/base/frameworks/file-analysis/bifs/stop.bro b/testing/btest/scripts/base/frameworks/file-analysis/bifs/stop.bro new file mode 100644 index 0000000000..c12ff08590 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/file-analysis/bifs/stop.bro @@ -0,0 +1,9 @@ +# @TEST-EXEC: bro -r $TRACES/http/get.trace $SCRIPTS/file-analysis-test.bro %INPUT >get.out +# @TEST-EXEC: btest-diff get.out +# @TEST-EXEC: test ! -s Cx92a0ym5R8-file + +hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info) + { + if ( trig != FileAnalysis::TRIGGER_NEW ) return; + FileAnalysis::stop(info$file_id); + } diff --git a/testing/btest/scripts/base/frameworks/file-analysis/ftp.bro b/testing/btest/scripts/base/frameworks/file-analysis/ftp.bro new file mode 100644 index 0000000000..9c2134bfa8 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/file-analysis/ftp.bro @@ -0,0 +1,10 @@ +# @TEST-EXEC: bro -r $TRACES/ftp/retr.trace $SCRIPTS/file-analysis-test.bro %INPUT >out +# @TEST-EXEC: btest-diff out +# @TEST-EXEC: btest-diff thefile + +redef test_file_analysis_source = "FTP_DATA"; + +redef test_get_file_name = function(info: FileAnalysis::Info): string + { + return "thefile"; + }; diff --git a/testing/btest/scripts/base/frameworks/file-analysis/http/get.bro b/testing/btest/scripts/base/frameworks/file-analysis/http/get.bro new file mode 100644 index 0000000000..e9439bd3b8 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/file-analysis/http/get.bro @@ -0,0 +1,13 @@ +# @TEST-EXEC: bro -r $TRACES/http/get.trace $SCRIPTS/file-analysis-test.bro %INPUT >get.out +# @TEST-EXEC: bro -r $TRACES/http/get-gzip.trace $SCRIPTS/file-analysis-test.bro %INPUT >get-gzip.out +# @TEST-EXEC: btest-diff get.out +# @TEST-EXEC: btest-diff get-gzip.out +# @TEST-EXEC: btest-diff Cx92a0ym5R8-file +# @TEST-EXEC: btest-diff kg59rqyYxN-file + +redef test_file_analysis_source = "HTTP"; + +redef test_get_file_name = function(info: FileAnalysis::Info): string + { + return fmt("%s-file", info$file_id); + }; diff --git a/testing/btest/scripts/base/frameworks/file-analysis/http/partial-content.bro b/testing/btest/scripts/base/frameworks/file-analysis/http/partial-content.bro new file mode 100644 index 0000000000..6773c6aa11 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/file-analysis/http/partial-content.bro @@ -0,0 +1,25 @@ +# @TEST-EXEC: bro -r $TRACES/http/206_example_a.pcap $SCRIPTS/file-analysis-test.bro %INPUT >a.out +# @TEST-EXEC: btest-diff a.out +# @TEST-EXEC: wc -c 7gZBKVUgy4l-file0 | sed 's/^[ \t]* //g' >a.size +# @TEST-EXEC: btest-diff a.size + +# @TEST-EXEC: bro -r $TRACES/http/206_example_b.pcap $SCRIPTS/file-analysis-test.bro %INPUT >b.out +# @TEST-EXEC: btest-diff b.out +# @TEST-EXEC: wc -c oDwT1BbzjM1-file0 | sed 's/^[ \t]* //g' >b.size +# @TEST-EXEC: btest-diff b.size + +# @TEST-EXEC: bro -r $TRACES/http/206_example_c.pcap $SCRIPTS/file-analysis-test.bro %INPUT >c.out +# @TEST-EXEC: btest-diff c.out +# @TEST-EXEC: wc -c uHS14uhRKGe-file0 | sed 's/^[ \t]* //g' >c.size +# @TEST-EXEC: btest-diff c.size + +global cnt: count = 0; + +redef test_file_analysis_source = "HTTP"; + +redef test_get_file_name = function(info: FileAnalysis::Info): string + { + local rval: string = fmt("%s-file%d", info$file_id, cnt); + ++cnt; + return rval; + }; diff --git a/testing/btest/scripts/base/frameworks/file-analysis/http/pipeline.bro b/testing/btest/scripts/base/frameworks/file-analysis/http/pipeline.bro new file mode 100644 index 0000000000..1bbfdb6b13 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/file-analysis/http/pipeline.bro @@ -0,0 +1,14 @@ +# @TEST-EXEC: bro -r $TRACES/http/pipelined-requests.trace $SCRIPTS/file-analysis-test.bro %INPUT >out +# @TEST-EXEC: btest-diff out +# @TEST-EXEC: btest-diff aFQKI8SPOL2-file +# @TEST-EXEC: btest-diff CCU3vUEr06l-file +# @TEST-EXEC: btest-diff HCzA0dVwDPj-file +# @TEST-EXEC: btest-diff a1Zu1fteVEf-file +# @TEST-EXEC: btest-diff xXlF7wFdsR-file + +redef test_file_analysis_source = "HTTP"; + +redef test_get_file_name = function(info: FileAnalysis::Info): string + { + return fmt("%s-file", info$file_id); + }; diff --git a/testing/btest/scripts/base/frameworks/file-analysis/http/post.bro b/testing/btest/scripts/base/frameworks/file-analysis/http/post.bro new file mode 100644 index 0000000000..55fd1ed9c0 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/file-analysis/http/post.bro @@ -0,0 +1,11 @@ +# @TEST-EXEC: bro -r $TRACES/http/post.trace $SCRIPTS/file-analysis-test.bro %INPUT >out +# @TEST-EXEC: btest-diff out +# @TEST-EXEC: btest-diff v5HLI7MxPQh-file +# @TEST-EXEC: btest-diff PZS1XGHkIf1-file + +redef test_file_analysis_source = "HTTP"; + +redef test_get_file_name = function(info: FileAnalysis::Info): string + { + return fmt("%s-file", info$file_id); + }; diff --git a/testing/btest/scripts/base/frameworks/file-analysis/input/basic.bro b/testing/btest/scripts/base/frameworks/file-analysis/input/basic.bro new file mode 100644 index 0000000000..3a9b529ab0 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/file-analysis/input/basic.bro @@ -0,0 +1,51 @@ +# @TEST-EXEC: btest-bg-run bro bro -b $SCRIPTS/file-analysis-test.bro %INPUT +# @TEST-EXEC: btest-bg-wait 8 +# @TEST-EXEC: btest-diff bro/.stdout +# @TEST-EXEC: diff -q bro/nYgPNGLrZf9-file input.log + +redef exit_only_after_terminate = T; + +redef test_get_file_name = function(info: FileAnalysis::Info): string + { + return fmt("%s-file", info$file_id); + }; + +@TEST-START-FILE input.log +#separator \x09 +#path ssh +#fields b i e c p sn a d t iv s sc ss se vc ve ns +#types bool int enum count port subnet addr double time interval string table table table vector vector string +T -42 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY 4242 +@TEST-END-FILE + +module A; + +type Val: record { + s: string; +}; + +event line(description: Input::EventDescription, tpe: Input::Event, s: string) + { + FileAnalysis::data_stream(description$source, s); + } + +event Input::end_of_data(name: string, source: string) + { + FileAnalysis::eof(source); + } + +event bro_init() + { + Input::add_event([$source="../input.log", $reader=Input::READER_BINARY, + $mode=Input::MANUAL, $name="input", $fields=Val, + $ev=line, $want_record=F]); + Input::remove("input"); + } + +hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info) + &priority=-10 + { + if ( trig != FileAnalysis::TRIGGER_EOF ) return; + terminate(); + } + diff --git a/testing/btest/scripts/base/frameworks/file-analysis/irc.bro b/testing/btest/scripts/base/frameworks/file-analysis/irc.bro new file mode 100644 index 0000000000..69fd2a93c9 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/file-analysis/irc.bro @@ -0,0 +1,10 @@ +# @TEST-EXEC: bro -r $TRACES/irc-dcc-send.trace $SCRIPTS/file-analysis-test.bro %INPUT >out +# @TEST-EXEC: btest-diff out +# @TEST-EXEC: btest-diff thefile + +redef test_file_analysis_source = "IRC_DATA"; + +redef test_get_file_name = function(info: FileAnalysis::Info): string + { + return "thefile"; + }; diff --git a/testing/btest/scripts/base/frameworks/file-analysis/logging.bro b/testing/btest/scripts/base/frameworks/file-analysis/logging.bro new file mode 100644 index 0000000000..275fc20ea5 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/file-analysis/logging.bro @@ -0,0 +1,9 @@ +# @TEST-EXEC: bro -r $TRACES/http/get.trace $SCRIPTS/file-analysis-test.bro %INPUT +# @TEST-EXEC: btest-diff file_analysis.log + +redef test_file_analysis_source = "HTTP"; + +redef test_get_file_name = function(info: FileAnalysis::Info): string + { + return fmt("%s-file", info$file_id); + }; diff --git a/testing/btest/scripts/base/frameworks/file-analysis/smtp.bro b/testing/btest/scripts/base/frameworks/file-analysis/smtp.bro new file mode 100644 index 0000000000..64c5e9f0d7 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/file-analysis/smtp.bro @@ -0,0 +1,16 @@ +# @TEST-EXEC: bro -r $TRACES/smtp.trace $SCRIPTS/file-analysis-test.bro %INPUT >out +# @TEST-EXEC: btest-diff out +# @TEST-EXEC: btest-diff thefile0 +# @TEST-EXEC: btest-diff thefile1 +# @TEST-EXEC: btest-diff thefile2 + +redef test_file_analysis_source = "SMTP"; + +global mycnt: count = 0; + +redef test_get_file_name = function(info: FileAnalysis::Info): string + { + local rval: string = fmt("thefile%d", mycnt); + ++mycnt; + return rval; + }; diff --git a/testing/btest/scripts/base/frameworks/input/tableevent.bro b/testing/btest/scripts/base/frameworks/input/tableevent.bro index a409803440..c83b1361c1 100644 --- a/testing/btest/scripts/base/frameworks/input/tableevent.bro +++ b/testing/btest/scripts/base/frameworks/input/tableevent.bro @@ -35,7 +35,6 @@ global destination: table[int] of Val = table(); event line(description: Input::TableDescription, tpe: Input::Event, left: Idx, right: bool) { - print outfile, description; print outfile, tpe; print outfile, left; print outfile, right; diff --git a/testing/btest/scripts/base/frameworks/metrics/notice.bro b/testing/btest/scripts/base/frameworks/metrics/notice.bro deleted file mode 100644 index 0ac9faa956..0000000000 --- a/testing/btest/scripts/base/frameworks/metrics/notice.bro +++ /dev/null @@ -1,24 +0,0 @@ -# @TEST-EXEC: bro %INPUT -# @TEST-EXEC: btest-diff notice.log - - -redef enum Notice::Type += { - Test_Notice, -}; - -redef enum Metrics::ID += { - TEST_METRIC, -}; - -event bro_init() &priority=5 - { - Metrics::add_filter(TEST_METRIC, - [$name="foo-bar", - $break_interval=3secs, - $note=Test_Notice, - $notice_threshold=2, - $log=F]); - Metrics::add_data(TEST_METRIC, [$host=1.2.3.4], 3); - Metrics::add_data(TEST_METRIC, [$host=6.5.4.3], 2); - Metrics::add_data(TEST_METRIC, [$host=7.2.1.5], 1); - } diff --git a/testing/btest/scripts/base/frameworks/notice/mail-alarms.bro b/testing/btest/scripts/base/frameworks/notice/mail-alarms.bro index 3116b1025a..0970ec0c76 100644 --- a/testing/btest/scripts/base/frameworks/notice/mail-alarms.bro +++ b/testing/btest/scripts/base/frameworks/notice/mail-alarms.bro @@ -1,7 +1,11 @@ # @TEST-EXEC: bro -C -r $TRACES/web.trace %INPUT # @TEST-EXEC: btest-diff alarm-mail.txt -redef Notice::policy += { [$action = Notice::ACTION_ALARM, $priority = 1 ] }; +hook Notice::policy(n: Notice::Info) &priority=1 + { + add n$actions[Notice::ACTION_ALARM]; + } + redef Notice::force_email_summaries = T; redef enum Notice::Type += { diff --git a/testing/btest/scripts/base/frameworks/software/version-parsing.bro b/testing/btest/scripts/base/frameworks/software/version-parsing.bro index 03327a25cd..2b406f22b8 100644 --- a/testing/btest/scripts/base/frameworks/software/version-parsing.bro +++ b/testing/btest/scripts/base/frameworks/software/version-parsing.bro @@ -21,7 +21,7 @@ global matched_software: table[string] of Software::Description = { ["Java1.2.2-JDeveloper"] = [$name="Java", $version=[$major=1,$minor=2,$minor2=2,$addl="JDeveloper"], $unparsed_version=""], ["Java/1.6.0_13"] = - [$name="Java", $version=[$major=1,$minor=6,$minor2=0,$addl="13"], $unparsed_version=""], + [$name="Java", $version=[$major=1,$minor=6,$minor2=0,$minor3=13], $unparsed_version=""], ["Python-urllib/3.1"] = [$name="Python-urllib", $version=[$major=3,$minor=1], $unparsed_version=""], ["libwww-perl/5.820"] = @@ -39,9 +39,9 @@ global matched_software: table[string] of Software::Description = { ["The Bat! (v2.00.9) Personal"] = [$name="The Bat!", $version=[$major=2,$minor=0,$minor2=9,$addl="Personal"], $unparsed_version=""], ["Flash/10,2,153,1"] = - [$name="Flash", $version=[$major=10,$minor=2,$minor2=153,$addl="1"], $unparsed_version=""], + [$name="Flash", $version=[$major=10,$minor=2,$minor2=153,$minor3=1], $unparsed_version=""], ["mt2/1.2.3.967 Oct 13 2010-13:40:24 ord-pixel-x2 pid 0x35a3 13731"] = - [$name="mt2", $version=[$major=1,$minor=2,$minor2=3,$addl="967"], $unparsed_version=""], + [$name="mt2", $version=[$major=1,$minor=2,$minor2=3,$minor3=967,$addl="Oct"], $unparsed_version=""], ["CacheFlyServe v26b"] = [$name="CacheFlyServe", $version=[$major=26,$addl="b"], $unparsed_version=""], ["Apache/2.0.46 (Win32) mod_ssl/2.0.46 OpenSSL/0.9.7b mod_jk2/2.0.4"] = @@ -54,7 +54,7 @@ global matched_software: table[string] of Software::Description = { ["Mozilla/5.0 (iPhone; U; CPU iPhone OS 4_3_2 like Mac OS X; en-us) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8H7 Safari/6533.18.5"] = [$name="Safari", $version=[$major=5,$minor=0,$minor2=2,$addl="Mobile"], $unparsed_version=""], ["Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_7; en-US) AppleWebKit/534.16 (KHTML, like Gecko) Chrome/10.0.648.205 Safari/534.16"] = - [$name="Chrome", $version=[$major=10,$minor=0,$minor2=648,$addl="205"], $unparsed_version=""], + [$name="Chrome", $version=[$major=10,$minor=0,$minor2=648,$minor3=205], $unparsed_version=""], ["Opera/9.80 (Windows NT 6.1; U; sv) Presto/2.7.62 Version/11.01"] = [$name="Opera", $version=[$major=11,$minor=1], $unparsed_version=""], ["Mozilla/5.0 (Windows; U; Windows NT 5.1; de; rv:1.9.2.11) Gecko/20101013 Lightning/1.0b2 Thunderbird/3.1.5"] = @@ -62,7 +62,7 @@ global matched_software: table[string] of Software::Description = { ["iTunes/9.0 (Macintosh; Intel Mac OS X 10.5.8) AppleWebKit/531.9"] = [$name="iTunes", $version=[$major=9,$minor=0,$addl="Macintosh"], $unparsed_version=""], ["Java1.3.1_04"] = - [$name="Java", $version=[$major=1,$minor=3,$minor2=1,$addl="04"], $unparsed_version=""], + [$name="Java", $version=[$major=1,$minor=3,$minor2=1,$minor3=4], $unparsed_version=""], ["Mozilla/5.0 (Linux; U; Android 2.3.3; zh-tw; HTC Pyramid Build/GRI40) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1"] = [$name="Safari", $version=[$major=4,$minor=0,$addl="Mobile"], $unparsed_version=""], ["Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_6; en-us) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27"] = diff --git a/testing/btest/scripts/base/protocols/conn/contents-default-extract.test b/testing/btest/scripts/base/protocols/conn/contents-default-extract.test index 82f46b62c8..b53081826c 100644 --- a/testing/btest/scripts/base/protocols/conn/contents-default-extract.test +++ b/testing/btest/scripts/base/protocols/conn/contents-default-extract.test @@ -1,3 +1,3 @@ -# @TEST-EXEC: bro -f "tcp port 21" -r $TRACES/ipv6-ftp.trace "Conn::default_extract=T" +# @TEST-EXEC: bro -f "tcp port 21" -r $TRACES/ftp/ipv6.trace "Conn::default_extract=T" # @TEST-EXEC: btest-diff contents_[2001:470:1f11:81f:c999:d94:aa7c:2e3e]:49185-[2001:470:4867:99::21]:21_orig.dat # @TEST-EXEC: btest-diff contents_[2001:470:1f11:81f:c999:d94:aa7c:2e3e]:49185-[2001:470:4867:99::21]:21_resp.dat diff --git a/testing/btest/scripts/base/protocols/conn/polling.test b/testing/btest/scripts/base/protocols/conn/polling.test index a6fbc35f66..f855326e77 100644 --- a/testing/btest/scripts/base/protocols/conn/polling.test +++ b/testing/btest/scripts/base/protocols/conn/polling.test @@ -1,6 +1,6 @@ -# @TEST-EXEC: bro -b -r $TRACES/http-100-continue.trace %INPUT >out1 +# @TEST-EXEC: bro -b -r $TRACES/http/100-continue.trace %INPUT >out1 # @TEST-EXEC: btest-diff out1 -# @TEST-EXEC: bro -b -r $TRACES/http-100-continue.trace %INPUT stop_cnt=2 >out2 +# @TEST-EXEC: bro -b -r $TRACES/http/100-continue.trace %INPUT stop_cnt=2 >out2 # @TEST-EXEC: btest-diff out2 @load base/protocols/conn diff --git a/testing/btest/scripts/base/protocols/ftp/ftp-extract.bro b/testing/btest/scripts/base/protocols/ftp/ftp-extract.bro new file mode 100644 index 0000000000..9ae5280757 --- /dev/null +++ b/testing/btest/scripts/base/protocols/ftp/ftp-extract.bro @@ -0,0 +1,12 @@ +# This tests FTP file extraction. +# +# @TEST-EXEC: bro -r $TRACES/ftp/ipv4.trace %INPUT +# @TEST-EXEC: btest-diff conn.log +# @TEST-EXEC: btest-diff ftp.log +# @TEST-EXEC: btest-diff ftp-item-Rqjkzoroau4-0.dat +# @TEST-EXEC: btest-diff ftp-item-BTsa70Ua9x7-1.dat +# @TEST-EXEC: btest-diff ftp-item-VLQvJybrm38-2.dat +# @TEST-EXEC: btest-diff ftp-item-zrfwSs9K1yk-3.dat + +redef FTP::logged_commands += {"LIST"}; +redef FTP::extract_file_types=/.*/; diff --git a/testing/btest/scripts/base/protocols/ftp/ftp-ipv4.bro b/testing/btest/scripts/base/protocols/ftp/ftp-ipv4.bro index 5cb8b808d5..cb58d4af8a 100644 --- a/testing/btest/scripts/base/protocols/ftp/ftp-ipv4.bro +++ b/testing/btest/scripts/base/protocols/ftp/ftp-ipv4.bro @@ -1,6 +1,6 @@ # This tests both active and passive FTP over IPv4. # -# @TEST-EXEC: bro -r $TRACES/ftp-ipv4.trace +# @TEST-EXEC: bro -r $TRACES/ftp/ipv4.trace # @TEST-EXEC: btest-diff conn.log # @TEST-EXEC: btest-diff ftp.log diff --git a/testing/btest/scripts/base/protocols/ftp/ftp-ipv6.bro b/testing/btest/scripts/base/protocols/ftp/ftp-ipv6.bro index 7ce31808c9..87dfa7e052 100644 --- a/testing/btest/scripts/base/protocols/ftp/ftp-ipv6.bro +++ b/testing/btest/scripts/base/protocols/ftp/ftp-ipv6.bro @@ -1,6 +1,6 @@ # This tests both active and passive FTP over IPv6. # -# @TEST-EXEC: bro -r $TRACES/ipv6-ftp.trace +# @TEST-EXEC: bro -r $TRACES/ftp/ipv6.trace # @TEST-EXEC: btest-diff conn.log # @TEST-EXEC: btest-diff ftp.log diff --git a/testing/btest/scripts/base/protocols/http/100-continue.bro b/testing/btest/scripts/base/protocols/http/100-continue.bro index d1d34c1abe..ed9e4970fe 100644 --- a/testing/btest/scripts/base/protocols/http/100-continue.bro +++ b/testing/btest/scripts/base/protocols/http/100-continue.bro @@ -3,7 +3,7 @@ # a given request. The http scripts should also be able log such replies # in a way that correlates the final response with the request. # -# @TEST-EXEC: bro -r $TRACES/http-100-continue.trace %INPUT +# @TEST-EXEC: bro -r $TRACES/http/100-continue.trace %INPUT # @TEST-EXEC: test ! -f weird.log # @TEST-EXEC: btest-diff http.log diff --git a/testing/btest/scripts/base/protocols/http/http-extract-files.bro b/testing/btest/scripts/base/protocols/http/http-extract-files.bro index 4338cddb47..ce9d3e7e04 100644 --- a/testing/btest/scripts/base/protocols/http/http-extract-files.bro +++ b/testing/btest/scripts/base/protocols/http/http-extract-files.bro @@ -1,5 +1,5 @@ # @TEST-EXEC: bro -C -r $TRACES/web.trace %INPUT # @TEST-EXEC: btest-diff http.log -# @TEST-EXEC: btest-diff http-item_141.42.64.125:56730-125.190.109.199:80_resp_1.dat +# @TEST-EXEC: btest-diff http-item-BFymS6bFgT3-0.dat -redef HTTP::extract_file_types += /text\/html/; \ No newline at end of file +redef HTTP::extract_file_types += /text\/html/; diff --git a/testing/btest/scripts/base/protocols/http/http-header-crlf.bro b/testing/btest/scripts/base/protocols/http/http-header-crlf.bro index 06081d94e5..c9ba7afba3 100644 --- a/testing/btest/scripts/base/protocols/http/http-header-crlf.bro +++ b/testing/btest/scripts/base/protocols/http/http-header-crlf.bro @@ -2,7 +2,7 @@ # it gets confused whether it's in a header or not; it shouldn't report # the http_no_crlf_in_header_list wierd. # -# @TEST-EXEC: bro -r $TRACES/http-byteranges.trace %INPUT +# @TEST-EXEC: bro -r $TRACES/http/byteranges.trace %INPUT # @TEST-EXEC: test ! -f weird.log # The base analysis scripts are loaded by default. diff --git a/testing/btest/scripts/base/protocols/http/http-methods.bro b/testing/btest/scripts/base/protocols/http/http-methods.bro index 59045c1cc5..5ab89bbe4d 100644 --- a/testing/btest/scripts/base/protocols/http/http-methods.bro +++ b/testing/btest/scripts/base/protocols/http/http-methods.bro @@ -1,6 +1,6 @@ # This tests that the HTTP analyzer handles strange HTTP methods properly. # -# @TEST-EXEC: bro -r $TRACES/http-methods.trace %INPUT +# @TEST-EXEC: bro -r $TRACES/http/methods.trace %INPUT # @TEST-EXEC: btest-diff weird.log # @TEST-EXEC: btest-diff http.log diff --git a/testing/btest/scripts/base/protocols/http/http-mime-and-md5.bro b/testing/btest/scripts/base/protocols/http/http-mime-and-md5.bro index dd01e62413..b35e491b4d 100644 --- a/testing/btest/scripts/base/protocols/http/http-mime-and-md5.bro +++ b/testing/btest/scripts/base/protocols/http/http-mime-and-md5.bro @@ -1,21 +1,6 @@ -# This tests md5 calculation for a specified mime type. The http.log -# will normalize mime types other than the target type to prevent sensitivity -# to varying versions of libmagic. +# This tests md5 calculation for a specified mime type. -# @TEST-EXEC: bro -r $TRACES/http-pipelined-requests.trace %INPUT > output +# @TEST-EXEC: bro -r $TRACES/http/pipelined-requests.trace %INPUT > output # @TEST-EXEC: btest-diff http.log redef HTTP::generate_md5 += /image\/png/; - -event bro_init() - { - Log::remove_default_filter(HTTP::LOG); - Log::add_filter(HTTP::LOG, [$name="normalized-mime-types", - $pred=function(rec: HTTP::Info): bool - { - if ( rec?$mime_type && HTTP::generate_md5 != rec$mime_type ) - rec$mime_type = "FAKE_MIME"; - return T; - } - ]); - } diff --git a/testing/btest/scripts/base/protocols/http/http-pipelining.bro b/testing/btest/scripts/base/protocols/http/http-pipelining.bro index 9875683269..bb392b1c4b 100644 --- a/testing/btest/scripts/base/protocols/http/http-pipelining.bro +++ b/testing/btest/scripts/base/protocols/http/http-pipelining.bro @@ -1,4 +1,4 @@ -# @TEST-EXEC: bro -r $TRACES/http-pipelined-requests.trace %INPUT > output +# @TEST-EXEC: bro -r $TRACES/http/pipelined-requests.trace %INPUT > output # @TEST-EXEC: btest-diff http.log # mime type is irrelevant to this test, so filter it out diff --git a/testing/btest/scripts/base/protocols/irc/dcc-extract.test b/testing/btest/scripts/base/protocols/irc/dcc-extract.test index b6bf43ac50..8a6680f99b 100644 --- a/testing/btest/scripts/base/protocols/irc/dcc-extract.test +++ b/testing/btest/scripts/base/protocols/irc/dcc-extract.test @@ -4,9 +4,9 @@ # @TEST-EXEC: bro -r $TRACES/irc-dcc-send.trace %INPUT # @TEST-EXEC: btest-diff irc.log -# @TEST-EXEC: btest-diff irc-dcc-item_192.168.1.77:57655-209.197.168.151:1024_1.dat +# @TEST-EXEC: btest-diff irc-dcc-item-wqKMAamJVSb-0.dat # @TEST-EXEC: bro -r $TRACES/irc-dcc-send.trace %INPUT IRC::extraction_prefix="test" -# @TEST-EXEC: test -e test_192.168.1.77:57655-209.197.168.151:1024_1.dat +# @TEST-EXEC: test -e test-wqKMAamJVSb-0.dat redef IRC::extract_file_types=/.*/; diff --git a/testing/btest/scripts/base/protocols/smtp/mime-extract.test b/testing/btest/scripts/base/protocols/smtp/mime-extract.test index 8dc8bc9e31..54e50d0459 100644 --- a/testing/btest/scripts/base/protocols/smtp/mime-extract.test +++ b/testing/btest/scripts/base/protocols/smtp/mime-extract.test @@ -1,24 +1,11 @@ # @TEST-EXEC: bro -r $TRACES/smtp.trace %INPUT # @TEST-EXEC: btest-diff smtp_entities.log -# @TEST-EXEC: btest-diff smtp-entity_10.10.1.4:1470-74.53.140.153:25_1.dat -# @TEST-EXEC: btest-diff smtp-entity_10.10.1.4:1470-74.53.140.153:25_2.dat +# @TEST-EXEC: btest-diff smtp-entity-cwR7l6Zctxb-0.dat +# @TEST-EXEC: btest-diff smtp-entity-Ltd7QO7jEv3-1.dat # @TEST-EXEC: bro -r $TRACES/smtp.trace %INPUT SMTP::extraction_prefix="test" -# @TEST-EXEC: test -e test_10.10.1.4:1470-74.53.140.153:25_1.dat -# @TEST-EXEC: test -e test_10.10.1.4:1470-74.53.140.153:25_2.dat +# @TEST-EXEC: test -e test-cwR7l6Zctxb-0.dat +# @TEST-EXEC: test -e test-Ltd7QO7jEv3-1.dat @load base/protocols/smtp redef SMTP::extract_file_types=/text\/plain/; - -event bro_init() - { - Log::remove_default_filter(SMTP::ENTITIES_LOG); - Log::add_filter(SMTP::ENTITIES_LOG, [$name="normalized-mime-types", - $pred=function(rec: SMTP::EntityInfo): bool - { - if ( rec?$mime_type ) - rec$mime_type = "FAKE_MIME"; - return T; - } - ]); - } diff --git a/testing/btest/scripts/base/protocols/smtp/mime.test b/testing/btest/scripts/base/protocols/smtp/mime.test index 2b80e148ff..8e7a336987 100644 --- a/testing/btest/scripts/base/protocols/smtp/mime.test +++ b/testing/btest/scripts/base/protocols/smtp/mime.test @@ -1,22 +1,6 @@ -# Checks logging of mime types and md5 calculation. Mime type in the log -# is normalized to prevent sensitivity to libmagic version. - # @TEST-EXEC: bro -r $TRACES/smtp.trace %INPUT # @TEST-EXEC: btest-diff smtp_entities.log @load base/protocols/smtp redef SMTP::generate_md5=/text\/plain/; - -event bro_init() - { - Log::remove_default_filter(SMTP::ENTITIES_LOG); - Log::add_filter(SMTP::ENTITIES_LOG, [$name="normalized-mime-types", - $pred=function(rec: SMTP::EntityInfo): bool - { - if ( rec?$mime_type ) - rec$mime_type = "FAKE_MIME"; - return T; - } - ]); - } diff --git a/testing/btest/scripts/policy/protocols/ssl/extract-certs-pem.bro b/testing/btest/scripts/policy/protocols/ssl/extract-certs-pem.bro new file mode 100644 index 0000000000..acc414a33c --- /dev/null +++ b/testing/btest/scripts/policy/protocols/ssl/extract-certs-pem.bro @@ -0,0 +1,6 @@ +# @TEST-EXEC: bro -r $TRACES/ssl.v3.trace %INPUT +# @TEST-EXEC: btest-diff certs-remote.pem + +@load protocols/ssl/extract-certs-pem + +redef SSL::extract_certs_pem = ALL_HOSTS; diff --git a/testing/btest/signatures/bad-eval-condition.bro b/testing/btest/signatures/bad-eval-condition.bro index 34997b1124..2b3fef76fe 100644 --- a/testing/btest/signatures/bad-eval-condition.bro +++ b/testing/btest/signatures/bad-eval-condition.bro @@ -1,4 +1,4 @@ -# @TEST-EXEC-FAIL: bro -r $TRACES/ftp-ipv4.trace %INPUT +# @TEST-EXEC-FAIL: bro -r $TRACES/ftp/ipv4.trace %INPUT # @TEST-EXEC: btest-diff .stderr @load-sigs blah.sig diff --git a/testing/btest/signatures/dpd.bro b/testing/btest/signatures/dpd.bro index d6ae02cb50..a302603bd2 100644 --- a/testing/btest/signatures/dpd.bro +++ b/testing/btest/signatures/dpd.bro @@ -1,7 +1,7 @@ -# @TEST-EXEC: bro -b -s myftp -r $TRACES/ftp-ipv4.trace %INPUT >dpd-ipv4.out -# @TEST-EXEC: bro -b -s myftp -r $TRACES/ipv6-ftp.trace %INPUT >dpd-ipv6.out -# @TEST-EXEC: bro -b -r $TRACES/ftp-ipv4.trace %INPUT >nosig-ipv4.out -# @TEST-EXEC: bro -b -r $TRACES/ipv6-ftp.trace %INPUT >nosig-ipv6.out +# @TEST-EXEC: bro -b -s myftp -r $TRACES/ftp/ipv4.trace %INPUT >dpd-ipv4.out +# @TEST-EXEC: bro -b -s myftp -r $TRACES/ftp/ipv6.trace %INPUT >dpd-ipv6.out +# @TEST-EXEC: bro -b -r $TRACES/ftp/ipv4.trace %INPUT >nosig-ipv4.out +# @TEST-EXEC: bro -b -r $TRACES/ftp/ipv6.trace %INPUT >nosig-ipv6.out # @TEST-EXEC: btest-diff dpd-ipv4.out # @TEST-EXEC: btest-diff dpd-ipv6.out # @TEST-EXEC: btest-diff nosig-ipv4.out diff --git a/testing/btest/signatures/eval-condition.bro b/testing/btest/signatures/eval-condition.bro index f3f1171da6..a14003b691 100644 --- a/testing/btest/signatures/eval-condition.bro +++ b/testing/btest/signatures/eval-condition.bro @@ -1,4 +1,4 @@ -# @TEST-EXEC: bro -r $TRACES/ftp-ipv4.trace %INPUT +# @TEST-EXEC: bro -r $TRACES/ftp/ipv4.trace %INPUT # @TEST-EXEC: btest-diff conn.log @load-sigs blah.sig diff --git a/testing/scripts/diff-canonifier b/testing/scripts/diff-canonifier index 3cb213a3f7..aef9173cc8 100755 --- a/testing/scripts/diff-canonifier +++ b/testing/scripts/diff-canonifier @@ -2,4 +2,5 @@ # # Default canonifier used with the tests in testing/btest/*. -`dirname $0`/diff-remove-timestamps +`dirname $0`/diff-remove-timestamps \ + | `dirname $0`/diff-remove-mime-types diff --git a/testing/scripts/file-analysis-test.bro b/testing/scripts/file-analysis-test.bro new file mode 100644 index 0000000000..a52a8ec29b --- /dev/null +++ b/testing/scripts/file-analysis-test.bro @@ -0,0 +1,112 @@ + +global test_file_analysis_source: string = "" &redef; + +global test_file_actions: set[FileAnalysis::ActionArgs]; + +global test_get_file_name: function(info: FileAnalysis::Info): string = + function(info: FileAnalysis::Info): string { return ""; } &redef; + +global test_print_file_data_events: bool = F &redef; + +event file_chunk(info: FileAnalysis::Info, data: string, off: count) + { + if ( test_print_file_data_events ) + print "file_chunk", info$file_id, |data|, off, data; + } + +event file_stream(info: FileAnalysis::Info, data: string) + { + if ( test_print_file_data_events ) + print "file_stream", info$file_id, |data|, data; + } + +hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info) + { + print trig; + + switch ( trig ) { + case FileAnalysis::TRIGGER_NEW: + print info$file_id, info$seen_bytes, info$missing_bytes; + + if ( test_file_analysis_source == "" || + info$source == test_file_analysis_source ) + { + for ( act in test_file_actions ) + FileAnalysis::add_action(info$file_id, act); + + local filename: string = test_get_file_name(info); + if ( filename != "" ) + FileAnalysis::add_action(info$file_id, + [$act=FileAnalysis::ACTION_EXTRACT, + $extract_filename=filename]); + FileAnalysis::add_action(info$file_id, + [$act=FileAnalysis::ACTION_DATA_EVENT, + $chunk_event=file_chunk, + $stream_event=file_stream]); + + } + break; + + case FileAnalysis::TRIGGER_BOF_BUFFER: + if ( info?$bof_buffer ) + print info$bof_buffer[0:10]; + break; + + case FileAnalysis::TRIGGER_TYPE: + # not actually printing the values due to libmagic variances + if ( info?$file_type ) + print "file type is set"; + if ( info?$mime_type ) + print "mime type is set"; + break; + + case FileAnalysis::TRIGGER_EOF: + fallthrough; + case FileAnalysis::TRIGGER_DONE: + print info$file_id, info$seen_bytes, info$missing_bytes; + if ( info?$conns ) + for ( cid in info$conns ) + print cid; + + if ( info?$total_bytes ) + print "total bytes: " + fmt("%s", info$total_bytes); + if ( info?$source ) + print "source: " + info$source; + + for ( act in info$actions ) + switch ( act$act ) { + case FileAnalysis::ACTION_MD5: + if ( info$actions[act]?$md5 ) + print fmt("MD5: %s", info$actions[act]$md5); + break; + case FileAnalysis::ACTION_SHA1: + if ( info$actions[act]?$sha1 ) + print fmt("SHA1: %s", info$actions[act]$sha1); + break; + case FileAnalysis::ACTION_SHA256: + if ( info$actions[act]?$sha256 ) + print fmt("SHA256: %s", info$actions[act]$sha256); + break; + } + break; + } + } + +hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info) + &priority=-5 + { + if ( trig != FileAnalysis::TRIGGER_TYPE ) return; + + # avoids libmagic variances across systems + if ( info?$mime_type ) + info$mime_type = "set"; + if ( info?$file_type ) + info$file_type = "set"; + } + +event bro_init() + { + add test_file_actions[[$act=FileAnalysis::ACTION_MD5]]; + add test_file_actions[[$act=FileAnalysis::ACTION_SHA1]]; + add test_file_actions[[$act=FileAnalysis::ACTION_SHA256]]; + }