Mirror/zeek
1
0
Fork 0
mirror of https://github.com/zeek/zeek.git synced 2025-10-04 15:48:19 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

Merging master.

Browse source
This commit is contained in:
Seth Hall 2012-11-21 12:18:03 -05:00
commit 3546d93f36
1015 changed files with 214684 additions and 4605 deletions
Download patch file Download diff file Expand all files Collapse all files

1020
CHANGES
View file

File diff suppressed because it is too large Load diff

64
CMakeLists.txt
View file

@ -88,25 +88,60 @@ if (LIBGEOIP_FOUND)
list(APPEND OPTLIBS ${LibGeoIP_LIBRARY})
endif ()
set(USE_PERFTOOLS false)
set(HAVE_PERFTOOLS false)
set(USE_PERFTOOLS_DEBUG false)
set(USE_PERFTOOLS_TCMALLOC false)
find_package(GooglePerftools)
if (NOT DISABLE_PERFTOOLS)
find_package(GooglePerftools)
endif ()
if (GOOGLEPERFTOOLS_FOUND)
include_directories(BEFORE ${GooglePerftools_INCLUDE_DIR})
set(USE_PERFTOOLS true)
set(HAVE_PERFTOOLS true)
# Non-Linux systems may not be well-supported by gperftools, so
# require explicit request from user to enable it in that case.
if (${CMAKE_SYSTEM_NAME} MATCHES "Linux" OR ENABLE_PERFTOOLS)
set(USE_PERFTOOLS_TCMALLOC true)
if (ENABLE_PERFTOOLS_DEBUG)
# Enable heap debugging with perftools.
set(USE_PERFTOOLS_DEBUG true)
list(APPEND OPTLIBS ${GooglePerftools_LIBRARIES_DEBUG})
else ()
# Link in tcmalloc for better performance.
list(APPEND OPTLIBS ${GooglePerftools_LIBRARIES})
if (ENABLE_PERFTOOLS_DEBUG)
# Enable heap debugging with perftools.
set(USE_PERFTOOLS_DEBUG true)
include_directories(BEFORE ${GooglePerftools_INCLUDE_DIR})
list(APPEND OPTLIBS ${GooglePerftools_LIBRARIES_DEBUG})
else ()
# Link in tcmalloc for better performance.
list(APPEND OPTLIBS ${GooglePerftools_LIBRARIES})
endif ()
endif ()
endif ()
set(USE_DATASERIES false)
find_package(Lintel)
find_package(DataSeries)
find_package(LibXML2)
if (NOT DISABLE_DATASERIES AND
LINTEL_FOUND AND DATASERIES_FOUND AND LIBXML2_FOUND)
set(USE_DATASERIES true)
include_directories(BEFORE ${Lintel_INCLUDE_DIR})
include_directories(BEFORE ${DataSeries_INCLUDE_DIR})
include_directories(BEFORE ${LibXML2_INCLUDE_DIR})
list(APPEND OPTLIBS ${Lintel_LIBRARIES})
list(APPEND OPTLIBS ${DataSeries_LIBRARIES})
list(APPEND OPTLIBS ${LibXML2_LIBRARIES})
endif()
set(USE_ELASTICSEARCH false)
set(USE_CURL false)
find_package(LibCURL)
if (NOT DISABLE_ELASTICSEARCH AND LIBCURL_FOUND)
set(USE_ELASTICSEARCH true)
set(USE_CURL true)
include_directories(BEFORE ${LibCURL_INCLUDE_DIR})
list(APPEND OPTLIBS ${LibCURL_LIBRARIES})
endif()
if (ENABLE_PERFTOOLS_DEBUG)
# Just a no op to prevent CMake from complaining about manually-specified
# ENABLE_PERFTOOLS_DEBUG not being used if google perftools weren't found
@ -196,8 +231,13 @@ message(
"\nAux. Tools: ${INSTALL_AUX_TOOLS}"
"\n"
"\nGeoIP: ${USE_GEOIP}"
"\nGoogle perftools: ${USE_PERFTOOLS}"
"\ngperftools found: ${HAVE_PERFTOOLS}"
"\n tcmalloc: ${USE_PERFTOOLS_TCMALLOC}"
"\n debugging: ${USE_PERFTOOLS_DEBUG}"
"\ncURL: ${USE_CURL}"
"\n"
"\nDataSeries: ${USE_DATASERIES}"
"\nElasticSearch: ${USE_ELASTICSEARCH}"
"\n"
"\n================================================================\n"
)

144
DocSourcesList.cmake Normal file
View file

@ -0,0 +1,144 @@
# DO NOT EDIT
# This file is auto-generated from the genDocSourcesList.sh script.
#
# This is a list of Bro script sources for which to generate reST documentation.
# It will be included inline in the CMakeLists.txt found in the same directory
# in order to create Makefile targets that define how to generate reST from
# a given Bro script.
#
# Note: any path prefix of the script (2nd argument of rest_target macro)
# will be used to derive what path under scripts/ the generated documentation
# will be placed.
set(psd ${PROJECT_SOURCE_DIR}/scripts)
rest_target(${CMAKE_CURRENT_SOURCE_DIR} example.bro internal)
rest_target(${psd} base/init-default.bro internal)
rest_target(${psd} base/init-bare.bro internal)
rest_target(${CMAKE_BINARY_DIR}/src base/bro.bif.bro)
rest_target(${CMAKE_BINARY_DIR}/src base/const.bif.bro)
rest_target(${CMAKE_BINARY_DIR}/src base/event.bif.bro)
rest_target(${CMAKE_BINARY_DIR}/src base/logging.bif.bro)
rest_target(${CMAKE_BINARY_DIR}/src base/reporter.bif.bro)
rest_target(${CMAKE_BINARY_DIR}/src base/strings.bif.bro)
rest_target(${CMAKE_BINARY_DIR}/src base/types.bif.bro)
rest_target(${psd} base/frameworks/cluster/main.bro)
rest_target(${psd} base/frameworks/cluster/nodes/manager.bro)
rest_target(${psd} base/frameworks/cluster/nodes/proxy.bro)
rest_target(${psd} base/frameworks/cluster/nodes/worker.bro)
rest_target(${psd} base/frameworks/cluster/setup-connections.bro)
rest_target(${psd} base/frameworks/communication/main.bro)
rest_target(${psd} base/frameworks/control/main.bro)
rest_target(${psd} base/frameworks/dpd/main.bro)
rest_target(${psd} base/frameworks/intel/main.bro)
rest_target(${psd} base/frameworks/logging/main.bro)
rest_target(${psd} base/frameworks/logging/postprocessors/scp.bro)
rest_target(${psd} base/frameworks/logging/postprocessors/sftp.bro)
rest_target(${psd} base/frameworks/logging/writers/ascii.bro)
rest_target(${psd} base/frameworks/logging/writers/dataseries.bro)
rest_target(${psd} base/frameworks/metrics/cluster.bro)
rest_target(${psd} base/frameworks/metrics/main.bro)
rest_target(${psd} base/frameworks/metrics/non-cluster.bro)
rest_target(${psd} base/frameworks/notice/actions/add-geodata.bro)
rest_target(${psd} base/frameworks/notice/actions/drop.bro)
rest_target(${psd} base/frameworks/notice/actions/email_admin.bro)
rest_target(${psd} base/frameworks/notice/actions/page.bro)
rest_target(${psd} base/frameworks/notice/actions/pp-alarms.bro)
rest_target(${psd} base/frameworks/notice/cluster.bro)
rest_target(${psd} base/frameworks/notice/extend-email/hostnames.bro)
rest_target(${psd} base/frameworks/notice/main.bro)
rest_target(${psd} base/frameworks/notice/weird.bro)
rest_target(${psd} base/frameworks/packet-filter/main.bro)
rest_target(${psd} base/frameworks/packet-filter/netstats.bro)
rest_target(${psd} base/frameworks/reporter/main.bro)
rest_target(${psd} base/frameworks/signatures/main.bro)
rest_target(${psd} base/frameworks/software/main.bro)
rest_target(${psd} base/protocols/conn/contents.bro)
rest_target(${psd} base/protocols/conn/inactivity.bro)
rest_target(${psd} base/protocols/conn/main.bro)
rest_target(${psd} base/protocols/dns/consts.bro)
rest_target(${psd} base/protocols/dns/main.bro)
rest_target(${psd} base/protocols/ftp/file-extract.bro)
rest_target(${psd} base/protocols/ftp/main.bro)
rest_target(${psd} base/protocols/ftp/utils-commands.bro)
rest_target(${psd} base/protocols/http/file-extract.bro)
rest_target(${psd} base/protocols/http/file-hash.bro)
rest_target(${psd} base/protocols/http/file-ident.bro)
rest_target(${psd} base/protocols/http/main.bro)
rest_target(${psd} base/protocols/http/utils.bro)
rest_target(${psd} base/protocols/irc/dcc-send.bro)
rest_target(${psd} base/protocols/irc/main.bro)
rest_target(${psd} base/protocols/smtp/entities-excerpt.bro)
rest_target(${psd} base/protocols/smtp/entities.bro)
rest_target(${psd} base/protocols/smtp/main.bro)
rest_target(${psd} base/protocols/ssh/main.bro)
rest_target(${psd} base/protocols/ssl/consts.bro)
rest_target(${psd} base/protocols/ssl/main.bro)
rest_target(${psd} base/protocols/ssl/mozilla-ca-list.bro)
rest_target(${psd} base/protocols/syslog/consts.bro)
rest_target(${psd} base/protocols/syslog/main.bro)
rest_target(${psd} base/utils/addrs.bro)
rest_target(${psd} base/utils/conn-ids.bro)
rest_target(${psd} base/utils/directions-and-hosts.bro)
rest_target(${psd} base/utils/files.bro)
rest_target(${psd} base/utils/numbers.bro)
rest_target(${psd} base/utils/paths.bro)
rest_target(${psd} base/utils/patterns.bro)
rest_target(${psd} base/utils/site.bro)
rest_target(${psd} base/utils/strings.bro)
rest_target(${psd} base/utils/thresholds.bro)
rest_target(${psd} policy/frameworks/communication/listen.bro)
rest_target(${psd} policy/frameworks/control/controllee.bro)
rest_target(${psd} policy/frameworks/control/controller.bro)
rest_target(${psd} policy/frameworks/dpd/detect-protocols.bro)
rest_target(${psd} policy/frameworks/dpd/packet-segment-logging.bro)
rest_target(${psd} policy/frameworks/metrics/conn-example.bro)
rest_target(${psd} policy/frameworks/metrics/http-example.bro)
rest_target(${psd} policy/frameworks/metrics/ssl-example.bro)
rest_target(${psd} policy/frameworks/software/version-changes.bro)
rest_target(${psd} policy/frameworks/software/vulnerable.bro)
rest_target(${psd} policy/integration/barnyard2/main.bro)
rest_target(${psd} policy/integration/barnyard2/types.bro)
rest_target(${psd} policy/misc/analysis-groups.bro)
rest_target(${psd} policy/misc/capture-loss.bro)
rest_target(${psd} policy/misc/loaded-scripts.bro)
rest_target(${psd} policy/misc/profiling.bro)
rest_target(${psd} policy/misc/stats.bro)
rest_target(${psd} policy/misc/trim-trace-file.bro)
rest_target(${psd} policy/protocols/conn/known-hosts.bro)
rest_target(${psd} policy/protocols/conn/known-services.bro)
rest_target(${psd} policy/protocols/conn/weirds.bro)
rest_target(${psd} policy/protocols/dns/auth-addl.bro)
rest_target(${psd} policy/protocols/dns/detect-external-names.bro)
rest_target(${psd} policy/protocols/ftp/detect.bro)
rest_target(${psd} policy/protocols/ftp/software.bro)
rest_target(${psd} policy/protocols/http/detect-MHR.bro)
rest_target(${psd} policy/protocols/http/detect-intel.bro)
rest_target(${psd} policy/protocols/http/detect-sqli.bro)
rest_target(${psd} policy/protocols/http/detect-webapps.bro)
rest_target(${psd} policy/protocols/http/header-names.bro)
rest_target(${psd} policy/protocols/http/software-browser-plugins.bro)
rest_target(${psd} policy/protocols/http/software.bro)
rest_target(${psd} policy/protocols/http/var-extraction-cookies.bro)
rest_target(${psd} policy/protocols/http/var-extraction-uri.bro)
rest_target(${psd} policy/protocols/smtp/blocklists.bro)
rest_target(${psd} policy/protocols/smtp/detect-suspicious-orig.bro)
rest_target(${psd} policy/protocols/smtp/software.bro)
rest_target(${psd} policy/protocols/ssh/detect-bruteforcing.bro)
rest_target(${psd} policy/protocols/ssh/geo-data.bro)
rest_target(${psd} policy/protocols/ssh/interesting-hostnames.bro)
rest_target(${psd} policy/protocols/ssh/software.bro)
rest_target(${psd} policy/protocols/ssl/cert-hash.bro)
rest_target(${psd} policy/protocols/ssl/expiring-certs.bro)
rest_target(${psd} policy/protocols/ssl/extract-certs-pem.bro)
rest_target(${psd} policy/protocols/ssl/known-certs.bro)
rest_target(${psd} policy/protocols/ssl/validate-certs.bro)
rest_target(${psd} policy/tuning/defaults/packet-fragments.bro)
rest_target(${psd} policy/tuning/defaults/warnings.bro)
rest_target(${psd} policy/tuning/track-all-assets.bro)
rest_target(${psd} site/local-manager.bro)
rest_target(${psd} site/local-proxy.bro)
rest_target(${psd} site/local-worker.bro)
rest_target(${psd} site/local.bro)
rest_target(${psd} test-all-policy.bro)

300
INSTALL
View file

@ -1,81 +1,303 @@
.. _CMake: http://www.cmake.org
.. _SWIG: http://www.swig.org
.. _Xcode: https://developer.apple.com/xcode/
.. _MacPorts: http://www.macports.org
.. _Fink: http://www.finkproject.org
.. _Homebrew: http://mxcl.github.com/homebrew
.. _bro downloads page: http://bro-ids.org/download/index.html
==============
Installing Bro
==============
Bro can be downloaded in either pre-built binary package or
source code forms.
Prerequisites
=============
Bro relies on the following libraries and tools, which need to be installed
Bro requires the following libraries and tools to be installed
before you begin:
* CMake 2.6.3 or greater http://www.cmake.org
* Libpcap http://www.tcpdump.org
* Libpcap (headers and libraries) http://www.tcpdump.org
* OpenSSL libraries http://www.openssl.org
* OpenSSL (headers and libraries) http://www.openssl.org
* SWIG http://www.swig.org
* BIND8 library
* Libmagic
* Libz
Bro can make uses of some optional libraries if they are found at
installation time:
* Bash (for BroControl)
* LibGeoIP For geo-locating IP addresses.
To build Bro from source, the following additional dependencies are required:
Bro also needs the following tools, but on most systems they will
already come preinstalled:
* CMake 2.6.3 or greater http://www.cmake.org
* SWIG http://www.swig.org
* Bash (For Bro Control).
* BIND8 (headers and libraries)
* Bison (GNU Parser Generator)
* Flex (Fast Lexical Analyzer)
* Perl (Used only during the Bro build process)
* Libpcap headers http://www.tcpdump.org
* OpenSSL headers http://www.openssl.org
* libmagic headers
* zlib headers
* Perl
Bro can make use of some optional libraries and tools if they are found at
build time:
* LibGeoIP (for geo-locating IP addresses)
* gperftools (tcmalloc is used to improve memory and CPU usage)
* sendmail (for BroControl)
* ipsumdump (for trace-summary) http://www.cs.ucla.edu/~kohler/ipsumdump
* Ruby executable, library, and headers (for Broccoli Ruby bindings)
Installation
============
Installing From Pre-Built Binary Release Packages
=================================================
To build and install into ``/usr/local/bro``::
See the `bro downloads page`_ for currently supported/targeted platforms.
* RPM
.. console::
sudo yum localinstall Bro-*.rpm
* DEB
.. console::
sudo gdebi Bro-*.deb
* MacOS Disk Image with Installer
Just open the ``Bro-*.dmg`` and then run the ``.pkg`` installer.
Everything installed by the package will go into ``/opt/bro``.
The primary install prefix for binary packages is ``/opt/bro``.
Non-MacOS packages that include BroControl also put variable/runtime
data (e.g. Bro logs) in ``/var/opt/bro``.
Installing From Source
======================
Required Dependencies
~~~~~~~~~~~~~~~~~~~~~
The following dependencies are required to build Bro:
* RPM/RedHat-based Linux:
.. console::
sudo yum install cmake make gcc gcc-c++ flex bison libpcap-devel openssl-devel python-devel swig zlib-devel file-devel
* DEB/Debian-based Linux:
.. console::
sudo apt-get install cmake make gcc g++ flex bison libpcap-dev libssl-dev python-dev swig zlib1g-dev libmagic-dev
* FreeBSD
Most required dependencies should come with a minimal FreeBSD install
except for the following.
.. console::
sudo pkg_add -r bash cmake swig bison python
Note that ``bash`` needs to be in ``PATH``, which by default it is
not. The FreeBSD package installs the binary into
``/usr/local/bin``.
* Mac OS X
Compiling source code on Macs requires first downloading Xcode_,
then going through its "Preferences..." -> "Downloads" menus to
install the "Command Line Tools" component.
Lion (10.7) and Mountain Lion (10.8) come with all required
dependencies except for CMake_, SWIG_, and ``libmagic``.
Distributions of these dependencies can likely be obtained from your
preferred Mac OS X package management system (e.g. MacPorts_, Fink_,
or Homebrew_).
Specifically for MacPorts, the ``swig``, ``swig-ruby``, ``swig-python``
and ``file`` packages provide the required dependencies.
Optional Dependencies
~~~~~~~~~~~~~~~~~~~~~
Bro can use libGeoIP for geo-locating IP addresses, and sendmail for
sending emails.
* RedHat Enterprise Linux:
.. console::
sudo yum install geoip-devel sendmail
* CentOS Linux:
.. console::
sudo yum install GeoIP-devel sendmail
* DEB/Debian-based Linux:
.. console::
sudo apt-get install libgeoip-dev sendmail
* Ports-based FreeBSD
.. console::
sudo pkg_add -r GeoIP
sendmail is typically already available.
* Mac OS X
Vanilla OS X installations don't ship with libGeoIP, but
if installed from your preferred package management system (e.g. MacPorts,
Fink, or Homebrew), they should be automatically detected and Bro will
compile against them.
Additional steps may be needed to :doc:`get the right GeoIP database <geoip>`.
Compiling Bro Source Code
~~~~~~~~~~~~~~~~~~~~~~~~~
Bro releases are bundled into source packages for convenience and
available from the `bro downloads page`_.
Alternatively, the latest Bro development version can be obtained through git
repositories hosted at `git.bro-ids.org <http://git.bro-ids.org>`_. See
our `git development documentation
<http://bro-ids.org/development/process.html>`_ for comprehensive
information on Bro's use of git revision control, but the short story
for downloading the full source code experience for Bro via git is:
.. console::
git clone --recursive git://git.bro-ids.org/bro
.. note:: If you choose to clone the ``bro`` repository non-recursively for
a "minimal Bro experience", be aware that compiling it depends on
BinPAC, which has its own ``binpac`` repository. Either install it
first or initialize/update the cloned ``bro`` repository's
``aux/binpac`` submodule.
The typical way to build and install from source is (for more options,
run ``./configure --help``):
.. console::
./configure
make
make install
This will first build Bro into a directory inside the distribution
called ``build/``, using default build options. It then installs all
required files into ``/usr/local/bro``, including the Bro binary in
``/usr/local/bro/bin/bro``.
You can specify a different installation directory with::
./configure --prefix=<dir>
Note that ``/usr`` and ``/opt/bro`` are the standard prefixes for
binary Bro packages to be installed, so those are typically not good
choices unless you are creating such a package.
Run ``./configure --help`` for more options.
The default installation path is ``/usr/local/bro``, which would typically
require root privileges when doing the ``make install``. A different
installation path can be chosen by specifying the ``--prefix`` option.
Note that ``/usr`` and ``/opt/bro`` are the
standard prefixes for binary Bro packages to be installed, so those are
typically not good choices unless you are creating such a package.
Depending on the Bro package you downloaded, there may be auxiliary
tools and libraries available in the ``aux/`` directory. All of them
except for ``aux/bro-aux`` will also be built and installed by doing
``make install``. To install the programs that come in the
``aux/bro-aux`` directory, use ``make install-aux``. There are
tools and libraries available in the ``aux/`` directory. Some of them
will be automatically built and installed along with Bro. There are
``--disable-*`` options that can be given to the configure script to
turn off unwanted auxiliary projects.
turn off unwanted auxiliary projects that would otherwise be installed
automatically. Finally, use ``make install-aux`` to install some of
the other programs that are in the ``aux/bro-aux`` directory.
OpenBSD users, please see our `FAQ
<http://www.bro-ids.org/documentation/faq.html>` if you are having
OpenBSD users, please see our FAQ at
http://www.bro-ids.org/documentation/faq.html if you are having
problems installing Bro.
Upgrading From a Previous Version of Bro
========================================
If you're doing an upgrade install (rather than a fresh install),
there's two suggested approaches: either install Bro using the same
installation prefix directory as before, or pick a new prefix and copy
local customizations over.
Re-Use Previous Install Prefix
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
If you choose to configure and install Bro with the same prefix
directory as before, local customization and configuration to files in
``$prefix/share/bro/site`` and ``$prefix/etc`` won't be overwritten
(``$prefix`` indicating the root of where Bro was installed). Also, logs
generated at run-time won't be touched by the upgrade. (But making
a backup of local changes before upgrading is still recommended.)
After upgrading, remember to check ``$prefix/share/bro/site`` and
``$prefix/etc`` for ``.example`` files, which indicate the
distribution's version of the file differs from the local one, which may
include local changes. Review the differences, and make adjustments
as necessary (for differences that aren't the result of a local change,
use the new version's).
Pick a New Install prefix
~~~~~~~~~~~~~~~~~~~~~~~~~
If you want to install the newer version in a different prefix
directory than before, you can just copy local customization and
configuration files from ``$prefix/share/bro/site`` and ``$prefix/etc``
to the new location (``$prefix`` indicating the root of where Bro was
originally installed). Make sure to review the files for difference
before copying and make adjustments as necessary (for differences that
aren't the result of a local change, use the new version's). Of
particular note, the copied version of ``$prefix/etc/broctl.cfg`` is
likely to need changes to the ``SpoolDir`` and ``LogDir`` settings.
Configure the Run-Time Environment
==================================
Just remember that you may need to adjust your ``PATH`` environment variable
according to the platform/shell/package you're using. For example:
Bourne-Shell Syntax:
.. console::
export PATH=/usr/local/bro/bin:$PATH
C-Shell Syntax:
.. console::
setenv PATH /usr/local/bro/bin:$PATH
Or substitute ``/opt/bro/bin`` instead if you installed from a binary package.
Running Bro
===========
Bro is a complex program and it takes a bit of time to get familiar
with it. A good place for newcomers to start is the Quickstart Guide
with it. A good place for newcomers to start is the Quick Start Guide
at http://www.bro-ids.org/documentation/quickstart.html.
For developers that wish to run Bro directly from the ``build/``

174
NEWS
View file

@ -3,22 +3,152 @@ Release Notes
=============
This document summarizes the most important changes in the current Bro
release. For a complete list of changes, see the ``CHANGES`` file.
release. For a complete list of changes, see the ``CHANGES`` file
(note that submodules, such as BroControl and Broccoli, come with
their own CHANGES.)
Bro 2.2
-------
New Functionality
~~~~~~~~~~~~~~~~~
- GridFTP support. TODO: Extend.
- ssl.log now also records the subject client and issuer certificates.
Changed Functionality
~~~~~~~~~~~~~~~~~~~~~
- We removed the following, already deprecated, functionality:
* Scripting language:
- &disable_print_hook attribute.
* BiF functions:
- parse_dotted_addr(), dump_config(),
make_connection_persistent(), generate_idmef(),
split_complete()
- Removed a now unused argument from "do_split" helper function.
- "this" is no longer a reserved keyword.
- The Input Framework's update_finished event has been renamed to
end_of_data. It will now not only fire after table-reads have been
completed, but also after the last event of a whole-file-read (or
whole-db-read, etc.).
- Renamed the option defining the frequency of alarm summary mails to
'Logging::default_alarm_mail_interval'. When using BroControl, the
value can now be set with the new broctl.cfg option
"MailAlarmsInterval".
Bro 2.1
-------
- Dependencies:
New Functionality
~~~~~~~~~~~~~~~~~
- Bro now comes with extensive IPv6 support. Past versions offered
only basic IPv6 functionality that was rarely used in practice as it
had to be enabled explicitly. IPv6 support is now fully integrated
into all parts of Bro including protocol analysis and the scripting
language. It's on by default and no longer requires any special
configuration.
Some of the most significant enhancements include support for IPv6
fragment reassembly, support for following IPv6 extension header
chains, and support for tunnel decapsulation (6to4 and Teredo). The
DNS analyzer now handles AAAA records properly, and DNS lookups that
Bro itself performs now include AAAA queries, so that, for example,
the result returned by script-level lookups is a set that can
contain both IPv4 and IPv6 addresses. Support for the most common
ICMPv6 message types has been added. Also, the FTP EPSV and EPRT
commands are now handled properly. Internally, the way IP addresses
are stored has been improved, so Bro can handle both IPv4
and IPv6 by default without any special configuration.
In addition to Bro itself, the other Bro components have also been
made IPv6-aware by default. In particular, significant changes were
made to trace-summary, PySubnetTree, and Broccoli to support IPv6.
- Bro now decapsulates tunnels via its new tunnel framework located in
scripts/base/frameworks/tunnels. It currently supports Teredo,
AYIYA, IP-in-IP (both IPv4 and IPv6), and SOCKS. For all these, it
logs the outer tunnel connections in both conn.log and tunnel.log,
and then proceeds to analyze the inner payload as if it were not
tunneled, including also logging that session in conn.log. For
SOCKS, it generates a new socks.log in addition with more
information.
- Bro now features a flexible input framework that allows users to
integrate external information in real-time into Bro while it's
processing network traffic. The most direct use-case at the moment
is reading data from ASCII files into Bro tables, with updates
picked up automatically when the file changes during runtime. See
doc/input.rst for more information.
Internally, the input framework is structured around the notion of
"reader plugins" that make it easy to interface to different data
sources. We will add more in the future.
- BroControl now has built-in support for host-based load-balancing
when using either PF_RING, Myricom cards, or individual interfaces.
Instead of adding a separate worker entry in node.cfg for each Bro
worker process on each worker host, it is now possible to just
specify the number of worker processes on each host and BroControl
configures everything correctly (including any neccessary enviroment
variables for the balancers).
This change adds three new keywords to the node.cfg file (to be used
with worker entries): lb_procs (specifies number of workers on a
host), lb_method (specifies what type of load balancing to use:
pf_ring, myricom, or interfaces), and lb_interfaces (used only with
"lb_method=interfaces" to specify which interfaces to load-balance
on).
- Bro's default ASCII log format is not exactly the most efficient way
for storing and searching large volumes of data. An alternatives,
Bro now comes with experimental support for two alternative output
formats:
* DataSeries: an efficient binary format for recording structured
bulk data. DataSeries is developed and maintained at HP Labs.
See doc/logging-dataseries for more information.
* ElasticSearch: a distributed RESTful, storage engine and search
engine built on top of Apache Lucene. It scales very well, both
for distributed indexing and distributed searching. See
doc/logging-elasticsearch.rst for more information.
Note that at this point, we consider Bro's support for these two
formats as prototypes for collecting experience with alternative
outputs. We do not yet recommend them for production (but welcome
feedback!)
Changed Functionality
~~~~~~~~~~~~~~~~~~~~~
The following summarizes the most important differences in existing
functionality. Note that this list is not complete, see CHANGES for
the full set.
- Changes in dependencies:
* Bro now requires CMake >= 2.6.3.
* Bro now links in tcmalloc (part of Google perftools) if found at
configure time. Doing so can significantly improve memory and
CPU use.
* On Linux, Bro now links in tcmalloc (part of Google perftools)
if found at configure time. Doing so can significantly improve
memory and CPU use.
- Bro now supports IPv6 out of the box; the configure switch
--enable-brov6 is gone.
On the other platforms, the new configure option
--enable-perftools can be used to enable linking to tcmalloc.
(Note that perftools's support for non-Linux platforms may be
less reliable).
- The configure switch --enable-brov6 is gone.
- DNS name lookups performed by Bro now also query AAAA records. The
results of the A and AAAA queries for a given hostname are combined
@ -35,20 +165,44 @@ Bro 2.1
- The syntax for IPv6 literals changed from "2607:f8b0:4009:802::1012"
to "[2607:f8b0:4009:802::1012]".
- Bro now spawn threads for doing its logging. From a user's
- Bro now spawns threads for doing its logging. From a user's
perspective not much should change, except that the OS may now show
a bunch of Bro threads.
- We renamed the configure option --enable-perftools to
--enable-perftool-debug to indicate that the switch is only relevant
--enable-perftools-debug to indicate that the switch is only relevant
for debugging the heap.
- Bro's ICMP analyzer now handles both IPv4 and IPv6 messages with a
joint set of events. The `icmp_conn` record got a new boolean field
'v6' that indicates whether the ICMP message is v4 or v6.
- Log postprocessor scripts get an additional argument indicating the
type of the log writer in use (e.g., "ascii").
- BroControl's make-archive-name script also receives the writer
type, but as its 2nd(!) argument. If you're using a custom version
of that script, you need to adapt it. See the shipped version for
details.
- Signature files can now be loaded via the new "@load-sigs"
directive. In contrast to the existing (and still supported)
signature_files constant, this can be used to load signatures
relative to the current script (e.g., "@load-sigs ./foo.sig").
- The options "tunnel_port" and "parse_udp_tunnels" have been removed.
Bro now supports decapsulating tunnels directly for protocols it
understands.
- ASCII logs now record the time when they were opened/closed at the
beginning and end of the file, respectively (wall clock). The
options LogAscii::header_prefix and LogAscii::include_header have
been renamed to LogAscii::meta_prefix and LogAscii::include_meta,
respectively.
- The ASCII writers "header_*" options have been renamed to "meta_*"
(because there's now also a footer).
TODO: Extend.
Bro 2.0
-------

2
VERSION
View file

@ -1 +1 @@
2.0-306
2.1-167

2
aux/binpac

@ -1 +1 @@
Subproject commit 71c37019bc371eb7863fb6aa47a7daa4540f4f1f
Subproject commit 2fd9086c9dc0e76f6ff1ae04a60cbbce60507aab

2
aux/bro-aux

@ -1 +1 @@
Subproject commit d885987e7968669e34504b0403ac89bd13928e9a
Subproject commit bea556198b69d30d64c0cf1b594e6de71176df6f

2
aux/broccoli

@ -1 +1 @@
Subproject commit 55f368b0ad283b2e7d68ef72922b5d9683e2a880
Subproject commit 907210ce1470724fb386f939cc1b10a4caa2ae39

2
aux/broctl

@ -1 +1 @@
Subproject commit ff35c3c144885902c898bf8b47e351c7b8d55e10
Subproject commit 834131cd0ec0f63cce9de818726fe6167dedbf34

2
aux/btest

@ -1 +1 @@
Subproject commit 045a02749b20b3c5896497959e6fda02d060508f
Subproject commit d83e10c5f76cbfdf81c843575351fbc7b544fc93

2
cmake

@ -1 +1 @@
Subproject commit 49278736c1404cb8c077272b80312c947e68bf52
Subproject commit 14537f56d66b18ab9d5024f798caf4d1f356fc67

13
config.h.in
View file

@ -114,6 +114,15 @@
/* Analyze Mobile IPv6 traffic */
#cmakedefine ENABLE_MOBILE_IPV6
/* Use libCurl. */
#cmakedefine USE_CURL
/* Use the DataSeries writer. */
#cmakedefine USE_DATASERIES
/* Use the ElasticSearch writer. */
#cmakedefine USE_ELASTICSEARCH
/* Version number of package */
#define VERSION "@VERSION@"
@ -162,6 +171,10 @@
#ifndef HAVE_IPPROTO_IPV6
#define IPPROTO_IPV6 41
#endif
#cmakedefine HAVE_IPPROTO_IPV4
#ifndef HAVE_IPPROTO_IPV4
#define IPPROTO_IPV4 4
#endif
#cmakedefine HAVE_IPPROTO_ROUTING
#ifndef HAVE_IPPROTO_ROUTING
#define IPPROTO_ROUTING 43

37
configure vendored
View file

@ -1,7 +1,7 @@
#!/bin/sh
# Convenience wrapper for easily viewing/setting options that
# the project's CMake scripts will recognize
set -e
command="$0 $*"
# check for `cmake` command
@ -29,12 +29,17 @@ Usage: $0 [OPTION]... [VAR=VALUE]...
Optional Features:
--enable-debug compile in debugging mode
--enable-mobile-ipv6 analyze mobile IPv6 features defined by RFC 6275
--enable-perftools force use of Google perftools on non-Linux systems
(automatically on when perftools is present on Linux)
--enable-perftools-debug use Google's perftools for debugging
--disable-broccoli don't build or install the Broccoli library
--disable-broctl don't install Broctl
--disable-auxtools don't build or install auxilliary tools
--disable-auxtools don't build or install auxiliary tools
--disable-perftools don't try to build with Google Perftools
--disable-python don't try to build python bindings for broccoli
--disable-ruby don't try to build ruby bindings for broccoli
--disable-dataseries don't use the optional DataSeries log writer
--disable-elasticsearch don't use the optional ElasticSearch log writer
Required Packages in Non-Standard Locations:
--with-openssl=PATH path to OpenSSL install root
@ -56,6 +61,9 @@ Usage: $0 [OPTION]... [VAR=VALUE]...
--with-ruby-lib=PATH path to ruby library
--with-ruby-inc=PATH path to ruby headers
--with-swig=PATH path to SWIG executable
--with-dataseries=PATH path to DataSeries and Lintel libraries
--with-xml2=PATH path to libxml2 installation (for DataSeries)
--with-curl=PATH path to libcurl install root (for ElasticSearch)
Packaging Options (for developers):
--binary-package toggle special logic for binary packaging
@ -95,6 +103,7 @@ append_cache_entry PY_MOD_INSTALL_DIR PATH $prefix/lib/broctl
append_cache_entry BRO_SCRIPT_INSTALL_PATH STRING $prefix/share/bro
append_cache_entry BRO_ETC_INSTALL_DIR PATH $prefix/etc
append_cache_entry ENABLE_DEBUG BOOL false
append_cache_entry ENABLE_PERFTOOLS BOOL false
append_cache_entry ENABLE_PERFTOOLS_DEBUG BOOL false
append_cache_entry BinPAC_SKIP_INSTALL BOOL true
append_cache_entry BUILD_SHARED_LIBS BOOL true
@ -103,6 +112,7 @@ append_cache_entry INSTALL_BROCCOLI BOOL true
append_cache_entry INSTALL_BROCTL BOOL true
append_cache_entry CPACK_SOURCE_IGNORE_FILES STRING
append_cache_entry ENABLE_MOBILE_IPV6 BOOL false
append_cache_entry DISABLE_PERFTOOLS BOOL false
# parse arguments
while [ $# -ne 0 ]; do
@ -142,7 +152,11 @@ while [ $# -ne 0 ]; do
--enable-mobile-ipv6)
append_cache_entry ENABLE_MOBILE_IPV6 BOOL true
;;
--enable-perftools)
append_cache_entry ENABLE_PERFTOOLS BOOL true
;;
--enable-perftools-debug)
append_cache_entry ENABLE_PERFTOOLS BOOL true
append_cache_entry ENABLE_PERFTOOLS_DEBUG BOOL true
;;
--disable-broccoli)
@ -154,12 +168,21 @@ while [ $# -ne 0 ]; do
--disable-auxtools)
append_cache_entry INSTALL_AUX_TOOLS BOOL false
;;
--disable-perftools)
append_cache_entry DISABLE_PERFTOOLS BOOL true
;;
--disable-python)
append_cache_entry DISABLE_PYTHON_BINDINGS BOOL true
;;
--disable-ruby)
append_cache_entry DISABLE_RUBY_BINDINGS BOOL true
;;
--disable-dataseries)
append_cache_entry DISABLE_DATASERIES BOOL true
;;
--disable-elasticsearch)
append_cache_entry DISABLE_ELASTICSEARCH BOOL true
;;
--with-openssl=*)
append_cache_entry OpenSSL_ROOT_DIR PATH $optarg
;;
@ -213,6 +236,16 @@ while [ $# -ne 0 ]; do
--with-swig=*)
append_cache_entry SWIG_EXECUTABLE PATH $optarg
;;
--with-dataseries=*)
append_cache_entry DataSeries_ROOT_DIR PATH $optarg
append_cache_entry Lintel_ROOT_DIR PATH $optarg
;;
--with-xml2=*)
append_cache_entry LibXML2_ROOT_DIR PATH $optarg
;;
--with-curl=*)
append_cache_entry LibCURL_ROOT_DIR PATH $optarg
;;
--binary-package)
append_cache_entry BINARY_PACKAGING_MODE BOOL true
;;

2
doc/ext/bro_lexer/bro.py
View file

@ -29,7 +29,7 @@ class BroLexer(RegexLexer):
r'|vector)\b', Keyword.Type),
(r'(T|F)\b', Keyword.Constant),
(r'(&)((?:add|delete|expire)_func|attr|(create|read|write)_expire'
r'|default|disable_print_hook|raw_output|encrypt|group|log'
r'|default|raw_output|encrypt|group|log'
r'|mergeable|optional|persistent|priority|redef'
r'|rotate_(?:interval|size)|synchronized)\b', bygroups(Punctuation,
Keyword)),

BIN
doc/ext/bro_lexer/bro.pyc
View file

Binary file not shown.

11
doc/faq.rst
View file

@ -46,7 +46,7 @@ directions:
http://securityonion.blogspot.com/2011/10/when-is-full-packet-capture-not-full.html
What does an error message like ``internal error: NB-DNS error`` mean?
---------------------------------------------------------------------------------------------------------------------------------
----------------------------------------------------------------------
That often means that DNS is not set up correctly on the system
running Bro. Try verifying from the command line that DNS lookups
@ -65,6 +65,15 @@ Generally, please note that we do not regularly test OpenBSD builds.
We appreciate any patches that improve Bro's support for this
platform.
How do BroControl options affect Bro script variables?
------------------------------------------------------
Some (but not all) BroControl options override a corresponding Bro script variable.
For example, setting the BroControl option "LogRotationInterval" will override
the value of the Bro script variable "Log::default_rotation_interval".
See the :doc:`BroControl Documentation <components/broctl/README>` to find out
which BroControl options override Bro script variables, and for more discussion
on site-specific customization.
Usage
=====

3
doc/index.rst
View file

@ -11,8 +11,8 @@ Guides
:maxdepth: 1
INSTALL
quickstart
upgrade
quickstart
faq
reporting-problems
@ -24,6 +24,7 @@ Frameworks
notice
logging
input
cluster
signatures

407
doc/input.rst Normal file
View file

@ -0,0 +1,407 @@
==============================================
Loading Data into Bro with the Input Framework
==============================================
.. rst-class:: opening
Bro now features a flexible input framework that allows users
to import data into Bro. Data is either read into Bro tables or
converted to events which can then be handled by scripts.
This document gives an overview of how to use the input framework
with some examples. For more complex scenarios it is
worthwhile to take a look at the unit tests in
``testing/btest/scripts/base/frameworks/input/``.
.. contents::
Reading Data into Tables
========================
Probably the most interesting use-case of the input framework is to
read data into a Bro table.
By default, the input framework reads the data in the same format
as it is written by the logging framework in Bro - a tab-separated
ASCII file.
We will show the ways to read files into Bro with a simple example.
For this example we assume that we want to import data from a blacklist
that contains server IP addresses as well as the timestamp and the reason
for the block.
An example input file could look like this:
::
#fields ip timestamp reason
192.168.17.1 1333252748 Malware host
192.168.27.2 1330235733 Botnet server
192.168.250.3 1333145108 Virus detected
To read a file into a Bro table, two record types have to be defined.
One contains the types and names of the columns that should constitute the
table keys and the second contains the types and names of the columns that
should constitute the table values.
In our case, we want to be able to lookup IPs. Hence, our key record
only contains the server IP. All other elements should be stored as
the table content.
The two records are defined as:
.. code:: bro
type Idx: record {
ip: addr;
};
type Val: record {
timestamp: time;
reason: string;
};
Note that the names of the fields in the record definitions have to correspond
to the column names listed in the '#fields' line of the log file, in this
case 'ip', 'timestamp', and 'reason'.
The log file is read into the table with a simple call of the ``add_table``
function:
.. code:: bro
global blacklist: table[addr] of Val = table();
Input::add_table([$source="blacklist.file", $name="blacklist", $idx=Idx, $val=Val, $destination=blacklist]);
Input::remove("blacklist");
With these three lines we first create an empty table that should contain the
blacklist data and then instruct the input framework to open an input stream
named ``blacklist`` to read the data into the table. The third line removes the
input stream again, because we do not need it any more after the data has been
read.
Because some data files can - potentially - be rather big, the input framework
works asynchronously. A new thread is created for each new input stream.
This thread opens the input data file, converts the data into a Bro format and
sends it back to the main Bro thread.
Because of this, the data is not immediately accessible. Depending on the
size of the data source it might take from a few milliseconds up to a few
seconds until all data is present in the table. Please note that this means
that when Bro is running without an input source or on very short captured
files, it might terminate before the data is present in the system (because
Bro already handled all packets before the import thread finished).
Subsequent calls to an input source are queued until the previous action has
been completed. Because of this, it is, for example, possible to call
``add_table`` and ``remove`` in two subsequent lines: the ``remove`` action
will remain queued until the first read has been completed.
Once the input framework finishes reading from a data source, it fires
the ``end_of_data`` event. Once this event has been received all data
from the input file is available in the table.
.. code:: bro
event Input::end_of_data(name: string, source: string) {
# now all data is in the table
print blacklist;
}
The table can also already be used while the data is still being read - it
just might not contain all lines in the input file when the event has not
yet fired. After it has been populated it can be used like any other Bro
table and blacklist entries can easily be tested:
.. code:: bro
if ( 192.168.18.12 in blacklist )
# take action
Re-reading and streaming data
-----------------------------
For many data sources, like for many blacklists, the source data is continually
changing. For these cases, the Bro input framework supports several ways to
deal with changing data files.
The first, very basic method is an explicit refresh of an input stream. When
an input stream is open, the function ``force_update`` can be called. This
will trigger a complete refresh of the table; any changed elements from the
file will be updated. After the update is finished the ``end_of_data``
event will be raised.
In our example the call would look like:
.. code:: bro
Input::force_update("blacklist");
The input framework also supports two automatic refresh modes. The first mode
continually checks if a file has been changed. If the file has been changed, it
is re-read and the data in the Bro table is updated to reflect the current
state. Each time a change has been detected and all the new data has been
read into the table, the ``end_of_data`` event is raised.
The second mode is a streaming mode. This mode assumes that the source data
file is an append-only file to which new data is continually appended. Bro
continually checks for new data at the end of the file and will add the new
data to the table. If newer lines in the file have the same index as previous
lines, they will overwrite the values in the output table. Because of the
nature of streaming reads (data is continually added to the table),
the ``end_of_data`` event is never raised when using streaming reads.
The reading mode can be selected by setting the ``mode`` option of the
add_table call. Valid values are ``MANUAL`` (the default), ``REREAD``
and ``STREAM``.
Hence, when adding ``$mode=Input::REREAD`` to the previous example, the
blacklist table will always reflect the state of the blacklist input file.
.. code:: bro
Input::add_table([$source="blacklist.file", $name="blacklist", $idx=Idx, $val=Val, $destination=blacklist, $mode=Input::REREAD]);
Receiving change events
-----------------------
When re-reading files, it might be interesting to know exactly which lines in
the source files have changed.
For this reason, the input framework can raise an event each time when a data
item is added to, removed from or changed in a table.
The event definition looks like this:
.. code:: bro
event entry(description: Input::TableDescription, tpe: Input::Event, left: Idx, right: Val) {
# act on values
}
The event has to be specified in ``$ev`` in the ``add_table`` call:
.. code:: bro
Input::add_table([$source="blacklist.file", $name="blacklist", $idx=Idx, $val=Val, $destination=blacklist, $mode=Input::REREAD, $ev=entry]);
The ``description`` field of the event contains the arguments that were
originally supplied to the add_table call. Hence, the name of the stream can,
for example, be accessed with ``description$name``. ``tpe`` is an enum
containing the type of the change that occurred.
If a line that was not previously present in the table has been added,
then ``tpe`` will contain ``Input::EVENT_NEW``. In this case ``left`` contains
the index of the added table entry and ``right`` contains the values of the
added entry.
If a table entry that already was present is altered during the re-reading or
streaming read of a file, ``tpe`` will contain ``Input::EVENT_CHANGED``. In
this case ``left`` contains the index of the changed table entry and ``right``
contains the values of the entry before the change. The reason for this is
that the table already has been updated when the event is raised. The current
value in the table can be ascertained by looking up the current table value.
Hence it is possible to compare the new and the old values of the table.
If a table element is removed because it was no longer present during a
re-read, then ``tpe`` will contain ``Input::REMOVED``. In this case ``left``
contains the index and ``right`` the values of the removed element.
Filtering data during import
----------------------------
The input framework also allows a user to filter the data during the import.
To this end, predicate functions are used. A predicate function is called
before a new element is added/changed/removed from a table. The predicate
can either accept or veto the change by returning true for an accepted
change and false for a rejected change. Furthermore, it can alter the data
before it is written to the table.
The following example filter will reject to add entries to the table when
they were generated over a month ago. It will accept all changes and all
removals of values that are already present in the table.
.. code:: bro
Input::add_table([$source="blacklist.file", $name="blacklist", $idx=Idx, $val=Val, $destination=blacklist, $mode=Input::REREAD,
$pred(typ: Input::Event, left: Idx, right: Val) = {
if ( typ != Input::EVENT_NEW ) {
return T;
}
return ( ( current_time() - right$timestamp ) < (30 day) );
}]);
To change elements while they are being imported, the predicate function can
manipulate ``left`` and ``right``. Note that predicate functions are called
before the change is committed to the table. Hence, when a table element is
changed (``tpe`` is ``INPUT::EVENT_CHANGED``), ``left`` and ``right``
contain the new values, but the destination (``blacklist`` in our example)
still contains the old values. This allows predicate functions to examine
the changes between the old and the new version before deciding if they
should be allowed.
Different readers
-----------------
The input framework supports different kinds of readers for different kinds
of source data files. At the moment, the default reader reads ASCII files
formatted in the Bro log file format (tab-separated values). At the moment,
Bro comes with two other readers. The ``RAW`` reader reads a file that is
split by a specified record separator (usually newline). The contents are
returned line-by-line as strings; it can, for example, be used to read
configuration files and the like and is probably
only useful in the event mode and not for reading data to tables.
Another included reader is the ``BENCHMARK`` reader, which is being used
to optimize the speed of the input framework. It can generate arbitrary
amounts of semi-random data in all Bro data types supported by the input
framework.
In the future, the input framework will get support for new data sources
like, for example, different databases.
Add_table options
-----------------
This section lists all possible options that can be used for the add_table
function and gives a short explanation of their use. Most of the options
already have been discussed in the previous sections.
The possible fields that can be set for a table stream are:
``source``
A mandatory string identifying the source of the data.
For the ASCII reader this is the filename.
``name``
A mandatory name for the filter that can later be used
to manipulate it further.
``idx``
Record type that defines the index of the table.
``val``
Record type that defines the values of the table.
``reader``
The reader used for this stream. Default is ``READER_ASCII``.
``mode``
The mode in which the stream is opened. Possible values are
``MANUAL``, ``REREAD`` and ``STREAM``. Default is ``MANUAL``.
``MANUAL`` means that the file is not updated after it has
been read. Changes to the file will not be reflected in the
data Bro knows. ``REREAD`` means that the whole file is read
again each time a change is found. This should be used for
files that are mapped to a table where individual lines can
change. ``STREAM`` means that the data from the file is
streamed. Events / table entries will be generated as new
data is appended to the file.
``destination``
The destination table.
``ev``
Optional event that is raised, when values are added to,
changed in, or deleted from the table. Events are passed an
Input::Event description as the first argument, the index
record as the second argument and the values as the third
argument.
``pred``
Optional predicate, that can prevent entries from being added
to the table and events from being sent.
``want_record``
Boolean value, that defines if the event wants to receive the
fields inside of a single record value, or individually
(default). This can be used if ``val`` is a record
containing only one type. In this case, if ``want_record`` is
set to false, the table will contain elements of the type
contained in ``val``.
Reading Data to Events
======================
The second supported mode of the input framework is reading data to Bro
events instead of reading them to a table using event streams.
Event streams work very similarly to table streams that were already
discussed in much detail. To read the blacklist of the previous example
into an event stream, the following Bro code could be used:
.. code:: bro
type Val: record {
ip: addr;
timestamp: time;
reason: string;
};
event blacklistentry(description: Input::EventDescription, tpe: Input::Event, ip: addr, timestamp: time, reason: string) {
# work with event data
}
event bro_init() {
Input::add_event([$source="blacklist.file", $name="blacklist", $fields=Val, $ev=blacklistentry]);
}
The main difference in the declaration of the event stream is, that an event
stream needs no separate index and value declarations -- instead, all source
data types are provided in a single record definition.
Apart from this, event streams work exactly the same as table streams and
support most of the options that are also supported for table streams.
The options that can be set when creating an event stream with
``add_event`` are:
``source``
A mandatory string identifying the source of the data.
For the ASCII reader this is the filename.
``name``
A mandatory name for the stream that can later be used
to remove it.
``fields``
Name of a record type containing the fields, which should be
retrieved from the input stream.
``ev``
The event which is fired, after a line has been read from the
input source. The first argument that is passed to the event
is an Input::Event structure, followed by the data, either
inside of a record (if ``want_record is set``) or as
individual fields. The Input::Event structure can contain
information, if the received line is ``NEW``, has been
``CHANGED`` or ``DELETED``. Since the ASCII reader cannot
track this information for event filters, the value is
always ``NEW`` at the moment.
``mode``
The mode in which the stream is opened. Possible values are
``MANUAL``, ``REREAD`` and ``STREAM``. Default is ``MANUAL``.
``MANUAL`` means that the file is not updated after it has
been read. Changes to the file will not be reflected in the
data Bro knows. ``REREAD`` means that the whole file is read
again each time a change is found. This should be used for
files that are mapped to a table where individual lines can
change. ``STREAM`` means that the data from the file is
streamed. Events / table entries will be generated as new
data is appended to the file.
``reader``
The reader used for this stream. Default is ``READER_ASCII``.
``want_record``
Boolean value, that defines if the event wants to receive the
fields inside of a single record value, or individually
(default). If this is set to true, the event will receive a
single record of the type provided in ``fields``.

125
doc/intel.rst Normal file
View file

@ -0,0 +1,125 @@
Intel Framework
===============
Intro
-----
Intelligence data is critical to the process of monitoring for
security purposes. There is always data which will be discovered
through the incident response process and data which is shared through
private communities. The goals of Bro's Intelligence Framework are to
consume that data, make it available for matching, and provide
infrastructure around improving performance, memory utilization, and
generally making all of this easier.
Data in the Intelligence Framework is the atomic piece of intelligence
such as an IP address or an e-mail address along with a suite of
metadata about it such as a freeform source field, a freeform
descriptive field and a URL which might lead to more information about
the specific item. The metadata in the default scripts has been
deliberately kept minimal so that the community can find the
appropriate fields that need added by writing scripts which extend the
base record using the normal record extension mechanism.
Quick Start
-----------
Load the package of scripts that sends data into the Intelligence
Framework to be checked by loading this script in local.bro::
@load policy/frameworks/intel
(TODO: find some good mechanism for getting setup with good data
quickly)
Refer to the "Loading Intelligence" section below to see the format
for Intelligence Framework text files, then load those text files with
this line in local.bro::
redef Intel::read_files += { "/somewhere/yourdata.txt" };
The data itself only needs to reside on the manager if running in a
cluster.
Architecture
------------
The Intelligence Framework can be thought of as containing three
separate portions. The first part is how intelligence is loaded,
followed by the mechanism for indicating to the intelligence framework
that a piece of data which needs to be checked has been seen, and
thirdly the part where a positive match has been discovered.
Loading Intelligence
********************
Intelligence data can only be loaded through plain text files using
the Input Framework conventions. Additionally, on clusters the
manager is the only node that needs the intelligence data. The
intelligence framework has distribution mechanisms which will push
data out to all of the nodes that need it.
Here is an example of the intelligence data format. Note that all
whitespace separators are literal tabs and fields containing only a
hyphen a considered to be null values.::
#fields host net str str_type meta.source meta.desc meta.url
1.2.3.4 - - - source1 Sending phishing email http://source1.com/badhosts/1.2.3.4
- 31.131.248.0/21 - - spamhaus-drop SBL154982 - -
- - a.b.com Intel::DOMAIN source2 Name used for data exfiltration -
For more examples of built in `str_type` values, please refer to the
autogenerated documentation for the intelligence framework (TODO:
figure out how to do this link).
To load the data once files are created, use the following example
code to define files to load with your own file names of course::
redef Intel::read_files += {
"/somewhere/feed1.txt",
"/somewhere/feed2.txt",
};
Remember, the files only need to be present on the file system of the
manager node on cluster deployments.
Seen Data
*********
When some bit of data is extracted (such as an email address in the
"From" header in a message over SMTP), the Intelligence Framework
needs to be informed that this data was discovered and it's presence
should be checked within the intelligence data set. This is
accomplished through the Intel::seen (TODO: do a reference link)
function.
Typically users won't need to work with this function due to built in
hook scripts that Bro ships with that will "see" data and send it into
the intelligence framework. A user may only need to load the entire
package of hook scripts as a module or pick and choose specific
scripts to load. Keep in mind that as more data is sent into the
intelligence framework, the CPU load consumed by Bro will increase
depending on how many times the Intel::seen function is being called
which is heavily traffic dependent.
The full package of hook scripts that Bro ships with for sending this
"seen" data into the intelligence framework can be loading by adding
this line to local.bro::
@load policy/frameworks/intel
Intelligence Matches
********************
Against all hopes, most networks will eventually have a hit on
intelligence data which could indicate a possible compromise or other
unwanted activity. The Intelligence Framework provides an event that
is generated whenever a match is discovered named Intel::match (TODO:
make a link to inline docs). Due to design restrictions placed upon
the intelligence framework, there is no assurance as to where this
event will be generated. It could be generated on the worker where
the data was seen or on the manager. When the Intel::match event is
handled, only the data given as event arguments to the event can be
assured since the host where the data was seen may not be where
Intel::match is handled.

186
doc/logging-dataseries.rst Normal file
View file

@ -0,0 +1,186 @@
=============================
Binary Output with DataSeries
=============================
.. rst-class:: opening
Bro's default ASCII log format is not exactly the most efficient
way for storing and searching large volumes of data. An an
alternative, Bro comes with experimental support for `DataSeries
<http://www.hpl.hp.com/techreports/2009/HPL-2009-323.html>`_
output, an efficient binary format for recording structured bulk
data. DataSeries is developed and maintained at HP Labs.
.. contents::
Installing DataSeries
---------------------
To use DataSeries, its libraries must be available at compile-time,
along with the supporting *Lintel* package. Generally, both are
distributed on `HP Labs' web site
<http://tesla.hpl.hp.com/opensource/>`_. Currently, however, you need
to use recent development versions for both packages, which you can
download from github like this::
git clone http://github.com/dataseries/Lintel
git clone http://github.com/dataseries/DataSeries
To build and install the two into ``<prefix>``, do::
( cd Lintel && mkdir build && cd build && cmake -DCMAKE_INSTALL_PREFIX=<prefix> .. && make && make install )
( cd DataSeries && mkdir build && cd build && cmake -DCMAKE_INSTALL_PREFIX=<prefix> .. && make && make install )
Please refer to the packages' documentation for more information about
the installation process. In particular, there's more information on
required and optional `dependencies for Lintel
<https://raw.github.com/dataseries/Lintel/master/doc/dependencies.txt>`_
and `dependencies for DataSeries
<https://raw.github.com/dataseries/DataSeries/master/doc/dependencies.txt>`_.
For users on RedHat-style systems, you'll need the following::
yum install libxml2-devel boost-devel
Compiling Bro with DataSeries Support
-------------------------------------
Once you have installed DataSeries, Bro's ``configure`` should pick it
up automatically as long as it finds it in a standard system location.
Alternatively, you can specify the DataSeries installation prefix
manually with ``--with-dataseries=<prefix>``. Keep an eye on
``configure``'s summary output, if it looks like the following, Bro
found DataSeries and will compile in the support::
# ./configure --with-dataseries=/usr/local
[...]
====================| Bro Build Summary |=====================
[...]
DataSeries: true
[...]
================================================================
Activating DataSeries
---------------------
The direct way to use DataSeries is to switch *all* log files over to
the binary format. To do that, just add ``redef
Log::default_writer=Log::WRITER_DATASERIES;`` to your ``local.bro``.
For testing, you can also just pass that on the command line::
bro -r trace.pcap Log::default_writer=Log::WRITER_DATASERIES
With that, Bro will now write all its output into DataSeries files
``*.ds``. You can inspect these using DataSeries's set of command line
tools, which its installation process installs into ``<prefix>/bin``.
For example, to convert a file back into an ASCII representation::
$ ds2txt conn.log
[... We skip a bunch of metadata here ...]
ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto service duration orig_bytes resp_bytes conn_state local_orig missed_bytes history orig_pkts orig_ip_bytes resp_pkts resp_ip_bytes
1300475167.096535 CRCC5OdDlXe 141.142.220.202 5353 224.0.0.251 5353 udp dns 0.000000 0 0 S0 F 0 D 1 73 0 0
1300475167.097012 o7XBsfvo3U1 fe80::217:f2ff:fed7:cf65 5353 ff02::fb 5353 udp 0.000000 0 0 S0 F 0 D 1 199 0 0
1300475167.099816 pXPi1kPMgxb 141.142.220.50 5353 224.0.0.251 5353 udp 0.000000 0 0 S0 F 0 D 1 179 0 0
1300475168.853899 R7sOc16woCj 141.142.220.118 43927 141.142.2.2 53 udp dns 0.000435 38 89 SF F 0 Dd 1 66 1 117
1300475168.854378 Z6dfHVmt0X7 141.142.220.118 37676 141.142.2.2 53 udp dns 0.000420 52 99 SF F 0 Dd 1 80 1 127
1300475168.854837 k6T92WxgNAh 141.142.220.118 40526 141.142.2.2 53 udp dns 0.000392 38 183 SF F 0 Dd 1 66 1 211
[...]
(``--skip-all`` suppresses the metadata.)
Note that the ASCII conversion is *not* equivalent to Bro's default
output format.
You can also switch only individual files over to DataSeries by adding
code like this to your ``local.bro``:
.. code:: bro
event bro_init()
{
local f = Log::get_filter(Conn::LOG, "default"); # Get default filter for connection log.
f$writer = Log::WRITER_DATASERIES; # Change writer type.
Log::add_filter(Conn::LOG, f); # Replace filter with adapted version.
}
Bro's DataSeries writer comes with a few tuning options, see
:doc:`scripts/base/frameworks/logging/writers/dataseries`.
Working with DataSeries
=======================
Here are a few examples of using DataSeries command line tools to work
with the output files.
* Printing CSV::
$ ds2txt --csv conn.log
ts,uid,id.orig_h,id.orig_p,id.resp_h,id.resp_p,proto,service,duration,orig_bytes,resp_bytes,conn_state,local_orig,missed_bytes,history,orig_pkts,orig_ip_bytes,resp_pkts,resp_ip_bytes
1258790493.773208,ZTtgbHvf4s3,192.168.1.104,137,192.168.1.255,137,udp,dns,3.748891,350,0,S0,F,0,D,7,546,0,0
1258790451.402091,pOY6Rw7lhUd,192.168.1.106,138,192.168.1.255,138,udp,,0.000000,0,0,S0,F,0,D,1,229,0,0
1258790493.787448,pn5IiEslca9,192.168.1.104,138,192.168.1.255,138,udp,,2.243339,348,0,S0,F,0,D,2,404,0,0
1258790615.268111,D9slyIu3hFj,192.168.1.106,137,192.168.1.255,137,udp,dns,3.764626,350,0,S0,F,0,D,7,546,0,0
[...]
Add ``--separator=X`` to set a different separator.
* Extracting a subset of columns::
$ ds2txt --select '*' ts,id.resp_h,id.resp_p --skip-all conn.log
1258790493.773208 192.168.1.255 137
1258790451.402091 192.168.1.255 138
1258790493.787448 192.168.1.255 138
1258790615.268111 192.168.1.255 137
1258790615.289842 192.168.1.255 138
[...]
* Filtering rows::
$ ds2txt --where '*' 'duration > 5 && id.resp_p > 1024' --skip-all conn.ds
1258790631.532888 V8mV5WLITu5 192.168.1.105 55890 239.255.255.250 1900 udp 15.004568 798 0 S0 F 0 D 6 966 0 0
1258792413.439596 tMcWVWQptvd 192.168.1.105 55890 239.255.255.250 1900 udp 15.004581 798 0 S0 F 0 D 6 966 0 0
1258794195.346127 cQwQMRdBrKa 192.168.1.105 55890 239.255.255.250 1900 udp 15.005071 798 0 S0 F 0 D 6 966 0 0
1258795977.253200 i8TEjhWd2W8 192.168.1.105 55890 239.255.255.250 1900 udp 15.004824 798 0 S0 F 0 D 6 966 0 0
1258797759.160217 MsLsBA8Ia49 192.168.1.105 55890 239.255.255.250 1900 udp 15.005078 798 0 S0 F 0 D 6 966 0 0
1258799541.068452 TsOxRWJRGwf 192.168.1.105 55890 239.255.255.250 1900 udp 15.004082 798 0 S0 F 0 D 6 966 0 0
[...]
* Calculate some statistics:
Mean/stddev/min/max over a column::
$ dsstatgroupby '*' basic duration from conn.ds
# Begin DSStatGroupByModule
# processed 2159 rows, where clause eliminated 0 rows
# count(*), mean(duration), stddev, min, max
2159, 42.7938, 1858.34, 0, 86370
[...]
Quantiles of total connection volume::
$ dsstatgroupby '*' quantile 'orig_bytes + resp_bytes' from conn.ds
[...]
2159 data points, mean 24616 +- 343295 [0,1.26615e+07]
quantiles about every 216 data points:
10%: 0, 124, 317, 348, 350, 350, 601, 798, 1469
tails: 90%: 1469, 95%: 7302, 99%: 242629, 99.5%: 1226262
[...]
The ``man`` pages for these tools show further options, and their
``-h`` option gives some more information (either can be a bit cryptic
unfortunately though).
Deficiencies
------------
Due to limitations of the DataSeries format, one cannot inspect its
files before they have been fully written. In other words, when using
DataSeries, it's currently not possible to inspect the live log
files inside the spool directory before they are rotated to their
final location. It seems that this could be fixed with some effort,
and we will work with DataSeries development team on that if the
format gains traction among Bro users.
Likewise, we're considering writing custom command line tools for
interacting with DataSeries files, making that a bit more convenient
than what the standard utilities provide.

89
doc/logging-elasticsearch.rst Normal file
View file

@ -0,0 +1,89 @@
=========================================
Indexed Logging Output with ElasticSearch
=========================================
.. rst-class:: opening
Bro's default ASCII log format is not exactly the most efficient
way for searching large volumes of data. ElasticSearch
is a new data storage technology for dealing with tons of data.
It's also a search engine built on top of Apache's Lucene
project. It scales very well, both for distributed indexing and
distributed searching.
.. contents::
Warning
-------
This writer plugin is still in testing and is not yet recommended for
production use! The approach to how logs are handled in the plugin is "fire
and forget" at this time, there is no error handling if the server fails to
respond successfully to the insertion request.
Installing ElasticSearch
------------------------
Download the latest version from: <http://www.elasticsearch.org/download/>.
Once extracted, start ElasticSearch with::
# ./bin/elasticsearch
For more detailed information, refer to the ElasticSearch installation
documentation: http://www.elasticsearch.org/guide/reference/setup/installation.html
Compiling Bro with ElasticSearch Support
----------------------------------------
First, ensure that you have libcurl installed the run configure.::
# ./configure
[...]
====================| Bro Build Summary |=====================
[...]
cURL: true
[...]
ElasticSearch: true
[...]
================================================================
Activating ElasticSearch
------------------------
The easiest way to enable ElasticSearch output is to load the tuning/logs-to-
elasticsearch.bro script. If you are using BroControl, the following line in
local.bro will enable it.
.. console::
@load tuning/logs-to-elasticsearch
With that, Bro will now write most of its logs into ElasticSearch in addition
to maintaining the Ascii logs like it would do by default. That script has
some tunable options for choosing which logs to send to ElasticSearch, refer
to the autogenerated script documentation for those options.
There is an interface being written specifically to integrate with the data
that Bro outputs into ElasticSearch named Brownian. It can be found here::
https://github.com/grigorescu/Brownian
Tuning
------
A common problem encountered with ElasticSearch is too many files being held
open. The ElasticSearch website has some suggestions on how to increase the
open file limit.
- http://www.elasticsearch.org/tutorials/2011/04/06/too-many-open-files.html
TODO
----
Lots.
- Perform multicast discovery for server.
- Better error detection.
- Better defaults (don't index loaded-plugins, for instance).
-

11
doc/logging.rst
View file

@ -373,3 +373,14 @@ record, care must be given to when and how long data is stored.
Normally data saved to the connection record will remain there for the
duration of the connection and from a practical perspective it's not
uncommon to need to delete that data before the end of the connection.
Other Writers
-------------
Bro supports the following output formats other than ASCII:
.. toctree::
:maxdepth: 1
logging-dataseries
logging-elasticsearch

184
doc/quickstart.rst
View file

@ -1,9 +1,3 @@
.. _CMake: http://www.cmake.org
.. _SWIG: http://www.swig.org
.. _MacPorts: http://www.macports.org
.. _Fink: http://www.finkproject.org
.. _Homebrew: http://mxcl.github.com/homebrew
.. _bro downloads page: http://bro-ids.org/download/index.html
=================
Quick Start Guide
@ -22,178 +16,11 @@ Installation
Bro works on most modern, Unix-based systems and requires no custom
hardware. It can be downloaded in either pre-built binary package or
source code forms.
source code forms. See :doc:`Installing Bro <INSTALL>` for instructions
on how to install Bro.
Pre-Built Binary Release Packages
---------------------------------
See the `bro downloads page`_ for currently supported/targeted platforms.
* RPM
.. console::
sudo yum localinstall Bro-*.rpm
* DEB
.. console::
sudo gdebi Bro-*.deb
* MacOS Disk Image with Installer
Just open the ``Bro-*.dmg`` and then run the ``.pkg`` installer.
Everything installed by the package will go into ``/opt/bro``.
The primary install prefix for binary packages is ``/opt/bro``.
Non-MacOS packages that include BroControl also put variable/runtime
data (e.g. Bro logs) in ``/var/opt/bro``.
Building From Source
--------------------
Required Dependencies
~~~~~~~~~~~~~~~~~~~~~
The following dependencies are required to build Bro:
* RPM/RedHat-based Linux:
.. console::
sudo yum install cmake make gcc gcc-c++ flex bison libpcap-devel openssl-devel python-devel swig zlib-devel file-devel
* DEB/Debian-based Linux:
.. console::
sudo apt-get install cmake make gcc g++ flex bison libpcap-dev libssl-dev python-dev swig zlib1g-dev libmagic-dev
* FreeBSD
Most required dependencies should come with a minimal FreeBSD install
except for the following.
.. console::
sudo pkg_add -r bash cmake swig bison python
Note that ``bash`` needs to be in ``PATH``, which by default it is
not. The FreeBSD package installs the binary into
``/usr/local/bin``.
* Mac OS X
Snow Leopard (10.6) comes with all required dependencies except for CMake_.
Lion (10.7) comes with all required dependencies except for CMake_ and SWIG_.
Distributions of these dependencies can be obtained from the project websites
linked above, but they're also likely available from your preferred Mac OS X
package management system (e.g. MacPorts_, Fink_, or Homebrew_).
Note that the MacPorts ``swig`` package may not include any specific
language support so you may need to also install ``swig-ruby`` and
``swig-python``.
Optional Dependencies
~~~~~~~~~~~~~~~~~~~~~
Bro can use libGeoIP for geo-locating IP addresses, and sendmail for
sending emails.
* RedHat Enterprise Linux:
.. console::
sudo yum install geoip-devel sendmail
* CentOS Linux:
.. console::
sudo yum install GeoIP-devel sendmail
* DEB/Debian-based Linux:
.. console::
sudo apt-get install libgeoip-dev sendmail
* Ports-based FreeBSD
.. console::
sudo pkg_add -r GeoIP
sendmail is typically already available.
* Mac OS X
Vanilla OS X installations don't ship with libmagic or libGeoIP, but
if installed from your preferred package management system (e.g. MacPorts,
Fink, or Homebrew), they should be automatically detected and Bro will compile
against them.
Additional steps may be needed to :doc:`get the right GeoIP database <geoip>`
Compiling Bro Source Code
~~~~~~~~~~~~~~~~~~~~~~~~~
Bro releases are bundled into source packages for convenience and
available from the `bro downloads page`_.
The latest Bro development versions are obtainable through git
repositories hosted at `git.bro-ids.org <http://git.bro-ids.org>`_. See
our `git development documentation
<http://bro-ids.org/development/process.html>`_ for comprehensive
information on Bro's use of git revision control, but the short story
for downloading the full source code experience for Bro via git is:
.. console::
git clone --recursive git://git.bro-ids.org/bro
.. note:: If you choose to clone the ``bro`` repository non-recursively for
a "minimal Bro experience", be aware that compiling it depends on
BinPAC, which has its own ``binpac`` repository. Either install it
first or initialize/update the cloned ``bro`` repository's
``aux/binpac`` submodule.
See the ``INSTALL`` file included with the source code for more information
on compiling, but this is the typical way to build and install from source
(of course, changing the value of the ``--prefix`` option to point to the
desired root install path):
.. console::
./configure --prefix=/desired/install/path
make
make install
The default installation prefix is ``/usr/local/bro``, which would typically
require root privileges when doing the ``make install``.
Configure the Run-Time Environment
----------------------------------
Just remember that you may need to adjust your ``PATH`` environment variable
according to the platform/shell/package you're using. For example:
Bourne-Shell Syntax:
.. console::
export PATH=/usr/local/bro/bin:$PATH
C-Shell Syntax:
.. console::
setenv PATH /usr/local/bro/bin:$PATH
Or substitute ``/opt/bro/bin`` instead if you installed from a binary package.
.. note:: Below, ``$PREFIX`` is used to reference the Bro installation
root directory.
Using BroControl
================
@ -202,9 +29,6 @@ BroControl is an interactive shell for easily operating/managing Bro
installations on a single system or even across multiple systems in a
traffic-monitoring cluster.
.. note:: Below, ``$PREFIX`` is used to reference the Bro installation
root directory.
A Minimal Starting Configuration
--------------------------------

33
doc/scripts/DocSourcesList.cmake
View file

@ -19,6 +19,7 @@ rest_target(${psd} base/init-bare.bro internal)
rest_target(${CMAKE_BINARY_DIR}/src base/bro.bif.bro)
rest_target(${CMAKE_BINARY_DIR}/src base/const.bif.bro)
rest_target(${CMAKE_BINARY_DIR}/src base/event.bif.bro)
rest_target(${CMAKE_BINARY_DIR}/src base/input.bif.bro)
rest_target(${CMAKE_BINARY_DIR}/src base/logging.bif.bro)
rest_target(${CMAKE_BINARY_DIR}/src base/reporter.bif.bro)
rest_target(${CMAKE_BINARY_DIR}/src base/strings.bif.bro)
@ -31,11 +32,20 @@ rest_target(${psd} base/frameworks/cluster/setup-connections.bro)
rest_target(${psd} base/frameworks/communication/main.bro)
rest_target(${psd} base/frameworks/control/main.bro)
rest_target(${psd} base/frameworks/dpd/main.bro)
rest_target(${psd} base/frameworks/input/main.bro)
rest_target(${psd} base/frameworks/input/readers/ascii.bro)
rest_target(${psd} base/frameworks/input/readers/benchmark.bro)
rest_target(${psd} base/frameworks/input/readers/raw.bro)
rest_target(${psd} base/frameworks/intel/cluster.bro)
rest_target(${psd} base/frameworks/intel/input.bro)
rest_target(${psd} base/frameworks/intel/main.bro)
rest_target(${psd} base/frameworks/logging/main.bro)
rest_target(${psd} base/frameworks/logging/postprocessors/scp.bro)
rest_target(${psd} base/frameworks/logging/postprocessors/sftp.bro)
rest_target(${psd} base/frameworks/logging/writers/ascii.bro)
rest_target(${psd} base/frameworks/logging/writers/dataseries.bro)
rest_target(${psd} base/frameworks/logging/writers/elasticsearch.bro)
rest_target(${psd} base/frameworks/logging/writers/none.bro)
rest_target(${psd} base/frameworks/metrics/cluster.bro)
rest_target(${psd} base/frameworks/metrics/main.bro)
rest_target(${psd} base/frameworks/metrics/non-cluster.bro)
@ -53,12 +63,16 @@ rest_target(${psd} base/frameworks/packet-filter/netstats.bro)
rest_target(${psd} base/frameworks/reporter/main.bro)
rest_target(${psd} base/frameworks/signatures/main.bro)
rest_target(${psd} base/frameworks/software/main.bro)
rest_target(${psd} base/frameworks/tunnels/main.bro)
rest_target(${psd} base/misc/find-checksum-offloading.bro)
rest_target(${psd} base/protocols/conn/contents.bro)
rest_target(${psd} base/protocols/conn/inactivity.bro)
rest_target(${psd} base/protocols/conn/main.bro)
rest_target(${psd} base/protocols/conn/polling.bro)
rest_target(${psd} base/protocols/dns/consts.bro)
rest_target(${psd} base/protocols/dns/main.bro)
rest_target(${psd} base/protocols/ftp/file-extract.bro)
rest_target(${psd} base/protocols/ftp/gridftp.bro)
rest_target(${psd} base/protocols/ftp/main.bro)
rest_target(${psd} base/protocols/ftp/utils-commands.bro)
rest_target(${psd} base/protocols/http/file-extract.bro)
@ -68,9 +82,13 @@ rest_target(${psd} base/protocols/http/main.bro)
rest_target(${psd} base/protocols/http/utils.bro)
rest_target(${psd} base/protocols/irc/dcc-send.bro)
rest_target(${psd} base/protocols/irc/main.bro)
rest_target(${psd} base/protocols/modbus/consts.bro)
rest_target(${psd} base/protocols/modbus/main.bro)
rest_target(${psd} base/protocols/smtp/entities-excerpt.bro)
rest_target(${psd} base/protocols/smtp/entities.bro)
rest_target(${psd} base/protocols/smtp/main.bro)
rest_target(${psd} base/protocols/socks/consts.bro)
rest_target(${psd} base/protocols/socks/main.bro)
rest_target(${psd} base/protocols/ssh/main.bro)
rest_target(${psd} base/protocols/ssl/consts.bro)
rest_target(${psd} base/protocols/ssl/main.bro)
@ -87,11 +105,21 @@ rest_target(${psd} base/utils/patterns.bro)
rest_target(${psd} base/utils/site.bro)
rest_target(${psd} base/utils/strings.bro)
rest_target(${psd} base/utils/thresholds.bro)
rest_target(${psd} base/utils/urls.bro)
rest_target(${psd} policy/frameworks/communication/listen.bro)
rest_target(${psd} policy/frameworks/control/controllee.bro)
rest_target(${psd} policy/frameworks/control/controller.bro)
rest_target(${psd} policy/frameworks/dpd/detect-protocols.bro)
rest_target(${psd} policy/frameworks/dpd/packet-segment-logging.bro)
rest_target(${psd} policy/frameworks/intel/conn-established.bro)
rest_target(${psd} policy/frameworks/intel/dns.bro)
rest_target(${psd} policy/frameworks/intel/http-host-header.bro)
rest_target(${psd} policy/frameworks/intel/http-url.bro)
rest_target(${psd} policy/frameworks/intel/http-user-agents.bro)
rest_target(${psd} policy/frameworks/intel/smtp-url-extraction.bro)
rest_target(${psd} policy/frameworks/intel/smtp.bro)
rest_target(${psd} policy/frameworks/intel/ssl.bro)
rest_target(${psd} policy/frameworks/intel/where-locations.bro)
rest_target(${psd} policy/frameworks/metrics/conn-example.bro)
rest_target(${psd} policy/frameworks/metrics/http-example.bro)
rest_target(${psd} policy/frameworks/metrics/ssl-example.bro)
@ -99,6 +127,7 @@ rest_target(${psd} policy/frameworks/software/version-changes.bro)
rest_target(${psd} policy/frameworks/software/vulnerable.bro)
rest_target(${psd} policy/integration/barnyard2/main.bro)
rest_target(${psd} policy/integration/barnyard2/types.bro)
rest_target(${psd} policy/integration/collective-intel/main.bro)
rest_target(${psd} policy/misc/analysis-groups.bro)
rest_target(${psd} policy/misc/capture-loss.bro)
rest_target(${psd} policy/misc/loaded-scripts.bro)
@ -113,7 +142,6 @@ rest_target(${psd} policy/protocols/dns/detect-external-names.bro)
rest_target(${psd} policy/protocols/ftp/detect.bro)
rest_target(${psd} policy/protocols/ftp/software.bro)
rest_target(${psd} policy/protocols/http/detect-MHR.bro)
rest_target(${psd} policy/protocols/http/detect-intel.bro)
rest_target(${psd} policy/protocols/http/detect-sqli.bro)
rest_target(${psd} policy/protocols/http/detect-webapps.bro)
rest_target(${psd} policy/protocols/http/header-names.bro)
@ -121,6 +149,8 @@ rest_target(${psd} policy/protocols/http/software-browser-plugins.bro)
rest_target(${psd} policy/protocols/http/software.bro)
rest_target(${psd} policy/protocols/http/var-extraction-cookies.bro)
rest_target(${psd} policy/protocols/http/var-extraction-uri.bro)
rest_target(${psd} policy/protocols/modbus/known-masters-slaves.bro)
rest_target(${psd} policy/protocols/modbus/track-memmap.bro)
rest_target(${psd} policy/protocols/smtp/blocklists.bro)
rest_target(${psd} policy/protocols/smtp/detect-suspicious-orig.bro)
rest_target(${psd} policy/protocols/smtp/software.bro)
@ -135,6 +165,7 @@ rest_target(${psd} policy/protocols/ssl/known-certs.bro)
rest_target(${psd} policy/protocols/ssl/validate-certs.bro)
rest_target(${psd} policy/tuning/defaults/packet-fragments.bro)
rest_target(${psd} policy/tuning/defaults/warnings.bro)
rest_target(${psd} policy/tuning/logs-to-elasticsearch.bro)
rest_target(${psd} policy/tuning/track-all-assets.bro)
rest_target(${psd} site/local-manager.bro)
rest_target(${psd} site/local-proxy.bro)

28
doc/scripts/builtins.rst
View file

@ -55,8 +55,8 @@ The Bro scripting language supports the following built-in types.
A temporal type representing a relative time. An ``interval``
constant can be written as a numeric constant followed by a time
unit where the time unit is one of ``usec``, ``sec``, ``min``,
``hr``, or ``day`` which respectively represent microseconds,
unit where the time unit is one of ``usec``, ``msec``, ``sec``, ``min``,
``hr``, or ``day`` which respectively represent microseconds, milliseconds,
seconds, minutes, hours, and days. Whitespace between the numeric
constant and time unit is optional. Appending the letter "s" to the
time unit in order to pluralize it is also optional (to no semantic
@ -95,14 +95,14 @@ The Bro scripting language supports the following built-in types.
and embedded.
In exact matching the ``==`` equality relational operator is used
with one :bro:type:`string` operand and one :bro:type:`pattern`
operand to check whether the full string exactly matches the
pattern. In this case, the ``^`` beginning-of-line and ``$``
end-of-line anchors are redundant since pattern is implicitly
anchored to the beginning and end of the line to facilitate an exact
match. For example::
with one :bro:type:`pattern` operand and one :bro:type:`string`
operand (order of operands does not matter) to check whether the full
string exactly matches the pattern. In exact matching, the ``^``
beginning-of-line and ``$`` end-of-line anchors are redundant since
the pattern is implicitly anchored to the beginning and end of the
line to facilitate an exact match. For example::
"foo" == /foo|bar/
/foo|bar/ == "foo"
yields true, while::
@ -110,9 +110,9 @@ The Bro scripting language supports the following built-in types.
yields false. The ``!=`` operator would yield the negation of ``==``.
In embedded matching the ``in`` operator is again used with one
:bro:type:`string` operand and one :bro:type:`pattern` operand
(which must be on the left-hand side), but tests whether the pattern
In embedded matching the ``in`` operator is used with one
:bro:type:`pattern` operand (which must be on the left-hand side) and
one :bro:type:`string` operand, but tests whether the pattern
appears anywhere within the given string. For example::
/foo|bar/ in "foobar"
@ -600,10 +600,6 @@ scripting language supports the following built-in attributes.
.. TODO: needs to be documented.
.. bro:attr:: &disable_print_hook
Deprecated. Will be removed.
.. bro:attr:: &raw_output
Opens a file in raw mode, i.e., non-ASCII characters are not

92
doc/signatures.rst
View file

@ -51,13 +51,18 @@ This script contains a default event handler that raises
:bro:enum:`Signatures::Sensitive_Signature` :doc:`Notices <notice>`
(as well as others; see the beginning of the script).
As signatures are independent of Bro's policy scripts, they are put
into their own file(s). There are two ways to specify which files
contain signatures: By using the ``-s`` flag when you invoke Bro, or
by extending the Bro variable :bro:id:`signature_files` using the ``+=``
operator. If a signature file is given without a path, it is searched
along the normal ``BROPATH``. The default extension of the file name
is ``.sig``, and Bro appends that automatically when necessary.
As signatures are independent of Bro's policy scripts, they are put into
their own file(s). There are three ways to specify which files contain
signatures: By using the ``-s`` flag when you invoke Bro, or by
extending the Bro variable :bro:id:`signature_files` using the ``+=``
operator, or by using the ``@load-sigs`` directive inside a Bro script.
If a signature file is given without a full path, it is searched for
along the normal ``BROPATH``. Additionally, the ``@load-sigs``
directive can be used to load signature files in a path relative to the
Bro script in which it's placed, e.g. ``@load-sigs ./mysigs.sig`` will
expect that signature file in the same directory as the Bro script. The
default extension of the file name is ``.sig``, and Bro appends that
automatically when necessary.
Signature language
==================
@ -78,9 +83,8 @@ Header Conditions
~~~~~~~~~~~~~~~~~
Header conditions limit the applicability of the signature to a subset
of traffic that contains matching packet headers. For TCP, this match
is performed only for the first packet of a connection. For other
protocols, it is done on each individual packet.
of traffic that contains matching packet headers. This type of matching
is performed only for the first packet of a connection.
There are pre-defined header conditions for some of the most used
header fields. All of them generally have the format ``<keyword> <cmp>
@ -90,14 +94,22 @@ one of ``==``, ``!=``, ``<``, ``<=``, ``>``, ``>=``; and
against. The following keywords are defined:
``src-ip``/``dst-ip <cmp> <address-list>``
Source and destination address, respectively. Addresses can be
given as IP addresses or CIDR masks.
Source and destination address, respectively. Addresses can be given
as IPv4 or IPv6 addresses or CIDR masks. For IPv6 addresses/masks
the colon-hexadecimal representation of the address must be enclosed
in square brackets (e.g. ``[fe80::1]`` or ``[fe80::0]/16``).
``src-port``/``dst-port`` ``<int-list>``
``src-port``/``dst-port <cmp> <int-list>``
Source and destination port, respectively.
``ip-proto tcp|udp|icmp``
IP protocol.
``ip-proto <cmp> tcp|udp|icmp|icmp6|ip|ip6``
IPv4 header's Protocol field or the Next Header field of the final
IPv6 header (i.e. either Next Header field in the fixed IPv6 header
if no extension headers are present or that field from the last
extension header in the chain). Note that the IP-in-IP forms of
tunneling are automatically decapsulated by default and signatures
apply to only the inner-most packet, so specifying ``ip`` or ``ip6``
is a no-op.
For lists of multiple values, they are sequentially compared against
the corresponding header field. If at least one of the comparisons
@ -111,20 +123,22 @@ condition can be defined either as
header <proto>[<offset>:<size>] [& <integer>] <cmp> <value-list>
This compares the value found at the given position of the packet
header with a list of values. ``offset`` defines the position of the
value within the header of the protocol defined by ``proto`` (which
can be ``ip``, ``tcp``, ``udp`` or ``icmp``). ``size`` is either 1, 2,
or 4 and specifies the value to have a size of this many bytes. If the
optional ``& <integer>`` is given, the packet's value is first masked
with the integer before it is compared to the value-list. ``cmp`` is
one of ``==``, ``!=``, ``<``, ``<=``, ``>``, ``>=``. ``value-list`` is
a list of comma-separated integers similar to those described above.
The integers within the list may be followed by an additional ``/
mask`` where ``mask`` is a value from 0 to 32. This corresponds to the
CIDR notation for netmasks and is translated into a corresponding
bitmask applied to the packet's value prior to the comparison (similar
to the optional ``& integer``).
This compares the value found at the given position of the packet header
with a list of values. ``offset`` defines the position of the value
within the header of the protocol defined by ``proto`` (which can be
``ip``, ``ip6``, ``tcp``, ``udp``, ``icmp`` or ``icmp6``). ``size`` is
either 1, 2, or 4 and specifies the value to have a size of this many
bytes. If the optional ``& <integer>`` is given, the packet's value is
first masked with the integer before it is compared to the value-list.
``cmp`` is one of ``==``, ``!=``, ``<``, ``<=``, ``>``, ``>=``.
``value-list`` is a list of comma-separated integers similar to those
described above. The integers within the list may be followed by an
additional ``/ mask`` where ``mask`` is a value from 0 to 32. This
corresponds to the CIDR notation for netmasks and is translated into a
corresponding bitmask applied to the packet's value prior to the
comparison (similar to the optional ``& integer``). IPv6 address values
are not allowed in the value-list, though you can still inspect any 1,
2, or 4 byte section of an IPv6 header using this keyword.
Putting it all together, this is an example condition that is
equivalent to ``dst-ip == 1.2.3.4/16, 5.6.7.8/24``:
@ -133,8 +147,8 @@ equivalent to ``dst-ip == 1.2.3.4/16, 5.6.7.8/24``:
header ip[16:4] == 1.2.3.4/16, 5.6.7.8/24
Internally, the predefined header conditions are in fact just
short-cuts and mapped into a generic condition.
Note that the analogous example for IPv6 isn't currently possible since
4 bytes is the max width of a value that can be compared.
Content Conditions
~~~~~~~~~~~~~~~~~~
@ -224,20 +238,10 @@ matched. The following context conditions are defined:
confirming the match. If false is returned, no signature match is
going to be triggered. The function has to be of type ``function
cond(state: signature_state, data: string): bool``. Here,
``content`` may contain the most recent content chunk available at
``data`` may contain the most recent content chunk available at
the time the signature was matched. If no such chunk is available,
``content`` will be the empty string. ``signature_state`` is
defined as follows:
.. code:: bro
type signature_state: record {
id: string; # ID of the signature
conn: connection; # Current connection
is_orig: bool; # True if current endpoint is originator
payload_size: count; # Payload size of the first packet
};
``data`` will be the empty string. See :bro:type:`signature_state`
for its definition.
``payload-size <cmp> <integer>``
Compares the integer to the size of the payload of a packet. For

73
doc/upgrade.rst
View file

@ -1,19 +1,80 @@
=============================
Upgrading From Bro 1.5 to 2.0
=============================
==========================================
Upgrading From the Previous Version of Bro
==========================================
.. rst-class:: opening
This guide details differences between Bro versions 1.5 and 2.0
This guide details specific differences between Bro versions
that may be important for users to know as they work on updating
their Bro deployment/configuration to the later version.
.. contents::
Introduction
============
Upgrading From Bro 2.0 to 2.1
=============================
In Bro 2.1, IPv6 is enabled by default. Therefore, when building Bro from
source, the "--enable-brov6" configure option has been removed because it
is no longer relevant.
Other configure changes include renaming the "--enable-perftools" option
to "--enable-perftools-debug" to indicate that the option is only relevant
for debugging the heap. One other change involves what happens when
tcmalloc (part of Google perftools) is found at configure time. On Linux,
it will automatically be linked with Bro, but on other platforms you
need to use the "--enable-perftools" option to enable linking to tcmalloc.
There are a couple of changes to the Bro scripting language to better
support IPv6. First, IPv6 literals appearing in a Bro script must now be
enclosed in square brackets (for example, ``[fe80::db15]``). For subnet
literals, the slash "/" appears after the closing square bracket (for
example, ``[fe80:1234::]/32``). Second, when an IP address variable or IP
address literal is enclosed in pipes (for example, ``|[fe80::db15]|``) the
result is now the size of the address in bits (32 for IPv4 and 128 for IPv6).
In the Bro scripting language, "match" and "using" are no longer reserved
keywords.
Some built-in functions have been removed: "addr_to_count" (use
"addr_to_counts" instead), "bro_has_ipv6" (this is no longer relevant
because Bro now always supports IPv6), "active_connection" (use
"connection_exists" instead), and "connection_record" (use "lookup_connection"
instead).
The "NFS3::mode2string" built-in function has been renamed to "file_mode".
Some built-in functions have been changed: "exit" (now takes the exit code
as a parameter), "to_port" (now takes a string as parameter instead
of a count and transport protocol, but "count_to_port" is still available),
"connect" (now takes an additional string parameter specifying the zone of
a non-global IPv6 address), and "listen" (now takes three additional
parameters to enable listening on IPv6 addresses).
Some Bro script variables have been renamed: "LogAscii::header_prefix"
has been renamed to "LogAscii::meta_prefix", "LogAscii::include_header"
has been renamed to "LogAscii::include_meta".
Some Bro script variables have been removed: "tunnel_port",
"parse_udp_tunnels", "use_connection_compressor", "cc_handle_resets",
"cc_handle_only_syns", and "cc_instantiate_on_data".
A couple events have changed: the "icmp_redirect" event now includes
the target and destination addresses and any Neighbor Discovery options
in the message, and the last parameter of the "dns_AAAA_reply" event has
been removed because it was unused.
The format of the ASCII log files has changed very slightly. Two new lines
are automatically added, one to record the time when the log was opened,
and the other to record the time when the log was closed.
In BroControl, the option (in broctl.cfg) "CFlowAddr" was renamed
to "CFlowAddress".
Upgrading From Bro 1.5 to 2.0
=============================
As the version number jump suggests, Bro 2.0 is a major upgrade and
lots of things have changed. Most importantly, we have rewritten

16
pkg/make-mac-packages
View file

@ -3,7 +3,13 @@
# This script creates binary packages for Mac OS X.
# They can be found in ../build/ after running.
./check-cmake || { exit 1; }
cmake -P /dev/stdin << "EOF"
if ( ${CMAKE_VERSION} VERSION_LESS 2.8.9 )
message(FATAL_ERROR "CMake >= 2.8.9 required to build package")
endif ()
EOF
[ $? -ne 0 ] && exit 1;
type sw_vers > /dev/null 2>&1 || {
echo "Unable to get Mac OS X version" >&2;
@ -34,26 +40,26 @@ prefix=/opt/bro
cd ..
# Minimum Bro
CMAKE_OSX_ARCHITECTURES=${arch} ./configure --prefix=${prefix} \
CMAKE_PREFIX_PATH=/usr CMAKE_OSX_ARCHITECTURES=${arch} ./configure --prefix=${prefix} \
--disable-broccoli --disable-broctl --pkg-name-prefix=Bro-minimal \
--binary-package
( cd build && make package )
# Full Bro package
CMAKE_OSX_ARCHITECTURES=${arch} ./configure --prefix=${prefix} \
CMAKE_PREFIX_PATH=/usr CMAKE_OSX_ARCHITECTURES=${arch} ./configure --prefix=${prefix} \
--pkg-name-prefix=Bro --binary-package
( cd build && make package )
# Broccoli
cd aux/broccoli
CMAKE_OSX_ARCHITECTURES=${arch} ./configure --prefix=${prefix} \
CMAKE_PREFIX_PATH=/usr CMAKE_OSX_ARCHITECTURES=${arch} ./configure --prefix=${prefix} \
--binary-package
( cd build && make package && mv *.dmg ../../../build/ )
cd ../..
# Broctl
cd aux/broctl
CMAKE_OSX_ARCHITECTURES=${arch} ./configure --prefix=${prefix} \
CMAKE_PREFIX_PATH=/usr CMAKE_OSX_ARCHITECTURES=${arch} ./configure --prefix=${prefix} \
--binary-package
( cd build && make package && mv *.dmg ../../../build/ )
cd ../..

3
scripts/base/frameworks/cluster/main.bro
View file

@ -77,6 +77,9 @@ export {
node_type: NodeType;
## The IP address of the cluster node.
ip: addr;
## If the *ip* field is a non-global IPv6 address, this field
## can specify a particular :rfc:`4007` ``zone_id``.
zone_id: string &default="";
## The port to which the this local node can connect when
## establishing communication.
p: port;

4
scripts/base/frameworks/cluster/nodes/manager.bro
View file

@ -13,8 +13,12 @@
## Turn off remote logging since this is the manager and should only log here.
redef Log::enable_remote_logging = F;
## Log rotation interval.
redef Log::default_rotation_interval = 1 hrs;
## Alarm summary mail interval.
redef Log::default_mail_alarms_interval = 24 hrs;
## Use the cluster's archive logging script.
redef Log::default_rotation_postprocessor_cmd = "archive-log";

25
scripts/base/frameworks/cluster/setup-connections.bro
View file

@ -19,23 +19,26 @@ event bro_init() &priority=9
# Connections from the control node for runtime control and update events.
# Every node in a cluster is eligible for control from this host.
if ( n$node_type == CONTROL )
Communication::nodes["control"] = [$host=n$ip, $connect=F,
$class="control", $events=control_events];
Communication::nodes["control"] = [$host=n$ip, $zone_id=n$zone_id,
$connect=F, $class="control",
$events=control_events];
if ( me$node_type == MANAGER )
{
if ( n$node_type == WORKER && n$manager == node )
Communication::nodes[i] =
[$host=n$ip, $connect=F,
[$host=n$ip, $zone_id=n$zone_id, $connect=F,
$class=i, $events=worker2manager_events, $request_logs=T];
if ( n$node_type == PROXY && n$manager == node )
Communication::nodes[i] =
[$host=n$ip, $connect=F,
[$host=n$ip, $zone_id=n$zone_id, $connect=F,
$class=i, $events=proxy2manager_events, $request_logs=T];
if ( n$node_type == TIME_MACHINE && me?$time_machine && me$time_machine == i )
Communication::nodes["time-machine"] = [$host=nodes[i]$ip, $p=nodes[i]$p,
Communication::nodes["time-machine"] = [$host=nodes[i]$ip,
$zone_id=nodes[i]$zone_id,
$p=nodes[i]$p,
$connect=T, $retry=1min,
$events=tm2manager_events];
}
@ -44,7 +47,8 @@ event bro_init() &priority=9
{
if ( n$node_type == WORKER && n$proxy == node )
Communication::nodes[i] =
[$host=n$ip, $connect=F, $class=i, $sync=T, $auth=T, $events=worker2proxy_events];
[$host=n$ip, $zone_id=n$zone_id, $connect=F, $class=i,
$sync=T, $auth=T, $events=worker2proxy_events];
# accepts connections from the previous one.
# (This is not ideal for setups with many proxies)
@ -53,16 +57,18 @@ event bro_init() &priority=9
{
if ( n?$proxy )
Communication::nodes[i]
= [$host=n$ip, $p=n$p,
= [$host=n$ip, $zone_id=n$zone_id, $p=n$p,
$connect=T, $auth=F, $sync=T, $retry=1mins];
else if ( me?$proxy && me$proxy == i )
Communication::nodes[me$proxy]
= [$host=nodes[i]$ip, $connect=F, $auth=T, $sync=T];
= [$host=nodes[i]$ip, $zone_id=nodes[i]$zone_id,
$connect=F, $auth=T, $sync=T];
}
# Finally the manager, to send it status updates.
if ( n$node_type == MANAGER && me$manager == i )
Communication::nodes["manager"] = [$host=nodes[i]$ip,
$zone_id=nodes[i]$zone_id,
$p=nodes[i]$p,
$connect=T, $retry=1mins,
$class=node,
@ -72,6 +78,7 @@ event bro_init() &priority=9
{
if ( n$node_type == MANAGER && me$manager == i )
Communication::nodes["manager"] = [$host=nodes[i]$ip,
$zone_id=nodes[i]$zone_id,
$p=nodes[i]$p,
$connect=T, $retry=1mins,
$class=node,
@ -79,6 +86,7 @@ event bro_init() &priority=9
if ( n$node_type == PROXY && me$proxy == i )
Communication::nodes["proxy"] = [$host=nodes[i]$ip,
$zone_id=nodes[i]$zone_id,
$p=nodes[i]$p,
$connect=T, $retry=1mins,
$sync=T, $class=node,
@ -87,6 +95,7 @@ event bro_init() &priority=9
if ( n$node_type == TIME_MACHINE &&
me?$time_machine && me$time_machine == i )
Communication::nodes["time-machine"] = [$host=nodes[i]$ip,
$zone_id=nodes[i]$zone_id,
$p=nodes[i]$p,
$connect=T,
$retry=1min,

28
scripts/base/frameworks/communication/main.bro
View file

@ -2,6 +2,7 @@
##! and/or transfer events.
@load base/frameworks/packet-filter
@load base/utils/addrs
module Communication;
@ -10,7 +11,8 @@ export {
## The communication logging stream identifier.
redef enum Log::ID += { LOG };
## Which interface to listen on (0.0.0.0 for any interface).
## Which interface to listen on. The addresses ``0.0.0.0`` and ``[::]``
## are wildcards.
const listen_interface = 0.0.0.0 &redef;
## Which port to listen on.
@ -19,6 +21,19 @@ export {
## This defines if a listening socket should use SSL.
const listen_ssl = F &redef;
## Defines if a listening socket can bind to IPv6 addresses.
const listen_ipv6 = F &redef;
## If :bro:id:`Communication::listen_interface` is a non-global
## IPv6 address and requires a specific :rfc:`4007` ``zone_id``,
## it can be specified here.
const listen_ipv6_zone_id = "" &redef;
## Defines the interval at which to retry binding to
## :bro:id:`Communication::listen_interface` on
## :bro:id:`Communication::listen_port` if it's already in use.
const listen_retry = 30 secs &redef;
## Default compression level. Compression level is 0-9, with 0 = no
## compression.
global compression_level = 0 &redef;
@ -27,7 +42,7 @@ export {
type Info: record {
## The network time at which a communication event occurred.
ts: time &log;
## The peer name (if any) for which a communication event is concerned.
## The peer name (if any) with which a communication event is concerned.
peer: string &log &optional;
## Where the communication event message originated from, that is,
## either from the scripting layer or inside the Bro process.
@ -52,6 +67,10 @@ export {
## Remote address.
host: addr;
## If the *host* field is a non-global IPv6 address, this field
## can specify a particular :rfc:`4007` ``zone_id``.
zone_id: string &optional;
## Port of the remote Bro communication endpoint if we are initiating
## the connection based on the :bro:id:`connect` field.
p: port &optional;
@ -160,7 +179,7 @@ event remote_log(level: count, src: count, msg: string)
# This is a core generated event.
event remote_log_peer(p: event_peer, level: count, src: count, msg: string)
{
local rmsg = fmt("[#%d/%s:%d] %s", p$id, p$host, p$p, msg);
local rmsg = fmt("[#%d/%s:%d] %s", p$id, addr_to_uri(p$host), p$p, msg);
do_script_log_common(level, src, rmsg);
}
@ -178,7 +197,8 @@ function connect_peer(peer: string)
p = node$p;
local class = node?$class ? node$class : "";
local id = connect(node$host, p, class, node$retry, node$ssl);
local zone_id = node?$zone_id ? node$zone_id : "";
local id = connect(node$host, zone_id, p, class, node$retry, node$ssl);
if ( id == PEER_ID_NONE )
Log::write(Communication::LOG, [$ts = network_time(),

4
scripts/base/frameworks/control/main.bro
View file

@ -11,6 +11,10 @@ export {
## The port of the host that will be controlled.
const host_port = 0/tcp &redef;
## If :bro:id:`Control::host` is a non-global IPv6 address and
## requires a specific :rfc:`4007` ``zone_id``, it can be set here.
const zone_id = "" &redef;
## The command that is being done. It's typically set on the
## command line.
const cmd = "" &redef;

61
scripts/base/frameworks/dpd/dpd.sig
View file

@ -149,3 +149,64 @@ signature dpd_ssl_client {
payload /^(\x16\x03[\x00\x01\x02]..\x01...\x03[\x00\x01\x02]|...?\x01[\x00\x01\x02][\x02\x03]).*/
tcp-state originator
}
signature dpd_ayiya {
ip-proto = udp
payload /^..\x11\x29/
enable "ayiya"
}
signature dpd_teredo {
ip-proto = udp
payload /^(\x00\x00)|(\x00\x01)|([\x60-\x6f])/
enable "teredo"
}
signature dpd_socks4_client {
ip-proto == tcp
# '32' is a rather arbitrary max length for the user name.
payload /^\x04[\x01\x02].{0,32}\x00/
tcp-state originator
}
signature dpd_socks4_server {
ip-proto == tcp
requires-reverse-signature dpd_socks4_client
payload /^\x00[\x5a\x5b\x5c\x5d]/
tcp-state responder
enable "socks"
}
signature dpd_socks4_reverse_client {
ip-proto == tcp
# '32' is a rather arbitrary max length for the user name.
payload /^\x04[\x01\x02].{0,32}\x00/
tcp-state responder
}
signature dpd_socks4_reverse_server {
ip-proto == tcp
requires-reverse-signature dpd_socks4_reverse_client
payload /^\x00[\x5a\x5b\x5c\x5d]/
tcp-state originator
enable "socks"
}
signature dpd_socks5_client {
ip-proto == tcp
# Watch for a few authentication methods to reduce false positives.
payload /^\x05.[\x00\x01\x02]/
tcp-state originator
}
signature dpd_socks5_server {
ip-proto == tcp
requires-reverse-signature dpd_socks5_client
# Watch for a single authentication method to be chosen by the server or
# the server to indicate the no authentication is required.
payload /^\x05(\x00|\x01[\x00\x01\x02])/
tcp-state responder
enable "socks"
}

6
scripts/base/frameworks/dpd/main.bro
View file

@ -3,8 +3,7 @@
module DPD;
## Add the DPD signatures to the signature framework.
redef signature_files += "base/frameworks/dpd/dpd.sig";
@load-sigs ./dpd.sig
export {
## Add the DPD logging stream identifier.
@ -105,5 +104,8 @@ event protocol_violation(c: connection, atype: count, aid: count,
reason: string) &priority=-5
{
if ( c?$dpd )
{
Log::write(DPD::LOG, c$dpd);
delete c$dpd;
}
}

5
scripts/base/frameworks/input/__load__.bro Normal file
View file

@ -0,0 +1,5 @@
@load ./main
@load ./readers/ascii
@load ./readers/raw
@load ./readers/benchmark

158
scripts/base/frameworks/input/main.bro Normal file
View file

@ -0,0 +1,158 @@
##! The input framework provides a way to read previously stored data either
##! as an event stream or into a bro table.
module Input;
export {
## The default input reader used. Defaults to `READER_ASCII`.
const default_reader = READER_ASCII &redef;
## The default reader mode used. Defaults to `MANUAL`.
const default_mode = MANUAL &redef;
## Flag that controls if the input framework accepts records
## that contain types that are not supported (at the moment
## file and function). If true, the input framework will
## warn in these cases, but continue. If false, it will
## abort. Defaults to false (abort)
const accept_unsupported_types = F &redef;
## TableFilter description type used for the `table` method.
type TableDescription: record {
## Common definitions for tables and events
## String that allows the reader to find the source.
## For `READER_ASCII`, this is the filename.
source: string;
## Reader to use for this stream
reader: Reader &default=default_reader;
## Read mode to use for this stream
mode: Mode &default=default_mode;
## Descriptive name. Used to remove a stream at a later time
name: string;
# Special definitions for tables
## Table which will receive the data read by the input framework
destination: any;
## Record that defines the values used as the index of the table
idx: any;
## Record that defines the values used as the elements of the table
## If val is undefined, destination has to be a set.
val: any &optional;
## Defines if the value of the table is a record (default), or a single value. Val
## can only contain one element when this is set to false.
want_record: bool &default=T;
## The event that is raised each time a value is added to, changed in or removed
## from the table. The event will receive an Input::Event enum as the first
## argument, the idx record as the second argument and the value (record) as the
## third argument.
ev: any &optional; # event containing idx, val as values.
## Predicate function that can decide if an insertion, update or removal should
## really be executed. Parameters are the same as for the event. If true is
## returned, the update is performed. If false is returned, it is skipped.
pred: function(typ: Input::Event, left: any, right: any): bool &optional;
## A key/value table that will be passed on the reader.
## Interpretation of the values is left to the writer, but
## usually they will be used for configuration purposes.
config: table[string] of string &default=table();
};
## EventFilter description type used for the `event` method.
type EventDescription: record {
## Common definitions for tables and events
## String that allows the reader to find the source.
## For `READER_ASCII`, this is the filename.
source: string;
## Reader to use for this steam
reader: Reader &default=default_reader;
## Read mode to use for this stream
mode: Mode &default=default_mode;
## Descriptive name. Used to remove a stream at a later time
name: string;
# Special definitions for events
## Record describing the fields to be retrieved from the source input.
fields: any;
## If want_record if false, the event receives each value in fields as a separate argument.
## If it is set to true (default), the event receives all fields in a single record value.
want_record: bool &default=T;
## The event that is raised each time a new line is received from the reader.
## The event will receive an Input::Event enum as the first element, and the fields as the following arguments.
ev: any;
## A key/value table that will be passed on the reader.
## Interpretation of the values is left to the writer, but
## usually they will be used for configuration purposes.
config: table[string] of string &default=table();
};
## Create a new table input from a given source. Returns true on success.
##
## description: `TableDescription` record describing the source.
global add_table: function(description: Input::TableDescription) : bool;
## Create a new event input from a given source. Returns true on success.
##
## description: `TableDescription` record describing the source.
global add_event: function(description: Input::EventDescription) : bool;
## Remove a input stream. Returns true on success and false if the named stream was
## not found.
##
## id: string value identifying the stream to be removed
global remove: function(id: string) : bool;
## Forces the current input to be checked for changes.
## Returns true on success and false if the named stream was not found
##
## id: string value identifying the stream
global force_update: function(id: string) : bool;
## Event that is called, when the end of a data source has been reached, including
## after an update.
global end_of_data: event(name: string, source:string);
}
@load base/input.bif
module Input;
function add_table(description: Input::TableDescription) : bool
{
return __create_table_stream(description);
}
function add_event(description: Input::EventDescription) : bool
{
return __create_event_stream(description);
}
function remove(id: string) : bool
{
return __remove_stream(id);
}
function force_update(id: string) : bool
{
return __force_update(id);
}

21
scripts/base/frameworks/input/readers/ascii.bro Normal file
View file

@ -0,0 +1,21 @@
##! Interface for the ascii input reader.
##!
##! The defaults are set to match Bro's ASCII output.
module InputAscii;
export {
## Separator between fields.
## Please note that the separator has to be exactly one character long
const separator = "\t" &redef;
## Separator between set elements.
## Please note that the separator has to be exactly one character long
const set_separator = "," &redef;
## String to use for empty fields.
const empty_field = "(empty)" &redef;
## String to use for an unset &optional field.
const unset_field = "-" &redef;
}

23
scripts/base/frameworks/input/readers/benchmark.bro Normal file
View file

@ -0,0 +1,23 @@
##! Interface for the ascii input reader.
module InputBenchmark;
export {
## multiplication factor for each second
const factor = 1.0 &redef;
## spread factor between lines
const spread = 0 &redef;
## spreading where usleep = 1000000 / autospread * num_lines
const autospread = 0.0 &redef;
## addition factor for each heartbeat
const addfactor = 0 &redef;
## stop spreading at x lines per heartbeat
const stopspreadat = 0 &redef;
## 1 -> enable timed spreading
const timedspread = 0.0 &redef;
}

9
scripts/base/frameworks/input/readers/raw.bro Normal file
View file

@ -0,0 +1,9 @@
##! Interface for the raw input reader.
module InputRaw;
export {
## Separator between input records.
## Please note that the separator has to be exactly one character long
const record_separator = "\n" &redef;
}

10
scripts/base/frameworks/intel/__load__.bro
View file

@ -1 +1,11 @@
@load ./main
# The cluster framework must be loaded first.
@load base/frameworks/cluster
@if ( Cluster::is_enabled() )
@load ./cluster
@endif
# This needs cluster support to only read on the manager.
@load ./input

61
scripts/base/frameworks/intel/cluster.bro Normal file
View file

@ -0,0 +1,61 @@
##! Cluster transparency support for the intelligence framework. This is mostly oriented
##! toward distributing intelligence information across clusters.
@load base/frameworks/cluster
@load ./input
module Intel;
redef record Item += {
## This field is used internally for cluster transparency to avoid
## re-dispatching intelligence items over and over from workers.
first_dispatch: bool &default=T;
};
# If this process is not a manager process, we don't want the full metadata
@if ( Cluster::local_node_type() != Cluster::MANAGER )
redef have_full_data = F;
@endif
global cluster_new_item: event(item: Item);
# Primary intelligence distribution comes from manager.
redef Cluster::manager2worker_events += /^Intel::(cluster_new_item)$/;
# If a worker finds intelligence and adds it, it should share it back to the manager.
redef Cluster::worker2manager_events += /^Intel::(cluster_new_item|match_no_items)$/;
@if ( Cluster::local_node_type() == Cluster::MANAGER )
event Intel::match_no_items(s: Seen) &priority=5
{
event Intel::match(s, Intel::get_items(s));
}
event remote_connection_handshake_done(p: event_peer)
{
# When a worker connects, send it the complete minimal data store.
# It will be kept up to date after this by the cluster_new_item event.
if ( Cluster::nodes[p$descr]$node_type == Cluster::WORKER )
{
send_id(p, "Intel::min_data_store");
}
}
@endif
event Intel::cluster_new_item(item: Intel::Item) &priority=5
{
# Ignore locally generated events to avoid event storms.
if ( is_remote_event() )
Intel::insert(item);
}
event Intel::new_item(item: Intel::Item) &priority=5
{
# The cluster manager always rebroadcasts intelligence.
# Workers redistribute it if it was locally generated.
if ( Cluster::local_node_type() == Cluster::MANAGER ||
item$first_dispatch )
{
item$first_dispatch=F;
event Intel::cluster_new_item(item);
}
}

33
scripts/base/frameworks/intel/input.bro Normal file
View file

@ -0,0 +1,33 @@
@load ./main
module Intel;
export {
## Intelligence files that will be read off disk. The files are
## reread everytime they are updated so updates much be atomic with
## "mv" instead of writing the file in place.
const read_files: set[string] = {} &redef;
}
event Intel::read_entry(desc: Input::EventDescription, tpe: Input::Event, item: Intel::Item)
{
Intel::insert(item);
}
event bro_init() &priority=5
{
if ( ! Cluster::is_enabled() ||
Cluster::local_node_type() == Cluster::MANAGER )
{
for ( a_file in read_files )
{
Input::add_event([$source=a_file,
$reader=Input::READER_ASCII,
$mode=Input::REREAD,
$name=cat("intel-", a_file),
$fields=Intel::Item,
$ev=Intel::read_entry]);
}
}
}

552
scripts/base/frameworks/intel/main.bro
View file

@ -1,323 +1,345 @@
##! The intelligence framework provides a way to store and query IP addresses,
##! strings (with a subtype), and numeric (with a subtype) data. Metadata
##! also be associated with the intelligence like tags which are arbitrary
##! strings, time values, and longer descriptive strings.
# Example string subtypes:
# url
# email
# domain
# software
# user_name
# file_name
# file_md5
# x509_md5
# Example tags:
# infrastructure
# malicious
# sensitive
# canary
# friend
##! and strings (with a str_type). Metadata can
##! also be associated with the intelligence like for making more informed
##! decisions about matching and handling of intelligence.
@load base/frameworks/notice
module Intel;
export {
## The intel logging stream identifier.
redef enum Log::ID += { LOG };
redef enum Notice::Type += {
## This notice should be used in all detector scripts to indicate
## an intelligence based detection.
Detection,
## String data needs to be further categoried since it could represent
## and number of types of data.
type StrType: enum {
## A complete URL without the prefix "http://".
URL,
## User-Agent string, typically HTTP or mail message body.
USER_AGENT,
## Email address.
EMAIL,
## DNS domain name.
DOMAIN,
## A user name.
USER_NAME,
## File hash which is non-hash type specific. It's up to the user to query
## for any relevant hash types.
FILE_HASH,
## Certificate SHA-1 hash.
CERT_HASH,
};
## Record type used for logging information from the intelligence framework.
## Primarily for problems or oddities with inserting and querying data.
## This is important since the content of the intelligence framework can
## change quite dramatically during runtime and problems may be introduced
## into the data.
type Info: record {
## The current network time.
ts: time &log;
## Represents the severity of the message.
## This value should be one of: "info", "warn", "error"
level: string &log;
## The message.
message: string &log;
};
## Record to represent metadata associated with a single piece of
## intelligence.
## Data about an :bro:type:`Intel::Item`
type MetaData: record {
## A description for the data.
## An arbitrary string value representing the data source. Typically,
## the convention for this field will be the source name and feed name
## separated by a hyphen. For example: "source1-c&c".
source: string;
## A freeform description for the data.
desc: string &optional;
## A URL where more information may be found about the intelligence.
## A URL for more information about the data.
url: string &optional;
## The time at which the data was first declared to be intelligence.
first_seen: time &optional;
## When this data was most recent inserted into the framework.
latest_seen: time &optional;
## Arbitrary text tags for the data.
tags: set[string];
};
## Record to represent a singular piece of intelligence.
## Represents a piece of intelligence.
type Item: record {
## If the data is an IP address, this hold the address.
ip: addr &optional;
## If the data is textual, this holds the text.
str: string &optional;
## If the data is numeric, this holds the number.
num: int &optional;
## The subtype of the data for when either the $str or $num fields are
## given. If one of those fields are given, this field must be present.
subtype: string &optional;
## The IP address if the intelligence is about an IP address.
host: addr &optional;
## The network if the intelligence is about a CIDR block.
net: subnet &optional;
## The string if the intelligence is about a string.
str: string &optional;
## The type of data that is in the string if the $str field is set.
str_type: StrType &optional;
## The next five fields are temporary until a better model for
## attaching metadata to an intelligence item is created.
desc: string &optional;
url: string &optional;
first_seen: time &optional;
latest_seen: time &optional;
tags: set[string];
## These single string tags are throw away until pybroccoli supports sets.
tag1: string &optional;
tag2: string &optional;
tag3: string &optional;
## Metadata for the item. Typically represents more deeply \
## descriptive data for a piece of intelligence.
meta: MetaData;
};
## Record model used for constructing queries against the intelligence
## framework.
type QueryItem: record {
## If an IP address is being queried for, this field should be given.
ip: addr &optional;
## If a string is being queried for, this field should be given.
str: string &optional;
## If numeric data is being queried for, this field should be given.
num: int &optional;
## If either a string or number is being queried for, this field should
## indicate the subtype of the data.
subtype: string &optional;
## A set of tags where if a single metadata record attached to an item
## has any one of the tags defined in this field, it will match.
or_tags: set[string] &optional;
## A set of tags where a single metadata record attached to an item
## must have all of the tags defined in this field.
and_tags: set[string] &optional;
## The predicate can be given when searching for a match. It will
## be tested against every :bro:type:`Intel::MetaData` item associated
## with the data being matched on. If it returns T a single time, the
## matcher will consider that the item has matched. This field can
## be used for constructing arbitrarily complex queries that may not
## be possible with the $or_tags or $and_tags fields.
pred: function(meta: Intel::MetaData): bool &optional;
## Enum to represent where data came from when it was discovered.
## The convenction is to prefix the name with "IN_".
type Where: enum {
## A catchall value to represent data of unknown provenance.
IN_ANYWHERE,
};
## Function to insert data into the intelligence framework.
##
## item: The data item.
##
## Returns: T if the data was successfully inserted into the framework,
## otherwise it returns F.
global insert: function(item: Item): bool;
## The $host field and combination of $str and $str_type fields are mutually
## exclusive. These records *must* represent either an IP address being
## seen or a string being seen.
type Seen: record {
## The IP address if the data seen is an IP address.
host: addr &log &optional;
## The string if the data is about a string.
str: string &log &optional;
## The type of data that is in the string if the $str field is set.
str_type: StrType &log &optional;
## A wrapper for the :bro:id:`Intel::insert` function. This is primarily
## used as the external API for inserting data into the intelligence
## using Broccoli.
global insert_event: event(item: Item);
## Where the data was discovered.
where: Where &log;
## Function for matching data within the intelligence framework.
global matcher: function(item: QueryItem): bool;
## If the data was discovered within a connection, the
## connection record should go into get to give context to the data.
conn: connection &optional;
};
## Record used for the logging framework representing a positive
## hit within the intelligence framework.
type Info: record {
## Timestamp when the data was discovered.
ts: time &log;
## If a connection was associated with this intelligence hit,
## this is the uid for the connection
uid: string &log &optional;
## If a connection was associated with this intelligence hit,
## this is the conn_id for the connection.
id: conn_id &log &optional;
## Where the data was seen.
seen: Seen &log;
## Sources which supplied data that resulted in this match.
sources: set[string] &log;
};
## Intelligence data manipulation functions.
global insert: function(item: Item);
## Function to declare discovery of a piece of data in order to check
## it against known intelligence for matches.
global seen: function(s: Seen);
## Event to represent a match in the intelligence data from data that was seen.
## On clusters there is no assurance as to where this event will be generated
## so do not assume that arbitrary global state beyond the given data
## will be available.
##
## This is the primary mechanism where a user will take actions based on data
## within the intelligence framework.
global match: event(s: Seen, items: set[Item]);
global log_intel: event(rec: Info);
}
type MetaDataStore: table[count] of MetaData;
# Internal handler for matches with no metadata available.
global match_no_items: event(s: Seen);
# Internal events for cluster data distribution
global new_item: event(item: Item);
global updated_item: event(item: Item);
# Optionally store metadata. This is used internally depending on
# if this is a cluster deployment or not.
const have_full_data = T &redef;
# The in memory data structure for holding intelligence.
type DataStore: record {
ip_data: table[addr] of MetaDataStore;
# The first string is the actual value and the second string is the subtype.
string_data: table[string, string] of MetaDataStore;
int_data: table[int, string] of MetaDataStore;
net_data: table[subnet] of set[MetaData];
string_data: table[string, StrType] of set[MetaData];
};
global data_store: DataStore;
global data_store: DataStore &redef;
event bro_init()
# The in memory data structure for holding the barest matchable intelligence.
# This is primarily for workers to do the initial quick matches and store
# a minimal amount of data for the full match to happen on the manager.
type MinDataStore: record {
net_data: set[subnet];
string_data: set[string, StrType];
};
global min_data_store: MinDataStore &redef;
event bro_init() &priority=5
{
Log::create_stream(Intel::LOG, [$columns=Info]);
Log::create_stream(LOG, [$columns=Info, $ev=log_intel]);
}
function insert(item: Item): bool
function find(s: Seen): bool
{
local err_msg = "";
if ( (item?$str || item?$num) && ! item?$subtype )
err_msg = "You must provide a subtype to insert_sync or this item doesn't make sense.";
if ( err_msg == "" )
if ( s?$host &&
((have_full_data && s$host in data_store$net_data) ||
(s$host in min_data_store$net_data)))
{
# Create and fill out the meta data item.
local meta: MetaData;
if ( item?$first_seen )
meta$first_seen = item$first_seen;
if ( item?$latest_seen )
meta$latest_seen = item$latest_seen;
if ( item?$tags )
meta$tags = item$tags;
if ( item?$desc )
meta$desc = item$desc;
if ( item?$url )
meta$url = item$url;
return T;
}
else if ( s?$str && s?$str_type &&
((have_full_data && [s$str, s$str_type] in data_store$string_data) ||
([s$str, s$str_type] in min_data_store$string_data)))
{
return T;
}
else
{
return F;
}
}
function get_items(s: Seen): set[Item]
{
local item: Item;
local return_data: set[Item] = set();
# This is hopefully only temporary until pybroccoli supports sets.
if ( item?$tag1 )
add item$tags[item$tag1];
if ( item?$tag2 )
add item$tags[item$tag2];
if ( item?$tag3 )
add item$tags[item$tag3];
if ( ! have_full_data )
{
# A reporter warning should be generated here because this function
# should never be called from a host that doesn't have the full data.
# TODO: do a reporter warning.
return return_data;
}
if ( item?$ip )
if ( s?$host )
{
# See if the host is known about and it has meta values
if ( s$host in data_store$net_data )
{
if ( item$ip !in data_store$ip_data )
data_store$ip_data[item$ip] = table();
data_store$ip_data[item$ip][|data_store$ip_data[item$ip]|] = meta;
return T;
for ( m in data_store$net_data[s$host] )
{
# TODO: the lookup should be finding all and not just most specific
# and $host/$net should have the correct value.
item = [$host=s$host, $meta=m];
add return_data[item];
}
}
else if ( item?$str )
}
else if ( s?$str && s?$str_type )
{
# See if the string is known about and it has meta values
if ( [s$str, s$str_type] in data_store$string_data )
{
if ( [item$str, item$subtype] !in data_store$string_data )
data_store$string_data[item$str, item$subtype] = table();
data_store$string_data[item$str, item$subtype][|data_store$string_data[item$str, item$subtype]|] = meta;
return T;
for ( m in data_store$string_data[s$str, s$str_type] )
{
item = [$str=s$str, $str_type=s$str_type, $meta=m];
add return_data[item];
}
}
else if ( item?$num )
{
if ( [item$num, item$subtype] !in data_store$int_data )
data_store$int_data[item$num, item$subtype] = table();
}
data_store$int_data[item$num, item$subtype][|data_store$int_data[item$num, item$subtype]|] = meta;
return T;
return return_data;
}
function Intel::seen(s: Seen)
{
if ( find(s) )
{
if ( have_full_data )
{
local items = get_items(s);
event Intel::match(s, items);
}
else
err_msg = "Failed to insert intelligence item for some unknown reason.";
}
if ( err_msg != "" )
Log::write(Intel::LOG, [$ts=network_time(), $level="warn", $message=fmt(err_msg)]);
return F;
}
event insert_event(item: Item)
{
insert(item);
}
function match_item_with_metadata(item: QueryItem, meta: MetaData): bool
{
if ( item?$and_tags )
{
local matched = T;
# Every tag given has to match in a single MetaData entry.
for ( tag in item$and_tags )
{
if ( tag !in meta$tags )
matched = F;
event Intel::match_no_items(s);
}
if ( matched )
}
}
function has_meta(check: MetaData, metas: set[MetaData]): bool
{
local check_hash = md5_hash(check);
for ( m in metas )
{
if ( check_hash == md5_hash(m) )
return T;
}
else if ( item?$or_tags )
{
# For OR tags, only a single tag has to match.
for ( tag in item$or_tags )
{
if ( tag in meta$tags )
return T;
}
}
else if ( item?$pred )
return item$pred(meta);
# This indicates some sort of failure in the query
# The records must not be equivalent if we made it this far.
return F;
}
function matcher(item: QueryItem): bool
event Intel::match(s: Seen, items: set[Item]) &priority=5
{
local err_msg = "";
if ( ! (item?$ip || item?$str || item?$num) )
err_msg = "You must supply one of the $ip, $str, or $num fields to search on";
else if ( (item?$or_tags || item?$and_tags) && item?$pred )
err_msg = "You can't match with both tags and a predicate.";
else if ( item?$or_tags && item?$and_tags )
err_msg = "You can't match with both OR'd together tags and AND'd together tags";
else if ( (item?$str || item?$num) && ! item?$subtype )
err_msg = "You must provide a subtype to matcher or this item doesn't make sense.";
else if ( item?$str && item?$num )
err_msg = "You must only provide $str or $num, not both.";
local empty_set: set[string] = set();
local info: Info = [$ts=network_time(), $seen=s, $sources=empty_set];
local meta: MetaData;
if ( err_msg == "" )
if ( s?$conn )
{
if ( item?$ip )
{
if ( item$ip in data_store$ip_data )
{
if ( ! item?$and_tags && ! item?$or_tags && ! item?$pred )
return T;
for ( i in data_store$ip_data[item$ip] )
{
meta = data_store$ip_data[item$ip][i];
if ( match_item_with_metadata(item, meta) )
return T;
}
}
}
else if ( item?$str )
{
if ( [item$str, item$subtype] in data_store$string_data )
{
if ( ! item?$and_tags && ! item?$or_tags && ! item?$pred )
return T;
for ( i in data_store$string_data[item$str, item$subtype] )
{
meta = data_store$string_data[item$str, item$subtype][i];
if ( match_item_with_metadata(item, meta) )
return T;
}
}
}
else if ( item?$num )
{
if ( [item$num, item$subtype] in data_store$int_data )
{
if ( ! item?$and_tags && ! item?$or_tags && ! item?$pred )
return T;
for ( i in data_store$int_data[item$num, item$subtype] )
{
meta = data_store$int_data[item$num, item$subtype][i];
if ( match_item_with_metadata(item, meta) )
return T;
}
}
}
else
err_msg = "Failed to query intelligence data for some unknown reason.";
info$uid = s$conn$uid;
info$id = s$conn$id;
}
if ( err_msg != "" )
Log::write(Intel::LOG, [$ts=network_time(), $level="error", $message=fmt(err_msg)]);
return F;
for ( item in items )
add info$sources[item$meta$source];
Log::write(Intel::LOG, info);
}
function insert(item: Item)
{
if ( item?$str && !item?$str_type )
{
event reporter_warning(network_time(), fmt("You must provide a str_type for strings or this item doesn't make sense. Item: %s", item), "");
return;
}
# Create and fill out the meta data item.
local meta = item$meta;
local metas: set[MetaData];
if ( item?$host )
{
local host = mask_addr(item$host, is_v4_addr(item$host) ? 32 : 128);
if ( have_full_data )
{
if ( host !in data_store$net_data )
data_store$net_data[host] = set();
metas = data_store$net_data[host];
}
add min_data_store$net_data[host];
}
else if ( item?$net )
{
if ( have_full_data )
{
if ( item$net !in data_store$net_data )
data_store$net_data[item$net] = set();
metas = data_store$net_data[item$net];
}
add min_data_store$net_data[item$net];
}
else if ( item?$str )
{
if ( have_full_data )
{
if ( [item$str, item$str_type] !in data_store$string_data )
data_store$string_data[item$str, item$str_type] = set();
metas = data_store$string_data[item$str, item$str_type];
}
add min_data_store$string_data[item$str, item$str_type];
}
local updated = F;
if ( have_full_data )
{
for ( m in metas )
{
if ( meta$source == m$source )
{
if ( has_meta(meta, metas) )
{
# It's the same item being inserted again.
return;
}
else
{
# Same source, different metadata means updated item.
updated = T;
}
}
}
add metas[item$meta];
}
if ( updated )
event Intel::updated_item(item);
else
event Intel::new_item(item);
}

3
scripts/base/frameworks/logging/__load__.bro
View file

@ -1,3 +1,6 @@
@load ./main
@load ./postprocessors
@load ./writers/ascii
@load ./writers/dataseries
@load ./writers/elasticsearch
@load ./writers/none

48
scripts/base/frameworks/logging/main.bro
View file

@ -60,6 +60,9 @@ export {
## Default rotation interval. Zero disables rotation.
const default_rotation_interval = 0secs &redef;
## Default alarm summary mail interval. Zero disables alarm summary mails.
const default_mail_alarms_interval = 0secs &redef;
## Default naming format for timestamps embedded into filenames.
## Uses a ``strftime()`` style.
const default_rotation_date_format = "%Y-%m-%d-%H-%M-%S" &redef;
@ -96,6 +99,12 @@ export {
## file name. Generally, filenames are expected to given
## without any extensions; writers will add appropiate
## extensions automatically.
##
## If this path is found to conflict with another filter's
## for the same writer type, it is automatically corrected
## by appending "-N", where N is the smallest integer greater
## or equal to 2 that allows the corrected path name to not
## conflict with another filter's.
path: string &optional;
## A function returning the output path for recording entries
@ -115,7 +124,10 @@ export {
## rec: An instance of the streams's ``columns`` type with its
## fields set to the values to be logged.
##
## Returns: The path to be used for the filter.
## Returns: The path to be used for the filter, which will be subject
## to the same automatic correction rules as the *path*
## field of :bro:type:`Log::Filter` in the case of conflicts
## with other filters trying to use the same writer/path pair.
path_func: function(id: ID, path: string, rec: any): string &optional;
## Subset of column names to record. If not given, all
@ -138,6 +150,11 @@ export {
## Callback function to trigger for rotated files. If not set, the
## default comes out of :bro:id:`Log::default_rotation_postprocessors`.
postprocessor: function(info: RotationInfo) : bool &optional;
## A key/value table that will be passed on to the writer.
## Interpretation of the values is left to the writer, but
## usually they will be used for configuration purposes.
config: table[string] of string &default=table();
};
## Sentinel value for indicating that a filter was not found when looked up.
@ -313,6 +330,11 @@ export {
## Log::default_rotation_postprocessor_cmd
## Log::default_rotation_postprocessors
global run_rotation_postprocessor_cmd: function(info: RotationInfo, npath: string) : bool;
## The streams which are currently active and not disabled.
## This table is not meant to be modified by users! Only use it for
## examining which streams are active.
global active_streams: table[ID] of Stream = table();
}
# We keep a script-level copy of all filters so that we can manipulate them.
@ -327,20 +349,23 @@ function __default_rotation_postprocessor(info: RotationInfo) : bool
{
if ( info$writer in default_rotation_postprocessors )
return default_rotation_postprocessors[info$writer](info);
else
# Return T by default so that postprocessor-less writers don't shutdown.
return T;
}
function default_path_func(id: ID, path: string, rec: any) : string
{
# The suggested path value is a previous result of this function
# or a filter path explicitly set by the user, so continue using it.
if ( path != "" )
return path;
local id_str = fmt("%s", id);
local parts = split1(id_str, /::/);
if ( |parts| == 2 )
{
# The suggested path value is a previous result of this function
# or a filter path explicitly set by the user, so continue using it.
if ( path != "" )
return path;
# Example: Notice::LOG -> "notice"
if ( parts[2] == "LOG" )
{
@ -376,13 +401,16 @@ function run_rotation_postprocessor_cmd(info: RotationInfo, npath: string) : boo
if ( pp_cmd == "" )
return T;
# Turn, e.g., Log::WRITER_ASCII into "ascii".
local writer = subst_string(to_lower(fmt("%s", info$writer)), "log::writer_", "");
# The date format is hard-coded here to provide a standardized
# script interface.
system(fmt("%s %s %s %s %s %d",
system(fmt("%s %s %s %s %s %d %s",
pp_cmd, npath, info$path,
strftime("%y-%m-%d_%H.%M.%S", info$open),
strftime("%y-%m-%d_%H.%M.%S", info$close),
info$terminating));
info$terminating, writer));
return T;
}
@ -392,11 +420,15 @@ function create_stream(id: ID, stream: Stream) : bool
if ( ! __create_stream(id, stream) )
return F;
active_streams[id] = stream;
return add_default_filter(id);
}
function disable_stream(id: ID) : bool
{
delete active_streams[id];
return __disable_stream(id);
}

11
scripts/base/frameworks/logging/writers/ascii.bro
View file

@ -8,12 +8,13 @@ export {
## into files. This is primarily for debugging purposes.
const output_to_stdout = F &redef;
## If true, include a header line with column names and description
## of the other ASCII logging options that were used.
const include_header = T &redef;
## If true, include lines with log meta information such as column names with
## types, the values of ASCII logging options that in use, and the time when the
## file was opened and closes (the latter at the end).
const include_meta = T &redef;
## Prefix for the header line if included.
const header_prefix = "#" &redef;
## Prefix for lines with meta information.
const meta_prefix = "#" &redef;
## Separator between fields.
const separator = "\t" &redef;

60
scripts/base/frameworks/logging/writers/dataseries.bro Normal file
View file

@ -0,0 +1,60 @@
##! Interface for the DataSeries log writer.
module LogDataSeries;
export {
## Compression to use with the DS output file. Options are:
##
## 'none' -- No compression.
## 'lzf' -- LZF compression. Very quick, but leads to larger output files.
## 'lzo' -- LZO compression. Very fast decompression times.
## 'gz' -- GZIP compression. Slower than LZF, but also produces smaller output.
## 'bz2' -- BZIP2 compression. Slower than GZIP, but also produces smaller output.
const compression = "gz" &redef;
## The extent buffer size.
## Larger values here lead to better compression and more efficient writes, but
## also increase the lag between the time events are received and the time they
## are actually written to disk.
const extent_size = 65536 &redef;
## Should we dump the XML schema we use for this DS file to disk?
## If yes, the XML schema shares the name of the logfile, but has
## an XML ending.
const dump_schema = F &redef;
## How many threads should DataSeries spawn to perform compression?
## Note that this dictates the number of threads per log stream. If
## you're using a lot of streams, you may want to keep this number
## relatively small.
##
## Default value is 1, which will spawn one thread / stream.
##
## Maximum is 128, minimum is 1.
const num_threads = 1 &redef;
## Should time be stored as an integer or a double?
## Storing time as a double leads to possible precision issues and
## can (significantly) increase the size of the resulting DS log.
## That said, timestamps stored in double form are consistent
## with the rest of Bro, including the standard ASCII log. Hence, we
## use them by default.
const use_integer_for_time = F &redef;
}
# Default function to postprocess a rotated DataSeries log file. It moves the
# rotated file to a new name that includes a timestamp with the opening time, and
# then runs the writer's default postprocessor command on it.
function default_rotation_postprocessor_func(info: Log::RotationInfo) : bool
{
# Move file to name including both opening and closing time.
local dst = fmt("%s.%s.ds", info$path,
strftime(Log::default_rotation_date_format, info$open));
system(fmt("/bin/mv %s %s", info$fname, dst));
# Run default postprocessor.
return Log::run_rotation_postprocessor_cmd(info, dst);
}
redef Log::default_rotation_postprocessors += { [Log::WRITER_DATASERIES] = default_rotation_postprocessor_func };

48
scripts/base/frameworks/logging/writers/elasticsearch.bro Normal file
View file

@ -0,0 +1,48 @@
##! Log writer for sending logs to an ElasticSearch server.
##!
##! Note: This module is in testing and is not yet considered stable!
##!
##! There is one known memory issue. If your elasticsearch server is
##! running slowly and taking too long to return from bulk insert
##! requests, the message queue to the writer thread will continue
##! growing larger and larger giving the appearance of a memory leak.
module LogElasticSearch;
export {
## Name of the ES cluster
const cluster_name = "elasticsearch" &redef;
## ES Server
const server_host = "127.0.0.1" &redef;
## ES Port
const server_port = 9200 &redef;
## Name of the ES index
const index_prefix = "bro" &redef;
## The ES type prefix comes before the name of the related log.
## e.g. prefix = "bro\_" would create types of bro_dns, bro_software, etc.
const type_prefix = "" &redef;
## The time before an ElasticSearch transfer will timeout. Note that
## the fractional part of the timeout will be ignored. In particular, time
## specifications less than a second result in a timeout value of 0, which
## means "no timeout."
const transfer_timeout = 2secs;
## The batch size is the number of messages that will be queued up before
## they are sent to be bulk indexed.
const max_batch_size = 1000 &redef;
## The maximum amount of wall-clock time that is allowed to pass without
## finishing a bulk log send. This represents the maximum delay you
## would like to have with your logs before they are sent to ElasticSearch.
const max_batch_interval = 1min &redef;
## The maximum byte size for a buffered JSON string to send to the bulk
## insert API.
const max_byte_size = 1024 * 1024 &redef;
}

17
scripts/base/frameworks/logging/writers/none.bro Normal file
View file

@ -0,0 +1,17 @@
##! Interface for the None log writer. Thiis writer is mainly for debugging.
module LogNone;
export {
## If true, output debugging output that can be useful for unit
## testing the logging framework.
const debug = F &redef;
}
function default_rotation_postprocessor_func(info: Log::RotationInfo) : bool
{
return T;
}
redef Log::default_rotation_postprocessors += { [Log::WRITER_NONE] = default_rotation_postprocessor_func };

2
scripts/base/frameworks/notice/actions/pp-alarms.bro
View file

@ -101,7 +101,7 @@ event bro_init()
# This replaces the standard non-pretty-printing filter.
Log::add_filter(Notice::ALARM_LOG,
[$name="alarm-mail", $writer=Log::WRITER_NONE,
$interv=Log::default_rotation_interval,
$interv=Log::default_mail_alarms_interval,
$postprocessor=pp_postprocessor]);
}

5
scripts/base/frameworks/notice/cluster.bro
View file

@ -23,7 +23,10 @@ redef Cluster::worker2manager_events += /Notice::cluster_notice/;
@if ( Cluster::local_node_type() != Cluster::MANAGER )
# The notice policy is completely handled by the manager and shouldn't be
# done by workers or proxies to save time for packet processing.
redef policy = {};
event bro_init() &priority=11
{
Notice::policy = table();
}
event Notice::begin_suppression(n: Notice::Info)
{

37
scripts/base/frameworks/reporter/main.bro
View file

@ -36,24 +36,55 @@ export {
## Not all reporter messages will have locations in them though.
location: string &log &optional;
};
## Tunable for sending reporter warning messages to STDERR. The option to
## turn it off is presented here in case Bro is being run by some
## external harness and shouldn't output anything to the console.
const warnings_to_stderr = T &redef;
## Tunable for sending reporter error messages to STDERR. The option to
## turn it off is presented here in case Bro is being run by some
## external harness and shouldn't output anything to the console.
const errors_to_stderr = T &redef;
}
global stderr: file;
event bro_init() &priority=5
{
Log::create_stream(Reporter::LOG, [$columns=Info]);
if ( errors_to_stderr || warnings_to_stderr )
stderr = open("/dev/stderr");
}
event reporter_info(t: time, msg: string, location: string)
event reporter_info(t: time, msg: string, location: string) &priority=-5
{
Log::write(Reporter::LOG, [$ts=t, $level=INFO, $message=msg, $location=location]);
}
event reporter_warning(t: time, msg: string, location: string)
event reporter_warning(t: time, msg: string, location: string) &priority=-5
{
if ( warnings_to_stderr )
{
if ( t > double_to_time(0.0) )
print stderr, fmt("WARNING: %.6f %s (%s)", t, msg, location);
else
print stderr, fmt("WARNING: %s (%s)", msg, location);
}
Log::write(Reporter::LOG, [$ts=t, $level=WARNING, $message=msg, $location=location]);
}
event reporter_error(t: time, msg: string, location: string)
event reporter_error(t: time, msg: string, location: string) &priority=-5
{
if ( errors_to_stderr )
{
if ( t > double_to_time(0.0) )
print stderr, fmt("ERROR: %.6f %s (%s)", t, msg, location);
else
print stderr, fmt("ERROR: %s (%s)", msg, location);
}
Log::write(Reporter::LOG, [$ts=t, $level=ERROR, $message=msg, $location=location]);
}

1
scripts/base/frameworks/tunnels/__load__.bro Normal file
View file

@ -0,0 +1 @@
@load ./main

149
scripts/base/frameworks/tunnels/main.bro Normal file
View file

@ -0,0 +1,149 @@
##! This script handles the tracking/logging of tunnels (e.g. Teredo,
##! AYIYA, or IP-in-IP such as 6to4 where "IP" is either IPv4 or IPv6).
##!
##! For any connection that occurs over a tunnel, information about its
##! encapsulating tunnels is also found in the *tunnel* field of
##! :bro:type:`connection`.
module Tunnel;
export {
## The tunnel logging stream identifier.
redef enum Log::ID += { LOG };
## Types of interesting activity that can occur with a tunnel.
type Action: enum {
## A new tunnel (encapsulating "connection") has been seen.
DISCOVER,
## A tunnel connection has closed.
CLOSE,
## No new connections over a tunnel happened in the amount of
## time indicated by :bro:see:`Tunnel::expiration_interval`.
EXPIRE,
};
## The record type which contains column fields of the tunnel log.
type Info: record {
## Time at which some tunnel activity occurred.
ts: time &log;
## The unique identifier for the tunnel, which may correspond
## to a :bro:type:`connection`'s *uid* field for non-IP-in-IP tunnels.
## This is optional because there could be numerous connections
## for payload proxies like SOCKS but we should treat it as a single
## tunnel.
uid: string &log &optional;
## The tunnel "connection" 4-tuple of endpoint addresses/ports.
## For an IP tunnel, the ports will be 0.
id: conn_id &log;
## The type of tunnel.
tunnel_type: Tunnel::Type &log;
## The type of activity that occurred.
action: Action &log;
};
## Logs all tunnels in an encapsulation chain with action
## :bro:see:`Tunnel::DISCOVER` that aren't already in the
## :bro:id:`Tunnel::active` table and adds them if not.
global register_all: function(ecv: EncapsulatingConnVector);
## Logs a single tunnel "connection" with action
## :bro:see:`Tunnel::DISCOVER` if it's not already in the
## :bro:id:`Tunnel::active` table and adds it if not.
global register: function(ec: EncapsulatingConn);
## Logs a single tunnel "connection" with action
## :bro:see:`Tunnel::EXPIRE` and removes it from the
## :bro:id:`Tunnel::active` table.
##
## t: A table of tunnels.
##
## idx: The index of the tunnel table corresponding to the tunnel to expire.
##
## Returns: 0secs, which when this function is used as an
## :bro:attr:`&expire_func`, indicates to remove the element at
## *idx* immediately.
global expire: function(t: table[conn_id] of Info, idx: conn_id): interval;
## Removes a single tunnel from the :bro:id:`Tunnel::active` table
## and logs the closing/expiration of the tunnel.
##
## tunnel: The tunnel which has closed or expired.
##
## action: The specific reason for the tunnel ending.
global close: function(tunnel: Info, action: Action);
## The amount of time a tunnel is not used in establishment of new
## connections before it is considered inactive/expired.
const expiration_interval = 1hrs &redef;
## Currently active tunnels. That is, tunnels for which new, encapsulated
## connections have been seen in the interval indicated by
## :bro:see:`Tunnel::expiration_interval`.
global active: table[conn_id] of Info = table() &read_expire=expiration_interval &expire_func=expire;
}
const ayiya_ports = { 5072/udp };
redef dpd_config += { [ANALYZER_AYIYA] = [$ports = ayiya_ports] };
const teredo_ports = { 3544/udp };
redef dpd_config += { [ANALYZER_TEREDO] = [$ports = teredo_ports] };
redef likely_server_ports += { ayiya_ports, teredo_ports };
event bro_init() &priority=5
{
Log::create_stream(Tunnel::LOG, [$columns=Info]);
}
function register_all(ecv: EncapsulatingConnVector)
{
for ( i in ecv )
register(ecv[i]);
}
function register(ec: EncapsulatingConn)
{
if ( ec$cid !in active )
{
local tunnel: Info;
tunnel$ts = network_time();
if ( ec?$uid )
tunnel$uid = ec$uid;
tunnel$id = ec$cid;
tunnel$action = DISCOVER;
tunnel$tunnel_type = ec$tunnel_type;
active[ec$cid] = tunnel;
Log::write(LOG, tunnel);
}
}
function close(tunnel: Info, action: Action)
{
tunnel$action = action;
tunnel$ts = network_time();
Log::write(LOG, tunnel);
delete active[tunnel$id];
}
function expire(t: table[conn_id] of Info, idx: conn_id): interval
{
close(t[idx], EXPIRE);
return 0secs;
}
event new_connection(c: connection) &priority=5
{
if ( c?$tunnel )
register_all(c$tunnel);
}
event tunnel_changed(c: connection, e: EncapsulatingConnVector) &priority=5
{
register_all(e);
}
event connection_state_remove(c: connection) &priority=-5
{
if ( c$id in active )
close(active[c$id], CLOSE);
}

294
scripts/base/init-bare.bro
View file

@ -115,6 +115,61 @@ type icmp_context: record {
DF: bool; ##< True if the packets *don't fragment* flag is set.
};
## Values extracted from a Prefix Information option in an ICMPv6 neighbor
## discovery message as specified by :rfc:`4861`.
##
## .. bro:see:: icmp6_nd_option
type icmp6_nd_prefix_info: record {
## Number of leading bits of the *prefix* that are valid.
prefix_len: count;
## Flag indicating the prefix can be used for on-link determination.
L_flag: bool;
## Autonomous address-configuration flag.
A_flag: bool;
## Length of time in seconds that the prefix is valid for purpose of
## on-link determination (0xffffffff represents infinity).
valid_lifetime: interval;
## Length of time in seconds that the addresses generated from the prefix
## via stateless address autoconfiguration remain preferred
## (0xffffffff represents infinity).
preferred_lifetime: interval;
## An IP address or prefix of an IP address. Use the *prefix_len* field
## to convert this into a :bro:type:`subnet`.
prefix: addr;
};
## Options extracted from ICMPv6 neighbor discovery messages as specified
## by :rfc:`4861`.
##
## .. bro:see:: icmp_router_solicitation icmp_router_advertisement
## icmp_neighbor_advertisement icmp_neighbor_solicitation icmp_redirect
## icmp6_nd_options
type icmp6_nd_option: record {
## 8-bit identifier of the type of option.
otype: count;
## 8-bit integer representing the length of the option (including the type
## and length fields) in units of 8 octets.
len: count;
## Source Link-Layer Address (Type 1) or Target Link-Layer Address (Type 2).
## Byte ordering of this is dependent on the actual link-layer.
link_address: string &optional;
## Prefix Information (Type 3).
prefix: icmp6_nd_prefix_info &optional;
## Redirected header (Type 4). This field contains the context of the
## original, redirected packet.
redirect: icmp_context &optional;
## Recommended MTU for the link (Type 5).
mtu: count &optional;
## The raw data of the option (everything after type & length fields),
## useful for unknown option types or when the full option payload is
## truncated in the captured packet. In those cases, option fields
## won't be pre-extracted into the fields above.
payload: string &optional;
};
## A type alias for a vector of ICMPv6 neighbor discovery message options.
type icmp6_nd_options: vector of icmp6_nd_option;
# A DNS mapping between IP address and hostname resolved by Bro's internal
# resolver.
#
@ -178,9 +233,35 @@ type endpoint_stats: record {
## use ``count``. That should be changed.
type AnalyzerID: count;
## Statistics about an endpoint.
module Tunnel;
export {
## Records the identity of an encapsulating parent of a tunneled connection.
type EncapsulatingConn: record {
## The 4-tuple of the encapsulating "connection". In case of an IP-in-IP
## tunnel the ports will be set to 0. The direction (i.e., orig and
## resp) are set according to the first tunneled packet seen
## and not according to the side that established the tunnel.
cid: conn_id;
## The type of tunnel.
tunnel_type: Tunnel::Type;
## A globally unique identifier that, for non-IP-in-IP tunnels,
## cross-references the *uid* field of :bro:type:`connection`.
uid: string &optional;
} &log;
} # end export
module GLOBAL;
## A type alias for a vector of encapsulating "connections", i.e for when
## there are tunnels within tunnels.
##
## todo::Where is this used?
## .. todo:: We need this type definition only for declaring builtin functions
## via ``bifcl``. We should extend ``bifcl`` to understand composite types
## directly and then remove this alias.
type EncapsulatingConnVector: vector of Tunnel::EncapsulatingConn;
## Statistics about a :bro:type:`connection` endpoint.
##
## .. bro:see:: connection
type endpoint: record {
size: count; ##< Logical size of data sent (for TCP: derived from sequence numbers).
## Endpoint state. For TCP connection, one of the constants:
@ -194,12 +275,15 @@ type endpoint: record {
## Number of IP-level bytes sent. Only set if :bro:id:`use_conn_size_analyzer` is
## true.
num_bytes_ip: count &optional;
## The current IPv6 flow label that the connection endpoint is using.
## Always 0 if the connection is over IPv4.
flow_label: count;
};
# A connection. This is Bro's basic connection type describing IP- and
# transport-layer information about the conversation. Note that Bro uses a
# liberal interpreation of "connection" and associates instances of this type
# also with UDP and ICMP flows.
## A connection. This is Bro's basic connection type describing IP- and
## transport-layer information about the conversation. Note that Bro uses a
## liberal interpreation of "connection" and associates instances of this type
## also with UDP and ICMP flows.
type connection: record {
id: conn_id; ##< The connection's identifying 4-tuple.
orig: endpoint; ##< Statistics about originator side.
@ -219,11 +303,17 @@ type connection: record {
service: set[string];
addl: string; ##< Deprecated.
hot: count; ##< Deprecated.
history: string; ##< State history of TCP connections. See *history* in :bro:see:`Conn::Info`.
history: string; ##< State history of connections. See *history* in :bro:see:`Conn::Info`.
## A globally unique connection identifier. For each connection, Bro creates an ID
## that is very likely unique across independent Bro runs. These IDs can thus be
## used to tag and locate information associated with that connection.
uid: string;
## If the connection is tunneled, this field contains information about
## the encapsulating "connection(s)" with the outermost one starting
## at index zero. It's also always the first such enapsulation seen
## for the connection unless the :bro:id:`tunnel_changed` event is handled
## and re-assigns this field to the new encapsulation.
tunnel: EncapsulatingConnVector &optional;
};
## Fields of a SYN packet.
@ -612,7 +702,9 @@ function add_signature_file(sold: string, snew: string): string
}
## Signature files to read. Use ``redef signature_files += "foo.sig"`` to
## extend. Signature files will be searched relative to ``BROPATH``.
## extend. Signature files added this way will be searched relative to
## ``BROPATH``. Using the ``@load-sigs`` directive instead is preferred
## since that can search paths relative to the current script.
global signature_files = "" &add_func = add_signature_file;
## ``p0f`` fingerprint file to use. Will be searched relative to ``BROPATH``.
@ -721,7 +813,7 @@ const tcp_storm_interarrival_thresh = 1 sec &redef;
## peer's ACKs. Set to zero to turn off this determination.
##
## .. bro:see:: tcp_max_above_hole_without_any_acks tcp_excessive_data_without_further_acks
const tcp_max_initial_window = 4096;
const tcp_max_initial_window = 4096 &redef;
## If we're not seeing our peer's ACKs, the maximum volume of data above a sequence
## hole that we'll tolerate before assuming that there's been a packet drop and we
@ -729,7 +821,7 @@ const tcp_max_initial_window = 4096;
## up.
##
## .. bro:see:: tcp_max_initial_window tcp_excessive_data_without_further_acks
const tcp_max_above_hole_without_any_acks = 4096;
const tcp_max_above_hole_without_any_acks = 4096 &redef;
## If we've seen this much data without any of it being acked, we give up
## on that connection to avoid memory exhaustion due to buffering all that
@ -738,7 +830,7 @@ const tcp_max_above_hole_without_any_acks = 4096;
## has in fact gone too far, but for now we just make this quite beefy.
##
## .. bro:see:: tcp_max_initial_window tcp_max_above_hole_without_any_acks
const tcp_excessive_data_without_further_acks = 10 * 1024 * 1024;
const tcp_excessive_data_without_further_acks = 10 * 1024 * 1024 &redef;
## For services without an a handler, these sets define originator-side ports that
## still trigger reassembly.
@ -866,18 +958,9 @@ const frag_timeout = 0.0 sec &redef;
const packet_sort_window = 0 usecs &redef;
## If positive, indicates the encapsulation header size that should
## be skipped. This either applies to all packets, or if
## :bro:see:`tunnel_port` is set, only to packets on that port.
##
## .. :bro:see:: tunnel_port
## be skipped. This applies to all packets.
const encap_hdr_size = 0 &redef;
## A UDP port that specifies which connections to apply :bro:see:`encap_hdr_size`
## to.
##
## .. :bro:see:: encap_hdr_size
const tunnel_port = 0/udp &redef;
## Whether to use the ``ConnSize`` analyzer to count the number of packets and
## IP-level bytes transfered by each endpoint. If true, these values are returned
## in the connection's :bro:see:`endpoint` record value.
@ -954,16 +1037,19 @@ const IPPROTO_MOBILITY = 135; ##< IPv6 mobility header.
## Values extracted from an IPv6 extension header's (e.g. hop-by-hop or
## destination option headers) option field.
##
## .. bro:see:: ip6_hdr ip6_hdr_chain ip6_hopopts ip6_dstopts
## .. bro:see:: ip6_hdr ip6_ext_hdr ip6_hopopts ip6_dstopts
type ip6_option: record {
otype: count; ##< Option type.
len: count; ##< Option data length.
data: string; ##< Option data.
};
## A type alias for a vector of IPv6 options.
type ip6_options: vector of ip6_option;
## Values extracted from an IPv6 Hop-by-Hop options extension header.
##
## .. bro:see:: pkt_hdr ip4_hdr ip6_hdr ip6_hdr_chain ip6_option
## .. bro:see:: pkt_hdr ip4_hdr ip6_hdr ip6_ext_hdr ip6_option
type ip6_hopopts: record {
## Protocol number of the next header (RFC 1700 et seq., IANA assigned
## number), e.g. :bro:id:`IPPROTO_ICMP`.
@ -971,12 +1057,12 @@ type ip6_hopopts: record {
## Length of header in 8-octet units, excluding first unit.
len: count;
## The TLV encoded options;
options: vector of ip6_option;
options: ip6_options;
};
## Values extracted from an IPv6 Destination options extension header.
##
## .. bro:see:: pkt_hdr ip4_hdr ip6_hdr ip6_hdr_chain ip6_option
## .. bro:see:: pkt_hdr ip4_hdr ip6_hdr ip6_ext_hdr ip6_option
type ip6_dstopts: record {
## Protocol number of the next header (RFC 1700 et seq., IANA assigned
## number), e.g. :bro:id:`IPPROTO_ICMP`.
@ -984,12 +1070,12 @@ type ip6_dstopts: record {
## Length of header in 8-octet units, excluding first unit.
len: count;
## The TLV encoded options;
options: vector of ip6_option;
options: ip6_options;
};
## Values extracted from an IPv6 Routing extension header.
##
## .. bro:see:: pkt_hdr ip4_hdr ip6_hdr ip6_hdr_chain
## .. bro:see:: pkt_hdr ip4_hdr ip6_hdr ip6_ext_hdr
type ip6_routing: record {
## Protocol number of the next header (RFC 1700 et seq., IANA assigned
## number), e.g. :bro:id:`IPPROTO_ICMP`.
@ -1006,7 +1092,7 @@ type ip6_routing: record {
## Values extracted from an IPv6 Fragment extension header.
##
## .. bro:see:: pkt_hdr ip4_hdr ip6_hdr ip6_hdr_chain
## .. bro:see:: pkt_hdr ip4_hdr ip6_hdr ip6_ext_hdr
type ip6_fragment: record {
## Protocol number of the next header (RFC 1700 et seq., IANA assigned
## number), e.g. :bro:id:`IPPROTO_ICMP`.
@ -1025,7 +1111,7 @@ type ip6_fragment: record {
## Values extracted from an IPv6 Authentication extension header.
##
## .. bro:see:: pkt_hdr ip4_hdr ip6_hdr ip6_hdr_chain
## .. bro:see:: pkt_hdr ip4_hdr ip6_hdr ip6_ext_hdr
type ip6_ah: record {
## Protocol number of the next header (RFC 1700 et seq., IANA assigned
## number), e.g. :bro:id:`IPPROTO_ICMP`.
@ -1036,15 +1122,15 @@ type ip6_ah: record {
rsv: count;
## Security Parameter Index.
spi: count;
## Sequence number.
seq: count;
## Authentication data.
data: string;
## Sequence number, unset in the case that *len* field is zero.
seq: count &optional;
## Authentication data, unset in the case that *len* field is zero.
data: string &optional;
};
## Values extracted from an IPv6 ESP extension header.
##
## .. bro:see:: pkt_hdr ip4_hdr ip6_hdr ip6_hdr_chain
## .. bro:see:: pkt_hdr ip4_hdr ip6_hdr ip6_ext_hdr
type ip6_esp: record {
## Security Parameters Index.
spi: count;
@ -1054,7 +1140,7 @@ type ip6_esp: record {
## Values extracted from an IPv6 Mobility Binding Refresh Request message.
##
## .. bro:see:: ip6_mobility_hdr ip6_hdr ip6_hdr_chain ip6_mobility_msg
## .. bro:see:: ip6_mobility_hdr ip6_hdr ip6_ext_hdr ip6_mobility_msg
type ip6_mobility_brr: record {
## Reserved.
rsv: count;
@ -1064,7 +1150,7 @@ type ip6_mobility_brr: record {
## Values extracted from an IPv6 Mobility Home Test Init message.
##
## .. bro:see:: ip6_mobility_hdr ip6_hdr ip6_hdr_chain ip6_mobility_msg
## .. bro:see:: ip6_mobility_hdr ip6_hdr ip6_ext_hdr ip6_mobility_msg
type ip6_mobility_hoti: record {
## Reserved.
rsv: count;
@ -1076,7 +1162,7 @@ type ip6_mobility_hoti: record {
## Values extracted from an IPv6 Mobility Care-of Test Init message.
##
## .. bro:see:: ip6_mobility_hdr ip6_hdr ip6_hdr_chain ip6_mobility_msg
## .. bro:see:: ip6_mobility_hdr ip6_hdr ip6_ext_hdr ip6_mobility_msg
type ip6_mobility_coti: record {
## Reserved.
rsv: count;
@ -1088,7 +1174,7 @@ type ip6_mobility_coti: record {
## Values extracted from an IPv6 Mobility Home Test message.
##
## .. bro:see:: ip6_mobility_hdr ip6_hdr ip6_hdr_chain ip6_mobility_msg
## .. bro:see:: ip6_mobility_hdr ip6_hdr ip6_ext_hdr ip6_mobility_msg
type ip6_mobility_hot: record {
## Home Nonce Index.
nonce_idx: count;
@ -1102,7 +1188,7 @@ type ip6_mobility_hot: record {
## Values extracted from an IPv6 Mobility Care-of Test message.
##
## .. bro:see:: ip6_mobility_hdr ip6_hdr ip6_hdr_chain ip6_mobility_msg
## .. bro:see:: ip6_mobility_hdr ip6_hdr ip6_ext_hdr ip6_mobility_msg
type ip6_mobility_cot: record {
## Care-of Nonce Index.
nonce_idx: count;
@ -1116,7 +1202,7 @@ type ip6_mobility_cot: record {
## Values extracted from an IPv6 Mobility Binding Update message.
##
## .. bro:see:: ip6_mobility_hdr ip6_hdr ip6_hdr_chain ip6_mobility_msg
## .. bro:see:: ip6_mobility_hdr ip6_hdr ip6_ext_hdr ip6_mobility_msg
type ip6_mobility_bu: record {
## Sequence number.
seq: count;
@ -1136,7 +1222,7 @@ type ip6_mobility_bu: record {
## Values extracted from an IPv6 Mobility Binding Acknowledgement message.
##
## .. bro:see:: ip6_mobility_hdr ip6_hdr ip6_hdr_chain ip6_mobility_msg
## .. bro:see:: ip6_mobility_hdr ip6_hdr ip6_ext_hdr ip6_mobility_msg
type ip6_mobility_back: record {
## Status.
status: count;
@ -1152,7 +1238,7 @@ type ip6_mobility_back: record {
## Values extracted from an IPv6 Mobility Binding Error message.
##
## .. bro:see:: ip6_mobility_hdr ip6_hdr ip6_hdr_chain ip6_mobility_msg
## .. bro:see:: ip6_mobility_hdr ip6_hdr ip6_ext_hdr ip6_mobility_msg
type ip6_mobility_be: record {
## Status.
status: count;
@ -1164,7 +1250,7 @@ type ip6_mobility_be: record {
## Values extracted from an IPv6 Mobility header's message data.
##
## .. bro:see:: ip6_mobility_hdr ip6_hdr ip6_hdr_chain
## .. bro:see:: ip6_mobility_hdr ip6_hdr ip6_ext_hdr
type ip6_mobility_msg: record {
## The type of message from the header's MH Type field.
id: count;
@ -1188,7 +1274,7 @@ type ip6_mobility_msg: record {
## Values extracted from an IPv6 Mobility header.
##
## .. bro:see:: pkt_hdr ip4_hdr ip6_hdr ip6_hdr_chain
## .. bro:see:: pkt_hdr ip4_hdr ip6_hdr ip6_ext_hdr
type ip6_mobility_hdr: record {
## Protocol number of the next header (RFC 1700 et seq., IANA assigned
## number), e.g. :bro:id:`IPPROTO_ICMP`.
@ -1229,9 +1315,12 @@ type ip6_ext_hdr: record {
mobility: ip6_mobility_hdr &optional;
};
## A type alias for a vector of IPv6 extension headers.
type ip6_ext_hdr_chain: vector of ip6_ext_hdr;
## Values extracted from an IPv6 header.
##
## .. bro:see:: pkt_hdr ip4_hdr ip6_hdr_chain ip6_hopopts ip6_dstopts
## .. bro:see:: pkt_hdr ip4_hdr ip6_ext_hdr ip6_hopopts ip6_dstopts
## ip6_routing ip6_fragment ip6_ah ip6_esp
type ip6_hdr: record {
class: count; ##< Traffic class.
@ -1243,7 +1332,7 @@ type ip6_hdr: record {
hlim: count; ##< Hop limit.
src: addr; ##< Source address.
dst: addr; ##< Destination address.
exts: vector of ip6_ext_hdr; ##< Extension header chain.
exts: ip6_ext_hdr_chain; ##< Extension header chain.
};
## Values extracted from an IPv4 header.
@ -1312,6 +1401,42 @@ type pkt_hdr: record {
icmp: icmp_hdr &optional; ##< The ICMP header if an ICMP packet.
};
## A Teredo origin indication header. See :rfc:`4380` for more information
## about the Teredo protocol.
##
## .. bro:see:: teredo_bubble teredo_origin_indication teredo_authentication
## teredo_hdr
type teredo_auth: record {
id: string; ##< Teredo client identifier.
value: string; ##< HMAC-SHA1 over shared secret key between client and
##< server, nonce, confirmation byte, origin indication
##< (if present), and the IPv6 packet.
nonce: count; ##< Nonce chosen by Teredo client to be repeated by
##< Teredo server.
confirm: count; ##< Confirmation byte to be set to 0 by Teredo client
##< and non-zero by server if client needs new key.
};
## A Teredo authentication header. See :rfc:`4380` for more information
## about the Teredo protocol.
##
## .. bro:see:: teredo_bubble teredo_origin_indication teredo_authentication
## teredo_hdr
type teredo_origin: record {
p: port; ##< Unobfuscated UDP port of Teredo client.
a: addr; ##< Unobfuscated IPv4 address of Teredo client.
};
## A Teredo packet header. See :rfc:`4380` for more information about the
## Teredo protocol.
##
## .. bro:see:: teredo_bubble teredo_origin_indication teredo_authentication
type teredo_hdr: record {
auth: teredo_auth &optional; ##< Teredo authentication header.
origin: teredo_origin &optional; ##< Teredo origin indication header.
hdr: pkt_hdr; ##< IPv6 and transport protocol headers.
};
## Definition of "secondary filters". A secondary filter is a BPF filter given as
## index in this table. For each such filter, the corresponding event is raised for
## all matching packets.
@ -1789,6 +1914,14 @@ export {
};
} # end export
module Threading;
export {
## The heartbeat interval used by the threading framework.
## Changing this should usually not be neccessary and will break several tests.
const heartbeat_interval = 1.0 secs &redef;
}
module GLOBAL;
## An NTP message.
@ -2311,11 +2444,37 @@ type bittorrent_benc_dir: table[string] of bittorrent_benc_value;
## bt_tracker_response_not_ok
type bt_tracker_headers: table[string] of string;
type ModbusCoils: vector of bool;
type ModbusRegisters: vector of count;
type ModbusHeaders: record {
tid: count;
pid: count;
uid: count;
function_code: count;
};
module SOCKS;
export {
## This record is for a SOCKS client or server to provide either a
## name or an address to represent a desired or established connection.
type Address: record {
host: addr &optional;
name: string &optional;
} &log;
}
module GLOBAL;
@load base/event.bif
## BPF filter the user has set via the -f command line options. Empty if none.
const cmd_line_bpf_filter = "" &redef;
## The maximum number of open files to keep cached at a given time.
## If set to zero, this is automatically determined by inspecting
## the current/maximum limit on open files for the process.
const max_files_in_cache = 0 &redef;
## Deprecated.
const log_rotate_interval = 0 sec &redef;
@ -2599,11 +2758,41 @@ const record_all_packets = F &redef;
## .. bro:see:: conn_stats
const ignore_keep_alive_rexmit = F &redef;
## Whether the analysis engine parses IP packets encapsulated in
## UDP tunnels.
##
## .. bro:see:: tunnel_port
const parse_udp_tunnels = F &redef;
module Tunnel;
export {
## The maximum depth of a tunnel to decapsulate until giving up.
## Setting this to zero will disable all types of tunnel decapsulation.
const max_depth: count = 2 &redef;
## Toggle whether to do IPv{4,6}-in-IPv{4,6} decapsulation.
const enable_ip = T &redef;
## Toggle whether to do IPv{4,6}-in-AYIYA decapsulation.
const enable_ayiya = T &redef;
## Toggle whether to do IPv6-in-Teredo decapsulation.
const enable_teredo = T &redef;
## With this option set, the Teredo analysis will first check to see if
## other protocol analyzers have confirmed that they think they're
## parsing the right protocol and only continue with Teredo tunnel
## decapsulation if nothing else has yet confirmed. This can help
## reduce false positives of UDP traffic (e.g. DNS) that also happens
## to have a valid Teredo encapsulation.
const yielding_teredo_decapsulation = T &redef;
## With this set, the Teredo analyzer waits until it sees both sides
## of a connection using a valid Teredo encapsulation before issuing
## a :bro:see:`protocol_confirmation`. If it's false, the first
## occurence of a packet with valid Teredo encapsulation causes a
## confirmation. Both cases are still subject to effects of
## :bro:see:`Tunnel::yielding_teredo_decapsulation`.
const delay_teredo_confirmation = T &redef;
## How often to cleanup internal state for inactive IP tunnels.
const ip_tunnel_timeout = 24hrs &redef;
} # end export
module GLOBAL;
## Number of bytes per packet to capture from live interfaces.
const snaplen = 8192 &redef;
@ -2611,3 +2800,6 @@ const snaplen = 8192 &redef;
# Load the logging framework here because it uses fairly deep integration with
# BiFs and script-land defined types.
@load base/frameworks/logging
@load base/frameworks/input

6
scripts/base/init-default.bro
View file

@ -14,6 +14,7 @@
@load base/utils/patterns
@load base/utils/strings
@load base/utils/thresholds
@load base/utils/urls
# This has some deep interplay between types and BiFs so it's
# loaded in base/init-bare.bro
@ -30,13 +31,18 @@
@load base/frameworks/intel
@load base/frameworks/reporter
@load base/frameworks/protocols
@load base/frameworks/tunnels
@load base/protocols/conn
@load base/protocols/dns
@load base/protocols/ftp
@load base/protocols/http
@load base/protocols/irc
@load base/protocols/modbus
@load base/protocols/smtp
@load base/protocols/socks
@load base/protocols/ssh
@load base/protocols/ssl
@load base/protocols/syslog
@load base/misc/find-checksum-offloading

57
scripts/base/misc/find-checksum-offloading.bro Normal file
View file

@ -0,0 +1,57 @@
##! Discover cases where the local interface is sniffed and outbound packets
##! have checksum offloading. Load this script to receive a notice if it's
##! likely that checksum offload effects are being seen on a live interface or
##! in a packet trace file.
@load base/frameworks/notice
module ChecksumOffloading;
export {
## The interval which is used for checking packet statistics
## to see if checksum offloading is affecting analysis.
const check_interval = 10secs &redef;
}
# Keep track of how many bad checksums have been seen.
global bad_checksums = 0;
# Track to see if this script is done so that messages aren't created multiple times.
global done = F;
event ChecksumOffloading::check()
{
if ( done )
return;
local pkts_recvd = net_stats()$pkts_recvd;
if ( (bad_checksums*1.0 / net_stats()$pkts_recvd*1.0) > 0.05 )
{
local packet_src = reading_traces() ? "trace file likely has" : "interface is likely receiving";
local message = fmt("Your %s invalid IP checksums, most likely from NIC checksum offloading.", packet_src);
Reporter::warning(message);
done = T;
}
else if ( pkts_recvd < 20 )
{
# Keep scheduling this event until we've seen some lower threshold of
# total packets.
schedule check_interval { ChecksumOffloading::check() };
}
}
event bro_init()
{
schedule check_interval { ChecksumOffloading::check() };
}
event net_weird(name: string)
{
if ( name == "bad_IP_checksum" )
++bad_checksums;
}
event bro_done()
{
event ChecksumOffloading::check();
}

1
scripts/base/protocols/conn/__load__.bro
View file

@ -1,3 +1,4 @@
@load ./main
@load ./contents
@load ./inactivity
@load ./polling

41
scripts/base/protocols/conn/main.bro
View file

@ -17,7 +17,7 @@ export {
type Info: record {
## This is the time of the first packet.
ts: time &log;
## A unique identifier of a connection.
## A unique identifier of the connection.
uid: string &log;
## The connection's 4-tuple of endpoint addresses/ports.
id: conn_id &log;
@ -61,14 +61,14 @@ export {
## be left empty at all times.
local_orig: bool &log &optional;
## Indicates the number of bytes missed in content gaps which is
## Indicates the number of bytes missed in content gaps, which is
## representative of packet loss. A value other than zero will
## normally cause protocol analysis to fail but some analysis may
## have been completed prior to the packet loss.
missed_bytes: count &log &default=0;
## Records the state history of connections as a string of letters.
## For TCP connections the meaning of those letters is:
## The meaning of those letters is:
##
## ====== ====================================================
## Letter Meaning
@ -83,24 +83,29 @@ export {
## i inconsistent packet (e.g. SYN+RST bits both set)
## ====== ====================================================
##
## If the letter is in upper case it means the event comes from the
## originator and lower case then means the responder.
## Also, there is compression. We only record one "d" in each direction,
## for instance. I.e., we just record that data went in that direction.
## This history is not meant to encode how much data that happened to
## be.
## If the event comes from the originator, the letter is in upper-case; if it comes
## from the responder, it's in lower-case. Multiple packets of the same type will
## only be noted once (e.g. we only record one "d" in each direction, regardless of
## how many data packets were seen.)
history: string &log &optional;
## Number of packets the originator sent.
## Number of packets that the originator sent.
## Only set if :bro:id:`use_conn_size_analyzer` = T
orig_pkts: count &log &optional;
## Number IP level bytes the originator sent (as seen on the wire,
## Number of IP level bytes that the originator sent (as seen on the wire,
## taken from IP total_length header field).
## Only set if :bro:id:`use_conn_size_analyzer` = T
orig_ip_bytes: count &log &optional;
## Number of packets the responder sent. See ``orig_pkts``.
## Number of packets that the responder sent.
## Only set if :bro:id:`use_conn_size_analyzer` = T
resp_pkts: count &log &optional;
## Number IP level bytes the responder sent. See ``orig_pkts``.
## Number og IP level bytes that the responder sent (as seen on the wire,
## taken from IP total_length header field).
## Only set if :bro:id:`use_conn_size_analyzer` = T
resp_ip_bytes: count &log &optional;
## If this connection was over a tunnel, indicate the
## *uid* values for any encapsulating parent connections
## used over the lifetime of this inner connection.
tunnel_parents: set[string] &log;
};
## Event that can be handled to access the :bro:type:`Conn::Info`
@ -190,6 +195,8 @@ function set_conn(c: connection, eoc: bool)
c$conn$ts=c$start_time;
c$conn$uid=c$uid;
c$conn$id=c$id;
if ( c?$tunnel && |c$tunnel| > 0 )
add c$conn$tunnel_parents[c$tunnel[|c$tunnel|-1]$uid];
c$conn$proto=get_port_transport_proto(c$id$resp_p);
if( |Site::local_nets| > 0 )
c$conn$local_orig=Site::is_local_addr(c$id$orig_h);
@ -228,6 +235,14 @@ event content_gap(c: connection, is_orig: bool, seq: count, length: count) &prio
c$conn$missed_bytes = c$conn$missed_bytes + length;
}
event tunnel_changed(c: connection, e: EncapsulatingConnVector) &priority=5
{
set_conn(c, F);
if ( |e| > 0 )
add c$conn$tunnel_parents[e[|e|-1]$uid];
c$tunnel = e;
}
event connection_state_remove(c: connection) &priority=5
{
set_conn(c, T);

49
scripts/base/protocols/conn/polling.bro Normal file
View file

@ -0,0 +1,49 @@
##! Implements a generic way to poll connections looking for certain features
##! (e.g. monitor bytes transferred). The specific feature of a connection
##! to look for, the polling interval, and the code to execute if the feature
##! is found are all controlled by user-defined callback functions.
module ConnPolling;
export {
## Starts monitoring a given connection.
##
## c: The connection to watch.
##
## callback: A callback function that takes as arguments the monitored
## *connection*, and counter *cnt* that increments each time the
## callback is called. It returns an interval indicating how long
## in the future to schedule an event which will call the
## callback. A negative return interval causes polling to stop.
##
## cnt: The initial value of a counter which gets passed to *callback*.
##
## i: The initial interval at which to schedule the next callback.
## May be ``0secs`` to poll right away.
global watch: function(c: connection,
callback: function(c: connection, cnt: count): interval,
cnt: count, i: interval);
}
event ConnPolling::check(c: connection,
callback: function(c: connection, cnt: count): interval,
cnt: count)
{
if ( ! connection_exists(c$id) )
return;
lookup_connection(c$id); # updates the conn val
local next_interval = callback(c, cnt);
if ( next_interval < 0secs )
return;
watch(c, callback, cnt + 1, next_interval);
}
function watch(c: connection,
callback: function(c: connection, cnt: count): interval,
cnt: count, i: interval)
{
schedule i { ConnPolling::check(c, callback, cnt) };
}

36
scripts/base/protocols/dns/main.bro
View file

@ -46,27 +46,29 @@ export {
AA: bool &log &default=F;
## The Truncation bit specifies that the message was truncated.
TC: bool &log &default=F;
## The Recursion Desired bit indicates to a name server to recursively
## purse the query.
## The Recursion Desired bit in a request message indicates that
## the client wants recursive service for this query.
RD: bool &log &default=F;
## The Recursion Available bit in a response message indicates if
## The Recursion Available bit in a response message indicates that
## the name server supports recursive queries.
RA: bool &log &default=F;
## A reserved field that is currently supposed to be zero in all
## queries and responses.
Z: count &log &default=0;
## The set of resource descriptions in answer of the query.
## The set of resource descriptions in the query answer.
answers: vector of string &log &optional;
## The caching intervals of the associated RRs described by the
## ``answers`` field.
TTLs: vector of interval &log &optional;
## The DNS query was rejected by the server.
rejected: bool &log &default=F;
## This value indicates if this request/response pair is ready to be
## logged.
ready: bool &default=F;
## The total number of resource records in a reply message's answer
## section.
total_answers: count &optional;
total_answers: count &default=0;
## The total number of resource records in a reply message's answer,
## authority, and additional sections.
total_replies: count &optional;
@ -151,11 +153,11 @@ function set_session(c: connection, msg: dns_msg, is_query: bool)
c$dns = c$dns_state$pending[msg$id];
c$dns$rcode = msg$rcode;
c$dns$rcode_name = base_errors[msg$rcode];
if ( ! is_query )
{
c$dns$rcode = msg$rcode;
c$dns$rcode_name = base_errors[msg$rcode];
if ( ! c$dns?$total_answers )
c$dns$total_answers = msg$num_answers;
@ -175,10 +177,13 @@ function set_session(c: connection, msg: dns_msg, is_query: bool)
}
}
event dns_message(c: connection, is_orig: bool, msg: dns_msg, len: count) &priority=5
{
set_session(c, msg, is_orig);
}
event DNS::do_reply(c: connection, msg: dns_msg, ans: dns_answer, reply: string) &priority=5
{
set_session(c, msg, F);
if ( ans$answer_type == DNS_ANS )
{
c$dns$AA = msg$AA;
@ -198,7 +203,8 @@ event DNS::do_reply(c: connection, msg: dns_msg, ans: dns_answer, reply: string)
c$dns$TTLs[|c$dns$TTLs|] = ans$TTL;
}
if ( c$dns?$answers && |c$dns$answers| == c$dns$total_answers )
if ( c$dns?$answers && c$dns?$total_answers &&
|c$dns$answers| == c$dns$total_answers )
{
add c$dns_state$finished_answers[c$dns$trans_id];
# Indicate this request/reply pair is ready to be logged.
@ -219,8 +225,6 @@ event DNS::do_reply(c: connection, msg: dns_msg, ans: dns_answer, reply: string)
event dns_request(c: connection, msg: dns_msg, query: string, qtype: count, qclass: count) &priority=5
{
set_session(c, msg, T);
c$dns$RD = msg$RD;
c$dns$TC = msg$TC;
c$dns$qclass = qclass;
@ -310,11 +314,9 @@ event dns_SRV_reply(c: connection, msg: dns_msg, ans: dns_answer) &priority=5
#
# }
event dns_rejected(c: connection, msg: dns_msg,
query: string, qtype: count, qclass: count) &priority=5
event dns_rejected(c: connection, msg: dns_msg, query: string, qtype: count, qclass: count) &priority=5
{
set_session(c, msg, F);
c$dns$rejected = T;
}
event connection_state_remove(c: connection) &priority=-5

1
scripts/base/protocols/ftp/__load__.bro
View file

@ -1,3 +1,4 @@
@load ./utils-commands
@load ./main
@load ./file-extract
@load ./gridftp

121
scripts/base/protocols/ftp/gridftp.bro Normal file
View file

@ -0,0 +1,121 @@
##! A detection script for GridFTP data and control channels.
##!
##! GridFTP control channels are identified by FTP control channels
##! that successfully negotiate the GSSAPI method of an AUTH request
##! and for which the exchange involved an encoded TLS/SSL handshake,
##! indicating the GSI mechanism for GSSAPI was used. This analysis
##! is all supported internally, this script simple adds the "gridftp"
##! label to the *service* field of the control channel's
##! :bro:type:`connection` record.
##!
##! GridFTP data channels are identified by a heuristic that relies on
##! the fact that default settings for GridFTP clients typically
##! mutally authenticate the data channel with TLS/SSL and negotiate a
##! NULL bulk cipher (no encryption). Connections with those
##! attributes are then polled for two minutes with decreasing frequency
##! to check if the transfer sizes are large enough to indicate a
##! GridFTP data channel that would be undesireable to analyze further
##! (e.g. stop TCP reassembly). A side effect is that true connection
##! sizes are not logged, but at the benefit of saving CPU cycles that
##! otherwise go to analyzing the large (and likely benign) connections.
@load ./main
@load base/protocols/conn
@load base/protocols/ssl
@load base/frameworks/notice
module GridFTP;
export {
## Number of bytes transferred before guessing a connection is a
## GridFTP data channel.
const size_threshold = 1073741824 &redef;
## Max number of times to check whether a connection's size exceeds the
## :bro:see:`GridFTP::size_threshold`.
const max_poll_count = 15 &redef;
## Whether to skip further processing of the GridFTP data channel once
## detected, which may help performance.
const skip_data = T &redef;
## Base amount of time between checking whether a GridFTP data connection
## has transferred more than :bro:see:`GridFTP::size_threshold` bytes.
const poll_interval = 1sec &redef;
## The amount of time the base :bro:see:`GridFTP::poll_interval` is
## increased by each poll interval. Can be used to make more frequent
## checks at the start of a connection and gradually slow down.
const poll_interval_increase = 1sec &redef;
## Raised when a GridFTP data channel is detected.
##
## c: The connection pertaining to the GridFTP data channel.
global data_channel_detected: event(c: connection);
## The initial criteria used to determine whether to start polling
## the connection for the :bro:see:`GridFTP::size_threshold` to have
## been exceeded. This is called in a :bro:see:`ssl_established` event
## handler and by default looks for both a client and server certificate
## and for a NULL bulk cipher. One way in which this function could be
## redefined is to make it also consider client/server certificate issuer
## subjects.
##
## c: The connection which may possibly be a GridFTP data channel.
##
## Returns: true if the connection should be further polled for an
## exceeded :bro:see:`GridFTP::size_threshold`, else false.
const data_channel_initial_criteria: function(c: connection): bool &redef;
}
redef record FTP::Info += {
last_auth_requested: string &optional;
};
event ftp_request(c: connection, command: string, arg: string) &priority=4
{
if ( command == "AUTH" && c?$ftp )
c$ftp$last_auth_requested = arg;
}
function size_callback(c: connection, cnt: count): interval
{
if ( c$orig$size > size_threshold || c$resp$size > size_threshold )
{
add c$service["gridftp-data"];
event GridFTP::data_channel_detected(c);
if ( skip_data )
skip_further_processing(c$id);
return -1sec;
}
if ( cnt >= max_poll_count )
return -1sec;
return poll_interval + poll_interval_increase * cnt;
}
event ssl_established(c: connection) &priority=5
{
# If an FTP client requests AUTH GSSAPI and later an SSL handshake
# finishes, it's likely a GridFTP control channel, so add service label.
if ( c?$ftp && c$ftp?$last_auth_requested &&
/GSSAPI/ in c$ftp$last_auth_requested )
add c$service["gridftp"];
}
function data_channel_initial_criteria(c: connection): bool
{
return ( c?$ssl && c$ssl?$client_subject && c$ssl?$subject &&
c$ssl?$cipher && /WITH_NULL/ in c$ssl$cipher );
}
event ssl_established(c: connection) &priority=-3
{
# By default GridFTP data channels do mutual authentication and
# negotiate a cipher suite with a NULL bulk cipher.
if ( data_channel_initial_criteria(c) )
ConnPolling::watch(c, size_callback, 0, 0secs);
}

10
scripts/base/protocols/ftp/main.bro
View file

@ -7,6 +7,7 @@
@load ./utils-commands
@load base/utils/paths
@load base/utils/numbers
@load base/utils/addrs
module FTP;
@ -29,7 +30,9 @@ export {
type Info: record {
## Time when the command was sent.
ts: time &log;
## Unique ID for the connection.
uid: string &log;
## The connection's 4-tuple of endpoint addresses/ports.
id: conn_id &log;
## User name for the current FTP session.
user: string &log &default="<unknown>";
@ -169,12 +172,7 @@ function ftp_message(s: Info)
local arg = s$cmdarg$arg;
if ( s$cmdarg$cmd in file_cmds )
{
if ( is_v4_addr(s$id$resp_h) )
arg = fmt("ftp://%s%s", s$id$resp_h, build_path_compressed(s$cwd, arg));
else
arg = fmt("ftp://[%s]%s", s$id$resp_h, build_path_compressed(s$cwd, arg));
}
arg = fmt("ftp://%s%s", addr_to_uri(s$id$resp_h), build_path_compressed(s$cwd, arg));
s$ts=s$cmdarg$ts;
s$command=s$cmdarg$cmd;

3
scripts/base/protocols/http/file-ident.bro
View file

@ -6,7 +6,8 @@
@load ./utils
# Add the magic number signatures to the core signature set.
redef signature_files += "base/protocols/http/file-ident.sig";
@load-sigs ./file-ident.sig
# Ignore the signatures used to match files
redef Signatures::ignored_ids += /^matchfile-/;

2
scripts/base/protocols/http/main.bro
View file

@ -23,7 +23,9 @@ export {
type Info: record {
## Timestamp for when the request happened.
ts: time &log;
## Unique ID for the connection.
uid: string &log;
## The connection's 4-tuple of endpoint addresses/ports.
id: conn_id &log;
## Represents the pipelined depth into the connection of this
## request/response transaction.

3
scripts/base/protocols/http/utils.bro
View file

@ -1,6 +1,7 @@
##! Utilities specific for HTTP processing.
@load ./main
@load base/utils/addrs
module HTTP;
@ -51,7 +52,7 @@ function extract_keys(data: string, kv_splitter: pattern): string_vec
function build_url(rec: Info): string
{
local uri = rec?$uri ? rec$uri : "/<missed_request>";
local host = rec?$host ? rec$host : fmt("%s", rec$id$resp_h);
local host = rec?$host ? rec$host : addr_to_uri(rec$id$resp_h);
if ( rec$id$resp_p != 80/tcp )
host = fmt("%s:%s", host, rec$id$resp_p);
return fmt("%s%s", host, uri);

2
scripts/base/protocols/irc/main.bro
View file

@ -13,7 +13,9 @@ export {
type Info: record {
## Timestamp when the command was seen.
ts: time &log;
## Unique ID for the connection.
uid: string &log;
## The connection's 4-tuple of endpoint addresses/ports.
id: conn_id &log;
## Nick name given for the connection.
nick: string &log &optional;

2
scripts/base/protocols/modbus/__load__.bro Normal file
View file

@ -0,0 +1,2 @@
@load ./consts
@load ./main

67
scripts/base/protocols/modbus/consts.bro Normal file
View file

@ -0,0 +1,67 @@
module Modbus;
export {
## Standard defined Modbus function codes.
const function_codes = {
[0x01] = "READ_COILS",
[0x02] = "READ_DISCRETE_INPUTS",
[0x03] = "READ_HOLDING_REGISTERS",
[0x04] = "READ_INPUT_REGISTERS",
[0x05] = "WRITE_SINGLE_COIL",
[0x06] = "WRITE_SINGLE_REGISTER",
[0x07] = "READ_EXCEPTION_STATUS",
[0x08] = "DIAGNOSTICS",
[0x0B] = "GET_COMM_EVENT_COUNTER",
[0x0C] = "GET_COMM_EVENT_LOG",
[0x0F] = "WRITE_MULTIPLE_COILS",
[0x10] = "WRITE_MULTIPLE_REGISTERS",
[0x11] = "REPORT_SLAVE_ID",
[0x14] = "READ_FILE_RECORD",
[0x15] = "WRITE_FILE_RECORD",
[0x16] = "MASK_WRITE_REGISTER",
[0x17] = "READ_WRITE_MULTIPLE_REGISTERS",
[0x18] = "READ_FIFO_QUEUE",
[0x2B] = "ENCAP_INTERFACE_TRANSPORT",
# Machine/vendor/network specific functions
[0x09] = "PROGRAM_484",
[0x0A] = "POLL_484",
[0x0D] = "PROGRAM_584_984",
[0x0E] = "POLL_584_984",
[0x12] = "PROGRAM_884_U84",
[0x13] = "RESET_COMM_LINK_884_U84",
[0x28] = "PROGRAM_CONCEPT",
[0x7D] = "FIRMWARE_REPLACEMENT",
[0x7E] = "PROGRAM_584_984_2",
[0x7F] = "REPORT_LOCAL_ADDRESS",
# Exceptions
[0x81] = "READ_COILS_EXCEPTION",
[0x82] = "READ_DISCRETE_INPUTS_EXCEPTION",
[0x83] = "READ_HOLDING_REGISTERS_EXCEPTION",
[0x84] = "READ_INPUT_REGISTERS_EXCEPTION",
[0x85] = "WRITE_SINGLE_COIL_EXCEPTION",
[0x86] = "WRITE_SINGLE_REGISTER_EXCEPTION",
[0x87] = "READ_EXCEPTION_STATUS_EXCEPTION",
[0x8F] = "WRITE_MULTIPLE_COILS_EXCEPTION",
[0x90] = "WRITE_MULTIPLE_REGISTERS_EXCEPTION",
[0x94] = "READ_FILE_RECORD_EXCEPTION",
[0x95] = "WRITE_FILE_RECORD_EXCEPTION",
[0x96] = "MASK_WRITE_REGISTER_EXCEPTION",
[0x97] = "READ_WRITE_MULTIPLE_REGISTERS_EXCEPTION",
[0x98] = "READ_FIFO_QUEUE_EXCEPTION",
} &default=function(i: count):string { return fmt("unknown-%d", i); } &redef;
const exception_codes = {
[0x01] = "ILLEGAL_FUNCTION",
[0x02] = "ILLEGAL_DATA_ADDRESS",
[0x03] = "ILLEGAL_DATA_VALUE",
[0x04] = "SLAVE_DEVICE_FAILURE",
[0x05] = "ACKNOWLEDGE",
[0x06] = "SLAVE_DEVICE_BUSY",
[0x08] = "MEMORY_PARITY_ERROR",
[0x0A] = "GATEWAY_PATH_UNAVAILABLE",
[0x0B] = "GATEWAY_TARGET_DEVICE_FAILED_TO_RESPOND",
} &default=function(i: count):string { return fmt("unknown-%d", i); } &redef;
}

71
scripts/base/protocols/modbus/main.bro Normal file
View file

@ -0,0 +1,71 @@
##! Base Modbus analysis script.
module Modbus;
@load ./consts
export {
redef enum Log::ID += { LOG };
type Info: record {
## Time of the request.
ts: time &log;
## Unique identifier for the connnection.
uid: string &log;
## Identifier for the connection.
id: conn_id &log;
## The name of the function message that was sent.
func: string &log &optional;
## The exception if the response was a failure.
exception: string &log &optional;
};
## Event that can be handled to access the Modbus record as it is sent on
## to the logging framework.
global log_modbus: event(rec: Info);
}
redef record connection += {
modbus: Info &optional;
};
# Configure DPD and the packet filter.
redef capture_filters += { ["modbus"] = "tcp port 502" };
redef dpd_config += { [ANALYZER_MODBUS] = [$ports = set(502/tcp)] };
redef likely_server_ports += { 502/tcp };
event bro_init() &priority=5
{
Log::create_stream(Modbus::LOG, [$columns=Info, $ev=log_modbus]);
}
event modbus_message(c: connection, headers: ModbusHeaders, is_orig: bool) &priority=5
{
if ( ! c?$modbus )
{
c$modbus = [$ts=network_time(), $uid=c$uid, $id=c$id];
}
c$modbus$ts = network_time();
c$modbus$func = function_codes[headers$function_code];
}
event modbus_message(c: connection, headers: ModbusHeaders, is_orig: bool) &priority=-5
{
# Only log upon replies.
# Also, don't log now if this is an exception (log in the exception event handler)
if ( ! is_orig && ( headers$function_code <= 0x81 || headers$function_code >= 0x98 ) )
Log::write(LOG, c$modbus);
}
event modbus_exception(c: connection, headers: ModbusHeaders, code: count) &priority=5
{
c$modbus$exception = exception_codes[code];
}
event modbus_exception(c: connection, headers: ModbusHeaders, code: count) &priority=-5
{
Log::write(LOG, c$modbus);
delete c$modbus$exception;
}

28
scripts/base/protocols/smtp/main.bro
View file

@ -9,33 +9,51 @@ export {
redef enum Log::ID += { LOG };
type Info: record {
## Time when the message was first seen.
ts: time &log;
## Unique ID for the connection.
uid: string &log;
## The connection's 4-tuple of endpoint addresses/ports.
id: conn_id &log;
## This is a number that indicates the number of messages deep into
## this connection where this particular message was transferred.
## A count to represent the depth of this message transaction in a single
## connection where multiple messages were transferred.
trans_depth: count &log;
## Contents of the Helo header.
helo: string &log &optional;
## Contents of the From header.
mailfrom: string &log &optional;
## Contents of the Rcpt header.
rcptto: set[string] &log &optional;
## Contents of the Date header.
date: string &log &optional;
## Contents of the From header.
from: string &log &optional;
## Contents of the To header.
to: set[string] &log &optional;
## Contents of the ReplyTo header.
reply_to: string &log &optional;
## Contents of the MsgID header.
msg_id: string &log &optional;
## Contents of the In-Reply-To header.
in_reply_to: string &log &optional;
## Contents of the Subject header.
subject: string &log &optional;
## Contents of the X-Origininating-IP header.
x_originating_ip: addr &log &optional;
## Contents of the first Received header.
first_received: string &log &optional;
## Contents of the second Received header.
second_received: string &log &optional;
## The last message the server sent to the client.
## The last message that the server sent to the client.
last_reply: string &log &optional;
## The message transmission path, as extracted from the headers.
path: vector of addr &log &optional;
## Value of the User-Agent header from the client.
user_agent: string &log &optional;
## Indicate if the "Received: from" headers should still be processed.
## Indicates if the "Received: from" headers should still be processed.
process_received_from: bool &default=T;
## Indicates if client activity has been seen, but not yet logged
## Indicates if client activity has been seen, but not yet logged.
has_client_activity: bool &default=F;
};

2
scripts/base/protocols/socks/__load__.bro Normal file
View file

@ -0,0 +1,2 @@
@load ./consts
@load ./main

40
scripts/base/protocols/socks/consts.bro Normal file
View file

@ -0,0 +1,40 @@
module SOCKS;
export {
type RequestType: enum {
CONNECTION = 1,
PORT = 2,
UDP_ASSOCIATE = 3,
};
const v5_authentication_methods: table[count] of string = {
[0] = "No Authentication Required",
[1] = "GSSAPI",
[2] = "Username/Password",
[3] = "Challenge-Handshake Authentication Protocol",
[5] = "Challenge-Response Authentication Method",
[6] = "Secure Sockets Layer",
[7] = "NDS Authentication",
[8] = "Multi-Authentication Framework",
[255] = "No Acceptable Methods",
} &default=function(i: count):string { return fmt("unknown-%d", i); };
const v4_status: table[count] of string = {
[0x5a] = "succeeded",
[0x5b] = "general SOCKS server failure",
[0x5c] = "request failed because client is not running identd",
[0x5d] = "request failed because client's identd could not confirm the user ID string in the request",
} &default=function(i: count):string { return fmt("unknown-%d", i); };
const v5_status: table[count] of string = {
[0] = "succeeded",
[1] = "general SOCKS server failure",
[2] = "connection not allowed by ruleset",
[3] = "Network unreachable",
[4] = "Host unreachable",
[5] = "Connection refused",
[6] = "TTL expired",
[7] = "Command not supported",
[8] = "Address type not supported",
} &default=function(i: count):string { return fmt("unknown-%d", i); };
}

92
scripts/base/protocols/socks/main.bro Normal file
View file

@ -0,0 +1,92 @@
@load base/frameworks/tunnels
@load ./consts
module SOCKS;
export {
redef enum Log::ID += { LOG };
type Info: record {
## Time when the proxy connection was first detected.
ts: time &log;
## Unique ID for the tunnel - may correspond to connection uid or be non-existent.
uid: string &log;
## The connection's 4-tuple of endpoint addresses/ports.
id: conn_id &log;
## Protocol version of SOCKS.
version: count &log;
## Username for the proxy if extracted from the network..
user: string &log &optional;
## Server status for the attempt at using the proxy.
status: string &log &optional;
## Client requested SOCKS address. Could be an address, a name or both.
request: SOCKS::Address &log &optional;
## Client requested port.
request_p: port &log &optional;
## Server bound address. Could be an address, a name or both.
bound: SOCKS::Address &log &optional;
## Server bound port.
bound_p: port &log &optional;
};
## Event that can be handled to access the SOCKS
## record as it is sent on to the logging framework.
global log_socks: event(rec: Info);
}
event bro_init() &priority=5
{
Log::create_stream(SOCKS::LOG, [$columns=Info, $ev=log_socks]);
}
redef record connection += {
socks: SOCKS::Info &optional;
};
# Configure DPD
redef capture_filters += { ["socks"] = "tcp port 1080" };
redef dpd_config += { [ANALYZER_SOCKS] = [$ports = set(1080/tcp)] };
redef likely_server_ports += { 1080/tcp };
function set_session(c: connection, version: count)
{
if ( ! c?$socks )
c$socks = [$ts=network_time(), $id=c$id, $uid=c$uid, $version=version];
}
event socks_request(c: connection, version: count, request_type: count,
sa: SOCKS::Address, p: port, user: string) &priority=5
{
set_session(c, version);
c$socks$request = sa;
c$socks$request_p = p;
# Copy this conn_id and set the orig_p to zero because in the case of SOCKS proxies there will
# be potentially many source ports since a new proxy connection is established for each
# proxied connection. We treat this as a singular "tunnel".
local cid = copy(c$id);
cid$orig_p = 0/tcp;
Tunnel::register([$cid=cid, $tunnel_type=Tunnel::SOCKS, $payload_proxy=T]);
}
event socks_reply(c: connection, version: count, reply: count, sa: SOCKS::Address, p: port) &priority=5
{
set_session(c, version);
if ( version == 5 )
c$socks$status = v5_status[reply];
else if ( version == 4 )
c$socks$status = v4_status[reply];
c$socks$bound = sa;
c$socks$bound_p = p;
}
event socks_reply(c: connection, version: count, reply: count, sa: SOCKS::Address, p: port) &priority=-5
{
# This will handle the case where the analyzer failed in some way and was removed. We probably
# don't want to log these connections.
if ( "SOCKS" in c$service )
Log::write(SOCKS::LOG, c$socks);
}

10
scripts/base/protocols/ssh/main.bro
View file

@ -27,21 +27,23 @@ export {
type Info: record {
## Time when the SSH connection began.
ts: time &log;
## Unique ID for the connection.
uid: string &log;
## The connection's 4-tuple of endpoint addresses/ports.
id: conn_id &log;
## Indicates if the login was heuristically guessed to be "success"
## or "failure".
status: string &log &optional;
## Direction of the connection. If the client was a local host
## logging into an external host, this would be OUTBOUD. INBOUND
## logging into an external host, this would be OUTBOUND. INBOUND
## would be set for the opposite situation.
# TODO: handle local-local and remote-remote better.
direction: Direction &log &optional;
## Software string given by the client.
## Software string from the client.
client: string &log &optional;
## Software string given by the server.
## Software string from the server.
server: string &log &optional;
## Amount of data returned from the server. This is currently
## Amount of data returned from the server. This is currently
## the only measure of the success heuristic and it is logged to
## assist analysts looking at the logs to make their own determination
## about the success on a case-by-case basis.

2
scripts/base/protocols/ssl/consts.bro
View file

@ -81,6 +81,8 @@ export {
[35] = "SessionTicket TLS",
[40] = "extended_random",
[13172] = "next_protocol_negotiation",
[13175] = "origin_bound_certificates",
[13180] = "encrypted_client_certificates",
[65281] = "renegotiation_info"
} &default=function(i: count):string { return fmt("unknown-%d", i); };

68
scripts/base/protocols/ssl/main.bro
View file

@ -10,13 +10,15 @@ export {
redef enum Log::ID += { LOG };
type Info: record {
## Time when the SSL connection began.
## Time when the SSL connection was first detected.
ts: time &log;
uid: string &log;
## Unique ID for the connection.
uid: string &log;
## The connection's 4-tuple of endpoint addresses/ports.
id: conn_id &log;
## SSL/TLS version the server offered.
## SSL/TLS version that the server offered.
version: string &log &optional;
## SSL/TLS cipher suite the server chose.
## SSL/TLS cipher suite that the server chose.
cipher: string &log &optional;
## Value of the Server Name Indicator SSL/TLS extension. It
## indicates the server name that the client was requesting.
@ -25,19 +27,32 @@ export {
session_id: string &log &optional;
## Subject of the X.509 certificate offered by the server.
subject: string &log &optional;
## Subject of the signer of the X.509 certificate offered by the server.
issuer_subject: string &log &optional;
## NotValidBefore field value from the server certificate.
not_valid_before: time &log &optional;
## NotValidAfter field value from the serve certificate.
## NotValidAfter field value from the server certificate.
not_valid_after: time &log &optional;
## Last alert that was seen during the connection.
last_alert: string &log &optional;
## Subject of the X.509 certificate offered by the client.
client_subject: string &log &optional;
## Subject of the signer of the X.509 certificate offered by the client.
client_issuer_subject: string &log &optional;
## Full binary server certificate stored in DER format.
cert: string &optional;
## Chain of certificates offered by the server to validate its
## complete signing chain.
cert_chain: vector of string &optional;
## Full binary client certificate stored in DER format.
client_cert: string &optional;
## Chain of certificates offered by the client to validate its
## complete signing chain.
client_cert_chain: vector of string &optional;
## The analyzer ID used for the analyzer instance attached
## to each connection. It is not used for logging since it's a
## meaningless arbitrary number.
@ -82,7 +97,8 @@ redef Protocols::common_ports += { ["SSL"] = ports };
function set_session(c: connection)
{
if ( ! c?$ssl )
c$ssl = [$ts=network_time(), $uid=c$uid, $id=c$id, $cert_chain=vector()];
c$ssl = [$ts=network_time(), $uid=c$uid, $id=c$id, $cert_chain=vector(),
$client_cert_chain=vector()];
}
function finish(c: connection)
@ -116,22 +132,40 @@ event x509_certificate(c: connection, is_orig: bool, cert: X509, chain_idx: coun
# We aren't doing anything with client certificates yet.
if ( is_orig )
return;
if ( chain_idx == 0 )
{
# Save the primary cert.
c$ssl$cert = der_cert;
if ( chain_idx == 0 )
{
# Save the primary cert.
c$ssl$client_cert = der_cert;
# Also save other certificate information about the primary cert.
c$ssl$subject = cert$subject;
c$ssl$not_valid_before = cert$not_valid_before;
c$ssl$not_valid_after = cert$not_valid_after;
# Also save other certificate information about the primary cert.
c$ssl$client_subject = cert$subject;
c$ssl$client_issuer_subject = cert$issuer;
}
else
{
# Otherwise, add it to the cert validation chain.
c$ssl$client_cert_chain[|c$ssl$client_cert_chain|] = der_cert;
}
}
else
{
# Otherwise, add it to the cert validation chain.
c$ssl$cert_chain[|c$ssl$cert_chain|] = der_cert;
if ( chain_idx == 0 )
{
# Save the primary cert.
c$ssl$cert = der_cert;
# Also save other certificate information about the primary cert.
c$ssl$subject = cert$subject;
c$ssl$issuer_subject = cert$issuer;
c$ssl$not_valid_before = cert$not_valid_before;
c$ssl$not_valid_after = cert$not_valid_after;
}
else
{
# Otherwise, add it to the cert validation chain.
c$ssl$cert_chain[|c$ssl$cert_chain|] = der_cert;
}
}
}

14
scripts/base/protocols/ssl/mozilla-ca-list.bro
View file

File diff suppressed because one or more lines are too long

4
scripts/base/protocols/syslog/main.bro
View file

@ -10,9 +10,11 @@ export {
redef enum Log::ID += { LOG };
type Info: record {
## Timestamp of when the syslog message was seen.
## Timestamp when the syslog message was seen.
ts: time &log;
## Unique ID for the connection.
uid: string &log;
## The connection's 4-tuple of endpoint addresses/ports.
id: conn_id &log;
## Protocol over which the message was seen.
proto: transport_proto &log;

15
scripts/base/utils/addrs.bro
View file

@ -98,3 +98,18 @@ function find_ip_addresses(input: string): string_array
}
return output;
}
## Returns the string representation of an IP address suitable for inclusion
## in a URI. For IPv4, this does no special formatting, but for IPv6, the
## address is included in square brackets.
##
## a: the address to make suitable for URI inclusion.
##
## Returns: the string representation of *a* suitable for URI inclusion.
function addr_to_uri(a: addr): string
{
if ( is_v4_addr(a) )
return fmt("%s", a);
else
return fmt("[%s]", a);
}

5
scripts/base/utils/files.bro
View file

@ -1,10 +1,11 @@
@load ./addrs
## This function can be used to generate a consistent filename for when
## contents of a file, stream, or connection are being extracted to disk.
function generate_extraction_filename(prefix: string, c: connection, suffix: string): string
{
local conn_info = fmt("%s:%d-%s:%d",
c$id$orig_h, c$id$orig_p, c$id$resp_h, c$id$resp_p);
local conn_info = fmt("%s:%d-%s:%d", addr_to_uri(c$id$orig_h), c$id$orig_p,
addr_to_uri(c$id$resp_h), c$id$resp_p);
if ( prefix != "" )
conn_info = fmt("%s_%s", prefix, conn_info);

2
scripts/base/utils/patterns.bro
View file

@ -1,5 +1,7 @@
##! Functions for creating and working with patterns.
module GLOBAL;
## Given a pattern as a string with two tildes (~~) contained in it, it will
## return a pattern with string set's elements OR'd together where the
## double-tilde was given (this function only works at or before init time).

16
scripts/base/utils/site.bro
View file

@ -10,15 +10,19 @@ export {
const private_address_space: set[subnet] = {
10.0.0.0/8,
192.168.0.0/16,
172.16.0.0/12,
100.64.0.0/10, # RFC6598 Carrier Grade NAT
127.0.0.0/8,
172.16.0.0/12
[fe80::]/10,
[::1]/128,
} &redef;
## Networks that are considered "local".
const local_nets: set[subnet] &redef;
## This is used for retrieving the subnet when you multiple
## :bro:id:`Site::local_nets`. A membership query can be done with an
## This is used for retrieving the subnet when using multiple entries in
## :bro:id:`Site::local_nets`. It's populated automatically from there.
## A membership query can be done with an
## :bro:type:`addr` and the table will yield the subnet it was found
## within.
global local_nets_table: table[subnet] of subnet = {};
@ -40,27 +44,33 @@ export {
## Function that returns true if an address corresponds to one of
## the local networks, false if not.
## The function inspects :bro:id:`Site::local_nets`.
global is_local_addr: function(a: addr): bool;
## Function that returns true if an address corresponds to one of
## the neighbor networks, false if not.
## The function inspects :bro:id:`Site::neighbor_nets`.
global is_neighbor_addr: function(a: addr): bool;
## Function that returns true if an address corresponds to one of
## the private/unrouted networks, false if not.
## The function inspects :bro:id:`Site::private_address_space`.
global is_private_addr: function(a: addr): bool;
## Function that returns true if a host name is within a local
## DNS zone.
## The function inspects :bro:id:`Site::local_zones`.
global is_local_name: function(name: string): bool;
## Function that returns true if a host name is within a neighbor
## DNS zone.
## The function inspects :bro:id:`Site::neighbor_zones`.
global is_neighbor_name: function(name: string): bool;
## Function that returns a common separated list of email addresses
## that are considered administrators for the IP address provided as
## an argument.
## The function inspects :bro:id:`Site::local_admins`.
global get_emails: function(a: addr): string;
}

25
scripts/base/utils/urls.bro Normal file
View file

@ -0,0 +1,25 @@
## Functions for URL handling.
## A regular expression for matching and extracting URLs.
const url_regex = /^([a-zA-Z\-]{3,5})(:\/\/[^\/?#"'\r\n><]*)([^?#"'\r\n><]*)([^[:blank:]\r\n"'><]*|\??[^"'\r\n><]*)/ &redef;
## Extracts URLs discovered in arbitrary text.
function find_all_urls(s: string): string_set
{
return find_all(s, url_regex);
}
## Extracts URLs discovered in arbitrary text without
## the URL scheme included.
function find_all_urls_without_scheme(s: string): string_set
{
local urls = find_all_urls(s);
local return_urls: set[string] = set();
for ( url in urls )
{
local no_scheme = sub(url, /^([a-zA-Z\-]{3,5})(:\/\/)/, "");
add return_urls[no_scheme];
}
return return_urls;
}

3
scripts/policy/frameworks/communication/listen.bro
View file

@ -8,5 +8,6 @@ module Communication;
event bro_init() &priority=-10
{
enable_communication();
listen(listen_interface, listen_port, listen_ssl);
listen(listen_interface, listen_port, listen_ssl, listen_ipv6,
listen_ipv6_zone_id, listen_retry);
}

4
scripts/policy/frameworks/control/controller.bro
View file

@ -25,8 +25,8 @@ event bro_init() &priority=5
# Establish the communication configuration and only request response
# messages.
Communication::nodes["control"] = [$host=host, $p=host_port,
$sync=F, $connect=T,
Communication::nodes["control"] = [$host=host, $zone_id=zone_id,
$p=host_port, $sync=F, $connect=T,
$class="control", $events=Control::controllee_events];
}

8
scripts/policy/frameworks/intel/__load__.bro Normal file
View file

@ -0,0 +1,8 @@
@load ./conn-established
@load ./dns
@load ./http-host-header
@load ./http-url
@load ./http-user-agents
@load ./ssl
@load ./smtp
@load ./smtp-url-extraction

8
scripts/policy/frameworks/intel/conn-established.bro Normal file
View file

@ -0,0 +1,8 @@
@load base/frameworks/intel
@load ./where-locations
event connection_established(c: connection)
{
Intel::seen([$host=c$id$orig_h, $conn=c, $where=Conn::IN_ORIG]);
Intel::seen([$host=c$id$resp_h, $conn=c, $where=Conn::IN_RESP]);
}

10
scripts/policy/frameworks/intel/dns.bro Normal file
View file

@ -0,0 +1,10 @@
@load base/frameworks/intel
@load ./where-locations
event dns_request(c: connection, msg: dns_msg, query: string, qtype: count, qclass: count)
{
Intel::seen([$str=query,
$str_type=Intel::DOMAIN,
$conn=c,
$where=DNS::IN_REQUEST]);
}

11
scripts/policy/frameworks/intel/http-host-header.bro Normal file
View file

@ -0,0 +1,11 @@
@load base/frameworks/intel
@load ./where-locations
event http_header(c: connection, is_orig: bool, name: string, value: string)
{
if ( is_orig && name == "HOST" )
Intel::seen([$str=value,
$str_type=Intel::DOMAIN,
$conn=c,
$where=HTTP::IN_HOST_HEADER]);
}

12
scripts/policy/frameworks/intel/http-url.bro Normal file
View file

@ -0,0 +1,12 @@
@load base/frameworks/intel
@load base/protocols/http/utils
@load ./where-locations
event http_message_done(c: connection, is_orig: bool, stat: http_message_stat)
{
if ( is_orig && c?$http )
Intel::seen([$str=HTTP::build_url(c$http),
$str_type=Intel::URL,
$conn=c,
$where=HTTP::IN_URL]);
}

12
scripts/policy/frameworks/intel/http-user-agents.bro Normal file
View file

@ -0,0 +1,12 @@
@load base/frameworks/intel
@load ./where-locations
event http_header(c: connection, is_orig: bool, name: string, value: string)
{
if ( is_orig && name == "USER-AGENT" )
Intel::seen([$str=value,
$str_type=Intel::USER_AGENT,
$conn=c,
$where=HTTP::IN_USER_AGENT_HEADER]);
}

15
scripts/policy/frameworks/intel/smtp-url-extraction.bro Normal file
View file

@ -0,0 +1,15 @@
@load base/frameworks/intel
@load base/utils/urls
@load ./where-locations
event mime_segment_data(c: connection, length: count, data: string) &priority=3
{
local urls = find_all_urls_without_scheme(data);
for ( url in urls )
{
Intel::seen([$str=url,
$str_type=Intel::URL,
$conn=c,
$where=SMTP::IN_MESSAGE]);
}
}

71
scripts/policy/frameworks/intel/smtp.bro Normal file
View file

@ -0,0 +1,71 @@
@load base/frameworks/intel
@load base/protocols/smtp
@load ./where-locations
event mime_end_entity(c: connection)
{
if ( c?$smtp )
{
if ( c$smtp?$path )
{
local path = c$smtp$path;
for ( i in path )
{
Intel::seen([$host=path[i],
$conn=c,
$where=SMTP::IN_RECEIVED_HEADER]);
}
}
if ( c$smtp?$user_agent )
Intel::seen([$str=c$smtp$user_agent,
$str_type=Intel::USER_AGENT,
$conn=c,
$where=SMTP::IN_HEADER]);
if ( c$smtp?$x_originating_ip )
Intel::seen([$host=c$smtp$x_originating_ip,
$conn=c,
$where=SMTP::IN_X_ORIGINATING_IP_HEADER]);
if ( c$smtp?$mailfrom )
Intel::seen([$str=c$smtp$mailfrom,
$str_type=Intel::EMAIL,
$conn=c,
$where=SMTP::IN_MAIL_FROM]);
if ( c$smtp?$rcptto )
{
for ( rcptto in c$smtp$rcptto )
{
Intel::seen([$str=rcptto,
$str_type=Intel::EMAIL,
$conn=c,
$where=SMTP::IN_RCPT_TO]);
}
}
if ( c$smtp?$from )
Intel::seen([$str=c$smtp$from,
$str_type=Intel::EMAIL,
$conn=c,
$where=SMTP::IN_FROM]);
if ( c$smtp?$to )
{
for ( email_to in c$smtp$to )
{
Intel::seen([$str=email_to,
$str_type=Intel::EMAIL,
$conn=c,
$where=SMTP::IN_TO]);
}
}
if ( c$smtp?$reply_to )
Intel::seen([$str=c$smtp$reply_to,
$str_type=Intel::EMAIL,
$conn=c,
$where=SMTP::IN_REPLY_TO]);
}
}

34
scripts/policy/frameworks/intel/ssl.bro Normal file
View file

@ -0,0 +1,34 @@
@load base/frameworks/intel
@load base/protocols/ssl
@load ./where-locations
event x509_certificate(c: connection, is_orig: bool, cert: X509, chain_idx: count, chain_len: count, der_cert: string)
{
if ( chain_idx == 0 )
{
if ( /emailAddress=/ in cert$subject )
{
local email = sub(cert$subject, /^.*emailAddress=/, "");
email = sub(email, /,.*$/, "");
Intel::seen([$str=email,
$str_type=Intel::EMAIL,
$conn=c,
$where=(is_orig ? SSL::IN_CLIENT_CERT : SSL::IN_SERVER_CERT)]);
}
Intel::seen([$str=sha1_hash(der_cert),
$str_type=Intel::CERT_HASH,
$conn=c,
$where=(is_orig ? SSL::IN_CLIENT_CERT : SSL::IN_SERVER_CERT)]);
}
}
event ssl_extension(c: connection, is_orig: bool, code: count, val: string)
{
if ( is_orig && SSL::extensions[code] == "server_name" &&
c?$ssl && c$ssl?$server_name )
Intel::seen([$str=c$ssl$server_name,
$str_type=Intel::DOMAIN,
$conn=c,
$where=SSL::IN_SERVER_NAME]);
}

25
scripts/policy/frameworks/intel/where-locations.bro Normal file
View file

@ -0,0 +1,25 @@
@load base/frameworks/intel
export {
redef enum Intel::Where += {
Conn::IN_ORIG,
Conn::IN_RESP,
DNS::IN_REQUEST,
DNS::IN_RESPONSE,
HTTP::IN_HOST_HEADER,
HTTP::IN_USER_AGENT_HEADER,
HTTP::IN_URL,
SMTP::IN_MAIL_FROM,
SMTP::IN_RCPT_TO,
SMTP::IN_FROM,
SMTP::IN_TO,
SMTP::IN_RECEIVED_HEADER,
SMTP::IN_REPLY_TO,
SMTP::IN_X_ORIGINATING_IP_HEADER,
SMTP::IN_MESSAGE,
SSL::IN_SERVER_CERT,
SSL::IN_CLIENT_CERT,
SSL::IN_SERVER_NAME,
SMTP::IN_HEADER,
};
}

6
scripts/policy/integration/collective-intel/README Normal file
View file

@ -0,0 +1,6 @@
Collective Intelligence Framework Integration
=============================================
The scripts in this module are for deeper integration with the Collective Intelligence
Framework (CIF) since Bro's Intel framework doesn't natively behave the same as CIF nor
does it store and maintain the same data in all cases.

Some files were not shown because too many files have changed in this diff Show more