diff --git a/CHANGES b/CHANGES index 582694a79a..2cbebd0255 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,104 @@ +2.2-117 | 2014-01-23 14:18:19 -0800 + + * Fixing initialization context in anonymous functions. (Robin + Sommer) + +2.2-115 | 2014-01-22 12:11:18 -0800 + + * Add unit tests for new Bro Manual docs. (Jon Siwek) + + * New content for the "Using Bro" section of the manual. (Rafael + Bonilla/Jon Siwek) + +2.2-105 | 2014-01-20 12:16:48 -0800 + + * Support GRE tunnel decapsulation, including enhanced GRE headers. + GRE tunnels are treated just like IP-in-IP tunnels by parsing past + the GRE header in between the delivery and payload IP packets. + Addresses BIT-867. (Jon Siwek) + + * Simplify FragReassembler memory management. (Jon Siwek) + +2.2-102 | 2014-01-20 12:00:29 -0800 + + * Include file information (MIME type and description) into notice + emails if available. (Justin Azoff) + +2.2-100 | 2014-01-20 11:54:58 -0800 + + * Fix caching of recently validated SSL certifcates. (Justin Azoff) + +2.2-98 | 2014-01-20 11:50:32 -0800 + + * For notice suppresion, instead of storing the entire notice in + Notice::suppressing, just store the time the notice should be + suppressed until. This saves significant memory but can no longer + raise end_suppression, which has been removed. (Justin Azoff) + +2.2-96 | 2014-01-20 11:41:07 -0800 + + * Integrate libmagic 5.16. Bro now now always relies on + builtin/shipped magic library/database. (Jon Siwek) + + * Bro now requires a CMake 2.8.x, but no longer a pre-installed + libmagic. (Jon Siwek) + +2.2-93 | 2014-01-13 09:16:51 -0800 + + * Fixing compile problems with some versions of libc++. Reported by + Craig Leres. (Robin Sommer) + +2.2-91 | 2014-01-13 01:33:28 -0800 + + * Improve GeoIP City database support. When trying to open a city + database, it now considers both the "REV0" and "REV1" versions of + the city database instead of just the former. (Jon Siwek) + + * Broxygen init fixes. Addresses BIT-1110. (Jon Siwek) + + - Don't check mtime of bro binary if BRO_DISABLE_BROXYGEN env var set. + + - Fix failure to locate bro binary if invoking from a relative + path and '.' isn't in PATH. + + * Fix for packet writing to make it use the global snap length. + (Seth Hall) + + * Fix for traffic with TCP segmentation offloading with IP header + len field being set to zero. (Seth Hall) + + * Canonify output of a unit test. (Jon Siwek) + + * A set of documentation updates. (Daniel Thayer) + + - Fix typo in Bro 2.2 NEWS on string indexing. + - Fix typo in the Quick Start Guide, and clarified the + instructions about modifying crontab. + - Add/fix documentation for missing/misnamed event parameters. + - Fix typos in BIF documentation of hexstr_to_bytestring. + - Update the documentation of types and attributes. + - Documented the new substring extraction functionality. + - Clarified the description of "&priority" and "void". + +2.2-75 | 2013-12-18 08:36:50 -0800 + + * Fixing segfault with mismatching set &default in record fields. + (Robin Sommer) + +2.2-74 | 2013-12-16 08:49:55 -0800 + + * Improve warnings emitted from raw/execute input reader. (Jon + Siwek) + + * Further improve core.when-interpreter-exceptions unit test. (Jon + Siwek) + +2.2-72 | 2013-12-12 07:12:47 -0800 + + * Improve the core.when-interpreter-exceptions unit test to prevent + it from occasionally timing out. (Jon Siwek) + 2.2-70 | 2013-12-10 15:02:50 -0800 * Fix (harmless) uninitialized field in basename/dirname util diff --git a/CMakeLists.txt b/CMakeLists.txt index 88cee2ec29..483f88babf 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,5 +1,10 @@ project(Bro C CXX) -cmake_minimum_required(VERSION 2.6.3 FATAL_ERROR) + +# When changing the minimum version here, also adapt +# cmake/BroPluginDynamic and +# aux/bro-aux/plugin-support/skeleton/CMakeLists.txt +cmake_minimum_required(VERSION 2.8.0 FATAL_ERROR) + include(cmake/CommonCMakeConfig.cmake) ######################################################################## @@ -16,17 +21,21 @@ endif () get_filename_component(BRO_SCRIPT_INSTALL_PATH ${BRO_SCRIPT_INSTALL_PATH} ABSOLUTE) +set(BRO_PLUGIN_INSTALL_PATH ${BRO_ROOT_DIR}/lib/bro/plugins CACHE STRING "Installation path for plugins" FORCE) set(BRO_MAGIC_INSTALL_PATH ${BRO_ROOT_DIR}/share/bro/magic) set(BRO_MAGIC_SOURCE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/magic/database) + configure_file(bro-path-dev.in ${CMAKE_CURRENT_BINARY_DIR}/bro-path-dev) file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/bro-path-dev.sh "export BROPATH=`${CMAKE_CURRENT_BINARY_DIR}/bro-path-dev`\n" "export BROMAGIC=\"${BRO_MAGIC_SOURCE_PATH}\"\n" + "export BRO_PLUGIN_PATH=\"${CMAKE_CURRENT_BINARY_DIR}/src:${BRO_PLUGIN_INSTALL_PATH}\"\n" "export PATH=\"${CMAKE_CURRENT_BINARY_DIR}/src\":$PATH\n") file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/bro-path-dev.csh "setenv BROPATH `${CMAKE_CURRENT_BINARY_DIR}/bro-path-dev`\n" "setenv BROMAGIC \"${BRO_MAGIC_SOURCE_PATH}\"\n" + "setenv BRO_PLUGIN_PATH \"${CMAKE_CURRENT_BINARY_DIR}/src:${BRO_PLUGIN_INSTALL_PATH}\"\n" "setenv PATH \"${CMAKE_CURRENT_BINARY_DIR}/src\":$PATH\n") file(STRINGS "${CMAKE_CURRENT_SOURCE_DIR}/VERSION" VERSION LIMIT_COUNT 1) @@ -39,6 +48,32 @@ set(VERSION_MAJ_MIN "${VERSION_MAJOR}.${VERSION_MINOR}") ######################################################################## ## Dependency Configuration +include(ExternalProject) + +# LOG_* options to ExternalProject_Add appear in CMake 2.8.3. If +# available, using them hides external project configure/build output. +if("${CMAKE_VERSION}" VERSION_GREATER 2.8.2) + set(EXTERNAL_PROJECT_LOG_OPTIONS + LOG_DOWNLOAD 1 LOG_UPDATE 1 LOG_CONFIGURE 1 LOG_BUILD 1 LOG_INSTALL 1) +else() + set(EXTERNAL_PROJECT_LOG_OPTIONS) +endif() + +set(LIBMAGIC_PREFIX ${CMAKE_CURRENT_BINARY_DIR}/libmagic-prefix) +set(LIBMAGIC_INCLUDE_DIR ${LIBMAGIC_PREFIX}/include) +set(LIBMAGIC_LIB_DIR ${LIBMAGIC_PREFIX}/lib) +set(LIBMAGIC_LIBRARY ${LIBMAGIC_LIB_DIR}/libmagic.a) +ExternalProject_Add(libmagic + PREFIX ${LIBMAGIC_PREFIX} + URL ${CMAKE_CURRENT_SOURCE_DIR}/src/3rdparty/file-5.16.tar.gz + CONFIGURE_COMMAND ./configure --enable-static --disable-shared + --prefix=${LIBMAGIC_PREFIX} + --includedir=${LIBMAGIC_INCLUDE_DIR} + --libdir=${LIBMAGIC_LIB_DIR} + BUILD_IN_SOURCE 1 + ${EXTERNAL_PROJECT_LOG_OPTIONS} +) + include(FindRequiredPackage) # Check cache value first to avoid displaying "Found sed" messages everytime @@ -57,7 +92,6 @@ FindRequiredPackage(BISON) FindRequiredPackage(PCAP) FindRequiredPackage(OpenSSL) FindRequiredPackage(BIND) -FindRequiredPackage(LibMagic) FindRequiredPackage(ZLIB) if (NOT BinPAC_ROOT_DIR AND @@ -73,18 +107,12 @@ if (MISSING_PREREQS) message(FATAL_ERROR "Configuration aborted due to missing prerequisites") endif () -set(libmagic_req 5.04) -if ( LibMagic_VERSION VERSION_LESS ${libmagic_req} ) - message(FATAL_ERROR "libmagic of at least version ${libmagic_req} required " - "(found ${LibMagic_VERSION})") -endif () - include_directories(BEFORE ${PCAP_INCLUDE_DIR} ${OpenSSL_INCLUDE_DIR} ${BIND_INCLUDE_DIR} ${BinPAC_INCLUDE_DIR} - ${LibMagic_INCLUDE_DIR} + ${LIBMAGIC_INCLUDE_DIR} ${ZLIB_INCLUDE_DIR} ) @@ -163,7 +191,7 @@ set(brodeps ${PCAP_LIBRARY} ${OpenSSL_LIBRARIES} ${BIND_LIBRARY} - ${LibMagic_LIBRARY} + ${LIBMAGIC_LIBRARY} ${ZLIB_LIBRARY} ${OPTLIBS} ) @@ -182,6 +210,10 @@ include(MiscTests) include(PCAPTests) include(OpenSSLTests) include(CheckNameserCompat) +include(GetArchitecture) + +# Tell the plugin code that we're building as part of the main tree. +set(BRO_PLUGIN_INTERNAL_BUILD true CACHE INTERNAL "" FORCE) configure_file(${CMAKE_CURRENT_SOURCE_DIR}/config.h.in ${CMAKE_CURRENT_BINARY_DIR}/config.h) diff --git a/NEWS b/NEWS index 524cac14e0..df0bd07f4c 100644 --- a/NEWS +++ b/NEWS @@ -9,9 +9,21 @@ Bro 2.3 [In progress] +Dependencies +------------ + +- Bro no longer requires a pre-installed libmagic (because it now + ships its own). + +- Compiling from source now needs a CMake version >= 2.8.0. + New Functionality ----------------- +- Support for GRE tunnel decapsulation, including enhanced GRE + headers. GRE tunnels are treated just like IP-in-IP tunnels by + parsing past the GRE header in between the delivery and payload IP + packets. Changed Functionality --------------------- @@ -22,6 +34,8 @@ Changed Functionality - ssl_client_hello() now receives a vector of ciphers, instead of a set, to preserve their order. +- Notice::end_suppression() has been removed. + Bro 2.2 ======= @@ -198,9 +212,9 @@ New Functionality global s = MySet([$c=1], [$c=2]); - Strings now support the subscript operator to extract individual - characters and substrings (e.g., ``s[4]``, ``s[1,5]``). The index + characters and substrings (e.g., ``s[4]``, ``s[1:5]``). The index expression can take up to two indices for the start and end index of - the substring to return (e.g. ``mystring[1,3]``). + the substring to return (e.g. ``mystring[1:3]``). - Functions now support default parameters, e.g.:: diff --git a/VERSION b/VERSION index 066d37e8c2..33ad1f2a99 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.2-70 +2.2-117 diff --git a/aux/binpac b/aux/binpac index 54b321009b..896ddedde5 160000 --- a/aux/binpac +++ b/aux/binpac @@ -1 +1 @@ -Subproject commit 54b321009b750268526419bdbd841f421c839313 +Subproject commit 896ddedde55c48ec2163577fc258b49c418abb3e diff --git a/aux/bro-aux b/aux/bro-aux index ebf9c0d88a..77234b4ba1 160000 --- a/aux/bro-aux +++ b/aux/bro-aux @@ -1 +1 @@ -Subproject commit ebf9c0d88ae8230845b91f15755156f93ff21aa8 +Subproject commit 77234b4ba1c5ad0eda64554a7f16a0d79df9ca52 diff --git a/cmake b/cmake index e7a46cb82e..67da63d43e 160000 --- a/cmake +++ b/cmake @@ -1 +1 @@ -Subproject commit e7a46cb82ee10aa522c4d88115baf10181277d20 +Subproject commit 67da63d43e734111b324d8ed045e188e0a28ebf2 diff --git a/config.h.in b/config.h.in index 2d065f755e..d3889a2d90 100644 --- a/config.h.in +++ b/config.h.in @@ -209,3 +209,14 @@ /* Common IPv6 extension structure */ #cmakedefine HAVE_IP6_EXT + +/* String with host architecture (e.g., "linux-x86_64") */ +#define HOST_ARCHITECTURE "@HOST_ARCHITECTURE@" + +/* String with extension of dynamic libraries (e.g., ".so") */ +#define DYNAMIC_PLUGIN_SUFFIX "@CMAKE_SHARED_MODULE_SUFFIX@" + +/* True if we're building outside of the main Bro source code tree. */ +#ifndef BRO_PLUGIN_INTERNAL_BUILD +#define BRO_PLUGIN_INTERNAL_BUILD @BRO_PLUGIN_INTERNAL_BUILD@ +#endif diff --git a/doc/_static/basic.css b/doc/_static/basic.css index 1332c7b048..26e3450b65 100644 --- a/doc/_static/basic.css +++ b/doc/_static/basic.css @@ -439,8 +439,17 @@ td.linenos pre { color: #aaa; } +.highlight-guess { + overflow:auto; +} + +.highlight-none { + overflow:auto; +} + table.highlighttable { margin-left: 0.5em; + overflow:scroll; } table.highlighttable td { diff --git a/doc/broids/index.rst b/doc/broids/index.rst new file mode 100644 index 0000000000..46e0d6ded6 --- /dev/null +++ b/doc/broids/index.rst @@ -0,0 +1,80 @@ + +.. _bro-ids: + +======= +Bro IDS +======= + +An Intrusion Detection System (IDS) allows you to detect suspicious +activities happening on your network as a result of a past or active +attack. Because of its programming capabilities, Bro can easily be +configured to behave like traditional IDSs and detect common attacks +with well known patterns, or you can create your own scripts to detect +conditions specific to your particular case. + +In the following sections, we present a few examples of common uses of +Bro as an IDS. + +------------------------------------------------ +Detecting an FTP Bruteforce attack and notifying +------------------------------------------------ + +For the purpose of this exercise, we define FTP bruteforcing as too many +rejected usernames and passwords occurring from a single address. We +start by defining a threshold for the number of attempts and a +monitoring interval in minutes as well as a new notice type. + +.. btest-include:: ${BRO_SRC_ROOT}/scripts/policy/protocols/ftp/detect-bruteforcing.bro + :lines: 9-25 + +Now, using the ftp_reply event, we check for error codes from the `500 +series `_ +for the "USER" and "PASS" commands, representing rejected usernames or +passwords. For this, we can use the :bro:see:`FTP::parse_ftp_reply_code` +function to break down the reply code and check if the first digit is a +"5" or not. If true, we then use the :ref:`Summary Statistics Framework +` to keep track of the number of failed attempts. + +.. btest-include:: ${BRO_SRC_ROOT}/scripts/policy/protocols/ftp/detect-bruteforcing.bro + :lines: 52-60 + +Next, we use the SumStats framework to raise a notice of the attack of +the attack when the number of failed attempts exceeds the specified +threshold during the measuring interval. + +.. btest-include:: ${BRO_SRC_ROOT}/scripts/policy/protocols/ftp/detect-bruteforcing.bro + :lines: 28-50 + +Below is the final code for our script. + +.. btest-include:: ${BRO_SRC_ROOT}/scripts/policy/protocols/ftp/detect-bruteforcing.bro + +.. btest:: ftp-bruteforce + + @TEST-EXEC: btest-rst-cmd bro -r ${TRACES}/ftp/bruteforce.pcap protocols/ftp/detect-bruteforcing.bro + @TEST-EXEC: btest-rst-include notice.log + +As a final note, the :doc:`detect-bruteforcing.bro +` script above is +include with Bro out of the box, so you only need to load it at startup +to instruct Bro to detect and notify of FTP bruteforce attacks. + +------------- +Other Attacks +------------- + +Detecting SQL Injection attacks +------------------------------- + +Checking files against known malware hashes +------------------------------------------- + +Files transmitted on your network could either be completely harmless or +contain viruses and other threats. One possible action against this +threat is to compute the hashes of the files and compare them against a +list of known malware hashes. Bro simplifies this task by offering a +:doc:`detect-MHR.bro ` +script that creates and compares hashes against the `Malware Hash +Registry `_ maintained by Team +Cymru. You only need to load this script along with your other scripts +at startup time. diff --git a/doc/devel/plugins.rst b/doc/devel/plugins.rst new file mode 100644 index 0000000000..e9b8bf337c --- /dev/null +++ b/doc/devel/plugins.rst @@ -0,0 +1,339 @@ + +=================== +Writing Bro Plugins +=================== + +Bro is internally moving to a plugin structure that enables extending +the system dynamically, without modifying the core code base. That way +custom code remains self-contained and can be maintained, compiled, +and installed independently. Currently, plugins can add the following +functionality to Bro: + + - Bro scripts. + + - Builtin functions/events/types for the scripting language. + + - Protocol and file analyzers. + + - Packet sources and packet dumpers. TODO: Not yet. + + - Logging framework backends. TODO: Not yet. + + - Input framework readers. TODO: Not yet. + +A plugin's functionality is available to the user just as if Bro had +the corresponding code built-in. Indeed, internally many of Bro's +pieces are structured as plugins as well, they are just statically +compiled into the binary rather than loaded dynamically at runtime, as +external plugins are. + +Quick Start +=========== + +Writing a basic plugin is quite straight-forward as long as one +follows a few conventions. In the following we walk through adding a +new built-in function (bif) to Bro; we'll add `a `rot13(s: string) : +string``, a function that rotates every character in a string by 13 +places. + +A plugin comes in the form of a directory following a certain +structure. To get started, Bro's distribution provides a helper script +``aux/bro-aux/plugin-support/init-plugin`` that creates a skeleton +plugin that can then be customized. Let's use that:: + + # mkdir rot13-plugin + # cd rot13-plugin + # init-plugin Demo Rot13 + +As you can see the script takes two arguments. The first is a +namespace the plugin will live in, and the second a descriptive name +for the plugin itself. Bro uses the combination of the two to identify +a plugin. The namespace serves to avoid naming conflicts between +plugins written by independent developers; pick, e.g., the name of +your organisation (and note that the namespace ``Bro`` is reserved for +functionality distributed by the Bro Project). In our example, the +plugin will be called ``Demo::Rot13``. + +The ``init-plugin`` script puts a number of files in place. The full +layout is described later. For now, all we need is +``src/functions.bif``. It's initially empty, but we'll add our new bif +there as follows:: + + # cat scripts/functions.bif + module CaesarCipher; + + function camel_case%(s: string%) : string + %{ + char* rot13 = copy_string(s->CheckString()); + + for ( char* p = rot13; *p; p++ ) + { + char b = islower(*p) ? 'a' : 'A'; + *p = (*p - b + 13) % 26 + b; + } + + return new StringVal(strlen(rot13), rot13); + %} + +The syntax of this file is just like any other ``*.bif`` file; we +won't go into it here. + +Now we can already compile our plugin, we just need to tell the +Makefile put in place by ``init-plugin`` where the Bro source tree is +located (Bro needs to have been built there first):: + + # make BRO=/path/to/bro/dist + [... cmake output ...] + +Now our ``rot13-plugin`` directory has everything that it needs +for Bro to recognize it as a dynamic plugin. Once we point Bro to it, +it will pull it in automatically, as we can check with the ``-N`` +option: + + # export BRO_PLUGIN_PATH=/path/to/rot13-plugin + # bro -N + [...] + Plugin: Demo::Rot13 - (dynamic, version 1) + [...] + +That looks quite good, except for the dummy description that we should +replace with something nicer so that users will know what our plugin +is about. We do this by editing the ``config.description`` line in +``src/Plugin.cc``, like this: + + [...] + plugin::Configuration Configure() + { + plugin::Configuration config; + config.name = "Demo::Rot13"; + config.description = "Caesar cipher rotating a string's characters by 13 places."; + config.version.major = 1; + config.version.minor = 0; + return config; + } + [...] + + # make + [...] + # bro -N | grep Rot13 + Plugin: Demo::Rot13 - Caesar cipher rotating a string's characters by 13 places. (dynamic, version 1) + +Better. Bro can also show us what exactly the plugin provides with the +more verbose option ``-NN``:: + + # bro -NN + [...] + Plugin: Demo::Rot13 - Caesar cipher rotating a string's characters by 13 places. (dynamic, version 1) + [Function] CaesarCipher::rot13 + [...] + +There's our function. Now let's use it:: + + # bro -e 'print CaesarCipher::rot13("Hello")' + Uryyb + +It works. We next install the plugin along with Bro itself, so that it +will find it directly without needing the ``BRO_PLUGIN_PATH`` +environment variable. If we first unset the variable, the function +will no longer be available:: + + # unset BRO_PLUGIN_PATH + # bro -e 'print CaesarCipher::rot13("Hello")' + error in , line 1: unknown identifier CaesarCipher::rot13, at or near "CaesarCipher::rot13" + +Once we install it, it works again:: + + # make install + # bro -e 'print CaesarCipher::rot13("Hello")' + Uryyb + +The installed version went into +``/lib/bro/plugins/Demo_Rot13``. + +We can distribute the plugin in either source or binary form by using +the Makefile's ``sdist`` and ``bdist`` target, respectively. Both +create corrsponding tarballs:: + + # make sdist + [...] + Source distribution in build/sdist/Demo_Rot13.tar.gz + + # make bdist + [...] + Binary distribution in build/Demo_Rot13-darwin-x86_64.tar.gz + +The source archive will contain everything in the plugin directory +except any generated files. The binary archive will contain anything +needed to install and run the plugin, i.e., just what ``make install`` +puts into place as well. As the binary distribution is +platform-dependent, its name includes the OS and architecture the +plugin was built on. + +Plugin Directory Layout +======================= + +A plugin's directory needs to follow a set of conventions so that Bro +(1) recognizes it as a plugin, and (2) knows what to load. While +``init-plugin`` takes care of most of this, the following is the full +story. We'll use ```` to represent a plugin's top-level +directory. + +``/__bro_plugin__`` + A file that marks a directory as containing a Bro plugin. The file + must exist, and its content must consist of a single line with the + qualified name of the plugin (e.g., "Demo::Rot13"). + +``/lib/--.so`` + The shared library containing the plugin's compiled code. Bro will + load this in dynamically at run-time if OS and architecture match + the current platform. + +``lib/bif/`` + Directory with auto-generated Bro scripts that declare the plugins + bif elements. The files here are produced by ``bifcl``. + +``scripts/`` + A directory with the plugin's custom Bro scripts. When the plugin + gets activated, this directory will be automatically added to + ``BROPATH``, so that any scripts/modules inside can be + ``@load``ed. + +``scripts``/__load__.bro + A Bro script that will be loaded immediately when the plugin gets + activated. See below for more information on activating plugins. + +By convention, a plugin should put its custom scripts into sub folders +of ``scripts/``, i.e., ``scripts//