diff --git a/CMakeLists.txt b/CMakeLists.txt index 88cee2ec29..7feaa4d1aa 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,5 +1,5 @@ project(Bro C CXX) -cmake_minimum_required(VERSION 2.6.3 FATAL_ERROR) +cmake_minimum_required(VERSION 2.8.0 FATAL_ERROR) include(cmake/CommonCMakeConfig.cmake) ######################################################################## @@ -39,6 +39,26 @@ set(VERSION_MAJ_MIN "${VERSION_MAJOR}.${VERSION_MINOR}") ######################################################################## ## Dependency Configuration +include(ExternalProject) + +set(LIBMAGIC_PREFIX ${CMAKE_CURRENT_BINARY_DIR}/libmagic-prefix) +set(LIBMAGIC_INCLUDE_DIR ${LIBMAGIC_PREFIX}/include) +set(LIBMAGIC_LIB_DIR ${LIBMAGIC_PREFIX}/lib) +set(LIBMAGIC_LIBRARY ${LIBMAGIC_LIB_DIR}/libmagic.a) +ExternalProject_Add(libmagic + PREFIX ${LIBMAGIC_PREFIX} + URL ${CMAKE_CURRENT_SOURCE_DIR}/src/3rdparty/file-5.16.tar.gz + CONFIGURE_COMMAND ./configure --enable-static --disable-shared + --prefix=${LIBMAGIC_PREFIX} + --includedir=${LIBMAGIC_INCLUDE_DIR} + --libdir=${LIBMAGIC_LIB_DIR} + BUILD_IN_SOURCE 1 + LOG_DOWNLOAD 1 + LOG_CONFIGURE 1 + LOG_BUILD 1 + LOG_INSTALL 1 +) + include(FindRequiredPackage) # Check cache value first to avoid displaying "Found sed" messages everytime @@ -57,7 +77,6 @@ FindRequiredPackage(BISON) FindRequiredPackage(PCAP) FindRequiredPackage(OpenSSL) FindRequiredPackage(BIND) -FindRequiredPackage(LibMagic) FindRequiredPackage(ZLIB) if (NOT BinPAC_ROOT_DIR AND @@ -73,18 +92,12 @@ if (MISSING_PREREQS) message(FATAL_ERROR "Configuration aborted due to missing prerequisites") endif () -set(libmagic_req 5.04) -if ( LibMagic_VERSION VERSION_LESS ${libmagic_req} ) - message(FATAL_ERROR "libmagic of at least version ${libmagic_req} required " - "(found ${LibMagic_VERSION})") -endif () - include_directories(BEFORE ${PCAP_INCLUDE_DIR} ${OpenSSL_INCLUDE_DIR} ${BIND_INCLUDE_DIR} ${BinPAC_INCLUDE_DIR} - ${LibMagic_INCLUDE_DIR} + ${LIBMAGIC_INCLUDE_DIR} ${ZLIB_INCLUDE_DIR} ) @@ -163,7 +176,7 @@ set(brodeps ${PCAP_LIBRARY} ${OpenSSL_LIBRARIES} ${BIND_LIBRARY} - ${LibMagic_LIBRARY} + ${LIBMAGIC_LIBRARY} ${ZLIB_LIBRARY} ${OPTLIBS} ) diff --git a/doc/install/install.rst b/doc/install/install.rst index 7532a82cea..3678d948c2 100644 --- a/doc/install/install.rst +++ b/doc/install/install.rst @@ -29,14 +29,13 @@ before you begin: * Libpcap (http://www.tcpdump.org) * OpenSSL libraries (http://www.openssl.org) * BIND8 library - * Libmagic 5.04 or greater * Libz * Bash (for BroControl) * Python (for BroControl) To build Bro from source, the following additional dependencies are required: - * CMake 2.6.3 or greater (http://www.cmake.org) + * CMake 2.8.0 or greater (http://www.cmake.org) * Make * C/C++ compiler * SWIG (http://www.swig.org) @@ -44,7 +43,6 @@ To build Bro from source, the following additional dependencies are required: * Flex (Fast Lexical Analyzer) * Libpcap headers (http://www.tcpdump.org) * OpenSSL headers (http://www.openssl.org) - * libmagic headers * zlib headers * Perl @@ -55,13 +53,13 @@ that ``bash`` and ``python`` are in your ``PATH``): .. console:: - sudo yum install cmake make gcc gcc-c++ flex bison libpcap-devel openssl-devel python-devel swig zlib-devel file-devel + sudo yum install cmake make gcc gcc-c++ flex bison libpcap-devel openssl-devel python-devel swig zlib-devel * DEB/Debian-based Linux: .. console:: - sudo apt-get install cmake make gcc g++ flex bison libpcap-dev libssl-dev python-dev swig zlib1g-dev libmagic-dev + sudo apt-get install cmake make gcc g++ flex bison libpcap-dev libssl-dev python-dev swig zlib1g-dev * FreeBSD: @@ -78,15 +76,11 @@ that ``bash`` and ``python`` are in your ``PATH``): then going through its "Preferences..." -> "Downloads" menus to install the "Command Line Tools" component. - Lion (10.7) and Mountain Lion (10.8) come with all required - dependencies except for CMake_, SWIG_, and ``libmagic``. - + OS X comes with all required dependencies except for CMake_ and SWIG_. Distributions of these dependencies can likely be obtained from your preferred Mac OS X package management system (e.g. MacPorts_, Fink_, - or Homebrew_). - - Specifically for MacPorts, the ``cmake``, ``swig``, - ``swig-python`` and ``file`` packages provide the required dependencies. + or Homebrew_). Specifically for MacPorts, the ``cmake``, ``swig``, + ``swig-python`` and packages provide the required dependencies. Optional Dependencies diff --git a/magic b/magic index e87fe13a7b..a29b831d8e 160000 --- a/magic +++ b/magic @@ -1 +1 @@ -Subproject commit e87fe13a7b776182ffc8c75076d42702f5c28fed +Subproject commit a29b831d8ee63b7dbc64109e6dbd3f9aca6d5c82 diff --git a/src/3rdparty b/src/3rdparty index 12b5cb446c..5191111430 160000 --- a/src/3rdparty +++ b/src/3rdparty @@ -1 +1 @@ -Subproject commit 12b5cb446c8128bb22e5cbd7baa7d53669539487 +Subproject commit 51911114309cb9ac957502b412dc585e94e8fcbd diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 8e22b504e4..c85b3b526f 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -388,6 +388,9 @@ install(TARGETS bro DESTINATION bin) set(BRO_EXE bro CACHE STRING "Bro executable binary" FORCE) +# External libmagic project must be built before bro. +add_dependencies(bro libmagic) + # Target to create all the autogenerated files. add_custom_target(generate_outputs_stage1) add_dependencies(generate_outputs_stage1 ${bro_ALL_GENERATED_OUTPUTS}) diff --git a/src/util.cc b/src/util.cc index ad55e3f75e..5a0b55a34a 100644 --- a/src/util.cc +++ b/src/util.cc @@ -1649,48 +1649,28 @@ void operator delete[](void* v) #endif -// Being selective of which components of MAGIC_NO_CHECK_BUILTIN are actually -// known to be problematic, but keeping rest of libmagic's builtin checks. -#define DISABLE_LIBMAGIC_BUILTIN_CHECKS ( \ -/* MAGIC_NO_CHECK_COMPRESS | */ \ -/* MAGIC_NO_CHECK_TAR | */ \ -/* MAGIC_NO_CHECK_SOFT | */ \ -/* MAGIC_NO_CHECK_APPTYPE | */ \ -/* MAGIC_NO_CHECK_ELF | */ \ -/* MAGIC_NO_CHECK_TEXT | */ \ - MAGIC_NO_CHECK_CDF | \ - MAGIC_NO_CHECK_TOKENS \ -/* MAGIC_NO_CHECK_ENCODING */ \ -) - void bro_init_magic(magic_t* cookie_ptr, int flags) { if ( ! cookie_ptr || *cookie_ptr ) return; - *cookie_ptr = magic_open(flags|DISABLE_LIBMAGIC_BUILTIN_CHECKS); + *cookie_ptr = magic_open(flags); - // Use our custom database for mime types, but the default database - // from libmagic for the verbose file type. - const char* database = (flags & MAGIC_MIME) ? bro_magic_path() : 0; + // Always use Bro's custom magic database. + const char* database = bro_magic_path(); if ( ! *cookie_ptr ) { const char* err = magic_error(*cookie_ptr); - if ( ! err ) - err = "unknown"; - - reporter->InternalError("can't init libmagic: %s", err); + reporter->InternalError("can't init libmagic: %s", + err ? err : "unknown"); } else if ( magic_load(*cookie_ptr, database) < 0 ) { const char* err = magic_error(*cookie_ptr); - if ( ! err ) - err = "unknown"; - - const char* db_name = database ? database : ""; - reporter->InternalError("can't load magic file %s: %s", db_name, err); + reporter->InternalError("can't load magic file %s: %s", database, + err ? err : "unknown"); magic_close(*cookie_ptr); *cookie_ptr = 0; } diff --git a/testing/btest/Baseline/core.tunnels.gtp.outer_ip_frag/http.log b/testing/btest/Baseline/core.tunnels.gtp.outer_ip_frag/http.log index 20408a08fe..e312a8e6ba 100644 --- a/testing/btest/Baseline/core.tunnels.gtp.outer_ip_frag/http.log +++ b/testing/btest/Baseline/core.tunnels.gtp.outer_ip_frag/http.log @@ -6,5 +6,5 @@ #open 2013-08-26-19-02-18 #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied orig_fuids orig_mime_types resp_fuids resp_mime_types #types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] vector[string] vector[string] vector[string] vector[string] -1333458850.375568 CjhGID4nQcgTWjvg4c 10.131.47.185 1923 79.101.110.141 80 1 GET o-o.preferred.telekomrs-beg1.v2.lscache8.c.youtube.com /videoplayback?upn=MTU2MDY5NzQ5OTM0NTI3NDY4NDc&sparams=algorithm,burst,cp,factor,id,ip,ipbits,itag,source,upn,expire&fexp=912300,907210&algorithm=throttle-factor&itag=34&ip=212.0.0.0&burst=40&sver=3&signature=832FB1042E20780CFCA77A4DB5EA64AC593E8627.D1166C7E8365732E52DAFD68076DAE0146E0AE01&source=youtube&expire=1333484980&key=yt1&ipbits=8&factor=1.25&cp=U0hSSFRTUl9NSkNOMl9MTVZKOjh5eEN2SG8tZF84&id=ebf1e932d4bd1286&cm2=1 http://s.ytimg.com/yt/swfbin/watch_as3-vflqrJwOA.swf Mozilla/5.0 (Windows NT 5.1) AppleWebKit/535.11 (KHTML, like Gecko; X-SBLSP) Chrome/17.0.963.83 Safari/535.11 0 56320 206 Partial Content - - - (empty) - - - - - FNJkBA1b8FSHt5N8jl application/octet-stream +1333458850.375568 CjhGID4nQcgTWjvg4c 10.131.47.185 1923 79.101.110.141 80 1 GET o-o.preferred.telekomrs-beg1.v2.lscache8.c.youtube.com /videoplayback?upn=MTU2MDY5NzQ5OTM0NTI3NDY4NDc&sparams=algorithm,burst,cp,factor,id,ip,ipbits,itag,source,upn,expire&fexp=912300,907210&algorithm=throttle-factor&itag=34&ip=212.0.0.0&burst=40&sver=3&signature=832FB1042E20780CFCA77A4DB5EA64AC593E8627.D1166C7E8365732E52DAFD68076DAE0146E0AE01&source=youtube&expire=1333484980&key=yt1&ipbits=8&factor=1.25&cp=U0hSSFRTUl9NSkNOMl9MTVZKOjh5eEN2SG8tZF84&id=ebf1e932d4bd1286&cm2=1 http://s.ytimg.com/yt/swfbin/watch_as3-vflqrJwOA.swf Mozilla/5.0 (Windows NT 5.1) AppleWebKit/535.11 (KHTML, like Gecko; X-SBLSP) Chrome/17.0.963.83 Safari/535.11 0 56320 206 Partial Content - - - (empty) - - - - - FNJkBA1b8FSHt5N8jl binary #close 2013-08-26-19-02-18 diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.bifs.set_timeout_interval/bro..stdout b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.bifs.set_timeout_interval/bro..stdout index e78f5c8c17..1cce5e77f0 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.bifs.set_timeout_interval/bro..stdout +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.bifs.set_timeout_interval/bro..stdout @@ -11,7 +11,7 @@ source: HTTP FILE_NEW file #1, 0, 0 MIME_TYPE -application/octet-stream +binary FILE_OVER_NEW_CONNECTION FILE_TIMEOUT FILE_TIMEOUT diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.partial-content/b.out b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.partial-content/b.out index 9c05f311f3..306fa3493c 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.partial-content/b.out +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.partial-content/b.out @@ -11,7 +11,7 @@ source: HTTP FILE_NEW file #1, 0, 0 MIME_TYPE -application/octet-stream +binary FILE_OVER_NEW_CONNECTION FILE_TIMEOUT FILE_STATE_REMOVE diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.irc/out b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.irc/out index fcd30b2253..2fde3f5073 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.irc/out +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.irc/out @@ -10,7 +10,7 @@ file #1, 0, 0 FILE_BOF_BUFFER \0\0^Ex\0\0^J\xf0\0\0^P MIME_TYPE -application/octet-stream +binary FILE_OVER_NEW_CONNECTION FILE_STATE_REMOVE file #1, 124, 0