Integrate libmagic 5.16 via CMake ExternalProject.

- Bumps CMake requirement to CMake 2.8.
- Bro now always relies on builtin/shipped magic library/database.

CMakeLists.txt

@@ -1,5 +1,5 @@
 project(Bro C CXX)
-cmake_minimum_required(VERSION 2.6.3 FATAL_ERROR)
+cmake_minimum_required(VERSION 2.8.0 FATAL_ERROR)
 include(cmake/CommonCMakeConfig.cmake)
 
 ########################################################################
@@ -39,6 +39,26 @@ set(VERSION_MAJ_MIN "${VERSION_MAJOR}.${VERSION_MINOR}")
 ########################################################################
 ## Dependency Configuration
 
+include(ExternalProject)
+
+set(LIBMAGIC_PREFIX ${CMAKE_CURRENT_BINARY_DIR}/libmagic-prefix)
+set(LIBMAGIC_INCLUDE_DIR ${LIBMAGIC_PREFIX}/include)
+set(LIBMAGIC_LIB_DIR ${LIBMAGIC_PREFIX}/lib)
+set(LIBMAGIC_LIBRARY ${LIBMAGIC_LIB_DIR}/libmagic.a)
+ExternalProject_Add(libmagic
+    PREFIX ${LIBMAGIC_PREFIX}
+    URL ${CMAKE_CURRENT_SOURCE_DIR}/src/3rdparty/file-5.16.tar.gz
+    CONFIGURE_COMMAND ./configure --enable-static --disable-shared
+                                  --prefix=${LIBMAGIC_PREFIX}
+                                  --includedir=${LIBMAGIC_INCLUDE_DIR}
+                                  --libdir=${LIBMAGIC_LIB_DIR}
+    BUILD_IN_SOURCE 1
+    LOG_DOWNLOAD 1
+    LOG_CONFIGURE 1
+    LOG_BUILD 1
+    LOG_INSTALL 1
+)
+
 include(FindRequiredPackage)
 
 # Check cache value first to avoid displaying "Found sed" messages everytime
@@ -57,7 +77,6 @@ FindRequiredPackage(BISON)
 FindRequiredPackage(PCAP)
 FindRequiredPackage(OpenSSL)
 FindRequiredPackage(BIND)
-FindRequiredPackage(LibMagic)
 FindRequiredPackage(ZLIB)
 
 if (NOT BinPAC_ROOT_DIR AND
@@ -73,18 +92,12 @@ if (MISSING_PREREQS)
     message(FATAL_ERROR "Configuration aborted due to missing prerequisites")
 endif ()
 
-set(libmagic_req 5.04)
-if ( LibMagic_VERSION VERSION_LESS ${libmagic_req} )
-    message(FATAL_ERROR "libmagic of at least version ${libmagic_req} required "
-                        "(found ${LibMagic_VERSION})")
-endif ()
-
 include_directories(BEFORE
                     ${PCAP_INCLUDE_DIR}
                     ${OpenSSL_INCLUDE_DIR}
                     ${BIND_INCLUDE_DIR}
                     ${BinPAC_INCLUDE_DIR}
-                    ${LibMagic_INCLUDE_DIR}
+                    ${LIBMAGIC_INCLUDE_DIR}
                     ${ZLIB_INCLUDE_DIR}
 )
 
@@ -163,7 +176,7 @@ set(brodeps
     ${PCAP_LIBRARY}
     ${OpenSSL_LIBRARIES}
     ${BIND_LIBRARY}
-    ${LibMagic_LIBRARY}
+    ${LIBMAGIC_LIBRARY}
     ${ZLIB_LIBRARY}
     ${OPTLIBS}
 )

doc/install/install.rst

@@ -29,14 +29,13 @@ before you begin:
     * Libpcap                           (http://www.tcpdump.org)
     * OpenSSL libraries                 (http://www.openssl.org)
     * BIND8 library
-    * Libmagic 5.04 or greater
     * Libz
     * Bash (for BroControl)
     * Python (for BroControl)
 
 To build Bro from source, the following additional dependencies are required:
 
-    * CMake 2.6.3 or greater            (http://www.cmake.org)
+    * CMake 2.8.0 or greater            (http://www.cmake.org)
     * Make
     * C/C++ compiler
     * SWIG                              (http://www.swig.org)
@@ -44,7 +43,6 @@ To build Bro from source, the following additional dependencies are required:
     * Flex  (Fast Lexical Analyzer)
     * Libpcap headers                   (http://www.tcpdump.org)
     * OpenSSL headers                   (http://www.openssl.org)
-    * libmagic headers
     * zlib headers
     * Perl
 
@@ -55,13 +53,13 @@ that ``bash`` and ``python`` are in your ``PATH``):
 
   .. console::
 
-     sudo yum install cmake make gcc gcc-c++ flex bison libpcap-devel openssl-devel python-devel swig zlib-devel file-devel
+     sudo yum install cmake make gcc gcc-c++ flex bison libpcap-devel openssl-devel python-devel swig zlib-devel
 
 * DEB/Debian-based Linux:
 
   .. console::
 
-     sudo apt-get install cmake make gcc g++ flex bison libpcap-dev libssl-dev python-dev swig zlib1g-dev libmagic-dev
+     sudo apt-get install cmake make gcc g++ flex bison libpcap-dev libssl-dev python-dev swig zlib1g-dev
 
 * FreeBSD:
 
@@ -78,15 +76,11 @@ that ``bash`` and ``python`` are in your ``PATH``):
   then going through its "Preferences..." -> "Downloads" menus to
   install the "Command Line Tools" component.
 
-  Lion (10.7) and Mountain Lion (10.8) come with all required
+  OS X comes with all required dependencies except for CMake_ and SWIG_.
-  dependencies except for CMake_, SWIG_, and ``libmagic``.
-
   Distributions of these dependencies can likely be obtained from your
   preferred Mac OS X package management system (e.g. MacPorts_, Fink_,
-  or Homebrew_).
+  or Homebrew_).  Specifically for MacPorts, the ``cmake``, ``swig``,
-
+  ``swig-python`` and packages provide the required dependencies.
-  Specifically for MacPorts, the ``cmake``, ``swig``,
-  ``swig-python`` and ``file`` packages provide the required dependencies.
 
 
 Optional Dependencies

magic

@@ -1 +1 @@
-Subproject commit e87fe13a7b776182ffc8c75076d42702f5c28fed
+Subproject commit a29b831d8ee63b7dbc64109e6dbd3f9aca6d5c82

src/3rdparty

@@ -1 +1 @@
-Subproject commit 12b5cb446c8128bb22e5cbd7baa7d53669539487
+Subproject commit 51911114309cb9ac957502b412dc585e94e8fcbd

src/CMakeLists.txt

@@ -388,6 +388,9 @@ install(TARGETS bro DESTINATION bin)
 set(BRO_EXE bro
     CACHE STRING "Bro executable binary" FORCE)
 
+# External libmagic project must be built before bro.
+add_dependencies(bro libmagic)
+
 # Target to create all the autogenerated files.
 add_custom_target(generate_outputs_stage1)
 add_dependencies(generate_outputs_stage1 ${bro_ALL_GENERATED_OUTPUTS})

src/util.cc

@@ -1649,48 +1649,28 @@ void operator delete[](void* v)
 
 #endif
 
-// Being selective of which components of MAGIC_NO_CHECK_BUILTIN are actually
-// known to be problematic, but keeping rest of libmagic's builtin checks.
-#define DISABLE_LIBMAGIC_BUILTIN_CHECKS  ( \
-/*  MAGIC_NO_CHECK_COMPRESS | */ \
-/*  MAGIC_NO_CHECK_TAR  | */ \
-/*  MAGIC_NO_CHECK_SOFT | */ \
-/*  MAGIC_NO_CHECK_APPTYPE  | */ \
-/*  MAGIC_NO_CHECK_ELF  | */ \
-/*  MAGIC_NO_CHECK_TEXT | */ \
-    MAGIC_NO_CHECK_CDF  | \
-    MAGIC_NO_CHECK_TOKENS  \
-/*  MAGIC_NO_CHECK_ENCODING */ \
-)
-
 void bro_init_magic(magic_t* cookie_ptr, int flags)
 	{
 	if ( ! cookie_ptr || *cookie_ptr )
 		return;
 
-	*cookie_ptr = magic_open(flags|DISABLE_LIBMAGIC_BUILTIN_CHECKS);
+	*cookie_ptr = magic_open(flags);
 
-	// Use our custom database for mime types, but the default database
+	// Always use Bro's custom magic database.
-	// from libmagic for the verbose file type.
+	const char* database = bro_magic_path();
-	const char* database = (flags & MAGIC_MIME) ? bro_magic_path() : 0;
 
 	if ( ! *cookie_ptr )
 		{
 		const char* err = magic_error(*cookie_ptr);
-		if ( ! err )
+		reporter->InternalError("can't init libmagic: %s",
-			err = "unknown";
+		                        err ? err : "unknown");
-
-		reporter->InternalError("can't init libmagic: %s", err);
 		}
 
 	else if ( magic_load(*cookie_ptr, database) < 0 )
 		{
 		const char* err = magic_error(*cookie_ptr);
-		if ( ! err )
+		reporter->InternalError("can't load magic file %s: %s", database,
-			err = "unknown";
+		                        err ? err : "unknown");
-
-		const char* db_name = database ? database : "<default>";
-		reporter->InternalError("can't load magic file %s: %s", db_name, err);
 		magic_close(*cookie_ptr);
 		*cookie_ptr = 0;
 		}

testing/btest/Baseline/core.tunnels.gtp.outer_ip_frag/http.log

@@ -6,5 +6,5 @@
 #open	2013-08-26-19-02-18
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	trans_depth	method	host	uri	referrer	user_agent	request_body_len	response_body_len	status_code	status_msg	info_code	info_msg	filename	tags	username	password	proxied	orig_fuids	orig_mime_types	resp_fuids	resp_mime_types
 #types	time	string	addr	port	addr	port	count	string	string	string	string	string	count	count	count	string	count	string	string	table[enum]	string	string	table[string]	vector[string]	vector[string]	vector[string]	vector[string]
-1333458850.375568	CjhGID4nQcgTWjvg4c	10.131.47.185	1923	79.101.110.141	80	1	GET	o-o.preferred.telekomrs-beg1.v2.lscache8.c.youtube.com	/videoplayback?upn=MTU2MDY5NzQ5OTM0NTI3NDY4NDc&sparams=algorithm,burst,cp,factor,id,ip,ipbits,itag,source,upn,expire&fexp=912300,907210&algorithm=throttle-factor&itag=34&ip=212.0.0.0&burst=40&sver=3&signature=832FB1042E20780CFCA77A4DB5EA64AC593E8627.D1166C7E8365732E52DAFD68076DAE0146E0AE01&source=youtube&expire=1333484980&key=yt1&ipbits=8&factor=1.25&cp=U0hSSFRTUl9NSkNOMl9MTVZKOjh5eEN2SG8tZF84&id=ebf1e932d4bd1286&cm2=1	http://s.ytimg.com/yt/swfbin/watch_as3-vflqrJwOA.swf	Mozilla/5.0 (Windows NT 5.1) AppleWebKit/535.11 (KHTML, like Gecko; X-SBLSP) Chrome/17.0.963.83 Safari/535.11	0	56320	206	Partial Content	-	-	-	(empty)	-	-	-	-	-	FNJkBA1b8FSHt5N8jl	application/octet-stream
+1333458850.375568	CjhGID4nQcgTWjvg4c	10.131.47.185	1923	79.101.110.141	80	1	GET	o-o.preferred.telekomrs-beg1.v2.lscache8.c.youtube.com	/videoplayback?upn=MTU2MDY5NzQ5OTM0NTI3NDY4NDc&sparams=algorithm,burst,cp,factor,id,ip,ipbits,itag,source,upn,expire&fexp=912300,907210&algorithm=throttle-factor&itag=34&ip=212.0.0.0&burst=40&sver=3&signature=832FB1042E20780CFCA77A4DB5EA64AC593E8627.D1166C7E8365732E52DAFD68076DAE0146E0AE01&source=youtube&expire=1333484980&key=yt1&ipbits=8&factor=1.25&cp=U0hSSFRTUl9NSkNOMl9MTVZKOjh5eEN2SG8tZF84&id=ebf1e932d4bd1286&cm2=1	http://s.ytimg.com/yt/swfbin/watch_as3-vflqrJwOA.swf	Mozilla/5.0 (Windows NT 5.1) AppleWebKit/535.11 (KHTML, like Gecko; X-SBLSP) Chrome/17.0.963.83 Safari/535.11	0	56320	206	Partial Content	-	-	-	(empty)	-	-	-	-	-	FNJkBA1b8FSHt5N8jl	binary
 #close	2013-08-26-19-02-18

testing/btest/Baseline/scripts.base.frameworks.file-analysis.bifs.set_timeout_interval/bro..stdout

@@ -11,7 +11,7 @@ source: HTTP
 FILE_NEW
 file #1, 0, 0
 MIME_TYPE
-application/octet-stream
+binary
 FILE_OVER_NEW_CONNECTION
 FILE_TIMEOUT
 FILE_TIMEOUT

testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.partial-content/b.out

@@ -11,7 +11,7 @@ source: HTTP
 FILE_NEW
 file #1, 0, 0
 MIME_TYPE
-application/octet-stream
+binary
 FILE_OVER_NEW_CONNECTION
 FILE_TIMEOUT
 FILE_STATE_REMOVE

testing/btest/Baseline/scripts.base.frameworks.file-analysis.irc/out

@@ -10,7 +10,7 @@ file #1, 0, 0
 FILE_BOF_BUFFER
 \0\0^Ex\0\0^J\xf0\0\0^P
 MIME_TYPE
-application/octet-stream
+binary
 FILE_OVER_NEW_CONNECTION
 FILE_STATE_REMOVE
 file #1, 124, 0