Replace libmagic w/ Bro signatures for file MIME type identification.

Notable changes:

- libmagic is no longer used at all.  All MIME type detection is
  done through new Bro signatures, and there's no longer a means to get
  verbose file type descriptions (e.g. "PNG image data, 1435 x 170").
  The majority of the default file magic signatures are derived
  from the default magic database of libmagic ~5.17.

- File magic signatures consist of two new constructs in the
  signature rule parsing grammar: "file-magic" gives a regular
  expression to match against, and "file-mime" gives the MIME type
  string of content that matches the magic and an optional strength
  value for the match.

- Modified signature/rule syntax for identifiers: they can no longer
  start with a '-', which made for ambiguous syntax when doing negative
  strength values in "file-mime".  Also brought syntax for Bro script
  identifiers in line with reality (they can't start with numbers or
  include '-' at all).

- A new Built-In Function, "file_magic", can be used to get all
  file magic matches and their corresponding strength against a given
  chunk of data

- The second parameter of the "identify_data" Built-In Function
  can no longer be used to get verbose file type descriptions, though it
  can still be used to get the strongest matching file magic signature.

- The "file_transferred" event's "descr" parameter no longer
  contains verbose file type descriptions.

- The BROMAGIC environment variable no longer changes any behavior
  in Bro as magic databases are no longer used/installed.

- Reverted back to minimum requirement of CMake 2.6.3 from 2.8.0
  (it's back to being the same requirement as the Bro v2.2 release).
  The bump was to accomodate building libmagic as an external project,
  which is no longer needed.

Addresses BIT-1143.

.gitmodules

@@ -16,9 +16,6 @@
 [submodule "cmake"]
 	path = cmake
 	url = git://git.bro.org/cmake
-[submodule "magic"]
-	path = magic
-	url = git://git.bro.org/bromagic
 [submodule "src/3rdparty"]
 	path = src/3rdparty
 	url = git://git.bro.org/bro-3rdparty

CMakeLists.txt

@@ -1,5 +1,5 @@
 project(Bro C CXX)
-cmake_minimum_required(VERSION 2.8.0 FATAL_ERROR)
+cmake_minimum_required(VERSION 2.6.3 FATAL_ERROR)
 include(cmake/CommonCMakeConfig.cmake)
 
 ########################################################################
@@ -16,17 +16,12 @@ endif ()
 get_filename_component(BRO_SCRIPT_INSTALL_PATH ${BRO_SCRIPT_INSTALL_PATH}
     ABSOLUTE)
 
-set(BRO_MAGIC_INSTALL_PATH ${BRO_ROOT_DIR}/share/bro/magic)
-set(BRO_MAGIC_SOURCE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/magic/database)
-
 configure_file(bro-path-dev.in ${CMAKE_CURRENT_BINARY_DIR}/bro-path-dev)
 file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/bro-path-dev.sh
      "export BROPATH=`${CMAKE_CURRENT_BINARY_DIR}/bro-path-dev`\n"
-     "export BROMAGIC=\"${BRO_MAGIC_SOURCE_PATH}\"\n"
      "export PATH=\"${CMAKE_CURRENT_BINARY_DIR}/src\":$PATH\n")
 file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/bro-path-dev.csh
      "setenv BROPATH `${CMAKE_CURRENT_BINARY_DIR}/bro-path-dev`\n"
-     "setenv BROMAGIC \"${BRO_MAGIC_SOURCE_PATH}\"\n"
      "setenv PATH \"${CMAKE_CURRENT_BINARY_DIR}/src\":$PATH\n")
 
 file(STRINGS "${CMAKE_CURRENT_SOURCE_DIR}/VERSION" VERSION LIMIT_COUNT 1)
@@ -39,32 +34,6 @@ set(VERSION_MAJ_MIN "${VERSION_MAJOR}.${VERSION_MINOR}")
 ########################################################################
 ## Dependency Configuration
 
-include(ExternalProject)
-
-# LOG_* options to ExternalProject_Add appear in CMake 2.8.3.  If
-# available, using them hides external project configure/build output.
-if("${CMAKE_VERSION}" VERSION_GREATER 2.8.2)
-    set(EXTERNAL_PROJECT_LOG_OPTIONS
-        LOG_DOWNLOAD 1 LOG_UPDATE 1 LOG_CONFIGURE 1 LOG_BUILD 1 LOG_INSTALL 1)
-else()
-    set(EXTERNAL_PROJECT_LOG_OPTIONS)
-endif()
-
-set(LIBMAGIC_PREFIX ${CMAKE_CURRENT_BINARY_DIR}/libmagic-prefix)
-set(LIBMAGIC_INCLUDE_DIR ${LIBMAGIC_PREFIX}/include)
-set(LIBMAGIC_LIB_DIR ${LIBMAGIC_PREFIX}/lib)
-set(LIBMAGIC_LIBRARY ${LIBMAGIC_LIB_DIR}/libmagic.a)
-ExternalProject_Add(libmagic
-    PREFIX ${LIBMAGIC_PREFIX}
-    URL ${CMAKE_CURRENT_SOURCE_DIR}/src/3rdparty/file-5.17.tar.gz
-    CONFIGURE_COMMAND ./configure --enable-static --disable-shared
-                                  --prefix=${LIBMAGIC_PREFIX}
-                                  --includedir=${LIBMAGIC_INCLUDE_DIR}
-                                  --libdir=${LIBMAGIC_LIB_DIR}
-    BUILD_IN_SOURCE 1
-    ${EXTERNAL_PROJECT_LOG_OPTIONS}
-)
-
 include(FindRequiredPackage)
 
 # Check cache value first to avoid displaying "Found sed" messages everytime
@@ -103,7 +72,6 @@ include_directories(BEFORE
                     ${OpenSSL_INCLUDE_DIR}
                     ${BIND_INCLUDE_DIR}
                     ${BinPAC_INCLUDE_DIR}
-                    ${LIBMAGIC_INCLUDE_DIR}
                     ${ZLIB_INCLUDE_DIR}
 )
 
@@ -182,7 +150,6 @@ set(brodeps
     ${PCAP_LIBRARY}
     ${OpenSSL_LIBRARIES}
     ${BIND_LIBRARY}
-    ${LIBMAGIC_LIBRARY}
     ${ZLIB_LIBRARY}
     ${OPTLIBS}
 )
@@ -220,10 +187,6 @@ CheckOptionalBuildSources(aux/broctl   Broctl   INSTALL_BROCTL)
 CheckOptionalBuildSources(aux/bro-aux  Bro-Aux  INSTALL_AUX_TOOLS)
 CheckOptionalBuildSources(aux/broccoli Broccoli INSTALL_BROCCOLI)
 
-install(DIRECTORY ./magic/database/
-        DESTINATION ${BRO_MAGIC_INSTALL_PATH}
-)
-
 ########################################################################
 ## Packaging Setup
 

doc/CMakeLists.txt

@@ -14,8 +14,6 @@ if (NOT ${retval} EQUAL 0)
     message(FATAL_ERROR "Problem setting BROPATH")
 endif ()
 
-set(BROMAGIC ${BRO_MAGIC_SOURCE_PATH})
-
 # Configure the Sphinx config file (expand variables CMake might know about).
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/conf.py.in
                ${CMAKE_CURRENT_BINARY_DIR}/conf.py
@@ -34,7 +32,6 @@ add_custom_target(sphinxdoc
             ${CMAKE_CURRENT_SOURCE_DIR}/ ${SPHINX_INPUT_DIR}
     # Use Bro/Broxygen to dynamically generate reST for all Bro scripts.
     COMMAND BROPATH=${BROPATH}
-            BROMAGIC=${BROMAGIC}
             ${CMAKE_BINARY_DIR}/src/bro
             -X ${CMAKE_CURRENT_BINARY_DIR}/broxygen.conf
             broxygen >/dev/null

doc/frameworks/signatures.rst

@@ -64,8 +64,8 @@ expect that signature file in the same directory as the Bro script. The
 default extension of the file name is ``.sig``, and Bro appends that
 automatically when necessary.
 
-Signature language
+Signature Language for Network Traffic
-==================
+======================================
 
 Let's look at the format of a signature more closely. Each individual
 signature has the format ``signature <id> { <attributes> }``. ``<id>``
@@ -286,6 +286,44 @@ two actions defined:
     connection (``"http"``, ``"ftp"``, etc.). This is used by Bro's
     dynamic protocol detection to activate analyzers on the fly.
 
+Signature Language for File Content
+===================================
+
+The signature framework can also be used to identify MIME types of files
+irrespective of the network protocol/connection over which the file is
+transferred.  A special type of signature can be written for this
+purpose and will be used automatically by the :doc:`Files Framework
+<file-analysis>` or by Bro scripts that use the :bro:see:`file_magic`
+built-in function.
+
+Conditions
+----------
+
+File signatures use a single type of content condition in the form of a
+regular expression:
+
+``file-magic /<regular expression>/``
+
+This is analogous to the ``payload`` content condition for the network
+traffic signature language described above.  The difference is that
+``payload`` signatures are applied to payloads of network connections,
+but ``file-magic`` can be applied to any arbitrary data, it does not
+have to be tied to a network protocol/connection.
+
+Actions
+-------
+
+Upon matching a chunk of data, file signatures use the following action
+to get information about that data's MIME type:
+
+``file-mime <string> [, <integer>]``
+
+The arguments include the MIME type string associated with the file
+magic regular expression and an optional "strength" as a signed integer.
+Since multiple file magic signatures may match against a given chunk of
+data, the strength value may be used to help choose a "winner".  Higher
+values are considered stronger.
+
 Things to keep in mind when writing signatures
 ==============================================
 

doc/install/install.rst

@@ -35,7 +35,7 @@ before you begin:
 
 To build Bro from source, the following additional dependencies are required:
 
-    * CMake 2.8.0 or greater            (http://www.cmake.org)
+    * CMake 2.6.3 or greater            (http://www.cmake.org)
     * Make
     * C/C++ compiler
     * SWIG                              (http://www.swig.org)

magic

@@ -1 +0,0 @@
-Subproject commit 99c6b89230e2b9b0e781c42b0b9412d2ab4e14b2

scripts/base/frameworks/files/__load__.bro

@@ -1 +1,2 @@
 @load ./main.bro
+@load ./magic

scripts/base/frameworks/files/magic/__load__.bro

@@ -0,0 +1,2 @@
+@load-sigs ./general
+@load-sigs ./libmagic

scripts/base/frameworks/files/magic/general.sig

@@ -0,0 +1,11 @@
+# General purpose file magic signatures.
+
+signature file-plaintext {
+    file-magic /([[:print:][:space:]]+)/
+    file-mime "text/plain", -20
+}
+
+signature file-binary {
+    file-magic /(.*)([^[:print:][:space:]]+)/
+    file-mime "binary", -10
+}

scripts/base/init-bare.bro

@@ -60,6 +60,23 @@ type addr_vec: vector of addr;
 ##    directly and then remove this alias.
 type table_string_of_string: table[string] of string;
 
+## A structure indicating a MIME type and strength of a match against
+## file magic signatures.
+##
+## :bro:see:`file_magic`
+type mime_match: record {
+	strength: int;    ##< How strongly the signature matched.  Used for
+	                  ##< prioritization when multiple file magic signatures
+	                  ##< match.
+	mime:     string; ##< The MIME type of the file magic signature match.
+};
+
+## A vector of file magic signature matches, ordered by strength of
+## the signature, strongest first.
+##
+## :bro:see:`file_magic`
+type mime_matches: vector of mime_match;
+
 ## A connection's transport-layer protocol. Note that Bro uses the term
 ## "connection" broadly, using flow semantics for ICMP and UDP.
 type transport_proto: enum {

scripts/base/init-default.bro

@@ -58,6 +58,5 @@
 @load base/files/extract
 @load base/files/unified2
 
-
 @load base/misc/find-checksum-offloading
 @load base/misc/find-filtered-trace

src/CMakeLists.txt

@@ -387,9 +387,6 @@ install(TARGETS bro DESTINATION bin)
 set(BRO_EXE bro
     CACHE STRING "Bro executable binary" FORCE)
 
-# External libmagic project must be built before bro.
-add_dependencies(bro libmagic)
-
 # Target to create all the autogenerated files.
 add_custom_target(generate_outputs_stage1)
 add_dependencies(generate_outputs_stage1 ${bro_ALL_GENERATED_OUTPUTS})

src/NetVar.cc

@@ -20,6 +20,8 @@ TableType* string_set;
 TableType* string_array;
 TableType* count_set;
 VectorType* string_vec;
+VectorType* mime_matches;
+RecordType* mime_match;
 
 int watchdog_interval;
 
@@ -330,6 +332,8 @@ void init_net_var()
 	string_set = internal_type("string_set")->AsTableType();
 	string_array = internal_type("string_array")->AsTableType();
 	string_vec = internal_type("string_vec")->AsVectorType();
+	mime_match = internal_type("mime_match")->AsRecordType();
+	mime_matches = internal_type("mime_matches")->AsVectorType();
 
 	ignore_checksums = opt_internal_int("ignore_checksums");
 	partial_connection_ok = opt_internal_int("partial_connection_ok");

src/NetVar.h

@@ -23,6 +23,8 @@ extern TableType* string_set;
 extern TableType* string_array;
 extern TableType* count_set;
 extern VectorType* string_vec;
+extern VectorType* mime_matches;
+extern RecordType* mime_match;
 
 extern int watchdog_interval;
 

src/Rule.cc

@@ -38,7 +38,7 @@ Rule::~Rule()
 const char* Rule::TypeToString(Rule::PatternType type)
 	{
 	static const char* labels[] = {
-		"Payload", "HTTP-REQUEST", "HTTP-REQUEST-BODY",
+		"File Magic", "Payload", "HTTP-REQUEST", "HTTP-REQUEST-BODY",
 		"HTTP-REQUEST-HEADER", "HTTP-REPLY-BODY",
 		"HTTP-REPLY-HEADER", "FTP", "Finger",
 	};

src/Rule.h

@@ -37,7 +37,7 @@ public:
 	unsigned int Index() const	{ return idx; }
 
 	enum PatternType {
-		PAYLOAD, HTTP_REQUEST, HTTP_REQUEST_BODY, HTTP_REQUEST_HEADER,
+		FILE_MAGIC, PAYLOAD, HTTP_REQUEST, HTTP_REQUEST_BODY, HTTP_REQUEST_HEADER,
 		HTTP_REPLY_BODY, HTTP_REPLY_HEADER, FTP, FINGER, TYPES,
 	};
 

src/RuleAction.cc

@@ -35,6 +35,11 @@ void RuleActionEvent::PrintDebug()
 	fprintf(stderr, "	RuleActionEvent: |%s|\n", msg);
 	}
 
+void RuleActionMIME::PrintDebug()
+	{
+	fprintf(stderr, "	RuleActionMIME: |%s|\n", mime);
+	}
+
 RuleActionAnalyzer::RuleActionAnalyzer(const char* arg_analyzer)
 	{
 	string str(arg_analyzer);

src/RuleAction.h

@@ -36,6 +36,31 @@ private:
 	const char* msg;
 };
 
+class RuleActionMIME : public RuleAction {
+public:
+	RuleActionMIME(const char* arg_mime, int arg_strength = 0)
+		{ mime = copy_string(arg_mime); strength = arg_strength; }
+
+	virtual ~RuleActionMIME()
+		{ delete [] mime; }
+
+	virtual void DoAction(const Rule* parent, RuleEndpointState* state,
+	                      const u_char* data, int len)
+		{ }
+
+	virtual void PrintDebug();
+
+	string GetMIME() const
+		{ return mime; }
+
+	int GetStrength() const
+		{ return strength; }
+
+private:
+	const char* mime;
+	int strength;
+};
+
 // Base class for enable/disable actions.
 class RuleActionAnalyzer : public RuleAction {
 public:

src/RuleMatcher.cc

@@ -186,6 +186,15 @@ RuleEndpointState::~RuleEndpointState()
 		delete matched_text[j];
 	}
 
+RuleFileMagicState::~RuleFileMagicState()
+	{
+	loop_over_list(matchers, i)
+		{
+		delete matchers[i]->state;
+		delete matchers[i];
+		}
+	}
+
 RuleMatcher::RuleMatcher(int arg_RE_level)
 	{
 	root = new RuleHdrTest(RuleHdrTest::NOPROT, 0, 0, RuleHdrTest::EQ,
@@ -564,6 +573,127 @@ static inline bool compare(const vector<IPPrefix>& prefixes, const IPAddr& a,
 	return false;
 	}
 
+RuleFileMagicState* RuleMatcher::InitFileMagic() const
+	{
+	RuleFileMagicState* state = new RuleFileMagicState();
+
+	if ( rule_bench == 3 )
+		return state;
+
+	loop_over_list(root->psets[Rule::FILE_MAGIC], i)
+		{
+		RuleHdrTest::PatternSet* set = root->psets[Rule::FILE_MAGIC][i];
+		assert(set->re);
+		RuleFileMagicState::Matcher* m = new RuleFileMagicState::Matcher;
+		m->state = new RE_Match_State(set->re);
+		state->matchers.append(m);
+		}
+
+	// Save some memory.
+	state->matchers.resize(0);
+	return state;
+	}
+
+RuleMatcher::MIME_Matches* RuleMatcher::Match(RuleFileMagicState* state,
+                                              const u_char* data, uint64 len,
+                                              MIME_Matches* rval) const
+	{
+	if ( ! rval )
+		rval = new MIME_Matches();
+
+	if ( ! state )
+		{
+		reporter->Warning("RuleFileMagicState not initialized yet.");
+		return rval;
+		}
+
+	if ( rule_bench >= 2 )
+		return rval;
+
+#ifdef DEBUG
+	if ( debug_logger.IsEnabled(DBG_RULES) )
+		{
+		const char* s = fmt_bytes(reinterpret_cast<const char*>(data),
+		                          min(40, static_cast<int>(len)));
+		DBG_LOG(DBG_RULES, "Matching %s rules on |%s%s|",
+		        Rule::TypeToString(Rule::FILE_MAGIC), s,
+		        len > 40 ? "..." : "");
+		}
+#endif
+
+	bool newmatch = false;
+
+	loop_over_list(state->matchers, x)
+		{
+		RuleFileMagicState::Matcher* m = state->matchers[x];
+
+		if ( m->state->Match(data, len, true, false, true) )
+			newmatch = true;
+		}
+
+	if ( ! newmatch )
+		return rval;
+
+	DBG_LOG(DBG_RULES, "New pattern match found");
+
+	AcceptingSet accepted;
+	int_list matchpos;
+
+	loop_over_list(state->matchers, y)
+		{
+		RuleFileMagicState::Matcher* m = state->matchers[y];
+		const AcceptingSet* ac = m->state->Accepted();
+
+		loop_over_list(*ac, k)
+			{
+			if ( ! accepted.is_member((*ac)[k]) )
+				{
+				accepted.append((*ac)[k]);
+				matchpos.append((*m->state->MatchPositions())[k]);
+				}
+			}
+		}
+
+	// Find rules for which patterns have matched.
+	rule_list matched;
+
+	loop_over_list(accepted, i)
+		{
+		Rule* r = Rule::rule_table[accepted[i] - 1];
+
+		DBG_LOG(DBG_RULES, "Checking rule: %v", r->id);
+
+		loop_over_list(r->patterns, j)
+			{
+			if ( ! accepted.is_member(r->patterns[j]->id) )
+				continue;
+
+			if ( (unsigned int) matchpos[i] >
+			     r->patterns[j]->offset + r->patterns[j]->depth )
+				continue;
+
+			DBG_LOG(DBG_RULES, "All patterns of rule satisfied");
+			}
+
+		if ( ! matched.is_member(r) )
+			matched.append(r);
+		}
+
+	loop_over_list(matched, j)
+		{
+		Rule* r = matched[j];
+
+		loop_over_list(r->actions, rai)
+			{
+			const RuleActionMIME* ram = dynamic_cast<const RuleActionMIME*>(r->actions[rai]);
+			set<string>& ss = (*rval)[ram->GetStrength()];
+			ss.insert(ram->GetMIME());
+			}
+		}
+
+	return rval;
+	}
+
 RuleEndpointState* RuleMatcher::InitEndpoint(analyzer::Analyzer* analyzer,
 						const IP_Hdr* ip, int caplen,
 						RuleEndpointState* opposite,
@@ -1010,6 +1140,15 @@ void RuleMatcher::ClearEndpointState(RuleEndpointState* state)
 		state->matchers[j]->state->Clear();
 	}
 
+void RuleMatcher::ClearFileMagicState(RuleFileMagicState* state) const
+	{
+	if ( rule_bench == 3 )
+		return;
+
+	loop_over_list(state->matchers, j)
+		state->matchers[j]->state->Clear();
+	}
+
 void RuleMatcher::PrintDebug()
 	{
 	loop_over_list(rules, i)

src/RuleMatcher.h

@@ -3,6 +3,10 @@
 
 #include <limits.h>
 #include <vector>
+#include <map>
+#include <functional>
+#include <set>
+#include <string>
 
 #include "IPAddr.h"
 #include "BroString.h"
@@ -191,6 +195,31 @@ private:
 	int_list matched_rules;		// Rules for which all conditions have matched
 };
 
+/**
+ * A state object used for matching file magic signatures.
+ */
+class RuleFileMagicState {
+friend class RuleMatcher;
+public:
+
+	~RuleFileMagicState();
+
+private:
+
+	// Ctor is private; use RuleMatcher::InitFileMagic() for instantiation.
+	RuleFileMagicState()
+		{ }
+
+	struct Matcher {
+		RE_Match_State* state;
+	};
+
+	declare(PList, Matcher);
+	typedef PList(Matcher) matcher_list;
+
+	matcher_list matchers;
+};
+
 
 // RuleMatcher is the main class which builds up the data structures
 // and performs the actual matching.
@@ -205,6 +234,42 @@ public:
 	// Parse the given files and built up data structures.
 	bool ReadFiles(const name_list& files);
 
+	/**
+	 * Inititialize a state object for matching file magic signatures.
+	 * @return A state object that can be used for file magic mime type
+	 *         identification.
+	 */
+	RuleFileMagicState* InitFileMagic() const;
+
+	/**
+	 * Data structure containing a set of matching file magic signatures.
+	 * Ordered from greatest to least strength.  Matches of the same strength
+	 * will be in the set in lexicographic order of the MIME type string.
+	 */
+	typedef map<int, set<string>, std::greater<int> > MIME_Matches;
+
+	/**
+	 * Matches a chunk of data against file magic signatures.
+	 * @param state A state object previously returned from
+	 *              RuleMatcher::InitFileMagic()
+	 * @param data Chunk of data to match signatures against.
+	 * @param len Length of \a data in bytes.
+	 * @param matches An optional pre-existing match result object to
+	 *                modify with additional matches.  If it's a null
+	 *                pointer, one will be instantiated and returned from
+	 *                this method.
+	 * @return The results of the signature matching.
+	 */
+	MIME_Matches* Match(RuleFileMagicState* state, const u_char* data,
+	                   uint64 len, MIME_Matches* matches = 0) const;
+
+
+	/**
+	 * Resets a state object used with matching file magic signatures.
+	 * @param state The state object to reset to an initial condition.
+	 */
+	void ClearFileMagicState(RuleFileMagicState* state) const;
+
 	// Initialize the matching state for a endpoind of a connection based on
 	// the given packet (which should be the first packet encountered for
 	// this endpoint). If the matching is triggered by an PIA, a pointer to

src/analyzer/protocol/file/File.cc

@@ -3,6 +3,7 @@
 #include "File.h"
 
 #include "file_analysis/Manager.h"
+#include "RuleMatcher.h"
 #include "Reporter.h"
 #include "util.h"
 
@@ -48,14 +49,17 @@ void File_Analyzer::Done()
 
 void File_Analyzer::Identify()
 	{
-	const char* desc = bro_magic_buffer(magic_desc_cookie, buffer, buffer_len);
+	RuleFileMagicState* fms = rule_matcher->InitFileMagic();
-	const char* mime = bro_magic_buffer(magic_mime_cookie, buffer, buffer_len);
+	RuleMatcher::MIME_Matches matches;
-
+	rule_matcher->Match(fms, reinterpret_cast<const u_char*>(buffer),
+	                    buffer_len, &matches);
+	string match = matches.empty() ? "<unknown>"
+	                               : *(matches.begin()->second.begin());
 	val_list* vl = new val_list;
 	vl->append(BuildConnVal());
 	vl->append(new StringVal(buffer_len, buffer));
-	vl->append(new StringVal(desc ? desc : "<unknown>"));
+	vl->append(new StringVal("<unknown>"));
-	vl->append(new StringVal(mime ? mime : "<unknown>"));
+	vl->append(new StringVal(match));
 	ConnectionEvent(file_transferred, vl);
 	}
 

src/analyzer/protocol/file/events.bif

@@ -1,3 +1,12 @@
-## TODO.
+## Generated when a TCP connection associated w/ file data transfer is seen
+## typically (e.g. as happens w/ FTP or IRC).
 ##
+## c: The connection over which file data is transferred.
+##
+## prefix: Up to 1024 bytes of the file data.
+##
+## descr: Deprecated/unused argument.
+##
+## mime_type: MIME type of the file or "<unknown>" if no file magic signatures
+##            matched.
 event file_transferred%(c: connection, prefix: string, descr: string, mime_type: string%);

src/bro.bif

@@ -835,30 +835,66 @@ function syslog%(s: string%): any
 	return 0;
 	%}
 
-%%{
+## Determines the MIME type of a piece of data using Bro's file magic
-extern "C" {
+## signatures.
-#include <magic.h>
-}
-%%}
-
-## Determines the MIME type of a piece of data using ``libmagic``.
 ##
 ## data: The data to find the MIME type for.
 ##
-## return_mime: If true, the function returns a short MIME type string (e.g.,
+## return_mime: Deprecated argument; does nothing, except emit a warning
-##              ``text/plain`` instead of a more elaborate textual description).
+##              when false.
 ##
-## Returns: The MIME type of *data*, or "<unknown>" if there was an error.
+## Returns: The MIME type of *data*, or "<unknown>" if there was an error
-function identify_data%(data: string, return_mime: bool%): string
+##          or no match.  This is the strongest signature match.
+##
+## .. bro:see:: file_magic
+function identify_data%(data: string, return_mime: bool &default=T%): string
 	%{
-	magic_t* magic = return_mime ? &magic_mime_cookie : &magic_desc_cookie;
+	if ( ! return_mime )
+		reporter->Warning("identify_data() builtin-function only returns MIME types, but verbose file info requested");
 
-	if( ! *magic )
+	static RuleFileMagicState* fms = rule_matcher->InitFileMagic();
+	rule_matcher->ClearFileMagicState(fms);
+	RuleMatcher::MIME_Matches matches;
+	rule_matcher->Match(fms, data->Bytes(), data->Len(), &matches);
+
+	if ( matches.empty() )
 		return new StringVal("<unknown>");
 
-	const char* desc = bro_magic_buffer(*magic, data->Bytes(), data->Len());
+	return new StringVal(*(matches.begin()->second.begin()));
+	%}
 
-	return new StringVal(desc ? desc : "<unknown>");
+## Determines the MIME type of a piece of data using Bro's file magic
+## signatures.
+##
+## data: The data for which to find matching MIME types.
+##
+## Returns: All matching signatures, in order of strength.
+##
+## .. bro:see:: identify_data
+function file_magic%(data: string%): mime_matches
+	%{
+	static RuleFileMagicState* fms = rule_matcher->InitFileMagic();
+	rule_matcher->ClearFileMagicState(fms);
+	RuleMatcher::MIME_Matches matches;
+	rule_matcher->Match(fms, data->Bytes(), data->Len(), &matches);
+	VectorVal* rval = new VectorVal(mime_matches);
+
+	for ( RuleMatcher::MIME_Matches::const_iterator it = matches.begin();
+	      it != matches.end(); ++it )
+		{
+		RecordVal* element = new RecordVal(mime_match);
+
+		for ( set<string>::const_iterator it2 = it->second.begin();
+		      it2 != it->second.end(); ++it2 )
+			{
+			element->Assign(0, new Val(it->first, TYPE_INT));
+			element->Assign(1, new StringVal(*it2));
+			}
+
+		rval->Assign(rval->Size(), element);
+		}
+
+	return rval;
 	%}
 
 ## Performs an entropy test on the given data.

src/file_analysis/File.cc

@@ -10,6 +10,7 @@
 #include "Val.h"
 #include "Type.h"
 #include "Event.h"
+#include "RuleMatcher.h"
 
 #include "analyzer/Analyzer.h"
 #include "analyzer/Manager.h"
@@ -279,20 +280,17 @@ bool File::BufferBOF(const u_char* data, uint64 len)
 
 bool File::DetectMIME(const u_char* data, uint64 len)
 	{
-	const char* mime = bro_magic_buffer(magic_mime_cookie, data, len);
+	static RuleFileMagicState* fms = rule_matcher->InitFileMagic();
+	rule_matcher->ClearFileMagicState(fms);
+	RuleMatcher::MIME_Matches matches;
+	rule_matcher->Match(fms, data, len, &matches);
 
-	if ( mime )
+	if ( matches.empty() )
-		{
+		return false;
-		const char* mime_end = strchr(mime, ';');
 
-		if ( mime_end )
+	val->Assign(mime_type_idx, new StringVal(*matches.begin()->second.begin()));
-			// strip off charset
-			val->Assign(mime_type_idx, new StringVal(mime_end - mime, mime));
-		else
-			val->Assign(mime_type_idx, new StringVal(mime));
-		}
 
-	return mime;
+	return true;
 	}
 
 void File::ReplayBOF()

src/file_analysis/File.h

@@ -225,11 +225,12 @@ protected:
 	void ReplayBOF();
 
 	/**
-	 * Does mime type detection and assigns type (if available) to \c mime_type
+	 * Does mime type detection via file magic signatures and assigns
+	 * strongest matching mime type (if available) to \c mime_type
 	 * field in #val.
 	 * @param data pointer to a chunk of file data.
 	 * @param len number of bytes in the data chunk.
-	 * @return whether mime type was available.
+	 * @return whether a mime type match was found.
 	 */
 	bool DetectMIME(const u_char* data, uint64 len);
 

src/main.cc

@@ -23,7 +23,6 @@ extern "C" {
 #endif
 
 #include <openssl/md5.h>
-#include <magic.h>
 
 extern "C" void OPENSSL_add_all_algorithms_conf(void);
 
@@ -69,9 +68,6 @@ extern "C" void OPENSSL_add_all_algorithms_conf(void);
 
 Brofiler brofiler;
 
-magic_t magic_desc_cookie = 0;
-magic_t magic_mime_cookie = 0;
-
 #ifndef HAVE_STRSEP
 extern "C" {
 char* strsep(char**, const char*);
@@ -218,7 +214,6 @@ void usage()
 #endif
 
 	fprintf(stderr, "    $BROPATH                       | file search path (%s)\n", bro_path());
-	fprintf(stderr, "    $BROMAGIC                      | libmagic mime magic database search path (%s)\n", bro_magic_path());
 	fprintf(stderr, "    $BRO_PREFIXES                  | prefix list (%s)\n", bro_prefixes().c_str());
 	fprintf(stderr, "    $BRO_DNS_FAKE                  | disable DNS lookups (%s)\n", bro_dns_fake());
 	fprintf(stderr, "    $BRO_SEED_FILE                 | file to load seeds from (not set)\n");
@@ -778,9 +773,6 @@ int main(int argc, char** argv)
 	curl_global_init(CURL_GLOBAL_ALL);
 #endif
 
-	bro_init_magic(&magic_desc_cookie, MAGIC_NONE);
-	bro_init_magic(&magic_mime_cookie, MAGIC_MIME);
-
 	int r = sqlite3_initialize();
 
 	if ( r != SQLITE_OK )

src/rule-parse.y

@@ -34,6 +34,7 @@ static uint8_t mask_to_len(uint32_t mask)
 %token TOK_ENABLE
 %token TOK_EVAL
 %token TOK_EVENT
+%token TOK_MIME
 %token TOK_HEADER
 %token TOK_IDENT
 %token TOK_INT
@@ -61,9 +62,9 @@ static uint8_t mask_to_len(uint32_t mask)
 
 %type <str> TOK_STRING TOK_IDENT TOK_POLICY_SYMBOL TOK_PATTERN pattern string
 %type <val> TOK_INT TOK_TCP_STATE_SYM TOK_IP_OPTION_SYM TOK_COMP
-%type <val> integer ipoption_list tcpstate_list
+%type <val> integer ipoption_list tcpstate_list opt_strength
 %type <rule> rule
-%type <bl> TOK_BOOL
+%type <bl> TOK_BOOL opt_negate
 %type <hdr_test> hdr_expr
 %type <range> range rangeopt
 %type <vallist> value_list
@@ -186,6 +187,9 @@ rule_attr:
 	|	TOK_EVENT string
 			{ current_rule->AddAction(new RuleActionEvent($2)); }
 
+	|	TOK_MIME string opt_strength
+			{ current_rule->AddAction(new RuleActionMIME($2, $3)); }
+
 	|	TOK_ENABLE TOK_STRING
 			{ current_rule->AddAction(new RuleActionEnable($2)); }
 
@@ -359,6 +363,20 @@ integer:
 			{ $$ = id_to_uint($1); }
 	;
 
+opt_negate:
+		'-'
+			{ $$ = true; }
+	|
+			{ $$ = false; }
+	;
+
+opt_strength:
+		',' opt_negate TOK_INT
+			{ $$ = $2 ? -$3 : $3; }
+	|
+			{ $$ = 0; }
+	;
+
 string:
 		TOK_STRING
 			{ $$ = $1; }

src/rule-scan.l

@@ -21,11 +21,13 @@ D	[0-9]+
 H	[0-9a-fA-F]+
 HEX {H}
 STRING	\"([^\n\"]|\\\")*\"
-ID	([0-9a-zA-Z_-]+::)*[0-9a-zA-Z_-]+
+IDCOMPONENT [0-9a-zA-Z_][0-9a-zA-Z_-]*
+ID	{IDCOMPONENT}(::{IDCOMPONENT})*
 IP6	("["({HEX}:){7}{HEX}"]")|("["0x{HEX}({HEX}|:)*"::"({HEX}|:)*"]")|("["({HEX}|:)*"::"({HEX}|:)*"]")|("["({HEX}|:)*"::"({HEX}|:)*({D}"."){3}{D}"]")
 RE	\/(\\\/)?([^/]|[^\\]\\\/)*\/
 META	\.[^ \t]+{WS}[^\n]+
-PID	([0-9a-zA-Z_-]|"::")+
+PIDCOMPONENT [A-Za-z_][A-Za-z_0-9]*
+PID  {PIDCOMPONENT}(::{PIDCOMPONENT})*
 
 %option nounput nodefault
 
@@ -50,7 +52,7 @@ PID	([0-9a-zA-Z_-]|"::")+
 	return TOK_IP6;
 	}
 
-[!\]\[{}&:,]	return rules_text[0];
+[!\]\[{}&:,-]	return rules_text[0];
 
 "<="	{ rules_lval.val = RuleHdrTest::LE; return TOK_COMP; }
 ">="	{ rules_lval.val = RuleHdrTest::GE; return TOK_COMP; }
@@ -116,6 +118,7 @@ dst-port	return TOK_DST_PORT;
 enable		return TOK_ENABLE;
 eval		return TOK_EVAL;
 event		return TOK_EVENT;
+file-mime	return TOK_MIME;
 header		return TOK_HEADER;
 ip-options	return TOK_IP_OPTIONS;
 ip-proto	return TOK_IP_PROTO;
@@ -129,6 +132,7 @@ src-port	return TOK_SRC_PORT;
 tcp-state	return TOK_TCP_STATE;
 active		return TOK_ACTIVE;
 
+file-magic { rules_lval.val = Rule::FILE_MAGIC; return TOK_PATTERN_TYPE; }
 payload	{ rules_lval.val = Rule::PAYLOAD; return TOK_PATTERN_TYPE; }
 http-request	{ rules_lval.val = Rule::HTTP_REQUEST; return TOK_PATTERN_TYPE; }
 http-request-body	{ rules_lval.val = Rule::HTTP_REQUEST_BODY; return TOK_PATTERN_TYPE; }

src/util-config.h.in

@@ -1,2 +1 @@
 #define BRO_SCRIPT_INSTALL_PATH "@BRO_SCRIPT_INSTALL_PATH@"
-#define BRO_MAGIC_INSTALL_PATH "@BRO_MAGIC_INSTALL_PATH@"

src/util.cc

@@ -911,16 +911,6 @@ const char* bro_path()
 	return path;
 	}
 
-const char* bro_magic_path()
-	{
-	const char* path = getenv("BROMAGIC");
-
-	if ( ! path )
-		path = BRO_MAGIC_INSTALL_PATH;
-
-	return path;
-	}
-
 string bro_prefixes()
 	{
 	string rval;
@@ -1649,45 +1639,6 @@ void operator delete[](void* v)
 
 #endif
 
-void bro_init_magic(magic_t* cookie_ptr, int flags)
-	{
-	if ( ! cookie_ptr || *cookie_ptr )
-		return;
-
-	*cookie_ptr = magic_open(flags);
-
-	// Always use Bro's custom magic database.
-	const char* database = bro_magic_path();
-
-	if ( ! *cookie_ptr )
-		{
-		const char* err = magic_error(*cookie_ptr);
-		reporter->InternalError("can't init libmagic: %s",
-		                        err ? err : "unknown");
-		}
-
-	else if ( magic_load(*cookie_ptr, database) < 0 )
-		{
-		const char* err = magic_error(*cookie_ptr);
-		reporter->InternalError("can't load magic file %s: %s", database,
-		                        err ? err : "unknown");
-		magic_close(*cookie_ptr);
-		*cookie_ptr = 0;
-		}
-	}
-
-const char* bro_magic_buffer(magic_t cookie, const void* buffer, size_t length)
-	{
-	const char* rval = magic_buffer(cookie, buffer, length);
-	if ( ! rval )
-		{
-		const char* err = magic_error(cookie);
-		reporter->Error("magic_buffer error: %s", err ? err : "unknown");
-		}
-
-	return rval;
-	}
-
 const char* canonify_name(const char* name)
 	{
 	unsigned int len = strlen(name);

src/util.h

@@ -22,7 +22,6 @@
 #include <stdlib.h>
 #include <string.h>
 #include <stdarg.h>
-#include <magic.h>
 #include <libgen.h>
 #include "config.h"
 
@@ -483,12 +482,6 @@ struct CompareString
 		}
 	};
 
-extern magic_t magic_desc_cookie;
-extern magic_t magic_mime_cookie;
-
-void bro_init_magic(magic_t* cookie_ptr, int flags);
-const char* bro_magic_buffer(magic_t cookie, const void* buffer, size_t length);
-
 /**
  * Canonicalizes a name by converting it to uppercase letters and replacing
  * all non-alphanumeric characters with an underscore.

testing/btest/Baseline/bifs.identify_data/out

@@ -1,4 +1,2 @@
-ASCII text, with no line terminators
 text/plain
-PNG image data
 image/png

testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log

@@ -3,7 +3,7 @@
 #empty_field	(empty)
 #unset_field	-
 #path	loaded_scripts
-#open	2013-10-30-16-52-11
+#open	2014-03-03-20-45-31
 #fields	name
 #types	string
 scripts/base/init-bare.bro
@@ -94,6 +94,7 @@ scripts/base/init-bare.bro
       build/scripts/base/bif/file_analysis.bif.bro
       scripts/base/utils/site.bro
         scripts/base/utils/patterns.bro
+    scripts/base/frameworks/files/magic/__load__.bro
   build/scripts/base/bif/__load__.bro
     build/scripts/base/bif/bloom-filter.bif.bro
     build/scripts/base/bif/broxygen.bif.bro
@@ -101,4 +102,4 @@ scripts/base/init-bare.bro
     build/scripts/base/bif/top-k.bif.bro
 scripts/policy/misc/loaded-scripts.bro
   scripts/base/utils/paths.bro
-#close	2013-10-30-16-52-11
+#close	2014-03-03-20-45-31

testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log

@@ -3,7 +3,7 @@
 #empty_field	(empty)
 #unset_field	-
 #path	loaded_scripts
-#open	2014-01-31-22-54-38
+#open	2014-03-03-20-45-54
 #fields	name
 #types	string
 scripts/base/init-bare.bro
@@ -94,6 +94,7 @@ scripts/base/init-bare.bro
       build/scripts/base/bif/file_analysis.bif.bro
       scripts/base/utils/site.bro
         scripts/base/utils/patterns.bro
+    scripts/base/frameworks/files/magic/__load__.bro
   build/scripts/base/bif/__load__.bro
     build/scripts/base/bif/bloom-filter.bif.bro
     build/scripts/base/bif/broxygen.bif.bro
@@ -222,4 +223,4 @@ scripts/base/init-default.bro
   scripts/base/misc/find-checksum-offloading.bro
   scripts/base/misc/find-filtered-trace.bro
 scripts/policy/misc/loaded-scripts.bro
-#close	2014-01-31-22-54-38
+#close	2014-03-03-20-45-54

testing/btest/Baseline/doc.sphinx.file_extraction/btest-doc.sphinx.file_extraction#1

@@ -6,9 +6,9 @@
 
       # bro -r http/bro.org.pcap file_extraction.bro
       Extracting file HTTP-FiIpIB2hRQSDBOSJRg.html
+      Extracting file HTTP-FMG4bMmVV64eOsCb.txt
       Extracting file HTTP-FnaT2a3UDd093opCB9.txt
       Extracting file HTTP-FsvATF146kf1Emc21j.txt
       Extracting file HTTP-FkMQHg2nBr44fc5h63.txt
-      Extracting file HTTP-FfQGqj4Fhh3pH7nVQj.txt
       [...]
 

testing/btest/Baseline/doc.sphinx.mimestats/btest-doc.sphinx.mimestats#1

@@ -16,16 +16,16 @@
       #empty_field	(empty)
       #unset_field	-
       #path	mime_metrics
-      #open	2014-01-21-21-35-28
+      #open	2014-03-03-22-45-14
       #fields	ts	ts_delta	mtype	uniq_hosts	hits	bytes
       #types	time	interval	string	count	count	count
       1389719059.311698	300.000000	text/html	1	4	53070
       1389719059.311698	300.000000	image/jpeg	1	1	186859
-      1389719059.311698	300.000000	text/troff	1	1	2957
+      1389719059.311698	300.000000	text/troff	1	1	3180
       1389719059.311698	300.000000	application/pgp-signature	1	1	836
-      1389719059.311698	300.000000	text/plain	1	12	114205
+      1389719059.311698	300.000000	text/plain	1	12	113982
       1389719059.311698	300.000000	image/gif	1	1	172
       1389719059.311698	300.000000	image/png	1	9	82176
       1389719059.311698	300.000000	image/x-icon	1	2	2300
-      #close	2014-01-21-21-35-28
+      #close	2014-03-03-22-45-14
 

testing/btest/Baseline/scripts.base.frameworks.file-analysis.ftp/out

@@ -3,7 +3,7 @@ file #0, 0, 0
 FILE_BOF_BUFFER
 The Nationa
 MIME_TYPE
-text/x-pascal
+text/plain
 FILE_OVER_NEW_CONNECTION
 FILE_STATE_REMOVE
 file #0, 16557, 0

testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.partial-content/c.out

@@ -1,7 +1,7 @@
 FILE_NEW
 file #0, 0, 0
 MIME_TYPE
-application/octet-stream
+text/plain
 FILE_OVER_NEW_CONNECTION
 FILE_OVER_NEW_CONNECTION
 FILE_STATE_REMOVE

testing/btest/bifs/identify_data.bro

@@ -6,11 +6,9 @@ event bro_init()
 	{
 	# plain text
 	local a = "This is a test";
-	print identify_data(a, F);
 	print identify_data(a, T);
 
 	# PNG image
 	local b = "\x89\x50\x4e\x47\x0d\x0a\x1a\x0a\x00";
-	print identify_data(b, F);
 	print identify_data(b, T);
 	}