Replace libmagic w/ Bro signatures for file MIME type identification.

Notable changes:

- libmagic is no longer used at all.  All MIME type detection is
  done through new Bro signatures, and there's no longer a means to get
  verbose file type descriptions (e.g. "PNG image data, 1435 x 170").
  The majority of the default file magic signatures are derived
  from the default magic database of libmagic ~5.17.

- File magic signatures consist of two new constructs in the
  signature rule parsing grammar: "file-magic" gives a regular
  expression to match against, and "file-mime" gives the MIME type
  string of content that matches the magic and an optional strength
  value for the match.

- Modified signature/rule syntax for identifiers: they can no longer
  start with a '-', which made for ambiguous syntax when doing negative
  strength values in "file-mime".  Also brought syntax for Bro script
  identifiers in line with reality (they can't start with numbers or
  include '-' at all).

- A new Built-In Function, "file_magic", can be used to get all
  file magic matches and their corresponding strength against a given
  chunk of data

- The second parameter of the "identify_data" Built-In Function
  can no longer be used to get verbose file type descriptions, though it
  can still be used to get the strongest matching file magic signature.

- The "file_transferred" event's "descr" parameter no longer
  contains verbose file type descriptions.

- The BROMAGIC environment variable no longer changes any behavior
  in Bro as magic databases are no longer used/installed.

- Reverted back to minimum requirement of CMake 2.6.3 from 2.8.0
  (it's back to being the same requirement as the Bro v2.2 release).
  The bump was to accomodate building libmagic as an external project,
  which is no longer needed.

Addresses BIT-1143.

src/RuleMatcher.cc

@@ -186,6 +186,15 @@ RuleEndpointState::~RuleEndpointState()
 		delete matched_text[j];
 	}
 
+RuleFileMagicState::~RuleFileMagicState()
+	{
+	loop_over_list(matchers, i)
+		{
+		delete matchers[i]->state;
+		delete matchers[i];
+		}
+	}
+
 RuleMatcher::RuleMatcher(int arg_RE_level)
 	{
 	root = new RuleHdrTest(RuleHdrTest::NOPROT, 0, 0, RuleHdrTest::EQ,
@@ -564,6 +573,127 @@ static inline bool compare(const vector<IPPrefix>& prefixes, const IPAddr& a,
 	return false;
 	}
 
+RuleFileMagicState* RuleMatcher::InitFileMagic() const
+	{
+	RuleFileMagicState* state = new RuleFileMagicState();
+
+	if ( rule_bench == 3 )
+		return state;
+
+	loop_over_list(root->psets[Rule::FILE_MAGIC], i)
+		{
+		RuleHdrTest::PatternSet* set = root->psets[Rule::FILE_MAGIC][i];
+		assert(set->re);
+		RuleFileMagicState::Matcher* m = new RuleFileMagicState::Matcher;
+		m->state = new RE_Match_State(set->re);
+		state->matchers.append(m);
+		}
+
+	// Save some memory.
+	state->matchers.resize(0);
+	return state;
+	}
+
+RuleMatcher::MIME_Matches* RuleMatcher::Match(RuleFileMagicState* state,
+                                              const u_char* data, uint64 len,
+                                              MIME_Matches* rval) const
+	{
+	if ( ! rval )
+		rval = new MIME_Matches();
+
+	if ( ! state )
+		{
+		reporter->Warning("RuleFileMagicState not initialized yet.");
+		return rval;
+		}
+
+	if ( rule_bench >= 2 )
+		return rval;
+
+#ifdef DEBUG
+	if ( debug_logger.IsEnabled(DBG_RULES) )
+		{
+		const char* s = fmt_bytes(reinterpret_cast<const char*>(data),
+		                          min(40, static_cast<int>(len)));
+		DBG_LOG(DBG_RULES, "Matching %s rules on |%s%s|",
+		        Rule::TypeToString(Rule::FILE_MAGIC), s,
+		        len > 40 ? "..." : "");
+		}
+#endif
+
+	bool newmatch = false;
+
+	loop_over_list(state->matchers, x)
+		{
+		RuleFileMagicState::Matcher* m = state->matchers[x];
+
+		if ( m->state->Match(data, len, true, false, true) )
+			newmatch = true;
+		}
+
+	if ( ! newmatch )
+		return rval;
+
+	DBG_LOG(DBG_RULES, "New pattern match found");
+
+	AcceptingSet accepted;
+	int_list matchpos;
+
+	loop_over_list(state->matchers, y)
+		{
+		RuleFileMagicState::Matcher* m = state->matchers[y];
+		const AcceptingSet* ac = m->state->Accepted();
+
+		loop_over_list(*ac, k)
+			{
+			if ( ! accepted.is_member((*ac)[k]) )
+				{
+				accepted.append((*ac)[k]);
+				matchpos.append((*m->state->MatchPositions())[k]);
+				}
+			}
+		}
+
+	// Find rules for which patterns have matched.
+	rule_list matched;
+
+	loop_over_list(accepted, i)
+		{
+		Rule* r = Rule::rule_table[accepted[i] - 1];
+
+		DBG_LOG(DBG_RULES, "Checking rule: %v", r->id);
+
+		loop_over_list(r->patterns, j)
+			{
+			if ( ! accepted.is_member(r->patterns[j]->id) )
+				continue;
+
+			if ( (unsigned int) matchpos[i] >
+			     r->patterns[j]->offset + r->patterns[j]->depth )
+				continue;
+
+			DBG_LOG(DBG_RULES, "All patterns of rule satisfied");
+			}
+
+		if ( ! matched.is_member(r) )
+			matched.append(r);
+		}
+
+	loop_over_list(matched, j)
+		{
+		Rule* r = matched[j];
+
+		loop_over_list(r->actions, rai)
+			{
+			const RuleActionMIME* ram = dynamic_cast<const RuleActionMIME*>(r->actions[rai]);
+			set<string>& ss = (*rval)[ram->GetStrength()];
+			ss.insert(ram->GetMIME());
+			}
+		}
+
+	return rval;
+	}
+
 RuleEndpointState* RuleMatcher::InitEndpoint(analyzer::Analyzer* analyzer,
 						const IP_Hdr* ip, int caplen,
 						RuleEndpointState* opposite,
@@ -1010,6 +1140,15 @@ void RuleMatcher::ClearEndpointState(RuleEndpointState* state)
 		state->matchers[j]->state->Clear();
 	}
 
+void RuleMatcher::ClearFileMagicState(RuleFileMagicState* state) const
+	{
+	if ( rule_bench == 3 )
+		return;
+
+	loop_over_list(state->matchers, j)
+		state->matchers[j]->state->Clear();
+	}
+
 void RuleMatcher::PrintDebug()
 	{
 	loop_over_list(rules, i)