Merge remote-tracking branch 'origin/topic/jsiwek/gh-554-file-signature-optimizations'

* origin/topic/jsiwek/gh-554-file-signature-optimizations:
  GH-554: don't init PIA endpoint matchers if there's only file-magic
  GH-554: remove use of file magic in protocol-based signature logic

src/RuleMatcher.cc

@@ -205,6 +205,7 @@ RuleMatcher::RuleMatcher(int arg_RE_level)
 				new maskedvalue_list);
 	RE_level = arg_RE_level;
 	parse_error = false;
+	has_non_file_magic_rule = false;
 	}
 
 RuleMatcher::~RuleMatcher()
@@ -285,6 +286,25 @@ void RuleMatcher::BuildRulesTree()
 		if ( ! rule->Active() )
 			continue;
 
+		const auto& pats = rule->patterns;
+
+		if ( ! has_non_file_magic_rule )
+			{
+			if ( pats.length() > 0 )
+				{
+				for ( const auto& p : pats )
+					{
+					if ( p->type != Rule::FILE_MAGIC )
+						{
+						has_non_file_magic_rule = true;
+						break;
+						}
+					}
+				}
+			else
+				has_non_file_magic_rule = true;
+			}
+
 		rule->SortHdrTests();
 		InsertRuleIntoTree(rule, 0, root, 0);
 		}
@@ -732,7 +752,7 @@ RuleEndpointState* RuleMatcher::InitEndpoint(analyzer::Analyzer* analyzer,
 		// pattern matching to do.
 		if ( hdr_test->level <= RE_level )
 			{
-			for ( int i = 0; i < Rule::TYPES; ++i )
+			for ( int i = Rule::PAYLOAD; i < Rule::TYPES; ++i )
 				{
 				for ( const auto& set : hdr_test->psets[i] )
 					{

src/RuleMatcher.h

@@ -286,6 +286,8 @@ public:
 	void AddRule(Rule* rule);
 	void SetParseError()		{ parse_error = true; }
 
+	bool HasNonFileMagicRule() const	{ return has_non_file_magic_rule; }
+
 	// Interface to for getting some statistics
 	struct Stats {
 		unsigned int matchers;	// # distinct RE matchers
@@ -356,6 +358,7 @@ private:
 	                                   const AcceptingMatchSet& ams);
 
 	int RE_level;
+	bool has_non_file_magic_rule;
 	bool parse_error;
 	RuleHdrTest* root;
 	rule_list rules;

src/analyzer/protocol/pia/PIA.cc

@@ -130,6 +130,9 @@ void PIA::DoMatch(const u_char* data, int len, bool is_orig, bool bol, bool eol,
 	if ( ! rule_matcher )
 		return;
 
+	if ( ! rule_matcher->HasNonFileMagicRule() )
+		return;
+
 	if ( ! MatcherInitialized(is_orig) )
 		InitEndpointMatcher(AsAnalyzer(), ip, len, is_orig, this);