Merge remote-tracking branch 'origin/topic/jsiwek/bit-844'

BIT-844 #merged

* origin/topic/jsiwek/bit-844:
  Remove stale signature benchmarking code (-L command-line option).
  BIT-844: fix UDP payload signatures to match packet-wise

CHANGES

@@ -1,4 +1,12 @@
 
+2.3-685 | 2015-04-09 14:52:11 -0700
+
+  * Remove stale signature benchmarking code (-L command-line option).
+    (Jon Siwek)
+
+  * BIT-844: fix UDP payload signatures to match packet-wise. (Jon
+    Siwek)
+
 2.3-682 | 2015-04-09 12:07:00 -0700
 
   * Fixing input readers' component type. (Robin Sommer)

VERSION

@@ -1 +1 @@
-2.3-682
+2.3-685

aux/binpac

@@ -1 +1 @@
-Subproject commit ab50e5115bc0d217552a63f15382e45ed608f5fc
+Subproject commit 544330932e7cd4615d6d19f63907e8aa2acebb9e

src/RE.cc

@@ -20,9 +20,6 @@ int case_insensitive = 0;
 extern int RE_parse(void);
 extern void RE_set_input(const char* str);
 
-// If true, the set-wise matching always returns false - for benchmarking.
-extern int rule_bench;
-
 Specific_RE_Matcher::Specific_RE_Matcher(match_type arg_mt, int arg_multiline)
 : equiv_class(NUM_SYM)
 	{
@@ -279,9 +276,6 @@ inline void RE_Match_State::AddMatches(const AcceptingSet& as,
 bool RE_Match_State::Match(const u_char* bv, int n,
 				bool bol, bool eol, bool clear)
 	{
-	if ( rule_bench > 0 )
-		return false;
-
 	if ( current_pos == -1 )
 		{
 		// First call to Match().

src/RuleMatcher.cc

@@ -577,9 +577,6 @@ RuleFileMagicState* RuleMatcher::InitFileMagic() const
 	{
 	RuleFileMagicState* state = new RuleFileMagicState();
 
-	if ( rule_bench == 3 )
-		return state;
-
 	loop_over_list(root->psets[Rule::FILE_MAGIC], i)
 		{
 		RuleHdrTest::PatternSet* set = root->psets[Rule::FILE_MAGIC][i];
@@ -630,9 +627,6 @@ RuleMatcher::MIME_Matches* RuleMatcher::Match(RuleFileMagicState* state,
 		return rval;
 		}
 
-	if ( rule_bench >= 2 )
-		return rval;
-
 #ifdef DEBUG
 	if ( debug_logger.IsEnabled(DBG_RULES) )
 		{
@@ -712,9 +706,6 @@ RuleEndpointState* RuleMatcher::InitEndpoint(analyzer::Analyzer* analyzer,
 	RuleEndpointState* state =
 		new RuleEndpointState(analyzer, from_orig, opposite, pia);
 
-	if ( rule_bench == 3 )
-		return state;
-
 	rule_hdr_test_list tests;
 	tests.append(root);
 
@@ -837,9 +828,6 @@ void RuleMatcher::Match(RuleEndpointState* state, Rule::PatternType type,
 	// for 'accepted' (that depends on the average number of matching
 	// patterns).
 
-	if ( rule_bench >= 2 )
-		return;
-
 	bool newmatch = false;
 
 #ifdef DEBUG
@@ -956,9 +944,6 @@ void RuleMatcher::Match(RuleEndpointState* state, Rule::PatternType type,
 
 void RuleMatcher::FinishEndpoint(RuleEndpointState* state)
 	{
-	if ( rule_bench == 3 )
-		return;
-
 	// Send EOL to payload matchers.
 	Match(state, Rule::PAYLOAD, (const u_char *) "", 0, false, true, false);
 
@@ -1110,15 +1095,7 @@ void RuleMatcher::ExecRule(Rule* rule, RuleEndpointState* state, bool eos)
 
 void RuleMatcher::ClearEndpointState(RuleEndpointState* state)
 	{
-	if ( rule_bench == 3 )
-		return;
-
-	ExecPureRules(state, 1);
 	state->payload_size = -1;
-	state->matched_by_patterns.clear();
-	loop_over_list(state->matched_text, i)
-		delete state->matched_text[i];
-	state->matched_text.clear();
 
 	loop_over_list(state->matchers, j)
 		state->matchers[j]->state->Clear();
@@ -1126,9 +1103,6 @@ void RuleMatcher::ClearEndpointState(RuleEndpointState* state)
 
 void RuleMatcher::ClearFileMagicState(RuleFileMagicState* state) const
 	{
-	if ( rule_bench == 3 )
-		return;
-
 	loop_over_list(state->matchers, j)
 		state->matchers[j]->state->Clear();
 	}
@@ -1496,8 +1470,12 @@ void RuleMatcherState::ClearMatchState(bool orig)
 	if ( ! rule_matcher )
 		return;
 
+	if ( orig )
+		{
 		if ( orig_match_state )
 			rule_matcher->ClearEndpointState(orig_match_state);
-	if ( resp_match_state )
+		}
+
+	else if ( resp_match_state )
 		rule_matcher->ClearEndpointState(resp_match_state);
 	}

src/RuleMatcher.h

@@ -22,8 +22,6 @@
 
 //#define MATCHER_PRINT_STATS
 
-extern int rule_bench;
-
 // Parser interface:
 
 extern void rules_error(const char* msg);

src/analyzer/protocol/pia/PIA.cc

@@ -81,7 +81,7 @@ void PIA::PIA_Done()
 	}
 
 void PIA::PIA_DeliverPacket(int len, const u_char* data, bool is_orig, uint64 seq,
-				const IP_Hdr* ip, int caplen)
+				const IP_Hdr* ip, int caplen, bool clear_state)
 	{
 	if ( pkt_buffer.state == SKIPPING )
 		return;
@@ -108,6 +108,9 @@ void PIA::PIA_DeliverPacket(int len, const u_char* data, bool is_orig, uint64 se
 	// FIXME: I'm not sure why it does not work with eol=true...
 	DoMatch(data, len, is_orig, true, false, false, ip);
 
+	if ( clear_state )
+		RuleMatcherState::ClearMatchState(is_orig);
+
 	pkt_buffer.state = new_state;
 
 	current_packet.data = 0;

src/analyzer/protocol/pia/PIA.h

@@ -42,7 +42,7 @@ public:
 protected:
 	void PIA_Done();
 	void PIA_DeliverPacket(int len, const u_char* data, bool is_orig,
-				uint64 seq, const IP_Hdr* ip, int caplen);
+				uint64 seq, const IP_Hdr* ip, int caplen, bool clear_state);
 
 	enum State { INIT, BUFFERING, MATCHING_ONLY, SKIPPING } state;
 
@@ -109,7 +109,7 @@ protected:
 					uint64 seq, const IP_Hdr* ip, int caplen)
 		{
 		Analyzer::DeliverPacket(len, data, is_orig, seq, ip, caplen);
-		PIA_DeliverPacket(len, data, is_orig, seq, ip, caplen);
+		PIA_DeliverPacket(len, data, is_orig, seq, ip, caplen, true);
 		}
 
 	virtual void ActivateAnalyzer(analyzer::Tag tag, const Rule* rule);
@@ -154,7 +154,7 @@ protected:
 					uint64 seq, const IP_Hdr* ip, int caplen)
 		{
 		Analyzer::DeliverPacket(len, data, is_orig, seq, ip, caplen);
-		PIA_DeliverPacket(len, data, is_orig, seq, ip, caplen);
+		PIA_DeliverPacket(len, data, is_orig, seq, ip, caplen, false);
 		}
 
 	virtual void DeliverStream(int len, const u_char* data, bool is_orig);

src/main.cc

@@ -117,7 +117,6 @@ SampleLogger* sample_logger = 0;
 int signal_val = 0;
 int optimize = 0;
 int do_notice_analysis = 0;
-int rule_bench = 0;
 extern char version[];
 char* command_line_policy = 0;
 vector<string> params;
@@ -195,7 +194,6 @@ void usage()
 	fprintf(stderr, "    -F|--force-dns                 | force DNS\n");
 	fprintf(stderr, "    -I|--print-id <ID name>        | print out given ID\n");
 	fprintf(stderr, "    -K|--md5-hashkey <hashkey>     | set key for MD5-keyed hashing\n");
-	fprintf(stderr, "    -L|--rule-benchmark            | benchmark for rules\n");
 	fprintf(stderr, "    -N|--print-plugins             | print available plugins and exit (-NN for verbose)\n");
 	fprintf(stderr, "    -O|--optimize                  | optimize policy script\n");
 	fprintf(stderr, "    -P|--prime-dns                 | prime DNS\n");
@@ -503,7 +501,6 @@ int main(int argc, char** argv)
 		{"save-seeds",		required_argument,	0,	'H'},
 		{"set-seed",		required_argument,	0,	'J'},
 		{"md5-hashkey",		required_argument,	0,	'K'},
-		{"rule-benchmark",	no_argument,		0,	'L'},
 		{"print-plugins",	no_argument,		0,	'N'},
 		{"optimize",		no_argument,		0,	'O'},
 		{"prime-dns",		no_argument,		0,	'P'},
@@ -668,10 +665,6 @@ int main(int argc, char** argv)
 			hmac_key_set = 1;
 			break;
 
-		case 'L':
-			++rule_bench;
-			break;
-
 		case 'N':
 			++print_plugins;
 			break;

testing/btest/Baseline/signatures.udp-packetwise-match/out

@@ -0,0 +1,6 @@
+signature match, Found XXXX, XXXX
+signature match, Found ^XXXX, XXXX
+signature match, Found .*XXXX, XXXX
+signature match, Found YYYY, YYYY
+signature match, Found ^YYYY, YYYY
+signature match, Found .*YYYY, YYYY

testing/btest/signatures/udp-packetwise-match.bro

@@ -0,0 +1,53 @@
+# @TEST-EXEC: bro -r $TRACES/udp-signature-test.pcap %INPUT >out
+# @TEST-EXEC: btest-diff out
+
+@load-sigs test.sig
+
+@TEST-START-FILE test.sig
+signature xxxx {
+ ip-proto = udp
+ payload /XXXX/
+ event "Found XXXX"
+}
+
+signature axxxx {
+ ip-proto = udp
+ payload /^XXXX/
+ event "Found ^XXXX"
+}
+
+signature sxxxx {
+ ip-proto = udp
+ payload /.*XXXX/
+ event "Found .*XXXX"
+}
+
+signature yyyy {
+ ip-proto = udp
+ payload /YYYY/
+ event "Found YYYY"
+}
+
+signature ayyyy {
+ ip-proto = udp
+ payload /^YYYY/
+ event "Found ^YYYY"
+}
+
+signature syyyy {
+ ip-proto = udp
+ payload /.*YYYY/
+ event "Found .*YYYY"
+}
+
+signature nope {
+ ip-proto = udp
+ payload /.*nope/
+ event "Found .*nope"
+}
+@TEST-END-FILE
+
+event signature_match(state: signature_state, msg: string, data: string)
+	{
+	print "signature match", msg, data;
+	}