Merge branch 'fastpath' of ssh://git.bro-ids.org/bro into fastpath

doc/signatures.rst

@@ -229,20 +229,10 @@ matched. The following context conditions are defined:
     confirming the match. If false is returned, no signature match is
     going to be triggered. The function has to be of type ``function
     cond(state: signature_state, data: string): bool``. Here,
-    ``content`` may contain the most recent content chunk available at
+    ``data`` may contain the most recent content chunk available at
     the time the signature was matched. If no such chunk is available,
-    ``content`` will be the empty string. ``signature_state`` is
+    ``data`` will be the empty string. See :bro:type:`signature_state`
-    defined as follows:
+    for its definition.
-
-    .. code:: bro
-
-        type signature_state: record {
-            id: string;          # ID of the signature
-            conn: connection;    # Current connection
-            is_orig: bool;       # True if current endpoint is originator
-            payload_size: count; # Payload size of the first packet
-            };
-
 
 ``payload-size <cmp> <integer>``
     Compares the integer to the size of the payload of a packet. For

src/RuleCondition.cc

@@ -126,6 +126,23 @@ RuleConditionEval::RuleConditionEval(const char* func)
 		rules_error("unknown identifier", func);
 		return;
 		}
+
+	if ( id->Type()->Tag() == TYPE_FUNC )
+		{
+		// validate argument quantity and type
+		FuncType* f = id->Type()->AsFuncType();
+
+		if ( f->YieldType()->Tag() != TYPE_BOOL )
+			rules_error("eval function type must yield a 'bool'", func);
+
+		TypeList tl;
+		tl.Append(internal_type("signature_state")->Ref());
+		tl.Append(base_type(TYPE_STRING));
+
+		if ( ! f->CheckArgs(tl.Types()) )
+			rules_error("eval function parameters must be a 'signature_state' "
+			            "and a 'string' type", func);
+		}
 	}
 
 bool RuleConditionEval::DoMatch(Rule* rule, RuleEndpointState* state,

testing/btest/Baseline/analyzers.conn-size-cc/conn.log

@@ -1,5 +0,0 @@
-1128727430.350788 ? 141.42.64.125 125.190.109.199 other 56729 12345 tcp ? ? S0 X 1 60 0 0 cc=1
-1144876538.705610 5.921003 169.229.147.203 239.255.255.253 other 49370 427 udp 147 ? S0 X 3 231 0 0
-1144876599.397603 0.815763 192.150.186.169 194.64.249.244 http 53063 80 tcp 377 445 SF X 6 677 5 713
-1144876709.032670 9.000191 169.229.147.43 239.255.255.253 other 49370 427 udp 196 ? S0 X 4 308 0 0
-1144876697.068273 0.000650 192.150.186.169 192.150.186.15 icmp-unreach 3 3 icmp 56 ? OTH X 2 112 0 0

testing/btest/Baseline/analyzers.conn-size/conn.log

@@ -1,5 +0,0 @@
-1128727430.350788 ? 141.42.64.125 125.190.109.199 other 56729 12345 tcp ? ? S0 X 1 60 0 0
-1144876538.705610 5.921003 169.229.147.203 239.255.255.253 other 49370 427 udp 147 ? S0 X 3 231 0 0
-1144876599.397603 0.815763 192.150.186.169 194.64.249.244 http 53063 80 tcp 377 445 SF X 6 697 5 713
-1144876709.032670 9.000191 169.229.147.43 239.255.255.253 other 49370 427 udp 196 ? S0 X 4 308 0 0
-1144876697.068273 0.000650 192.150.186.169 192.150.186.15 icmp-unreach 3 3 icmp 56 ? OTH X 2 112 0 0

testing/btest/Baseline/signatures.bad-eval-condition/.stderr

@@ -0,0 +1,2 @@
+error: Error in signature (./blah.sig:6): eval function parameters must be a 'signature_state' and a 'string' type (mark_conn)
+

testing/btest/Baseline/signatures.eval-condition/conn.log

@@ -0,0 +1,14 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	conn
+#open	2012-08-23-16-41-23
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	service	duration	orig_bytes	resp_bytes	conn_state	local_orig	missed_bytes	history	orig_pkts	orig_ip_bytes	resp_pkts	resp_ip_bytes	tunnel_parents
+#types	time	string	addr	port	addr	port	enum	string	interval	count	count	string	bool	count	string	count	count	count	count	table[string]
+1329843175.736107	arKYeMETxOg	141.142.220.235	37604	199.233.217.249	56666	tcp	ftp-data	0.112432	0	342	SF	-	0	ShAdfFa	4	216	4	562	(empty)
+1329843179.871641	k6kgXLOoSKl	141.142.220.235	59378	199.233.217.249	56667	tcp	ftp-data	0.111218	0	77	SF	-	0	ShAdfFa	4	216	4	297	(empty)
+1329843194.151526	nQcgTWjvg4c	199.233.217.249	61920	141.142.220.235	33582	tcp	ftp-data	0.056211	342	0	SF	-	0	ShADaFf	5	614	3	164	(empty)
+1329843197.783443	j4u32Pc5bif	199.233.217.249	61918	141.142.220.235	37835	tcp	ftp-data	0.056005	77	0	SF	-	0	ShADaFf	5	349	3	164	(empty)
+1329843161.968492	UWkUyAuUGXf	141.142.220.235	50003	199.233.217.249	21	tcp	ftp,blah	38.055625	180	3146	SF	-	0	ShAdDfFa	38	2164	25	4458	(empty)
+#close	2012-08-23-16-41-23

testing/btest/analyzers/conn-size-cc.bro

@@ -1,2 +0,0 @@
-# @TEST-EXEC: bro -C -r ${TRACES}/conn-size.trace tcp udp icmp report_conn_size_analyzer=T
-# @TEST-EXEC: btest-diff conn.log

testing/btest/analyzers/conn-size.bro

@@ -1,2 +0,0 @@
-# @TEST-EXEC: bro -C -r ${TRACES}/conn-size.trace tcp udp icmp report_conn_size_analyzer=T
-# @TEST-EXEC: btest-diff conn.log

testing/btest/btest.cfg

@@ -1,5 +1,5 @@
 [btest]
-TestDirs    = doc bifs language core scripts istate coverage
+TestDirs    = doc bifs language core scripts istate coverage signatures
 TmpDir      = %(testbase)s/.tmp
 BaselineDir = %(testbase)s/Baseline
 IgnoreDirs  = .svn CVS .tmp

testing/btest/core/leaks/basic-cluster.bro

@@ -1,21 +1,22 @@
 # Needs perftools support.
 #
+# @TEST-SERIALIZE: comm
 # @TEST-GROUP: leaks
-
+#
 # @TEST-REQUIRES: bro  --help 2>&1 | grep -q mem-leaks
-
+#
 # @TEST-EXEC: btest-bg-run manager-1 HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local BROPATH=$BROPATH:.. CLUSTER_NODE=manager-1 bro -m %INPUT
 # @TEST-EXEC: btest-bg-run proxy-1   HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local BROPATH=$BROPATH:.. CLUSTER_NODE=proxy-1   bro -m %INPUT
 # @TEST-EXEC: sleep 1
 # @TEST-EXEC: btest-bg-run worker-1  HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local BROPATH=$BROPATH:.. CLUSTER_NODE=worker-1  bro -m -r $TRACES/web.trace --pseudo-realtime %INPUT
 # @TEST-EXEC: btest-bg-run worker-2  HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local BROPATH=$BROPATH:.. CLUSTER_NODE=worker-2  bro -m -r $TRACES/web.trace --pseudo-realtime %INPUT
-# @TEST-EXEC: btest-bg-wait 40
+# @TEST-EXEC: btest-bg-wait 60
 # @TEST-EXEC: btest-diff manager-1/metrics.log
 
 @TEST-START-FILE cluster-layout.bro
 redef Cluster::nodes = {
-	["manager-1"] = [$node_type=Cluster::MANAGER, $ip=127.0.0.1, $p=37757/tcp, $workers=set("worker-1")],
+	["manager-1"] = [$node_type=Cluster::MANAGER, $ip=127.0.0.1, $p=37757/tcp, $workers=set("worker-1", "worker-2")],
-	["proxy-1"] = [$node_type=Cluster::PROXY,     $ip=127.0.0.1, $p=37758/tcp, $manager="manager-1", $workers=set("worker-1")],
+	["proxy-1"] = [$node_type=Cluster::PROXY,     $ip=127.0.0.1, $p=37758/tcp, $manager="manager-1", $workers=set("worker-1", "worker-2")],
 	["worker-1"] = [$node_type=Cluster::WORKER,   $ip=127.0.0.1, $p=37760/tcp, $manager="manager-1", $proxy="proxy-1", $interface="eth0"],
 	["worker-2"] = [$node_type=Cluster::WORKER,   $ip=127.0.0.1, $p=37761/tcp, $manager="manager-1", $proxy="proxy-1", $interface="eth1"],
 };
@@ -32,13 +33,6 @@ event bro_init() &priority=5
 	Metrics::add_filter(TEST_METRIC, 
 		[$name="foo-bar",
 		 $break_interval=3secs]);
-		
-	if ( Cluster::local_node_type() == Cluster::WORKER )
-		{
-		Metrics::add_data(TEST_METRIC, [$host=1.2.3.4], 3);
-		Metrics::add_data(TEST_METRIC, [$host=6.5.4.3], 2);
-		Metrics::add_data(TEST_METRIC, [$host=7.2.1.5], 1);
-		}
 	}
 
 event remote_connection_closed(p: event_peer)
@@ -46,9 +40,25 @@ event remote_connection_closed(p: event_peer)
 	terminate();
 	}
 
+global ready_for_data: event();
+
+redef Cluster::manager2worker_events += /ready_for_data/;
+
+@if ( Cluster::local_node_type() == Cluster::WORKER )
+
+event ready_for_data()
+	{
+	Metrics::add_data(TEST_METRIC, [$host=1.2.3.4], 3);
+	Metrics::add_data(TEST_METRIC, [$host=6.5.4.3], 2);
+	Metrics::add_data(TEST_METRIC, [$host=7.2.1.5], 1);
+	}
+
+@endif
+
 @if ( Cluster::local_node_type() == Cluster::MANAGER )
 
 global n = 0;
+global peer_count = 0;
 
 event Metrics::log_metrics(rec: Metrics::Info)
 	{
@@ -60,4 +70,14 @@ event Metrics::log_metrics(rec: Metrics::Info)
 		}
 	}
 
+event remote_connection_handshake_done(p: event_peer)
+	{
+	print p;
+	peer_count = peer_count + 1;
+	if ( peer_count == 3 )
+		{
+		event ready_for_data();
+		}
+	}
+
 @endif

testing/btest/core/leaks/remote.bro

@@ -1,5 +1,6 @@
 # Needs perftools support.
 #
+# @TEST-SERIALIZE: comm
 # @TEST-GROUP: leaks
 #
 # @TEST-REQUIRES: bro  --help 2>&1 | grep -q mem-leaks

testing/btest/coverage/bare-mode-errors.test

@@ -8,7 +8,7 @@
 # @TEST-SERIALIZE: comm
 #
 # @TEST-EXEC: test -d $DIST/scripts
-# @TEST-EXEC: for script in `find $DIST/scripts -name \*\.bro -not -path '*/site/*'`; do echo $script; if echo "$script" | egrep -q 'communication/listen|controllee'; then rm -rf load_attempt .bgprocs; btest-bg-run load_attempt bro -b $script; btest-bg-wait -k 2; cat load_attempt/.stderr >>allerrors; else bro -b $script 2>>allerrors; fi done || exit 0
+# @TEST-EXEC: for script in `find $DIST/scripts/ -name \*\.bro -not -path '*/site/*'`; do echo $script; if echo "$script" | egrep -q 'communication/listen|controllee'; then rm -rf load_attempt .bgprocs; btest-bg-run load_attempt bro -b $script; btest-bg-wait -k 2; cat load_attempt/.stderr >>allerrors; else bro -b $script 2>>allerrors; fi done || exit 0
 # @TEST-EXEC: cat allerrors | grep -v "received termination signal" | sort | uniq > unique_errors
 # @TEST-EXEC: if [ $(grep -c CURL_INCLUDE_DIR-NOTFOUND $BUILD/CMakeCache.txt) -ne 0 ]; then cp unique_errors unique_errors_no_elasticsearch; fi
 # @TEST-EXEC: if [ $(grep -c CURL_INCLUDE_DIR-NOTFOUND $BUILD/CMakeCache.txt) -ne 0 ]; then btest-diff unique_errors_no_elasticsearch; else btest-diff unique_errors; fi

testing/btest/scripts/base/frameworks/metrics/basic-cluster.bro

@@ -5,13 +5,13 @@
 # @TEST-EXEC: sleep 1
 # @TEST-EXEC: btest-bg-run worker-1  BROPATH=$BROPATH:.. CLUSTER_NODE=worker-1 bro %INPUT
 # @TEST-EXEC: btest-bg-run worker-2  BROPATH=$BROPATH:.. CLUSTER_NODE=worker-2 bro %INPUT
-# @TEST-EXEC: btest-bg-wait 20
+# @TEST-EXEC: btest-bg-wait 30
 # @TEST-EXEC: btest-diff manager-1/metrics.log
 
 @TEST-START-FILE cluster-layout.bro
 redef Cluster::nodes = {
-	["manager-1"] = [$node_type=Cluster::MANAGER, $ip=127.0.0.1, $p=37757/tcp, $workers=set("worker-1")],
+	["manager-1"] = [$node_type=Cluster::MANAGER, $ip=127.0.0.1, $p=37757/tcp, $workers=set("worker-1", "worker-2")],
-	["proxy-1"] = [$node_type=Cluster::PROXY,     $ip=127.0.0.1, $p=37758/tcp, $manager="manager-1", $workers=set("worker-1")],
+	["proxy-1"] = [$node_type=Cluster::PROXY,     $ip=127.0.0.1, $p=37758/tcp, $manager="manager-1", $workers=set("worker-1", "worker-2")],
 	["worker-1"] = [$node_type=Cluster::WORKER,   $ip=127.0.0.1, $p=37760/tcp, $manager="manager-1", $proxy="proxy-1", $interface="eth0"],
 	["worker-2"] = [$node_type=Cluster::WORKER,   $ip=127.0.0.1, $p=37761/tcp, $manager="manager-1", $proxy="proxy-1", $interface="eth1"],
 };
@@ -28,13 +28,6 @@ event bro_init() &priority=5
 	Metrics::add_filter(TEST_METRIC, 
 		[$name="foo-bar",
 		 $break_interval=3secs]);
-		
-	if ( Cluster::local_node_type() == Cluster::WORKER )
-		{
-		Metrics::add_data(TEST_METRIC, [$host=1.2.3.4], 3);
-		Metrics::add_data(TEST_METRIC, [$host=6.5.4.3], 2);
-		Metrics::add_data(TEST_METRIC, [$host=7.2.1.5], 1);
-		}
 	}
 
 event remote_connection_closed(p: event_peer)
@@ -42,9 +35,25 @@ event remote_connection_closed(p: event_peer)
 	terminate();
 	}
 
+global ready_for_data: event();
+
+redef Cluster::manager2worker_events += /ready_for_data/;
+
+@if ( Cluster::local_node_type() == Cluster::WORKER )
+
+event ready_for_data()
+	{
+	Metrics::add_data(TEST_METRIC, [$host=1.2.3.4], 3);
+	Metrics::add_data(TEST_METRIC, [$host=6.5.4.3], 2);
+	Metrics::add_data(TEST_METRIC, [$host=7.2.1.5], 1);
+	}
+
+@endif
+
 @if ( Cluster::local_node_type() == Cluster::MANAGER )
 
 global n = 0;
+global peer_count = 0;
 
 event Metrics::log_metrics(rec: Metrics::Info)
 	{
@@ -56,4 +65,14 @@ event Metrics::log_metrics(rec: Metrics::Info)
 		}
 	}
 
+event remote_connection_handshake_done(p: event_peer)
+	{
+	print p;
+	peer_count = peer_count + 1;
+	if ( peer_count == 3 )
+		{
+		event ready_for_data();
+		}
+	}
+
 @endif

testing/btest/signatures/bad-eval-condition.bro

@@ -0,0 +1,22 @@
+# @TEST-EXEC-FAIL: bro -r $TRACES/ftp-ipv4.trace %INPUT
+# @TEST-EXEC: btest-diff .stderr
+
+@load-sigs blah.sig
+
+@TEST-START-FILE blah.sig
+signature blah
+	{
+	ip-proto == tcp
+	src-port == 21
+	payload /.*/
+	eval mark_conn
+	}
+@TEST-END-FILE
+
+# wrong function signature for use with signature 'eval' conditions
+# needs to be reported
+function mark_conn(state: signature_state): bool
+	{
+	add state$conn$service["blah"];
+	return T;
+	}

testing/btest/signatures/eval-condition.bro

@@ -0,0 +1,20 @@
+# @TEST-EXEC: bro -r $TRACES/ftp-ipv4.trace %INPUT
+# @TEST-EXEC: btest-diff conn.log
+
+@load-sigs blah.sig
+
+@TEST-START-FILE blah.sig
+signature blah
+	{
+	ip-proto == tcp
+	src-port == 21
+	payload /.*/
+	eval mark_conn
+	}
+@TEST-END-FILE
+
+function mark_conn(state: signature_state, data: string): bool
+	{
+	add state$conn$service["blah"];
+	return T;
+	}