Merge remote-tracking branch 'origin/master' into topic/johanna/netcontrol-improvements

CHANGES

@@ -1,4 +1,36 @@
 
+2.4-424 | 2016-03-24 13:38:47 -0700
+
+  * Only load openflow/netcontrol if compiled with broker. (Johanna Amann)
+
+  * Adding canonifier to test. (Robin Sommer)
+
+2.4-422 | 2016-03-21 19:48:30 -0700
+
+  * Adapt to recent change in CAF CMake script. (Matthias Vallentin)
+
+  * Deprecate --with-libcaf in favor of --with-caf, as already done in
+    Broker. (Matthias Vallentin)
+
+2.4-418 | 2016-03-21 12:22:15 -0700
+
+  * Add protocol confirmation to MySQL analyzer. (Vlad Grigorescu)
+
+  * Check that there is only one of &read_expire, &write_expire,
+    &create_expire. (Johanna Amann)
+
+  * Fixed &read_expire for subnet-indexed tables, plus test case. (Jan
+    Grashoefer)
+
+  * Add filter_subnet_table() that works similar to matching_subnet()
+    but returns a filtered view of the original set/table only
+    containing the changed subnets. (Jan Grashoefer)
+
+  * Fix bug in tablue values' tracking read operations. (Johanna
+    Amann)
+
+  * Update TLS constants and extensions from IANA. (Johanna Amann)
+
 2.4-406 | 2016-03-11 14:27:47 -0800
 
   * Add NetControl and OpenFlow frameworks.  (Johanna Amann)

NEWS

@@ -49,6 +49,9 @@ New Functionality
   - matching_subnets(subnet, table) returns all subnets of the set or table
     that contain the given subnet.
 
+  - filter_subnet_table(subnet, table) works like check_subnet, but returns
+    a table containing all matching entries.
+
 - Several built-in functions for handling IP addresses and subnets were added:
 
   - is_v4_subnet(subnet) checks whether a subnet specification is IPv4.

VERSION

@@ -1 +1 @@
-2.4-406
+2.4-424

aux/broker

@@ -1 +1 @@
-Subproject commit fe35cde8f07ff7cf6decd2fb761cffc32e763d2d
+Subproject commit 6684ab5109f526fb535013760f17a4c8dff093ae

configure

@@ -276,8 +276,12 @@ while [ $# -ne 0 ]; do
         --with-swig=*)
             append_cache_entry SWIG_EXECUTABLE      PATH    $optarg
             ;;
+        --with-caf=*)
+            append_cache_entry CAF_ROOT_DIR      PATH   $optarg
+            ;;
         --with-libcaf=*)
-            append_cache_entry LIBCAF_ROOT_DIR      PATH   $optarg
+            echo "warning: --with-libcaf deprecated, use --with-caf instead"
+            append_cache_entry CAF_ROOT_DIR      PATH   $optarg
             ;;
         --with-rocksdb=*)
             append_cache_entry ROCKSDB_ROOT_DIR     PATH   $optarg

scripts/base/init-default.bro

@@ -37,8 +37,10 @@
 @load base/frameworks/reporter
 @load base/frameworks/sumstats
 @load base/frameworks/tunnels
+@ifdef ( BrokerComm::enable )
 @load base/frameworks/openflow
 @load base/frameworks/netcontrol
+@endif
 
 @load base/protocols/conn
 @load base/protocols/dhcp

scripts/base/protocols/ssl/consts.bro

@@ -109,7 +109,7 @@ export {
 		[7] = "client_authz",
 		[8] = "server_authz",
 		[9] = "cert_type",
-		[10] = "elliptic_curves",
+		[10] = "elliptic_curves", # new name: supported_groups - draft-ietf-tls-negotiated-ff-dhe
 		[11] = "ec_point_formats",
 		[12] = "srp",
 		[13] = "signature_algorithms",
@@ -120,9 +120,10 @@ export {
 		[18] = "signed_certificate_timestamp",
 		[19] = "client_certificate_type",
 		[20] = "server_certificate_type",
-		[21] = "padding", # temporary till 2016-03-12
+		[21] = "padding",
 		[22] = "encrypt_then_mac",
 		[23] = "extended_master_secret",
+		[24] = "token_binding", # temporary till 2017-02-04 - draft-ietf-tokbind-negotiation
 		[35] = "SessionTicket TLS",
 		[40] = "extended_random",
 		[13172] = "next_protocol_negotiation",
@@ -165,7 +166,10 @@ export {
 		[26] = "brainpoolP256r1",
 		[27] = "brainpoolP384r1",
 		[28] = "brainpoolP512r1",
-		# draft-ietf-tls-negotiated-ff-dhe-05
+		# Temporary till 2017-03-01 - draft-ietf-tls-rfc4492bis
+		[29] = "ecdh_x25519",
+		[30] = "ecdh_x448",
+		# draft-ietf-tls-negotiated-ff-dhe-10
 		[256] = "ffdhe2048",
 		[257] = "ffdhe3072",
 		[258] = "ffdhe4096",

src/Attr.cc

@@ -375,12 +375,33 @@ void Attributes::CheckAttr(Attr* a)
 	case ATTR_EXPIRE_READ:
 	case ATTR_EXPIRE_WRITE:
 	case ATTR_EXPIRE_CREATE:
+		{
 		if ( type->Tag() != TYPE_TABLE )
 			{
 			Error("expiration only applicable to tables");
 			break;
 			}
 
+		int num_expires = 0;
+		if ( attrs )
+			{
+			loop_over_list(*attrs, i)
+				{
+				Attr* a = (*attrs)[i];
+				if ( a->Tag() == ATTR_EXPIRE_READ ||
+				     a->Tag() == ATTR_EXPIRE_WRITE ||
+				     a->Tag() == ATTR_EXPIRE_CREATE )
+					num_expires++;
+				}
+			}
+
+		if ( num_expires > 1 )
+			{
+			Error("set/table can only have one of &read_expire, &write_expire, &create_expire");
+			break;
+			}
+		}
+
 #if 0
 		//### not easy to test this w/o knowing the ID.
 		if ( ! IsGlobal() )

src/PrefixTable.cc

@@ -62,9 +62,9 @@ void* PrefixTable::Insert(const Val* value, void* data)
 	}
 	}
 
-list<IPPrefix> PrefixTable::FindAll(const IPAddr& addr, int width) const
+list<tuple<IPPrefix,void*>> PrefixTable::FindAll(const IPAddr& addr, int width) const
 	{
-	std::list<IPPrefix> out;
+	std::list<tuple<IPPrefix,void*>> out;
 	prefix_t* prefix = MakePrefix(addr, width);
 
 	int elems = 0;
@@ -73,14 +73,14 @@ list<IPPrefix> PrefixTable::FindAll(const IPAddr& addr, int width) const
 	patricia_search_all(tree, prefix, &list, &elems);
 
 	for ( int i = 0; i < elems; ++i )
-		out.push_back(PrefixToIPPrefix(list[i]->prefix));
+		out.push_back(std::make_tuple(PrefixToIPPrefix(list[i]->prefix), list[i]->data));
 
 	Deref_Prefix(prefix);
 	free(list);
 	return out;
 	}
 
-list<IPPrefix> PrefixTable::FindAll(const SubNetVal* value) const
+list<tuple<IPPrefix,void*>> PrefixTable::FindAll(const SubNetVal* value) const
 	{
 	return FindAll(value->AsSubNet().Prefix(), value->AsSubNet().LengthIPv6());
 	}

src/PrefixTable.h

@@ -37,8 +37,8 @@ public:
 	void* Lookup(const Val* value, bool exact = false) const;
 
 	// Returns list of all found matches or empty list otherwise.
-	list<IPPrefix> FindAll(const IPAddr& addr, int width) const;
-	list<IPPrefix> FindAll(const SubNetVal* value) const;
+	list<tuple<IPPrefix,void*>> FindAll(const IPAddr& addr, int width) const;
+	list<tuple<IPPrefix,void*>> FindAll(const SubNetVal* value) const;
 
 	// Returns pointer to data or nil if not found.
 	void* Remove(const IPAddr& addr, int width);

src/Val.cc

@@ -1787,7 +1787,16 @@ Val* TableVal::Lookup(Val* index, bool use_default_val)
 		{
 		TableEntryVal* v = (TableEntryVal*) subnets->Lookup(index);
 		if ( v )
+			{
+			if ( attrs && attrs->FindAttr(ATTR_EXPIRE_READ) )
+					{
+					v->SetExpireAccess(network_time);
+					if ( LoggingAccess() && expire_time )
+						ReadOperation(index, v);
+					}
+
 			return v->Value() ? v->Value() : this;
+			}
 
 		if ( ! use_default_val )
 			return 0;
@@ -1810,9 +1819,7 @@ Val* TableVal::Lookup(Val* index, bool use_default_val)
 
 			if ( v )
 				{
-				if ( attrs &&
-				     ! (attrs->FindAttr(ATTR_EXPIRE_WRITE) ||
-					attrs->FindAttr(ATTR_EXPIRE_CREATE)) )
+				if ( attrs && attrs->FindAttr(ATTR_EXPIRE_READ) )
 					{
 					v->SetExpireAccess(network_time);
 					if ( LoggingAccess() && expire_time )
@@ -1833,6 +1840,57 @@ Val* TableVal::Lookup(Val* index, bool use_default_val)
 	return def;
 	}
 
+VectorVal* TableVal::LookupSubnets(const SubNetVal* search)
+	{
+	if ( ! subnets )
+		reporter->InternalError("LookupSubnets called on wrong table type");
+
+	VectorVal* result = new VectorVal(internal_type("subnet_vec")->AsVectorType());
+
+	auto matches = subnets->FindAll(search);
+	for ( auto element : matches )
+		{
+		SubNetVal* s = new SubNetVal(get<0>(element));
+		result->Assign(result->Size(), s);
+		}
+
+	return result;
+	}
+
+TableVal* TableVal::LookupSubnetValues(const SubNetVal* search)
+	{
+	if ( ! subnets )
+		reporter->InternalError("LookupSubnetValues called on wrong table type");
+
+	TableVal* nt = new TableVal(this->Type()->Ref()->AsTableType());
+
+	auto matches = subnets->FindAll(search);
+	for ( auto element : matches )
+		{
+		SubNetVal* s = new SubNetVal(get<0>(element));
+		TableEntryVal* entry = reinterpret_cast<TableEntryVal*>(get<1>(element));
+
+		if ( entry && entry->Value() )
+			nt->Assign(s, entry->Value()->Ref());
+		else
+			nt->Assign(s, 0); // set
+
+		if ( entry )
+			{
+			if ( attrs && attrs->FindAttr(ATTR_EXPIRE_READ) )
+				{
+				entry->SetExpireAccess(network_time);
+				if ( LoggingAccess() && expire_time )
+					ReadOperation(s, entry);
+				}
+			}
+
+		Unref(s); // assign does not consume index
+		}
+
+	return nt;
+	}
+
 bool TableVal::UpdateTimestamp(Val* index)
 	{
 	TableEntryVal* v;
@@ -1854,7 +1912,7 @@ bool TableVal::UpdateTimestamp(Val* index)
 		return false;
 
 	v->SetExpireAccess(network_time);
-	if ( attrs->FindAttr(ATTR_EXPIRE_READ) )
+	if ( LoggingAccess() && attrs->FindAttr(ATTR_EXPIRE_READ) )
 		ReadOperation(index, v);
 
 	return true;
@@ -2478,7 +2536,7 @@ bool TableVal::DoUnserialize(UnserialInfo* info)
 		}
 
 	// If necessary, activate the expire timer.
-	if ( attrs)
+	if ( attrs )
 		{
 		CheckExpireAttr(ATTR_EXPIRE_READ);
 		CheckExpireAttr(ATTR_EXPIRE_WRITE);

src/Val.h

@@ -790,6 +790,16 @@ public:
 	// need to Ref/Unref it when calling the default function.
 	Val* Lookup(Val* index, bool use_default_val = true);
 
+	// For a table[subnet]/set[subnet], return all subnets that cover
+	// the given subnet.
+	// Causes an internal error if called for any other kind of table.
+	VectorVal* LookupSubnets(const SubNetVal* s);
+
+	// For a set[subnet]/table[subnet], return a new table that only contains
+	// entries that cover the given subnet.
+	// Causes an internal error if called for any other kind of table.
+	TableVal* LookupSubnetValues(const SubNetVal* s);
+
 	// Sets the timestamp for the given index to network time.
 	// Returns false if index does not exist.
 	bool UpdateTimestamp(Val* index);

src/analyzer/protocol/mysql/mysql-analyzer.pac

@@ -19,6 +19,9 @@ refine flow MySQL_Flow += {
 
 	function proc_mysql_handshake_response_packet(msg: Handshake_Response_Packet): bool
 		%{
+		if ( ${msg.version} == 9 || ${msg.version == 10} )
+			connection()->bro_analyzer()->ProtocolConfirmation();
+
 		if ( mysql_handshake )
 			{
 			if ( ${msg.version} == 10 )

src/bro.bif

@@ -1031,7 +1031,7 @@ function clear_table%(v: any%): any
 	return 0;
 	%}
 
-## Gets all subnets that match a given subnet from a set/table[subnet]
+## Gets all subnets that contain a given subnet from a set/table[subnet]
 ##
 ## search: the subnet to search for.
 ##
@@ -1046,23 +1046,26 @@ function matching_subnets%(search: subnet, t: any%): subnet_vec
 		return nullptr;
 		}
 
-	const PrefixTable* pt = t->AsTableVal()->Subnets();
-	if ( ! pt )
+	return t->AsTableVal()->LookupSubnets(search);
+	%}
+
+## For a set[subnet]/table[subnet], create a new table that contains all entries that
+## contain a given subnet.
+##
+## search: the subnet to search for.
+##
+## t: the set[subnet] or table[subnet].
+##
+## Returns: A new table that contains all the entries that cover the subnet searched for.
+function filter_subnet_table%(search: subnet, t: any%): any
+	%{
+	if ( t->Type()->Tag() != TYPE_TABLE || ! t->Type()->AsTableType()->IsSubNetIndex() )
 		{
-		reporter->Error("matching_subnets encountered nonexisting prefix table.");
+		reporter->Error("filter_subnet_table needs to be called on a set[subnet]/table[subnet].");
 		return nullptr;
 		}
 
-	VectorVal* result_v = new VectorVal(internal_type("subnet_vec")->AsVectorType());
-
-	auto matches = pt->FindAll(search);
-	for ( auto element : matches )
-		{
-		SubNetVal* s = new SubNetVal(element);
-		result_v->Assign(result_v->Size(), s);
-		}
-
-	return result_v;
+	return t->AsTableVal()->LookupSubnetValues(search);
 	%}
 
 ## Checks if a specific subnet is a member of a set/table[subnet].
@@ -1078,14 +1081,14 @@ function check_subnet%(search: subnet, t: any%): bool
 	%{
 	if ( t->Type()->Tag() != TYPE_TABLE || ! t->Type()->AsTableType()->IsSubNetIndex() )
 		{
-		reporter->Error("matching_subnets needs to be called on a set[subnet]/table[subnet].");
+		reporter->Error("check_subnet needs to be called on a set[subnet]/table[subnet].");
 		return nullptr;
 		}
 
 	const PrefixTable* pt = t->AsTableVal()->Subnets();
 	if ( ! pt )
 		{
-		reporter->Error("matching_subnets encountered nonexisting prefix table.");
+		reporter->Error("check_subnet encountered nonexisting prefix table.");
 		return nullptr;
 		}
 

src/broker/CMakeLists.txt

@@ -10,8 +10,8 @@ if ( ROCKSDB_INCLUDE_DIR )
     include_directories(BEFORE ${ROCKSDB_INCLUDE_DIR})
 endif ()
 
-include_directories(BEFORE ${LIBCAF_INCLUDE_DIR_CORE})
-include_directories(BEFORE ${LIBCAF_INCLUDE_DIR_IO})
+include_directories(BEFORE ${CAF_INCLUDE_DIR_CORE})
+include_directories(BEFORE ${CAF_INCLUDE_DIR_IO})
 
 set(comm_SRCS
     Data.cc

testing/btest/Baseline/bifs.filter_subnet_table/output

@@ -0,0 +1,20 @@
+{
+10.0.0.0/8,
+10.2.0.2/31,
+10.2.0.0/16
+}
+{
+[10.0.0.0/8] = a,
+[10.2.0.2/31] = c,
+[10.2.0.0/16] = b
+}
+{
+[10.0.0.0/8] = a,
+[10.3.0.0/16] = e
+}
+{
+
+}
+{
+
+}

testing/btest/Baseline/language.expire_multiple-2/output

@@ -0,0 +1 @@
+error in /Users/johanna/bro/master/testing/btest/.tmp/language.expire_multiple-2/expire_multiple.test, line 2: set/table can only have one of &read_expire, &write_expire, &create_expire (&write_expire=1.0 sec, &create_expire=3.0 secs)

testing/btest/Baseline/language.expire_multiple-3/output

@@ -0,0 +1 @@
+error in /Users/johanna/bro/master/testing/btest/.tmp/language.expire_multiple-3/expire_multiple.test, line 2: set/table can only have one of &read_expire, &write_expire, &create_expire (&write_expire=1.0 sec, &read_expire=3.0 secs)

testing/btest/Baseline/language.expire_multiple/output

@@ -0,0 +1 @@
+error in /Users/johanna/bro/master/testing/btest/.tmp/language.expire_multiple/expire_multiple.test, line 4: set/table can only have one of &read_expire, &write_expire, &create_expire (&create_expire=1.0 sec, &read_expire=1.0 sec)

testing/btest/Baseline/language.expire_subnet/output

@@ -0,0 +1,27 @@
+All:
+0 --> zero
+2 --> two
+4 --> four
+1 --> one
+3 --> three
+192.168.3.0/24 --> three
+192.168.0.0/16 --> zero
+192.168.4.0/24 --> four
+192.168.1.0/24 --> one
+192.168.2.0/24 --> two
+Time: 0 secs
+
+Accessed table nums: two; three
+Accessed table nets: two; three, zero
+Time: 7.0 secs 518.0 msecs 828.0 usecs
+
+Expired Num: 0 --> zero at 8.0 secs 835.0 msecs 30.0 usecs
+Expired Num: 4 --> four at 8.0 secs 835.0 msecs 30.0 usecs
+Expired Num: 1 --> one at 8.0 secs 835.0 msecs 30.0 usecs
+Expired Subnet: 192.168.4.0/24 --> four at 8.0 secs 835.0 msecs 30.0 usecs
+Expired Subnet: 192.168.1.0/24 --> one at 8.0 secs 835.0 msecs 30.0 usecs
+Expired Num: 2 --> two at 15.0 secs 150.0 msecs 681.0 usecs
+Expired Num: 3 --> three at 15.0 secs 150.0 msecs 681.0 usecs
+Expired Subnet: 192.168.3.0/24 --> three at 15.0 secs 150.0 msecs 681.0 usecs
+Expired Subnet: 192.168.0.0/16 --> zero at 15.0 secs 150.0 msecs 681.0 usecs
+Expired Subnet: 192.168.2.0/24 --> two at 15.0 secs 150.0 msecs 681.0 usecs

testing/btest/bifs/filter_subnet_table.bro

@@ -0,0 +1,49 @@
+# @TEST-EXEC: bro -b %INPUT >output
+# @TEST-EXEC: btest-diff output
+
+global testa: set[subnet] = {
+	10.0.0.0/8,
+	10.2.0.0/16,
+	10.2.0.2/31,
+	10.1.0.0/16,
+	10.3.0.0/16,
+	5.0.0.0/8,
+	5.5.0.0/25,
+	5.2.0.0/32,
+	7.2.0.0/32,
+	[2607:f8b0:4008:807::200e]/64,
+	[2607:f8b0:4007:807::200e]/64,
+	[2607:f8b0:4007:807::200e]/128
+};
+
+global testb: table[subnet] of string = {
+	[10.0.0.0/8] = "a",
+	[10.2.0.0/16] = "b",
+	[10.2.0.2/31] = "c",
+	[10.1.0.0/16] = "d",
+	[10.3.0.0/16] = "e",
+	[5.0.0.0/8] = "f",
+	[5.5.0.0/25] = "g",
+	[5.2.0.0/32] = "h",
+	[7.2.0.0/32] = "i",
+	[[2607:f8b0:4008:807::200e]/64] = "j",
+	[[2607:f8b0:4007:807::200e]/64] = "k",
+	[[2607:f8b0:4007:807::200e]/128] = "l"
+};
+
+
+event bro_init()
+	{
+	local c = filter_subnet_table(10.2.0.2/32, testa);
+	print c;
+	c = filter_subnet_table(10.2.0.2/32, testb);
+	print c;
+	c = filter_subnet_table(10.3.0.2/32, testb);
+	print c;
+	c = filter_subnet_table(1.0.0.0/8, testb);
+	print c;
+
+	local unspecified: table[subnet] of string = table();
+	c = filter_subnet_table(10.2.0.2/32, unspecified);
+	print c;
+	}

testing/btest/language/expire_multiple.test

@@ -0,0 +1,12 @@
+# @TEST-EXEC-FAIL: bro -b %INPUT >output 2>&1
+# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff output
+
+global s: set[string] &create_expire=1secs &read_expire=1secs;
+
+# @TEST-START-NEXT:
+
+global s: set[string] &write_expire=1secs &create_expire=3secs;
+
+# @TEST-START-NEXT:
+
+global s: set[string] &write_expire=1secs &read_expire=3secs;

testing/btest/language/expire_subnet.test

@@ -0,0 +1,96 @@
+# @TEST-EXEC: bro -C -r $TRACES/var-services-std-ports.trace %INPUT >output
+# @TEST-EXEC: btest-diff output
+
+redef table_expire_interval = 1sec;
+
+global start_time: time;
+
+function time_past(): interval
+	{
+	return network_time() - start_time;
+	}
+
+function expire_nums(tbl: table[count] of string, idx: count): interval
+	{
+	print fmt("Expired Num: %s --> %s at %s", idx, tbl[idx], time_past());
+	return 0sec;
+	}
+
+function expire_nets(tbl: table[subnet] of string, idx: subnet): interval
+	{
+	print fmt("Expired Subnet: %s --> %s at %s", idx, tbl[idx], time_past());
+	return 0sec;
+	}
+
+global nums: table[count] of string &read_expire=8sec &expire_func=expire_nums;
+global nets: table[subnet] of string &read_expire=8sec &expire_func=expire_nets;
+global step: count;
+
+### Test ###
+
+function execute_test()
+	{
+	local num_a = nums[2];
+	local num_b = nums[3];
+
+	local net_a = nets[192.168.2.0/24];
+	#local net_b = nets[192.168.3.0/24];
+	local nets_b = "";
+	local nets_b_tbl: table[subnet] of string;
+
+	nets_b_tbl = filter_subnet_table(192.168.3.0/24, nets);
+	for ( idx in nets_b_tbl )
+		nets_b += cat(", ", nets_b_tbl[idx]);
+	nets_b = nets_b[2:];
+	
+	# writing resets expire as expected
+	#nets[192.168.2.0/24] = "accessed";
+	#nets[192.168.3.0/24] = "accessed";
+
+	print fmt("Accessed table nums: %s; %s", num_a, num_b);
+	print fmt("Accessed table nets: %s; %s", net_a, nets_b);
+	print fmt("Time: %s", time_past());
+	print "";
+	}
+
+### Events ###
+
+event bro_init()
+	{
+	step = 0;
+
+	nums[0] = "zero";
+	nums[1] = "one";
+	nums[2] = "two";
+	nums[3] = "three";
+	nums[4] = "four";
+
+	nets[192.168.0.0/16] = "zero";
+	nets[192.168.1.0/24] = "one";
+	nets[192.168.2.0/24] = "two";
+	nets[192.168.3.0/24] = "three";
+	nets[192.168.4.0/24] = "four";
+	}
+
+event new_packet(c: connection, p: pkt_hdr)
+	{
+	if ( step == 0 )
+		{
+		++step;
+		start_time = network_time();
+
+		print "All:";
+		for ( num in nums )
+			print fmt("%s --> %s", num, nums[num]);
+		for ( net in nets )
+			print fmt("%s --> %s", net, nets[net]);
+		print fmt("Time: %s", time_past());
+		print "";
+		}
+
+	if ( (time_past() > 7sec) && (step == 1) )
+		{
+		++step;
+		execute_test();
+		}
+	}