Merge remote-tracking branch 'origin/master' into topic/johanna/md5-fips

CHANGES

@@ -1,4 +1,19 @@
 
+2.6-95 | 2019-01-23 09:49:35 -0800
+
+  * GH-219: fix |x| operator int overflow / floating point type inconsistency
+    (Jon Siwek, Corelight)
+
+2.6-92 | 2019-01-22 08:53:36 -0800
+
+  * GH-151: fix hash calculation for nested sets
+
+    Hash key construction of nested sets depended on the order in
+    which their elements are iterated, which varied even between sets
+    containing equivalent elements.  The iteration order is now sorted
+    by each element's hash value (or, on collision, by full key) such
+    that equivalent sets no longer hash differently. (Jon Siwek, Corelight)
+
 2.6-89 | 2019-01-18 15:17:34 -0800
 
   * Pre-allocate and re-use Vals for bool, int, count, enum and empty string (Jon Siwek, Corelight)

VERSION

@@ -1 +1 @@
-2.6-89
+2.6-95

aux/broker

@@ -1 +1 @@
-Subproject commit c7b1dfd38ec6c42729f8c462eef6457a8dd948b6
+Subproject commit bf734622dceaafaf7a481185efd22bd7cc805f9b

src/3rdparty

@@ -1 +1 @@
-Subproject commit b822eeed58c4a1ee3781f1f8c8a19fd590dc4a04
+Subproject commit 6e93c5546a4770d513fb57213d7b29e39e12bf4d

src/CompHash.cc

@@ -7,6 +7,9 @@
 #include "Reporter.h"
 #include "Func.h"
 
+#include <vector>
+#include <map>
+
 CompositeHash::CompositeHash(TypeList* composite_type)
 	{
 	type = composite_type;
@@ -174,12 +177,44 @@ char* CompositeHash::SingleValHash(int type_check, char* kp0,
 			{
 			int* kp = AlignAndPadType<int>(kp0);
 			TableVal* tv = v->AsTableVal();
-			ListVal* lv = tv->ConvertToList();
 			*kp = tv->Size();
 			kp1 = reinterpret_cast<char*>(kp+1);
-			for ( int i = 0; i < tv->Size(); ++i )
+
+			auto tbl = tv->AsTable();
+			auto it = tbl->InitForIteration();
+			ListVal* lv = new ListVal(TYPE_ANY);
+
+			struct HashKeyComparer {
+				bool operator()(const HashKey* a, const HashKey* b) const
+					{
+					if ( a->Hash() != b->Hash() )
+						return a->Hash() < b->Hash();
+					if ( a->Size() != b->Size() )
+						return a->Size() < b->Size();
+					return strncmp(static_cast<const char*>(a->Key()),
+					               static_cast<const char*>(b->Key()),
+					               a->Size()) < 0;
+					}
+			};
+
+			std::map<HashKey*, int, HashKeyComparer> hashkeys;
+			HashKey* k;
+			auto idx = 0;
+
+			while ( tbl->NextEntry(k, it) )
 				{
-				Val* key = lv->Index(i);
+				hashkeys[k] = idx++;
+				lv->Append(tv->RecoverIndex(k));
+				}
+
+			for ( auto& kv : hashkeys )
+				delete kv.first;
+
+			for ( auto& kv : hashkeys )
+				{
+				auto idx = kv.second;
+				Val* key = lv->Index(idx);
+
 				if ( ! (kp1 = SingleValHash(type_check, kp1, key->Type(), key,
 				                            false)) )
 					{

src/Expr.cc

@@ -1360,7 +1360,10 @@ SizeExpr::SizeExpr(Expr* arg_op) : UnaryExpr(EXPR_SIZE, arg_op)
 	if ( IsError() )
 		return;
 
-	SetType(base_type(TYPE_COUNT));
+	if ( op->Type()->InternalType() == TYPE_INTERNAL_DOUBLE )
+		SetType(op->Type()->Ref());
+	else
+		SetType(base_type(TYPE_COUNT));
 	}
 
 Val* SizeExpr::Eval(Frame* f) const

src/Val.cc

@@ -425,7 +425,7 @@ Val* Val::SizeVal() const
 		return val_mgr->GetCount(val.uint_val);
 
 	case TYPE_INTERNAL_DOUBLE:
-		return new Val(fabs(val.double_val), TYPE_DOUBLE);
+		return new Val(fabs(val.double_val), type->Tag());
 
 	case TYPE_INTERNAL_OTHER:
 		if ( type->Tag() == TYPE_FUNC )

src/parse.y

@@ -728,7 +728,12 @@ expr:
 	|       '|' expr '|'	%prec '('
 			{
 			set_location(@1, @3);
-			$$ = new SizeExpr($2);
+			auto e = $2;
+
+			if ( IsIntegral(e->Type()->Tag()) )
+				e = new ArithCoerceExpr(e, TYPE_INT);
+
+			$$ = new SizeExpr(e);
 			}
 
 	|       expr TOK_AS type

testing/btest/Baseline/language.interval/out

@@ -15,6 +15,7 @@ add different time units (PASS)
 subtract different time units (PASS)
 absolute value (PASS)
 absolute value (PASS)
+absolute value (PASS)
 assignment operator (PASS)
 assignment operator (PASS)
 multiplication operator (PASS)

testing/btest/Baseline/language.record-index-complex-fields/output

@@ -10,18 +10,18 @@ a
 }
 }
 {
-[a=4, tags_v=[0, 1], tags_t={
-[two] = 2,
-[one] = 1
-}, tags_s={
-b,
-a
-}],
 [a=13, tags_v=[, , 2, 3], tags_t={
 [five] = 5,
 [four] = 4
 }, tags_s={
 c,
 d
+}],
+[a=4, tags_v=[0, 1], tags_t={
+[two] = 2,
+[one] = 1
+}, tags_s={
+b,
+a
 }]
 }

testing/btest/Baseline/language.sizeof/output

@@ -2,6 +2,7 @@ IPv4 Address 1.2.3.4: 32
 IPv6 Address ::1: 128
 Boolean T: 1
 Count 10: 10
+Expr: 4
 Double -1.23: 1.230000
 Enum ENUM3: 2
 File 21.000000

testing/btest/language/interval.bro

@@ -70,8 +70,9 @@ event bro_init()
 	test_case( "compare different time units", in13 >= in35 );
 	test_case( "add different time units", in13 + in14 == 4min );
 	test_case( "subtract different time units", in24 - in23 == 0sec );
-	test_case( "absolute value", |in25| == 2.0*3600 );
-	test_case( "absolute value", |in36| == 2.5*86400 );
+	test_case( "absolute value", |in25| == 2hr );
+	test_case( "absolute value", |in36| == 2.5day );
+	test_case( "absolute value", |5sec - 9sec| == 4sec );
 	in34 += 2hr;
 	test_case( "assignment operator", in34 == 122min );
 	in34 -= 2hr;

testing/btest/language/nested-sets.bro

@@ -0,0 +1,19 @@
+# @TEST-EXEC: for i in `seq 21`; do echo 0 >> random.seed; done
+# @TEST-EXEC: test `bro -b -G random.seed %INPUT` = "pass"
+
+type r: record {
+	b: set[count];
+};
+
+global foo: set[r];
+global bar = set(1,3,5);
+
+add foo[record($b=bar)];
+
+bar = set(5,3,1);
+delete foo[record($b=bar)];
+
+if ( |foo| > 0 )
+	print "fail";
+else
+	print "pass";

testing/btest/language/sizeof.bro

@@ -64,6 +64,10 @@ print fmt("Boolean %s: %d", b, |b|);
 # Size of count: identity.
 print fmt("Count %s: %d", c, |c|);
 
+# Size of integral arithmetic expression should coerce to int before absolute
+# value operation to help prevent common unsigned int overflow situations.
+print fmt("Expr: %d", |5 - 9|);
+
 # Size of double: returns absolute value.
 print fmt("Double %s: %f", d, |d|);
 

testing/btest/language/time.bro

@@ -27,7 +27,7 @@ event bro_init()
 	test_case( "inequality", t1 != t3 );
 	test_case( "equality", t1 == t4 );
 	test_case( "subtract time", t2 - t1 == 3sec);
-	test_case( "size operator", |t5| == 1234567890.0 );
+	test_case( "size operator", |t5| == t5 );
 
 }