Merge remote-tracking branch 'origin/topic/jsiwek/gh-151'

* origin/topic/jsiwek/gh-151:
  GH-151: fix hash calculation for nested sets

CHANGES

@@ -1,4 +1,14 @@
 
+2.6-92 | 2019-01-22 08:53:36 -0800
+
+  * GH-151: fix hash calculation for nested sets
+
+    Hash key construction of nested sets depended on the order in
+    which their elements are iterated, which varied even between sets
+    containing equivalent elements.  The iteration order is now sorted
+    by each element's hash value (or, on collision, by full key) such
+    that equivalent sets no longer hash differently. (Jon Siwek, Corelight)
+
 2.6-89 | 2019-01-18 15:17:34 -0800
 
   * Pre-allocate and re-use Vals for bool, int, count, enum and empty string (Jon Siwek, Corelight)

VERSION

@@ -1 +1 @@
-2.6-89
+2.6-92

src/CompHash.cc

@@ -7,6 +7,9 @@
 #include "Reporter.h"
 #include "Func.h"
 
+#include <vector>
+#include <map>
+
 CompositeHash::CompositeHash(TypeList* composite_type)
 	{
 	type = composite_type;
@@ -174,12 +177,44 @@ char* CompositeHash::SingleValHash(int type_check, char* kp0,
 			{
 			int* kp = AlignAndPadType<int>(kp0);
 			TableVal* tv = v->AsTableVal();
-			ListVal* lv = tv->ConvertToList();
 			*kp = tv->Size();
 			kp1 = reinterpret_cast<char*>(kp+1);
-			for ( int i = 0; i < tv->Size(); ++i )
+
+			auto tbl = tv->AsTable();
+			auto it = tbl->InitForIteration();
+			ListVal* lv = new ListVal(TYPE_ANY);
+
+			struct HashKeyComparer {
+				bool operator()(const HashKey* a, const HashKey* b) const
 					{
-				Val* key = lv->Index(i);
+					if ( a->Hash() != b->Hash() )
+						return a->Hash() < b->Hash();
+					if ( a->Size() != b->Size() )
+						return a->Size() < b->Size();
+					return strncmp(static_cast<const char*>(a->Key()),
+					               static_cast<const char*>(b->Key()),
+					               a->Size()) < 0;
+					}
+			};
+
+			std::map<HashKey*, int, HashKeyComparer> hashkeys;
+			HashKey* k;
+			auto idx = 0;
+
+			while ( tbl->NextEntry(k, it) )
+				{
+				hashkeys[k] = idx++;
+				lv->Append(tv->RecoverIndex(k));
+				}
+
+			for ( auto& kv : hashkeys )
+				delete kv.first;
+
+			for ( auto& kv : hashkeys )
+				{
+				auto idx = kv.second;
+				Val* key = lv->Index(idx);
+
 				if ( ! (kp1 = SingleValHash(type_check, kp1, key->Type(), key,
 				                            false)) )
 					{

testing/btest/Baseline/language.record-index-complex-fields/output

@@ -10,18 +10,18 @@ a
 }
 }
 {
-[a=4, tags_v=[0, 1], tags_t={
-[two] = 2,
-[one] = 1
-}, tags_s={
-b,
-a
-}],
 [a=13, tags_v=[, , 2, 3], tags_t={
 [five] = 5,
 [four] = 4
 }, tags_s={
 c,
 d
+}],
+[a=4, tags_v=[0, 1], tags_t={
+[two] = 2,
+[one] = 1
+}, tags_s={
+b,
+a
 }]
 }

testing/btest/language/nested-sets.bro

@@ -0,0 +1,19 @@
+# @TEST-EXEC: for i in `seq 21`; do echo 0 >> random.seed; done
+# @TEST-EXEC: test `bro -b -G random.seed %INPUT` = "pass"
+
+type r: record {
+	b: set[count];
+};
+
+global foo: set[r];
+global bar = set(1,3,5);
+
+add foo[record($b=bar)];
+
+bar = set(5,3,1);
+delete foo[record($b=bar)];
+
+if ( |foo| > 0 )
+	print "fail";
+else
+	print "pass";