GH-839: Fix use of &optional sub-records within table/set indices

src/CompHash.cc

@@ -509,6 +509,9 @@ int CompositeHash::SingleTypeKeySize(Type* bt, const Val* v,
 
 		case TYPE_RECORD:
 			{
+			if ( ! v )
+				return (optional && ! calc_static_size) ? sz : 0;
+
 			const RecordVal* rv = v ? v->AsRecordVal() : nullptr;
 			RecordType* rt = bt->AsRecordType();
 			int num_fields = rt->NumFields();

testing/btest/Baseline/language.table-optional-subrecord-index/out

@@ -0,0 +1,18 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+2, {
+[subrec=[str=hi]],
+[subrec=<uninitialized>]
+}
+T
+T
+T
+T
+T
+0, {
+
+}
+T
+T
+T
+T
+T

testing/btest/language/table-optional-subrecord-index.zeek

@@ -0,0 +1,35 @@
+# @TEST-EXEC: zeek -b %INPUT >out
+# @TEST-EXEC: btest-diff out
+# @TEST-DOC: Check functionality of a table/set index consisting of optional sub-records.
+
+type SubRec: record {
+	str: string;
+};
+
+type Rec: record {
+	subrec: SubRec &optional;
+};
+
+global myset: set[Rec] = set();
+
+local i = Rec();
+local j = Rec($subrec=SubRec($str="hi"));
+add myset[i];
+add myset[j];
+print |myset|, myset;
+
+# All membership tests below are expected to evaluate to true.
+print i in myset;
+print j in myset;
+print Rec() in myset;
+print Rec($subrec=SubRec($str="hi")) in myset;
+print Rec($subrec=SubRec($str="no")) !in myset;
+
+delete myset[i];
+delete myset[j];
+print |myset|, myset;
+print i !in myset;
+print j !in myset;
+print Rec() !in myset;
+print Rec($subrec=SubRec($str="hi")) !in myset;
+print Rec($subrec=SubRec($str="no")) !in myset;