Support table deserialization in from_json()

This allows additional data roundtripping through JSON since to_json() already
supports tables. There are some subtleties around the formatting of strings in
JSON object keys, for which this adds a bit of helper infrastructure.

This also expands the language.table test to verify the roundtrips, and adapts
bif.from_json to include a table in the test record.

src/Val.cc

@@ -330,7 +330,7 @@ TableValPtr Val::GetRecordFields() {
 }
 
 // A predicate to identify those types we render as a string in JSON.
-static bool IsQuotedJSONType(const TypePtr& t) {
+static bool UsesJSONStringType(const TypePtr& t) {
     if ( t == nullptr )
         return false;
 
@@ -440,8 +440,10 @@ static void BuildJSON(json::detail::NullDoubleWriter& writer, Val* val, bool onl
                     BuildJSON(key_writer, entry_key, only_loggable, re);
                     string key_str = buffer.GetString();
 
-                    if ( key_str.length() >= 2 && key_str[0] == '"' && key_str[key_str.length() - 1] == '"' )
+                    // Strip the quotes for any type we render as a string. This
-                        // Strip quotes.
+                    // makes the JSON object's keys look more natural, yielding
+                    // '{ "foo": ... }', not '{ "\"foo\"": ... }', for such types.
+                    if ( UsesJSONStringType(entry_key->GetType()) )
                         key_str = key_str.substr(1, key_str.length() - 2);
 
                     BuildJSON(writer, entry->GetVal().get(), only_loggable, re, key_str);
@@ -1064,16 +1066,14 @@ static std::variant<ValPtr, std::string> BuildVal(const rapidjson::Value& j, con
         }
 
         case TYPE_TABLE: {
+            auto tt = t->AsTableType(); // The table vs set type does not matter below
+            auto tv = make_intrusive<TableVal>(IntrusivePtr{NewRef{}, tt});
+            auto tl = tt->GetIndices();
+
+            if ( t->IsSet() ) {
                 if ( ! j.IsArray() )
                     return mismatch_err();
 
-            if ( ! t->IsSet() )
-                return util::fmt("tables are not supported");
-
-            auto tt = t->AsSetType();
-            auto tl = tt->GetIndices();
-            auto tv = make_intrusive<TableVal>(IntrusivePtr{NewRef{}, tt});
-
                 for ( const auto& item : j.GetArray() ) {
                     std::variant<ValPtr, std::string> v;
 
@@ -1084,7 +1084,6 @@ static std::variant<ValPtr, std::string> BuildVal(const rapidjson::Value& j, con
 
                     if ( ! get_if<ValPtr>(&v) )
                         return v;
-
                     if ( ! std::get<ValPtr>(v) )
                         continue;
 
@@ -1093,6 +1092,44 @@ static std::variant<ValPtr, std::string> BuildVal(const rapidjson::Value& j, con
 
                 return tv;
             }
+            else {
+                if ( ! j.IsObject() )
+                    return mismatch_err();
+
+                for ( auto it = j.MemberBegin(); it != j.MemberEnd(); ++it ) {
+                    rapidjson::Document idxstr;
+                    idxstr.Parse(it->name.GetString(), it->name.GetStringLength());
+
+                    std::variant<ValPtr, std::string> idx;
+
+                    if ( tl->GetTypes().size() > 1 )
+                        idx = BuildVal(idxstr, tl, key_func);
+                    else if ( UsesJSONStringType(tl->GetPureType()) )
+                        // Parse this with the quotes the string came with. This
+                        // mirrors the quote-stripping in BuildJSON().
+                        idx = BuildVal(it->name, tl->GetPureType(), key_func);
+                    else
+                        // Parse the string's content, not the full JSON string.
+                        idx = BuildVal(idxstr, tl->GetPureType(), key_func);
+
+                    if ( ! get_if<ValPtr>(&idx) )
+                        return idx;
+                    if ( ! std::get<ValPtr>(idx) )
+                        continue;
+
+                    auto v = BuildVal(it->value, tt->Yield(), key_func);
+
+                    if ( ! get_if<ValPtr>(&v) )
+                        return v;
+                    if ( ! std::get<ValPtr>(v) )
+                        continue;
+
+                    tv->Assign(std::move(std::get<ValPtr>(idx)), std::move(std::get<ValPtr>(v)));
+                }
+
+                return tv;
+            }
+        }
 
         case TYPE_RECORD: {
             if ( ! j.IsObject() )

testing/btest/Baseline/bifs.from_json-5/.stderr

@@ -1,2 +1,2 @@
 ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
-error in <...>/from_json.zeek, line 4: tables are not supported (from_json([], to_any_coerce table_string_of_string, from_json_default_key_mapper))
+error in <...>/from_json.zeek, line 4: cannot convert JSON type 'array' to Zeek type 'table' (from_json([], to_any_coerce table_string_of_string, from_json_default_key_mapper))

testing/btest/Baseline/bifs.from_json/.stdout

@@ -5,4 +5,6 @@ aa:bb::/32,
 }, se={
 [192.168.0.1, 80/tcp] ,
 [2001:db8::1, 8080/udp] 
+}, tbl={
+[192.168.0.1, 80/tcp] = foo
 }], valid=T]

testing/btest/Baseline/language.table/out

@@ -53,21 +53,29 @@ table index non-membership (PASS)
 table index lookup (PASS)
 table index reduced size (PASS)
 table index iteration (PASS)
+table index JSON roundtrip success (PASS)
+table index JSON roundtrip correct (PASS)
 vector index size (PASS)
 vector index membership (PASS)
 vector index non-membership (PASS)
 vector index lookup (PASS)
 vector index reduced size (PASS)
 vector index iteration (PASS)
+vector index JSON roundtrip success (PASS)
+vector index JSON roundtrip (PASS)
 set index size (PASS)
 set index membership (PASS)
 set index non-membership (PASS)
 set index lookup (PASS)
 set index reduced size (PASS)
 set index iteration (PASS)
+set index JSON roundtrip success (PASS)
+set index JSON roundtrip (PASS)
 pattern index size (PASS)
 pattern index membership (PASS)
 pattern index non-membership (PASS)
 pattern index lookup (PASS)
 pattern index reduced size (PASS)
 pattern index iteration (PASS)
+pattern index JSON roundtrip success (PASS)
+pattern index JSON roundtrip (PASS)

testing/btest/bifs/from_json.zeek

@@ -29,11 +29,12 @@ type Foo: record {
 	re: pattern;
 	su: subnet_set;
 	se: set[addr, port];
+	tbl: table[addr, port] of string;
 };
 
 event zeek_init()
 	{
-	local json = "{\"hello\":\"world\",\"t\":true,\"f\":false,\"se\":[[\"192.168.0.1\", \"80/tcp\"], [\"2001:db8::1\", \"8080/udp\"]],\"n\":null,\"i\":123,\"pi\":3.1416,\"a\":[\"1\",\"2\",\"3\",\"4\"],\"su\":[\"[aa:bb::0]/32\",\"192.168.0.0/16\"],\"c1\":\"A::Blue\",\"p\":\"1500/tcp\",\"it\":5000,\"ad\":\"127.0.0.1\",\"s\":\"[::1/128]\",\"re\":\"/a/\",\"ti\":1681652265.042767}";
+	local json = "{\"hello\":\"world\",\"t\":true,\"f\":false,\"se\":[[\"192.168.0.1\", \"80/tcp\"], [\"2001:db8::1\", \"8080/udp\"]],\"n\":null,\"i\":123,\"pi\":3.1416,\"a\":[\"1\",\"2\",\"3\",\"4\"],\"su\":[\"[aa:bb::0]/32\",\"192.168.0.0/16\"],\"c1\":\"A::Blue\",\"p\":\"1500/tcp\",\"it\":5000,\"ad\":\"127.0.0.1\",\"s\":\"[::1/128]\",\"re\":\"/a/\",\"ti\":1681652265.042767,\"tbl\":{\"[\\\"192.168.0.1\\\",\\\"80/tcp\\\"]\":\"foo\"}}";
 	print from_json(json, Foo);
 	}
 

testing/btest/language/table.zeek

@@ -162,94 +162,130 @@ function basic_functionality()
 	test_case( "!in operator", [cid, T] !in t11 );
 }
 
-function complex_index_types()
+type tss_table: table[table[string] of string] of string;
+
+function complex_index_type_table()
 {
 	# Initialization
-	local t1: table[table[string] of string] of string = {
+	local t: tss_table = {
 		[table(["k1"] = "v1")] = "res1"
 	};
 
 	# Adding a member
-	t1[table(["k2"] = "v2")] = "res2";
+	t[table(["k2"] = "v2")] = "res2";
 
 	# Various checks, including membership test and lookup
-	test_case( "table index size", |t1| == 2 );
+	test_case( "table index size", |t| == 2 );
-	test_case( "table index membership", table(["k2"] = "v2") in t1 );
+	test_case( "table index membership", table(["k2"] = "v2") in t );
-	test_case( "table index non-membership", table(["k2"] = "v3") !in t1 );
+	test_case( "table index non-membership", table(["k2"] = "v3") !in t );
-	test_case( "table index lookup", t1[table(["k2"] = "v2")] == "res2" );
+	test_case( "table index lookup", t[table(["k2"] = "v2")] == "res2" );
 
 	# Member deletion
-	delete t1[table(["k1"] = "v1")];
+	delete t[table(["k1"] = "v1")];
-	test_case( "table index reduced size", |t1| == 1 );
+	test_case( "table index reduced size", |t| == 1 );
 
 	# Iteration
-	for ( ti in t1 )
+	for ( ti in t )
 		{
 		test_case( "table index iteration", to_json(ti) == to_json(table(["k2"] = "v2")) );
 		break;
 		}
 
-	# As above, for other index types
+	# JSON serialize/unserialize
-	local t2: table[vector of string] of string = {
+	local fjr = from_json(to_json(t), tss_table);
+	test_case( "table index JSON roundtrip success", fjr$valid );
+	test_case( "table index JSON roundtrip correct", to_json(t) == to_json(fjr$v) );
+}
+
+type vs_table: table[vector of string] of string;
+
+function complex_index_type_vector()
+{
+	local t: vs_table = {
 		[vector("v1", "v2")] = "res1"
 	};
 
-	t2[vector("v3", "v4")] = "res2";
+	t[vector("v3", "v4")] = "res2";
-	test_case( "vector index size", |t2| == 2 );
+	test_case( "vector index size", |t| == 2 );
-	test_case( "vector index membership", vector("v3", "v4") in t2 );
+	test_case( "vector index membership", vector("v3", "v4") in t );
-	test_case( "vector index non-membership", vector("v4", "v5") !in t2 );
+	test_case( "vector index non-membership", vector("v4", "v5") !in t );
-	test_case( "vector index lookup", t2[vector("v3", "v4")] == "res2" );
+	test_case( "vector index lookup", t[vector("v3", "v4")] == "res2" );
 
-	delete t2[vector("v1", "v2")];
+	delete t[vector("v1", "v2")];
-	test_case( "vector index reduced size", |t2| == 1 );
+	test_case( "vector index reduced size", |t| == 1 );
 
-	for ( vi in t2 )
+	for ( vi in t )
 		{
 		test_case( "vector index iteration", to_json(vi) == to_json(vector("v3", "v4")) );
 		break;
 		}
 
-	local t3: table[set[string]] of string = {
+	local fjr = from_json(to_json(t), vs_table);
+	test_case( "vector index JSON roundtrip success", fjr$valid );
+	test_case( "vector index JSON roundtrip", to_json(t) == to_json(fjr$v) );
+}
+
+type ss_table: table[set[string]] of string;
+
+function complex_index_type_set()
+{
+	local t: ss_table = {
 		[set("s1", "s2")] = "res1"
 	};
 
-	t3[set("s3", "s4")] = "res2";
+	t[set("s3", "s4")] = "res2";
-	test_case( "set index size", |t3| == 2 );
+	test_case( "set index size", |t| == 2 );
-	test_case( "set index membership", set("s3", "s4") in t3 );
+	test_case( "set index membership", set("s3", "s4") in t );
-	test_case( "set index non-membership", set("s4", "s5") !in t3 );
+	test_case( "set index non-membership", set("s4", "s5") !in t );
-	test_case( "set index lookup", t3[set("s3", "s4")] == "res2" );
+	test_case( "set index lookup", t[set("s3", "s4")] == "res2" );
 
-	delete t3[set("s1", "s2")];
+	delete t[set("s1", "s2")];
-	test_case( "set index reduced size", |t3| == 1 );
+	test_case( "set index reduced size", |t| == 1 );
 
-	for ( si in t3 )
+	for ( si in t )
 		{
 		test_case( "set index iteration", to_json(si) == to_json(set("s3", "s4")) );
 		break;
 		}
 
-	local t4: table[pattern] of string = {
+	local fjr = from_json(to_json(t), ss_table);
+	test_case( "set index JSON roundtrip success", fjr$valid );
+	test_case( "set index JSON roundtrip", to_json(t) == to_json(fjr$v) );
+}
+
+type tp_table: table[pattern] of string;
+
+function complex_index_type_pattern()
+{
+	local t: tp_table = {
 		[/pat1/] = "res1"
 	};
 
-	t4[/pat2/] = "res2";
+	t[/pat2/] = "res2";
-	test_case( "pattern index size", |t4| == 2 );
+	test_case( "pattern index size", |t| == 2 );
-	test_case( "pattern index membership", /pat2/ in t4 );
+	test_case( "pattern index membership", /pat2/ in t );
-	test_case( "pattern index non-membership", /pat3/ !in t4 );
+	test_case( "pattern index non-membership", /pat3/ !in t );
-	test_case( "pattern index lookup", t4[/pat2/] == "res2" );
+	test_case( "pattern index lookup", t[/pat2/] == "res2" );
 
-	delete t4[/pat1/];
+	delete t[/pat1/];
-	test_case( "pattern index reduced size", |t4| == 1 );
+	test_case( "pattern index reduced size", |t| == 1 );
 
-	for ( pi in t4 )
+	for ( pi in t )
 		{
 		test_case( "pattern index iteration", to_json(pi) == to_json(/pat2/) );
 		break;
 		}
+
+	local fjr = from_json(to_json(t), tp_table);
+	test_case( "pattern index JSON roundtrip success", fjr$valid );
+	test_case( "pattern index JSON roundtrip", to_json(t) == to_json(fjr$v) );
 }
 
 event zeek_init()
 {
 	basic_functionality();
-	complex_index_types();
+	complex_index_type_table();
+	complex_index_type_vector();
+	complex_index_type_set();
+	complex_index_type_pattern();
 }