mirror of
https://github.com/zeek/zeek.git
synced 2025-10-09 01:58:20 +00:00
Support table deserialization in from_json()
This allows additional data roundtripping through JSON since to_json() already supports tables. There are some subtleties around the formatting of strings in JSON object keys, for which this adds a bit of helper infrastructure. This also expands the language.table test to verify the roundtrips, and adapts bif.from_json to include a table in the test record.
This commit is contained in:
Christian Kreibich
parent
df645e9bb2
commit
92c1098e97
6 changed files with 148 additions and 64 deletions
83
src/Val.cc
83
src/Val.cc
|
|
@ -330,7 +330,7 @@ TableValPtr Val::GetRecordFields() {
|
|||
}
|
||||
|
||||
// A predicate to identify those types we render as a string in JSON.
|
||||
static bool IsQuotedJSONType(const TypePtr& t) {
|
||||
static bool UsesJSONStringType(const TypePtr& t) {
|
||||
if ( t == nullptr )
|
||||
return false;
|
||||
|
||||
|
|
@ -440,8 +440,10 @@ static void BuildJSON(json::detail::NullDoubleWriter& writer, Val* val, bool onl
|
|||
BuildJSON(key_writer, entry_key, only_loggable, re);
|
||||
string key_str = buffer.GetString();
|
||||
|
||||
if ( key_str.length() >= 2 && key_str[0] == '"' && key_str[key_str.length() - 1] == '"' )
|
||||
// Strip quotes.
|
||||
// Strip the quotes for any type we render as a string. This
|
||||
// makes the JSON object's keys look more natural, yielding
|
||||
// '{ "foo": ... }', not '{ "\"foo\"": ... }', for such types.
|
||||
if ( UsesJSONStringType(entry_key->GetType()) )
|
||||
key_str = key_str.substr(1, key_str.length() - 2);
|
||||
|
||||
BuildJSON(writer, entry->GetVal().get(), only_loggable, re, key_str);
|
||||
|
|
@ -1064,34 +1066,69 @@ static std::variant<ValPtr, std::string> BuildVal(const rapidjson::Value& j, con
|
|||
}
|
||||
|
||||
case TYPE_TABLE: {
|
||||
if ( ! j.IsArray() )
|
||||
return mismatch_err();
|
||||
|
||||
if ( ! t->IsSet() )
|
||||
return util::fmt("tables are not supported");
|
||||
|
||||
auto tt = t->AsSetType();
|
||||
auto tl = tt->GetIndices();
|
||||
auto tt = t->AsTableType(); // The table vs set type does not matter below
|
||||
auto tv = make_intrusive<TableVal>(IntrusivePtr{NewRef{}, tt});
|
||||
auto tl = tt->GetIndices();
|
||||
|
||||
for ( const auto& item : j.GetArray() ) {
|
||||
std::variant<ValPtr, std::string> v;
|
||||
if ( t->IsSet() ) {
|
||||
if ( ! j.IsArray() )
|
||||
return mismatch_err();
|
||||
|
||||
if ( tl->GetTypes().size() == 1 )
|
||||
v = BuildVal(item, tl->GetPureType(), key_func);
|
||||
else
|
||||
v = BuildVal(item, tl, key_func);
|
||||
for ( const auto& item : j.GetArray() ) {
|
||||
std::variant<ValPtr, std::string> v;
|
||||
|
||||
if ( ! get_if<ValPtr>(&v) )
|
||||
return v;
|
||||
if ( tl->GetTypes().size() == 1 )
|
||||
v = BuildVal(item, tl->GetPureType(), key_func);
|
||||
else
|
||||
v = BuildVal(item, tl, key_func);
|
||||
|
||||
if ( ! std::get<ValPtr>(v) )
|
||||
continue;
|
||||
if ( ! get_if<ValPtr>(&v) )
|
||||
return v;
|
||||
if ( ! std::get<ValPtr>(v) )
|
||||
continue;
|
||||
|
||||
tv->Assign(std::move(std::get<ValPtr>(v)), nullptr);
|
||||
tv->Assign(std::move(std::get<ValPtr>(v)), nullptr);
|
||||
}
|
||||
|
||||
return tv;
|
||||
}
|
||||
else {
|
||||
if ( ! j.IsObject() )
|
||||
return mismatch_err();
|
||||
|
||||
return tv;
|
||||
for ( auto it = j.MemberBegin(); it != j.MemberEnd(); ++it ) {
|
||||
rapidjson::Document idxstr;
|
||||
idxstr.Parse(it->name.GetString(), it->name.GetStringLength());
|
||||
|
||||
std::variant<ValPtr, std::string> idx;
|
||||
|
||||
if ( tl->GetTypes().size() > 1 )
|
||||
idx = BuildVal(idxstr, tl, key_func);
|
||||
else if ( UsesJSONStringType(tl->GetPureType()) )
|
||||
// Parse this with the quotes the string came with. This
|
||||
// mirrors the quote-stripping in BuildJSON().
|
||||
idx = BuildVal(it->name, tl->GetPureType(), key_func);
|
||||
else
|
||||
// Parse the string's content, not the full JSON string.
|
||||
idx = BuildVal(idxstr, tl->GetPureType(), key_func);
|
||||
|
||||
if ( ! get_if<ValPtr>(&idx) )
|
||||
return idx;
|
||||
if ( ! std::get<ValPtr>(idx) )
|
||||
continue;
|
||||
|
||||
auto v = BuildVal(it->value, tt->Yield(), key_func);
|
||||
|
||||
if ( ! get_if<ValPtr>(&v) )
|
||||
return v;
|
||||
if ( ! std::get<ValPtr>(v) )
|
||||
continue;
|
||||
|
||||
tv->Assign(std::move(std::get<ValPtr>(idx)), std::move(std::get<ValPtr>(v)));
|
||||
}
|
||||
|
||||
return tv;
|
||||
}
|
||||
}
|
||||
|
||||
case TYPE_RECORD: {
|
||||
|
|
|
|||
|
|
@ -1,2 +1,2 @@
|
|||
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
|
||||
error in <...>/from_json.zeek, line 4: tables are not supported (from_json([], to_any_coerce table_string_of_string, from_json_default_key_mapper))
|
||||
error in <...>/from_json.zeek, line 4: cannot convert JSON type 'array' to Zeek type 'table' (from_json([], to_any_coerce table_string_of_string, from_json_default_key_mapper))
|
||||
|
|
|
|||
|
|
@ -5,4 +5,6 @@ aa:bb::/32,
|
|||
}, se={
|
||||
[192.168.0.1, 80/tcp] ,
|
||||
[2001:db8::1, 8080/udp]
|
||||
}, tbl={
|
||||
[192.168.0.1, 80/tcp] = foo
|
||||
}], valid=T]
|
||||
|
|
|
|||
|
|
@ -53,21 +53,29 @@ table index non-membership (PASS)
|
|||
table index lookup (PASS)
|
||||
table index reduced size (PASS)
|
||||
table index iteration (PASS)
|
||||
table index JSON roundtrip success (PASS)
|
||||
table index JSON roundtrip correct (PASS)
|
||||
vector index size (PASS)
|
||||
vector index membership (PASS)
|
||||
vector index non-membership (PASS)
|
||||
vector index lookup (PASS)
|
||||
vector index reduced size (PASS)
|
||||
vector index iteration (PASS)
|
||||
vector index JSON roundtrip success (PASS)
|
||||
vector index JSON roundtrip (PASS)
|
||||
set index size (PASS)
|
||||
set index membership (PASS)
|
||||
set index non-membership (PASS)
|
||||
set index lookup (PASS)
|
||||
set index reduced size (PASS)
|
||||
set index iteration (PASS)
|
||||
set index JSON roundtrip success (PASS)
|
||||
set index JSON roundtrip (PASS)
|
||||
pattern index size (PASS)
|
||||
pattern index membership (PASS)
|
||||
pattern index non-membership (PASS)
|
||||
pattern index lookup (PASS)
|
||||
pattern index reduced size (PASS)
|
||||
pattern index iteration (PASS)
|
||||
pattern index JSON roundtrip success (PASS)
|
||||
pattern index JSON roundtrip (PASS)
|
||||
|
|
|
|||
|
|
@ -29,11 +29,12 @@ type Foo: record {
|
|||
re: pattern;
|
||||
su: subnet_set;
|
||||
se: set[addr, port];
|
||||
tbl: table[addr, port] of string;
|
||||
};
|
||||
|
||||
event zeek_init()
|
||||
{
|
||||
local json = "{\"hello\":\"world\",\"t\":true,\"f\":false,\"se\":[[\"192.168.0.1\", \"80/tcp\"], [\"2001:db8::1\", \"8080/udp\"]],\"n\":null,\"i\":123,\"pi\":3.1416,\"a\":[\"1\",\"2\",\"3\",\"4\"],\"su\":[\"[aa:bb::0]/32\",\"192.168.0.0/16\"],\"c1\":\"A::Blue\",\"p\":\"1500/tcp\",\"it\":5000,\"ad\":\"127.0.0.1\",\"s\":\"[::1/128]\",\"re\":\"/a/\",\"ti\":1681652265.042767}";
|
||||
local json = "{\"hello\":\"world\",\"t\":true,\"f\":false,\"se\":[[\"192.168.0.1\", \"80/tcp\"], [\"2001:db8::1\", \"8080/udp\"]],\"n\":null,\"i\":123,\"pi\":3.1416,\"a\":[\"1\",\"2\",\"3\",\"4\"],\"su\":[\"[aa:bb::0]/32\",\"192.168.0.0/16\"],\"c1\":\"A::Blue\",\"p\":\"1500/tcp\",\"it\":5000,\"ad\":\"127.0.0.1\",\"s\":\"[::1/128]\",\"re\":\"/a/\",\"ti\":1681652265.042767,\"tbl\":{\"[\\\"192.168.0.1\\\",\\\"80/tcp\\\"]\":\"foo\"}}";
|
||||
print from_json(json, Foo);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -162,94 +162,130 @@ function basic_functionality()
|
|||
test_case( "!in operator", [cid, T] !in t11 );
|
||||
}
|
||||
|
||||
function complex_index_types()
|
||||
type tss_table: table[table[string] of string] of string;
|
||||
|
||||
function complex_index_type_table()
|
||||
{
|
||||
# Initialization
|
||||
local t1: table[table[string] of string] of string = {
|
||||
local t: tss_table = {
|
||||
[table(["k1"] = "v1")] = "res1"
|
||||
};
|
||||
|
||||
# Adding a member
|
||||
t1[table(["k2"] = "v2")] = "res2";
|
||||
t[table(["k2"] = "v2")] = "res2";
|
||||
|
||||
# Various checks, including membership test and lookup
|
||||
test_case( "table index size", |t1| == 2 );
|
||||
test_case( "table index membership", table(["k2"] = "v2") in t1 );
|
||||
test_case( "table index non-membership", table(["k2"] = "v3") !in t1 );
|
||||
test_case( "table index lookup", t1[table(["k2"] = "v2")] == "res2" );
|
||||
test_case( "table index size", |t| == 2 );
|
||||
test_case( "table index membership", table(["k2"] = "v2") in t );
|
||||
test_case( "table index non-membership", table(["k2"] = "v3") !in t );
|
||||
test_case( "table index lookup", t[table(["k2"] = "v2")] == "res2" );
|
||||
|
||||
# Member deletion
|
||||
delete t1[table(["k1"] = "v1")];
|
||||
test_case( "table index reduced size", |t1| == 1 );
|
||||
delete t[table(["k1"] = "v1")];
|
||||
test_case( "table index reduced size", |t| == 1 );
|
||||
|
||||
# Iteration
|
||||
for ( ti in t1 )
|
||||
for ( ti in t )
|
||||
{
|
||||
test_case( "table index iteration", to_json(ti) == to_json(table(["k2"] = "v2")) );
|
||||
break;
|
||||
}
|
||||
|
||||
# As above, for other index types
|
||||
local t2: table[vector of string] of string = {
|
||||
# JSON serialize/unserialize
|
||||
local fjr = from_json(to_json(t), tss_table);
|
||||
test_case( "table index JSON roundtrip success", fjr$valid );
|
||||
test_case( "table index JSON roundtrip correct", to_json(t) == to_json(fjr$v) );
|
||||
}
|
||||
|
||||
type vs_table: table[vector of string] of string;
|
||||
|
||||
function complex_index_type_vector()
|
||||
{
|
||||
local t: vs_table = {
|
||||
[vector("v1", "v2")] = "res1"
|
||||
};
|
||||
|
||||
t2[vector("v3", "v4")] = "res2";
|
||||
test_case( "vector index size", |t2| == 2 );
|
||||
test_case( "vector index membership", vector("v3", "v4") in t2 );
|
||||
test_case( "vector index non-membership", vector("v4", "v5") !in t2 );
|
||||
test_case( "vector index lookup", t2[vector("v3", "v4")] == "res2" );
|
||||
t[vector("v3", "v4")] = "res2";
|
||||
test_case( "vector index size", |t| == 2 );
|
||||
test_case( "vector index membership", vector("v3", "v4") in t );
|
||||
test_case( "vector index non-membership", vector("v4", "v5") !in t );
|
||||
test_case( "vector index lookup", t[vector("v3", "v4")] == "res2" );
|
||||
|
||||
delete t2[vector("v1", "v2")];
|
||||
test_case( "vector index reduced size", |t2| == 1 );
|
||||
delete t[vector("v1", "v2")];
|
||||
test_case( "vector index reduced size", |t| == 1 );
|
||||
|
||||
for ( vi in t2 )
|
||||
for ( vi in t )
|
||||
{
|
||||
test_case( "vector index iteration", to_json(vi) == to_json(vector("v3", "v4")) );
|
||||
break;
|
||||
}
|
||||
|
||||
local t3: table[set[string]] of string = {
|
||||
local fjr = from_json(to_json(t), vs_table);
|
||||
test_case( "vector index JSON roundtrip success", fjr$valid );
|
||||
test_case( "vector index JSON roundtrip", to_json(t) == to_json(fjr$v) );
|
||||
}
|
||||
|
||||
type ss_table: table[set[string]] of string;
|
||||
|
||||
function complex_index_type_set()
|
||||
{
|
||||
local t: ss_table = {
|
||||
[set("s1", "s2")] = "res1"
|
||||
};
|
||||
|
||||
t3[set("s3", "s4")] = "res2";
|
||||
test_case( "set index size", |t3| == 2 );
|
||||
test_case( "set index membership", set("s3", "s4") in t3 );
|
||||
test_case( "set index non-membership", set("s4", "s5") !in t3 );
|
||||
test_case( "set index lookup", t3[set("s3", "s4")] == "res2" );
|
||||
t[set("s3", "s4")] = "res2";
|
||||
test_case( "set index size", |t| == 2 );
|
||||
test_case( "set index membership", set("s3", "s4") in t );
|
||||
test_case( "set index non-membership", set("s4", "s5") !in t );
|
||||
test_case( "set index lookup", t[set("s3", "s4")] == "res2" );
|
||||
|
||||
delete t3[set("s1", "s2")];
|
||||
test_case( "set index reduced size", |t3| == 1 );
|
||||
delete t[set("s1", "s2")];
|
||||
test_case( "set index reduced size", |t| == 1 );
|
||||
|
||||
for ( si in t3 )
|
||||
for ( si in t )
|
||||
{
|
||||
test_case( "set index iteration", to_json(si) == to_json(set("s3", "s4")) );
|
||||
break;
|
||||
}
|
||||
|
||||
local t4: table[pattern] of string = {
|
||||
local fjr = from_json(to_json(t), ss_table);
|
||||
test_case( "set index JSON roundtrip success", fjr$valid );
|
||||
test_case( "set index JSON roundtrip", to_json(t) == to_json(fjr$v) );
|
||||
}
|
||||
|
||||
type tp_table: table[pattern] of string;
|
||||
|
||||
function complex_index_type_pattern()
|
||||
{
|
||||
local t: tp_table = {
|
||||
[/pat1/] = "res1"
|
||||
};
|
||||
|
||||
t4[/pat2/] = "res2";
|
||||
test_case( "pattern index size", |t4| == 2 );
|
||||
test_case( "pattern index membership", /pat2/ in t4 );
|
||||
test_case( "pattern index non-membership", /pat3/ !in t4 );
|
||||
test_case( "pattern index lookup", t4[/pat2/] == "res2" );
|
||||
t[/pat2/] = "res2";
|
||||
test_case( "pattern index size", |t| == 2 );
|
||||
test_case( "pattern index membership", /pat2/ in t );
|
||||
test_case( "pattern index non-membership", /pat3/ !in t );
|
||||
test_case( "pattern index lookup", t[/pat2/] == "res2" );
|
||||
|
||||
delete t4[/pat1/];
|
||||
test_case( "pattern index reduced size", |t4| == 1 );
|
||||
delete t[/pat1/];
|
||||
test_case( "pattern index reduced size", |t| == 1 );
|
||||
|
||||
for ( pi in t4 )
|
||||
for ( pi in t )
|
||||
{
|
||||
test_case( "pattern index iteration", to_json(pi) == to_json(/pat2/) );
|
||||
break;
|
||||
}
|
||||
|
||||
local fjr = from_json(to_json(t), tp_table);
|
||||
test_case( "pattern index JSON roundtrip success", fjr$valid );
|
||||
test_case( "pattern index JSON roundtrip", to_json(t) == to_json(fjr$v) );
|
||||
}
|
||||
|
||||
event zeek_init()
|
||||
{
|
||||
basic_functionality();
|
||||
complex_index_types();
|
||||
complex_index_type_table();
|
||||
complex_index_type_vector();
|
||||
complex_index_type_set();
|
||||
complex_index_type_pattern();
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue