Mirror/zeek
1
0
Fork 0
mirror of https://github.com/zeek/zeek.git synced 2025-10-09 01:58:20 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

Support table deserialization in from_json()

Browse source 
This allows additional data roundtripping through JSON since to_json() already
supports tables. There are some subtleties around the formatting of strings in
JSON object keys, for which this adds a bit of helper infrastructure.

This also expands the language.table test to verify the roundtrips, and adapts
bif.from_json to include a table in the test record.
This commit is contained in:
Christian Kreibich 2024-06-28 15:24:52 -07:00
parent df645e9bb2
commit 92c1098e97
6 changed files with 148 additions and 64 deletions
Download patch file Download diff file Expand all files Collapse all files

59
src/Val.cc
View file

@ -330,7 +330,7 @@ TableValPtr Val::GetRecordFields() {
}
// A predicate to identify those types we render as a string in JSON.
static bool IsQuotedJSONType(const TypePtr& t) {
static bool UsesJSONStringType(const TypePtr& t) {
if ( t == nullptr )
return false;
@ -440,8 +440,10 @@ static void BuildJSON(json::detail::NullDoubleWriter& writer, Val* val, bool onl
BuildJSON(key_writer, entry_key, only_loggable, re);
string key_str = buffer.GetString();
if ( key_str.length() >= 2 && key_str[0] == '"' && key_str[key_str.length() - 1] == '"' )
// Strip quotes.
// Strip the quotes for any type we render as a string. This
// makes the JSON object's keys look more natural, yielding
// '{ "foo": ... }', not '{ "\"foo\"": ... }', for such types.
if ( UsesJSONStringType(entry_key->GetType()) )
key_str = key_str.substr(1, key_str.length() - 2);
BuildJSON(writer, entry->GetVal().get(), only_loggable, re, key_str);
@ -1064,16 +1066,14 @@ static std::variant<ValPtr, std::string> BuildVal(const rapidjson::Value& j, con
}
case TYPE_TABLE: {
auto tt = t->AsTableType(); // The table vs set type does not matter below
auto tv = make_intrusive<TableVal>(IntrusivePtr{NewRef{}, tt});
auto tl = tt->GetIndices();
if ( t->IsSet() ) {
if ( ! j.IsArray() )
return mismatch_err();
if ( ! t->IsSet() )
return util::fmt("tables are not supported");
auto tt = t->AsSetType();
auto tl = tt->GetIndices();
auto tv = make_intrusive<TableVal>(IntrusivePtr{NewRef{}, tt});
for ( const auto& item : j.GetArray() ) {
std::variant<ValPtr, std::string> v;
@ -1084,7 +1084,6 @@ static std::variant<ValPtr, std::string> BuildVal(const rapidjson::Value& j, con
if ( ! get_if<ValPtr>(&v) )
return v;
if ( ! std::get<ValPtr>(v) )
continue;
@ -1093,6 +1092,44 @@ static std::variant<ValPtr, std::string> BuildVal(const rapidjson::Value& j, con
return tv;
}
else {
if ( ! j.IsObject() )
return mismatch_err();
for ( auto it = j.MemberBegin(); it != j.MemberEnd(); ++it ) {
rapidjson::Document idxstr;
idxstr.Parse(it->name.GetString(), it->name.GetStringLength());
std::variant<ValPtr, std::string> idx;
if ( tl->GetTypes().size() > 1 )
idx = BuildVal(idxstr, tl, key_func);
else if ( UsesJSONStringType(tl->GetPureType()) )
// Parse this with the quotes the string came with. This
// mirrors the quote-stripping in BuildJSON().
idx = BuildVal(it->name, tl->GetPureType(), key_func);
else
// Parse the string's content, not the full JSON string.
idx = BuildVal(idxstr, tl->GetPureType(), key_func);
if ( ! get_if<ValPtr>(&idx) )
return idx;
if ( ! std::get<ValPtr>(idx) )
continue;
auto v = BuildVal(it->value, tt->Yield(), key_func);
if ( ! get_if<ValPtr>(&v) )
return v;
if ( ! std::get<ValPtr>(v) )
continue;
tv->Assign(std::move(std::get<ValPtr>(idx)), std::move(std::get<ValPtr>(v)));
}
return tv;
}
}
case TYPE_RECORD: {
if ( ! j.IsObject() )

2
testing/btest/Baseline/bifs.from_json-5/.stderr
View file

@ -1,2 +1,2 @@
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
error in <...>/from_json.zeek, line 4: tables are not supported (from_json([], to_any_coerce table_string_of_string, from_json_default_key_mapper))
error in <...>/from_json.zeek, line 4: cannot convert JSON type 'array' to Zeek type 'table' (from_json([], to_any_coerce table_string_of_string, from_json_default_key_mapper))

2
testing/btest/Baseline/bifs.from_json/.stdout
View file

@ -5,4 +5,6 @@ aa:bb::/32,
}, se={
[192.168.0.1, 80/tcp] ,
[2001:db8::1, 8080/udp]
}, tbl={
[192.168.0.1, 80/tcp] = foo
}], valid=T]

8
testing/btest/Baseline/language.table/out
View file

@ -53,21 +53,29 @@ table index non-membership (PASS)
table index lookup (PASS)
table index reduced size (PASS)
table index iteration (PASS)
table index JSON roundtrip success (PASS)
table index JSON roundtrip correct (PASS)
vector index size (PASS)
vector index membership (PASS)
vector index non-membership (PASS)
vector index lookup (PASS)
vector index reduced size (PASS)
vector index iteration (PASS)
vector index JSON roundtrip success (PASS)
vector index JSON roundtrip (PASS)
set index size (PASS)
set index membership (PASS)
set index non-membership (PASS)
set index lookup (PASS)
set index reduced size (PASS)
set index iteration (PASS)
set index JSON roundtrip success (PASS)
set index JSON roundtrip (PASS)
pattern index size (PASS)
pattern index membership (PASS)
pattern index non-membership (PASS)
pattern index lookup (PASS)
pattern index reduced size (PASS)
pattern index iteration (PASS)
pattern index JSON roundtrip success (PASS)
pattern index JSON roundtrip (PASS)

3
testing/btest/bifs/from_json.zeek
View file

@ -29,11 +29,12 @@ type Foo: record {
re: pattern;
su: subnet_set;
se: set[addr, port];
tbl: table[addr, port] of string;
};
event zeek_init()
{
local json = "{\"hello\":\"world\",\"t\":true,\"f\":false,\"se\":[[\"192.168.0.1\", \"80/tcp\"], [\"2001:db8::1\", \"8080/udp\"]],\"n\":null,\"i\":123,\"pi\":3.1416,\"a\":[\"1\",\"2\",\"3\",\"4\"],\"su\":[\"[aa:bb::0]/32\",\"192.168.0.0/16\"],\"c1\":\"A::Blue\",\"p\":\"1500/tcp\",\"it\":5000,\"ad\":\"127.0.0.1\",\"s\":\"[::1/128]\",\"re\":\"/a/\",\"ti\":1681652265.042767}";
local json = "{\"hello\":\"world\",\"t\":true,\"f\":false,\"se\":[[\"192.168.0.1\", \"80/tcp\"], [\"2001:db8::1\", \"8080/udp\"]],\"n\":null,\"i\":123,\"pi\":3.1416,\"a\":[\"1\",\"2\",\"3\",\"4\"],\"su\":[\"[aa:bb::0]/32\",\"192.168.0.0/16\"],\"c1\":\"A::Blue\",\"p\":\"1500/tcp\",\"it\":5000,\"ad\":\"127.0.0.1\",\"s\":\"[::1/128]\",\"re\":\"/a/\",\"ti\":1681652265.042767,\"tbl\":{\"[\\\"192.168.0.1\\\",\\\"80/tcp\\\"]\":\"foo\"}}";
print from_json(json, Foo);
}

114
testing/btest/language/table.zeek
View file

@ -162,94 +162,130 @@ function basic_functionality()
test_case( "!in operator", [cid, T] !in t11 );
}
function complex_index_types()
type tss_table: table[table[string] of string] of string;
function complex_index_type_table()
{
# Initialization
local t1: table[table[string] of string] of string = {
local t: tss_table = {
[table(["k1"] = "v1")] = "res1"
};
# Adding a member
t1[table(["k2"] = "v2")] = "res2";
t[table(["k2"] = "v2")] = "res2";
# Various checks, including membership test and lookup
test_case( "table index size", |t1| == 2 );
test_case( "table index membership", table(["k2"] = "v2") in t1 );
test_case( "table index non-membership", table(["k2"] = "v3") !in t1 );
test_case( "table index lookup", t1[table(["k2"] = "v2")] == "res2" );
test_case( "table index size", |t| == 2 );
test_case( "table index membership", table(["k2"] = "v2") in t );
test_case( "table index non-membership", table(["k2"] = "v3") !in t );
test_case( "table index lookup", t[table(["k2"] = "v2")] == "res2" );
# Member deletion
delete t1[table(["k1"] = "v1")];
test_case( "table index reduced size", |t1| == 1 );
delete t[table(["k1"] = "v1")];
test_case( "table index reduced size", |t| == 1 );
# Iteration
for ( ti in t1 )
for ( ti in t )
{
test_case( "table index iteration", to_json(ti) == to_json(table(["k2"] = "v2")) );
break;
}
# As above, for other index types
local t2: table[vector of string] of string = {
# JSON serialize/unserialize
local fjr = from_json(to_json(t), tss_table);
test_case( "table index JSON roundtrip success", fjr$valid );
test_case( "table index JSON roundtrip correct", to_json(t) == to_json(fjr$v) );
}
type vs_table: table[vector of string] of string;
function complex_index_type_vector()
{
local t: vs_table = {
[vector("v1", "v2")] = "res1"
};
t2[vector("v3", "v4")] = "res2";
test_case( "vector index size", |t2| == 2 );
test_case( "vector index membership", vector("v3", "v4") in t2 );
test_case( "vector index non-membership", vector("v4", "v5") !in t2 );
test_case( "vector index lookup", t2[vector("v3", "v4")] == "res2" );
t[vector("v3", "v4")] = "res2";
test_case( "vector index size", |t| == 2 );
test_case( "vector index membership", vector("v3", "v4") in t );
test_case( "vector index non-membership", vector("v4", "v5") !in t );
test_case( "vector index lookup", t[vector("v3", "v4")] == "res2" );
delete t2[vector("v1", "v2")];
test_case( "vector index reduced size", |t2| == 1 );
delete t[vector("v1", "v2")];
test_case( "vector index reduced size", |t| == 1 );
for ( vi in t2 )
for ( vi in t )
{
test_case( "vector index iteration", to_json(vi) == to_json(vector("v3", "v4")) );
break;
}
local t3: table[set[string]] of string = {
local fjr = from_json(to_json(t), vs_table);
test_case( "vector index JSON roundtrip success", fjr$valid );
test_case( "vector index JSON roundtrip", to_json(t) == to_json(fjr$v) );
}
type ss_table: table[set[string]] of string;
function complex_index_type_set()
{
local t: ss_table = {
[set("s1", "s2")] = "res1"
};
t3[set("s3", "s4")] = "res2";
test_case( "set index size", |t3| == 2 );
test_case( "set index membership", set("s3", "s4") in t3 );
test_case( "set index non-membership", set("s4", "s5") !in t3 );
test_case( "set index lookup", t3[set("s3", "s4")] == "res2" );
t[set("s3", "s4")] = "res2";
test_case( "set index size", |t| == 2 );
test_case( "set index membership", set("s3", "s4") in t );
test_case( "set index non-membership", set("s4", "s5") !in t );
test_case( "set index lookup", t[set("s3", "s4")] == "res2" );
delete t3[set("s1", "s2")];
test_case( "set index reduced size", |t3| == 1 );
delete t[set("s1", "s2")];
test_case( "set index reduced size", |t| == 1 );
for ( si in t3 )
for ( si in t )
{
test_case( "set index iteration", to_json(si) == to_json(set("s3", "s4")) );
break;
}
local t4: table[pattern] of string = {
local fjr = from_json(to_json(t), ss_table);
test_case( "set index JSON roundtrip success", fjr$valid );
test_case( "set index JSON roundtrip", to_json(t) == to_json(fjr$v) );
}
type tp_table: table[pattern] of string;
function complex_index_type_pattern()
{
local t: tp_table = {
[/pat1/] = "res1"
};
t4[/pat2/] = "res2";
test_case( "pattern index size", |t4| == 2 );
test_case( "pattern index membership", /pat2/ in t4 );
test_case( "pattern index non-membership", /pat3/ !in t4 );
test_case( "pattern index lookup", t4[/pat2/] == "res2" );
t[/pat2/] = "res2";
test_case( "pattern index size", |t| == 2 );
test_case( "pattern index membership", /pat2/ in t );
test_case( "pattern index non-membership", /pat3/ !in t );
test_case( "pattern index lookup", t[/pat2/] == "res2" );
delete t4[/pat1/];
test_case( "pattern index reduced size", |t4| == 1 );
delete t[/pat1/];
test_case( "pattern index reduced size", |t| == 1 );
for ( pi in t4 )
for ( pi in t )
{
test_case( "pattern index iteration", to_json(pi) == to_json(/pat2/) );
break;
}
local fjr = from_json(to_json(t), tp_table);
test_case( "pattern index JSON roundtrip success", fjr$valid );
test_case( "pattern index JSON roundtrip", to_json(t) == to_json(fjr$v) );
}
event zeek_init()
{
basic_functionality();
complex_index_types();
complex_index_type_table();
complex_index_type_vector();
complex_index_type_set();
complex_index_type_pattern();
}