Mirror/zeek
1
0
Fork 0
mirror of https://github.com/zeek/zeek.git synced 2025-10-02 06:38:20 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Merge remote-tracking branch 'origin/topic/timw/json-interval-conversion'

Browse source 
* origin/topic/timw/json-interval-conversion:
  Add interval_as_double argument to control how intervals are converted to JSON
  Add btest for round-trip JSON conversion
  Allow comparing two PatternVals
  Handle conversion between data from Val::ToJSON and ValFromJSON better
This commit is contained in:
Tim Wojtulewicz 2024-12-03 10:10:15 -07:00
commit deb2acaeda
8 changed files with 220 additions and 23 deletions
Download patch file Download diff file Expand all files Collapse all files

10
CHANGES
View file

@ -1,3 +1,13 @@
7.1.0-dev.654 | 2024-12-03 10:10:15 -0700
* Add interval_as_double argument to control how intervals are converted to JSON (Tim Wojtulewicz, Corelight)
* Add btest for round-trip JSON conversion (Tim Wojtulewicz, Corelight)
* Allow comparing two PatternVals (Tim Wojtulewicz, Corelight)
* Handle conversion between data from Val::ToJSON and ValFromJSON better (Tim Wojtulewicz, Corelight)
7.1.0-dev.649 | 2024-12-02 13:43:26 +0100
* added new Cluster:: BiFs to script optimization tracking (Vern Paxson, Corelight)

2
VERSION
View file

@ -1 +1 @@
7.1.0-dev.649
7.1.0-dev.654

20
src/Expr.cc
View file

@ -1965,6 +1965,7 @@ EqExpr::EqExpr(ExprTag arg_tag, ExprPtr arg_op1, ExprPtr arg_op2)
case TYPE_ADDR:
case TYPE_SUBNET:
case TYPE_ERROR:
case TYPE_PATTERN:
case TYPE_FUNC: break;
case TYPE_ENUM:
@ -1996,12 +1997,19 @@ EqExpr::EqExpr(ExprTag arg_tag, ExprPtr arg_op1, ExprPtr arg_op2)
ValPtr EqExpr::Fold(Val* v1, Val* v2) const {
if ( op1->GetType()->Tag() == TYPE_PATTERN ) {
auto re = v1->As<PatternVal*>();
const String* s = v2->AsString();
if ( tag == EXPR_EQ )
return val_mgr->Bool(re->MatchExactly(s));
else
return val_mgr->Bool(! re->MatchExactly(s));
if ( op2->GetType()->Tag() == TYPE_PATTERN ) {
auto re1 = v1->As<PatternVal*>();
auto re2 = v2->As<PatternVal*>();
return val_mgr->Bool(strcmp(re1->Get()->PatternText(), re2->Get()->PatternText()) == 0);
}
else {
auto re = v1->As<PatternVal*>();
const String* s = v2->AsString();
if ( tag == EXPR_EQ )
return val_mgr->Bool(re->MatchExactly(s));
else
return val_mgr->Bool(! re->MatchExactly(s));
}
}
else if ( op1->GetType()->Tag() == TYPE_FUNC ) {
auto res = v1->AsFunc() == v2->AsFunc();

71
src/Val.cc
View file

@ -351,7 +351,7 @@ static bool UsesJSONStringType(const TypePtr& t) {
// This is a static method in this file to avoid including rapidjson's headers
// in Val.h, because they're huge.
static void BuildJSON(json::detail::NullDoubleWriter& writer, Val* val, bool only_loggable = false,
RE_Matcher* re = nullptr, const string& key = "") {
RE_Matcher* re = nullptr, const string& key = "", bool interval_as_double = false) {
if ( ! key.empty() )
writer.Key(key);
@ -387,7 +387,6 @@ static void BuildJSON(json::detail::NullDoubleWriter& writer, Val* val, bool onl
}
case TYPE_PATTERN:
case TYPE_INTERVAL:
case TYPE_ADDR:
case TYPE_SUBNET: {
ODesc d;
@ -397,6 +396,18 @@ static void BuildJSON(json::detail::NullDoubleWriter& writer, Val* val, bool onl
break;
}
case TYPE_INTERVAL: {
if ( interval_as_double )
writer.Double(val->AsInterval());
else {
ODesc d;
d.SetStyle(RAW_STYLE);
val->Describe(&d);
writer.String(reinterpret_cast<const char*>(d.Bytes()), d.Len());
}
break;
}
case TYPE_FILE:
case TYPE_FUNC:
case TYPE_ENUM:
@ -433,11 +444,11 @@ static void BuildJSON(json::detail::NullDoubleWriter& writer, Val* val, bool onl
Val* entry_key = lv->Length() == 1 ? lv->Idx(0).get() : lv.get();
if ( tval->GetType()->IsSet() )
BuildJSON(writer, entry_key, only_loggable, re);
BuildJSON(writer, entry_key, only_loggable, re, "", interval_as_double);
else {
rapidjson::StringBuffer buffer;
json::detail::NullDoubleWriter key_writer(buffer);
BuildJSON(key_writer, entry_key, only_loggable, re);
BuildJSON(key_writer, entry_key, only_loggable, re, "", interval_as_double);
string key_str = buffer.GetString();
// Strip the quotes for any type we render as a string. This
@ -446,7 +457,7 @@ static void BuildJSON(json::detail::NullDoubleWriter& writer, Val* val, bool onl
if ( UsesJSONStringType(entry_key->GetType()) )
key_str = key_str.substr(1, key_str.length() - 2);
BuildJSON(writer, entry->GetVal().get(), only_loggable, re, key_str);
BuildJSON(writer, entry->GetVal().get(), only_loggable, re, key_str, interval_as_double);
}
}
@ -481,7 +492,7 @@ static void BuildJSON(json::detail::NullDoubleWriter& writer, Val* val, bool onl
else
key_str = field_name;
BuildJSON(writer, value.get(), only_loggable, re, key_str);
BuildJSON(writer, value.get(), only_loggable, re, key_str, interval_as_double);
}
}
@ -495,7 +506,7 @@ static void BuildJSON(json::detail::NullDoubleWriter& writer, Val* val, bool onl
auto* lval = val->AsListVal();
size_t size = lval->Length();
for ( size_t i = 0; i < size; i++ )
BuildJSON(writer, lval->Idx(i).get(), only_loggable, re);
BuildJSON(writer, lval->Idx(i).get(), only_loggable, re, "", interval_as_double);
writer.EndArray();
break;
@ -507,7 +518,7 @@ static void BuildJSON(json::detail::NullDoubleWriter& writer, Val* val, bool onl
auto* vval = val->AsVectorVal();
size_t size = vval->SizeVal()->AsCount();
for ( size_t i = 0; i < size; i++ )
BuildJSON(writer, vval->ValAt(i).get(), only_loggable, re);
BuildJSON(writer, vval->ValAt(i).get(), only_loggable, re, "", interval_as_double);
writer.EndArray();
break;
@ -528,11 +539,11 @@ static void BuildJSON(json::detail::NullDoubleWriter& writer, Val* val, bool onl
}
}
StringValPtr Val::ToJSON(bool only_loggable, RE_Matcher* re) {
StringValPtr Val::ToJSON(bool only_loggable, RE_Matcher* re, bool interval_as_double) {
rapidjson::StringBuffer buffer;
json::detail::NullDoubleWriter writer(buffer);
BuildJSON(writer, this, only_loggable, re, "");
BuildJSON(writer, this, only_loggable, re, "", interval_as_double);
return make_intrusive<StringVal>(buffer.GetString());
}
@ -938,10 +949,44 @@ static std::variant<ValPtr, std::string> BuildVal(const rapidjson::Value& j, con
}
case TYPE_INTERVAL: {
if ( ! j.IsNumber() )
return mismatch_err();
if ( j.IsNumber() )
return make_intrusive<IntervalVal>(j.GetDouble());
return make_intrusive<IntervalVal>(j.GetDouble());
if ( j.IsString() ) {
auto parts = util::split(j.GetString(), " ");
// Strip out any empty items. This can happen if there are
// strings of spaces in the original string.
parts.erase(std::remove_if(parts.begin(), parts.end(), [](auto x) { return x.empty(); }), parts.end());
if ( (parts.size() % 2) != 0 )
return "wrong interval format, must be pairs of values with units";
double interval_secs = 0.0;
for ( size_t i = 0; i < parts.size(); i += 2 ) {
auto value = std::stod(std::string{parts[i]});
const auto& unit = parts[i + 1];
if ( unit == "day" || unit == "days" )
interval_secs += (value * Days);
else if ( unit == "hr" || unit == "hrs" )
interval_secs += (value * Hours);
else if ( unit == "min" || unit == "mins" )
interval_secs += (value * Minutes);
else if ( unit == "sec" || unit == "secs" )
interval_secs += (value * Seconds);
else if ( unit == "msec" || unit == "msecs" )
interval_secs += (value * Milliseconds);
else if ( unit == "usec" || unit == "usecs" )
interval_secs += (value * Microseconds);
else
return util::fmt("wrong interval format, invalid unit type %s", unit.data());
}
return make_intrusive<IntervalVal>(interval_secs, Seconds);
}
return mismatch_err();
}
case TYPE_PORT: {

5
src/Val.h
View file

@ -248,9 +248,12 @@ public:
* first match on any record field name in the resulting output. See the
* to_json() BiF for context.
*
* @param interval_as_double If true, interval values will be written as
* doubles instead of the broken-out version with units.
*
* @return JSON data representing the Val.
*/
StringValPtr ToJSON(bool only_loggable = false, RE_Matcher* re = nullptr);
StringValPtr ToJSON(bool only_loggable = false, RE_Matcher* re = nullptr, bool interval_as_double = false);
template<typename T>
T As() {

8
src/zeek.bif
View file

@ -5103,14 +5103,18 @@ function anonymize_addr%(a: addr, cl: IPAddrAnonymizationClass%): addr
## rendered name. The default pattern strips a leading
## underscore.
##
## interval_as_double: If T, interval values will be logged as doubles
## instead of the broken-out version with units as strings.
##
## returns: a JSON formatted string.
##
## .. zeek:see:: fmt cat cat_sep string_cat print_raw from_json
function to_json%(val: any, only_loggable: bool &default=F, field_escape_pattern: pattern &default=/^_/%): string
function to_json%(val: any, only_loggable: bool &default=F, field_escape_pattern: pattern &default=/^_/, interval_as_double: bool &default=F%): string
%{
return val->ToJSON(only_loggable, field_escape_pattern);
return val->ToJSON(only_loggable, field_escape_pattern, interval_as_double);
%}
## A function to convert a JSON string into Zeek values of a given type.
##
## Implicit conversion from JSON to Zeek types is implemented for:

25
testing/btest/Baseline/scripts.base.utils.json-roundtrip/output Normal file
View file

@ -0,0 +1,25 @@
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
Valid conversion of Foo: 1
hello, T
t, T
f, T
n, T
m, T
def, T
i, T
pi, T
a, [T, T, T, T]
c1, T
p, T
ti, T
it, T
ad, T
s, T
re, T
su, T
se, T
Valid conversion of IntervalOnly: 1
it, T

102
testing/btest/scripts/base/utils/json-roundtrip.test Normal file
View file

@ -0,0 +1,102 @@
# @TEST-DOCS: Test round-trip JSON encoding and decoding using the Zeek methods
# @TEST-EXEC: zeek -b %INPUT >output
# @TEST-EXEC: btest-diff output
type Color: enum {
Red = 10,
White = 20,
Blue = 30
};
type Foo: record {
hello: string;
t: bool;
f: bool;
n: count &optional;
m: count &optional; # not in input
def: count &default = 123;
i: int;
pi: double;
a: string_vec;
c1: Color;
p: port;
ti: time;
it: interval;
ad: addr;
s: subnet;
re: pattern;
su: subnet_set;
se: set[addr, port];
tbl: table[addr, port] of string;
};
type IntervalOnly: record {
it: interval;
};
event zeek_init()
{
local f: Foo;
f$hello = "world";
f$t = T;
f$f = F;
f$n = 0;
f$i = 123;
f$pi = 3.1416;
f$a = ["1", "2", "3", "4"];
f$c1 = Blue;
f$p = 1500/tcp;
f$ti = double_to_time(1681652265.042767);
f$it = double_to_interval(2*24*3600 + 2*3600 + 2*60 + 2*1.0 + 2*0.1 + 2*0.0001);
f$ad = 127.0.0.1;
f$s = 10.0.0.1/24;
f$re = /a/;
f$su = [[aa:bb::0]/32, 192.168.0.0/16];
f$se = [[192.168.0.1, 80/tcp], [[2001:db8::1], 8080/udp]];
f$tbl[192.168.100.1, 80/tcp] = "foo";
local f_json = to_json(f);
local f2 = from_json(f_json, Foo);
print fmt("Valid conversion of Foo: %d", f2$valid);
print "";
local f2_v : Foo = f2$v;
print "hello", f$hello == f2_v$hello;
print "t", f$t == f2_v$t;
print "f", f$f == f2_v$f;
print "n", f$n == f2_v$n;
print "m", (! f?$m);
print "def", f$def == f2_v$def;
print "i", f$i == f2_v$i;
print "pi", f$pi == f2_v$pi;
print "a", f$a == f2_v$a;
print "c1", f$c1 == f2_v$c1;
print "p", f$p == f2_v$p;
print "ti", f$ti == f2_v$ti;
print "it", f$it == f2_v$it;
print "ad", f$ad == f2_v$ad;
print "s", f$s == f2_v$s;
print "re", f$re == f2_v$re;
print "su", f$su == f2_v$su;
print "se", f$se == f2_v$se;
# TODO: direct comparisons of tables isn't allowed. This will have to wait.
# print f$tbl == f2_v$tbl;
local io: IntervalOnly;
io$it = double_to_interval(2*24*3600 + 2*3600 + 2*60 + 2*1.0 + 2*0.1 + 2*0.0001);
# Test round-trip conversion of intervals as doubles.
local io_json = to_json(io, F, /^_/, T);
local io2 = from_json(io_json, IntervalOnly);
print "";
print fmt("Valid conversion of IntervalOnly: %d", f2$valid);
print "";
local io2_v : IntervalOnly = io2$v;
print "it", io$it == io2_v$it;
}