diff --git a/CHANGES b/CHANGES index addcb68cc8..02f88a42ca 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,13 @@ +7.1.0-dev.654 | 2024-12-03 10:10:15 -0700 + + * Add interval_as_double argument to control how intervals are converted to JSON (Tim Wojtulewicz, Corelight) + + * Add btest for round-trip JSON conversion (Tim Wojtulewicz, Corelight) + + * Allow comparing two PatternVals (Tim Wojtulewicz, Corelight) + + * Handle conversion between data from Val::ToJSON and ValFromJSON better (Tim Wojtulewicz, Corelight) + 7.1.0-dev.649 | 2024-12-02 13:43:26 +0100 * added new Cluster:: BiFs to script optimization tracking (Vern Paxson, Corelight) diff --git a/VERSION b/VERSION index 59410cf314..77840937bb 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -7.1.0-dev.649 +7.1.0-dev.654 diff --git a/src/Expr.cc b/src/Expr.cc index bdac67c9be..878002d598 100644 --- a/src/Expr.cc +++ b/src/Expr.cc @@ -1965,6 +1965,7 @@ EqExpr::EqExpr(ExprTag arg_tag, ExprPtr arg_op1, ExprPtr arg_op2) case TYPE_ADDR: case TYPE_SUBNET: case TYPE_ERROR: + case TYPE_PATTERN: case TYPE_FUNC: break; case TYPE_ENUM: @@ -1996,12 +1997,19 @@ EqExpr::EqExpr(ExprTag arg_tag, ExprPtr arg_op1, ExprPtr arg_op2) ValPtr EqExpr::Fold(Val* v1, Val* v2) const { if ( op1->GetType()->Tag() == TYPE_PATTERN ) { - auto re = v1->As(); - const String* s = v2->AsString(); - if ( tag == EXPR_EQ ) - return val_mgr->Bool(re->MatchExactly(s)); - else - return val_mgr->Bool(! re->MatchExactly(s)); + if ( op2->GetType()->Tag() == TYPE_PATTERN ) { + auto re1 = v1->As(); + auto re2 = v2->As(); + return val_mgr->Bool(strcmp(re1->Get()->PatternText(), re2->Get()->PatternText()) == 0); + } + else { + auto re = v1->As(); + const String* s = v2->AsString(); + if ( tag == EXPR_EQ ) + return val_mgr->Bool(re->MatchExactly(s)); + else + return val_mgr->Bool(! re->MatchExactly(s)); + } } else if ( op1->GetType()->Tag() == TYPE_FUNC ) { auto res = v1->AsFunc() == v2->AsFunc(); diff --git a/src/Val.cc b/src/Val.cc index bdbe3ceb35..d62b486464 100644 --- a/src/Val.cc +++ b/src/Val.cc @@ -351,7 +351,7 @@ static bool UsesJSONStringType(const TypePtr& t) { // This is a static method in this file to avoid including rapidjson's headers // in Val.h, because they're huge. static void BuildJSON(json::detail::NullDoubleWriter& writer, Val* val, bool only_loggable = false, - RE_Matcher* re = nullptr, const string& key = "") { + RE_Matcher* re = nullptr, const string& key = "", bool interval_as_double = false) { if ( ! key.empty() ) writer.Key(key); @@ -387,7 +387,6 @@ static void BuildJSON(json::detail::NullDoubleWriter& writer, Val* val, bool onl } case TYPE_PATTERN: - case TYPE_INTERVAL: case TYPE_ADDR: case TYPE_SUBNET: { ODesc d; @@ -397,6 +396,18 @@ static void BuildJSON(json::detail::NullDoubleWriter& writer, Val* val, bool onl break; } + case TYPE_INTERVAL: { + if ( interval_as_double ) + writer.Double(val->AsInterval()); + else { + ODesc d; + d.SetStyle(RAW_STYLE); + val->Describe(&d); + writer.String(reinterpret_cast(d.Bytes()), d.Len()); + } + break; + } + case TYPE_FILE: case TYPE_FUNC: case TYPE_ENUM: @@ -433,11 +444,11 @@ static void BuildJSON(json::detail::NullDoubleWriter& writer, Val* val, bool onl Val* entry_key = lv->Length() == 1 ? lv->Idx(0).get() : lv.get(); if ( tval->GetType()->IsSet() ) - BuildJSON(writer, entry_key, only_loggable, re); + BuildJSON(writer, entry_key, only_loggable, re, "", interval_as_double); else { rapidjson::StringBuffer buffer; json::detail::NullDoubleWriter key_writer(buffer); - BuildJSON(key_writer, entry_key, only_loggable, re); + BuildJSON(key_writer, entry_key, only_loggable, re, "", interval_as_double); string key_str = buffer.GetString(); // Strip the quotes for any type we render as a string. This @@ -446,7 +457,7 @@ static void BuildJSON(json::detail::NullDoubleWriter& writer, Val* val, bool onl if ( UsesJSONStringType(entry_key->GetType()) ) key_str = key_str.substr(1, key_str.length() - 2); - BuildJSON(writer, entry->GetVal().get(), only_loggable, re, key_str); + BuildJSON(writer, entry->GetVal().get(), only_loggable, re, key_str, interval_as_double); } } @@ -481,7 +492,7 @@ static void BuildJSON(json::detail::NullDoubleWriter& writer, Val* val, bool onl else key_str = field_name; - BuildJSON(writer, value.get(), only_loggable, re, key_str); + BuildJSON(writer, value.get(), only_loggable, re, key_str, interval_as_double); } } @@ -495,7 +506,7 @@ static void BuildJSON(json::detail::NullDoubleWriter& writer, Val* val, bool onl auto* lval = val->AsListVal(); size_t size = lval->Length(); for ( size_t i = 0; i < size; i++ ) - BuildJSON(writer, lval->Idx(i).get(), only_loggable, re); + BuildJSON(writer, lval->Idx(i).get(), only_loggable, re, "", interval_as_double); writer.EndArray(); break; @@ -507,7 +518,7 @@ static void BuildJSON(json::detail::NullDoubleWriter& writer, Val* val, bool onl auto* vval = val->AsVectorVal(); size_t size = vval->SizeVal()->AsCount(); for ( size_t i = 0; i < size; i++ ) - BuildJSON(writer, vval->ValAt(i).get(), only_loggable, re); + BuildJSON(writer, vval->ValAt(i).get(), only_loggable, re, "", interval_as_double); writer.EndArray(); break; @@ -528,11 +539,11 @@ static void BuildJSON(json::detail::NullDoubleWriter& writer, Val* val, bool onl } } -StringValPtr Val::ToJSON(bool only_loggable, RE_Matcher* re) { +StringValPtr Val::ToJSON(bool only_loggable, RE_Matcher* re, bool interval_as_double) { rapidjson::StringBuffer buffer; json::detail::NullDoubleWriter writer(buffer); - BuildJSON(writer, this, only_loggable, re, ""); + BuildJSON(writer, this, only_loggable, re, "", interval_as_double); return make_intrusive(buffer.GetString()); } @@ -938,10 +949,44 @@ static std::variant BuildVal(const rapidjson::Value& j, con } case TYPE_INTERVAL: { - if ( ! j.IsNumber() ) - return mismatch_err(); + if ( j.IsNumber() ) + return make_intrusive(j.GetDouble()); - return make_intrusive(j.GetDouble()); + if ( j.IsString() ) { + auto parts = util::split(j.GetString(), " "); + + // Strip out any empty items. This can happen if there are + // strings of spaces in the original string. + parts.erase(std::remove_if(parts.begin(), parts.end(), [](auto x) { return x.empty(); }), parts.end()); + + if ( (parts.size() % 2) != 0 ) + return "wrong interval format, must be pairs of values with units"; + + double interval_secs = 0.0; + for ( size_t i = 0; i < parts.size(); i += 2 ) { + auto value = std::stod(std::string{parts[i]}); + const auto& unit = parts[i + 1]; + + if ( unit == "day" || unit == "days" ) + interval_secs += (value * Days); + else if ( unit == "hr" || unit == "hrs" ) + interval_secs += (value * Hours); + else if ( unit == "min" || unit == "mins" ) + interval_secs += (value * Minutes); + else if ( unit == "sec" || unit == "secs" ) + interval_secs += (value * Seconds); + else if ( unit == "msec" || unit == "msecs" ) + interval_secs += (value * Milliseconds); + else if ( unit == "usec" || unit == "usecs" ) + interval_secs += (value * Microseconds); + else + return util::fmt("wrong interval format, invalid unit type %s", unit.data()); + } + + return make_intrusive(interval_secs, Seconds); + } + + return mismatch_err(); } case TYPE_PORT: { diff --git a/src/Val.h b/src/Val.h index 8d5d8f3d52..bc6de01ddd 100644 --- a/src/Val.h +++ b/src/Val.h @@ -248,9 +248,12 @@ public: * first match on any record field name in the resulting output. See the * to_json() BiF for context. * + * @param interval_as_double If true, interval values will be written as + * doubles instead of the broken-out version with units. + * * @return JSON data representing the Val. */ - StringValPtr ToJSON(bool only_loggable = false, RE_Matcher* re = nullptr); + StringValPtr ToJSON(bool only_loggable = false, RE_Matcher* re = nullptr, bool interval_as_double = false); template T As() { diff --git a/src/zeek.bif b/src/zeek.bif index 6cc5276e48..ee14c0ddce 100644 --- a/src/zeek.bif +++ b/src/zeek.bif @@ -5103,14 +5103,18 @@ function anonymize_addr%(a: addr, cl: IPAddrAnonymizationClass%): addr ## rendered name. The default pattern strips a leading ## underscore. ## +## interval_as_double: If T, interval values will be logged as doubles +## instead of the broken-out version with units as strings. +## ## returns: a JSON formatted string. ## ## .. zeek:see:: fmt cat cat_sep string_cat print_raw from_json -function to_json%(val: any, only_loggable: bool &default=F, field_escape_pattern: pattern &default=/^_/%): string +function to_json%(val: any, only_loggable: bool &default=F, field_escape_pattern: pattern &default=/^_/, interval_as_double: bool &default=F%): string %{ - return val->ToJSON(only_loggable, field_escape_pattern); + return val->ToJSON(only_loggable, field_escape_pattern, interval_as_double); %} + ## A function to convert a JSON string into Zeek values of a given type. ## ## Implicit conversion from JSON to Zeek types is implemented for: diff --git a/testing/btest/Baseline/scripts.base.utils.json-roundtrip/output b/testing/btest/Baseline/scripts.base.utils.json-roundtrip/output new file mode 100644 index 0000000000..89017ab79a --- /dev/null +++ b/testing/btest/Baseline/scripts.base.utils.json-roundtrip/output @@ -0,0 +1,25 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +Valid conversion of Foo: 1 + +hello, T +t, T +f, T +n, T +m, T +def, T +i, T +pi, T +a, [T, T, T, T] +c1, T +p, T +ti, T +it, T +ad, T +s, T +re, T +su, T +se, T + +Valid conversion of IntervalOnly: 1 + +it, T diff --git a/testing/btest/scripts/base/utils/json-roundtrip.test b/testing/btest/scripts/base/utils/json-roundtrip.test new file mode 100644 index 0000000000..4caa06c05e --- /dev/null +++ b/testing/btest/scripts/base/utils/json-roundtrip.test @@ -0,0 +1,102 @@ +# @TEST-DOCS: Test round-trip JSON encoding and decoding using the Zeek methods +# @TEST-EXEC: zeek -b %INPUT >output +# @TEST-EXEC: btest-diff output + +type Color: enum { + Red = 10, + White = 20, + Blue = 30 +}; + +type Foo: record { + hello: string; + t: bool; + f: bool; + n: count &optional; + m: count &optional; # not in input + def: count &default = 123; + i: int; + pi: double; + a: string_vec; + c1: Color; + p: port; + ti: time; + it: interval; + ad: addr; + s: subnet; + re: pattern; + su: subnet_set; + se: set[addr, port]; + tbl: table[addr, port] of string; +}; + +type IntervalOnly: record { + it: interval; +}; + +event zeek_init() + { + + local f: Foo; + f$hello = "world"; + f$t = T; + f$f = F; + f$n = 0; + f$i = 123; + f$pi = 3.1416; + f$a = ["1", "2", "3", "4"]; + f$c1 = Blue; + f$p = 1500/tcp; + f$ti = double_to_time(1681652265.042767); + f$it = double_to_interval(2*24*3600 + 2*3600 + 2*60 + 2*1.0 + 2*0.1 + 2*0.0001); + f$ad = 127.0.0.1; + f$s = 10.0.0.1/24; + f$re = /a/; + f$su = [[aa:bb::0]/32, 192.168.0.0/16]; + f$se = [[192.168.0.1, 80/tcp], [[2001:db8::1], 8080/udp]]; + f$tbl[192.168.100.1, 80/tcp] = "foo"; + + local f_json = to_json(f); + + local f2 = from_json(f_json, Foo); + print fmt("Valid conversion of Foo: %d", f2$valid); + print ""; + + local f2_v : Foo = f2$v; + + print "hello", f$hello == f2_v$hello; + print "t", f$t == f2_v$t; + print "f", f$f == f2_v$f; + print "n", f$n == f2_v$n; + print "m", (! f?$m); + print "def", f$def == f2_v$def; + print "i", f$i == f2_v$i; + print "pi", f$pi == f2_v$pi; + print "a", f$a == f2_v$a; + print "c1", f$c1 == f2_v$c1; + print "p", f$p == f2_v$p; + print "ti", f$ti == f2_v$ti; + print "it", f$it == f2_v$it; + print "ad", f$ad == f2_v$ad; + print "s", f$s == f2_v$s; + print "re", f$re == f2_v$re; + print "su", f$su == f2_v$su; + print "se", f$se == f2_v$se; + +# TODO: direct comparisons of tables isn't allowed. This will have to wait. +# print f$tbl == f2_v$tbl; + + local io: IntervalOnly; + io$it = double_to_interval(2*24*3600 + 2*3600 + 2*60 + 2*1.0 + 2*0.1 + 2*0.0001); + + # Test round-trip conversion of intervals as doubles. + local io_json = to_json(io, F, /^_/, T); + local io2 = from_json(io_json, IntervalOnly); + + print ""; + print fmt("Valid conversion of IntervalOnly: %d", f2$valid); + print ""; + + local io2_v : IntervalOnly = io2$v; + print "it", io$it == io2_v$it; +}