Merge remote-tracking branch 'origin/topic/timw/json-interval-conversion'

* origin/topic/timw/json-interval-conversion:
  Add interval_as_double argument to control how intervals are converted to JSON
  Add btest for round-trip JSON conversion
  Allow comparing two PatternVals
  Handle conversion between data from Val::ToJSON and ValFromJSON better

CHANGES

@@ -1,3 +1,13 @@
+7.1.0-dev.654 | 2024-12-03 10:10:15 -0700
+
+  * Add interval_as_double argument to control how intervals are converted to JSON (Tim Wojtulewicz, Corelight)
+
+  * Add btest for round-trip JSON conversion (Tim Wojtulewicz, Corelight)
+
+  * Allow comparing two PatternVals (Tim Wojtulewicz, Corelight)
+
+  * Handle conversion between data from Val::ToJSON and ValFromJSON better (Tim Wojtulewicz, Corelight)
+
 7.1.0-dev.649 | 2024-12-02 13:43:26 +0100
 
   * added new Cluster:: BiFs to script optimization tracking (Vern Paxson, Corelight)

VERSION

@@ -1 +1 @@
-7.1.0-dev.649
+7.1.0-dev.654

src/Expr.cc

@@ -1965,6 +1965,7 @@ EqExpr::EqExpr(ExprTag arg_tag, ExprPtr arg_op1, ExprPtr arg_op2)
             case TYPE_ADDR:
             case TYPE_SUBNET:
             case TYPE_ERROR:
+            case TYPE_PATTERN:
             case TYPE_FUNC: break;
 
             case TYPE_ENUM:
@@ -1996,12 +1997,19 @@ EqExpr::EqExpr(ExprTag arg_tag, ExprPtr arg_op1, ExprPtr arg_op2)
 
 ValPtr EqExpr::Fold(Val* v1, Val* v2) const {
     if ( op1->GetType()->Tag() == TYPE_PATTERN ) {
-        auto re = v1->As<PatternVal*>();
+        if ( op2->GetType()->Tag() == TYPE_PATTERN ) {
-        const String* s = v2->AsString();
+            auto re1 = v1->As<PatternVal*>();
-        if ( tag == EXPR_EQ )
+            auto re2 = v2->As<PatternVal*>();
-            return val_mgr->Bool(re->MatchExactly(s));
+            return val_mgr->Bool(strcmp(re1->Get()->PatternText(), re2->Get()->PatternText()) == 0);
-        else
+        }
-            return val_mgr->Bool(! re->MatchExactly(s));
+        else {
+            auto re = v1->As<PatternVal*>();
+            const String* s = v2->AsString();
+            if ( tag == EXPR_EQ )
+                return val_mgr->Bool(re->MatchExactly(s));
+            else
+                return val_mgr->Bool(! re->MatchExactly(s));
+        }
     }
     else if ( op1->GetType()->Tag() == TYPE_FUNC ) {
         auto res = v1->AsFunc() == v2->AsFunc();

src/Val.cc

@@ -351,7 +351,7 @@ static bool UsesJSONStringType(const TypePtr& t) {
 // This is a static method in this file to avoid including rapidjson's headers
 // in Val.h, because they're huge.
 static void BuildJSON(json::detail::NullDoubleWriter& writer, Val* val, bool only_loggable = false,
-                      RE_Matcher* re = nullptr, const string& key = "") {
+                      RE_Matcher* re = nullptr, const string& key = "", bool interval_as_double = false) {
     if ( ! key.empty() )
         writer.Key(key);
 
@@ -387,7 +387,6 @@ static void BuildJSON(json::detail::NullDoubleWriter& writer, Val* val, bool onl
         }
 
         case TYPE_PATTERN:
-        case TYPE_INTERVAL:
         case TYPE_ADDR:
         case TYPE_SUBNET: {
             ODesc d;
@@ -397,6 +396,18 @@ static void BuildJSON(json::detail::NullDoubleWriter& writer, Val* val, bool onl
             break;
         }
 
+        case TYPE_INTERVAL: {
+            if ( interval_as_double )
+                writer.Double(val->AsInterval());
+            else {
+                ODesc d;
+                d.SetStyle(RAW_STYLE);
+                val->Describe(&d);
+                writer.String(reinterpret_cast<const char*>(d.Bytes()), d.Len());
+            }
+            break;
+        }
+
         case TYPE_FILE:
         case TYPE_FUNC:
         case TYPE_ENUM:
@@ -433,11 +444,11 @@ static void BuildJSON(json::detail::NullDoubleWriter& writer, Val* val, bool onl
                 Val* entry_key = lv->Length() == 1 ? lv->Idx(0).get() : lv.get();
 
                 if ( tval->GetType()->IsSet() )
-                    BuildJSON(writer, entry_key, only_loggable, re);
+                    BuildJSON(writer, entry_key, only_loggable, re, "", interval_as_double);
                 else {
                     rapidjson::StringBuffer buffer;
                     json::detail::NullDoubleWriter key_writer(buffer);
-                    BuildJSON(key_writer, entry_key, only_loggable, re);
+                    BuildJSON(key_writer, entry_key, only_loggable, re, "", interval_as_double);
                     string key_str = buffer.GetString();
 
                     // Strip the quotes for any type we render as a string. This
@@ -446,7 +457,7 @@ static void BuildJSON(json::detail::NullDoubleWriter& writer, Val* val, bool onl
                     if ( UsesJSONStringType(entry_key->GetType()) )
                         key_str = key_str.substr(1, key_str.length() - 2);
 
-                    BuildJSON(writer, entry->GetVal().get(), only_loggable, re, key_str);
+                    BuildJSON(writer, entry->GetVal().get(), only_loggable, re, key_str, interval_as_double);
                 }
             }
 
@@ -481,7 +492,7 @@ static void BuildJSON(json::detail::NullDoubleWriter& writer, Val* val, bool onl
                     else
                         key_str = field_name;
 
-                    BuildJSON(writer, value.get(), only_loggable, re, key_str);
+                    BuildJSON(writer, value.get(), only_loggable, re, key_str, interval_as_double);
                 }
             }
 
@@ -495,7 +506,7 @@ static void BuildJSON(json::detail::NullDoubleWriter& writer, Val* val, bool onl
             auto* lval = val->AsListVal();
             size_t size = lval->Length();
             for ( size_t i = 0; i < size; i++ )
-                BuildJSON(writer, lval->Idx(i).get(), only_loggable, re);
+                BuildJSON(writer, lval->Idx(i).get(), only_loggable, re, "", interval_as_double);
 
             writer.EndArray();
             break;
@@ -507,7 +518,7 @@ static void BuildJSON(json::detail::NullDoubleWriter& writer, Val* val, bool onl
             auto* vval = val->AsVectorVal();
             size_t size = vval->SizeVal()->AsCount();
             for ( size_t i = 0; i < size; i++ )
-                BuildJSON(writer, vval->ValAt(i).get(), only_loggable, re);
+                BuildJSON(writer, vval->ValAt(i).get(), only_loggable, re, "", interval_as_double);
 
             writer.EndArray();
             break;
@@ -528,11 +539,11 @@ static void BuildJSON(json::detail::NullDoubleWriter& writer, Val* val, bool onl
     }
 }
 
-StringValPtr Val::ToJSON(bool only_loggable, RE_Matcher* re) {
+StringValPtr Val::ToJSON(bool only_loggable, RE_Matcher* re, bool interval_as_double) {
     rapidjson::StringBuffer buffer;
     json::detail::NullDoubleWriter writer(buffer);
 
-    BuildJSON(writer, this, only_loggable, re, "");
+    BuildJSON(writer, this, only_loggable, re, "", interval_as_double);
 
     return make_intrusive<StringVal>(buffer.GetString());
 }
@@ -938,10 +949,44 @@ static std::variant<ValPtr, std::string> BuildVal(const rapidjson::Value& j, con
         }
 
         case TYPE_INTERVAL: {
-            if ( ! j.IsNumber() )
+            if ( j.IsNumber() )
-                return mismatch_err();
+                return make_intrusive<IntervalVal>(j.GetDouble());
 
-            return make_intrusive<IntervalVal>(j.GetDouble());
+            if ( j.IsString() ) {
+                auto parts = util::split(j.GetString(), " ");
+
+                // Strip out any empty items. This can happen if there are
+                // strings of spaces in the original string.
+                parts.erase(std::remove_if(parts.begin(), parts.end(), [](auto x) { return x.empty(); }), parts.end());
+
+                if ( (parts.size() % 2) != 0 )
+                    return "wrong interval format, must be pairs of values with units";
+
+                double interval_secs = 0.0;
+                for ( size_t i = 0; i < parts.size(); i += 2 ) {
+                    auto value = std::stod(std::string{parts[i]});
+                    const auto& unit = parts[i + 1];
+
+                    if ( unit == "day" || unit == "days" )
+                        interval_secs += (value * Days);
+                    else if ( unit == "hr" || unit == "hrs" )
+                        interval_secs += (value * Hours);
+                    else if ( unit == "min" || unit == "mins" )
+                        interval_secs += (value * Minutes);
+                    else if ( unit == "sec" || unit == "secs" )
+                        interval_secs += (value * Seconds);
+                    else if ( unit == "msec" || unit == "msecs" )
+                        interval_secs += (value * Milliseconds);
+                    else if ( unit == "usec" || unit == "usecs" )
+                        interval_secs += (value * Microseconds);
+                    else
+                        return util::fmt("wrong interval format, invalid unit type %s", unit.data());
+                }
+
+                return make_intrusive<IntervalVal>(interval_secs, Seconds);
+            }
+
+            return mismatch_err();
         }
 
         case TYPE_PORT: {

src/Val.h

@@ -248,9 +248,12 @@ public:
      * first match on any record field name in the resulting output.  See the
      * to_json() BiF for context.
      *
+     * @param interval_as_double If true, interval values will be written as
+     * doubles instead of the broken-out version with units.
+     *
      * @return  JSON data representing the Val.
      */
-    StringValPtr ToJSON(bool only_loggable = false, RE_Matcher* re = nullptr);
+    StringValPtr ToJSON(bool only_loggable = false, RE_Matcher* re = nullptr, bool interval_as_double = false);
 
     template<typename T>
     T As() {

src/zeek.bif

@@ -5103,14 +5103,18 @@ function anonymize_addr%(a: addr, cl: IPAddrAnonymizationClass%): addr
 ##                       rendered name. The default pattern strips a leading
 ##                       underscore.
 ##
+## interval_as_double: If T, interval values will be logged as doubles
+##                     instead of the broken-out version with units as strings.
+##
 ## returns: a JSON formatted string.
 ##
 ## .. zeek:see:: fmt cat cat_sep string_cat print_raw from_json
-function to_json%(val: any, only_loggable: bool &default=F, field_escape_pattern: pattern &default=/^_/%): string
+function to_json%(val: any, only_loggable: bool &default=F, field_escape_pattern: pattern &default=/^_/, interval_as_double: bool &default=F%): string
 	%{
-	return val->ToJSON(only_loggable, field_escape_pattern);
+	return val->ToJSON(only_loggable, field_escape_pattern, interval_as_double);
 	%}
 
+
 ## A function to convert a JSON string into Zeek values of a given type.
 ##
 ## Implicit conversion from JSON to Zeek types is implemented for:

testing/btest/Baseline/scripts.base.utils.json-roundtrip/output

@@ -0,0 +1,25 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+Valid conversion of Foo: 1
+
+hello, T
+t, T
+f, T
+n, T
+m, T
+def, T
+i, T
+pi, T
+a, [T, T, T, T]
+c1, T
+p, T
+ti, T
+it, T
+ad, T
+s, T
+re, T
+su, T
+se, T
+
+Valid conversion of IntervalOnly: 1
+
+it, T

testing/btest/scripts/base/utils/json-roundtrip.test

@@ -0,0 +1,102 @@
+# @TEST-DOCS: Test round-trip JSON encoding and decoding using the Zeek methods
+# @TEST-EXEC: zeek -b %INPUT >output
+# @TEST-EXEC: btest-diff output
+
+type Color: enum {
+	Red = 10,
+	White = 20,
+	Blue = 30
+};
+
+type Foo: record {
+	hello: string;
+	t: bool;
+	f: bool;
+	n: count &optional;
+	m: count &optional;  # not in input
+	def: count &default = 123;
+	i: int;
+	pi: double;
+	a: string_vec;
+	c1: Color;
+	p: port;
+	ti: time;
+	it: interval;
+	ad: addr;
+	s: subnet;
+	re: pattern;
+	su: subnet_set;
+	se: set[addr, port];
+	tbl: table[addr, port] of string;
+};
+
+type IntervalOnly: record {
+	it: interval;
+};
+
+event zeek_init()
+	{
+
+	local f: Foo;
+	f$hello = "world";
+	f$t = T;
+	f$f = F;
+	f$n = 0;
+	f$i = 123;
+	f$pi = 3.1416;
+	f$a = ["1", "2", "3", "4"];
+	f$c1 = Blue;
+	f$p = 1500/tcp;
+	f$ti = double_to_time(1681652265.042767);
+	f$it = double_to_interval(2*24*3600 + 2*3600 +  2*60 + 2*1.0 + 2*0.1 + 2*0.0001);
+	f$ad = 127.0.0.1;
+	f$s = 10.0.0.1/24;
+	f$re = /a/;
+	f$su = [[aa:bb::0]/32, 192.168.0.0/16];
+	f$se = [[192.168.0.1, 80/tcp], [[2001:db8::1], 8080/udp]];
+	f$tbl[192.168.100.1, 80/tcp] = "foo";
+
+	local f_json = to_json(f);
+
+	local f2 = from_json(f_json, Foo);
+	print fmt("Valid conversion of Foo: %d", f2$valid);
+	print "";
+
+	local f2_v : Foo = f2$v;
+
+	print "hello", f$hello == f2_v$hello;
+	print "t", f$t == f2_v$t;
+	print "f", f$f == f2_v$f;
+	print "n", f$n == f2_v$n;
+	print "m", (! f?$m);
+	print "def", f$def == f2_v$def;
+	print "i", f$i == f2_v$i;
+	print "pi", f$pi == f2_v$pi;
+	print "a", f$a == f2_v$a;
+	print "c1", f$c1 == f2_v$c1;
+	print "p", f$p == f2_v$p;
+	print "ti", f$ti == f2_v$ti;
+	print "it", f$it == f2_v$it;
+	print "ad", f$ad == f2_v$ad;
+	print "s", f$s == f2_v$s;
+	print "re", f$re == f2_v$re;
+	print "su", f$su == f2_v$su;
+	print "se", f$se == f2_v$se;
+
+# TODO: direct comparisons of tables isn't allowed. This will have to wait.
+#	print f$tbl == f2_v$tbl;
+
+	local io: IntervalOnly;
+	io$it = double_to_interval(2*24*3600 + 2*3600 +  2*60 + 2*1.0 + 2*0.1 + 2*0.0001);
+
+	# Test round-trip conversion of intervals as doubles.
+	local io_json = to_json(io, F, /^_/, T);
+	local io2 = from_json(io_json, IntervalOnly);
+
+	print "";
+	print fmt("Valid conversion of IntervalOnly: %d", f2$valid);
+	print "";
+
+	local io2_v : IntervalOnly = io2$v;
+	print "it", io$it == io2_v$it;
+}