diff --git a/CHANGES b/CHANGES index f3e17c9da7..e4b699d482 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,7 @@ +6.0.0-dev.518 | 2023-05-09 10:19:46 +0200 + + * Implement from_json bif (Fupeng Zhao) + 6.0.0-dev.516 | 2023-05-05 14:08:15 -0700 * BTest baseline updates for -O gen-C++ (Vern Paxson, Corelight) diff --git a/NEWS b/NEWS index 959dd0849e..119e5b3ba2 100644 --- a/NEWS +++ b/NEWS @@ -91,6 +91,19 @@ New Functionality To disable this functionality, pass ``--disable-javascript`` to configure. +- A new bif ``from_json()`` can be used to parse JSON strings into records. + + type A: record { a: addr; }; + local p = from_json({\"a\": \"192.168.0.1\"}", A); + if ( p$valid ) + print (p$v as A) + + Implicit conversion from JSON to Zeek types is implemented for bool, int, count, + real, interval (number as seconds) and time (number as unix timestamp), port + (strings in "80/tcp" notation), patterns, addr, subnet, enum, sets, vectors + and records similar to the rules of the input framework. Optional or default + record fields are allowed to be missing or null in the input. + - Zeek now provides native "Community ID" support with a new bif called ``community_id_v1()``. Two policy scripts ``protocols/conn/community-id-logging`` and ``frameworks/notice/community-id`` extend the respective logs with a diff --git a/VERSION b/VERSION index 5c7b605b6c..303a1e609d 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -6.0.0-dev.516 +6.0.0-dev.518 diff --git a/scripts/base/init-bare.zeek b/scripts/base/init-bare.zeek index 7220c538f9..b044892ed8 100644 --- a/scripts/base/init-bare.zeek +++ b/scripts/base/init-bare.zeek @@ -1094,6 +1094,14 @@ type entropy_test_result: record { serial_correlation: double; ##< Serial correlation coefficient. }; +## Return type for from_json BIF. +## +## .. zeek:see:: from_json +type from_json_result: record { + v: any &optional; ##< Parsed value. + valid: bool; ##< True if parsing was successful. +}; + # TCP values for :zeek:see:`endpoint` *state* field. # todo:: these should go into an enum to make them autodoc'able. const TCP_INACTIVE = 0; ##< Endpoint is still inactive. diff --git a/src/Val.cc b/src/Val.cc index 9b94c2dd97..816e583a30 100644 --- a/src/Val.cc +++ b/src/Val.cc @@ -6,6 +6,9 @@ #include #include +#define RAPIDJSON_HAS_STDSTRING 1 +#include +#include #include #include #include @@ -1061,6 +1064,333 @@ StringValPtr StringVal::Replace(RE_Matcher* re, const String& repl, bool do_all) return make_intrusive(new String(true, result, r - result)); } +static std::variant BuildVal(const rapidjson::Value& j, const TypePtr& t) + { + auto mismatch_err = [t, &j]() + { + std::string json_type; + switch ( j.GetType() ) + { + case rapidjson::Type::kNullType: + json_type = "null"; + break; + case rapidjson::Type::kFalseType: + case rapidjson::Type::kTrueType: + json_type = "bool"; + break; + case rapidjson::Type::kObjectType: + json_type = "object"; + break; + case rapidjson::Type::kArrayType: + json_type = "array"; + break; + case rapidjson::Type::kStringType: + json_type = "string"; + break; + case rapidjson::Type::kNumberType: + json_type = "number"; + break; + default: + json_type = "unknown"; + } + + return util::fmt("cannot convert JSON type '%s' to Zeek type '%s'", json_type.c_str(), + type_name(t->Tag())); + }; + + if ( j.IsNull() ) + return Val::nil; + + switch ( t->Tag() ) + { + case TYPE_BOOL: + { + if ( ! j.IsBool() ) + return mismatch_err(); + + return val_mgr->Bool(j.GetBool()); + } + + case TYPE_INT: + { + if ( ! j.IsInt64() ) + return mismatch_err(); + + return val_mgr->Int(j.GetInt64()); + } + + case TYPE_COUNT: + { + if ( ! j.IsUint64() ) + return mismatch_err(); + + return val_mgr->Count(j.GetUint64()); + } + + case TYPE_TIME: + { + if ( ! j.IsNumber() ) + return mismatch_err(); + + return make_intrusive(j.GetDouble()); + } + + case TYPE_DOUBLE: + { + if ( ! j.IsNumber() ) + return mismatch_err(); + + return make_intrusive(j.GetDouble()); + } + + case TYPE_INTERVAL: + { + if ( ! j.IsNumber() ) + return mismatch_err(); + + return make_intrusive(j.GetDouble()); + } + + case TYPE_PORT: + { + if ( ! j.IsString() ) + return mismatch_err(); + + int port = 0; + if ( j.GetStringLength() > 0 && j.GetStringLength() < 10 ) + { + char* slash; + errno = 0; + port = strtol(j.GetString(), &slash, 10); + if ( ! errno ) + { + ++slash; + if ( util::streq(slash, "tcp") ) + return val_mgr->Port(port, TRANSPORT_TCP); + else if ( util::streq(slash, "udp") ) + return val_mgr->Port(port, TRANSPORT_UDP); + else if ( util::streq(slash, "icmp") ) + return val_mgr->Port(port, TRANSPORT_ICMP); + else if ( util::streq(slash, "unknown") ) + return val_mgr->Port(port, TRANSPORT_UNKNOWN); + } + } + + return "wrong port format, must be /[0-9]{1,5}\\/(tcp|udp|icmp|unknown)/"; + } + + case TYPE_PATTERN: + { + if ( ! j.IsString() ) + return mismatch_err(); + + std::string candidate(j.GetString(), j.GetStringLength()); + if ( candidate.size() > 2 && candidate.front() == candidate.back() && + candidate.back() == '/' ) + { + // Remove the '/'s + candidate.erase(0, 1); + candidate.erase(candidate.size() - 1); + } + + auto re = std::make_unique(candidate.c_str()); + if ( ! re->Compile() ) + return "error compiling pattern"; + + return make_intrusive(re.release()); + } + + case TYPE_ADDR: + case TYPE_SUBNET: + { + if ( ! j.IsString() ) + return mismatch_err(); + + int width = 0; + std::string candidate; + + if ( t->Tag() == TYPE_ADDR ) + candidate = std::string(j.GetString(), j.GetStringLength()); + else + { + std::string_view subnet_sv(j.GetString(), j.GetStringLength()); + auto pos = subnet_sv.find('/'); + if ( pos == subnet_sv.npos ) + return util::fmt("invalid value for subnet: '%s'", j.GetString()); + + candidate = std::string(j.GetString(), pos); + + errno = 0; + char* end; + width = strtol(subnet_sv.data() + pos + 1, &end, 10); + if ( subnet_sv.data() + pos + 1 == end || errno ) + return util::fmt("invalid value for subnet: '%s'", j.GetString()); + } + + if ( candidate.front() == '[' ) + candidate.erase(0, 1); + if ( candidate.back() == ']' ) + candidate.erase(candidate.size() - 1); + + if ( t->Tag() == TYPE_ADDR ) + return make_intrusive(candidate); + else + return make_intrusive(candidate.c_str(), width); + } + + case TYPE_ENUM: + { + if ( ! j.IsString() ) + return mismatch_err(); + + auto et = t->AsEnumType(); + auto intval = et->Lookup({j.GetString(), j.GetStringLength()}); + + if ( intval < 0 ) + return util::fmt("'%s' is not a valid enum for '%s'.", j.GetString(), + et->GetName().c_str()); + + return et->GetEnumVal(intval); + } + + case TYPE_STRING: + { + if ( ! j.IsString() ) + return mismatch_err(); + + return make_intrusive(j.GetStringLength(), j.GetString()); + } + + case TYPE_TABLE: + { + if ( ! j.IsArray() ) + return mismatch_err(); + + if ( ! t->IsSet() ) + return util::fmt("tables are not supported"); + + auto tt = t->AsSetType(); + auto tl = tt->GetIndices(); + auto tv = make_intrusive(IntrusivePtr{NewRef{}, tt}); + + for ( const auto& item : j.GetArray() ) + { + std::variant v; + + if ( tl->GetTypes().size() == 1 ) + v = BuildVal(item, tl->GetPureType()); + else + v = BuildVal(item, tl); + + if ( ! get_if(&v) ) + return v; + + if ( ! std::get(v) ) + continue; + + tv->Assign(std::move(std::get(v)), nullptr); + } + + return tv; + } + + case TYPE_RECORD: + { + if ( ! j.IsObject() ) + return mismatch_err(); + + auto rt = t->AsRecordType(); + auto rv = make_intrusive(IntrusivePtr{NewRef{}, rt}); + for ( int i = 0; i < rt->NumFields(); ++i ) + { + auto td_i = rt->FieldDecl(i); + auto m_it = j.FindMember(td_i->id); + bool has_member = m_it != j.MemberEnd(); + bool member_is_null = has_member && m_it->value.IsNull(); + + if ( ! has_member || member_is_null ) + { + if ( ! td_i->GetAttr(detail::ATTR_OPTIONAL) && + ! td_i->GetAttr(detail::ATTR_DEFAULT) ) + return util::fmt("required field %s$%s is %s in JSON", t->GetName().c_str(), + td_i->id, member_is_null ? "null" : "missing"); + + continue; + } + + auto v = BuildVal(m_it->value, td_i->type); + if ( ! get_if(&v) ) + return v; + + rv->Assign(i, std::move(std::get(v))); + } + + return rv; + } + + case TYPE_LIST: + { + if ( ! j.IsArray() ) + return mismatch_err(); + + auto lt = t->AsTypeList(); + + if ( j.GetArray().Size() < lt->GetTypes().size() ) + return "index type doesn't match"; + + auto lv = make_intrusive(TYPE_ANY); + + for ( size_t i = 0; i < lt->GetTypes().size(); i++ ) + { + auto v = BuildVal(j.GetArray()[i], lt->GetTypes()[i]); + if ( ! get_if(&v) ) + return v; + + lv->Append(std::move(std::get(v))); + } + + return lv; + } + + case TYPE_VECTOR: + { + if ( ! j.IsArray() ) + return mismatch_err(); + + auto vt = t->AsVectorType(); + auto vv = make_intrusive(IntrusivePtr{NewRef{}, vt}); + for ( const auto& item : j.GetArray() ) + { + auto v = BuildVal(item, vt->Yield()); + if ( ! get_if(&v) ) + return v; + + if ( ! std::get(v) ) + continue; + + vv->Assign(vv->Size(), std::move(std::get(v))); + } + + return vv; + } + + default: + return util::fmt("type '%s' unsupport", type_name(t->Tag())); + } + } + +std::variant ValFromJSON(std::string_view json_str, const TypePtr& t) + { + rapidjson::Document doc; + rapidjson::ParseResult ok = doc.Parse(json_str.data(), json_str.length()); + + if ( ! ok ) + return util::fmt("JSON parse error: %s Offset: %lu", rapidjson::GetParseError_En(ok.Code()), + ok.Offset()); + + return BuildVal(doc, t); + } + ValPtr StringVal::DoClone(CloneState* state) { // We could likely treat this type as immutable and return a reference diff --git a/src/Val.h b/src/Val.h index 7290a6eacd..0cd9dc3eb6 100644 --- a/src/Val.h +++ b/src/Val.h @@ -6,6 +6,7 @@ #include #include #include +#include #include #include "zeek/IntrusivePtr.h" @@ -1794,4 +1795,9 @@ extern bool can_cast_value_to_type(const Val* v, Type* t); // specific instance later. extern bool can_cast_value_to_type(const Type* s, Type* t); +// Parses a JSON string into arbitrary Zeek data using std::variant to simulate functional exception +// handling. Returns a ValPtr if parsing was successful, or a std::string containing an error +// message if an error occurred. +extern std::variant ValFromJSON(std::string_view json_str, const TypePtr& t); + } // namespace zeek diff --git a/src/zeek.bif b/src/zeek.bif index 924cb720e1..65b6df3054 100644 --- a/src/zeek.bif +++ b/src/zeek.bif @@ -5550,12 +5550,67 @@ function anonymize_addr%(a: addr, cl: IPAddrAnonymizationClass%): addr ## ## returns: a JSON formatted string. ## -## .. zeek:see:: fmt cat cat_sep string_cat print_raw +## .. zeek:see:: fmt cat cat_sep string_cat print_raw from_json function to_json%(val: any, only_loggable: bool &default=F, field_escape_pattern: pattern &default=/^_/%): string %{ return val->ToJSON(only_loggable, field_escape_pattern); %} +## A function to convert a JSON string into Zeek values of a given type. +## +## Implicit conversion from JSON to Zeek types is implemented for: +## +## - bool +## - int, count, real +## - interval from numbers as seconds +## - time from numbers as unix timestamp +## - port from strings in "80/tcp" notation +## - addr, subnet +## - enum +## - sets +## - vectors +## - records (from JSON objects) +## +## Optional or default record fields are allowed to be missing or null in the input. +## +## s: The JSON string to parse. +## +## t: Type of Zeek data. +## +## returns: A value of type t. +## +## .. zeek:see:: to_json +function from_json%(s: string, t: any%): from_json_result + %{ + static auto result_type = zeek::id::find_type("from_json_result"); + static auto v_idx = result_type->FieldOffset("v"); + static auto valid_idx = result_type->FieldOffset("valid"); + + auto rval = zeek::make_intrusive(result_type); + + if ( t->GetType()->Tag() != zeek::TYPE_TYPE ) + { + rval->Assign(valid_idx, false); + zeek::emit_builtin_error("from_json() requires a type argument"); + return rval; + } + + auto res = zeek::detail::ValFromJSON(s->ToStdStringView(), t->AsType()->AsTypeType()->GetType()); + + if ( auto val = std::get_if(&res) ) + { + rval->Assign(v_idx, *val); + rval->Assign(valid_idx, true); + } + else + { + rval->Assign(valid_idx, false); + zeek::emit_builtin_error(std::get(res).c_str()); + } + + return rval; + %} + ## Compresses a given path by removing '..'s and the parent directory it ## references and also removing dual '/'s and extraneous '/./'s. ## diff --git a/testing/btest/Baseline/bifs.from_json-10/.stderr b/testing/btest/Baseline/bifs.from_json-10/.stderr new file mode 100644 index 0000000000..49d861c74c --- /dev/null +++ b/testing/btest/Baseline/bifs.from_json-10/.stderr @@ -0,0 +1 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. diff --git a/testing/btest/Baseline/bifs.from_json-10/.stdout b/testing/btest/Baseline/bifs.from_json-10/.stdout new file mode 100644 index 0000000000..c7202a240c --- /dev/null +++ b/testing/btest/Baseline/bifs.from_json-10/.stdout @@ -0,0 +1,6 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +[v={ +fe80::/64, +192.168.0.0/16 +}, valid=T] +[v=[1, 3, 4], valid=T] diff --git a/testing/btest/Baseline/bifs.from_json-11/.stderr b/testing/btest/Baseline/bifs.from_json-11/.stderr new file mode 100644 index 0000000000..1438154b7a --- /dev/null +++ b/testing/btest/Baseline/bifs.from_json-11/.stderr @@ -0,0 +1,3 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +error in <...>/from_json.zeek, line 8: required field Foo$hello is missing in JSON (from_json({"t":null}, to_any_coerceFoo)) +error in <...>/from_json.zeek, line 9: required field Foo$hello is null in JSON (from_json({"hello": null, "t": true}, to_any_coerceFoo)) diff --git a/testing/btest/Baseline/bifs.from_json-11/.stdout b/testing/btest/Baseline/bifs.from_json-11/.stdout new file mode 100644 index 0000000000..d288024480 --- /dev/null +++ b/testing/btest/Baseline/bifs.from_json-11/.stdout @@ -0,0 +1,3 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +[v=, valid=F] +[v=, valid=F] diff --git a/testing/btest/Baseline/bifs.from_json-12/.stderr b/testing/btest/Baseline/bifs.from_json-12/.stderr new file mode 100644 index 0000000000..49d861c74c --- /dev/null +++ b/testing/btest/Baseline/bifs.from_json-12/.stderr @@ -0,0 +1 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. diff --git a/testing/btest/Baseline/bifs.from_json-12/.stdout b/testing/btest/Baseline/bifs.from_json-12/.stdout new file mode 100644 index 0000000000..7673a47ea5 --- /dev/null +++ b/testing/btest/Baseline/bifs.from_json-12/.stdout @@ -0,0 +1,2 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +[v=[hello=Hello!], valid=T] diff --git a/testing/btest/Baseline/bifs.from_json-2/.stderr b/testing/btest/Baseline/bifs.from_json-2/.stderr new file mode 100644 index 0000000000..1ce3c885b8 --- /dev/null +++ b/testing/btest/Baseline/bifs.from_json-2/.stderr @@ -0,0 +1,2 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +error in <...>/from_json.zeek, line 4: from_json() requires a type argument (from_json([], to_any_coerce10)) diff --git a/testing/btest/Baseline/bifs.from_json-2/.stdout b/testing/btest/Baseline/bifs.from_json-2/.stdout new file mode 100644 index 0000000000..aee95c8a8e --- /dev/null +++ b/testing/btest/Baseline/bifs.from_json-2/.stdout @@ -0,0 +1,2 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +[v=, valid=F] diff --git a/testing/btest/Baseline/bifs.from_json-3/.stderr b/testing/btest/Baseline/bifs.from_json-3/.stderr new file mode 100644 index 0000000000..cd9437efcb --- /dev/null +++ b/testing/btest/Baseline/bifs.from_json-3/.stderr @@ -0,0 +1,2 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +error in <...>/from_json.zeek, line 4: JSON parse error: Missing a closing quotation mark in string. Offset: 5 (from_json({"hel, to_any_coercestring_vec)) diff --git a/testing/btest/Baseline/bifs.from_json-3/.stdout b/testing/btest/Baseline/bifs.from_json-3/.stdout new file mode 100644 index 0000000000..aee95c8a8e --- /dev/null +++ b/testing/btest/Baseline/bifs.from_json-3/.stdout @@ -0,0 +1,2 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +[v=, valid=F] diff --git a/testing/btest/Baseline/bifs.from_json-4/.stderr b/testing/btest/Baseline/bifs.from_json-4/.stderr new file mode 100644 index 0000000000..210255a35f --- /dev/null +++ b/testing/btest/Baseline/bifs.from_json-4/.stderr @@ -0,0 +1,3 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +error in <...>/from_json.zeek, line 9: cannot convert JSON type 'array' to Zeek type 'bool' (from_json([], to_any_coercebool_t)) +error in <...>/from_json.zeek, line 10: cannot convert JSON type 'string' to Zeek type 'bool' (from_json({"a": "hello"}, to_any_coerceFoo)) diff --git a/testing/btest/Baseline/bifs.from_json-4/.stdout b/testing/btest/Baseline/bifs.from_json-4/.stdout new file mode 100644 index 0000000000..d288024480 --- /dev/null +++ b/testing/btest/Baseline/bifs.from_json-4/.stdout @@ -0,0 +1,3 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +[v=, valid=F] +[v=, valid=F] diff --git a/testing/btest/Baseline/bifs.from_json-5/.stderr b/testing/btest/Baseline/bifs.from_json-5/.stderr new file mode 100644 index 0000000000..b08347df71 --- /dev/null +++ b/testing/btest/Baseline/bifs.from_json-5/.stderr @@ -0,0 +1,2 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +error in <...>/from_json.zeek, line 4: tables are not supported (from_json([], to_any_coercetable_string_of_string)) diff --git a/testing/btest/Baseline/bifs.from_json-5/.stdout b/testing/btest/Baseline/bifs.from_json-5/.stdout new file mode 100644 index 0000000000..aee95c8a8e --- /dev/null +++ b/testing/btest/Baseline/bifs.from_json-5/.stdout @@ -0,0 +1,2 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +[v=, valid=F] diff --git a/testing/btest/Baseline/bifs.from_json-6/.stderr b/testing/btest/Baseline/bifs.from_json-6/.stderr new file mode 100644 index 0000000000..fa015b8031 --- /dev/null +++ b/testing/btest/Baseline/bifs.from_json-6/.stderr @@ -0,0 +1,2 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +error in <...>/from_json.zeek, line 5: wrong port format, must be <...>/(tcp|udp|icmp|unknown)/ (from_json("80", to_any_coerceport_t)) diff --git a/testing/btest/Baseline/bifs.from_json-6/.stdout b/testing/btest/Baseline/bifs.from_json-6/.stdout new file mode 100644 index 0000000000..aee95c8a8e --- /dev/null +++ b/testing/btest/Baseline/bifs.from_json-6/.stdout @@ -0,0 +1,2 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +[v=, valid=F] diff --git a/testing/btest/Baseline/bifs.from_json-7/.stderr b/testing/btest/Baseline/bifs.from_json-7/.stderr new file mode 100644 index 0000000000..b3789904c6 --- /dev/null +++ b/testing/btest/Baseline/bifs.from_json-7/.stderr @@ -0,0 +1,3 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +error in <...>/from_json.zeek, line 5: index type doesn't match (from_json([[1, false], [2]], to_any_coerceset_t)) +error in <...>/from_json.zeek, line 6: cannot convert JSON type 'number' to Zeek type 'bool' (from_json([[1, false], [2, 1]], to_any_coerceset_t)) diff --git a/testing/btest/Baseline/bifs.from_json-7/.stdout b/testing/btest/Baseline/bifs.from_json-7/.stdout new file mode 100644 index 0000000000..d288024480 --- /dev/null +++ b/testing/btest/Baseline/bifs.from_json-7/.stdout @@ -0,0 +1,3 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +[v=, valid=F] +[v=, valid=F] diff --git a/testing/btest/Baseline/bifs.from_json-8/.stderr b/testing/btest/Baseline/bifs.from_json-8/.stderr new file mode 100644 index 0000000000..18bb4fa2eb --- /dev/null +++ b/testing/btest/Baseline/bifs.from_json-8/.stderr @@ -0,0 +1,3 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +error: error compiling pattern /^?(.|\n)*(([[:print:]]{-}[[:alnum:]]foo))/ +error in <...>/from_json.zeek, line 5: error compiling pattern (from_json("/([[:print:]]{-}[[:alnum:]]foo)/", to_any_coercepattern_t)) diff --git a/testing/btest/Baseline/bifs.from_json-8/.stdout b/testing/btest/Baseline/bifs.from_json-8/.stdout new file mode 100644 index 0000000000..aee95c8a8e --- /dev/null +++ b/testing/btest/Baseline/bifs.from_json-8/.stdout @@ -0,0 +1,2 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +[v=, valid=F] diff --git a/testing/btest/Baseline/bifs.from_json-9/.stderr b/testing/btest/Baseline/bifs.from_json-9/.stderr new file mode 100644 index 0000000000..60cbecd370 --- /dev/null +++ b/testing/btest/Baseline/bifs.from_json-9/.stderr @@ -0,0 +1,2 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +error in <...>/from_json.zeek, line 7: 'Yellow' is not a valid enum for 'Color'. (from_json("Yellow", to_any_coerceColor)) diff --git a/testing/btest/Baseline/bifs.from_json-9/.stdout b/testing/btest/Baseline/bifs.from_json-9/.stdout new file mode 100644 index 0000000000..aee95c8a8e --- /dev/null +++ b/testing/btest/Baseline/bifs.from_json-9/.stdout @@ -0,0 +1,2 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +[v=, valid=F] diff --git a/testing/btest/Baseline/bifs.from_json/.stderr b/testing/btest/Baseline/bifs.from_json/.stderr new file mode 100644 index 0000000000..49d861c74c --- /dev/null +++ b/testing/btest/Baseline/bifs.from_json/.stderr @@ -0,0 +1 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. diff --git a/testing/btest/Baseline/bifs.from_json/.stdout b/testing/btest/Baseline/bifs.from_json/.stdout new file mode 100644 index 0000000000..24f35f7b9b --- /dev/null +++ b/testing/btest/Baseline/bifs.from_json/.stdout @@ -0,0 +1,8 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +[v=[hello=world, t=T, f=F, n=, m=, def=123, i=123, pi=3.1416, a=[1, 2, 3, 4], c1=A::Blue, p=1500/tcp, ti=1681652265.042767, it=1.0 hr 23.0 mins 20.0 secs, ad=127.0.0.1, s=::1/128, re=/^?(a)$?/, su={ +aa:bb::/32, +192.168.0.0/16 +}, se={ +[192.168.0.1, 80/tcp] , +[2001:db8::1, 8080/udp] +}], valid=T] diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.from_json/out b/testing/btest/Baseline/scripts.base.frameworks.input.from_json/out new file mode 100644 index 0000000000..6376dd8577 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.from_json/out @@ -0,0 +1,7 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +Valid: [ip=192.168.0.1, source=local, timestamp=1990-09-22T12:13:14, timestamp_parsed=654005594.0] ({"ip": "192.168.0.1", "source": "local", "timestamp": "1990-09-22T12:13:14"}) +Valid: [ip=192.168.0.1, source=local, timestamp=1990-09-23T13:14:15, timestamp_parsed=654095655.0] ({"ip": "192.168.0.1", "source": "local", "timestamp": "1990-09-23T13:14:15"}) +Valid: [ip=192.168.0.2, source=local, timestamp=, timestamp_parsed=] ({"ip": "192.168.0.2", "source": "local"}) +Invalid: '{"source": "local"}' +Invalid: '{... ]' +Valid: [ip=8.8.4.4, source=remote, timestamp=, timestamp_parsed=] ({"ip": "8.8.4.4", "source": "remote"}) diff --git a/testing/btest/bifs/from_json.zeek b/testing/btest/bifs/from_json.zeek new file mode 100644 index 0000000000..3deefd323c --- /dev/null +++ b/testing/btest/bifs/from_json.zeek @@ -0,0 +1,137 @@ +# @TEST-EXEC: ASAN_OPTIONS="$ASAN_OPTIONS,detect_leaks=0" zeek -b %INPUT +# @TEST-EXEC: TEST_DIFF_CANONIFIER= btest-diff .stdout +# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff .stderr + +module A; + +type Color: enum { + Red = 10, + White = 20, + Blue = 30 +}; + +type Foo: record { + hello: string; + t: bool; + f: bool; + n: count &optional; + m: count &optional; # not in input + def: count &default = 123; + i: int; + pi: double; + a: string_vec; + c1: Color; + p: port; + ti: time; + it: interval; + ad: addr; + s: subnet; + re: pattern; + su: subnet_set; + se: set[addr, port]; +}; + +event zeek_init() + { + local json = "{\"hello\":\"world\",\"t\":true,\"f\":false,\"se\":[[\"192.168.0.1\", \"80/tcp\"], [\"2001:db8::1\", \"8080/udp\"]],\"n\":null,\"i\":123,\"pi\":3.1416,\"a\":[\"1\",\"2\",\"3\",\"4\"],\"su\":[\"[aa:bb::0]/32\",\"192.168.0.0/16\"],\"c1\":\"A::Blue\",\"p\":\"1500/tcp\",\"it\":5000,\"ad\":\"127.0.0.1\",\"s\":\"[::1/128]\",\"re\":\"/a/\",\"ti\":1681652265.042767}"; + print from_json(json, Foo); + } + +@TEST-START-NEXT +# argument type mismatch +event zeek_init() + { + print from_json("[]", 10); + } + +@TEST-START-NEXT +# JSON parse error +event zeek_init() + { + print from_json("{\"hel", string_vec); + } + +@TEST-START-NEXT +type bool_t: bool; +type Foo: record { + a: bool; +}; + +# type mismatch error +event zeek_init() + { + print from_json("[]", bool_t); + print from_json("{\"a\": \"hello\"}", Foo); + } + +@TEST-START-NEXT +# type unsupport error +event zeek_init() + { + print from_json("[]", table_string_of_string); + } + +@TEST-START-NEXT +type port_t: port; +# wrong port format +event zeek_init() + { + print from_json("\"80\"", port_t); + } + +@TEST-START-NEXT +type set_t: set[int, bool]; +# index type doesn't match +event zeek_init() + { + print from_json("[[1, false], [2]]", set_t); + print from_json("[[1, false], [2, 1]]", set_t); + } + +@TEST-START-NEXT +type pattern_t: pattern; +# pattern compile error +event zeek_init() + { + print from_json("\"/([[:print:]]{-}[[:alnum:]]foo)/\"", pattern_t); + } + +@TEST-START-NEXT +type Color: enum { + Red = 10 +}; +# enum error +event zeek_init() + { + print from_json("\"Yellow\"", Color); + } + +@TEST-START-NEXT +# container null +event zeek_init() + { + print from_json("[\"fe80::/64\",null,\"192.168.0.0/16\"]", subnet_set); + print from_json("[\"1\",null,\"3\",\"4\"]", string_vec); + } + +@TEST-START-NEXT +type Foo: record { + hello: string; + t: bool; +}; +# record field null or missing +event zeek_init() + { + print from_json("{\"t\":null}", Foo); + print from_json("{\"hello\": null, \"t\": true}", Foo); + } + +@TEST-START-NEXT +type Foo: record { + hello: string; +}; +# extra fields are alright +event zeek_init() + { + print from_json("{\"hello\": \"Hello!\", \"t\": true}", Foo); + } diff --git a/testing/btest/scripts/base/frameworks/input/from_json.zeek b/testing/btest/scripts/base/frameworks/input/from_json.zeek new file mode 100644 index 0000000000..547738ad29 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/input/from_json.zeek @@ -0,0 +1,71 @@ +# @TEST-DOC: Reading a jsonl file using the raw input reader and parsing via from_json() +# @TEST-EXEC: zeek -b %INPUT >out +# @TEST-EXEC: TEST_DIFF_CANONIFIER= btest-diff out + +@TEST-START-FILE denylist.jsonl +{"ip": "192.168.0.1", "source": "local", "timestamp": "1990-09-22T12:13:14"} +{"ip": "192.168.0.1", "source": "local", "timestamp": "1990-09-23T13:14:15"} +{"ip": "192.168.0.2", "source": "local"} +{"source": "local"} +{... ] +{"ip": "8.8.4.4", "source": "remote"} +@TEST-END-FILE + +redef exit_only_after_terminate = T; + +module A; + +type Line: record { + l: string; +}; + +type Deny: record { + ip: addr; + source: string; + timestamp: string &optional; + timestamp_parsed: time &optional; +}; + +event line(description: Input::EventDescription, tpe: Input::Event, line: string) + { + local r = from_json(line, Deny); + if ( r$valid ) + { + local deny = r$v as Deny; + if ( deny?$timestamp ) + deny$timestamp_parsed = strptime("%Y-%m-%dT%H:%M:%S", deny$timestamp); + + print fmt("Valid: %s (%s)", deny, line); + } + else + print fmt("Invalid: '%s'", line); + } + +event die() + { + if ( zeek_is_terminating() ) + return; + + print "error: test timeout"; + exit(1); + } + +event zeek_init() + { + Input::add_event([ + $source="denylist.jsonl", + $name="denylist", + $reader=Input::READER_RAW, + $fields=Line, + $ev=line, + $want_record=F + ]); + + schedule 5sec { die() }; + } + +event Input::end_of_data(name: string, source:string) + { + Input::remove("deny"); + terminate(); + }