diff --git a/CHANGES b/CHANGES index 5f73bfa690..bd18f26349 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,11 @@ +2.6-586 | 2019-07-11 11:15:40 -0700 + + * Convert all JSON output to use an external library for better consistency (Tim Wojtulewicz, Corelight) + + See NEWS for more details; this makes to_json a bif and causes slight changes in its + output, as well as the output of the JSON logger. + 2.6-576 | 2019-07-10 18:38:54 -0700 * Remove unused option: chunked_io_buffer_soft_cap (Jon Siwek, Corelight) diff --git a/NEWS b/NEWS index f59147d173..a4e19ab60e 100644 --- a/NEWS +++ b/NEWS @@ -347,6 +347,16 @@ Changed Functionality of each other on separate cluster nodes to all be logged rather than suppressed and de-duplicated into a single notice. + +- to_json is now a bif, no longer a script. Loading base/utils/json.zeek is no + longer necessary and has been deprecated. to_json should yield much better, always + valid json. There are some small differences in output; unnecessary spaces are removed + and port values are rendered differently, now including the port and the protocol. + +- The output of the JSON logger now uses an external library to generate json. There + are small changes to the output; most visibly double numbers are now rounded slightly + differently. The way in which port values are rendered does _not_ change for JSON logs. + Removed Functionality --------------------- diff --git a/VERSION b/VERSION index e16e9a62a0..4f1f474483 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.6-576 +2.6-586 diff --git a/doc b/doc index fb4942d261..bcfe6ffc88 160000 --- a/doc +++ b/doc @@ -1 +1 @@ -Subproject commit fb4942d261ebd2c8160a0862b527f3639ebf1245 +Subproject commit bcfe6ffc88e0a89e7ade664113d458bae9e5e5fc diff --git a/scripts/base/frameworks/openflow/plugins/ryu.zeek b/scripts/base/frameworks/openflow/plugins/ryu.zeek index cc400293a0..08e8c8d022 100644 --- a/scripts/base/frameworks/openflow/plugins/ryu.zeek +++ b/scripts/base/frameworks/openflow/plugins/ryu.zeek @@ -3,7 +3,6 @@ @load base/frameworks/openflow @load base/utils/active-http @load base/utils/exec -@load base/utils/json module OpenFlow; diff --git a/scripts/base/utils/json.zeek b/scripts/base/utils/json.zeek index 5bce89d18b..a9662e74a2 100644 --- a/scripts/base/utils/json.zeek +++ b/scripts/base/utils/json.zeek @@ -1,109 +1,2 @@ -##! Functions to assist with generating JSON data from Zeek data scructures. -# We might want to implement this in core somtime, this looks... hacky at best. - -@load base/utils/strings - -## A function to convert arbitrary Zeek data into a JSON string. -## -## v: The value to convert to JSON. Typically a record. -## -## only_loggable: If the v value is a record this will only cause -## fields with the &log attribute to be included in the JSON. -## -## returns: a JSON formatted string. -function to_json(v: any, only_loggable: bool &default=F, field_escape_pattern: pattern &default=/^_/): string - { - local tn = type_name(v); - switch ( tn ) - { - case "type": - return ""; - - case "string": - return cat("\"", gsub(gsub(clean(v), /\\/, "\\\\"), /\"/, "\\\""), "\""); - - case "port": - return cat(port_to_count(to_port(cat(v)))); - - case "enum": - fallthrough; - case "interval": - fallthrough; - case "addr": - fallthrough; - case "subnet": - return cat("\"", v, "\""); - - case "int": - fallthrough; - case "count": - fallthrough; - case "time": - return cat(v); - - case "double": - return fmt("%.16g", v); - - case "bool": - local bval: bool = v; - return bval ? "true" : "false"; - - default: - break; - } - - if ( /^record/ in tn ) - { - local rec_parts: string_vec = vector(); - - local ft = record_fields(v); - for ( field, field_desc in ft ) - { - # replace the escape pattern in the field. - if( field_escape_pattern in field ) - field = cat(sub(field, field_escape_pattern, "")); - if ( field_desc?$value && (!only_loggable || field_desc$log) ) - { - local onepart = cat("\"", field, "\": ", to_json(field_desc$value, only_loggable)); - rec_parts += onepart; - } - } - return cat("{", join_string_vec(rec_parts, ", "), "}"); - } - - # None of the following are supported. - else if ( /^set/ in tn ) - { - local set_parts: string_vec = vector(); - local sa: set[bool] = v; - for ( sv in sa ) - { - set_parts += to_json(sv, only_loggable); - } - return cat("[", join_string_vec(set_parts, ", "), "]"); - } - else if ( /^table/ in tn ) - { - local tab_parts: vector of string = vector(); - local ta: table[bool] of any = v; - for ( ti, tv in ta ) - { - local ts = to_json(ti); - local if_quotes = (ts[0] == "\"") ? "" : "\""; - tab_parts += cat(if_quotes, ts, if_quotes, ": ", to_json(tv, only_loggable)); - } - return cat("{", join_string_vec(tab_parts, ", "), "}"); - } - else if ( /^vector/ in tn ) - { - local vec_parts: string_vec = vector(); - local va: vector of any = v; - for ( vi in va ) - { - vec_parts += to_json(va[vi], only_loggable); - } - return cat("[", join_string_vec(vec_parts, ", "), "]"); - } - - return "\"\""; - } +## This file is deprecated in favor of to_json in zeek.bif +@deprecated="Remove in 3.1. to_json is now always available as a built-in function." diff --git a/src/3rdparty b/src/3rdparty index 785e581f00..1e9d49362d 160000 --- a/src/3rdparty +++ b/src/3rdparty @@ -1 +1 @@ -Subproject commit 785e581f00a1efae3fca7a62fb15d8756c5aedb1 +Subproject commit 1e9d49362d2c3bb2f43abcd8eebe47be045659a5 diff --git a/src/Val.cc b/src/Val.cc index 57bfbb3a5e..09cbc80691 100644 --- a/src/Val.cc +++ b/src/Val.cc @@ -27,6 +27,16 @@ #include "broker/Data.h" +#include "3rdparty/json.hpp" +#include "3rdparty/fifo_map.hpp" + +// Define a class for use with the json library that orders the keys in the same order that +// they were inserted. By default, the json library orders them alphabetically and we don't +// want it like that. +template +using json_fifo_map = nlohmann::fifo_map, A>; +using ZeekJson = nlohmann::basic_json; + Val::Val(Func* f) { val.func_val = f; @@ -380,6 +390,274 @@ bool Val::WouldOverflow(const BroType* from_type, const BroType* to_type, const return false; } +TableVal* Val::GetRecordFields() + { + TableVal* fields = new TableVal(internal_type("record_field_table")->AsTableType()); + + auto t = Type(); + + if ( t->Tag() != TYPE_RECORD && t->Tag() != TYPE_TYPE ) + { + reporter->Error("non-record value/type passed to record_fields"); + return fields; + } + + RecordType* rt = nullptr; + RecordVal* rv = nullptr; + + if ( t->Tag() == TYPE_RECORD ) + { + rt = t->AsRecordType(); + rv = AsRecordVal(); + } + else + { + t = t->AsTypeType()->Type(); + + if ( t->Tag() != TYPE_RECORD ) + { + reporter->Error("non-record value/type passed to record_fields"); + return fields; + } + + rt = t->AsRecordType(); + } + + for ( int i = 0; i < rt->NumFields(); ++i ) + { + BroType* ft = rt->FieldType(i); + TypeDecl* fd = rt->FieldDecl(i); + Val* fv = nullptr; + + if ( rv ) + fv = rv->Lookup(i); + + if ( fv ) + ::Ref(fv); + + bool logged = (fd->attrs && fd->FindAttr(ATTR_LOG) != 0); + + RecordVal* nr = new RecordVal(internal_type("record_field")->AsRecordType()); + + if ( ft->Tag() == TYPE_RECORD ) + nr->Assign(0, new StringVal("record " + ft->GetName())); + else + nr->Assign(0, new StringVal(type_name(ft->Tag()))); + + nr->Assign(1, val_mgr->GetBool(logged)); + nr->Assign(2, fv); + nr->Assign(3, rt->FieldDefault(i)); + + Val* field_name = new StringVal(rt->FieldName(i)); + fields->Assign(field_name, nr); + Unref(field_name); + } + + return fields; + } + +// This is a static method in this file to avoid including json.hpp in Val.h since it's huge. +static ZeekJson BuildJSON(Val* val, bool only_loggable=false, RE_Matcher* re=new RE_Matcher("^_")) + { + ZeekJson j; + BroType* type = val->Type(); + switch ( type->Tag() ) + { + case TYPE_BOOL: + j = val->AsBool(); + break; + + case TYPE_INT: + j = val->AsInt(); + break; + + case TYPE_COUNT: + j = val->AsCount(); + break; + + case TYPE_COUNTER: + j = val->AsCounter(); + break; + + case TYPE_TIME: + j = val->AsTime(); + break; + + case TYPE_DOUBLE: + j = val->AsDouble(); + break; + + case TYPE_PORT: + { + auto* pval = val->AsPortVal(); + j["port"] = pval->Port(); + j["proto"] = pval->Protocol(); + break; + } + + case TYPE_PATTERN: + case TYPE_INTERVAL: + case TYPE_ADDR: + case TYPE_SUBNET: + { + ODesc d; + d.SetStyle(RAW_STYLE); + val->Describe(&d); + + auto* bs = new BroString(1, d.TakeBytes(), d.Len()); + j = string((char*)bs->Bytes(), bs->Len()); + + delete bs; + break; + } + + case TYPE_FILE: + case TYPE_FUNC: + case TYPE_ENUM: + case TYPE_STRING: + { + ODesc d; + d.SetStyle(RAW_STYLE); + val->Describe(&d); + + auto* bs = new BroString(1, d.TakeBytes(), d.Len()); + j = json_escape_utf8(string((char*)bs->Bytes(), bs->Len())); + + delete bs; + break; + } + + case TYPE_TABLE: + { + auto* table = val->AsTable(); + auto* tval = val->AsTableVal(); + + if ( tval->Type()->IsSet() ) + j = ZeekJson::array(); + else + j = ZeekJson::object(); + + HashKey* k; + auto c = table->InitForIteration(); + while ( table->NextEntry(k, c) ) + { + auto lv = tval->RecoverIndex(k); + delete k; + + if ( tval->Type()->IsSet() ) + { + auto* value = lv->Index(0)->Ref(); + j.push_back(BuildJSON(value, only_loggable, re)); + Unref(value); + } + else + { + ZeekJson key_json; + Val* entry_value; + if ( lv->Length() == 1 ) + { + Val* entry_key = lv->Index(0)->Ref(); + entry_value = tval->Lookup(entry_key, true); + key_json = BuildJSON(entry_key, only_loggable, re); + Unref(entry_key); + } + else + { + entry_value = tval->Lookup(lv, true); + key_json = BuildJSON(lv, only_loggable, re); + } + + string key_string; + if ( key_json.is_string() ) + key_string = key_json; + else + key_string = key_json.dump(); + + j[key_string] = BuildJSON(entry_value, only_loggable, re); + } + + Unref(lv); + } + + break; + } + + case TYPE_RECORD: + { + j = ZeekJson::object(); + auto* rval = val->AsRecordVal(); + TableVal* fields = rval->GetRecordFields(); + auto* field_indexes = fields->ConvertToPureList(); + int num_indexes = field_indexes->Length(); + + for ( int i = 0; i < num_indexes; ++i ) + { + Val* key = field_indexes->Index(i); + auto* key_field = fields->Lookup(key)->AsRecordVal(); + + auto* key_val = key->AsStringVal(); + string key_string; + if ( re->MatchAnywhere(key_val->AsString()) != 0 ) + { + key_val = key_val->Substitute(re, new StringVal(""), 0)->AsStringVal(); + key_string = key_val->ToStdString(); + delete key_val; + } + else + key_string = key_val->ToStdString(); + + Val* value = key_field->Lookup("value", true); + + if ( value && ( ! only_loggable || key_field->Lookup("log")->AsBool() ) ) + j[key_string] = BuildJSON(value, only_loggable, re); + } + + delete fields; + break; + } + + case TYPE_LIST: + { + j = ZeekJson::array(); + auto* lval = val->AsListVal(); + size_t size = lval->Length(); + for (size_t i = 0; i < size; i++) + j.push_back(BuildJSON(lval->Index(i), only_loggable, re)); + + break; + } + + case TYPE_VECTOR: + { + j = ZeekJson::array(); + auto* vval = val->AsVectorVal(); + size_t size = vval->SizeVal()->AsCount(); + for (size_t i = 0; i < size; i++) + j.push_back(BuildJSON(vval->Lookup(i), only_loggable, re)); + + break; + } + + case TYPE_OPAQUE: + { + j = ZeekJson::object(); + auto* oval = val->AsOpaqueVal(); + j["opaque_type"] = OpaqueMgr::mgr()->TypeID(oval); + break; + } + + default: break; + } + + return j; + } + +StringVal* Val::ToJSON(bool only_loggable, RE_Matcher* re) + { + ZeekJson j = BuildJSON(this, only_loggable, re); + return new StringVal(j.dump()); + } + IntervalVal::IntervalVal(double quantity, double units) : Val(quantity * units, TYPE_INTERVAL) { @@ -491,6 +769,18 @@ uint32 PortVal::Port() const return p & ~PORT_SPACE_MASK; } +string PortVal::Protocol() const + { + if ( IsUDP() ) + return "udp"; + else if ( IsTCP() ) + return "tcp"; + else if ( IsICMP() ) + return "icmp"; + else + return "unknown"; + } + int PortVal::IsTCP() const { return (val.uint_val & PORT_SPACE_MASK) == TCP_PORT_MASK; @@ -510,14 +800,8 @@ void PortVal::ValDescribe(ODesc* d) const { uint32 p = static_cast(val.uint_val); d->Add(p & ~PORT_SPACE_MASK); - if ( IsUDP() ) - d->Add("/udp"); - else if ( IsTCP() ) - d->Add("/tcp"); - else if ( IsICMP() ) - d->Add("/icmp"); - else - d->Add("/unknown"); + d->Add("/"); + d->Add(Protocol()); } Val* PortVal::DoClone(CloneState* state) @@ -713,6 +997,12 @@ StringVal::StringVal(const string& s) : Val(TYPE_STRING) val.string_val = new BroString(reinterpret_cast(s.data()), s.length(), 1); } +string StringVal::ToStdString() const + { + auto* bs = AsString(); + return string((char*)bs->Bytes(), bs->Len()); + } + StringVal* StringVal::ToUpper() { val.string_val->ToUpper(); @@ -734,6 +1024,92 @@ unsigned int StringVal::MemoryAllocation() const return padded_sizeof(*this) + val.string_val->MemoryAllocation(); } +Val* StringVal::Substitute(RE_Matcher* re, StringVal* repl, bool do_all) + { + const u_char* s = Bytes(); + int offset = 0; + int n = Len(); + + // cut_points is a set of pairs of indices in str that should + // be removed/replaced. A pair means "delete starting + // at offset x, up to but not including offset y". + List(ptr_compat_int) cut_points; // where RE matches pieces of str + + int size = 0; // size of result + + while ( n > 0 ) + { + // Find next match offset. + int end_of_match; + while ( n > 0 && + (end_of_match = re->MatchPrefix(&s[offset], n)) <= 0 ) + { + // This character is going to be copied to the result. + ++size; + + // Move on to next character. + ++offset; + --n; + } + + if ( n <= 0 ) + break; + + // s[offset .. offset+end_of_match-1] matches re. + cut_points.append(offset); + cut_points.append(offset + end_of_match); + + offset += end_of_match; + n -= end_of_match; + + if ( ! do_all ) + { + // We've now done the first substitution - finished. + // Include the remainder of the string in the result. + size += n; + break; + } + } + + // size now reflects amount of space copied. Factor in amount + // of space for replacement text. + int num_cut_points = cut_points.length() / 2; + size += num_cut_points * repl->Len(); + + // And a final NUL for good health. + ++size; + + byte_vec result = new u_char[size]; + byte_vec r = result; + + // Copy it all over. + int start_offset = 0; + for ( int i = 0; i < cut_points.length(); i += 2 /* loop over pairs */ ) + { + int num_to_copy = cut_points[i] - start_offset; + memcpy(r, s + start_offset, num_to_copy); + + r += num_to_copy; + start_offset = cut_points[i+1]; + + // Now add in replacement text. + memcpy(r, repl->Bytes(), repl->Len()); + r += repl->Len(); + } + + // Copy final trailing characters. + int num_to_copy = Len() - start_offset; + memcpy(r, s + start_offset, num_to_copy); + r += num_to_copy; + + // Final NUL. No need to increment r, since the length + // computed from it in the next statement does not include + // the NUL. + r[0] = '\0'; + + return new StringVal(new BroString(1, result, r - result)); + } + Val* StringVal::DoClone(CloneState* state) { // We could likely treat this type as immutable and return a reference diff --git a/src/Val.h b/src/Val.h index 3b895dab14..fb85ce2794 100644 --- a/src/Val.h +++ b/src/Val.h @@ -20,6 +20,7 @@ #include "Notifier.h" #include "IPAddr.h" #include "DebugLogger.h" +#include "RE.h" // We have four different port name spaces: TCP, UDP, ICMP, and UNKNOWN. // We distinguish between them based on the bits specified in the *_PORT_MASK @@ -34,7 +35,6 @@ class Val; class Func; class BroFile; -class RE_Matcher; class PrefixTable; class PortVal; @@ -347,6 +347,10 @@ public: static bool WouldOverflow(const BroType* from_type, const BroType* to_type, const Val* val); + TableVal* GetRecordFields(); + + StringVal* ToJSON(bool only_loggable=false, RE_Matcher* re=new RE_Matcher("^_")); + protected: friend class EnumType; @@ -530,6 +534,7 @@ public: // Returns the port number in host order (not including the mask). uint32 Port() const; + string Protocol() const; // Tests for protocol types. int IsTCP() const; @@ -632,10 +637,13 @@ public: // char* ExpandedString(int format = BroString::EXPANDED_STRING) // { return AsString()->ExpandedString(format); } + std::string ToStdString() const; StringVal* ToUpper(); unsigned int MemoryAllocation() const override; + Val* Substitute(RE_Matcher* re, StringVal* repl, bool do_all); + protected: friend class Val; StringVal() {} diff --git a/src/strings.bif b/src/strings.bif index f2661f8cc9..42630e4b6b 100644 --- a/src/strings.bif +++ b/src/strings.bif @@ -351,91 +351,6 @@ Val* do_split(StringVal* str_val, RE_Matcher* re, int incl_sep, int max_num_sep) return a; } -Val* do_sub(StringVal* str_val, RE_Matcher* re, StringVal* repl, int do_all) - { - const u_char* s = str_val->Bytes(); - int offset = 0; - int n = str_val->Len(); - - // cut_points is a set of pairs of indices in str that should - // be removed/replaced. A pair means "delete starting - // at offset x, up to but not including offset y". - List(ptr_compat_int) cut_points; // where RE matches pieces of str - - int size = 0; // size of result - - while ( n > 0 ) - { - // Find next match offset. - int end_of_match; - while ( n > 0 && - (end_of_match = re->MatchPrefix(&s[offset], n)) <= 0 ) - { - // This character is going to be copied to the result. - ++size; - - // Move on to next character. - ++offset; - --n; - } - - if ( n <= 0 ) - break; - - // s[offset .. offset+end_of_match-1] matches re. - cut_points.append(offset); - cut_points.append(offset + end_of_match); - - offset += end_of_match; - n -= end_of_match; - - if ( ! do_all ) - { - // We've now done the first substitution - finished. - // Include the remainder of the string in the result. - size += n; - break; - } - } - - // size now reflects amount of space copied. Factor in amount - // of space for replacement text. - int num_cut_points = cut_points.length() / 2; - size += num_cut_points * repl->Len(); - - // And a final NUL for good health. - ++size; - - byte_vec result = new u_char[size]; - byte_vec r = result; - - // Copy it all over. - int start_offset = 0; - for ( int i = 0; i < cut_points.length(); i += 2 /* loop over pairs */ ) - { - int num_to_copy = cut_points[i] - start_offset; - memcpy(r, s + start_offset, num_to_copy); - - r += num_to_copy; - start_offset = cut_points[i+1]; - - // Now add in replacement text. - memcpy(r, repl->Bytes(), repl->Len()); - r += repl->Len(); - } - - // Copy final trailing characters. - int num_to_copy = str_val->Len() - start_offset; - memcpy(r, s + start_offset, num_to_copy); - r += num_to_copy; - - // Final NUL. No need to increment r, since the length - // computed from it in the next statement does not include - // the NUL. - r[0] = '\0'; - - return new StringVal(new BroString(1, result, r - result)); - } %%} ## Splits a string into an array of strings according to a pattern. @@ -535,7 +450,7 @@ function split_string_n%(str: string, re: pattern, ## .. zeek:see:: gsub subst_string function sub%(str: string, re: pattern, repl: string%): string %{ - return do_sub(str, re, repl, 0); + return str->Substitute(re, repl, false); %} ## Substitutes a given replacement string for all occurrences of a pattern @@ -552,7 +467,7 @@ function sub%(str: string, re: pattern, repl: string%): string ## .. zeek:see:: sub subst_string function gsub%(str: string, re: pattern, repl: string%): string %{ - return do_sub(str, re, repl, 1); + return str->Substitute(re, repl, true); %} diff --git a/src/threading/Formatter.cc b/src/threading/Formatter.cc index 395a7fefa6..5fa8ab6fa9 100644 --- a/src/threading/Formatter.cc +++ b/src/threading/Formatter.cc @@ -111,3 +111,14 @@ string Formatter::Render(double d) return buf; } +string Formatter::Render(TransportProto proto) + { + if ( proto == TRANSPORT_UDP ) + return "udp"; + else if ( proto == TRANSPORT_TCP ) + return "tcp"; + else if ( proto == TRANSPORT_ICMP ) + return "icmp"; + else + return "unknown"; + } diff --git a/src/threading/Formatter.h b/src/threading/Formatter.h index c564f3c945..53ca22254b 100644 --- a/src/threading/Formatter.h +++ b/src/threading/Formatter.h @@ -112,6 +112,17 @@ public: */ static string Render(double d); + /** + * Convert a transport protocol into a string. + * + * This is a helper function that formatter implementations may use. + * + * @param proto The transport protocol. + * + * @return An ASCII representation of the protocol. + */ + static string Render(TransportProto proto); + /** * Convert a string into a TransportProto. The string must be one of * \c tcp, \c udp, \c icmp, or \c unknown. diff --git a/src/threading/formatters/JSON.cc b/src/threading/formatters/JSON.cc index a324a08530..58b1ffd779 100644 --- a/src/threading/formatters/JSON.cc +++ b/src/threading/formatters/JSON.cc @@ -11,7 +11,7 @@ #include #include -#include "./JSON.h" +#include "JSON.h" using namespace threading::formatter; @@ -27,78 +27,83 @@ JSON::~JSON() bool JSON::Describe(ODesc* desc, int num_fields, const Field* const * fields, Value** vals) const { - if ( surrounding_braces ) - desc->AddRaw("{"); + ZeekJson j = ZeekJson::object(); for ( int i = 0; i < num_fields; i++ ) { - const u_char* bytes = desc->Bytes(); - int len = desc->Len(); + if ( vals[i]->present ) + { + ZeekJson new_entry = BuildJSON(vals[i]); + if ( new_entry.is_null() ) + return false; - if ( i > 0 && - len > 0 && - bytes[len-1] != ',' && - bytes[len-1] != '{' && - bytes[len-1] != '[' && - vals[i]->present ) - desc->AddRaw(","); - - if ( ! Describe(desc, vals[i], fields[i]->name) ) - return false; + j[fields[i]->name] = new_entry; + } } - if ( surrounding_braces ) - desc->AddRaw("}"); + desc->Add(j.dump()); return true; } bool JSON::Describe(ODesc* desc, Value* val, const string& name) const { + if ( desc->IsBinary() ) + { + GetThread()->Error("json formatter: binary format not supported"); + return false; + } + if ( ! val->present ) return true; - if ( name.size() ) - { - desc->AddRaw("\"", 1); - desc->Add(name); - desc->AddRaw("\":", 2); - } + ZeekJson j = BuildJSON(val, name); + if ( j.is_null() ) + return false; + desc->Add(j.dump()); + return true; + } + +threading::Value* JSON::ParseValue(const string& s, const string& name, TypeTag type, TypeTag subtype) const + { + GetThread()->Error("JSON formatter does not support parsing yet."); + return nullptr; + } + +ZeekJson JSON::BuildJSON(Value* val, const string& name) const + { + ZeekJson j; switch ( val->type ) { case TYPE_BOOL: - desc->AddRaw(val->val.int_val == 0 ? "false" : "true"); + j = val->val.int_val != 0; break; case TYPE_INT: - desc->Add(val->val.int_val); + j = val->val.int_val; break; case TYPE_COUNT: case TYPE_COUNTER: - desc->Add(val->val.uint_val); + j = val->val.uint_val; break; case TYPE_PORT: - desc->Add(val->val.port_val.port); + j = val->val.port_val.port; break; case TYPE_SUBNET: - desc->AddRaw("\"", 1); - desc->Add(Render(val->val.subnet_val)); - desc->AddRaw("\"", 1); + j = Formatter::Render(val->val.subnet_val); break; case TYPE_ADDR: - desc->AddRaw("\"", 1); - desc->Add(Render(val->val.addr_val)); - desc->AddRaw("\"", 1); + j = Formatter::Render(val->val.addr_val); break; case TYPE_DOUBLE: case TYPE_INTERVAL: - desc->Add(val->val.double_val); + j = val->val.double_val; break; case TYPE_TIME: @@ -110,15 +115,13 @@ bool JSON::Describe(ODesc* desc, Value* val, const string& name) const time_t the_time = time_t(floor(val->val.double_val)); struct tm t; - desc->AddRaw("\"", 1); - if ( ! gmtime_r(&the_time, &t) || ! strftime(buffer, sizeof(buffer), "%Y-%m-%dT%H:%M:%S", &t) ) { GetThread()->Error(GetThread()->Fmt("json formatter: failure getting time: (%lf)", val->val.double_val)); // This was a failure, doesn't really matter what gets put here // but it should probably stand out... - desc->Add("2000-01-01T00:00:00.000000"); + j = "2000-01-01T00:00:00.000000"; } else { @@ -129,20 +132,17 @@ bool JSON::Describe(ODesc* desc, Value* val, const string& name) const frac += 1; snprintf(buffer2, sizeof(buffer2), "%s.%06.0fZ", buffer, fabs(frac) * 1000000); - desc->Add(buffer2); + j = buffer2; } - - desc->AddRaw("\"", 1); } else if ( timestamps == TS_EPOCH ) - desc->Add(val->val.double_val); + j = val->val.double_val; else if ( timestamps == TS_MILLIS ) { // ElasticSearch uses milliseconds for timestamps - uint64_t ts = (uint64_t) (val->val.double_val * 1000); - desc->Add(ts); + j = (uint64_t) (val->val.double_val * 1000); } break; @@ -153,74 +153,40 @@ bool JSON::Describe(ODesc* desc, Value* val, const string& name) const case TYPE_FILE: case TYPE_FUNC: { - desc->AddRaw("\"", 1); - - for ( int i = 0; i < val->val.string_val.length; ++i ) - { - char c = val->val.string_val.data[i]; - - // 2byte Unicode escape special characters. - if ( c < 32 || c > 126 || c == '\n' || c == '"' || c == '\'' || c == '\\' || c == '&' ) - { - desc->AddRaw("\\u00", 4); - char hex[2] = {'0', '0'}; - bytetohex(c, hex); - desc->AddRaw(hex, 1); - desc->AddRaw(hex + 1, 1); - } - else - desc->AddRaw(&c, 1); - } - - desc->AddRaw("\"", 1); + j = json_escape_utf8(string(val->val.string_val.data, val->val.string_val.length)); break; } case TYPE_TABLE: { - desc->AddRaw("[", 1); + j = ZeekJson::array(); - for ( int j = 0; j < val->val.set_val.size; j++ ) - { - if ( j > 0 ) - desc->AddRaw(",", 1); + for ( int idx = 0; idx < val->val.set_val.size; idx++ ) + j.push_back(BuildJSON(val->val.set_val.vals[idx])); - Describe(desc, val->val.set_val.vals[j]); - } - - desc->AddRaw("]", 1); break; } case TYPE_VECTOR: { - desc->AddRaw("[", 1); + j = ZeekJson::array(); - for ( int j = 0; j < val->val.vector_val.size; j++ ) - { - if ( j > 0 ) - desc->AddRaw(",", 1); - Describe(desc, val->val.vector_val.vals[j]); - } + for ( int idx = 0; idx < val->val.vector_val.size; idx++ ) + j.push_back(BuildJSON(val->val.vector_val.vals[idx])); - desc->AddRaw("]", 1); break; } default: - return false; + break; } - return true; - } + if ( ! name.empty() && ! j.is_null() ) + { + ZeekJson j2 = ZeekJson::object(); + j2[name] = j; + return j2; + } -threading::Value* JSON::ParseValue(const string& s, const string& name, TypeTag type, TypeTag subtype) const - { - GetThread()->Error("JSON formatter does not support parsing yet."); - return NULL; - } - -void JSON::SurroundingBraces(bool use_braces) - { - surrounding_braces = use_braces; + return j; } diff --git a/src/threading/formatters/JSON.h b/src/threading/formatters/JSON.h index 4984f74067..8b9ab1ed6e 100644 --- a/src/threading/formatters/JSON.h +++ b/src/threading/formatters/JSON.h @@ -4,9 +4,19 @@ #define THREADING_FORMATTERS_JSON_H #include "../Formatter.h" +#include "3rdparty/json.hpp" +#include "3rdparty/fifo_map.hpp" + namespace threading { namespace formatter { +// Define a class for use with the json library that orders the keys in the same order that +// they were inserted. By default, the json library orders them alphabetically and we don't +// want it like that. +template +using json_fifo_map = nlohmann::fifo_map, A>; +using ZeekJson = nlohmann::basic_json; + /** * A thread-safe class for converting values into a JSON representation * and vice versa. @@ -27,9 +37,10 @@ public: threading::Value** vals) const override; threading::Value* ParseValue(const string& s, const string& name, TypeTag type, TypeTag subtype = TYPE_ERROR) const override; - void SurroundingBraces(bool use_braces); - private: + + ZeekJson BuildJSON(Value* val, const string& name = "") const; + TimeFormat timestamps; bool surrounding_braces; }; diff --git a/src/util.cc b/src/util.cc index 2a6a5c37c4..9f51fc4b64 100644 --- a/src/util.cc +++ b/src/util.cc @@ -1872,3 +1872,90 @@ char* zeekenv(const char* name) return getenv(it->second); } + +static string json_escape_byte(char c) + { + char hex[2] = {'0', '0'}; + bytetohex(c, hex); + + string result = "\\x"; + result.append(hex, 2); + + return result; + } + +string json_escape_utf8(const string& val) + { + string result; + result.reserve(val.length()); + + size_t char_start = 0; + size_t idx; + for ( idx = 0; idx < val.length(); ) + { + // Normal ASCII characters plus a few of the control characters can be inserted directly. The rest of + // the control characters should be escaped as regular bytes. + if ( ( val[idx] >= 32 && val[idx] <= 127 ) || + val[idx] == '\b' || val[idx] == '\f' || val[idx] == '\n' || val[idx] == '\r' || val[idx] == '\t' ) + { + result.push_back(val[idx]); + ++idx; + continue; + } + else if ( val[idx] >= 0 && val[idx] < 32 ) + { + result.append(json_escape_byte(val[idx])); + ++idx; + continue; + } + + // The next bit is based on the table at https://en.wikipedia.org/wiki/UTF-8#Description. + // If next character is 11110xxx, this is a 4-byte UTF-8 + int char_size = 0; + if ( (val[idx] & 0xF8) == 0xF0 ) char_size = 4; + + // If next character is 1110xxxx, this is a 3-byte UTF-8 + else if ( (val[idx] & 0xF0) == 0xE0 ) char_size = 3; + + // If next character is 110xxxxx, this is a 2-byte UTF-8 + else if ( (val[idx] & 0xE0) == 0xC0 ) char_size = 2; + + // This byte isn't a continuation byte, insert it as a byte and continue. + if ( char_size == 0) + { + result.append(json_escape_byte(val[idx])); + ++idx; + continue; + } + + // If we don't have enough bytes to get to the end of character, give up and insert all of the rest + // of them as escaped values. + if ( char_size > (val.length() - idx) ) + break; + + // Loop through the rest of the supposed character and see if this is a valid character. + size_t c_idx = idx + 1; + for ( ; c_idx < idx + char_size; c_idx++ ) + if ( (val[c_idx] & 0xC0) != 0x80 ) break; + + // if we didn't make it to the end of the character without finding an error, insert just this + // character and skip ahead. Otherwise insert all of the bytes for this character into the result. + if ( c_idx != idx + char_size ) + { + result.append(json_escape_byte(val[idx])); + ++idx; + continue; + } + else + { + for ( size_t step = 0; step < char_size; step++, idx++ ) + result.push_back(val[idx]); + } + } + + if ( idx != val.length() ) + for ( ; idx < val.length(); ++idx ) + result.append(json_escape_byte(val[idx])); + + return result; + } diff --git a/src/util.h b/src/util.h index f019f4cbc1..661e18a111 100644 --- a/src/util.h +++ b/src/util.h @@ -565,4 +565,12 @@ std::unique_ptr build_unique (Args&&... args) { return std::unique_ptr(new T(std::forward(args)...)); } +/** + * Escapes bytes in a string that are not valid UTF8 characters with \xYY format. Used + * by the JSON writer and BIF methods. + * @param val the input string to be escaped + * @return the escaped string + */ +std::string json_escape_utf8(const std::string& val); + #endif diff --git a/src/zeek.bif b/src/zeek.bif index da11a18681..8b7a047189 100644 --- a/src/zeek.bif +++ b/src/zeek.bif @@ -1981,68 +1981,7 @@ function lookup_ID%(id: string%) : any ## Returns: A table that describes the fields of a record. function record_fields%(rec: any%): record_field_table %{ - TableVal* fields = new TableVal(record_field_table); - - auto t = rec->Type(); - - if ( t->Tag() != TYPE_RECORD && t->Tag() != TYPE_TYPE ) - { - reporter->Error("non-record value/type passed to record_fields"); - return fields; - } - - RecordType* rt = nullptr; - RecordVal* rv = nullptr; - - if ( t->Tag() == TYPE_RECORD ) - { - rt = t->AsRecordType(); - rv = rec->AsRecordVal(); - } - else - { - t = t->AsTypeType()->Type(); - - if ( t->Tag() != TYPE_RECORD ) - { - reporter->Error("non-record value/type passed to record_fields"); - return fields; - } - - rt = t->AsRecordType(); - } - - for ( int i = 0; i < rt->NumFields(); ++i ) - { - BroType* ft = rt->FieldType(i); - TypeDecl* fd = rt->FieldDecl(i); - Val* fv = nullptr; - - if ( rv ) - fv = rv->Lookup(i); - - if ( fv ) - Ref(fv); - - bool logged = (fd->attrs && fd->FindAttr(ATTR_LOG) != 0); - - RecordVal* nr = new RecordVal(record_field); - - if ( ft->Tag() == TYPE_RECORD ) - nr->Assign(0, new StringVal("record " + ft->GetName())); - else - nr->Assign(0, new StringVal(type_name(ft->Tag()))); - - nr->Assign(1, val_mgr->GetBool(logged)); - nr->Assign(2, fv); - nr->Assign(3, rt->FieldDefault(i)); - - Val* field_name = new StringVal(rt->FieldName(i)); - fields->Assign(field_name, nr); - Unref(field_name); - } - - return fields; + return rec->GetRecordFields(); %} ## Enables detailed collection of profiling statistics. Statistics include @@ -5100,3 +5039,16 @@ function anonymize_addr%(a: addr, cl: IPAddrAnonymizationClass%): addr (enum ip_addr_anonymization_class_t) anon_class)); } %} + +## A function to convert arbitrary Zeek data into a JSON string. +## +## v: The value to convert to JSON. Typically a record. +## +## only_loggable: If the v value is a record this will only cause +## fields with the &log attribute to be included in the JSON. +## +## returns: a JSON formatted string. +function to_json%(val: any, only_loggable: bool &default=F, field_escape_pattern: pattern &default=/^_/%): string + %{ + return val->ToJSON(only_loggable, field_escape_pattern); + %} diff --git a/testing/btest/Baseline/coverage.bare-mode-errors/errors b/testing/btest/Baseline/coverage.bare-mode-errors/errors index e69de29bb2..c87b897c61 100644 --- a/testing/btest/Baseline/coverage.bare-mode-errors/errors +++ b/testing/btest/Baseline/coverage.bare-mode-errors/errors @@ -0,0 +1 @@ +warning in /Users/tim/Desktop/projects/zeek/testing/btest/../../scripts//base/utils/json.zeek, line 2: deprecated script loaded from command line arguments ="Remove in 3.1. to_json is now always available as a built-in function." diff --git a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log index 6fbc95e4ec..b78d8a2480 100644 --- a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log @@ -249,7 +249,6 @@ scripts/base/init-default.zeek scripts/base/frameworks/openflow/main.zeek scripts/base/frameworks/openflow/plugins/__load__.zeek scripts/base/frameworks/openflow/plugins/ryu.zeek - scripts/base/utils/json.zeek scripts/base/frameworks/openflow/plugins/log.zeek scripts/base/frameworks/openflow/plugins/broker.zeek scripts/base/frameworks/openflow/non-cluster.zeek diff --git a/testing/btest/Baseline/coverage.init-default/missing_loads b/testing/btest/Baseline/coverage.init-default/missing_loads index 893a603972..87361a686a 100644 --- a/testing/btest/Baseline/coverage.init-default/missing_loads +++ b/testing/btest/Baseline/coverage.init-default/missing_loads @@ -8,3 +8,4 @@ -./frameworks/openflow/cluster.zeek -./frameworks/packet-filter/cluster.zeek -./frameworks/sumstats/cluster.zeek +-./utils/json.zeek diff --git a/testing/btest/Baseline/plugins.hooks/output b/testing/btest/Baseline/plugins.hooks/output index ef7ba59161..9b70daf0c8 100644 --- a/testing/btest/Baseline/plugins.hooks/output +++ b/testing/btest/Baseline/plugins.hooks/output @@ -821,7 +821,6 @@ 0.000000 MetaHookPost LoadFile(0, base<...>/input.bif.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, base<...>/intel) -> -1 0.000000 MetaHookPost LoadFile(0, base<...>/irc) -> -1 -0.000000 MetaHookPost LoadFile(0, base<...>/json.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, base<...>/krb) -> -1 0.000000 MetaHookPost LoadFile(0, base<...>/logging) -> -1 0.000000 MetaHookPost LoadFile(0, base<...>/logging.bif.zeek) -> -1 @@ -1712,7 +1711,6 @@ 0.000000 MetaHookPre LoadFile(0, base<...>/input.bif.zeek) 0.000000 MetaHookPre LoadFile(0, base<...>/intel) 0.000000 MetaHookPre LoadFile(0, base<...>/irc) -0.000000 MetaHookPre LoadFile(0, base<...>/json.zeek) 0.000000 MetaHookPre LoadFile(0, base<...>/krb) 0.000000 MetaHookPre LoadFile(0, base<...>/logging) 0.000000 MetaHookPre LoadFile(0, base<...>/logging.bif.zeek) @@ -2611,7 +2609,6 @@ 0.000000 | HookLoadFile base<...>/input.bif.zeek 0.000000 | HookLoadFile base<...>/intel 0.000000 | HookLoadFile base<...>/irc -0.000000 | HookLoadFile base<...>/json.zeek 0.000000 | HookLoadFile base<...>/krb 0.000000 | HookLoadFile base<...>/logging 0.000000 | HookLoadFile base<...>/logging.bif.zeek diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-double/json.log b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-double/json.log index 49b3c5d172..bb0f950b13 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-double/json.log +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-double/json.log @@ -1,22 +1,22 @@ -{"d":2.153226e+09} -{"d":2.153226e+09} -{"d":2.153226e+09} +{"d":2153226000.0} +{"d":2153226000.1} +{"d":2153226000.123457} {"d":1.0} {"d":1.1} -{"d":1.123457} -{"d":-1.123457} +{"d":1.123456789} +{"d":-1.123456789} {"d":1.1234} {"d":0.1234} {"d":50000.0} {"d":-50000.0} -{"d":3.140000e+15} -{"d":-3.140000e+15} -{"d":1.790000e+308} -{"d":-1.790000e+308} -{"d":0.000012} -{"d":0} -{"d":-0} -{"d":inf} -{"d":-inf} -{"d":0.0} -{"d":nan} +{"d":3.14e+15} +{"d":-3.14e+15} +{"d":1.79e+308} +{"d":-1.79e+308} +{"d":1.23456789e-05} +{"d":2.23e-308} +{"d":-2.23e-308} +{"d":null} +{"d":null} +{"d":-0.0} +{"d":null} diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-double/test.log b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-double/test.log index 9d5dd6ecf0..21457f916c 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-double/test.log +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-double/test.log @@ -3,7 +3,7 @@ #empty_field (empty) #unset_field - #path test -#open 2017-11-06-19-58-08 +#open 2019-07-01-17-40-55 #fields d #types double 2153226000.0 @@ -28,4 +28,4 @@ inf -inf 0.0 nan -#close 2017-11-06-19-58-08 +#close 2019-07-01-17-40-55 diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-json-utf8/ssh.log b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-json-utf8/ssh.log new file mode 100644 index 0000000000..b1a395906f --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-json-utf8/ssh.log @@ -0,0 +1,12 @@ +{"s":"a"} +{"s":"\b\f\n\r\t\\x00\\x15"} +{"s":"ñ"} +{"s":"\\xc3("} +{"s":"\\xa0\\xa1"} +{"s":"₡"} +{"s":"\\xe2(\\xa1"} +{"s":"\\xe2\\x82("} +{"s":"𐌼"} +{"s":"\\xf0(\\x8c\\xbc"} +{"s":"\\xf0\\x90(\\xbc"} +{"s":"\\xf0(\\x8c("} diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-json/ssh.log b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-json/ssh.log index 16ba5ddb60..52c5e4856c 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-json/ssh.log +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-json/ssh.log @@ -1 +1 @@ -{"b":true,"i":-42,"e":"SSH::LOG","c":21,"p":123,"sn":"10.0.0.0/24","a":"1.2.3.4","d":3.14,"t":1215620010.54321,"iv":100.0,"s":"hurz","sc":[2,4,1,3],"ss":["BB","AA","CC"],"se":[],"vc":[10,20,30],"ve":[],"f":"SSH::foo\u000a{ \u000aif (0 < SSH::i) \u000a\u0009return (Foo);\u000aelse\u000a\u0009return (Bar);\u000a\u000a}"} +{"b":true,"i":-42,"e":"SSH::LOG","c":21,"p":123,"sn":"10.0.0.0/24","a":"1.2.3.4","d":3.14,"t":1215620010.54321,"iv":100.0,"s":"hurz","sc":[2,4,1,3],"ss":["BB","AA","CC"],"se":[],"vc":[10,20,30],"ve":[],"f":"SSH::foo\n{ \nif (0 < SSH::i) \n\treturn (Foo);\nelse\n\treturn (Bar);\n\n}"} diff --git a/testing/btest/Baseline/scripts.base.frameworks.openflow.ryu-basic/.stdout b/testing/btest/Baseline/scripts.base.frameworks.openflow.ryu-basic/.stdout index 31cb56f05a..20db16d9ff 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.openflow.ryu-basic/.stdout +++ b/testing/btest/Baseline/scripts.base.frameworks.openflow.ryu-basic/.stdout @@ -1,22 +1,22 @@ http://127.0.0.1:8080/stats/flowentry/clear/42 http://127.0.0.1:8080/stats/flowentry/add -{"priority": 0, "idle_timeout": 0, "hard_timeout": 0, "match": {}, "actions": [{"port": 3, "type": "OUTPUT"}, {"port": 7, "type": "OUTPUT"}], "cookie": 4398046511105, "flags": 0, "dpid": 42} +{"priority":0,"idle_timeout":0,"hard_timeout":0,"match":{},"actions":[{"port":3,"type":"OUTPUT"},{"port":7,"type":"OUTPUT"}],"cookie":4398046511105,"flags":0,"dpid":42} Flow_mod_success http://127.0.0.1:8080/stats/flowentry/add -{"priority": 5, "idle_timeout": 30, "hard_timeout": 0, "match": {"nw_dst": "74.53.140.153/32", "tp_dst": 25, "tp_src": 1470, "nw_proto": 6, "dl_type": 2048, "nw_src": "10.10.1.4/32"}, "actions": [], "cookie": 4398046511146, "flags": 0, "dpid": 42} +{"priority":5,"idle_timeout":30,"hard_timeout":0,"match":{"nw_dst":"74.53.140.153/32","tp_dst":25,"tp_src":1470,"nw_proto":6,"dl_type":2048,"nw_src":"10.10.1.4/32"},"actions":[],"cookie":4398046511146,"flags":0,"dpid":42} http://127.0.0.1:8080/stats/flowentry/add -{"priority": 5, "idle_timeout": 30, "hard_timeout": 0, "match": {"nw_dst": "10.10.1.4/32", "tp_dst": 1470, "tp_src": 25, "nw_proto": 6, "dl_type": 2048, "nw_src": "74.53.140.153/32"}, "actions": [], "cookie": 4398046511146, "flags": 0, "dpid": 42} +{"priority":5,"idle_timeout":30,"hard_timeout":0,"match":{"nw_dst":"10.10.1.4/32","tp_dst":1470,"tp_src":25,"nw_proto":6,"dl_type":2048,"nw_src":"74.53.140.153/32"},"actions":[],"cookie":4398046511146,"flags":0,"dpid":42} Flow_mod_success Flow_mod_success http://127.0.0.1:8080/stats/flowentry/add -{"priority": 5, "idle_timeout": 30, "hard_timeout": 0, "match": {"nw_dst": "192.168.133.102/32", "tp_dst": 25, "tp_src": 49648, "nw_proto": 6, "dl_type": 2048, "nw_src": "192.168.133.100/32"}, "actions": [], "cookie": 4398046511146, "flags": 0, "dpid": 42} +{"priority":5,"idle_timeout":30,"hard_timeout":0,"match":{"nw_dst":"192.168.133.102/32","tp_dst":25,"tp_src":49648,"nw_proto":6,"dl_type":2048,"nw_src":"192.168.133.100/32"},"actions":[],"cookie":4398046511146,"flags":0,"dpid":42} http://127.0.0.1:8080/stats/flowentry/add -{"priority": 5, "idle_timeout": 30, "hard_timeout": 0, "match": {"nw_dst": "192.168.133.100/32", "tp_dst": 49648, "tp_src": 25, "nw_proto": 6, "dl_type": 2048, "nw_src": "192.168.133.102/32"}, "actions": [], "cookie": 4398046511146, "flags": 0, "dpid": 42} +{"priority":5,"idle_timeout":30,"hard_timeout":0,"match":{"nw_dst":"192.168.133.100/32","tp_dst":49648,"tp_src":25,"nw_proto":6,"dl_type":2048,"nw_src":"192.168.133.102/32"},"actions":[],"cookie":4398046511146,"flags":0,"dpid":42} Flow_mod_success Flow_mod_success http://127.0.0.1:8080/stats/flowentry/add -{"priority": 5, "idle_timeout": 30, "hard_timeout": 0, "match": {"nw_dst": "17.167.150.73/32", "tp_dst": 443, "tp_src": 49655, "nw_proto": 6, "dl_type": 2048, "nw_src": "192.168.133.100/32"}, "actions": [], "cookie": 4398046511146, "flags": 0, "dpid": 42} +{"priority":5,"idle_timeout":30,"hard_timeout":0,"match":{"nw_dst":"17.167.150.73/32","tp_dst":443,"tp_src":49655,"nw_proto":6,"dl_type":2048,"nw_src":"192.168.133.100/32"},"actions":[],"cookie":4398046511146,"flags":0,"dpid":42} http://127.0.0.1:8080/stats/flowentry/add -{"priority": 5, "idle_timeout": 30, "hard_timeout": 0, "match": {"nw_dst": "192.168.133.100/32", "tp_dst": 49655, "tp_src": 443, "nw_proto": 6, "dl_type": 2048, "nw_src": "17.167.150.73/32"}, "actions": [], "cookie": 4398046511146, "flags": 0, "dpid": 42} +{"priority":5,"idle_timeout":30,"hard_timeout":0,"match":{"nw_dst":"192.168.133.100/32","tp_dst":49655,"tp_src":443,"nw_proto":6,"dl_type":2048,"nw_src":"17.167.150.73/32"},"actions":[],"cookie":4398046511146,"flags":0,"dpid":42} Flow_mod_success Flow_mod_success diff --git a/testing/btest/Baseline/scripts.base.utils.json/output b/testing/btest/Baseline/scripts.base.utils.json/output index 43513e43f9..2d2e56253f 100644 --- a/testing/btest/Baseline/scripts.base.utils.json/output +++ b/testing/btest/Baseline/scripts.base.utils.json/output @@ -8,32 +8,33 @@ true "-12.0 hrs" "hello" "" -65535 -1 -123 -0 +{"port":65535,"proto":"tcp"} +{"port":1,"proto":"udp"} +{"port":123,"proto":"icmp"} +{"port":0,"proto":"unknown"} "1.2.3.4" "ffff:1234::1" "123.123.123.123" "192.0.0.0/8" "fe80::/64" "Red" -{"s": "test", "c": 100} -{"s": "test"} -{"s": "test"} -{"m": {"s": "test"}} +"/^?(^abcd)$?/" +{"s":"test","c":100} +{"s":"test"} +{"s":"test"} +{"m":{"s":"test"}} [] -[2, 1] +[2,1] ["1.2.3.4"] -[[true, false]] -[{"s": "test"}] +[[true,false]] +[{"s":"test"}] [] -[2, 1] +[2,1] ["1.2.3.4"] -[{"s": "test"}] -[{"s": "test"}] +[{"s":"test"}] +[{"s":"test"}] {} -{"2": "10.2.2.2", "1": "10.1.1.1"} -{"10.1.1.1": {"a": 1}, "10.2.2.2": {"b": 2}} -{"10.1.1.1": [1, 2], "10.2.2.2": [3, 5]} -{"1": {"s": "test"}} +{"2":"10.2.2.2","1":"10.1.1.1"} +{"10.1.1.1":{"a":1},"10.2.2.2":{"b":2}} +{"10.1.1.1":[1,2],"10.2.2.2":[3,5]} +{"1":{"s":"test"}} diff --git a/testing/btest/scripts/base/frameworks/logging/ascii-json-utf8.zeek b/testing/btest/scripts/base/frameworks/logging/ascii-json-utf8.zeek new file mode 100644 index 0000000000..dabe86073d --- /dev/null +++ b/testing/btest/scripts/base/frameworks/logging/ascii-json-utf8.zeek @@ -0,0 +1,59 @@ +# +# @TEST-EXEC: zeek -b %INPUT +# @TEST-EXEC: btest-diff ssh.log +# +# Testing all possible types. + +redef LogAscii::use_json = T; + + +module SSH; + +export { + redef enum Log::ID += { LOG }; + + type Log: record { + s: string; + } &log; +} + +event zeek_init() +{ + Log::create_stream(SSH::LOG, [$columns=Log]); + + # Strings taken from https://stackoverflow.com/a/3886015 + + # Valid ASCII and valid ASCII control characters + Log::write(SSH::LOG, [$s="a"]); + Log::write(SSH::LOG, [$s="\b\f\n\r\t\x00\x15"]); + + # Valid 2 Octet Sequence + Log::write(SSH::LOG, [$s="\xc3\xb1"]); + + # Invalid 2 Octet Sequence + Log::write(SSH::LOG, [$s="\xc3\x28"]); + + # Invalid Sequence Identifier + Log::write(SSH::LOG, [$s="\xa0\xa1"]); + + # Valid 3 Octet Sequence + Log::write(SSH::LOG, [$s="\xe2\x82\xa1"]); + + # Invalid 3 Octet Sequence (in 2nd Octet) + Log::write(SSH::LOG, [$s="\xe2\x28\xa1"]); + + # Invalid 3 Octet Sequence (in 3rd Octet) + Log::write(SSH::LOG, [$s="\xe2\x82\x28"]); + + # Valid 4 Octet Sequence + Log::write(SSH::LOG, [$s="\xf0\x90\x8c\xbc"]); + + # Invalid 4 Octet Sequence (in 2nd Octet) + Log::write(SSH::LOG, [$s="\xf0\x28\x8c\xbc"]); + + # Invalid 4 Octet Sequence (in 3rd Octet) + Log::write(SSH::LOG, [$s="\xf0\x90\x28\xbc"]); + + # Invalid 4 Octet Sequence (in 4th Octet) + Log::write(SSH::LOG, [$s="\xf0\x28\x8c\x28"]); +} diff --git a/testing/btest/scripts/base/utils/json.test b/testing/btest/scripts/base/utils/json.test index 8d34ed98b1..6e7854b744 100644 --- a/testing/btest/scripts/base/utils/json.test +++ b/testing/btest/scripts/base/utils/json.test @@ -72,6 +72,9 @@ event zeek_init() local e: color = Red; print to_json(e); + local p: pattern = /^abcd/; + print to_json(p); + # ######################### # Test the container types: