Merge remote-tracking branch 'origin/topic/timw/150-to-json'

* origin/topic/timw/150-to-json:
  Update submodules for JSON work
  Update unit tests for JSON logger to match new output
  Modify JSON log writer to use the external JSON library
  Update unit test output to match json.zeek being deprecated and slight format changes to JSON output
  Add proper JSON serialization via C++, deprecate json.zeek
  Add new method for escaping UTF8 strings for JSON output
  Move do_sub method from zeek.bif to StringVal class method
  Move record_fields method from zeek.bif to Val class method
  Add ToStdString method for StringVal

CHANGES

@@ -1,4 +1,11 @@
 
+2.6-586 | 2019-07-11 11:15:40 -0700
+
+  * Convert all JSON output to use an external library for better consistency (Tim Wojtulewicz, Corelight)
+
+    See NEWS for more details; this makes to_json a bif and causes slight changes in its
+    output, as well as the output of the JSON logger.
+
 2.6-576 | 2019-07-10 18:38:54 -0700
 
   * Remove unused option: chunked_io_buffer_soft_cap (Jon Siwek, Corelight)

NEWS

@@ -347,6 +347,16 @@ Changed Functionality
   of each other on separate cluster nodes to all be logged rather
   than suppressed and de-duplicated into a single notice.
 
+
+- to_json is now a bif, no longer a script. Loading base/utils/json.zeek is no
+  longer necessary and has been deprecated. to_json should yield much better, always
+  valid json. There are some small differences in output; unnecessary spaces are removed
+  and port values are rendered differently, now including the port and the protocol.
+
+- The output of the JSON logger now uses an external library to generate json. There
+  are small changes to the output; most visibly double numbers are now rounded slightly
+  differently. The way in which port values are rendered does _not_ change for JSON logs.
+
 Removed Functionality
 ---------------------
 

VERSION

@@ -1 +1 @@
-2.6-576
+2.6-586

doc

@@ -1 +1 @@
-Subproject commit fb4942d261ebd2c8160a0862b527f3639ebf1245
+Subproject commit bcfe6ffc88e0a89e7ade664113d458bae9e5e5fc

scripts/base/frameworks/openflow/plugins/ryu.zeek

@@ -3,7 +3,6 @@
 @load base/frameworks/openflow
 @load base/utils/active-http
 @load base/utils/exec
-@load base/utils/json
 
 module OpenFlow;
 

scripts/base/utils/json.zeek

@@ -1,109 +1,2 @@
-##! Functions to assist with generating JSON data from Zeek data scructures.
+## This file is deprecated in favor of to_json in zeek.bif
-# We might want to implement this in core somtime, this looks... hacky at best.
+@deprecated="Remove in 3.1.  to_json is now always available as a built-in function."
-
-@load base/utils/strings
-
-## A function to convert arbitrary Zeek data into a JSON string.
-##
-## v: The value to convert to JSON.  Typically a record.
-##
-## only_loggable: If the v value is a record this will only cause
-##                fields with the &log attribute to be included in the JSON.
-##
-## returns: a JSON formatted string.
-function to_json(v: any, only_loggable: bool &default=F, field_escape_pattern: pattern &default=/^_/): string
-	{
-	local tn = type_name(v);
-	switch ( tn )
-		{
-		case "type":
-		return "";
-
-		case "string":
-		return cat("\"", gsub(gsub(clean(v), /\\/, "\\\\"), /\"/, "\\\""), "\"");
-
-		case "port":
-		return cat(port_to_count(to_port(cat(v))));
-
-		case "enum":
-		fallthrough;
-		case "interval":
-		fallthrough;
-		case "addr":
-		fallthrough;
-		case "subnet":
-		return cat("\"", v, "\"");
-
-		case "int":
-		fallthrough;
-		case "count":
-		fallthrough;
-		case "time":
-		return cat(v);
-
-		case "double":
-		return fmt("%.16g", v);
-
-		case "bool":
-		local bval: bool = v;
-		return bval ? "true" : "false";
-
-		default:
-		break;
-		}
-
-	if ( /^record/ in tn )
-		{
-		local rec_parts: string_vec = vector();
-
-		local ft = record_fields(v);
-		for ( field, field_desc in ft )
-			{
-			# replace the escape pattern in the field.
-			if( field_escape_pattern in field )
-				field = cat(sub(field, field_escape_pattern, ""));
-			if ( field_desc?$value && (!only_loggable || field_desc$log) )
-				{
-				local onepart = cat("\"", field, "\": ", to_json(field_desc$value, only_loggable));
-				rec_parts += onepart;
-				}
-			}
-			return cat("{", join_string_vec(rec_parts, ", "), "}");
-		}
-
-	# None of the following are supported.
-	else if ( /^set/ in tn )
-		{
-		local set_parts: string_vec = vector();
-		local sa: set[bool] = v;
-		for ( sv in sa )
-			{
-			set_parts += to_json(sv, only_loggable);
-			}
-		return cat("[", join_string_vec(set_parts, ", "), "]");
-		}
-	else if ( /^table/ in tn )
-		{
-		local tab_parts: vector of string = vector();
-		local ta: table[bool] of any = v;
-		for ( ti, tv in ta )
-			{
-			local ts = to_json(ti);
-			local if_quotes = (ts[0] == "\"") ? "" : "\"";
-			tab_parts += cat(if_quotes, ts, if_quotes, ": ", to_json(tv, only_loggable));
-			}
-		return cat("{", join_string_vec(tab_parts, ", "), "}");
-		}
-	else if ( /^vector/ in tn )
-		{
-		local vec_parts: string_vec = vector();
-		local va: vector of any = v;
-		for ( vi in va )
-			{
-			vec_parts += to_json(va[vi], only_loggable);
-			}
-		return cat("[", join_string_vec(vec_parts, ", "), "]");
-		}
-
-	return "\"\"";
-	}

src/3rdparty

@@ -1 +1 @@
-Subproject commit 785e581f00a1efae3fca7a62fb15d8756c5aedb1
+Subproject commit 1e9d49362d2c3bb2f43abcd8eebe47be045659a5

src/Val.cc

@@ -27,6 +27,16 @@
 
 #include "broker/Data.h"
 
+#include "3rdparty/json.hpp"
+#include "3rdparty/fifo_map.hpp"
+
+// Define a class for use with the json library that orders the keys in the same order that
+// they were inserted. By default, the json library orders them alphabetically and we don't
+// want it like that.
+template<class K, class V, class compare, class A>
+using json_fifo_map = nlohmann::fifo_map<K, V, nlohmann::fifo_map_compare<K>, A>;
+using ZeekJson = nlohmann::basic_json<json_fifo_map>;
+
 Val::Val(Func* f)
 	{
 	val.func_val = f;
@@ -380,6 +390,274 @@ bool Val::WouldOverflow(const BroType* from_type, const BroType* to_type, const
 	return false;
 	}
 
+TableVal* Val::GetRecordFields()
+	{
+	TableVal* fields = new TableVal(internal_type("record_field_table")->AsTableType());
+
+	auto t = Type();
+
+	if ( t->Tag() != TYPE_RECORD && t->Tag() != TYPE_TYPE )
+		{
+		reporter->Error("non-record value/type passed to record_fields");
+		return fields;
+		}
+
+	RecordType* rt = nullptr;
+	RecordVal* rv = nullptr;
+
+	if ( t->Tag() == TYPE_RECORD )
+		{
+		rt = t->AsRecordType();
+		rv = AsRecordVal();
+		}
+	else
+		{
+		t = t->AsTypeType()->Type();
+
+		if ( t->Tag() != TYPE_RECORD )
+			{
+			reporter->Error("non-record value/type passed to record_fields");
+			return fields;
+			}
+
+		rt = t->AsRecordType();
+		}
+
+	for ( int i = 0; i < rt->NumFields(); ++i )
+		{
+		BroType* ft = rt->FieldType(i);
+		TypeDecl* fd = rt->FieldDecl(i);
+		Val* fv = nullptr;
+
+		if ( rv )
+			fv = rv->Lookup(i);
+
+		if ( fv )
+			::Ref(fv);
+
+		bool logged = (fd->attrs && fd->FindAttr(ATTR_LOG) != 0);
+
+		RecordVal* nr = new RecordVal(internal_type("record_field")->AsRecordType());
+
+		if ( ft->Tag() == TYPE_RECORD )
+			nr->Assign(0, new StringVal("record " + ft->GetName()));
+		else
+			nr->Assign(0, new StringVal(type_name(ft->Tag())));
+
+		nr->Assign(1, val_mgr->GetBool(logged));
+		nr->Assign(2, fv);
+		nr->Assign(3, rt->FieldDefault(i));
+
+		Val* field_name = new StringVal(rt->FieldName(i));
+		fields->Assign(field_name, nr);
+		Unref(field_name);
+		}
+
+	return fields;
+	}
+
+// This is a static method in this file to avoid including json.hpp in Val.h since it's huge.
+static ZeekJson BuildJSON(Val* val, bool only_loggable=false, RE_Matcher* re=new RE_Matcher("^_"))
+	{
+	ZeekJson j;
+	BroType* type = val->Type();
+	switch ( type->Tag() )
+		{
+		case TYPE_BOOL:
+			j = val->AsBool();
+			break;
+
+		case TYPE_INT:
+			j = val->AsInt();
+			break;
+
+		case TYPE_COUNT:
+			j = val->AsCount();
+			break;
+
+		case TYPE_COUNTER:
+			j = val->AsCounter();
+			break;
+
+		case TYPE_TIME:
+			j = val->AsTime();
+			break;
+
+		case TYPE_DOUBLE:
+			j = val->AsDouble();
+			break;
+
+		case TYPE_PORT:
+			{
+			auto* pval = val->AsPortVal();
+			j["port"] = pval->Port();
+			j["proto"] = pval->Protocol();
+			break;
+			}
+
+		case TYPE_PATTERN:
+		case TYPE_INTERVAL:
+		case TYPE_ADDR:
+		case TYPE_SUBNET:
+			{
+			ODesc d;
+			d.SetStyle(RAW_STYLE);
+			val->Describe(&d);
+
+			auto* bs = new BroString(1, d.TakeBytes(), d.Len());
+			j = string((char*)bs->Bytes(), bs->Len());
+
+			delete bs;
+			break;
+			}
+
+		case TYPE_FILE:
+		case TYPE_FUNC:
+		case TYPE_ENUM:
+		case TYPE_STRING:
+			{
+			ODesc d;
+			d.SetStyle(RAW_STYLE);
+			val->Describe(&d);
+
+			auto* bs = new BroString(1, d.TakeBytes(), d.Len());
+			j = json_escape_utf8(string((char*)bs->Bytes(), bs->Len()));
+
+			delete bs;
+			break;
+			}
+
+		case TYPE_TABLE:
+			{
+			auto* table = val->AsTable();
+			auto* tval = val->AsTableVal();
+
+			if ( tval->Type()->IsSet() )
+				j = ZeekJson::array();
+			else
+				j = ZeekJson::object();
+
+			HashKey* k;
+			auto c = table->InitForIteration();
+			while ( table->NextEntry(k, c) )
+				{
+				auto lv = tval->RecoverIndex(k);
+				delete k;
+
+				if ( tval->Type()->IsSet() )
+					{
+					auto* value = lv->Index(0)->Ref();
+					j.push_back(BuildJSON(value, only_loggable, re));
+					Unref(value);
+					}
+				else
+					{
+					ZeekJson key_json;
+					Val* entry_value;
+					if ( lv->Length() == 1 )
+						{
+						Val* entry_key = lv->Index(0)->Ref();
+						entry_value = tval->Lookup(entry_key, true);
+						key_json = BuildJSON(entry_key, only_loggable, re);
+						Unref(entry_key);
+						}
+					else
+						{
+						entry_value = tval->Lookup(lv, true);
+						key_json = BuildJSON(lv, only_loggable, re);
+						}
+
+					string key_string;
+					if ( key_json.is_string() )
+						key_string = key_json;
+					else
+						key_string = key_json.dump();
+
+					j[key_string] = BuildJSON(entry_value, only_loggable, re);
+					}
+
+				Unref(lv);
+				}
+
+			break;
+			}
+
+		case TYPE_RECORD:
+			{
+			j = ZeekJson::object();
+			auto* rval = val->AsRecordVal();
+			TableVal* fields = rval->GetRecordFields();
+			auto* field_indexes = fields->ConvertToPureList();
+			int num_indexes = field_indexes->Length();
+
+			for ( int i = 0; i < num_indexes; ++i )
+				{
+				Val* key = field_indexes->Index(i);
+				auto* key_field = fields->Lookup(key)->AsRecordVal();
+
+				auto* key_val = key->AsStringVal();
+				string key_string;
+				if ( re->MatchAnywhere(key_val->AsString()) != 0 )
+					{
+					key_val = key_val->Substitute(re, new StringVal(""), 0)->AsStringVal();
+					key_string = key_val->ToStdString();
+					delete key_val;
+					}
+				else
+					key_string = key_val->ToStdString();
+
+				Val* value = key_field->Lookup("value", true);
+
+				if ( value && ( ! only_loggable || key_field->Lookup("log")->AsBool() ) )
+					j[key_string] = BuildJSON(value, only_loggable, re);
+				}
+
+			delete fields;
+			break;
+			}
+
+		case TYPE_LIST:
+			{
+			j = ZeekJson::array();
+			auto* lval = val->AsListVal();
+			size_t size = lval->Length();
+			for (size_t i = 0; i < size; i++)
+				j.push_back(BuildJSON(lval->Index(i), only_loggable, re));
+
+			break;
+			}
+
+		case TYPE_VECTOR:
+			{
+			j = ZeekJson::array();
+			auto* vval = val->AsVectorVal();
+			size_t size = vval->SizeVal()->AsCount();
+			for (size_t i = 0; i < size; i++)
+				j.push_back(BuildJSON(vval->Lookup(i), only_loggable, re));
+
+			break;
+			}
+
+		case TYPE_OPAQUE:
+			{
+			j = ZeekJson::object();
+			auto* oval = val->AsOpaqueVal();
+			j["opaque_type"] = OpaqueMgr::mgr()->TypeID(oval);
+			break;
+			}
+
+		default: break;
+		}
+
+	return j;
+	}
+
+StringVal* Val::ToJSON(bool only_loggable, RE_Matcher* re)
+	{
+	ZeekJson j = BuildJSON(this, only_loggable, re);
+	return new StringVal(j.dump());
+	}
+
 IntervalVal::IntervalVal(double quantity, double units) :
 	Val(quantity * units, TYPE_INTERVAL)
 	{
@@ -491,6 +769,18 @@ uint32 PortVal::Port() const
 	return p & ~PORT_SPACE_MASK;
 	}
 
+string PortVal::Protocol() const
+	{
+	if ( IsUDP() )
+		return "udp";
+	else if ( IsTCP() )
+		return "tcp";
+	else if ( IsICMP() )
+		return "icmp";
+	else
+		return "unknown";
+	}
+
 int PortVal::IsTCP() const
 	{
 	return (val.uint_val & PORT_SPACE_MASK) == TCP_PORT_MASK;
@@ -510,14 +800,8 @@ void PortVal::ValDescribe(ODesc* d) const
 	{
 	uint32 p = static_cast<uint32>(val.uint_val);
 	d->Add(p & ~PORT_SPACE_MASK);
-	if ( IsUDP() )
+	d->Add("/");
-		d->Add("/udp");
+	d->Add(Protocol());
-	else if ( IsTCP() )
-		d->Add("/tcp");
-	else if ( IsICMP() )
-		d->Add("/icmp");
-	else
-		d->Add("/unknown");
 	}
 
 Val* PortVal::DoClone(CloneState* state)
@@ -713,6 +997,12 @@ StringVal::StringVal(const string& s) : Val(TYPE_STRING)
 	val.string_val = new BroString(reinterpret_cast<const u_char*>(s.data()), s.length(), 1);
 	}
 
+string StringVal::ToStdString() const
+	{
+	auto* bs = AsString();
+	return string((char*)bs->Bytes(), bs->Len());
+	}
+
 StringVal* StringVal::ToUpper()
 	{
 	val.string_val->ToUpper();
@@ -734,6 +1024,92 @@ unsigned int StringVal::MemoryAllocation() const
 	return padded_sizeof(*this) + val.string_val->MemoryAllocation();
 	}
 
+Val* StringVal::Substitute(RE_Matcher* re, StringVal* repl, bool do_all)
+	{
+	const u_char* s = Bytes();
+	int offset = 0;
+	int n = Len();
+
+	// cut_points is a set of pairs of indices in str that should
+	// be removed/replaced.  A pair <x,y> means "delete starting
+	// at offset x, up to but not including offset y".
+	List(ptr_compat_int) cut_points;	// where RE matches pieces of str
+
+	int size = 0;	// size of result
+
+	while ( n > 0 )
+		{
+		// Find next match offset.
+		int end_of_match;
+		while ( n > 0 &&
+		        (end_of_match = re->MatchPrefix(&s[offset], n)) <= 0 )
+			{
+			// This character is going to be copied to the result.
+			++size;
+
+			// Move on to next character.
+			++offset;
+			--n;
+			}
+
+		if ( n <= 0 )
+			break;
+
+		// s[offset .. offset+end_of_match-1] matches re.
+		cut_points.append(offset);
+		cut_points.append(offset + end_of_match);
+
+		offset += end_of_match;
+		n -= end_of_match;
+
+		if ( ! do_all )
+			{
+			// We've now done the first substitution - finished.
+			// Include the remainder of the string in the result.
+			size += n;
+			break;
+			}
+		}
+
+	// size now reflects amount of space copied.  Factor in amount
+	// of space for replacement text.
+	int num_cut_points = cut_points.length() / 2;
+	size += num_cut_points * repl->Len();
+
+	// And a final NUL for good health.
+	++size;
+
+	byte_vec result = new u_char[size];
+	byte_vec r = result;
+
+	// Copy it all over.
+	int start_offset = 0;
+	for ( int i = 0; i < cut_points.length(); i += 2 /* loop over pairs */ )
+		{
+		int num_to_copy = cut_points[i] - start_offset;
+		memcpy(r, s + start_offset, num_to_copy);
+
+		r += num_to_copy;
+		start_offset = cut_points[i+1];
+
+		// Now add in replacement text.
+		memcpy(r, repl->Bytes(), repl->Len());
+		r += repl->Len();
+		}
+
+	// Copy final trailing characters.
+	int num_to_copy = Len() - start_offset;
+	memcpy(r, s + start_offset, num_to_copy);
+	r += num_to_copy;
+
+	// Final NUL.  No need to increment r, since the length
+	// computed from it in the next statement does not include
+	// the NUL.
+	r[0] = '\0';
+
+	return new StringVal(new BroString(1, result, r - result));
+	}
+
 Val* StringVal::DoClone(CloneState* state)
 	{
 	// We could likely treat this type as immutable and return a reference

src/Val.h

@@ -20,6 +20,7 @@
 #include "Notifier.h"
 #include "IPAddr.h"
 #include "DebugLogger.h"
+#include "RE.h"
 
 // We have four different port name spaces: TCP, UDP, ICMP, and UNKNOWN.
 // We distinguish between them based on the bits specified in the *_PORT_MASK
@@ -34,7 +35,6 @@
 class Val;
 class Func;
 class BroFile;
-class RE_Matcher;
 class PrefixTable;
 
 class PortVal;
@@ -347,6 +347,10 @@ public:
 
 	static bool WouldOverflow(const BroType* from_type, const BroType* to_type, const Val* val);
 
+	TableVal* GetRecordFields();
+
+	StringVal* ToJSON(bool only_loggable=false, RE_Matcher* re=new RE_Matcher("^_"));
+
 protected:
 
 	friend class EnumType;
@@ -530,6 +534,7 @@ public:
 
 	// Returns the port number in host order (not including the mask).
 	uint32 Port() const;
+	string Protocol() const;
 
 	// Tests for protocol types.
 	int IsTCP() const;
@@ -632,10 +637,13 @@ public:
 	// char* ExpandedString(int format = BroString::EXPANDED_STRING)
 	// 	{ return AsString()->ExpandedString(format); }
 
+	std::string ToStdString() const;
 	StringVal* ToUpper();
 
 	unsigned int MemoryAllocation() const override;
 
+	Val* Substitute(RE_Matcher* re, StringVal* repl, bool do_all);
+
 protected:
 	friend class Val;
 	StringVal()	{}

src/strings.bif

@@ -351,91 +351,6 @@ Val* do_split(StringVal* str_val, RE_Matcher* re, int incl_sep, int max_num_sep)
 	return a;
 	}
 
-Val* do_sub(StringVal* str_val, RE_Matcher* re, StringVal* repl, int do_all)
-	{
-	const u_char* s = str_val->Bytes();
-	int offset = 0;
-	int n = str_val->Len();
-
-	// cut_points is a set of pairs of indices in str that should
-	// be removed/replaced.  A pair <x,y> means "delete starting
-	// at offset x, up to but not including offset y".
-	List(ptr_compat_int) cut_points;	// where RE matches pieces of str
-
-	int size = 0;	// size of result
-
-	while ( n > 0 )
-		{
-		// Find next match offset.
-		int end_of_match;
-		while ( n > 0 &&
-			(end_of_match = re->MatchPrefix(&s[offset], n)) <= 0 )
-			{
-			// This character is going to be copied to the result.
-			++size;
-
-			// Move on to next character.
-			++offset;
-			--n;
-			}
-
-		if ( n <= 0 )
-			break;
-
-		// s[offset .. offset+end_of_match-1] matches re.
-		cut_points.append(offset);
-		cut_points.append(offset + end_of_match);
-
-		offset += end_of_match;
-		n -= end_of_match;
-
-		if ( ! do_all )
-			{
-			// We've now done the first substitution - finished.
-			// Include the remainder of the string in the result.
-			size += n;
-			break;
-			}
-		}
-
-	// size now reflects amount of space copied.  Factor in amount
-	// of space for replacement text.
-	int num_cut_points = cut_points.length() / 2;
-	size += num_cut_points * repl->Len();
-
-	// And a final NUL for good health.
-	++size;
-
-	byte_vec result = new u_char[size];
-	byte_vec r = result;
-
-	// Copy it all over.
-	int start_offset = 0;
-	for ( int i = 0; i < cut_points.length(); i += 2 /* loop over pairs */ )
-		{
-		int num_to_copy = cut_points[i] - start_offset;
-		memcpy(r, s + start_offset, num_to_copy);
-
-		r += num_to_copy;
-		start_offset = cut_points[i+1];
-
-		// Now add in replacement text.
-		memcpy(r, repl->Bytes(), repl->Len());
-		r += repl->Len();
-		}
-
-	// Copy final trailing characters.
-	int num_to_copy = str_val->Len() - start_offset;
-	memcpy(r, s + start_offset, num_to_copy);
-	r += num_to_copy;
-
-	// Final NUL.  No need to increment r, since the length
-	// computed from it in the next statement does not include
-	// the NUL.
-	r[0] = '\0';
-
-	return new StringVal(new BroString(1, result, r - result));
-	}
 %%}
 
 ## Splits a string into an array of strings according to a pattern.
@@ -535,7 +450,7 @@ function split_string_n%(str: string, re: pattern,
 ## .. zeek:see:: gsub subst_string
 function sub%(str: string, re: pattern, repl: string%): string
 	%{
-	return do_sub(str, re, repl, 0);
+	return str->Substitute(re, repl, false);
 	%}
 
 ## Substitutes a given replacement string for all occurrences of a pattern
@@ -552,7 +467,7 @@ function sub%(str: string, re: pattern, repl: string%): string
 ## .. zeek:see:: sub subst_string
 function gsub%(str: string, re: pattern, repl: string%): string
 	%{
-	return do_sub(str, re, repl, 1);
+	return str->Substitute(re, repl, true);
 	%}
 
 

src/threading/Formatter.cc

@@ -111,3 +111,14 @@ string Formatter::Render(double d)
 	return buf;
 	}
 
+string Formatter::Render(TransportProto proto)
+	{
+	if ( proto == TRANSPORT_UDP )
+		return "udp";
+	else if ( proto == TRANSPORT_TCP )
+		return "tcp";
+	else if ( proto == TRANSPORT_ICMP )
+		return "icmp";
+	else
+		return "unknown";
+	}

src/threading/Formatter.h

@@ -112,6 +112,17 @@ public:
 	 */
 	static string Render(double d);
 
+	/**
+	 * Convert a transport protocol into a string.
+	 *
+	 * This is a helper function that formatter implementations may use.
+	 *
+	 * @param proto The transport protocol.
+	 *
+	 * @return An ASCII representation of the protocol.
+	 */
+	static string Render(TransportProto proto);
+
 	/**
 	 * Convert a string into a TransportProto. The string must be one of
 	 * \c tcp, \c udp, \c icmp, or \c unknown.

src/threading/formatters/JSON.cc

@@ -11,7 +11,7 @@
 #include <math.h>
 #include <stdint.h>
 
-#include "./JSON.h"
+#include "JSON.h"
 
 using namespace threading::formatter;
 
@@ -27,78 +27,83 @@ JSON::~JSON()
 bool JSON::Describe(ODesc* desc, int num_fields, const Field* const * fields,
                     Value** vals) const
 	{
-	if ( surrounding_braces )
+	ZeekJson j = ZeekJson::object();
-		desc->AddRaw("{");
 
 	for ( int i = 0; i < num_fields; i++ )
 		{
-		const u_char* bytes = desc->Bytes();
+		if ( vals[i]->present )
-		int len = desc->Len();
+			{
+			ZeekJson new_entry = BuildJSON(vals[i]);
+			if ( new_entry.is_null() )
+				return false;
 
-		if ( i > 0 &&
+			j[fields[i]->name] = new_entry;
-		     len > 0 &&
+			}
-		     bytes[len-1] != ',' &&
-		     bytes[len-1] != '{' &&
-		     bytes[len-1] != '[' &&
-		     vals[i]->present )
-			desc->AddRaw(",");
-
-		if ( ! Describe(desc, vals[i], fields[i]->name) )
-			return false;
 		}
 
-	if ( surrounding_braces )
+	desc->Add(j.dump());
-		desc->AddRaw("}");
 
 	return true;
 	}
 
 bool JSON::Describe(ODesc* desc, Value* val, const string& name) const
 	{
+	if ( desc->IsBinary() )
+		{
+		GetThread()->Error("json formatter: binary format not supported");
+		return false;
+		}
+
 	if ( ! val->present )
 		return true;
 
-	if ( name.size() )
+	ZeekJson j = BuildJSON(val, name);
-		{
+	if ( j.is_null() )
-		desc->AddRaw("\"", 1);
+		return false;
-		desc->Add(name);
-		desc->AddRaw("\":", 2);
-		}
 
+	desc->Add(j.dump());
+	return true;
+	}
+
+threading::Value* JSON::ParseValue(const string& s, const string& name, TypeTag type, TypeTag subtype) const
+	{
+	GetThread()->Error("JSON formatter does not support parsing yet.");
+	return nullptr;
+	}
+
+ZeekJson JSON::BuildJSON(Value* val, const string& name) const
+	{
+	ZeekJson j;
 	switch ( val->type )
 		{
 		case TYPE_BOOL:
-			desc->AddRaw(val->val.int_val == 0 ? "false" : "true");
+			j = val->val.int_val != 0;
 			break;
 
 		case TYPE_INT:
-			desc->Add(val->val.int_val);
+			j = val->val.int_val;
 			break;
 
 		case TYPE_COUNT:
 		case TYPE_COUNTER:
-			desc->Add(val->val.uint_val);
+			j = val->val.uint_val;
 			break;
 
 		case TYPE_PORT:
-			desc->Add(val->val.port_val.port);
+			j = val->val.port_val.port;
 			break;
 
 		case TYPE_SUBNET:
-			desc->AddRaw("\"", 1);
+			j = Formatter::Render(val->val.subnet_val);
-			desc->Add(Render(val->val.subnet_val));
-			desc->AddRaw("\"", 1);
 			break;
 
 		case TYPE_ADDR:
-			desc->AddRaw("\"", 1);
+			j = Formatter::Render(val->val.addr_val);
-			desc->Add(Render(val->val.addr_val));
-			desc->AddRaw("\"", 1);
 			break;
 
 		case TYPE_DOUBLE:
 		case TYPE_INTERVAL:
-			desc->Add(val->val.double_val);
+			j = val->val.double_val;
 			break;
 
 		case TYPE_TIME:
@@ -110,15 +115,13 @@ bool JSON::Describe(ODesc* desc, Value* val, const string& name) const
 				time_t the_time = time_t(floor(val->val.double_val));
 				struct tm t;
 
-				desc->AddRaw("\"", 1);
-
 				if ( ! gmtime_r(&the_time, &t) ||
 				     ! strftime(buffer, sizeof(buffer), "%Y-%m-%dT%H:%M:%S", &t) )
 					{
 					GetThread()->Error(GetThread()->Fmt("json formatter: failure getting time: (%lf)", val->val.double_val));
 					// This was a failure, doesn't really matter what gets put here
 					// but it should probably stand out...
-					desc->Add("2000-01-01T00:00:00.000000");
+					j = "2000-01-01T00:00:00.000000";
 					}
 				else
 					{
@@ -129,20 +132,17 @@ bool JSON::Describe(ODesc* desc, Value* val, const string& name) const
 						frac += 1;
 
 					snprintf(buffer2, sizeof(buffer2), "%s.%06.0fZ", buffer, fabs(frac) * 1000000);
-					desc->Add(buffer2);
+					j = buffer2;
 					}
-
-				desc->AddRaw("\"", 1);
 				}
 
 			else if ( timestamps == TS_EPOCH )
-				desc->Add(val->val.double_val);
+				j = val->val.double_val;
 
 			else if ( timestamps == TS_MILLIS )
 				{
 				// ElasticSearch uses milliseconds for timestamps
-				uint64_t ts = (uint64_t) (val->val.double_val * 1000);
+				j = (uint64_t) (val->val.double_val * 1000);
-				desc->Add(ts);
 				}
 
 			break;
@@ -153,74 +153,40 @@ bool JSON::Describe(ODesc* desc, Value* val, const string& name) const
 		case TYPE_FILE:
 		case TYPE_FUNC:
 			{
-			desc->AddRaw("\"", 1);
+			j = json_escape_utf8(string(val->val.string_val.data, val->val.string_val.length));
-
-			for ( int i = 0; i < val->val.string_val.length; ++i )
-				{
-				char c = val->val.string_val.data[i];
-
-				// 2byte Unicode escape special characters.
-				if ( c < 32 || c > 126 || c == '\n' || c == '"' || c == '\'' || c == '\\' || c == '&' )
-					{
-					desc->AddRaw("\\u00", 4);
-					char hex[2] = {'0', '0'};
-					bytetohex(c, hex);
-					desc->AddRaw(hex, 1);
-					desc->AddRaw(hex + 1, 1);
-					}
-				else
-					desc->AddRaw(&c, 1);
-				}
-
-			desc->AddRaw("\"", 1);
 			break;
 			}
 
 		case TYPE_TABLE:
 			{
-			desc->AddRaw("[", 1);
+			j = ZeekJson::array();
 
-			for ( int j = 0; j < val->val.set_val.size; j++ )
+			for ( int idx = 0; idx < val->val.set_val.size; idx++ )
-				{
+				j.push_back(BuildJSON(val->val.set_val.vals[idx]));
-				if ( j > 0 )
-					desc->AddRaw(",", 1);
 
-				Describe(desc, val->val.set_val.vals[j]);
-				}
-
-			desc->AddRaw("]", 1);
 			break;
 			}
 
 		case TYPE_VECTOR:
 			{
-			desc->AddRaw("[", 1);
+			j = ZeekJson::array();
 
-			for ( int j = 0; j < val->val.vector_val.size; j++ )
+			for ( int idx = 0; idx < val->val.vector_val.size; idx++ )
-				{
+				j.push_back(BuildJSON(val->val.vector_val.vals[idx]));
-				if ( j > 0 )
-					desc->AddRaw(",", 1);
-				Describe(desc, val->val.vector_val.vals[j]);
-				}
 
-			desc->AddRaw("]", 1);
 			break;
 			}
 
 		default:
-			return false;
+			break;
 		}
 
-	return true;
+	if ( ! name.empty() && ! j.is_null() )
-	}
+		{
+		ZeekJson j2 = ZeekJson::object();
+		j2[name] = j;
+		return j2;
+		}
 
-threading::Value* JSON::ParseValue(const string& s, const string& name, TypeTag type, TypeTag subtype) const
+	return j;
-	{
-	GetThread()->Error("JSON formatter does not support parsing yet.");
-	return NULL;
-	}
-
-void JSON::SurroundingBraces(bool use_braces)
-	{
-	surrounding_braces = use_braces;
 	}

src/threading/formatters/JSON.h

@@ -4,9 +4,19 @@
 #define THREADING_FORMATTERS_JSON_H
 
 #include "../Formatter.h"
+#include "3rdparty/json.hpp"
+#include "3rdparty/fifo_map.hpp"
+
 
 namespace threading { namespace formatter {
 
+// Define a class for use with the json library that orders the keys in the same order that
+// they were inserted. By default, the json library orders them alphabetically and we don't
+// want it like that.
+template<class K, class V, class compare, class A>
+using json_fifo_map = nlohmann::fifo_map<K, V, nlohmann::fifo_map_compare<K>, A>;
+using ZeekJson = nlohmann::basic_json<json_fifo_map>;
+
 /**
   * A thread-safe class for converting values into a JSON representation
   * and vice versa.
@@ -27,9 +37,10 @@ public:
 	                      threading::Value** vals) const override;
 	threading::Value* ParseValue(const string& s, const string& name, TypeTag type, TypeTag subtype = TYPE_ERROR) const override;
 
-	void SurroundingBraces(bool use_braces);
-
 private:
+
+	ZeekJson BuildJSON(Value* val, const string& name = "") const;
+
 	TimeFormat timestamps;
 	bool surrounding_braces;
 };

src/util.cc

@@ -1872,3 +1872,90 @@ char* zeekenv(const char* name)
 
 	return getenv(it->second);
 	}
+
+static string json_escape_byte(char c)
+	{
+	char hex[2] = {'0', '0'};
+	bytetohex(c, hex);
+
+	string result = "\\x";
+	result.append(hex, 2);
+	
+	return result;
+	}
+
+string json_escape_utf8(const string& val)
+	{
+	string result;
+	result.reserve(val.length());
+	
+	size_t char_start = 0;
+	size_t idx;
+	for ( idx = 0; idx < val.length(); )
+		{
+		// Normal ASCII characters plus a few of the control characters can be inserted directly. The rest of
+		// the control characters should be escaped as regular bytes.
+		if ( ( val[idx] >= 32 && val[idx] <= 127 ) ||
+		       val[idx] == '\b' || val[idx] == '\f' || val[idx] == '\n' || val[idx] == '\r' || val[idx] == '\t' )
+			{
+			result.push_back(val[idx]);
+			++idx;
+			continue;
+			}
+		else if ( val[idx] >= 0 && val[idx] < 32 )
+			{
+			result.append(json_escape_byte(val[idx]));
+			++idx;
+			continue;
+			}
+
+		// The next bit is based on the table at https://en.wikipedia.org/wiki/UTF-8#Description.
+		// If next character is 11110xxx, this is a 4-byte UTF-8
+		int char_size = 0;
+		if ( (val[idx] & 0xF8) == 0xF0 ) char_size = 4;
+		
+		// If next character is 1110xxxx, this is a 3-byte UTF-8
+		else if ( (val[idx] & 0xF0) == 0xE0 ) char_size = 3;
+		
+		// If next character is 110xxxxx, this is a 2-byte UTF-8
+		else if ( (val[idx] & 0xE0) == 0xC0 ) char_size = 2;
+		
+		// This byte isn't a continuation byte, insert it as a byte and continue.
+		if ( char_size == 0)
+			{
+			result.append(json_escape_byte(val[idx]));
+			++idx;
+			continue;
+			}
+		
+		// If we don't have enough bytes to get to the end of character, give up and insert all of the rest
+		// of them as escaped values.
+		if ( char_size > (val.length() - idx) )
+			break;
+		
+		// Loop through the rest of the supposed character and see if this is a valid character.
+		size_t c_idx = idx + 1;
+		for ( ; c_idx < idx + char_size; c_idx++ )
+			if ( (val[c_idx] & 0xC0) != 0x80 ) break;
+		
+		// if we didn't make it to the end of the character without finding an error, insert just this
+		// character and skip ahead. Otherwise insert all of the bytes for this character into the result.
+		if ( c_idx != idx + char_size )
+			{
+			result.append(json_escape_byte(val[idx]));
+			++idx;
+			continue;
+			}
+		else
+			{
+			for ( size_t step = 0; step < char_size; step++, idx++ )
+				result.push_back(val[idx]);
+			}
+		}
+	
+	if ( idx != val.length() )
+		for ( ; idx < val.length(); ++idx )
+			result.append(json_escape_byte(val[idx]));
+	
+	return result;
+	}

src/util.h

@@ -565,4 +565,12 @@ std::unique_ptr<T> build_unique (Args&&... args) {
 	return std::unique_ptr<T>(new T(std::forward<Args>(args)...));
 }
 
+/**
+ * Escapes bytes in a string that are not valid UTF8 characters with \xYY format. Used
+ * by the JSON writer and BIF methods.
+ * @param val the input string to be escaped
+ * @return the escaped string
+ */
+std::string json_escape_utf8(const std::string& val);
+
 #endif

src/zeek.bif

@@ -1981,68 +1981,7 @@ function lookup_ID%(id: string%) : any
 ## Returns: A table that describes the fields of a record.
 function record_fields%(rec: any%): record_field_table
 	%{
-	TableVal* fields = new TableVal(record_field_table);
+	return rec->GetRecordFields();
-
-	auto t = rec->Type();
-
-	if ( t->Tag() != TYPE_RECORD && t->Tag() != TYPE_TYPE )
-		{
-		reporter->Error("non-record value/type passed to record_fields");
-		return fields;
-		}
-
-	RecordType* rt = nullptr;
-	RecordVal* rv = nullptr;
-
-	if ( t->Tag() == TYPE_RECORD )
-		{
-		rt = t->AsRecordType();
-		rv = rec->AsRecordVal();
-		}
-	else
-		{
-		t = t->AsTypeType()->Type();
-
-		if ( t->Tag() != TYPE_RECORD )
-			{
-			reporter->Error("non-record value/type passed to record_fields");
-			return fields;
-			}
-
-		rt = t->AsRecordType();
-		}
-
-	for ( int i = 0; i < rt->NumFields(); ++i )
-		{
-		BroType* ft = rt->FieldType(i);
-		TypeDecl* fd = rt->FieldDecl(i);
-		Val* fv = nullptr;
-
-		if ( rv )
-			fv = rv->Lookup(i);
-
-		if ( fv )
-			Ref(fv);
-
-		bool logged = (fd->attrs && fd->FindAttr(ATTR_LOG) != 0);
-
-		RecordVal* nr = new RecordVal(record_field);
-
-		if ( ft->Tag() == TYPE_RECORD )
-			nr->Assign(0, new StringVal("record " + ft->GetName()));
-		else
-			nr->Assign(0, new StringVal(type_name(ft->Tag())));
-
-		nr->Assign(1, val_mgr->GetBool(logged));
-		nr->Assign(2, fv);
-		nr->Assign(3, rt->FieldDefault(i));
-
-		Val* field_name = new StringVal(rt->FieldName(i));
-		fields->Assign(field_name, nr);
-		Unref(field_name);
-		}
-
-	return fields;
 	%}
 
 ## Enables detailed collection of profiling statistics. Statistics include
@@ -5100,3 +5039,16 @@ function anonymize_addr%(a: addr, cl: IPAddrAnonymizationClass%): addr
 			(enum ip_addr_anonymization_class_t) anon_class));
 		}
 	%}
+
+## A function to convert arbitrary Zeek data into a JSON string.
+##
+## v: The value to convert to JSON.  Typically a record.
+##
+## only_loggable: If the v value is a record this will only cause
+##                fields with the &log attribute to be included in the JSON.
+##
+## returns: a JSON formatted string.
+function to_json%(val: any, only_loggable: bool &default=F, field_escape_pattern: pattern &default=/^_/%): string
+	%{
+	return val->ToJSON(only_loggable, field_escape_pattern);
+	%}

testing/btest/Baseline/coverage.bare-mode-errors/errors

@@ -0,0 +1 @@
+warning in /Users/tim/Desktop/projects/zeek/testing/btest/../../scripts//base/utils/json.zeek, line 2: deprecated script loaded from command line arguments ="Remove in 3.1.  to_json is now always available as a built-in function."

testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log

@@ -249,7 +249,6 @@ scripts/base/init-default.zeek
     scripts/base/frameworks/openflow/main.zeek
     scripts/base/frameworks/openflow/plugins/__load__.zeek
       scripts/base/frameworks/openflow/plugins/ryu.zeek
-        scripts/base/utils/json.zeek
       scripts/base/frameworks/openflow/plugins/log.zeek
       scripts/base/frameworks/openflow/plugins/broker.zeek
     scripts/base/frameworks/openflow/non-cluster.zeek

testing/btest/Baseline/coverage.init-default/missing_loads

@@ -8,3 +8,4 @@
 -./frameworks/openflow/cluster.zeek
 -./frameworks/packet-filter/cluster.zeek
 -./frameworks/sumstats/cluster.zeek
+-./utils/json.zeek

testing/btest/Baseline/plugins.hooks/output

@@ -821,7 +821,6 @@
 0.000000   MetaHookPost  LoadFile(0, base<...>/input.bif.zeek) -> -1
 0.000000   MetaHookPost  LoadFile(0, base<...>/intel) -> -1
 0.000000   MetaHookPost  LoadFile(0, base<...>/irc) -> -1
-0.000000   MetaHookPost  LoadFile(0, base<...>/json.zeek) -> -1
 0.000000   MetaHookPost  LoadFile(0, base<...>/krb) -> -1
 0.000000   MetaHookPost  LoadFile(0, base<...>/logging) -> -1
 0.000000   MetaHookPost  LoadFile(0, base<...>/logging.bif.zeek) -> -1
@@ -1712,7 +1711,6 @@
 0.000000   MetaHookPre   LoadFile(0, base<...>/input.bif.zeek)
 0.000000   MetaHookPre   LoadFile(0, base<...>/intel)
 0.000000   MetaHookPre   LoadFile(0, base<...>/irc)
-0.000000   MetaHookPre   LoadFile(0, base<...>/json.zeek)
 0.000000   MetaHookPre   LoadFile(0, base<...>/krb)
 0.000000   MetaHookPre   LoadFile(0, base<...>/logging)
 0.000000   MetaHookPre   LoadFile(0, base<...>/logging.bif.zeek)
@@ -2611,7 +2609,6 @@
 0.000000 | HookLoadFile  base<...>/input.bif.zeek
 0.000000 | HookLoadFile  base<...>/intel
 0.000000 | HookLoadFile  base<...>/irc
-0.000000 | HookLoadFile  base<...>/json.zeek
 0.000000 | HookLoadFile  base<...>/krb
 0.000000 | HookLoadFile  base<...>/logging
 0.000000 | HookLoadFile  base<...>/logging.bif.zeek

testing/btest/Baseline/scripts.base.frameworks.logging.ascii-double/json.log

@@ -1,22 +1,22 @@
-{"d":2.153226e+09}
+{"d":2153226000.0}
-{"d":2.153226e+09}
+{"d":2153226000.1}
-{"d":2.153226e+09}
+{"d":2153226000.123457}
 {"d":1.0}
 {"d":1.1}
-{"d":1.123457}
+{"d":1.123456789}
-{"d":-1.123457}
+{"d":-1.123456789}
 {"d":1.1234}
 {"d":0.1234}
 {"d":50000.0}
 {"d":-50000.0}
-{"d":3.140000e+15}
+{"d":3.14e+15}
-{"d":-3.140000e+15}
+{"d":-3.14e+15}
-{"d":1.790000e+308}
+{"d":1.79e+308}
-{"d":-1.790000e+308}
+{"d":-1.79e+308}
-{"d":0.000012}
+{"d":1.23456789e-05}
-{"d":0}
+{"d":2.23e-308}
-{"d":-0}
+{"d":-2.23e-308}
-{"d":inf}
+{"d":null}
-{"d":-inf}
+{"d":null}
-{"d":0.0}
+{"d":-0.0}
-{"d":nan}
+{"d":null}

testing/btest/Baseline/scripts.base.frameworks.logging.ascii-double/test.log

@@ -3,7 +3,7 @@
 #empty_field	(empty)
 #unset_field	-
 #path	test
-#open	2017-11-06-19-58-08
+#open	2019-07-01-17-40-55
 #fields	d
 #types	double
 2153226000.0
@@ -28,4 +28,4 @@ inf
 -inf
 0.0
 nan
-#close	2017-11-06-19-58-08
+#close	2019-07-01-17-40-55

testing/btest/Baseline/scripts.base.frameworks.logging.ascii-json-utf8/ssh.log

@@ -0,0 +1,12 @@
+{"s":"a"}
+{"s":"\b\f\n\r\t\\x00\\x15"}
+{"s":"ñ"}
+{"s":"\\xc3("}
+{"s":"\\xa0\\xa1"}
+{"s":"₡"}
+{"s":"\\xe2(\\xa1"}
+{"s":"\\xe2\\x82("}
+{"s":"𐌼"}
+{"s":"\\xf0(\\x8c\\xbc"}
+{"s":"\\xf0\\x90(\\xbc"}
+{"s":"\\xf0(\\x8c("}

testing/btest/Baseline/scripts.base.frameworks.logging.ascii-json/ssh.log

@@ -1 +1 @@
-{"b":true,"i":-42,"e":"SSH::LOG","c":21,"p":123,"sn":"10.0.0.0/24","a":"1.2.3.4","d":3.14,"t":1215620010.54321,"iv":100.0,"s":"hurz","sc":[2,4,1,3],"ss":["BB","AA","CC"],"se":[],"vc":[10,20,30],"ve":[],"f":"SSH::foo\u000a{ \u000aif (0 < SSH::i) \u000a\u0009return (Foo);\u000aelse\u000a\u0009return (Bar);\u000a\u000a}"}
+{"b":true,"i":-42,"e":"SSH::LOG","c":21,"p":123,"sn":"10.0.0.0/24","a":"1.2.3.4","d":3.14,"t":1215620010.54321,"iv":100.0,"s":"hurz","sc":[2,4,1,3],"ss":["BB","AA","CC"],"se":[],"vc":[10,20,30],"ve":[],"f":"SSH::foo\n{ \nif (0 < SSH::i) \n\treturn (Foo);\nelse\n\treturn (Bar);\n\n}"}

testing/btest/Baseline/scripts.base.frameworks.openflow.ryu-basic/.stdout

@@ -1,22 +1,22 @@
 http://127.0.0.1:8080/stats/flowentry/clear/42
 http://127.0.0.1:8080/stats/flowentry/add
-{"priority": 0, "idle_timeout": 0, "hard_timeout": 0, "match": {}, "actions": [{"port": 3, "type": "OUTPUT"}, {"port": 7, "type": "OUTPUT"}], "cookie": 4398046511105, "flags": 0, "dpid": 42}
+{"priority":0,"idle_timeout":0,"hard_timeout":0,"match":{},"actions":[{"port":3,"type":"OUTPUT"},{"port":7,"type":"OUTPUT"}],"cookie":4398046511105,"flags":0,"dpid":42}
 Flow_mod_success
 http://127.0.0.1:8080/stats/flowentry/add
-{"priority": 5, "idle_timeout": 30, "hard_timeout": 0, "match": {"nw_dst": "74.53.140.153/32", "tp_dst": 25, "tp_src": 1470, "nw_proto": 6, "dl_type": 2048, "nw_src": "10.10.1.4/32"}, "actions": [], "cookie": 4398046511146, "flags": 0, "dpid": 42}
+{"priority":5,"idle_timeout":30,"hard_timeout":0,"match":{"nw_dst":"74.53.140.153/32","tp_dst":25,"tp_src":1470,"nw_proto":6,"dl_type":2048,"nw_src":"10.10.1.4/32"},"actions":[],"cookie":4398046511146,"flags":0,"dpid":42}
 http://127.0.0.1:8080/stats/flowentry/add
-{"priority": 5, "idle_timeout": 30, "hard_timeout": 0, "match": {"nw_dst": "10.10.1.4/32", "tp_dst": 1470, "tp_src": 25, "nw_proto": 6, "dl_type": 2048, "nw_src": "74.53.140.153/32"}, "actions": [], "cookie": 4398046511146, "flags": 0, "dpid": 42}
+{"priority":5,"idle_timeout":30,"hard_timeout":0,"match":{"nw_dst":"10.10.1.4/32","tp_dst":1470,"tp_src":25,"nw_proto":6,"dl_type":2048,"nw_src":"74.53.140.153/32"},"actions":[],"cookie":4398046511146,"flags":0,"dpid":42}
 Flow_mod_success
 Flow_mod_success
 http://127.0.0.1:8080/stats/flowentry/add
-{"priority": 5, "idle_timeout": 30, "hard_timeout": 0, "match": {"nw_dst": "192.168.133.102/32", "tp_dst": 25, "tp_src": 49648, "nw_proto": 6, "dl_type": 2048, "nw_src": "192.168.133.100/32"}, "actions": [], "cookie": 4398046511146, "flags": 0, "dpid": 42}
+{"priority":5,"idle_timeout":30,"hard_timeout":0,"match":{"nw_dst":"192.168.133.102/32","tp_dst":25,"tp_src":49648,"nw_proto":6,"dl_type":2048,"nw_src":"192.168.133.100/32"},"actions":[],"cookie":4398046511146,"flags":0,"dpid":42}
 http://127.0.0.1:8080/stats/flowentry/add
-{"priority": 5, "idle_timeout": 30, "hard_timeout": 0, "match": {"nw_dst": "192.168.133.100/32", "tp_dst": 49648, "tp_src": 25, "nw_proto": 6, "dl_type": 2048, "nw_src": "192.168.133.102/32"}, "actions": [], "cookie": 4398046511146, "flags": 0, "dpid": 42}
+{"priority":5,"idle_timeout":30,"hard_timeout":0,"match":{"nw_dst":"192.168.133.100/32","tp_dst":49648,"tp_src":25,"nw_proto":6,"dl_type":2048,"nw_src":"192.168.133.102/32"},"actions":[],"cookie":4398046511146,"flags":0,"dpid":42}
 Flow_mod_success
 Flow_mod_success
 http://127.0.0.1:8080/stats/flowentry/add
-{"priority": 5, "idle_timeout": 30, "hard_timeout": 0, "match": {"nw_dst": "17.167.150.73/32", "tp_dst": 443, "tp_src": 49655, "nw_proto": 6, "dl_type": 2048, "nw_src": "192.168.133.100/32"}, "actions": [], "cookie": 4398046511146, "flags": 0, "dpid": 42}
+{"priority":5,"idle_timeout":30,"hard_timeout":0,"match":{"nw_dst":"17.167.150.73/32","tp_dst":443,"tp_src":49655,"nw_proto":6,"dl_type":2048,"nw_src":"192.168.133.100/32"},"actions":[],"cookie":4398046511146,"flags":0,"dpid":42}
 http://127.0.0.1:8080/stats/flowentry/add
-{"priority": 5, "idle_timeout": 30, "hard_timeout": 0, "match": {"nw_dst": "192.168.133.100/32", "tp_dst": 49655, "tp_src": 443, "nw_proto": 6, "dl_type": 2048, "nw_src": "17.167.150.73/32"}, "actions": [], "cookie": 4398046511146, "flags": 0, "dpid": 42}
+{"priority":5,"idle_timeout":30,"hard_timeout":0,"match":{"nw_dst":"192.168.133.100/32","tp_dst":49655,"tp_src":443,"nw_proto":6,"dl_type":2048,"nw_src":"17.167.150.73/32"},"actions":[],"cookie":4398046511146,"flags":0,"dpid":42}
 Flow_mod_success
 Flow_mod_success

testing/btest/Baseline/scripts.base.utils.json/output

@@ -8,32 +8,33 @@ true
 "-12.0 hrs"
 "hello"
 ""
-65535
+{"port":65535,"proto":"tcp"}
-1
+{"port":1,"proto":"udp"}
-123
+{"port":123,"proto":"icmp"}
-0
+{"port":0,"proto":"unknown"}
 "1.2.3.4"
 "ffff:1234::1"
 "123.123.123.123"
 "192.0.0.0/8"
 "fe80::/64"
 "Red"
-{"s": "test", "c": 100}
+"/^?(^abcd)$?/"
-{"s": "test"}
+{"s":"test","c":100}
-{"s": "test"}
+{"s":"test"}
-{"m": {"s": "test"}}
+{"s":"test"}
+{"m":{"s":"test"}}
 []
-[2, 1]
+[2,1]
 ["1.2.3.4"]
-[[true, false]]
+[[true,false]]
-[{"s": "test"}]
+[{"s":"test"}]
 []
-[2, 1]
+[2,1]
 ["1.2.3.4"]
-[{"s": "test"}]
+[{"s":"test"}]
-[{"s": "test"}]
+[{"s":"test"}]
 {}
-{"2": "10.2.2.2", "1": "10.1.1.1"}
+{"2":"10.2.2.2","1":"10.1.1.1"}
-{"10.1.1.1": {"a": 1}, "10.2.2.2": {"b": 2}}
+{"10.1.1.1":{"a":1},"10.2.2.2":{"b":2}}
-{"10.1.1.1": [1, 2], "10.2.2.2": [3, 5]}
+{"10.1.1.1":[1,2],"10.2.2.2":[3,5]}
-{"1": {"s": "test"}}
+{"1":{"s":"test"}}

testing/btest/scripts/base/frameworks/logging/ascii-json-utf8.zeek

@@ -0,0 +1,59 @@
+#
+# @TEST-EXEC: zeek -b %INPUT
+# @TEST-EXEC: btest-diff ssh.log
+#
+# Testing all possible types.
+
+redef LogAscii::use_json = T;
+
+
+module SSH;
+
+export {
+	redef enum Log::ID += { LOG };
+
+	type Log: record {
+		s: string;
+	} &log;
+}
+
+event zeek_init()
+{
+	Log::create_stream(SSH::LOG, [$columns=Log]);
+
+	# Strings taken from https://stackoverflow.com/a/3886015
+
+	# Valid ASCII and valid ASCII control characters
+	Log::write(SSH::LOG, [$s="a"]);
+	Log::write(SSH::LOG, [$s="\b\f\n\r\t\x00\x15"]);
+
+	# Valid 2 Octet Sequence
+	Log::write(SSH::LOG, [$s="\xc3\xb1"]);
+	
+	# Invalid 2 Octet Sequence
+	Log::write(SSH::LOG, [$s="\xc3\x28"]);
+	
+	# Invalid Sequence Identifier
+	Log::write(SSH::LOG, [$s="\xa0\xa1"]);
+	
+	# Valid 3 Octet Sequence
+	Log::write(SSH::LOG, [$s="\xe2\x82\xa1"]);
+	
+	# Invalid 3 Octet Sequence (in 2nd Octet)
+	Log::write(SSH::LOG, [$s="\xe2\x28\xa1"]);
+	
+	# Invalid 3 Octet Sequence (in 3rd Octet)
+	Log::write(SSH::LOG, [$s="\xe2\x82\x28"]);
+	
+	# Valid 4 Octet Sequence
+	Log::write(SSH::LOG, [$s="\xf0\x90\x8c\xbc"]);
+	
+	# Invalid 4 Octet Sequence (in 2nd Octet)
+	Log::write(SSH::LOG, [$s="\xf0\x28\x8c\xbc"]);
+	
+	# Invalid 4 Octet Sequence (in 3rd Octet)
+	Log::write(SSH::LOG, [$s="\xf0\x90\x28\xbc"]);
+	
+	# Invalid 4 Octet Sequence (in 4th Octet)
+	Log::write(SSH::LOG, [$s="\xf0\x28\x8c\x28"]);
+}

testing/btest/scripts/base/utils/json.test

@@ -72,6 +72,9 @@ event zeek_init()
 	local e: color = Red;
 	print to_json(e);
 
+	local p: pattern = /^abcd/;
+	print to_json(p);
+
 	# #########################
 	# Test the container types: