Merge remote-tracking branch 'origin/topic/timw/595-json-perf'

* origin/topic/timw/595-json-perf:
  Update COPYING.3rdparty
  Use json::emplace to avoid some extra calls to operator[]
  Use tessil/unordered-map instead of nlohmann/fifo-map to mitigate performance issues when logging JSON

CHANGES

@@ -1,4 +1,13 @@
 
+3.1.0-dev.161 | 2019-10-01 16:41:35 -0700
+
+  * Update COPYING.3rdparty (Tim Wojtulewicz, Corelight)
+
+  * Use json::emplace to avoid some extra calls to operator[] (Tim Wojtulewicz, Corelight)
+
+  * Use tessil/unordered-map instead of nlohmann/fifo-map to improve JSON
+    logging performance (Tim Wojtulewicz, Corelight)
+
 3.1.0-dev.156 | 2019-10-01 09:05:49 +0000
 
   * Improve RecordVal JSON formatting to no longer create a record

COPYING.3rdparty

@@ -115,18 +115,18 @@ DEALINGS IN THE SOFTWARE.
 
 ==============================================================================
 
-%%% fifo_map.hpp
+%%% tsl-ordered-map
 
 ==============================================================================
 
-Copyright (c) 2015-2017 Niels Lohmann.
+Copyright (c) 2017 Tessil
 
-Permission is hereby granted, free of charge, to any person obtaining a copy of
+Permission is hereby granted, free of charge, to any person obtaining a copy
-this software and associated documentation files (the "Software"), to deal in
+of this software and associated documentation files (the "Software"), to deal
-the Software without restriction, including without limitation the rights to
+in the Software without restriction, including without limitation the rights
-use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-of the Software, and to permit persons to whom the Software is furnished to do
+copies of the Software, and to permit persons to whom the Software is
-so, subject to the following conditions:
+furnished to do so, subject to the following conditions:
 
 The above copyright notice and this permission notice shall be included in all
 copies or substantial portions of the Software.

VERSION

@@ -1 +1 @@
-3.1.0-dev.156
+3.1.0-dev.161

aux/broker

@@ -1 +1 @@
-Subproject commit e4142d5c488968558f96bdd6f63ae7671226a66e
+Subproject commit bb826b6eac14fa15486eb12557890f94f860f845

src/3rdparty

@@ -1 +1 @@
-Subproject commit ff6438b894c3d70a017a86025eb5ab0770d7a0a7
+Subproject commit e3a7602300a3449698aa71fb9689c0a1a792e3a4

src/CMakeLists.txt

@@ -441,12 +441,17 @@ install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/
 )
 
 install(FILES
-        ${CMAKE_CURRENT_SOURCE_DIR}/3rdparty/fifo_map.hpp
         ${CMAKE_CURRENT_SOURCE_DIR}/3rdparty/json.hpp
         ${CMAKE_CURRENT_SOURCE_DIR}/3rdparty/sqlite3.h
         DESTINATION include/zeek/3rdparty
 )
 
+install(FILES
+        ${CMAKE_CURRENT_SOURCE_DIR}/3rdparty/tsl-ordered-map/ordered_map.h
+        ${CMAKE_CURRENT_SOURCE_DIR}/3rdparty/tsl-ordered-map/ordered_hash.h
+        DESTINATION include/zeek/3rdparty/tsl-ordered-map
+)
+
 ########################################################################
 ## Clang-tidy target now that we have all of the sources
 

src/Val.cc

@@ -28,14 +28,19 @@
 #include "broker/Data.h"
 
 #include "3rdparty/json.hpp"
-#include "3rdparty/fifo_map.hpp"
+#include "3rdparty/tsl-ordered-map/ordered_map.h"
+
 
 // Define a class for use with the json library that orders the keys in the same order that
 // they were inserted. By default, the json library orders them alphabetically and we don't
 // want it like that.
-template<class K, class V, class compare, class A>
+template<class Key, class T, class Ignore, class Allocator,
-using json_fifo_map = nlohmann::fifo_map<K, V, nlohmann::fifo_map_compare<K>, A>;
+         class Hash = std::hash<Key>, class KeyEqual = std::equal_to<Key>,
-using ZeekJson = nlohmann::basic_json<json_fifo_map>;
+         class AllocatorPair = typename std::allocator_traits<Allocator>::template rebind_alloc<std::pair<Key, T>>,
+         class ValueTypeContainer = std::vector<std::pair<Key, T>, AllocatorPair>>
+using ordered_map = tsl::ordered_map<Key, T, Hash, KeyEqual, AllocatorPair, ValueTypeContainer>;
+
+using ZeekJson = nlohmann::basic_json<ordered_map>;
 
 Val::Val(Func* f)
 	{
@@ -493,8 +498,8 @@ static ZeekJson BuildJSON(Val* val, bool only_loggable=false, RE_Matcher* re=new
 		case TYPE_PORT:
 			{
 			auto* pval = val->AsPortVal();
-			j["port"] = pval->Port();
+			j.emplace("port", pval->Port());
-			j["proto"] = pval->Protocol();
+			j.emplace("proto", pval->Protocol());
 			break;
 			}
 
@@ -568,7 +573,7 @@ static ZeekJson BuildJSON(Val* val, bool only_loggable=false, RE_Matcher* re=new
 					else
 						key_string = key_json.dump();
 
-					j[key_string] = BuildJSON(entry_value, only_loggable, re);
+					j.emplace(key_string, BuildJSON(entry_value, only_loggable, re));
 					}
 
 				Unref(entry_key);
@@ -603,7 +608,7 @@ static ZeekJson BuildJSON(Val* val, bool only_loggable=false, RE_Matcher* re=new
 				Val* value = rval->LookupWithDefault(i);
 
 				if ( value && ( ! only_loggable || rt->FieldHasAttr(i, ATTR_LOG) ) )
-					j[key_string] = BuildJSON(value, only_loggable, re);
+					j.emplace(key_string, BuildJSON(value, only_loggable, re));
 
 				Unref(value);
 				}
@@ -635,9 +640,8 @@ static ZeekJson BuildJSON(Val* val, bool only_loggable=false, RE_Matcher* re=new
 
 		case TYPE_OPAQUE:
 			{
-			j = ZeekJson::object();
 			auto* oval = val->AsOpaqueVal();
-			j["opaque_type"] = OpaqueMgr::mgr()->TypeID(oval);
+			j = { { "opaque_type", OpaqueMgr::mgr()->TypeID(oval) } };
 			break;
 			}
 

src/threading/formatters/JSON.cc

@@ -37,7 +37,7 @@ bool JSON::Describe(ODesc* desc, int num_fields, const Field* const * fields,
 			if ( new_entry.is_null() )
 				return false;
 
-			j[fields[i]->name] = new_entry;
+			j.emplace(fields[i]->name, new_entry);
 			}
 		}
 
@@ -186,11 +186,7 @@ ZeekJson JSON::BuildJSON(Value* val, const string& name) const
 		}
 
 	if ( ! name.empty() && ! j.is_null() )
-		{
+		return { { name, j } };
-		ZeekJson j2 = ZeekJson::object();
-		j2[name] = j;
-		return j2;
-		}
 
 	return j;
 	}

src/threading/formatters/JSON.h

@@ -4,7 +4,7 @@
 
 #include "../Formatter.h"
 #include "3rdparty/json.hpp"
-#include "3rdparty/fifo_map.hpp"
+#include "3rdparty/tsl-ordered-map/ordered_map.h"
 
 
 namespace threading { namespace formatter {
@@ -12,9 +12,13 @@ namespace threading { namespace formatter {
 // Define a class for use with the json library that orders the keys in the same order that
 // they were inserted. By default, the json library orders them alphabetically and we don't
 // want it like that.
-template<class K, class V, class compare, class A>
+template<class Key, class T, class Ignore, class Allocator,
-using json_fifo_map = nlohmann::fifo_map<K, V, nlohmann::fifo_map_compare<K>, A>;
+         class Hash = std::hash<Key>, class KeyEqual = std::equal_to<Key>,
-using ZeekJson = nlohmann::basic_json<json_fifo_map>;
+         class AllocatorPair = typename std::allocator_traits<Allocator>::template rebind_alloc<std::pair<Key, T>>,
+         class ValueTypeContainer = std::vector<std::pair<Key, T>, AllocatorPair>>
+using ordered_map = tsl::ordered_map<Key, T, Hash, KeyEqual, AllocatorPair, ValueTypeContainer>;
+
+using ZeekJson = nlohmann::basic_json<ordered_map>;
 
 /**
   * A thread-safe class for converting values into a JSON representation

testing/btest/Baseline/scripts.base.utils.json/output

@@ -40,3 +40,4 @@ true
 {"10.1.1.1":{"a":1},"10.2.2.2":{"b":2}}
 {"10.1.1.1":[1,2],"10.2.2.2":[3,5]}
 {"1":{"s":"test"}}
+{"opaque_type":"TopkVal"}

testing/btest/scripts/base/utils/json.test

@@ -128,4 +128,8 @@ event zeek_init()
 	print to_json(ta3);
 	print to_json(ta4);
 	print to_json(ta5, T);
+
+	# Opaque
+	local o1: opaque of topk = topk_init(5);
+	print to_json(o1);
 	}