From c8f2d52d9197d459511a0072a576f390e2301f18 Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Wed, 25 Sep 2019 16:11:32 -0700 Subject: [PATCH 1/3] Use tessil/unordered-map instead of nlohmann/fifo-map to mitigate performance issues when logging JSON --- src/3rdparty | 2 +- src/CMakeLists.txt | 7 ++++++- src/Val.cc | 13 +++++++++---- src/threading/formatters/JSON.h | 12 ++++++++---- 4 files changed, 24 insertions(+), 10 deletions(-) diff --git a/src/3rdparty b/src/3rdparty index c1eab215ae..1030737715 160000 --- a/src/3rdparty +++ b/src/3rdparty @@ -1 +1 @@ -Subproject commit c1eab215ae34b2bc03fcb8c787b386a25e00bf3e +Subproject commit 10307377157d71c0ad0d3fd33ccf7e04ce8f65ff diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index f2009c536f..564fd7acb3 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -441,12 +441,17 @@ install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/ ) install(FILES - ${CMAKE_CURRENT_SOURCE_DIR}/3rdparty/fifo_map.hpp ${CMAKE_CURRENT_SOURCE_DIR}/3rdparty/json.hpp ${CMAKE_CURRENT_SOURCE_DIR}/3rdparty/sqlite3.h DESTINATION include/zeek/3rdparty ) +install(FILES + ${CMAKE_CURRENT_SOURCE_DIR}/3rdparty/tsl-ordered-map/ordered_map.h + ${CMAKE_CURRENT_SOURCE_DIR}/3rdparty/tsl-ordered-map/ordered_hash.h + DESTINATION include/zeek/3rdparty/tsl-ordered-map +) + ######################################################################## ## Clang-tidy target now that we have all of the sources diff --git a/src/Val.cc b/src/Val.cc index 6a27dbfbc7..2da784de9b 100644 --- a/src/Val.cc +++ b/src/Val.cc @@ -28,14 +28,19 @@ #include "broker/Data.h" #include "3rdparty/json.hpp" -#include "3rdparty/fifo_map.hpp" +#include "3rdparty/tsl-ordered-map/ordered_map.h" + // Define a class for use with the json library that orders the keys in the same order that // they were inserted. By default, the json library orders them alphabetically and we don't // want it like that. -template -using json_fifo_map = nlohmann::fifo_map, A>; -using ZeekJson = nlohmann::basic_json; +template, class KeyEqual = std::equal_to, + class AllocatorPair = typename std::allocator_traits::template rebind_alloc>, + class ValueTypeContainer = std::vector, AllocatorPair>> +using ordered_map = tsl::ordered_map; + +using ZeekJson = nlohmann::basic_json; Val::Val(Func* f) { diff --git a/src/threading/formatters/JSON.h b/src/threading/formatters/JSON.h index 1015eb54c6..71edadc61d 100644 --- a/src/threading/formatters/JSON.h +++ b/src/threading/formatters/JSON.h @@ -4,7 +4,7 @@ #include "../Formatter.h" #include "3rdparty/json.hpp" -#include "3rdparty/fifo_map.hpp" +#include "3rdparty/tsl-ordered-map/ordered_map.h" namespace threading { namespace formatter { @@ -12,9 +12,13 @@ namespace threading { namespace formatter { // Define a class for use with the json library that orders the keys in the same order that // they were inserted. By default, the json library orders them alphabetically and we don't // want it like that. -template -using json_fifo_map = nlohmann::fifo_map, A>; -using ZeekJson = nlohmann::basic_json; +template, class KeyEqual = std::equal_to, + class AllocatorPair = typename std::allocator_traits::template rebind_alloc>, + class ValueTypeContainer = std::vector, AllocatorPair>> +using ordered_map = tsl::ordered_map; + +using ZeekJson = nlohmann::basic_json; /** * A thread-safe class for converting values into a JSON representation From d4d1009b5ca666b72d0aae109658a630359f5c23 Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Wed, 25 Sep 2019 16:12:50 -0700 Subject: [PATCH 2/3] Use json::emplace to avoid some extra calls to operator[] --- src/Val.cc | 11 +++++------ src/threading/formatters/JSON.cc | 8 ++------ testing/btest/Baseline/scripts.base.utils.json/output | 1 + testing/btest/scripts/base/utils/json.test | 4 ++++ 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/Val.cc b/src/Val.cc index 2da784de9b..a6958b35b5 100644 --- a/src/Val.cc +++ b/src/Val.cc @@ -494,8 +494,8 @@ static ZeekJson BuildJSON(Val* val, bool only_loggable=false, RE_Matcher* re=new case TYPE_PORT: { auto* pval = val->AsPortVal(); - j["port"] = pval->Port(); - j["proto"] = pval->Protocol(); + j.emplace("port", pval->Port()); + j.emplace("proto", pval->Protocol()); break; } @@ -577,7 +577,7 @@ static ZeekJson BuildJSON(Val* val, bool only_loggable=false, RE_Matcher* re=new else key_string = key_json.dump(); - j[key_string] = BuildJSON(entry_value, only_loggable, re); + j.emplace(key_string, BuildJSON(entry_value, only_loggable, re)); } Unref(lv); @@ -614,7 +614,7 @@ static ZeekJson BuildJSON(Val* val, bool only_loggable=false, RE_Matcher* re=new Val* value = key_field->Lookup("value", true); if ( value && ( ! only_loggable || key_field->Lookup("log")->AsBool() ) ) - j[key_string] = BuildJSON(value, only_loggable, re); + j.emplace(key_string, BuildJSON(value, only_loggable, re)); } delete fields; @@ -646,9 +646,8 @@ static ZeekJson BuildJSON(Val* val, bool only_loggable=false, RE_Matcher* re=new case TYPE_OPAQUE: { - j = ZeekJson::object(); auto* oval = val->AsOpaqueVal(); - j["opaque_type"] = OpaqueMgr::mgr()->TypeID(oval); + j = { { "opaque_type", OpaqueMgr::mgr()->TypeID(oval) } }; break; } diff --git a/src/threading/formatters/JSON.cc b/src/threading/formatters/JSON.cc index 58b1ffd779..096e823a10 100644 --- a/src/threading/formatters/JSON.cc +++ b/src/threading/formatters/JSON.cc @@ -37,7 +37,7 @@ bool JSON::Describe(ODesc* desc, int num_fields, const Field* const * fields, if ( new_entry.is_null() ) return false; - j[fields[i]->name] = new_entry; + j.emplace(fields[i]->name, new_entry); } } @@ -182,11 +182,7 @@ ZeekJson JSON::BuildJSON(Value* val, const string& name) const } if ( ! name.empty() && ! j.is_null() ) - { - ZeekJson j2 = ZeekJson::object(); - j2[name] = j; - return j2; - } + return { { name, j } }; return j; } diff --git a/testing/btest/Baseline/scripts.base.utils.json/output b/testing/btest/Baseline/scripts.base.utils.json/output index 2d2e56253f..3be9649902 100644 --- a/testing/btest/Baseline/scripts.base.utils.json/output +++ b/testing/btest/Baseline/scripts.base.utils.json/output @@ -38,3 +38,4 @@ true {"10.1.1.1":{"a":1},"10.2.2.2":{"b":2}} {"10.1.1.1":[1,2],"10.2.2.2":[3,5]} {"1":{"s":"test"}} +{"opaque_type":"TopkVal"} diff --git a/testing/btest/scripts/base/utils/json.test b/testing/btest/scripts/base/utils/json.test index 6e7854b744..bec4a665b6 100644 --- a/testing/btest/scripts/base/utils/json.test +++ b/testing/btest/scripts/base/utils/json.test @@ -122,4 +122,8 @@ event zeek_init() print to_json(ta3); print to_json(ta4); print to_json(ta5, T); + + # Opaque + local o1: opaque of topk = topk_init(5); + print to_json(o1); } From a96ef569385321b8eed0ddde6053cb75ee6f3a49 Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Fri, 27 Sep 2019 10:25:18 -0700 Subject: [PATCH 3/3] Update COPYING.3rdparty --- COPYING.3rdparty | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/COPYING.3rdparty b/COPYING.3rdparty index 8c66c27e5d..52b69deecb 100644 --- a/COPYING.3rdparty +++ b/COPYING.3rdparty @@ -115,18 +115,18 @@ DEALINGS IN THE SOFTWARE. ============================================================================== -%%% fifo_map.hpp +%%% tsl-ordered-map ============================================================================== -Copyright (c) 2015-2017 Niels Lohmann. +Copyright (c) 2017 Tessil -Permission is hereby granted, free of charge, to any person obtaining a copy of -this software and associated documentation files (the "Software"), to deal in -the Software without restriction, including without limitation the rights to -use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies -of the Software, and to permit persons to whom the Software is furnished to do -so, subject to the following conditions: +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.