diff --git a/CHANGES b/CHANGES index e57405cbdb..6805083eee 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,13 @@ +3.1.0-dev.161 | 2019-10-01 16:41:35 -0700 + + * Update COPYING.3rdparty (Tim Wojtulewicz, Corelight) + + * Use json::emplace to avoid some extra calls to operator[] (Tim Wojtulewicz, Corelight) + + * Use tessil/unordered-map instead of nlohmann/fifo-map to improve JSON + logging performance (Tim Wojtulewicz, Corelight) + 3.1.0-dev.156 | 2019-10-01 09:05:49 +0000 * Improve RecordVal JSON formatting to no longer create a record diff --git a/COPYING.3rdparty b/COPYING.3rdparty index 8c66c27e5d..52b69deecb 100644 --- a/COPYING.3rdparty +++ b/COPYING.3rdparty @@ -115,18 +115,18 @@ DEALINGS IN THE SOFTWARE. ============================================================================== -%%% fifo_map.hpp +%%% tsl-ordered-map ============================================================================== -Copyright (c) 2015-2017 Niels Lohmann. +Copyright (c) 2017 Tessil -Permission is hereby granted, free of charge, to any person obtaining a copy of -this software and associated documentation files (the "Software"), to deal in -the Software without restriction, including without limitation the rights to -use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies -of the Software, and to permit persons to whom the Software is furnished to do -so, subject to the following conditions: +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. diff --git a/VERSION b/VERSION index ba95c64e8f..733b67b077 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.1.0-dev.156 +3.1.0-dev.161 diff --git a/aux/broker b/aux/broker index e4142d5c48..bb826b6eac 160000 --- a/aux/broker +++ b/aux/broker @@ -1 +1 @@ -Subproject commit e4142d5c488968558f96bdd6f63ae7671226a66e +Subproject commit bb826b6eac14fa15486eb12557890f94f860f845 diff --git a/src/3rdparty b/src/3rdparty index ff6438b894..e3a7602300 160000 --- a/src/3rdparty +++ b/src/3rdparty @@ -1 +1 @@ -Subproject commit ff6438b894c3d70a017a86025eb5ab0770d7a0a7 +Subproject commit e3a7602300a3449698aa71fb9689c0a1a792e3a4 diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index f2009c536f..564fd7acb3 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -441,12 +441,17 @@ install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/ ) install(FILES - ${CMAKE_CURRENT_SOURCE_DIR}/3rdparty/fifo_map.hpp ${CMAKE_CURRENT_SOURCE_DIR}/3rdparty/json.hpp ${CMAKE_CURRENT_SOURCE_DIR}/3rdparty/sqlite3.h DESTINATION include/zeek/3rdparty ) +install(FILES + ${CMAKE_CURRENT_SOURCE_DIR}/3rdparty/tsl-ordered-map/ordered_map.h + ${CMAKE_CURRENT_SOURCE_DIR}/3rdparty/tsl-ordered-map/ordered_hash.h + DESTINATION include/zeek/3rdparty/tsl-ordered-map +) + ######################################################################## ## Clang-tidy target now that we have all of the sources diff --git a/src/Val.cc b/src/Val.cc index 73f4473433..930d88d5e1 100644 --- a/src/Val.cc +++ b/src/Val.cc @@ -28,14 +28,19 @@ #include "broker/Data.h" #include "3rdparty/json.hpp" -#include "3rdparty/fifo_map.hpp" +#include "3rdparty/tsl-ordered-map/ordered_map.h" + // Define a class for use with the json library that orders the keys in the same order that // they were inserted. By default, the json library orders them alphabetically and we don't // want it like that. -template -using json_fifo_map = nlohmann::fifo_map, A>; -using ZeekJson = nlohmann::basic_json; +template, class KeyEqual = std::equal_to, + class AllocatorPair = typename std::allocator_traits::template rebind_alloc>, + class ValueTypeContainer = std::vector, AllocatorPair>> +using ordered_map = tsl::ordered_map; + +using ZeekJson = nlohmann::basic_json; Val::Val(Func* f) { @@ -493,8 +498,8 @@ static ZeekJson BuildJSON(Val* val, bool only_loggable=false, RE_Matcher* re=new case TYPE_PORT: { auto* pval = val->AsPortVal(); - j["port"] = pval->Port(); - j["proto"] = pval->Protocol(); + j.emplace("port", pval->Port()); + j.emplace("proto", pval->Protocol()); break; } @@ -568,7 +573,7 @@ static ZeekJson BuildJSON(Val* val, bool only_loggable=false, RE_Matcher* re=new else key_string = key_json.dump(); - j[key_string] = BuildJSON(entry_value, only_loggable, re); + j.emplace(key_string, BuildJSON(entry_value, only_loggable, re)); } Unref(entry_key); @@ -603,7 +608,7 @@ static ZeekJson BuildJSON(Val* val, bool only_loggable=false, RE_Matcher* re=new Val* value = rval->LookupWithDefault(i); if ( value && ( ! only_loggable || rt->FieldHasAttr(i, ATTR_LOG) ) ) - j[key_string] = BuildJSON(value, only_loggable, re); + j.emplace(key_string, BuildJSON(value, only_loggable, re)); Unref(value); } @@ -635,9 +640,8 @@ static ZeekJson BuildJSON(Val* val, bool only_loggable=false, RE_Matcher* re=new case TYPE_OPAQUE: { - j = ZeekJson::object(); auto* oval = val->AsOpaqueVal(); - j["opaque_type"] = OpaqueMgr::mgr()->TypeID(oval); + j = { { "opaque_type", OpaqueMgr::mgr()->TypeID(oval) } }; break; } diff --git a/src/threading/formatters/JSON.cc b/src/threading/formatters/JSON.cc index b5d9c9904b..919ecd4a2b 100644 --- a/src/threading/formatters/JSON.cc +++ b/src/threading/formatters/JSON.cc @@ -37,7 +37,7 @@ bool JSON::Describe(ODesc* desc, int num_fields, const Field* const * fields, if ( new_entry.is_null() ) return false; - j[fields[i]->name] = new_entry; + j.emplace(fields[i]->name, new_entry); } } @@ -186,11 +186,7 @@ ZeekJson JSON::BuildJSON(Value* val, const string& name) const } if ( ! name.empty() && ! j.is_null() ) - { - ZeekJson j2 = ZeekJson::object(); - j2[name] = j; - return j2; - } + return { { name, j } }; return j; } diff --git a/src/threading/formatters/JSON.h b/src/threading/formatters/JSON.h index 1015eb54c6..71edadc61d 100644 --- a/src/threading/formatters/JSON.h +++ b/src/threading/formatters/JSON.h @@ -4,7 +4,7 @@ #include "../Formatter.h" #include "3rdparty/json.hpp" -#include "3rdparty/fifo_map.hpp" +#include "3rdparty/tsl-ordered-map/ordered_map.h" namespace threading { namespace formatter { @@ -12,9 +12,13 @@ namespace threading { namespace formatter { // Define a class for use with the json library that orders the keys in the same order that // they were inserted. By default, the json library orders them alphabetically and we don't // want it like that. -template -using json_fifo_map = nlohmann::fifo_map, A>; -using ZeekJson = nlohmann::basic_json; +template, class KeyEqual = std::equal_to, + class AllocatorPair = typename std::allocator_traits::template rebind_alloc>, + class ValueTypeContainer = std::vector, AllocatorPair>> +using ordered_map = tsl::ordered_map; + +using ZeekJson = nlohmann::basic_json; /** * A thread-safe class for converting values into a JSON representation diff --git a/testing/btest/Baseline/scripts.base.utils.json/output b/testing/btest/Baseline/scripts.base.utils.json/output index 4d9004b33c..8757a51433 100644 --- a/testing/btest/Baseline/scripts.base.utils.json/output +++ b/testing/btest/Baseline/scripts.base.utils.json/output @@ -40,3 +40,4 @@ true {"10.1.1.1":{"a":1},"10.2.2.2":{"b":2}} {"10.1.1.1":[1,2],"10.2.2.2":[3,5]} {"1":{"s":"test"}} +{"opaque_type":"TopkVal"} diff --git a/testing/btest/scripts/base/utils/json.test b/testing/btest/scripts/base/utils/json.test index 8797cc193b..3572bd3e07 100644 --- a/testing/btest/scripts/base/utils/json.test +++ b/testing/btest/scripts/base/utils/json.test @@ -128,4 +128,8 @@ event zeek_init() print to_json(ta3); print to_json(ta4); print to_json(ta5, T); + + # Opaque + local o1: opaque of topk = topk_init(5); + print to_json(o1); }