Mirror/zeek
1
0
Fork 0
mirror of https://github.com/zeek/zeek.git synced 2025-10-17 14:08:20 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

Merge remote-tracking branch 'origin/topic/timw/595-json-perf'

Browse source 
* origin/topic/timw/595-json-perf:
  Update COPYING.3rdparty
  Use json::emplace to avoid some extra calls to operator[]
  Use tessil/unordered-map instead of nlohmann/fifo-map to mitigate performance issues when logging JSON
This commit is contained in:
Jon Siwek 2019-10-01 16:41:35 -07:00
commit a5d71ed2d2
11 changed files with 55 additions and 32 deletions
Download patch file Download diff file Expand all files Collapse all files

2
src/3rdparty

@ -1 +1 @@
Subproject commit ff6438b894c3d70a017a86025eb5ab0770d7a0a7
Subproject commit e3a7602300a3449698aa71fb9689c0a1a792e3a4

7
src/CMakeLists.txt
View file

@ -441,12 +441,17 @@ install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/
)
install(FILES
${CMAKE_CURRENT_SOURCE_DIR}/3rdparty/fifo_map.hpp
${CMAKE_CURRENT_SOURCE_DIR}/3rdparty/json.hpp
${CMAKE_CURRENT_SOURCE_DIR}/3rdparty/sqlite3.h
DESTINATION include/zeek/3rdparty
)
install(FILES
${CMAKE_CURRENT_SOURCE_DIR}/3rdparty/tsl-ordered-map/ordered_map.h
${CMAKE_CURRENT_SOURCE_DIR}/3rdparty/tsl-ordered-map/ordered_hash.h
DESTINATION include/zeek/3rdparty/tsl-ordered-map
)
########################################################################
## Clang-tidy target now that we have all of the sources

24
src/Val.cc
View file

@ -28,14 +28,19 @@
#include "broker/Data.h"
#include "3rdparty/json.hpp"
#include "3rdparty/fifo_map.hpp"
#include "3rdparty/tsl-ordered-map/ordered_map.h"
// Define a class for use with the json library that orders the keys in the same order that
// they were inserted. By default, the json library orders them alphabetically and we don't
// want it like that.
template<class K, class V, class compare, class A>
using json_fifo_map = nlohmann::fifo_map<K, V, nlohmann::fifo_map_compare<K>, A>;
using ZeekJson = nlohmann::basic_json<json_fifo_map>;
template<class Key, class T, class Ignore, class Allocator,
class Hash = std::hash<Key>, class KeyEqual = std::equal_to<Key>,
class AllocatorPair = typename std::allocator_traits<Allocator>::template rebind_alloc<std::pair<Key, T>>,
class ValueTypeContainer = std::vector<std::pair<Key, T>, AllocatorPair>>
using ordered_map = tsl::ordered_map<Key, T, Hash, KeyEqual, AllocatorPair, ValueTypeContainer>;
using ZeekJson = nlohmann::basic_json<ordered_map>;
Val::Val(Func* f)
{
@ -493,8 +498,8 @@ static ZeekJson BuildJSON(Val* val, bool only_loggable=false, RE_Matcher* re=new
case TYPE_PORT:
{
auto* pval = val->AsPortVal();
j["port"] = pval->Port();
j["proto"] = pval->Protocol();
j.emplace("port", pval->Port());
j.emplace("proto", pval->Protocol());
break;
}
@ -568,7 +573,7 @@ static ZeekJson BuildJSON(Val* val, bool only_loggable=false, RE_Matcher* re=new
else
key_string = key_json.dump();
j[key_string] = BuildJSON(entry_value, only_loggable, re);
j.emplace(key_string, BuildJSON(entry_value, only_loggable, re));
}
Unref(entry_key);
@ -603,7 +608,7 @@ static ZeekJson BuildJSON(Val* val, bool only_loggable=false, RE_Matcher* re=new
Val* value = rval->LookupWithDefault(i);
if ( value && ( ! only_loggable || rt->FieldHasAttr(i, ATTR_LOG) ) )
j[key_string] = BuildJSON(value, only_loggable, re);
j.emplace(key_string, BuildJSON(value, only_loggable, re));
Unref(value);
}
@ -635,9 +640,8 @@ static ZeekJson BuildJSON(Val* val, bool only_loggable=false, RE_Matcher* re=new
case TYPE_OPAQUE:
{
j = ZeekJson::object();
auto* oval = val->AsOpaqueVal();
j["opaque_type"] = OpaqueMgr::mgr()->TypeID(oval);
j = { { "opaque_type", OpaqueMgr::mgr()->TypeID(oval) } };
break;
}

8
src/threading/formatters/JSON.cc
View file

@ -37,7 +37,7 @@ bool JSON::Describe(ODesc* desc, int num_fields, const Field* const * fields,
if ( new_entry.is_null() )
return false;
j[fields[i]->name] = new_entry;
j.emplace(fields[i]->name, new_entry);
}
}
@ -186,11 +186,7 @@ ZeekJson JSON::BuildJSON(Value* val, const string& name) const
}
if ( ! name.empty() && ! j.is_null() )
{
ZeekJson j2 = ZeekJson::object();
j2[name] = j;
return j2;
}
return { { name, j } };
return j;
}

12
src/threading/formatters/JSON.h
View file

@ -4,7 +4,7 @@
#include "../Formatter.h"
#include "3rdparty/json.hpp"
#include "3rdparty/fifo_map.hpp"
#include "3rdparty/tsl-ordered-map/ordered_map.h"
namespace threading { namespace formatter {
@ -12,9 +12,13 @@ namespace threading { namespace formatter {
// Define a class for use with the json library that orders the keys in the same order that
// they were inserted. By default, the json library orders them alphabetically and we don't
// want it like that.
template<class K, class V, class compare, class A>
using json_fifo_map = nlohmann::fifo_map<K, V, nlohmann::fifo_map_compare<K>, A>;
using ZeekJson = nlohmann::basic_json<json_fifo_map>;
template<class Key, class T, class Ignore, class Allocator,
class Hash = std::hash<Key>, class KeyEqual = std::equal_to<Key>,
class AllocatorPair = typename std::allocator_traits<Allocator>::template rebind_alloc<std::pair<Key, T>>,
class ValueTypeContainer = std::vector<std::pair<Key, T>, AllocatorPair>>
using ordered_map = tsl::ordered_map<Key, T, Hash, KeyEqual, AllocatorPair, ValueTypeContainer>;
using ZeekJson = nlohmann::basic_json<ordered_map>;
/**
* A thread-safe class for converting values into a JSON representation