Mirror/zeek
1
0
Fork 0
mirror of https://github.com/zeek/zeek.git synced 2025-10-02 22:58:20 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

Merge remote-tracking branch 'origin/topic/timw/595-json-perf'

Browse source 
* origin/topic/timw/595-json-perf:
  Update COPYING.3rdparty
  Use json::emplace to avoid some extra calls to operator[]
  Use tessil/unordered-map instead of nlohmann/fifo-map to mitigate performance issues when logging JSON
This commit is contained in:
Jon Siwek 2019-10-01 16:41:35 -07:00
commit a5d71ed2d2
11 changed files with 55 additions and 32 deletions
Download patch file Download diff file Expand all files Collapse all files

9
CHANGES
View file

@ -1,4 +1,13 @@
3.1.0-dev.161 | 2019-10-01 16:41:35 -0700
* Update COPYING.3rdparty (Tim Wojtulewicz, Corelight)
* Use json::emplace to avoid some extra calls to operator[] (Tim Wojtulewicz, Corelight)
* Use tessil/unordered-map instead of nlohmann/fifo-map to improve JSON
logging performance (Tim Wojtulewicz, Corelight)
3.1.0-dev.156 | 2019-10-01 09:05:49 +0000
* Improve RecordVal JSON formatting to no longer create a record

16
COPYING.3rdparty
View file

@ -115,18 +115,18 @@ DEALINGS IN THE SOFTWARE.
==============================================================================
%%% fifo_map.hpp
%%% tsl-ordered-map
==============================================================================
Copyright (c) 2015-2017 Niels Lohmann.
Copyright (c) 2017 Tessil
Permission is hereby granted, free of charge, to any person obtaining a copy of
this software and associated documentation files (the "Software"), to deal in
the Software without restriction, including without limitation the rights to
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
of the Software, and to permit persons to whom the Software is furnished to do
so, subject to the following conditions:
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

2
VERSION
View file

@ -1 +1 @@
3.1.0-dev.156
3.1.0-dev.161

2
aux/broker

@ -1 +1 @@
Subproject commit e4142d5c488968558f96bdd6f63ae7671226a66e
Subproject commit bb826b6eac14fa15486eb12557890f94f860f845

2
src/3rdparty

@ -1 +1 @@
Subproject commit ff6438b894c3d70a017a86025eb5ab0770d7a0a7
Subproject commit e3a7602300a3449698aa71fb9689c0a1a792e3a4

7
src/CMakeLists.txt
View file

@ -441,12 +441,17 @@ install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/
)
install(FILES
${CMAKE_CURRENT_SOURCE_DIR}/3rdparty/fifo_map.hpp
${CMAKE_CURRENT_SOURCE_DIR}/3rdparty/json.hpp
${CMAKE_CURRENT_SOURCE_DIR}/3rdparty/sqlite3.h
DESTINATION include/zeek/3rdparty
)
install(FILES
${CMAKE_CURRENT_SOURCE_DIR}/3rdparty/tsl-ordered-map/ordered_map.h
${CMAKE_CURRENT_SOURCE_DIR}/3rdparty/tsl-ordered-map/ordered_hash.h
DESTINATION include/zeek/3rdparty/tsl-ordered-map
)
########################################################################
## Clang-tidy target now that we have all of the sources

24
src/Val.cc
View file

@ -28,14 +28,19 @@
#include "broker/Data.h"
#include "3rdparty/json.hpp"
#include "3rdparty/fifo_map.hpp"
#include "3rdparty/tsl-ordered-map/ordered_map.h"
// Define a class for use with the json library that orders the keys in the same order that
// they were inserted. By default, the json library orders them alphabetically and we don't
// want it like that.
template<class K, class V, class compare, class A>
using json_fifo_map = nlohmann::fifo_map<K, V, nlohmann::fifo_map_compare<K>, A>;
using ZeekJson = nlohmann::basic_json<json_fifo_map>;
template<class Key, class T, class Ignore, class Allocator,
class Hash = std::hash<Key>, class KeyEqual = std::equal_to<Key>,
class AllocatorPair = typename std::allocator_traits<Allocator>::template rebind_alloc<std::pair<Key, T>>,
class ValueTypeContainer = std::vector<std::pair<Key, T>, AllocatorPair>>
using ordered_map = tsl::ordered_map<Key, T, Hash, KeyEqual, AllocatorPair, ValueTypeContainer>;
using ZeekJson = nlohmann::basic_json<ordered_map>;
Val::Val(Func* f)
{
@ -493,8 +498,8 @@ static ZeekJson BuildJSON(Val* val, bool only_loggable=false, RE_Matcher* re=new
case TYPE_PORT:
{
auto* pval = val->AsPortVal();
j["port"] = pval->Port();
j["proto"] = pval->Protocol();
j.emplace("port", pval->Port());
j.emplace("proto", pval->Protocol());
break;
}
@ -568,7 +573,7 @@ static ZeekJson BuildJSON(Val* val, bool only_loggable=false, RE_Matcher* re=new
else
key_string = key_json.dump();
j[key_string] = BuildJSON(entry_value, only_loggable, re);
j.emplace(key_string, BuildJSON(entry_value, only_loggable, re));
}
Unref(entry_key);
@ -603,7 +608,7 @@ static ZeekJson BuildJSON(Val* val, bool only_loggable=false, RE_Matcher* re=new
Val* value = rval->LookupWithDefault(i);
if ( value && ( ! only_loggable || rt->FieldHasAttr(i, ATTR_LOG) ) )
j[key_string] = BuildJSON(value, only_loggable, re);
j.emplace(key_string, BuildJSON(value, only_loggable, re));
Unref(value);
}
@ -635,9 +640,8 @@ static ZeekJson BuildJSON(Val* val, bool only_loggable=false, RE_Matcher* re=new
case TYPE_OPAQUE:
{
j = ZeekJson::object();
auto* oval = val->AsOpaqueVal();
j["opaque_type"] = OpaqueMgr::mgr()->TypeID(oval);
j = { { "opaque_type", OpaqueMgr::mgr()->TypeID(oval) } };
break;
}

8
src/threading/formatters/JSON.cc
View file

@ -37,7 +37,7 @@ bool JSON::Describe(ODesc* desc, int num_fields, const Field* const * fields,
if ( new_entry.is_null() )
return false;
j[fields[i]->name] = new_entry;
j.emplace(fields[i]->name, new_entry);
}
}
@ -186,11 +186,7 @@ ZeekJson JSON::BuildJSON(Value* val, const string& name) const
}
if ( ! name.empty() && ! j.is_null() )
{
ZeekJson j2 = ZeekJson::object();
j2[name] = j;
return j2;
}
return { { name, j } };
return j;
}

12
src/threading/formatters/JSON.h
View file

@ -4,7 +4,7 @@
#include "../Formatter.h"
#include "3rdparty/json.hpp"
#include "3rdparty/fifo_map.hpp"
#include "3rdparty/tsl-ordered-map/ordered_map.h"
namespace threading { namespace formatter {
@ -12,9 +12,13 @@ namespace threading { namespace formatter {
// Define a class for use with the json library that orders the keys in the same order that
// they were inserted. By default, the json library orders them alphabetically and we don't
// want it like that.
template<class K, class V, class compare, class A>
using json_fifo_map = nlohmann::fifo_map<K, V, nlohmann::fifo_map_compare<K>, A>;
using ZeekJson = nlohmann::basic_json<json_fifo_map>;
template<class Key, class T, class Ignore, class Allocator,
class Hash = std::hash<Key>, class KeyEqual = std::equal_to<Key>,
class AllocatorPair = typename std::allocator_traits<Allocator>::template rebind_alloc<std::pair<Key, T>>,
class ValueTypeContainer = std::vector<std::pair<Key, T>, AllocatorPair>>
using ordered_map = tsl::ordered_map<Key, T, Hash, KeyEqual, AllocatorPair, ValueTypeContainer>;
using ZeekJson = nlohmann::basic_json<ordered_map>;
/**
* A thread-safe class for converting values into a JSON representation

1
testing/btest/Baseline/scripts.base.utils.json/output
View file

@ -40,3 +40,4 @@ true
{"10.1.1.1":{"a":1},"10.2.2.2":{"b":2}}
{"10.1.1.1":[1,2],"10.2.2.2":[3,5]}
{"1":{"s":"test"}}
{"opaque_type":"TopkVal"}

4
testing/btest/scripts/base/utils/json.test
View file

@ -128,4 +128,8 @@ event zeek_init()
print to_json(ta3);
print to_json(ta4);
print to_json(ta5, T);
# Opaque
local o1: opaque of topk = topk_init(5);
print to_json(o1);
}