Mirror/zeek
1
0
Fork 0
mirror of https://github.com/zeek/zeek.git synced 2025-10-02 14:48:21 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

Optimize initial memory consumption

Browse source
This commit is contained in:
Elad Solomon 2021-12-16 08:22:19 +00:00 • committed by Tomer Lev
parent eb08b696bf
commit 3f349b8a37
10 changed files with 49 additions and 22 deletions
Download patch file Download diff file Expand all files Collapse all files

4
CMakeLists.txt
View file

@ -180,6 +180,10 @@ if ( NOT ZEEK_LOG_DIR )
set(ZEEK_LOG_DIR ${ZEEK_ROOT_DIR}/logs)
endif ()
if ( NOT MSVC )
set(HAVE_SUPERVISOR true)
endif ()
install(DIRECTORY DESTINATION ${ZEEK_ETC_INSTALL_DIR})
install(DIRECTORY DESTINATION ${ZEEK_STATE_DIR})
install(DIRECTORY DESTINATION ${ZEEK_SPOOL_DIR})

6
src/CMakeLists.txt
View file

@ -177,6 +177,8 @@ gen_zam_target(${GEN_ZAM_SRC})
## Including subdirectories.
########################################################################
option(USE_SQLITE "Should Zeek use SQLite?" ON)
set(bro_SUBDIR_LIBS CACHE INTERNAL "subdir libraries" FORCE)
set(bro_SUBDIR_DEPS CACHE INTERNAL "subdir dependencies" FORCE)
set(bro_PLUGIN_LIBS CACHE INTERNAL "plugin libraries" FORCE)
@ -441,7 +443,7 @@ set(THIRD_PARTY_SRCS
3rdparty/modp_numtoa.c
3rdparty/patricia.c
3rdparty/setsignal.c
3rdparty/sqlite3.c
$<$<BOOL:USE_SQLITE>:3rdparty/sqlite3.c>
3rdparty/strsep.c
)
@ -679,7 +681,7 @@ install(FILES
${CMAKE_CURRENT_SOURCE_DIR}/3rdparty/modp_numtoa.h
${CMAKE_CURRENT_SOURCE_DIR}/3rdparty/patricia.h
${CMAKE_CURRENT_SOURCE_DIR}/3rdparty/setsignal.h
${CMAKE_CURRENT_SOURCE_DIR}/3rdparty/sqlite3.h
$<$<BOOL:USE_SQLITE>:${CMAKE_CURRENT_SOURCE_DIR}/3rdparty/sqlite3.h>
${CMAKE_CURRENT_SOURCE_DIR}/3rdparty/doctest.h
DESTINATION include/zeek/3rdparty
)

21
src/Val.cc
View file

@ -3963,17 +3963,9 @@ ValManager::ValManager()
for ( auto i = 0u; i < PREALLOCATED_INTS; ++i )
ints[i] = Val::MakeInt(PREALLOCATED_INT_LOWEST + i);
for ( auto i = 0u; i < ports.size(); ++i )
{
auto& arr = ports[i];
auto port_type = (TransportProto)i;
for ( auto j = 0u; j < arr.size(); ++j )
arr[j] = IntrusivePtr{AdoptRef{}, new PortVal(PortVal::Mask(j, port_type))};
}
}
const PortValPtr& ValManager::Port(uint32_t port_num, TransportProto port_type) const
const PortValPtr& ValManager::Port(uint32_t port_num, TransportProto port_type)
{
if ( port_num >= 65536 )
{
@ -3981,10 +3973,17 @@ const PortValPtr& ValManager::Port(uint32_t port_num, TransportProto port_type)
port_num = 0;
}
return ports[port_type][port_num];
std::pair key{port_num, port_type};
if (ports.find(key) == ports.end())
{
ports[key] = IntrusivePtr{AdoptRef{}, new PortVal(PortVal::Mask(port_num, port_type))};
}
const PortValPtr& ValManager::Port(uint32_t port_num) const
return ports[key];
}
const PortValPtr& ValManager::Port(uint32_t port_num)
{
auto mask = port_num & PORT_SPACE_MASK;
port_num &= ~PORT_SPACE_MASK;

17
src/Val.h
View file

@ -298,9 +298,15 @@ protected:
class ValManager
{
public:
#ifdef _MSC_VER
static constexpr zeek_uint_t PREALLOCATED_COUNTS = 1;
static constexpr zeek_uint_t PREALLOCATED_INTS = 1;
static constexpr zeek_int_t PREALLOCATED_INT_LOWEST = 0;
#else
static constexpr zeek_uint_t PREALLOCATED_COUNTS = 4096;
static constexpr zeek_uint_t PREALLOCATED_INTS = 512;
static constexpr zeek_int_t PREALLOCATED_INT_LOWEST = -255;
#endif
static constexpr zeek_int_t PREALLOCATED_INT_HIGHEST = PREALLOCATED_INT_LOWEST +
PREALLOCATED_INTS - 1;
@ -327,13 +333,13 @@ public:
inline const StringValPtr& EmptyString() const { return empty_string; }
// Port number given in host order.
const PortValPtr& Port(uint32_t port_num, TransportProto port_type) const;
const PortValPtr& Port(uint32_t port_num, TransportProto port_type);
// Host-order port number already masked with port space protocol mask.
const PortValPtr& Port(uint32_t port_num) const;
const PortValPtr& Port(uint32_t port_num);
private:
std::array<std::array<PortValPtr, 65536>, NUM_PORT_SPACES> ports;
std::map<std::pair<uint32_t, TransportProto>, PortValPtr> ports;
std::array<ValPtr, PREALLOCATED_COUNTS> counts;
std::array<ValPtr, PREALLOCATED_INTS> ints;
StringValPtr empty_string;
@ -1124,9 +1130,8 @@ public:
AddedField(field);
}
// For int types, we provide both [u]int32_t and [u]int64_t versions
// for convenience, since sometimes the caller has one rather
// than the other.
// For int types, we provide both [u]int32_t and [u]int64_t versions for
// convenience, since sometimes the caller has one rather than the other.
void Assign(int field, int32_t new_val)
{
(*record_val)[field] = ZVal(zeek_int_t(new_val));

4
src/input/readers/CMakeLists.txt
View file

@ -4,4 +4,6 @@ add_subdirectory(benchmark)
add_subdirectory(binary)
add_subdirectory(config)
add_subdirectory(raw)
add_subdirectory(sqlite)
if (USE_SQLITE)
add_subdirectory(sqlite)
endif()

4
src/logging/writers/CMakeLists.txt
View file

@ -1,4 +1,6 @@
add_subdirectory(ascii)
add_subdirectory(none)
add_subdirectory(sqlite)
if (USE_SQLITE)
add_subdirectory(sqlite)
endif()

2
src/supervisor/Supervisor.h
View file

@ -2,6 +2,8 @@
#pragma once
#include "zeek/zeek-config.h"
#include <sys/types.h>
#include <chrono>
#include <cstdint>

3
src/util.cc
View file

@ -2434,6 +2434,9 @@ void get_memory_usage(uint64_t* total, uint64_t* malloced)
// In KB.
ret_total = r.ru_maxrss * 1024;
if ( malloced )
*malloced = r.ru_ixrss * 1024;
#endif
if ( total )

7
src/zeek-setup.cc
View file

@ -16,7 +16,9 @@
#include <list>
#include <optional>
#ifdef USE_SQLITE
#include "zeek/3rdparty/sqlite3.h"
#endif
#define DOCTEST_CONFIG_IMPLEMENT
@ -645,11 +647,12 @@ SetupResult setup(int argc, char** argv, Options* zopts)
// FIXME: On systems that don't provide /dev/urandom, OpenSSL doesn't
// seed the PRNG. We should do this here (but at least Linux, FreeBSD
// and Solaris provide /dev/urandom).
#ifdef USE_SQLITE
int r = sqlite3_initialize();
if ( r != SQLITE_OK )
reporter->Error("Failed to initialize sqlite3: %s", sqlite3_errstr(r));
#endif
timer_mgr = new TimerMgr();
@ -1121,7 +1124,9 @@ int cleanup(bool did_run_loop)
run_state::detail::delete_run();
terminate_zeek();
#ifdef USE_SQLITE
sqlite3_shutdown();
#endif
do_ssl_deinit();

3
zeek-config.h.in
View file

@ -135,6 +135,9 @@
/* Use the ElasticSearch writer. */
#cmakedefine USE_ELASTICSEARCH
/* Use the sqlite reader/writer. */
#cmakedefine USE_SQLITE
/* Version number of package */
#define VERSION "@VERSION@"