Optimize initial memory consumption

CMakeLists.txt

@@ -180,6 +180,10 @@ if ( NOT ZEEK_LOG_DIR )
     set(ZEEK_LOG_DIR ${ZEEK_ROOT_DIR}/logs)
 endif ()
 
+if ( NOT MSVC )
+    set(HAVE_SUPERVISOR true)
+endif ()
+
 install(DIRECTORY DESTINATION ${ZEEK_ETC_INSTALL_DIR})
 install(DIRECTORY DESTINATION ${ZEEK_STATE_DIR})
 install(DIRECTORY DESTINATION ${ZEEK_SPOOL_DIR})

src/CMakeLists.txt

@@ -177,6 +177,8 @@ gen_zam_target(${GEN_ZAM_SRC})
 ## Including subdirectories.
 ########################################################################
 
+option(USE_SQLITE "Should Zeek use SQLite?" ON)
+
 set(bro_SUBDIR_LIBS CACHE INTERNAL "subdir libraries" FORCE)
 set(bro_SUBDIR_DEPS CACHE INTERNAL "subdir dependencies" FORCE)
 set(bro_PLUGIN_LIBS CACHE INTERNAL "plugin libraries" FORCE)
@@ -441,7 +443,7 @@ set(THIRD_PARTY_SRCS
     3rdparty/modp_numtoa.c
     3rdparty/patricia.c
     3rdparty/setsignal.c
-    3rdparty/sqlite3.c
+    $<$<BOOL:USE_SQLITE>:3rdparty/sqlite3.c>
     3rdparty/strsep.c
 )
 
@@ -679,7 +681,7 @@ install(FILES
         ${CMAKE_CURRENT_SOURCE_DIR}/3rdparty/modp_numtoa.h
         ${CMAKE_CURRENT_SOURCE_DIR}/3rdparty/patricia.h
         ${CMAKE_CURRENT_SOURCE_DIR}/3rdparty/setsignal.h
-        ${CMAKE_CURRENT_SOURCE_DIR}/3rdparty/sqlite3.h
+        $<$<BOOL:USE_SQLITE>:${CMAKE_CURRENT_SOURCE_DIR}/3rdparty/sqlite3.h>
         ${CMAKE_CURRENT_SOURCE_DIR}/3rdparty/doctest.h
         DESTINATION include/zeek/3rdparty
 )

src/Val.cc

@@ -3963,17 +3963,9 @@ ValManager::ValManager()
 	for ( auto i = 0u; i < PREALLOCATED_INTS; ++i )
 		ints[i] = Val::MakeInt(PREALLOCATED_INT_LOWEST + i);
 
-	for ( auto i = 0u; i < ports.size(); ++i )
-		{
-		auto& arr = ports[i];
-		auto port_type = (TransportProto)i;
-
-		for ( auto j = 0u; j < arr.size(); ++j )
-			arr[j] = IntrusivePtr{AdoptRef{}, new PortVal(PortVal::Mask(j, port_type))};
-		}
 	}
 
-const PortValPtr& ValManager::Port(uint32_t port_num, TransportProto port_type) const
+const PortValPtr& ValManager::Port(uint32_t port_num, TransportProto port_type)
 	{
 	if ( port_num >= 65536 )
 		{
@@ -3981,10 +3973,17 @@ const PortValPtr& ValManager::Port(uint32_t port_num, TransportProto port_type)
 		port_num = 0;
 		}
 
-	return ports[port_type][port_num];
+
+	std::pair key{port_num, port_type};
+    if (ports.find(key) == ports.end())
+	{
+		ports[key] = IntrusivePtr{AdoptRef{}, new PortVal(PortVal::Mask(port_num, port_type))};
 	}
 
-const PortValPtr& ValManager::Port(uint32_t port_num) const
+	return ports[key];
+	}
+
+const PortValPtr& ValManager::Port(uint32_t port_num)
 	{
 	auto mask = port_num & PORT_SPACE_MASK;
 	port_num &= ~PORT_SPACE_MASK;

src/Val.h

@@ -298,9 +298,15 @@ protected:
 class ValManager
 	{
 public:
+#ifdef _MSC_VER
+	static constexpr zeek_uint_t PREALLOCATED_COUNTS = 1;
+	static constexpr zeek_uint_t PREALLOCATED_INTS = 1;
+	static constexpr zeek_int_t PREALLOCATED_INT_LOWEST = 0;
+#else
 	static constexpr zeek_uint_t PREALLOCATED_COUNTS = 4096;
 	static constexpr zeek_uint_t PREALLOCATED_INTS = 512;
 	static constexpr zeek_int_t PREALLOCATED_INT_LOWEST = -255;
+#endif
 	static constexpr zeek_int_t PREALLOCATED_INT_HIGHEST = PREALLOCATED_INT_LOWEST +
 	                                                       PREALLOCATED_INTS - 1;
 
@@ -327,13 +333,13 @@ public:
 	inline const StringValPtr& EmptyString() const { return empty_string; }
 
 	// Port number given in host order.
-	const PortValPtr& Port(uint32_t port_num, TransportProto port_type) const;
+	const PortValPtr& Port(uint32_t port_num, TransportProto port_type);
 
 	// Host-order port number already masked with port space protocol mask.
-	const PortValPtr& Port(uint32_t port_num) const;
+	const PortValPtr& Port(uint32_t port_num);
 
 private:
-	std::array<std::array<PortValPtr, 65536>, NUM_PORT_SPACES> ports;
+	std::map<std::pair<uint32_t, TransportProto>, PortValPtr> ports;
 	std::array<ValPtr, PREALLOCATED_COUNTS> counts;
 	std::array<ValPtr, PREALLOCATED_INTS> ints;
 	StringValPtr empty_string;
@@ -1124,9 +1130,8 @@ public:
 		AddedField(field);
 		}
 
-	// For int types, we provide both [u]int32_t and [u]int64_t  versions
+	// For int types, we provide both [u]int32_t and [u]int64_t versions for
-	// for convenience, since sometimes the caller has one rather
+	// convenience, since sometimes the caller has one rather than the other.
-	// than the other.
 	void Assign(int field, int32_t new_val)
 		{
 		(*record_val)[field] = ZVal(zeek_int_t(new_val));

src/input/readers/CMakeLists.txt

@@ -4,4 +4,6 @@ add_subdirectory(benchmark)
 add_subdirectory(binary)
 add_subdirectory(config)
 add_subdirectory(raw)
+if (USE_SQLITE)
   add_subdirectory(sqlite)
+endif()

src/logging/writers/CMakeLists.txt

@@ -1,4 +1,6 @@
 
 add_subdirectory(ascii)
 add_subdirectory(none)
+if (USE_SQLITE)
     add_subdirectory(sqlite)
+endif()

src/supervisor/Supervisor.h

@@ -2,6 +2,8 @@
 
 #pragma once
 
+#include "zeek/zeek-config.h"
+
 #include <sys/types.h>
 #include <chrono>
 #include <cstdint>

src/util.cc

@@ -2434,6 +2434,9 @@ void get_memory_usage(uint64_t* total, uint64_t* malloced)
 
 	// In KB.
 	ret_total = r.ru_maxrss * 1024;
+
+	if ( malloced )
+		*malloced = r.ru_ixrss * 1024;
 #endif
 
 	if ( total )

src/zeek-setup.cc

@@ -16,7 +16,9 @@
 #include <list>
 #include <optional>
 
+#ifdef USE_SQLITE
 #include "zeek/3rdparty/sqlite3.h"
+#endif
 
 #define DOCTEST_CONFIG_IMPLEMENT
 
@@ -645,11 +647,12 @@ SetupResult setup(int argc, char** argv, Options* zopts)
 	// FIXME: On systems that don't provide /dev/urandom, OpenSSL doesn't
 	// seed the PRNG. We should do this here (but at least Linux, FreeBSD
 	// and Solaris provide /dev/urandom).
-
+#ifdef USE_SQLITE
 	int r = sqlite3_initialize();
 
 	if ( r != SQLITE_OK )
 		reporter->Error("Failed to initialize sqlite3: %s", sqlite3_errstr(r));
+#endif
 
 	timer_mgr = new TimerMgr();
 
@@ -1121,7 +1124,9 @@ int cleanup(bool did_run_loop)
 	run_state::detail::delete_run();
 	terminate_zeek();
 
+#ifdef USE_SQLITE
 	sqlite3_shutdown();
+#endif
 
 	do_ssl_deinit();
 

zeek-config.h.in

@@ -135,6 +135,9 @@
 /* Use the ElasticSearch writer. */
 #cmakedefine USE_ELASTICSEARCH
 
+/* Use the sqlite reader/writer. */
+#cmakedefine USE_SQLITE
+
 /* Version number of package */
 #define VERSION "@VERSION@"