From c3b9756576ee13c2131138bc7dd78bb1acc7dda4 Mon Sep 17 00:00:00 2001 From: Elad Solomon Date: Sun, 12 Sep 2021 12:19:46 +0000 Subject: [PATCH] Zeek Statistics - Added statistics hook - Fixed rlimit usage - Removed POpen for windows implementation layer - Completed statistics plugin --- scripts/base/init-bare.zeek | 11 ++++ src/CMakeLists.txt | 1 + src/statistics/CMakeLists.txt | 9 ++++ src/statistics/Plugin.cc | 54 +++++++++++++++++++ src/statistics/Plugin.h | 29 ++++++++++ src/statistics/statistics.bif | 29 ++++++++++ .../canonified_loaded_scripts.log | 1 + .../canonified_loaded_scripts.log | 1 + 8 files changed, 135 insertions(+) create mode 100644 src/statistics/CMakeLists.txt create mode 100644 src/statistics/Plugin.cc create mode 100644 src/statistics/Plugin.h create mode 100644 src/statistics/statistics.bif diff --git a/scripts/base/init-bare.zeek b/scripts/base/init-bare.zeek index 519db3922d..f6f87769df 100644 --- a/scripts/base/init-bare.zeek +++ b/scripts/base/init-bare.zeek @@ -792,6 +792,17 @@ type ReporterStats: record { weirds_by_type: table[string] of count; }; +## Statistics about how many times each event name is queued. +## +## .. zeek:see:: Statistics::get_and_reset_event_name_statistics +type EventNameCounter: record { + ## Name of the zeek event. + name: string &log; + ## Times it was queued, as captured by event hook. + times_queued: count &log; +} &log; +type EventNameStats: vector of EventNameCounter; + ## Table type used to map variable names to their memory allocation. ## ## .. todo:: We need this type definition only for declaring builtin functions diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 4b86fcdf10..9fd5d2c743 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -192,6 +192,7 @@ add_subdirectory(file_analysis) add_subdirectory(input) add_subdirectory(iosource) add_subdirectory(logging) +add_subdirectory(statistics) add_subdirectory(probabilistic) add_subdirectory(session) diff --git a/src/statistics/CMakeLists.txt b/src/statistics/CMakeLists.txt new file mode 100644 index 0000000000..11538b5cf0 --- /dev/null +++ b/src/statistics/CMakeLists.txt @@ -0,0 +1,9 @@ + +include(ZeekPlugin) + +include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR}) + +zeek_plugin_begin(Zeek Statistics) +zeek_plugin_cc(Plugin.cc) +bif_target(statistics.bif) +zeek_plugin_end() diff --git a/src/statistics/Plugin.cc b/src/statistics/Plugin.cc new file mode 100644 index 0000000000..18cb71645c --- /dev/null +++ b/src/statistics/Plugin.cc @@ -0,0 +1,54 @@ + +#include + +#include "Plugin.h" + +#include "zeek/Func.h" +#include "zeek/Event.h" +#include "zeek/Conn.h" +#include "zeek/Desc.h" +#include "zeek/threading/Formatter.h" +#include "zeek/RunState.h" + +#include "statistics.bif.h" + +namespace zeek::plugin::statistics { Plugin plugin; } + +using namespace zeek::plugin::statistics; + +zeek::plugin::Configuration Plugin::Configure() + { + zeek::plugin::Configuration config; + config.name = "Statistics"; + config.description = "Statistics module"; + config.version.major = 1; + config.version.minor = 0; + config.version.patch = 0; + return config; + } + +bool Plugin::HookQueueEvent(zeek::Event* event) + { + const char* name = event->Handler()->Name(); + + std::lock_guard scopedLock(m_lock); + if (m_eventNameCounters.find(name) == m_eventNameCounters.end()) + { + m_eventNameCounters[name] = 0; + } + m_eventNameCounters[name]++; + return false; + } + +std::unordered_map Plugin::GetAndResetEventStatistics() + { + std::lock_guard scopedLock(m_lock); + std::unordered_map result(m_eventNameCounters); + m_eventNameCounters.clear(); + return result; + } + +void Plugin::StartEventNamesStatisticsMonitor() + { + EnableHook(zeek::plugin::HOOK_QUEUE_EVENT); + } diff --git a/src/statistics/Plugin.h b/src/statistics/Plugin.h new file mode 100644 index 0000000000..0c396d8b07 --- /dev/null +++ b/src/statistics/Plugin.h @@ -0,0 +1,29 @@ + +#pragma once + +#include +#include +#include +#include "zeek/plugin/Plugin.h" + +namespace zeek::plugin::statistics { + +class Plugin : public zeek::plugin::Plugin +{ +protected: + bool HookQueueEvent(zeek::Event* event) override; + + zeek::plugin::Configuration Configure() override; + +public: + std::unordered_map GetAndResetEventStatistics(); + void StartEventNamesStatisticsMonitor(); + +private: + std::unordered_map m_eventNameCounters; + std::mutex m_lock; + +}; + +extern Plugin plugin; +} diff --git a/src/statistics/statistics.bif b/src/statistics/statistics.bif new file mode 100644 index 0000000000..7f2f14c9a2 --- /dev/null +++ b/src/statistics/statistics.bif @@ -0,0 +1,29 @@ + +%%{ +#include "zeek/statistics/Plugin.h" +%%} + +function Statistics::get_and_reset_event_name_statistics%(%): EventNameStats + %{ + auto rval = zeek::make_intrusive(zeek::id::find_type("EventNameStats")); + auto stats = zeek::plugin::statistics::plugin.GetAndResetEventStatistics(); + const auto& recordType = zeek::id::find_type("EventNameCounter"); + + auto i = 0; + for (auto& eventCounter : stats) + { + auto eventStatRecord = zeek::make_intrusive(recordType); + eventStatRecord->Assign(0, zeek::make_intrusive(eventCounter.first)); + eventStatRecord->Assign(1, zeek::val_mgr->Count(eventCounter.second)); + rval->Assign(i, std::move(eventStatRecord)); + ++i; + } + + return rval; + %} + +function Statistics::start_event_name_statistics_monitor%(%): any + %{ + zeek::plugin::statistics::plugin.StartEventNamesStatisticsMonitor(); + return nullptr; + %} \ No newline at end of file diff --git a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log index 429bc44f5a..cda12d3d6a 100644 --- a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log @@ -125,6 +125,7 @@ scripts/base/init-frameworks-and-bifs.zeek build/scripts/base/bif/telemetry.bif.zeek build/scripts/base/bif/zeekygen.bif.zeek build/scripts/base/bif/pcap.bif.zeek + build/scripts/base/bif/statistics.bif.zeek build/scripts/base/bif/bloom-filter.bif.zeek build/scripts/base/bif/cardinality-counter.bif.zeek build/scripts/base/bif/top-k.bif.zeek diff --git a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log index 75cc1c72d4..66c433fb47 100644 --- a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log @@ -125,6 +125,7 @@ scripts/base/init-frameworks-and-bifs.zeek build/scripts/base/bif/telemetry.bif.zeek build/scripts/base/bif/zeekygen.bif.zeek build/scripts/base/bif/pcap.bif.zeek + build/scripts/base/bif/statistics.bif.zeek build/scripts/base/bif/bloom-filter.bif.zeek build/scripts/base/bif/cardinality-counter.bif.zeek build/scripts/base/bif/top-k.bif.zeek