Mirror/zeek
1
0
Fork 0
mirror of https://github.com/zeek/zeek.git synced 2025-10-10 10:38:20 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

Make telemetry metrics out of MsgThread statistics

Browse source
This commit is contained in:
Tim Wojtulewicz 2024-08-05 13:06:55 -07:00
parent a6843067e9
commit 73f71e652d
7 changed files with 155 additions and 82 deletions
Download patch file Download diff file Expand all files Collapse all files

121
src/threading/Manager.cc
View file

@ -23,6 +23,9 @@ void HeartbeatTimer::Dispatch(double t, bool is_expire) {
} // namespace detail
static std::vector<uint64_t> pending_bucket_brackets = {1, 10, 100,
1000, 10000, std::numeric_limits<uint64_t>::infinity()};
Manager::Manager() {
DBG_LOG(DBG_THREADING, "Creating thread manager ...");
@ -38,14 +41,128 @@ Manager::~Manager() {
}
void Manager::InitPostScript() {
static auto get_message_thread_stats = []() -> const BucketedMessages* {
if ( ! thread_mgr->terminating ) {
double now = util::current_time();
if ( thread_mgr->bucketed_messages_last_updated < now - 1 ) {
thread_mgr->current_bucketed_messages.pending_in_total = 0;
thread_mgr->current_bucketed_messages.pending_out_total = 0;
for ( auto& m : thread_mgr->current_bucketed_messages.pending_in )
m.second = 0;
for ( auto& m : thread_mgr->current_bucketed_messages.pending_out )
m.second = 0;
MsgThread::Stats thread_stats;
for ( const auto& t : thread_mgr->msg_threads ) {
t->GetStats(&thread_stats);
thread_mgr->current_bucketed_messages.sent_in_total += thread_stats.sent_in;
thread_mgr->current_bucketed_messages.sent_out_total += thread_stats.sent_out;
thread_mgr->current_bucketed_messages.pending_in_total += thread_stats.pending_in;
thread_mgr->current_bucketed_messages.pending_out_total += thread_stats.pending_out;
for ( auto upper_limit : pending_bucket_brackets ) {
if ( thread_stats.pending_in < upper_limit ) {
thread_mgr->current_bucketed_messages.pending_in[upper_limit]++;
break;
}
}
for ( auto upper_limit : pending_bucket_brackets ) {
if ( thread_stats.pending_out < upper_limit ) {
thread_mgr->current_bucketed_messages.pending_out[upper_limit]++;
break;
}
}
}
thread_mgr->bucketed_messages_last_updated = now;
}
}
return &thread_mgr->current_bucketed_messages;
};
num_threads_metric =
telemetry_mgr->GaugeInstance("zeek", "active_threads", {}, "Number of active threads", "",
telemetry_mgr->GaugeInstance("zeek", "msgthread_active_threads", {}, "Number of active threads", "",
[]() -> prometheus::ClientMetric {
prometheus::ClientMetric metric;
metric.gauge.value =
thread_mgr ? static_cast<double>(thread_mgr->all_threads.size()) : 0.0;
return metric;
});
total_threads_metric = telemetry_mgr->CounterInstance("zeek", "msgthread_threads", {}, "Total number of threads");
total_messages_in_metric =
telemetry_mgr->CounterInstance("zeek", "msgthread_in_messages", {}, "Number of inbound messages received", "",
[]() -> prometheus::ClientMetric {
auto* s = get_message_thread_stats();
prometheus::ClientMetric metric;
metric.gauge.value = static_cast<double>(s->sent_in_total);
return metric;
});
total_messages_in_metric =
telemetry_mgr->CounterInstance("zeek", "msgthread_out_messages", {}, "Number of outbound messages sent", "",
[]() -> prometheus::ClientMetric {
auto* s = get_message_thread_stats();
prometheus::ClientMetric metric;
metric.gauge.value = static_cast<double>(s->sent_out_total);
return metric;
});
pending_messages_in_metric =
telemetry_mgr->GaugeInstance("zeek", "msgthread_pending_in_messages", {}, "Pending number of inbound messages",
"", []() -> prometheus::ClientMetric {
auto* s = get_message_thread_stats();
prometheus::ClientMetric metric;
metric.gauge.value = static_cast<double>(s->pending_in_total);
return metric;
});
pending_messages_out_metric =
telemetry_mgr->GaugeInstance("zeek", "msgthread_pending_out_messages", {},
"Pending number of outbound messages", "", []() -> prometheus::ClientMetric {
auto* s = get_message_thread_stats();
prometheus::ClientMetric metric;
metric.gauge.value = static_cast<double>(s->pending_out_total);
return metric;
});
pending_message_in_buckets_fam =
telemetry_mgr->GaugeFamily("zeek", "msgthread_pending_messages_in_buckets", {"lt"},
"Number of threads with pending inbound messages split into buckets");
pending_message_out_buckets_fam =
telemetry_mgr->GaugeFamily("zeek", "msgthread_pending_messages_out_buckets", {"lt"},
"Number of threads with pending outbound messages split into buckets");
for ( auto upper_limit : pending_bucket_brackets ) {
std::string upper_limit_str;
if ( upper_limit == std::numeric_limits<uint64_t>::infinity() )
upper_limit_str = "inf";
else
upper_limit_str = std::to_string(upper_limit);
current_bucketed_messages.pending_in[upper_limit] = 0;
current_bucketed_messages.pending_out[upper_limit] = 0;
pending_message_in_buckets[upper_limit] =
pending_message_in_buckets_fam->GetOrAdd({{"lt", upper_limit_str}},
[upper_limit]() -> prometheus::ClientMetric {
auto* s = get_message_thread_stats();
prometheus::ClientMetric metric;
metric.gauge.value =
static_cast<double>(s->pending_in.at(upper_limit));
return metric;
});
pending_message_out_buckets[upper_limit] =
pending_message_out_buckets_fam->GetOrAdd({{"lt", upper_limit_str}},
[upper_limit]() -> prometheus::ClientMetric {
auto* s = get_message_thread_stats();
prometheus::ClientMetric metric;
metric.gauge.value =
static_cast<double>(s->pending_out.at(upper_limit));
return metric;
});
}
}
void Manager::Terminate() {
@ -90,6 +207,8 @@ void Manager::AddThread(BasicThread* thread) {
if ( ! heartbeat_timer_running )
StartHeartbeatTimer();
total_threads_metric->Inc();
}
void Manager::AddMsgThread(MsgThread* thread) {