Mirror/zeek
1
0
Fork 0
mirror of https://github.com/zeek/zeek.git synced 2025-10-14 04:28:20 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

Merge remote-tracking branch 'origin/topic/jsiwek/gh-780-indefinite-log-buffer'

Browse source 
* origin/topic/jsiwek/gh-780-indefinite-log-buffer:
  GH-780: Prevent log batches from indefinite buffering
This commit is contained in:
Johanna Amann 2020-02-07 09:31:50 -08:00
commit 49a21d73d0
19 changed files with 237 additions and 23 deletions
Download patch file Download diff file Expand all files Collapse all files

9
src/broker/Manager.cc
View file

@ -131,7 +131,6 @@ Manager::Manager(bool arg_reading_pcaps)
after_zeek_init = false;
peer_count = 0;
log_batch_size = 0;
log_batch_interval = 0;
log_topic_func = nullptr;
vector_of_data_type = nullptr;
log_id_type = nullptr;
@ -147,7 +146,6 @@ void Manager::InitPostScript()
DBG_LOG(DBG_BROKER, "Initializing");
log_batch_size = get_option("Broker::log_batch_size")->AsCount();
log_batch_interval = get_option("Broker::log_batch_interval")->AsInterval();
default_log_topic_prefix =
get_option("Broker::default_log_topic_prefix")->AsString()->CheckString();
log_topic_func = get_option("Broker::log_topic")->AsFunc();
@ -580,8 +578,7 @@ bool Manager::PublishLogWrite(EnumVal* stream, EnumVal* writer, string path, int
auto& pending_batch = lb.msgs[topic];
pending_batch.emplace_back(msg.move_data());
if ( lb.message_count >= log_batch_size ||
(network_time - lb.last_flush >= log_batch_interval ) )
if ( lb.message_count >= log_batch_size )
statistics.num_logs_outgoing += lb.Flush(bstate->endpoint, log_batch_size);
return true;
@ -608,7 +605,6 @@ size_t Manager::LogBuffer::Flush(broker::endpoint& endpoint, size_t log_batch_si
}
auto rval = message_count;
last_flush = network_time;
message_count = 0;
return rval;
}
@ -619,8 +615,9 @@ size_t Manager::FlushLogBuffers()
auto rval = 0u;
for ( auto& lb : log_buffers )
rval += lb.Flush(bstate->endpoint, log_batch_interval);
rval += lb.Flush(bstate->endpoint, log_batch_size);
statistics.num_logs_outgoing += rval;
return rval;
}

2
src/broker/Manager.h
View file

@ -354,7 +354,6 @@ private:
struct LogBuffer {
// Indexed by topic string.
std::unordered_map<std::string, broker::vector> msgs;
double last_flush;
size_t message_count;
size_t Flush(broker::endpoint& endpoint, size_t batch_size);
@ -389,7 +388,6 @@ private:
int peer_count;
size_t log_batch_size;
double log_batch_interval;
Func* log_topic_func;
VectorType* vector_of_data_type;
EnumType* log_id_type;