Merge remote-tracking branch 'origin/topic/jgras/fix-cluster-started-btesting'

* origin/topic/jgras/fix-cluster-started-btesting:
  Make worker restart in cluster btest race-proof
This commit is contained in:
Arne Welzel 2024-03-06 10:38:55 +01:00
commit eeeaffb42c
3 changed files with 19 additions and 9 deletions

View file

@ -1,3 +1,7 @@
7.0.0-dev.45 | 2024-03-06 10:38:55 +0100
* Make worker restart in cluster btest race-proof (Jan Grashoefer, Corelight)
7.0.0-dev.43 | 2024-03-05 19:23:09 +0100 7.0.0-dev.43 | 2024-03-05 19:23:09 +0100
* highwayhash: Point to github.com/google/highwayhash, bump to master (Arne Welzel, Corelight) * highwayhash: Point to github.com/google/highwayhash, bump to master (Arne Welzel, Corelight)

View file

@ -1 +1 @@
7.0.0-dev.43 7.0.0-dev.45

View file

@ -20,25 +20,28 @@ redef exit_only_after_terminate=T;
redef Log::default_rotation_interval = 0secs; redef Log::default_rotation_interval = 0secs;
global topic = "test-topic"; global topic = "test-topic";
global restart_worker1_signal: event();
@if ( Supervisor::is_supervisor() ) @if ( Supervisor::is_supervisor() )
global supervisor_output_file: file; global supervisor_output_file: file;
global worker1_starts = 0; global worker1_restart_signals = 0;
event Cluster::Experimental::node_fully_connected(name: string, id: string, resending: bool) event restart_worker1_signal()
{ {
if ( name != "worker-1" ) # Wait for the signal to be raised twice, which means worker-1 is fully connected and
# the cluster is started.
if ( ++worker1_restart_signals < 2 )
return; return;
# Restart worker-1 twice # Shut down once we restarted worker-1 twice.
if ( ++worker1_starts >= 3 ) if ( worker1_restart_signals > 3 )
{ {
terminate(); terminate();
return; return;
} }
print supervisor_output_file, fmt("restarting %s", name); print supervisor_output_file, "restarting worker-1";
Supervisor::restart("worker-1"); Supervisor::restart("worker-1");
} }
@ -91,13 +94,16 @@ event Cluster::Experimental::node_fully_connected(name: string, id: string, rese
{ {
print "node fully connected"; print "node fully connected";
if ( Cluster::node == "manager" ) if ( Cluster::node == "manager" && name == "worker-1" )
Broker::publish(topic, Cluster::Experimental::node_fully_connected, name, id, resending); Broker::publish(topic, restart_worker1_signal);
} }
event Cluster::Experimental::cluster_started() event Cluster::Experimental::cluster_started()
{ {
print "cluster_started"; print "cluster_started";
if ( Cluster::node == "manager" )
Broker::publish(topic, restart_worker1_signal);
} }
event zeek_done() event zeek_done()