From b120f39bd7d83386b52c954959f4084fd9e6a448 Mon Sep 17 00:00:00 2001 From: Michael Dopheide Date: Thu, 24 Oct 2024 17:40:51 -0500 Subject: [PATCH 1/4] When auto-generating metrics ports for worker nodes, get them more uniform across instances. --- .../management/controller/main.zeek | 23 ++++++++++--------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/scripts/policy/frameworks/management/controller/main.zeek b/scripts/policy/frameworks/management/controller/main.zeek index efc603e227..4cf8a4ceef 100644 --- a/scripts/policy/frameworks/management/controller/main.zeek +++ b/scripts/policy/frameworks/management/controller/main.zeek @@ -414,15 +414,17 @@ function config_assign_metrics_ports(config: Management::Configuration) [Supervisor::WORKER] = 3, }; - local p = port_to_count(Management::Controller::auto_assign_metrics_start_port); - local ports_set: set[count]; + local instance_metrics_start_port: table[string] of count; + local instance_ports_set: table[string] of set[count]; local node: Management::Node; # Pre-populate agents ports, if we have them: for ( inst in config$instances ) { + instance_metrics_start_port[inst$name] = port_to_count(Management::Controller::auto_assign_metrics_start_port); + instance_ports_set[inst$name] = {}; if ( inst?$listen_port ) - add ports_set[port_to_count(inst$listen_port)]; + add instance_ports_set[inst$name][port_to_count(inst$listen_port)]; } # Pre-populate nodes with pre-defined metrics ports, as well @@ -430,11 +432,10 @@ function config_assign_metrics_ports(config: Management::Configuration) for ( node in config$nodes ) { if ( node?$p ) - add ports_set[port_to_count(node$p)]; - + add instance_ports_set[node$instance][port_to_count(node$p)]; if ( node?$metrics_port ) { - add ports_set[port_to_count(node$metrics_port)]; + add instance_ports_set[node$instance][port_to_count(node$metrics_port)]; add new_nodes[node]; } } @@ -468,15 +469,15 @@ function config_assign_metrics_ports(config: Management::Configuration) node = nodes[i]; # Find next available port ... - while ( p in ports_set ) - ++p; + while ( instance_metrics_start_port[node$instance] in instance_ports_set[node$instance] ) + ++instance_metrics_start_port[node$instance]; - node$metrics_port = count_to_port(p, tcp); + node$metrics_port = count_to_port(instance_metrics_start_port[node$instance], tcp); add new_nodes[node]; - add ports_set[p]; + add instance_ports_set[node$instance][instance_metrics_start_port[node$instance]]; # ... and consume it. - ++p; + ++instance_metrics_start_port[node$instance]; } config$nodes = new_nodes; From 0c0769b1b217adef3b1c9c8681e8b097628f5cec Mon Sep 17 00:00:00 2001 From: Michael Dopheide Date: Wed, 6 Nov 2024 14:03:54 -0600 Subject: [PATCH 2/4] Support multiple instances per host addr in auto metrics generation --- .../management/controller/main.zeek | 31 ++++++++++++------- 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/scripts/policy/frameworks/management/controller/main.zeek b/scripts/policy/frameworks/management/controller/main.zeek index 4cf8a4ceef..4a342fe02b 100644 --- a/scripts/policy/frameworks/management/controller/main.zeek +++ b/scripts/policy/frameworks/management/controller/main.zeek @@ -414,28 +414,34 @@ function config_assign_metrics_ports(config: Management::Configuration) [Supervisor::WORKER] = 3, }; - local instance_metrics_start_port: table[string] of count; - local instance_ports_set: table[string] of set[count]; + local instance_metrics_start_port: table[addr] of count; + local instance_ports_set: table[addr] of set[count]; + local instance_addr_lookup: table[string] of addr; local node: Management::Node; + local node_addr: addr; # Pre-populate agents ports, if we have them: for ( inst in config$instances ) { - instance_metrics_start_port[inst$name] = port_to_count(Management::Controller::auto_assign_metrics_start_port); - instance_ports_set[inst$name] = {}; + # build instance name -> addr lookup table + instance_addr_lookup[inst$name] = inst$host; + + instance_metrics_start_port[inst$host] = port_to_count(Management::Controller::auto_assign_metrics_start_port); + instance_ports_set[inst$host] = {}; if ( inst?$listen_port ) - add instance_ports_set[inst$name][port_to_count(inst$listen_port)]; + add instance_ports_set[inst$host][port_to_count(inst$listen_port)]; } # Pre-populate nodes with pre-defined metrics ports, as well # as their Broker ports: for ( node in config$nodes ) + node_addr = instance_addr_lookup[node$instance]; { if ( node?$p ) - add instance_ports_set[node$instance][port_to_count(node$p)]; + add instance_ports_set[node_addr][port_to_count(node$p)]; if ( node?$metrics_port ) { - add instance_ports_set[node$instance][port_to_count(node$metrics_port)]; + add instance_ports_set[node_addr][port_to_count(node$metrics_port)]; add new_nodes[node]; } } @@ -467,17 +473,18 @@ function config_assign_metrics_ports(config: Management::Configuration) for ( i in nodes ) { node = nodes[i]; + node_addr = instance_addr_lookup[node$instance]; # Find next available port ... - while ( instance_metrics_start_port[node$instance] in instance_ports_set[node$instance] ) - ++instance_metrics_start_port[node$instance]; + while ( instance_metrics_start_port[node_addr] in instance_ports_set[node_addr] ) + ++instance_metrics_start_port[node_addr]; - node$metrics_port = count_to_port(instance_metrics_start_port[node$instance], tcp); + node$metrics_port = count_to_port(instance_metrics_start_port[node_addr], tcp); add new_nodes[node]; - add instance_ports_set[node$instance][instance_metrics_start_port[node$instance]]; + add instance_ports_set[node_addr][instance_metrics_start_port[node_addr]]; # ... and consume it. - ++instance_metrics_start_port[node$instance]; + ++instance_metrics_start_port[node_addr]; } config$nodes = new_nodes; From ea88257d4dc6ff5114b9582f36a95355ce3a5844 Mon Sep 17 00:00:00 2001 From: Christian Kreibich Date: Thu, 30 Jan 2025 16:36:14 -0800 Subject: [PATCH 3/4] Management framework: move up addition of agent IPs into deployable cluster configs Since the changes to port autoassignment in the preceding commits leverage agent IP address information, we need to ensure that this information is available at the time of autoassignment. The controller learns IP addresses from connecting agents, and previously used that information at deploy time. This moves the augmentation of the cluster config up to port autoassignment time. --- .../management/controller/main.zeek | 39 ++++++++++--------- 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/scripts/policy/frameworks/management/controller/main.zeek b/scripts/policy/frameworks/management/controller/main.zeek index 4a342fe02b..b2407d532f 100644 --- a/scripts/policy/frameworks/management/controller/main.zeek +++ b/scripts/policy/frameworks/management/controller/main.zeek @@ -186,24 +186,6 @@ global g_configs: table[ConfigState] of Management::Configuration function config_deploy_to_agents(config: Management::Configuration, req: Management::Request::Request) { - # Make any final changes to the configuration we send off. - - # If needed, fill in agent IP address info as learned from their peerings. - # XXX this will need revisiting when we support host names. - local instances: set[Management::Instance]; - - for ( inst in config$instances ) - { - if ( inst$name in g_instances_known - && inst$host == 0.0.0.0 - && g_instances_known[inst$name]$host != 0.0.0.0 ) - inst$host = g_instances_known[inst$name]$host; - - add instances[inst]; - } - - config$instances = instances; - for ( name in g_instances ) { if ( name !in g_instances_ready ) @@ -1037,6 +1019,27 @@ event Management::Controller::API::stage_configuration_request(reqid: string, co g_configs[STAGED] = config; config_copy = copy(config); + # The staged config is preserved as the client sent it to us. For the + # ready-to-deploy version we fill in additional details here. + # + # One such bit of information is that we know the IP addresses of + # instances that connected to the controller from their Broker peering. + # + # XXX this will need revisiting when we support host names. + local instances: set[Management::Instance]; + + for ( inst in config_copy$instances ) + { + if ( inst$name in g_instances_known + && inst$host == 0.0.0.0 + && g_instances_known[inst$name]$host != 0.0.0.0 ) + inst$host = g_instances_known[inst$name]$host; + + add instances[inst]; + } + + config_copy$instances = instances; + if ( Management::Controller::auto_assign_broker_ports ) config_assign_broker_ports(config_copy); if ( Management::Controller::auto_assign_metrics_ports ) From 93eb99ca2e402d5b04e4fdcb6c856e3436cd95c9 Mon Sep 17 00:00:00 2001 From: Christian Kreibich Date: Fri, 31 Jan 2025 15:10:58 -0800 Subject: [PATCH 4/4] Bump cluster testsuite to pull in updated Prometheus tests --- testing/external/commit-hash.zeek-testing-cluster | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testing/external/commit-hash.zeek-testing-cluster b/testing/external/commit-hash.zeek-testing-cluster index c12bacc038..9aaa812bab 100644 --- a/testing/external/commit-hash.zeek-testing-cluster +++ b/testing/external/commit-hash.zeek-testing-cluster @@ -1 +1 @@ -43966c3a8c1a1a9d2cc3c77aebdbded602bf2cb3 +2f042c950d493a164fd2c7ef5172213bb1205642