Skip to content

Commit a4ff571

Browse files
feat: Support deploying Gateway LBs with region property on LoadBalancerConfiguration
Signed-off-by: Aleksander Aleksic <aleksander.aleksic@nordicsemi.no>
1 parent 250024d commit a4ff571

25 files changed

+1639
-71
lines changed

apis/gateway/v1beta1/loadbalancerconfig_types.go

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -198,6 +198,20 @@ type LoadBalancerConfigurationSpec struct {
198198
// +optional
199199
MergingMode *LoadBalancerConfigMergeMode `json:"mergingMode,omitempty"`
200200

201+
// region is the AWS region where the load balancer will be deployed. When unset, the controller's default region is used.
202+
// When set to a different region, vpcId, vpcSelector, or loadBalancerSubnets with identifiers must be set so the VPC can be resolved.
203+
// +optional
204+
Region *string `json:"region,omitempty"`
205+
206+
// vpcId is the VPC ID in the target region. Used when region is set (and especially when it differs from the controller default).
207+
// +optional
208+
VpcID *string `json:"vpcId,omitempty"`
209+
210+
// vpcSelector selects the VPC in the target region by tags. Each key is a tag name; the value list is the allowed tag values.
211+
// A VPC matches if it has each tag key with one of the corresponding values. Exactly one VPC must match in the target region.
212+
// +optional
213+
VpcSelector *map[string][]string `json:"vpcSelector,omitempty"`
214+
201215
// +kubebuilder:validation:MinLength=1
202216
// +kubebuilder:validation:MaxLength=32
203217
// loadBalancerName defines the name of the LB to provision. If unspecified, it will be automatically generated.

apis/gateway/v1beta1/zz_generated.deepcopy.go

Lines changed: 30 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

config/crd/gateway/gateway.k8s.aws_loadbalancerconfigurations.yaml

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -267,6 +267,12 @@ spec:
267267
- prefer-gateway
268268
- prefer-gateway-class
269269
type: string
270+
region:
271+
description: region is the AWS region where the load balancer will
272+
be deployed. When unset, the controller's default region is used.
273+
When set to a different region, vpcId, vpcSelector, or loadBalancerSubnets
274+
with identifiers must be set so the VPC can be resolved.
275+
type: string
270276
minimumLoadBalancerCapacity:
271277
description: MinimumLoadBalancerCapacity define the capacity reservation
272278
for LoadBalancers
@@ -317,6 +323,21 @@ spec:
317323
type: string
318324
description: Tags the AWS Tags on all related resources to the gateway.
319325
type: object
326+
vpcId:
327+
description: vpcId is the VPC ID in the target region. Used when
328+
region is set (and especially when it differs from the controller
329+
default).
330+
type: string
331+
vpcSelector:
332+
additionalProperties:
333+
items:
334+
type: string
335+
type: array
336+
description: vpcSelector selects the VPC in the target region by
337+
tags. Each key is a tag name; the value list is the allowed tag
338+
values. A VPC matches if it has each tag key with one of the corresponding
339+
values. Exactly one VPC must match in the target region.
340+
type: object
320341
wafV2:
321342
description: WAFv2 define the AWS WAFv2 settings for a Gateway [Application
322343
Load Balancer]

controllers/gateway/gateway_controller.go

Lines changed: 85 additions & 20 deletions
Large diffs are not rendered by default.

docs/guide/gateway/loadbalancerconfig.md

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,26 @@ Defines the LoadBalancer Scheme.
6565

6666
**Default** internal
6767

68+
#### Region
69+
70+
`region`
71+
72+
The AWS region where the load balancer will be deployed. When unset, the controller's default region (from `--aws-region` or environment) is used. When set to a different region, you must specify the VPC in that region using one of: `vpcId`, `vpcSelector`, or `loadBalancerSubnets` with subnet identifiers so the controller can resolve the VPC.
73+
74+
**Default** Controller's default region
75+
76+
#### VpcID
77+
78+
`vpcId`
79+
80+
The VPC ID in the target region. Used when `region` is set, especially when it differs from the controller default. Required (or use `vpcSelector` / `loadBalancerSubnets` with identifiers) when deploying to a non-default region.
81+
82+
#### VpcSelector
83+
84+
`vpcSelector`
85+
86+
Selects the VPC in the target region by tags. Same shape as `loadBalancerSubnetsSelector`: each key is a tag name, the value list is the allowed tag values. A VPC matches if it has each tag key with one of the corresponding values. Exactly one VPC must match in the target region. Use when `region` is set and you prefer tag-based selection over a fixed `vpcId`.
87+
6888
#### IpAddressType
6989

7090
`ipAddressType`

docs/guide/gateway/spec.md

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -460,6 +460,9 @@ _Appears in:_
460460
| Field | Description | Default | Validation |
461461
| --- | --- | --- | --- |
462462
| `mergingMode` _[LoadBalancerConfigMergeMode](#loadbalancerconfigmergemode)_ | mergingMode defines the merge behavior when both the Gateway and GatewayClass have a defined LoadBalancerConfiguration.<br />This field is only honored for the configuration attached to the GatewayClass. | | Enum: [prefer-gateway prefer-gateway-class] <br /> |
463+
| `region` _string_ | region is the AWS region where the load balancer will be deployed. When unset, the controller's default region is used. When set to a different region, set vpcId, vpcSelector, or loadBalancerSubnets with identifiers so the VPC can be resolved. | | |
464+
| `vpcId` _string_ | vpcId is the VPC ID in the target region. Used when region is set (especially when it differs from the controller default). | | |
465+
| `vpcSelector` _map[string][]string_ | vpcSelector selects the VPC in the target region by tags. Each key is a tag name; the value list is the allowed tag values. Exactly one VPC must match in the target region. | | |
463466
| `loadBalancerName` _string_ | loadBalancerName defines the name of the LB to provision. If unspecified, it will be automatically generated. | | MaxLength: 32 <br />MinLength: 1 <br /> |
464467
| `scheme` _[LoadBalancerScheme](#loadbalancerscheme)_ | scheme defines the type of LB to provision. If unspecified, it will be automatically inferred. | | Enum: [internal internet-facing] <br /> |
465468
| `ipAddressType` _[LoadBalancerIpAddressType](#loadbalanceripaddresstype)_ | loadBalancerIPType defines what kind of load balancer to provision (ipv4, dual stack) | | Enum: [ipv4 dualstack dualstack-without-public-ipv4] <br /> |
@@ -472,7 +475,6 @@ _Appears in:_
472475
| `securityGroups` _string_ | securityGroups an optional list of security group ids or names to apply to the LB | | |
473476
| `securityGroupPrefixes` _string_ | securityGroupPrefixes an optional list of prefixes that are allowed to access the LB. | | |
474477
| `sourceRanges` _string_ | sourceRanges an optional list of CIDRs that are allowed to access the LB. | | |
475-
| `vpcId` _string_ | vpcId is the ID of the VPC for the load balancer. | | |
476478
| `loadBalancerAttributes` _[LoadBalancerAttribute](#loadbalancerattribute) array_ | LoadBalancerAttributes defines the attribute of LB | | |
477479
| `tags` _map[string]string_ | Tags the AWS Tags on all related resources to the gateway. | | |
478480
| `enableICMP` _boolean_ | EnableICMP [Network LoadBalancer]<br />enables the creation of security group rules to the managed security group<br />to allow explicit ICMP traffic for Path MTU discovery for IPv4 and dual-stack VPCs | | |

helm/aws-load-balancer-controller/crds/gateway-crds.yaml

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -722,6 +722,12 @@ spec:
722722
- prefer-gateway
723723
- prefer-gateway-class
724724
type: string
725+
region:
726+
description: region is the AWS region where the load balancer will
727+
be deployed. When unset, the controller's default region is used.
728+
When set to a different region, vpcId, vpcSelector, or loadBalancerSubnets
729+
with identifiers must be set so the VPC can be resolved.
730+
type: string
725731
minimumLoadBalancerCapacity:
726732
description: MinimumLoadBalancerCapacity define the capacity reservation
727733
for LoadBalancers
@@ -772,6 +778,20 @@ spec:
772778
type: string
773779
description: Tags the AWS Tags on all related resources to the gateway.
774780
type: object
781+
vpcId:
782+
description: vpcId is the VPC ID in the target region. Used when region
783+
is set (and especially when it differs from the controller default).
784+
type: string
785+
vpcSelector:
786+
additionalProperties:
787+
items:
788+
type: string
789+
type: array
790+
description: vpcSelector selects the VPC in the target region by tags.
791+
Each key is a tag name; the value list is the allowed tag values.
792+
A VPC matches if it has each tag key with one of the corresponding
793+
values. Exactly one VPC must match in the target region.
794+
type: object
775795
wafV2:
776796
description: WAFv2 define the AWS WAFv2 settings for a Gateway [Application
777797
Load Balancer]

main.go

Lines changed: 33 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,8 @@ import (
3030
elbv2gw "sigs.k8s.io/aws-load-balancer-controller/apis/gateway/v1beta1"
3131
"sigs.k8s.io/aws-load-balancer-controller/controllers/gateway"
3232
"sigs.k8s.io/aws-load-balancer-controller/pkg/aws/services"
33+
"sigs.k8s.io/aws-load-balancer-controller/pkg/deploy"
34+
gatewaypkg "sigs.k8s.io/aws-load-balancer-controller/pkg/gateway"
3335
gateway_constants "sigs.k8s.io/aws-load-balancer-controller/pkg/gateway/constants"
3436
"sigs.k8s.io/aws-load-balancer-controller/pkg/gateway/referencecounter"
3537
"sigs.k8s.io/aws-load-balancer-controller/pkg/gateway/routeutils"
@@ -119,6 +121,9 @@ type gatewayControllerConfig struct {
119121
targetGroupCollector awsmetrics.TargetGroupCollector
120122
targetGroupARNMapper shared_utils.TargetGroupARNMapper
121123
certDiscovery certs.CertDiscovery
124+
cloudProvider gatewaypkg.CloudProvider
125+
stackDeployerFactoryALB func(cloud services.Cloud) deploy.StackDeployer
126+
stackDeployerFactoryNLB func(cloud services.Cloud) deploy.StackDeployer
122127
}
123128

124129
func main() {
@@ -198,7 +203,11 @@ func main() {
198203

199204
tgArnMapper := shared_utils.NewTargetGroupNameToArnMapper(cloud.ELBV2())
200205

201-
tgbResManager := targetgroupbinding.NewDefaultResourceManager(mgr.GetClient(), cloud.ELBV2(),
206+
elbv2ForRegion := func(region string) (services.ELBV2, error) {
207+
return aws.NewELBV2ForRegion(controllerCFG.AWSConfig, region, awsMetricsCollector, ctrl.Log.WithName("elbv2-"+region), aws.DefaultLbStabilizationTime)
208+
}
209+
210+
tgbResManager := targetgroupbinding.NewDefaultResourceManager(mgr.GetClient(), cloud.ELBV2(), cloud.Region(), elbv2ForRegion,
202211
podInfoRepo, networkingManager, vpcInfoProvider, multiClusterManager, lbcMetricsCollector,
203212
cloud.VpcID(), controllerCFG.FeatureGates.Enabled(config.EndpointsFailOpen), controllerCFG.EnableEndpointSlices,
204213
mgr.GetEventRecorderFor("targetGroupBinding"), ctrl.Log, controllerCFG.MaxTargetsPerTargetGroup)
@@ -263,6 +272,20 @@ func main() {
263272
serviceReferenceCounter := referencecounter.NewServiceReferenceCounter()
264273
certDiscovery := certs.NewACMCertDiscovery(cloud.ACM(), controllerCFG.IngressConfig.AllowedCertificateAuthorityARNs, ctrl.Log.WithName("gateway-cert-discovery"))
265274

275+
cloudProvider := gatewaypkg.NewDefaultCloudProvider(cloud, subnetResolver, vpcInfoProvider, controllerCFG.AWSConfig, controllerCFG, mgr.GetClient(), awsMetricsCollector, ctrl.Log.WithName("cloud-provider"))
276+
// Use region-scoped networking SG manager and reconciler per cloud so the deployer lists/finds SGs and reconciles ingress in the target region (fixes cross-region Duplicate and DescribeSecurityGroups NotFound).
277+
stackDeployerFactoryALB := func(c services.Cloud) deploy.StackDeployer {
278+
deployLogger := ctrl.Log.WithName("deploy").WithName(c.Region())
279+
networkingSGManagerForCloud := networking.NewDefaultSecurityGroupManager(c.EC2(), deployLogger)
280+
sgReconcilerForCloud := networking.NewDefaultSecurityGroupReconciler(networkingSGManagerForCloud, deployLogger)
281+
return deploy.NewDefaultStackDeployer(c, mgr.GetClient(), networkingManager, networkingSGManagerForCloud, sgReconcilerForCloud, elbv2deploy.NewDefaultTaggingManager(c.ELBV2(), c.VpcID(), controllerCFG.FeatureGates, c.RGT(), ctrl.Log), controllerCFG, gateway_constants.ALBGatewayTagPrefix, ctrl.Log, lbcMetricsCollector, gateway_constants.ALBGatewayController, true, targetGroupCollector, false)
282+
}
283+
stackDeployerFactoryNLB := func(c services.Cloud) deploy.StackDeployer {
284+
deployLogger := ctrl.Log.WithName("deploy").WithName(c.Region())
285+
networkingSGManagerForCloud := networking.NewDefaultSecurityGroupManager(c.EC2(), deployLogger)
286+
sgReconcilerForCloud := networking.NewDefaultSecurityGroupReconciler(networkingSGManagerForCloud, deployLogger)
287+
return deploy.NewDefaultStackDeployer(c, mgr.GetClient(), networkingManager, networkingSGManagerForCloud, sgReconcilerForCloud, elbv2deploy.NewDefaultTaggingManager(c.ELBV2(), c.VpcID(), controllerCFG.FeatureGates, c.RGT(), ctrl.Log), controllerCFG, gateway_constants.NLBGatewayTagPrefix, ctrl.Log, lbcMetricsCollector, gateway_constants.NLBGatewayController, true, targetGroupCollector, true)
288+
}
266289
gwControllerConfig := &gatewayControllerConfig{
267290
cloud: cloud,
268291
k8sClient: mgr.GetClient(),
@@ -282,6 +305,9 @@ func main() {
282305
targetGroupCollector: targetGroupCollector,
283306
targetGroupARNMapper: tgArnMapper,
284307
certDiscovery: certDiscovery,
308+
cloudProvider: cloudProvider,
309+
stackDeployerFactoryALB: stackDeployerFactoryALB,
310+
stackDeployerFactoryNLB: stackDeployerFactoryNLB,
285311
}
286312

287313
enabledControllers := sets.Set[string]{}
@@ -441,8 +467,8 @@ func main() {
441467
}
442468
corewebhook.NewServiceMutator(controllerCFG.ServiceConfig.LoadBalancerClass, ctrl.Log, lbcMetricsCollector).SetupWithManager(mgr)
443469
elbv2webhook.NewIngressClassParamsValidator(lbcMetricsCollector).SetupWithManager(mgr)
444-
elbv2webhook.NewTargetGroupBindingMutator(cloud.ELBV2(), ctrl.Log, lbcMetricsCollector).SetupWithManager(mgr)
445-
elbv2webhook.NewTargetGroupBindingValidator(mgr.GetClient(), cloud.ELBV2(), cloud.VpcID(), ctrl.Log, lbcMetricsCollector).SetupWithManager(mgr)
470+
elbv2webhook.NewTargetGroupBindingMutator(cloud.ELBV2(), cloud.Region(), elbv2ForRegion, ctrl.Log, lbcMetricsCollector).SetupWithManager(mgr)
471+
elbv2webhook.NewTargetGroupBindingValidator(mgr.GetClient(), cloud.ELBV2(), cloud.Region(), elbv2ForRegion, cloud.VpcID(), ctrl.Log, lbcMetricsCollector).SetupWithManager(mgr)
446472
networkingwebhook.NewIngressValidator(mgr.GetClient(), controllerCFG.IngressConfig, ctrl.Log, lbcMetricsCollector).SetupWithManager(mgr)
447473

448474
// Setup GlobalAccelerator validator only if enabled
@@ -514,6 +540,8 @@ func setupGatewayController(ctx context.Context, mgr ctrl.Manager, cfg *gatewayC
514540
cfg.reconcileCounters,
515541
cfg.targetGroupCollector,
516542
cfg.targetGroupARNMapper,
543+
cfg.cloudProvider,
544+
cfg.stackDeployerFactoryNLB,
517545
)
518546
case gateway_constants.ALBGatewayController:
519547
reconciler = gateway.NewALBGatewayReconciler(
@@ -538,6 +566,8 @@ func setupGatewayController(ctx context.Context, mgr ctrl.Manager, cfg *gatewayC
538566
cfg.reconcileCounters,
539567
cfg.targetGroupCollector,
540568
cfg.targetGroupARNMapper,
569+
cfg.cloudProvider,
570+
cfg.stackDeployerFactoryALB,
541571
)
542572
default:
543573
return fmt.Errorf("unknown controller type: %s", controllerType)

pkg/aws/region.go

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
package aws
2+
3+
import (
4+
"context"
5+
"time"
6+
7+
"github.com/aws/aws-sdk-go-v2/feature/ec2/imds"
8+
"github.com/go-logr/logr"
9+
"github.com/pkg/errors"
10+
epresolver "sigs.k8s.io/aws-load-balancer-controller/pkg/aws/endpoints"
11+
"sigs.k8s.io/aws-load-balancer-controller/pkg/aws/provider"
12+
"sigs.k8s.io/aws-load-balancer-controller/pkg/aws/services"
13+
aws_metrics "sigs.k8s.io/aws-load-balancer-controller/pkg/metrics/aws"
14+
)
15+
16+
// NewCloudForRegion creates a Cloud for the given region and vpcID without using EC2 metadata.
17+
// The base cfg is copied; only Region and VpcID are set. Use this when deploying to a non-default region.
18+
func NewCloudForRegion(cfg CloudConfig, region, vpcID string, clusterName string, metricsCollector *aws_metrics.Collector, logger logr.Logger, lbStabilizationTime time.Duration) (services.Cloud, error) {
19+
cfgForRegion := cfg
20+
cfgForRegion.Region = region
21+
cfgForRegion.VpcID = vpcID
22+
return NewCloud(cfgForRegion, clusterName, metricsCollector, logger, nil, lbStabilizationTime)
23+
}
24+
25+
// NewEC2ClientForRegion returns an EC2 client configured for the given region.
26+
// Used for VPC resolution (e.g. DescribeVpcs, DescribeSubnets) in that region before creating a full Cloud.
27+
func NewEC2ClientForRegion(cfg CloudConfig, region string, metricsCollector *aws_metrics.Collector, logger logr.Logger) (services.EC2, error) {
28+
awsClientsProvider, err := newClientsProviderForRegion(cfg, region, metricsCollector)
29+
if err != nil {
30+
return nil, err
31+
}
32+
return services.NewEC2(awsClientsProvider), nil
33+
}
34+
35+
// NewELBV2ForRegion returns an ELBV2 client configured for the given region.
36+
// Used by webhooks and model builders that need to describe/validate resources in a non-default region.
37+
// This does not require a VPC ID or full Cloud — only the region is needed for API calls like DescribeTargetGroups.
38+
func NewELBV2ForRegion(cfg CloudConfig, region string, metricsCollector *aws_metrics.Collector, logger logr.Logger, lbStabilizationTime time.Duration) (services.ELBV2, error) {
39+
awsClientsProvider, err := newClientsProviderForRegion(cfg, region, metricsCollector)
40+
if err != nil {
41+
return nil, err
42+
}
43+
cloud := &regionStubCloud{region: region}
44+
elbv2 := services.NewELBV2(awsClientsProvider, cloud, lbStabilizationTime)
45+
cloud.elbv2 = elbv2
46+
return elbv2, nil
47+
}
48+
49+
func newClientsProviderForRegion(cfg CloudConfig, region string, metricsCollector *aws_metrics.Collector) (provider.AWSClientsProvider, error) {
50+
cfgForRegion := cfg
51+
cfgForRegion.Region = region
52+
endpointsResolver := epresolver.NewResolver(cfgForRegion.AWSEndpoints)
53+
configGenerator := NewAWSConfigGenerator(cfgForRegion, imds.EndpointModeStateIPv4, metricsCollector)
54+
awsConfig, err := configGenerator.GenerateAWSConfig()
55+
if err != nil {
56+
return nil, err
57+
}
58+
return provider.NewDefaultAWSClientsProvider(awsConfig, endpointsResolver)
59+
}
60+
61+
// regionStubCloud is a minimal Cloud implementation used only by NewELBV2ForRegion.
62+
// Only Region() and ELBV2() are meaningful; other methods are unused by the ELBV2 client
63+
// for basic operations like DescribeTargetGroups.
64+
type regionStubCloud struct {
65+
region string
66+
elbv2 services.ELBV2
67+
}
68+
69+
func (c *regionStubCloud) Region() string { return c.region }
70+
func (c *regionStubCloud) VpcID() string { return "" }
71+
func (c *regionStubCloud) ELBV2() services.ELBV2 { return c.elbv2 }
72+
func (c *regionStubCloud) EC2() services.EC2 { return nil }
73+
func (c *regionStubCloud) ACM() services.ACM { return nil }
74+
func (c *regionStubCloud) WAFv2() services.WAFv2 { return nil }
75+
func (c *regionStubCloud) WAFRegional() services.WAFRegional { return nil }
76+
func (c *regionStubCloud) Shield() services.Shield { return nil }
77+
func (c *regionStubCloud) RGT() services.RGT { return nil }
78+
func (c *regionStubCloud) GlobalAccelerator() services.GlobalAccelerator { return nil }
79+
func (c *regionStubCloud) GetAssumedRoleELBV2(_ context.Context, _ string, _ string) (services.ELBV2, error) {
80+
return nil, errors.New("AssumeRole is not supported for cross-region stub cloud; use a full Cloud instead")
81+
}
82+
83+
var _ services.Cloud = &regionStubCloud{}

0 commit comments

Comments
 (0)