diff --git a/openshift-tests/ccm-aws-tests/go.mod b/openshift-tests/ccm-aws-tests/go.mod index bc14fcc74..3fe96a468 100644 --- a/openshift-tests/ccm-aws-tests/go.mod +++ b/openshift-tests/ccm-aws-tests/go.mod @@ -17,7 +17,7 @@ require ( k8s.io/api v0.36.0 k8s.io/apimachinery v0.36.0 k8s.io/client-go v0.36.0 - k8s.io/cloud-provider-aws/tests/e2e v0.0.0-20260507005622-746099eda51c + k8s.io/cloud-provider-aws/tests/e2e v0.0.0-20260606003233-c34d66ed717a k8s.io/kubernetes v1.36.0 k8s.io/pod-security-admission v0.36.0 ) diff --git a/openshift-tests/ccm-aws-tests/go.sum b/openshift-tests/ccm-aws-tests/go.sum index d6e3ecfa1..66e331587 100644 --- a/openshift-tests/ccm-aws-tests/go.sum +++ b/openshift-tests/ccm-aws-tests/go.sum @@ -275,8 +275,8 @@ k8s.io/client-go v0.36.0 h1:pOYi7C4RHChYjMiHpZSpSbIM6ZxVbRXBy7CuiIwqA3c= k8s.io/client-go v0.36.0/go.mod h1:ZKKcpwF0aLYfkHFCjillCKaTK/yBkEDHTDXCFY6AS9Y= k8s.io/cloud-provider v0.36.0 h1:PtiHsId1lBJixCbl5T+gUzbgOYAPschYj8tEAxxe0Ts= k8s.io/cloud-provider v0.36.0/go.mod h1:y/3sksoC0taJZR0PcAAYUqVyD6Jzu2X0lD4yCEPXPuI= -k8s.io/cloud-provider-aws/tests/e2e v0.0.0-20260507005622-746099eda51c h1:mXxJGVHCgE0diY1xgPg1mlmGuDhj2o8KzehT+YcOED8= -k8s.io/cloud-provider-aws/tests/e2e v0.0.0-20260507005622-746099eda51c/go.mod h1:BVLb2sMjVqNAN4t7bADRqtpdVA7brv8LcNas/kdKUJo= +k8s.io/cloud-provider-aws/tests/e2e v0.0.0-20260606003233-c34d66ed717a h1:3vIWSg8oFCQUal5G8Xj8m2hM7VrSJyqxaZkX1b6ICbM= +k8s.io/cloud-provider-aws/tests/e2e v0.0.0-20260606003233-c34d66ed717a/go.mod h1:vu0ofZopcm0LmGHdelw1LaP7oZdFZjgQ6k2fN+/MO5g= k8s.io/component-base v0.36.0 h1:hFjEktssxiJhrK1zfybkH4kJOi8iZuF+mIDCqS5+jRo= k8s.io/component-base v0.36.0/go.mod h1:JZvIfcNHk+uck+8LhJzhSBtydWXaZNQwX2OdL+Mnwsk= k8s.io/component-helpers v0.36.0 h1:KznLAOD7oPxjaeheW4SOQijz9UtMO8Nvp89+lR8FYks= diff --git a/openshift-tests/ccm-aws-tests/vendor/k8s.io/cloud-provider-aws/tests/e2e/cloudconfig.go b/openshift-tests/ccm-aws-tests/vendor/k8s.io/cloud-provider-aws/tests/e2e/cloudconfig.go new file mode 100644 index 000000000..fcb3d07fa --- /dev/null +++ b/openshift-tests/ccm-aws-tests/vendor/k8s.io/cloud-provider-aws/tests/e2e/cloudconfig.go @@ -0,0 +1,386 @@ +/* +Copyright 2026 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "context" + "fmt" + "path/filepath" + "strings" + "time" + + appsv1 "k8s.io/api/apps/v1" + v1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + clientset "k8s.io/client-go/kubernetes" + "k8s.io/kubernetes/test/e2e/framework" +) + +const ( + // ccmDaemonSetName is the name of the CCM DaemonSet in kube-system namespace. + ccmDaemonSetName = "aws-cloud-controller-manager" + // ccmNamespace is the namespace where CCM daemonset is deployed by kops. + ccmNamespace = "kube-system" + // ccmLabelSelector is the label selector to identify CCM pods. + ccmLabelSelector = "k8s-app=aws-cloud-controller-manager" + // tempConfigMapName is the name used for the injected config map with the modified cloud config. + tempConfigMapName = "aws-cloud-config-e2e" + // defaultRestartTimeout is the default timeout for waiting for CCM pods to restart and become ready after config changes. + defaultRestartTimeout = 3 * time.Minute + // pollInterval is the interval for polling pod readiness after CCM restart. + pollInterval = 5 * time.Second +) + +// cloudConfigManager helps to change the CCM cloud configuration for e2e tests. +// It is engineered to work with kops-provisioned clusters that run CI tests. +// It modifies the CCM cloud config at runtime by creating a temporary ConfigMap and +// patching the DaemonSet to use it instead of the original hostPath volume. +type cloudConfigManager struct { + // State tracking for restoration + originalVolume *v1.Volume + originalVolumeMount *v1.VolumeMount + configMapKey string + + // restartTimeout defines for how long to wait for CCM pods to become ready + // after configuration changes. + restartTimeout time.Duration +} + +// cloudConfigManagerOption is a functional option for configuring ccmCloudConfigManager. +type cloudConfigManagerOption func(*cloudConfigManager) + +// withRestartTimeout sets the timeout for waiting for CCM pods to become ready after configuration changes. +func withRestartTimeout(timeout time.Duration) cloudConfigManagerOption { + return func(m *cloudConfigManager) { + m.restartTimeout = timeout + } +} + +// newCloudConfigManager creates a new CCM cloud config manager with the provided options. +func newCloudConfigManager(opts ...cloudConfigManagerOption) *cloudConfigManager { + m := &cloudConfigManager{ + restartTimeout: defaultRestartTimeout, + } + + // Apply options + for _, opt := range opts { + opt(m) + } + + return m +} + +// setCloudConfig modifies the CCM cloud configuration with the provided content, storing the original +// configuration state for later restoration. +// +// Steps: +// - Finds the cloud-config path from CCM args +// - Creates a temporary ConfigMap with the desired config +// - Patches the CCM DaemonSet to use the ConfigMap instead of hostPath +// - Adds subPath to volumeMount to mount ConfigMap key as a file +// - Restarts CCM pods and waits for them to become ready +// +// Parameters: +// - ctx: Context for the operation +// - cs: Kubernetes clientset +// - configContent: The cloud config file content to set. +// +// Returns error if any step fails or if CCM is not using hostPath. +func (m *cloudConfigManager) setCloudConfig(ctx context.Context, cs clientset.Interface, configContent string) error { + framework.Logf("=== Setting CCM cloud configuration ===") + + // Get CCM DaemonSet + ds, err := cs.AppsV1().DaemonSets(ccmNamespace).Get(ctx, ccmDaemonSetName, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("failed to get CCM DaemonSet: %w", err) + } + + // Find cloud-config file path from CCM container args + if len(ds.Spec.Template.Spec.Containers) == 0 { + return fmt.Errorf("CCM DaemonSet has no containers") + } + cloudConfigPath := getCloudConfigPath(ds.Spec.Template.Spec.Containers[0]) + + // Check if CCM actually uses a cloud config. If not, return an error as it's an unsupported scenario. + if cloudConfigPath == "" { + return fmt.Errorf("CCM does not use --cloud-config flag") + } + framework.Logf("Cloud config path from CCM args: %s", cloudConfigPath) + + // Find the volume and volumeMount for cloud config + _, volumeIdx, err := m.findCloudConfigVolume(ds, cloudConfigPath) + if err != nil { + return err + } + + // Verify it's hostPath-based + if m.originalVolume.HostPath == nil { + return fmt.Errorf("CCM cloud config is not hostPath-based (only hostPath configs are supported)") + } + framework.Logf("Current config mount: HostPath=%s", m.originalVolume.HostPath.Path) + + // Create ConfigMap and patch DaemonSet. This also stores a copy of the original volume for later restoration. + if err := m.createConfigMapAndPatchDaemonSet(ctx, cs, ds, volumeIdx, configContent); err != nil { + return err + } + + // Restart CCM pods and wait for them to be ready + return restartCCMPods(ctx, cs, m.restartTimeout) +} + +// restoreCloudConfig restores the original CCM cloud configuration as saved by setCloudConfig. +// This restores the hostPath volume created by kops and deletes the temporary ConfigMap injected. +func (m *cloudConfigManager) restoreCloudConfig(ctx context.Context, cs clientset.Interface) error { + framework.Logf("=== Restoring original CCM configuration ===") + + ds, err := cs.AppsV1().DaemonSets(ccmNamespace).Get(ctx, ccmDaemonSetName, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("failed to get DaemonSet: %w", err) + } + + if m.originalVolume == nil { + return fmt.Errorf("no original volume to restore") + } + + // Restore original volume + for i, vol := range ds.Spec.Template.Spec.Volumes { + if vol.Name == m.originalVolume.Name { + ds.Spec.Template.Spec.Volumes[i] = *m.originalVolume + framework.Logf("Restored original volume: %s", m.originalVolume.Name) + break + } + } + + // Restore original volumeMount (remove subPath) + if m.originalVolumeMount != nil { + for i, container := range ds.Spec.Template.Spec.Containers { + for j, mount := range container.VolumeMounts { + if mount.Name == m.originalVolume.Name { + ds.Spec.Template.Spec.Containers[i].VolumeMounts[j] = *m.originalVolumeMount + framework.Logf("Restored original volumeMount") + break + } + } + } + } + + // Update DaemonSet + _, err = cs.AppsV1().DaemonSets(ccmNamespace).Update(ctx, ds, metav1.UpdateOptions{}) + if err != nil { + return fmt.Errorf("failed to restore DaemonSet: %w", err) + } + framework.Logf("Restored DaemonSet to use hostPath") + + // Delete temporary ConfigMap + err = cs.CoreV1().ConfigMaps(ccmNamespace).Delete(ctx, tempConfigMapName, metav1.DeleteOptions{}) + if err != nil && !apierrors.IsNotFound(err) { + return fmt.Errorf("failed to delete ConfigMap: %w", err) + } + framework.Logf("Deleted temporary ConfigMap %s", tempConfigMapName) + + // Restart CCM pods to load original config + return restartCCMPods(ctx, cs, m.restartTimeout) +} + +// getCloudConfigPath extracts the cloud config path from CCM container arguments. +// Returns empty string if --cloud-config flag is not found. +func getCloudConfigPath(container v1.Container) string { + for i, arg := range container.Args { + if arg == "--cloud-config" && i+1 < len(container.Args) { + return container.Args[i+1] + } else if strings.HasPrefix(arg, "--cloud-config=") { + return strings.TrimPrefix(arg, "--cloud-config=") + } + } + + return "" // No cloud-config flag found +} + +// findCloudConfigVolume locates the volume and volumeMount for the cloud config in the CCM DaemonSet. +// It returns the volume name, volume index, and ConfigMap key. +func (m *cloudConfigManager) findCloudConfigVolume(ds *appsv1.DaemonSet, cloudConfigPath string) (volumeName string, volumeIdx int, err error) { + if len(ds.Spec.Template.Spec.Containers) == 0 { + return "", -1, fmt.Errorf("CCM DaemonSet has no containers") + } + container := ds.Spec.Template.Spec.Containers[0] + + // Find volumeMount that matches the cloud config path + for _, mount := range container.VolumeMounts { + if mount.MountPath == cloudConfigPath { + volumeName = mount.Name + // If SubPath is set, that's the ConfigMap key; otherwise use basename of path + if mount.SubPath != "" { + m.configMapKey = mount.SubPath + } else { + m.configMapKey = filepath.Base(cloudConfigPath) + } + framework.Logf("Found cloud config volumeMount: name=%s, mountPath=%s, subPath=%s, key=%s", + volumeName, mount.MountPath, mount.SubPath, m.configMapKey) + break + } + } + if volumeName == "" { + return "", -1, fmt.Errorf("cloud config volumeMount not found for path %s", cloudConfigPath) + } + + // Find the volume by name + for i, vol := range ds.Spec.Template.Spec.Volumes { + if vol.Name == volumeName { + volumeIdx = i + m.originalVolume = vol.DeepCopy() + framework.Logf("Found cloud config volume: name=%s, type=%s", vol.Name, getVolumeType(vol)) + return volumeName, volumeIdx, nil + } + } + + return "", -1, fmt.Errorf("cloud config volume not found for name %s", volumeName) +} + +// createConfigMapAndPatchDaemonSet creates a new ConfigMap with cloud config and patches +// the CCM DaemonSet to use it instead of hostPath. +func (m *cloudConfigManager) createConfigMapAndPatchDaemonSet(ctx context.Context, cs clientset.Interface, ds *appsv1.DaemonSet, volumeIdx int, configContent string) error { + framework.Logf("Creating temporary ConfigMap and patching DaemonSet") + + // Create ConfigMap with the detected key + cm := &v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: tempConfigMapName, + Namespace: ccmNamespace, + }, + Data: map[string]string{ + m.configMapKey: configContent, + }, + } + _, err := cs.CoreV1().ConfigMaps(ccmNamespace).Create(ctx, cm, metav1.CreateOptions{}) + if err != nil { + if apierrors.IsAlreadyExists(err) { + // ConfigMap already exists, update it + framework.Logf("ConfigMap %s already exists, updating...", tempConfigMapName) + _, err = cs.CoreV1().ConfigMaps(ccmNamespace).Update(ctx, cm, metav1.UpdateOptions{}) + if err != nil { + return fmt.Errorf("failed to update existing ConfigMap: %w", err) + } + framework.Logf("Updated ConfigMap %s with key '%s'", tempConfigMapName, m.configMapKey) + } else { + return fmt.Errorf("failed to create ConfigMap: %w", err) + } + } else { + framework.Logf("Created ConfigMap %s with key '%s'", tempConfigMapName, m.configMapKey) + } + + // Patch DaemonSet volume to use ConfigMap + ds.Spec.Template.Spec.Volumes[volumeIdx] = v1.Volume{ + Name: m.originalVolume.Name, + VolumeSource: v1.VolumeSource{ + ConfigMap: &v1.ConfigMapVolumeSource{ + LocalObjectReference: v1.LocalObjectReference{ + Name: tempConfigMapName, + }, + }, + }, + } + + // Find and patch volumeMount to use subPath + // ConfigMap mounts as directory, hostPath as file + // Without subPath, /etc/kubernetes/cloud.config becomes a directory + for i, container := range ds.Spec.Template.Spec.Containers { + for j, mount := range container.VolumeMounts { + if mount.Name == m.originalVolume.Name { + // Save original volumeMount for rollback + m.originalVolumeMount = mount.DeepCopy() + // Add subPath to mount ConfigMap key as file + ds.Spec.Template.Spec.Containers[i].VolumeMounts[j].SubPath = m.configMapKey + framework.Logf("Added subPath=%s to volumeMount", m.configMapKey) + break + } + } + } + + _, err = cs.AppsV1().DaemonSets(ccmNamespace).Update(ctx, ds, metav1.UpdateOptions{}) + if err != nil { + return fmt.Errorf("failed to update DaemonSet: %w", err) + } + framework.Logf("Patched DaemonSet to use ConfigMap") + + return nil +} + +// restartCCMPods restarts all CCM pods and waits for them to become ready. +func restartCCMPods(ctx context.Context, cs clientset.Interface, timeout time.Duration) error { + framework.Logf("Restarting CCM pods") + + ccmPods, err := cs.CoreV1().Pods(ccmNamespace).List(ctx, metav1.ListOptions{ + LabelSelector: ccmLabelSelector, + }) + if err != nil { + return fmt.Errorf("failed to list CCM pods: %w", err) + } + + for _, pod := range ccmPods.Items { + err = cs.CoreV1().Pods(ccmNamespace).Delete(ctx, pod.Name, metav1.DeleteOptions{}) + // Ignore NotFound errors - pod may already be deleting due to DaemonSet update + if err != nil && !apierrors.IsNotFound(err) { + return fmt.Errorf("failed to delete pod %s: %w", pod.Name, err) + } + } + + // Wait for new pods to be Running AND Ready + framework.Logf("Waiting for CCM pods to become ready (timeout: %v)", timeout) + err = wait.PollUntilContextTimeout(ctx, pollInterval, timeout, true, func(ctx context.Context) (bool, error) { + pods, err := cs.CoreV1().Pods(ccmNamespace).List(ctx, metav1.ListOptions{ + LabelSelector: ccmLabelSelector, + }) + if err != nil || len(pods.Items) == 0 { + return false, nil + } + for _, pod := range pods.Items { + if pod.Status.Phase != v1.PodRunning { + return false, nil + } + // Check container ready status + ready := false + for _, condition := range pod.Status.Conditions { + if condition.Type == v1.PodReady && condition.Status == v1.ConditionTrue { + ready = true + break + } + } + if !ready { + return false, nil + } + } + return true, nil + }) + if err != nil { + return fmt.Errorf("CCM pods did not become ready within %v: %w", timeout, err) + } + framework.Logf("CCM restarted successfully") + + return nil +} + +// getVolumeType returns a string describing the type of volume. +func getVolumeType(vol v1.Volume) string { + if vol.ConfigMap != nil { + return "ConfigMap" + } else if vol.HostPath != nil { + return "HostPath" + } + return "Unknown" +} diff --git a/openshift-tests/ccm-aws-tests/vendor/k8s.io/cloud-provider-aws/tests/e2e/loadbalancer.go b/openshift-tests/ccm-aws-tests/vendor/k8s.io/cloud-provider-aws/tests/e2e/loadbalancer.go index 217706061..ffd4ee169 100644 --- a/openshift-tests/ccm-aws-tests/vendor/k8s.io/cloud-provider-aws/tests/e2e/loadbalancer.go +++ b/openshift-tests/ccm-aws-tests/vendor/k8s.io/cloud-provider-aws/tests/e2e/loadbalancer.go @@ -36,6 +36,8 @@ import ( "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/aws/retry" "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/ec2" + ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types" elbv2 "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2" elbv2types "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2/types" ) @@ -45,6 +47,7 @@ const ( annotationLBInternal = "service.beta.kubernetes.io/aws-load-balancer-internal" annotationLBTargetNodeLabels = "service.beta.kubernetes.io/aws-load-balancer-target-node-labels" annotationLBTargetGroupAttributes = "service.beta.kubernetes.io/aws-load-balancer-target-group-attributes" + annotationLBSecurityGroups = "service.beta.kubernetes.io/aws-load-balancer-security-groups" ) var ( @@ -56,354 +59,647 @@ var ( } ) -// loadbalancer tests -var _ = Describe("[cloud-provider-aws-e2e] loadbalancer", func() { - f := framework.NewDefaultFramework("cloud-provider-aws") - f.NamespacePodSecurityEnforceLevel = admissionapi.LevelPrivileged +// loadBalancerTestCases defines the structure for different load balancer test scenarios, +// including configuration, hooks for dynamic behavior, and test verification controls. +type loadBalancerTestCases struct { + // Overall test case configuration. + name string + resourceSuffix string + extraAnnotations map[string]string + listenerCount int + + // Hooks + // HookPostServiceConfig hook runs after the service manifest is created, and before the service is created. + hookPostServiceConfig func(cfg *e2eTestConfig) + // HookPostServiceCreate hook runs after the test is run. + hookPostServiceCreate func(cfg *e2eTestConfig) + // HookPreTest hook runs before the test is run. + hookPreTest func(cfg *e2eTestConfig) + + // Flags to override default test behavior. + overrideTestRunInClusterReachableHTTP bool + requireAffinity bool + + // Test verification + skipTestFailure bool +} - var ( - cs clientset.Interface - ns *v1.Namespace - ) +// runTestCase is a helper function that executes a load balancer test case +func runTestCase(ctx context.Context, tc loadBalancerTestCases, cs clientset.Interface, ns *v1.Namespace) { + By("setting up test environment and discovering worker nodes") + e2e := newE2eTestConfig(cs) + e2e.discoverClusterWorkerNode() + framework.Logf("[SETUP] Test case: %s", tc.name) + framework.Logf("[SETUP] Worker nodes discovered: %d nodes, selector: %s, sample node: %s", e2e.nodeCount, e2e.nodeSelector, e2e.nodeSingleSample) - BeforeEach(func() { - cs = f.ClientSet - ns = f.Namespace - }) + loadBalancerCreateTimeout := e2eservice.GetServiceLoadBalancerCreationTimeout(ctx, cs) + framework.Logf("[CONFIG] AWS load balancer timeout: %s", loadBalancerCreateTimeout) - AfterEach(func() { - // After each test + By("building service configuration with annotations") + + serviceName := "lbconfig-test" + if len(tc.resourceSuffix) > 0 { + serviceName = serviceName + "-" + tc.resourceSuffix + } + framework.Logf("[CONFIG] Service name: %s, namespace: %s", serviceName, ns.Name) + e2e.LBJig = e2eservice.NewTestJig(cs, ns.Name, serviceName) + + // Hook annotations to support dynamic config + e2e.svc = e2e.buildService(tc.listenerCount, tc.extraAnnotations) + framework.Logf("[CONFIG] Service ports: %d, extra annotations: %v", len(e2e.svc.Spec.Ports), tc.extraAnnotations) + + if tc.hookPostServiceConfig != nil { + By("executing hook post-service-config: applying service configuration") + framework.Logf("[HOOK] Executing post-service-config hook") + tc.hookPostServiceConfig(e2e) + framework.Logf("[HOOK] Final service annotations: %v", e2e.svc.Annotations) + } + + By("creating LoadBalancer service in Kubernetes") + if _, err := e2e.LBJig.Client.CoreV1().Services(e2e.LBJig.Namespace).Create(context.TODO(), e2e.svc, metav1.CreateOptions{}); err != nil { + framework.ExpectNoError(fmt.Errorf("failed to create LoadBalancer Service %q: %v", e2e.svc.Name, err)) + } + framework.Logf("[K8S] LoadBalancer service created successfully") + + By("waiting for AWS load balancer provisioning") + var err error + e2e.svc, err = e2e.LBJig.WaitForLoadBalancer(ctx, loadBalancerCreateTimeout) + // Collect comprehensive debugging information when LoadBalancer provisioning fails + if err != nil { + serviceName := e2e.LBJig.Name + if e2e.svc != nil { + serviceName = e2e.svc.Name + } + framework.Logf("ERROR: LoadBalancer provisioning failed for service %q: %v", serviceName, err) + framework.Logf("ERROR: LoadBalancer provisioning timeout reached after %v", loadBalancerCreateTimeout) + + // Ensure we have detailed debugging information before failing + framework.Logf("=== LoadBalancer Provisioning Failure Debug Information ===") + gatherEventosOnFailure(e2e.ctx, e2e.kubeClient, e2e.LBJig.Namespace, e2e.LBJig.Name) + framework.Logf("=== End of LoadBalancer Provisioning Failure Debug Information ===") + + // Fail the test immediately to prevent further execution + framework.ExpectNoError(err, "LoadBalancer provisioning failed - check debug information above") + } + framework.Logf("[AWS] Load balancer provisioned successfully") + + By("creating backend server pods") + _, err = e2e.LBJig.Run(ctx, e2e.buildDeployment(tc.requireAffinity)) + if err != nil { + serviceName := e2e.LBJig.Name + if e2e.svc != nil { + serviceName = e2e.svc.Name + } + framework.Logf("ERROR: LoadBalancer provisioning failed for service %q: %v", serviceName, err) + framework.Logf("ERROR: LoadBalancer provisioning timeout reached after %v", loadBalancerCreateTimeout) + + // Ensure we have detailed debugging information before failing + framework.Logf("=== LoadBalancer Provisioning Failure Debug Information ===") + gatherEventosOnFailure(e2e.ctx, e2e.kubeClient, e2e.LBJig.Namespace, e2e.LBJig.Name) + framework.Logf("=== End of LoadBalancer Provisioning Failure Debug Information ===") + + // Fail the test immediately to prevent further execution + framework.ExpectNoError(err, "LoadBalancer provisioning failed - check debug information above") + } + + framework.Logf("[K8S] Backend pods created, affinity required: %t", tc.requireAffinity) + + if tc.hookPostServiceCreate != nil { + By("executing hook post-service-create: applying service configuration") + tc.hookPostServiceCreate(e2e) + } + + By("collecting service and load balancer information") + if e2e.svc == nil { + framework.Logf("=== Service Validation Error Debug Information ===") + gatherEventosOnFailure(e2e.ctx, e2e.kubeClient, e2e.LBJig.Namespace, e2e.LBJig.Name) + framework.Logf("=== End of Service Validation Error Debug Information ===") + framework.Failf("Service is nil after LoadBalancer provisioning for service %s", e2e.LBJig.Name) + } + if len(e2e.svc.Spec.Ports) == 0 { + framework.Logf("=== Service Ports Error Debug Information ===") + framework.Logf("Service spec: %+v", e2e.svc.Spec) + gatherEventosOnFailure(e2e.ctx, e2e.kubeClient, e2e.LBJig.Namespace, e2e.LBJig.Name) + framework.Logf("=== End of Service Ports Error Debug Information ===") + framework.Failf("No ports found in service spec for service %s/%s", e2e.svc.Namespace, e2e.svc.Name) + } + if len(e2e.svc.Status.LoadBalancer.Ingress) == 0 { + framework.Logf("=== LoadBalancer Ingress Error Debug Information ===") + framework.Logf("Service status: %+v", e2e.svc.Status) + gatherEventosOnFailure(e2e.ctx, e2e.kubeClient, e2e.LBJig.Namespace, e2e.LBJig.Name) + framework.Logf("=== End of LoadBalancer Ingress Error Debug Information ===") + framework.Failf("No ingress found in LoadBalancer status for service %s/%s", e2e.svc.Namespace, e2e.svc.Name) + } + + svcPort := int(e2e.svc.Spec.Ports[0].Port) + ingressAddress := e2eservice.GetIngressPoint(&e2e.svc.Status.LoadBalancer.Ingress[0]) + framework.Logf("[LB-INFO] Ingress address: %s, port: %d", ingressAddress, svcPort) + + if ingressAddress == "" { + framework.Logf("=== Empty Ingress Address Debug Information ===") + framework.Logf("LoadBalancer ingress[0]: %+v", e2e.svc.Status.LoadBalancer.Ingress[0]) + gatherEventosOnFailure(e2e.ctx, e2e.kubeClient, e2e.LBJig.Namespace, e2e.LBJig.Name) + framework.Logf("=== End of Empty Ingress Address Debug Information ===") + framework.Failf("LoadBalancer ingress address is empty for service %s/%s", e2e.svc.Namespace, e2e.svc.Name) + } + + if tc.hookPreTest != nil { + By("executing pre-test hook") + tc.hookPreTest(e2e) + } + + // overrideTestRunInClusterReachableHTTP changes the default test function to run the client in the cluster. + if tc.overrideTestRunInClusterReachableHTTP { + By("testing HTTP connectivity for internal load balancer") + framework.Logf("[TEST] Running internal connectivity test from node: %s", e2e.nodeSingleSample) + err := inClusterTestReachableHTTP(cs, ns.Name, e2e.nodeSingleSample, ingressAddress, svcPort) + if err != nil && tc.skipTestFailure { + Skip(err.Error()) + } + framework.ExpectNoError(err) + } else { + By("testing HTTP connectivity for external/internet-facing load balancer") + framework.Logf("[TEST] Running external connectivity test to %s:%d", ingressAddress, svcPort) + e2eservice.TestReachableHTTP(ctx, ingressAddress, svcPort, e2eservice.LoadBalancerLagTimeoutAWS) + } + framework.Logf("[TEST] HTTP connectivity test completed successfully") + + // Update the service to cluster IP + By("cleaning up: converting service to ClusterIP") + _, err = e2e.LBJig.UpdateService(ctx, func(s *v1.Service) { + s.Spec.Type = v1.ServiceTypeClusterIP }) + framework.ExpectNoError(err) + + // Wait for the load balancer to be destroyed asynchronously + By("cleaning up: waiting for load balancer destruction") + framework.Logf("[CLEANUP] Waiting for load balancer destruction") + _, err = e2e.LBJig.WaitForLoadBalancerDestroy(ctx, ingressAddress, svcPort, loadBalancerCreateTimeout) + framework.ExpectNoError(err) + framework.Logf("[CLEANUP] Load balancer destroyed successfully") +} - type loadBalancerTestCases struct { - // Overall test case configuration. - name string - resourceSuffix string - extraAnnotations map[string]string - listenerCount int - - // Hooks - // HookPostServiceConfig hook runs after the service manifest is created, and before the service is created. - hookPostServiceConfig func(cfg *e2eTestConfig) - // HookPostServiceCreate hook runs after the test is run. - hookPostServiceCreate func(cfg *e2eTestConfig) - // HookPreTest hook runs before the test is run. - hookPreTest func(cfg *e2eTestConfig) - - // Flags to override default test behavior. - overrideTestRunInClusterReachableHTTP bool - requireAffinity bool - - // Test verification - skipTestFailure bool - } - cases := []loadBalancerTestCases{ - { - name: "CLB should be reachable with default configurations", - resourceSuffix: "", - extraAnnotations: map[string]string{}, +// clbTests: Tests related to CLB load balancers +var clbTests = []loadBalancerTestCases{ + { + name: "CLB should be reachable with default configurations", + resourceSuffix: "", + extraAnnotations: map[string]string{}, + }, + // Hairpining traffic test for CLB. + { + name: "CLB internal should be reachable with hairpinning traffic", + resourceSuffix: "hp-clb-int", + extraAnnotations: map[string]string{ + annotationLBInternal: "true", }, - { - name: "NLB should be reachable with default configurations", - resourceSuffix: "nlb", - extraAnnotations: map[string]string{annotationLBType: "nlb"}, + hookPostServiceConfig: func(cfg *e2eTestConfig) { + framework.Logf("running hook post-service-config patching service annotations to enforce LB pins/selects target to a single node: kubernetes.io/hostname=%s", cfg.nodeSingleSample) + if cfg.svc.Annotations == nil { + cfg.svc.Annotations = map[string]string{} + } + cfg.svc.Annotations[annotationLBTargetNodeLabels] = fmt.Sprintf("kubernetes.io/hostname=%s", cfg.nodeSingleSample) }, - { - name: "NLB should be reachable with target-node-labels", - resourceSuffix: "sg-nd", - extraAnnotations: map[string]string{annotationLBType: "nlb"}, - hookPostServiceConfig: func(cfg *e2eTestConfig) { - framework.Logf("running hook post-service-config patching service annotations to test node label selector") - if cfg.svc.Annotations == nil { - cfg.svc.Annotations = map[string]string{} - } - cfg.svc.Annotations[annotationLBTargetNodeLabels] = cfg.nodeSelector - }, - hookPostServiceCreate: func(cfg *e2eTestConfig) { - framework.Logf("running hook post-service-create to validate the number of targets in the load balancer selected") - if len(cfg.svc.Status.LoadBalancer.Ingress) == 0 { - framework.Failf("No ingress found in LoadBalancer status for service %s/%s", cfg.svc.Namespace, cfg.svc.Name) - } - lbDNS := cfg.svc.Status.LoadBalancer.Ingress[0].Hostname - framework.ExpectNoError(getLBTargetCount(cfg.ctx, lbDNS, cfg.nodeCount), "AWS LB target count validation failed") - }, + overrideTestRunInClusterReachableHTTP: true, + requireAffinity: true, + }, +} + +// nlbTests: Generic NLB tests that test NLB functionality and are not specific to a security group configuration layout +var nlbTests = []loadBalancerTestCases{ + { + name: "NLB should be reachable with default configurations", + resourceSuffix: "nlb", + extraAnnotations: map[string]string{annotationLBType: "nlb"}, + }, + { + name: "NLB should be reachable with target-node-labels", + resourceSuffix: "sg-nd", + extraAnnotations: map[string]string{annotationLBType: "nlb"}, + hookPostServiceConfig: func(cfg *e2eTestConfig) { + framework.Logf("running hook post-service-config patching service annotations to test node label selector") + if cfg.svc.Annotations == nil { + cfg.svc.Annotations = map[string]string{} + } + cfg.svc.Annotations[annotationLBTargetNodeLabels] = cfg.nodeSelector }, - // Hairpining traffic test for CLB. - { - name: "CLB internal should be reachable with hairpinning traffic", - resourceSuffix: "hp-clb-int", - extraAnnotations: map[string]string{ - annotationLBInternal: "true", - }, - hookPostServiceConfig: func(cfg *e2eTestConfig) { - framework.Logf("running hook post-service-config patching service annotations to enforce LB pins/selects target to a single node: kubernetes.io/hostname=%s", cfg.nodeSingleSample) - if cfg.svc.Annotations == nil { - cfg.svc.Annotations = map[string]string{} - } - cfg.svc.Annotations[annotationLBTargetNodeLabels] = fmt.Sprintf("kubernetes.io/hostname=%s", cfg.nodeSingleSample) - }, - overrideTestRunInClusterReachableHTTP: true, - requireAffinity: true, + hookPostServiceCreate: func(cfg *e2eTestConfig) { + framework.Logf("running hook post-service-create to validate the number of targets in the load balancer selected") + if len(cfg.svc.Status.LoadBalancer.Ingress) == 0 { + framework.Failf("No ingress found in LoadBalancer status for service %s/%s", cfg.svc.Namespace, cfg.svc.Name) + } + lbDNS := cfg.svc.Status.LoadBalancer.Ingress[0].Hostname + framework.ExpectNoError(getLBTargetCount(cfg.ctx, lbDNS, cfg.nodeCount), "AWS LB target count validation failed") }, - // Hairpining traffic test for NLB. - // The target type instance (default) sets the preserve client IP attribute to true, - // the NLB target group attributes are set to preserve_client_ip.enabled=false to allow hairpining traffic. - // The test also validates the target group attributes are set correctly to AWS resource. - { - name: "NLB internal should be reachable with hairpinning traffic", - resourceSuffix: "hp-nlb-int", - extraAnnotations: map[string]string{ - annotationLBType: "nlb", - annotationLBInternal: "true", - annotationLBTargetGroupAttributes: "preserve_client_ip.enabled=false", - }, - listenerCount: 1, - overrideTestRunInClusterReachableHTTP: true, - requireAffinity: true, - hookPostServiceConfig: func(cfg *e2eTestConfig) { - framework.Logf("running hook post-service-config patching service annotations to enforce LB pins/selects target to a single node: kubernetes.io/hostname=%s", cfg.nodeSingleSample) - if cfg.svc.Annotations == nil { - cfg.svc.Annotations = map[string]string{} - } - cfg.svc.Annotations[annotationLBTargetNodeLabels] = fmt.Sprintf("kubernetes.io/hostname=%s", cfg.nodeSingleSample) - }, - hookPreTest: func(e2e *e2eTestConfig) { - framework.Logf("running hook pre-test: verify target group attributes are set correctly to AWS resource") + }, + // Hairpining traffic test for NLB. + // The target type instance (default) sets the preserve client IP attribute to true, + // the NLB target group attributes are set to preserve_client_ip.enabled=false to allow hairpining traffic. + // The test also validates the target group attributes are set correctly to AWS resource. + { + name: "NLB internal should be reachable with hairpinning traffic", + resourceSuffix: "hp-nlb-int", + extraAnnotations: map[string]string{ + annotationLBType: "nlb", + annotationLBInternal: "true", + annotationLBTargetGroupAttributes: "preserve_client_ip.enabled=false", + }, + listenerCount: 1, + overrideTestRunInClusterReachableHTTP: true, + requireAffinity: true, + hookPostServiceConfig: func(cfg *e2eTestConfig) { + framework.Logf("running hook post-service-config patching service annotations to enforce LB pins/selects target to a single node: kubernetes.io/hostname=%s", cfg.nodeSingleSample) + if cfg.svc.Annotations == nil { + cfg.svc.Annotations = map[string]string{} + } + cfg.svc.Annotations[annotationLBTargetNodeLabels] = fmt.Sprintf("kubernetes.io/hostname=%s", cfg.nodeSingleSample) + }, + hookPreTest: func(e2e *e2eTestConfig) { + framework.Logf("running hook pre-test: verify target group attributes are set correctly to AWS resource") - if e2e.svc.Status.LoadBalancer.Ingress[0].Hostname == "" && e2e.svc.Status.LoadBalancer.Ingress[0].IP == "" { - framework.Failf("LoadBalancer ingress is empty (no hostname or IP) for service %s/%s", e2e.svc.Namespace, e2e.svc.Name) - } + if e2e.svc.Status.LoadBalancer.Ingress[0].Hostname == "" && e2e.svc.Status.LoadBalancer.Ingress[0].IP == "" { + framework.Failf("LoadBalancer ingress is empty (no hostname or IP) for service %s/%s", e2e.svc.Namespace, e2e.svc.Name) + } - hostAddr := e2eservice.GetIngressPoint(&e2e.svc.Status.LoadBalancer.Ingress[0]) - framework.Logf("Load balancer's ingress address: %s", hostAddr) + hostAddr := e2eservice.GetIngressPoint(&e2e.svc.Status.LoadBalancer.Ingress[0]) + framework.Logf("Load balancer's ingress address: %s", hostAddr) - if hostAddr == "" { - framework.Failf("Unable to get LoadBalancer ingress address for service %s/%s", e2e.svc.Namespace, e2e.svc.Name) - } + if hostAddr == "" { + framework.Failf("Unable to get LoadBalancer ingress address for service %s/%s", e2e.svc.Namespace, e2e.svc.Name) + } - elbClient, err := getAWSClientLoadBalancer(e2e.ctx) - framework.ExpectNoError(err, "failed to create AWS ELB client") + elbClient, err := getAWSClientLoadBalancer(e2e.ctx) + framework.ExpectNoError(err, "failed to create AWS ELB client") - // DescribeLoadBalancers API doesn't support filtering by DNS name directly - // Use AWS SDK paginator to search through all load balancers - foundLB, err := getAWSLoadBalancerFromDNSName(e2e.ctx, elbClient, hostAddr) - framework.ExpectNoError(err, "failed to find load balancer with DNS name %s", hostAddr) - if foundLB == nil { - framework.Failf("Found load balancer is nil for DNS name %s", hostAddr) - } + // DescribeLoadBalancers API doesn't support filtering by DNS name directly + // Use AWS SDK paginator to search through all load balancers + foundLB, err := getAWSLoadBalancerFromDNSName(e2e.ctx, elbClient, hostAddr) + framework.ExpectNoError(err, "failed to find load balancer with DNS name %s", hostAddr) + if foundLB == nil { + framework.Failf("Found load balancer is nil for DNS name %s", hostAddr) + } - lbARN := aws.ToString(foundLB.LoadBalancerArn) - if lbARN == "" { - framework.Failf("Load balancer ARN is empty for DNS name %s", hostAddr) - } - framework.Logf("Found load balancer: %s with ARN: %s", aws.ToString(foundLB.LoadBalancerName), lbARN) - - // lookup target group ARN from load balancer ARN - targetGroups, err := elbClient.DescribeTargetGroups(e2e.ctx, &elbv2.DescribeTargetGroupsInput{ - LoadBalancerArn: aws.String(lbARN), - }) - framework.ExpectNoError(err, "failed to describe target groups") - gomega.Expect(len(targetGroups.TargetGroups)).To(gomega.Equal(1)) - - targetGroupAttributes, err := elbClient.DescribeTargetGroupAttributes(e2e.ctx, &elbv2.DescribeTargetGroupAttributesInput{ - TargetGroupArn: aws.String(aws.ToString(targetGroups.TargetGroups[0].TargetGroupArn)), - }) - framework.ExpectNoError(err, "failed to describe target group attributes") - - // verify if the target group attributes are set correctly - - annotationToDict := map[string]string{} - for _, v := range strings.Split(e2e.svc.Annotations[annotationLBTargetGroupAttributes], ",") { - parts := strings.Split(v, "=") - annotationToDict[parts[0]] = parts[1] - } - framework.Logf("TG attribute Annotation to dict: %v", annotationToDict) + lbARN := aws.ToString(foundLB.LoadBalancerArn) + if lbARN == "" { + framework.Failf("Load balancer ARN is empty for DNS name %s", hostAddr) + } + framework.Logf("Found load balancer: %s with ARN: %s", aws.ToString(foundLB.LoadBalancerName), lbARN) - framework.Logf("=== All Target Group Attributes from AWS ===") - for _, attr := range targetGroupAttributes.Attributes { - framework.Logf(" %s=%s", aws.ToString(attr.Key), aws.ToString(attr.Value)) - } + // lookup target group ARN from load balancer ARN + targetGroups, err := elbClient.DescribeTargetGroups(e2e.ctx, &elbv2.DescribeTargetGroupsInput{ + LoadBalancerArn: aws.String(lbARN), + }) + framework.ExpectNoError(err, "failed to describe target groups") + gomega.Expect(len(targetGroups.TargetGroups)).To(gomega.Equal(1)) - framework.Logf("=== Expected Target Group Attributes from Annotation ===") - for key, value := range annotationToDict { - framework.Logf(" %s=%s", key, value) - } + targetGroupAttributes, err := elbClient.DescribeTargetGroupAttributes(e2e.ctx, &elbv2.DescribeTargetGroupAttributesInput{ + TargetGroupArn: aws.String(aws.ToString(targetGroups.TargetGroups[0].TargetGroupArn)), + }) + framework.ExpectNoError(err, "failed to describe target group attributes") + + // verify if the target group attributes are set correctly + + annotationToDict := map[string]string{} + for _, v := range strings.Split(e2e.svc.Annotations[annotationLBTargetGroupAttributes], ",") { + parts := strings.Split(v, "=") + annotationToDict[parts[0]] = parts[1] + } + framework.Logf("TG attribute Annotation to dict: %v", annotationToDict) + + framework.Logf("=== All Target Group Attributes from AWS ===") + for _, attr := range targetGroupAttributes.Attributes { + framework.Logf(" %s=%s", aws.ToString(attr.Key), aws.ToString(attr.Value)) + } + + framework.Logf("=== Expected Target Group Attributes from Annotation ===") + for key, value := range annotationToDict { + framework.Logf(" %s=%s", key, value) + } - // Check if our expected attributes are present and match - framework.Logf("=== Verifying Target Group Attributes ===") - for _, attr := range targetGroupAttributes.Attributes { - if expectedValue, ok := annotationToDict[aws.ToString(attr.Key)]; ok { - actualValue := aws.ToString(attr.Value) - framework.Logf("Checking attribute: %s", aws.ToString(attr.Key)) - framework.Logf(" Expected: %s", expectedValue) - framework.Logf(" Actual: %s", actualValue) - - if actualValue != expectedValue { - framework.Failf("Target group attribute mismatch for %s: expected %s, got %s", aws.ToString(attr.Key), expectedValue, actualValue) - } else { - framework.Logf("✓ Target group attribute %s matches expected value %s", aws.ToString(attr.Key), expectedValue) - } + // Check if our expected attributes are present and match + framework.Logf("=== Verifying Target Group Attributes ===") + for _, attr := range targetGroupAttributes.Attributes { + if expectedValue, ok := annotationToDict[aws.ToString(attr.Key)]; ok { + actualValue := aws.ToString(attr.Value) + framework.Logf("Checking attribute: %s", aws.ToString(attr.Key)) + framework.Logf(" Expected: %s", expectedValue) + framework.Logf(" Actual: %s", actualValue) + + if actualValue != expectedValue { + framework.Failf("Target group attribute mismatch for %s: expected %s, got %s", aws.ToString(attr.Key), expectedValue, actualValue) + } else { + framework.Logf("✓ Target group attribute %s matches expected value %s", aws.ToString(attr.Key), expectedValue) } } - }, + } }, - } + }, +} + +// managedSgModeNLBTests: NLB tests related to managed/BYO security groups, requiring NLBSecurityGroupMode=Managed in the cloud config +var managedSgModeNLBTests = []loadBalancerTestCases{ + // Test creation of NLB with Bring Your Own (BYO) Security Group from the start. + // This test validates: + // - NLB can be created with BYO SG annotation from the start + // - NLB has ONLY the BYO SG attached (no managed SG) + // - NLB is reachable with BYO SG + { + name: "NLB created with BYO Security Group should have BYO Security Group associated", + resourceSuffix: "nlb-byo", + listenerCount: 1, + extraAnnotations: map[string]string{ + annotationLBType: "nlb", + }, + hookPostServiceConfig: func(cfg *e2eTestConfig) { + framework.Logf("Creating BYO security group for NLB") - serviceNameBase := "lbconfig-test" - for _, tc := range cases { - It(tc.name, func(ctx context.Context) { - By("setting up test environment and discovering worker nodes") - e2e := newE2eTestConfig(cs) - e2e.discoverClusterWorkerNode() - framework.Logf("[SETUP] Test case: %s", tc.name) - framework.Logf("[SETUP] Worker nodes discovered: %d nodes, selector: %s, sample node: %s", e2e.nodeCount, e2e.nodeSelector, e2e.nodeSingleSample) + securityGroupName := cfg.svc.Namespace + "-" + cfg.svc.Name + "-nlb-byo-sg" + var err error + cfg.byoSecurityGroupID, err = createSecurityGroup(cfg.ctx, cfg, securityGroupName, fmt.Sprintf("BYO Security Group for NLB e2e test service %s/%s", cfg.svc.Namespace, cfg.svc.Name)) + framework.ExpectNoError(err, "Failed to create BYO security group") + framework.Logf("Created BYO security group: %s", cfg.byoSecurityGroupID) + + DeferCleanup(func(ctx context.Context) { + if cfg.byoSecurityGroupID != "" { + framework.Logf("Cleaning up BYO security group: %s", cfg.byoSecurityGroupID) + framework.Logf("Waiting for ENIs to be detached from security group...") + gomega.Eventually(ctx, func() error { + return deleteSecurityGroup(ctx, cfg.byoSecurityGroupID) + }, 2*time.Minute, 5*time.Second).Should(gomega.Succeed(), "Failed to delete BYO security group") + framework.Logf("✓ Deleted BYO security group: %s", cfg.byoSecurityGroupID) + } + }) - loadBalancerCreateTimeout := e2eservice.GetServiceLoadBalancerCreationTimeout(ctx, cs) - framework.Logf("[CONFIG] AWS load balancer timeout: %s", loadBalancerCreateTimeout) + framework.ExpectNoError(authorizeSecurityGroupToPorts(cfg.ctx, cfg.byoSecurityGroupID, cfg.svc.Spec.Ports), "Failed to authorize BYO security group to service ports") - By("building service configuration with annotations") - serviceName := serviceNameBase - if len(tc.resourceSuffix) > 0 { - serviceName = serviceName + "-" + tc.resourceSuffix - } - framework.Logf("[CONFIG] Service name: %s, namespace: %s", serviceName, ns.Name) - e2e.LBJig = e2eservice.NewTestJig(cs, ns.Name, serviceName) - - // Hook annotations to support dynamic config - e2e.svc = e2e.buildService(tc.listenerCount, tc.extraAnnotations) - framework.Logf("[CONFIG] Service ports: %d, extra annotations: %v", len(e2e.svc.Spec.Ports), tc.extraAnnotations) - - if tc.hookPostServiceConfig != nil { - By("executing hook post-service-config: applying service configuration") - framework.Logf("[HOOK] Executing post-service-config hook") - tc.hookPostServiceConfig(e2e) - framework.Logf("[HOOK] Final service annotations: %v", e2e.svc.Annotations) + cfg.svc.Annotations[annotationLBSecurityGroups] = cfg.byoSecurityGroupID + framework.Logf("Added BYO SG annotation: %s=%s", annotationLBSecurityGroups, cfg.byoSecurityGroupID) + }, + hookPreTest: func(cfg *e2eTestConfig) { + framework.Logf("Verifying NLB has only BYO SG attached") + + if len(cfg.svc.Status.LoadBalancer.Ingress) == 0 { + framework.Failf("No ingress found in LoadBalancer status for service %s/%s", cfg.svc.Namespace, cfg.svc.Name) } + lbDNS := cfg.svc.Status.LoadBalancer.Ingress[0].Hostname + framework.Logf("NLB DNS: %s", lbDNS) + + attachedSGs, err := getLoadBalancerSecurityGroups(cfg.ctx, lbDNS) + framework.ExpectNoError(err, "Failed to get NLB security groups") + framework.Logf("NLB %s has security groups: %+v", lbDNS, attachedSGs) - By("creating LoadBalancer service in Kubernetes") - if _, err := e2e.LBJig.Client.CoreV1().Services(e2e.LBJig.Namespace).Create(context.TODO(), e2e.svc, metav1.CreateOptions{}); err != nil { - framework.ExpectNoError(fmt.Errorf("failed to create LoadBalancer Service %q: %v", e2e.svc.Name, err)) + if len(attachedSGs) != 1 { + framework.Failf("Expected NLB to have exactly 1 security group (BYO SG), got %d: %v", len(attachedSGs), attachedSGs) } - framework.Logf("[K8S] LoadBalancer service created successfully") + if attachedSGs[0] != cfg.byoSecurityGroupID { + framework.Failf("Expected NLB to have BYO SG %q, got %q", cfg.byoSecurityGroupID, attachedSGs[0]) + } + framework.Logf("✓ Verified: NLB has only the BYO SG attached: %s", cfg.byoSecurityGroupID) + }, + }, + // Test transition of NLB from Managed Security Group to Bring Your Own (BYO) Security Group (SG) + // This test validates: + // - NLB created with managed (cluster-owned) SG + // - After annotation is added to use BYO SG the NLB should transition to + // have BYO SG attached and managed SG deleted + // - NLB is reachable + { + name: "NLB should transition from Managed SG to BYO Security Group", + resourceSuffix: "nlb-m2b", + listenerCount: 1, + extraAnnotations: map[string]string{ + annotationLBType: "nlb", + }, + hookPostServiceConfig: func(cfg *e2eTestConfig) { + framework.Logf("Creating BYO security group for later transition") - By("waiting for AWS load balancer provisioning") + securityGroupName := cfg.svc.Namespace + "-" + cfg.svc.Name + "-nlb-byo-sg" var err error - e2e.svc, err = e2e.LBJig.WaitForLoadBalancer(ctx, loadBalancerCreateTimeout) - // Collect comprehensive debugging information when LoadBalancer provisioning fails - if err != nil { - serviceName := e2e.LBJig.Name - if e2e.svc != nil { - serviceName = e2e.svc.Name + cfg.byoSecurityGroupID, err = createSecurityGroup(cfg.ctx, cfg, securityGroupName, fmt.Sprintf("BYO Security Group for NLB e2e test service %s/%s", cfg.svc.Namespace, cfg.svc.Name)) + framework.ExpectNoError(err, "Failed to create BYO security group") + framework.Logf("Created BYO security group: %s", cfg.byoSecurityGroupID) + + DeferCleanup(func(ctx context.Context) { + if cfg.byoSecurityGroupID != "" { + framework.Logf("Cleaning up BYO security group: %s", cfg.byoSecurityGroupID) + framework.Logf("Waiting for ENIs to be detached from security group...") + gomega.Eventually(ctx, func() error { + return deleteSecurityGroup(ctx, cfg.byoSecurityGroupID) + }, 2*time.Minute, 5*time.Second).Should(gomega.Succeed(), "Failed to delete BYO security group after waiting for ENI detachment") + framework.Logf("✓ Deleted BYO security group: %s", cfg.byoSecurityGroupID) } - framework.Logf("ERROR: LoadBalancer provisioning failed for service %q: %v", serviceName, err) - framework.Logf("ERROR: LoadBalancer provisioning timeout reached after %v", loadBalancerCreateTimeout) - - // Ensure we have detailed debugging information before failing - framework.Logf("=== LoadBalancer Provisioning Failure Debug Information ===") - gatherEventosOnFailure(e2e.ctx, e2e.kubeClient, e2e.LBJig.Namespace, e2e.LBJig.Name) - framework.Logf("=== End of LoadBalancer Provisioning Failure Debug Information ===") + }) - // Fail the test immediately to prevent further execution - framework.ExpectNoError(err, "LoadBalancer provisioning failed - check debug information above") + framework.ExpectNoError(authorizeSecurityGroupToPorts(cfg.ctx, cfg.byoSecurityGroupID, cfg.svc.Spec.Ports), "Failed to authorize BYO security group to service ports") + }, + hookPreTest: func(cfg *e2eTestConfig) { + framework.Logf("Transitioning from managed SG to BYO SG") + lbDNS := cfg.svc.Status.LoadBalancer.Ingress[0].Hostname + + // Step 1: Verify NLB has managed SG + framework.Logf("Step 1: Verifying NLB has managed security group") + managedSGs, err := getLoadBalancerSecurityGroups(cfg.ctx, lbDNS) + framework.ExpectNoError(err, "Failed to get load balancer security groups") + framework.Logf("NLB %s has security groups: %+v", lbDNS, managedSGs) + + if len(managedSGs) != 1 { + framework.Failf("Expected NLB to have 1 managed security group, got %d", len(managedSGs)) } - framework.Logf("[AWS] Load balancer provisioned successfully") + managedSGID := managedSGs[0] + framework.Logf("Managed SG ID: %s", managedSGID) + + // Step 2: Add BYO SG annotation + framework.Logf("Step 2: Adding BYO SG annotation to service") + cfg.svc.Annotations[annotationLBSecurityGroups] = cfg.byoSecurityGroupID + newSvc, err := cfg.kubeClient.CoreV1().Services(cfg.LBJig.Namespace).Update(cfg.ctx, cfg.svc, metav1.UpdateOptions{}) + framework.ExpectNoError(err, "Failed to update Kubernetes Service with BYO SG annotation") + cfg.svc = newSvc + framework.Logf("Updated service with BYO SG annotation: %s=%s", annotationLBSecurityGroups, cfg.byoSecurityGroupID) + + // Step 3: Wait for controller to process the update and verify NLB now has BYO SG + framework.Logf("Step 3: Waiting for controller to update NLB security groups and verifying BYO SG is attached") + gomega.Eventually(cfg.ctx, func() ([]string, error) { + return getLoadBalancerSecurityGroups(cfg.ctx, lbDNS) + }, 2*time.Minute, 5*time.Second).Should(gomega.And( + gomega.HaveLen(1), + gomega.ContainElement(cfg.byoSecurityGroupID), + ), "NLB should have exactly 1 security group (BYO SG) after annotation update") + framework.Logf("✓ Verified: NLB has only the BYO SG attached: %s", cfg.byoSecurityGroupID) + + // Verify managed SG was deleted + framework.Logf("Verifying managed SG %s was deleted", managedSGID) + gomega.Eventually(cfg.ctx, func() error { + _, err := getSecurityGroup(cfg.ctx, managedSGID) + return err + }, 2*time.Minute, 5*time.Second).Should(gomega.And( + gomega.HaveOccurred(), + gomega.MatchError(gomega.ContainSubstring("InvalidGroup.NotFound")), + ), "Managed SG should be deleted after transition to BYO SG") + framework.Logf("✓ Verified: Managed SG %s was deleted", managedSGID) + }, + }, + // Test transition of NLB from BYO Security Group to managed Security Group (SG) + // This test validates: + // - NLB created with BYO SG + // - After annotation for BYO SG is removed the NLB should transition to + // have a new managed SG attached and the BYO SG deassociated (but not deleted) + // - NLB is reachable + { + name: "NLB should transition from BYO SG to Managed Security Group", + resourceSuffix: "nlb-b2m", + listenerCount: 1, + extraAnnotations: map[string]string{ + annotationLBType: "nlb", + }, + hookPostServiceConfig: func(cfg *e2eTestConfig) { + framework.Logf("running hook post-service-config to create BYO security group before service creation") - By("creating backend server pods") - _, err = e2e.LBJig.Run(ctx, e2e.buildDeployment(tc.requireAffinity)) - if err != nil { - serviceName := e2e.LBJig.Name - if e2e.svc != nil { - serviceName = e2e.svc.Name + // Create BYO security group + securityGroupName := cfg.svc.Namespace + "-" + cfg.svc.Name + "-nlb-byo-sg" + var err error + cfg.byoSecurityGroupID, err = createSecurityGroup(cfg.ctx, cfg, securityGroupName, fmt.Sprintf("BYO Security Group for NLB e2e test service %s/%s", cfg.svc.Namespace, cfg.svc.Name)) + framework.ExpectNoError(err, "Failed to create BYO security group") + framework.Logf("Created BYO security group: %s", cfg.byoSecurityGroupID) + + DeferCleanup(func(ctx context.Context) { + if cfg.byoSecurityGroupID != "" { + framework.Logf("Cleaning up BYO security group: %s", cfg.byoSecurityGroupID) + framework.Logf("Waiting for ENIs to be detached from security group...") + gomega.Eventually(ctx, func() error { + return deleteSecurityGroup(ctx, cfg.byoSecurityGroupID) + }, 2*time.Minute, 5*time.Second).Should(gomega.Succeed(), "Failed to delete BYO security group after waiting for ENI detachment") + framework.Logf("✓ Deleted BYO security group: %s", cfg.byoSecurityGroupID) } - framework.Logf("ERROR: LoadBalancer provisioning failed for service %q: %v", serviceName, err) - framework.Logf("ERROR: LoadBalancer provisioning timeout reached after %v", loadBalancerCreateTimeout) + }) - // Ensure we have detailed debugging information before failing - framework.Logf("=== LoadBalancer Provisioning Failure Debug Information ===") - gatherEventosOnFailure(e2e.ctx, e2e.kubeClient, e2e.LBJig.Namespace, e2e.LBJig.Name) - framework.Logf("=== End of LoadBalancer Provisioning Failure Debug Information ===") + framework.ExpectNoError(authorizeSecurityGroupToPorts(cfg.ctx, cfg.byoSecurityGroupID, cfg.svc.Spec.Ports), "Failed to authorize BYO security group to service ports") - // Fail the test immediately to prevent further execution - framework.ExpectNoError(err, "LoadBalancer provisioning failed - check debug information above") + cfg.svc.Annotations[annotationLBSecurityGroups] = cfg.byoSecurityGroupID + framework.Logf("Added BYO SG annotation: %s=%s", annotationLBSecurityGroups, cfg.byoSecurityGroupID) + }, + hookPreTest: func(cfg *e2eTestConfig) { + framework.Logf("running hook pre-test to transition from BYO SG to managed SG") + lbDNS := cfg.svc.Status.LoadBalancer.Ingress[0].Hostname + + // Step 1: Verify NLB has BYO SG + framework.Logf("Step 1: Verifying NLB has BYO security group") + byoSGs, err := getLoadBalancerSecurityGroups(cfg.ctx, lbDNS) + framework.ExpectNoError(err, "Failed to get load balancer security groups") + framework.Logf("NLB %s has security groups: %+v", lbDNS, byoSGs) + + if len(byoSGs) != 1 { + framework.Failf("Expected NLB to have exactly 1 security group (BYO SG), got %d: %v", len(byoSGs), byoSGs) + } + if byoSGs[0] != cfg.byoSecurityGroupID { + framework.Failf("Expected NLB to have BYO SG %q, got %q", cfg.byoSecurityGroupID, byoSGs[0]) } + framework.Logf("✓ Verified: NLB has BYO SG attached: %s", cfg.byoSecurityGroupID) + + // Step 2: Remove BYO SG annotation + framework.Logf("Step 2: Removing BYO SG annotation from service") + delete(cfg.svc.Annotations, annotationLBSecurityGroups) + newSvc, err := cfg.kubeClient.CoreV1().Services(cfg.LBJig.Namespace).Update(cfg.ctx, cfg.svc, metav1.UpdateOptions{}) + framework.ExpectNoError(err, "Failed to update Kubernetes Service to remove BYO SG annotation") + cfg.svc = newSvc + framework.Logf("Removed BYO SG annotation from service") + + // Step 3: Wait for controller to process the update and verify NLB has new managed SG + framework.Logf("Step 3: Waiting for controller to update NLB security groups and verifying new managed SG is attached") + err = wait.PollUntilContextTimeout(cfg.ctx, 5*time.Second, 2*time.Minute, true, func(ctx context.Context) (bool, error) { + managedSGs, err := getLoadBalancerSecurityGroups(cfg.ctx, lbDNS) + if err != nil { + framework.Logf("Failed to get load balancer security groups: %v", err) + return false, nil + } + if len(managedSGs) == 0 { + framework.Logf("No security groups attached yet") + return false, nil + } + if managedSGs[0] == cfg.byoSecurityGroupID { + framework.Logf("Still has BYO SG attached") + return false, nil + } + framework.Logf("NLB %s has security groups after removing BYO annotation: %+v", lbDNS, managedSGs) + return true, nil + }) + framework.ExpectNoError(err, "Failed waiting for NLB to get new managed SG") + + // Verify BYO SG still exists but is not attached + framework.Logf("Verifying BYO SG %s still exists", cfg.byoSecurityGroupID) + byoSG, err := getSecurityGroup(cfg.ctx, cfg.byoSecurityGroupID) + framework.ExpectNoError(err, "Failed to get BYO security group") + gomega.Expect(byoSG).ToNot(gomega.BeNil(), "BYO SG should still exist after transition to managed SG") + framework.Logf("✓ Verified: BYO SG %s still exists and was not deleted", cfg.byoSecurityGroupID) + }, + }, +} - framework.Logf("[K8S] Backend pods created, affinity required: %t", tc.requireAffinity) +// execute load balancer tests +var _ = Describe("[cloud-provider-aws-e2e]", func() { + f := framework.NewDefaultFramework("cloud-provider-aws") + f.NamespacePodSecurityEnforceLevel = admissionapi.LevelPrivileged - if tc.hookPostServiceCreate != nil { - By("executing hook post-service-create: applying service configuration") - tc.hookPostServiceCreate(e2e) - } + var ( + cs clientset.Interface + ns *v1.Namespace + ) - By("collecting service and load balancer information") - if e2e.svc == nil { - framework.Logf("=== Service Validation Error Debug Information ===") - gatherEventosOnFailure(e2e.ctx, e2e.kubeClient, e2e.LBJig.Namespace, e2e.LBJig.Name) - framework.Logf("=== End of Service Validation Error Debug Information ===") - framework.Failf("Service is nil after LoadBalancer provisioning for service %s", e2e.LBJig.Name) - } - if len(e2e.svc.Spec.Ports) == 0 { - framework.Logf("=== Service Ports Error Debug Information ===") - framework.Logf("Service spec: %+v", e2e.svc.Spec) - gatherEventosOnFailure(e2e.ctx, e2e.kubeClient, e2e.LBJig.Namespace, e2e.LBJig.Name) - framework.Logf("=== End of Service Ports Error Debug Information ===") - framework.Failf("No ports found in service spec for service %s/%s", e2e.svc.Namespace, e2e.svc.Name) - } - if len(e2e.svc.Status.LoadBalancer.Ingress) == 0 { - framework.Logf("=== LoadBalancer Ingress Error Debug Information ===") - framework.Logf("Service status: %+v", e2e.svc.Status) - gatherEventosOnFailure(e2e.ctx, e2e.kubeClient, e2e.LBJig.Namespace, e2e.LBJig.Name) - framework.Logf("=== End of LoadBalancer Ingress Error Debug Information ===") - framework.Failf("No ingress found in LoadBalancer status for service %s/%s", e2e.svc.Namespace, e2e.svc.Name) - } + BeforeEach(func() { + cs = f.ClientSet + ns = f.Namespace + }) - svcPort := int(e2e.svc.Spec.Ports[0].Port) - ingressAddress := e2eservice.GetIngressPoint(&e2e.svc.Status.LoadBalancer.Ingress[0]) - framework.Logf("[LB-INFO] Ingress address: %s, port: %d", ingressAddress, svcPort) + AfterEach(func() { + // After each test + }) - if ingressAddress == "" { - framework.Logf("=== Empty Ingress Address Debug Information ===") - framework.Logf("LoadBalancer ingress[0]: %+v", e2e.svc.Status.LoadBalancer.Ingress[0]) - gatherEventosOnFailure(e2e.ctx, e2e.kubeClient, e2e.LBJig.Namespace, e2e.LBJig.Name) - framework.Logf("=== End of Empty Ingress Address Debug Information ===") - framework.Failf("LoadBalancer ingress address is empty for service %s/%s", e2e.svc.Namespace, e2e.svc.Name) - } + // run all load balancer tests with the default configuration + Context("loadbalancer", func() { + // CLB tests + for _, tc := range clbTests { + It(tc.name, func(ctx context.Context) { + runTestCase(ctx, tc, cs, ns) + }) + } - if tc.hookPreTest != nil { - By("executing pre-test hook") - tc.hookPreTest(e2e) - } + // Generic NLB tests + for _, tc := range nlbTests { + It(tc.name, func(ctx context.Context) { + runTestCase(ctx, tc, cs, ns) + }) + } - // overrideTestRunInClusterReachableHTTP changes the default test function to run the client in the cluster. - if tc.overrideTestRunInClusterReachableHTTP { - By("testing HTTP connectivity for internal load balancer") - framework.Logf("[TEST] Running internal connectivity test from node: %s", e2e.nodeSingleSample) - err := inClusterTestReachableHTTP(cs, ns.Name, e2e.nodeSingleSample, ingressAddress, svcPort) - if err != nil && tc.skipTestFailure { - Skip(err.Error()) - } - framework.ExpectNoError(err) - } else { - By("testing HTTP connectivity for external/internet-facing load balancer") - framework.Logf("[TEST] Running external connectivity test to %s:%d", ingressAddress, svcPort) - e2eservice.TestReachableHTTP(ctx, ingressAddress, svcPort, e2eservice.LoadBalancerLagTimeoutAWS) - } - framework.Logf("[TEST] HTTP connectivity test completed successfully") + // NLBSecurityGroupMode=Managed specific tests + for _, tc := range managedSgModeNLBTests { + It(tc.name, func(ctx context.Context) { + runTestCase(ctx, tc, cs, ns) + }) + } + }) + + // Run relevant NLB tests with NLBSecurityGroupMode disabled via cloud config override + Context("[nlb-security-group-mode-disabled] loadbalancer", Serial, Ordered, func() { + cloudConfigMgr := newCloudConfigManager(withRestartTimeout(3 * time.Minute)) - // Update the service to cluster IP - By("cleaning up: converting service to ClusterIP") - _, err = e2e.LBJig.UpdateService(ctx, func(s *v1.Service) { - s.Spec.Type = v1.ServiceTypeClusterIP + BeforeAll(func(ctx context.Context) { + // Disable NLB managed security group mode by setting an empty config (default CCM configuration) + err := cloudConfigMgr.setCloudConfig(ctx, cs, "") + framework.ExpectNoError(err, "Failed to disable NLB managed security group mode") + }) + + // Only run nlb tests not tied to NLBSecurityGroupMode=Managed + for _, tc := range nlbTests { + It(tc.name, func(ctx context.Context) { + runTestCase(ctx, tc, cs, ns) }) - framework.ExpectNoError(err) - - // Wait for the load balancer to be destroyed asynchronously - By("cleaning up: waiting for load balancer destruction") - framework.Logf("[CLEANUP] Waiting for load balancer destruction") - _, err = e2e.LBJig.WaitForLoadBalancerDestroy(ctx, ingressAddress, svcPort, loadBalancerCreateTimeout) - framework.ExpectNoError(err) - framework.Logf("[CLEANUP] Load balancer destroyed successfully") + } + + AfterAll(func(ctx context.Context) { + // Restore original CCM configuration + err := cloudConfigMgr.restoreCloudConfig(ctx, cs) + framework.ExpectNoError(err, "Failed to restore original CCM configuration") }) - } + }) }) type e2eTestConfig struct { @@ -416,6 +712,7 @@ type e2eTestConfig struct { cfgPodProtocol v1.Protocol cfgDefaultAnnotations map[string]string LBJig *e2eservice.TestJig + byoSecurityGroupID string // service instance svc *v1.Service @@ -998,3 +1295,153 @@ func gatherEventosOnFailure(ctx context.Context, cs clientset.Interface, namespa gatherControllerLogs(ctx, cs, namespace, resourceName) gatherServiceStatus(ctx, cs, namespace, resourceName) } + +// getAWSClientEC2 creates an EC2 client for AWS operations +func getAWSClientEC2(ctx context.Context) (*ec2.Client, error) { + cfg, err := config.LoadDefaultConfig(ctx, + config.WithRetryer(func() aws.Retryer { + return retry.AddWithMaxAttempts(retry.NewStandard(), 10) + }), + ) + if err != nil { + return nil, fmt.Errorf("failed to load AWS config: %w", err) + } + return ec2.NewFromConfig(cfg), nil +} + +// createSecurityGroup creates a security group for testing +func createSecurityGroup(ctx context.Context, cfg *e2eTestConfig, name, description string) (string, error) { + ec2Client, err := getAWSClientEC2(ctx) + if err != nil { + return "", err + } + + // Get VPC ID from any node in the cluster + nodes, err := cfg.kubeClient.CoreV1().Nodes().List(ctx, metav1.ListOptions{}) + if err != nil { + return "", fmt.Errorf("failed to list nodes: %w", err) + } + if len(nodes.Items) == 0 { + return "", fmt.Errorf("no nodes found in cluster") + } + + // Extract VPC ID from node's provider ID + // Provider ID format: aws:///us-west-2a/i-0123456789abcdef0 + providerID := nodes.Items[0].Spec.ProviderID + if providerID == "" { + return "", fmt.Errorf("node %s has no provider ID", nodes.Items[0].Name) + } + + // Get instance details to find VPC ID + instanceID := providerID[strings.LastIndex(providerID, "/")+1:] + describeResult, err := ec2Client.DescribeInstances(ctx, &ec2.DescribeInstancesInput{ + InstanceIds: []string{instanceID}, + }) + if err != nil { + return "", fmt.Errorf("failed to describe instance: %w", err) + } + if len(describeResult.Reservations) == 0 || len(describeResult.Reservations[0].Instances) == 0 { + return "", fmt.Errorf("instance not found: %s", instanceID) + } + + vpcID := describeResult.Reservations[0].Instances[0].VpcId + if vpcID == nil { + return "", fmt.Errorf("instance has no VPC ID") + } + + // Create security group + result, err := ec2Client.CreateSecurityGroup(ctx, &ec2.CreateSecurityGroupInput{ + GroupName: aws.String(name), + Description: aws.String(description), + VpcId: vpcID, + }) + if err != nil { + return "", fmt.Errorf("failed to create security group: %w", err) + } + + return aws.ToString(result.GroupId), nil +} + +// authorizeSecurityGroupToPorts adds ingress rules for the given service ports +func authorizeSecurityGroupToPorts(ctx context.Context, securityGroupID string, ports []v1.ServicePort) error { + ec2Client, err := getAWSClientEC2(ctx) + if err != nil { + return err + } + + permissions := make([]ec2types.IpPermission, 0, len(ports)) + for _, port := range ports { + protocol := strings.ToLower(string(port.Protocol)) + permissions = append(permissions, ec2types.IpPermission{ + IpProtocol: aws.String(protocol), + FromPort: aws.Int32(port.Port), + ToPort: aws.Int32(port.Port), + IpRanges: []ec2types.IpRange{ + {CidrIp: aws.String("0.0.0.0/0")}, + }, + }) + } + + _, err = ec2Client.AuthorizeSecurityGroupIngress(ctx, &ec2.AuthorizeSecurityGroupIngressInput{ + GroupId: aws.String(securityGroupID), + IpPermissions: permissions, + }) + if err != nil { + return fmt.Errorf("failed to authorize security group ingress: %w", err) + } + + return nil +} + +// getLoadBalancerSecurityGroups retrieves security groups attached to a load balancer +func getLoadBalancerSecurityGroups(ctx context.Context, lbDNS string) ([]string, error) { + elbClient, err := getAWSClientLoadBalancer(ctx) + if err != nil { + return nil, err + } + + lb, err := getAWSLoadBalancerFromDNSName(ctx, elbClient, lbDNS) + if err != nil { + return nil, err + } + + return lb.SecurityGroups, nil +} + +// deleteSecurityGroup deletes a security group +func deleteSecurityGroup(ctx context.Context, securityGroupID string) error { + ec2Client, err := getAWSClientEC2(ctx) + if err != nil { + return err + } + + _, err = ec2Client.DeleteSecurityGroup(ctx, &ec2.DeleteSecurityGroupInput{ + GroupId: aws.String(securityGroupID), + }) + if err != nil { + return fmt.Errorf("failed to delete security group %s: %w", securityGroupID, err) + } + + return nil +} + +// getSecurityGroup retrieves a security group by ID +func getSecurityGroup(ctx context.Context, securityGroupID string) (*ec2types.SecurityGroup, error) { + ec2Client, err := getAWSClientEC2(ctx) + if err != nil { + return nil, err + } + + result, err := ec2Client.DescribeSecurityGroups(ctx, &ec2.DescribeSecurityGroupsInput{ + GroupIds: []string{securityGroupID}, + }) + if err != nil { + return nil, fmt.Errorf("failed to describe security group: %w", err) + } + + if len(result.SecurityGroups) == 0 { + return nil, fmt.Errorf("security group not found: %s", securityGroupID) + } + + return &result.SecurityGroups[0], nil +} diff --git a/openshift-tests/ccm-aws-tests/vendor/modules.txt b/openshift-tests/ccm-aws-tests/vendor/modules.txt index c10bb0464..59daa403b 100644 --- a/openshift-tests/ccm-aws-tests/vendor/modules.txt +++ b/openshift-tests/ccm-aws-tests/vendor/modules.txt @@ -1141,7 +1141,7 @@ k8s.io/client-go/util/workqueue # k8s.io/cloud-provider v0.35.1 => k8s.io/cloud-provider v0.36.0 ## explicit; go 1.26.0 k8s.io/cloud-provider/service/helpers -# k8s.io/cloud-provider-aws/tests/e2e v0.0.0-20260507005622-746099eda51c +# k8s.io/cloud-provider-aws/tests/e2e v0.0.0-20260606003233-c34d66ed717a ## explicit; go 1.26.0 k8s.io/cloud-provider-aws/tests/e2e # k8s.io/component-base v0.36.0