-
Notifications
You must be signed in to change notification settings - Fork 77
DNM: OTE/CCM-AWS: update e2e fetching upstream exports #447
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -6,7 +6,6 @@ import ( | |
| "strings" | ||
| "time" | ||
|
|
||
| elbv2 "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2" | ||
| elbv2types "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2/types" | ||
| . "github.com/onsi/ginkgo/v2" | ||
| . "github.com/onsi/gomega" | ||
|
|
@@ -18,6 +17,8 @@ import ( | |
| "k8s.io/kubernetes/test/e2e/framework" | ||
| e2eservice "k8s.io/kubernetes/test/e2e/framework/service" | ||
| admissionapi "k8s.io/pod-security-admission/api" | ||
|
|
||
| ccme2e "k8s.io/cloud-provider-aws/tests/e2e" | ||
| ) | ||
|
|
||
| const ( | ||
|
|
@@ -118,21 +119,21 @@ var _ = Describe(fmt.Sprintf("%s NLB [OCPFeatureGate:%s]", e2eTestPrefixLoadBala | |
| It("should create NLB service with security group attached", func(ctx context.Context) { | ||
| isNLBFeatureEnabled(ctx) | ||
|
|
||
| By("creatomg required AWS clients") | ||
| elbClient, err := createAWSClientLoadBalancer(ctx) | ||
| framework.ExpectNoError(err, "failed to create AWS ELB client") | ||
| By("creating E2E test helper") | ||
| e2e, err := ccme2e.NewE2ETestHelper(ctx, cs) | ||
| framework.ExpectNoError(err, "failed to create E2E test helper") | ||
|
|
||
| By("creating test service and deployment configuration") | ||
| serviceName := "nlb-sg-crt" | ||
| _, jig, err := createServiceNLB(ctx, cs, ns, serviceName, map[int32]int32{80: 8080}) | ||
| framework.ExpectNoError(err, "failed to create NLB service load balancer") | ||
|
|
||
| foundLB, lbDNS, err := getNLBMetaFromName(ctx, cs, ns, serviceName, elbClient) | ||
| foundLB, lbDNS, err := getNLBMetaFromName(ctx, cs, ns, serviceName, e2e) | ||
| framework.ExpectNoError(err, "failed to get NLB metadata") | ||
| Expect(foundLB).NotTo(BeNil(), "found load balancer is nil") | ||
|
|
||
| DeferCleanup(func(cleanupCtx context.Context) { | ||
| err := deleteServiceAndWaitForLoadBalancerDeletion(cleanupCtx, jig, lbDNS) | ||
| DeferCleanup(func() { | ||
| err := deleteServiceAndWaitForLoadBalancerDeletion(e2e, jig, lbDNS) | ||
| framework.ExpectNoError(err, "failed to delete service and wait for load balancer deletion") | ||
| }) | ||
|
|
||
|
|
@@ -165,9 +166,9 @@ var _ = Describe(fmt.Sprintf("%s NLB [OCPFeatureGate:%s]", e2eTestPrefixLoadBala | |
| It("should have security groups attached to default ingress controller NLB", func(ctx context.Context) { | ||
| isNLBFeatureEnabled(ctx) | ||
|
|
||
| By("creatomg required AWS clients") | ||
| elbClient, err := createAWSClientLoadBalancer(ctx) | ||
| framework.ExpectNoError(err, "failed to create AWS ELB client") | ||
| By("creating E2E test helper") | ||
| e2e, err := ccme2e.NewE2ETestHelper(ctx, cs) | ||
| framework.ExpectNoError(err, "failed to create E2E test helper") | ||
|
|
||
| By("getting default ingress controller service") | ||
| ingressNamespace := "openshift-ingress" | ||
|
|
@@ -204,26 +205,23 @@ var _ = Describe(fmt.Sprintf("%s NLB [OCPFeatureGate:%s]", e2eTestPrefixLoadBala | |
| } | ||
| framework.Logf("Default ingress controller is using NLB type") | ||
|
|
||
| // Eventually ensures that the load balancer is found and is in the Active state. | ||
| var foundLB *elbv2types.LoadBalancer | ||
| err = wait.PollUntilContextTimeout(ctx, 10*time.Second, 3*time.Minute, true, func(pollCtx context.Context) (bool, error) { | ||
| lb, err := getAWSLoadBalancerFromDNSName(pollCtx, elbClient, lbDNS) | ||
| Eventually(ctx, func(ctx context.Context) error { | ||
| lb, err := e2e.GetAWSHelper().GetLoadBalancerFromDNSName(lbDNS) | ||
| if err != nil { | ||
| framework.Logf("Failed to find load balancer with DNS %s: %v", lbDNS, err) | ||
| return false, nil | ||
| } | ||
| if lb != nil && lb.State != nil && lb.State.Code == elbv2types.LoadBalancerStateEnumActive { | ||
| foundLB = lb | ||
| return true, nil | ||
| return err | ||
| } | ||
| if lb == nil { | ||
| framework.Logf("Load balancer %s not returned yet", lbDNS) | ||
| return false, nil | ||
| return fmt.Errorf("load balancer not found") | ||
| } | ||
| framework.Logf("Load balancer not yet active, current state: %v", lb.State) | ||
| return false, nil | ||
| }) | ||
| framework.ExpectNoError(err, "failed to find active load balancer") | ||
| Expect(foundLB).NotTo(BeNil(), "found load balancer is nil") | ||
| if lb.State == nil || lb.State.Code != elbv2types.LoadBalancerStateEnumActive { | ||
| return fmt.Errorf("load balancer state is not Active: %v", lb.State) | ||
| } | ||
| foundLB = lb | ||
| return nil | ||
| }).WithTimeout(20*time.Minute).Should(Succeed(), | ||
| "load balancer should be found and in the Active state") | ||
|
|
||
| By("verifying security groups are attached to the default ingress NLB") | ||
| Expect(len(foundLB.SecurityGroups)).To(BeNumerically(">", 0), | ||
|
|
@@ -253,24 +251,21 @@ var _ = Describe(fmt.Sprintf("%s NLB [OCPFeatureGate:%s]", e2eTestPrefixLoadBala | |
| It("should update security group rules when service is updated", func(ctx context.Context) { | ||
| isNLBFeatureEnabled(ctx) | ||
|
|
||
| By("creatomg required AWS clients") | ||
| ec2Client, err := createAWSClientEC2(ctx) | ||
| framework.ExpectNoError(err, "failed to create AWS EC2 client") | ||
|
|
||
| elbClient, err := createAWSClientLoadBalancer(ctx) | ||
| framework.ExpectNoError(err, "failed to create AWS ELB client") | ||
| By("creating E2E test helper") | ||
| e2e, err := ccme2e.NewE2ETestHelper(ctx, cs) | ||
| framework.ExpectNoError(err, "failed to create E2E test helper") | ||
|
|
||
| By("creating test service and deployment configuration") | ||
| serviceName := "nlb-sg-upd" | ||
| _, jig, err := createServiceNLB(ctx, cs, ns, serviceName, map[int32]int32{80: 8080}) | ||
| framework.ExpectNoError(err, "failed to create NLB service load balancer") | ||
|
|
||
| foundLB, lbDNS, err := getNLBMetaFromName(ctx, cs, ns, serviceName, elbClient) | ||
| foundLB, lbDNS, err := getNLBMetaFromName(ctx, cs, ns, serviceName, e2e) | ||
| framework.ExpectNoError(err, "failed to get NLB metadata") | ||
| Expect(foundLB).NotTo(BeNil(), "found load balancer is nil") | ||
|
|
||
| DeferCleanup(func(cleanupCtx context.Context) { | ||
| err := deleteServiceAndWaitForLoadBalancerDeletion(cleanupCtx, jig, lbDNS) | ||
| DeferCleanup(func() { | ||
| err := deleteServiceAndWaitForLoadBalancerDeletion(e2e, jig, lbDNS) | ||
| framework.ExpectNoError(err, "failed to delete service and wait for load balancer deletion") | ||
| }) | ||
|
Comment on lines
-272
to
270
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I think we will have an issue here if we pass the existing
or, better
From K8s blog on testing https://www.kubernetes.dev/blog/2023/04/12/e2e-testing-best-practices-reloaded/:
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Good suggestion Federico, that's totally make sense as we are refactoring the e2e library to standardize the utilization of the retries preventing duplicated function retries. The last update I've reviewed, still validating the overall context as well checking if the cleanup will succeed |
||
|
|
||
|
|
@@ -280,7 +275,7 @@ var _ = Describe(fmt.Sprintf("%s NLB [OCPFeatureGate:%s]", e2eTestPrefixLoadBala | |
| framework.Logf("Load balancer has %d security group(s) attached before update", len(foundLB.SecurityGroups)) | ||
|
|
||
| By("getting security group rules") | ||
| originalSGRules, err := getAWSSecurityGroupRules(ctx, ec2Client, foundLB.SecurityGroups) | ||
| originalSGRules, err := getAWSSecurityGroupRules(ctx, e2e.GetAWSHelper().GetEC2Client(), foundLB.SecurityGroups) | ||
| framework.ExpectNoError(err, "failed to get security group rules") | ||
|
Comment on lines
+278
to
279
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Don't require the security-group rule count to increase. The behavioral contract here is “ports 80 and 443 are present after the update.” AWS/CCM can replace or consolidate rules without increasing the total rule count, so Also applies to: 297-320 🤖 Prompt for AI Agents |
||
|
|
||
| By("updating service: adding a new port") | ||
|
|
@@ -297,7 +292,7 @@ var _ = Describe(fmt.Sprintf("%s NLB [OCPFeatureGate:%s]", e2eTestPrefixLoadBala | |
|
|
||
| By("waiting for security group rules to include the new port 443") | ||
| Eventually(ctx, func(ctx context.Context) ([]int32, error) { | ||
| foundLB, err = getAWSLoadBalancerFromDNSName(ctx, elbClient, lbDNS) | ||
| foundLB, err = e2e.GetAWSHelper().GetLoadBalancerFromDNSName(lbDNS) | ||
| if err != nil { | ||
| framework.Logf("Error finding load balancer: %v", err) | ||
| return nil, err | ||
|
|
@@ -311,7 +306,7 @@ var _ = Describe(fmt.Sprintf("%s NLB [OCPFeatureGate:%s]", e2eTestPrefixLoadBala | |
| return nil, fmt.Errorf("load balancer has no security groups attached") | ||
| } | ||
|
|
||
| currentSGRules, err := getAWSSecurityGroupRules(ctx, ec2Client, foundLB.SecurityGroups) | ||
| currentSGRules, err := getAWSSecurityGroupRules(ctx, e2e.GetAWSHelper().GetEC2Client(), foundLB.SecurityGroups) | ||
| if err != nil { | ||
| framework.Logf("failed to get security group rules to calculate the diff: %v", err) | ||
| return nil, err | ||
|
|
@@ -364,20 +359,17 @@ var _ = Describe(fmt.Sprintf("%s NLB [OCPFeatureGate:%s]", e2eTestPrefixLoadBala | |
| It("should cleanup security groups when service is deleted", func(ctx context.Context) { | ||
| isNLBFeatureEnabled(ctx) | ||
|
|
||
| By("creatomg required AWS clients") | ||
| ec2Client, err := createAWSClientEC2(ctx) | ||
| framework.ExpectNoError(err, "failed to create AWS EC2 client") | ||
|
|
||
| elbClient, err := createAWSClientLoadBalancer(ctx) | ||
| framework.ExpectNoError(err, "failed to create AWS ELB client") | ||
| By("creating E2E test helper") | ||
| e2e, err := ccme2e.NewE2ETestHelper(ctx, cs) | ||
| framework.ExpectNoError(err, "failed to create E2E test helper") | ||
|
|
||
| By("creating test service and deployment configuration") | ||
| serviceName := "nlb-sg-cleanup-test" | ||
|
|
||
| _, jig, err := createServiceNLB(ctx, cs, ns, serviceName, map[int32]int32{80: 8080}) | ||
| framework.ExpectNoError(err, "failed to create NLB service load balancer") | ||
|
|
||
| foundLB, lbDNS, err := getNLBMetaFromName(ctx, cs, ns, serviceName, elbClient) | ||
| foundLB, lbDNS, err := getNLBMetaFromName(ctx, cs, ns, serviceName, e2e) | ||
| framework.ExpectNoError(err, "failed to get NLB metadata") | ||
| Expect(foundLB).NotTo(BeNil(), "found load balancer is nil") | ||
|
|
||
|
|
@@ -393,20 +385,20 @@ var _ = Describe(fmt.Sprintf("%s NLB [OCPFeatureGate:%s]", e2eTestPrefixLoadBala | |
|
|
||
| By("verifying security groups exist before deletion") | ||
| for _, sgID := range securityGroupIDs { | ||
| exists, err := securityGroupExists(ctx, ec2Client, sgID) | ||
| exists, err := securityGroupExists(ctx, e2e.GetAWSHelper().GetEC2Client(), sgID) | ||
| framework.ExpectNoError(err, "failed to check security group %s", sgID) | ||
| Expect(exists).To(BeTrue(), "security group %s should exist before deletion", sgID) | ||
| } | ||
|
|
||
| err = deleteServiceAndWaitForLoadBalancerDeletion(ctx, jig, lbDNS) | ||
| err = deleteServiceAndWaitForLoadBalancerDeletion(e2e, jig, lbDNS) | ||
| framework.ExpectNoError(err, "failed to delete service and wait for load balancer deletion") | ||
|
|
||
| By("verifying managed security groups are cleaned up") | ||
| // Poll for security group cleanup with timeout (AWS cleanup can take time) | ||
| err = wait.PollUntilContextTimeout(ctx, 10*time.Second, 3*time.Minute, true, func(ctx context.Context) (bool, error) { | ||
| allDeleted := true | ||
| for _, sgID := range securityGroupIDs { | ||
| exists, err := securityGroupExists(ctx, ec2Client, sgID) | ||
| exists, err := securityGroupExists(ctx, e2e.GetAWSHelper().GetEC2Client(), sgID) | ||
| if err != nil { | ||
| framework.Logf("Error checking security group %s: %v", sgID, err) | ||
| return false, err | ||
|
|
@@ -447,12 +439,9 @@ var _ = Describe(fmt.Sprintf("%s NLB [OCPFeatureGate:%s]", e2eTestPrefixLoadBala | |
| It("should have correct security group rules for service ports", func(ctx context.Context) { | ||
| isNLBFeatureEnabled(ctx) | ||
|
|
||
| By("creatomg required AWS clients") | ||
| ec2Client, err := createAWSClientEC2(ctx) | ||
| framework.ExpectNoError(err, "failed to create AWS EC2 client") | ||
|
|
||
| elbClient, err := createAWSClientLoadBalancer(ctx) | ||
| framework.ExpectNoError(err, "failed to create AWS ELB client") | ||
| By("creating E2E test helper") | ||
| e2e, err := ccme2e.NewE2ETestHelper(ctx, cs) | ||
| framework.ExpectNoError(err, "failed to create E2E test helper") | ||
|
|
||
| By("creating test service and deployment configuration") | ||
| serviceName := "nlb-sg-rules-test" | ||
|
|
@@ -465,12 +454,12 @@ var _ = Describe(fmt.Sprintf("%s NLB [OCPFeatureGate:%s]", e2eTestPrefixLoadBala | |
| lbDNS := svc.Status.LoadBalancer.Ingress[0].Hostname | ||
| framework.Logf("Load balancer DNS: %s", lbDNS) | ||
|
|
||
| foundLB, lbDNS, err := getNLBMetaFromName(ctx, cs, ns, serviceName, elbClient) | ||
| foundLB, lbDNS, err := getNLBMetaFromName(ctx, cs, ns, serviceName, e2e) | ||
| framework.ExpectNoError(err, "failed to get NLB metadata") | ||
| Expect(foundLB).NotTo(BeNil(), "found load balancer is nil") | ||
|
|
||
| DeferCleanup(func(cleanupCtx context.Context) { | ||
| err := deleteServiceAndWaitForLoadBalancerDeletion(cleanupCtx, jig, lbDNS) | ||
| DeferCleanup(func() { | ||
| err := deleteServiceAndWaitForLoadBalancerDeletion(e2e, jig, lbDNS) | ||
| framework.ExpectNoError(err, "failed to delete service and wait for load balancer deletion") | ||
| }) | ||
|
|
||
|
|
@@ -480,7 +469,7 @@ var _ = Describe(fmt.Sprintf("%s NLB [OCPFeatureGate:%s]", e2eTestPrefixLoadBala | |
| framework.Logf("Load balancer has %d security group(s) attached", len(foundLB.SecurityGroups)) | ||
|
|
||
| By("inspecting security group rules") | ||
| currentSGRules, err := getAWSSecurityGroupRules(ctx, ec2Client, foundLB.SecurityGroups) | ||
| currentSGRules, err := getAWSSecurityGroupRules(ctx, e2e.GetAWSHelper().GetEC2Client(), foundLB.SecurityGroups) | ||
| framework.ExpectNoError(err, "failed to get security group rules to calculate the diff") | ||
| Expect(len(currentSGRules)).To(BeNumerically(">", 0), "security group rules should not be empty") | ||
|
|
||
|
|
@@ -543,42 +532,51 @@ func createServiceNLB(ctx context.Context, cs clientset.Interface, ns *v1.Namesp | |
| } | ||
|
|
||
| // getNLBMetaFromName gets the NLB metadata (AWS API object) from the service/loadbalancer name. | ||
| func getNLBMetaFromName(ctx context.Context, cs clientset.Interface, ns *v1.Namespace, serviceName string, elbc *elbv2.Client) (*elbv2types.LoadBalancer, string, error) { | ||
| func getNLBMetaFromName(ctx context.Context, cs clientset.Interface, ns *v1.Namespace, serviceName string, e2e *ccme2e.E2ETestHelper) (*elbv2types.LoadBalancer, string, error) { | ||
| By("getting service to extract load balancer DNS name") | ||
| svc, err := cs.CoreV1().Services(ns.Name).Get(ctx, serviceName, metav1.GetOptions{}) | ||
| framework.ExpectNoError(err, "failed to get service %s", serviceName) | ||
| if err != nil { | ||
| return nil, "", fmt.Errorf("failed to get service %s: %v", serviceName, err) | ||
| } | ||
|
|
||
| By("extracting load balancer DNS name") | ||
| Expect(len(svc.Status.LoadBalancer.Ingress)).To(BeNumerically(">", 0), | ||
| "no ingress entry found in LoadBalancer status") | ||
|
|
||
| if len(svc.Status.LoadBalancer.Ingress) == 0 { | ||
| return nil, "", fmt.Errorf("no ingress entry found in LoadBalancer status") | ||
| } | ||
| lbDNS := svc.Status.LoadBalancer.Ingress[0].Hostname | ||
| Expect(lbDNS).NotTo(BeEmpty(), "Ingress Hostname must not be empty") | ||
| if lbDNS == "" { | ||
| return nil, "", fmt.Errorf("Ingress Hostname must not be empty") | ||
| } | ||
| framework.Logf("Load balancer DNS: %s", lbDNS) | ||
|
|
||
| foundLB, err := getAWSLoadBalancerFromDNSName(ctx, elbc, lbDNS) | ||
| framework.ExpectNoError(err, "failed to find load balancer with DNS name %s", lbDNS) | ||
| Expect(foundLB).NotTo(BeNil(), "found load balancer is nil") | ||
| // Lookup the Load Balancer from the DNS name with default retries. | ||
| foundLB, err := e2e.GetAWSHelper().GetLoadBalancerFromDNSNameWithRetry(lbDNS) | ||
| if err != nil { | ||
| e2e.GatherServiceInfo() | ||
| return nil, "", fmt.Errorf("failed to find load balancer with DNS name %s: %v", lbDNS, err) | ||
| } | ||
| if foundLB == nil { | ||
| return nil, "", fmt.Errorf("found load balancer is nil") | ||
| } | ||
|
|
||
| return foundLB, lbDNS, nil | ||
| } | ||
|
|
||
| // deleteServiceAndWaitForLoadBalancerDeletion deletes the service and waits for the load balancer to be deleted. | ||
| func deleteServiceAndWaitForLoadBalancerDeletion(ctx context.Context, jig *e2eservice.TestJig, lbDNS string) error { | ||
| func deleteServiceAndWaitForLoadBalancerDeletion(e2e *ccme2e.E2ETestHelper, jig *e2eservice.TestJig, lbDNS string) error { | ||
| By("deleting the service") | ||
| // using deletion context to avoid leaking resources | ||
| ctx := context.TODO() | ||
| err := jig.Client.CoreV1().Services(jig.Namespace).Delete(ctx, jig.Name, metav1.DeleteOptions{}) | ||
| framework.ExpectNoError(err, "failed to delete service") | ||
|
|
||
| By("waiting for load balancer to be destroyed") | ||
| loadBalancerCreateTimeout := e2eservice.GetServiceLoadBalancerCreationTimeout(ctx, jig.Client) | ||
|
|
||
| // Get ELB client once before polling | ||
| elbClient, err := createAWSClientLoadBalancer(ctx) | ||
| framework.ExpectNoError(err, "failed to create AWS ELB client") | ||
|
|
||
| // Poll for load balancer deletion | ||
| // Get the load balancer from the DNS name with retry toleranting | ||
| // errors when not found as we expect the load balancer to be deleted. | ||
| err = wait.PollUntilContextTimeout(ctx, 5*time.Second, loadBalancerCreateTimeout, true, func(ctx context.Context) (bool, error) { | ||
| foundLB, err := getAWSLoadBalancerFromDNSName(ctx, elbClient, lbDNS) | ||
| foundLB, err := e2e.GetAWSHelper().GetLoadBalancerFromDNSName(lbDNS) | ||
| if err != nil { | ||
| // Check if the error indicates the load balancer was not found (i.e., successfully deleted) | ||
| if strings.Contains(err.Error(), "no load balancer found with DNS name") { | ||
|
|
||
Uh oh!
There was an error while loading. Please reload this page.