diff --git a/test/e2e/crds/v1alpha1/backendtrafficpolicy.go b/test/e2e/crds/v1alpha1/backendtrafficpolicy.go index a1f938c2..3a551910 100644 --- a/test/e2e/crds/v1alpha1/backendtrafficpolicy.go +++ b/test/e2e/crds/v1alpha1/backendtrafficpolicy.go @@ -20,7 +20,6 @@ package v1alpha1 import ( "context" "fmt" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -63,17 +62,14 @@ spec: By("create GatewayProxy") err = s.CreateResourceFromString(s.GetGatewayProxySpec()) Expect(err).NotTo(HaveOccurred(), "creating GatewayProxy") - time.Sleep(5 * time.Second) By("create GatewayClass") err = s.CreateResourceFromString(s.GetGatewayClassYaml()) Expect(err).NotTo(HaveOccurred(), "creating GatewayClass") - time.Sleep(5 * time.Second) By("create Gateway") err = s.CreateResourceFromString(s.GetGatewayYaml()) Expect(err).NotTo(HaveOccurred(), "creating Gateway") - time.Sleep(5 * time.Second) By("create HTTPRoute") s.ApplyHTTPRoute(types.NamespacedName{Namespace: s.Namespace(), Name: "httpbin"}, fmt.Sprintf(defaultHTTPRoute, s.Namespace(), s.Namespace())) @@ -232,28 +228,28 @@ spec: scaffold.WithExpectedStatus(200), }, }) - time.Sleep(2 * time.Second) - - ups, err := s.DefaultDataplaneResource().Upstream().List(context.Background()) - Expect(err).ToNot(HaveOccurred(), "listing upstreams") - Expect(ups).NotTo(BeEmpty(), "upstreams should not be empty") - - var target *adctypes.Upstream - for _, u := range ups { - if u.Checks != nil { - target = u - break + Eventually(func(g Gomega) { + ups, err := s.DefaultDataplaneResource().Upstream().List(context.Background()) + g.Expect(err).ToNot(HaveOccurred(), "listing upstreams") + g.Expect(ups).NotTo(BeEmpty(), "upstreams should not be empty") + + var target *adctypes.Upstream + for _, u := range ups { + if u.Checks != nil { + target = u + break + } } - } - Expect(target).NotTo(BeNil(), "upstream with health check should exist") - Expect(target.Checks.Active).NotTo(BeNil(), "active health check should be configured") - Expect(target.Checks.Active.HTTPPath).To(Equal("/get"), "active health check http path") - Expect(target.Checks.Active.Healthy.Interval).To(Equal(1), "active healthy interval") - Expect(target.Checks.Active.Healthy.HTTPStatuses).To(Equal([]int{200}), "active healthy http codes") - Expect(target.Checks.Active.Unhealthy.Interval).To(Equal(1), "active unhealthy interval") - Expect(target.Checks.Active.Unhealthy.HTTPFailures).To(Equal(2), "active unhealthy http failures") - Expect(target.Checks.Active.Unhealthy.HTTPStatuses).To(Equal([]int{500}), "active unhealthy http codes") - Expect(target.Checks.Passive).To(BeNil(), "passive health check should not be configured") + g.Expect(target).NotTo(BeNil(), "upstream with health check should exist") + g.Expect(target.Checks.Active).NotTo(BeNil(), "active health check should be configured") + g.Expect(target.Checks.Active.HTTPPath).To(Equal("/get"), "active health check http path") + g.Expect(target.Checks.Active.Healthy.Interval).To(Equal(1), "active healthy interval") + g.Expect(target.Checks.Active.Healthy.HTTPStatuses).To(Equal([]int{200}), "active healthy http codes") + g.Expect(target.Checks.Active.Unhealthy.Interval).To(Equal(1), "active unhealthy interval") + g.Expect(target.Checks.Active.Unhealthy.HTTPFailures).To(Equal(2), "active unhealthy http failures") + g.Expect(target.Checks.Active.Unhealthy.HTTPStatuses).To(Equal([]int{500}), "active unhealthy http codes") + g.Expect(target.Checks.Passive).To(BeNil(), "passive health check should not be configured") + }).WithTimeout(scaffold.DefaultTimeout).ProbeEvery(scaffold.DefaultInterval).Should(Succeed()) }) It("should configure active and passive health checks on upstream", func() { @@ -268,31 +264,31 @@ spec: scaffold.WithExpectedStatus(200), }, }) - time.Sleep(2 * time.Second) - - ups, err := s.DefaultDataplaneResource().Upstream().List(context.Background()) - Expect(err).ToNot(HaveOccurred(), "listing upstreams") - Expect(ups).NotTo(BeEmpty(), "upstreams should not be empty") - - var target *adctypes.Upstream - for _, u := range ups { - if u.Checks != nil && u.Checks.Passive != nil { - target = u - break + Eventually(func(g Gomega) { + ups, err := s.DefaultDataplaneResource().Upstream().List(context.Background()) + g.Expect(err).ToNot(HaveOccurred(), "listing upstreams") + g.Expect(ups).NotTo(BeEmpty(), "upstreams should not be empty") + + var target *adctypes.Upstream + for _, u := range ups { + if u.Checks != nil && u.Checks.Passive != nil { + target = u + break + } } - } - Expect(target).NotTo(BeNil(), "upstream with active and passive health check should exist") - - // Verify active health check - Expect(target.Checks.Active).NotTo(BeNil(), "active health check should be configured") - Expect(target.Checks.Active.HTTPPath).To(Equal("/get"), "active health check http path") - Expect(target.Checks.Active.Healthy.HTTPStatuses).To(Equal([]int{200}), "active healthy http codes") - Expect(target.Checks.Active.Unhealthy.HTTPFailures).To(Equal(2), "active unhealthy http failures") - - // Verify passive health check - Expect(target.Checks.Passive.Healthy.HTTPStatuses).To(Equal([]int{200}), "passive healthy http codes") - Expect(target.Checks.Passive.Unhealthy.HTTPStatuses).To(Equal([]int{502, 503}), "passive unhealthy http codes") - Expect(target.Checks.Passive.Unhealthy.HTTPFailures).To(Equal(3), "passive unhealthy http failures") + g.Expect(target).NotTo(BeNil(), "upstream with active and passive health check should exist") + + // Verify active health check + g.Expect(target.Checks.Active).NotTo(BeNil(), "active health check should be configured") + g.Expect(target.Checks.Active.HTTPPath).To(Equal("/get"), "active health check http path") + g.Expect(target.Checks.Active.Healthy.HTTPStatuses).To(Equal([]int{200}), "active healthy http codes") + g.Expect(target.Checks.Active.Unhealthy.HTTPFailures).To(Equal(2), "active unhealthy http failures") + + // Verify passive health check + g.Expect(target.Checks.Passive.Healthy.HTTPStatuses).To(Equal([]int{200}), "passive healthy http codes") + g.Expect(target.Checks.Passive.Unhealthy.HTTPStatuses).To(Equal([]int{502, 503}), "passive unhealthy http codes") + g.Expect(target.Checks.Passive.Unhealthy.HTTPFailures).To(Equal(3), "passive unhealthy http failures") + }).WithTimeout(scaffold.DefaultTimeout).ProbeEvery(scaffold.DefaultInterval).Should(Succeed()) }) It("should remove health check when policy is deleted", func() { @@ -307,32 +303,33 @@ spec: scaffold.WithExpectedStatus(200), }, }) - time.Sleep(2 * time.Second) - // Verify health check is present on the target upstream - ups, err := s.DefaultDataplaneResource().Upstream().List(context.Background()) - Expect(err).ToNot(HaveOccurred()) - hasHealthCheck := false - for _, u := range ups { - if u.Checks != nil { - hasHealthCheck = true - break + Eventually(func(g Gomega) { + ups, err := s.DefaultDataplaneResource().Upstream().List(context.Background()) + g.Expect(err).ToNot(HaveOccurred()) + hasHealthCheck := false + for _, u := range ups { + if u.Checks != nil { + hasHealthCheck = true + break + } } - } - Expect(hasHealthCheck).To(BeTrue(), "upstream should have health check before policy deletion") + g.Expect(hasHealthCheck).To(BeTrue(), "upstream should have health check before policy deletion") + }).WithTimeout(scaffold.DefaultTimeout).ProbeEvery(scaffold.DefaultInterval).Should(Succeed()) // Delete the policy err = s.DeleteResourceFromString(policyWithActiveHealthCheck) Expect(err).NotTo(HaveOccurred(), "deleting BackendTrafficPolicy") - time.Sleep(3 * time.Second) // Verify health check is removed from the target upstream - ups, err = s.DefaultDataplaneResource().Upstream().List(context.Background()) - Expect(err).ToNot(HaveOccurred()) - Expect(ups).NotTo(BeEmpty(), "upstreams should still exist after policy deletion") - for _, u := range ups { - Expect(u.Checks).To(BeNil(), "upstream should not have health check after policy deletion") - } + Eventually(func(g Gomega) { + ups, err := s.DefaultDataplaneResource().Upstream().List(context.Background()) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(ups).NotTo(BeEmpty(), "upstreams should still exist after policy deletion") + for _, u := range ups { + g.Expect(u.Checks).To(BeNil(), "upstream should not have health check after policy deletion") + } + }).WithTimeout(scaffold.DefaultTimeout).ProbeEvery(scaffold.DefaultInterval).Should(Succeed()) }) }) }) @@ -379,7 +376,6 @@ spec: By("create GatewayProxy") err := s.CreateResourceFromString(s.GetGatewayProxySpec()) Expect(err).NotTo(HaveOccurred(), "creating GatewayProxy") - time.Sleep(5 * time.Second) By("create IngressClass with GatewayProxy reference") err = s.CreateResourceFromString(fmt.Sprintf(defaultIngressClass, s.GetControllerName(), s.Namespace())) diff --git a/test/e2e/crds/v1alpha1/consumer.go b/test/e2e/crds/v1alpha1/consumer.go index 6454fcf6..f8c0634e 100644 --- a/test/e2e/crds/v1alpha1/consumer.go +++ b/test/e2e/crds/v1alpha1/consumer.go @@ -80,17 +80,14 @@ spec: By("create GatewayProxy") err = s.CreateResourceFromString(s.GetGatewayProxySpec()) Expect(err).NotTo(HaveOccurred(), "creating GatewayProxy") - time.Sleep(time.Second) By("create GatewayClass") err = s.CreateResourceFromString(s.GetGatewayClassYaml()) Expect(err).NotTo(HaveOccurred(), "creating GatewayClass") - time.Sleep(time.Second) By("create Gateway") err = s.CreateResourceFromString(s.GetGatewayYaml()) Expect(err).NotTo(HaveOccurred(), "creating Gateway") - time.Sleep(time.Second) By("create HTTPRoute") s.ApplyHTTPRoute(types.NamespacedName{Namespace: s.Namespace(), Name: "httpbin"}, fmt.Sprintf(defaultHTTPRoute, s.Namespace())) diff --git a/test/e2e/crds/v1alpha1/gatewayproxy.go b/test/e2e/crds/v1alpha1/gatewayproxy.go index a2c76d9d..0f0aca7f 100644 --- a/test/e2e/crds/v1alpha1/gatewayproxy.go +++ b/test/e2e/crds/v1alpha1/gatewayproxy.go @@ -117,17 +117,14 @@ spec: err = s.CreateResourceFromString(fmt.Sprintf(gatewayProxySpec, framework.ProviderType, s.AdminKey())) } Expect(err).NotTo(HaveOccurred(), "creating GatewayProxy") - time.Sleep(5 * time.Second) By("create GatewayClass") err = s.CreateResourceFromString(s.GetGatewayClassYaml()) Expect(err).NotTo(HaveOccurred(), "creating GatewayClass") - time.Sleep(5 * time.Second) By("create Gateway") err = s.CreateResourceFromString(s.GetGatewayYaml()) Expect(err).NotTo(HaveOccurred(), "creating Gateway") - time.Sleep(5 * time.Second) By("create HTTPRoute") s.ApplyHTTPRoute(types.NamespacedName{Namespace: s.Namespace(), Name: "httpbin"}, fmt.Sprintf(httpRouteSpec, gatewayName)) diff --git a/test/e2e/crds/v2/basic.go b/test/e2e/crds/v2/basic.go index 7b197e10..83c64614 100644 --- a/test/e2e/crds/v2/basic.go +++ b/test/e2e/crds/v2/basic.go @@ -67,12 +67,10 @@ var _ = Describe("APISIX Standalone Basic Tests", Label("apisix.apache.org", "v2 By("create GatewayProxy") err := s.CreateResourceFromString(s.GetGatewayProxySpec()) Expect(err).NotTo(HaveOccurred(), "creating GatewayProxy") - time.Sleep(5 * time.Second) By("create IngressClass") err = s.CreateResourceFromStringWithNamespace(s.GetIngressClassYaml(), "") Expect(err).NotTo(HaveOccurred(), "creating IngressClass") - time.Sleep(5 * time.Second) const apisixRouteSpec = ` apiVersion: apisix.apache.org/v2 diff --git a/test/e2e/crds/v2/consumer.go b/test/e2e/crds/v2/consumer.go index 3b8eb2e7..805e5262 100644 --- a/test/e2e/crds/v2/consumer.go +++ b/test/e2e/crds/v2/consumer.go @@ -72,12 +72,10 @@ var _ = Describe("Test ApisixConsumer", Label("apisix.apache.org", "v2", "apisix By("create GatewayProxy") err := s.CreateResourceFromString(s.GetGatewayProxySpec()) Expect(err).NotTo(HaveOccurred(), "creating GatewayProxy") - time.Sleep(5 * time.Second) By("create IngressClass") err = s.CreateResourceFromStringWithNamespace(s.GetIngressClassYaml(), "") Expect(err).NotTo(HaveOccurred(), "creating IngressClass") - time.Sleep(5 * time.Second) }) Context("Test KeyAuth", func() { diff --git a/test/e2e/crds/v2/globalrule.go b/test/e2e/crds/v2/globalrule.go index 6dcb1690..649435ad 100644 --- a/test/e2e/crds/v2/globalrule.go +++ b/test/e2e/crds/v2/globalrule.go @@ -31,6 +31,21 @@ import ( var _ = Describe("Test GlobalRule", Label("apisix.apache.org", "v2", "apisixglobalrule"), func() { s := scaffold.NewDefaultScaffold() + // globalRuleAccepted polls until the named ApisixGlobalRule reports the + // accepted/synced status, replacing fixed sleeps before status checks. + globalRuleAccepted := func(name string, extraSubstrings ...string) { + matchers := []OmegaMatcher{ + ContainSubstring(`status: "True"`), + ContainSubstring("message: The global rule has been accepted and synced to APISIX"), + } + for _, sub := range extraSubstrings { + matchers = append(matchers, ContainSubstring(sub)) + } + s.RetryAssertion(func() (string, error) { + return s.GetResourceYaml("ApisixGlobalRule", name) + }).Should(And(matchers...)) + } + var defaultRouteSpec = ` apiVersion: apisix.apache.org/v2 kind: ApisixRoute @@ -55,17 +70,14 @@ spec: By("create GatewayProxy") err := s.CreateResourceFromString(s.GetGatewayProxySpec()) Expect(err).NotTo(HaveOccurred(), "creating GatewayProxy") - time.Sleep(5 * time.Second) By("create IngressClass") err = s.CreateResourceFromStringWithNamespace(s.GetIngressClassYaml(), "") Expect(err).NotTo(HaveOccurred(), "creating IngressClass") - time.Sleep(5 * time.Second) By("create ApisixRoute") err = s.CreateResourceFromString(fmt.Sprintf(defaultRouteSpec, s.Namespace())) Expect(err).NotTo(HaveOccurred(), "creating ApisixRoute") - time.Sleep(5 * time.Second) By("verify Ingress works") Eventually(func() int { @@ -99,34 +111,35 @@ spec: Expect(err).NotTo(HaveOccurred(), "creating ApisixGlobalRule") By("verify ApisixGlobalRule status condition") - time.Sleep(5 * time.Second) - gryaml, err := s.GetResourceYaml("ApisixGlobalRule", "test-global-rule-response-rewrite") - Expect(err).NotTo(HaveOccurred(), "getting ApisixGlobalRule yaml") - Expect(gryaml).To(ContainSubstring(`status: "True"`)) - Expect(gryaml).To(ContainSubstring("message: The global rule has been accepted and synced to APISIX")) + globalRuleAccepted("test-global-rule-response-rewrite") By("verify global rule is applied - response should have custom headers") - resp := s.NewAPISIXClient(). - GET("/get"). - WithHost("globalrule.example.com"). - Expect(). - Status(http.StatusOK) - resp.Header("X-Global-Rule").IsEqual("test-response-rewrite") - resp.Header("X-Global-Test").IsEqual("enabled") + s.RequestAssert(&scaffold.RequestAssert{ + Method: "GET", + Path: "/get", + Host: "globalrule.example.com", + Checks: []scaffold.ResponseCheckFunc{ + scaffold.WithExpectedStatus(http.StatusOK), + scaffold.WithExpectedHeader("X-Global-Rule", "test-response-rewrite"), + scaffold.WithExpectedHeader("X-Global-Test", "enabled"), + }, + }) By("delete ApisixGlobalRule") err = s.DeleteResource("ApisixGlobalRule", "test-global-rule-response-rewrite") Expect(err).NotTo(HaveOccurred(), "deleting ApisixGlobalRule") - time.Sleep(5 * time.Second) By("verify global rule is removed - response should not have custom headers") - resp = s.NewAPISIXClient(). - GET("/get"). - WithHost("globalrule.example.com"). - Expect(). - Status(http.StatusOK) - resp.Header("X-Global-Rule").IsEmpty() - resp.Header("X-Global-Test").IsEmpty() + s.RequestAssert(&scaffold.RequestAssert{ + Method: "GET", + Path: "/get", + Host: "globalrule.example.com", + Checks: []scaffold.ResponseCheckFunc{ + scaffold.WithExpectedStatus(http.StatusOK), + scaffold.WithExpectedNotHeader("X-Global-Rule"), + scaffold.WithExpectedNotHeader("X-Global-Test"), + }, + }) }) It("Test GlobalRule update", func() { @@ -166,41 +179,38 @@ spec: Expect(err).NotTo(HaveOccurred(), "creating ApisixGlobalRule") By("verify initial ApisixGlobalRule status condition") - time.Sleep(5 * time.Second) - gryaml, err := s.GetResourceYaml("ApisixGlobalRule", "test-global-rule-update") - Expect(err).NotTo(HaveOccurred(), "getting ApisixGlobalRule yaml") - Expect(gryaml).To(ContainSubstring(`status: "True"`)) - Expect(gryaml).To(ContainSubstring("message: The global rule has been accepted and synced to APISIX")) + globalRuleAccepted("test-global-rule-update") By("verify initial configuration") - resp := s.NewAPISIXClient(). - GET("/get"). - WithHost("globalrule.example.com"). - Expect(). - Status(http.StatusOK) - resp.Header("X-Update-Test").IsEqual("version1") - resp.Header("X-New-Header").IsEmpty() + s.RequestAssert(&scaffold.RequestAssert{ + Method: "GET", + Path: "/get", + Host: "globalrule.example.com", + Checks: []scaffold.ResponseCheckFunc{ + scaffold.WithExpectedStatus(http.StatusOK), + scaffold.WithExpectedHeader("X-Update-Test", "version1"), + scaffold.WithExpectedNotHeader("X-New-Header"), + }, + }) By("update ApisixGlobalRule") err = s.CreateResourceFromString(fmt.Sprintf(updatedGlobalRuleYaml, s.Namespace())) Expect(err).NotTo(HaveOccurred(), "updating ApisixGlobalRule") By("verify updated ApisixGlobalRule status condition") - time.Sleep(5 * time.Second) - gryaml, err = s.GetResourceYaml("ApisixGlobalRule", "test-global-rule-update") - Expect(err).NotTo(HaveOccurred(), "getting updated ApisixGlobalRule yaml") - Expect(gryaml).To(ContainSubstring(`status: "True"`)) - Expect(gryaml).To(ContainSubstring("message: The global rule has been accepted and synced to APISIX")) - Expect(gryaml).To(ContainSubstring("observedGeneration: 2")) + globalRuleAccepted("test-global-rule-update", "observedGeneration: 2") By("verify updated configuration") - resp = s.NewAPISIXClient(). - GET("/get"). - WithHost("globalrule.example.com"). - Expect(). - Status(http.StatusOK) - resp.Header("X-Update-Test").IsEqual("version2") - resp.Header("X-New-Header").IsEqual("added") + s.RequestAssert(&scaffold.RequestAssert{ + Method: "GET", + Path: "/get", + Host: "globalrule.example.com", + Checks: []scaffold.ResponseCheckFunc{ + scaffold.WithExpectedStatus(http.StatusOK), + scaffold.WithExpectedHeader("X-Update-Test", "version2"), + scaffold.WithExpectedHeader("X-New-Header", "added"), + }, + }) By("delete ApisixGlobalRule") err = s.DeleteResource("ApisixGlobalRule", "test-global-rule-update") @@ -249,57 +259,55 @@ spec: Expect(err).NotTo(HaveOccurred(), "creating ApisixGlobalRule with response-rewrite") By("verify both ApisixGlobalRule status conditions") - time.Sleep(5 * time.Second) - - proxyRewriteYaml, err := s.GetResourceYaml("ApisixGlobalRule", "test-global-rule-proxy-rewrite") - Expect(err).NotTo(HaveOccurred(), "getting proxy-rewrite ApisixGlobalRule yaml") - Expect(proxyRewriteYaml).To(ContainSubstring(`status: "True"`)) - Expect(proxyRewriteYaml).To(ContainSubstring("message: The global rule has been accepted and synced to APISIX")) - - responseRewriteYaml, err := s.GetResourceYaml("ApisixGlobalRule", "test-global-rule-response-rewrite-multi") - Expect(err).NotTo(HaveOccurred(), "getting response-rewrite ApisixGlobalRule yaml") - Expect(responseRewriteYaml).To(ContainSubstring(`status: "True"`)) - Expect(responseRewriteYaml).To(ContainSubstring("message: The global rule has been accepted and synced to APISIX")) + globalRuleAccepted("test-global-rule-proxy-rewrite") + globalRuleAccepted("test-global-rule-response-rewrite-multi") By("verify both global rules are applied on GET request") - getResp := s.NewAPISIXClient(). - GET("/get"). - WithHost("globalrule.example.com"). - Expect(). - Status(http.StatusOK) - getResp.Header("X-Global-Multi").IsEqual("test-multi-rule") - getResp.Header("X-Response-Type").IsEqual("rewrite") - getResp.Body().Contains(`"X-Global-Proxy": "test"`) + s.RequestAssert(&scaffold.RequestAssert{ + Method: "GET", + Path: "/get", + Host: "globalrule.example.com", + Checks: []scaffold.ResponseCheckFunc{ + scaffold.WithExpectedStatus(http.StatusOK), + scaffold.WithExpectedHeader("X-Global-Multi", "test-multi-rule"), + scaffold.WithExpectedHeader("X-Response-Type", "rewrite"), + scaffold.WithExpectedBodyContains(`"X-Global-Proxy": "test"`), + }, + }) By("delete proxy-rewrite ApisixGlobalRule") err = s.DeleteResource("ApisixGlobalRule", "test-global-rule-proxy-rewrite") Expect(err).NotTo(HaveOccurred(), "deleting proxy-rewrite ApisixGlobalRule") - time.Sleep(5 * time.Second) By("verify only response-rewrite global rule remains - proxy-rewrite headers should be removed") - getRespAfterProxyDelete := s.NewAPISIXClient(). - GET("/get"). - WithHost("globalrule.example.com"). - Expect(). - Status(http.StatusOK) - getRespAfterProxyDelete.Header("X-Global-Multi").IsEqual("test-multi-rule") - getRespAfterProxyDelete.Header("X-Response-Type").IsEqual("rewrite") - getRespAfterProxyDelete.Body().NotContains(`"X-Global-Proxy": "test"`) + s.RequestAssert(&scaffold.RequestAssert{ + Method: "GET", + Path: "/get", + Host: "globalrule.example.com", + Checks: []scaffold.ResponseCheckFunc{ + scaffold.WithExpectedStatus(http.StatusOK), + scaffold.WithExpectedHeader("X-Global-Multi", "test-multi-rule"), + scaffold.WithExpectedHeader("X-Response-Type", "rewrite"), + scaffold.WithExpectedBodyNotContains(`"X-Global-Proxy": "test"`), + }, + }) By("delete response-rewrite ApisixGlobalRule") err = s.DeleteResource("ApisixGlobalRule", "test-global-rule-response-rewrite-multi") Expect(err).NotTo(HaveOccurred(), "deleting response-rewrite ApisixGlobalRule") - time.Sleep(5 * time.Second) By("verify all global rules are removed") - finalResp := s.NewAPISIXClient(). - GET("/get"). - WithHost("globalrule.example.com"). - Expect(). - Status(http.StatusOK) - finalResp.Header("X-Global-Multi").IsEmpty() - finalResp.Header("X-Response-Type").IsEmpty() - finalResp.Body().NotContains(`"X-Global-Proxy": "test"`) + s.RequestAssert(&scaffold.RequestAssert{ + Method: "GET", + Path: "/get", + Host: "globalrule.example.com", + Checks: []scaffold.ResponseCheckFunc{ + scaffold.WithExpectedStatus(http.StatusOK), + scaffold.WithExpectedNotHeader("X-Global-Multi"), + scaffold.WithExpectedNotHeader("X-Response-Type"), + scaffold.WithExpectedBodyNotContains(`"X-Global-Proxy": "test"`), + }, + }) }) It("Test GlobalRule with plugin using secretRef", func() { @@ -336,19 +344,18 @@ spec: Expect(err).NotTo(HaveOccurred(), "creating ApisixGlobalRule with secretRef") By("verify ApisixGlobalRule status condition") - time.Sleep(5 * time.Second) - gryaml, err := s.GetResourceYaml("ApisixGlobalRule", "test-global-rule-with-secret") - Expect(err).NotTo(HaveOccurred(), "getting ApisixGlobalRule yaml") - Expect(gryaml).To(ContainSubstring(`status: "True"`)) - Expect(gryaml).To(ContainSubstring("message: The global rule has been accepted and synced to APISIX")) + globalRuleAccepted("test-global-rule-with-secret") By("verify global rule with secret is applied") - resp := s.NewAPISIXClient(). - GET("/get"). - WithHost("globalrule.example.com"). - Expect(). - Status(http.StatusOK) - resp.Body().Contains("GlobalRule with secret test") + s.RequestAssert(&scaffold.RequestAssert{ + Method: "GET", + Path: "/get", + Host: "globalrule.example.com", + Checks: []scaffold.ResponseCheckFunc{ + scaffold.WithExpectedStatus(http.StatusOK), + scaffold.WithExpectedBodyContains("GlobalRule with secret test"), + }, + }) By("update Secret") updatedSecretYaml := ` @@ -363,26 +370,29 @@ stringData: ` err = s.CreateResourceFromString(fmt.Sprintf(updatedSecretYaml, s.Namespace())) Expect(err).NotTo(HaveOccurred(), "updating Secret") - time.Sleep(5 * time.Second) By("verify global rule with updated secret") - resp = s.NewAPISIXClient(). - GET("/get"). - WithHost("globalrule.example.com"). - Expect(). - Status(http.StatusOK) - resp.Body().Contains("GlobalRule with secret test updated") + s.RequestAssert(&scaffold.RequestAssert{ + Method: "GET", + Path: "/get", + Host: "globalrule.example.com", + Checks: []scaffold.ResponseCheckFunc{ + scaffold.WithExpectedStatus(http.StatusOK), + scaffold.WithExpectedBodyContains("GlobalRule with secret test updated"), + }, + }) By("delete Secret") err = s.DeleteResource("Secret", "echo-secret") Expect(err).NotTo(HaveOccurred(), "deleting Secret") - time.Sleep(5 * time.Second) By("verify ApisixGlobalRule status shows error after secret deletion") - gryaml, err = s.GetResourceYaml("ApisixGlobalRule", "test-global-rule-with-secret") - Expect(err).NotTo(HaveOccurred(), "getting ApisixGlobalRule yaml") - Expect(gryaml).To(ContainSubstring(`status: "False"`)) - Expect(gryaml).To(ContainSubstring("failed to get Secret")) + s.RetryAssertion(func() (string, error) { + return s.GetResourceYaml("ApisixGlobalRule", "test-global-rule-with-secret") + }).Should(And( + ContainSubstring(`status: "False"`), + ContainSubstring("failed to get Secret"), + )) By("delete ApisixGlobalRule") err = s.DeleteResource("ApisixGlobalRule", "test-global-rule-with-secret") diff --git a/test/e2e/crds/v2/pluginconfig.go b/test/e2e/crds/v2/pluginconfig.go index 250bb5a2..13867f1b 100644 --- a/test/e2e/crds/v2/pluginconfig.go +++ b/test/e2e/crds/v2/pluginconfig.go @@ -42,12 +42,10 @@ var _ = Describe("Test ApisixPluginConfig", Label("apisix.apache.org", "v2", "ap By("create GatewayProxy") err := s.CreateResourceFromString(s.GetGatewayProxySpec()) Expect(err).NotTo(HaveOccurred(), "creating GatewayProxy") - time.Sleep(5 * time.Second) By("create IngressClass") err = s.CreateResourceFromStringWithNamespace(s.GetIngressClassYaml(), "") Expect(err).NotTo(HaveOccurred(), "creating IngressClass") - time.Sleep(5 * time.Second) }) It("Basic ApisixPluginConfig test", func() { @@ -190,12 +188,17 @@ spec: By("update ApisixPluginConfig") applier.MustApplyAPIv2(types.NamespacedName{Namespace: s.Namespace(), Name: "test-plugin-config-update"}, &apisixPluginConfig, fmt.Sprintf(apisixPluginConfigSpecV2, s.Namespace())) - time.Sleep(5 * time.Second) By("verify updated plugin config works") - resp = s.NewAPISIXClient().GET("/get").Expect().Status(http.StatusOK) - resp.Header("X-Version").IsEqual("v2") - resp.Header("X-Updated").IsEqual("true") + s.RequestAssert(&scaffold.RequestAssert{ + Method: "GET", + Path: "/get", + Checks: []scaffold.ResponseCheckFunc{ + scaffold.WithExpectedStatus(http.StatusOK), + scaffold.WithExpectedHeader("X-Version", "v2"), + scaffold.WithExpectedHeader("X-Updated", "true"), + }, + }) By("delete resources") err := s.DeleteResource("ApisixRoute", "test-route-update") @@ -384,7 +387,6 @@ spec: By("apply ApisixPluginConfig in default namespace") err := s.CreateResourceFromStringWithNamespace(fmt.Sprintf(crossNamespaceApisixPluginConfigSpec, s.Namespace()), "default") Expect(err).NotTo(HaveOccurred(), "creating default/cross-ns-plugin-config") - time.Sleep(5 * time.Second) By("apply ApisixRoute in test namespace that references ApisixPluginConfig in default namespace") var apisixRoute apiv2.ApisixRoute diff --git a/test/e2e/crds/v2/route.go b/test/e2e/crds/v2/route.go index d228885d..2859b3df 100644 --- a/test/e2e/crds/v2/route.go +++ b/test/e2e/crds/v2/route.go @@ -53,12 +53,10 @@ var _ = Describe("Test ApisixRoute", Label("apisix.apache.org", "v2", "apisixrou By("create GatewayProxy") err := s.CreateResourceFromString(s.GetGatewayProxySpec()) Expect(err).NotTo(HaveOccurred(), "creating GatewayProxy") - time.Sleep(5 * time.Second) By("create IngressClass") err = s.CreateResourceFromStringWithNamespace(s.GetIngressClassYaml(), "") Expect(err).NotTo(HaveOccurred(), "creating IngressClass") - time.Sleep(5 * time.Second) }) Context("Test ApisixRoute", func() { @@ -228,28 +226,32 @@ spec: By("apply ApisixRoute with plugins") applier.MustApplyAPIv2(types.NamespacedName{Namespace: s.Namespace(), Name: "default"}, &apisixRoute, fmt.Sprintf(apisixRouteSpecPart0, s.Namespace(), s.Namespace())+apisixRouteSpecPart1) - time.Sleep(5 * time.Second) By("verify plugin works") - resp := s.NewAPISIXClient().GET("/get").Expect().Status(http.StatusOK) - resp.Header("X-Global-Rule").IsEqual("test-response-rewrite") - resp.Header("X-Global-Test").IsEqual("enabled") + s.RequestAssert(&scaffold.RequestAssert{ + Method: "GET", + Path: "/get", + Checks: []scaffold.ResponseCheckFunc{ + scaffold.WithExpectedStatus(http.StatusOK), + scaffold.WithExpectedHeader("X-Global-Rule", "test-response-rewrite"), + scaffold.WithExpectedHeader("X-Global-Test", "enabled"), + }, + }) By("remove plugin") applier.MustApplyAPIv2(types.NamespacedName{Namespace: s.Namespace(), Name: "default"}, &apisixRoute, fmt.Sprintf(apisixRouteSpecPart0, s.Namespace(), s.Namespace())) - time.Sleep(5 * time.Second) By("verify no plugin works") s.RequestAssert(&scaffold.RequestAssert{ Method: "GET", Path: "/get", - Check: scaffold.WithExpectedStatus(http.StatusOK), + Checks: []scaffold.ResponseCheckFunc{ + scaffold.WithExpectedStatus(http.StatusOK), + scaffold.WithExpectedNotHeader("X-Global-Rule"), + scaffold.WithExpectedNotHeader("X-Global-Test"), + }, }) - - resp = s.NewAPISIXClient().GET("/get").Expect().Status(http.StatusOK) - resp.Header("X-Global-Rule").IsEmpty() - resp.Header("X-Global-Test").IsEmpty() }) It("Test ApisixRoute match by vars", func() { @@ -1464,15 +1466,21 @@ spec: fmt.Sprintf(apisixRouteSpec, s.Namespace(), s.Namespace()), ) By("wait for WebSocket server to be ready") - time.Sleep(10 * time.Second) - By("verify WebSocket connection") u = url.URL{ Scheme: "ws", Host: s.ApisixHTTPEndpoint(), Path: "/echo", } headers = http.Header{"Host": []string{"httpbin.org"}} + Eventually(func() int { + _, resp, _ := websocket.DefaultDialer.Dial(u.String(), headers) + if resp == nil { + return 0 + } + return resp.StatusCode + }).WithTimeout(scaffold.DefaultTimeout).ProbeEvery(scaffold.DefaultInterval).Should(Equal(http.StatusSwitchingProtocols)) + By("verify WebSocket connection") conn, resp, err := websocket.DefaultDialer.Dial(u.String(), headers) Expect(err).ShouldNot(HaveOccurred(), "WebSocket handshake") Expect(resp.StatusCode).Should(Equal(http.StatusSwitchingProtocols)) @@ -1686,27 +1694,20 @@ spec: createApisixRoute(routeName, upstreamName) By("verify access to multiple services") - time.Sleep(7 * time.Second) hasHttpbin := false hasNginx := false - for range 20 { - expect := s.NewAPISIXClient().GET("/ip"). + Eventually(func(g Gomega) { + body := s.NewAPISIXClient().GET("/ip"). WithHeader("Host", "httpbin.org"). WithHeader("X-Foo", "bar"). - Expect() - - body := expect.Body().Raw() + Expect().Body().Raw() if strings.Contains(body, "Hello") { hasNginx = true } else if strings.Contains(body, `"origin"`) { hasHttpbin = true } - if hasHttpbin && hasNginx { - break - } - time.Sleep(100 * time.Millisecond) - } - assert.True(GinkgoT(), hasHttpbin && hasNginx, "both httpbin and nginx should be accessed at least once") + g.Expect(hasHttpbin && hasNginx).To(BeTrue(), "both httpbin and nginx should be accessed at least once") + }).WithTimeout(scaffold.DefaultTimeout).ProbeEvery(scaffold.DefaultInterval).Should(Succeed()) }) It("should be able to use backends and upstreams together", func() { @@ -1759,26 +1760,20 @@ spec: routeSpec, ) By("verify access to multiple services") - time.Sleep(7 * time.Second) hasNginx := false hasHttpbin := false - for range 20 { - expect := s.NewAPISIXClient().GET("/ip"). + Eventually(func(g Gomega) { + body := s.NewAPISIXClient().GET("/ip"). WithHeader("Host", "httpbin.org"). WithHeader("X-Foo", "bar"). - Expect() - body := expect.Body().Raw() + Expect().Body().Raw() if strings.Contains(body, "Hello") { hasNginx = true } else if strings.Contains(body, `"origin"`) { hasHttpbin = true } - if hasNginx && hasHttpbin { - break - } - time.Sleep(100 * time.Millisecond) - } - assert.True(GinkgoT(), hasNginx && hasHttpbin, "both nginx and httpbin should be accessed at least once") + g.Expect(hasNginx && hasHttpbin).To(BeTrue(), "both nginx and httpbin should be accessed at least once") + }).WithTimeout(scaffold.DefaultTimeout).ProbeEvery(scaffold.DefaultInterval).Should(Succeed()) }) }) }) @@ -1942,12 +1937,12 @@ spec: By("apply apisixroute") applier.MustApplyAPIv2(types.NamespacedName{Namespace: s.Namespace(), Name: "default"}, new(apiv2.ApisixRoute), fmt.Sprintf(apisixRouteSpec, s.Namespace(), s.Namespace())) - time.Sleep(6 * time.Second) - services, err := s.DefaultDataplaneResource().Service().List(context.Background()) - Expect(err).ShouldNot(HaveOccurred(), "list services") - assert.Len(GinkgoT(), services, 1, "there should be one service") - service := services[0] - Expect(service.Upstream.TLS).ShouldNot(BeNil(), "check tls in service") + Eventually(func(g Gomega) { + services, err := s.DefaultDataplaneResource().Service().List(context.Background()) + g.Expect(err).ShouldNot(HaveOccurred(), "list services") + g.Expect(services).To(HaveLen(1), "there should be one service") + g.Expect(services[0].Upstream.TLS).ShouldNot(BeNil(), "check tls in service") + }).WithTimeout(scaffold.DefaultTimeout).ProbeEvery(scaffold.DefaultInterval).Should(Succeed()) }) }) diff --git a/test/e2e/crds/v2/status.go b/test/e2e/crds/v2/status.go index 6988c94d..4530f2d0 100644 --- a/test/e2e/crds/v2/status.go +++ b/test/e2e/crds/v2/status.go @@ -48,12 +48,10 @@ var _ = Describe("Test apisix.apache.org/v2 Status", Label("apisix.apache.org", By("create GatewayProxy") err := s.CreateResourceFromString(s.GetGatewayProxySpec()) Expect(err).NotTo(HaveOccurred(), "creating GatewayProxy") - time.Sleep(5 * time.Second) By("create IngressClass") err = s.CreateResourceFromStringWithNamespace(s.GetIngressClassYaml(), "") Expect(err).NotTo(HaveOccurred(), "creating IngressClass") - time.Sleep(5 * time.Second) }) const ar = ` apiVersion: apisix.apache.org/v2 @@ -270,17 +268,16 @@ spec: err := s.CreateResourceFromString(arWithInvalidIngressClass) Expect(err).NotTo(HaveOccurred(), "creating ApisixRoute with invalid IngressClass") - for range 10 { + Consistently(func(g Gomega) { output, err := s.GetOutputFromString("ar", "ar-with-invalid-ingressclass", "-o", "yaml") - Expect(err).NotTo(HaveOccurred(), "getting ApisixRoute output") - Expect(output).ShouldNot( + g.Expect(err).NotTo(HaveOccurred(), "getting ApisixRoute output") + g.Expect(output).ShouldNot( Or( ContainSubstring(`status: "True"`), ContainSubstring(`status: "False"`), ), ) - time.Sleep(1 * time.Second) - } + }).WithTimeout(10 * time.Second).ProbeEvery(time.Second).Should(Succeed()) }) }) }) diff --git a/test/e2e/crds/v2/streamroute.go b/test/e2e/crds/v2/streamroute.go index ced79f7e..151f6cd8 100644 --- a/test/e2e/crds/v2/streamroute.go +++ b/test/e2e/crds/v2/streamroute.go @@ -41,12 +41,10 @@ var _ = Describe("Test ApisixRoute With StreamRoute", Label("apisix.apache.org", gatewayProxy := s.GetGatewayProxySpec() err := s.CreateResourceFromString(gatewayProxy) Expect(err).NotTo(HaveOccurred(), "creating GatewayProxy") - time.Sleep(5 * time.Second) By("create IngressClass") err = s.CreateResourceFromStringWithNamespace(s.GetIngressClassYaml(), "") Expect(err).NotTo(HaveOccurred(), "creating IngressClass") - time.Sleep(5 * time.Second) }) Context("TCP Proxy", func() { @@ -130,18 +128,20 @@ spec: dnsSvc := s.NewCoreDNSService() err := s.CreateResourceFromString(fmt.Sprintf(apisixRoute, s.Namespace(), dnsSvc.Name, dnsSvc.Spec.Ports[0].Port)) Expect(err).NotTo(HaveOccurred(), "creating ApisixRoute") - time.Sleep(20 * time.Second) svc := s.GetDataplaneService() // test dns query - output, err := s.RunDigDNSClientFromK8s(fmt.Sprintf("@%s", svc.Name), "-p", "9200", "github.com") - Expect(err).NotTo(HaveOccurred(), "dig github.com via apisix udp proxy") - Expect(output).To(ContainSubstring("ADDITIONAL SECTION")) - - time.Sleep(3 * time.Second) - output = s.GetDeploymentLogs(scaffold.CoreDNSDeployment) - Expect(output).To(ContainSubstring("github.com. udp")) + Eventually(func(g Gomega) { + output, err := s.RunDigDNSClientFromK8s(fmt.Sprintf("@%s", svc.Name), "-p", "9200", "github.com") + g.Expect(err).NotTo(HaveOccurred(), "dig github.com via apisix udp proxy") + g.Expect(output).To(ContainSubstring("ADDITIONAL SECTION")) + }).WithTimeout(scaffold.DefaultTimeout).ProbeEvery(scaffold.DefaultInterval).Should(Succeed()) + + Eventually(func(g Gomega) { + output := s.GetDeploymentLogs(scaffold.CoreDNSDeployment) + g.Expect(output).To(ContainSubstring("github.com. udp")) + }).WithTimeout(scaffold.DefaultTimeout).ProbeEvery(scaffold.DefaultInterval).Should(Succeed()) }) }) diff --git a/test/e2e/crds/v2/tls.go b/test/e2e/crds/v2/tls.go index b8282d76..5a64cebc 100644 --- a/test/e2e/crds/v2/tls.go +++ b/test/e2e/crds/v2/tls.go @@ -68,12 +68,10 @@ var _ = Describe("Test ApisixTls", Label("apisix.apache.org", "v2", "apisixtls") By("create GatewayProxy") err := s.CreateResourceFromString(s.GetGatewayProxySpec()) Expect(err).NotTo(HaveOccurred(), "creating GatewayProxy") - time.Sleep(5 * time.Second) By("create IngressClass") err = s.CreateResourceFromStringWithNamespace(s.GetIngressClassYaml(), "") Expect(err).NotTo(HaveOccurred(), "creating IngressClass") - time.Sleep(5 * time.Second) By("create ApisixRoute for TLS testing") var apisixRoute apiv2.ApisixRoute diff --git a/test/e2e/crds/v2/upstream.go b/test/e2e/crds/v2/upstream.go index a86c4a8d..a95e35ab 100644 --- a/test/e2e/crds/v2/upstream.go +++ b/test/e2e/crds/v2/upstream.go @@ -20,7 +20,6 @@ package v2 import ( "context" "fmt" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -41,12 +40,10 @@ var _ = Describe("Test ApisixUpstream", Label("apisix.apache.org", "v2", "apisix By("create GatewayProxy") err := s.CreateResourceFromString(s.GetGatewayProxySpec()) Expect(err).NotTo(HaveOccurred(), "creating GatewayProxy") - time.Sleep(5 * time.Second) By("create IngressClass") err = s.CreateResourceFromStringWithNamespace(s.GetIngressClassYaml(), "") Expect(err).NotTo(HaveOccurred(), "creating IngressClass") - time.Sleep(5 * time.Second) }) Context("Health Check", func() { @@ -111,25 +108,25 @@ spec: Path: "/ip", Host: "httpbin.org", }) - time.Sleep(2 * time.Second) - - ups, err := s.Deployer.DefaultDataplaneResource().Upstream().List(context.Background()) - Expect(err).ToNot(HaveOccurred(), "listing upstreams") - Expect(ups).To(HaveLen(1), "the number of upstreams") - Expect(ups[0].Nodes).To(HaveLen(3), "the number of upstream nodes") - Expect(ups[0].Checks).ToNot(BeNil(), "the healthcheck configuration") - Expect(ups[0].Checks.Active).ToNot(BeNil(), "the active healthcheck configuration") - Expect(ups[0].Checks.Active.Healthy).ToNot(BeNil(), "the active healthy configuration") - Expect(ups[0].Checks.Active.Unhealthy).ToNot(BeNil(), "the active unhealthy configuration") - Expect(ups[0].Checks.Active.Healthy.Interval).To(Equal(1), "the healthy interval") - Expect(ups[0].Checks.Active.Healthy.HTTPStatuses).To(Equal([]int{200}), "the healthy http status") - Expect(ups[0].Checks.Active.Unhealthy.Interval).To(Equal(1), "the unhealthy interval") - Expect(ups[0].Checks.Active.Unhealthy.HTTPFailures).To(Equal(2), "the unhealthy http failures") - Expect(ups[0].Checks.Passive).ToNot(BeNil(), "the passive healthcheck configuration") - Expect(ups[0].Checks.Passive.Healthy).ToNot(BeNil(), "the passive healthy configuration") - Expect(ups[0].Checks.Passive.Unhealthy).ToNot(BeNil(), "the passive unhealthy configuration") - Expect(ups[0].Checks.Passive.Healthy.HTTPStatuses).To(Equal([]int{200}), "the passive healthy http status") - Expect(ups[0].Checks.Passive.Unhealthy.HTTPStatuses).To(Equal([]int{502}), "the passive unhealthy http status") + Eventually(func(g Gomega) { + ups, err := s.Deployer.DefaultDataplaneResource().Upstream().List(context.Background()) + g.Expect(err).ToNot(HaveOccurred(), "listing upstreams") + g.Expect(ups).To(HaveLen(1), "the number of upstreams") + g.Expect(ups[0].Nodes).To(HaveLen(3), "the number of upstream nodes") + g.Expect(ups[0].Checks).ToNot(BeNil(), "the healthcheck configuration") + g.Expect(ups[0].Checks.Active).ToNot(BeNil(), "the active healthcheck configuration") + g.Expect(ups[0].Checks.Active.Healthy).ToNot(BeNil(), "the active healthy configuration") + g.Expect(ups[0].Checks.Active.Unhealthy).ToNot(BeNil(), "the active unhealthy configuration") + g.Expect(ups[0].Checks.Active.Healthy.Interval).To(Equal(1), "the healthy interval") + g.Expect(ups[0].Checks.Active.Healthy.HTTPStatuses).To(Equal([]int{200}), "the healthy http status") + g.Expect(ups[0].Checks.Active.Unhealthy.Interval).To(Equal(1), "the unhealthy interval") + g.Expect(ups[0].Checks.Active.Unhealthy.HTTPFailures).To(Equal(2), "the unhealthy http failures") + g.Expect(ups[0].Checks.Passive).ToNot(BeNil(), "the passive healthcheck configuration") + g.Expect(ups[0].Checks.Passive.Healthy).ToNot(BeNil(), "the passive healthy configuration") + g.Expect(ups[0].Checks.Passive.Unhealthy).ToNot(BeNil(), "the passive unhealthy configuration") + g.Expect(ups[0].Checks.Passive.Healthy.HTTPStatuses).To(Equal([]int{200}), "the passive healthy http status") + g.Expect(ups[0].Checks.Passive.Unhealthy.HTTPStatuses).To(Equal([]int{502}), "the passive unhealthy http status") + }).WithTimeout(scaffold.DefaultTimeout).ProbeEvery(scaffold.DefaultInterval).Should(Succeed()) for range 100 { s.NewAPISIXClient().GET("/ip").WithHost("httpbin.org").Expect().Status(200) diff --git a/test/e2e/gatewayapi/controller.go b/test/e2e/gatewayapi/controller.go index 817e9e17..492ffabc 100644 --- a/test/e2e/gatewayapi/controller.go +++ b/test/e2e/gatewayapi/controller.go @@ -93,7 +93,6 @@ spec: ), fmt.Sprintf("checking %s condition status", resourType), ) - time.Sleep(3 * time.Second) } var beforeEach = func(s *scaffold.Scaffold) { By(fmt.Sprintf("create GatewayClass for controller %s", s.GetControllerName())) @@ -104,29 +103,30 @@ spec: gatewayProxy := fmt.Sprintf(gatewayProxyYaml, gatewayProxyName, s.Deployer.GetAdminEndpoint(), s.AdminKey()) err := s.CreateResourceFromStringWithNamespace(gatewayProxy, gatewayName) Expect(err).NotTo(HaveOccurred(), "creating GatewayProxy") - time.Sleep(5 * time.Second) gatewayClassName := s.Namespace() err = s.CreateResourceFromStringWithNamespace(fmt.Sprintf(defautlGatewayClass, gatewayClassName, gatewayName, s.GetControllerName()), gatewayName) Expect(err).NotTo(HaveOccurred(), "creating GatewayClass") - time.Sleep(10 * time.Second) By("check GatewayClass condition") - gcyaml, err := s.GetResourceYamlFromNamespace(types.KindGatewayClass, gatewayClassName, s.Namespace()) - Expect(err).NotTo(HaveOccurred(), "getting GatewayClass yaml") - Expect(gcyaml).To(ContainSubstring(`status: "True"`), "checking GatewayClass condition status") - Expect(gcyaml).To(ContainSubstring("message: the gatewayclass has been accepted by the apisix-ingress-controller"), "checking GatewayClass condition message") + s.RetryAssertion(func() (string, error) { + return s.GetResourceYamlFromNamespace(types.KindGatewayClass, gatewayClassName, s.Namespace()) + }).Should(And( + ContainSubstring(`status: "True"`), + ContainSubstring("message: the gatewayclass has been accepted by the apisix-ingress-controller"), + ), "checking GatewayClass condition") By("create Gateway") err = s.CreateResourceFromStringWithNamespace(fmt.Sprintf(defautlGateway, gatewayName, gatewayName, gatewayClassName, gatewayProxyName), gatewayName) Expect(err).NotTo(HaveOccurred(), "creating Gateway") - time.Sleep(10 * time.Second) By("check Gateway condition") - gwyaml, err := s.GetResourceYamlFromNamespace(types.KindGateway, gatewayName, gatewayName) - Expect(err).NotTo(HaveOccurred(), "getting Gateway yaml") - Expect(gwyaml).To(ContainSubstring(`status: "True"`), "checking Gateway condition status") - Expect(gwyaml).To(ContainSubstring("message: the gateway has been accepted by the apisix-ingress-controller"), "checking Gateway condition message") + s.RetryAssertion(func() (string, error) { + return s.GetResourceYamlFromNamespace(types.KindGateway, gatewayName, gatewayName) + }).Should(And( + ContainSubstring(`status: "True"`), + ContainSubstring("message: the gateway has been accepted by the apisix-ingress-controller"), + ), "checking Gateway condition") } Context("Create resource with first controller", func() { @@ -212,7 +212,12 @@ spec: }) It("Apply resource ", func() { ResourceApplied(s2, types.KindHTTPRoute, "httpbin2", s2.Namespace(), fmt.Sprintf(route2, s2.Namespace(), s2.Namespace()), 1) - time.Sleep(5 * time.Second) + + s2.RetryAssertion(func() int { + routes, _ := s2.DefaultDataplaneResource().Route().List(s2.Context) + return len(routes) + }).WithInterval(3*time.Second).Should(Equal(1), "checking route count") + routes, err := s2.DefaultDataplaneResource().Route().List(s2.Context) Expect(err).NotTo(HaveOccurred()) Expect(routes).To(HaveLen(1)) diff --git a/test/e2e/gatewayapi/gateway.go b/test/e2e/gatewayapi/gateway.go index a5f19cd7..da00b918 100644 --- a/test/e2e/gatewayapi/gateway.go +++ b/test/e2e/gatewayapi/gateway.go @@ -118,41 +118,41 @@ spec: gatewayProxy := fmt.Sprintf(gatewayProxyYaml, s.Namespace(), s.Deployer.GetAdminEndpoint(), s.AdminKey()) err := s.CreateResourceFromString(gatewayProxy) Expect(err).NotTo(HaveOccurred(), "creating GatewayProxy") - time.Sleep(5 * time.Second) By("create GatewayClass") err = s.CreateResourceFromStringWithNamespace(fmt.Sprintf(defaultGatewayClass, gatewayClassName, s.GetControllerName()), "") Expect(err).NotTo(HaveOccurred(), "creating GatewayClass") - time.Sleep(5 * time.Second) By("check GatewayClass condition") - gcyaml, err := s.GetResourceYaml("GatewayClass", gatewayClassName) - Expect(err).NotTo(HaveOccurred(), "getting GatewayClass yaml") - Expect(gcyaml).To(ContainSubstring(`status: "True"`), "checking GatewayClass condition status") - Expect(gcyaml).To(ContainSubstring("message: the gatewayclass has been accepted by the apisix-ingress-controller"), "checking GatewayClass condition message") + s.RetryAssertion(func() (string, error) { + return s.GetResourceYaml("GatewayClass", gatewayClassName) + }).Should(And( + ContainSubstring(`status: "True"`), + ContainSubstring("message: the gatewayclass has been accepted by the apisix-ingress-controller"), + ), "checking GatewayClass condition") By("create Gateway") gatewayName := s.Namespace() err = s.CreateResourceFromStringWithNamespace(fmt.Sprintf(defaultGateway, gatewayName, gatewayClassName), s. Namespace()) Expect(err).NotTo(HaveOccurred(), "creating Gateway") - time.Sleep(5 * time.Second) By("check Gateway condition") - gwyaml, err := s.GetResourceYaml("Gateway", gatewayName) - Expect(err).NotTo(HaveOccurred(), "getting Gateway yaml") - Expect(gwyaml).To(ContainSubstring(`status: "True"`), "checking Gateway condition status") - Expect(gwyaml).To(ContainSubstring("message: the gateway has been accepted by the apisix-ingress-controller"), "checking Gateway condition message") + s.RetryAssertion(func() (string, error) { + return s.GetResourceYaml("Gateway", gatewayName) + }).Should(And( + ContainSubstring(`status: "True"`), + ContainSubstring("message: the gateway has been accepted by the apisix-ingress-controller"), + ), "checking Gateway condition") By("create Gateway with not accepted GatewayClass") err = s.CreateResourceFromStringWithNamespace(noClassGateway, s.Namespace()) Expect(err).NotTo(HaveOccurred(), "creating Gateway") - time.Sleep(5 * time.Second) By("check Gateway condition") - gwyaml, err = s.GetResourceYaml("Gateway", "apisix-not-class") - Expect(err).NotTo(HaveOccurred(), "getting Gateway yaml") - Expect(gwyaml).To(ContainSubstring(`status: Unknown`), "checking Gateway condition status") + s.RetryAssertion(func() (string, error) { + return s.GetResourceYaml("Gateway", "apisix-not-class") + }).Should(ContainSubstring(`status: Unknown`), "checking Gateway condition status") }) }) @@ -162,7 +162,6 @@ spec: gatewayProxy := fmt.Sprintf(gatewayProxyYaml, s.Namespace(), s.Deployer.GetAdminEndpoint(), s.AdminKey()) err := s.CreateResourceFromString(gatewayProxy) Expect(err).NotTo(HaveOccurred(), "creating GatewayProxy") - time.Sleep(5 * time.Second) By("create secret") secretName := _secretName @@ -204,19 +203,19 @@ spec: By("create GatewayClass") err = s.CreateResourceFromStringWithNamespace(fmt.Sprintf(defaultGatewayClass, gatewayClassName, s.GetControllerName()), "") Expect(err).NotTo(HaveOccurred(), "creating GatewayClass") - time.Sleep(5 * time.Second) By("create Gateway") err = s.CreateResourceFromStringWithNamespace(defaultGateway, s.Namespace()) Expect(err).NotTo(HaveOccurred(), "creating Gateway") - time.Sleep(10 * time.Second) - - tls, err := s.DefaultDataplaneResource().SSL().List(context.Background()) - assert.Nil(GinkgoT(), err, "list tls error") - assert.Len(GinkgoT(), tls, 1, "tls number not expect") - assert.Len(GinkgoT(), tls[0].Certificates, 1, "length of certificates not expect") - assert.Equal(GinkgoT(), Cert, tls[0].Certificates[0].Certificate, "tls cert not expect") - assert.ElementsMatch(GinkgoT(), []string{host}, tls[0].Snis) + + Eventually(func(g Gomega) { + tls, err := s.DefaultDataplaneResource().SSL().List(context.Background()) + g.Expect(err).NotTo(HaveOccurred(), "list tls error") + g.Expect(tls).To(HaveLen(1), "tls number not expect") + g.Expect(tls[0].Certificates).To(HaveLen(1), "length of certificates not expect") + g.Expect(tls[0].Certificates[0].Certificate).To(Equal(Cert), "tls cert not expect") + g.Expect(tls[0].Snis).To(ConsistOf(host)) + }).WithTimeout(scaffold.DefaultTimeout).ProbeEvery(scaffold.DefaultInterval).Should(Succeed()) }) It("Gateway SSL with and without hostname", func() { @@ -224,7 +223,6 @@ spec: gatewayProxy := fmt.Sprintf(gatewayProxyYaml, s.Namespace(), s.Deployer.GetAdminEndpoint(), s.AdminKey()) err := s.CreateResourceFromString(gatewayProxy) Expect(err).NotTo(HaveOccurred(), "creating GatewayProxy") - time.Sleep(5 * time.Second) secretName := _secretName createSecret(s, secretName) @@ -278,7 +276,6 @@ spec: By("create GatewayClass") err = s.CreateResourceFromStringWithNamespace(fmt.Sprintf(defaultGatewayClass, gatewayClassName, s.GetControllerName()), "") Expect(err).NotTo(HaveOccurred(), "creating GatewayClass") - time.Sleep(5 * time.Second) By("create Gateway") err = s.CreateResourceFromStringWithNamespace(defaultGateway, s.Namespace()) diff --git a/test/e2e/gatewayapi/gatewayclass.go b/test/e2e/gatewayapi/gatewayclass.go index 4ecc8c19..ff6b75ee 100644 --- a/test/e2e/gatewayapi/gatewayclass.go +++ b/test/e2e/gatewayapi/gatewayclass.go @@ -65,22 +65,24 @@ spec: By("create default GatewayClass") err := s.CreateResourceFromString(fmt.Sprintf(defautlGatewayClass, gatewayClassName, s.GetControllerName())) Expect(err).NotTo(HaveOccurred(), "creating GatewayClass") - time.Sleep(5 * time.Second) - gcyaml, err := s.GetResourceYaml("GatewayClass", gatewayClassName) - Expect(err).NotTo(HaveOccurred(), "getting GatewayClass yaml") - Expect(gcyaml).To(ContainSubstring(`status: "True"`), "checking GatewayClass condition status") - Expect(gcyaml).To(ContainSubstring("message: the gatewayclass has been accepted by the apisix-ingress-controller"), "checking GatewayClass condition message") + s.RetryAssertion(func() (string, error) { + return s.GetResourceYaml("GatewayClass", gatewayClassName) + }).Should(And( + ContainSubstring(`status: "True"`), + ContainSubstring("message: the gatewayclass has been accepted by the apisix-ingress-controller"), + ), "checking GatewayClass condition") By("create GatewayClass with not accepted") err = s.CreateResourceFromString(noGatewayClass) Expect(err).NotTo(HaveOccurred(), "creating GatewayClass") - time.Sleep(5 * time.Second) - gcyaml, err = s.GetResourceYaml("GatewayClass", "apisix-not-accepeted") - Expect(err).NotTo(HaveOccurred(), "getting GatewayClass yaml") - Expect(gcyaml).To(ContainSubstring(`status: Unknown`), "checking GatewayClass condition status") - Expect(gcyaml).To(ContainSubstring("message: Waiting for controller"), "checking GatewayClass condition message") + s.RetryAssertion(func() (string, error) { + return s.GetResourceYaml("GatewayClass", "apisix-not-accepeted") + }).Should(And( + ContainSubstring(`status: Unknown`), + ContainSubstring("message: Waiting for controller"), + ), "checking GatewayClass condition") }) It("Delete GatewayClass", func() { @@ -98,7 +100,6 @@ spec: gatewayName := s.Namespace() err = s.CreateResourceFromString(fmt.Sprintf(defaultGateway, gatewayName, gatewayClassName)) Expect(err).NotTo(HaveOccurred(), "creating Gateway") - time.Sleep(time.Second) By("try to delete the GatewayClass") _, err = s.RunKubectlAndGetOutput("delete", "GatewayClass", gatewayClassName, "--wait=false") @@ -107,27 +108,30 @@ spec: _, err = s.GetResourceYaml("GatewayClass", gatewayClassName) Expect(err).NotTo(HaveOccurred(), "get resource yaml") - output, err := s.RunKubectlAndGetOutput("describe", "GatewayClass", gatewayClassName) - Expect(err).NotTo(HaveOccurred(), "describe GatewayClass apisix") - Expect(output).To(And( - ContainSubstring("Warning"), - ContainSubstring("DeletionBlocked"), - ContainSubstring("gatewayclass-controller"), - ContainSubstring("the GatewayClass is still used by Gateways"), - )) + Eventually(func(g Gomega) { + output, err := s.RunKubectlAndGetOutput("describe", "GatewayClass", gatewayClassName) + g.Expect(err).NotTo(HaveOccurred(), "describe GatewayClass apisix") + g.Expect(output).To(And( + ContainSubstring("Warning"), + ContainSubstring("DeletionBlocked"), + ContainSubstring("gatewayclass-controller"), + ContainSubstring("the GatewayClass is still used by Gateways"), + )) + }).WithTimeout(scaffold.DefaultTimeout).ProbeEvery(scaffold.DefaultInterval).Should(Succeed()) By("delete the Gateway") err = s.DeleteResource("Gateway", gatewayName) Expect(err).NotTo(HaveOccurred(), "deleting Gateway") - time.Sleep(time.Second) By("try to delete the GatewayClass again") err = s.DeleteResource("GatewayClass", gatewayClassName) Expect(err).NotTo(HaveOccurred()) - _, err = s.GetResourceYaml("GatewayClass", gatewayClassName) - Expect(err).To(HaveOccurred(), "get resource yaml") - Expect(err.Error()).To(ContainSubstring("not found")) + Eventually(func(g Gomega) { + _, err := s.GetResourceYaml("GatewayClass", gatewayClassName) + g.Expect(err).To(HaveOccurred(), "get resource yaml") + g.Expect(err.Error()).To(ContainSubstring("not found")) + }).WithTimeout(scaffold.DefaultTimeout).ProbeEvery(scaffold.DefaultInterval).Should(Succeed()) }) }) }) diff --git a/test/e2e/gatewayapi/gatewayproxy.go b/test/e2e/gatewayapi/gatewayproxy.go index 99283cac..1ed877e7 100644 --- a/test/e2e/gatewayapi/gatewayproxy.go +++ b/test/e2e/gatewayapi/gatewayproxy.go @@ -19,7 +19,6 @@ package gatewayapi import ( "fmt" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -133,29 +132,30 @@ spec: gatewayClassName = s.Namespace() err := s.CreateResourceFromStringWithNamespace(fmt.Sprintf(defaultGatewayClass, gatewayClassName, s.GetControllerName()), "") Expect(err).NotTo(HaveOccurred(), "creating GatewayClass") - time.Sleep(5 * time.Second) By("Check GatewayClass condition") - gcYaml, err := s.GetResourceYaml(types.KindGatewayClass, gatewayClassName) - Expect(err).NotTo(HaveOccurred(), "getting GatewayClass yaml") - Expect(gcYaml).To(ContainSubstring(`status: "True"`), "checking GatewayClass condition status") - Expect(gcYaml).To(ContainSubstring("message: the gatewayclass has been accepted by the apisix-ingress-controller"), "checking GatewayClass condition message") + s.RetryAssertion(func() (string, error) { + return s.GetResourceYaml(types.KindGatewayClass, gatewayClassName) + }).Should(And( + ContainSubstring(`status: "True"`), + ContainSubstring("message: the gatewayclass has been accepted by the apisix-ingress-controller"), + ), "checking GatewayClass condition") By("Create GatewayProxy with enabled plugin") err = s.CreateResourceFromString(fmt.Sprintf(gatewayProxyWithEnabledPlugin, s.Namespace(), s.Deployer.GetAdminEndpoint(), s.AdminKey())) Expect(err).NotTo(HaveOccurred(), "creating GatewayProxy with enabled plugin") - time.Sleep(5 * time.Second) By("Create Gateway with GatewayProxy") err = s.CreateResourceFromStringWithNamespace(fmt.Sprintf(gatewayWithProxy, s.Namespace(), gatewayClassName, s.Namespace()), s.Namespace()) Expect(err).NotTo(HaveOccurred(), "creating Gateway with GatewayProxy") - time.Sleep(5 * time.Second) By("check Gateway condition") - gwyaml, err := s.GetResourceYaml(types.KindGateway, s.Namespace()) - Expect(err).NotTo(HaveOccurred(), "getting Gateway yaml") - Expect(gwyaml).To(ContainSubstring(`status: "True"`), "checking Gateway condition status") - Expect(gwyaml).To(ContainSubstring("message: the gateway has been accepted by the apisix-ingress-controller"), "checking Gateway condition message") + s.RetryAssertion(func() (string, error) { + return s.GetResourceYaml(types.KindGateway, s.Namespace()) + }).Should(And( + ContainSubstring(`status: "True"`), + ContainSubstring("message: the gateway has been accepted by the apisix-ingress-controller"), + ), "checking Gateway condition") }) Context("Test Gateway with enabled GatewayProxy plugin", func() { diff --git a/test/e2e/gatewayapi/httproute.go b/test/e2e/gatewayapi/httproute.go index e9432002..8e869bc5 100644 --- a/test/e2e/gatewayapi/httproute.go +++ b/test/e2e/gatewayapi/httproute.go @@ -2159,7 +2159,6 @@ spec: }) It("HTTPRoute Canary", func() { s.ResourceApplied("HTTPRoute", "httpbin", fmt.Sprintf(sameWeiht, s.Namespace()), 1) - time.Sleep(5 * time.Second) s.RetryAssertion(func() int { var ( @@ -2368,7 +2367,6 @@ spec: By("apply services and HTTPRoute") err := s.CreateResourceFromStringWithNamespace(fmt.Sprintf(servicesSpec, s.Namespace()), s.Namespace()) Expect(err).ShouldNot(HaveOccurred(), "apply services and HTTPRoute") - time.Sleep(10 * time.Second) By("verify load balancing works") s.RequestAssert(&scaffold.RequestAssert{ @@ -2392,7 +2390,6 @@ spec: case http.StatusMovedPermanently: upstreamHosts["mock.api7.ai"]++ } - time.Sleep(100 * time.Millisecond) // Small delay between requests } By("verify both upstreams received requests") diff --git a/test/e2e/gatewayapi/status.go b/test/e2e/gatewayapi/status.go index 999ec31c..2a7dce48 100644 --- a/test/e2e/gatewayapi/status.go +++ b/test/e2e/gatewayapi/status.go @@ -88,24 +88,23 @@ spec: By("create GatewayProxy") err := s.CreateResourceFromString(s.GetGatewayProxySpec()) Expect(err).NotTo(HaveOccurred(), "creating GatewayProxy") - time.Sleep(5 * time.Second) By("create GatewayClass") gatewayClassName := s.Namespace() err = s.CreateResourceFromStringWithNamespace(fmt.Sprintf(gatewayClass, gatewayClassName, s.GetControllerName()), "") Expect(err).NotTo(HaveOccurred(), "creating GatewayClass") - time.Sleep(5 * time.Second) By("create Gateway") err = s.CreateResourceFromStringWithNamespace(fmt.Sprintf(defaultGateway, gatewayClassName), s.Namespace()) Expect(err).NotTo(HaveOccurred(), "creating Gateway") - time.Sleep(5 * time.Second) By("check Gateway condition") - gwyaml, err := s.GetResourceYaml("Gateway", "apisix") - Expect(err).NotTo(HaveOccurred(), "getting Gateway yaml") - Expect(gwyaml).To(ContainSubstring(`status: "True"`), "checking Gateway condition status") - Expect(gwyaml).To(ContainSubstring("message: the gateway has been accepted by the apisix-ingress-controller"), "checking Gateway condition message") + s.RetryAssertion(func() (string, error) { + return s.GetResourceYaml("Gateway", "apisix") + }).Should(And( + ContainSubstring(`status: "True"`), + ContainSubstring("message: the gateway has been accepted by the apisix-ingress-controller"), + ), "checking Gateway condition") }) It("dataplane unavailable", func() { diff --git a/test/e2e/gatewayapi/tcproute.go b/test/e2e/gatewayapi/tcproute.go index c8ac7ba7..080bfdb6 100644 --- a/test/e2e/gatewayapi/tcproute.go +++ b/test/e2e/gatewayapi/tcproute.go @@ -87,7 +87,6 @@ spec: By("creating TCPRoute") Expect(s.CreateResourceFromString(fmt.Sprintf(tcpRoute, s.Namespace()))). NotTo(HaveOccurred(), "creating TCPRoute") - time.Sleep(2 * time.Second) By("verifying TCPRoute is functional") s.HTTPOverTCPConnectAssert(true, time.Minute*3) // should be able to connect diff --git a/test/e2e/gatewayapi/udproute.go b/test/e2e/gatewayapi/udproute.go index 1c2ea7c5..f59737f2 100644 --- a/test/e2e/gatewayapi/udproute.go +++ b/test/e2e/gatewayapi/udproute.go @@ -19,7 +19,6 @@ package gatewayapi import ( "fmt" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -90,9 +89,9 @@ spec: Expect(err).NotTo(HaveOccurred(), "dig github.com via apisix udp proxy") Expect(output).To(ContainSubstring("ADDITIONAL SECTION")) - time.Sleep(3 * time.Second) - output = s.GetDeploymentLogs(scaffold.CoreDNSDeployment) - Expect(output).To(ContainSubstring("github.com. udp")) + Eventually(func(g Gomega) { + g.Expect(s.GetDeploymentLogs(scaffold.CoreDNSDeployment)).To(ContainSubstring("github.com. udp")) + }).WithTimeout(scaffold.DefaultTimeout).ProbeEvery(scaffold.DefaultInterval).Should(Succeed()) }) }) }) diff --git a/test/e2e/ingress/annotations.go b/test/e2e/ingress/annotations.go index a65cddbc..1a522c9b 100644 --- a/test/e2e/ingress/annotations.go +++ b/test/e2e/ingress/annotations.go @@ -162,7 +162,6 @@ spec: By("create IngressClass") err := s.CreateResourceFromStringWithNamespace(s.GetIngressClassYaml(), "") Expect(err).NotTo(HaveOccurred(), "creating IngressClass") - time.Sleep(5 * time.Second) }) It("retries", func() { Expect(s.CreateResourceFromString(fmt.Sprintf(ingressRetries, s.Namespace()))).ShouldNot(HaveOccurred(), "creating Ingress") @@ -606,7 +605,6 @@ spec: By("create IngressClass") err := s.CreateResourceFromStringWithNamespace(s.GetIngressClassYaml(), "") Expect(err).NotTo(HaveOccurred(), "creating IngressClass") - time.Sleep(5 * time.Second) }) It("redirect", func() { Expect(s.CreateResourceFromString(fmt.Sprintf(tohttps, s.Namespace()))).ShouldNot(HaveOccurred(), "creating Ingress") @@ -643,17 +641,16 @@ spec: It("csrf", func() { Expect(s.CreateResourceFromString(fmt.Sprintf(ingressCSRF, s.Namespace()))).ShouldNot(HaveOccurred(), "creating Ingress") - time.Sleep(5 * time.Second) - By("Request without CSRF token should fail") - msg401 := s.NewAPISIXClient(). - POST("/anything"). - WithHeader("Host", "httpbin.example"). - Expect(). - Status(http.StatusUnauthorized). - Body(). - Raw() - Expect(msg401).To(ContainSubstring("no csrf token in headers"), "checking error message") + s.RequestAssert(&scaffold.RequestAssert{ + Method: "POST", + Path: "/anything", + Host: "httpbin.example", + Checks: []scaffold.ResponseCheckFunc{ + scaffold.WithExpectedStatus(http.StatusUnauthorized), + scaffold.WithExpectedBodyContains("no csrf token in headers"), + }, + }) By("GET request should succeed and return CSRF token in cookie") resp := s.NewAPISIXClient(). @@ -1128,7 +1125,6 @@ spec: By("create IngressClass") err = s.CreateResourceFromStringWithNamespace(s.GetIngressClassYaml(), "") Expect(err).NotTo(HaveOccurred(), "creating IngressClass") - time.Sleep(5 * time.Second) }) AfterEach(func() { s.DeleteNamespace(ns) @@ -1176,7 +1172,6 @@ spec: By("create IngressClass") err := s.CreateResourceFromStringWithNamespace(s.GetIngressClassYaml(), "") Expect(err).NotTo(HaveOccurred(), "creating IngressClass") - time.Sleep(5 * time.Second) }) It("regex match", func() { Expect(s.CreateResourceFromString(fmt.Sprintf(ingressRegex, s.Namespace()))).ShouldNot(HaveOccurred(), "creating Ingress") diff --git a/test/e2e/ingress/ingress.go b/test/e2e/ingress/ingress.go index 93cb486a..f1493541 100644 --- a/test/e2e/ingress/ingress.go +++ b/test/e2e/ingress/ingress.go @@ -78,7 +78,6 @@ spec: By("create GatewayProxy") err := s.CreateResourceFromStringWithNamespace(gatewayProxy, s.Namespace()) Expect(err).NotTo(HaveOccurred(), "creating GatewayProxy") - time.Sleep(5 * time.Second) secretName := _secretName host := "api6.com" @@ -126,20 +125,31 @@ spec: By("create IngressClass") err = s.CreateResourceFromStringWithNamespace(fmt.Sprintf(defaultIngressClass, s.Namespace(), s.GetControllerName(), s.Namespace()), "") Expect(err).NotTo(HaveOccurred(), "creating IngressClass") - time.Sleep(5 * time.Second) By("create Ingress with TLS") err = s.CreateResourceFromStringWithNamespace(tlsIngress, s.Namespace()) Expect(err).NotTo(HaveOccurred(), "creating Ingress with TLS") - time.Sleep(5 * time.Second) By("check TLS configuration") - tls, err := s.DefaultDataplaneResource().SSL().List(context.Background()) - assert.Nil(GinkgoT(), err, "list tls error") - assert.Len(GinkgoT(), tls, 1, "tls number not expect") - assert.Len(GinkgoT(), tls[0].Certificates, 1, "length of certificates not expect") - assert.Equal(GinkgoT(), Cert, tls[0].Certificates[0].Certificate, "tls cert not expect") - assert.ElementsMatch(GinkgoT(), []string{host}, tls[0].Snis) + s.RetryAssertion(func() error { + tls, err := s.DefaultDataplaneResource().SSL().List(context.Background()) + if err != nil { + return err + } + if len(tls) != 1 { + return fmt.Errorf("expected 1 tls, got %d", len(tls)) + } + if len(tls[0].Certificates) != 1 { + return fmt.Errorf("expected 1 certificate, got %d", len(tls[0].Certificates)) + } + if tls[0].Certificates[0].Certificate != Cert { + return fmt.Errorf("tls cert not expect") + } + if !assert.ObjectsAreEqual([]string{host}, tls[0].Snis) { + return fmt.Errorf("expected snis %v, got %v", []string{host}, tls[0].Snis) + } + return nil + }).ShouldNot(HaveOccurred(), "check TLS configuration") }) }) @@ -262,20 +272,16 @@ spec: gatewayProxy := fmt.Sprintf(gatewayProxyYaml, s.Namespace(), s.Deployer.GetAdminEndpoint(), s.AdminKey()) err := s.CreateResourceFromStringWithNamespace(gatewayProxy, s.Namespace()) Expect(err).NotTo(HaveOccurred(), "creating GatewayProxy") - time.Sleep(5 * time.Second) By("create Default IngressClass") err = s.CreateResourceFromStringWithNamespace(fmt.Sprintf(defaultIngressClass, s.GetControllerName(), s.Namespace()), s.Namespace()) Expect(err).NotTo(HaveOccurred(), "creating Default IngressClass") - time.Sleep(5 * time.Second) - }) It("Service Endpoints Changed", func() { By("create Ingress without IngressClass") err := s.CreateResourceFromStringWithNamespace(fmt.Sprintf(defaultIngress, s.Namespace()), s.Namespace()) Expect(err).NotTo(HaveOccurred(), "creating Ingress without IngressClass") - time.Sleep(5 * time.Second) By("verify default ingress") s.RequestAssert(&scaffold.RequestAssert{ @@ -348,7 +354,6 @@ spec: ingressName := s.Namespace() + "-external" err := s.CreateResourceFromStringWithNamespace(fmt.Sprintf(ingressWithExternalName, ingressName), s.Namespace()) Expect(err).NotTo(HaveOccurred(), "creating Ingress without IngressClass") - time.Sleep(5 * time.Second) By("create Ingress") err = s.CreateResourceFromStringWithNamespace(fmt.Sprintf(defaultIngress, s.Namespace()), s.Namespace()) @@ -379,7 +384,6 @@ spec: Expect(err).NotTo(HaveOccurred(), "deleting Ingress without IngressClass") s.Deployer.ScaleIngress(1) - time.Sleep(1 * time.Minute) s.RequestAssert(&scaffold.RequestAssert{ Method: "GET", @@ -389,11 +393,17 @@ spec: Interval: time.Second * 10, Timeout: 3 * time.Minute, }) - s.NewAPISIXClient(). - GET("/get"). - WithHost("default.example.com"). - Expect(). - Status(404) + // The deleted Ingress is reconciled away asynchronously after the + // controller is scaled back up, so poll until the route is gone + // instead of checking once. + s.RequestAssert(&scaffold.RequestAssert{ + Method: "GET", + Path: "/get", + Host: "default.example.com", + Check: scaffold.WithExpectedStatus(http.StatusNotFound), + Interval: time.Second * 2, + Timeout: time.Minute, + }) }) It("IngressClassName Change", func() { @@ -459,12 +469,10 @@ spec: By("create GatewayProxy") err := s.CreateResourceFromString(s.GetGatewayProxySpec()) Expect(err).NotTo(HaveOccurred(), "creating GatewayProxy") - time.Sleep(5 * time.Second) By("create IngressClass") err = s.CreateResourceFromStringWithNamespace(s.GetIngressClassYaml(), s.Namespace()) Expect(err).NotTo(HaveOccurred(), "creating GatewayProxy") - time.Sleep(5 * time.Second) By("create Ingress with annotation-defined class") ingressName := s.Namespace() + "-annotation" @@ -613,25 +621,25 @@ spec: By("create GatewayProxy") err := s.CreateResourceFromStringWithNamespace(gatewayProxy, s.Namespace()) Expect(err).NotTo(HaveOccurred(), "creating GatewayProxy") - time.Sleep(5 * time.Second) By("create IngressClass with GatewayProxy reference") err = s.CreateResourceFromStringWithNamespace(fmt.Sprintf(ingressClassWithProxy, s.GetControllerName(), s.Namespace()), "") Expect(err).NotTo(HaveOccurred(), "creating IngressClass with GatewayProxy") - time.Sleep(5 * time.Second) By("create Ingress with GatewayProxy IngressClass") err = s.CreateResourceFromStringWithNamespace(testIngress, s.Namespace()) Expect(err).NotTo(HaveOccurred(), "creating Ingress with GatewayProxy IngressClass") - time.Sleep(5 * time.Second) By("verify HTTP request") - resp := s.NewAPISIXClient(). - GET("/get"). - WithHost("proxy.example.com"). - Expect(). - Status(200) - resp.Header("X-Proxy-Test").IsEqual("enabled") + s.RequestAssert(&scaffold.RequestAssert{ + Method: "GET", + Path: "/get", + Host: "proxy.example.com", + Checks: []scaffold.ResponseCheckFunc{ + scaffold.WithExpectedStatus(http.StatusOK), + scaffold.WithExpectedHeader("X-Proxy-Test", "enabled"), + }, + }) }) It("Test IngressClass with GatewayProxy using Secret", func() { @@ -647,31 +655,30 @@ stringData: `, s.AdminKey()) err := s.CreateResourceFromStringWithNamespace(adminSecret, s.Namespace()) Expect(err).NotTo(HaveOccurred(), "creating admin secret") - time.Sleep(5 * time.Second) By("create GatewayProxy with Secret reference") gatewayProxy := fmt.Sprintf(gatewayProxyWithSecretYaml, s.Namespace(), s.Deployer.GetAdminEndpoint()) err = s.CreateResourceFromStringWithNamespace(gatewayProxy, s.Namespace()) Expect(err).NotTo(HaveOccurred(), "creating GatewayProxy with Secret") - time.Sleep(5 * time.Second) By("create IngressClass with GatewayProxy reference") err = s.CreateResourceFromStringWithNamespace(fmt.Sprintf(ingressClassWithProxySecret, s.GetControllerName(), s.Namespace()), "") Expect(err).NotTo(HaveOccurred(), "creating IngressClass with GatewayProxy") - time.Sleep(5 * time.Second) By("create Ingress with GatewayProxy IngressClass") err = s.CreateResourceFromStringWithNamespace(testIngressWithSecret, s.Namespace()) Expect(err).NotTo(HaveOccurred(), "creating Ingress with GatewayProxy IngressClass") - time.Sleep(5 * time.Second) By("verify HTTP request") - resp := s.NewAPISIXClient(). - GET("/get"). - WithHost("proxy-secret.example.com"). - Expect(). - Status(200) - resp.Header("X-Proxy-Test").IsEqual("enabled") + s.RequestAssert(&scaffold.RequestAssert{ + Method: "GET", + Path: "/get", + Host: "proxy-secret.example.com", + Checks: []scaffold.ResponseCheckFunc{ + scaffold.WithExpectedStatus(http.StatusOK), + scaffold.WithExpectedHeader("X-Proxy-Test", "enabled"), + }, + }) }) }) @@ -797,12 +804,10 @@ spec: By("create GatewayProxy") err := s.CreateResourceFromStringWithNamespace(getGatewayProxySpec(), s.Namespace()) Expect(err).NotTo(HaveOccurred(), "creating GatewayProxy") - time.Sleep(5 * time.Second) By("create IngressClass") err = s.CreateResourceFromStringWithNamespace(fmt.Sprintf(ingressClassSpec, s.Namespace(), s.GetControllerName(), s.Namespace()), s.Namespace()) Expect(err).NotTo(HaveOccurred(), "creating GatewayClass") - time.Sleep(5 * time.Second) }) It("HTTPRoutePolicy targetRef an Ingress", func() { @@ -979,12 +984,10 @@ spec: gatewayProxy := fmt.Sprintf(gatewayProxyYaml, s.Namespace(), s.Deployer.GetAdminEndpoint(), s.AdminKey()) err := s.CreateResourceFromStringWithNamespace(gatewayProxy, s.Namespace()) Expect(err).NotTo(HaveOccurred(), "creating GatewayProxy") - time.Sleep(5 * time.Second) By("create IngressClass") err = s.CreateResourceFromStringWithNamespace(fmt.Sprintf(ingressClass, s.Namespace(), s.GetControllerName(), s.Namespace()), s.Namespace()) Expect(err).NotTo(HaveOccurred(), "creating IngressClass") - time.Sleep(5 * time.Second) }) It("Should sync Ingress when GatewayProxy is updated", func() { @@ -1100,7 +1103,6 @@ spec: By("create IngressClass") err := s.CreateResourceFromStringWithNamespace(fmt.Sprintf(ingressClass, s.Namespace(), s.GetControllerName(), s.Namespace()), s.Namespace()) Expect(err).NotTo(HaveOccurred(), "creating IngressClass") - time.Sleep(5 * time.Second) }) It("Ingress With HTTPS Backend", func() { diff --git a/test/e2e/webhook/apisixconsumer.go b/test/e2e/webhook/apisixconsumer.go index 364e8784..76984619 100644 --- a/test/e2e/webhook/apisixconsumer.go +++ b/test/e2e/webhook/apisixconsumer.go @@ -20,7 +20,6 @@ package webhook import ( "fmt" "strings" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -39,12 +38,10 @@ var _ = Describe("Test ApisixConsumer Webhook", Label("webhook"), func() { By("creating GatewayProxy") err := s.CreateResourceFromString(s.GetGatewayProxySpec()) Expect(err).NotTo(HaveOccurred(), "creating GatewayProxy") - time.Sleep(5 * time.Second) By("creating IngressClass") err = s.CreateResourceFromStringWithNamespace(s.GetIngressClassYaml(), "") Expect(err).NotTo(HaveOccurred(), "creating IngressClass") - time.Sleep(5 * time.Second) }) It("should warn on missing authentication secrets", func() { //nolint:dupl @@ -64,9 +61,11 @@ spec: name: %s ` - output, err := s.CreateResourceFromStringAndGetOutput(fmt.Sprintf(consumerYAML, consumerName, s.Namespace(), s.Namespace(), missingSecret)) - Expect(err).ShouldNot(HaveOccurred()) - Expect(output).To(ContainSubstring(fmt.Sprintf("Warning: Referenced Secret '%s/%s' not found", s.Namespace(), missingSecret))) + Eventually(func(g Gomega) { + output, err := s.CreateResourceFromStringAndGetOutput(fmt.Sprintf(consumerYAML, consumerName, s.Namespace(), s.Namespace(), missingSecret)) + g.Expect(err).ShouldNot(HaveOccurred()) + g.Expect(output).To(ContainSubstring(fmt.Sprintf("Warning: Referenced Secret '%s/%s' not found", s.Namespace(), missingSecret))) + }).WithTimeout(scaffold.DefaultTimeout).ProbeEvery(scaffold.DefaultInterval).Should(Succeed()) By("creating referenced secret") secretYAML := fmt.Sprintf(` @@ -78,14 +77,14 @@ stringData: username: demo password: demo `, missingSecret) - err = s.CreateResourceFromString(secretYAML) + err := s.CreateResourceFromString(secretYAML) Expect(err).NotTo(HaveOccurred(), "creating basic auth secret") - time.Sleep(2 * time.Second) - - output, err = s.CreateResourceFromStringAndGetOutput(fmt.Sprintf(consumerYAML, consumerName, s.Namespace(), s.Namespace(), missingSecret)) - Expect(err).ShouldNot(HaveOccurred()) - Expect(output).NotTo(ContainSubstring(fmt.Sprintf("Warning: Referenced Secret '%s/%s' not found", s.Namespace(), missingSecret))) + Eventually(func(g Gomega) { + output, err := s.CreateResourceFromStringAndGetOutput(fmt.Sprintf(consumerYAML, consumerName, s.Namespace(), s.Namespace(), missingSecret)) + g.Expect(err).ShouldNot(HaveOccurred()) + g.Expect(output).NotTo(ContainSubstring(fmt.Sprintf("Warning: Referenced Secret '%s/%s' not found", s.Namespace(), missingSecret))) + }).WithTimeout(scaffold.DefaultTimeout).ProbeEvery(scaffold.DefaultInterval).Should(Succeed()) }) It("should reject invalid plugin config during ADC validation", func() { diff --git a/test/e2e/webhook/apisixroute.go b/test/e2e/webhook/apisixroute.go index 2e498f5f..fe2a92db 100644 --- a/test/e2e/webhook/apisixroute.go +++ b/test/e2e/webhook/apisixroute.go @@ -19,7 +19,6 @@ package webhook import ( "fmt" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -37,12 +36,10 @@ var _ = Describe("Test ApisixRoute Webhook", Label("webhook"), func() { By("creating GatewayProxy") err := s.CreateResourceFromString(s.GetGatewayProxySpec()) Expect(err).NotTo(HaveOccurred(), "creating GatewayProxy") - time.Sleep(5 * time.Second) By("creating IngressClass") err = s.CreateResourceFromStringWithNamespace(s.GetIngressClassYaml(), "") Expect(err).NotTo(HaveOccurred(), "creating IngressClass") - time.Sleep(5 * time.Second) }) It("should warn on missing service references", func() { //nolint:dupl @@ -68,9 +65,11 @@ spec: servicePort: 80 ` - output, err := s.CreateResourceFromStringAndGetOutput(fmt.Sprintf(routeYAML, routeName, s.Namespace(), s.Namespace(), missingService)) - Expect(err).ShouldNot(HaveOccurred()) - Expect(output).To(ContainSubstring(fmt.Sprintf("Warning: Referenced Service '%s/%s' not found", s.Namespace(), missingService))) + Eventually(func(g Gomega) { + output, err := s.CreateResourceFromStringAndGetOutput(fmt.Sprintf(routeYAML, routeName, s.Namespace(), s.Namespace(), missingService)) + g.Expect(err).ShouldNot(HaveOccurred()) + g.Expect(output).To(ContainSubstring(fmt.Sprintf("Warning: Referenced Service '%s/%s' not found", s.Namespace(), missingService))) + }).WithTimeout(scaffold.DefaultTimeout).ProbeEvery(scaffold.DefaultInterval).Should(Succeed()) By("creating referenced Service") serviceYAML := fmt.Sprintf(` @@ -87,14 +86,14 @@ spec: targetPort: 80 type: ClusterIP `, missingService) - err = s.CreateResourceFromString(serviceYAML) + err := s.CreateResourceFromString(serviceYAML) Expect(err).NotTo(HaveOccurred(), "creating backend service placeholder") - time.Sleep(2 * time.Second) - - output, err = s.CreateResourceFromStringAndGetOutput(fmt.Sprintf(routeYAML, routeName, s.Namespace(), s.Namespace(), missingService)) - Expect(err).ShouldNot(HaveOccurred()) - Expect(output).NotTo(ContainSubstring(fmt.Sprintf("Warning: Referenced Service '%s/%s' not found", s.Namespace(), missingService))) + Eventually(func(g Gomega) { + output, err := s.CreateResourceFromStringAndGetOutput(fmt.Sprintf(routeYAML, routeName, s.Namespace(), s.Namespace(), missingService)) + g.Expect(err).ShouldNot(HaveOccurred()) + g.Expect(output).NotTo(ContainSubstring(fmt.Sprintf("Warning: Referenced Service '%s/%s' not found", s.Namespace(), missingService))) + }).WithTimeout(scaffold.DefaultTimeout).ProbeEvery(scaffold.DefaultInterval).Should(Succeed()) }) It("should reject routes that fail ADC validation", func() { diff --git a/test/e2e/webhook/apisixtls.go b/test/e2e/webhook/apisixtls.go index 0d24d0f8..0fd7f3dd 100644 --- a/test/e2e/webhook/apisixtls.go +++ b/test/e2e/webhook/apisixtls.go @@ -19,7 +19,6 @@ package webhook import ( "fmt" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -37,12 +36,10 @@ var _ = Describe("Test ApisixTls Webhook", Label("webhook"), func() { By("creating GatewayProxy") err := s.CreateResourceFromString(s.GetGatewayProxySpec()) Expect(err).NotTo(HaveOccurred(), "creating GatewayProxy") - time.Sleep(5 * time.Second) By("creating IngressClass") err = s.CreateResourceFromStringWithNamespace(s.GetIngressClassYaml(), "") Expect(err).NotTo(HaveOccurred(), "creating IngressClass") - time.Sleep(5 * time.Second) }) It("should warn on missing TLS secrets", func() { @@ -68,26 +65,28 @@ spec: namespace: %s ` - output, err := s.CreateResourceFromStringAndGetOutput(fmt.Sprintf(tlsYAML, tlsName, s.Namespace(), s.Namespace(), serverSecret, s.Namespace(), clientSecret, s.Namespace())) - Expect(err).ShouldNot(HaveOccurred()) - Expect(output).To(ContainSubstring(fmt.Sprintf("Warning: Referenced Secret '%s/%s' not found", s.Namespace(), serverSecret))) - Expect(output).To(ContainSubstring(fmt.Sprintf("Warning: Referenced Secret '%s/%s' not found", s.Namespace(), clientSecret))) + Eventually(func(g Gomega) { + output, err := s.CreateResourceFromStringAndGetOutput(fmt.Sprintf(tlsYAML, tlsName, s.Namespace(), s.Namespace(), serverSecret, s.Namespace(), clientSecret, s.Namespace())) + g.Expect(err).ShouldNot(HaveOccurred()) + g.Expect(output).To(ContainSubstring(fmt.Sprintf("Warning: Referenced Secret '%s/%s' not found", s.Namespace(), serverSecret))) + g.Expect(output).To(ContainSubstring(fmt.Sprintf("Warning: Referenced Secret '%s/%s' not found", s.Namespace(), clientSecret))) + }).WithTimeout(scaffold.DefaultTimeout).ProbeEvery(scaffold.DefaultInterval).Should(Succeed()) By("creating referenced TLS secrets with valid certificate material") serverCert, serverKey := s.GenerateCert(GinkgoT(), []string{"webhook.example.com"}) - err = s.NewKubeTlsSecret(serverSecret, serverCert.String(), serverKey.String()) + err := s.NewKubeTlsSecret(serverSecret, serverCert.String(), serverKey.String()) Expect(err).NotTo(HaveOccurred(), "creating server TLS secret") caCert, _, _, _, _ := s.GenerateMACert(GinkgoT(), []string{"webhook.example.com"}) err = s.NewClientCASecret(clientSecret, caCert.String(), "") Expect(err).NotTo(HaveOccurred(), "creating client CA secret") - time.Sleep(2 * time.Second) - - output, err = s.CreateResourceFromStringAndGetOutput(fmt.Sprintf(tlsYAML, tlsName, s.Namespace(), s.Namespace(), serverSecret, s.Namespace(), clientSecret, s.Namespace())) - Expect(err).ShouldNot(HaveOccurred()) - Expect(output).NotTo(ContainSubstring(fmt.Sprintf("Warning: Referenced Secret '%s/%s' not found", s.Namespace(), serverSecret))) - Expect(output).NotTo(ContainSubstring(fmt.Sprintf("Warning: Referenced Secret '%s/%s' not found", s.Namespace(), clientSecret))) + Eventually(func(g Gomega) { + output, err := s.CreateResourceFromStringAndGetOutput(fmt.Sprintf(tlsYAML, tlsName, s.Namespace(), s.Namespace(), serverSecret, s.Namespace(), clientSecret, s.Namespace())) + g.Expect(err).ShouldNot(HaveOccurred()) + g.Expect(output).NotTo(ContainSubstring(fmt.Sprintf("Warning: Referenced Secret '%s/%s' not found", s.Namespace(), serverSecret))) + g.Expect(output).NotTo(ContainSubstring(fmt.Sprintf("Warning: Referenced Secret '%s/%s' not found", s.Namespace(), clientSecret))) + }).WithTimeout(scaffold.DefaultTimeout).ProbeEvery(scaffold.DefaultInterval).Should(Succeed()) }) It("should reject invalid TLS material during ADC validation", func() { @@ -137,12 +136,12 @@ spec: err = s.NewKubeTlsSecret(serverSecret, serverCert.String(), serverKey.String()) Expect(err).NotTo(HaveOccurred(), "creating valid server TLS secret") - // Wait for the webhook cache to reflect the recreated Secret before submitting ApisixTls. - time.Sleep(2 * time.Second) - By("creating corrected ApisixTls") - err = s.CreateResourceFromString(tlsYAML) - Expect(err).NotTo(HaveOccurred(), "creating corrected ApisixTls") + // Retry until the webhook cache reflects the recreated Secret. + Eventually(func(g Gomega) { + err := s.CreateResourceFromString(tlsYAML) + g.Expect(err).NotTo(HaveOccurred(), "creating corrected ApisixTls") + }).WithTimeout(scaffold.DefaultTimeout).ProbeEvery(scaffold.DefaultInterval).Should(Succeed()) }) It("should reject TLS update with invalid certificate material", func() { diff --git a/test/e2e/webhook/consumer.go b/test/e2e/webhook/consumer.go index 6a051e25..4338b3ac 100644 --- a/test/e2e/webhook/consumer.go +++ b/test/e2e/webhook/consumer.go @@ -19,7 +19,6 @@ package webhook import ( "fmt" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -37,17 +36,14 @@ var _ = Describe("Test Consumer Webhook", Label("webhook"), func() { By("creating GatewayProxy") err := s.CreateResourceFromString(s.GetGatewayProxySpec()) Expect(err).NotTo(HaveOccurred(), "creating GatewayProxy") - time.Sleep(5 * time.Second) By("creating GatewayClass") err = s.CreateResourceFromString(s.GetGatewayClassYaml()) Expect(err).NotTo(HaveOccurred(), "creating GatewayClass") - time.Sleep(2 * time.Second) By("creating Gateway") err = s.CreateResourceFromString(s.GetGatewayYaml()) Expect(err).NotTo(HaveOccurred(), "creating Gateway") - time.Sleep(5 * time.Second) }) It("should warn on missing secret references", func() { @@ -68,9 +64,11 @@ spec: name: %s ` - output, err := s.CreateResourceFromStringAndGetOutput(fmt.Sprintf(consumerYAML, consumerName, gatewayName, missingSecret)) - Expect(err).ShouldNot(HaveOccurred()) - Expect(output).To(ContainSubstring(fmt.Sprintf("Warning: Referenced Secret '%s/%s' not found", s.Namespace(), missingSecret))) + Eventually(func(g Gomega) { + output, err := s.CreateResourceFromStringAndGetOutput(fmt.Sprintf(consumerYAML, consumerName, gatewayName, missingSecret)) + g.Expect(err).ShouldNot(HaveOccurred()) + g.Expect(output).To(ContainSubstring(fmt.Sprintf("Warning: Referenced Secret '%s/%s' not found", s.Namespace(), missingSecret))) + }).WithTimeout(scaffold.DefaultTimeout).ProbeEvery(scaffold.DefaultInterval).Should(Succeed()) By("creating referenced secret") secretYAML := fmt.Sprintf(` @@ -81,14 +79,14 @@ metadata: stringData: token: %s `, missingSecret, s.AdminKey()) - err = s.CreateResourceFromString(secretYAML) + err := s.CreateResourceFromString(secretYAML) Expect(err).NotTo(HaveOccurred(), "creating consumer secret") - time.Sleep(2 * time.Second) - - output, err = s.CreateResourceFromStringAndGetOutput(fmt.Sprintf(consumerYAML, consumerName, gatewayName, missingSecret)) - Expect(err).ShouldNot(HaveOccurred()) - Expect(output).NotTo(ContainSubstring(fmt.Sprintf("Warning: Referenced Secret '%s/%s' not found", s.Namespace(), missingSecret))) + Eventually(func(g Gomega) { + output, err := s.CreateResourceFromStringAndGetOutput(fmt.Sprintf(consumerYAML, consumerName, gatewayName, missingSecret)) + g.Expect(err).ShouldNot(HaveOccurred()) + g.Expect(output).NotTo(ContainSubstring(fmt.Sprintf("Warning: Referenced Secret '%s/%s' not found", s.Namespace(), missingSecret))) + }).WithTimeout(scaffold.DefaultTimeout).ProbeEvery(scaffold.DefaultInterval).Should(Succeed()) }) It("should reject invalid plugin config during ADC validation", func() { diff --git a/test/e2e/webhook/gateway.go b/test/e2e/webhook/gateway.go index 78178014..97d149f6 100644 --- a/test/e2e/webhook/gateway.go +++ b/test/e2e/webhook/gateway.go @@ -19,7 +19,6 @@ package webhook import ( "fmt" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -39,8 +38,6 @@ var _ = Describe("Test Gateway Webhook", Label("webhook"), func() { err := s.CreateResourceFromString(s.GetGatewayClassYaml()) Expect(err).ShouldNot(HaveOccurred()) - time.Sleep(2 * time.Second) - By("creating Gateway referencing a missing GatewayProxy") missingName := "missing-proxy" gwYAML := ` @@ -61,28 +58,31 @@ spec: name: %s ` - output, err := s.CreateResourceFromStringAndGetOutput(fmt.Sprintf(gwYAML, s.Namespace(), s.Namespace(), missingName)) - Expect(err).ShouldNot(HaveOccurred()) - Expect(output).To(ContainSubstring(fmt.Sprintf("Warning: Referenced GatewayProxy '%s/%s' not found.", s.Namespace(), missingName))) - - time.Sleep(2 * time.Second) + Eventually(func(g Gomega) { + output, err := s.CreateResourceFromStringAndGetOutput(fmt.Sprintf(gwYAML, s.Namespace(), s.Namespace(), missingName)) + g.Expect(err).ShouldNot(HaveOccurred()) + g.Expect(output).To(ContainSubstring(fmt.Sprintf("Warning: Referenced GatewayProxy '%s/%s' not found.", s.Namespace(), missingName))) + }).WithTimeout(scaffold.DefaultTimeout).ProbeEvery(scaffold.DefaultInterval).Should(Succeed()) By("updating Gateway to reference another missing GatewayProxy") missingName2 := "missing-proxy-2" - output, err = s.CreateResourceFromStringAndGetOutput(fmt.Sprintf(gwYAML, s.Namespace(), s.Namespace(), missingName2)) - Expect(err).ShouldNot(HaveOccurred()) - Expect(output).To(ContainSubstring(fmt.Sprintf("Warning: Referenced GatewayProxy '%s/%s' not found.", s.Namespace(), missingName2))) + Eventually(func(g Gomega) { + output, err := s.CreateResourceFromStringAndGetOutput(fmt.Sprintf(gwYAML, s.Namespace(), s.Namespace(), missingName2)) + g.Expect(err).ShouldNot(HaveOccurred()) + g.Expect(output).To(ContainSubstring(fmt.Sprintf("Warning: Referenced GatewayProxy '%s/%s' not found.", s.Namespace(), missingName2))) + }).WithTimeout(scaffold.DefaultTimeout).ProbeEvery(scaffold.DefaultInterval).Should(Succeed()) By("create GatewayProxy") err = s.CreateResourceFromString(s.GetGatewayProxySpec()) Expect(err).NotTo(HaveOccurred(), "creating GatewayProxy") - time.Sleep(5 * time.Second) By("updating Gateway to reference an existing GatewayProxy") existingName := "apisix-proxy-config" - output, err = s.CreateResourceFromStringAndGetOutput(fmt.Sprintf(gwYAML, s.Namespace(), s.Namespace(), existingName)) - Expect(err).ShouldNot(HaveOccurred()) - Expect(output).NotTo(ContainSubstring(fmt.Sprintf("Warning: Referenced GatewayProxy '%s/%s' not found.", s.Namespace(), existingName))) + Eventually(func(g Gomega) { + output, err := s.CreateResourceFromStringAndGetOutput(fmt.Sprintf(gwYAML, s.Namespace(), s.Namespace(), existingName)) + g.Expect(err).ShouldNot(HaveOccurred()) + g.Expect(output).NotTo(ContainSubstring(fmt.Sprintf("Warning: Referenced GatewayProxy '%s/%s' not found.", s.Namespace(), existingName))) + }).WithTimeout(scaffold.DefaultTimeout).ProbeEvery(scaffold.DefaultInterval).Should(Succeed()) By("delete Gateway") err = s.DeleteResource("Gateway", s.Namespace()) diff --git a/test/e2e/webhook/gatewayproxy.go b/test/e2e/webhook/gatewayproxy.go index 4f2d12e4..888d84fe 100644 --- a/test/e2e/webhook/gatewayproxy.go +++ b/test/e2e/webhook/gatewayproxy.go @@ -19,7 +19,6 @@ package webhook import ( "fmt" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -59,10 +58,12 @@ spec: missingSecret := "missing-admin-secret" gpName := "webhook-gateway-proxy" - output, err := s.CreateResourceFromStringAndGetOutput(fmt.Sprintf(gatewayProxyTemplate, gpName, missingService, missingSecret)) - Expect(err).ShouldNot(HaveOccurred()) - Expect(output).To(ContainSubstring(fmt.Sprintf("Warning: Referenced Service '%s/%s' not found", s.Namespace(), missingService))) - Expect(output).To(ContainSubstring(fmt.Sprintf("Warning: Referenced Secret '%s/%s' not found", s.Namespace(), missingSecret))) + Eventually(func(g Gomega) { + output, err := s.CreateResourceFromStringAndGetOutput(fmt.Sprintf(gatewayProxyTemplate, gpName, missingService, missingSecret)) + g.Expect(err).ShouldNot(HaveOccurred()) + g.Expect(output).To(ContainSubstring(fmt.Sprintf("Warning: Referenced Service '%s/%s' not found", s.Namespace(), missingService))) + g.Expect(output).To(ContainSubstring(fmt.Sprintf("Warning: Referenced Secret '%s/%s' not found", s.Namespace(), missingSecret))) + }).WithTimeout(scaffold.DefaultTimeout).ProbeEvery(scaffold.DefaultInterval).Should(Succeed()) By("creating the referenced Service and Secret without the required key") serviceYAML := fmt.Sprintf(` @@ -79,7 +80,7 @@ spec: targetPort: 9180 type: ClusterIP `, missingService) - err = s.CreateResourceFromString(serviceYAML) + err := s.CreateResourceFromString(serviceYAML) Expect(err).NotTo(HaveOccurred(), "creating placeholder service") secretWithoutKey := fmt.Sprintf(` @@ -93,16 +94,16 @@ stringData: err = s.CreateResourceFromString(secretWithoutKey) Expect(err).NotTo(HaveOccurred(), "creating placeholder secret without token key") - time.Sleep(2 * time.Second) - By("delete and reapply the GatewayProxy, because gatewayproxy has no change") - err = s.DeleteResource("GatewayProxy", gpName) - Expect(err).ShouldNot(HaveOccurred()) + Eventually(func(g Gomega) { + err := s.DeleteResource("GatewayProxy", gpName) + g.Expect(err).ShouldNot(HaveOccurred()) - output, err = s.CreateResourceFromStringAndGetOutput(fmt.Sprintf(gatewayProxyTemplate, gpName, missingService, missingSecret)) - Expect(err).ShouldNot(HaveOccurred()) - Expect(output).NotTo(ContainSubstring(fmt.Sprintf("Warning: Referenced Service '%s/%s' not found", s.Namespace(), missingService))) - Expect(output).To(ContainSubstring(fmt.Sprintf("Warning: Secret key 'token' not found in Secret '%s/%s'", s.Namespace(), missingSecret))) + output, err := s.CreateResourceFromStringAndGetOutput(fmt.Sprintf(gatewayProxyTemplate, gpName, missingService, missingSecret)) + g.Expect(err).ShouldNot(HaveOccurred()) + g.Expect(output).NotTo(ContainSubstring(fmt.Sprintf("Warning: Referenced Service '%s/%s' not found", s.Namespace(), missingService))) + g.Expect(output).To(ContainSubstring(fmt.Sprintf("Warning: Secret key 'token' not found in Secret '%s/%s'", s.Namespace(), missingSecret))) + }).WithTimeout(scaffold.DefaultTimeout).ProbeEvery(scaffold.DefaultInterval).Should(Succeed()) By("updating the Secret to include the expected key") secretWithKey := fmt.Sprintf(` @@ -116,16 +117,16 @@ stringData: err = s.CreateResourceFromString(secretWithKey) Expect(err).NotTo(HaveOccurred(), "adding token key to secret") - time.Sleep(2 * time.Second) - By("delete and reapply the GatewayProxy, because gatewayproxy has no change") - err = s.DeleteResource("GatewayProxy", gpName) - Expect(err).ShouldNot(HaveOccurred()) - - output, err = s.CreateResourceFromStringAndGetOutput(fmt.Sprintf(gatewayProxyTemplate, gpName, missingService, missingSecret)) - Expect(err).ShouldNot(HaveOccurred()) - Expect(output).NotTo(ContainSubstring(fmt.Sprintf("Warning: Referenced Service '%s/%s' not found", s.Namespace(), missingService))) - Expect(output).NotTo(ContainSubstring(fmt.Sprintf("Warning: Secret key 'token' not found in Secret '%s/%s'", s.Namespace(), missingSecret))) + Eventually(func(g Gomega) { + err := s.DeleteResource("GatewayProxy", gpName) + g.Expect(err).ShouldNot(HaveOccurred()) + + output, err := s.CreateResourceFromStringAndGetOutput(fmt.Sprintf(gatewayProxyTemplate, gpName, missingService, missingSecret)) + g.Expect(err).ShouldNot(HaveOccurred()) + g.Expect(output).NotTo(ContainSubstring(fmt.Sprintf("Warning: Referenced Service '%s/%s' not found", s.Namespace(), missingService))) + g.Expect(output).NotTo(ContainSubstring(fmt.Sprintf("Warning: Secret key 'token' not found in Secret '%s/%s'", s.Namespace(), missingSecret))) + }).WithTimeout(scaffold.DefaultTimeout).ProbeEvery(scaffold.DefaultInterval).Should(Succeed()) }) Context("GatewayProxy configuration conflicts", func() { @@ -163,15 +164,15 @@ stringData: err := s.CreateResourceFromString(fmt.Sprintf(gatewayProxyTemplate, initialProxy, serviceName, secretName)) Expect(err).ShouldNot(HaveOccurred(), "creating initial GatewayProxy") - time.Sleep(2 * time.Second) - - err = s.CreateResourceFromString(fmt.Sprintf(gatewayProxyTemplate, conflictingProxy, serviceName, secretName)) - Expect(err).Should(HaveOccurred(), "expecting conflict for duplicated GatewayProxy") - Expect(err.Error()).To(ContainSubstring("gateway proxy configuration conflict")) - Expect(err.Error()).To(ContainSubstring(fmt.Sprintf("%s/%s", s.Namespace(), conflictingProxy))) - Expect(err.Error()).To(ContainSubstring(fmt.Sprintf("%s/%s", s.Namespace(), initialProxy))) - Expect(err.Error()).To(ContainSubstring("Service")) - Expect(err.Error()).To(ContainSubstring("AdminKey secret")) + Eventually(func(g Gomega) { + err := s.CreateResourceFromString(fmt.Sprintf(gatewayProxyTemplate, conflictingProxy, serviceName, secretName)) + g.Expect(err).Should(HaveOccurred(), "expecting conflict for duplicated GatewayProxy") + g.Expect(err.Error()).To(ContainSubstring("gateway proxy configuration conflict")) + g.Expect(err.Error()).To(ContainSubstring(fmt.Sprintf("%s/%s", s.Namespace(), conflictingProxy))) + g.Expect(err.Error()).To(ContainSubstring(fmt.Sprintf("%s/%s", s.Namespace(), initialProxy))) + g.Expect(err.Error()).To(ContainSubstring("Service")) + g.Expect(err.Error()).To(ContainSubstring("AdminKey secret")) + }).WithTimeout(scaffold.DefaultTimeout).ProbeEvery(scaffold.DefaultInterval).Should(Succeed()) Expect(s.DeleteResource("GatewayProxy", initialProxy)).ShouldNot(HaveOccurred()) Expect(s.DeleteResource("Service", serviceName)).ShouldNot(HaveOccurred()) @@ -207,15 +208,15 @@ spec: err := s.CreateResourceFromString(fmt.Sprintf(gatewayProxyTemplate, existingProxy, endpointA, endpointB, inlineKey)) Expect(err).ShouldNot(HaveOccurred(), "creating GatewayProxy with inline AdminKey") - time.Sleep(2 * time.Second) - - err = s.CreateResourceFromString(fmt.Sprintf(gatewayProxyTemplate, conflictingProxy, endpointB, endpointC, inlineKey)) - Expect(err).Should(HaveOccurred(), "expecting conflict for overlapping endpoints with shared AdminKey") - Expect(err.Error()).To(ContainSubstring("gateway proxy configuration conflict")) - Expect(err.Error()).To(ContainSubstring(fmt.Sprintf("%s/%s", s.Namespace(), conflictingProxy))) - Expect(err.Error()).To(ContainSubstring(fmt.Sprintf("%s/%s", s.Namespace(), existingProxy))) - Expect(err.Error()).To(ContainSubstring("control plane endpoints")) - Expect(err.Error()).To(ContainSubstring("inline AdminKey value")) + Eventually(func(g Gomega) { + err := s.CreateResourceFromString(fmt.Sprintf(gatewayProxyTemplate, conflictingProxy, endpointB, endpointC, inlineKey)) + g.Expect(err).Should(HaveOccurred(), "expecting conflict for overlapping endpoints with shared AdminKey") + g.Expect(err.Error()).To(ContainSubstring("gateway proxy configuration conflict")) + g.Expect(err.Error()).To(ContainSubstring(fmt.Sprintf("%s/%s", s.Namespace(), conflictingProxy))) + g.Expect(err.Error()).To(ContainSubstring(fmt.Sprintf("%s/%s", s.Namespace(), existingProxy))) + g.Expect(err.Error()).To(ContainSubstring("control plane endpoints")) + g.Expect(err.Error()).To(ContainSubstring("inline AdminKey value")) + }).WithTimeout(scaffold.DefaultTimeout).ProbeEvery(scaffold.DefaultInterval).Should(Succeed()) }) It("should reject GatewayProxy update that creates conflict with another GatewayProxy", func() { @@ -254,19 +255,19 @@ stringData: err := s.CreateResourceFromString(fmt.Sprintf(gatewayProxyTemplate, proxyA, sharedServiceName, sharedSecretName)) Expect(err).ShouldNot(HaveOccurred(), "creating GatewayProxy A with shared Service and Secret") - time.Sleep(2 * time.Second) - - err = s.CreateResourceFromString(fmt.Sprintf(gatewayProxyTemplate, proxyB, uniqueServiceName, sharedSecretName)) - Expect(err).ShouldNot(HaveOccurred(), "creating GatewayProxy B with unique Service but same Secret") - - time.Sleep(2 * time.Second) + Eventually(func(g Gomega) { + err := s.CreateResourceFromString(fmt.Sprintf(gatewayProxyTemplate, proxyB, uniqueServiceName, sharedSecretName)) + g.Expect(err).ShouldNot(HaveOccurred(), "creating GatewayProxy B with unique Service but same Secret") + }).WithTimeout(scaffold.DefaultTimeout).ProbeEvery(scaffold.DefaultInterval).Should(Succeed()) By("updating GatewayProxy B to use the same Service as GatewayProxy A, causing conflict") - err = s.CreateResourceFromString(fmt.Sprintf(gatewayProxyTemplate, proxyB, sharedServiceName, sharedSecretName)) - Expect(err).Should(HaveOccurred(), "expecting conflict when updating to same Service") - Expect(err.Error()).To(ContainSubstring("gateway proxy configuration conflict")) - Expect(err.Error()).To(ContainSubstring(fmt.Sprintf("%s/%s", s.Namespace(), proxyA))) - Expect(err.Error()).To(ContainSubstring(fmt.Sprintf("%s/%s", s.Namespace(), proxyB))) + Eventually(func(g Gomega) { + err := s.CreateResourceFromString(fmt.Sprintf(gatewayProxyTemplate, proxyB, sharedServiceName, sharedSecretName)) + g.Expect(err).Should(HaveOccurred(), "expecting conflict when updating to same Service") + g.Expect(err.Error()).To(ContainSubstring("gateway proxy configuration conflict")) + g.Expect(err.Error()).To(ContainSubstring(fmt.Sprintf("%s/%s", s.Namespace(), proxyA))) + g.Expect(err.Error()).To(ContainSubstring(fmt.Sprintf("%s/%s", s.Namespace(), proxyB))) + }).WithTimeout(scaffold.DefaultTimeout).ProbeEvery(scaffold.DefaultInterval).Should(Succeed()) }) }) }) diff --git a/test/e2e/webhook/helpers.go b/test/e2e/webhook/helpers.go index eec76ec7..cea2f534 100644 --- a/test/e2e/webhook/helpers.go +++ b/test/e2e/webhook/helpers.go @@ -19,7 +19,6 @@ package webhook import ( "fmt" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -50,17 +49,14 @@ func setupGatewayResources(s *scaffold.Scaffold) { By("creating GatewayProxy") err := s.CreateResourceFromString(s.GetGatewayProxySpec()) Expect(err).NotTo(HaveOccurred(), "creating GatewayProxy") - time.Sleep(5 * time.Second) By("creating GatewayClass") err = s.CreateResourceFromString(s.GetGatewayClassYaml()) Expect(err).NotTo(HaveOccurred(), "creating GatewayClass") - time.Sleep(2 * time.Second) By("creating Gateway") err = s.CreateResourceFromString(s.GetGatewayYaml()) Expect(err).NotTo(HaveOccurred(), "creating Gateway") - time.Sleep(5 * time.Second) } func verifyMissingBackendWarnings(s *scaffold.Scaffold, tc routeWebhookTestCase) { @@ -88,15 +84,16 @@ spec: missingBackendWarning := fmt.Sprintf("Warning: Referenced Service '%s/%s' not found", gatewayName, tc.missingService) mirrorBackendWarning := fmt.Sprintf("Warning: Referenced Service '%s/%s' not found", gatewayName, tc.mirrorService) - output, err := s.CreateResourceFromStringAndGetOutput(routeYAML) - Expect(err).ShouldNot(HaveOccurred()) - Expect(output).To(ContainSubstring(missingBackendWarning)) - Expect(output).To(ContainSubstring(mirrorBackendWarning)) + Eventually(func(g Gomega) { + output, err := s.CreateResourceFromStringAndGetOutput(routeYAML) + g.Expect(err).ShouldNot(HaveOccurred()) + g.Expect(output).To(ContainSubstring(missingBackendWarning)) + g.Expect(output).To(ContainSubstring(mirrorBackendWarning)) + }).WithTimeout(scaffold.DefaultTimeout).ProbeEvery(scaffold.DefaultInterval).Should(Succeed()) By("delete the " + tc.routeKind) - err = s.DeleteResource(tc.routeKind, tc.routeName) + err := s.DeleteResource(tc.routeKind, tc.routeName) Expect(err).NotTo(HaveOccurred()) - time.Sleep(2 * time.Second) By(fmt.Sprintf("creating referenced backend services for %s", tc.routeKind)) serviceYAML := ` @@ -122,24 +119,22 @@ spec: err = s.CreateResourceFromString(mirrorService) Expect(err).NotTo(HaveOccurred(), "creating mirror backend service") - time.Sleep(2 * time.Second) - - output, err = s.CreateResourceFromStringAndGetOutput(routeYAML) - Expect(err).ShouldNot(HaveOccurred()) - Expect(output).NotTo(ContainSubstring(missingBackendWarning)) - Expect(output).NotTo(ContainSubstring(mirrorBackendWarning)) + Eventually(func(g Gomega) { + output, err := s.CreateResourceFromStringAndGetOutput(routeYAML) + g.Expect(err).ShouldNot(HaveOccurred()) + g.Expect(output).NotTo(ContainSubstring(missingBackendWarning)) + g.Expect(output).NotTo(ContainSubstring(mirrorBackendWarning)) + }).WithTimeout(scaffold.DefaultTimeout).ProbeEvery(scaffold.DefaultInterval).Should(Succeed()) } func setupSimpleGatewayWithProtocol(s *scaffold.Scaffold, protocol, listenerName string, port int) { By("creating GatewayProxy") err := s.CreateResourceFromString(s.GetGatewayProxySpec()) Expect(err).NotTo(HaveOccurred(), "creating GatewayProxy") - time.Sleep(5 * time.Second) By("creating GatewayClass") err = s.CreateResourceFromString(s.GetGatewayClassYaml()) Expect(err).NotTo(HaveOccurred(), "creating GatewayClass") - time.Sleep(2 * time.Second) gatewayYAML := fmt.Sprintf(` apiVersion: gateway.networking.k8s.io/v1 @@ -165,7 +160,6 @@ spec: By(fmt.Sprintf("creating Gateway with %s listener", protocol)) err = s.CreateResourceFromString(gatewayYAML) Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("creating %s-capable Gateway", protocol)) - time.Sleep(5 * time.Second) } func verifySimpleRouteMissingBackendWarnings(s *scaffold.Scaffold, tc simpleRouteWebhookTestCase) { @@ -187,14 +181,15 @@ spec: missingBackendWarning := fmt.Sprintf("Warning: Referenced Service '%s/%s' not found", gatewayName, tc.missingService) - output, err := s.CreateResourceFromStringAndGetOutput(routeYAML) - Expect(err).ShouldNot(HaveOccurred()) - Expect(output).To(ContainSubstring(missingBackendWarning)) + Eventually(func(g Gomega) { + output, err := s.CreateResourceFromStringAndGetOutput(routeYAML) + g.Expect(err).ShouldNot(HaveOccurred()) + g.Expect(output).To(ContainSubstring(missingBackendWarning)) + }).WithTimeout(scaffold.DefaultTimeout).ProbeEvery(scaffold.DefaultInterval).Should(Succeed()) By("delete the " + tc.routeKind) - err = s.DeleteResource(tc.routeKind, tc.routeName) + err := s.DeleteResource(tc.routeKind, tc.routeName) Expect(err).NotTo(HaveOccurred()) - time.Sleep(2 * time.Second) By("creating referenced backend service") serviceYAML := ` @@ -229,11 +224,11 @@ spec: err = s.CreateResourceFromString(backendService) Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("creating %s backend service", tc.servicePortName)) - time.Sleep(2 * time.Second) - - output, err = s.CreateResourceFromStringAndGetOutput(routeYAML) - Expect(err).ShouldNot(HaveOccurred()) - Expect(output).NotTo(ContainSubstring(missingBackendWarning)) + Eventually(func(g Gomega) { + output, err := s.CreateResourceFromStringAndGetOutput(routeYAML) + g.Expect(err).ShouldNot(HaveOccurred()) + g.Expect(output).NotTo(ContainSubstring(missingBackendWarning)) + }).WithTimeout(scaffold.DefaultTimeout).ProbeEvery(scaffold.DefaultInterval).Should(Succeed()) } func expectAdmissionDenied(s *scaffold.Scaffold, resourceType, resourceName string, err error) { diff --git a/test/e2e/webhook/ingressclass.go b/test/e2e/webhook/ingressclass.go index 8192a51b..352cfb5a 100644 --- a/test/e2e/webhook/ingressclass.go +++ b/test/e2e/webhook/ingressclass.go @@ -19,7 +19,6 @@ package webhook import ( "fmt" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -52,28 +51,31 @@ spec: scope: "Namespace" ` - output, err := s.CreateResourceFromStringAndGetOutput(fmt.Sprintf(icYAML, s.GetControllerName(), missingName, s.Namespace())) - Expect(err).ShouldNot(HaveOccurred()) - Expect(output).To(ContainSubstring(fmt.Sprintf("Warning: Referenced GatewayProxy '%s/%s' not found.", s.Namespace(), missingName))) - - time.Sleep(2 * time.Second) + Eventually(func(g Gomega) { + output, err := s.CreateResourceFromStringAndGetOutput(fmt.Sprintf(icYAML, s.GetControllerName(), missingName, s.Namespace())) + g.Expect(err).ShouldNot(HaveOccurred()) + g.Expect(output).To(ContainSubstring(fmt.Sprintf("Warning: Referenced GatewayProxy '%s/%s' not found.", s.Namespace(), missingName))) + }).WithTimeout(scaffold.DefaultTimeout).ProbeEvery(scaffold.DefaultInterval).Should(Succeed()) By("updating IngressClass to reference another missing GatewayProxy") missingName2 := "missing-proxy-2" - output, err = s.CreateResourceFromStringAndGetOutput(fmt.Sprintf(icYAML, s.GetControllerName(), missingName2, s.Namespace())) - Expect(err).ShouldNot(HaveOccurred()) - Expect(output).To(ContainSubstring(fmt.Sprintf("Warning: Referenced GatewayProxy '%s/%s' not found.", s.Namespace(), missingName2))) + Eventually(func(g Gomega) { + output, err := s.CreateResourceFromStringAndGetOutput(fmt.Sprintf(icYAML, s.GetControllerName(), missingName2, s.Namespace())) + g.Expect(err).ShouldNot(HaveOccurred()) + g.Expect(output).To(ContainSubstring(fmt.Sprintf("Warning: Referenced GatewayProxy '%s/%s' not found.", s.Namespace(), missingName2))) + }).WithTimeout(scaffold.DefaultTimeout).ProbeEvery(scaffold.DefaultInterval).Should(Succeed()) By("create GatewayProxy") - err = s.CreateResourceFromString(s.GetGatewayProxySpec()) + err := s.CreateResourceFromString(s.GetGatewayProxySpec()) Expect(err).NotTo(HaveOccurred(), "creating GatewayProxy") - time.Sleep(5 * time.Second) By("updating IngressClass to reference an existing GatewayProxy") existingName := "apisix-proxy-config" - output, err = s.CreateResourceFromStringAndGetOutput(fmt.Sprintf(icYAML, s.GetControllerName(), existingName, s.Namespace())) - Expect(err).ShouldNot(HaveOccurred()) - Expect(output).NotTo(ContainSubstring(fmt.Sprintf("Warning: Referenced GatewayProxy '%s/%s' not found.", s.Namespace(), existingName))) + Eventually(func(g Gomega) { + output, err := s.CreateResourceFromStringAndGetOutput(fmt.Sprintf(icYAML, s.GetControllerName(), existingName, s.Namespace())) + g.Expect(err).ShouldNot(HaveOccurred()) + g.Expect(output).NotTo(ContainSubstring(fmt.Sprintf("Warning: Referenced GatewayProxy '%s/%s' not found.", s.Namespace(), existingName))) + }).WithTimeout(scaffold.DefaultTimeout).ProbeEvery(scaffold.DefaultInterval).Should(Succeed()) By("deleting IngressClass") err = s.DeleteResource("IngressClass", "apisix-with-missing") diff --git a/test/e2e/webhook/ssl_conflict.go b/test/e2e/webhook/ssl_conflict.go index 8c0ff4bb..583a30d4 100644 --- a/test/e2e/webhook/ssl_conflict.go +++ b/test/e2e/webhook/ssl_conflict.go @@ -19,7 +19,6 @@ package webhook import ( "fmt" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -37,17 +36,14 @@ var _ = Describe("Test SSL/TLS Conflict Detection", Label("webhook"), func() { By("creating GatewayProxy") err := s.CreateResourceFromString(s.GetGatewayProxySpec()) Expect(err).NotTo(HaveOccurred(), "creating GatewayProxy") - time.Sleep(5 * time.Second) By("creating GatewayClass") err = s.CreateResourceFromString(s.GetGatewayClassYaml()) Expect(err).NotTo(HaveOccurred(), "creating GatewayClass") - time.Sleep(2 * time.Second) By("creating IngressClass") err = s.CreateResourceFromStringWithNamespace(s.GetIngressClassYaml(), "") Expect(err).NotTo(HaveOccurred(), "creating IngressClass") - time.Sleep(2 * time.Second) }) Context("ApisixTls conflict detection", func() { @@ -59,7 +55,6 @@ var _ = Describe("Test SSL/TLS Conflict Detection", Label("webhook"), func() { By("creating two different TLS secrets") createApisixTLSSecret(s, secretA, host, "creating secret A") createApisixTLSSecret(s, secretB, host, "creating secret B") - time.Sleep(2 * time.Second) By("creating first ApisixTls with certificate A") tlsAYAML := fmt.Sprintf(` @@ -79,8 +74,6 @@ spec: err := s.CreateResourceFromString(tlsAYAML) Expect(err).NotTo(HaveOccurred(), "creating ApisixTls A") - time.Sleep(2 * time.Second) - By("attempting to create second ApisixTls with certificate B for same host") tlsBYAML := fmt.Sprintf(` apiVersion: apisix.apache.org/v2 @@ -96,11 +89,13 @@ spec: name: %s namespace: %s `, s.Namespace(), s.Namespace(), host, secretB, s.Namespace()) - err = s.CreateResourceFromString(tlsBYAML) - Expect(err).Should(HaveOccurred(), "expecting conflict when creating ApisixTls B") - Expect(err.Error()).To(ContainSubstring("SSL configuration conflicts detected")) - Expect(err.Error()).To(ContainSubstring(host)) - Expect(err.Error()).To(ContainSubstring("ApisixTls")) + Eventually(func(g Gomega) { + err = s.CreateResourceFromString(tlsBYAML) + g.Expect(err).Should(HaveOccurred(), "expecting conflict when creating ApisixTls B") + g.Expect(err.Error()).To(ContainSubstring("SSL configuration conflicts detected")) + g.Expect(err.Error()).To(ContainSubstring(host)) + g.Expect(err.Error()).To(ContainSubstring("ApisixTls")) + }).WithTimeout(scaffold.DefaultTimeout).ProbeEvery(scaffold.DefaultInterval).Should(Succeed()) }) It("should allow ApisixTls with same certificate for same host", func() { @@ -110,8 +105,6 @@ spec: By("creating a shared TLS secret") createKubeTLSSecret(s, sharedSecret, host, "creating shared secret") - time.Sleep(2 * time.Second) - By("creating first ApisixTls with shared certificate") tls1YAML := fmt.Sprintf(` apiVersion: apisix.apache.org/v2 @@ -130,8 +123,6 @@ spec: err := s.CreateResourceFromString(tls1YAML) Expect(err).NotTo(HaveOccurred(), "creating first ApisixTls") - time.Sleep(2 * time.Second) - By("creating second ApisixTls with same certificate for same host") tls2YAML := fmt.Sprintf(` apiVersion: apisix.apache.org/v2 @@ -147,8 +138,10 @@ spec: name: %s namespace: %s `, s.Namespace(), s.Namespace(), host, sharedSecret, s.Namespace()) - err = s.CreateResourceFromString(tls2YAML) - Expect(err).NotTo(HaveOccurred(), "second ApisixTls should be allowed with same certificate") + Eventually(func(g Gomega) { + err = s.CreateResourceFromString(tls2YAML) + g.Expect(err).NotTo(HaveOccurred(), "second ApisixTls should be allowed with same certificate") + }).WithTimeout(scaffold.DefaultTimeout).ProbeEvery(scaffold.DefaultInterval).Should(Succeed()) }) }) @@ -162,8 +155,6 @@ spec: createKubeTLSSecret(s, secretA, host, "creating secret A") createKubeTLSSecret(s, secretB, host, "creating secret B") - time.Sleep(2 * time.Second) - By("creating ApisixTls with certificate A") tlsYAML := fmt.Sprintf(` apiVersion: apisix.apache.org/v2 @@ -182,8 +173,6 @@ spec: err := s.CreateResourceFromString(tlsYAML) Expect(err).NotTo(HaveOccurred(), "creating ApisixTls") - time.Sleep(2 * time.Second) - By("attempting to create Gateway with certificate B for same host") hostname := host gatewayYAML := fmt.Sprintf(` @@ -209,10 +198,12 @@ spec: kind: GatewayProxy name: apisix-proxy-config `, s.Namespace(), s.Namespace(), hostname, secretB) - err = s.CreateResourceFromString(gatewayYAML) - Expect(err).Should(HaveOccurred(), "expecting conflict when creating Gateway") - Expect(err.Error()).To(ContainSubstring("SSL configuration conflicts detected")) - Expect(err.Error()).To(ContainSubstring(host)) + Eventually(func(g Gomega) { + err = s.CreateResourceFromString(gatewayYAML) + g.Expect(err).Should(HaveOccurred(), "expecting conflict when creating Gateway") + g.Expect(err.Error()).To(ContainSubstring("SSL configuration conflicts detected")) + g.Expect(err.Error()).To(ContainSubstring(host)) + }).WithTimeout(scaffold.DefaultTimeout).ProbeEvery(scaffold.DefaultInterval).Should(Succeed()) }) It("should allow Gateway with same certificate as existing ApisixTls", func() { @@ -222,8 +213,6 @@ spec: By("creating a shared TLS secret") createKubeTLSSecret(s, sharedSecret, host, "creating shared secret") - time.Sleep(2 * time.Second) - By("creating ApisixTls with shared certificate") tlsYAML := fmt.Sprintf(` apiVersion: apisix.apache.org/v2 @@ -242,8 +231,6 @@ spec: err := s.CreateResourceFromString(tlsYAML) Expect(err).NotTo(HaveOccurred(), "creating ApisixTls") - time.Sleep(2 * time.Second) - By("creating Gateway with same certificate") hostname := host gatewayYAML := fmt.Sprintf(` @@ -269,8 +256,10 @@ spec: kind: GatewayProxy name: apisix-proxy-config `, s.Namespace(), s.Namespace(), hostname, sharedSecret) - err = s.CreateResourceFromString(gatewayYAML) - Expect(err).NotTo(HaveOccurred(), "Gateway should be allowed with same certificate") + Eventually(func(g Gomega) { + err = s.CreateResourceFromString(gatewayYAML) + g.Expect(err).NotTo(HaveOccurred(), "Gateway should be allowed with same certificate") + }).WithTimeout(scaffold.DefaultTimeout).ProbeEvery(scaffold.DefaultInterval).Should(Succeed()) }) It("should reject ApisixTls when Gateway without hostname uses different certificate", func() { @@ -282,8 +271,6 @@ spec: createKubeTLSSecret(s, secretA, host, "creating secret A") createKubeTLSSecret(s, secretB, host, "creating secret B") - time.Sleep(2 * time.Second) - By("creating Gateway without explicit hostname using certificate A") gatewayYAML := fmt.Sprintf(` apiVersion: gateway.networking.k8s.io/v1 @@ -310,8 +297,6 @@ spec: err := s.CreateResourceFromString(gatewayYAML) Expect(err).NotTo(HaveOccurred(), "creating Gateway without hostname") - time.Sleep(2 * time.Second) - By("attempting to create ApisixTls with certificate B for same host") tlsYAML := fmt.Sprintf(` apiVersion: apisix.apache.org/v2 @@ -327,10 +312,12 @@ spec: name: %s namespace: %s `, s.Namespace(), s.Namespace(), host, secretB, s.Namespace()) - err = s.CreateResourceFromString(tlsYAML) - Expect(err).Should(HaveOccurred(), "expecting conflict when creating ApisixTls without hostname on existing Gateway") - Expect(err.Error()).To(ContainSubstring("SSL configuration conflicts detected")) - Expect(err.Error()).To(ContainSubstring(host)) + Eventually(func(g Gomega) { + err = s.CreateResourceFromString(tlsYAML) + g.Expect(err).Should(HaveOccurred(), "expecting conflict when creating ApisixTls without hostname on existing Gateway") + g.Expect(err.Error()).To(ContainSubstring("SSL configuration conflicts detected")) + g.Expect(err.Error()).To(ContainSubstring(host)) + }).WithTimeout(scaffold.DefaultTimeout).ProbeEvery(scaffold.DefaultInterval).Should(Succeed()) }) }) @@ -344,8 +331,6 @@ spec: createKubeTLSSecret(s, secretA, host, "creating secret A") createKubeTLSSecret(s, secretB, host, "creating secret B") - time.Sleep(2 * time.Second) - By("attempting to create Gateway with two listeners using different certificates for same host") hostname := host gatewayYAML := fmt.Sprintf(` @@ -379,10 +364,12 @@ spec: kind: GatewayProxy name: apisix-proxy-config `, s.Namespace(), s.Namespace(), hostname, secretA, hostname, secretB) - err := s.CreateResourceFromString(gatewayYAML) - Expect(err).Should(HaveOccurred(), "expecting self-conflict in Gateway") - Expect(err.Error()).To(ContainSubstring("SSL configuration conflicts detected")) - Expect(err.Error()).To(ContainSubstring(host)) + Eventually(func(g Gomega) { + err := s.CreateResourceFromString(gatewayYAML) + g.Expect(err).Should(HaveOccurred(), "expecting self-conflict in Gateway") + g.Expect(err.Error()).To(ContainSubstring("SSL configuration conflicts detected")) + g.Expect(err.Error()).To(ContainSubstring(host)) + }).WithTimeout(scaffold.DefaultTimeout).ProbeEvery(scaffold.DefaultInterval).Should(Succeed()) }) }) @@ -396,8 +383,6 @@ spec: createKubeTLSSecret(s, secretA, host, "creating secret A") createKubeTLSSecret(s, secretB, host, "creating secret B") - time.Sleep(2 * time.Second) - By("creating a backend service for Ingress") serviceYAML := fmt.Sprintf(` apiVersion: v1 @@ -415,8 +400,6 @@ spec: err := s.CreateResourceFromString(serviceYAML) Expect(err).NotTo(HaveOccurred(), "creating service") - time.Sleep(2 * time.Second) - By("attempting to create Ingress with two TLS configs using different certificates for same host") ingressYAML := fmt.Sprintf(` apiVersion: networking.k8s.io/v1 @@ -445,10 +428,12 @@ spec: port: number: 80 `, s.Namespace(), s.Namespace(), host, secretA, host, secretB, host) - err = s.CreateResourceFromString(ingressYAML) - Expect(err).Should(HaveOccurred(), "expecting self-conflict in Ingress") - Expect(err.Error()).To(ContainSubstring("SSL configuration conflicts detected")) - Expect(err.Error()).To(ContainSubstring(host)) + Eventually(func(g Gomega) { + err = s.CreateResourceFromString(ingressYAML) + g.Expect(err).Should(HaveOccurred(), "expecting self-conflict in Ingress") + g.Expect(err.Error()).To(ContainSubstring("SSL configuration conflicts detected")) + g.Expect(err.Error()).To(ContainSubstring(host)) + }).WithTimeout(scaffold.DefaultTimeout).ProbeEvery(scaffold.DefaultInterval).Should(Succeed()) }) It("should reject Ingress with conflicting certificate against existing ApisixTls", func() { @@ -460,8 +445,6 @@ spec: createKubeTLSSecret(s, secretA, host, "creating secret A") createKubeTLSSecret(s, secretB, host, "creating secret B") - time.Sleep(2 * time.Second) - By("creating ApisixTls with certificate A") tlsYAML := fmt.Sprintf(` apiVersion: apisix.apache.org/v2 @@ -480,8 +463,6 @@ spec: err := s.CreateResourceFromString(tlsYAML) Expect(err).NotTo(HaveOccurred(), "creating ApisixTls") - time.Sleep(2 * time.Second) - By("creating a backend service for Ingress") serviceYAML := fmt.Sprintf(` apiVersion: v1 @@ -499,8 +480,6 @@ spec: err = s.CreateResourceFromString(serviceYAML) Expect(err).NotTo(HaveOccurred(), "creating service") - time.Sleep(2 * time.Second) - By("attempting to create Ingress with certificate B for same host") ingressYAML := fmt.Sprintf(` apiVersion: networking.k8s.io/v1 @@ -526,10 +505,12 @@ spec: port: number: 80 `, s.Namespace(), s.Namespace(), host, secretB, host) - err = s.CreateResourceFromString(ingressYAML) - Expect(err).Should(HaveOccurred(), "expecting conflict when creating Ingress") - Expect(err.Error()).To(ContainSubstring("SSL configuration conflicts detected")) - Expect(err.Error()).To(ContainSubstring(host)) + Eventually(func(g Gomega) { + err = s.CreateResourceFromString(ingressYAML) + g.Expect(err).Should(HaveOccurred(), "expecting conflict when creating Ingress") + g.Expect(err.Error()).To(ContainSubstring("SSL configuration conflicts detected")) + g.Expect(err.Error()).To(ContainSubstring(host)) + }).WithTimeout(scaffold.DefaultTimeout).ProbeEvery(scaffold.DefaultInterval).Should(Succeed()) }) It("should allow Ingress with same certificate as existing Gateway", func() { @@ -539,8 +520,6 @@ spec: By("creating a shared TLS secret") createKubeTLSSecret(s, sharedSecret, host, "creating shared secret") - time.Sleep(2 * time.Second) - By("creating Gateway with shared certificate") hostname := host gatewayYAML := fmt.Sprintf(` @@ -569,8 +548,6 @@ spec: err := s.CreateResourceFromString(gatewayYAML) Expect(err).NotTo(HaveOccurred(), "creating Gateway") - time.Sleep(2 * time.Second) - By("creating a backend service for Ingress") serviceYAML := fmt.Sprintf(` apiVersion: v1 @@ -588,8 +565,6 @@ spec: err = s.CreateResourceFromString(serviceYAML) Expect(err).NotTo(HaveOccurred(), "creating service") - time.Sleep(2 * time.Second) - By("creating Ingress with same certificate") ingressYAML := fmt.Sprintf(` apiVersion: networking.k8s.io/v1 @@ -615,8 +590,10 @@ spec: port: number: 80 `, s.Namespace(), s.Namespace(), host, sharedSecret, host) - err = s.CreateResourceFromString(ingressYAML) - Expect(err).NotTo(HaveOccurred(), "Ingress should be allowed with same certificate") + Eventually(func(g Gomega) { + err = s.CreateResourceFromString(ingressYAML) + g.Expect(err).NotTo(HaveOccurred(), "Ingress should be allowed with same certificate") + }).WithTimeout(scaffold.DefaultTimeout).ProbeEvery(scaffold.DefaultInterval).Should(Succeed()) }) It("should reject Ingress when Gateway without hostname uses different certificate", func() { @@ -628,8 +605,6 @@ spec: createKubeTLSSecret(s, secretA, host, "creating secret A") createKubeTLSSecret(s, secretB, host, "creating secret B") - time.Sleep(2 * time.Second) - By("creating Gateway without explicit hostname using certificate A") gatewayYAML := fmt.Sprintf(` apiVersion: gateway.networking.k8s.io/v1 @@ -656,8 +631,6 @@ spec: err := s.CreateResourceFromString(gatewayYAML) Expect(err).NotTo(HaveOccurred(), "creating Gateway without hostname") - time.Sleep(2 * time.Second) - By("creating a backend service for Ingress") serviceYAML := fmt.Sprintf(` apiVersion: v1 @@ -675,8 +648,6 @@ spec: err = s.CreateResourceFromString(serviceYAML) Expect(err).NotTo(HaveOccurred(), "creating service") - time.Sleep(2 * time.Second) - By("attempting to create Ingress without explicit host using certificate B") ingressYAML := fmt.Sprintf(` apiVersion: networking.k8s.io/v1 @@ -699,10 +670,12 @@ spec: port: number: 80 `, s.Namespace(), s.Namespace(), secretB) - err = s.CreateResourceFromString(ingressYAML) - Expect(err).Should(HaveOccurred(), "expecting conflict when creating Ingress without hostname") - Expect(err.Error()).To(ContainSubstring("SSL configuration conflicts detected")) - Expect(err.Error()).To(ContainSubstring(host)) + Eventually(func(g Gomega) { + err = s.CreateResourceFromString(ingressYAML) + g.Expect(err).Should(HaveOccurred(), "expecting conflict when creating Ingress without hostname") + g.Expect(err.Error()).To(ContainSubstring("SSL configuration conflicts detected")) + g.Expect(err.Error()).To(ContainSubstring(host)) + }).WithTimeout(scaffold.DefaultTimeout).ProbeEvery(scaffold.DefaultInterval).Should(Succeed()) }) }) @@ -737,8 +710,6 @@ spec: err := s.CreateResourceFromStringWithNamespace(defaultIngressClassYAML, "") Expect(err).NotTo(HaveOccurred(), "creating default IngressClass") - time.Sleep(2 * time.Second) - By("creating backend service for default ingress test") serviceYAML := fmt.Sprintf(` apiVersion: v1 @@ -756,8 +727,6 @@ spec: err = s.CreateResourceFromString(serviceYAML) Expect(err).NotTo(HaveOccurred(), "creating service") - time.Sleep(2 * time.Second) - By("creating baseline Ingress with certificate A") ingressAYAML := fmt.Sprintf(` apiVersion: networking.k8s.io/v1 @@ -785,8 +754,6 @@ spec: err = s.CreateResourceFromString(ingressAYAML) Expect(err).NotTo(HaveOccurred(), "creating baseline Ingress") - time.Sleep(2 * time.Second) - By("attempting to create second Ingress with conflicting certificate via default class") ingressBYAML := fmt.Sprintf(` apiVersion: networking.k8s.io/v1 @@ -811,10 +778,12 @@ spec: port: number: 80 `, s.Namespace(), host, secretB, host) - err = s.CreateResourceFromString(ingressBYAML) - Expect(err).Should(HaveOccurred(), "expecting conflict when creating second Ingress") - Expect(err.Error()).To(ContainSubstring("SSL configuration conflicts detected")) - Expect(err.Error()).To(ContainSubstring(host)) + Eventually(func(g Gomega) { + err = s.CreateResourceFromString(ingressBYAML) + g.Expect(err).Should(HaveOccurred(), "expecting conflict when creating second Ingress") + g.Expect(err.Error()).To(ContainSubstring("SSL configuration conflicts detected")) + g.Expect(err.Error()).To(ContainSubstring(host)) + }).WithTimeout(scaffold.DefaultTimeout).ProbeEvery(scaffold.DefaultInterval).Should(Succeed()) }) It("should reject ApisixTls without explicit class when default class uses a different certificate", func() { @@ -847,8 +816,6 @@ spec: err := s.CreateResourceFromStringWithNamespace(defaultIngressClassYAML, "") Expect(err).NotTo(HaveOccurred(), "creating default IngressClass") - time.Sleep(2 * time.Second) - By("creating baseline ApisixTls without explicit ingress class") tlsAYAML := fmt.Sprintf(` apiVersion: apisix.apache.org/v2 @@ -866,8 +833,6 @@ spec: err = s.CreateResourceFromString(tlsAYAML) Expect(err).NotTo(HaveOccurred(), "creating baseline ApisixTls") - time.Sleep(2 * time.Second) - By("attempting to create ApisixTls with conflicting certificate without class override") tlsBYAML := fmt.Sprintf(` apiVersion: apisix.apache.org/v2 @@ -882,10 +847,12 @@ spec: name: %s namespace: %s `, s.Namespace(), host, secretB, s.Namespace()) - err = s.CreateResourceFromString(tlsBYAML) - Expect(err).Should(HaveOccurred(), "expecting conflict when creating second ApisixTls") - Expect(err.Error()).To(ContainSubstring("SSL configuration conflicts detected")) - Expect(err.Error()).To(ContainSubstring(host)) + Eventually(func(g Gomega) { + err = s.CreateResourceFromString(tlsBYAML) + g.Expect(err).Should(HaveOccurred(), "expecting conflict when creating second ApisixTls") + g.Expect(err.Error()).To(ContainSubstring("SSL configuration conflicts detected")) + g.Expect(err.Error()).To(ContainSubstring(host)) + }).WithTimeout(scaffold.DefaultTimeout).ProbeEvery(scaffold.DefaultInterval).Should(Succeed()) }) }) @@ -917,8 +884,6 @@ spec: err := s.CreateResourceFromString(tlsYAML) Expect(err).NotTo(HaveOccurred(), "creating baseline ApisixTls for ingress update") - time.Sleep(2 * time.Second) - By("creating backend service for ingress update test") serviceYAML := fmt.Sprintf(` apiVersion: v1 @@ -936,8 +901,6 @@ spec: err = s.CreateResourceFromString(serviceYAML) Expect(err).NotTo(HaveOccurred(), "creating service") - time.Sleep(2 * time.Second) - By("creating initial Ingress with matching certificate") ingressBaseYAML := fmt.Sprintf(` apiVersion: networking.k8s.io/v1 @@ -966,8 +929,6 @@ spec: err = s.CreateResourceFromString(ingressBaseYAML) Expect(err).NotTo(HaveOccurred(), "creating initial Ingress") - time.Sleep(2 * time.Second) - By("attempting to update Ingress to use conflicting certificate B") ingressUpdatedYAML := fmt.Sprintf(` apiVersion: networking.k8s.io/v1 @@ -993,10 +954,12 @@ spec: port: number: 80 `, s.Namespace(), s.Namespace(), host, secretB, host) - err = s.CreateResourceFromString(ingressUpdatedYAML) - Expect(err).Should(HaveOccurred(), "expecting conflict when updating Ingress certificate") - Expect(err.Error()).To(ContainSubstring("SSL configuration conflicts detected")) - Expect(err.Error()).To(ContainSubstring(host)) + Eventually(func(g Gomega) { + err = s.CreateResourceFromString(ingressUpdatedYAML) + g.Expect(err).Should(HaveOccurred(), "expecting conflict when updating Ingress certificate") + g.Expect(err.Error()).To(ContainSubstring("SSL configuration conflicts detected")) + g.Expect(err.Error()).To(ContainSubstring(host)) + }).WithTimeout(scaffold.DefaultTimeout).ProbeEvery(scaffold.DefaultInterval).Should(Succeed()) }) It("should reject Gateway update that switches to a conflicting certificate", func() { @@ -1026,8 +989,6 @@ spec: err := s.CreateResourceFromString(tlsYAML) Expect(err).NotTo(HaveOccurred(), "creating baseline ApisixTls for gateway update") - time.Sleep(2 * time.Second) - By("creating initial Gateway using certificate A") gatewayBaseYAML := fmt.Sprintf(` apiVersion: gateway.networking.k8s.io/v1 @@ -1055,8 +1016,6 @@ spec: err = s.CreateResourceFromString(gatewayBaseYAML) Expect(err).NotTo(HaveOccurred(), "creating initial Gateway") - time.Sleep(2 * time.Second) - By("attempting to update Gateway to use conflicting certificate B") gatewayUpdatedYAML := fmt.Sprintf(` apiVersion: gateway.networking.k8s.io/v1 @@ -1081,10 +1040,12 @@ spec: kind: GatewayProxy name: apisix-proxy-config `, s.Namespace(), s.Namespace(), host, secretB) - err = s.CreateResourceFromString(gatewayUpdatedYAML) - Expect(err).Should(HaveOccurred(), "expecting conflict when updating Gateway certificate") - Expect(err.Error()).To(ContainSubstring("SSL configuration conflicts detected")) - Expect(err.Error()).To(ContainSubstring(host)) + Eventually(func(g Gomega) { + err = s.CreateResourceFromString(gatewayUpdatedYAML) + g.Expect(err).Should(HaveOccurred(), "expecting conflict when updating Gateway certificate") + g.Expect(err.Error()).To(ContainSubstring("SSL configuration conflicts detected")) + g.Expect(err.Error()).To(ContainSubstring(host)) + }).WithTimeout(scaffold.DefaultTimeout).ProbeEvery(scaffold.DefaultInterval).Should(Succeed()) }) }) @@ -1100,8 +1061,6 @@ spec: createKubeTLSSecret(s, secretB, host, "creating secret B") createKubeTLSSecret(s, secretC, host, "creating secret C") - time.Sleep(2 * time.Second) - By("creating Gateway with certificate A") hostname := host gatewayYAML := fmt.Sprintf(` @@ -1130,8 +1089,6 @@ spec: err := s.CreateResourceFromString(gatewayYAML) Expect(err).NotTo(HaveOccurred(), "creating Gateway with cert A") - time.Sleep(2 * time.Second) - By("attempting to create ApisixTls with certificate B") tlsYAML := fmt.Sprintf(` apiVersion: apisix.apache.org/v2 @@ -1147,9 +1104,11 @@ spec: name: %s namespace: %s `, s.Namespace(), s.Namespace(), host, secretB, s.Namespace()) - err = s.CreateResourceFromString(tlsYAML) - Expect(err).Should(HaveOccurred(), "expecting conflict when creating ApisixTls with different cert") - Expect(err.Error()).To(ContainSubstring("SSL configuration conflicts detected")) + Eventually(func(g Gomega) { + err = s.CreateResourceFromString(tlsYAML) + g.Expect(err).Should(HaveOccurred(), "expecting conflict when creating ApisixTls with different cert") + g.Expect(err.Error()).To(ContainSubstring("SSL configuration conflicts detected")) + }).WithTimeout(scaffold.DefaultTimeout).ProbeEvery(scaffold.DefaultInterval).Should(Succeed()) By("creating a backend service") serviceYAML := fmt.Sprintf(` @@ -1168,8 +1127,6 @@ spec: err = s.CreateResourceFromString(serviceYAML) Expect(err).NotTo(HaveOccurred(), "creating service") - time.Sleep(2 * time.Second) - By("attempting to create Ingress with certificate C") ingressYAML := fmt.Sprintf(` apiVersion: networking.k8s.io/v1 @@ -1195,9 +1152,11 @@ spec: port: number: 80 `, s.Namespace(), s.Namespace(), host, secretC, host) - err = s.CreateResourceFromString(ingressYAML) - Expect(err).Should(HaveOccurred(), "expecting conflict when creating Ingress with different cert") - Expect(err.Error()).To(ContainSubstring("SSL configuration conflicts detected")) + Eventually(func(g Gomega) { + err = s.CreateResourceFromString(ingressYAML) + g.Expect(err).Should(HaveOccurred(), "expecting conflict when creating Ingress with different cert") + g.Expect(err.Error()).To(ContainSubstring("SSL configuration conflicts detected")) + }).WithTimeout(scaffold.DefaultTimeout).ProbeEvery(scaffold.DefaultInterval).Should(Succeed()) }) }) })