diff --git a/test/e2e/upgrade/ingress_controller.go b/test/e2e/upgrade/ingress_controller.go new file mode 100644 index 000000000000..ac55d9a64998 --- /dev/null +++ b/test/e2e/upgrade/ingress_controller.go @@ -0,0 +1,224 @@ +package upgrade + +import ( + "fmt" + "time" + + g "github.com/onsi/ginkgo" + o "github.com/onsi/gomega" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + kapierrs "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/kubernetes/test/e2e/framework" + "k8s.io/kubernetes/test/e2e/upgrades" + + routev1 "github.com/openshift/api/route/v1" + operatorclientset "github.com/openshift/client-go/operator/clientset/versioned" + routeclientset "github.com/openshift/client-go/route/clientset/versioned" + "github.com/openshift/origin/test/extended/util/url" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// IngressControllerUpgradeTest tests that the default ingress controller is +// available before and after a cluster upgrade. During a master-only upgrade, +// it will test that the ingress controller remains available during as well. +type IngressControllerUpgradeTest struct { + urlTester *url.Tester + routeTest *url.Test +} + +func (IngressControllerUpgradeTest) Name() string { return "ingress-controller-upgrade" } + +// Setup creates a route and makes sure it's reachable through the default +// ingress controller. +func (t *IngressControllerUpgradeTest) Setup(f *framework.Framework) { + var ( + routeName = "ingress-controller-test" + deploymentName = "ingress-controller-test" + serviceName = "ingress-controller-test" + ) + + ns := f.Namespace.Name + + g.By("creating an HTTP echo server deployment " + deploymentName + " in namespace " + ns) + _, err := f.ClientSet.AppsV1().Deployments(ns).Create(&appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: deploymentName, + Namespace: ns, + }, + Spec: appsv1.DeploymentSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "http-echo", + "deployment": deploymentName, + }, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "app": "http-echo", + "deployment": deploymentName, + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "http-echo", + Image: "openshift/origin-node", + Command: []string{ + "/bin/socat", + "TCP4-LISTEN:8080,reuseaddr,fork", + `EXEC:'/bin/bash -c "printf \\"HTTP/1.0 200 OK\\\\r\\\\n\\\\r\\\\n\\"; sed -e \\"/^\\\\\r/q\\""'`, + }, + Ports: []corev1.ContainerPort{ + { + ContainerPort: 8080, + Name: "http", + Protocol: corev1.ProtocolTCP, + }, + }, + }, + }, + }, + }, + }, + }) + o.Expect(err).ToNot(o.HaveOccurred()) + + g.By("creating a service " + serviceName + " in namespace " + ns) + _, err = f.ClientSet.CoreV1().Services(ns).Create(&corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: serviceName, + Namespace: ns, + }, + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{ + { + Port: 8080, + Name: "http", + Protocol: corev1.ProtocolTCP, + }, + }, + Selector: map[string]string{ + "app": "http-echo", + }, + }, + }) + o.Expect(err).ToNot(o.HaveOccurred()) + + g.By("creating a route " + routeName + " in namespace " + ns) + restconfig, err := framework.LoadConfig() + o.Expect(err).NotTo(o.HaveOccurred()) + routeclient := routeclientset.NewForConfigOrDie(restconfig).RouteV1() + _, err = routeclient.Routes(ns).Create(&routev1.Route{ + ObjectMeta: metav1.ObjectMeta{ + Name: routeName, + Namespace: ns, + }, + Spec: routev1.RouteSpec{ + Host: "www.example.com", + To: routev1.RouteTargetReference{ + Kind: "Service", + Name: serviceName, + }, + }, + }) + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("looking up the ingress controller's endpoint publishing strategy") + foundLoadBalancerServiceStrategyType := false + operatorclient := operatorclientset.NewForConfigOrDie(restconfig).OperatorV1() + err = wait.PollImmediate(2*time.Second, 30*time.Second, func() (bool, error) { + ic, err := operatorclient.IngressControllers("openshift-ingress-operator").Get("default", metav1.GetOptions{}) + if kapierrs.IsNotFound(err) { + return false, nil + } + o.Expect(err).NotTo(o.HaveOccurred()) + if ic.Status.EndpointPublishingStrategy == nil { + return false, nil + } + if ic.Status.EndpointPublishingStrategy.Type == "LoadBalancerService" { + foundLoadBalancerServiceStrategyType = true + } + return true, nil + }) + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("looking up the ingress controller's endpoint") + routerServiceName := "router-internal-default" + if foundLoadBalancerServiceStrategyType { + routerServiceName = "router-default" + } + var endpoint string + err = wait.PollImmediate(2*time.Second, 60*time.Second, func() (bool, error) { + svc, err := f.ClientSet.CoreV1().Services("openshift-ingress").Get(routerServiceName, metav1.GetOptions{}) + if kapierrs.IsNotFound(err) { + return false, nil + } + o.Expect(err).NotTo(o.HaveOccurred()) + if svc.Spec.Type == corev1.ServiceTypeLoadBalancer { + if len(svc.Status.LoadBalancer.Ingress) != 0 { + if len(svc.Status.LoadBalancer.Ingress[0].IP) != 0 { + endpoint = svc.Status.LoadBalancer.Ingress[0].IP + return true, nil + } + if len(svc.Status.LoadBalancer.Ingress[0].Hostname) != 0 { + endpoint = svc.Status.LoadBalancer.Ingress[0].Hostname + return true, nil + } + } + return false, nil + } + endpoint = svc.Spec.ClusterIP + return true, nil + }) + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By(fmt.Sprintf("hitting the route")) + t.urlTester = url.NewTester(f.ClientSet, ns) + t.routeTest = url.Expect("GET", "http://www.example.com").Through(endpoint).HasStatusCode(200) + t.urlTester.Within(3*time.Minute, t.routeTest) +} + +// Test runs a connectivity check. +func (t *IngressControllerUpgradeTest) Test(f *framework.Framework, done <-chan struct{}, upgrade upgrades.UpgradeType) { + switch upgrade { + case upgrades.MasterUpgrade, upgrades.NodeUpgrade, upgrades.ClusterUpgrade: + t.test(f, done, true) + default: + t.test(f, done, false) + } +} + +// Teardown cleans up any remaining resources. +func (t *IngressControllerUpgradeTest) Teardown(f *framework.Framework) { + // Rely on the namespace deletion to clean everything up. +} + +func (t *IngressControllerUpgradeTest) test(f *framework.Framework, done <-chan struct{}, testDuringDisruption bool) { + if testDuringDisruption { + g.By("continuously hitting the route") + wait.Until(func() { + // TODO: Use Within once we have confidence that this + // test is not flaky. + // t.urlTester.Within(1*time.Minute, t.routeTest) + defer g.GinkgoRecover() + response := t.urlTester.WithErrorPassthrough(true).Response(t.routeTest) + if response == nil { + framework.Logf("got nil response") + } else if len(response.Error) > 0 { + framework.Logf("got error for route: %v", response.Error) + } + }, framework.Poll, done) + } else { + g.By("waiting for upgrade to finish without checking if the route remains up") + <-done + } + + g.By("hitting the route again") + wait.Until(func() { + t.urlTester.Within(1*time.Minute, t.routeTest) + }, framework.Poll, done) +} diff --git a/test/e2e/upgrade/upgrade.go b/test/e2e/upgrade/upgrade.go index 1d5757802d85..467ca7cf2816 100644 --- a/test/e2e/upgrade/upgrade.go +++ b/test/e2e/upgrade/upgrade.go @@ -50,6 +50,7 @@ func AllTests() []upgrades.Test { // &upgrades.MySqlUpgradeTest{}, // &upgrades.EtcdUpgradeTest{}, // &upgrades.CassandraUpgradeTest{}, + &IngressControllerUpgradeTest{}, } }