diff --git a/hack/tests/e2e-ansible.sh b/hack/tests/e2e-ansible.sh index 05e5d95775..78b81ae28a 100755 --- a/hack/tests/e2e-ansible.sh +++ b/hack/tests/e2e-ansible.sh @@ -1,215 +1,26 @@ #!/usr/bin/env bash -set -eu +set -o errexit +set -o nounset +set -o pipefail -source hack/lib/test_lib.sh -source hack/lib/image_lib.sh -source ./hack/lib/common.sh +source ./hack/lib/test_lib.sh +source ./hack/lib/image_lib.sh + +# install SDK binaries +make install # ansible proxy test require a running cluster; run during e2e instead go test -count=1 ./internal/ansible/proxy/... -DEST_IMAGE="quay.io/example/memcached-operator:v0.0.2" -ROOTDIR="$(pwd)" -TMPDIR="$(mktemp -d)" -trap_add 'rm -rf $TMPDIR' EXIT - -setup_envs $tmp_sdk_root - -deploy_operator() { - header_text "Running deploy operator" - IMG=$DEST_IMAGE make deploy - kubectl create clusterrolebinding memcached-operator-system-metrics-reader --clusterrole=memcached-operator-metrics-reader --serviceaccount=default:default -} - -remove_operator() { - pushd $TMPDIR/memcached-operator - header_text "Running remove operator" - kubectl delete --ignore-not-found clusterrolebinding memcached-operator-system-metrics-reader - make undeploy - popd -} - -operator_logs() { - header_text "Getting Pod logs" - kubectl describe pods - header_text "Getting events" - kubectl get events - header_text "Getting operator logs" - kubectl logs deployment/memcached-operator-controller-manager -c manager -} - -test_operator() { - header_text "Testing operator metrics" - header_text "wait for operator pod to run" - if ! timeout 1m kubectl rollout status deployment/memcached-operator-controller-manager; - then - error_text "FAIL: Failed to run" - operator_logs - exit 1 - fi - - header_text "verify that metrics service was created" - if ! timeout 60s bash -c -- "until kubectl get service/memcached-operator-controller-manager-metrics-service > /dev/null 2>&1; do sleep 1; done"; - then - error_text "FAIL: Failed to get metrics service" - operator_logs - exit 1 - fi - - header_text "verify that the metrics endpoint exists" - serviceaccount_secret=$(kubectl get serviceaccounts default -n default -o jsonpath='{.secrets[0].name}') - token=$(kubectl get secret ${serviceaccount_secret} -n default -o jsonpath={.data.token} | base64 -d) - - # verify that the metrics endpoint exists - if ! timeout 60s bash -c -- "until kubectl run --attach --rm --restart=Never --namespace=default test-metrics --image=${METRICS_TEST_IMAGE} -- -sfkH \"Authorization: Bearer ${token}\" https://memcached-operator-controller-manager-metrics-service:8443/metrics; do sleep 1; done"; - then - error_text "Failed to verify that metrics endpoint exists" - operator_logs - exit 1 - fi - - header_text "create custom resource (Memcached CR)" - kubectl create -f config/samples/ansible_v1alpha1_memcached.yaml - if ! timeout 60s bash -c -- 'until kubectl get deployment -l app=memcached | grep memcached; do sleep 1; done'; - then - error_text "FAIL: Failed to verify to create memcached Deployment" - operator_logs - exit 1 - fi - - header_text "Wait for Operator Pod" - if ! timeout 60s bash -c -- "until kubectl get pod -l control-plane=controller-manager; do sleep 1; done" - then - error_text "FAIL: Operator pod does not exist." - operator_logs - exit 1 - fi - - header_text "Ensure no liveness probe fail events" - # We can't directly hit the endpoint, which is not publicly exposed. If k8s sees a failing endpoint, it will create a "Killing" event. - live_pod=$(kubectl get pod -l control-plane=controller-manager -o jsonpath="{..metadata.name}") - if kubectl get events --field-selector involvedObject.name=$live_pod | grep Killing - then - error_text "FAIL: Operator pod killed due to failed liveness probe." - kubectl get events --field-selector involvedObject.name=$live_pod,reason=Killing - operator_logs - exit 1 - fi +# create test directories +test_dir=./test +tests=$test_dir/e2e-ansible - header_text "Verify that a config map owned by the CR has been created." - if ! timeout 1m bash -c -- "until kubectl get configmap test-blacklist-watches > /dev/null 2>&1; do sleep 1; done"; - then - error_text "FAIL: Unable to retrieve config map test-blacklist-watches." - operator_logs - exit 1 - fi +export TRACE=1 +export GO111MODULE=on - header_text "Verify that config map requests skip the cache." - if ! kubectl logs deployment/memcached-operator-controller-manager -c manager | grep -e "Skipping cache lookup\".*"Path\":\"\/api\/v1\/namespaces\/default\/configmaps\/test-blacklist-watches\"; - then - error_text "FAIL: test-blacklist-watches should not be accessible with the cache." - operator_logs - exit 1 - fi - - - header_text "verify that metrics reflect cr creation" - if ! timeout 60s bash -c -- "until kubectl run --attach --rm --restart=Never --namespace=default test-metrics --image=${METRICS_TEST_IMAGE} -- -sfkH \"Authorization: Bearer ${token}\" https://memcached-operator-controller-manager-metrics-service:8443/metrics | grep memcached-sample; do sleep 1; done"; - then - error_text "Failed to verify that metrics reflect cr creation" - operator_logs - exit 1 - fi - - header_text "get memcached deploy by labels" - memcached_deployment=$(kubectl get deployment -l app=memcached -o jsonpath="{..metadata.name}") - if ! timeout 1m kubectl rollout status deployment/${memcached_deployment}; - then - error_text "FAIL: Failed memcached Deployment failed rollout" - kubectl logs deployment/${memcached_deployment} - exit 1 - fi - - header_text "create a configmap that the finalizer should remove" - kubectl create configmap deleteme - trap_add 'kubectl delete --ignore-not-found configmap deleteme' EXIT - - header_text "delete custom resource (Memcached CR)" - kubectl delete -f ${OPERATORDIR}/config/samples/ansible_v1alpha1_memcached.yaml --wait=true - header_text "if the finalizer did not delete the configmap..." - if kubectl get configmap deleteme 2> /dev/null; - then - error_text "FAIL: the finalizer did not delete the configmap" - operator_logs - exit 1 - fi - - header_text "The deployment should get garbage collected, so we expect to fail getting the deployment." - if ! timeout 60s bash -c -- "while kubectl get deployment ${memcached_deployment} 2> /dev/null; do sleep 1; done"; - then - error_text "FAIL: memcached Deployment did not get garbage collected" - operator_logs - exit 1 - fi - - header_text "Ensure that no errors appear in the log" - if kubectl logs deployment/memcached-operator-controller-manager -c manager| grep -i error; - then - error_text "FAIL: the operator log includes errors" - operator_logs - exit 1 - fi -} - -header_text "Creating and building the operator" -pushd "$TMPDIR" -mkdir memcached-operator -pushd memcached-operator -operator-sdk init --plugins ansible.sdk.operatorframework.io/v1 \ - --domain example.com \ - --group ansible \ - --version v1alpha1 \ - --kind Memcached \ - --generate-playbook \ - --generate-role -cp "$ROOTDIR/test/ansible-memcached/tasks.yml" roles/memcached/tasks/main.yml -cp "$ROOTDIR/test/ansible-memcached/defaults.yml" roles/memcached/defaults/main.yml -cp -a "$ROOTDIR/test/ansible-memcached/memfin" roles/ -marker=$(tail -n1 watches.yaml) -sed -i'.bak' -e '$ d' watches.yaml;rm -f watches.yaml.bak -cat "$ROOTDIR/test/ansible-memcached/watches-finalizer.yaml" >> watches.yaml -echo $marker >> watches.yaml -header_text "Adding a second Kind to test watching multiple GVKs" -operator-sdk create api --kind=Foo --group ansible --version=v1alpha1 -sed -i".bak" -e 's/# FIXME.*/role: \/dev\/null/g' watches.yaml;rm -f watches.yaml.bak - -sed -i".bak" -E -e 's/(FROM quay.io\/operator-framework\/ansible-operator)(:.*)?/\1:dev/g' Dockerfile; rm -f Dockerfile.bak -IMG=$DEST_IMAGE make docker-build -# If using a kind cluster, load the image into all nodes. -load_image_if_kind "$DEST_IMAGE" -make kustomize -if [ -f ./bin/kustomize ] ; then - KUSTOMIZE="$(realpath ./bin/kustomize)" -else - KUSTOMIZE="$(which kustomize)" -fi -pushd config/default -${KUSTOMIZE} edit set namespace default -popd - -# kind has an issue with certain image registries (ex. redhat's), so use a -# different test pod image. -METRICS_TEST_IMAGE="curlimages/curl:latest" -docker pull "$METRICS_TEST_IMAGE" -# If using a kind cluster, load the metrics test image into all nodes. -load_image_if_kind "$METRICS_TEST_IMAGE" - -OPERATORDIR="$(pwd)" - -trap_add 'remove_operator' EXIT -deploy_operator -test_operator +# set default envvars +setup_envs $tmp_sdk_root -popd -popd +go test $tests -v -ginkgo.v diff --git a/test/e2e-ansible/e2e_ansible_cluster_test.go b/test/e2e-ansible/e2e_ansible_cluster_test.go new file mode 100644 index 0000000000..53caed15a5 --- /dev/null +++ b/test/e2e-ansible/e2e_ansible_cluster_test.go @@ -0,0 +1,385 @@ +// Copyright 2020 The Operator-SDK Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writifng, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e_ansible_test + +import ( + "encoding/base64" + "fmt" + "path/filepath" + "strings" + "time" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + kbtestutils "sigs.k8s.io/kubebuilder/test/e2e/utils" + + testutils "github.com/operator-framework/operator-sdk/test/internal" +) + +var _ = Describe("Running ansible projects", func() { + var controllerPodName string + var memcachedSampleFile string + var fooSampleFile string + var memfinSampleFile string + var memcachedDeployment string + + Context("built with operator-sdk", func() { + BeforeEach(func() { + By("checking samples") + memcachedSampleFile = filepath.Join(tc.Dir, "config", "samples", + fmt.Sprintf("%s_%s_%s.yaml", tc.Group, tc.Version, strings.ToLower(tc.Kind))) + fooSampleFile = filepath.Join(tc.Dir, "config", "samples", + fmt.Sprintf("%s_%s_foo.yaml", tc.Group, tc.Version)) + memfinSampleFile = filepath.Join(tc.Dir, "config", "samples", + fmt.Sprintf("%s_%s_memfin.yaml", tc.Group, tc.Version)) + + By("enabling Prometheus via the kustomization.yaml") + Expect(kbtestutils.UncommentCode( + filepath.Join(tc.Dir, "config", "default", "kustomization.yaml"), + "#- ../prometheus", "#")).To(Succeed()) + + By("deploying project on the cluster") + err := tc.Make("deploy", "IMG="+tc.ImageName) + Expect(err).Should(Succeed()) + }) + AfterEach(func() { + By("deleting Curl Pod created") + _, _ = tc.Kubectl.Delete(false, "pod", "curl") + + By("deleting CR instances created") + _, _ = tc.Kubectl.Delete(false, "-f", memcachedSampleFile) + _, _ = tc.Kubectl.Delete(false, "-f", fooSampleFile) + _, _ = tc.Kubectl.Delete(false, "-f", memfinSampleFile) + + By("cleaning up permissions") + _, _ = tc.Kubectl.Command("delete", "clusterrolebinding", + fmt.Sprintf("metrics-%s", tc.TestSuffix)) + + By("undeploy project") + _ = tc.Make("undeploy") + + By("ensuring that the namespace was deleted") + verifyNamespaceDeleted := func() error { + _, err := tc.Kubectl.Command("get", "namespace", tc.Kubectl.Namespace) + if strings.Contains(err.Error(), "(NotFound): namespaces") { + return err + } + return nil + } + Eventually(verifyNamespaceDeleted, 2*time.Minute, time.Second).ShouldNot(Succeed()) + }) + + It("should run correctly in a cluster", func() { + By("checking if the Operator project Pod is running") + verifyControllerUp := func() error { + By("getting the controller-manager pod name") + podOutput, err := tc.Kubectl.Get( + true, + "pods", "-l", "control-plane=controller-manager", + "-o", "go-template={{ range .items }}{{ if not .metadata.deletionTimestamp }}{{ .metadata.name }}"+ + "{{ \"\\n\" }}{{ end }}{{ end }}") + Expect(err).NotTo(HaveOccurred()) + + By("ensuring the created controller-manager Pod") + podNames := kbtestutils.GetNonEmptyLines(podOutput) + if len(podNames) != 1 { + return fmt.Errorf("expect 1 controller pods running, but got %d", len(podNames)) + } + controllerPodName = podNames[0] + Expect(controllerPodName).Should(ContainSubstring("controller-manager")) + + By("checking the controller-manager Pod is running") + status, err := tc.Kubectl.Get( + true, + "pods", controllerPodName, "-o", "jsonpath={.status.phase}") + Expect(err).NotTo(HaveOccurred()) + if status != "Running" { + return fmt.Errorf("controller pod in %s status", status) + } + return nil + } + Eventually(verifyControllerUp, 2*time.Minute, time.Second).Should(Succeed()) + + By("ensuring the created ServiceMonitor for the manager") + _, err := tc.Kubectl.Get( + true, + "ServiceMonitor", + fmt.Sprintf("e2e-%s-controller-manager-metrics-monitor", tc.TestSuffix)) + Expect(err).NotTo(HaveOccurred()) + + By("ensuring the created metrics Service for the manager") + _, err = tc.Kubectl.Get( + true, + "Service", + fmt.Sprintf("e2e-%s-controller-manager-metrics-service", tc.TestSuffix)) + Expect(err).NotTo(HaveOccurred()) + + By("create custom resource (Memcached CR)") + _, err = tc.Kubectl.Apply(false, "-f", memcachedSampleFile) + Expect(err).NotTo(HaveOccurred()) + + By("create custom resource (Foo CR)") + _, err = tc.Kubectl.Apply(false, "-f", fooSampleFile) + Expect(err).NotTo(HaveOccurred()) + + By("create custom resource (Memfin CR)") + _, err = tc.Kubectl.Apply(false, "-f", memfinSampleFile) + Expect(err).NotTo(HaveOccurred()) + + By("ensuring the CR gets reconciled") + managerContainerLogs := func() string { + logOutput, err := tc.Kubectl.Logs(controllerPodName, "-c", "manager") + Expect(err).NotTo(HaveOccurred()) + return logOutput + } + Eventually(managerContainerLogs, time.Minute, time.Second).Should(ContainSubstring( + "Ansible-runner exited successfully")) + Eventually(managerContainerLogs, time.Minute, time.Second).ShouldNot(ContainSubstring("failed=1")) + + By("ensuring no liveness probe fail events") + verifyControllerProbe := func() string { + By("getting the controller-manager events") + eventsOutput, err := tc.Kubectl.Get( + true, + "events", "--field-selector", fmt.Sprintf("involvedObject.name=%s", controllerPodName)) + Expect(err).NotTo(HaveOccurred()) + return eventsOutput + } + Eventually(verifyControllerProbe, time.Minute, time.Second).ShouldNot(ContainSubstring("Killing")) + + By("getting memcached deploy by labels") + getMencachedDeploument := func() string { + memcachedDeployment, err = tc.Kubectl.Get( + false, "deployment", + "-l", "app=memcached", "-o", "jsonpath={..metadata.name}") + Expect(err).NotTo(HaveOccurred()) + return memcachedDeployment + } + Eventually(getMencachedDeploument, 2*time.Minute, time.Second).ShouldNot(BeEmpty()) + + By("checking the Memcached CR deployment status") + verifyCRUp := func() string { + output, err := tc.Kubectl.Command( + "rollout", "status", "deployment", memcachedDeployment) + Expect(err).NotTo(HaveOccurred()) + return output + } + Eventually(verifyCRUp, time.Minute, time.Second).Should(ContainSubstring("successfully rolled out")) + + By("ensuring the created Service for the Memcached CR") + crServiceName, err := tc.Kubectl.Get( + false, + "Service", "-l", "app=memcached") + Expect(err).NotTo(HaveOccurred()) + Expect(len(crServiceName)).NotTo(BeIdenticalTo(0)) + + By("Verifying that a config map owned by the CR has been created") + verifyConfigMap := func() error { + _, err = tc.Kubectl.Get( + false, + "configmap", "test-blacklist-watches") + return err + } + Eventually(verifyConfigMap, time.Minute*2, time.Second).Should(Succeed()) + + By("Ensuring that config map requests skip the cache.") + checkSkipCache := func() string { + logOutput, err := tc.Kubectl.Logs(controllerPodName, "-c", "manager") + Expect(err).NotTo(HaveOccurred()) + return logOutput + } + Eventually(checkSkipCache, time.Minute, time.Second).Should(ContainSubstring("\"Skipping cache " + + "lookup\",\"resource\":{\"IsResourceRequest\":true," + + "\"Path\":\"/api/v1/namespaces/default/configmaps/test-blacklist-watches\"")) + + By("scaling deployment replicas to 2") + _, err = tc.Kubectl.Command( + "scale", "deployment", memcachedDeployment, "--replicas", "2") + Expect(err).NotTo(HaveOccurred()) + + By("verifying the deployment automatically scales back down to 1") + verifyMemcachedScalesBack := func() error { + replicas, err := tc.Kubectl.Get( + false, + "deployment", memcachedDeployment, "-o", "jsonpath={..spec.replicas}") + Expect(err).NotTo(HaveOccurred()) + if replicas != "1" { + return fmt.Errorf("memcached(CR) deployment with %s replicas", replicas) + } + return nil + } + Eventually(verifyMemcachedScalesBack, time.Minute, time.Second).Should(Succeed()) + + By("updating size to 2 in the CR manifest") + testutils.ReplaceInFile(memcachedSampleFile, "size: 1", "size: 2") + + By("applying CR manifest with size: 2") + _, err = tc.Kubectl.Apply(false, "-f", memcachedSampleFile) + Expect(err).NotTo(HaveOccurred()) + + By("ensuring the CR gets reconciled after patching it") + managerContainerLogsAfterUpdateCR := func() string { + logOutput, err := tc.Kubectl.Logs(controllerPodName, "-c", "manager") + Expect(err).NotTo(HaveOccurred()) + return logOutput + } + Eventually(managerContainerLogsAfterUpdateCR, time.Minute, time.Second).Should( + ContainSubstring("Ansible-runner exited successfully")) + Eventually(managerContainerLogs, time.Minute, time.Second).ShouldNot(ContainSubstring("failed=1")) + + By("checking Deployment replicas spec is equals 2") + verifyMemcachedPatch := func() error { + replicas, err := tc.Kubectl.Get( + false, + "deployment", memcachedDeployment, "-o", "jsonpath={..spec.replicas}") + Expect(err).NotTo(HaveOccurred()) + if replicas != "2" { + return fmt.Errorf("memcached(CR) deployment with %s replicas", replicas) + } + return nil + } + Eventually(verifyMemcachedPatch, time.Minute, time.Second).Should(Succeed()) + + By("granting permissions to access the metrics and read the token") + _, err = tc.Kubectl.Command( + "create", + "clusterrolebinding", + fmt.Sprintf("metrics-%s", tc.TestSuffix), + fmt.Sprintf("--clusterrole=e2e-%s-metrics-reader", tc.TestSuffix), + fmt.Sprintf("--serviceaccount=%s:default", tc.Kubectl.Namespace)) + Expect(err).NotTo(HaveOccurred()) + + By("getting the token") + b64Token, err := tc.Kubectl.Get( + true, + "secrets", + "-o=jsonpath={.items[0].data.token}") + Expect(err).NotTo(HaveOccurred()) + token, err := base64.StdEncoding.DecodeString(strings.TrimSpace(b64Token)) + Expect(err).NotTo(HaveOccurred()) + Expect(token).NotTo(HaveLen(0)) + + By("creating a pod with curl image") + // todo: the flag --generator=run-pod/v1 is deprecated, however, shows that besides + // it should not make any difference and work locally successfully when the flag is removed + // travis has been failing and the curl pod is not found when the flag is not used + cmdOpts := []string{ + "run", "--generator=run-pod/v1", "curl", "--image=curlimages/curl:7.68.0", "--restart=OnFailure", "--", + "curl", "-v", "-k", "-H", fmt.Sprintf(`Authorization: Bearer %s`, token), + fmt.Sprintf("https://e2e-%v-controller-manager-metrics-service.e2e-%v-system.svc:8443/metrics", + tc.TestSuffix, tc.TestSuffix), + } + _, err = tc.Kubectl.CommandInNamespace(cmdOpts...) + Expect(err).NotTo(HaveOccurred()) + + By("validating the curl pod running as expected") + verifyCurlUp := func() error { + // Validate pod status + status, err := tc.Kubectl.Get( + true, + "pods", "curl", "-o", "jsonpath={.status.phase}") + Expect(err).NotTo(HaveOccurred()) + if status != "Completed" && status != "Succeeded" { + return fmt.Errorf("curl pod in %s status", status) + } + return nil + } + Eventually(verifyCurlUp, 4*time.Minute, time.Second).Should(Succeed()) + + By("checking metrics endpoint serving as expected") + getCurlLogs := func() string { + logOutput, err := tc.Kubectl.Logs("curl") + Expect(err).NotTo(HaveOccurred()) + return logOutput + } + Eventually(getCurlLogs, time.Minute, time.Second).Should(ContainSubstring("< HTTP/2 200")) + + By("getting the CR namespace token") + crNamespace, err := tc.Kubectl.Get( + false, + tc.Kind, + fmt.Sprintf("%s-sample", strings.ToLower(tc.Kind)), + "-o=jsonpath={..metadata.namespace}") + Expect(err).NotTo(HaveOccurred()) + Expect(crNamespace).NotTo(HaveLen(0)) + + By("ensuring the operator metrics contains a `resource_created_at` metric for the Memcached CR") + metricExportedMemcachedCR := fmt.Sprintf("resource_created_at_seconds{group=\"%s\","+ + "kind=\"%s\","+ + "name=\"%s-sample\","+ + "namespace=\"%s\","+ + "version=\"%s\"}", + fmt.Sprintf("%s.%s", tc.Group, tc.Domain), + tc.Kind, + strings.ToLower(tc.Kind), + crNamespace, + tc.Version) + Eventually(getCurlLogs, time.Minute, time.Second).Should(ContainSubstring(metricExportedMemcachedCR)) + + By("ensuring the operator metrics contains a `resource_created_at` metric for the Foo CR") + metricExportedFooCR := fmt.Sprintf("resource_created_at_seconds{group=\"%s\","+ + "kind=\"%s\","+ + "name=\"%s-sample\","+ + "namespace=\"%s\","+ + "version=\"%s\"}", + fmt.Sprintf("%s.%s", tc.Group, tc.Domain), + "Foo", + strings.ToLower("Foo"), + crNamespace, + tc.Version) + Eventually(getCurlLogs, time.Minute, time.Second).Should(ContainSubstring(metricExportedFooCR)) + + By("ensuring the operator metrics contains a `resource_created_at` metric for the Memfin CR") + metricExportedMemfinCR := fmt.Sprintf("resource_created_at_seconds{group=\"%s\","+ + "kind=\"%s\","+ + "name=\"%s-sample\","+ + "namespace=\"%s\","+ + "version=\"%s\"}", + fmt.Sprintf("%s.%s", tc.Group, tc.Domain), + "Memfin", + strings.ToLower("Memfin"), + crNamespace, + tc.Version) + Eventually(getCurlLogs, time.Minute, time.Second).Should(ContainSubstring(metricExportedMemfinCR)) + + By("creating a configmap that the finalizer should remove") + _, err = tc.Kubectl.Command("create", "configmap", "deleteme") + Expect(err).NotTo(HaveOccurred()) + + By("deleting Memcached CR manifest") + _, err = tc.Kubectl.Delete(false, "-f", memcachedSampleFile) + Expect(err).NotTo(HaveOccurred()) + + By("ensuring the CR gets reconciled successfully") + managerContainerLogsAfterDeleteCR := func() string { + logOutput, err := tc.Kubectl.Logs(controllerPodName, "-c", "manager") + Expect(err).NotTo(HaveOccurred()) + return logOutput + } + Eventually(managerContainerLogsAfterDeleteCR, time.Minute, time.Second).Should(ContainSubstring( + "Ansible-runner exited successfully")) + Eventually(managerContainerLogsAfterDeleteCR).ShouldNot(ContainSubstring("error")) + + By("ensuring that Memchaced Deployment was removed") + getMemcachedDeployment := func() error { + _, err := tc.Kubectl.Get( + false, "deployment", + memcachedDeployment) + return err + } + Eventually(getMemcachedDeployment, time.Minute*2, time.Second).ShouldNot(Succeed()) + }) + }) +}) diff --git a/test/e2e-ansible/e2e_ansible_local_test.go b/test/e2e-ansible/e2e_ansible_local_test.go new file mode 100644 index 0000000000..78d83a75a1 --- /dev/null +++ b/test/e2e-ansible/e2e_ansible_local_test.go @@ -0,0 +1,50 @@ +// Copyright 2020 The Operator-SDK Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e_ansible_test + +import ( + "os/exec" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +var _ = Describe("Running ansible projects", func() { + Context("built with operator-sdk", func() { + + BeforeEach(func() { + By("installing CRD's") + err := tc.Make("install") + Expect(err).NotTo(HaveOccurred()) + }) + + AfterEach(func() { + By("uninstalling CRD's") + err := tc.Make("uninstall") + Expect(err).NotTo(HaveOccurred()) + }) + + It("should run correctly locally", func() { + By("running the project") + cmd := exec.Command("make", "run") + err := cmd.Start() + Expect(err).NotTo(HaveOccurred()) + + By("killing the project") + err = cmd.Process.Kill() + Expect(err).NotTo(HaveOccurred()) + }) + }) +}) diff --git a/test/e2e-ansible/e2e_ansible_olm_test.go b/test/e2e-ansible/e2e_ansible_olm_test.go new file mode 100644 index 0000000000..843f4a7f28 --- /dev/null +++ b/test/e2e-ansible/e2e_ansible_olm_test.go @@ -0,0 +1,81 @@ +// Copyright 2020 The Operator-SDK Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e_ansible_test + +import ( + "os/exec" + "path" + "path/filepath" + "strings" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + testutils "github.com/operator-framework/operator-sdk/test/internal" +) + +var _ = Describe("Integrating ansible Projects with OLM", func() { + Context("with operator-sdk", func() { + const operatorVersion = "0.0.1" + + BeforeEach(func() { + By("turning off interactive prompts for all generation tasks.") + replace := "operator-sdk generate kustomize manifests" + testutils.ReplaceInFile(filepath.Join(tc.Dir, "Makefile"), replace, replace+" --interactive=false") + }) + + It("should generate and run a valid OLM bundle and packagemanifests", func() { + By("building the bundle") + err := tc.Make("bundle", "IMG="+tc.ImageName) + Expect(err).NotTo(HaveOccurred()) + + By("building the operator bundle image") + // Use the existing image tag but with a "-bundle" suffix. + imageSplit := strings.SplitN(tc.ImageName, ":", 2) + bundleImage := path.Join("quay.io", imageSplit[0]+"-bundle") + if len(imageSplit) == 2 { + bundleImage += ":" + imageSplit[1] + } + err = tc.Make("bundle-build", "BUNDLE_IMG="+bundleImage) + Expect(err).NotTo(HaveOccurred()) + + By("loading the project image into Kind cluster") + err = tc.LoadImageToKindClusterWithName(bundleImage) + Expect(err).Should(Succeed()) + + By("adding the 'packagemanifests' rule to the Makefile") + err = tc.AddPackagemanifestsTarget() + Expect(err).Should(Succeed()) + + By("generating the operator package manifests") + err = tc.Make("packagemanifests", "IMG="+tc.ImageName) + Expect(err).NotTo(HaveOccurred()) + + By("running the package") + runPkgManCmd := exec.Command(tc.BinaryName, "run", "packagemanifests", + "--install-mode", "AllNamespaces", + "--version", operatorVersion, + "--timeout", "4m") + _, err = tc.Run(runPkgManCmd) + Expect(err).NotTo(HaveOccurred()) + + By("destroying the deployed package manifests-formatted operator") + cleanupPkgManCmd := exec.Command(tc.BinaryName, "cleanup", projectName, + "--timeout", "4m") + _, err = tc.Run(cleanupPkgManCmd) + Expect(err).NotTo(HaveOccurred()) + }) + }) +}) diff --git a/test/e2e-ansible/e2e_ansible_suite_test.go b/test/e2e-ansible/e2e_ansible_suite_test.go new file mode 100644 index 0000000000..d61aedf3a0 --- /dev/null +++ b/test/e2e-ansible/e2e_ansible_suite_test.go @@ -0,0 +1,312 @@ +// Copyright 2020 The Operator-SDK Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e_ansible_test + +import ( + "fmt" + "path/filepath" + "strings" + "testing" + "time" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/operator-framework/operator-sdk/internal/version" + testutils "github.com/operator-framework/operator-sdk/test/internal" +) + +// TestE2EAnsible ensures the ansible projects built with the SDK tool by using its binary. +func TestE2EAnsible(t *testing.T) { + if testing.Short() { + t.Skip("skipping Operator SDK E2E Ansible Suite testing in short mode") + } + RegisterFailHandler(Fail) + RunSpecs(t, "E2EAnsible Suite") +} + +var ( + tc testutils.TestContext + // isPromethuesManagedBySuite is true when the suite tests is installing/uninstalling the Prometheus + isPromethuesManagedBySuite = true + // isOLMManagedBySuite is true when the suite tests is installing/uninstalling the OLM + isOLMManagedBySuite = true + // kubectx stores the k8s context from where the tests are running + kubectx string + // projectName is the name of the test project + projectName string +) + +// BeforeSuite run before any specs are run to perform the required actions for all e2e ansible tests. +var _ = BeforeSuite(func(done Done) { + var err error + + By("creating a new test context") + tc, err = testutils.NewTestContext("GO111MODULE=on") + Expect(err).NotTo(HaveOccurred()) + Expect(tc.Prepare()).To(Succeed()) + projectName = filepath.Base(tc.Dir) + + By("checking the cluster type") + kubectx, err = tc.Kubectl.Command("config", "current-context") + Expect(err).Should(Succeed()) + + By("checking API resources applied on Cluster") + output, err := tc.Kubectl.Command("api-resources") + Expect(err).NotTo(HaveOccurred()) + if strings.Contains(output, "servicemonitors") { + isPromethuesManagedBySuite = false + } + if strings.Contains(output, "clusterserviceversions") { + isOLMManagedBySuite = false + } + + if isPromethuesManagedBySuite { + By("installing Prometheus") + Expect(tc.InstallPrometheusOperManager()).To(Succeed()) + + By("ensuring provisioned Prometheus Manager Service") + Eventually(func() error { + _, err := tc.Kubectl.Get( + false, + "Service", "prometheus-operator") + return err + }, 3*time.Minute, time.Second).Should(Succeed()) + } + + if isOLMManagedBySuite { + By("installing OLM") + Expect(tc.InstallOLM()).To(Succeed()) + } + + By("setting domain and GKV") + tc.Domain = "example.com" + tc.Version = "v1alpha1" + tc.Group = "ansible" + tc.Kind = "Memcached" + + By("initializing a ansible project") + err = tc.Init( + "--plugins", "ansible", + "--project-version", "3-alpha", + "--domain", tc.Domain) + Expect(err).Should(Succeed()) + + By("creating the Memcached API") + err = tc.CreateAPI( + "--group", tc.Group, + "--version", tc.Version, + "--kind", tc.Kind, + "--generate-playbook", + "--generate-role") + Expect(err).Should(Succeed()) + + By("replacing project Dockerfile to use ansible base image with the dev tag") + version := strings.TrimSuffix(version.Version, "+git") + testutils.ReplaceInFile(filepath.Join(tc.Dir, "Dockerfile"), version, "dev") + + By("adding Memcached mock task to the role") + testutils.ReplaceInFile(filepath.Join(tc.Dir, "roles", strings.ToLower(tc.Kind), "tasks", "main.yml"), + fmt.Sprintf("# tasks file for %s", tc.Kind), memcachedWithBlackListTask) + + By("setting defaults to Memcached") + testutils.ReplaceInFile(filepath.Join(tc.Dir, "roles", strings.ToLower(tc.Kind), "defaults", "main.yml"), + fmt.Sprintf("# defaults file for %s", tc.Kind), "size: 1") + + By("updating Memcached sample") + memcachedSampleFile := filepath.Join(tc.Dir, "config", "samples", + fmt.Sprintf("%s_%s_%s.yaml", tc.Group, tc.Version, strings.ToLower(tc.Kind))) + testutils.ReplaceInFile(memcachedSampleFile, "foo: bar", "size: 1") + + By("creating an API definition to add a task to delete the config map") + err = tc.CreateAPI( + "--group", tc.Group, + "--version", tc.Version, + "--kind", "Memfin", + "--generate-role") + Expect(err).Should(Succeed()) + + By("adding task to delete config map") + testutils.ReplaceInFile(filepath.Join(tc.Dir, "roles", "memfin", "tasks", "main.yml"), + "# tasks file for Memfin", taskToDeleteConfigMap) + + By("adding to watches finalizer and blacklist") + testutils.ReplaceInFile(filepath.Join(tc.Dir, "watches.yaml"), + "playbook: playbooks/memcached.yml", memcachedWatchCustomizations) + + By("create API to test watching multiple GVKs") + err = tc.CreateAPI( + "--group", tc.Group, + "--version", tc.Version, + "--kind", "Foo", + "--generate-role") + Expect(err).Should(Succeed()) + + By("adding RBAC permissions for the Memcached Kind") + testutils.ReplaceInFile(filepath.Join(tc.Dir, "config", "rbac", "role.yaml"), + "# +kubebuilder:scaffold:rules", rolesForBaseOperator) + + By("checking the kustomize setup") + err = tc.Make("kustomize") + Expect(err).Should(Succeed()) + + By("building the project image") + err = tc.Make("docker-build", "IMG="+tc.ImageName) + Expect(err).Should(Succeed()) + + if isRunningOnKind() { + By("loading the project image into Kind cluster") + err = tc.LoadImageToKindCluster() + Expect(err).Should(Succeed()) + } + + close(done) +}, 360) + +// AfterSuite run after all the specs have run, regardless of whether any tests have failed to ensures that +// all be cleaned up +var _ = AfterSuite(func() { + if isPromethuesManagedBySuite { + By("uninstalling Prometheus") + tc.UninstallPrometheusOperManager() + } + if isOLMManagedBySuite { + By("uninstalling OLM") + tc.UninstallOLM() + } + + By("destroying container image and work dir") + tc.Destroy() +}) + +// isRunningOnKind returns true when the tests are executed in a Kind Cluster +func isRunningOnKind() bool { + return strings.Contains(kubectx, "kind") +} + +const memcachedWithBlackListTask = `- name: start memcached + community.kubernetes.k8s: + definition: + kind: Deployment + apiVersion: apps/v1 + metadata: + name: '{{ ansible_operator_meta.name }}-memcached' + namespace: '{{ ansible_operator_meta.namespace }}' + labels: + app: memcached + spec: + replicas: "{{size}}" + selector: + matchLabels: + app: memcached + template: + metadata: + labels: + app: memcached + spec: + containers: + - name: memcached + command: + - memcached + - -m=64 + - -o + - modern + - -v + image: "docker.io/memcached:1.4.36-alpine" + ports: + - containerPort: 11211 + readinessProbe: + tcpSocket: + port: 11211 + initialDelaySeconds: 3 + periodSeconds: 3 + +- operator_sdk.util.k8s_status: + api_version: ansible.example.com/v1alpha1 + kind: Memcached + name: "{{ ansible_operator_meta.name }}" + namespace: "{{ ansible_operator_meta.namespace }}" + status: + test: "hello world" + +- community.kubernetes.k8s: + definition: + kind: Secret + apiVersion: v1 + metadata: + name: test-secret + namespace: "{{ ansible_operator_meta.namespace }}" + data: + test: aGVsbG8K +- name: Get cluster api_groups + set_fact: + api_groups: "{{ lookup('community.kubernetes.k8s', cluster_info='api_groups', kubeconfig=lookup('env', 'K8S_AUTH_KUBECONFIG')) }}" + +- name: create project if projects are available + community.kubernetes.k8s: + definition: + apiVersion: project.openshift.io/v1 + kind: Project + metadata: + name: testing-foo + when: "'project.openshift.io' in api_groups" + +- name: Create ConfigMap to test blacklisted watches + community.kubernetes.k8s: + definition: + kind: ConfigMap + apiVersion: v1 + metadata: + name: test-blacklist-watches + namespace: "{{ ansible_operator_meta.namespace }}" + data: + arbitrary: afdasdfsajsafj + state: present` + +const taskToDeleteConfigMap = `- name: delete configmap for test + community.kubernetes.k8s: + kind: ConfigMap + api_version: v1 + name: deleteme + namespace: default + state: absent` + +const memcachedWatchCustomizations = `playbook: playbooks/memcached.yml + finalizer: + name: finalizer.ansible.example.com + role: memfin + blacklist: + - group: "" + version: v1 + kind: ConfigMap` + +const rolesForBaseOperator = ` + ## + ## Apply customize roles for base operator + ## + - apiGroups: + - "" + resources: + - configmaps + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +# +kubebuilder:scaffold:rules +`