From 052b7322bc0b2eebc00b3d1e4e1dea7a80c279d6 Mon Sep 17 00:00:00 2001 From: DQ Date: Tue, 15 Apr 2025 13:26:26 +1000 Subject: [PATCH 1/9] feat: add integration tests and update workflow for testing --- .github/workflows/test-e2e.yml | 5 ++ Makefile | 6 +- go.mod | 5 ++ go.sum | 12 ++++ test/integration/integration_test.go | 66 ++++++++++++++++++++++ test/integration/main_test.go | 82 ++++++++++++++++++++++++++++ test/utils/resources/cache.go | 46 ++++++++++++++++ test/utils/utils.go | 1 + 8 files changed, 222 insertions(+), 1 deletion(-) create mode 100644 test/integration/integration_test.go create mode 100644 test/integration/main_test.go create mode 100644 test/utils/resources/cache.go diff --git a/.github/workflows/test-e2e.yml b/.github/workflows/test-e2e.yml index b2eda8c..89b197a 100644 --- a/.github/workflows/test-e2e.yml +++ b/.github/workflows/test-e2e.yml @@ -17,6 +17,11 @@ jobs: with: go-version-file: go.mod + - name: Running integration tests + run: | + go mod tidy + make test-integration + - name: Install the latest version of kind run: | curl -Lo ./kind https://kind.sigs.k8s.io/dl/latest/kind-linux-amd64 diff --git a/Makefile b/Makefile index b11d448..81e7f6c 100644 --- a/Makefile +++ b/Makefile @@ -59,7 +59,7 @@ vet: ## Run go vet against code. .PHONY: test test: manifests generate fmt vet setup-envtest ## Run tests. - KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test $$(go list ./... | grep -v /e2e) -coverprofile ./cover.out -covermode=atomic -coverpkg=./... + KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test $$(go list ./... | grep -v /e2e | grep -v /integration) -coverprofile ./cover.out -covermode=atomic -coverpkg=./... # TODO(user): To use a different vendor for e2e tests, modify the setup under 'tests/e2e'. # The default setup assumes Kind is pre-installed and builds/loads the Manager Docker image locally. @@ -77,6 +77,10 @@ test-e2e: manifests generate fmt vet ## Run the e2e tests. Expected an isolated } go test ./test/e2e/ -v -ginkgo.v +.Phony: test-integration +test-integration: manifests generate fmt vet ## Run the integration tests. Expected an isolated environment using Kind. + go test ./test/integration + .PHONY: lint lint: golangci-lint ## Run golangci-lint linter $(GOLANGCI_LINT) run diff --git a/go.mod b/go.mod index 3e045bb..433d7cd 100644 --- a/go.mod +++ b/go.mod @@ -16,6 +16,7 @@ require ( k8s.io/apimachinery v0.32.1 k8s.io/client-go v0.32.1 sigs.k8s.io/controller-runtime v0.20.4 + sigs.k8s.io/e2e-framework v0.6.0 ) require ( @@ -46,14 +47,17 @@ require ( github.com/google/go-cmp v0.6.0 // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db // indirect + github.com/gorilla/websocket v1.5.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/mailru/easyjson v0.7.7 // indirect + github.com/moby/spdystream v0.5.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/client_golang v1.19.1 // indirect @@ -63,6 +67,7 @@ require ( github.com/spf13/cobra v1.8.1 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/stoewer/go-strcase v1.3.0 // indirect + github.com/vladimirvivien/gexe v0.4.1 // indirect github.com/x448/float16 v0.8.4 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0 // indirect go.opentelemetry.io/otel v1.28.0 // indirect diff --git a/go.sum b/go.sum index 14ebb5d..d2ea84a 100644 --- a/go.sum +++ b/go.sum @@ -2,6 +2,8 @@ cel.dev/expr v0.18.0 h1:CJ6drgk+Hf96lkLikr4rFf19WrU0BOWEihyZnI2TAzo= cel.dev/expr v0.18.0/go.mod h1:MrpN08Q+lEBs+bGYdLxxHkZoUSsCp0nSKTs0nTymJgw= github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8TVTI= github.com/antlr4-go/antlr/v4 v4.13.0/go.mod h1:pfChB/xh/Unjila75QW7+VU4TSnWnnk9UTnmpPaOR2g= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a h1:idn718Q4B6AGu/h5Sxe66HYVdqdGu2l9Iebqhi/AEoA= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -67,6 +69,8 @@ github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db h1:097atOisP2aRj7vFgY github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= +github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 h1:bkypFPDjIYGfCYD5mRBvpqxfYX1YCS1PXdKYWi8FsN0= github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0/go.mod h1:P+Lt/0by1T8bfcF3z737NnSbmxQAppXMRziHUxPOC8k= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= @@ -86,6 +90,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/moby/spdystream v0.5.0 h1:7r0J1Si3QO/kjRitvSLVVFUjxMEb/YLj6S9FF62JBCU= +github.com/moby/spdystream v0.5.0/go.mod h1:xBAYlnt/ay+11ShkdFKNAG7LsyK/tmNBVvVOwrfMgdI= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -93,6 +99,8 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= github.com/onsi/ginkgo/v2 v2.22.0 h1:Yed107/8DjTr0lKCNt7Dn8yQ6ybuDRQoMGrNFKzMfHg= github.com/onsi/ginkgo/v2 v2.22.0/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo= github.com/onsi/gomega v1.36.1 h1:bJDPBO7ibjxcbHMgSCoo4Yj18UWbKDlLwX1x9sybDcw= @@ -130,6 +138,8 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/vladimirvivien/gexe v0.4.1 h1:W9gWkp8vSPjDoXDu04Yp4KljpVMaSt8IQuHswLDd5LY= +github.com/vladimirvivien/gexe v0.4.1/go.mod h1:3gjgTqE2c0VyHnU5UOIwk7gyNzZDGulPb/DJPgcw64E= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -243,6 +253,8 @@ sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.0 h1:CPT0ExVicCzcp sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.0/go.mod h1:Ve9uj1L+deCXFrPOk1LpFXqTg7LCFzFso6PA48q/XZw= sigs.k8s.io/controller-runtime v0.20.4 h1:X3c+Odnxz+iPTRobG4tp092+CvBU9UK0t/bRf+n0DGU= sigs.k8s.io/controller-runtime v0.20.4/go.mod h1:xg2XB0K5ShQzAgsoujxuKN4LNXR2LfwwHsPj7Iaw+XY= +sigs.k8s.io/e2e-framework v0.6.0 h1:p7hFzHnLKO7eNsWGI2AbC1Mo2IYxidg49BiT4njxkrM= +sigs.k8s.io/e2e-framework v0.6.0/go.mod h1:IREnCHnKgRCioLRmNi0hxSJ1kJ+aAdjEKK/gokcZu4k= sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 h1:/Rv+M11QRah1itp8VhT6HoVx1Ray9eB4DBr+K+/sCJ8= sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3/go.mod h1:18nIHnGi6636UCz6m8i4DhaJ65T6EruyzmoQqI2BVDo= sigs.k8s.io/structured-merge-diff/v4 v4.4.2 h1:MdmvkGuXi/8io6ixD5wud3vOLwc1rj0aNqRlpuvjmwA= diff --git a/test/integration/integration_test.go b/test/integration/integration_test.go new file mode 100644 index 0000000..668a58e --- /dev/null +++ b/test/integration/integration_test.go @@ -0,0 +1,66 @@ +package integration + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "k8s.io/apimachinery/pkg/util/wait" + k8sresources "sigs.k8s.io/e2e-framework/klient/k8s/resources" + "sigs.k8s.io/e2e-framework/pkg/envconf" + "sigs.k8s.io/e2e-framework/pkg/features" + + v1 "github.com/Azure/operation-cache-controller/api/v1" + "github.com/Azure/operation-cache-controller/test/utils/resources" +) + +type cacheKey struct{} + +var CacheFeature = features.New("appsv1/deployment/cache"). + Setup(func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { + // start a deployment + cache := resources.SampleCache + cache.Namespace = testNamespace + if err := c.Client().Resources().Create(ctx, &cache); err != nil { + t.Fatal(err) + } + time.Sleep(2 * time.Second) + + return ctx + }). + Assess("create cache", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { + var cache v1.Cache + if err := cfg.Client().Resources().Get(ctx, resources.SampleCache.Name, testNamespace, &cache); err != nil { + t.Fatal(err) + } + assert.Equal(t, "cache-sample", cache.Name) + if err := wait.PollUntilContextTimeout(ctx, 5*time.Second, 60*time.Second, true, func(ctx context.Context) (bool, error) { + var ops v1.OperationList + if err := cfg.Client().Resources().List(ctx, &ops, k8sresources.WithFieldSelector(fmt.Sprintf("metadata.namespace=%s", testNamespace))); err != nil { + return false, err + } + var countOwnedOps int + for _, op := range ops.Items { + op := op + if op.ObjectMeta.GetOwnerReferences()[0].Name == cache.Name { + countOwnedOps++ + } + } + if countOwnedOps != int(cache.Status.KeepAliveCount) { + return false, nil + } + return true, nil + }); err != nil { + t.Fatal(err, "operations not meet the expected count") + } + return context.WithValue(ctx, cacheKey{}, &cache) + }). + Teardown(func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { + cache := ctx.Value(cacheKey{}).(*v1.Cache) + if err := cfg.Client().Resources().Delete(ctx, cache); err != nil { + t.Fatal(err) + } + return ctx + }).Feature() diff --git a/test/integration/main_test.go b/test/integration/main_test.go new file mode 100644 index 0000000..5710f91 --- /dev/null +++ b/test/integration/main_test.go @@ -0,0 +1,82 @@ +package integration + +import ( + "context" + "os" + "os/exec" + "testing" + + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/client-go/kubernetes/scheme" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + "sigs.k8s.io/e2e-framework/pkg/env" + "sigs.k8s.io/e2e-framework/pkg/envconf" + "sigs.k8s.io/e2e-framework/pkg/envfuncs" + "sigs.k8s.io/e2e-framework/support/kind" + + v1 "github.com/Azure/operation-cache-controller/api/v1" + "github.com/Azure/operation-cache-controller/test/utils" +) + +var testenv env.Environment + +// projectImage is the name of the image which will be build and loaded +// with the code source changes to be tested. +var projectImage = "example.com/operation-cache-controller:v0.0.1" +var kindClusterName = "integration-test-cluster" +var testNamespace = "deploy-controller-system" + +func init() { + log.SetLogger(zap.New(zap.WriteTo(os.Stdout), zap.UseDevMode(true))) + utilruntime.Must(v1.AddToScheme(scheme.Scheme)) +} + +func TestMain(m *testing.M) { + // Create a new test environment configuration + testenv = env.New() + + // Setup the test environment with Kind cluster and necessary resources + testenv = testenv.Setup( + envfuncs.CreateCluster(kind.NewProvider(), kindClusterName), + envfuncs.LoadDockerImageToCluster(kindClusterName, projectImage), + envfuncs.CreateNamespace(testNamespace), + InstallCRD, + DeployControllerManager, + ) + + // Teardown the test environment + testenv = testenv.Finish( + envfuncs.DeleteNamespace(testNamespace), + UninstallCRD, + envfuncs.DestroyCluster(kindClusterName), + ) + + // Run the tests + os.Exit(testenv.Run(m)) +} + +func InstallCRD(ctx context.Context, cfg *envconf.Config) (context.Context, error) { + // Install the CRD in the test environment + cmd := exec.Command("make", "install") + _, err := utils.Run(cmd) + return ctx, err +} + +func DeployControllerManager(ctx context.Context, cfg *envconf.Config) (context.Context, error) { + // Deploy the controller manager in the test environment + cmd := exec.Command("make", "deploy") + _, err := utils.Run(cmd) + return ctx, err +} + +func UninstallCRD(ctx context.Context, cfg *envconf.Config) (context.Context, error) { + // Uninstall the CRD in the test environment + cmd := exec.Command("make", "uninstall") + _, err := utils.Run(cmd) + return ctx, err +} +func TestRealCluster(t *testing.T) { + // Run the integration tests against the Kind cluster + testenv.Test(t, CacheFeature) +} diff --git a/test/utils/resources/cache.go b/test/utils/resources/cache.go new file mode 100644 index 0000000..f17d37a --- /dev/null +++ b/test/utils/resources/cache.go @@ -0,0 +1,46 @@ +package resources + +import ( + appsv1 "github.com/Azure/operation-cache-controller/api/v1" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +var SampleCache appsv1.Cache = appsv1.Cache{ + ObjectMeta: metav1.ObjectMeta{Name: "cache-sample"}, + Spec: appsv1.CacheSpec{ + Strategy: "", + OperationTemplate: appsv1.OperationSpec{ + Applications: []appsv1.ApplicationSpec{ + { + Name: "test-app", + Provision: batchv1.JobSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "test-app-provision", + Image: "aksartifactsmsftprodeastus.azurecr.io/devinfra/underlay:sha256:05c616236b999cb9e610cd7afede57adc90516f3db6f30408c72a907889ad8dc", + }, + }, + }, + }, + }, + Teardown: batchv1.JobSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "test-app-provision", + Image: "aksartifactsmsftprodeastus.azurecr.io/devinfra/underlay:sha256:05c616236b999cb9e610cd7afede57adc90516f3db6f30408c72a907889ad8dc", + }, + }, + }, + }, + }, + }, + }, + }, + }, +} diff --git a/test/utils/utils.go b/test/utils/utils.go index 04a5141..93d5edd 100644 --- a/test/utils/utils.go +++ b/test/utils/utils.go @@ -198,6 +198,7 @@ func GetProjectDir() (string, error) { return wd, err } wd = strings.Replace(wd, "/test/e2e", "", -1) + wd = strings.Replace(wd, "/test/integration", "", -1) return wd, nil } From 040ee4218bb5314d9dae071adda0883ea863bddc Mon Sep 17 00:00:00 2001 From: DQ Date: Tue, 15 Apr 2025 13:38:53 +1000 Subject: [PATCH 2/9] refactor: disable lll linter and clean up integration test variable shadowing --- .golangci.yml | 2 +- test/integration/integration_test.go | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index 6b29746..308dd7f 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -30,7 +30,7 @@ linters: - gosimple - govet - ineffassign - - lll + # - lll - misspell - nakedret - prealloc diff --git a/test/integration/integration_test.go b/test/integration/integration_test.go index 668a58e..a7b7aab 100644 --- a/test/integration/integration_test.go +++ b/test/integration/integration_test.go @@ -43,7 +43,6 @@ var CacheFeature = features.New("appsv1/deployment/cache"). } var countOwnedOps int for _, op := range ops.Items { - op := op if op.ObjectMeta.GetOwnerReferences()[0].Name == cache.Name { countOwnedOps++ } From 655433444c0311e001ea11ef94fb33452c165514 Mon Sep 17 00:00:00 2001 From: DQ Date: Tue, 15 Apr 2025 14:24:38 +1000 Subject: [PATCH 3/9] feat: update DeployControllerManager to include image parameter --- test/integration/main_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/integration/main_test.go b/test/integration/main_test.go index 5710f91..47095d6 100644 --- a/test/integration/main_test.go +++ b/test/integration/main_test.go @@ -2,6 +2,7 @@ package integration import ( "context" + "fmt" "os" "os/exec" "testing" @@ -65,7 +66,7 @@ func InstallCRD(ctx context.Context, cfg *envconf.Config) (context.Context, erro func DeployControllerManager(ctx context.Context, cfg *envconf.Config) (context.Context, error) { // Deploy the controller manager in the test environment - cmd := exec.Command("make", "deploy") + cmd := exec.Command("make", "deploy", fmt.Sprintf("IMG=%s", projectImage)) _, err := utils.Run(cmd) return ctx, err } From 9d64641fcdce426478b337ed7fc1bb7aafc36ce0 Mon Sep 17 00:00:00 2001 From: DQ Date: Tue, 15 Apr 2025 14:33:41 +1000 Subject: [PATCH 4/9] feat: implement BuildImage function to build Docker image for controller manager --- test/integration/main_test.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/test/integration/main_test.go b/test/integration/main_test.go index 47095d6..c5ccdf2 100644 --- a/test/integration/main_test.go +++ b/test/integration/main_test.go @@ -57,6 +57,13 @@ func TestMain(m *testing.M) { os.Exit(testenv.Run(m)) } +func BuildImage(ctx context.Context, cfg *envconf.Config) (context.Context, error) { + // Build the Docker image for the controller manager + cmd := exec.Command("make", "docker-build", fmt.Sprintf("IMG=%s", projectImage)) + _, err := utils.Run(cmd) + return ctx, err +} + func InstallCRD(ctx context.Context, cfg *envconf.Config) (context.Context, error) { // Install the CRD in the test environment cmd := exec.Command("make", "install") From 3bc2df4d53e6b28a436ea81566c706b2d671a25b Mon Sep 17 00:00:00 2001 From: DQ Date: Tue, 15 Apr 2025 16:09:54 +1000 Subject: [PATCH 5/9] fix: update cache key annotations and test namespace for consistency --- internal/utils/controller/const.go | 6 +++--- test/integration/main_test.go | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/internal/utils/controller/const.go b/internal/utils/controller/const.go index 0f34cfd..2916951 100644 --- a/internal/utils/controller/const.go +++ b/internal/utils/controller/const.go @@ -1,12 +1,12 @@ package controller const ( - LabelNameCacheKey = "github.com/Azure/operation-cache-controller/cache-key" + LabelNameCacheKey = "operation-cache-controller.azure.github.com/cache-key" ) const ( - AnnotationNameCacheMode = "github.com/Azure/operation-cache-controller/cache-mode" - AnnotationNameCacheKey = "github.com/Azure/operation-cache-controller/cache-key-annotation" + AnnotationNameCacheMode = "operation-cache-controller.azure.github.com/cache-mode" + AnnotationNameCacheKey = "operation-cache-controller.azure.github.com/cache-key" AnnotationValueTrue = "true" AnnotationValueFalse = "false" ) diff --git a/test/integration/main_test.go b/test/integration/main_test.go index c5ccdf2..85304a5 100644 --- a/test/integration/main_test.go +++ b/test/integration/main_test.go @@ -26,7 +26,7 @@ var testenv env.Environment // with the code source changes to be tested. var projectImage = "example.com/operation-cache-controller:v0.0.1" var kindClusterName = "integration-test-cluster" -var testNamespace = "deploy-controller-system" +var testNamespace = "operation-cache-controller-test" func init() { log.SetLogger(zap.New(zap.WriteTo(os.Stdout), zap.UseDevMode(true))) @@ -42,6 +42,7 @@ func TestMain(m *testing.M) { envfuncs.CreateCluster(kind.NewProvider(), kindClusterName), envfuncs.LoadDockerImageToCluster(kindClusterName, projectImage), envfuncs.CreateNamespace(testNamespace), + BuildImage, InstallCRD, DeployControllerManager, ) From 347dcc1843c71791cdd461319c4e4dde5945b2e8 Mon Sep 17 00:00:00 2001 From: DQ Date: Tue, 15 Apr 2025 16:12:09 +1000 Subject: [PATCH 6/9] fix: reorder BuildImage function call in test environment setup --- test/integration/main_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/main_test.go b/test/integration/main_test.go index 85304a5..bf60ff0 100644 --- a/test/integration/main_test.go +++ b/test/integration/main_test.go @@ -39,10 +39,10 @@ func TestMain(m *testing.M) { // Setup the test environment with Kind cluster and necessary resources testenv = testenv.Setup( + BuildImage, envfuncs.CreateCluster(kind.NewProvider(), kindClusterName), envfuncs.LoadDockerImageToCluster(kindClusterName, projectImage), envfuncs.CreateNamespace(testNamespace), - BuildImage, InstallCRD, DeployControllerManager, ) From 5fc53a401de17496fc3f8187dea0c2ed0c9f08a7 Mon Sep 17 00:00:00 2001 From: DQ Date: Tue, 15 Apr 2025 17:16:27 +1000 Subject: [PATCH 7/9] fix: update test namespace for consistency in integration tests --- test/integration/main_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/main_test.go b/test/integration/main_test.go index bf60ff0..e407c45 100644 --- a/test/integration/main_test.go +++ b/test/integration/main_test.go @@ -26,7 +26,7 @@ var testenv env.Environment // with the code source changes to be tested. var projectImage = "example.com/operation-cache-controller:v0.0.1" var kindClusterName = "integration-test-cluster" -var testNamespace = "operation-cache-controller-test" +var testNamespace = "operation-cache-controller" func init() { log.SetLogger(zap.New(zap.WriteTo(os.Stdout), zap.UseDevMode(true))) From 927d0cdc3448c39f285ec01fc8fafaa1c9a5b941 Mon Sep 17 00:00:00 2001 From: DQ Date: Tue, 15 Apr 2025 17:37:33 +1000 Subject: [PATCH 8/9] feat: add integration tests workflow for end-to-end testing --- .github/workflows/test-e2e.yml | 5 ----- .github/workflows/test-integration.yml | 23 +++++++++++++++++++++++ 2 files changed, 23 insertions(+), 5 deletions(-) create mode 100644 .github/workflows/test-integration.yml diff --git a/.github/workflows/test-e2e.yml b/.github/workflows/test-e2e.yml index 89b197a..b2eda8c 100644 --- a/.github/workflows/test-e2e.yml +++ b/.github/workflows/test-e2e.yml @@ -17,11 +17,6 @@ jobs: with: go-version-file: go.mod - - name: Running integration tests - run: | - go mod tidy - make test-integration - - name: Install the latest version of kind run: | curl -Lo ./kind https://kind.sigs.k8s.io/dl/latest/kind-linux-amd64 diff --git a/.github/workflows/test-integration.yml b/.github/workflows/test-integration.yml new file mode 100644 index 0000000..ce9ba06 --- /dev/null +++ b/.github/workflows/test-integration.yml @@ -0,0 +1,23 @@ +name: Integration Tests + +on: + push: + pull_request: + +jobs: + test-e2e: + name: Run on Ubuntu + runs-on: ubuntu-latest + steps: + - name: Clone the code + uses: actions/checkout@v4 + + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version-file: go.mod + + - name: Running integration tests + run: | + go mod tidy + make test-integration From 6d6b7d56392a53600e113573b2cefce5c358d79f Mon Sep 17 00:00:00 2001 From: DQ Date: Wed, 16 Apr 2025 14:25:57 +1000 Subject: [PATCH 9/9] refactor: reorganize integration tests and remove unused cache resources --- test/integration/integration_test.go | 49 +++++++++++------------- test/integration/main_test.go | 7 ++-- test/utils/resources.go | 57 ++++++++++++++++++++++++++++ test/utils/resources/cache.go | 46 ---------------------- 4 files changed, 83 insertions(+), 76 deletions(-) create mode 100644 test/utils/resources.go delete mode 100644 test/utils/resources/cache.go diff --git a/test/integration/integration_test.go b/test/integration/integration_test.go index a7b7aab..0c895db 100644 --- a/test/integration/integration_test.go +++ b/test/integration/integration_test.go @@ -2,63 +2,60 @@ package integration import ( "context" - "fmt" "testing" "time" "github.com/stretchr/testify/assert" "k8s.io/apimachinery/pkg/util/wait" - k8sresources "sigs.k8s.io/e2e-framework/klient/k8s/resources" "sigs.k8s.io/e2e-framework/pkg/envconf" "sigs.k8s.io/e2e-framework/pkg/features" v1 "github.com/Azure/operation-cache-controller/api/v1" - "github.com/Azure/operation-cache-controller/test/utils/resources" + rqutils "github.com/Azure/operation-cache-controller/internal/utils/controller/requirement" + "github.com/Azure/operation-cache-controller/test/utils" ) -type cacheKey struct{} +type requirementKey struct{} -var CacheFeature = features.New("appsv1/deployment/cache"). +const ( + testRequirementName = "test-requirement" +) + +var CacheFeature = features.New("Simple Requirements"). Setup(func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { // start a deployment - cache := resources.SampleCache - cache.Namespace = testNamespace - if err := c.Client().Resources().Create(ctx, &cache); err != nil { + requiremnt := utils.NewRequirement(testRequirementName, utils.TestNamespcae) + requiremnt.Namespace = utils.TestNamespcae + if err := c.Client().Resources().Create(ctx, requiremnt); err != nil { t.Fatal(err) } time.Sleep(2 * time.Second) return ctx }). - Assess("create cache", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { - var cache v1.Cache - if err := cfg.Client().Resources().Get(ctx, resources.SampleCache.Name, testNamespace, &cache); err != nil { + Assess("create requirement", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { + var requirement v1.Requirement + if err := cfg.Client().Resources().Get(ctx, testRequirementName, utils.TestNamespcae, &requirement); err != nil { t.Fatal(err) } - assert.Equal(t, "cache-sample", cache.Name) - if err := wait.PollUntilContextTimeout(ctx, 5*time.Second, 60*time.Second, true, func(ctx context.Context) (bool, error) { - var ops v1.OperationList - if err := cfg.Client().Resources().List(ctx, &ops, k8sresources.WithFieldSelector(fmt.Sprintf("metadata.namespace=%s", testNamespace))); err != nil { + assert.Equal(t, testRequirementName, requirement.Name) + if err := wait.PollUntilContextTimeout(ctx, 10*time.Second, 60*time.Second, true, func(ctx context.Context) (bool, error) { + requirement := &v1.Requirement{} + if err := cfg.Client().Resources().Get(ctx, testRequirementName, utils.TestNamespcae, requirement); err != nil { return false, err } - var countOwnedOps int - for _, op := range ops.Items { - if op.ObjectMeta.GetOwnerReferences()[0].Name == cache.Name { - countOwnedOps++ - } - } - if countOwnedOps != int(cache.Status.KeepAliveCount) { + if requirement.Status.Phase != rqutils.PhaseReady { return false, nil } return true, nil }); err != nil { - t.Fatal(err, "operations not meet the expected count") + t.Fatal(err, "operations not ready") } - return context.WithValue(ctx, cacheKey{}, &cache) + return context.WithValue(ctx, requirementKey{}, &requirement) }). Teardown(func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { - cache := ctx.Value(cacheKey{}).(*v1.Cache) - if err := cfg.Client().Resources().Delete(ctx, cache); err != nil { + requirement := ctx.Value(requirementKey{}).(*v1.Requirement) + if err := cfg.Client().Resources().Delete(ctx, requirement); err != nil { t.Fatal(err) } return ctx diff --git a/test/integration/main_test.go b/test/integration/main_test.go index e407c45..5a1d799 100644 --- a/test/integration/main_test.go +++ b/test/integration/main_test.go @@ -26,7 +26,6 @@ var testenv env.Environment // with the code source changes to be tested. var projectImage = "example.com/operation-cache-controller:v0.0.1" var kindClusterName = "integration-test-cluster" -var testNamespace = "operation-cache-controller" func init() { log.SetLogger(zap.New(zap.WriteTo(os.Stdout), zap.UseDevMode(true))) @@ -42,15 +41,14 @@ func TestMain(m *testing.M) { BuildImage, envfuncs.CreateCluster(kind.NewProvider(), kindClusterName), envfuncs.LoadDockerImageToCluster(kindClusterName, projectImage), - envfuncs.CreateNamespace(testNamespace), + envfuncs.CreateNamespace(utils.TestNamespcae), InstallCRD, DeployControllerManager, ) // Teardown the test environment testenv = testenv.Finish( - envfuncs.DeleteNamespace(testNamespace), - UninstallCRD, + envfuncs.DeleteNamespace(utils.TestNamespcae), envfuncs.DestroyCluster(kindClusterName), ) @@ -86,6 +84,7 @@ func UninstallCRD(ctx context.Context, cfg *envconf.Config) (context.Context, er return ctx, err } func TestRealCluster(t *testing.T) { + // Create a new test environment configuration // Run the integration tests against the Kind cluster testenv.Test(t, CacheFeature) } diff --git a/test/utils/resources.go b/test/utils/resources.go new file mode 100644 index 0000000..e6abe1c --- /dev/null +++ b/test/utils/resources.go @@ -0,0 +1,57 @@ +package utils + +import ( + appsv1 "github.com/Azure/operation-cache-controller/api/v1" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + TestNamespcae = "operation-cache-controller-system" +) + +func NewTestJobSpec(name string) batchv1.JobSpec { + return batchv1.JobSpec{ + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{}, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: name, + Image: "mcr.microsoft.com/azurelinux/busybox:1.36", + Command: []string{"echo", name + " job"}, + }, + }, + }, + }, + } +} + +func NewTestApplicationSpec(name string) appsv1.ApplicationSpec { + return appsv1.ApplicationSpec{ + Name: name, + Provision: NewTestJobSpec("provision"), + Teardown: NewTestJobSpec("teardown"), + } +} + +func NewSimpleOperationSpec(name string) *appsv1.OperationSpec { + return &appsv1.OperationSpec{ + Applications: []appsv1.ApplicationSpec{NewTestApplicationSpec("app1")}, + } +} + +func NewRequirement(name, namespace string) *appsv1.Requirement { + return &appsv1.Requirement{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: appsv1.RequirementSpec{ + Template: appsv1.OperationSpec{ + Applications: []appsv1.ApplicationSpec{NewTestApplicationSpec("app1")}, + }, + }, + } +} diff --git a/test/utils/resources/cache.go b/test/utils/resources/cache.go deleted file mode 100644 index f17d37a..0000000 --- a/test/utils/resources/cache.go +++ /dev/null @@ -1,46 +0,0 @@ -package resources - -import ( - appsv1 "github.com/Azure/operation-cache-controller/api/v1" - batchv1 "k8s.io/api/batch/v1" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -var SampleCache appsv1.Cache = appsv1.Cache{ - ObjectMeta: metav1.ObjectMeta{Name: "cache-sample"}, - Spec: appsv1.CacheSpec{ - Strategy: "", - OperationTemplate: appsv1.OperationSpec{ - Applications: []appsv1.ApplicationSpec{ - { - Name: "test-app", - Provision: batchv1.JobSpec{ - Template: corev1.PodTemplateSpec{ - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: "test-app-provision", - Image: "aksartifactsmsftprodeastus.azurecr.io/devinfra/underlay:sha256:05c616236b999cb9e610cd7afede57adc90516f3db6f30408c72a907889ad8dc", - }, - }, - }, - }, - }, - Teardown: batchv1.JobSpec{ - Template: corev1.PodTemplateSpec{ - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: "test-app-provision", - Image: "aksartifactsmsftprodeastus.azurecr.io/devinfra/underlay:sha256:05c616236b999cb9e610cd7afede57adc90516f3db6f30408c72a907889ad8dc", - }, - }, - }, - }, - }, - }, - }, - }, - }, -}