SPLAT-2060: Create e2e tests for vSphere Data Disk feature

Dockerfile

@@ -1,7 +1,10 @@
 FROM registry.ci.openshift.org/openshift/release:golang-1.23 AS builder
 WORKDIR /go/src/github.com/openshift/machine-api-operator
 COPY . .
-RUN NO_DOCKER=1 make build
+RUN NO_DOCKER=1 make build && \
+    mkdir -p /tmp/build && \
+    cp /go/src/github.com/openshift/machine-api-operator/bin/machine-api-tests-ext /tmp/build/machine-api-tests-ext && \
+    gzip /tmp/build/machine-api-tests-ext
 
 FROM registry.ci.openshift.org/openshift/origin-v4.0:base
 COPY --from=builder /go/src/github.com/openshift/machine-api-operator/install manifests
@@ -10,5 +13,8 @@ COPY --from=builder /go/src/github.com/openshift/machine-api-operator/bin/nodeli
 COPY --from=builder /go/src/github.com/openshift/machine-api-operator/bin/machine-healthcheck .
 COPY --from=builder /go/src/github.com/openshift/machine-api-operator/bin/machineset ./machineset-controller
 COPY --from=builder /go/src/github.com/openshift/machine-api-operator/bin/vsphere ./machine-controller-manager
+COPY --from=builder /tmp/build/machine-api-tests-ext.gz .
 
-LABEL io.openshift.release.operator true
+LABEL io.k8s.display-name="OpenShift Machine API Operator" \
+      io.openshift.release.operator=true \
+      io.openshift.tags="openshift,tests,e2e,e2e-extension"

Dockerfile.rhel

@@ -1,7 +1,10 @@
 FROM registry.ci.openshift.org/ocp/builder:rhel-9-golang-1.23-openshift-4.19 AS builder
 WORKDIR /go/src/github.com/openshift/machine-api-operator
 COPY . .
-RUN NO_DOCKER=1 make build
+RUN NO_DOCKER=1 make build && \
+    mkdir -p /tmp/build && \
+    cp /go/src/github.com/openshift/machine-api-operator/bin/machine-api-tests-ext /tmp/build/machine-api-tests-ext && \
+    gzip /tmp/build/machine-api-tests-ext
 
 FROM registry.ci.openshift.org/ocp/4.19:base-rhel9
 COPY --from=builder /go/src/github.com/openshift/machine-api-operator/install manifests
@@ -10,5 +13,8 @@ COPY --from=builder /go/src/github.com/openshift/machine-api-operator/bin/nodeli
 COPY --from=builder /go/src/github.com/openshift/machine-api-operator/bin/machine-healthcheck .
 COPY --from=builder /go/src/github.com/openshift/machine-api-operator/bin/machineset ./machineset-controller
 COPY --from=builder /go/src/github.com/openshift/machine-api-operator/bin/vsphere ./machine-controller-manager
+COPY --from=builder /tmp/build/machine-api-tests-ext.gz .
 
-LABEL io.openshift.release.operator true
+LABEL io.k8s.display-name="OpenShift Machine API Operator" \
+      io.openshift.release.operator=true \
+      io.openshift.tags="openshift,tests,e2e,e2e-extension"

Makefile

@@ -65,7 +65,7 @@ vendor:
 check: verify-crds-sync lint fmt vet test ## Run code validations
 
 .PHONY: build
-build: machine-api-operator nodelink-controller machine-healthcheck machineset vsphere ## Build binaries
+build: machine-api-operator nodelink-controller machine-healthcheck machineset vsphere machine-api-tests-ext ## Build binaries
 
 .PHONY: machine-api-operator
 machine-api-operator:
@@ -87,6 +87,10 @@ vsphere:
 machineset:
 	$(DOCKER_CMD) ./hack/go-build.sh machineset
 
+.PHONY: machine-api-tests-ext
+machine-api-tests-ext:
+	$(DOCKER_CMD) ./hack/go-build.sh machine-api-tests-ext
+
 .PHONY: test-e2e
 test-e2e: ## Run openshift specific e2e tests
 	./hack/e2e.sh test-e2e

cmd/machine-api-tests-ext/main.go

@@ -0,0 +1,98 @@
+package main
+
+import (
+	"flag"
+	"os"
+	"regexp"
+	"strings"
+
+	"github.com/openshift-eng/openshift-tests-extension/pkg/cmd"
+	e "github.com/openshift-eng/openshift-tests-extension/pkg/extension"
+	"github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests"
+	g "github.com/openshift-eng/openshift-tests-extension/pkg/ginkgo"
+	"github.com/spf13/cobra"
+	"github.com/spf13/pflag"
+	utilflag "k8s.io/component-base/cli/flag"
+	"k8s.io/component-base/logs"
+	"k8s.io/kubernetes/test/e2e/framework"
+
+	// If using ginkgo, import your tests here
+	_ "github.com/openshift/machine-api-operator/test/e2e/vsphere"
+)
+
+func main() {
+	logs.InitLogs()
+	defer logs.FlushLogs()
+	pflag.CommandLine.SetNormalizeFunc(utilflag.WordSepNormalizeFunc)
+
+	// These flags are used to pull in the default values to test context - required
+	// so tests run correctly, even if the underlying flags aren't used.
+	framework.RegisterCommonFlags(flag.CommandLine)
+	framework.RegisterClusterFlags(flag.CommandLine)
+
+	// Create our registry of openshift-tests extensions
+	extensionRegistry := e.NewRegistry()
+	kubeTestsExtension := e.NewExtension("openshift", "payload", "machine-api-operator")
+	extensionRegistry.Register(kubeTestsExtension)
+
+	// Carve up the kube tests into our openshift suites...
+	kubeTestsExtension.AddSuite(e.Suite{
+		Name: "mao/conformance/parallel",
+		Parents: []string{
+			"openshift/conformance/parallel",
+		},
+		Qualifiers: []string{`!labels.exists(l, l == "Serial") && labels.exists(l, l == "Conformance")`},
+	})
+
+	kubeTestsExtension.AddSuite(e.Suite{
+		Name: "mao/conformance/serial",
+		Parents: []string{
+			"openshift/conformance/serial",
+		},
+		Qualifiers: []string{`labels.exists(l, l == "Serial") && labels.exists(l, l == "Conformance")`},
+	})
+
+	// Build our specs from ginkgo
+	specs, err := g.BuildExtensionTestSpecsFromOpenShiftGinkgoSuite()
+	if err != nil {
+		panic(err)
+	}
+
+	// Initialization for kube ginkgo test framework needs to run before all tests execute
+	specs.AddBeforeAll(func() {
+		if err := initializeTestFramework(os.Getenv("TEST_PROVIDER")); err != nil {
+			panic(err)
+		}
+	})
+
+	// Let's scan for tests with a platform label and create the rule for them such as [platform:vsphere]
+	foundPlatforms := make(map[string]string)
+	for _, test := range specs.Select(extensiontests.NameContains("[platform:")).Names() {
+		re := regexp.MustCompile(`\[platform:[a-z]*]`)
+		match := re.FindStringSubmatch(test)
+		for _, platformDef := range match {
+			if _, ok := foundPlatforms[platformDef]; !ok {
+				platform := platformDef[strings.Index(platformDef, ":")+1 : len(platformDef)-1]
+				foundPlatforms[platformDef] = platform
+				specs.Select(extensiontests.NameContains(platformDef)).
+					Include(extensiontests.PlatformEquals(platform))
+			}
+		}
+
+	}
+
+	kubeTestsExtension.AddSpecs(specs)
+
+	// Cobra stuff
+	root := &cobra.Command{
+		Long: "Machine API Operator tests extension for OpenShift",
+	}
+
+	root.AddCommand(cmd.DefaultExtensionCommands(extensionRegistry)...)
+
+	if err := func() error {
+		return root.Execute()
+	}(); err != nil {
+		os.Exit(1)
+	}
+}

cmd/machine-api-tests-ext/provider.go

@@ -0,0 +1,112 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/onsi/ginkgo/v2"
+	"github.com/onsi/gomega"
+	corev1 "k8s.io/api/core/v1"
+	kclientset "k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/tools/clientcmd"
+	"k8s.io/kubernetes/openshift-hack/e2e"
+	conformancetestdata "k8s.io/kubernetes/test/conformance/testdata"
+	"k8s.io/kubernetes/test/e2e/framework"
+	"k8s.io/kubernetes/test/e2e/framework/testfiles"
+	e2etestingmanifests "k8s.io/kubernetes/test/e2e/testing-manifests"
+	testfixtures "k8s.io/kubernetes/test/fixtures"
+
+	// this appears to inexplicably auto-register global flags.
+	_ "k8s.io/kubernetes/test/e2e/storage/drivers"
+
+	// these are loading important global flags that we need to get and set
+	_ "k8s.io/kubernetes/test/e2e"
+	_ "k8s.io/kubernetes/test/e2e/lifecycle"
+)
+
+// copied directly from github.com/openshift/kubernetes/openshift-hack/cmd/k8s-tests-ext/provider.go
+// I attempted to use the clusterdiscovery.InitializeTestFramework in origin but it has too many additional parameters
+// that as an test-ext, I felt we shouldn't have to load all that.  Hopefully origin's test-ext frameworks gets enhanced
+// to have a simple way to initialize all this w/o having to copy/pasta like the openshift/kubernetes project did.
+func initializeTestFramework(provider string) error {
+	providerInfo := &ClusterConfiguration{}
+	if err := json.Unmarshal([]byte(provider), &providerInfo); err != nil {
+		return fmt.Errorf("provider must be a JSON object with the 'type' key at a minimum: %v", err)
+	}
+	if len(providerInfo.ProviderName) == 0 {
+		return fmt.Errorf("provider must be a JSON object with the 'type' key")
+	}
+	config := &ClusterConfiguration{}
+	if err := json.Unmarshal([]byte(provider), config); err != nil {
+		return fmt.Errorf("provider must decode into the ClusterConfig object: %v", err)
+	}
+
+	// update testContext with loaded config
+	testContext := &framework.TestContext
+	testContext.Provider = config.ProviderName
+	testContext.CloudConfig = framework.CloudConfig{
+		ProjectID:   config.ProjectID,
+		Region:      config.Region,
+		Zone:        config.Zone,
+		Zones:       config.Zones,
+		NumNodes:    config.NumNodes,
+		MultiMaster: config.MultiMaster,
+		MultiZone:   config.MultiZone,
+		ConfigFile:  config.ConfigFile,
+	}
+	testContext.AllowedNotReadyNodes = -1
+	testContext.MinStartupPods = -1
+	testContext.MaxNodesToGather = 0
+	testContext.KubeConfig = os.Getenv("KUBECONFIG")
+
+	if ad := os.Getenv("ARTIFACT_DIR"); len(strings.TrimSpace(ad)) == 0 {
+		if err := os.Setenv("ARTIFACT_DIR", filepath.Join(os.TempDir(), "artifacts")); err != nil {
+			return fmt.Errorf("unable to set ARTIFACT_DIR: %v", err)
+		}
+	}
+
+	testContext.DeleteNamespace = os.Getenv("DELETE_NAMESPACE") != "false"
+	testContext.VerifyServiceAccount = true
+	testfiles.AddFileSource(e2etestingmanifests.GetE2ETestingManifestsFS())
+	testfiles.AddFileSource(testfixtures.GetTestFixturesFS())
+	testfiles.AddFileSource(conformancetestdata.GetConformanceTestdataFS())
+	testContext.KubectlPath = "kubectl"
+	testContext.KubeConfig = os.Getenv("KUBECONFIG")
+
+	// "debian" is used when not set. At least GlusterFS tests need "custom".
+	// (There is no option for "rhel" or "centos".)
+	testContext.NodeOSDistro = "custom"
+	testContext.MasterOSDistro = "custom"
+
+	// load and set the host variable for kubectl
+	clientConfig := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(&clientcmd.ClientConfigLoadingRules{ExplicitPath: testContext.KubeConfig}, &clientcmd.ConfigOverrides{})
+	cfg, err := clientConfig.ClientConfig()
+	if err != nil {
+		return err
+	}
+	testContext.Host = cfg.Host
+
+	// Ensure that Kube tests run privileged (like they do upstream)
+	testContext.CreateTestingNS = func(ctx context.Context, baseName string, c kclientset.Interface, labels map[string]string) (*corev1.Namespace, error) {
+		return e2e.CreateTestingNS(ctx, baseName, c, labels, true)
+	}
+
+	gomega.RegisterFailHandler(ginkgo.Fail)
+
+	framework.AfterReadingAllFlags(testContext)
+	testContext.DumpLogsOnFailure = true
+
+	// these constants are taken from kube e2e and used by tests
+	testContext.IPFamily = "ipv4"
+	if config.HasIPv6 && !config.HasIPv4 {
+		testContext.IPFamily = "ipv6"
+	}
+
+	testContext.ReportDir = os.Getenv("TEST_JUNIT_DIR")
+
+	return nil
+}

cmd/machine-api-tests-ext/types.go

@@ -0,0 +1,47 @@
+package main
+
+// copied directly from github.com/openshift/origin/test/extended/util/cluster/cluster.go
+type ClusterConfiguration struct {
+	ProviderName string `json:"type"`
+
+	// These fields (and the "type" tag for ProviderName) chosen to match
+	// upstream's e2e.CloudConfig.
+	ProjectID   string
+	Region      string
+	Zone        string
+	NumNodes    int
+	MultiMaster bool
+	MultiZone   bool
+	Zones       []string
+	ConfigFile  string
+
+	// Disconnected is set for test jobs without external internet connectivity
+	Disconnected bool
+
+	// SingleReplicaTopology is set for disabling disruptive tests or tests
+	// that require high availability
+	SingleReplicaTopology bool
+
+	// NetworkPlugin is the "official" plugin name
+	NetworkPlugin string
+	// NetworkPluginMode is an optional sub-identifier for the NetworkPlugin.
+	// (Currently it is only used for OpenShiftSDN.)
+	NetworkPluginMode string `json:",omitempty"`
+
+	// HasIPv4 and HasIPv6 determine whether IPv4-specific, IPv6-specific,
+	// and dual-stack-specific tests are run
+	HasIPv4 bool
+	HasIPv6 bool
+
+	// HasSCTP determines whether SCTP connectivity tests can be run in the cluster
+	HasSCTP bool
+
+	// IsProxied determines whether we are accessing the cluster through an HTTP proxy
+	IsProxied bool
+
+	// IsIBMROKS determines whether the cluster is Managed IBM Cloud (ROKS)
+	IsIBMROKS bool
+
+	// IsNoOptionalCapabilities indicates the cluster has no optional capabilities enabled
+	HasNoOptionalCapabilities bool
+}