CSPL-2094, 2342: Upgrade Strategy for LM and CM

kuttl/kuttl-test-helm-upgrade.yaml

@@ -0,0 +1,10 @@
+# Entrypoint for helm automation
+apiVersion: kuttl.dev/v1beta1
+kind: TestSuite
+testDirs:
+- ./kuttl/tests/upgrade
+parallel: 3
+timeout: 5000
+startKIND: false
+artifactsDir: kuttl-artifacts
+kindNodeCache: false

kuttl/tests/upgrade/c3-with-operator/00-assert.yaml

@@ -0,0 +1,9 @@
+---
+# assert for splunk operator deployment to be ready
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: splunk-operator-controller-manager
+status:
+  readyReplicas: 1
+  availableReplicas: 1

kuttl/tests/upgrade/c3-with-operator/00-install-c3.yaml

@@ -0,0 +1,6 @@
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  - command: helm install splunk-c3 $HELM_REPO_PATH/splunk-enterprise -f c3_config.yaml --set splunk-operator.splunkOperator.image.repository=${KUTTL_SPLUNK_OPERATOR_IMAGE} --set splunk-operator.image.repository=${KUTTL_SPLUNK_ENTERPRISE_IMAGE} --namespace ${NAMESPACE}
+    namespaced: true

kuttl/tests/upgrade/c3-with-operator/01-assert.yaml

@@ -0,0 +1,9 @@
+---
+# assert for splunk operator pod to be ready
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: splunk-operator-controller-manager
+status:
+  readyReplicas: 1
+  availableReplicas: 1

kuttl/tests/upgrade/c3-with-operator/01-upgrade-splunk-image.yaml

@@ -0,0 +1,6 @@
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  - command: helm upgrade splunk-c3 $HELM_REPO_PATH/splunk-enterprise --reuse-values -f c3_config.yaml --set splunk-operator.splunkOperator.image.repository=${KUTTL_SPLUNK_OPERATOR_IMAGE} --set splunk-operator.image.repository=${KUTTL_SPLUNK_ENTERPRISE_NEW_IMAGE} --namespace ${NAMESPACE} 
+    namespaced: true

kuttl/tests/upgrade/c3-with-operator/02-assert.yaml

@@ -0,0 +1,24 @@
+---
+# assert for cluster manager custom resource to be ready
+apiVersion: enterprise.splunk.com/v4
+kind: ClusterManager
+metadata:
+  name: cm
+status:
+  phase: Ready
+
+---
+# check if stateful sets are created
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: splunk-cm-cluster-manager
+status:
+  replicas: 1
+
+---
+# check if secret object are created
+apiVersion: v1
+kind: Secret
+metadata:
+  name: splunk-cm-cluster-manager-secret-v1

kuttl/tests/upgrade/c3-with-operator/03-assert.yaml

@@ -0,0 +1,40 @@
+---
+# assert for SearchHeadCluster custom resource to be ready
+apiVersion: enterprise.splunk.com/v4
+kind: SearchHeadCluster
+metadata:
+  name: shc
+status:
+  phase: Ready
+
+---
+# check if secret object are created
+apiVersion: v1
+kind: Secret
+metadata:
+  name: splunk-shc-deployer-secret-v1
+
+---
+# check if secret object are created
+apiVersion: v1
+kind: Secret
+metadata:
+  name: splunk-shc-search-head-secret-v1
+
+---
+# check for stateful set and replicas as configured
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: splunk-shc-search-head
+status:
+  replicas: 3
+
+---
+# check for statefull set
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: splunk-shc-deployer
+status:
+  replicas: 1

kuttl/tests/upgrade/c3-with-operator/04-assert.yaml

@@ -0,0 +1,24 @@
+---
+# assert for indexer cluster custom resource to be ready
+apiVersion: enterprise.splunk.com/v4
+kind: IndexerCluster
+metadata:
+  name: idxc
+status:
+  phase: Ready
+
+---
+# check for stateful set and replicas as configured
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: splunk-idxc-indexer
+status:
+  replicas: 3
+
+---
+# check if secret object are created
+apiVersion: v1
+kind: Secret
+metadata:
+  name: splunk-idxc-indexer-secret-v1

kuttl/tests/upgrade/c3-with-operator/05-uninstall-c3.yaml

@@ -0,0 +1,5 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  - command: helm uninstall splunk-c3 --namespace ${NAMESPACE}
+    namespaced: true

kuttl/tests/upgrade/c3-with-operator/c3_config.yaml

@@ -0,0 +1,50 @@
+splunk-operator:
+  enabled: true
+  splunkOperator:
+    clusterWideAccess: false
+
+sva:
+    c3:
+      enabled: true
+
+      clusterManager:
+        name: cm
+
+      indexerClusters:
+        - name: idxc
+
+      searchHeadClusters:
+        - name: shc
+
+
+indexerCluster:
+  enabled: true
+
+  additionalLabels:
+    label: "true"
+
+  additionalAnnotations:
+    annotation: "true"
+    service.beta.kubernetes.io/azure-load-balancer-internal: "true"
+
+  serviceTemplate:
+    spec:
+      type: LoadBalancer
+
+clusterManager:
+  enabled: true
+
+  additionalLabels:
+    label: "true"
+
+  additionalAnnotations:
+    annotation: "true"
+
+searchHeadCluster:
+  enabled: true
+
+  additionalLabels:
+    label: "true"
+
+  additionalAnnotations:
+    annotation: "true"

pkg/splunk/enterprise/clustermanager.go

@@ -23,6 +23,7 @@ import (
 
 	enterpriseApi "github.com/splunk/splunk-operator/api/v4"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+	rclient "sigs.k8s.io/controller-runtime/pkg/client"
 
 	"github.com/go-logr/logr"
 	splclient "github.com/splunk/splunk-operator/pkg/splunk/client"
@@ -31,6 +32,8 @@ import (
 	splutil "github.com/splunk/splunk-operator/pkg/splunk/util"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
+	k8serrors "k8s.io/apimachinery/pkg/api/errors"
+
 	"k8s.io/apimachinery/pkg/types"
 	"sigs.k8s.io/controller-runtime/pkg/log"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
@@ -177,6 +180,12 @@ func ApplyClusterManager(ctx context.Context, client splcommon.ControllerClient,
 		return result, err
 	}
 
+	// check if the ClusterManager is ready for version upgrade, if required
+	continueReconcile, err := isClusterManagerReadyForUpgrade(ctx, client, cr)
+	if err != nil || !continueReconcile {
+		return result, err
+	}
+
 	clusterManagerManager := splctrl.DefaultStatefulSetPodManager{}
 	phase, err := clusterManagerManager.Update(ctx, client, statefulSet, 1)
 	if err != nil {
@@ -434,3 +443,132 @@ func VerifyCMisMultisite(ctx context.Context, cr *enterpriseApi.ClusterManager,
 	}
 	return extraEnv, err
 }
+
+// isClusterManagerReadyForUpgrade checks if ClusterManager can be upgraded if a version upgrade is in-progress
+// No-operation otherwise; returns bool, err accordingly
+func isClusterManagerReadyForUpgrade(ctx context.Context, c splcommon.ControllerClient, cr *enterpriseApi.ClusterManager) (bool, error) {
+	reqLogger := log.FromContext(ctx)
+	scopedLog := reqLogger.WithName("isClusterManagerReadyForUpgrade").WithValues("name", cr.GetName(), "namespace", cr.GetNamespace())
+	eventPublisher, _ := newK8EventPublisher(c, cr)
+
+	// check if a LicenseManager is attached to the instance
+	licenseManagerRef := cr.Spec.LicenseManagerRef
+	if licenseManagerRef.Name == "" {
+		return true, nil
+	}
+
+	namespacedName := types.NamespacedName{
+		Namespace: cr.GetNamespace(),
+		Name:      GetSplunkStatefulsetName(SplunkClusterManager, cr.GetName()),
+	}
+
+	// check if the stateful set is created at this instance
+	statefulSet := &appsv1.StatefulSet{}
+	err := c.Get(ctx, namespacedName, statefulSet)
+	if err != nil && k8serrors.IsNotFound(err) {
+		return true, nil
+	}
+
+	namespacedName = types.NamespacedName{Namespace: cr.GetNamespace(), Name: licenseManagerRef.Name}
+	licenseManager := &enterpriseApi.LicenseManager{}
+
+	// get the license manager referred in cluster manager
+	err = c.Get(ctx, namespacedName, licenseManager)
+	if err != nil {
+		if k8serrors.IsNotFound(err) {
+			return true, nil
+		}
+		eventPublisher.Warning(ctx, "isClusterManagerReadyForUpgrade", fmt.Sprintf("Could not find the License Manager. Reason %v", err))
+		scopedLog.Error(err, "Unable to get licenseManager")
+		return false, err
+	}
+
+	lmImage, err := getCurrentImage(ctx, c, cr, SplunkLicenseManager)
+	if err != nil {
+		eventPublisher.Warning(ctx, "isClusterManagerReadyForUpgrade", fmt.Sprintf("Could not get the License Manager Image. Reason %v", err))
+		scopedLog.Error(err, "Unable to get licenseManager current image")
+		return false, err
+	}
+
+	cmImage, err := getCurrentImage(ctx, c, cr, SplunkClusterManager)
+	if err != nil {
+		eventPublisher.Warning(ctx, "isClusterManagerReadyForUpgrade", fmt.Sprintf("Could not get the Cluster Manager Image. Reason %v", err))
+		scopedLog.Error(err, "Unable to get clusterManager current image")
+		return false, err
+	}
+
+	// check if an image upgrade is happening and whether the ClusterManager is ready for the upgrade
+	if (cr.Spec.Image != cmImage) && (licenseManager.Status.Phase != enterpriseApi.PhaseReady || lmImage != cr.Spec.Image) {
+		return false, nil
+	}
+
+	return true, nil
+}
+
+// changeClusterManagerAnnotations updates the splunk/image-tag field of the ClusterManager annotations to trigger the reconcile loop
+// on update, and returns error if something is wrong
+func changeClusterManagerAnnotations(ctx context.Context, c splcommon.ControllerClient, cr *enterpriseApi.LicenseManager) error {
+	reqLogger := log.FromContext(ctx)
+	scopedLog := reqLogger.WithName("changeClusterManagerAnnotations").WithValues("name", cr.GetName(), "namespace", cr.GetNamespace())
+	eventPublisher, _ := newK8EventPublisher(c, cr)
+
+	clusterManagerInstance := &enterpriseApi.ClusterManager{}
+	if len(cr.Spec.ClusterManagerRef.Name) > 0 {
+		// if the LicenseManager holds the ClusterManagerRef
+		namespacedName := types.NamespacedName{
+			Namespace: cr.GetNamespace(),
+			Name:      cr.Spec.ClusterManagerRef.Name,
+		}
+		err := c.Get(ctx, namespacedName, clusterManagerInstance)
+		if err != nil {
+			if k8serrors.IsNotFound(err) {
+				return nil
+			}
+			return err
+		}
+	} else {
+		// List out all the ClusterManager instances in the namespace
+		opts := []rclient.ListOption{
+			rclient.InNamespace(cr.GetNamespace()),
+		}
+		objectList := enterpriseApi.ClusterManagerList{}
+		err := c.List(ctx, &objectList, opts...)
+		if err != nil {
+			if err.Error() == "NotFound" {
+				return nil
+			}
+			return err
+		}
+
+		if len(objectList.Items) == 0 {
+			return nil
+		}
+
+		// check if instance has the required LicenseManagerRef
+		for _, cm := range objectList.Items {
+			if cm.Spec.LicenseManagerRef.Name == cr.GetName() {
+				clusterManagerInstance = &cm
+				break
+			}
+		}
+
+		if len(clusterManagerInstance.GetName()) == 0 {
+			return nil
+		}
+	}
+
+	image, err := getCurrentImage(ctx, c, cr, SplunkLicenseManager)
+	if err != nil {
+		eventPublisher.Warning(ctx, "changeClusterManagerAnnotations", fmt.Sprintf("Could not get the LicenseManager Image. Reason %v", err))
+		scopedLog.Error(err, "Get LicenseManager Image failed with", "error", err)
+		return err
+	}
+	err = changeAnnotations(ctx, c, image, clusterManagerInstance)
+	if err != nil {
+		eventPublisher.Warning(ctx, "changeClusterManagerAnnotations", fmt.Sprintf("Could not update annotations. Reason %v", err))
+		scopedLog.Error(err, "ClusterManager types update after changing annotations failed with", "error", err)
+		return err
+	}
+
+	return nil
+}